[ { "path": ".coveralls.yml", "content": "service_name: travis-pro\n" }, { "path": ".github/CODEOWNERS", "content": "* @nats-io/server\n*.go @nats-io/server-dev\ndoc/ @nats-io/server-dev\ntest/ @nats-io/server-dev\nlocksordering.txt @nats-io/server-dev\n" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\n# NATS.io\ncommunity_bridge: nats-io\n\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n - name: Discussion\n url: https://github.com/nats-io/nats-server/discussions\n about: Ideal for ideas, feedback, or longer form questions.\n - name: Chat\n url: https://slack.nats.io\n about: Ideal for short, one-off questions, general conversation, and meeting other NATS users!\n" }, { "path": ".github/ISSUE_TEMPLATE/defect.yml", "content": "---\nname: Defect\ndescription: Report a defect, such as a bug or regression.\nlabels:\n - defect\nbody:\n - type: textarea\n id: observed\n attributes:\n label: Observed behavior\n description: Describe the unexpected behavior or performance regression you are observing.\n validations:\n required: true\n - type: textarea\n id: expected\n attributes:\n label: Expected behavior\n description: Describe the expected behavior or performance characteristics.\n validations:\n required: true\n - type: textarea\n id: versions\n attributes:\n label: Server and client version\n description: |-\n **One of the first things we are likely to ask you is if the defect reproduces on the latest version, so please check that first!**\n For the server, use `nats-server --version`, check the startup log output, or the image tag pulled from Docker.\n For the CLI client, use `nats --version`.\n For language-specific clients, check the version downloaded by the language dependency manager.\n validations:\n required: true\n - type: textarea\n id: environment\n attributes:\n label: Host environment\n description: |-\n Specify any relevant details about the host environment the server and/or client was running in,\n such as operating system, CPU architecture, container runtime, etc.\n validations:\n required: false\n - type: textarea\n id: steps\n attributes:\n label: Steps to reproduce\n description: Provide as many concrete steps to reproduce the defect. Please also include stream info and consumer info if this issue is JetStream-related.\n validations:\n required: false\n" }, { "path": ".github/ISSUE_TEMPLATE/proposal.yml", "content": "---\nname: Proposal\ndescription: Propose an enhancement or new feature.\nlabels:\n - proposal\nbody:\n - type: textarea\n id: change\n attributes:\n label: Proposed change\n description: This could be a behavior change, enhanced API, or a new feature.\n validations:\n required: true\n - type: textarea\n id: usecase\n attributes:\n label: Use case\n description: What is the use case or general motivation for this proposal?\n validations:\n required: true\n - type: textarea\n id: contribute\n attributes:\n label: Contribution\n description: |-\n Are you intending or interested in contributing code for this proposal if accepted?\n validations:\n required: false\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "\n\nSigned-off-by: Your Name \n" }, { "path": ".github/actions/nightly-release/action.yaml", "content": "name: Nightly Docker Releaser\ndescription: Builds nightly docker images\n\ninputs:\n hub_username:\n description: Docker hub username\n required: true\n\n hub_password:\n description: Docker hub password\n required: true\n\n name:\n description: The name of the build\n default: nightly\n required: false\n\nruns:\n using: composite\n steps:\n - name: Log in to Docker Hub\n shell: bash\n run: docker login -u \"${{ inputs.hub_username }}\" -p \"${{ inputs.hub_password }}\"\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Generate build metadata\n shell: sh\n run: |\n echo \"BUILD_DATE=$(date +%Y%m%d)\" >> $GITHUB_ENV\n echo \"GIT_COMMIT=$(cd ${{ inputs.workdir }}/nats-server && git rev-parse --short HEAD)\" >> $GITHUB_ENV\n echo \"BUILD_NAME=$(echo ${{ inputs.name }} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9.-]/-/g')\" >> $GITHUB_ENV\n\n - name: Build and push Docker images\n uses: docker/build-push-action@v6\n with:\n context: \"${{ github.workspace }}/src/github.com/nats-io\"\n file: \"${{ github.workspace }}/src/github.com/nats-io/nats-server/docker/Dockerfile.nightly\"\n build-args: |\n VERSION=nightly-${{ env.BUILD_DATE }}\n GIT_COMMIT=${{ env.GIT_COMMIT }}\n platforms: linux/amd64,linux/arm64\n push: true\n tags: |\n synadia/nats-server:${{ env.BUILD_NAME }}\n synadia/nats-server:${{ env.BUILD_NAME }}-${{ env.BUILD_DATE }}\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"gomod\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"" }, { "path": ".github/workflows/claude.yml", "content": "name: Claude Code\n\n# GITHUB_TOKEN is neutered — all GitHub API access uses the App token instead.\npermissions: {}\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n pull_request_target:\n types: [opened, reopened]\n\njobs:\n claude:\n name: Claude Review\n uses: synadia-io/ai-workflows/.github/workflows/claude.yml@v2\n if: contains(\n fromJson('[\"OWNER\",\"MEMBER\",\"COLLABORATOR\"]'),\n github.event.comment.author_association || github.event.pull_request.author_association\n )\n with:\n gh_app_id: ${{ vars.CLAUDE_GH_APP_ID }}\n checkout_mode: \"base\"\n review_focus: |\n Additionally focus on:\n - Performance implications (hot paths, allocations, lock contention)\n - Concurrency safety (goroutine leaks, race conditions, deadlocks)\n - Raft consensus and JetStream clustering correctness\n - Security boundaries (authentication, authorization, TLS handling)\n secrets:\n claude_oauth_token: ${{ secrets.CLAUDE_OAUTH_TOKEN }}\n gh_app_private_key: ${{ secrets.CLAUDE_GH_APP_PRIVATE_KEY }}\n" }, { "path": ".github/workflows/cov.yaml", "content": "name: NATS Server Code Coverage\non:\n workflow_dispatch: {}\n\n schedule:\n - cron: \"40 4 * * *\"\n\npermissions:\n contents: read\n\njobs:\n nightly_coverage:\n runs-on: ubuntu-latest\n\n env:\n GOPATH: /home/runner/work/nats-server\n GO111MODULE: \"on\"\n\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n with:\n path: src/github.com/nats-io/nats-server\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n cache-dependency-path: src/github.com/nats-io/nats-server/go.sum\n\n - name: Run code coverage\n shell: bash --noprofile --norc -x -eo pipefail {0}\n run: |\n set -e\n cd src/github.com/nats-io/nats-server\n ./scripts/cov.sh upload\n set +e\n\n - name: Convert coverage.out to coverage.lcov\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit e4612787670fc5b5f49026b8c29c5569921de1db = tag v1.2.0\n uses: jandelgado/gcov2lcov-action@e4612787670fc5b5f49026b8c29c5569921de1db\n with:\n infile: acc.out\n working-directory: src/github.com/nats-io/nats-server\n\n - name: Coveralls\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit 5cbfd81b66ca5d10c19b062c04de0199c215fb6e = tag v2.3.7\n uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e\n with:\n github-token: ${{ secrets.github_token }}\n file: src/github.com/nats-io/nats-server/coverage.lcov\n" }, { "path": ".github/workflows/long-tests.yaml", "content": "name: NATS Server Long Tests\n\non:\n # Allow manual trigger (any branch)\n workflow_dispatch:\n # Run daily at 12:30 on default branch\n schedule:\n - cron: \"30 12 * * *\"\n\npermissions:\n contents: read\n\nconcurrency:\n # At most one of these workflow per ref running\n group: ${{ github.workflow }}-${{ github.ref }}\n # New one cancels in-progress one\n cancel-in-progress: true\n\njobs:\n js-long:\n name: Long JetStream tests\n runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run tests\n run: go test -race -v -run='^TestLong.*' ./server -tags=include_js_long_tests -count=1 -vet=off -timeout=60m -shuffle on -p 1 -failfast\n" }, { "path": ".github/workflows/mqtt-test.yaml", "content": "name: MQTT External Tests\non: [pull_request]\n\npermissions:\n contents: read\n\njobs:\n test:\n env:\n GOPATH: /home/runner/work/nats-server\n GO111MODULE: \"on\"\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n with:\n path: src/github.com/nats-io/nats-server\n\n - name: Setup Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n cache-dependency-path: src/github.com/nats-io/nats-server/go.sum\n\n - name: Set up testing tools and environment\n shell: bash --noprofile --norc -eo pipefail {0}\n id: setup\n run: |\n wget https://github.com/hivemq/mqtt-cli/releases/download/v4.20.0/mqtt-cli-4.20.0.deb\n sudo apt install ./mqtt-cli-4.20.0.deb\n go install github.com/ConnectEverything/mqtt-test@v0.1.0\n\n - name: Run tests (3 times to detect flappers)\n shell: bash --noprofile --norc -eo pipefail {0}\n run: |\n cd src/github.com/nats-io/nats-server\n go test -v --count=3 --run='TestXMQTT' ./server\n\n - name: Run tests with --race\n shell: bash --noprofile --norc -eo pipefail {0}\n run: |\n cd src/github.com/nats-io/nats-server\n go test -v --race --failfast --run='TestXMQTT' ./server\n\n - name: Run benchmarks\n shell: bash --noprofile --norc -eo pipefail {0}\n run: |\n cd src/github.com/nats-io/nats-server\n go test --run='-' --count=3 --bench 'BenchmarkXMQTT' --benchtime=100x ./server\n\n # TODO: compare benchmarks\n" }, { "path": ".github/workflows/nightly.yaml", "content": "name: Docker Nightly\non:\n workflow_dispatch:\n inputs:\n target:\n description: \"Override source branch (optional)\"\n type: string\n required: false\n\n schedule:\n - cron: \"40 4 * * *\"\n\npermissions:\n contents: read\n\njobs:\n run:\n runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }}\n permissions:\n contents: write\n steps:\n - name: Checkout NATS\n uses: actions/checkout@v6\n with:\n path: src/github.com/nats-io/nats-server\n ref: ${{ inputs.target || 'main' }}\n fetch-depth: 0\n fetch-tags: true\n\n - name: Checkout NSC\n uses: actions/checkout@v6\n with:\n repository: nats-io/nsc\n path: src/github.com/nats-io/nsc\n fetch-depth: 1\n fetch-tags: true\n\n - name: Checkout NATS CLI\n uses: actions/checkout@v6\n with:\n repository: nats-io/natscli\n path: src/github.com/nats-io/natscli\n fetch-depth: 1\n fetch-tags: true\n\n - name: Build Docker nightly\n uses: ./src/github.com/nats-io/nats-server/.github/actions/nightly-release\n with:\n name: ${{ inputs.target || 'nightly' }}\n hub_username: \"${{ secrets.DOCKER_USERNAME }}\"\n hub_password: \"${{ secrets.DOCKER_PASSWORD }}\"\n" }, { "path": ".github/workflows/release.yaml", "content": "name: NATS Server Releases\non:\n push:\n tags:\n - v*\n\npermissions:\n contents: read\n\njobs:\n run:\n name: GitHub Release\n runs-on: ${{ vars.GHA_WORKER_RELEASE || 'ubuntu-latest' }}\n permissions:\n contents: write\n steps:\n - name: Checkout code\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n fetch-tags: true\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: \"stable\"\n\n - name: Check version matches tag\n env:\n TRAVIS_TAG: ${{ github.ref_name }}\n run: |\n go test -race -v -run=TestVersionMatchesTag ./server -ldflags=\"-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG\" -count=1 -vet=off\n\n - name: Install cosign\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit ba7bc0a3fef59531c69a25acd34668d6d3fe6f22 = tag v4.1.0\n uses: sigstore/cosign-installer@ba7bc0a3fef59531c69a25acd34668d6d3fe6f22\n\n - name: Install syft\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit 57aae528053a48a3f6235f2d9461b05fbcb7366d = tag v0.23.1\n uses: anchore/sbom-action/download-syft@57aae528053a48a3f6235f2d9461b05fbcb7366d\n with:\n syft-version: \"v1.42.2\"\n\n - name: Create release\n uses: goreleaser/goreleaser-action@v7\n with:\n distribution: goreleaser\n version: \"~> v2\"\n args: release --clean\n env:\n GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/stale-issues.yaml", "content": "name: Stale Issues\non:\n schedule:\n - cron: \"30 1 * * *\"\n\npermissions:\n issues: write\n pull-requests: write\n\njobs:\n stale:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/stale@v10\n with:\n stale-issue-label: stale\n stale-pr-label: stale\n days-before-stale: 56 # Mark stale after 8 weeks (56 days) of inactivity\n days-before-close: -1 # Disable auto-closing\n exempt-all-milestones: true # Any issue/PR within a milestone will be omitted\n" }, { "path": ".github/workflows/tests.yaml", "content": "name: NATS Server Tests\n\non:\n push:\n pull_request:\n workflow_dispatch:\n\npermissions:\n contents: read\n\nenv:\n RACE: ${{ (github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') && github.event_name != 'pull_request') && '-race' || '' }}\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }}\n\njobs:\n signoffs:\n name: Sign-offs\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n if: github.ref != 'refs/heads/main'\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Check all branch commits are signed off\n if: github.event_name != 'pull_request'\n run: |\n for c in $(git rev-list --no-merges ${{ github.event.before }}..${{ github.event.after }}); do\n msg=\"$(git log -1 --pretty=%B \"$c\")\"\n if ! printf '%s\\n' \"$msg\" | grep -q '^Signed-off-by:'; then\n echo \"::error ::Commit $c has not been signed off in the commit message with a \\`Signed-off-by: Your Name \\` line\"\n git log -1 --pretty=format:\"%h %s\" \"$c\"\n missing=1\n fi\n done\n exit $missing\n\n - name: Check all PR commits are signed off\n if: github.event_name == 'pull_request'\n run: |\n for c in $(git rev-list origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }}); do\n msg=\"$(git log -1 --pretty=%B \"$c\")\"\n if ! printf '%s\\n' \"$msg\" | grep -q '^Signed-off-by:'; then\n echo \"::error ::Commit $c has not been signed off in the commit message with a \\`Signed-off-by: Your Name \\` line\"\n git log -1 --pretty=format:\"%h %s\" \"$c\"\n missing=1\n fi\n done\n exit $missing\n\n - name: Check PR description is signed off\n if: github.event_name == 'pull_request'\n env:\n PR_DESC: ${{ github.event.pull_request.body }}\n run: |\n grep -Pq '^Signed-off-by:\\s*(?!Your Name|.*)' <<<\"$PR_DESC\" || {\n echo \"::error ::Pull request has not been signed off in the PR description with a \\`Signed-off-by:\\` line\"\n exit 1\n }\n\n lint:\n name: Lint\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version-file: \"go.mod\"\n\n - name: Run golangci-lint\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit 1481404843c368bc19ca9406f87d6e0fc97bdcfd = tag v7.0.0\n uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd\n with:\n version: v2.9.0\n skip-cache: true\n skip-save-cache: true\n args: --timeout=5m --config=.golangci.yml\n\n build-latest:\n name: Build (Latest Go)\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Build NATS Server 64-bit\n run: GOARCH=amd64 GOOS=linux go build\n\n - name: Build NATS Server 32-bit\n run: GOARCH=386 GOOS=linux go build\n\n build-supported:\n name: Build (Minimum Go)\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version-file: \"go.mod\"\n\n - name: Build NATS Server 64-bit\n run: GOARCH=amd64 GOOS=linux go build\n\n - name: Build NATS Server 32-bit\n run: GOARCH=386 GOOS=linux go build\n\n # Using GitHub-supplied workers for Windows for now.\n # Note that the below testing steps depend on the Linux build\n # only, as the Windows builds take a fair bit longer to set up.\n build-windows:\n name: Build (Minimum Go, ${{ matrix.os }})\n strategy:\n fail-fast: false\n matrix:\n os: [windows-2022, windows-2025]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version-file: \"go.mod\"\n\n - name: Build NATS Server\n run: go build\n\n store:\n name: Test Stores\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh store_tests\n\n js-no-cluster:\n name: Test JetStream\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_tests\n\n raft:\n name: Test Raft\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh raft_tests\n\n js-consumers:\n name: Test JetStream Consumers\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_consumer_tests\n\n js-cluster-1:\n name: Test JetStream Cluster 1\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_cluster_tests_1\n\n js-cluster-2:\n name: Test JetStream Cluster 2\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_cluster_tests_2\n\n js-cluster-3:\n name: Test JetStream Cluster 3\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_cluster_tests_3\n\n js-cluster-4:\n name: Test JetStream Cluster 4\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_cluster_tests_4\n\n js-supercluster:\n name: Test JetStream Supercluster\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh js_super_cluster_tests\n\n no-race-1:\n name: Test No-Race 1\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh no_race_1_tests\n\n no-race-2:\n name: Test No-Race 2\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh no_race_2_tests\n\n mqtt:\n name: Test MQTT\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh mqtt_tests\n\n msgtrace:\n name: Test Message Tracing\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh msgtrace_tests\n\n jwt:\n name: Test JWT\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh jwt_tests\n\n server-pkg-non-js:\n name: Test Remaining Server Tests\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_LARGE || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh srv_pkg_non_js_tests\n\n non-server-pkg:\n name: Test Other Packages\n needs: [build-latest, build-supported, lint]\n runs-on: ${{ vars.GHA_WORKER_MEDIUM || 'ubuntu-latest' }}\n timeout-minutes: 30\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n\n - name: Install Go\n uses: actions/setup-go@v6\n with:\n go-version: stable\n\n - name: Run unit tests\n run: ./scripts/runTestsOnTravis.sh non_srv_pkg_tests\n" }, { "path": ".github/workflows/vuln.yaml", "content": "name: Vulnerability Scan\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n schedule:\n - cron: \"0 6 * * *\"\n\npermissions:\n contents: read\n security-events: write\n\njobs:\n govulncheck:\n runs-on: ${{ vars.GHA_WORKER_SMALL || 'ubuntu-latest' }}\n\n steps:\n - name: Run govulncheck\n # Use commit hash here to avoid a re-tagging attack, as this is a third-party action\n # Commit b625fbe08f3bccbe446d94fbf87fcc875a4f50ee = tag v1.0.4\n uses: golang/govulncheck-action@b625fbe08f3bccbe446d94fbf87fcc875a4f50ee\n with:\n go-package: ./...\n output-format: sarif\n output-file: govulncheck.sarif\n\n - name: Upload SARIF results\n if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork\n uses: github/codeql-action/upload-sarif@v4\n with:\n sarif_file: govulncheck.sarif\n" }, { "path": ".gitignore", "content": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\n\n# Folders\n_obj\n_test\ndist\n\n# Configuration Files\n*.conf\n*.cfg\n\n# Architecture specific extensions/prefixes\n*.[568vq]\n[568vq].out\n\n*.cgo1.go\n*.cgo2.c\n_cgo_defun.c\n_cgo_gotypes.go\n_cgo_export.*\n\n_testmain.go\n\n*.exe\n\n# Eclipse\n.project\n\n# IntelliJ\n.idea/\n\n# Emacs\n*~\n\\#*\\#\n.\\#*\n\n# Visual Studio Code\n.vscode\n\n# Mac\n.DS_Store\n\n# bin\nnats-server\ngnatsd\ncheck\n\n# coverage\ncoverage.out\n\n# Cross compiled binaries\npkg\n" }, { "path": ".golangci.yml", "content": "version: \"2\"\nrun:\n concurrency: 4\n modules-download-mode: readonly\n issues-exit-code: 1\n tests: true\noutput:\n formats:\n text:\n path: stdout\n print-linter-name: true\n print-issued-lines: true\nlinters:\n default: none\n enable:\n - forbidigo\n - govet\n - ineffassign\n - misspell\n - staticcheck\n - unused\n settings:\n errcheck:\n check-type-assertions: false\n check-blank: false\n forbidigo:\n forbid:\n - pattern: ^fmt\\.Print(f|ln)?$\n govet:\n settings:\n printf:\n funcs:\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf\n misspell:\n locale: US\n staticcheck:\n checks:\n - all\n - -QF*\n - -ST1003\n - -ST1016\n exclusions:\n generated: lax\n presets:\n - comments\n - common-false-positives\n - legacy\n - std-error-handling\n rules:\n - linters:\n - forbidigo\n path: main.go\n - linters:\n - forbidigo\n source: \"nats-server: v%s\"\n - linters:\n - forbidigo\n path: server/opts.go\n - linters:\n - forbidigo\n path: _test.go\n paths:\n - .github\n - doc\n - docker\n - logos\n - scripts\n - util\n - third_party$\n - builtin$\n - examples$\nformatters:\n enable:\n - goimports\n exclusions:\n generated: lax\n paths:\n - .github\n - doc\n - docker\n - logos\n - scripts\n - util\n - third_party$\n - builtin$\n - examples$\n" }, { "path": ".goreleaser.yml", "content": "project_name: nats-server\nversion: 2\n\nrelease:\n github:\n owner: '{{ envOrDefault \"GITHUB_REPOSITORY_OWNER\" \"nats-io\" }}'\n name: '{{ envOrDefault \"GITHUB_REPOSITORY_NAME\" \"nats-server\" }}'\n name_template: \"Release {{.Tag}}\"\n draft: true\n\nchangelog:\n disable: true\n\nbuilds:\n - main: .\n binary: nats-server\n flags:\n - -trimpath\n ldflags:\n - -w -X 'github.com/nats-io/nats-server/v2/server.gitCommit={{.ShortCommit}}' -X 'github.com/nats-io/nats-server/v2/server.serverVersion={{.Tag}}'\n env:\n # This is the toolchain version we use for releases. To override, set the env var, e.g.:\n # GORELEASER_TOOLCHAIN=\"go1.22.8\" TARGET='linux_amd64' goreleaser build --snapshot --clean --single-target\n - GOTOOLCHAIN={{ envOrDefault \"GORELEASER_TOOLCHAIN\" \"go1.26.1\" }}\n - GO111MODULE=on\n - CGO_ENABLED=0\n goos:\n - darwin\n - linux\n - windows\n - freebsd\n goarch:\n - amd64\n - arm\n - arm64\n - 386\n - loong64\n - mips64le\n - s390x\n - ppc64le\n # RISC-V currently only supported on Linux\n - riscv64\n goarm:\n - 6\n - 7\n ignore:\n - goos: windows\n goarch: arm\n - goos: darwin\n goarch: 386\n - goos: freebsd\n goarch: arm\n - goos: freebsd\n goarch: arm64\n - goos: freebsd\n goarch: 386\n - goos: darwin\n goarch: riscv64\n - goos: windows\n goarch: riscv64\n - goos: freebsd\n goarch: riscv64\n mod_timestamp: \"{{ .CommitTimestamp }}\"\n\nnfpms:\n - file_name_template: \"{{.ProjectName}}-{{.Tag}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}\"\n homepage: https://nats.io\n description: High-Performance server for NATS, the cloud native messaging system.\n maintainer: Ivan Kozlovic \n license: Apache 2.0\n vendor: Synadia Inc.\n formats:\n - deb\n - rpm\n mtime: \"{{ .CommitDate }}\"\n contents:\n - src: /usr/bin/nats-server\n dst: /usr/local/bin/nats-server\n type: \"symlink\"\n rpm:\n buildhost: synadia.com\n\narchives:\n - name_template: \"{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}\"\n id: targz-archives\n wrap_in_directory: true\n formats: [\"tar.gz\"]\n format_overrides:\n - goos: windows\n formats: [\"zip\"]\n builds_info:\n owner: root\n group: root\n mode: 0755\n files:\n - src: README.md\n info:\n owner: root\n group: root\n mode: 0644\n mtime: \"{{ .CommitDate }}\"\n - src: LICENSE\n info:\n owner: root\n group: root\n mode: 0644\n mtime: \"{{ .CommitDate }}\"\n\nchecksum:\n name_template: \"SHA256SUMS\"\n algorithm: sha256\n\nsboms:\n - artifacts: binary\n documents:\n [\n \"{{.ProjectName}}-{{.Tag}}-{{.Os}}-{{.Arch}}{{if .Arm}}{{.Arm}}{{end}}.sbom.spdx.json\",\n ]\n" }, { "path": "AMBASSADORS.md", "content": "# Ambassadors\n\nThe NATS ambassador program recognizes community members that go above and beyond in their contributions to the community and the ecosystem. Learn more [here](https://nats.io/community#nats-ambassador-program).\n\n- [Maurice van Veen](https://nats.io/community#maurice-van-veen) \n" }, { "path": "CODE-OF-CONDUCT.md", "content": "## Community Code of Conduct\n\nNATS follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing\n\nThanks for your interest in contributing! This document contains `nats-io/nats-server` specific contributing details. If you are a first-time contributor, please refer to the general [NATS Contributor Guide](https://nats.io/contributing/) to get a comprehensive overview of contributing to the NATS project.\n\n## Getting started\n\nThere are three general ways you can contribute to this repo:\n\n- Proposing an enhancement or new feature\n- Reporting a bug or regression\n- Contributing changes to the source code\n\nFor the first two, refer to the [GitHub Issues](https://github.com/nats-io/nats-server/issues/new/choose) which guides you through the available options along with the needed information to collect.\n\n## Contributing changes\n\n_Prior to opening a pull request, it is recommended to open an issue first to ensure the maintainers can review intended changes. This saves time for both you and us. Exceptions to this rule include fixing non-functional source such as code comments, documentation or other supporting files._\n\nProposing source code changes is done through GitHub's standard pull request workflow.\n\nIf your branch is a work-in-progress then please start by creating your pull requests as draft, by clicking the down-arrow next to the `Create pull request` button and instead selecting `Create draft pull request`.\n\nThis will defer the automatic process of requesting a review from the NATS team and significantly reduces noise until you are ready. Once you are happy, you can click the `Ready for review` button.\n\n### Guidelines\n\nA good pull request includes:\n\n- A succinct yet descriptive title describing, in a few words, what is fixed/improved/optimised by this change.\n- A high-level description of the changes, including an overview of _why_ the changes are relevant or what problem you are trying to solve. Include links to any issues that are related by adding comments like `Resolves #NNN` to your description. See [Linking a Pull Request to an Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) for more information.\n- An up-to-date parent commit. Please make sure you are pulling in the latest `main` branch and rebasing your work on top of it, i.e. `git rebase main`.\n- Unit tests where appropriate. Bug fixes will benefit from the addition of regression tests and performance improvements will benefit from the addition of new benchmarks where possible. New features will **NOT** be accepted without suitable test coverage!\n- No more commits than necessary. Sometimes having multiple commits is useful for telling a story or isolating changes from one another, but please squash down any unnecessary commits that may just be for clean-up, comments or small changes.\n- No additional external dependencies that aren't absolutely essential. Please do everything you can to avoid pulling in additional libraries/dependencies into `go.mod` as we will be very critical of these.\n\n### Sign-off\n\nIn order to accept a contribution, you will first need to certify that the contribution is your original work and that you license the work to the project under the [Apache-2.0 license](https://github.com/nats-io/nats-server/blob/main/LICENSE).\n\nThis is done by using `Signed-off-by: Your Name ` statements, which should appear in **both** your commit messages and your PR description. Please note that we can only accept sign-offs under a legal name. Nicknames and aliases are not permitted.\n\nTo perform a sign-off when committing with `git`, use `git commit -s` (or `--signoff`) to add the `Signed-off-by:` trailer to your commit message.\n\n## Get help\n\nIf you have questions about the contribution process, please start a [GitHub discussion](https://github.com/nats-io/nats-server/discussions), join the [NATS Slack](https://slack.nats.io/), or send your question to the [NATS Google Group](https://groups.google.com/forum/#!forum/natsio).\n" }, { "path": "DEPENDENCIES.md", "content": "# External Dependencies\n\nThis file lists the dependencies used in this repository.\n\n| Dependency | License |\n|-|-|\n| Go | BSD 3-Clause \"New\" or \"Revised\" License |\n| github.com/nats-io/nats-server/v2 | Apache License 2.0 |\n| github.com/google/go-tpm | Apache License 2.0 |\n| github.com/klauspost/compress | BSD 3-Clause \"New\" or \"Revised\" License |\n| github.com/minio/highwayhash | Apache License 2.0 |\n| github.com/nats-io/jwt/v2 | Apache License 2.0 |\n| github.com/nats-io/nats.go | Apache License 2.0 |\n| github.com/nats-io/nkeys | Apache License 2.0 |\n| github.com/nats-io/nuid | Apache License 2.0 |\n| golang.org/x/crypto | BSD 3-Clause \"New\" or \"Revised\" License |\n| golang.org/x/sys | BSD 3-Clause \"New\" or \"Revised\" License |\n| golang.org/x/time | BSD 3-Clause \"New\" or \"Revised\" License |\n" }, { "path": "GOVERNANCE.md", "content": "# NATS Server Governance\n\nNATS Server is part of the NATS project and is subject to the [NATS Governance](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md).\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MAINTAINERS.md", "content": "# Maintainers\n\nMaintainership is on a per project basis. Reference [NATS Governance Model](https://github.com/nats-io/nats-general/blob/main/GOVERNANCE.md).\n\n### Maintainers\n - Derek Collison [@derekcollison](https://github.com/derekcollison)\n - Ivan Kozlovic [@kozlovic](https://github.com/kozlovic)\n - Waldemar Quevedo [@wallyqs](https://github.com/wallyqs)\n - Oleg Shaldybin [@olegshaldybin](https://github.com/olegshaldybin)\n - R.I. Pienaar [@ripienaar](https://github.com/ripienaar)\n" }, { "path": "README.md", "content": "

\n \"NATS\n

\n\n[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices. NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)). NATS has over [40 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi. NATS can secure and simplify design and operation of modern distributed systems.\n\n[![License][License-Image]][License-Url] [![Build][Build-Status-Image]][Build-Status-Url] [![Release][Release-Image]][Release-Url] [![Slack][Slack-Image]][Slack-Url] [![Coverage][Coverage-Image]][Coverage-Url] [![Docker Downloads][Docker-Image]][Docker-Url] [![GitHub Downloads][GitHub-Image]][Somsubhra-URL] [![CII Best Practices][CIIBestPractices-Image]][CIIBestPractices-Url] [![Artifact Hub][ArtifactHub-Image]][ArtifactHub-Url]\n\n## Documentation\n\n- [Official Website](https://nats.io)\n- [Official Documentation](https://docs.nats.io)\n- [FAQ](https://docs.nats.io/reference/faq)\n- Watch [a video overview](https://rethink.synadia.com/episodes/1/) of NATS.\n- Watch [this video from SCALE 13x](https://www.youtube.com/watch?v=sm63oAVPqAM) to learn more about its origin story and design philosophy.\n\n## Contact\n\n- [Twitter](https://twitter.com/nats_io): Follow us on Twitter!\n- [Google Groups](https://groups.google.com/forum/#!forum/natsio): Where you can ask questions\n- [Slack](https://natsio.slack.com): Click [here](https://slack.nats.io) to join. You can ask questions to our maintainers and to the rich and active community.\n\n## Contributing\n\nIf you are interested in contributing to NATS, read about our...\n\n- [Contributing guide](./CONTRIBUTING.md)\n- [Report issues or propose Pull Requests](https://github.com/nats-io)\n\n[License-Url]: https://www.apache.org/licenses/LICENSE-2.0\n[License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg\n[Docker-Image]: https://img.shields.io/docker/pulls/_/nats.svg\n[Docker-Url]: https://hub.docker.com/_/nats\n[Slack-Image]: https://img.shields.io/badge/chat-on%20slack-green\n[Slack-Url]: https://slack.nats.io\n[Fossa-Url]: https://app.fossa.io/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server?ref=badge_shield\n[Fossa-Image]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnats-io%2Fnats-server.svg?type=shield\n[Build-Status-Url]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml\n[Build-Status-Image]: https://github.com/nats-io/nats-server/actions/workflows/tests.yaml/badge.svg?branch=main\n[Release-Url]: https://github.com/nats-io/nats-server/releases/latest\n[Release-Image]: https://img.shields.io/github/v/release/nats-io/nats-server\n[Coverage-Url]: https://coveralls.io/r/nats-io/nats-server?branch=main\n[Coverage-image]: https://coveralls.io/repos/github/nats-io/nats-server/badge.svg?branch=main\n[ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-server\n[ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-server\n[CIIBestPractices-Url]: https://bestpractices.coreinfrastructure.org/projects/1895\n[CIIBestPractices-Image]: https://bestpractices.coreinfrastructure.org/projects/1895/badge\n[ArtifactHub-Url]: https://artifacthub.io/packages/helm/nats/nats\n[ArtifactHub-Image]: https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/nats\n[GitHub-Release]: https://github.com/nats-io/nats-server/releases/\n[GitHub-Image]: https://img.shields.io/github/downloads/nats-io/nats-server/total.svg?logo=github\n[Somsubhra-url]: https://somsubhra.github.io/github-release-stats/?username=nats-io&repository=nats-server\n\n## Roadmap\n\nThe NATS product roadmap can be found [here](https://nats.io/about/#roadmap).\n\n## Adopters\n\nWho uses NATS? See our [list of users](https://nats.io/#who-uses-nats) on [https://nats.io](https://nats.io).\n\n## Security\n\n### Security Audit\n\nA third party security audit was performed by Trail of Bits following engagement by the Open Source Technology Improvement Fund (OSTIF). You can see the [full report from April 2025 here](https://github.com/trailofbits/publications/blob/master/reviews/2025-04-ostif-nats-securityreview.pdf).\n\n### Reporting Security Vulnerabilities\n\nIf you've found a vulnerability or a potential vulnerability in the NATS server, please let us know at\n[nats-security](mailto:security@nats.io).\n\n## License\n\nUnless otherwise noted, the NATS source files are distributed\nunder the Apache Version 2.0 license found in the LICENSE file.\n" }, { "path": "RELEASES.md", "content": "# Release Workflow\n\n- As of NATS 2.11, the server follows a 6-month release cycle. 2.11 was released in March 2025.\n- Version tagging follows semantic versioning with security fixes clearly marked.\n- The `main` branch is for active development of features and bug fixes. Release branches are created for preparing patch releases. Releases are tagged with semantic versions\n- The current and previous minor series are supported in terms of bug fixes and security patches.\n- Binaries are released via [GitHub Releases](https://github.com/nats-io/nats-server/releases) and Docker images are available as an [official image](https://hub.docker.com/_/nats).\n- Preview releases for the next minor version, e.g. 2.14.0, become available once feature development is complete.\n- Tagged releases as well as nightly Docker images of `main` (for testing) are available in the [synadia/nats-server](https://hub.docker.com/r/synadia/nats-server/tags) Docker repository.\n" }, { "path": "TODO.md", "content": "\n# General\n\n- [ ] Auth for queue groups?\n- [ ] Blacklist or ERR escalation to close connection for auth/permissions\n- [ ] Protocol updates, MAP, MPUB, etc\n- [ ] Multiple listen endpoints\n- [ ] Websocket / HTTP2 strategy\n- [ ] T series reservations\n- [ ] _SYS. server events?\n- [ ] No downtime restart\n- [ ] Signal based reload of configuration\n- [ ] brew, apt-get, rpm, chocately (windows)\n- [ ] IOVec pools and writev for high fanout?\n- [ ] Modify cluster support for single message across routes between pub/sub and d-queue\n- [ ] Memory limits/warnings?\n- [ ] Limit number of subscriptions a client can have, total memory usage etc.\n- [ ] Multi-tenant accounts with isolation of subject space\n- [ ] Pedantic state\n- [X] _SYS.> reserved for server events?\n- [X] Listen configure key vs addr and port\n- [X] Add ENV and variable support to dconf? ucl?\n- [X] Buffer pools/sync pools?\n- [X] Multiple Authorization / Access\n- [X] Write dynamic socket buffer sizes\n- [X] Read dynamic socket buffer sizes\n- [X] Info updates contain other implicit route servers\n- [X] Sublist better at high concurrency, cache uses writelock always currently\n- [X] Switch to 1.4/1.5 and use maps vs hashmaps in sublist\n- [X] NewSource on Rand to lower lock contention on QueueSubs, or redesign!\n- [X] Default sort by cid on connz\n- [X] Track last activity time per connection?\n- [X] Add total connections to varz so we won't miss spikes, etc.\n- [X] Add starttime and uptime to connz list.\n- [X] Gossip Protocol for discovery for clustering\n- [X] Add in HTTP requests to varz?\n- [X] Add favico and help link for monitoring?\n- [X] Better user/pass support using bcrypt etc.\n- [X] SSL/TLS support\n- [X] Add support for / to point to varz, connz, etc..\n- [X] Support sort options for /connz via nats-top\n- [X] Dropped message statistics (slow consumers)\n- [X] Add current time to each monitoring endpoint\n- [X] varz uptime do days and only integer secs\n- [X] Place version in varz (same info sent to clients)\n- [X] Place server ID/UUID in varz\n- [X] nats-top equivalent, utils\n- [X] Connz report routes (/routez)\n- [X] Docker\n- [X] Remove reliance on `ps`\n- [X] Syslog support\n- [X] Client support for language and version\n- [X] Fix benchmarks on linux\n- [X] Daemon mode? Won't fix\n" }, { "path": "conf/fuzz.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build gofuzz\n\npackage conf\n\nfunc Fuzz(data []byte) int {\n\t_, err := Parse(string(data))\n\tif err != nil {\n\t\treturn 0\n\t}\n\treturn 1\n}\n" }, { "path": "conf/lex.go", "content": "// Copyright 2013-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Customized heavily from\n// https://github.com/BurntSushi/toml/blob/master/lex.go, which is based on\n// Rob Pike's talk: http://cuddle.googlecode.com/hg/talk/lex.html\n\n// The format supported is less restrictive than today's formats.\n// Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //)\n// Also supports key value assignments using '=' or ':' or whiteSpace()\n// e.g. foo = 2, foo : 2, foo 2\n// maps can be assigned with no key separator as well\n// semicolons as value terminators in key/value assignments are optional\n//\n// see lex_test.go for more examples.\n\npackage conf\n\nimport (\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"strings\"\n\t\"unicode\"\n\t\"unicode/utf8\"\n)\n\ntype itemType int\n\nconst (\n\titemError itemType = iota\n\titemNIL // used in the parser to indicate no type\n\titemEOF\n\titemKey\n\titemText\n\titemString\n\titemBool\n\titemInteger\n\titemFloat\n\titemDatetime\n\titemArrayStart\n\titemArrayEnd\n\titemMapStart\n\titemMapEnd\n\titemCommentStart\n\titemVariable\n\titemInclude\n)\n\nconst (\n\teof = 0\n\tmapStart = '{'\n\tmapEnd = '}'\n\tkeySepEqual = '='\n\tkeySepColon = ':'\n\tarrayStart = '['\n\tarrayEnd = ']'\n\tarrayValTerm = ','\n\tmapValTerm = ','\n\tcommentHashStart = '#'\n\tcommentSlashStart = '/'\n\tdqStringStart = '\"'\n\tdqStringEnd = '\"'\n\tsqStringStart = '\\''\n\tsqStringEnd = '\\''\n\toptValTerm = ';'\n\ttopOptStart = '{'\n\ttopOptValTerm = ','\n\ttopOptTerm = '}'\n\tblockStart = '('\n\tblockEnd = ')'\n\tmapEndString = string(mapEnd)\n)\n\ntype stateFn func(lx *lexer) stateFn\n\ntype lexer struct {\n\tinput string\n\tstart int\n\tpos int\n\twidth int\n\tline int\n\tstate stateFn\n\titems chan item\n\n\t// A stack of state functions used to maintain context.\n\t// The idea is to reuse parts of the state machine in various places.\n\t// For example, values can appear at the top level or within arbitrarily\n\t// nested arrays. The last state on the stack is used after a value has\n\t// been lexed. Similarly for comments.\n\tstack []stateFn\n\n\t// Used for processing escapable substrings in double-quoted and raw strings\n\tstringParts []string\n\tstringStateFn stateFn\n\n\t// lstart is the start position of the current line.\n\tlstart int\n\n\t// ilstart is the start position of the line from the current item.\n\tilstart int\n}\n\ntype item struct {\n\ttyp itemType\n\tval string\n\tline int\n\tpos int\n}\n\nfunc (lx *lexer) nextItem() item {\n\tfor {\n\t\tselect {\n\t\tcase item := <-lx.items:\n\t\t\treturn item\n\t\tdefault:\n\t\t\tlx.state = lx.state(lx)\n\t\t}\n\t}\n}\n\nfunc lex(input string) *lexer {\n\tlx := &lexer{\n\t\tinput: input,\n\t\tstate: lexTop,\n\t\tline: 1,\n\t\titems: make(chan item, 10),\n\t\tstack: make([]stateFn, 0, 10),\n\t\tstringParts: []string{},\n\t}\n\treturn lx\n}\n\nfunc (lx *lexer) push(state stateFn) {\n\tlx.stack = append(lx.stack, state)\n}\n\nfunc (lx *lexer) pop() stateFn {\n\tif len(lx.stack) == 0 {\n\t\treturn lx.errorf(\"BUG in lexer: no states to pop.\")\n\t}\n\tli := len(lx.stack) - 1\n\tlast := lx.stack[li]\n\tlx.stack = lx.stack[0:li]\n\treturn last\n}\n\nfunc (lx *lexer) emit(typ itemType) {\n\tval := strings.Join(lx.stringParts, \"\") + lx.input[lx.start:lx.pos]\n\t// Position of item in line where it started.\n\tpos := lx.pos - lx.ilstart - len(val)\n\tlx.items <- item{typ, val, lx.line, pos}\n\tlx.start = lx.pos\n\tlx.ilstart = lx.lstart\n}\n\nfunc (lx *lexer) emitString() {\n\tvar finalString string\n\tif len(lx.stringParts) > 0 {\n\t\tfinalString = strings.Join(lx.stringParts, \"\") + lx.input[lx.start:lx.pos]\n\t\tlx.stringParts = []string{}\n\t} else {\n\t\tfinalString = lx.input[lx.start:lx.pos]\n\t}\n\t// Position of string in line where it started.\n\tpos := lx.pos - lx.ilstart - len(finalString)\n\tlx.items <- item{itemString, finalString, lx.line, pos}\n\tlx.start = lx.pos\n\tlx.ilstart = lx.lstart\n}\n\nfunc (lx *lexer) addCurrentStringPart(offset int) {\n\tlx.stringParts = append(lx.stringParts, lx.input[lx.start:lx.pos-offset])\n\tlx.start = lx.pos\n}\n\nfunc (lx *lexer) addStringPart(s string) stateFn {\n\tlx.stringParts = append(lx.stringParts, s)\n\tlx.start = lx.pos\n\treturn lx.stringStateFn\n}\n\nfunc (lx *lexer) hasEscapedParts() bool {\n\treturn len(lx.stringParts) > 0\n}\n\nfunc (lx *lexer) next() (r rune) {\n\tif lx.pos >= len(lx.input) {\n\t\tlx.width = 0\n\t\treturn eof\n\t}\n\n\tif lx.input[lx.pos] == '\\n' {\n\t\tlx.line++\n\n\t\t// Mark start position of current line.\n\t\tlx.lstart = lx.pos\n\t}\n\tr, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:])\n\tlx.pos += lx.width\n\n\treturn r\n}\n\n// ignore skips over the pending input before this point.\nfunc (lx *lexer) ignore() {\n\tlx.start = lx.pos\n\tlx.ilstart = lx.lstart\n}\n\n// backup steps back one rune. Can be called only once per call of next.\nfunc (lx *lexer) backup() {\n\tlx.pos -= lx.width\n\tif lx.pos < len(lx.input) && lx.input[lx.pos] == '\\n' {\n\t\tlx.line--\n\t}\n}\n\n// peek returns but does not consume the next rune in the input.\nfunc (lx *lexer) peek() rune {\n\tr := lx.next()\n\tlx.backup()\n\treturn r\n}\n\n// errorf stops all lexing by emitting an error and returning `nil`.\n// Note that any value that is a character is escaped if it's a special\n// character (new lines, tabs, etc.).\nfunc (lx *lexer) errorf(format string, values ...any) stateFn {\n\tfor i, value := range values {\n\t\tif v, ok := value.(rune); ok {\n\t\t\tvalues[i] = escapeSpecial(v)\n\t\t}\n\t}\n\n\t// Position of error in current line.\n\tpos := lx.pos - lx.lstart\n\tlx.items <- item{\n\t\titemError,\n\t\tfmt.Sprintf(format, values...),\n\t\tlx.line,\n\t\tpos,\n\t}\n\treturn nil\n}\n\n// lexTop consumes elements at the top level of data structure.\nfunc lexTop(lx *lexer) stateFn {\n\tr := lx.next()\n\tif unicode.IsSpace(r) {\n\t\treturn lexSkip(lx, lexTop)\n\t}\n\n\tswitch r {\n\tcase topOptStart:\n\t\tlx.push(lexTop)\n\t\treturn lexSkip(lx, lexBlockStart)\n\tcase commentHashStart:\n\t\tlx.push(lexTop)\n\t\treturn lexCommentStart\n\tcase commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexTop)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase eof:\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\n\t// At this point, the only valid item can be a key, so we back up\n\t// and let the key lexer do the rest.\n\tlx.backup()\n\tlx.push(lexTopValueEnd)\n\treturn lexKeyStart\n}\n\n// lexTopValueEnd is entered whenever a top-level value has been consumed.\n// It must see only whitespace, and will turn back to lexTop upon a new line.\n// If it sees EOF, it will quit the lexer successfully.\nfunc lexTopValueEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == commentHashStart:\n\t\t// a comment will read to a new line for us.\n\t\tlx.push(lexTop)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexTop)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase isWhitespace(r):\n\t\treturn lexTopValueEnd\n\tcase isNL(r) || r == eof || r == optValTerm || r == topOptValTerm || r == topOptTerm:\n\t\tlx.ignore()\n\t\treturn lexTop\n\t}\n\treturn lx.errorf(\"Expected a top-level value to end with a new line, \"+\n\t\t\"comment or EOF, but got '%v' instead.\", r)\n}\n\nfunc lexBlockStart(lx *lexer) stateFn {\n\tr := lx.next()\n\tif unicode.IsSpace(r) {\n\t\treturn lexSkip(lx, lexBlockStart)\n\t}\n\n\tswitch r {\n\tcase topOptStart:\n\t\tlx.push(lexBlockEnd)\n\t\treturn lexSkip(lx, lexBlockStart)\n\tcase topOptTerm:\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\tcase commentHashStart:\n\t\tlx.push(lexBlockStart)\n\t\treturn lexCommentStart\n\tcase commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexBlockStart)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase eof:\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\n\t// At this point, the only valid item can be a key, so we back up\n\t// and let the key lexer do the rest.\n\tlx.backup()\n\tlx.push(lexBlockValueEnd)\n\treturn lexKeyStart\n}\n\n// lexBlockValueEnd is entered whenever a block-level value has been consumed.\n// It must see only whitespace, and will turn back to lexBlockStart upon a new line.\n// If it sees EOF, it will quit the lexer successfully.\nfunc lexBlockValueEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == commentHashStart:\n\t\t// a comment will read to a new line for us.\n\t\tlx.push(lexBlockValueEnd)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexBlockValueEnd)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase isWhitespace(r):\n\t\treturn lexBlockValueEnd\n\tcase isNL(r) || r == optValTerm || r == topOptValTerm:\n\t\tlx.ignore()\n\t\treturn lexBlockStart\n\tcase r == topOptTerm:\n\t\tlx.backup()\n\t\treturn lexBlockEnd\n\t}\n\treturn lx.errorf(\"Expected a block-level value to end with a new line, \"+\n\t\t\"comment or EOF, but got '%v' instead.\", r)\n}\n\n// lexBlockEnd is entered whenever a block-level value has been consumed.\n// It must see only whitespace, and will turn back to lexTop upon a \"}\".\nfunc lexBlockEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == commentHashStart:\n\t\t// a comment will read to a new line for us.\n\t\tlx.push(lexBlockStart)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexBlockStart)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase isNL(r) || isWhitespace(r):\n\t\treturn lexBlockEnd\n\tcase r == optValTerm || r == topOptValTerm:\n\t\tlx.ignore()\n\t\treturn lexBlockStart\n\tcase r == topOptTerm:\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\t}\n\treturn lx.errorf(\"Expected a block-level to end with a '}', but got '%v' instead.\", r)\n}\n\n// lexKeyStart consumes a key name up until the first non-whitespace character.\n// lexKeyStart will ignore whitespace. It will also eat enclosing quotes.\nfunc lexKeyStart(lx *lexer) stateFn {\n\tr := lx.peek()\n\tswitch {\n\tcase isKeySeparator(r):\n\t\treturn lx.errorf(\"Unexpected key separator '%v'\", r)\n\tcase unicode.IsSpace(r):\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexKeyStart)\n\tcase r == dqStringStart:\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexDubQuotedKey)\n\tcase r == sqStringStart:\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexQuotedKey)\n\t}\n\tlx.ignore()\n\tlx.next()\n\treturn lexKey\n}\n\n// lexDubQuotedKey consumes the text of a key between quotes.\nfunc lexDubQuotedKey(lx *lexer) stateFn {\n\tr := lx.peek()\n\tif r == dqStringEnd {\n\t\tlx.emit(itemKey)\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexKeyEnd)\n\t} else if r == eof {\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\tlx.next()\n\treturn lexDubQuotedKey\n}\n\n// lexQuotedKey consumes the text of a key between quotes.\nfunc lexQuotedKey(lx *lexer) stateFn {\n\tr := lx.peek()\n\tif r == sqStringEnd {\n\t\tlx.emit(itemKey)\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexKeyEnd)\n\t} else if r == eof {\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\tlx.next()\n\treturn lexQuotedKey\n}\n\n// keyCheckKeyword will check for reserved keywords as the key value when the key is\n// separated with a space.\nfunc (lx *lexer) keyCheckKeyword(fallThrough, push stateFn) stateFn {\n\tkey := strings.ToLower(lx.input[lx.start:lx.pos])\n\tswitch key {\n\tcase \"include\":\n\t\tlx.ignore()\n\t\tif push != nil {\n\t\t\tlx.push(push)\n\t\t}\n\t\treturn lexIncludeStart\n\t}\n\tlx.emit(itemKey)\n\treturn fallThrough\n}\n\n// lexIncludeStart will consume the whitespace til the start of the value.\nfunc lexIncludeStart(lx *lexer) stateFn {\n\tr := lx.next()\n\tif isWhitespace(r) {\n\t\treturn lexSkip(lx, lexIncludeStart)\n\t}\n\tlx.backup()\n\treturn lexInclude\n}\n\n// lexIncludeQuotedString consumes the inner contents of a string. It assumes that the\n// beginning '\"' has already been consumed and ignored. It will not interpret any\n// internal contents.\nfunc lexIncludeQuotedString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == sqStringEnd:\n\t\tlx.backup()\n\t\tlx.emit(itemInclude)\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\tcase r == eof:\n\t\treturn lx.errorf(\"Unexpected EOF in quoted include\")\n\t}\n\treturn lexIncludeQuotedString\n}\n\n// lexIncludeDubQuotedString consumes the inner contents of a string. It assumes that the\n// beginning '\"' has already been consumed and ignored. It will not interpret any\n// internal contents.\nfunc lexIncludeDubQuotedString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == dqStringEnd:\n\t\tlx.backup()\n\t\tlx.emit(itemInclude)\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\tcase r == eof:\n\t\treturn lx.errorf(\"Unexpected EOF in double quoted include\")\n\t}\n\treturn lexIncludeDubQuotedString\n}\n\n// lexIncludeString consumes the inner contents of a raw string.\nfunc lexIncludeString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase isNL(r) || r == eof || r == optValTerm || r == mapEnd || isWhitespace(r):\n\t\tlx.backup()\n\t\tlx.emit(itemInclude)\n\t\treturn lx.pop()\n\tcase r == sqStringEnd:\n\t\tlx.backup()\n\t\tlx.emit(itemInclude)\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\t}\n\treturn lexIncludeString\n}\n\n// lexInclude will consume the include value.\nfunc lexInclude(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == sqStringStart:\n\t\tlx.ignore() // ignore the \" or '\n\t\treturn lexIncludeQuotedString\n\tcase r == dqStringStart:\n\t\tlx.ignore() // ignore the \" or '\n\t\treturn lexIncludeDubQuotedString\n\tcase r == arrayStart:\n\t\treturn lx.errorf(\"Expected include value but found start of an array\")\n\tcase r == mapStart:\n\t\treturn lx.errorf(\"Expected include value but found start of a map\")\n\tcase r == blockStart:\n\t\treturn lx.errorf(\"Expected include value but found start of a block\")\n\tcase unicode.IsDigit(r), r == '-':\n\t\treturn lx.errorf(\"Expected include value but found start of a number\")\n\tcase r == '\\\\':\n\t\treturn lx.errorf(\"Expected include value but found escape sequence\")\n\tcase isNL(r):\n\t\treturn lx.errorf(\"Expected include value but found new line\")\n\t}\n\tlx.backup()\n\treturn lexIncludeString\n}\n\n// lexKey consumes the text of a key. Assumes that the first character (which\n// is not whitespace) has already been consumed.\nfunc lexKey(lx *lexer) stateFn {\n\tr := lx.peek()\n\tif unicode.IsSpace(r) {\n\t\t// Spaces signal we could be looking at a keyword, e.g. include.\n\t\t// Keywords will eat the keyword and set the appropriate return stateFn.\n\t\treturn lx.keyCheckKeyword(lexKeyEnd, nil)\n\t} else if isKeySeparator(r) || r == eof {\n\t\tlx.emit(itemKey)\n\t\treturn lexKeyEnd\n\t}\n\tlx.next()\n\treturn lexKey\n}\n\n// lexKeyEnd consumes the end of a key (up to the key separator).\n// Assumes that the first whitespace character after a key (or the '=' or ':'\n// separator) has NOT been consumed.\nfunc lexKeyEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase unicode.IsSpace(r):\n\t\treturn lexSkip(lx, lexKeyEnd)\n\tcase isKeySeparator(r):\n\t\treturn lexSkip(lx, lexValue)\n\tcase r == eof:\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\t// We start the value here\n\tlx.backup()\n\treturn lexValue\n}\n\n// lexValue starts the consumption of a value anywhere a value is expected.\n// lexValue will ignore whitespace.\n// After a value is lexed, the last state on the next is popped and returned.\nfunc lexValue(lx *lexer) stateFn {\n\t// We allow whitespace to precede a value, but NOT new lines.\n\t// In array syntax, the array states are responsible for ignoring new lines.\n\tr := lx.next()\n\tif isWhitespace(r) {\n\t\treturn lexSkip(lx, lexValue)\n\t}\n\n\tswitch {\n\tcase r == arrayStart:\n\t\tlx.ignore()\n\t\tlx.emit(itemArrayStart)\n\t\treturn lexArrayValue\n\tcase r == mapStart:\n\t\tlx.ignore()\n\t\tlx.emit(itemMapStart)\n\t\treturn lexMapKeyStart\n\tcase r == sqStringStart:\n\t\tlx.ignore() // ignore the \" or '\n\t\treturn lexQuotedString\n\tcase r == dqStringStart:\n\t\tlx.ignore() // ignore the \" or '\n\t\tlx.stringStateFn = lexDubQuotedString\n\t\treturn lexDubQuotedString\n\tcase r == '-':\n\t\treturn lexNegNumberStart\n\tcase r == blockStart:\n\t\tlx.ignore()\n\t\treturn lexBlock\n\tcase unicode.IsDigit(r):\n\t\tlx.backup() // avoid an extra state and use the same as above\n\t\treturn lexNumberOrDateOrStringOrIPStart\n\tcase r == '.': // special error case, be kind to users\n\t\treturn lx.errorf(\"Floats must start with a digit\")\n\tcase isNL(r):\n\t\treturn lx.errorf(\"Expected value but found new line\")\n\t}\n\tlx.backup()\n\tlx.stringStateFn = lexString\n\treturn lexString\n}\n\n// lexArrayValue consumes one value in an array. It assumes that '[' or ','\n// have already been consumed. All whitespace and new lines are ignored.\nfunc lexArrayValue(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase unicode.IsSpace(r):\n\t\treturn lexSkip(lx, lexArrayValue)\n\tcase r == commentHashStart:\n\t\tlx.push(lexArrayValue)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexArrayValue)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase r == arrayValTerm:\n\t\treturn lx.errorf(\"Unexpected array value terminator '%v'.\", arrayValTerm)\n\tcase r == arrayEnd:\n\t\treturn lexArrayEnd\n\t}\n\n\tlx.backup()\n\tlx.push(lexArrayValueEnd)\n\treturn lexValue\n}\n\n// lexArrayValueEnd consumes the cruft between values of an array. Namely,\n// it ignores whitespace and expects either a ',' or a ']'.\nfunc lexArrayValueEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase isWhitespace(r):\n\t\treturn lexSkip(lx, lexArrayValueEnd)\n\tcase r == commentHashStart:\n\t\tlx.push(lexArrayValueEnd)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexArrayValueEnd)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase r == arrayValTerm || isNL(r):\n\t\treturn lexSkip(lx, lexArrayValue) // Move onto next\n\tcase r == arrayEnd:\n\t\treturn lexArrayEnd\n\t}\n\treturn lx.errorf(\"Expected an array value terminator %q or an array \"+\n\t\t\"terminator %q, but got '%v' instead.\", arrayValTerm, arrayEnd, r)\n}\n\n// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has\n// just been consumed.\nfunc lexArrayEnd(lx *lexer) stateFn {\n\tlx.ignore()\n\tlx.emit(itemArrayEnd)\n\treturn lx.pop()\n}\n\n// lexMapKeyStart consumes a key name up until the first non-whitespace\n// character.\n// lexMapKeyStart will ignore whitespace.\nfunc lexMapKeyStart(lx *lexer) stateFn {\n\tr := lx.peek()\n\tswitch {\n\tcase isKeySeparator(r):\n\t\treturn lx.errorf(\"Unexpected key separator '%v'.\", r)\n\tcase r == arrayEnd:\n\t\treturn lx.errorf(\"Unexpected array end '%v' processing map.\", r)\n\tcase unicode.IsSpace(r):\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapKeyStart)\n\tcase r == mapEnd:\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapEnd)\n\tcase r == commentHashStart:\n\t\tlx.next()\n\t\tlx.push(lexMapKeyStart)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\tlx.next()\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexMapKeyStart)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\tcase r == sqStringStart:\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapQuotedKey)\n\tcase r == dqStringStart:\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapDubQuotedKey)\n\tcase r == eof:\n\t\treturn lx.errorf(\"Unexpected EOF processing map.\")\n\t}\n\tlx.ignore()\n\tlx.next()\n\treturn lexMapKey\n}\n\n// lexMapQuotedKey consumes the text of a key between quotes.\nfunc lexMapQuotedKey(lx *lexer) stateFn {\n\tif r := lx.peek(); r == eof {\n\t\treturn lx.errorf(\"Unexpected EOF processing quoted map key.\")\n\t} else if r == sqStringEnd {\n\t\tlx.emit(itemKey)\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapKeyEnd)\n\t}\n\tlx.next()\n\treturn lexMapQuotedKey\n}\n\n// lexMapDubQuotedKey consumes the text of a key between quotes.\nfunc lexMapDubQuotedKey(lx *lexer) stateFn {\n\tif r := lx.peek(); r == eof {\n\t\treturn lx.errorf(\"Unexpected EOF processing double quoted map key.\")\n\t} else if r == dqStringEnd {\n\t\tlx.emit(itemKey)\n\t\tlx.next()\n\t\treturn lexSkip(lx, lexMapKeyEnd)\n\t}\n\tlx.next()\n\treturn lexMapDubQuotedKey\n}\n\n// lexMapKey consumes the text of a key. Assumes that the first character (which\n// is not whitespace) has already been consumed.\nfunc lexMapKey(lx *lexer) stateFn {\n\tif r := lx.peek(); r == eof {\n\t\treturn lx.errorf(\"Unexpected EOF processing map key.\")\n\t} else if unicode.IsSpace(r) {\n\t\t// Spaces signal we could be looking at a keyword, e.g. include.\n\t\t// Keywords will eat the keyword and set the appropriate return stateFn.\n\t\treturn lx.keyCheckKeyword(lexMapKeyEnd, lexMapValueEnd)\n\t} else if isKeySeparator(r) {\n\t\tlx.emit(itemKey)\n\t\treturn lexMapKeyEnd\n\t}\n\tlx.next()\n\treturn lexMapKey\n}\n\n// lexMapKeyEnd consumes the end of a key (up to the key separator).\n// Assumes that the first whitespace character after a key (or the '='\n// separator) has NOT been consumed.\nfunc lexMapKeyEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase unicode.IsSpace(r):\n\t\treturn lexSkip(lx, lexMapKeyEnd)\n\tcase isKeySeparator(r):\n\t\treturn lexSkip(lx, lexMapValue)\n\t}\n\t// We start the value here\n\tlx.backup()\n\treturn lexMapValue\n}\n\n// lexMapValue consumes one value in a map. It assumes that '{' or ','\n// have already been consumed. All whitespace and new lines are ignored.\n// Map values can be separated by ',' or simple NLs.\nfunc lexMapValue(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase unicode.IsSpace(r):\n\t\treturn lexSkip(lx, lexMapValue)\n\tcase r == mapValTerm:\n\t\treturn lx.errorf(\"Unexpected map value terminator %q.\", mapValTerm)\n\tcase r == mapEnd:\n\t\treturn lexSkip(lx, lexMapEnd)\n\t}\n\tlx.backup()\n\tlx.push(lexMapValueEnd)\n\treturn lexValue\n}\n\n// lexMapValueEnd consumes the cruft between values of a map. Namely,\n// it ignores whitespace and expects either a ',' or a '}'.\nfunc lexMapValueEnd(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase isWhitespace(r):\n\t\treturn lexSkip(lx, lexMapValueEnd)\n\tcase r == commentHashStart:\n\t\tlx.push(lexMapValueEnd)\n\t\treturn lexCommentStart\n\tcase r == commentSlashStart:\n\t\trn := lx.next()\n\t\tif rn == commentSlashStart {\n\t\t\tlx.push(lexMapValueEnd)\n\t\t\treturn lexCommentStart\n\t\t}\n\t\tlx.backup()\n\t\tfallthrough\n\tcase r == optValTerm || r == mapValTerm || isNL(r):\n\t\treturn lexSkip(lx, lexMapKeyStart) // Move onto next\n\tcase r == mapEnd:\n\t\treturn lexSkip(lx, lexMapEnd)\n\t}\n\treturn lx.errorf(\"Expected a map value terminator %q or a map \"+\n\t\t\"terminator %q, but got '%v' instead.\", mapValTerm, mapEnd, r)\n}\n\n// lexMapEnd finishes the lexing of a map. It assumes that a '}' has\n// just been consumed.\nfunc lexMapEnd(lx *lexer) stateFn {\n\tlx.ignore()\n\tlx.emit(itemMapEnd)\n\treturn lx.pop()\n}\n\n// Checks if the unquoted string was actually a boolean\nfunc (lx *lexer) isBool() bool {\n\tstr := strings.ToLower(lx.input[lx.start:lx.pos])\n\treturn str == \"true\" || str == \"false\" ||\n\t\tstr == \"on\" || str == \"off\" ||\n\t\tstr == \"yes\" || str == \"no\"\n}\n\n// Check if the unquoted string is a variable reference, starting with $.\nfunc (lx *lexer) isVariable() bool {\n\tif lx.start >= len(lx.input) {\n\t\treturn false\n\t}\n\tif lx.input[lx.start] == '$' {\n\t\tlx.start += 1\n\t\treturn true\n\t}\n\treturn false\n}\n\n// lexQuotedString consumes the inner contents of a string. It assumes that the\n// beginning '\"' has already been consumed and ignored. It will not interpret any\n// internal contents.\nfunc lexQuotedString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == sqStringEnd:\n\t\tlx.backup()\n\t\tlx.emit(itemString)\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\tcase r == eof:\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\treturn lexQuotedString\n}\n\n// lexDubQuotedString consumes the inner contents of a string. It assumes that the\n// beginning '\"' has already been consumed and ignored. It will not interpret any\n// internal contents.\nfunc lexDubQuotedString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == '\\\\':\n\t\tlx.addCurrentStringPart(1)\n\t\treturn lexStringEscape\n\tcase r == dqStringEnd:\n\t\tlx.backup()\n\t\tlx.emitString()\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\tcase r == eof:\n\t\tif lx.pos > lx.start {\n\t\t\treturn lx.errorf(\"Unexpected EOF.\")\n\t\t}\n\t\tlx.emit(itemEOF)\n\t\treturn nil\n\t}\n\treturn lexDubQuotedString\n}\n\n// lexString consumes the inner contents of a raw string.\nfunc lexString(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == '\\\\':\n\t\tlx.addCurrentStringPart(1)\n\t\treturn lexStringEscape\n\t// Termination of non-quoted strings\n\tcase isNL(r) || r == eof || r == optValTerm ||\n\t\tr == arrayValTerm || r == arrayEnd || r == mapEnd ||\n\t\tisWhitespace(r):\n\n\t\tlx.backup()\n\t\tif lx.hasEscapedParts() {\n\t\t\tlx.emitString()\n\t\t} else if lx.isBool() {\n\t\t\tlx.emit(itemBool)\n\t\t} else if lx.isVariable() {\n\t\t\tlx.emit(itemVariable)\n\t\t} else {\n\t\t\tlx.emitString()\n\t\t}\n\t\treturn lx.pop()\n\tcase r == sqStringEnd:\n\t\tlx.backup()\n\t\tlx.emitString()\n\t\tlx.next()\n\t\tlx.ignore()\n\t\treturn lx.pop()\n\t}\n\treturn lexString\n}\n\n// lexBlock consumes the inner contents as a string. It assumes that the\n// beginning '(' has already been consumed and ignored. It will continue\n// processing until it finds a ')' on a new line by itself.\nfunc lexBlock(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == blockEnd:\n\t\tlx.backup()\n\t\tlx.backup()\n\n\t\t// Looking for a ')' character on a line by itself, if the previous\n\t\t// character isn't a new line, then break so we keep processing the block.\n\t\tif lx.next() != '\\n' {\n\t\t\tlx.next()\n\t\t\tbreak\n\t\t}\n\t\tlx.next()\n\n\t\t// Make sure the next character is a new line or an eof. We want a ')' on a\n\t\t// bare line by itself.\n\t\tswitch lx.next() {\n\t\tcase '\\n', eof:\n\t\t\tlx.backup()\n\t\t\tlx.backup()\n\t\t\tlx.emit(itemString)\n\t\t\tlx.next()\n\t\t\tlx.ignore()\n\t\t\treturn lx.pop()\n\t\t}\n\t\tlx.backup()\n\tcase r == eof:\n\t\treturn lx.errorf(\"Unexpected EOF processing block.\")\n\t}\n\treturn lexBlock\n}\n\n// lexStringEscape consumes an escaped character. It assumes that the preceding\n// '\\\\' has already been consumed.\nfunc lexStringEscape(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch r {\n\tcase 'x':\n\t\treturn lexStringBinary\n\tcase 't':\n\t\treturn lx.addStringPart(\"\\t\")\n\tcase 'n':\n\t\treturn lx.addStringPart(\"\\n\")\n\tcase 'r':\n\t\treturn lx.addStringPart(\"\\r\")\n\tcase '\"':\n\t\treturn lx.addStringPart(\"\\\"\")\n\tcase '\\\\':\n\t\treturn lx.addStringPart(\"\\\\\")\n\t}\n\treturn lx.errorf(\"Invalid escape character '%v'. Only the following \"+\n\t\t\"escape characters are allowed: \\\\xXX, \\\\t, \\\\n, \\\\r, \\\\\\\", \\\\\\\\.\", r)\n}\n\n// lexStringBinary consumes two hexadecimal digits following '\\x'. It assumes\n// that the '\\x' has already been consumed.\nfunc lexStringBinary(lx *lexer) stateFn {\n\tr := lx.next()\n\tif isNL(r) {\n\t\treturn lx.errorf(\"Expected two hexadecimal digits after '\\\\x', but hit end of line\")\n\t}\n\tr = lx.next()\n\tif isNL(r) {\n\t\treturn lx.errorf(\"Expected two hexadecimal digits after '\\\\x', but hit end of line\")\n\t}\n\toffset := lx.pos - 2\n\tbyteString, err := hex.DecodeString(lx.input[offset:lx.pos])\n\tif err != nil {\n\t\treturn lx.errorf(\"Expected two hexadecimal digits after '\\\\x', but got '%s'\", lx.input[offset:lx.pos])\n\t}\n\tlx.addStringPart(string(byteString))\n\treturn lx.stringStateFn\n}\n\n// lexNumberOrDateOrStringOrIPStart consumes either a (positive)\n// integer, a float, a datetime, or IP, or String that started with a\n// number. It assumes that NO negative sign has been consumed, that\n// is triggered above.\nfunc lexNumberOrDateOrStringOrIPStart(lx *lexer) stateFn {\n\tr := lx.next()\n\tif !unicode.IsDigit(r) {\n\t\tif r == '.' {\n\t\t\treturn lx.errorf(\"Floats must start with a digit, not '.'.\")\n\t\t}\n\t\treturn lx.errorf(\"Expected a digit but got '%v'.\", r)\n\t}\n\treturn lexNumberOrDateOrStringOrIP\n}\n\n// lexNumberOrDateOrStringOrIP consumes either a (positive) integer,\n// float, datetime, IP or string without quotes that starts with a\n// number.\nfunc lexNumberOrDateOrStringOrIP(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == '-':\n\t\tif lx.pos-lx.start != 5 {\n\t\t\treturn lx.errorf(\"All ISO8601 dates must be in full Zulu form.\")\n\t\t}\n\t\treturn lexDateAfterYear\n\tcase unicode.IsDigit(r):\n\t\treturn lexNumberOrDateOrStringOrIP\n\tcase r == '.':\n\t\t// Assume float at first, but could be IP\n\t\treturn lexFloatStart\n\tcase isNumberSuffix(r):\n\t\treturn lexConvenientNumber\n\tcase !(isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r)):\n\t\t// Treat it as a string value once we get a rune that\n\t\t// is not a number.\n\t\tlx.stringStateFn = lexString\n\t\treturn lexString\n\t}\n\tlx.backup()\n\tlx.emit(itemInteger)\n\treturn lx.pop()\n}\n\n// lexConvenientNumber is when we have a suffix, e.g. 1k or 1Mb\nfunc lexConvenientNumber(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase r == 'b' || r == 'B' || r == 'i' || r == 'I':\n\t\treturn lexConvenientNumber\n\t}\n\tlx.backup()\n\tif isNL(r) || r == eof || r == mapEnd || r == optValTerm || r == mapValTerm || isWhitespace(r) || unicode.IsDigit(r) {\n\t\tlx.emit(itemInteger)\n\t\treturn lx.pop()\n\t}\n\t// This is not a number, so treat it as a string.\n\tlx.stringStateFn = lexString\n\treturn lexString\n}\n\n// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format.\n// It assumes that \"YYYY-\" has already been consumed.\nfunc lexDateAfterYear(lx *lexer) stateFn {\n\tformats := []rune{\n\t\t// digits are '0'.\n\t\t// everything else is direct equality.\n\t\t'0', '0', '-', '0', '0',\n\t\t'T',\n\t\t'0', '0', ':', '0', '0', ':', '0', '0',\n\t\t'Z',\n\t}\n\tfor _, f := range formats {\n\t\tr := lx.next()\n\t\tif f == '0' {\n\t\t\tif !unicode.IsDigit(r) {\n\t\t\t\treturn lx.errorf(\"Expected digit in ISO8601 datetime, \"+\n\t\t\t\t\t\"but found '%v' instead.\", r)\n\t\t\t}\n\t\t} else if f != r {\n\t\t\treturn lx.errorf(\"Expected '%v' in ISO8601 datetime, \"+\n\t\t\t\t\"but found '%v' instead.\", f, r)\n\t\t}\n\t}\n\tlx.emit(itemDatetime)\n\treturn lx.pop()\n}\n\n// lexNegNumberStart consumes either an integer or a float. It assumes that a\n// negative sign has already been read, but that *no* digits have been consumed.\n// lexNegNumberStart will move to the appropriate integer or float states.\nfunc lexNegNumberStart(lx *lexer) stateFn {\n\t// we MUST see a digit. Even floats have to start with a digit.\n\tr := lx.next()\n\tif !unicode.IsDigit(r) {\n\t\tif r == '.' {\n\t\t\treturn lx.errorf(\"Floats must start with a digit, not '.'.\")\n\t\t}\n\t\treturn lx.errorf(\"Expected a digit but got '%v'.\", r)\n\t}\n\treturn lexNegNumber\n}\n\n// lexNegNumber consumes a negative integer or a float after seeing the first digit.\nfunc lexNegNumber(lx *lexer) stateFn {\n\tr := lx.next()\n\tswitch {\n\tcase unicode.IsDigit(r):\n\t\treturn lexNegNumber\n\tcase r == '.':\n\t\treturn lexFloatStart\n\tcase isNumberSuffix(r):\n\t\treturn lexConvenientNumber\n\t}\n\tlx.backup()\n\tlx.emit(itemInteger)\n\treturn lx.pop()\n}\n\n// lexFloatStart starts the consumption of digits of a float after a '.'.\n// Namely, at least one digit is required.\nfunc lexFloatStart(lx *lexer) stateFn {\n\tr := lx.next()\n\tif !unicode.IsDigit(r) {\n\t\treturn lx.errorf(\"Floats must have a digit after the '.', but got \"+\n\t\t\t\"'%v' instead.\", r)\n\t}\n\treturn lexFloat\n}\n\n// lexFloat consumes the digits of a float after a '.'.\n// Assumes that one digit has been consumed after a '.' already.\nfunc lexFloat(lx *lexer) stateFn {\n\tr := lx.next()\n\tif unicode.IsDigit(r) {\n\t\treturn lexFloat\n\t}\n\n\t// Not a digit, if its another '.', need to see if we falsely assumed a float.\n\tif r == '.' {\n\t\treturn lexIPAddr\n\t}\n\n\tlx.backup()\n\tlx.emit(itemFloat)\n\treturn lx.pop()\n}\n\n// lexIPAddr consumes IP addrs, like 127.0.0.1:4222\nfunc lexIPAddr(lx *lexer) stateFn {\n\tr := lx.next()\n\tif unicode.IsDigit(r) || r == '.' || r == ':' || r == '-' {\n\t\treturn lexIPAddr\n\t}\n\tlx.backup()\n\tlx.emit(itemString)\n\treturn lx.pop()\n}\n\n// lexCommentStart begins the lexing of a comment. It will emit\n// itemCommentStart and consume no characters, passing control to lexComment.\nfunc lexCommentStart(lx *lexer) stateFn {\n\tlx.ignore()\n\tlx.emit(itemCommentStart)\n\treturn lexComment\n}\n\n// lexComment lexes an entire comment. It assumes that '#' has been consumed.\n// It will consume *up to* the first new line character, and pass control\n// back to the last state on the stack.\nfunc lexComment(lx *lexer) stateFn {\n\tr := lx.peek()\n\tif isNL(r) || r == eof {\n\t\tlx.emit(itemText)\n\t\treturn lx.pop()\n\t}\n\tlx.next()\n\treturn lexComment\n}\n\n// lexSkip ignores all slurped input and moves on to the next state.\nfunc lexSkip(lx *lexer, nextState stateFn) stateFn {\n\treturn func(lx *lexer) stateFn {\n\t\tlx.ignore()\n\t\treturn nextState\n\t}\n}\n\n// Tests to see if we have a number suffix\nfunc isNumberSuffix(r rune) bool {\n\treturn r == 'k' || r == 'K' || r == 'm' || r == 'M' || r == 'g' || r == 'G' || r == 't' || r == 'T' || r == 'p' || r == 'P' || r == 'e' || r == 'E'\n}\n\n// Tests for both key separators\nfunc isKeySeparator(r rune) bool {\n\treturn r == keySepEqual || r == keySepColon\n}\n\n// isWhitespace returns true if `r` is a whitespace character according\n// to the spec.\nfunc isWhitespace(r rune) bool {\n\treturn r == '\\t' || r == ' '\n}\n\nfunc isNL(r rune) bool {\n\treturn r == '\\n' || r == '\\r'\n}\n\nfunc (itype itemType) String() string {\n\tswitch itype {\n\tcase itemError:\n\t\treturn \"Error\"\n\tcase itemNIL:\n\t\treturn \"NIL\"\n\tcase itemEOF:\n\t\treturn \"EOF\"\n\tcase itemText:\n\t\treturn \"Text\"\n\tcase itemString:\n\t\treturn \"String\"\n\tcase itemBool:\n\t\treturn \"Bool\"\n\tcase itemInteger:\n\t\treturn \"Integer\"\n\tcase itemFloat:\n\t\treturn \"Float\"\n\tcase itemDatetime:\n\t\treturn \"DateTime\"\n\tcase itemKey:\n\t\treturn \"Key\"\n\tcase itemArrayStart:\n\t\treturn \"ArrayStart\"\n\tcase itemArrayEnd:\n\t\treturn \"ArrayEnd\"\n\tcase itemMapStart:\n\t\treturn \"MapStart\"\n\tcase itemMapEnd:\n\t\treturn \"MapEnd\"\n\tcase itemCommentStart:\n\t\treturn \"CommentStart\"\n\tcase itemVariable:\n\t\treturn \"Variable\"\n\tcase itemInclude:\n\t\treturn \"Include\"\n\t}\n\tpanic(fmt.Sprintf(\"BUG: Unknown type '%s'.\", itype.String()))\n}\n\nfunc (item item) String() string {\n\treturn fmt.Sprintf(\"(%s, '%s', %d, %d)\", item.typ.String(), item.val, item.line, item.pos)\n}\n\nfunc escapeSpecial(c rune) string {\n\tswitch c {\n\tcase '\\n':\n\t\treturn \"\\\\n\"\n\t}\n\treturn string(c)\n}\n" }, { "path": "conf/lex_test.go", "content": "package conf\n\nimport \"testing\"\n\n// Test to make sure we get what we expect.\nfunc expect(t *testing.T, lx *lexer, items []item) {\n\tt.Helper()\n\tfor i := 0; i < len(items); i++ {\n\t\titem := lx.nextItem()\n\t\t_ = item.String()\n\t\tif item.typ == itemEOF {\n\t\t\tbreak\n\t\t}\n\t\tif item != items[i] {\n\t\t\tt.Fatalf(\"Testing: '%s'\\nExpected %q, received %q\\n\",\n\t\t\t\tlx.input, items[i], item)\n\t\t}\n\t\tif item.typ == itemError {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestPlainValue(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleKeyStringValues(t *testing.T) {\n\t// Double quotes\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 7},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = \\\"bar\\\"\")\n\texpect(t, lx, expectedItems)\n\n\t// Single quotes\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 7},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 'bar'\")\n\texpect(t, lx, expectedItems)\n\n\t// No spaces\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 5},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo='bar'\")\n\texpect(t, lx, expectedItems)\n\n\t// NL\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 5},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo='bar'\\r\\n\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=\\t'bar'\\t\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestComplexStringValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\\\\r\\\\n \\\\t\", 1, 7},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\n\tlx := lex(\"foo = 'bar\\\\r\\\\n \\\\t'\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestStringStartingWithNumber(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"3xyz\", 1, 6},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\n\tlx := lex(`foo = 3xyz`)\n\texpect(t, lx, expectedItems)\n\n\tlx = lex(`foo = 3xyz,`)\n\texpect(t, lx, expectedItems)\n\n\tlx = lex(`foo = 3xyz;`)\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 2, 9},\n\t\t{itemString, \"3xyz\", 2, 15},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tcontent := `\n foo = 3xyz\n `\n\tlx = lex(content)\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"map\", 2, 9},\n\t\t{itemMapStart, \"\", 2, 14},\n\t\t{itemKey, \"foo\", 3, 11},\n\t\t{itemString, \"3xyz\", 3, 17},\n\t\t{itemMapEnd, \"\", 3, 22},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tcontent = `\n map {\n foo = 3xyz}\n `\n\tlx = lex(content)\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"map\", 2, 9},\n\t\t{itemMapStart, \"\", 2, 14},\n\t\t{itemKey, \"foo\", 3, 11},\n\t\t{itemString, \"3xyz\", 3, 17},\n\t\t{itemMapEnd, \"\", 4, 10},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tcontent = `\n map {\n foo = 3xyz;\n }\n `\n\tlx = lex(content)\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"map\", 2, 9},\n\t\t{itemMapStart, \"\", 2, 14},\n\t\t{itemKey, \"foo\", 3, 11},\n\t\t{itemString, \"3xyz\", 3, 17},\n\t\t{itemKey, \"bar\", 4, 11},\n\t\t{itemString, \"4wqs\", 4, 17},\n\t\t{itemMapEnd, \"\", 5, 10},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tcontent = `\n map {\n foo = 3xyz,\n bar = 4wqs\n }\n `\n\tlx = lex(content)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBinaryString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"e\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = \\\\x65\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBinaryStringLatin1(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\\xe9\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = \\\\xe9\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleKeyIntegerValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = 123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=123\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo=123\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleKeyNegativeIntegerValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"-123\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = -123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"-123\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=-123\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo=-123\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestConvenientIntegerValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1k\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = 1k\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1K\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1K\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1m\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1m\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1M\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1M\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1g\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1g\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1G\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1G\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1MB\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1MB\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"1Gb\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1Gb\")\n\texpect(t, lx, expectedItems)\n\n\t// Negative versions\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"-1m\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = -1m\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"-1GB\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = -1GB \")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"1Ghz\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 1Ghz\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"2Pie\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 2Pie\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"3Mbs\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 3Mbs,\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"4Gb\", 1, 6},\n\t\t{itemKey, \"bar\", 1, 11},\n\t\t{itemString, \"5Gø\", 1, 17},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = 4Gb, bar = 5Gø\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleKeyFloatValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemFloat, \"22.2\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = 22.2\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemFloat, \"22.2\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=22.2\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo=22.2\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadBinaryStringEndingAfterZeroHexChars(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Expected two hexadecimal digits after '\\\\x', but hit end of line\", 2, 1},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = xyz\\\\x\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadBinaryStringEndingAfterOneHexChar(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Expected two hexadecimal digits after '\\\\x', but hit end of line\", 2, 1},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = xyz\\\\xF\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadBinaryStringWithZeroHexChars(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Expected two hexadecimal digits after '\\\\x', but got ']\\\"'\", 1, 12},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(`foo = \"[\\x]\"`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadBinaryStringWithOneHexChar(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Expected two hexadecimal digits after '\\\\x', but got 'e]'\", 1, 12},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(`foo = \"[\\xe]\"`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadFloatValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Floats must start with a digit\", 1, 7},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = .2\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadKey(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemError, \"Unexpected key separator ':'\", 1, 1},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\" :foo = 22\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleKeyBoolValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemBool, \"true\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = true\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemBool, \"true\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=true\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo=true\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestComments(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemCommentStart, \"\", 1, 1},\n\t\t{itemText, \" This is a comment\", 1, 1},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"# This is a comment\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"# This is a comment\\r\\n\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemCommentStart, \"\", 1, 2},\n\t\t{itemText, \" This is a comment\", 1, 2},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"// This is a comment\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestTopValuesWithComments(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 6},\n\t\t{itemCommentStart, \"\", 1, 12},\n\t\t{itemText, \" This is a comment\", 1, 12},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\n\tlx := lex(\"foo = 123 // This is a comment\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 4},\n\t\t{itemCommentStart, \"\", 1, 12},\n\t\t{itemText, \" This is a comment\", 1, 12},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo=123 # This is a comment\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestRawString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"bar\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = bar\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(`foo = bar' `)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestDateValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemDatetime, \"2016-05-04T18:53:41Z\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\n\tlx := lex(\"foo = 2016-05-04T18:53:41Z\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestVariableValues(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemVariable, \"bar\", 1, 7},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = $bar\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemVariable, \"bar\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo =$bar\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemVariable, \"bar\", 1, 5},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo $bar\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestArrays(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemArrayStart, \"\", 1, 7},\n\t\t{itemInteger, \"1\", 1, 7},\n\t\t{itemInteger, \"2\", 1, 10},\n\t\t{itemInteger, \"3\", 1, 13},\n\t\t{itemString, \"bar\", 1, 17},\n\t\t{itemArrayEnd, \"\", 1, 22},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = [1, 2, 3, 'bar']\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemArrayStart, \"\", 1, 7},\n\t\t{itemInteger, \"1\", 1, 7},\n\t\t{itemInteger, \"2\", 1, 9},\n\t\t{itemInteger, \"3\", 1, 11},\n\t\t{itemString, \"bar\", 1, 14},\n\t\t{itemArrayEnd, \"\", 1, 19},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = [1,2,3,'bar']\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemArrayStart, \"\", 1, 7},\n\t\t{itemInteger, \"1\", 1, 7},\n\t\t{itemInteger, \"2\", 1, 10},\n\t\t{itemInteger, \"3\", 1, 12},\n\t\t{itemString, \"bar\", 1, 15},\n\t\t{itemArrayEnd, \"\", 1, 20},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo = [1, 2,3,'bar']\")\n\texpect(t, lx, expectedItems)\n}\n\nvar mlArray = `\n# top level comment\nfoo = [\n 1, # One\n 2, // Two\n 3 # Three\n 'bar' ,\n \"bar\"\n]\n`\n\nfunc TestMultilineArrays(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemCommentStart, \"\", 2, 2},\n\t\t{itemText, \" top level comment\", 2, 2},\n\t\t{itemKey, \"foo\", 3, 1},\n\t\t{itemArrayStart, \"\", 3, 8},\n\t\t{itemInteger, \"1\", 4, 2},\n\t\t{itemCommentStart, \"\", 4, 6},\n\t\t{itemText, \" One\", 4, 6},\n\t\t{itemInteger, \"2\", 5, 2},\n\t\t{itemCommentStart, \"\", 5, 7},\n\t\t{itemText, \" Two\", 5, 7},\n\t\t{itemInteger, \"3\", 6, 2},\n\t\t{itemCommentStart, \"\", 6, 5},\n\t\t{itemText, \" Three\", 6, 5},\n\t\t{itemString, \"bar\", 7, 3},\n\t\t{itemString, \"bar\", 8, 3},\n\t\t{itemArrayEnd, \"\", 9, 2},\n\t\t{itemEOF, \"\", 9, 0},\n\t}\n\tlx := lex(mlArray)\n\texpect(t, lx, expectedItems)\n}\n\nvar mlArrayNoSep = `\n# top level comment\nfoo = [\n 1 // foo\n 2\n 3\n 'bar'\n \"bar\"\n]\n`\n\nfunc TestMultilineArraysNoSep(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemCommentStart, \"\", 2, 2},\n\t\t{itemText, \" top level comment\", 2, 2},\n\t\t{itemKey, \"foo\", 3, 1},\n\t\t{itemArrayStart, \"\", 3, 8},\n\t\t{itemInteger, \"1\", 4, 2},\n\t\t{itemCommentStart, \"\", 4, 6},\n\t\t{itemText, \" foo\", 4, 6},\n\t\t{itemInteger, \"2\", 5, 2},\n\t\t{itemInteger, \"3\", 6, 2},\n\t\t{itemString, \"bar\", 7, 3},\n\t\t{itemString, \"bar\", 8, 3},\n\t\t{itemArrayEnd, \"\", 9, 2},\n\t\t{itemEOF, \"\", 9, 0},\n\t}\n\tlx := lex(mlArrayNoSep)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSimpleMap(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 7},\n\t\t{itemKey, \"ip\", 1, 7},\n\t\t{itemString, \"127.0.0.1\", 1, 11},\n\t\t{itemKey, \"port\", 1, 23},\n\t\t{itemInteger, \"4242\", 1, 30},\n\t\t{itemMapEnd, \"\", 1, 35},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\n\tlx := lex(\"foo = {ip='127.0.0.1', port = 4242}\")\n\texpect(t, lx, expectedItems)\n}\n\nvar mlMap = `\nfoo = {\n ip = '127.0.0.1' # the IP\n port= 4242 // the port\n}\n`\n\nfunc TestMultilineMap(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemMapStart, \"\", 2, 8},\n\t\t{itemKey, \"ip\", 3, 3},\n\t\t{itemString, \"127.0.0.1\", 3, 9},\n\t\t{itemCommentStart, \"\", 3, 21},\n\t\t{itemText, \" the IP\", 3, 21},\n\t\t{itemKey, \"port\", 4, 3},\n\t\t{itemInteger, \"4242\", 4, 9},\n\t\t{itemCommentStart, \"\", 4, 16},\n\t\t{itemText, \" the port\", 4, 16},\n\t\t{itemMapEnd, \"\", 5, 2},\n\t\t{itemEOF, \"\", 5, 0},\n\t}\n\n\tlx := lex(mlMap)\n\texpect(t, lx, expectedItems)\n}\n\nvar nestedMap = `\nfoo = {\n host = {\n ip = '127.0.0.1'\n port= 4242\n }\n}\n`\n\nfunc TestNestedMaps(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemMapStart, \"\", 2, 8},\n\t\t{itemKey, \"host\", 3, 3},\n\t\t{itemMapStart, \"\", 3, 11},\n\t\t{itemKey, \"ip\", 4, 5},\n\t\t{itemString, \"127.0.0.1\", 4, 11},\n\t\t{itemKey, \"port\", 5, 5},\n\t\t{itemInteger, \"4242\", 5, 11},\n\t\t{itemMapEnd, \"\", 6, 4},\n\t\t{itemMapEnd, \"\", 7, 2},\n\t\t{itemEOF, \"\", 7, 0},\n\t}\n\n\tlx := lex(nestedMap)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestQuotedKeys(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo : 123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 1},\n\t\t{itemInteger, \"123\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"'foo' : 123\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"\\\"foo\\\" : 123\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestQuotedKeysWithSpace(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \" foo\", 1, 1},\n\t\t{itemInteger, \"123\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"' foo' : 123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \" foo\", 1, 1},\n\t\t{itemInteger, \"123\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"\\\" foo\\\" : 123\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestColonKeySep(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo : 123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo:123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 5},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo: 123\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 6},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo: 123\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestWhitespaceKeySep(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 4},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo 123\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo 123\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo\\t123\")\n\texpect(t, lx, expectedItems)\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemInteger, \"123\", 1, 5},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo\\t\\t123\\r\\n\")\n\texpect(t, lx, expectedItems)\n}\n\nvar escString = `\nfoo = \\t\nbar = \\r\nbaz = \\n\nq = \\\"\nbs = \\\\\n`\n\nfunc TestEscapedString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemString, \"\\t\", 2, 9},\n\t\t{itemKey, \"bar\", 3, 1},\n\t\t{itemString, \"\\r\", 3, 9},\n\t\t{itemKey, \"baz\", 4, 1},\n\t\t{itemString, \"\\n\", 4, 9},\n\t\t{itemKey, \"q\", 5, 1},\n\t\t{itemString, \"\\\"\", 5, 9},\n\t\t{itemKey, \"bs\", 6, 1},\n\t\t{itemString, \"\\\\\", 6, 9},\n\t\t{itemEOF, \"\", 6, 0},\n\t}\n\tlx := lex(escString)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringES(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\\\\end\", 1, 8},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \"\\\\end\"`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringSE(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"start\\\\\", 1, 8},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \"start\\\\\"`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringEE(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"Eq\", 1, 12},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\x45\\x71`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringSEE(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"startEq\", 1, 12},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = start\\x45\\x71`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringSES(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"start|end\", 1, 9},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = start\\x7Cend`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringEES(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"<>end\", 1, 12},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\x3c\\x3eend`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestCompoundStringESE(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\", 1, 12},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\x3cmiddle\\x3E`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBadStringEscape(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemError, \"Invalid escape character 'y'. Only the following escape characters are allowed: \\\\xXX, \\\\t, \\\\n, \\\\r, \\\\\\\", \\\\\\\\.\", 1, 8},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\y`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestNonBool(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\\\\true\", 1, 7},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\\\true`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestNonVariable(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\\\\$var\", 1, 7},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \\\\$var`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestEmptyStringDQ(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\", 1, 7},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = \"\"`)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestEmptyStringSQ(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"\", 1, 7},\n\t\t{itemEOF, \"\", 2, 0},\n\t}\n\tlx := lex(`foo = ''`)\n\texpect(t, lx, expectedItems)\n}\n\nvar nestedWhitespaceMap = `\nfoo {\n host {\n ip = '127.0.0.1'\n port= 4242\n }\n}\n`\n\nfunc TestNestedWhitespaceMaps(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemMapStart, \"\", 2, 7},\n\t\t{itemKey, \"host\", 3, 3},\n\t\t{itemMapStart, \"\", 3, 10},\n\t\t{itemKey, \"ip\", 4, 5},\n\t\t{itemString, \"127.0.0.1\", 4, 11},\n\t\t{itemKey, \"port\", 5, 5},\n\t\t{itemInteger, \"4242\", 5, 11},\n\t\t{itemMapEnd, \"\", 6, 4},\n\t\t{itemMapEnd, \"\", 7, 2},\n\t\t{itemEOF, \"\", 7, 0},\n\t}\n\n\tlx := lex(nestedWhitespaceMap)\n\texpect(t, lx, expectedItems)\n}\n\nvar semicolons = `\nfoo = 123;\nbar = 'baz';\nbaz = 'boo'\nmap {\n id = 1;\n}\n`\n\nfunc TestOptionalSemicolons(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemInteger, \"123\", 2, 7},\n\t\t{itemKey, \"bar\", 3, 1},\n\t\t{itemString, \"baz\", 3, 8},\n\t\t{itemKey, \"baz\", 4, 1},\n\t\t{itemString, \"boo\", 4, 8},\n\t\t{itemKey, \"map\", 5, 1},\n\t\t{itemMapStart, \"\", 5, 6},\n\t\t{itemKey, \"id\", 6, 2},\n\t\t{itemInteger, \"1\", 6, 7},\n\t\t{itemMapEnd, \"\", 7, 2},\n\t\t{itemEOF, \"\", 8, 0},\n\t}\n\n\tlx := lex(semicolons)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSemicolonChaining(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemString, \"1\", 1, 5},\n\t\t{itemKey, \"bar\", 1, 9},\n\t\t{itemFloat, \"2.2\", 1, 13},\n\t\t{itemKey, \"baz\", 1, 18},\n\t\t{itemBool, \"true\", 1, 22},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\n\tlx := lex(\"foo='1'; bar=2.2; baz=true;\")\n\texpect(t, lx, expectedItems)\n}\n\nvar noquotes = `\nfoo = 123\nbar = baz\nbaz=boo\nmap {\n id:one\n id2 : onetwo\n}\nt true\nf false\ntstr \"true\"\ntkey = two\nfkey = five # This should be a string\n`\n\nfunc TestNonQuotedStrings(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemInteger, \"123\", 2, 7},\n\t\t{itemKey, \"bar\", 3, 1},\n\t\t{itemString, \"baz\", 3, 7},\n\t\t{itemKey, \"baz\", 4, 1},\n\t\t{itemString, \"boo\", 4, 5},\n\t\t{itemKey, \"map\", 5, 1},\n\t\t{itemMapStart, \"\", 5, 6},\n\t\t{itemKey, \"id\", 6, 2},\n\t\t{itemString, \"one\", 6, 5},\n\t\t{itemKey, \"id2\", 7, 2},\n\t\t{itemString, \"onetwo\", 7, 8},\n\t\t{itemMapEnd, \"\", 8, 2},\n\t\t{itemKey, \"t\", 9, 1},\n\t\t{itemBool, \"true\", 9, 3},\n\t\t{itemKey, \"f\", 10, 1},\n\t\t{itemBool, \"false\", 10, 3},\n\t\t{itemKey, \"tstr\", 11, 1},\n\t\t{itemString, \"true\", 11, 7},\n\t\t{itemKey, \"tkey\", 12, 1},\n\t\t{itemString, \"two\", 12, 8},\n\t\t{itemKey, \"fkey\", 13, 1},\n\t\t{itemString, \"five\", 13, 8},\n\t\t{itemCommentStart, \"\", 13, 14},\n\t\t{itemText, \" This should be a string\", 13, 14},\n\t\t{itemEOF, \"\", 14, 0},\n\t}\n\tlx := lex(noquotes)\n\texpect(t, lx, expectedItems)\n}\n\nvar danglingquote = `\nlisten: \"localhost:4242\n\nhttp: localhost:8222\n`\n\nfunc TestDanglingQuotedString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"listen\", 2, 1},\n\t\t{itemError, \"Unexpected EOF.\", 5, 1},\n\t}\n\tlx := lex(danglingquote)\n\texpect(t, lx, expectedItems)\n}\n\nvar keydanglingquote = `\nfoo = \"\nlisten: \"\n\nhttp: localhost:8222\n\n\"\n`\n\nfunc TestKeyDanglingQuotedString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemString, \"\\nlisten: \", 3, 8},\n\t\t{itemKey, \"http\", 5, 1},\n\t\t{itemString, \"localhost:8222\", 5, 7},\n\t\t{itemError, \"Unexpected EOF.\", 8, 1},\n\t}\n\tlx := lex(keydanglingquote)\n\texpect(t, lx, expectedItems)\n}\n\nvar danglingsquote = `\nlisten: 'localhost:4242\n\nhttp: localhost:8222\n`\n\nfunc TestDanglingSingleQuotedString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"listen\", 2, 1},\n\t\t{itemError, \"Unexpected EOF.\", 5, 1},\n\t}\n\tlx := lex(danglingsquote)\n\texpect(t, lx, expectedItems)\n}\n\nvar keydanglingsquote = `\nfoo = '\nlisten: '\n\nhttp: localhost:8222\n\n'\n`\n\nfunc TestKeyDanglingSingleQuotedString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 2, 1},\n\t\t{itemString, \"\\nlisten: \", 3, 8},\n\t\t{itemKey, \"http\", 5, 1},\n\t\t{itemString, \"localhost:8222\", 5, 7},\n\t\t{itemError, \"Unexpected EOF.\", 8, 1},\n\t}\n\tlx := lex(keydanglingsquote)\n\texpect(t, lx, expectedItems)\n}\n\nvar mapdanglingbracket = `\nlisten = 4222\n\ncluster = {\n\n foo = bar\n\n`\n\nfunc TestMapDanglingBracket(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"listen\", 2, 1},\n\t\t{itemInteger, \"4222\", 2, 10},\n\t\t{itemKey, \"cluster\", 4, 1},\n\t\t{itemMapStart, \"\", 4, 12},\n\t\t{itemKey, \"foo\", 6, 3},\n\t\t{itemString, \"bar\", 6, 9},\n\t\t{itemError, \"Unexpected EOF processing map.\", 8, 1},\n\t}\n\tlx := lex(mapdanglingbracket)\n\texpect(t, lx, expectedItems)\n}\n\nvar blockdanglingparens = `\nlisten = 4222\n\nquote = (\n\n foo = bar\n\n`\n\nfunc TestBlockDanglingParens(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"listen\", 2, 1},\n\t\t{itemInteger, \"4222\", 2, 10},\n\t\t{itemKey, \"quote\", 4, 1},\n\t\t{itemError, \"Unexpected EOF processing block.\", 8, 1},\n\t}\n\tlx := lex(blockdanglingparens)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestMapQuotedKeys(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 7},\n\t\t{itemKey, \"bar\", 1, 8},\n\t\t{itemInteger, \"4242\", 1, 15},\n\t\t{itemMapEnd, \"\", 1, 20},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = {'bar' = 4242}\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo = {\\\"bar\\\" = 4242}\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestSpecialCharsMapQuotedKeys(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 7},\n\t\t{itemKey, \"bar-1.2.3\", 1, 8},\n\t\t{itemMapStart, \"\", 1, 22},\n\t\t{itemKey, \"port\", 1, 23},\n\t\t{itemInteger, \"4242\", 1, 28},\n\t\t{itemMapEnd, \"\", 1, 34},\n\t\t{itemMapEnd, \"\", 1, 35},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"foo = {'bar-1.2.3' = { port:4242 }}\")\n\texpect(t, lx, expectedItems)\n\tlx = lex(\"foo = {\\\"bar-1.2.3\\\" = { port:4242 }}\")\n\texpect(t, lx, expectedItems)\n}\n\nvar mlnestedmap = `\nsystems {\n allinone {\n description: \"This is a description.\"\n }\n}\n`\n\nfunc TestDoubleNestedMapsNewLines(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"systems\", 2, 1},\n\t\t{itemMapStart, \"\", 2, 10},\n\t\t{itemKey, \"allinone\", 3, 3},\n\t\t{itemMapStart, \"\", 3, 13},\n\t\t{itemKey, \"description\", 4, 5},\n\t\t{itemString, \"This is a description.\", 4, 19},\n\t\t{itemMapEnd, \"\", 5, 4},\n\t\t{itemMapEnd, \"\", 6, 2},\n\t\t{itemEOF, \"\", 7, 0},\n\t}\n\tlx := lex(mlnestedmap)\n\texpect(t, lx, expectedItems)\n}\n\nvar blockexample = `\nnumbers (\n1234567890\n)\n`\n\nfunc TestBlockString(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"numbers\", 2, 1},\n\t\t{itemString, \"\\n1234567890\\n\", 4, 10},\n\t}\n\tlx := lex(blockexample)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestBlockStringEOF(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"numbers\", 2, 1},\n\t\t{itemString, \"\\n1234567890\\n\", 4, 10},\n\t}\n\tblockbytes := []byte(blockexample[0 : len(blockexample)-1])\n\tblockbytes = append(blockbytes, 0)\n\tlx := lex(string(blockbytes))\n\texpect(t, lx, expectedItems)\n}\n\nvar mlblockexample = `\nnumbers (\n 12(34)56\n (\n 7890\n )\n)\n`\n\nfunc TestBlockStringMultiLine(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"numbers\", 2, 1},\n\t\t{itemString, \"\\n 12(34)56\\n (\\n 7890\\n )\\n\", 7, 10},\n\t}\n\tlx := lex(mlblockexample)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestUnquotedIPAddr(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \"127.0.0.1:4222\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"listen: 127.0.0.1:4222\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \"127.0.0.1\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen: 127.0.0.1\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \"apcera.me:80\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen: apcera.me:80\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \"nats.io:-1\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen: nats.io:-1\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemInteger, \"-1\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen: -1\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \":-1\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen: :-1\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemString, \":80\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen = :80\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"listen\", 1, 0},\n\t\t{itemArrayStart, \"\", 1, 10},\n\t\t{itemString, \"localhost:4222\", 1, 10},\n\t\t{itemString, \"localhost:4333\", 1, 26},\n\t\t{itemArrayEnd, \"\", 1, 41},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"listen = [localhost:4222, localhost:4333]\")\n\texpect(t, lx, expectedItems)\n}\n\nvar arrayOfMaps = `\nauthorization {\n users = [\n {user: alice, password: foo}\n {user: bob, password: bar}\n ]\n timeout: 0.5\n}\n`\n\nfunc TestArrayOfMaps(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"authorization\", 2, 1},\n\t\t{itemMapStart, \"\", 2, 16},\n\t\t{itemKey, \"users\", 3, 5},\n\t\t{itemArrayStart, \"\", 3, 14},\n\t\t{itemMapStart, \"\", 4, 8},\n\t\t{itemKey, \"user\", 4, 8},\n\t\t{itemString, \"alice\", 4, 14},\n\t\t{itemKey, \"password\", 4, 21},\n\t\t{itemString, \"foo\", 4, 31},\n\t\t{itemMapEnd, \"\", 4, 35},\n\t\t{itemMapStart, \"\", 5, 8},\n\t\t{itemKey, \"user\", 5, 8},\n\t\t{itemString, \"bob\", 5, 14},\n\t\t{itemKey, \"password\", 5, 21},\n\t\t{itemString, \"bar\", 5, 31},\n\t\t{itemMapEnd, \"\", 5, 35},\n\t\t{itemArrayEnd, \"\", 6, 6},\n\t\t{itemKey, \"timeout\", 7, 5},\n\t\t{itemFloat, \"0.5\", 7, 14},\n\t\t{itemMapEnd, \"\", 8, 2},\n\t\t{itemEOF, \"\", 9, 0},\n\t}\n\tlx := lex(arrayOfMaps)\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestInclude(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemInclude, \"users.conf\", 1, 9},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx := lex(\"include \\\"users.conf\\\"\")\n\texpect(t, lx, expectedItems)\n\n\tlx = lex(\"include 'users.conf'\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemInclude, \"users.conf\", 1, 8},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"include users.conf\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestMapInclude(t *testing.T) {\n\texpectedItems := []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 5},\n\t\t{itemInclude, \"users.conf\", 1, 14},\n\t\t{itemMapEnd, \"\", 1, 26},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\n\tlx := lex(\"foo { include users.conf }\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 5},\n\t\t{itemInclude, \"users.conf\", 1, 13},\n\t\t{itemMapEnd, \"\", 1, 24},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo {include users.conf}\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 5},\n\t\t{itemInclude, \"users.conf\", 1, 15},\n\t\t{itemMapEnd, \"\", 1, 28},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo { include 'users.conf' }\")\n\texpect(t, lx, expectedItems)\n\n\texpectedItems = []item{\n\t\t{itemKey, \"foo\", 1, 0},\n\t\t{itemMapStart, \"\", 1, 5},\n\t\t{itemInclude, \"users.conf\", 1, 15},\n\t\t{itemMapEnd, \"\", 1, 27},\n\t\t{itemEOF, \"\", 1, 0},\n\t}\n\tlx = lex(\"foo { include \\\"users.conf\\\"}\")\n\texpect(t, lx, expectedItems)\n}\n\nfunc TestJSONCompat(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tinput string\n\t\texpected []item\n\t}{\n\t\t{\n\t\t\tname: \"should omit initial and final brackets at top level with a single item\",\n\t\t\tinput: `\n {\n \"http_port\": 8223\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 3, 28},\n\t\t\t\t{itemInteger, \"8223\", 3, 40},\n\t\t\t\t{itemKey, \"}\", 4, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should omit trailing commas at top level with two items\",\n\t\t\tinput: `\n {\n \"http_port\": 8223,\n \"port\": 4223\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 3, 28},\n\t\t\t\t{itemInteger, \"8223\", 3, 40},\n\t\t\t\t{itemKey, \"port\", 4, 28},\n\t\t\t\t{itemInteger, \"4223\", 4, 35},\n\t\t\t\t{itemKey, \"}\", 5, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should omit trailing commas at top level with multiple items\",\n\t\t\tinput: `\n {\n \"http_port\": 8223,\n \"port\": 4223,\n \"max_payload\": \"5MB\",\n \"debug\": true,\n \"max_control_line\": 1024\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 3, 28},\n\t\t\t\t{itemInteger, \"8223\", 3, 40},\n\t\t\t\t{itemKey, \"port\", 4, 28},\n\t\t\t\t{itemInteger, \"4223\", 4, 35},\n\t\t\t\t{itemKey, \"max_payload\", 5, 28},\n\t\t\t\t{itemString, \"5MB\", 5, 43},\n\t\t\t\t{itemKey, \"debug\", 6, 28},\n\t\t\t\t{itemBool, \"true\", 6, 36},\n\t\t\t\t{itemKey, \"max_control_line\", 7, 28},\n\t\t\t\t{itemInteger, \"1024\", 7, 47},\n\t\t\t\t{itemKey, \"}\", 8, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should support JSON not prettified\",\n\t\t\tinput: `{\"http_port\": 8224,\"port\": 4224}\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 1, 2},\n\t\t\t\t{itemInteger, \"8224\", 1, 14},\n\t\t\t\t{itemKey, \"port\", 1, 20},\n\t\t\t\t{itemInteger, \"4224\", 1, 27},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should support JSON not prettified with final bracket after newline\",\n\t\t\tinput: `{\"http_port\": 8225,\"port\": 4225\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 1, 2},\n\t\t\t\t{itemInteger, \"8225\", 1, 14},\n\t\t\t\t{itemKey, \"port\", 1, 20},\n\t\t\t\t{itemInteger, \"4225\", 1, 27},\n\t\t\t\t{itemKey, \"}\", 2, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should support uglified JSON with inner blocks\",\n\t\t\tinput: `{\"http_port\": 8227,\"port\": 4227,\"write_deadline\": \"1h\",\"cluster\": {\"port\": 6222,\"routes\": [\"nats://127.0.0.1:4222\",\"nats://127.0.0.1:4223\",\"nats://127.0.0.1:4224\"]}}\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 1, 2},\n\t\t\t\t{itemInteger, \"8227\", 1, 14},\n\t\t\t\t{itemKey, \"port\", 1, 20},\n\t\t\t\t{itemInteger, \"4227\", 1, 27},\n\t\t\t\t{itemKey, \"write_deadline\", 1, 33},\n\t\t\t\t{itemString, \"1h\", 1, 51},\n\t\t\t\t{itemKey, \"cluster\", 1, 56},\n\t\t\t\t{itemMapStart, \"\", 1, 67},\n\t\t\t\t{itemKey, \"port\", 1, 68},\n\t\t\t\t{itemInteger, \"6222\", 1, 75},\n\t\t\t\t{itemKey, \"routes\", 1, 81},\n\t\t\t\t{itemArrayStart, \"\", 1, 91},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4222\", 1, 92},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4223\", 1, 116},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4224\", 1, 140},\n\t\t\t\t{itemArrayEnd, \"\", 1, 163},\n\t\t\t\t{itemMapEnd, \"\", 1, 164},\n\t\t\t\t{itemKey, \"}\", 14, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should support prettified JSON with inner blocks\",\n\t\t\tinput: `\n {\n \"http_port\": 8227,\n \"port\": 4227,\n \"write_deadline\": \"1h\",\n \"cluster\": {\n \"port\": 6222,\n \"routes\": [\n \"nats://127.0.0.1:4222\",\n \"nats://127.0.0.1:4223\",\n \"nats://127.0.0.1:4224\"\n ]\n }\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"http_port\", 3, 28},\n\t\t\t\t{itemInteger, \"8227\", 3, 40},\n\t\t\t\t{itemKey, \"port\", 4, 28},\n\t\t\t\t{itemInteger, \"4227\", 4, 35},\n\t\t\t\t{itemKey, \"write_deadline\", 5, 28},\n\t\t\t\t{itemString, \"1h\", 5, 46},\n\t\t\t\t{itemKey, \"cluster\", 6, 28},\n\t\t\t\t{itemMapStart, \"\", 6, 39},\n\t\t\t\t{itemKey, \"port\", 7, 30},\n\t\t\t\t{itemInteger, \"6222\", 7, 37},\n\t\t\t\t{itemKey, \"routes\", 8, 30},\n\t\t\t\t{itemArrayStart, \"\", 8, 40},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4222\", 9, 32},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4223\", 10, 32},\n\t\t\t\t{itemString, \"nats://127.0.0.1:4224\", 11, 32},\n\t\t\t\t{itemArrayEnd, \"\", 12, 30},\n\t\t\t\t{itemMapEnd, \"\", 13, 28},\n\t\t\t\t{itemKey, \"}\", 14, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"should support JSON with blocks\",\n\t\t\tinput: `{\n \"jetstream\": {\n \"store_dir\": \"/tmp/nats\"\n \"max_mem\": 1000000,\n },\n \"port\": 4222,\n \"server_name\": \"nats1\"\n }\n `,\n\t\t\texpected: []item{\n\t\t\t\t{itemKey, \"jetstream\", 2, 28},\n\t\t\t\t{itemMapStart, \"\", 2, 41},\n\t\t\t\t{itemKey, \"store_dir\", 3, 30},\n\t\t\t\t{itemString, \"/tmp/nats\", 3, 43},\n\t\t\t\t{itemKey, \"max_mem\", 4, 30},\n\t\t\t\t{itemInteger, \"1000000\", 4, 40},\n\t\t\t\t{itemMapEnd, \"\", 5, 28},\n\t\t\t\t{itemKey, \"port\", 6, 28},\n\t\t\t\t{itemInteger, \"4222\", 6, 35},\n\t\t\t\t{itemKey, \"server_name\", 7, 28},\n\t\t\t\t{itemString, \"nats1\", 7, 43},\n\t\t\t\t{itemKey, \"}\", 8, 25},\n\t\t\t\t{itemEOF, \"\", 0, 0},\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tlx := lex(test.input)\n\t\t\texpect(t, lx, test.expected)\n\t\t})\n\t}\n}\n" }, { "path": "conf/parse.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package conf supports a configuration file format used by gnatsd. It is\n// a flexible format that combines the best of traditional\n// configuration formats and newer styles such as JSON and YAML.\npackage conf\n\n// The format supported is less restrictive than today's formats.\n// Supports mixed Arrays [], nested Maps {}, multiple comment types (# and //)\n// Also supports key value assignments using '=' or ':' or whiteSpace()\n// e.g. foo = 2, foo : 2, foo 2\n// maps can be assigned with no key separator as well\n// semicolons as value terminators in key/value assignments are optional\n//\n// see parse_test.go for more examples.\n\nimport (\n\t\"crypto/sha256\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\t\"unicode\"\n)\n\nconst _EMPTY_ = \"\"\n\ntype parser struct {\n\tmapping map[string]any\n\tlx *lexer\n\n\t// The current scoped context, can be array or map\n\tctx any\n\n\t// stack of contexts, either map or array/slice stack\n\tctxs []any\n\n\t// Keys stack\n\tkeys []string\n\n\t// Keys stack as items\n\tikeys []item\n\n\t// The config file path, empty by default.\n\tfp string\n\n\t// pedantic reports error when configuration is not correct.\n\tpedantic bool\n\n\t// Tracks environment variable references, to avoid cycles\n\tenvVarReferences map[string]bool\n}\n\n// Parse will return a map of keys to any, although concrete types\n// underly them. The values supported are string, bool, int64, float64, DateTime.\n// Arrays and nested Maps are also supported.\nfunc Parse(data string) (map[string]any, error) {\n\tp, err := parse(data, \"\", false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn p.mapping, nil\n}\n\n// ParseWithChecks is equivalent to Parse but runs in pedantic mode.\nfunc ParseWithChecks(data string) (map[string]any, error) {\n\tp, err := parse(data, \"\", true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn p.mapping, nil\n}\n\n// ParseFile is a helper to open file, etc. and parse the contents.\nfunc ParseFile(fp string) (map[string]any, error) {\n\tdata, err := os.ReadFile(fp)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error opening config file: %v\", err)\n\t}\n\n\tp, err := parse(string(data), fp, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn p.mapping, nil\n}\n\n// ParseFileWithChecks is equivalent to ParseFile but runs in pedantic mode.\nfunc ParseFileWithChecks(fp string) (map[string]any, error) {\n\tdata, err := os.ReadFile(fp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tp, err := parse(string(data), fp, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn p.mapping, nil\n}\n\n// configDigest returns a digest for the parsed config.\nfunc configDigest(m map[string]any) (string, error) {\n\tdigest := sha256.New()\n\te := json.NewEncoder(digest)\n\tif err := e.Encode(m); err != nil {\n\t\treturn _EMPTY_, err\n\t}\n\treturn fmt.Sprintf(\"sha256:%x\", digest.Sum(nil)), nil\n}\n\n// ParseFileWithChecksDigest returns the processed config and a digest\n// that represents the configuration.\nfunc ParseFileWithChecksDigest(fp string) (map[string]any, string, error) {\n\tm, err := ParseFileWithChecks(fp)\n\tif err != nil {\n\t\treturn nil, _EMPTY_, err\n\t}\n\tdigest, err := configDigest(m)\n\tif err != nil {\n\t\treturn nil, _EMPTY_, err\n\t}\n\treturn m, digest, nil\n}\n\ntype token struct {\n\titem item\n\tvalue any\n\tusedVariable bool\n\tsourceFile string\n}\n\nfunc (t *token) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(t.value)\n}\n\nfunc (t *token) Value() any {\n\treturn t.value\n}\n\nfunc (t *token) Line() int {\n\treturn t.item.line\n}\n\nfunc (t *token) IsUsedVariable() bool {\n\treturn t.usedVariable\n}\n\nfunc (t *token) SourceFile() string {\n\treturn t.sourceFile\n}\n\nfunc (t *token) Position() int {\n\treturn t.item.pos\n}\n\nfunc newParser(data, fp string, pedantic bool) *parser {\n\treturn &parser{\n\t\tmapping: make(map[string]any),\n\t\tlx: lex(data),\n\t\tctxs: make([]any, 0, 4),\n\t\tkeys: make([]string, 0, 4),\n\t\tikeys: make([]item, 0, 4),\n\t\tfp: filepath.Dir(fp),\n\t\tpedantic: pedantic,\n\t\tenvVarReferences: make(map[string]bool),\n\t}\n}\n\nfunc parse(data, fp string, pedantic bool) (*parser, error) {\n\tp := newParser(data, fp, pedantic)\n\tif err := p.parse(fp); err != nil {\n\t\treturn nil, err\n\t}\n\treturn p, nil\n}\n\nfunc parseEnv(data string, parent *parser) (*parser, error) {\n\tp := newParser(data, \"\", false)\n\tp.envVarReferences = parent.envVarReferences\n\tif err := p.parse(\"\"); err != nil {\n\t\treturn nil, err\n\t}\n\treturn p, nil\n}\n\nfunc (p *parser) parse(fp string) error {\n\tp.pushContext(p.mapping)\n\n\tvar prevItem item\n\tfor {\n\t\tit := p.next()\n\t\tif it.typ == itemEOF {\n\t\t\t// Here we allow the final character to be a bracket '}'\n\t\t\t// in order to support JSON like configurations.\n\t\t\tif prevItem.typ == itemKey && prevItem.val != mapEndString {\n\t\t\t\treturn fmt.Errorf(\"config is invalid (%s:%d:%d)\", fp, it.line, it.pos)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tprevItem = it\n\t\tif err := p.processItem(it, fp); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (p *parser) next() item {\n\treturn p.lx.nextItem()\n}\n\nfunc (p *parser) pushContext(ctx any) {\n\tp.ctxs = append(p.ctxs, ctx)\n\tp.ctx = ctx\n}\n\nfunc (p *parser) popContext() any {\n\tif len(p.ctxs) == 0 {\n\t\tpanic(\"BUG in parser, context stack empty\")\n\t}\n\tli := len(p.ctxs) - 1\n\tlast := p.ctxs[li]\n\tp.ctxs = p.ctxs[0:li]\n\tp.ctx = p.ctxs[len(p.ctxs)-1]\n\treturn last\n}\n\nfunc (p *parser) pushKey(key string) {\n\tp.keys = append(p.keys, key)\n}\n\nfunc (p *parser) popKey() string {\n\tif len(p.keys) == 0 {\n\t\tpanic(\"BUG in parser, keys stack empty\")\n\t}\n\tli := len(p.keys) - 1\n\tlast := p.keys[li]\n\tp.keys = p.keys[0:li]\n\treturn last\n}\n\nfunc (p *parser) pushItemKey(key item) {\n\tp.ikeys = append(p.ikeys, key)\n}\n\nfunc (p *parser) popItemKey() item {\n\tif len(p.ikeys) == 0 {\n\t\tpanic(\"BUG in parser, item keys stack empty\")\n\t}\n\tli := len(p.ikeys) - 1\n\tlast := p.ikeys[li]\n\tp.ikeys = p.ikeys[0:li]\n\treturn last\n}\n\nfunc (p *parser) processItem(it item, fp string) error {\n\tsetValue := func(it item, v any) {\n\t\tif p.pedantic {\n\t\t\tp.setValue(&token{it, v, false, fp})\n\t\t} else {\n\t\t\tp.setValue(v)\n\t\t}\n\t}\n\n\tswitch it.typ {\n\tcase itemError:\n\t\treturn fmt.Errorf(\"Parse error on line %d: '%s'\", it.line, it.val)\n\tcase itemKey:\n\t\t// Keep track of the keys as items and strings,\n\t\t// we do this in order to be able to still support\n\t\t// includes without many breaking changes.\n\t\tp.pushKey(it.val)\n\n\t\tif p.pedantic {\n\t\t\tp.pushItemKey(it)\n\t\t}\n\tcase itemMapStart:\n\t\tnewCtx := make(map[string]any)\n\t\tp.pushContext(newCtx)\n\tcase itemMapEnd:\n\t\tsetValue(it, p.popContext())\n\tcase itemString:\n\t\t// FIXME(dlc) sanitize string?\n\t\tsetValue(it, it.val)\n\tcase itemInteger:\n\t\tlastDigit := 0\n\t\tfor _, r := range it.val {\n\t\t\tif !unicode.IsDigit(r) && r != '-' {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tlastDigit++\n\t\t}\n\t\tnumStr := it.val[:lastDigit]\n\t\tnum, err := strconv.ParseInt(numStr, 10, 64)\n\t\tif err != nil {\n\t\t\tif e, ok := err.(*strconv.NumError); ok &&\n\t\t\t\te.Err == strconv.ErrRange {\n\t\t\t\treturn fmt.Errorf(\"integer '%s' is out of the range\", it.val)\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"expected integer, but got '%s'\", it.val)\n\t\t}\n\t\t// Process a suffix\n\t\tsuffix := strings.ToLower(strings.TrimSpace(it.val[lastDigit:]))\n\n\t\tswitch suffix {\n\t\tcase \"\":\n\t\t\tsetValue(it, num)\n\t\tcase \"k\":\n\t\t\tsetValue(it, num*1000)\n\t\tcase \"kb\", \"ki\", \"kib\":\n\t\t\tsetValue(it, num*1024)\n\t\tcase \"m\":\n\t\t\tsetValue(it, num*1000*1000)\n\t\tcase \"mb\", \"mi\", \"mib\":\n\t\t\tsetValue(it, num*1024*1024)\n\t\tcase \"g\":\n\t\t\tsetValue(it, num*1000*1000*1000)\n\t\tcase \"gb\", \"gi\", \"gib\":\n\t\t\tsetValue(it, num*1024*1024*1024)\n\t\tcase \"t\":\n\t\t\tsetValue(it, num*1000*1000*1000*1000)\n\t\tcase \"tb\", \"ti\", \"tib\":\n\t\t\tsetValue(it, num*1024*1024*1024*1024)\n\t\tcase \"p\":\n\t\t\tsetValue(it, num*1000*1000*1000*1000*1000)\n\t\tcase \"pb\", \"pi\", \"pib\":\n\t\t\tsetValue(it, num*1024*1024*1024*1024*1024)\n\t\tcase \"e\":\n\t\t\tsetValue(it, num*1000*1000*1000*1000*1000*1000)\n\t\tcase \"eb\", \"ei\", \"eib\":\n\t\t\tsetValue(it, num*1024*1024*1024*1024*1024*1024)\n\t\t}\n\tcase itemFloat:\n\t\tnum, err := strconv.ParseFloat(it.val, 64)\n\t\tif err != nil {\n\t\t\tif e, ok := err.(*strconv.NumError); ok &&\n\t\t\t\te.Err == strconv.ErrRange {\n\t\t\t\treturn fmt.Errorf(\"float '%s' is out of the range\", it.val)\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"expected float, but got '%s'\", it.val)\n\t\t}\n\t\tsetValue(it, num)\n\tcase itemBool:\n\t\tswitch strings.ToLower(it.val) {\n\t\tcase \"true\", \"yes\", \"on\":\n\t\t\tsetValue(it, true)\n\t\tcase \"false\", \"no\", \"off\":\n\t\t\tsetValue(it, false)\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"expected boolean value, but got '%s'\", it.val)\n\t\t}\n\n\tcase itemDatetime:\n\t\tdt, err := time.Parse(\"2006-01-02T15:04:05Z\", it.val)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\n\t\t\t\t\"expected Zulu formatted DateTime, but got '%s'\", it.val)\n\t\t}\n\t\tsetValue(it, dt)\n\tcase itemArrayStart:\n\t\tvar array = make([]any, 0)\n\t\tp.pushContext(array)\n\tcase itemArrayEnd:\n\t\tarray := p.ctx\n\t\tp.popContext()\n\t\tsetValue(it, array)\n\tcase itemVariable:\n\t\tvalue, found, err := p.lookupVariable(it.val)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"variable reference for '%s' on line %d could not be parsed: %s\",\n\t\t\t\tit.val, it.line, err)\n\t\t}\n\t\tif !found {\n\t\t\treturn fmt.Errorf(\"variable reference for '%s' on line %d can not be found\",\n\t\t\t\tit.val, it.line)\n\t\t}\n\n\t\tif p.pedantic {\n\t\t\tswitch tk := value.(type) {\n\t\t\tcase *token:\n\t\t\t\t// Mark the looked up variable as used, and make\n\t\t\t\t// the variable reference become handled as a token.\n\t\t\t\ttk.usedVariable = true\n\t\t\t\tp.setValue(&token{it, tk.Value(), false, fp})\n\t\t\tdefault:\n\t\t\t\t// Special case to add position context to bcrypt references.\n\t\t\t\tp.setValue(&token{it, value, false, fp})\n\t\t\t}\n\t\t} else {\n\t\t\tp.setValue(value)\n\t\t}\n\tcase itemInclude:\n\t\tvar (\n\t\t\tm map[string]any\n\t\t\terr error\n\t\t)\n\t\tif p.pedantic {\n\t\t\tm, err = ParseFileWithChecks(filepath.Join(p.fp, it.val))\n\t\t} else {\n\t\t\tm, err = ParseFile(filepath.Join(p.fp, it.val))\n\t\t}\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error parsing include file '%s', %v\", it.val, err)\n\t\t}\n\t\tfor k, v := range m {\n\t\t\tp.pushKey(k)\n\n\t\t\tif p.pedantic {\n\t\t\t\tswitch tk := v.(type) {\n\t\t\t\tcase *token:\n\t\t\t\t\tp.pushItemKey(tk.item)\n\t\t\t\t}\n\t\t\t}\n\t\t\tp.setValue(v)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Used to map an environment value into a temporary map to pass to secondary Parse call.\nconst pkey = \"pk\"\n\n// We special case raw strings here that are bcrypt'd. This allows us not to force quoting the strings\nconst bcryptPrefix = \"2a$\"\n\n// lookupVariable will lookup a variable reference. It will use block scoping on keys\n// it has seen before, with the top level scoping being the environment variables. We\n// ignore array contexts and only process the map contexts..\n//\n// Returns true for ok if it finds something, similar to map.\nfunc (p *parser) lookupVariable(varReference string) (any, bool, error) {\n\t// Do special check to see if it is a raw bcrypt string.\n\tif strings.HasPrefix(varReference, bcryptPrefix) {\n\t\treturn \"$\" + varReference, true, nil\n\t}\n\n\t// Loop through contexts currently on the stack.\n\tfor i := len(p.ctxs) - 1; i >= 0; i-- {\n\t\tctx := p.ctxs[i]\n\t\t// Process if it is a map context\n\t\tif m, ok := ctx.(map[string]any); ok {\n\t\t\tif v, ok := m[varReference]; ok {\n\t\t\t\treturn v, ok, nil\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we are here, we have exhausted our context maps and still not found anything.\n\t// Detect reference cycles\n\tif p.envVarReferences[varReference] {\n\t\treturn nil, false, fmt.Errorf(\"variable reference cycle for '%s'\", varReference)\n\t}\n\tp.envVarReferences[varReference] = true\n\tdefer delete(p.envVarReferences, varReference)\n\n\t// Parse from the environment\n\tif vStr, ok := os.LookupEnv(varReference); ok {\n\t\t// Everything we get here will be a string value, so we need to process as a parser would.\n\t\tif subp, err := parseEnv(fmt.Sprintf(\"%s=%s\", pkey, vStr), p); err == nil {\n\t\t\tv, ok := subp.mapping[pkey]\n\t\t\treturn v, ok, nil\n\t\t} else {\n\t\t\treturn nil, false, err\n\t\t}\n\t}\n\treturn nil, false, nil\n}\n\nfunc (p *parser) setValue(val any) {\n\t// Test to see if we are on an array or a map\n\n\t// Array processing\n\tif ctx, ok := p.ctx.([]any); ok {\n\t\tp.ctx = append(ctx, val)\n\t\tp.ctxs[len(p.ctxs)-1] = p.ctx\n\t}\n\n\t// Map processing\n\tif ctx, ok := p.ctx.(map[string]any); ok {\n\t\tkey := p.popKey()\n\n\t\tif p.pedantic {\n\t\t\t// Change the position to the beginning of the key\n\t\t\t// since more useful when reporting errors.\n\t\t\tswitch v := val.(type) {\n\t\t\tcase *token:\n\t\t\t\tit := p.popItemKey()\n\t\t\t\tv.item.pos = it.pos\n\t\t\t\tv.item.line = it.line\n\t\t\t\tctx[key] = v\n\t\t\t}\n\t\t} else {\n\t\t\t// FIXME(dlc), make sure to error if redefining same key?\n\t\t\tctx[key] = val\n\t\t}\n\t}\n}\n" }, { "path": "conf/parse_test.go", "content": "package conf\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\n// Test to make sure we get what we expect.\n\nfunc test(t *testing.T, data string, ex map[string]any) {\n\tt.Helper()\n\tm, err := Parse(data)\n\tif err != nil {\n\t\tt.Fatalf(\"Received err: %v\\n\", err)\n\t}\n\tif m == nil {\n\t\tt.Fatal(\"Received nil map\")\n\t}\n\n\tif !reflect.DeepEqual(m, ex) {\n\t\tt.Fatalf(\"Not Equal:\\nReceived: '%+v'\\nExpected: '%+v'\\n\", m, ex)\n\t}\n\n\t// Also test ParseWithChecks\n\tn, err := ParseWithChecks(data)\n\tif err != nil {\n\t\tt.Fatalf(\"Received err: %v\\n\", err)\n\t}\n\tif n == nil {\n\t\tt.Fatal(\"Received nil map\")\n\t}\n\t// Compare with the original results.\n\ta, err := json.Marshal(m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tb, err := json.Marshal(n)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !bytes.Equal(b, a) {\n\t\tt.Fatalf(\"Not Equal:\\nReceived: '%+v'\\nExpected: '%+v'\\n\", string(a), string(b))\n\t}\n}\n\nfunc TestSimpleTopLevel(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": \"1\",\n\t\t\"bar\": float64(2.2),\n\t\t\"baz\": true,\n\t\t\"boo\": int64(22),\n\t}\n\ttest(t, \"foo='1'; bar=2.2; baz=true; boo=22\", ex)\n}\n\nfunc TestBools(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": true,\n\t}\n\ttest(t, \"foo=true\", ex)\n\ttest(t, \"foo=TRUE\", ex)\n\ttest(t, \"foo=true\", ex)\n\ttest(t, \"foo=yes\", ex)\n\ttest(t, \"foo=on\", ex)\n}\n\nvar varSample = `\n index = 22\n foo = $index\n`\n\nfunc TestSimpleVariable(t *testing.T) {\n\tex := map[string]any{\n\t\t\"index\": int64(22),\n\t\t\"foo\": int64(22),\n\t}\n\ttest(t, varSample, ex)\n}\n\nvar varNestedSample = `\n index = 22\n nest {\n index = 11\n foo = $index\n }\n bar = $index\n`\n\nfunc TestNestedVariable(t *testing.T) {\n\tex := map[string]any{\n\t\t\"index\": int64(22),\n\t\t\"nest\": map[string]any{\n\t\t\t\"index\": int64(11),\n\t\t\t\"foo\": int64(11),\n\t\t},\n\t\t\"bar\": int64(22),\n\t}\n\ttest(t, varNestedSample, ex)\n}\n\nfunc TestMissingVariable(t *testing.T) {\n\t_, err := Parse(\"foo=$index\")\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error for a missing variable, got none\")\n\t}\n\tif !strings.HasPrefix(err.Error(), \"variable reference\") {\n\t\tt.Fatalf(\"Wanted a variable reference err, got %q\\n\", err)\n\t}\n}\n\nfunc TestEnvVariable(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": int64(22),\n\t}\n\tevar := \"__UNIQ22__\"\n\tos.Setenv(evar, \"22\")\n\tdefer os.Unsetenv(evar)\n\ttest(t, fmt.Sprintf(\"foo = $%s\", evar), ex)\n}\n\nfunc TestEnvVariableString(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": \"xyz\",\n\t}\n\tevar := \"__UNIQ22__\"\n\tos.Setenv(evar, \"xyz\")\n\tdefer os.Unsetenv(evar)\n\ttest(t, fmt.Sprintf(\"foo = $%s\", evar), ex)\n}\n\nfunc TestEnvVariableStringStartingWithNumber(t *testing.T) {\n\tevar := \"__UNIQ22__\"\n\tos.Setenv(evar, \"3xyz\")\n\tdefer os.Unsetenv(evar)\n\n\t_, err := Parse(\"foo = $%s\")\n\tif err == nil {\n\t\tt.Fatalf(\"Expected err not being able to process string: %v\\n\", err)\n\t}\n}\n\nfunc TestEnvVariableStringStartingWithNumberAndSizeUnit(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": \"3Gyz\",\n\t}\n\tevar := \"__UNIQ22__\"\n\tos.Setenv(evar, \"3Gyz\")\n\tdefer os.Unsetenv(evar)\n\ttest(t, fmt.Sprintf(\"foo = $%s\", evar), ex)\n}\n\nfunc TestEnvVariableStringStartingWithNumberUsingQuotes(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": \"3xyz\",\n\t}\n\tevar := \"__UNIQ22__\"\n\tos.Setenv(evar, \"'3xyz'\")\n\tdefer os.Unsetenv(evar)\n\ttest(t, fmt.Sprintf(\"foo = $%s\", evar), ex)\n}\n\nfunc TestBcryptVariable(t *testing.T) {\n\tex := map[string]any{\n\t\t\"password\": \"$2a$11$ooo\",\n\t}\n\ttest(t, \"password: $2a$11$ooo\", ex)\n}\n\nvar easynum = `\nk = 8k\nkb = 4kb\nki = 3ki\nkib = 4ki\nm = 1m\nmb = 2MB\nmi = 2Mi\nmib = 64MiB\ng = 2g\ngb = 22GB\ngi = 22Gi\ngib = 22GiB\ntb = 22TB\nti = 22Ti\ntib = 22TiB\npb = 22PB\npi = 22Pi\npib = 22PiB\n`\n\nfunc TestConvenientNumbers(t *testing.T) {\n\tex := map[string]any{\n\t\t\"k\": int64(8 * 1000),\n\t\t\"kb\": int64(4 * 1024),\n\t\t\"ki\": int64(3 * 1024),\n\t\t\"kib\": int64(4 * 1024),\n\t\t\"m\": int64(1000 * 1000),\n\t\t\"mb\": int64(2 * 1024 * 1024),\n\t\t\"mi\": int64(2 * 1024 * 1024),\n\t\t\"mib\": int64(64 * 1024 * 1024),\n\t\t\"g\": int64(2 * 1000 * 1000 * 1000),\n\t\t\"gb\": int64(22 * 1024 * 1024 * 1024),\n\t\t\"gi\": int64(22 * 1024 * 1024 * 1024),\n\t\t\"gib\": int64(22 * 1024 * 1024 * 1024),\n\t\t\"tb\": int64(22 * 1024 * 1024 * 1024 * 1024),\n\t\t\"ti\": int64(22 * 1024 * 1024 * 1024 * 1024),\n\t\t\"tib\": int64(22 * 1024 * 1024 * 1024 * 1024),\n\t\t\"pb\": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024),\n\t\t\"pi\": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024),\n\t\t\"pib\": int64(22 * 1024 * 1024 * 1024 * 1024 * 1024),\n\t}\n\ttest(t, easynum, ex)\n}\n\nfunc TestParseFileWithChecksDigestPreservesConfigKeyUsedAsVariable(t *testing.T) {\n\tconfFile := filepath.Join(t.TempDir(), \"nats.conf\")\n\tif err := os.WriteFile(confFile, []byte(`\n\t\tport = 4222\n\t\tmonitor_port = $port\n\t`), 0666); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tm, _, err := ParseFileWithChecksDigest(confFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif _, ok := m[\"port\"]; !ok {\n\t\tt.Fatal(\"expected config key used as a variable reference to be preserved\")\n\t}\n}\n\nvar sample1 = `\nfoo {\n host {\n ip = '127.0.0.1'\n port = 4242\n }\n servers = [ \"a.com\", \"b.com\", \"c.com\"]\n}\n`\n\nfunc TestSample1(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": map[string]any{\n\t\t\t\"host\": map[string]any{\n\t\t\t\t\"ip\": \"127.0.0.1\",\n\t\t\t\t\"port\": int64(4242),\n\t\t\t},\n\t\t\t\"servers\": []any{\"a.com\", \"b.com\", \"c.com\"},\n\t\t},\n\t}\n\ttest(t, sample1, ex)\n}\n\nvar cluster = `\ncluster {\n port: 4244\n\n authorization {\n user: route_user\n password: top_secret\n timeout: 1\n }\n\n # Routes are actively solicited and connected to from this server.\n # Other servers can connect to us if they supply the correct credentials\n # in their routes definitions from above.\n\n // Test both styles of comments\n\n routes = [\n nats-route://foo:bar@apcera.me:4245\n nats-route://foo:bar@apcera.me:4246\n ]\n}\n`\n\nfunc TestSample2(t *testing.T) {\n\tex := map[string]any{\n\t\t\"cluster\": map[string]any{\n\t\t\t\"port\": int64(4244),\n\t\t\t\"authorization\": map[string]any{\n\t\t\t\t\"user\": \"route_user\",\n\t\t\t\t\"password\": \"top_secret\",\n\t\t\t\t\"timeout\": int64(1),\n\t\t\t},\n\t\t\t\"routes\": []any{\n\t\t\t\t\"nats-route://foo:bar@apcera.me:4245\",\n\t\t\t\t\"nats-route://foo:bar@apcera.me:4246\",\n\t\t\t},\n\t\t},\n\t}\n\n\ttest(t, cluster, ex)\n}\n\nvar sample3 = `\nfoo {\n expr = '(true == \"false\")'\n text = 'This is a multi-line\ntext block.'\n}\n`\n\nfunc TestSample3(t *testing.T) {\n\tex := map[string]any{\n\t\t\"foo\": map[string]any{\n\t\t\t\"expr\": \"(true == \\\"false\\\")\",\n\t\t\t\"text\": \"This is a multi-line\\ntext block.\",\n\t\t},\n\t}\n\ttest(t, sample3, ex)\n}\n\nvar sample4 = `\n array [\n { abc: 123 }\n { xyz: \"word\" }\n ]\n`\n\nfunc TestSample4(t *testing.T) {\n\tex := map[string]any{\n\t\t\"array\": []any{\n\t\t\tmap[string]any{\"abc\": int64(123)},\n\t\t\tmap[string]any{\"xyz\": \"word\"},\n\t\t},\n\t}\n\ttest(t, sample4, ex)\n}\n\nvar sample5 = `\n now = 2016-05-04T18:53:41Z\n gmt = false\n\n`\n\nfunc TestSample5(t *testing.T) {\n\tdt, _ := time.Parse(\"2006-01-02T15:04:05Z\", \"2016-05-04T18:53:41Z\")\n\tex := map[string]any{\n\t\t\"now\": dt,\n\t\t\"gmt\": false,\n\t}\n\ttest(t, sample5, ex)\n}\n\nfunc TestIncludes(t *testing.T) {\n\tex := map[string]any{\n\t\t\"listen\": \"127.0.0.1:4222\",\n\t\t\"authorization\": map[string]any{\n\t\t\t\"ALICE_PASS\": \"$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q\",\n\t\t\t\"BOB_PASS\": \"$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly\",\n\t\t\t\"users\": []any{\n\t\t\t\tmap[string]any{\n\t\t\t\t\t\"user\": \"alice\",\n\t\t\t\t\t\"password\": \"$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q\"},\n\t\t\t\tmap[string]any{\n\t\t\t\t\t\"user\": \"bob\",\n\t\t\t\t\t\"password\": \"$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly\"},\n\t\t\t},\n\t\t\t\"timeout\": float64(0.5),\n\t\t},\n\t}\n\n\tm, err := ParseFile(\"simple.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received err: %v\\n\", err)\n\t}\n\tif m == nil {\n\t\tt.Fatal(\"Received nil map\")\n\t}\n\n\tif !reflect.DeepEqual(m, ex) {\n\t\tt.Fatalf(\"Not Equal:\\nReceived: '%+v'\\nExpected: '%+v'\\n\", m, ex)\n\t}\n}\n\nvar varIncludedVariablesSample = `\nauthorization {\n\n include \"./includes/passwords.conf\"\n\n CAROL_PASS: foo\n\n users = [\n {user: alice, password: $ALICE_PASS}\n {user: bob, password: $BOB_PASS}\n {user: carol, password: $CAROL_PASS}\n ]\n}\n`\n\nfunc TestIncludeVariablesWithChecks(t *testing.T) {\n\tp, err := parse(varIncludedVariablesSample, \"\", true)\n\tif err != nil {\n\t\tt.Fatalf(\"Received err: %v\\n\", err)\n\t}\n\tkey := \"authorization\"\n\tm, ok := p.mapping[key]\n\tif !ok {\n\t\tt.Errorf(\"Expected %q to be in the config\", key)\n\t}\n\texpectKeyVal := func(t *testing.T, m any, expectedKey string, expectedVal string, expectedLine, expectedPos int) {\n\t\tt.Helper()\n\t\ttk := m.(*token)\n\t\tv := tk.Value()\n\t\tvv := v.(map[string]any)\n\t\tvalue, ok := vv[expectedKey]\n\t\tif !ok {\n\t\t\tt.Errorf(\"Expected key %q\", expectedKey)\n\t\t}\n\t\ttk, ok = value.(*token)\n\t\tif !ok {\n\t\t\tt.Fatalf(\"Expected token %v\", value)\n\t\t}\n\t\tif tk.Line() != expectedLine {\n\t\t\tt.Errorf(\"Expected token to be at line %d, got: %d\", expectedLine, tk.Line())\n\t\t}\n\t\tif tk.Position() != expectedPos {\n\t\t\tt.Errorf(\"Expected token to be at position %d, got: %d\", expectedPos, tk.Position())\n\t\t}\n\t\tv = tk.Value()\n\t\tif v != expectedVal {\n\t\t\tt.Errorf(\"Expected %q, got: %s\", expectedVal, v)\n\t\t}\n\t}\n\texpectKeyVal(t, m, \"ALICE_PASS\", \"$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q\", 2, 1)\n\texpectKeyVal(t, m, \"BOB_PASS\", \"$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly\", 3, 1)\n\texpectKeyVal(t, m, \"CAROL_PASS\", \"foo\", 6, 3)\n}\n\nfunc TestParserNoInfiniteLoop(t *testing.T) {\n\tfor _, test := range []string{`A@@Føøøø?˛ø:{øøøø˙˙`, `include \"9/�`} {\n\t\tif _, err := Parse(test); err == nil {\n\t\t\tt.Fatal(\"expected an error\")\n\t\t} else if !strings.Contains(err.Error(), \"Unexpected EOF\") {\n\t\t\tt.Fatal(\"expected unexpected eof error\")\n\t\t}\n\t}\n}\n\nfunc TestParseWithNoValuesAreInvalid(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconf string\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"invalid key without values\",\n\t\t\t`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`,\n\t\t\t\"config is invalid (:1:41)\",\n\t\t},\n\t\t{\n\t\t\t\"invalid untrimmed key without values\",\n\t\t\t` aaaaaaaaaaaaaaaaaaaaaaaaaaa`,\n\t\t\t\"config is invalid (:1:41)\",\n\t\t},\n\t\t{\n\t\t\t\"invalid untrimmed key without values\",\n\t\t\t` aaaaaaaaaaaaaaaaaaaaaaaaaaa `,\n\t\t\t\"config is invalid (:1:41)\",\n\t\t},\n\t\t{\n\t\t\t\"invalid keys after comments\",\n\t\t\t`\n \t\t# with comments and no spaces to create key values\n \t\t# is also an invalid config.\n \t\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n `,\n\t\t\t\"config is invalid (:5:25)\",\n\t\t},\n\t\t{\n\t\t\t\"comma separated without values are invalid\",\n\t\t\t`\n a,a,a,a,a,a,a,a,a,a,a\n `,\n\t\t\t\"config is invalid (:3:25)\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif _, err := parse(test.conf, \"\", true); err == nil {\n\t\t\t\tt.Error(\"expected an error\")\n\t\t\t} else if !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Errorf(\"expected invalid conf error, got: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseWithNoValuesEmptyConfigsAreValid(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconf string\n\t}{\n\t\t{\n\t\t\t\"empty conf\",\n\t\t\t\"\",\n\t\t},\n\t\t{\n\t\t\t\"empty conf with line breaks\",\n\t\t\t`\n\n\n `,\n\t\t},\n\t\t{\n\t\t\t\"just comments with no values\",\n\t\t\t`\n # just comments with no values\n # is still valid.\n `,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif _, err := parse(test.conf, \"\", true); err != nil {\n\t\t\t\tt.Errorf(\"unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseWithTrailingBracketsAreValid(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconf string\n\t}{\n\t\t{\n\t\t\t\"empty conf\",\n\t\t\t\"{}\",\n\t\t},\n\t\t{\n\t\t\t\"just comments with no values\",\n\t\t\t`\n {\n # comments in the body\n }\n `,\n\t\t},\n\t\t{\n\t\t\t// trailing brackets accidentally can become keys,\n\t\t\t// this is valid since needed to support JSON like configs..\n\t\t\t\"trailing brackets after config\",\n\t\t\t`\n accounts { users = [{}]}\n }\n `,\n\t\t},\n\t\t{\n\t\t\t\"wrapped in brackets\",\n\t\t\t`{\n accounts { users = [{}]}\n }\n `,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif _, err := parse(test.conf, \"\", true); err != nil {\n\t\t\t\tt.Errorf(\"unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseWithNoValuesIncludes(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tinput string\n\t\tincludes map[string]string\n\t\terr string\n\t\tlinepos string\n\t}{\n\t\t{\n\t\t\t`# includes\n\t\t\taccounts {\n foo { include 'foo.conf'}\n bar { users = [{user = \"bar\"}] }\n quux { include 'quux.conf'}\n }\n `,\n\t\t\tmap[string]string{\n\t\t\t\t\"foo.conf\": ``,\n\t\t\t\t\"quux.conf\": `?????????????`,\n\t\t\t},\n\t\t\t\"error parsing include file 'quux.conf', config is invalid\",\n\t\t\t\"quux.conf:1:1\",\n\t\t},\n\t\t{\n\t\t\t`# includes\n\t\t\taccounts {\n foo { include 'foo.conf'}\n bar { include 'bar.conf'}\n quux { include 'quux.conf'}\n }\n `,\n\t\t\tmap[string]string{\n\t\t\t\t\"foo.conf\": ``, // Empty configs are ok\n\t\t\t\t\"bar.conf\": `AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`,\n\t\t\t\t\"quux.conf\": `\n # just some comments,\n # and no key values also ok.\n `,\n\t\t\t},\n\t\t\t\"error parsing include file 'bar.conf', config is invalid\",\n\t\t\t\"bar.conf:1:34\",\n\t\t},\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tsdir := t.TempDir()\n\t\t\tf, err := os.CreateTemp(sdir, \"nats.conf-\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif test.includes != nil {\n\t\t\t\tfor includeFile, contents := range test.includes {\n\t\t\t\t\tinf, err := os.Create(filepath.Join(sdir, includeFile))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t\tif err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil {\n\t\t\t\t\t\tt.Error(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, err := parse(test.input, f.Name(), true); err == nil {\n\t\t\t\tt.Error(\"expected an error\")\n\t\t\t} else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) {\n\t\t\t\tt.Errorf(\"expected invalid conf error, got: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJSONParseCompat(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tinput string\n\t\tincludes map[string]string\n\t\texpected map[string]any\n\t}{\n\t\t{\n\t\t\t\"JSON with nested blocks\",\n\t\t\t`\n {\n \"http_port\": 8227,\n \"port\": 4227,\n \"write_deadline\": \"1h\",\n \"cluster\": {\n \"port\": 6222,\n \"routes\": [\n \"nats://127.0.0.1:4222\",\n \"nats://127.0.0.1:4223\",\n \"nats://127.0.0.1:4224\"\n ]\n }\n }\n `,\n\t\t\tnil,\n\t\t\tmap[string]any{\n\t\t\t\t\"http_port\": int64(8227),\n\t\t\t\t\"port\": int64(4227),\n\t\t\t\t\"write_deadline\": \"1h\",\n\t\t\t\t\"cluster\": map[string]any{\n\t\t\t\t\t\"port\": int64(6222),\n\t\t\t\t\t\"routes\": []any{\n\t\t\t\t\t\t\"nats://127.0.0.1:4222\",\n\t\t\t\t\t\t\"nats://127.0.0.1:4223\",\n\t\t\t\t\t\t\"nats://127.0.0.1:4224\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"JSON with nested blocks\",\n\t\t\t`{\n \"jetstream\": {\n \"store_dir\": \"/tmp/nats\"\n \"max_mem\": 1000000,\n },\n \"port\": 4222,\n \"server_name\": \"nats1\"\n }\n `,\n\t\t\tnil,\n\t\t\tmap[string]any{\n\t\t\t\t\"jetstream\": map[string]any{\n\t\t\t\t\t\"store_dir\": \"/tmp/nats\",\n\t\t\t\t\t\"max_mem\": int64(1_000_000),\n\t\t\t\t},\n\t\t\t\t\"port\": int64(4222),\n\t\t\t\t\"server_name\": \"nats1\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"JSON empty object in one line\",\n\t\t\t`{}`,\n\t\t\tnil,\n\t\t\tmap[string]any{},\n\t\t},\n\t\t{\n\t\t\t\"JSON empty object with line breaks\",\n\t\t\t`\n {\n }\n `,\n\t\t\tnil,\n\t\t\tmap[string]any{},\n\t\t},\n\t\t{\n\t\t\t\"JSON includes\",\n\t\t\t`\n accounts {\n foo { include 'foo.json' }\n bar { include 'bar.json' }\n quux { include 'quux.json' }\n }\n `,\n\t\t\tmap[string]string{\n\t\t\t\t\"foo.json\": `{ \"users\": [ {\"user\": \"foo\"} ] }`,\n\t\t\t\t\"bar.json\": `{\n \"users\": [ {\"user\": \"bar\"} ]\n }`,\n\t\t\t\t\"quux.json\": `{}`,\n\t\t\t},\n\t\t\tmap[string]any{\n\t\t\t\t\"accounts\": map[string]any{\n\t\t\t\t\t\"foo\": map[string]any{\n\t\t\t\t\t\t\"users\": []any{\n\t\t\t\t\t\t\tmap[string]any{\n\t\t\t\t\t\t\t\t\"user\": \"foo\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t\"bar\": map[string]any{\n\t\t\t\t\t\t\"users\": []any{\n\t\t\t\t\t\t\tmap[string]any{\n\t\t\t\t\t\t\t\t\"user\": \"bar\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t\"quux\": map[string]any{},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsdir := t.TempDir()\n\t\t\tf, err := os.CreateTemp(sdir, \"nats.conf-\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif test.includes != nil {\n\t\t\t\tfor includeFile, contents := range test.includes {\n\t\t\t\t\tinf, err := os.Create(filepath.Join(sdir, includeFile))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t\tif err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil {\n\t\t\t\t\t\tt.Error(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tm, err := ParseFile(f.Name())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(m, test.expected) {\n\t\t\t\tt.Fatalf(\"Not Equal:\\nReceived: '%+v'\\nExpected: '%+v'\\n\", m, test.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestBlocks(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tinput string\n\t\texpected map[string]any\n\t\terr string\n\t\tlinepos string\n\t}{\n\t\t{\n\t\t\t\"inline block\",\n\t\t\t`{ listen: 0.0.0.0:4222 }`,\n\t\t\tmap[string]any{\n\t\t\t\t\"listen\": \"0.0.0.0:4222\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"newline block\",\n\t\t\t`{\n\t\t\t\tlisten: 0.0.0.0:4222\n\t\t\t }`,\n\t\t\tmap[string]any{\n\t\t\t\t\"listen\": \"0.0.0.0:4222\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"newline block with trailing comment\",\n\t\t\t`\n\t\t\t{\n\t\t\t\tlisten: 0.0.0.0:4222\n\t\t\t}\n\t\t\t# wibble\n\t\t\t`,\n\t\t\tmap[string]any{\n\t\t\t\t\"listen\": \"0.0.0.0:4222\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"nested newline blocks with trailing comment\",\n\t\t\t`\n\t\t\t{\n\t\t\t\t{\n\t\t\t\t\tlisten: 0.0.0.0:4222 // random comment\n\t\t\t\t}\n\t\t\t\t# wibble1\n\t\t\t}\n\t\t\t# wibble2\n\t\t\t`,\n\t\t\tmap[string]any{\n\t\t\t\t\"listen\": \"0.0.0.0:4222\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"top line values in block scope\",\n\t\t\t`\n\t\t\t{\n\t\t\t \"debug\": False\n\t\t\t \"prof_port\": 8221\n\t\t\t \"server_name\": \"aws-useast2-natscj1-1\"\n\t\t\t}\n\t\t\t`,\n\t\t\tmap[string]any{\n\t\t\t\t\"debug\": false,\n\t\t\t\t\"prof_port\": int64(8221),\n\t\t\t\t\"server_name\": \"aws-useast2-natscj1-1\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"comment in block scope after value parse\",\n\t\t\t`\n\t\t\t{\n\t\t\t \"debug\": False\n\t\t\t \"server_name\": \"gcp-asianortheast3-natscj1-1\"\n\n\t\t\t # Profile port specification.\n\t\t\t \"prof_port\": 8221\n\t\t\t}\n\t\t\t`,\n\t\t\tmap[string]any{\n\t\t\t\t\"debug\": false,\n\t\t\t\t\"prof_port\": int64(8221),\n\t\t\t\t\"server_name\": \"gcp-asianortheast3-natscj1-1\",\n\t\t\t},\n\t\t\t\"\", \"\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tf, err := os.CreateTemp(t.TempDir(), \"nats.conf-\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif m, err := ParseFile(f.Name()); err == nil {\n\t\t\t\tif !reflect.DeepEqual(m, test.expected) {\n\t\t\t\t\tt.Fatalf(\"Not Equal:\\nReceived: '%+v'\\nExpected: '%+v'\\n\", m, test.expected)\n\t\t\t\t}\n\t\t\t} else if !strings.Contains(err.Error(), test.err) || !strings.Contains(err.Error(), test.linepos) {\n\t\t\t\tt.Errorf(\"expected invalid conf error, got: %v\", err)\n\t\t\t} else if err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseDigest(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tinput string\n\t\tincludes map[string]string\n\t\tdigest string\n\t}{\n\t\t{\n\t\t\t`foo = bar`,\n\t\t\tnil,\n\t\t\t\"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de\",\n\t\t},\n\t\t{\n\t\t\t`# Comments and whitespace have no effect\n foo = bar\n `,\n\t\t\tnil,\n\t\t\t\"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de\",\n\t\t},\n\t\t{\n\t\t\t`# Syntax changes have no effect\n 'foo': 'bar'\n `,\n\t\t\tnil,\n\t\t\t\"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de\",\n\t\t},\n\t\t{\n\t\t\t`# Syntax changes have no effect\n { 'foo': 'bar' }\n `,\n\t\t\tnil,\n\t\t\t\"sha256:226e49e13d16e5e8aa0d62e58cd63361bf097d3e2b2444aa3044334628a2e8de\",\n\t\t},\n\t\t{\n\t\t\t`# substitutions\n BAR_USERS = { users = [ {user = \"bar\"} ]}\n hello = 'world'\n accounts {\n QUUX_USERS = [ { user: quux }]\n bar = $BAR_USERS\n quux = { users = $QUUX_USERS }\n }\n very { nested { env { VAR = 'NESTED', quux = $VAR }}}\n `,\n\t\t\tnil,\n\t\t\t\"sha256:bddb282343249c26d3edcef9bfaa9d2711505fc67210380e871405ba394a9186\",\n\t\t},\n\t\t{\n\t\t\t`# substitutions, same as previous one without env vars.\n hello = 'world'\n accounts {\n bar = { users = [ { user = \"bar\" } ]}\n quux = { users = [ { user: quux } ]}\n }\n very { nested { env { quux = 'NESTED' }}}\n `,\n\t\t\tnil,\n\t\t\t\"sha256:34f8faf3f269fe7509edc4742f20c8c4a7ad51fe21f8b361764314b533ac3ab5\",\n\t\t},\n\t\t{\n\t\t\t`# substitutions\n BAR_USERS = { users = [ {user = \"foo\"} ]}\n bar = $BAR_USERS\n accounts {\n users = $BAR_USERS\n }\n `,\n\t\t\tnil,\n\t\t\t\"sha256:b0e2ba0b8ec2cb75681b5c5d61789cda0c9942c2f9fe16cdb7f6703364485360\",\n\t\t},\n\t\t{\n\t\t\t`# substitutions\n bar = { users = [ {user = \"foo\"} ]}\n accounts {\n users = { users = [ {user = \"foo\"} ]}\n }\n `,\n\t\t\tnil,\n\t\t\t\"sha256:f5d943b4ed22b80c6199203f8a7eaa8eb68ef7b2d46ef6b1b26f05e21f8beb13\",\n\t\t},\n\t\t{\n\t\t\t`# includes\n\t\t\taccounts {\n foo { include 'foo.conf'}\n bar { users = [{user = \"bar\"}] }\n quux { include 'quux.conf'}\n }\n `,\n\t\t\tmap[string]string{\n\t\t\t\t\"foo.conf\": ` users = [{user = \"foo\"}]`,\n\t\t\t\t\"quux.conf\": ` users = [{user = \"quux\"}]`,\n\t\t\t},\n\t\t\t\"sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9\",\n\t\t},\n\t\t{\n\t\t\t`# includes\n\t\t\taccounts {\n foo { include 'foo.conf'}\n bar { include 'bar.conf'}\n quux { include 'quux.conf'}\n }\n `,\n\t\t\tmap[string]string{\n\t\t\t\t\"foo.conf\": ` users = [{user = \"foo\"}]`,\n\t\t\t\t\"bar.conf\": ` users = [{user = \"bar\"}]`,\n\t\t\t\t\"quux.conf\": ` users = [{user = \"quux\"}]`,\n\t\t\t},\n\t\t\t\"sha256:e72d70c91b64b0f880f86decb95ec2600cbdcf8bdcd2355fce5ebc54a84a77e9\",\n\t\t},\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tsdir := t.TempDir()\n\t\t\tf, err := os.CreateTemp(sdir, \"nats.conf-\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif err := os.WriteFile(f.Name(), []byte(test.input), 066); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif test.includes != nil {\n\t\t\t\tfor includeFile, contents := range test.includes {\n\t\t\t\t\tinf, err := os.Create(filepath.Join(sdir, includeFile))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t\tif err := os.WriteFile(inf.Name(), []byte(contents), 066); err != nil {\n\t\t\t\t\t\tt.Error(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t_, digest, err := ParseFileWithChecksDigest(f.Name())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif digest != test.digest {\n\t\t\t\tt.Errorf(\"\\ngot: %s\\nexpected: %s\", digest, test.digest)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "doc/README.md", "content": "# Architecture Decision Records\n\nThe NATS ADR documents have moved to their [own repository](https://github.com/nats-io/nats-architecture-and-design/)\n" }, { "path": "docker/Dockerfile.nightly", "content": "FROM --platform=$BUILDPLATFORM golang:alpine AS builder\n\nARG VERSION=\"nightly\"\nARG GIT_COMMIT\n\nARG TARGETOS\nARG TARGETARCH\n\nENV GOOS=$TARGETOS \\\n GOARCH=$TARGETARCH \\\n GO111MODULE=on \\\n CGO_ENABLED=0\n\nRUN apk add --no-cache ca-certificates\nRUN update-ca-certificates\n\nWORKDIR /src\nRUN mkdir -p /src/out/$GOOS/$GOARCH\n\nCOPY ./nats-server/ /src/nats-server/\nCOPY ./nsc/ /src/nsc/\nCOPY ./natscli/ /src/natscli/\n\nRUN cd /src/nats-server && go build -trimpath \\\n -ldflags \"-w -X server.serverVersion=${VERSION},server.gitCommit=${GIT_COMMIT}\" \\\n -o /src/out/$TARGETOS/$TARGETARCH/nats-server .\n\nRUN cd /src/natscli && go build -trimpath \\\n -ldflags \"-w -X main.version=${VERSION}\" \\\n -o /src/out/$TARGETOS/$TARGETARCH/nats ./nats\n\nRUN cd /src/nsc && go build -trimpath \\\n -o /src/out/$TARGETOS/$TARGETARCH/nsc .\n\nFROM --platform=$TARGETPLATFORM alpine:latest\n\nARG TARGETOS\nARG TARGETARCH\n\nCOPY ./nats-server/docker/nats-server.conf /nats/conf/nats-server.conf\nCOPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/\nCOPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nats-server /bin/nats-server\nCOPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nats /bin/nats\nCOPY --from=builder /src/out/$TARGETOS/$TARGETARCH/nsc /bin/nsc\n\nEXPOSE 4222 8222 6222 5222\n\nENTRYPOINT [\"/bin/nats-server\"]\nCMD [\"-c\", \"/nats/conf/nats-server.conf\"]\n" }, { "path": "go.mod", "content": "module github.com/nats-io/nats-server/v2\n\ngo 1.25.0\n\ntoolchain go1.25.8\n\nrequire (\n\tgithub.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op\n\tgithub.com/google/go-tpm v0.9.8\n\tgithub.com/klauspost/compress v1.18.4\n\tgithub.com/nats-io/jwt/v2 v2.8.1\n\tgithub.com/nats-io/nats.go v1.49.0\n\tgithub.com/nats-io/nkeys v0.4.15\n\tgithub.com/nats-io/nuid v1.0.1\n\tgolang.org/x/crypto v0.49.0\n\tgolang.org/x/sys v0.42.0\n\tgolang.org/x/time v0.15.0\n)\n\n// We don't usually pin non-tagged commits but so far no release has\n// been made that includes https://github.com/minio/highwayhash/pull/29.\n// This will be updated if a new tag covers this in the future.\nrequire github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76\n" }, { "path": "go.sum", "content": "github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=\ngithub.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=\ngithub.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=\ngithub.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=\ngithub.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=\ngithub.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=\ngithub.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=\ngithub.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=\ngithub.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=\ngithub.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=\ngithub.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=\ngithub.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=\ngithub.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=\ngithub.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=\ngithub.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=\ngithub.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=\ngolang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=\ngolang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=\ngolang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=\ngolang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=\ngolang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=\ngolang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=\n" }, { "path": "internal/antithesis/noop.go", "content": "// Copyright 2022-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// This file is used iff the `enable_antithesis_sdk` build tag is not present\n//go:build !enable_antithesis_sdk\n\npackage antithesis\n\nimport (\n\t\"testing\"\n)\n\n// AssertUnreachable this implementation is a NOOP\nfunc AssertUnreachable(_ testing.TB, _ string, _ map[string]any) {}\n\n// Assert this implementation is a NOOP\nfunc Assert(_ testing.TB, _ bool, _ string, _ map[string]any) {}\n" }, { "path": "internal/antithesis/test_assert.go", "content": "// Copyright 2022-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// This file is used iff the `enable_antithesis_sdk` build tag is present\n//go:build enable_antithesis_sdk\n\npackage antithesis\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"runtime/debug\"\n\t\"testing\"\n\n\t\"github.com/antithesishq/antithesis-sdk-go/assert\"\n)\n\n// This file provides assertions utility functions suitable for use in tests.\n// It is a thin wrapper around the Antithesis SDK.\n//\n// Notice that unlike other assertions libraries, a violation does not halt execution or fail the test.\n// The effects of violating an assertion, are:\n// 1. Print the violation message, prefixed by the test name in which it happened\n// 2. Print the stack for the calling goroutine so each failed test has a clear trace of the failure\n// 3. Invoke the underlying Antithesis assertion\n//\n// N.B. Enabling this module outside for tests running outside of Antithesis will enable 1 and 2 above, but not 3.\n// therefore it can be useful to output additional test failure details when running tests locally or in CI.\n\n// AssertUnreachable is used to flag code branches that should not get invoked ever.\n// Example:\n//\n// pubAck, err := js.Publish(...)\n//\n//\tif err != nil {\n//\t antithesis.AssertUnreachable(t, \"Publish failed\", map[string]any{\"error\": err.Error()})\n//\t t.Fatalf(\"Publish failed with error: %s\", err)\n//\t}\nfunc AssertUnreachable(t testing.TB, message string, details map[string]any) {\n\t// Always print a message\n\tfmt.Printf(\"{*} [%s] Assert Unreachable violation: %s\\n\", t.Name(), message)\n\tif details != nil && len(details) > 0 {\n\t\tfmt.Printf(\"{*} Details:\\n\")\n\t\tjsonDetails, err := json.MarshalIndent(details, \"\", \" \")\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tfmt.Println(string(jsonDetails))\n\t}\n\n\t// Always print the stack trace\n\tfmt.Printf(\"{*} Stack trace:\\n\")\n\tdebug.PrintStack()\n\n\t// N.B. as of today, message (as-is) is the unique identifier of the event\n\t// Therefore this will de-duplicate the same assertion failing in 2 different tests\n\t// But not the same assertion failing at 2 different lines of the same test\n\tmessageWithTestName := fmt.Sprintf(\"[%s] %s\", t.Name(), message)\n\n\t// Fire assertion violation event (if Antithesis is enabled)\n\tassert.Unreachable(messageWithTestName, details)\n}\n\n// Assert is used to check that some given condition is always true,\n// Example:\n//\n//\tantithesis.Assert(t, sequence > lastSequence, \"Non-monotonic stream sequence number\", map[string]any{\n//\t \"stream\": streamName,\n//\t \"connection_id\": nc.Id(),\n//\t \"sequence\": sequence,\n//\t \"lastSequence\": lastSequence,\n//\t})\nfunc Assert(t testing.TB, condition bool, message string, details map[string]any) {\n\t// Condition is true, nothing to do\n\tif condition {\n\t\treturn\n\t}\n\n\t// Always print a message\n\tfmt.Printf(\"{*} [%s] Assert violation: %s\\n\", t.Name(), message)\n\tif details != nil && len(details) > 0 {\n\t\tfmt.Printf(\"{*} Details:\\n\")\n\t\tjsonDetails, err := json.MarshalIndent(details, \"\", \" \")\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tfmt.Println(string(jsonDetails))\n\t}\n\n\t// Always print the stack trace\n\tfmt.Printf(\"{*} Stack trace:\\n\")\n\tdebug.PrintStack()\n\n\t// N.B. as of today, message (as-is) is the unique identifier of the event\n\t// Therefore this will de-duplicate the same assertion failing in 2 different tests\n\t// But not the same assertion failing at 2 different lines of the same test\n\tmessageWithTestName := fmt.Sprintf(\"[%s] %s\", t.Name(), message)\n\n\t// Fire assertion violation event (if Antithesis is enabled)\n\tassert.AlwaysOrUnreachable(false, messageWithTestName, details)\n}\n" }, { "path": "internal/fastrand/LICENSE", "content": "Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." }, { "path": "internal/fastrand/fastrand.go", "content": "// Copyright 2020-2023 The LevelDB-Go, Pebble and NATS Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style license that can be found in\n// the LICENSE file.\n\npackage fastrand\n\nimport _ \"unsafe\" // required by go:linkname\n\n// Uint32 returns a lock free uint32 value.\n//\n//go:linkname Uint32 runtime.fastrand\nfunc Uint32() uint32\n\n// Uint32n returns a lock free uint32 value in the interval [0, n).\n//\n//go:linkname Uint32n runtime.fastrandn\nfunc Uint32n(n uint32) uint32\n\n// Uint64 returns a lock free uint64 value.\nfunc Uint64() uint64 {\n\tv := uint64(Uint32())\n\treturn v<<32 | uint64(Uint32())\n}\n" }, { "path": "internal/fastrand/fastrand_test.go", "content": "// Copyright 2020-23 The LevelDB-Go, Pebble and NATS Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style license that can be found in\n// the LICENSE file.\n\npackage fastrand\n\nimport (\n\t\"math/rand\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype defaultRand struct {\n\tmu sync.Mutex\n\tsrc rand.Source64\n}\n\nfunc newDefaultRand() *defaultRand {\n\tr := &defaultRand{\n\t\tsrc: rand.New(rand.NewSource(time.Now().UnixNano())),\n\t}\n\treturn r\n}\n\nfunc (r *defaultRand) Uint32() uint32 {\n\tr.mu.Lock()\n\ti := uint32(r.src.Uint64())\n\tr.mu.Unlock()\n\treturn i\n}\n\nfunc (r *defaultRand) Uint64() uint64 {\n\tr.mu.Lock()\n\ti := uint64(r.src.Uint64())\n\tr.mu.Unlock()\n\treturn i\n}\n\nfunc BenchmarkFastRand32(b *testing.B) {\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tUint32()\n\t\t}\n\t})\n}\n\nfunc BenchmarkFastRand64(b *testing.B) {\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tUint64()\n\t\t}\n\t})\n}\n\nfunc BenchmarkDefaultRand32(b *testing.B) {\n\tr := newDefaultRand()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tr.Uint32()\n\t\t}\n\t})\n}\n\nfunc BenchmarkDefaultRand64(b *testing.B) {\n\tr := newDefaultRand()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tr.Uint64()\n\t\t}\n\t})\n}\n" }, { "path": "internal/ldap/dn.go", "content": "// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com)\n// Portions copyright (c) 2015-2016 go-ldap Authors\npackage ldap\n\nimport (\n\t\"bytes\"\n\t\"crypto/x509/pkix\"\n\t\"encoding/asn1\"\n\tenchex \"encoding/hex\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n)\n\nvar attributeTypeNames = map[string]string{\n\t\"2.5.4.3\": \"CN\",\n\t\"2.5.4.5\": \"SERIALNUMBER\",\n\t\"2.5.4.6\": \"C\",\n\t\"2.5.4.7\": \"L\",\n\t\"2.5.4.8\": \"ST\",\n\t\"2.5.4.9\": \"STREET\",\n\t\"2.5.4.10\": \"O\",\n\t\"2.5.4.11\": \"OU\",\n\t\"2.5.4.17\": \"POSTALCODE\",\n\t// FIXME: Add others.\n\t\"0.9.2342.19200300.100.1.25\": \"DC\",\n}\n\n// AttributeTypeAndValue represents an attributeTypeAndValue from https://tools.ietf.org/html/rfc4514\ntype AttributeTypeAndValue struct {\n\t// Type is the attribute type\n\tType string\n\t// Value is the attribute value\n\tValue string\n}\n\n// RelativeDN represents a relativeDistinguishedName from https://tools.ietf.org/html/rfc4514\ntype RelativeDN struct {\n\tAttributes []*AttributeTypeAndValue\n}\n\n// DN represents a distinguishedName from https://tools.ietf.org/html/rfc4514\ntype DN struct {\n\tRDNs []*RelativeDN\n}\n\n// FromCertSubject takes a pkix.Name from a cert and returns a DN\n// that uses the same set. Does not support multi value RDNs.\nfunc FromCertSubject(subject pkix.Name) (*DN, error) {\n\tdn := &DN{\n\t\tRDNs: make([]*RelativeDN, 0),\n\t}\n\tfor i := len(subject.Names) - 1; i >= 0; i-- {\n\t\tname := subject.Names[i]\n\t\toidString := name.Type.String()\n\t\ttypeName, ok := attributeTypeNames[oidString]\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"invalid type name: %+v\", name)\n\t\t}\n\t\tv, ok := name.Value.(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"invalid type value: %+v\", v)\n\t\t}\n\t\trdn := &RelativeDN{\n\t\t\tAttributes: []*AttributeTypeAndValue{\n\t\t\t\t{\n\t\t\t\t\tType: typeName,\n\t\t\t\t\tValue: v,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tdn.RDNs = append(dn.RDNs, rdn)\n\t}\n\treturn dn, nil\n}\n\n// FromRawCertSubject takes a raw subject from a certificate\n// and uses asn1.Unmarshal to get the individual RDNs in the\n// original order, including multi-value RDNs.\nfunc FromRawCertSubject(rawSubject []byte) (*DN, error) {\n\tdn := &DN{\n\t\tRDNs: make([]*RelativeDN, 0),\n\t}\n\tvar rdns pkix.RDNSequence\n\t_, err := asn1.Unmarshal(rawSubject, &rdns)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor i := len(rdns) - 1; i >= 0; i-- {\n\t\trdn := rdns[i]\n\t\tif len(rdn) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tr := &RelativeDN{}\n\t\tattrs := make([]*AttributeTypeAndValue, 0)\n\t\tfor j := len(rdn) - 1; j >= 0; j-- {\n\t\t\tatv := rdn[j]\n\n\t\t\ttypeName := \"\"\n\t\t\tname := atv.Type.String()\n\t\t\ttypeName, ok := attributeTypeNames[name]\n\t\t\tif !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"invalid type name: %+v\", name)\n\t\t\t}\n\t\t\tvalue, ok := atv.Value.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"invalid type value: %+v\", atv.Value)\n\t\t\t}\n\t\t\tattr := &AttributeTypeAndValue{\n\t\t\t\tType: typeName,\n\t\t\t\tValue: value,\n\t\t\t}\n\t\t\tattrs = append(attrs, attr)\n\t\t}\n\t\tr.Attributes = attrs\n\t\tdn.RDNs = append(dn.RDNs, r)\n\t}\n\n\treturn dn, nil\n}\n\n// ParseDN returns a distinguishedName or an error.\n// The function respects https://tools.ietf.org/html/rfc4514\nfunc ParseDN(str string) (*DN, error) {\n\tdn := new(DN)\n\tdn.RDNs = make([]*RelativeDN, 0)\n\trdn := new(RelativeDN)\n\trdn.Attributes = make([]*AttributeTypeAndValue, 0)\n\tbuffer := bytes.Buffer{}\n\tattribute := new(AttributeTypeAndValue)\n\tescaping := false\n\n\tunescapedTrailingSpaces := 0\n\tstringFromBuffer := func() string {\n\t\ts := buffer.String()\n\t\ts = s[0 : len(s)-unescapedTrailingSpaces]\n\t\tbuffer.Reset()\n\t\tunescapedTrailingSpaces = 0\n\t\treturn s\n\t}\n\n\tfor i := 0; i < len(str); i++ {\n\t\tchar := str[i]\n\t\tswitch {\n\t\tcase escaping:\n\t\t\tunescapedTrailingSpaces = 0\n\t\t\tescaping = false\n\t\t\tswitch char {\n\t\t\tcase ' ', '\"', '#', '+', ',', ';', '<', '=', '>', '\\\\':\n\t\t\t\tbuffer.WriteByte(char)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Not a special character, assume hex encoded octet\n\t\t\tif len(str) == i+1 {\n\t\t\t\treturn nil, errors.New(\"got corrupted escaped character\")\n\t\t\t}\n\n\t\t\tdst := []byte{0}\n\t\t\tn, err := enchex.Decode([]byte(dst), []byte(str[i:i+2]))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to decode escaped character: %s\", err)\n\t\t\t} else if n != 1 {\n\t\t\t\treturn nil, fmt.Errorf(\"expected 1 byte when un-escaping, got %d\", n)\n\t\t\t}\n\t\t\tbuffer.WriteByte(dst[0])\n\t\t\ti++\n\t\tcase char == '\\\\':\n\t\t\tunescapedTrailingSpaces = 0\n\t\t\tescaping = true\n\t\tcase char == '=':\n\t\t\tattribute.Type = stringFromBuffer()\n\t\t\t// Special case: If the first character in the value is # the following data\n\t\t\t// is BER encoded. Throw an error since not supported right now.\n\t\t\tif len(str) > i+1 && str[i+1] == '#' {\n\t\t\t\treturn nil, errors.New(\"unsupported BER encoding\")\n\t\t\t}\n\t\tcase char == ',' || char == '+':\n\t\t\t// We're done with this RDN or value, push it\n\t\t\tif len(attribute.Type) == 0 {\n\t\t\t\treturn nil, errors.New(\"incomplete type, value pair\")\n\t\t\t}\n\t\t\tattribute.Value = stringFromBuffer()\n\t\t\trdn.Attributes = append(rdn.Attributes, attribute)\n\t\t\tattribute = new(AttributeTypeAndValue)\n\t\t\tif char == ',' {\n\t\t\t\tdn.RDNs = append(dn.RDNs, rdn)\n\t\t\t\trdn = new(RelativeDN)\n\t\t\t\trdn.Attributes = make([]*AttributeTypeAndValue, 0)\n\t\t\t}\n\t\tcase char == ' ' && buffer.Len() == 0:\n\t\t\t// ignore unescaped leading spaces\n\t\t\tcontinue\n\t\tdefault:\n\t\t\tif char == ' ' {\n\t\t\t\t// Track unescaped spaces in case they are trailing and we need to remove them\n\t\t\t\tunescapedTrailingSpaces++\n\t\t\t} else {\n\t\t\t\t// Reset if we see a non-space char\n\t\t\t\tunescapedTrailingSpaces = 0\n\t\t\t}\n\t\t\tbuffer.WriteByte(char)\n\t\t}\n\t}\n\tif buffer.Len() > 0 {\n\t\tif len(attribute.Type) == 0 {\n\t\t\treturn nil, errors.New(\"DN ended with incomplete type, value pair\")\n\t\t}\n\t\tattribute.Value = stringFromBuffer()\n\t\trdn.Attributes = append(rdn.Attributes, attribute)\n\t\tdn.RDNs = append(dn.RDNs, rdn)\n\t}\n\treturn dn, nil\n}\n\n// Equal returns true if the DNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch).\n// Returns true if they have the same number of relative distinguished names\n// and corresponding relative distinguished names (by position) are the same.\nfunc (d *DN) Equal(other *DN) bool {\n\tif len(d.RDNs) != len(other.RDNs) {\n\t\treturn false\n\t}\n\tfor i := range d.RDNs {\n\t\tif !d.RDNs[i].Equal(other.RDNs[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// RDNsMatch returns true if the individual RDNs of the DNs\n// are the same regardless of ordering.\nfunc (d *DN) RDNsMatch(other *DN) bool {\n\tif len(d.RDNs) != len(other.RDNs) {\n\t\treturn false\n\t}\n\nCheckNextRDN:\n\tfor _, irdn := range d.RDNs {\n\t\tfor _, ordn := range other.RDNs {\n\t\t\tif (len(irdn.Attributes) == len(ordn.Attributes)) &&\n\t\t\t\t(irdn.hasAllAttributes(ordn.Attributes) && ordn.hasAllAttributes(irdn.Attributes)) {\n\t\t\t\t// Found the RDN, check if next one matches.\n\t\t\t\tcontinue CheckNextRDN\n\t\t\t}\n\t\t}\n\n\t\t// Could not find a matching individual RDN, auth fails.\n\t\treturn false\n\t}\n\treturn true\n}\n\n// AncestorOf returns true if the other DN consists of at least one RDN followed by all the RDNs of the current DN.\n// \"ou=widgets,o=acme.com\" is an ancestor of \"ou=sprockets,ou=widgets,o=acme.com\"\n// \"ou=widgets,o=acme.com\" is not an ancestor of \"ou=sprockets,ou=widgets,o=foo.com\"\n// \"ou=widgets,o=acme.com\" is not an ancestor of \"ou=widgets,o=acme.com\"\nfunc (d *DN) AncestorOf(other *DN) bool {\n\tif len(d.RDNs) >= len(other.RDNs) {\n\t\treturn false\n\t}\n\t// Take the last `len(d.RDNs)` RDNs from the other DN to compare against\n\totherRDNs := other.RDNs[len(other.RDNs)-len(d.RDNs):]\n\tfor i := range d.RDNs {\n\t\tif !d.RDNs[i].Equal(otherRDNs[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Equal returns true if the RelativeDNs are equal as defined by rfc4517 4.2.15 (distinguishedNameMatch).\n// Relative distinguished names are the same if and only if they have the same number of AttributeTypeAndValues\n// and each attribute of the first RDN is the same as the attribute of the second RDN with the same attribute type.\n// The order of attributes is not significant.\n// Case of attribute types is not significant.\nfunc (r *RelativeDN) Equal(other *RelativeDN) bool {\n\tif len(r.Attributes) != len(other.Attributes) {\n\t\treturn false\n\t}\n\treturn r.hasAllAttributes(other.Attributes) && other.hasAllAttributes(r.Attributes)\n}\n\nfunc (r *RelativeDN) hasAllAttributes(attrs []*AttributeTypeAndValue) bool {\n\tfor _, attr := range attrs {\n\t\tfound := false\n\t\tfor _, myattr := range r.Attributes {\n\t\t\tif myattr.Equal(attr) {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Equal returns true if the AttributeTypeAndValue is equivalent to the specified AttributeTypeAndValue\n// Case of the attribute type is not significant\nfunc (a *AttributeTypeAndValue) Equal(other *AttributeTypeAndValue) bool {\n\treturn strings.EqualFold(a.Type, other.Type) && a.Value == other.Value\n}\n" }, { "path": "internal/ldap/dn_test.go", "content": "// Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com)\n// Portions copyright (c) 2015-2016 go-ldap Authors\n// Static-Check Fixes Copyright 2024 The NATS Authors\n\npackage ldap\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestSuccessfulDNParsing(t *testing.T) {\n\ttestcases := map[string]DN{\n\t\t\"\": {[]*RelativeDN{}},\n\t\t\"cn=Jim\\\\2C \\\\22Hasse Hö\\\\22 Hansson!,dc=dummy,dc=com\": {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\"cn\", \"Jim, \\\"Hasse Hö\\\" Hansson!\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"dc\", \"dummy\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"dc\", \"com\"}}}}},\n\t\t\"UID=jsmith,DC=example,DC=net\": {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\"UID\", \"jsmith\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"DC\", \"example\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"DC\", \"net\"}}}}},\n\t\t\"OU=Sales+CN=J. Smith,DC=example,DC=net\": {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{\n\t\t\t\t{\"OU\", \"Sales\"},\n\t\t\t\t{\"CN\", \"J. Smith\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"DC\", \"example\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"DC\", \"net\"}}}}},\n\t\t//\n\t\t// \"1.3.6.1.4.1.1466.0=#04024869\": {[]*RelativeDN{\n\t\t// \t{[]*AttributeTypeAndValue{{\"1.3.6.1.4.1.1466.0\", \"Hi\"}}}}},\n\t\t// \"1.3.6.1.4.1.1466.0=#04024869,DC=net\": {[]*RelativeDN{\n\t\t// \t{[]*AttributeTypeAndValue{{\"1.3.6.1.4.1.1466.0\", \"Hi\"}}},\n\t\t// \t{[]*AttributeTypeAndValue{{\"DC\", \"net\"}}}}},\n\t\t\"CN=Lu\\\\C4\\\\8Di\\\\C4\\\\87\": {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\"CN\", \"Lučić\"}}}}},\n\t\t\" CN = Lu\\\\C4\\\\8Di\\\\C4\\\\87 \": {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\"CN\", \"Lučić\"}}}}},\n\t\t` A = 1 , B = 2 `: {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\"A\", \"1\"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\"B\", \"2\"}}}}},\n\t\t` A = 1 + B = 2 `: {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{\n\t\t\t\t{\"A\", \"1\"},\n\t\t\t\t{\"B\", \"2\"}}}}},\n\t\t` \\ \\ A\\ \\ = \\ \\ 1\\ \\ , \\ \\ B\\ \\ = \\ \\ 2\\ \\ `: {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{{\" A \", \" 1 \"}}},\n\t\t\t{[]*AttributeTypeAndValue{{\" B \", \" 2 \"}}}}},\n\t\t` \\ \\ A\\ \\ = \\ \\ 1\\ \\ + \\ \\ B\\ \\ = \\ \\ 2\\ \\ `: {[]*RelativeDN{\n\t\t\t{[]*AttributeTypeAndValue{\n\t\t\t\t{\" A \", \" 1 \"},\n\t\t\t\t{\" B \", \" 2 \"}}}}},\n\t}\n\n\tfor test, answer := range testcases {\n\t\tdn, err := ParseDN(test)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t\tcontinue\n\t\t}\n\t\tif !reflect.DeepEqual(dn, &answer) {\n\t\t\tt.Errorf(\"Parsed DN %s is not equal to the expected structure\", test)\n\t\t\tt.Logf(\"Expected:\")\n\t\t\tfor _, rdn := range answer.RDNs {\n\t\t\t\tfor _, attribs := range rdn.Attributes {\n\t\t\t\t\tt.Logf(\"#%v\\n\", attribs)\n\t\t\t\t}\n\t\t\t}\n\t\t\tt.Logf(\"Actual:\")\n\t\t\tfor _, rdn := range dn.RDNs {\n\t\t\t\tfor _, attribs := range rdn.Attributes {\n\t\t\t\t\tt.Logf(\"#%v\\n\", attribs)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestErrorDNParsing(t *testing.T) {\n\ttestcases := map[string]string{\n\t\t\"*\": \"DN ended with incomplete type, value pair\",\n\t\t\"cn=Jim\\\\0Test\": \"failed to decode escaped character: encoding/hex: invalid byte: U+0054 'T'\",\n\t\t\"cn=Jim\\\\0\": \"got corrupted escaped character\",\n\t\t\"DC=example,=net\": \"DN ended with incomplete type, value pair\",\n\t\t// \"1=#0402486\": \"failed to decode BER encoding: encoding/hex: odd length hex string\",\n\t\t\"test,DC=example,DC=com\": \"incomplete type, value pair\",\n\t\t\"=test,DC=example,DC=com\": \"incomplete type, value pair\",\n\t}\n\n\tfor test, answer := range testcases {\n\t\t_, err := ParseDN(test)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected %s to fail parsing but succeeded\\n\", test)\n\t\t} else if err.Error() != answer {\n\t\t\tt.Errorf(\"Unexpected error on %s:\\n%s\\nvs.\\n%s\\n\", test, answer, err.Error())\n\t\t}\n\t}\n}\n\nfunc TestDNEqual(t *testing.T) {\n\ttestcases := []struct {\n\t\tA string\n\t\tB string\n\t\tEqual bool\n\t}{\n\t\t// Exact match\n\t\t{\"\", \"\", true},\n\t\t{\"o=A\", \"o=A\", true},\n\t\t{\"o=A\", \"o=B\", false},\n\n\t\t{\"o=A,o=B\", \"o=A,o=B\", true},\n\t\t{\"o=A,o=B\", \"o=A,o=C\", false},\n\n\t\t{\"o=A+o=B\", \"o=A+o=B\", true},\n\t\t{\"o=A+o=B\", \"o=A+o=C\", false},\n\n\t\t// Case mismatch in type is ignored\n\t\t{\"o=A\", \"O=A\", true},\n\t\t{\"o=A,o=B\", \"o=A,O=B\", true},\n\t\t{\"o=A+o=B\", \"o=A+O=B\", true},\n\n\t\t// Case mismatch in value is significant\n\t\t{\"o=a\", \"O=A\", false},\n\t\t{\"o=a,o=B\", \"o=A,O=B\", false},\n\t\t{\"o=a+o=B\", \"o=A+O=B\", false},\n\n\t\t// Multi-valued RDN order mismatch is ignored\n\t\t{\"o=A+o=B\", \"O=B+o=A\", true},\n\t\t// Number of RDN attributes is significant\n\t\t{\"o=A+o=B\", \"O=B+o=A+O=B\", false},\n\n\t\t// Missing values are significant\n\t\t{\"o=A+o=B\", \"O=B+o=A+O=C\", false}, // missing values matter\n\t\t{\"o=A+o=B+o=C\", \"O=B+o=A\", false}, // missing values matter\n\n\t\t// Whitespace tests\n\t\t// Matching\n\t\t{\n\t\t\t\"cn=John Doe, ou=People, dc=sun.com\",\n\t\t\t\"cn=John Doe, ou=People, dc=sun.com\",\n\t\t\ttrue,\n\t\t},\n\t\t// Difference in leading/trailing chars is ignored\n\t\t{\n\t\t\t\"cn=John Doe, ou=People, dc=sun.com\",\n\t\t\t\"cn=John Doe,ou=People,dc=sun.com\",\n\t\t\ttrue,\n\t\t},\n\t\t// Difference in values is significant\n\t\t{\n\t\t\t\"cn=John Doe, ou=People, dc=sun.com\",\n\t\t\t\"cn=John Doe, ou=People, dc=sun.com\",\n\t\t\tfalse,\n\t\t},\n\t}\n\n\tfor i, tc := range testcases {\n\t\ta, err := ParseDN(tc.A)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%d: %v\", i, err)\n\t\t\tcontinue\n\t\t}\n\t\tb, err := ParseDN(tc.B)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%d: %v\", i, err)\n\t\t\tcontinue\n\t\t}\n\t\tif expected, actual := tc.Equal, a.Equal(b); expected != actual {\n\t\t\tt.Errorf(\"%d: when comparing '%s' and '%s' expected %v, got %v\", i, tc.A, tc.B, expected, actual)\n\t\t\tcontinue\n\t\t}\n\t\tif expected, actual := tc.Equal, b.Equal(a); expected != actual {\n\t\t\tt.Errorf(\"%d: when comparing '%s' and '%s' expected %v, got %v\", i, tc.A, tc.B, expected, actual)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestDNAncestor(t *testing.T) {\n\ttestcases := []struct {\n\t\tA string\n\t\tB string\n\t\tAncestor bool\n\t}{\n\t\t// Exact match returns false\n\t\t{\"\", \"\", false},\n\t\t{\"o=A\", \"o=A\", false},\n\t\t{\"o=A,o=B\", \"o=A,o=B\", false},\n\t\t{\"o=A+o=B\", \"o=A+o=B\", false},\n\n\t\t// Mismatch\n\t\t{\"ou=C,ou=B,o=A\", \"ou=E,ou=D,ou=B,o=A\", false},\n\n\t\t// Descendant\n\t\t{\"ou=C,ou=B,o=A\", \"ou=E,ou=C,ou=B,o=A\", true},\n\t}\n\n\tfor i, tc := range testcases {\n\t\ta, err := ParseDN(tc.A)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%d: %v\", i, err)\n\t\t\tcontinue\n\t\t}\n\t\tb, err := ParseDN(tc.B)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%d: %v\", i, err)\n\t\t\tcontinue\n\t\t}\n\t\tif expected, actual := tc.Ancestor, a.AncestorOf(b); expected != actual {\n\t\t\tt.Errorf(\"%d: when comparing '%s' and '%s' expected %v, got %v\", i, tc.A, tc.B, expected, actual)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n" }, { "path": "internal/ocsp/ocsp.go", "content": "// Copyright 2019-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage testhelper\n\nimport (\n\t\"crypto\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/base64\"\n\t\"encoding/pem\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"golang.org/x/crypto/ocsp\"\n)\n\nconst (\n\tdefaultResponseTTL = 4 * time.Second\n\tdefaultAddress = \"127.0.0.1:8888\"\n)\n\nfunc NewOCSPResponderCustomAddress(t *testing.T, issuerCertPEM, issuerKeyPEM string, addr string) *http.Server {\n\tt.Helper()\n\treturn NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, addr, defaultResponseTTL, \"\")\n}\n\nfunc NewOCSPResponder(t *testing.T, issuerCertPEM, issuerKeyPEM string) *http.Server {\n\tt.Helper()\n\treturn NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, \"\")\n}\n\nfunc NewOCSPResponderDesignatedCustomAddress(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, addr string) *http.Server {\n\tt.Helper()\n\treturn NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, addr, defaultResponseTTL, \"\")\n}\n\nfunc NewOCSPResponderPreferringHTTPMethod(t *testing.T, issuerCertPEM, issuerKeyPEM, method string) *http.Server {\n\tt.Helper()\n\treturn NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, defaultResponseTTL, method)\n}\n\nfunc NewOCSPResponderCustomTimeout(t *testing.T, issuerCertPEM, issuerKeyPEM string, responseTTL time.Duration) *http.Server {\n\tt.Helper()\n\treturn NewOCSPResponderBase(t, issuerCertPEM, issuerCertPEM, issuerKeyPEM, false, defaultAddress, responseTTL, \"\")\n}\n\nfunc NewOCSPResponderBase(t *testing.T, issuerCertPEM, respCertPEM, respKeyPEM string, embed bool, addr string, responseTTL time.Duration, method string) *http.Server {\n\tt.Helper()\n\tvar mu sync.Mutex\n\tstatus := make(map[string]int)\n\n\tissuerCert := parseCertPEM(t, issuerCertPEM)\n\trespCert := parseCertPEM(t, respCertPEM)\n\trespKey := parseKeyPEM(t, respKeyPEM)\n\n\tmux := http.NewServeMux()\n\t// The \"/statuses/\" endpoint is for directly setting a key-value pair in\n\t// the CA's status database.\n\tmux.HandleFunc(\"/statuses/\", func(rw http.ResponseWriter, r *http.Request) {\n\t\tdefer r.Body.Close()\n\n\t\tkey := r.URL.Path[len(\"/statuses/\"):]\n\t\tswitch r.Method {\n\t\tcase \"GET\":\n\t\t\tmu.Lock()\n\t\t\tn, ok := status[key]\n\t\t\tif !ok {\n\t\t\t\tn = ocsp.Unknown\n\t\t\t}\n\t\t\tmu.Unlock()\n\n\t\t\tfmt.Fprintf(rw, \"%s %d\", key, n)\n\t\tcase \"POST\":\n\t\t\tdata, err := io.ReadAll(r.Body)\n\t\t\tif err != nil {\n\t\t\t\thttp.Error(rw, err.Error(), http.StatusBadRequest)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tn, err := strconv.Atoi(string(data))\n\t\t\tif err != nil {\n\t\t\t\thttp.Error(rw, err.Error(), http.StatusBadRequest)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tmu.Lock()\n\t\t\tstatus[key] = n\n\t\t\tmu.Unlock()\n\n\t\t\tfmt.Fprintf(rw, \"%s %d\", key, n)\n\t\tdefault:\n\t\t\thttp.Error(rw, \"Method Not Allowed\", http.StatusMethodNotAllowed)\n\t\t\treturn\n\t\t}\n\t})\n\t// The \"/\" endpoint is for normal OCSP requests. This actually parses an\n\t// OCSP status request and signs a response with a CA. Lightly based off:\n\t// https://www.ietf.org/rfc/rfc2560.txt\n\tmux.HandleFunc(\"/\", func(rw http.ResponseWriter, r *http.Request) {\n\t\tvar reqData []byte\n\t\tvar err error\n\n\t\tswitch {\n\t\tcase r.Method == \"GET\":\n\t\t\tif method != \"\" && r.Method != method {\n\t\t\t\thttp.Error(rw, \"\", http.StatusBadRequest)\n\t\t\t\treturn\n\t\t\t}\n\t\t\treqData, err = base64.StdEncoding.DecodeString(r.URL.Path[1:])\n\t\tcase r.Method == \"POST\":\n\t\t\tif method != \"\" && r.Method != method {\n\t\t\t\thttp.Error(rw, \"\", http.StatusBadRequest)\n\t\t\t\treturn\n\t\t\t}\n\t\t\treqData, err = io.ReadAll(r.Body)\n\t\t\tr.Body.Close()\n\t\tdefault:\n\t\t\thttp.Error(rw, \"Method Not Allowed\", http.StatusMethodNotAllowed)\n\t\t\treturn\n\t\t}\n\t\tif err != nil {\n\t\t\thttp.Error(rw, err.Error(), http.StatusBadRequest)\n\t\t\treturn\n\t\t}\n\n\t\tocspReq, err := ocsp.ParseRequest(reqData)\n\t\tif err != nil {\n\t\t\thttp.Error(rw, err.Error(), http.StatusBadRequest)\n\t\t\treturn\n\t\t}\n\n\t\tmu.Lock()\n\t\tn, ok := status[ocspReq.SerialNumber.String()]\n\t\tif !ok {\n\t\t\tn = ocsp.Unknown\n\t\t}\n\t\tmu.Unlock()\n\n\t\ttmpl := ocsp.Response{\n\t\t\tStatus: n,\n\t\t\tSerialNumber: ocspReq.SerialNumber,\n\t\t\tThisUpdate: time.Now(),\n\t\t}\n\t\tif responseTTL != 0 {\n\t\t\ttmpl.NextUpdate = tmpl.ThisUpdate.Add(responseTTL)\n\t\t}\n\t\tif embed {\n\t\t\ttmpl.Certificate = respCert\n\t\t}\n\t\trespData, err := ocsp.CreateResponse(issuerCert, respCert, tmpl, respKey)\n\t\tif err != nil {\n\t\t\thttp.Error(rw, err.Error(), http.StatusInternalServerError)\n\t\t\treturn\n\t\t}\n\n\t\trw.Header().Set(\"Content-Type\", \"application/ocsp-response\")\n\t\trw.Header().Set(\"Content-Length\", fmt.Sprint(len(respData)))\n\n\t\tfmt.Fprint(rw, string(respData))\n\t})\n\n\tsrv := &http.Server{\n\t\tAddr: addr,\n\t\tHandler: mux,\n\t\tReadTimeout: time.Second * 5,\n\t}\n\tgo srv.ListenAndServe()\n\ttime.Sleep(1 * time.Second)\n\treturn srv\n}\n\nfunc parseCertPEM(t *testing.T, certPEM string) *x509.Certificate {\n\tt.Helper()\n\tblock := parsePEM(t, certPEM)\n\n\tcert, err := x509.ParseCertificate(block.Bytes)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to parse cert '%s': %s\", certPEM, err)\n\t}\n\treturn cert\n}\n\nfunc parseKeyPEM(t *testing.T, keyPEM string) crypto.Signer {\n\tt.Helper()\n\tblock := parsePEM(t, keyPEM)\n\n\tkey, err := x509.ParsePKCS8PrivateKey(block.Bytes)\n\tif err != nil {\n\t\tkey, err = x509.ParsePKCS1PrivateKey(block.Bytes)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to parse ikey %s: %s\", keyPEM, err)\n\t\t}\n\t}\n\tkeyc := key.(crypto.Signer)\n\treturn keyc\n}\n\nfunc parsePEM(t *testing.T, pemPath string) *pem.Block {\n\tt.Helper()\n\tdata, err := os.ReadFile(pemPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tblock, _ := pem.Decode(data)\n\tif block == nil {\n\t\tt.Fatalf(\"failed to decode PEM %s\", pemPath)\n\t}\n\treturn block\n}\n\nfunc GetOCSPStatus(s tls.ConnectionState) (*ocsp.Response, error) {\n\tif len(s.VerifiedChains) == 0 {\n\t\treturn nil, fmt.Errorf(\"missing TLS verified chains\")\n\t}\n\tchain := s.VerifiedChains[0]\n\n\tif got, want := len(chain), 2; got < want {\n\t\treturn nil, fmt.Errorf(\"incomplete cert chain, got %d, want at least %d\", got, want)\n\t}\n\tleaf, issuer := chain[0], chain[1]\n\n\tresp, err := ocsp.ParseResponseForCert(s.OCSPResponse, leaf, issuer)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse OCSP response: %w\", err)\n\t}\n\tif err := resp.CheckSignatureFrom(issuer); err != nil {\n\t\treturn resp, err\n\t}\n\treturn resp, nil\n}\n\nfunc SetOCSPStatus(t *testing.T, ocspURL, certPEM string, status int) {\n\tt.Helper()\n\n\tcert := parseCertPEM(t, certPEM)\n\n\thc := &http.Client{Timeout: 10 * time.Second}\n\tresp, err := hc.Post(\n\t\tfmt.Sprintf(\"%s/statuses/%s\", ocspURL, cert.SerialNumber),\n\t\t\"\",\n\t\tstrings.NewReader(fmt.Sprint(status)),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer resp.Body.Close()\n\n\tdata, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to read OCSP HTTP response body: %s\", err)\n\t}\n\n\tif got, want := resp.Status, \"200 OK\"; got != want {\n\t\tt.Error(strings.TrimSpace(string(data)))\n\t\tt.Fatalf(\"unexpected OCSP HTTP set status, got %q, want %q\", got, want)\n\t}\n}\n" }, { "path": "internal/testhelper/logging.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage testhelper\n\n// These routines need to be accessible in both the server and test\n// directories, and tests importing a package don't get exported symbols from\n// _test.go files in the imported package, so we put them here where they can\n// be used freely.\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n)\n\ntype DummyLogger struct {\n\tsync.Mutex\n\tMsg string\n\tAllMsgs []string\n}\n\nfunc (l *DummyLogger) CheckContent(t *testing.T, expectedStr string) {\n\tt.Helper()\n\tl.Lock()\n\tdefer l.Unlock()\n\tif l.Msg != expectedStr {\n\t\tt.Fatalf(\"Expected log to be: %v, got %v\", expectedStr, l.Msg)\n\t}\n}\n\nfunc (l *DummyLogger) aggregate() {\n\tif l.AllMsgs != nil {\n\t\tl.AllMsgs = append(l.AllMsgs, l.Msg)\n\t}\n}\n\nfunc (l *DummyLogger) Noticef(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\nfunc (l *DummyLogger) Errorf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\nfunc (l *DummyLogger) Warnf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\nfunc (l *DummyLogger) Fatalf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\nfunc (l *DummyLogger) Debugf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\nfunc (l *DummyLogger) Tracef(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tl.Msg = fmt.Sprintf(format, v...)\n\tl.aggregate()\n}\n\n// NewDummyLogger creates a dummy logger and allows to ask for logs to be\n// retained instead of just keeping the most recent. Use retain to provide an\n// initial size estimate on messages (not to provide a max capacity).\nfunc NewDummyLogger(retain uint) *DummyLogger {\n\tl := &DummyLogger{}\n\tif retain > 0 {\n\t\tl.AllMsgs = make([]string, 0, retain)\n\t}\n\treturn l\n}\n\nfunc (l *DummyLogger) Drain() {\n\tl.Lock()\n\tdefer l.Unlock()\n\tif l.AllMsgs == nil {\n\t\treturn\n\t}\n\tl.AllMsgs = make([]string, 0, len(l.AllMsgs))\n}\n\nfunc (l *DummyLogger) CheckForProhibited(t *testing.T, reason, needle string) {\n\tt.Helper()\n\tl.Lock()\n\tdefer l.Unlock()\n\n\tif l.AllMsgs == nil {\n\t\tt.Fatal(\"DummyLogger.CheckForProhibited called without AllMsgs being collected\")\n\t}\n\n\t// Collect _all_ matches, rather than have to re-test repeatedly.\n\t// This will particularly help with less deterministic tests with multiple matches.\n\tshouldFail := false\n\tfor i := range l.AllMsgs {\n\t\tif strings.Contains(l.AllMsgs[i], needle) {\n\t\t\tt.Errorf(\"log contains %s: %v\", reason, l.AllMsgs[i])\n\t\t\tshouldFail = true\n\t\t}\n\t}\n\tif shouldFail {\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "locksordering.txt", "content": "Here is the list of some established lock ordering.\n\nIn this list, A -> B means that you can have A.Lock() then B.Lock(), not the opposite.\n\njetStream -> jsAccount -> Server -> client -> Account\n\njetStream -> jsAccount -> stream -> consumer\n\nA lock to protect jetstream account's usage has been introduced: jsAccount.usageMu.\nThis lock is independent and can be invoked under any other lock: jsAccount -> jsa.usageMu, stream -> jsa.usageMu, etc...\n\nA lock to protect the account's leafnodes list was also introduced to\nallow that lock to be held and the acquire a client lock which is not\npossible with the normal account lock.\n\naccountLeafList -> client\n\nAccountResolver interface has various implementations, but assume: AccountResolver -> Server\n\nA reloadMu lock was added to prevent newly connecting clients racing with the configuration reload.\nThis must be taken out as soon as a reload is about to happen before any other locks:\n\n reloadMu -> Server\n reloadMu -> optsMu\n\nThe \"jscmMu\" lock in the Account is used to serialise calls to checkJetStreamMigrate and\nclearObserverState so that they cannot interleave which would leave Raft nodes in\ninconsistent observer states.\n\n jscmMu -> Account -> jsAccount\n jscmMu -> stream.clsMu\n jscmMu -> RaftNode\n\nThe \"clsMu\" lock protects the consumer list on a stream, used for signalling consumer activity.\n\n stream -> clsMu\n\nThe \"clMu\", \"ddMu\" and \"batchMu\" locks protect clustered, dedupe and batch state respectively.\nThe stream lock (`mset.mu`) is optional, but if holding \"clMu\", \"ddMu\" or \"batchMu\",\nlocking the stream lock afterward would violate locking order.\n\n stream -> clMu\n stream -> batchMu -> clMu\n stream -> ddMu\n\nThe \"mset.batches.mu\" lock protects the batching state without needing to hold the stream lock.\nIf \"clMu\" is used to commit a batch, it should only be acquired while already holding the batch lock.\n\n mset.batches.mu -> clMu\n" }, { "path": "logger/log.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package logger provides logging facilities for the NATS server\npackage logger\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\n// Default file permissions for log files.\nconst defaultLogPerms = os.FileMode(0640)\n\n// Logger is the server logger\ntype Logger struct {\n\tsync.Mutex\n\tlogger *log.Logger\n\tdebug bool\n\ttrace bool\n\tinfoLabel string\n\twarnLabel string\n\terrorLabel string\n\tfatalLabel string\n\tdebugLabel string\n\ttraceLabel string\n\tfl *fileLogger\n}\n\ntype LogOption interface {\n\tisLoggerOption()\n}\n\n// LogUTC controls whether timestamps in the log output should be UTC or local time.\ntype LogUTC bool\n\nfunc (l LogUTC) isLoggerOption() {}\n\nfunc logFlags(time bool, opts ...LogOption) int {\n\tflags := 0\n\tif time {\n\t\tflags = log.LstdFlags | log.Lmicroseconds\n\t}\n\n\tfor _, opt := range opts {\n\t\tswitch v := opt.(type) {\n\t\tcase LogUTC:\n\t\t\tif time && bool(v) {\n\t\t\t\tflags |= log.LUTC\n\t\t\t}\n\t\t}\n\t}\n\n\treturn flags\n}\n\n// NewStdLogger creates a logger with output directed to Stderr\nfunc NewStdLogger(time, debug, trace, colors, pid bool, opts ...LogOption) *Logger {\n\tflags := logFlags(time, opts...)\n\n\tpre := \"\"\n\tif pid {\n\t\tpre = pidPrefix()\n\t}\n\n\tl := &Logger{\n\t\tlogger: log.New(os.Stderr, pre, flags),\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t}\n\n\tif colors {\n\t\tsetColoredLabelFormats(l)\n\t} else {\n\t\tsetPlainLabelFormats(l)\n\t}\n\n\treturn l\n}\n\n// NewFileLogger creates a logger with output directed to a file\nfunc NewFileLogger(filename string, time, debug, trace, pid bool, opts ...LogOption) *Logger {\n\tflags := logFlags(time, opts...)\n\n\tpre := \"\"\n\tif pid {\n\t\tpre = pidPrefix()\n\t}\n\n\tfl, err := newFileLogger(filename, pre, time)\n\tif err != nil {\n\t\tlog.Fatalf(\"error opening file: %v\", err)\n\t\treturn nil\n\t}\n\n\tl := &Logger{\n\t\tlogger: log.New(fl, pre, flags),\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t\tfl: fl,\n\t}\n\tfl.Lock()\n\tfl.l = l\n\tfl.Unlock()\n\n\tsetPlainLabelFormats(l)\n\treturn l\n}\n\ntype writerAndCloser interface {\n\tWrite(b []byte) (int, error)\n\tClose() error\n\tName() string\n}\n\ntype fileLogger struct {\n\tout int64\n\tcanRotate int32\n\tsync.Mutex\n\tl *Logger\n\tf writerAndCloser\n\tlimit int64\n\tolimit int64\n\tpid string\n\ttime bool\n\tclosed bool\n\tmaxNumFiles int\n}\n\nfunc newFileLogger(filename, pidPrefix string, time bool) (*fileLogger, error) {\n\tfileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE\n\tf, err := os.OpenFile(filename, fileflags, defaultLogPerms)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstats, err := f.Stat()\n\tif err != nil {\n\t\tf.Close()\n\t\treturn nil, err\n\t}\n\tfl := &fileLogger{\n\t\tcanRotate: 0,\n\t\tf: f,\n\t\tout: stats.Size(),\n\t\tpid: pidPrefix,\n\t\ttime: time,\n\t}\n\treturn fl, nil\n}\n\nfunc (l *fileLogger) setLimit(limit int64) {\n\tl.Lock()\n\tl.olimit, l.limit = limit, limit\n\tatomic.StoreInt32(&l.canRotate, 1)\n\trotateNow := l.out > l.limit\n\tl.Unlock()\n\tif rotateNow {\n\t\tl.l.Noticef(\"Rotating logfile...\")\n\t}\n}\n\nfunc (l *fileLogger) setMaxNumFiles(max int) {\n\tl.Lock()\n\tl.maxNumFiles = max\n\tl.Unlock()\n}\n\nfunc (l *fileLogger) logDirect(label, format string, v ...any) int {\n\tvar entrya = [256]byte{}\n\tvar entry = entrya[:0]\n\tif l.pid != \"\" {\n\t\tentry = append(entry, l.pid...)\n\t}\n\tif l.time {\n\t\tnow := time.Now()\n\t\tyear, month, day := now.Date()\n\t\thour, min, sec := now.Clock()\n\t\tmicrosec := now.Nanosecond() / 1000\n\t\tentry = append(entry, fmt.Sprintf(\"%04d/%02d/%02d %02d:%02d:%02d.%06d \",\n\t\t\tyear, month, day, hour, min, sec, microsec)...)\n\t}\n\tentry = append(entry, label...)\n\tentry = append(entry, fmt.Sprintf(format, v...)...)\n\tentry = append(entry, '\\r', '\\n')\n\tl.f.Write(entry)\n\treturn len(entry)\n}\n\nfunc (l *fileLogger) logPurge(fname string) {\n\tvar backups []string\n\tlDir := filepath.Dir(fname)\n\tlBase := filepath.Base(fname)\n\tentries, err := os.ReadDir(lDir)\n\tif err != nil {\n\t\tl.logDirect(l.l.errorLabel, \"Unable to read directory %q for log purge (%v), will attempt next rotation\", lDir, err)\n\t\treturn\n\t}\n\tfor _, entry := range entries {\n\t\tif entry.IsDir() || entry.Name() == lBase || !strings.HasPrefix(entry.Name(), lBase) {\n\t\t\tcontinue\n\t\t}\n\t\tif stamp, found := strings.CutPrefix(entry.Name(), fmt.Sprintf(\"%s%s\", lBase, \".\")); found {\n\t\t\t_, err := time.Parse(\"2006:01:02:15:04:05.999999999\", strings.Replace(stamp, \".\", \":\", 5))\n\t\t\tif err == nil {\n\t\t\t\tbackups = append(backups, entry.Name())\n\t\t\t}\n\t\t}\n\t}\n\tcurrBackups := len(backups)\n\tmaxBackups := l.maxNumFiles - 1\n\tif currBackups > maxBackups {\n\t\t// backups sorted oldest to latest based on timestamped lexical filename (ReadDir)\n\t\tfor i := 0; i < currBackups-maxBackups; i++ {\n\t\t\tif err := os.Remove(filepath.Join(lDir, string(os.PathSeparator), backups[i])); err != nil {\n\t\t\t\tl.logDirect(l.l.errorLabel, \"Unable to remove backup log file %q (%v), will attempt next rotation\", backups[i], err)\n\t\t\t\t// Bail fast, we'll try again next rotation\n\t\t\t\treturn\n\t\t\t}\n\t\t\tl.logDirect(l.l.infoLabel, \"Purged log file %q\", backups[i])\n\t\t}\n\t}\n}\n\nfunc (l *fileLogger) Write(b []byte) (int, error) {\n\tif atomic.LoadInt32(&l.canRotate) == 0 {\n\t\tn, err := l.f.Write(b)\n\t\tif err == nil {\n\t\t\tatomic.AddInt64(&l.out, int64(n))\n\t\t}\n\t\treturn n, err\n\t}\n\tl.Lock()\n\tn, err := l.f.Write(b)\n\tif err == nil {\n\t\tl.out += int64(n)\n\t\tif l.out > l.limit {\n\t\t\tif err := l.f.Close(); err != nil {\n\t\t\t\tl.limit *= 2\n\t\t\t\tl.logDirect(l.l.errorLabel, \"Unable to close logfile for rotation (%v), will attempt next rotation at size %v\", err, l.limit)\n\t\t\t\tl.Unlock()\n\t\t\t\treturn n, err\n\t\t\t}\n\t\t\tfname := l.f.Name()\n\t\t\tnow := time.Now()\n\t\t\tbak := fmt.Sprintf(\"%s.%04d.%02d.%02d.%02d.%02d.%02d.%09d\", fname,\n\t\t\t\tnow.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(),\n\t\t\t\tnow.Second(), now.Nanosecond())\n\t\t\tos.Rename(fname, bak)\n\t\t\tfileflags := os.O_WRONLY | os.O_APPEND | os.O_CREATE\n\t\t\tf, err := os.OpenFile(fname, fileflags, defaultLogPerms)\n\t\t\tif err != nil {\n\t\t\t\tl.Unlock()\n\t\t\t\tpanic(fmt.Sprintf(\"Unable to re-open the logfile %q after rotation: %v\", fname, err))\n\t\t\t}\n\t\t\tl.f = f\n\t\t\tn := l.logDirect(l.l.infoLabel, \"Rotated log, backup saved as %q\", bak)\n\t\t\tl.out = int64(n)\n\t\t\tl.limit = l.olimit\n\t\t\tif l.maxNumFiles > 0 {\n\t\t\t\tl.logPurge(fname)\n\t\t\t}\n\t\t}\n\t}\n\tl.Unlock()\n\treturn n, err\n}\n\nfunc (l *fileLogger) close() error {\n\tl.Lock()\n\tif l.closed {\n\t\tl.Unlock()\n\t\treturn nil\n\t}\n\tl.closed = true\n\tl.Unlock()\n\treturn l.f.Close()\n}\n\n// SetSizeLimit sets the size of a logfile after which a backup\n// is created with the file name + \"year.month.day.hour.min.sec.nanosec\"\n// and the current log is truncated.\nfunc (l *Logger) SetSizeLimit(limit int64) error {\n\tl.Lock()\n\tif l.fl == nil {\n\t\tl.Unlock()\n\t\treturn fmt.Errorf(\"can set log size limit only for file logger\")\n\t}\n\tfl := l.fl\n\tl.Unlock()\n\tfl.setLimit(limit)\n\treturn nil\n}\n\n// SetMaxNumFiles sets the number of archived log files that will be retained\nfunc (l *Logger) SetMaxNumFiles(max int) error {\n\tl.Lock()\n\tif l.fl == nil {\n\t\tl.Unlock()\n\t\treturn fmt.Errorf(\"can set log max number of files only for file logger\")\n\t}\n\tfl := l.fl\n\tl.Unlock()\n\tfl.setMaxNumFiles(max)\n\treturn nil\n}\n\n// NewTestLogger creates a logger with output directed to Stderr with a prefix.\n// Useful for tracing in tests when multiple servers are in the same pid\nfunc NewTestLogger(prefix string, time bool) *Logger {\n\tflags := 0\n\tif time {\n\t\tflags = log.LstdFlags | log.Lmicroseconds\n\t}\n\tl := &Logger{\n\t\tlogger: log.New(os.Stderr, prefix, flags),\n\t\tdebug: true,\n\t\ttrace: true,\n\t}\n\tsetColoredLabelFormats(l)\n\treturn l\n}\n\n// Close implements the io.Closer interface to clean up\n// resources in the server's logger implementation.\n// Caller must ensure threadsafety.\nfunc (l *Logger) Close() error {\n\tif l.fl != nil {\n\t\treturn l.fl.close()\n\t}\n\treturn nil\n}\n\n// Generate the pid prefix string\nfunc pidPrefix() string {\n\treturn fmt.Sprintf(\"[%d] \", os.Getpid())\n}\n\nfunc setPlainLabelFormats(l *Logger) {\n\tl.infoLabel = \"[INF] \"\n\tl.debugLabel = \"[DBG] \"\n\tl.warnLabel = \"[WRN] \"\n\tl.errorLabel = \"[ERR] \"\n\tl.fatalLabel = \"[FTL] \"\n\tl.traceLabel = \"[TRC] \"\n}\n\nfunc setColoredLabelFormats(l *Logger) {\n\tcolorFormat := \"[\\x1b[%sm%s\\x1b[0m] \"\n\tl.infoLabel = fmt.Sprintf(colorFormat, \"32\", \"INF\")\n\tl.debugLabel = fmt.Sprintf(colorFormat, \"36\", \"DBG\")\n\tl.warnLabel = fmt.Sprintf(colorFormat, \"0;93\", \"WRN\")\n\tl.errorLabel = fmt.Sprintf(colorFormat, \"31\", \"ERR\")\n\tl.fatalLabel = fmt.Sprintf(colorFormat, \"31\", \"FTL\")\n\tl.traceLabel = fmt.Sprintf(colorFormat, \"33\", \"TRC\")\n}\n\n// Noticef logs a notice statement\nfunc (l *Logger) Noticef(format string, v ...any) {\n\tl.logger.Printf(l.infoLabel+format, v...)\n}\n\n// Warnf logs a notice statement\nfunc (l *Logger) Warnf(format string, v ...any) {\n\tl.logger.Printf(l.warnLabel+format, v...)\n}\n\n// Errorf logs an error statement\nfunc (l *Logger) Errorf(format string, v ...any) {\n\tl.logger.Printf(l.errorLabel+format, v...)\n}\n\n// Fatalf logs a fatal error\nfunc (l *Logger) Fatalf(format string, v ...any) {\n\tl.logger.Fatalf(l.fatalLabel+format, v...)\n}\n\n// Debugf logs a debug statement\nfunc (l *Logger) Debugf(format string, v ...any) {\n\tif l.debug {\n\t\tl.logger.Printf(l.debugLabel+format, v...)\n\t}\n}\n\n// Tracef logs a trace statement\nfunc (l *Logger) Tracef(format string, v ...any) {\n\tif l.trace {\n\t\tl.logger.Printf(l.traceLabel+format, v...)\n\t}\n}\n" }, { "path": "logger/log_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage logger\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestStdLogger(t *testing.T) {\n\tlogger := NewStdLogger(false, false, false, false, false)\n\n\tflags := logger.logger.Flags()\n\tif flags != 0 {\n\t\tt.Fatalf(\"Expected %q, received %q\\n\", 0, flags)\n\t}\n\n\tif logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.debug)\n\t}\n\n\tif logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.trace)\n\t}\n}\n\nfunc TestStdLoggerWithDebugTraceAndTime(t *testing.T) {\n\tlogger := NewStdLogger(true, true, true, false, false)\n\n\tflags := logger.logger.Flags()\n\tif flags != log.LstdFlags|log.Lmicroseconds {\n\t\tt.Fatalf(\"Expected %d, received %d\\n\", log.LstdFlags, flags)\n\t}\n\n\tif !logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.debug)\n\t}\n\n\tif !logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.trace)\n\t}\n}\n\nfunc TestStdLoggerNotice(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, false, false, false, false)\n\t\tlogger.Noticef(\"foo\")\n\t}, \"[INF] foo\\n\")\n}\n\nfunc TestStdLoggerNoticeWithColor(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, false, false, true, false)\n\t\tlogger.Noticef(\"foo\")\n\t}, \"[\\x1b[32mINF\\x1b[0m] foo\\n\")\n}\n\nfunc TestStdLoggerDebug(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, true, false, false, false)\n\t\tlogger.Debugf(\"foo %s\", \"bar\")\n\t}, \"[DBG] foo bar\\n\")\n}\n\nfunc TestStdLoggerDebugWithOutDebug(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, false, false, false, false)\n\t\tlogger.Debugf(\"foo\")\n\t}, \"\")\n}\n\nfunc TestStdLoggerTrace(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, false, true, false, false)\n\t\tlogger.Tracef(\"foo\")\n\t}, \"[TRC] foo\\n\")\n}\n\nfunc TestStdLoggerTraceWithOutDebug(t *testing.T) {\n\texpectOutput(t, func() {\n\t\tlogger := NewStdLogger(false, false, false, false, false)\n\t\tlogger.Tracef(\"foo\")\n\t}, \"\")\n}\n\nfunc TestFileLogger(t *testing.T) {\n\ttmpDir := t.TempDir()\n\tfile := createFileAtDir(t, tmpDir, \"nats-server:log_\")\n\tfile.Close()\n\n\tlogger := NewFileLogger(file.Name(), false, false, false, false)\n\tdefer logger.Close()\n\tlogger.Noticef(\"foo\")\n\n\tbuf, err := os.ReadFile(file.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read logfile: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length logfile\")\n\t}\n\n\tif string(buf) != \"[INF] foo\\n\" {\n\t\tt.Fatalf(\"Expected '%s', received '%s'\\n\", \"[INFO] foo\", string(buf))\n\t}\n\n\tfile = createFileAtDir(t, tmpDir, \"nats-server:log_\")\n\tfile.Close()\n\n\tlogger = NewFileLogger(file.Name(), true, false, true, true)\n\tdefer logger.Close()\n\tlogger.Errorf(\"foo\")\n\n\tbuf, err = os.ReadFile(file.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read logfile: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length logfile\")\n\t}\n\tstr := string(buf)\n\terrMsg := fmt.Sprintf(\"Expected '%s', received '%s'\\n\", \"[pid] [ERR] foo\", str)\n\tpidEnd := strings.Index(str, \" \")\n\tinfoStart := strings.LastIndex(str, \"[ERR]\")\n\tif pidEnd == -1 || infoStart == -1 {\n\t\tt.Fatalf(\"%v\", errMsg)\n\t}\n\tpid := str[0:pidEnd]\n\tif pid[0] != '[' || pid[len(pid)-1] != ']' {\n\t\tt.Fatalf(\"%v\", errMsg)\n\t}\n\n\tdate := str[pidEnd:infoStart]\n\tdateRegExp := \"[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}\"\n\treg, err := regexp.Compile(dateRegExp)\n\tif err != nil {\n\t\tt.Fatalf(\"Compile date regexp error: %v\", err)\n\t}\n\tif matched := reg.Match([]byte(date)); !matched {\n\t\tt.Fatalf(\"Date string '%s' does not match '%s'\", date, dateRegExp)\n\t}\n\n\tif !strings.HasSuffix(str, \"[ERR] foo\\n\") {\n\t\tt.Fatalf(\"%v\", errMsg)\n\t}\n}\n\nfunc TestFileLoggerSizeLimit(t *testing.T) {\n\t// Create std logger\n\tlogger := NewStdLogger(true, false, false, false, true)\n\tif err := logger.SetSizeLimit(1000); err == nil ||\n\t\t!strings.Contains(err.Error(), \"only for file logger\") {\n\t\tt.Fatalf(\"Expected error about being able to use only for file logger, got %v\", err)\n\t}\n\tlogger.Close()\n\n\ttmpDir := t.TempDir()\n\n\tfile := createFileAtDir(t, tmpDir, \"log_\")\n\tfile.Close()\n\n\tlogger = NewFileLogger(file.Name(), true, false, false, true)\n\tdefer logger.Close()\n\tlogger.SetSizeLimit(1000)\n\n\tfor i := 0; i < 50; i++ {\n\t\tlogger.Noticef(\"This is a line in the log file\")\n\t}\n\n\tfiles, err := os.ReadDir(tmpDir)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading logs dir: %v\", err)\n\t}\n\tif len(files) == 1 {\n\t\tt.Fatalf(\"Expected file to have been rotated\")\n\t}\n\tlastBackup := files[len(files)-1]\n\tif err := logger.Close(); err != nil {\n\t\tt.Fatalf(\"Error closing log: %v\", err)\n\t}\n\tcontent, err := os.ReadFile(file.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Error loading latest log: %v\", err)\n\t}\n\tif !bytes.Contains(content, []byte(\"Rotated log\")) ||\n\t\t!bytes.Contains(content, []byte(lastBackup.Name())) {\n\t\tt.Fatalf(\"Should be statement about rotated log and backup name, got %s\", content)\n\t}\n\n\ttmpDir = t.TempDir()\n\n\t// Recreate logger and don't set a limit\n\tfile = createFileAtDir(t, tmpDir, \"log_\")\n\tfile.Close()\n\tlogger = NewFileLogger(file.Name(), true, false, false, true)\n\tdefer logger.Close()\n\tfor i := 0; i < 50; i++ {\n\t\tlogger.Noticef(\"This is line %d in the log file\", i+1)\n\t}\n\tfiles, err = os.ReadDir(tmpDir)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading logs dir: %v\", err)\n\t}\n\tif len(files) != 1 {\n\t\tt.Fatalf(\"Expected file to not be rotated\")\n\t}\n\n\t// Now set a limit that is below current size\n\tlogger.SetSizeLimit(1000)\n\t// Should have triggered rotation\n\tfiles, err = os.ReadDir(tmpDir)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading logs dir: %v\", err)\n\t}\n\tif len(files) <= 1 {\n\t\tt.Fatalf(\"Expected file to have been rotated\")\n\t}\n\tif err := logger.Close(); err != nil {\n\t\tt.Fatalf(\"Error closing log: %v\", err)\n\t}\n\tlastBackup = files[len(files)-1]\n\tcontent, err = os.ReadFile(file.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Error loading latest log: %v\", err)\n\t}\n\tif !bytes.Contains(content, []byte(\"Rotated log\")) ||\n\t\t!bytes.Contains(content, []byte(lastBackup.Name())) {\n\t\tt.Fatalf(\"Should be statement about rotated log and backup name, got %s\", content)\n\t}\n\n\tlogger = NewFileLogger(file.Name(), true, false, false, true)\n\tdefer logger.Close()\n\tlogger.SetSizeLimit(1000)\n\n\t// Check error on rotate.\n\tlogger.Lock()\n\tlogger.fl.Lock()\n\tfailClose := &fileLogFailClose{logger.fl.f, true}\n\tlogger.fl.f = failClose\n\tlogger.fl.Unlock()\n\tlogger.Unlock()\n\t// Write a big line that will force rotation.\n\t// Since we fail to close the log file, we should have bumped the limit to 2000\n\tlogger.Noticef(\"This is a big line: %v\", make([]byte, 1000))\n\n\t// Remove the failure\n\tfailClose.fail = false\n\t// Write a big line that makes rotation happen\n\tlogger.Noticef(\"This is a big line: %v\", make([]byte, 2000))\n\t// Close\n\tlogger.Close()\n\n\tfiles, err = os.ReadDir(tmpDir)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading logs dir: %v\", err)\n\t}\n\tlastBackup = files[len(files)-1]\n\tcontent, err = os.ReadFile(filepath.Join(tmpDir, lastBackup.Name()))\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading backup file: %v\", err)\n\t}\n\tif !bytes.Contains(content, []byte(\"on purpose\")) || !bytes.Contains(content, []byte(\"size 2000\")) {\n\t\tt.Fatalf(\"Expected error that file could not rotated and max size bumped to 2000, got %s\", content)\n\t}\n}\n\ntype fileLogFailClose struct {\n\twriterAndCloser\n\tfail bool\n}\n\nfunc (l *fileLogFailClose) Close() error {\n\tif l.fail {\n\t\treturn fmt.Errorf(\"on purpose\")\n\t}\n\treturn l.writerAndCloser.Close()\n}\n\nfunc expectOutput(t *testing.T, f func(), expected string) {\n\told := os.Stderr // keep backup of the real stderr\n\tr, w, _ := os.Pipe()\n\tos.Stderr = w\n\n\tf()\n\n\toutC := make(chan string)\n\t// copy the output in a separate goroutine so printing can't block indefinitely\n\tgo func() {\n\t\tvar buf bytes.Buffer\n\t\tio.Copy(&buf, r)\n\t\toutC <- buf.String()\n\t}()\n\n\tos.Stderr.Close()\n\tos.Stderr = old // restoring the real stdout\n\tout := <-outC\n\tif out != expected {\n\t\tt.Fatalf(\"Expected '%s', received '%s'\\n\", expected, out)\n\t}\n}\n\nfunc createFileAtDir(t *testing.T, dir, prefix string) *os.File {\n\tt.Helper()\n\tf, err := os.CreateTemp(dir, prefix)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn f\n}\n" }, { "path": "logger/syslog.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage logger\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"log/syslog\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n)\n\n// SysLogger provides a system logger facility\ntype SysLogger struct {\n\twriter *syslog.Writer\n\tdebug bool\n\ttrace bool\n}\n\n// SetSyslogName sets the name to use for the syslog.\n// Currently used only on Windows.\nfunc SetSyslogName(name string) {}\n\n// GetSysLoggerTag generates the tag name for use in syslog statements. If\n// the executable is linked, the name of the link will be used as the tag,\n// otherwise, the name of the executable is used. \"nats-server\" is the default\n// for the NATS server.\nfunc GetSysLoggerTag() string {\n\tprocName := os.Args[0]\n\tif strings.ContainsRune(procName, os.PathSeparator) {\n\t\tparts := strings.FieldsFunc(procName, func(c rune) bool {\n\t\t\treturn c == os.PathSeparator\n\t\t})\n\t\tprocName = parts[len(parts)-1]\n\t}\n\treturn procName\n}\n\n// NewSysLogger creates a new system logger\nfunc NewSysLogger(debug, trace bool) *SysLogger {\n\tw, err := syslog.New(syslog.LOG_DAEMON|syslog.LOG_NOTICE, GetSysLoggerTag())\n\tif err != nil {\n\t\tlog.Fatalf(\"error connecting to syslog: %q\", err.Error())\n\t}\n\n\treturn &SysLogger{\n\t\twriter: w,\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t}\n}\n\n// NewRemoteSysLogger creates a new remote system logger\nfunc NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger {\n\tnetwork, addr := getNetworkAndAddr(fqn)\n\tw, err := syslog.Dial(network, addr, syslog.LOG_DEBUG, GetSysLoggerTag())\n\tif err != nil {\n\t\tlog.Fatalf(\"error connecting to syslog: %q\", err.Error())\n\t}\n\n\treturn &SysLogger{\n\t\twriter: w,\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t}\n}\n\nfunc getNetworkAndAddr(fqn string) (network, addr string) {\n\tu, err := url.Parse(fqn)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tnetwork = u.Scheme\n\tif network == \"udp\" || network == \"tcp\" {\n\t\taddr = u.Host\n\t} else if network == \"unix\" {\n\t\taddr = u.Path\n\t} else {\n\t\tlog.Fatalf(\"error invalid network type: %q\", u.Scheme)\n\t}\n\n\treturn\n}\n\n// Noticef logs a notice statement\nfunc (l *SysLogger) Noticef(format string, v ...any) {\n\tl.writer.Notice(fmt.Sprintf(format, v...))\n}\n\n// Warnf logs a warning statement\nfunc (l *SysLogger) Warnf(format string, v ...any) {\n\tl.writer.Warning(fmt.Sprintf(format, v...))\n}\n\n// Fatalf logs a fatal error\nfunc (l *SysLogger) Fatalf(format string, v ...any) {\n\tl.writer.Crit(fmt.Sprintf(format, v...))\n}\n\n// Errorf logs an error statement\nfunc (l *SysLogger) Errorf(format string, v ...any) {\n\tl.writer.Err(fmt.Sprintf(format, v...))\n}\n\n// Debugf logs a debug statement\nfunc (l *SysLogger) Debugf(format string, v ...any) {\n\tif l.debug {\n\t\tl.writer.Debug(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Tracef logs a trace statement\nfunc (l *SysLogger) Tracef(format string, v ...any) {\n\tif l.trace {\n\t\tl.writer.Notice(fmt.Sprintf(format, v...))\n\t}\n}\n" }, { "path": "logger/syslog_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage logger\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\nvar serverFQN string\n\nfunc TestSysLogger(t *testing.T) {\n\tlogger := NewSysLogger(false, false)\n\n\tif logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.debug)\n\t}\n\n\tif logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.trace)\n\t}\n}\n\nfunc TestSysLoggerWithDebugAndTrace(t *testing.T) {\n\tlogger := NewSysLogger(true, true)\n\n\tif !logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.debug)\n\t}\n\n\tif !logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.trace)\n\t}\n}\n\nfunc testTag(t *testing.T, exePath, expected string) {\n\tos.Args[0] = exePath\n\tif result := GetSysLoggerTag(); result != expected {\n\t\tt.Fatalf(\"Expected %s, received %s\", expected, result)\n\t}\n}\n\nfunc restoreArg(orig string) {\n\tos.Args[0] = orig\n}\n\nfunc TestSysLoggerTagGen(t *testing.T) {\n\torigArg := os.Args[0]\n\tdefer restoreArg(origArg)\n\n\ttestTag(t, \"nats-server\", \"nats-server\")\n\ttestTag(t, filepath.Join(\".\", \"nats-server\"), \"nats-server\")\n\ttestTag(t, filepath.Join(\"home\", \"bin\", \"nats-server\"), \"nats-server\")\n\ttestTag(t, filepath.Join(\"..\", \"..\", \"nats-server\"), \"nats-server\")\n\ttestTag(t, \"nats-server.service1\", \"nats-server.service1\")\n\ttestTag(t, \"nats-server_service1\", \"nats-server_service1\")\n\ttestTag(t, \"nats-server-service1\", \"nats-server-service1\")\n\ttestTag(t, \"nats-server service1\", \"nats-server service1\")\n}\n\nfunc TestSysLoggerTag(t *testing.T) {\n\torigArg := os.Args[0]\n\tdefer restoreArg(origArg)\n\n\tos.Args[0] = \"ServerLoggerTag\"\n\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, true)\n\tlogger.Noticef(\"foo\")\n\n\tline := <-done\n\tdata := strings.Split(line, \"[\")\n\tif len(data) != 2 {\n\t\tt.Fatalf(\"Unexpected syslog line %s\\n\", line)\n\t}\n\n\tif !strings.Contains(data[0], os.Args[0]) {\n\t\tt.Fatalf(\"Expected '%s', received '%s'\\n\", os.Args[0], data[0])\n\t}\n}\n\nfunc TestRemoteSysLogger(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, true)\n\n\tif !logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.debug)\n\t}\n\n\tif !logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.trace)\n\t}\n\tlogger.Noticef(\"foo\")\n\t<-done\n}\n\nfunc TestRemoteSysLoggerNotice(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, true)\n\n\tlogger.Noticef(\"foo %s\", \"bar\")\n\texpectSyslogOutput(t, <-done, \"foo bar\\n\")\n}\n\nfunc TestRemoteSysLoggerDebug(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, true)\n\n\tlogger.Debugf(\"foo %s\", \"qux\")\n\texpectSyslogOutput(t, <-done, \"foo qux\\n\")\n}\n\nfunc TestRemoteSysLoggerDebugDisabled(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, false, false)\n\n\tlogger.Debugf(\"foo %s\", \"qux\")\n\trcvd := <-done\n\tif rcvd != \"\" {\n\t\tt.Fatalf(\"Unexpected syslog response %s\\n\", rcvd)\n\t}\n}\n\nfunc TestRemoteSysLoggerTrace(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, true)\n\n\tlogger.Tracef(\"foo %s\", \"qux\")\n\texpectSyslogOutput(t, <-done, \"foo qux\\n\")\n}\n\nfunc TestRemoteSysLoggerTraceDisabled(t *testing.T) {\n\tdone := make(chan string)\n\tstartServer(done)\n\tlogger := NewRemoteSysLogger(serverFQN, true, false)\n\n\tlogger.Tracef(\"foo %s\", \"qux\")\n\trcvd := <-done\n\tif rcvd != \"\" {\n\t\tt.Fatalf(\"Unexpected syslog response %s\\n\", rcvd)\n\t}\n}\n\nfunc TestGetNetworkAndAddrUDP(t *testing.T) {\n\tn, a := getNetworkAndAddr(\"udp://foo.com:1000\")\n\n\tif n != \"udp\" {\n\t\tt.Fatalf(\"Unexpected network %s\\n\", n)\n\t}\n\n\tif a != \"foo.com:1000\" {\n\t\tt.Fatalf(\"Unexpected addr %s\\n\", a)\n\t}\n}\n\nfunc TestGetNetworkAndAddrTCP(t *testing.T) {\n\tn, a := getNetworkAndAddr(\"tcp://foo.com:1000\")\n\n\tif n != \"tcp\" {\n\t\tt.Fatalf(\"Unexpected network %s\\n\", n)\n\t}\n\n\tif a != \"foo.com:1000\" {\n\t\tt.Fatalf(\"Unexpected addr %s\\n\", a)\n\t}\n}\n\nfunc TestGetNetworkAndAddrUnix(t *testing.T) {\n\tn, a := getNetworkAndAddr(\"unix:///foo.sock\")\n\n\tif n != \"unix\" {\n\t\tt.Fatalf(\"Unexpected network %s\\n\", n)\n\t}\n\n\tif a != \"/foo.sock\" {\n\t\tt.Fatalf(\"Unexpected addr %s\\n\", a)\n\t}\n}\nfunc expectSyslogOutput(t *testing.T, line string, expected string) {\n\tdata := strings.Split(line, \"]: \")\n\tif len(data) != 2 {\n\t\tt.Fatalf(\"Unexpected syslog line %s\\n\", line)\n\t}\n\n\tif data[1] != expected {\n\t\tt.Fatalf(\"Expected '%s', received '%s'\\n\", expected, data[1])\n\t}\n}\n\nfunc runSyslog(c net.PacketConn, done chan<- string) {\n\tvar buf [4096]byte\n\tvar rcvd string\n\tfor {\n\t\tn, _, err := c.ReadFrom(buf[:])\n\t\tif err != nil || n == 0 {\n\t\t\tbreak\n\t\t}\n\t\trcvd += string(buf[:n])\n\t}\n\tdone <- rcvd\n}\n\nfunc startServer(done chan<- string) {\n\tc, e := net.ListenPacket(\"udp\", \"127.0.0.1:0\")\n\tif e != nil {\n\t\tlog.Fatalf(\"net.ListenPacket failed udp :0 %v\", e)\n\t}\n\n\tserverFQN = fmt.Sprintf(\"udp://%s\", c.LocalAddr().String())\n\tc.SetReadDeadline(time.Now().Add(100 * time.Millisecond))\n\tgo runSyslog(c, done)\n}\n" }, { "path": "logger/syslog_windows.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package logger logs to the windows event log\npackage logger\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"golang.org/x/sys/windows/svc/eventlog\"\n)\n\nvar natsEventSource = \"NATS-Server\"\n\n// SetSyslogName sets the name to use for the system log event source\nfunc SetSyslogName(name string) {\n\tnatsEventSource = name\n}\n\n// SysLogger logs to the windows event logger\ntype SysLogger struct {\n\twriter *eventlog.Log\n\tdebug bool\n\ttrace bool\n}\n\n// NewSysLogger creates a log using the windows event logger\nfunc NewSysLogger(debug, trace bool) *SysLogger {\n\tif err := eventlog.InstallAsEventCreate(natsEventSource, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil {\n\t\tif !strings.Contains(err.Error(), \"registry key already exists\") {\n\t\t\tpanic(fmt.Sprintf(\"could not access event log: %v\", err))\n\t\t}\n\t}\n\n\tw, err := eventlog.Open(natsEventSource)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not open event log: %v\", err))\n\t}\n\n\treturn &SysLogger{\n\t\twriter: w,\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t}\n}\n\n// NewRemoteSysLogger creates a remote event logger\nfunc NewRemoteSysLogger(fqn string, debug, trace bool) *SysLogger {\n\tw, err := eventlog.OpenRemote(fqn, natsEventSource)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"could not open event log: %v\", err))\n\t}\n\n\treturn &SysLogger{\n\t\twriter: w,\n\t\tdebug: debug,\n\t\ttrace: trace,\n\t}\n}\n\nfunc formatMsg(tag, format string, v ...any) string {\n\torig := fmt.Sprintf(format, v...)\n\treturn fmt.Sprintf(\"pid[%d][%s]: %s\", os.Getpid(), tag, orig)\n}\n\n// Noticef logs a notice statement\nfunc (l *SysLogger) Noticef(format string, v ...any) {\n\tl.writer.Info(1, formatMsg(\"NOTICE\", format, v...))\n}\n\n// Warnf logs a warning statement\nfunc (l *SysLogger) Warnf(format string, v ...any) {\n\tl.writer.Info(1, formatMsg(\"WARN\", format, v...))\n}\n\n// Fatalf logs a fatal error\nfunc (l *SysLogger) Fatalf(format string, v ...any) {\n\tmsg := formatMsg(\"FATAL\", format, v...)\n\tl.writer.Error(5, msg)\n\tpanic(msg)\n}\n\n// Errorf logs an error statement\nfunc (l *SysLogger) Errorf(format string, v ...any) {\n\tl.writer.Error(2, formatMsg(\"ERROR\", format, v...))\n}\n\n// Debugf logs a debug statement\nfunc (l *SysLogger) Debugf(format string, v ...any) {\n\tif l.debug {\n\t\tl.writer.Info(3, formatMsg(\"DEBUG\", format, v...))\n\t}\n}\n\n// Tracef logs a trace statement\nfunc (l *SysLogger) Tracef(format string, v ...any) {\n\tif l.trace {\n\t\tl.writer.Info(4, formatMsg(\"TRACE\", format, v...))\n\t}\n}\n" }, { "path": "logger/syslog_windows_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage logger\n\nimport (\n\t\"os/exec\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"golang.org/x/sys/windows/svc/eventlog\"\n)\n\n// Skips testing if we do not have privledges to run this test.\n// This lets us skip the tests for general (non admin/system) users.\nfunc checkPrivledges(t *testing.T) {\n\tsrc := \"NATS-eventlog-testsource\"\n\tdefer eventlog.Remove(src)\n\tif err := eventlog.InstallAsEventCreate(src, eventlog.Info|eventlog.Error|eventlog.Warning); err != nil {\n\t\tif strings.Contains(err.Error(), \"Access is denied\") {\n\t\t\t// Skip this test because elevated privileges are required.\n\t\t\tt.SkipNow()\n\t\t}\n\t\t// let the tests report other types of errors\n\t}\n}\n\n// lastLogEntryContains reads the last entry (/c:1 /rd:true) written\n// to the event log by the NATS-Server source, returning true if the\n// passed text was found, false otherwise.\nfunc lastLogEntryContains(t *testing.T, text string) bool {\n\tvar output []byte\n\tvar err error\n\n\tcmd := exec.Command(\"wevtutil.exe\", \"qe\", \"Application\", \"/q:*[System[Provider[@Name='NATS-Server']]]\",\n\t\t\"/rd:true\", \"/c:1\")\n\tif output, err = cmd.Output(); err != nil {\n\t\tt.Fatalf(\"Unable to execute command: %v\", err)\n\t}\n\treturn strings.Contains(string(output), text)\n}\n\n// TestSysLogger tests event logging on windows\nfunc TestSysLogger(t *testing.T) {\n\tcheckPrivledges(t)\n\tlogger := NewSysLogger(false, false)\n\tif logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.debug)\n\t}\n\n\tif logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", false, logger.trace)\n\t}\n\tlogger.Noticef(\"%s\", \"Noticef\")\n\tif !lastLogEntryContains(t, \"[NOTICE]: Noticef\") {\n\t\tt.Fatalf(\"missing log entry\")\n\t}\n\n\tlogger.Errorf(\"%s\", \"Errorf\")\n\tif !lastLogEntryContains(t, \"[ERROR]: Errorf\") {\n\t\tt.Fatalf(\"missing log entry\")\n\t}\n\n\tlogger.Tracef(\"%s\", \"Tracef\")\n\tif lastLogEntryContains(t, \"Tracef\") {\n\t\tt.Fatalf(\"should not contain log entry\")\n\t}\n\n\tlogger.Debugf(\"%s\", \"Debugf\")\n\tif lastLogEntryContains(t, \"Debugf\") {\n\t\tt.Fatalf(\"should not contain log entry\")\n\t}\n}\n\n// TestSysLoggerWithDebugAndTrace tests event logging\nfunc TestSysLoggerWithDebugAndTrace(t *testing.T) {\n\tcheckPrivledges(t)\n\tlogger := NewSysLogger(true, true)\n\tif !logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.debug)\n\t}\n\n\tif !logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.trace)\n\t}\n\n\tlogger.Tracef(\"%s\", \"Tracef\")\n\tif !lastLogEntryContains(t, \"[TRACE]: Tracef\") {\n\t\tt.Fatalf(\"missing log entry\")\n\t}\n\n\tlogger.Debugf(\"%s\", \"Debugf\")\n\tif !lastLogEntryContains(t, \"[DEBUG]: Debugf\") {\n\t\tt.Fatalf(\"missing log entry\")\n\t}\n}\n\n// TestRemoteSysLoggerWithDebugAndTrace tests remote event logging\nfunc TestRemoteSysLoggerWithDebugAndTrace(t *testing.T) {\n\tcheckPrivledges(t)\n\tlogger := NewRemoteSysLogger(\"\", true, true)\n\tif !logger.debug {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.debug)\n\t}\n\n\tif !logger.trace {\n\t\tt.Fatalf(\"Expected %t, received %t\\n\", true, logger.trace)\n\t}\n\tlogger.Tracef(\"NATS %s\", \"[TRACE]: Remote Noticef\")\n\tif !lastLogEntryContains(t, \"Remote Noticef\") {\n\t\tt.Fatalf(\"missing log entry\")\n\t}\n}\n\nfunc TestSysLoggerFatalf(t *testing.T) {\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tif !lastLogEntryContains(t, \"[FATAL]: Fatalf\") {\n\t\t\t\tt.Fatalf(\"missing log entry\")\n\t\t\t}\n\t\t}\n\t}()\n\n\tcheckPrivledges(t)\n\tlogger := NewSysLogger(true, true)\n\tlogger.Fatalf(\"%s\", \"Fatalf\")\n\tt.Fatalf(\"did not panic when expected to\")\n}\n" }, { "path": "main.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\n//go:generate go run server/errors_gen.go\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nvar usageStr = `\nUsage: nats-server [options]\n\nServer Options:\n -a, --addr, --net Bind to host address (default: 0.0.0.0)\n -p, --port Use port for clients (default: 4222)\n -n, --name\n --server_name Server name (default: auto)\n -P, --pid File to store PID\n -m, --http_port Use port for http monitoring\n -ms,--https_port Use port for https monitoring\n -c, --config Configuration file\n -t Test configuration and exit\n -sl,--signal [=] Send signal to nats-server process (ldm, stop, quit, term, reopen, reload)\n can be either a PID (e.g. 1) or the path to a PID file (e.g. /var/run/nats-server.pid)\n --client_advertise Client URL to advertise to other servers\n --ports_file_dir Creates a ports file in the specified directory (_.ports).\n\nLogging Options:\n -l, --log File to redirect log output\n -T, --logtime Timestamp log entries (default: true)\n -s, --syslog Log to syslog or windows event log\n -r, --remote_syslog Syslog server addr (udp://localhost:514)\n -D, --debug Enable debugging output\n -V, --trace Trace the raw protocol\n -VV Verbose trace (traces system account as well)\n -DV Debug and trace\n -DVV Debug and verbose trace (traces system account as well)\n --log_size_limit Logfile size limit (default: auto)\n --max_traced_msg_len Maximum printable length for traced messages (default: unlimited)\n\nJetStream Options:\n -js, --jetstream Enable JetStream functionality\n -sd, --store_dir Set the storage directory\n\nAuthorization Options:\n --user User required for connections\n --pass Password required for connections\n --auth Authorization token required for connections\n\nTLS Options:\n --tls Enable TLS, do not verify clients (default: false)\n --tlscert Server certificate file\n --tlskey Private key for server certificate\n --tlsverify Enable TLS, verify client certificates\n --tlscacert Client certificate CA for verification\n\nCluster Options:\n --routes Routes to solicit and connect\n --cluster Cluster URL for solicited routes\n --cluster_name Cluster Name, if not set one will be dynamically generated\n --no_advertise Do not advertise known cluster information to clients\n --cluster_advertise Cluster URL to advertise to other servers\n --connect_retries For implicit routes, number of connect retries\n --cluster_listen Cluster url from which members can solicit routes\n\nProfiling Options:\n --profile Profiling HTTP port\n\nCommon Options:\n -h, --help Show this message\n -v, --version Show version\n --help_tls TLS help\n`\n\n// usage will print out the flag options for the server.\nfunc usage() {\n\tfmt.Printf(\"%s\\n\", usageStr)\n\tos.Exit(0)\n}\n\nfunc main() {\n\texe := \"nats-server\"\n\n\t// Create a FlagSet and sets the usage\n\tfs := flag.NewFlagSet(exe, flag.ExitOnError)\n\tfs.Usage = usage\n\n\t// Configure the options from the flags/config file\n\topts, err := server.ConfigureOptions(fs, os.Args[1:],\n\t\tserver.PrintServerAndExit,\n\t\tfs.Usage,\n\t\tserver.PrintTLSHelpAndDie)\n\tif err != nil {\n\t\tserver.PrintAndDie(fmt.Sprintf(\"%s: %s\", exe, err))\n\t} else if opts.CheckConfig {\n\t\tfmt.Fprintf(os.Stderr, \"%s: configuration file %s is valid (%s)\\n\", exe, opts.ConfigFile, opts.ConfigDigest())\n\t\tos.Exit(0)\n\t}\n\n\t// Create the server with appropriate options.\n\ts, err := server.NewServer(opts)\n\tif err != nil {\n\t\tserver.PrintAndDie(fmt.Sprintf(\"%s: %s\", exe, err))\n\t}\n\n\t// Configure the logger based on the flags.\n\ts.ConfigureLogger()\n\n\t// Start things up. Block here until done.\n\tif err := server.Run(s); err != nil {\n\t\tserver.PrintAndDie(err.Error())\n\t}\n\n\ts.WaitForShutdown()\n}\n" }, { "path": "scripts/cov.sh", "content": "#!/bin/bash\n# Run from directory above via ./scripts/cov.sh\n\ncheck_file () {\n # If the content of the file is simply \"mode: atomic\", then it means that the\n # code coverage did not complete due to a panic in one of the tests.\n if [[ $(cat ./cov/$2) == \"mode: atomic\" ]]; then\n echo \"#############################################\"\n echo \"## Code coverage for $1 package failed ##\"\n echo \"#############################################\"\n exit 1\n fi\n}\n\n# Do not globally set the -e flag because we don't a flapper to prevent the push to coverall.\n\nexport GO111MODULE=\"on\"\n\ngo install github.com/wadey/gocovmerge@latest\n# Fail fast by checking if we can run gocovmerge\ngocovmerge\nif [[ $? != 0 ]]; then\n echo \"Unable to run gocovmerge\"\n exit 1\nfi\n\nrm -rf ./cov\nmkdir cov\n#\n# Since it is difficult to get a full run without a flapper, do not use `-failfast`.\n# It is better to have one flapper or two and still get the report than have\n# to re-run the whole code coverage. One or two failed tests should not affect\n# so much the code coverage.\n#\n# However, we need to take into account that if there is a panic in one test, all\n# other tests in that package will not run, which then would cause the code coverage\n# to drastically be lowered. In that case, we don't want the code coverage to be\n# uploaded.\n#\ngo test -v -covermode=atomic -coverprofile=./cov/conf.out ./conf -timeout=1h -tags=skip_no_race_tests\ncheck_file \"conf\" \"conf.out\"\ngo test -v -covermode=atomic -coverprofile=./cov/internal.out ./internal/ldap -timeout=1h -tags=skip_no_race_tests\ncheck_file \"internal\" \"internal.out\"\ngo test -v -covermode=atomic -coverprofile=./cov/log.out ./logger -timeout=1h -tags=skip_no_race_tests\ncheck_file \"logger\" \"log.out\"\ngo test -v -covermode=atomic -coverprofile=./cov/server_avl.out ./server/avl -timeout=1h -tags=skip_no_race_tests\ncheck_file \"server_avl\" \"server_avl.out\"\ngo test -v -covermode=atomic -coverprofile=./cov/server.out ./server -timeout=1h -tags=skip_no_race_tests\ncheck_file \"server\" \"server.out\"\ngo test -v -covermode=atomic -coverprofile=./cov/test.out -coverpkg=./server ./test -timeout=1h -tags=skip_no_race_tests\ncheck_file \"test\" \"test.out\"\n\n# At this point, if that fails, we want the caller to know about the failure.\nset -e\ngocovmerge ./cov/*.out > acc.out\nrm -rf ./cov\n\n# If no argument passed, launch a browser to see the results.\nif [[ $1 == \"\" ]]; then\n go tool cover -html=acc.out\nfi\nset +e\n" }, { "path": "scripts/runTestsOnTravis.sh", "content": "#!/bin/sh\n\nset -ex\n\nif [ \"$1\" = \"compile\" ]; then\n # First check that NATS builds.\n go build -v;\n\n # Now run the linters.\n go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.4;\n golangci-lint run;\n if [ \"$TRAVIS_TAG\" != \"\" ]; then\n go test -v -run=TestVersionMatchesTag ./server -ldflags=\"-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG\" -count=1 -vet=off\n fi\n\nelif [ \"$1\" = \"build_only\" ]; then\n go build -v;\n\nelif [ \"$1\" = \"no_race_1_tests\" ]; then\n\n # Run tests without the `-race` flag. By convention, those tests start\n # with `TestNoRace`. This will include all test files that have the\n # \"&& !skip_no_race_1_tests\" in the go build directive.\n\n go test -v -p=1 -run=TestNoRace ./... -tags=skip_no_race_2_tests -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"no_race_2_tests\" ]; then\n\n # Run tests without the `-race` flag. By convention, those tests start\n # with `TestNoRace`. This will include all test files that have the\n # \"&& !skip_no_race_2_tests\" in the go build directive.\n\n go test -v -p=1 -run=TestNoRace ./... -tags=skip_no_race_1_tests -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"store_tests\" ]; then\n\n # Run store tests. By convention, all file store tests start with `TestFileStore`,\n # and memory store tests start with `TestMemStore`.\n\n go test $RACE -v -p=1 -run=Test\\(Store\\|FileStore\\|MemStore\\) ./server -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_tests\" ]; then\n\n # Run JetStream non-clustered tests. By convention, all JS tests start\n # with `TestJetStream`. We exclude the clustered, super-clustered and\n # consumer tests by using the appropriate tags.\n\n go test $RACE -v -p=1 -run=TestJetStream ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4,skip_js_super_cluster_tests,skip_js_consumer_tests -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_consumer_tests\" ]; then\n\n # Run JetStream consumer tests. All tests start with TestJetStreamConsumer prefix.\n\n go test $RACE -v -p=1 -run=TestJetStreamConsumer ./server -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"raft_tests\" ]; then\n\n # Run the RAFT tests. All tests start with TestNRG prefix.\n\n go test $RACE -v -p=1 ./server/... -run=\"^TestNRG\" -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_cluster_tests_1\" ]; then\n\n # Run JetStream clustered tests. By convention, all JS cluster tests\n # start with `TestJetStreamCluster`. Will run the first batch of tests,\n # excluding others with use of proper tags.\n\n go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests_2,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_cluster_tests_2\" ]; then\n\n # Run JetStream clustered tests. By convention, all JS cluster tests\n # start with `TestJetStreamCluster`. Will run the second batch of tests,\n # excluding others with use of proper tags.\n\n go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_3,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_cluster_tests_3\" ]; then\n\n # Run JetStream clustered tests. By convention, all JS cluster tests\n # start with `TestJetStreamCluster`. Will run the third batch of tests,\n # excluding others with use of proper tags.\n #\n\n go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_4 -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_cluster_tests_4\" ]; then\n\n # Run JetStream clustered tests. By convention, all JS cluster tests\n # start with `TestJetStreamCluster`. Will run the third batch of tests,\n # excluding others with use of proper tags.\n #\n\n go test $RACE -v -p=1 -run=TestJetStreamCluster ./server -tags=skip_js_cluster_tests,skip_js_cluster_tests_2,skip_js_cluster_tests_3 -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"js_super_cluster_tests\" ]; then\n\n # Run JetStream super clustered tests. By convention, all JS super cluster\n # tests start with `TestJetStreamSuperCluster`.\n\n go test $RACE -v -p=1 -run=TestJetStreamSuperCluster ./server -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"mqtt_tests\" ]; then\n\n # Run MQTT tests. By convention, all MQTT tests start with `TestMQTT`.\n\n go test $RACE -v -p=1 -run=TestMQTT ./server -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"msgtrace_tests\" ]; then\n\n # Run Message Tracing tests. By convention, all message tracing tests start with `TestMsgTrace`.\n\n go test $RACE -v -p=1 -run=TestMsgTrace ./server -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"srv_pkg_non_js_tests\" ]; then\n\n # Run all non JetStream tests in the server package. We exclude the\n # store tests by using the `skip_store_tests` build tag, the JS tests\n # by using `skip_js_tests`, MQTT tests by using `skip_mqtt_tests` and\n # message tracing tests by using `skip_msgtrace_tests`.\n # Ignore JWT and NRG tests here as they have their own matrix run.\n\n # Also including the ldflag with the version since this includes the `TestVersionMatchesTag`.\n go test $RACE -v -p=1 ./server/... -run=\"^Test(N[^R]|NR[^G]|J[^W]|JW[^T]|[^JN])\" -ldflags=\"-X=github.com/nats-io/nats-server/v2/server.serverVersion=$TRAVIS_TAG\" -tags=skip_store_tests,skip_js_tests,skip_mqtt_tests,skip_msgtrace_tests,skip_no_race_tests -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"jwt_tests\" ]; then\n\n # Run the JWT tests. All tests start with TestJWT.\n\n go test $RACE -v -p=1 ./server/... -run=\"^TestJWT\" -count=1 -vet=off -timeout=30m -failfast\n\nelif [ \"$1\" = \"non_srv_pkg_tests\" ]; then\n\n # Run all tests of all non server package.\n\n go test $RACE -v -p=1 $(go list ./... | grep -v \"/server\") -count=1 -vet=off -timeout=30m -failfast\n\nfi\n" }, { "path": "scripts/updateCopyrights.sh", "content": "#!/bin/bash\n\n# Ensure script is run at the root of a git repository\ngit rev-parse --is-inside-work-tree &>/dev/null || { echo \"Not inside a git repository\"; exit 1; }\n\n# Find all .go files tracked by git\ngit ls-files \"*.go\" | while read -r file; do\n # Skip files that don't have a copyright belonging to \"The NATS Authors\"\n current_copyright=$(grep -oE \"^// Copyright [0-9]{4}(-[0-9]{4})? The NATS Authors\" \"$file\" || echo \"\")\n [[ -z \"$current_copyright\" ]] && continue\n\n # Get the last commit year for the file, ignore commit messages containing the word \"copyright\"\n last_year=$(git log --follow --format=\"%ad\" --date=format:%Y --grep=\"(C|c)opyright\" --invert-grep -n 1 -- \"$file\")\n existing_years=$(echo \"$current_copyright\" | grep -oE \"[0-9]{4}(-[0-9]{4})?\")\n\n # Determine the new copyright range\n if [[ \"$existing_years\" =~ ^([0-9]{4})-([0-9]{4})$ ]]; then\n first_year=${BASH_REMATCH[1]}\n new_copyright=\"// Copyright $first_year-$last_year The NATS Authors\"\n elif [[ \"$existing_years\" =~ ^([0-9]{4})$ ]]; then\n first_year=${BASH_REMATCH[1]}\n if [[ \"$first_year\" == \"$last_year\" ]]; then\n new_copyright=\"// Copyright $first_year The NATS Authors\"\n else\n new_copyright=\"// Copyright $first_year-$last_year The NATS Authors\"\n fi\n else\n continue # If the format is somehow incorrect, skip the file\n fi\n\n # Update the first line\n if sed --version &>/dev/null; then\n # Linux sed\n sed -i \"1s|^// Copyright.*|$new_copyright|\" \"$file\"\n else\n # BSD/macOS sed, needs -i ''\n sed -i '' \"1s|^// Copyright.*|$new_copyright|\" \"$file\"\n fi\ndone\n" }, { "path": "server/README-MQTT.md", "content": "**MQTT Implementation Overview**\n\nRevision 1.1\n\nAuthors: Ivan Kozlovic, Lev Brouk\n\nNATS Server currently supports most of MQTT 3.1.1. This document describes how\nit is implemented.\n\nIt is strongly recommended to review the [MQTT v3.1.1\nspecifications](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html)\nand get a detailed understanding before proceeding with this document.\n\n# Contents\n\n1. [Concepts](#1-concepts)\n - [Server, client](#server-client)\n - [Connection, client ID, session](#connection-client-id-session)\n - [Packets, messages, and subscriptions](#packets-messages-and-subscriptions)\n - [Quality of Service (QoS), publish identifier (PI)](#quality-of-service-qos-publish-identifier-pi)\n - [Retained message](#retained-message)\n - [Will message](#will-message)\n2. [Use of JetStream](#2-use-of-jetstream)\n - [JetStream API](#jetstream-api)\n - [Streams](#streams)\n - [Consumers and Internal NATS Subscriptions](#consumers-and-internal-nats-subscriptions)\n3. [Lifecycles](#3-lifecycles)\n - [Connection, Session](#connection-session)\n - [Subscription](#subscription)\n - [Message](#message)\n - [Retained messages](#retained-messages)\n4. [Implementation Notes](#4-implementation-notes)\n - [Hooking into NATS I/O](#hooking-into-nats-io)\n - [Session Management](#session-management)\n - [Processing QoS acks: PUBACK, PUBREC, PUBCOMP](#processing-qos-acks-puback-pubrec-pubcomp)\n - [Subject Wildcards](#subject-wildcards)\n5. [Known issues](#5-known-issues)\n\n# 1. Concepts\n\n## Server, client\n\nIn the MQTT specification there are concepts of **Client** and **Server**, used\nsomewhat interchangeably with those of **Sender** and **Receiver**. A **Server**\nacts as a **Receiver** when it gets `PUBLISH` messages from a **Sender**\n**Client**, and acts as a **Sender** when it delivers them to subscribed\n**Clients**.\n\nIn the NATS server implementation there are also concepts (types) `server` and\n`client`. `client` is an internal representation of a (connected) client and\nruns its own read and write loops. Both of these have an `mqtt` field that if\nset makes them behave as MQTT-compliant.\n\nThe code and comments may sometimes be confusing as they refer to `server` and\n`client` sometimes ambiguously between MQTT and NATS.\n\n## Connection, client ID, session\n\nWhen an MQTT client connects to a server, it must send a `CONNECT` packet to\ncreate an **MQTT Connection**. The packet must include a **Client Identifier**.\nThe server will then create or load a previously saved **Session** for the (hash\nof) the client ID.\n\n## Packets, messages, and subscriptions\n\nThe low level unit of transmission in MQTT is a **Packet**. Examples of packets\nare: `CONNECT`, `SUBSCRIBE`, `SUBACK`, `PUBLISH`, `PUBCOMP`, etc.\n\nAn **MQTT Message** starts with a `PUBLISH` packet that a client sends to the\nserver. It is then matched against the current **MQTT Subscriptions** and is\ndelivered to them as appropriate. During the message delivery the server acts as\nan MQTT client, and the receiver acts as an MQTT server.\n\nInternally we use **NATS Messages** and **NATS Subscriptions** to facilitate\nmessage delivery. This may be somewhat confusing as the code refers to `msg` and\n`sub`. What may be even more confusing is that some MQTT packets (specifically,\n`PUBREL`) are represented as NATS messages, and that the original MQTT packet\n\"metadata\" may be encoded as NATS message headers.\n\n## Quality of Service (QoS), publish identifier (PI)\n\nMQTT specifies 3 levels of quality of service (**QoS**):\n\n- `0` for at most once. A single delivery attempt.\n- `1` for at least once. Will try to redeliver until acknowledged by the\n receiver.\n- `2` for exactly once. See the [SPEC REF] for the acknowledgement flow.\n\nQoS 1 and 2 messages need to be identified with publish identifiers (**PI**s). A\nPI is a 16-bit integer that must uniquely identify a message for the duration of\nthe required exchange of acknowledgment packets.\n\nNote that the QoS applies separately to the transmission of a message from a\nsender client to the server, and from the server to the receiving client. There\nis no protocol-level acknowledgements between the receiver and the original\nsender. The sender passes the ownership of messages to the server, and the\nserver then delivers them at maximum possible QoS to the receivers\n(subscribers). The PIs for in-flight outgoing messages are issued and stored per\nsession.\n\n## Retained message\n\nA **Retained Message** is not part of any MQTT session and is not removed when the\nsession that produced it goes away. Instead, the server needs to persist a\n_single_ retained message per topic. When a subscription is started, the server\nneeds to send the “matching” retained messages, that is, messages that would\nhave been delivered to the new subscription should that subscription had been\nrunning prior to the publication of this message.\n\nRetained messages are removed when the server receives a retained message with\nan empty body. Still, this retained message that serves as a “delete” of a\nretained message will be processed as a normal published message.\n\nRetained messages can have QoS.\n\n## Will message\n\nThe `CONNECT` packet can contain information about a **Will Message** that needs to\nbe sent to any client subscribing on the Will topic/subject in the event that\nthe client is disconnected implicitly, that is, not as a result as the client\nsending the `DISCONNECT` packet.\n\nWill messages can have the retain flag and QoS.\n\n# 2. Use of JetStream\n\nThe MQTT implementation relies heavily on JetStream. We use it to:\n\n- Persist (and restore) the [Session](#connection-client-id-session) state.\n- Store and retrieve [Retained messages](#retained-message).\n- Persist incoming [QoS 1 and\n 2](#quality-of-service-qos-publish-identifier-pi) messages, and\n re-deliver if needed.\n- Store and de-duplicate incoming [QoS\n 2](#quality-of-service-qos-publish-identifier-pi) messages.\n- Persist and re-deliver outgoing [QoS\n 2](#quality-of-service-qos-publish-identifier-pi) `PUBREL` packets.\n\nHere is the overview of how we set up and use JetStream **streams**,\n**consumers**, and **internal NATS subscriptions**.\n\n## JetStream API\n\nAll interactions with JetStream are performed via `mqttJSA` that sends NATS\nrequests to JetStream. Most are processed synchronously and await a response,\nsome (e.g. `jsa.sendAck()`) are sent asynchronously. JetStream API is usually\nreferred to as `jsa` in the code. No special locking is required to use `jsa`,\nhowever the asynchronous use of JetStream may create race conditions with\ndelivery callbacks.\n\n## Streams\n\nWe create the following streams unless they already exist. Failing to ensure the\nstreams would prevent the client from connecting.\n\nEach stream is created with a replica value that is determined by the size of\nthe cluster but limited to 3. It can also be overwritten by the stream_replicas\noption in the MQTT configuration block.\n\nThe streams are created the first time an Account Session Manager is initialized\nand are used by all sessions in it. Note that to avoid race conditions, some\nsubscriptions are created first. The streams are never deleted. See\n`mqttCreateAccountSessionManager()` for details.\n\n1. `$MQTT_sess` stores persisted **Session** records. It filters on\n `\"$MQTT.sess.>` subject and has a “limits” policy with `MaxMsgsPer` setting\n of 1.\n2. `$MQTT_msgs` is used for **QoS 1 and 2 message delivery**.\n It filters on `$MQTT.msgs.>` subject and has an “interest” policy.\n3. `$MQTT_rmsgs` stores **Retained Messages**. They are all\n stored (and filtered) on a single subject `$MQTT.rmsg`. This stream has a\n limits policy.\n4. `$MQTT_qos2in` stores and deduplicates **Incoming QoS 2 Messages**. It\n filters on `$MQTT.qos2.in.>` and has a \"limits\" policy with `MaxMsgsPer` of\n 1.\n5. `$MQTT_out` stores **Outgoing QoS 2** `PUBREL` packets. It filters on\n `$MQTT.out.>` and has a \"interest\" retention policy.\n\n## Consumers and Internal NATS Subscriptions\n\n### Account Scope\n\n- A durable consumer for [Retained Messages](#retained-message) -\n `$MQTT_rmsgs_`\n- A subscription to handle all [jsa](#jetstream-api) replies for the account.\n- A subscription to replies to \"session persist\" requests, so that we can detect\n the use of a session with the same client ID anywhere in the cluster.\n- 2 subscriptions to support [retained messages](#retained-message):\n `$MQTT.sub.` for the messages themselves, and one to receive replies to\n \"delete retained message\" JS API (on the JS reply subject var).\n\n### Session Scope\n\nWhen a new QoS 2 MQTT subscription is detected in a session, we ensure that\nthere is a durable consumer for [QoS\n2](#quality-of-service-qos-publish-identifier-pi) `PUBREL`s out for delivery -\n`$MQTT_PUBREL_`\n\n### Subscription Scope\n\nFor all MQTT subscriptions, regardless of their QoS, we create internal NATS subscriptions to\n\n- `subject` (directly encoded from `topic`). This subscription is used to\n deliver QoS 0 messages, and messages originating from NATS.\n- if needed, `subject fwc` complements `subject` for topics like `topic.#` to\n include `topic` itself, see [top-level wildcards](#subject-wildcards)\n\nFor QoS 1 or 2 MQTT subscriptions we ensure:\n\n- A durable consumer for messages out for delivery - `_`\n- An internal subscription to `$MQTT.sub.` to deliver the messages to the\n receiving client.\n\n### (Old) Notes\n\nAs indicated before, for a QoS1 or QoS2 subscription, the server will create a\nJetStream consumer with the appropriate subject filter. If the subscription\nalready existed, then only the NATS subscription is created for the JetStream\nconsumer’s delivery subject.\n\nNote that JS consumers can be created with an “Replicas” override, which from\nrecent discussion is problematic with “Interest” policy streams, which\n“$MQTT_msgs” is.\n\nWe do handle situations where a subscription on the same subject filter is sent\nwith a different QoS as per MQTT specifications. If the existing was on QoS 1 or\n2, and the “new” is for QoS 0, then we delete the existing JS consumer.\n\nSubscriptions that are QoS 0 have a NATS subscription with the callback function\nbeing `mqttDeliverMsgCbQos0()`; while QoS 1 and 2 have a NATS subscription with\ncallback `mqttDeliverMsgCbQos12()`. Both those functions have comments that\ndescribe the reason for their existence and what they are doing. For instance\nthe `mqttDeliverMsgCbQos0()` callback will reject any producing client that is\nof type JETSTREAM, so that it handles only non JetStream (QoS 1 and 2) messages.\n\nBoth these functions end-up calling mqttDeliver() which will first enqueue the\npossible retained messages buffer before delivering any new message. The message\nitself being delivered is serialized in MQTT format and enqueued to the client’s\noutbound buffer and call to addToPCD is made so that it is flushed out of the\nreadloop.\n\n# 3. Lifecycles\n\n## Connection, Session\n\nAn MQTT connection is created when a listening MQTT server receives a `CONNECT`\npacket. See `mqttProcessConnect()`. A connection is associated with a session.\nSteps:\n\n1. Ensure that we have an `AccountSessionManager` so we can have an\n `mqttSession`. Lazily initialize JetStream streams, and internal consumers\n and subscriptions. See `getOrCreateMQTTAccountSessionManager()`.\n2. Find and disconnect any previous session/client for the same ID. See\n `mqttProcessConnect()`.\n3. Ensure we have an `mqttSession` - create a new or load a previously persisted\n one. If the clean flag is set in `CONNECT`, clean the session. see\n `mqttSession.clear()`\n4. Initialize session's subscriptions, if any.\n5. Always send back a `CONNACK` packet. If there were errors in previous steps,\n include the error.\n\nAn MQTT connection can be closed for a number of reasons, including receiving a\n`DISCONNECT` from the client, explicit internal errors processing MQTT packets,\nor the server receiving another `CONNECT` packet with the same client ID. See\n`mqttHandleClosedClient()` and `mqttHandleWill()`. Steps:\n\n1. Send out the Will Message if applicable (if not caused by a `DISCONNECT` packet)\n2. Delete the JetStream consumers for to QoS 1 and 2 packet delivery through\n JS API calls (if \"clean\" session flag is set)\n3. Delete the session record from the “$MQTT_sess” stream, based on recorded\n stream sequence. (if \"clean\" session flag is set)\n4. Close the client connection.\n\nOn an explicit disconnect, that is, the client sends the DISCONNECT packet, the\nserver will NOT send the Will, as per specifications.\n\nFor sessions that had the “clean” flag, the JS consumers corresponding to QoS 1\nsubscriptions are deleted through JS API calls, the session record is then\ndeleted (based on recorded stream sequence) from the “$MQTT_sess” stream.\n\nFinally, the client connection is closed\n\nSessions are persisted on disconnect, and on subscriptions changes.\n\n## Subscription\n\nReceiving an MQTT `SUBSCRIBE` packet creates new subscriptions, or updates\nexisting subscriptions in a session. Each `SUBSCRIBE` packet may contain several\nspecific subscriptions (`topic` + QoS in each). We always respond with a\n`SUBACK`, which may indicate which subscriptions errored out.\n\nFor each subscription in the packet, we:\n\n1. Ignore it if `topic` starts with `$MQTT.sub.`.\n2. Set up QoS 0 message delivery - an internal NATS subscription on `topic`.\n3. Replay any retained messages for `topic`, once as QoS 0.\n4. If we already have a subscription on `topic`, update its QoS\n5. If this is a QoS 2 subscription in the session, ensure we have the [PUBREL\n consumer](#session-scope) for the session.\n6. If this is a QoS 1 or 2 subscription, ensure we have the [Message\n consumer](#subscription-scope) for this subscription (or delete one if it\n exists and this is now a QoS 0 sub).\n7. Add an extra subscription for the [top-level wildcard](#subject-wildcards) case.\n8. Update the session, persist it if changed.\n\nWhen a session is restored (no clean flag), we go through the same steps to\nre-subscribe to its stored subscription, except step #8 which would have been\nredundant.\n\nWhen we get an `UNSUBSCRIBE` packet, it can contain multiple subscriptions to\nunsubscribe. The parsing will generate a slice of mqttFilter objects that\ncontain the “filter” (the topic with possibly wildcard of the subscription) and\nthe QoS value. The server goes through the list and deletes the JS consumer (if\nQoS 1 or 2) and unsubscribes the NATS subscription for the delivery subject (if\nit was a QoS 1 or 2) or on the actual topic/subject. In case of the “#”\nwildcard, the server will handle the “level up” subscriptions that NATS had to\ncreate.\n\nAgain, we update the session and persist it as needed in the `$MQTT_sess`\nstream.\n\n## Message\n\n1. Detect an incoming PUBLISH packet, parse and check the message QoS. Fill out\n the session's `mqttPublish` struct that contains information about the\n published message. (see `mqttParse()`, `mqttParsePub()`)\n2. Process the message according to its QoS (see `mqttProcessPub()`)\n\n - QoS 0:\n - Initiate message delivery\n - QoS 1:\n - Initiate message delivery\n - Send back a `PUBACK`\n - QoS 2:\n - Store the message in `$MQTT_qos2in` stream, using a PI-specific subject.\n Since `MaxMsgsPer` is set to 1, we will ignore duplicates on the PI.\n - Send back a `PUBREC`\n - \"Wait\" for a `PUBREL`, then initiate message delivery\n - Remove the previously stored QoS2 message\n - Send back a `PUBCOMP`\n\n3. Initiate message delivery (see `mqttInitiateMsgDelivery()`)\n\n - Convert the MQTT `topic` into a NATS `subject` using\n `mqttTopicToNATSPubSubject()` function. If there is a known subject\n mapping, then we select the new subject using `selectMappedSubject()`\n function and then convert back this subject into an MQTT topic using\n `natsSubjectToMQTTTopic()` function.\n - Re-serialize the `PUBLISH` packet received as a NATS message. Use NATS\n headers for the metadata, and the deliverable MQTT `PUBLISH` packet as the\n contents.\n - Publish the messages as `subject` (and `subject fwc` if applicable, see\n [subject wildcards](#subject-wildcards)). Use the \"standard\" NATS\n `c.processInboundClientMsg()` to do that. `processInboundClientMsg()` will\n distribute the message to any NATS subscriptions (including routes,\n gateways, leafnodes) and the relevant MQTT subscriptions.\n - Check for retained messages, process as needed. See\n `c.processInboundClientMsg()` calling `c.mqttHandlePubRetain()` For MQTT\n clients.\n - If the message QoS is 1 or 2, store it in `$MQTT_msgs` stream as\n `$MQTT.msgs.` for \"at least once\" delivery with retries.\n\n4. Let NATS and JetStream deliver to the internal subscriptions, and to the\n receiving clients. See `mqttDeliverMsgCb...()`\n\n - The NATS message posted to `subject` (and `subject fwc`) will be delivered\n to each relevant internal subscription by calling `mqttDeliverMsgCbQoS0()`.\n The function has access to both the publishing and the receiving clients.\n\n - Ignore all irrelevant invocations. Specifically, do nothing if the\n message needs to be delivered with a higher QoS - that will be handled by\n the other, `...QoS12` callback. Note that if the original message was\n publuished with a QoS 1 or 2, but the subscription has its maximum QoS\n set to 0, the message will be delivered by this callback.\n - Ignore \"reserved\" subscriptions, as per MQTT spec.\n - Decode delivery `topic` from the NATS `subject`.\n - Write (enqueue) outgoing `PUBLISH` packet.\n - **DONE for QoS 0**\n\n - The NATS message posted to JetStream as `$MQTT.msgs.subject` will be\n consumed by subscription-specific consumers. Note that MQTT subscriptions\n with max QoS 0 do not have JetStream consumers. They are handled by the\n QoS0 callback.\n\n The consumers will deliver it to the `$MQTT.sub.`\n subject for their respective NATS subscriptions by calling\n `mqttDeliverMsgCbQoS12()`. This callback too has access to both the\n publishing and the receiving clients.\n\n - Ignore \"reserved\" subscriptions, as per MQTT spec.\n - See if this is a re-delivery from JetStream by checking `sess.cpending`\n for the JS reply subject. If so, use the existing PI and treat this as a\n duplicate redelivery.\n - Otherwise, assign the message a new PI (see `trackPublish()` and\n `bumpPI()`) and store it in `sess.cpending` and `sess.pendingPublish`,\n along with the JS reply subject that can be used to remove this pending\n message from the consumer once it's delivered to the receipient.\n - Decode delivery `topic` from the NATS `subject`.\n - Write (enqueue) outgoing `PUBLISH` packet.\n\n5. QoS 1: \"Wait\" for a `PUBACK`. See `mqttProcessPubAck()`.\n\n - When received, remove the PI from the tracking maps, send an ACK to\n consumer to remove the message.\n - **DONE for QoS 1**\n\n6. QoS 2: \"Wait\" for a `PUBREC`. When received, we need to do all the same\n things as in the QoS 1 `PUBACK` case, but we need to send out a `PUBREL`, and\n continue using the same PI until the delivery flow is complete and we get\n back a `PUBCOMP`. For that, we add the PI to `sess.pendingPubRel`, and to\n `sess.cpending` with the PubRel consumer durable name.\n\n We also compose and store a headers-only NATS message signifying a `PUBREL`\n out for delivery, and store it in the `$MQTT_qos2out` stream, as\n `$MQTT.qos2.out.`.\n\n7. QoS 2: Deliver `PUBREL`. The PubRel session-specific consumer will publish to\n internal subscription on `$MQTT.qos2.delivery`, calling\n `mqttDeliverPubRelCb()`. We store the ACK reply subject in `cpending` to\n remove the JS message on `PUBCOMP`, compose and send out a `PUBREL` packet.\n\n8. QoS 2: \"Wait\" for a `PUBCOMP`. See `mqttProcessPubComp()`.\n - When received, remove the PI from the tracking maps, send an ACK to\n consumer to remove the `PUBREL` message.\n - **DONE for QoS 2**\n\n## Retained messages\n\nWhen we process an inbound `PUBLISH` and submit it to\n`processInboundClientMsg()` function, for MQTT clients it will invoke\n`mqttHandlePubRetain()` which checks if the published message is “retained” or\nnot.\n\nIf it is, then we construct a record representing the retained message and store\nit in the `$MQTT_rmsg` stream, under the single `$MQTT.rmsg` subject. The stored\nrecord (in JSON) contains information about the subject, topic, MQTT flags, user\nthat produced this message and the message content itself. It is stored and the\nstream sequence is remembered in the memory structure that contains retained\nmessages.\n\nNote that when creating an account session manager, the retained messages stream\nis read from scratch to load all the messages through the use of a JS consumer.\nThe associated subscription will process the recovered retained messages or any\nnew that comes from the network.\n\nA retained message is added to a map and a subscription is created and inserted\ninto a sublist that will be used to perform a ReverseMatch() when a subscription\nis started and we want to find all retained messages that the subscription would\nhave received if it had been running prior to the message being published.\n\nIf a retained message on topic “foo” already exists, then the server has to\ndelete the old message at the stream sequence we saved when storing it.\n\nThis could have been done with having retained messages stored under\n`$MQTT.rmsg.` as opposed to all under a single subject, and make use of\nthe `MaxMsgsPer` field set to 1. The `MaxMsgsPer` option was introduced well into\nthe availability of MQTT and changes to the sessions was made in [PR\n#2501](https://github.com/nats-io/nats-server/pull/2501), with a conversion of\nexisting streams such as `$MQTT*sess*` into a single stream with unique\nsubjects, but the changes were not made to the retained messages stream.\n\nThere are also subscriptions for the handling of retained messages which are\nmessages that are asked by the publisher to be retained by the MQTT server to be\ndelivered to matching subscriptions when they start. There is a single message\nper topic. Retained messages are deleted when the user sends a retained message\n(there is a flag in the PUBLISH protocol) on a given topic with an empty body.\nThe difficulty with retained messages is to handle them in a cluster since all\nservers need to be aware of their presence so that they can deliver them to\nsubscriptions that those servers may become the leader for.\n\n- `$MQTT_rmsgs` which has a “limits” policy and holds retained messages, all\n under `$MQTT.rmsg` single subject. Not sure why I did not use MaxMsgsPer for\n this stream and not filter `$MQTT.rmsg.>`.\n\nThe first step when processing a new subscription is to gather the retained\nmessages that would be a match for this subscription. To do so, the server will\nserialize into a buffer all messages for the account session manager’s sublist’s\nReverseMatch result. We use the returned subscriptions’ subject to find from a\nmap appropriate retained message (see `serializeRetainedMsgsForSub()` for\ndetails).\n\n# 4. Implementation Notes\n\n## Hooking into NATS I/O\n\n### Starting the accept loop\n\nThe MQTT accept loop is started when the server detects that an MQTT port has\nbeen defined in the configuration file. It works similarly to all other accept\nloops. Note that for MQTT over websocket, the websocket port has to be defined\nand MQTT clients will connect to that port instead of the MQTT port and need to\nprovide `/mqtt` as part of the URL to redirect the creation of the client to an\nMQTT client (with websocket support) instead of a regular NATS with websocket.\nSee the branching done in `startWebsocketServer()`. See `startMQTT()`.\n\n### Starting the read/write loops\n\nWhen a TCP connection is accepted, the internal go routine will invoke\n`createMQTTClient()`. This function will set a `c.mqtt` object that will make it\nbecome an MQTT client (through the `isMqtt()` helper function). The `readLoop()`\nand `writeLoop()` are started similarly to other clients. However, the read loop\nwill branch out to `mqttParse()` instead when detecting that this is an MQTT\nclient.\n\n## Session Management\n\n### Account Session Manager\n\n`mqttAccountSessionManager` is an object that holds the state of all sessions in\nan account. It also manages the lifecycle of JetStream streams and internal\nsubscriptions for processing JS API replies, session updates, etc. See\n`mqttCreateAccountSessionManager()`. It is lazily initialized upon the first\nMQTT `CONNECT` packet received. Account session manager is referred to as `asm`\nin the code.\n\nNote that creating the account session manager (and attempting to create the\nstreams) is done only once per account on a given server, since once created the\naccount session manager for a given account would be found in the sessions map\nof the mqttSessionManager object.\n\n### Find and disconnect previous session/client\n\nOnce all that is done, we now go to the creation of the session object itself.\nFor that, we first need to make sure that it does not already exist, meaning\nthat it is registered on the server - or anywhere in the cluster. Note that MQTT\ndictates that if a session with the same ID connects, the OLD session needs to\nbe closed, not the new one being created. NATS Server complies with this\nrequirement.\n\nOnce a session is detected to already exists, the old one (as described above)\nis closed and the new one accepted, however, the session ID is maintained in a\nflappers map so that we detect situations where sessions with the same ID are\nstarted multiple times causing the previous one to be closed. When that\ndetection occurs, the newly created session is put in “jail” for a second to\navoid a very rapid succession of connect/disconnect. This has already been seen\nby users since there was some issue there where we would schedule the connection\nclosed instead of waiting in place which was causing a panic.\n\nWe also protect from multiple clients on a given server trying to connect with\nthe same ID at the “same time” while the processing of a CONNECT of a session is\nnot yet finished. This is done with the use of a sessLocked map, keyed by the\nsession ID.\n\n### Create or restore the session\n\nIf everything is good up to that point, the server will either create or restore\na session from the stream. This is done in the `createOrRestoreSession()`\nfunction. The client/session ID is hashed and added to the session’s stream\nsubject along with the JS domain to prevent clients connecting from different\ndomains to “pollute” the session stream of a given domain.\n\nSince each session constitutes a subject and the stream has a maximum of 1\nmessage per subject, we attempt to load the last message on the formed subject.\nIf we don’t find it, then the session object is created “empty”, while if we\nfind a record, we create the session object based on the record persisted on the\nstream.\n\nIf the session was restored from the JS stream, we keep track of the stream\nsequence where the record was located. When we save the session (even if it\nalready exists) we will use this sequence number to set the\n`JSExpectedLastSubjSeq` header so that we handle possibly different servers in a\n(super)cluster to detect the race of clients trying to use the same session ID,\nsince only one of the write should succeed. On success, the session’s new\nsequence is remembered by the server that did the write.\n\nWhen created or restored, the CONNACK can now be sent back to the client, and if\nthere were any recovered subscriptions, they are now processed.\n\n## Processing QoS acks: PUBACK, PUBREC, PUBCOMP\n\nWhen the server delivers a message with QoS 1 or 2 (also a `PUBREL` for QoS 2) to a subscribed client, the client will send back an acknowledgement. See `mqttProcessPubAck()`, `mqttProcessPubRec()`, and `mqttProcessPubComp()`\n\nWhile the specific logic for each packet differs, these handlers all update the\nsession's PI mappings (`cpending`, `pendingPublish`, `pendingPubRel`), and if\nneeded send an ACK to JetStream to remove the message from its consumer and stop\nthe re-delivery attempts.\n\n## Subject Wildcards\n\nNote that MQTT subscriptions have wildcards too, the `“+”` wildcard is equivalent\nto NATS’s `“*”` wildcard, however, MQTT’s wildcard `“#”` is similar to `“>”`, except\nthat it also includes the level above. That is, a subscription on `“foo/#”` would\nreceive messages on `“foo/bar/baz”`, but also on `“foo”`.\n\nSo, for MQTT subscriptions enging with a `'#'` we are forced to create 2\ninternal NATS subscriptions, one on `“foo”` and one on `“foo.>”`.\n\n# 5. Known issues\n- \"active\" redelivery for QoS from JetStream (compliant, just a note)\n- JetStream QoS redelivery happens out of (original) order\n- finish delivery of in-flight messages after UNSUB\n- finish delivery of in-flight messages after a reconnect\n- consider replacing `$MQTT_msgs` with `$MQTT_out`.\n- consider using unique `$MQTT.rmsg.>` and `MaxMsgsPer` for retained messages.\n- add a cli command to list/clean old sessions\n" }, { "path": "server/README.md", "content": "# Tests\n\nTests that run on Travis have been split into jobs that run in their own VM in parallel. This reduces the overall running time but also is allowing recycling of a job when we get a flapper as opposed to have to recycle the whole test suite.\n\n## JetStream Tests\n\nFor JetStream tests, we need to observe a naming convention so that no tests are omitted when running on Travis.\n\nThe script `runTestsOnTravis.sh` will run a given job based on the definition found in \"`.travis.yml`\".\n\nAs for the naming convention:\n\n- All JetStream test name should start with `TestJetStream`\n- Cluster tests should go into `jetstream_cluster_test.go` and start with `TestJetStreamCluster`\n- Super-cluster tests should go into `jetstream_super_cluster_test.go` and start with `TestJetStreamSuperCluster`\n\nNot following this convention means that some tests may not be executed on Travis.\n" }, { "path": "server/accounts.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"net/textproto\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// For backwards compatibility with NATS < 2.0, users who are not explicitly defined into an\n// account will be grouped in the default global account.\nconst globalAccountName = DEFAULT_GLOBAL_ACCOUNT\n\nconst defaultMaxSubLimitReportThreshold = int64(2 * time.Second)\n\nvar maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold\n\n// Account are subject namespace definitions. By default no messages are shared between accounts.\n// You can share via Exports and Imports of Streams and Services.\ntype Account struct {\n\t// Total stats for the account.\n\tstats struct {\n\t\tsync.Mutex\n\t\tstats // Totals\n\t\tgw stats // Gateways\n\t\trt stats // Routes\n\t\tln stats // Leafnodes\n\t}\n\n\tgwReplyMapping\n\tName string\n\tNkey string\n\tIssuer string\n\tclaimJWT string\n\tupdated time.Time\n\tmu sync.RWMutex\n\tsqmu sync.Mutex\n\tsl *Sublist\n\tic *client\n\tsq *sendq\n\tisid uint64\n\tetmr *time.Timer\n\tctmr *time.Timer\n\tstrack map[string]sconns\n\tnrclients int32\n\tsysclients int32\n\tnleafs int32\n\tnrleafs int32\n\tclients map[*client]struct{}\n\trm map[string]int32\n\tlqws map[string]int32\n\tusersRevoked map[string]int64\n\tmappings []*mapping\n\thasMapped atomic.Bool\n\tlmu sync.RWMutex\n\tlleafs []*client\n\tleafClusters map[string]uint64\n\timports importMap\n\texports exportMap\n\tjs *jsAccount\n\tjsLimits map[string]JetStreamAccountLimits\n\tnrgAccount string\n\tlimits\n\texpired atomic.Bool\n\tincomplete bool\n\tsigningKeys map[string]jwt.Scope\n\textAuth *jwt.ExternalAuthorization\n\tsrv *Server // server this account is registered with (possibly nil)\n\tlds string // loop detection subject for leaf nodes\n\tsiReply []byte // service reply prefix, will form wildcard subscription.\n\teventIds *nuid.NUID\n\teventIdsMu sync.Mutex\n\tdefaultPerms *Permissions\n\ttags jwt.TagList\n\tnameTag string\n\tlastLimErr int64\n\troutePoolIdx int\n\t// If the trace destination is specified and a message with a traceParentHdr\n\t// is received, and has the least significant bit of the last token set to 1,\n\t// then if traceDestSampling is > 0 and < 100, a random value will be selected\n\t// and if it falls between 0 and that value, message tracing will be triggered.\n\ttraceDest string\n\ttraceDestSampling int\n\t// Guarantee that only one goroutine can be running either checkJetStreamMigrate\n\t// or clearObserverState at a given time for this account to prevent interleaving.\n\tjscmMu sync.Mutex\n}\n\nconst (\n\taccDedicatedRoute = -1\n\taccTransitioningToDedicatedRoute = -2\n)\n\n// Account based limits.\ntype limits struct {\n\tmpay int32\n\tmsubs int32\n\tmconns int32\n\tmleafs int32\n\tdisallowBearer bool\n}\n\n// Used to track remote clients and leafnodes per remote server.\ntype sconns struct {\n\tconns int32\n\tleafs int32\n}\n\n// clampInt64ToInt32 safely converts an int64 limit to int32,\n// clamping values to the [math.MinInt32, math.MaxInt32] range.\nfunc clampInt64ToInt32(v int64) int32 {\n\treturn int32(max(math.MinInt32, min(math.MaxInt32, v)))\n}\n\n// Import stream mapping struct\ntype streamImport struct {\n\tacc *Account\n\tfrom string\n\tto string\n\ttr *subjectTransform\n\trtr *subjectTransform\n\tclaim *jwt.Import\n\tusePub bool\n\tinvalid bool\n\t// This is `allow_trace` and when true and message tracing is happening,\n\t// we will trace egresses past the account boundary, if `false`, we stop\n\t// at the account boundary.\n\tatrc bool\n}\n\nconst ClientInfoHdr = \"Nats-Request-Info\"\n\n// Import service mapping struct\ntype serviceImport struct {\n\tacc *Account\n\tclaim *jwt.Import\n\tse *serviceExport\n\tsid []byte\n\tfrom string\n\tto string\n\ttr *subjectTransform\n\tts int64\n\trt ServiceRespType\n\tlatency *serviceLatency\n\tm1 *ServiceLatency\n\trc *client\n\tusePub bool\n\tresponse bool\n\tinvalid bool\n\tshare bool\n\ttracking bool\n\tdidDeliver bool\n\tatrc bool // allow trace (got from service export)\n\ttrackingHdr http.Header // header from request\n}\n\n// This is used to record when we create a mapping for implicit service\n// imports. We use this to clean up entries that are not singletons when\n// we detect that interest is no longer present. The key to the map will\n// be the actual interest. We record the mapped subject and the account.\ntype serviceRespEntry struct {\n\tacc *Account\n\tmsub string\n}\n\n// ServiceRespType represents the types of service request response types.\ntype ServiceRespType uint8\n\n// Service response types. Defaults to a singleton.\nconst (\n\tSingleton ServiceRespType = iota\n\tStreamed\n\tChunked\n)\n\n// String helper.\nfunc (rt ServiceRespType) String() string {\n\tswitch rt {\n\tcase Singleton:\n\t\treturn \"Singleton\"\n\tcase Streamed:\n\t\treturn \"Streamed\"\n\tcase Chunked:\n\t\treturn \"Chunked\"\n\t}\n\treturn \"Unknown ServiceResType\"\n}\n\n// exportAuth holds configured approvals or boolean indicating an\n// auth token is required for import.\ntype exportAuth struct {\n\ttokenReq bool\n\taccountPos uint\n\tapproved map[string]*Account\n\tactsRevoked map[string]int64\n}\n\n// streamExport\ntype streamExport struct {\n\texportAuth\n}\n\n// serviceExport holds additional information for exported services.\ntype serviceExport struct {\n\texportAuth\n\tacc *Account\n\trespType ServiceRespType\n\tlatency *serviceLatency\n\trtmr *time.Timer\n\trespThresh time.Duration\n\t// This is `allow_trace` and when true and message tracing is happening,\n\t// when processing a service import we will go through account boundary\n\t// and trace egresses on that other account. If `false`, we stop at the\n\t// account boundary.\n\tatrc bool\n}\n\n// Used to track service latency.\ntype serviceLatency struct {\n\tsampling int8 // percentage from 1-100 or 0 to indicate triggered by header\n\tsubject string\n}\n\n// exportMap tracks the exported streams and services.\ntype exportMap struct {\n\tstreams map[string]*streamExport\n\tservices map[string]*serviceExport\n\tresponses map[string]*serviceImport\n}\n\n// importMap tracks the imported streams and services.\n// For services we will also track the response mappings as well.\ntype importMap struct {\n\tstreams []*streamImport\n\tservices map[string][]*serviceImport\n\trrMap map[string][]*serviceRespEntry\n}\n\n// NewAccount creates a new unlimited account with the given name.\nfunc NewAccount(name string) *Account {\n\ta := &Account{\n\t\tName: name,\n\t\tlimits: limits{-1, -1, -1, -1, false},\n\t\teventIds: nuid.New(),\n\t}\n\treturn a\n}\n\nfunc (a *Account) String() string {\n\treturn a.Name\n}\n\nfunc (a *Account) setTraceDest(dest string) {\n\ta.mu.Lock()\n\ta.traceDest = dest\n\ta.mu.Unlock()\n}\n\nfunc (a *Account) getTraceDestAndSampling() (string, int) {\n\ta.mu.RLock()\n\tdest := a.traceDest\n\tsampling := a.traceDestSampling\n\ta.mu.RUnlock()\n\treturn dest, sampling\n}\n\n// Used to create shallow copies of accounts for transfer\n// from opts to real accounts in server struct.\n// Account `na` write lock is expected to be held on entry\n// while account `a` is the one from the Options struct\n// being loaded/reloaded and do not need locking.\nfunc (a *Account) shallowCopy(na *Account) {\n\tna.Nkey = a.Nkey\n\tna.Issuer = a.Issuer\n\tna.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling\n\tna.nrgAccount = a.nrgAccount\n\n\tif a.imports.streams != nil {\n\t\tna.imports.streams = make([]*streamImport, 0, len(a.imports.streams))\n\t\tfor _, v := range a.imports.streams {\n\t\t\tsi := *v\n\t\t\tna.imports.streams = append(na.imports.streams, &si)\n\t\t}\n\t}\n\tif a.imports.services != nil {\n\t\tna.imports.services = make(map[string][]*serviceImport)\n\t\tfor k, v := range a.imports.services {\n\t\t\tsis := make([]*serviceImport, 0, len(v))\n\t\t\tfor _, si := range v {\n\t\t\t\tcsi := *si\n\t\t\t\tsis = append(sis, &csi)\n\t\t\t}\n\t\t\tna.imports.services[k] = sis\n\t\t}\n\t}\n\tif a.exports.streams != nil {\n\t\tna.exports.streams = make(map[string]*streamExport)\n\t\tfor k, v := range a.exports.streams {\n\t\t\tif v != nil {\n\t\t\t\tse := *v\n\t\t\t\tna.exports.streams[k] = &se\n\t\t\t} else {\n\t\t\t\tna.exports.streams[k] = nil\n\t\t\t}\n\t\t}\n\t}\n\tif a.exports.services != nil {\n\t\tna.exports.services = make(map[string]*serviceExport)\n\t\tfor k, v := range a.exports.services {\n\t\t\tif v != nil {\n\t\t\t\tse := *v\n\t\t\t\tna.exports.services[k] = &se\n\t\t\t} else {\n\t\t\t\tna.exports.services[k] = nil\n\t\t\t}\n\t\t}\n\t}\n\tna.mappings = a.mappings\n\tna.hasMapped.Store(len(na.mappings) > 0)\n\n\t// JetStream\n\tna.jsLimits = a.jsLimits\n\t// Server config account limits.\n\tna.limits = a.limits\n}\n\n// nextEventID uses its own lock for better concurrency.\nfunc (a *Account) nextEventID() string {\n\ta.eventIdsMu.Lock()\n\tid := a.eventIds.Next()\n\ta.eventIdsMu.Unlock()\n\treturn id\n}\n\n// Returns a slice of clients stored in the account, or nil if none is present.\n// Lock is held on entry.\nfunc (a *Account) getClientsLocked() []*client {\n\tif len(a.clients) == 0 {\n\t\treturn nil\n\t}\n\tclients := make([]*client, 0, len(a.clients))\n\tfor c := range a.clients {\n\t\tclients = append(clients, c)\n\t}\n\treturn clients\n}\n\n// Returns a slice of clients stored in the account, or nil if none is present.\nfunc (a *Account) getClients() []*client {\n\ta.mu.RLock()\n\tclients := a.getClientsLocked()\n\ta.mu.RUnlock()\n\treturn clients\n}\n\n// Returns a slice of external (non-internal) clients stored in the account, or nil if none is present.\n// Lock is held on entry.\nfunc (a *Account) getExternalClientsLocked() []*client {\n\tif len(a.clients) == 0 {\n\t\treturn nil\n\t}\n\tvar clients []*client\n\tfor c := range a.clients {\n\t\tif !isInternalClient(c.kind) {\n\t\t\tclients = append(clients, c)\n\t\t}\n\t}\n\treturn clients\n}\n\n// Called to track a remote server and connections and leafnodes it\n// has for this account.\nfunc (a *Account) updateRemoteServer(m *AccountNumConns) []*client {\n\ta.mu.Lock()\n\tif a.strack == nil {\n\t\ta.strack = make(map[string]sconns)\n\t}\n\t// This does not depend on receiving all updates since each one is idempotent.\n\t// FIXME(dlc) - We should cleanup when these both go to zero.\n\tprev := a.strack[m.Server.ID]\n\ta.strack[m.Server.ID] = sconns{conns: int32(m.Conns), leafs: int32(m.LeafNodes)}\n\ta.nrclients += int32(m.Conns) - prev.conns\n\ta.nrleafs += int32(m.LeafNodes) - prev.leafs\n\n\tmtce := a.mconns != jwt.NoLimit && (len(a.clients)-int(a.sysclients)+int(a.nrclients) > int(a.mconns))\n\t// If we are over here some have snuck in and we need to rebalance.\n\t// All others will probably be doing the same thing but better to be\n\t// conservative and bit harsh here. Clients will reconnect if we over compensate.\n\tvar clients []*client\n\tif mtce {\n\t\tclients = a.getExternalClientsLocked()\n\n\t\t// Sort in reverse chronological.\n\t\tslices.SortFunc(clients, func(i, j *client) int { return -i.start.Compare(j.start) })\n\t\tover := (len(a.clients) - int(a.sysclients) + int(a.nrclients)) - int(a.mconns)\n\t\tif over < len(clients) {\n\t\t\tclients = clients[:over]\n\t\t}\n\t}\n\t// Now check leafnodes.\n\tmtlce := a.mleafs != jwt.NoLimit && (a.nleafs+a.nrleafs > a.mleafs)\n\tif mtlce {\n\t\t// Take ones from the end.\n\t\ta.lmu.RLock()\n\t\tleafs := a.lleafs\n\t\tover := int(a.nleafs + a.nrleafs - a.mleafs)\n\t\tif over < len(leafs) {\n\t\t\tleafs = leafs[len(leafs)-over:]\n\t\t}\n\t\tclients = append(clients, leafs...)\n\t\ta.lmu.RUnlock()\n\t}\n\ta.mu.Unlock()\n\n\t// If we have exceeded our max clients this will be populated.\n\treturn clients\n}\n\n// Removes tracking for a remote server that has shutdown.\nfunc (a *Account) removeRemoteServer(sid string) {\n\ta.mu.Lock()\n\tif a.strack != nil {\n\t\tprev := a.strack[sid]\n\t\tdelete(a.strack, sid)\n\t\ta.nrclients -= prev.conns\n\t\ta.nrleafs -= prev.leafs\n\t}\n\ta.mu.Unlock()\n}\n\n// When querying for subject interest this is the number of\n// expected responses. We need to actually check that the entry\n// has active connections.\nfunc (a *Account) expectedRemoteResponses() (expected int32) {\n\ta.mu.RLock()\n\tfor _, sc := range a.strack {\n\t\tif sc.conns > 0 || sc.leafs > 0 {\n\t\t\texpected++\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\treturn\n}\n\n// Clears eventing and tracking for this account.\nfunc (a *Account) clearEventing() {\n\ta.mu.Lock()\n\ta.nrclients = 0\n\t// Now clear state\n\tclearTimer(&a.etmr)\n\tclearTimer(&a.ctmr)\n\ta.clients = nil\n\ta.strack = nil\n\ta.mu.Unlock()\n}\n\n// GetName will return the accounts name.\nfunc (a *Account) GetName() string {\n\tif a == nil {\n\t\treturn \"n/a\"\n\t}\n\ta.mu.RLock()\n\tname := a.Name\n\ta.mu.RUnlock()\n\treturn name\n}\n\n// getNameTag will return the name tag or the account name if not set.\nfunc (a *Account) getNameTag() string {\n\tif a == nil {\n\t\treturn _EMPTY_\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn a.getNameTagLocked()\n}\n\n// getNameTagLocked will return the name tag or the account name if not set.\n// Lock should be held.\nfunc (a *Account) getNameTagLocked() string {\n\tif a == nil {\n\t\treturn _EMPTY_\n\t}\n\tnameTag := a.nameTag\n\tif nameTag == _EMPTY_ {\n\t\tnameTag = a.Name\n\t}\n\treturn nameTag\n}\n\n// NumConnections returns active number of clients for this account for\n// all known servers.\nfunc (a *Account) NumConnections() int {\n\ta.mu.RLock()\n\tnc := len(a.clients) - int(a.sysclients) + int(a.nrclients)\n\ta.mu.RUnlock()\n\treturn nc\n}\n\n// NumRemoteConnections returns the number of client or leaf connections that\n// are not on this server.\nfunc (a *Account) NumRemoteConnections() int {\n\ta.mu.RLock()\n\tnc := int(a.nrclients + a.nrleafs)\n\ta.mu.RUnlock()\n\treturn nc\n}\n\n// NumLocalConnections returns active number of clients for this account\n// on this server.\nfunc (a *Account) NumLocalConnections() int {\n\ta.mu.RLock()\n\tnlc := a.numLocalConnections()\n\ta.mu.RUnlock()\n\treturn nlc\n}\n\n// Do not account for the system accounts.\nfunc (a *Account) numLocalConnections() int {\n\treturn len(a.clients) - int(a.sysclients) - int(a.nleafs)\n}\n\n// This is for extended local interest.\n// Lock should not be held.\nfunc (a *Account) numLocalAndLeafConnections() int {\n\ta.mu.RLock()\n\tnlc := len(a.clients) - int(a.sysclients)\n\ta.mu.RUnlock()\n\treturn nlc\n}\n\nfunc (a *Account) numLocalLeafNodes() int {\n\treturn int(a.nleafs)\n}\n\n// MaxTotalConnectionsReached returns if we have reached our limit for number of connections.\nfunc (a *Account) MaxTotalConnectionsReached() bool {\n\tvar mtce bool\n\ta.mu.RLock()\n\tif a.mconns != jwt.NoLimit {\n\t\tmtce = len(a.clients)-int(a.sysclients)+int(a.nrclients) >= int(a.mconns)\n\t}\n\ta.mu.RUnlock()\n\treturn mtce\n}\n\n// MaxActiveConnections return the set limit for the account system\n// wide for total number of active connections.\nfunc (a *Account) MaxActiveConnections() int {\n\ta.mu.RLock()\n\tmconns := int(a.mconns)\n\ta.mu.RUnlock()\n\treturn mconns\n}\n\n// MaxTotalLeafNodesReached returns if we have reached our limit for number of leafnodes.\nfunc (a *Account) MaxTotalLeafNodesReached() bool {\n\ta.mu.RLock()\n\tmtc := a.maxTotalLeafNodesReached()\n\ta.mu.RUnlock()\n\treturn mtc\n}\n\nfunc (a *Account) maxTotalLeafNodesReached() bool {\n\tif a.mleafs != jwt.NoLimit {\n\t\treturn a.nleafs+a.nrleafs >= a.mleafs\n\t}\n\treturn false\n}\n\n// NumLeafNodes returns the active number of local and remote\n// leaf node connections.\nfunc (a *Account) NumLeafNodes() int {\n\ta.mu.RLock()\n\tnln := int(a.nleafs + a.nrleafs)\n\ta.mu.RUnlock()\n\treturn nln\n}\n\n// NumRemoteLeafNodes returns the active number of remote\n// leaf node connections.\nfunc (a *Account) NumRemoteLeafNodes() int {\n\ta.mu.RLock()\n\tnrn := int(a.nrleafs)\n\ta.mu.RUnlock()\n\treturn nrn\n}\n\n// MaxActiveLeafNodes return the set limit for the account system\n// wide for total number of leavenode connections.\n// NOTE: these are tracked separately.\nfunc (a *Account) MaxActiveLeafNodes() int {\n\ta.mu.RLock()\n\tmleafs := int(a.mleafs)\n\ta.mu.RUnlock()\n\treturn mleafs\n}\n\n// RoutedSubs returns how many subjects we would send across a route when first\n// connected or expressing interest. Local client subs.\nfunc (a *Account) RoutedSubs() int {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn len(a.rm)\n}\n\n// TotalSubs returns total number of Subscriptions for this account.\nfunc (a *Account) TotalSubs() int {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif a.sl == nil {\n\t\treturn 0\n\t}\n\treturn int(a.sl.Count())\n}\n\nfunc (a *Account) shouldLogMaxSubErr() bool {\n\tif a == nil {\n\t\treturn true\n\t}\n\ta.mu.RLock()\n\tlast := a.lastLimErr\n\ta.mu.RUnlock()\n\tif now := time.Now().UnixNano(); now-last >= maxSubLimitReportThreshold {\n\t\ta.mu.Lock()\n\t\ta.lastLimErr = now\n\t\ta.mu.Unlock()\n\t\treturn true\n\t}\n\treturn false\n}\n\n// MapDest is for mapping published subjects for clients.\ntype MapDest struct {\n\tSubject string `json:\"subject\"`\n\tWeight uint8 `json:\"weight\"`\n\tCluster string `json:\"cluster,omitempty\"`\n}\n\nfunc NewMapDest(subject string, weight uint8) *MapDest {\n\treturn &MapDest{subject, weight, _EMPTY_}\n}\n\n// destination is for internal representation for a weighted mapped destination.\ntype destination struct {\n\ttr *subjectTransform\n\tweight uint8\n}\n\n// mapping is an internal entry for mapping subjects.\ntype mapping struct {\n\tsrc string\n\twc bool\n\tdests []*destination\n\tcdests map[string][]*destination\n}\n\n// AddMapping adds in a simple route mapping from src subject to dest subject\n// for inbound client messages.\nfunc (a *Account) AddMapping(src, dest string) error {\n\treturn a.AddWeightedMappings(src, NewMapDest(dest, 100))\n}\n\n// AddWeightedMappings will add in a weighted mappings for the destinations.\nfunc (a *Account) AddWeightedMappings(src string, dests ...*MapDest) error {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\n\tif !IsValidSubject(src) {\n\t\treturn ErrBadSubject\n\t}\n\n\tm := &mapping{src: src, wc: subjectHasWildcard(src), dests: make([]*destination, 0, len(dests)+1)}\n\tseen := make(map[string]struct{})\n\n\tvar tw = make(map[string]uint8)\n\tfor _, d := range dests {\n\t\tif _, ok := seen[d.Subject]; ok {\n\t\t\treturn fmt.Errorf(\"duplicate entry for %q\", d.Subject)\n\t\t}\n\t\tseen[d.Subject] = struct{}{}\n\t\tif d.Weight > 100 {\n\t\t\treturn fmt.Errorf(\"individual weights need to be <= 100\")\n\t\t}\n\t\ttw[d.Cluster] += d.Weight\n\t\tif tw[d.Cluster] > 100 {\n\t\t\treturn fmt.Errorf(\"total weight needs to be <= 100\")\n\t\t}\n\t\terr := ValidateMapping(src, d.Subject)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttr, err := NewSubjectTransform(src, d.Subject)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif d.Cluster == _EMPTY_ {\n\t\t\tm.dests = append(m.dests, &destination{tr, d.Weight})\n\t\t} else {\n\t\t\t// We have a cluster scoped filter.\n\t\t\tif m.cdests == nil {\n\t\t\t\tm.cdests = make(map[string][]*destination)\n\t\t\t}\n\t\t\tad := m.cdests[d.Cluster]\n\t\t\tad = append(ad, &destination{tr, d.Weight})\n\t\t\tm.cdests[d.Cluster] = ad\n\t\t}\n\t}\n\n\tprocessDestinations := func(dests []*destination) ([]*destination, error) {\n\t\tvar ltw uint8\n\t\tfor _, d := range dests {\n\t\t\tltw += d.weight\n\t\t}\n\t\t// Auto add in original at weight difference if all entries weight does not total to 100.\n\t\t// Iff the src was not already added in explicitly, meaning they want loss.\n\t\t_, haveSrc := seen[src]\n\t\tif ltw != 100 && !haveSrc {\n\t\t\tdest := src\n\t\t\tif m.wc {\n\t\t\t\t// We need to make the appropriate markers for the wildcards etc.\n\t\t\t\tdest = transformTokenize(dest)\n\t\t\t}\n\t\t\ttr, err := NewSubjectTransform(src, dest)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\taw := 100 - ltw\n\t\t\tif len(dests) == 0 {\n\t\t\t\taw = 100\n\t\t\t}\n\t\t\tdests = append(dests, &destination{tr, aw})\n\t\t}\n\t\tslices.SortFunc(dests, func(i, j *destination) int { return cmp.Compare(i.weight, j.weight) })\n\n\t\tvar lw uint8\n\t\tfor _, d := range dests {\n\t\t\td.weight += lw\n\t\t\tlw = d.weight\n\t\t}\n\t\treturn dests, nil\n\t}\n\n\tvar err error\n\tif m.dests, err = processDestinations(m.dests); err != nil {\n\t\treturn err\n\t}\n\n\t// Option cluster scoped destinations\n\tfor cluster, dests := range m.cdests {\n\t\tif dests, err = processDestinations(dests); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.cdests[cluster] = dests\n\t}\n\n\t// Replace an old one if it exists.\n\tfor i, em := range a.mappings {\n\t\tif em.src == src {\n\t\t\ta.mappings[i] = m\n\t\t\treturn nil\n\t\t}\n\t}\n\t// If we did not replace add to the end.\n\ta.mappings = append(a.mappings, m)\n\ta.hasMapped.Store(len(a.mappings) > 0)\n\n\t// If we have connected leafnodes make sure to update.\n\tif a.nleafs > 0 {\n\t\t// Need to release because lock ordering is client -> account\n\t\ta.mu.Unlock()\n\t\t// Now grab the leaf list lock. We can hold client lock under this one.\n\t\ta.lmu.RLock()\n\t\tfor _, lc := range a.lleafs {\n\t\t\tlc.forceAddToSmap(src)\n\t\t}\n\t\ta.lmu.RUnlock()\n\t\ta.mu.Lock()\n\t}\n\treturn nil\n}\n\n// RemoveMapping will remove an existing mapping.\nfunc (a *Account) RemoveMapping(src string) bool {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\tfor i, m := range a.mappings {\n\t\tif m.src == src {\n\t\t\t// Swap last one into this spot. Its ok to change order.\n\t\t\ta.mappings[i] = a.mappings[len(a.mappings)-1]\n\t\t\ta.mappings[len(a.mappings)-1] = nil // gc\n\t\t\ta.mappings = a.mappings[:len(a.mappings)-1]\n\t\t\ta.hasMapped.Store(len(a.mappings) > 0)\n\t\t\t// If we have connected leafnodes make sure to update.\n\t\t\tif a.nleafs > 0 {\n\t\t\t\t// Need to release because lock ordering is client -> account\n\t\t\t\ta.mu.Unlock()\n\t\t\t\t// Now grab the leaf list lock. We can hold client lock under this one.\n\t\t\t\ta.lmu.RLock()\n\t\t\t\tfor _, lc := range a.lleafs {\n\t\t\t\t\tlc.forceRemoveFromSmap(src)\n\t\t\t\t}\n\t\t\t\ta.lmu.RUnlock()\n\t\t\t\ta.mu.Lock()\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Indicates we have mapping entries.\nfunc (a *Account) hasMappings() bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\treturn a.hasMapped.Load()\n}\n\n// This performs the logic to map to a new dest subject based on mappings.\n// Should only be called from processInboundClientMsg or service import processing.\nfunc (a *Account) selectMappedSubject(dest string) (string, bool) {\n\tif !a.hasMappings() {\n\t\treturn dest, false\n\t}\n\n\ta.mu.RLock()\n\t// In case we have to tokenize for subset matching.\n\ttsa := [32]string{}\n\ttts := tsa[:0]\n\n\tvar m *mapping\n\tfor _, rm := range a.mappings {\n\t\tif !rm.wc && rm.src == dest {\n\t\t\tm = rm\n\t\t\tbreak\n\t\t} else {\n\t\t\t// tokenize and reuse for subset matching.\n\t\t\tif len(tts) == 0 {\n\t\t\t\tstart := 0\n\t\t\t\tsubject := dest\n\t\t\t\tfor i := 0; i < len(subject); i++ {\n\t\t\t\t\tif subject[i] == btsep {\n\t\t\t\t\t\ttts = append(tts, subject[start:i])\n\t\t\t\t\t\tstart = i + 1\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttts = append(tts, subject[start:])\n\t\t\t}\n\t\t\tif isSubsetMatch(tts, rm.src) {\n\t\t\t\tm = rm\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif m == nil {\n\t\ta.mu.RUnlock()\n\t\treturn dest, false\n\t}\n\n\t// The selected destination for the mapping.\n\tvar d *destination\n\tvar ndest string\n\n\tdests := m.dests\n\tif len(m.cdests) > 0 {\n\t\tcn := a.srv.cachedClusterName()\n\t\tdests = m.cdests[cn]\n\t\tif dests == nil {\n\t\t\t// Fallback to main if we do not match the cluster.\n\t\t\tdests = m.dests\n\t\t}\n\t}\n\n\t// Optimize for single entry case.\n\tif len(dests) == 1 && dests[0].weight == 100 {\n\t\td = dests[0]\n\t} else {\n\t\tw := uint8(fastrand.Uint32n(100))\n\t\tfor _, rm := range dests {\n\t\t\tif w < rm.weight {\n\t\t\t\td = rm\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif d != nil {\n\t\tif len(d.tr.dtokmftokindexesargs) == 0 {\n\t\t\tndest = d.tr.dest\n\t\t} else {\n\t\t\tndest = d.tr.TransformTokenizedSubject(tts)\n\t\t}\n\t}\n\n\ta.mu.RUnlock()\n\treturn ndest, true\n}\n\n// SubscriptionInterest returns true if this account has a matching subscription\n// for the given `subject`.\nfunc (a *Account) SubscriptionInterest(subject string) bool {\n\treturn a.Interest(subject) > 0\n}\n\n// Interest returns the number of subscriptions for a given subject that match.\nfunc (a *Account) Interest(subject string) int {\n\tvar nms int\n\ta.mu.RLock()\n\tif a.sl != nil {\n\t\tnp, nq := a.sl.NumInterest(subject)\n\t\tnms = np + nq\n\t}\n\ta.mu.RUnlock()\n\treturn nms\n}\n\n// addClient keeps our accounting of local active clients or leafnodes updated.\n// Returns previous total.\nfunc (a *Account) addClient(c *client) int {\n\ta.mu.Lock()\n\tn := len(a.clients)\n\n\t// Could come here earlier than the account is registered with the server.\n\t// Make sure we can still track clients.\n\tif a.clients == nil {\n\t\ta.clients = make(map[*client]struct{})\n\t}\n\ta.clients[c] = struct{}{}\n\n\t// If we did not add it, we are done\n\tif n == len(a.clients) {\n\t\ta.mu.Unlock()\n\t\treturn n\n\t}\n\tif c.kind != CLIENT && c.kind != LEAF {\n\t\ta.sysclients++\n\t} else if c.kind == LEAF {\n\t\ta.nleafs++\n\t}\n\ta.mu.Unlock()\n\n\t// If we added a new leaf use the list lock and add it to the list.\n\tif c.kind == LEAF {\n\t\ta.lmu.Lock()\n\t\ta.lleafs = append(a.lleafs, c)\n\t\ta.lmu.Unlock()\n\t}\n\n\tif c != nil && c.srv != nil {\n\t\tc.srv.accConnsUpdate(a)\n\t}\n\n\treturn n\n}\n\n// For registering clusters for remote leafnodes.\n// We only register as the hub.\nfunc (a *Account) registerLeafNodeCluster(cluster string) {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\tif a.leafClusters == nil {\n\t\ta.leafClusters = make(map[string]uint64)\n\t}\n\ta.leafClusters[cluster]++\n}\n\n// Check to see if we already have this cluster registered.\nfunc (a *Account) hasLeafNodeCluster(cluster string) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn a.leafClusters[cluster] > 0\n}\n\n// Check to see if this cluster is isolated, meaning the only one.\n// Read Lock should be held.\nfunc (a *Account) isLeafNodeClusterIsolated(cluster string) bool {\n\tif cluster == _EMPTY_ {\n\t\treturn false\n\t}\n\tif len(a.leafClusters) > 1 {\n\t\treturn false\n\t}\n\treturn a.leafClusters[cluster] == uint64(a.nleafs)\n}\n\n// Helper function to remove leaf nodes. If number of leafnodes gets large\n// this may need to be optimized out of linear search but believe number\n// of active leafnodes per account scope to be small and therefore cache friendly.\n// Lock should not be held on general account lock.\nfunc (a *Account) removeLeafNode(c *client) {\n\t// Make sure we hold the list lock as well.\n\ta.lmu.Lock()\n\tdefer a.lmu.Unlock()\n\n\tll := len(a.lleafs)\n\tfor i, l := range a.lleafs {\n\t\tif l == c {\n\t\t\ta.lleafs[i] = a.lleafs[ll-1]\n\t\t\tif ll == 1 {\n\t\t\t\ta.lleafs = nil\n\t\t\t} else {\n\t\t\t\ta.lleafs = a.lleafs[:ll-1]\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// removeClient keeps our accounting of local active clients updated.\nfunc (a *Account) removeClient(c *client) int {\n\ta.mu.Lock()\n\tn := len(a.clients)\n\tdelete(a.clients, c)\n\t// If we did not actually remove it, we are done.\n\tif n == len(a.clients) {\n\t\ta.mu.Unlock()\n\t\treturn n\n\t}\n\tif c.kind != CLIENT && c.kind != LEAF {\n\t\ta.sysclients--\n\t} else if c.kind == LEAF {\n\t\ta.nleafs--\n\t\t// Need to do cluster accounting here.\n\t\t// Do cluster accounting if we are a hub.\n\t\tif c.isHubLeafNode() {\n\t\t\tcluster := c.remoteCluster()\n\t\t\tif count := a.leafClusters[cluster]; count > 1 {\n\t\t\t\ta.leafClusters[cluster]--\n\t\t\t} else if count == 1 {\n\t\t\t\tdelete(a.leafClusters, cluster)\n\t\t\t}\n\t\t}\n\t}\n\ta.mu.Unlock()\n\n\tif c.kind == LEAF {\n\t\ta.removeLeafNode(c)\n\t}\n\n\tif c != nil && c.srv != nil {\n\t\tc.srv.accConnsUpdate(a)\n\t}\n\n\treturn n\n}\n\nfunc setExportAuth(ea *exportAuth, subject string, accounts []*Account, accountPos uint) error {\n\tif accountPos > 0 {\n\t\ttoken := strings.Split(subject, tsep)\n\t\tif len(token) < int(accountPos) || token[accountPos-1] != \"*\" {\n\t\t\treturn ErrInvalidSubject\n\t\t}\n\t}\n\tea.accountPos = accountPos\n\t// empty means auth required but will be import token.\n\tif accounts == nil {\n\t\treturn nil\n\t}\n\tif len(accounts) == 0 {\n\t\tea.tokenReq = true\n\t\treturn nil\n\t}\n\tif ea.approved == nil {\n\t\tea.approved = make(map[string]*Account, len(accounts))\n\t}\n\tfor _, acc := range accounts {\n\t\tea.approved[acc.Name] = acc\n\t}\n\treturn nil\n}\n\n// AddServiceExport will configure the account with the defined export.\nfunc (a *Account) AddServiceExport(subject string, accounts []*Account) error {\n\treturn a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, 0)\n}\n\n// AddServiceExport will configure the account with the defined export.\nfunc (a *Account) addServiceExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {\n\treturn a.addServiceExportWithResponseAndAccountPos(subject, Singleton, accounts, accountPos)\n}\n\n// AddServiceExportWithResponse will configure the account with the defined export and response type.\nfunc (a *Account) AddServiceExportWithResponse(subject string, respType ServiceRespType, accounts []*Account) error {\n\treturn a.addServiceExportWithResponseAndAccountPos(subject, respType, accounts, 0)\n}\n\n// AddServiceExportWithresponse will configure the account with the defined export and response type.\nfunc (a *Account) addServiceExportWithResponseAndAccountPos(\n\tsubject string, respType ServiceRespType, accounts []*Account, accountPos uint) error {\n\tif a == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\ta.mu.Lock()\n\tif a.exports.services == nil {\n\t\ta.exports.services = make(map[string]*serviceExport)\n\t}\n\n\tse := a.exports.services[subject]\n\t// Always create a service export\n\tif se == nil {\n\t\tse = &serviceExport{}\n\t}\n\n\tif respType != Singleton {\n\t\tse.respType = respType\n\t}\n\n\tif accounts != nil || accountPos > 0 {\n\t\tif err := setExportAuth(&se.exportAuth, subject, accounts, accountPos); err != nil {\n\t\t\ta.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\tlrt := a.lowestServiceExportResponseTime()\n\tse.acc = a\n\tse.respThresh = DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD\n\ta.exports.services[subject] = se\n\n\tvar clients []*client\n\tnlrt := a.lowestServiceExportResponseTime()\n\tif nlrt != lrt && len(a.clients) > 0 {\n\t\tclients = a.getClientsLocked()\n\t}\n\t// Need to release because lock ordering is client -> Account\n\ta.mu.Unlock()\n\tif len(clients) > 0 {\n\t\tupdateAllClientsServiceExportResponseTime(clients, nlrt)\n\t}\n\treturn nil\n}\n\n// TrackServiceExport will enable latency tracking of the named service.\n// Results will be published in this account to the given results subject.\nfunc (a *Account) TrackServiceExport(service, results string) error {\n\treturn a.TrackServiceExportWithSampling(service, results, DEFAULT_SERVICE_LATENCY_SAMPLING)\n}\n\n// TrackServiceExportWithSampling will enable latency tracking of the named service for the given\n// sampling rate (1-100). Results will be published in this account to the given results subject.\nfunc (a *Account) TrackServiceExportWithSampling(service, results string, sampling int) error {\n\tif a == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\tif sampling != 0 { // 0 means triggered by header\n\t\tif sampling < 1 || sampling > 100 {\n\t\t\treturn ErrBadSampling\n\t\t}\n\t}\n\tif !IsValidPublishSubject(results) {\n\t\treturn ErrBadPublishSubject\n\t}\n\t// Don't loop back on outselves.\n\tif a.IsExportService(results) {\n\t\treturn ErrBadPublishSubject\n\t}\n\n\tif a.srv != nil && !a.srv.EventsEnabled() {\n\t\treturn ErrNoSysAccount\n\t}\n\n\ta.mu.Lock()\n\tif a.exports.services == nil {\n\t\ta.mu.Unlock()\n\t\treturn ErrMissingService\n\t}\n\tea, ok := a.exports.services[service]\n\tif !ok {\n\t\ta.mu.Unlock()\n\t\treturn ErrMissingService\n\t}\n\tif ea == nil {\n\t\tea = &serviceExport{}\n\t\ta.exports.services[service] = ea\n\t} else if ea.respType != Singleton {\n\t\ta.mu.Unlock()\n\t\treturn ErrBadServiceType\n\t}\n\tea.latency = &serviceLatency{\n\t\tsampling: int8(sampling),\n\t\tsubject: results,\n\t}\n\ts := a.srv\n\ta.mu.Unlock()\n\n\tif s == nil {\n\t\treturn nil\n\t}\n\n\t// Now track down the imports and add in latency as needed to enable.\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\tacc.mu.Lock()\n\t\tfor _, ims := range acc.imports.services {\n\t\t\tfor _, im := range ims {\n\t\t\t\tif im != nil && im.acc.Name == a.Name && subjectIsSubsetMatch(im.to, service) {\n\t\t\t\t\tim.latency = ea.latency\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tacc.mu.Unlock()\n\t\treturn true\n\t})\n\n\treturn nil\n}\n\n// UnTrackServiceExport will disable latency tracking of the named service.\nfunc (a *Account) UnTrackServiceExport(service string) {\n\tif a == nil || (a.srv != nil && !a.srv.EventsEnabled()) {\n\t\treturn\n\t}\n\n\ta.mu.Lock()\n\tif a.exports.services == nil {\n\t\ta.mu.Unlock()\n\t\treturn\n\t}\n\tea, ok := a.exports.services[service]\n\tif !ok || ea == nil || ea.latency == nil {\n\t\ta.mu.Unlock()\n\t\treturn\n\t}\n\t// We have latency here.\n\tea.latency = nil\n\ts := a.srv\n\ta.mu.Unlock()\n\n\tif s == nil {\n\t\treturn\n\t}\n\n\t// Now track down the imports and clean them up.\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\tacc.mu.Lock()\n\t\tfor _, ims := range acc.imports.services {\n\t\t\tfor _, im := range ims {\n\t\t\t\tif im != nil && im.acc.Name == a.Name {\n\t\t\t\t\tif subjectIsSubsetMatch(im.to, service) {\n\t\t\t\t\t\tim.latency, im.m1 = nil, nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tacc.mu.Unlock()\n\t\treturn true\n\t})\n}\n\n// IsExportService will indicate if this service exists. Will check wildcard scenarios.\nfunc (a *Account) IsExportService(service string) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\t_, ok := a.exports.services[service]\n\tif ok {\n\t\treturn true\n\t}\n\ttokens := strings.Split(service, tsep)\n\tfor subj := range a.exports.services {\n\t\tif isSubsetMatch(tokens, subj) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// IsExportServiceTracking will indicate if given publish subject is an export service with tracking enabled.\nfunc (a *Account) IsExportServiceTracking(service string) bool {\n\ta.mu.RLock()\n\tea, ok := a.exports.services[service]\n\tif ok && ea == nil {\n\t\ta.mu.RUnlock()\n\t\treturn false\n\t}\n\tif ok && ea != nil && ea.latency != nil {\n\t\ta.mu.RUnlock()\n\t\treturn true\n\t}\n\t// FIXME(dlc) - Might want to cache this is in the hot path checking for latency tracking.\n\ttokens := strings.Split(service, tsep)\n\tfor subj, ea := range a.exports.services {\n\t\tif isSubsetMatch(tokens, subj) && ea != nil && ea.latency != nil {\n\t\t\ta.mu.RUnlock()\n\t\t\treturn true\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\treturn false\n}\n\n// ServiceLatency is the JSON message sent out in response to latency tracking for\n// an accounts exported services. Additional client info is available in requestor\n// and responder. Note that for a requestor, the only information shared by default\n// is the RTT used to calculate the total latency. The requestor's account can\n// designate to share the additional information in the service import.\ntype ServiceLatency struct {\n\tTypedEvent\n\tStatus int `json:\"status\"`\n\tError string `json:\"description,omitempty\"`\n\tRequestor *ClientInfo `json:\"requestor,omitempty\"`\n\tResponder *ClientInfo `json:\"responder,omitempty\"`\n\tRequestHeader http.Header `json:\"header,omitempty\"` // only contains header(s) triggering the measurement\n\tRequestStart time.Time `json:\"start\"`\n\tServiceLatency time.Duration `json:\"service\"`\n\tSystemLatency time.Duration `json:\"system\"`\n\tTotalLatency time.Duration `json:\"total\"`\n}\n\n// ServiceLatencyType is the NATS Event Type for ServiceLatency\nconst ServiceLatencyType = \"io.nats.server.metric.v1.service_latency\"\n\n// NATSTotalTime is a helper function that totals the NATS latencies.\nfunc (m1 *ServiceLatency) NATSTotalTime() time.Duration {\n\treturn m1.Requestor.RTT + m1.Responder.RTT + m1.SystemLatency\n}\n\n// Merge function to merge m1 and m2 (requestor and responder) measurements\n// when there are two samples. This happens when the requestor and responder\n// are on different servers.\n//\n// m2 ServiceLatency is correct, so use that.\n// m1 TotalLatency is correct, so use that.\n// Will use those to back into NATS latency.\nfunc (m1 *ServiceLatency) merge(m2 *ServiceLatency) {\n\trtt := time.Duration(0)\n\tif m2.Responder != nil {\n\t\trtt = m2.Responder.RTT\n\t}\n\tm1.SystemLatency = m1.ServiceLatency - (m2.ServiceLatency + rtt)\n\tm1.ServiceLatency = m2.ServiceLatency\n\tm1.Responder = m2.Responder\n\tsanitizeLatencyMetric(m1)\n}\n\n// sanitizeLatencyMetric adjusts latency metric values that could go\n// negative in some edge conditions since we estimate client RTT\n// for both requestor and responder.\n// These numbers are never meant to be negative, it just could be\n// how we back into the values based on estimated RTT.\nfunc sanitizeLatencyMetric(sl *ServiceLatency) {\n\tif sl.ServiceLatency < 0 {\n\t\tsl.ServiceLatency = 0\n\t}\n\tif sl.SystemLatency < 0 {\n\t\tsl.SystemLatency = 0\n\t}\n}\n\n// Used for transporting remote latency measurements.\ntype remoteLatency struct {\n\tAccount string `json:\"account\"`\n\tReqId string `json:\"req_id\"`\n\tM2 ServiceLatency `json:\"m2\"`\n\trespThresh time.Duration\n}\n\n// sendLatencyResult will send a latency result and clear the si of the requestor(rc).\nfunc (a *Account) sendLatencyResult(si *serviceImport, sl *ServiceLatency) {\n\tsl.Type = ServiceLatencyType\n\tsl.ID = a.nextEventID()\n\tsl.Time = time.Now().UTC()\n\ta.mu.Lock()\n\tlsubj := si.latency.subject\n\tsi.rc = nil\n\ta.mu.Unlock()\n\n\ta.srv.sendInternalAccountMsg(a, lsubj, sl)\n}\n\n// Used to send a bad request metric when we do not have a reply subject\nfunc (a *Account) sendBadRequestTrackingLatency(si *serviceImport, requestor *client, header http.Header) {\n\tsl := &ServiceLatency{\n\t\tStatus: 400,\n\t\tError: \"Bad Request\",\n\t\tRequestor: requestor.getClientInfo(si.share),\n\t}\n\tsl.RequestHeader = header\n\tsl.RequestStart = time.Now().Add(-sl.Requestor.RTT).UTC()\n\ta.sendLatencyResult(si, sl)\n}\n\n// Used to send a latency result when the requestor interest was lost before the\n// response could be delivered.\nfunc (a *Account) sendReplyInterestLostTrackLatency(si *serviceImport) {\n\tsl := &ServiceLatency{\n\t\tStatus: 408,\n\t\tError: \"Request Timeout\",\n\t}\n\ta.mu.RLock()\n\trc, share, ts := si.rc, si.share, si.ts\n\tsl.RequestHeader = si.trackingHdr\n\ta.mu.RUnlock()\n\tif rc != nil {\n\t\tsl.Requestor = rc.getClientInfo(share)\n\t}\n\tsl.RequestStart = time.Unix(0, ts-int64(sl.Requestor.RTT)).UTC()\n\ta.sendLatencyResult(si, sl)\n}\n\nfunc (a *Account) sendBackendErrorTrackingLatency(si *serviceImport, reason rsiReason) {\n\tsl := &ServiceLatency{}\n\ta.mu.RLock()\n\trc, share, ts := si.rc, si.share, si.ts\n\tsl.RequestHeader = si.trackingHdr\n\ta.mu.RUnlock()\n\tif rc != nil {\n\t\tsl.Requestor = rc.getClientInfo(share)\n\t}\n\tvar reqRTT time.Duration\n\tif sl.Requestor != nil {\n\t\treqRTT = sl.Requestor.RTT\n\t}\n\tsl.RequestStart = time.Unix(0, ts-int64(reqRTT)).UTC()\n\tif reason == rsiNoDelivery {\n\t\tsl.Status = 503\n\t\tsl.Error = \"Service Unavailable\"\n\t} else if reason == rsiTimeout {\n\t\tsl.Status = 504\n\t\tsl.Error = \"Service Timeout\"\n\t}\n\ta.sendLatencyResult(si, sl)\n}\n\n// sendTrackingLatency will send out the appropriate tracking information for the\n// service request/response latency. This is called when the requestor's server has\n// received the response.\n// TODO(dlc) - holding locks for RTTs may be too much long term. Should revisit.\nfunc (a *Account) sendTrackingLatency(si *serviceImport, responder *client) bool {\n\ta.mu.RLock()\n\trc := si.rc\n\ta.mu.RUnlock()\n\tif rc == nil {\n\t\treturn true\n\t}\n\n\tts := time.Now()\n\tserviceRTT := time.Duration(ts.UnixNano() - si.ts)\n\trequestor := si.rc\n\n\tsl := &ServiceLatency{\n\t\tStatus: 200,\n\t\tRequestor: requestor.getClientInfo(si.share),\n\t\tResponder: responder.getClientInfo(true),\n\t}\n\tvar respRTT, reqRTT time.Duration\n\tif sl.Responder != nil {\n\t\trespRTT = sl.Responder.RTT\n\t}\n\tif sl.Requestor != nil {\n\t\treqRTT = sl.Requestor.RTT\n\t}\n\tsl.RequestStart = time.Unix(0, si.ts-int64(reqRTT)).UTC()\n\tsl.ServiceLatency = serviceRTT - respRTT\n\tsl.TotalLatency = reqRTT + serviceRTT\n\tif respRTT > 0 {\n\t\tsl.SystemLatency = time.Since(ts)\n\t\tsl.TotalLatency += sl.SystemLatency\n\t}\n\tsl.RequestHeader = si.trackingHdr\n\tsanitizeLatencyMetric(sl)\n\n\tsl.Type = ServiceLatencyType\n\tsl.ID = a.nextEventID()\n\tsl.Time = time.Now().UTC()\n\n\t// If we are expecting a remote measurement, store our sl here.\n\t// We need to account for the race between this and us receiving the\n\t// remote measurement.\n\t// FIXME(dlc) - We need to clean these up but this should happen\n\t// already with the auto-expire logic.\n\tif responder != nil && responder.kind != CLIENT {\n\t\tsi.acc.mu.Lock()\n\t\tif si.m1 != nil {\n\t\t\tm1, m2 := sl, si.m1\n\t\t\tm1.merge(m2)\n\t\t\tsi.acc.mu.Unlock()\n\t\t\ta.srv.sendInternalAccountMsg(a, si.latency.subject, m1)\n\t\t\ta.mu.Lock()\n\t\t\tsi.rc = nil\n\t\t\ta.mu.Unlock()\n\t\t\treturn true\n\t\t}\n\t\tsi.m1 = sl\n\t\tsi.acc.mu.Unlock()\n\t\treturn false\n\t} else {\n\t\ta.srv.sendInternalAccountMsg(a, si.latency.subject, sl)\n\t\ta.mu.Lock()\n\t\tsi.rc = nil\n\t\ta.mu.Unlock()\n\t}\n\treturn true\n}\n\n// This will check to make sure our response lower threshold is set\n// properly in any clients doing rrTracking.\nfunc updateAllClientsServiceExportResponseTime(clients []*client, lrt time.Duration) {\n\tfor _, c := range clients {\n\t\tc.mu.Lock()\n\t\tif c.rrTracking != nil && lrt != c.rrTracking.lrt {\n\t\t\tc.rrTracking.lrt = lrt\n\t\t\tif c.rrTracking.ptmr.Stop() {\n\t\t\t\tc.rrTracking.ptmr.Reset(lrt)\n\t\t\t}\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n}\n\n// Will select the lowest respThresh from all service exports.\n// Read lock should be held.\nfunc (a *Account) lowestServiceExportResponseTime() time.Duration {\n\t// Lowest we will allow is 5 minutes. Its an upper bound for this function.\n\tlrt := 5 * time.Minute\n\tfor _, se := range a.exports.services {\n\t\tif se.respThresh < lrt {\n\t\t\tlrt = se.respThresh\n\t\t}\n\t}\n\treturn lrt\n}\n\n// AddServiceImportWithClaim will add in the service import via the jwt claim.\nfunc (a *Account) AddServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import) error {\n\treturn a.addServiceImportWithClaim(destination, from, to, imClaim, false)\n}\n\n// addServiceImportWithClaim will add in the service import via the jwt claim.\n// It will also skip the authorization check in cases where internal is true\nfunc (a *Account) addServiceImportWithClaim(destination *Account, from, to string, imClaim *jwt.Import, internal bool) error {\n\tif destination == nil {\n\t\treturn ErrMissingAccount\n\t}\n\t// Empty means use from.\n\tif to == _EMPTY_ {\n\t\tto = from\n\t}\n\tif !IsValidSubject(from) || !IsValidSubject(to) {\n\t\treturn ErrInvalidSubject\n\t}\n\n\t// First check to see if the account has authorized us to route to the \"to\" subject.\n\tif !internal && !destination.checkServiceImportAuthorized(a, to, imClaim) {\n\t\treturn ErrServiceImportAuthorization\n\t}\n\n\t// Check if this introduces a cycle before proceeding.\n\t// From will be the mapped subject.\n\t// If the 'to' has a wildcard make sure we pre-transform the 'from' before we check for cycles, e.g. '$1'\n\tfromT := from\n\tif subjectHasWildcard(to) {\n\t\tfromT, _ = transformUntokenize(from)\n\t}\n\tif err := a.serviceImportFormsCycle(destination, fromT); err != nil {\n\t\treturn err\n\t}\n\n\t_, err := a.addServiceImport(destination, from, to, imClaim)\n\n\treturn err\n}\n\nconst MaxAccountCycleSearchDepth = 1024\n\nfunc (a *Account) serviceImportFormsCycle(dest *Account, from string) error {\n\treturn dest.checkServiceImportsForCycles(from, map[string]bool{a.Name: true})\n}\n\nfunc (a *Account) checkServiceImportsForCycles(from string, visited map[string]bool) error {\n\tif len(visited) >= MaxAccountCycleSearchDepth {\n\t\treturn ErrCycleSearchDepth\n\t}\n\ta.mu.RLock()\n\tfor _, sis := range a.imports.services {\n\t\tfor _, si := range sis {\n\t\t\tif SubjectsCollide(from, si.to) {\n\t\t\t\ta.mu.RUnlock()\n\t\t\t\tif visited[si.acc.Name] {\n\t\t\t\t\treturn ErrImportFormsCycle\n\t\t\t\t}\n\t\t\t\t// Push ourselves and check si.acc\n\t\t\t\tvisited[a.Name] = true\n\t\t\t\t// Make a copy to not overwrite the passed value.\n\t\t\t\tf := from\n\t\t\t\tif subjectIsSubsetMatch(si.from, f) {\n\t\t\t\t\tf = si.from\n\t\t\t\t}\n\t\t\t\tif err := si.acc.checkServiceImportsForCycles(f, visited); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\ta.mu.RLock()\n\t\t\t}\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\treturn nil\n}\n\nfunc (a *Account) streamImportFormsCycle(dest *Account, to string) error {\n\treturn dest.checkStreamImportsForCycles(to, map[string]bool{a.Name: true})\n}\n\n// Lock should be held.\nfunc (a *Account) hasServiceExportMatching(to string) bool {\n\tfor subj := range a.exports.services {\n\t\tif subjectIsSubsetMatch(to, subj) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Lock should be held.\nfunc (a *Account) hasStreamExportMatching(to string) bool {\n\tfor subj := range a.exports.streams {\n\t\tif subjectIsSubsetMatch(to, subj) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool) error {\n\tif len(visited) >= MaxAccountCycleSearchDepth {\n\t\treturn ErrCycleSearchDepth\n\t}\n\n\ta.mu.RLock()\n\n\tif !a.hasStreamExportMatching(to) {\n\t\ta.mu.RUnlock()\n\t\treturn nil\n\t}\n\n\tfor _, si := range a.imports.streams {\n\t\tif SubjectsCollide(to, si.to) {\n\t\t\ta.mu.RUnlock()\n\t\t\tif visited[si.acc.Name] {\n\t\t\t\treturn ErrImportFormsCycle\n\t\t\t}\n\t\t\t// Push ourselves and check si.acc\n\t\t\tvisited[a.Name] = true\n\t\t\t// Make a copy to not overwrite the passed value.\n\t\t\tt := to\n\t\t\tif subjectIsSubsetMatch(si.to, t) {\n\t\t\t\tt = si.to\n\t\t\t}\n\t\t\tif err := si.acc.checkStreamImportsForCycles(t, visited); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ta.mu.RLock()\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\treturn nil\n}\n\n// SetServiceImportSharing will allow sharing of information about requests with the export account.\n// Used for service latency tracking at the moment.\nfunc (a *Account) SetServiceImportSharing(destination *Account, to string, allow bool) error {\n\treturn a.setServiceImportSharing(destination, to, true, allow)\n}\n\n// setServiceImportSharing will allow sharing of information about requests with the export account.\nfunc (a *Account) setServiceImportSharing(destination *Account, to string, check, allow bool) error {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\tif check && a.isClaimAccount() {\n\t\treturn fmt.Errorf(\"claim based accounts can not be updated directly\")\n\t}\n\t// We can't use getServiceImportForAccountLocked() here since we are looking\n\t// for the service import with the si.to == to, which may not be the key\n\t// for the service import in the map.\n\tfor _, sis := range a.imports.services {\n\t\tfor _, si := range sis {\n\t\t\tif si.acc.Name == destination.Name && si.to == to {\n\t\t\t\tsi.share = allow\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\treturn fmt.Errorf(\"service import not found\")\n}\n\n// AddServiceImport will add a route to an account to send published messages / requests\n// to the destination account. From is the local subject to map, To is the\n// subject that will appear on the destination account. Destination will need\n// to have an import rule to allow access via addService.\nfunc (a *Account) AddServiceImport(destination *Account, from, to string) error {\n\treturn a.AddServiceImportWithClaim(destination, from, to, nil)\n}\n\n// NumPendingReverseResponses returns the number of response mappings we have for all outstanding\n// requests for service imports.\nfunc (a *Account) NumPendingReverseResponses() int {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn len(a.imports.rrMap)\n}\n\n// NumPendingAllResponses return the number of all responses outstanding for service exports.\nfunc (a *Account) NumPendingAllResponses() int {\n\treturn a.NumPendingResponses(_EMPTY_)\n}\n\n// NumPendingResponses returns the number of responses outstanding for service exports\n// on this account. An empty filter string returns all responses regardless of which export.\n// If you specify the filter we will only return ones that are for that export.\n// NOTE this is only for what this server is tracking.\nfunc (a *Account) NumPendingResponses(filter string) int {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif filter == _EMPTY_ {\n\t\treturn len(a.exports.responses)\n\t}\n\tse := a.getServiceExport(filter)\n\tif se == nil {\n\t\treturn 0\n\t}\n\tvar nre int\n\tfor _, si := range a.exports.responses {\n\t\tif si.se == se {\n\t\t\tnre++\n\t\t}\n\t}\n\treturn nre\n}\n\n// NumServiceImports returns the number of service imports we have configured.\nfunc (a *Account) NumServiceImports() int {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn len(a.imports.services)\n}\n\n// Reason why we are removing this response serviceImport.\ntype rsiReason int\n\nconst (\n\trsiOk = rsiReason(iota)\n\trsiNoDelivery\n\trsiTimeout\n)\n\n// removeRespServiceImport removes a response si mapping and the reverse entries for interest detection.\nfunc (a *Account) removeRespServiceImport(si *serviceImport, reason rsiReason) {\n\tif si == nil {\n\t\treturn\n\t}\n\n\ta.mu.Lock()\n\tc := a.ic\n\tdelete(a.exports.responses, si.from)\n\tdest, to, tracking, rc, didDeliver := si.acc, si.to, si.tracking, si.rc, si.didDeliver\n\ta.mu.Unlock()\n\n\t// If we have a sid make sure to unsub.\n\tif len(si.sid) > 0 && c != nil {\n\t\tc.processUnsub(si.sid)\n\t}\n\n\tif tracking && rc != nil && !didDeliver {\n\t\ta.sendBackendErrorTrackingLatency(si, reason)\n\t}\n\n\tdest.checkForReverseEntry(to, si, false)\n}\n\nfunc (a *Account) getServiceImportForAccountLocked(dstAccName, subject string) *serviceImport {\n\tsis, ok := a.imports.services[subject]\n\tif !ok {\n\t\treturn nil\n\t}\n\tif len(sis) == 1 && sis[0].acc.Name == dstAccName {\n\t\treturn sis[0]\n\t}\n\tfor _, si := range sis {\n\t\tif si.acc.Name == dstAccName {\n\t\t\treturn si\n\t\t}\n\t}\n\treturn nil\n}\n\n// removeServiceImport will remove the route by subject.\nfunc (a *Account) removeServiceImport(dstAccName, subject string) {\n\ta.mu.Lock()\n\tsis, ok := a.imports.services[subject]\n\tif !ok {\n\t\ta.mu.Unlock()\n\t\treturn\n\t}\n\tvar si *serviceImport\n\tif len(sis) == 1 {\n\t\tsi = sis[0]\n\t\tif si.acc.Name != dstAccName {\n\t\t\tsi = nil\n\t\t} else {\n\t\t\tdelete(a.imports.services, subject)\n\t\t}\n\t} else {\n\t\tfor i, esi := range sis {\n\t\t\tif esi.acc.Name == dstAccName {\n\t\t\t\tsi = esi\n\t\t\t\tlast := len(sis) - 1\n\t\t\t\tif i != last {\n\t\t\t\t\tsis[i] = sis[last]\n\t\t\t\t}\n\t\t\t\tsis = sis[:last]\n\t\t\t\ta.imports.services[subject] = sis\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif si == nil {\n\t\ta.mu.Unlock()\n\t\treturn\n\t}\n\tvar sid []byte\n\tc := a.ic\n\tif c != nil && si.sid != nil {\n\t\tsid = si.sid\n\t}\n\ta.mu.Unlock()\n\n\tif sid != nil {\n\t\tc.processUnsub(sid)\n\t}\n}\n\n// This tracks responses to service requests mappings. This is used for cleanup.\nfunc (a *Account) addReverseRespMapEntry(acc *Account, reply, from string) {\n\ta.mu.Lock()\n\tif a.imports.rrMap == nil {\n\t\ta.imports.rrMap = make(map[string][]*serviceRespEntry)\n\t}\n\tsre := &serviceRespEntry{acc, from}\n\tsra := a.imports.rrMap[reply]\n\ta.imports.rrMap[reply] = append(sra, sre)\n\ta.mu.Unlock()\n}\n\n// checkForReverseEntries is for when we are trying to match reverse entries to a wildcard.\n// This will be called from checkForReverseEntry when the reply arg is a wildcard subject.\n// This will usually be called in a go routine since we need to walk all the entries.\nfunc (a *Account) checkForReverseEntries(reply string, checkInterest, recursed bool) {\n\tif subjectIsLiteral(reply) {\n\t\ta._checkForReverseEntry(reply, nil, checkInterest, recursed)\n\t\treturn\n\t}\n\n\ta.mu.RLock()\n\tif len(a.imports.rrMap) == 0 {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\n\tvar _rs [64]string\n\trs := _rs[:0]\n\tif n := len(a.imports.rrMap); n > cap(rs) {\n\t\trs = make([]string, 0, n)\n\t}\n\n\tfor k := range a.imports.rrMap {\n\t\trs = append(rs, k)\n\t}\n\ta.mu.RUnlock()\n\n\ttsa := [32]string{}\n\ttts := tokenizeSubjectIntoSlice(tsa[:0], reply)\n\n\trsa := [32]string{}\n\tfor _, r := range rs {\n\t\trts := tokenizeSubjectIntoSlice(rsa[:0], r)\n\t\t// isSubsetMatchTokenized is heavy so make sure we do this without the lock.\n\t\tif isSubsetMatchTokenized(rts, tts) {\n\t\t\ta._checkForReverseEntry(r, nil, checkInterest, recursed)\n\t\t}\n\t}\n}\n\n// This checks for any response map entries. If you specify an si we will only match and\n// clean up for that one, otherwise we remove them all.\nfunc (a *Account) checkForReverseEntry(reply string, si *serviceImport, checkInterest bool) {\n\ta._checkForReverseEntry(reply, si, checkInterest, false)\n}\n\n// Callers should use checkForReverseEntry instead. This function exists to help prevent\n// infinite recursion.\nfunc (a *Account) _checkForReverseEntry(reply string, si *serviceImport, checkInterest, recursed bool) {\n\ta.mu.RLock()\n\tif len(a.imports.rrMap) == 0 {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\n\tif subjectHasWildcard(reply) {\n\t\tif recursed {\n\t\t\t// If we have reached this condition then it is because the reverse entries also\n\t\t\t// contain wildcards (that shouldn't happen but a client *could* provide an inbox\n\t\t\t// prefix that is illegal because it ends in a wildcard character), at which point\n\t\t\t// we will end up with infinite recursion between this func and checkForReverseEntries.\n\t\t\t// To avoid a stack overflow panic, we'll give up instead.\n\t\t\ta.mu.RUnlock()\n\t\t\treturn\n\t\t}\n\n\t\tdoInline := len(a.imports.rrMap) <= 64\n\t\ta.mu.RUnlock()\n\n\t\tif doInline {\n\t\t\ta.checkForReverseEntries(reply, checkInterest, true)\n\t\t} else {\n\t\t\tgo a.checkForReverseEntries(reply, checkInterest, true)\n\t\t}\n\t\treturn\n\t}\n\n\tif sres := a.imports.rrMap[reply]; sres == nil {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\n\t// If we are here we have an entry we should check.\n\t// If requested we will first check if there is any\n\t// interest for this subject for the entire account.\n\t// If there is we can not delete any entries yet.\n\t// Note that if we are here reply has to be a literal subject.\n\tif checkInterest {\n\t\t// If interest still exists we can not clean these up yet.\n\t\tif a.sl.HasInterest(reply) {\n\t\t\ta.mu.RUnlock()\n\t\t\treturn\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\n\t// Delete the appropriate entries here based on optional si.\n\ta.mu.Lock()\n\t// We need a new lookup here because we have released the lock.\n\tsres := a.imports.rrMap[reply]\n\tif si == nil {\n\t\tdelete(a.imports.rrMap, reply)\n\t} else if sres != nil {\n\t\t// Find the one we are looking for..\n\t\tfor i, sre := range sres {\n\t\t\tif sre.msub == si.from {\n\t\t\t\tsres = append(sres[:i], sres[i+1:]...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif len(sres) > 0 {\n\t\t\ta.imports.rrMap[si.to] = sres\n\t\t} else {\n\t\t\tdelete(a.imports.rrMap, si.to)\n\t\t}\n\t}\n\ta.mu.Unlock()\n\n\t// If we are here we no longer have interest and we have\n\t// response entries that we should clean up.\n\tif si == nil {\n\t\t// sres is now known to have been removed from a.imports.rrMap, so we\n\t\t// can safely (data race wise) iterate through.\n\t\tfor _, sre := range sres {\n\t\t\tacc := sre.acc\n\t\t\tvar trackingCleanup bool\n\t\t\tvar rsi *serviceImport\n\t\t\tacc.mu.Lock()\n\t\t\tc := acc.ic\n\t\t\tif rsi = acc.exports.responses[sre.msub]; rsi != nil && !rsi.didDeliver {\n\t\t\t\tdelete(acc.exports.responses, rsi.from)\n\t\t\t\ttrackingCleanup = rsi.tracking && rsi.rc != nil\n\t\t\t}\n\t\t\tacc.mu.Unlock()\n\t\t\t// If we are doing explicit subs for all responses (e.g. bound to leafnode)\n\t\t\t// we will have a non-empty sid here.\n\t\t\tif rsi != nil && len(rsi.sid) > 0 && c != nil {\n\t\t\t\tc.processUnsub(rsi.sid)\n\t\t\t}\n\t\t\tif trackingCleanup {\n\t\t\t\tacc.sendReplyInterestLostTrackLatency(rsi)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Checks to see if a potential service import subject is already overshadowed.\nfunc (a *Account) serviceImportShadowed(from string) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif a.imports.services[from] != nil {\n\t\treturn true\n\t}\n\t// We did not find a direct match, so check individually.\n\tfor subj := range a.imports.services {\n\t\tif subjectIsSubsetMatch(from, subj) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Internal check to see if a service import exists.\nfunc (a *Account) serviceImportExists(dstAccName, from string) bool {\n\ta.mu.RLock()\n\tdup := a.getServiceImportForAccountLocked(dstAccName, from)\n\ta.mu.RUnlock()\n\treturn dup != nil\n}\n\n// Add a service import.\n// This does no checks and should only be called by the msg processing code.\n// Use AddServiceImport from above if responding to user input or config changes, etc.\nfunc (a *Account) addServiceImport(dest *Account, from, to string, claim *jwt.Import) (*serviceImport, error) {\n\trt := Singleton\n\tvar lat *serviceLatency\n\n\tif dest == nil {\n\t\treturn nil, ErrMissingAccount\n\t}\n\n\tvar atrc bool\n\tdest.mu.RLock()\n\tse := dest.getServiceExport(to)\n\tif se != nil {\n\t\trt = se.respType\n\t\tlat = se.latency\n\t\tatrc = se.atrc\n\t}\n\tdestAccName := dest.Name\n\tdest.mu.RUnlock()\n\n\ta.mu.Lock()\n\tif a.imports.services == nil {\n\t\ta.imports.services = make(map[string][]*serviceImport)\n\t} else if dup := a.getServiceImportForAccountLocked(destAccName, from); dup != nil {\n\t\ta.mu.Unlock()\n\t\treturn nil, fmt.Errorf(\"duplicate service import subject %q, previously used in import for account %q, subject %q\",\n\t\t\tfrom, dup.acc.Name, dup.to)\n\t}\n\n\tif to == _EMPTY_ {\n\t\tto = from\n\t}\n\t// Check to see if we have a wildcard\n\tvar (\n\t\tusePub bool\n\t\ttr *subjectTransform\n\t\terr error\n\t)\n\n\tif subjectHasWildcard(to) {\n\t\t// If to and from match, then we use the published subject.\n\t\tif to == from {\n\t\t\tusePub = true\n\t\t} else {\n\t\t\tto, _ = transformUntokenize(to)\n\t\t\t// Create a transform. Do so in reverse such that $ symbols only exist in to\n\t\t\tif tr, err = NewSubjectTransformStrict(to, transformTokenize(from)); err != nil {\n\t\t\t\ta.mu.Unlock()\n\t\t\t\treturn nil, fmt.Errorf(\"failed to create mapping transform for service import subject from %q to %q: %v\",\n\t\t\t\t\tfrom, to, err)\n\t\t\t} else {\n\t\t\t\t// un-tokenize and reverse transform so we get the transform needed\n\t\t\t\tfrom, _ = transformUntokenize(from)\n\t\t\t\ttr = tr.reverse()\n\t\t\t}\n\t\t}\n\t}\n\tvar share bool\n\tif claim != nil {\n\t\tshare = claim.Share\n\t}\n\tsi := &serviceImport{dest, claim, se, nil, from, to, tr, 0, rt, lat, nil, nil, usePub, false, false, share, false, false, atrc, nil}\n\tsis := a.imports.services[from]\n\tsis = append(sis, si)\n\ta.imports.services[from] = sis\n\ta.mu.Unlock()\n\n\tif err := a.addServiceImportSub(si); err != nil {\n\t\ta.removeServiceImport(destAccName, si.from)\n\t\treturn nil, err\n\t}\n\treturn si, nil\n}\n\n// Returns the internal client, will create one if not present.\n// Lock should be held.\nfunc (a *Account) internalClient() *client {\n\tif a.ic == nil && a.srv != nil {\n\t\ta.ic = a.srv.createInternalAccountClient()\n\t\ta.ic.acc = a\n\t}\n\treturn a.ic\n}\n\n// Internal account scoped subscriptions.\nfunc (a *Account) subscribeInternal(subject string, cb msgHandler) (*subscription, error) {\n\treturn a.subscribeInternalEx(subject, cb, false)\n}\n\n// Unsubscribe from an internal account subscription.\nfunc (a *Account) unsubscribeInternal(sub *subscription) {\n\tif ic := a.internalClient(); ic != nil {\n\t\tic.processUnsub(sub.sid)\n\t}\n}\n\n// Creates internal subscription for service import responses.\nfunc (a *Account) subscribeServiceImportResponse(subject string) (*subscription, error) {\n\treturn a.subscribeInternalEx(subject, a.processServiceImportResponse, true)\n}\n\nfunc (a *Account) subscribeInternalEx(subject string, cb msgHandler, ri bool) (*subscription, error) {\n\ta.mu.Lock()\n\ta.isid++\n\tc, sid := a.internalClient(), strconv.FormatUint(a.isid, 10)\n\ta.mu.Unlock()\n\n\t// This will happen in parsing when the account has not been properly setup.\n\tif c == nil {\n\t\treturn nil, fmt.Errorf(\"no internal account client\")\n\t}\n\n\treturn c.processSubEx([]byte(subject), nil, []byte(sid), cb, false, false, ri)\n}\n\n// This will add an account subscription that matches the \"from\" from a service import entry.\nfunc (a *Account) addServiceImportSub(si *serviceImport) error {\n\ta.mu.Lock()\n\tc := a.internalClient()\n\t// This will happen in parsing when the account has not been properly setup.\n\tif c == nil {\n\t\ta.mu.Unlock()\n\t\treturn nil\n\t}\n\tif si.sid != nil {\n\t\ta.mu.Unlock()\n\t\treturn fmt.Errorf(\"duplicate call to create subscription for service import\")\n\t}\n\ta.isid++\n\tsid := strconv.FormatUint(a.isid, 10)\n\tsi.sid = []byte(sid)\n\tsubject := si.from\n\ta.mu.Unlock()\n\n\tcb := func(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) {\n\t\tc.pa.delivered = c.processServiceImport(si, acc, msg)\n\t}\n\tsub, err := c.processSubEx([]byte(subject), nil, []byte(sid), cb, true, true, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Leafnodes introduce a new way to introduce messages into the system. Therefore forward import subscription\n\t// This is similar to what initLeafNodeSmapAndSendSubs does\n\t// TODO we need to consider performing this update as we get client subscriptions.\n\t// This behavior would result in subscription propagation only where actually used.\n\ta.updateLeafNodes(sub, 1)\n\treturn nil\n}\n\n// Remove all the subscriptions associated with service imports.\nfunc (a *Account) removeAllServiceImportSubs() {\n\ta.mu.RLock()\n\tvar sids [][]byte\n\tfor _, sis := range a.imports.services {\n\t\tfor _, si := range sis {\n\t\t\tif si.sid != nil {\n\t\t\t\tsids = append(sids, si.sid)\n\t\t\t\tsi.sid = nil\n\t\t\t}\n\t\t}\n\t}\n\tc := a.ic\n\ta.ic = nil\n\ta.mu.RUnlock()\n\n\tif c == nil {\n\t\treturn\n\t}\n\tfor _, sid := range sids {\n\t\tc.processUnsub(sid)\n\t}\n\tc.closeConnection(InternalClient)\n}\n\n// Add in subscriptions for all registered service imports.\nfunc (a *Account) addAllServiceImportSubs() {\n\tvar sis [32]*serviceImport\n\tserviceImports := sis[:0]\n\ta.mu.RLock()\n\tfor _, sis := range a.imports.services {\n\t\tserviceImports = append(serviceImports, sis...)\n\t}\n\ta.mu.RUnlock()\n\tfor _, si := range serviceImports {\n\t\ta.addServiceImportSub(si)\n\t}\n}\n\nvar (\n\t// header where all information is encoded in one value.\n\ttrcUber = textproto.CanonicalMIMEHeaderKey(\"Uber-Trace-Id\")\n\ttrcCtx = textproto.CanonicalMIMEHeaderKey(\"Traceparent\")\n\ttrcB3 = textproto.CanonicalMIMEHeaderKey(\"B3\")\n\t// openzipkin header to check\n\ttrcB3Sm = textproto.CanonicalMIMEHeaderKey(\"X-B3-Sampled\")\n\ttrcB3Id = textproto.CanonicalMIMEHeaderKey(\"X-B3-TraceId\")\n\t// additional header needed to include when present\n\ttrcB3PSId = textproto.CanonicalMIMEHeaderKey(\"X-B3-ParentSpanId\")\n\ttrcB3SId = textproto.CanonicalMIMEHeaderKey(\"X-B3-SpanId\")\n\ttrcCtxSt = textproto.CanonicalMIMEHeaderKey(\"Tracestate\")\n\ttrcUberCtxPrefix = textproto.CanonicalMIMEHeaderKey(\"Uberctx-\")\n)\n\nfunc newB3Header(h http.Header) http.Header {\n\tretHdr := http.Header{}\n\tif v, ok := h[trcB3Sm]; ok {\n\t\tretHdr[trcB3Sm] = v\n\t}\n\tif v, ok := h[trcB3Id]; ok {\n\t\tretHdr[trcB3Id] = v\n\t}\n\tif v, ok := h[trcB3PSId]; ok {\n\t\tretHdr[trcB3PSId] = v\n\t}\n\tif v, ok := h[trcB3SId]; ok {\n\t\tretHdr[trcB3SId] = v\n\t}\n\treturn retHdr\n}\n\nfunc newUberHeader(h http.Header, tId []string) http.Header {\n\tretHdr := http.Header{trcUber: tId}\n\tfor k, v := range h {\n\t\tif strings.HasPrefix(k, trcUberCtxPrefix) {\n\t\t\tretHdr[k] = v\n\t\t}\n\t}\n\treturn retHdr\n}\n\nfunc newTraceCtxHeader(h http.Header, tId []string) http.Header {\n\tretHdr := http.Header{trcCtx: tId}\n\tif v, ok := h[trcCtxSt]; ok {\n\t\tretHdr[trcCtxSt] = v\n\t}\n\treturn retHdr\n}\n\n// Helper to determine when to sample. When header has a value, sampling is driven by header\nfunc shouldSample(l *serviceLatency, c *client) (bool, http.Header) {\n\tif l == nil {\n\t\treturn false, nil\n\t}\n\tif l.sampling < 0 {\n\t\treturn false, nil\n\t}\n\tif l.sampling >= 100 {\n\t\treturn true, nil\n\t}\n\tif l.sampling > 0 && rand.Int31n(100) <= int32(l.sampling) {\n\t\treturn true, nil\n\t}\n\th := c.parseState.getHeader()\n\tif len(h) == 0 {\n\t\treturn false, nil\n\t}\n\tif tId := h[trcUber]; len(tId) != 0 {\n\t\t// sample 479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1\n\t\ttk := strings.Split(tId[0], \":\")\n\t\tif len(tk) == 4 && len(tk[3]) > 0 && len(tk[3]) <= 2 {\n\t\t\tdst := [2]byte{}\n\t\t\tsrc := [2]byte{'0', tk[3][0]}\n\t\t\tif len(tk[3]) == 2 {\n\t\t\t\tsrc[1] = tk[3][1]\n\t\t\t}\n\t\t\tif _, err := hex.Decode(dst[:], src[:]); err == nil && dst[0]&1 == 1 {\n\t\t\t\treturn true, newUberHeader(h, tId)\n\t\t\t}\n\t\t}\n\t\treturn false, nil\n\t} else if sampled := h[trcB3Sm]; len(sampled) != 0 && sampled[0] == \"1\" {\n\t\treturn true, newB3Header(h) // allowed\n\t} else if len(sampled) != 0 && sampled[0] == \"0\" {\n\t\treturn false, nil // denied\n\t} else if _, ok := h[trcB3Id]; ok {\n\t\t// sample 80f198ee56343ba864fe8b2a57d3eff7\n\t\t// presence (with X-B3-Sampled not being 0) means sampling left to recipient\n\t\treturn true, newB3Header(h)\n\t} else if b3 := h[trcB3]; len(b3) != 0 {\n\t\t// sample 80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90\n\t\t// sample 0\n\t\ttk := strings.Split(b3[0], \"-\")\n\t\tif len(tk) > 2 && tk[2] == \"0\" {\n\t\t\treturn false, nil // denied\n\t\t} else if len(tk) == 1 && tk[0] == \"0\" {\n\t\t\treturn false, nil // denied\n\t\t}\n\t\treturn true, http.Header{trcB3: b3} // sampling allowed or left to recipient of header\n\t} else if tId := h[trcCtx]; len(tId) != 0 {\n\t\tvar sample bool\n\t\t// sample 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\n\t\ttk := strings.Split(tId[0], \"-\")\n\t\tif len(tk) == 4 && len([]byte(tk[3])) == 2 {\n\t\t\tif hexVal, err := strconv.ParseInt(tk[3], 16, 8); err == nil {\n\t\t\t\tsample = hexVal&0x1 == 0x1\n\t\t\t}\n\t\t}\n\t\tif sample {\n\t\t\treturn true, newTraceCtxHeader(h, tId)\n\t\t} else {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\treturn false, nil\n}\n\n// Used to mimic client like replies.\nconst (\n\treplyPrefix = \"_R_.\"\n\treplyPrefixLen = len(replyPrefix)\n\tbaseServerLen = 10\n\treplyLen = 6\n\tminReplyLen = 15\n\tdigits = \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\"\n\tbase = 62\n)\n\n// This is where all service export responses are handled.\nfunc (a *Account) processServiceImportResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\ta.mu.RLock()\n\tif a.expired.Load() || len(a.exports.responses) == 0 {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\tsi := a.exports.responses[subject]\n\n\tif si == nil || si.invalid {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\ta.mu.RUnlock()\n\n\t// Send for normal processing.\n\tc.processServiceImport(si, a, msg)\n}\n\n// Will create the response prefix for fast generation of responses.\n// A wildcard subscription may be used handle interest graph propagation\n// for all service replies, unless we are bound to a leafnode.\n// Lock should be held.\nfunc (a *Account) createRespWildcard() {\n\tvar b = [baseServerLen]byte{'_', 'R', '_', '.'}\n\trn := fastrand.Uint64()\n\tfor i, l := replyPrefixLen, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\ta.siReply = append(b[:], '.')\n}\n\n// Test whether this is a tracked reply.\nfunc isTrackedReply(reply []byte) bool {\n\tlreply := len(reply) - 1\n\treturn lreply > 3 && reply[lreply-1] == '.' && reply[lreply] == 'T'\n}\n\n// Generate a new service reply from the wildcard prefix.\n// FIXME(dlc) - probably do not have to use rand here. about 25ns per.\nfunc (a *Account) newServiceReply(tracking bool) []byte {\n\ta.mu.Lock()\n\ts := a.srv\n\trn := fastrand.Uint64()\n\n\t// Check if we need to create the reply here.\n\tvar createdSiReply bool\n\tif a.siReply == nil {\n\t\ta.createRespWildcard()\n\t\tcreatedSiReply = true\n\t}\n\treplyPre := a.siReply\n\ta.mu.Unlock()\n\n\t// If we created the siReply and we are not bound to a leafnode\n\t// we need to do the wildcard subscription.\n\tif createdSiReply {\n\t\ta.subscribeServiceImportResponse(string(append(replyPre, '>')))\n\t}\n\n\tvar b [replyLen]byte\n\tfor i, l := 0, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\t// Make sure to copy.\n\treply := make([]byte, 0, len(replyPre)+len(b))\n\treply = append(reply, replyPre...)\n\treply = append(reply, b[:]...)\n\n\tif tracking && s.sys != nil {\n\t\t// Add in our tracking identifier. This allows the metrics to get back to only\n\t\t// this server without needless SUBS/UNSUBS.\n\t\treply = append(reply, '.')\n\t\treply = append(reply, s.sys.shash...)\n\t\treply = append(reply, '.', 'T')\n\t}\n\n\treturn reply\n}\n\n// Checks if a serviceImport was created to map responses.\nfunc (si *serviceImport) isRespServiceImport() bool {\n\treturn si != nil && si.response\n}\n\n// Sets the response threshold timer for a service export.\n// Account lock should be held\nfunc (se *serviceExport) setResponseThresholdTimer() {\n\tif se.rtmr != nil {\n\t\treturn // Already set\n\t}\n\tse.rtmr = time.AfterFunc(se.respThresh, se.checkExpiredResponses)\n}\n\n// Account lock should be held\nfunc (se *serviceExport) clearResponseThresholdTimer() bool {\n\tif se.rtmr == nil {\n\t\treturn true\n\t}\n\tstopped := se.rtmr.Stop()\n\tse.rtmr = nil\n\treturn stopped\n}\n\n// checkExpiredResponses will check for any pending responses that need to\n// be cleaned up.\nfunc (se *serviceExport) checkExpiredResponses() {\n\tacc := se.acc\n\tif acc == nil {\n\t\tse.clearResponseThresholdTimer()\n\t\treturn\n\t}\n\n\tvar expired []*serviceImport\n\tmints := time.Now().UnixNano() - int64(se.respThresh)\n\n\t// TODO(dlc) - Should we release lock while doing this? Or only do these in batches?\n\t// Should we break this up for responses only from this service export?\n\t// Responses live on acc directly for fast inbound processsing for the _R_ wildcard.\n\t// We could do another indirection at this level but just to get to the service export?\n\tvar totalResponses int\n\tacc.mu.RLock()\n\tfor _, si := range acc.exports.responses {\n\t\tif si.se == se {\n\t\t\ttotalResponses++\n\t\t\tif si.ts <= mints {\n\t\t\t\texpired = append(expired, si)\n\t\t\t}\n\t\t}\n\t}\n\tacc.mu.RUnlock()\n\n\tfor _, si := range expired {\n\t\tacc.removeRespServiceImport(si, rsiTimeout)\n\t}\n\n\t// Pull out expired to determine if we have any left for timer.\n\ttotalResponses -= len(expired)\n\n\t// Redo timer as needed.\n\tacc.mu.Lock()\n\tif totalResponses > 0 && se.rtmr != nil {\n\t\tse.rtmr.Stop()\n\t\tse.rtmr.Reset(se.respThresh)\n\t} else {\n\t\tse.clearResponseThresholdTimer()\n\t}\n\tacc.mu.Unlock()\n}\n\n// ServiceExportResponseThreshold returns the current threshold.\nfunc (a *Account) ServiceExportResponseThreshold(export string) (time.Duration, error) {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\tse := a.getServiceExport(export)\n\tif se == nil {\n\t\treturn 0, fmt.Errorf(\"no export defined for %q\", export)\n\t}\n\treturn se.respThresh, nil\n}\n\n// SetServiceExportResponseThreshold sets the maximum time the system will a response to be delivered\n// from a service export responder.\nfunc (a *Account) SetServiceExportResponseThreshold(export string, maxTime time.Duration) error {\n\ta.mu.Lock()\n\tif a.isClaimAccount() {\n\t\ta.mu.Unlock()\n\t\treturn fmt.Errorf(\"claim based accounts can not be updated directly\")\n\t}\n\tlrt := a.lowestServiceExportResponseTime()\n\tse := a.getServiceExport(export)\n\tif se == nil {\n\t\ta.mu.Unlock()\n\t\treturn fmt.Errorf(\"no export defined for %q\", export)\n\t}\n\tse.respThresh = maxTime\n\n\tvar clients []*client\n\tnlrt := a.lowestServiceExportResponseTime()\n\tif nlrt != lrt && len(a.clients) > 0 {\n\t\tclients = a.getClientsLocked()\n\t}\n\t// Need to release because lock ordering is client -> Account\n\ta.mu.Unlock()\n\tif len(clients) > 0 {\n\t\tupdateAllClientsServiceExportResponseTime(clients, nlrt)\n\t}\n\treturn nil\n}\n\nfunc (a *Account) SetServiceExportAllowTrace(export string, allowTrace bool) error {\n\ta.mu.Lock()\n\tse := a.getServiceExport(export)\n\tif se == nil {\n\t\ta.mu.Unlock()\n\t\treturn fmt.Errorf(\"no export defined for %q\", export)\n\t}\n\tse.atrc = allowTrace\n\ta.mu.Unlock()\n\treturn nil\n}\n\n// This is for internal service import responses.\nfunc (a *Account) addRespServiceImport(dest *Account, to string, osi *serviceImport, tracking bool, header http.Header) *serviceImport {\n\tnrr := string(osi.acc.newServiceReply(tracking))\n\n\ta.mu.Lock()\n\trt := osi.rt\n\n\t// dest is the requestor's account. a is the service responder with the export.\n\t// Marked as internal here, that is how we distinguish.\n\tsi := &serviceImport{dest, nil, osi.se, nil, nrr, to, nil, 0, rt, nil, nil, nil, false, true, false, osi.share, false, false, false, nil}\n\n\tif a.exports.responses == nil {\n\t\ta.exports.responses = make(map[string]*serviceImport)\n\t}\n\ta.exports.responses[nrr] = si\n\n\t// Always grab time and make sure response threshold timer is running.\n\tsi.ts = time.Now().UnixNano()\n\tif osi.se != nil {\n\t\tosi.se.setResponseThresholdTimer()\n\t}\n\n\tif rt == Singleton && tracking {\n\t\tsi.latency = osi.latency\n\t\tsi.tracking = true\n\t\tsi.trackingHdr = header\n\t}\n\ta.mu.Unlock()\n\n\t// We do add in the reverse map such that we can detect loss of interest and do proper\n\t// cleanup of this si as interest goes away.\n\tdest.addReverseRespMapEntry(a, to, nrr)\n\n\treturn si\n}\n\n// AddStreamImportWithClaim will add in the stream import from a specific account with optional token.\nfunc (a *Account) AddStreamImportWithClaim(account *Account, from, prefix string, imClaim *jwt.Import) error {\n\treturn a.addStreamImportWithClaim(account, from, prefix, false, imClaim)\n}\n\nfunc (a *Account) addStreamImportWithClaim(account *Account, from, prefix string, allowTrace bool, imClaim *jwt.Import) error {\n\tif account == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\t// First check to see if the account has authorized export of the subject.\n\tif !account.checkStreamImportAuthorized(a, from, imClaim) {\n\t\treturn ErrStreamImportAuthorization\n\t}\n\n\t// Check prefix if it exists and make sure its a literal.\n\t// Append token separator if not already present.\n\tif prefix != _EMPTY_ {\n\t\t// Make sure there are no wildcards here, this prefix needs to be a literal\n\t\t// since it will be prepended to a publish subject.\n\t\tif !subjectIsLiteral(prefix) {\n\t\t\treturn ErrStreamImportBadPrefix\n\t\t}\n\t\tif prefix[len(prefix)-1] != btsep {\n\t\t\tprefix = prefix + string(btsep)\n\t\t}\n\t}\n\n\treturn a.addMappedStreamImportWithClaim(account, from, prefix+from, allowTrace, imClaim)\n}\n\n// AddMappedStreamImport helper for AddMappedStreamImportWithClaim\nfunc (a *Account) AddMappedStreamImport(account *Account, from, to string) error {\n\treturn a.AddMappedStreamImportWithClaim(account, from, to, nil)\n}\n\n// AddMappedStreamImportWithClaim will add in the stream import from a specific account with optional token.\nfunc (a *Account) AddMappedStreamImportWithClaim(account *Account, from, to string, imClaim *jwt.Import) error {\n\treturn a.addMappedStreamImportWithClaim(account, from, to, false, imClaim)\n}\n\nfunc (a *Account) addMappedStreamImportWithClaim(account *Account, from, to string, allowTrace bool, imClaim *jwt.Import) error {\n\tif account == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\t// First check to see if the account has authorized export of the subject.\n\tif !account.checkStreamImportAuthorized(a, from, imClaim) {\n\t\treturn ErrStreamImportAuthorization\n\t}\n\n\tif to == _EMPTY_ {\n\t\tto = from\n\t}\n\n\t// Check if this forms a cycle.\n\tif err := a.streamImportFormsCycle(account, to); err != nil {\n\t\treturn err\n\t}\n\n\tif err := a.streamImportFormsCycle(account, from); err != nil {\n\t\treturn err\n\t}\n\n\tvar (\n\t\tusePub bool\n\t\ttr *subjectTransform\n\t\terr error\n\t)\n\tif subjectHasWildcard(from) {\n\t\tif to == from {\n\t\t\tusePub = true\n\t\t} else {\n\t\t\t// Create a transform\n\t\t\tif tr, err = NewSubjectTransformStrict(from, transformTokenize(to)); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to create mapping transform for stream import subject from %q to %q: %v\",\n\t\t\t\t\tfrom, to, err)\n\t\t\t}\n\t\t\tto, _ = transformUntokenize(to)\n\t\t}\n\t}\n\n\ta.mu.Lock()\n\tif a.isStreamImportDuplicate(account, from) {\n\t\ta.mu.Unlock()\n\t\treturn ErrStreamImportDuplicate\n\t}\n\tif imClaim != nil {\n\t\tallowTrace = imClaim.AllowTrace\n\t}\n\ta.imports.streams = append(a.imports.streams, &streamImport{account, from, to, tr, nil, imClaim, usePub, false, allowTrace})\n\ta.mu.Unlock()\n\treturn nil\n}\n\n// isStreamImportDuplicate checks for duplicate.\n// Lock should be held.\nfunc (a *Account) isStreamImportDuplicate(acc *Account, from string) bool {\n\tfor _, si := range a.imports.streams {\n\t\tif si.acc == acc && si.from == from {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// AddStreamImport will add in the stream import from a specific account.\nfunc (a *Account) AddStreamImport(account *Account, from, prefix string) error {\n\treturn a.addStreamImportWithClaim(account, from, prefix, false, nil)\n}\n\n// IsPublicExport is a placeholder to denote a public export.\nvar IsPublicExport = []*Account(nil)\n\n// AddStreamExport will add an export to the account. If accounts is nil\n// it will signify a public export, meaning anyone can import.\nfunc (a *Account) AddStreamExport(subject string, accounts []*Account) error {\n\treturn a.addStreamExportWithAccountPos(subject, accounts, 0)\n}\n\n// AddStreamExport will add an export to the account. If accounts is nil\n// it will signify a public export, meaning anyone can import.\n// if accountPos is > 0, all imports will be granted where the following holds:\n// strings.Split(subject, tsep)[accountPos] == account id will be granted.\nfunc (a *Account) addStreamExportWithAccountPos(subject string, accounts []*Account, accountPos uint) error {\n\tif a == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\n\tif a.exports.streams == nil {\n\t\ta.exports.streams = make(map[string]*streamExport)\n\t}\n\tea := a.exports.streams[subject]\n\tif accounts != nil || accountPos > 0 {\n\t\tif ea == nil {\n\t\t\tea = &streamExport{}\n\t\t}\n\t\tif err := setExportAuth(&ea.exportAuth, subject, accounts, accountPos); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\ta.exports.streams[subject] = ea\n\treturn nil\n}\n\n// Check if another account is authorized to import from us.\nfunc (a *Account) checkStreamImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {\n\t// Find the subject in the exports list.\n\ta.mu.RLock()\n\tauth := a.checkStreamImportAuthorizedNoLock(account, subject, imClaim)\n\ta.mu.RUnlock()\n\treturn auth\n}\n\nfunc (a *Account) checkStreamImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {\n\tif a.exports.streams == nil || !IsValidSubject(subject) {\n\t\treturn false\n\t}\n\treturn a.checkStreamExportApproved(account, subject, imClaim)\n}\n\nfunc (a *Account) checkAuth(ea *exportAuth, account *Account, imClaim *jwt.Import, tokens []string) bool {\n\t// if ea is nil or ea.approved is nil, that denotes a public export\n\tif ea == nil || (len(ea.approved) == 0 && !ea.tokenReq && ea.accountPos == 0) {\n\t\treturn true\n\t}\n\t// Check if the export is protected and enforces presence of importing account identity\n\tif ea.accountPos > 0 {\n\t\treturn ea.accountPos <= uint(len(tokens)) && tokens[ea.accountPos-1] == account.Name\n\t}\n\t// Check if token required\n\tif ea.tokenReq {\n\t\treturn a.checkActivation(account, imClaim, ea, true)\n\t}\n\tif ea.approved == nil {\n\t\treturn false\n\t}\n\t// If we have a matching account we are authorized\n\t_, ok := ea.approved[account.Name]\n\treturn ok\n}\n\nfunc (a *Account) checkStreamExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {\n\t// Check direct match of subject first\n\tea, ok := a.exports.streams[subject]\n\tif ok {\n\t\t// if ea is nil or eq.approved is nil, that denotes a public export\n\t\tif ea == nil {\n\t\t\treturn true\n\t\t}\n\t\treturn a.checkAuth(&ea.exportAuth, account, imClaim, nil)\n\t}\n\n\t// ok if we are here we did not match directly so we need to test each one.\n\t// The import subject arg has to take precedence, meaning the export\n\t// has to be a true subset of the import claim. We already checked for\n\t// exact matches above.\n\ttokens := strings.Split(subject, tsep)\n\tfor subj, ea := range a.exports.streams {\n\t\tif isSubsetMatch(tokens, subj) {\n\t\t\tif ea == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\treturn a.checkAuth(&ea.exportAuth, account, imClaim, tokens)\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *Account) checkServiceExportApproved(account *Account, subject string, imClaim *jwt.Import) bool {\n\t// Check direct match of subject first\n\tse, ok := a.exports.services[subject]\n\tif ok {\n\t\t// if se is nil or eq.approved is nil, that denotes a public export\n\t\tif se == nil {\n\t\t\treturn true\n\t\t}\n\t\treturn a.checkAuth(&se.exportAuth, account, imClaim, nil)\n\t}\n\t// ok if we are here we did not match directly so we need to test each one.\n\t// The import subject arg has to take precedence, meaning the export\n\t// has to be a true subset of the import claim. We already checked for\n\t// exact matches above.\n\ttokens := strings.Split(subject, tsep)\n\tfor subj, se := range a.exports.services {\n\t\tif isSubsetMatch(tokens, subj) {\n\t\t\tif se == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\treturn a.checkAuth(&se.exportAuth, account, imClaim, tokens)\n\t\t}\n\t}\n\treturn false\n}\n\n// Helper function to get a serviceExport.\n// Lock should be held on entry.\nfunc (a *Account) getServiceExport(subj string) *serviceExport {\n\tse, ok := a.exports.services[subj]\n\t// The export probably has a wildcard, so lookup that up.\n\tif !ok {\n\t\tse = a.getWildcardServiceExport(subj)\n\t}\n\treturn se\n}\n\n// This helper is used when trying to match a serviceExport record that is\n// represented by a wildcard.\n// Lock should be held on entry.\nfunc (a *Account) getWildcardServiceExport(from string) *serviceExport {\n\ttokens := strings.Split(from, tsep)\n\tfor subj, se := range a.exports.services {\n\t\tif isSubsetMatch(tokens, subj) {\n\t\t\treturn se\n\t\t}\n\t}\n\treturn nil\n}\n\n// These are import stream specific versions for when an activation expires.\nfunc (a *Account) streamActivationExpired(exportAcc *Account, subject string) {\n\ta.mu.RLock()\n\tif a.expired.Load() || a.imports.streams == nil {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\tvar si *streamImport\n\tfor _, si = range a.imports.streams {\n\t\tif si.acc == exportAcc && si.from == subject {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif si == nil || si.invalid {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\ta.mu.RUnlock()\n\n\tif si.acc.checkActivation(a, si.claim, nil, false) {\n\t\t// The token has been updated most likely and we are good to go.\n\t\treturn\n\t}\n\n\ta.mu.Lock()\n\tsi.invalid = true\n\tclients := a.getClientsLocked()\n\tawcsti := map[string]struct{}{a.Name: {}}\n\ta.mu.Unlock()\n\tfor _, c := range clients {\n\t\tc.processSubsOnConfigReload(awcsti)\n\t}\n}\n\n// These are import service specific versions for when an activation expires.\nfunc (a *Account) serviceActivationExpired(dstAcc *Account, subject string) {\n\ta.mu.RLock()\n\tif a.expired.Load() || a.imports.services == nil {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\tsi := a.getServiceImportForAccountLocked(dstAcc.Name, subject)\n\tif si == nil || si.invalid {\n\t\ta.mu.RUnlock()\n\t\treturn\n\t}\n\ta.mu.RUnlock()\n\n\tif si.acc.checkActivation(a, si.claim, nil, false) {\n\t\t// The token has been updated most likely and we are good to go.\n\t\treturn\n\t}\n\n\ta.mu.Lock()\n\tsi.invalid = true\n\ta.mu.Unlock()\n}\n\n// Fires for expired activation tokens. We could track this with timers etc.\n// Instead we just re-analyze where we are and if we need to act.\nfunc (a *Account) activationExpired(exportAcc *Account, subject string, kind jwt.ExportType) {\n\tswitch kind {\n\tcase jwt.Stream:\n\t\ta.streamActivationExpired(exportAcc, subject)\n\tcase jwt.Service:\n\t\ta.serviceActivationExpired(exportAcc, subject)\n\t}\n}\n\nfunc isRevoked(revocations map[string]int64, subject string, issuedAt int64) bool {\n\tif len(revocations) == 0 {\n\t\treturn false\n\t}\n\tif t, ok := revocations[subject]; !ok || t < issuedAt {\n\t\tif t, ok := revocations[jwt.All]; !ok || t < issuedAt {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// checkActivation will check the activation token for validity.\n// ea may only be nil in cases where revocation may not be checked, say triggered by expiration timer.\nfunc (a *Account) checkActivation(importAcc *Account, claim *jwt.Import, ea *exportAuth, expTimer bool) bool {\n\tif claim == nil || claim.Token == _EMPTY_ {\n\t\treturn false\n\t}\n\t// Create a quick clone so we can inline Token JWT.\n\tclone := *claim\n\n\tvr := jwt.CreateValidationResults()\n\tclone.Validate(importAcc.Name, vr)\n\tif vr.IsBlocking(true) {\n\t\treturn false\n\t}\n\tact, err := jwt.DecodeActivationClaims(clone.Token)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif !a.isIssuerClaimTrusted(act) {\n\t\treturn false\n\t}\n\tvr = jwt.CreateValidationResults()\n\tact.Validate(vr)\n\tif vr.IsBlocking(true) {\n\t\treturn false\n\t}\n\tif act.Expires != 0 {\n\t\ttn := time.Now().Unix()\n\t\tif act.Expires <= tn {\n\t\t\treturn false\n\t\t}\n\t\tif expTimer {\n\t\t\texpiresAt := time.Duration(act.Expires - tn)\n\t\t\ttime.AfterFunc(expiresAt*time.Second, func() {\n\t\t\t\timportAcc.activationExpired(a, string(act.ImportSubject), claim.Type)\n\t\t\t})\n\t\t}\n\t}\n\tif ea == nil {\n\t\treturn true\n\t}\n\t// Check for token revocation..\n\treturn !isRevoked(ea.actsRevoked, act.Subject, act.IssuedAt)\n}\n\n// Returns true if the activation claim is trusted. That is the issuer matches\n// the account or is an entry in the signing keys.\nfunc (a *Account) isIssuerClaimTrusted(claims *jwt.ActivationClaims) bool {\n\t// if no issuer account, issuer is the account\n\tif claims.IssuerAccount == _EMPTY_ {\n\t\treturn true\n\t}\n\t// If the IssuerAccount is not us, then this is considered an error.\n\tif a.Name != claims.IssuerAccount {\n\t\tif a.srv != nil {\n\t\t\ta.srv.Errorf(\"Invalid issuer account %q in activation claim (subject: %q - type: %q) for account %q\",\n\t\t\t\tclaims.IssuerAccount, claims.Activation.ImportSubject, claims.Activation.ImportType, a.Name)\n\t\t}\n\t\treturn false\n\t}\n\t_, ok := a.hasIssuerNoLock(claims.Issuer)\n\treturn ok\n}\n\n// Returns true if `a` and `b` stream imports are the same. Note that the\n// check is done with the account's name, not the pointer. This is used\n// during config reload where we are comparing current and new config\n// in which pointers are different.\n// Acquires `a` read lock, but `b` is assumed to not be accessed\n// by anyone but the caller (`b` is not registered anywhere).\nfunc (a *Account) checkStreamImportsEqual(b *Account) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\n\tif len(a.imports.streams) != len(b.imports.streams) {\n\t\treturn false\n\t}\n\t// Load the b imports into a map index by what we are looking for.\n\tbm := make(map[string]*streamImport, len(b.imports.streams))\n\tfor _, bim := range b.imports.streams {\n\t\tbm[bim.acc.Name+bim.from+bim.to] = bim\n\t}\n\tfor _, aim := range a.imports.streams {\n\t\tif bim, ok := bm[aim.acc.Name+aim.from+aim.to]; !ok {\n\t\t\treturn false\n\t\t} else if aim.atrc != bim.atrc {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Returns true if `a` and `b` stream exports are the same.\n// Acquires `a` read lock, but `b` is assumed to not be accessed\n// by anyone but the caller (`b` is not registered anywhere).\nfunc (a *Account) checkStreamExportsEqual(b *Account) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif len(a.exports.streams) != len(b.exports.streams) {\n\t\treturn false\n\t}\n\tfor subj, aea := range a.exports.streams {\n\t\tbea, ok := b.exports.streams[subj]\n\t\tif !ok {\n\t\t\treturn false\n\t\t}\n\t\tif !isStreamExportEqual(aea, bea) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc isStreamExportEqual(a, b *streamExport) bool {\n\tif a == nil && b == nil {\n\t\treturn true\n\t}\n\tif (a == nil && b != nil) || (a != nil && b == nil) {\n\t\treturn false\n\t}\n\treturn isExportAuthEqual(&a.exportAuth, &b.exportAuth)\n}\n\n// Returns true if `a` and `b` service exports are the same.\n// Acquires `a` read lock, but `b` is assumed to not be accessed\n// by anyone but the caller (`b` is not registered anywhere).\nfunc (a *Account) checkServiceExportsEqual(b *Account) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif len(a.exports.services) != len(b.exports.services) {\n\t\treturn false\n\t}\n\tfor subj, aea := range a.exports.services {\n\t\tbea, ok := b.exports.services[subj]\n\t\tif !ok {\n\t\t\treturn false\n\t\t}\n\t\tif !isServiceExportEqual(aea, bea) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc isServiceExportEqual(a, b *serviceExport) bool {\n\tif a == nil && b == nil {\n\t\treturn true\n\t}\n\tif (a == nil && b != nil) || (a != nil && b == nil) {\n\t\treturn false\n\t}\n\tif !isExportAuthEqual(&a.exportAuth, &b.exportAuth) {\n\t\treturn false\n\t}\n\tif a.acc.Name != b.acc.Name {\n\t\treturn false\n\t}\n\tif a.respType != b.respType {\n\t\treturn false\n\t}\n\tif a.latency != nil || b.latency != nil {\n\t\tif (a.latency != nil && b.latency == nil) || (a.latency == nil && b.latency != nil) {\n\t\t\treturn false\n\t\t}\n\t\tif a.latency.sampling != b.latency.sampling {\n\t\t\treturn false\n\t\t}\n\t\tif a.latency.subject != b.latency.subject {\n\t\t\treturn false\n\t\t}\n\t}\n\tif a.atrc != b.atrc {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Returns true if `a` and `b` exportAuth structures are equal.\n// Both `a` and `b` are guaranteed to be non-nil.\n// Locking is handled by the caller.\nfunc isExportAuthEqual(a, b *exportAuth) bool {\n\tif a.tokenReq != b.tokenReq {\n\t\treturn false\n\t}\n\tif a.accountPos != b.accountPos {\n\t\treturn false\n\t}\n\tif len(a.approved) != len(b.approved) {\n\t\treturn false\n\t}\n\tfor ak, av := range a.approved {\n\t\tif bv, ok := b.approved[ak]; !ok || av.Name != bv.Name {\n\t\t\treturn false\n\t\t}\n\t}\n\tif len(a.actsRevoked) != len(b.actsRevoked) {\n\t\treturn false\n\t}\n\tfor ak, av := range a.actsRevoked {\n\t\tif bv, ok := b.actsRevoked[ak]; !ok || av != bv {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Check if another account is authorized to route requests to this service.\nfunc (a *Account) checkServiceImportAuthorized(account *Account, subject string, imClaim *jwt.Import) bool {\n\ta.mu.RLock()\n\tauthorized := a.checkServiceImportAuthorizedNoLock(account, subject, imClaim)\n\ta.mu.RUnlock()\n\treturn authorized\n}\n\n// Check if another account is authorized to route requests to this service.\nfunc (a *Account) checkServiceImportAuthorizedNoLock(account *Account, subject string, imClaim *jwt.Import) bool {\n\t// Find the subject in the services list.\n\tif a.exports.services == nil {\n\t\treturn false\n\t}\n\treturn a.checkServiceExportApproved(account, subject, imClaim)\n}\n\n// IsExpired returns expiration status.\nfunc (a *Account) IsExpired() bool {\n\treturn a.expired.Load()\n}\n\n// Called when an account has expired.\nfunc (a *Account) expiredTimeout() {\n\t// Mark expired first.\n\ta.expired.Store(true)\n\n\t// Collect the clients and expire them.\n\tcs := a.getClients()\n\tfor _, c := range cs {\n\t\tif !isInternalClient(c.kind) {\n\t\t\tc.accountAuthExpired()\n\t\t}\n\t}\n}\n\n// Sets the expiration timer for an account JWT that has it set.\nfunc (a *Account) setExpirationTimer(d time.Duration) {\n\ta.etmr = time.AfterFunc(d, a.expiredTimeout)\n}\n\n// Lock should be held\nfunc (a *Account) clearExpirationTimer() bool {\n\tif a.etmr == nil {\n\t\treturn true\n\t}\n\tstopped := a.etmr.Stop()\n\ta.etmr = nil\n\treturn stopped\n}\n\n// checkUserRevoked will check if a user has been revoked.\nfunc (a *Account) checkUserRevoked(nkey string, issuedAt int64) bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn isRevoked(a.usersRevoked, nkey, issuedAt)\n}\n\n// failBearer will return if bearer token are allowed (false) or disallowed (true)\nfunc (a *Account) failBearer() bool {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn a.disallowBearer\n}\n\n// Check expiration and set the proper state as needed.\nfunc (a *Account) checkExpiration(claims *jwt.ClaimsData) {\n\ta.mu.Lock()\n\tdefer a.mu.Unlock()\n\n\ta.clearExpirationTimer()\n\tif claims.Expires == 0 {\n\t\ta.expired.Store(false)\n\t\treturn\n\t}\n\ttn := time.Now().Unix()\n\tif claims.Expires <= tn {\n\t\ta.expired.Store(true)\n\t\treturn\n\t}\n\texpiresAt := time.Duration(claims.Expires - tn)\n\ta.setExpirationTimer(expiresAt * time.Second)\n\ta.expired.Store(false)\n}\n\n// hasIssuer returns true if the issuer matches the account\n// If the issuer is a scoped signing key, the scope will be returned as well\n// issuer or it is a signing key for the account.\nfunc (a *Account) hasIssuer(issuer string) (jwt.Scope, bool) {\n\ta.mu.RLock()\n\tscope, ok := a.hasIssuerNoLock(issuer)\n\ta.mu.RUnlock()\n\treturn scope, ok\n}\n\n// hasIssuerNoLock is the unlocked version of hasIssuer\nfunc (a *Account) hasIssuerNoLock(issuer string) (jwt.Scope, bool) {\n\tscope, ok := a.signingKeys[issuer]\n\treturn scope, ok\n}\n\n// Returns the loop detection subject used for leafnodes\nfunc (a *Account) getLDSubject() string {\n\ta.mu.RLock()\n\tlds := a.lds\n\ta.mu.RUnlock()\n\treturn lds\n}\n\n// Placeholder for signaling token auth required.\nvar tokenAuthReq = []*Account{}\n\nfunc authAccounts(tokenReq bool) []*Account {\n\tif tokenReq {\n\t\treturn tokenAuthReq\n\t}\n\treturn nil\n}\n\n// SetAccountResolver will assign the account resolver.\nfunc (s *Server) SetAccountResolver(ar AccountResolver) {\n\ts.mu.Lock()\n\ts.accResolver = ar\n\ts.mu.Unlock()\n}\n\n// AccountResolver returns the registered account resolver.\nfunc (s *Server) AccountResolver() AccountResolver {\n\ts.mu.RLock()\n\tar := s.accResolver\n\ts.mu.RUnlock()\n\treturn ar\n}\n\n// isClaimAccount returns if this account is backed by a JWT claim.\n// Lock should be held.\nfunc (a *Account) isClaimAccount() bool {\n\treturn a.claimJWT != _EMPTY_\n}\n\n// UpdateAccountClaims will update an existing account with new claims.\n// This will replace any exports or imports previously defined.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) UpdateAccountClaims(a *Account, ac *jwt.AccountClaims) {\n\ts.updateAccountClaimsWithRefresh(a, ac, true)\n}\n\nfunc (a *Account) traceLabel() string {\n\tif a == nil {\n\t\treturn _EMPTY_\n\t}\n\tif a.nameTag != _EMPTY_ {\n\t\treturn fmt.Sprintf(\"%s/%s\", a.Name, a.nameTag)\n\t}\n\treturn a.Name\n}\n\n// Check if an account has external auth set.\n// Operator/Account Resolver only.\nfunc (a *Account) hasExternalAuth() bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn a.extAuth != nil\n}\n\n// Deterimine if this is an external auth user.\nfunc (a *Account) isExternalAuthUser(userID string) bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif a.extAuth != nil {\n\t\tfor _, u := range a.extAuth.AuthUsers {\n\t\t\tif userID == u {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Return the external authorization xkey if external authorization is enabled and the xkey is set.\n// Operator/Account Resolver only.\nfunc (a *Account) externalAuthXKey() string {\n\tif a == nil {\n\t\treturn _EMPTY_\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif a.extAuth != nil && a.extAuth.XKey != _EMPTY_ {\n\t\treturn a.extAuth.XKey\n\t}\n\treturn _EMPTY_\n}\n\n// Check if an account switch for external authorization is allowed.\nfunc (a *Account) isAllowedAcount(acc string) bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tif a.extAuth != nil {\n\t\t// if we have a single allowed account, and we have a wildcard\n\t\t// we accept it\n\t\tif len(a.extAuth.AllowedAccounts) == 1 &&\n\t\t\ta.extAuth.AllowedAccounts[0] == jwt.AnyAccount {\n\t\t\treturn true\n\t\t}\n\t\t// otherwise must match exactly\n\t\tfor _, a := range a.extAuth.AllowedAccounts {\n\t\t\tif a == acc {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// updateAccountClaimsWithRefresh will update an existing account with new claims.\n// If refreshImportingAccounts is true it will also update incomplete dependent accounts\n// This will replace any exports or imports previously defined.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaims, refreshImportingAccounts bool) {\n\tif a == nil {\n\t\treturn\n\t}\n\ts.Debugf(\"Updating account claims: %s/%s\", a.Name, ac.Name)\n\ta.checkExpiration(ac.Claims())\n\n\ta.mu.Lock()\n\t// Clone to update, only select certain fields.\n\told := &Account{Name: a.Name, exports: a.exports, limits: a.limits, signingKeys: a.signingKeys}\n\n\t// overwrite claim meta data\n\ta.nameTag = ac.Name\n\ta.tags = ac.Tags\n\n\t// Grab trace label under lock.\n\ttl := a.traceLabel()\n\n\tvar td string\n\tvar tds int\n\tif ac.Trace != nil {\n\t\t// Update trace destination and sampling\n\t\ttd, tds = string(ac.Trace.Destination), ac.Trace.Sampling\n\t\tif !IsValidPublishSubject(td) {\n\t\t\ttd, tds = _EMPTY_, 0\n\t\t} else if tds <= 0 || tds > 100 {\n\t\t\ttds = 100\n\t\t}\n\t}\n\ta.traceDest, a.traceDestSampling = td, tds\n\n\t// Check for external authorization.\n\tif ac.HasExternalAuthorization() {\n\t\ta.extAuth = &jwt.ExternalAuthorization{}\n\t\ta.extAuth.AuthUsers.Add(ac.Authorization.AuthUsers...)\n\t\ta.extAuth.AllowedAccounts.Add(ac.Authorization.AllowedAccounts...)\n\t\ta.extAuth.XKey = ac.Authorization.XKey\n\t}\n\n\t// Reset exports and imports here.\n\n\t// Exports is creating a whole new map.\n\ta.exports = exportMap{}\n\n\t// Imports are checked unlocked in processInbound, so we can't change out the struct here. Need to process inline.\n\tif a.imports.streams != nil {\n\t\told.imports.streams = a.imports.streams\n\t\ta.imports.streams = nil\n\t}\n\tif a.imports.services != nil {\n\t\told.imports.services = make(map[string][]*serviceImport, len(a.imports.services))\n\t\tfor k, v := range a.imports.services {\n\t\t\tsis := append([]*serviceImport(nil), v...)\n\t\t\told.imports.services[k] = sis\n\t\t\tdelete(a.imports.services, k)\n\t\t}\n\t}\n\n\talteredScope := map[string]struct{}{}\n\n\t// update account signing keys\n\ta.signingKeys = nil\n\t_, strict := s.strictSigningKeyUsage[a.Issuer]\n\tif len(ac.SigningKeys) > 0 || !strict {\n\t\ta.signingKeys = make(map[string]jwt.Scope)\n\t}\n\tsignersChanged := false\n\tfor k, scope := range ac.SigningKeys {\n\t\ta.signingKeys[k] = scope\n\t}\n\tif !strict {\n\t\ta.signingKeys[a.Name] = nil\n\t}\n\tif len(a.signingKeys) != len(old.signingKeys) {\n\t\tsignersChanged = true\n\t}\n\tfor k, scope := range a.signingKeys {\n\t\tif oldScope, ok := old.signingKeys[k]; !ok {\n\t\t\tsignersChanged = true\n\t\t} else if !reflect.DeepEqual(scope, oldScope) {\n\t\t\tsignersChanged = true\n\t\t\talteredScope[k] = struct{}{}\n\t\t}\n\t}\n\t// collect mappings that need to be removed\n\tremoveList := []string{}\n\tfor _, m := range a.mappings {\n\t\tif _, ok := ac.Mappings[jwt.Subject(m.src)]; !ok {\n\t\t\tremoveList = append(removeList, m.src)\n\t\t}\n\t}\n\ta.mu.Unlock()\n\n\tfor sub, wm := range ac.Mappings {\n\t\tmappings := make([]*MapDest, len(wm))\n\t\tfor i, m := range wm {\n\t\t\tmappings[i] = &MapDest{\n\t\t\t\tSubject: string(m.Subject),\n\t\t\t\tWeight: m.GetWeight(),\n\t\t\t\tCluster: m.Cluster,\n\t\t\t}\n\t\t}\n\t\t// This will overwrite existing entries\n\t\ta.AddWeightedMappings(string(sub), mappings...)\n\t}\n\t// remove mappings\n\tfor _, rmMapping := range removeList {\n\t\ta.RemoveMapping(rmMapping)\n\t}\n\n\t// Re-register system exports/imports.\n\tif a == s.SystemAccount() {\n\t\ts.addSystemAccountExports(a)\n\t} else {\n\t\ts.registerSystemImports(a)\n\t}\n\n\tjsEnabled := s.JetStreamEnabled()\n\n\tstreamTokenExpirationChanged := false\n\tserviceTokenExpirationChanged := false\n\n\tfor _, e := range ac.Exports {\n\t\tswitch e.Type {\n\t\tcase jwt.Stream:\n\t\t\ts.Debugf(\"Adding stream export %q for %s\", e.Subject, tl)\n\t\t\tif err := a.addStreamExportWithAccountPos(\n\t\t\t\tstring(e.Subject), authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {\n\t\t\t\ts.Debugf(\"Error adding stream export to account [%s]: %v\", tl, err.Error())\n\t\t\t}\n\t\tcase jwt.Service:\n\t\t\ts.Debugf(\"Adding service export %q for %s\", e.Subject, tl)\n\t\t\trt := Singleton\n\t\t\tswitch e.ResponseType {\n\t\t\tcase jwt.ResponseTypeStream:\n\t\t\t\trt = Streamed\n\t\t\tcase jwt.ResponseTypeChunked:\n\t\t\t\trt = Chunked\n\t\t\t}\n\t\t\tif err := a.addServiceExportWithResponseAndAccountPos(\n\t\t\t\tstring(e.Subject), rt, authAccounts(e.TokenReq), e.AccountTokenPosition); err != nil {\n\t\t\t\ts.Debugf(\"Error adding service export to account [%s]: %v\", tl, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsub := string(e.Subject)\n\t\t\tif e.Latency != nil {\n\t\t\t\tif err := a.TrackServiceExportWithSampling(sub, string(e.Latency.Results), int(e.Latency.Sampling)); err != nil {\n\t\t\t\t\thdrNote := _EMPTY_\n\t\t\t\t\tif e.Latency.Sampling == jwt.Headers {\n\t\t\t\t\t\thdrNote = \" (using headers)\"\n\t\t\t\t\t}\n\t\t\t\t\ts.Debugf(\"Error adding latency tracking%s for service export to account [%s]: %v\", hdrNote, tl, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif e.ResponseThreshold != 0 {\n\t\t\t\t// Response threshold was set in options.\n\t\t\t\tif err := a.SetServiceExportResponseThreshold(sub, e.ResponseThreshold); err != nil {\n\t\t\t\t\ts.Debugf(\"Error adding service export response threshold for [%s]: %v\", tl, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := a.SetServiceExportAllowTrace(sub, e.AllowTrace); err != nil {\n\t\t\t\ts.Debugf(\"Error adding allow_trace for %q: %v\", sub, err)\n\t\t\t}\n\t\t}\n\n\t\tvar revocationChanged *bool\n\t\tvar ea *exportAuth\n\n\t\ta.mu.Lock()\n\t\tswitch e.Type {\n\t\tcase jwt.Stream:\n\t\t\trevocationChanged = &streamTokenExpirationChanged\n\t\t\tif se, ok := a.exports.streams[string(e.Subject)]; ok && se != nil {\n\t\t\t\tea = &se.exportAuth\n\t\t\t}\n\t\tcase jwt.Service:\n\t\t\trevocationChanged = &serviceTokenExpirationChanged\n\t\t\tif se, ok := a.exports.services[string(e.Subject)]; ok && se != nil {\n\t\t\t\tea = &se.exportAuth\n\t\t\t}\n\t\t}\n\t\tif ea != nil {\n\t\t\toldRevocations := ea.actsRevoked\n\t\t\tif len(e.Revocations) == 0 {\n\t\t\t\t// remove all, no need to evaluate existing imports\n\t\t\t\tea.actsRevoked = nil\n\t\t\t} else if len(oldRevocations) == 0 {\n\t\t\t\t// add all, existing imports need to be re evaluated\n\t\t\t\tea.actsRevoked = e.Revocations\n\t\t\t\t*revocationChanged = true\n\t\t\t} else {\n\t\t\t\tea.actsRevoked = e.Revocations\n\t\t\t\t// diff, existing imports need to be conditionally re evaluated, depending on:\n\t\t\t\t// if a key was added, or it's timestamp increased\n\t\t\t\tfor k, t := range e.Revocations {\n\t\t\t\t\tif tOld, ok := oldRevocations[k]; !ok || tOld < t {\n\t\t\t\t\t\t*revocationChanged = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ta.mu.Unlock()\n\t}\n\tvar incompleteImports []*jwt.Import\n\tfor _, i := range ac.Imports {\n\t\tacc, err := s.lookupAccount(i.Account)\n\t\tif acc == nil || err != nil {\n\t\t\ts.Errorf(\"Can't locate account [%s] for import of [%v] %s (err=%v)\", i.Account, i.Subject, i.Type, err)\n\t\t\tincompleteImports = append(incompleteImports, i)\n\t\t\tcontinue\n\t\t}\n\t\t// Capture trace labels.\n\t\tacc.mu.RLock()\n\t\tatl := acc.traceLabel()\n\t\tacc.mu.RUnlock()\n\t\t// Grab from and to\n\t\tfrom, to := string(i.Subject), i.GetTo()\n\t\tswitch i.Type {\n\t\tcase jwt.Stream:\n\t\t\tif i.LocalSubject != _EMPTY_ {\n\t\t\t\t// set local subject implies to is empty\n\t\t\t\tto = string(i.LocalSubject)\n\t\t\t\ts.Debugf(\"Adding stream import %s:%q for %s:%q\", atl, from, tl, to)\n\t\t\t\terr = a.AddMappedStreamImportWithClaim(acc, from, to, i)\n\t\t\t} else {\n\t\t\t\ts.Debugf(\"Adding stream import %s:%q for %s:%q\", atl, from, tl, to)\n\t\t\t\terr = a.AddStreamImportWithClaim(acc, from, to, i)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\ts.Debugf(\"Error adding stream import to account [%s]: %v\", tl, err.Error())\n\t\t\t\tincompleteImports = append(incompleteImports, i)\n\t\t\t}\n\t\tcase jwt.Service:\n\t\t\tif i.LocalSubject != _EMPTY_ {\n\t\t\t\tfrom = string(i.LocalSubject)\n\t\t\t\tto = string(i.Subject)\n\t\t\t}\n\t\t\ts.Debugf(\"Adding service import %s:%q for %s:%q\", atl, from, tl, to)\n\t\t\tif err := a.AddServiceImportWithClaim(acc, from, to, i); err != nil {\n\t\t\t\ts.Debugf(\"Error adding service import to account [%s]: %v\", tl, err.Error())\n\t\t\t\tincompleteImports = append(incompleteImports, i)\n\t\t\t}\n\t\t}\n\t}\n\t// Now let's apply any needed changes from import/export changes.\n\tif !a.checkStreamImportsEqual(old) {\n\t\tawcsti := map[string]struct{}{a.Name: {}}\n\t\tfor _, c := range a.getClients() {\n\t\t\tc.processSubsOnConfigReload(awcsti)\n\t\t}\n\t}\n\t// Now check if stream exports have changed.\n\tif !a.checkStreamExportsEqual(old) || signersChanged || streamTokenExpirationChanged {\n\t\tclients := map[*client]struct{}{}\n\t\t// We need to check all accounts that have an import claim from this account.\n\t\tawcsti := map[string]struct{}{}\n\n\t\t// We must only allow one goroutine to go through here, otherwise we could deadlock\n\t\t// due to locking two accounts in succession.\n\t\ts.mu.Lock()\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\t// Move to the next if this account is actually account \"a\".\n\t\t\tif acc.Name == a.Name {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tacc.mu.Lock()\n\t\t\tfor _, im := range acc.imports.streams {\n\t\t\t\tif im != nil && im.acc.Name == a.Name {\n\t\t\t\t\t// Check for if we are still authorized for an import.\n\t\t\t\t\tim.invalid = !a.checkStreamImportAuthorized(acc, im.from, im.claim)\n\t\t\t\t\tawcsti[acc.Name] = struct{}{}\n\t\t\t\t\tfor c := range acc.clients {\n\t\t\t\t\t\tclients[c] = struct{}{}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tacc.mu.Unlock()\n\t\t\treturn true\n\t\t})\n\t\ts.mu.Unlock()\n\t\t// Now walk clients.\n\t\tfor c := range clients {\n\t\t\tc.processSubsOnConfigReload(awcsti)\n\t\t}\n\t}\n\t// Now check if service exports have changed.\n\tif !a.checkServiceExportsEqual(old) || signersChanged || serviceTokenExpirationChanged {\n\t\t// We must only allow one goroutine to go through here, otherwise we could deadlock\n\t\t// due to locking two accounts in succession.\n\t\ts.mu.Lock()\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\t// Move to the next if this account is actually account \"a\".\n\t\t\tif acc.Name == a.Name {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tacc.mu.Lock()\n\t\t\tfor _, sis := range acc.imports.services {\n\t\t\t\tfor _, si := range sis {\n\t\t\t\t\tif si != nil && si.acc.Name == a.Name {\n\t\t\t\t\t\t// Check for if we are still authorized for an import.\n\t\t\t\t\t\tsi.invalid = !a.checkServiceImportAuthorized(acc, si.to, si.claim)\n\t\t\t\t\t\t// Make sure we should still be tracking latency and if we\n\t\t\t\t\t\t// are allowed to trace.\n\t\t\t\t\t\tif !si.response {\n\t\t\t\t\t\t\ta.mu.RLock()\n\t\t\t\t\t\t\tif se := a.getServiceExport(si.to); se != nil {\n\t\t\t\t\t\t\t\tif si.latency != nil {\n\t\t\t\t\t\t\t\t\tsi.latency = se.latency\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t// Update allow trace.\n\t\t\t\t\t\t\t\tsi.atrc = se.atrc\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\ta.mu.RUnlock()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tacc.mu.Unlock()\n\t\t\treturn true\n\t\t})\n\t\ts.mu.Unlock()\n\t}\n\n\t// Now make sure we shutdown the old service import subscriptions.\n\tvar sids [][]byte\n\ta.mu.RLock()\n\tc := a.ic\n\tif c != nil {\n\t\tfor _, sis := range old.imports.services {\n\t\t\tfor _, si := range sis {\n\t\t\t\tif si.sid != nil {\n\t\t\t\t\tsids = append(sids, si.sid)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\ta.mu.RUnlock()\n\tfor _, sid := range sids {\n\t\tc.processUnsub(sid)\n\t}\n\n\t// Now do limits if they are present.\n\ta.mu.Lock()\n\ta.msubs = clampInt64ToInt32(ac.Limits.Subs)\n\ta.mpay = clampInt64ToInt32(ac.Limits.Payload)\n\ta.mconns = clampInt64ToInt32(ac.Limits.Conn)\n\ta.mleafs = clampInt64ToInt32(ac.Limits.LeafNodeConn)\n\ta.disallowBearer = ac.Limits.DisallowBearer\n\t// Check for any revocations\n\tif len(ac.Revocations) > 0 {\n\t\t// We will always replace whatever we had with most current, so no\n\t\t// need to look at what we have.\n\t\ta.usersRevoked = make(map[string]int64, len(ac.Revocations))\n\t\tfor pk, t := range ac.Revocations {\n\t\t\ta.usersRevoked[pk] = t\n\t\t}\n\t} else {\n\t\ta.usersRevoked = nil\n\t}\n\ta.defaultPerms = buildPermissionsFromJwt(&ac.DefaultPermissions)\n\ta.incomplete = len(incompleteImports) != 0\n\tfor _, i := range incompleteImports {\n\t\ts.incompleteAccExporterMap.Store(i.Account, struct{}{})\n\t}\n\tif a.srv == nil {\n\t\ta.srv = s\n\t}\n\n\tif ac.Limits.IsJSEnabled() {\n\t\ttoUnlimited := func(value int64) int64 {\n\t\t\tif value > 0 {\n\t\t\t\treturn value\n\t\t\t}\n\t\t\treturn -1\n\t\t}\n\t\tif ac.Limits.JetStreamLimits.DiskStorage != 0 || ac.Limits.JetStreamLimits.MemoryStorage != 0 {\n\t\t\t// JetStreamAccountLimits and jwt.JetStreamLimits use same value for unlimited\n\t\t\ta.jsLimits = map[string]JetStreamAccountLimits{\n\t\t\t\t_EMPTY_: {\n\t\t\t\t\tMaxMemory: ac.Limits.JetStreamLimits.MemoryStorage,\n\t\t\t\t\tMaxStore: ac.Limits.JetStreamLimits.DiskStorage,\n\t\t\t\t\tMaxStreams: int(ac.Limits.JetStreamLimits.Streams),\n\t\t\t\t\tMaxConsumers: int(ac.Limits.JetStreamLimits.Consumer),\n\t\t\t\t\tMemoryMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.MemoryMaxStreamBytes),\n\t\t\t\t\tStoreMaxStreamBytes: toUnlimited(ac.Limits.JetStreamLimits.DiskMaxStreamBytes),\n\t\t\t\t\tMaxBytesRequired: ac.Limits.JetStreamLimits.MaxBytesRequired,\n\t\t\t\t\tMaxAckPending: int(toUnlimited(ac.Limits.JetStreamLimits.MaxAckPending)),\n\t\t\t\t},\n\t\t\t}\n\t\t} else {\n\t\t\ta.jsLimits = map[string]JetStreamAccountLimits{}\n\t\t\tfor t, l := range ac.Limits.JetStreamTieredLimits {\n\t\t\t\ta.jsLimits[t] = JetStreamAccountLimits{\n\t\t\t\t\tMaxMemory: l.MemoryStorage,\n\t\t\t\t\tMaxStore: l.DiskStorage,\n\t\t\t\t\tMaxStreams: int(l.Streams),\n\t\t\t\t\tMaxConsumers: int(l.Consumer),\n\t\t\t\t\tMemoryMaxStreamBytes: toUnlimited(l.MemoryMaxStreamBytes),\n\t\t\t\t\tStoreMaxStreamBytes: toUnlimited(l.DiskMaxStreamBytes),\n\t\t\t\t\tMaxBytesRequired: l.MaxBytesRequired,\n\t\t\t\t\tMaxAckPending: int(toUnlimited(l.MaxAckPending)),\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else if a.jsLimits != nil {\n\t\t// covers failed update followed by disable\n\t\ta.jsLimits = nil\n\t}\n\n\ta.updated = time.Now()\n\tclients := a.getClientsLocked()\n\tajs := a.js\n\ta.mu.Unlock()\n\n\t// Sort in chronological order so that most recent connections over the limit are pruned.\n\tif a.MaxTotalConnectionsReached() {\n\t\tslices.SortFunc(clients, func(i, j *client) int { return i.start.Compare(j.start) })\n\t}\n\n\t// If JetStream is enabled for this server we will call into configJetStream for the account\n\t// regardless of enabled or disabled. It handles both cases.\n\tif jsEnabled {\n\t\tif err := s.configJetStream(a, nil); err != nil {\n\t\t\ts.Errorf(\"Error configuring jetstream for account [%s]: %v\", tl, err.Error())\n\t\t\ta.mu.Lock()\n\t\t\t// Absent reload of js server cfg, this is going to be broken until js is disabled\n\t\t\ta.incomplete = true\n\t\t\ta.mu.Unlock()\n\t\t} else {\n\t\t\ta.mu.Lock()\n\t\t\t// Refresh reference, we've just enabled JetStream, so it would have been nil before.\n\t\t\tajs = a.js\n\t\t\ta.mu.Unlock()\n\t\t}\n\t} else if a.jsLimits != nil {\n\t\t// We do not have JS enabled for this server, but the account has it enabled so setup\n\t\t// our imports properly. This allows this server to proxy JS traffic correctly.\n\t\ts.checkJetStreamExports()\n\t\ta.enableAllJetStreamServiceImportsAndMappings()\n\t}\n\n\tif ajs != nil {\n\t\t// Check whether the account NRG status changed. If it has then we need to notify the\n\t\t// Raft groups running on the system so that they can move their subs if needed.\n\t\ta.mu.Lock()\n\t\tprevious := a.nrgAccount\n\t\tswitch ac.ClusterTraffic {\n\t\tcase \"system\", _EMPTY_:\n\t\t\ta.nrgAccount = _EMPTY_\n\t\tcase \"owner\":\n\t\t\ta.nrgAccount = a.Name\n\t\tdefault:\n\t\t\ts.Errorf(\"Account claim for %q has invalid value %q for cluster traffic account\", a.Name, ac.ClusterTraffic)\n\t\t}\n\t\tchanged := a.nrgAccount != previous\n\t\ta.mu.Unlock()\n\t\tif changed {\n\t\t\ts.updateNRGAccountStatus()\n\t\t}\n\t}\n\n\t// client list is in chronological order (older cids at the beginning of the list).\n\tcount := 0\n\tfor _, c := range clients {\n\t\ta.mu.RLock()\n\t\texceeded := a.mconns != jwt.NoLimit && count >= int(a.mconns)\n\t\ta.mu.RUnlock()\n\t\t// Only kick non-internal clients.\n\t\tif !isInternalClient(c.kind) {\n\t\t\tif exceeded {\n\t\t\t\tc.maxAccountConnExceeded()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcount++\n\t\t}\n\t\tc.mu.Lock()\n\t\tc.applyAccountLimits()\n\t\t// if we have an nkey user we are a callout user - save\n\t\t// the issuedAt, and nkey user id to honor revocations\n\t\tvar nkeyUserID string\n\t\tvar issuedAt int64\n\t\tif c.user != nil {\n\t\t\tissuedAt = c.user.Issued\n\t\t\tnkeyUserID = c.user.Nkey\n\t\t}\n\t\ttheJWT := c.opts.JWT\n\t\tc.mu.Unlock()\n\t\t// Check for being revoked here. We use ac one to avoid the account lock.\n\t\tif (ac.Revocations != nil || ac.Limits.DisallowBearer) && theJWT != _EMPTY_ {\n\t\t\tif juc, err := jwt.DecodeUserClaims(theJWT); err != nil {\n\t\t\t\tc.Debugf(\"User JWT not valid: %v\", err)\n\t\t\t\tc.authViolation()\n\t\t\t\tcontinue\n\t\t\t} else if juc.BearerToken && ac.Limits.DisallowBearer {\n\t\t\t\tc.Debugf(\"Bearer User JWT not allowed\")\n\t\t\t\tc.authViolation()\n\t\t\t\tcontinue\n\t\t\t} else if ok := ac.IsClaimRevoked(juc); ok {\n\t\t\t\tc.sendErrAndDebug(\"User Authentication Revoked\")\n\t\t\t\tc.closeConnection(Revocation)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// if we extracted nkeyUserID and issuedAt we are a callout type\n\t\t// calloutIAT should only be set if we are in callout scenario as\n\t\t// the user JWT is _NOT_ associated with the client for callouts,\n\t\t// so we rely on the calloutIAT to know when the JWT was issued\n\t\t// revocations simply state that JWT issued before or by that date\n\t\t// are not valid\n\t\tif ac.Revocations != nil && nkeyUserID != _EMPTY_ && issuedAt > 0 {\n\t\t\tseconds, ok := ac.Revocations[jwt.All]\n\t\t\tif ok && seconds >= issuedAt {\n\t\t\t\tc.sendErrAndDebug(\"User Authentication Revoked\")\n\t\t\t\tc.closeConnection(Revocation)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tseconds, ok = ac.Revocations[nkeyUserID]\n\t\t\tif ok && seconds >= issuedAt {\n\t\t\t\tc.sendErrAndDebug(\"User Authentication Revoked\")\n\t\t\t\tc.closeConnection(Revocation)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check if the signing keys changed, might have to evict\n\tif signersChanged {\n\t\tfor _, c := range clients {\n\t\t\tc.mu.Lock()\n\t\t\tif c.user == nil {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsk := c.user.SigningKey\n\t\t\tc.mu.Unlock()\n\t\t\tif sk == _EMPTY_ {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif _, ok := alteredScope[sk]; ok {\n\t\t\t\tc.closeConnection(AuthenticationViolation)\n\t\t\t} else if _, ok := a.hasIssuer(sk); !ok {\n\t\t\t\tc.closeConnection(AuthenticationViolation)\n\t\t\t}\n\t\t}\n\t}\n\n\tif _, ok := s.incompleteAccExporterMap.Load(old.Name); ok && refreshImportingAccounts {\n\t\ts.incompleteAccExporterMap.Delete(old.Name)\n\t\ts.accounts.Range(func(key, value any) bool {\n\t\t\tacc := value.(*Account)\n\t\t\tacc.mu.RLock()\n\t\t\tincomplete := acc.incomplete\n\t\t\tname := acc.Name\n\t\t\tlabel := acc.traceLabel()\n\t\t\t// Must use jwt in account or risk failing on fetch\n\t\t\t// This jwt may not be the same that caused exportingAcc to be in incompleteAccExporterMap\n\t\t\tclaimJWT := acc.claimJWT\n\t\t\tacc.mu.RUnlock()\n\t\t\tif incomplete && name != old.Name {\n\t\t\t\tif accClaims, _, err := s.verifyAccountClaims(claimJWT); err == nil {\n\t\t\t\t\t// Since claimJWT has not changed, acc can become complete\n\t\t\t\t\t// but it won't alter incomplete for it's dependents accounts.\n\t\t\t\t\ts.updateAccountClaimsWithRefresh(acc, accClaims, false)\n\t\t\t\t\t// old.Name was deleted before ranging over accounts\n\t\t\t\t\t// If it exists again, UpdateAccountClaims set it for failed imports of acc.\n\t\t\t\t\t// So there was one import of acc that imported this account and failed again.\n\t\t\t\t\t// Since this account just got updated, the import itself may be in error. So trace that.\n\t\t\t\t\tif _, ok := s.incompleteAccExporterMap.Load(old.Name); ok {\n\t\t\t\t\t\ts.incompleteAccExporterMap.Delete(old.Name)\n\t\t\t\t\t\ts.Errorf(\"Account %s has issues importing account %s\", label, old.Name)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n}\n\n// Helper to build an internal account structure from a jwt.AccountClaims.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) buildInternalAccount(ac *jwt.AccountClaims) *Account {\n\tacc := NewAccount(ac.Subject)\n\tacc.Issuer = ac.Issuer\n\t// Set this here since we are placing in s.tmpAccounts below and may be\n\t// referenced by an route RS+, etc.\n\ts.setAccountSublist(acc)\n\n\t// We don't want to register an account that is in the process of\n\t// being built, however, to solve circular import dependencies, we\n\t// need to store it here.\n\tif v, loaded := s.tmpAccounts.LoadOrStore(ac.Subject, acc); loaded {\n\t\treturn v.(*Account)\n\t}\n\n\t// Update based on claims.\n\ts.UpdateAccountClaims(acc, ac)\n\n\treturn acc\n}\n\n// Helper to build Permissions from jwt.Permissions\n// or return nil if none were specified\nfunc buildPermissionsFromJwt(uc *jwt.Permissions) *Permissions {\n\tif uc == nil {\n\t\treturn nil\n\t}\n\tvar p *Permissions\n\tif len(uc.Pub.Allow) > 0 || len(uc.Pub.Deny) > 0 {\n\t\tp = &Permissions{}\n\t\tp.Publish = &SubjectPermission{}\n\t\tp.Publish.Allow = uc.Pub.Allow\n\t\tp.Publish.Deny = uc.Pub.Deny\n\t}\n\tif len(uc.Sub.Allow) > 0 || len(uc.Sub.Deny) > 0 {\n\t\tif p == nil {\n\t\t\tp = &Permissions{}\n\t\t}\n\t\tp.Subscribe = &SubjectPermission{}\n\t\tp.Subscribe.Allow = uc.Sub.Allow\n\t\tp.Subscribe.Deny = uc.Sub.Deny\n\t}\n\tif uc.Resp != nil {\n\t\tif p == nil {\n\t\t\tp = &Permissions{}\n\t\t}\n\t\tp.Response = &ResponsePermission{\n\t\t\tMaxMsgs: uc.Resp.MaxMsgs,\n\t\t\tExpires: uc.Resp.Expires,\n\t\t}\n\t\tvalidateResponsePermissions(p)\n\t}\n\treturn p\n}\n\n// Helper to build internal NKeyUser.\nfunc buildInternalNkeyUser(uc *jwt.UserClaims, acts map[string]struct{}, acc *Account) *NkeyUser {\n\tnu := &NkeyUser{Nkey: uc.Subject, Account: acc, AllowedConnectionTypes: acts, Issued: uc.IssuedAt}\n\tif uc.IssuerAccount != _EMPTY_ {\n\t\tnu.SigningKey = uc.Issuer\n\t}\n\n\t// Now check for permissions.\n\tvar p = buildPermissionsFromJwt(&uc.Permissions)\n\tif p == nil && acc.defaultPerms != nil {\n\t\tp = acc.defaultPerms.clone()\n\t}\n\tnu.Permissions = p\n\treturn nu\n}\n\nfunc fetchAccount(res AccountResolver, name string) (string, error) {\n\tif !nkeys.IsValidPublicAccountKey(name) {\n\t\treturn _EMPTY_, fmt.Errorf(\"will only fetch valid account keys\")\n\t}\n\treturn res.Fetch(copyString(name))\n}\n\n// AccountResolver interface. This is to fetch Account JWTs by public nkeys\ntype AccountResolver interface {\n\tFetch(name string) (string, error)\n\tStore(name, jwt string) error\n\tIsReadOnly() bool\n\tStart(server *Server) error\n\tIsTrackingUpdate() bool\n\tReload() error\n\tClose()\n}\n\n// Default implementations of IsReadOnly/Start so only need to be written when changed\ntype resolverDefaultsOpsImpl struct{}\n\nfunc (*resolverDefaultsOpsImpl) IsReadOnly() bool {\n\treturn true\n}\n\nfunc (*resolverDefaultsOpsImpl) IsTrackingUpdate() bool {\n\treturn false\n}\n\nfunc (*resolverDefaultsOpsImpl) Start(*Server) error {\n\treturn nil\n}\n\nfunc (*resolverDefaultsOpsImpl) Reload() error {\n\treturn nil\n}\n\nfunc (*resolverDefaultsOpsImpl) Close() {\n}\n\nfunc (*resolverDefaultsOpsImpl) Store(_, _ string) error {\n\treturn fmt.Errorf(\"store operation not supported for URL Resolver\")\n}\n\n// MemAccResolver is a memory only resolver.\n// Mostly for testing.\ntype MemAccResolver struct {\n\tsm sync.Map\n\tresolverDefaultsOpsImpl\n}\n\n// Fetch will fetch the account jwt claims from the internal sync.Map.\nfunc (m *MemAccResolver) Fetch(name string) (string, error) {\n\tif j, ok := m.sm.Load(name); ok {\n\t\treturn j.(string), nil\n\t}\n\treturn _EMPTY_, ErrMissingAccount\n}\n\n// Store will store the account jwt claims in the internal sync.Map.\nfunc (m *MemAccResolver) Store(name, jwt string) error {\n\tm.sm.Store(name, jwt)\n\treturn nil\n}\n\nfunc (m *MemAccResolver) IsReadOnly() bool {\n\treturn false\n}\n\n// URLAccResolver implements an http fetcher.\ntype URLAccResolver struct {\n\turl string\n\tc *http.Client\n\tresolverDefaultsOpsImpl\n}\n\n// NewURLAccResolver returns a new resolver for the given base URL.\nfunc NewURLAccResolver(url string) (*URLAccResolver, error) {\n\tif !strings.HasSuffix(url, \"/\") {\n\t\turl += \"/\"\n\t}\n\t// FIXME(dlc) - Make timeout and others configurable.\n\t// We create our own transport to amortize TLS.\n\ttr := &http.Transport{\n\t\tMaxIdleConns: 10,\n\t\tIdleConnTimeout: 30 * time.Second,\n\t}\n\tur := &URLAccResolver{\n\t\turl: url,\n\t\tc: &http.Client{Timeout: DEFAULT_ACCOUNT_FETCH_TIMEOUT, Transport: tr},\n\t}\n\treturn ur, nil\n}\n\n// Fetch will fetch the account jwt claims from the base url, appending the\n// account name onto the end.\nfunc (ur *URLAccResolver) Fetch(name string) (string, error) {\n\turl := ur.url + name\n\tresp, err := ur.c.Get(url)\n\tif err != nil {\n\t\treturn _EMPTY_, fmt.Errorf(\"could not fetch <%q>: %v\", redactURLString(url), err)\n\t} else if resp == nil {\n\t\treturn _EMPTY_, fmt.Errorf(\"could not fetch <%q>: no response\", redactURLString(url))\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn _EMPTY_, fmt.Errorf(\"could not fetch <%q>: %v\", redactURLString(url), resp.Status)\n\t}\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn _EMPTY_, err\n\t}\n\treturn string(body), nil\n}\n\n// Resolver based on nats for synchronization and backing directory for storage.\ntype DirAccResolver struct {\n\t*DirJWTStore\n\t*Server\n\tsyncInterval time.Duration\n\tfetchTimeout time.Duration\n}\n\nfunc (dr *DirAccResolver) IsTrackingUpdate() bool {\n\treturn true\n}\n\nfunc (dr *DirAccResolver) Reload() error {\n\treturn dr.DirJWTStore.Reload()\n}\n\n// ServerAPIClaimUpdateResponse is the response to $SYS.REQ.ACCOUNT..CLAIMS.UPDATE and $SYS.REQ.CLAIMS.UPDATE\ntype ServerAPIClaimUpdateResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *ClaimUpdateStatus `json:\"data,omitempty\"`\n\tError *ClaimUpdateError `json:\"error,omitempty\"`\n}\n\ntype ClaimUpdateError struct {\n\tAccount string `json:\"account,omitempty\"`\n\tCode int `json:\"code\"`\n\tDescription string `json:\"description,omitempty\"`\n}\n\ntype ClaimUpdateStatus struct {\n\tAccount string `json:\"account,omitempty\"`\n\tCode int `json:\"code,omitempty\"`\n\tMessage string `json:\"message,omitempty\"`\n}\n\nfunc respondToUpdate(s *Server, respSubj string, acc string, message string, err error) {\n\tif err == nil {\n\t\tif acc == _EMPTY_ {\n\t\t\ts.Debugf(\"%s\", message)\n\t\t} else {\n\t\t\ts.Debugf(\"%s - %s\", message, acc)\n\t\t}\n\t} else {\n\t\tif acc == _EMPTY_ {\n\t\t\ts.Errorf(\"%s - %s\", message, err)\n\t\t} else {\n\t\t\ts.Errorf(\"%s - %s - %s\", message, acc, err)\n\t\t}\n\t}\n\tif respSubj == _EMPTY_ {\n\t\treturn\n\t}\n\n\tresponse := ServerAPIClaimUpdateResponse{\n\t\tServer: &ServerInfo{},\n\t}\n\n\tif err == nil {\n\t\tresponse.Data = &ClaimUpdateStatus{\n\t\t\tAccount: acc,\n\t\t\tCode: http.StatusOK,\n\t\t\tMessage: message,\n\t\t}\n\t} else {\n\t\tresponse.Error = &ClaimUpdateError{\n\t\t\tAccount: acc,\n\t\t\tCode: http.StatusInternalServerError,\n\t\t\tDescription: fmt.Sprintf(\"%s - %v\", message, err),\n\t\t}\n\t}\n\n\ts.sendInternalMsgLocked(respSubj, _EMPTY_, response.Server, response)\n}\n\nfunc handleListRequest(store *DirJWTStore, s *Server, reply string) {\n\tif reply == _EMPTY_ {\n\t\treturn\n\t}\n\taccIds := make([]string, 0, 1024)\n\tif err := store.PackWalk(1, func(partialPackMsg string) {\n\t\tif tk := strings.Split(partialPackMsg, \"|\"); len(tk) == 2 {\n\t\t\taccIds = append(accIds, tk[0])\n\t\t}\n\t}); err != nil {\n\t\t// let them timeout\n\t\ts.Errorf(\"list request error: %v\", err)\n\t} else {\n\t\ts.Debugf(\"list request responded with %d account ids\", len(accIds))\n\t\tserver := &ServerInfo{}\n\t\tresponse := map[string]any{\"server\": server, \"data\": accIds}\n\t\ts.sendInternalMsgLocked(reply, _EMPTY_, server, response)\n\t}\n}\n\nfunc handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) {\n\tvar accIds []any\n\tvar subj, sysAccName string\n\tif sysAcc := s.SystemAccount(); sysAcc != nil {\n\t\tsysAccName = sysAcc.GetName()\n\t}\n\t// Only operator and operator signing key are allowed to delete\n\tgk, err := jwt.DecodeGeneric(string(msg))\n\tif err == nil {\n\t\tsubj = gk.Subject\n\t\tif store.deleteType == NoDelete {\n\t\t\terr = fmt.Errorf(\"delete must be enabled in server config\")\n\t\t} else if subj != gk.Issuer {\n\t\t\terr = fmt.Errorf(\"not self signed\")\n\t\t} else if _, ok := store.operator[gk.Issuer]; !ok {\n\t\t\terr = fmt.Errorf(\"not trusted\")\n\t\t} else if list, ok := gk.Data[\"accounts\"]; !ok {\n\t\t\terr = fmt.Errorf(\"malformed request\")\n\t\t} else if accIds, ok = list.([]any); !ok {\n\t\t\terr = fmt.Errorf(\"malformed request\")\n\t\t} else {\n\t\t\tfor _, entry := range accIds {\n\t\t\t\tif acc, ok := entry.(string); !ok ||\n\t\t\t\t\tacc == _EMPTY_ || !nkeys.IsValidPublicAccountKey(acc) {\n\t\t\t\t\terr = fmt.Errorf(\"malformed request\")\n\t\t\t\t\tbreak\n\t\t\t\t} else if acc == sysAccName {\n\t\t\t\t\terr = fmt.Errorf(\"not allowed to delete system account\")\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil {\n\t\trespondToUpdate(s, reply, _EMPTY_, fmt.Sprintf(\"delete accounts request by %s failed\", subj), err)\n\t\treturn\n\t}\n\terrs := []string{}\n\tpassCnt := 0\n\tfor _, acc := range accIds {\n\t\tif err := store.delete(acc.(string)); err != nil {\n\t\t\terrs = append(errs, err.Error())\n\t\t} else {\n\t\t\tpassCnt++\n\t\t}\n\t}\n\tif len(errs) == 0 {\n\t\trespondToUpdate(s, reply, _EMPTY_, fmt.Sprintf(\"deleted %d accounts\", passCnt), nil)\n\t} else {\n\t\trespondToUpdate(s, reply, _EMPTY_, fmt.Sprintf(\"deleted %d accounts, failed for %d\", passCnt, len(errs)),\n\t\t\terrors.New(strings.Join(errs, \"\\n\")))\n\t}\n}\n\nfunc getOperatorKeys(s *Server) (string, map[string]struct{}, bool, error) {\n\tvar op string\n\tvar strict bool\n\tkeys := make(map[string]struct{})\n\tif opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 {\n\t\top = opts.TrustedOperators[0].Subject\n\t\tstrict = opts.TrustedOperators[0].StrictSigningKeyUsage\n\t\tif !strict {\n\t\t\tkeys[opts.TrustedOperators[0].Subject] = struct{}{}\n\t\t}\n\t\tfor _, key := range opts.TrustedOperators[0].SigningKeys {\n\t\t\tkeys[key] = struct{}{}\n\t\t}\n\t}\n\tif len(keys) == 0 {\n\t\treturn _EMPTY_, nil, false, fmt.Errorf(\"no operator key found\")\n\t}\n\treturn op, keys, strict, nil\n}\n\nfunc claimValidate(claim *jwt.AccountClaims) error {\n\tvr := &jwt.ValidationResults{}\n\tclaim.Validate(vr)\n\tif vr.IsBlocking(false) {\n\t\treturn fmt.Errorf(\"validation errors: %v\", vr.Errors())\n\t}\n\treturn nil\n}\n\nfunc removeCb(s *Server, pubKey string) {\n\tv, ok := s.accounts.Load(pubKey)\n\tif !ok {\n\t\treturn\n\t}\n\ta := v.(*Account)\n\ts.Debugf(\"Disable account %s due to remove\", pubKey)\n\ta.mu.Lock()\n\t// lock out new clients\n\ta.msubs = 0\n\ta.mpay = 0\n\ta.mconns = 0\n\ta.mleafs = 0\n\ta.updated = time.Now()\n\tjsa := a.js\n\ta.mu.Unlock()\n\t// set the account to be expired and disconnect clients\n\ta.expiredTimeout()\n\t// For JS, we need also to disable it.\n\tif js := s.getJetStream(); js != nil && jsa != nil {\n\t\tjs.disableJetStream(jsa)\n\t\t// Remove JetStream state in memory, this will be reset\n\t\t// on the changed callback from the account in case it is\n\t\t// enabled again.\n\t\ta.js = nil\n\t}\n\t// We also need to remove all ServerImport subscriptions\n\ta.removeAllServiceImportSubs()\n\ta.mu.Lock()\n\ta.clearExpirationTimer()\n\ta.mu.Unlock()\n}\n\nfunc (dr *DirAccResolver) Start(s *Server) error {\n\top, opKeys, strict, err := getOperatorKeys(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdr.Lock()\n\tdefer dr.Unlock()\n\tdr.Server = s\n\tdr.operator = opKeys\n\tdr.DirJWTStore.changed = func(pubKey string) {\n\t\tif v, ok := s.accounts.Load(pubKey); ok {\n\t\t\tif theJwt, err := dr.LoadAcc(pubKey); err != nil {\n\t\t\t\ts.Errorf(\"DirResolver - Update got error on load: %v\", err)\n\t\t\t} else {\n\t\t\t\tacc := v.(*Account)\n\t\t\t\tif err = s.updateAccountWithClaimJWT(acc, theJwt); err != nil {\n\t\t\t\t\ts.Errorf(\"DirResolver - Update for account %q resulted in error %v\", pubKey, err)\n\t\t\t\t} else {\n\t\t\t\t\tif _, jsa, err := acc.checkForJetStream(); err != nil {\n\t\t\t\t\t\tif !IsNatsErr(err, JSNotEnabledForAccountErr) {\n\t\t\t\t\t\t\ts.Warnf(\"DirResolver - Error checking for JetStream support for account %q: %v\", pubKey, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if jsa == nil {\n\t\t\t\t\t\tif err = s.configJetStream(acc, nil); err != nil {\n\t\t\t\t\t\t\ts.Errorf(\"DirResolver - Error configuring JetStream for account %q: %v\", pubKey, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tdr.DirJWTStore.deleted = func(pubKey string) {\n\t\tremoveCb(s, pubKey)\n\t}\n\tpackRespIb := s.newRespInbox()\n\tfor _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {\n\t\t// subscribe to account jwt update requests\n\t\tif _, err := s.sysSubscribe(fmt.Sprintf(reqSub, \"*\"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {\n\t\t\tvar pubKey string\n\t\t\ttk := strings.Split(subj, tsep)\n\t\t\tif len(tk) == accUpdateTokensNew {\n\t\t\t\tpubKey = tk[accReqAccIndex]\n\t\t\t} else if len(tk) == accUpdateTokensOld {\n\t\t\t\tpubKey = tk[accUpdateAccIdxOld]\n\t\t\t} else {\n\t\t\t\ts.Debugf(\"DirResolver - jwt update skipped due to bad subject %q\", subj)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {\n\t\t\t\trespondToUpdate(s, resp, \"n/a\", \"jwt update resulted in error\", err)\n\t\t\t} else if err := claimValidate(claim); err != nil {\n\t\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt validation failed\", err)\n\t\t\t} else if claim.Subject != pubKey {\n\t\t\t\terr := errors.New(\"subject does not match jwt content\")\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t\t\t} else if claim.Issuer == op && strict {\n\t\t\t\terr := errors.New(\"operator requires issuer to be a signing key\")\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t\t\t} else if err := dr.save(pubKey, string(msg)); err != nil {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t\t\t} else {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt updated\", nil)\n\t\t\t}\n\t\t}); err != nil {\n\t\t\treturn fmt.Errorf(\"error setting up update handling: %v\", err)\n\t\t}\n\t}\n\tif _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {\n\t\t// As this is a raw message, we need to extract payload and only decode claims from it,\n\t\t// in case request is sent with headers.\n\t\t_, msg = c.msgParts(msg)\n\t\tif claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {\n\t\t\trespondToUpdate(s, resp, \"n/a\", \"jwt update resulted in error\", err)\n\t\t} else if claim.Issuer == op && strict {\n\t\t\terr := errors.New(\"operator requires issuer to be a signing key\")\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update resulted in error\", err)\n\t\t} else if err := claimValidate(claim); err != nil {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt validation failed\", err)\n\t\t} else if err := dr.save(claim.Subject, string(msg)); err != nil {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update resulted in error\", err)\n\t\t} else {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt updated\", nil)\n\t\t}\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up update handling: %v\", err)\n\t}\n\t// respond to lookups with our version\n\tif _, err := s.sysSubscribe(fmt.Sprintf(accLookupReqSubj, \"*\"), func(_ *subscription, _ *client, _ *Account, subj, reply string, msg []byte) {\n\t\tif reply == _EMPTY_ {\n\t\t\treturn\n\t\t}\n\t\ttk := strings.Split(subj, tsep)\n\t\tif len(tk) != accLookupReqTokens {\n\t\t\treturn\n\t\t}\n\t\taccName := tk[accReqAccIndex]\n\t\tif theJWT, err := dr.DirJWTStore.LoadAcc(accName); err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\ts.Debugf(\"DirResolver - Could not find account %q\", accName)\n\t\t\t\t// Reply with empty response to signal absence of JWT to others.\n\t\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, nil)\n\t\t\t} else {\n\t\t\t\ts.Errorf(\"DirResolver - Error looking up account %q: %v\", accName, err)\n\t\t\t}\n\t\t} else {\n\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(theJWT))\n\t\t}\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up lookup request handling: %v\", err)\n\t}\n\t// respond to pack requests with one or more pack messages\n\t// an empty message signifies the end of the response responder.\n\tif _, err := s.sysSubscribeQ(accPackReqSubj, \"responder\", func(_ *subscription, _ *client, _ *Account, _, reply string, theirHash []byte) {\n\t\tif reply == _EMPTY_ {\n\t\t\treturn\n\t\t}\n\t\tourHash := dr.DirJWTStore.Hash()\n\t\tif bytes.Equal(theirHash, ourHash[:]) {\n\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})\n\t\t\ts.Debugf(\"DirResolver - Pack request matches hash %x\", ourHash[:])\n\t\t} else if err := dr.DirJWTStore.PackWalk(1, func(partialPackMsg string) {\n\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte(partialPackMsg))\n\t\t}); err != nil {\n\t\t\t// let them timeout\n\t\t\ts.Errorf(\"DirResolver - Pack request error: %v\", err)\n\t\t} else {\n\t\t\ts.Debugf(\"DirResolver - Pack request hash %x - finished responding with hash %x\", theirHash, ourHash)\n\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, []byte{})\n\t\t}\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up pack request handling: %v\", err)\n\t}\n\t// respond to list requests with one message containing all account ids\n\tif _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {\n\t\thandleListRequest(dr.DirJWTStore, s, reply)\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up list request handling: %v\", err)\n\t}\n\tif _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\t\t// As this is a raw message, we need to extract payload and only decode claims from it,\n\t\t// in case request is sent with headers.\n\t\t_, msg = c.msgParts(msg)\n\t\thandleDeleteRequest(dr.DirJWTStore, s, msg, reply)\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up delete request handling: %v\", err)\n\t}\n\t// embed pack responses into store\n\tif _, err := s.sysSubscribe(packRespIb, func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) {\n\t\thash := dr.DirJWTStore.Hash()\n\t\tif len(msg) == 0 { // end of response stream\n\t\t\ts.Debugf(\"DirResolver - Merging finished and resulting in: %x\", dr.DirJWTStore.Hash())\n\t\t\treturn\n\t\t} else if err := dr.DirJWTStore.Merge(string(msg)); err != nil {\n\t\t\ts.Errorf(\"DirResolver - Merging resulted in error: %v\", err)\n\t\t} else {\n\t\t\ts.Debugf(\"DirResolver - Merging succeeded and changed %x to %x\", hash, dr.DirJWTStore.Hash())\n\t\t}\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up pack response handling: %v\", err)\n\t}\n\t// periodically send out pack message\n\tquit := s.quitCh\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\t\tticker := time.NewTicker(dr.syncInterval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-quit:\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t}\n\t\t\tourHash := dr.DirJWTStore.Hash()\n\t\t\ts.Debugf(\"DirResolver - Checking store state: %x\", ourHash)\n\t\t\ts.sendInternalMsgLocked(accPackReqSubj, packRespIb, nil, ourHash[:])\n\t\t}\n\t})\n\ts.Noticef(\"Managing all jwt in exclusive directory %s\", dr.directory)\n\treturn nil\n}\n\nfunc (dr *DirAccResolver) Fetch(name string) (string, error) {\n\tif theJWT, err := dr.LoadAcc(name); theJWT != _EMPTY_ {\n\t\treturn theJWT, nil\n\t} else {\n\t\tdr.Lock()\n\t\tsrv := dr.Server\n\t\tto := dr.fetchTimeout\n\t\tdr.Unlock()\n\t\tif srv == nil {\n\t\t\treturn _EMPTY_, err\n\t\t}\n\t\treturn srv.fetch(dr, name, to) // lookup from other server\n\t}\n}\n\nfunc (dr *DirAccResolver) Store(name, jwt string) error {\n\treturn dr.saveIfNewer(name, jwt)\n}\n\ntype DirResOption func(s *DirAccResolver) error\n\n// limits the amount of time spent waiting for an account fetch to complete\nfunc FetchTimeout(to time.Duration) DirResOption {\n\treturn func(r *DirAccResolver) error {\n\t\tif to <= time.Duration(0) {\n\t\t\treturn fmt.Errorf(\"Fetch timeout %v is too smal\", to)\n\t\t}\n\t\tr.fetchTimeout = to\n\t\treturn nil\n\t}\n}\n\nfunc (dr *DirAccResolver) apply(opts ...DirResOption) error {\n\tfor _, o := range opts {\n\t\tif err := o(dr); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete deleteType, opts ...DirResOption) (*DirAccResolver, error) {\n\tif limit == 0 {\n\t\tlimit = math.MaxInt64\n\t}\n\tif syncInterval <= 0 {\n\t\tsyncInterval = time.Minute\n\t}\n\tstore, err := NewExpiringDirJWTStore(path, false, true, delete, 0, limit, false, 0, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tres := &DirAccResolver{store, nil, syncInterval, DEFAULT_ACCOUNT_FETCH_TIMEOUT}\n\tif err := res.apply(opts...); err != nil {\n\t\treturn nil, err\n\t}\n\treturn res, nil\n}\n\n// Caching resolver using nats for lookups and making use of a directory for storage\ntype CacheDirAccResolver struct {\n\tDirAccResolver\n\tttl time.Duration\n}\n\nfunc (s *Server) fetch(res AccountResolver, name string, timeout time.Duration) (string, error) {\n\tif s == nil {\n\t\treturn _EMPTY_, ErrNoAccountResolver\n\t}\n\trespC := make(chan []byte, 1)\n\taccountLookupRequest := fmt.Sprintf(accLookupReqSubj, name)\n\ts.mu.Lock()\n\tif s.sys == nil || s.sys.replies == nil {\n\t\ts.mu.Unlock()\n\t\treturn _EMPTY_, fmt.Errorf(\"eventing shut down\")\n\t}\n\t// Resolver will wait for detected active servers to reply\n\t// before serving an error in case there weren't any found.\n\texpectedServers := len(s.sys.servers)\n\treplySubj := s.newRespInbox()\n\treplies := s.sys.replies\n\n\t// Store our handler.\n\treplies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {\n\t\tvar clone []byte\n\t\tisEmpty := len(msg) == 0\n\t\tif !isEmpty {\n\t\t\tclone = make([]byte, len(msg))\n\t\t\tcopy(clone, msg)\n\t\t}\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\texpectedServers--\n\t\t// Skip empty responses until getting all the available servers.\n\t\tif isEmpty && expectedServers > 0 {\n\t\t\treturn\n\t\t}\n\t\t// Use the first valid response if there is still interest or\n\t\t// one of the empty responses to signal that it was not found.\n\t\tif _, ok := replies[replySubj]; ok {\n\t\t\tselect {\n\t\t\tcase respC <- clone:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}\n\ts.sendInternalMsg(accountLookupRequest, replySubj, nil, []byte{})\n\tquit := s.quitCh\n\ts.mu.Unlock()\n\tvar err error\n\tvar theJWT string\n\tselect {\n\tcase <-quit:\n\t\terr = errors.New(\"fetching jwt failed due to shutdown\")\n\tcase <-time.After(timeout):\n\t\terr = errors.New(\"fetching jwt timed out\")\n\tcase m := <-respC:\n\t\tif len(m) == 0 {\n\t\t\terr = errors.New(\"account jwt not found\")\n\t\t} else if err = res.Store(name, string(m)); err == nil {\n\t\t\ttheJWT = string(m)\n\t\t}\n\t}\n\ts.mu.Lock()\n\tdelete(replies, replySubj)\n\ts.mu.Unlock()\n\tclose(respC)\n\treturn theJWT, err\n}\n\nfunc NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, opts ...DirResOption) (*CacheDirAccResolver, error) {\n\tif limit <= 0 {\n\t\tlimit = 1_000\n\t}\n\tstore, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tres := &CacheDirAccResolver{DirAccResolver{store, nil, 0, DEFAULT_ACCOUNT_FETCH_TIMEOUT}, ttl}\n\tif err := res.apply(opts...); err != nil {\n\t\treturn nil, err\n\t}\n\treturn res, nil\n}\n\nfunc (dr *CacheDirAccResolver) Start(s *Server) error {\n\top, opKeys, strict, err := getOperatorKeys(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdr.Lock()\n\tdefer dr.Unlock()\n\tdr.Server = s\n\tdr.operator = opKeys\n\tdr.DirJWTStore.changed = func(pubKey string) {\n\t\tif v, ok := s.accounts.Load(pubKey); !ok {\n\t\t} else if theJwt, err := dr.LoadAcc(pubKey); err != nil {\n\t\t\ts.Errorf(\"DirResolver - Update got error on load: %v\", err)\n\t\t} else if err := s.updateAccountWithClaimJWT(v.(*Account), theJwt); err != nil {\n\t\t\ts.Errorf(\"DirResolver - Update resulted in error %v\", err)\n\t\t}\n\t}\n\tdr.DirJWTStore.deleted = func(pubKey string) {\n\t\tremoveCb(s, pubKey)\n\t}\n\tfor _, reqSub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {\n\t\t// subscribe to account jwt update requests\n\t\tif _, err := s.sysSubscribe(fmt.Sprintf(reqSub, \"*\"), func(_ *subscription, _ *client, _ *Account, subj, resp string, msg []byte) {\n\t\t\tvar pubKey string\n\t\t\ttk := strings.Split(subj, tsep)\n\t\t\tif len(tk) == accUpdateTokensNew {\n\t\t\t\tpubKey = tk[accReqAccIndex]\n\t\t\t} else if len(tk) == accUpdateTokensOld {\n\t\t\t\tpubKey = tk[accUpdateAccIdxOld]\n\t\t\t} else {\n\t\t\t\ts.Debugf(\"DirResolver - jwt update cache skipped due to bad subject %q\", subj)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update cache resulted in error\", err)\n\t\t\t} else if claim.Subject != pubKey {\n\t\t\t\terr := errors.New(\"subject does not match jwt content\")\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update cache resulted in error\", err)\n\t\t\t} else if claim.Issuer == op && strict {\n\t\t\t\terr := errors.New(\"operator requires issuer to be a signing key\")\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update cache resulted in error\", err)\n\t\t\t} else if _, ok := s.accounts.Load(pubKey); !ok {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update cache skipped\", nil)\n\t\t\t} else if err := claimValidate(claim); err != nil {\n\t\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update cache validation failed\", err)\n\t\t\t} else if err := dr.save(pubKey, string(msg)); err != nil {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt update cache resulted in error\", err)\n\t\t\t} else {\n\t\t\t\trespondToUpdate(s, resp, pubKey, \"jwt updated cache\", nil)\n\t\t\t}\n\t\t}); err != nil {\n\t\t\treturn fmt.Errorf(\"error setting up update handling: %v\", err)\n\t\t}\n\t}\n\tif _, err := s.sysSubscribe(accClaimsReqSubj, func(_ *subscription, c *client, _ *Account, _, resp string, msg []byte) {\n\t\t// As this is a raw message, we need to extract payload and only decode claims from it,\n\t\t// in case request is sent with headers.\n\t\t_, msg = c.msgParts(msg)\n\t\tif claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {\n\t\t\trespondToUpdate(s, resp, \"n/a\", \"jwt update cache resulted in error\", err)\n\t\t} else if claim.Issuer == op && strict {\n\t\t\terr := errors.New(\"operator requires issuer to be a signing key\")\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update cache resulted in error\", err)\n\t\t} else if _, ok := s.accounts.Load(claim.Subject); !ok {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update cache skipped\", nil)\n\t\t} else if err := claimValidate(claim); err != nil {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update cache validation failed\", err)\n\t\t} else if err := dr.save(claim.Subject, string(msg)); err != nil {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt update cache resulted in error\", err)\n\t\t} else {\n\t\t\trespondToUpdate(s, resp, claim.Subject, \"jwt updated cache\", nil)\n\t\t}\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up update handling: %v\", err)\n\t}\n\t// respond to list requests with one message containing all account ids\n\tif _, err := s.sysSubscribe(accListReqSubj, func(_ *subscription, _ *client, _ *Account, _, reply string, _ []byte) {\n\t\thandleListRequest(dr.DirJWTStore, s, reply)\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up list request handling: %v\", err)\n\t}\n\tif _, err := s.sysSubscribe(accDeleteReqSubj, func(_ *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\t\t// As this is a raw message, we need to extract payload and only decode claims from it,\n\t\t// in case request is sent with headers.\n\t\t_, msg = c.msgParts(msg)\n\t\thandleDeleteRequest(dr.DirJWTStore, s, msg, reply)\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error setting up list request handling: %v\", err)\n\t}\n\ts.Noticef(\"Managing some jwt in exclusive directory %s\", dr.directory)\n\treturn nil\n}\n\nfunc (dr *CacheDirAccResolver) Reload() error {\n\treturn dr.DirAccResolver.Reload()\n}\n" }, { "path": "server/accounts_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc simpleAccountServer(t *testing.T) (*Server, *Account, *Account) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\t// Now create two accounts.\n\tf, err := s.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account 'foo': %v\", err)\n\t}\n\tb, err := s.RegisterAccount(\"$bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account 'bar': %v\", err)\n\t}\n\treturn s, f, b\n}\n\nfunc TestRegisterDuplicateAccounts(t *testing.T) {\n\ts, _, _ := simpleAccountServer(t)\n\tif _, err := s.RegisterAccount(\"$foo\"); err == nil {\n\t\tt.Fatal(\"Expected an error registering 'foo' twice\")\n\t}\n}\n\nfunc TestAccountIsolation(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tcfoo, crFoo, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error register client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error register client with 'bar' account: %v\", err)\n\t}\n\n\t// Make sure they are different accounts/sl.\n\tif cfoo.acc == cbar.acc {\n\t\tt.Fatalf(\"Error, accounts the same for both clients\")\n\t}\n\n\t// Now do quick test that makes sure messages do not cross over.\n\t// setup bar as a foo subscriber.\n\tcbar.parseAsync(\"SUB foo 1\\r\\nPING\\r\\nPING\\r\\n\")\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response incorrect: %q\", l)\n\t}\n\n\tcfoo.parseAsync(\"SUB foo 1\\r\\nPUB foo 5\\r\\nhello\\r\\nPING\\r\\n\")\n\tl, err = crFoo.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'foo' from server: %v\", err)\n\t}\n\n\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crFoo, []byte(\"hello\\r\\n\"), t)\n\n\t// Now make sure nothing shows up on bar.\n\tl, err = crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response incorrect: %q\", l)\n\t}\n}\n\nfunc TestAccountIsolationExportImport(t *testing.T) {\n\tcheckIsolation := func(t *testing.T, pubSubj string, ncExp, ncImp *nats.Conn) {\n\t\t// We keep track of 2 subjects.\n\t\t// One subject (pubSubj) is based off the stream import.\n\t\t// The other subject \"fizz\" is not imported and should be isolated.\n\n\t\tgotSubjs := map[string]int{\n\t\t\tpubSubj: 0,\n\t\t\t\"fizz\": 0,\n\t\t}\n\t\tcount := int32(0)\n\t\tch := make(chan struct{}, 1)\n\t\tif _, err := ncImp.Subscribe(\">\", func(m *nats.Msg) {\n\t\t\tgotSubjs[m.Subject] += 1\n\t\t\tif n := atomic.AddInt32(&count, 1); n == 3 {\n\t\t\t\tch <- struct{}{}\n\t\t\t}\n\t\t}); err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\t// Since both prod and cons use same server, flushing here will ensure\n\t\t// that the interest is registered and known at the time we publish.\n\t\tncImp.Flush()\n\n\t\tif err := ncExp.Publish(pubSubj, []byte(fmt.Sprintf(\"ncExp pub %s\", pubSubj))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif err := ncImp.Publish(pubSubj, []byte(fmt.Sprintf(\"ncImp pub %s\", pubSubj))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif err := ncExp.Publish(\"fizz\", []byte(\"ncExp pub fizz\")); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif err := ncImp.Publish(\"fizz\", []byte(\"ncImp pub fizz\")); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\twantSubjs := map[string]int{\n\t\t\t// Subscriber ncImp should receive publishes from ncExp and ncImp.\n\t\t\tpubSubj: 2,\n\t\t\t// Subscriber ncImp should only receive the publish from ncImp.\n\t\t\t\"fizz\": 1,\n\t\t}\n\n\t\t// Wait for at least the 3 expected messages\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"Expected 3 messages, got %v\", atomic.LoadInt32(&count))\n\t\t}\n\t\t// But now wait a bit to see if subscription receives more than expected.\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\tif got, want := len(gotSubjs), len(wantSubjs); got != want {\n\t\t\tt.Fatalf(\"unexpected subjs len, got=%d; want=%d\", got, want)\n\t\t}\n\n\t\tfor key, gotCnt := range gotSubjs {\n\t\t\tif wantCnt := wantSubjs[key]; gotCnt != wantCnt {\n\t\t\t\tt.Errorf(\"unexpected receive count for subject %q, got=%d, want=%d\", key, gotCnt, wantCnt)\n\t\t\t}\n\t\t}\n\t}\n\n\tcases := []struct {\n\t\tname string\n\t\texp, imp string\n\t\tpubSubj string\n\t}{\n\t\t{\n\t\t\tname: \"export literal, import literal\",\n\t\t\texp: \"foo\", imp: \"foo\",\n\t\t\tpubSubj: \"foo\",\n\t\t},\n\t\t{\n\t\t\tname: \"export full wildcard, import literal\",\n\t\t\texp: \"foo.>\", imp: \"foo.bar\",\n\t\t\tpubSubj: \"foo.bar\",\n\t\t},\n\t\t{\n\t\t\tname: \"export full wildcard, import sublevel full wildcard\",\n\t\t\texp: \"foo.>\", imp: \"foo.bar.>\",\n\t\t\tpubSubj: \"foo.bar.whizz\",\n\t\t},\n\t\t{\n\t\t\tname: \"export full wildcard, import full wildcard\",\n\t\t\texp: \"foo.>\", imp: \"foo.>\",\n\t\t\tpubSubj: \"foo.bar\",\n\t\t},\n\t\t{\n\t\t\tname: \"export partial wildcard, import partial wildcard\",\n\t\t\texp: \"foo.*\", imp: \"foo.*\",\n\t\t\tpubSubj: \"foo.bar\",\n\t\t},\n\t\t{\n\t\t\tname: \"export mid partial wildcard, import mid partial wildcard\",\n\t\t\texp: \"foo.*.bar\", imp: \"foo.*.bar\",\n\t\t\tpubSubj: \"foo.whizz.bar\",\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(fmt.Sprintf(\"%s jwt\", c.name), func(t *testing.T) {\n\t\t\t// Setup NATS server.\n\t\t\ts := opTrustBasicSetup()\n\t\t\tdefer s.Shutdown()\n\t\t\ts.Start()\n\t\t\tif err := s.readyForConnections(5 * time.Second); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tbuildMemAccResolver(s)\n\n\t\t\t// Setup exporter account.\n\t\t\taccExpPair, accExpPub := createKey(t)\n\t\t\taccExpClaims := jwt.NewAccountClaims(accExpPub)\n\t\t\tif c.exp != \"\" {\n\t\t\t\taccExpClaims.Limits.WildcardExports = true\n\t\t\t\taccExpClaims.Exports.Add(&jwt.Export{\n\t\t\t\t\tName: fmt.Sprintf(\"%s-stream-export\", c.exp),\n\t\t\t\t\tSubject: jwt.Subject(c.exp),\n\t\t\t\t\tType: jwt.Stream,\n\t\t\t\t})\n\t\t\t}\n\t\t\taccExpJWT, err := accExpClaims.Encode(oKp)\n\t\t\trequire_NoError(t, err)\n\t\t\taddAccountToMemResolver(s, accExpPub, accExpJWT)\n\n\t\t\t// Setup importer account.\n\t\t\taccImpPair, accImpPub := createKey(t)\n\t\t\taccImpClaims := jwt.NewAccountClaims(accImpPub)\n\t\t\tif c.imp != \"\" {\n\t\t\t\taccImpClaims.Imports.Add(&jwt.Import{\n\t\t\t\t\tName: fmt.Sprintf(\"%s-stream-import\", c.imp),\n\t\t\t\t\tSubject: jwt.Subject(c.imp),\n\t\t\t\t\tAccount: accExpPub,\n\t\t\t\t\tType: jwt.Stream,\n\t\t\t\t})\n\t\t\t}\n\t\t\taccImpJWT, err := accImpClaims.Encode(oKp)\n\t\t\trequire_NoError(t, err)\n\t\t\taddAccountToMemResolver(s, accImpPub, accImpJWT)\n\n\t\t\t// Connect with different accounts.\n\t\t\tncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, accExpPair),\n\t\t\t\tnats.Name(fmt.Sprintf(\"nc-exporter-%s\", c.exp)))\n\t\t\tdefer ncExp.Close()\n\t\t\tncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, accImpPair),\n\t\t\t\tnats.Name(fmt.Sprintf(\"nc-importer-%s\", c.imp)))\n\t\t\tdefer ncImp.Close()\n\n\t\t\tcheckIsolation(t, c.pubSubj, ncExp, ncImp)\n\t\t\tif t.Failed() {\n\t\t\t\tt.Logf(\"exported=%q; imported=%q\", c.exp, c.imp)\n\t\t\t}\n\t\t})\n\n\t\tt.Run(fmt.Sprintf(\"%s conf\", c.name), func(t *testing.T) {\n\t\t\t// Setup NATS server.\n\t\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\n\t\t\t\taccounts: {\n\t\t\t\t\taccExp: {\n\t\t\t\t\t\tusers: [{user: accExp, password: accExp}]\n\t\t\t\t\t\texports: [{stream: %q}]\n\t\t\t\t\t}\n\t\t\t\t\taccImp: {\n\t\t\t\t\t\tusers: [{user: accImp, password: accImp}]\n\t\t\t\t\t\timports: [{stream: {account: accExp, subject: %q}}]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t\tc.exp, c.imp,\n\t\t\t)))\n\t\t\ts, _ := RunServerWithConfig(cf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Connect with different accounts.\n\t\t\tncExp := natsConnect(t, s.ClientURL(), nats.UserInfo(\"accExp\", \"accExp\"),\n\t\t\t\tnats.Name(fmt.Sprintf(\"nc-exporter-%s\", c.exp)))\n\t\t\tdefer ncExp.Close()\n\t\t\tncImp := natsConnect(t, s.ClientURL(), nats.UserInfo(\"accImp\", \"accImp\"),\n\t\t\t\tnats.Name(fmt.Sprintf(\"nc-importer-%s\", c.imp)))\n\t\t\tdefer ncImp.Close()\n\n\t\t\tcheckIsolation(t, c.pubSubj, ncExp, ncImp)\n\t\t\tif t.Failed() {\n\t\t\t\tt.Logf(\"exported=%q; imported=%q\", c.exp, c.imp)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMultiAccountsIsolation(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: 127.0.0.1:-1\n\taccounts: {\n\t\tPUBLIC: {\n\t\t\tusers:[{user: public, password: public}]\n\t\t\texports: [\n\t\t\t { stream: orders.client.stream.> }\n\t\t\t { stream: orders.client2.stream.> }\n\t\t\t]\n\t\t}\n\t\tCLIENT: {\n\t\t\tusers:[{user: client, password: client}]\n\t\t\timports: [\n\t\t\t { stream: { account: PUBLIC, subject: orders.client.stream.> }}\n\t\t\t]\n\t\t}\n\t\tCLIENT2: {\n\t\t\tusers:[{user: client2, password: client2}]\n\t\t\timports: [\n\t\t\t { stream: { account: PUBLIC, subject: orders.client2.stream.> }}\n\t\t\t]\n\t\t}\n\t}`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\t// Create a connection for CLIENT and subscribe on orders.>\n\tclientnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"client\", \"client\"))\n\tdefer clientnc.Close()\n\n\tclientsub := natsSubSync(t, clientnc, \"orders.>\")\n\tnatsFlush(t, clientnc)\n\n\t// Now same for CLIENT2.\n\tclient2nc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"client2\", \"client2\"))\n\tdefer client2nc.Close()\n\n\tclient2sub := natsSubSync(t, client2nc, \"orders.>\")\n\tnatsFlush(t, client2nc)\n\n\t// Now create a connection for PUBLIC\n\tpublicnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"public\", \"public\"))\n\tdefer publicnc.Close()\n\t// Publish on 'orders.client.stream.entry', so only CLIENT should receive it.\n\tnatsPub(t, publicnc, \"orders.client.stream.entry\", []byte(\"test1\"))\n\n\t// Verify that clientsub gets it.\n\tmsg := natsNexMsg(t, clientsub, time.Second)\n\trequire_Equal(t, string(msg.Data), \"test1\")\n\n\t// And also verify that client2sub does NOT get it.\n\t_, err := client2sub.NextMsg(100 * time.Microsecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\tclientsub.Unsubscribe()\n\tnatsFlush(t, clientnc)\n\tclient2sub.Unsubscribe()\n\tnatsFlush(t, client2nc)\n\n\t// Now have both accounts subscribe to \"orders.*.stream.entry\"\n\tclientsub = natsSubSync(t, clientnc, \"orders.*.stream.entry\")\n\tnatsFlush(t, clientnc)\n\n\tclient2sub = natsSubSync(t, client2nc, \"orders.*.stream.entry\")\n\tnatsFlush(t, client2nc)\n\n\t// Using the PUBLIC account, publish on the \"CLIENT\" subject\n\tnatsPub(t, publicnc, \"orders.client.stream.entry\", []byte(\"test2\"))\n\tnatsFlush(t, publicnc)\n\n\tmsg = natsNexMsg(t, clientsub, time.Second)\n\trequire_Equal(t, string(msg.Data), \"test2\")\n\n\t_, err = client2sub.NextMsg(100 * time.Microsecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n}\n\nfunc TestAccountFromOptions(t *testing.T) {\n\topts := defaultServerOptions\n\topts.Accounts = []*Account{NewAccount(\"foo\"), NewAccount(\"bar\")}\n\ts := New(&opts)\n\tdefer s.Shutdown()\n\n\tta := s.numReservedAccounts() + 2\n\tif la := s.numAccounts(); la != ta {\n\t\tt.Fatalf(\"Expected to have a server with %d active accounts, got %v\", ta, la)\n\t}\n\t// Check that sl is filled in.\n\tfooAcc, _ := s.LookupAccount(\"foo\")\n\tbarAcc, _ := s.LookupAccount(\"bar\")\n\tif fooAcc == nil || barAcc == nil {\n\t\tt.Fatalf(\"Error retrieving accounts for 'foo' and 'bar'\")\n\t}\n\tif fooAcc.sl == nil || barAcc.sl == nil {\n\t\tt.Fatal(\"Expected Sublists to be filled in on Opts.Accounts\")\n\t}\n}\n\n// Clients used to be able to ask that the account be forced to be new.\n// This was for dynamic sandboxes for demo environments but was never really used.\n// Make sure it always errors if set.\nfunc TestNewAccountAndRequireNewAlwaysError(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\tA: { users: [ {user: ua, password: pa} ] },\n\t\t\tB: { users: [ {user: ub, password: pb} ] },\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Success case\n\tc, _, _ := newClientForServer(s)\n\tconnectOp := \"CONNECT {\\\"user\\\":\\\"ua\\\", \\\"pass\\\":\\\"pa\\\"}\\r\\n\"\n\terr := c.parse([]byte(connectOp))\n\trequire_NoError(t, err)\n\tc.close()\n\n\t// Simple cases, any setting of account or new_account always errors.\n\t// Even with proper auth.\n\tc, cr, _ := newClientForServer(s)\n\tconnectOp = \"CONNECT {\\\"user\\\":\\\"ua\\\", \\\"pass\\\":\\\"pa\\\", \\\"account\\\":\\\"ANY\\\"}\\r\\n\"\n\tc.parseAsync(connectOp)\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR 'Authorization Violation'\") {\n\t\tt.Fatalf(\"Expected an error, got %q\", l)\n\t}\n\tc.close()\n\n\t// new_account with proper credentials.\n\tc, cr, _ = newClientForServer(s)\n\tconnectOp = \"CONNECT {\\\"user\\\":\\\"ua\\\", \\\"pass\\\":\\\"pa\\\", \\\"new_account\\\":true}\\r\\n\"\n\tc.parseAsync(connectOp)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR 'Authorization Violation'\") {\n\t\tt.Fatalf(\"Expected an error, got %q\", l)\n\t}\n\tc.close()\n\n\t// switch acccounts with proper credentials.\n\tc, cr, _ = newClientForServer(s)\n\tconnectOp = \"CONNECT {\\\"user\\\":\\\"ua\\\", \\\"pass\\\":\\\"pa\\\", \\\"account\\\":\\\"B\\\"}\\r\\n\"\n\tc.parseAsync(connectOp)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR 'Authorization Violation'\") {\n\t\tt.Fatalf(\"Expected an error, got %q\", l)\n\t}\n\tc.close()\n\n\t// Even if correct account designation, still make sure we error.\n\tc, cr, _ = newClientForServer(s)\n\tconnectOp = \"CONNECT {\\\"user\\\":\\\"ua\\\", \\\"pass\\\":\\\"pa\\\", \\\"account\\\":\\\"A\\\"}\\r\\n\"\n\tc.parseAsync(connectOp)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR 'Authorization Violation'\") {\n\t\tt.Fatalf(\"Expected an error, got %q\", l)\n\t}\n\tc.close()\n}\n\nfunc accountNameExists(name string, accounts []*Account) bool {\n\tfor _, acc := range accounts {\n\t\tif strings.Compare(acc.Name, name) == 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc TestAccountSimpleConfig(t *testing.T) {\n\tcfg1 := `\n\t\taccounts = [foo, bar]\n\t`\n\n\tconfFileName := createConfFile(t, []byte(cfg1))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error processing config file: %v\", err)\n\t}\n\tif la := len(opts.Accounts); la != 2 {\n\t\tt.Fatalf(\"Expected to see 2 accounts in opts, got %d\", la)\n\t}\n\tif !accountNameExists(\"foo\", opts.Accounts) {\n\t\tt.Fatal(\"Expected a 'foo' account\")\n\t}\n\tif !accountNameExists(\"bar\", opts.Accounts) {\n\t\tt.Fatal(\"Expected a 'bar' account\")\n\t}\n\n\tcfg2 := `\n\t\taccounts = [foo, foo]\n\t`\n\n\t// Make sure double entries is an error.\n\tconfFileName = createConfFile(t, []byte(cfg2))\n\t_, err = ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error with double account entries\")\n\t}\n}\n\nfunc TestAccountParseConfig(t *testing.T) {\n\ttraceDest := \"my.trace.dest\"\n\ttraceSampling := 50\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(`\n accounts {\n synadia {\n users = [\n {user: alice, password: foo}\n {user: bob, password: bar}\n ]\n }\n nats.io {\n\t\tmsg_trace: {dest: %q, sampling: %d%%}\n users = [\n {user: derek, password: foo}\n {user: ivan, password: bar}\n ]\n }\n }\n `, traceDest, traceSampling)))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error processing config file: %v\", err)\n\t}\n\n\tif la := len(opts.Accounts); la != 2 {\n\t\tt.Fatalf(\"Expected to see 2 accounts in opts, got %d\", la)\n\t}\n\n\tif lu := len(opts.Users); lu != 4 {\n\t\tt.Fatalf(\"Expected 4 total Users, got %d\", lu)\n\t}\n\n\tvar natsAcc *Account\n\tfor _, acc := range opts.Accounts {\n\t\tif acc.Name == \"nats.io\" {\n\t\t\tnatsAcc = acc\n\t\t\tbreak\n\t\t}\n\t}\n\tif natsAcc == nil {\n\t\tt.Fatalf(\"Error retrieving account for 'nats.io'\")\n\t}\n\ttd, tds := natsAcc.getTraceDestAndSampling()\n\trequire_Equal[string](t, td, traceDest)\n\trequire_Equal[int](t, tds, traceSampling)\n\n\tfor _, u := range opts.Users {\n\t\tif u.Username == \"derek\" {\n\t\t\tif u.Account != natsAcc {\n\t\t\t\tt.Fatalf(\"Expected to see the 'nats.io' account, but received %+v\", u.Account)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestAccountParseConfigDuplicateUsers(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n accounts {\n synadia {\n users = [\n {user: alice, password: foo}\n {user: bob, password: bar}\n ]\n }\n nats.io {\n users = [\n {user: alice, password: bar}\n ]\n }\n }\n `))\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error with double user entries\")\n\t}\n}\n\nfunc TestAccountParseConfigImportsExports(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/accounts.conf\")\n\tif err != nil {\n\t\tt.Fatal(\"parsing failed: \", err)\n\t}\n\tif la := len(opts.Accounts); la != 3 {\n\t\tt.Fatalf(\"Expected to see 3 accounts in opts, got %d\", la)\n\t}\n\tif lu := len(opts.Nkeys); lu != 4 {\n\t\tt.Fatalf(\"Expected 4 total Nkey users, got %d\", lu)\n\t}\n\tif lu := len(opts.Users); lu != 0 {\n\t\tt.Fatalf(\"Expected no Users, got %d\", lu)\n\t}\n\tvar natsAcc, synAcc *Account\n\tfor _, acc := range opts.Accounts {\n\t\tif acc.Name == \"nats.io\" {\n\t\t\tnatsAcc = acc\n\t\t} else if acc.Name == \"synadia\" {\n\t\t\tsynAcc = acc\n\t\t}\n\t}\n\tif natsAcc == nil {\n\t\tt.Fatalf(\"Error retrieving account for 'nats.io'\")\n\t}\n\tif natsAcc.Nkey != \"AB5UKNPVHDWBP5WODG742274I3OGY5FM3CBIFCYI4OFEH7Y23GNZPXFE\" {\n\t\tt.Fatalf(\"Expected nats account to have an nkey, got %q\\n\", natsAcc.Nkey)\n\t}\n\t// Check user assigned to the correct account.\n\tfor _, nk := range opts.Nkeys {\n\t\tif nk.Nkey == \"UBRYMDSRTC6AVJL6USKKS3FIOE466GMEU67PZDGOWYSYHWA7GSKO42VW\" {\n\t\t\tif nk.Account != natsAcc {\n\t\t\t\tt.Fatalf(\"Expected user to be associated with natsAcc, got %q\\n\", nk.Account.Name)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Now check for the imports and exports of streams and services.\n\tif lis := len(natsAcc.imports.streams); lis != 2 {\n\t\tt.Fatalf(\"Expected 2 imported streams, got %d\\n\", lis)\n\t}\n\tfor _, si := range natsAcc.imports.streams {\n\t\tif si.from == \"public.synadia\" {\n\t\t\trequire_True(t, si.atrc)\n\t\t} else {\n\t\t\trequire_False(t, si.atrc)\n\t\t}\n\t}\n\tif lis := len(natsAcc.imports.services); lis != 1 {\n\t\tt.Fatalf(\"Expected 1 imported service, got %d\\n\", lis)\n\t}\n\tif les := len(natsAcc.exports.services); les != 4 {\n\t\tt.Fatalf(\"Expected 4 exported services, got %d\\n\", les)\n\t}\n\tif les := len(natsAcc.exports.streams); les != 0 {\n\t\tt.Fatalf(\"Expected no exported streams, got %d\\n\", les)\n\t}\n\n\tea := natsAcc.exports.services[\"nats.time\"]\n\tif ea == nil {\n\t\tt.Fatalf(\"Expected to get a non-nil exportAuth for service\")\n\t}\n\tif ea.respType != Streamed {\n\t\tt.Fatalf(\"Expected to get a Streamed response type, got %q\", ea.respType)\n\t}\n\trequire_True(t, ea.atrc)\n\tea = natsAcc.exports.services[\"nats.photo\"]\n\tif ea == nil {\n\t\tt.Fatalf(\"Expected to get a non-nil exportAuth for service\")\n\t}\n\tif ea.respType != Chunked {\n\t\tt.Fatalf(\"Expected to get a Chunked response type, got %q\", ea.respType)\n\t}\n\trequire_False(t, ea.atrc)\n\tea = natsAcc.exports.services[\"nats.add\"]\n\tif ea == nil {\n\t\tt.Fatalf(\"Expected to get a non-nil exportAuth for service\")\n\t}\n\tif ea.respType != Singleton {\n\t\tt.Fatalf(\"Expected to get a Singleton response type, got %q\", ea.respType)\n\t}\n\trequire_True(t, ea.atrc)\n\n\tif synAcc == nil {\n\t\tt.Fatalf(\"Error retrieving account for 'synadia'\")\n\t}\n\n\tif lis := len(synAcc.imports.streams); lis != 0 {\n\t\tt.Fatalf(\"Expected no imported streams, got %d\\n\", lis)\n\t}\n\tif lis := len(synAcc.imports.services); lis != 1 {\n\t\tt.Fatalf(\"Expected 1 imported service, got %d\\n\", lis)\n\t}\n\tif les := len(synAcc.exports.services); les != 2 {\n\t\tt.Fatalf(\"Expected 2 exported service, got %d\\n\", les)\n\t}\n\tif les := len(synAcc.exports.streams); les != 2 {\n\t\tt.Fatalf(\"Expected 2 exported streams, got %d\\n\", les)\n\t}\n}\n\nfunc TestImportExportConfigFailures(t *testing.T) {\n\t// Import from unknow account\n\tcf := createConfFile(t, []byte(`\n accounts {\n nats.io {\n imports = [{stream: {account: \"synadia\", subject:\"foo\"}}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with import from unknown account\")\n\t}\n\t// Import a service with no account.\n\tcf = createConfFile(t, []byte(`\n accounts {\n nats.io {\n imports = [{service: subject:\"foo.*\"}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with import of a service with no account\")\n\t}\n\t// Import a service with a wildcard subject.\n\tcf = createConfFile(t, []byte(`\n accounts {\n nats.io {\n imports = [{service: {account: \"nats.io\", subject:\"foo.*\"}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with import of a service with wildcard subject\")\n\t}\n\t// Export with unknown keyword.\n\tcf = createConfFile(t, []byte(`\n accounts {\n nats.io {\n exports = [{service: \"foo.*\", wat:true}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with export with unknown keyword\")\n\t}\n\t// Import with unknown keyword.\n\tcf = createConfFile(t, []byte(`\n accounts {\n nats.io {\n imports = [{stream: {account: nats.io, subject: \"foo.*\"}, wat:true}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with import with unknown keyword\")\n\t}\n\t// Export with an account.\n\tcf = createConfFile(t, []byte(`\n accounts {\n nats.io {\n exports = [{service: {account: nats.io, subject:\"foo.*\"}}]\n }\n }\n `))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with export with account\")\n\t}\n}\n\nfunc TestImportAuthorized(t *testing.T) {\n\t_, foo, bar := simpleAccountServer(t)\n\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \">\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.*\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.>\", nil), false, t)\n\n\tfoo.AddStreamExport(\"foo\", IsPublicExport)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"bar\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*\", nil), false, t)\n\n\tfoo.AddStreamExport(\"*\", []*Account{bar})\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"bar\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"baz\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \">\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.*\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*.*\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*.>\", nil), false, t)\n\n\t// Reset and test '>' public export\n\t_, foo, bar = simpleAccountServer(t)\n\tfoo.AddStreamExport(\">\", nil)\n\t// Everything should work.\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"bar\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"baz\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \">\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.*\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*.*\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"*.>\", nil), true, t)\n\n\t_, foo, bar = simpleAccountServer(t)\n\tfoo.addStreamExportWithAccountPos(\"foo.*\", []*Account{}, 2)\n\tfoo.addStreamExportWithAccountPos(\"bar.*.foo\", []*Account{}, 2)\n\tif err := foo.addStreamExportWithAccountPos(\"baz.*.>\", []*Account{}, 3); err == nil {\n\t\tt.Fatal(\"expected error\")\n\t}\n\tcheckBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf(\"foo.%s\", bar.Name), nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf(\"bar.%s.foo\", bar.Name), nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, fmt.Sprintf(\"baz.foo.%s\", bar.Name), nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.X\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"bar.X.foo\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"baz.foo.X\", nil), false, t)\n\n\tfoo.addServiceExportWithAccountPos(\"a.*\", []*Account{}, 2)\n\tfoo.addServiceExportWithAccountPos(\"b.*.a\", []*Account{}, 2)\n\tif err := foo.addServiceExportWithAccountPos(\"c.*.>\", []*Account{}, 3); err == nil {\n\t\tt.Fatal(\"expected error\")\n\t}\n\tcheckBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf(\"a.%s\", bar.Name), nil), true, t)\n\tcheckBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf(\"b.%s.a\", bar.Name), nil), true, t)\n\tcheckBool(foo.checkServiceImportAuthorized(bar, fmt.Sprintf(\"c.a.%s\", bar.Name), nil), false, t)\n\tcheckBool(foo.checkServiceImportAuthorized(bar, \"a.X\", nil), false, t)\n\tcheckBool(foo.checkServiceImportAuthorized(bar, \"b.X.a\", nil), false, t)\n\tcheckBool(foo.checkServiceImportAuthorized(bar, \"c.a.X\", nil), false, t)\n\n\t// Reset and test pwc and fwc\n\ts, foo, bar := simpleAccountServer(t)\n\tfoo.AddStreamExport(\"foo.*.baz.>\", []*Account{bar})\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar.baz.1\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar.baz.*\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.*.baz.1.1\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.22.baz.22\", nil), true, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar.baz\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bar, \"foo.bar.*.*\", nil), false, t)\n\n\t// Make sure we match the account as well\n\n\tfb, _ := s.RegisterAccount(\"foobar\")\n\tbz, _ := s.RegisterAccount(\"baz\")\n\n\tcheckBool(foo.checkStreamImportAuthorized(fb, \"foo.bar.baz.1\", nil), false, t)\n\tcheckBool(foo.checkStreamImportAuthorized(bz, \"foo.bar.baz.1\", nil), false, t)\n}\n\nfunc TestSimpleMapping(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Test first that trying to import with no matching export permission returns an error.\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"foo\", \"import\"); err != ErrStreamImportAuthorization {\n\t\tt.Fatalf(\"Expected error of ErrAccountImportAuthorization but got %v\", err)\n\t}\n\n\t// Now map the subject space between foo and bar.\n\t// Need to do export first.\n\tif err := cfoo.acc.AddStreamExport(\"foo\", nil); err != nil { // Public with no accounts defined.\n\t\tt.Fatalf(\"Error adding account export to client foo: %v\", err)\n\t}\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"foo\", \"import\"); err != nil {\n\t\tt.Fatalf(\"Error adding account import to client bar: %v\", err)\n\t}\n\n\t// Normal and Queue Subscription on bar client.\n\tif err := cbar.parse([]byte(\"SUB import.foo 1\\r\\nSUB import.foo bar 2\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// Now publish our message.\n\tcfoo.parseAsync(\"PUB foo 5\\r\\nhello\\r\\n\")\n\n\tcheckMsg := func(l, sid string) {\n\t\tt.Helper()\n\t\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\t\tif len(mraw) == 0 {\n\t\t\tt.Fatalf(\"No message received\")\n\t\t}\n\t\tmatches := mraw[0]\n\t\tif matches[SUB_INDEX] != \"import.foo\" {\n\t\t\tt.Fatalf(\"Did not get correct subject: wanted %q, got %q\", \"import.foo\", matches[SUB_INDEX])\n\t\t}\n\t\tif matches[SID_INDEX] != sid {\n\t\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t\t}\n\t}\n\n\t// Now check we got the message from normal subscription.\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tcheckMsg(l, \"1\")\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n\n\tl, err = crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tcheckMsg(l, \"2\")\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n\n\t// We should have 2 subscriptions in both. Normal and Queue Subscriber\n\t// for barAcc which are local, and 2 that are shadowed in fooAcc.\n\t// Now make sure that when we unsubscribe we clean up properly for both.\n\tif bslc := barAcc.sl.Count(); bslc != 2 {\n\t\tt.Fatalf(\"Expected 2 normal subscriptions on barAcc, got %d\", bslc)\n\t}\n\tif fslc := fooAcc.sl.Count(); fslc != 2 {\n\t\tt.Fatalf(\"Expected 2 shadowed subscriptions on fooAcc, got %d\", fslc)\n\t}\n\n\t// Now unsubscribe.\n\tif err := cbar.parse([]byte(\"UNSUB 1\\r\\nUNSUB 2\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// We should have zero on both.\n\tif bslc := barAcc.sl.Count(); bslc != 0 {\n\t\tt.Fatalf(\"Expected no normal subscriptions on barAcc, got %d\", bslc)\n\t}\n\tif fslc := fooAcc.sl.Count(); fslc != 0 {\n\t\tt.Fatalf(\"Expected no shadowed subscriptions on fooAcc, got %d\", fslc)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/1159\nfunc TestStreamImportLengthBug(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, _, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\tif err := cfoo.acc.AddStreamExport(\"client.>\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account export to client foo: %v\", err)\n\t}\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"client.>\", \"events.>\"); err == nil {\n\t\tt.Fatalf(\"Expected an error when using a stream import prefix with a wildcard\")\n\t}\n\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"client.>\", \"events\"); err != nil {\n\t\tt.Fatalf(\"Error adding account import to client bar: %v\", err)\n\t}\n\n\tif err := cbar.parse([]byte(\"SUB events.> 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// Also make sure that we will get an error from a config version.\n\t// JWT will be updated separately.\n\tcf := createConfFile(t, []byte(`\n\taccounts {\n\t foo {\n\t exports = [{stream: \"client.>\"}]\n\t }\n\t bar {\n\t imports = [{stream: {account: \"foo\", subject:\"client.>\"}, prefix:\"events.>\"}]\n\t }\n\t}\n\t`))\n\tif _, err := ProcessConfigFile(cf); err == nil {\n\t\tt.Fatalf(\"Expected an error with import with wildcard prefix\")\n\t}\n}\n\nfunc TestShadowSubsCleanupOnClientClose(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\t// Now map the subject space between foo and bar.\n\t// Need to do export first.\n\tif err := fooAcc.AddStreamExport(\"foo\", nil); err != nil { // Public with no accounts defined.\n\t\tt.Fatalf(\"Error adding account export to client foo: %v\", err)\n\t}\n\tif err := barAcc.AddStreamImport(fooAcc, \"foo\", \"import\"); err != nil {\n\t\tt.Fatalf(\"Error adding account import to client bar: %v\", err)\n\t}\n\n\tcbar, _, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Normal and Queue Subscription on bar client.\n\tif err := cbar.parse([]byte(\"SUB import.foo 1\\r\\nSUB import.foo bar 2\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\tif fslc := fooAcc.sl.Count(); fslc != 2 {\n\t\tt.Fatalf(\"Expected 2 shadowed subscriptions on fooAcc, got %d\", fslc)\n\t}\n\n\t// Now close cbar and make sure we remove shadows.\n\tcbar.closeConnection(ClientClosed)\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif fslc := fooAcc.sl.Count(); fslc != 0 {\n\t\t\treturn fmt.Errorf(\"Number of shadow subscriptions is %d\", fslc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoPrefixWildcardMapping(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\tif err := cfoo.acc.AddStreamExport(\">\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding stream export to client foo: %v\", err)\n\t}\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"*\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import to client bar: %v\", err)\n\t}\n\n\t// Normal Subscription on bar client for literal \"foo\".\n\tcbar.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\t_, err := crBar.ReadString('\\n') // Make sure subscriptions were processed.\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// Now publish our message.\n\tcfoo.parseAsync(\"PUB foo 5\\r\\nhello\\r\\n\")\n\n\t// Now check we got the message from normal subscription.\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n}\n\nfunc TestPrefixWildcardMapping(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\tif err := cfoo.acc.AddStreamExport(\">\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding stream export to client foo: %v\", err)\n\t}\n\t// Checking that trailing '.' is accepted, tested that it is auto added above.\n\tif err := cbar.acc.AddStreamImport(fooAcc, \"*\", \"pub.imports.\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import to client bar: %v\", err)\n\t}\n\n\t// Normal Subscription on bar client for wildcard.\n\tcbar.parseAsync(\"SUB pub.imports.* 1\\r\\nPING\\r\\n\")\n\t_, err := crBar.ReadString('\\n') // Make sure subscriptions were processed.\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// Now publish our message.\n\tcfoo.parseAsync(\"PUB foo 5\\r\\nhello\\r\\n\")\n\n\t// Now check we got the messages from wildcard subscription.\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"pub.imports.foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n}\n\nfunc TestPrefixWildcardMappingWithLiteralSub(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\tif err := fooAcc.AddStreamExport(\">\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding stream export to client foo: %v\", err)\n\t}\n\tif err := barAcc.AddStreamImport(fooAcc, \"*\", \"pub.imports.\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import to client bar: %v\", err)\n\t}\n\n\t// Normal Subscription on bar client for wildcard.\n\tcbar.parseAsync(\"SUB pub.imports.foo 1\\r\\nPING\\r\\n\")\n\t_, err := crBar.ReadString('\\n') // Make sure subscriptions were processed.\n\tif err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\n\t// Now publish our message.\n\tcfoo.parseAsync(\"PUB foo 5\\r\\nhello\\r\\n\")\n\n\t// Now check we got the messages from wildcard subscription.\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"pub.imports.foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n}\n\nfunc TestMultipleImportsAndSingleWCSub(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\tif err := fooAcc.AddStreamExport(\"foo\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding stream export to account foo: %v\", err)\n\t}\n\tif err := fooAcc.AddStreamExport(\"bar\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding stream export to account foo: %v\", err)\n\t}\n\n\tif err := barAcc.AddStreamImport(fooAcc, \"foo\", \"pub.\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import to account bar: %v\", err)\n\t}\n\tif err := barAcc.AddStreamImport(fooAcc, \"bar\", \"pub.\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import to account bar: %v\", err)\n\t}\n\n\t// Wildcard Subscription on bar client for both imports.\n\tcbar.parse([]byte(\"SUB pub.* 1\\r\\n\"))\n\n\t// Now publish a message on 'foo' and 'bar'\n\tcfoo.parseAsync(\"PUB foo 5\\r\\nhello\\r\\nPUB bar 5\\r\\nworld\\r\\n\")\n\n\t// Now check we got the messages from the wildcard subscription.\n\tl, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"pub.foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"hello\\r\\n\"), t)\n\n\tl, err = crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw = msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches = mraw[0]\n\tif matches[SUB_INDEX] != \"pub.bar\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"world\\r\\n\"), t)\n\n\t// Check subscription count.\n\tif fslc := fooAcc.sl.Count(); fslc != 2 {\n\t\tt.Fatalf(\"Expected 2 shadowed subscriptions on fooAcc, got %d\", fslc)\n\t}\n\tif bslc := barAcc.sl.Count(); bslc != 1 {\n\t\tt.Fatalf(\"Expected 1 normal subscriptions on barAcc, got %d\", bslc)\n\t}\n\n\t// Now unsubscribe.\n\tif err := cbar.parse([]byte(\"UNSUB 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'bar' from server: %v\", err)\n\t}\n\t// We should have zero on both.\n\tif bslc := barAcc.sl.Count(); bslc != 0 {\n\t\tt.Fatalf(\"Expected no normal subscriptions on barAcc, got %d\", bslc)\n\t}\n\tif fslc := fooAcc.sl.Count(); fslc != 0 {\n\t\tt.Fatalf(\"Expected no shadowed subscriptions on fooAcc, got %d\", fslc)\n\t}\n}\n\n// Make sure the AddServiceExport function is additive if called multiple times.\nfunc TestAddServiceExport(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tbazAcc, err := s.RegisterAccount(\"$baz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account 'baz': %v\", err)\n\t}\n\tdefer s.Shutdown()\n\n\tif err := fooAcc.AddServiceExport(\"test.request\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr := fooAcc.exports.services[\"test.request\"]\n\tif len(tr.approved) != 0 {\n\t\tt.Fatalf(\"Expected no authorized accounts, got %d\", len(tr.approved))\n\t}\n\tif err := fooAcc.AddServiceExport(\"test.request\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr = fooAcc.exports.services[\"test.request\"]\n\tif tr == nil {\n\t\tt.Fatalf(\"Expected authorized accounts, got nil\")\n\t}\n\tif ls := len(tr.approved); ls != 1 {\n\t\tt.Fatalf(\"Expected 1 authorized accounts, got %d\", ls)\n\t}\n\tif err := fooAcc.AddServiceExport(\"test.request\", []*Account{bazAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr = fooAcc.exports.services[\"test.request\"]\n\tif tr == nil {\n\t\tt.Fatalf(\"Expected authorized accounts, got nil\")\n\t}\n\tif ls := len(tr.approved); ls != 2 {\n\t\tt.Fatalf(\"Expected 2 authorized accounts, got %d\", ls)\n\t}\n}\n\nfunc TestServiceExportWithWildcards(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpublic bool\n\t}{\n\t\t{\n\t\t\tname: \"public\",\n\t\t\tpublic: true,\n\t\t},\n\t\t{\n\t\t\tname: \"private\",\n\t\t\tpublic: false,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ts, fooAcc, barAcc := simpleAccountServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tvar accs []*Account\n\t\t\tif !test.public {\n\t\t\t\taccs = []*Account{barAcc}\n\t\t\t}\n\t\t\t// Add service export with a wildcard\n\t\t\tif err := fooAcc.AddServiceExport(\"ngs.update.*\", accs); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding account service export: %v\", err)\n\t\t\t}\n\t\t\t// Import on bar account\n\t\t\tif err := barAcc.AddServiceImport(fooAcc, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding account service import: %v\", err)\n\t\t\t}\n\n\t\t\tcfoo, crFoo, _ := newClientForServer(s)\n\t\t\tdefer cfoo.close()\n\n\t\t\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\t\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t\t\t}\n\t\t\tcbar, crBar, _ := newClientForServer(s)\n\t\t\tdefer cbar.close()\n\n\t\t\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\t\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t\t\t}\n\n\t\t\t// Now setup the responder under cfoo\n\t\t\tcfoo.parse([]byte(\"SUB ngs.update.* 1\\r\\n\"))\n\n\t\t\t// Now send the request. Remember we expect the request on our local ngs.update.\n\t\t\t// We added the route with that \"from\" and will map it to \"ngs.update.$bar\"\n\t\t\tcbar.parseAsync(\"SUB reply 11\\r\\nPUB ngs.update reply 4\\r\\nhelp\\r\\n\")\n\n\t\t\t// Now read the request from crFoo\n\t\t\tl, err := crFoo.ReadString('\\n')\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t\t\t}\n\n\t\t\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\t\t\tif len(mraw) == 0 {\n\t\t\t\tt.Fatalf(\"No message received\")\n\t\t\t}\n\t\t\tmatches := mraw[0]\n\t\t\tif matches[SUB_INDEX] != \"ngs.update.$bar\" {\n\t\t\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t\t\t}\n\t\t\tif matches[SID_INDEX] != \"1\" {\n\t\t\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t\t\t}\n\t\t\t// Make sure this looks like _INBOX\n\t\t\tif !strings.HasPrefix(matches[REPLY_INDEX], \"_R_.\") {\n\t\t\t\tt.Fatalf(\"Expected an _R_.* like reply, got '%s'\", matches[REPLY_INDEX])\n\t\t\t}\n\t\t\tcheckPayload(crFoo, []byte(\"help\\r\\n\"), t)\n\n\t\t\treplyOp := fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", matches[REPLY_INDEX])\n\t\t\tcfoo.parseAsync(replyOp)\n\n\t\t\t// Now read the response from crBar\n\t\t\tl, err = crBar.ReadString('\\n')\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t\t\t}\n\t\t\tmraw = msgPat.FindAllStringSubmatch(l, -1)\n\t\t\tif len(mraw) == 0 {\n\t\t\t\tt.Fatalf(\"No message received\")\n\t\t\t}\n\t\t\tmatches = mraw[0]\n\t\t\tif matches[SUB_INDEX] != \"reply\" {\n\t\t\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t\t\t}\n\t\t\tif matches[SID_INDEX] != \"11\" {\n\t\t\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t\t\t}\n\t\t\tif matches[REPLY_INDEX] != \"\" {\n\t\t\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t\t\t}\n\t\t\tcheckPayload(crBar, []byte(\"22\\r\\n\"), t)\n\n\t\t\tif nr := barAcc.NumPendingAllResponses(); nr != 0 {\n\t\t\t\tt.Fatalf(\"Expected no responses on barAcc, got %d\", nr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAccountAddServiceImportRace(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tif err := fooAcc.AddServiceExport(\"foo.*\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\n\ttotal := 100\n\terrCh := make(chan error, total)\n\tfor i := 0; i < 100; i++ {\n\t\tgo func(i int) {\n\t\t\terr := barAcc.AddServiceImport(fooAcc, fmt.Sprintf(\"foo.%d\", i), \"\")\n\t\t\terrCh <- err // nil is a valid value.\n\t\t}(i)\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\terr := <-errCh\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error adding account service import: %v\", err)\n\t\t}\n\t}\n\n\tbarAcc.mu.Lock()\n\tlens := len(barAcc.imports.services)\n\tc := barAcc.internalClient()\n\tbarAcc.mu.Unlock()\n\tif lens != total {\n\t\tt.Fatalf(\"Expected %d imported services, got %d\", total, lens)\n\t}\n\tc.mu.Lock()\n\tlens = len(c.subs)\n\tc.mu.Unlock()\n\tif lens != total {\n\t\tt.Fatalf(\"Expected %d subscriptions in internal client, got %d\", total, lens)\n\t}\n}\n\nfunc TestServiceImportWithWildcards(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tif err := fooAcc.AddServiceExport(\"test.*\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\t// We can not map wildcards atm, so if we supply a to mapping and a wildcard we should fail.\n\tif err := barAcc.AddServiceImport(fooAcc, \"test.*\", \"foo\"); err == nil {\n\t\tt.Fatalf(\"Expected error adding account service import with wildcard and mapping, got none\")\n\t}\n\tif err := barAcc.AddServiceImport(fooAcc, \"test.>\", \"\"); err == nil {\n\t\tt.Fatalf(\"Expected error adding account service import with broader wildcard, got none\")\n\t}\n\t// This should work.\n\tif err := barAcc.AddServiceImport(fooAcc, \"test.*\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding account service import: %v\", err)\n\t}\n\t// Make sure we can send and receive.\n\tcfoo, crFoo, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\t// Now setup the resonder under cfoo\n\tcfoo.parse([]byte(\"SUB test.* 1\\r\\n\"))\n\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Now send the request.\n\tgo cbar.parse([]byte(\"SUB bar 11\\r\\nPUB test.22 bar 4\\r\\nhelp\\r\\n\"))\n\n\t// Now read the request from crFoo\n\tl, err := crFoo.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"test.22\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\t// Make sure this looks like _INBOX\n\tif !strings.HasPrefix(matches[REPLY_INDEX], \"_R_.\") {\n\t\tt.Fatalf(\"Expected an _R_.* like reply, got '%s'\", matches[REPLY_INDEX])\n\t}\n\tcheckPayload(crFoo, []byte(\"help\\r\\n\"), t)\n\n\treplyOp := fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", matches[REPLY_INDEX])\n\tgo cfoo.parse([]byte(replyOp))\n\n\t// Now read the response from crBar\n\tl, err = crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw = msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches = mraw[0]\n\tif matches[SUB_INDEX] != \"bar\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"11\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tif matches[REPLY_INDEX] != \"\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"22\\r\\n\"), t)\n\n\t// Remove the service import with the wildcard and make sure hasWC is cleared.\n\tbarAcc.removeServiceImport(fooAcc.Name, \"test.*\")\n\n\tbarAcc.mu.Lock()\n\tdefer barAcc.mu.Unlock()\n\tif len(barAcc.imports.services) != 0 {\n\t\tt.Fatalf(\"Expected no imported services, got %d\", len(barAcc.imports.services))\n\t}\n}\n\n// Make sure the AddStreamExport function is additive if called multiple times.\nfunc TestAddStreamExport(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tbazAcc, err := s.RegisterAccount(\"$baz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account 'baz': %v\", err)\n\t}\n\tdefer s.Shutdown()\n\n\tif err := fooAcc.AddStreamExport(\"test.request\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr := fooAcc.exports.streams[\"test.request\"]\n\tif tr != nil {\n\t\tt.Fatalf(\"Expected no authorized accounts, got %d\", len(tr.approved))\n\t}\n\tif err := fooAcc.AddStreamExport(\"test.request\", []*Account{barAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr = fooAcc.exports.streams[\"test.request\"]\n\tif tr == nil {\n\t\tt.Fatalf(\"Expected authorized accounts, got nil\")\n\t}\n\tif ls := len(tr.approved); ls != 1 {\n\t\tt.Fatalf(\"Expected 1 authorized accounts, got %d\", ls)\n\t}\n\tif err := fooAcc.AddStreamExport(\"test.request\", []*Account{bazAcc}); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\ttr = fooAcc.exports.streams[\"test.request\"]\n\tif tr == nil {\n\t\tt.Fatalf(\"Expected authorized accounts, got nil\")\n\t}\n\tif ls := len(tr.approved); ls != 2 {\n\t\tt.Fatalf(\"Expected 2 authorized accounts, got %d\", ls)\n\t}\n}\n\nfunc TestCrossAccountRequestReply(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, crFoo, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Add in the service export for the requests. Make it public.\n\tif err := cfoo.acc.AddServiceExport(\"test.request\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\n\t// Test addServiceImport to make sure it requires accounts.\n\tif err := cbar.acc.AddServiceImport(nil, \"foo\", \"test.request\"); err != ErrMissingAccount {\n\t\tt.Fatalf(\"Expected ErrMissingAccount but received %v.\", err)\n\t}\n\tif err := cbar.acc.AddServiceImport(fooAcc, \"foo\", \"test..request.\"); err != ErrInvalidSubject {\n\t\tt.Fatalf(\"Expected ErrInvalidSubject but received %v.\", err)\n\t}\n\n\t// Now add in the route mapping for request to be routed to the foo account.\n\tif err := cbar.acc.AddServiceImport(fooAcc, \"foo\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding account service import to client bar: %v\", err)\n\t}\n\n\t// Now setup the resonder under cfoo\n\tcfoo.parse([]byte(\"SUB test.request 1\\r\\n\"))\n\n\t// Now send the request. Remember we expect the request on our local foo. We added the route\n\t// with that \"from\" and will map it to \"test.request\"\n\tcbar.parseAsync(\"SUB bar 11\\r\\nPUB foo bar 4\\r\\nhelp\\r\\n\")\n\n\t// Now read the request from crFoo\n\tl, err := crFoo.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\tif matches[SUB_INDEX] != \"test.request\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\t// Make sure this looks like _INBOX\n\tif !strings.HasPrefix(matches[REPLY_INDEX], \"_R_.\") {\n\t\tt.Fatalf(\"Expected an _R_.* like reply, got '%s'\", matches[REPLY_INDEX])\n\t}\n\tcheckPayload(crFoo, []byte(\"help\\r\\n\"), t)\n\n\treplyOp := fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", matches[REPLY_INDEX])\n\tcfoo.parseAsync(replyOp)\n\n\t// Now read the response from crBar\n\tl, err = crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw = msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches = mraw[0]\n\tif matches[SUB_INDEX] != \"bar\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"11\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tif matches[REPLY_INDEX] != \"\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\", matches[SID_INDEX])\n\t}\n\tcheckPayload(crBar, []byte(\"22\\r\\n\"), t)\n\n\tif nr := barAcc.NumPendingAllResponses(); nr != 0 {\n\t\tt.Fatalf(\"Expected no responses on barAcc, got %d\", nr)\n\t}\n}\n\nfunc TestAccountRequestReplyTrackLatency(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\t// Run server in Go routine. We need this one running for internal sending of msgs.\n\ts.Start()\n\t// Wait for accept loop(s) to be started\n\tif err := s.readyForConnections(10 * time.Second); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcfoo, crFoo, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Add in the service export for the requests. Make it public.\n\tif err := fooAcc.AddServiceExport(\"track.service\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\n\t// Now let's add in tracking\n\n\t// First check we get an error if service does not exist.\n\tif err := fooAcc.TrackServiceExport(\"track.wrong\", \"results\"); err != ErrMissingService {\n\t\tt.Fatalf(\"Expected error enabling tracking latency for wrong service\")\n\t}\n\t// Check results should be a valid subject\n\tif err := fooAcc.TrackServiceExport(\"track.service\", \"results.*\"); err != ErrBadPublishSubject {\n\t\tt.Fatalf(\"Expected error enabling tracking latency for bad results subject\")\n\t}\n\t// Make sure we can not loop around on ourselves..\n\tif err := fooAcc.TrackServiceExport(\"track.service\", \"track.service\"); err != ErrBadPublishSubject {\n\t\tt.Fatalf(\"Expected error enabling tracking latency for same subject\")\n\t}\n\t// Check bad sampling\n\tif err := fooAcc.TrackServiceExportWithSampling(\"track.service\", \"results\", -1); err != ErrBadSampling {\n\t\tt.Fatalf(\"Expected error enabling tracking latency for bad sampling\")\n\t}\n\tif err := fooAcc.TrackServiceExportWithSampling(\"track.service\", \"results\", 101); err != ErrBadSampling {\n\t\tt.Fatalf(\"Expected error enabling tracking latency for bad sampling\")\n\t}\n\n\t// Now let's add in tracking for real. This will be 100%\n\tif err := fooAcc.TrackServiceExport(\"track.service\", \"results\"); err != nil {\n\t\tt.Fatalf(\"Error enabling tracking latency: %v\", err)\n\t}\n\n\t// Now add in the route mapping for request to be routed to the foo account.\n\tif err := barAcc.AddServiceImport(fooAcc, \"req\", \"track.service\"); err != nil {\n\t\tt.Fatalf(\"Error adding account service import to client bar: %v\", err)\n\t}\n\n\t// Now setup the responder under cfoo and the listener for the results\n\tcfoo.parse([]byte(\"SUB track.service 1\\r\\nSUB results 2\\r\\n\"))\n\n\treadFooMsg := func() ([]byte, string) {\n\t\tt.Helper()\n\t\tl, err := crFoo.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading from client 'foo': %v\", err)\n\t\t}\n\t\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\t\tif len(mraw) == 0 {\n\t\t\tt.Fatalf(\"No message received\")\n\t\t}\n\t\tmsg := mraw[0]\n\t\tmsgSize, _ := strconv.Atoi(msg[LEN_INDEX])\n\t\treturn grabPayload(crFoo, msgSize), msg[REPLY_INDEX]\n\t}\n\n\tstart := time.Now()\n\n\t// Now send the request. Remember we expect the request on our local foo. We added the route\n\t// with that \"from\" and will map it to \"test.request\"\n\tcbar.parseAsync(\"SUB resp 11\\r\\nPUB req resp 4\\r\\nhelp\\r\\n\")\n\n\t// Now read the request from crFoo\n\t_, reply := readFooMsg()\n\treplyOp := fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", reply)\n\n\tserviceTime := 25 * time.Millisecond\n\n\t// We will wait a bit to check latency results\n\tgo func() {\n\t\ttime.Sleep(serviceTime)\n\t\tcfoo.parseAsync(replyOp)\n\t}()\n\n\t// Now read the response from crBar\n\t_, err := crBar.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\n\t// Now let's check that we got the sampling results\n\trMsg, _ := readFooMsg()\n\n\t// Unmarshal and check it.\n\tvar sl ServiceLatency\n\terr = json.Unmarshal(rMsg, &sl)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse latency json: %v\\n\", err)\n\t}\n\tstartDelta := sl.RequestStart.Sub(start)\n\tif startDelta > 5*time.Millisecond {\n\t\tt.Fatalf(\"Bad start delta %v\", startDelta)\n\t}\n\tif sl.ServiceLatency < serviceTime {\n\t\tt.Fatalf(\"Bad service latency: %v\", sl.ServiceLatency)\n\t}\n\tif sl.TotalLatency < sl.ServiceLatency {\n\t\tt.Fatalf(\"Bad total latency: %v\", sl.ServiceLatency)\n\t}\n}\n\n// This will test for leaks in the remote latency tracking via client.rrTracking\nfunc TestAccountTrackLatencyRemoteLeaks(t *testing.T) {\n\toptsA, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\toptsA.NoSigs, optsA.NoLog = true, true\n\toptsA.ServerName = \"A\"\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\toptsB.ServerName = \"B\"\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\tsrvs := []*Server{srvA, srvB}\n\n\t// Now add in the accounts and setup tracking.\n\tfor _, s := range srvs {\n\t\ts.SetSystemAccount(globalAccountName)\n\t\tfooAcc, _ := s.RegisterAccount(\"$foo\")\n\t\tfooAcc.AddServiceExport(\"track.service\", nil)\n\t\tfooAcc.TrackServiceExport(\"track.service\", \"results\")\n\t\tbarAcc, _ := s.RegisterAccount(\"$bar\")\n\t\tif err := barAcc.AddServiceImport(fooAcc, \"req\", \"track.service\"); err != nil {\n\t\t\tt.Fatalf(\"Failed to import: %v\", err)\n\t\t}\n\t}\n\n\tgetClient := func(s *Server, name string) *client {\n\t\tt.Helper()\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\tfor _, c := range s.clients {\n\t\t\tc.mu.Lock()\n\t\t\tn := c.opts.Name\n\t\t\tc.mu.Unlock()\n\t\t\tif n == name {\n\t\t\t\treturn c\n\t\t\t}\n\t\t}\n\t\tt.Fatalf(\"Did not find client %q on server %q\", name, s.info.ID)\n\t\treturn nil\n\t}\n\n\t// Test with a responder on second server, srvB. but they will not respond.\n\tcfooNC := natsConnect(t, srvB.ClientURL(), nats.Name(\"foo\"))\n\tdefer cfooNC.Close()\n\tcfoo := getClient(srvB, \"foo\")\n\tfooAcc, _ := srvB.LookupAccount(\"$foo\")\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\t// Set new limits\n\tfor _, srv := range srvs {\n\t\tfooAcc, _ := srv.LookupAccount(\"$foo\")\n\t\terr := fooAcc.SetServiceExportResponseThreshold(\"track.service\", 5*time.Millisecond)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t\t}\n\t}\n\n\t// Now setup the responder under cfoo and the listener for the results\n\ttime.Sleep(50 * time.Millisecond)\n\tbaseSubs := int(srvA.NumSubscriptions())\n\tfooSub := natsSubSync(t, cfooNC, \"track.service\")\n\tnatsFlush(t, cfooNC)\n\t// Wait for it to propagate.\n\tcheckExpectedSubs(t, baseSubs+1, srvA)\n\n\tcbarNC := natsConnect(t, srvA.ClientURL(), nats.Name(\"bar\"))\n\tdefer cbarNC.Close()\n\tcbar := getClient(srvA, \"bar\")\n\n\tbarAcc, _ := srvA.LookupAccount(\"$bar\")\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\treadFooMsg := func() {\n\t\tt.Helper()\n\t\tif _, err := fooSub.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not receive foo msg: %v\", err)\n\t\t}\n\t}\n\n\t// Send 2 requests\n\tnatsSubSync(t, cbarNC, \"resp\")\n\n\tnatsPubReq(t, cbarNC, \"req\", \"resp\", []byte(\"help\"))\n\tnatsPubReq(t, cbarNC, \"req\", \"resp\", []byte(\"help\"))\n\n\treadFooMsg()\n\treadFooMsg()\n\n\tvar rc *client\n\t// Pull out first client\n\tsrvB.mu.Lock()\n\tfor _, rc = range srvB.clients {\n\t\tif rc != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tsrvB.mu.Unlock()\n\n\ttracking := func() int {\n\t\trc.mu.Lock()\n\t\tvar nt int\n\t\tif rc.rrTracking != nil {\n\t\t\tnt = len(rc.rrTracking.rmap)\n\t\t}\n\t\trc.mu.Unlock()\n\t\treturn nt\n\t}\n\n\texpectTracking := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif numTracking := tracking(); numTracking != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected to have %d tracking replies, got %d\", expected, numTracking)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\texpectTracking(2)\n\t// Make sure these remote tracking replies honor the current respThresh for a service export.\n\ttime.Sleep(10 * time.Millisecond)\n\texpectTracking(0)\n\t// Also make sure tracking is removed\n\trc.mu.Lock()\n\tremoved := rc.rrTracking == nil\n\trc.mu.Unlock()\n\tif !removed {\n\t\tt.Fatalf(\"Expected the rrTracking to be removed\")\n\t}\n\n\t// Now let's test that a lower response threshold is picked up.\n\tfSub := natsSubSync(t, cfooNC, \"foo\")\n\tnatsFlush(t, cfooNC)\n\n\t// Wait for it to propagate.\n\tcheckExpectedSubs(t, baseSubs+4, srvA)\n\n\t// queue up some first. We want to test changing when rrTracking exists.\n\tnatsPubReq(t, cbarNC, \"req\", \"resp\", []byte(\"help\"))\n\treadFooMsg()\n\texpectTracking(1)\n\n\tfor _, s := range srvs {\n\t\tfooAcc, _ := s.LookupAccount(\"$foo\")\n\t\tbarAcc, _ := s.LookupAccount(\"$bar\")\n\t\tfooAcc.AddServiceExport(\"foo\", nil)\n\t\tfooAcc.TrackServiceExport(\"foo\", \"foo.results\")\n\t\tfooAcc.SetServiceExportResponseThreshold(\"foo\", time.Millisecond)\n\t\tbarAcc.AddServiceImport(fooAcc, \"foo\", \"foo\")\n\t}\n\n\tnatsSubSync(t, cbarNC, \"reply\")\n\tnatsPubReq(t, cbarNC, \"foo\", \"reply\", []byte(\"help\"))\n\tif _, err := fSub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Did not receive foo msg: %v\", err)\n\t}\n\texpectTracking(2)\n\n\trc.mu.Lock()\n\tlrt := rc.rrTracking.lrt\n\trc.mu.Unlock()\n\tif lrt != time.Millisecond {\n\t\tt.Fatalf(\"Expected lrt of %v, got %v\", time.Millisecond, lrt)\n\t}\n\n\t// Now make sure we clear on close.\n\trc.closeConnection(ClientClosed)\n\n\t// Actual tear down will be not inline.\n\tcheckFor(t, time.Second, 5*time.Millisecond, func() error {\n\t\trc.mu.Lock()\n\t\tremoved = rc.rrTracking == nil\n\t\trc.mu.Unlock()\n\t\tif !removed {\n\t\t\treturn fmt.Errorf(\"Expected the rrTracking to be removed after client close\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestCrossAccountServiceResponseTypes(t *testing.T) {\n\ts, fooAcc, barAcc := simpleAccountServer(t)\n\tdefer s.Shutdown()\n\n\tcfoo, crFoo, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\tcbar, crBar, _ := newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Add in the service export for the requests. Make it public.\n\tif err := fooAcc.AddServiceExportWithResponse(\"test.request\", Streamed, nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\t// Now add in the route mapping for request to be routed to the foo account.\n\tif err := barAcc.AddServiceImport(fooAcc, \"foo\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding account service import to client bar: %v\", err)\n\t}\n\n\t// Now setup the resonder under cfoo\n\tcfoo.parse([]byte(\"SUB test.request 1\\r\\n\"))\n\n\t// Now send the request. Remember we expect the request on our local foo. We added the route\n\t// with that \"from\" and will map it to \"test.request\"\n\tcbar.parseAsync(\"SUB bar 11\\r\\nPUB foo bar 4\\r\\nhelp\\r\\n\")\n\n\t// Now read the request from crFoo\n\tl, err := crFoo.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\n\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches := mraw[0]\n\treply := matches[REPLY_INDEX]\n\tif !strings.HasPrefix(reply, \"_R_.\") {\n\t\tt.Fatalf(\"Expected an _R_.* like reply, got '%s'\", reply)\n\t}\n\tcrFoo.ReadString('\\n')\n\n\treplyOp := fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", matches[REPLY_INDEX])\n\tvar mReply []byte\n\tfor i := 0; i < 10; i++ {\n\t\tmReply = append(mReply, replyOp...)\n\t}\n\n\tcfoo.parseAsync(string(mReply))\n\n\tvar buf []byte\n\tfor i := 0; i < 20; i++ {\n\t\tb, err := crBar.ReadBytes('\\n')\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading response: %v\", err)\n\t\t}\n\t\tbuf = append(buf[:], b...)\n\t\tif mraw = msgPat.FindAllStringSubmatch(string(buf), -1); len(mraw) == 10 {\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(mraw) != 10 {\n\t\tt.Fatalf(\"Expected a response but got %d\", len(mraw))\n\t}\n\n\t// Also make sure the response map gets cleaned up when interest goes away.\n\tcbar.closeConnection(ClientClosed)\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif nr := barAcc.NumPendingAllResponses(); nr != 0 {\n\t\t\treturn fmt.Errorf(\"Number of responses is %d\", nr)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now test bogus reply subjects are handled and do not accumulate the response maps.\n\tcbar, _, _ = newClientForServer(s)\n\tdefer cbar.close()\n\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\t// Do not create any interest in the reply subject 'bar'. Just send a request.\n\tcbar.parseAsync(\"PUB foo bar 4\\r\\nhelp\\r\\n\")\n\n\t// Now read the request from crFoo\n\tl, err = crFoo.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading from client 'bar': %v\", err)\n\t}\n\tmraw = msgPat.FindAllStringSubmatch(l, -1)\n\tif len(mraw) == 0 {\n\t\tt.Fatalf(\"No message received\")\n\t}\n\tmatches = mraw[0]\n\treply = matches[REPLY_INDEX]\n\tif !strings.HasPrefix(reply, \"_R_.\") {\n\t\tt.Fatalf(\"Expected an _R_.* like reply, got '%s'\", reply)\n\t}\n\tcrFoo.ReadString('\\n')\n\n\treplyOp = fmt.Sprintf(\"PUB %s 2\\r\\n22\\r\\n\", matches[REPLY_INDEX])\n\n\tcfoo.parseAsync(replyOp)\n\n\t// Now wait for a bit, the reply should trip a no interest condition\n\t// which should clean this up.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif nr := fooAcc.NumPendingAllResponses(); nr != 0 {\n\t\t\treturn fmt.Errorf(\"Number of responses is %d\", nr)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Also make sure the response map entry is gone as well.\n\tfooAcc.mu.RLock()\n\tlrm := len(fooAcc.exports.responses)\n\tfooAcc.mu.RUnlock()\n\n\tif lrm != 0 {\n\t\tt.Fatalf(\"Expected the responses to be cleared, got %d entries\", lrm)\n\t}\n}\n\nfunc TestAccountMapsUsers(t *testing.T) {\n\t// Used for the nkey users to properly sign.\n\tseed1 := \"SUAPM67TC4RHQLKBX55NIQXSMATZDOZK6FNEOSS36CAYA7F7TY66LP4BOM\"\n\tseed2 := \"SUAIS5JPX4X4GJ7EIIJEQ56DH2GWPYJRPWN5XJEDENJOZHCBLI7SEPUQDE\"\n\n\tconfFileName := createConfFile(t, []byte(`\n accounts {\n synadia {\n users = [\n {user: derek, password: foo},\n {nkey: UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR}\n ]\n }\n nats {\n users = [\n {user: ivan, password: bar},\n {nkey: UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH}\n ]\n }\n }\n `))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error parsing config file: %v\", err)\n\t}\n\topts.NoSigs = true\n\ts := New(opts)\n\tdefer s.Shutdown()\n\tsynadia, _ := s.LookupAccount(\"synadia\")\n\tnats, _ := s.LookupAccount(\"nats\")\n\n\tif synadia == nil || nats == nil {\n\t\tt.Fatalf(\"Expected non nil accounts during lookup\")\n\t}\n\n\t// Make sure a normal log in maps the accounts correctly.\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\tconnectOp := []byte(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"foo\\\"}\\r\\n\")\n\tc.parse(connectOp)\n\tif c.acc != synadia {\n\t\tt.Fatalf(\"Expected the client's account to match 'synadia', got %v\", c.acc)\n\t}\n\n\tc, _, _ = newClientForServer(s)\n\tdefer c.close()\n\tconnectOp = []byte(\"CONNECT {\\\"user\\\":\\\"ivan\\\",\\\"pass\\\":\\\"bar\\\"}\\r\\n\")\n\tc.parse(connectOp)\n\tif c.acc != nats {\n\t\tt.Fatalf(\"Expected the client's account to match 'nats', got %v\", c.acc)\n\t}\n\n\t// Now test nkeys as well.\n\tkp, _ := nkeys.FromSeed([]byte(seed1))\n\tpubKey, _ := kp.PublicKey()\n\n\tc, cr, l := newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\tvar info nonceInfo\n\terr = json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err := kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n\tif c.acc != synadia {\n\t\tt.Fatalf(\"Expected the nkey client's account to match 'synadia', got %v\", c.acc)\n\t}\n\n\t// Now nats account nkey user.\n\tkp, _ = nkeys.FromSeed([]byte(seed2))\n\tpubKey, _ = kp.PublicKey()\n\n\tc, cr, l = newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\terr = json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err = kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig = base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs = fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n\tif c.acc != nats {\n\t\tt.Fatalf(\"Expected the nkey client's account to match 'nats', got %v\", c.acc)\n\t}\n}\n\nfunc TestAccountGlobalDefault(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tif acc, _ := s.LookupAccount(globalAccountName); acc == nil {\n\t\tt.Fatalf(\"Expected a global default account on a new server, got none.\")\n\t}\n\t// Make sure we can not create one with same name..\n\tif _, err := s.RegisterAccount(globalAccountName); err == nil {\n\t\tt.Fatalf(\"Expected error trying to create a new reserved account\")\n\t}\n\n\t// Make sure we can not define one in a config file either.\n\tconfFileName := createConfFile(t, []byte(`accounts { $G {} }`))\n\n\tif _, err := ProcessConfigFile(confFileName); err == nil {\n\t\tt.Fatalf(\"Expected an error parsing config file with reserved account\")\n\t}\n}\n\nfunc TestAccountCheckStreamImportsEqual(t *testing.T) {\n\t// Create bare accounts for this test\n\tfooAcc := NewAccount(\"foo\")\n\tif err := fooAcc.AddStreamExport(\">\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding stream export: %v\", err)\n\t}\n\n\tbarAcc := NewAccount(\"bar\")\n\tif err := barAcc.AddStreamImport(fooAcc, \"foo\", \"myPrefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tbazAcc := NewAccount(\"baz\")\n\tif err := bazAcc.AddStreamImport(fooAcc, \"foo\", \"myPrefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif !barAcc.checkStreamImportsEqual(bazAcc) {\n\t\tt.Fatal(\"Expected stream imports to be the same\")\n\t}\n\n\tif err := bazAcc.AddStreamImport(fooAcc, \"foo.>\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif barAcc.checkStreamImportsEqual(bazAcc) {\n\t\tt.Fatal(\"Expected stream imports to be different\")\n\t}\n\tif err := barAcc.AddStreamImport(fooAcc, \"foo.>\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif !barAcc.checkStreamImportsEqual(bazAcc) {\n\t\tt.Fatal(\"Expected stream imports to be the same\")\n\t}\n\n\t// Create another account that is named \"foo\". We want to make sure\n\t// that the comparison still works (based on account name, not pointer)\n\tnewFooAcc := NewAccount(\"foo\")\n\tif err := newFooAcc.AddStreamExport(\">\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding stream export: %v\", err)\n\t}\n\tbatAcc := NewAccount(\"bat\")\n\tif err := batAcc.AddStreamImport(newFooAcc, \"foo\", \"myPrefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif err := batAcc.AddStreamImport(newFooAcc, \"foo.>\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif !batAcc.checkStreamImportsEqual(barAcc) {\n\t\tt.Fatal(\"Expected stream imports to be the same\")\n\t}\n\tif !batAcc.checkStreamImportsEqual(bazAcc) {\n\t\tt.Fatal(\"Expected stream imports to be the same\")\n\t}\n\n\t// Test with account with different \"from\"\n\texpAcc := NewAccount(\"new_acc\")\n\tif err := expAcc.AddStreamExport(\">\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding stream export: %v\", err)\n\t}\n\taAcc := NewAccount(\"a\")\n\tif err := aAcc.AddStreamImport(expAcc, \"bar\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tbAcc := NewAccount(\"b\")\n\tif err := bAcc.AddStreamImport(expAcc, \"baz\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif aAcc.checkStreamImportsEqual(bAcc) {\n\t\tt.Fatal(\"Expected stream imports to be different\")\n\t}\n\n\t// Test with account with different \"prefix\"\n\taAcc = NewAccount(\"a\")\n\tif err := aAcc.AddStreamImport(expAcc, \"bar\", \"prefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tbAcc = NewAccount(\"b\")\n\tif err := bAcc.AddStreamImport(expAcc, \"bar\", \"diff_prefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif aAcc.checkStreamImportsEqual(bAcc) {\n\t\tt.Fatal(\"Expected stream imports to be different\")\n\t}\n\n\t// Test with account with different \"name\"\n\texpAcc = NewAccount(\"diff_name\")\n\tif err := expAcc.AddStreamExport(\">\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding stream export: %v\", err)\n\t}\n\tbAcc = NewAccount(\"b\")\n\tif err := bAcc.AddStreamImport(expAcc, \"bar\", \"prefix\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\tif aAcc.checkStreamImportsEqual(bAcc) {\n\t\tt.Fatal(\"Expected stream imports to be different\")\n\t}\n}\n\nfunc TestAccountNoDeadlockOnQueueSubRouteMapUpdate(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc.QueueSubscribeSync(\"foo\", \"bar\")\n\n\tvar accs []*Account\n\tfor i := 0; i < 10; i++ {\n\t\tacc, _ := s.RegisterAccount(fmt.Sprintf(\"acc%d\", i))\n\t\tacc.mu.Lock()\n\t\taccs = append(accs, acc)\n\t}\n\n\topts2 := DefaultOptions()\n\topts2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", opts.Cluster.Host, opts.Cluster.Port))\n\ts2 := RunServer(opts2)\n\tdefer s2.Shutdown()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tfor _, acc := range accs {\n\t\t\tacc.mu.Unlock()\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tnc.QueueSubscribeSync(\"foo\", \"bar\")\n\tnc.Flush()\n\n\twg.Wait()\n}\n\nfunc TestAccountDuplicateServiceImportSubject(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfooAcc, _ := s.RegisterAccount(\"foo\")\n\tfooAcc.AddServiceExport(\"remote1\", nil)\n\tfooAcc.AddServiceExport(\"remote2\", nil)\n\n\tbarAcc, _ := s.RegisterAccount(\"bar\")\n\tif err := barAcc.AddServiceImport(fooAcc, \"foo\", \"remote1\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\tif err := barAcc.AddServiceImport(fooAcc, \"foo\", \"remote2\"); err == nil || !strings.Contains(err.Error(), \"duplicate\") {\n\t\tt.Fatalf(\"Expected an error about duplicate service import subject, got %q\", err)\n\t}\n\n\t// However, it is allowed to add more than one service import\n\t// on the same subject for different exporting accounts\n\tbazAcc, _ := s.RegisterAccount(\"baz\")\n\tbazAcc.AddServiceExport(\"remote1\", nil)\n\tif err := barAcc.AddServiceImport(bazAcc, \"foo\", \"remote1\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\tbatAcc, _ := s.RegisterAccount(\"bat\")\n\tbatAcc.AddServiceExport(\"remote1\", nil)\n\tif err := barAcc.AddServiceImport(batAcc, \"foo\", \"remote1\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// But again, can't add for one that has been already added, say bazAcc.\n\tif err := barAcc.AddServiceImport(bazAcc, \"foo\", \"remote1\"); err == nil || !strings.Contains(err.Error(), \"duplicate\") {\n\t\tt.Fatalf(\"Expected an error about duplicate service import subject, got %q\", err)\n\t}\n}\n\nfunc TestAccountRemoveServiceImport(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfooAcc, _ := s.RegisterAccount(\"foo\")\n\tfooAcc.AddServiceExport(\"remote1\", nil)\n\n\tbarAcc, _ := s.RegisterAccount(\"bar\")\n\terr := barAcc.AddServiceImport(fooAcc, \"foo\", \"remote1\")\n\trequire_NoError(t, err)\n\n\tbazAcc, _ := s.RegisterAccount(\"baz\")\n\tbazAcc.AddServiceExport(\"remote1\", nil)\n\terr = barAcc.AddServiceImport(bazAcc, \"foo\", \"remote1\")\n\trequire_NoError(t, err)\n\n\tbatAcc, _ := s.RegisterAccount(\"bat\")\n\tbatAcc.AddServiceExport(\"remote1\", nil)\n\terr = barAcc.AddServiceImport(batAcc, \"foo\", \"remote1\")\n\trequire_NoError(t, err)\n\n\t// Check that we can get the service import we expect\n\tcheckGetSI := func(acc, subj string) bool {\n\t\tbarAcc.mu.RLock()\n\t\tdefer barAcc.mu.RUnlock()\n\t\tsi := barAcc.getServiceImportForAccountLocked(acc, subj)\n\t\tif si == nil {\n\t\t\treturn false\n\t\t}\n\t\treturn si.acc.Name == acc\n\t}\n\trequire_False(t, checkGetSI(\"nonsense\", \"foo\"))\n\trequire_False(t, checkGetSI(\"foo\", \"nonsense\"))\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\trequire_True(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_True(t, checkGetSI(\"bat\", \"foo\"))\n\n\t// Remove service imports one by one and check expected map content.\n\n\t// Try first to remove one that does not exists, because of subject\n\t// or because of account.\n\tbarAcc.removeServiceImport(\"nonsense\", \"foo\")\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\trequire_True(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_True(t, checkGetSI(\"bat\", \"foo\"))\n\tbarAcc.removeServiceImport(\"baz\", \"nonsense\")\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\trequire_True(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_True(t, checkGetSI(\"bat\", \"foo\"))\n\n\t// Remove the 2nd account that was added\n\tbarAcc.removeServiceImport(\"baz\", \"foo\")\n\trequire_False(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\trequire_True(t, checkGetSI(\"bat\", \"foo\"))\n\n\t// Remove the last account that was added\n\tbarAcc.removeServiceImport(\"bat\", \"foo\")\n\trequire_False(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_False(t, checkGetSI(\"bat\", \"foo\"))\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\n\t// There is only one left, make sure that we still properly check\n\t// for appropriate subject and account name.\n\tbarAcc.removeServiceImport(\"nonsense\", \"foo\")\n\trequire_False(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_False(t, checkGetSI(\"bat\", \"foo\"))\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\tbarAcc.removeServiceImport(\"foo\", \"nonsense\")\n\trequire_False(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_False(t, checkGetSI(\"bat\", \"foo\"))\n\trequire_True(t, checkGetSI(\"foo\", \"foo\"))\n\n\t// Remove the first account that was added\n\tbarAcc.removeServiceImport(\"foo\", \"foo\")\n\trequire_False(t, checkGetSI(\"baz\", \"foo\"))\n\trequire_False(t, checkGetSI(\"bat\", \"foo\"))\n\trequire_False(t, checkGetSI(\"foo\", \"foo\"))\n\n\t// Make sure that the service import map is cleared\n\tbarAcc.mu.RLock()\n\tsis, ok := barAcc.imports.services[\"foo\"]\n\tbarAcc.mu.RUnlock()\n\trequire_False(t, ok)\n\trequire_Len(t, len(sis), 0)\n}\n\nfunc TestAccountMultipleServiceImportsWithSameSubjectFromDifferentAccounts(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts: {\n\t\t\tSVC-E: {\n\t\t\t\tusers: [ { user:svc-e, password:svc-e } ]\n\t\t\t\texports: [ { accounts: [CLIENTS], service: \"SvcReq.>\", response_type: singleton } ]\n\t\t\t}\n\n\t\t\tSVC-W: {\n\t\t\t\tusers: [ { user:svc-w, password:svc-w } ]\n\t\t\t\texports: [ { accounts: [CLIENTS], service: \"SvcReq.>\", response_type: singleton } ]\n\t\t\t}\n\n\t\t\tCLIENTS: {\n\t\t\t\tusers: [ { user:cl, password:cl } ]\n\t\t\t\timports: [\n\t\t\t\t\t{ service: { account: SVC-E, subject: \"SvcReq.>\" } }\n\t\t\t\t\t{ service: { account: SVC-W, subject: \"SvcReq.>\" } }\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tlisten: \"127.0.0.1:-1\"\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tncE := natsConnect(t, s.ClientURL(), nats.UserInfo(\"svc-e\", \"svc-e\"))\n\tdefer ncE.Close()\n\tnatsSub(t, ncE, \"SvcReq.>\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From SVC-E\"))\n\t})\n\tnatsFlush(t, ncE)\n\n\tncW := natsConnect(t, s.ClientURL(), nats.UserInfo(\"svc-w\", \"svc-w\"))\n\tdefer ncW.Close()\n\tnatsSub(t, ncW, \"SvcReq.>\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From SVC-W\"))\n\t})\n\tnatsFlush(t, ncW)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"cl\", \"cl\"))\n\tdefer nc.Close()\n\tsub := natsSubSync(t, nc, nats.NewInbox())\n\tnatsPubReq(t, nc, \"SvcReq.Test\", sub.Subject, []byte(\"test\"))\n\tmsg1 := natsNexMsg(t, sub, time.Second)\n\tmsg2 := natsNexMsg(t, sub, time.Second)\n\tm1 := string(msg1.Data)\n\tm2 := string(msg2.Data)\n\tif (m1 == \"From SVC-E\" && m2 == \"From SVC-W\") || (m1 == \"From SVC-W\" && m2 == \"From SVC-E\") {\n\t\t// Ok\n\t\treturn\n\t}\n\tt.Fatalf(\"Unexpected responses: m1=%s m2=%s\", m1, m2)\n}\n\nfunc TestMultipleStreamImportsWithSameSubjectDifferentPrefix(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfooAcc, _ := s.RegisterAccount(\"foo\")\n\tfooAcc.AddStreamExport(\"test\", nil)\n\n\tbarAcc, _ := s.RegisterAccount(\"bar\")\n\tbarAcc.AddStreamExport(\"test\", nil)\n\n\timportAcc, _ := s.RegisterAccount(\"import\")\n\n\tif err := importAcc.AddStreamImport(fooAcc, \"test\", \"foo\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := importAcc.AddStreamImport(barAcc, \"test\", \"bar\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now make sure we can see messages from both.\n\tcimport, crImport, _ := newClientForServer(s)\n\tdefer cimport.close()\n\tif err := cimport.registerWithAccount(importAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'import' account: %v\", err)\n\t}\n\tif err := cimport.parse([]byte(\"SUB *.test 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'import' from server: %v\", err)\n\t}\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\tcbar, _, _ := newClientForServer(s)\n\tdefer cbar.close()\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\treadMsg := func() {\n\t\tt.Helper()\n\t\tl, err := crImport.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading msg header from client 'import': %v\", err)\n\t\t}\n\t\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\t\tif len(mraw) == 0 {\n\t\t\tt.Fatalf(\"No message received\")\n\t\t}\n\t\t// Consume msg body too.\n\t\tif _, err = crImport.ReadString('\\n'); err != nil {\n\t\t\tt.Fatalf(\"Error reading msg body from client 'import': %v\", err)\n\t\t}\n\t}\n\n\tcbar.parseAsync(\"PUB test 9\\r\\nhello-bar\\r\\n\")\n\treadMsg()\n\n\tcfoo.parseAsync(\"PUB test 9\\r\\nhello-foo\\r\\n\")\n\treadMsg()\n}\n\n// This should work with prefixes that are different but we also want it to just work with same subject\n// being imported from multiple accounts.\nfunc TestMultipleStreamImportsWithSameSubject(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfooAcc, _ := s.RegisterAccount(\"foo\")\n\tfooAcc.AddStreamExport(\"test\", nil)\n\n\tbarAcc, _ := s.RegisterAccount(\"bar\")\n\tbarAcc.AddStreamExport(\"test\", nil)\n\n\timportAcc, _ := s.RegisterAccount(\"import\")\n\n\tif err := importAcc.AddStreamImport(fooAcc, \"test\", \"\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Since we allow this now, make sure we do detect a duplicate import from same account etc.\n\t// That should be not allowed.\n\tif err := importAcc.AddStreamImport(fooAcc, \"test\", \"\"); err != ErrStreamImportDuplicate {\n\t\tt.Fatalf(\"Expected ErrStreamImportDuplicate but got %v\", err)\n\t}\n\n\tif err := importAcc.AddStreamImport(barAcc, \"test\", \"\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now make sure we can see messages from both.\n\tcimport, crImport, _ := newClientForServer(s)\n\tdefer cimport.close()\n\tif err := cimport.registerWithAccount(importAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'import' account: %v\", err)\n\t}\n\tif err := cimport.parse([]byte(\"SUB test 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error for client 'import' from server: %v\", err)\n\t}\n\n\tcfoo, _, _ := newClientForServer(s)\n\tdefer cfoo.close()\n\tif err := cfoo.registerWithAccount(fooAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'foo' account: %v\", err)\n\t}\n\n\tcbar, _, _ := newClientForServer(s)\n\tdefer cbar.close()\n\tif err := cbar.registerWithAccount(barAcc); err != nil {\n\t\tt.Fatalf(\"Error registering client with 'bar' account: %v\", err)\n\t}\n\n\treadMsg := func() {\n\t\tt.Helper()\n\t\tl, err := crImport.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading msg header from client 'import': %v\", err)\n\t\t}\n\t\tmraw := msgPat.FindAllStringSubmatch(l, -1)\n\t\tif len(mraw) == 0 {\n\t\t\tt.Fatalf(\"No message received\")\n\t\t}\n\t\t// Consume msg body too.\n\t\tif _, err = crImport.ReadString('\\n'); err != nil {\n\t\t\tt.Fatalf(\"Error reading msg body from client 'import': %v\", err)\n\t\t}\n\t}\n\n\tcbar.parseAsync(\"PUB test 9\\r\\nhello-bar\\r\\n\")\n\treadMsg()\n\n\tcfoo.parseAsync(\"PUB test 9\\r\\nhello-foo\\r\\n\")\n\treadMsg()\n}\n\nfunc TestAccountBasicRouteMapping(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT)\n\tacc.AddMapping(\"foo\", \"bar\")\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tfsub, _ := nc.SubscribeSync(\"foo\")\n\tbsub, _ := nc.SubscribeSync(\"bar\")\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\tt.Fatalf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t}\n\t}\n\n\tcheckPending(fsub, 0)\n\tcheckPending(bsub, 1)\n\n\tacc.RemoveMapping(\"foo\")\n\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending(fsub, 1)\n\tcheckPending(bsub, 1)\n}\n\nfunc TestAccountWildcardRouteMapping(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT)\n\n\taddMap := func(src, dest string) {\n\t\tt.Helper()\n\t\tif err := acc.AddMapping(src, dest); err != nil {\n\t\t\tt.Fatalf(\"Error adding mapping: %v\", err)\n\t\t}\n\t}\n\n\taddMap(\"foo.*.*\", \"bar.$2.$1\")\n\taddMap(\"bar.*.>\", \"baz.$1.>\")\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tpub := func(subj string) {\n\t\tt.Helper()\n\t\terr := nc.Publish(subj, nil)\n\t\tif err == nil {\n\t\t\terr = nc.Flush()\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t\t}\n\t}\n\n\tfsub, _ := nc.SubscribeSync(\"foo.>\")\n\tbsub, _ := nc.SubscribeSync(\"bar.>\")\n\tzsub, _ := nc.SubscribeSync(\"baz.>\")\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\tt.Fatalf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t}\n\t}\n\n\tpub(\"foo.1.2\")\n\n\tcheckPending(fsub, 0)\n\tcheckPending(bsub, 1)\n\tcheckPending(zsub, 0)\n}\n\nfunc TestAccountRouteMappingChangesAfterClientStart(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Create the client first then add in mapping.\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tnc.Flush()\n\n\tacc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT)\n\tacc.AddMapping(\"foo\", \"bar\")\n\n\tfsub, _ := nc.SubscribeSync(\"foo\")\n\tbsub, _ := nc.SubscribeSync(\"bar\")\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\tt.Fatalf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t}\n\t}\n\n\tcheckPending(fsub, 0)\n\tcheckPending(bsub, 1)\n\n\tacc.RemoveMapping(\"foo\")\n\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending(fsub, 1)\n\tcheckPending(bsub, 1)\n}\n\nfunc TestAccountSimpleWeightedRouteMapping(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT)\n\tacc.AddWeightedMappings(\"foo\", NewMapDest(\"bar\", 50))\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tfsub, _ := nc.SubscribeSync(\"foo\")\n\tbsub, _ := nc.SubscribeSync(\"bar\")\n\n\ttotal := 500\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo\", nil)\n\t}\n\tnc.Flush()\n\n\tfpending, _, _ := fsub.Pending()\n\tbpending, _, _ := bsub.Pending()\n\n\th := total / 2\n\ttp := h / 5\n\tmin, max := h-tp, h+tp\n\tif fpending < min || fpending > max {\n\t\tt.Fatalf(\"Expected about %d msgs, got %d and %d\", h, fpending, bpending)\n\t}\n}\n\nfunc TestAccountMultiWeightedRouteMappings(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(DEFAULT_GLOBAL_ACCOUNT)\n\n\t// Check failures for bad weights.\n\tshouldErr := func(rds ...*MapDest) {\n\t\tt.Helper()\n\t\tif acc.AddWeightedMappings(\"foo\", rds...) == nil {\n\t\t\tt.Fatalf(\"Expected an error, got none\")\n\t\t}\n\t}\n\tshouldNotErr := func(rds ...*MapDest) {\n\t\tt.Helper()\n\t\tif err := acc.AddWeightedMappings(\"foo\", rds...); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tshouldErr(NewMapDest(\"bar\", 150))\n\tshouldNotErr(NewMapDest(\"bar\", 50))\n\tshouldNotErr(NewMapDest(\"bar\", 50), NewMapDest(\"baz\", 50))\n\t// Same dest duplicated should error.\n\tshouldErr(NewMapDest(\"bar\", 50), NewMapDest(\"bar\", 50))\n\t// total over 100\n\tshouldErr(NewMapDest(\"bar\", 50), NewMapDest(\"baz\", 60))\n\n\tacc.RemoveMapping(\"foo\")\n\n\t// 20 for original, you can leave it off will be auto-added.\n\tshouldNotErr(NewMapDest(\"bar\", 50), NewMapDest(\"baz\", 30))\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tfsub, _ := nc.SubscribeSync(\"foo\")\n\tbsub, _ := nc.SubscribeSync(\"bar\")\n\tzsub, _ := nc.SubscribeSync(\"baz\")\n\n\t// For checking later.\n\trds := []struct {\n\t\tsub *nats.Subscription\n\t\tw uint8\n\t}{\n\t\t{fsub, 20},\n\t\t{bsub, 50},\n\t\t{zsub, 30},\n\t}\n\n\ttotal := 5000\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo\", nil)\n\t}\n\tnc.Flush()\n\n\tfor _, rd := range rds {\n\t\tpending, _, _ := rd.sub.Pending()\n\t\texpected := total / int(100/rd.w)\n\t\ttp := expected / 5 // 20%\n\t\tmin, max := expected-tp, expected+tp\n\t\tif pending < min || pending > max {\n\t\t\tt.Fatalf(\"Expected about %d msgs for %q, got %d\", expected, rd.sub.Subject, pending)\n\t\t}\n\t}\n}\n\nfunc TestGlobalAccountRouteMappingsConfiguration(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n\tmappings = {\n\t\tfoo: bar\n\t\tfoo.*: [ { dest: bar.v1.$1, weight: 40% }, { destination: baz.v2.$1, weight: 20 } ]\n\t\tbar.*.*: RAB.$2.$1\n }\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tbsub, _ := nc.SubscribeSync(\"bar\")\n\tfsub1, _ := nc.SubscribeSync(\"bar.v1.>\")\n\tfsub2, _ := nc.SubscribeSync(\"baz.v2.>\")\n\tzsub, _ := nc.SubscribeSync(\"RAB.>\")\n\tf22sub, _ := nc.SubscribeSync(\"foo.*\")\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\tt.Fatalf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t}\n\t}\n\n\tnc.Publish(\"foo\", nil)\n\tnc.Publish(\"bar.11.22\", nil)\n\n\ttotal := 500\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo.22\", nil)\n\t}\n\tnc.Flush()\n\n\tcheckPending(bsub, 1)\n\tcheckPending(zsub, 1)\n\n\tfpending, _, _ := f22sub.Pending()\n\tfpending1, _, _ := fsub1.Pending()\n\tfpending2, _, _ := fsub2.Pending()\n\n\tif fpending1 < fpending2 || fpending < fpending2 {\n\t\tt.Fatalf(\"Loadbalancing seems off for the foo.* mappings: %d and %d and %d\", fpending, fpending1, fpending2)\n\t}\n}\n\nfunc TestAccountRouteMappingsConfiguration(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n\taccounts {\n\t\tsynadia {\n\t\t\tusers = [{user: derek, password: foo}]\n\t\t\tmappings = {\n\t\t\t\tfoo: bar\n\t\t\t\tfoo.*: [ { dest: bar.v1.$1, weight: 40% }, { destination: baz.v2.$1, weight: 20 } ]\n\t\t\t\tbar.*.*: RAB.$2.$1\n\t\t }\n\t\t}\n\t}\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\t// We test functionality above, so for this one just make sure we have mappings for the account.\n\tacc, _ := s.LookupAccount(\"synadia\")\n\tif !acc.hasMappings() {\n\t\tt.Fatalf(\"Account %q does not have mappings\", \"synadia\")\n\t}\n\n\taz, err := s.Accountz(&AccountzOptions{\"synadia\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting Accountz: %v\", err)\n\t}\n\tif az.Account == nil {\n\t\tt.Fatalf(\"Expected an Account\")\n\t}\n\tif len(az.Account.Mappings) != 3 {\n\t\tt.Fatalf(\"Expected %d mappings, saw %d\", 3, len(az.Account.Mappings))\n\t}\n}\n\nfunc TestAccountRouteMappingsWithLossInjection(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n\tmappings = {\n\t\tfoo: { dest: foo, weight: 80% }\n\t\tbar: { dest: bar, weight: 0% }\n }\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(\"foo\")\n\n\ttotal := 1000\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo\", nil)\n\t}\n\tnc.Flush()\n\n\tif pending, _, _ := sub.Pending(); pending == total {\n\t\tt.Fatalf(\"Expected some loss and pending to not be same as sent\")\n\t}\n\n\tsub, _ = nc.SubscribeSync(\"bar\")\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"bar\", nil)\n\t}\n\tnc.Flush()\n\n\tif pending, _, _ := sub.Pending(); pending != 0 {\n\t\tt.Fatalf(\"Expected all messages to be dropped and pending to be 0, got %d\", pending)\n\t}\n}\n\nfunc TestAccountRouteMappingsWithOriginClusterFilter(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n\tmappings = {\n\t\tfoo: { dest: bar, cluster: SYN, weight: 100% }\n }\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(\"foo\")\n\n\ttotal := 1000\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo\", nil)\n\t}\n\tnc.Flush()\n\n\tif pending, _, _ := sub.Pending(); pending != total {\n\t\tt.Fatalf(\"Expected pending to be %d, got %d\", total, pending)\n\t}\n\n\ts.setClusterName(\"SYN\")\n\tsub, _ = nc.SubscribeSync(\"bar\")\n\tfor i := 0; i < total; i++ {\n\t\tnc.Publish(\"foo\", nil)\n\t}\n\tnc.Flush()\n\n\tif pending, _, _ := sub.Pending(); pending != total {\n\t\tt.Fatalf(\"Expected pending to be %d, got %d\", total, pending)\n\t}\n}\n\nfunc TestAccountServiceImportWithRouteMappings(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n accounts {\n foo {\n users = [{user: derek, password: foo}]\n exports = [{service: \"request\"}]\n }\n bar {\n users = [{user: ivan, password: bar}]\n imports = [{service: {account: \"foo\", subject:\"request\"}}]\n }\n }\n `))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(\"foo\")\n\tacc.AddMapping(\"request\", \"request.v2\")\n\n\t// Create the service client first.\n\tncFoo := natsConnect(t, fmt.Sprintf(\"nats://derek:foo@%s:%d\", opts.Host, opts.Port))\n\tdefer ncFoo.Close()\n\n\tfooSub := natsSubSync(t, ncFoo, \"request.v2\")\n\tncFoo.Flush()\n\n\t// Requestor\n\tncBar := natsConnect(t, fmt.Sprintf(\"nats://ivan:bar@%s:%d\", opts.Host, opts.Port))\n\tdefer ncBar.Close()\n\n\tncBar.Publish(\"request\", nil)\n\tncBar.Flush()\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif n, _, _ := fooSub.Pending(); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected a request for %q, but got %d\", fooSub.Subject, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestAccountImportsWithWildcardSupport(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n accounts {\n foo {\n users = [{user: derek, password: foo}]\n exports = [\n { service: \"request.*\" }\n { stream: \"events.>\" }\n { stream: \"info.*.*.>\" }\n ]\n }\n bar {\n users = [{user: ivan, password: bar}]\n imports = [\n { service: {account: \"foo\", subject:\"request.*\"}, to:\"my.request.*\"}\n { stream: {account: \"foo\", subject:\"events.>\"}, to:\"foo.events.>\"}\n { stream: {account: \"foo\", subject:\"info.*.*.>\"}, to:\"foo.info.$2.$1.>\"}\n ]\n }\n }\n `))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tncFoo := natsConnect(t, fmt.Sprintf(\"nats://derek:foo@%s:%d\", opts.Host, opts.Port))\n\tdefer ncFoo.Close()\n\n\tncBar := natsConnect(t, fmt.Sprintf(\"nats://ivan:bar@%s:%d\", opts.Host, opts.Port))\n\tdefer ncBar.Close()\n\n\t// Create subscriber for the service endpoint in foo.\n\t_, err := ncFoo.QueueSubscribe(\"request.*\", \"t22\", func(m *nats.Msg) {\n\t\tif m.Subject != \"request.22\" {\n\t\t\tt.Fatalf(\"Expected literal subject for request, got %q\", m.Subject)\n\t\t}\n\t\tm.Respond([]byte(\"yes!\"))\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tncFoo.Flush()\n\n\t// Now test service import.\n\tresp, err := ncBar.Request(\"my.request.22\", []byte(\"yes?\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\tif string(resp.Data) != \"yes!\" {\n\t\tt.Fatalf(\"Expected a response of %q, got %q\", \"yes!\", resp.Data)\n\t}\n\n\t// Now test stream imports.\n\tesub, _ := ncBar.SubscribeSync(\"foo.events.*\") // subset\n\tisub, _ := ncBar.SubscribeSync(\"foo.info.>\")\n\tncBar.Flush()\n\n\t// Now publish some stream events.\n\tncFoo.Publish(\"events.22\", nil)\n\tncFoo.Publish(\"info.11.22.bar\", nil)\n\tncFoo.Flush()\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckPending(esub, 1)\n\tcheckPending(isub, 1)\n\n\t// Now check to make sure the subjects are correct etc.\n\tm, err := esub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif m.Subject != \"foo.events.22\" {\n\t\tt.Fatalf(\"Incorrect subject for stream import, expected %q, got %q\", \"foo.events.22\", m.Subject)\n\t}\n\n\tm, err = isub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif m.Subject != \"foo.info.22.11.bar\" {\n\t\tt.Fatalf(\"Incorrect subject for stream import, expected %q, got %q\", \"foo.info.22.11.bar\", m.Subject)\n\t}\n}\n\n// duplicates TestJWTAccountImportsWithWildcardSupport (jwt_test.go) in config\nfunc TestAccountImportsWithWildcardSupportStreamAndService(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n accounts {\n foo {\n users = [{user: derek, password: foo}]\n exports = [\n { service: \"$request.*.$in.*.>\" }\n { stream: \"$events.*.$in.*.>\" }\n ]\n }\n bar {\n users = [{user: ivan, password: bar}]\n imports = [\n { service: {account: \"foo\", subject:\"$request.*.$in.*.>\"}, to:\"my.request.$2.$1.>\"}\n { stream: {account: \"foo\", subject:\"$events.*.$in.*.>\"}, to:\"my.events.$2.$1.>\"}\n ]\n }\n }\n `))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tncFoo := natsConnect(t, fmt.Sprintf(\"nats://derek:foo@%s:%d\", opts.Host, opts.Port))\n\tdefer ncFoo.Close()\n\n\tncBar := natsConnect(t, fmt.Sprintf(\"nats://ivan:bar@%s:%d\", opts.Host, opts.Port))\n\tdefer ncBar.Close()\n\n\t// Create subscriber for the service endpoint in foo.\n\t_, err := ncFoo.Subscribe(\"$request.>\", func(m *nats.Msg) {\n\t\tif m.Subject != \"$request.2.$in.1.bar\" {\n\t\t\tt.Fatalf(\"Expected literal subject for request, got %q\", m.Subject)\n\t\t}\n\t\tm.Respond([]byte(\"yes!\"))\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tncFoo.Flush()\n\n\t// Now test service import.\n\tif resp, err := ncBar.Request(\"my.request.1.2.bar\", []byte(\"yes?\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t} else if string(resp.Data) != \"yes!\" {\n\t\tt.Fatalf(\"Expected a response of %q, got %q\", \"yes!\", resp.Data)\n\t}\n\tsubBar, err := ncBar.SubscribeSync(\"my.events.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\tncBar.Flush()\n\n\tncFoo.Publish(\"$events.1.$in.2.bar\", nil)\n\n\tm, err := subBar.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\tif m.Subject != \"my.events.2.1.bar\" {\n\t\tt.Fatalf(\"Expected literal subject for request, got %q\", m.Subject)\n\t}\n}\n\nfunc BenchmarkNewRouteReply(b *testing.B) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\tg := s.globalAccount()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tg.newServiceReply(false)\n\t}\n}\n\nfunc TestSamplingHeader(t *testing.T) {\n\ttest := func(expectSampling bool, h http.Header) {\n\t\tt.Helper()\n\t\tb := strings.Builder{}\n\t\tb.WriteString(\"\\r\\n\") // simulate status line\n\t\th.Write(&b)\n\t\tb.WriteString(\"\\r\\n\")\n\t\thdrString := b.String()\n\t\tc := &client{parseState: parseState{msgBuf: []byte(hdrString), pa: pubArg{hdr: len(hdrString)}}}\n\t\tsample, hdr := shouldSample(&serviceLatency{0, \"foo\"}, c)\n\t\tif expectSampling {\n\t\t\tif !sample {\n\t\t\t\tt.Fatal(\"Expected to sample\")\n\t\t\t} else if hdr == nil {\n\t\t\t\tt.Fatal(\"Expected a header\")\n\t\t\t}\n\t\t\tfor k, v := range h {\n\t\t\t\tif hdr.Get(k) != v[0] {\n\t\t\t\t\tt.Fatal(\"Expect header to match\")\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tif sample {\n\t\t\t\tt.Fatal(\"Expected not to sample\")\n\t\t\t} else if hdr != nil {\n\t\t\t\tt.Fatal(\"Expected no header\")\n\t\t\t}\n\t\t}\n\t}\n\n\ttest(false, http.Header{\"Uber-Trace-Id\": []string{\"0:0:0:0\"}})\n\ttest(false, http.Header{\"Uber-Trace-Id\": []string{\"0:0:0:00\"}}) // one byte encoded as two hex digits\n\ttest(true, http.Header{\"Uber-Trace-Id\": []string{\"0:0:0:1\"}})\n\ttest(true, http.Header{\"Uber-Trace-Id\": []string{\"0:0:0:01\"}})\n\ttest(true, http.Header{\"Uber-Trace-Id\": []string{\"0:0:0:5\"}}) // debug and sample\n\ttest(true, http.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:1\"}})\n\ttest(true, http.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:479fefe9525eddb:0:1\"}})\n\ttest(false, http.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:5adb976bfc1f95c1:479fefe9525eddb:0\"}})\n\ttest(false, http.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:479fefe9525eddb:0:0\"}})\n\n\ttest(true, http.Header{\"X-B3-Sampled\": []string{\"1\"}})\n\ttest(false, http.Header{\"X-B3-Sampled\": []string{\"0\"}})\n\ttest(true, http.Header{\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"}}) // decision left to recipient\n\ttest(false, http.Header{\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"}, \"X-B3-Sampled\": []string{\"0\"}})\n\ttest(true, http.Header{\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"}, \"X-B3-Sampled\": []string{\"1\"}})\n\n\ttest(false, http.Header{\"B3\": []string{\"0\"}}) // deny only\n\ttest(false, http.Header{\"B3\": []string{\"0-0-0-0\"}})\n\ttest(false, http.Header{\"B3\": []string{\"0-0-0\"}})\n\ttest(true, http.Header{\"B3\": []string{\"0-0-1-0\"}})\n\ttest(true, http.Header{\"B3\": []string{\"0-0-1\"}})\n\ttest(true, http.Header{\"B3\": []string{\"0-0-d\"}}) // debug is not a deny\n\ttest(true, http.Header{\"B3\": []string{\"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1\"}})\n\ttest(true, http.Header{\"B3\": []string{\"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90\"}})\n\ttest(false, http.Header{\"B3\": []string{\"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-0-05e3ac9a4f6e3b90\"}})\n\n\ttest(true, http.Header{\"traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}})\n\ttest(true, http.Header{\"traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-27\"}})\n\ttest(false, http.Header{\"traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\"}})\n\ttest(false, http.Header{\"traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-22\"}})\n}\n\nfunc TestAccountSystemPermsWithGlobalAccess(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts {\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Make sure we can connect with no auth to global account as normal.\n\tnc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Make sure we can connect to the system account with correct credentials.\n\tsc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer sc.Close()\n}\n\nconst importSubscriptionOverlapTemplate = `\nlisten: 127.0.0.1:-1\naccounts: {\n ACCOUNT_X: {\n\tusers: [\n\t {user: publisher}\n\t]\n\texports: [\n\t {stream: %s}\n\t]\n },\n ACCOUNT_Y: {\n\tusers: [\n\t {user: subscriber}\n\t]\n\timports: [\n\t {stream: {account: ACCOUNT_X, subject: %s }, %s}\n\t]\n }\n}`\n\nfunc TestImportSubscriptionPartialOverlapWithPrefix(t *testing.T) {\n\tcf := createConfFile(t, []byte(fmt.Sprintf(importSubscriptionOverlapTemplate, \">\", \">\", \"prefix: myprefix\")))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tncX := natsConnect(t, fmt.Sprintf(\"nats://%s:%s@127.0.0.1:%d\", \"publisher\", \"\", opts.Port))\n\tdefer ncX.Close()\n\n\tncY := natsConnect(t, fmt.Sprintf(\"nats://%s:%s@127.0.0.1:%d\", \"subscriber\", \"\", opts.Port))\n\tdefer ncY.Close()\n\n\tfor _, subj := range []string{\">\", \"myprefix.*\", \"myprefix.>\", \"myprefix.test\", \"*.>\", \"*.*\", \"*.test\"} {\n\t\tt.Run(subj, func(t *testing.T) {\n\t\t\tsub, err := ncY.SubscribeSync(subj)\n\t\t\tsub.AutoUnsubscribe(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, ncY.Flush())\n\n\t\t\tncX.Publish(\"test\", []byte(\"hello\"))\n\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, string(m.Data) == \"hello\")\n\t\t})\n\t}\n}\n\nfunc TestImportSubscriptionPartialOverlapWithTransform(t *testing.T) {\n\tcf := createConfFile(t, []byte(fmt.Sprintf(importSubscriptionOverlapTemplate, \"*.*.>\", \"*.*.>\", \"to: myprefix.$2.$1.>\")))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tncX := natsConnect(t, fmt.Sprintf(\"nats://%s:%s@127.0.0.1:%d\", \"publisher\", \"\", opts.Port))\n\tdefer ncX.Close()\n\n\tncY := natsConnect(t, fmt.Sprintf(\"nats://%s:%s@127.0.0.1:%d\", \"subscriber\", \"\", opts.Port))\n\tdefer ncY.Close()\n\n\tfor _, subj := range []string{\">\", \"*.*.*.>\", \"*.2.*.>\", \"*.*.1.>\", \"*.2.1.>\", \"*.*.*.*\", \"*.2.1.*\", \"*.*.*.test\",\n\t\t\"*.*.1.test\", \"*.2.*.test\", \"*.2.1.test\", \"myprefix.*.*.*\", \"myprefix.>\", \"myprefix.*.>\", \"myprefix.*.*.>\",\n\t\t\"myprefix.2.>\", \"myprefix.2.1.>\", \"myprefix.*.1.>\", \"myprefix.2.*.>\", \"myprefix.2.1.*\", \"myprefix.*.*.test\",\n\t\t\"myprefix.2.1.test\"} {\n\t\tt.Run(subj, func(t *testing.T) {\n\t\t\tsub, err := ncY.SubscribeSync(subj)\n\t\t\tsub.AutoUnsubscribe(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, ncY.Flush())\n\n\t\t\tncX.Publish(\"1.2.test\", []byte(\"hello\"))\n\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, string(m.Data) == \"hello\")\n\t\t\trequire_Equal(t, m.Subject, \"myprefix.2.1.test\")\n\t\t})\n\t}\n}\n\nfunc TestAccountLimitsServerConfig(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tport: -1\n\tmax_connections: 10\n\taccounts {\n\t\tMAXC {\n\t\t\tusers = [{user: derek, password: foo}]\n\t\t\tlimits {\n\t\t\t\tmax_connections: 5\n\t\t\t\tmax_subs: 10\n\t\t\t\tmax_payload: 32k\n\t\t\t\tmax_leafnodes: 1\n\t\t\t}\n\t\t}\n\t}\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tacc, err := s.lookupAccount(\"MAXC\")\n\trequire_NoError(t, err)\n\n\tif mc := acc.MaxActiveConnections(); mc != 5 {\n\t\tt.Fatalf(\"Did not set MaxActiveConnections properly, expected 5, got %d\", mc)\n\t}\n\tif mlc := acc.MaxActiveLeafNodes(); mlc != 1 {\n\t\tt.Fatalf(\"Did not set MaxActiveLeafNodes properly, expected 1, got %d\", mlc)\n\t}\n\n\t// Do quick test on connections, but if they are registered above should be good.\n\tfor i := 0; i < 5; i++ {\n\t\tc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"derek\", \"foo\"))\n\t\trequire_NoError(t, err)\n\t\tdefer c.Close()\n\t}\n\n\t// Should fail.\n\t_, err = nats.Connect(s.ClientURL(), nats.UserInfo(\"derek\", \"foo\"))\n\trequire_Error(t, err)\n}\n\n// Connections being closed should be the newer ones in case of JWT limits.\nfunc TestAccountMaxConnectionsDisconnectsNewestFirst(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n port: -1\n server_name: A\n accounts {\n TEST {\n users = [\n {user: user1, password: foo}\n {user: user2, password: foo}\n {user: user3, password: foo}\n ]\n limits {\n max_connections: 3\n }\n }\n }\n `))\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tvar conns []*nats.Conn\n\n\tdisconnects := make([]chan error, 0)\n\tfor i := 1; i <= 3; i++ {\n\t\tdisconnectCh := make(chan error)\n\t\tc, err := nats.Connect(\n\t\t\ts.ClientURL(),\n\t\t\tnats.UserInfo(fmt.Sprintf(\"user%d\", i), \"foo\"),\n\t\t\tnats.DisconnectErrHandler(func(_ *nats.Conn, err error) {\n\t\t\t\tdisconnectCh <- err\n\t\t\t}),\n\t\t\tnats.NoReconnect(),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer c.Close()\n\t\tconns = append(conns, c)\n\t\tdisconnects = append(disconnects, disconnectCh)\n\t\t// Small delay to ensure distinct start times.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\tacc, err := s.lookupAccount(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, acc.NumConnections(), 3)\n\n\t// Force account update to trigger connection limit enforcement.\n\taccClaims := jwt.NewAccountClaims(acc.Name)\n\taccClaims.Limits.Conn = 2\n\ts.UpdateAccountClaims(acc, accClaims)\n\n\t// Wait for disconnections from the most recent client.\n\tdisconnectCh := disconnects[2]\n\tselect {\n\tcase <-disconnectCh:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected newest connection to disconnect!\")\n\t}\n\n\t// Check which connections are still active\n\t// The newest connection (last one created) should be disconnected first.\n\tactiveConnections := 0\n\tvar connected []int\n\tfor i, conn := range conns {\n\t\tif !conn.IsClosed() {\n\t\t\tactiveConnections++\n\t\t\tconnected = append(connected, i)\n\t\t}\n\t}\n\trequire_Equal(t, activeConnections, 2)\n\trequire_Equal(t, len(connected), 2)\n\n\t// The first two connections should still be connected.\n\trequire_Equal(t, connected[0], 0)\n\trequire_Equal(t, connected[1], 1)\n\n\t// The newest connection should be closed.\n\trequire_True(t, conns[2].IsClosed())\n}\n\nfunc TestAccountUpdateRemoteServerDisconnectsNewestFirst(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n port: -1\n accounts {\n TEST {\n users = [{user: dummy, password: foo}]\n limits {\n max_connections: 5\n }\n }\n }\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tacc, err := s.lookupAccount(\"TEST\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\tc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"dummy\", \"foo\"))\n\t\trequire_NoError(t, err)\n\t\tdefer c.Close()\n\n\t\t// Small delay to ensure distinct start times.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\trequire_Equal(t, acc.NumConnections(), 3)\n\n\t// Simulate remote server reporting connections that would exceed the limit.\n\t// Remote has 4 + we have 3, meaning that 2 will have to be disconnected.\n\tremoteServerMsg := &AccountNumConns{\n\t\tServer: ServerInfo{\n\t\t\tID: \"fake-server-1\",\n\t\t\tName: \"fake-nats-1\",\n\t\t},\n\t\tAccountStat: AccountStat{\n\t\t\tAccount: \"TEST\",\n\t\t\tConns: 4,\n\t\t},\n\t}\n\ttoDisconnect := acc.updateRemoteServer(remoteServerMsg)\n\n\t// Should want to disconnect 2 clients (7 total - 5 max conns limit = 2 over).\n\trequire_Equal(t, len(toDisconnect), 2)\n\n\t// Verify the returned clients are in reverse chronological order.\n\tvar startTimes []time.Time\n\tfor _, c := range toDisconnect {\n\t\tstartTimes = append(startTimes, c.start)\n\t}\n\tif !startTimes[0].After(startTimes[1]) {\n\t\tt.Fatalf(\"Expected clients to be in reverse chronological order: %v is before %v\", startTimes[0], startTimes[1])\n\t}\n\n\t// The clients to disconnect should be the newest ones we created.\n\tdisconnected := make(map[uint64]bool)\n\tfor _, c := range toDisconnect {\n\t\tdisconnected[c.cid] = true\n\t}\n\n\t// Get all current clients and find the oldest ones.\n\tallClients := acc.getClients()\n\tbyStartTime := make([]*client, len(allClients))\n\tcopy(byStartTime, allClients)\n\tslices.SortFunc(byStartTime, func(i, j *client) int {\n\t\treturn i.start.Compare(j.start)\n\t})\n\n\t// The last two clients in the sorted list should be the ones selected for disconnection.\n\trequire_False(t, disconnected[byStartTime[0].cid])\n\trequire_True(t, disconnected[byStartTime[1].cid])\n\trequire_True(t, disconnected[byStartTime[2].cid])\n}\n\nfunc TestAccountMaxConnectionsDuringLameDuckMode(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n port: -1\n accounts {\n TEST {\n users = [\n {user: user, password: user}\n ]\n limits {\n max_connections: 3\n }\n }\n }\n\t\tcluster {\n\t\t listen: 127.0.0.1:7248\n\t\t name: \"abc\"\n\t\t}\n\t\tno_auth_user: user\n\t\tlame_duck_grace_period: 1s\n `))\n\n\toptsA, err := ProcessConfigFile(cf)\n\trequire_NoError(t, err)\n\toptsA.NoSigs, optsA.NoLog = true, true\n\toptsA.ServerName = \"A\"\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\toptsB.ServerName = \"B\"\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tdisconnects := make([]chan error, 0)\n\tfor range 3 {\n\t\tdisconnectCh := make(chan error)\n\t\tc, err := nats.Connect(\n\t\t\tsrvA.ClientURL(),\n\t\t\tnats.UserInfo(\"user\", \"user\"),\n\t\t\tnats.DisconnectErrHandler(func(c *nats.Conn, err error) {\n\t\t\t\tdisconnectCh <- err\n\t\t\t}),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer c.Close()\n\t\tdisconnects = append(disconnects, disconnectCh)\n\t\t// Small delay to ensure distinct start times.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\tacc, err := srvA.lookupAccount(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, acc.NumConnections(), 3)\n\n\tgo srvA.LameDuckShutdown()\n\n\tfor _, disconnectCh := range disconnects {\n\t\tselect {\n\t\tcase err := <-disconnectCh:\n\t\t\trequire_Equal(t, err.Error(), \"EOF\")\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatal(\"Expected LDM reconnect\")\n\t\t}\n\t}\n}\n\nfunc TestAccountUserSubPermsWithQueueGroups(t *testing.T) {\n\tcf := createConfFile(t, []byte(`\n\tlisten: 127.0.0.1:-1\n\n\tauthorization {\n\tusers = [\n\t\t{ user: user, password: \"pass\",\n\t\t\tpermissions: {\n\t\t\t\tpublish: \"foo.restricted\"\n\t\t\t\tsubscribe: { allow: \"foo.>\", deny: \"foo.restricted\" }\n\t\t\t\tallow_responses: { max: 1, ttl: 0s }\n\t\t\t}\n\t\t}\n\t]}\n `))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"user\", \"pass\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// qsub solo.\n\tqsub, err := nc.QueueSubscribeSync(\"foo.>\", \"qg\")\n\trequire_NoError(t, err)\n\n\terr = nc.Publish(\"foo.restricted\", []byte(\"RESTRICTED\"))\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\t// Expect no msgs.\n\tcheckSubsPending(t, qsub, 0)\n}\n\nfunc TestAccountImportCycle(t *testing.T) {\n\ttmpl := `\n\tport: -1\n\taccounts: {\n\t\tCP: {\n\t\t\tusers: [\n\t\t\t\t{user: cp, password: cp},\n\t\t\t],\n\t\t\texports: [\n\t\t\t\t{service: \"q1.>\", response_type: Singleton},\n\t\t\t\t{service: \"q2.>\", response_type: Singleton},\n\t\t\t\t%s\n\t\t\t],\n\t\t},\n\t\tA: {\n\t\t\tusers: [\n\t\t\t\t{user: a, password: a},\n\t\t\t],\n\t\t\timports: [\n\t\t\t\t{service: {account: CP, subject: \"q1.>\"}},\n\t\t\t\t{service: {account: CP, subject: \"q2.>\"}},\n\t\t\t\t%s\n\t\t\t]\n\t\t},\n\t}\n\t`\n\tcf := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, _EMPTY_)))\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\tncCp, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"cp\", \"cp\"))\n\trequire_NoError(t, err)\n\tdefer ncCp.Close()\n\tncA, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"a\", \"a\"))\n\trequire_NoError(t, err)\n\tdefer ncA.Close()\n\t// setup responder\n\tnatsSub(t, ncCp, \"q1.>\", func(m *nats.Msg) { m.Respond([]byte(\"reply\")) })\n\t// setup requestor\n\tib := \"q2.inbox\"\n\tsubAResp, err := ncA.SubscribeSync(ib)\n\tncA.Flush()\n\trequire_NoError(t, err)\n\treq := func() {\n\t\tt.Helper()\n\t\t// send request\n\t\terr = ncA.PublishRequest(\"q1.a\", ib, []byte(\"test\"))\n\t\tncA.Flush()\n\t\trequire_NoError(t, err)\n\t\tmRep, err := subAResp.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, string(mRep.Data), \"reply\")\n\t}\n\treq()\n\n\t// Update the config and do a config reload and make sure it all still work\n\tchangeCurrentConfigContentWithNewContent(t, cf, []byte(\n\t\tfmt.Sprintf(tmpl, `{service: \"q3.>\", response_type: Singleton},`, `{service: {account: CP, subject: \"q3.>\"}},`)))\n\terr = s.Reload()\n\trequire_NoError(t, err)\n\treq()\n}\n\nfunc TestAccountImportOwnExport(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts: {\n\t\t\tA: {\n\t\t\texports: [\n\t\t\t\t{ service: echo, accounts: [A], latency: { subject: \"latency.echo\" } }\n\t\t\t],\n\t\t\timports: [\n\t\t\t\t{ service: { account: A, subject: echo } }\n\t\t\t]\n\n\t\t\tusers: [\n\t\t\t\t{ user: user, pass: pass }\n\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"user\", \"pass\"))\n\tdefer nc.Close()\n\n\tnatsSub(t, nc, \"echo\", func(m *nats.Msg) { m.Respond(nil) })\n\t_, err := nc.Request(\"echo\", []byte(\"request\"), time.Second)\n\trequire_NoError(t, err)\n}\n\n// Test for a bug that would cause duplicate deliveries in certain situations when\n// service export/imports and leafnodes involved.\n// https://github.com/nats-io/nats-server/issues/3191\nfunc TestAccountImportDuplicateResponseDeliveryWithLeafnodes(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tusers = [{user: A, password: P}]\n\t\t\t\texports: [ { service: \"foo\", response_type: stream } ]\n\t\t\t}\n\t\t\tB: {\n\t\t\t\tusers = [{user: B, password: P}]\n\t\t\t\timports: [ { service: {account: \"A\", subject:\"foo\"} } ]\n\t\t\t}\n\t\t}\n\t\tleaf { listen: \"127.0.0.1:17222\" }\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Requestors will connect to account B.\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"B\", \"P\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// By sending a request (regardless of no responders), this will trigger a wildcard _R_ subscription since\n\t// we do not have a leafnode connected.\n\tnc.PublishRequest(\"foo\", \"reply\", nil)\n\tnc.Flush()\n\n\t// Now connect the LN. This will be where the service responder lives.\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [ { url: \"nats://A:P@127.0.0.1:17222\" } ]\n\t\t}\n\t`))\n\tln, _ := RunServerWithConfig(conf)\n\tdefer ln.Shutdown()\n\tcheckLeafNodeConnected(t, s)\n\n\t// Now attach a responder to the LN.\n\tlnc, err := nats.Connect(ln.ClientURL())\n\trequire_NoError(t, err)\n\tdefer lnc.Close()\n\n\tlnc.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"bar\"))\n\t})\n\tlnc.Flush()\n\tcheckSubInterest(t, s, \"A\", \"foo\", time.Second)\n\n\t// Make sure it works, but request only wants one, so need second test to show failure, but\n\t// want to make sure we are wired up correctly.\n\t_, err = nc.Request(\"foo\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Now setup inbox reply so we can check if we get multiple responses.\n\treply := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\n\tnc.PublishRequest(\"foo\", reply, nil)\n\n\t// Do another to make sure we know the other request will have been processed too.\n\t_, err = nc.Request(\"foo\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tif n, _, _ := sub.Pending(); n > 1 {\n\t\tt.Fatalf(\"Expected only 1 response, got %d\", n)\n\t}\n}\n\nfunc TestAccountReloadServiceImportPanic(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers = [ { user: \"a\", pass: \"p\" } ]\n\t\t\t\texports [ { service: \"HELP\" } ]\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers = [ { user: \"b\", pass: \"p\" } ]\n\t\t\t\timports [ { service: { account: A, subject: \"HELP\"} } ]\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Now connect up the subscriber for HELP. No-op for this test.\n\tnc, _ := jsClientConnect(t, s, nats.UserInfo(\"a\", \"p\"))\n\tdefer nc.Close()\n\n\t_, err := nc.Subscribe(\"HELP\", func(m *nats.Msg) { m.Respond([]byte(\"OK\")) })\n\trequire_NoError(t, err)\n\t// Make sure the subscription is processed before using a new connection to publish.\n\tnatsFlush(t, nc)\n\n\t// Now create connection to account b where we will publish to HELP.\n\tnc, _ = jsClientConnect(t, s, nats.UserInfo(\"b\", \"p\"))\n\tdefer nc.Close()\n\n\t// We want to continually be publishing messages that will trigger the service import while calling reload.\n\tdone := make(chan bool)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\tvar requests, responses atomic.Uint64\n\treply := nats.NewInbox()\n\t_, err = nc.Subscribe(reply, func(m *nats.Msg) { responses.Add(1) })\n\trequire_NoError(t, err)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tnc.PublishRequest(\"HELP\", reply, []byte(\"HELP\"))\n\t\t\t\trequests.Add(1)\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Perform a bunch of reloads.\n\tfor i := 0; i < 1000; i++ {\n\t\trequire_NoError(t, s.Reload())\n\t}\n\n\tclose(done)\n\twg.Wait()\n\n\ttotalRequests := requests.Load()\n\tcheckFor(t, 20*time.Second, 250*time.Millisecond, func() error {\n\t\tresp := responses.Load()\n\t\tif resp == totalRequests {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Have not received all responses, want %d got %d\", totalRequests, resp)\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/4674\nfunc TestAccountServiceAndStreamExportDoubleDelivery(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\ttenant1: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ { user: \"one\", password: \"one\" } ]\n\t\t\t\texports: [\n\t\t\t\t\t{ stream: \"DW.>\" }\n\t\t\t\t\t{ service: \"DW.>\" }\n\t\t\t\t]\n \t\t\t}\n\t\t\tglobal: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ { user: \"global\", password: \"global\" } ]\n\t\t\t\timports: [\n\t\t\t\t\t{ stream: { account: tenant1, subject: \"DW.>\" }, prefix: tenant1 }\n\t\t\t\t\t{ service: { account: tenant1, subject: \"DW.>\" }, to: \"tenant1.DW.>\" }\n\t\t\t\t]\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Now connect up the subscriber for HELP. No-op for this test.\n\tnc, _ := jsClientConnect(t, s, nats.UserInfo(\"one\", \"one\"))\n\tdefer nc.Close()\n\n\tvar msgs atomic.Int32\n\t_, err := nc.Subscribe(\">\", func(m *nats.Msg) {\n\t\tmsgs.Add(1)\n\t})\n\trequire_NoError(t, err)\n\n\tnc.Publish(\"DW.test.123\", []byte(\"test\"))\n\ttime.Sleep(200 * time.Millisecond)\n\trequire_Equal(t, msgs.Load(), 1)\n}\n\nfunc TestAccountServiceImportNoResponders(t *testing.T) {\n\t// Setup NATS server.\n\tcf := createConfFile(t, []byte(`\n\t\t\t\tport: -1\n\t\t\t\taccounts: {\n\t\t\t\t\taccExp: {\n\t\t\t\t\t\tusers: [{user: accExp, password: accExp}]\n\t\t\t\t\t\texports: [{service: \"foo\"}]\n\t\t\t\t\t}\n\t\t\t\t\taccImp: {\n\t\t\t\t\t\tusers: [{user: accImp, password: accImp}]\n\t\t\t\t\t\timports: [{service: {account: accExp, subject: \"foo\"}}]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\t// Connect to the import account. We will not setup any responders, so a request should\n\t// error out with ErrNoResponders.\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"accImp\", \"accImp\"))\n\tdefer nc.Close()\n\n\t_, err := nc.Request(\"foo\", []byte(\"request\"), 250*time.Millisecond)\n\trequire_Error(t, err, nats.ErrNoResponders)\n}\n" }, { "path": "server/ats/ats.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// ats controls the go routines for the access time service.\n// This allows more efficient unixnano operations for cache access times.\n// We will have one per binary (usually per server).\npackage ats\n\nimport (\n\t\"sync/atomic\"\n\t\"time\"\n)\n\n// Update every 100ms for gathering access time in unix nano.\nconst TickInterval = 100 * time.Millisecond\n\nvar (\n\t// Our unix nano time.\n\tutime atomic.Int64\n\t// How may registered users do we have, controls lifetime of Go routine.\n\trefs atomic.Int64\n\t// To signal the shutdown of the Go routine.\n\tdone chan struct{}\n)\n\nfunc init() {\n\t// Initialize our done chan.\n\tdone = make(chan struct{}, 1)\n}\n\n// Register usage. This will happen on filestore creation.\nfunc Register() {\n\tif v := refs.Add(1); v == 1 {\n\t\t// This is the first to register (could also go up and down),\n\t\t// so spin up Go routine and grab initial time.\n\t\tutime.Store(time.Now().UnixNano())\n\n\t\tgo func() {\n\t\t\tticker := time.NewTicker(TickInterval)\n\t\t\tdefer ticker.Stop()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-ticker.C:\n\t\t\t\t\tutime.Store(time.Now().UnixNano())\n\t\t\t\tcase <-done:\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n}\n\n// Unregister usage. We will shutdown the go routine if no more registered users.\nfunc Unregister() {\n\tif v := refs.Add(-1); v == 0 {\n\t\tdone <- struct{}{}\n\t} else if v < 0 {\n\t\trefs.Store(0)\n\t\tpanic(\"unbalanced unregister for access time state\")\n\t}\n}\n\n// Will load the access time from an atomic.\n// If no one has registered this will return 0 or stale data.\n// It is the responsibility of the user to properly register and unregister.\nfunc AccessTime() int64 {\n\t// Return last updated time.\n\tv := utime.Load()\n\tif v == 0 {\n\t\t// Always register a time, the worst case is a stale time.\n\t\t// On startup, we can register in parallel and could previously panic.\n\t\tv = time.Now().UnixNano()\n\t\tutime.Store(v)\n\t}\n\treturn v\n}\n" }, { "path": "server/ats/ats_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ats\n\nimport (\n\t\"runtime\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestNotRunningValue(t *testing.T) {\n\t// Set back to zero in case this test gets run multiple times via --count.\n\tutime.Store(0)\n\tat := AccessTime()\n\tif at == 0 {\n\t\tt.Fatal(\"Expected non-zero access time\")\n\t}\n\n\tatn := AccessTime()\n\tif atn != at {\n\t\tt.Fatal(\"Did not expect updates to access time\")\n\t}\n}\n\nfunc TestRegisterAndUnregister(t *testing.T) {\n\tngrp := runtime.NumGoroutine()\n\n\tRegister()\n\tif r := refs.Load(); r != 1 {\n\t\tt.Fatalf(\"Expected refs to be 1, got %d\", r)\n\t}\n\t// Make sure we have a non-zero time.\n\tat := AccessTime()\n\tif at == 0 {\n\t\tt.Fatal(\"Expected non-zero access time\")\n\t}\n\t// Make sure we are updating access time.\n\ttime.Sleep(2 * TickInterval)\n\tatn := AccessTime()\n\tif atn == at {\n\t\tt.Fatal(\"Expected access time to be updated but it was not\")\n\t}\n\n\t// Now unregister.\n\tUnregister()\n\tif r := refs.Load(); r != 0 {\n\t\tt.Fatalf(\"Expected refs to be 0, got %d\", r)\n\t}\n\ttime.Sleep(TickInterval)\n\n\tat = AccessTime()\n\ttime.Sleep(2 * TickInterval)\n\tatn = AccessTime()\n\tif atn != at {\n\t\tt.Fatal(\"Did not expect updates to access time\")\n\t}\n\n\t// Check that we have no additional go routines running.\n\tngra := runtime.NumGoroutine()\n\tif ngra != ngrp {\n\t\tt.Fatalf(\"Expected same number of go routines after removing all registered: %d vs %d\", ngrp, ngra)\n\t}\n\n\t// Check that we spin back up on going from zero to one registered after spinning down.\n\tRegister()\n\tdefer Unregister()\n\n\tat = AccessTime()\n\ttime.Sleep(2 * TickInterval)\n\tatn = AccessTime()\n\tif atn == at {\n\t\tt.Fatal(\"Expected access time to be updated but it was not\")\n\t}\n\tngra = runtime.NumGoroutine()\n\tif ngra != ngrp+1 {\n\t\tt.Fatalf(\"Expected to see additional go routine: %d vs %d\", ngrp, ngra)\n\t}\n}\n\nfunc TestUnbalancedUnregister(t *testing.T) {\n\tdefer func() {\n\t\tif r := recover(); r == nil {\n\t\t\tt.Errorf(\"Expected function to panic, but it did not\")\n\t\t}\n\t}()\n\tUnregister()\n}\n" }, { "path": "server/auth.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"crypto/sha256\"\n\t\"crypto/subtle\"\n\t\"crypto/tls\"\n\t\"crypto/x509/pkix\"\n\t\"encoding/asn1\"\n\t\"encoding/base64\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"regexp\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/internal/ldap\"\n\t\"github.com/nats-io/nkeys\"\n\t\"golang.org/x/crypto/bcrypt\"\n)\n\n// Authentication is an interface for implementing authentication\ntype Authentication interface {\n\t// Check if a client is authorized to connect\n\tCheck(c ClientAuthentication) bool\n}\n\n// ClientAuthentication is an interface for client authentication\ntype ClientAuthentication interface {\n\t// GetOpts gets options associated with a client\n\tGetOpts() *ClientOpts\n\t// GetTLSConnectionState if TLS is enabled, TLS ConnectionState, nil otherwise\n\tGetTLSConnectionState() *tls.ConnectionState\n\t// RegisterUser optionally map a user after auth.\n\tRegisterUser(*User)\n\t// RemoteAddress expose the connection information of the client\n\tRemoteAddress() net.Addr\n\t// GetNonce is the nonce presented to the user in the INFO line\n\tGetNonce() []byte\n\t// Kind indicates what type of connection this is matching defined constants like CLIENT, ROUTER, GATEWAY, LEAF etc\n\tKind() int\n}\n\n// NkeyUser is for multiple nkey based users\ntype NkeyUser struct {\n\tNkey string `json:\"user\"`\n\tIssued int64 `json:\"issued,omitempty\"` // this is a copy of the issued at (iat) field in the jwt\n\tPermissions *Permissions `json:\"permissions,omitempty\"`\n\tAccount *Account `json:\"account,omitempty\"`\n\tSigningKey string `json:\"signing_key,omitempty\"`\n\tAllowedConnectionTypes map[string]struct{} `json:\"connection_types,omitempty\"`\n\tProxyRequired bool `json:\"proxy_required,omitempty\"`\n}\n\n// User is for multiple accounts/users.\ntype User struct {\n\tUsername string `json:\"user\"`\n\tPassword string `json:\"password\"`\n\tPermissions *Permissions `json:\"permissions,omitempty\"`\n\tAccount *Account `json:\"account,omitempty\"`\n\tConnectionDeadline time.Time `json:\"connection_deadline,omitempty\"`\n\tAllowedConnectionTypes map[string]struct{} `json:\"connection_types,omitempty\"`\n\tProxyRequired bool `json:\"proxy_required,omitempty\"`\n}\n\n// clone performs a deep copy of the User struct, returning a new clone with\n// all values copied.\nfunc (u *User) clone() *User {\n\tif u == nil {\n\t\treturn nil\n\t}\n\tclone := &User{}\n\t*clone = *u\n\t// Account is not cloned because it is always by reference to an existing struct.\n\tclone.Permissions = u.Permissions.clone()\n\n\tif u.AllowedConnectionTypes != nil {\n\t\tclone.AllowedConnectionTypes = make(map[string]struct{})\n\t\tfor k, v := range u.AllowedConnectionTypes {\n\t\t\tclone.AllowedConnectionTypes[k] = v\n\t\t}\n\t}\n\n\treturn clone\n}\n\n// clone performs a deep copy of the NkeyUser struct, returning a new clone with\n// all values copied.\nfunc (n *NkeyUser) clone() *NkeyUser {\n\tif n == nil {\n\t\treturn nil\n\t}\n\tclone := &NkeyUser{}\n\t*clone = *n\n\t// Account is not cloned because it is always by reference to an existing struct.\n\tclone.Permissions = n.Permissions.clone()\n\n\tif n.AllowedConnectionTypes != nil {\n\t\tclone.AllowedConnectionTypes = make(map[string]struct{})\n\t\tfor k, v := range n.AllowedConnectionTypes {\n\t\t\tclone.AllowedConnectionTypes[k] = v\n\t\t}\n\t}\n\n\treturn clone\n}\n\n// SubjectPermission is an individual allow and deny struct for publish\n// and subscribe authorizations.\ntype SubjectPermission struct {\n\tAllow []string `json:\"allow,omitempty\"`\n\tDeny []string `json:\"deny,omitempty\"`\n}\n\n// ResponsePermission can be used to allow responses to any reply subject\n// that is received on a valid subscription.\ntype ResponsePermission struct {\n\tMaxMsgs int `json:\"max\"`\n\tExpires time.Duration `json:\"ttl\"`\n}\n\n// Permissions are the allowed subjects on a per\n// publish or subscribe basis.\ntype Permissions struct {\n\tPublish *SubjectPermission `json:\"publish\"`\n\tSubscribe *SubjectPermission `json:\"subscribe\"`\n\tResponse *ResponsePermission `json:\"responses,omitempty\"`\n}\n\n// RoutePermissions are similar to user permissions\n// but describe what a server can import/export from and to\n// another server.\ntype RoutePermissions struct {\n\tImport *SubjectPermission `json:\"import\"`\n\tExport *SubjectPermission `json:\"export\"`\n}\n\n// clone will clone an individual subject permission.\nfunc (p *SubjectPermission) clone() *SubjectPermission {\n\tif p == nil {\n\t\treturn nil\n\t}\n\tclone := &SubjectPermission{}\n\tif p.Allow != nil {\n\t\tclone.Allow = make([]string, len(p.Allow))\n\t\tcopy(clone.Allow, p.Allow)\n\t}\n\tif p.Deny != nil {\n\t\tclone.Deny = make([]string, len(p.Deny))\n\t\tcopy(clone.Deny, p.Deny)\n\t}\n\treturn clone\n}\n\n// clone performs a deep copy of the Permissions struct, returning a new clone\n// with all values copied.\nfunc (p *Permissions) clone() *Permissions {\n\tif p == nil {\n\t\treturn nil\n\t}\n\tclone := &Permissions{}\n\tif p.Publish != nil {\n\t\tclone.Publish = p.Publish.clone()\n\t}\n\tif p.Subscribe != nil {\n\t\tclone.Subscribe = p.Subscribe.clone()\n\t}\n\tif p.Response != nil {\n\t\tclone.Response = &ResponsePermission{\n\t\t\tMaxMsgs: p.Response.MaxMsgs,\n\t\t\tExpires: p.Response.Expires,\n\t\t}\n\t}\n\treturn clone\n}\n\n// checkAuthforWarnings will look for insecure settings and log concerns.\n// Lock is assumed held.\nfunc (s *Server) checkAuthforWarnings() {\n\twarn := false\n\topts := s.getOpts()\n\tif opts.Password != _EMPTY_ && !isBcrypt(opts.Password) {\n\t\twarn = true\n\t}\n\tfor _, u := range s.users {\n\t\t// Skip warn if using TLS certs based auth\n\t\t// unless a password has been left in the config.\n\t\tif u.Password == _EMPTY_ && opts.TLSMap {\n\t\t\tcontinue\n\t\t}\n\t\t// Check if this is our internal sys client created on the fly.\n\t\tif s.sysAccOnlyNoAuthUser != _EMPTY_ && u.Username == s.sysAccOnlyNoAuthUser {\n\t\t\tcontinue\n\t\t}\n\t\tif !isBcrypt(u.Password) {\n\t\t\twarn = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif warn {\n\t\t// Warning about using plaintext passwords.\n\t\ts.Warnf(\"Plaintext passwords detected, use nkeys or bcrypt\")\n\t}\n}\n\n// If Users or Nkeys options have definitions without an account defined,\n// assign them to the default global account.\n// Lock should be held.\nfunc (s *Server) assignGlobalAccountToOrphanUsers(nkeys map[string]*NkeyUser, users map[string]*User) {\n\tfor _, u := range users {\n\t\tif u.Account == nil {\n\t\t\tu.Account = s.gacc\n\t\t}\n\t}\n\tfor _, u := range nkeys {\n\t\tif u.Account == nil {\n\t\t\tu.Account = s.gacc\n\t\t}\n\t}\n}\n\n// If the given permissions has a ResponsePermission\n// set, ensure that defaults are set (if values are 0)\n// and that a Publish permission is set, and Allow\n// is disabled if not explicitly set.\nfunc validateResponsePermissions(p *Permissions) {\n\tif p == nil || p.Response == nil {\n\t\treturn\n\t}\n\tif p.Publish == nil {\n\t\tp.Publish = &SubjectPermission{}\n\t}\n\tif p.Publish.Allow == nil {\n\t\t// We turn off the blanket allow statement.\n\t\tp.Publish.Allow = []string{}\n\t}\n\t// If there is a response permission, ensure\n\t// that if value is 0, we set the default value.\n\tif p.Response.MaxMsgs == 0 {\n\t\tp.Response.MaxMsgs = DEFAULT_ALLOW_RESPONSE_MAX_MSGS\n\t}\n\tif p.Response.Expires == 0 {\n\t\tp.Response.Expires = DEFAULT_ALLOW_RESPONSE_EXPIRATION\n\t}\n}\n\n// configureAuthorization will do any setup needed for authorization.\n// Lock is assumed held.\nfunc (s *Server) configureAuthorization() {\n\topts := s.getOpts()\n\tif opts == nil {\n\t\treturn\n\t}\n\n\t// Check for multiple users first\n\t// This just checks and sets up the user map if we have multiple users.\n\tif opts.CustomClientAuthentication != nil {\n\t\ts.info.AuthRequired = true\n\t} else if s.trustedKeys != nil {\n\t\ts.info.AuthRequired = true\n\t} else if opts.Nkeys != nil || opts.Users != nil {\n\t\ts.nkeys, s.users = s.buildNkeysAndUsersFromOptions(opts.Nkeys, opts.Users)\n\t\ts.info.AuthRequired = true\n\t} else if opts.Username != _EMPTY_ || opts.Authorization != _EMPTY_ {\n\t\ts.info.AuthRequired = true\n\t} else {\n\t\ts.users = nil\n\t\ts.nkeys = nil\n\t\ts.info.AuthRequired = false\n\t}\n\n\t// Do similar for websocket config\n\ts.wsConfigAuth(&opts.Websocket)\n\t// And for mqtt config\n\ts.mqttConfigAuth(&opts.MQTT)\n\n\t// Check for server configured auth callouts.\n\tif opts.AuthCallout != nil {\n\t\ts.mu.Unlock()\n\t\t// Give operator log entries if not valid account and auth_users.\n\t\t_, err := s.lookupAccount(opts.AuthCallout.Account)\n\t\ts.mu.Lock()\n\t\tif err != nil {\n\t\t\ts.Errorf(\"Authorization callout account %q not valid\", opts.AuthCallout.Account)\n\t\t}\n\t\tfor _, u := range opts.AuthCallout.AuthUsers {\n\t\t\t// Check for user in users and nkeys since this is server config.\n\t\t\tvar found bool\n\t\t\tif len(s.users) > 0 {\n\t\t\t\t_, found = s.users[u]\n\t\t\t}\n\t\t\tif !found && len(s.nkeys) > 0 {\n\t\t\t\t_, found = s.nkeys[u]\n\t\t\t}\n\t\t\tif !found {\n\t\t\t\ts.Errorf(\"Authorization callout user %q not valid: %v\", u, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Takes the given slices of NkeyUser and User options and build\n// corresponding maps used by the server. The users are cloned\n// so that server does not reference options.\n// The global account is assigned to users that don't have an\n// existing account.\n// Server lock is held on entry.\nfunc (s *Server) buildNkeysAndUsersFromOptions(nko []*NkeyUser, uo []*User) (map[string]*NkeyUser, map[string]*User) {\n\tvar nkeys map[string]*NkeyUser\n\tvar users map[string]*User\n\n\tif nko != nil {\n\t\tnkeys = make(map[string]*NkeyUser, len(nko))\n\t\tfor _, u := range nko {\n\t\t\tcopy := u.clone()\n\t\t\tif u.Account != nil {\n\t\t\t\tif v, ok := s.accounts.Load(u.Account.Name); ok {\n\t\t\t\t\tcopy.Account = v.(*Account)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif copy.Permissions != nil {\n\t\t\t\tvalidateResponsePermissions(copy.Permissions)\n\t\t\t}\n\t\t\tnkeys[u.Nkey] = copy\n\t\t}\n\t}\n\tif uo != nil {\n\t\tusers = make(map[string]*User, len(uo))\n\t\tfor _, u := range uo {\n\t\t\tcopy := u.clone()\n\t\t\tif u.Account != nil {\n\t\t\t\tif v, ok := s.accounts.Load(u.Account.Name); ok {\n\t\t\t\t\tcopy.Account = v.(*Account)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif copy.Permissions != nil {\n\t\t\t\tvalidateResponsePermissions(copy.Permissions)\n\t\t\t}\n\t\t\tusers[u.Username] = copy\n\t\t}\n\t}\n\ts.assignGlobalAccountToOrphanUsers(nkeys, users)\n\treturn nkeys, users\n}\n\n// checkAuthentication will check based on client type and\n// return boolean indicating if client is authorized.\nfunc (s *Server) checkAuthentication(c *client) bool {\n\tswitch c.kind {\n\tcase CLIENT:\n\t\treturn s.isClientAuthorized(c)\n\tcase ROUTER:\n\t\treturn s.isRouterAuthorized(c)\n\tcase GATEWAY:\n\t\treturn s.isGatewayAuthorized(c)\n\tcase LEAF:\n\t\treturn s.isLeafNodeAuthorized(c)\n\tdefault:\n\t\treturn false\n\t}\n}\n\n// isClientAuthorized will check the client against the proper authorization method and data.\n// This could be nkey, token, or username/password based.\nfunc (s *Server) isClientAuthorized(c *client) bool {\n\topts := s.getOpts()\n\n\t// Check custom auth first, then jwts, then nkeys, then\n\t// multiple users with TLS map if enabled, then token,\n\t// then single user/pass.\n\tif opts.CustomClientAuthentication != nil && !opts.CustomClientAuthentication.Check(c) {\n\t\treturn false\n\t}\n\n\tif opts.CustomClientAuthentication == nil && !s.processClientOrLeafAuthentication(c, opts) {\n\t\treturn false\n\t}\n\n\tif c.kind == CLIENT || c.kind == LEAF {\n\t\t// Generate an event if we have a system account.\n\t\ts.accountConnectEvent(c)\n\t}\n\n\treturn true\n}\n\n// returns false if the client needs to be disconnected\nfunc (c *client) matchesPinnedCert(tlsPinnedCerts PinnedCertSet) bool {\n\tif tlsPinnedCerts == nil {\n\t\treturn true\n\t}\n\ttlsState := c.GetTLSConnectionState()\n\tif tlsState == nil || len(tlsState.PeerCertificates) == 0 || tlsState.PeerCertificates[0] == nil {\n\t\tc.Debugf(\"Failed pinned cert test as client did not provide a certificate\")\n\t\treturn false\n\t}\n\tsha := sha256.Sum256(tlsState.PeerCertificates[0].RawSubjectPublicKeyInfo)\n\tkeyId := hex.EncodeToString(sha[:])\n\tif _, ok := tlsPinnedCerts[keyId]; !ok {\n\t\tc.Debugf(\"Failed pinned cert test for key id: %s\", keyId)\n\t\treturn false\n\t}\n\treturn true\n}\n\nvar (\n\tmustacheRE = regexp.MustCompile(`{{2}([^}]+)}{2}`)\n\tmaxPermTemplateSubjectExpansions = 4096\n\terrPermTemplateExpansionLimit error = fmt.Errorf(\"template expansion exceeds limit\")\n)\n\nfunc processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.UserClaims, acc *Account) (jwt.UserPermissionLimits, error) {\n\tnArrayCartesianProduct := func(a ...[]string) [][]string {\n\t\tc := 1\n\t\tfor _, a := range a {\n\t\t\tc *= len(a)\n\t\t}\n\t\tif c == 0 {\n\t\t\treturn nil\n\t\t}\n\t\tp := make([][]string, c)\n\t\tb := make([]string, c*len(a))\n\t\tn := make([]int, len(a))\n\t\ts := 0\n\t\tfor i := range p {\n\t\t\te := s + len(a)\n\t\t\tpi := b[s:e]\n\t\t\tp[i] = pi\n\t\t\ts = e\n\t\t\tfor j, n := range n {\n\t\t\t\tpi[j] = a[j][n]\n\t\t\t}\n\t\t\tfor j := len(n) - 1; j >= 0; j-- {\n\t\t\t\tn[j]++\n\t\t\t\tif n[j] < len(a[j]) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tn[j] = 0\n\t\t\t}\n\t\t}\n\t\treturn p\n\t}\n\tisTag := func(op string) []string {\n\t\tif len(op) >= 4 && strings.EqualFold(\"tag(\", op[:4]) && strings.HasSuffix(op, \")\") {\n\t\t\tv := strings.TrimPrefix(op, \"tag(\")\n\t\t\tv = strings.TrimSuffix(v, \")\")\n\t\t\treturn []string{\"tag\", v}\n\t\t} else if len(op) >= 12 && strings.EqualFold(\"account-tag(\", op[:12]) && strings.HasSuffix(op, \")\") {\n\t\t\tv := strings.TrimPrefix(op, \"account-tag(\")\n\t\t\tv = strings.TrimSuffix(v, \")\")\n\t\t\treturn []string{\"account-tag\", v}\n\t\t}\n\t\treturn nil\n\t}\n\tapplyTemplate := func(list jwt.StringList, failOnBadSubject bool) (jwt.StringList, error) {\n\t\tfound := false\n\tFOR_FIND:\n\t\tfor i := 0; i < len(list); i++ {\n\t\t\t// check if templates are present\n\t\t\tif mustacheRE.MatchString(list[i]) {\n\t\t\t\tfound = true\n\t\t\t\tbreak FOR_FIND\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\treturn list, nil\n\t\t}\n\t\t// process the templates\n\t\temittedList := make([]string, 0, len(list))\n\t\tfor i := 0; i < len(list); i++ {\n\t\t\t// find all the templates {{}} in this acl\n\t\t\ttokens := mustacheRE.FindAllString(list[i], -1)\n\t\t\tsrcs := make([]string, len(tokens))\n\t\t\tvalues := make([][]string, len(tokens))\n\t\t\thasTags := false\n\t\t\tfor tokenNum, tk := range tokens {\n\t\t\t\tsrcs[tokenNum] = tk\n\t\t\t\top := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(tk, \"{{\"), \"}}\"))\n\t\t\t\tif strings.EqualFold(\"name()\", op) {\n\t\t\t\t\tvalues[tokenNum] = []string{ujwt.Name}\n\t\t\t\t} else if strings.EqualFold(\"subject()\", op) {\n\t\t\t\t\tvalues[tokenNum] = []string{ujwt.Subject}\n\t\t\t\t} else if strings.EqualFold(\"account-name()\", op) {\n\t\t\t\t\tacc.mu.RLock()\n\t\t\t\t\tvalues[tokenNum] = []string{acc.nameTag}\n\t\t\t\t\tacc.mu.RUnlock()\n\t\t\t\t} else if strings.EqualFold(\"account-subject()\", op) {\n\t\t\t\t\t// this always has an issuer account since this is a scoped signer\n\t\t\t\t\tvalues[tokenNum] = []string{ujwt.IssuerAccount}\n\t\t\t\t} else if isTag(op) != nil {\n\t\t\t\t\thasTags = true\n\t\t\t\t\tmatch := isTag(op)\n\t\t\t\t\tvar tags jwt.TagList\n\t\t\t\t\tif match[0] == \"account-tag\" {\n\t\t\t\t\t\tacc.mu.RLock()\n\t\t\t\t\t\ttags = acc.tags\n\t\t\t\t\t\tacc.mu.RUnlock()\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttags = ujwt.Tags\n\t\t\t\t\t}\n\t\t\t\t\ttagPrefix := fmt.Sprintf(\"%s:\", strings.ToLower(match[1]))\n\t\t\t\t\tvar valueList []string\n\t\t\t\t\tfor _, tag := range tags {\n\t\t\t\t\t\tif strings.HasPrefix(tag, tagPrefix) {\n\t\t\t\t\t\t\ttagValue := strings.TrimPrefix(tag, tagPrefix)\n\t\t\t\t\t\t\tvalueList = append(valueList, tagValue)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif len(valueList) != 0 {\n\t\t\t\t\t\tvalues[tokenNum] = valueList\n\t\t\t\t\t} else if failOnBadSubject {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"generated invalid subject %q: %q is not defined\", list[i], match[1])\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// generate an invalid subject?\n\t\t\t\t\t\tvalues[tokenNum] = []string{\" \"}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\treturn nil, fmt.Errorf(\"template operation in %q: %q is not defined\", list[i], op)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !hasTags {\n\t\t\t\tsubj := list[i]\n\t\t\t\tfor idx, m := range srcs {\n\t\t\t\t\tsubj = strings.Replace(subj, m, values[idx][0], -1)\n\t\t\t\t}\n\t\t\t\tif IsValidSubject(subj) {\n\t\t\t\t\temittedList = append(emittedList, subj)\n\t\t\t\t} else if failOnBadSubject {\n\t\t\t\t\treturn nil, fmt.Errorf(\"generated invalid subject\")\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\texpCount := 1\n\t\t\t\tfor _, v := range values {\n\t\t\t\t\tif len(v) == 0 {\n\t\t\t\t\t\texpCount = 0\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tif expCount > maxPermTemplateSubjectExpansions/len(v) {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"%w: %d\", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions)\n\t\t\t\t\t}\n\t\t\t\t\texpCount *= len(v)\n\t\t\t\t}\n\t\t\t\tif len(emittedList) > maxPermTemplateSubjectExpansions-expCount {\n\t\t\t\t\treturn nil, fmt.Errorf(\"%w: %d\", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions)\n\t\t\t\t}\n\t\t\t\ta := nArrayCartesianProduct(values...)\n\t\t\t\tfor _, aa := range a {\n\t\t\t\t\tsubj := list[i]\n\t\t\t\t\tfor j := 0; j < len(srcs); j++ {\n\t\t\t\t\t\tsubj = strings.Replace(subj, srcs[j], aa[j], -1)\n\t\t\t\t\t}\n\t\t\t\t\tif IsValidSubject(subj) {\n\t\t\t\t\t\temittedList = append(emittedList, subj)\n\t\t\t\t\t} else if failOnBadSubject {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"generated invalid subject\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn emittedList, nil\n\t}\n\n\tsubAllowWasNotEmpty := len(lim.Permissions.Sub.Allow) > 0\n\tpubAllowWasNotEmpty := len(lim.Permissions.Pub.Allow) > 0\n\n\tvar err error\n\tif lim.Permissions.Sub.Allow, err = applyTemplate(lim.Permissions.Sub.Allow, false); err != nil {\n\t\treturn jwt.UserPermissionLimits{}, err\n\t} else if lim.Permissions.Sub.Deny, err = applyTemplate(lim.Permissions.Sub.Deny, true); err != nil {\n\t\treturn jwt.UserPermissionLimits{}, err\n\t} else if lim.Permissions.Pub.Allow, err = applyTemplate(lim.Permissions.Pub.Allow, false); err != nil {\n\t\treturn jwt.UserPermissionLimits{}, err\n\t} else if lim.Permissions.Pub.Deny, err = applyTemplate(lim.Permissions.Pub.Deny, true); err != nil {\n\t\treturn jwt.UserPermissionLimits{}, err\n\t}\n\n\t// if pub/sub allow were not empty, but are empty post template processing, add in a \"deny >\" to compensate\n\tif subAllowWasNotEmpty && len(lim.Permissions.Sub.Allow) == 0 {\n\t\tlim.Permissions.Sub.Deny.Add(\">\")\n\t}\n\tif pubAllowWasNotEmpty && len(lim.Permissions.Pub.Allow) == 0 {\n\t\tlim.Permissions.Pub.Deny.Add(\">\")\n\t}\n\treturn lim, nil\n}\n\nfunc (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (authorized bool) {\n\tvar (\n\t\tnkey *NkeyUser\n\t\tjuc *jwt.UserClaims\n\t\tacc *Account\n\t\tuser *User\n\t\tok bool\n\t\terr error\n\t\tao bool // auth override\n\t)\n\n\t// Little helper that will log the error as a debug statement, set the auth error in\n\t// the connection and return false to indicate authentication failure.\n\tsetProxyAuthError := func(err error) bool {\n\t\tc.Debugf(err.Error())\n\t\tc.setAuthError(err)\n\t\treturn false\n\t}\n\n\t// Indicate if this connection came from a trusted proxy. Note that if\n\t// trustedProxy could be false even if the connection is proxied, but it\n\t// means that there was no trusted proxy configured.\n\ttrustedProxy, ok := s.proxyCheck(c, opts)\n\tif trustedProxy && !ok {\n\t\treturn setProxyAuthError(ErrAuthProxyNotTrusted)\n\t}\n\n\tvar proxyRequired bool\n\t// Check if we have auth callouts enabled at the server level or in the bound account.\n\tdefer func() {\n\t\tauthErr := c.getAuthError()\n\t\tif authErr == nil {\n\t\t\tauthErr = ErrAuthentication\n\t\t}\n\t\treason := getAuthErrClosedState(authErr).String()\n\t\t// No-op\n\t\tif juc == nil && opts.AuthCallout == nil {\n\t\t\tif !authorized {\n\t\t\t\ts.sendAccountAuthErrorEvent(c, c.acc, reason)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\t// We have a juc, check if externally managed, i.e. should be delegated\n\t\t// to the auth callout service.\n\t\tif juc != nil && !acc.hasExternalAuth() {\n\t\t\tif !authorized {\n\t\t\t\ts.sendAccountAuthErrorEvent(c, c.acc, reason)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\t// Check config-mode. The global account is a condition since users that\n\t\t// are not found in the config are implicitly bound to the global account.\n\t\t// This means those users should be implicitly delegated to auth callout\n\t\t// if configured. Exclude LEAF connections from this check.\n\t\tif c.kind != LEAF && juc == nil && opts.AuthCallout != nil && c.acc.Name != globalAccountName {\n\t\t\t// If no allowed accounts are defined, then all accounts are in scope.\n\t\t\t// Otherwise see if the account is in the list.\n\t\t\tdelegated := len(opts.AuthCallout.AllowedAccounts) == 0\n\t\t\tif !delegated {\n\t\t\t\tfor _, n := range opts.AuthCallout.AllowedAccounts {\n\t\t\t\t\tif n == c.acc.Name {\n\t\t\t\t\t\tdelegated = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Not delegated, so return with previous authorized result.\n\t\t\tif !delegated {\n\t\t\t\tif !authorized {\n\t\t\t\t\ts.sendAccountAuthErrorEvent(c, c.acc, reason)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\t// We have auth callout set here.\n\t\tvar skip bool\n\t\t// Check if we are on the list of auth_users.\n\t\tuserID := c.getRawAuthUser()\n\t\tif juc != nil {\n\t\t\tskip = acc.isExternalAuthUser(userID)\n\t\t} else {\n\t\t\tfor _, u := range opts.AuthCallout.AuthUsers {\n\t\t\t\tif userID == u {\n\t\t\t\t\tskip = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// If we are here we have an auth callout defined and we have failed auth so far\n\t\t// so we will callout to our auth backend for processing.\n\t\tif !skip {\n\t\t\tauthorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy)\n\t\t}\n\t\t// Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject.\n\t\tif authorized {\n\t\t\tvar authAccountName string\n\t\t\tif juc == nil && opts.AuthCallout != nil {\n\t\t\t\tauthAccountName = opts.AuthCallout.Account\n\t\t\t} else if juc != nil {\n\t\t\t\tauthAccountName = acc.Name\n\t\t\t}\n\t\t\tc.mu.Lock()\n\t\t\tif c.acc != nil && c.acc.Name == authAccountName {\n\t\t\t\tc.mergeDenyPermissions(pub, []string{AuthCalloutSubject})\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t} else {\n\t\t\t// If we are here we failed external authorization.\n\t\t\t// Send an account scoped event. Server config mode acc will be nil,\n\t\t\t// so lookup the auth callout assigned account, that is where this will be sent.\n\t\t\tif acc == nil {\n\t\t\t\tacc, _ = s.lookupAccount(opts.AuthCallout.Account)\n\t\t\t}\n\t\t\ts.sendAccountAuthErrorEvent(c, acc, reason)\n\t\t}\n\t}()\n\n\ts.mu.Lock()\n\tauthRequired := s.info.AuthRequired\n\tif !authRequired {\n\t\t// If no auth required for regular clients, then check if\n\t\t// we have an override for MQTT or Websocket clients.\n\t\tswitch c.clientType() {\n\t\tcase MQTT:\n\t\t\tauthRequired = s.mqtt.authOverride\n\t\tcase WS:\n\t\t\tauthRequired = s.websocket.authOverride\n\t\t}\n\t}\n\tif !authRequired {\n\t\t// TODO(dlc) - If they send us credentials should we fail?\n\t\ts.mu.Unlock()\n\t\tif c.kind == LEAF {\n\t\t\t// Auth is not required, register the leaf node with the selected account.\n\t\t\t// Otherwise, auth needs to match client auth.\n\t\t\treturn s.registerLeafWithAccount(c, opts.LeafNode.Account)\n\t\t}\n\t\treturn true\n\t}\n\tvar (\n\t\tusername string\n\t\tpassword string\n\t\ttoken string\n\t\tnoAuthUser string\n\t\tpinnedAcounts map[string]struct{}\n\t)\n\ttlsMap := opts.TLSMap\n\tif c.kind == CLIENT {\n\t\tswitch c.clientType() {\n\t\tcase MQTT:\n\t\t\tmo := &opts.MQTT\n\t\t\t// Always override TLSMap.\n\t\t\ttlsMap = mo.TLSMap\n\t\t\t// The rest depends on if there was any auth override in\n\t\t\t// the mqtt's config.\n\t\t\tif s.mqtt.authOverride {\n\t\t\t\tnoAuthUser = mo.NoAuthUser\n\t\t\t\tusername = mo.Username\n\t\t\t\tpassword = mo.Password\n\t\t\t\ttoken = mo.Token\n\t\t\t\tao = true\n\t\t\t}\n\t\tcase WS:\n\t\t\two := &opts.Websocket\n\t\t\t// Always override TLSMap.\n\t\t\ttlsMap = wo.TLSMap\n\t\t\t// The rest depends on if there was any auth override in\n\t\t\t// the websocket's config.\n\t\t\tif s.websocket.authOverride {\n\t\t\t\tnoAuthUser = wo.NoAuthUser\n\t\t\t\tusername = wo.Username\n\t\t\t\tpassword = wo.Password\n\t\t\t\ttoken = wo.Token\n\t\t\t\tao = true\n\t\t\t}\n\t\t}\n\t} else {\n\t\ttlsMap = opts.LeafNode.TLSMap\n\t}\n\n\tif !ao {\n\t\tnoAuthUser = opts.NoAuthUser\n\t\t// If a leaf connects using websocket, and websocket{} block has a no_auth_user\n\t\t// use that one instead.\n\t\tif c.kind == LEAF && c.isWebsocket() && opts.Websocket.NoAuthUser != _EMPTY_ {\n\t\t\tnoAuthUser = opts.Websocket.NoAuthUser\n\t\t}\n\t\tusername = opts.Username\n\t\tpassword = opts.Password\n\t\ttoken = opts.Authorization\n\t}\n\n\t// Check if we have trustedKeys defined in the server. If so we require a user jwt.\n\tif s.trustedKeys != nil {\n\t\tif c.opts.JWT == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {\n\t\t\tc.opts.JWT = opts.DefaultSentinel\n\t\t}\n\t\tif c.opts.JWT == _EMPTY_ {\n\t\t\ts.mu.Unlock()\n\t\t\tc.Debugf(\"Authentication requires a user JWT\")\n\t\t\treturn false\n\t\t}\n\t\t// So we have a valid user jwt here.\n\t\tjuc, err = jwt.DecodeUserClaims(c.opts.JWT)\n\t\tif err != nil {\n\t\t\ts.mu.Unlock()\n\t\t\tc.Debugf(\"User JWT not valid: %v\", err)\n\t\t\treturn false\n\t\t}\n\t\tif proxyRequired = juc.ProxyRequired; proxyRequired && !trustedProxy {\n\t\t\ts.mu.Unlock()\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\tvr := jwt.CreateValidationResults()\n\t\tjuc.Validate(vr)\n\t\tif vr.IsBlocking(true) {\n\t\t\ts.mu.Unlock()\n\t\t\tc.Debugf(\"User JWT no longer valid: %+v\", vr)\n\t\t\treturn false\n\t\t}\n\t\tpinnedAcounts = opts.resolverPinnedAccounts\n\t}\n\n\t// Check if we have nkeys or users for client.\n\thasNkeys := len(s.nkeys) > 0\n\thasUsers := len(s.users) > 0\n\tif hasNkeys {\n\t\tif (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ &&\n\t\t\tc.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ && c.opts.Nkey == _EMPTY_ {\n\t\t\tif _, exists := s.nkeys[noAuthUser]; exists {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.opts.Nkey = noAuthUser\n\t\t\t\tc.mu.Unlock()\n\t\t\t}\n\t\t}\n\t\tif c.opts.Nkey != _EMPTY_ {\n\t\t\tnkey, ok = s.nkeys[c.opts.Nkey]\n\t\t\tif !ok || !c.connectionTypeAllowed(nkey.AllowedConnectionTypes) {\n\t\t\t\ts.mu.Unlock()\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\tif hasUsers && nkey == nil {\n\t\t// Check if we are tls verify and are mapping users from the client_certificate.\n\t\tif tlsMap {\n\t\t\tauthorized := checkClientTLSCertSubject(c, func(u string, certDN *ldap.DN, _ bool) (string, bool) {\n\t\t\t\t// First do literal lookup using the resulting string representation\n\t\t\t\t// of RDNSequence as implemented by the pkix package from Go.\n\t\t\t\tif u != _EMPTY_ {\n\t\t\t\t\tusr, ok := s.users[u]\n\t\t\t\t\tif !ok || !c.connectionTypeAllowed(usr.AllowedConnectionTypes) {\n\t\t\t\t\t\treturn _EMPTY_, false\n\t\t\t\t\t}\n\t\t\t\t\tuser = usr\n\t\t\t\t\treturn usr.Username, true\n\t\t\t\t}\n\n\t\t\t\tif certDN == nil {\n\t\t\t\t\treturn _EMPTY_, false\n\t\t\t\t}\n\n\t\t\t\t// Look through the accounts for a DN that is equal to the one\n\t\t\t\t// presented by the certificate.\n\t\t\t\tdns := make(map[*User]*ldap.DN)\n\t\t\t\tfor _, usr := range s.users {\n\t\t\t\t\tif !c.connectionTypeAllowed(usr.AllowedConnectionTypes) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// TODO: Use this utility to make a full validation pass\n\t\t\t\t\t// on start in case tlsmap feature is being used.\n\t\t\t\t\tinputDN, err := ldap.ParseDN(usr.Username)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif inputDN.Equal(certDN) {\n\t\t\t\t\t\tuser = usr\n\t\t\t\t\t\treturn usr.Username, true\n\t\t\t\t\t}\n\n\t\t\t\t\t// In case it did not match exactly, then collect the DNs\n\t\t\t\t\t// and try to match later in case the DN was reordered.\n\t\t\t\t\tdns[usr] = inputDN\n\t\t\t\t}\n\n\t\t\t\t// Check in case the DN was reordered.\n\t\t\t\tfor usr, inputDN := range dns {\n\t\t\t\t\tif inputDN.RDNsMatch(certDN) {\n\t\t\t\t\t\tuser = usr\n\t\t\t\t\t\treturn usr.Username, true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn _EMPTY_, false\n\t\t\t})\n\t\t\tif !authorized {\n\t\t\t\ts.mu.Unlock()\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif c.opts.Username != _EMPTY_ {\n\t\t\t\ts.Warnf(\"User %q found in connect proto, but user required from cert\", c.opts.Username)\n\t\t\t}\n\t\t\t// Already checked that the client didn't send a user in connect\n\t\t\t// but we set it here to be able to identify it in the logs.\n\t\t\tc.opts.Username = user.Username\n\t\t} else {\n\t\t\tif (c.kind == CLIENT || c.kind == LEAF) && noAuthUser != _EMPTY_ &&\n\t\t\t\tc.opts.Username == _EMPTY_ && c.opts.Password == _EMPTY_ && c.opts.Token == _EMPTY_ {\n\t\t\t\tif u, exists := s.users[noAuthUser]; exists {\n\t\t\t\t\tc.mu.Lock()\n\t\t\t\t\tc.opts.Username = u.Username\n\t\t\t\t\tc.opts.Password = u.Password\n\t\t\t\t\tc.mu.Unlock()\n\t\t\t\t}\n\t\t\t}\n\t\t\tif c.opts.Username != _EMPTY_ {\n\t\t\t\tuser, ok = s.users[c.opts.Username]\n\t\t\t\tif !ok || !c.connectionTypeAllowed(user.AllowedConnectionTypes) {\n\t\t\t\t\ts.mu.Unlock()\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\t// If we have a jwt and a userClaim, make sure we have the Account, etc associated.\n\t// We need to look up the account. This will use an account resolver if one is present.\n\tif juc != nil {\n\t\tissuer := juc.Issuer\n\t\tif juc.IssuerAccount != _EMPTY_ {\n\t\t\tissuer = juc.IssuerAccount\n\t\t}\n\t\tif pinnedAcounts != nil {\n\t\t\tif _, ok := pinnedAcounts[issuer]; !ok {\n\t\t\t\tc.Debugf(\"Account %s not listed as operator pinned account\", issuer)\n\t\t\t\tatomic.AddUint64(&s.pinnedAccFail, 1)\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif acc, err = s.LookupAccount(issuer); acc == nil {\n\t\t\tc.Debugf(\"Account JWT lookup error: %v\", err)\n\t\t\treturn false\n\t\t}\n\t\tacc.mu.RLock()\n\t\taissuer := acc.Issuer\n\t\tacc.mu.RUnlock()\n\t\tif !s.isTrustedIssuer(aissuer) {\n\t\t\tc.Debugf(\"Account JWT not signed by trusted operator\")\n\t\t\treturn false\n\t\t}\n\t\tif scope, ok := acc.hasIssuer(juc.Issuer); !ok {\n\t\t\tc.Debugf(\"User JWT issuer is not known\")\n\t\t\treturn false\n\t\t} else if scope != nil {\n\t\t\tif err := scope.ValidateScopedSigner(juc); err != nil {\n\t\t\t\tc.Debugf(\"User JWT is not valid: %v\", err)\n\t\t\t\treturn false\n\t\t\t} else if uSc, ok := scope.(*jwt.UserScope); !ok {\n\t\t\t\tc.Debugf(\"User JWT is not valid\")\n\t\t\t\treturn false\n\t\t\t} else if juc.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, juc, acc); err != nil {\n\t\t\t\tc.Debugf(\"User JWT generated invalid permissions\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif acc.IsExpired() {\n\t\t\tc.Debugf(\"Account JWT has expired\")\n\t\t\treturn false\n\t\t}\n\t\tif juc.BearerToken && acc.failBearer() {\n\t\t\tc.Debugf(\"Account does not allow bearer tokens\")\n\t\t\treturn false\n\t\t}\n\t\t// We check the allowed connection types, but only after processing\n\t\t// of scoped signer (so that it updates `juc` with what is defined\n\t\t// in the account.\n\t\tallowedConnTypes, err := convertAllowedConnectionTypes(juc.AllowedConnectionTypes)\n\t\tif err != nil {\n\t\t\t// We got an error, which means some connection types were unknown. As long as\n\t\t\t// a valid one is returned, we proceed with auth. If not, we have to reject.\n\t\t\t// In other words, suppose that JWT allows \"WEBSOCKET\" in the array. No error\n\t\t\t// is returned and allowedConnTypes will contain \"WEBSOCKET\" only.\n\t\t\t// Client will be rejected if not a websocket client, or proceed with rest of\n\t\t\t// auth if it is.\n\t\t\t// Now suppose JWT allows \"WEBSOCKET, MQTT\" and say MQTT is not known by this\n\t\t\t// server. In this case, allowedConnTypes would contain \"WEBSOCKET\" and we\n\t\t\t// would get `err` indicating that \"MQTT\" is an unknown connection type.\n\t\t\t// If a websocket client connects, it should still be allowed, since after all\n\t\t\t// the admin wanted to allow websocket and mqtt connection types.\n\t\t\t// However, say that the JWT only allows \"MQTT\" (and again suppose this server\n\t\t\t// does not know about MQTT connection type), then since the allowedConnTypes\n\t\t\t// map would be empty (no valid types found), and since empty means allow-all,\n\t\t\t// then we should reject because the intent was to allow connections for this\n\t\t\t// user only as an MQTT client.\n\t\t\tc.Debugf(\"%v\", err)\n\t\t\tif len(allowedConnTypes) == 0 {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif !c.connectionTypeAllowed(allowedConnTypes) {\n\t\t\tc.Debugf(\"Connection type not allowed\")\n\t\t\treturn false\n\t\t}\n\t\t// skip validation of nonce when presented with a bearer token\n\t\t// FIXME: if BearerToken is only for WSS, need check for server with that port enabled\n\t\tif !juc.BearerToken {\n\t\t\t// Verify the signature against the nonce.\n\t\t\tif c.opts.Sig == _EMPTY_ {\n\t\t\t\tc.Debugf(\"Signature missing\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tsig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig)\n\t\t\tif err != nil {\n\t\t\t\t// Allow fallback to normal base64.\n\t\t\t\tsig, err = base64.StdEncoding.DecodeString(c.opts.Sig)\n\t\t\t\tif err != nil {\n\t\t\t\t\tc.Debugf(\"Signature not valid base64\")\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\tpub, err := nkeys.FromPublicKey(juc.Subject)\n\t\t\tif err != nil {\n\t\t\t\tc.Debugf(\"User nkey not valid: %v\", err)\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif err := pub.Verify(c.nonce, sig); err != nil {\n\t\t\t\tc.Debugf(\"Signature not verified\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif acc.checkUserRevoked(juc.Subject, juc.IssuedAt) {\n\t\t\tc.Debugf(\"User authentication revoked\")\n\t\t\treturn false\n\t\t}\n\t\tif !validateSrc(juc, c.host) {\n\t\t\tc.Errorf(\"Bad src Ip %s\", c.host)\n\t\t\treturn false\n\t\t}\n\t\tallowNow, validFor := validateTimes(juc)\n\t\tif !allowNow {\n\t\t\tc.Errorf(\"Outside connect times\")\n\t\t\treturn false\n\t\t}\n\n\t\tnkey = buildInternalNkeyUser(juc, allowedConnTypes, acc)\n\t\tif err := c.RegisterNkeyUser(nkey); err != nil {\n\t\t\treturn false\n\t\t}\n\n\t\t// Warn about JetStream restrictions\n\t\tif c.perms != nil {\n\t\t\tdeniedPub := []string{}\n\t\t\tdeniedSub := []string{}\n\t\t\tfor _, sub := range denyAllJs {\n\t\t\t\tif c.perms.pub.deny != nil {\n\t\t\t\t\tif c.perms.pub.deny.HasInterest(sub) {\n\t\t\t\t\t\tdeniedPub = append(deniedPub, sub)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif c.perms.sub.deny != nil {\n\t\t\t\t\tif c.perms.sub.deny.HasInterest(sub) {\n\t\t\t\t\t\tdeniedSub = append(deniedSub, sub)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(deniedPub) > 0 || len(deniedSub) > 0 {\n\t\t\t\tc.Noticef(\"Connected %s has JetStream denied on pub: %v sub: %v\", c.kindString(), deniedPub, deniedSub)\n\t\t\t}\n\t\t}\n\n\t\t// Hold onto the user's public key.\n\t\tc.mu.Lock()\n\t\tc.pubKey = juc.Subject\n\t\tc.tags = juc.Tags\n\t\tc.nameTag = juc.Name\n\t\tc.mu.Unlock()\n\n\t\t// Check if we need to set an auth timer if the user jwt expires.\n\t\tc.setExpiration(juc.Claims(), validFor)\n\n\t\tacc.mu.RLock()\n\t\tc.Debugf(\"Authenticated JWT: %s %q (claim-name: %q, claim-tags: %q) \"+\n\t\t\t\"signed with %q by Account %q (claim-name: %q, claim-tags: %q) signed with %q has mappings %t accused %p\",\n\t\t\tc.kindString(), juc.Subject, juc.Name, juc.Tags, juc.Issuer, issuer, acc.nameTag, acc.tags, acc.Issuer, acc.hasMappings(), acc)\n\t\tacc.mu.RUnlock()\n\t\treturn true\n\t}\n\n\tif nkey != nil {\n\t\tif proxyRequired = nkey.ProxyRequired; proxyRequired && !trustedProxy {\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\t// If we did not match noAuthUser check signature which is required.\n\t\tif nkey.Nkey != noAuthUser {\n\t\t\tif c.opts.Sig == _EMPTY_ {\n\t\t\t\tc.Debugf(\"Signature missing\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tsig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig)\n\t\t\tif err != nil {\n\t\t\t\t// Allow fallback to normal base64.\n\t\t\t\tsig, err = base64.StdEncoding.DecodeString(c.opts.Sig)\n\t\t\t\tif err != nil {\n\t\t\t\t\tc.Debugf(\"Signature not valid base64\")\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\tpub, err := nkeys.FromPublicKey(c.opts.Nkey)\n\t\t\tif err != nil {\n\t\t\t\tc.Debugf(\"User nkey not valid: %v\", err)\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif err := pub.Verify(c.nonce, sig); err != nil {\n\t\t\t\tc.Debugf(\"Signature not verified\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif err := c.RegisterNkeyUser(nkey); err != nil {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\tif user != nil {\n\t\tif proxyRequired = user.ProxyRequired; proxyRequired && !trustedProxy {\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\tok = comparePasswords(user.Password, c.opts.Password)\n\t\t// If we are authorized, register the user which will properly setup any permissions\n\t\t// for pub/sub authorizations.\n\t\tif ok {\n\t\t\tc.RegisterUser(user)\n\t\t}\n\t\treturn ok\n\t}\n\n\t// Check for the use of simple auth.\n\tif c.kind == CLIENT || c.kind == LEAF {\n\t\tif proxyRequired = opts.ProxyRequired; proxyRequired && !trustedProxy {\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\tif token != _EMPTY_ {\n\t\t\treturn comparePasswords(token, c.opts.Token)\n\t\t} else if username != _EMPTY_ {\n\t\t\tif username != c.opts.Username {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn comparePasswords(password, c.opts.Password)\n\t\t}\n\t}\n\treturn false\n}\n\n// If there are configured trusted proxies and this connection comes\n// from a proxy whose signature can be verified by one of the known\n// trusted key, this function will return `true, true`. If the signature\n// cannot be verified by any, it will return `true, false`.\n// If the connectio is not proxied, or there are no configured trusted\n// proxies, then this function returns `false, false`.\n//\n// Server lock MUST NOT be held on entry since this function will grab\n// the read lock to extract the list of proxy trusted keys. The signature\n// verification process will be done outside of the lock.\nfunc (s *Server) proxyCheck(c *client, opts *Options) (bool, bool) {\n\t// If there is no signature or no configured trusted proxy, return false.\n\tpsig := c.opts.ProxySig\n\tif psig == _EMPTY_ || opts.Proxies == nil || len(opts.Proxies.Trusted) == 0 {\n\t\treturn false, false\n\t}\n\t// Decode the signature.\n\tsig, err := base64.RawURLEncoding.DecodeString(psig)\n\tif err != nil {\n\t\tc.Debugf(\"Proxy signature not valid base64\")\n\t\treturn true, false\n\t}\n\t// Go through the trusted keys and verify the signature.\n\ts.mu.RLock()\n\tkeys := slices.Clone(s.proxiesKeyPairs)\n\ts.mu.RUnlock()\n\tfor _, kp := range keys {\n\t\t// We stop at the first that is valid.\n\t\tif err := kp.Verify(c.nonce, sig); err == nil {\n\t\t\tpub, _ := kp.PublicKey()\n\t\t\t// Track which proxy public key is used by this connection.\n\t\t\tc.mu.Lock()\n\t\t\tc.proxyKey = pub\n\t\t\tcid := c.cid\n\t\t\tc.mu.Unlock()\n\t\t\t// Track this proxied connection so that it can be closed\n\t\t\t// if the trusted key is removed on configuration reload.\n\t\t\ts.mu.Lock()\n\t\t\tif s.proxiedConns == nil {\n\t\t\t\ts.proxiedConns = make(map[string]map[uint64]*client)\n\t\t\t}\n\t\t\tclients := s.proxiedConns[pub]\n\t\t\tif clients == nil {\n\t\t\t\tclients = make(map[uint64]*client)\n\t\t\t}\n\t\t\tclients[cid] = c\n\t\t\ts.proxiedConns[pub] = clients\n\t\t\ts.mu.Unlock()\n\t\t\treturn true, true\n\t\t}\n\t}\n\t// We could not verify the signature, so indicate failure.\n\treturn true, false\n}\n\nfunc getTLSAuthDCs(rdns *pkix.RDNSequence) string {\n\tdcOID := asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25}\n\tdcs := []string{}\n\tfor _, rdn := range *rdns {\n\t\tif len(rdn) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, atv := range rdn {\n\t\t\tvalue, ok := atv.Value.(string)\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif atv.Type.Equal(dcOID) {\n\t\t\t\tdcs = append(dcs, \"DC=\"+value)\n\t\t\t}\n\t\t}\n\t}\n\treturn strings.Join(dcs, \",\")\n}\n\ntype tlsMapAuthFn func(string, *ldap.DN, bool) (string, bool)\n\nfunc checkClientTLSCertSubject(c *client, fn tlsMapAuthFn) bool {\n\ttlsState := c.GetTLSConnectionState()\n\tif tlsState == nil {\n\t\tc.Debugf(\"User required in cert, no TLS connection state\")\n\t\treturn false\n\t}\n\tif len(tlsState.PeerCertificates) == 0 {\n\t\tc.Debugf(\"User required in cert, no peer certificates found\")\n\t\treturn false\n\t}\n\tcert := tlsState.PeerCertificates[0]\n\tif len(tlsState.PeerCertificates) > 1 {\n\t\tc.Debugf(\"Multiple peer certificates found, selecting first\")\n\t}\n\n\thasSANs := len(cert.DNSNames) > 0\n\thasEmailAddresses := len(cert.EmailAddresses) > 0\n\thasSubject := len(cert.Subject.String()) > 0\n\thasURIs := len(cert.URIs) > 0\n\tif !hasEmailAddresses && !hasSubject && !hasURIs {\n\t\tc.Debugf(\"User required in cert, none found\")\n\t\treturn false\n\t}\n\n\tswitch {\n\tcase hasEmailAddresses:\n\t\tfor _, u := range cert.EmailAddresses {\n\t\t\tif match, ok := fn(u, nil, false); ok {\n\t\t\t\tc.Debugf(\"Using email found in cert for auth [%q]\", match)\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tfallthrough\n\tcase hasSANs:\n\t\tfor _, u := range cert.DNSNames {\n\t\t\tif match, ok := fn(u, nil, true); ok {\n\t\t\t\tc.Debugf(\"Using SAN found in cert for auth [%q]\", match)\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tfallthrough\n\tcase hasURIs:\n\t\tfor _, u := range cert.URIs {\n\t\t\tif match, ok := fn(u.String(), nil, false); ok {\n\t\t\t\tc.Debugf(\"Using URI found in cert for auth [%q]\", match)\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\n\t// Use the string representation of the full RDN Sequence including\n\t// the domain components in case there are any.\n\trdn := cert.Subject.ToRDNSequence().String()\n\n\t// Match using the raw subject to avoid ignoring attributes.\n\t// https://github.com/golang/go/issues/12342\n\tdn, err := ldap.FromRawCertSubject(cert.RawSubject)\n\tif err == nil {\n\t\tif match, ok := fn(_EMPTY_, dn, false); ok {\n\t\t\tc.Debugf(\"Using DistinguishedNameMatch for auth [%q]\", match)\n\t\t\treturn true\n\t\t}\n\t\tc.Debugf(\"DistinguishedNameMatch could not be used for auth [%q]\", rdn)\n\t}\n\n\tvar rdns pkix.RDNSequence\n\tif _, err := asn1.Unmarshal(cert.RawSubject, &rdns); err == nil {\n\t\t// If found domain components then include roughly following\n\t\t// the order from https://tools.ietf.org/html/rfc2253\n\t\t//\n\t\t// NOTE: The original sequence from string representation by ToRDNSequence does not follow\n\t\t// the correct ordering, so this addition ofdomainComponents would likely be deprecated in\n\t\t// another release in favor of using the correct ordered as parsed by the go-ldap library.\n\t\t//\n\t\tdcs := getTLSAuthDCs(&rdns)\n\t\tif len(dcs) > 0 {\n\t\t\tu := strings.Join([]string{rdn, dcs}, \",\")\n\t\t\tif match, ok := fn(u, nil, false); ok {\n\t\t\t\tc.Debugf(\"Using RDNSequence for auth [%q]\", match)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tc.Debugf(\"RDNSequence could not be used for auth [%q]\", u)\n\t\t}\n\t}\n\n\t// If no match, then use the string representation of the RDNSequence\n\t// from the subject without the domainComponents.\n\tif match, ok := fn(rdn, nil, false); ok {\n\t\tc.Debugf(\"Using certificate subject for auth [%q]\", match)\n\t\treturn true\n\t}\n\n\tc.Debugf(\"User in cert [%q], not found\", rdn)\n\treturn false\n}\n\nfunc dnsAltNameLabels(dnsAltName string) []string {\n\treturn strings.Split(strings.ToLower(dnsAltName), \".\")\n}\n\n// Check DNS name according to https://tools.ietf.org/html/rfc6125#section-6.4.1\nfunc dnsAltNameMatches(dnsAltNameLabels []string, urls []*url.URL) bool {\nURLS:\n\tfor _, url := range urls {\n\t\tif url == nil {\n\t\t\tcontinue URLS\n\t\t}\n\t\thostLabels := strings.Split(strings.ToLower(url.Hostname()), \".\")\n\t\t// Following https://tools.ietf.org/html/rfc6125#section-6.4.3, should not => will not, may => will not\n\t\t// The wildcard * never matches multiple label and only matches the left most label.\n\t\tif len(hostLabels) != len(dnsAltNameLabels) {\n\t\t\tcontinue URLS\n\t\t}\n\t\ti := 0\n\t\t// only match wildcard on left most label\n\t\tif dnsAltNameLabels[0] == \"*\" {\n\t\t\ti++\n\t\t}\n\t\tfor ; i < len(dnsAltNameLabels); i++ {\n\t\t\tif dnsAltNameLabels[i] != hostLabels[i] {\n\t\t\t\tcontinue URLS\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}\n\treturn false\n}\n\n// checkRouterAuth checks optional router authorization which can be nil or username/password.\nfunc (s *Server) isRouterAuthorized(c *client) bool {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Check custom auth first, then TLS map if enabled\n\t// then single user/pass.\n\tif opts.CustomRouterAuthentication != nil {\n\t\treturn opts.CustomRouterAuthentication.Check(c)\n\t}\n\n\tif opts.Cluster.TLSMap || opts.Cluster.TLSCheckKnownURLs {\n\t\treturn checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) {\n\t\t\tif user == _EMPTY_ {\n\t\t\t\treturn _EMPTY_, false\n\t\t\t}\n\t\t\tif opts.Cluster.TLSCheckKnownURLs && isDNSAltName {\n\t\t\t\tif dnsAltNameMatches(dnsAltNameLabels(user), opts.Routes) {\n\t\t\t\t\treturn _EMPTY_, true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif opts.Cluster.TLSMap && opts.Cluster.Username == user {\n\t\t\t\treturn _EMPTY_, true\n\t\t\t}\n\t\t\treturn _EMPTY_, false\n\t\t})\n\t}\n\n\tif opts.Cluster.Username == _EMPTY_ {\n\t\treturn true\n\t}\n\n\tif opts.Cluster.Username != c.opts.Username {\n\t\treturn false\n\t}\n\tif !comparePasswords(opts.Cluster.Password, c.opts.Password) {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// isGatewayAuthorized checks optional gateway authorization which can be nil or username/password.\nfunc (s *Server) isGatewayAuthorized(c *client) bool {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Check whether TLS map is enabled, otherwise use single user/pass.\n\tif opts.Gateway.TLSMap || opts.Gateway.TLSCheckKnownURLs {\n\t\treturn checkClientTLSCertSubject(c, func(user string, _ *ldap.DN, isDNSAltName bool) (string, bool) {\n\t\t\tif user == _EMPTY_ {\n\t\t\t\treturn _EMPTY_, false\n\t\t\t}\n\t\t\tif opts.Gateway.TLSCheckKnownURLs && isDNSAltName {\n\t\t\t\tlabels := dnsAltNameLabels(user)\n\t\t\t\tfor _, gw := range opts.Gateway.Gateways {\n\t\t\t\t\tif gw != nil && dnsAltNameMatches(labels, gw.URLs) {\n\t\t\t\t\t\treturn _EMPTY_, true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif opts.Gateway.TLSMap && opts.Gateway.Username == user {\n\t\t\t\treturn _EMPTY_, true\n\t\t\t}\n\t\t\treturn _EMPTY_, false\n\t\t})\n\t}\n\n\tif opts.Gateway.Username == _EMPTY_ {\n\t\treturn true\n\t}\n\n\tif opts.Gateway.Username != c.opts.Username {\n\t\treturn false\n\t}\n\treturn comparePasswords(opts.Gateway.Password, c.opts.Password)\n}\n\nfunc (s *Server) registerLeafWithAccount(c *client, account string) bool {\n\tvar err error\n\tacc := s.globalAccount()\n\tif account != _EMPTY_ {\n\t\tacc, err = s.lookupAccount(account)\n\t\tif err != nil {\n\t\t\ts.Errorf(\"authentication of user %q failed, unable to lookup account %q: %v\",\n\t\t\t\tc.opts.Username, account, err)\n\t\t\treturn false\n\t\t}\n\t}\n\tif err = c.registerWithAccount(acc); err != nil {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// isLeafNodeAuthorized will check for auth for an inbound leaf node connection.\nfunc (s *Server) isLeafNodeAuthorized(c *client) bool {\n\topts := s.getOpts()\n\n\tsetProxyAuthError := func(err error) bool {\n\t\tc.Debugf(err.Error())\n\t\tc.setAuthError(err)\n\t\treturn false\n\t}\n\n\tisAuthorized := func(username, password, account string, proxyRequired bool) bool {\n\t\ttrustedProxy, ok := s.proxyCheck(c, opts)\n\t\tif trustedProxy && !ok {\n\t\t\treturn setProxyAuthError(ErrAuthProxyNotTrusted)\n\t\t}\n\t\t// A given user may not be required, but if the boolean is set at the\n\t\t// authorization top-level, then override.\n\t\tif !proxyRequired && opts.LeafNode.ProxyRequired {\n\t\t\tproxyRequired = true\n\t\t}\n\t\tif proxyRequired && !trustedProxy {\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\tif username != c.opts.Username {\n\t\t\treturn false\n\t\t}\n\t\tif !comparePasswords(password, c.opts.Password) {\n\t\t\treturn false\n\t\t}\n\t\treturn s.registerLeafWithAccount(c, account)\n\t}\n\n\t// If leafnodes config has an authorization{} stanza, this takes precedence.\n\t// The user in CONNECT must match. We will bind to the account associated\n\t// with that user (from the leafnode's authorization{} config).\n\tif opts.LeafNode.Username != _EMPTY_ {\n\t\treturn isAuthorized(opts.LeafNode.Username, opts.LeafNode.Password, opts.LeafNode.Account,\n\t\t\topts.LeafNode.ProxyRequired)\n\t} else if opts.LeafNode.Nkey != _EMPTY_ {\n\t\ttrustedProxy, ok := s.proxyCheck(c, opts)\n\t\tif trustedProxy && !ok {\n\t\t\treturn false\n\t\t}\n\t\tif opts.LeafNode.ProxyRequired && !trustedProxy {\n\t\t\treturn setProxyAuthError(ErrAuthProxyRequired)\n\t\t}\n\t\tif c.opts.Nkey != opts.LeafNode.Nkey {\n\t\t\treturn false\n\t\t}\n\t\tif c.opts.Sig == _EMPTY_ {\n\t\t\tc.Debugf(\"Signature missing\")\n\t\t\treturn false\n\t\t}\n\t\tsig, err := base64.RawURLEncoding.DecodeString(c.opts.Sig)\n\t\tif err != nil {\n\t\t\t// Allow fallback to normal base64.\n\t\t\tsig, err = base64.StdEncoding.DecodeString(c.opts.Sig)\n\t\t\tif err != nil {\n\t\t\t\tc.Debugf(\"Signature not valid base64\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tpub, err := nkeys.FromPublicKey(c.opts.Nkey)\n\t\tif err != nil {\n\t\t\tc.Debugf(\"User nkey not valid: %v\", err)\n\t\t\treturn false\n\t\t}\n\t\tif err := pub.Verify(c.nonce, sig); err != nil {\n\t\t\tc.Debugf(\"Signature not verified\")\n\t\t\treturn false\n\t\t}\n\t\treturn s.registerLeafWithAccount(c, opts.LeafNode.Account)\n\t} else if len(opts.LeafNode.Users) > 0 {\n\t\tif opts.LeafNode.TLSMap {\n\t\t\tvar user *User\n\t\t\tfound := checkClientTLSCertSubject(c, func(u string, _ *ldap.DN, _ bool) (string, bool) {\n\t\t\t\t// This is expected to be a very small array.\n\t\t\t\tfor _, usr := range opts.LeafNode.Users {\n\t\t\t\t\tif u == usr.Username {\n\t\t\t\t\t\tuser = usr\n\t\t\t\t\t\treturn u, true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn _EMPTY_, false\n\t\t\t})\n\t\t\tif !found {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif c.opts.Username != _EMPTY_ {\n\t\t\t\ts.Warnf(\"User %q found in connect proto, but user required from cert\", c.opts.Username)\n\t\t\t}\n\t\t\tc.opts.Username = user.Username\n\t\t\t// EMPTY will result in $G\n\t\t\taccName := _EMPTY_\n\t\t\tif user.Account != nil {\n\t\t\t\taccName = user.Account.GetName()\n\t\t\t}\n\t\t\t// This will authorize since are using an existing user,\n\t\t\t// but it will also register with proper account.\n\t\t\treturn isAuthorized(user.Username, user.Password, accName, user.ProxyRequired)\n\t\t}\n\n\t\t// This is expected to be a very small array.\n\t\tfor _, u := range opts.LeafNode.Users {\n\t\t\tif u.Username == c.opts.Username {\n\t\t\t\tvar accName string\n\t\t\t\tif u.Account != nil {\n\t\t\t\t\taccName = u.Account.Name\n\t\t\t\t}\n\t\t\t\treturn isAuthorized(u.Username, u.Password, accName, u.ProxyRequired)\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\n\t// We are here if we accept leafnode connections without any credentials.\n\n\t// Still, if the CONNECT has some user info, we will bind to the\n\t// user's account or to the specified default account (if provided)\n\t// or to the global account.\n\treturn s.isClientAuthorized(c)\n}\n\n// Support for bcrypt stored passwords and tokens.\nvar validBcryptPrefix = regexp.MustCompile(`^\\$2[abxy]\\$\\d{2}\\$.*`)\n\n// isBcrypt checks whether the given password or token is bcrypted.\nfunc isBcrypt(password string) bool {\n\tif strings.HasPrefix(password, \"$\") {\n\t\treturn validBcryptPrefix.MatchString(password)\n\t}\n\n\treturn false\n}\n\nfunc comparePasswords(serverPassword, clientPassword string) bool {\n\t// Check to see if the server password is a bcrypt hash\n\tif isBcrypt(serverPassword) {\n\t\tif err := bcrypt.CompareHashAndPassword([]byte(serverPassword), []byte(clientPassword)); err != nil {\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\t// stringToBytes should be constant-time near enough compared to\n\t\t// turning a string into []byte normally.\n\t\tspass := stringToBytes(serverPassword)\n\t\tcpass := stringToBytes(clientPassword)\n\t\tif subtle.ConstantTimeCompare(spass, cpass) == 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc validateAuth(o *Options) error {\n\tif err := validatePinnedCerts(o.TLSPinnedCerts); err != nil {\n\t\treturn err\n\t}\n\tfor _, u := range o.Users {\n\t\tif err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor _, u := range o.Nkeys {\n\t\tif err := validateAllowedConnectionTypes(u.AllowedConnectionTypes); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn validateNoAuthUser(o, o.NoAuthUser)\n}\n\nfunc validateAllowedConnectionTypes(m map[string]struct{}) error {\n\tfor ct := range m {\n\t\tctuc := strings.ToUpper(ct)\n\t\tswitch ctuc {\n\t\tcase jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket,\n\t\t\tjwt.ConnectionTypeLeafnode, jwt.ConnectionTypeLeafnodeWS,\n\t\t\tjwt.ConnectionTypeMqtt, jwt.ConnectionTypeMqttWS,\n\t\t\tjwt.ConnectionTypeInProcess:\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"unknown connection type %q\", ct)\n\t\t}\n\t\tif ctuc != ct {\n\t\t\tdelete(m, ct)\n\t\t\tm[ctuc] = struct{}{}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validateNoAuthUser(o *Options, noAuthUser string) error {\n\tif noAuthUser == _EMPTY_ {\n\t\treturn nil\n\t}\n\tif len(o.TrustedOperators) > 0 {\n\t\treturn fmt.Errorf(\"no_auth_user not compatible with Trusted Operator\")\n\t}\n\n\tif o.Nkeys == nil && o.Users == nil {\n\t\treturn fmt.Errorf(`no_auth_user: \"%s\" present, but users/nkeys are not defined`, noAuthUser)\n\t}\n\tfor _, u := range o.Users {\n\t\tif u.Username == noAuthUser {\n\t\t\treturn nil\n\t\t}\n\t}\n\tfor _, u := range o.Nkeys {\n\t\tif u.Nkey == noAuthUser {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn fmt.Errorf(\n\t\t`no_auth_user: \"%s\" not present as user or nkey in authorization block or account configuration`,\n\t\tnoAuthUser)\n}\n\nfunc validateProxies(o *Options) error {\n\tif o.Proxies == nil {\n\t\treturn nil\n\t}\n\tfor _, p := range o.Proxies.Trusted {\n\t\tif !nkeys.IsValidPublicKey(p.Key) {\n\t\t\treturn fmt.Errorf(\"proxy trusted key %q is invalid\", p.Key)\n\t\t}\n\t}\n\treturn nil\n}\n\n// Create a list of nkeys.KeyPair corresponding to the public keys\n// of the Proxies.TrustedKeys list.\n// Server lock must be held on entry.\nfunc (s *Server) processProxiesTrustedKeys() {\n\t// We could be here on reload.\n\tif s.proxiesKeyPairs != nil {\n\t\ts.proxiesKeyPairs = s.proxiesKeyPairs[:0]\n\t}\n\tif opts := s.getOpts(); opts.Proxies == nil {\n\t\treturn\n\t}\n\tfor _, p := range s.getOpts().Proxies.Trusted {\n\t\t// Can't fail since we have already checked that it was a valid key.\n\t\tkp, _ := nkeys.FromPublicKey(p.Key)\n\t\ts.proxiesKeyPairs = append(s.proxiesKeyPairs, kp)\n\t}\n}\n\n// Returns the connection's `ClosedState` for the given authenication error.\nfunc getAuthErrClosedState(authErr error) ClosedState {\n\tswitch authErr {\n\tcase ErrAuthProxyNotTrusted:\n\t\treturn ProxyNotTrusted\n\tcase ErrAuthProxyRequired:\n\t\treturn ProxyRequired\n\tdefault:\n\t\treturn AuthenticationViolation\n\t}\n}\n" }, { "path": "server/auth_callout.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/pem\"\n\t\"errors\"\n\t\"fmt\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nconst (\n\tAuthCalloutSubject = \"$SYS.REQ.USER.AUTH\"\n\tAuthRequestSubject = \"nats-authorization-request\"\n\tAuthRequestXKeyHeader = \"Nats-Server-Xkey\"\n)\n\nfunc titleCase(m string) string {\n\tr := []rune(m)\n\tif len(r) == 0 {\n\t\treturn _EMPTY_\n\t}\n\treturn string(append([]rune{unicode.ToUpper(r[0])}, r[1:]...))\n}\n\n// Process a callout on this client's behalf.\nfunc (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool) (authorized bool, errStr string) {\n\tisOperatorMode := len(opts.TrustedKeys) > 0\n\n\t// this is the account the user connected in, or the one running the callout\n\tvar acc *Account\n\tif !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.Account != _EMPTY_ {\n\t\taname := opts.AuthCallout.Account\n\t\tvar err error\n\t\tacc, err = s.LookupAccount(aname)\n\t\tif err != nil {\n\t\t\terrStr = fmt.Sprintf(\"No valid account %q for auth callout request: %v\", aname, err)\n\t\t\ts.Warnf(errStr)\n\t\t\treturn false, errStr\n\t\t}\n\t} else {\n\t\tacc = c.acc\n\t}\n\tif acc == nil {\n\t\t// FIX for https://github.com/nats-io/nats-server/issues/7841\n\t\t// hand rolled creds on leafnode became crasher here\n\t\terrStr = fmt.Sprintf(\"%s not mapped to a callout account\", c.kindString())\n\t\ts.Warnf(errStr)\n\t\treturn false, errStr\n\t}\n\n\t// Check if we have been requested to encrypt.\n\tvar xkp nkeys.KeyPair\n\tvar xkey string\n\tvar pubAccXKey string\n\tif !isOperatorMode && opts.AuthCallout != nil && opts.AuthCallout.XKey != _EMPTY_ {\n\t\tpubAccXKey = opts.AuthCallout.XKey\n\t} else if isOperatorMode {\n\t\tpubAccXKey = acc.externalAuthXKey()\n\t}\n\t// If set grab server's xkey keypair and public key.\n\tif pubAccXKey != _EMPTY_ {\n\t\t// These are only set on creation, so lock not needed.\n\t\txkp, xkey = s.xkp, s.info.XKey\n\t}\n\n\t// FIXME: so things like the server ID that get assigned, are used as a sort of nonce - but\n\t// reality is that the keypair here, is generated, so the response generated a JWT has to be\n\t// this user - no replay possible\n\t// Create a keypair for the user. We will expect this public user to be in the signed response.\n\t// This prevents replay attacks.\n\tukp, _ := nkeys.CreateUser()\n\tpub, _ := ukp.PublicKey()\n\n\treply := s.newRespInbox()\n\trespCh := make(chan string, 1)\n\n\tdecodeResponse := func(rc *client, rmsg []byte, acc *Account) (*jwt.UserClaims, error) {\n\t\taccount := acc.Name\n\t\t_, msg := rc.msgParts(rmsg)\n\n\t\t// This signals not authorized.\n\t\t// Since this is an account subscription will always have \"\\r\\n\".\n\t\tif len(msg) <= LEN_CR_LF {\n\t\t\treturn nil, fmt.Errorf(\"auth callout violation: %q on account %q\", \"no reason supplied\", account)\n\t\t}\n\t\t// Strip trailing CRLF.\n\t\tmsg = msg[:len(msg)-LEN_CR_LF]\n\t\tencrypted := false\n\t\t// If we sent an encrypted request the response could be encrypted as well.\n\t\t// we are expecting the input to be `eyJ` if it is a JWT\n\t\tif xkp != nil && len(msg) > 0 && !bytes.HasPrefix(msg, []byte(jwtPrefix)) {\n\t\t\tvar err error\n\t\t\tmsg, err = xkp.Open(msg, pubAccXKey)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error decrypting auth callout response on account %q: %v\", account, err)\n\t\t\t}\n\t\t\tencrypted = true\n\t\t}\n\n\t\tcr, err := jwt.DecodeAuthorizationResponseClaims(string(msg))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvr := jwt.CreateValidationResults()\n\t\tcr.Validate(vr)\n\t\tif len(vr.Issues) > 0 {\n\t\t\treturn nil, fmt.Errorf(\"authorization response had validation errors: %v\", vr.Issues[0])\n\t\t}\n\n\t\t// the subject is the user id\n\t\tif cr.Subject != pub {\n\t\t\treturn nil, errors.New(\"auth callout violation: auth callout response is not for expected user\")\n\t\t}\n\n\t\t// check the audience to be the server ID\n\t\tif cr.Audience != s.info.ID {\n\t\t\treturn nil, errors.New(\"auth callout violation: auth callout response is not for server\")\n\t\t}\n\n\t\t// check if had an error message from the auth account\n\t\tif cr.Error != _EMPTY_ {\n\t\t\treturn nil, fmt.Errorf(\"auth callout service returned an error: %v\", cr.Error)\n\t\t}\n\n\t\t// if response is encrypted none of this is needed\n\t\tif isOperatorMode && !encrypted {\n\t\t\tpkStr := cr.Issuer\n\t\t\tif cr.IssuerAccount != _EMPTY_ {\n\t\t\t\tpkStr = cr.IssuerAccount\n\t\t\t}\n\t\t\tif pkStr != account {\n\t\t\t\tif _, ok := acc.signingKeys[pkStr]; !ok {\n\t\t\t\t\treturn nil, errors.New(\"auth callout signing key is unknown\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn jwt.DecodeUserClaims(cr.Jwt)\n\t}\n\n\t// getIssuerAccount returns the issuer (as per JWT) - it also asserts that\n\t// only in operator mode we expect to receive `issuer_account`.\n\tgetIssuerAccount := func(arc *jwt.UserClaims, account string) (string, error) {\n\t\t// Make sure correct issuer.\n\t\tvar issuer string\n\t\tif opts.AuthCallout != nil {\n\t\t\tissuer = opts.AuthCallout.Issuer\n\t\t} else {\n\t\t\t// Operator mode is who we send the request on unless switching accounts.\n\t\t\tissuer = acc.Name\n\t\t}\n\n\t\t// the jwt issuer can be a signing key\n\t\tjwtIssuer := arc.Issuer\n\t\tif arc.IssuerAccount != _EMPTY_ {\n\t\t\tif !isOperatorMode {\n\t\t\t\t// this should be invalid - effectively it would allow the auth callout\n\t\t\t\t// to issue on another account which may be allowed given the configuration\n\t\t\t\t// where the auth callout account can handle multiple different ones..\n\t\t\t\treturn _EMPTY_, fmt.Errorf(\"error non operator mode account %q: attempted to use issuer_account\", account)\n\t\t\t}\n\t\t\tjwtIssuer = arc.IssuerAccount\n\t\t}\n\n\t\tif jwtIssuer != issuer {\n\t\t\tif !isOperatorMode {\n\t\t\t\treturn _EMPTY_, fmt.Errorf(\"wrong issuer for auth callout response on account %q, expected %q got %q\", account, issuer, jwtIssuer)\n\t\t\t} else if !acc.isAllowedAcount(jwtIssuer) {\n\t\t\t\treturn _EMPTY_, fmt.Errorf(\"account %q not permitted as valid account option for auth callout for account %q\",\n\t\t\t\t\tarc.Issuer, account)\n\t\t\t}\n\t\t}\n\t\treturn jwtIssuer, nil\n\t}\n\n\tgetExpirationAndAllowedConnections := func(arc *jwt.UserClaims, account string) (time.Duration, map[string]struct{}, error) {\n\t\tallowNow, expiration := validateTimes(arc)\n\t\tif !allowNow {\n\t\t\tc.Errorf(\"Outside connect times\")\n\t\t\treturn 0, nil, fmt.Errorf(\"authorized user on account %q outside of valid connect times\", account)\n\t\t}\n\n\t\tallowedConnTypes, err := convertAllowedConnectionTypes(arc.User.AllowedConnectionTypes)\n\t\tif err != nil {\n\t\t\tc.Debugf(\"%v\", err)\n\t\t\tif len(allowedConnTypes) == 0 {\n\t\t\t\treturn 0, nil, fmt.Errorf(\"authorized user on account %q using invalid connection type\", account)\n\t\t\t}\n\t\t}\n\t\treturn expiration, allowedConnTypes, nil\n\t}\n\n\tassignAccountAndPermissions := func(arc *jwt.UserClaims, account string) (*Account, error) {\n\t\t// Apply to this client.\n\t\tvar err error\n\t\tissuerAccount, err := getIssuerAccount(arc, account)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// if we are not in operator mode, they can specify placement as a tag\n\t\tvar placement string\n\t\tif !isOperatorMode {\n\t\t\t// only allow placement if we are not in operator mode\n\t\t\tplacement = arc.Audience\n\t\t} else {\n\t\t\tplacement = issuerAccount\n\t\t}\n\n\t\ttargetAcc, err := s.LookupAccount(placement)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"no valid account %q for auth callout response on account %q: %v\", placement, account, err)\n\t\t}\n\t\tif isOperatorMode {\n\t\t\t// this will validate the signing key that emitted the user, and if it is a signing\n\t\t\t// key it assigns the permissions from the target account\n\t\t\tif scope, ok := targetAcc.hasIssuer(arc.Issuer); !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"user JWT issuer %q is not known\", arc.Issuer)\n\t\t\t} else if scope != nil {\n\t\t\t\t// this possibly has to be different because it could just be a plain issued by a non-scoped signing key\n\t\t\t\tif err := scope.ValidateScopedSigner(arc); err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"user JWT is not valid: %v\", err)\n\t\t\t\t} else if uSc, ok := scope.(*jwt.UserScope); !ok {\n\t\t\t\t\treturn nil, fmt.Errorf(\"user JWT is not a valid scoped user\")\n\t\t\t\t} else if arc.User.UserPermissionLimits, err = processUserPermissionsTemplate(uSc.Template, arc, targetAcc); err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"user JWT generated invalid permissions: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn targetAcc, nil\n\t}\n\n\tprocessReply := func(_ *subscription, rc *client, racc *Account, subject, reply string, rmsg []byte) {\n\t\tarc, err := decodeResponse(rc, rmsg, racc)\n\t\tif err != nil {\n\t\t\tc.authViolation()\n\t\t\trespCh <- titleCase(err.Error())\n\t\t\treturn\n\t\t}\n\t\t// If the caller had established that the user should go through a proxy,\n\t\t// or if the `arc` JWT requires it, and we don't have a trusted proxy,\n\t\t// reject the connection.\n\t\tif (proxyRequired || arc.ProxyRequired) && !trustedProxy {\n\t\t\terr = ErrAuthProxyRequired\n\t\t\tc.setAuthError(err)\n\t\t\tc.authViolation()\n\t\t\trespCh <- titleCase(err.Error())\n\t\t\treturn\n\t\t}\n\t\tvr := jwt.CreateValidationResults()\n\t\tarc.Validate(vr)\n\t\tif len(vr.Issues) > 0 {\n\t\t\tc.authViolation()\n\t\t\trespCh <- fmt.Sprintf(\"Error validating user JWT: %v\", vr.Issues[0])\n\t\t\treturn\n\t\t}\n\n\t\t// Make sure that the user is what we requested.\n\t\tif arc.Subject != pub {\n\t\t\tc.authViolation()\n\t\t\trespCh <- fmt.Sprintf(\"Expected authorized user of %q but got %q on account %q\", pub, arc.Subject, racc.Name)\n\t\t\treturn\n\t\t}\n\n\t\texpiration, allowedConnTypes, err := getExpirationAndAllowedConnections(arc, racc.Name)\n\t\tif err != nil {\n\t\t\tc.authViolation()\n\t\t\trespCh <- titleCase(err.Error())\n\t\t\treturn\n\t\t}\n\n\t\ttargetAcc, err := assignAccountAndPermissions(arc, racc.Name)\n\t\tif err != nil {\n\t\t\tc.authViolation()\n\t\t\trespCh <- titleCase(err.Error())\n\t\t\treturn\n\t\t}\n\n\t\t// the JWT is cleared, because if in operator mode it may hold the JWT\n\t\t// for the bearer token that connected to the callout if in operator mode\n\t\t// the permissions are already set on the client, this prevents a decode\n\t\t// on c.RegisterNKeyUser which would have wrong values\n\t\tc.mu.Lock()\n\t\tc.opts.JWT = _EMPTY_\n\t\tc.mu.Unlock()\n\n\t\t// Build internal user and bind to the targeted account.\n\t\tnkuser := buildInternalNkeyUser(arc, allowedConnTypes, targetAcc)\n\t\tif err := c.RegisterNkeyUser(nkuser); err != nil {\n\t\t\tc.authViolation()\n\t\t\trespCh <- fmt.Sprintf(\"Could not register auth callout user: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// See if the response wants to override the username.\n\t\tif arc.Name != _EMPTY_ {\n\t\t\tc.mu.Lock()\n\t\t\tc.opts.Username = arc.Name\n\t\t\t// Clear any others.\n\t\t\tc.opts.Nkey = _EMPTY_\n\t\t\tc.pubKey = _EMPTY_\n\t\t\tc.opts.Token = _EMPTY_\n\t\t\tc.mu.Unlock()\n\t\t}\n\n\t\t// Check if we need to set an auth timer if the user jwt expires.\n\t\tc.setExpiration(arc.Claims(), expiration)\n\n\t\trespCh <- _EMPTY_\n\t}\n\n\t// create a subscription to receive a response from the authcallout\n\tsub, err := acc.subscribeInternal(reply, processReply)\n\tif err != nil {\n\t\terrStr = fmt.Sprintf(\"Error setting up reply subscription for auth request: %v\", err)\n\t\ts.Warnf(errStr)\n\t\treturn false, errStr\n\t}\n\tdefer acc.unsubscribeInternal(sub)\n\n\t// Build our request claims - jwt subject should be nkey\n\tjwtSub := acc.Name\n\tif opts.AuthCallout != nil {\n\t\tjwtSub = opts.AuthCallout.Issuer\n\t}\n\n\t// The public key of the server, if set is available on Varz.Key\n\t// This means that when a service connects, it can now peer\n\t// authenticate if it wants to - but that also means that it needs to be\n\t// listening to cluster changes\n\tclaim := jwt.NewAuthorizationRequestClaims(jwtSub)\n\tclaim.Audience = AuthRequestSubject\n\t// Set expected public user nkey.\n\tclaim.UserNkey = pub\n\n\ts.mu.RLock()\n\tclaim.Server = jwt.ServerID{\n\t\tName: s.info.Name,\n\t\tHost: s.info.Host,\n\t\tID: s.info.ID,\n\t\tVersion: s.info.Version,\n\t\tCluster: s.info.Cluster,\n\t}\n\ts.mu.RUnlock()\n\n\t// Tags\n\tclaim.Server.Tags = s.getOpts().Tags\n\n\t// Check if we have been requested to encrypt.\n\t// FIXME: possibly this public key also needs to be on the\n\t// Varz, because then it can be peer verified?\n\tif xkp != nil {\n\t\tclaim.Server.XKey = xkey\n\t}\n\n\tauthTimeout := secondsToDuration(s.getOpts().AuthTimeout)\n\tclaim.Expires = time.Now().Add(time.Duration(authTimeout)).UTC().Unix()\n\n\t// Grab client info for the request.\n\tc.mu.Lock()\n\tc.fillClientInfo(&claim.ClientInformation)\n\tc.fillConnectOpts(&claim.ConnectOptions)\n\t// If we have a sig in the client opts, fill in nonce.\n\tif claim.ConnectOptions.SignedNonce != _EMPTY_ {\n\t\tclaim.ClientInformation.Nonce = string(c.nonce)\n\t}\n\n\t// TLS\n\tif c.flags.isSet(handshakeComplete) && c.nc != nil {\n\t\tvar ct jwt.ClientTLS\n\t\tconn := c.nc.(*tls.Conn)\n\t\tcs := conn.ConnectionState()\n\t\tct.Version = tlsVersion(cs.Version)\n\t\tct.Cipher = tls.CipherSuiteName(cs.CipherSuite)\n\t\t// Check verified chains.\n\t\tfor _, vs := range cs.VerifiedChains {\n\t\t\tvar certs []string\n\t\t\tfor _, c := range vs {\n\t\t\t\tblk := &pem.Block{\n\t\t\t\t\tType: \"CERTIFICATE\",\n\t\t\t\t\tBytes: c.Raw,\n\t\t\t\t}\n\t\t\t\tcerts = append(certs, string(pem.EncodeToMemory(blk)))\n\t\t\t}\n\t\t\tct.VerifiedChains = append(ct.VerifiedChains, certs)\n\t\t}\n\t\t// If we do not have verified chains put in peer certs.\n\t\tif len(ct.VerifiedChains) == 0 {\n\t\t\tfor _, c := range cs.PeerCertificates {\n\t\t\t\tblk := &pem.Block{\n\t\t\t\t\tType: \"CERTIFICATE\",\n\t\t\t\t\tBytes: c.Raw,\n\t\t\t\t}\n\t\t\t\tct.Certs = append(ct.Certs, string(pem.EncodeToMemory(blk)))\n\t\t\t}\n\t\t}\n\t\tclaim.TLS = &ct\n\t}\n\tc.mu.Unlock()\n\n\tb, err := claim.Encode(s.kp)\n\tif err != nil {\n\t\terrStr = fmt.Sprintf(\"Error encoding auth request claim on account %q: %v\", acc.Name, err)\n\t\ts.Warnf(errStr)\n\t\treturn false, errStr\n\t}\n\treq := []byte(b)\n\tvar hdr []byte\n\n\t// Check if we have been asked to encrypt.\n\tif xkp != nil {\n\t\treq, err = xkp.Seal([]byte(req), pubAccXKey)\n\t\tif err != nil {\n\t\t\terrStr = fmt.Sprintf(\"Error encrypting auth request claim on account %q: %v\", acc.Name, err)\n\t\t\ts.Warnf(errStr)\n\t\t\treturn false, errStr\n\t\t}\n\t\thdr = genHeader(hdr, AuthRequestXKeyHeader, xkey)\n\t}\n\n\t// Send out our request.\n\tif err := s.sendInternalAccountMsgWithReply(acc, AuthCalloutSubject, reply, hdr, req, false); err != nil {\n\t\terrStr = fmt.Sprintf(\"Error sending authorization request: %v\", err)\n\t\ts.Debugf(errStr)\n\t\treturn false, errStr\n\t}\n\tselect {\n\tcase errStr = <-respCh:\n\t\tif authorized = errStr == _EMPTY_; !authorized {\n\t\t\ts.Warnf(errStr)\n\t\t}\n\tcase <-time.After(authTimeout):\n\t\ts.Debugf(fmt.Sprintf(\"Authorization callout response not received in time on account %q\", acc.Name))\n\t}\n\n\treturn authorized, errStr\n}\n\n// Fill in client information for the request.\n// Lock should be held.\nfunc (c *client) fillClientInfo(ci *jwt.ClientInformation) {\n\tif c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {\n\t\treturn\n\t}\n\n\t// Do it this way to fail to compile if fields are added to jwt.ClientInformation.\n\t*ci = jwt.ClientInformation{\n\t\tHost: c.host,\n\t\tID: c.cid,\n\t\tUser: c.getRawAuthUser(),\n\t\tName: c.opts.Name,\n\t\tTags: c.tags,\n\t\tNameTag: c.nameTag,\n\t\tKind: c.kindString(),\n\t\tType: c.clientTypeString(),\n\t\tMQTT: c.getMQTTClientID(),\n\t}\n}\n\n// Fill in client options.\n// Lock should be held.\nfunc (c *client) fillConnectOpts(opts *jwt.ConnectOptions) {\n\tif c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {\n\t\treturn\n\t}\n\n\to := c.opts\n\n\t// Do it this way to fail to compile if fields are added to jwt.ClientInformation.\n\t*opts = jwt.ConnectOptions{\n\t\tJWT: o.JWT,\n\t\tNkey: o.Nkey,\n\t\tSignedNonce: o.Sig,\n\t\tToken: o.Token,\n\t\tUsername: o.Username,\n\t\tPassword: o.Password,\n\t\tName: o.Name,\n\t\tLang: o.Lang,\n\t\tVersion: o.Version,\n\t\tProtocol: o.Protocol,\n\t}\n}\n" }, { "path": "server/auth_callout_test.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"encoding/pem\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\n// Helper function to decode an auth request.\nfunc decodeAuthRequest(t *testing.T, ejwt []byte) (string, *jwt.ServerID, *jwt.ClientInformation, *jwt.ConnectOptions, *jwt.ClientTLS) {\n\tt.Helper()\n\tac, err := jwt.DecodeAuthorizationRequestClaims(string(ejwt))\n\trequire_NoError(t, err)\n\treturn ac.UserNkey, &ac.Server, &ac.ClientInformation, &ac.ConnectOptions, ac.TLS\n}\n\nfunc TestTitleCaseEmptyString(t *testing.T) {\n\tdefer require_NoPanic(t)\n\trequire_Equal(t, titleCase(_EMPTY_), _EMPTY_)\n}\n\nconst (\n\tauthCalloutPub = \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\"\n\tauthCalloutSeed = \"SUAP277QP7U4JMFFPVZHLJYEQJ2UHOTYVEIZJYAWRJXQLP4FRSEHYZJJOU\"\n\tauthCalloutIssuer = \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\tauthCalloutIssuerSeed = \"SAANDLKMXL6CUS3CP52WIXBEDN6YJ545GDKC65U5JZPPV6WH6ESWUA6YAI\"\n\tauthCalloutIssuerSK = \"SAAE46BB675HKZKSVJEUZAKKWIV6BJJO6XYE46Z3ZHO7TCI647M3V42IJE\"\n)\n\nfunc serviceResponse(t *testing.T, userID string, serverID string, uJwt string, errMsg string, expires time.Duration) []byte {\n\tcr := jwt.NewAuthorizationResponseClaims(userID)\n\tcr.Audience = serverID\n\tcr.Error = errMsg\n\tcr.Jwt = uJwt\n\tif expires != 0 {\n\t\tcr.Expires = time.Now().Add(expires).Unix()\n\t}\n\taa, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\ttoken, err := cr.Encode(aa)\n\trequire_NoError(t, err)\n\treturn []byte(token)\n}\n\nfunc newScopedRole(t *testing.T, role string, pub []string, sub []string, allowResponses bool) (*jwt.UserScope, nkeys.KeyPair) {\n\takp, pk := createKey(t)\n\tr := jwt.NewUserScope()\n\tr.Key = pk\n\tr.Template.Sub.Allow.Add(sub...)\n\tr.Template.Pub.Allow.Add(pub...)\n\tif allowResponses {\n\t\tr.Template.Resp = &jwt.ResponsePermission{\n\t\t\tMaxMsgs: 1,\n\t\t\tExpires: time.Second * 3,\n\t\t}\n\t}\n\tr.Role = role\n\treturn r, akp\n}\n\n// Will create a signed user jwt as an authorized user.\nfunc createAuthUser(t *testing.T, user, name, account, issuerAccount string, akp nkeys.KeyPair, expires time.Duration, limits *jwt.UserPermissionLimits) string {\n\tt.Helper()\n\n\tif akp == nil {\n\t\tvar err error\n\t\takp, err = nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\t\trequire_NoError(t, err)\n\t}\n\n\tuc := jwt.NewUserClaims(user)\n\tif issuerAccount != \"\" {\n\t\tif _, err := nkeys.FromPublicKey(issuerAccount); err != nil {\n\t\t\tt.Fatalf(\"issuer account is not a public key: %v\", err)\n\t\t}\n\t\tuc.IssuerAccount = issuerAccount\n\t}\n\t// The callout uses the audience as the target account\n\t// only if in non-operator mode, otherwise the user JWT has\n\t// correct attribution - issuer or issuer_account\n\tif _, err := nkeys.FromPublicKey(account); err != nil {\n\t\t// if it is not a public key, set the audience\n\t\tuc.Audience = account\n\t}\n\n\tif name != _EMPTY_ {\n\t\tuc.Name = name\n\t}\n\tif expires != 0 {\n\t\tuc.Expires = time.Now().Add(expires).Unix()\n\t}\n\tif limits != nil {\n\t\tuc.UserPermissionLimits = *limits\n\t}\n\n\tvr := jwt.CreateValidationResults()\n\tuc.Validate(vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\n\ttok, err := uc.Encode(akp)\n\trequire_NoError(t, err)\n\n\treturn tok\n}\n\ntype authTest struct {\n\tt *testing.T\n\tsrv *Server\n\tconf string\n\tauthClient *nats.Conn\n\tclients []*nats.Conn\n}\n\nfunc NewAuthTest(t *testing.T, config string, authHandler nats.MsgHandler, clientOptions ...nats.Option) *authTest {\n\ta := &authTest{t: t}\n\ta.conf = createConfFile(t, []byte(config))\n\ta.srv, _ = RunServerWithConfig(a.conf)\n\n\tvar err error\n\ta.authClient = a.ConnectCallout(clientOptions...)\n\t_, err = a.authClient.Subscribe(AuthCalloutSubject, authHandler)\n\trequire_NoError(t, err)\n\treturn a\n}\n\nfunc (at *authTest) NewClient(clientOptions ...nats.Option) (*nats.Conn, error) {\n\tconn, err := nats.Connect(at.srv.ClientURL(), clientOptions...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tat.clients = append(at.clients, conn)\n\treturn conn, nil\n}\n\nfunc (at *authTest) ConnectCallout(clientOptions ...nats.Option) *nats.Conn {\n\tconn, err := at.NewClient(clientOptions...)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"callout client failed: %w\", err)\n\t}\n\trequire_NoError(at.t, err)\n\treturn conn\n}\n\nfunc (at *authTest) Connect(clientOptions ...nats.Option) *nats.Conn {\n\tconn, err := at.NewClient(clientOptions...)\n\trequire_NoError(at.t, err)\n\treturn conn\n}\n\nfunc (at *authTest) WSNewClient(clientOptions ...nats.Option) (*nats.Conn, error) {\n\tpi := at.srv.PortsInfo(10 * time.Millisecond)\n\trequire_False(at.t, pi == nil)\n\n\t// test cert is SAN to DNS localhost, not local IPs returned by server in test environments\n\twssUrl := strings.Replace(pi.WebSocket[0], \"127.0.0.1\", \"localhost\", 1)\n\n\t// Seeing 127.0.1.1 in some test environments...\n\twssUrl = strings.Replace(wssUrl, \"127.0.1.1\", \"localhost\", 1)\n\n\tconn, err := nats.Connect(wssUrl, clientOptions...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tat.clients = append(at.clients, conn)\n\treturn conn, nil\n}\n\nfunc (at *authTest) WSConnect(clientOptions ...nats.Option) *nats.Conn {\n\tconn, err := at.WSNewClient(clientOptions...)\n\trequire_NoError(at.t, err)\n\treturn conn\n}\n\nfunc (at *authTest) RequireConnectError(clientOptions ...nats.Option) {\n\t_, err := at.NewClient(clientOptions...)\n\trequire_Error(at.t, err)\n}\n\nfunc (at *authTest) Cleanup() {\n\tif at.authClient != nil {\n\t\tat.authClient.Close()\n\t}\n\tif at.srv != nil {\n\t\tat.srv.Shutdown()\n\t\tremoveFile(at.t, at.conf)\n\t}\n}\n\nfunc TestAuthCalloutBasics(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\t# users that will power the auth callout service.\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\tcallouts := uint32(0)\n\thandler := func(m *nats.Msg) {\n\t\tatomic.AddUint32(&callouts, 1)\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"A\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\t// Allow dlc user.\n\t\tif (opts.Username == \"dlc\" && opts.Password == \"zzz\") || opts.Token == \"SECRET_TOKEN\" {\n\t\t\tvar j jwt.UserPermissionLimits\n\t\t\tj.Pub.Allow.Add(\"$SYS.>\")\n\t\t\tj.Payload = 1024\n\t\t\tif opts.Token == \"SECRET_TOKEN\" {\n\t\t\t\t// Token MUST NOT be exposed in user info.\n\t\t\t\trequire_Equal(t, ci.User, \"[REDACTED]\")\n\t\t\t}\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, &j)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Username == \"proxy\" {\n\t\t\tvar j jwt.UserPermissionLimits\n\t\t\tj.ProxyRequired = true\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, &j)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\t// Nil response signals no authentication.\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\n\t// This one should fail since bad password.\n\tat.RequireConnectError(nats.UserInfo(\"dlc\", \"xxx\"))\n\n\t// This one should fail because it will require to be proxied and it is not.\n\tat.RequireConnectError(nats.UserInfo(\"proxy\", \"xxx\"))\n\n\t// This one will use callout since not defined in server config.\n\tnc := at.Connect(nats.UserInfo(\"dlc\", \"zzz\"))\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\n\tdlc := &UserInfo{\n\t\tUserID: \"dlc\",\n\t\tAccount: globalAccountName,\n\t\tAccountName: globalAccountName,\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tAllow: []string{\"$SYS.>\"},\n\t\t\t\tDeny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account.\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{},\n\t\t},\n\t}\n\texpires := userInfo.Expires\n\tuserInfo.Expires = 0\n\tif !reflect.DeepEqual(dlc, userInfo) {\n\t\tt.Fatalf(\"User info for %q did not match\", \"dlc\")\n\t}\n\tif expires > 10*time.Minute || expires < (10*time.Minute-5*time.Second) {\n\t\tt.Fatalf(\"Expected expires of ~%v, got %v\", 10*time.Minute, expires)\n\t}\n\n\t// Callout with a token should also work, regardless of it being redacted in the user info.\n\tnc.Close()\n\tnc = at.Connect(nats.Token(\"SECRET_TOKEN\"))\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\tdlc = &UserInfo{\n\t\t// Token MUST NOT be exposed in user info.\n\t\tUserID: \"[REDACTED]\",\n\t\tAccount: globalAccountName,\n\t\tAccountName: globalAccountName,\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tAllow: []string{\"$SYS.>\"},\n\t\t\t\tDeny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account.\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{},\n\t\t},\n\t}\n\texpires = userInfo.Expires\n\tuserInfo.Expires = 0\n\tif !reflect.DeepEqual(dlc, userInfo) {\n\t\tt.Fatalf(\"User info for %q did not match\", \"dlc\")\n\t}\n\tif expires > 10*time.Minute || expires < (10*time.Minute-5*time.Second) {\n\t\tt.Fatalf(\"Expected expires of ~%v, got %v\", 10*time.Minute, expires)\n\t}\n}\n\nfunc TestAuthCalloutMultiAccounts(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: ZZ\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t\tBAR {}\n\t\t\tBAZ {}\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"ZZ\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\t// Allow dlc user and map to the BAZ account.\n\t\tif opts.Username == \"dlc\" && opts.Password == \"zzz\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"BAZ\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\t// Nil response signals no authentication.\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\n\t// This one will use callout since not defined in server config.\n\tnc := at.Connect(nats.UserInfo(\"dlc\", \"zzz\"))\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo := response.Data.(*UserInfo)\n\n\trequire_True(t, userInfo.UserID == \"dlc\")\n\trequire_True(t, userInfo.Account == \"BAZ\")\n}\n\nfunc TestAuthCalloutAllowedAccounts(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: ZZ\n\t\taccounts {\n\t\t\tAUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO { users [ {user: \"foo\", password: \"pwd\"} ] }\n\t\t\tBAR {}\n\t\t\tSYS { users [ {user: \"sys\", password: \"pwd\"} ] }\n\t\t}\n\t\tsystem_account: SYS\n\t\tno_auth_user: foo\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t\tallowed_accounts: [ BAR ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tt.Helper()\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"ZZ\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\t// Allow dlc user and map to the BAZ account.\n\t\tif opts.Username == \"dlc\" && opts.Password == \"zzz\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"BAR\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\t// Nil response signals no authentication.\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tcheck := func(at *authTest, user, password, account string) {\n\t\tt.Helper()\n\n\t\tvar nc *nats.Conn\n\t\t// Assume no auth user.\n\t\tif password == \"\" {\n\t\t\tnc = at.Connect()\n\t\t} else {\n\t\t\tnc = at.Connect(nats.UserInfo(user, password))\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\t\terr = json.Unmarshal(resp.Data, &response)\n\t\trequire_NoError(t, err)\n\t\tuserInfo := response.Data.(*UserInfo)\n\n\t\trequire_True(t, userInfo.UserID == user)\n\t\trequire_True(t, userInfo.Account == account)\n\t}\n\n\ttests := []struct {\n\t\tuser string\n\t\tpassword string\n\t\taccount string\n\t}{\n\t\t{\"dlc\", \"zzz\", \"BAR\"},\n\t\t{\"foo\", \"\", \"FOO\"},\n\t\t{\"sys\", \"pwd\", \"SYS\"},\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\n\tfor _, test := range tests {\n\t\tt.Run(test.user, func(t *testing.T) {\n\t\t\tat.t = t\n\t\t\tcheck(at, test.user, test.password, test.account)\n\t\t})\n\t}\n}\n\nfunc TestAuthCalloutClientTLSCerts(t *testing.T) {\n\tconf := `\n\t\tlisten: \"localhost:-1\"\n\t\tserver_name: T\n\n\t\ttls {\n\t\t\tcert_file = \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\tkey_file = \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\tca_file = \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\tverify = true\n\t\t}\n\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, ci, _, ctls := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"T\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\trequire_True(t, ctls != nil)\n\t\t// Zero since we are verified and will be under verified chains.\n\t\trequire_True(t, len(ctls.Certs) == 0)\n\t\trequire_True(t, len(ctls.VerifiedChains) == 1)\n\t\t// Since we have a CA.\n\t\trequire_True(t, len(ctls.VerifiedChains[0]) == 2)\n\t\tblk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0]))\n\t\tcert, err := x509.ParseCertificate(blk.Bytes)\n\t\trequire_NoError(t, err)\n\t\tif strings.HasPrefix(cert.Subject.String(), \"CN=example.com\") {\n\t\t\t// Override blank name here, server will substitute.\n\t\t\tujwt := createAuthUser(t, user, \"dlc\", \"FOO\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler,\n\t\tnats.UserInfo(\"auth\", \"pwd\"),\n\t\tnats.ClientCert(\"../test/configs/certs/tlsauth/client2.pem\", \"../test/configs/certs/tlsauth/client2-key.pem\"),\n\t\tnats.RootCAs(\"../test/configs/certs/tlsauth/ca.pem\"))\n\tdefer ac.Cleanup()\n\n\t// Will use client cert to determine user.\n\tnc := ac.Connect(\n\t\tnats.ClientCert(\"../test/configs/certs/tlsauth/client2.pem\", \"../test/configs/certs/tlsauth/client2-key.pem\"),\n\t\tnats.RootCAs(\"../test/configs/certs/tlsauth/ca.pem\"),\n\t)\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo := response.Data.(*UserInfo)\n\n\trequire_True(t, userInfo.UserID == \"dlc\")\n\trequire_True(t, userInfo.Account == \"FOO\")\n}\n\nfunc TestAuthCalloutVerifiedUserCalloutsWithSig(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tusers: [\n\t\t\t\t{ user: \"auth\", password: \"pwd\" }\n \t\t\t\t{ nkey: \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\" }\n \t\t\t]\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\t# users that will power the auth callout service.\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\tcallouts := uint32(0)\n\thandler := func(m *nats.Msg) {\n\t\tatomic.AddUint32(&callouts, 1)\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"A\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\trequire_True(t, opts.SignedNonce != _EMPTY_)\n\t\trequire_True(t, ci.Nonce != _EMPTY_)\n\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 0, nil)\n\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t}\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tseedFile := createTempFile(t, _EMPTY_)\n\tdefer removeFile(t, seedFile.Name())\n\tseedFile.WriteString(authCalloutSeed)\n\tnkeyOpt, err := nats.NkeyOptionFromSeed(seedFile.Name())\n\trequire_NoError(t, err)\n\n\tnc := ac.Connect(nkeyOpt)\n\tdefer nc.Close()\n\n\t// Make sure that the callout was called.\n\tif atomic.LoadUint32(&callouts) != 1 {\n\t\tt.Fatalf(\"Expected callout to be called\")\n\t}\n\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\n\tdlc := &UserInfo{\n\t\tUserID: \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\",\n\t\tAccount: globalAccountName,\n\t\tAccountName: globalAccountName,\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tDeny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account.\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{},\n\t\t},\n\t}\n\tif !reflect.DeepEqual(dlc, userInfo) {\n\t\tt.Fatalf(\"User info for %q did not match\", \"dlc\")\n\t}\n}\n\n// For creating the authorized users in operator mode.\nfunc createAuthServiceUser(t *testing.T, accKp nkeys.KeyPair) (pub, creds string) {\n\tt.Helper()\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"auth-service\"\n\tuclaim.Subject = upub\n\tvr := jwt.ValidationResults{}\n\tuclaim.Validate(&vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn upub, genCredsFile(t, ujwt, seed)\n}\n\nfunc createBasicAccountUser(t *testing.T, accKp nkeys.KeyPair) (creds string) {\n\treturn createBasicAccount(t, \"auth-client\", accKp, true)\n}\n\nfunc createBasicAccountLeaf(t *testing.T, accKp nkeys.KeyPair) (creds string) {\n\treturn createBasicAccount(t, \"auth-leaf\", accKp, false)\n}\n\nfunc createBasicAccountBearer(t *testing.T, accKp nkeys.KeyPair) string {\n\tt.Helper()\n\tukp, _ := nkeys.CreateUser()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"default_sentinel\"\n\tuclaim.Subject = upub\n\tuclaim.BearerToken = true\n\n\tuclaim.Permissions.Pub.Deny.Add(\">\")\n\tuclaim.Permissions.Sub.Deny.Add(\">\")\n\n\tvr := jwt.ValidationResults{}\n\tuclaim.Validate(&vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn ujwt\n}\n\nfunc createBasicAccount(t *testing.T, name string, accKp nkeys.KeyPair, addDeny bool) (creds string) {\n\tt.Helper()\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub\n\tuclaim.Name = name\n\tif addDeny {\n\t\t// For these deny all permission\n\t\tuclaim.Permissions.Pub.Deny.Add(\">\")\n\t\tuclaim.Permissions.Sub.Deny.Add(\">\")\n\t}\n\tvr := jwt.ValidationResults{}\n\tuclaim.Validate(&vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn genCredsFile(t, ujwt, seed)\n}\n\nfunc createScopedUser(t *testing.T, accKp nkeys.KeyPair, sk nkeys.KeyPair) (creds string) {\n\tt.Helper()\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tapk, _ := accKp.PublicKey()\n\tuclaim.IssuerAccount = apk\n\tuclaim.Subject = upub\n\tuclaim.Name = \"scoped-user\"\n\tuclaim.SetScoped(true)\n\n\t// Uncomment this to set the sub limits\n\t// uclaim.Limits.Subs = 0\n\tvr := jwt.ValidationResults{}\n\tuclaim.Validate(&vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\tujwt, err := uclaim.Encode(sk)\n\trequire_NoError(t, err)\n\treturn genCredsFile(t, ujwt, seed)\n}\n\nfunc TestAuthCalloutOperatorNoServerConfigCalloutAllowed(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: MEM\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n `, ojwt)))\n\tdefer removeFile(t, conf)\n\n\topts := LoadConfig(conf)\n\t_, err := NewServer(opts)\n\trequire_Error(t, err, errors.New(\"operators do not allow authorization callouts to be configured directly\"))\n}\n\nfunc TestAuthCalloutOperatorModeBasics(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// TEST account.\n\ttkp, tpub := createKey(t)\n\ttSigningKp, tSigningPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(tpub)\n\taccClaim.Name = \"TEST\"\n\taccClaim.SigningKeys.Add(tSigningPub)\n\tscope, scopedKp := newScopedRole(t, \"foo\", []string{\"foo.>\", \"$SYS.REQ.USER.INFO\"}, []string{\"foo.>\", \"_INBOX.>\"}, false)\n\taccClaim.SigningKeys.AddScopedSigner(scope)\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH service account.\n\takp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tapub, err := akp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(apub)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(tpub)\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdefaultSentinel := createBasicAccountBearer(t, akp)\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n default_sentinel: %s\n `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt, defaultSentinel)\n\n\tconst secretToken = \"--XX--\"\n\tconst dummyToken = \"--ZZ--\"\n\tconst skKeyToken = \"--SK--\"\n\tconst scopedToken = \"--Scoped--\"\n\tconst badScopedToken = \"--BADScoped--\"\n\tconst defaultToken = \"--Default--\"\n\n\tdkp, notAllowAccountPub := createKey(t)\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Token == secretToken {\n\t\t\tujwt := createAuthUser(t, user, \"dlc\", tpub, \"\", tkp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == dummyToken {\n\t\t\tujwt := createAuthUser(t, user, \"dummy\", notAllowAccountPub, \"\", dkp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == skKeyToken {\n\t\t\tujwt := createAuthUser(t, user, \"sk\", tpub, tpub, tSigningKp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == scopedToken {\n\t\t\t// must have no limits set\n\t\t\tujwt := createAuthUser(t, user, \"scoped\", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{})\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == badScopedToken {\n\t\t\t// limits are nil - here which result in a default user - this will fail scoped\n\t\t\tujwt := createAuthUser(t, user, \"bad-scoped\", tpub, tpub, scopedKp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == defaultToken {\n\t\t\tujwt := createAuthUser(t, user, \"default\", tpub, \"\", tkp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\tresp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\texpected := &UserInfo{\n\t\tUserID: upub,\n\t\tAccount: apub,\n\t\tAccountName: \"AUTH\",\n\t\tUserName: \"auth-service\",\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tDeny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account.\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{},\n\t\t},\n\t}\n\tif !reflect.DeepEqual(expected, userInfo) {\n\t\tt.Fatalf(\"User info did not match expected, expected auto-deny permissions on callout subject\")\n\t}\n\n\t// Bearer token etc..\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createBasicAccountUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\t// We require a token.\n\tac.RequireConnectError(nats.UserCredentials(creds))\n\n\t// Send correct token. This should switch us to the test account.\n\tnc := ac.Connect(nats.UserCredentials(creds), nats.Token(secretToken))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\n\t// Make sure we switch accounts.\n\tif userInfo.Account != tpub {\n\t\tt.Fatalf(\"Expected to be switched to %q, but got %q\", tpub, userInfo.Account)\n\t}\n\n\t// Now make sure that if the authorization service switches to an account that is not allowed, we reject.\n\tac.RequireConnectError(nats.UserCredentials(creds), nats.Token(dummyToken))\n\n\t// Send the signing key token. This should switch us to the test account, but the user\n\t// is signed with the account signing key\n\tnc = ac.Connect(nats.UserCredentials(creds), nats.Token(skKeyToken))\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\tif userInfo.Account != tpub {\n\t\tt.Fatalf(\"Expected to be switched to %q, but got %q\", tpub, userInfo.Account)\n\t}\n\n\t// bad scoped user\n\tac.RequireConnectError(nats.UserCredentials(creds), nats.Token(badScopedToken))\n\n\t// Send the signing key token. This should switch us to the test account, but the user\n\t// is signed with the account signing key\n\tnc = ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\tif userInfo.Account != tpub {\n\t\tt.Fatalf(\"Expected to be switched to %q, but got %q\", tpub, userInfo.Account)\n\t}\n\trequire_True(t, len(userInfo.Permissions.Publish.Allow) == 2)\n\tslices.Sort(userInfo.Permissions.Publish.Allow)\n\trequire_Equal(t, \"foo.>\", userInfo.Permissions.Publish.Allow[1])\n\tslices.Sort(userInfo.Permissions.Subscribe.Allow)\n\trequire_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2)\n\trequire_Equal(t, \"foo.>\", userInfo.Permissions.Subscribe.Allow[1])\n\n\t// this connects without a credential, so will be assigned the default sentinel\n\tnc = ac.Connect(nats.Token(defaultToken))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\tui := response.Data.(*UserInfo)\n\trequire_Equal(t, \"default\", ui.UserID)\n\trequire_NoError(t, err)\n\n}\n\nfunc testAuthCalloutScopedUser(t *testing.T, allowAnyAccount bool) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// TEST account.\n\t_, tpub := createKey(t)\n\t_, tSigningPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(tpub)\n\taccClaim.Name = \"TEST\"\n\taccClaim.SigningKeys.Add(tSigningPub)\n\tscope, scopedKp := newScopedRole(t, \"foo\", []string{\"foo.>\", \"$SYS.REQ.USER.INFO\"}, []string{\"foo.>\", \"_INBOX.>\"}, true)\n\tscope.Template.Limits.Subs = 10\n\tscope.Template.Limits.Payload = 512\n\taccClaim.SigningKeys.AddScopedSigner(scope)\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH service account.\n\takp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tapub, err := akp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(apub)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tif allowAnyAccount {\n\t\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\t} else {\n\t\tauthClaim.Authorization.AllowedAccounts.Add(tpub)\n\t}\n\t// the scope for the bearer token which has no permissions\n\tsentinelScope, authKP := newScopedRole(t, \"sentinel\", nil, nil, false)\n\tsentinelScope.Template.Sub.Deny.Add(\">\")\n\tsentinelScope.Template.Pub.Deny.Add(\">\")\n\tsentinelScope.Template.Limits.Subs = 0\n\tsentinelScope.Template.Payload = 0\n\tauthClaim.SigningKeys.AddScopedSigner(sentinelScope)\n\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n listen: 127.0.0.1:-1\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: {\n %s: %s\n %s: %s\n %s: %s\n }\n `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt)\n\n\tconst scopedToken = \"--Scoped--\"\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Token == scopedToken {\n\t\t\t// must have no limits set\n\t\t\tujwt := createAuthUser(t, user, \"scoped\", tpub, tpub, scopedKp, 0, &jwt.UserPermissionLimits{})\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\tresp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\texpected := &UserInfo{\n\t\tUserID: upub,\n\t\tAccount: apub,\n\t\tAccountName: \"AUTH\",\n\t\tUserName: \"auth-service\",\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tDeny: []string{AuthCalloutSubject}, // Will be auto-added since in auth account.\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{},\n\t\t},\n\t}\n\tif !reflect.DeepEqual(expected, userInfo) {\n\t\tt.Fatalf(\"User info did not match expected, expected auto-deny permissions on callout subject\")\n\t}\n\n\t// Bearer token - this has no permissions see sentinelScope\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createScopedUser(t, akp, authKP)\n\tdefer removeFile(t, creds)\n\n\t// Send the signing key token. This should switch us to the test account, but the user\n\t// is signed with the account signing key\n\tnc := ac.Connect(nats.UserCredentials(creds), nats.Token(scopedToken))\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\tif userInfo.Account != tpub {\n\t\tt.Fatalf(\"Expected to be switched to %q, but got %q\", tpub, userInfo.Account)\n\t}\n\trequire_True(t, len(userInfo.Permissions.Publish.Allow) == 2)\n\tslices.Sort(userInfo.Permissions.Publish.Allow)\n\trequire_Equal(t, \"foo.>\", userInfo.Permissions.Publish.Allow[1])\n\tslices.Sort(userInfo.Permissions.Subscribe.Allow)\n\trequire_True(t, len(userInfo.Permissions.Subscribe.Allow) == 2)\n\trequire_Equal(t, \"foo.>\", userInfo.Permissions.Subscribe.Allow[1])\n\n\t_, err = nc.Subscribe(\"foo.>\", func(msg *nats.Msg) {\n\t\tt.Log(\"got request on foo.>\")\n\t\trequire_NoError(t, msg.Respond(nil))\n\t})\n\trequire_NoError(t, err)\n\n\tm, err := nc.Request(\"foo.bar\", nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, m)\n\tt.Log(\"go response from foo.bar\")\n\n\tnc.Close()\n}\n\nfunc TestAuthCalloutScopedUserAssignedAccount(t *testing.T) {\n\ttestAuthCalloutScopedUser(t, false)\n}\n\nfunc TestAuthCalloutScopedUserAllAccount(t *testing.T) {\n\ttestAuthCalloutScopedUser(t, true)\n}\n\nconst (\n\tcurveSeed = \"SXAAXMRAEP6JWWHNB6IKFL554IE6LZVT6EY5MBRICPILTLOPHAG73I3YX4\"\n\tcurvePublic = \"XAB3NANV3M6N7AHSQP2U5FRWKKUT7EG2ZXXABV4XVXYQRJGM4S2CZGHT\"\n)\n\nfunc TestAuthCalloutServerConfigEncryption(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\t# users that will power the auth callout service.\n\t\t\t\tauth_users: [ auth ]\n\t\t\t\t# This is a public xkey (x25519). The auth service has the private key.\n\t\t\t\txkey: \"%s\"\n\t\t\t}\n\t\t}\n\t`\n\tconf := fmt.Sprintf(tmpl, curvePublic)\n\n\trkp, err := nkeys.FromCurveSeed([]byte(curveSeed))\n\trequire_NoError(t, err)\n\n\thandler := func(m *nats.Msg) {\n\t\t// This will be encrypted.\n\t\t_, err := jwt.DecodeAuthorizationRequestClaims(string(m.Data))\n\t\trequire_Error(t, err)\n\n\t\txkey := m.Header.Get(AuthRequestXKeyHeader)\n\t\trequire_True(t, xkey != _EMPTY_)\n\t\tdecrypted, err := rkp.Open(m.Data, xkey)\n\t\trequire_NoError(t, err)\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, decrypted)\n\t\t// The header xkey must match the signed xkey in server info.\n\t\trequire_True(t, si.XKey == xkey)\n\t\trequire_True(t, si.Name == \"A\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\t// Allow dlc user.\n\t\tif opts.Username == \"dlc\" && opts.Password == \"zzz\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Username == \"dlc\" && opts.Password == \"xxx\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, nil)\n\t\t\t// Encrypt this response.\n\t\t\tdata, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, \"\", 0), si.XKey) // Server's public xkey.\n\t\t\trequire_NoError(t, err)\n\t\t\tm.Respond(data)\n\t\t} else {\n\t\t\t// Nil response signals no authentication.\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tnc := ac.Connect(nats.UserInfo(\"dlc\", \"zzz\"))\n\tdefer nc.Close()\n\n\t// Authorization services can optionally encrypt the responses using the server's public xkey.\n\tnc = ac.Connect(nats.UserInfo(\"dlc\", \"xxx\"))\n\tdefer nc.Close()\n}\n\nfunc TestAuthCalloutOperatorModeEncryption(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// TEST account.\n\ttkp, tpub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(tpub)\n\taccClaim.Name = \"TEST\"\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH service account.\n\takp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tapub, err := akp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(apub)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(tpub)\n\tauthClaim.Authorization.XKey = curvePublic\n\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, spub, apub, authJwt, tpub, accJwt, spub, sysJwt)\n\n\trkp, err := nkeys.FromCurveSeed([]byte(curveSeed))\n\trequire_NoError(t, err)\n\n\tconst tokenA = \"--XX--\"\n\tconst tokenB = \"--ZZ--\"\n\n\thandler := func(m *nats.Msg) {\n\t\t// Make sure this is an encrypted request.\n\t\tif bytes.HasPrefix(m.Data, []byte(jwtPrefix)) {\n\t\t\tt.Fatalf(\"Request not encrypted\")\n\t\t}\n\t\txkey := m.Header.Get(AuthRequestXKeyHeader)\n\t\trequire_True(t, xkey != _EMPTY_)\n\t\tdecrypted, err := rkp.Open(m.Data, xkey)\n\t\trequire_NoError(t, err)\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, decrypted)\n\t\t// The header xkey must match the signed xkey in server info.\n\t\trequire_True(t, si.XKey == xkey)\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\tif opts.Token == tokenA {\n\t\t\tujwt := createAuthUser(t, user, \"dlc\", tpub, \"\", tkp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == tokenB {\n\t\t\tujwt := createAuthUser(t, user, \"rip\", tpub, \"\", tkp, 0, nil)\n\t\t\t// Encrypt this response.\n\t\t\tdata, err := rkp.Seal(serviceResponse(t, user, si.ID, ujwt, \"\", 0), si.XKey) // Server's public xkey.\n\t\t\trequire_NoError(t, err)\n\t\t\tm.Respond(data)\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\n\t// Bearer token etc..\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createBasicAccountUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\t// This will receive an encrypted request to the auth service but send plaintext response.\n\tnc := ac.Connect(nats.UserCredentials(creds), nats.Token(tokenA))\n\tdefer nc.Close()\n\n\t// This will receive an encrypted request to the auth service and send an encrypted response.\n\tnc = ac.Connect(nats.UserCredentials(creds), nats.Token(tokenB))\n\tdefer nc.Close()\n}\n\nfunc TestAuthCalloutServerTags(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tserver_tags: [\"foo\", \"bar\"]\n\t\tauthorization {\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\ttch := make(chan jwt.TagList, 1)\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\ttch <- si.Tags\n\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, nil)\n\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tnc := ac.Connect()\n\tdefer nc.Close()\n\n\ttags := <-tch\n\trequire_True(t, len(tags) == 2)\n\trequire_True(t, tags.Contains(\"foo\"))\n\trequire_True(t, tags.Contains(\"bar\"))\n}\n\nfunc TestAuthCalloutServerClusterAndVersion(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t\tcluster { name: HUB }\n\t`\n\tch := make(chan string, 2)\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tch <- si.Cluster\n\t\tch <- si.Version\n\t\tujwt := createAuthUser(t, user, _EMPTY_, globalAccountName, \"\", nil, 10*time.Minute, nil)\n\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tnc := ac.Connect()\n\tdefer nc.Close()\n\n\tcluster := <-ch\n\trequire_True(t, cluster == \"HUB\")\n\n\tversion := <-ch\n\trequire_True(t, len(version) > 0)\n\tok, err := versionAtLeastCheckError(version, 2, 10, 0)\n\trequire_NoError(t, err)\n\trequire_True(t, ok)\n}\n\nfunc TestAuthCalloutErrorResponse(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"BAD AUTH\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tac.RequireConnectError(nats.UserInfo(\"dlc\", \"zzz\"))\n}\n\nfunc TestAuthCalloutAuthUserFailDoesNotInvokeCallout(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\tauthorization {\n\t\t\tusers: [ { user: \"auth\", password: \"pwd\" } ]\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\tcallouts := uint32(0)\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tatomic.AddUint32(&callouts, 1)\n\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"WRONG PASSWORD\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\tac.RequireConnectError(nats.UserInfo(\"auth\", \"zzz\"))\n\n\tif atomic.LoadUint32(&callouts) != 0 {\n\t\tt.Fatalf(\"Expected callout to not be called\")\n\t}\n}\n\nfunc TestAuthCalloutAuthErrEvents(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t\tBAR {}\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\t// Allow dlc user and map to the BAZ account.\n\t\tif opts.Username == \"dlc\" && opts.Password == \"zzz\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Username == \"dlc\" {\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"WRONG PASSWORD\", 0))\n\t\t} else if opts.Username == \"rip\" {\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"BAD CREDS\", 0))\n\t\t} else if opts.Username == \"proxy\" {\n\t\t\tvar j jwt.UserPermissionLimits\n\t\t\tj.ProxyRequired = true\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, &j)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// This is where the event fires, in this account.\n\tsub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\t// This one will use callout since not defined in server config.\n\tnc := ac.Connect(nats.UserInfo(\"dlc\", \"zzz\"))\n\tdefer nc.Close()\n\tcheckSubsPending(t, sub, 0)\n\n\tcheckAuthErrEvent := func(user, pass, reason string) {\n\t\tac.RequireConnectError(nats.UserInfo(user, pass))\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\t// Convert both reasons to lower case to do the comparison.\n\t\tdmr := strings.ToLower(dm.Reason)\n\t\tr := strings.ToLower(reason)\n\n\t\tif !strings.Contains(dmr, r) {\n\t\t\tt.Fatalf(\"Expected %q reason, but got %q\", r, dmr)\n\t\t}\n\t}\n\n\tcheckAuthErrEvent(\"dlc\", \"xxx\", \"WRONG PASSWORD\")\n\tcheckAuthErrEvent(\"rip\", \"abc\", \"BAD CREDS\")\n\t// The auth callout uses as the reason the error string, not a closed state.\n\tcheckAuthErrEvent(\"proxy\", \"proxy\", ErrAuthProxyRequired.Error())\n}\n\nfunc TestAuthCalloutConnectEvents(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t\tBAR {}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth, admin ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\t// Allow dlc user and map to the BAZ account.\n\t\tif opts.Username == \"dlc\" && opts.Password == \"zzz\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Username == \"rip\" && opts.Password == \"xxx\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"BAR\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"BAD CREDS\", 0))\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// Setup system user.\n\tsnc := ac.Connect(nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\t// Allow this connect event to pass us by..\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Watch for connect events.\n\tcsub, err := snc.SubscribeSync(fmt.Sprintf(connectEventSubj, \"*\"))\n\trequire_NoError(t, err)\n\n\t// Watch for disconnect events.\n\tdsub, err := snc.SubscribeSync(fmt.Sprintf(disconnectEventSubj, \"*\"))\n\trequire_NoError(t, err)\n\n\t// Connections updates. Old\n\tacOldSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjOld, \"*\"))\n\trequire_NoError(t, err)\n\n\t// Connections updates. New\n\tacNewSub, err := snc.SubscribeSync(fmt.Sprintf(accConnsEventSubjNew, \"*\"))\n\trequire_NoError(t, err)\n\n\tsnc.Flush()\n\n\tcheckConnectEvents := func(user, pass, acc string) {\n\t\tnc := ac.Connect(nats.UserInfo(user, pass))\n\t\trequire_NoError(t, err)\n\n\t\tm, err := csub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar cm ConnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &cm)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, cm.Client.User == user)\n\t\trequire_True(t, cm.Client.Account == acc)\n\n\t\t// Check that we have updates, 1 each, for the connections updates.\n\t\tm, err = acOldSub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar anc AccountNumConns\n\t\terr = json.Unmarshal(m.Data, &anc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, anc.AccountStat.Account == acc)\n\t\trequire_True(t, anc.AccountStat.Conns == 1)\n\n\t\tm, err = acNewSub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\terr = json.Unmarshal(m.Data, &anc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, anc.AccountStat.Account == acc)\n\t\trequire_True(t, anc.AccountStat.Conns == 1)\n\n\t\t// Force the disconnect.\n\t\tnc.Close()\n\n\t\tm, err = dsub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\tm, err = acOldSub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(m.Data, &anc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, anc.AccountStat.Account == acc)\n\t\trequire_True(t, anc.AccountStat.Conns == 0)\n\n\t\tm, err = acNewSub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(m.Data, &anc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, anc.AccountStat.Account == acc)\n\t\trequire_True(t, anc.AccountStat.Conns == 0)\n\n\t\t// Make sure no double events sent.\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\tcheckSubsPending(t, csub, 0)\n\t\tcheckSubsPending(t, dsub, 0)\n\t\tcheckSubsPending(t, acOldSub, 0)\n\t\tcheckSubsPending(t, acNewSub, 0)\n\t}\n\n\tcheckConnectEvents(\"dlc\", \"zzz\", \"FOO\")\n\tcheckConnectEvents(\"rip\", \"xxx\", \"BAR\")\n}\n\nfunc TestAuthCalloutBadServer(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, _, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tskp, err := nkeys.CreateServer()\n\t\trequire_NoError(t, err)\n\t\tspk, err := skp.PublicKey()\n\t\trequire_NoError(t, err)\n\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\tm.Respond(serviceResponse(t, user, spk, ujwt, \"\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// This is where the event fires, in this account.\n\tsub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\tcheckAuthErrEvent := func(user, pass, reason string) {\n\t\tac.RequireConnectError(nats.UserInfo(user, pass))\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\tif !strings.Contains(dm.Reason, reason) {\n\t\t\tt.Fatalf(\"Expected %q reason, but got %q\", reason, dm.Reason)\n\t\t}\n\t}\n\tcheckAuthErrEvent(\"hello\", \"world\", \"response is not for server\")\n}\n\nfunc TestAuthCalloutBadUser(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\t_, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tkp, err := nkeys.CreateUser()\n\t\trequire_NoError(t, err)\n\t\tupk, err := kp.PublicKey()\n\t\trequire_NoError(t, err)\n\t\tujwt := createAuthUser(t, upk, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\tm.Respond(serviceResponse(t, upk, si.ID, ujwt, \"\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// This is where the event fires, in this account.\n\tsub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\tcheckAuthErrEvent := func(user, pass, reason string) {\n\t\tac.RequireConnectError(nats.UserInfo(user, pass))\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\tif !strings.Contains(dm.Reason, reason) {\n\t\t\tt.Fatalf(\"Expected %q reason, but got %q\", reason, dm.Reason)\n\t\t}\n\t}\n\tcheckAuthErrEvent(\"hello\", \"world\", \"auth callout response is not for expected user\")\n}\n\nfunc TestAuthCalloutExpiredUser(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, time.Second*-5, nil)\n\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// This is where the event fires, in this account.\n\tsub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\tcheckAuthErrEvent := func(user, pass, reason string) {\n\t\tac.RequireConnectError(nats.UserInfo(user, pass))\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\tif !strings.Contains(dm.Reason, reason) {\n\t\t\tt.Fatalf(\"Expected %q reason, but got %q\", reason, dm.Reason)\n\t\t}\n\t}\n\tcheckAuthErrEvent(\"hello\", \"world\", \"claim is expired\")\n}\n\nfunc TestAuthCalloutExpiredResponse(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", time.Second*-5))\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer ac.Cleanup()\n\n\t// This is where the event fires, in this account.\n\tsub, err := ac.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\tcheckAuthErrEvent := func(user, pass, reason string) {\n\t\tac.RequireConnectError(nats.UserInfo(user, pass))\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar dm DisconnectEventMsg\n\t\terr = json.Unmarshal(m.Data, &dm)\n\t\trequire_NoError(t, err)\n\n\t\tif !strings.Contains(dm.Reason, reason) {\n\t\t\tt.Fatalf(\"Expected %q reason, but got %q\", reason, dm.Reason)\n\t\t}\n\t}\n\tcheckAuthErrEvent(\"hello\", \"world\", \"claim is expired\")\n}\n\nfunc TestAuthCalloutOperator_AnyAccount(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// A account.\n\takp, apk := createKey(t)\n\taClaim := jwt.NewAccountClaims(apk)\n\taClaim.Name = \"A\"\n\taJwt, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// B account.\n\tbkp, bpk := createKey(t)\n\tbClaim := jwt.NewAccountClaims(bpk)\n\tbClaim.Name = \"B\"\n\tbJwt, err := bClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH callout service account.\n\tckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tcpk, err := ckp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, ckp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(cpk)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, spub, cpk, authJwt, apk, aJwt, bpk, bJwt, spub, sysJwt)\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Token == \"PutMeInA\" {\n\t\t\tujwt := createAuthUser(t, user, \"user_a\", apk, \"\", akp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else if opts.Token == \"PutMeInB\" {\n\t\t\tujwt := createAuthUser(t, user, \"user_b\", bpk, \"\", bkp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\tresp, err := ac.authClient.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\t// Bearer token etc..\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createBasicAccountUser(t, ckp)\n\tdefer removeFile(t, creds)\n\n\t// We require a token.\n\tac.RequireConnectError(nats.UserCredentials(creds))\n\n\t// Send correct token. This should switch us to the A account.\n\tnc := ac.Connect(nats.UserCredentials(creds), nats.Token(\"PutMeInA\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo := response.Data.(*UserInfo)\n\trequire_Equal(t, userInfo.Account, apk)\n\n\tnc = ac.Connect(nats.UserCredentials(creds), nats.Token(\"PutMeInB\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo = response.Data.(*UserInfo)\n\trequire_Equal(t, userInfo.Account, bpk)\n}\n\nfunc TestAuthCalloutWSClientTLSCerts(t *testing.T) {\n\tconf := `\n\t\tserver_name: T\n\t\tlisten: \"localhost:-1\"\n\n\t\ttls {\n\t\t\tcert_file = \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\tkey_file = \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\tca_file = \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\tverify = true\n\t\t}\n\n\t\twebsocket: {\n\t\t\tlisten: \"localhost:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file = \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\t\tkey_file = \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\t\tca_file = \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\tverify = true\n\t\t\t}\n\t\t}\n\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, ci, _, ctls := decodeAuthRequest(t, m.Data)\n\t\trequire_Equal(t, si.Name, \"T\")\n\t\trequire_Equal(t, ci.Host, \"127.0.0.1\")\n\t\trequire_NotEqual(t, ctls, nil)\n\t\t// Zero since we are verified and will be under verified chains.\n\t\trequire_Equal(t, len(ctls.Certs), 0)\n\t\trequire_Equal(t, len(ctls.VerifiedChains), 1)\n\t\t// Since we have a CA.\n\t\trequire_Equal(t, len(ctls.VerifiedChains[0]), 2)\n\t\tblk, _ := pem.Decode([]byte(ctls.VerifiedChains[0][0]))\n\t\tcert, err := x509.ParseCertificate(blk.Bytes)\n\t\trequire_NoError(t, err)\n\t\tif strings.HasPrefix(cert.Subject.String(), \"CN=example.com\") {\n\t\t\t// Override blank name here, server will substitute.\n\t\t\tujwt := createAuthUser(t, user, \"dlc\", \"FOO\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler,\n\t\tnats.UserInfo(\"auth\", \"pwd\"),\n\t\tnats.ClientCert(\"../test/configs/certs/tlsauth/client2.pem\", \"../test/configs/certs/tlsauth/client2-key.pem\"),\n\t\tnats.RootCAs(\"../test/configs/certs/tlsauth/ca.pem\"))\n\tdefer ac.Cleanup()\n\n\t// Will use client cert to determine user.\n\tnc := ac.WSConnect(\n\t\tnats.ClientCert(\"../test/configs/certs/tlsauth/client2.pem\", \"../test/configs/certs/tlsauth/client2-key.pem\"),\n\t\tnats.RootCAs(\"../test/configs/certs/tlsauth/ca.pem\"),\n\t)\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo := response.Data.(*UserInfo)\n\n\trequire_Equal(t, userInfo.UserID, \"dlc\")\n\trequire_Equal(t, userInfo.Account, \"FOO\")\n}\n\nfunc testConfClientClose(t *testing.T, respondNil bool) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: ZZ\n\t\taccounts {\n AUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tif respondNil {\n\t\t\tm.Respond(nil)\n\t\t} else {\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"not today\", 0))\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\n\t// This one will use callout since not defined in server config.\n\t_, err := at.NewClient(nats.UserInfo(\"a\", \"x\"))\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR))\n}\n\nfunc TestAuthCallout_ClientAuthErrorConf(t *testing.T) {\n\ttestConfClientClose(t, true)\n\ttestConfClientClose(t, false)\n}\n\nfunc testAuthCall_ClientAuthErrorOperatorMode(t *testing.T, respondNil bool) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH service account.\n\takp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tapub, err := akp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(apub)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\n\t// the scope for the bearer token which has no permissions\n\tsentinelScope, authKP := newScopedRole(t, \"sentinel\", nil, nil, false)\n\tsentinelScope.Template.Sub.Deny.Add(\">\")\n\tsentinelScope.Template.Pub.Deny.Add(\">\")\n\tsentinelScope.Template.Limits.Subs = 0\n\tsentinelScope.Template.Payload = 0\n\tauthClaim.SigningKeys.AddScopedSigner(sentinelScope)\n\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n listen: 127.0.0.1:-1\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: {\n %s: %s\n %s: %s\n }\n `, ojwt, spub, apub, authJwt, spub, sysJwt)\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, _, _ := decodeAuthRequest(t, m.Data)\n\t\tif respondNil {\n\t\t\tm.Respond(nil)\n\t\t} else {\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, \"\", \"not today\", 0))\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\n\t// Bearer token - this has no permissions see sentinelScope\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createScopedUser(t, akp, authKP)\n\tdefer removeFile(t, creds)\n\n\t// Send the signing key token. This should switch us to the test account, but the user\n\t// is signed with the account signing key\n\t_, err = ac.NewClient(nats.UserCredentials(creds))\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(strings.ToLower(err.Error()), nats.AUTHORIZATION_ERR))\n}\n\nfunc TestAuthCallout_ClientAuthErrorOperatorMode(t *testing.T) {\n\ttestAuthCall_ClientAuthErrorOperatorMode(t, true)\n\ttestAuthCall_ClientAuthErrorOperatorMode(t, false)\n}\n\nfunc TestOperatorModeUserRevocation(t *testing.T) {\n\tskp, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// TEST account.\n\ttkp, tpub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(tpub)\n\taccClaim.Name = \"TEST\"\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH service account.\n\takp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tapub, err := akp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(apub)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(tpub)\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tjwtDir, err := os.MkdirTemp(\"\", \"\")\n\trequire_NoError(t, err)\n\tdefer func() {\n\t\t_ = os.RemoveAll(jwtDir)\n\t}()\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: \"full\"\n\t\t\tdir: %s\n\t\t\tallow_delete: false\n\t\t\tinterval: \"2m\"\n\t\t\ttimeout: \"1.9s\"\n\t\t}\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, spub, jwtDir, apub, authJwt, tpub, accJwt, spub, sysJwt)\n\n\tconst token = \"--secret--\"\n\n\tusers := make(map[string]string)\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Token == token {\n\t\t\t// must have no limits set\n\t\t\tujwt := createAuthUser(t, user, \"user\", tpub, tpub, tkp, 0, &jwt.UserPermissionLimits{})\n\t\t\tusers[opts.Name] = ujwt\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tac := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer ac.Cleanup()\n\n\t// create a system user\n\t_, sysCreds := createAuthServiceUser(t, skp)\n\tdefer removeFile(t, sysCreds)\n\t// connect the system user\n\tsysNC, err := ac.NewClient(nats.UserCredentials(sysCreds))\n\trequire_NoError(t, err)\n\tdefer sysNC.Close()\n\n\t// Bearer token etc..\n\t// This is used by all users, and the customization will be in other connect args.\n\t// This needs to also be bound to the authorization account.\n\tcreds = createBasicAccountUser(t, akp)\n\tdefer removeFile(t, creds)\n\n\tvar fwg sync.WaitGroup\n\n\t// connect three clients\n\tnc := ac.Connect(nats.UserCredentials(creds), nats.Name(\"first\"), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif err != nil && strings.Contains(err.Error(), \"authentication revoked\") {\n\t\t\tfwg.Done()\n\t\t}\n\t}))\n\tfwg.Add(1)\n\n\tvar swg sync.WaitGroup\n\t// connect another user\n\tncA := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.Name(\"second\"), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif err != nil && strings.Contains(err.Error(), \"authentication revoked\") {\n\t\t\tswg.Done()\n\t\t}\n\t}))\n\tswg.Add(1)\n\n\tncB := ac.Connect(nats.UserCredentials(creds), nats.Token(token), nats.NoReconnect(), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif err != nil && strings.Contains(err.Error(), \"authentication revoked\") {\n\t\t\tswg.Done()\n\t\t}\n\t}))\n\tswg.Add(1)\n\n\trequire_NoError(t, err)\n\n\t// revoke the user first - look at the JWT we issued\n\tuc, err := jwt.DecodeUserClaims(users[\"first\"])\n\trequire_NoError(t, err)\n\t// revoke the user in account\n\taccClaim.Revocations = make(map[string]int64)\n\taccClaim.Revocations.Revoke(uc.Subject, time.Now().Add(time.Minute))\n\taccJwt, err = accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\t// send the update request\n\tupdateAccount(t, sysNC, accJwt)\n\n\t// wait for the user to be disconnected with the error we expect\n\tfwg.Wait()\n\trequire_Equal(t, nc.IsConnected(), false)\n\n\t// update the account to remove any revocations\n\taccClaim.Revocations = make(map[string]int64)\n\taccJwt, err = accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\tupdateAccount(t, sysNC, accJwt)\n\t// we should still be connected on the other 2 clients\n\trequire_Equal(t, ncA.IsConnected(), true)\n\trequire_Equal(t, ncB.IsConnected(), true)\n\n\t// update the jwt and revoke all users\n\taccClaim.Revocations.Revoke(jwt.All, time.Now().Add(time.Minute))\n\taccJwt, err = accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\tupdateAccount(t, sysNC, accJwt)\n\n\tswg.Wait()\n\trequire_Equal(t, ncA.IsConnected(), false)\n\trequire_Equal(t, ncB.IsConnected(), false)\n}\n\nfunc updateAccount(t *testing.T, sys *nats.Conn, jwtToken string) {\n\tac, err := jwt.DecodeAccountClaims(jwtToken)\n\trequire_NoError(t, err)\n\tr, err := sys.Request(fmt.Sprintf(`$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE`, ac.Subject), []byte(jwtToken), time.Second*2)\n\trequire_NoError(t, err)\n\n\ttype data struct {\n\t\tAccount string `json:\"account\"`\n\t\tCode int `json:\"code\"`\n\t}\n\ttype serverResponse struct {\n\t\tData data `json:\"data\"`\n\t}\n\n\tvar response serverResponse\n\terr = json.Unmarshal(r.Data, &response)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, response.Data)\n\trequire_Equal(t, response.Data.Code, int(200))\n}\n\nfunc TestAuthCalloutLeafNodeAndOperatorMode(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// A account.\n\takp, apk := createKey(t)\n\taClaim := jwt.NewAccountClaims(apk)\n\taClaim.Name = \"A\"\n\taJwt, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH callout service account.\n\tckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\n\tcpk, err := ckp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, ckp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(cpk)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n `, ojwt, spub, cpk, authJwt, apk, aJwt, spub, sysJwt)\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif (opts.Username == \"leaf\" && opts.Password == \"pwd\") || (opts.Token == \"token\") {\n\t\t\tujwt := createAuthUser(t, user, \"user_a\", apk, \"\", akp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer at.Cleanup()\n\n\tucreds := createBasicAccountUser(t, ckp)\n\tdefer removeFile(t, ucreds)\n\n\t// This should switch us to the A account.\n\tnc := at.Connect(nats.UserCredentials(ucreds), nats.Token(\"token\"))\n\tdefer nc.Close()\n\n\tnatsSub(t, nc, \"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"here\"))\n\t})\n\tnatsFlush(t, nc)\n\n\t// Create creds for the leaf account.\n\tlcreds := createBasicAccountLeaf(t, ckp)\n\tdefer removeFile(t, lcreds)\n\n\thopts := at.srv.getOpts()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tup string\n\t\tok bool\n\t}{\n\t\t{\"bad token\", \"tokenx\", false},\n\t\t{\"bad username and password\", \"leaf:pwdx\", false},\n\t\t{\"token\", \"token\", true},\n\t\t{\"username and password\", \"leaf:pwd\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tserver_name: \"LEAF\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://%s@127.0.0.1:%d\"\n\t\t\t\t\t\t\tcredentials: \"%s\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`, test.up, hopts.LeafNode.Port, lcreds)))\n\t\t\tleaf, _ := RunServerWithConfig(lconf)\n\t\t\tdefer leaf.Shutdown()\n\n\t\t\tif !test.ok {\n\t\t\t\t// Expect failure to connect. Wait a bit before checking.\n\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tcheckLeafNodeConnected(t, leaf)\n\n\t\t\tcheckSubInterest(t, leaf, globalAccountName, \"foo\", time.Second)\n\n\t\t\tncl := natsConnect(t, leaf.ClientURL())\n\t\t\tdefer ncl.Close()\n\n\t\t\tresp, err := ncl.Request(\"foo\", []byte(\"hello\"), time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, string(resp.Data), \"here\")\n\t\t})\n\t}\n}\n\nfunc TestAuthCalloutLeafNodeAndConfigMode(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tAUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tA {}\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\t# Needs to be a public account nkey, will work for both server config and operator mode.\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif (opts.Username == \"leaf\" && opts.Password == \"pwd\") || (opts.Token == \"token\") {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"A\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\n\t// This should switch us to the A account.\n\tnc := at.Connect(nats.Token(\"token\"))\n\tdefer nc.Close()\n\n\tnatsSub(t, nc, \"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"here\"))\n\t})\n\tnatsFlush(t, nc)\n\n\thopts := at.srv.getOpts()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tup string\n\t\tok bool\n\t}{\n\t\t{\"bad token\", \"tokenx\", false},\n\t\t{\"bad username and password\", \"leaf:pwdx\", false},\n\t\t{\"token\", \"token\", true},\n\t\t{\"username and password\", \"leaf:pwd\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tserver_name: \"LEAF\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [{url: \"nats://%s@127.0.0.1:%d\"}]\n\t\t\t\t}\n\t\t\t`, test.up, hopts.LeafNode.Port)))\n\t\t\tleaf, _ := RunServerWithConfig(lconf)\n\t\t\tdefer leaf.Shutdown()\n\n\t\t\tif !test.ok {\n\t\t\t\t// Expect failure to connect. Wait a bit before checking.\n\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tcheckLeafNodeConnected(t, leaf)\n\n\t\t\tcheckSubInterest(t, leaf, globalAccountName, \"foo\", time.Second)\n\n\t\t\tncl := natsConnect(t, leaf.ClientURL())\n\t\t\tdefer ncl.Close()\n\n\t\t\tresp, err := ncl.Request(\"foo\", []byte(\"hello\"), time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, string(resp.Data), \"here\")\n\t\t})\n\t}\n\n}\n\nfunc TestAuthCalloutProxyRequiredInUserNotInAuthJWT(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: A\n\t\taccounts {\n\t\t\tAUTH: {\n\t\t\t\tusers: [ { user: auth, password: auth } ]\n\t\t\t}\n\t\t\tAPP: {\n\t\t\t\tusers: [\n\t\t\t\t\t{ user: user, password: pwd }\n\t\t\t\t\t{ user: proxy, password: pwd, proxy_required: true }\n\t\t\t\t]\n\t\t\t}\n\t\t\tSYS: {}\n\t\t}\n\t\tsystem_account: SYS\n\n\t\tauthorization {\n\t\t\ttimeout: 1s\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\tauth_users: [ auth ]\n\t\t\t\taccount: AUTH\n\t\t\t}\n\t\t}\n\t`\n\tvar invoked atomic.Int32\n\thandler := func(m *nats.Msg) {\n\t\tinvoked.Add(1)\n\t\tuser, si, ci, opts, _ := decodeAuthRequest(t, m.Data)\n\t\trequire_True(t, si.Name == \"A\")\n\t\trequire_True(t, ci.Host == \"127.0.0.1\")\n\t\tif opts.Username == \"proxy\" {\n\t\t\t// If we don't set a ProxyRequired property explicitly here, but the\n\t\t\t// user has it, so it should still be rejected.\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"APP\", _EMPTY_, nil, 10*time.Minute, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, _EMPTY_, 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"auth\"))\n\tdefer at.Cleanup()\n\n\tsub, err := at.authClient.SubscribeSync(authErrorAccountEventSubj)\n\trequire_NoError(t, err)\n\n\t// It should fail, even if the auth callout does not require proxy in its JWT.\n\t// In other words, we reject if not proxied and the user config or the auth JWT\n\t// requires proxy connection.\n\tat.RequireConnectError(nats.UserInfo(\"proxy\", \"pwd\"))\n\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tvar dm DisconnectEventMsg\n\terr = json.Unmarshal(m.Data, &dm)\n\trequire_NoError(t, err)\n\n\t// Convert both reasons to lower case to do the comparison.\n\tdmr := strings.ToLower(dm.Reason)\n\tr := strings.ToLower(ErrAuthProxyRequired.Error())\n\tif !strings.Contains(dmr, r) {\n\t\tt.Fatalf(\"Expected %q reason, but got %q\", r, dmr)\n\t}\n\n\t// Auth callout should have been invoked once.\n\trequire_Equal(t, 1, int(invoked.Load()))\n}\n\nfunc TestAuthCalloutOperatorModeMismatchedCalloutCreds(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH callout service account.\n\tckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\tcpk, err := ckp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, _ := createAuthServiceUser(t, ckp)\n\n\tauthClaim := jwt.NewAccountClaims(cpk)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, spub, cpk, authJwt, spub, sysJwt)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create mismatched creds for the auth service user:\n\t// JWT with the auth user's public key, but a different seed.\n\tukpBad, _ := nkeys.CreateUser()\n\tseedBad, _ := ukpBad.Seed()\n\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"auth-service\"\n\tuclaim.Subject = upub // auth user's public key\n\tujwt, err := uclaim.Encode(ckp)\n\trequire_NoError(t, err)\n\tbadCreds := genCredsFile(t, ujwt, seedBad)\n\tdefer removeFile(t, badCreds)\n\n\t// Client auth will time out because the callout service didn't connect.\n\t_, err = nats.Connect(s.ClientURL(), nats.UserCredentials(badCreds), nats.MaxReconnects(0))\n\trequire_Error(t, err)\n\n\t// Server should still be running.\n\ttime.Sleep(500 * time.Millisecond)\n\trequire_True(t, s.Running())\n}\n\nfunc TestAuthCalloutLeafNodeOperatorModeMismatchedCreds(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// A account.\n\takp, apk := createKey(t)\n\taClaim := jwt.NewAccountClaims(apk)\n\taClaim.Name = \"A\"\n\taJwt, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// AUTH callout service account.\n\tckp, err := nkeys.FromSeed([]byte(authCalloutIssuerSeed))\n\trequire_NoError(t, err)\n\tcpk, err := ckp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// The authorized user for the service.\n\tupub, creds := createAuthServiceUser(t, ckp)\n\tdefer removeFile(t, creds)\n\n\tauthClaim := jwt.NewAccountClaims(cpk)\n\tauthClaim.Name = \"AUTH\"\n\tauthClaim.EnableExternalAuthorization(upub)\n\tauthClaim.Authorization.AllowedAccounts.Add(\"*\")\n\tauthJwt, err := authClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n `, ojwt, spub, cpk, authJwt, apk, aJwt, spub, sysJwt)\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Token == \"token\" {\n\t\t\tujwt := createAuthUser(t, user, \"user_a\", apk, \"\", akp, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserCredentials(creds))\n\tdefer at.Cleanup()\n\n\t// Good leaf node connection.\n\tlcreds := createBasicAccountLeaf(t, ckp)\n\tdefer removeFile(t, lcreds)\n\n\thopts := at.srv.getOpts()\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"LEAF\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://token@127.0.0.1:%d\"\n\t\t\t\t\tcredentials: \"%s\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hopts.LeafNode.Port, lcreds)))\n\tleaf, _ := RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\tcheckLeafNodeConnected(t, leaf)\n\n\t// Now create a leaf with mismatched JWT/seed - JWT from one user,\n\t// seed from another. This should not crash the server.\n\tukp1, _ := nkeys.CreateUser()\n\tseed1, _ := ukp1.Seed()\n\n\tukp2, _ := nkeys.CreateUser()\n\tupub2, _ := ukp2.PublicKey()\n\n\t// Create JWT for user2 but pair with seed from user1.\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub2\n\tuclaim.Name = \"mismatched-leaf\"\n\tujwt, err := uclaim.Encode(ckp)\n\trequire_NoError(t, err)\n\tbadCreds := genCredsFile(t, ujwt, seed1)\n\tdefer removeFile(t, badCreds)\n\n\tlconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"BAD_LEAF\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://token@127.0.0.1:%d\"\n\t\t\t\t\tcredentials: \"%s\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hopts.LeafNode.Port, badCreds)))\n\tleaf2, _ := RunServerWithConfig(lconf2)\n\tdefer leaf2.Shutdown()\n\n\t// Bad leaf should fail to connect but NOT crash the server.\n\ttime.Sleep(50 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf2, 0)\n\n\t// Verify the hub server is still running and healthy.\n\tcheckLeafNodeConnectedCount(t, at.srv, 1)\n}\n\nfunc TestAuthCalloutRegisterWithAccountAfterClose(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: ZZ\n\t\taccounts {\n\t\t\tAUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO {}\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\t// Track whether the auth callout handler was called and collect the\n\t// client pointer for manual registration later.\n\ttype calloutInfo struct {\n\t\tuser string\n\t\tserverID string\n\t\tprocessed chan struct{}\n\t}\n\tvar ci calloutInfo\n\tci.processed = make(chan struct{})\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Username == \"zombie\" && opts.Password == \"pwd\" {\n\t\t\tci.user = user\n\t\t\tci.serverID = si.ID\n\t\t\tclose(ci.processed)\n\t\t\t// Do NOT respond. Let the auth callout timeout.\n\t\t\t// This simulates the case where the response arrives late.\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\ts := at.srv\n\n\tfooAcc, err := s.lookupAccount(\"FOO\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, fooAcc.NumLocalConnections(), 0)\n\tbaseServerClients := s.NumClients()\n\n\t// Connect a client that will go through auth callout. The handler will\n\t// NOT respond, so the callout will timeout and the client will be closed.\n\tvar connectErr error\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tdefer close(done)\n\t\t_, connectErr = nats.Connect(s.ClientURL(),\n\t\t\tnats.UserInfo(\"zombie\", \"pwd\"),\n\t\t\tnats.Timeout(5*time.Second),\n\t\t\tnats.MaxReconnects(0),\n\t\t)\n\t}()\n\n\t// Wait for the auth handler to be invoked (request received).\n\t<-ci.processed\n\n\t// Wait for auth timeout (1s) + closeConnection to complete.\n\t<-done\n\trequire_Error(t, connectErr)\n\trequire_Equal(t, fooAcc.NumLocalConnections(), 0)\n\trequire_Equal(t, s.NumClients(), baseServerClients)\n\n\t// Manually create a client, like auth callout would, so we have a reference to it and can close it.\n\tglobalAcc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tc := &client{srv: s, kind: CLIENT, acc: globalAcc}\n\tc.initClient()\n\tglobalAcc.addClient(c)\n\tc.setNoReconnect()\n\tc.closeConnection(ClientClosed)\n\n\t// Simulate what processReply does after the auth callout response arrives:\n\t// It calls c.RegisterNkeyUser which calls registerWithAccount(targetAcc).\n\t// registerWithAccount should not add the client as we've closed it above.\n\trequire_Error(t, c.registerWithAccount(fooAcc), ErrConnectionClosed)\n\trequire_Equal(t, fooAcc.NumLocalConnections(), 0)\n}\n\nfunc TestAuthCalloutZombieInflatesAccountConnections(t *testing.T) {\n\tconf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: ZZ\n\t\taccounts {\n\t\t\tAUTH { users [ {user: \"auth\", password: \"pwd\"} ] }\n\t\t\tFOO { limits { max_conn: 5 } }\n\t\t}\n\t\tauthorization {\n\t\t\ttimeout: 1\n\t\t\tauth_callout {\n\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\taccount: AUTH\n\t\t\t\tauth_users: [ auth ]\n\t\t\t}\n\t\t}\n\t`\n\n\thandler := func(m *nats.Msg) {\n\t\tuser, si, _, opts, _ := decodeAuthRequest(t, m.Data)\n\t\tif opts.Username == \"legit\" && opts.Password == \"pwd\" {\n\t\t\tujwt := createAuthUser(t, user, _EMPTY_, \"FOO\", \"\", nil, 0, nil)\n\t\t\tm.Respond(serviceResponse(t, user, si.ID, ujwt, \"\", 0))\n\t\t} else {\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tat := NewAuthTest(t, conf, handler, nats.UserInfo(\"auth\", \"pwd\"))\n\tdefer at.Cleanup()\n\ts := at.srv\n\n\tglobalAcc := s.globalAccount()\n\tfooAcc, err := s.lookupAccount(\"FOO\")\n\trequire_NoError(t, err)\n\n\tfooAcc.mu.RLock()\n\tmconns := fooAcc.mconns\n\tfooAcc.mu.RUnlock()\n\trequire_Equal(t, mconns, 5)\n\n\t// Step 1: Inject zombie connections into FOO account.\n\t// This simulates what happens when processReply calls registerWithAccount\n\t// on a client that has already been through closeConnection.\n\tfor range 3 {\n\t\tc := &client{srv: s, kind: CLIENT, acc: globalAcc}\n\t\tc.initClient()\n\t\tglobalAcc.addClient(c)\n\t\tc.closeConnection(ClientClosed)\n\t\trequire_Error(t, c.registerWithAccount(fooAcc), ErrConnectionClosed)\n\t}\n\trequire_Equal(t, fooAcc.NumLocalConnections(), 0)\n\n\t// Step 2: Connect legitimate clients. FOO has max_connections: 5.\n\t// With 3 zombies, we should only be able to connect 2 real clients\n\t// before hitting the limit, even though there are 0 real connections.\n\tfor range 5 {\n\t\tnc, err := nats.Connect(s.ClientURL(),\n\t\t\tnats.UserInfo(\"legit\", \"pwd\"),\n\t\t\tnats.MaxReconnects(0),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\t//goland:noinspection GoDeferInLoop\n\t\tdefer nc.Close()\n\t}\n\trequire_Equal(t, fooAcc.NumLocalConnections(), 5)\n}\n" }, { "path": "server/auth_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc TestUserCloneNilPermissions(t *testing.T) {\n\tuser := &User{\n\t\tUsername: \"foo\",\n\t\tPassword: \"bar\",\n\t}\n\n\tclone := user.clone()\n\n\tif !reflect.DeepEqual(user, clone) {\n\t\tt.Fatalf(\"Cloned Users are incorrect.\\nexpected: %+v\\ngot: %+v\",\n\t\t\tuser, clone)\n\t}\n\n\tclone.Password = \"baz\"\n\tif reflect.DeepEqual(user, clone) {\n\t\tt.Fatal(\"Expected Users to be different\")\n\t}\n}\n\nfunc TestUserClone(t *testing.T) {\n\tuser := &User{\n\t\tUsername: \"foo\",\n\t\tPassword: \"bar\",\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tAllow: []string{\"foo\"},\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{\n\t\t\t\tAllow: []string{\"bar\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tclone := user.clone()\n\n\tif !reflect.DeepEqual(user, clone) {\n\t\tt.Fatalf(\"Cloned Users are incorrect.\\nexpected: %+v\\ngot: %+v\",\n\t\t\tuser, clone)\n\t}\n\n\tclone.Permissions.Subscribe.Allow = []string{\"baz\"}\n\tif reflect.DeepEqual(user, clone) {\n\t\tt.Fatal(\"Expected Users to be different\")\n\t}\n}\n\nfunc TestUserClonePermissionsNoLists(t *testing.T) {\n\tuser := &User{\n\t\tUsername: \"foo\",\n\t\tPassword: \"bar\",\n\t\tPermissions: &Permissions{},\n\t}\n\n\tclone := user.clone()\n\n\tif clone.Permissions.Publish != nil {\n\t\tt.Fatalf(\"Expected Publish to be nil, got: %v\", clone.Permissions.Publish)\n\t}\n\tif clone.Permissions.Subscribe != nil {\n\t\tt.Fatalf(\"Expected Subscribe to be nil, got: %v\", clone.Permissions.Subscribe)\n\t}\n}\n\nfunc TestUserCloneNoPermissions(t *testing.T) {\n\tuser := &User{\n\t\tUsername: \"foo\",\n\t\tPassword: \"bar\",\n\t}\n\n\tclone := user.clone()\n\n\tif clone.Permissions != nil {\n\t\tt.Fatalf(\"Expected Permissions to be nil, got: %v\", clone.Permissions)\n\t}\n}\n\nfunc TestUserCloneNil(t *testing.T) {\n\tuser := (*User)(nil)\n\tclone := user.clone()\n\tif clone != nil {\n\t\tt.Fatalf(\"Expected nil, got: %+v\", clone)\n\t}\n}\n\nfunc TestUserUnknownAllowedConnectionType(t *testing.T) {\n\to := DefaultOptions()\n\to.Users = []*User{{\n\t\tUsername: \"user\",\n\t\tPassword: \"pwd\",\n\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, \"someNewType\"}),\n\t}}\n\t_, err := NewServer(o)\n\tif err == nil || !strings.Contains(err.Error(), \"connection type\") {\n\t\tt.Fatalf(\"Expected error about unknown connection type, got %v\", err)\n\t}\n\n\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{\"websocket\"})\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ts.mu.Lock()\n\tuser := s.opts.Users[0]\n\ts.mu.Unlock()\n\tfor act := range user.AllowedConnectionTypes {\n\t\tif act != jwt.ConnectionTypeWebsocket {\n\t\t\tt.Fatalf(\"Expected map to have been updated with proper case, got %v\", act)\n\t\t}\n\t}\n\t// Same with NKey user now.\n\to.Users = nil\n\to.Nkeys = []*NkeyUser{{\n\t\tNkey: \"somekey\",\n\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, \"someNewType\"}),\n\t}}\n\t_, err = NewServer(o)\n\tif err == nil || !strings.Contains(err.Error(), \"connection type\") {\n\t\tt.Fatalf(\"Expected error about unknown connection type, got %v\", err)\n\t}\n\to.Nkeys[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{\"websocket\"})\n\ts, err = NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ts.mu.Lock()\n\tnkey := s.opts.Nkeys[0]\n\ts.mu.Unlock()\n\tfor act := range nkey.AllowedConnectionTypes {\n\t\tif act != jwt.ConnectionTypeWebsocket {\n\t\t\tt.Fatalf(\"Expected map to have been updated with proper case, got %v\", act)\n\t\t}\n\t}\n}\n\nfunc TestDNSAltNameMatching(t *testing.T) {\n\tfor idx, test := range []struct {\n\t\taltName string\n\t\turls []string\n\t\tmatch bool\n\t}{\n\t\t{\"foo\", []string{\"FOO\"}, true},\n\t\t{\"foo\", []string{\"..\"}, false},\n\t\t{\"foo\", []string{\".\"}, false},\n\t\t{\"Foo\", []string{\"foO\"}, true},\n\t\t{\"FOO\", []string{\"foo\"}, true},\n\t\t{\"foo1\", []string{\"bar\"}, false},\n\t\t{\"multi\", []string{\"m\", \"mu\", \"mul\", \"multi\"}, true},\n\t\t{\"multi\", []string{\"multi\", \"m\", \"mu\", \"mul\"}, true},\n\t\t{\"foo.bar\", []string{\"foo\", \"foo.bar.bar\", \"foo.baz\"}, false},\n\t\t{\"foo.Bar\", []string{\"foo\", \"bar.foo\", \"Foo.Bar\"}, true},\n\t\t{\"foo.*\", []string{\"foo\", \"bar.foo\", \"Foo.Bar\"}, false}, // only match left most\n\t\t{\"f*.bar\", []string{\"foo\", \"bar.foo\", \"Foo.Bar\"}, false},\n\t\t{\"*.bar\", []string{\"foo.bar\"}, true},\n\t\t{\"*\", []string{\"baz.bar\", \"bar\", \"z.y\"}, true},\n\t\t{\"*\", []string{\"bar\"}, true},\n\t\t{\"*\", []string{\".\"}, false},\n\t\t{\"*\", []string{\"\"}, true},\n\t\t{\"*\", []string{\"*\"}, true},\n\t\t{\"bar.*\", []string{\"bar.*\"}, true},\n\t\t{\"*.Y-X-red-mgmt.default.svc\", []string{\"A.Y-X-red-mgmt.default.svc\"}, true},\n\t\t{\"*.Y-X-green-mgmt.default.svc\", []string{\"A.Y-X-green-mgmt.default.svc\"}, true},\n\t\t{\"*.Y-X-blue-mgmt.default.svc\", []string{\"A.Y-X-blue-mgmt.default.svc\"}, true},\n\t\t{\"Y-X-red-mgmt\", []string{\"Y-X-red-mgmt\"}, true},\n\t\t{\"Y-X-red-mgmt\", []string{\"X-X-red-mgmt\"}, false},\n\t\t{\"Y-X-red-mgmt\", []string{\"Y-X-green-mgmt\"}, false},\n\t\t{\"Y-X-red-mgmt\", []string{\"Y\"}, false},\n\t\t{\"Y-X-red-mgmt\", []string{\"Y-X\"}, false},\n\t\t{\"Y-X-red-mgmt\", []string{\"Y-X-red\"}, false},\n\t\t{\"Y-X-red-mgmt\", []string{\"X-red-mgmt\"}, false},\n\t\t{\"Y-X-green-mgmt\", []string{\"Y-X-green-mgmt\"}, true},\n\t\t{\"Y-X-blue-mgmt\", []string{\"Y-X-blue-mgmt\"}, true},\n\t\t{\"connect.Y.local\", []string{\"connect.Y.local\"}, true},\n\t\t{\"connect.Y.local\", []string{\".Y.local\"}, false},\n\t\t{\"connect.Y.local\", []string{\"..local\"}, false},\n\t\t{\"gcp.Y.local\", []string{\"gcp.Y.local\"}, true},\n\t\t{\"uswest1.gcp.Y.local\", []string{\"uswest1.gcp.Y.local\"}, true},\n\t} {\n\t\turlSet := make([]*url.URL, len(test.urls))\n\t\tfor i, u := range test.urls {\n\t\t\tvar err error\n\t\t\turlSet[i], err = url.Parse(\"nats://\" + u)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\tif dnsAltNameMatches(dnsAltNameLabels(test.altName), urlSet) != test.match {\n\t\t\tt.Fatal(\"Test\", idx, \"Match miss match, expected:\", test.match)\n\t\t}\n\t}\n}\n\nfunc TestProcessUserPermissionsTemplateMalformedOpDoesNotPanic(t *testing.T) {\n\tdefer require_NoPanic(t)\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Permissions.Sub.Deny = jwt.StringList{\"foo.{{x}}\"}\n\n\t_, err := processUserPermissionsTemplate(lim, &jwt.UserClaims{}, &Account{})\n\trequire_Error(t, err)\n}\n\nfunc TestProcessUserPermissionsTemplateUnknownAllowOpDoesNotPanic(t *testing.T) {\n\tdefer require_NoPanic(t)\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Permissions.Pub.Allow = jwt.StringList{\"foo.{{unknown()}}\"}\n\n\t_, err := processUserPermissionsTemplate(lim, &jwt.UserClaims{}, &Account{})\n\trequire_Error(t, err)\n}\n\nfunc TestProcessUserPermissionsTemplateRejectsExcessiveTagExpansions(t *testing.T) {\n\tdefer require_NoPanic(t)\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Permissions.Pub.Allow = jwt.StringList{\"foo.{{tag(a)}}.{{tag(b)}}\"}\n\n\tuc := &jwt.UserClaims{}\n\tfor i := range 128 {\n\t\tuc.Tags = append(uc.Tags, fmt.Sprintf(\"a:v%d\", i))\n\t\tuc.Tags = append(uc.Tags, fmt.Sprintf(\"b:v%d\", i))\n\t}\n\n\t_, err := processUserPermissionsTemplate(lim, uc, &Account{})\n\trequire_Error(t, err)\n}\n\nfunc TestNoAuthUser(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tFOO { users [{user: \"foo\", password: \"pwd1\"}] }\n\t\t\tBAR { users [{user: \"bar\", password: \"pwd2\"}] }\n\t\t}\n\t\tno_auth_user: \"foo\"\n\t`))\n\tdefer os.Remove(conf)\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tusrInfo string\n\t\tok bool\n\t\taccount string\n\t}{\n\t\t{\"valid user/pwd\", \"bar:pwd2@\", true, \"BAR\"},\n\t\t{\"invalid pwd\", \"bar:wrong@\", false, _EMPTY_},\n\t\t{\"some token\", \"sometoken@\", false, _EMPTY_},\n\t\t{\"user used without pwd\", \"bar@\", false, _EMPTY_}, // will be treated as a token\n\t\t{\"user with empty password\", \"bar:@\", false, _EMPTY_},\n\t\t{\"no user\", _EMPTY_, true, \"FOO\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\turl := fmt.Sprintf(\"nats://%s127.0.0.1:%d\", test.usrInfo, o.Port)\n\t\t\tnc, err := nats.Connect(url)\n\t\t\tif err != nil {\n\t\t\t\tif test.ok {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else if !test.ok {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatalf(\"Should have failed, did not\")\n\t\t\t}\n\t\t\tvar accName string\n\t\t\ts.mu.Lock()\n\t\t\tfor _, c := range s.clients {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tif c.acc != nil {\n\t\t\t\t\taccName = c.acc.Name\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ts.mu.Unlock()\n\t\t\tnc.Close()\n\t\t\tcheckClientsCount(t, s, 0)\n\t\t\tif accName != test.account {\n\t\t\t\tt.Fatalf(\"The account should have been %q, got %q\", test.account, accName)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNoAuthUserNkey(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tFOO { users [{user: \"foo\", password: \"pwd1\"}] }\n\t\t\tBAR { users [{nkey: \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\"}] }\n\t\t}\n\t\tno_auth_user: \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\"\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Make sure we connect ok and to the correct account.\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\tuserInfo := response.Data.(*UserInfo)\n\trequire_Equal(t, userInfo.UserID, \"UBO2MQV67TQTVIRV3XFTEZOACM4WLOCMCDMAWN5QVN5PI2N6JHTVDRON\")\n\trequire_Equal(t, userInfo.Account, \"BAR\")\n}\n\nfunc TestUserConnectionDeadline(t *testing.T) {\n\tclientAuth := &DummyAuth{\n\t\tt: t,\n\t\tregister: true,\n\t\tdeadline: time.Now().Add(50 * time.Millisecond),\n\t}\n\n\topts := DefaultOptions()\n\topts.CustomClientAuthentication = clientAuth\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tvar dcerr error\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\n\tnc, err := nats.Connect(\n\t\ts.ClientURL(),\n\t\tnats.UserInfo(\"valid\", _EMPTY_),\n\t\tnats.NoReconnect(),\n\t\tnats.ErrorHandler(func(nc *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tdcerr = err\n\t\t\tcancel()\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected client to connect, got: %s\", err)\n\t}\n\n\t<-ctx.Done()\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif nc.IsConnected() {\n\t\t\treturn fmt.Errorf(\"Expected to be disconnected\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tif dcerr == nil || dcerr.Error() != \"nats: authentication expired\" {\n\t\tt.Fatalf(\"Expected a auth expired error: got: %v\", dcerr)\n\t}\n}\n\nfunc TestNoAuthUserNoConnectProto(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tA { users [{user: \"foo\", password: \"pwd\"}] }\n\t\t}\n\t\tauthorization { timeout: 1 }\n\t\tno_auth_user: \"foo\"\n\t`))\n\tdefer os.Remove(conf)\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tcheckClients := func(n int) {\n\t\tt.Helper()\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tif nc := s.NumClients(); nc != n {\n\t\t\tt.Fatalf(\"Expected %d clients, got %d\", n, nc)\n\t\t}\n\t}\n\n\tconn, err := net.Dial(\"tcp\", net.JoinHostPort(o.Host, fmt.Sprintf(\"%d\", o.Port)))\n\trequire_NoError(t, err)\n\tdefer conn.Close()\n\tcheckClientsCount(t, s, 1)\n\n\t// With no auth user we should not require a CONNECT.\n\t// Make sure we are good on not sending CONN first.\n\t_, err = conn.Write([]byte(\"PUB foo 2\\r\\nok\\r\\n\"))\n\trequire_NoError(t, err)\n\tcheckClients(1)\n\tconn.Close()\n\n\t// Now make sure we still do get timed out though.\n\tconn, err = net.Dial(\"tcp\", net.JoinHostPort(o.Host, fmt.Sprintf(\"%d\", o.Port)))\n\trequire_NoError(t, err)\n\tdefer conn.Close()\n\tcheckClientsCount(t, s, 1)\n\n\ttime.Sleep(1200 * time.Millisecond)\n\tcheckClientsCount(t, s, 0)\n}\n\ntype captureProxyRequiredLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *captureProxyRequiredLogger) Debugf(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, ErrAuthProxyRequired.Error()) {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestAuthProxyRequired(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tauthorization {\n\t\t\tuser: user\n\t\t\tpassword: pwd\n\t\t\tproxy_required: true\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tl := &captureProxyRequiredLogger{ch: make(chan string, 1)}\n\ts.SetLogger(l, true, false)\n\n\t_, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\trequire_True(t, errors.Is(err, nats.ErrAuthorization))\n\n\tcheckLog := func() {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase <-l.ch:\n\t\t\treturn\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Did not get log statement\")\n\t\t}\n\t}\n\tcheckLog()\n\n\tdrainLog := func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-l.ch:\n\t\t\tdefault:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\ts.Shutdown()\n\tdrainLog()\n\n\tnkUsr1, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tnkPub1, err := nkUsr1.PublicKey()\n\trequire_NoError(t, err)\n\n\tnkUsr2, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tnkPub2, err := nkUsr2.PublicKey()\n\trequire_NoError(t, err)\n\n\tconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tauthorization {\n\t\t\tusers: [\n\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t{nkey: \"%s\", proxy_required: true}\n\t\t\t\t{nkey: \"%s\", proxy_required: false}\n\t\t\t]\n\t\t}\n\t`, nkPub1, nkPub2))\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\tcheckClients := func() {\n\t\tt.Helper()\n\t\t// Should connect ok.\n\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"user1\", \"pwd1\"))\n\t\tnc.Close()\n\n\t\t// Should not, since it requires going through proxy.\n\t\t_, err = nats.Connect(s.ClientURL(), nats.UserInfo(\"user2\", \"pwd2\"))\n\t\trequire_True(t, errors.Is(err, nats.ErrAuthorization))\n\t\tcheckLog()\n\n\t\t// Should connect ok.\n\t\tnc = natsConnect(t, s.ClientURL(), nats.UserInfo(\"user3\", \"pwd3\"))\n\t\tnc.Close()\n\n\t\t// Should not, since it requires going through proxy.\n\t\t_, err = nats.Connect(s.ClientURL(), nats.Nkey(nkPub1, func(nonce []byte) ([]byte, error) {\n\t\t\treturn nkUsr1.Sign(nonce)\n\t\t}))\n\t\trequire_True(t, errors.Is(err, nats.ErrAuthorization))\n\t\tcheckLog()\n\n\t\t// Should connect ok.\n\t\tnc = natsConnect(t, s.ClientURL(), nats.Nkey(nkPub2, func(nonce []byte) ([]byte, error) {\n\t\t\treturn nkUsr2.Sign(nonce)\n\t\t}))\n\t\tnc.Close()\n\t}\n\tcheckClients()\n\ts.Shutdown()\n\tdrainLog()\n\n\tconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t\t{nkey: \"%s\", proxy_required: true}\n\t\t\t\t\t{nkey: \"%s\", proxy_required: false}\n\t\t\t\t]\n\t\t\t}\n\t\t\tSYS { users: [ {user:sys, password: pwd} ] }\n\t\t}\n\t\tsystem_account: SYS\n\t`, nkPub1, nkPub2))\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\tsnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"sys\", \"pwd\"))\n\tdefer snc.Close()\n\tsub := natsSubSync(t, snc, \"$SYS.SERVER.*.CLIENT.AUTH.ERR\")\n\tsub2 := natsSubSync(t, snc, \"$SYS.ACCOUNT.A.DISCONNECT\")\n\tnatsFlush(t, snc)\n\n\tcheckClients()\n\n\t// We should get 2 authentication error messages, saying that\n\t// the reason is \"ProxyRequired\".\n\tfor range 2 {\n\t\tvar de DisconnectEventMsg\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\terr = json.Unmarshal(msg.Data, &de)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, de.Reason, ProxyRequired.String())\n\t}\n\n\t// We should get 3 disconnect events since only 3 have connected\n\t// ok and then just closed.\n\tfor range 3 {\n\t\tvar de DisconnectEventMsg\n\t\tmsg := natsNexMsg(t, sub2, time.Second)\n\t\terr = json.Unmarshal(msg.Data, &de)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, de.Reason, ClientClosed.String())\n\t}\n\t// Make sure there is no more.\n\tif msg, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Should have received only 3 disconnect messages, got another: %s\", msg.Data)\n\t}\n\n\ts.Shutdown()\n\tdrainLog()\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tuser: user\n\t\t\t\tpassword: pwd\n\t\t\t\tproxy_required: true\n\t\t\t}\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\tlconf := createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes [ {url: \"nats://user:pwd@127.0.0.1:%d\"} ]\n\t\t}\n\t`, o.LeafNode.Port))\n\tleaf, _ := RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\n\ttime.Sleep(125 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\tcheckLog()\n\tleaf.Shutdown()\n\ts.Shutdown()\n\tdrainLog()\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tauthorization {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\tlconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes [ {url: \"nats://user2:pwd@127.0.0.1:%d\"} ]\n\t\t}\n\t`, o.LeafNode.Port))\n\tleaf, _ = RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\n\ttime.Sleep(125 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\tcheckLog()\n\tleaf.Shutdown()\n\ts.Shutdown()\n\tdrainLog()\n\n\tconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tnkey: %s\n\t\t\t\tproxy_required: true\n\t\t\t}\n\t\t}\n\t`, nkPub1))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\tnkSeed1, err := nkUsr1.Seed()\n\trequire_NoError(t, err)\n\n\tlconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t\tnkey: \"%s\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o.LeafNode.Port, nkSeed1))\n\tleaf, _ = RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\n\ttime.Sleep(125 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\tcheckLog()\n\tleaf.Shutdown()\n\ts.Shutdown()\n\tdrainLog()\n\n\t// Check with operator mode.\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: OP\n\t\toperator = \"../test/configs/nkeys/op.jwt\"\n\t\tresolver = MEMORY\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.SetLogger(l, true, false)\n\n\t_, akp := createAccount(s)\n\tkp, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tpub, err := kp.PublicKey()\n\trequire_NoError(t, err)\n\tnuc := jwt.NewUserClaims(pub)\n\tnuc.ProxyRequired = true\n\tujwt, err := nuc.Encode(akp)\n\trequire_NoError(t, err)\n\n\tlopts := &DefaultTestOptions\n\tu, err := url.Parse(fmt.Sprintf(\"nats://%s:%d\", o.LeafNode.Host, o.LeafNode.Port))\n\trequire_NoError(t, err)\n\tremote := &RemoteLeafOpts{URLs: []*url.URL{u}}\n\tremote.SignatureCB = func(nonce []byte) (string, []byte, error) {\n\t\tsig, err := kp.Sign(nonce)\n\t\treturn ujwt, sig, err\n\t}\n\tlopts.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\tlopts.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\tleaf = RunServer(lopts)\n\tdefer leaf.Shutdown()\n\n\ttime.Sleep(125 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\tcheckLog()\n\tleaf.Shutdown()\n\n\t// Try with an user.\n\tdrainLog()\n\n\t_, err = nats.Connect(s.ClientURL(), createUserCredsEx(t, nuc, akp))\n\trequire_True(t, errors.Is(err, nats.ErrAuthorization))\n\tcheckLog()\n\n\tdrainLog()\n\n\t// Try with creds file.\n\tkp, err = nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tuseed, err := kp.Seed()\n\trequire_NoError(t, err)\n\tpub, err = kp.PublicKey()\n\trequire_NoError(t, err)\n\tnuc = jwt.NewUserClaims(pub)\n\tnuc.ProxyRequired = true\n\tujwt, err = nuc.Encode(akp)\n\trequire_NoError(t, err)\n\tcredsFile := genCredsFile(t, ujwt, useed)\n\n\tlconf = createConfFile(t, fmt.Appendf(nil, `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o.LeafNode.Port, credsFile))\n\tleaf, _ = RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\n\ttime.Sleep(125 * time.Millisecond)\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\tcheckLog()\n\tleaf.Shutdown()\n\n\ts.Shutdown()\n\tdrainLog()\n}\n" }, { "path": "server/avl/norace_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests\n\npackage avl\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"testing\"\n\t\"time\"\n)\n\n// Print Results: go test -v --args --results\nvar printResults = flag.Bool(\"results\", false, \"Enable Results Logging\")\n\n// SequenceSet memory tests vs dmaps.\nfunc TestNoRaceSeqSetSizeComparison(t *testing.T) {\n\t// Create 5M random entries (dupes possible but ok for this test) out of 8M range.\n\tnum := 5_000_000\n\tmax := 7_000_000\n\n\tseqs := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tn := uint64(rand.Int63n(int64(max + 1)))\n\t\tseqs = append(seqs, n)\n\t}\n\n\truntime.GC()\n\t// Disable to get stable results.\n\tgcp := debug.SetGCPercent(-1)\n\tdefer debug.SetGCPercent(gcp)\n\n\tmem := runtime.MemStats{}\n\truntime.ReadMemStats(&mem)\n\tinUseBefore := mem.HeapInuse\n\n\tdmap := make(map[uint64]struct{}, num)\n\tfor _, n := range seqs {\n\t\tdmap[n] = struct{}{}\n\t}\n\truntime.ReadMemStats(&mem)\n\tdmapUse := mem.HeapInuse - inUseBefore\n\tinUseBefore = mem.HeapInuse\n\n\t// Now do SequenceSet on same dataset.\n\tvar sset SequenceSet\n\tfor _, n := range seqs {\n\t\tsset.Insert(n)\n\t}\n\n\truntime.ReadMemStats(&mem)\n\tseqSetUse := mem.HeapInuse - inUseBefore\n\n\tif seqSetUse > 2*1024*1024 {\n\t\tt.Fatalf(\"Expected SequenceSet size to be < 2M, got %v\", friendlyBytes(seqSetUse))\n\t}\n\tif seqSetUse*50 > dmapUse {\n\t\tt.Fatalf(\"Expected SequenceSet to be at least 50x better then dmap approach: %v vs %v\",\n\t\t\tfriendlyBytes(seqSetUse),\n\t\t\tfriendlyBytes(dmapUse),\n\t\t)\n\t}\n}\n\nfunc TestNoRaceSeqSetEncodeLarge(t *testing.T) {\n\tnum := 2_500_000\n\tmax := 5_000_000\n\n\tdmap := make(map[uint64]struct{}, num)\n\tvar ss SequenceSet\n\tfor i := 0; i < num; i++ {\n\t\tn := uint64(rand.Int63n(int64(max + 1)))\n\t\tss.Insert(n)\n\t\tdmap[n] = struct{}{}\n\t}\n\n\t// Disable to get stable results.\n\tgcp := debug.SetGCPercent(-1)\n\tdefer debug.SetGCPercent(gcp)\n\n\t// In general should be about the same, but can see some variability.\n\texpected := time.Millisecond\n\n\tstart := time.Now()\n\tb := ss.Encode(nil)\n\n\tif elapsed := time.Since(start); elapsed > expected {\n\t\tt.Fatalf(\"Expected encode of %d items with encoded size %v to take less than %v, got %v\",\n\t\t\tnum, friendlyBytes(len(b)), expected, elapsed)\n\t} else {\n\t\tlogResults(\"Encode time for %d items was %v, encoded size is %v\\n\", num, elapsed, friendlyBytes(len(b)))\n\t}\n\n\tstart = time.Now()\n\tss2, _, err := Decode(b)\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > expected {\n\t\tt.Fatalf(\"Expected decode to take less than %v, got %v\", expected, elapsed)\n\t} else {\n\t\tlogResults(\"Decode time is %v\\n\", elapsed)\n\t}\n\trequire_True(t, ss.Nodes() == ss2.Nodes())\n\trequire_True(t, ss.Size() == ss2.Size())\n}\n\nfunc TestNoRaceSeqSetRelativeSpeed(t *testing.T) {\n\t// Create 1M random entries (dupes possible but ok for this test) out of 3M range.\n\tnum := 1_000_000\n\tmax := 3_000_000\n\n\tseqs := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tn := uint64(rand.Int63n(int64(max + 1)))\n\t\tseqs = append(seqs, n)\n\t}\n\n\tstart := time.Now()\n\t// Now do SequenceSet on same dataset.\n\tvar sset SequenceSet\n\tfor _, n := range seqs {\n\t\tsset.Insert(n)\n\t}\n\tssInsertElapsed := time.Since(start)\n\tlogResults(\"Inserts SequenceSet: %v for %d items\\n\", ssInsertElapsed, num)\n\n\tstart = time.Now()\n\tfor _, n := range seqs {\n\t\tif ok := sset.Exists(n); !ok {\n\t\t\tt.Fatalf(\"Should exist\")\n\t\t}\n\t}\n\tssLookupElapsed := time.Since(start)\n\tlogResults(\"Lookups: %v\\n\", ssLookupElapsed)\n\n\t// Now do a map.\n\tdmap := make(map[uint64]struct{})\n\tstart = time.Now()\n\tfor _, n := range seqs {\n\t\tdmap[n] = struct{}{}\n\t}\n\tmapInsertElapsed := time.Since(start)\n\tlogResults(\"Inserts Map[uint64]: %v for %d items\\n\", mapInsertElapsed, num)\n\n\tstart = time.Now()\n\tfor _, n := range seqs {\n\t\tif _, ok := dmap[n]; !ok {\n\t\t\tt.Fatalf(\"Should exist\")\n\t\t}\n\t}\n\tmapLookupElapsed := time.Since(start)\n\tlogResults(\"Lookups: %v\\n\", mapLookupElapsed)\n\n\t// In general we are between 1.5 and 1.75 times slower atm then a straight map.\n\t// Let's test an upper bound of 2x for now.\n\tif mapInsertElapsed*2 <= ssInsertElapsed {\n\t\tt.Fatalf(\"Expected SequenceSet insert to be no more than 2x slower (%v vs %v)\", mapInsertElapsed, ssInsertElapsed)\n\t}\n\n\tif mapLookupElapsed*3 <= ssLookupElapsed {\n\t\tt.Fatalf(\"Expected SequenceSet lookups to be no more than 3x slower (%v vs %v)\", mapLookupElapsed, ssLookupElapsed)\n\t}\n}\n\n// friendlyBytes returns a string with the given bytes int64\n// represented as a size, such as 1KB, 10MB, etc...\nfunc friendlyBytes[T int | uint64 | int64](bytes T) string {\n\tfbytes := float64(bytes)\n\tbase := 1024\n\tpre := []string{\"K\", \"M\", \"G\", \"T\", \"P\", \"E\"}\n\tif fbytes < float64(base) {\n\t\treturn fmt.Sprintf(\"%v B\", fbytes)\n\t}\n\texp := int(math.Log(fbytes) / math.Log(float64(base)))\n\tindex := exp - 1\n\treturn fmt.Sprintf(\"%.2f %sB\", fbytes/math.Pow(float64(base), float64(exp)), pre[index])\n}\n\nfunc logResults(format string, args ...any) {\n\tif *printResults {\n\t\tfmt.Printf(format, args...)\n\t}\n}\n" }, { "path": "server/avl/seqset.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage avl\n\nimport (\n\t\"cmp\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"math/bits\"\n\t\"slices\"\n)\n\n// SequenceSet is a memory and encoding optimized set for storing unsigned ints.\n//\n// SequenceSet is ~80-100 times more efficient memory wise than a map[uint64]struct{}.\n// SequenceSet is ~1.75 times slower at inserts than the same map.\n// SequenceSet is not thread safe.\n//\n// We use an AVL tree with nodes that hold bitmasks for set membership.\n//\n// Encoding will convert to a space optimized encoding using bitmasks.\ntype SequenceSet struct {\n\troot *node // root node\n\tsize int // number of items\n\tnodes int // number of nodes\n\t// Having this here vs on the stack in Insert/Delete\n\t// makes a difference in memory usage.\n\tchanged bool\n}\n\n// Insert will insert the sequence into the set.\n// The tree will be balanced inline.\nfunc (ss *SequenceSet) Insert(seq uint64) {\n\tif ss.root = ss.root.insert(seq, &ss.changed, &ss.nodes); ss.changed {\n\t\tss.changed = false\n\t\tss.size++\n\t}\n}\n\n// Exists will return true iff the sequence is a member of this set.\nfunc (ss *SequenceSet) Exists(seq uint64) bool {\n\tfor n := ss.root; n != nil; {\n\t\tif seq < n.base {\n\t\t\tn = n.l\n\t\t\tcontinue\n\t\t} else if seq >= n.base+numEntries {\n\t\t\tn = n.r\n\t\t\tcontinue\n\t\t}\n\t\treturn n.exists(seq)\n\t}\n\treturn false\n}\n\n// SetInitialMin should be used to set the initial minimum sequence when known.\n// This will more effectively utilize space versus self selecting.\n// The set should be empty.\nfunc (ss *SequenceSet) SetInitialMin(min uint64) error {\n\tif !ss.IsEmpty() {\n\t\treturn ErrSetNotEmpty\n\t}\n\tss.root, ss.nodes = &node{base: min, h: 1}, 1\n\treturn nil\n}\n\n// Delete will remove the sequence from the set.\n// Will optionally remove nodes and rebalance.\n// Returns where the sequence was set.\nfunc (ss *SequenceSet) Delete(seq uint64) bool {\n\tif ss == nil || ss.root == nil {\n\t\treturn false\n\t}\n\tss.root = ss.root.delete(seq, &ss.changed, &ss.nodes)\n\tif ss.changed {\n\t\tss.changed = false\n\t\tss.size--\n\t\tif ss.size == 0 {\n\t\t\tss.Empty()\n\t\t}\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Size returns the number of items in the set.\nfunc (ss *SequenceSet) Size() int {\n\treturn ss.size\n}\n\n// Nodes returns the number of nodes in the tree.\nfunc (ss *SequenceSet) Nodes() int {\n\treturn ss.nodes\n}\n\n// Empty will clear all items from a set.\nfunc (ss *SequenceSet) Empty() {\n\tss.root = nil\n\tss.size = 0\n\tss.nodes = 0\n}\n\n// IsEmpty is a fast check of the set being empty.\nfunc (ss *SequenceSet) IsEmpty() bool {\n\tif ss == nil || ss.root == nil {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Range will invoke the given function for each item in the set.\n// They will range over the set in ascending order.\n// If the callback returns false we terminate the iteration.\nfunc (ss *SequenceSet) Range(f func(uint64) bool) {\n\tss.root.iter(f)\n}\n\n// Heights returns the left and right heights of the tree.\nfunc (ss *SequenceSet) Heights() (l, r int) {\n\tif ss.root == nil {\n\t\treturn 0, 0\n\t}\n\tif ss.root.l != nil {\n\t\tl = ss.root.l.h\n\t}\n\tif ss.root.r != nil {\n\t\tr = ss.root.r.h\n\t}\n\treturn l, r\n}\n\n// Returns min, max and number of set items.\nfunc (ss *SequenceSet) State() (min, max, num uint64) {\n\tif ss == nil || ss.root == nil {\n\t\treturn 0, 0, 0\n\t}\n\tmin, max = ss.MinMax()\n\treturn min, max, uint64(ss.Size())\n}\n\n// MinMax will return the minunum and maximum values in the set.\nfunc (ss *SequenceSet) MinMax() (min, max uint64) {\n\tif ss.root == nil {\n\t\treturn 0, 0\n\t}\n\tfor l := ss.root; l != nil; l = l.l {\n\t\tif l.l == nil {\n\t\t\tmin = l.min()\n\t\t}\n\t}\n\tfor r := ss.root; r != nil; r = r.r {\n\t\tif r.r == nil {\n\t\t\tmax = r.max()\n\t\t}\n\t}\n\treturn min, max\n}\n\nfunc clone(src *node, target **node) {\n\tif src == nil {\n\t\treturn\n\t}\n\tn := &node{base: src.base, bits: src.bits, h: src.h}\n\t*target = n\n\tclone(src.l, &n.l)\n\tclone(src.r, &n.r)\n}\n\n// Clone will return a clone of the given SequenceSet.\nfunc (ss *SequenceSet) Clone() *SequenceSet {\n\tif ss == nil {\n\t\treturn nil\n\t}\n\tcss := &SequenceSet{nodes: ss.nodes, size: ss.size}\n\tclone(ss.root, &css.root)\n\n\treturn css\n}\n\n// Union will union this SequenceSet with ssa.\nfunc (ss *SequenceSet) Union(ssa ...*SequenceSet) {\n\tfor _, sa := range ssa {\n\t\tsa.root.nodeIter(func(n *node) {\n\t\t\tfor nb, b := range n.bits {\n\t\t\t\tfor pos := uint64(0); b != 0; pos++ {\n\t\t\t\t\tif b&1 == 1 {\n\t\t\t\t\t\tseq := n.base + (uint64(nb) * uint64(bitsPerBucket)) + pos\n\t\t\t\t\t\tss.Insert(seq)\n\t\t\t\t\t}\n\t\t\t\t\tb >>= 1\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Union will return a union of all sets.\nfunc Union(ssa ...*SequenceSet) *SequenceSet {\n\tif len(ssa) == 0 {\n\t\treturn nil\n\t}\n\t// Sort so we can clone largest.\n\tslices.SortFunc(ssa, func(i, j *SequenceSet) int { return -cmp.Compare(i.Size(), j.Size()) }) // reverse order\n\tss := ssa[0].Clone()\n\n\t// Insert the rest through range call.\n\tfor i := 1; i < len(ssa); i++ {\n\t\tssa[i].Range(func(n uint64) bool {\n\t\t\tss.Insert(n)\n\t\t\treturn true\n\t\t})\n\t}\n\treturn ss\n}\n\nconst (\n\t// Magic is used to identify the encode binary state..\n\tmagic = uint8(22)\n\t// Version\n\tversion = uint8(2)\n\t// hdrLen\n\thdrLen = 2\n\t// minimum length of an encoded SequenceSet.\n\tminLen = 2 + 8 // magic + version + num nodes + num entries.\n)\n\n// EncodeLen returns the bytes needed for encoding.\nfunc (ss SequenceSet) EncodeLen() int {\n\treturn minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2))\n}\n\nfunc (ss SequenceSet) Encode(buf []byte) []byte {\n\tnn, encLen := ss.Nodes(), ss.EncodeLen()\n\n\tif cap(buf) < encLen {\n\t\tbuf = make([]byte, encLen)\n\t} else {\n\t\tbuf = buf[:encLen]\n\t}\n\n\t// TODO(dlc) - Go 1.19 introduced Append to not have to keep track.\n\t// Once 1.20 is out we could change this over.\n\t// Also binary.Write() is way slower, do not use.\n\n\tvar le = binary.LittleEndian\n\tbuf[0], buf[1] = magic, version\n\ti := hdrLen\n\tle.PutUint32(buf[i:], uint32(nn))\n\tle.PutUint32(buf[i+4:], uint32(ss.size))\n\ti += 8\n\tss.root.nodeIter(func(n *node) {\n\t\tle.PutUint64(buf[i:], n.base)\n\t\ti += 8\n\t\tfor _, b := range n.bits {\n\t\t\tle.PutUint64(buf[i:], b)\n\t\t\ti += 8\n\t\t}\n\t\tle.PutUint16(buf[i:], uint16(n.h))\n\t\ti += 2\n\t})\n\treturn buf[:i]\n}\n\n// ErrBadEncoding is returned when we can not decode properly.\nvar (\n\tErrBadEncoding = errors.New(\"ss: bad encoding\")\n\tErrBadVersion = errors.New(\"ss: bad version\")\n\tErrSetNotEmpty = errors.New(\"ss: set not empty\")\n)\n\n// Decode returns the sequence set and number of bytes read from the buffer on success.\nfunc Decode(buf []byte) (*SequenceSet, int, error) {\n\tif len(buf) < minLen || buf[0] != magic {\n\t\treturn nil, -1, ErrBadEncoding\n\t}\n\n\tswitch v := buf[1]; v {\n\tcase 1:\n\t\treturn decodev1(buf)\n\tcase 2:\n\t\treturn decodev2(buf)\n\tdefault:\n\t\treturn nil, -1, ErrBadVersion\n\t}\n}\n\n// Helper to decode v2.\nfunc decodev2(buf []byte) (*SequenceSet, int, error) {\n\tvar le = binary.LittleEndian\n\tindex := 2\n\tnn := int(le.Uint32(buf[index:]))\n\tsz := int(le.Uint32(buf[index+4:]))\n\tindex += 8\n\n\texpectedLen := minLen + (nn * ((numBuckets+1)*8 + 2))\n\tif len(buf) < expectedLen {\n\t\treturn nil, -1, ErrBadEncoding\n\t}\n\n\tss, nodes := SequenceSet{size: sz}, make([]node, nn)\n\n\tfor i := 0; i < nn; i++ {\n\t\tn := &nodes[i]\n\t\tn.base = le.Uint64(buf[index:])\n\t\tindex += 8\n\t\tfor bi := range n.bits {\n\t\t\tn.bits[bi] = le.Uint64(buf[index:])\n\t\t\tindex += 8\n\t\t}\n\t\tn.h = int(le.Uint16(buf[index:]))\n\t\tindex += 2\n\t\tss.insertNode(n)\n\t}\n\n\treturn &ss, index, nil\n}\n\n// Helper to decode v1 into v2 which has fixed buckets of 32 vs 64 originally.\nfunc decodev1(buf []byte) (*SequenceSet, int, error) {\n\tvar le = binary.LittleEndian\n\tindex := 2\n\tnn := int(le.Uint32(buf[index:]))\n\tsz := int(le.Uint32(buf[index+4:]))\n\tindex += 8\n\n\tconst v1NumBuckets = 64\n\n\texpectedLen := minLen + (nn * ((v1NumBuckets+1)*8 + 2))\n\tif len(buf) < expectedLen {\n\t\treturn nil, -1, ErrBadEncoding\n\t}\n\n\tvar ss SequenceSet\n\tfor i := 0; i < nn; i++ {\n\t\tbase := le.Uint64(buf[index:])\n\t\tindex += 8\n\t\tfor nb := uint64(0); nb < v1NumBuckets; nb++ {\n\t\t\tn := le.Uint64(buf[index:])\n\t\t\t// Walk all set bits and insert sequences manually for this decode from v1.\n\t\t\tfor pos := uint64(0); n != 0; pos++ {\n\t\t\t\tif n&1 == 1 {\n\t\t\t\t\tseq := base + (nb * uint64(bitsPerBucket)) + pos\n\t\t\t\t\tss.Insert(seq)\n\t\t\t\t}\n\t\t\t\tn >>= 1\n\t\t\t}\n\t\t\tindex += 8\n\t\t}\n\t\t// Skip over encoded height.\n\t\tindex += 2\n\t}\n\n\t// Sanity check.\n\tif ss.Size() != sz {\n\t\treturn nil, -1, ErrBadEncoding\n\t}\n\n\treturn &ss, index, nil\n\n}\n\n// insertNode places a decoded node into the tree.\n// These should be done in tree order as defined by Encode()\n// This allows us to not have to calculate height or do rebalancing.\n// So much better performance this way.\nfunc (ss *SequenceSet) insertNode(n *node) {\n\tss.nodes++\n\n\tif ss.root == nil {\n\t\tss.root = n\n\t\treturn\n\t}\n\t// Walk our way to the insertion point.\n\tfor p := ss.root; p != nil; {\n\t\tif n.base < p.base {\n\t\t\tif p.l == nil {\n\t\t\t\tp.l = n\n\t\t\t\treturn\n\t\t\t}\n\t\t\tp = p.l\n\t\t} else {\n\t\t\tif p.r == nil {\n\t\t\t\tp.r = n\n\t\t\t\treturn\n\t\t\t}\n\t\t\tp = p.r\n\t\t}\n\t}\n}\n\nconst (\n\tbitsPerBucket = 64 // bits in uint64\n\tnumBuckets = 32\n\tnumEntries = numBuckets * bitsPerBucket\n)\n\ntype node struct {\n\t//v dvalue\n\tbase uint64\n\tbits [numBuckets]uint64\n\tl *node\n\tr *node\n\th int\n}\n\n// Set the proper bit.\n// seq should have already been qualified and inserted should be non nil.\nfunc (n *node) set(seq uint64, inserted *bool) {\n\tseq -= n.base\n\ti := seq / bitsPerBucket\n\tmask := uint64(1) << (seq % bitsPerBucket)\n\tif (n.bits[i] & mask) == 0 {\n\t\tn.bits[i] |= mask\n\t\t*inserted = true\n\t}\n}\n\nfunc (n *node) insert(seq uint64, inserted *bool, nodes *int) *node {\n\tif n == nil {\n\t\tbase := (seq / numEntries) * numEntries\n\t\tn := &node{base: base, h: 1}\n\t\tn.set(seq, inserted)\n\t\t*nodes++\n\t\treturn n\n\t}\n\n\tif seq < n.base {\n\t\tn.l = n.l.insert(seq, inserted, nodes)\n\t} else if seq >= n.base+numEntries {\n\t\tn.r = n.r.insert(seq, inserted, nodes)\n\t} else {\n\t\tn.set(seq, inserted)\n\t}\n\n\tn.h = maxH(n) + 1\n\n\t// Don't make a function, impacts performance.\n\tif bf := balanceF(n); bf > 1 {\n\t\t// Left unbalanced.\n\t\tif balanceF(n.l) < 0 {\n\t\t\tn.l = n.l.rotateL()\n\t\t}\n\t\treturn n.rotateR()\n\t} else if bf < -1 {\n\t\t// Right unbalanced.\n\t\tif balanceF(n.r) > 0 {\n\t\t\tn.r = n.r.rotateR()\n\t\t}\n\t\treturn n.rotateL()\n\t}\n\treturn n\n}\n\nfunc (n *node) rotateL() *node {\n\tr := n.r\n\tif r != nil {\n\t\tn.r = r.l\n\t\tr.l = n\n\t\tn.h = maxH(n) + 1\n\t\tr.h = maxH(r) + 1\n\t} else {\n\t\tn.r = nil\n\t\tn.h = maxH(n) + 1\n\t}\n\treturn r\n}\n\nfunc (n *node) rotateR() *node {\n\tl := n.l\n\tif l != nil {\n\t\tn.l = l.r\n\t\tl.r = n\n\t\tn.h = maxH(n) + 1\n\t\tl.h = maxH(l) + 1\n\t} else {\n\t\tn.l = nil\n\t\tn.h = maxH(n) + 1\n\t}\n\treturn l\n}\n\nfunc balanceF(n *node) int {\n\tif n == nil {\n\t\treturn 0\n\t}\n\tvar lh, rh int\n\tif n.l != nil {\n\t\tlh = n.l.h\n\t}\n\tif n.r != nil {\n\t\trh = n.r.h\n\t}\n\treturn lh - rh\n}\n\nfunc maxH(n *node) int {\n\tif n == nil {\n\t\treturn 0\n\t}\n\tvar lh, rh int\n\tif n.l != nil {\n\t\tlh = n.l.h\n\t}\n\tif n.r != nil {\n\t\trh = n.r.h\n\t}\n\tif lh > rh {\n\t\treturn lh\n\t}\n\treturn rh\n}\n\n// Clear the proper bit.\n// seq should have already been qualified and deleted should be non nil.\n// Will return true if this node is now empty.\nfunc (n *node) clear(seq uint64, deleted *bool) bool {\n\tseq -= n.base\n\ti := seq / bitsPerBucket\n\tmask := uint64(1) << (seq % bitsPerBucket)\n\tif (n.bits[i] & mask) != 0 {\n\t\tn.bits[i] &^= mask\n\t\t*deleted = true\n\t}\n\tfor _, b := range n.bits {\n\t\tif b != 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (n *node) delete(seq uint64, deleted *bool, nodes *int) *node {\n\tif n == nil {\n\t\treturn nil\n\t}\n\n\tif seq < n.base {\n\t\tn.l = n.l.delete(seq, deleted, nodes)\n\t} else if seq >= n.base+numEntries {\n\t\tn.r = n.r.delete(seq, deleted, nodes)\n\t} else if empty := n.clear(seq, deleted); empty {\n\t\t*nodes--\n\t\tif n.l == nil {\n\t\t\tn = n.r\n\t\t} else if n.r == nil {\n\t\t\tn = n.l\n\t\t} else {\n\t\t\t// We have both children.\n\t\t\tn.r = n.r.insertNodePrev(n.l)\n\t\t\tn = n.r\n\t\t}\n\t}\n\n\tif n != nil {\n\t\tn.h = maxH(n) + 1\n\t}\n\n\t// Check balance.\n\tif bf := balanceF(n); bf > 1 {\n\t\t// Left unbalanced.\n\t\tif balanceF(n.l) < 0 {\n\t\t\tn.l = n.l.rotateL()\n\t\t}\n\t\treturn n.rotateR()\n\t} else if bf < -1 {\n\t\t// right unbalanced.\n\t\tif balanceF(n.r) > 0 {\n\t\t\tn.r = n.r.rotateR()\n\t\t}\n\t\treturn n.rotateL()\n\t}\n\n\treturn n\n}\n\n// Will insert nn into the node assuming it is less than all other nodes in n.\n// Will re-calculate height and balance.\nfunc (n *node) insertNodePrev(nn *node) *node {\n\tif n.l == nil {\n\t\tn.l = nn\n\t} else {\n\t\tn.l = n.l.insertNodePrev(nn)\n\t}\n\tn.h = maxH(n) + 1\n\n\t// Check balance.\n\tif bf := balanceF(n); bf > 1 {\n\t\t// Left unbalanced.\n\t\tif balanceF(n.l) < 0 {\n\t\t\tn.l = n.l.rotateL()\n\t\t}\n\t\treturn n.rotateR()\n\t} else if bf < -1 {\n\t\t// right unbalanced.\n\t\tif balanceF(n.r) > 0 {\n\t\t\tn.r = n.r.rotateR()\n\t\t}\n\t\treturn n.rotateL()\n\t}\n\treturn n\n}\n\nfunc (n *node) exists(seq uint64) bool {\n\tseq -= n.base\n\ti := seq / bitsPerBucket\n\tmask := uint64(1) << (seq % bitsPerBucket)\n\treturn n.bits[i]&mask != 0\n}\n\n// Return minimum sequence in the set.\n// This node can not be empty.\nfunc (n *node) min() uint64 {\n\tfor i, b := range n.bits {\n\t\tif b != 0 {\n\t\t\treturn n.base +\n\t\t\t\tuint64(i*bitsPerBucket) +\n\t\t\t\tuint64(bits.TrailingZeros64(b))\n\t\t}\n\t}\n\treturn 0\n}\n\n// Return maximum sequence in the set.\n// This node can not be empty.\nfunc (n *node) max() uint64 {\n\tfor i := numBuckets - 1; i >= 0; i-- {\n\t\tif b := n.bits[i]; b != 0 {\n\t\t\treturn n.base +\n\t\t\t\tuint64(i*bitsPerBucket) +\n\t\t\t\tuint64(bitsPerBucket-bits.LeadingZeros64(b>>1))\n\t\t}\n\t}\n\treturn 0\n}\n\n// This is done in tree order.\nfunc (n *node) nodeIter(f func(n *node)) {\n\tif n == nil {\n\t\treturn\n\t}\n\tf(n)\n\tn.l.nodeIter(f)\n\tn.r.nodeIter(f)\n}\n\n// iter will iterate through the set's items in this node.\n// If the supplied function returns false we terminate the iteration.\nfunc (n *node) iter(f func(uint64) bool) bool {\n\tif n == nil {\n\t\treturn true\n\t}\n\n\tif ok := n.l.iter(f); !ok {\n\t\treturn false\n\t}\n\tfor num := n.base; num < n.base+numEntries; num++ {\n\t\tif n.exists(num) {\n\t\t\tif ok := f(num); !ok {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\tif ok := n.r.iter(f); !ok {\n\t\treturn false\n\t}\n\n\treturn true\n}\n" }, { "path": "server/avl/seqset_test.go", "content": "// Copyright 2023 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage avl\n\nimport (\n\t\"encoding/base64\"\n\t\"math/rand\"\n\t\"testing\"\n)\n\nfunc TestSeqSetBasics(t *testing.T) {\n\tvar ss SequenceSet\n\n\tseqs := []uint64{22, 222, 2000, 2, 2, 4}\n\tfor _, seq := range seqs {\n\t\tss.Insert(seq)\n\t\trequire_True(t, ss.Exists(seq))\n\t}\n\n\trequire_True(t, ss.Nodes() == 1)\n\trequire_True(t, ss.Size() == len(seqs)-1) // We have one dup in there.\n\tlh, rh := ss.Heights()\n\trequire_True(t, lh == 0)\n\trequire_True(t, rh == 0)\n}\n\nfunc TestSeqSetLeftLean(t *testing.T) {\n\tvar ss SequenceSet\n\n\tfor i := uint64(4 * numEntries); i > 0; i-- {\n\t\tss.Insert(i)\n\t}\n\n\trequire_True(t, ss.Nodes() == 5)\n\trequire_True(t, ss.Size() == 4*numEntries)\n\tlh, rh := ss.Heights()\n\trequire_True(t, lh == 2)\n\trequire_True(t, rh == 1)\n}\n\nfunc TestSeqSetRightLean(t *testing.T) {\n\tvar ss SequenceSet\n\n\tfor i := uint64(0); i < uint64(4*numEntries); i++ {\n\t\tss.Insert(i)\n\t}\n\n\trequire_True(t, ss.Nodes() == 4)\n\trequire_True(t, ss.Size() == 4*numEntries)\n\tlh, rh := ss.Heights()\n\trequire_True(t, lh == 1)\n\trequire_True(t, rh == 2)\n}\n\nfunc TestSeqSetCorrectness(t *testing.T) {\n\t// Generate 100k sequences across 500k range.\n\tnum := 100_000\n\tmax := 500_000\n\n\tset := make(map[uint64]struct{}, num)\n\tvar ss SequenceSet\n\tfor i := 0; i < num; i++ {\n\t\tn := uint64(rand.Int63n(int64(max + 1)))\n\t\tss.Insert(n)\n\t\tset[n] = struct{}{}\n\t}\n\n\tfor i := uint64(0); i <= uint64(max); i++ {\n\t\t_, exists := set[i]\n\t\trequire_True(t, ss.Exists(i) == exists)\n\t}\n}\n\nfunc TestSeqSetRange(t *testing.T) {\n\tnum := 2*numEntries + 22\n\tnums := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tnums = append(nums, uint64(i))\n\t}\n\trand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })\n\n\tvar ss SequenceSet\n\tfor _, n := range nums {\n\t\tss.Insert(n)\n\t}\n\n\tnums = nums[:0]\n\tss.Range(func(n uint64) bool {\n\t\tnums = append(nums, n)\n\t\treturn true\n\t})\n\trequire_True(t, len(nums) == num)\n\tfor i := uint64(0); i < uint64(num); i++ {\n\t\trequire_True(t, nums[i] == i)\n\t}\n\n\t// Test truncating the range call.\n\tnums = nums[:0]\n\tss.Range(func(n uint64) bool {\n\t\tif n >= 10 {\n\t\t\treturn false\n\t\t}\n\t\tnums = append(nums, n)\n\t\treturn true\n\t})\n\trequire_True(t, len(nums) == 10)\n\tfor i := uint64(0); i < 10; i++ {\n\t\trequire_True(t, nums[i] == i)\n\t}\n}\n\nfunc TestSeqSetDelete(t *testing.T) {\n\tvar ss SequenceSet\n\n\t// Simple single node.\n\tseqs := []uint64{22, 222, 2222, 2, 2, 4}\n\tfor _, seq := range seqs {\n\t\tss.Insert(seq)\n\t}\n\n\tfor _, seq := range seqs {\n\t\tss.Delete(seq)\n\t\trequire_True(t, !ss.Exists(seq))\n\t}\n\trequire_True(t, ss.root == nil)\n}\n\nfunc TestSeqSetInsertAndDeletePedantic(t *testing.T) {\n\tvar ss SequenceSet\n\n\tnum := 50*numEntries + 22\n\tnums := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tnums = append(nums, uint64(i))\n\t}\n\trand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })\n\n\t// Make sure always balanced.\n\ttestBalanced := func() {\n\t\tt.Helper()\n\t\t// Check heights.\n\t\tss.root.nodeIter(func(n *node) {\n\t\t\tif n != nil && n.h != maxH(n)+1 {\n\t\t\t\tt.Fatalf(\"Node height is wrong: %+v\", n)\n\t\t\t}\n\t\t})\n\t\t// Check balance factor.\n\t\tif bf := balanceF(ss.root); bf > 1 || bf < -1 {\n\t\t\tt.Fatalf(\"Unbalanced tree\")\n\t\t}\n\t}\n\n\tfor _, n := range nums {\n\t\tss.Insert(n)\n\t\ttestBalanced()\n\t}\n\trequire_True(t, ss.root != nil)\n\n\tfor _, n := range nums {\n\t\tss.Delete(n)\n\t\ttestBalanced()\n\t\trequire_True(t, !ss.Exists(n))\n\t\tif ss.Size() > 0 {\n\t\t\trequire_True(t, ss.root != nil)\n\t\t}\n\t}\n\trequire_True(t, ss.root == nil)\n}\n\nfunc TestSeqSetMinMax(t *testing.T) {\n\tvar ss SequenceSet\n\n\t// Simple single node.\n\tseqs := []uint64{22, 222, 2222, 2, 2, 4}\n\tfor _, seq := range seqs {\n\t\tss.Insert(seq)\n\t}\n\n\tmin, max := ss.MinMax()\n\trequire_True(t, min == 2 && max == 2222)\n\n\t// Multi-node\n\tss.Empty()\n\n\tnum := 22*numEntries + 22\n\tnums := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tnums = append(nums, uint64(i))\n\t}\n\trand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })\n\tfor _, n := range nums {\n\t\tss.Insert(n)\n\t}\n\n\tmin, max = ss.MinMax()\n\trequire_True(t, min == 0 && max == uint64(num-1))\n}\n\nfunc TestSeqSetClone(t *testing.T) {\n\t// Generate 100k sequences across 500k range.\n\tnum := 100_000\n\tmax := 500_000\n\n\tvar ss SequenceSet\n\tfor i := 0; i < num; i++ {\n\t\tss.Insert(uint64(rand.Int63n(int64(max + 1))))\n\t}\n\n\tssc := ss.Clone()\n\trequire_True(t, ss.Size() == ssc.Size())\n\trequire_True(t, ss.Nodes() == ssc.Nodes())\n}\n\nfunc TestSeqSetUnion(t *testing.T) {\n\tvar ss1, ss2 SequenceSet\n\n\tseqs1 := []uint64{22, 222, 2222, 2, 2, 4}\n\tfor _, seq := range seqs1 {\n\t\tss1.Insert(seq)\n\t}\n\n\tseqs2 := []uint64{33, 333, 3333, 3, 33_333, 333_333}\n\tfor _, seq := range seqs2 {\n\t\tss2.Insert(seq)\n\t}\n\n\tss := Union(&ss1, &ss2)\n\trequire_True(t, ss.Size() == 11)\n\n\tseqs := append(seqs1, seqs2...)\n\tfor _, n := range seqs {\n\t\trequire_True(t, ss.Exists(n))\n\t}\n}\n\nfunc TestSeqSetFirst(t *testing.T) {\n\tvar ss SequenceSet\n\n\tseqs := []uint64{22, 222, 2222, 222_222}\n\tfor _, seq := range seqs {\n\t\t// Normal case where we pick first/base.\n\t\tss.Insert(seq)\n\t\trequire_True(t, ss.root.base == (seq/numEntries)*numEntries)\n\t\tss.Empty()\n\t\t// Where we set the minimum start value.\n\t\tss.SetInitialMin(seq)\n\t\tss.Insert(seq)\n\t\trequire_True(t, ss.root.base == seq)\n\t\tss.Empty()\n\t}\n}\n\n// Test that we can union with nodes vs individual sequence insertion.\nfunc TestSeqSetDistinctUnion(t *testing.T) {\n\t// Distinct sets.\n\tvar ss1 SequenceSet\n\tseqs1 := []uint64{1, 10, 100, 200}\n\tfor _, seq := range seqs1 {\n\t\tss1.Insert(seq)\n\t}\n\n\tvar ss2 SequenceSet\n\tseqs2 := []uint64{5000, 6100, 6200, 6222}\n\tfor _, seq := range seqs2 {\n\t\tss2.Insert(seq)\n\t}\n\n\tss := ss1.Clone()\n\tallSeqs := append(seqs1, seqs2...)\n\n\tss.Union(&ss2)\n\trequire_True(t, ss.Size() == len(allSeqs))\n\tfor _, seq := range allSeqs {\n\t\trequire_True(t, ss.Exists(seq))\n\t}\n}\n\nfunc TestSeqSetDecodeV1(t *testing.T) {\n\t// Encoding from v1 which was 64 buckets.\n\tseqs := []uint64{22, 222, 2222, 222_222, 2_222_222}\n\tencStr := `\nFgEDAAAABQAAAABgAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAADgIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAA==\n`\n\n\tenc, err := base64.StdEncoding.DecodeString(encStr)\n\trequire_NoError(t, err)\n\n\tss, _, err := Decode(enc)\n\trequire_NoError(t, err)\n\n\trequire_True(t, ss.Size() == len(seqs))\n\tfor _, seq := range seqs {\n\t\trequire_True(t, ss.Exists(seq))\n\t}\n}\n\nfunc require_NoError(t *testing.T, err error) {\n\tt.Helper()\n\tif err != nil {\n\t\tt.Fatalf(\"require no error, but got: %v\", err)\n\t}\n}\n\nfunc require_True(t *testing.T, b bool) {\n\tt.Helper()\n\tif !b {\n\t\tt.Fatalf(\"require true\")\n\t}\n}\n" }, { "path": "server/benchmark_publish_test.go", "content": "// Copyright 2022 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc BenchmarkPublish(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tseed = 12345\n\t\tminMessages = 10_000\n\t\tsubject = \"S\"\n\t\tqueue = \"Q\"\n\t)\n\n\tconst (\n\t\tKB = 1024\n\t\tMB = KB * KB\n\t)\n\n\ttype SubscriberType string\n\tconst (\n\t\tAsync SubscriberType = \"Async\"\n\t\tQueueAsync SubscriberType = \"AsyncQueue\"\n\t\tNone SubscriberType = \"None\"\n\t)\n\n\tbenchmarksCases := []struct {\n\t\tmessageSize int\n\t}{\n\t\t{0},\n\t\t{1},\n\t\t{32},\n\t\t{128},\n\t\t{512},\n\t\t{4 * KB},\n\t\t{32 * KB},\n\t\t{128 * KB},\n\t\t{512 * KB},\n\t\t{1 * MB},\n\t}\n\n\t// All the cases above are run for each of the subscriber cases below\n\tsubscribersCases := []struct {\n\t\tnumSubs int\n\t\tsubType SubscriberType\n\t}{\n\t\t{0, None},\n\t\t{1, Async},\n\t\t{1, QueueAsync},\n\t\t{10, Async},\n\t\t{10, QueueAsync},\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\t\tbcName := fmt.Sprintf(\n\t\t\t\"MsgSz=%db\",\n\t\t\tbc.messageSize,\n\t\t)\n\n\t\tb.Run(\n\t\t\tbcName,\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, sc := range subscribersCases {\n\n\t\t\t\t\tscName := fmt.Sprintf(\"Subs=%dx%v\", sc.numSubs, sc.subType)\n\t\t\t\t\tif sc.subType == None {\n\t\t\t\t\t\tscName = fmt.Sprintf(\"Subs=%v\", sc.subType)\n\t\t\t\t\t}\n\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tscName,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t// Skip short runs, benchmark gets re-executed with a larger N\n\t\t\t\t\t\t\tif b.N < minMessages {\n\t\t\t\t\t\t\t\tb.ResetTimer()\n\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Running %s/%s with %d ops\", bcName, scName, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tsubErrors := uint64(0)\n\t\t\t\t\t\t\thandleSubError := func(_ *nats.Conn, _ *nats.Subscription, _ error) {\n\t\t\t\t\t\t\t\tatomic.AddUint64(&subErrors, 1)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Start single server (no JS)\n\t\t\t\t\t\t\topts := DefaultTestOptions\n\t\t\t\t\t\t\topts.Port = -1\n\t\t\t\t\t\t\ts := RunServer(&opts)\n\t\t\t\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\t\t\t\t// Create subscribers\n\t\t\t\t\t\t\tfor i := 0; i < sc.numSubs; i++ {\n\t\t\t\t\t\t\t\tsubConn, connErr := nats.Connect(s.ClientURL(), nats.ErrorHandler(handleSubError))\n\t\t\t\t\t\t\t\tif connErr != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to connect: %v\", connErr)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tdefer subConn.Close()\n\n\t\t\t\t\t\t\t\tvar sub *nats.Subscription\n\t\t\t\t\t\t\t\tvar subErr error\n\n\t\t\t\t\t\t\t\tswitch sc.subType {\n\t\t\t\t\t\t\t\tcase None:\n\t\t\t\t\t\t\t\t\t// No subscription\n\t\t\t\t\t\t\t\tcase Async:\n\t\t\t\t\t\t\t\t\tsub, subErr = subConn.Subscribe(subject, func(*nats.Msg) {})\n\t\t\t\t\t\t\t\tcase QueueAsync:\n\t\t\t\t\t\t\t\t\tsub, subErr = subConn.QueueSubscribe(subject, queue, func(*nats.Msg) {})\n\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Unknow subscribers type: %v\", sc.subType)\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tif subErr != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to subscribe: %v\", subErr)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tdefer sub.Unsubscribe()\n\t\t\t\t\t\t\t\t// Do not drop messages due to slow subscribers:\n\t\t\t\t\t\t\t\tsub.SetPendingLimits(-1, -1)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Create publisher connection\n\t\t\t\t\t\t\tnc, err := nats.Connect(s.ClientURL())\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to connect: %v\", err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\t\t\t\t\t\tmessage := make([]byte, bc.messageSize)\n\t\t\t\t\t\t\tvar published, errors int\n\n\t\t\t\t\t\t\t// Benchmark starts here\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t\t\t\t\trng.Read(message)\n\t\t\t\t\t\t\t\tpubErr := nc.Publish(subject, message)\n\t\t\t\t\t\t\t\tif pubErr != nil {\n\t\t\t\t\t\t\t\t\terrors++\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\tpublished++\n\t\t\t\t\t\t\t\t\tb.SetBytes(int64(bc.messageSize))\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Benchmark ends here\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tif published+errors != b.N {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Something doesn't add up: %d + %d != %d\", published, errors, b.N)\n\t\t\t\t\t\t\t} else if subErrors > 0 {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Subscribers errors: %d\", subErrors)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n" }, { "path": "server/certidp/certidp.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage certidp\n\nimport (\n\t\"crypto/sha256\"\n\t\"crypto/x509\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"golang.org/x/crypto/ocsp\"\n)\n\nconst (\n\tDefaultAllowedClockSkew = 30 * time.Second\n\tDefaultOCSPResponderTimeout = 2 * time.Second\n\tDefaultTTLUnsetNextUpdate = 1 * time.Hour\n)\n\ntype StatusAssertion int\n\nvar (\n\tStatusAssertionStrToVal = map[string]StatusAssertion{\n\t\t\"good\": ocsp.Good,\n\t\t\"revoked\": ocsp.Revoked,\n\t\t\"unknown\": ocsp.Unknown,\n\t}\n\tStatusAssertionValToStr = map[StatusAssertion]string{\n\t\tocsp.Good: \"good\",\n\t\tocsp.Revoked: \"revoked\",\n\t\tocsp.Unknown: \"unknown\",\n\t}\n\tStatusAssertionIntToVal = map[int]StatusAssertion{\n\t\t0: ocsp.Good,\n\t\t1: ocsp.Revoked,\n\t\t2: ocsp.Unknown,\n\t}\n)\n\n// GetStatusAssertionStr returns the corresponding string representation of the StatusAssertion.\nfunc GetStatusAssertionStr(sa int) string {\n\t// If the provided status assertion value is not found in the map (StatusAssertionIntToVal),\n\t// the function defaults to \"unknown\" to avoid defaulting to \"good,\" which is the default iota value\n\t// for the ocsp.StatusAssertion enumeration (https://pkg.go.dev/golang.org/x/crypto/ocsp#pkg-constants).\n\t// This ensures that we don't unintentionally default to \"good\" when there's no map entry.\n\tv, ok := StatusAssertionIntToVal[sa]\n\tif !ok {\n\t\t// set unknown as fallback\n\t\tv = ocsp.Unknown\n\t}\n\n\treturn StatusAssertionValToStr[v]\n}\n\nfunc (sa StatusAssertion) MarshalJSON() ([]byte, error) {\n\t// This ensures that we don't unintentionally default to \"good\" when there's no map entry.\n\t// (see more details in the GetStatusAssertionStr() comment)\n\tstr, ok := StatusAssertionValToStr[sa]\n\tif !ok {\n\t\t// set unknown as fallback\n\t\tstr = StatusAssertionValToStr[ocsp.Unknown]\n\t}\n\treturn json.Marshal(str)\n}\n\nfunc (sa *StatusAssertion) UnmarshalJSON(in []byte) error {\n\t// This ensures that we don't unintentionally default to \"good\" when there's no map entry.\n\t// (see more details in the GetStatusAssertionStr() comment)\n\tv, ok := StatusAssertionStrToVal[strings.ReplaceAll(string(in), \"\\\"\", \"\")]\n\tif !ok {\n\t\t// set unknown as fallback\n\t\tv = StatusAssertionStrToVal[\"unknown\"]\n\t}\n\t*sa = v\n\treturn nil\n}\n\ntype ChainLink struct {\n\tLeaf *x509.Certificate\n\tIssuer *x509.Certificate\n\tOCSPWebEndpoints *[]*url.URL\n}\n\n// OCSPPeerConfig holds the parsed OCSP peer configuration section of TLS configuration\ntype OCSPPeerConfig struct {\n\tVerify bool\n\tTimeout float64\n\tClockSkew float64\n\tWarnOnly bool\n\tUnknownIsGood bool\n\tAllowWhenCAUnreachable bool\n\tTTLUnsetNextUpdate float64\n}\n\nfunc NewOCSPPeerConfig() *OCSPPeerConfig {\n\treturn &OCSPPeerConfig{\n\t\tVerify: false,\n\t\tTimeout: DefaultOCSPResponderTimeout.Seconds(),\n\t\tClockSkew: DefaultAllowedClockSkew.Seconds(),\n\t\tWarnOnly: false,\n\t\tUnknownIsGood: false,\n\t\tAllowWhenCAUnreachable: false,\n\t\tTTLUnsetNextUpdate: DefaultTTLUnsetNextUpdate.Seconds(),\n\t}\n}\n\n// Log is a neutral method of passing server loggers to plugins\ntype Log struct {\n\tDebugf func(format string, v ...any)\n\tNoticef func(format string, v ...any)\n\tWarnf func(format string, v ...any)\n\tErrorf func(format string, v ...any)\n\tTracef func(format string, v ...any)\n}\n\ntype CertInfo struct {\n\tSubject string `json:\"subject,omitempty\"`\n\tIssuer string `json:\"issuer,omitempty\"`\n\tFingerprint string `json:\"fingerprint,omitempty\"`\n\tRaw []byte `json:\"raw,omitempty\"`\n}\n\nvar OCSPPeerUsage = `\nFor client, leaf spoke (remotes), and leaf hub connections, you may enable OCSP peer validation:\n\n tls {\n ...\n # mTLS must be enabled (with exception of Leaf remotes)\n verify: true\n ...\n # short form enables peer verify and takes option defaults\n ocsp_peer: true\n\n # long form includes settable options\n ocsp_peer {\n # Enable OCSP peer validation (default false)\n verify: true\n\n # OCSP responder timeout in seconds (may be fractional, default 2 seconds)\n ca_timeout: 2\n\n # Allowed skew between server and OCSP responder time in seconds (may be fractional, default 30 seconds)\n allowed_clockskew: 30\n\n # Warn-only and never reject connections (default false)\n warn_only: false\n\n # Treat response Unknown status as valid certificate (default false)\n unknown_is_good: false\n\n # Warn-only if no CA response can be obtained and no cached revocation exists (default false)\n allow_when_ca_unreachable: false\n\n # If response NextUpdate unset by CA, set a default cache TTL in seconds from ThisUpdate (default 1 hour)\n cache_ttl_when_next_update_unset: 3600\n }\n ...\n }\n\nNote: OCSP validation for route and gateway connections is enabled using the 'ocsp' configuration option.\n`\n\n// GenerateFingerprint returns a base64-encoded SHA256 hash of the raw certificate\nfunc GenerateFingerprint(cert *x509.Certificate) string {\n\tdata := sha256.Sum256(cert.Raw)\n\treturn base64.StdEncoding.EncodeToString(data[:])\n}\n\nfunc getWebEndpoints(uris []string) []*url.URL {\n\tvar urls []*url.URL\n\tfor _, uri := range uris {\n\t\tendpoint, err := url.ParseRequestURI(uri)\n\t\tif err != nil {\n\t\t\t// skip invalid URLs\n\t\t\tcontinue\n\t\t}\n\t\tif endpoint.Scheme != \"http\" && endpoint.Scheme != \"https\" {\n\t\t\t// skip non-web URLs\n\t\t\tcontinue\n\t\t}\n\t\turls = append(urls, endpoint)\n\t}\n\treturn urls\n}\n\n// GetSubjectDNForm returns RDN sequence concatenation of the certificate's subject to be\n// used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes.\nfunc GetSubjectDNForm(cert *x509.Certificate) string {\n\tif cert == nil {\n\t\treturn \"\"\n\t}\n\treturn strings.TrimSuffix(fmt.Sprintf(\"%s+\", cert.Subject.ToRDNSequence()), \"+\")\n}\n\n// GetIssuerDNForm returns RDN sequence concatenation of the certificate's issuer to be\n// used in logs, events, etc. Should never be used for reliable cache matching or other crypto purposes.\nfunc GetIssuerDNForm(cert *x509.Certificate) string {\n\tif cert == nil {\n\t\treturn \"\"\n\t}\n\treturn strings.TrimSuffix(fmt.Sprintf(\"%s+\", cert.Issuer.ToRDNSequence()), \"+\")\n}\n\n// CertOCSPEligible checks if the certificate's issuer has populated AIA with OCSP responder endpoint(s)\n// and is thus eligible for OCSP validation\nfunc CertOCSPEligible(link *ChainLink) bool {\n\tif link == nil || link.Leaf.Raw == nil || len(link.Leaf.Raw) == 0 {\n\t\treturn false\n\t}\n\tif len(link.Leaf.OCSPServer) == 0 {\n\t\treturn false\n\t}\n\turls := getWebEndpoints(link.Leaf.OCSPServer)\n\tif len(urls) == 0 {\n\t\treturn false\n\t}\n\tlink.OCSPWebEndpoints = &urls\n\treturn true\n}\n\n// GetLeafIssuerCert returns the issuer certificate of the leaf (positional) certificate in the chain\nfunc GetLeafIssuerCert(chain []*x509.Certificate, leafPos int) *x509.Certificate {\n\tif len(chain) == 0 || leafPos < 0 {\n\t\treturn nil\n\t}\n\t// self-signed certificate or too-big leafPos\n\tif leafPos >= len(chain)-1 {\n\t\treturn nil\n\t}\n\t// returns pointer to issuer cert or nil\n\treturn (chain)[leafPos+1]\n}\n\n// OCSPResponseCurrent checks if the OCSP response is current (i.e. not expired and not future effective)\nfunc OCSPResponseCurrent(ocspr *ocsp.Response, opts *OCSPPeerConfig, log *Log) bool {\n\tskew := time.Duration(opts.ClockSkew * float64(time.Second))\n\tif skew < 0*time.Second {\n\t\tskew = DefaultAllowedClockSkew\n\t}\n\tnow := time.Now().UTC()\n\t// Typical effectivity check based on CA response ThisUpdate and NextUpdate semantics\n\tif !ocspr.NextUpdate.IsZero() && ocspr.NextUpdate.Before(now.Add(-1*skew)) {\n\t\tt := ocspr.NextUpdate.Format(time.RFC3339Nano)\n\t\tnt := now.Format(time.RFC3339Nano)\n\t\tlog.Debugf(DbgResponseExpired, t, nt, skew)\n\t\treturn false\n\t}\n\t// CA responder can assert NextUpdate unset, in which case use config option to set a default cache TTL\n\tif ocspr.NextUpdate.IsZero() {\n\t\tttl := time.Duration(opts.TTLUnsetNextUpdate * float64(time.Second))\n\t\tif ttl < 0*time.Second {\n\t\t\tttl = DefaultTTLUnsetNextUpdate\n\t\t}\n\t\texpiryTime := ocspr.ThisUpdate.Add(ttl)\n\t\tif expiryTime.Before(now.Add(-1 * skew)) {\n\t\t\tt := expiryTime.Format(time.RFC3339Nano)\n\t\t\tnt := now.Format(time.RFC3339Nano)\n\t\t\tlog.Debugf(DbgResponseTTLExpired, t, nt, skew)\n\t\t\treturn false\n\t\t}\n\t}\n\tif ocspr.ThisUpdate.After(now.Add(skew)) {\n\t\tt := ocspr.ThisUpdate.Format(time.RFC3339Nano)\n\t\tnt := now.Format(time.RFC3339Nano)\n\t\tlog.Debugf(DbgResponseFutureDated, t, nt, skew)\n\t\treturn false\n\t}\n\treturn true\n}\n\n// ValidDelegationCheck checks if the CA OCSP Response was signed by a valid CA Issuer delegate as per (RFC 6960, section 4.2.2.2)\n// If a valid delegate or direct-signed by CA Issuer, true returned.\nfunc ValidDelegationCheck(iss *x509.Certificate, ocspr *ocsp.Response) bool {\n\t// This call assumes prior successful parse and signature validation of the OCSP response\n\t// The Go OCSP library (as of x/crypto/ocsp v0.9) will detect and perform a 1-level delegate signature check but does not\n\t// implement the additional criteria for delegation specified in RFC 6960, section 4.2.2.2.\n\tif iss == nil || ocspr == nil {\n\t\treturn false\n\t}\n\t// not a delegation, no-op\n\tif ocspr.Certificate == nil {\n\t\treturn true\n\t}\n\t// delegate is self-same with CA Issuer, not a delegation although response issued in that form\n\tif ocspr.Certificate.Equal(iss) {\n\t\treturn true\n\t}\n\t// we need to verify CA Issuer stamped id-kp-OCSPSigning on delegate\n\tdelegatedSigner := false\n\tfor _, keyUseExt := range ocspr.Certificate.ExtKeyUsage {\n\t\tif keyUseExt == x509.ExtKeyUsageOCSPSigning {\n\t\t\tdelegatedSigner = true\n\t\t\tbreak\n\t\t}\n\t}\n\treturn delegatedSigner\n}\n" }, { "path": "server/certidp/certidp_test.go", "content": "// Copyright 2023 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage certidp\n\nimport \"testing\"\n\n// Checks the return values of the function GetStatusAssertionStr\nfunc TestGetStatusAssertionStr(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput int\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"GoodStatus\",\n\t\t\tinput: 0,\n\t\t\texpected: \"good\",\n\t\t},\n\t\t{\n\t\t\tname: \"RevokedStatus\",\n\t\t\tinput: 1,\n\t\t\texpected: \"revoked\",\n\t\t},\n\t\t{\n\t\t\tname: \"UnknownStatus\",\n\t\t\tinput: 2,\n\t\t\texpected: \"unknown\",\n\t\t},\n\t\t// Invalid status assertion value.\n\t\t{\n\t\t\tname: \"InvalidStatus\",\n\t\t\tinput: 42,\n\t\t\texpected: \"unknown\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot := GetStatusAssertionStr(tt.input)\n\t\t\tif got != tt.expected {\n\t\t\t\tt.Errorf(\"Expected GetStatusAssertionStr: %v, got %v\", tt.expected, got)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/certidp/messages.go", "content": "// Copyright 2023 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage certidp\n\nvar (\n\t// Returned errors\n\tErrIllegalPeerOptsConfig = \"expected map to define OCSP peer options, got [%T]\"\n\tErrIllegalCacheOptsConfig = \"expected map to define OCSP peer cache options, got [%T]\"\n\tErrParsingPeerOptFieldGeneric = \"error parsing tls peer config, unknown field [%q]\"\n\tErrParsingPeerOptFieldTypeConversion = \"error parsing tls peer config, conversion error: %s\"\n\tErrParsingCacheOptFieldTypeConversion = \"error parsing OCSP peer cache config, conversion error: %s\"\n\tErrUnableToPlugTLSEmptyConfig = \"unable to plug TLS verify connection, config is nil\"\n\tErrMTLSRequired = \"OCSP peer verification for client connections requires TLS verify (mTLS) to be enabled\"\n\tErrUnableToPlugTLSClient = \"unable to register client OCSP verification\"\n\tErrUnableToPlugTLSServer = \"unable to register server OCSP verification\"\n\tErrCannotWriteCompressed = \"error writing to compression writer: %w\"\n\tErrCannotReadCompressed = \"error reading compression reader: %w\"\n\tErrTruncatedWrite = \"short write on body (%d != %d)\"\n\tErrCannotCloseWriter = \"error closing compression writer: %w\"\n\tErrParsingCacheOptFieldGeneric = \"error parsing OCSP peer cache config, unknown field [%q]\"\n\tErrUnknownCacheType = \"error parsing OCSP peer cache config, unknown type [%s]\"\n\tErrInvalidChainlink = \"invalid chain link\"\n\tErrBadResponderHTTPStatus = \"bad OCSP responder http status: [%d]\"\n\tErrNoAvailOCSPServers = \"no available OCSP servers\"\n\tErrFailedWithAllRequests = \"exhausted OCSP responders: %w\"\n\n\t// Direct logged errors\n\tErrLoadCacheFail = \"Unable to load OCSP peer cache: %s\"\n\tErrSaveCacheFail = \"Unable to save OCSP peer cache: %s\"\n\tErrBadCacheTypeConfig = \"Unimplemented OCSP peer cache type [%v]\"\n\tErrResponseCompressFail = \"Unable to compress OCSP response for key [%s]: %s\"\n\tErrResponseDecompressFail = \"Unable to decompress OCSP response for key [%s]: %s\"\n\tErrPeerEmptyNoEvent = \"Peer certificate is nil, cannot send OCSP peer reject event\"\n\tErrPeerEmptyAutoReject = \"Peer certificate is nil, rejecting OCSP peer\"\n\n\t// Debug information\n\tDbgPlugTLSForKind = \"Plugging TLS OCSP peer for [%s]\"\n\tDbgNumServerChains = \"Peer OCSP enabled: %d TLS server chain(s) will be evaluated\"\n\tDbgNumClientChains = \"Peer OCSP enabled: %d TLS client chain(s) will be evaluated\"\n\tDbgLinksInChain = \"Chain [%d]: %d total link(s)\"\n\tDbgSelfSignedValid = \"Chain [%d] is self-signed, thus peer is valid\"\n\tDbgValidNonOCSPChain = \"Chain [%d] has no OCSP eligible links, thus peer is valid\"\n\tDbgChainIsOCSPEligible = \"Chain [%d] has %d OCSP eligible link(s)\"\n\tDbgChainIsOCSPValid = \"Chain [%d] is OCSP valid for all eligible links, thus peer is valid\"\n\tDbgNoOCSPValidChains = \"No OCSP valid chains, thus peer is invalid\"\n\tDbgCheckingCacheForCert = \"Checking OCSP peer cache for [%s], key [%s]\"\n\tDbgCurrentResponseCached = \"Cached OCSP response is current, status [%s]\"\n\tDbgExpiredResponseCached = \"Cached OCSP response is expired, status [%s]\"\n\tDbgOCSPValidPeerLink = \"OCSP verify pass for [%s]\"\n\tDbgCachingResponse = \"Caching OCSP response for [%s], key [%s]\"\n\tDbgAchievedCompression = \"OCSP response compression ratio: [%f]\"\n\tDbgCacheHit = \"OCSP peer cache hit for key [%s]\"\n\tDbgCacheMiss = \"OCSP peer cache miss for key [%s]\"\n\tDbgPreservedRevocation = \"Revoked OCSP response for key [%s] preserved by cache policy\"\n\tDbgDeletingCacheResponse = \"Deleting OCSP peer cached response for key [%s]\"\n\tDbgStartingCache = \"Starting OCSP peer cache\"\n\tDbgStoppingCache = \"Stopping OCSP peer cache\"\n\tDbgLoadingCache = \"Loading OCSP peer cache [%s]\"\n\tDbgNoCacheFound = \"No OCSP peer cache found, starting with empty cache\"\n\tDbgSavingCache = \"Saving OCSP peer cache [%s]\"\n\tDbgCacheSaved = \"Saved OCSP peer cache successfully (%d bytes)\"\n\tDbgMakingCARequest = \"Trying OCSP responder url [%s]\"\n\tDbgResponseExpired = \"OCSP response NextUpdate [%s] is before now [%s] with clockskew [%s]\"\n\tDbgResponseTTLExpired = \"OCSP response cache expiry [%s] is before now [%s] with clockskew [%s]\"\n\tDbgResponseFutureDated = \"OCSP response ThisUpdate [%s] is before now [%s] with clockskew [%s]\"\n\tDbgCacheSaveTimerExpired = \"OCSP peer cache save timer expired\"\n\tDbgCacheDirtySave = \"OCSP peer cache is dirty, saving\"\n\n\t// Returned to peer as TLS reject reason\n\tMsgTLSClientRejectConnection = \"client not OCSP valid\"\n\tMsgTLSServerRejectConnection = \"server not OCSP valid\"\n\n\t// Expected runtime errors (direct logged)\n\tErrCAResponderCalloutFail = \"Attempt to obtain OCSP response from CA responder for [%s] failed: %s\"\n\tErrNewCAResponseNotCurrent = \"New OCSP CA response obtained for [%s] but not current\"\n\tErrCAResponseParseFailed = \"Could not parse OCSP CA response for [%s]: %s\"\n\tErrOCSPInvalidPeerLink = \"OCSP verify fail for [%s] with CA status [%s]\"\n\n\t// Policy override warnings (direct logged)\n\tMsgAllowWhenCAUnreachableOccurred = \"Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; no cached revocation so allowing\"\n\tMsgAllowWhenCAUnreachableOccurredCachedRevoke = \"Failed to obtain OCSP CA response for [%s] but AllowWhenCAUnreachable set; cached revocation exists so rejecting\"\n\tMsgAllowWarnOnlyOccurred = \"OCSP verify fail for [%s] but WarnOnly is true so allowing\"\n\n\t// Info (direct logged)\n\tMsgCacheOnline = \"OCSP peer cache online, type [%s]\"\n\tMsgCacheOffline = \"OCSP peer cache offline, type [%s]\"\n\n\t// OCSP cert invalid reasons (debug and event reasons)\n\tMsgFailedOCSPResponseFetch = \"Failed OCSP response fetch\"\n\tMsgOCSPResponseNotEffective = \"OCSP response not in effectivity window\"\n\tMsgFailedOCSPResponseParse = \"Failed OCSP response parse\"\n\tMsgOCSPResponseInvalidStatus = \"Invalid OCSP response status: %s\"\n\tMsgOCSPResponseDelegationInvalid = \"Invalid OCSP response delegation: %s\"\n\tMsgCachedOCSPResponseInvalid = \"Invalid cached OCSP response for [%s] with fingerprint [%s]\"\n)\n" }, { "path": "server/certidp/ocsp_responder.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage certidp\n\nimport (\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"golang.org/x/crypto/ocsp\"\n)\n\nfunc FetchOCSPResponse(link *ChainLink, opts *OCSPPeerConfig, log *Log) ([]byte, error) {\n\tif link == nil || link.Leaf == nil || link.Issuer == nil || opts == nil || log == nil {\n\t\treturn nil, errors.New(ErrInvalidChainlink)\n\t}\n\n\ttimeout := time.Duration(opts.Timeout * float64(time.Second))\n\tif timeout <= 0*time.Second {\n\t\ttimeout = DefaultOCSPResponderTimeout\n\t}\n\n\tgetRequestBytes := func(u string, hc *http.Client) ([]byte, error) {\n\t\tresp, err := hc.Get(u)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tif resp.StatusCode != http.StatusOK {\n\t\t\treturn nil, fmt.Errorf(ErrBadResponderHTTPStatus, resp.StatusCode)\n\t\t}\n\t\treturn io.ReadAll(resp.Body)\n\t}\n\n\t// Request documentation:\n\t// https://tools.ietf.org/html/rfc6960#appendix-A.1\n\n\treqDER, err := ocsp.CreateRequest(link.Leaf, link.Issuer, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treqEnc := encodeOCSPRequest(reqDER)\n\n\tresponders := *link.OCSPWebEndpoints\n\n\tif len(responders) == 0 {\n\t\treturn nil, errors.New(ErrNoAvailOCSPServers)\n\t}\n\n\tvar raw []byte\n\thc := &http.Client{\n\t\tTimeout: timeout,\n\t}\n\tfor _, u := range responders {\n\t\tresponderURL := u.String()\n\t\tlog.Debugf(DbgMakingCARequest, responderURL)\n\t\tresponderURL = strings.TrimSuffix(responderURL, \"/\")\n\t\traw, err = getRequestBytes(fmt.Sprintf(\"%s/%s\", responderURL, reqEnc), hc)\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(ErrFailedWithAllRequests, err)\n\t}\n\n\treturn raw, nil\n}\n\n// encodeOCSPRequest encodes the OCSP request in base64 and URL-encodes it.\n// This is needed to fulfill the OCSP responder's requirements for the request format. (X.690)\nfunc encodeOCSPRequest(reqDER []byte) string {\n\treqEnc := base64.StdEncoding.EncodeToString(reqDER)\n\treturn url.QueryEscape(reqEnc)\n}\n" }, { "path": "server/certidp/ocsp_responder_test.go", "content": "package certidp\n\nimport (\n\t\"encoding/base64\"\n\t\"net/url\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestEncodeOCSPRequest(t *testing.T) {\n\tdata := []byte(\"test data for OCSP request\")\n\tencoded := encodeOCSPRequest(data)\n\n\tif strings.ContainsAny(encoded, \"+/=\") {\n\t\tt.Errorf(\"URL contains unescaped characters: %s\", encoded)\n\t}\n\n\tdecodedURL, err := url.QueryUnescape(encoded)\n\tif err != nil {\n\t\tt.Errorf(\"failed to url-decode request: %v\", err)\n\t}\n\tdecodedData, err := base64.StdEncoding.DecodeString(decodedURL)\n\tif err != nil {\n\t\tt.Errorf(\"failed to base64-decode request: %v\", err)\n\t}\n\tif string(decodedData) != string(data) {\n\t\tt.Errorf(\"Decoded data does not match original: got %s, want %s\", decodedData, data)\n\t}\n}\n" }, { "path": "server/certstore/certstore.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage certstore\n\nimport (\n\t\"crypto\"\n\t\"crypto/x509\"\n\t\"io\"\n\t\"runtime\"\n\t\"strings\"\n)\n\ntype StoreType int\n\nconst MATCHBYEMPTY = 0\nconst STOREEMPTY = 0\n\nconst (\n\twindowsCurrentUser StoreType = iota + 1\n\twindowsLocalMachine\n)\n\nvar StoreMap = map[string]StoreType{\n\t\"windowscurrentuser\": windowsCurrentUser,\n\t\"windowslocalmachine\": windowsLocalMachine,\n}\n\nvar StoreOSMap = map[StoreType]string{\n\twindowsCurrentUser: \"windows\",\n\twindowsLocalMachine: \"windows\",\n}\n\ntype MatchByType int\n\nconst (\n\tmatchByIssuer MatchByType = iota + 1\n\tmatchBySubject\n\tmatchByThumbprint\n)\n\nvar MatchByMap = map[string]MatchByType{\n\t\"issuer\": matchByIssuer,\n\t\"subject\": matchBySubject,\n\t\"thumbprint\": matchByThumbprint,\n}\n\nvar Usage = `\nIn place of cert_file and key_file you may use the windows certificate store:\n\n tls {\n cert_store: \"WindowsCurrentUser\"\n cert_match_by: \"Subject\"\n cert_match: \"MyServer123\"\n }\n`\n\nfunc ParseCertStore(certStore string) (StoreType, error) {\n\tcertStoreType, exists := StoreMap[strings.ToLower(certStore)]\n\tif !exists {\n\t\treturn 0, ErrBadCertStore\n\t}\n\tvalidOS, exists := StoreOSMap[certStoreType]\n\tif !exists || validOS != runtime.GOOS {\n\t\treturn 0, ErrOSNotCompatCertStore\n\t}\n\treturn certStoreType, nil\n}\n\nfunc ParseCertMatchBy(certMatchBy string) (MatchByType, error) {\n\tcertMatchByType, exists := MatchByMap[strings.ToLower(certMatchBy)]\n\tif !exists {\n\t\treturn 0, ErrBadMatchByType\n\t}\n\treturn certMatchByType, nil\n}\n\nfunc GetLeafIssuer(leaf *x509.Certificate, vOpts x509.VerifyOptions) (issuer *x509.Certificate) {\n\tchains, err := leaf.Verify(vOpts)\n\tif err != nil || len(chains) == 0 {\n\t\tissuer = nil\n\t} else {\n\t\tissuer = chains[0][1]\n\t}\n\treturn\n}\n\n// credential provides access to a public key and is a crypto.Signer.\ntype credential interface {\n\t// Public returns the public key corresponding to the leaf certificate.\n\tPublic() crypto.PublicKey\n\t// Sign signs digest with the private key.\n\tSign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error)\n}\n" }, { "path": "server/certstore/certstore_other.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage certstore\n\nimport (\n\t\"crypto\"\n\t\"crypto/tls\"\n\t\"io\"\n)\n\nvar _ = MATCHBYEMPTY\n\n// otherKey implements crypto.Signer and crypto.Decrypter to satisfy linter on platforms that don't implement certstore\ntype otherKey struct{}\n\nfunc TLSConfig(_ StoreType, _ MatchByType, _ string, _ []string, _ bool, _ *tls.Config) error {\n\treturn ErrOSNotCompatCertStore\n}\n\n// Public always returns nil public key since this is a stub on non-supported platform\nfunc (k otherKey) Public() crypto.PublicKey {\n\treturn nil\n}\n\n// Sign always returns a nil signature since this is a stub on non-supported platform\nfunc (k otherKey) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) {\n\t_, _, _ = rand, digest, opts\n\treturn nil, nil\n}\n\n// Verify interface conformance.\nvar _ credential = &otherKey{}\n" }, { "path": "server/certstore/certstore_windows.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Adapted, updated, and enhanced from CertToStore, https://github.com/google/certtostore/releases/tag/v1.0.2\n// Apache License, Version 2.0, Copyright 2017 Google Inc.\n\npackage certstore\n\nimport (\n\t\"bytes\"\n\t\"crypto\"\n\t\"crypto/ecdsa\"\n\t\"crypto/elliptic\"\n\t\"crypto/rsa\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/big\"\n\t\"reflect\"\n\t\"sync\"\n\t\"syscall\"\n\t\"unicode/utf16\"\n\t\"unsafe\"\n\n\t\"golang.org/x/crypto/cryptobyte\"\n\t\"golang.org/x/crypto/cryptobyte/asn1\"\n\t\"golang.org/x/sys/windows\"\n)\n\nconst (\n\t// wincrypt.h constants\n\twinAcquireCached = windows.CRYPT_ACQUIRE_CACHE_FLAG\n\twinAcquireSilent = windows.CRYPT_ACQUIRE_SILENT_FLAG\n\twinAcquireOnlyNCryptKey = windows.CRYPT_ACQUIRE_ONLY_NCRYPT_KEY_FLAG\n\twinEncodingX509ASN = windows.X509_ASN_ENCODING\n\twinEncodingPKCS7 = windows.PKCS_7_ASN_ENCODING\n\twinCertStoreProvSystem = windows.CERT_STORE_PROV_SYSTEM\n\twinCertStoreCurrentUser = windows.CERT_SYSTEM_STORE_CURRENT_USER\n\twinCertStoreLocalMachine = windows.CERT_SYSTEM_STORE_LOCAL_MACHINE\n\twinCertStoreReadOnly = windows.CERT_STORE_READONLY_FLAG\n\twinInfoIssuerFlag = windows.CERT_INFO_ISSUER_FLAG\n\twinInfoSubjectFlag = windows.CERT_INFO_SUBJECT_FLAG\n\twinCompareNameStrW = windows.CERT_COMPARE_NAME_STR_W\n\twinCompareShift = windows.CERT_COMPARE_SHIFT\n\n\t// Reference https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore\n\twinFindIssuerStr = windows.CERT_FIND_ISSUER_STR_W\n\twinFindSubjectStr = windows.CERT_FIND_SUBJECT_STR_W\n\twinFindHashStr = windows.CERT_FIND_HASH_STR\n\n\twinNcryptKeySpec = windows.CERT_NCRYPT_KEY_SPEC\n\n\twinBCryptPadPKCS1 uintptr = 0x2\n\twinBCryptPadPSS uintptr = 0x8 // Modern TLS 1.2+\n\twinBCryptPadPSSSalt uint32 = 32 // default 20, 32 optimal for typical SHA256 hash\n\n\twinRSA1Magic = 0x31415352 // \"RSA1\" BCRYPT_RSAPUBLIC_MAGIC\n\n\twinECS1Magic = 0x31534345 // \"ECS1\" BCRYPT_ECDSA_PUBLIC_P256_MAGIC\n\twinECS3Magic = 0x33534345 // \"ECS3\" BCRYPT_ECDSA_PUBLIC_P384_MAGIC\n\twinECS5Magic = 0x35534345 // \"ECS5\" BCRYPT_ECDSA_PUBLIC_P521_MAGIC\n\n\twinECK1Magic = 0x314B4345 // \"ECK1\" BCRYPT_ECDH_PUBLIC_P256_MAGIC\n\twinECK3Magic = 0x334B4345 // \"ECK3\" BCRYPT_ECDH_PUBLIC_P384_MAGIC\n\twinECK5Magic = 0x354B4345 // \"ECK5\" BCRYPT_ECDH_PUBLIC_P521_MAGIC\n\n\twinCryptENotFound = windows.CRYPT_E_NOT_FOUND\n\n\tproviderMSSoftware = \"Microsoft Software Key Storage Provider\"\n)\n\nvar (\n\twinBCryptRSAPublicBlob = winWide(\"RSAPUBLICBLOB\")\n\twinBCryptECCPublicBlob = winWide(\"ECCPUBLICBLOB\")\n\n\twinNCryptAlgorithmGroupProperty = winWide(\"Algorithm Group\") // NCRYPT_ALGORITHM_GROUP_PROPERTY\n\twinNCryptUniqueNameProperty = winWide(\"Unique Name\") // NCRYPT_UNIQUE_NAME_PROPERTY\n\twinNCryptECCCurveNameProperty = winWide(\"ECCCurveName\") // NCRYPT_ECC_CURVE_NAME_PROPERTY\n\n\twinCurveIDs = map[uint32]elliptic.Curve{\n\t\twinECS1Magic: elliptic.P256(), // BCRYPT_ECDSA_PUBLIC_P256_MAGIC\n\t\twinECS3Magic: elliptic.P384(), // BCRYPT_ECDSA_PUBLIC_P384_MAGIC\n\t\twinECS5Magic: elliptic.P521(), // BCRYPT_ECDSA_PUBLIC_P521_MAGIC\n\t\twinECK1Magic: elliptic.P256(), // BCRYPT_ECDH_PUBLIC_P256_MAGIC\n\t\twinECK3Magic: elliptic.P384(), // BCRYPT_ECDH_PUBLIC_P384_MAGIC\n\t\twinECK5Magic: elliptic.P521(), // BCRYPT_ECDH_PUBLIC_P521_MAGIC\n\t}\n\n\twinCurveNames = map[string]elliptic.Curve{\n\t\t\"nistP256\": elliptic.P256(), // BCRYPT_ECC_CURVE_NISTP256\n\t\t\"nistP384\": elliptic.P384(), // BCRYPT_ECC_CURVE_NISTP384\n\t\t\"nistP521\": elliptic.P521(), // BCRYPT_ECC_CURVE_NISTP521\n\t}\n\n\twinAlgIDs = map[crypto.Hash]*uint16{\n\t\tcrypto.SHA1: winWide(\"SHA1\"), // BCRYPT_SHA1_ALGORITHM\n\t\tcrypto.SHA256: winWide(\"SHA256\"), // BCRYPT_SHA256_ALGORITHM\n\t\tcrypto.SHA384: winWide(\"SHA384\"), // BCRYPT_SHA384_ALGORITHM\n\t\tcrypto.SHA512: winWide(\"SHA512\"), // BCRYPT_SHA512_ALGORITHM\n\t}\n\n\t// MY is well-known system store on Windows that holds personal certificates. Read\n\t// More about the CA locations here:\n\t// https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/wcf/certificate-of-clientcertificate-element?redirectedfrom=MSDN\n\t// https://superuser.com/questions/217719/what-are-the-windows-system-certificate-stores\n\t// https://docs.microsoft.com/en-us/windows/win32/seccrypto/certificate-stores\n\t// https://learn.microsoft.com/en-us/windows/win32/seccrypto/system-store-locations\n\t// https://stackoverflow.com/questions/63286085/which-x509-storename-refers-to-the-certificates-stored-beneath-trusted-root-cert#:~:text=4-,StoreName.,is%20%22Intermediate%20Certification%20Authorities%22.\n\twinMyStore = winWide(\"MY\")\n\twinIntermediateCAStore = winWide(\"CA\")\n\twinRootStore = winWide(\"Root\")\n\twinAuthRootStore = winWide(\"AuthRoot\")\n\n\t// These DLLs must be available on all Windows hosts\n\twinCrypt32 = windows.NewLazySystemDLL(\"crypt32.dll\")\n\twinNCrypt = windows.NewLazySystemDLL(\"ncrypt.dll\")\n\n\twinCertFindCertificateInStore = winCrypt32.NewProc(\"CertFindCertificateInStore\")\n\twinCertVerifyTimeValidity = winCrypt32.NewProc(\"CertVerifyTimeValidity\")\n\twinCryptAcquireCertificatePrivateKey = winCrypt32.NewProc(\"CryptAcquireCertificatePrivateKey\")\n\twinNCryptExportKey = winNCrypt.NewProc(\"NCryptExportKey\")\n\twinNCryptOpenStorageProvider = winNCrypt.NewProc(\"NCryptOpenStorageProvider\")\n\twinNCryptGetProperty = winNCrypt.NewProc(\"NCryptGetProperty\")\n\twinNCryptSignHash = winNCrypt.NewProc(\"NCryptSignHash\")\n\n\twinFnGetProperty = winGetProperty\n)\n\nfunc init() {\n\tfor _, d := range []*windows.LazyDLL{\n\t\twinCrypt32, winNCrypt,\n\t} {\n\t\tif err := d.Load(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\tfor _, p := range []*windows.LazyProc{\n\t\twinCertFindCertificateInStore, winCryptAcquireCertificatePrivateKey,\n\t\twinNCryptExportKey, winNCryptOpenStorageProvider,\n\t\twinNCryptGetProperty, winNCryptSignHash,\n\t} {\n\t\tif err := p.Find(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n\ntype winPKCS1PaddingInfo struct {\n\tpszAlgID *uint16\n}\n\ntype winPSSPaddingInfo struct {\n\tpszAlgID *uint16\n\tcbSalt uint32\n}\n\n// createCACertsPool generates a CertPool from the Windows certificate store,\n// adding all matching certificates from the caCertsMatch array to the pool.\n// All matching certificates (vs first) are added to the pool based on a user\n// request. If no certificates are found an error is returned.\nfunc createCACertsPool(cs *winCertStore, storeType uint32, caCertsMatch []string, skipInvalid bool) (*x509.CertPool, error) {\n\tvar errs []error\n\tcaPool := x509.NewCertPool()\n\tfor _, s := range caCertsMatch {\n\t\tlfs, err := cs.caCertsBySubjectMatch(s, storeType, skipInvalid)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t} else {\n\t\t\tfor _, lf := range lfs {\n\t\t\t\tcaPool.AddCert(lf)\n\t\t\t}\n\t\t}\n\t}\n\t// If every lookup failed return the errors.\n\tif len(errs) == len(caCertsMatch) {\n\t\treturn nil, fmt.Errorf(\"unable to match any CA certificate: %v\", errs)\n\t}\n\treturn caPool, nil\n}\n\n// TLSConfig fulfills the same function as reading cert and key pair from\n// pem files but sources the Windows certificate store instead. The\n// certMatchBy and certMatch fields search the \"MY\" certificate location\n// for the first certificate that matches the certMatch field. The\n// caCertsMatch field is used to search the Trusted Root, Third Party Root,\n// and Intermediate Certificate Authority locations for certificates with\n// Subjects matching the provided strings. If a match is found, the\n// certificate is added to the pool that is used to verify the certificate\n// chain.\nfunc TLSConfig(certStore StoreType, certMatchBy MatchByType, certMatch string, caCertsMatch []string, skipInvalid bool, config *tls.Config) error {\n\tvar (\n\t\tleaf *x509.Certificate\n\t\tleafCtx *windows.CertContext\n\t\tpk *winKey\n\t\tvOpts = x509.VerifyOptions{}\n\t\tchains [][]*x509.Certificate\n\t\tchain []*x509.Certificate\n\t\trawChain [][]byte\n\t)\n\n\t// By StoreType, open a store\n\tif certStore == windowsCurrentUser || certStore == windowsLocalMachine {\n\t\tvar scope uint32\n\t\tcs, err := winOpenCertStore(providerMSSoftware)\n\t\tif err != nil || cs == nil {\n\t\t\treturn err\n\t\t}\n\t\tif certStore == windowsCurrentUser {\n\t\t\tscope = winCertStoreCurrentUser\n\t\t}\n\t\tif certStore == windowsLocalMachine {\n\t\t\tscope = winCertStoreLocalMachine\n\t\t}\n\n\t\t// certByIssuer or certBySubject\n\t\tif certMatchBy == matchBySubject || certMatchBy == MATCHBYEMPTY {\n\t\t\tleaf, leafCtx, err = cs.certBySubject(certMatch, scope, skipInvalid)\n\t\t} else if certMatchBy == matchByIssuer {\n\t\t\tleaf, leafCtx, err = cs.certByIssuer(certMatch, scope, skipInvalid)\n\t\t} else if certMatchBy == matchByThumbprint {\n\t\t\tleaf, leafCtx, err = cs.certByThumbprint(certMatch, scope, skipInvalid)\n\t\t} else {\n\t\t\treturn ErrBadMatchByType\n\t\t}\n\t\tif err != nil {\n\t\t\t// pass through error from cert search\n\t\t\treturn err\n\t\t}\n\t\tif leaf == nil || leafCtx == nil {\n\t\t\treturn ErrFailedCertSearch\n\t\t}\n\t\tpk, err = cs.certKey(leafCtx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif pk == nil {\n\t\t\treturn ErrNoPrivateKeyStoreRef\n\t\t}\n\t\t// Look for CA Certificates\n\t\tif len(caCertsMatch) != 0 {\n\t\t\tcaPool, err := createCACertsPool(cs, scope, caCertsMatch, skipInvalid)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tconfig.ClientCAs = caPool\n\t\t}\n\t} else {\n\t\treturn ErrBadCertStore\n\t}\n\n\t// Get intermediates in the cert store for the found leaf IFF there is a full chain of trust in the store\n\t// otherwise just use leaf as the final chain.\n\t//\n\t// Using std lib Verify as a reliable way to get valid chains out of the win store for the leaf; however,\n\t// using empty options since server TLS stanza could be TLS role as server identity or client identity.\n\tchains, err := leaf.Verify(vOpts)\n\tif err != nil || len(chains) == 0 {\n\t\tchains = append(chains, []*x509.Certificate{leaf})\n\t}\n\n\t// We have at least one verified chain so pop the first chain and remove the self-signed CA cert (if present)\n\t// from the end of the chain\n\tchain = chains[0]\n\tif len(chain) > 1 {\n\t\tchain = chain[:len(chain)-1]\n\t}\n\n\t// For tls.Certificate.Certificate need a [][]byte from []*x509.Certificate\n\t// Approximate capacity for efficiency\n\trawChain = make([][]byte, 0, len(chain))\n\tfor _, link := range chain {\n\t\trawChain = append(rawChain, link.Raw)\n\t}\n\n\ttlsCert := tls.Certificate{\n\t\tCertificate: rawChain,\n\t\tPrivateKey: pk,\n\t\tLeaf: leaf,\n\t}\n\tconfig.Certificates = []tls.Certificate{tlsCert}\n\n\t// note: pk is a windows pointer (not freed by Go) but needs to live the life of the server for Signing.\n\t// The cert context (leafCtx) windows pointer must not be freed underneath the pk so also life of the server.\n\treturn nil\n}\n\n// winWide returns a pointer to uint16 representing the equivalent\n// to a Windows LPCWSTR.\nfunc winWide(s string) *uint16 {\n\tw := utf16.Encode([]rune(s))\n\tw = append(w, 0)\n\treturn &w[0]\n}\n\n// winOpenProvider gets a provider handle for subsequent calls\nfunc winOpenProvider(provider string) (uintptr, error) {\n\tvar hProv uintptr\n\tpname := winWide(provider)\n\t// Open the provider, the last parameter is not used\n\tr, _, err := winNCryptOpenStorageProvider.Call(uintptr(unsafe.Pointer(&hProv)), uintptr(unsafe.Pointer(pname)), 0)\n\tif r == 0 {\n\t\treturn hProv, nil\n\t}\n\treturn hProv, fmt.Errorf(\"NCryptOpenStorageProvider returned %X: %v\", r, err)\n}\n\n// winFindCert wraps the CertFindCertificateInStore library call. Note that any cert context passed\n// into prev will be freed. If no certificate was found, nil will be returned.\nfunc winFindCert(store windows.Handle, enc, findFlags, findType uint32, para *uint16, prev *windows.CertContext) (*windows.CertContext, error) {\n\th, _, err := winCertFindCertificateInStore.Call(\n\t\tuintptr(store),\n\t\tuintptr(enc),\n\t\tuintptr(findFlags),\n\t\tuintptr(findType),\n\t\tuintptr(unsafe.Pointer(para)),\n\t\tuintptr(unsafe.Pointer(prev)),\n\t)\n\tif h == 0 {\n\t\t// Actual error, or simply not found?\n\t\tif errno, ok := err.(syscall.Errno); ok && errno == syscall.Errno(winCryptENotFound) {\n\t\t\treturn nil, ErrFailedCertSearch\n\t\t}\n\t\treturn nil, ErrFailedCertSearch\n\t}\n\t// nolint:govet\n\treturn (*windows.CertContext)(unsafe.Pointer(h)), nil\n}\n\n// winVerifyCertValid wraps the CertVerifyTimeValidity and simply returns true if the certificate is valid\nfunc winVerifyCertValid(timeToVerify *windows.Filetime, certInfo *windows.CertInfo) bool {\n\t// this function does not document returning errors / setting lasterror\n\tr, _, _ := winCertVerifyTimeValidity.Call(\n\t\tuintptr(unsafe.Pointer(timeToVerify)),\n\t\tuintptr(unsafe.Pointer(certInfo)),\n\t)\n\treturn r == 0\n}\n\n// winCertStore is a store implementation for the Windows Certificate Store\ntype winCertStore struct {\n\tProv uintptr\n\tProvName string\n\tstores map[string]*winStoreHandle\n\tmu sync.Mutex\n}\n\n// winOpenCertStore creates a winCertStore\nfunc winOpenCertStore(provider string) (*winCertStore, error) {\n\tcngProv, err := winOpenProvider(provider)\n\tif err != nil {\n\t\t// pass through error from winOpenProvider\n\t\treturn nil, err\n\t}\n\n\twcs := &winCertStore{\n\t\tProv: cngProv,\n\t\tProvName: provider,\n\t\tstores: make(map[string]*winStoreHandle),\n\t}\n\n\treturn wcs, nil\n}\n\n// winCertContextToX509 creates an x509.Certificate from a Windows cert context.\nfunc winCertContextToX509(ctx *windows.CertContext) (*x509.Certificate, error) {\n\tvar der []byte\n\tslice := (*reflect.SliceHeader)(unsafe.Pointer(&der))\n\tslice.Data = uintptr(unsafe.Pointer(ctx.EncodedCert))\n\tslice.Len = int(ctx.Length)\n\tslice.Cap = int(ctx.Length)\n\treturn x509.ParseCertificate(der)\n}\n\n// certByIssuer matches and returns the first certificate found by passed issuer.\n// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies\n// current user's personal certs or local machine's personal certs using storeType.\n// See CERT_FIND_ISSUER_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore\nfunc (w *winCertStore) certByIssuer(issuer string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) {\n\treturn w.certSearch(winFindIssuerStr, issuer, winMyStore, storeType, skipInvalid)\n}\n\n// certBySubject matches and returns the first certificate found by passed subject field.\n// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies\n// current user's personal certs or local machine's personal certs using storeType.\n// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore\nfunc (w *winCertStore) certBySubject(subject string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) {\n\treturn w.certSearch(winFindSubjectStr, subject, winMyStore, storeType, skipInvalid)\n}\n\n// certByThumbprint matches and returns the first certificate found by passed SHA1 thumbprint.\n// CertContext pointer returned allows subsequent key operations like Sign. Caller specifies\n// current user's personal certs or local machine's personal certs using storeType.\n// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore\nfunc (w *winCertStore) certByThumbprint(hash string, storeType uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) {\n\treturn w.certSearch(winFindHashStr, hash, winMyStore, storeType, skipInvalid)\n}\n\n// caCertsBySubjectMatch matches and returns all matching certificates of the subject field.\n//\n// The following locations are searched:\n// 1) Root (Trusted Root Certification Authorities)\n// 2) AuthRoot (Third-Party Root Certification Authorities)\n// 3) CA (Intermediate Certification Authorities)\n//\n// Caller specifies current user's personal certs or local machine's personal certs using storeType.\n// See CERT_FIND_SUBJECT_STR description at https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-certfindcertificateinstore\nfunc (w *winCertStore) caCertsBySubjectMatch(subject string, storeType uint32, skipInvalid bool) ([]*x509.Certificate, error) {\n\tvar (\n\t\tleaf *x509.Certificate\n\t\tsearchLocations = [3]*uint16{winRootStore, winAuthRootStore, winIntermediateCAStore}\n\t\trv []*x509.Certificate\n\t)\n\t// surprisingly, an empty string returns a result. We'll treat this as an error.\n\tif subject == \"\" {\n\t\treturn nil, ErrBadCaCertMatchField\n\t}\n\tfor _, sr := range searchLocations {\n\t\tvar err error\n\t\tif leaf, _, err = w.certSearch(winFindSubjectStr, subject, sr, storeType, skipInvalid); err == nil {\n\t\t\trv = append(rv, leaf)\n\t\t} else {\n\t\t\t// Ignore the failed search from a single location. Errors we catch include\n\t\t\t// ErrFailedX509Extract (resulting from a malformed certificate) and errors\n\t\t\t// around invalid attributes, unsupported algorithms, etc. These are corner\n\t\t\t// cases as certificates with these errors shouldn't have been allowed\n\t\t\t// to be added to the store in the first place.\n\t\t\tif err != ErrFailedCertSearch {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\t// Not found anywhere\n\tif len(rv) == 0 {\n\t\treturn nil, ErrFailedCertSearch\n\t}\n\treturn rv, nil\n}\n\n// certSearch is a helper function to lookup certificates based on search type and match value.\n// store is used to specify which store to perform the lookup in (system or user).\nfunc (w *winCertStore) certSearch(searchType uint32, matchValue string, searchRoot *uint16, store uint32, skipInvalid bool) (*x509.Certificate, *windows.CertContext, error) {\n\t// store handle to \"MY\" store\n\th, err := w.storeHandle(store, searchRoot)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tvar prev *windows.CertContext\n\tvar cert *x509.Certificate\n\n\ti, err := windows.UTF16PtrFromString(matchValue)\n\tif err != nil {\n\t\treturn nil, nil, ErrFailedCertSearch\n\t}\n\n\t// pass 0 as the third parameter because it is not used\n\t// https://msdn.microsoft.com/en-us/library/windows/desktop/aa376064(v=vs.85).aspx\n\n\tfor {\n\t\tnc, err := winFindCert(h, winEncodingX509ASN|winEncodingPKCS7, 0, searchType, i, prev)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tif nc != nil {\n\t\t\t// certificate found\n\t\t\tprev = nc\n\n\t\t\tvar now *windows.Filetime\n\t\t\tif skipInvalid && !winVerifyCertValid(now, nc.CertInfo) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Extract the DER-encoded certificate from the cert context\n\t\t\txc, err := winCertContextToX509(nc)\n\t\t\tif err == nil {\n\t\t\t\tcert = xc\n\t\t\t\tbreak\n\t\t\t} else {\n\t\t\t\treturn nil, nil, ErrFailedX509Extract\n\t\t\t}\n\t\t} else {\n\t\t\treturn nil, nil, ErrFailedCertSearch\n\t\t}\n\t}\n\n\tif cert == nil {\n\t\treturn nil, nil, ErrFailedX509Extract\n\t}\n\n\treturn cert, prev, nil\n}\n\ntype winStoreHandle struct {\n\thandle *windows.Handle\n}\n\nfunc winNewStoreHandle(provider uint32, store *uint16) (*winStoreHandle, error) {\n\tvar s winStoreHandle\n\tif s.handle != nil {\n\t\treturn &s, nil\n\t}\n\tst, err := windows.CertOpenStore(\n\t\twinCertStoreProvSystem,\n\t\t0,\n\t\t0,\n\t\tprovider|winCertStoreReadOnly,\n\t\tuintptr(unsafe.Pointer(store)))\n\tif err != nil {\n\t\treturn nil, ErrBadCryptoStoreProvider\n\t}\n\ts.handle = &st\n\treturn &s, nil\n}\n\n// winKey implements crypto.Signer and crypto.Decrypter for key based operations.\ntype winKey struct {\n\thandle uintptr\n\tpub crypto.PublicKey\n\tContainer string\n\tAlgorithmGroup string\n}\n\n// Public exports a public key to implement crypto.Signer\nfunc (k winKey) Public() crypto.PublicKey {\n\treturn k.pub\n}\n\n// Sign returns the signature of a hash to implement crypto.Signer\nfunc (k winKey) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) {\n\tswitch k.AlgorithmGroup {\n\tcase \"ECDSA\", \"ECDH\":\n\t\treturn winSignECDSA(k.handle, digest)\n\tcase \"RSA\":\n\t\thf := opts.HashFunc()\n\t\talgID, ok := winAlgIDs[hf]\n\t\tif !ok {\n\t\t\treturn nil, ErrBadRSAHashAlgorithm\n\t\t}\n\t\tswitch opts.(type) {\n\t\tcase *rsa.PSSOptions:\n\t\t\treturn winSignRSAPSSPadding(k.handle, digest, algID)\n\t\tdefault:\n\t\t\treturn winSignRSAPKCS1Padding(k.handle, digest, algID)\n\t\t}\n\tdefault:\n\t\treturn nil, ErrBadSigningAlgorithm\n\t}\n}\n\nfunc winSignECDSA(kh uintptr, digest []byte) ([]byte, error) {\n\tvar size uint32\n\t// Obtain the size of the signature\n\tr, _, _ := winNCryptSignHash.Call(\n\t\tkh,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\t0,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrStoreECDSASigningError\n\t}\n\n\t// Obtain the signature data\n\tbuf := make([]byte, size)\n\tr, _, _ = winNCryptSignHash.Call(\n\t\tkh,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\tuintptr(unsafe.Pointer(&buf[0])),\n\t\tuintptr(size),\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrStoreECDSASigningError\n\t}\n\tif len(buf) != int(size) {\n\t\treturn nil, ErrStoreECDSASigningError\n\t}\n\n\treturn winPackECDSASigValue(bytes.NewReader(buf[:size]), int(size/2))\n}\n\nfunc winPackECDSASigValue(r io.Reader, digestLength int) ([]byte, error) {\n\tsigR := make([]byte, digestLength)\n\tif _, err := io.ReadFull(r, sigR); err != nil {\n\t\treturn nil, ErrStoreECDSASigningError\n\t}\n\n\tsigS := make([]byte, digestLength)\n\tif _, err := io.ReadFull(r, sigS); err != nil {\n\t\treturn nil, ErrStoreECDSASigningError\n\t}\n\n\tvar b cryptobyte.Builder\n\tb.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) {\n\t\tb.AddASN1BigInt(new(big.Int).SetBytes(sigR))\n\t\tb.AddASN1BigInt(new(big.Int).SetBytes(sigS))\n\t})\n\treturn b.Bytes()\n}\n\nfunc winSignRSAPKCS1Padding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) {\n\t// PKCS#1 v1.5 padding for some TLS 1.2\n\tpadInfo := winPKCS1PaddingInfo{pszAlgID: algID}\n\tvar size uint32\n\t// Obtain the size of the signature\n\tr, _, _ := winNCryptSignHash.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(&padInfo)),\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\t0,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\twinBCryptPadPKCS1)\n\tif r != 0 {\n\t\treturn nil, ErrStoreRSASigningError\n\t}\n\n\t// Obtain the signature data\n\tsig := make([]byte, size)\n\tr, _, _ = winNCryptSignHash.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(&padInfo)),\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\tuintptr(unsafe.Pointer(&sig[0])),\n\t\tuintptr(size),\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\twinBCryptPadPKCS1)\n\tif r != 0 {\n\t\treturn nil, ErrStoreRSASigningError\n\t}\n\n\treturn sig[:size], nil\n}\n\nfunc winSignRSAPSSPadding(kh uintptr, digest []byte, algID *uint16) ([]byte, error) {\n\t// PSS padding for TLS 1.3 and some TLS 1.2\n\tpadInfo := winPSSPaddingInfo{pszAlgID: algID, cbSalt: winBCryptPadPSSSalt}\n\n\tvar size uint32\n\t// Obtain the size of the signature\n\tr, _, _ := winNCryptSignHash.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(&padInfo)),\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\t0,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\twinBCryptPadPSS)\n\tif r != 0 {\n\t\treturn nil, ErrStoreRSASigningError\n\t}\n\n\t// Obtain the signature data\n\tsig := make([]byte, size)\n\tr, _, _ = winNCryptSignHash.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(&padInfo)),\n\t\tuintptr(unsafe.Pointer(&digest[0])),\n\t\tuintptr(len(digest)),\n\t\tuintptr(unsafe.Pointer(&sig[0])),\n\t\tuintptr(size),\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\twinBCryptPadPSS)\n\tif r != 0 {\n\t\treturn nil, ErrStoreRSASigningError\n\t}\n\n\treturn sig[:size], nil\n}\n\n// certKey wraps CryptAcquireCertificatePrivateKey. It obtains the CNG private\n// key of a known certificate and returns a pointer to a winKey which implements\n// both crypto.Signer. When a nil cert context is passed\n// a nil key is intentionally returned, to model the expected behavior of a\n// non-existent cert having no private key.\n// https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-cryptacquirecertificateprivatekey\nfunc (w *winCertStore) certKey(cert *windows.CertContext) (*winKey, error) {\n\t// Return early if a nil cert was passed.\n\tif cert == nil {\n\t\treturn nil, nil\n\t}\n\tvar (\n\t\tkh uintptr\n\t\tspec uint32\n\t\tmustFree int\n\t)\n\tr, _, _ := winCryptAcquireCertificatePrivateKey.Call(\n\t\tuintptr(unsafe.Pointer(cert)),\n\t\twinAcquireCached|winAcquireSilent|winAcquireOnlyNCryptKey,\n\t\t0, // Reserved, must be null.\n\t\tuintptr(unsafe.Pointer(&kh)),\n\t\tuintptr(unsafe.Pointer(&spec)),\n\t\tuintptr(unsafe.Pointer(&mustFree)),\n\t)\n\t// If the function succeeds, the return value is nonzero (TRUE).\n\tif r == 0 {\n\t\treturn nil, ErrNoPrivateKeyStoreRef\n\t}\n\tif mustFree != 0 {\n\t\treturn nil, ErrNoPrivateKeyStoreRef\n\t}\n\tif spec != winNcryptKeySpec {\n\t\treturn nil, ErrNoPrivateKeyStoreRef\n\t}\n\n\treturn winKeyMetadata(kh)\n}\n\nfunc winKeyMetadata(kh uintptr) (*winKey, error) {\n\t// uc is used to populate the unique container name attribute of the private key\n\tuc, err := winGetPropertyStr(kh, winNCryptUniqueNameProperty)\n\tif err != nil {\n\t\t// unable to determine key unique name\n\t\treturn nil, ErrExtractingPrivateKeyMetadata\n\t}\n\n\talg, err := winGetPropertyStr(kh, winNCryptAlgorithmGroupProperty)\n\tif err != nil {\n\t\t// unable to determine key algorithm\n\t\treturn nil, ErrExtractingPrivateKeyMetadata\n\t}\n\n\tvar pub crypto.PublicKey\n\n\tswitch alg {\n\tcase \"ECDSA\", \"ECDH\":\n\t\tbuf, err := winExport(kh, winBCryptECCPublicBlob)\n\t\tif err != nil {\n\t\t\t// failed to export ECC public key\n\t\t\treturn nil, ErrExtractingECCPublicKey\n\t\t}\n\t\tpub, err = unmarshalECC(buf, kh)\n\t\tif err != nil {\n\t\t\treturn nil, ErrExtractingECCPublicKey\n\t\t}\n\tcase \"RSA\":\n\t\tbuf, err := winExport(kh, winBCryptRSAPublicBlob)\n\t\tif err != nil {\n\t\t\treturn nil, ErrExtractingRSAPublicKey\n\t\t}\n\t\tpub, err = winUnmarshalRSA(buf)\n\t\tif err != nil {\n\t\t\treturn nil, ErrExtractingRSAPublicKey\n\t\t}\n\tdefault:\n\t\treturn nil, ErrBadPublicKeyAlgorithm\n\t}\n\n\treturn &winKey{handle: kh, pub: pub, Container: uc, AlgorithmGroup: alg}, nil\n}\n\nfunc winGetProperty(kh uintptr, property *uint16) ([]byte, error) {\n\tvar strSize uint32\n\tr, _, _ := winNCryptGetProperty.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(property)),\n\t\t0,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&strSize)),\n\t\t0,\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrExtractPropertyFromKey\n\t}\n\n\tbuf := make([]byte, strSize)\n\tr, _, _ = winNCryptGetProperty.Call(\n\t\tkh,\n\t\tuintptr(unsafe.Pointer(property)),\n\t\tuintptr(unsafe.Pointer(&buf[0])),\n\t\tuintptr(strSize),\n\t\tuintptr(unsafe.Pointer(&strSize)),\n\t\t0,\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrExtractPropertyFromKey\n\t}\n\n\treturn buf, nil\n}\n\nfunc winGetPropertyStr(kh uintptr, property *uint16) (string, error) {\n\tbuf, err := winFnGetProperty(kh, property)\n\tif err != nil {\n\t\treturn \"\", ErrExtractPropertyFromKey\n\t}\n\tuc := bytes.ReplaceAll(buf, []byte{0x00}, []byte(\"\"))\n\treturn string(uc), nil\n}\n\nfunc winExport(kh uintptr, blobType *uint16) ([]byte, error) {\n\tvar size uint32\n\t// When obtaining the size of a public key, most parameters are not required\n\tr, _, _ := winNCryptExportKey.Call(\n\t\tkh,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(blobType)),\n\t\t0,\n\t\t0,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrExtractingPublicKey\n\t}\n\n\t// Place the exported key in buf now that we know the size required\n\tbuf := make([]byte, size)\n\tr, _, _ = winNCryptExportKey.Call(\n\t\tkh,\n\t\t0,\n\t\tuintptr(unsafe.Pointer(blobType)),\n\t\t0,\n\t\tuintptr(unsafe.Pointer(&buf[0])),\n\t\tuintptr(size),\n\t\tuintptr(unsafe.Pointer(&size)),\n\t\t0)\n\tif r != 0 {\n\t\treturn nil, ErrExtractingPublicKey\n\t}\n\treturn buf, nil\n}\n\nfunc unmarshalECC(buf []byte, kh uintptr) (*ecdsa.PublicKey, error) {\n\t// BCRYPT_ECCKEY_BLOB from bcrypt.h\n\theader := struct {\n\t\tMagic uint32\n\t\tKey uint32\n\t}{}\n\n\tr := bytes.NewReader(buf)\n\tif err := binary.Read(r, binary.LittleEndian, &header); err != nil {\n\t\treturn nil, ErrExtractingECCPublicKey\n\t}\n\n\tcurve, ok := winCurveIDs[header.Magic]\n\tif !ok {\n\t\t// Fix for b/185945636, where despite specifying the curve, nCrypt returns\n\t\t// an incorrect response with BCRYPT_ECDSA_PUBLIC_GENERIC_MAGIC.\n\t\tvar err error\n\t\tcurve, err = winCurveName(kh)\n\t\tif err != nil {\n\t\t\t// unsupported header magic or cannot match the curve by name\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tkeyX := make([]byte, header.Key)\n\tif n, err := r.Read(keyX); n != int(header.Key) || err != nil {\n\t\t// failed to read key X\n\t\treturn nil, ErrExtractingECCPublicKey\n\t}\n\n\tkeyY := make([]byte, header.Key)\n\tif n, err := r.Read(keyY); n != int(header.Key) || err != nil {\n\t\t// failed to read key Y\n\t\treturn nil, ErrExtractingECCPublicKey\n\t}\n\n\tpub := &ecdsa.PublicKey{\n\t\tCurve: curve,\n\t\tX: new(big.Int).SetBytes(keyX),\n\t\tY: new(big.Int).SetBytes(keyY),\n\t}\n\treturn pub, nil\n}\n\n// winCurveName reads the curve name property and returns the corresponding curve.\nfunc winCurveName(kh uintptr) (elliptic.Curve, error) {\n\tcn, err := winGetPropertyStr(kh, winNCryptECCCurveNameProperty)\n\tif err != nil {\n\t\t// unable to determine the curve property name\n\t\treturn nil, ErrExtractPropertyFromKey\n\t}\n\tcurve, ok := winCurveNames[cn]\n\tif !ok {\n\t\t// unknown curve name\n\t\treturn nil, ErrBadECCCurveName\n\t}\n\treturn curve, nil\n}\n\nfunc winUnmarshalRSA(buf []byte) (*rsa.PublicKey, error) {\n\t// BCRYPT_RSA_BLOB from bcrypt.h\n\theader := struct {\n\t\tMagic uint32\n\t\tBitLength uint32\n\t\tPublicExpSize uint32\n\t\tModulusSize uint32\n\t\tUnusedPrime1 uint32\n\t\tUnusedPrime2 uint32\n\t}{}\n\n\tr := bytes.NewReader(buf)\n\tif err := binary.Read(r, binary.LittleEndian, &header); err != nil {\n\t\treturn nil, ErrExtractingRSAPublicKey\n\t}\n\n\tif header.Magic != winRSA1Magic {\n\t\t// invalid header magic\n\t\treturn nil, ErrExtractingRSAPublicKey\n\t}\n\n\tif header.PublicExpSize > 8 {\n\t\t// unsupported public exponent size\n\t\treturn nil, ErrExtractingRSAPublicKey\n\t}\n\n\texp := make([]byte, 8)\n\tif n, err := r.Read(exp[8-header.PublicExpSize:]); n != int(header.PublicExpSize) || err != nil {\n\t\t// failed to read public exponent\n\t\treturn nil, ErrExtractingRSAPublicKey\n\t}\n\n\tmod := make([]byte, header.ModulusSize)\n\tif n, err := r.Read(mod); n != int(header.ModulusSize) || err != nil {\n\t\t// failed to read modulus\n\t\treturn nil, ErrExtractingRSAPublicKey\n\t}\n\n\tpub := &rsa.PublicKey{\n\t\tN: new(big.Int).SetBytes(mod),\n\t\tE: int(binary.BigEndian.Uint64(exp)),\n\t}\n\treturn pub, nil\n}\n\n// storeHandle returns a handle to a given cert store, opening the handle as needed.\nfunc (w *winCertStore) storeHandle(provider uint32, store *uint16) (windows.Handle, error) {\n\tw.mu.Lock()\n\tdefer w.mu.Unlock()\n\n\tkey := fmt.Sprintf(\"%d%s\", provider, windows.UTF16PtrToString(store))\n\tvar err error\n\tif w.stores[key] == nil {\n\t\tw.stores[key], err = winNewStoreHandle(provider, store)\n\t\tif err != nil {\n\t\t\treturn 0, ErrBadCryptoStoreProvider\n\t\t}\n\t}\n\treturn *w.stores[key].handle, nil\n}\n\n// Verify interface conformance.\nvar _ credential = &winKey{}\n" }, { "path": "server/certstore/errors.go", "content": "package certstore\n\nimport (\n\t\"errors\"\n)\n\nvar (\n\t// ErrBadCryptoStoreProvider represents inablity to establish link with a certificate store\n\tErrBadCryptoStoreProvider = errors.New(\"unable to open certificate store or store not available\")\n\n\t// ErrBadRSAHashAlgorithm represents a bad or unsupported RSA hash algorithm\n\tErrBadRSAHashAlgorithm = errors.New(\"unsupported RSA hash algorithm\")\n\n\t// ErrBadSigningAlgorithm represents a bad or unsupported signing algorithm\n\tErrBadSigningAlgorithm = errors.New(\"unsupported signing algorithm\")\n\n\t// ErrStoreRSASigningError represents an error returned from store during RSA signature\n\tErrStoreRSASigningError = errors.New(\"unable to obtain RSA signature from store\")\n\n\t// ErrStoreECDSASigningError represents an error returned from store during ECDSA signature\n\tErrStoreECDSASigningError = errors.New(\"unable to obtain ECDSA signature from store\")\n\n\t// ErrNoPrivateKeyStoreRef represents an error getting a handle to a private key in store\n\tErrNoPrivateKeyStoreRef = errors.New(\"unable to obtain private key handle from store\")\n\n\t// ErrExtractingPrivateKeyMetadata represents a family of errors extracting metadata about the private key in store\n\tErrExtractingPrivateKeyMetadata = errors.New(\"unable to extract private key metadata\")\n\n\t// ErrExtractingECCPublicKey represents an error exporting ECC-type public key from store\n\tErrExtractingECCPublicKey = errors.New(\"unable to extract ECC public key from store\")\n\n\t// ErrExtractingRSAPublicKey represents an error exporting RSA-type public key from store\n\tErrExtractingRSAPublicKey = errors.New(\"unable to extract RSA public key from store\")\n\n\t// ErrExtractingPublicKey represents a general error exporting public key from store\n\tErrExtractingPublicKey = errors.New(\"unable to extract public key from store\")\n\n\t// ErrBadPublicKeyAlgorithm represents a bad or unsupported public key algorithm\n\tErrBadPublicKeyAlgorithm = errors.New(\"unsupported public key algorithm\")\n\n\t// ErrExtractPropertyFromKey represents a general failure to extract a metadata property field\n\tErrExtractPropertyFromKey = errors.New(\"unable to extract property from key\")\n\n\t// ErrBadECCCurveName represents an ECC signature curve name that is bad or unsupported\n\tErrBadECCCurveName = errors.New(\"unsupported ECC curve name\")\n\n\t// ErrFailedCertSearch represents not able to find certificate in store\n\tErrFailedCertSearch = errors.New(\"unable to find certificate in store\")\n\n\t// ErrFailedX509Extract represents not being able to extract x509 certificate from found cert in store\n\tErrFailedX509Extract = errors.New(\"unable to extract x509 from certificate\")\n\n\t// ErrBadMatchByType represents unknown CERT_MATCH_BY passed\n\tErrBadMatchByType = errors.New(\"cert match by type not implemented\")\n\n\t// ErrBadCertStore represents unknown CERT_STORE passed\n\tErrBadCertStore = errors.New(\"cert store type not implemented\")\n\n\t// ErrConflictCertFileAndStore represents ambiguous configuration of both file and store\n\tErrConflictCertFileAndStore = errors.New(\"'cert_file' and 'cert_store' may not both be configured\")\n\n\t// ErrBadCertStoreField represents malformed cert_store option\n\tErrBadCertStoreField = errors.New(\"expected 'cert_store' to be a valid non-empty string\")\n\n\t// ErrBadCertMatchByField represents malformed cert_match_by option\n\tErrBadCertMatchByField = errors.New(\"expected 'cert_match_by' to be a valid non-empty string\")\n\n\t// ErrBadCertMatchField represents malformed cert_match option\n\tErrBadCertMatchField = errors.New(\"expected 'cert_match' to be a valid non-empty string\")\n\n\t// ErrBadCaCertMatchField represents malformed cert_match option\n\tErrBadCaCertMatchField = errors.New(\"expected 'ca_certs_match' to be a valid non-empty string array\")\n\n\t// ErrBadCertMatchSkipInvalidField represents malformed cert_match_skip_invalid option\n\tErrBadCertMatchSkipInvalidField = errors.New(\"expected 'cert_match_skip_invalid' to be a boolean\")\n\n\t// ErrOSNotCompatCertStore represents cert_store passed that exists but is not valid on current OS\n\tErrOSNotCompatCertStore = errors.New(\"cert_store not compatible with current operating system\")\n)\n" }, { "path": "server/certstore_windows_test.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage server\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc runPowershellScript(scriptFile string, args []string) error {\n\tpsExec, _ := exec.LookPath(\"powershell.exe\")\n\n\texecArgs := []string{psExec, \"-command\", fmt.Sprintf(\"& '%s'\", scriptFile)}\n\tif len(args) > 0 {\n\t\texecArgs = append(execArgs, args...)\n\t}\n\n\tcmdImport := &exec.Cmd{\n\t\tPath: psExec,\n\t\tArgs: execArgs,\n\t\tStdout: os.Stdout,\n\t\tStderr: os.Stderr,\n\t}\n\treturn cmdImport.Run()\n}\n\nfunc runConfiguredLeaf(t *testing.T, hubPort int, certStore string, matchBy string, match string, caMatch string, expectedLeafCount int) {\n\n\t// Fire up the leaf\n\tu, err := url.Parse(fmt.Sprintf(\"nats://localhost:%d\", hubPort))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\n\tconfigStr := fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tcert_store: \"%s\"\n\t\t\t\t\t\tcert_match_by: \"%s\"\n\t\t\t\t\t\tcert_match: \"%s\"\n\t\t\t\t\t\tca_certs_match: %s\n\n\t\t\t\t\t\t# Test settings that succeed should be equivalent to:\n\t\t\t\t\t\t# cert_file: \"../test/configs/certs/tlsauth/client.pem\"\n\t\t\t\t\t\t# key_file: \"../test/configs/certs/tlsauth/client-key.pem\"\n\t\t\t\t\t\t# ca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String(), certStore, matchBy, match, caMatch)\n\n\tleafConfig := createConfFile(t, []byte(configStr))\n\tdefer removeFile(t, leafConfig)\n\tleafServer, _ := RunServerWithConfig(leafConfig)\n\tdefer leafServer.Shutdown()\n\n\t// After client verify, hub will match by SAN email, SAN dns, and Subject (in that order)\n\t// Our test client specifies Subject only so we should match on that...\n\n\t// A little settle time\n\ttime.Sleep(1 * time.Second)\n\tcheckLeafNodeConnectedCount(t, leafServer, expectedLeafCount)\n}\n\n// TestLeafTLSWindowsCertStore tests the topology of two NATS Servers connected as leaf and hub with authentication of\n// leaf to hub via mTLS with leaf's certificate and signing key provisioned in the Windows certificate store.\nfunc TestLeafTLSWindowsCertStore(t *testing.T) {\n\n\t// Client Identity (client.pem)\n\t// Issuer: O = NATS CA, OU = NATS.io, CN = localhost\n\t// Subject: OU = NATS.io, CN = example.com\n\n\t// Make sure windows cert store is reset to avoid conflict with other tests\n\terr := runPowershellScript(\"../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell cert delete to succeed: %s\", err.Error())\n\t}\n\n\t// Provision Windows cert store with client cert and secret\n\terr = runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-client.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell provision to succeed: %s\", err.Error())\n\t}\n\n\terr = runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell provision CA to succeed: %s\", err.Error())\n\t}\n\n\t// Fire up the hub\n\thubConfig := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify_and_map: true\n\t\t\t}\n\t\t}\n\n\t\taccounts: {\n\t\t\tAcctA: {\n\t\t\t users: [ {user: \"OU = NATS.io, CN = example.com\"} ]\n\t\t\t},\n\t\t\tAcctB: {\n\t\t\t users: [ {user: UserB1} ]\n\t\t\t},\n\t\t\tSYS: {\n\t\t\t\tusers: [ {user: System} ]\n\t\t\t}\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`))\n\tdefer removeFile(t, hubConfig)\n\thubServer, hubOptions := RunServerWithConfig(hubConfig)\n\tdefer hubServer.Shutdown()\n\n\ttestCases := []struct {\n\t\tcertStore string\n\t\tcertMatchBy string\n\t\tcertMatch string\n\t\tcaCertsMatch string\n\t\texpectedLeafCount int\n\t}{\n\t\t// Test subject and issuer\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"\\\"NATS CA\\\"\", 1},\n\t\t{\"WindowsCurrentUser\", \"Issuer\", \"NATS CA\", \"\\\"NATS CA\\\"\", 1},\n\t\t{\"WindowsCurrentUser\", \"Issuer\", \"Frodo Baggins, Inc.\", \"\\\"NATS CA\\\"\", 0},\n\t\t{\"WindowsCurrentUser\", \"Thumbprint\", \"7e44f478114a2e29b98b00beb1b3687d8dc0e481\", \"\\\"NATS CA\\\"\", 0},\n\t\t// Test CAs, NATS CA is valid, others are missing\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"NATS CA\\\"]\", 1},\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"GlobalSign\\\"]\", 0},\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"Missing NATS Cert\\\"]\", 0},\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"NATS CA\\\", \\\"Missing NATS Cert1\\\"]\", 1},\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"Missing Cert2\\\",\\\"NATS CA\\\"]\", 1},\n\t\t{\"WindowsCurrentUser\", \"Subject\", \"example.com\", \"[\\\"Missing, Cert3\\\",\\\"Missing NATS Cert4\\\"]\", 0},\n\t}\n\tfor _, tc := range testCases {\n\t\ttestName := fmt.Sprintf(\"%s by %s match %s\", tc.certStore, tc.certMatchBy, tc.certMatch)\n\t\tt.Run(fmt.Sprintf(testName, tc.certStore, tc.certMatchBy, tc.certMatch, tc.caCertsMatch), func(t *testing.T) {\n\t\t\tdefer func() {\n\t\t\t\tif r := recover(); r != nil {\n\t\t\t\t\tif tc.expectedLeafCount != 0 {\n\t\t\t\t\t\tt.Fatalf(\"did not expect panic: %s\", testName)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif !strings.Contains(fmt.Sprintf(\"%v\", r), \"Error processing configuration file\") {\n\t\t\t\t\t\t\tt.Fatalf(\"did not expect unknown panic: %s\", testName)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t\trunConfiguredLeaf(t, hubOptions.LeafNode.Port,\n\t\t\t\ttc.certStore, tc.certMatchBy, tc.certMatch,\n\t\t\t\ttc.caCertsMatch, tc.expectedLeafCount)\n\t\t})\n\t}\n}\n\n// TestServerTLSWindowsCertStore tests the topology of a NATS server requiring TLS and gettings it own server\n// cert identity (as used when accepting NATS client connections and negotiating TLS) from Windows certificate store.\nfunc TestServerTLSWindowsCertStore(t *testing.T) {\n\n\t// Server Identity (server.pem)\n\t// Issuer: O = NATS CA, OU = NATS.io, CN = localhost\n\t// Subject: OU = NATS.io Operators, CN = localhost\n\n\t// Make sure windows cert store is reset to avoid conflict with other tests\n\terr := runPowershellScript(\"../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell cert delete to succeed: %s\", err.Error())\n\t}\n\n\t// Provision Windows cert store with server cert and secret\n\terr = runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-server.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell provision to succeed: %s\", err.Error())\n\t}\n\n\terr = runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-ca.ps1\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell provision CA to succeed: %s\", err.Error())\n\t}\n\n\t// Fire up the server\n\tsrvConfig := createConfFile(t, []byte(`\n\tlisten: \"localhost:-1\"\n\ttls {\n\t\tcert_store: \"WindowsCurrentUser\"\n\t\tcert_match_by: \"Subject\"\n\t\tcert_match: \"NATS.io Operators\"\n\t\tca_certs_match: [\"NATS CA\"]\n\t\ttimeout: 5\n\t}\n\t`))\n\tdefer removeFile(t, srvConfig)\n\tsrvServer, _ := RunServerWithConfig(srvConfig)\n\tif srvServer == nil {\n\t\tt.Fatalf(\"expected to be able start server with cert store configuration\")\n\t}\n\tdefer srvServer.Shutdown()\n\n\ttestCases := []struct {\n\t\tclientCA string\n\t\texpect bool\n\t}{\n\t\t{\"../test/configs/certs/tlsauth/ca.pem\", true},\n\t\t{\"../test/configs/certs/tlsauth/client.pem\", false},\n\t}\n\tfor _, tc := range testCases {\n\t\tt.Run(fmt.Sprintf(\"Client CA: %s\", tc.clientCA), func(t *testing.T) {\n\t\t\tnc, _ := nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA))\n\t\t\terr := nc.Publish(\"foo\", []byte(\"hello TLS server-authenticated server\"))\n\t\t\tif (err != nil) == tc.expect {\n\t\t\t\tt.Fatalf(\"expected publish result %v to TLS authenticated server\", tc.expect)\n\t\t\t}\n\t\t\tnc.Close()\n\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\tnc, _ = nats.Connect(srvServer.clientConnectURLs[0], nats.RootCAs(tc.clientCA))\n\t\t\t\terr = nc.Publish(\"foo\", []byte(\"hello TLS server-authenticated server\"))\n\t\t\t\tif (err != nil) == tc.expect {\n\t\t\t\t\tt.Fatalf(\"expected repeated connection result %v to TLS authenticated server\", tc.expect)\n\t\t\t\t}\n\t\t\t\tnc.Close()\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestServerIgnoreExpiredCerts tests if the server skips expired certificates in configuration, and finds non-expired ones\nfunc TestServerIgnoreExpiredCerts(t *testing.T) {\n\n\t// Server Identities: expired.pem; not-expired.pem\n\t// Issuer: OU = NATS.io, CN = localhost\n\t// Subject: OU = NATS.io Operators, CN = localhost\n\n\ttestCases := []struct {\n\t\tcertFile string\n\t\texpect bool\n\t}{\n\t\t{\"expired.p12\", false},\n\t\t{\"not-expired.p12\", true},\n\t}\n\tfor _, tc := range testCases {\n\t\tt.Run(fmt.Sprintf(\"Server certificate: %s\", tc.certFile), func(t *testing.T) {\n\t\t\t// Make sure windows cert store is reset to avoid conflict with other tests\n\t\t\terr := runPowershellScript(\"../test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1\", nil)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"expected powershell cert delete to succeed: %s\", err.Error())\n\t\t\t}\n\n\t\t\t// Provision Windows cert store with server cert and secret\n\t\t\terr = runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-server.ps1\", []string{tc.certFile})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"expected powershell provision to succeed: %s\", err.Error())\n\t\t\t}\n\t\t\t// Fire up the server\n\t\t\tsrvConfig := createConfFile(t, []byte(`\n\t\t\tlisten: \"localhost:-1\"\n\t\t\ttls {\n\t\t\t\tcert_store: \"WindowsCurrentUser\"\n\t\t\t\tcert_match_by: \"Subject\"\n\t\t\t\tcert_match: \"NATS.io Operators\"\n\t\t\t\tcert_match_skip_invalid: true\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t\t`))\n\t\t\tdefer removeFile(t, srvConfig)\n\t\t\tcfg, _ := ProcessConfigFile(srvConfig)\n\t\t\tif (cfg != nil) == tc.expect {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif tc.expect == false {\n\t\t\t\tt.Fatalf(\"expected server start to fail with expired certificate\")\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"expected server to start with non expired certificate\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWindowsTLS12ECDSA(t *testing.T) {\n\terr := runPowershellScript(\"../test/configs/certs/tlsauth/certstore/import-p12-server.ps1\", []string{\"ecdsa_server.pfx\"})\n\tif err != nil {\n\t\tt.Fatalf(\"expected powershell provision to succeed: %v\", err)\n\t}\n\n\tconfig := createConfFile(t, []byte(`\n\tlisten: \"localhost:-1\"\n\ttls {\n\t\tcert_store: \"WindowsCurrentUser\"\n\t\tcert_match_by: \"Thumbprint\"\n\t\tcert_match: \"4F8AF21756E5DBBD54619BBB6F3CC5D455ED4468\"\n\t\tcert_match_skip_invalid: true\n\t\ttimeout: 5\n\t}\n\t`))\n\tdefer removeFile(t, config)\n\n\tsrv, _ := RunServerWithConfig(config)\n\tif srv == nil {\n\t\tt.Fatalf(\"expected to be able start server with cert store configuration\")\n\t}\n\tdefer srv.Shutdown()\n\n\tfor name, version := range map[string]uint16{\n\t\t\"TLS 1.3\": tls.VersionTLS13,\n\t\t\"TLS 1.2\": tls.VersionTLS12,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\ttc := &tls.Config{MaxVersion: version, MinVersion: version, InsecureSkipVerify: true}\n\n\t\t\tif _, err = nats.Connect(srv.clientConnectURLs[0], nats.Secure(tc)); err != nil {\n\t\t\t\tt.Fatalf(\"connection with %s: %v\", name, err)\n\t\t\t}\n\n\t\t\tt.Logf(\"successful connection with %s\", name)\n\t\t})\n\t}\n}\n" }, { "path": "server/ciphersuites.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"crypto/fips140\"\n\t\"crypto/tls\"\n)\n\nfunc init() {\n\tfor _, cs := range tls.CipherSuites() {\n\t\tcipherMap[cs.Name] = cs\n\t\tcipherMapByID[cs.ID] = cs\n\t}\n\tfor _, cs := range tls.InsecureCipherSuites() {\n\t\tcipherMap[cs.Name] = cs\n\t\tcipherMapByID[cs.ID] = cs\n\t}\n}\n\nvar cipherMap = map[string]*tls.CipherSuite{}\nvar cipherMapByID = map[uint16]*tls.CipherSuite{}\n\nfunc defaultCipherSuites() []uint16 {\n\tciphers := tls.CipherSuites()\n\tdefaults := make([]uint16, 0, len(ciphers))\n\tfor _, cs := range ciphers {\n\t\tdefaults = append(defaults, cs.ID)\n\t}\n\treturn defaults\n}\n\n// Where we maintain available curve preferences\nvar curvePreferenceMap = map[string]tls.CurveID{\n\t\"X25519MLKEM768\": tls.X25519MLKEM768,\n\t\"X25519\": tls.X25519,\n\t\"CurveP256\": tls.CurveP256,\n\t\"CurveP384\": tls.CurveP384,\n\t\"CurveP521\": tls.CurveP521,\n}\n\n// reorder to default to the highest level of security. See:\n// https://blog.bracebin.com/achieving-perfect-ssl-labs-score-with-go\nfunc defaultCurvePreferences() []tls.CurveID {\n\tif fips140.Enabled() {\n\t\t// X25519 is not FIPS-approved by itself, but it is when\n\t\t// combined with MLKEM768.\n\t\treturn []tls.CurveID{\n\t\t\ttls.X25519MLKEM768, // post-quantum\n\t\t\ttls.CurveP256,\n\t\t\ttls.CurveP384,\n\t\t\ttls.CurveP521,\n\t\t}\n\t}\n\treturn []tls.CurveID{\n\t\ttls.X25519MLKEM768, // post-quantum\n\t\ttls.X25519, // faster than P256, arguably more secure\n\t\ttls.CurveP256,\n\t\ttls.CurveP384,\n\t\ttls.CurveP521,\n\t}\n}\n" }, { "path": "server/client.go", "content": "// Copyright 2012-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n)\n\n// Type of client connection.\nconst (\n\t// CLIENT is an end user.\n\tCLIENT = iota\n\t// ROUTER represents another server in the cluster.\n\tROUTER\n\t// GATEWAY is a link between 2 clusters.\n\tGATEWAY\n\t// SYSTEM is an internal system client.\n\tSYSTEM\n\t// LEAF is for leaf node connections.\n\tLEAF\n\t// JETSTREAM is an internal jetstream client.\n\tJETSTREAM\n\t// ACCOUNT is for the internal client for accounts.\n\tACCOUNT\n)\n\n// Internal clients. kind should be SYSTEM, JETSTREAM or ACCOUNT\nfunc isInternalClient(kind int) bool {\n\treturn kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT\n}\n\n// Extended type of a CLIENT connection. This is returned by c.clientType()\n// and indicate what type of client connection we are dealing with.\n// If invoked on a non CLIENT connection, NON_CLIENT type is returned.\nconst (\n\t// If the connection is not a CLIENT connection.\n\tNON_CLIENT = iota\n\t// Regular NATS client.\n\tNATS\n\t// MQTT client.\n\tMQTT\n\t// Websocket client.\n\tWS\n)\n\nconst (\n\t// ClientProtoZero is the original Client protocol from 2009.\n\t// http://nats.io/documentation/internals/nats-protocol/\n\tClientProtoZero = iota\n\t// ClientProtoInfo signals a client can receive more then the original INFO block.\n\t// This can be used to update clients on other cluster members, etc.\n\tClientProtoInfo\n)\n\nconst (\n\tpingProto = \"PING\" + _CRLF_\n\tpongProto = \"PONG\" + _CRLF_\n\terrProto = \"-ERR '%s'\" + _CRLF_\n\tokProto = \"+OK\" + _CRLF_\n)\n\n// TLS Hanshake client types\nconst (\n\ttlsHandshakeLeaf = \"leafnode\"\n\ttlsHandshakeMQTT = \"mqtt\"\n)\n\nconst (\n\t// Scratch buffer size for the processMsg() calls.\n\tmsgScratchSize = 1024\n\tmsgHeadProto = \"RMSG \"\n\tmsgHeadProtoLen = len(msgHeadProto)\n\n\t// For controlling dynamic buffer sizes.\n\tstartBufSize = 512 // For INFO/CONNECT block\n\tminBufSize = 64 // Smallest to shrink to for PING/PONG\n\tmaxBufSize = 65536 // 64k\n\tshortsToShrink = 2 // Trigger to shrink dynamic buffers\n\tmaxFlushPending = 10 // Max fsps to have in order to wait for writeLoop\n\treadLoopReport = 2 * time.Second\n\n\t// Server should not send a PING (for RTT) before the first PONG has\n\t// been sent to the client. However, in case some client libs don't\n\t// send CONNECT+PING, cap the maximum time before server can send\n\t// the RTT PING.\n\tmaxNoRTTPingBeforeFirstPong = 2 * time.Second\n\n\t// For stalling fast producers\n\tstallClientMinDuration = 2 * time.Millisecond\n\tstallClientMaxDuration = 5 * time.Millisecond\n\tstallTotalAllowed = 10 * time.Millisecond\n)\n\nvar readLoopReportThreshold = readLoopReport\n\n// Represent client booleans with a bitmask\ntype clientFlag uint16\n\nconst (\n\thdrLine = \"NATS/1.0\\r\\n\"\n\temptyHdrLine = \"NATS/1.0\\r\\n\\r\\n\"\n)\n\n// Some client state represented as flags\nconst (\n\tconnectReceived clientFlag = 1 << iota // The CONNECT proto has been received\n\tinfoReceived // The INFO protocol has been received\n\tfirstPongSent // The first PONG has been sent\n\thandshakeComplete // For TLS clients, indicate that the handshake is complete\n\tflushOutbound // Marks client as having a flushOutbound call in progress.\n\tnoReconnect // Indicate that on close, this connection should not attempt a reconnect\n\tcloseConnection // Marks that closeConnection has already been called.\n\tconnMarkedClosed // Marks that markConnAsClosed has already been called.\n\twriteLoopStarted // Marks that the writeLoop has been started.\n\tskipFlushOnClose // Marks that flushOutbound() should not be called on connection close.\n\texpectConnect // Marks if this connection is expected to send a CONNECT\n\tconnectProcessFinished // Marks if this connection has finished the connect process.\n\tcompressionNegotiated // Marks if this connection has negotiated compression level with remote.\n\tdidTLSFirst // Marks if this connection requested and was accepted doing the TLS handshake first (prior to INFO).\n\tisSlowConsumer // Marks connection as a slow consumer.\n)\n\n// set the flag (would be equivalent to set the boolean to true)\nfunc (cf *clientFlag) set(c clientFlag) {\n\t*cf |= c\n}\n\n// clear the flag (would be equivalent to set the boolean to false)\nfunc (cf *clientFlag) clear(c clientFlag) {\n\t*cf &= ^c\n}\n\n// isSet returns true if the flag is set, false otherwise\nfunc (cf clientFlag) isSet(c clientFlag) bool {\n\treturn cf&c != 0\n}\n\n// setIfNotSet will set the flag `c` only if that flag was not already\n// set and return true to indicate that the flag has been set. Returns\n// false otherwise.\nfunc (cf *clientFlag) setIfNotSet(c clientFlag) bool {\n\tif *cf&c == 0 {\n\t\t*cf |= c\n\t\treturn true\n\t}\n\treturn false\n}\n\n// ClosedState is the reason client was closed. This will\n// be passed into calls to clearConnection, but will only\n// be stored in ConnInfo for monitoring.\ntype ClosedState int\n\nconst (\n\tClientClosed = ClosedState(iota + 1)\n\tAuthenticationTimeout\n\tAuthenticationViolation\n\tTLSHandshakeError\n\tSlowConsumerPendingBytes\n\tSlowConsumerWriteDeadline\n\tWriteError\n\tReadError\n\tParseError\n\tStaleConnection\n\tProtocolViolation\n\tBadClientProtocolVersion\n\tWrongPort\n\tMaxAccountConnectionsExceeded\n\tMaxConnectionsExceeded\n\tMaxPayloadExceeded\n\tMaxControlLineExceeded\n\tMaxSubscriptionsExceeded\n\tDuplicateRoute\n\tRouteRemoved\n\tServerShutdown\n\tAuthenticationExpired\n\tWrongGateway\n\tMissingAccount\n\tRevocation\n\tInternalClient\n\tMsgHeaderViolation\n\tNoRespondersRequiresHeaders\n\tClusterNameConflict\n\tDuplicateRemoteLeafnodeConnection\n\tDuplicateClientID\n\tDuplicateServerName\n\tMinimumVersionRequired\n\tClusterNamesIdentical\n\tKicked\n\tProxyNotTrusted\n\tProxyRequired\n)\n\n// Some flags passed to processMsgResults\nconst pmrNoFlag int = 0\nconst (\n\tpmrCollectQueueNames int = 1 << iota\n\tpmrIgnoreEmptyQueueFilter\n\tpmrAllowSendFromRouteToRoute\n\tpmrMsgImportedFromService\n)\n\ntype WriteTimeoutPolicy uint8\n\nconst (\n\tWriteTimeoutPolicyDefault = iota\n\tWriteTimeoutPolicyClose\n\tWriteTimeoutPolicyRetry\n)\n\n// String returns a human-friendly value. Only used in varz.\nfunc (p WriteTimeoutPolicy) String() string {\n\tswitch p {\n\tcase WriteTimeoutPolicyClose:\n\t\treturn \"close\"\n\tcase WriteTimeoutPolicyRetry:\n\t\treturn \"retry\"\n\tdefault:\n\t\treturn _EMPTY_\n\t}\n}\n\ntype client struct {\n\t// Here first because of use of atomics, and memory alignment.\n\tstats\n\tgwReplyMapping\n\tkind int\n\tsrv *Server\n\tacc *Account\n\tperms *permissions\n\tin readCache\n\tparseState\n\topts ClientOpts\n\trrTracking *rrTracking\n\tmpay int32\n\tmsubs int32\n\tmcl int32\n\tmu sync.Mutex\n\tcid uint64\n\tstart time.Time\n\tnonce []byte\n\tpubKey string\n\tnc net.Conn\n\tncs atomic.Value\n\tncsAcc atomic.Value\n\tncsUser atomic.Value\n\tout outbound\n\tuser *NkeyUser\n\thost string\n\tport uint16\n\tsubs map[string]*subscription\n\treplies map[string]*resp\n\tmperms *msgDeny\n\tdarray []string\n\tpcd map[*client]struct{}\n\tatmr *time.Timer\n\texpires time.Time\n\tping pinfo\n\tmsgb [msgScratchSize]byte\n\tlast time.Time\n\tlastIn time.Time\n\tproxyKey string\n\n\trepliesSincePrune uint16\n\tlastReplyPrune time.Time\n\n\theaders bool\n\n\trtt time.Duration\n\trttStart time.Time\n\n\troute *route\n\tgw *gateway\n\tleaf *leaf\n\tws *websocket\n\tmqtt *mqtt\n\n\tflags clientFlag // Compact booleans into a single field. Size will be increased when needed.\n\n\trref byte\n\n\ttrace bool\n\techo bool\n\tnoIcb bool\n\tiproc bool // In-Process connection, set at creation and immutable.\n\n\ttags jwt.TagList\n\tnameTag string\n\n\ttlsTo *time.Timer\n\n\t// Authentication error override. This is used because the authentication\n\t// stack is simply returning a boolean, and the only authentication error\n\t// reported is the generic `ErrAuthentication`. In the authentication code,\n\t// if we want to report a different error, we can now set this field\n\t// and `authViolation()` will use that one.\n\tauthErr error\n}\n\ntype rrTracking struct {\n\trmap map[string]*remoteLatency\n\tptmr *time.Timer\n\tlrt time.Duration\n}\n\n// Struct for PING initiation from the server.\ntype pinfo struct {\n\ttmr *time.Timer\n\tout int\n}\n\n// outbound holds pending data for a socket.\ntype outbound struct {\n\tnb net.Buffers // Pending buffers for send, each has fixed capacity as per nbPool below.\n\twnb net.Buffers // Working copy of \"nb\", reused on each flushOutbound call, partial writes may leave entries here for next iteration.\n\tpb int64 // Total pending/queued bytes.\n\tfsp int32 // Flush signals that are pending per producer from readLoop's pcd.\n\twtp WriteTimeoutPolicy // What do we do on a write timeout?\n\tsg *sync.Cond // To signal writeLoop that there is data to flush.\n\twdl time.Duration // Snapshot of write deadline.\n\tmp int64 // Snapshot of max pending for client.\n\tlft time.Duration // Last flush time for Write.\n\tstc chan struct{} // Stall chan we create to slow down producers on overrun, e.g. fan-in.\n\tcw *s2.Writer\n}\n\nconst nbMaxVectorSize = 1024 // == IOV_MAX on Linux/Darwin and most other Unices (except Solaris/AIX)\n\nconst nbPoolSizeSmall = 512 // Underlying array size of small buffer\nconst nbPoolSizeMedium = 4096 // Underlying array size of medium buffer\nconst nbPoolSizeLarge = 65536 // Underlying array size of large buffer\n\nvar nbPoolSmall = &sync.Pool{\n\tNew: func() any {\n\t\tb := [nbPoolSizeSmall]byte{}\n\t\treturn &b\n\t},\n}\n\nvar nbPoolMedium = &sync.Pool{\n\tNew: func() any {\n\t\tb := [nbPoolSizeMedium]byte{}\n\t\treturn &b\n\t},\n}\n\nvar nbPoolLarge = &sync.Pool{\n\tNew: func() any {\n\t\tb := [nbPoolSizeLarge]byte{}\n\t\treturn &b\n\t},\n}\n\n// nbPoolGet returns a frame that is a best-effort match for the given size.\n// Once a pooled frame is no longer needed, it should be recycled by passing\n// it to nbPoolPut.\nfunc nbPoolGet(sz int) []byte {\n\tswitch {\n\tcase sz <= nbPoolSizeSmall:\n\t\treturn nbPoolSmall.Get().(*[nbPoolSizeSmall]byte)[:0]\n\tcase sz <= nbPoolSizeMedium:\n\t\treturn nbPoolMedium.Get().(*[nbPoolSizeMedium]byte)[:0]\n\tdefault:\n\t\treturn nbPoolLarge.Get().(*[nbPoolSizeLarge]byte)[:0]\n\t}\n}\n\n// nbPoolPut recycles a frame that was retrieved from nbPoolGet. It is not\n// safe to return multiple slices referring to chunks of the same underlying\n// array as this may create overlaps when the buffers are returned to their\n// original size, resulting in race conditions.\nfunc nbPoolPut(b []byte) {\n\tswitch cap(b) {\n\tcase nbPoolSizeSmall:\n\t\tb := (*[nbPoolSizeSmall]byte)(b[0:nbPoolSizeSmall])\n\t\tnbPoolSmall.Put(b)\n\tcase nbPoolSizeMedium:\n\t\tb := (*[nbPoolSizeMedium]byte)(b[0:nbPoolSizeMedium])\n\t\tnbPoolMedium.Put(b)\n\tcase nbPoolSizeLarge:\n\t\tb := (*[nbPoolSizeLarge]byte)(b[0:nbPoolSizeLarge])\n\t\tnbPoolLarge.Put(b)\n\tdefault:\n\t\t// Ignore frames that are the wrong size, this might happen\n\t\t// with WebSocket/MQTT messages as they are framed\n\t}\n}\n\ntype perm struct {\n\tallow *Sublist\n\tdeny *Sublist\n}\n\ntype permissions struct {\n\t// Have these 2 first for memory alignment due to the use of atomic.\n\tpcsz int32\n\tprun int32\n\tsub perm\n\tpub perm\n\tresp *ResponsePermission\n\tpcache sync.Map\n}\n\n// This is used to dynamically track responses and reply subjects\n// for dynamic permissioning.\ntype resp struct {\n\tt time.Time\n\tn int\n}\n\n// msgDeny is used when a user permission for subscriptions has a deny\n// clause but a subscription could be made that is of broader scope.\n// e.g. deny = \"foo\", but user subscribes to \"*\". That subscription should\n// succeed but no message sent on foo should be delivered.\ntype msgDeny struct {\n\tdeny *Sublist\n\tdcache map[string]bool\n}\n\n// routeTarget collects information regarding routes and queue groups for\n// sending information to a remote.\ntype routeTarget struct {\n\tsub *subscription\n\tqs []byte\n\t_qs [32]byte\n}\n\nconst (\n\tmaxResultCacheSize = 512\n\tmaxDenyPermCacheSize = 256\n\tmaxPermCacheSize = 128\n\tpruneSize = 32\n\trouteTargetInit = 8\n\treplyPermLimit = 4096\n\treplyPruneTime = time.Second\n)\n\n// Represent read cache booleans with a bitmask\ntype readCacheFlag uint16\n\nconst (\n\thasMappings readCacheFlag = 1 << iota // For account subject mappings.\n\tswitchToCompression readCacheFlag = 1 << 1\n)\n\nconst sysGroup = \"_sys_\"\n\n// Used in readloop to cache hot subject lookups and group statistics.\ntype readCache struct {\n\t// These are for clients who are bound to a single account.\n\tgenid uint64\n\tresults map[string]*SublistResult\n\n\t// This is for routes and gateways to have their own L1 as well that is account aware.\n\tpacache map[string]*perAccountCache\n\n\t// This is for when we deliver messages across a route. We use this structure\n\t// to make sure to only send one message and properly scope to queues as needed.\n\trts []routeTarget\n\n\t// These are all temporary totals for an invocation of a read in readloop.\n\tmsgs int32\n\tbytes int32\n\tsubs int32\n\n\trsz int32 // Read buffer size\n\tsrs int32 // Short reads, used for dynamic buffer resizing.\n\n\t// These are for readcache flags to avoid locks.\n\tflags readCacheFlag\n\n\t// Capture the time we started processing our readLoop.\n\tstart time.Time\n\n\t// Total time stalled so far for readLoop processing.\n\ttst time.Duration\n}\n\n// set the flag (would be equivalent to set the boolean to true)\nfunc (rcf *readCacheFlag) set(c readCacheFlag) {\n\t*rcf |= c\n}\n\n// clear the flag (would be equivalent to set the boolean to false)\nfunc (rcf *readCacheFlag) clear(c readCacheFlag) {\n\t*rcf &= ^c\n}\n\n// isSet returns true if the flag is set, false otherwise\nfunc (rcf readCacheFlag) isSet(c readCacheFlag) bool {\n\treturn rcf&c != 0\n}\n\nconst (\n\tdefaultMaxPerAccountCacheSize = 8192\n\tdefaultClosedSubsCheckInterval = 5 * time.Minute\n)\n\nvar (\n\tmaxPerAccountCacheSize = defaultMaxPerAccountCacheSize\n\tclosedSubsCheckInterval = defaultClosedSubsCheckInterval\n)\n\n// perAccountCache is for L1 semantics for inbound messages from a route or gateway to mimic the performance of clients.\ntype perAccountCache struct {\n\tacc *Account\n\tresults *SublistResult\n\tgenid uint64\n}\n\nfunc (c *client) String() (id string) {\n\tloaded := c.ncs.Load()\n\tif loaded != nil {\n\t\treturn loaded.(string)\n\t}\n\n\treturn _EMPTY_\n}\n\n// GetNonce returns the nonce that was presented to the user on connection\nfunc (c *client) GetNonce() []byte {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\treturn c.nonce\n}\n\n// GetName returns the application supplied name for the connection.\nfunc (c *client) GetName() string {\n\tc.mu.Lock()\n\tname := c.opts.Name\n\tc.mu.Unlock()\n\treturn name\n}\n\n// GetOpts returns the client options provided by the application.\nfunc (c *client) GetOpts() *ClientOpts {\n\treturn &c.opts\n}\n\n// GetTLSConnectionState returns the TLS ConnectionState if TLS is enabled, nil\n// otherwise. Implements the ClientAuth interface.\nfunc (c *client) GetTLSConnectionState() *tls.ConnectionState {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif c.nc == nil {\n\t\treturn nil\n\t}\n\ttc, ok := c.nc.(*tls.Conn)\n\tif !ok {\n\t\treturn nil\n\t}\n\tstate := tc.ConnectionState()\n\treturn &state\n}\n\n// For CLIENT connections, this function returns the client type, that is,\n// NATS (for regular clients), MQTT or WS for websocket.\n// If this is invoked for a non CLIENT connection, NON_CLIENT is returned.\n//\n// This function does not lock the client and accesses fields that are supposed\n// to be immutable and therefore it can be invoked outside of the client's lock.\nfunc (c *client) clientType() int {\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tif c.isMqtt() {\n\t\t\treturn MQTT\n\t\t} else if c.isWebsocket() {\n\t\t\treturn WS\n\t\t}\n\t\treturn NATS\n\tdefault:\n\t\treturn NON_CLIENT\n\t}\n}\n\nvar clientTypeStringMap = map[int]string{\n\tNON_CLIENT: _EMPTY_,\n\tNATS: \"nats\",\n\tWS: \"websocket\",\n\tMQTT: \"mqtt\",\n}\n\nfunc (c *client) clientTypeString() string {\n\tif typeStringVal, ok := clientTypeStringMap[c.clientType()]; ok {\n\t\treturn typeStringVal\n\t}\n\treturn _EMPTY_\n}\n\n// This is the main subscription struct that indicates\n// interest in published messages.\n// FIXME(dlc) - This is getting bloated for normal subs, need\n// to optionally have an opts section for non-normal stuff.\ntype subscription struct {\n\tclient *client\n\tim *streamImport // This is for import stream support.\n\trsi bool\n\tsi bool\n\tshadow []*subscription // This is to track shadowed accounts.\n\ticb msgHandler\n\tsubject []byte\n\tqueue []byte\n\tsid []byte\n\torigin []byte\n\tnm int64\n\tmax int64\n\tqw int32\n\tclosed int32\n\tmqtt *mqttSub\n}\n\n// Indicate that this subscription is closed.\n// This is used in pruning of route and gateway cache items.\nfunc (s *subscription) close() {\n\tatomic.StoreInt32(&s.closed, 1)\n}\n\n// Return true if this subscription was unsubscribed\n// or its connection has been closed.\nfunc (s *subscription) isClosed() bool {\n\treturn atomic.LoadInt32(&s.closed) == 1\n}\n\ntype ClientOpts struct {\n\tEcho bool `json:\"echo\"`\n\tVerbose bool `json:\"verbose\"`\n\tPedantic bool `json:\"pedantic\"`\n\tTLSRequired bool `json:\"tls_required\"`\n\tNkey string `json:\"nkey,omitempty\"`\n\tJWT string `json:\"jwt,omitempty\"`\n\tSig string `json:\"sig,omitempty\"`\n\tToken string `json:\"auth_token,omitempty\"`\n\tUsername string `json:\"user,omitempty\"`\n\tPassword string `json:\"pass,omitempty\"`\n\tName string `json:\"name\"`\n\tLang string `json:\"lang\"`\n\tVersion string `json:\"version\"`\n\tProtocol int `json:\"protocol\"`\n\tAccount string `json:\"account,omitempty\"`\n\tAccountNew bool `json:\"new_account,omitempty\"`\n\tHeaders bool `json:\"headers,omitempty\"`\n\tNoResponders bool `json:\"no_responders,omitempty\"`\n\n\t// Routes and Leafnodes only\n\tImport *SubjectPermission `json:\"import,omitempty\"`\n\tExport *SubjectPermission `json:\"export,omitempty\"`\n\n\t// Leafnodes\n\tRemoteAccount string `json:\"remote_account,omitempty\"`\n\n\t// Proxy would include its own nonce signature.\n\tProxySig string `json:\"proxy_sig,omitempty\"`\n}\n\nvar defaultOpts = ClientOpts{Verbose: true, Pedantic: true, Echo: true}\nvar internalOpts = ClientOpts{Verbose: false, Pedantic: false, Echo: false}\n\nfunc (c *client) setTraceLevel() {\n\tif c.kind == SYSTEM && !(atomic.LoadInt32(&c.srv.logging.traceSysAcc) != 0) {\n\t\tc.trace = false\n\t} else {\n\t\tc.trace = (atomic.LoadInt32(&c.srv.logging.trace) != 0)\n\t}\n}\n\n// Lock should be held\nfunc (c *client) initClient() {\n\ts := c.srv\n\tc.cid = atomic.AddUint64(&s.gcid, 1)\n\n\t// Outbound data structure setup\n\tc.out.sg = sync.NewCond(&(c.mu))\n\topts := s.getOpts()\n\t// Snapshots to avoid mutex access in fast paths.\n\tc.out.wdl = opts.WriteDeadline\n\tswitch {\n\tcase c.kind == ROUTER && opts.Cluster.WriteDeadline > 0:\n\t\tc.out.wdl = opts.Cluster.WriteDeadline\n\tcase c.kind == GATEWAY && opts.Gateway.WriteDeadline > 0:\n\t\tc.out.wdl = opts.Gateway.WriteDeadline\n\tcase c.kind == LEAF && opts.LeafNode.WriteDeadline > 0:\n\t\tc.out.wdl = opts.LeafNode.WriteDeadline\n\t}\n\tswitch c.kind {\n\tcase ROUTER:\n\t\tif c.out.wtp = opts.Cluster.WriteTimeout; c.out.wtp == WriteTimeoutPolicyDefault {\n\t\t\tc.out.wtp = WriteTimeoutPolicyRetry\n\t\t}\n\tcase LEAF:\n\t\tif c.out.wtp = opts.LeafNode.WriteTimeout; c.out.wtp == WriteTimeoutPolicyDefault {\n\t\t\tc.out.wtp = WriteTimeoutPolicyRetry\n\t\t}\n\tcase GATEWAY:\n\t\tif c.out.wtp = opts.Gateway.WriteTimeout; c.out.wtp == WriteTimeoutPolicyDefault {\n\t\t\tc.out.wtp = WriteTimeoutPolicyRetry\n\t\t}\n\tdefault:\n\t\tif c.out.wtp = opts.WriteTimeout; c.out.wtp == WriteTimeoutPolicyDefault {\n\t\t\tc.out.wtp = WriteTimeoutPolicyClose\n\t\t}\n\t}\n\tc.out.mp = opts.MaxPending\n\t// Snapshot max control line since currently can not be changed on reload and we\n\t// were checking it on each call to parse. If this changes and we allow MaxControlLine\n\t// to be reloaded without restart, this code will need to change.\n\tc.mcl = int32(opts.MaxControlLine)\n\tif c.mcl == 0 {\n\t\tc.mcl = MAX_CONTROL_LINE_SIZE\n\t}\n\n\tc.subs = make(map[string]*subscription)\n\tc.echo = true\n\n\tc.setTraceLevel()\n\n\t// This is a scratch buffer used for processMsg()\n\t// The msg header starts with \"RMSG \", which can be used\n\t// for both local and routes.\n\t// in bytes that is [82 77 83 71 32].\n\tc.msgb = [msgScratchSize]byte{82, 77, 83, 71, 32}\n\n\t// This is to track pending clients that have data to be flushed\n\t// after we process inbound msgs from our own connection.\n\tc.pcd = make(map[*client]struct{})\n\n\t// snapshot the string version of the connection\n\tvar conn string\n\tif c.nc != nil {\n\t\tif addr := c.nc.RemoteAddr(); addr != nil {\n\t\t\tif conn = addr.String(); conn != _EMPTY_ {\n\t\t\t\thost, port, _ := net.SplitHostPort(conn)\n\t\t\t\tiPort, _ := strconv.ParseUint(port, 10, 16)\n\t\t\t\tc.host, c.port = host, uint16(iPort)\n\t\t\t\tif c.isWebsocket() && c.ws.clientIP != _EMPTY_ {\n\t\t\t\t\tcip := c.ws.clientIP\n\t\t\t\t\t// Surround IPv6 addresses with square brackets, as\n\t\t\t\t\t// net.JoinHostPort would do...\n\t\t\t\t\tif strings.Contains(cip, \":\") {\n\t\t\t\t\t\tcip = \"[\" + cip + \"]\"\n\t\t\t\t\t}\n\t\t\t\t\tconn = fmt.Sprintf(\"%s/%s\", cip, conn)\n\t\t\t\t}\n\t\t\t\t// Now that we have extracted host and port, escape\n\t\t\t\t// the string because it is going to be used in Sprintf\n\t\t\t\tconn = strings.ReplaceAll(conn, \"%\", \"%%\")\n\t\t\t}\n\t\t}\n\t}\n\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tswitch c.clientType() {\n\t\tcase NATS:\n\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - cid:%d\", conn, c.cid))\n\t\tcase WS:\n\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - wid:%d\", conn, c.cid))\n\t\tcase MQTT:\n\t\t\tvar ws string\n\t\t\tif c.isWebsocket() {\n\t\t\t\tws = \"_ws\"\n\t\t\t}\n\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - mid%s:%d\", conn, ws, c.cid))\n\t\t}\n\tcase ROUTER:\n\t\tc.ncs.Store(fmt.Sprintf(\"%s - rid:%d\", conn, c.cid))\n\tcase GATEWAY:\n\t\tc.ncs.Store(fmt.Sprintf(\"%s - gid:%d\", conn, c.cid))\n\tcase LEAF:\n\t\tvar ws string\n\t\tif c.isWebsocket() {\n\t\t\tws = \"_ws\"\n\t\t}\n\t\tc.ncs.Store(fmt.Sprintf(\"%s - lid%s:%d\", conn, ws, c.cid))\n\tcase SYSTEM:\n\t\tc.ncs.Store(\"SYSTEM\")\n\tcase JETSTREAM:\n\t\tc.ncs.Store(\"JETSTREAM\")\n\tcase ACCOUNT:\n\t\tc.ncs.Store(\"ACCOUNT\")\n\t}\n}\n\n// RemoteAddress expose the Address of the client connection,\n// nil when not connected or unknown\nfunc (c *client) RemoteAddress() net.Addr {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.nc == nil {\n\t\treturn nil\n\t}\n\n\treturn c.nc.RemoteAddr()\n}\n\n// Helper function to report errors.\nfunc (c *client) reportErrRegisterAccount(acc *Account, err error) {\n\tif err == ErrTooManyAccountConnections {\n\t\tc.maxAccountConnExceeded()\n\t\treturn\n\t}\n\tc.Errorf(\"Problem registering with account %q: %s\", acc.Name, err)\n\tc.sendErr(\"Failed Account Registration\")\n}\n\n// Kind returns the client kind and will be one of the defined constants like CLIENT, ROUTER, GATEWAY, LEAF\nfunc (c *client) Kind() int {\n\tc.mu.Lock()\n\tkind := c.kind\n\tc.mu.Unlock()\n\n\treturn kind\n}\n\n// registerWithAccount will register the given user with a specific\n// account. This will change the subject namespace.\nfunc (c *client) registerWithAccount(acc *Account) error {\n\tif acc == nil {\n\t\treturn ErrBadAccount\n\t}\n\tacc.mu.RLock()\n\tbad := acc.sl == nil\n\tacc.mu.RUnlock()\n\tif bad {\n\t\treturn ErrBadAccount\n\t}\n\t// If we were previously registered, usually to $G, do accounting here to remove.\n\tif c.acc != nil {\n\t\tif prev := c.acc.removeClient(c); prev == 1 && c.srv != nil {\n\t\t\tc.srv.decActiveAccounts()\n\t\t}\n\t}\n\n\tc.mu.Lock()\n\t// This check does not apply to SYSTEM or JETSTREAM or ACCOUNT clients (because they don't have a `nc`...)\n\tif c.isClosed() && !isInternalClient(c.kind) {\n\t\tc.mu.Unlock()\n\t\treturn ErrConnectionClosed\n\t}\n\tkind := c.kind\n\tsrv := c.srv\n\tc.acc = acc\n\tc.applyAccountLimits()\n\tc.mu.Unlock()\n\n\t// Check if we have a max connections violation\n\tif kind == CLIENT && acc.MaxTotalConnectionsReached() {\n\t\treturn ErrTooManyAccountConnections\n\t} else if kind == LEAF {\n\t\t// Check if we are already connected to this cluster.\n\t\tif rc := c.remoteCluster(); rc != _EMPTY_ && acc.hasLeafNodeCluster(rc) {\n\t\t\treturn ErrLeafNodeLoop\n\t\t}\n\t\tif acc.MaxTotalLeafNodesReached() {\n\t\t\treturn ErrTooManyAccountConnections\n\t\t}\n\t}\n\n\t// Add in new one.\n\tif prev := acc.addClient(c); prev == 0 && srv != nil {\n\t\tsrv.incActiveAccounts()\n\t}\n\n\treturn nil\n}\n\n// Helper to determine if we have met or exceeded max subs.\nfunc (c *client) subsAtLimit() bool {\n\treturn c.msubs != jwt.NoLimit && len(c.subs) >= int(c.msubs)\n}\n\nfunc minLimit(value *int32, limit int32) bool {\n\tv := atomic.LoadInt32(value)\n\tif v != jwt.NoLimit {\n\t\tif limit != jwt.NoLimit {\n\t\t\tif limit < v {\n\t\t\t\tatomic.StoreInt32(value, limit)\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t} else if limit != jwt.NoLimit {\n\t\tatomic.StoreInt32(value, limit)\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Apply account limits\n// Lock is held on entry.\n// FIXME(dlc) - Should server be able to override here?\nfunc (c *client) applyAccountLimits() {\n\tif c.acc == nil || (c.kind != CLIENT && c.kind != LEAF) {\n\t\treturn\n\t}\n\tatomic.StoreInt32(&c.mpay, jwt.NoLimit)\n\tc.msubs = jwt.NoLimit\n\tif c.opts.JWT != _EMPTY_ { // user jwt implies account\n\t\tif uc, _ := jwt.DecodeUserClaims(c.opts.JWT); uc != nil {\n\t\t\tatomic.StoreInt32(&c.mpay, clampInt64ToInt32(uc.Limits.Payload))\n\t\t\tc.msubs = clampInt64ToInt32(uc.Limits.Subs)\n\t\t\tif uc.IssuerAccount != _EMPTY_ && uc.IssuerAccount != uc.Issuer {\n\t\t\t\tif scope, ok := c.acc.signingKeys[uc.Issuer]; ok {\n\t\t\t\t\tif userScope, ok := scope.(*jwt.UserScope); ok {\n\t\t\t\t\t\t// if signing key disappeared or changed and we don't get here, the client will be disconnected\n\t\t\t\t\t\tc.mpay = clampInt64ToInt32(userScope.Template.Limits.Payload)\n\t\t\t\t\t\tc.msubs = clampInt64ToInt32(userScope.Template.Limits.Subs)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tc.acc.mu.RLock()\n\tminLimit(&c.mpay, c.acc.mpay)\n\tminLimit(&c.msubs, c.acc.msubs)\n\tc.acc.mu.RUnlock()\n\n\ts := c.srv\n\topts := s.getOpts()\n\tmPay := opts.MaxPayload\n\t// options encode unlimited differently\n\tif mPay == 0 {\n\t\tmPay = jwt.NoLimit\n\t}\n\tmSubs := int32(opts.MaxSubs)\n\tif mSubs == 0 {\n\t\tmSubs = jwt.NoLimit\n\t}\n\twasUnlimited := c.mpay == jwt.NoLimit\n\tif minLimit(&c.mpay, mPay) && !wasUnlimited {\n\t\tc.Errorf(\"Max Payload set to %d from server overrides account or user config\", opts.MaxPayload)\n\t}\n\twasUnlimited = c.msubs == jwt.NoLimit\n\tif minLimit(&c.msubs, mSubs) && !wasUnlimited {\n\t\tc.Errorf(\"Max Subscriptions set to %d from server overrides account or user config\", opts.MaxSubs)\n\t}\n\tif c.subsAtLimit() {\n\t\tgo func() {\n\t\t\tc.maxSubsExceeded()\n\t\t\ttime.Sleep(20 * time.Millisecond)\n\t\t\tc.closeConnection(MaxSubscriptionsExceeded)\n\t\t}()\n\t}\n}\n\n// RegisterUser allows auth to call back into a new client\n// with the authenticated user. This is used to map\n// any permissions into the client and setup accounts.\nfunc (c *client) RegisterUser(user *User) {\n\t// Register with proper account and sublist.\n\tif user.Account != nil {\n\t\tif err := c.registerWithAccount(user.Account); err != nil {\n\t\t\tc.reportErrRegisterAccount(user.Account, err)\n\t\t\treturn\n\t\t}\n\t}\n\n\tc.mu.Lock()\n\n\t// Assign permissions.\n\tif user.Permissions == nil {\n\t\t// Reset perms to nil in case client previously had them.\n\t\tc.perms = nil\n\t\tc.mperms = nil\n\t} else {\n\t\tc.setPermissions(user.Permissions)\n\t}\n\n\t// allows custom authenticators to set a username to be reported in\n\t// server events and more\n\tif user.Username != _EMPTY_ {\n\t\tc.opts.Username = user.Username\n\t}\n\n\t// if a deadline time stamp is set we start a timer to disconnect the user at that time\n\tif !user.ConnectionDeadline.IsZero() {\n\t\tc.setExpirationTimerUnlocked(time.Until(user.ConnectionDeadline))\n\t}\n\n\tc.mu.Unlock()\n}\n\n// RegisterNkeyUser allows auth to call back into a new nkey\n// client with the authenticated user. This is used to map\n// any permissions into the client and setup accounts.\nfunc (c *client) RegisterNkeyUser(user *NkeyUser) error {\n\t// Register with proper account and sublist.\n\tif user.Account != nil {\n\t\tif err := c.registerWithAccount(user.Account); err != nil {\n\t\t\tc.reportErrRegisterAccount(user.Account, err)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tc.mu.Lock()\n\tc.user = user\n\t// Assign permissions.\n\tif user.Permissions == nil {\n\t\t// Reset perms to nil in case client previously had them.\n\t\tc.perms = nil\n\t\tc.mperms = nil\n\t} else {\n\t\tc.setPermissions(user.Permissions)\n\t}\n\tc.mu.Unlock()\n\treturn nil\n}\n\nfunc splitSubjectQueue(sq string) ([]byte, []byte, error) {\n\tvals := strings.Fields(strings.TrimSpace(sq))\n\ts := []byte(vals[0])\n\tvar q []byte\n\tif len(vals) == 2 {\n\t\tq = []byte(vals[1])\n\t} else if len(vals) > 2 {\n\t\treturn nil, nil, fmt.Errorf(\"invalid subject-queue %q\", sq)\n\t}\n\treturn s, q, nil\n}\n\n// Initializes client.perms structure.\n// Lock is held on entry.\nfunc (c *client) setPermissions(perms *Permissions) {\n\tif perms == nil {\n\t\treturn\n\t}\n\tc.perms = &permissions{}\n\n\t// Loop over publish permissions\n\tif perms.Publish != nil {\n\t\tif perms.Publish.Allow != nil {\n\t\t\tc.perms.pub.allow = NewSublistWithCache()\n\t\t}\n\t\tfor _, pubSubject := range perms.Publish.Allow {\n\t\t\tsub := &subscription{subject: []byte(pubSubject)}\n\t\t\tc.perms.pub.allow.Insert(sub)\n\t\t}\n\t\tif len(perms.Publish.Deny) > 0 {\n\t\t\tc.perms.pub.deny = NewSublistWithCache()\n\t\t}\n\t\tfor _, pubSubject := range perms.Publish.Deny {\n\t\t\tsub := &subscription{subject: []byte(pubSubject)}\n\t\t\tc.perms.pub.deny.Insert(sub)\n\t\t}\n\t}\n\n\t// Check if we are allowed to send responses.\n\tif perms.Response != nil {\n\t\trp := *perms.Response\n\t\tc.perms.resp = &rp\n\t\tc.replies = make(map[string]*resp)\n\t}\n\n\t// Loop over subscribe permissions\n\tif perms.Subscribe != nil {\n\t\tvar err error\n\t\tif len(perms.Subscribe.Allow) > 0 {\n\t\t\tc.perms.sub.allow = NewSublistWithCache()\n\t\t}\n\t\tfor _, subSubject := range perms.Subscribe.Allow {\n\t\t\tsub := &subscription{}\n\t\t\tsub.subject, sub.queue, err = splitSubjectQueue(subSubject)\n\t\t\tif err != nil {\n\t\t\t\tc.Errorf(\"%s\", err.Error())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.perms.sub.allow.Insert(sub)\n\t\t}\n\t\tif len(perms.Subscribe.Deny) > 0 {\n\t\t\tc.perms.sub.deny = NewSublistWithCache()\n\t\t\t// Also hold onto this array for later.\n\t\t\tc.darray = perms.Subscribe.Deny\n\t\t}\n\t\tfor _, subSubject := range perms.Subscribe.Deny {\n\t\t\tsub := &subscription{}\n\t\t\tsub.subject, sub.queue, err = splitSubjectQueue(subSubject)\n\t\t\tif err != nil {\n\t\t\t\tc.Errorf(\"%s\", err.Error())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.perms.sub.deny.Insert(sub)\n\t\t}\n\t}\n\n\t// If we are a leafnode and we are the hub copy the extracted perms\n\t// to resend back to soliciting server. These are reversed from the\n\t// way routes interpret them since this is how the soliciting server\n\t// will receive these back in an update INFO.\n\tif c.isHubLeafNode() {\n\t\tc.opts.Import = perms.Subscribe\n\t\tc.opts.Export = perms.Publish\n\t}\n}\n\n// Build public permissions from internal ones.\n// Used for user info requests.\nfunc (c *client) publicPermissions() *Permissions {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.perms == nil {\n\t\treturn nil\n\t}\n\tperms := &Permissions{\n\t\tPublish: &SubjectPermission{},\n\t\tSubscribe: &SubjectPermission{},\n\t}\n\n\t_subs := [32]*subscription{}\n\n\t// Publish\n\tif c.perms.pub.allow != nil {\n\t\tsubs := _subs[:0]\n\t\tc.perms.pub.allow.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tperms.Publish.Allow = append(perms.Publish.Allow, string(sub.subject))\n\t\t}\n\t}\n\tif c.perms.pub.deny != nil {\n\t\tsubs := _subs[:0]\n\t\tc.perms.pub.deny.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tperms.Publish.Deny = append(perms.Publish.Deny, string(sub.subject))\n\t\t}\n\t}\n\t// Subsribe\n\tif c.perms.sub.allow != nil {\n\t\tsubs := _subs[:0]\n\t\tc.perms.sub.allow.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tperms.Subscribe.Allow = append(perms.Subscribe.Allow, string(sub.subject))\n\t\t}\n\t}\n\tif c.perms.sub.deny != nil {\n\t\tsubs := _subs[:0]\n\t\tc.perms.sub.deny.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tperms.Subscribe.Deny = append(perms.Subscribe.Deny, string(sub.subject))\n\t\t}\n\t}\n\t// Responses.\n\tif c.perms.resp != nil {\n\t\trp := *c.perms.resp\n\t\tperms.Response = &rp\n\t}\n\n\treturn perms\n}\n\ntype denyType int\n\nconst (\n\tpub = denyType(iota + 1)\n\tsub\n\tboth\n)\n\n// Merge client.perms structure with additional pub deny permissions\n// Lock is held on entry.\nfunc (c *client) mergeDenyPermissions(what denyType, denyPubs []string) {\n\tif len(denyPubs) == 0 {\n\t\treturn\n\t}\n\tif c.perms == nil {\n\t\tc.perms = &permissions{}\n\t}\n\tvar perms []*perm\n\tswitch what {\n\tcase pub:\n\t\tperms = []*perm{&c.perms.pub}\n\tcase sub:\n\t\tperms = []*perm{&c.perms.sub}\n\tcase both:\n\t\tperms = []*perm{&c.perms.pub, &c.perms.sub}\n\t}\n\tfor _, p := range perms {\n\t\tif p.deny == nil {\n\t\t\tp.deny = NewSublistWithCache()\n\t\t}\n\tFOR_DENY:\n\t\tfor _, subj := range denyPubs {\n\t\t\tr := p.deny.Match(subj)\n\t\t\tfor _, v := range r.qsubs {\n\t\t\t\tfor _, s := range v {\n\t\t\t\t\tif string(s.subject) == subj {\n\t\t\t\t\t\tcontinue FOR_DENY\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, s := range r.psubs {\n\t\t\t\tif string(s.subject) == subj {\n\t\t\t\t\tcontinue FOR_DENY\n\t\t\t\t}\n\t\t\t}\n\t\t\tsub := &subscription{subject: []byte(subj)}\n\t\t\tp.deny.Insert(sub)\n\t\t}\n\t}\n}\n\n// Merge client.perms structure with additional pub deny permissions\n// Client lock must not be held on entry\nfunc (c *client) mergeDenyPermissionsLocked(what denyType, denyPubs []string) {\n\tc.mu.Lock()\n\tc.mergeDenyPermissions(what, denyPubs)\n\tc.mu.Unlock()\n}\n\n// Check to see if we have an expiration for the user JWT via base claims.\n// FIXME(dlc) - Clear on connect with new JWT.\nfunc (c *client) setExpiration(claims *jwt.ClaimsData, validFor time.Duration) {\n\tif claims.Expires == 0 {\n\t\tif validFor != 0 {\n\t\t\tc.setExpirationTimer(validFor)\n\t\t}\n\t\treturn\n\t}\n\texpiresAt := time.Duration(0)\n\ttn := time.Now().Unix()\n\tif claims.Expires > tn {\n\t\texpiresAt = time.Duration(claims.Expires-tn) * time.Second\n\t}\n\tif validFor != 0 && validFor < expiresAt {\n\t\tc.setExpirationTimer(validFor)\n\t} else {\n\t\tc.setExpirationTimer(expiresAt)\n\t}\n}\n\n// This will load up the deny structure used for filtering delivered\n// messages based on a deny clause for subscriptions.\n// Lock should be held.\nfunc (c *client) loadMsgDenyFilter() {\n\tc.mperms = &msgDeny{NewSublistWithCache(), make(map[string]bool)}\n\tfor _, sub := range c.darray {\n\t\tc.mperms.deny.Insert(&subscription{subject: []byte(sub)})\n\t}\n}\n\n// writeLoop is the main socket write functionality.\n// Runs in its own Go routine.\nfunc (c *client) writeLoop() {\n\tdefer c.srv.grWG.Done()\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tc.flags.set(writeLoopStarted)\n\tc.mu.Unlock()\n\n\t// Used to check that we did flush from last wake up.\n\twaitOk := true\n\tvar closed bool\n\n\t// Main loop. Will wait to be signaled and then will use\n\t// buffered outbound structure for efficient writev to the underlying socket.\n\tfor {\n\t\tc.mu.Lock()\n\t\tif closed = c.isClosed(); !closed {\n\t\t\towtf := c.out.fsp > 0 && c.out.pb < maxBufSize && c.out.fsp < maxFlushPending\n\t\t\tif waitOk && (c.out.pb == 0 || owtf) {\n\t\t\t\tc.out.sg.Wait()\n\t\t\t\t// Check that connection has not been closed while lock was released\n\t\t\t\t// in the conditional wait.\n\t\t\t\tclosed = c.isClosed()\n\t\t\t}\n\t\t}\n\t\tif closed {\n\t\t\tc.flushAndClose(false)\n\t\t\tc.mu.Unlock()\n\n\t\t\t// We should always call closeConnection() to ensure that state is\n\t\t\t// properly cleaned-up. It will be a no-op if already done.\n\t\t\tc.closeConnection(WriteError)\n\n\t\t\t// Now explicitly call reconnect(). Thanks to ref counting, we know\n\t\t\t// that the reconnect will execute only after connection has been\n\t\t\t// removed from the server state.\n\t\t\tc.reconnect()\n\t\t\treturn\n\t\t}\n\t\t// Flush data\n\t\twaitOk = c.flushOutbound()\n\t\tc.mu.Unlock()\n\t}\n}\n\n// flushClients will make sure to flush any clients we may have\n// sent to during processing. We pass in a budget as a time.Duration\n// for how much time to spend in place flushing for this client.\nfunc (c *client) flushClients(budget time.Duration) time.Time {\n\tlast := time.Now()\n\n\t// Check pending clients for flush.\n\tfor cp := range c.pcd {\n\t\t// TODO(dlc) - Wonder if it makes more sense to create a new map?\n\t\tdelete(c.pcd, cp)\n\n\t\t// Queue up a flush for those in the set\n\t\tcp.mu.Lock()\n\t\t// Update last activity for message delivery\n\t\tcp.last = last\n\t\t// Remove ourselves from the pending list.\n\t\tcp.out.fsp--\n\n\t\t// Just ignore if this was closed.\n\t\tif cp.isClosed() {\n\t\t\tcp.mu.Unlock()\n\t\t\tcontinue\n\t\t}\n\n\t\tif budget > 0 && cp.out.lft < 2*budget && cp.flushOutbound() {\n\t\t\tbudget -= cp.out.lft\n\t\t} else {\n\t\t\tcp.flushSignal()\n\t\t}\n\n\t\tcp.mu.Unlock()\n\t}\n\treturn last\n}\n\n// readLoop is the main socket read functionality.\n// Runs in its own Go routine.\nfunc (c *client) readLoop(pre []byte) {\n\t// Grab the connection off the client, it will be cleared on a close.\n\t// We check for that after the loop, but want to avoid a nil dereference\n\tc.mu.Lock()\n\ts := c.srv\n\tdefer s.grWG.Done()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tnc := c.nc\n\tws := c.isWebsocket()\n\tif c.isMqtt() {\n\t\tc.mqtt.r = &mqttReader{reader: nc}\n\t}\n\tc.in.rsz = startBufSize\n\n\t// Check the per-account-cache for closed subscriptions\n\tcpacc := c.kind == ROUTER || c.kind == GATEWAY\n\t// Last per-account-cache check for closed subscriptions\n\tlpacc := time.Now()\n\tacc := c.acc\n\tvar masking bool\n\tif ws {\n\t\tmasking = c.ws.maskread\n\t}\n\tcheckCompress := c.kind == ROUTER || c.kind == LEAF\n\tc.mu.Unlock()\n\n\tdefer func() {\n\t\tif c.isMqtt() {\n\t\t\ts.mqttHandleClosedClient(c)\n\t\t}\n\t\t// These are used only in the readloop, so we can set them to nil\n\t\t// on exit of the readLoop.\n\t\tc.in.results, c.in.pacache = nil, nil\n\t}()\n\n\t// Start read buffer.\n\tb := make([]byte, c.in.rsz)\n\n\t// Websocket clients will return several slices if there are multiple\n\t// websocket frames in the blind read. For non WS clients though, we\n\t// will always have 1 slice per loop iteration. So we define this here\n\t// so non WS clients will use bufs[0] = b[:n].\n\tvar _bufs [1][]byte\n\tbufs := _bufs[:1]\n\n\tvar wsr *wsReadInfo\n\tif ws {\n\t\twsr = &wsReadInfo{mask: masking}\n\t\twsr.init()\n\t}\n\n\tvar decompress bool\n\tvar reader io.Reader\n\treader = nc\n\n\tfor {\n\t\tvar n int\n\t\tvar err error\n\n\t\t// If we have a pre buffer parse that first.\n\t\tif len(pre) > 0 {\n\t\t\tb = pre\n\t\t\tn = len(pre)\n\t\t\tpre = nil\n\t\t} else {\n\t\t\tn, err = reader.Read(b)\n\t\t\t// If we have any data we will try to parse and exit at the end.\n\t\t\tif n == 0 && err != nil {\n\t\t\t\tc.closeConnection(closedStateForErr(err))\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tif ws {\n\t\t\tbufs, err = c.wsRead(wsr, reader, b[:n])\n\t\t\tif bufs == nil && err != nil {\n\t\t\t\tif err != io.EOF {\n\t\t\t\t\tc.Errorf(\"read error: %v\", err)\n\t\t\t\t}\n\t\t\t\tc.closeConnection(closedStateForErr(err))\n\t\t\t\treturn\n\t\t\t} else if bufs == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else {\n\t\t\tbufs[0] = b[:n]\n\t\t}\n\n\t\t// Check if the account has mappings and if so set the local readcache flag.\n\t\t// We check here to make sure any changes such as config reload are reflected here.\n\t\tif c.kind == CLIENT || c.kind == LEAF {\n\t\t\tif acc.hasMappings() {\n\t\t\t\tc.in.flags.set(hasMappings)\n\t\t\t} else {\n\t\t\t\tc.in.flags.clear(hasMappings)\n\t\t\t}\n\t\t}\n\n\t\tc.in.start = time.Now()\n\n\t\t// Clear inbound stats cache\n\t\tc.in.msgs = 0\n\t\tc.in.bytes = 0\n\t\tc.in.subs = 0\n\n\t\t// Main call into parser for inbound data. This will generate callouts\n\t\t// to process messages, etc.\n\t\tfor i := 0; i < len(bufs); i++ {\n\t\t\tif err := c.parse(bufs[i]); err != nil {\n\t\t\t\tif err == ErrMinimumVersionRequired {\n\t\t\t\t\t// Special case here, currently only for leaf node connections.\n\t\t\t\t\t// processLeafConnect() already sent the rejection and closed\n\t\t\t\t\t// the connection, so there is nothing else to do here.\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif dur := time.Since(c.in.start); dur >= readLoopReportThreshold {\n\t\t\t\t\tc.Warnf(\"Readloop processing time: %v\", dur)\n\t\t\t\t}\n\t\t\t\t// Need to call flushClients because some of the clients have been\n\t\t\t\t// assigned messages and their \"fsp\" incremented, and need now to be\n\t\t\t\t// decremented and their writeLoop signaled.\n\t\t\t\tc.flushClients(0)\n\t\t\t\t// handled inline\n\t\t\t\tif err != ErrMaxPayload && err != ErrAuthentication {\n\t\t\t\t\tc.Error(err)\n\t\t\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Clear total stalled time here.\n\t\t\tif c.in.tst >= stallClientMaxDuration {\n\t\t\t\tc.rateLimitFormatWarnf(\"Producer was stalled for a total of %v\", c.in.tst.Round(time.Millisecond))\n\t\t\t}\n\t\t\tc.in.tst = 0\n\t\t}\n\n\t\t// If we are a ROUTER/LEAF and have processed an INFO, it is possible that\n\t\t// we are asked to switch to compression now.\n\t\tif checkCompress && c.in.flags.isSet(switchToCompression) {\n\t\t\tc.in.flags.clear(switchToCompression)\n\t\t\t// For now we support only s2 compression...\n\t\t\treader = s2.NewReader(nc)\n\t\t\tdecompress = true\n\t\t}\n\n\t\t// Updates stats for client and server that were collected\n\t\t// from parsing through the buffer.\n\t\tif c.in.msgs > 0 {\n\t\t\tinMsgs := int64(c.in.msgs)\n\t\t\tinBytes := int64(c.in.bytes)\n\n\t\t\tatomic.AddInt64(&c.inMsgs, inMsgs)\n\t\t\tatomic.AddInt64(&c.inBytes, inBytes)\n\n\t\t\tif acc != nil {\n\t\t\t\tacc.stats.Lock()\n\t\t\t\tacc.stats.inMsgs += inMsgs\n\t\t\t\tacc.stats.inBytes += inBytes\n\t\t\t\tif c.kind == LEAF {\n\t\t\t\t\tacc.stats.ln.inMsgs += int64(inMsgs)\n\t\t\t\t\tacc.stats.ln.inBytes += int64(inBytes)\n\t\t\t\t}\n\t\t\t\tacc.stats.Unlock()\n\t\t\t}\n\n\t\t\tatomic.AddInt64(&s.inMsgs, inMsgs)\n\t\t\tatomic.AddInt64(&s.inBytes, inBytes)\n\t\t}\n\n\t\t// Signal to writeLoop to flush to socket.\n\t\tlast := c.flushClients(0)\n\n\t\t// Update activity, check read buffer size.\n\t\tc.mu.Lock()\n\n\t\t// Activity based on interest changes or data/msgs.\n\t\t// Also update last receive activity for ping sender\n\t\tif c.in.msgs > 0 || c.in.subs > 0 {\n\t\t\tc.last = last\n\t\t\tc.lastIn = last\n\t\t}\n\n\t\tif n >= cap(b) {\n\t\t\tc.in.srs = 0\n\t\t} else if n < cap(b)/2 { // divide by 2 b/c we want less than what we would shrink to.\n\t\t\tc.in.srs++\n\t\t}\n\n\t\t// Update read buffer size as/if needed.\n\t\tif n >= cap(b) && cap(b) < maxBufSize {\n\t\t\t// Grow\n\t\t\tc.in.rsz = int32(cap(b) * 2)\n\t\t\tb = make([]byte, c.in.rsz)\n\t\t} else if n < cap(b) && cap(b) > minBufSize && c.in.srs > shortsToShrink {\n\t\t\t// Shrink, for now don't accelerate, ping/pong will eventually sort it out.\n\t\t\tc.in.rsz = int32(cap(b) / 2)\n\t\t\tb = make([]byte, c.in.rsz)\n\t\t}\n\t\t// re-snapshot the account since it can change during reload, etc.\n\t\tacc = c.acc\n\t\t// Refresh nc because in some cases, we have upgraded c.nc to TLS.\n\t\tif nc != c.nc {\n\t\t\tnc = c.nc\n\t\t\tif decompress && nc != nil {\n\t\t\t\t// For now we support only s2 compression...\n\t\t\t\treader.(*s2.Reader).Reset(nc)\n\t\t\t} else if !decompress {\n\t\t\t\treader = nc\n\t\t\t}\n\t\t}\n\t\tc.mu.Unlock()\n\n\t\t// Connection was closed\n\t\tif nc == nil {\n\t\t\treturn\n\t\t}\n\n\t\tif dur := time.Since(c.in.start); dur >= readLoopReportThreshold {\n\t\t\tc.Warnf(\"Readloop processing time: %v\", dur)\n\t\t}\n\n\t\t// We could have had a read error from above but still read some data.\n\t\t// If so do the close here unconditionally.\n\t\tif err != nil {\n\t\t\tc.closeConnection(closedStateForErr(err))\n\t\t\treturn\n\t\t}\n\n\t\tif cpacc && (c.in.start.Sub(lpacc)) >= closedSubsCheckInterval {\n\t\t\tc.pruneClosedSubFromPerAccountCache()\n\t\t\tlpacc = time.Now()\n\t\t}\n\t}\n}\n\n// Returns the appropriate closed state for a given read error.\nfunc closedStateForErr(err error) ClosedState {\n\tif err == io.EOF {\n\t\treturn ClientClosed\n\t}\n\treturn ReadError\n}\n\n// collapsePtoNB will either returned framed WebSocket buffers or it will\n// return a reference to c.out.nb.\nfunc (c *client) collapsePtoNB() (net.Buffers, int64) {\n\tif c.isWebsocket() {\n\t\treturn c.wsCollapsePtoNB()\n\t}\n\treturn c.out.nb, c.out.pb\n}\n\n// flushOutbound will flush outbound buffer to a client.\n// Will return true if data was attempted to be written.\n// Lock must be held\nfunc (c *client) flushOutbound() bool {\n\tif c.flags.isSet(flushOutbound) {\n\t\t// For CLIENT connections, it is possible that the readLoop calls\n\t\t// flushOutbound(). If writeLoop and readLoop compete and we are\n\t\t// here we should release the lock to reduce the risk of spinning.\n\t\tc.mu.Unlock()\n\t\truntime.Gosched()\n\t\tc.mu.Lock()\n\t\treturn false\n\t}\n\tc.flags.set(flushOutbound)\n\tdefer func() {\n\t\t// Check flushAndClose() for explanation on why we do this.\n\t\tif c.isClosed() {\n\t\t\tfor i := range c.out.wnb {\n\t\t\t\tnbPoolPut(c.out.wnb[i])\n\t\t\t}\n\t\t\tc.out.wnb = nil\n\t\t}\n\t\tc.flags.clear(flushOutbound)\n\t}()\n\n\t// Check for nothing to do.\n\tif c.nc == nil || c.srv == nil || c.out.pb == 0 {\n\t\treturn true // true because no need to queue a signal.\n\t}\n\n\t// In the case of a normal socket connection, \"collapsed\" is just a ref\n\t// to \"nb\". In the case of WebSockets, additional framing is added to\n\t// anything that is waiting in \"nb\". Also keep a note of how many bytes\n\t// were queued before we release the mutex.\n\tcollapsed, attempted := c.collapsePtoNB()\n\n\t// Frustratingly, (net.Buffers).WriteTo() modifies the receiver so we\n\t// can't work on \"nb\" directly — while the mutex is unlocked during IO,\n\t// something else might call queueOutbound and modify it. So instead we\n\t// need a working copy — we'll operate on \"wnb\" instead. Note that in\n\t// the case of a partial write, \"wnb\" may have remaining data from the\n\t// previous write, and in the case of WebSockets, that data may already\n\t// be framed, so we are careful not to re-frame \"wnb\" here. Instead we\n\t// will just frame up \"nb\" and append it onto whatever is left on \"wnb\".\n\t// \"nb\" will be set to nil so that we can manipulate \"collapsed\" outside\n\t// of the client's lock, which is interesting in case of compression.\n\tc.out.nb = nil\n\n\t// In case it goes away after releasing the lock.\n\tnc := c.nc\n\n\t// Capture this (we change the value in some tests)\n\twdl := c.out.wdl\n\n\t// Check for compression\n\tcw := c.out.cw\n\tif cw != nil {\n\t\t// We will have to adjust once we have compressed, so remove for now.\n\t\tc.out.pb -= attempted\n\t\tif c.isWebsocket() {\n\t\t\tc.ws.fs -= attempted\n\t\t}\n\t}\n\n\t// Do NOT hold lock during actual IO.\n\tc.mu.Unlock()\n\n\t// Compress outside of the lock\n\tif cw != nil {\n\t\tvar err error\n\t\tbb := bytes.Buffer{}\n\n\t\tcw.Reset(&bb)\n\t\tfor _, buf := range collapsed {\n\t\t\tif err == nil {\n\t\t\t\t_, err = cw.Write(buf)\n\t\t\t}\n\t\t\t// Return always after consumed or error.\n\t\t\tnbPoolPut(buf)\n\t\t}\n\t\tif err == nil {\n\t\t\terr = cw.Close()\n\t\t}\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Error compressing data: %v\", err)\n\t\t\t// We need to grab the lock now before marking as closed and exiting\n\t\t\tc.mu.Lock()\n\t\t\tc.markConnAsClosed(WriteError)\n\t\t\treturn false\n\t\t}\n\t\tcollapsed = append(net.Buffers(nil), bb.Bytes())\n\t\tattempted = int64(len(collapsed[0]))\n\t}\n\n\t// This is safe to do outside of the lock since \"collapsed\" is no longer\n\t// referenced in c.out.nb (which can be modified in queueOutboud() while\n\t// the lock is released).\n\tc.out.wnb = append(c.out.wnb, collapsed...)\n\tvar _orig [nbMaxVectorSize][]byte\n\torig := append(_orig[:0], c.out.wnb...)\n\n\t// Since WriteTo is lopping things off the beginning, we need to remember\n\t// the start position of the underlying array so that we can get back to it.\n\t// Otherwise we'll always \"slide forward\" and that will result in reallocs.\n\tstartOfWnb := c.out.wnb[0:]\n\n\t// flush here\n\tstart := time.Now()\n\n\tvar n int64 // Total bytes written\n\tvar wn int64 // Bytes written per loop\n\tvar err error // Error from last write, if any\n\tfor len(c.out.wnb) > 0 {\n\t\t// Limit the number of vectors to no more than nbMaxVectorSize,\n\t\t// which if 1024, will mean a maximum of 64MB in one go.\n\t\twnb := c.out.wnb\n\t\tif len(wnb) > nbMaxVectorSize {\n\t\t\twnb = wnb[:nbMaxVectorSize]\n\t\t}\n\t\tconsumed := len(wnb)\n\n\t\t// Actual write to the socket. The deadline applies to each batch\n\t\t// rather than the total write, such that the configured deadline\n\t\t// can be tuned to a known maximum quantity (64MB).\n\t\tnc.SetWriteDeadline(time.Now().Add(wdl))\n\t\twn, err = wnb.WriteTo(nc)\n\t\tnc.SetWriteDeadline(time.Time{})\n\n\t\t// Update accounting, move wnb slice onwards if needed, or stop\n\t\t// if a write error was reported that wasn't a short write.\n\t\tn += wn\n\t\tc.out.wnb = c.out.wnb[consumed-len(wnb):]\n\t\tif err != nil && err != io.ErrShortWrite {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tlft := time.Since(start)\n\n\t// Re-acquire client lock.\n\tc.mu.Lock()\n\n\t// Adjust if we were compressing.\n\tif cw != nil {\n\t\tc.out.pb += attempted\n\t\tif c.isWebsocket() {\n\t\t\tc.ws.fs += attempted\n\t\t}\n\t}\n\n\t// At this point, \"wnb\" has been mutated by WriteTo and any consumed\n\t// buffers have been lopped off the beginning, so in order to return\n\t// them to the pool, we need to look at the difference between \"orig\"\n\t// and \"wnb\".\n\tfor i := 0; i < len(orig)-len(c.out.wnb); i++ {\n\t\tnbPoolPut(orig[i])\n\t}\n\n\t// At this point it's possible that \"nb\" has been modified by another\n\t// call to queueOutbound while the lock was released, so we'll leave\n\t// those for the next iteration. Meanwhile it's possible that we only\n\t// managed a partial write of \"wnb\", so we'll shift anything that\n\t// remains up to the beginning of the array to prevent reallocating.\n\t// Anything left in \"wnb\" has already been framed for WebSocket conns\n\t// so leave them alone for the next call to flushOutbound.\n\tc.out.wnb = append(startOfWnb[:0], c.out.wnb...)\n\n\t// If we've written everything but the underlying array of our working\n\t// buffer has grown excessively then free it — the GC will tidy it up\n\t// and we can allocate a new one next time.\n\tif len(c.out.wnb) == 0 && cap(c.out.wnb) > nbPoolSizeLarge*8 {\n\t\tc.out.wnb = nil\n\t}\n\n\t// Ignore ErrShortWrite errors, they will be handled as partials.\n\tvar gotWriteTimeout bool\n\tif err != nil && err != io.ErrShortWrite {\n\t\t// Handle timeout error (slow consumer) differently\n\t\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\t\tgotWriteTimeout = true\n\t\t\tif closed := c.handleWriteTimeout(n, attempted, len(orig)); closed {\n\t\t\t\treturn true\n\t\t\t}\n\t\t} else {\n\t\t\t// Other errors will cause connection to be closed.\n\t\t\t// For clients, report as debug but for others report as error.\n\t\t\treport := c.Debugf\n\t\t\tif c.kind != CLIENT {\n\t\t\t\treport = c.Errorf\n\t\t\t}\n\t\t\treport(\"Error flushing: %v\", err)\n\t\t\tc.markConnAsClosed(WriteError)\n\t\t\treturn true\n\t\t}\n\t}\n\n\t// Update flush time statistics.\n\tc.out.lft = lft\n\n\t// Subtract from pending bytes and messages.\n\tc.out.pb -= n\n\tif c.isWebsocket() {\n\t\tc.ws.fs -= n\n\t}\n\n\t// Check that if there is still data to send and writeLoop is in wait,\n\t// then we need to signal.\n\tif c.out.pb > 0 {\n\t\tc.flushSignal()\n\t}\n\n\t// Check if we have a stalled gate and if so and we are recovering release\n\t// any stalled producers. Only kind==CLIENT will stall.\n\tif c.out.stc != nil && (n == attempted || c.out.pb < c.out.mp/4*3) {\n\t\tclose(c.out.stc)\n\t\tc.out.stc = nil\n\t}\n\t// Check if the connection is recovering from being a slow consumer.\n\tif !gotWriteTimeout && c.flags.isSet(isSlowConsumer) {\n\t\tc.Noticef(\"Slow Consumer Recovered: Flush took %.3fs with %d chunks of %d total bytes.\", time.Since(start).Seconds(), len(orig), attempted)\n\t\tc.flags.clear(isSlowConsumer)\n\t}\n\n\treturn true\n}\n\n// This is invoked from flushOutbound() for io/timeout error (slow consumer).\n// Returns a boolean to indicate if the connection has been closed or not.\n// Lock is held on entry.\nfunc (c *client) handleWriteTimeout(written, attempted int64, numChunks int) bool {\n\tif tlsConn, ok := c.nc.(*tls.Conn); ok {\n\t\tif !tlsConn.ConnectionState().HandshakeComplete {\n\t\t\t// Likely a TLSTimeout error instead...\n\t\t\tc.markConnAsClosed(TLSHandshakeError)\n\t\t\t// Would need to coordinate with tlstimeout()\n\t\t\t// to avoid double logging, so skip logging\n\t\t\t// here, and don't report a slow consumer error.\n\t\t\treturn true\n\t\t}\n\t} else if c.flags.isSet(expectConnect) && !c.flags.isSet(connectReceived) {\n\t\t// Under some conditions, a connection may hit a slow consumer write deadline\n\t\t// before the authorization timeout. If that is the case, then we handle\n\t\t// as slow consumer though we do not increase the counter as that can be\n\t\t// misleading.\n\t\tc.markConnAsClosed(SlowConsumerWriteDeadline)\n\t\treturn true\n\t}\n\talreadySC := c.flags.isSet(isSlowConsumer)\n\tscState := \"Detected\"\n\tif alreadySC {\n\t\tscState = \"State\"\n\t}\n\n\t// Aggregate slow consumers.\n\tatomic.AddInt64(&c.srv.slowConsumers, 1)\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tc.srv.scStats.clients.Add(1)\n\tcase ROUTER:\n\t\t// Only count each Slow Consumer event once.\n\t\tif !alreadySC {\n\t\t\tc.srv.scStats.routes.Add(1)\n\t\t}\n\tcase GATEWAY:\n\t\tc.srv.scStats.gateways.Add(1)\n\tcase LEAF:\n\t\tc.srv.scStats.leafs.Add(1)\n\t}\n\tif c.acc != nil {\n\t\tc.acc.stats.Lock()\n\t\tc.acc.stats.slowConsumers++\n\t\tc.acc.stats.Unlock()\n\t}\n\tc.Noticef(\"Slow Consumer %s: WriteDeadline of %v exceeded with %d chunks of %d total bytes.\",\n\t\tscState, c.out.wdl, numChunks, attempted)\n\n\t// We always close CLIENT connections, or when nothing was written at all...\n\tif c.out.wtp == WriteTimeoutPolicyClose || written == 0 {\n\t\tc.markConnAsClosed(SlowConsumerWriteDeadline)\n\t\treturn true\n\t} else {\n\t\tc.flags.setIfNotSet(isSlowConsumer)\n\t}\n\treturn false\n}\n\n// Marks this connection has closed with the given reason.\n// Sets the connMarkedClosed flag and skipFlushOnClose depending on the reason.\n// Depending on the kind of connection, the connection will be saved.\n// If a writeLoop has been started, the final flush will be done there, otherwise\n// flush and close of TCP connection is done here in place.\n// Returns true if closed in place, flase otherwise.\n// Lock is held on entry.\nfunc (c *client) markConnAsClosed(reason ClosedState) {\n\t// Possibly set skipFlushOnClose flag even if connection has already been\n\t// mark as closed. The rationale is that a connection may be closed with\n\t// a reason that justifies a flush (say after sending an -ERR), but then\n\t// the flushOutbound() gets a write error. If that happens, connection\n\t// being lost, there is no reason to attempt to flush again during the\n\t// teardown when the writeLoop exits.\n\tvar skipFlush bool\n\tswitch reason {\n\tcase ReadError, WriteError, SlowConsumerPendingBytes, SlowConsumerWriteDeadline, TLSHandshakeError:\n\t\tc.flags.set(skipFlushOnClose)\n\t\tskipFlush = true\n\tcase StaleConnection:\n\t\t// Track stale connections statistics.\n\t\tatomic.AddInt64(&c.srv.staleConnections, 1)\n\t\tswitch c.kind {\n\t\tcase CLIENT:\n\t\t\tc.srv.staleStats.clients.Add(1)\n\t\tcase ROUTER:\n\t\t\tc.srv.staleStats.routes.Add(1)\n\t\tcase GATEWAY:\n\t\t\tc.srv.staleStats.gateways.Add(1)\n\t\tcase LEAF:\n\t\t\tc.srv.staleStats.leafs.Add(1)\n\t\t}\n\t}\n\tif c.flags.isSet(connMarkedClosed) {\n\t\treturn\n\t}\n\tc.flags.set(connMarkedClosed)\n\t// For a websocket client, unless we are told not to flush, enqueue\n\t// a websocket CloseMessage based on the reason.\n\tif !skipFlush && c.isWebsocket() && !c.ws.closeSent {\n\t\tc.wsEnqueueCloseMessage(reason)\n\t}\n\t// Be consistent with the creation: for routes, gateways and leaf,\n\t// we use Noticef on create, so use that too for delete.\n\tif c.srv != nil {\n\t\tif c.kind == LEAF || c.kind == ROUTER || c.kind == GATEWAY {\n\t\t\tvar tags []string\n\t\t\tvar remoteName string\n\t\t\tswitch {\n\t\t\tcase c.kind == LEAF && c.leaf != nil:\n\t\t\t\tremoteName = c.leaf.remoteServer\n\t\t\tcase c.kind == ROUTER && c.route != nil:\n\t\t\t\tremoteName = c.route.remoteName\n\t\t\tcase c.kind == GATEWAY && c.gw != nil:\n\t\t\t\tremoteName = c.gw.remoteName\n\t\t\t}\n\t\t\tif remoteName != _EMPTY_ {\n\t\t\t\ttags = append(tags, fmt.Sprintf(\"Remote: %s\", remoteName))\n\t\t\t}\n\t\t\tif len(tags) > 0 {\n\t\t\t\tc.Noticef(\"%s connection closed: %s - %s\", c.kindString(), reason, strings.Join(tags, \", \"))\n\t\t\t} else {\n\t\t\t\tc.Noticef(\"%s connection closed: %s\", c.kindString(), reason)\n\t\t\t}\n\t\t} else { // Client, System, Jetstream, and Account connections.\n\t\t\tc.Debugf(\"%s connection closed: %s\", c.kindString(), reason)\n\t\t}\n\t}\n\n\t// Save off the connection if its a client or leafnode.\n\tif c.kind == CLIENT || c.kind == LEAF {\n\t\tif nc := c.nc; nc != nil && c.srv != nil {\n\t\t\t// TODO: May want to send events to single go routine instead\n\t\t\t// of creating a new go routine for each save.\n\t\t\t// Pass the c.subs as a reference. It may be set to nil in\n\t\t\t// closeConnection.\n\t\t\tgo c.srv.saveClosedClient(c, nc, c.subs, reason)\n\t\t}\n\t}\n\t// If writeLoop exists, let it do the final flush, close and teardown.\n\tif c.flags.isSet(writeLoopStarted) {\n\t\t// Since we want the writeLoop to do the final flush and tcp close,\n\t\t// we want the reconnect to be done there too. However, it should'nt\n\t\t// happen before the connection has been removed from the server\n\t\t// state (end of closeConnection()). This ref count allows us to\n\t\t// guarantee that.\n\t\tc.rref++\n\t\tc.flushSignal()\n\t\treturn\n\t}\n\t// Flush (if skipFlushOnClose is not set) and close in place. If flushing,\n\t// use a small WriteDeadline.\n\tc.flushAndClose(true)\n}\n\n// flushSignal will use server to queue the flush IO operation to a pool of flushers.\n// Lock must be held.\nfunc (c *client) flushSignal() {\n\t// Check that sg is not nil, which will happen if the connection is closed.\n\tif c.out.sg != nil {\n\t\tc.out.sg.Signal()\n\t}\n}\n\n// Traces a message.\n// Will NOT check if tracing is enabled, does NOT need the client lock.\nfunc (c *client) traceMsgInternal(msg []byte, delivered bool, hdrSize int) {\n\topts := c.srv.getOpts()\n\tmaxTrace := opts.MaxTracedMsgLen\n\theadersOnly := opts.TraceHeaders\n\tsuffix := LEN_CR_LF\n\n\t// If TraceHeaders is enabled, extract only the header portion of the msg.\n\t// If a header is present, it ends with an additional trailing CRLF.\n\tif headersOnly {\n\t\tif hdrSize > 0 && len(msg) >= hdrSize {\n\t\t\tmsg = msg[:hdrSize]\n\t\t\tsuffix += LEN_CR_LF\n\t\t} else {\n\t\t\t// No headers present, so nothing to trace.\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Do not emit a log line for zero-length payloads.\n\tl := len(msg) - suffix\n\tif l <= 0 {\n\t\treturn\n\t}\n\n\tconst (\n\t\ttraceInPrefix string = \"<<-\"\n\t\ttraceOutPrefix string = \"->>\"\n\t)\n\tvar prefix string\n\tif delivered {\n\t\tprefix = traceOutPrefix\n\t} else {\n\t\tprefix = traceInPrefix\n\t}\n\tif maxTrace > 0 && l > maxTrace {\n\t\ttm := fmt.Sprintf(\"%q\", msg[:maxTrace])\n\t\tc.Tracef(\"%s MSG_PAYLOAD: [\\\"%s...\\\"]\", prefix, tm[1:len(tm)-1])\n\t} else {\n\t\tc.Tracef(\"%s MSG_PAYLOAD: [%q]\", prefix, msg[:l])\n\t}\n}\n\nfunc (c *client) traceMsg(msg []byte) {\n\tc.traceMsgInternal(msg, false, c.pa.hdr)\n}\n\nfunc (c *client) traceMsgDelivery(msg []byte, hdrSize int) {\n\tc.traceMsgInternal(msg, true, hdrSize)\n}\n\n// Traces an incoming operation.\n// Will NOT check if tracing is enabled, does NOT need the client lock.\nfunc (c *client) traceInOp(op string, arg []byte) {\n\tc.traceOp(\"<<- %s\", op, arg)\n}\n\n// Traces an outgoing operation.\n// Will NOT check if tracing is enabled, does NOT need the client lock.\nfunc (c *client) traceOutOp(op string, arg []byte) {\n\tc.traceOp(\"->> %s\", op, arg)\n}\n\nfunc (c *client) traceOp(format, op string, arg []byte) {\n\topa := []any{}\n\tif op != _EMPTY_ {\n\t\topa = append(opa, op)\n\t}\n\tif arg != nil {\n\t\topa = append(opa, bytesToString(arg))\n\t}\n\tc.Tracef(format, opa)\n}\n\n// Process the information messages from Clients and other Routes.\nfunc (c *client) processInfo(arg []byte) error {\n\tinfo := Info{}\n\tif err := json.Unmarshal(arg, &info); err != nil {\n\t\treturn err\n\t}\n\tswitch c.kind {\n\tcase ROUTER:\n\t\tc.processRouteInfo(&info)\n\tcase GATEWAY:\n\t\tc.processGatewayInfo(&info)\n\tcase LEAF:\n\t\tc.processLeafnodeInfo(&info)\n\t}\n\treturn nil\n}\n\nfunc (c *client) processErr(errStr string) {\n\tclose := true\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tc.Errorf(\"Client Error %s\", errStr)\n\tcase ROUTER:\n\t\tc.Errorf(\"Route Error %s\", errStr)\n\tcase GATEWAY:\n\t\tc.Errorf(\"Gateway Error %s\", errStr)\n\tcase LEAF:\n\t\tc.Errorf(\"Leafnode Error %s\", errStr)\n\t\tc.leafProcessErr(errStr)\n\t\tclose = false\n\tcase JETSTREAM:\n\t\tc.Errorf(\"JetStream Error %s\", errStr)\n\t}\n\tif close {\n\t\tc.closeConnection(ParseError)\n\t}\n}\n\n// Matcher for pass/password and auth_token fields.\nvar prefixAuthPat = regexp.MustCompile(`\"?\\s*(?:auth_token\\S*?|pass\\S*?)\"?\\s*[:=]\\s*\"?([^\",\\r\\n}]*)`)\n\n// Exact matcher for fields sig, proxy_sig and nkey.\n// Overlapping field \"sig\" does not match inside \"proxy_sig\".\nvar exactAuthPat = regexp.MustCompile(`(?:^|[^A-Za-z0-9_])\"?\\s*(?:proxy_sig|nkey|sig)\"?\\s*[:=]\\s*\"?([^\",\\r\\n}]*)`)\n\n// removeSecretsFromTrace removes any notion of passwords/tokens from trace\n// messages for logging.\nfunc removeSecretsFromTrace(arg []byte) []byte {\n\tbuf := redact(prefixAuthPat, arg)\n\treturn redact(exactAuthPat, buf)\n}\n\nfunc redact(pat *regexp.Regexp, proto []byte) []byte {\n\tm := pat.FindAllSubmatchIndex(proto, -1)\n\tif len(m) == 0 {\n\t\treturn proto\n\t}\n\t// Take a copy of the connect proto just for the trace message.\n\tvar _arg [4096]byte\n\tbuf := append(_arg[:0], proto...)\n\tredactedPass := []byte(\"[REDACTED]\")\n\tfor i := len(m) - 1; i >= 0; i-- {\n\t\tmatch := m[i]\n\t\tif len(match) < 4 {\n\t\t\tcontinue\n\t\t}\n\t\tstart, end := match[2], match[3]\n\t\t// Replace value substring.\n\t\tbuf = append(buf[:start], append(redactedPass, buf[end:]...)...)\n\t}\n\treturn buf\n}\n\n// Returns the RTT by computing the elapsed time since now and `start`.\n// On Windows VM where I (IK) run tests, time.Since() will return 0\n// (I suspect some time granularity issues). So return at minimum 1ns.\nfunc computeRTT(start time.Time) time.Duration {\n\trtt := time.Since(start)\n\tif rtt <= 0 {\n\t\trtt = time.Nanosecond\n\t}\n\treturn rtt\n}\n\n// processConnect will process a client connect op.\nfunc (c *client) processConnect(arg []byte) error {\n\tsupportsHeaders := c.srv.supportsHeaders()\n\tc.mu.Lock()\n\t// If we can't stop the timer because the callback is in progress...\n\tif !c.clearAuthTimer() {\n\t\t// wait for it to finish and handle sending the failure back to\n\t\t// the client.\n\t\tfor !c.isClosed() {\n\t\t\tc.mu.Unlock()\n\t\t\ttime.Sleep(25 * time.Millisecond)\n\t\t\tc.mu.Lock()\n\t\t}\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\tc.last = time.Now().UTC()\n\t// Estimate RTT to start.\n\tif c.kind == CLIENT {\n\t\tc.rtt = computeRTT(c.start)\n\t\tif c.srv != nil {\n\t\t\tc.clearPingTimer()\n\t\t\tc.setFirstPingTimer()\n\t\t}\n\t}\n\tkind := c.kind\n\tsrv := c.srv\n\n\t// Moved unmarshalling of clients' Options under the lock.\n\t// The client has already been added to the server map, so it is possible\n\t// that other routines lookup the client, and access its options under\n\t// the client's lock, so unmarshalling the options outside of the lock\n\t// would cause data RACEs.\n\tif err := json.Unmarshal(arg, &c.opts); err != nil {\n\t\tc.mu.Unlock()\n\t\treturn err\n\t}\n\t// Indicate that the CONNECT protocol has been received, and that the\n\t// server now knows which protocol this client supports.\n\tfirstConnect := !c.flags.isSet(connectReceived)\n\tc.flags.set(connectReceived)\n\t// Capture these under lock\n\tc.echo = c.opts.Echo\n\tproto := c.opts.Protocol\n\tverbose := c.opts.Verbose\n\tlang := c.opts.Lang\n\taccount := c.opts.Account\n\taccountNew := c.opts.AccountNew\n\n\t// if websocket client, maybe some options through cookies\n\tif ws := c.ws; ws != nil {\n\t\t// if JWT not in the CONNECT, use the cookie JWT (possibly empty).\n\t\tif c.opts.JWT == _EMPTY_ {\n\t\t\tc.opts.JWT = ws.cookieJwt\n\t\t}\n\t\t// if user not in the CONNECT, use the cookie user (possibly empty)\n\t\tif c.opts.Username == _EMPTY_ {\n\t\t\tc.opts.Username = ws.cookieUsername\n\t\t}\n\t\t// if pass not in the CONNECT, use the cookie password (possibly empty).\n\t\tif c.opts.Password == _EMPTY_ {\n\t\t\tc.opts.Password = ws.cookiePassword\n\t\t}\n\t\t// if token not in the CONNECT, use the cookie token (possibly empty).\n\t\tif c.opts.Token == _EMPTY_ {\n\t\t\tc.opts.Token = ws.cookieToken\n\t\t}\n\t}\n\n\t// when not in operator mode, discard the jwt\n\tif srv != nil && srv.trustedKeys == nil {\n\t\tc.opts.JWT = _EMPTY_\n\t}\n\tujwt := c.opts.JWT\n\n\t// For headers both client and server need to support.\n\tc.headers = supportsHeaders && c.opts.Headers\n\tc.mu.Unlock()\n\n\tif srv != nil {\n\t\t// Applicable to clients only:\n\t\t// As soon as c.opts is unmarshalled and if the proto is at\n\t\t// least ClientProtoInfo, we need to increment the following counter.\n\t\t// This is decremented when client is removed from the server's\n\t\t// clients map.\n\t\tif kind == CLIENT && proto >= ClientProtoInfo {\n\t\t\tsrv.mu.Lock()\n\t\t\tsrv.cproto++\n\t\t\tsrv.mu.Unlock()\n\t\t}\n\n\t\t// Check for Auth\n\t\tif ok := srv.checkAuthentication(c); !ok {\n\t\t\t// We may fail here because we reached max limits on an account.\n\t\t\tif ujwt != _EMPTY_ {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tacc := c.acc\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tsrv.mu.Lock()\n\t\t\t\ttooManyAccCons := acc != nil && acc != srv.gacc\n\t\t\t\tsrv.mu.Unlock()\n\t\t\t\tif tooManyAccCons {\n\t\t\t\t\treturn ErrTooManyAccountConnections\n\t\t\t\t}\n\t\t\t}\n\t\t\tc.authViolation()\n\t\t\treturn ErrAuthentication\n\t\t}\n\n\t\t// Check for Account designation, we used to have this as an optional feature for dynamic\n\t\t// sandbox environments. Now its considered an error.\n\t\tif accountNew || account != _EMPTY_ {\n\t\t\tc.authViolation()\n\t\t\treturn ErrAuthentication\n\t\t}\n\n\t\t// If no account designation.\n\t\t// Do this only for CLIENT and LEAF connections.\n\t\tif c.acc == nil && (c.kind == CLIENT || c.kind == LEAF) {\n\t\t\t// By default register with the global account.\n\t\t\tc.registerWithAccount(srv.globalAccount())\n\t\t}\n\n\t\t// Initialize user info used in logs.\n\t\tc.mu.Lock()\n\t\tacc := c.acc\n\t\tif c.getRawAuthUser() != _EMPTY_ {\n\t\t\tc.ncsUser.Store(c.getAuthUserLabel())\n\t\t}\n\t\tc.mu.Unlock()\n\t\tif acc != nil {\n\t\t\tacc.mu.RLock()\n\t\t\tc.ncsAcc.Store(acc.traceLabel())\n\t\t\tacc.mu.RUnlock()\n\t\t}\n\n\t\t// Enable logging connection details and auth info for this client.\n\t\tif c.kind == CLIENT && firstConnect && c.srv != nil {\n\t\t\tvar ncs string\n\t\t\tif c.opts.Version != _EMPTY_ {\n\t\t\t\tncs = fmt.Sprintf(\"v%s\", c.opts.Version)\n\t\t\t}\n\t\t\tif c.opts.Lang != _EMPTY_ {\n\t\t\t\tif c.opts.Version == _EMPTY_ {\n\t\t\t\t\tncs = c.opts.Lang\n\t\t\t\t} else {\n\t\t\t\t\tncs = fmt.Sprintf(\"%s:%s\", ncs, c.opts.Lang)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif c.opts.Name != _EMPTY_ {\n\t\t\t\tif c.opts.Version == _EMPTY_ && c.opts.Lang == _EMPTY_ {\n\t\t\t\t\tncs = c.opts.Name\n\t\t\t\t} else {\n\t\t\t\t\tncs = fmt.Sprintf(\"%s:%s\", ncs, c.opts.Name)\n\t\t\t\t}\n\t\t\t}\n\t\t\tvar acs string\n\t\t\taccl := c.ncsAcc.Load()\n\t\t\tauthUser := c.ncsUser.Load()\n\t\t\tif accl != nil && authUser != nil {\n\t\t\t\tacs = fmt.Sprintf(\"%s/%s\", accl, authUser)\n\t\t\t}\n\t\t\tswitch {\n\t\t\tcase ncs != _EMPTY_ && acs != _EMPTY_:\n\t\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - %q - %q\", c, ncs, acs))\n\t\t\tcase ncs != _EMPTY_:\n\t\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - %q\", c, ncs))\n\t\t\tcase acs != _EMPTY_:\n\t\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - %q\", c, acs))\n\t\t\t}\n\t\t}\n\t}\n\n\tswitch kind {\n\tcase CLIENT:\n\t\t// Check client protocol request if it exists.\n\t\tif proto < ClientProtoZero || proto > ClientProtoInfo {\n\t\t\tc.sendErr(ErrBadClientProtocol.Error())\n\t\t\tc.closeConnection(BadClientProtocolVersion)\n\t\t\treturn ErrBadClientProtocol\n\t\t}\n\t\t// Check to see that if no_responders is requested\n\t\t// they have header support on as well.\n\t\tc.mu.Lock()\n\t\tmisMatch := c.opts.NoResponders && !c.headers\n\t\tc.mu.Unlock()\n\t\tif misMatch {\n\t\t\tc.sendErr(ErrNoRespondersRequiresHeaders.Error())\n\t\t\tc.closeConnection(NoRespondersRequiresHeaders)\n\t\t\treturn ErrNoRespondersRequiresHeaders\n\t\t}\n\t\tif verbose {\n\t\t\tc.sendOK()\n\t\t}\n\tcase ROUTER:\n\t\t// Delegate the rest of processing to the route\n\t\treturn c.processRouteConnect(srv, arg, lang)\n\tcase GATEWAY:\n\t\t// Delegate the rest of processing to the gateway\n\t\treturn c.processGatewayConnect(arg)\n\tcase LEAF:\n\t\t// Delegate the rest of processing to the leaf node\n\t\treturn c.processLeafNodeConnect(srv, arg, lang)\n\t}\n\treturn nil\n}\n\nfunc (c *client) sendErrAndErr(err string) {\n\tc.sendErr(err)\n\tc.RateLimitErrorf(err)\n}\n\nfunc (c *client) sendErrAndDebug(err string) {\n\tc.sendErr(err)\n\tc.RateLimitDebugf(err)\n}\n\nfunc (c *client) authTimeout() {\n\tc.sendErrAndDebug(\"Authentication Timeout\")\n\tc.closeConnection(AuthenticationTimeout)\n}\n\nfunc (c *client) authExpired() {\n\tc.sendErrAndDebug(\"User Authentication Expired\")\n\tc.closeConnection(AuthenticationExpired)\n}\n\nfunc (c *client) accountAuthExpired() {\n\tc.sendErrAndDebug(\"Account Authentication Expired\")\n\tc.closeConnection(AuthenticationExpired)\n}\n\nfunc (c *client) authViolation() {\n\tauthErr := c.getAuthError()\n\tif authErr == nil {\n\t\tauthErr = ErrAuthentication\n\t}\n\treason := getAuthErrClosedState(authErr)\n\n\tvar s *Server\n\tif s = c.srv; s != nil {\n\t\tdefer s.sendAuthErrorEvent(c, reason.String())\n\t\tif c.getRawAuthUser() != _EMPTY_ {\n\t\t\tc.Errorf(\"%v - %s\", ErrAuthentication, c.getAuthUser())\n\t\t} else {\n\t\t\tc.Errorf(ErrAuthentication.Error())\n\t\t}\n\t}\n\tif c.isMqtt() {\n\t\tc.mqttEnqueueConnAck(mqttConnAckRCNotAuthorized, false)\n\t} else {\n\t\t// Send this to client, regardless of the authErr override.\n\t\tc.sendErr(\"Authorization Violation\")\n\t}\n\tc.closeConnection(reason)\n}\n\nfunc (c *client) maxAccountConnExceeded() {\n\tc.sendErrAndErr(ErrTooManyAccountConnections.Error())\n\tc.closeConnection(MaxAccountConnectionsExceeded)\n}\n\nfunc (c *client) maxConnExceeded() {\n\tc.sendErrAndErr(ErrTooManyConnections.Error())\n\tc.closeConnection(MaxConnectionsExceeded)\n}\n\nfunc (c *client) maxSubsExceeded() {\n\tif c.acc.shouldLogMaxSubErr() {\n\t\tc.Errorf(ErrTooManySubs.Error())\n\t}\n\tc.sendErr(ErrTooManySubs.Error())\n}\n\nfunc (c *client) maxPayloadViolation(sz int, max int32) {\n\tc.Errorf(\"%s: %d vs %d\", ErrMaxPayload.Error(), sz, max)\n\tc.sendErr(\"Maximum Payload Violation\")\n\tc.closeConnection(MaxPayloadExceeded)\n}\n\n// queueOutbound queues data for a clientconnection.\n// Lock should be held.\nfunc (c *client) queueOutbound(data []byte) {\n\t// Do not keep going if closed\n\tif c.isClosed() {\n\t\treturn\n\t}\n\n\t// Add to pending bytes total.\n\tc.out.pb += int64(len(data))\n\n\t// Take a copy of the slice ref so that we can chop bits off the beginning\n\t// without affecting the original \"data\" slice.\n\ttoBuffer := data\n\n\t// All of the queued []byte have a fixed capacity, so if there's a []byte\n\t// at the tail of the buffer list that isn't full yet, we should top that\n\t// up first. This helps to ensure we aren't pulling more []bytes from the\n\t// pool than we need to.\n\tif len(c.out.nb) > 0 {\n\t\tlast := &c.out.nb[len(c.out.nb)-1]\n\t\tif free := cap(*last) - len(*last); free > 0 {\n\t\t\tif l := len(toBuffer); l < free {\n\t\t\t\tfree = l\n\t\t\t}\n\t\t\t*last = append(*last, toBuffer[:free]...)\n\t\t\ttoBuffer = toBuffer[free:]\n\t\t}\n\t}\n\n\t// Now we can push the rest of the data into new []bytes from the pool\n\t// in fixed size chunks. This ensures we don't go over the capacity of any\n\t// of the buffers and end up reallocating.\n\tfor len(toBuffer) > 0 {\n\t\tnew := nbPoolGet(len(toBuffer))\n\t\tn := copy(new[:cap(new)], toBuffer)\n\t\tc.out.nb = append(c.out.nb, new[:n])\n\t\ttoBuffer = toBuffer[n:]\n\t}\n\n\t// Check for slow consumer via pending bytes limit.\n\t// ok to return here, client is going away.\n\tif c.kind == CLIENT && c.out.pb > c.out.mp {\n\t\t// Perf wise, it looks like it is faster to optimistically add than\n\t\t// checking current pb+len(data) and then add to pb.\n\t\tc.out.pb -= int64(len(data))\n\n\t\t// Increment the total and client's slow consumer counters.\n\t\tatomic.AddInt64(&c.srv.slowConsumers, 1)\n\t\tc.srv.scStats.clients.Add(1)\n\t\tif c.acc != nil {\n\t\t\tc.acc.stats.Lock()\n\t\t\tc.acc.stats.slowConsumers++\n\t\t\tc.acc.stats.Unlock()\n\t\t}\n\t\tc.Noticef(\"Slow Consumer Detected: MaxPending of %d Exceeded\", c.out.mp)\n\t\tc.markConnAsClosed(SlowConsumerPendingBytes)\n\t\treturn\n\t}\n\n\t// Check here if we should create a stall channel if we are falling behind.\n\t// We do this here since if we wait for consumer's writeLoop it could be\n\t// too late with large number of fan in producers.\n\t// If the outbound connection is > 75% of maximum pending allowed, create a stall gate.\n\tif c.out.pb > c.out.mp/4*3 && c.out.stc == nil {\n\t\tc.out.stc = make(chan struct{})\n\t}\n}\n\n// Assume the lock is held upon entry.\nfunc (c *client) enqueueProtoAndFlush(proto []byte, doFlush bool) {\n\tif c.isClosed() {\n\t\treturn\n\t}\n\tc.queueOutbound(proto)\n\tif !(doFlush && c.flushOutbound()) {\n\t\tc.flushSignal()\n\t}\n}\n\n// Queues and then flushes the connection. This should only be called when\n// the writeLoop cannot be started yet. Use enqueueProto() otherwise.\n// Lock is held on entry.\nfunc (c *client) sendProtoNow(proto []byte) {\n\tc.enqueueProtoAndFlush(proto, true)\n}\n\n// Enqueues the given protocol and signal the writeLoop if necessary.\n// Lock is held on entry.\nfunc (c *client) enqueueProto(proto []byte) {\n\tc.enqueueProtoAndFlush(proto, false)\n}\n\n// Assume the lock is held upon entry.\nfunc (c *client) sendPong() {\n\tif c.trace {\n\t\tc.traceOutOp(\"PONG\", nil)\n\t}\n\tc.enqueueProto([]byte(pongProto))\n}\n\n// Used to kick off a RTT measurement for latency tracking.\nfunc (c *client) sendRTTPing() bool {\n\tc.mu.Lock()\n\tsent := c.sendRTTPingLocked()\n\tc.mu.Unlock()\n\treturn sent\n}\n\n// Used to kick off a RTT measurement for latency tracking.\n// This is normally called only when the caller has checked that\n// the c.rtt is 0 and wants to force an update by sending a PING.\n// Client lock held on entry.\nfunc (c *client) sendRTTPingLocked() bool {\n\tif c.isMqtt() {\n\t\treturn false\n\t}\n\t// Most client libs send a CONNECT+PING and wait for a PONG from the\n\t// server. So if firstPongSent flag is set, it is ok for server to\n\t// send the PING. But in case we have client libs that don't do that,\n\t// allow the send of the PING if more than 2 secs have elapsed since\n\t// the client TCP connection was accepted.\n\tif !c.isClosed() &&\n\t\t(c.flags.isSet(firstPongSent) || time.Since(c.start) > maxNoRTTPingBeforeFirstPong) {\n\t\tc.sendPing()\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Assume the lock is held upon entry.\nfunc (c *client) sendPing() {\n\tc.rttStart = time.Now().UTC()\n\tc.ping.out++\n\tif c.trace {\n\t\tc.traceOutOp(\"PING\", nil)\n\t}\n\tc.enqueueProto([]byte(pingProto))\n}\n\n// Generates the INFO to be sent to the client with the client ID included.\n// info arg will be copied since passed by value.\n// Assume lock is held.\nfunc (c *client) generateClientInfoJSON(info Info, includeClientIP bool) []byte {\n\tinfo.CID = c.cid\n\tif includeClientIP {\n\t\tinfo.ClientIP = c.host\n\t}\n\tinfo.MaxPayload = c.mpay\n\tif c.isWebsocket() {\n\t\tinfo.ClientConnectURLs = info.WSConnectURLs\n\t\t// Otherwise lame duck info can panic\n\t\tif c.srv != nil {\n\t\t\tws := &c.srv.websocket\n\t\t\tinfo.TLSAvailable, info.TLSRequired = ws.tls, ws.tls\n\t\t\tinfo.Host, info.Port = ws.host, ws.port\n\t\t}\n\t}\n\tinfo.WSConnectURLs = nil\n\treturn generateInfoJSON(&info)\n}\n\nfunc (c *client) sendErr(err string) {\n\tc.mu.Lock()\n\tif c.trace {\n\t\tc.traceOutOp(\"-ERR\", []byte(err))\n\t}\n\tif !c.isMqtt() {\n\t\tc.enqueueProto([]byte(fmt.Sprintf(errProto, err)))\n\t}\n\tc.mu.Unlock()\n}\n\nfunc (c *client) sendOK() {\n\tc.mu.Lock()\n\tif c.trace {\n\t\tc.traceOutOp(\"OK\", nil)\n\t}\n\tc.enqueueProto([]byte(okProto))\n\tc.mu.Unlock()\n}\n\nfunc (c *client) processPing() {\n\tc.mu.Lock()\n\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\tc.sendPong()\n\n\t// Record this to suppress us sending one if this\n\t// is within a given time interval for activity.\n\tc.lastIn = time.Now()\n\n\t// If not a CLIENT, we are done. Also the CONNECT should\n\t// have been received, but make sure it is so before proceeding\n\tif c.kind != CLIENT || !c.flags.isSet(connectReceived) {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\t// If we are here, the CONNECT has been received so we know\n\t// if this client supports async INFO or not.\n\tvar (\n\t\tsendConnectInfo bool\n\t\tsrv = c.srv\n\t)\n\t// For the first PING (so firstPongSet is false) and for clients\n\t// that support async INFO protocols, we will send one with ConnectInfo=true,\n\t// the name of the account the client is bound to, and if the\n\t// account is the system account.\n\tif !c.flags.isSet(firstPongSent) {\n\t\t// Flip the flag.\n\t\tc.flags.set(firstPongSent)\n\t\t// Evaluate if we should send the INFO protocol.\n\t\tsendConnectInfo = srv != nil && c.opts.Protocol >= ClientProtoInfo\n\t}\n\tc.mu.Unlock()\n\n\tif sendConnectInfo {\n\t\tsrv.mu.Lock()\n\t\tinfo := srv.copyInfo()\n\t\tc.mu.Lock()\n\t\tinfo.RemoteAccount = c.acc.Name\n\t\tinfo.IsSystemAccount = c.acc == srv.SystemAccount()\n\t\tinfo.ConnectInfo = true\n\t\tc.enqueueProto(c.generateClientInfoJSON(info, true))\n\t\tc.mu.Unlock()\n\t\tsrv.mu.Unlock()\n\t}\n}\n\nfunc (c *client) processPong() {\n\tc.mu.Lock()\n\tc.ping.out = 0\n\tc.rtt = computeRTT(c.rttStart)\n\tsrv := c.srv\n\treorderGWs := c.kind == GATEWAY && c.gw.outbound\n\tvar ri *routeInfo\n\t// For a route with pooling, we may be instructed to start a new route.\n\tif c.kind == ROUTER && c.route != nil && c.route.startNewRoute != nil {\n\t\tri = c.route.startNewRoute\n\t\tc.route.startNewRoute = nil\n\t}\n\t// If compression is currently active for a route/leaf connection, if the\n\t// compression configuration is s2_auto, check if we should change\n\t// the compression level.\n\tif c.kind == ROUTER && needsCompression(c.route.compression) {\n\t\tc.updateS2AutoCompressionLevel(&srv.getOpts().Cluster.Compression, &c.route.compression)\n\t} else if c.kind == LEAF && needsCompression(c.leaf.compression) {\n\t\tvar co *CompressionOpts\n\t\tif r := c.leaf.remote; r != nil {\n\t\t\tco = &r.Compression\n\t\t} else {\n\t\t\tco = &srv.getOpts().LeafNode.Compression\n\t\t}\n\t\tc.updateS2AutoCompressionLevel(co, &c.leaf.compression)\n\t}\n\tc.mu.Unlock()\n\tif reorderGWs {\n\t\tsrv.gateway.orderOutboundConnections()\n\t}\n\tif ri != nil {\n\t\tsrv.startGoRoutine(func() {\n\t\t\tsrv.connectToRoute(ri.url, ri.rtype, true, ri.gossipMode, _EMPTY_)\n\t\t})\n\t}\n}\n\n// Select the s2 compression level based on the client's current RTT and the configured\n// RTT thresholds slice. If current level is different than selected one, save the\n// new compression level string and create a new s2 writer.\n// Lock held on entry.\nfunc (c *client) updateS2AutoCompressionLevel(co *CompressionOpts, compression *string) {\n\tif co.Mode != CompressionS2Auto {\n\t\treturn\n\t}\n\tif cm := selectS2AutoModeBasedOnRTT(c.rtt, co.RTTThresholds); cm != *compression {\n\t\t*compression = cm\n\t\tc.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...)\n\t}\n}\n\n// Will return the parts from the raw wire msg.\n// We return the `hdr` as a slice that is capped to the length of the headers\n// so that if the caller later tries to append to the returned header slice it\n// does not affect the message content.\nfunc (c *client) msgParts(data []byte) (hdr []byte, msg []byte) {\n\tif c != nil && c.pa.hdr > 0 {\n\t\treturn data[:c.pa.hdr:c.pa.hdr], data[c.pa.hdr:]\n\t}\n\treturn nil, data\n}\n\n// Header pubs take form HPUB [reply] \\r\\n\nfunc (c *client) processHeaderPub(arg, remaining []byte) error {\n\tif !c.headers {\n\t\treturn ErrMsgHeadersNotSupported\n\t}\n\n\t// Unroll splitArgs to avoid runtime/heap issues\n\ta := [MAX_HPUB_ARGS][]byte{}\n\targs := a[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 3:\n\t\tc.pa.subject = args[0]\n\t\tc.pa.reply = nil\n\t\tc.pa.hdr = parseSize(args[1])\n\t\tc.pa.size = parseSize(args[2])\n\t\tc.pa.hdb = args[1]\n\t\tc.pa.szb = args[2]\n\tcase 4:\n\t\tc.pa.subject = args[0]\n\t\tc.pa.reply = args[1]\n\t\tc.pa.hdr = parseSize(args[2])\n\t\tc.pa.size = parseSize(args[3])\n\t\tc.pa.hdb = args[2]\n\t\tc.pa.szb = args[3]\n\tdefault:\n\t\treturn fmt.Errorf(\"processHeaderPub Parse Error: %q\", arg)\n\t}\n\tif c.pa.hdr < 0 {\n\t\treturn fmt.Errorf(\"processHeaderPub Bad or Missing Header Size: %q\", arg)\n\t}\n\t// If number overruns an int64, parseSize() will have returned a negative value\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processHeaderPub Bad or Missing Total Size: %q\", arg)\n\t}\n\tif c.pa.hdr > c.pa.size {\n\t\treturn fmt.Errorf(\"processHeaderPub Header Size larger then TotalSize: %q\", arg)\n\t}\n\tmaxPayload := atomic.LoadInt32(&c.mpay)\n\t// Use int64() to avoid int32 overrun...\n\tif maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) {\n\t\t// If we are given the remaining read buffer (since we do blind reads\n\t\t// we may have the beginning of the message header/payload), we will\n\t\t// look for the tracing header and if found, we will generate a\n\t\t// trace event with the max payload ingress error.\n\t\t// Do this only for CLIENT connections.\n\t\tif c.kind == CLIENT && c.pa.hdr > 0 && len(remaining) > 0 {\n\t\t\thdr := remaining[:min(len(remaining), c.pa.hdr)]\n\t\t\tif td := getHeader(MsgTraceDest, hdr); len(td) > 0 {\n\t\t\t\tc.initAndSendIngressErrEvent(hdr, string(td), ErrMaxPayload)\n\t\t\t}\n\t\t}\n\t\tc.maxPayloadViolation(c.pa.size, maxPayload)\n\t\treturn ErrMaxPayload\n\t}\n\tif c.opts.Pedantic && !IsValidLiteralSubject(bytesToString(c.pa.subject)) {\n\t\tc.sendErr(\"Invalid Publish Subject\")\n\t}\n\treturn nil\n}\n\nfunc (c *client) processPub(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\ta := [MAX_PUB_ARGS][]byte{}\n\targs := a[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 2:\n\t\tc.pa.subject = args[0]\n\t\tc.pa.reply = nil\n\t\tc.pa.size = parseSize(args[1])\n\t\tc.pa.szb = args[1]\n\tcase 3:\n\t\tc.pa.subject = args[0]\n\t\tc.pa.reply = args[1]\n\t\tc.pa.size = parseSize(args[2])\n\t\tc.pa.szb = args[2]\n\tdefault:\n\t\treturn fmt.Errorf(\"processPub Parse Error: %q\", arg)\n\t}\n\t// If number overruns an int64, parseSize() will have returned a negative value\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processPub Bad or Missing Size: %q\", arg)\n\t}\n\tmaxPayload := atomic.LoadInt32(&c.mpay)\n\t// Use int64() to avoid int32 overrun...\n\tif maxPayload != jwt.NoLimit && int64(c.pa.size) > int64(maxPayload) {\n\t\tc.maxPayloadViolation(c.pa.size, maxPayload)\n\t\treturn ErrMaxPayload\n\t}\n\tif c.opts.Pedantic && !IsValidLiteralSubject(bytesToString(c.pa.subject)) {\n\t\tc.sendErr(\"Invalid Publish Subject\")\n\t}\n\treturn nil\n}\n\nfunc splitArg(arg []byte) [][]byte {\n\ta := [MAX_MSG_ARGS][]byte{}\n\targs := a[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\treturn args\n}\n\nfunc (c *client) parseSub(argo []byte, noForward bool) error {\n\t// Copy so we do not reference a potentially large buffer\n\t// FIXME(dlc) - make more efficient.\n\targ := make([]byte, len(argo))\n\tcopy(arg, argo)\n\targs := splitArg(arg)\n\tvar (\n\t\tsubject []byte\n\t\tqueue []byte\n\t\tsid []byte\n\t)\n\tswitch len(args) {\n\tcase 2:\n\t\tsubject = args[0]\n\t\tqueue = nil\n\t\tsid = args[1]\n\tcase 3:\n\t\tsubject = args[0]\n\t\tqueue = args[1]\n\t\tsid = args[2]\n\tdefault:\n\t\treturn fmt.Errorf(\"processSub Parse Error: %q\", arg)\n\t}\n\t// If there was an error, it has been sent to the client. We don't return an\n\t// error here to not close the connection as a parsing error.\n\tc.processSub(subject, queue, sid, nil, noForward)\n\treturn nil\n}\n\nfunc (c *client) processSub(subject, queue, bsid []byte, cb msgHandler, noForward bool) (*subscription, error) {\n\treturn c.processSubEx(subject, queue, bsid, cb, noForward, false, false)\n}\n\nfunc (c *client) processSubEx(subject, queue, bsid []byte, cb msgHandler, noForward, si, rsi bool) (*subscription, error) {\n\t// Create the subscription\n\tsub := &subscription{client: c, subject: subject, queue: queue, sid: bsid, icb: cb, si: si, rsi: rsi}\n\n\tc.mu.Lock()\n\n\t// Indicate activity.\n\tc.in.subs++\n\n\t// Grab connection type, account and server info.\n\tkind := c.kind\n\tacc := c.acc\n\tsrv := c.srv\n\n\tsid := bytesToString(sub.sid)\n\n\t// This check does not apply to SYSTEM or JETSTREAM or ACCOUNT clients (because they don't have a `nc`...)\n\t// When a connection is closed though, we set c.subs to nil. So check for the map to not be nil.\n\tif (c.isClosed() && !isInternalClient(kind)) || (c.subs == nil) {\n\t\tc.mu.Unlock()\n\t\treturn nil, ErrConnectionClosed\n\t}\n\n\t// Check permissions if applicable.\n\tif kind == CLIENT {\n\t\t// First do a pass whether queue subscription is valid. This does not necessarily\n\t\t// mean that it will not be able to plain subscribe.\n\t\t//\n\t\t// allow = [\"foo\"] -> can subscribe or queue subscribe to foo using any queue\n\t\t// allow = [\"foo v1\"] -> can only queue subscribe to 'foo v1', no plain subs allowed.\n\t\t// allow = [\"foo\", \"foo v1\"] -> can subscribe to 'foo' but can only queue subscribe to 'foo v1'\n\t\t//\n\t\tif sub.queue != nil {\n\t\t\tif !c.canSubscribe(string(sub.subject), string(sub.queue)) || string(sub.queue) == sysGroup {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.subPermissionViolation(sub)\n\t\t\t\treturn nil, ErrSubscribePermissionViolation\n\t\t\t}\n\t\t} else if !c.canSubscribe(string(sub.subject)) {\n\t\t\tc.mu.Unlock()\n\t\t\tc.subPermissionViolation(sub)\n\t\t\treturn nil, ErrSubscribePermissionViolation\n\t\t}\n\n\t\tif opts := srv.getOpts(); opts != nil && opts.MaxSubTokens > 0 {\n\t\t\tif len(bytes.Split(sub.subject, []byte(tsep))) > int(opts.MaxSubTokens) {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.maxTokensViolation(sub)\n\t\t\t\treturn nil, ErrTooManySubTokens\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check if we have a maximum on the number of subscriptions.\n\tif c.subsAtLimit() {\n\t\tc.mu.Unlock()\n\t\tc.maxSubsExceeded()\n\t\treturn nil, ErrTooManySubs\n\t}\n\n\tvar updateGWs bool\n\tvar err error\n\n\t// Subscribe here.\n\tes := c.subs[sid]\n\tif es == nil {\n\t\tc.subs[sid] = sub\n\t\tif acc != nil && acc.sl != nil {\n\t\t\terr = acc.sl.Insert(sub)\n\t\t\tif err != nil {\n\t\t\t\tdelete(c.subs, sid)\n\t\t\t} else {\n\t\t\t\tupdateGWs = c.srv.gateway.enabled\n\t\t\t}\n\t\t}\n\t}\n\t// Unlocked from here onward\n\tc.mu.Unlock()\n\n\tif err != nil {\n\t\tc.sendErr(\"Invalid Subject\")\n\t\treturn nil, ErrMalformedSubject\n\t} else if c.opts.Verbose && kind != SYSTEM {\n\t\tc.sendOK()\n\t}\n\n\t// If it was already registered, return it.\n\tif es != nil {\n\t\treturn es, nil\n\t}\n\n\t// No account just return.\n\tif acc == nil {\n\t\treturn sub, nil\n\t}\n\n\tif err := c.addShadowSubscriptions(acc, sub); err != nil {\n\t\tc.Errorf(err.Error())\n\t}\n\n\tif noForward {\n\t\treturn sub, nil\n\t}\n\n\t// If we are routing and this is a local sub, add to the route map for the associated account.\n\tif kind == CLIENT || kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT {\n\t\tsrv.updateRouteSubscriptionMap(acc, sub, 1)\n\t\tif updateGWs {\n\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, 1)\n\t\t}\n\t}\n\t// Now check on leafnode updates.\n\tacc.updateLeafNodes(sub, 1)\n\treturn sub, nil\n}\n\n// Used to pass stream import matches to addShadowSub\ntype ime struct {\n\tim *streamImport\n\toverlapSubj string\n\tdyn bool\n}\n\n// If the client's account has stream imports and there are matches for this\n// subscription's subject, then add shadow subscriptions in the other accounts\n// that export this subject.\nfunc (c *client) addShadowSubscriptions(acc *Account, sub *subscription) error {\n\tif acc == nil {\n\t\treturn ErrMissingAccount\n\t}\n\n\tvar (\n\t\t_ims [16]ime\n\t\tims = _ims[:0]\n\t\timTsa [32]string\n\t\ttokens []string\n\t\ttsa [32]string\n\t\thasWC bool\n\t\ttokensModified bool\n\t)\n\n\tacc.mu.RLock()\n\t// If this is from a service import, ignore.\n\tif sub.si {\n\t\tacc.mu.RUnlock()\n\t\treturn nil\n\t}\n\tsubj := bytesToString(sub.subject)\n\tif len(acc.imports.streams) > 0 {\n\t\ttokens = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\tfor _, tk := range tokens {\n\t\t\tif tk == pwcs {\n\t\t\t\thasWC = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !hasWC && tokens[len(tokens)-1] == fwcs {\n\t\t\thasWC = true\n\t\t}\n\t}\n\t// Loop over the import subjects. We have 4 scenarios. If we have an\n\t// exact match or a superset match we should use the from field from\n\t// the import. If we are a subset or overlap, we have to dynamically calculate\n\t// the subject. On overlap, ime requires the overlap subject.\n\tfor _, im := range acc.imports.streams {\n\t\tif im.invalid {\n\t\t\tcontinue\n\t\t}\n\t\tif subj == im.to {\n\t\t\tims = append(ims, ime{im, _EMPTY_, false})\n\t\t\tcontinue\n\t\t}\n\t\tif tokensModified {\n\t\t\t// re-tokenize subj to overwrite modifications from a previous iteration\n\t\t\ttokens = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\t\ttokensModified = false\n\t\t}\n\t\timTokens := tokenizeSubjectIntoSlice(imTsa[:0], im.to)\n\n\t\tif isSubsetMatchTokenized(tokens, imTokens) {\n\t\t\tims = append(ims, ime{im, _EMPTY_, true})\n\t\t} else if hasWC {\n\t\t\tif isSubsetMatchTokenized(imTokens, tokens) {\n\t\t\t\tims = append(ims, ime{im, _EMPTY_, false})\n\t\t\t} else {\n\t\t\t\timTokensLen := len(imTokens)\n\t\t\t\tfor i, t := range tokens {\n\t\t\t\t\tif i >= imTokensLen {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tif t == pwcs && imTokens[i] != fwcs {\n\t\t\t\t\t\ttokens[i] = imTokens[i]\n\t\t\t\t\t\ttokensModified = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttokensLen := len(tokens)\n\t\t\t\tlastIdx := tokensLen - 1\n\t\t\t\tif tokens[lastIdx] == fwcs {\n\t\t\t\t\tif imTokensLen >= tokensLen {\n\t\t\t\t\t\t// rewrite \">\" in tokens to be more specific\n\t\t\t\t\t\ttokens[lastIdx] = imTokens[lastIdx]\n\t\t\t\t\t\ttokensModified = true\n\t\t\t\t\t\tif imTokensLen > tokensLen {\n\t\t\t\t\t\t\t// copy even more specific parts from import\n\t\t\t\t\t\t\ttokens = append(tokens, imTokens[tokensLen:]...)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif isSubsetMatchTokenized(tokens, imTokens) {\n\t\t\t\t\t// As isSubsetMatchTokenized was already called with tokens and imTokens,\n\t\t\t\t\t// we wouldn't be here if it where not for tokens being modified.\n\t\t\t\t\t// Hence, Join to re compute the subject string\n\t\t\t\t\tims = append(ims, ime{im, strings.Join(tokens, tsep), true})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tacc.mu.RUnlock()\n\n\tvar shadow []*subscription\n\n\tif len(ims) > 0 {\n\t\tshadow = make([]*subscription, 0, len(ims))\n\t}\n\n\t// Now walk through collected stream imports that matched.\n\tfor i := 0; i < len(ims); i++ {\n\t\time := &ims[i]\n\t\t// We will create a shadow subscription.\n\t\tnsub, err := c.addShadowSub(sub, ime)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tshadow = append(shadow, nsub)\n\t}\n\n\tif shadow != nil {\n\t\tc.mu.Lock()\n\t\tsub.shadow = shadow\n\t\tc.mu.Unlock()\n\t}\n\n\treturn nil\n}\n\n// Add in the shadow subscription.\nfunc (c *client) addShadowSub(sub *subscription, ime *ime) (*subscription, error) {\n\tc.mu.Lock()\n\tnsub := *sub // copy\n\tc.mu.Unlock()\n\n\tim := ime.im\n\tnsub.im = im\n\n\tif !im.usePub && ime.dyn && im.tr != nil {\n\t\tif im.rtr == nil {\n\t\t\tim.rtr = im.tr.reverse()\n\t\t}\n\t\ts := bytesToString(nsub.subject)\n\t\tif ime.overlapSubj != _EMPTY_ {\n\t\t\ts = ime.overlapSubj\n\t\t}\n\t\tsubj := im.rtr.TransformSubject(s)\n\n\t\tnsub.subject = []byte(subj)\n\t} else if !im.usePub || (im.usePub && ime.overlapSubj != _EMPTY_) || !ime.dyn {\n\t\tif ime.overlapSubj != _EMPTY_ {\n\t\t\tnsub.subject = []byte(ime.overlapSubj)\n\t\t} else {\n\t\t\tnsub.subject = []byte(im.from)\n\t\t}\n\t}\n\t// Else use original subject\n\n\tc.Debugf(\"Creating import subscription on %q from account %q\", nsub.subject, im.acc.Name)\n\n\tif err := im.acc.sl.Insert(&nsub); err != nil {\n\t\terrs := fmt.Sprintf(\"Could not add shadow import subscription for account %q\", im.acc.Name)\n\t\tc.Debugf(errs)\n\t\treturn nil, errors.New(errs)\n\t}\n\n\t// Update our route map here. But only if we are not a leaf node or a hub leafnode.\n\tif c.kind != LEAF || c.isHubLeafNode() {\n\t\tc.srv.updateRemoteSubscription(im.acc, &nsub, 1)\n\t} else if c.kind == LEAF {\n\t\t// Update all leafnodes that connect to this server. Note that we could have\n\t\t// used the updateLeafNodes() function since when it does invoke updateSmap()\n\t\t// this function already takes care of not sending to a spoke leafnode since\n\t\t// the `nsub` here is already from a spoke leafnode, but to be explicit, we\n\t\t// use this version that updates only leafnodes that connect to this server.\n\t\tim.acc.updateLeafNodesEx(&nsub, 1, true)\n\t}\n\n\treturn &nsub, nil\n}\n\n// canSubscribe determines if the client is authorized to subscribe to the\n// given subject. Assumes caller is holding lock.\nfunc (c *client) canSubscribe(subject string, optQueue ...string) bool {\n\tif c.perms == nil {\n\t\treturn true\n\t}\n\n\tallowed, checkAllow := true, true\n\n\t// Optional queue group.\n\tvar queue string\n\tif len(optQueue) > 0 {\n\t\tqueue = optQueue[0]\n\t}\n\n\t// For CLIENT connections that are MQTT, or other types of connections, we will\n\t// implicitly allow anything that starts with the \"$MQTT.\" prefix. However,\n\t// we don't just return here, we skip the check for \"allow\" but will check \"deny\".\n\tif (c.isMqtt() || (c.kind != CLIENT)) && strings.HasPrefix(subject, mqttPrefix) {\n\t\tcheckAllow = false\n\t}\n\t// Check allow list. If no allow list that means all are allowed. Deny can overrule.\n\tif checkAllow && c.perms.sub.allow != nil {\n\t\tr := c.perms.sub.allow.Match(subject)\n\t\tallowed = len(r.psubs) > 0\n\t\tif queue != _EMPTY_ && len(r.qsubs) > 0 {\n\t\t\t// If the queue appears in the allow list, then DO allow.\n\t\t\tallowed = queueMatches(queue, r.qsubs)\n\t\t}\n\t\t// Leafnodes operate slightly differently in that they allow broader scoped subjects.\n\t\t// They will prune based on publish perms before sending to a leafnode client.\n\t\tif !allowed && c.kind == LEAF && subjectHasWildcard(subject) {\n\t\t\tr := c.perms.sub.allow.ReverseMatch(subject)\n\t\t\tallowed = len(r.psubs) != 0\n\t\t}\n\t}\n\t// If we have a deny list and we think we are allowed, check that as well.\n\tif allowed && c.perms.sub.deny != nil {\n\t\tr := c.perms.sub.deny.Match(subject)\n\t\tallowed = len(r.psubs) == 0\n\n\t\tif queue != _EMPTY_ && len(r.qsubs) > 0 {\n\t\t\t// If the queue appears in the deny list, then DO NOT allow.\n\t\t\tallowed = !queueMatches(queue, r.qsubs)\n\t\t}\n\n\t\t// We use the actual subscription to signal us to spin up the deny mperms\n\t\t// and cache. We check if the subject is a wildcard that contains any of\n\t\t// the deny clauses.\n\t\t// FIXME(dlc) - We could be smarter and track when these go away and remove.\n\t\tif allowed && c.mperms == nil && subjectHasWildcard(subject) {\n\t\t\t// Whip through the deny array and check if this wildcard subject is within scope.\n\t\t\tfor _, sub := range c.darray {\n\t\t\t\tif subjectIsSubsetMatch(sub, subject) {\n\t\t\t\t\tc.loadMsgDenyFilter()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn allowed\n}\n\nfunc queueMatches(queue string, qsubs [][]*subscription) bool {\n\tif len(qsubs) == 0 {\n\t\treturn true\n\t}\n\tfor _, qsub := range qsubs {\n\t\tqs := qsub[0]\n\t\tqname := bytesToString(qs.queue)\n\n\t\t// NOTE: '*' and '>' tokens can also be valid\n\t\t// queue names so we first check against the\n\t\t// literal name. e.g. v1.* == v1.*\n\t\tif queue == qname || (subjectHasWildcard(qname) && subjectIsSubsetMatch(queue, qname)) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Low level unsubscribe for a given client.\nfunc (c *client) unsubscribe(acc *Account, sub *subscription, force, remove bool) {\n\tif s := c.srv; s != nil && s.isShuttingDown() {\n\t\treturn\n\t}\n\n\tc.mu.Lock()\n\tif !force && sub.max > 0 && sub.nm < sub.max {\n\t\tc.Debugf(\"Deferring actual UNSUB(%s): %d max, %d received\", sub.subject, sub.max, sub.nm)\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\tif c.trace {\n\t\tc.traceOp(\"<-> %s\", \"DELSUB\", sub.sid)\n\t}\n\n\t// Remove accounting if requested. This will be false when we close a connection\n\t// with open subscriptions.\n\tif remove {\n\t\tdelete(c.subs, bytesToString(sub.sid))\n\t\tif acc != nil {\n\t\t\tacc.sl.Remove(sub)\n\t\t}\n\t}\n\n\t// Check to see if we have shadow subscriptions.\n\tvar updateRoute bool\n\tvar isSpokeLeaf bool\n\tshadowSubs := sub.shadow\n\tsub.shadow = nil\n\tif len(shadowSubs) > 0 {\n\t\tisSpokeLeaf = c.isSpokeLeafNode()\n\t\tupdateRoute = !isSpokeLeaf && (c.kind == CLIENT || c.kind == SYSTEM || c.kind == LEAF || c.kind == JETSTREAM) && c.srv != nil\n\t}\n\tsub.close()\n\tc.mu.Unlock()\n\n\t// Process shadow subs if we have them.\n\tfor _, nsub := range shadowSubs {\n\t\tif err := nsub.im.acc.sl.Remove(nsub); err != nil {\n\t\t\tc.Debugf(\"Could not remove shadow import subscription for account %q\", nsub.im.acc.Name)\n\t\t}\n\t\tif updateRoute {\n\t\t\tc.srv.updateRemoteSubscription(nsub.im.acc, nsub, -1)\n\t\t} else if isSpokeLeaf {\n\t\t\tnsub.im.acc.updateLeafNodesEx(nsub, -1, true)\n\t\t}\n\t}\n\n\t// Now check to see if this was part of a respMap entry for service imports.\n\t// We can skip subscriptions on reserved replies.\n\tif acc != nil && !isReservedReply(sub.subject) {\n\t\tacc.checkForReverseEntry(string(sub.subject), nil, true)\n\t}\n}\n\nfunc (c *client) processUnsub(arg []byte) error {\n\targs := splitArg(arg)\n\tvar sid []byte\n\tmax := int64(-1)\n\n\tswitch len(args) {\n\tcase 1:\n\t\tsid = args[0]\n\tcase 2:\n\t\tsid = args[0]\n\t\tmax = int64(parseSize(args[1]))\n\tdefault:\n\t\treturn fmt.Errorf(\"processUnsub Parse Error: %q\", arg)\n\t}\n\n\tvar sub *subscription\n\tvar ok, unsub bool\n\n\tc.mu.Lock()\n\n\t// Indicate activity.\n\tc.in.subs++\n\n\t// Grab connection type.\n\tkind := c.kind\n\tsrv := c.srv\n\tvar acc *Account\n\n\tupdateGWs := false\n\tif sub, ok = c.subs[string(sid)]; ok {\n\t\tacc = c.acc\n\t\tif max > 0 && max > sub.nm {\n\t\t\tsub.max = max\n\t\t} else {\n\t\t\t// Clear it here to override\n\t\t\tsub.max = 0\n\t\t\tunsub = true\n\t\t}\n\t\tupdateGWs = srv.gateway.enabled\n\t}\n\tc.mu.Unlock()\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\n\tif unsub {\n\t\tc.unsubscribe(acc, sub, false, true)\n\t\tif acc != nil && (kind == CLIENT || kind == SYSTEM || kind == ACCOUNT || kind == JETSTREAM) {\n\t\t\tsrv.updateRouteSubscriptionMap(acc, sub, -1)\n\t\t\tif updateGWs {\n\t\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, -1)\n\t\t\t}\n\t\t}\n\t\t// Now check on leafnode updates.\n\t\tacc.updateLeafNodes(sub, -1)\n\t}\n\n\treturn nil\n}\n\n// checkDenySub will check if we are allowed to deliver this message in the\n// presence of deny clauses for subscriptions. Deny clauses will not prevent\n// larger scoped wildcard subscriptions, so we need to check at delivery time.\n// Lock should be held.\nfunc (c *client) checkDenySub(subject string) bool {\n\tif denied, ok := c.mperms.dcache[subject]; ok {\n\t\treturn denied\n\t} else if np, _ := c.mperms.deny.NumInterest(subject); np != 0 {\n\t\tc.mperms.dcache[subject] = true\n\t\treturn true\n\t} else {\n\t\tc.mperms.dcache[subject] = false\n\t}\n\tif len(c.mperms.dcache) > maxDenyPermCacheSize {\n\t\tc.pruneDenyCache()\n\t}\n\treturn false\n}\n\n// Create a message header for routes or leafnodes. Header and origin cluster aware.\nfunc (c *client) msgHeaderForRouteOrLeaf(subj, reply []byte, rt *routeTarget, acc *Account) []byte {\n\thasHeader := c.pa.hdr > 0\n\tsubclient := rt.sub.client\n\tcanReceiveHeader := subclient.headers\n\n\tmh := c.msgb[:msgHeadProtoLen]\n\tkind := subclient.kind\n\tvar lnoc bool\n\n\tif kind == ROUTER {\n\t\t// If we are coming from a leaf with an origin cluster we need to handle differently\n\t\t// if we can. We will send a route based LMSG which has origin cluster and headers\n\t\t// by default.\n\t\tif c.kind == LEAF && c.remoteCluster() != _EMPTY_ {\n\t\t\tsubclient.mu.Lock()\n\t\t\tlnoc = subclient.route.lnoc\n\t\t\tsubclient.mu.Unlock()\n\t\t}\n\t\tif lnoc {\n\t\t\tmh[0] = 'L'\n\t\t\tmh = append(mh, c.remoteCluster()...)\n\t\t\tmh = append(mh, ' ')\n\t\t} else {\n\t\t\t// Router (and Gateway) nodes are RMSG. Set here since leafnodes may rewrite.\n\t\t\tmh[0] = 'R'\n\t\t}\n\t\tif len(subclient.route.accName) == 0 {\n\t\t\tmh = append(mh, acc.Name...)\n\t\t\tmh = append(mh, ' ')\n\t\t}\n\t} else {\n\t\t// Leaf nodes are LMSG\n\t\tmh[0] = 'L'\n\t\t// Remap subject if its a shadow subscription, treat like a normal client.\n\t\tif rt.sub.im != nil {\n\t\t\tif rt.sub.im.tr != nil {\n\t\t\t\tto := rt.sub.im.tr.TransformSubject(bytesToString(subj))\n\t\t\t\tsubj = []byte(to)\n\t\t\t} else if !rt.sub.im.usePub {\n\t\t\t\tsubj = []byte(rt.sub.im.to)\n\t\t\t}\n\t\t}\n\t}\n\tmh = append(mh, subj...)\n\tmh = append(mh, ' ')\n\n\tif len(rt.qs) > 0 {\n\t\tif len(reply) > 0 {\n\t\t\tmh = append(mh, \"+ \"...) // Signal that there is a reply.\n\t\t\tmh = append(mh, reply...)\n\t\t\tmh = append(mh, ' ')\n\t\t} else {\n\t\t\tmh = append(mh, \"| \"...) // Only queues\n\t\t}\n\t\tmh = append(mh, rt.qs...)\n\t} else if len(reply) > 0 {\n\t\tmh = append(mh, reply...)\n\t\tmh = append(mh, ' ')\n\t}\n\n\tif lnoc {\n\t\t// leafnode origin LMSG always have a header entry even if zero.\n\t\tif c.pa.hdr <= 0 {\n\t\t\tmh = append(mh, '0')\n\t\t} else {\n\t\t\tmh = append(mh, c.pa.hdb...)\n\t\t}\n\t\tmh = append(mh, ' ')\n\t\tmh = append(mh, c.pa.szb...)\n\t} else if hasHeader {\n\t\tif canReceiveHeader {\n\t\t\tmh[0] = 'H'\n\t\t\tmh = append(mh, c.pa.hdb...)\n\t\t\tmh = append(mh, ' ')\n\t\t\tmh = append(mh, c.pa.szb...)\n\t\t} else {\n\t\t\t// If we are here we need to truncate the payload size\n\t\t\tnsz := strconv.Itoa(c.pa.size - c.pa.hdr)\n\t\t\tmh = append(mh, nsz...)\n\t\t}\n\t} else {\n\t\tmh = append(mh, c.pa.szb...)\n\t}\n\treturn append(mh, _CRLF_...)\n}\n\n// Create a message header for clients. Header aware.\nfunc (c *client) msgHeader(subj, reply []byte, sub *subscription) []byte {\n\t// See if we should do headers. We have to have a headers msg and\n\t// the client we are going to deliver to needs to support headers as well.\n\thasHeader := c.pa.hdr > 0\n\tcanReceiveHeader := sub.client != nil && sub.client.headers\n\n\tvar mh []byte\n\tif hasHeader && canReceiveHeader {\n\t\tmh = c.msgb[:msgHeadProtoLen]\n\t\tmh[0] = 'H'\n\t} else {\n\t\tmh = c.msgb[1:msgHeadProtoLen]\n\t}\n\tmh = append(mh, subj...)\n\tmh = append(mh, ' ')\n\n\tif len(sub.sid) > 0 {\n\t\tmh = append(mh, sub.sid...)\n\t\tmh = append(mh, ' ')\n\t}\n\tif reply != nil {\n\t\tmh = append(mh, reply...)\n\t\tmh = append(mh, ' ')\n\t}\n\tif hasHeader {\n\t\tif canReceiveHeader {\n\t\t\tmh = append(mh, c.pa.hdb...)\n\t\t\tmh = append(mh, ' ')\n\t\t\tmh = append(mh, c.pa.szb...)\n\t\t} else {\n\t\t\t// If we are here we need to truncate the payload size\n\t\t\tnsz := strconv.Itoa(c.pa.size - c.pa.hdr)\n\t\t\tmh = append(mh, nsz...)\n\t\t}\n\t} else {\n\t\tmh = append(mh, c.pa.szb...)\n\t}\n\tmh = append(mh, _CRLF_...)\n\treturn mh\n}\n\nfunc (c *client) stalledWait(producer *client) {\n\t// Check to see if we have exceeded our total wait time per readLoop invocation.\n\tif producer.in.tst > stallTotalAllowed {\n\t\treturn\n\t}\n\n\t// Grab stall channel which the slow consumer will close when caught up.\n\tstall := c.out.stc\n\n\t// Calculate stall time.\n\tttl := stallClientMinDuration\n\tif c.out.pb >= c.out.mp {\n\t\tttl = stallClientMaxDuration\n\t}\n\n\tc.mu.Unlock()\n\tdefer c.mu.Lock()\n\n\t// Track per client and total client stalls.\n\tatomic.AddInt64(&c.stalls, 1)\n\tif c.srv != nil {\n\t\tatomic.AddInt64(&c.srv.stalls, 1)\n\t}\n\n\t// Now check if we are close to total allowed.\n\tif producer.in.tst+ttl > stallTotalAllowed {\n\t\tttl = stallTotalAllowed - producer.in.tst\n\t}\n\tdelay := time.NewTimer(ttl)\n\tdefer delay.Stop()\n\n\tstart := time.Now()\n\tselect {\n\tcase <-stall:\n\tcase <-delay.C:\n\t\tproducer.Debugf(\"Timed out of fast producer stall (%v)\", ttl)\n\t}\n\tproducer.in.tst += time.Since(start)\n}\n\n// Used to treat maps as efficient set\nvar needFlush = struct{}{}\n\n// deliverMsg will deliver a message to a matching subscription and its underlying client.\n// We process all connection/client types. mh is the part that will be protocol/client specific.\nfunc (c *client) deliverMsg(prodIsMQTT bool, sub *subscription, acc *Account, subject, reply, mh, msg []byte, gwrply bool) bool {\n\t// Check if message tracing is enabled.\n\tmt, traceOnly := c.isMsgTraceEnabled()\n\n\tclient := sub.client\n\t// Check sub client and check echo. Only do this if not a service import.\n\tif client == nil || (c == client && !client.echo && !sub.si) {\n\t\tif client != nil && mt != nil {\n\t\t\tclient.mu.Lock()\n\t\t\tmt.addEgressEvent(client, sub, errMsgTraceNoEcho)\n\t\t\tclient.mu.Unlock()\n\t\t}\n\t\treturn false\n\t}\n\n\tclient.mu.Lock()\n\n\t// Check if we have a subscribe deny clause. This will trigger us to check the subject\n\t// for a match against the denied subjects.\n\tif client.mperms != nil && client.checkDenySub(string(subject)) {\n\t\tmt.addEgressEvent(client, sub, errMsgTraceSubDeny)\n\t\tclient.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// New race detector forces this now.\n\tif sub.isClosed() {\n\t\tmt.addEgressEvent(client, sub, errMsgTraceSubClosed)\n\t\tclient.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Check if we are a leafnode and have perms to check.\n\tif client.kind == LEAF && client.perms != nil {\n\t\tvar subjectToCheck []byte\n\t\tif subject[0] == '_' && bytes.HasPrefix(subject, []byte(gwReplyPrefix)) {\n\t\t\tsubjectToCheck = subject[gwSubjectOffset:]\n\t\t} else if subject[0] == '$' && bytes.HasPrefix(subject, []byte(oldGWReplyPrefix)) {\n\t\t\tsubjectToCheck = subject[oldGWReplyStart:]\n\t\t} else {\n\t\t\tsubjectToCheck = subject\n\t\t}\n\t\tif !client.pubAllowedFullCheck(string(subjectToCheck), true, true) {\n\t\t\tmt.addEgressEvent(client, sub, errMsgTracePubViolation)\n\t\t\tclient.mu.Unlock()\n\t\t\tclient.Debugf(\"Not permitted to deliver to %q\", subjectToCheck)\n\t\t\treturn false\n\t\t}\n\t}\n\n\tvar mtErr string\n\tif mt != nil {\n\t\t// For non internal subscription, and if the remote does not support\n\t\t// the tracing feature...\n\t\tif sub.icb == nil && !client.msgTraceSupport() {\n\t\t\tif traceOnly {\n\t\t\t\t// We are not sending the message at all because the user\n\t\t\t\t// expects a trace-only and the remote does not support\n\t\t\t\t// tracing, which means that it would process/deliver this\n\t\t\t\t// message, which may break applications.\n\t\t\t\t// Add the Egress with the no-support error message.\n\t\t\t\tmt.addEgressEvent(client, sub, errMsgTraceOnlyNoSupport)\n\t\t\t\tclient.mu.Unlock()\n\t\t\t\treturn false\n\t\t\t}\n\t\t\t// If we are doing delivery, we will still forward the message,\n\t\t\t// but we add an error to the Egress event to hint that one should\n\t\t\t// not expect a tracing event from that remote.\n\t\t\tmtErr = errMsgTraceNoSupport\n\t\t}\n\t\t// For ROUTER, GATEWAY and LEAF, even if we intend to do tracing only,\n\t\t// we will still deliver the message. The remote side will\n\t\t// generate an event based on what happened on that server.\n\t\tif traceOnly && (client.kind == ROUTER || client.kind == GATEWAY || client.kind == LEAF) {\n\t\t\ttraceOnly = false\n\t\t}\n\t\t// If we skip delivery and this is not for a service import, we are done.\n\t\tif traceOnly && (sub.icb == nil || c.noIcb) {\n\t\t\tmt.addEgressEvent(client, sub, _EMPTY_)\n\t\t\tclient.mu.Unlock()\n\t\t\t// Although the message is not actually delivered, for the\n\t\t\t// purpose of \"didDeliver\", we need to return \"true\" here.\n\t\t\treturn true\n\t\t}\n\t}\n\n\tsrv := client.srv\n\n\t// We don't want to bump the number of delivered messages to the subscription\n\t// if we are doing trace-only (since really we are not sending it to the sub).\n\tif !traceOnly {\n\t\tsub.nm++\n\t}\n\n\t// Check if we should auto-unsubscribe.\n\tif sub.max > 0 {\n\t\tif client.kind == ROUTER && sub.nm >= sub.max {\n\t\t\t// The only router based messages that we will see here are remoteReplies.\n\t\t\t// We handle these slightly differently.\n\t\t\tdefer client.removeReplySub(sub)\n\t\t} else {\n\t\t\t// For routing..\n\t\t\tshouldForward := client.kind == CLIENT || client.kind == SYSTEM && client.srv != nil\n\t\t\t// If we are at the exact number, unsubscribe but\n\t\t\t// still process the message in hand, otherwise\n\t\t\t// unsubscribe and drop message on the floor.\n\t\t\tif sub.nm == sub.max {\n\t\t\t\tclient.Debugf(\"Auto-unsubscribe limit of %d reached for sid '%s'\", sub.max, sub.sid)\n\t\t\t\t// Due to defer, reverse the code order so that execution\n\t\t\t\t// is consistent with other cases where we unsubscribe.\n\t\t\t\tif shouldForward {\n\t\t\t\t\tdefer srv.updateRemoteSubscription(client.acc, sub, -1)\n\t\t\t\t}\n\t\t\t\tdefer client.unsubscribe(client.acc, sub, true, true)\n\t\t\t} else if sub.nm > sub.max {\n\t\t\t\tclient.Debugf(\"Auto-unsubscribe limit [%d] exceeded\", sub.max)\n\t\t\t\tmt.addEgressEvent(client, sub, errMsgTraceAutoSubExceeded)\n\t\t\t\tclient.mu.Unlock()\n\t\t\t\tclient.unsubscribe(client.acc, sub, true, true)\n\t\t\t\tif shouldForward {\n\t\t\t\t\tsrv.updateRemoteSubscription(client.acc, sub, -1)\n\t\t\t\t}\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check here if we have a header with our message. If this client can not\n\t// support we need to strip the headers from the payload.\n\t// The actual header would have been processed correctly for us, so just\n\t// need to update payload.\n\thdrSize := c.pa.hdr\n\tif c.pa.hdr > 0 && !sub.client.headers {\n\t\tmsg = msg[c.pa.hdr:]\n\t}\n\n\t// Update statistics\n\n\t// The msg includes the CR_LF, so pull back out for accounting.\n\tmsgSize := int64(len(msg))\n\t// MQTT producers send messages without CR_LF, so don't remove it for them.\n\tif !prodIsMQTT {\n\t\tmsgSize -= int64(LEN_CR_LF)\n\t}\n\n\t// We do not update the outbound stats if we are doing trace only since\n\t// this message will not be sent out.\n\t// Also do not update on internal callbacks.\n\tif !traceOnly && sub.icb == nil {\n\t\t// No atomic needed since accessed under client lock.\n\t\t// Monitor is reading those also under client's lock.\n\t\tclient.outMsgs++\n\t\tclient.outBytes += msgSize\n\t}\n\n\t// Check for internal subscriptions.\n\tif sub.icb != nil && !c.noIcb {\n\t\tif gwrply {\n\t\t\t// We will store in the account, not the client since it will likely\n\t\t\t// be a different client that will send the reply.\n\t\t\tsrv.trackGWReply(nil, client.acc, reply, c.pa.reply)\n\t\t}\n\t\tclient.mu.Unlock()\n\n\t\t// For service imports, track if we delivered.\n\t\tdidDeliver := true\n\n\t\t// Internal account clients are for service imports and need the '\\r\\n'.\n\t\tstart := time.Now()\n\t\tif client.kind == ACCOUNT {\n\t\t\tsub.icb(sub, c, acc, string(subject), string(reply), msg)\n\t\t\t// If we are a service import check to make sure we delivered the message somewhere.\n\t\t\tif sub.si {\n\t\t\t\tdidDeliver = c.pa.delivered\n\t\t\t}\n\t\t} else {\n\t\t\tsub.icb(sub, c, acc, string(subject), string(reply), msg[:msgSize])\n\t\t}\n\t\tif dur := time.Since(start); dur >= readLoopReportThreshold {\n\t\t\tsrv.Warnf(\"Internal subscription on %q took too long: %v\", subject, dur)\n\t\t}\n\n\t\treturn didDeliver\n\t}\n\n\t// If we are a client and we detect that the consumer we are\n\t// sending to is in a stalled state, go ahead and wait here\n\t// with a limit.\n\tif c.kind == CLIENT && client.out.stc != nil {\n\t\tif srv.getOpts().NoFastProducerStall {\n\t\t\tmt.addEgressEvent(client, sub, errMsgTraceFastProdNoStall)\n\t\t\tclient.mu.Unlock()\n\t\t\treturn false\n\t\t}\n\t\tclient.stalledWait(c)\n\t}\n\n\t// Check for closed connection\n\tif client.isClosed() {\n\t\tmt.addEgressEvent(client, sub, errMsgTraceClientClosed)\n\t\tclient.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// We have passed cases where we could possibly fail to deliver.\n\t// Do not call for service-import.\n\tif mt != nil && sub.icb == nil {\n\t\tmt.addEgressEvent(client, sub, mtErr)\n\t}\n\n\t// Do a fast check here to see if we should be tracking this from a latency\n\t// perspective. This will be for a request being received for an exported service.\n\t// This needs to be from a non-client (otherwise tracking happens at requestor).\n\t//\n\t// Also this check captures if the original reply (c.pa.reply) is a GW routed\n\t// reply (since it is known to be > minReplyLen). If that is the case, we need to\n\t// track the binding between the routed reply and the reply set in the message\n\t// header (which is c.pa.reply without the GNR routing prefix).\n\tif client.kind == CLIENT && len(c.pa.reply) > minReplyLen {\n\t\tif gwrply {\n\t\t\t// Note that we keep track of the GW routed reply in the destination\n\t\t\t// connection (`client`). The routed reply subject is in `c.pa.reply`,\n\t\t\t// should that change, we would have to pass the GW routed reply as\n\t\t\t// a parameter of deliverMsg().\n\t\t\tsrv.trackGWReply(client, nil, reply, c.pa.reply)\n\t\t}\n\n\t\t// If we do not have a registered RTT queue that up now.\n\t\tif client.rtt == 0 {\n\t\t\tclient.sendRTTPingLocked()\n\t\t}\n\t\t// FIXME(dlc) - We may need to optimize this.\n\t\t// We will have tagged this with a suffix ('.T') if we are tracking. This is\n\t\t// needed from sampling. Not all will be tracked.\n\t\tif c.kind != CLIENT && isTrackedReply(c.pa.reply) {\n\t\t\tclient.trackRemoteReply(string(subject), string(c.pa.reply))\n\t\t}\n\t}\n\n\t// Queue to outbound buffer\n\tclient.queueOutbound(mh)\n\tclient.queueOutbound(msg)\n\tif prodIsMQTT {\n\t\t// Need to add CR_LF since MQTT producers don't send CR_LF\n\t\tclient.queueOutbound([]byte(CR_LF))\n\t}\n\n\t// If we are tracking dynamic publish permissions that track reply subjects,\n\t// do that accounting here. We only look at client.replies which will be non-nil.\n\t// Only reply subject permissions if the client is not already allowed to publish to the reply subject.\n\tif client.replies != nil && len(reply) > 0 && !client.pubAllowedFullCheck(string(reply), true, true) {\n\t\tclient.replies[string(reply)] = &resp{time.Now(), 0}\n\t\tclient.repliesSincePrune++\n\t\tif client.repliesSincePrune > replyPermLimit || time.Since(client.lastReplyPrune) > replyPruneTime {\n\t\t\tclient.pruneReplyPerms()\n\t\t}\n\t}\n\n\t// Check outbound threshold and queue IO flush if needed.\n\t// This is specifically looking at situations where we are getting behind and may want\n\t// to intervene before this producer goes back to top of readloop. We are in the producer's\n\t// readloop go routine at this point.\n\t// FIXME(dlc) - We may call this alot, maybe suppress after first call?\n\tif len(client.out.nb) != 0 {\n\t\tclient.flushSignal()\n\t}\n\n\t// Add the data size we are responsible for here. This will be processed when we\n\t// return to the top of the readLoop.\n\tc.addToPCD(client)\n\n\tif client.trace {\n\t\tclient.traceOutOp(bytesToString(mh[:len(mh)-LEN_CR_LF]), nil)\n\t\tclient.traceMsgDelivery(msg, hdrSize)\n\t}\n\n\tclient.mu.Unlock()\n\n\treturn true\n}\n\n// Add the given sub's client to the list of clients that need flushing.\n// This must be invoked from `c`'s readLoop. No lock for c is required,\n// however, `client` lock must be held on entry. This holds true even\n// if `client` is same than `c`.\nfunc (c *client) addToPCD(client *client) {\n\tif _, ok := c.pcd[client]; !ok {\n\t\tclient.out.fsp++\n\t\tc.pcd[client] = needFlush\n\t}\n}\n\n// This will track a remote reply for an exported service that has requested\n// latency tracking.\n// Lock assumed to be held.\nfunc (c *client) trackRemoteReply(subject, reply string) {\n\ta := c.acc\n\tif a == nil {\n\t\treturn\n\t}\n\n\tvar lrt time.Duration\n\tvar respThresh time.Duration\n\n\ta.mu.RLock()\n\tse := a.getServiceExport(subject)\n\tif se != nil {\n\t\tlrt = a.lowestServiceExportResponseTime()\n\t\trespThresh = se.respThresh\n\t}\n\ta.mu.RUnlock()\n\n\tif se == nil {\n\t\treturn\n\t}\n\n\tif c.rrTracking == nil {\n\t\tc.rrTracking = &rrTracking{\n\t\t\trmap: make(map[string]*remoteLatency),\n\t\t\tptmr: time.AfterFunc(lrt, c.pruneRemoteTracking),\n\t\t\tlrt: lrt,\n\t\t}\n\t}\n\trl := remoteLatency{\n\t\tAccount: a.Name,\n\t\tReqId: reply,\n\t\trespThresh: respThresh,\n\t}\n\trl.M2.RequestStart = time.Now().UTC()\n\tc.rrTracking.rmap[reply] = &rl\n}\n\n// pruneRemoteTracking will prune any remote tracking objects\n// that are too old. These are orphaned when a service is not\n// sending reponses etc.\n// Lock should be held upon entry.\nfunc (c *client) pruneRemoteTracking() {\n\tc.mu.Lock()\n\tif c.rrTracking == nil {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tnow := time.Now()\n\tfor subject, rl := range c.rrTracking.rmap {\n\t\tif now.After(rl.M2.RequestStart.Add(rl.respThresh)) {\n\t\t\tdelete(c.rrTracking.rmap, subject)\n\t\t}\n\t}\n\tif len(c.rrTracking.rmap) > 0 {\n\t\tt := c.rrTracking.ptmr\n\t\tt.Stop()\n\t\tt.Reset(c.rrTracking.lrt)\n\t} else {\n\t\tc.rrTracking.ptmr.Stop()\n\t\tc.rrTracking = nil\n\t}\n\tc.mu.Unlock()\n}\n\n// pruneReplyPerms will remove any stale or expired entries\n// in our reply cache. We make sure to not check too often.\nfunc (c *client) pruneReplyPerms() {\n\t// Make sure we do not check too often.\n\tif c.perms.resp == nil {\n\t\treturn\n\t}\n\n\tmm := c.perms.resp.MaxMsgs\n\tttl := c.perms.resp.Expires\n\tnow := time.Now()\n\n\tfor k, resp := range c.replies {\n\t\tif mm > 0 && resp.n >= mm {\n\t\t\tdelete(c.replies, k)\n\t\t} else if ttl > 0 && now.Sub(resp.t) > ttl {\n\t\t\tdelete(c.replies, k)\n\t\t}\n\t}\n\n\tc.repliesSincePrune = 0\n\tc.lastReplyPrune = now\n}\n\n// pruneDenyCache will prune the deny cache via randomly\n// deleting items. Doing so pruneSize items at a time.\n// Lock must be held for this one since it is shared under\n// deliverMsg.\nfunc (c *client) pruneDenyCache() {\n\tr := 0\n\tfor subject := range c.mperms.dcache {\n\t\tdelete(c.mperms.dcache, subject)\n\t\tif r++; r > pruneSize {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// prunePubPermsCache will prune the cache via randomly\n// deleting items. Doing so pruneSize items at a time.\nfunc (c *client) prunePubPermsCache() {\n\t// With parallel additions to the cache, it is possible that this function\n\t// would not be able to reduce the cache to its max size in one go. We\n\t// will try a few times but will release/reacquire the \"lock\" at each\n\t// attempt to give a chance to another go routine to take over and not\n\t// have this go routine do too many attempts.\n\tfor i := 0; i < 5; i++ {\n\t\t// There is a case where we can invoke this from multiple go routines,\n\t\t// (in deliverMsg() if sub.client is a LEAF), so we make sure to prune\n\t\t// from only one go routine at a time.\n\t\tif !atomic.CompareAndSwapInt32(&c.perms.prun, 0, 1) {\n\t\t\treturn\n\t\t}\n\t\tconst maxPruneAtOnce = 1000\n\t\tr := 0\n\t\tc.perms.pcache.Range(func(k, _ any) bool {\n\t\t\tc.perms.pcache.Delete(k)\n\t\t\tif r++; (r > pruneSize && atomic.LoadInt32(&c.perms.pcsz) < int32(maxPermCacheSize)) ||\n\t\t\t\t(r > maxPruneAtOnce) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tn := atomic.AddInt32(&c.perms.pcsz, -int32(r))\n\t\tatomic.StoreInt32(&c.perms.prun, 0)\n\t\tif n <= int32(maxPermCacheSize) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// pubAllowed checks on publish permissioning.\n// Lock should not be held.\nfunc (c *client) pubAllowed(subject string) bool {\n\treturn c.pubAllowedFullCheck(subject, true, false)\n}\n\n// pubAllowedFullCheck checks on all publish permissioning depending\n// on the flag for dynamic reply permissions.\nfunc (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bool {\n\tif c.perms == nil || (c.perms.pub.allow == nil && c.perms.pub.deny == nil) {\n\t\treturn true\n\t}\n\t// Check if published subject is allowed if we have permissions in place.\n\tv, ok := c.perms.pcache.Load(subject)\n\tif ok {\n\t\treturn v.(bool)\n\t}\n\tallowed, checkAllow := true, true\n\t// For CLIENT connections that are MQTT, or other types of connections, we will\n\t// implicitly allow anything that starts with the \"$MQTT.\" prefix. However,\n\t// we don't just return here, we skip the check for \"allow\" but will check \"deny\".\n\tif (c.isMqtt() || c.kind != CLIENT) && strings.HasPrefix(subject, mqttPrefix) {\n\t\tcheckAllow = false\n\t}\n\t// Cache miss, check allow then deny as needed.\n\tif checkAllow && c.perms.pub.allow != nil {\n\t\tnp, _ := c.perms.pub.allow.NumInterest(subject)\n\t\tallowed = np != 0\n\t}\n\t// If we have a deny list and are currently allowed, check that as well.\n\tif allowed && c.perms.pub.deny != nil {\n\t\tnp, _ := c.perms.pub.deny.NumInterest(subject)\n\t\tallowed = np == 0\n\t}\n\n\t// If we are tracking reply subjects\n\t// dynamically, check to see if we are allowed here but avoid pcache.\n\t// We need to acquire the lock though.\n\tif !allowed && fullCheck && c.perms.resp != nil {\n\t\tif !hasLock {\n\t\t\tc.mu.Lock()\n\t\t}\n\t\tif resp := c.replies[subject]; resp != nil {\n\t\t\tresp.n++\n\t\t\t// Check if we have sent too many responses.\n\t\t\tif c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs {\n\t\t\t\tdelete(c.replies, subject)\n\t\t\t} else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires {\n\t\t\t\tdelete(c.replies, subject)\n\t\t\t} else {\n\t\t\t\tallowed = true\n\t\t\t}\n\t\t}\n\t\tif !hasLock {\n\t\t\tc.mu.Unlock()\n\t\t}\n\t} else {\n\t\t// Update our cache here.\n\t\tc.perms.pcache.Store(subject, allowed)\n\t\tif n := atomic.AddInt32(&c.perms.pcsz, 1); n > maxPermCacheSize {\n\t\t\tc.prunePubPermsCache()\n\t\t}\n\t}\n\treturn allowed\n}\n\n// Test whether a reply subject is a service import reply.\nfunc isServiceReply(reply []byte) bool {\n\t// This function is inlined and checking this way is actually faster\n\t// than byte-by-byte comparison.\n\treturn len(reply) > 3 && bytesToString(reply[:4]) == replyPrefix\n}\n\n// Test whether a reply subject is a service import or a gateway routed reply.\nfunc isReservedReply(reply []byte) bool {\n\tif isServiceReply(reply) {\n\t\treturn true\n\t}\n\trLen := len(reply)\n\t// Faster to check with string([:]) than byte-by-byte\n\tif rLen > jsAckPreLen && bytesToString(reply[:jsAckPreLen]) == jsAckPre {\n\t\treturn true\n\t} else if rLen > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// This will decide to call the client code or router code.\nfunc (c *client) processInboundMsg(msg []byte) {\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tc.processInboundClientMsg(msg)\n\tcase ROUTER:\n\t\tc.processInboundRoutedMsg(msg)\n\tcase GATEWAY:\n\t\tc.processInboundGatewayMsg(msg)\n\tcase LEAF:\n\t\tc.processInboundLeafMsg(msg)\n\t}\n}\n\n// selectMappedSubject will choose the mapped subject based on the client's inbound subject.\nfunc (c *client) selectMappedSubject() bool {\n\tnsubj, changed := c.acc.selectMappedSubject(bytesToString(c.pa.subject))\n\tif changed {\n\t\tc.pa.mapped = c.pa.subject\n\t\tc.pa.subject = []byte(nsubj)\n\t}\n\treturn changed\n}\n\n// clientNRGPrefix is used in processInboundClientMsg to detect if publishes\n// are being made from normal clients to NRG subjects.\nvar clientNRGPrefix = []byte(\"$NRG.\")\n\n// processInboundClientMsg is called to process an inbound msg from a client.\n// Return if the message was delivered, and if the message was not delivered\n// due to a permission issue.\nfunc (c *client) processInboundClientMsg(msg []byte) (bool, bool) {\n\t// Update statistics\n\t// The msg includes the CR_LF, so pull back out for accounting.\n\tc.in.msgs++\n\tc.in.bytes += int32(len(msg) - LEN_CR_LF)\n\n\t// Check that client (could be here with SYSTEM) is not publishing on reserved \"$GNR\" prefix.\n\tif c.kind == CLIENT && hasGWRoutedReplyPrefix(c.pa.subject) {\n\t\tc.pubPermissionViolation(c.pa.subject)\n\t\treturn false, true\n\t}\n\n\t// Mostly under testing scenarios.\n\tc.mu.Lock()\n\tif c.srv == nil || c.acc == nil {\n\t\tc.mu.Unlock()\n\t\treturn false, false\n\t}\n\tacc := c.acc\n\tgenidAddr := &acc.sl.genid\n\n\t// Check pub permissions\n\tif c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) && !c.pubAllowedFullCheck(string(c.pa.subject), true, true) {\n\t\tc.mu.Unlock()\n\t\tc.pubPermissionViolation(c.pa.subject)\n\t\treturn false, true\n\t}\n\tc.mu.Unlock()\n\n\t// Check if the client is trying to publish to reserved NRG subjects.\n\t// Doesn't apply to NRGs themselves as they use SYSTEM-kind clients instead.\n\tif c.kind == CLIENT && bytes.HasPrefix(c.pa.subject, clientNRGPrefix) && acc != c.srv.SystemAccount() {\n\t\tc.pubPermissionViolation(c.pa.subject)\n\t\treturn false, true\n\t}\n\n\t// Now check for reserved replies. These are used for service imports.\n\tif c.kind == CLIENT && len(c.pa.reply) > 0 && isReservedReply(c.pa.reply) {\n\t\tc.replySubjectViolation(c.pa.reply)\n\t\treturn false, true\n\t}\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\n\t// If MQTT client, check for retain flag now that we have passed permissions check\n\tif c.isMqtt() {\n\t\tc.mqttHandlePubRetain()\n\t}\n\n\t// Doing this inline as opposed to create a function (which otherwise has a measured\n\t// performance impact reported in our bench)\n\tvar isGWRouted bool\n\tif c.kind != CLIENT {\n\t\tif atomic.LoadInt32(&acc.gwReplyMapping.check) > 0 {\n\t\t\tacc.mu.RLock()\n\t\t\tc.pa.subject, isGWRouted = acc.gwReplyMapping.get(c.pa.subject)\n\t\t\tacc.mu.RUnlock()\n\t\t}\n\t} else if atomic.LoadInt32(&c.gwReplyMapping.check) > 0 {\n\t\tc.mu.Lock()\n\t\tc.pa.subject, isGWRouted = c.gwReplyMapping.get(c.pa.subject)\n\t\tc.mu.Unlock()\n\t}\n\n\t// If we have an exported service and we are doing remote tracking, check this subject\n\t// to see if we need to report the latency.\n\tif c.rrTracking != nil {\n\t\tc.mu.Lock()\n\t\trl := c.rrTracking.rmap[string(c.pa.subject)]\n\t\tif rl != nil {\n\t\t\tdelete(c.rrTracking.rmap, bytesToString(c.pa.subject))\n\t\t}\n\t\tc.mu.Unlock()\n\n\t\tif rl != nil {\n\t\t\tsl := &rl.M2\n\t\t\t// Fill this in and send it off to the other side.\n\t\t\tsl.Status = 200\n\t\t\tsl.Responder = c.getClientInfo(true)\n\t\t\tsl.ServiceLatency = time.Since(sl.RequestStart) - sl.Responder.RTT\n\t\t\tsl.TotalLatency = sl.ServiceLatency + sl.Responder.RTT\n\t\t\tsanitizeLatencyMetric(sl)\n\t\t\tlsub := remoteLatencySubjectForResponse(c.pa.subject)\n\t\t\tc.srv.sendInternalAccountMsg(nil, lsub, rl) // Send to SYS account\n\t\t}\n\t}\n\n\t// If the subject was converted to the gateway routed subject, then handle it now\n\t// and be done with the rest of this function.\n\tif isGWRouted {\n\t\tc.handleGWReplyMap(msg)\n\t\treturn true, false\n\t}\n\n\t// Match the subscriptions. We will use our own L1 map if\n\t// it's still valid, avoiding contention on the shared sublist.\n\tvar r *SublistResult\n\tvar ok bool\n\n\tgenid := atomic.LoadUint64(genidAddr)\n\tif genid == c.in.genid && c.in.results != nil {\n\t\tr, ok = c.in.results[string(c.pa.subject)]\n\t} else {\n\t\t// Reset our L1 completely.\n\t\tc.in.results = make(map[string]*SublistResult)\n\t\tc.in.genid = genid\n\t}\n\n\t// Go back to the sublist data structure.\n\tif !ok {\n\t\t// Match may use the subject here to populate a cache, so can not use bytesToString here.\n\t\tr = acc.sl.Match(string(c.pa.subject))\n\t\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\t\t// Prune the results cache. Keeps us from unbounded growth. Random delete.\n\t\t\tif len(c.in.results) >= maxResultCacheSize {\n\t\t\t\tn := 0\n\t\t\t\tfor subject := range c.in.results {\n\t\t\t\t\tdelete(c.in.results, subject)\n\t\t\t\t\tif n++; n > pruneSize {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Then add the new cache entry.\n\t\t\tc.in.results[string(c.pa.subject)] = r\n\t\t}\n\t}\n\n\t// Indication if we attempted to deliver the message to anyone.\n\tvar didDeliver bool\n\tvar qnames [][]byte\n\n\t// Check for no interest, short circuit if so.\n\t// This is the fanout scale.\n\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\tflag := pmrNoFlag\n\t\t// If there are matching queue subs and we are in gateway mode,\n\t\t// we need to keep track of the queue names the messages are\n\t\t// delivered to. When sending to the GWs, the RMSG will include\n\t\t// those names so that the remote clusters do not deliver messages\n\t\t// to their queue subs of the same names.\n\t\tif len(r.qsubs) > 0 && c.srv.gateway.enabled &&\n\t\t\tatomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 {\n\t\t\tflag |= pmrCollectQueueNames\n\t\t}\n\t\tdidDeliver, qnames = c.processMsgResults(acc, r, msg, c.pa.deliver, c.pa.subject, c.pa.reply, flag)\n\t}\n\n\t// Now deal with gateways\n\tif c.srv.gateway.enabled {\n\t\treply := c.pa.reply\n\t\tif len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {\n\t\t\treply = append(reply, '@')\n\t\t\treply = append(reply, c.pa.deliver...)\n\t\t}\n\t\tdidDeliver = c.sendMsgToGateways(acc, msg, c.pa.subject, reply, qnames, false) || didDeliver\n\t}\n\n\t// Check to see if we did not deliver to anyone and the client has a reply subject set\n\t// and wants notification of no_responders.\n\tif !didDeliver && len(c.pa.reply) > 0 {\n\t\tc.mu.Lock()\n\t\tif c.opts.NoResponders {\n\t\t\tif sub := c.subForReply(c.pa.reply); sub != nil {\n\t\t\t\thdrLen := 32 /* header without the subject */ + len(c.pa.subject)\n\t\t\t\tproto := fmt.Sprintf(\"HMSG %s %s %d %d\\r\\nNATS/1.0 503\\r\\nNats-Subject: %s\\r\\n\\r\\n\\r\\n\", c.pa.reply, sub.sid, hdrLen, hdrLen, c.pa.subject)\n\t\t\t\tc.queueOutbound([]byte(proto))\n\t\t\t\tc.addToPCD(c)\n\t\t\t}\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n\n\treturn didDeliver, false\n}\n\n// Return the subscription for this reply subject. Only look at normal subs for this client.\nfunc (c *client) subForReply(reply []byte) *subscription {\n\tr := c.acc.sl.Match(string(reply))\n\tfor _, sub := range r.psubs {\n\t\tif sub.client == c {\n\t\t\treturn sub\n\t\t}\n\t}\n\treturn nil\n}\n\n// This is invoked knowing that c.pa.subject has been set to the gateway routed subject.\n// This function will send the message to possibly LEAFs and directly back to the origin\n// gateway.\nfunc (c *client) handleGWReplyMap(msg []byte) bool {\n\t// Check for leaf nodes\n\tif c.srv.gwLeafSubs.Count() > 0 {\n\t\tif r := c.srv.gwLeafSubs.MatchBytes(c.pa.subject); len(r.psubs) > 0 {\n\t\t\tc.processMsgResults(c.acc, r, msg, c.pa.deliver, c.pa.subject, c.pa.reply, pmrNoFlag)\n\t\t}\n\t}\n\tif c.srv.gateway.enabled {\n\t\treply := c.pa.reply\n\t\tif len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {\n\t\t\treply = append(reply, '@')\n\t\t\treply = append(reply, c.pa.deliver...)\n\t\t}\n\t\tc.sendMsgToGateways(c.acc, msg, c.pa.subject, reply, nil, false)\n\t}\n\treturn true\n}\n\n// Used to setup the response map for a service import request that has a reply subject.\nfunc (c *client) setupResponseServiceImport(acc *Account, si *serviceImport, tracking bool, header http.Header) *serviceImport {\n\trsi := si.acc.addRespServiceImport(acc, string(c.pa.reply), si, tracking, header)\n\tif si.latency != nil {\n\t\tif c.rtt == 0 {\n\t\t\t// We have a service import that we are tracking but have not established RTT.\n\t\t\tc.sendRTTPing()\n\t\t}\n\t\tsi.acc.mu.Lock()\n\t\trsi.rc = c\n\t\tsi.acc.mu.Unlock()\n\t}\n\treturn rsi\n}\n\n// Will remove a header if present.\nfunc removeHeaderIfPresent(hdr []byte, key string) []byte {\n\tstart := getHeaderKeyIndex(key, hdr)\n\t// key can't be first and we want to check that it is preceded by a '\\n'\n\tif start < 1 || hdr[start-1] != '\\n' {\n\t\treturn hdr\n\t}\n\tindex := start + len(key)\n\tif index >= len(hdr) || hdr[index] != ':' {\n\t\treturn hdr\n\t}\n\tend := bytes.Index(hdr[start:], []byte(_CRLF_))\n\tif end < 0 {\n\t\treturn hdr\n\t}\n\thdr = append(hdr[:start], hdr[start+end+len(_CRLF_):]...)\n\tif len(hdr) <= len(emptyHdrLine) {\n\t\treturn nil\n\t}\n\treturn hdr\n}\n\nfunc removeHeaderIfPrefixPresent(hdr []byte, prefix string) []byte {\n\tvar index int\n\tfor {\n\t\tif index >= len(hdr) {\n\t\t\treturn hdr\n\t\t}\n\n\t\tstart := bytes.Index(hdr[index:], []byte(prefix))\n\t\tif start < 0 {\n\t\t\treturn hdr\n\t\t}\n\t\tindex += start\n\t\tif index < 1 || hdr[index-1] != '\\n' {\n\t\t\treturn hdr\n\t\t}\n\n\t\tend := bytes.Index(hdr[index+len(prefix):], []byte(_CRLF_))\n\t\tif end < 0 {\n\t\t\treturn hdr\n\t\t}\n\n\t\thdr = append(hdr[:index], hdr[index+end+len(prefix)+len(_CRLF_):]...)\n\t\tif len(hdr) <= len(emptyHdrLine) {\n\t\t\treturn nil\n\t\t}\n\t}\n}\n\n// Generate a new header based on optional original header and key value.\n// More used in JetStream layers.\nfunc genHeader(hdr []byte, key, value string) []byte {\n\tvar bb bytes.Buffer\n\tif len(hdr) > LEN_CR_LF {\n\t\tbb.Write(hdr[:len(hdr)-LEN_CR_LF])\n\t} else {\n\t\tbb.WriteString(hdrLine)\n\t}\n\thttp.Header{key: []string{value}}.Write(&bb)\n\tbb.WriteString(CR_LF)\n\treturn bb.Bytes()\n}\n\n// This will set a header for the message.\n// Lock does not need to be held but this should only be called\n// from the inbound go routine. We will update the pubArgs.\n// This will replace any previously set header and not add to it per normal spec.\nfunc (c *client) setHeader(key, value string, msg []byte) []byte {\n\tvar bb bytes.Buffer\n\tvar omi int\n\t// Write original header if present.\n\tif c.pa.hdr > LEN_CR_LF {\n\t\tomi = c.pa.hdr\n\t\thdr := removeHeaderIfPresent(msg[:c.pa.hdr-LEN_CR_LF], key)\n\t\tif len(hdr) == 0 {\n\t\t\tbb.WriteString(hdrLine)\n\t\t} else {\n\t\t\tbb.Write(hdr)\n\t\t}\n\t} else {\n\t\tbb.WriteString(hdrLine)\n\t}\n\thttp.Header{key: []string{value}}.Write(&bb)\n\tbb.WriteString(CR_LF)\n\tnhdr := bb.Len()\n\t// Put the original message back.\n\t// FIXME(dlc) - This is inefficient.\n\tbb.Write(msg[omi:])\n\tnsize := bb.Len() - LEN_CR_LF\n\t// MQTT producers don't have CRLF, so add it back.\n\tif c.isMqtt() {\n\t\tnsize += LEN_CR_LF\n\t}\n\t// Update pubArgs\n\t// If others will use this later we need to save and restore original.\n\tc.pa.hdr = nhdr\n\tc.pa.size = nsize\n\tc.pa.hdb = []byte(strconv.Itoa(nhdr))\n\tc.pa.szb = []byte(strconv.Itoa(nsize))\n\treturn bb.Bytes()\n}\n\n// Will return a copy of the value for the header denoted by key or nil if it does not exist.\n// If you know that it is safe to refer to the underlying hdr slice for the period that the\n// return value is used, then sliceHeader() will be faster.\nfunc getHeader(key string, hdr []byte) []byte {\n\tv := sliceHeader(key, hdr)\n\tif v == nil {\n\t\treturn nil\n\t}\n\treturn append(make([]byte, 0, len(v)), v...)\n}\n\n// Will return the sliced value for the header denoted by key or nil if it does not exists.\n// This function ignores errors and tries to achieve speed and no additional allocations.\nfunc sliceHeader(key string, hdr []byte) []byte {\n\tif len(hdr) == 0 {\n\t\treturn nil\n\t}\n\tindex := getHeaderKeyIndex(key, hdr)\n\tif index == -1 {\n\t\treturn nil\n\t}\n\t// Skip over the key and the : separator.\n\tindex += len(key) + 1\n\thdrLen := len(hdr)\n\t// Skip over whitespace before the value.\n\tfor index < hdrLen && hdr[index] == ' ' {\n\t\tindex++\n\t}\n\t// Collect together the rest of the value until we hit a CRLF.\n\tstart := index\n\tfor index < hdrLen {\n\t\tif hdr[index] == '\\r' && index < hdrLen-1 && hdr[index+1] == '\\n' {\n\t\t\tbreak\n\t\t}\n\t\tindex++\n\t}\n\treturn hdr[start:index:index]\n}\n\n// getHeaderKeyIndex returns an index into the header slice for the given key.\n// Returns -1 if not found.\nfunc getHeaderKeyIndex(key string, hdr []byte) int {\n\tif len(hdr) == 0 {\n\t\treturn -1\n\t}\n\tbkey := stringToBytes(key)\n\tkeyLen, hdrLen := len(key), len(hdr)\n\tvar offset int\n\tfor {\n\t\tindex := bytes.Index(hdr[offset:], bkey)\n\t\t// Check that we have enough characters, this will handle the -1 case of the key not\n\t\t// being found and will also handle not having enough characters for trailing CRLF.\n\t\tif index < 2 {\n\t\t\treturn -1\n\t\t}\n\t\tindex += offset\n\t\t// There should be a terminating CRLF.\n\t\tif index >= hdrLen-1 || hdr[index-1] != '\\n' || hdr[index-2] != '\\r' {\n\t\t\toffset = index + keyLen\n\t\t\tcontinue\n\t\t}\n\t\t// The key should be immediately followed by a : separator.\n\t\tif index+keyLen >= hdrLen {\n\t\t\treturn -1\n\t\t}\n\t\tif hdr[index+keyLen] != ':' {\n\t\t\toffset = index + keyLen\n\t\t\tcontinue\n\t\t}\n\t\treturn index\n\t}\n}\n\n// setHeader will replace the value of the first existing key `key`\n// with the given value `val`, or add this new key at the end of\n// the headers.\n//\n// Note: If the key does not exist, or if it exists but the new value\n// would make the resulting byte slice larger than the original one,\n// a new byte slice is returned and the original is left untouched.\n// This is to prevent situations where caller may have a `hdr` and\n// `msg` that are the parts of an underlying buffer. Extending the\n// `hdr` would otherwise overwrite the `msg` part.\n//\n// If the new value is smaller, then the original `hdr` byte slice\n// is modified.\nfunc setHeader(key, val string, hdr []byte) []byte {\n\tstart := getHeaderKeyIndex(key, hdr)\n\tif start >= 0 {\n\t\tvalStart := start + len(key) + 1\n\t\t// Preserve single whitespace if used.\n\t\thdrLen := len(hdr)\n\t\tif valStart < hdrLen && hdr[valStart] == ' ' {\n\t\t\tvalStart++\n\t\t}\n\t\tvalEnd := bytes.Index(hdr[valStart:], []byte(\"\\r\"))\n\t\tif valEnd < 0 {\n\t\t\treturn hdr // malformed headers\n\t\t}\n\t\tvalEnd += valStart\n\t\t// Length of the existing value (before the `\\r`)\n\t\toldValLen := valEnd - valStart\n\t\t// This is how many extra bytes we need for the new value.\n\t\t// If <= 0, it means that we need less and so will reuse the `hdr` buffer.\n\t\tif extra := len(val) - oldValLen; extra > 0 {\n\t\t\t// Check that we don't overflow an \"int\".\n\t\t\tif rem := math.MaxInt - hdrLen; rem < extra {\n\t\t\t\t// We don't grow, and return the existing header.\n\t\t\t\treturn hdr\n\t\t\t}\n\t\t\t// The new size is the old size plus the extra bytes.\n\t\t\tnewHdrSize := hdrLen + extra\n\t\t\tnewHdr := make([]byte, newHdrSize)\n\t\t\t// Copy the parts from `hdr` and `val` into the new buffer.\n\t\t\tn := copy(newHdr, hdr[:valStart])\n\t\t\tn += copy(newHdr[n:], val)\n\t\t\tcopy(newHdr[n:], hdr[valEnd:])\n\t\t\treturn newHdr\n\t\t}\n\t\t// We can write in place since it fits in the existing `hdr` buffer.\n\t\tn := copy(hdr[valStart:], val)\n\t\tn += copy(hdr[valStart+n:], hdr[valEnd:])\n\t\thdr = hdr[:valStart+n]\n\t\treturn hdr\n\t}\n\tif len(hdr) > 0 && bytes.HasSuffix(hdr, []byte(\"\\r\\n\")) {\n\t\thdr = hdr[:len(hdr)-2]\n\t\tval += \"\\r\\n\"\n\t}\n\t// Create the new buffer based on length of existing one and\n\t// length of the new \": \\r\\n\". Protect against \"int\" overflow.\n\tnewSize := uint64(len(hdr)) + uint64(len(key)) + 1 + 1 + uint64(len(val)) + 2\n\tif newSize > uint64(math.MaxInt) {\n\t\t// We don't grow, and return the existing header.\n\t\treturn hdr\n\t}\n\tnewHdr := make([]byte, 0, int(newSize))\n\tnewHdr = append(newHdr, hdr...)\n\treturn fmt.Appendf(newHdr, \"%s: %s\\r\\n\", key, val)\n}\n\n// For bytes.HasPrefix below.\nvar (\n\tjsRequestNextPreB = []byte(jsRequestNextPre)\n\tjsDirectGetPreB = []byte(jsDirectGetPre)\n)\n\n// processServiceImport is an internal callback when a subscription matches an imported service\n// from another account. This includes response mappings as well.\nfunc (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byte) bool {\n\t// If we are a GW and this is not a direct serviceImport ignore.\n\tisResponse := si.isRespServiceImport()\n\tif (c.kind == GATEWAY || c.kind == ROUTER) && !isResponse {\n\t\treturn false\n\t}\n\t// Detect cycles and ignore (return) when we detect one.\n\tif len(c.pa.psi) > 0 {\n\t\tfor i := len(c.pa.psi) - 1; i >= 0; i-- {\n\t\t\tif psi := c.pa.psi[i]; psi.se == si.se {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\tacc.mu.RLock()\n\tvar checkJS bool\n\tshouldReturn := si.invalid || acc.sl == nil\n\tif !shouldReturn && !isResponse && si.to == jsAllAPI {\n\t\tif bytes.HasPrefix(c.pa.subject, jsDirectGetPreB) || bytes.HasPrefix(c.pa.subject, jsRequestNextPreB) {\n\t\t\tcheckJS = true\n\t\t}\n\t}\n\tsiAcc := si.acc\n\tallowTrace := si.atrc\n\tacc.mu.RUnlock()\n\n\t// We have a special case where JetStream pulls in all service imports through one export.\n\t// However the GetNext for consumers and DirectGet for streams are a no-op and causes buildups of service imports,\n\t// response service imports and rrMap entries which all will need to simply expire.\n\t// TODO(dlc) - Come up with something better.\n\tif shouldReturn || (checkJS && si.se != nil && si.se.acc == c.srv.SystemAccount()) {\n\t\treturn false\n\t}\n\n\tmt, traceOnly := c.isMsgTraceEnabled()\n\n\tvar nrr []byte\n\tvar rsi *serviceImport\n\n\t// Check if there is a reply present and set up a response.\n\ttracking, headers := shouldSample(si.latency, c)\n\tif len(c.pa.reply) > 0 {\n\t\t// Special case for now, need to formalize.\n\t\t// TODO(dlc) - Formalize as a service import option for reply rewrite.\n\t\t// For now we can't do $JS.ACK since that breaks pull consumers across accounts.\n\t\tif !bytes.HasPrefix(c.pa.reply, []byte(jsAckPre)) {\n\t\t\tif rsi = c.setupResponseServiceImport(acc, si, tracking, headers); rsi != nil {\n\t\t\t\tnrr = []byte(rsi.from)\n\t\t\t}\n\t\t} else {\n\t\t\t// This only happens when we do a pull subscriber that trampolines through another account.\n\t\t\t// Normally this code is not called.\n\t\t\tnrr = c.pa.reply\n\t\t}\n\t} else if !isResponse && si.latency != nil && tracking {\n\t\t// Check to see if this was a bad request with no reply and we were supposed to be tracking.\n\t\tsiAcc.sendBadRequestTrackingLatency(si, c, headers)\n\t}\n\n\t// Send tracking info here if we are tracking this response.\n\t// This is always a response.\n\tvar didSendTL bool\n\tif si.tracking && !si.didDeliver {\n\t\t// Stamp that we attempted delivery.\n\t\tsi.didDeliver = true\n\t\tdidSendTL = acc.sendTrackingLatency(si, c)\n\t}\n\n\t// Pick correct \"to\" subject. If we matched on a wildcard use the literal publish subject.\n\tto, subject := si.to, string(c.pa.subject)\n\n\tif si.tr != nil {\n\t\t// FIXME(dlc) - This could be slow, may want to look at adding cache to bare transforms?\n\t\tto = si.tr.TransformSubject(subject)\n\t} else if si.usePub {\n\t\tto = subject\n\t}\n\n\t// Copy our pubArg since this gets modified as we process the service import itself.\n\tpacopy := c.pa\n\n\t// Now check to see if this account has mappings that could affect the service import.\n\t// Can't use non-locked trick like in processInboundClientMsg, so just call into selectMappedSubject\n\t// so we only lock once.\n\tnsubj, changed := siAcc.selectMappedSubject(to)\n\tif changed {\n\t\tc.pa.mapped = []byte(to)\n\t\tto = nsubj\n\t}\n\n\t// Set previous service import to detect chaining.\n\tlpsi := len(c.pa.psi)\n\thadPrevSi, share := lpsi > 0, si.share\n\tif hadPrevSi {\n\t\tshare = c.pa.psi[lpsi-1].share\n\t}\n\tc.pa.psi = append(c.pa.psi, si)\n\n\t// Place our client info for the request in the original message.\n\t// This will survive going across routes, etc.\n\tif !isResponse {\n\t\tisSysImport := siAcc == c.srv.SystemAccount()\n\t\tvar ci *ClientInfo\n\t\tif hadPrevSi && c.pa.hdr >= 0 {\n\t\t\tvar cis ClientInfo\n\t\t\tif err := json.Unmarshal(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr]), &cis); err == nil {\n\t\t\t\tci = &cis\n\t\t\t\tci.Service = acc.Name\n\t\t\t\t// Check if we are moving into a share details account from a non-shared\n\t\t\t\t// and add in server and cluster details.\n\t\t\t\tif !share && (si.share || isSysImport) {\n\t\t\t\t\tc.addServerAndClusterInfo(ci)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if c.kind != LEAF || c.pa.hdr < 0 || len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) == 0 {\n\t\t\tci = c.getClientInfo(share)\n\t\t\t// Fast batch requires knowledge of the original reply subject.\n\t\t\tif bytes.HasSuffix(c.pa.reply, []byte(FastBatchSuffix)) {\n\t\t\t\tci.Reply = bytesToString(c.pa.reply)\n\t\t\t}\n\t\t\t// If we did not share but the imports destination is the system account add in the server and cluster info.\n\t\t\tif !share && isSysImport {\n\t\t\t\tc.addServerAndClusterInfo(ci)\n\t\t\t}\n\t\t} else if c.kind == LEAF && (si.share || isSysImport) {\n\t\t\t// We have a leaf header here for ci, augment as above.\n\t\t\tci = c.getClientInfo(si.share)\n\t\t\tif !si.share && isSysImport {\n\t\t\t\tc.addServerAndClusterInfo(ci)\n\t\t\t}\n\t\t}\n\t\t// Set clientInfo if present.\n\t\tif ci != nil {\n\t\t\tif b, _ := json.Marshal(ci); b != nil {\n\t\t\t\tmsg = c.setHeader(ClientInfoHdr, bytesToString(b), msg)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Set our optional subject(to) and reply.\n\tif !isResponse && to != subject {\n\t\tc.pa.subject = []byte(to)\n\t}\n\tc.pa.reply = nrr\n\n\tif changed && c.isMqtt() && c.pa.hdr > 0 {\n\t\tc.srv.mqttStoreQoSMsgForAccountOnNewSubject(c.pa.hdr, msg, siAcc.GetName(), to)\n\t}\n\n\t// FIXME(dlc) - Do L1 cache trick like normal client?\n\trr := siAcc.sl.Match(to)\n\n\t// If we are a route or gateway or leafnode and this message is flipped to a queue subscriber we\n\t// need to handle that since the processMsgResults will want a queue filter.\n\tflags := pmrMsgImportedFromService\n\tif c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF {\n\t\tflags |= pmrIgnoreEmptyQueueFilter\n\t}\n\n\t// We will be calling back into processMsgResults since we are now being called as a normal sub.\n\t// We need to take care of the c.in.rts, so save off what is there and use a local version. We\n\t// will put back what was there after.\n\n\torts := c.in.rts\n\n\tvar lrts [routeTargetInit]routeTarget\n\tc.in.rts = lrts[:0]\n\n\tvar skipProcessing bool\n\t// If message tracing enabled, add the service import trace.\n\tif mt != nil {\n\t\tmt.addServiceImportEvent(siAcc.GetName(), string(pacopy.subject), to)\n\t\t// If we are not allowing tracing and doing trace only, we stop at this level.\n\t\tif !allowTrace {\n\t\t\tif traceOnly {\n\t\t\t\tskipProcessing = true\n\t\t\t} else {\n\t\t\t\t// We are going to do normal processing, and possibly chainning\n\t\t\t\t// with other server imports, but the rest won't be traced.\n\t\t\t\t// We do so by setting the c.pa.trace to nil (it will be restored\n\t\t\t\t// with c.pa = pacopy).\n\t\t\t\tc.pa.trace = nil\n\t\t\t\t// We also need to disable the message trace headers so that\n\t\t\t\t// if the message is routed, it does not initialize tracing in the\n\t\t\t\t// remote.\n\t\t\t\tmsg = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, msg)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar didDeliver bool\n\n\tif !skipProcessing {\n\t\t// If this is not a gateway connection but gateway is enabled,\n\t\t// try to send this converted message to all gateways.\n\t\tif c.srv.gateway.enabled {\n\t\t\tflags |= pmrCollectQueueNames\n\t\t\tvar queues [][]byte\n\t\t\tdidDeliver, queues = c.processMsgResults(siAcc, rr, msg, c.pa.deliver, []byte(to), nrr, flags)\n\t\t\tdidDeliver = c.sendMsgToGateways(siAcc, msg, []byte(to), nrr, queues, false) || didDeliver\n\t\t} else {\n\t\t\tdidDeliver, _ = c.processMsgResults(siAcc, rr, msg, c.pa.deliver, []byte(to), nrr, flags)\n\t\t}\n\t}\n\n\t// Restore to original values.\n\tc.in.rts = orts\n\tc.pa = pacopy\n\n\t// Before we undo didDeliver based on tracing and last mile, mark in the c.pa which informs us of no responders status.\n\t// If we override due to tracing and traceOnly we do not want to send back a no responders.\n\tc.pa.delivered = didDeliver\n\n\t// If this was a message trace but we skip last-mile delivery, we need to\n\t// do the remove, so:\n\tif mt != nil && traceOnly && didDeliver {\n\t\tdidDeliver = false\n\t}\n\n\t// Determine if we should remove this service import. This is for response service imports.\n\t// We will remove if we did not deliver, or if we are a response service import and we are\n\t// a singleton, or we have an EOF message.\n\tshouldRemove := !didDeliver || (isResponse && (si.rt == Singleton || len(msg) == LEN_CR_LF))\n\t// If we are tracking and we did not actually send the latency info we need to suppress the removal.\n\tif si.tracking && !didSendTL {\n\t\tshouldRemove = false\n\t}\n\t// If we are streamed or chunked we need to update our timestamp to avoid cleanup.\n\tif si.rt != Singleton && didDeliver {\n\t\tacc.mu.Lock()\n\t\tsi.ts = time.Now().UnixNano()\n\t\tacc.mu.Unlock()\n\t}\n\n\t// Cleanup of a response service import\n\tif shouldRemove {\n\t\treason := rsiOk\n\t\tif !didDeliver {\n\t\t\treason = rsiNoDelivery\n\t\t}\n\t\tif isResponse {\n\t\t\tacc.removeRespServiceImport(si, reason)\n\t\t} else {\n\t\t\t// This is a main import and since we could not even deliver to the exporting account\n\t\t\t// go ahead and remove the respServiceImport we created above.\n\t\t\tsiAcc.removeRespServiceImport(rsi, reason)\n\t\t}\n\t}\n\n\treturn didDeliver\n}\n\nfunc (c *client) addSubToRouteTargets(sub *subscription) {\n\tif c.in.rts == nil {\n\t\tc.in.rts = make([]routeTarget, 0, routeTargetInit)\n\t}\n\n\tfor i := range c.in.rts {\n\t\trt := &c.in.rts[i]\n\t\tif rt.sub.client == sub.client {\n\t\t\tif sub.queue != nil {\n\t\t\t\trt.qs = append(rt.qs, sub.queue...)\n\t\t\t\trt.qs = append(rt.qs, ' ')\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\tvar rt *routeTarget\n\tlrts := len(c.in.rts)\n\n\t// If we are here we do not have the sub yet in our list\n\t// If we have to grow do so here.\n\tif lrts == cap(c.in.rts) {\n\t\tc.in.rts = append(c.in.rts, routeTarget{})\n\t}\n\n\tc.in.rts = c.in.rts[:lrts+1]\n\trt = &c.in.rts[lrts]\n\trt.sub = sub\n\trt.qs = rt._qs[:0]\n\tif sub.queue != nil {\n\t\trt.qs = append(rt.qs, sub.queue...)\n\t\trt.qs = append(rt.qs, ' ')\n\t}\n}\n\n// This processes the sublist results for a given message.\n// Returns if the message was delivered to at least target and queue filters.\nfunc (c *client) processMsgResults(acc *Account, r *SublistResult, msg, deliver, subject, reply []byte, flags int) (bool, [][]byte) {\n\t// For sending messages across routes and leafnodes.\n\t// Reset if we have one since we reuse this data structure.\n\tif c.in.rts != nil {\n\t\tc.in.rts = c.in.rts[:0]\n\t}\n\n\tvar rplyHasGWPrefix bool\n\tvar creply = reply\n\n\t// If the reply subject is a GW routed reply, we will perform some\n\t// tracking in deliverMsg(). We also want to send to the user the\n\t// reply without the prefix. `creply` will be set to that and be\n\t// used to create the message header for client connections.\n\tif rplyHasGWPrefix = isGWRoutedReply(reply); rplyHasGWPrefix {\n\t\tcreply = reply[gwSubjectOffset:]\n\t}\n\n\t// With JetStream we now have times where we want to match a subscription\n\t// on one subject, but deliver it with another. e.g. JetStream deliverables.\n\t// This only works for last mile, meaning to a client. For other types we need\n\t// to use the original subject.\n\tsubj := subject\n\tif len(deliver) > 0 {\n\t\tsubj = deliver\n\t}\n\n\t// Check for JetStream encoded reply subjects.\n\t// For now these will only be on $JS.ACK prefixed reply subjects.\n\tvar remapped bool\n\tif len(creply) > 0 && c.kind != CLIENT && !isInternalClient(c.kind) && bytes.HasPrefix(creply, []byte(jsAckPre)) {\n\t\t// We need to rewrite the subject and the reply.\n\t\t// But, we must be careful that the stream name, consumer name, and subject can contain '@' characters.\n\t\t// JS ACK contains at least 8 dots, find the first @ after this prefix.\n\t\t// - $JS.ACK.......\n\t\tcounter := 0\n\t\tli := bytes.IndexFunc(creply, func(rn rune) bool {\n\t\t\tif rn == '.' {\n\t\t\t\tcounter++\n\t\t\t} else if rn == '@' {\n\t\t\t\treturn counter >= 8\n\t\t\t}\n\t\t\treturn false\n\t\t})\n\t\tif li != -1 && li < len(creply)-1 {\n\t\t\tremapped = true\n\t\t\tsubj, creply = creply[li+1:], creply[:li]\n\t\t}\n\t}\n\n\tvar didDeliver bool\n\n\t// delivery subject for clients\n\tvar dsubj []byte\n\t// Used as scratch if mapping\n\tvar _dsubj [128]byte\n\n\t// For stats, we will keep track of the number of messages that have been\n\t// delivered and then multiply by the size of that message and update\n\t// server and account stats in a \"single\" operation (instead of per-sub).\n\t// However, we account for situations where the message is possibly changed\n\t// by having an extra size\n\tvar dlvMsgs int64\n\tvar dlvExtraSize int64\n\tvar dlvRouteMsgs int64\n\tvar dlvLeafMsgs int64\n\n\t// We need to know if this is a MQTT producer because they send messages\n\t// without CR_LF (we otherwise remove the size of CR_LF from message size).\n\tprodIsMQTT := c.isMqtt()\n\n\tupdateStats := func() {\n\t\tif dlvMsgs == 0 {\n\t\t\treturn\n\t\t}\n\n\t\ttotalBytes := dlvMsgs*int64(len(msg)) + dlvExtraSize\n\t\trouteBytes := dlvRouteMsgs*int64(len(msg)) + dlvExtraSize\n\t\tleafBytes := dlvLeafMsgs*int64(len(msg)) + dlvExtraSize\n\n\t\t// For non MQTT producers, remove the CR_LF * number of messages\n\t\tif !prodIsMQTT {\n\t\t\ttotalBytes -= dlvMsgs * int64(LEN_CR_LF)\n\t\t\trouteBytes -= dlvRouteMsgs * int64(LEN_CR_LF)\n\t\t\tleafBytes -= dlvLeafMsgs * int64(LEN_CR_LF)\n\t\t}\n\n\t\tif acc != nil {\n\t\t\tacc.stats.Lock()\n\t\t\tacc.stats.outMsgs += dlvMsgs\n\t\t\tacc.stats.outBytes += totalBytes\n\t\t\tif dlvRouteMsgs > 0 {\n\t\t\t\tacc.stats.rt.outMsgs += dlvRouteMsgs\n\t\t\t\tacc.stats.rt.outBytes += routeBytes\n\t\t\t}\n\t\t\tif dlvLeafMsgs > 0 {\n\t\t\t\tacc.stats.ln.outMsgs += dlvLeafMsgs\n\t\t\t\tacc.stats.ln.outBytes += leafBytes\n\t\t\t}\n\t\t\tacc.stats.Unlock()\n\t\t}\n\n\t\tif srv := c.srv; srv != nil {\n\t\t\tatomic.AddInt64(&srv.outMsgs, dlvMsgs)\n\t\t\tatomic.AddInt64(&srv.outBytes, totalBytes)\n\t\t}\n\t}\n\n\tmt, traceOnly := c.isMsgTraceEnabled()\n\n\t// Loop over all normal subscriptions that match.\n\tfor _, sub := range r.psubs {\n\t\t// Check if this is a send to a ROUTER. We now process\n\t\t// these after everything else.\n\t\tswitch sub.client.kind {\n\t\tcase ROUTER:\n\t\t\tif (c.kind != ROUTER && !c.isSpokeLeafNode()) || (flags&pmrAllowSendFromRouteToRoute != 0) {\n\t\t\t\tc.addSubToRouteTargets(sub)\n\t\t\t}\n\t\t\tcontinue\n\t\tcase GATEWAY:\n\t\t\t// Never send to gateway from here.\n\t\t\tcontinue\n\t\tcase LEAF:\n\t\t\t// We handle similarly to routes and use the same data structures.\n\t\t\t// Leaf node delivery audience is different however.\n\t\t\t// Also leaf nodes are always no echo, so we make sure we are not\n\t\t\t// going to send back to ourselves here. For messages from routes we want\n\t\t\t// to suppress in general unless we know from the hub or its a service reply.\n\t\t\tif c != sub.client && (c.kind != ROUTER || sub.client.isHubLeafNode() || isServiceReply(c.pa.subject)) {\n\t\t\t\tc.addSubToRouteTargets(sub)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Assume delivery subject is the normal subject to this point.\n\t\tdsubj = subj\n\n\t\t// We may need to disable tracing, by setting c.pa.trace to `nil`\n\t\t// before the call to deliverMsg, if so, this will indicate that\n\t\t// we need to put it back.\n\t\tvar restorePaTrace bool\n\n\t\t// Check for stream import mapped subs (shadow subs). These apply to local subs only.\n\t\tif sub.im != nil {\n\t\t\t// If this message was a service import do not re-export to an exported stream.\n\t\t\tif flags&pmrMsgImportedFromService != 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif sub.im.tr != nil {\n\t\t\t\tto := sub.im.tr.TransformSubject(bytesToString(subject))\n\t\t\t\tdsubj = append(_dsubj[:0], to...)\n\t\t\t} else if sub.im.usePub {\n\t\t\t\tdsubj = append(_dsubj[:0], subj...)\n\t\t\t} else {\n\t\t\t\tdsubj = append(_dsubj[:0], sub.im.to...)\n\t\t\t}\n\n\t\t\tif mt != nil {\n\t\t\t\tmt.addStreamExportEvent(sub.client, dsubj)\n\t\t\t\t// If allow_trace is false...\n\t\t\t\tif !sub.im.atrc {\n\t\t\t\t\t// If we are doing only message tracing, we can move to the\n\t\t\t\t\t// next sub.\n\t\t\t\t\tif traceOnly {\n\t\t\t\t\t\t// Although the message was not delivered, for the purpose\n\t\t\t\t\t\t// of didDeliver, we need to set to true (to avoid possible\n\t\t\t\t\t\t// no responders).\n\t\t\t\t\t\tdidDeliver = true\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// If we are delivering the message, we need to disable tracing\n\t\t\t\t\t// before calling deliverMsg().\n\t\t\t\t\tc.pa.trace, restorePaTrace = nil, true\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Make sure deliver is set if inbound from a route.\n\t\t\tif remapped && (c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF) {\n\t\t\t\tdeliver = subj\n\t\t\t}\n\t\t\t// If we are mapping for a deliver subject we will reverse roles.\n\t\t\t// The original subj we set from above is correct for the msg header,\n\t\t\t// but we need to transform the deliver subject to properly route.\n\t\t\tif len(deliver) > 0 {\n\t\t\t\tdsubj, subj = subj, dsubj\n\t\t\t}\n\t\t}\n\n\t\t// Remap to the original subject if internal.\n\t\tif sub.icb != nil && sub.rsi {\n\t\t\tdsubj = subject\n\t\t}\n\n\t\t// Normal delivery\n\t\tmh := c.msgHeader(dsubj, creply, sub)\n\t\tif c.deliverMsg(prodIsMQTT, sub, acc, dsubj, creply, mh, msg, rplyHasGWPrefix) {\n\t\t\t// We don't count internal deliveries, so do only when sub.icb is nil.\n\t\t\tif sub.icb == nil {\n\t\t\t\tdlvMsgs++\n\t\t\t}\n\t\t\tdidDeliver = true\n\t\t}\n\t\tif restorePaTrace {\n\t\t\tc.pa.trace = mt\n\t\t}\n\t}\n\n\t// Set these up to optionally filter based on the queue lists.\n\t// This is for messages received from routes which will have directed\n\t// guidance on which queue groups we should deliver to.\n\tqf := c.pa.queues\n\n\t// Declared here because of goto.\n\tvar queues [][]byte\n\n\tvar leafOrigin string\n\tswitch c.kind {\n\tcase ROUTER:\n\t\tif len(c.pa.origin) > 0 {\n\t\t\t// Picture a message sent from a leafnode to a server that then routes\n\t\t\t// this message: CluserA -leaf-> HUB1 -route-> HUB2\n\t\t\t// Here we are in HUB2, so c.kind is a ROUTER, but the message will\n\t\t\t// contain a c.pa.origin set to \"ClusterA\" to indicate that this message\n\t\t\t// originated from that leafnode cluster.\n\t\t\tleafOrigin = bytesToString(c.pa.origin)\n\t\t}\n\tcase LEAF:\n\t\tleafOrigin = c.remoteCluster()\n\t}\n\n\t// For all routes/leaf/gateway connections, we may still want to send messages to\n\t// leaf nodes or routes even if there are no queue filters since we collect\n\t// them above and do not process inline like normal clients.\n\t// However, do select queue subs if asked to ignore empty queue filter.\n\tif (c.kind == LEAF || c.kind == ROUTER || c.kind == GATEWAY) && len(qf) == 0 && flags&pmrIgnoreEmptyQueueFilter == 0 {\n\t\tgoto sendToRoutesOrLeafs\n\t}\n\n\t// Process queue subs\n\tfor i := 0; i < len(r.qsubs); i++ {\n\t\tqsubs := r.qsubs[i]\n\t\t// If we have a filter check that here. We could make this a map or someting more\n\t\t// complex but linear search since we expect queues to be small. Should be faster\n\t\t// and more cache friendly.\n\t\tif qf != nil && len(qsubs) > 0 {\n\t\t\ttqn := qsubs[0].queue\n\t\t\tfor _, qn := range qf {\n\t\t\t\tif bytes.Equal(qn, tqn) {\n\t\t\t\t\tgoto selectQSub\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\tselectQSub:\n\t\t// We will hold onto remote or lead qsubs when we are coming from\n\t\t// a route or a leaf node just in case we can no longer do local delivery.\n\t\tvar rsub, sub *subscription\n\t\tvar _ql [32]*subscription\n\n\t\tsrc := c.kind\n\t\t// If we just came from a route we want to prefer local subs.\n\t\t// So only select from local subs but remember the first rsub\n\t\t// in case all else fails.\n\t\tif src == ROUTER {\n\t\t\tql := _ql[:0]\n\t\t\tfor i := 0; i < len(qsubs); i++ {\n\t\t\t\tsub = qsubs[i]\n\t\t\t\tif dst := sub.client.kind; dst == LEAF || dst == ROUTER {\n\t\t\t\t\t// If the destination is a LEAF, we first need to make sure\n\t\t\t\t\t// that we would not pick one that was the origin of this\n\t\t\t\t\t// message.\n\t\t\t\t\tif dst == LEAF && leafOrigin != _EMPTY_ && leafOrigin == sub.client.remoteCluster() {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// If we have assigned a ROUTER rsub already, replace if\n\t\t\t\t\t// the destination is a LEAF since we want to favor that.\n\t\t\t\t\tif rsub == nil || (rsub.client.kind == ROUTER && dst == LEAF) {\n\t\t\t\t\t\trsub = sub\n\t\t\t\t\t} else if dst == LEAF {\n\t\t\t\t\t\t// We already have a LEAF and this is another one.\n\t\t\t\t\t\t// Flip a coin to see if we swap it or not.\n\t\t\t\t\t\t// See https://github.com/nats-io/nats-server/issues/6040\n\t\t\t\t\t\tif fastrand.Uint32()%2 == 1 {\n\t\t\t\t\t\t\trsub = sub\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tql = append(ql, sub)\n\t\t\t\t}\n\t\t\t}\n\t\t\tqsubs = ql\n\t\t}\n\n\t\tsindex := 0\n\t\tlqs := len(qsubs)\n\t\tif lqs > 1 {\n\t\t\tsindex = int(fastrand.Uint32() % uint32(lqs))\n\t\t}\n\n\t\t// Find a subscription that is able to deliver this message starting at a random index.\n\t\t// Note that if the message came from a ROUTER, we will only have CLIENT or LEAF\n\t\t// queue subs here, otherwise we can have all types.\n\t\tfor i := 0; i < lqs; i++ {\n\t\t\tif sindex+i < lqs {\n\t\t\t\tsub = qsubs[sindex+i]\n\t\t\t} else {\n\t\t\t\tsub = qsubs[(sindex+i)%lqs]\n\t\t\t}\n\t\t\tif sub == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// If we are a spoke leaf node make sure to not forward across routes.\n\t\t\t// This mimics same behavior for normal subs above.\n\t\t\tif c.kind == LEAF && c.isSpokeLeafNode() && sub.client.kind == ROUTER {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// We have taken care of preferring local subs for a message from a route above.\n\t\t\t// Here we just care about a client or leaf and skipping a leaf and preferring locals.\n\t\t\tif dst := sub.client.kind; dst == ROUTER || dst == LEAF {\n\t\t\t\tif (src == LEAF || src == CLIENT) && dst == LEAF {\n\t\t\t\t\t// If we come from a LEAF and are about to pick a LEAF connection,\n\t\t\t\t\t// make sure this is not the same leaf cluster.\n\t\t\t\t\tif src == LEAF && leafOrigin != _EMPTY_ && leafOrigin == sub.client.remoteCluster() {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// Remember that leaf in case we don't find any other candidate.\n\t\t\t\t\t// We already start randomly in lqs slice, so we don't need\n\t\t\t\t\t// to do a random swap if we already have an rsub like we do\n\t\t\t\t\t// when src == ROUTER above.\n\t\t\t\t\tif rsub == nil {\n\t\t\t\t\t\trsub = sub\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\t// We want to favor qsubs in our own cluster. If the routed\n\t\t\t\t\t// qsub has an origin, it means that is on behalf of a leaf.\n\t\t\t\t\t// We need to treat it differently.\n\t\t\t\t\tif len(sub.origin) > 0 {\n\t\t\t\t\t\t// If we already have an rsub, nothing to do. Also, do\n\t\t\t\t\t\t// not pick a routed qsub for a LEAF origin cluster\n\t\t\t\t\t\t// that is the same than where the message comes from.\n\t\t\t\t\t\tif rsub == nil && (leafOrigin == _EMPTY_ || leafOrigin != bytesToString(sub.origin)) {\n\t\t\t\t\t\t\trsub = sub\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// This is a qsub that is local on the remote server (or\n\t\t\t\t\t// we are connected to an older server and we don't know).\n\t\t\t\t\t// Pick this one and be done.\n\t\t\t\t\trsub = sub\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Assume delivery subject is normal subject to this point.\n\t\t\tdsubj = subj\n\n\t\t\t// We may need to disable tracing, by setting c.pa.trace to `nil`\n\t\t\t// before the call to deliverMsg, if so, this will indicate that\n\t\t\t// we need to put it back.\n\t\t\tvar restorePaTrace bool\n\t\t\tvar skipDelivery bool\n\n\t\t\t// Check for stream import mapped subs. These apply to local subs only.\n\t\t\tif sub.im != nil {\n\t\t\t\t// If this message was a service import do not re-export to an exported stream.\n\t\t\t\tif flags&pmrMsgImportedFromService != 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif sub.im.tr != nil {\n\t\t\t\t\tto := sub.im.tr.TransformSubject(bytesToString(subject))\n\t\t\t\t\tdsubj = append(_dsubj[:0], to...)\n\t\t\t\t} else if sub.im.usePub {\n\t\t\t\t\tdsubj = append(_dsubj[:0], subj...)\n\t\t\t\t} else {\n\t\t\t\t\tdsubj = append(_dsubj[:0], sub.im.to...)\n\t\t\t\t}\n\n\t\t\t\tif mt != nil {\n\t\t\t\t\tmt.addStreamExportEvent(sub.client, dsubj)\n\t\t\t\t\t// If allow_trace is false...\n\t\t\t\t\tif !sub.im.atrc {\n\t\t\t\t\t\t// If we are doing only message tracing, we are done\n\t\t\t\t\t\t// with this queue group.\n\t\t\t\t\t\tif traceOnly {\n\t\t\t\t\t\t\tskipDelivery = true\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t// If we are delivering, we need to disable tracing\n\t\t\t\t\t\t\t// before the call to deliverMsg()\n\t\t\t\t\t\t\tc.pa.trace, restorePaTrace = nil, true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Make sure deliver is set if inbound from a route.\n\t\t\t\tif remapped && (c.kind == GATEWAY || c.kind == ROUTER || c.kind == LEAF) {\n\t\t\t\t\tdeliver = subj\n\t\t\t\t}\n\t\t\t\t// If we are mapping for a deliver subject we will reverse roles.\n\t\t\t\t// The original subj we set from above is correct for the msg header,\n\t\t\t\t// but we need to transform the deliver subject to properly route.\n\t\t\t\tif len(deliver) > 0 {\n\t\t\t\t\tdsubj, subj = subj, dsubj\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar delivered bool\n\t\t\tif !skipDelivery {\n\t\t\t\tmh := c.msgHeader(dsubj, creply, sub)\n\t\t\t\tdelivered = c.deliverMsg(prodIsMQTT, sub, acc, subject, creply, mh, msg, rplyHasGWPrefix)\n\t\t\t\tif restorePaTrace {\n\t\t\t\t\tc.pa.trace = mt\n\t\t\t\t}\n\t\t\t}\n\t\t\tif skipDelivery || delivered {\n\t\t\t\t// Update only if not skipped.\n\t\t\t\tif !skipDelivery && sub.icb == nil {\n\t\t\t\t\tdlvMsgs++\n\t\t\t\t\tswitch sub.client.kind {\n\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\tdlvRouteMsgs++\n\t\t\t\t\tcase LEAF:\n\t\t\t\t\t\tdlvLeafMsgs++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Do the rest even when message delivery was skipped.\n\t\t\t\tdidDeliver = true\n\t\t\t\t// Clear rsub\n\t\t\t\trsub = nil\n\t\t\t\tif flags&pmrCollectQueueNames != 0 {\n\t\t\t\t\tqueues = append(queues, sub.queue)\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif rsub != nil {\n\t\t\t// We are here if we have selected a leaf or route as the destination,\n\t\t\t// or if we tried to deliver to a local qsub but failed.\n\t\t\tc.addSubToRouteTargets(rsub)\n\t\t\tif flags&pmrCollectQueueNames != 0 {\n\t\t\t\tqueues = append(queues, rsub.queue)\n\t\t\t}\n\t\t}\n\t}\n\nsendToRoutesOrLeafs:\n\n\t// If no messages for routes or leafnodes return here.\n\tif len(c.in.rts) == 0 {\n\t\tupdateStats()\n\t\treturn didDeliver, queues\n\t}\n\n\t// If we do have a deliver subject we need to do something with it.\n\t// Again this is when JetStream (but possibly others) wants the system\n\t// to rewrite the delivered subject. The way we will do that is place it\n\t// at the end of the reply subject if it exists. But only if this wasn't\n\t// already performed, otherwise we'd end up with a duplicate '@' suffix\n\t// resulting in a protocol error.\n\tif len(deliver) > 0 && len(reply) > 0 && !remapped {\n\t\treply = append(reply, '@')\n\t\treply = append(reply, deliver...)\n\t}\n\n\t// Copy off original pa in case it changes.\n\tpa := c.pa\n\n\tif mt != nil {\n\t\t// We are going to replace \"pa\" with our copy of c.pa, but to restore\n\t\t// to the original copy of c.pa, we need to save it again.\n\t\tcpa := pa\n\t\tmsg = mt.setOriginAccountHeaderIfNeeded(c, acc, msg)\n\t\tdefer func() { c.pa = cpa }()\n\t\t// Update pa with our current c.pa state.\n\t\tpa = c.pa\n\t}\n\n\t// We address by index to avoid struct copy.\n\t// We have inline structs for memory layout and cache coherency.\n\tfor i := range c.in.rts {\n\t\trt := &c.in.rts[i]\n\t\tdc := rt.sub.client\n\t\tdmsg, hset := msg, false\n\n\t\t// Check if we have an origin cluster set from a leafnode message.\n\t\t// If so make sure we do not send it back to the same cluster for a different\n\t\t// leafnode. Cluster wide no echo.\n\t\tif dc.kind == LEAF {\n\t\t\t// Check two scenarios. One is inbound from a route (c.pa.origin),\n\t\t\t// and the other is leaf to leaf. In both case, leafOrigin is the one\n\t\t\t// to use for the comparison.\n\t\t\tif leafOrigin != _EMPTY_ && leafOrigin == dc.remoteCluster() {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// We need to check if this is a request that has a stamped client information header.\n\t\t\t// This will contain an account but will represent the account from the leafnode. If\n\t\t\t// they are not named the same this would cause an account lookup failure trying to\n\t\t\t// process the request for something like JetStream or other system services that rely\n\t\t\t// on the client info header. We can just check for reply and the presence of a header\n\t\t\t// to avoid slow downs for all traffic.\n\t\t\tif len(c.pa.reply) > 0 && c.pa.hdr >= 0 {\n\t\t\t\tdmsg, hset = c.checkLeafClientInfoHeader(msg)\n\t\t\t}\n\t\t}\n\n\t\tif mt != nil {\n\t\t\tdmsg = mt.setHopHeader(c, dmsg)\n\t\t\thset = true\n\t\t}\n\n\t\tmh := c.msgHeaderForRouteOrLeaf(subject, reply, rt, acc)\n\t\tif c.deliverMsg(prodIsMQTT, rt.sub, acc, subject, reply, mh, dmsg, false) {\n\t\t\tif rt.sub.icb == nil {\n\t\t\t\tdlvMsgs++\n\t\t\t\tswitch dc.kind {\n\t\t\t\tcase ROUTER:\n\t\t\t\t\tdlvRouteMsgs++\n\t\t\t\tcase LEAF:\n\t\t\t\t\tdlvLeafMsgs++\n\t\t\t\t}\n\t\t\t\tdlvExtraSize += int64(len(dmsg) - len(msg))\n\t\t\t}\n\t\t\tdidDeliver = true\n\t\t}\n\n\t\t// If we set the header reset the origin pub args.\n\t\tif hset {\n\t\t\tc.pa = pa\n\t\t}\n\t}\n\tupdateStats()\n\treturn didDeliver, queues\n}\n\n// Check and swap accounts on a client info header destined across a leafnode.\nfunc (c *client) checkLeafClientInfoHeader(msg []byte) (dmsg []byte, setHdr bool) {\n\tif c.pa.hdr < 0 || len(msg) < c.pa.hdr {\n\t\treturn msg, false\n\t}\n\tcir := sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])\n\tif len(cir) == 0 {\n\t\treturn msg, false\n\t}\n\n\tdmsg = msg\n\n\tvar ci ClientInfo\n\tif err := json.Unmarshal(cir, &ci); err == nil {\n\t\tif v, _ := c.srv.leafRemoteAccounts.Load(ci.Account); v != nil {\n\t\t\tremoteAcc := v.(string)\n\t\t\tif ci.Account != remoteAcc {\n\t\t\t\tci.Account = remoteAcc\n\t\t\t\tif b, _ := json.Marshal(ci); b != nil {\n\t\t\t\t\tdmsg, setHdr = c.setHeader(ClientInfoHdr, bytesToString(b), msg), true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn dmsg, setHdr\n}\n\nfunc (c *client) pubPermissionViolation(subject []byte) {\n\terrTxt := fmt.Sprintf(\"Permissions Violation for Publish to %q\", subject)\n\tif mt, _ := c.isMsgTraceEnabled(); mt != nil {\n\t\tmt.setIngressError(errTxt)\n\t}\n\tc.sendErr(errTxt)\n\tc.Errorf(\"Publish Violation - Subject %q\", subject)\n}\n\nfunc (c *client) subPermissionViolation(sub *subscription) {\n\terrTxt := fmt.Sprintf(\"Permissions Violation for Subscription to %q\", sub.subject)\n\tlogTxt := fmt.Sprintf(\"Subscription Violation - Subject %q, SID %s\", sub.subject, sub.sid)\n\n\tif sub.queue != nil {\n\t\terrTxt = fmt.Sprintf(\"Permissions Violation for Subscription to %q using queue %q\", sub.subject, sub.queue)\n\t\tlogTxt = fmt.Sprintf(\"Subscription Violation - Subject %q, Queue: %q, SID %s\", sub.subject, sub.queue, sub.sid)\n\t}\n\n\tc.sendErr(errTxt)\n\tc.Errorf(logTxt)\n}\n\nfunc (c *client) replySubjectViolation(reply []byte) {\n\terrTxt := fmt.Sprintf(\"Permissions Violation for Publish with Reply of %q\", reply)\n\tif mt, _ := c.isMsgTraceEnabled(); mt != nil {\n\t\tmt.setIngressError(errTxt)\n\t}\n\tc.sendErr(errTxt)\n\tc.Errorf(\"Publish Violation - Reply %q\", reply)\n}\n\nfunc (c *client) maxTokensViolation(sub *subscription) {\n\terrTxt := fmt.Sprintf(\"Permissions Violation for Subscription to %q, too many tokens\", sub.subject)\n\tlogTxt := fmt.Sprintf(\"Subscription Violation Too Many Tokens - Subject %q, SID %s\", sub.subject, sub.sid)\n\tc.sendErr(errTxt)\n\tc.Errorf(logTxt)\n}\n\nfunc (c *client) processPingTimer() {\n\tc.mu.Lock()\n\tc.ping.tmr = nil\n\t// Check if connection is still opened\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\tc.Debugf(\"%s Ping Timer\", c.kindString())\n\n\tvar sendPing bool\n\n\topts := c.srv.getOpts()\n\tpingInterval := opts.PingInterval\n\tif c.kind == ROUTER && opts.Cluster.PingInterval > 0 {\n\t\tpingInterval = opts.Cluster.PingInterval\n\t}\n\tif c.isWebsocket() && opts.Websocket.PingInterval > 0 {\n\t\tpingInterval = opts.Websocket.PingInterval\n\t}\n\tpingInterval = adjustPingInterval(c.kind, pingInterval)\n\tnow := time.Now()\n\tneedRTT := c.rtt == 0 || now.Sub(c.rttStart) > DEFAULT_RTT_MEASUREMENT_INTERVAL\n\n\t// Do not delay PINGs for ROUTER, GATEWAY or spoke LEAF connections.\n\tif c.kind == ROUTER || c.kind == GATEWAY || c.isSpokeLeafNode() {\n\t\tsendPing = true\n\t} else {\n\t\t// If we received client data or a ping from the other side within the PingInterval,\n\t\t// then there is no need to send a ping.\n\t\tif delta := now.Sub(c.lastIn); delta < pingInterval && !needRTT {\n\t\t\tc.Debugf(\"Delaying PING due to remote client data or ping %v ago\", delta.Round(time.Second))\n\t\t} else {\n\t\t\tsendPing = true\n\t\t}\n\t}\n\n\tif sendPing {\n\t\t// Check for violation\n\t\tmaxPingsOut := opts.MaxPingsOut\n\t\tif c.kind == ROUTER && opts.Cluster.MaxPingsOut > 0 {\n\t\t\tmaxPingsOut = opts.Cluster.MaxPingsOut\n\t\t}\n\t\tif c.ping.out+1 > maxPingsOut {\n\t\t\tc.Debugf(\"Stale Client Connection - Closing\")\n\t\t\tc.enqueueProto([]byte(fmt.Sprintf(errProto, \"Stale Connection\")))\n\t\t\tc.mu.Unlock()\n\t\t\tc.closeConnection(StaleConnection)\n\t\t\treturn\n\t\t}\n\t\t// Send PING\n\t\tc.sendPing()\n\t}\n\n\t// Reset to fire again.\n\tc.setPingTimer()\n\tc.mu.Unlock()\n}\n\n// Returns the smallest value between the given `d` and some max value\n// based on the connection kind.\nfunc adjustPingInterval(kind int, d time.Duration) time.Duration {\n\tswitch kind {\n\tcase ROUTER:\n\t\tif d > routeMaxPingInterval {\n\t\t\treturn routeMaxPingInterval\n\t\t}\n\tcase GATEWAY:\n\t\tif d > gatewayMaxPingInterval {\n\t\t\treturn gatewayMaxPingInterval\n\t\t}\n\t}\n\treturn d\n}\n\n// This is used when a connection cannot yet start to send PINGs because\n// the remote would not be able to handle them (case of compression,\n// or outbound gateway, etc...), but we still want to close the connection\n// if the timer has not been reset by the time we reach the time equivalent\n// to have sent the max number of pings.\n//\n// Lock should be held\nfunc (c *client) watchForStaleConnection(pingInterval time.Duration, pingMax int) {\n\tc.ping.tmr = time.AfterFunc(pingInterval*time.Duration(pingMax+1), func() {\n\t\tc.mu.Lock()\n\t\tc.Debugf(\"Stale Client Connection - Closing\")\n\t\tc.enqueueProto([]byte(fmt.Sprintf(errProto, \"Stale Connection\")))\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(StaleConnection)\n\t})\n}\n\n// Lock should be held\nfunc (c *client) setPingTimer() {\n\tif c.srv == nil {\n\t\treturn\n\t}\n\topts := c.srv.getOpts()\n\td := opts.PingInterval\n\tif c.kind == ROUTER && opts.Cluster.PingInterval > 0 {\n\t\td = opts.Cluster.PingInterval\n\t}\n\tif c.isWebsocket() && opts.Websocket.PingInterval > 0 {\n\t\td = opts.Websocket.PingInterval\n\t}\n\td = adjustPingInterval(c.kind, d)\n\tc.ping.tmr = time.AfterFunc(d, c.processPingTimer)\n}\n\n// Lock should be held\nfunc (c *client) clearPingTimer() {\n\tif c.ping.tmr == nil {\n\t\treturn\n\t}\n\tc.ping.tmr.Stop()\n\tc.ping.tmr = nil\n}\n\nfunc (c *client) clearTlsToTimer() {\n\tif c.tlsTo == nil {\n\t\treturn\n\t}\n\tc.tlsTo.Stop()\n\tc.tlsTo = nil\n}\n\n// Lock should be held\nfunc (c *client) setAuthTimer(d time.Duration) {\n\tc.atmr = time.AfterFunc(d, c.authTimeout)\n}\n\n// Lock should be held\nfunc (c *client) clearAuthTimer() bool {\n\tif c.atmr == nil {\n\t\treturn true\n\t}\n\tstopped := c.atmr.Stop()\n\tc.atmr = nil\n\treturn stopped\n}\n\n// We may reuse atmr for expiring user jwts,\n// so check connectReceived.\n// Lock assume held on entry.\nfunc (c *client) awaitingAuth() bool {\n\treturn !c.flags.isSet(connectReceived) && c.atmr != nil\n}\n\n// This will set the atmr for the JWT expiration time.\n// We will lock on entry.\nfunc (c *client) setExpirationTimer(d time.Duration) {\n\tc.mu.Lock()\n\tc.setExpirationTimerUnlocked(d)\n\tc.mu.Unlock()\n}\n\n// This will set the atmr for the JWT expiration time. client lock should be held before call\nfunc (c *client) setExpirationTimerUnlocked(d time.Duration) {\n\tc.atmr = time.AfterFunc(d, c.authExpired)\n\t// This is an JWT expiration.\n\tif c.flags.isSet(connectReceived) {\n\t\tc.expires = time.Now().Add(d).Truncate(time.Second)\n\t}\n}\n\n// Return when this client expires via a claim, or 0 if not set.\nfunc (c *client) claimExpiration() time.Duration {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif c.expires.IsZero() {\n\t\treturn 0\n\t}\n\treturn time.Until(c.expires).Truncate(time.Second)\n}\n\n// Possibly flush the connection and then close the low level connection.\n// The boolean `minimalFlush` indicates if the flush operation should have a\n// minimal write deadline.\n// Lock is held on entry.\nfunc (c *client) flushAndClose(minimalFlush bool) {\n\tif !c.flags.isSet(skipFlushOnClose) && c.out.pb > 0 {\n\t\tif minimalFlush {\n\t\t\tconst lowWriteDeadline = 100 * time.Millisecond\n\n\t\t\t// Reduce the write deadline if needed.\n\t\t\tif c.out.wdl > lowWriteDeadline {\n\t\t\t\tc.out.wdl = lowWriteDeadline\n\t\t\t}\n\t\t}\n\t\tc.flushOutbound()\n\t}\n\tfor i := range c.out.nb {\n\t\tnbPoolPut(c.out.nb[i])\n\t}\n\tc.out.nb = nil\n\t// We can't touch c.out.wnb when a flushOutbound is in progress since it\n\t// is accessed outside the lock there. If in progress, the cleanup will be\n\t// done in flushOutbound when detecting that connection is closed.\n\tif !c.flags.isSet(flushOutbound) {\n\t\tfor i := range c.out.wnb {\n\t\t\tnbPoolPut(c.out.wnb[i])\n\t\t}\n\t\tc.out.wnb = nil\n\t}\n\t// This seem to be important (from experimentation) for the GC to release\n\t// the connection.\n\tc.out.sg = nil\n\n\t// Close the low level connection.\n\tif c.nc != nil {\n\t\t// Starting with Go 1.16, the low level close will set its own deadline\n\t\t// of 5 seconds, so setting our own deadline does not work. Instead,\n\t\t// we will close the TLS connection in separate go routine.\n\t\tnc := c.nc\n\t\tc.nc = nil\n\t\tif _, ok := nc.(*tls.Conn); ok {\n\t\t\tgo func() { nc.Close() }()\n\t\t} else {\n\t\t\tnc.Close()\n\t\t}\n\t}\n}\n\nvar kindStringMap = map[int]string{\n\tCLIENT: \"Client\",\n\tROUTER: \"Router\",\n\tGATEWAY: \"Gateway\",\n\tLEAF: \"Leafnode\",\n\tJETSTREAM: \"JetStream\",\n\tACCOUNT: \"Account\",\n\tSYSTEM: \"System\",\n}\n\nfunc (c *client) kindString() string {\n\tif kindStringVal, ok := kindStringMap[c.kind]; ok {\n\t\treturn kindStringVal\n\t}\n\treturn \"Unknown Type\"\n}\n\n// swapAccountAfterReload will check to make sure the bound account for this client\n// is current. Under certain circumstances after a reload we could be pointing to\n// an older one.\nfunc (c *client) swapAccountAfterReload() {\n\tc.mu.Lock()\n\tsrv := c.srv\n\tan := c.acc.GetName()\n\tc.mu.Unlock()\n\tif srv == nil {\n\t\treturn\n\t}\n\tif acc, _ := srv.LookupAccount(an); acc != nil {\n\t\tc.mu.Lock()\n\t\tif c.acc != acc {\n\t\t\tc.acc = acc\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n}\n\n// processSubsOnConfigReload removes any subscriptions the client has that are no\n// longer authorized, and checks for imports (accounts) due to a config reload.\nfunc (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) {\n\tc.mu.Lock()\n\tvar (\n\t\tcheckPerms = c.perms != nil\n\t\tcheckAcc = c.acc != nil\n\t\tacc = c.acc\n\t)\n\tif !checkPerms && !checkAcc {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tvar (\n\t\t_subs [32]*subscription\n\t\tsubs = _subs[:0]\n\t\t_removed [32]*subscription\n\t\tremoved = _removed[:0]\n\t\tsrv = c.srv\n\t)\n\tif checkAcc {\n\t\t// We actually only want to check if stream imports have changed.\n\t\tif _, ok := awcsti[acc.Name]; !ok {\n\t\t\tcheckAcc = false\n\t\t}\n\t}\n\t// We will clear any mperms we have here. It will rebuild on the fly with canSubscribe,\n\t// so we do that here as we collect them. We will check result down below.\n\tc.mperms = nil\n\t// Collect client's subs under the lock\n\tfor _, sub := range c.subs {\n\t\t// Just checking to rebuild mperms under the lock, will collect removed though here.\n\t\t// Only collect under subs array of canSubscribe and checkAcc true.\n\t\tcanSub := c.canSubscribe(string(sub.subject))\n\t\tcanQSub := sub.queue != nil && c.canSubscribe(string(sub.subject), string(sub.queue))\n\n\t\tif !canSub && !canQSub {\n\t\t\tremoved = append(removed, sub)\n\t\t} else if checkAcc {\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t}\n\tc.mu.Unlock()\n\n\t// This list is all subs who are allowed and we need to check accounts.\n\tfor _, sub := range subs {\n\t\tc.mu.Lock()\n\t\toldShadows := sub.shadow\n\t\tsub.shadow = nil\n\t\tc.mu.Unlock()\n\t\tc.addShadowSubscriptions(acc, sub)\n\t\tfor _, nsub := range oldShadows {\n\t\t\tnsub.im.acc.sl.Remove(nsub)\n\t\t}\n\t}\n\n\t// Unsubscribe all that need to be removed and report back to client and logs.\n\tfor _, sub := range removed {\n\t\tc.unsubscribe(acc, sub, true, true)\n\t\tc.sendErr(fmt.Sprintf(\"Permissions Violation for Subscription to %q (sid %q)\", sub.subject, sub.sid))\n\t\tsrv.Noticef(\"Removed sub %q (sid %q) for %s - not authorized\", sub.subject, sub.sid, c.getAuthUser())\n\t}\n}\n\n// Allows us to count up all the queue subscribers during close.\ntype qsub struct {\n\tsub *subscription\n\tn int32\n}\n\nfunc (c *client) closeConnection(reason ClosedState) {\n\tc.mu.Lock()\n\tif c.flags.isSet(closeConnection) {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\t// Note that we may have markConnAsClosed() invoked before closeConnection(),\n\t// so don't set this to 1, instead bump the count.\n\tc.rref++\n\tc.flags.set(closeConnection)\n\tc.clearAuthTimer()\n\tc.clearPingTimer()\n\tc.clearTlsToTimer()\n\tc.markConnAsClosed(reason)\n\n\t// Unblock anyone who is potentially stalled waiting on us.\n\tif c.out.stc != nil {\n\t\tclose(c.out.stc)\n\t\tc.out.stc = nil\n\t}\n\n\t// If we have remote latency tracking running shut that down.\n\tif c.rrTracking != nil {\n\t\tc.rrTracking.ptmr.Stop()\n\t\tc.rrTracking = nil\n\t}\n\n\t// If we are shutting down, no need to do all the accounting on subs, etc.\n\t// During LDM we'll still do the accounting, otherwise account limits could close others after this reconnects.\n\tif reason == ServerShutdown && c.srv.isShuttingDown() {\n\t\ts := c.srv\n\t\tc.mu.Unlock()\n\t\tif s != nil {\n\t\t\t// Unregister\n\t\t\ts.removeClient(c)\n\t\t}\n\t\treturn\n\t}\n\n\tvar (\n\t\tkind = c.kind\n\t\tsrv = c.srv\n\t\tnoReconnect = c.flags.isSet(noReconnect)\n\t\tacc = c.acc\n\t\tspoke bool\n\t)\n\n\t// Snapshot for use if we are a client connection.\n\t// FIXME(dlc) - we can just stub in a new one for client\n\t// and reference existing one.\n\tvar subs []*subscription\n\tif kind == CLIENT || kind == LEAF || kind == JETSTREAM {\n\t\tvar _subs [32]*subscription\n\t\tsubs = _subs[:0]\n\t\t// Do not set c.subs to nil or delete the sub from c.subs here because\n\t\t// it will be needed in saveClosedClient (which has been started as a\n\t\t// go routine in markConnAsClosed). Cleanup will be done there.\n\t\tfor _, sub := range c.subs {\n\t\t\t// Auto-unsubscribe subscriptions must be unsubscribed forcibly.\n\t\t\tsub.max = 0\n\t\t\tsub.close()\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t\tspoke = c.isSpokeLeafNode()\n\t}\n\n\tc.mu.Unlock()\n\n\t// Remove client's or leaf node or jetstream subscriptions.\n\tif acc != nil && (kind == CLIENT || kind == LEAF || kind == JETSTREAM) {\n\t\tacc.sl.RemoveBatch(subs)\n\t} else if kind == ROUTER {\n\t\tc.removeRemoteSubs()\n\t}\n\n\tif srv != nil {\n\t\t// Unregister\n\t\tsrv.removeClient(c)\n\n\t\tif acc != nil {\n\t\t\t// Update remote subscriptions.\n\t\t\tif kind == CLIENT || kind == LEAF || kind == JETSTREAM {\n\t\t\t\tqsubs := map[string]*qsub{}\n\t\t\t\tfor _, sub := range subs {\n\t\t\t\t\t// Call unsubscribe here to cleanup shadow subscriptions and such.\n\t\t\t\t\tc.unsubscribe(acc, sub, true, false)\n\t\t\t\t\t// Update route as normal for a normal subscriber.\n\t\t\t\t\tif sub.queue == nil {\n\t\t\t\t\t\tif !spoke {\n\t\t\t\t\t\t\tsrv.updateRouteSubscriptionMap(acc, sub, -1)\n\t\t\t\t\t\t\tif srv.gateway.enabled {\n\t\t\t\t\t\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, -1)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tacc.updateLeafNodes(sub, -1)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// We handle queue subscribers special in case we\n\t\t\t\t\t\t// have a bunch we can just send one update to the\n\t\t\t\t\t\t// connected routes.\n\t\t\t\t\t\tnum := int32(1)\n\t\t\t\t\t\tif kind == LEAF {\n\t\t\t\t\t\t\tnum = sub.qw\n\t\t\t\t\t\t}\n\t\t\t\t\t\tkey := keyFromSub(sub)\n\t\t\t\t\t\tif esub, ok := qsubs[key]; ok {\n\t\t\t\t\t\t\tesub.n += num\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tqsubs[key] = &qsub{sub, num}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Process any qsubs here.\n\t\t\t\tfor _, esub := range qsubs {\n\t\t\t\t\tif !spoke {\n\t\t\t\t\t\tsrv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n))\n\t\t\t\t\t\tif srv.gateway.enabled {\n\t\t\t\t\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tacc.updateLeafNodes(esub.sub, -(esub.n))\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Always remove from the account, otherwise we can leak clients.\n\t\t\t// Note that SYSTEM and ACCOUNT types from above cleanup their own subs.\n\t\t\tif prev := acc.removeClient(c); prev == 1 {\n\t\t\t\tsrv.decActiveAccounts()\n\t\t\t}\n\t\t}\n\t}\n\n\t// Now that we are done with subscriptions, clear the field so that the\n\t// connection can be released and gc'ed.\n\tif kind == CLIENT || kind == LEAF {\n\t\tc.mu.Lock()\n\t\tc.subs = nil\n\t\tc.mu.Unlock()\n\t}\n\n\t// Don't reconnect connections that have been marked with\n\t// the no reconnect flag.\n\tif noReconnect {\n\t\treturn\n\t}\n\n\tc.reconnect()\n}\n\n// Depending on the kind of connections, this may attempt to recreate a connection.\n// The actual reconnect attempt will be started in a go routine.\nfunc (c *client) reconnect() {\n\tvar (\n\t\tretryImplicit bool\n\t\tgwName string\n\t\tgwIsOutbound bool\n\t\tgwCfg *gatewayCfg\n\t\tleafCfg *leafNodeCfg\n\t)\n\n\tc.mu.Lock()\n\t// Decrease the ref count and perform the reconnect only if == 0.\n\tc.rref--\n\tif c.flags.isSet(noReconnect) || c.rref > 0 {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tif c.route != nil {\n\t\t// A route is marked as solicited if it was given an URL to connect to,\n\t\t// which would be the case even with implicit (due to gossip), so mark this\n\t\t// as a retry for a route that is solicited and not explicit.\n\t\tretryImplicit = c.route.retry || (c.route.didSolicit && c.route.routeType == Implicit)\n\t}\n\tkind := c.kind\n\tswitch kind {\n\tcase GATEWAY:\n\t\tgwName = c.gw.name\n\t\tgwIsOutbound = c.gw.outbound\n\t\tgwCfg = c.gw.cfg\n\tcase LEAF:\n\t\tif c.isSolicitedLeafNode() {\n\t\t\tleafCfg = c.leaf.remote\n\t\t}\n\t}\n\tsrv := c.srv\n\tc.mu.Unlock()\n\n\t// Check for a solicited route. If it was, start up a reconnect unless\n\t// we are already connected to the other end.\n\tif didSolicit := c.isSolicitedRoute(); didSolicit || retryImplicit {\n\t\tsrv.mu.Lock()\n\t\tdefer srv.mu.Unlock()\n\n\t\t// Capture these under lock\n\t\tc.mu.Lock()\n\t\trid := c.route.remoteID\n\t\trtype := c.route.routeType\n\t\trurl := c.route.url\n\t\taccName := string(c.route.accName)\n\t\tcheckRID := accName == _EMPTY_ && srv.getOpts().Cluster.PoolSize < 1 && rid != _EMPTY_\n\t\tc.mu.Unlock()\n\n\t\t// It is possible that the server is being shutdown.\n\t\t// If so, don't try to reconnect\n\t\tif !srv.isRunning() {\n\t\t\treturn\n\t\t}\n\n\t\tif checkRID && srv.routes[rid] != nil {\n\t\t\t// This is the case of \"no pool\". Make sure that the registered one\n\t\t\t// is upgraded to solicited if the connection trying to reconnect\n\t\t\t// was a solicited one.\n\t\t\tif didSolicit {\n\t\t\t\tif remote := srv.routes[rid][0]; remote != nil {\n\t\t\t\t\tupgradeRouteToSolicited(remote, rurl, rtype)\n\t\t\t\t}\n\t\t\t}\n\t\t\tsrv.Debugf(\"Not attempting reconnect for solicited route, already connected to %q\", rid)\n\t\t\treturn\n\t\t} else if rid == srv.info.ID {\n\t\t\tsrv.Debugf(\"Detected route to self, ignoring %q\", rurl.Redacted())\n\t\t\treturn\n\t\t} else if rtype != Implicit || retryImplicit {\n\t\t\tsrv.Debugf(\"Attempting reconnect for solicited route %q\", rurl.Redacted())\n\t\t\t// Keep track of this go-routine so we can wait for it on\n\t\t\t// server shutdown.\n\t\t\tsrv.startGoRoutine(func() { srv.reConnectToRoute(rurl, rtype, accName) })\n\t\t}\n\t} else if srv != nil && kind == GATEWAY && gwIsOutbound {\n\t\tif gwCfg != nil {\n\t\t\tsrv.Debugf(\"Attempting reconnect for gateway %q\", gwName)\n\t\t\t// Run this as a go routine since we may be called within\n\t\t\t// the solicitGateway itself if there was an error during\n\t\t\t// the creation of the gateway connection.\n\t\t\tsrv.startGoRoutine(func() { srv.reconnectGateway(gwCfg) })\n\t\t} else {\n\t\t\tsrv.Debugf(\"Gateway %q not in configuration, not attempting reconnect\", gwName)\n\t\t}\n\t} else if leafCfg != nil {\n\t\t// Check if this is a solicited leaf node. Start up a reconnect.\n\t\tsrv.startGoRoutine(func() { srv.reConnectToRemoteLeafNode(leafCfg) })\n\t}\n}\n\n// Set the noReconnect flag. This is used before a call to closeConnection()\n// to prevent the connection to reconnect (routes, gateways).\nfunc (c *client) setNoReconnect() {\n\tc.mu.Lock()\n\tc.flags.set(noReconnect)\n\tc.mu.Unlock()\n}\n\n// Returns the client's RTT value with the protection of the client's lock.\nfunc (c *client) getRTTValue() time.Duration {\n\tc.mu.Lock()\n\trtt := c.rtt\n\tc.mu.Unlock()\n\treturn rtt\n}\n\n// This function is used by ROUTER and GATEWAY connections to\n// look for a subject on a given account (since these type of\n// connections are not bound to a specific account).\n// If the c.pa.subject is found in the cache, the cached result\n// is returned, otherwse, we match the account's sublist and update\n// the cache. The cache is pruned if reaching a certain size.\nfunc (c *client) getAccAndResultFromCache() (*Account, *SublistResult) {\n\tvar (\n\t\tacc *Account\n\t\tpac *perAccountCache\n\t\tr *SublistResult\n\t\tok bool\n\t)\n\t// Check our cache.\n\tif pac, ok = c.in.pacache[string(c.pa.pacache)]; ok {\n\t\t// Check the genid to see if it's still valid.\n\t\t// Since v2.10.0, the config reload of accounts has been fixed\n\t\t// and an account's sublist pointer should not change, so no need to\n\t\t// lock to access it.\n\t\tsl := pac.acc.sl\n\n\t\tif genid := atomic.LoadUint64(&sl.genid); genid != pac.genid {\n\t\t\tok = false\n\t\t\tclear(c.in.pacache)\n\t\t} else {\n\t\t\tacc = pac.acc\n\t\t\tr = pac.results\n\t\t}\n\t}\n\n\tif !ok {\n\t\tif c.kind == ROUTER && len(c.route.accName) > 0 {\n\t\t\tif acc = c.acc; acc == nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t} else {\n\t\t\t// Match correct account and sublist.\n\t\t\tif acc, _ = c.srv.LookupAccount(bytesToString(c.pa.account)); acc == nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t}\n\t\tsl := acc.sl\n\n\t\t// Match against the account sublist.\n\t\tr = sl.MatchBytes(c.pa.subject)\n\n\t\t// Check if we need to prune. This should give us a perAccountCache struct\n\t\t// to reuse instead of having to allocate a new one.\n\t\t// Previously we would have removed multiple entries but now we will only\n\t\t// prune the minimum number required to maintain the cache size, so that\n\t\t// we reduce the amount of GC pressure and maintain cache stability as best\n\t\t// as possible.\n\t\tif len(c.in.pacache) >= maxPerAccountCacheSize {\n\t\t\tfor cacheKey, p := range c.in.pacache {\n\t\t\t\tdelete(c.in.pacache, cacheKey)\n\t\t\t\tpac = p\n\t\t\t\tif len(c.in.pacache) < maxPerAccountCacheSize {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// If we can reuse the pac from earlier (i.e. we loaded one but it was an\n\t\t// old generation or we pruned the cache) then do so.\n\t\tif pac == nil {\n\t\t\tpac = &perAccountCache{}\n\t\t}\n\t\tpac.acc = acc\n\t\tpac.results = r\n\t\tpac.genid = atomic.LoadUint64(&sl.genid)\n\n\t\t// Store in our cache,make sure to do so after we prune.\n\t\tc.in.pacache[string(c.pa.pacache)] = pac\n\t}\n\treturn acc, r\n}\n\n// Account will return the associated account for this client.\nfunc (c *client) Account() *Account {\n\tif c == nil {\n\t\treturn nil\n\t}\n\tc.mu.Lock()\n\tacc := c.acc\n\tc.mu.Unlock()\n\treturn acc\n}\n\n// pruneClosedSubFromPerAccountCache remove entries that contain subscriptions\n// that have been closed.\nfunc (c *client) pruneClosedSubFromPerAccountCache() {\n\tfor cacheKey, pac := range c.in.pacache {\n\t\tfor _, sub := range pac.results.psubs {\n\t\t\tif sub.isClosed() {\n\t\t\t\tgoto REMOVE\n\t\t\t}\n\t\t}\n\t\tfor _, qsub := range pac.results.qsubs {\n\t\t\tfor _, sub := range qsub {\n\t\t\t\tif sub.isClosed() {\n\t\t\t\t\tgoto REMOVE\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcontinue\n\tREMOVE:\n\t\tdelete(c.in.pacache, cacheKey)\n\t}\n}\n\n// Returns our service account for this request.\nfunc (ci *ClientInfo) serviceAccount() string {\n\tif ci == nil {\n\t\treturn _EMPTY_\n\t}\n\tif ci.Service != _EMPTY_ {\n\t\treturn ci.Service\n\t}\n\treturn ci.Account\n}\n\n// Add in our server and cluster information to this client info.\nfunc (c *client) addServerAndClusterInfo(ci *ClientInfo) {\n\tif ci == nil {\n\t\treturn\n\t}\n\t// Server\n\tif c.kind != LEAF {\n\t\tci.Server = c.srv.Name()\n\t} else if c.kind == LEAF {\n\t\tci.Server = c.leaf.remoteServer\n\t}\n\t// Cluster\n\tci.Cluster = c.srv.cachedClusterName()\n\t// If we have gateways fill in cluster alternates.\n\t// These will be in RTT asc order.\n\tif c.srv.gateway.enabled {\n\t\tvar gws []*client\n\t\tc.srv.getOutboundGatewayConnections(&gws)\n\t\tfor _, c := range gws {\n\t\t\tc.mu.Lock()\n\t\t\tcn := c.gw.name\n\t\t\tc.mu.Unlock()\n\t\t\tci.Alternates = append(ci.Alternates, cn)\n\t\t}\n\t}\n}\n\n// Grabs the information for this client.\nfunc (c *client) getClientInfo(detailed bool) *ClientInfo {\n\tif c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {\n\t\treturn nil\n\t}\n\n\t// Result\n\tvar ci ClientInfo\n\n\tif detailed {\n\t\tc.addServerAndClusterInfo(&ci)\n\t}\n\n\tc.mu.Lock()\n\t// RTT and Account are always added.\n\tci.Account = accForClient(c)\n\tci.RTT = c.rtt\n\t// Detailed signals additional opt in.\n\tif detailed {\n\t\tci.Start = &c.start\n\t\tci.Host = c.host\n\t\tci.ID = c.cid\n\t\tci.Name = c.opts.Name\n\t\tci.User = c.getRawAuthUser()\n\t\tci.Lang = c.opts.Lang\n\t\tci.Version = c.opts.Version\n\t\tci.Jwt = c.opts.JWT\n\t\tci.IssuerKey = issuerForClient(c)\n\t\tci.NameTag = c.nameTag\n\t\tci.Tags = c.tags\n\t\tci.Kind = c.kindString()\n\t\tci.ClientType = c.clientTypeString()\n\t}\n\tc.mu.Unlock()\n\treturn &ci\n}\n\nfunc (c *client) doTLSServerHandshake(typ string, tlsConfig *tls.Config, timeout float64, pCerts PinnedCertSet) error {\n\t_, err := c.doTLSHandshake(typ, false, nil, tlsConfig, _EMPTY_, timeout, pCerts)\n\treturn err\n}\n\nfunc (c *client) doTLSClientHandshake(typ string, url *url.URL, tlsConfig *tls.Config, tlsName string, timeout float64, pCerts PinnedCertSet) (bool, error) {\n\treturn c.doTLSHandshake(typ, true, url, tlsConfig, tlsName, timeout, pCerts)\n}\n\n// Performs either server or client side (if solicit is true) TLS Handshake.\n// On error, the TLS handshake error has been logged and the connection\n// has been closed.\n//\n// Lock is held on entry.\nfunc (c *client) doTLSHandshake(typ string, solicit bool, url *url.URL, tlsConfig *tls.Config, tlsName string, timeout float64, pCerts PinnedCertSet) (bool, error) {\n\tvar host string\n\tvar resetTLSName bool\n\tvar err error\n\n\t// Capture kind for some debug/error statements.\n\tkind := c.kind\n\n\t// If we solicited, we will act like the client, otherwise the server.\n\tif solicit {\n\t\tc.Debugf(\"Starting TLS %s client handshake\", typ)\n\t\tif tlsConfig.ServerName == _EMPTY_ {\n\t\t\t// If the given url is a hostname, use this hostname for the\n\t\t\t// ServerName. If it is an IP, use the cfg's tlsName. If none\n\t\t\t// is available, resort to current IP.\n\t\t\thost = url.Hostname()\n\t\t\tif tlsName != _EMPTY_ && net.ParseIP(host) != nil {\n\t\t\t\thost = tlsName\n\t\t\t}\n\t\t\ttlsConfig.ServerName = host\n\t\t}\n\t\tc.nc = tls.Client(c.nc, tlsConfig)\n\t} else {\n\t\tif kind == CLIENT {\n\t\t\tc.Debugf(\"Starting TLS client connection handshake\")\n\t\t} else {\n\t\t\tc.Debugf(\"Starting TLS %s server handshake\", typ)\n\t\t}\n\t\tc.nc = tls.Server(c.nc, tlsConfig)\n\t}\n\n\tconn := c.nc.(*tls.Conn)\n\n\t// Setup the timeout\n\tttl := secondsToDuration(timeout)\n\tc.tlsTo = time.AfterFunc(ttl, func() { tlsTimeout(c, conn) })\n\tconn.SetReadDeadline(time.Now().Add(ttl))\n\n\tc.mu.Unlock()\n\tif err = conn.Handshake(); err != nil {\n\t\tif solicit {\n\t\t\t// Based on type of error, possibly clear the saved tlsName\n\t\t\t// See: https://github.com/nats-io/nats-server/issues/1256\n\t\t\t// NOTE: As of Go 1.20, the HostnameError is wrapped so cannot\n\t\t\t// type assert to check directly.\n\t\t\tvar hostnameErr x509.HostnameError\n\t\t\tif errors.As(err, &hostnameErr) {\n\t\t\t\tif host == tlsName {\n\t\t\t\t\tresetTLSName = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else if !c.matchesPinnedCert(pCerts) {\n\t\terr = ErrCertNotPinned\n\t}\n\n\tif err != nil {\n\t\tvar detail string\n\t\tvar subjs []string\n\t\tif ve, ok := err.(*tls.CertificateVerificationError); ok {\n\t\t\tfor _, cert := range ve.UnverifiedCertificates {\n\t\t\t\tfp := sha256.Sum256(cert.Raw)\n\t\t\t\tfph := hex.EncodeToString(fp[:])\n\t\t\t\tsubjs = append(subjs, fmt.Sprintf(\"%s SHA-256: %s\", cert.Subject.String(), fph))\n\t\t\t}\n\t\t}\n\t\tif len(subjs) > 0 {\n\t\t\tdetail = fmt.Sprintf(\" (%s)\", strings.Join(subjs, \"; \"))\n\t\t}\n\t\tif kind == CLIENT {\n\t\t\tc.Errorf(\"TLS handshake error: %v%s\", err, detail)\n\t\t} else {\n\t\t\tc.Errorf(\"TLS %s handshake error: %v%s\", typ, err, detail)\n\t\t}\n\t\tc.closeConnection(TLSHandshakeError)\n\n\t\t// Grab the lock before returning since the caller was holding the lock on entry\n\t\tc.mu.Lock()\n\t\t// Returning any error is fine. Since the connection is closed ErrConnectionClosed\n\t\t// is appropriate.\n\t\treturn resetTLSName, ErrConnectionClosed\n\t}\n\n\t// Reset the read deadline\n\tconn.SetReadDeadline(time.Time{})\n\n\t// Re-Grab lock\n\tc.mu.Lock()\n\n\t// To be consistent with client, set this flag to indicate that handshake is done\n\tc.flags.set(handshakeComplete)\n\n\t// The connection still may have been closed on success handshake due\n\t// to a race with tls timeout. If that the case, return error indicating\n\t// that the connection is closed.\n\tif c.isClosed() {\n\t\terr = ErrConnectionClosed\n\t}\n\n\treturn false, err\n}\n\n// getRawAuthUserLock returns the raw auth user for the client.\n// Will acquire the client lock.\nfunc (c *client) getRawAuthUserLock() string {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.getRawAuthUser()\n}\n\n// getRawAuthUser returns the raw auth user for the client.\n// Lock should be held.\nfunc (c *client) getRawAuthUser() string {\n\tswitch {\n\tcase c.opts.Nkey != _EMPTY_:\n\t\treturn c.opts.Nkey\n\tcase c.opts.Username != _EMPTY_:\n\t\treturn c.opts.Username\n\tcase c.opts.JWT != _EMPTY_:\n\t\treturn c.pubKey\n\tcase c.opts.Token != _EMPTY_:\n\t\treturn \"[REDACTED]\"\n\tdefault:\n\t\treturn _EMPTY_\n\t}\n}\n\n// getAuthUser returns the auth user for the client.\n// Lock should be held.\nfunc (c *client) getAuthUser() string {\n\tswitch {\n\tcase c.opts.Nkey != _EMPTY_:\n\t\treturn fmt.Sprintf(\"Nkey %q\", c.opts.Nkey)\n\tcase c.opts.Username != _EMPTY_:\n\t\treturn fmt.Sprintf(\"User %q\", c.opts.Username)\n\tcase c.opts.JWT != _EMPTY_:\n\t\treturn fmt.Sprintf(\"JWT User %q\", c.pubKey)\n\tcase c.opts.Token != _EMPTY_:\n\t\treturn fmt.Sprintf(\"Token %q\", \"[REDACTED]\")\n\tdefault:\n\t\treturn `User \"N/A\"`\n\t}\n}\n\n// getAuthUserLabel returns a label for the auth user for the client.\nfunc (c *client) getAuthUserLabel() string {\n\tswitch {\n\tcase c.opts.Nkey != _EMPTY_:\n\t\treturn fmt.Sprintf(\"nkey:%s\", c.opts.Nkey)\n\tcase c.opts.Username != _EMPTY_:\n\t\treturn fmt.Sprintf(\"user:%s\", c.opts.Username)\n\tcase c.opts.JWT != _EMPTY_:\n\t\treturn fmt.Sprintf(\"jwt:%s\", c.pubKey)\n\tcase c.opts.Token != _EMPTY_:\n\t\treturn \"token\"\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\n// Given an array of strings, this function converts it to a map as long\n// as all the content (converted to upper-case) matches some constants.\n\n// Converts the given array of strings to a map of string.\n// The strings are converted to upper-case and added to the map only\n// if the server recognize them as valid connection types.\n// If there are unknown connection types, the map of valid ones is returned\n// along with an error that contains the name of the unknown.\nfunc convertAllowedConnectionTypes(cts []string) (map[string]struct{}, error) {\n\tvar unknown []string\n\tm := make(map[string]struct{}, len(cts))\n\tfor _, i := range cts {\n\t\ti = strings.ToUpper(i)\n\t\tswitch i {\n\t\tcase jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket,\n\t\t\tjwt.ConnectionTypeLeafnode, jwt.ConnectionTypeLeafnodeWS,\n\t\t\tjwt.ConnectionTypeMqtt, jwt.ConnectionTypeMqttWS,\n\t\t\tjwt.ConnectionTypeInProcess:\n\t\t\tm[i] = struct{}{}\n\t\tdefault:\n\t\t\tunknown = append(unknown, i)\n\t\t}\n\t}\n\tvar err error\n\t// We will still return the map of valid ones.\n\tif len(unknown) != 0 {\n\t\terr = fmt.Errorf(\"invalid connection types %q\", unknown)\n\t}\n\treturn m, err\n}\n\n// This will return true if the connection is of a type present in the given `acts` map.\n// Note that so far this is used only for CLIENT or LEAF connections.\n// But a CLIENT can be standard or websocket (and other types in the future).\nfunc (c *client) connectionTypeAllowed(acts map[string]struct{}) bool {\n\t// Empty means all type of clients are allowed\n\tif len(acts) == 0 {\n\t\treturn true\n\t}\n\tvar want string\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tswitch c.clientType() {\n\t\tcase NATS:\n\t\t\tif c.iproc {\n\t\t\t\twant = jwt.ConnectionTypeInProcess\n\t\t\t} else {\n\t\t\t\twant = jwt.ConnectionTypeStandard\n\t\t\t}\n\t\tcase WS:\n\t\t\twant = jwt.ConnectionTypeWebsocket\n\t\tcase MQTT:\n\t\t\tif c.isWebsocket() {\n\t\t\t\twant = jwt.ConnectionTypeMqttWS\n\t\t\t} else {\n\t\t\t\twant = jwt.ConnectionTypeMqtt\n\t\t\t}\n\t\t}\n\tcase LEAF:\n\t\tif c.isWebsocket() {\n\t\t\twant = jwt.ConnectionTypeLeafnodeWS\n\t\t} else {\n\t\t\twant = jwt.ConnectionTypeLeafnode\n\t\t}\n\t}\n\t_, ok := acts[want]\n\treturn ok\n}\n\n// isClosed returns true if either closeConnection or connMarkedClosed\n// flag have been set, or if `nc` is nil, which may happen in tests.\nfunc (c *client) isClosed() bool {\n\treturn c.flags.isSet(closeConnection) || c.flags.isSet(connMarkedClosed) || c.nc == nil\n}\n\nfunc (c *client) format(format string) string {\n\tif s := c.String(); s != _EMPTY_ {\n\t\treturn fmt.Sprintf(\"%s - %s\", s, format)\n\t} else {\n\t\treturn format\n\t}\n}\n\nfunc (c *client) formatNoClientInfo(format string) string {\n\tacc := c.ncsAcc.Load()\n\tif acc != nil {\n\t\treturn fmt.Sprintf(\"%s - Account:%s\", format, acc)\n\t} else {\n\t\treturn format\n\t}\n}\n\nfunc (c *client) formatClientSuffix() string {\n\tuser := c.ncsUser.Load()\n\tif user == nil || user.(string) == _EMPTY_ {\n\t\treturn _EMPTY_\n\t}\n\treturn fmt.Sprintf(\" - %s\", user)\n}\n\n// Logging functionality scoped to a client or route.\nfunc (c *client) Error(err error) {\n\tc.srv.Errorf(c.format(err.Error()))\n}\n\nfunc (c *client) Errorf(format string, v ...any) {\n\tc.srv.Errorf(c.format(format), v...)\n}\n\nfunc (c *client) Debugf(format string, v ...any) {\n\tc.srv.Debugf(c.format(format), v...)\n}\n\nfunc (c *client) Noticef(format string, v ...any) {\n\tc.srv.Noticef(c.format(format), v...)\n}\n\nfunc (c *client) Tracef(format string, v ...any) {\n\tc.srv.Tracef(c.format(format), v...)\n}\n\nfunc (c *client) Warnf(format string, v ...any) {\n\tc.srv.Warnf(c.format(format), v...)\n}\n\nfunc (c *client) RateLimitErrorf(format string, v ...any) {\n\t// Do the check before adding the client info to the format...\n\tstatement := fmt.Sprintf(c.formatNoClientInfo(format), v...)\n\tif _, loaded := c.srv.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\tif s := c.String(); s != _EMPTY_ {\n\t\tc.srv.Errorf(\"%s - %s%s\", c, statement, c.formatClientSuffix())\n\t} else {\n\t\tc.srv.Errorf(\"%s%s\", statement, c.formatClientSuffix())\n\t}\n}\n\nfunc (c *client) rateLimitFormatWarnf(format string, v ...any) {\n\t// Do the check before adding the client info to the format...\n\tformat = c.formatNoClientInfo(format)\n\tif _, loaded := c.srv.rateLimitLogging.LoadOrStore(format, time.Now()); loaded {\n\t\treturn\n\t}\n\tstatement := fmt.Sprintf(format, v...)\n\tif s := c.String(); s != _EMPTY_ {\n\t\tc.srv.Warnf(\"%s - %s%s\", c, statement, c.formatClientSuffix())\n\t} else {\n\t\tc.srv.Warnf(\"%s%s\", statement, c.formatClientSuffix())\n\t}\n}\n\nfunc (c *client) RateLimitWarnf(format string, v ...any) {\n\t// Do the check before adding the client info to the format...\n\tstatement := fmt.Sprintf(c.formatNoClientInfo(format), v...)\n\tif _, loaded := c.srv.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\tif s := c.String(); s != _EMPTY_ {\n\t\tc.srv.Warnf(\"%s - %s%s\", c, statement, c.formatClientSuffix())\n\t} else {\n\t\tc.srv.Warnf(\"%s%s\", statement, c.formatClientSuffix())\n\t}\n}\n\nfunc (c *client) RateLimitDebugf(format string, v ...any) {\n\t// Do the check before adding the client info to the format...\n\tstatement := fmt.Sprintf(c.formatNoClientInfo(format), v...)\n\tif _, loaded := c.srv.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\tif s := c.String(); s != _EMPTY_ {\n\t\tc.srv.Debugf(\"%s - %s%s\", c, statement, c.formatClientSuffix())\n\t} else {\n\t\tc.srv.Debugf(\"%s%s\", statement, c.formatClientSuffix())\n\t}\n}\n\n// Set the very first PING to a lower interval to capture the initial RTT.\n// After that the PING interval will be set to the user defined value.\n// Client lock should be held.\nfunc (c *client) setFirstPingTimer() {\n\ts := c.srv\n\tif s == nil {\n\t\treturn\n\t}\n\topts := s.getOpts()\n\td := opts.PingInterval\n\n\tif c.kind == ROUTER && opts.Cluster.PingInterval > 0 {\n\t\td = opts.Cluster.PingInterval\n\t}\n\tif c.isWebsocket() && opts.Websocket.PingInterval > 0 {\n\t\td = opts.Websocket.PingInterval\n\t}\n\tif !opts.DisableShortFirstPing {\n\t\tif c.kind != CLIENT {\n\t\t\tif d > firstPingInterval {\n\t\t\t\td = firstPingInterval\n\t\t\t}\n\t\t\td = adjustPingInterval(c.kind, d)\n\t\t} else if d > firstClientPingInterval {\n\t\t\td = firstClientPingInterval\n\t\t}\n\t}\n\t// We randomize the first one by an offset up to 20%, e.g. 2m ~= max 24s.\n\taddDelay := rand.Int63n(int64(d / 5))\n\td += time.Duration(addDelay)\n\t// In the case of ROUTER/LEAF and when compression is configured, it is possible\n\t// that this timer was already set, but just to detect a stale connection\n\t// since we have to delay the first PING after compression negotiation\n\t// occurred.\n\tif c.ping.tmr != nil {\n\t\tc.ping.tmr.Stop()\n\t}\n\tc.ping.tmr = time.AfterFunc(d, c.processPingTimer)\n}\n\n// Sets this error as the authentication error. To be used in authViolation()\n// to report an error different of `ErrAuthentication`.\nfunc (c *client) setAuthError(err error) {\n\tc.mu.Lock()\n\tc.authErr = err\n\tc.mu.Unlock()\n}\n\n// Returns the authentication error set in the connection, possibly nil.\nfunc (c *client) getAuthError() error {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.authErr\n}\n" }, { "path": "server/client_proxyproto.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// PROXY protocol v2 constants\nconst (\n\t// Protocol signature (12 bytes)\n\tproxyProtoV2Sig = \"\\x0D\\x0A\\x0D\\x0A\\x00\\x0D\\x0A\\x51\\x55\\x49\\x54\\x0A\"\n\n\t// Version and command byte format: version(4 bits) | command(4 bits)\n\tproxyProtoV2VerMask = 0xF0\n\tproxyProtoV2Ver = 0x20 // Version 2\n\n\t// Commands\n\tproxyProtoCmdMask = 0x0F\n\tproxyProtoCmdLocal = 0x00 // LOCAL command (health check, use original connection)\n\tproxyProtoCmdProxy = 0x01 // PROXY command (proxied connection)\n\n\t// Address family and protocol byte format: family(4 bits) | protocol(4 bits)\n\tproxyProtoFamilyMask = 0xF0\n\tproxyProtoFamilyUnspec = 0x00 // Unspecified\n\tproxyProtoFamilyInet = 0x10 // IPv4\n\tproxyProtoFamilyInet6 = 0x20 // IPv6\n\tproxyProtoFamilyUnix = 0x30 // Unix socket\n\tproxyProtoProtoMask = 0x0F\n\tproxyProtoProtoUnspec = 0x00 // Unspecified\n\tproxyProtoProtoStream = 0x01 // TCP/STREAM\n\tproxyProtoProtoDatagram = 0x02 // UDP/DGRAM\n\n\t// Address sizes\n\tproxyProtoAddrSizeIPv4 = 12 // 4 (src IP) + 4 (dst IP) + 2 (src port) + 2 (dst port)\n\tproxyProtoAddrSizeIPv6 = 36 // 16 (src IP) + 16 (dst IP) + 2 (src port) + 2 (dst port)\n\n\t// Header sizes\n\tproxyProtoV2HeaderSize = 16 // Fixed header: 12 (sig) + 1 (ver/cmd) + 1 (fam/proto) + 2 (addr len)\n\n\t// Timeout for reading PROXY protocol header\n\tproxyProtoReadTimeout = 5 * time.Second\n)\n\n// PROXY protocol v1 constants\nconst (\n\tproxyProtoV1Prefix = \"PROXY \"\n\tproxyProtoV1MaxLineLen = 107 // Maximum line length including CRLF\n\tproxyProtoV1TCP4 = \"TCP4\"\n\tproxyProtoV1TCP6 = \"TCP6\"\n\tproxyProtoV1Unknown = \"UNKNOWN\"\n)\n\nvar (\n\t// Errors\n\terrProxyProtoInvalid = errors.New(\"invalid PROXY protocol header\")\n\terrProxyProtoUnsupported = errors.New(\"unsupported PROXY protocol feature\")\n\terrProxyProtoTimeout = errors.New(\"timeout reading PROXY protocol header\")\n\terrProxyProtoUnrecognized = errors.New(\"unrecognized PROXY protocol format\")\n)\n\n// proxyProtoAddr contains the address information extracted from PROXY protocol header\ntype proxyProtoAddr struct {\n\tsrcIP net.IP\n\tsrcPort uint16\n\tdstIP net.IP\n\tdstPort uint16\n}\n\n// String implements net.Addr interface\nfunc (p *proxyProtoAddr) String() string {\n\treturn net.JoinHostPort(p.srcIP.String(), fmt.Sprintf(\"%d\", p.srcPort))\n}\n\n// Network implements net.Addr interface\nfunc (p *proxyProtoAddr) Network() string {\n\tif p.srcIP.To4() != nil {\n\t\treturn \"tcp4\"\n\t}\n\treturn \"tcp6\"\n}\n\n// proxyConn wraps a net.Conn to override RemoteAddr() with the address\n// extracted from the PROXY protocol header\ntype proxyConn struct {\n\tnet.Conn\n\tremoteAddr net.Addr\n}\n\n// RemoteAddr returns the original client address extracted from PROXY protocol\nfunc (pc *proxyConn) RemoteAddr() net.Addr {\n\treturn pc.remoteAddr\n}\n\n// detectProxyProtoVersion reads the first bytes and determines protocol version.\n// Returns 1 for v1, 2 for v2, or error.\n// The first 6 bytes read are returned so they can be used by the parser.\nfunc detectProxyProtoVersion(conn net.Conn) (version int, header []byte, err error) {\n\t// Read first 6 bytes to check for \"PROXY \" or v2 signature\n\theader = make([]byte, 6)\n\tif _, err = io.ReadFull(conn, header); err != nil {\n\t\treturn 0, nil, fmt.Errorf(\"failed to read protocol version: %w\", err)\n\t}\n\tswitch bytesToString(header) {\n\tcase proxyProtoV1Prefix:\n\t\treturn 1, header, nil\n\tcase proxyProtoV2Sig[:6]:\n\t\treturn 2, header, nil\n\tdefault:\n\t\treturn 0, nil, errProxyProtoUnrecognized\n\t}\n}\n\n// readProxyProtoV1Header parses PROXY protocol v1 text format.\n// Expects the \"PROXY \" prefix (6 bytes) to have already been consumed.\n// Returns any bytes that were read past the trailing CRLF so the caller can\n// replay them into the next protocol layer.\nfunc readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, []byte, error) {\n\t// Read rest of line (max 107 bytes total, already read 6)\n\tmaxRemaining := proxyProtoV1MaxLineLen - 6\n\n\t// Read up to maxRemaining bytes at once (more efficient than byte-by-byte)\n\tbuf := make([]byte, maxRemaining)\n\tvar line []byte\n\tvar remaining []byte\n\n\tfor len(line) < maxRemaining {\n\t\t// Read available data\n\t\tn, err := conn.Read(buf[len(line):])\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"failed to read v1 line: %w\", err)\n\t\t}\n\n\t\tline = buf[:len(line)+n]\n\n\t\t// Look for CRLF in what we've read so far\n\t\tfor i := 0; i < len(line)-1; i++ {\n\t\t\tif line[i] == '\\r' && line[i+1] == '\\n' {\n\t\t\t\t// Found CRLF - keep any over-read bytes for the client parser.\n\t\t\t\tremaining = append(remaining, line[i+2:]...)\n\t\t\t\tline = line[:i]\n\t\t\t\tgoto foundCRLF\n\t\t\t}\n\t\t}\n\t}\n\n\t// Exceeded max length without finding CRLF\n\treturn nil, nil, fmt.Errorf(\"%w: v1 line too long\", errProxyProtoInvalid)\n\nfoundCRLF:\n\t// Get parts from the protocol\n\tparts := strings.Fields(string(line))\n\n\t// Validate format\n\tif len(parts) < 1 {\n\t\treturn nil, nil, fmt.Errorf(\"%w: invalid v1 format\", errProxyProtoInvalid)\n\t}\n\n\t// Handle UNKNOWN (health check, like v2 LOCAL)\n\tif parts[0] == proxyProtoV1Unknown {\n\t\treturn nil, remaining, nil\n\t}\n\n\t// Must have exactly 5 parts: protocol, src-ip, dst-ip, src-port, dst-port\n\tif len(parts) != 5 {\n\t\treturn nil, nil, fmt.Errorf(\"%w: invalid v1 format\", errProxyProtoInvalid)\n\t}\n\n\tprotocol := parts[0]\n\tsrcIP := net.ParseIP(parts[1])\n\tdstIP := net.ParseIP(parts[2])\n\n\tif srcIP == nil || dstIP == nil {\n\t\treturn nil, nil, fmt.Errorf(\"%w: invalid address\", errProxyProtoInvalid)\n\t}\n\n\t// Parse ports\n\tsrcPort, err := strconv.ParseUint(parts[3], 10, 16)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"invalid source port: %w\", err)\n\t}\n\n\tdstPort, err := strconv.ParseUint(parts[4], 10, 16)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"invalid dest port: %w\", err)\n\t}\n\n\t// Validate protocol matches IP version\n\tif protocol == proxyProtoV1TCP4 && srcIP.To4() == nil {\n\t\treturn nil, nil, fmt.Errorf(\"%w: TCP4 with IPv6 address\", errProxyProtoInvalid)\n\t}\n\tif protocol == proxyProtoV1TCP6 && srcIP.To4() != nil {\n\t\treturn nil, nil, fmt.Errorf(\"%w: TCP6 with IPv4 address\", errProxyProtoInvalid)\n\t}\n\tif protocol != proxyProtoV1TCP4 && protocol != proxyProtoV1TCP6 {\n\t\treturn nil, nil, fmt.Errorf(\"%w: invalid protocol %s\", errProxyProtoInvalid, protocol)\n\t}\n\n\treturn &proxyProtoAddr{\n\t\tsrcIP: srcIP,\n\t\tsrcPort: uint16(srcPort),\n\t\tdstIP: dstIP,\n\t\tdstPort: uint16(dstPort),\n\t}, remaining, nil\n}\n\n// readProxyProtoHeader reads and parses PROXY protocol (v1 or v2) from the connection.\n// Automatically detects version and routes to appropriate parser.\n// If the command is LOCAL/UNKNOWN (health check), it returns nil for addr and no error.\n// If the command is PROXY, it returns the parsed address information.\n// It also returns any bytes that were read past the v1 header terminator so the\n// caller can replay them into the normal client parser.\n// The connection must be fresh (no data read yet).\nfunc readProxyProtoHeader(conn net.Conn) (*proxyProtoAddr, []byte, error) {\n\t// Set read deadline to prevent hanging on slow/malicious clients\n\tif err := conn.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout)); err != nil {\n\t\treturn nil, nil, err\n\t}\n\tdefer conn.SetReadDeadline(time.Time{})\n\n\t// Detect version\n\tversion, firstBytes, err := detectProxyProtoVersion(conn)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tswitch version {\n\tcase 1:\n\t\t// v1 parser expects \"PROXY \" prefix already consumed\n\t\treturn readProxyProtoV1Header(conn)\n\tcase 2:\n\t\t// Read rest of v2 signature (bytes 6-11, total 6 more bytes)\n\t\tremaining := make([]byte, 6)\n\t\tif _, err := io.ReadFull(conn, remaining); err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"failed to read v2 signature: %w\", err)\n\t\t}\n\n\t\t// Verify full signature\n\t\tfullSig := string(firstBytes) + string(remaining)\n\t\tif fullSig != proxyProtoV2Sig {\n\t\t\treturn nil, nil, fmt.Errorf(\"%w: invalid signature\", errProxyProtoInvalid)\n\t\t}\n\n\t\t// Read rest of header: ver/cmd, fam/proto, addr-len (4 bytes)\n\t\theader := make([]byte, 4)\n\t\tif _, err := io.ReadFull(conn, header); err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"failed to read v2 header: %w\", err)\n\t\t}\n\n\t\t// Continue with parsing\n\t\taddr, err := parseProxyProtoV2Header(conn, header)\n\t\treturn addr, nil, err\n\tdefault:\n\t\treturn nil, nil, fmt.Errorf(\"unsupported PROXY protocol version: %d\", version)\n\t}\n}\n\n// readProxyProtoV2Header is kept for backward compatibility and direct testing.\n// It reads and parses a PROXY protocol v2 header from the connection.\n// If the command is LOCAL (health check), it returns nil for addr and no error.\n// If the command is PROXY, it returns the parsed address information.\n// The connection must be fresh (no data read yet).\nfunc readProxyProtoV2Header(conn net.Conn) (*proxyProtoAddr, error) {\n\t// Set read deadline to prevent hanging on slow/malicious clients\n\tif err := conn.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout)); err != nil {\n\t\treturn nil, err\n\t}\n\tdefer conn.SetReadDeadline(time.Time{})\n\n\t// Read fixed header (16 bytes)\n\theader := make([]byte, proxyProtoV2HeaderSize)\n\tif _, err := io.ReadFull(conn, header); err != nil {\n\t\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\t\treturn nil, errProxyProtoTimeout\n\t\t}\n\t\treturn nil, fmt.Errorf(\"failed to read PROXY protocol header: %w\", err)\n\t}\n\n\t// Validate signature (first 12 bytes)\n\tif string(header[:12]) != proxyProtoV2Sig {\n\t\treturn nil, fmt.Errorf(\"%w: invalid signature\", errProxyProtoInvalid)\n\t}\n\n\t// Continue with parsing after signature\n\treturn parseProxyProtoV2Header(conn, header[12:16])\n}\n\n// parseProxyProtoV2Header parses v2 protocol after signature has been validated.\n// header contains the 4 bytes: ver/cmd, fam/proto, addr-len (2 bytes).\nfunc parseProxyProtoV2Header(conn net.Conn, header []byte) (*proxyProtoAddr, error) {\n\t// Parse version and command\n\tverCmd := header[0]\n\tversion := verCmd & proxyProtoV2VerMask\n\tcommand := verCmd & proxyProtoCmdMask\n\n\tif version != proxyProtoV2Ver {\n\t\treturn nil, fmt.Errorf(\"%w: invalid version 0x%02x\", errProxyProtoInvalid, version)\n\t}\n\n\t// Parse address family and protocol\n\tfamProto := header[1]\n\tfamily := famProto & proxyProtoFamilyMask\n\tprotocol := famProto & proxyProtoProtoMask\n\n\t// Parse address length (big-endian uint16)\n\taddrLen := binary.BigEndian.Uint16(header[2:4])\n\n\t// Handle LOCAL command (health check)\n\tif command == proxyProtoCmdLocal {\n\t\t// For LOCAL, we should skip the address data if any\n\t\tif addrLen > 0 {\n\t\t\t// Discard the address data\n\t\t\tif _, err := io.CopyN(io.Discard, conn, int64(addrLen)); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to discard LOCAL command address data: %w\", err)\n\t\t\t}\n\t\t}\n\t\treturn nil, nil // nil addr indicates LOCAL command\n\t}\n\n\t// Handle PROXY command\n\tif command != proxyProtoCmdProxy {\n\t\treturn nil, fmt.Errorf(\"unknown PROXY protocol command: 0x%02x\", command)\n\t}\n\n\t// Validate protocol (we only support STREAM/TCP)\n\tif protocol != proxyProtoProtoStream {\n\t\treturn nil, fmt.Errorf(\"%w: only STREAM protocol supported\", errProxyProtoUnsupported)\n\t}\n\n\t// Parse address data based on family\n\tvar addr *proxyProtoAddr\n\tvar err error\n\tswitch family {\n\tcase proxyProtoFamilyInet:\n\t\taddr, err = parseIPv4Addr(conn, addrLen)\n\tcase proxyProtoFamilyInet6:\n\t\taddr, err = parseIPv6Addr(conn, addrLen)\n\tcase proxyProtoFamilyUnspec:\n\t\t// UNSPEC family with PROXY command is valid but rare\n\t\t// Just skip the address data\n\t\tif addrLen > 0 {\n\t\t\tif _, err := io.CopyN(io.Discard, conn, int64(addrLen)); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to discard UNSPEC address address data: %w\", err)\n\t\t\t}\n\t\t}\n\t\treturn nil, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"%w: unsupported address family 0x%02x\", errProxyProtoUnsupported, family)\n\t}\n\treturn addr, err\n}\n\n// parseIPv4Addr parses IPv4 address data from PROXY protocol header\nfunc parseIPv4Addr(conn net.Conn, addrLen uint16) (*proxyProtoAddr, error) {\n\t// IPv4: 4 (src IP) + 4 (dst IP) + 2 (src port) + 2 (dst port) = 12 bytes minimum\n\tif addrLen < proxyProtoAddrSizeIPv4 {\n\t\treturn nil, fmt.Errorf(\"IPv4 address data too short: %d bytes\", addrLen)\n\t}\n\taddrData := make([]byte, addrLen)\n\tif _, err := io.ReadFull(conn, addrData); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to read IPv4 address data: %w\", err)\n\t}\n\treturn &proxyProtoAddr{\n\t\tsrcIP: net.IP(addrData[0:4]),\n\t\tdstIP: net.IP(addrData[4:8]),\n\t\tsrcPort: binary.BigEndian.Uint16(addrData[8:10]),\n\t\tdstPort: binary.BigEndian.Uint16(addrData[10:12]),\n\t}, nil\n}\n\n// parseIPv6Addr parses IPv6 address data from PROXY protocol header\nfunc parseIPv6Addr(conn net.Conn, addrLen uint16) (*proxyProtoAddr, error) {\n\t// IPv6: 16 (src IP) + 16 (dst IP) + 2 (src port) + 2 (dst port) = 36 bytes minimum\n\tif addrLen < proxyProtoAddrSizeIPv6 {\n\t\treturn nil, fmt.Errorf(\"IPv6 address data too short: %d bytes\", addrLen)\n\t}\n\taddrData := make([]byte, addrLen)\n\tif _, err := io.ReadFull(conn, addrData); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to read IPv6 address data: %w\", err)\n\t}\n\treturn &proxyProtoAddr{\n\t\tsrcIP: net.IP(addrData[0:16]),\n\t\tdstIP: net.IP(addrData[16:32]),\n\t\tsrcPort: binary.BigEndian.Uint16(addrData[32:34]),\n\t\tdstPort: binary.BigEndian.Uint16(addrData[34:36]),\n\t}, nil\n}\n" }, { "path": "server/client_proxyproto_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\n// mockConn is a mock net.Conn for testing\ntype mockConn struct {\n\tnet.Conn\n\treadBuf *bytes.Buffer\n\twriteBuf *bytes.Buffer\n\tclosed bool\n\tdeadline time.Time\n}\n\nfunc newMockConn(data []byte) *mockConn {\n\treturn &mockConn{\n\t\treadBuf: bytes.NewBuffer(data),\n\t\twriteBuf: &bytes.Buffer{},\n\t}\n}\n\nfunc (m *mockConn) Read(b []byte) (int, error) {\n\tif m.closed {\n\t\treturn 0, fmt.Errorf(\"connection closed\")\n\t}\n\tif !m.deadline.IsZero() && time.Now().After(m.deadline) {\n\t\treturn 0, &net.OpError{Op: \"read\", Err: fmt.Errorf(\"timeout\")}\n\t}\n\treturn m.readBuf.Read(b)\n}\n\nfunc (m *mockConn) Write(b []byte) (int, error) {\n\tif m.closed {\n\t\treturn 0, fmt.Errorf(\"connection closed\")\n\t}\n\treturn m.writeBuf.Write(b)\n}\n\nfunc (m *mockConn) Close() error {\n\tm.closed = true\n\treturn nil\n}\n\nfunc (m *mockConn) LocalAddr() net.Addr {\n\treturn &net.TCPAddr{IP: net.ParseIP(\"127.0.0.1\"), Port: 4222}\n}\n\nfunc (m *mockConn) RemoteAddr() net.Addr {\n\treturn &net.TCPAddr{IP: net.ParseIP(\"192.168.1.100\"), Port: 54321}\n}\n\nfunc (m *mockConn) SetDeadline(t time.Time) error {\n\tm.deadline = t\n\treturn nil\n}\n\nfunc (m *mockConn) SetReadDeadline(t time.Time) error {\n\tm.deadline = t\n\treturn nil\n}\n\nfunc (m *mockConn) SetWriteDeadline(t time.Time) error {\n\treturn nil\n}\n\n// buildProxyV2Header builds a valid PROXY protocol v2 header\nfunc buildProxyV2Header(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, family byte) []byte {\n\tt.Helper()\n\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig) // Write signature\n\tbuf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy) // Write version and command (version 2, PROXY command)\n\tbuf.WriteByte(family | proxyProtoProtoStream) // Write family and protocol\n\n\tsrc := net.ParseIP(srcIP)\n\tdst := net.ParseIP(dstIP)\n\tvar addrData []byte\n\tswitch family {\n\tcase proxyProtoFamilyInet:\n\t\t// IPv4: 12 bytes\n\t\taddrData = make([]byte, proxyProtoAddrSizeIPv4)\n\t\tcopy(addrData[0:4], src.To4())\n\t\tcopy(addrData[4:8], dst.To4())\n\t\tbinary.BigEndian.PutUint16(addrData[8:10], srcPort)\n\t\tbinary.BigEndian.PutUint16(addrData[10:12], dstPort)\n\tcase proxyProtoFamilyInet6:\n\t\t// IPv6: 36 bytes\n\t\taddrData = make([]byte, proxyProtoAddrSizeIPv6)\n\t\tcopy(addrData[0:16], src.To16())\n\t\tcopy(addrData[16:32], dst.To16())\n\t\tbinary.BigEndian.PutUint16(addrData[32:34], srcPort)\n\t\tbinary.BigEndian.PutUint16(addrData[34:36], dstPort)\n\tdefault:\n\t\tt.Fatalf(\"unsupported address family: %d\", family)\n\t}\n\n\taddrLen := make([]byte, 2)\n\tbinary.BigEndian.PutUint16(addrLen, uint16(len(addrData)))\n\tbuf.Write(addrLen)\n\tbuf.Write(addrData)\n\n\treturn buf.Bytes()\n}\n\n// buildProxyV2LocalHeader builds a PROXY protocol v2 LOCAL command header\nfunc buildProxyV2LocalHeader() []byte {\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig) // // Write signature\n\tbuf.WriteByte(proxyProtoV2Ver | proxyProtoCmdLocal) // // Write version and command (version 2, LOCAL command)\n\tbuf.WriteByte(proxyProtoFamilyUnspec | proxyProtoProtoUnspec) // // Write family and protocol (UNSPEC)\n\tbuf.WriteByte(0)\n\tbuf.WriteByte(0)\n\treturn buf.Bytes()\n}\n\n// buildProxyV1Header builds a valid PROXY protocol v1 header (text format)\nfunc buildProxyV1Header(t *testing.T, protocol, srcIP, dstIP string, srcPort, dstPort uint16) []byte {\n\tt.Helper()\n\n\tif protocol != \"TCP4\" && protocol != \"TCP6\" && protocol != \"UNKNOWN\" {\n\t\tt.Fatalf(\"invalid protocol: %s\", protocol)\n\t}\n\n\tvar line string\n\tif protocol == \"UNKNOWN\" {\n\t\tline = \"PROXY UNKNOWN\\r\\n\"\n\t} else {\n\t\tline = fmt.Sprintf(\"PROXY %s %s %s %d %d\\r\\n\", protocol, srcIP, dstIP, srcPort, dstPort)\n\t}\n\n\treturn []byte(line)\n}\n\nfunc TestClientProxyProtoV2ParseIPv4(t *testing.T) {\n\theader := buildProxyV2Header(t, \"192.168.1.50\", \"10.0.0.1\", 12345, 4222, proxyProtoFamilyInet)\n\tconn := newMockConn(header)\n\n\taddr, err := readProxyProtoV2Header(conn)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr)\n\n\trequire_Equal(t, addr.srcIP.String(), \"192.168.1.50\")\n\trequire_Equal(t, addr.srcPort, 12345)\n\n\trequire_Equal(t, addr.dstIP.String(), \"10.0.0.1\")\n\trequire_Equal(t, addr.dstPort, 4222)\n\n\t// Test String() and Network() methods\n\trequire_Equal(t, addr.String(), \"192.168.1.50:12345\")\n\trequire_Equal(t, addr.Network(), \"tcp4\")\n}\n\nfunc TestClientProxyProtoV2ParseIPv6(t *testing.T) {\n\theader := buildProxyV2Header(t, \"2001:db8::1\", \"2001:db8::2\", 54321, 4222, proxyProtoFamilyInet6)\n\tconn := newMockConn(header)\n\n\taddr, err := readProxyProtoV2Header(conn)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr)\n\n\trequire_Equal(t, addr.srcIP.String(), \"2001:db8::1\")\n\trequire_Equal(t, addr.srcPort, 54321)\n\n\trequire_Equal(t, addr.dstIP.String(), \"2001:db8::2\")\n\trequire_Equal(t, addr.dstPort, 4222)\n\n\t// Test Network() method for IPv6\n\trequire_Equal(t, addr.String(), \"[2001:db8::1]:54321\")\n\trequire_Equal(t, addr.Network(), \"tcp6\")\n}\n\nfunc TestClientProxyProtoV2ParseLocalCommand(t *testing.T) {\n\theader := buildProxyV2LocalHeader()\n\tconn := newMockConn(header)\n\n\taddr, err := readProxyProtoV2Header(conn)\n\trequire_NoError(t, err)\n\trequire_True(t, addr == nil)\n}\n\nfunc TestClientProxyProtoV2InvalidSignature(t *testing.T) {\n\t// Create invalid signature\n\theader := []byte(\"INVALID_SIG_\")\n\theader = append(header, []byte{0x20, 0x11, 0x00, 0x0C}...)\n\tconn := newMockConn(header)\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV2InvalidVersion(t *testing.T) {\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig)\n\tbuf.WriteByte(0x10 | proxyProtoCmdProxy) // Version 1 instead of 2\n\tbuf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoStream)\n\tbuf.WriteByte(0)\n\tbuf.WriteByte(0)\n\n\tconn := newMockConn(buf.Bytes())\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV2UnsupportedFamily(t *testing.T) {\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig)\n\tbuf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy)\n\tbuf.WriteByte(proxyProtoFamilyUnix | proxyProtoProtoStream) // Unix socket family\n\tbuf.WriteByte(0)\n\tbuf.WriteByte(0)\n\n\tconn := newMockConn(buf.Bytes())\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, errProxyProtoUnsupported)\n}\n\nfunc TestClientProxyProtoV2UnsupportedProtocol(t *testing.T) {\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig)\n\tbuf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy)\n\tbuf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoDatagram) // UDP instead of TCP\n\tbuf.WriteByte(0)\n\tbuf.WriteByte(12)\n\n\tconn := newMockConn(buf.Bytes())\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, errProxyProtoUnsupported)\n}\n\nfunc TestClientProxyProtoV2TruncatedHeader(t *testing.T) {\n\theader := buildProxyV2Header(t, \"192.168.1.50\", \"10.0.0.1\", 12345, 4222, proxyProtoFamilyInet)\n\t// Only send first 10 bytes (incomplete header)\n\tconn := newMockConn(header[:10])\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, io.ErrUnexpectedEOF)\n}\n\nfunc TestClientProxyProtoV2ShortAddressData(t *testing.T) {\n\tbuf := &bytes.Buffer{}\n\tbuf.WriteString(proxyProtoV2Sig)\n\tbuf.WriteByte(proxyProtoV2Ver | proxyProtoCmdProxy)\n\tbuf.WriteByte(proxyProtoFamilyInet | proxyProtoProtoStream)\n\t// Set address length to 12 but don't provide data\n\tbuf.WriteByte(0)\n\tbuf.WriteByte(12)\n\t// Only provide 5 bytes instead of 12\n\tbuf.Write([]byte{1, 2, 3, 4, 5})\n\n\tconn := newMockConn(buf.Bytes())\n\n\t_, err := readProxyProtoV2Header(conn)\n\trequire_Error(t, err, io.ErrUnexpectedEOF)\n}\n\nfunc TestProxyConnRemoteAddr(t *testing.T) {\n\t// Create a real TCP connection for testing\n\toriginalAddr := &net.TCPAddr{IP: net.ParseIP(\"192.168.1.100\"), Port: 54321}\n\n\t// Create proxy address\n\tproxyAddr := &proxyProtoAddr{\n\t\tsrcIP: net.ParseIP(\"10.0.0.50\"),\n\t\tsrcPort: 12345,\n\t\tdstIP: net.ParseIP(\"10.0.0.1\"),\n\t\tdstPort: 4222,\n\t}\n\n\tmockConn := newMockConn(nil)\n\twrapped := &proxyConn{\n\t\tConn: mockConn,\n\t\tremoteAddr: proxyAddr,\n\t}\n\n\t// Verify RemoteAddr returns the proxied address\n\taddr := wrapped.RemoteAddr()\n\trequire_Equal(t, addr.String(), \"10.0.0.50:12345\")\n\trequire_Equal(t, mockConn.RemoteAddr().String(), originalAddr.String())\n}\n\nfunc TestClientProxyProtoV2EndToEnd(t *testing.T) {\n\t// Start a test server with PROXY protocol enabled\n\topts := DefaultOptions()\n\topts.Port = -1 // Random port\n\topts.ProxyProtocol = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Get the server's listening port\n\taddr := s.Addr().String()\n\n\t// Connect to the server\n\tconn, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer conn.Close()\n\n\t// Send PROXY protocol header\n\tclientIP := \"203.0.113.50\"\n\tclientPort := uint16(54321)\n\theader := buildProxyV2Header(t, clientIP, \"127.0.0.1\", clientPort, 4222, proxyProtoFamilyInet)\n\n\t_, err = conn.Write(header)\n\trequire_NoError(t, err)\n\n\t// Send CONNECT message\n\tconnectMsg := \"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"protocol\\\":1}\\r\\n\"\n\t_, err = conn.Write([]byte(connectMsg))\n\trequire_NoError(t, err)\n\n\t// Read INFO and +OK\n\tbuf := make([]byte, 4096)\n\tn, err := conn.Read(buf)\n\trequire_NoError(t, err)\n\n\tresponse := string(buf[:n])\n\trequire_True(t, strings.Contains(response, \"INFO\"))\n\n\t// Give server time to process\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Check server's client list to verify the IP was extracted correctly\n\ts.mu.Lock()\n\tclients := s.clients\n\ts.mu.Unlock()\n\trequire_True(t, len(clients) != 0)\n\n\t// Find our client\n\tvar foundClient *client\n\tfor _, c := range clients {\n\t\tc.mu.Lock()\n\t\tif c.host == clientIP && c.port == clientPort {\n\t\t\tfoundClient = c\n\t\t}\n\t\tc.mu.Unlock()\n\t\tif foundClient != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_NotNil(t, foundClient)\n}\n\nfunc TestClientProxyProtoV2LocalCommandEndToEnd(t *testing.T) {\n\t// Start a test server with PROXY protocol enabled\n\topts := DefaultOptions()\n\topts.Port = -1 // Random port\n\topts.ProxyProtocol = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Get the server's listening port\n\taddr := s.Addr().String()\n\n\t// Connect to the server\n\tconn, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer conn.Close()\n\n\t// Send PROXY protocol LOCAL header (health check)\n\theader := buildProxyV2LocalHeader()\n\n\t_, err = conn.Write(header)\n\trequire_NoError(t, err)\n\n\t// Send CONNECT message\n\tconnectMsg := \"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"protocol\\\":1}\\r\\n\"\n\t_, err = conn.Write([]byte(connectMsg))\n\trequire_NoError(t, err)\n\n\t// Read INFO and +OK\n\tbuf := make([]byte, 4096)\n\tn, err := conn.Read(buf)\n\trequire_NoError(t, err)\n\n\tresponse := string(buf[:n])\n\trequire_True(t, strings.Contains(response, \"INFO\"))\n\n\t// Connection should work normally with LOCAL command\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Verify at least one client is connected\n\ts.mu.Lock()\n\tnumClients := len(s.clients)\n\ts.mu.Unlock()\n\trequire_NotEqual(t, numClients, 0)\n}\n\n// ============================================================================\n// PROXY Protocol v1 Tests\n// ============================================================================\n\nfunc TestClientProxyProtoV1ParseTCP4(t *testing.T) {\n\theader := buildProxyV1Header(t, \"TCP4\", \"192.168.1.50\", \"10.0.0.1\", 12345, 4222)\n\tconn := newMockConn(header)\n\n\taddr, _, err := readProxyProtoHeader(conn)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr)\n\n\trequire_Equal(t, addr.srcIP.String(), \"192.168.1.50\")\n\trequire_Equal(t, addr.srcPort, 12345)\n\n\trequire_Equal(t, addr.dstIP.String(), \"10.0.0.1\")\n\trequire_Equal(t, addr.dstPort, 4222)\n}\n\nfunc TestClientProxyProtoV1ParseTCP6(t *testing.T) {\n\theader := buildProxyV1Header(t, \"TCP6\", \"2001:db8::1\", \"2001:db8::2\", 54321, 4222)\n\tconn := newMockConn(header)\n\n\taddr, _, err := readProxyProtoHeader(conn)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr)\n\n\trequire_Equal(t, addr.srcIP.String(), \"2001:db8::1\")\n\trequire_Equal(t, addr.srcPort, 54321)\n\n\trequire_Equal(t, addr.dstIP.String(), \"2001:db8::2\")\n\trequire_Equal(t, addr.dstPort, 4222)\n}\n\nfunc TestClientProxyProtoV1ParseUnknown(t *testing.T) {\n\theader := buildProxyV1Header(t, \"UNKNOWN\", \"\", \"\", 0, 0)\n\tconn := newMockConn(header)\n\n\taddr, _, err := readProxyProtoHeader(conn)\n\trequire_NoError(t, err)\n\trequire_True(t, addr == nil)\n}\n\nfunc TestClientProxyProtoV1PreservesCoalescedClientBytes(t *testing.T) {\n\theader := buildProxyV1Header(t, \"TCP4\", \"192.168.1.50\", \"10.0.0.1\", 12345, 4222)\n\tconnect := []byte(\"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"protocol\\\":1}\\r\\n\")\n\tconn := newMockConn(append(header, connect...))\n\n\taddr, remaining, err := readProxyProtoHeader(conn)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr)\n\trequire_Equal(t, string(remaining), string(connect))\n\n\trest, err := io.ReadAll(conn)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(rest), 0)\n}\n\nfunc TestClientProxyProtoV1InvalidFormat(t *testing.T) {\n\t// Missing fields\n\theader := []byte(\"PROXY TCP4 192.168.1.1\\r\\n\")\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV1LineTooLong(t *testing.T) {\n\t// Create a line longer than 107 bytes\n\tlongIP := strings.Repeat(\"1234567890\", 12) // 120 chars\n\theader := fmt.Appendf(nil, \"PROXY TCP4 %s 10.0.0.1 12345 443\\r\\n\", longIP)\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV1InvalidIP(t *testing.T) {\n\theader := []byte(\"PROXY TCP4 not.an.ip.addr 10.0.0.1 12345 443\\r\\n\")\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV1MismatchedProtocol(t *testing.T) {\n\t// TCP4 with IPv6 address\n\theader := buildProxyV1Header(t, \"TCP4\", \"2001:db8::1\", \"2001:db8::2\", 12345, 443)\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_Error(t, err, errProxyProtoInvalid)\n\n\t// TCP6 with IPv4 address\n\theader2 := buildProxyV1Header(t, \"TCP6\", \"192.168.1.1\", \"10.0.0.1\", 12345, 443)\n\tconn2 := newMockConn(header2)\n\n\t_, _, err = readProxyProtoHeader(conn2)\n\trequire_Error(t, err, errProxyProtoInvalid)\n}\n\nfunc TestClientProxyProtoV1InvalidPort(t *testing.T) {\n\theader := []byte(\"PROXY TCP4 192.168.1.1 10.0.0.1 99999 443\\r\\n\")\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_True(t, err != nil)\n}\n\nfunc TestClientProxyProtoV1EndToEnd(t *testing.T) {\n\t// Start a test server with PROXY protocol enabled\n\topts := DefaultOptions()\n\topts.Port = -1 // Random port\n\topts.ProxyProtocol = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Get the server's listening port\n\taddr := s.Addr().String()\n\n\t// Connect to the server\n\tconn, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer conn.Close()\n\n\t// Send the PROXY protocol header and first client commands in one write so\n\t// the parser must preserve bytes read past the end of the v1 header.\n\tclientIP, clientPort := \"203.0.113.50\", uint16(54321)\n\theader := buildProxyV1Header(t, \"TCP4\", clientIP, \"127.0.0.1\", clientPort, 4222)\n\tpayload := append(header, []byte(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":false,\\\"protocol\\\":1}\\r\\nPING\\r\\n\")...)\n\n\t_, err = conn.Write(payload)\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, conn.SetReadDeadline(time.Now().Add(2*time.Second)))\n\tcr := bufio.NewReader(conn)\n\n\tline, err := cr.ReadString('\\n')\n\trequire_NoError(t, err)\n\trequire_True(t, strings.HasPrefix(line, \"INFO \"))\n\n\tline, err = cr.ReadString('\\n')\n\trequire_NoError(t, err)\n\trequire_True(t, strings.HasPrefix(line, \"+OK\"))\n\n\texpectPong(t, cr)\n\n\t// Give server time to process\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Check server's client list to verify the IP was extracted correctly\n\ts.mu.Lock()\n\tclients := s.clients\n\tvar foundClient *client\n\tfor _, c := range clients {\n\t\tc.mu.Lock()\n\t\tif c.host == clientIP && c.port == clientPort {\n\t\t\tfoundClient = c\n\t\t}\n\t\tc.mu.Unlock()\n\t\tif foundClient != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\ts.mu.Unlock()\n\trequire_NotNil(t, foundClient)\n}\n\n// ============================================================================\n// Mixed Protocol Version Tests\n// ============================================================================\n\nfunc TestClientProxyProtoVersionDetection(t *testing.T) {\n\t// Test v1 detection\n\tv1Header := buildProxyV1Header(t, \"TCP4\", \"192.168.1.1\", \"10.0.0.1\", 12345, 443)\n\tconn1 := newMockConn(v1Header)\n\n\taddr1, _, err := readProxyProtoHeader(conn1)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr1)\n\trequire_Equal(t, addr1.srcIP.String(), \"192.168.1.1\")\n\n\t// Test v2 detection\n\tv2Header := buildProxyV2Header(t, \"192.168.1.2\", \"10.0.0.1\", 54321, 443, proxyProtoFamilyInet)\n\tconn2 := newMockConn(v2Header)\n\n\taddr2, _, err := readProxyProtoHeader(conn2)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, addr2)\n\trequire_Equal(t, addr2.srcIP.String(), \"192.168.1.2\")\n}\n\nfunc TestClientProxyProtoUnrecognizedVersion(t *testing.T) {\n\t// Invalid header that doesn't match v1 or v2\n\theader := []byte(\"HELLO WORLD\\r\\n\")\n\tconn := newMockConn(header)\n\n\t_, _, err := readProxyProtoHeader(conn)\n\trequire_Error(t, err, errProxyProtoUnrecognized)\n}\n" }, { "path": "server/client_test.go", "content": "// Copyright 2012-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\ntype serverInfo struct {\n\tID string `json:\"server_id\"`\n\tHost string `json:\"host\"`\n\tPort uint `json:\"port\"`\n\tVersion string `json:\"version\"`\n\tAuthRequired bool `json:\"auth_required\"`\n\tTLSRequired bool `json:\"tls_required\"`\n\tMaxPayload int64 `json:\"max_payload\"`\n\tHeaders bool `json:\"headers\"`\n\tConnectURLs []string `json:\"connect_urls,omitempty\"`\n\tLameDuckMode bool `json:\"ldm,omitempty\"`\n\tCID uint64 `json:\"client_id,omitempty\"`\n}\n\ntype testAsyncClient struct {\n\t*client\n\tparseAsync func(string)\n\tquitCh chan bool\n}\n\nfunc (c *testAsyncClient) close() {\n\tc.client.closeConnection(ClientClosed)\n\tc.quitCh <- true\n}\n\nfunc (c *testAsyncClient) parse(proto []byte) error {\n\terr := c.client.parse(proto)\n\tc.client.flushClients(0)\n\treturn err\n}\n\nfunc (c *testAsyncClient) parseAndClose(proto []byte) {\n\tc.client.parse(proto)\n\tc.client.flushClients(0)\n\tc.closeConnection(ClientClosed)\n}\n\nfunc createClientAsync(ch chan *client, s *Server, cli net.Conn) {\n\t// Normally, those type of clients are used against non running servers.\n\t// However, some don't, which would then cause the writeLoop to be\n\t// started twice for the same client (since createClient() start both\n\t// read and write loop if it is detected as running).\n\tstartWriteLoop := !s.isRunning()\n\tif startWriteLoop {\n\t\ts.grWG.Add(1)\n\t}\n\tgo func() {\n\t\tc := s.createClient(cli)\n\t\t// Must be here to suppress +OK\n\t\tc.opts.Verbose = false\n\t\tif startWriteLoop {\n\t\t\tgo c.writeLoop()\n\t\t}\n\t\tch <- c\n\t}()\n}\n\nfunc newClientForServer(s *Server) (*testAsyncClient, *bufio.Reader, string) {\n\tcli, srv := net.Pipe()\n\tcr := bufio.NewReaderSize(cli, maxBufSize)\n\tch := make(chan *client)\n\tcreateClientAsync(ch, s, srv)\n\t// So failing tests don't just hang.\n\tcli.SetReadDeadline(time.Now().Add(10 * time.Second))\n\tl, _ := cr.ReadString('\\n')\n\t// Grab client\n\tc := <-ch\n\tparse, quitCh := genAsyncParser(c)\n\tasyncClient := &testAsyncClient{\n\t\tclient: c,\n\t\tparseAsync: parse,\n\t\tquitCh: quitCh,\n\t}\n\treturn asyncClient, cr, l\n}\n\nfunc genAsyncParser(c *client) (func(string), chan bool) {\n\tpab := make(chan []byte, 16)\n\tpas := func(cs string) { pab <- []byte(cs) }\n\tquit := make(chan bool)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase cs := <-pab:\n\t\t\t\tc.parse(cs)\n\t\t\t\tc.flushClients(0)\n\t\t\tcase <-quit:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\treturn pas, quit\n}\n\nvar defaultServerOptions = Options{\n\tHost: \"127.0.0.1\",\n\tPort: -1,\n\tTrace: true,\n\tDebug: true,\n\tDisableShortFirstPing: true,\n\tNoLog: true,\n\tNoSigs: true,\n}\n\nfunc rawSetup(serverOptions Options) (*Server, *testAsyncClient, *bufio.Reader, string) {\n\ts := New(&serverOptions)\n\tc, cr, l := newClientForServer(s)\n\treturn s, c, cr, l\n}\n\nfunc setUpClientWithResponse() (*testAsyncClient, string) {\n\t_, c, _, l := rawSetup(defaultServerOptions)\n\treturn c, l\n}\n\nfunc setupClient() (*Server, *testAsyncClient, *bufio.Reader) {\n\ts, c, cr, _ := rawSetup(defaultServerOptions)\n\treturn s, c, cr\n}\n\nfunc checkClientsCount(t *testing.T, s *Server, expected int) {\n\tt.Helper()\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tif nc := s.NumClients(); nc != expected {\n\t\t\treturn fmt.Errorf(\"The number of expected connections was %v, got %v\", expected, nc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkAccClientsCount(t *testing.T, acc *Account, expected int) {\n\tt.Helper()\n\tcheckFor(t, 4*time.Second, 10*time.Millisecond, func() error {\n\t\tif nc := acc.NumConnections(); nc != expected {\n\t\t\treturn fmt.Errorf(\"Expected account %q to have %v clients, got %v\",\n\t\t\t\tacc.Name, expected, nc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestAsyncClientWithRunningServer(t *testing.T) {\n\to := DefaultOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tbuf := make([]byte, 1000000)\n\twriteLoopTxt := fmt.Sprintf(\"writeLoop(%p)\", c.client)\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tn := runtime.Stack(buf, true)\n\t\tif count := strings.Count(string(buf[:n]), writeLoopTxt); count != 1 {\n\t\t\treturn fmt.Errorf(\"writeLoop for client should have been started only once: %v\", count)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestClientCreateAndInfo(t *testing.T) {\n\ts, c, _, l := rawSetup(defaultServerOptions)\n\tdefer c.close()\n\n\tif c.cid != 1 {\n\t\tt.Fatalf(\"Expected cid of 1 vs %d\\n\", c.cid)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatal(\"Expected state to be OP_START\")\n\t}\n\n\tif !strings.HasPrefix(l, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", l)\n\t}\n\t// Make sure payload is proper json\n\tvar info serverInfo\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\t// Sanity checks\n\tif info.MaxPayload != MAX_PAYLOAD_SIZE ||\n\t\tinfo.AuthRequired || info.TLSRequired ||\n\t\tint(info.Port) != s.opts.Port {\n\t\tt.Fatalf(\"INFO inconsistent: %+v\\n\", info)\n\t}\n}\n\nfunc TestClientNoResponderSupport(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Force header support if you want to do no_responders. Make sure headers are set.\n\tif err := c.parse([]byte(\"CONNECT {\\\"no_responders\\\":true}\\r\\n\")); err == nil {\n\t\tt.Fatalf(\"Expected error\")\n\t}\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tc.parseAsync(\"CONNECT {\\\"headers\\\":true, \\\"no_responders\\\":true}\\r\\nSUB reply 1\\r\\nPUB foo reply 2\\r\\nok\\r\\n\")\n\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\tam := hmsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a match for %q\", l)\n\t}\n\tcheckPayload(cr, []byte(\"NATS/1.0 503\\r\\nNats-Subject: foo\\r\\n\\r\\n\\r\\n\"), t)\n}\n\nfunc TestServerHeaderSupport(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, _, l := newClientForServer(s)\n\tdefer c.close()\n\n\tif !strings.HasPrefix(l, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", l)\n\t}\n\tvar info serverInfo\n\tif err := json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif !info.Headers {\n\t\tt.Fatalf(\"Expected by default for header support to be enabled\")\n\t}\n\n\topts.NoHeaderSupport = true\n\topts.Port = -1\n\ts = New(&opts)\n\n\tc, _, l = newClientForServer(s)\n\tdefer c.close()\n\n\tif err := json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Headers {\n\t\tt.Fatalf(\"Expected header support to be disabled\")\n\t}\n}\n\n// This test specifically is not testing how headers are encoded in a raw msg.\n// It wants to make sure the serve and clients agreement on when to use headers\n// is bi-directional and functions properly.\nfunc TestClientHeaderSupport(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Even though the server supports headers we need to explicitly say we do in the\n\t// CONNECT. If we do not we should get an error.\n\tif err := c.parse([]byte(\"CONNECT {}\\r\\nHPUB foo 0 2\\r\\nok\\r\\n\")); err != ErrMsgHeadersNotSupported {\n\t\tt.Fatalf(\"Expected to receive an error, got %v\", err)\n\t}\n\n\t// This should succeed.\n\tc, _, _ = newClientForServer(s)\n\tdefer c.close()\n\n\tif err := c.parse([]byte(\"CONNECT {\\\"headers\\\":true}\\r\\nHPUB foo 0 2\\r\\nok\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t}\n\n\t// Now start a server without support.\n\topts.NoHeaderSupport = true\n\topts.Port = -1\n\ts = New(&opts)\n\n\tc, _, _ = newClientForServer(s)\n\tdefer c.close()\n\tif err := c.parse([]byte(\"CONNECT {\\\"headers\\\":true}\\r\\nHPUB foo 0 2\\r\\nok\\r\\n\")); err != ErrMsgHeadersNotSupported {\n\t\tt.Fatalf(\"Expected to receive an error, got %v\", err)\n\t}\n}\n\nvar hmsgPat = regexp.MustCompile(`HMSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)[^\\S\\r\\n]+(\\d+)\\r\\n`)\n\nfunc TestClientHeaderDeliverMsg(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tconnect := \"CONNECT {\\\"headers\\\":true}\"\n\tsubOp := \"SUB foo 1\"\n\tpubOp := \"HPUB foo 12 14\\r\\nName:Derek\\r\\nOK\\r\\n\"\n\tcmd := strings.Join([]string{connect, subOp, pubOp}, \"\\r\\n\")\n\n\tc.parseAsync(cmd)\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\tam := hmsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a match for %q\", l)\n\t}\n\tmatches := am[0]\n\tif len(matches) != 7 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 7)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[HDR_INDEX] != \"12\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[HDR_INDEX])\n\t}\n\tif matches[TLEN_INDEX] != \"14\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[TLEN_INDEX])\n\t}\n\tcheckPayload(cr, []byte(\"Name:Derek\\r\\nOK\\r\\n\"), t)\n}\n\nvar smsgPat = regexp.MustCompile(`^MSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\r\\n`)\n\nfunc TestClientHeaderDeliverStrippedMsg(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tb, br, _ := newClientForServer(s)\n\tdefer b.close()\n\n\t// Does not support headers\n\tb.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\tif _, err := br.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\tconnect := \"CONNECT {\\\"headers\\\":true}\"\n\tpubOp := \"HPUB foo 12 14\\r\\nName:Derek\\r\\nOK\\r\\n\"\n\tcmd := strings.Join([]string{connect, pubOp}, \"\\r\\n\")\n\tc.parseAsync(cmd)\n\t// Read from 'b' client.\n\tl, err := br.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tam := smsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a correct match for %q\", l)\n\t}\n\tmatches := am[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"2\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n\tcheckPayload(br, []byte(\"OK\\r\\n\"), t)\n\tif br.Buffered() != 0 {\n\t\tt.Fatalf(\"Expected no extra bytes to be buffered, got %d\", br.Buffered())\n\t}\n}\n\nfunc TestClientHeaderDeliverQueueSubStrippedMsg(t *testing.T) {\n\topts := defaultServerOptions\n\ts := New(&opts)\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tb, br, _ := newClientForServer(s)\n\tdefer b.close()\n\n\t// Does not support headers\n\tb.parseAsync(\"SUB foo bar 1\\r\\nPING\\r\\n\")\n\tif _, err := br.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\tconnect := \"CONNECT {\\\"headers\\\":true}\"\n\tpubOp := \"HPUB foo 12 14\\r\\nName:Derek\\r\\nOK\\r\\n\"\n\tcmd := strings.Join([]string{connect, pubOp}, \"\\r\\n\")\n\tc.parseAsync(cmd)\n\t// Read from 'b' client.\n\tl, err := br.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tam := smsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a correct match for %q\", l)\n\t}\n\tmatches := am[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"2\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n\tcheckPayload(br, []byte(\"OK\\r\\n\"), t)\n}\n\nfunc TestNonTLSConnectionState(t *testing.T) {\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\tstate := c.GetTLSConnectionState()\n\tif state != nil {\n\t\tt.Error(\"GetTLSConnectionState() returned non-nil\")\n\t}\n}\n\nfunc TestClientConnect(t *testing.T) {\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\n\t// Basic Connect setting flags\n\tconnectOp := []byte(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true,\\\"tls_required\\\":false,\\\"echo\\\":false}\\r\\n\")\n\terr := c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Verbose: true, Pedantic: true, Echo: false}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n\n\t// Test that we can capture user/pass\n\tconnectOp = []byte(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"foo\\\"}\\r\\n\")\n\tc.opts = defaultOpts\n\terr = c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Username: \"derek\", Password: \"foo\"}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n\n\t// Test that we can capture client name\n\tconnectOp = []byte(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"foo\\\",\\\"name\\\":\\\"router\\\"}\\r\\n\")\n\tc.opts = defaultOpts\n\terr = c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Username: \"derek\", Password: \"foo\", Name: \"router\"}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n\n\t// Test that we correctly capture auth tokens\n\tconnectOp = []byte(\"CONNECT {\\\"auth_token\\\":\\\"YZZ222\\\",\\\"name\\\":\\\"router\\\"}\\r\\n\")\n\tc.opts = defaultOpts\n\terr = c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Token: \"YZZ222\", Name: \"router\"}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n}\n\nfunc TestClientConnectProto(t *testing.T) {\n\t_, c, r := setupClient()\n\tdefer c.close()\n\n\t// Basic Connect setting flags, proto should be zero (original proto)\n\tconnectOp := []byte(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true,\\\"tls_required\\\":false}\\r\\n\")\n\terr := c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Protocol: ClientProtoZero}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n\n\t// ProtoInfo\n\tconnectOp = []byte(fmt.Sprintf(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true,\\\"tls_required\\\":false,\\\"protocol\\\":%d}\\r\\n\", ClientProtoInfo))\n\terr = c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected state of OP_START vs %d\\n\", c.state)\n\t}\n\tif !reflect.DeepEqual(c.opts, ClientOpts{Echo: true, Verbose: true, Pedantic: true, Protocol: ClientProtoInfo}) {\n\t\tt.Fatalf(\"Did not parse connect options correctly: %+v\\n\", c.opts)\n\t}\n\tif c.opts.Protocol != ClientProtoInfo {\n\t\tt.Fatalf(\"Protocol should have been set to %v, but is set to %v\", ClientProtoInfo, c.opts.Protocol)\n\t}\n\n\t// Illegal Option\n\tconnectOp = []byte(\"CONNECT {\\\"protocol\\\":22}\\r\\n\")\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\t// The client here is using a pipe, we need to be dequeuing\n\t// data otherwise the server would be blocked trying to send\n\t// the error back to it.\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tif _, _, err := r.ReadLine(); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\terr = c.parse(connectOp)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected to receive an error\\n\")\n\t}\n\tif err != ErrBadClientProtocol {\n\t\tt.Fatalf(\"Expected err of %q, got %q\\n\", ErrBadClientProtocol, err)\n\t}\n\twg.Wait()\n}\n\nfunc TestRemoteAddress(t *testing.T) {\n\trc := &client{}\n\n\t// though in reality this will panic if it does not, adding coverage anyway\n\tif rc.RemoteAddress() != nil {\n\t\tt.Errorf(\"RemoteAddress() did not handle nil connection correctly\")\n\t}\n\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\taddr := c.RemoteAddress()\n\n\tif addr.Network() != \"pipe\" {\n\t\tt.Errorf(\"RemoteAddress() returned invalid network: %s\", addr.Network())\n\t}\n\n\tif addr.String() != \"pipe\" {\n\t\tt.Errorf(\"RemoteAddress() returned invalid string: %s\", addr.String())\n\t}\n}\n\nfunc TestClientPing(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\t// PING\n\tpingOp := \"PING\\r\\n\"\n\tc.parseAsync(pingOp)\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response incorrect: %s\\n\", l)\n\t}\n}\n\nvar msgPat = regexp.MustCompile(`MSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\r\\n`)\n\nconst (\n\tSUB_INDEX = 1\n\tSID_INDEX = 2\n\tREPLY_INDEX = 4\n\tLEN_INDEX = 5\n\tHDR_INDEX = 5\n\tTLEN_INDEX = 6\n)\n\nfunc grabPayload(cr *bufio.Reader, expected int) []byte {\n\td := make([]byte, expected)\n\tn, _ := cr.Read(d)\n\tcr.ReadString('\\n')\n\treturn d[:n]\n}\n\nfunc checkPayload(cr *bufio.Reader, expected []byte, t *testing.T) {\n\tt.Helper()\n\t// Read in payload\n\td := make([]byte, len(expected))\n\tn, err := cr.Read(d)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg payload from server: %v\\n\", err)\n\t}\n\tif n != len(expected) {\n\t\tt.Fatalf(\"Did not read correct amount of bytes: %d vs %d\\n\", n, len(expected))\n\t}\n\tif !bytes.Equal(d, expected) {\n\t\tt.Fatalf(\"Did not read correct payload:: <%s>\\n\", d)\n\t}\n}\n\nfunc TestClientSimplePubSub(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\t// SUB/PUB\n\tc.parseAsync(\"SUB foo 1\\r\\nPUB foo 5\\r\\nhello\\r\\nPING\\r\\n\")\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"5\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n\tcheckPayload(cr, []byte(\"hello\\r\\n\"), t)\n}\n\nfunc TestClientPubSubNoEcho(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\t// Specify no echo\n\tconnectOp := []byte(\"CONNECT {\\\"echo\\\":false}\\r\\n\")\n\terr := c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\t// SUB/PUB\n\tc.parseAsync(\"SUB foo 1\\r\\nPUB foo 5\\r\\nhello\\r\\nPING\\r\\n\")\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\t// We should not receive anything but a PONG since we specified no echo.\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response incorrect: %q\\n\", l)\n\t}\n}\n\nfunc TestClientSimplePubSubWithReply(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\t// SUB/PUB\n\tc.parseAsync(\"SUB foo 1\\r\\nPUB foo bar 5\\r\\nhello\\r\\nPING\\r\\n\")\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[REPLY_INDEX] != \"bar\" {\n\t\tt.Fatalf(\"Did not get correct reply subject: '%s'\\n\", matches[REPLY_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"5\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n}\n\nfunc TestClientNoBodyPubSubWithReply(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\t// SUB/PUB\n\tc.parseAsync(\"SUB foo 1\\r\\nPUB foo bar 0\\r\\n\\r\\nPING\\r\\n\")\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[REPLY_INDEX] != \"bar\" {\n\t\tt.Fatalf(\"Did not get correct reply subject: '%s'\\n\", matches[REPLY_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"0\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n}\n\nfunc TestClientPubWithQueueSub(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\tnum := 100\n\n\t// Queue SUB/PUB\n\tsubs := []byte(\"SUB foo g1 1\\r\\nSUB foo g1 2\\r\\n\")\n\tpubs := []byte(\"PUB foo bar 5\\r\\nhello\\r\\n\")\n\top := []byte{}\n\top = append(op, subs...)\n\tfor i := 0; i < num; i++ {\n\t\top = append(op, pubs...)\n\t}\n\n\tgo c.parseAndClose(op)\n\n\tvar n1, n2, received int\n\tfor ; ; received++ {\n\t\tl, err := cr.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\n\t\t// Count which sub\n\t\tswitch matches[SID_INDEX] {\n\t\tcase \"1\":\n\t\t\tn1++\n\t\tcase \"2\":\n\t\t\tn2++\n\t\t}\n\t\tcheckPayload(cr, []byte(\"hello\\r\\n\"), t)\n\t}\n\tif received != num {\n\t\tt.Fatalf(\"Received wrong # of msgs: %d vs %d\\n\", received, num)\n\t}\n\t// Threshold for randomness for now\n\tif n1 < 20 || n2 < 20 {\n\t\tt.Fatalf(\"Received wrong # of msgs per subscriber: %d - %d\\n\", n1, n2)\n\t}\n}\n\nfunc TestSplitSubjectQueue(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tsq string\n\t\twantSubject []byte\n\t\twantQueue []byte\n\t\twantErr bool\n\t}{\n\t\t{name: \"single subject\",\n\t\t\tsq: \"foo\", wantSubject: []byte(\"foo\"), wantQueue: nil},\n\t\t{name: \"subject and queue\",\n\t\t\tsq: \"foo bar\", wantSubject: []byte(\"foo\"), wantQueue: []byte(\"bar\")},\n\t\t{name: \"subject and queue with surrounding spaces\",\n\t\t\tsq: \" foo bar \", wantSubject: []byte(\"foo\"), wantQueue: []byte(\"bar\")},\n\t\t{name: \"subject and queue with extra spaces in the middle\",\n\t\t\tsq: \"foo bar\", wantSubject: []byte(\"foo\"), wantQueue: []byte(\"bar\")},\n\t\t{name: \"subject, queue, and extra token\",\n\t\t\tsq: \"foo bar fizz\", wantSubject: []byte(nil), wantQueue: []byte(nil), wantErr: true},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\tsub, que, err := splitSubjectQueue(c.sq)\n\t\t\tif err == nil && c.wantErr {\n\t\t\t\tt.Fatal(\"Expected error, but got nil\")\n\t\t\t}\n\t\t\tif err != nil && !c.wantErr {\n\t\t\t\tt.Fatalf(\"Expected nil error, but got %v\", err)\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(sub, c.wantSubject) {\n\t\t\t\tt.Fatalf(\"Expected to get subject %#v, but instead got %#v\", c.wantSubject, sub)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(que, c.wantQueue) {\n\t\t\t\tt.Fatalf(\"Expected to get queue %#v, but instead got %#v\", c.wantQueue, que)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTypeString(t *testing.T) {\n\tcases := []struct {\n\t\tintType int\n\t\tstringType string\n\t}{\n\t\t{\n\t\t\tintType: CLIENT,\n\t\t\tstringType: \"Client\",\n\t\t},\n\t\t{\n\t\t\tintType: ROUTER,\n\t\t\tstringType: \"Router\",\n\t\t},\n\t\t{\n\t\t\tintType: GATEWAY,\n\t\t\tstringType: \"Gateway\",\n\t\t},\n\t\t{\n\t\t\tintType: LEAF,\n\t\t\tstringType: \"Leafnode\",\n\t\t},\n\t\t{\n\t\t\tintType: JETSTREAM,\n\t\t\tstringType: \"JetStream\",\n\t\t},\n\t\t{\n\t\t\tintType: ACCOUNT,\n\t\t\tstringType: \"Account\",\n\t\t},\n\t\t{\n\t\t\tintType: SYSTEM,\n\t\t\tstringType: \"System\",\n\t\t},\n\t\t{\n\t\t\tintType: -1,\n\t\t\tstringType: \"Unknown Type\",\n\t\t},\n\t}\n\tfor _, cs := range cases {\n\t\tc := &client{kind: cs.intType}\n\t\ttypeStringVal := c.kindString()\n\n\t\tif typeStringVal != cs.stringType {\n\t\t\tt.Fatalf(\"Expected typeString value %q, but instead received %q\", cs.stringType, typeStringVal)\n\t\t}\n\t}\n}\n\nfunc TestQueueSubscribePermissions(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tperms *SubjectPermission\n\t\tsubject string\n\t\tqueue string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"plain subscription on foo\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"foo\"}},\n\t\t\tsubject: \"foo\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"queue subscribe with allowed group\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"foo bar\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"queue subscribe with wildcard allowed group\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"foo bar.*\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar.fizz\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"queue subscribe with full wildcard subject and subgroup\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"> bar.>\"}},\n\t\t\tsubject: \"whizz\",\n\t\t\tqueue: \"bar.bang\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"plain subscribe with full wildcard subject and subgroup\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"> bar.>\"}},\n\t\t\tsubject: \"whizz\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"whizz\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"deny plain subscription on foo\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\">\"}, Deny: []string{\"foo\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"bar\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"allow plain subscription, except foo\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\">\"}, Deny: []string{\"foo\"}},\n\t\t\tsubject: \"bar\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"deny everything\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\">\"}, Deny: []string{\">\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"bar\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"can only subscribe to queues v1\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"> v1.>\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"v1.prod\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"cannot subscribe to queues, plain subscribe ok\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\">\"}, Deny: []string{\"> >\"}},\n\t\t\tsubject: \"foo\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"cannot subscribe to queues, queue subscribe not ok\",\n\t\t\tperms: &SubjectPermission{Deny: []string{\"> >\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"bar\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"deny all queue subscriptions on dev or stg only\",\n\t\t\tperms: &SubjectPermission{Deny: []string{\"> *.dev\", \"> *.stg\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"allow only queue subscription on dev or stg\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"> *.dev\", \"> *.stg\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"bar\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"deny queue subscriptions with subject foo\",\n\t\t\tperms: &SubjectPermission{Deny: []string{\"foo >\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"bar\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"plain sub is allowed, but queue subscribe with queue not in list\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"foo bar\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"fizz\",\n\t\t\twant: \"-ERR 'Permissions Violation for Subscription to \\\"foo\\\" using queue \\\"fizz\\\"'\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"allow plain sub, but do queue subscribe\",\n\t\t\tperms: &SubjectPermission{Allow: []string{\"foo\"}},\n\t\t\tsubject: \"foo\",\n\t\t\tqueue: \"bar\",\n\t\t\twant: \"+OK\\r\\n\",\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\t_, client, r := setupClient()\n\t\t\tdefer client.close()\n\n\t\t\tclient.RegisterUser(&User{\n\t\t\t\tPermissions: &Permissions{Subscribe: c.perms},\n\t\t\t})\n\t\t\tconnect := []byte(\"CONNECT {\\\"verbose\\\":true}\\r\\n\")\n\t\t\tqsub := []byte(fmt.Sprintf(\"SUB %s %s 1\\r\\n\", c.subject, c.queue))\n\n\t\t\tgo client.parseAndClose(append(connect, qsub...))\n\n\t\t\tvar buf bytes.Buffer\n\t\t\tif _, err := io.Copy(&buf, r); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Extra OK is from the successful CONNECT.\n\t\t\twant := \"+OK\\r\\n\" + c.want\n\t\t\tif got := buf.String(); got != want {\n\t\t\t\tt.Fatalf(\"Expected to receive %q, but instead received %q\", want, got)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestClientPubWithQueueSubNoEcho(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc1, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Grab the client from server and set no echo by hand.\n\ts.mu.Lock()\n\tlc := len(s.clients)\n\tc := s.clients[s.gcid]\n\ts.mu.Unlock()\n\n\tif lc != 1 {\n\t\tt.Fatalf(\"Expected only 1 client but got %d\\n\", lc)\n\t}\n\tif c == nil {\n\t\tt.Fatal(\"Expected to retrieve client\\n\")\n\t}\n\tc.mu.Lock()\n\tc.echo = false\n\tc.mu.Unlock()\n\n\t// Queue sub on nc1.\n\t_, err = nc1.QueueSubscribe(\"foo\", \"bar\", func(*nats.Msg) {})\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc1.Flush()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tn := int32(0)\n\tcb := func(m *nats.Msg) {\n\t\tatomic.AddInt32(&n, 1)\n\t}\n\n\t_, err = nc2.QueueSubscribe(\"foo\", \"bar\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc2.Flush()\n\n\t// Now publish 100 messages on nc1 which does not allow echo.\n\tfor i := 0; i < 100; i++ {\n\t\tnc1.Publish(\"foo\", []byte(\"Hello\"))\n\t}\n\tnc1.Flush()\n\tnc2.Flush()\n\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tnum := atomic.LoadInt32(&n)\n\t\tif num != int32(100) {\n\t\t\treturn fmt.Errorf(\"Expected all the msgs to be received by nc2, got %d\\n\", num)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestClientUnSub(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\tnum := 1\n\n\t// SUB/PUB\n\tsubs := []byte(\"SUB foo 1\\r\\nSUB foo 2\\r\\n\")\n\tunsub := []byte(\"UNSUB 1\\r\\n\")\n\tpub := []byte(\"PUB foo bar 5\\r\\nhello\\r\\n\")\n\n\top := []byte{}\n\top = append(op, subs...)\n\top = append(op, unsub...)\n\top = append(op, pub...)\n\n\tgo c.parseAndClose(op)\n\n\tvar received int\n\tfor ; ; received++ {\n\t\tl, err := cr.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\t\tif matches[SID_INDEX] != \"2\" {\n\t\t\tt.Fatalf(\"Received msg on unsubscribed subscription!\\n\")\n\t\t}\n\t\tcheckPayload(cr, []byte(\"hello\\r\\n\"), t)\n\t}\n\tif received != num {\n\t\tt.Fatalf(\"Received wrong # of msgs: %d vs %d\\n\", received, num)\n\t}\n}\n\nfunc TestClientUnSubMax(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\n\tnum := 10\n\texp := 5\n\n\t// SUB/PUB\n\tsubs := []byte(\"SUB foo 1\\r\\n\")\n\tunsub := []byte(\"UNSUB 1 5\\r\\n\")\n\tpub := []byte(\"PUB foo bar 5\\r\\nhello\\r\\n\")\n\n\top := []byte{}\n\top = append(op, subs...)\n\top = append(op, unsub...)\n\tfor i := 0; i < num; i++ {\n\t\top = append(op, pub...)\n\t}\n\n\tgo c.parseAndClose(op)\n\n\tvar received int\n\tfor ; ; received++ {\n\t\tl, err := cr.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tmatches := msgPat.FindAllStringSubmatch(l, -1)[0]\n\t\tif matches[SID_INDEX] != \"1\" {\n\t\t\tt.Fatalf(\"Received msg on unsubscribed subscription!\\n\")\n\t\t}\n\t\tcheckPayload(cr, []byte(\"hello\\r\\n\"), t)\n\t}\n\tif received != exp {\n\t\tt.Fatalf(\"Received wrong # of msgs: %d vs %d\\n\", received, exp)\n\t}\n}\n\nfunc TestClientAutoUnsubExactReceived(t *testing.T) {\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\n\t// SUB/PUB\n\tsubs := []byte(\"SUB foo 1\\r\\n\")\n\tunsub := []byte(\"UNSUB 1 1\\r\\n\")\n\tpub := []byte(\"PUB foo bar 2\\r\\nok\\r\\n\")\n\n\top := []byte{}\n\top = append(op, subs...)\n\top = append(op, unsub...)\n\top = append(op, pub...)\n\n\tc.parse(op)\n\n\t// We should not have any subscriptions in place here.\n\tif len(c.subs) != 0 {\n\t\tt.Fatalf(\"Wrong number of subscriptions: expected 0, got %d\\n\", len(c.subs))\n\t}\n}\n\nfunc TestClientUnsubAfterAutoUnsub(t *testing.T) {\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\n\t// SUB/UNSUB/UNSUB\n\tsubs := []byte(\"SUB foo 1\\r\\n\")\n\tasub := []byte(\"UNSUB 1 1\\r\\n\")\n\tunsub := []byte(\"UNSUB 1\\r\\n\")\n\n\top := []byte{}\n\top = append(op, subs...)\n\top = append(op, asub...)\n\top = append(op, unsub...)\n\n\tc.parse(op)\n\n\t// We should not have any subscriptions in place here.\n\tif len(c.subs) != 0 {\n\t\tt.Fatalf(\"Wrong number of subscriptions: expected 0, got %d\\n\", len(c.subs))\n\t}\n}\n\nfunc TestClientRemoveSubsOnDisconnect(t *testing.T) {\n\ts, c, _ := setupClient()\n\tdefer c.close()\n\tsubs := []byte(\"SUB foo 1\\r\\nSUB bar 2\\r\\n\")\n\n\tc.parse(subs)\n\n\tif s.NumSubscriptions() != 2 {\n\t\tt.Fatalf(\"Should have 2 subscriptions, got %d\\n\", s.NumSubscriptions())\n\t}\n\tc.closeConnection(ClientClosed)\n\tcheckExpectedSubs(t, 0, s)\n}\n\nfunc TestClientDoesNotAddSubscriptionsWhenConnectionClosed(t *testing.T) {\n\t_, c, _ := setupClient()\n\tc.close()\n\tsubs := []byte(\"SUB foo 1\\r\\nSUB bar 2\\r\\n\")\n\n\tc.parse(subs)\n\n\tif c.acc.sl.Count() != 0 {\n\t\tt.Fatalf(\"Should have no subscriptions after close, got %d\\n\", c.acc.sl.Count())\n\t}\n}\n\nfunc TestClientMapRemoval(t *testing.T) {\n\ts, c, _ := setupClient()\n\tc.close()\n\n\tcheckClientsCount(t, s, 0)\n}\n\nfunc TestAuthorizationTimeout(t *testing.T) {\n\tserverOptions := DefaultOptions()\n\tserverOptions.Authorization = \"my_token\"\n\tserverOptions.AuthTimeout = 0.4\n\ts := RunServer(serverOptions)\n\tdefer s.Shutdown()\n\n\tconn, err := net.Dial(\"tcp\", net.JoinHostPort(serverOptions.Host, fmt.Sprintf(\"%d\", serverOptions.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error dialing server: %v\\n\", err)\n\t}\n\tdefer conn.Close()\n\tclient := bufio.NewReaderSize(conn, maxBufSize)\n\tif _, err := client.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\ttime.Sleep(3 * secondsToDuration(serverOptions.AuthTimeout))\n\tl, err := client.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\tif !strings.Contains(l, \"Authentication Timeout\") {\n\t\tt.Fatalf(\"Authentication Timeout response incorrect: %q\\n\", l)\n\t}\n}\n\n// This is from bug report #18\nfunc TestTwoTokenPubMatchSingleTokenSub(t *testing.T) {\n\t_, c, cr := setupClient()\n\tdefer c.close()\n\ttest := \"PUB foo.bar 5\\r\\nhello\\r\\nSUB foo 1\\r\\nPING\\r\\nPUB foo.bar 5\\r\\nhello\\r\\nPING\\r\\n\"\n\tc.parseAsync(test)\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response incorrect: %q\\n\", l)\n\t}\n\t// Expect just a pong, no match should exist here..\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"PONG\\r\\n\") {\n\t\tt.Fatalf(\"PONG response was expected, got: %q\\n\", l)\n\t}\n}\n\nfunc TestUnsubRace(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\",\n\t\ts.getOpts().Host,\n\t\ts.Addr().(*net.TCPAddr).Port,\n\t)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client to %s: %v\\n\", url, err)\n\t}\n\tdefer nc.Close()\n\n\tncp, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer ncp.Close()\n\n\tsub, _ := nc.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\t// Just eat it..\n\t})\n\tnc.Flush()\n\n\tvar wg sync.WaitGroup\n\n\twg.Add(1)\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tncp.Publish(\"foo\", []byte(\"hello\"))\n\t\t}\n\t\twg.Done()\n\t}()\n\n\ttime.Sleep(5 * time.Millisecond)\n\n\tsub.Unsubscribe()\n\n\twg.Wait()\n}\n\nfunc TestClientCloseTLSConnection(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/tls.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.TLSTimeout = 100\n\topts.NoLog = true\n\topts.NoSigs = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tendpoint := net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port))\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on dial: %v\", err)\n\t}\n\tdefer conn.Close()\n\tbr := bufio.NewReaderSize(conn, 100)\n\tif _, err := br.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Unexpected error reading INFO: %v\", err)\n\t}\n\n\ttlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})\n\tdefer tlsConn.Close()\n\tif err := tlsConn.Handshake(); err != nil {\n\t\tt.Fatalf(\"Unexpected error during handshake: %v\", err)\n\t}\n\tbr = bufio.NewReaderSize(tlsConn, 100)\n\tconnectOp := []byte(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"foo\\\",\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"tls_required\\\":true}\\r\\n\")\n\tif _, err := tlsConn.Write(connectOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tif _, err := tlsConn.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing PING: %v\", err)\n\t}\n\tif _, err := br.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Unexpected error reading PONG: %v\", err)\n\t}\n\n\t// Check that client is registered.\n\tcheckClientsCount(t, s, 1)\n\tvar cli *client\n\ts.mu.Lock()\n\tfor _, c := range s.clients {\n\t\tcli = c\n\t\tbreak\n\t}\n\ts.mu.Unlock()\n\tif cli == nil {\n\t\tt.Fatal(\"Did not register client on time\")\n\t}\n\t// Test GetTLSConnectionState\n\tstate := cli.GetTLSConnectionState()\n\tif state == nil {\n\t\tt.Error(\"GetTLSConnectionState() returned nil\")\n\t}\n\n\t// Test RemoteAddress\n\taddr := cli.RemoteAddress()\n\tif addr == nil {\n\t\tt.Error(\"RemoteAddress() returned nil\")\n\t}\n\n\tif addr.(*net.TCPAddr).IP.String() != \"127.0.0.1\" {\n\t\tt.Error(\"RemoteAddress() returned incorrect ip \" + addr.String())\n\t}\n\n\t// Fill the buffer. We want to timeout on write so that nc.Close()\n\t// would block due to a write that cannot complete.\n\tbuf := make([]byte, 64*1024)\n\tdone := false\n\tfor !done {\n\t\tcli.nc.SetWriteDeadline(time.Now().Add(time.Second))\n\t\tif _, err := cli.nc.Write(buf); err != nil {\n\t\t\tdone = true\n\t\t}\n\t\tcli.nc.SetWriteDeadline(time.Time{})\n\t}\n\tch := make(chan bool)\n\tgo func() {\n\t\tselect {\n\t\tcase <-ch:\n\t\t\treturn\n\t\tcase <-time.After(3 * time.Second):\n\t\t\tfmt.Println(\"!!!! closeConnection is blocked, test will hang !!!\")\n\t\t\treturn\n\t\t}\n\t}()\n\t// Close the client\n\tcli.closeConnection(ClientClosed)\n\tch <- true\n}\n\n// This tests issue #558\nfunc TestWildcardCharsInLiteralSubjectWorks(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tch := make(chan bool, 1)\n\t// This subject is a literal even though it contains `*` and `>`,\n\t// they are not treated as wildcards.\n\tsubj := \"foo.bar,*,>,baz\"\n\tcb := func(_ *nats.Msg) {\n\t\tch <- true\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tsub, err := nc.Subscribe(subj, cb)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\tif err := nc.Flush(); err != nil {\n\t\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t\t}\n\t\tif err := nc.LastError(); err != nil {\n\t\t\tt.Fatalf(\"Server reported error: %v\", err)\n\t\t}\n\t\tif err := nc.Publish(subj, []byte(\"msg\")); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"Should have received the message\")\n\t\t}\n\t\tif err := sub.Unsubscribe(); err != nil {\n\t\t\tt.Fatalf(\"Error on unsubscribe: %v\", err)\n\t\t}\n\t}\n}\n\n// This test ensures that coalescing into the fixed-size output\n// queues works as expected. When bytes are queued up, they should\n// not overflow a buffer until the capacity is exceeded, at which\n// point a new buffer should be added.\nfunc TestClientOutboundQueueCoalesce(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tclients := s.GlobalAccount().getClients()\n\tif len(clients) != 1 {\n\t\tt.Fatal(\"Expecting a client to exist\")\n\t}\n\tclient := clients[0]\n\tclient.mu.Lock()\n\tdefer client.mu.Unlock()\n\n\t// First up, queue something small into the queue.\n\tclient.queueOutbound([]byte{1, 2, 3, 4, 5})\n\n\tif len(client.out.nb) != 1 {\n\t\tt.Fatal(\"Expecting a single queued buffer\")\n\t}\n\tif l := len(client.out.nb[0]); l != 5 {\n\t\tt.Fatalf(\"Expecting only 5 bytes in the first queued buffer, found %d instead\", l)\n\t}\n\n\t// Then queue up a few more bytes, but not enough\n\t// to overflow into the next buffer.\n\tclient.queueOutbound([]byte{6, 7, 8, 9, 10})\n\n\tif len(client.out.nb) != 1 {\n\t\tt.Fatal(\"Expecting a single queued buffer\")\n\t}\n\tif l := len(client.out.nb[0]); l != 10 {\n\t\tt.Fatalf(\"Expecting 10 bytes in the first queued buffer, found %d instead\", l)\n\t}\n\n\t// Finally, queue up something that is guaranteed\n\t// to overflow.\n\tb := nbPoolSmall.Get().(*[nbPoolSizeSmall]byte)[:]\n\tb = b[:cap(b)]\n\tclient.queueOutbound(b)\n\tif len(client.out.nb) != 2 {\n\t\tt.Fatal(\"Expecting buffer to have overflowed\")\n\t}\n\tif l := len(client.out.nb[0]); l != cap(b) {\n\t\tt.Fatalf(\"Expecting %d bytes in the first queued buffer, found %d instead\", cap(b), l)\n\t}\n\tif l := len(client.out.nb[1]); l != 10 {\n\t\tt.Fatalf(\"Expecting 10 bytes in the second queued buffer, found %d instead\", l)\n\t}\n}\n\nfunc TestClientTraceRace(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Activate trace logging\n\ts.SetLogger(&DummyLogger{}, false, true)\n\n\tnc1, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\ttotal := 10000\n\tcount := 0\n\tch := make(chan bool, 1)\n\tif _, err := nc1.Subscribe(\"foo\", func(_ *nats.Msg) {\n\t\tcount++\n\t\tif count == total {\n\t\t\tch <- true\n\t\t}\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; i < total; i++ {\n\t\t\tnc1.Publish(\"bar\", []byte(\"hello\"))\n\t\t}\n\t}()\n\tfor i := 0; i < total; i++ {\n\t\tnc2.Publish(\"foo\", []byte(\"hello\"))\n\t}\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"Did not get all our messages\")\n\t}\n\twg.Wait()\n}\n\nfunc TestClientUserInfo(t *testing.T) {\n\tpnkey := \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"\n\tc := &client{\n\t\tcid: 1024,\n\t\topts: ClientOpts{\n\t\t\tNkey: pnkey,\n\t\t},\n\t}\n\tgot := c.getAuthUser()\n\texpected := `Nkey \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"`\n\tif got != expected {\n\t\tt.Errorf(\"Expected %q, got %q\", expected, got)\n\t}\n\n\tc = &client{\n\t\tcid: 1024,\n\t\topts: ClientOpts{\n\t\t\tUsername: \"foo\",\n\t\t},\n\t}\n\tgot = c.getAuthUser()\n\texpected = `User \"foo\"`\n\tif got != expected {\n\t\tt.Errorf(\"Expected %q, got %q\", expected, got)\n\t}\n\n\tc = &client{\n\t\tcid: 1024,\n\t\topts: ClientOpts{},\n\t}\n\tgot = c.getAuthUser()\n\texpected = `User \"N/A\"`\n\tif got != expected {\n\t\tt.Errorf(\"Expected %q, got %q\", expected, got)\n\t}\n\n\tc = &client{\n\t\tcid: 1024,\n\t\topts: ClientOpts{\n\t\t\tToken: \"s3cr3t!\",\n\t\t},\n\t}\n\tgot = c.getAuthUser()\n\texpected = `Token \"[REDACTED]\"`\n\tif got != expected {\n\t\tt.Errorf(\"Expected %q, got %q\", expected, got)\n\t}\n}\n\ntype captureWarnLogger struct {\n\tDummyLogger\n\twarn chan string\n}\n\nfunc (l *captureWarnLogger) Warnf(format string, v ...any) {\n\tselect {\n\tcase l.warn <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\nfunc TestReadloopWarning(t *testing.T) {\n\treadLoopReportThreshold = 100 * time.Millisecond\n\tdefer func() { readLoopReportThreshold = readLoopReport }()\n\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tl := &captureWarnLogger{warn: make(chan string, 1)}\n\ts.SetLogger(l, false, false)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc := natsConnect(t, url)\n\tdefer nc.Close()\n\tnatsSubSync(t, nc, \"foo\")\n\tnatsFlush(t, nc)\n\tcid, _ := nc.GetClientID()\n\n\tsender := natsConnect(t, url)\n\tdefer sender.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tc := s.getClient(cid)\n\tc.mu.Lock()\n\tgo func() {\n\t\tdefer wg.Done()\n\t\ttime.Sleep(250 * time.Millisecond)\n\t\tc.mu.Unlock()\n\t}()\n\n\tnatsPub(t, sender, \"foo\", make([]byte, 100))\n\tnatsFlush(t, sender)\n\n\tselect {\n\tcase warn := <-l.warn:\n\t\tif !strings.Contains(warn, \"Readloop\") {\n\t\t\tt.Fatalf(\"unexpected warning: %v\", warn)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"No warning printed\")\n\t}\n\twg.Wait()\n}\n\nfunc TestTraceMsg(t *testing.T) {\n\tc := &client{}\n\n\t// Enable message trace\n\tc.trace = true\n\n\tcases := []struct {\n\t\tDesc string\n\t\tMsg []byte\n\t\tWanted string\n\t\tMaxTracedMsgLen int\n\t}{\n\t\t{\n\t\t\tDesc: \"normal length\",\n\t\t\tMsg: []byte(fmt.Sprintf(\"normal%s\", CR_LF)),\n\t\t\tWanted: \"<<- MSG_PAYLOAD: [\\\"normal\\\"]\",\n\t\t\tMaxTracedMsgLen: 10,\n\t\t},\n\t\t{\n\t\t\tDesc: \"over length\",\n\t\t\tMsg: []byte(fmt.Sprintf(\"over length%s\", CR_LF)),\n\t\t\tWanted: \"<<- MSG_PAYLOAD: [\\\"over lengt...\\\"]\",\n\t\t\tMaxTracedMsgLen: 10,\n\t\t},\n\t\t{\n\t\t\tDesc: \"unlimited length\",\n\t\t\tMsg: []byte(fmt.Sprintf(\"unlimited length%s\", CR_LF)),\n\t\t\tWanted: \"<<- MSG_PAYLOAD: [\\\"unlimited length\\\"]\",\n\t\t\tMaxTracedMsgLen: 0,\n\t\t},\n\t\t{\n\t\t\tDesc: \"negative max traced msg len\",\n\t\t\tMsg: []byte(fmt.Sprintf(\"negative max traced msg len%s\", CR_LF)),\n\t\t\tWanted: \"<<- MSG_PAYLOAD: [\\\"negative max traced msg len\\\"]\",\n\t\t\tMaxTracedMsgLen: -1,\n\t\t},\n\t}\n\n\tfor _, ut := range cases {\n\t\tc.srv = &Server{\n\t\t\topts: &Options{MaxTracedMsgLen: ut.MaxTracedMsgLen},\n\t\t}\n\t\tc.srv.SetLogger(&DummyLogger{}, true, true)\n\n\t\tc.traceMsg(ut.Msg)\n\n\t\tgot := c.srv.logging.logger.(*DummyLogger).Msg\n\t\tif !reflect.DeepEqual(ut.Wanted, got) {\n\t\t\tt.Errorf(\"Desc: %s. Msg %q. Traced msg want: %s, got: %s\", ut.Desc, ut.Msg, ut.Wanted, got)\n\t\t}\n\t}\n}\n\nfunc TestTraceMsgHeadersOnly(t *testing.T) {\n\tc := &client{}\n\t// Enable message trace\n\tc.trace = true\n\n\thdr := fmt.Sprintf(`NATS/1.0%sFoo: 1%s%s`, CR_LF, CR_LF, CR_LF)\n\thdr2 := fmt.Sprintf(`NATS/1.0%sFoo: 1%sBar: 2%s%s`, CR_LF, CR_LF, CR_LF, CR_LF)\n\n\tcases := []struct {\n\t\tDesc string\n\t\tMsg []byte\n\t\tHdr int\n\t\tWanted string\n\t\tMaxTracedMsgLen int\n\t}{\n\t\t{\n\t\t\tDesc: \"payload only\",\n\t\t\tMsg: []byte(`test\\r\\n`),\n\t\t\tHdr: 0,\n\t\t\tWanted: _EMPTY_,\n\t\t\tMaxTracedMsgLen: 0,\n\t\t},\n\t\t{\n\t\t\tDesc: \"header only\",\n\t\t\tMsg: []byte(hdr),\n\t\t\tHdr: len(hdr),\n\t\t\tWanted: `<<- MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\"]`,\n\t\t\tMaxTracedMsgLen: 0,\n\t\t},\n\t\t{\n\t\t\tDesc: \"with header and payload\",\n\t\t\tMsg: []byte(fmt.Sprintf(\"%stest%s\", hdr, CR_LF)),\n\t\t\tHdr: len(hdr),\n\t\t\tWanted: `<<- MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\"]`,\n\t\t\tMaxTracedMsgLen: 0,\n\t\t},\n\t\t{\n\t\t\tDesc: \"max length\",\n\t\t\tMsg: []byte(hdr),\n\t\t\tHdr: len(hdr),\n\t\t\tWanted: `<<- MSG_PAYLOAD: [\"NATS/1...\"]`,\n\t\t\tMaxTracedMsgLen: 6,\n\t\t},\n\t\t{\n\t\t\tDesc: \"two headers max length\",\n\t\t\tMsg: []byte(hdr2),\n\t\t\tHdr: len(hdr2),\n\t\t\tWanted: `<<- MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\\r\\nBar...\"]`,\n\t\t\tMaxTracedMsgLen: 21,\n\t\t},\n\t}\n\n\tfor _, ut := range cases {\n\t\tt.Run(ut.Desc, func(t *testing.T) {\n\t\t\tc.srv = &Server{\n\t\t\t\topts: &Options{MaxTracedMsgLen: ut.MaxTracedMsgLen, TraceHeaders: true},\n\t\t\t}\n\t\t\tc.srv.SetLogger(&DummyLogger{}, true, true)\n\t\t\tc.pa.hdr = ut.Hdr\n\n\t\t\tc.traceMsg(ut.Msg)\n\n\t\t\tgot := c.srv.logging.logger.(*DummyLogger).Msg\n\t\t\trequire_Equal(t, string(ut.Wanted), got)\n\t\t})\n\t}\n}\n\nfunc TestTraceMsgDelivery(t *testing.T) {\n\tlogger := &DummyLogger{}\n\n\topts := DefaultOptions()\n\topts.Trace = true\n\ts := RunServer(opts)\n\ts.SetLogger(logger, true, true)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tncp, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer ncp.Close()\n\n\t_, err = nc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tm := nats.NewMsg(msg.Reply)\n\t\tm.Header[\"A\"] = []string{\"1\"}\n\t\tm.Header[\"B\"] = []string{\"2\"}\n\t\tm.Data = []byte(\"Hi Traced\")\n\t\tmsg.RespondMsg(m)\n\t})\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\tmsg := nats.NewMsg(\"foo\")\n\tmsg.Header = nats.Header{}\n\tmsg.Header[\"A\"] = []string{\"A:1\"}\n\tmsg.Header[\"B\"] = []string{\"B:2\"}\n\tmsg.Data = []byte(\"Hello Traced\")\n\t_, err = ncp.RequestMsg(msg, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\n\t// Wait for logging to settle and safely read the message.\n\ttime.Sleep(50 * time.Millisecond)\n\tlogger.Lock()\n\tm := logger.Msg\n\tlogger.Unlock()\n\trequire_Contains(t, m, \"->> MSG_PAYLOAD:\")\n\trequire_Contains(t, m, \"NATS/1.0\")\n\trequire_Contains(t, m, \"Hi Traced\")\n\n\t_, err = nc.Subscribe(\"bar\", func(msg *nats.Msg) {\n\t\tm := nats.NewMsg(msg.Reply)\n\t\tm.Data = []byte(\"Plain Response\")\n\t\tmsg.RespondMsg(m)\n\t})\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\tmsg = nats.NewMsg(\"bar\")\n\t_, err = ncp.RequestMsg(msg, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\n\t// Wait and safely read again.\n\ttime.Sleep(50 * time.Millisecond)\n\tlogger.Lock()\n\tm = logger.Msg\n\tlogger.Unlock()\n\trequire_Contains(t, m, \"->> MSG_PAYLOAD:\")\n\trequire_Contains(t, m, \"Plain Response\")\n}\n\nfunc TestTraceMsgDeliveryWithHeaders(t *testing.T) {\n\tc := &client{}\n\tc.trace = true\n\thdr := fmt.Sprintf(`NATS/1.0%sFoo: 1%s%s`, CR_LF, CR_LF, CR_LF)\n\thdr2 := fmt.Sprintf(`NATS/1.0%sFoo: bar%sBar: baz%s%s`, CR_LF, CR_LF, CR_LF, CR_LF)\n\n\tcases := []struct {\n\t\tname string\n\t\tmsg []byte\n\t\thdr int\n\t\ttraceDeliver bool\n\t\ttraceHeaders bool\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"delivery with headers enabled\",\n\t\t\tmsg: []byte(hdr),\n\t\t\thdr: len(hdr),\n\t\t\ttraceHeaders: true,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery with full message\",\n\t\t\tmsg: []byte(fmt.Sprintf(\"%stest%s\", hdr, CR_LF)),\n\t\t\thdr: len(hdr),\n\t\t\ttraceHeaders: false,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\\r\\n\\r\\ntest\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery with headers only\",\n\t\t\tmsg: []byte(fmt.Sprintf(\"%stest%s\", hdr, CR_LF)),\n\t\t\thdr: len(hdr),\n\t\t\ttraceHeaders: true,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery multiple headers\",\n\t\t\tmsg: []byte(hdr2),\n\t\t\thdr: len(hdr2),\n\t\t\ttraceHeaders: true,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: bar\\r\\nBar: baz\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery with payload but headers only tracing\",\n\t\t\tmsg: []byte(fmt.Sprintf(\"%spayload data%s\", hdr2, CR_LF)),\n\t\t\thdr: len(hdr2),\n\t\t\ttraceHeaders: true,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: bar\\r\\nBar: baz\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery no headers but tracing enabled\",\n\t\t\tmsg: []byte(fmt.Sprintf(\"plain message%s\", CR_LF)),\n\t\t\thdr: 0,\n\t\t\ttraceHeaders: false,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"plain message\"]`,\n\t\t},\n\t\t{\n\t\t\tname: \"delivery headers disabled but deliver enabled\",\n\t\t\tmsg: []byte(fmt.Sprintf(\"%spayload%s\", hdr, CR_LF)),\n\t\t\thdr: len(hdr),\n\t\t\ttraceHeaders: false,\n\t\t\texpected: `->> MSG_PAYLOAD: [\"NATS/1.0\\r\\nFoo: 1\\r\\n\\r\\npayload\"]`,\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tlogger := &DummyLogger{}\n\n\t\t\tc.srv = &Server{\n\t\t\t\topts: &Options{\n\t\t\t\t\tTrace: true,\n\t\t\t\t\tTraceHeaders: tc.traceHeaders,\n\t\t\t\t},\n\t\t\t}\n\t\t\tc.srv.SetLogger(logger, true, true)\n\t\t\tc.pa.hdr = tc.hdr\n\t\t\tc.traceMsgDelivery(tc.msg, tc.hdr)\n\t\t\tgot := logger.Msg\n\t\t\tif tc.expected == \"\" {\n\t\t\t\t// Disabled\n\t\t\t\trequire_Equal(t, tc.expected, got)\n\t\t\t} else {\n\t\t\t\trequire_True(t, strings.Contains(got, \"->> MSG_PAYLOAD:\"))\n\t\t\t\trequire_Equal(t, tc.expected, got)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestClientMaxPending(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxPending = math.MaxInt32 + 1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"foo\")\n\tnatsPub(t, nc, \"foo\", []byte(\"msg\"))\n\tnatsNexMsg(t, sub, 100*time.Millisecond)\n}\n\nfunc TestResponsePermissions(t *testing.T) {\n\tfor i, test := range []struct {\n\t\tname string\n\t\tperms *ResponsePermission\n\t}{\n\t\t{\"max_msgs\", &ResponsePermission{MaxMsgs: 2, Expires: time.Hour}},\n\t\t{\"no_expire_limit\", &ResponsePermission{MaxMsgs: 3, Expires: -1 * time.Millisecond}},\n\t\t{\"expire\", &ResponsePermission{MaxMsgs: 1000, Expires: 100 * time.Millisecond}},\n\t\t{\"no_msgs_limit\", &ResponsePermission{MaxMsgs: -1, Expires: 100 * time.Millisecond}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := DefaultOptions()\n\t\t\tu1 := &User{\n\t\t\t\tUsername: \"service\",\n\t\t\t\tPassword: \"pwd\",\n\t\t\t\tPermissions: &Permissions{Response: test.perms},\n\t\t\t}\n\t\t\tu2 := &User{Username: \"ivan\", Password: \"pwd\"}\n\t\t\topts.Users = []*User{u1, u2}\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tsvcNC := natsConnect(t, fmt.Sprintf(\"nats://service:pwd@%s:%d\", opts.Host, opts.Port))\n\t\t\tdefer svcNC.Close()\n\t\t\treqSub := natsSubSync(t, svcNC, \"request\")\n\t\t\tnatsFlush(t, svcNC)\n\n\t\t\tnc := natsConnect(t, fmt.Sprintf(\"nats://ivan:pwd@%s:%d\", opts.Host, opts.Port))\n\t\t\tdefer nc.Close()\n\n\t\t\treplySub := natsSubSync(t, nc, \"reply\")\n\n\t\t\tnatsPubReq(t, nc, \"request\", \"reply\", []byte(\"req1\"))\n\n\t\t\treq1 := natsNexMsg(t, reqSub, 100*time.Millisecond)\n\n\t\t\tcheckFailed := func(t *testing.T) {\n\t\t\t\tt.Helper()\n\t\t\t\tif reply, err := replySub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tif reply != nil {\n\t\t\t\t\t\tt.Fatalf(\"Expected to receive timeout, got reply=%q\", reply.Data)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tswitch i {\n\t\t\tcase 0:\n\t\t\t\t// Should allow only 2 replies...\n\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\tnatsPub(t, svcNC, req1.Reply, []byte(\"reply\"))\n\t\t\t\t}\n\t\t\t\tnatsNexMsg(t, replySub, 100*time.Millisecond)\n\t\t\t\tnatsNexMsg(t, replySub, 100*time.Millisecond)\n\t\t\t\t// The next should fail...\n\t\t\t\tcheckFailed(t)\n\t\t\tcase 1:\n\t\t\t\t// Expiration is set to -1ms, which should count as infinite...\n\t\t\t\tnatsPub(t, svcNC, req1.Reply, []byte(\"reply\"))\n\t\t\t\t// Sleep a bit before next send\n\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\tnatsPub(t, svcNC, req1.Reply, []byte(\"reply\"))\n\t\t\t\t// Make sure we receive both\n\t\t\t\tnatsNexMsg(t, replySub, 100*time.Millisecond)\n\t\t\t\tnatsNexMsg(t, replySub, 100*time.Millisecond)\n\t\t\tcase 2:\n\t\t\t\tfallthrough\n\t\t\tcase 3:\n\t\t\t\t// Expire set to 100ms so make sure we wait more between\n\t\t\t\t// next publish\n\t\t\t\tnatsPub(t, svcNC, req1.Reply, []byte(\"reply\"))\n\t\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t\t\tnatsPub(t, svcNC, req1.Reply, []byte(\"reply\"))\n\t\t\t\t// Should receive one, and fail on the other\n\t\t\t\tnatsNexMsg(t, replySub, 100*time.Millisecond)\n\t\t\t\tcheckFailed(t)\n\t\t\t}\n\t\t\t// When testing expiration, sleep before sending next reply\n\t\t\tif i >= 2 {\n\t\t\t\ttime.Sleep(400 * time.Millisecond)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestPingNotSentTooSoon(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tdoneCh := make(chan bool, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\ts.Connz(nil)\n\t\t\tselect {\n\t\t\tcase <-doneCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < 100; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL())\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tnc.Close()\n\t}\n\tclose(doneCh)\n\twg.Wait()\n\n\tc, br, _ := newClientForServer(s)\n\tdefer c.close()\n\tconnectOp := []byte(\"CONNECT {\\\"user\\\":\\\"ivan\\\",\\\"pass\\\":\\\"bar\\\"}\\r\\n\")\n\tc.parse(connectOp)\n\n\t// Since client has not send PING, having server try to send RTT ping\n\t// to client should not do anything\n\tif c.sendRTTPing() {\n\t\tt.Fatalf(\"RTT ping should not have been sent\")\n\t}\n\t// Used to move c.start here but it is often causing race conditions,\n\t// so we'll just wait instead.\n\ttime.Sleep(maxNoRTTPingBeforeFirstPong)\n\n\terrCh := make(chan error, 1)\n\tgo func() {\n\t\tl, _ := br.ReadString('\\n')\n\t\tif l != \"PING\\r\\n\" {\n\t\t\terrCh <- fmt.Errorf(\"expected to get PING, got %s\", l)\n\t\t\treturn\n\t\t}\n\t\terrCh <- nil\n\t}()\n\tif !c.sendRTTPing() {\n\t\tt.Fatalf(\"RTT ping should have been sent\")\n\t}\n\twg.Wait()\n\tif e := <-errCh; e != nil {\n\t\tt.Fatal(e.Error())\n\t}\n}\n\nfunc TestClientCheckUseOfGWReplyPrefix(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tech := make(chan error, 1)\n\tnc, err := nats.Connect(s.ClientURL(),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\t\tech <- e\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Expect to fail if publish on gateway reply prefix\n\tnc.Publish(gwReplyPrefix+\"anything\", []byte(\"should fail\"))\n\n\t// Wait for publish violation error\n\tselect {\n\tcase e := <-ech:\n\t\tif e == nil || !strings.Contains(strings.ToLower(e.Error()), \"violation for publish\") {\n\t\t\tt.Fatalf(\"Expected violation error, got %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive permissions violation error\")\n\t}\n\n\t// Now publish a message with a reply set to the prefix,\n\t// it should be rejected too.\n\tnc.PublishRequest(\"foo\", gwReplyPrefix+\"anything\", []byte(\"should fail\"))\n\n\t// Wait for publish violation error with reply\n\tselect {\n\tcase e := <-ech:\n\t\tif e == nil || !strings.Contains(strings.ToLower(e.Error()), \"violation for publish with reply\") {\n\t\t\tt.Fatalf(\"Expected violation error, got %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive permissions violation error\")\n\t}\n}\n\nfunc TestNoClientLeakOnSlowConsumer(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.Dial(\"tcp\", net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer c.Close()\n\n\tcr := bufio.NewReader(c)\n\n\t// Wait for INFO...\n\tline, _, _ := cr.ReadLine()\n\tvar info serverInfo\n\tif err = json.Unmarshal(line[5:], &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\n\t// Send our connect\n\tif _, err := c.Write([]byte(\"CONNECT {\\\"verbose\\\": false}\\r\\nSUB foo 1\\r\\nPING\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error sending CONNECT and SUB: %v\", err)\n\t}\n\t// Wait for PONG\n\tline, _, _ = cr.ReadLine()\n\tif string(line) != \"PONG\" {\n\t\tt.Fatalf(\"Expected 'PONG' but got %q\", line)\n\t}\n\n\t// Get the client from server map\n\tcli := s.GetClient(info.CID)\n\tif cli == nil {\n\t\tt.Fatalf(\"No client registered\")\n\t}\n\t// Change the write deadline to very low value\n\tcli.mu.Lock()\n\tcli.out.wdl = time.Nanosecond\n\tcli.mu.Unlock()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\t// Send some messages to cause write deadline error on \"cli\"\n\tpayload := make([]byte, 1000)\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPub(t, nc, \"foo\", payload)\n\t}\n\tnatsFlush(t, nc)\n\tnc.Close()\n\n\t// Now make sure that the number of clients goes to 0.\n\tcheckClientsCount(t, s, 0)\n}\n\nfunc TestClientSlowConsumerWithoutConnect(t *testing.T) {\n\topts := DefaultOptions()\n\topts.WriteDeadline = 100 * time.Millisecond\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"127.0.0.1:%d\", opts.Port)\n\tc, err := net.Dial(\"tcp\", url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\tdefer c.Close()\n\tc.Write([]byte(\"SUB foo 1\\r\\n\"))\n\n\tpayload := make([]byte, 10000)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tfor i := 0; i < 10000; i++ {\n\t\tnc.Publish(\"foo\", payload)\n\t}\n\tnc.Flush()\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t// Expect slow consumer..\n\t\tif n := atomic.LoadInt64(&s.slowConsumers); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 slow consumer, got: %v\", n)\n\t\t}\n\t\tif n := s.scStats.clients.Load(); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 slow consumer, got: %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\tvarz, err := s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif varz.SlowConsumersStats.Clients != 1 {\n\t\tt.Error(\"Expected a slow consumer client in varz\")\n\t}\n}\n\nfunc TestClientNoSlowConsumerIfConnectExpected(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Username = \"ivan\"\n\topts.Password = \"pass\"\n\t// Make it very slow so that the INFO sent to client fails...\n\topts.WriteDeadline = time.Nanosecond\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Expect server to close the connection, but will bump the slow\n\t// consumer count.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://ivan:pass@%s:%d\", opts.Host, opts.Port))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected connect error\")\n\t}\n\tif n := atomic.LoadInt64(&s.slowConsumers); n != 0 {\n\t\tt.Fatalf(\"Expected 0 slow consumer, got: %v\", n)\n\t}\n}\n\nfunc TestClientIPv6Address(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Host = \"0.0.0.0\"\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://[::1]:%v\", opts.Port))\n\t// Travis may not accept IPv6, in that case, skip the test.\n\tif err != nil {\n\t\tt.Skipf(\"Skipping test because could not connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tcid, _ := nc.GetClientID()\n\tc := s.GetClient(cid)\n\tc.mu.Lock()\n\tncs := c.String()\n\tc.mu.Unlock()\n\tif !strings.HasPrefix(ncs, \"[::1]\") {\n\t\tt.Fatalf(\"Wrong string representation of an IPv6 address: %q\", ncs)\n\t}\n}\n\nfunc TestPBNotIncreasedOnMaxPending(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxPending = 100\n\ts := &Server{opts: opts}\n\tc := &client{srv: s}\n\tc.initClient()\n\n\tc.mu.Lock()\n\tc.queueOutbound(make([]byte, 200))\n\tpb := c.out.pb\n\tc.mu.Unlock()\n\n\tif pb != 0 {\n\t\tt.Fatalf(\"c.out.pb should be 0, got %v\", pb)\n\t}\n}\n\ntype testConnWritePartial struct {\n\tnet.Conn\n\tpartial bool\n\tbuf bytes.Buffer\n}\n\nfunc (c *testConnWritePartial) Write(p []byte) (int, error) {\n\tn := len(p)\n\tif c.partial {\n\t\tn = n/2 + 1\n\t}\n\treturn c.buf.Write(p[:n])\n}\n\nfunc (c *testConnWritePartial) RemoteAddr() net.Addr {\n\treturn nil\n}\n\nfunc (c *testConnWritePartial) SetWriteDeadline(_ time.Time) error {\n\treturn nil\n}\n\nfunc TestFlushOutboundNoSliceReuseIfPartial(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxPending = 1024\n\ts := &Server{opts: opts}\n\n\tfakeConn := &testConnWritePartial{partial: true}\n\tc := &client{srv: s, nc: fakeConn}\n\tc.initClient()\n\n\tbufs := [][]byte{\n\t\t[]byte(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"),\n\t\t[]byte(\"------\"),\n\t\t[]byte(\"0123456789\"),\n\t}\n\texpected := bytes.Buffer{}\n\tfor _, buf := range bufs {\n\t\texpected.Write(buf)\n\t\tc.mu.Lock()\n\t\tc.queueOutbound(buf)\n\t\tc.flushOutbound()\n\t\tfakeConn.partial = false\n\t\tc.mu.Unlock()\n\t}\n\t// Ensure everything is flushed.\n\tfor done := false; !done; {\n\t\tc.mu.Lock()\n\t\tif c.out.pb > 0 {\n\t\t\tc.flushOutbound()\n\t\t} else {\n\t\t\tdone = true\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n\tif !bytes.Equal(expected.Bytes(), fakeConn.buf.Bytes()) {\n\t\tt.Fatalf(\"Expected\\n%q\\ngot\\n%q\", expected.String(), fakeConn.buf.String())\n\t}\n}\n\ntype captureNoticeLogger struct {\n\tDummyLogger\n\tnotices []string\n}\n\nfunc (l *captureNoticeLogger) Noticef(format string, v ...any) {\n\tl.Lock()\n\tl.notices = append(l.notices, fmt.Sprintf(format, v...))\n\tl.Unlock()\n}\n\nfunc TestCloseConnectionLogsReason(t *testing.T) {\n\to1 := DefaultOptions()\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\tl := &captureNoticeLogger{}\n\ts1.SetLogger(l, true, true)\n\n\to2 := DefaultOptions()\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\ts2.Shutdown()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif s1.NumRoutes() != 0 {\n\t\t\treturn fmt.Errorf(\"route still connected\")\n\t\t}\n\t\treturn nil\n\t})\n\t// Now check that s1 has logged that the connection is closed and that the reason is included.\n\tok := false\n\tl.Lock()\n\tfor _, n := range l.notices {\n\t\tif strings.Contains(n, \"connection closed: \"+ClientClosed.String()) {\n\t\t\tok = true\n\t\t\tbreak\n\t\t}\n\t}\n\tl.Unlock()\n\tif !ok {\n\t\tt.Fatal(\"Log does not contain closed reason\")\n\t}\n}\n\nfunc TestCloseConnectionVeryEarly(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuseTLS bool\n\t}{\n\t\t{\"no_tls\", false},\n\t\t{\"tls\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\tif test.useTLS {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\t\t\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\t\t\t}\n\t\t\t\ttlsConfig, err := GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t\to.TLSConfig = tlsConfig\n\t\t\t}\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// The issue was with a connection that would break right when\n\t\t\t// server was sending the INFO. Creating a bare TCP connection\n\t\t\t// and closing it right away won't help reproduce the problem.\n\t\t\t// So testing in 2 steps.\n\n\t\t\t// Get a normal TCP connection to the server.\n\t\t\tc, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", o.Port))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unable to create tcp connection\")\n\t\t\t}\n\t\t\t// Now close it.\n\t\t\tc.Close()\n\n\t\t\t// Wait that num clients falls to 0.\n\t\t\tcheckClientsCount(t, s, 0)\n\n\t\t\t// Call again with this closed connection. Alternatively, we\n\t\t\t// would have to call with a fake connection that implements\n\t\t\t// net.Conn but returns an error on Write.\n\t\t\ts.createClient(c)\n\n\t\t\t// This connection should not have been added to the server.\n\t\t\tcheckClientsCount(t, s, 0)\n\t\t})\n\t}\n}\n\ntype connAddrString struct {\n\tnet.Addr\n}\n\nfunc (a *connAddrString) String() string {\n\treturn \"[fe80::abc:def:ghi:123%utun0]:4222\"\n}\n\ntype connString struct {\n\tnet.Conn\n}\n\nfunc (c *connString) RemoteAddr() net.Addr {\n\treturn &connAddrString{}\n}\n\nfunc TestClientConnectionName(t *testing.T) {\n\ts, err := NewServer(DefaultOptions())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tl := &DummyLogger{}\n\ts.SetLogger(l, true, true)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tkind int\n\t\tkindStr string\n\t\tws bool\n\t\tmqtt bool\n\t}{\n\t\t{\"client\", CLIENT, \"cid:\", false, false},\n\t\t{\"ws client\", CLIENT, \"wid:\", true, false},\n\t\t{\"mqtt client\", CLIENT, \"mid:\", false, true},\n\t\t{\"route\", ROUTER, \"rid:\", false, false},\n\t\t{\"gateway\", GATEWAY, \"gid:\", false, false},\n\t\t{\"leafnode\", LEAF, \"lid:\", false, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc := &client{srv: s, nc: &connString{}, kind: test.kind}\n\t\t\tif test.ws {\n\t\t\t\tc.ws = &websocket{}\n\t\t\t}\n\t\t\tif test.mqtt {\n\t\t\t\tc.mqtt = &mqtt{}\n\t\t\t}\n\t\t\tc.initClient()\n\n\t\t\tif host := \"fe80::abc:def:ghi:123%utun0\"; host != c.host {\n\t\t\t\tt.Fatalf(\"expected host to be %q, got %q\", host, c.host)\n\t\t\t}\n\t\t\tif port := uint16(4222); port != c.port {\n\t\t\t\tt.Fatalf(\"expected port to be %v, got %v\", port, c.port)\n\t\t\t}\n\n\t\t\tcheckLog := func(suffix string) {\n\t\t\t\tt.Helper()\n\t\t\t\tl.Lock()\n\t\t\t\tmsg := l.Msg\n\t\t\t\tl.Unlock()\n\t\t\t\tif strings.Contains(msg, \"(MISSING)\") {\n\t\t\t\t\tt.Fatalf(\"conn name was not escaped properly, got MISSING: %s\", msg)\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(l.Msg, test.kindStr) {\n\t\t\t\t\tt.Fatalf(\"expected kind to be %q, got: %s\", test.kindStr, msg)\n\t\t\t\t}\n\t\t\t\tif !strings.HasSuffix(l.Msg, suffix) {\n\t\t\t\t\tt.Fatalf(\"expected statement to end with %q, got %s\", suffix, msg)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tc.Debugf(\"debug: %v\", 1)\n\t\t\tcheckLog(\" 1\")\n\t\t\tc.Tracef(\"trace: %s\", \"2\")\n\t\t\tcheckLog(\" 2\")\n\t\t\tc.Warnf(\"warn: %s %d\", \"3\", 4)\n\t\t\tcheckLog(\" 3 4\")\n\t\t\tc.Errorf(\"error: %v %s\", 5, \"6\")\n\t\t\tcheckLog(\" 5 6\")\n\t\t})\n\t}\n}\n\nfunc TestClientLimits(t *testing.T) {\n\taccKp, err := nkeys.CreateAccount()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account key: %v\", err)\n\t}\n\tuKp, err := nkeys.CreateUser()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating user key: %v\", err)\n\t}\n\tuPub, err := uKp.PublicKey()\n\tif err != nil {\n\t\tt.Fatalf(\"Error obtaining publicKey: %v\", err)\n\t}\n\ts, err := NewServer(DefaultOptions())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tfor _, test := range []struct {\n\t\tclient int32\n\t\tacc int32\n\t\tsrv int32\n\t\texpect int32\n\t}{\n\t\t// all identical\n\t\t{1, 1, 1, 1},\n\t\t{-1, -1, 0, -1},\n\t\t// only one value unlimited\n\t\t{1, -1, 0, 1},\n\t\t{-1, 1, 0, 1},\n\t\t{-1, -1, 1, 1},\n\t\t// all combinations of distinct values\n\t\t{1, 2, 3, 1},\n\t\t{1, 3, 2, 1},\n\t\t{2, 1, 3, 1},\n\t\t{2, 3, 1, 1},\n\t\t{3, 1, 2, 1},\n\t\t{3, 2, 1, 1},\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\ts.opts.MaxPayload = test.srv\n\t\t\ts.opts.MaxSubs = int(test.srv)\n\t\t\tc := &client{srv: s, acc: &Account{\n\t\t\t\tlimits: limits{mpay: test.acc, msubs: test.acc},\n\t\t\t}}\n\t\t\tuc := jwt.NewUserClaims(uPub)\n\t\t\tuc.Limits.Subs = int64(test.client)\n\t\t\tuc.Limits.Payload = int64(test.client)\n\t\t\tc.opts.JWT, err = uc.Encode(accKp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error encoding jwt: %v\", err)\n\t\t\t}\n\t\t\tc.applyAccountLimits()\n\t\t\tif c.mpay != test.expect {\n\t\t\t\tt.Fatalf(\"payload %d not as expected %d\", c.mpay, test.expect)\n\t\t\t}\n\t\t\tif c.msubs != test.expect {\n\t\t\t\tt.Fatalf(\"subscriber %d not as expected %d\", c.msubs, test.expect)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestClientClampMaxSubsErrReport(t *testing.T) {\n\tmaxSubLimitReportThreshold = int64(100 * time.Millisecond)\n\tdefer func() { maxSubLimitReportThreshold = defaultMaxSubLimitReportThreshold }()\n\n\to1 := DefaultOptions()\n\to1.MaxSubs = 1\n\to1.LeafNode.Host = \"127.0.0.1\"\n\to1.LeafNode.Port = -1\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts1.SetLogger(l, false, false)\n\n\to2 := DefaultOptions()\n\to2.Cluster.Name = \"xyz\"\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.LeafNode.Port))\n\to2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s1)\n\tcheckLeafNodeConnected(t, s2)\n\n\tnc := natsConnect(t, s2.ClientURL())\n\tdefer nc.Close()\n\tnatsSubSync(t, nc, \"foo\")\n\tnatsSubSync(t, nc, \"bar\")\n\n\t// Make sure we receive only 1\n\tcheck := func() {\n\t\tt.Helper()\n\t\tfor i := 0; i < 2; i++ {\n\t\t\tselect {\n\t\t\tcase errStr := <-l.errCh:\n\t\t\t\tif i > 0 {\n\t\t\t\t\tt.Fatalf(\"Should not have logged a second time: %s\", errStr)\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(errStr, \"maximum subscriptions\") {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %s\", errStr)\n\t\t\t\t}\n\t\t\tcase <-time.After(300 * time.Millisecond):\n\t\t\t\tif i == 0 {\n\t\t\t\t\tt.Fatal(\"Error should have been logged\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tcheck()\n\n\t// The above will have waited long enough to clear the report threshold.\n\t// So create two new subs and check again that we get only 1 report.\n\tnatsSubSync(t, nc, \"baz\")\n\tnatsSubSync(t, nc, \"bat\")\n\tcheck()\n}\n\nfunc TestClientDenySysGroupSub(t *testing.T) {\n\ts := RunServer(DefaultOptions())\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {}))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t_, err = nc.QueueSubscribeSync(\"foo\", sysGroup)\n\trequire_NoError(t, err)\n\tnc.Flush()\n\terr = nc.LastError()\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"Permissions Violation\")\n}\n\nfunc TestClientAuthRequiredNoAuthUser(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\tA: { users: [ { user: user, password: pass } ] }\n\t\t}\n\t\tno_auth_user: user\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tif nc.AuthRequired() {\n\t\tt.Fatalf(\"Expected AuthRequired to be false due to 'no_auth_user'\")\n\t}\n}\n\nfunc TestClientUserInfoReq(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tPERMS = {\n\t\t\tpublish = { allow: \"$SYS.REQ.>\", deny: \"$SYS.REQ.ACCOUNT.>\" }\n\t\t\tsubscribe = \"_INBOX.>\"\n\t\t\tallow_responses: true\n\t\t}\n\t\taccounts: {\n\t\t\tA: { users: [ { user: dlc, password: pass, permissions: $PERMS } ] }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tno_auth_user: dlc\n\t`))\n\tdefer removeFile(t, conf)\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(\"$SYS.REQ.USER.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\n\tdlc := &UserInfo{\n\t\tUserID: \"dlc\",\n\t\tAccount: \"A\",\n\t\tAccountName: \"A\",\n\t\tPermissions: &Permissions{\n\t\t\tPublish: &SubjectPermission{\n\t\t\t\tAllow: []string{\"$SYS.REQ.>\"},\n\t\t\t\tDeny: []string{\"$SYS.REQ.ACCOUNT.>\"},\n\t\t\t},\n\t\t\tSubscribe: &SubjectPermission{\n\t\t\t\tAllow: []string{\"_INBOX.>\"},\n\t\t\t},\n\t\t\tResponse: &ResponsePermission{\n\t\t\t\tMaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS,\n\t\t\t\tExpires: DEFAULT_ALLOW_RESPONSE_EXPIRATION,\n\t\t\t},\n\t\t},\n\t}\n\tif !reflect.DeepEqual(dlc, userInfo) {\n\t\tt.Fatalf(\"User info for %q did not match\", \"dlc\")\n\t}\n\n\t// Make sure system users work ok too.\n\tnc, err = nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(\"$SYS.REQ.USER.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tresponse = ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo = response.Data.(*UserInfo)\n\n\tadmin := &UserInfo{\n\t\tUserID: \"admin\",\n\t\tAccount: \"$SYS\",\n\t\tAccountName: \"$SYS\",\n\t}\n\tif !reflect.DeepEqual(admin, userInfo) {\n\t\tt.Fatalf(\"User info for %q did not match\", \"admin\")\n\t}\n}\n\nfunc TestTLSClientHandshakeFirst(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\ttimeout: \t1\n\t\t\tfirst: \t\t%s\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"true\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tconnect := func(tlsfirst, expectedOk bool) {\n\t\topts := []nats.Option{nats.RootCAs(\"../test/configs/certs/ca.pem\")}\n\t\tif tlsfirst {\n\t\t\topts = append(opts, nats.TLSHandshakeFirst())\n\t\t}\n\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", o.Port), opts...)\n\t\tif expectedOk {\n\t\t\tdefer nc.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif tlsfirst {\n\t\t\t\tcz, err := s.Connz(nil)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting connz: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !cz.Conns[0].TLSFirst {\n\t\t\t\t\tt.Fatal(\"Expected TLSFirst boolean to be set, it was not\")\n\t\t\t\t}\n\t\t\t}\n\t\t} else if !expectedOk && err == nil {\n\t\t\tnc.Close()\n\t\t\tt.Fatal(\"Expected error, got none\")\n\t\t}\n\t}\n\n\t// Server is TLS first, but client is not, so should fail.\n\tconnect(false, false)\n\n\t// Now client is TLS first too, so should work.\n\tconnect(true, true)\n\n\t// Config reload the server and disable tls first\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"false\"))\n\n\t// Now if client wants TLS first, connection should fail.\n\tconnect(true, false)\n\n\t// But if it does not, should be ok.\n\tconnect(false, true)\n\n\t// Config reload the server again and enable tls first\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"true\"))\n\n\t// If both client and server are TLS first, this should work.\n\tconnect(true, true)\n}\n\nfunc TestTLSClientHandshakeFirstFallbackDelayConfigValues(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\ttimeout: \t1\n\t\t\tfirst: \t\t%s\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tval string\n\t\tfirst bool\n\t\tdelay time.Duration\n\t}{\n\t\t{\"first as boolean true\", \"true\", true, 0},\n\t\t{\"first as boolean false\", \"false\", false, 0},\n\t\t{\"first as string true\", \"\\\"true\\\"\", true, 0},\n\t\t{\"first as string false\", \"\\\"false\\\"\", false, 0},\n\t\t{\"first as string on\", \"on\", true, 0},\n\t\t{\"first as string off\", \"off\", false, 0},\n\t\t{\"first as string auto\", \"auto\", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY},\n\t\t{\"first as string auto_fallback\", \"auto_fallback\", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY},\n\t\t{\"first as fallback duration\", \"300ms\", true, 300 * time.Millisecond},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.val)))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tif test.first {\n\t\t\t\tif !o.TLSHandshakeFirst {\n\t\t\t\t\tt.Fatal(\"Expected tls first to be true, was not\")\n\t\t\t\t}\n\t\t\t\tif test.delay != o.TLSHandshakeFirstFallback {\n\t\t\t\t\tt.Fatalf(\"Expected fallback delay to be %v, got %v\", test.delay, o.TLSHandshakeFirstFallback)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif o.TLSHandshakeFirst {\n\t\t\t\t\tt.Fatal(\"Expected tls first to be false, was not\")\n\t\t\t\t}\n\t\t\t\tif o.TLSHandshakeFirstFallback != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected fallback delay to be 0, got %v\", o.TLSHandshakeFirstFallback)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype pauseAfterDial struct {\n\tdelay time.Duration\n}\n\nfunc (d *pauseAfterDial) Dial(network, address string) (net.Conn, error) {\n\tc, err := net.Dial(network, address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttime.Sleep(d.delay)\n\treturn c, nil\n}\n\nfunc TestTLSClientHandshakeFirstFallbackDelay(t *testing.T) {\n\t// Using certificates with RSA 4K to make sure that the fallback does\n\t// not prevent a client with TLS first to successfully connect.\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/tls/benchmark-server-cert-rsa-4096.pem\"\n\t\t\tkey_file: \"./configs/certs/tls/benchmark-server-key-rsa-4096.pem\"\n\t\t\ttimeout: \t1\n\t\t\tfirst: \t\t%s\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"auto\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"tls://localhost:%d\", o.Port)\n\td := &pauseAfterDial{delay: DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY + 100*time.Millisecond}\n\n\t// Connect a client without \"TLS first\" and it should be accepted.\n\tnc, err := nats.Connect(url,\n\t\tnats.SetCustomDialer(d),\n\t\tnats.Secure(&tls.Config{\n\t\t\tServerName: \"reuben.nats.io\",\n\t\t\tMinVersion: tls.VersionTLS12,\n\t\t}),\n\t\tnats.RootCAs(\"./configs/certs/tls/benchmark-ca-cert.pem\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\t// Check that the TLS first in monitoring is set to false\n\tcs, err := s.Connz(nil)\n\trequire_NoError(t, err)\n\tif cs.Conns[0].TLSFirst {\n\t\tt.Fatal(\"Expected monitoring ConnInfo.TLSFirst to be false, it was not\")\n\t}\n\tnc.Close()\n\n\t// Wait for the client to be removed\n\tcheckClientsCount(t, s, 0)\n\n\t// Increase the fallback delay with config reload.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"\\\"1s\\\"\"))\n\n\t// This time, start the client with \"TLS first\".\n\t// We will also make sure that we did not wait for the fallback delay\n\t// in order to connect.\n\tstart := time.Now()\n\tnc, err = nats.Connect(url,\n\t\tnats.SetCustomDialer(d),\n\t\tnats.Secure(&tls.Config{\n\t\t\tServerName: \"reuben.nats.io\",\n\t\t\tMinVersion: tls.VersionTLS12,\n\t\t}),\n\t\tnats.RootCAs(\"./configs/certs/tls/benchmark-ca-cert.pem\"),\n\t\tnats.TLSHandshakeFirst())\n\trequire_NoError(t, err)\n\trequire_True(t, time.Since(start) < 500*time.Millisecond)\n\tdefer nc.Close()\n\n\t// Check that the TLS first in monitoring is set to true.\n\tcs, err = s.Connz(nil)\n\trequire_NoError(t, err)\n\tif !cs.Conns[0].TLSFirst {\n\t\tt.Fatal(\"Expected monitoring ConnInfo.TLSFirst to be true, it was not\")\n\t}\n\tnc.Close()\n}\n\nfunc TestTLSClientHandshakeFirstFallbackDelayAndAllowNonTLS(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\ttimeout: \t1\n\t\t\tfirst: \t\t%s\n\t\t}\n\t\tallow_non_tls: true\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"true\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// We first start with a server that has handshake first set to true\n\t// and allow_non_tls. In that case, only \"TLS first\" clients should be\n\t// accepted.\n\turl := fmt.Sprintf(\"tls://localhost:%d\", o.Port)\n\tnc, err := nats.Connect(url,\n\t\tnats.RootCAs(\"../test/configs/certs/ca.pem\"),\n\t\tnats.TLSHandshakeFirst())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\t// Check that the TLS first in monitoring is set to true\n\tcs, err := s.Connz(nil)\n\trequire_NoError(t, err)\n\tif !cs.Conns[0].TLSFirst {\n\t\tt.Fatal(\"Expected monitoring ConnInfo.TLSFirst to be true, it was not\")\n\t}\n\tnc.Close()\n\n\t// Client not using \"TLS First\" should fail.\n\tnc, err = nats.Connect(url, nats.RootCAs(\"../test/configs/certs/ca.pem\"))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected connection to fail, it did not\")\n\t}\n\n\t// And non TLS clients should also fail to connect.\n\tnc, err = nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.Port))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected connection to fail, it did not\")\n\t}\n\n\t// Now we will replace TLS first in server with a fallback delay.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"\\\"25ms\\\"\"))\n\n\t// Clients with \"TLS first\" should still be able to connect\n\tnc, err = nats.Connect(url,\n\t\tnats.RootCAs(\"../test/configs/certs/ca.pem\"),\n\t\tnats.TLSHandshakeFirst())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tcheckConnInfo := func(isTLS, isTLSFirst bool) {\n\t\tt.Helper()\n\t\tcs, err = s.Connz(nil)\n\t\trequire_NoError(t, err)\n\t\tconn := cs.Conns[0]\n\t\tif !isTLS {\n\t\t\tif conn.TLSVersion != _EMPTY_ {\n\t\t\t\tt.Fatalf(\"Being a non TLS client, there should not be TLSVersion set, got %v\", conn.TLSVersion)\n\t\t\t}\n\t\t\tif conn.TLSFirst {\n\t\t\t\tt.Fatal(\"Being a non TLS client, TLSFirst should not be set, but it was\")\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tif isTLSFirst && !conn.TLSFirst {\n\t\t\tt.Fatal(\"Expected monitoring ConnInfo.TLSFirst to be true, it was not\")\n\t\t} else if !isTLSFirst && conn.TLSFirst {\n\t\t\tt.Fatal(\"Expected monitoring ConnInfo.TLSFirst to be false, it was not\")\n\t\t}\n\t\tnc.Close()\n\n\t\tcheckClientsCount(t, s, 0)\n\t}\n\tcheckConnInfo(true, true)\n\n\t// Clients with TLS but not \"TLS first\" should also be able to connect.\n\tnc, err = nats.Connect(url, nats.RootCAs(\"../test/configs/certs/ca.pem\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tcheckConnInfo(true, false)\n\n\t// And non TLS clients should also be able to connect.\n\tnc, err = nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.Port))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tcheckConnInfo(false, false)\n}\n\nfunc TestTLSClientHandshakeFirstAndInProcessConnection(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\ttimeout: \t1\n\t\t\tfirst: \t\ttrue\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Check that we can create an in process connection that does not use TLS\n\tnc, err := nats.Connect(_EMPTY_, nats.InProcessServer(s))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tif nc.TLSRequired() {\n\t\tt.Fatalf(\"Shouldn't have required TLS for in-process connection\")\n\t}\n\tif _, err = nc.TLSConnectionState(); err == nil {\n\t\tt.Fatal(\"Should have got an error retrieving TLS connection state\")\n\t}\n\tnc.Close()\n\n\t// If the client wants TLS, it should get a TLS connection.\n\tnc, err = nats.Connect(_EMPTY_,\n\t\tnats.InProcessServer(s),\n\t\tnats.RootCAs(\"../test/configs/certs/ca.pem\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tif _, err = nc.TLSConnectionState(); err != nil {\n\t\tt.Fatal(\"Should have not got an error retrieving TLS connection state\")\n\t}\n\t// However, the server would not have sent that TLS was required,\n\t// but instead it is available.\n\tif nc.TLSRequired() {\n\t\tt.Fatalf(\"Shouldn't have required TLS for in-process connection\")\n\t}\n\tnc.Close()\n\n\t// The in-process connection with TLS and \"TLS first\" should also be working.\n\tnc, err = nats.Connect(_EMPTY_,\n\t\tnats.InProcessServer(s),\n\t\tnats.RootCAs(\"../test/configs/certs/ca.pem\"),\n\t\tnats.TLSHandshakeFirst())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tif !nc.TLSRequired() {\n\t\tt.Fatalf(\"The server should have sent that TLS is required\")\n\t}\n\tif _, err = nc.TLSConnectionState(); err != nil {\n\t\tt.Fatal(\"Should have not got an error retrieving TLS connection state\")\n\t}\n}\n\nfunc TestRemoveHeaderIfPrefixPresent(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\thdr = genHeader(hdr, \"a\", \"1\")\n\thdr = genHeader(hdr, JSExpectedStream, \"my-stream\")\n\thdr = genHeader(hdr, JSExpectedLastSeq, \"22\")\n\thdr = genHeader(hdr, \"b\", \"2\")\n\thdr = genHeader(hdr, JSExpectedLastSubjSeq, \"24\")\n\thdr = genHeader(hdr, JSExpectedLastMsgId, \"1\")\n\thdr = genHeader(hdr, \"c\", \"3\")\n\n\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Expected-\")\n\n\tif !bytes.Equal(hdr, []byte(\"NATS/1.0\\r\\na: 1\\r\\nb: 2\\r\\nc: 3\\r\\n\\r\\n\")) {\n\t\tt.Fatalf(\"Expected headers to be stripped, got %q\", hdr)\n\t}\n}\n\nfunc TestSliceHeader(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\thdr = genHeader(hdr, \"a\", \"1\")\n\thdr = genHeader(hdr, JSExpectedStream, \"my-stream\")\n\thdr = genHeader(hdr, JSExpectedLastSeq, \"22\")\n\thdr = genHeader(hdr, \"b\", \"2\")\n\thdr = genHeader(hdr, JSExpectedLastSubjSeq, \"24\")\n\thdr = genHeader(hdr, JSExpectedLastMsgId, \"1\")\n\thdr = genHeader(hdr, \"c\", \"3\")\n\n\tsliced := sliceHeader(JSExpectedLastSubjSeq, hdr)\n\tcopied := getHeader(JSExpectedLastSubjSeq, hdr)\n\n\trequire_NotNil(t, sliced)\n\trequire_Equal(t, cap(sliced), 2)\n\n\trequire_NotNil(t, copied)\n\trequire_Equal(t, cap(copied), len(copied))\n\n\trequire_True(t, bytes.Equal(sliced, copied))\n}\n\nfunc TestSliceHeaderOrderingPrefix(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t// These headers share the same prefix, the longer subject\n\t// must not invalidate the existence of the shorter one.\n\thdr = genHeader(hdr, JSExpectedLastSubjSeqSubj, \"foo\")\n\thdr = genHeader(hdr, JSExpectedLastSubjSeq, \"24\")\n\n\tsliced := sliceHeader(JSExpectedLastSubjSeq, hdr)\n\tcopied := getHeader(JSExpectedLastSubjSeq, hdr)\n\n\trequire_NotNil(t, sliced)\n\trequire_Equal(t, cap(sliced), 2)\n\n\trequire_NotNil(t, copied)\n\trequire_Equal(t, cap(copied), len(copied))\n\n\trequire_True(t, bytes.Equal(sliced, copied))\n}\n\nfunc TestSliceHeaderOrderingSuffix(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t// These headers share the same suffix, the longer subject\n\t// must not invalidate the existence of the shorter one.\n\thdr = genHeader(hdr, \"Previous-Nats-Msg-Id\", \"user\")\n\thdr = genHeader(hdr, \"Nats-Msg-Id\", \"control\")\n\n\tsliced := sliceHeader(\"Nats-Msg-Id\", hdr)\n\tcopied := getHeader(\"Nats-Msg-Id\", hdr)\n\n\trequire_NotNil(t, sliced)\n\trequire_NotNil(t, copied)\n\trequire_True(t, bytes.Equal(sliced, copied))\n\trequire_Equal(t, string(copied), \"control\")\n}\n\nfunc TestRemoveHeaderIfPresentOrderingPrefix(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t// These headers share the same prefix, the longer subject\n\t// must not invalidate the existence of the shorter one.\n\thdr = genHeader(hdr, JSExpectedLastSubjSeqSubj, \"foo\")\n\thdr = genHeader(hdr, JSExpectedLastSubjSeq, \"24\")\n\n\thdr = removeHeaderIfPresent(hdr, JSExpectedLastSubjSeq)\n\tehdr := genHeader(nil, JSExpectedLastSubjSeqSubj, \"foo\")\n\trequire_True(t, bytes.Equal(hdr, ehdr))\n}\n\nfunc TestRemoveHeaderIfPresentOrderingSuffix(t *testing.T) {\n\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t// These headers share the same suffix, the longer subject\n\t// must not invalidate the existence of the shorter one.\n\thdr = genHeader(hdr, \"Previous-Nats-Msg-Id\", \"user\")\n\thdr = genHeader(hdr, \"Nats-Msg-Id\", \"control\")\n\n\thdr = removeHeaderIfPresent(hdr, \"Nats-Msg-Id\")\n\tehdr := genHeader(nil, \"Previous-Nats-Msg-Id\", \"user\")\n\trequire_True(t, bytes.Equal(hdr, ehdr))\n}\n\nfunc TestMsgPartsCapsHdrSlice(t *testing.T) {\n\tc := &client{}\n\thdrContent := hdrLine + \"Key1: Val1\\r\\nKey2: Val2\\r\\n\\r\\n\"\n\tmsgBody := \"hello\\r\\n\"\n\tbuf := slices.Clone([]byte(hdrContent + msgBody))\n\tc.pa.hdr = len(hdrContent)\n\n\thdr, msg := c.msgParts(buf)\n\t// Make sure \"hdr\" and \"msg\" are as expected.\n\trequire_Equal(t, string(hdr), hdrContent)\n\trequire_Equal(t, string(msg), msgBody)\n\t// Previously, cap(hdr) would have been the same than the one of \"buf\",\n\t// but now this is not the case.\n\trequire_True(t, cap(hdr) < cap(buf))\n\t// Just to make sure, try to add something (smaller than \"hello\\r\\n\")\n\t// to \"hdr\" and make sure the \"msg\" content is not modified.\n\thdr = append(hdr, \"test\"...)\n\trequire_Equal(t, string(hdr), hdrContent+\"test\")\n\trequire_Equal(t, string(msg), \"hello\\r\\n\")\n}\n\nfunc TestSetHeaderDoesNotOverwriteUnderlyingBuffer(t *testing.T) {\n\tinitialHdrContent := \"NATS/1.0\\r\\nKey1: Val1\\r\\nKey2: Val2\\r\\n\\r\\n\"\n\tmsgBody := \"this is the message body\\r\\n\"\n\tfor _, test := range []struct {\n\t\tname string\n\t\tkey string\n\t\tval string\n\t\texpectedHdr string\n\t\tnewBuf bool\n\t}{\n\t\t{\"existing key new value larger\", \"Key1\", \"Val1Updated\", \"NATS/1.0\\r\\nKey1: Val1Updated\\r\\nKey2: Val2\\r\\n\\r\\n\", true},\n\t\t{\"existing key new value smaller\", \"Key1\", \"v1\", \"NATS/1.0\\r\\nKey1: v1\\r\\nKey2: Val2\\r\\n\\r\\n\", false},\n\t\t{\"new key\", \"Key3\", \"Val3\", \"NATS/1.0\\r\\nKey1: Val1\\r\\nKey2: Val2\\r\\nKey3: Val3\\r\\n\\r\\n\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tbuf := make([]byte, 0, len(initialHdrContent)+len(msgBody))\n\t\t\tbuf = append(buf, initialHdrContent...)\n\t\t\tmsgStart := len(buf)\n\t\t\tbuf = append(buf, msgBody...)\n\n\t\t\t// Simulate that we create slices out of the underlying buffer,\n\t\t\t// separating the header and body parts.\n\t\t\thdr, msg := buf[:msgStart], buf[msgStart:]\n\t\t\thdr = setHeader(test.key, test.val, hdr)\n\t\t\trequire_Equal(t, string(hdr), test.expectedHdr)\n\t\t\trequire_Equal(t, string(msg), msgBody)\n\t\t\tif test.newBuf {\n\t\t\t\t// The \"hdr\" part of the underlying buffer should not have\n\t\t\t\t// been changed.\n\t\t\t\trequire_Equal(t, string(buf[:len(initialHdrContent)]), initialHdrContent)\n\t\t\t} else {\n\t\t\t\t// The \"hdr\" part of the underlying buffer has been changed.\n\t\t\t\trequire_Equal(t, string(buf[:len(test.expectedHdr)]), test.expectedHdr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSetHeaderOrderingPrefix(t *testing.T) {\n\tfor _, space := range []bool{true, false} {\n\t\ttitle := \"Normal\"\n\t\tif !space {\n\t\t\ttitle = \"Trimmed\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t\t\t// These headers share the same prefix, the longer subject\n\t\t\t// must not invalidate the existence of the shorter one.\n\t\t\thdr = genHeader(hdr, JSExpectedLastSubjSeqSubj, \"foo\")\n\t\t\thdr = genHeader(hdr, JSExpectedLastSubjSeq, \"24\")\n\t\t\tif !space {\n\t\t\t\thdr = bytes.ReplaceAll(hdr, []byte(\" \"), nil)\n\t\t\t}\n\n\t\t\thdr = setHeader(JSExpectedLastSubjSeq, \"12\", hdr)\n\t\t\tehdr := genHeader(nil, JSExpectedLastSubjSeqSubj, \"foo\")\n\t\t\tehdr = genHeader(ehdr, JSExpectedLastSubjSeq, \"12\")\n\t\t\tif !space {\n\t\t\t\tehdr = bytes.ReplaceAll(ehdr, []byte(\" \"), nil)\n\t\t\t}\n\t\t\trequire_True(t, bytes.Equal(hdr, ehdr))\n\t\t})\n\t}\n}\n\nfunc TestSetHeaderOrderingSuffix(t *testing.T) {\n\tfor _, space := range []bool{true, false} {\n\t\ttitle := \"Normal\"\n\t\tif !space {\n\t\t\ttitle = \"Trimmed\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\thdr := []byte(\"NATS/1.0\\r\\n\\r\\n\")\n\n\t\t\t// These headers share the same suffix, the longer subject\n\t\t\t// must not invalidate the existence of the shorter one.\n\t\t\thdr = genHeader(hdr, \"Previous-Nats-Msg-Id\", \"user\")\n\t\t\thdr = genHeader(hdr, \"Nats-Msg-Id\", \"control\")\n\t\t\tif !space {\n\t\t\t\thdr = bytes.ReplaceAll(hdr, []byte(\" \"), nil)\n\t\t\t}\n\n\t\t\thdr = setHeader(\"Nats-Msg-Id\", \"other\", hdr)\n\t\t\tehdr := genHeader(nil, \"Previous-Nats-Msg-Id\", \"user\")\n\t\t\tehdr = genHeader(ehdr, \"Nats-Msg-Id\", \"other\")\n\t\t\tif !space {\n\t\t\t\tehdr = bytes.ReplaceAll(ehdr, []byte(\" \"), nil)\n\t\t\t}\n\t\t\trequire_True(t, bytes.Equal(hdr, ehdr))\n\t\t})\n\t}\n}\n\nfunc TestInProcessAllowedConnectionType(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"test\", password: \"pwd\", allowed_connection_types: [\"%s\"]}] }\n\t\t}\n\t\twrite_deadline: \"500ms\"\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tct string\n\t\tinProcessOnly bool\n\t}{\n\t\t{\"conf inprocess\", jwt.ConnectionTypeInProcess, true},\n\t\t{\"conf standard\", jwt.ConnectionTypeStandard, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.ct)))\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Create standard connection\n\t\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"test\", \"pwd\"))\n\t\t\tif test.inProcessOnly && err == nil {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatal(\"Expected standard connection to fail, it did not\")\n\t\t\t}\n\t\t\t// Works if nc is nil (which it will if only in-process are allowed)\n\t\t\tnc.Close()\n\n\t\t\t// Create inProcess connection\n\t\t\tnc, err = nats.Connect(_EMPTY_, nats.UserInfo(\"test\", \"pwd\"), nats.InProcessServer(s))\n\t\t\tif !test.inProcessOnly && err == nil {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatal(\"Expected in-process connection to fail, it did not\")\n\t\t\t}\n\t\t\t// Works if nc is nil (which it will if only standard are allowed)\n\t\t\tnc.Close()\n\t\t})\n\t}\n\tfor _, test := range []struct {\n\t\tname string\n\t\tct string\n\t\tinProcessOnly bool\n\t}{\n\t\t{\"jwt inprocess\", jwt.ConnectionTypeInProcess, true},\n\t\t{\"jwt standard\", jwt.ConnectionTypeStandard, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tskp, _ := nkeys.FromSeed(oSeed)\n\t\t\tspub, _ := skp.PublicKey()\n\n\t\t\to := defaultServerOptions\n\t\t\to.TrustedKeys = []string{spub}\n\t\t\to.WriteDeadline = 500 * time.Millisecond\n\t\t\ts := RunServer(&o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tbuildMemAccResolver(s)\n\n\t\t\tkp, _ := nkeys.CreateAccount()\n\t\t\taPub, _ := kp.PublicKey()\n\t\t\tclaim := jwt.NewAccountClaims(aPub)\n\t\t\taJwt, err := claim.Encode(oKp)\n\t\t\trequire_NoError(t, err)\n\n\t\t\taddAccountToMemResolver(s, aPub, aJwt)\n\n\t\t\tcreds := createUserWithLimit(t, kp, time.Time{},\n\t\t\t\tfunc(j *jwt.UserPermissionLimits) {\n\t\t\t\t\tj.AllowedConnectionTypes.Add(test.ct)\n\t\t\t\t})\n\t\t\t// Create standard connection\n\t\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(creds))\n\t\t\tif test.inProcessOnly && err == nil {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatal(\"Expected standard connection to fail, it did not\")\n\t\t\t}\n\t\t\t// Works if nc is nil (which it will if only in-process are allowed)\n\t\t\tnc.Close()\n\n\t\t\t// Create inProcess connection\n\t\t\tnc, err = nats.Connect(_EMPTY_, nats.UserCredentials(creds), nats.InProcessServer(s))\n\t\t\tif !test.inProcessOnly && err == nil {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatal(\"Expected in-process connection to fail, it did not\")\n\t\t\t}\n\t\t\t// Works if nc is nil (which it will if only standard are allowed)\n\t\t\tnc.Close()\n\t\t})\n\t}\n}\n\nfunc TestClientFlushOutboundNoSlowConsumer(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxPending = 1024 * 1024 * 140 // 140MB\n\topts.MaxPayload = 1024 * 1024 * 16 // 16MB\n\topts.WriteDeadline = time.Second * 30\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tdefer nc.Close()\n\n\tproxy := newNetProxy(0, 1024*1024*8, 1024*1024*8, s.ClientURL()) // 8MB/s\n\tdefer proxy.stop()\n\n\twait := make(chan error)\n\n\tnca, err := nats.Connect(proxy.clientURL(), nats.NoCallbacksAfterClientClose())\n\trequire_NoError(t, err)\n\tdefer nca.Close()\n\tnca.SetDisconnectErrHandler(func(c *nats.Conn, err error) {\n\t\twait <- err\n\t\tclose(wait)\n\t})\n\n\tncb, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer ncb.Close()\n\n\t_, err = nca.Subscribe(\"test\", func(msg *nats.Msg) {\n\t\twait <- nil\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish 128MB of data onto the test subject. This will\n\t// mean that the outbound queue for nca has more than 64MB,\n\t// which is the max we will send into a single writev call.\n\tpayload := make([]byte, 1024*1024*16) // 16MB\n\tfor i := 0; i < 8; i++ {\n\t\trequire_NoError(t, ncb.Publish(\"test\", payload))\n\t}\n\n\t// Get the client ID for nca.\n\tcid, err := nca.GetClientID()\n\trequire_NoError(t, err)\n\n\t// Check that the client queue has more than 64MB queued\n\t// up in it.\n\ts.mu.RLock()\n\tca := s.clients[cid]\n\ts.mu.RUnlock()\n\tca.mu.Lock()\n\tpba := ca.out.pb\n\tca.mu.Unlock()\n\trequire_True(t, pba > 1024*1024*64)\n\n\t// Wait for our messages to be delivered. This will take\n\t// a few seconds as the client is limited to 8MB/s, so it\n\t// can't deliver messages to us as quickly as the other\n\t// client can publish them.\n\tvar msgs int\n\tfor err := range wait {\n\t\trequire_NoError(t, err)\n\t\tmsgs++\n\t\tif msgs == 8 {\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_Equal(t, msgs, 8)\n}\n\nfunc TestClientRejectsNRGSubjects(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\tA: { users: [ { user: nat, password: pass } ] }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tno_auth_user: nat\n\t`))\n\tdefer removeFile(t, conf)\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tt.Run(\"System\", func(t *testing.T) {\n\t\tech := make(chan error, 1)\n\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"),\n\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\t\t\tech <- e\n\t\t\t}),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\n\t\t// System account clients are allowed to publish to these subjects.\n\t\trequire_NoError(t, nc.Publish(\"$NRG.foo\", nil))\n\t\trequire_NoChanRead(t, ech, time.Second)\n\t})\n\n\tt.Run(\"Normal\", func(t *testing.T) {\n\t\tech := make(chan error, 1)\n\t\tnc, err := nats.Connect(s.ClientURL(),\n\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\t\t\tech <- e\n\t\t\t}),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\n\t\t// Non-system clients should receive a pub permission error.\n\t\trequire_NoError(t, nc.Publish(\"$NRG.foo\", nil))\n\t\terr = require_ChanRead(t, ech, time.Second)\n\t\trequire_Error(t, err)\n\t\trequire_True(t, strings.HasPrefix(err.Error(), \"nats: permissions violation\"))\n\t})\n}\n\nfunc TestConnectionStringWithLogConnectionInfo(t *testing.T) {\n\topts := DefaultOptions()\n\ts, c, _, _ := rawSetup(*opts)\n\tdefer c.close()\n\tdefer s.Shutdown()\n\n\tc.kind = CLIENT\n\tconnectArg := []byte(\"{\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"version\\\":\\\"1.0.0\\\",\\\"lang\\\":\\\"go\\\",\\\"name\\\":\\\"test-client\\\"}\")\n\n\terr := c.processConnect(connectArg)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error on first processConnect: %v\", err)\n\t}\n\n\t// Get the connection string after first processConnect.\n\tfirstConnStr := c.ncs.Load()\n\tif firstConnStr == nil {\n\t\treturn\n\t}\n\tfirstStr := firstConnStr.(string)\n\tfirstLen := len(firstStr)\n\trequire_Equal(t, firstStr, `pipe - cid:1 - \"v1.0.0:go:test-client\"`)\n\n\t// Process connect multiple times.\n\tfor i := 0; i < 3; i++ {\n\t\terr = c.processConnect(connectArg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Received error on processConnect attempt %d: %v\", i+2, err)\n\t\t}\n\t}\n\n\t// Get the connection string after multiple calls.\n\tfinalConnStr := c.ncs.Load()\n\trequire_NotNil(t, finalConnStr)\n\n\tfinalStr := finalConnStr.(string)\n\trequire_Equal(t, firstStr, finalStr)\n\n\t// Now send a different connect over the same connection.\n\tconnectArg2 := []byte(\"{\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"version\\\":\\\"1.0.0\\\",\\\"lang\\\":\\\"go\\\",\\\"name\\\":\\\"test-client:new\\\"}\")\n\n\terr = c.processConnect(connectArg2)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error on processConnect: %v\", err)\n\t}\n\tfinalConnStr = c.ncs.Load()\n\trequire_NotNil(t, finalConnStr)\n\n\t// Check that it remains the same size after a different connect.\n\tfinalStr = finalConnStr.(string)\n\tfinalLen := len(finalStr)\n\tif finalLen > firstLen {\n\t\tt.Fatalf(\"Connection string grew from %d to %d characters\", firstLen, finalLen)\n\t}\n}\n\nfunc TestLogConnectionAuthInfo(t *testing.T) {\n\tt.Run(\"username_password\", func(t *testing.T) {\n\t\topts := DefaultOptions()\n\t\topts.Username = \"testuser\"\n\t\topts.Password = \"testpass\"\n\t\ts, c, _, _ := rawSetup(*opts)\n\t\tdefer c.close()\n\t\tdefer s.Shutdown()\n\n\t\tc.kind = CLIENT\n\t\tconnectArg := []byte(`{\"verbose\":false,\"pedantic\":false,\"user\":\"testuser\",\"pass\":\"testpass\"}`)\n\n\t\terr := c.processConnect(connectArg)\n\t\trequire_NoError(t, err)\n\n\t\tconnStr := c.ncs.Load()\n\t\trequire_NotNil(t, connStr)\n\t\tstr := connStr.(string)\n\t\trequire_Contains(t, str, `\"$G/user:testuser\"`)\n\t})\n\tt.Run(\"token\", func(t *testing.T) {\n\t\topts := DefaultOptions()\n\t\topts.Authorization = \"secret-token\"\n\t\ts, c, _, _ := rawSetup(*opts)\n\t\tdefer c.close()\n\t\tdefer s.Shutdown()\n\n\t\tc.kind = CLIENT\n\t\tconnectArg := []byte(`{\"verbose\":false,\"pedantic\":false,\"auth_token\":\"secret-token\"}`)\n\n\t\terr := c.processConnect(connectArg)\n\t\trequire_NoError(t, err)\n\n\t\tconnStr := c.ncs.Load()\n\t\trequire_NotNil(t, connStr)\n\t\tstr := connStr.(string)\n\t\trequire_Contains(t, str, `\"$G/token\"`)\n\t})\n\tt.Run(\"nkey\", func(t *testing.T) {\n\t\tkp, _ := nkeys.CreateUser()\n\t\tpub, _ := kp.PublicKey()\n\t\tnkey := string(pub)\n\n\t\topts := DefaultOptions()\n\t\topts.Nkeys = []*NkeyUser{{Nkey: nkey}}\n\t\ts, c, _, _ := rawSetup(*opts)\n\t\tdefer c.close()\n\t\tdefer s.Shutdown()\n\n\t\tc.kind = CLIENT\n\t\tnonce := make([]byte, 32)\n\t\tc.nonce = nonce\n\t\tsig, _ := kp.Sign(nonce)\n\t\tsigEncoded := base64.RawURLEncoding.EncodeToString(sig)\n\n\t\tconnectArg := fmt.Sprintf(`{\"verbose\":false,\"pedantic\":false,\"nkey\":\"%s\",\"sig\":\"%s\"}`, nkey, sigEncoded)\n\n\t\terr := c.processConnect([]byte(connectArg))\n\t\trequire_NoError(t, err)\n\n\t\tconnStr := c.ncs.Load()\n\t\trequire_NotNil(t, connStr)\n\t\tstr := connStr.(string)\n\t\trequire_Contains(t, str, fmt.Sprintf(`\"$G/nkey:%s\"`, nkey))\n\t})\n\tt.Run(\"combined_info_and_auth\", func(t *testing.T) {\n\t\topts := DefaultOptions()\n\t\topts.Username = \"testuser\"\n\t\topts.Password = \"testpass\"\n\t\ts, c, _, _ := rawSetup(*opts)\n\t\tdefer c.close()\n\t\tdefer s.Shutdown()\n\n\t\tc.kind = CLIENT\n\t\tconnectArg := []byte(`{\"verbose\":false,\"pedantic\":false,\"user\":\"testuser\",\"pass\":\"testpass\",\"version\":\"1.0.0\",\"lang\":\"go\",\"name\":\"test-client\"}`)\n\n\t\terr := c.processConnect(connectArg)\n\t\trequire_NoError(t, err)\n\n\t\tconnStr := c.ncs.Load()\n\t\trequire_NotNil(t, connStr)\n\t\tstr := connStr.(string)\n\t\trequire_Contains(t, str, `\"v1.0.0:go:test-client\"`)\n\t\trequire_Contains(t, str, `\"$G/user:testuser\"`)\n\t})\n\tt.Run(\"no_auth\", func(t *testing.T) {\n\t\topts := DefaultOptions()\n\t\ts, c, _, _ := rawSetup(*opts)\n\t\tdefer c.close()\n\t\tdefer s.Shutdown()\n\n\t\tc.kind = CLIENT\n\t\tconnectArg := []byte(`{\"verbose\":false,\"pedantic\":false}`)\n\n\t\terr := c.processConnect(connectArg)\n\t\trequire_NoError(t, err)\n\n\t\tconnStr := c.ncs.Load()\n\t\tif connStr != nil {\n\t\t\tstr := connStr.(string)\n\t\t\tif strings.Contains(str, \"$G/\") {\n\t\t\t\tt.Fatalf(\"Expected no auth info when no authentication provided, got: %s\", str)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestClientConfigureWriteTimeoutPolicy(t *testing.T) {\n\tfor name, policy := range map[string]WriteTimeoutPolicy{\n\t\t\"Default\": WriteTimeoutPolicyDefault,\n\t\t\"Retry\": WriteTimeoutPolicyRetry,\n\t\t\"Close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\topts := DefaultOptions()\n\t\t\topts.WriteTimeout = policy\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\t\t\tdefer nc.Close()\n\n\t\t\ts.mu.RLock()\n\t\t\tdefer s.mu.RUnlock()\n\n\t\t\tfor _, r := range s.clients {\n\t\t\t\tif policy == WriteTimeoutPolicyDefault {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, WriteTimeoutPolicyClose)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, policy)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestClientFlushOutboundWriteTimeoutPolicy relies on specifically having\n// written at least one byte in order to not trip the \"written == 0\" close\n// condition, so just setting an unrealistically low write deadline won't\n// work. Instead what we'll do is write the first byte very quickly and then\n// slow down, so that we can trip a more honest slow consumer condition.\ntype writeTimeoutPolicyWriter struct {\n\tnet.Conn\n\tdeadline time.Time\n\twritten int\n}\n\nfunc (w *writeTimeoutPolicyWriter) SetWriteDeadline(deadline time.Time) error {\n\tw.deadline = deadline\n\treturn w.Conn.SetWriteDeadline(deadline)\n}\n\nfunc (w *writeTimeoutPolicyWriter) Write(b []byte) (int, error) {\n\tif w.written == 0 {\n\t\tw.written++\n\t\treturn w.Conn.Write(b[:1])\n\t}\n\ttime.Sleep(time.Until(w.deadline) + 10*time.Millisecond)\n\treturn w.Conn.Write(b)\n}\n\nfunc TestClientFlushOutboundWriteTimeoutPolicy(t *testing.T) {\n\tfor name, policy := range map[string]WriteTimeoutPolicy{\n\t\t\"Retry\": WriteTimeoutPolicyRetry,\n\t\t\"Close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\topts := DefaultOptions()\n\t\t\topts.PingInterval = 250 * time.Millisecond\n\t\t\topts.WriteDeadline = 100 * time.Millisecond\n\t\t\topts.WriteTimeout = policy\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\t\t\tdefer nc1.Close()\n\n\t\t\t_, err := nc1.Subscribe(\"test\", func(_ *nats.Msg) {})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\t\t\tdefer nc2.Close()\n\n\t\t\tcid, err := nc1.GetClientID()\n\t\t\trequire_NoError(t, err)\n\n\t\t\tclient := s.getClient(cid)\n\t\t\tclient.mu.Lock()\n\t\t\tclient.out.wdl = 100 * time.Millisecond\n\t\t\tclient.nc = &writeTimeoutPolicyWriter{Conn: client.nc}\n\t\t\tclient.mu.Unlock()\n\n\t\t\trequire_NoError(t, nc2.Publish(\"test\", make([]byte, 1024*1024)))\n\n\t\t\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tclient.mu.Lock()\n\t\t\t\tdefer client.mu.Unlock()\n\t\t\t\tswitch {\n\t\t\t\tcase !client.flags.isSet(connMarkedClosed):\n\t\t\t\t\treturn fmt.Errorf(\"connection not closed yet\")\n\t\t\t\tcase policy == WriteTimeoutPolicyRetry && client.flags.isSet(isSlowConsumer):\n\t\t\t\t\t// Retry policy should have marked the client as a slow consumer and\n\t\t\t\t\t// continued to retry flushes.\n\t\t\t\t\treturn nil\n\t\t\t\tcase policy == WriteTimeoutPolicyClose && !client.flags.isSet(isSlowConsumer):\n\t\t\t\t\t// Close policy shouldn't have marked the client as a slow consumer,\n\t\t\t\t\t// it will just close it instead.\n\t\t\t\t\treturn nil\n\t\t\t\tdefault:\n\t\t\t\t\treturn fmt.Errorf(\"client not in correct state yet\")\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestFlushOutboundS2CompressionPoolBufferRecycling(t *testing.T) {\n\topts := DefaultOptions()\n\ts := &Server{opts: opts}\n\n\tfakeConn := &testConnWritePartial{}\n\tc := &client{srv: s, nc: fakeConn, kind: ROUTER}\n\tc.initClient()\n\tc.out.cw = s2.NewWriter(nil, s2.WriterConcurrency(1))\n\n\tpayload := make([]byte, 256)\n\tfor i := range payload {\n\t\tpayload[i] = byte(i)\n\t}\n\n\t// Queue multiple buffers per iteration so each leaked pool\n\t// buffer adds one extra allocation, making the leak obvious.\n\tconst numBuffers = 10\n\n\t// Warm up: run a few iterations to populate the pool.\n\tfor i := 0; i < 5; i++ {\n\t\tc.mu.Lock()\n\t\tfor j := 0; j < numBuffers; j++ {\n\t\t\tc.queueOutbound(payload)\n\t\t}\n\t\tc.flushOutbound()\n\t\tc.mu.Unlock()\n\t}\n\n\tallocs := testing.AllocsPerRun(100, func() {\n\t\tc.mu.Lock()\n\t\tfor j := 0; j < numBuffers; j++ {\n\t\t\tc.queueOutbound(payload)\n\t\t}\n\t\tc.flushOutbound()\n\t\tc.mu.Unlock()\n\t})\n\tif allocs > 15 {\n\t\tt.Fatalf(\"Too many allocs per iteration (%.1f); pool buffers are likely being leaked\", allocs)\n\t}\n}\n" }, { "path": "server/closed_conns_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc checkClosedConns(t *testing.T, s *Server, num int, wait time.Duration) {\n\tt.Helper()\n\tcheckFor(t, wait, 5*time.Millisecond, func() error {\n\t\tif nc := s.numClosedConns(); nc != num {\n\t\t\treturn fmt.Errorf(\"Closed conns expected to be %v, got %v\", num, nc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkTotalClosedConns(t *testing.T, s *Server, num uint64, wait time.Duration) {\n\tt.Helper()\n\tcheckFor(t, wait, 5*time.Millisecond, func() error {\n\t\tif nc := s.totalClosedConns(); nc != num {\n\t\t\treturn fmt.Errorf(\"Total closed conns expected to be %v, got %v\", num, nc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestClosedConnsAccounting(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxClosedClients = 10\n\topts.NoSystemAccount = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\twait := time.Second\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tid, _ := nc.GetClientID()\n\tnc.Close()\n\n\tcheckClosedConns(t, s, 1, wait)\n\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tif conns[0].Cid != id {\n\t\tt.Fatalf(\"Expected CID to be %d, got %d\\n\", id, conns[0].Cid)\n\t}\n\n\t// Now create 21 more\n\tfor i := 0; i < 21; i++ {\n\t\tnc, err = nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tnc.Close()\n\t\tcheckTotalClosedConns(t, s, uint64(i+2), wait)\n\t}\n\n\tcheckClosedConns(t, s, opts.MaxClosedClients, wait)\n\tcheckTotalClosedConns(t, s, 22, wait)\n\n\tconns = s.closedClients()\n\tif lc := len(conns); lc != opts.MaxClosedClients {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\",\n\t\t\topts.MaxClosedClients, lc)\n\t}\n\n\t// Set it to the start after overflow.\n\tcid := uint64(22 - opts.MaxClosedClients)\n\tfor _, ci := range conns {\n\t\tcid++\n\t\tif ci.Cid != cid {\n\t\t\tt.Fatalf(\"Expected cid of %d, got %d\\n\", cid, ci.Cid)\n\t\t}\n\t}\n}\n\nfunc TestClosedConnsSubsAccounting(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Now create some subscriptions\n\tnumSubs := 10\n\tfor i := 0; i < numSubs; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\tnc.Subscribe(subj, func(m *nats.Msg) {})\n\t}\n\tnc.Flush()\n\tnc.Close()\n\n\tcheckClosedConns(t, s, 1, 20*time.Millisecond)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be 1, got %d\\n\", lc)\n\t}\n\tci := conns[0]\n\n\tif len(ci.subs) != numSubs {\n\t\tt.Fatalf(\"Expected number of Subs to be %d, got %d\\n\", numSubs, len(ci.subs))\n\t}\n}\n\nfunc checkReason(t *testing.T, reason string, expected ClosedState) {\n\tif !strings.Contains(reason, expected.String()) {\n\t\tt.Fatalf(\"Expected closed connection with `%s` state, got `%s`\\n\",\n\t\t\texpected, reason)\n\t}\n}\n\nfunc TestClosedAuthorizationTimeout(t *testing.T) {\n\tserverOptions := DefaultOptions()\n\tserverOptions.Authorization = \"my_token\"\n\tserverOptions.AuthTimeout = 0.4\n\ts := RunServer(serverOptions)\n\tdefer s.Shutdown()\n\n\tconn, err := net.Dial(\"tcp\", net.JoinHostPort(serverOptions.Host, fmt.Sprintf(\"%d\", serverOptions.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error dialing server: %v\\n\", err)\n\t}\n\tdefer conn.Close()\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, AuthenticationTimeout)\n}\n\nfunc TestClosedAuthorizationViolation(t *testing.T) {\n\tserverOptions := DefaultOptions()\n\tserverOptions.Authorization = \"my_token\"\n\ts := RunServer(serverOptions)\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected failure for connection\")\n\t}\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, AuthenticationViolation)\n}\n\nfunc TestClosedUPAuthorizationViolation(t *testing.T) {\n\tserverOptions := DefaultOptions()\n\tserverOptions.Username = \"my_user\"\n\tserverOptions.Password = \"my_secret\"\n\ts := RunServer(serverOptions)\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected failure for connection\")\n\t}\n\n\turl2 := fmt.Sprintf(\"nats://my_user:wrong_pass@%s:%d\", opts.Host, opts.Port)\n\tnc, err = nats.Connect(url2)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected failure for connection\")\n\t}\n\n\tcheckClosedConns(t, s, 2, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 2 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 2, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, AuthenticationViolation)\n\tcheckReason(t, conns[1].Reason, AuthenticationViolation)\n}\n\nfunc TestClosedMaxPayload(t *testing.T) {\n\tserverOptions := DefaultOptions()\n\tserverOptions.MaxPayload = 100\n\n\ts := RunServer(serverOptions)\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\tendpoint := net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port))\n\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not make a raw connection to the server: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\t// This should trigger it.\n\tpub := \"PUB foo.bar 1024\\r\\n\"\n\tconn.Write([]byte(pub))\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, MaxPayloadExceeded)\n}\n\nfunc TestClosedTLSHandshake(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/tls.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.TLSVerify = true\n\topts.NoLog = true\n\topts.NoSigs = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected failure for connection\")\n\t}\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, TLSHandshakeError)\n}\n" }, { "path": "server/config_check_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestConfigCheck(t *testing.T) {\n\ttests := []struct {\n\t\t// name is the name of the test.\n\t\tname string\n\n\t\t// config is content of the configuration file.\n\t\tconfig string\n\n\t\t// warningErr is an error that does not prevent server from starting.\n\t\twarningErr error\n\n\t\t// errorLine is the location of the error.\n\t\terrorLine int\n\n\t\t// errorPos is the position of the error.\n\t\terrorPos int\n\n\t\t// warning errors also include a reason optionally.\n\t\treason string\n\n\t\t// newDefaultErr is a configuration error that includes source of error.\n\t\terr error\n\t}{\n\t\t{\n\t\t\tname: \"when unknown field is used at top level\",\n\t\t\tconfig: `\n monitor = \"127.0.0.1:4442\"\n `,\n\t\t\terr: errors.New(`unknown field \"monitor\"`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 17,\n\t\t},\n\t\t{\n\t\t\tname: \"when default permissions are used at top level\",\n\t\t\tconfig: `\n \"default_permissions\" {\n publish = [\"_SANDBOX.>\"]\n subscribe = [\"_SANDBOX.>\"]\n }\n `,\n\t\t\terr: errors.New(`unknown field \"default_permissions\"`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 18,\n\t\t},\n\t\t{\n\t\t\tname: \"when authorization config is empty\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t}\n\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when authorization config has unknown fields\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t foo = \"bar\"\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`unknown field \"foo\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when authorization config has unknown fields\",\n\t\t\tconfig: `\n\t\tport = 4222\n\n\t\tauthorization = {\n\t\t user = \"hello\"\n\t\t foo = \"bar\"\n\t\t password = \"world\"\n\t\t}\n\n\t\t`,\n\t\t\terr: errors.New(`unknown field \"foo\"`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has unknown fields\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = [\n\t\t {\n\t\t user = \"foo\"\n\t\t pass = \"bar\"\n\t\t token = \"quux\"\n\t\t }\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`unknown field \"token\"`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has unknown fields\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t subscribe = {}\n\t\t inboxes = {}\n\t\t publish = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field \"inboxes\" parsing permissions`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has unknown fields within allow or deny\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t subscribe = {\n\t\t allow = [\"hello\", \"world\"]\n\t\t deny = [\"foo\", \"bar\"]\n\t\t denied = \"_INBOX.>\"\n\t\t }\n\t\t publish = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field name \"denied\" parsing subject permissions, only 'allow' or 'deny' are permitted`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has unknown fields within allow or deny\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t publish = {\n\t\t allow = [\"hello\", \"world\"]\n\t\t deny = [\"foo\", \"bar\"]\n\t\t allowed = \"_INBOX.>\"\n\t\t }\n\t\t subscribe = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field name \"allowed\" parsing subject permissions, only 'allow' or 'deny' are permitted`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has unknown fields using arrays\",\n\t\t\tconfig: `\n\t\tauthorization {\n\n\t\t default_permissions {\n\t\t subscribe = [\"a\"]\n\t\t publish = [\"b\"]\n\t\t inboxes = [\"c\"]\n\t\t }\n\n\t\t users = [\n\t\t {\n\t\t user = \"foo\"\n\t\t pass = \"bar\"\n\t\t }\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field \"inboxes\" parsing permissions`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has unknown fields using strings\",\n\t\t\tconfig: `\n\t\tauthorization {\n\n\t\t default_permissions {\n\t\t subscribe = \"a\"\n\t\t requests = \"b\"\n\t\t publish = \"c\"\n\t\t }\n\n\t\t users = [\n\t\t {\n\t\t user = \"foo\"\n\t\t pass = \"bar\"\n\t\t }\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field \"requests\" parsing permissions`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config is empty\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = [\n\t\t {\n\t\t user = \"foo\", pass = \"bar\", permissions = {\n\t\t }\n\t\t }\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown permissions are included in user config\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = [\n\t\t {\n\t\t user = \"foo\", pass = \"bar\", permissions {\n\t\t inboxes = true\n\t\t }\n\t\t }\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Unknown field \"inboxes\" parsing permissions`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 11,\n\t\t},\n\t\t{\n\t\t\tname: \"when clustering config is empty\",\n\t\t\tconfig: `\n\t\tcluster = {\n\t\t}\n\t\t`,\n\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown option is in clustering config\",\n\t\t\tconfig: `\n\t\t# NATS Server Configuration\n\t\tport = 4222\n\n\t\tcluster = {\n\n\t\t port = 6222\n\n\t\t foo = \"bar\"\n\n\t\t authorization {\n\t\t user = \"hello\"\n\t\t pass = \"world\"\n\t\t }\n\n\t\t}\n\t\t`,\n\n\t\t\terr: errors.New(`unknown field \"foo\"`),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown option is in clustering authorization config\",\n\t\t\tconfig: `\n\t\tcluster = {\n\t\t authorization {\n\t\t foo = \"bar\"\n\t\t }\n\t\t}\n\t\t`,\n\n\t\t\terr: errors.New(`unknown field \"foo\"`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown option is in tls config\",\n\t\t\tconfig: `\n\t\ttls = {\n\t\t hello = \"world\"\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`error parsing tls config, unknown field \"hello\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown option is in cluster tls config\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t tls = {\n\t\t foo = \"bar\"\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`error parsing tls config, unknown field \"foo\"`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"when using cipher suites in the TLS config\",\n\t\t\tconfig: `\n\t\ttls = {\n\t\t cipher_suites: [\n\t\t\t\"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\",\n\t\t\t\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"\n\t\t ]\n\t\t preferences = []\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`error parsing tls config, unknown field \"preferences\"`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"when using curve preferences in the TLS config\",\n\t\t\tconfig: `\n\t\ttls = {\n\t\t curve_preferences: [\n\t\t\t\"CurveP256\",\n\t\t\t\"CurveP384\",\n\t\t\t\"CurveP521\"\n\t\t ]\n\t\t suites = []\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`error parsing tls config, unknown field \"suites\"`),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"when using curve preferences in the TLS config\",\n\t\t\tconfig: `\n\t\ttls = {\n\t\t curve_preferences: [\n\t\t\t\"CurveP5210000\"\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`unrecognized curve preference CurveP5210000`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"verify_cert_and_check_known_urls not support for clients\",\n\t\t\tconfig: `\n\t\ttls = {\n\t\t\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/key.pem\"\n\t\t\t\t\t verify_cert_and_check_known_urls: true\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(\"verify_cert_and_check_known_urls not supported in this context\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 10,\n\t\t},\n\t\t{\n\t\t\tname: \"when unknown option is in cluster config with defined routes\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t port = 6222\n\t\t routes = [\n\t\t nats://127.0.0.1:6222\n\t\t ]\n\t\t peers = []\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`unknown field \"peers\"`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when used as variable in authorization block it should not be considered as unknown field\",\n\t\t\tconfig: `\n\t\t# listen: 127.0.0.1:-1\n\t\tlisten: 127.0.0.1:4222\n\n\t\tauthorization {\n\t\t # Superuser can do anything.\n\t\t super_user = {\n\t\t publish = \">\"\n\t\t subscribe = \">\"\n\t\t }\n\n\t\t # Can do requests on foo or bar, and subscribe to anything\n\t\t # that is a response to an _INBOX.\n\t\t #\n\t\t # Notice that authorization filters can be singletons or arrays.\n\t\t req_pub_user = {\n\t\t publish = [\"req.foo\", \"req.bar\"]\n\t\t subscribe = \"_INBOX.>\"\n\t\t }\n\n\t\t # Setup a default user that can subscribe to anything, but has\n\t\t # no publish capabilities.\n\t\t default_user = {\n\t\t subscribe = \"PUBLIC.>\"\n\t\t }\n\n\t\t unused = \"hello\"\n\n\t\t # Default permissions if none presented. e.g. susan below.\n\t\t default_permissions: $default_user\n\n\t\t # Users listed with persmissions.\n\t\t users = [\n\t\t {user: alice, password: foo, permissions: $super_user}\n\t\t {user: bob, password: bar, permissions: $req_pub_user}\n\t\t {user: susan, password: baz}\n\t\t ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`unknown field \"unused\"`),\n\t\t\terrorLine: 27,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when used as variable in top level config it should not be considered as unknown field\",\n\t\t\tconfig: `\n\t\tmonitoring_port = 8222\n\n\t\thttp_port = $monitoring_port\n\n\t\tport = 4222\n\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when used as variable in cluster config it should not be considered as unknown field\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t clustering_port = 6222\n\t\t port = $clustering_port\n\t\t}\n\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting permissions within cluster authorization block\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t authorization {\n\t\t permissions = {\n\t\t publish = { allow = [\"foo\", \"bar\"] }\n\t\t }\n\t\t }\n\n\t\t permissions = {\n\t\t publish = { deny = [\"foo\", \"bar\"] }\n\t\t }\n\t\t}\n\t\t`,\n\t\t\twarningErr: errors.New(`invalid use of field \"authorization\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t\treason: `setting \"permissions\" within cluster authorization block is deprecated`,\n\t\t},\n\t\t{\n\t\t\tname: \"when write deadline is used with deprecated usage\",\n\t\t\tconfig: `\n write_deadline = 100\n\t\t`,\n\t\t\twarningErr: errors.New(`invalid use of field \"write_deadline\"`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 17,\n\t\t\treason: `write_deadline should be converted to a duration`,\n\t\t},\n\t\t/////////////////////\n\t\t// ACCOUNTS\t //\n\t\t/////////////////////\n\t\t{\n\t\t\tname: \"when accounts block is correctly configured\",\n\t\t\tconfig: `\n\t\thttp_port = 8222\n\n\t\taccounts {\n\n\t\t #\n\t\t # synadia > nats.io, cncf\n\t\t #\n\t\t synadia {\n\t\t # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA\n\t\t nkey = \"AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL\"\n\n\t\t users [\n\t\t {\n\t\t # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM\n\t\t nkey = \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t }\n\t\t ]\n\n\t\t exports = [\n\t\t { service: \"synadia.requests\", accounts: [nats, cncf] }\n\t\t ]\n\t\t }\n\n\t\t #\n\t\t # nats < synadia\n\t\t #\n\t\t nats {\n\t\t # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE\n\t\t nkey = \"ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS\"\n\n\t\t users [\n\t\t {\n\t\t # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM\n\t\t nkey = \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"\n\t\t }\n\t\t ]\n\n\t\t imports = [\n\t\t # This account has to send requests to 'nats.requests' subject\n\t\t { service: { account: \"synadia\", subject: \"synadia.requests\" }, to: \"nats.requests\" }\n\t\t ]\n\t\t }\n\n\t\t #\n\t\t # cncf < synadia\n\t\t #\n\t\t cncf {\n\t\t # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4\n\t\t nkey = \"AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT\"\n\n\t\t users [\n\t\t {\n\t\t # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ\n\t\t nkey = \"UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ\"\n\t\t }\n\t\t ]\n\n\t\t imports = [\n\t\t # This account has to send requests to 'synadia.requests' subject\n\t\t { service: { account: \"synadia\", subject: \"synadia.requests\" } }\n\t\t ]\n\t\t }\n\t\t}\n\t\t\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when nkey is invalid within accounts block\",\n\t\t\tconfig: `\n\t\taccounts {\n\n\t\t #\n\t\t # synadia > nats.io, cncf\n\t\t #\n\t\t synadia {\n\t\t # SAADJL5XAEM6BDYSWDTGVILJVY54CQXZM5ZLG4FRUAKB62HWRTPNSGXOHA\n\t\t nkey = \"AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL\"\n\n\t\t users [\n\t\t {\n\t\t # SUAEL6RU3BSDAFKOHNTEOK5Q6FTM5FTAMWVIKBET6FHPO4JRII3CYELVNM\n\t\t nkey = \"SCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t }\n\t\t ]\n\n\t\t exports = [\n\t\t { service: \"synadia.requests\", accounts: [nats, cncf] }\n\t\t ]\n\t\t }\n\n\t\t #\n\t\t # nats < synadia\n\t\t #\n\t\t nats {\n\t\t # SUAJTM55JH4BNYDA22DMDZJSRBRKVDGSLYK2HDIOCM3LPWCDXIDV5Q4CIE\n\t\t nkey = \"ADRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS\"\n\n\t\t users [\n\t\t {\n\t\t # SUADZTYQAKTY5NQM7XRB5XR3C24M6ROGZLBZ6P5HJJSSOFUGC5YXOOECOM\n\t\t nkey = \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"\n\t\t }\n\t\t ]\n\n\t\t imports = [\n\t\t # This account has to send requests to 'nats.requests' subject\n\t\t { service: { account: \"synadia\", subject: \"synadia.requests\" }, to: \"nats.requests\" }\n\t\t ]\n\t\t }\n\n\t\t #\n\t\t # cncf < synadia\n\t\t #\n\t\t cncf {\n\t\t # SAAFHDZX7SGZ2SWHPS22JRPPK5WX44NPLNXQHR5C5RIF6QRI3U65VFY6C4\n\t\t nkey = \"AD4YRVUJF2KASKPGRMNXTYKIYSCB3IHHB4Y2ME6B2PDIV5QJ23C2ZRIT\"\n\n\t\t users [\n\t\t {\n\t\t # SUAKINP3Z2BPUXWOFSW2FZC7TFJCMMU7DHKP2C62IJQUDASOCDSTDTRMJQ\n\t\t nkey = \"UB57IEMPG4KOTPFV5A66QKE2HZ3XBXFHVRCCVMJEWKECMVN2HSH3VTSJ\"\n\t\t }\n\t\t ]\n\n\t\t imports = [\n\t\t # This account has to send requests to 'synadia.requests' subject\n\t\t { service: { account: \"synadia\", subject: \"synadia.requests\" } }\n\t\t ]\n\t\t }\n\t\t}\n\t\t\t\t`,\n\t\t\terr: errors.New(`Not a valid public nkey for a user`),\n\t\t\terrorLine: 14,\n\t\t\terrorPos: 11,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block has unknown fields\",\n\t\t\tconfig: `\n\t\thttp_port = 8222\n\n\t\taccounts {\n foo = \"bar\"\n\t\t}`,\n\t\t\terr: errors.New(`Expected map entries for accounts`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 19,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts has a referenced config variable within same block\",\n\t\t\tconfig: `\n\t\t\t accounts {\n\t\t\t PERMISSIONS = {\n\t\t\t\tpublish = {\n\t\t\t\t allow = [\"foo\",\"bar\"]\n\t\t\t\t deny = [\"quux\"]\n\t\t\t\t}\n\t\t\t }\n\n\t\t\t synadia {\n\t\t\t\tnkey = \"AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL\"\n\n\t\t\t\tusers [\n\t\t\t\t {\n\t\t\t\t nkey = \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t permissions = $PERMISSIONS\n\t\t\t\t }\n\t\t\t\t]\n\t\t\t\texports = [\n\t\t\t\t { stream: \"synadia.>\" }\n\t\t\t\t]\n\t\t\t }\n\t\t\t }`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts has an unreferenced config variables within same block\",\n\t\t\tconfig: `\n\t\t\t accounts {\n\t\t\t PERMISSIONS = {\n\t\t\t\tpublish = {\n\t\t\t\t allow = [\"foo\",\"bar\"]\n\t\t\t\t deny = [\"quux\"]\n\t\t\t\t}\n\t\t\t }\n\n\t\t\t synadia {\n\t\t\t\tnkey = \"AC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL\"\n\n\t\t\t\tusers [\n\t\t\t\t {\n\t\t\t\t nkey = \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t }\n\t\t\t\t]\n\t\t\t\texports = [\n\t\t\t\t { stream: \"synadia.>\" }\n\t\t\t\t]\n\t\t\t }\n\t\t\t }`,\n\t\t\terr: errors.New(`unknown field \"publish\"`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block defines a global account\",\n\t\t\tconfig: `\n\t\thttp_port = 8222\n\n\t\taccounts {\n $G = {\n }\n\t\t}\n\t\t\t\t`,\n\t\t\terr: errors.New(`\"$G\" is a Reserved Account`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 19,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block uses an invalid public key\",\n\t\t\tconfig: `\n\t\taccounts {\n synadia = {\n nkey = \"invalid\"\n }\n\t\t}\n\t\t\t\t`,\n\t\t\terr: errors.New(`Not a valid public nkey for an account: \"invalid\"`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts list includes reserved account\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = [foo, bar, \"$G\"]\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`\"$G\" is a Reserved Account`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 26,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts list includes a dupe entry\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = [foo, bar, bar]\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Duplicate Account Entry: bar`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block includes a dupe user\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n users = [\n { user: \"foo\", pass: \"bar\" },\n { user: \"hello\", pass: \"world\" },\n { user: \"foo\", pass: \"bar\" }\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Duplicate user \"foo\" detected`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block imports are not a list\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n imports = true\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Imports should be an array, got bool`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block exports are not a list\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n exports = true\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Exports should be an array, got bool`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block imports items are not a map\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n imports = [\n false\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Import Items should be a map with type entry, got bool`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 23,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts block export items are not a map\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n exports = [\n false\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Export Items should be a map with type entry, got bool`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 23,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts exports has a stream name that is not a string\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n exports = [\n {\n stream: false\n }\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Expected stream name to be string, got bool`),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts exports has a service name that is not a string\",\n\t\t\tconfig: `\n\t\taccounts = {\n nats {\n exports = [\n {\n service: false\n }\n ]\n }\n }\n\t\t\t\t`,\n\t\t\terr: errors.New(`Expected service name to be string, got bool`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts imports stream without name\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n imports = [\n { stream: { }}\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Expect an account name and a subject`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when accounts imports service without name\",\n\t\t\tconfig: `\n port = 4222\n\n\t\taccounts = {\n nats {\n imports = [\n { service: { }}\n ]\n }\n }\n\n http_port = 8222\n\t\t\t\t`,\n\t\t\terr: errors.New(`Expect an account name and a subject`),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when account trace destination is of the wrong type\",\n\t\t\tconfig: `\n accounts {\n A { trace_dest: 123 }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Expected account message trace \"trace_dest\" to be a string or a map/struct, got int64`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 23,\n\t\t},\n\t\t{\n\t\t\tname: \"when account trace destination is not a valid destination\",\n\t\t\tconfig: `\n accounts {\n A { trace_dest: \"invalid..dest\" }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Trace destination \"invalid..dest\" is not valid`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 23,\n\t\t},\n\t\t{\n\t\t\tname: \"when account trace destination is not a valid publish subject\",\n\t\t\tconfig: `\n accounts {\n A { trace_dest: \"invalid.publish.*.subject\" }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Trace destination \"invalid.publish.*.subject\" is not valid`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 23,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace dest is wrong type\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: 123} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Field \"dest\" should be a string, got int64`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 35,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace dest is invalid\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"invalid..dest\"} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Trace destination \"invalid..dest\" is not valid`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 35,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace sampling is wrong type\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"acc.dest\", sampling: {wront: \"type\"}} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Trace destination sampling field \"sampling\" should be an integer or a percentage, got map[string]interface {}`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace sampling is wrong string\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"acc.dest\", sampling: abc%} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Invalid trace destination sampling value \"abc%\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace sampling is negative\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"acc.dest\", sampling: -1} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Ttrace destination sampling value -1 is invalid, needs to be [1..100]`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace sampling is zero\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"acc.dest\", sampling: 0} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Ttrace destination sampling value 0 is invalid, needs to be [1..100]`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace sampling is more than 100\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {dest: \"acc.dest\", sampling: 101} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Ttrace destination sampling value 101 is invalid, needs to be [1..100]`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"when account message trace has unknown field\",\n\t\t\tconfig: `\n accounts {\n A { msg_trace: {wrong: \"field\"} }\n }\n\t\t\t`,\n\t\t\terr: errors.New(`Unknown field \"wrong\" parsing account message trace map/struct \"msg_trace\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 35,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has both token and users\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t\ttoken = \"s3cr3t\"\n\t\t\tusers = [\n\t\t\t\t{\n\t\t\t\t\tuser = \"foo\"\n\t\t\t\t\tpass = \"bar\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Can not have a token and a users array`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has both token and user\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t\tuser = \"foo\"\n\t\t\tpass = \"bar\"\n\t\t\ttoken = \"baz\"\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Cannot have a user/pass and token`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has both user and users array\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t\tuser = \"user1\"\n\t\t\tpass = \"pwd1\"\n\t\t\tusers = [\n\t\t\t\t{\n\t\t\t\t\tuser = \"user2\"\n\t\t\t\t\tpass = \"pwd2\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Can not have a single user/pass and a users array`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization has duplicate users\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t\tusers = [\n\t\t\t\t{user: \"user1\", pass: \"pwd\"}\n\t\t\t\t{user: \"user2\", pass: \"pwd\"}\n\t\t\t\t{user: \"user1\", pass: \"pwd\"}\n\t\t\t]\n\t\t}\n\t\t`,\n\t\t\terr: fmt.Errorf(`Duplicate user %q detected`, \"user1\"),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization has duplicate nkeys\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t\tusers = [\n\t\t\t\t{nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX }\n\t\t\t\t{nkey: UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E }\n\t\t\t\t{nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX }\n\t\t\t]\n\t\t}\n\t\t`,\n\t\t\terr: fmt.Errorf(`Duplicate nkey %q detected`, \"UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX\"),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has users not as a list\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = false\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Expected users field to be an array, got false`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has users not as a map\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = [false]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Expected user entry to be a map/struct, got false`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 14,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization config has permissions not as a map\",\n\t\t\tconfig: `\n\t\tauthorization = {\n\t\t users = [{user: hello, pass: world}]\n permissions = false\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Expected permissions to be a map/struct, got false`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 19,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has invalid fields within allow\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t publish = {\n\t\t allow = [false, \"hello\", \"world\"]\n\t\t deny = [\"foo\", \"bar\"]\n\t\t }\n\t\t subscribe = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Subject in permissions array cannot be cast to string`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 18,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has invalid fields within deny\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t publish = {\n\t\t allow = [\"hello\", \"world\"]\n\t\t deny = [true, \"foo\", \"bar\"]\n\t\t }\n\t\t subscribe = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Subject in permissions array cannot be cast to string`),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 17,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions config has invalid type\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t publish = {\n\t\t allow = false\n\t\t }\n\t\t subscribe = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Expected subject permissions to be a subject, or array of subjects, got bool`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"when user authorization permissions subject is invalid\",\n\t\t\tconfig: `\n\t\tauthorization {\n\t\t permissions {\n\t\t publish = {\n\t\t allow = [\"foo..bar\"]\n\t\t }\n\t\t subscribe = {}\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`subject \"foo..bar\" is not a valid subject`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"when cluster config listen is invalid\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t listen = \"0.0.0.0:XXXX\"\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`could not parse port \"XXXX\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when cluster config includes multiple users\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t authorization {\n users = []\n }\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`Cluster authorization does not allow multiple users`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when cluster routes are invalid\",\n\t\t\tconfig: `\n\t\tcluster {\n routes = [\n \"0.0.0.0:XXXX\"\n # \"0.0.0.0:YYYY\"\n # \"0.0.0.0:ZZZZ\"\n ]\n\t\t}\n\t\t`,\n\t\t\terr: errors.New(`error parsing route url [\"0.0.0.0:XXXX\"]`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 22,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting invalid TLS config within cluster block\",\n\t\t\tconfig: `\n\t\tcluster {\n\t\t tls {\n\t\t }\n\t\t}\n\t\t`,\n\t\t\terr: nil,\n\t\t\terrorLine: 0,\n\t\t\terrorPos: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid lame_duck_duration type\",\n\t\t\tconfig: `\n\t\t\t\tlame_duck_duration: abc\n\t\t\t`,\n\t\t\terr: errors.New(`error parsing lame_duck_duration: time: invalid duration`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"lame_duck_duration too small\",\n\t\t\tconfig: `\n\t\t\t\tlame_duck_duration: \"5s\"\n\t\t\t`,\n\t\t\terr: errors.New(`invalid lame_duck_duration of 5s, minimum is 30 seconds`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid lame_duck_grace_period type\",\n\t\t\tconfig: `\n\t\t\t\tlame_duck_grace_period: abc\n\t\t\t`,\n\t\t\terr: errors.New(`error parsing lame_duck_grace_period: time: invalid duration`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"lame_duck_grace_period should be positive\",\n\t\t\tconfig: `\n\t\t\t\tlame_duck_grace_period: \"-5s\"\n\t\t\t`,\n\t\t\terr: errors.New(`invalid lame_duck_grace_period, needs to be positive`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when only setting TLS timeout for a leafnode remote\",\n\t\t\tconfig: `\n\t\tleafnodes {\n\t\t remotes = [\n\t\t {\n\t\t url: \"tls://nats:7422\"\n\t\t tls {\n\t\t timeout: 0.01\n\t\t }\n\t\t }\n\t\t ]\n\t\t}`,\n\t\t\terr: nil,\n\t\t\terrorLine: 0,\n\t\t\terrorPos: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"verify_cert_and_check_known_urls do not work for leaf nodes\",\n\t\t\tconfig: `\n\t\tleafnodes {\n\t\t remotes = [\n\t\t {\n\t\t url: \"tls://nats:7422\"\n\t\t tls {\n\t\t timeout: 0.01\n\t\t\t\tverify_cert_and_check_known_urls: true\n\t\t }\n\t\t }\n\t\t ]\n\t\t}`,\n\t\t\t//Unexpected error after processing config: /var/folders/9h/6g_c9l6n6bb8gp331d_9y0_w0000gn/T/057996446:8:5:\n\t\t\terr: errors.New(\"verify_cert_and_check_known_urls not supported in this context\"),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when leafnode remotes use wrong type\",\n\t\t\tconfig: `\n\t\tleafnodes {\n\t\t remotes: {\n \t url: \"tls://nats:7422\"\n\t\t }\n\t\t}`,\n\t\t\terr: errors.New(`Expected remotes field to be an array, got map[string]interface {}`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"when leafnode remotes url uses wrong type\",\n\t\t\tconfig: `\n\t\tleafnodes {\n\t\t remotes: [\n \t { urls: 1234 }\n\t\t ]\n\t\t}`,\n\t\t\terr: errors.New(`Expected remote leafnode url to be an array or string, got 1234`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 18,\n\t\t},\n\t\t{\n\t\t\tname: \"when leafnode min_version is wrong type\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tmin_version = 123\n\t\t\t\t}`,\n\t\t\terr: errors.New(`interface conversion: interface {} is int64, not string`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"when leafnode min_version has parsing error\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tmin_version = bad.version\n\t\t\t\t}`,\n\t\t\terr: errors.New(`invalid leafnode's minimum version: invalid semver`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"when leafnode min_version is too low\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tmin_version = 2.7.9\n\t\t\t\t}`,\n\t\t\terr: errors.New(`the minimum version should be at least 2.8.0`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting latency tracking with a system account\",\n\t\t\tconfig: `\n system_account: sys\n\n accounts {\n sys { users = [ {user: sys, pass: \"\" } ] }\n\n nats.io: {\n users = [ { user : bar, pass: \"\" } ]\n\n exports = [\n { service: \"nats.add\"\n response: singleton\n latency: {\n sampling: 100%\n subject: \"latency.tracking.add\"\n }\n }\n\n ]\n }\n }\n `,\n\t\t\terr: nil,\n\t\t\terrorLine: 0,\n\t\t\terrorPos: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting latency tracking with an invalid publish subject\",\n\t\t\tconfig: `\n system_account = sys\n accounts {\n sys { users = [ {user: sys, pass: \"\" } ] }\n\n nats.io: {\n users = [ { user : bar, pass: \"\" } ]\n\n exports = [\n { service: \"nats.add\"\n response: singleton\n latency: \"*\"\n }\n ]\n }\n }\n `,\n\t\t\terr: errors.New(`Error adding service latency sampling for \"nats.add\" on subject \"*\": invalid publish subject`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 17,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting latency tracking on a stream\",\n\t\t\tconfig: `\n system_account = sys\n accounts {\n sys { users = [ {user: sys, pass: \"\" } ] }\n\n nats.io: {\n users = [ { user : bar, pass: \"\" } ]\n\n exports = [\n { stream: \"nats.add\"\n latency: \"foo\"\n }\n ]\n }\n }\n `,\n\t\t\terr: errors.New(`Detected latency directive on non-service`),\n\t\t\terrorLine: 11,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting allow_trace on a stream export (after)\",\n\t\t\tconfig: `\n system_account = sys\n accounts {\n sys { users = [ {user: sys, pass: \"\" } ] }\n\n nats.io: {\n users = [ { user : bar, pass: \"\" } ]\n exports = [ { stream: \"nats.add\", allow_trace: true } ]\n }\n }\n `,\n\t\t\terr: errors.New(`Detected allow_trace directive on non-service`),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 55,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting allow_trace on a stream export (before)\",\n\t\t\tconfig: `\n system_account = sys\n accounts {\n sys { users = [ {user: sys, pass: \"\" } ] }\n\n nats.io: {\n users = [ { user : bar, pass: \"\" } ]\n exports = [ { allow_trace: true, stream: \"nats.add\" } ]\n }\n }\n `,\n\t\t\terr: errors.New(`Detected allow_trace directive on non-service`),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 35,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting allow_trace on a service import (after)\",\n\t\t\tconfig: `\n accounts {\n A: {\n users = [ {user: user1, pass: \"\"} ]\n exports = [{service: \"foo\"}]\n }\n B: {\n users = [ {user: user2, pass: \"\"} ]\n imports = [ { service: {account: \"A\", subject: \"foo\"}, allow_trace: true } ]\n }\n }\n `,\n\t\t\terr: errors.New(`Detected allow_trace directive on a non-stream`),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 76,\n\t\t},\n\t\t{\n\t\t\tname: \"when setting allow_trace on a service import (before)\",\n\t\t\tconfig: `\n accounts {\n A: {\n users = [ {user: user1, pass: \"\"} ]\n exports = [{service: \"foo\"}]\n }\n B: {\n users = [ {user: user2, pass: \"\"} ]\n imports = [ { allow_trace: true, service: {account: \"A\", subject: \"foo\"} } ]\n }\n }\n `,\n\t\t\terr: errors.New(`Detected allow_trace directive on a non-stream`),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 35,\n\t\t},\n\t\t{\n\t\t\tname: \"when using duplicate service import subject\",\n\t\t\tconfig: `\n\t\t\t\t\t\t\t\taccounts {\n\t\t\t\t\t\t\t\t\t A: {\n\t\t\t\t\t\t\t\t\t\t users = [ {user: user1, pass: \"\"} ]\n\t\t\t\t\t\t\t\t\t\t exports = [\n\t\t\t\t\t\t\t\t\t\t\t {service: \"remote1\"}\n\t\t\t\t\t\t\t\t\t\t\t {service: \"remote2\"}\n\t\t\t\t\t\t\t\t\t\t ]\n\t\t\t\t\t\t\t\t\t }\n\t\t\t\t\t\t\t\t\t B: {\n\t\t\t\t\t\t\t\t\t\t users = [ {user: user2, pass: \"\"} ]\n\t\t\t\t\t\t\t\t\t\t imports = [\n\t\t\t\t\t\t\t\t\t\t\t {service: {account: \"A\", subject: \"remote1\"}, to: \"local\"}\n\t\t\t\t\t\t\t\t\t\t\t {service: {account: \"A\", subject: \"remote2\"}, to: \"local\"}\n\t\t\t\t\t\t\t\t\t\t ]\n\t\t\t\t\t\t\t\t\t }\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t`,\n\t\t\terr: errors.New(`Duplicate service import subject \"local\", previously used in import for account \"A\", subject \"remote1\"`),\n\t\t\terrorLine: 14,\n\t\t\terrorPos: 71,\n\t\t},\n\t\t{\n\t\t\tname: \"mixing single and multi users in leafnode authorization\",\n\t\t\tconfig: `\n leafnodes {\n authorization {\n user: user1\n password: pwd\n users = [{user: user2, password: pwd}]\n }\n }\n `,\n\t\t\terr: errors.New(\"can not have a single user/pass and a users array\"),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 20,\n\t\t},\n\t\t{\n\t\t\tname: \"duplicate usernames in leafnode authorization\",\n\t\t\tconfig: `\n leafnodes {\n authorization {\n users = [\n {user: user, password: pwd}\n {user: user, password: pwd}\n ]\n }\n }\n `,\n\t\t\terr: errors.New(`duplicate user \"user\" detected in leafnode authorization`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"mqtt bad type\",\n\t\t\tconfig: `\n mqtt [\n\t\t\t\t\t\"wrong\"\n\t\t\t\t]\n\t\t\t`,\n\t\t\terr: errors.New(`Expected mqtt to be a map, got []interface {}`),\n\t\t\terrorLine: 2,\n\t\t\terrorPos: 17,\n\t\t},\n\t\t{\n\t\t\tname: \"mqtt bad listen\",\n\t\t\tconfig: `\n mqtt {\n listen: \"xxxxxxxx\"\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(`could not parse address string \"xxxxxxxx\"`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"mqtt bad host\",\n\t\t\tconfig: `\n mqtt {\n host: 1234\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(`interface conversion: interface {} is int64, not string`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"mqtt bad port\",\n\t\t\tconfig: `\n mqtt {\n port: \"abc\"\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(`interface conversion: interface {} is string, not int64`),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"mqtt bad TLS\",\n\t\t\tconfig: `\n mqtt {\n\t\t\t\t\tport: -1\n tls {\n cert_file: \"./configs/certs/server.pem\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(`missing 'key_file' in TLS configuration`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"connection types wrong type\",\n\t\t\tconfig: `\n authorization {\n users [\n {user: a, password: pwd, allowed_connection_types: 123}\n\t\t\t\t\t ]\n\t\t\t\t }\n\t\t\t`,\n\t\t\terr: errors.New(`error parsing allowed connection types: unsupported type int64`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"connection types content wrong type\",\n\t\t\tconfig: `\n authorization {\n users [\n {user: a, password: pwd, allowed_connection_types: [\n 123\n WEBSOCKET\n\t\t\t\t\t\t\t]}\n\t\t\t\t\t ]\n\t\t\t\t }\n\t\t\t`,\n\t\t\terr: errors.New(`error parsing allowed connection types: unsupported type in array int64`),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 32,\n\t\t},\n\t\t{\n\t\t\tname: \"connection types type unknown\",\n\t\t\tconfig: `\n authorization {\n users [\n {user: a, password: pwd, allowed_connection_types: [ \"UNKNOWN\" ]}\n\t\t\t\t\t ]\n\t\t\t\t }\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"invalid connection types [%q]\", \"UNKNOWN\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 53,\n\t\t},\n\t\t{\n\t\t\tname: \"websocket auth unknown var\",\n\t\t\tconfig: `\n\t\t\t\twebsocket {\n\t\t\t\t\tauthorization {\n unknown: \"field\"\n\t\t\t\t }\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"unknown field %q\", \"unknown\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 25,\n\t\t},\n\t\t{\n\t\t\tname: \"websocket bad tls\",\n\t\t\tconfig: `\n\t\t\t\twebsocket {\n tls {\n\t\t\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"missing 'key_file' in TLS configuration\"),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 21,\n\t\t},\n\t\t{\n\t\t\tname: \"verify_cert_and_check_known_urls not support for websockets\",\n\t\t\tconfig: `\n\t\t\t\twebsocket {\n tls {\n\t\t\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/key.pem\"\n\t\t\t\t\t verify_cert_and_check_known_urls: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"verify_cert_and_check_known_urls not supported in this context\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 10,\n\t\t},\n\t\t{\n\t\t\tname: \"ambiguous store dir\",\n\t\t\tconfig: `\n store_dir: \"foo\"\n jetstream {\n store_dir: \"bar\"\n }\n `,\n\t\t\terr: fmt.Errorf(`Duplicate 'store_dir' configuration`),\n\t\t},\n\t\t{\n\t\t\tname: \"token not supported in cluster\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\ttoken: \"my_token\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"Cluster authorization does not support tokens\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"token not supported in gateway\",\n\t\t\tconfig: `\n\t\t\t\tgateway {\n\t\t\t\t\tport: -1\n\t\t\t\t\tname: \"A\"\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\ttoken: \"my_token\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"Gateway authorization does not support tokens\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for cluster pool size\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tpool_size: \"abc\"\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is string, not int64\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for cluster accounts\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\taccounts: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing accounts: unsupported type int64\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for cluster compression\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"field %q should be a boolean or a structure, got int64\", \"compression\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for cluster compression mode\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not string\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for cluster compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\trtt_thresholds: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not []interface {}\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid durations for cluster compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\trtt_thresholds: [abc]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"time: invalid duration %q\", \"abc\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid durations for cluster ping interval\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tping_interval: -1\n\t\t\t\t\tping_max: 6\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(`invalid use of field \"ping_interval\": ping_interval should be converted to a duration`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid durations for cluster ping interval\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\tping_interval: '2m'\n\t\t\t\t\tping_max: 6\n\t\t\t\t}\n\t\t\t`,\n\t\t\twarningErr: fmt.Errorf(`Cluster 'ping_interval' will reset to 30s which is the max for routes`),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for leafnodes compression\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"field %q should be a boolean or a structure, got int64\", \"compression\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for leafnodes compression mode\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not string\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for leafnodes compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\trtt_thresholds: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not []interface {}\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid durations for leafnodes compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: {\n\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\trtt_thresholds: [abc]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"time: invalid duration %q\", \"abc\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for remote leafnodes compression\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:123\"\n\t\t\t\t\t\t\tcompression: 123\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"field %q should be a boolean or a structure, got int64\", \"compression\"),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for remote leafnodes compression mode\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:123\"\n\t\t\t\t\t\t\tcompression: {\n\t\t\t\t\t\t\t\tmode: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not string\"),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"wrong type for remote leafnodes compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:123\"\n\t\t\t\t\t\t\tcompression: {\n\t\t\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\t\t\trtt_thresholds: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"interface conversion: interface {} is int64, not []interface {}\"),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid durations for remote leafnodes compression rtt thresholds\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:123\"\n\t\t\t\t\t\t\tcompression: {\n\t\t\t\t\t\t\t\tmode: \"s2_auto\"\n\t\t\t\t\t\t\t\trtt_thresholds: [abc]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"time: invalid duration %q\", \"abc\"),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid duration for remote leafnode first info timeout\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:123\"\n\t\t\t\t\t\t\tfirst_info_timeout: abc\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing first_info_timeout: time: invalid duration %q\", \"abc\"),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"show warnings on empty configs without values\",\n\t\t\tconfig: ``,\n\t\t\twarningErr: errors.New(`config has no values or is empty`),\n\t\t\terrorLine: 0,\n\t\t\terrorPos: 0,\n\t\t\treason: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"show warnings on empty configs without values and only comments\",\n\t\t\tconfig: `# Valid file but has no usable values.\n `,\n\t\t\twarningErr: errors.New(`config has no values or is empty`),\n\t\t\terrorLine: 0,\n\t\t\terrorPos: 0,\n\t\t\treason: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"TLS handshake first, wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tfirst: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"field %q should be a boolean or a string, got int64\", \"first\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS handshake first, wrong value\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tfirst: \"123\"\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"field %q's value %q is invalid\", \"first\", \"123\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcerts: [\n\t\t\t\t\t { cert_file: \"configs/certs/server.pem\", key_file: \"configs/certs/key.pem\"},\n\t\t\t\t\t { cert_file: \"configs/certs/cert.new.pem\", key_file: \"configs/certs/key.new.pem\"},\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs, bad type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcerts: [\n\t\t\t\t\t { cert_file: \"configs/certs/server.pem\", key_file: 123 },\n\t\t\t\t\t { cert_file: \"configs/certs/cert.new.pem\", key_file: \"configs/certs/key.new.pem\"},\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing certificates config: unsupported type int64\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 49,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs, missing key_file\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcerts: [\n\t\t\t\t\t { cert_file: \"configs/certs/server.pem\" }\n\t\t\t\t\t { cert_file: \"configs/certs/cert.new.pem\", key_file: \"configs/certs/key.new.pem\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing certificates config: both 'cert_file' and 'cert_key' options are required\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 10,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs and single cert options at the same time\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/key.pem\"\n\t\t\t\t\tcerts: [\n\t\t\t\t\t { cert_file: \"configs/certs/server.pem\", key_file: \"configs/certs/key.pem\"},\n\t\t\t\t\t { cert_file: \"configs/certs/cert.new.pem\", key_file: \"configs/certs/key.new.pem\"},\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing tls config, cannot combine 'cert_file' option with 'certs' option\"),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs used but not configured, but cert_file configured\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/key.pem\"\n\t\t\t\t\tcerts: []\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"TLS multiple certs, missing bad path\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcerts: [\n\t\t\t\t\t { cert_file: \"configs/certs/cert.new.pem\", key_file: \"configs/certs/key.new.pem\"}\n\t\t\t\t\t { cert_file: \"configs/certs/server.pem\", key_file: \"configs/certs/key.new.pom\" }\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: fmt.Errorf(\"error parsing X509 certificate/key pair 2/2: open configs/certs/key.new.pom: no such file or directory\"),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: 123\n\t\t\t`,\n\t\t\terr: errors.New(\"expected proxies to be a map/struct, got int64\"),\n\t\t\terrorLine: 3,\n\t\t\terrorPos: 5,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies unknown field\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: {\n\t\t\t\t\tfield1: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"unknown field\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies trusted wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: {\n\t\t\t\t\ttrusted: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"expected proxies' trusted field to be an array, got int64\"),\n\t\t\terrorLine: 4,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies trusted key wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: {\n\t\t\t\t\ttrusted: [\n\t\t\t\t\t\t\"abc\",\n\t\t\t\t\t\t\"def\"\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"expected proxies' trusted entry to be a map/struct, got string\"),\n\t\t\terrorLine: 5,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies trusted unknown field\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: {\n\t\t\t\t\ttrusted: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t\t\t\tfield1: 123\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"unknown field\"),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"Proxies trusted key invalid\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tproxies: {\n\t\t\t\t\ttrusted: [\n\t\t\t\t\t\t{key: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"}\n\t\t\t\t\t\t{key: \"bad1\"}\n\t\t\t\t\t\t{key: \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"}\n\t\t\t\t\t\t{key: \"bad2\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"invalid proxy key\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"Auth user require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tuser: user\n\t\t\t\t\tpassword: pwd\n\t\t\t\t\tproxy_required: 123\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 6,\n\t\t},\n\t\t{\n\t\t\tname: \"Auth users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tusers: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tuser: user\n\t\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"Auth nkey users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tusers: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tnkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"Account users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA: {\n\t\t\t\t\t\tusers: [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tuser: user\n\t\t\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"Account nkey users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA: {\n\t\t\t\t\t\tusers: [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tnkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"Leafnode user require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\tuser: user\n\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 8,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"Leafnode neky user require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\tnkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 7,\n\t\t\terrorPos: 7,\n\t\t},\n\t\t{\n\t\t\tname: \"Leafnode users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\tusers: [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tuser: user\n\t\t\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 10,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"Leafnode nkey users require proxied wrong type\",\n\t\t\tconfig: `\n\t\t\t\tport: -1\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tauthorization {\n\t\t\t\t\t\tusers: [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tnkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\t\t\t\t\tproxy_required: 123\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"interface conversion\"),\n\t\t\terrorLine: 9,\n\t\t\terrorPos: 9,\n\t\t},\n\t\t{\n\t\t\tname: \"leafnode proxy with unsupported scheme\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"ftp://proxy.example.com:8080\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"proxy URL scheme must be http or https, got: ftp\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"leafnode proxy with missing host\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"proxy URL must specify a host\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"leafnode proxy with username but no password\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t\tusername: \"testuser\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"proxy username and password must both be specified or both be empty\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"leafnode proxy with password but no username\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t\tpassword: \"testpass\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"proxy username and password must both be specified or both be empty\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t\t{\n\t\t\tname: \"leafnode proxy with WSS URL but no TLS config\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"wss://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\terr: errors.New(\"proxy is configured but remote URL wss://127.0.0.1:7422 requires TLS and no TLS configuration is provided\"),\n\t\t\terrorLine: 6,\n\t\t\terrorPos: 8,\n\t\t},\n\t}\n\n\tcheckConfig := func(config string) error {\n\t\topts := &Options{\n\t\t\tCheckConfig: true,\n\t\t}\n\t\treturn opts.ProcessConfigFile(config)\n\t}\n\n\tcheckErr := func(t *testing.T, err, expectedErr error) {\n\t\tt.Helper()\n\t\tswitch {\n\t\tcase err == nil && expectedErr == nil:\n\t\t\t// OK\n\t\tcase err != nil && expectedErr == nil:\n\t\t\tt.Errorf(\"Unexpected error after processing config: %s\", err)\n\t\tcase err == nil && expectedErr != nil:\n\t\t\tt.Errorf(\"Expected %q error after processing invalid config but got nothing\", expectedErr)\n\t\t}\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(test.config))\n\t\t\terr := checkConfig(conf)\n\t\t\tvar expectedErr error\n\n\t\t\t// Check for either warnings or errors.\n\t\t\tif test.err != nil {\n\t\t\t\texpectedErr = test.err\n\t\t\t} else if test.warningErr != nil {\n\t\t\t\texpectedErr = test.warningErr\n\t\t\t}\n\n\t\t\tif err != nil && expectedErr != nil {\n\t\t\t\tvar msg string\n\n\t\t\t\tif test.errorPos > 0 {\n\t\t\t\t\tmsg = fmt.Sprintf(\"%s:%d:%d: %s\", conf, test.errorLine, test.errorPos, expectedErr.Error())\n\t\t\t\t\tif test.reason != \"\" {\n\t\t\t\t\t\tmsg += \": \" + test.reason\n\t\t\t\t\t}\n\t\t\t\t} else if test.warningErr != nil {\n\t\t\t\t\tmsg = expectedErr.Error()\n\t\t\t\t} else {\n\t\t\t\t\tmsg = test.reason\n\t\t\t\t}\n\n\t\t\t\tif !strings.Contains(err.Error(), msg) {\n\t\t\t\t\tt.Errorf(\"Expected:\\n%q\\ngot:\\n%q\", msg, err.Error())\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheckErr(t, err, expectedErr)\n\t\t})\n\t}\n}\n\nfunc TestConfigCheckIncludes(t *testing.T) {\n\t// Check happy path first.\n\topts := &Options{\n\t\tCheckConfig: true,\n\t}\n\terr := opts.ProcessConfigFile(\"./configs/include_conf_check_a.conf\")\n\tif err != nil {\n\t\tt.Errorf(\"Unexpected error processing include files with configuration check enabled: %v\", err)\n\t}\n\n\topts = &Options{\n\t\tCheckConfig: true,\n\t}\n\terr = opts.ProcessConfigFile(\"./configs/include_bad_conf_check_a.conf\")\n\tif err == nil {\n\t\tt.Errorf(\"Expected error processing include files with configuration check enabled: %v\", err)\n\t}\n\texpectedErr := `include_bad_conf_check_b.conf:10:19: unknown field \"monitoring_port\"` + \"\\n\"\n\tif err != nil && !strings.HasSuffix(err.Error(), expectedErr) {\n\t\tt.Errorf(\"Expected: \\n%q, got\\n: %q\", expectedErr, err.Error())\n\t}\n}\n\nfunc TestConfigCheckMultipleErrors(t *testing.T) {\n\topts := &Options{\n\t\tCheckConfig: true,\n\t}\n\terr := opts.ProcessConfigFile(\"./configs/multiple_errors.conf\")\n\tif err == nil {\n\t\tt.Errorf(\"Expected error processing config files with multiple errors check enabled: %v\", err)\n\t}\n\tcerr, ok := err.(*processConfigErr)\n\tif !ok {\n\t\tt.Fatalf(\"Expected a configuration process error\")\n\t}\n\tgot := len(cerr.Warnings())\n\texpected := 1\n\tif got != expected {\n\t\tt.Errorf(\"Expected a %d warning, got: %d\", expected, got)\n\t}\n\tgot = len(cerr.Errors())\n\t// Could be 7 or 8 errors depending on internal ordering of the parsing.\n\tif got != 7 && got != 8 {\n\t\tt.Errorf(\"Expected 7 or 8 errors, got: %d\", got)\n\t}\n\n\terrMsg := err.Error()\n\n\terrs := []string{\n\t\t`./configs/multiple_errors.conf:12:1: invalid use of field \"write_deadline\": write_deadline should be converted to a duration`,\n\t\t`./configs/multiple_errors.conf:2:1: Cannot have a user/pass and token`,\n\t\t`./configs/multiple_errors.conf:10:1: unknown field \"monitoring\"`,\n\t\t`./configs/multiple_errors.conf:67:3: Cluster authorization does not allow multiple users`,\n\t\t`./configs/multiple_errors.conf:21:5: Not a valid public nkey for an account: \"OC5GRL36RQV7MJ2GT6WQSCKDKJKYTK4T2LGLWJ2SEJKRDHFOQQWGGFQL\"`,\n\t\t`./configs/multiple_errors.conf:26:9: Not a valid public nkey for a user`,\n\t\t`./configs/multiple_errors.conf:36:5: Not a valid public nkey for an account: \"ODRZ42QBM7SXQDXXTSVWT2WLLFYOQGAFC4TO6WOAXHEKQHIXR4HFYJDS\"`,\n\t\t`./configs/multiple_errors.conf:41:9: Not a valid public nkey for a user`,\n\t}\n\tfor _, msg := range errs {\n\t\tfound := strings.Contains(errMsg, msg)\n\t\tif !found {\n\t\t\tt.Fatalf(\"Expected to find error %q\", msg)\n\t\t}\n\t}\n\tif got == 8 {\n\t\textra := \"./configs/multiple_errors.conf:54:5: Can not have a single user/pass and accounts\"\n\t\tif !strings.Contains(errMsg, extra) {\n\t\t\tt.Fatalf(\"Expected to find error %q (%s)\", extra, errMsg)\n\t\t}\n\t}\n}\n" }, { "path": "server/configs/certs/cert.new.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDnjCCAoagAwIBAgIJAM/HacKKaH7zMA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD\nVQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu\naW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIyODIxWhcN\nMjkxMDE0MTIyODIxWjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD\nVQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u\ncyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz\nLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBIAVUr4gCEBRVbA\nHSweCLExOrMZmII4AdvgIqT+svjBkJd+vdkbd5b/SC1HQx1E14kiRJ/JrZIZoMqi\n+7pK3kFM63Fkhkg8rWOxn0tQznSymKTpha5NdDWxnB0dXlXFCQG1e/cuDalR7UhF\nLPHiuK42gAvhivBcymDPV0hTYt4rHb71SQ1DwfCYzcLkDvDFA/W7kronaEhRyWn6\nuvZvHkdvScoubdzoW/kNBH4JYZw5svLzGz3z20rUGeLttF4ge5SCAz9unZk96HUO\nEFmUDvFmxdnTXrYINjraNvgu7fPFmVzupWrWPA/7U+cxOvm3qBMdqxFxNr7bq+Md\nUPKi3QIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB\nAQCCgFKP/bMi5cvasJOzXKLpwOneW+oItL2t/5NxPumIBMDo5NnShzsFLfGmujYw\nfjcMharynkkbz/oeDAm8h1mySAJrnqtabiWgaW8zbNJfJOrAq4Jvs9COMAzjJclL\n+h9GWtvVylDnsNtd18n1gA5OYv0A6YjuSrWINL8Sp5QvTF/5tT8jFrDOIjZl7m50\nlX4R70N9GLt2jIlKro+qdsi6qUZccuJmoQxUpG1iQcRNFHtWfDPr5KFEXaO6IoYt\nD1kYWmo/A3WQm7nbXeZw/zaSGSS0t6/hKZwm+gPCL6TEdDrjhjxpCZQTwaMd6jj2\nbvT2OA0ZpzUWyxnaX6u+cDfM\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/key.new.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8EgBVSviAIQFF\nVsAdLB4IsTE6sxmYgjgB2+AipP6y+MGQl3692Rt3lv9ILUdDHUTXiSJEn8mtkhmg\nyqL7ukreQUzrcWSGSDytY7GfS1DOdLKYpOmFrk10NbGcHR1eVcUJAbV79y4NqVHt\nSEUs8eK4rjaAC+GK8FzKYM9XSFNi3isdvvVJDUPB8JjNwuQO8MUD9buSuidoSFHJ\nafq69m8eR29Jyi5t3Ohb+Q0EfglhnDmy8vMbPfPbStQZ4u20XiB7lIIDP26dmT3o\ndQ4QWZQO8WbF2dNetgg2Oto2+C7t88WZXO6latY8D/tT5zE6+beoEx2rEXE2vtur\n4x1Q8qLdAgMBAAECggEBAI0zJrTi2Le6D+4zN4Gvgc0c280qcmkiO1KJ9jmMjv1n\n7kvUsf9vZUPgRkG2XO5ypyD7gJLtNMnwCvXBraQ5NcSwWkPampKG4ad8Vfs23LBk\nxUH9bqZDOzuopHSFF2ugEZK1icBM0HLJUQ1JWUZpRMNLaPex8+AQnloDXSg20QRE\nZ53CDYZVEVaLAYIkfgZ6QgBSbJ0ob/a+ZJkChWs3f6cpNHige4tR2RQ0aJFwvlwu\neC8G3RDNzUMhPf4TSf/XUlHXngyHKa7MS3feEtuKIQ4gb8hjESDcZznJUGG9qyMT\n7445mapc60t8z46o9iVeVJQmQxmjFUDkLcZ9ckcWfzECgYEA4OiolNUiTnXbvP0H\n3rxAnhx9GlyHiRaIZvgsAHH/6aMbJIXnH89cAR38PL79OaQpWZqK8Z1vrCHC8q7s\nWb1RehgnwPxtTl4RTZ0wj3Ri5BIq32nzgla0w/5RAmAw9HrYXjp4TR4Quc+bHLcr\nZM2gA8nZ4PG73rYvd/Mu3lCQiS8CgYEA1hGoaduGG7eHTMCct/kCupFfYXgEuUDh\nV/nO1SYJbIkMuSONNaQy1oUc5o0BB/JTcslZWqtuGa9snKPVd3/rJz75YGt1QR6u\nMukCYjbWGAWXfDh8G8aGSr3k1k3JmKNE7fI7L5DuvKCGoGjzzC+kclyoij8jwXk0\nzghaYPvM+bMCgYEA3/fwgR3p5vZJF19mqfEP7CP0lP7V3bd5qAi1UNA1h4VsrydF\nLRFCzr38hMWwx+jpYJicitU78s9AIon9RbRY4dwSIoV9mE/mrUK+q+y72eEZnpgU\n7ZPIuXCVXWdK+PsoYlWZnTo2b8ME9UiWxvBZy8wD05UGgFcu2CVsY+kYtfMCgYAs\nVn/xXPyL5Rlq9kH/gN3l0pJU18zyqdOCq0UBtN0i08gE2K44vAejkvKHdhEOmkxa\nbAXL19H4E/OFBhICrEYCXPK928PvdvFRrh1GRmFVnGLh4bki752FAYvSL05gBQET\n36YOlhA8lWsM8m/8jKmc3kAyUh2PxxD+05AUolK0LQKBgHqPgMj5rfiR7MoVIaXr\nK8h7g+HejppmCxzENc98hvIQIO2aWeC4Hi/EMe+XYSN/Gl6d5+aw80JYRgIv6dFV\nhDEEt8lEI4sFhKD0oG1dQo+BdDNXOLbpHq+RQRPHa8J/pmSjI5Joqb2mCaAcm/KJ\nnYE20ox06YMSb+ZrjP2p4qw3\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDc8YaxliKNsa8M\nidwQsDXS0MI4z+ZP1UMW9U/Vt2qTienYs3L6KqezRfwEH2AoE7XyvQdzePV3aN1g\nA0L3mAKiS2FN3DhxR6Y8D/JNb5YA6Pl0flaMKnWktma/SMOf7o7eS7Zt6ZmIvtBg\np1ZtRSZSH64KSqC467eGByHnXm25Z7Uf/2rO8K1jsmRwgDEJoJL5Z4va4pypX5Kq\n7OxWk8Za50Kuk6DtdRfIcnUjoDadsYzGHzF7kp1/31elQ2aUPp9e6X3aWu++gJv5\nbrBMsWeQ+ETmjCMPqxoXDM2AJqwP73irELU7POUR/q3gooqy16BvTp7TaLvAtKdP\nSK7kl1QVAgMBAAECggEAKVT1gZs4c5Fd0daoWGbeasWqz730s9KNbmoNuUlKorcW\nC/fl9m6sWJkrAApujpDBZNt/3VYvdVskrqVtjaveMkYvucUMugRRUMMa7OmHAjyT\nDfvkbOw0fc+xgO83yV9zUpqPUhh1JGOtz6b+QArDlHFguYQkdPODrYsACKxdkmU/\nA+ZJkC2Sdn0L7ARmROxvoelxV3HNbC+p8lqrIVarpni8oeDF2JxQfyJETl3Df099\n/KA/IQvXytEQBJpguh0j7sK81cI40TKO3kNWsCe46S2nkCuEeEums7BqAspmdNMS\ngBW0rpFA9Ep5AyT4+A2XhWFQAwf/PI26JUVDhbClAQKBgQD/XlONHf5tNiOEtW88\nWhd9ha7oLgoBhr4gFOcPM4MUStRDftRXY1ty0IGseV0Rj1xQRUp31Pl6Kep/udy1\nr/6ML/IQNaht1VJOf2+2v7oZFoLRGLvR3qSRFNVtfLWp1Dg5OPgyi00OpFnxz/6o\niuWlh1EWwSm9vPl91HcWPPcwBQKBgQDdfWfDfy8q9jExj8tVNh5++haeemprkZxk\nQuduIkahlbQ3V+SLaSWQrwzcB4PWL+45fnIWwK3Vmm2POBv+OWfKGjJniSd4Zrrf\nH+lSg4+hpivvoaYf1TsCTHy/af/5pYIkEUmbPM0EqwubyoW6oMAgGIAMyH6XBLaY\na4vWZo6g0QKBgFXDXhgUrLAM8JzPOk5wi1cSoI1FeQLON+gaXQdT63/TKbqJS9MV\ngU7sC8Da+ZC+LuiefMYF9ss6bJD84Mz8EGcQayFag/hvHjdSwTgE6AEo+EI1Jk7z\nkR1Qe+VLbs9cgI1nPqPq+LQkKjj1+7aq/zk6WtdjhBs+7iN+SYhkbTfpAoGAeCbv\ngz4beFLVnO1EgJU7Nea1HoOJ95CmJj2lDjnJ1x4/BUbI1FfV6QcNEs+A1VBrCwVt\nHqqnopiDlo35oY/CngBYF5JvtwEDnsbQ69IyuJ5Md1JZrCsgN78GbVAzbFo8nxRB\nudTh4wZm5byXjwZlMwQXctfQ1FvaMhUlZsl71gECgYA82osmLHxZZOiQrYJ7z302\nCmPohwcTtUASqeA0kQOX/OdFJ1IxWIFpzq2ffMipLTfzk7OeFS2hqvNo/NE8NHlR\n/79IycYwlO99T3e2yQhYKW5ZMT3Q8YtyLLPtBzPPtO/Tc8yKhjX8wc9w9sxq9kqg\ni5Z+6xfO+l/NBT/JHF4yEw==\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDnjCCAoagAwIBAgIJAOXpOWk41Pd7MA0GCSqGSIb3DQEBCwUAMIGDMQswCQYD\nVQQGEwJVUzELMAkGA1UECAwCQ0ExFDASBgNVBAcMC0xvcyBBbmdlbGVzMSQwIgYD\nVQQKDBtTeW5hZGlhIENvbW11bmljYXRpb25zIEluYy4xKzApBgNVBAMMIm5hdHMu\naW8vZW1haWxBZGRyZXNzPWRlcmVrQG5hdHMuaW8wHhcNMTkxMDE3MTIzMjQ5WhcN\nMjkxMDE0MTIzMjQ5WjCBgzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYD\nVQQHDAtMb3MgQW5nZWxlczEkMCIGA1UECgwbU3luYWRpYSBDb21tdW5pY2F0aW9u\ncyBJbmMuMSswKQYDVQQDDCJuYXRzLmlvL2VtYWlsQWRkcmVzcz1kZXJla0BuYXRz\nLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3PGGsZYijbGvDInc\nELA10tDCOM/mT9VDFvVP1bdqk4np2LNy+iqns0X8BB9gKBO18r0Hc3j1d2jdYANC\n95gCokthTdw4cUemPA/yTW+WAOj5dH5WjCp1pLZmv0jDn+6O3ku2bemZiL7QYKdW\nbUUmUh+uCkqguOu3hgch515tuWe1H/9qzvCtY7JkcIAxCaCS+WeL2uKcqV+Squzs\nVpPGWudCrpOg7XUXyHJ1I6A2nbGMxh8xe5Kdf99XpUNmlD6fXul92lrvvoCb+W6w\nTLFnkPhE5owjD6saFwzNgCasD+94qxC1OzzlEf6t4KKKstegb06e02i7wLSnT0iu\n5JdUFQIDAQABoxMwETAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IB\nAQDIMNgMqgGdXwZTIDxz4iq76iHdSHxZK4YwJr/4xRSq1uEddxmUfAQzo+gBboA5\nc0XxYxc0xViuueThnqdLhHmEyCs8uGFRLaQAI5Bq1PdMQP12m5fCWRAyKrCxdtli\nzm8ByDq7mWpkfMTd/rJGR4wCR9qI9Y5Bp6p4FBKZ3pzEanFXMV9IHhkm1BGh9tbe\nl6GQyBptEpfTiRwNCC/ympeiL3G8hfDCPkcLed5sQ+OhPe5iWMVPncZh/qehnUJK\nB5CIXcagcROFutsDYPCurKcfQOsfqulu0q95h7FQUOsIeU7jIlcLxIii19qjlTZh\nsFjsul5G7qqMEgIUsx3U985v\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-ca-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIID2zCCAsOgAwIBAgIUZj0PngA93uUSShcRndTQju/J88YwDQYJKoZIhvcNAQEL\nBQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB\nbmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250\nbzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3\nWhgPMjEyMzA3MjExNTI3NTdaMHwxEzARBgNVBAMMCm5hdHMuaW8uQ0ExIjAgBgNV\nBAsMGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWEx\nEDAOBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNB\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2cCyJL+DExUyZto2eFLm\nMBRSkQLxM9pOWB9O8TecHlPcc/SPGq/x9lpguJ/IiaUj+VffVWy236KW2JL5Xj83\nPZwhXi1yZzxlIBsKAgAUeNfWuTAc0K0Qm9pR5Wjv5eNcT0mw6JX0SPgUQAl9BSwU\nWvtMOTxOt0hBjHmZaEamp7nLmwogpvgPsrubD6U4O/vUQm3JTsbp2rFQxXPpkG19\n69PGsT37r0/w9Zv0xNAcB/zCWdNBXCTA2ACV2IpJedWm8Jrjcn3Kp4Fv3TKTsCZl\neWtfxCdljndk88+NFK7cEw7b9Bs5R5Zhu20C+Ne8vmMWhYbVBFYws5/jGzPBkVTD\n7wIDAQABo1MwUTAdBgNVHQ4EFgQUEqfeAemfeIp4MM4C7H1bJS+mra4wHwYDVR0j\nBBgwFoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDwYDVR0TAQH/BAUwAwEB/zANBgkq\nhkiG9w0BAQsFAAOCAQEAiamiPxOlZ0pwOJvv0ylDreHVk2kast67YlhAOcZoSMvi\ne2jbKL98U3+ZznGj21AKqEaOkO7UmKoJ/3QlrjgElXzcMUrUrJ1WNowlbXPlAhyL\nKhNthLKUr72Tv6wv5GZdAR6DaAwq3iYTbpnLq4oCnFHiXgDWgWyJDLsTGulWve/K\nGGM2JMcnacNgNC18uki440Wcfp0vGj9HhO6I/u63oGewZnIK87GQMQCt3JLFyiUc\nhrn9nWoixFWcJfCjBcMlwZXMIAlDdelU1/hWtSknKCs57GvZuACcicAYiYIkWCkd\np1pF4G0Ic6irAnLTqhdGwL4+5pjNd1Ih0Gezn9hJLg==\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-ca-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZwLIkv4MTFTJm\n2jZ4UuYwFFKRAvEz2k5YH07xN5weU9xz9I8ar/H2WmC4n8iJpSP5V99VbLbfopbY\nkvlePzc9nCFeLXJnPGUgGwoCABR419a5MBzQrRCb2lHlaO/l41xPSbDolfRI+BRA\nCX0FLBRa+0w5PE63SEGMeZloRqanucubCiCm+A+yu5sPpTg7+9RCbclOxunasVDF\nc+mQbX3r08axPfuvT/D1m/TE0BwH/MJZ00FcJMDYAJXYikl51abwmuNyfcqngW/d\nMpOwJmV5a1/EJ2WOd2Tzz40UrtwTDtv0GzlHlmG7bQL417y+YxaFhtUEVjCzn+Mb\nM8GRVMPvAgMBAAECggEASvFWfm5JMtqjP6HPiGbjoV2FMzJjiFiUiSCxXzSn7wom\nv+PGEsXGTWa6jiAz+SeUc38KNtDVOa+wIfani4fPP82J8GtMyfoPxdZ4gcq8QQDr\n/k1wRWOi6Tjg4cdVdXXkMcencs0VR73V3lpFpG+Qy+VcTQCUCOF96dZ59VkHh4a4\nCHX6PegOWwr0TSaCUacwhua+rPmCar/btAYv7Wp7+c+Zf7Rn2WYTV7ol4sYXR4ZQ\nSy/ROijeFTMkYNpaW/KOO2/pn3OJA8ycYH8UWZpsenPfIajC0Eka7BoEQpw6M8HR\nwRWrKwssBEs0psFiq9s8J+6resPgXfU/9pf+mTkTqQKBgQD8kktUqN+vYc6t22DE\ntSkg8y8OsGh9VTYfp7x5hu9qEC9t4mAKjqA/rRLiTXze/wInntreRTjjMb/NlqMy\nPvI0Z+dM1UuqcF1axgKrIYsgnLJWuunOhaj5K3LhiNcHznlCtN9601dbccwLlQhL\n5jdjnOuJ0i+Nh9v5oiu37SfldwKBgQDctWdbF4hJrBPnS6CcojuQj6ha10wYYe46\nZVcxKe5hFBs1q975YCHnEntyCDvXGfOTRgbKeZbwNhMvAc7Pp6eGMR/9SpiRwTt4\n567hUz56WXVmp4gSvxoNuYRlWiMI8rZkyKJ8KFipvHgRa8nuamh0QBB4ShEJiVk8\nfhaUiZeTSQKBgF0hAD/OKPR9Jv06J9tARVMN+Cr9ZvnXwqY3biqNU5gTMbndv7YE\n0xfHlG/3THTZKI09aMyOT6SOQn/m7HPpe9tQ+Jt/BnBpEDMZUgCR1MAIp0WNlAp/\nhEej+q8oiskpG9M56DFc3hgsxKT8pdt+nqvPP5ZI9xnDn5vTbTVbb9uPAoGAFRvU\ncsXhZwpqLOjyx4hMohrbQzTsNjjHjBY9LJqSDf7aS1vQy5ECLRN7cwCOmJgGz8MW\nyy6t3POPCiPmH74tK4xvPs5voSEWCw49j5dillkP/W1wejqEx2NC4l6okyaDg0gd\nIjrJoBJCeYgRnBfZPaUS7i5HSt40BrEYf8RZFuECgYAjSnvY8nYFRudsylYOu3TL\nAcGbAdpDfL2H4G9z7qEQC6t1LqqGNcfMan/qS4J/n5GCOedVWHcfCraROOMil52v\n3ZDXyyjGEO08XgKnoa2ZL/z2a6s077+hAAnbcywyi2Qz8Yfi6GnwYfU1u8SN0APd\nT71HPNsWkU4zkxmP4S0Olg==\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-cert-ed25519.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDHTCCAgWgAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsYwDQYJKoZIhvcNAQEL\nBQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB\nbmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250\nbzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3\nWhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM\nGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO\nBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMCow\nBQYDK2VwAyEAyyc9y9iZgWWSsPRahbeGxF6XN3VOFPZBvD/HQps6jr6jgZEwgY4w\nCwYDVR0PBAQDAgQwMBMGA1UdJQQMMAoGCCsGAQUFBwMBMCoGA1UdEQQjMCGCDnJl\ndWJlbi5uYXRzLmlvgg9yZXViZW4ubmF0cy5jb20wHQYDVR0OBBYEFBwkwMU8xuQO\nFN1Ck5o2qQ4Dz87ZMB8GA1UdIwQYMBaAFBKn3gHpn3iKeDDOAux9WyUvpq2uMA0G\nCSqGSIb3DQEBCwUAA4IBAQALjCynuxEobk1MYQAFhkrfAD29H6yRpOcKigHCZjTJ\nDnpupip1xvaFPPvhi4nxtuWcXgKpWEfd1jOPaiNV6lrefahitZpzcflD7wNOxqvx\nHau2U3lFnjnGaC0ppp66x26cQznp6YcTdxrJ1QF4vkOejxqNvaTzmiwzSPIIYm7+\niKVWT+Z86WKof3vAdsX/f148YH1YSPk0ykiBzlbLScbyWebbaydrAIpU01IkSvMo\nqDYu+Fba0tpONLe1BUklc608riwQjw9HiJJ2zJIAOBAUev5+48RP91/K111Ix1bl\nfGPT8/1TJbyGG2jeJwyLoSIu72aDnnIBfqGkVunRTmeg\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-cert-rsa-1024.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDkzCCAnugAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsMwDQYJKoZIhvcNAQEL\nBQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB\nbmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250\nbzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3\nWhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM\nGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO\nBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIGf\nMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCyHHaVHinB3jBsicR4hp7uopz0u3+O\nkUicIUSQXDcWiPzdvE+7YZ/s4+Ud4aw4g9q0wHzkZSaMg8nil4tCKmTrUKolVTVj\nCCCBmtqq3LwzNLapyoDJRyXsWqHt5TWYSxaf/UQT6sWOgqHOLrbd4J8F0sjxEniB\nGDHR1ZXpJCBaIQIDAQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggr\nBgEFBQcDATAqBgNVHREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMu\nY29tMB0GA1UdDgQWBBQk5kWOUcUNn7FppddLANe3droUlzAfBgNVHSMEGDAWgBQS\np94B6Z94ingwzgLsfVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEA2Njy2f1PUZRf\nG1/oZ0El7J8L6Ql1HmEC7tOTzbORg7U9uMHKqIFL/IXXAdAlE/EjFEA2riPO8cu/\nbvL2A4CapYzt2kDD9PPYfVtniRr7mv0EVntPwEvfiySMAEeZuW/M2liPfgPpQkhL\nfzwPeCOfqM8AjpyDab8NEGX5Bbf421oQorlENpm4PKQCXoUN5cWpBwuwWxj7yndj\n256MevLDKKe/ALSLQEo/2Jgpnmp7Qol0GtomCzsLgZ+ASuVtCsGTFmaRrsqVPspJ\noOl6qby5gYwN9TR8zfRYL1m1sbYROz+5+ofEoiTnaOoOSjiBIoYoMeSC/jvJQTPT\nVdD8QeQ6Og==\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-cert-rsa-2048.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEFzCCAv+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsQwDQYJKoZIhvcNAQEL\nBQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB\nbmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250\nbzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3\nWhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM\nGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO\nBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIB\nIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyx3O+Z6u8Y1SiuHu3szWbLvL\nWrZpSpEiZkll+wk5205S1FRcQLccfr4ubdtjOBdi+RzILCtkflUI01Dbqu6cV7/2\nyfLthxBeNDiXMhjyOFkYLwwE4w7CdTwWWsmW31oUH1rYXIDPoeb7WPF7w3NwaUJu\nZXnqM98LRgWDTmh+nsqDDW/bz1fYIdxcO9az6iBOnJ2AGWI2ur5GzWc4+gNMOZiZ\nXj657g0MbyVM4Gzyc4Au22hShZ/YorLP8NAiwNJamlrCFzlnZN/ePjuQPcI6glnb\noO9IAGfPdAOJptfayuPAZgUngzewB38yY0Q/rKG1GJKSkQ8X6/lXiWaRPZJjYwID\nAQABo4GRMIGOMAsGA1UdDwQEAwIEMDATBgNVHSUEDDAKBggrBgEFBQcDATAqBgNV\nHREEIzAhgg5yZXViZW4ubmF0cy5pb4IPcmV1YmVuLm5hdHMuY29tMB0GA1UdDgQW\nBBRtanJZScdSlsPsi58lBcpdj+bV/zAfBgNVHSMEGDAWgBQSp94B6Z94ingwzgLs\nfVslL6atrjANBgkqhkiG9w0BAQsFAAOCAQEAV4TZ3b8cYO7ZeRyoCQtCBAab9gNe\nkbQpWqICvkVQOk5Anq3opwAWk2FuIRs5KoT7ssckHpXwTwWLs+KuIVo+Fet19IH6\nBQfck1jwhzM04MA6zLO/F2j548XlrJy3IzViPM/VxwMMTt5YSoogrz/3TzzJPIe0\neQomf5HbpVgrf08pMVkdaI7PCd7N/CxeWiD5zEWqBu9FqofO188Kb/umx0VwgBju\ndX46MKO5TyUc91UrG3M35/r4Z7fd52SWWWFQiI7UBOl2L27samjHlJsKjyFoBF3Z\nalvnoUVzo7zwAYmhEdPYDNVceF4KtAFpGipoQPRMg83G87LgYBA4Sa6uKw==\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-cert-rsa-4096.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIFFzCCA/+gAwIBAgIUc31LCktokAIQGqLsSC2BlsLCTsUwDQYJKoZIhvcNAQEL\nBQAwfDETMBEGA1UEAwwKbmF0cy5pby5DQTEiMCAGA1UECwwZUGVyZm9ybWFuY2VB\nbmRSZWxpYWJpbGl0eTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UEBwwHVG9yb250\nbzEQMA4GA1UECAwHT250YXJpbzELMAkGA1UEBhMCQ0EwIBcNMjMwODE0MTUyNzU3\nWhgPMjEyMzA3MjExNTI3NTdaMHkxEDAOBgNVBAMMB25hdHMuaW8xIjAgBgNVBAsM\nGVBlcmZvcm1hbmNlQW5kUmVsaWFiaWxpdHkxEDAOBgNVBAoMB1N5bmFkaWExEDAO\nBgNVBAcMB1Rvcm9udG8xEDAOBgNVBAgMB09udGFyaW8xCzAJBgNVBAYTAkNBMIIC\nIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAj/rw0WpCnizUSs06NxSsjuYb\n6sWtYQjLd6O6SCoQwrPSY2Zv0u28RywBJkSiIbv12kWxrhqv+kDqhmt7tfwEdHAE\ngOl6E2P/CFDnyAJ4it5qgFRHcEItGp1Ap4ZrZ3OTl41DDe28giflHb7+VAzxUV6r\nzkHmrqkaJd41YrJoKyZ3u+/FHs6CudO+jGXC2ubH1i71ARXHn6tkmOTwFf1N49wB\nHr144xBmPyvH/A/elgXBjR28/7x64MO/qBZsr/lbTaS4YwN1+rod/HRc2GIiJxpS\ntB1bh/dXZCfa0QyhTzCNLG/j7IzetDrPBGzpw2WjufhkSuaxoMDWlDkkfqmQxtHJ\n5L+PqIiPT69tzEbfuS9Ogz7DW10CcpSdXW13sWCdSNCGEvPqLFka36q9B82V0GHz\ntmx8VqdfWSu4qMyVmTsxxzLTTxpQU4X1Q2RnT/1igbsOM620LuvrvnXlh5Rdg7tp\nT++QJ/b4xCCg62tv4VwORe27xHYXMeDn4aoRdyoI45/+ZK6yqwNAOrKKpse/M8uz\nmJK2i8pfEFmitIKoNYn3MR2dFrCqifZkFf9rX9A/1Ym+WKAPLmWdGp13fvTdzxQG\nY44f9tBL2RWsoGX++01XEwIiWz7kqObC0L8fz3EdIPaULX7MZiQrxzzhRCcJhyn/\naOrJfLYj0GAmIaHElHkCAwEAAaOBkTCBjjALBgNVHQ8EBAMCBDAwEwYDVR0lBAww\nCgYIKwYBBQUHAwEwKgYDVR0RBCMwIYIOcmV1YmVuLm5hdHMuaW+CD3JldWJlbi5u\nYXRzLmNvbTAdBgNVHQ4EFgQUbJSM8LNWmc9IgLe0X53yE3c3h1cwHwYDVR0jBBgw\nFoAUEqfeAemfeIp4MM4C7H1bJS+mra4wDQYJKoZIhvcNAQELBQADggEBAEGmLvEE\n+MTE1bHMbl/5QG+/xusmervIuxkfAfId0H+8TWB75y+yhVZpEdM7knfl+lexmtGQ\nGr4HNGTZhAZ3NYFaBw7nfeqO48He7gHUKfJA/zv7FREF3Fy+Qe/hydDJQzBzZfaU\n64XqhY6jOurpZhTAoOXfjpYzZaLi7+rdpTAbfxHCCAC8SxZD3++Q97ZeoT6en47O\n8SQQ7FIzWxs15k88oYalw51vZujxX7dz4l+LxsLXtlYW7ZM1163cgU7lF/jQqDcN\nz8X8jk1AjQY7AuFPuOzQ1hLXcZySm8rUG5pPgHrZ1QKmkFFWRaeCiO2hU794wI5C\nvIGR8lIhkNwEJnQ=\n-----END CERTIFICATE-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-key-ed25519.pem", "content": "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIJRCtUNxUuutNs9j8OtcwFw1xkbs+zxjHhpAqVuqDNo5\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-key-rsa-1024.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBALIcdpUeKcHeMGyJ\nxHiGnu6inPS7f46RSJwhRJBcNxaI/N28T7thn+zj5R3hrDiD2rTAfORlJoyDyeKX\ni0IqZOtQqiVVNWMIIIGa2qrcvDM0tqnKgMlHJexaoe3lNZhLFp/9RBPqxY6Coc4u\ntt3gnwXSyPESeIEYMdHVlekkIFohAgMBAAECgYAwt8RfyV5WnvXT2mMZLIlwcJ5J\n+rdLQcYAnsDoU7DlwxaXeBi/AlcCLtvOrpmy464A3t3KgzhmGu4vwo/ey0XK+nTQ\ntzORP/PXTaVC8DzJ8PnJmUaB7l+H7a88OSPLbjgnbpw4SyvDpKUHiiw0EDYC7L6Y\n1vvCOlnprptXbE5eeQJBAOpjwdBVWkVtmStjsxbxZsUTI7XKxS2VZinRLH0l5/hI\nhIHRxwy9oRbeNrf5815lGolTUD0mq+N0dJRlMop1yKsCQQDCiGDkH/pQqhB8ibmD\n0XNw0EzxJmPFACO/x49VCfCPE5p1FQhpyIl6JkyAFNN7Xs4HX8jMHTuvNgJVti61\nO0BjAkEAj0wr2vXDubyWrztF61nszcG0zFjKkeLL0fcLLvv0xQt4z3F0MyrgCH4U\nkAflLSm8voZMAQbagbXZ7DuuWY5G/wJAWyKnOdidXZL+3ElthwrmKVD86vEQRqe1\nF9C3HqDkeTM25mkvItfXSEmPB2Y6WY7luOCv4qhDYOdNmrgaE7+pfwJAcbV5ZVJW\nOZvH1ofsJVvUA8J58tzv1+KPb96pI3YRAu8xbMC0mzezPsYjg2wjaRgJ2C+7On27\nBaArNo75B20AkA==\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-key-rsa-2048.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLHc75nq7xjVKK\n4e7ezNZsu8tatmlKkSJmSWX7CTnbTlLUVFxAtxx+vi5t22M4F2L5HMgsK2R+VQjT\nUNuq7pxXv/bJ8u2HEF40OJcyGPI4WRgvDATjDsJ1PBZayZbfWhQfWthcgM+h5vtY\n8XvDc3BpQm5leeoz3wtGBYNOaH6eyoMNb9vPV9gh3Fw71rPqIE6cnYAZYja6vkbN\nZzj6A0w5mJlePrnuDQxvJUzgbPJzgC7baFKFn9iiss/w0CLA0lqaWsIXOWdk394+\nO5A9wjqCWdug70gAZ890A4mm19rK48BmBSeDN7AHfzJjRD+sobUYkpKRDxfr+VeJ\nZpE9kmNjAgMBAAECggEADicdEWKfsQAWZMv6X3bpZ/kr4b29F2+GdJcfrn7Fk8Tg\n25+nL/EyYJhy1r/HKZTjlhUN05oQbgcRztue+smWhjy/fvHY4CThU4Uv79MyKX/3\nwet1+bZBEXcm3ZuXUifOKCMl2Ug2b4MPN3LYG1XTWtpAo/x7N7MOb4oZzKBWVk0J\nGEfxvJVZyVe3libIN2WWZFQJ1620AxaWP2jZ83PhRR/TJIBiq+pI/lBIyt+nu8Z3\n3vzB020R2uvENOhnaDjzNVfnJxSvlAAQkN47zo2Tf4UnB0792Y7DEEjFKX4XCFVd\nzxmjmw7VcRcnyruDhCoRC6mNraaAHuMqPwuBoC6GtQKBgQD/7CV8XKzcPPfFJY9e\naHPzgXJwK+5u3jq7tNYUksVfv0s2lLQnRdbqAhHzxNYLQjdVd7J6t55h2z8scYaP\noB7TTwszhKZS2sQ/lcpfOFoFN6KjN0iOnXVFucGoQ36gexNqPw894NFKWX/RHrnZ\nUfL/OnOUPpra3w+WMxjUYQ5ivQKBgQDLLZDIpM8fSRKeqZzjv/eoKnoDhqmzPipj\nbvNXAkIr/nWfUHL9YRnpxX7PW8DqFWIYoM8b0uOKO4vLJnQUMyc2jPq1rPXEBrjk\nw+xKWCipKdPrqwttiLKme+ZArRT5CJ9qcqxSX0yNp69xwXtyHe+zY18F/a4+70wT\n5wJwYmhQnwKBgHpICTk8xtOMxg6K/c/sNMr65QU32Htc789Ufp3h6zDupC92KgZB\n1oiFaLKDMIq8ntfVk5ATQDgdnDfOHq9toIzyzbVWAmrAYNjI56NLt6eah7lY5vBN\nyAUC1sdhSJXBeOthKhU04IuX6/yto7t07piJA0SoDTHbNwVbcNe5cDg5AoGBALjR\njxVlDd+4mc5oHYXy1rZLRUg10+JvlyFyCLrKHCVmx9oO1Tr1fBvxggPfw+FraBtd\nFGiL8l2JAwXdydOiIHZ30Ys3dSxGrSOzsRqDjSEsIlEK+088/L2CkRWeHCjYliK/\ng08+zyVANtC0nrVU0/mLWCHb/AfVp4+nIMnYSmmjAoGAEMyBqq2AyUmx1xAmhw36\nLqgKy+vgHEAFRFPD8IttHFLOlUdXlvxoDq4xW2a7bJsJrs9ZrluRFKVh7QnSckmP\nJt/Plg+XYB3B2exD5Xyh9xNYNVW/Aqvg+NuiWeCGK/o7mUfGWd9qWrD2aw51m+X3\nSvtkgck1kulqPoUFG1b3R4k=\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/certs/tls/benchmark-server-key-rsa-4096.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCP+vDRakKeLNRK\nzTo3FKyO5hvqxa1hCMt3o7pIKhDCs9JjZm/S7bxHLAEmRKIhu/XaRbGuGq/6QOqG\na3u1/AR0cASA6XoTY/8IUOfIAniK3mqAVEdwQi0anUCnhmtnc5OXjUMN7byCJ+Ud\nvv5UDPFRXqvOQeauqRol3jVismgrJne778UezoK5076MZcLa5sfWLvUBFcefq2SY\n5PAV/U3j3AEevXjjEGY/K8f8D96WBcGNHbz/vHrgw7+oFmyv+VtNpLhjA3X6uh38\ndFzYYiInGlK0HVuH91dkJ9rRDKFPMI0sb+PsjN60Os8EbOnDZaO5+GRK5rGgwNaU\nOSR+qZDG0cnkv4+oiI9Pr23MRt+5L06DPsNbXQJylJ1dbXexYJ1I0IYS8+osWRrf\nqr0HzZXQYfO2bHxWp19ZK7iozJWZOzHHMtNPGlBThfVDZGdP/WKBuw4zrbQu6+u+\ndeWHlF2Du2lP75An9vjEIKDra2/hXA5F7bvEdhcx4OfhqhF3Kgjjn/5krrKrA0A6\nsoqmx78zy7OYkraLyl8QWaK0gqg1ifcxHZ0WsKqJ9mQV/2tf0D/Vib5YoA8uZZ0a\nnXd+9N3PFAZjjh/20EvZFaygZf77TVcTAiJbPuSo5sLQvx/PcR0g9pQtfsxmJCvH\nPOFEJwmHKf9o6sl8tiPQYCYhocSUeQIDAQABAoICAAEYUQ4KqeyJiJN0DvIIdc0q\nv1dwVEKlaOS6nvSkYJcWe9kL2E8JRfy1lIxTCneethPdoqhhZWljZn/fX+QRAVir\nBGxq5NCHxWgDCNble+oJydNlhgXldEcxnyJXBvM/trA49Q5X+pkeeWMvRsBiuSVx\nBqCi3CtZDQzw18TtzessF6OF2IuE2bYqP9anVyLH4rOvN5JKn9zH69PaLorq76rL\nYMp4JEFNKIs+R5RUFjwxBC6Q+r+9fM1qTQdtJOZMC293pfusylAoll315ZczXIah\nxYi/ThOmjtMrwWyEan1PcCIVzMI0X7o3q0eMQNU0ApmBY9k0+sYEnsJ8Um8l1icb\nWE/KBDzQk2mSoIzEdo99gYiuLAhi5WTEcQzeovZKHKkhRxLn3GUK/ZznpTuI0Voh\n1Qbfw05wf62yh8/nc747n5ULPgf8/nUH5G8jUeSis9gMGtFloqIxUEBphzIHwj7W\nFUIKoKpGkHL4Jkg9wXIe7lg5Qp+cdCrkWKKvQc57EbWxsM1Pjktw6X5ctNQclWG0\nWQJUw+BJhq6InKAV4aDW4/QlCOjvGtDDkYvwghE+ZgIzm9BARM4+DbeoPTmAkwwz\n6NTM8DYWvdcYVpgchEwphS6OFkOD1Dc6Wg2OUUtLjvl4NVf/NDYN8wIWNwmMm4aj\n+Is12NwoUqZ44E4pSfohAoIBAQDLloatDuObuLzInBqER2tByPNnDV9AVTR2rm6w\nTgBmslFHBCf/uSzq0ivh1oZmFfDhcqdrtheTL0BWXbFxR/GelDm9N4d/LqOGRyWV\nQmy5dGp2wE6gsZGZuvsbVst6eFO+zgDLvDMLrVqP+577g/QbrSR8gPz39WCgLpvf\nRcNpM2rHeKkm09IdUe0G+YEnW3fD7rxtMS+j8J57yuuxLO+SemGJuJQqGHYOukxF\nKMZxX2wMs1YMNwKEXY0/9M+LAtW9J3azkRp5xd+E2kzXGoWcl2nS/3+J6nh1+HWq\nSZE7SDWSimzut/aPTOM7YxLVqtANWHzVBBNpTbOMSrSZTZQJAoIBAQC1C/afOROb\nGQMb1LOt95AxvGyYo62mOkcPxhxQGojVIi+Yz8Q1ZWTuT5qyyWSofR3bE9mQwQGW\nKQ/znJKiwA48BS0osnmQi3Bi6eVD73NfwxMkL6zrGFxf336DNQi9mbidJvOaHNsT\nwXGPqRQXDcLa3s3WqerDzKJ2gxG22rtkoC0uCw+J9NlkJTTI9bJ9rjDZIh6mE/M0\n3ye19IgkBdbMv7FSjGVpovdqWZ2HELDYAXJftzuMPUO1GNYbyMHAHnWH0a1TQRM8\nELHzmPRaFBeegmpBVaenlxKoi72iwqrVFrEr5FLAKxLq/9RwizJ8FzySIpH/5+hU\nKy4mG93lmHjxAoIBAQDLOXs+jTpPW923M3yUxuYeSQYPvJ10jplMT1tWysZDvYS8\nqz1yW9qmnR4I1ihbB1Po+JZ/QsnNtsE2dViHiBV9AuGQLDopjtjVVXgCwsfdaIRN\n/jF+30JEfw3igIWlvy95rBHHThp2cZmRWM+eql2msvNVBT2AF4VY4K3f9rfV7+mL\nLLtNcuyvL/S3naB7NSccgte89/hiYfMSB8G2nvCW+2saGxJr4vcWRImWD9nnmiU1\nmF8w2ki88NXrHel/Dlll9FrdbN9M52T0LSW/I050vgB5C2q4tUGCIX7zeXRsBOzV\nVzDeKu0Ipuu9gGxwtY3xhH839FWcLGAqjvgwf+xhAoIBAQCuvWs9ZoNr0QpVFEiQ\nAj9kIa7W7DOwGtN3gAjXr6Sdwa8a2H1R5Bk0ghSXtxW2IXxtdI0qz35OhjdlM5u8\nBY43k+9wNkJqporEjWfA2B4NMWUKKhHFnu+ZgUbEMK3NAc9TrsKz3mH8gVqwA8rm\nLVwCj8UwCTQT4zBzHjI8wITZrFeu9vH6fx5LMDXwOGQcNcHj8LCQLvUv9KqJTgkQ\na6pUWDg3qlY/TRFrzi7iq9NjyJGxnFKXGpJ8+gm9K1kFquBZRKD7l/WOpbZ7nQdK\n4dWiIdGYWanFcWSK1MUlkKn9nTdHW8oau/g4ZM+QCGmjp3HIwiEUU6rDgiG6mm7j\nKPShAoIBAG1sFn8X841038Z/sp4JzYypQjj7g9UOTkghHcJHIZzRXLiTqW3lKxbt\nGz93DjWRxXD404hem8dOUf22VgfjB4z1mSrV5SWtLjLf5wD8gl0eUNKP2lZFLggz\nO6nHCLLzlKs2RH5pDo3c8qyjLRIfCXy0YGnr+9RErVJG+TVB/MOSgHagmkdVOYCH\nphw4EiwJ+rPFy/xm5D6+BuOyt7hw7boQsw3EHpZTyQHWObcIggjlopTX2VSG8Dx+\n/iQRTuVRVyNAhYwCuNtSh27zawWr+A40acFJsJpvkFkbZBEH1IqoCteUaiEVU1qm\n51lKgt3ZVAXuecJ1U/0u4HtC0QdEBGE=\n-----END PRIVATE KEY-----\n" }, { "path": "server/configs/one.creds", "content": "-----BEGIN NATS USER JWT-----\neyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJHRUNQVEpISE1TM01DTUtMVFBHWUdBTzQ1R1E2TjZRUFlXUTRHUExBRUIzM1ZDUkpOUlZRIiwiaWF0IjoxNjE2MjQ3MjMyLCJpc3MiOiJBQlZSWktKNlo3TklNUElZSlJDSEVZRlJVTzdFTk42TldPS1FERkxGREZWUFNNMzZVUFgyVUNQUCIsIm5hbWUiOiJvbmUiLCJzdWIiOiJVRENJQkdHR0hDSkJRUE9PNFNDSkpCSFpEUjM3TlFHR0NJV01ORzJEREFLSjZXTUtCTUFLWElNTyIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.VLhSDtGZEF_jdvgmhgkdISXAt5wFMMZxxwm5w8UrsnlM1hkUvtxBlTe4IP0xJIf4xf8JOR2Bmf73xUGJKUZECQ\n------END NATS USER JWT------\n\n************************* IMPORTANT *************************\nNKEY Seed printed below can be used to sign and prove identity.\nNKEYs are sensitive and should be treated as secrets.\n\n-----BEGIN USER NKEY SEED-----\nSUAPCDMU5TSHHLWUUZSOUABJXP2GXRCZVEOVWPSVM5XRSXYGQMRRFDYNMY\n------END USER NKEY SEED------\n\n*************************************************************\n" }, { "path": "server/const.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"regexp\"\n\t\"runtime/debug\"\n\t\"time\"\n)\n\n// Command is a signal used to control a running nats-server process.\ntype Command string\n\n// Valid Command values.\nconst (\n\tCommandStop = Command(\"stop\")\n\tCommandQuit = Command(\"quit\")\n\tCommandReopen = Command(\"reopen\")\n\tCommandReload = Command(\"reload\")\n\n\t// private for now\n\tcommandLDMode = Command(\"ldm\")\n\tcommandTerm = Command(\"term\")\n)\n\nvar (\n\t// gitCommit and serverVersion injected at build.\n\tgitCommit, serverVersion string\n\t// trustedKeys is a whitespace separated array of trusted operator's public nkeys.\n\ttrustedKeys string\n\t// SemVer regexp to validate the VERSION.\n\tsemVerRe = regexp.MustCompile(`^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$`)\n)\n\n// formatRevision formats a VCS revision string for display.\nfunc formatRevision(revision string) string {\n\tif len(revision) >= 7 {\n\t\treturn revision[:7]\n\t}\n\treturn revision\n}\n\nfunc init() {\n\t// Use build info if present, it would be if building using 'go build .'\n\t// or when using a release.\n\tif info, ok := debug.ReadBuildInfo(); ok {\n\t\tfor _, setting := range info.Settings {\n\t\t\tswitch setting.Key {\n\t\t\tcase \"vcs.revision\":\n\t\t\t\tgitCommit = formatRevision(setting.Value)\n\t\t\t}\n\t\t}\n\t}\n}\n\nconst (\n\t// VERSION is the current version for the server.\n\tVERSION = \"2.14.0-dev\"\n\n\t// PROTO is the currently supported protocol.\n\t// 0 was the original\n\t// 1 maintains proto 0, adds echo abilities for CONNECT from the client. Clients\n\t// should not send echo unless proto in INFO is >= 1.\n\tPROTO = 1\n\n\t// DEFAULT_PORT is the default port for client connections.\n\tDEFAULT_PORT = 4222\n\n\t// RANDOM_PORT is the value for port that, when supplied, will cause the\n\t// server to listen on a randomly-chosen available port. The resolved port\n\t// is available via the Addr() method.\n\tRANDOM_PORT = -1\n\n\t// DEFAULT_HOST defaults to all interfaces.\n\tDEFAULT_HOST = \"0.0.0.0\"\n\n\t// MAX_CONTROL_LINE_SIZE is the maximum allowed protocol control line size.\n\t// 4k should be plenty since payloads sans connect/info string are separate.\n\tMAX_CONTROL_LINE_SIZE = 4096\n\n\t// MAX_PAYLOAD_SIZE is the maximum allowed payload size. Should be using\n\t// something different if > 1MB payloads are needed.\n\tMAX_PAYLOAD_SIZE = (1024 * 1024)\n\n\t// MAX_PAYLOAD_MAX_SIZE is the size at which the server will warn about\n\t// max_payload being too high. In the future, the server may enforce/reject\n\t// max_payload above this value.\n\tMAX_PAYLOAD_MAX_SIZE = (8 * 1024 * 1024)\n\n\t// MAX_PENDING_SIZE is the maximum outbound pending bytes per client.\n\tMAX_PENDING_SIZE = (64 * 1024 * 1024)\n\n\t// DEFAULT_MAX_CONNECTIONS is the default maximum connections allowed.\n\tDEFAULT_MAX_CONNECTIONS = (64 * 1024)\n\n\t// TLS_TIMEOUT is the TLS wait time.\n\tTLS_TIMEOUT = 2 * time.Second\n\n\t// DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY is the default amount of\n\t// time for the server to wait for the TLS handshake with a client to\n\t// be initiated before falling back to sending the INFO protocol first.\n\t// See TLSHandshakeFirst and TLSHandshakeFirstFallback options.\n\tDEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY = 50 * time.Millisecond\n\n\t// AUTH_TIMEOUT is the authorization wait time.\n\tAUTH_TIMEOUT = 2 * time.Second\n\n\t// DEFAULT_PING_INTERVAL is how often pings are sent to clients, etc...\n\tDEFAULT_PING_INTERVAL = 2 * time.Minute\n\n\t// DEFAULT_PING_MAX_OUT is maximum allowed pings outstanding before disconnect.\n\tDEFAULT_PING_MAX_OUT = 2\n\n\t// CR_LF string\n\tCR_LF = \"\\r\\n\"\n\n\t// LEN_CR_LF hold onto the computed size.\n\tLEN_CR_LF = len(CR_LF)\n\n\t// DEFAULT_FLUSH_DEADLINE is the write/flush deadlines.\n\tDEFAULT_FLUSH_DEADLINE = 10 * time.Second\n\n\t// DEFAULT_HTTP_PORT is the default monitoring port.\n\tDEFAULT_HTTP_PORT = 8222\n\n\t// DEFAULT_HTTP_BASE_PATH is the default base path for monitoring.\n\tDEFAULT_HTTP_BASE_PATH = \"/\"\n\n\t// ACCEPT_MIN_SLEEP is the minimum acceptable sleep times on temporary errors.\n\tACCEPT_MIN_SLEEP = 10 * time.Millisecond\n\n\t// ACCEPT_MAX_SLEEP is the maximum acceptable sleep times on temporary errors\n\tACCEPT_MAX_SLEEP = 1 * time.Second\n\n\t// DEFAULT_ROUTE_CONNECT Route solicitation intervals.\n\tDEFAULT_ROUTE_CONNECT = 1 * time.Second\n\n\t// DEFAULT_ROUTE_CONNECT_MAX Route solicitation intervals (max).\n\tDEFAULT_ROUTE_CONNECT_MAX = 30 * time.Second\n\n\t// DEFAULT_ROUTE_RECONNECT Route reconnect delay.\n\tDEFAULT_ROUTE_RECONNECT = 1 * time.Second\n\n\t// DEFAULT_ROUTE_DIAL Route dial timeout.\n\tDEFAULT_ROUTE_DIAL = 1 * time.Second\n\n\t// DEFAULT_ROUTE_POOL_SIZE Route default pool size\n\tDEFAULT_ROUTE_POOL_SIZE = 3\n\n\t// DEFAULT_LEAF_NODE_RECONNECT LeafNode reconnect interval.\n\tDEFAULT_LEAF_NODE_RECONNECT = time.Second\n\n\t// DEFAULT_LEAF_TLS_TIMEOUT TLS timeout for LeafNodes\n\tDEFAULT_LEAF_TLS_TIMEOUT = 2 * time.Second\n\n\t// PROTO_SNIPPET_SIZE is the default size of proto to print on parse errors.\n\tPROTO_SNIPPET_SIZE = 32\n\n\t// MAX_CONTROL_LINE_SNIPPET_SIZE is the default size of proto to print on max control line errors.\n\tMAX_CONTROL_LINE_SNIPPET_SIZE = 128\n\n\t// MAX_MSG_ARGS Maximum possible number of arguments from MSG proto.\n\tMAX_MSG_ARGS = 4\n\n\t// MAX_RMSG_ARGS Maximum possible number of arguments from RMSG proto.\n\tMAX_RMSG_ARGS = 6\n\n\t// MAX_HMSG_ARGS Maximum possible number of arguments from HMSG proto.\n\tMAX_HMSG_ARGS = 7\n\n\t// MAX_PUB_ARGS Maximum possible number of arguments from PUB proto.\n\tMAX_PUB_ARGS = 3\n\n\t// MAX_HPUB_ARGS Maximum possible number of arguments from HPUB proto.\n\tMAX_HPUB_ARGS = 4\n\n\t// MAX_RSUB_ARGS Maximum possible number of arguments from a RS+/LS+ proto.\n\tMAX_RSUB_ARGS = 6\n\n\t// DEFAULT_MAX_CLOSED_CLIENTS is the maximum number of closed connections we hold onto.\n\tDEFAULT_MAX_CLOSED_CLIENTS = 10000\n\n\t// DEFAULT_LAME_DUCK_DURATION is the time in which the server spreads\n\t// the closing of clients when signaled to go in lame duck mode.\n\tDEFAULT_LAME_DUCK_DURATION = 2 * time.Minute\n\n\t// DEFAULT_LAME_DUCK_GRACE_PERIOD is the duration the server waits, after entering\n\t// lame duck mode, before starting closing client connections.\n\tDEFAULT_LAME_DUCK_GRACE_PERIOD = 10 * time.Second\n\n\t// DEFAULT_LEAFNODE_INFO_WAIT Route dial timeout.\n\tDEFAULT_LEAFNODE_INFO_WAIT = 1 * time.Second\n\n\t// DEFAULT_LEAFNODE_PORT is the default port for remote leafnode connections.\n\tDEFAULT_LEAFNODE_PORT = 7422\n\n\t// DEFAULT_CONNECT_ERROR_REPORTS is the number of attempts at which a\n\t// repeated failed route, gateway or leaf node connection is reported.\n\t// This is used for initial connection, that is, when the server has\n\t// never had a connection to the given endpoint. Once connected, and\n\t// if a disconnect occurs, DEFAULT_RECONNECT_ERROR_REPORTS is used\n\t// instead.\n\t// The default is to report every 3600 attempts (roughly every hour).\n\tDEFAULT_CONNECT_ERROR_REPORTS = 3600\n\n\t// DEFAULT_RECONNECT_ERROR_REPORTS is the default number of failed\n\t// attempt to reconnect a route, gateway or leaf node connection.\n\t// The default is to report every attempt.\n\tDEFAULT_RECONNECT_ERROR_REPORTS = 1\n\n\t// DEFAULT_RTT_MEASUREMENT_INTERVAL is how often we want to measure RTT from\n\t// this server to clients, routes, gateways or leafnode connections.\n\tDEFAULT_RTT_MEASUREMENT_INTERVAL = time.Hour\n\n\t// DEFAULT_ALLOW_RESPONSE_MAX_MSGS is the default number of responses allowed\n\t// for a reply subject.\n\tDEFAULT_ALLOW_RESPONSE_MAX_MSGS = 1\n\n\t// DEFAULT_ALLOW_RESPONSE_EXPIRATION is the default time allowed for a given\n\t// dynamic response permission.\n\tDEFAULT_ALLOW_RESPONSE_EXPIRATION = 2 * time.Minute\n\n\t// DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD is the default time that the system will\n\t// expect a service export response to be delivered. This is used in corner cases for\n\t// time based cleanup of reverse mapping structures.\n\tDEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD = 2 * time.Minute\n\n\t// DEFAULT_SERVICE_LATENCY_SAMPLING is the default sampling rate for service\n\t// latency metrics\n\tDEFAULT_SERVICE_LATENCY_SAMPLING = 100\n\n\t// DEFAULT_SYSTEM_ACCOUNT\n\tDEFAULT_SYSTEM_ACCOUNT = \"$SYS\"\n\n\t// DEFAULT GLOBAL_ACCOUNT\n\tDEFAULT_GLOBAL_ACCOUNT = \"$G\"\n\n\t// DEFAULT_FETCH_TIMEOUT is the default time that the system will wait for an account fetch to return.\n\tDEFAULT_ACCOUNT_FETCH_TIMEOUT = 1900 * time.Millisecond\n)\n" }, { "path": "server/consumer.go", "content": "// Copyright 2019-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/avl\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nuid\"\n\t\"golang.org/x/time/rate\"\n)\n\n// Headers sent with Request Timeout\nconst (\n\tJSPullRequestPendingMsgs = \"Nats-Pending-Messages\"\n\tJSPullRequestPendingBytes = \"Nats-Pending-Bytes\"\n\tJSPullRequestNatsPinId = \"Nats-Pin-Id\"\n)\n\nvar (\n\tvalidGroupName = regexp.MustCompile(`^[a-zA-Z0-9/_=-]{1,16}$`)\n)\n\n// Headers sent when batch size was completed, but there were remaining bytes.\nconst JsPullRequestRemainingBytesT = \"NATS/1.0 409 Batch Completed\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\"\n\ntype ConsumerInfo struct {\n\tStream string `json:\"stream_name\"`\n\tName string `json:\"name\"`\n\tCreated time.Time `json:\"created\"`\n\tConfig *ConsumerConfig `json:\"config,omitempty\"`\n\tDelivered SequenceInfo `json:\"delivered\"`\n\tAckFloor SequenceInfo `json:\"ack_floor\"`\n\tNumAckPending int `json:\"num_ack_pending\"`\n\tNumRedelivered int `json:\"num_redelivered\"`\n\tNumWaiting int `json:\"num_waiting\"`\n\tNumPending uint64 `json:\"num_pending\"`\n\tCluster *ClusterInfo `json:\"cluster,omitempty\"`\n\tPushBound bool `json:\"push_bound,omitempty\"`\n\tPaused bool `json:\"paused,omitempty\"`\n\tPauseRemaining time.Duration `json:\"pause_remaining,omitempty\"`\n\t// TimeStamp indicates when the info was gathered\n\tTimeStamp time.Time `json:\"ts\"`\n\tPriorityGroups []PriorityGroupState `json:\"priority_groups,omitempty\"`\n}\n\n// consumerInfoClusterResponse is a response used in a cluster to communicate the consumer info\n// back to the meta leader as part of a consumer list request.\ntype consumerInfoClusterResponse struct {\n\tConsumerInfo\n\tOfflineReason string `json:\"offline_reason,omitempty\"` // Reporting when a consumer is offline.\n}\n\ntype PriorityGroupState struct {\n\tGroup string `json:\"group\"`\n\tPinnedClientID string `json:\"pinned_client_id,omitempty\"`\n\tPinnedTS time.Time `json:\"pinned_ts,omitempty\"`\n}\n\ntype ConsumerConfig struct {\n\tDurable string `json:\"durable_name,omitempty\"`\n\tName string `json:\"name,omitempty\"`\n\tDescription string `json:\"description,omitempty\"`\n\tDeliverPolicy DeliverPolicy `json:\"deliver_policy\"`\n\tOptStartSeq uint64 `json:\"opt_start_seq,omitempty\"`\n\tOptStartTime *time.Time `json:\"opt_start_time,omitempty\"`\n\tAckPolicy AckPolicy `json:\"ack_policy\"`\n\tAckWait time.Duration `json:\"ack_wait,omitempty\"`\n\tMaxDeliver int `json:\"max_deliver,omitempty\"`\n\tBackOff []time.Duration `json:\"backoff,omitempty\"`\n\tFilterSubject string `json:\"filter_subject,omitempty\"`\n\tFilterSubjects []string `json:\"filter_subjects,omitempty\"`\n\tReplayPolicy ReplayPolicy `json:\"replay_policy\"`\n\tRateLimit uint64 `json:\"rate_limit_bps,omitempty\"` // Bits per sec\n\tSampleFrequency string `json:\"sample_freq,omitempty\"`\n\tMaxWaiting int `json:\"max_waiting,omitempty\"`\n\tMaxAckPending int `json:\"max_ack_pending,omitempty\"`\n\tFlowControl bool `json:\"flow_control,omitempty\"`\n\tHeadersOnly bool `json:\"headers_only,omitempty\"`\n\n\t// Pull based options.\n\tMaxRequestBatch int `json:\"max_batch,omitempty\"`\n\tMaxRequestExpires time.Duration `json:\"max_expires,omitempty\"`\n\tMaxRequestMaxBytes int `json:\"max_bytes,omitempty\"`\n\n\t// Push based consumers.\n\tDeliverSubject string `json:\"deliver_subject,omitempty\"`\n\tDeliverGroup string `json:\"deliver_group,omitempty\"`\n\tHeartbeat time.Duration `json:\"idle_heartbeat,omitempty\"`\n\n\t// Ephemeral inactivity threshold.\n\tInactiveThreshold time.Duration `json:\"inactive_threshold,omitempty\"`\n\n\t// Generally inherited by parent stream and other markers, now can be configured directly.\n\tReplicas int `json:\"num_replicas\"`\n\t// Force memory storage.\n\tMemoryStorage bool `json:\"mem_storage,omitempty\"`\n\n\t// Don't add to general clients.\n\tDirect bool `json:\"direct,omitempty\"`\n\n\t// Metadata is additional metadata for the Consumer.\n\tMetadata map[string]string `json:\"metadata,omitempty\"`\n\n\t// PauseUntil is for suspending the consumer until the deadline.\n\tPauseUntil *time.Time `json:\"pause_until,omitempty\"`\n\n\t// Priority groups\n\tPriorityGroups []string `json:\"priority_groups,omitempty\"`\n\tPriorityPolicy PriorityPolicy `json:\"priority_policy,omitempty\"`\n\tPinnedTTL time.Duration `json:\"priority_timeout,omitempty\"`\n}\n\n// SequenceInfo has both the consumer and the stream sequence and last activity.\ntype SequenceInfo struct {\n\tConsumer uint64 `json:\"consumer_seq\"`\n\tStream uint64 `json:\"stream_seq\"`\n\tLast *time.Time `json:\"last_active,omitempty\"`\n}\n\ntype CreateConsumerRequest struct {\n\tStream string `json:\"stream_name\"`\n\tConfig ConsumerConfig `json:\"config\"`\n\tAction ConsumerAction `json:\"action\"`\n\tPedantic bool `json:\"pedantic,omitempty\"`\n}\n\ntype ConsumerAction int\n\nconst (\n\tActionCreateOrUpdate ConsumerAction = iota\n\tActionUpdate\n\tActionCreate\n)\n\nconst (\n\tactionUpdateJSONString = `\"update\"`\n\tactionCreateJSONString = `\"create\"`\n\tactionCreateOrUpdateJSONString = `\"\"`\n)\n\nvar (\n\tactionUpdateJSONBytes = []byte(actionUpdateJSONString)\n\tactionCreateJSONBytes = []byte(actionCreateJSONString)\n\tactionCreateOrUpdateJSONBytes = []byte(actionCreateOrUpdateJSONString)\n)\n\nfunc (a ConsumerAction) String() string {\n\tswitch a {\n\tcase ActionCreateOrUpdate:\n\t\treturn actionCreateOrUpdateJSONString\n\tcase ActionCreate:\n\t\treturn actionCreateJSONString\n\tcase ActionUpdate:\n\t\treturn actionUpdateJSONString\n\t}\n\treturn actionCreateOrUpdateJSONString\n}\n\nfunc (a ConsumerAction) MarshalJSON() ([]byte, error) {\n\tswitch a {\n\tcase ActionCreate:\n\t\treturn actionCreateJSONBytes, nil\n\tcase ActionUpdate:\n\t\treturn actionUpdateJSONBytes, nil\n\tcase ActionCreateOrUpdate:\n\t\treturn actionCreateOrUpdateJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", a)\n\t}\n}\n\nfunc (a *ConsumerAction) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase actionCreateJSONString:\n\t\t*a = ActionCreate\n\tcase actionUpdateJSONString:\n\t\t*a = ActionUpdate\n\tcase actionCreateOrUpdateJSONString:\n\t\t*a = ActionCreateOrUpdate\n\tdefault:\n\t\treturn fmt.Errorf(\"unknown consumer action: %v\", string(data))\n\t}\n\treturn nil\n}\n\n// ConsumerNakOptions is for optional NAK values, e.g. delay.\ntype ConsumerNakOptions struct {\n\tDelay time.Duration `json:\"delay\"`\n}\n\n// PriorityPolicy determines policy for selecting messages based on priority.\ntype PriorityPolicy int\n\nconst (\n\t// No priority policy.\n\tPriorityNone PriorityPolicy = iota\n\t// Clients will get the messages only if certain criteria are specified.\n\tPriorityOverflow\n\t// Single client takes over handling of the messages, while others are on standby.\n\tPriorityPinnedClient\n\t// Clients with lowest priority will be selected first.\n\tPriorityPrioritized\n)\n\nconst (\n\tPriorityNoneJSONString = `\"none\"`\n\tPriorityOverflowJSONString = `\"overflow\"`\n\tPriorityPinnedClientJSONString = `\"pinned_client\"`\n\tPriorityPrioritizedJSONString = `\"prioritized\"`\n)\n\nvar (\n\tPriorityNoneJSONBytes = []byte(PriorityNoneJSONString)\n\tPriorityOverflowJSONBytes = []byte(PriorityOverflowJSONString)\n\tPriorityPinnedClientJSONBytes = []byte(PriorityPinnedClientJSONString)\n\tPriorityPrioritizedJSONBytes = []byte(PriorityPrioritizedJSONString)\n)\n\nfunc (pp PriorityPolicy) String() string {\n\tswitch pp {\n\tcase PriorityOverflow:\n\t\treturn PriorityOverflowJSONString\n\tcase PriorityPinnedClient:\n\t\treturn PriorityPinnedClientJSONString\n\tcase PriorityPrioritized:\n\t\treturn PriorityPrioritizedJSONString\n\tdefault:\n\t\treturn PriorityNoneJSONString\n\t}\n}\n\nfunc (pp PriorityPolicy) MarshalJSON() ([]byte, error) {\n\tswitch pp {\n\tcase PriorityOverflow:\n\t\treturn PriorityOverflowJSONBytes, nil\n\tcase PriorityPinnedClient:\n\t\treturn PriorityPinnedClientJSONBytes, nil\n\tcase PriorityPrioritized:\n\t\treturn PriorityPrioritizedJSONBytes, nil\n\tcase PriorityNone:\n\t\treturn PriorityNoneJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown priority policy: %v\", pp)\n\t}\n}\n\nfunc (pp *PriorityPolicy) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase PriorityOverflowJSONString:\n\t\t*pp = PriorityOverflow\n\tcase PriorityPinnedClientJSONString:\n\t\t*pp = PriorityPinnedClient\n\tcase PriorityPrioritizedJSONString:\n\t\t*pp = PriorityPrioritized\n\tcase PriorityNoneJSONString:\n\t\t*pp = PriorityNone\n\tdefault:\n\t\treturn fmt.Errorf(\"unknown priority policy: %v\", string(data))\n\t}\n\treturn nil\n}\n\n// DeliverPolicy determines how the consumer should select the first message to deliver.\ntype DeliverPolicy int\n\nconst (\n\t// DeliverAll will be the default so can be omitted from the request.\n\tDeliverAll DeliverPolicy = iota\n\t// DeliverLast will start the consumer with the last sequence received.\n\tDeliverLast\n\t// DeliverNew will only deliver new messages that are sent after the consumer is created.\n\tDeliverNew\n\t// DeliverByStartSequence will look for a defined starting sequence to start.\n\tDeliverByStartSequence\n\t// DeliverByStartTime will select the first messsage with a timestamp >= to StartTime.\n\tDeliverByStartTime\n\t// DeliverLastPerSubject will start the consumer with the last message for all subjects received.\n\tDeliverLastPerSubject\n)\n\nfunc (dp DeliverPolicy) String() string {\n\tswitch dp {\n\tcase DeliverAll:\n\t\treturn \"all\"\n\tcase DeliverLast:\n\t\treturn \"last\"\n\tcase DeliverNew:\n\t\treturn \"new\"\n\tcase DeliverByStartSequence:\n\t\treturn \"by_start_sequence\"\n\tcase DeliverByStartTime:\n\t\treturn \"by_start_time\"\n\tcase DeliverLastPerSubject:\n\t\treturn \"last_per_subject\"\n\tdefault:\n\t\treturn \"undefined\"\n\t}\n}\n\n// AckPolicy determines how the consumer should acknowledge delivered messages.\ntype AckPolicy int\n\nconst (\n\t// AckNone requires no acks for delivered messages.\n\tAckNone AckPolicy = iota\n\t// AckAll when acking a sequence number, this implicitly acks all sequences below this one as well.\n\tAckAll\n\t// AckExplicit requires ack or nack for all messages.\n\tAckExplicit\n)\n\nfunc (a AckPolicy) String() string {\n\tswitch a {\n\tcase AckNone:\n\t\treturn \"none\"\n\tcase AckAll:\n\t\treturn \"all\"\n\tdefault:\n\t\treturn \"explicit\"\n\t}\n}\n\n// ReplayPolicy determines how the consumer should replay messages it already has queued in the stream.\ntype ReplayPolicy int\n\nconst (\n\t// ReplayInstant will replay messages as fast as possible.\n\tReplayInstant ReplayPolicy = iota\n\t// ReplayOriginal will maintain the same timing as the messages were received.\n\tReplayOriginal\n)\n\nfunc (r ReplayPolicy) String() string {\n\tswitch r {\n\tcase ReplayInstant:\n\t\treturn replayInstantPolicyJSONString\n\tdefault:\n\t\treturn replayOriginalPolicyJSONString\n\t}\n}\n\n// OK\nconst OK = \"+OK\"\n\n// Ack responses. Note that a nil or no payload is same as AckAck\nvar (\n\t// Ack\n\tAckAck = []byte(\"+ACK\") // nil or no payload to ack subject also means ACK\n\tAckOK = []byte(OK) // deprecated but +OK meant ack as well.\n\n\t// Nack\n\tAckNak = []byte(\"-NAK\")\n\t// Progress indicator\n\tAckProgress = []byte(\"+WPI\")\n\t// Ack + Deliver the next message(s).\n\tAckNext = []byte(\"+NXT\")\n\t// Terminate delivery of the message.\n\tAckTerm = []byte(\"+TERM\")\n)\n\nconst (\n\t// reasons to supply when terminating messages using limits\n\tackTermLimitsReason = \"Message deleted by stream limits\"\n\tackTermUnackedLimitsReason = \"Unacknowledged message was deleted\"\n)\n\n// Calculate accurate replicas for the consumer config with the parent stream config.\nfunc (consCfg ConsumerConfig) replicas(strCfg *StreamConfig) int {\n\tif consCfg.Replicas == 0 || consCfg.Replicas > strCfg.Replicas {\n\t\tif !isDurableConsumer(&consCfg) && strCfg.Retention == LimitsPolicy && consCfg.Replicas == 0 {\n\t\t\t// Matches old-school ephemerals only, where the replica count is 0.\n\t\t\treturn 1\n\t\t}\n\t\treturn strCfg.Replicas\n\t}\n\treturn consCfg.Replicas\n}\n\n// Consumer is a jetstream consumer.\ntype consumer struct {\n\t// Atomic used to notify that we want to process an ack.\n\t// This will be checked in checkPending to abort processing\n\t// and let ack be processed in priority.\n\tawl int64\n\tleader atomic.Bool\n\tmu sync.RWMutex\n\tjs *jetStream\n\tmset *stream\n\tacc *Account\n\tsrv *Server\n\tclient *client\n\tsysc *client\n\tsid int\n\tname string\n\tstream string\n\tsseq uint64 // next stream sequence\n\tsubjf subjectFilters // subject filters and their sequences\n\tfilters *gsl.SimpleSublist // When we have multiple filters we will use LoadNextMsgMulti and pass this in.\n\tdseq uint64 // delivered consumer sequence\n\tadflr uint64 // ack delivery floor\n\tasflr uint64 // ack store floor\n\tchkflr uint64 // our check floor, interest streams only.\n\tnpc int64 // Num Pending Count\n\tnpf uint64 // Num Pending Floor Sequence\n\tdsubj string\n\tqgroup string\n\tlss *lastSeqSkipList\n\trlimit *rate.Limiter\n\treqSub *subscription\n\tresetSub *subscription\n\tackSub *subscription\n\tackReplyT string\n\tackSubj string\n\tnextMsgSubj string\n\tnextMsgReqs *ipQueue[*nextMsgReq]\n\tresetSubj string\n\tmaxp int\n\tpblimit int\n\tmaxpb int\n\tpbytes int\n\tfcsz int\n\tfcid string\n\tfcSub *subscription\n\toutq *jsOutQ\n\tpending map[uint64]*Pending\n\tptmr *time.Timer\n\tptmrEnd time.Time\n\trdq []uint64\n\trdqi avl.SequenceSet\n\trdc map[uint64]uint64\n\treplies map[uint64]string\n\tpendingDeliveries map[uint64]*jsPubMsg // Messages that can be delivered after achieving quorum.\n\twaitingDeliveries map[string]*waitingDelivery // (Optional) request timeout messages that need to wait for replicated deliveries first.\n\tmaxdc uint64\n\twaiting *waitQueue\n\tcfg ConsumerConfig\n\tici *ConsumerInfo\n\tstore ConsumerStore\n\tactive bool\n\treplay bool\n\tdtmr *time.Timer\n\tuptmr *time.Timer // Unpause timer\n\tgwdtmr *time.Timer\n\tdthresh time.Duration\n\tmch chan struct{} // Message channel\n\tqch chan struct{} // Quit channel\n\tmqch chan struct{} // The monitor's quit channel.\n\tinch chan bool // Interest change channel\n\tsfreq int32\n\tackEventT string\n\tnakEventT string\n\tdeliveryExcEventT string\n\tcreated time.Time\n\tldt time.Time\n\tlat time.Time\n\tlwqic time.Time\n\tclosed bool\n\n\t// Clustered.\n\tca *consumerAssignment\n\tnode RaftNode\n\tinfoSub *subscription\n\tlqsent time.Time\n\tprm map[string]struct{}\n\tprOk bool\n\tuch chan struct{}\n\tretention RetentionPolicy\n\n\tmonitorWg sync.WaitGroup\n\tinMonitor bool\n\n\t// R>1 proposals\n\tpch chan struct{}\n\tphead *proposal\n\tptail *proposal\n\n\t// Ack queue\n\tackMsgs *ipQueue[*jsAckMsg]\n\n\t// for stream signaling when multiple filters are set.\n\tsigSubs []string\n\n\t// Priority groups\n\t// Details described in ADR-42.\n\n\t// currentPinId is the current nuid for the pinned consumer.\n\t// If the Consumer is running in `PriorityPinnedClient` mode, server will\n\t// pick up a new nuid and assign it to first pending pull request.\n\tcurrentPinId string\n\t/// pinnedTtl is the remaining time before the current PinId expires.\n\tpinnedTtl *time.Timer\n\tpinnedTS time.Time\n\n\t// If standalone/single-server, the offline reason needs to be stored directly in the consumer.\n\t// Otherwise, if clustered it will be part of the consumer assignment.\n\tofflineReason string\n}\n\n// A single subject filter.\ntype subjectFilter struct {\n\tsubject string\n\ttokenizedSubject []string\n\thasWildcard bool\n}\n\ntype subjectFilters []*subjectFilter\n\n// subjects is a helper function used for updating consumers.\n// It is not used and should not be used in hotpath.\nfunc (s subjectFilters) subjects() []string {\n\tsubjects := make([]string, 0, len(s))\n\tfor _, filter := range s {\n\t\tsubjects = append(subjects, filter.subject)\n\t}\n\treturn subjects\n}\n\ntype proposal struct {\n\tdata []byte\n\tnext *proposal\n}\n\nconst (\n\t// JsAckWaitDefault is the default AckWait, only applicable on explicit ack policy consumers.\n\tJsAckWaitDefault = 30 * time.Second\n\t// JsDeleteWaitTimeDefault is the default amount of time we will wait for non-durable\n\t// consumers to be in an inactive state before deleting them.\n\tJsDeleteWaitTimeDefault = 5 * time.Second\n\t// JsFlowControlMaxPending specifies default pending bytes during flow control that can be outstanding.\n\tJsFlowControlMaxPending = 32 * 1024 * 1024\n\t// JsDefaultMaxAckPending is set for consumers with explicit ack that do not set the max ack pending.\n\tJsDefaultMaxAckPending = 1000\n\t// JsDefaultPinnedTTL is the default grace period for the pinned consumer to send a new request before a new pin\n\t// is picked by a server.\n\tJsDefaultPinnedTTL = 2 * time.Minute\n)\n\n// Helper function to set consumer config defaults from above.\nfunc setConsumerConfigDefaults(config *ConsumerConfig, streamCfg *StreamConfig, lim *JSLimitOpts, accLim *JetStreamAccountLimits, pedantic bool) *ApiError {\n\t// Setup default of -1, meaning no limit for MaxDeliver.\n\tif config.MaxDeliver == 0 || config.MaxDeliver < -1 {\n\t\tif pedantic && config.MaxDeliver < -1 {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_deliver must be set to -1\"))\n\t\t}\n\t\tconfig.MaxDeliver = -1\n\t}\n\t// Setup zero defaults.\n\tif config.MaxWaiting < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_waiting must not be negative\"))\n\t\t}\n\t\tconfig.MaxWaiting = 0\n\t}\n\tif config.MaxAckPending < -1 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_ack_pending must be set to -1\"))\n\t\t}\n\t\tconfig.MaxAckPending = -1\n\t}\n\tif config.MaxRequestBatch < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_batch must not be negative\"))\n\t\t}\n\t\tconfig.MaxRequestBatch = 0\n\t}\n\tif config.MaxRequestExpires < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_expires must not be negative\"))\n\t\t}\n\t\tconfig.MaxRequestExpires = 0\n\t}\n\tif config.MaxRequestMaxBytes < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_bytes must not be negative\"))\n\t\t}\n\t\tconfig.MaxRequestMaxBytes = 0\n\t}\n\tif config.Heartbeat < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"idle_heartbeat must not be negative\"))\n\t\t}\n\t\tconfig.Heartbeat = 0\n\t}\n\tif config.InactiveThreshold < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"inactive_threshold must not be negative\"))\n\t\t}\n\t\tconfig.InactiveThreshold = 0\n\t}\n\tif config.PinnedTTL < 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"priority_timeout must not be negative\"))\n\t\t}\n\t\tconfig.PinnedTTL = 0\n\t}\n\n\t// Set to default if not specified.\n\tif config.DeliverSubject == _EMPTY_ && config.MaxWaiting == 0 {\n\t\tconfig.MaxWaiting = JSWaitQueueDefaultMax\n\t}\n\t// Setup proper default for ack wait if we are in explicit ack mode.\n\tif config.AckWait == 0 && (config.AckPolicy == AckExplicit || config.AckPolicy == AckAll) {\n\t\tconfig.AckWait = JsAckWaitDefault\n\t}\n\t// If BackOff was specified that will override the AckWait and the MaxDeliver.\n\tif len(config.BackOff) > 0 {\n\t\tif pedantic && config.AckWait != config.BackOff[0] {\n\t\t\treturn NewJSPedanticError(errors.New(\"first backoff value has to equal batch AckWait\"))\n\t\t}\n\t\tconfig.AckWait = config.BackOff[0]\n\t}\n\tif config.MaxAckPending == 0 {\n\t\tif pedantic && streamCfg.ConsumerLimits.MaxAckPending > 0 {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_ack_pending must be set if it's configured in stream limits\"))\n\t\t}\n\t\tconfig.MaxAckPending = streamCfg.ConsumerLimits.MaxAckPending\n\t}\n\tif config.InactiveThreshold == 0 {\n\t\tif pedantic && streamCfg.ConsumerLimits.InactiveThreshold > 0 {\n\t\t\treturn NewJSPedanticError(errors.New(\"inactive_threshold must be set if it's configured in stream limits\"))\n\t\t}\n\t\tconfig.InactiveThreshold = streamCfg.ConsumerLimits.InactiveThreshold\n\t}\n\t// Set proper default for max ack pending if we are ack explicit and none has been set.\n\tif (config.AckPolicy == AckExplicit || config.AckPolicy == AckAll) && config.MaxAckPending == 0 {\n\t\tackPending := JsDefaultMaxAckPending\n\t\tif lim.MaxAckPending > 0 && lim.MaxAckPending < ackPending {\n\t\t\tackPending = lim.MaxAckPending\n\t\t}\n\t\tif accLim.MaxAckPending > 0 && accLim.MaxAckPending < ackPending {\n\t\t\tackPending = accLim.MaxAckPending\n\t\t}\n\t\tconfig.MaxAckPending = ackPending\n\t}\n\t// if applicable set max request batch size\n\tif config.DeliverSubject == _EMPTY_ && config.MaxRequestBatch == 0 && lim.MaxRequestBatch > 0 {\n\t\tif pedantic {\n\t\t\treturn NewJSPedanticError(errors.New(\"max_request_batch must be set if it's JetStream limits are set\"))\n\t\t}\n\t\tconfig.MaxRequestBatch = lim.MaxRequestBatch\n\t}\n\n\t// set the default value only if pinned policy is used.\n\tif config.PriorityPolicy == PriorityPinnedClient && config.PinnedTTL == 0 {\n\t\tconfig.PinnedTTL = JsDefaultPinnedTTL\n\t}\n\treturn nil\n}\n\n// Check the consumer config. If we are recovering don't check filter subjects.\nfunc checkConsumerCfg(\n\tconfig *ConsumerConfig,\n\tsrvLim *JSLimitOpts,\n\tcfg *StreamConfig,\n\t_ *Account,\n\taccLim *JetStreamAccountLimits,\n\tisRecovering bool,\n) *ApiError {\n\n\t// Check if replicas is defined but exceeds parent stream.\n\tif config.Replicas > 0 && config.Replicas > cfg.Replicas {\n\t\treturn NewJSConsumerReplicasExceedsStreamError()\n\t}\n\t// Check that it is not negative\n\tif config.Replicas < 0 {\n\t\treturn NewJSReplicasCountCannotBeNegativeError()\n\t}\n\t// If the stream is interest or workqueue retention make sure the replicas\n\t// match that of the stream. This is REQUIRED for now.\n\tif cfg.Retention == InterestPolicy || cfg.Retention == WorkQueuePolicy {\n\t\t// Only error here if not recovering.\n\t\t// We handle recovering in a different spot to allow consumer to come up\n\t\t// if previous version allowed it to be created. We do not want it to not come up.\n\t\tif !isRecovering && config.Replicas != 0 && config.Replicas != cfg.Replicas {\n\t\t\treturn NewJSConsumerReplicasShouldMatchStreamError()\n\t\t}\n\t}\n\n\tif _, err := config.AckPolicy.MarshalJSON(); err != nil {\n\t\treturn NewJSConsumerAckPolicyInvalidError()\n\t}\n\tif _, err := config.ReplayPolicy.MarshalJSON(); err != nil {\n\t\treturn NewJSConsumerReplayPolicyInvalidError()\n\t}\n\n\t// Check not negative AckWait/BackOff\n\tfor _, backoff := range config.BackOff {\n\t\tif backoff < 0 {\n\t\t\treturn NewJSConsumerBackOffNegativeError()\n\t\t}\n\t}\n\tif config.AckWait < 0 {\n\t\treturn NewJSConsumerAckWaitNegativeError()\n\t}\n\n\t// Check if we have a BackOff defined that MaxDeliver is within range etc.\n\tif lbo := len(config.BackOff); lbo > 0 && config.MaxDeliver != -1 && lbo > config.MaxDeliver {\n\t\treturn NewJSConsumerMaxDeliverBackoffError()\n\t}\n\n\tif len(config.Description) > JSMaxDescriptionLen {\n\t\treturn NewJSConsumerDescriptionTooLongError(JSMaxDescriptionLen)\n\t}\n\n\t// For now expect a literal subject if its not empty. Empty means work queue mode (pull mode).\n\tif config.DeliverSubject != _EMPTY_ {\n\t\tif !subjectIsLiteral(config.DeliverSubject) {\n\t\t\treturn NewJSConsumerDeliverToWildcardsError()\n\t\t}\n\t\tif !IsValidSubject(config.DeliverSubject) {\n\t\t\treturn NewJSConsumerInvalidDeliverSubjectError()\n\t\t}\n\t\tif deliveryFormsCycle(cfg, config.DeliverSubject) {\n\t\t\treturn NewJSConsumerDeliverCycleError()\n\t\t}\n\t\tif config.MaxWaiting != 0 {\n\t\t\treturn NewJSConsumerPushMaxWaitingError()\n\t\t}\n\t\tif config.MaxAckPending > 0 && config.AckPolicy == AckNone {\n\t\t\treturn NewJSConsumerMaxPendingAckPolicyRequiredError()\n\t\t}\n\t\tif config.Heartbeat > 0 && config.Heartbeat < 100*time.Millisecond {\n\t\t\treturn NewJSConsumerSmallHeartbeatError()\n\t\t}\n\t} else {\n\t\t// Pull mode with work queue retention from the stream requires an explicit ack.\n\t\tif config.AckPolicy == AckNone && cfg.Retention == WorkQueuePolicy {\n\t\t\treturn NewJSConsumerPullRequiresAckError()\n\t\t}\n\t\tif config.RateLimit > 0 {\n\t\t\treturn NewJSConsumerPullWithRateLimitError()\n\t\t}\n\t\tif config.MaxWaiting < 0 {\n\t\t\treturn NewJSConsumerMaxWaitingNegativeError()\n\t\t}\n\t\tif config.Heartbeat > 0 {\n\t\t\treturn NewJSConsumerHBRequiresPushError()\n\t\t}\n\t\tif config.FlowControl {\n\t\t\treturn NewJSConsumerFCRequiresPushError()\n\t\t}\n\t\tif config.MaxRequestBatch < 0 {\n\t\t\treturn NewJSConsumerMaxRequestBatchNegativeError()\n\t\t}\n\t\tif config.MaxRequestExpires != 0 && config.MaxRequestExpires < time.Millisecond {\n\t\t\treturn NewJSConsumerMaxRequestExpiresTooSmallError()\n\t\t}\n\t\tif srvLim.MaxRequestBatch > 0 && config.MaxRequestBatch > srvLim.MaxRequestBatch {\n\t\t\treturn NewJSConsumerMaxRequestBatchExceededError(srvLim.MaxRequestBatch)\n\t\t}\n\t}\n\tif srvLim.MaxAckPending > 0 && config.MaxAckPending > srvLim.MaxAckPending {\n\t\treturn NewJSConsumerMaxPendingAckExcessError(srvLim.MaxAckPending)\n\t}\n\tif accLim.MaxAckPending > 0 && config.MaxAckPending > accLim.MaxAckPending {\n\t\treturn NewJSConsumerMaxPendingAckExcessError(accLim.MaxAckPending)\n\t}\n\tif cfg.ConsumerLimits.MaxAckPending > 0 && config.MaxAckPending > cfg.ConsumerLimits.MaxAckPending {\n\t\treturn NewJSConsumerMaxPendingAckExcessError(cfg.ConsumerLimits.MaxAckPending)\n\t}\n\tif cfg.ConsumerLimits.InactiveThreshold > 0 && config.InactiveThreshold > cfg.ConsumerLimits.InactiveThreshold {\n\t\treturn NewJSConsumerInactiveThresholdExcessError(cfg.ConsumerLimits.InactiveThreshold)\n\t}\n\n\t// Direct need to be non-mapped ephemerals.\n\tif config.Direct {\n\t\tif config.DeliverSubject == _EMPTY_ {\n\t\t\treturn NewJSConsumerDirectRequiresPushError()\n\t\t}\n\t\tif isDurableConsumer(config) {\n\t\t\treturn NewJSConsumerDirectRequiresEphemeralError()\n\t\t}\n\t}\n\n\t// Do not allow specifying both FilterSubject and FilterSubjects,\n\t// as that's probably unintentional without any difference from passing\n\t// all filters in FilterSubjects.\n\tif config.FilterSubject != _EMPTY_ && len(config.FilterSubjects) > 0 {\n\t\treturn NewJSConsumerDuplicateFilterSubjectsError()\n\t}\n\n\tif config.FilterSubject != _EMPTY_ && !IsValidSubject(config.FilterSubject) {\n\t\treturn NewJSStreamInvalidConfigError(ErrBadSubject)\n\t}\n\n\t// We treat FilterSubjects: []string{\"\"} as a misconfig, so we validate against it.\n\tfor _, filter := range config.FilterSubjects {\n\t\tif filter == _EMPTY_ {\n\t\t\treturn NewJSConsumerEmptyFilterError()\n\t\t}\n\t}\n\tsubjectFilters := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects)\n\n\t// Check subject filters do not overlap.\n\tfor outer, subject := range subjectFilters {\n\t\tif !IsValidSubject(subject) {\n\t\t\treturn NewJSStreamInvalidConfigError(ErrBadSubject)\n\t\t}\n\t\tfor inner, ssubject := range subjectFilters {\n\t\t\tif inner != outer && subjectIsSubsetMatch(subject, ssubject) {\n\t\t\t\treturn NewJSConsumerOverlappingSubjectFiltersError()\n\t\t\t}\n\t\t}\n\t}\n\n\t// Helper function to formulate similar errors.\n\tbadStart := func(dp, start string) error {\n\t\treturn fmt.Errorf(\"consumer delivery policy is deliver %s, but optional start %s is also set\", dp, start)\n\t}\n\tnotSet := func(dp, notSet string) error {\n\t\treturn fmt.Errorf(\"consumer delivery policy is deliver %s, but optional %s is not set\", dp, notSet)\n\t}\n\n\t// Check on start position conflicts.\n\tswitch config.DeliverPolicy {\n\tcase DeliverAll:\n\t\tif config.OptStartSeq > 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"all\", \"sequence\"))\n\t\t}\n\t\tif config.OptStartTime != nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"all\", \"time\"))\n\t\t}\n\tcase DeliverLast:\n\t\tif config.OptStartSeq > 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"last\", \"sequence\"))\n\t\t}\n\t\tif config.OptStartTime != nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"last\", \"time\"))\n\t\t}\n\tcase DeliverLastPerSubject:\n\t\tif config.OptStartSeq > 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"last per subject\", \"sequence\"))\n\t\t}\n\t\tif config.OptStartTime != nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"last per subject\", \"time\"))\n\t\t}\n\t\tif config.FilterSubject == _EMPTY_ && len(config.FilterSubjects) == 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(notSet(\"last per subject\", \"filter subject\"))\n\t\t}\n\tcase DeliverNew:\n\t\tif config.OptStartSeq > 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"new\", \"sequence\"))\n\t\t}\n\t\tif config.OptStartTime != nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"new\", \"time\"))\n\t\t}\n\tcase DeliverByStartSequence:\n\t\tif config.OptStartSeq == 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(notSet(\"by start sequence\", \"start sequence\"))\n\t\t}\n\t\tif config.OptStartTime != nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"by start sequence\", \"time\"))\n\t\t}\n\tcase DeliverByStartTime:\n\t\tif config.OptStartTime == nil {\n\t\t\treturn NewJSConsumerInvalidPolicyError(notSet(\"by start time\", \"start time\"))\n\t\t}\n\t\tif config.OptStartSeq != 0 {\n\t\t\treturn NewJSConsumerInvalidPolicyError(badStart(\"by start time\", \"start sequence\"))\n\t\t}\n\t}\n\n\tif config.SampleFrequency != _EMPTY_ {\n\t\ts := strings.TrimSuffix(config.SampleFrequency, \"%\")\n\t\tif sampleFreq, err := strconv.Atoi(s); err != nil || sampleFreq < 0 {\n\t\t\treturn NewJSConsumerInvalidSamplingError(err)\n\t\t}\n\t}\n\n\t// We reject if flow control is set without heartbeats.\n\tif config.FlowControl && config.Heartbeat == 0 {\n\t\treturn NewJSConsumerWithFlowControlNeedsHeartbeatsError()\n\t}\n\n\tif config.Durable != _EMPTY_ && config.Name != _EMPTY_ {\n\t\tif config.Name != config.Durable {\n\t\t\treturn NewJSConsumerCreateDurableAndNameMismatchError()\n\t\t}\n\t}\n\n\tvar metadataLen int\n\tfor k, v := range config.Metadata {\n\t\tmetadataLen += len(k) + len(v)\n\t}\n\tif metadataLen > JSMaxMetadataLen {\n\t\treturn NewJSConsumerMetadataLengthError(fmt.Sprintf(\"%dKB\", JSMaxMetadataLen/1024))\n\t}\n\n\tif config.PriorityPolicy != PriorityNone {\n\t\tif config.DeliverSubject != \"\" {\n\t\t\treturn NewJSConsumerPushWithPriorityGroupError()\n\t\t}\n\t\tif len(config.PriorityGroups) == 0 {\n\t\t\treturn NewJSConsumerPriorityPolicyWithoutGroupError()\n\t\t}\n\n\t\tfor _, group := range config.PriorityGroups {\n\t\t\tif group == _EMPTY_ {\n\t\t\t\treturn NewJSConsumerEmptyGroupNameError()\n\t\t\t}\n\t\t\tif !validGroupName.MatchString(group) {\n\t\t\t\treturn NewJSConsumerInvalidGroupNameError()\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// If PriorityPolicy is None or not set, reject if PriorityGroups or PinnedTTL are set\n\t\tif len(config.PriorityGroups) > 0 {\n\t\t\treturn NewJSConsumerPriorityGroupWithPolicyNoneError()\n\t\t}\n\t\tif config.PinnedTTL > 0 {\n\t\t\treturn NewJSConsumerPinnedTTLWithoutPriorityPolicyNoneError()\n\t\t}\n\t}\n\n\t// For now don't allow preferred server in placement.\n\tif cfg.Placement != nil && cfg.Placement.Preferred != _EMPTY_ {\n\t\treturn NewJSStreamInvalidConfigError(fmt.Errorf(\"preferred server not permitted in placement\"))\n\t}\n\n\treturn nil\n}\n\nfunc (mset *stream) addConsumerWithAction(config *ConsumerConfig, action ConsumerAction, pedantic bool) (*consumer, error) {\n\treturn mset.addConsumerWithAssignment(config, _EMPTY_, nil, false, action, pedantic)\n}\n\nfunc (mset *stream) addConsumer(config *ConsumerConfig) (*consumer, error) {\n\treturn mset.addConsumerWithAction(config, ActionCreateOrUpdate, false)\n}\n\nfunc (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname string, ca *consumerAssignment, isRecovering bool, action ConsumerAction, pedantic bool) (*consumer, error) {\n\t// Check if this stream has closed.\n\tif mset.closed.Load() {\n\t\treturn nil, NewJSStreamInvalidError()\n\t}\n\n\tmset.mu.RLock()\n\ts, js, jsa, cfg, acc, lseq := mset.srv, mset.js, mset.jsa, mset.cfg, mset.acc, mset.lseq\n\tmset.mu.RUnlock()\n\n\t// If we do not have the consumer currently assigned to us in cluster mode we will proceed but warn.\n\t// This can happen on startup with restored state where on meta replay we still do not have\n\t// the assignment. Running in single server mode this always returns true.\n\tif oname != _EMPTY_ && !jsa.consumerAssigned(mset.name(), oname) {\n\t\ts.Debugf(\"Consumer %q > %q does not seem to be assigned to this server\", mset.name(), oname)\n\t}\n\n\tif config == nil {\n\t\treturn nil, NewJSConsumerConfigRequiredError()\n\t}\n\n\tselectedLimits, _, _, _ := acc.selectLimits(config.replicas(&cfg))\n\tif selectedLimits == nil {\n\t\treturn nil, NewJSNoLimitsError()\n\t}\n\n\tsrvLim := &s.getOpts().JetStreamLimits\n\t// Make sure we have sane defaults. Do so with the JS lock, otherwise a\n\t// badly timed meta snapshot can result in a race condition.\n\tmset.js.mu.Lock()\n\terr := setConsumerConfigDefaults(config, &cfg, srvLim, selectedLimits, pedantic)\n\tmset.js.mu.Unlock()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := checkConsumerCfg(config, srvLim, &cfg, acc, selectedLimits, isRecovering); err != nil {\n\t\treturn nil, err\n\t}\n\tsampleFreq := 0\n\tif config.SampleFrequency != _EMPTY_ {\n\t\t// Can't fail as checkConsumerCfg checks correct format\n\t\tsampleFreq, _ = strconv.Atoi(strings.TrimSuffix(config.SampleFrequency, \"%\"))\n\t}\n\n\t// Grab the client, account and server reference.\n\tc := mset.client\n\tif c == nil {\n\t\treturn nil, NewJSStreamInvalidError()\n\t}\n\tvar accName string\n\tc.mu.Lock()\n\ts, a := c.srv, c.acc\n\tif a != nil {\n\t\taccName = a.Name\n\t}\n\tc.mu.Unlock()\n\n\t// Hold mset lock here.\n\tmset.mu.Lock()\n\tif mset.client == nil || mset.store == nil || mset.consumers == nil {\n\t\tmset.mu.Unlock()\n\t\treturn nil, NewJSStreamInvalidError()\n\t}\n\n\t// If this one is durable and already exists, we let that be ok as long as only updating what should be allowed.\n\tvar cName string\n\tif isDurableConsumer(config) {\n\t\tcName = config.Durable\n\t} else if config.Name != _EMPTY_ {\n\t\tcName = config.Name\n\t}\n\tif cName != _EMPTY_ {\n\t\tif eo, ok := mset.consumers[cName]; ok {\n\t\t\tif action == ActionCreate {\n\t\t\t\tocfg := eo.config()\n\t\t\t\tcopyConsumerMetadata(config, &ocfg)\n\t\t\t\tif !reflect.DeepEqual(config, &ocfg) {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn nil, NewJSConsumerAlreadyExistsError()\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Check for overlapping subjects if we are a workqueue\n\t\t\tif cfg.Retention == WorkQueuePolicy {\n\t\t\t\tsubjects := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects)\n\t\t\t\tif !mset.partitionUnique(cName, subjects) {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn nil, NewJSConsumerWQConsumerNotUniqueError()\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t\terr := eo.updateConfig(config)\n\t\t\tif err == nil {\n\t\t\t\treturn eo, nil\n\t\t\t}\n\t\t\treturn nil, NewJSConsumerCreateError(err, Unless(err))\n\t\t}\n\t}\n\tif action == ActionUpdate {\n\t\tmset.mu.Unlock()\n\t\treturn nil, NewJSConsumerDoesNotExistError()\n\t}\n\n\t// If we're clustered we've already done this check, only do this if we're a standalone server.\n\t// But if we're standalone, only enforce if we're not recovering, since the MaxConsumers could've\n\t// been updated while we already had more consumers on disk.\n\tif !s.JetStreamIsClustered() && s.standAloneMode() && !isRecovering {\n\t\t// Check for any limits, if the config for the consumer sets a limit we check against that\n\t\t// but if not we use the value from account limits, if account limits is more restrictive\n\t\t// than stream config we prefer the account limits to handle cases where account limits are\n\t\t// updated during the lifecycle of the stream\n\t\tmaxc := cfg.MaxConsumers\n\t\tif maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) {\n\t\t\tmaxc = selectedLimits.MaxConsumers\n\t\t}\n\t\tif maxc > 0 && mset.numPublicConsumers() >= maxc {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn nil, NewJSMaximumConsumersLimitError()\n\t\t}\n\t}\n\n\t// Check on stream type conflicts with WorkQueues.\n\tif cfg.Retention == WorkQueuePolicy && !config.Direct {\n\t\t// Force explicit acks here.\n\t\tif config.AckPolicy != AckExplicit {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn nil, NewJSConsumerWQRequiresExplicitAckError()\n\t\t}\n\n\t\tif len(mset.consumers) > 0 {\n\t\t\tsubjects := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects)\n\t\t\tif len(subjects) == 0 {\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn nil, NewJSConsumerWQMultipleUnfilteredError()\n\t\t\t} else if !mset.partitionUnique(cName, subjects) {\n\t\t\t\t// Prior to v2.9.7, on a stream with WorkQueue policy, the servers\n\t\t\t\t// were not catching the error of having multiple consumers with\n\t\t\t\t// overlapping filter subjects depending on the scope, for instance\n\t\t\t\t// creating \"foo.*.bar\" and then \"foo.>\" was not detected, while\n\t\t\t\t// \"foo.>\" and then \"foo.*.bar\" would have been. Failing here\n\t\t\t\t// in recovery mode would leave the rejected consumer in a bad state,\n\t\t\t\t// so we will simply warn here, asking the user to remove this\n\t\t\t\t// consumer administratively. Otherwise, if this is the creation\n\t\t\t\t// of a new consumer, we will return the error.\n\t\t\t\tif isRecovering {\n\t\t\t\t\ts.Warnf(\"Consumer %q > %q has a filter subject that overlaps \"+\n\t\t\t\t\t\t\"with other consumers, which is not allowed for a stream \"+\n\t\t\t\t\t\t\"with WorkQueue policy, it should be administratively deleted\",\n\t\t\t\t\t\tcfg.Name, cName)\n\t\t\t\t} else {\n\t\t\t\t\t// We have a partition but it is not unique amongst the others.\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn nil, NewJSConsumerWQConsumerNotUniqueError()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif config.DeliverPolicy != DeliverAll {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn nil, NewJSConsumerWQConsumerNotDeliverAllError()\n\t\t}\n\t}\n\n\t// Set name, which will be durable name if set, otherwise we create one at random.\n\to := &consumer{\n\t\tmset: mset,\n\t\tjs: s.getJetStream(),\n\t\tacc: a,\n\t\tsrv: s,\n\t\tclient: s.createInternalJetStreamClient(),\n\t\tsysc: s.createInternalJetStreamClient(),\n\t\tcfg: *config,\n\t\tdsubj: config.DeliverSubject,\n\t\toutq: mset.outq,\n\t\tactive: true,\n\t\tqch: make(chan struct{}),\n\t\tmqch: make(chan struct{}),\n\t\tuch: make(chan struct{}, 1),\n\t\tmch: make(chan struct{}, 1),\n\t\tsfreq: int32(sampleFreq),\n\t\tmaxdc: uint64(max(config.MaxDeliver, 0)), // MaxDeliver is negative (-1) when infinite.\n\t\tmaxp: config.MaxAckPending,\n\t\tretention: cfg.Retention,\n\t\tcreated: time.Now().UTC(),\n\t}\n\n\t// Add created timestamp used for the store, must match that of the consumer assignment if it exists.\n\tif ca != nil {\n\t\tjs.mu.RLock()\n\t\to.created = ca.Created\n\t\tjs.mu.RUnlock()\n\t}\n\n\t// Bind internal client to the user account.\n\to.client.registerWithAccount(a)\n\t// Bind to the system account.\n\to.sysc.registerWithAccount(s.SystemAccount())\n\n\tif isDurableConsumer(config) {\n\t\tif len(config.Durable) > JSMaxNameLen {\n\t\t\tmset.mu.Unlock()\n\t\t\to.deleteWithoutAdvisory()\n\t\t\treturn nil, NewJSConsumerNameTooLongError(JSMaxNameLen)\n\t\t}\n\t\to.name = config.Durable\n\t} else if oname != _EMPTY_ {\n\t\to.name = oname\n\t} else {\n\t\tif config.Name != _EMPTY_ {\n\t\t\to.name = config.Name\n\t\t} else {\n\t\t\t// Legacy ephemeral auto-generated.\n\t\t\tfor {\n\t\t\t\to.name = createConsumerName()\n\t\t\t\tif _, ok := mset.consumers[o.name]; !ok {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tconfig.Name = o.name\n\t\t}\n\t}\n\t// Create ackMsgs queue now that we have a consumer name\n\to.ackMsgs = newIPQueue[*jsAckMsg](s, fmt.Sprintf(\"[ACC:%s] consumer '%s' on stream '%s' ackMsgs\", accName, o.name, cfg.Name))\n\n\t// Create our request waiting queue.\n\tif o.isPullMode() {\n\t\to.waiting = newWaitQueue(config.MaxWaiting)\n\t\t// Create our internal queue for next msg requests.\n\t\to.nextMsgReqs = newIPQueue[*nextMsgReq](s, fmt.Sprintf(\"[ACC:%s] consumer '%s' on stream '%s' pull requests\", accName, o.name, cfg.Name))\n\t}\n\n\t// already under lock, mset.Name() would deadlock\n\to.stream = cfg.Name\n\to.ackEventT = JSMetricConsumerAckPre + \".\" + o.stream + \".\" + o.name\n\to.nakEventT = JSAdvisoryConsumerMsgNakPre + \".\" + o.stream + \".\" + o.name\n\to.deliveryExcEventT = JSAdvisoryConsumerMaxDeliveryExceedPre + \".\" + o.stream + \".\" + o.name\n\n\tif !isValidName(o.name) {\n\t\tmset.mu.Unlock()\n\t\to.deleteWithoutAdvisory()\n\t\treturn nil, NewJSConsumerBadDurableNameError()\n\t}\n\n\t// Setup our storage if not a direct consumer.\n\tif !config.Direct {\n\t\tstore, err := mset.store.ConsumerStore(o.name, o.created, config)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\to.deleteWithoutAdvisory()\n\t\t\treturn nil, NewJSConsumerStoreFailedError(err)\n\t\t}\n\t\to.store = store\n\t}\n\n\tfor _, filter := range gatherSubjectFilters(o.cfg.FilterSubject, o.cfg.FilterSubjects) {\n\t\tsub := &subjectFilter{\n\t\t\tsubject: filter,\n\t\t\thasWildcard: subjectHasWildcard(filter),\n\t\t\ttokenizedSubject: tokenizeSubjectIntoSlice(nil, filter),\n\t\t}\n\t\to.subjf = append(o.subjf, sub)\n\t}\n\n\t// If we have multiple filter subjects, create a sublist which we will use\n\t// in calling store.LoadNextMsgMulti.\n\tif len(o.subjf) <= 1 {\n\t\to.filters = nil\n\t} else {\n\t\to.filters = gsl.NewSublist[struct{}]()\n\t\tfor _, filter := range o.subjf {\n\t\t\to.filters.Insert(filter.subject, struct{}{})\n\t\t}\n\t}\n\n\tif o.store != nil && o.store.HasState() {\n\t\t// Restore our saved state.\n\t\to.mu.Lock()\n\t\to.readStoredState()\n\n\t\treplicas := o.cfg.replicas(&mset.cfg)\n\n\t\t// Starting sequence represents the next sequence to be delivered, so decrement it\n\t\t// since that's the minimum amount the stream should have as its last sequence.\n\t\tsseq := o.sseq\n\t\tif sseq > 0 {\n\t\t\tsseq--\n\t\t}\n\n\t\to.mu.Unlock()\n\n\t\t// A stream observing data loss rolls back in its sequence. Check if we need to reconcile the consumer state\n\t\t// to ensure new messages aren't skipped.\n\t\t// Only performed for non-replicated consumers for now.\n\t\tif replicas == 1 && lseq < sseq && isRecovering {\n\t\t\ts.Warnf(\"JetStream consumer '%s > %s > %s' delivered sequence %d past last stream sequence of %d\",\n\t\t\t\to.acc.Name, o.stream, o.name, sseq, lseq)\n\n\t\t\to.mu.Lock()\n\t\t\to.reconcileStateWithStream(lseq)\n\n\t\t\t// Save the reconciled state\n\t\t\tstate := &ConsumerState{\n\t\t\t\tDelivered: SequencePair{\n\t\t\t\t\tStream: o.sseq - 1,\n\t\t\t\t\tConsumer: o.dseq - 1,\n\t\t\t\t},\n\t\t\t\tAckFloor: SequencePair{\n\t\t\t\t\tStream: o.asflr,\n\t\t\t\t\tConsumer: o.adflr,\n\t\t\t\t},\n\t\t\t\tPending: o.pending,\n\t\t\t\tRedelivered: o.rdc,\n\t\t\t}\n\t\t\terr := o.store.ForceUpdate(state)\n\t\t\to.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\ts.Errorf(\"JetStream consumer '%s > %s > %s' errored while updating state: %v\", o.acc.Name, o.stream, o.name, err)\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn nil, NewJSConsumerStoreFailedError(err)\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// Select starting sequence number\n\t\tif err := o.selectStartingSeqNo(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Now register with mset and create the ack subscription.\n\t// Check if we already have this one registered.\n\tif eo, ok := mset.consumers[o.name]; ok {\n\t\tmset.mu.Unlock()\n\t\tif !o.isDurable() || !o.isPushMode() {\n\t\t\to.name = _EMPTY_ // Prevent removal since same name.\n\t\t\to.deleteWithoutAdvisory()\n\t\t\treturn nil, NewJSConsumerNameExistError()\n\t\t}\n\t\t// If we are here we have already registered this durable. If it is still active that is an error.\n\t\tif eo.isActive() {\n\t\t\to.name = _EMPTY_ // Prevent removal since same name.\n\t\t\to.deleteWithoutAdvisory()\n\t\t\treturn nil, NewJSConsumerExistingActiveError()\n\t\t}\n\t\t// Since we are here this means we have a potentially new durable so we should update here.\n\t\t// Check that configs are the same.\n\t\tif !configsEqualSansDelivery(o.cfg, eo.cfg) {\n\t\t\to.name = _EMPTY_ // Prevent removal since same name.\n\t\t\to.deleteWithoutAdvisory()\n\t\t\treturn nil, NewJSConsumerReplacementWithDifferentNameError()\n\t\t}\n\t\t// Once we are here we have a replacement push-based durable.\n\t\teo.updateDeliverSubject(o.cfg.DeliverSubject)\n\t\treturn eo, nil\n\t}\n\n\t// Set up the ack subscription for this consumer. Will use wildcard for all acks.\n\t// We will remember the template to generate replies with sequence numbers and use\n\t// that to scanf them back in.\n\t// Escape '%' in consumer and stream names, as `pre` is used as a template later\n\t// in consumer.ackReply(), resulting in erroneous formatting of the ack subject.\n\tmn := strings.ReplaceAll(cfg.Name, \"%\", \"%%\")\n\tpre := fmt.Sprintf(jsAckT, mn, strings.ReplaceAll(o.name, \"%\", \"%%\"))\n\to.ackReplyT = fmt.Sprintf(\"%s.%%d.%%d.%%d.%%d.%%d\", pre)\n\to.ackSubj = fmt.Sprintf(\"%s.*.*.*.*.*\", pre)\n\to.nextMsgSubj = fmt.Sprintf(JSApiRequestNextT, mn, o.name)\n\to.resetSubj = fmt.Sprintf(JSApiConsumerResetT, mn, o.name)\n\n\t// Check/update the inactive threshold\n\to.updateInactiveThreshold(&o.cfg)\n\n\tif o.isPushMode() {\n\t\t// Check if we are running only 1 replica and that the delivery subject has interest.\n\t\t// Check in place here for interest. Will setup properly in setLeader.\n\t\tif config.replicas(&cfg) == 1 {\n\t\t\tinterest := o.acc.sl.HasInterest(o.cfg.DeliverSubject)\n\t\t\tif !o.hasDeliveryInterest(interest) {\n\t\t\t\t// Let the interest come to us eventually, but setup delete timer.\n\t\t\t\to.updateDeliveryInterest(false)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Set our ca.\n\tif ca != nil {\n\t\to.setConsumerAssignment(ca)\n\t}\n\n\t// Check if we have a rate limit set.\n\tif config.RateLimit != 0 {\n\t\to.setRateLimit(config.RateLimit)\n\t}\n\n\tmset.setConsumer(o)\n\tmset.mu.Unlock()\n\n\tif config.Direct || (!s.JetStreamIsClustered() && s.standAloneMode()) {\n\t\to.setLeader(true)\n\t}\n\n\t// This is always true in single server mode.\n\tif o.IsLeader() {\n\t\t// Send advisory.\n\t\tvar suppress bool\n\t\tif !s.standAloneMode() && ca == nil {\n\t\t\tsuppress = true\n\t\t} else if ca != nil {\n\t\t\tsuppress = ca.responded\n\t\t}\n\t\tif !suppress {\n\t\t\to.sendCreateAdvisory()\n\t\t}\n\t}\n\n\treturn o, nil\n}\n\n// Updates the consumer `dthresh` delete timer duration and set\n// cfg.InactiveThreshold to JsDeleteWaitTimeDefault for ephemerals\n// if not explicitly already specified by the user.\n// Lock should be held.\nfunc (o *consumer) updateInactiveThreshold(cfg *ConsumerConfig) {\n\t// Ephemerals will always have inactive thresholds.\n\tif !o.isDurable() && cfg.InactiveThreshold <= 0 {\n\t\t// Add in 1 sec of jitter above and beyond the default of 5s.\n\t\to.dthresh = JsDeleteWaitTimeDefault + 100*time.Millisecond + time.Duration(rand.Int63n(900))*time.Millisecond\n\t\t// Only stamp config with default sans jitter.\n\t\tcfg.InactiveThreshold = JsDeleteWaitTimeDefault\n\t} else if cfg.InactiveThreshold > 0 {\n\t\t// Add in up to 1 sec of jitter if pull mode.\n\t\tif o.isPullMode() {\n\t\t\to.dthresh = cfg.InactiveThreshold + 100*time.Millisecond + time.Duration(rand.Int63n(900))*time.Millisecond\n\t\t} else {\n\t\t\to.dthresh = cfg.InactiveThreshold\n\t\t}\n\t} else if cfg.InactiveThreshold <= 0 {\n\t\t// We accept InactiveThreshold be set to 0 (for durables)\n\t\to.dthresh = 0\n\t}\n}\n\n// Updates the paused state. If we are the leader and the pause deadline\n// hasn't passed yet then we will start a timer to kick the consumer once\n// that deadline is reached. Lock should be held.\nfunc (o *consumer) updatePauseState(cfg *ConsumerConfig) {\n\tif o.uptmr != nil {\n\t\tstopAndClearTimer(&o.uptmr)\n\t}\n\tif !o.isLeader() {\n\t\t// Only the leader will run the timer as only the leader will run\n\t\t// loopAndGatherMsgs.\n\t\treturn\n\t}\n\tif cfg.PauseUntil == nil || cfg.PauseUntil.IsZero() || cfg.PauseUntil.Before(time.Now()) {\n\t\t// Either the PauseUntil is unset (is effectively zero) or the\n\t\t// deadline has already passed, in which case there is nothing\n\t\t// to do.\n\t\treturn\n\t}\n\to.uptmr = time.AfterFunc(time.Until(*cfg.PauseUntil), func() {\n\t\to.mu.Lock()\n\t\tdefer o.mu.Unlock()\n\n\t\tstopAndClearTimer(&o.uptmr)\n\t\to.sendPauseAdvisoryLocked(&o.cfg)\n\t\to.signalNewMessages()\n\t})\n}\n\nfunc (o *consumer) consumerAssignment() *consumerAssignment {\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.ca\n}\n\nfunc (o *consumer) setConsumerAssignment(ca *consumerAssignment) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\to.ca = ca\n\tif ca == nil {\n\t\treturn\n\t}\n\t// Set our node.\n\to.node = ca.Group.node\n\n\t// Trigger update chan.\n\tselect {\n\tcase o.uch <- struct{}{}:\n\tdefault:\n\t}\n}\n\nfunc (o *consumer) monitorQuitC() <-chan struct{} {\n\tif o == nil {\n\t\treturn nil\n\t}\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\t// Recreate if a prior monitor routine was stopped.\n\tif o.mqch == nil {\n\t\to.mqch = make(chan struct{})\n\t}\n\treturn o.mqch\n}\n\n// signalMonitorQuit signals to exit the monitor loop. If there's no Raft node,\n// this will be the only way to stop the monitor goroutine.\nfunc (o *consumer) signalMonitorQuit() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\tif o.mqch != nil {\n\t\tclose(o.mqch)\n\t\to.mqch = nil\n\t}\n}\n\nfunc (o *consumer) updateC() <-chan struct{} {\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.uch\n}\n\n// checkQueueInterest will check on our interest's queue group status.\n// Lock should be held.\nfunc (o *consumer) checkQueueInterest() {\n\tif !o.active || o.cfg.DeliverSubject == _EMPTY_ {\n\t\treturn\n\t}\n\tsubj := o.dsubj\n\tif subj == _EMPTY_ {\n\t\tsubj = o.cfg.DeliverSubject\n\t}\n\n\tif rr := o.acc.sl.Match(subj); len(rr.qsubs) > 0 {\n\t\t// Just grab first\n\t\tif qsubs := rr.qsubs[0]; len(qsubs) > 0 {\n\t\t\tif sub := rr.qsubs[0][0]; len(sub.queue) > 0 {\n\t\t\t\to.qgroup = string(sub.queue)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// clears our node if we have one. When we scale down to 1.\nfunc (o *consumer) clearNode() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\tif o.node != nil {\n\t\to.node.Delete()\n\t\to.node = nil\n\t}\n}\n\n// IsLeader will return if we are the current leader.\nfunc (o *consumer) IsLeader() bool {\n\treturn o.isLeader()\n}\n\n// Lock should be held.\nfunc (o *consumer) isLeader() bool {\n\treturn o.leader.Load()\n}\n\nfunc (o *consumer) setLeader(isLeader bool) {\n\to.mu.RLock()\n\tmset, closed := o.mset, o.closed\n\tmovingToClustered := o.node != nil && o.pch == nil\n\tmovingToNonClustered := o.node == nil && o.pch != nil\n\twasLeader := o.leader.Swap(isLeader)\n\to.mu.RUnlock()\n\n\t// If we are here we have a change in leader status.\n\tif isLeader {\n\t\tif closed || mset == nil {\n\t\t\treturn\n\t\t}\n\n\t\tif wasLeader {\n\t\t\t// If we detect we are scaling up, make sure to create clustered routines and channels.\n\t\t\tif movingToClustered {\n\t\t\t\to.mu.Lock()\n\t\t\t\t// We are moving from R1 to clustered.\n\t\t\t\to.pch = make(chan struct{}, 1)\n\t\t\t\tgo o.loopAndForwardProposals(o.qch)\n\t\t\t\tif o.phead != nil {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase o.pch <- struct{}{}:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\t} else if movingToNonClustered {\n\t\t\t\t// We are moving from clustered to non-clustered now.\n\t\t\t\t// Set pch to nil so if we scale back up we will recreate the loopAndForward from above.\n\t\t\t\to.mu.Lock()\n\t\t\t\tpch := o.pch\n\t\t\t\to.pch = nil\n\t\t\t\tselect {\n\t\t\t\tcase pch <- struct{}{}:\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tmset.mu.RLock()\n\t\ts, jsa, stream := mset.srv, mset.jsa, mset.getCfgName()\n\t\tmset.mu.RUnlock()\n\n\t\to.mu.Lock()\n\t\to.rdq = nil\n\t\to.rdqi.Empty()\n\n\t\t// Restore our saved state.\n\t\t// During non-leader status we just update our underlying store when not clustered.\n\t\t// If clustered we need to propose our initial (possibly skipped ahead) o.sseq to the group.\n\t\tif o.node == nil || o.dseq > 1 || (o.store != nil && o.store.HasState()) {\n\t\t\to.readStoredState()\n\t\t} else if o.node != nil && o.sseq >= 1 {\n\t\t\to.updateSkipped(o.sseq)\n\t\t}\n\n\t\t// Setup initial num pending.\n\t\to.streamNumPending()\n\n\t\t// Cleanup lss when we take over in clustered mode.\n\t\tif o.hasSkipListPending() && o.sseq >= o.lss.resume {\n\t\t\to.lss = nil\n\t\t}\n\n\t\t// Do info sub.\n\t\tif o.infoSub == nil && jsa != nil {\n\t\t\tisubj := fmt.Sprintf(clusterConsumerInfoT, jsa.acc(), stream, o.name)\n\t\t\t// Note below the way we subscribe here is so that we can send requests to ourselves.\n\t\t\to.infoSub, _ = s.systemSubscribe(isubj, _EMPTY_, false, o.sysc, o.handleClusterConsumerInfoRequest)\n\t\t}\n\n\t\tvar err error\n\t\tif o.cfg.AckPolicy != AckNone {\n\t\t\tif o.ackSub, err = o.subscribeInternal(o.ackSubj, o.pushAck); err != nil {\n\t\t\t\to.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\t// Setup the internal sub for next message requests regardless.\n\t\t// Will error if wrong mode to provide feedback to users.\n\t\tif o.reqSub, err = o.subscribeInternal(o.nextMsgSubj, o.processNextMsgReq); err != nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t\tif o.resetSub, err = o.subscribeInternal(o.resetSubj, o.processResetReq); err != nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Check on flow control settings.\n\t\tif o.cfg.FlowControl {\n\t\t\to.setMaxPendingBytes(JsFlowControlMaxPending)\n\t\t\tfcsubj := fmt.Sprintf(jsFlowControl, stream, o.name)\n\t\t\tif o.fcSub, err = o.subscribeInternal(fcsubj, o.processFlowControl); err != nil {\n\t\t\t\to.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\t// If push mode, register for notifications on interest.\n\t\tif o.isPushMode() {\n\t\t\to.inch = make(chan bool, 8)\n\t\t\to.acc.sl.registerNotification(o.cfg.DeliverSubject, o.cfg.DeliverGroup, o.inch)\n\t\t\tif o.active = <-o.inch; o.active {\n\t\t\t\to.checkQueueInterest()\n\t\t\t}\n\n\t\t\t// Check gateways in case they are enabled.\n\t\t\tif s.gateway.enabled {\n\t\t\t\tif !o.active {\n\t\t\t\t\to.active = s.hasGatewayInterest(o.acc.Name, o.cfg.DeliverSubject)\n\t\t\t\t}\n\t\t\t\tstopAndClearTimer(&o.gwdtmr)\n\t\t\t\to.gwdtmr = time.AfterFunc(time.Second, func() { o.watchGWinterest() })\n\t\t\t}\n\t\t}\n\n\t\tif o.dthresh > 0 && (o.isPullMode() || !o.active) {\n\t\t\t// Pull consumer. We run the dtmr all the time for this one.\n\t\t\tstopAndClearTimer(&o.dtmr)\n\t\t\to.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive)\n\t\t}\n\n\t\t// Update the consumer pause tracking.\n\t\to.updatePauseState(&o.cfg)\n\n\t\t// If we are not in ReplayInstant mode mark us as in replay state until resolved.\n\t\tif o.cfg.ReplayPolicy != ReplayInstant {\n\t\t\to.replay = true\n\t\t}\n\n\t\t// Recreate quit channel.\n\t\to.qch = make(chan struct{})\n\t\tqch := o.qch\n\t\tnode := o.node\n\t\tif node != nil && o.pch == nil {\n\t\t\to.pch = make(chan struct{}, 1)\n\t\t}\n\t\tpullMode := o.isPullMode()\n\t\to.mu.Unlock()\n\n\t\t// Check if there are any pending we might need to clean up etc.\n\t\to.checkPending()\n\n\t\t// Snapshot initial info.\n\t\to.infoWithSnap(true)\n\n\t\t// These are the labels we will use to annotate our goroutines.\n\t\tlabels := pprofLabels{\n\t\t\t\"type\": \"consumer\",\n\t\t\t\"account\": mset.accName(),\n\t\t\t\"stream\": mset.name(),\n\t\t\t\"consumer\": o.name,\n\t\t}\n\n\t\t// Now start up Go routine to deliver msgs.\n\t\tgo func() {\n\t\t\tsetGoRoutineLabels(labels)\n\t\t\to.loopAndGatherMsgs(qch)\n\t\t}()\n\n\t\t// Now start up Go routine to process acks.\n\t\tgo func() {\n\t\t\tsetGoRoutineLabels(labels)\n\t\t\to.processInboundAcks(qch)\n\t\t}()\n\n\t\tif pullMode {\n\t\t\t// Now start up Go routine to process inbound next message requests.\n\t\t\tgo func() {\n\t\t\t\tsetGoRoutineLabels(labels)\n\t\t\t\to.processInboundNextMsgReqs(qch)\n\t\t\t}()\n\t\t}\n\n\t\t// If we are R>1 spin up our proposal loop.\n\t\tif node != nil {\n\t\t\t// Determine if we can send pending requests info to the group.\n\t\t\t// They must be on server versions >= 2.7.1\n\t\t\to.checkAndSetPendingRequestsOk()\n\t\t\to.checkPendingRequests()\n\t\t\tgo func() {\n\t\t\t\tsetGoRoutineLabels(labels)\n\t\t\t\to.loopAndForwardProposals(qch)\n\t\t\t}()\n\t\t}\n\n\t} else {\n\t\t// Shutdown the go routines and the subscriptions.\n\t\to.mu.Lock()\n\t\tif o.qch != nil {\n\t\t\tclose(o.qch)\n\t\t\to.qch = nil\n\t\t}\n\t\t// Stop any inactivity timers. Should only be running on leaders.\n\t\tstopAndClearTimer(&o.dtmr)\n\t\t// Stop any unpause timers. Should only be running on leaders.\n\t\tstopAndClearTimer(&o.uptmr)\n\t\t// Make sure to clear out any re-deliver queues\n\t\to.stopAndClearPtmr()\n\t\to.rdq = nil\n\t\to.rdqi.Empty()\n\t\to.pending = nil\n\t\to.resetPendingDeliveries()\n\t\t// ok if they are nil, we protect inside unsubscribe()\n\t\to.unsubscribe(o.ackSub)\n\t\to.unsubscribe(o.reqSub)\n\t\to.unsubscribe(o.resetSub)\n\t\to.unsubscribe(o.fcSub)\n\t\to.ackSub, o.reqSub, o.resetSub, o.fcSub = nil, nil, nil, nil\n\t\tif o.infoSub != nil {\n\t\t\to.srv.sysUnsubscribe(o.infoSub)\n\t\t\to.infoSub = nil\n\t\t}\n\t\t// Reset waiting if we are in pull mode.\n\t\tif o.isPullMode() {\n\t\t\to.waiting = newWaitQueue(o.cfg.MaxWaiting)\n\t\t\to.nextMsgReqs.drain()\n\t\t} else if o.srv.gateway.enabled {\n\t\t\tstopAndClearTimer(&o.gwdtmr)\n\t\t}\n\t\to.unassignPinId()\n\t\t// If we were the leader make sure to drain queued up acks.\n\t\tif wasLeader {\n\t\t\to.ackMsgs.drain()\n\t\t\t// Reset amount of acks that need to be processed.\n\t\t\tatomic.StoreInt64(&o.awl, 0)\n\t\t\t// Also remove any pending replies since we should not be the one to respond at this point.\n\t\t\to.replies = nil\n\t\t}\n\t\to.mu.Unlock()\n\t}\n}\n\n// This is coming on the wire so do not block here.\nfunc (o *consumer) handleClusterConsumerInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tgo o.infoWithSnapAndReply(false, reply)\n}\n\n// Lock should be held.\nfunc (o *consumer) subscribeInternal(subject string, cb msgHandler) (*subscription, error) {\n\tc := o.client\n\tif c == nil {\n\t\treturn nil, fmt.Errorf(\"invalid consumer\")\n\t}\n\tif !c.srv.EventsEnabled() {\n\t\treturn nil, ErrNoSysAccount\n\t}\n\tif cb == nil {\n\t\treturn nil, fmt.Errorf(\"undefined message handler\")\n\t}\n\n\to.sid++\n\n\t// Now create the subscription\n\treturn c.processSub([]byte(subject), nil, []byte(strconv.Itoa(o.sid)), cb, false)\n}\n\n// Unsubscribe from our subscription.\n// Lock should be held.\nfunc (o *consumer) unsubscribe(sub *subscription) {\n\tif sub == nil || o.client == nil {\n\t\treturn\n\t}\n\to.client.processUnsub(sub.sid)\n}\n\n// We need to make sure we protect access to the outq.\n// Do all advisory sends here.\nfunc (o *consumer) sendAdvisory(subject string, e any) {\n\tif o.acc == nil {\n\t\treturn\n\t}\n\n\t// If there is no one listening for this advisory then save ourselves the effort\n\t// and don't bother encoding the JSON or sending it.\n\tif sl := o.acc.sl; (sl != nil && !sl.HasInterest(subject)) && !o.srv.hasGatewayInterest(o.acc.Name, subject) {\n\t\treturn\n\t}\n\n\tj, err := json.Marshal(e)\n\tif err != nil {\n\t\treturn\n\t}\n\n\to.outq.sendMsg(subject, j)\n}\n\nfunc (o *consumer) sendDeleteAdvisoryLocked() {\n\te := JSConsumerActionAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerActionAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tAction: DeleteEvent,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\tsubj := JSAdvisoryConsumerDeletedPre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n}\n\nfunc (o *consumer) sendPinnedAdvisoryLocked(group string) {\n\te := JSConsumerGroupPinnedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerGroupPinnedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tAccount: o.acc.Name,\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t\tPinnedClientId: o.currentPinId,\n\t\tGroup: group,\n\t}\n\n\tsubj := JSAdvisoryConsumerPinnedPre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n\n}\nfunc (o *consumer) sendUnpinnedAdvisoryLocked(group string, reason string) {\n\te := JSConsumerGroupUnpinnedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerGroupUnpinnedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tAccount: o.acc.Name,\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t\tGroup: group,\n\t\tReason: reason,\n\t}\n\n\tsubj := JSAdvisoryConsumerUnpinnedPre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n\n}\n\nfunc (o *consumer) sendCreateAdvisory() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\te := JSConsumerActionAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerActionAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tAction: CreateEvent,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\tsubj := JSAdvisoryConsumerCreatedPre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n}\n\nfunc (o *consumer) sendPauseAdvisoryLocked(cfg *ConsumerConfig) {\n\te := JSConsumerPauseAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerPauseAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\tif cfg.PauseUntil != nil {\n\t\te.PauseUntil = *cfg.PauseUntil\n\t\te.Paused = time.Now().Before(e.PauseUntil)\n\t}\n\n\tsubj := JSAdvisoryConsumerPausePre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n}\n\n// Created returns created time.\nfunc (o *consumer) createdTime() time.Time {\n\to.mu.Lock()\n\tcreated := o.created\n\to.mu.Unlock()\n\treturn created\n}\n\n// Internal to allow creation time to be restored.\nfunc (o *consumer) setCreatedTime(created time.Time) {\n\to.mu.Lock()\n\to.created = created\n\to.mu.Unlock()\n}\n\n// This will check for extended interest in a subject. If we have local interest we just return\n// that, but in the absence of local interest and presence of gateways or service imports we need\n// to check those as well.\nfunc (o *consumer) hasDeliveryInterest(localInterest bool) bool {\n\to.mu.RLock()\n\tmset := o.mset\n\tif mset == nil {\n\t\to.mu.RUnlock()\n\t\treturn false\n\t}\n\tacc := o.acc\n\tdeliver := o.cfg.DeliverSubject\n\to.mu.RUnlock()\n\n\tif localInterest {\n\t\treturn true\n\t}\n\n\t// If we are here check gateways.\n\tif s := acc.srv; s != nil && s.hasGatewayInterest(acc.Name, deliver) {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (s *Server) hasGatewayInterest(account, subject string) bool {\n\tgw := s.gateway\n\tif !gw.enabled {\n\t\treturn false\n\t}\n\tgw.RLock()\n\tdefer gw.RUnlock()\n\tfor _, gwc := range gw.outo {\n\t\tpsi, qr := gwc.gatewayInterest(account, stringToBytes(subject))\n\t\tif psi || qr != nil {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// This processes an update to the local interest for a deliver subject.\nfunc (o *consumer) updateDeliveryInterest(localInterest bool) bool {\n\tinterest := o.hasDeliveryInterest(localInterest)\n\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tmset := o.mset\n\tif mset == nil || o.isPullMode() {\n\t\treturn false\n\t}\n\n\tif interest && !o.active {\n\t\to.signalNewMessages()\n\t}\n\t// Update active status, if not active clear any queue group we captured.\n\tif o.active = interest; !o.active {\n\t\to.qgroup = _EMPTY_\n\t} else {\n\t\to.checkQueueInterest()\n\t}\n\n\t// If the delete timer has already been set do not clear here and return.\n\t// Note that durable can now have an inactive threshold, so don't check\n\t// for durable status, instead check for dthresh > 0.\n\tif o.dtmr != nil && o.dthresh > 0 && !interest {\n\t\treturn true\n\t}\n\n\t// Stop and clear the delete timer always.\n\tstopAndClearTimer(&o.dtmr)\n\n\t// If we do not have interest anymore and have a delete threshold set, then set\n\t// a timer to delete us. We wait for a bit in case of server reconnect.\n\tif !interest && o.dthresh > 0 {\n\t\to.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive)\n\t\treturn true\n\t}\n\treturn false\n}\n\nconst (\n\tdefaultConsumerNotActiveStartInterval = 30 * time.Second\n\tdefaultConsumerNotActiveMaxInterval = 5 * time.Minute\n)\n\nvar (\n\tconsumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval\n\tconsumerNotActiveMaxInterval = defaultConsumerNotActiveMaxInterval\n)\n\n// deleteNotActive must only be called from time.AfterFunc or in its own\n// goroutine, as it can block on clean-up.\nfunc (o *consumer) deleteNotActive() {\n\t// Take a copy of these when the goroutine starts, mostly it avoids a\n\t// race condition with tests that modify these consts, such as\n\t// TestJetStreamClusterGhostEphemeralsAfterRestart.\n\tcnaMax := consumerNotActiveMaxInterval\n\tcnaStart := consumerNotActiveStartInterval\n\n\to.mu.Lock()\n\tif o.mset == nil {\n\t\to.mu.Unlock()\n\t\treturn\n\t}\n\t// Push mode just look at active.\n\tif o.isPushMode() {\n\t\t// If we are active simply return.\n\t\tif o.active {\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t} else {\n\t\t// Pull mode.\n\t\telapsed := time.Since(o.waiting.last)\n\t\tif elapsed < o.dthresh {\n\t\t\t// These need to keep firing so reset but use delta.\n\t\t\tif o.dtmr != nil {\n\t\t\t\to.dtmr.Reset(o.dthresh - elapsed)\n\t\t\t} else {\n\t\t\t\to.dtmr = time.AfterFunc(o.dthresh-elapsed, o.deleteNotActive)\n\t\t\t}\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t\t// Check if we still have valid requests waiting.\n\t\tif o.checkWaitingForInterest() {\n\t\t\tif o.dtmr != nil {\n\t\t\t\to.dtmr.Reset(o.dthresh)\n\t\t\t} else {\n\t\t\t\to.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive)\n\t\t\t}\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// We now know we have no waiting requests, and our last request was long ago.\n\t\t// However, based on AckWait the consumer could still be actively processing,\n\t\t// even if we haven't been informed if there were no acks in the meantime.\n\t\t// We must wait for the message that expires last and start counting down the\n\t\t// inactive threshold from there.\n\t\tnow := time.Now().UnixNano()\n\t\tl := len(o.cfg.BackOff)\n\t\tvar delay time.Duration\n\t\tvar ackWait time.Duration\n\t\tfor _, p := range o.pending {\n\t\t\tif l == 0 {\n\t\t\t\tackWait = o.ackWait(0)\n\t\t\t} else {\n\t\t\t\tbi := int(o.rdc[p.Sequence])\n\t\t\t\tif bi < 0 {\n\t\t\t\t\tbi = 0\n\t\t\t\t} else if bi >= l {\n\t\t\t\t\tbi = l - 1\n\t\t\t\t}\n\t\t\t\tackWait = o.ackWait(o.cfg.BackOff[bi])\n\t\t\t}\n\t\t\tif ts := p.Timestamp + ackWait.Nanoseconds() + o.dthresh.Nanoseconds(); ts > now {\n\t\t\t\tdelay = max(delay, time.Duration(ts-now))\n\t\t\t}\n\t\t}\n\t\t// We'll wait for the latest time we expect an ack, plus the inactive threshold.\n\t\t// Acknowledging a message will reset this back down to just the inactive threshold.\n\t\tif delay > 0 {\n\t\t\tif o.dtmr != nil {\n\t\t\t\to.dtmr.Reset(delay)\n\t\t\t} else {\n\t\t\t\to.dtmr = time.AfterFunc(delay, o.deleteNotActive)\n\t\t\t}\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t}\n\n\ts, js := o.mset.srv, o.srv.js.Load()\n\tacc, stream, name, isDirect := o.acc.Name, o.stream, o.name, o.cfg.Direct\n\tvar qch, cqch chan struct{}\n\tif o.srv != nil {\n\t\tqch = o.srv.quitCh\n\t}\n\toqch := o.qch\n\to.mu.Unlock()\n\tif js != nil {\n\t\tcqch = js.clusterQuitC()\n\t}\n\n\t// Useful for pprof.\n\tsetGoRoutineLabels(pprofLabels{\n\t\t\"account\": acc,\n\t\t\"stream\": stream,\n\t\t\"consumer\": name,\n\t})\n\n\t// We will delete locally regardless.\n\tdefer o.delete()\n\n\t// If we are clustered, check if we still have this consumer assigned.\n\t// If we do forward a proposal to delete ourselves to the metacontroller leader.\n\tif !isDirect && s.JetStreamIsClustered() {\n\t\tjs.mu.RLock()\n\t\tvar (\n\t\t\tcca consumerAssignment\n\t\t\tmeta RaftNode\n\t\t\tremoveEntry []byte\n\t\t)\n\t\tca, cc := js.consumerAssignment(acc, stream, name), js.cluster\n\t\tif ca != nil && cc != nil {\n\t\t\tmeta = cc.meta\n\t\t\tcca = *ca\n\t\t\tcca.Reply = _EMPTY_\n\t\t\tremoveEntry = encodeDeleteConsumerAssignment(&cca)\n\t\t\tmeta.ForwardProposal(removeEntry)\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t\tif ca != nil && cc != nil {\n\t\t\t// Check to make sure we went away.\n\t\t\t// Don't think this needs to be a monitored go routine.\n\t\t\tjitter := time.Duration(rand.Int63n(int64(cnaStart)))\n\t\t\tinterval := cnaStart + jitter\n\t\t\tticker := time.NewTicker(interval)\n\t\t\tdefer ticker.Stop()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-ticker.C:\n\t\t\t\tcase <-qch:\n\t\t\t\t\treturn\n\t\t\t\tcase <-cqch:\n\t\t\t\t\treturn\n\t\t\t\tcase <-oqch:\n\t\t\t\t\t// The consumer has stopped already, likely by an earlier delete proposal being applied.\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tif js.shuttingDown {\n\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tnca := js.consumerAssignment(acc, stream, name)\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t// Make sure this is the same consumer assignment, and not a new consumer with the same name.\n\t\t\t\tif nca != nil && reflect.DeepEqual(nca, ca) {\n\t\t\t\t\ts.Warnf(\"Consumer assignment for '%s > %s > %s' not cleaned up, retrying\", acc, stream, name)\n\t\t\t\t\tmeta.ForwardProposal(removeEntry)\n\t\t\t\t\tif interval < cnaMax {\n\t\t\t\t\t\tinterval *= 2\n\t\t\t\t\t\tticker.Reset(interval)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// We saw that consumer has been removed, all done.\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (o *consumer) watchGWinterest() {\n\tpa := o.isActive()\n\t// If there is no local interest...\n\tif o.hasNoLocalInterest() {\n\t\to.updateDeliveryInterest(false)\n\t\tif !pa && o.isActive() {\n\t\t\to.signalNewMessages()\n\t\t}\n\t}\n\n\t// We want this to always be running so we can also pick up on interest returning.\n\to.mu.Lock()\n\tif o.gwdtmr != nil {\n\t\to.gwdtmr.Reset(time.Second)\n\t} else {\n\t\tstopAndClearTimer(&o.gwdtmr)\n\t\to.gwdtmr = time.AfterFunc(time.Second, func() { o.watchGWinterest() })\n\t}\n\to.mu.Unlock()\n}\n\n// Config returns the consumer's configuration.\nfunc (o *consumer) config() ConsumerConfig {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.cfg\n}\n\n// Check if we have hit max deliveries. If so do notification and cleanup.\n// Return whether or not the max was hit.\n// Lock should be held.\nfunc (o *consumer) hasMaxDeliveries(seq uint64) bool {\n\tif o.maxdc == 0 {\n\t\treturn false\n\t}\n\tif dc := o.deliveryCount(seq); dc >= o.maxdc {\n\t\t// We have hit our max deliveries for this sequence.\n\t\t// Only send the advisory once.\n\t\tif dc == o.maxdc {\n\t\t\to.notifyDeliveryExceeded(seq, dc)\n\t\t}\n\t\t// Determine if we signal to start flow of messages again.\n\t\tif o.maxp > 0 && len(o.pending) >= o.maxp {\n\t\t\to.signalNewMessages()\n\t\t}\n\t\t// Make sure to remove from pending.\n\t\tif p, ok := o.pending[seq]; ok && p != nil {\n\t\t\tdelete(o.pending, seq)\n\t\t\to.updateDelivered(p.Sequence, seq, dc, p.Timestamp)\n\t\t}\n\t\t// Ensure redelivered state is set, if not already.\n\t\tif o.rdc == nil {\n\t\t\to.rdc = make(map[uint64]uint64)\n\t\t}\n\t\to.rdc[seq] = dc\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Force expiration of all pending.\n// Lock should be held.\nfunc (o *consumer) forceExpirePending() {\n\tvar expired []uint64\n\tfor seq := range o.pending {\n\t\tif !o.onRedeliverQueue(seq) && !o.hasMaxDeliveries(seq) {\n\t\t\texpired = append(expired, seq)\n\t\t}\n\t}\n\tif len(expired) > 0 {\n\t\tslices.Sort(expired)\n\t\to.addToRedeliverQueue(expired...)\n\t\t// Now we should update the timestamp here since we are redelivering.\n\t\t// We will use an incrementing time to preserve order for any other redelivery.\n\t\toff := time.Now().UnixNano() - o.pending[expired[0]].Timestamp\n\t\tfor _, seq := range expired {\n\t\t\tif p, ok := o.pending[seq]; ok && p != nil {\n\t\t\t\tp.Timestamp += off\n\t\t\t}\n\t\t}\n\t\to.resetPtmr(o.ackWait(0))\n\t}\n\to.signalNewMessages()\n}\n\n// Acquire proper locks and update rate limit.\n// Will use what is in config.\nfunc (o *consumer) setRateLimitNeedsLocks() {\n\to.mu.RLock()\n\tmset := o.mset\n\to.mu.RUnlock()\n\n\tif mset == nil {\n\t\treturn\n\t}\n\n\tmset.mu.RLock()\n\to.mu.Lock()\n\to.setRateLimit(o.cfg.RateLimit)\n\to.mu.Unlock()\n\tmset.mu.RUnlock()\n}\n\n// Set the rate limiter\n// Both mset and consumer lock should be held.\nfunc (o *consumer) setRateLimit(bps uint64) {\n\tif bps == 0 {\n\t\to.rlimit = nil\n\t\treturn\n\t}\n\n\t// TODO(dlc) - Make sane values or error if not sane?\n\t// We are configured in bits per sec so adjust to bytes.\n\trl := rate.Limit(bps / 8)\n\tmset := o.mset\n\n\t// Burst should be set to maximum msg size for this account, etc.\n\tvar burst int\n\t// We don't need to get cfgMu's rlock here since this function\n\t// is already invoked under mset.mu.RLock(), which superseeds cfgMu.\n\tif mset.cfg.MaxMsgSize > 0 {\n\t\tburst = int(mset.cfg.MaxMsgSize)\n\t} else if mset.jsa.account.limits.mpay > 0 {\n\t\tburst = int(mset.jsa.account.limits.mpay)\n\t} else {\n\t\ts := mset.jsa.account.srv\n\t\tburst = int(s.getOpts().MaxPayload)\n\t}\n\n\to.rlimit = rate.NewLimiter(rl, burst)\n}\n\n// Check if new consumer config allowed vs old.\nfunc (acc *Account) checkNewConsumerConfig(cfg, ncfg *ConsumerConfig) error {\n\tif reflect.DeepEqual(cfg, ncfg) {\n\t\treturn nil\n\t}\n\t// Something different, so check since we only allow certain things to be updated.\n\tif cfg.DeliverPolicy != ncfg.DeliverPolicy {\n\t\treturn errors.New(\"deliver policy can not be updated\")\n\t}\n\tif cfg.OptStartSeq != ncfg.OptStartSeq {\n\t\treturn errors.New(\"start sequence can not be updated\")\n\t}\n\tif cfg.OptStartTime != nil && ncfg.OptStartTime != nil {\n\t\t// Both have start times set, compare them directly:\n\t\tif !cfg.OptStartTime.Equal(*ncfg.OptStartTime) {\n\t\t\treturn errors.New(\"start time can not be updated\")\n\t\t}\n\t} else if cfg.OptStartTime != nil || ncfg.OptStartTime != nil {\n\t\t// At least one start time is set and the other is not\n\t\treturn errors.New(\"start time can not be updated\")\n\t}\n\tif cfg.AckPolicy != ncfg.AckPolicy {\n\t\treturn errors.New(\"ack policy can not be updated\")\n\t}\n\tif cfg.ReplayPolicy != ncfg.ReplayPolicy {\n\t\treturn errors.New(\"replay policy can not be updated\")\n\t}\n\tif cfg.Heartbeat != ncfg.Heartbeat {\n\t\treturn errors.New(\"heart beats can not be updated\")\n\t}\n\tif cfg.FlowControl != ncfg.FlowControl {\n\t\treturn errors.New(\"flow control can not be updated\")\n\t}\n\n\t// Deliver Subject is conditional on if its bound.\n\tif cfg.DeliverSubject != ncfg.DeliverSubject {\n\t\tif cfg.DeliverSubject == _EMPTY_ {\n\t\t\treturn errors.New(\"can not update pull consumer to push based\")\n\t\t}\n\t\tif ncfg.DeliverSubject == _EMPTY_ {\n\t\t\treturn errors.New(\"can not update push consumer to pull based\")\n\t\t}\n\t\tif acc.sl.HasInterest(cfg.DeliverSubject) {\n\t\t\treturn NewJSConsumerNameExistError()\n\t\t}\n\t}\n\n\tif cfg.MaxWaiting != ncfg.MaxWaiting {\n\t\treturn errors.New(\"max waiting can not be updated\")\n\t}\n\n\t// Check if BackOff is defined, MaxDeliver is within range.\n\tif lbo := len(ncfg.BackOff); lbo > 0 && ncfg.MaxDeliver != -1 && lbo > ncfg.MaxDeliver {\n\t\treturn NewJSConsumerMaxDeliverBackoffError()\n\t}\n\n\treturn nil\n}\n\n// Update the config based on the new config, or error if update not allowed.\nfunc (o *consumer) updateConfig(cfg *ConsumerConfig) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.closed || o.mset == nil {\n\t\treturn NewJSConsumerDoesNotExistError()\n\t}\n\n\tif err := o.acc.checkNewConsumerConfig(&o.cfg, cfg); err != nil {\n\t\treturn err\n\t}\n\n\t// Make sure we always store PauseUntil in UTC.\n\tif cfg.PauseUntil != nil {\n\t\tutc := (*cfg.PauseUntil).UTC()\n\t\tcfg.PauseUntil = &utc\n\t}\n\n\tif o.store != nil {\n\t\t// Update local state always.\n\t\tif err := o.store.UpdateConfig(cfg); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// DeliverSubject\n\tif cfg.DeliverSubject != o.cfg.DeliverSubject {\n\t\to.updateDeliverSubjectLocked(cfg.DeliverSubject)\n\t}\n\n\t// MaxAckPending\n\tif cfg.MaxAckPending != o.cfg.MaxAckPending {\n\t\to.maxp = cfg.MaxAckPending\n\t\to.signalNewMessages()\n\t\t// If MaxAckPending is lowered, we could have allocated a pending deliveries map of larger size.\n\t\t// Reset it here, so we can shrink the map.\n\t\tif cfg.MaxAckPending < o.cfg.MaxAckPending {\n\t\t\to.resetPendingDeliveries()\n\t\t}\n\t}\n\t// AckWait\n\tif cfg.AckWait != o.cfg.AckWait {\n\t\tif o.ptmr != nil {\n\t\t\to.resetPtmr(100 * time.Millisecond)\n\t\t}\n\t}\n\t// Rate Limit\n\tif cfg.RateLimit != o.cfg.RateLimit {\n\t\t// We need both locks here so do in Go routine.\n\t\tgo o.setRateLimitNeedsLocks()\n\t}\n\tif cfg.SampleFrequency != o.cfg.SampleFrequency {\n\t\ts := strings.TrimSuffix(cfg.SampleFrequency, \"%\")\n\t\tif sampleFreq, err := strconv.ParseInt(s, 10, 32); err == nil {\n\t\t\to.sfreq = int32(sampleFreq)\n\t\t}\n\t}\n\t// Set MaxDeliver if changed\n\tif cfg.MaxDeliver != o.cfg.MaxDeliver {\n\t\t// MaxDeliver is negative (-1) when infinite.\n\t\to.maxdc = uint64(max(cfg.MaxDeliver, 0))\n\t}\n\t// Set InactiveThreshold if changed.\n\tif val := cfg.InactiveThreshold; val != o.cfg.InactiveThreshold {\n\t\to.updateInactiveThreshold(cfg)\n\t\tstopAndClearTimer(&o.dtmr)\n\t\t// Restart timer only if we are the leader.\n\t\tif o.isLeader() && o.dthresh > 0 {\n\t\t\to.dtmr = time.AfterFunc(o.dthresh, o.deleteNotActive)\n\t\t}\n\t}\n\t// Check whether the pause has changed\n\t{\n\t\tvar old, new time.Time\n\t\tif o.cfg.PauseUntil != nil {\n\t\t\told = *o.cfg.PauseUntil\n\t\t}\n\t\tif cfg.PauseUntil != nil {\n\t\t\tnew = *cfg.PauseUntil\n\t\t}\n\t\tif !old.Equal(new) {\n\t\t\to.updatePauseState(cfg)\n\t\t\tif o.isLeader() {\n\t\t\t\to.sendPauseAdvisoryLocked(cfg)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check for Subject Filters update.\n\tnewSubjects := gatherSubjectFilters(cfg.FilterSubject, cfg.FilterSubjects)\n\tupdatedFilters := !subjectSliceEqual(newSubjects, o.subjf.subjects())\n\tif updatedFilters {\n\t\tnewSubjf := make(subjectFilters, 0, len(newSubjects))\n\t\tfor _, newFilter := range newSubjects {\n\t\t\tfs := &subjectFilter{\n\t\t\t\tsubject: newFilter,\n\t\t\t\thasWildcard: subjectHasWildcard(newFilter),\n\t\t\t\ttokenizedSubject: tokenizeSubjectIntoSlice(nil, newFilter),\n\t\t\t}\n\t\t\tnewSubjf = append(newSubjf, fs)\n\t\t}\n\t\t// Make sure we have correct signaling setup.\n\t\t// Consumer lock can not be held.\n\t\tmset := o.mset\n\t\to.mu.Unlock()\n\t\tmset.swapSigSubs(o, newSubjf.subjects())\n\t\to.mu.Lock()\n\n\t\t// When we're done with signaling, we can replace the subjects.\n\t\t// If filters were removed, set `o.subjf` to nil.\n\t\tif len(newSubjf) == 0 {\n\t\t\to.subjf = nil\n\t\t\to.filters = nil\n\t\t} else {\n\t\t\to.subjf = newSubjf\n\t\t\tif len(o.subjf) == 1 {\n\t\t\t\to.filters = nil\n\t\t\t} else {\n\t\t\t\to.filters = gsl.NewSublist[struct{}]()\n\t\t\t\tfor _, filter := range o.subjf {\n\t\t\t\t\to.filters.Insert(filter.subject, struct{}{})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Record new config for others that do not need special handling.\n\t// Allowed but considered no-op, [Description, SampleFrequency, MaxWaiting, HeadersOnly]\n\to.cfg = *cfg\n\n\tif updatedFilters {\n\t\t// Cleanup messages that lost interest.\n\t\tif o.retention == InterestPolicy {\n\t\t\to.mu.Unlock()\n\t\t\to.cleanupNoInterestMessages(o.mset, false)\n\t\t\to.mu.Lock()\n\t\t}\n\n\t\t// Re-calculate num pending on update.\n\t\to.streamNumPending()\n\t}\n\n\treturn nil\n}\n\n// This is a config change for the delivery subject for a\n// push based consumer.\nfunc (o *consumer) updateDeliverSubject(newDeliver string) {\n\t// Update the config and the dsubj\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\to.updateDeliverSubjectLocked(newDeliver)\n}\n\n// This is a config change for the delivery subject for a\n// push based consumer.\nfunc (o *consumer) updateDeliverSubjectLocked(newDeliver string) {\n\tif o.closed || o.isPullMode() || o.cfg.DeliverSubject == newDeliver {\n\t\treturn\n\t}\n\n\t// Force redeliver of all pending on change of delivery subject.\n\tif len(o.pending) > 0 {\n\t\to.forceExpirePending()\n\t}\n\n\to.acc.sl.clearNotification(o.dsubj, o.cfg.DeliverGroup, o.inch)\n\to.dsubj, o.cfg.DeliverSubject = newDeliver, newDeliver\n\t// When we register new one it will deliver to update state loop.\n\to.acc.sl.registerNotification(newDeliver, o.cfg.DeliverGroup, o.inch)\n}\n\n// Check that configs are equal but allow delivery subjects to be different.\nfunc configsEqualSansDelivery(a, b ConsumerConfig) bool {\n\t// These were copied in so can set Delivery here.\n\ta.DeliverSubject, b.DeliverSubject = _EMPTY_, _EMPTY_\n\treturn reflect.DeepEqual(a, b)\n}\n\n// Helper to send a reply to an ack.\nfunc (o *consumer) sendAckReply(subj string) {\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\to.outq.sendMsg(subj, nil)\n}\n\ntype jsAckMsg struct {\n\tsubject string\n\treply string\n\thdr int\n\tmsg []byte\n}\n\nvar jsAckMsgPool sync.Pool\n\nfunc newJSAckMsg(subj, reply string, hdr int, msg []byte) *jsAckMsg {\n\tvar m *jsAckMsg\n\tam := jsAckMsgPool.Get()\n\tif am != nil {\n\t\tm = am.(*jsAckMsg)\n\t} else {\n\t\tm = &jsAckMsg{}\n\t}\n\t// When getting something from a pool it is critical that all fields are\n\t// initialized. Doing this way guarantees that if someone adds a field to\n\t// the structure, the compiler will fail the build if this line is not updated.\n\t(*m) = jsAckMsg{subj, reply, hdr, msg}\n\treturn m\n}\n\nfunc (am *jsAckMsg) returnToPool() {\n\tif am == nil {\n\t\treturn\n\t}\n\tam.subject, am.reply, am.hdr, am.msg = _EMPTY_, _EMPTY_, -1, nil\n\tjsAckMsgPool.Put(am)\n}\n\n// Push the ack message to the consumer's ackMsgs queue\nfunc (o *consumer) pushAck(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tatomic.AddInt64(&o.awl, 1)\n\to.ackMsgs.push(newJSAckMsg(subject, reply, c.pa.hdr, copyBytes(rmsg)))\n}\n\n// Processes a message for the ack reply subject delivered with a message.\nfunc (o *consumer) processAck(subject, reply string, hdr int, rmsg []byte) {\n\tdefer atomic.AddInt64(&o.awl, -1)\n\n\tvar msg []byte\n\tif hdr > 0 {\n\t\tmsg = rmsg[hdr:]\n\t} else {\n\t\tmsg = rmsg\n\t}\n\n\tsseq, dseq, dc := ackReplyInfo(subject)\n\n\tskipAckReply := sseq == 0\n\n\tswitch {\n\tcase len(msg) == 0, bytes.Equal(msg, AckAck), bytes.Equal(msg, AckOK):\n\t\tif !o.processAckMsg(sseq, dseq, dc, reply, true) {\n\t\t\t// We handle replies for acks in updateAcks\n\t\t\tskipAckReply = true\n\t\t}\n\tcase bytes.HasPrefix(msg, AckNext):\n\t\to.processAckMsg(sseq, dseq, dc, _EMPTY_, true)\n\t\to.processNextMsgRequest(reply, msg[len(AckNext):])\n\t\tskipAckReply = true\n\tcase bytes.HasPrefix(msg, AckNak):\n\t\to.processNak(sseq, dseq, dc, msg)\n\tcase bytes.Equal(msg, AckProgress):\n\t\to.progressUpdate(sseq)\n\tcase bytes.HasPrefix(msg, AckTerm):\n\t\tvar reason string\n\t\tif buf := msg[len(AckTerm):]; len(buf) > 0 {\n\t\t\treason = string(bytes.TrimSpace(buf))\n\t\t}\n\t\tif !o.processTerm(sseq, dseq, dc, reason, reply) {\n\t\t\t// We handle replies for acks in updateAcks\n\t\t\tskipAckReply = true\n\t\t}\n\t}\n\n\t// Ack the ack if requested.\n\tif len(reply) > 0 && !skipAckReply {\n\t\to.sendAckReply(reply)\n\t}\n}\n\n// Used to process a working update to delay redelivery.\nfunc (o *consumer) progressUpdate(seq uint64) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif p, ok := o.pending[seq]; ok {\n\t\tp.Timestamp = time.Now().UnixNano()\n\t\t// Update store system.\n\t\to.updateDelivered(p.Sequence, seq, 1, p.Timestamp)\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) updateSkipped(seq uint64) {\n\t// Clustered mode and R>1 only.\n\tif o.node == nil || !o.isLeader() {\n\t\treturn\n\t}\n\tvar b [1 + 8]byte\n\tb[0] = byte(updateSkipOp)\n\tvar le = binary.LittleEndian\n\tle.PutUint64(b[1:], seq)\n\to.propose(b[:])\n}\n\nfunc (o *consumer) resetStartingSeq(seq uint64, reply string) (uint64, bool, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Reset to a specific sequence, or back to the ack floor.\n\tif seq == 0 {\n\t\tseq = o.asflr + 1\n\t} else if o.cfg.DeliverPolicy == DeliverAll {\n\t\t// Always allowed.\n\t\tgoto VALID\n\t} else if o.cfg.DeliverPolicy == DeliverByStartSequence {\n\t\t// Only allowed if not going below what's configured.\n\t\tif seq < o.cfg.OptStartSeq {\n\t\t\treturn 0, false, errors.New(\"below start seq\")\n\t\t}\n\t\tgoto VALID\n\t} else if o.cfg.DeliverPolicy == DeliverByStartTime && o.mset != nil {\n\t\t// Only allowed if not going below what's configured.\n\t\tnseq := o.mset.store.GetSeqFromTime(*o.cfg.OptStartTime)\n\t\tif seq < nseq {\n\t\t\treturn 0, false, errors.New(\"below start time\")\n\t\t}\n\t\tgoto VALID\n\t} else {\n\t\treturn 0, false, errors.New(\"not allowed\")\n\t}\n\nVALID:\n\t// Must be a minimum of 1.\n\tif seq <= 0 {\n\t\tseq = 1\n\t}\n\to.resetLocalStartingSeq(seq)\n\t// Clustered mode and R>1.\n\tif o.node != nil {\n\t\tb := make([]byte, 1+8+len(reply))\n\t\tb[0] = byte(resetSeqOp)\n\t\tvar le = binary.LittleEndian\n\t\tle.PutUint64(b[1:], seq)\n\t\tcopy(b[1+8:], reply)\n\t\to.propose(b[:])\n\t\treturn seq, false, nil\n\t} else if o.store != nil {\n\t\to.store.Reset(seq - 1)\n\t\t// Cleanup messages that lost interest.\n\t\tif o.retention == InterestPolicy {\n\t\t\tif mset := o.mset; mset != nil {\n\t\t\t\to.mu.Unlock()\n\t\t\t\tss := mset.state()\n\t\t\t\to.checkStateForInterestStream(&ss)\n\t\t\t\to.mu.Lock()\n\t\t\t}\n\t\t}\n\n\t\t// Recalculate pending, and re-trigger message delivery.\n\t\to.streamNumPending()\n\t\to.signalNewMessages()\n\t\treturn seq, true, nil\n\t}\n\treturn seq, false, nil\n}\n\n// Lock should be held.\nfunc (o *consumer) resetLocalStartingSeq(seq uint64) {\n\to.pending, o.rdc = nil, nil\n\to.rdq = nil\n\to.rdqi.Empty()\n\to.sseq, o.dseq = seq, 1\n\to.adflr, o.asflr = o.dseq-1, o.sseq-1\n\to.ldt, o.lat = time.Time{}, time.Time{}\n}\n\nfunc (o *consumer) loopAndForwardProposals(qch chan struct{}) {\n\t// On exit make sure we nil out pch.\n\tdefer func() {\n\t\to.mu.Lock()\n\t\to.pch = nil\n\t\to.mu.Unlock()\n\t}()\n\n\to.mu.RLock()\n\tnode, pch := o.node, o.pch\n\to.mu.RUnlock()\n\n\tif node == nil || pch == nil {\n\t\treturn\n\t}\n\n\tforwardProposals := func() error {\n\t\to.mu.Lock()\n\t\tif o.node == nil || !o.node.Leader() {\n\t\t\to.mu.Unlock()\n\t\t\treturn errors.New(\"no longer leader\")\n\t\t}\n\t\tproposal := o.phead\n\t\to.phead, o.ptail = nil, nil\n\t\to.mu.Unlock()\n\t\t// 256k max for now per batch.\n\t\tconst maxBatch = 256 * 1024\n\t\tvar entries []*Entry\n\t\tfor sz := 0; proposal != nil; proposal = proposal.next {\n\t\t\tentries = append(entries, newEntry(EntryNormal, proposal.data))\n\t\t\tsz += len(proposal.data)\n\t\t\tif sz > maxBatch {\n\t\t\t\tnode.ProposeMulti(entries)\n\t\t\t\t// We need to re-create `entries` because there is a reference\n\t\t\t\t// to it in the node's pae map.\n\t\t\t\tsz, entries = 0, nil\n\t\t\t}\n\t\t}\n\t\tif len(entries) > 0 {\n\t\t\tnode.ProposeMulti(entries)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// In case we have anything pending on entry.\n\tforwardProposals()\n\n\tfor {\n\t\tselect {\n\t\tcase <-qch:\n\t\t\tforwardProposals()\n\t\t\treturn\n\t\tcase <-pch:\n\t\t\tif err := forwardProposals(); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) propose(entry []byte) {\n\tp := &proposal{data: entry}\n\tif o.phead == nil {\n\t\to.phead = p\n\t} else {\n\t\to.ptail.next = p\n\t}\n\to.ptail = p\n\n\t// Kick our looper routine.\n\tselect {\n\tcase o.pch <- struct{}{}:\n\tdefault:\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) updateDelivered(dseq, sseq, dc uint64, ts int64) {\n\t// Clustered mode and R>1.\n\tif o.node != nil {\n\t\t// Inline for now, use variable compression.\n\t\tvar b [4*binary.MaxVarintLen64 + 1]byte\n\t\tb[0] = byte(updateDeliveredOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], dseq)\n\t\tn += binary.PutUvarint(b[n:], sseq)\n\t\tn += binary.PutUvarint(b[n:], dc)\n\t\tn += binary.PutVarint(b[n:], ts)\n\t\to.propose(b[:n])\n\t} else if o.store != nil {\n\t\to.store.UpdateDelivered(dseq, sseq, dc, ts)\n\t}\n\t// Update activity.\n\to.ldt = time.Now()\n}\n\n// Used to remember a pending ack reply in a replicated consumer.\n// Lock should be held.\nfunc (o *consumer) addAckReply(sseq uint64, reply string) {\n\tif o.replies == nil {\n\t\to.replies = make(map[uint64]string)\n\t}\n\to.replies[sseq] = reply\n}\n\n// Used to remember messages that need to be sent for a replicated consumer, after delivered quorum.\n// Lock should be held.\nfunc (o *consumer) addReplicatedQueuedMsg(pmsg *jsPubMsg) {\n\t// Is not explicitly limited in size, but will at most hold maximum ack pending.\n\tif o.pendingDeliveries == nil {\n\t\to.pendingDeliveries = make(map[uint64]*jsPubMsg)\n\t}\n\to.pendingDeliveries[pmsg.seq] = pmsg\n\n\t// Is not explicitly limited in size, but will at most hold maximum waiting requests.\n\tif o.waitingDeliveries == nil {\n\t\to.waitingDeliveries = make(map[string]*waitingDelivery)\n\t}\n\tif wd, ok := o.waitingDeliveries[pmsg.dsubj]; ok {\n\t\twd.seq = pmsg.seq\n\t} else {\n\t\twd := wdPool.Get().(*waitingDelivery)\n\t\twd.seq = pmsg.seq\n\t\to.waitingDeliveries[pmsg.dsubj] = wd\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) updateAcks(dseq, sseq uint64, reply string) {\n\tif o.node != nil {\n\t\t// Inline for now, use variable compression.\n\t\tvar b [2*binary.MaxVarintLen64 + 1]byte\n\t\tb[0] = byte(updateAcksOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], dseq)\n\t\tn += binary.PutUvarint(b[n:], sseq)\n\t\to.propose(b[:n])\n\t\tif reply != _EMPTY_ {\n\t\t\to.addAckReply(sseq, reply)\n\t\t}\n\t} else if o.store != nil {\n\t\to.store.UpdateAcks(dseq, sseq)\n\t\tif reply != _EMPTY_ {\n\t\t\t// Already locked so send direct.\n\t\t\to.outq.sendMsg(reply, nil)\n\t\t}\n\t}\n\t// Update activity.\n\to.lat = time.Now()\n}\n\n// Communicate to the cluster an addition of a pending request.\n// Lock should be held.\nfunc (o *consumer) addClusterPendingRequest(reply string) {\n\tif o.node == nil || !o.pendingRequestsOk() {\n\t\treturn\n\t}\n\tb := make([]byte, len(reply)+1)\n\tb[0] = byte(addPendingRequest)\n\tcopy(b[1:], reply)\n\to.propose(b)\n}\n\n// Communicate to the cluster a removal of a pending request.\n// Lock should be held.\nfunc (o *consumer) removeClusterPendingRequest(reply string) {\n\tif o.node == nil || !o.pendingRequestsOk() {\n\t\treturn\n\t}\n\tb := make([]byte, len(reply)+1)\n\tb[0] = byte(removePendingRequest)\n\tcopy(b[1:], reply)\n\to.propose(b)\n}\n\n// Set whether or not we can send pending requests to followers.\nfunc (o *consumer) setPendingRequestsOk(ok bool) {\n\to.mu.Lock()\n\to.prOk = ok\n\to.mu.Unlock()\n}\n\n// Lock should be held.\nfunc (o *consumer) pendingRequestsOk() bool {\n\treturn o.prOk\n}\n\n// Set whether or not we can send info about pending pull requests to our group.\n// Will require all peers have a minimum version.\nfunc (o *consumer) checkAndSetPendingRequestsOk() {\n\to.mu.RLock()\n\ts, isValid := o.srv, o.mset != nil\n\to.mu.RUnlock()\n\tif !isValid {\n\t\treturn\n\t}\n\n\tif ca := o.consumerAssignment(); ca != nil && len(ca.Group.Peers) > 1 {\n\t\tfor _, pn := range ca.Group.Peers {\n\t\t\tif si, ok := s.nodeToInfo.Load(pn); ok {\n\t\t\t\tif !versionAtLeast(si.(nodeInfo).version, 2, 7, 1) {\n\t\t\t\t\t// We expect all of our peers to eventually be up to date.\n\t\t\t\t\t// So check again in awhile.\n\t\t\t\t\ttime.AfterFunc(eventsHBInterval, func() { o.checkAndSetPendingRequestsOk() })\n\t\t\t\t\to.setPendingRequestsOk(false)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\to.setPendingRequestsOk(true)\n}\n\n// On leadership change make sure we alert the pending requests that they are no longer valid.\nfunc (o *consumer) checkPendingRequests() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\tif o.mset == nil || o.outq == nil {\n\t\treturn\n\t}\n\thdr := []byte(\"NATS/1.0 409 Leadership Change\\r\\n\\r\\n\")\n\tfor reply := range o.prm {\n\t\to.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t}\n\to.prm = nil\n}\n\n// This will release any pending pull requests if applicable.\n// Should be called only by the leader being deleted or stopped.\n// Lock should be held.\nfunc (o *consumer) releaseAnyPendingRequests(isAssigned bool) {\n\tif o.mset == nil || o.outq == nil || o.waiting.len() == 0 {\n\t\treturn\n\t}\n\twq := o.waiting\n\tfor wr := wq.head; wr != nil; {\n\t\tif !isAssigned {\n\t\t\thdr := []byte(\"NATS/1.0 409 Consumer Deleted\\r\\n\\r\\n\")\n\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t}\n\t\tnext := wr.next\n\t\twr.recycle()\n\t\twr = next\n\t}\n\t// Nil out old queue.\n\to.waiting = nil\n}\n\n// Process a NAK.\nfunc (o *consumer) processNak(sseq, dseq, dc uint64, nak []byte) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Check for out of range.\n\tif dseq <= o.adflr || dseq > o.dseq {\n\t\treturn\n\t}\n\t// If we are explicit ack make sure this is still on our pending list.\n\tif _, ok := o.pending[sseq]; !ok {\n\t\treturn\n\t}\n\n\t// Deliver an advisory\n\te := JSConsumerDeliveryNakAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerDeliveryNakAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tConsumerSeq: dseq,\n\t\tStreamSeq: sseq,\n\t\tDeliveries: dc,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\to.sendAdvisory(o.nakEventT, e)\n\n\t// Check to see if we have delays attached.\n\tif len(nak) > len(AckNak) {\n\t\targ := bytes.TrimSpace(nak[len(AckNak):])\n\t\tif len(arg) > 0 {\n\t\t\tvar d time.Duration\n\t\t\tvar err error\n\t\t\tif arg[0] == '{' {\n\t\t\t\tvar nd ConsumerNakOptions\n\t\t\t\tif err = json.Unmarshal(arg, &nd); err == nil {\n\t\t\t\t\td = nd.Delay\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\td, err = time.ParseDuration(string(arg))\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\t// Treat this as normal NAK.\n\t\t\t\to.srv.Warnf(\"JetStream consumer '%s > %s > %s' bad NAK delay value: %q\", o.acc.Name, o.stream, o.name, arg)\n\t\t\t} else {\n\t\t\t\t// We have a parsed duration that the user wants us to wait before retrying.\n\t\t\t\t// Make sure we are not on the rdq.\n\t\t\t\to.removeFromRedeliverQueue(sseq)\n\t\t\t\tif p, ok := o.pending[sseq]; ok {\n\t\t\t\t\t// now - ackWait is expired now, so offset from there.\n\t\t\t\t\tp.Timestamp = time.Now().Add(-o.cfg.AckWait).Add(d).UnixNano()\n\t\t\t\t\t// Update store system which will update followers as well.\n\t\t\t\t\to.updateDelivered(p.Sequence, sseq, dc, p.Timestamp)\n\t\t\t\t\tif o.ptmr != nil {\n\t\t\t\t\t\t// Want checkPending to run and figure out the next timer ttl.\n\t\t\t\t\t\t// TODO(dlc) - We could optimize this maybe a bit more and track when we expect the timer to fire.\n\t\t\t\t\t\to.resetPtmr(10 * time.Millisecond)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Nothing else for use to do now so return.\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// If already queued up also ignore.\n\tif !o.onRedeliverQueue(sseq) {\n\t\to.addToRedeliverQueue(sseq)\n\t}\n\n\to.signalNewMessages()\n}\n\n// Process a TERM\n// Returns `true` if the ack was processed in place and the sender can now respond\n// to the client, or `false` if there was an error or the ack is replicated (in which\n// case the reply will be sent later).\nfunc (o *consumer) processTerm(sseq, dseq, dc uint64, reason, reply string) bool {\n\t// Treat like an ack to suppress redelivery.\n\tackedInPlace := o.processAckMsg(sseq, dseq, dc, reply, false)\n\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Deliver an advisory\n\te := JSConsumerDeliveryTerminatedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerDeliveryTerminatedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tConsumerSeq: dseq,\n\t\tStreamSeq: sseq,\n\t\tDeliveries: dc,\n\t\tReason: reason,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\tsubj := JSAdvisoryConsumerMsgTerminatedPre + \".\" + o.stream + \".\" + o.name\n\to.sendAdvisory(subj, e)\n\treturn ackedInPlace\n}\n\n// Introduce a small delay in when timer fires to check pending.\n// Allows bursts to be treated in same time frame.\nconst ackWaitDelay = time.Millisecond\n\n// ackWait returns how long to wait to fire the pending timer.\nfunc (o *consumer) ackWait(next time.Duration) time.Duration {\n\tif next > 0 {\n\t\treturn next + ackWaitDelay\n\t}\n\treturn o.cfg.AckWait + ackWaitDelay\n}\n\n// Due to bug in calculation of sequences on restoring redelivered let's do quick sanity check.\n// Lock should be held.\nfunc (o *consumer) checkRedelivered() {\n\tvar shouldUpdateState bool\n\tfor sseq := range o.rdc {\n\t\tif sseq <= o.asflr {\n\t\t\tdelete(o.rdc, sseq)\n\t\t\to.removeFromRedeliverQueue(sseq)\n\t\t\tshouldUpdateState = true\n\t\t}\n\t}\n\tif shouldUpdateState {\n\t\tif err := o.writeStoreStateUnlocked(); err != nil && o.srv != nil && o.mset != nil && !o.closed {\n\t\t\ts, acc, mset, name := o.srv, o.acc, o.mset, o.name\n\t\t\ts.Warnf(\"Consumer '%s > %s > %s' error on write store state from check redelivered: %v\", acc, mset.getCfgName(), name, err)\n\t\t}\n\t}\n}\n\n// This will restore the state from disk.\n// Lock should be held.\nfunc (o *consumer) readStoredState() error {\n\tif o.store == nil {\n\t\treturn nil\n\t}\n\tstate, err := o.store.State()\n\tif err == nil {\n\t\to.applyState(state)\n\t\tif len(o.rdc) > 0 {\n\t\t\to.checkRedelivered()\n\t\t}\n\t}\n\treturn err\n}\n\n// Apply the consumer stored state.\n// Lock should be held.\nfunc (o *consumer) applyState(state *ConsumerState) {\n\tif state == nil {\n\t\treturn\n\t}\n\n\to.sseq = state.Delivered.Stream + 1\n\to.dseq = state.Delivered.Consumer + 1\n\to.adflr = state.AckFloor.Consumer\n\to.asflr = state.AckFloor.Stream\n\to.pending = state.Pending\n\to.rdc = state.Redelivered\n\n\t// Setup tracking timer if we have restored pending.\n\tif o.isLeader() && len(o.pending) > 0 {\n\t\t// This is on startup or leader change. We want to check pending\n\t\t// sooner in case there are inconsistencies etc. Pick between 500ms - 1.5s\n\t\tdelay := 500*time.Millisecond + time.Duration(rand.Int63n(1000))*time.Millisecond\n\n\t\t// If normal is lower than this just use that.\n\t\tif o.cfg.AckWait < delay {\n\t\t\tdelay = o.ackWait(0)\n\t\t}\n\t\to.resetPtmr(delay)\n\t}\n}\n\n// Sets our store state from another source. Used in clustered mode on snapshot restore.\n// Lock should be held.\nfunc (o *consumer) setStoreState(state *ConsumerState) error {\n\tif state == nil || o.store == nil {\n\t\treturn nil\n\t}\n\terr := o.store.Update(state)\n\tif err == nil {\n\t\to.applyState(state)\n\t} else if err == ErrStoreOldUpdate {\n\t\t// Our store already has a newer state, which is normal during recovery\n\t\t// when the consumer was loaded from disk before the meta snapshot state\n\t\t// was applied.\n\t\treturn nil\n\t}\n\treturn err\n}\n\n// Update our state to the store.\nfunc (o *consumer) writeStoreState() error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.writeStoreStateUnlocked()\n}\n\n// Update our state to the store.\n// Lock should be held.\nfunc (o *consumer) writeStoreStateUnlocked() error {\n\tif o.store == nil {\n\t\treturn nil\n\t}\n\tstate := ConsumerState{\n\t\tDelivered: SequencePair{\n\t\t\tConsumer: o.dseq - 1,\n\t\t\tStream: o.sseq - 1,\n\t\t},\n\t\tAckFloor: SequencePair{\n\t\t\tConsumer: o.adflr,\n\t\t\tStream: o.asflr,\n\t\t},\n\t\tPending: o.pending,\n\t\tRedelivered: o.rdc,\n\t}\n\treturn o.store.Update(&state)\n}\n\n// Returns an initial info. Only applicable for non-clustered consumers.\n// We will clear after we return it, so one shot.\nfunc (o *consumer) initialInfo() *ConsumerInfo {\n\to.mu.Lock()\n\tici := o.ici\n\to.ici = nil // gc friendly\n\to.mu.Unlock()\n\tif ici == nil {\n\t\tici = o.info()\n\t}\n\treturn ici\n}\n\n// Clears our initial info.\n// Used when we have a leader change in cluster mode but do not send a response.\nfunc (o *consumer) clearInitialInfo() {\n\to.mu.Lock()\n\to.ici = nil // gc friendly\n\to.mu.Unlock()\n}\n\n// Info returns our current consumer state.\nfunc (o *consumer) info() *ConsumerInfo {\n\treturn o.infoWithSnap(false)\n}\n\nfunc (o *consumer) infoWithSnap(snap bool) *ConsumerInfo {\n\treturn o.infoWithSnapAndReply(snap, _EMPTY_)\n}\n\nfunc (o *consumer) infoWithSnapAndReply(snap bool, reply string) *ConsumerInfo {\n\to.mu.Lock()\n\tmset := o.mset\n\tif o.closed || mset == nil || mset.srv == nil {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\tjs := o.js\n\tif js == nil {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t// Capture raftGroup.\n\tvar rg *raftGroup\n\tif o.ca != nil {\n\t\trg = o.ca.Group\n\t}\n\n\tpriorityGroups := []PriorityGroupState{}\n\t// TODO(jrm): when we introduce supporting many priority groups, we need to update assigning `o.currentNuid` for each group.\n\tif len(o.cfg.PriorityGroups) > 0 {\n\t\tpriorityGroups = append(priorityGroups, PriorityGroupState{\n\t\t\tGroup: o.cfg.PriorityGroups[0],\n\t\t\tPinnedClientID: o.currentPinId,\n\t\t\tPinnedTS: o.pinnedTS,\n\t\t})\n\t}\n\n\tnp, err := o.checkNumPending()\n\tif err != nil {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\n\tcfg := o.cfg\n\tinfo := &ConsumerInfo{\n\t\tStream: o.stream,\n\t\tName: o.name,\n\t\tCreated: o.created,\n\t\tConfig: &cfg,\n\t\tDelivered: SequenceInfo{\n\t\t\tConsumer: o.dseq - 1,\n\t\t\tStream: o.sseq - 1,\n\t\t},\n\t\tAckFloor: SequenceInfo{\n\t\t\tConsumer: o.adflr,\n\t\t\tStream: o.asflr,\n\t\t},\n\t\tNumAckPending: len(o.pending),\n\t\tNumRedelivered: len(o.rdc),\n\t\tNumPending: np,\n\t\tPushBound: o.isPushMode() && o.active,\n\t\tTimeStamp: time.Now().UTC(),\n\t\tPriorityGroups: priorityGroups,\n\t}\n\t// Reset redelivered for MaxDeliver 1. Redeliveries are disabled so must not report it (is confusing otherwise).\n\t// The state does still keep track of these messages.\n\tif o.cfg.MaxDeliver == 1 {\n\t\tinfo.NumRedelivered = 0\n\t}\n\tif o.cfg.PauseUntil != nil {\n\t\tp := *o.cfg.PauseUntil\n\t\tif info.Paused = time.Now().Before(p); info.Paused {\n\t\t\tinfo.PauseRemaining = time.Until(p)\n\t\t}\n\t}\n\n\t// We always need to pull certain data from our store.\n\tif o.store != nil {\n\t\tstate, err := o.store.BorrowState()\n\t\tif err != nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn nil\n\t\t}\n\n\t\t// If we are the leader we could have o.sseq that is skipped ahead.\n\t\t// To maintain consistency in reporting (e.g. jsz) we always take the state for our delivered/ackfloor stream sequence.\n\t\t// Only use skipped ahead o.sseq if we're a new consumer and have not yet replicated this state yet.\n\t\tleader := o.isLeader()\n\t\tif !leader || o.store.HasState() {\n\t\t\tinfo.Delivered.Consumer, info.Delivered.Stream = state.Delivered.Consumer, state.Delivered.Stream\n\t\t}\n\t\tinfo.AckFloor.Consumer, info.AckFloor.Stream = state.AckFloor.Consumer, state.AckFloor.Stream\n\t\tif !leader {\n\t\t\tinfo.NumAckPending = len(state.Pending)\n\t\t\tinfo.NumRedelivered = len(state.Redelivered)\n\t\t}\n\t}\n\n\t// Adjust active based on non-zero etc. Also make UTC here.\n\tif !o.ldt.IsZero() {\n\t\tldt := o.ldt.UTC() // This copies as well.\n\t\tinfo.Delivered.Last = &ldt\n\t}\n\tif !o.lat.IsZero() {\n\t\tlat := o.lat.UTC() // This copies as well.\n\t\tinfo.AckFloor.Last = &lat\n\t}\n\n\t// If we are a pull mode consumer, report on number of waiting requests.\n\tif o.isPullMode() {\n\t\to.processWaiting(false)\n\t\tinfo.NumWaiting = o.waiting.len()\n\t}\n\t// If we were asked to snapshot do so here.\n\tif snap {\n\t\to.ici = info\n\t}\n\tsysc := o.sysc\n\to.mu.Unlock()\n\n\t// Do cluster.\n\tif rg != nil {\n\t\tinfo.Cluster = js.clusterInfo(rg)\n\t}\n\n\t// If we have a reply subject send the response here.\n\tif reply != _EMPTY_ && sysc != nil {\n\t\tsysc.sendInternalMsg(reply, _EMPTY_, nil, info)\n\t}\n\n\treturn info\n}\n\n// Will signal us that new messages are available. Will break out of waiting.\nfunc (o *consumer) signalNewMessages() {\n\t// Kick our new message channel\n\tselect {\n\tcase o.mch <- struct{}{}:\n\tdefault:\n\t}\n}\n\n// shouldSample lets us know if we are sampling metrics on acks.\nfunc (o *consumer) shouldSample() bool {\n\tswitch {\n\tcase o.sfreq <= 0:\n\t\treturn false\n\tcase o.sfreq >= 100:\n\t\treturn true\n\t}\n\n\t// TODO(ripienaar) this is a tad slow so we need to rethink here, however this will only\n\t// hit for those with sampling enabled and its not the default\n\treturn rand.Int31n(100) <= o.sfreq\n}\n\nfunc (o *consumer) sampleAck(sseq, dseq, dc uint64) {\n\tif !o.shouldSample() {\n\t\treturn\n\t}\n\n\tnow := time.Now().UTC()\n\tunow := now.UnixNano()\n\n\te := JSConsumerAckMetric{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerAckMetricType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: now,\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tConsumerSeq: dseq,\n\t\tStreamSeq: sseq,\n\t\tDelay: unow - o.pending[sseq].Timestamp,\n\t\tDeliveries: dc,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\to.sendAdvisory(o.ackEventT, e)\n}\n\n// Process an ACK.\n// Returns `true` if the ack was processed in place and the sender can now respond\n// to the client, or `false` if there was an error or the ack is replicated (in which\n// case the reply will be sent later).\nfunc (o *consumer) processAckMsg(sseq, dseq, dc uint64, reply string, doSample bool) bool {\n\to.mu.Lock()\n\tif o.closed {\n\t\to.mu.Unlock()\n\t\treturn false\n\t}\n\n\tmset := o.mset\n\tif mset == nil || mset.closed.Load() {\n\t\to.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Check if this ack is above the current pointer to our next to deliver.\n\tif sseq >= o.sseq {\n\t\t// Let's make sure this is valid.\n\t\t// This is only received on the consumer leader, so should never be higher\n\t\t// than the last stream sequence. But could happen if we've just become\n\t\t// consumer leader, and we are not up-to-date on the stream yet.\n\t\tvar ss StreamState\n\t\tmset.store.FastState(&ss)\n\t\tif sseq > ss.LastSeq {\n\t\t\to.srv.Warnf(\"JetStream consumer '%s > %s > %s' ACK sequence %d past last stream sequence of %d\",\n\t\t\t\to.acc.Name, o.stream, o.name, sseq, ss.LastSeq)\n\t\t\t// FIXME(dlc) - For 2.11 onwards should we return an error here to the caller?\n\t\t}\n\t\t// Even though another leader must have delivered a message with this sequence, we must not adjust\n\t\t// the current pointer. This could otherwise result in a stuck consumer, where messages below this\n\t\t// sequence can't be redelivered, and we'll have incorrect pending state and ack floors.\n\t\to.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Let the owning stream know if we are interest or workqueue retention based.\n\t// If this consumer is clustered (o.node != nil) this will be handled by\n\t// processReplicatedAck after the ack has propagated.\n\tackInPlace := o.node == nil && o.retention != LimitsPolicy\n\n\tvar sgap, floor uint64\n\tvar needSignal bool\n\n\tswitch o.cfg.AckPolicy {\n\tcase AckExplicit:\n\t\tif p, ok := o.pending[sseq]; ok {\n\t\t\tif doSample {\n\t\t\t\to.sampleAck(sseq, dseq, dc)\n\t\t\t}\n\t\t\tif o.maxp > 0 && len(o.pending) >= o.maxp {\n\t\t\t\tneedSignal = true\n\t\t\t}\n\t\t\tdelete(o.pending, sseq)\n\t\t\t// Use the original deliver sequence from our pending record.\n\t\t\tdseq = p.Sequence\n\n\t\t\t// Only move floors if we matched an existing pending.\n\t\t\tif len(o.pending) == 0 {\n\t\t\t\to.adflr = o.dseq - 1\n\t\t\t\to.asflr = o.sseq - 1\n\t\t\t} else if dseq == o.adflr+1 {\n\t\t\t\to.adflr, o.asflr = dseq, sseq\n\t\t\t\tfor ss := sseq + 1; ss < o.sseq; ss++ {\n\t\t\t\t\tif p, ok := o.pending[ss]; ok {\n\t\t\t\t\t\tif p.Sequence > 0 {\n\t\t\t\t\t\t\to.adflr, o.asflr = p.Sequence-1, ss-1\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tdelete(o.rdc, sseq)\n\t\to.removeFromRedeliverQueue(sseq)\n\tcase AckAll:\n\t\t// no-op\n\t\tif dseq <= o.adflr || sseq <= o.asflr {\n\t\t\to.mu.Unlock()\n\t\t\t// Return true to let caller respond back to the client.\n\t\t\treturn true\n\t\t}\n\t\tif o.maxp > 0 && len(o.pending) >= o.maxp {\n\t\t\tneedSignal = true\n\t\t}\n\t\tsgap = sseq - o.asflr\n\t\tfloor = sseq // start at same and set lower as we go.\n\t\to.adflr, o.asflr = dseq, sseq\n\n\t\tremove := func(seq uint64) {\n\t\t\tdelete(o.pending, seq)\n\t\t\tdelete(o.rdc, seq)\n\t\t\to.removeFromRedeliverQueue(seq)\n\t\t\tif seq < floor {\n\t\t\t\tfloor = seq\n\t\t\t}\n\t\t}\n\t\t// Determine if smarter to walk all of pending vs the sequence range.\n\t\tif sgap > uint64(len(o.pending)) {\n\t\t\tfor seq := range o.pending {\n\t\t\t\tif seq <= sseq {\n\t\t\t\t\tremove(seq)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := sseq; seq > sseq-sgap && len(o.pending) > 0; seq-- {\n\t\t\t\tremove(seq)\n\t\t\t}\n\t\t}\n\tcase AckNone:\n\t\t// FIXME(dlc) - This is error but do we care?\n\t\to.mu.Unlock()\n\t\treturn ackInPlace\n\t}\n\n\t// No ack replication, so we set reply to \"\" so that updateAcks does not\n\t// send the reply. The caller will.\n\tif ackInPlace {\n\t\treply = _EMPTY_\n\t}\n\t// Update underlying store.\n\to.updateAcks(dseq, sseq, reply)\n\to.mu.Unlock()\n\n\tif ackInPlace {\n\t\tif sgap > 1 {\n\t\t\t// FIXME(dlc) - This can very inefficient, will need to fix.\n\t\t\tfor seq := sseq; seq >= floor; seq-- {\n\t\t\t\tmset.ackMsg(o, seq)\n\t\t\t}\n\t\t} else {\n\t\t\tmset.ackMsg(o, sseq)\n\t\t}\n\t}\n\n\t// If we had max ack pending set and were at limit we need to unblock ourselves.\n\tif needSignal {\n\t\to.signalNewMessages()\n\t}\n\treturn ackInPlace\n}\n\n// Determine if this is a truly filtered consumer. Modern clients will place filtered subjects\n// even if the stream only has a single non-wildcard subject designation.\n// Read lock should be held.\nfunc (o *consumer) isFiltered() bool {\n\tif o.subjf == nil {\n\t\treturn false\n\t}\n\t// If we are here we want to check if the filtered subject is\n\t// a direct match for our only listed subject.\n\tmset := o.mset\n\tif mset == nil {\n\t\treturn true\n\t}\n\n\t// Protect access to mset.cfg with the cfgMu mutex.\n\tmset.cfgMu.RLock()\n\tmsetSubjects := mset.cfg.Subjects\n\tmset.cfgMu.RUnlock()\n\n\t// `isFiltered` need to be performant, so we do\n\t// as any checks as possible to avoid unnecessary work.\n\t// Here we avoid iteration over slices if there is only one subject in stream\n\t// and one filter for the consumer.\n\tif len(msetSubjects) == 1 && len(o.subjf) == 1 {\n\t\treturn msetSubjects[0] != o.subjf[0].subject\n\t}\n\n\t// if the list is not equal length, we can return early, as this is filtered.\n\tif len(msetSubjects) != len(o.subjf) {\n\t\treturn true\n\t}\n\n\t// if in rare case scenario that user passed all stream subjects as consumer filters,\n\t// we need to do a more expensive operation.\n\t// reflect.DeepEqual would return false if the filters are the same, but in different order\n\t// so it can't be used here.\n\tcfilters := make(map[string]struct{}, len(o.subjf))\n\tfor _, val := range o.subjf {\n\t\tcfilters[val.subject] = struct{}{}\n\t}\n\tfor _, val := range msetSubjects {\n\t\tif _, ok := cfilters[val]; !ok {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Check if we need an ack for this store seq.\n// This is called for interest based retention streams to remove messages.\nfunc (o *consumer) needAck(sseq uint64, subj string) bool {\n\tvar needAck bool\n\tvar asflr, osseq uint64\n\tvar pending map[uint64]*Pending\n\tvar rdc map[uint64]uint64\n\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\n\tisFiltered := o.isFiltered()\n\tif isFiltered && o.mset == nil {\n\t\treturn false\n\t}\n\n\t// Check if we are filtered, and if so check if this is even applicable to us.\n\tif isFiltered {\n\t\tif subj == _EMPTY_ {\n\t\t\tvar err error\n\t\t\tif subj, err = o.mset.store.SubjectForSeq(sseq); err != nil {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif !o.isFilteredMatch(subj) {\n\t\t\treturn false\n\t\t}\n\t}\n\tif o.isLeader() {\n\t\tasflr, osseq = o.asflr, o.sseq\n\t\tpending, rdc = o.pending, o.rdc\n\t} else {\n\t\tif o.store == nil {\n\t\t\treturn false\n\t\t}\n\t\tstate, err := o.store.BorrowState()\n\t\tif err != nil || state == nil {\n\t\t\t// Fall back to what we track internally for now.\n\t\t\treturn sseq > o.asflr && !o.isFiltered()\n\t\t}\n\t\t// If loading state as here, the osseq is +1.\n\t\tasflr, osseq, pending, rdc = state.AckFloor.Stream, state.Delivered.Stream+1, state.Pending, state.Redelivered\n\t}\n\n\tswitch o.cfg.AckPolicy {\n\tcase AckNone, AckAll:\n\t\tneedAck = sseq > asflr\n\tcase AckExplicit:\n\t\tif sseq > asflr {\n\t\t\tif sseq >= osseq {\n\t\t\t\tneedAck = true\n\t\t\t} else {\n\t\t\t\t_, needAck = pending[sseq]\n\t\t\t}\n\t\t}\n\t}\n\n\t// Finally check if redelivery of this message is tracked.\n\t// If the message is not pending, it should be preserved if it reached max delivery.\n\tif !needAck {\n\t\t_, needAck = rdc[sseq]\n\t}\n\n\treturn needAck\n}\n\ntype PriorityGroup struct {\n\tGroup string `json:\"group,omitempty\"`\n\tMinPending int64 `json:\"min_pending,omitempty\"`\n\tMinAckPending int64 `json:\"min_ack_pending,omitempty\"`\n\tId string `json:\"id,omitempty\"`\n\tPriority int `json:\"priority,omitempty\"`\n}\n\n// Used in nextReqFromMsg, since the json.Unmarshal causes the request\n// struct to escape to the heap always. This should reduce GC pressure.\nvar jsGetNextPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &JSApiConsumerGetNextRequest{}\n\t},\n}\n\n// Helper for the next message requests.\nfunc nextReqFromMsg(msg []byte) (time.Time, int, int, bool, time.Duration, time.Time, *PriorityGroup, error) {\n\treq := bytes.TrimSpace(msg)\n\n\tswitch {\n\tcase len(req) == 0:\n\t\treturn time.Time{}, 1, 0, false, 0, time.Time{}, nil, nil\n\n\tcase req[0] == '{':\n\t\tcr := jsGetNextPool.Get().(*JSApiConsumerGetNextRequest)\n\t\tdefer func() {\n\t\t\t*cr = JSApiConsumerGetNextRequest{}\n\t\t\tjsGetNextPool.Put(cr)\n\t\t}()\n\t\tif err := json.Unmarshal(req, &cr); err != nil {\n\t\t\treturn time.Time{}, -1, 0, false, 0, time.Time{}, nil, err\n\t\t}\n\t\tvar hbt time.Time\n\t\tif cr.Heartbeat > 0 {\n\t\t\tif cr.Heartbeat*2 > cr.Expires {\n\t\t\t\treturn time.Time{}, 1, 0, false, 0, time.Time{}, nil, errors.New(\"heartbeat value too large\")\n\t\t\t}\n\t\t\thbt = time.Now().Add(cr.Heartbeat)\n\t\t}\n\t\tpriorityGroup := cr.PriorityGroup\n\t\tif cr.Expires == time.Duration(0) {\n\t\t\treturn time.Time{}, cr.Batch, cr.MaxBytes, cr.NoWait, cr.Heartbeat, hbt, &priorityGroup, nil\n\t\t}\n\t\treturn time.Now().Add(cr.Expires), cr.Batch, cr.MaxBytes, cr.NoWait, cr.Heartbeat, hbt, &priorityGroup, nil\n\tdefault:\n\t\tif n, err := strconv.Atoi(string(req)); err == nil {\n\t\t\treturn time.Time{}, n, 0, false, 0, time.Time{}, nil, nil\n\t\t}\n\t}\n\n\treturn time.Time{}, 1, 0, false, 0, time.Time{}, nil, nil\n}\n\n// Represents a request that is on the internal waiting queue\ntype waitingRequest struct {\n\tnext *waitingRequest\n\tacc *Account\n\tinterest string\n\treply string\n\tn int // For batching\n\td int // num delivered\n\tb int // For max bytes tracking\n\texpires time.Time\n\treceived time.Time\n\thb time.Duration\n\thbt time.Time\n\tnoWait bool\n\tpriorityGroup *PriorityGroup\n}\n\n// sync.Pool for waiting requests.\nvar wrPool = sync.Pool{\n\tNew: func() any {\n\t\treturn new(waitingRequest)\n\t},\n}\n\n// Recycle this request. This request can not be accessed after this call.\nfunc (wr *waitingRequest) recycleIfDone() bool {\n\tif wr != nil && wr.n <= 0 {\n\t\twr.recycle()\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Force a recycle.\nfunc (wr *waitingRequest) recycle() {\n\tif wr != nil {\n\t\twr.next, wr.acc, wr.interest, wr.reply = nil, nil, _EMPTY_, _EMPTY_\n\t\twrPool.Put(wr)\n\t}\n}\n\n// Represents an (optional) request timeout that's sent after waiting for replicated deliveries.\ntype waitingDelivery struct {\n\tseq uint64\n\tpn int // Pending messages.\n\tpb int // Pending bytes.\n}\n\n// sync.Pool for waiting deliveries.\nvar wdPool = sync.Pool{\n\tNew: func() any {\n\t\treturn new(waitingDelivery)\n\t},\n}\n\n// Force a recycle.\nfunc (wd *waitingDelivery) recycle() {\n\tif wd != nil {\n\t\twd.seq, wd.pn, wd.pb = 0, 0, 0\n\t\twdPool.Put(wd)\n\t}\n}\n\n// waiting queue for requests that are waiting for new messages to arrive.\ntype waitQueue struct {\n\tn, max int\n\tlast time.Time\n\thead *waitingRequest\n\ttail *waitingRequest\n}\n\n// Create a new ring buffer with at most max items.\nfunc newWaitQueue(max int) *waitQueue {\n\treturn &waitQueue{max: max}\n}\n\nvar (\n\terrWaitQueueFull = errors.New(\"wait queue is full\")\n\terrWaitQueueNil = errors.New(\"wait queue is nil\")\n)\n\n// insertSorted inserts wr at the correct position based on priority\nfunc (wq *waitQueue) insertSorted(wr *waitingRequest) {\n\n\t// Handle empty queue\n\tif wq.head == nil {\n\t\twq.head = wr\n\t\twq.tail = wr\n\t\twr.next = nil\n\t\treturn\n\t}\n\tinsertAtPosition(wr, wq)\n}\n\nfunc (wq *waitQueue) addPrioritized(wr *waitingRequest) error {\n\tif wq == nil {\n\t\treturn errWaitQueueNil\n\t}\n\tif wq.isFull() {\n\t\treturn errWaitQueueFull\n\t}\n\n\twq.insertSorted(wr)\n\twq.n++\n\twq.last = wr.received\n\treturn nil\n}\n\n// Adds in a new request.\nfunc (wq *waitQueue) add(wr *waitingRequest) error {\n\tif wq == nil {\n\t\treturn errWaitQueueNil\n\t}\n\tif wq.isFull() {\n\t\treturn errWaitQueueFull\n\t}\n\tif wq.head == nil {\n\t\twq.head = wr\n\t} else {\n\t\twq.tail.next = wr\n\t}\n\t// Always set tail.\n\twq.tail = wr\n\t// Make sure nil\n\twr.next = nil\n\n\t// Track last active via when we receive a request.\n\twq.last = wr.received\n\twq.n++\n\treturn nil\n}\n\nfunc (wq *waitQueue) isFull() bool {\n\tif wq == nil {\n\t\treturn false\n\t}\n\treturn wq.n == wq.max\n}\n\nfunc (wq *waitQueue) isEmpty() bool {\n\tif wq == nil {\n\t\treturn true\n\t}\n\treturn wq.n == 0\n}\n\nfunc (wq *waitQueue) len() int {\n\tif wq == nil {\n\t\treturn 0\n\t}\n\treturn wq.n\n}\n\n// Peek will return the next request waiting or nil if empty.\nfunc (wq *waitQueue) peek() *waitingRequest {\n\tif wq == nil {\n\t\treturn nil\n\t}\n\treturn wq.head\n}\n\nfunc (wq *waitQueue) cycle() {\n\twr := wq.peek()\n\tif wr != nil {\n\t\t// Always remove current now on a pop, and move to end if still valid.\n\t\t// If we were the only one don't need to remove since this can be a no-op.\n\t\twq.removeCurrent()\n\t\twq.add(wr)\n\t}\n}\n\nfunc (wq *waitQueue) popOrPopAndRequeue(priority PriorityPolicy) *waitingRequest {\n\tif wq == nil || wq.head == nil {\n\t\treturn nil\n\t}\n\n\tif priority == PriorityPrioritized {\n\t\treturn wq.popAndRequeue()\n\t}\n\treturn wq.pop()\n}\n\n// pop will return the next request and move the read cursor.\n// This will now place a request that still has pending items at the ends of the list.\nfunc (wq *waitQueue) pop() *waitingRequest {\n\twr := wq.peek()\n\tif wr != nil {\n\t\twr.d++\n\t\twr.n--\n\t\t// Always remove current now on a pop, and move to end if still valid.\n\t\t// If we were the only one don't need to remove since this can be a no-op.\n\t\tif wr.n > 0 && wq.n > 1 {\n\t\t\twq.removeCurrent()\n\t\t\twq.add(wr)\n\t\t} else if wr.n <= 0 {\n\t\t\twq.removeCurrent()\n\t\t}\n\t}\n\treturn wr\n}\n\n// popAndRequeue pops the head element and requeues it at the end of its priority group.\n// This maintains FIFO order within the same priority level.\nfunc (wq *waitQueue) popAndRequeue() *waitingRequest {\n\tif wq == nil || wq.head == nil {\n\t\treturn nil\n\t}\n\n\t// Save the head\n\twr := wq.head\n\n\tif wr == nil {\n\t\treturn wr\n\t}\n\n\twr.d++\n\twr.n--\n\n\tif wr.n > 0 && wq.n > 1 {\n\t\tif wr.next == nil {\n\t\t\treturn wr\n\t\t}\n\t\twq.head = wq.head.next\n\t\twr.next = nil\n\n\t\tinsertAtPosition(wr, wq)\n\n\t} else if wr.n <= 0 {\n\t\twq.removeCurrent()\n\t\treturn wr\n\n\t}\n\n\treturn wr\n}\n\nfunc insertAtPosition(wr *waitingRequest, wq *waitQueue) {\n\tpriority := math.MaxInt32\n\tif wr.priorityGroup != nil {\n\t\tpriority = wr.priorityGroup.Priority\n\t}\n\n\tvar prev *waitingRequest\n\tcurrent := wq.head\n\tfor current != nil {\n\t\tcurrentPriority := math.MaxInt32\n\t\tif current.priorityGroup != nil {\n\t\t\tcurrentPriority = current.priorityGroup.Priority\n\t\t}\n\t\tif currentPriority > priority {\n\t\t\tbreak\n\t\t}\n\t\tprev = current\n\t\tcurrent = current.next\n\t}\n\n\tif prev == nil {\n\t\t// All remaining elements have higher priority\n\t\twr.next = wq.head\n\t\twq.head = wr\n\t} else {\n\t\twr.next = prev.next\n\t\tprev.next = wr\n\t\tif wr.next == nil {\n\t\t\twq.tail = wr\n\t\t}\n\t}\n}\n\n// Removes the current read pointer (head FIFO) entry.\nfunc (wq *waitQueue) removeCurrent() {\n\twq.remove(nil, wq.head)\n}\n\n// Remove the wr element from the wait queue.\nfunc (wq *waitQueue) remove(pre, wr *waitingRequest) {\n\tif wr == nil {\n\t\treturn\n\t}\n\tif pre != nil {\n\t\tpre.next = wr.next\n\t} else if wr == wq.head {\n\t\t// We are removing head here.\n\t\twq.head = wr.next\n\t}\n\t// Check if wr was our tail.\n\tif wr == wq.tail {\n\t\t// Check if we need to assign to pre.\n\t\tif wr.next == nil {\n\t\t\twq.tail = pre\n\t\t} else {\n\t\t\twq.tail = wr.next\n\t\t}\n\t}\n\twq.n--\n}\n\n// Return the map of pending requests keyed by the reply subject.\n// No-op if push consumer or invalid etc.\nfunc (o *consumer) pendingRequests() map[string]*waitingRequest {\n\tif o.waiting == nil {\n\t\treturn nil\n\t}\n\twq, m := o.waiting, make(map[string]*waitingRequest)\n\tfor wr := wq.head; wr != nil; wr = wr.next {\n\t\tm[wr.reply] = wr\n\t}\n\n\treturn m\n}\n\nfunc (o *consumer) setPinnedTimer(priorityGroup string) {\n\tif o.pinnedTtl != nil {\n\t\to.pinnedTtl.Reset(o.cfg.PinnedTTL)\n\t} else {\n\t\to.pinnedTtl = time.AfterFunc(o.cfg.PinnedTTL, func() {\n\t\t\to.mu.Lock()\n\t\t\t// Skip if already unset.\n\t\t\tif o.currentPinId == _EMPTY_ {\n\t\t\t\to.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\to.unassignPinId()\n\t\t\to.sendUnpinnedAdvisoryLocked(priorityGroup, \"timeout\")\n\t\t\to.mu.Unlock()\n\t\t\to.signalNewMessages()\n\t\t})\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) assignNewPinId(wr *waitingRequest) {\n\tif wr.priorityGroup == nil || wr.priorityGroup.Group == _EMPTY_ {\n\t\treturn\n\t}\n\to.currentPinId = nuid.Next()\n\to.pinnedTS = time.Now().UTC()\n\twr.priorityGroup.Id = o.currentPinId\n\to.setPinnedTimer(wr.priorityGroup.Group)\n\to.sendPinnedAdvisoryLocked(wr.priorityGroup.Group)\n}\n\n// Lock should be held.\nfunc (o *consumer) unassignPinId() {\n\to.currentPinId = _EMPTY_\n\to.pinnedTS = time.Time{}\n\tif o.pinnedTtl != nil {\n\t\to.pinnedTtl.Stop()\n\t\to.pinnedTtl = nil\n\t}\n}\n\n// Return next waiting request. This will check for expirations but not noWait or interest.\n// That will be handled by processWaiting.\n// Lock should be held.\nfunc (o *consumer) nextWaiting(sz int) *waitingRequest {\n\tif o.waiting == nil || o.waiting.isEmpty() {\n\t\treturn nil\n\t}\n\n\t// Check if server needs to assign a new pin id.\n\tneedNewPin := o.currentPinId == _EMPTY_ && o.cfg.PriorityPolicy == PriorityPinnedClient\n\n\tnumCycled := 0\n\tfor wr := o.waiting.peek(); !o.waiting.isEmpty(); wr = o.waiting.peek() {\n\t\tif wr == nil {\n\t\t\tbreak\n\t\t}\n\t\t// Check if we have max bytes set.\n\t\tif wr.b > 0 {\n\t\t\tif sz <= wr.b {\n\t\t\t\twr.b -= sz\n\t\t\t\t// If we are right now at zero, set batch to 1 to deliver this one but stop after.\n\t\t\t\tif wr.b == 0 {\n\t\t\t\t\twr.n = 1\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Since we can't send that message to the requestor, we need to\n\t\t\t\t// notify that we are closing the request.\n\t\t\t\tconst maxBytesT = \"NATS/1.0 409 Message Size Exceeds MaxBytes\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\"\n\t\t\t\thdr := fmt.Appendf(nil, maxBytesT, JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\t// Remove the current one, no longer valid due to max bytes limit.\n\t\t\t\to.waiting.removeCurrent()\n\t\t\t\tif o.node != nil {\n\t\t\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t\t\t}\n\t\t\t\twr.recycle()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif wr.expires.IsZero() || time.Now().Before(wr.expires) {\n\t\t\tif needNewPin {\n\t\t\t\tif wr.priorityGroup.Id == _EMPTY_ {\n\t\t\t\t\to.assignNewPinId(wr)\n\t\t\t\t} else {\n\t\t\t\t\t// There is pin id set, but not a matching one. Send a notification to the client and remove the request.\n\t\t\t\t\t// Probably this is the old pin id.\n\t\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 423 Nats-Wrong-Pin-Id\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\t\to.waiting.removeCurrent()\n\t\t\t\t\tif o.node != nil {\n\t\t\t\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t\t\t\t}\n\t\t\t\t\twr.recycle()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t} else if o.currentPinId != _EMPTY_ {\n\t\t\t\t// Check if we have a match on the currentNuid\n\t\t\t\tif wr.priorityGroup != nil && wr.priorityGroup.Id == o.currentPinId {\n\t\t\t\t\t// If we have a match, we do nothing here and will deliver the message later down the code path.\n\t\t\t\t} else if wr.priorityGroup.Id == _EMPTY_ {\n\t\t\t\t\to.waiting.cycle()\n\t\t\t\t\tnumCycled++\n\t\t\t\t\tif numCycled >= o.waiting.len() {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\t// There is pin id set, but not a matching one. Send a notification to the client and remove the request.\n\t\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 423 Nats-Wrong-Pin-Id\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\t\to.waiting.removeCurrent()\n\t\t\t\t\tif o.node != nil {\n\t\t\t\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t\t\t\t}\n\t\t\t\t\twr.recycle()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif o.cfg.PriorityPolicy == PriorityOverflow {\n\t\t\t\tif wr.priorityGroup != nil &&\n\t\t\t\t\t// If both limits are zero we don't cycle and the request will be fulfilled.\n\t\t\t\t\t(wr.priorityGroup.MinPending > 0 || wr.priorityGroup.MinAckPending > 0) &&\n\t\t\t\t\t// We need to check o.npc+1, because before calling nextWaiting, we do o.npc--\n\t\t\t\t\t// If one OR the other limit is exceeded, we want to fulfill the request.\n\t\t\t\t\t// This is an inverted check. For clarity, we check the positive condition and negate.\n\t\t\t\t\t!((wr.priorityGroup.MinPending > 0 && wr.priorityGroup.MinPending <= o.npc+1) ||\n\t\t\t\t\t\t(wr.priorityGroup.MinAckPending > 0 && wr.priorityGroup.MinAckPending <= int64(len(o.pending)))) {\n\t\t\t\t\to.waiting.cycle()\n\t\t\t\t\tnumCycled++\n\t\t\t\t\t// We're done cycling through the requests.\n\t\t\t\t\tif numCycled >= o.waiting.len() {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif wr.acc.sl.HasInterest(wr.interest) {\n\t\t\t\treturn o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)\n\t\t\t} else if time.Since(wr.received) < defaultGatewayRecentSubExpiration && (o.srv.leafNodeEnabled || o.srv.gateway.enabled) {\n\t\t\t\treturn o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)\n\t\t\t} else if o.srv.gateway.enabled && o.srv.hasGatewayInterest(wr.acc.Name, wr.interest) {\n\t\t\t\treturn o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)\n\t\t\t}\n\t\t} else {\n\t\t\t// We do check for expiration in `processWaiting`, but it is possible to hit the expiry here, and not there.\n\t\t\trdWait := o.replicateDeliveries()\n\t\t\tif rdWait {\n\t\t\t\t// Check if we need to send the timeout after pending replicated deliveries, or can do so immediately.\n\t\t\t\tif wd, ok := o.waitingDeliveries[wr.reply]; !ok {\n\t\t\t\t\trdWait = false\n\t\t\t\t} else {\n\t\t\t\t\twd.pn, wd.pb = wr.n, wr.b\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !rdWait {\n\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 408 Request Timeout\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t}\n\t\t\to.waiting.removeCurrent()\n\t\t\tif o.node != nil {\n\t\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t\t}\n\t\t\twr.recycle()\n\t\t\tcontinue\n\n\t\t}\n\t\tif wr.interest != wr.reply {\n\t\t\tconst intExpT = \"NATS/1.0 408 Interest Expired\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\"\n\t\t\thdr := fmt.Appendf(nil, intExpT, JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t}\n\t\t// Remove the current one, no longer valid.\n\t\to.waiting.removeCurrent()\n\t\tif o.node != nil {\n\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t}\n\t\twr.recycle()\n\t}\n\n\treturn nil\n}\n\n// Next message request.\ntype nextMsgReq struct {\n\treply string\n\tmsg []byte\n}\n\nvar nextMsgReqPool sync.Pool\n\nfunc newNextMsgReq(reply string, msg []byte) *nextMsgReq {\n\tvar nmr *nextMsgReq\n\tm := nextMsgReqPool.Get()\n\tif m != nil {\n\t\tnmr = m.(*nextMsgReq)\n\t} else {\n\t\tnmr = &nextMsgReq{}\n\t}\n\t// When getting something from a pool it is critical that all fields are\n\t// initialized. Doing this way guarantees that if someone adds a field to\n\t// the structure, the compiler will fail the build if this line is not updated.\n\t(*nmr) = nextMsgReq{reply, msg}\n\treturn nmr\n}\n\nfunc (nmr *nextMsgReq) returnToPool() {\n\tif nmr == nil {\n\t\treturn\n\t}\n\tnmr.reply, nmr.msg = _EMPTY_, nil\n\tnextMsgReqPool.Put(nmr)\n}\n\n// processNextMsgReq will process a request for the next message available. A nil message payload means deliver\n// a single message. If the payload is a formal request or a number parseable with Atoi(), then we will send a\n// batch of messages without requiring another request to this endpoint, or an ACK.\nfunc (o *consumer) processNextMsgReq(_ *subscription, c *client, _ *Account, _, reply string, rmsg []byte) {\n\tif reply == _EMPTY_ {\n\t\treturn\n\t}\n\n\t// Short circuit error here.\n\tif o.nextMsgReqs == nil {\n\t\thdr := []byte(\"NATS/1.0 409 Consumer is push based\\r\\n\\r\\n\")\n\t\to.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\n\thdr, msg := c.msgParts(rmsg)\n\tif errorOnRequiredApiLevel(hdr) {\n\t\thdr = []byte(\"NATS/1.0 412 Required Api Level\\r\\n\\r\\n\")\n\t\to.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\to.nextMsgReqs.push(newNextMsgReq(reply, copyBytes(msg)))\n}\n\n// processResetReq will reset a consumer to a new starting sequence.\nfunc (o *consumer) processResetReq(_ *subscription, c *client, a *Account, _, reply string, rmsg []byte) {\n\tif reply == _EMPTY_ {\n\t\treturn\n\t}\n\n\ts := o.srv\n\tvar resp = JSApiConsumerResetResponse{ApiResponse: ApiResponse{Type: JSApiConsumerResetResponseType}}\n\n\thdr, msg := c.msgParts(rmsg)\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendInternalAccountMsg(a, reply, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// An empty message resets back to the ack floor, otherwise a custom sequence can be used.\n\tvar req JSApiConsumerResetRequest\n\tif len(msg) > 0 {\n\t\tif err := json.Unmarshal(msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendInternalAccountMsg(a, reply, s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\tresetSeq, canRespond, err := o.resetStartingSeq(req.Seq, reply)\n\tif err != nil {\n\t\tresp.Error = NewJSConsumerInvalidResetError(err)\n\t\ts.sendInternalAccountMsg(a, reply, s.jsonResponse(&resp))\n\t} else if canRespond {\n\t\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.info())\n\t\tresp.ResetSeq = resetSeq\n\t\ts.sendInternalAccountMsg(a, reply, s.jsonResponse(&resp))\n\t}\n}\n\nfunc (o *consumer) processNextMsgRequest(reply string, msg []byte) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tmset := o.mset\n\tif mset == nil {\n\t\treturn\n\t}\n\n\tsendErr := func(status int, description string) {\n\t\thdr := fmt.Appendf(nil, \"NATS/1.0 %d %s\\r\\n\\r\\n\", status, description)\n\t\to.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t}\n\n\tif o.isPushMode() || o.waiting == nil {\n\t\tsendErr(409, \"Consumer is push based\")\n\t\treturn\n\t}\n\n\t// Check payload here to see if they sent in batch size or a formal request.\n\texpires, batchSize, maxBytes, noWait, hb, hbt, priorityGroup, err := nextReqFromMsg(msg)\n\tif err != nil {\n\t\tsendErr(400, fmt.Sprintf(\"Bad Request - %v\", err))\n\t\treturn\n\t}\n\n\t// Check for request limits\n\tif o.cfg.MaxRequestBatch > 0 && batchSize > o.cfg.MaxRequestBatch {\n\t\tsendErr(409, fmt.Sprintf(\"Exceeded MaxRequestBatch of %d\", o.cfg.MaxRequestBatch))\n\t\treturn\n\t}\n\n\tif !expires.IsZero() && o.cfg.MaxRequestExpires > 0 && expires.After(time.Now().Add(o.cfg.MaxRequestExpires)) {\n\t\tsendErr(409, fmt.Sprintf(\"Exceeded MaxRequestExpires of %v\", o.cfg.MaxRequestExpires))\n\t\treturn\n\t}\n\n\tif maxBytes > 0 && o.cfg.MaxRequestMaxBytes > 0 && maxBytes > o.cfg.MaxRequestMaxBytes {\n\t\tsendErr(409, fmt.Sprintf(\"Exceeded MaxRequestMaxBytes of %v\", o.cfg.MaxRequestMaxBytes))\n\t\treturn\n\t}\n\n\tif priorityGroup != nil {\n\t\tif (priorityGroup.MinPending != 0 || priorityGroup.MinAckPending != 0) && o.cfg.PriorityPolicy != PriorityOverflow {\n\t\t\tsendErr(400, \"Bad Request - Not a Overflow Priority consumer\")\n\t\t}\n\n\t\tif priorityGroup.Id != _EMPTY_ && o.cfg.PriorityPolicy != PriorityPinnedClient {\n\t\t\tsendErr(400, \"Bad Request - Not a Pinned Client Priority consumer\")\n\t\t}\n\t\tif priorityGroup.Priority < 0 || priorityGroup.Priority > 9 {\n\t\t\tsendErr(400, \"Bad Request - Priority must be between 0 and 9\")\n\t\t\treturn\n\t\t}\n\t}\n\n\tif priorityGroup != nil && o.cfg.PriorityPolicy != PriorityNone {\n\t\tif priorityGroup.Group == _EMPTY_ {\n\t\t\tsendErr(400, \"Bad Request - Priority Group missing\")\n\t\t\treturn\n\t\t}\n\t\tif !slices.Contains(o.cfg.PriorityGroups, priorityGroup.Group) {\n\t\t\tsendErr(400, \"Bad Request - Invalid Priority Group\")\n\t\t\treturn\n\t\t}\n\n\t\tif o.currentPinId != _EMPTY_ {\n\t\t\tif priorityGroup.Id == o.currentPinId {\n\t\t\t\to.setPinnedTimer(priorityGroup.Group)\n\t\t\t} else if priorityGroup.Id != _EMPTY_ {\n\t\t\t\tsendErr(423, \"Nats-Pin-Id mismatch\")\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we have the max number of requests already pending try to expire.\n\tif o.waiting.isFull() {\n\t\t// Try to expire some of the requests.\n\t\t// We do not want to push too hard here so at maximum process once per sec.\n\t\tif time.Since(o.lwqic) > time.Second {\n\t\t\to.processWaiting(false)\n\t\t}\n\t}\n\n\t// If the request is for noWait and we have pending requests already, check if we have room.\n\tif noWait {\n\t\tmsgsPending := o.numPending() + uint64(len(o.rdq))\n\t\t// If no pending at all, decide what to do with request.\n\t\t// If no expires was set then fail.\n\t\tif msgsPending == 0 && expires.IsZero() {\n\t\t\to.waiting.last = time.Now()\n\t\t\tsendErr(404, \"No Messages\")\n\t\t\treturn\n\t\t}\n\t\tif msgsPending > 0 {\n\t\t\t_, _, batchPending, _ := o.processWaiting(false)\n\t\t\tif msgsPending < uint64(batchPending) {\n\t\t\t\to.waiting.last = time.Now()\n\t\t\t\tsendErr(408, \"Requests Pending\")\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// If we are here this should be considered a one-shot situation.\n\t\t// We will wait for expires but will return as soon as we have any messages.\n\t}\n\n\t// If we receive this request though an account export, we need to track that interest subject and account.\n\tacc, interest := trackDownAccountAndInterest(o.acc, reply)\n\n\t// Create a waiting request.\n\twr := wrPool.Get().(*waitingRequest)\n\twr.acc, wr.interest, wr.reply, wr.n, wr.d, wr.noWait, wr.expires, wr.hb, wr.hbt, wr.priorityGroup = acc, interest, reply, batchSize, 0, noWait, expires, hb, hbt, priorityGroup\n\twr.b = maxBytes\n\twr.received = time.Now()\n\n\tif o.cfg.PriorityPolicy == PriorityPrioritized {\n\t\tif err := o.waiting.addPrioritized(wr); err != nil {\n\t\t\tif hb == 0 {\n\t\t\t\tsendErr(409, \"Exceeded MaxWaiting\")\n\t\t\t}\n\t\t\twr.recycle()\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tif err := o.waiting.add(wr); err != nil {\n\t\t\t// If the client has a heartbeat interval set, don't bother responding with a 409,\n\t\t\t// otherwise we can end up in a hot loop with the client re-requesting instead of\n\t\t\t// waiting for the missing heartbeats instead and retrying.\n\t\t\tif hb == 0 {\n\t\t\t\tsendErr(409, \"Exceeded MaxWaiting\")\n\t\t\t}\n\t\t\twr.recycle()\n\t\t\treturn\n\t\t}\n\t}\n\to.signalNewMessages()\n\t// If we are clustered update our followers about this request.\n\tif o.node != nil {\n\t\to.addClusterPendingRequest(wr.reply)\n\t}\n}\n\nfunc trackDownAccountAndInterest(acc *Account, interest string) (*Account, string) {\n\tfor strings.HasPrefix(interest, replyPrefix) {\n\t\toa := acc\n\t\toa.mu.RLock()\n\t\tif oa.exports.responses == nil {\n\t\t\toa.mu.RUnlock()\n\t\t\tbreak\n\t\t}\n\t\tsi := oa.exports.responses[interest]\n\t\tif si == nil {\n\t\t\toa.mu.RUnlock()\n\t\t\tbreak\n\t\t}\n\t\tacc, interest = si.acc, si.to\n\t\toa.mu.RUnlock()\n\t}\n\treturn acc, interest\n}\n\n// Return current delivery count for a given sequence.\nfunc (o *consumer) deliveryCount(seq uint64) uint64 {\n\tif o.rdc == nil {\n\t\treturn 1\n\t}\n\tif dc := o.rdc[seq]; dc >= 1 {\n\t\treturn dc\n\t}\n\treturn 1\n}\n\n// Increase the delivery count for this message.\n// ONLY used on redelivery semantics.\n// Lock should be held.\nfunc (o *consumer) incDeliveryCount(sseq uint64) uint64 {\n\tif o.rdc == nil {\n\t\to.rdc = make(map[uint64]uint64)\n\t}\n\to.rdc[sseq] += 1\n\treturn o.rdc[sseq] + 1\n}\n\n// Used if we have to adjust on failed delivery or bad lookups.\n// Those failed attempts should not increase deliver count.\n// Lock should be held.\nfunc (o *consumer) decDeliveryCount(sseq uint64) {\n\tif o.rdc == nil {\n\t\to.rdc = make(map[uint64]uint64)\n\t}\n\to.rdc[sseq] -= 1\n}\n\n// send a delivery exceeded advisory.\nfunc (o *consumer) notifyDeliveryExceeded(sseq, dc uint64) {\n\te := JSConsumerDeliveryExceededAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerDeliveryExceededAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: o.stream,\n\t\tConsumer: o.name,\n\t\tStreamSeq: sseq,\n\t\tDeliveries: dc,\n\t\tDomain: o.srv.getOpts().JetStreamDomain,\n\t}\n\n\to.sendAdvisory(o.deliveryExcEventT, e)\n}\n\n// Check if the candidate subject matches a filter if its present.\n// Lock should be held.\nfunc (o *consumer) isFilteredMatch(subj string) bool {\n\t// No filter is automatic match.\n\tif o.subjf == nil {\n\t\treturn true\n\t}\n\tfor _, filter := range o.subjf {\n\t\tif !filter.hasWildcard && subj == filter.subject {\n\t\t\treturn true\n\t\t}\n\t}\n\t// It's quicker to first check for non-wildcard filters, then\n\t// iterate again to check for subset match.\n\ttsa := [32]string{}\n\ttts := tokenizeSubjectIntoSlice(tsa[:0], subj)\n\tfor _, filter := range o.subjf {\n\t\tif isSubsetMatchTokenized(tts, filter.tokenizedSubject) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Check if the candidate filter subject is equal to or a subset match\n// of one of the filter subjects.\n// Lock should be held.\nfunc (o *consumer) isEqualOrSubsetMatch(subj string) bool {\n\tfor _, filter := range o.subjf {\n\t\tif !filter.hasWildcard && subj == filter.subject {\n\t\t\treturn true\n\t\t}\n\t}\n\ttsa := [32]string{}\n\ttts := tokenizeSubjectIntoSlice(tsa[:0], subj)\n\tfor _, filter := range o.subjf {\n\t\tif isSubsetMatchTokenized(filter.tokenizedSubject, tts) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nvar (\n\terrMaxAckPending = errors.New(\"max ack pending reached\")\n\terrBadConsumer = errors.New(\"consumer not valid\")\n\terrNoInterest = errors.New(\"consumer requires interest for delivery subject when ephemeral\")\n)\n\n// Get next available message from underlying store.\n// Is partition aware and redeliver aware.\n// Lock should be held.\nfunc (o *consumer) getNextMsg() (*jsPubMsg, uint64, error) {\n\tif o.mset == nil || o.mset.store == nil {\n\t\treturn nil, 0, errBadConsumer\n\t}\n\t// Process redelivered messages before looking at possibly \"skip list\" (deliver last per subject)\n\tif o.hasRedeliveries() {\n\t\tvar seq, dc uint64\n\t\tfor seq = o.getNextToRedeliver(); seq > 0; seq = o.getNextToRedeliver() {\n\t\t\tdc = o.incDeliveryCount(seq)\n\t\t\tif o.maxdc > 0 && dc > o.maxdc {\n\t\t\t\t// Only send once\n\t\t\t\tif dc == o.maxdc+1 {\n\t\t\t\t\to.notifyDeliveryExceeded(seq, dc-1)\n\t\t\t\t}\n\t\t\t\t// Make sure to remove from pending.\n\t\t\t\tif p, ok := o.pending[seq]; ok && p != nil {\n\t\t\t\t\tdelete(o.pending, seq)\n\t\t\t\t\to.updateDelivered(p.Sequence, seq, dc, p.Timestamp)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpmsg := getJSPubMsgFromPool()\n\t\t\tsm, err := o.mset.store.LoadMsg(seq, &pmsg.StoreMsg)\n\t\t\tif sm == nil || err != nil {\n\t\t\t\tpmsg.returnToPool()\n\t\t\t\tpmsg, dc = nil, 0\n\t\t\t\t// Adjust back deliver count.\n\t\t\t\to.decDeliveryCount(seq)\n\t\t\t}\n\t\t\t// Message was scheduled for redelivery but was removed in the meantime.\n\t\t\tif err == ErrStoreMsgNotFound || err == errDeletedMsg {\n\t\t\t\t// This is a race condition where the message is still in o.pending and\n\t\t\t\t// scheduled for redelivery, but it has been removed from the stream.\n\t\t\t\t// o.processTerm is called in a goroutine so could run after we get here.\n\t\t\t\t// That will correct the pending state and delivery/ack floors, so just skip here.\n\t\t\t\tpmsg.returnToPool()\n\t\t\t\tpmsg = nil\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn pmsg, dc, err\n\t\t}\n\t}\n\n\t// Check if we have max pending.\n\tif o.maxp > 0 && len(o.pending) >= o.maxp {\n\t\t// maxp only set when ack policy != AckNone and user set MaxAckPending\n\t\t// Stall if we have hit max pending.\n\t\treturn nil, 0, errMaxAckPending\n\t}\n\n\tif o.hasSkipListPending() {\n\t\tseq := o.lss.seqs[0]\n\t\tif len(o.lss.seqs) == 1 {\n\t\t\to.sseq = o.lss.resume\n\t\t\to.lss = nil\n\t\t\to.updateSkipped(o.sseq)\n\t\t} else {\n\t\t\to.lss.seqs = o.lss.seqs[1:]\n\t\t\to.sseq = seq\n\t\t}\n\t\tpmsg := getJSPubMsgFromPool()\n\t\tsm, err := o.mset.store.LoadMsg(seq, &pmsg.StoreMsg)\n\t\tif sm == nil || err != nil {\n\t\t\tpmsg.returnToPool()\n\t\t\tpmsg = nil\n\t\t}\n\t\to.sseq++\n\t\treturn pmsg, 1, err\n\t}\n\n\tvar sseq uint64\n\tvar err error\n\tvar sm *StoreMsg\n\tvar pmsg = getJSPubMsgFromPool()\n\n\t// Grab next message applicable to us.\n\tfilters, subjf, fseq := o.filters, o.subjf, o.sseq\n\t// Check if we are multi-filtered or not.\n\tif filters != nil {\n\t\tsm, sseq, err = o.mset.store.LoadNextMsgMulti(filters, fseq, &pmsg.StoreMsg)\n\t} else if len(subjf) > 0 { // Means single filtered subject since o.filters means > 1.\n\t\tfilter, wc := subjf[0].subject, subjf[0].hasWildcard\n\t\tsm, sseq, err = o.mset.store.LoadNextMsg(filter, wc, fseq, &pmsg.StoreMsg)\n\t} else {\n\t\t// No filter here.\n\t\tsm, sseq, err = o.mset.store.LoadNextMsg(_EMPTY_, false, fseq, &pmsg.StoreMsg)\n\t}\n\tif sm == nil {\n\t\tpmsg.returnToPool()\n\t\tpmsg = nil\n\t}\n\t// Check if we should move our o.sseq.\n\tif sseq >= o.sseq {\n\t\t// If we are moving step by step then sseq == o.sseq.\n\t\t// If we have jumped we should update skipped for other replicas.\n\t\tif sseq != o.sseq && err == ErrStoreEOF {\n\t\t\to.updateSkipped(sseq + 1)\n\t\t}\n\t\to.sseq = sseq + 1\n\t}\n\treturn pmsg, 1, err\n}\n\n// Will check for expiration and lack of interest on waiting requests.\n// Will also do any heartbeats and return the next expiration or HB interval.\nfunc (o *consumer) processWaiting(eos bool) (int, int, int, time.Time) {\n\tvar fexp time.Time\n\tif o.srv == nil || o.waiting.isEmpty() {\n\t\treturn 0, 0, 0, fexp\n\t}\n\t// Mark our last check time.\n\to.lwqic = time.Now()\n\n\tvar expired, brp int\n\ts, now := o.srv, time.Now()\n\n\twq := o.waiting\n\tremove := func(pre, wr *waitingRequest) *waitingRequest {\n\t\texpired++\n\t\tif o.node != nil {\n\t\t\to.removeClusterPendingRequest(wr.reply)\n\t\t}\n\t\tnext := wr.next\n\t\twq.remove(pre, wr)\n\t\twr.recycle()\n\t\treturn next\n\t}\n\n\tvar pre *waitingRequest\n\tfor wr := wq.head; wr != nil; {\n\t\t// Check expiration.\n\t\texpires := !wr.expires.IsZero() && now.After(wr.expires)\n\t\tif (eos && wr.noWait) || expires {\n\t\t\trdWait := o.replicateDeliveries()\n\t\t\tif rdWait {\n\t\t\t\t// Check if we need to send the timeout after pending replicated deliveries, or can do so immediately.\n\t\t\t\tif wd, ok := o.waitingDeliveries[wr.reply]; !ok {\n\t\t\t\t\trdWait = false\n\t\t\t\t} else {\n\t\t\t\t\twd.pn, wd.pb = wr.n, wr.b\n\t\t\t\t}\n\t\t\t\t// If we still need to wait for replicated deliveries, remove from waiting list.\n\t\t\t\tif rdWait {\n\t\t\t\t\twr = remove(pre, wr)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Normally it's a timeout.\n\t\t\tif expires {\n\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 408 Request Timeout\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)\n\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\twr = remove(pre, wr)\n\t\t\t\tcontinue\n\t\t\t} else if wr.expires.IsZero() || wr.d > 0 {\n\t\t\t\t// But if we're NoWait without expiry, we've reached the end of the stream, and we've not delivered any messages.\n\t\t\t\t// Return no messages instead, which is the same as if we'd rejected the pull request initially.\n\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 404 No Messages\\r\\n\\r\\n\")\n\t\t\t\to.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\twr = remove(pre, wr)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// Now check interest.\n\t\tinterest := wr.acc.sl.HasInterest(wr.interest)\n\t\tif !interest && (s.leafNodeEnabled || s.gateway.enabled) {\n\t\t\t// If we are here check on gateways and leaf nodes (as they can mask gateways on the other end).\n\t\t\t// If we have interest or the request is too young break and do not expire.\n\t\t\tif time.Since(wr.received) < defaultGatewayRecentSubExpiration {\n\t\t\t\tinterest = true\n\t\t\t} else if s.gateway.enabled && s.hasGatewayInterest(wr.acc.Name, wr.interest) {\n\t\t\t\tinterest = true\n\t\t\t}\n\t\t}\n\t\t// Check if we have interest.\n\t\tif !interest {\n\t\t\t// No more interest here so go ahead and remove this one from our list.\n\t\t\twr = remove(pre, wr)\n\t\t\tcontinue\n\t\t}\n\n\t\t// If interest, update batch pending requests counter and update fexp timer.\n\t\tbrp += wr.n\n\t\tif !wr.hbt.IsZero() {\n\t\t\tif now.After(wr.hbt) {\n\t\t\t\t// Fire off a heartbeat here.\n\t\t\t\to.sendIdleHeartbeat(wr.reply)\n\t\t\t\t// Update next HB.\n\t\t\t\twr.hbt = now.Add(wr.hb)\n\t\t\t}\n\t\t\tif fexp.IsZero() || wr.hbt.Before(fexp) {\n\t\t\t\tfexp = wr.hbt\n\t\t\t}\n\t\t}\n\t\tif !wr.expires.IsZero() && (fexp.IsZero() || wr.expires.Before(fexp)) {\n\t\t\tfexp = wr.expires\n\t\t}\n\t\t// Update pre and wr here.\n\t\tpre = wr\n\t\twr = wr.next\n\t}\n\n\treturn expired, wq.len(), brp, fexp\n}\n\n// Will check to make sure those waiting still have registered interest.\nfunc (o *consumer) checkWaitingForInterest() bool {\n\to.processWaiting(true)\n\treturn o.waiting.len() > 0\n}\n\n// Lock should be held.\nfunc (o *consumer) hbTimer() (time.Duration, *time.Timer) {\n\tif o.cfg.Heartbeat == 0 {\n\t\treturn 0, nil\n\t}\n\treturn o.cfg.Heartbeat, time.NewTimer(o.cfg.Heartbeat)\n}\n\n// Check here for conditions when our ack floor may have drifted below the streams first sequence.\n// In general this is accounted for in normal operations, but if the consumer misses the signal from\n// the stream it will not clear the message and move the ack state.\n// Should only be called from consumer leader.\nfunc (o *consumer) checkAckFloor() {\n\to.mu.RLock()\n\tmset, closed, asflr, numPending := o.mset, o.closed, o.asflr, len(o.pending)\n\to.mu.RUnlock()\n\n\tif asflr == 0 || closed || mset == nil {\n\t\treturn\n\t}\n\n\tvar ss StreamState\n\tmset.store.FastState(&ss)\n\n\t// If our floor is equal or greater that is normal and nothing for us to do.\n\tif ss.FirstSeq == 0 || asflr >= ss.FirstSeq-1 {\n\t\treturn\n\t}\n\n\t// Check which linear space is less to walk.\n\tif ss.FirstSeq-asflr-1 < uint64(numPending) {\n\t\t// Process all messages that no longer exist.\n\t\tfor seq := asflr + 1; seq < ss.FirstSeq; seq++ {\n\t\t\t// Check if this message was pending.\n\t\t\to.mu.RLock()\n\t\t\tp, isPending := o.pending[seq]\n\t\t\trdc := o.deliveryCount(seq)\n\t\t\to.mu.RUnlock()\n\t\t\t// If it was pending for us, get rid of it.\n\t\t\tif isPending {\n\t\t\t\to.processTerm(seq, p.Sequence, rdc, ackTermLimitsReason, _EMPTY_)\n\t\t\t}\n\t\t}\n\t} else if numPending > 0 {\n\t\t// here it is shorter to walk pending.\n\t\t// toTerm is seq, dseq, rcd for each entry.\n\t\ttoTerm := make([]uint64, 0, numPending*3)\n\t\to.mu.RLock()\n\t\tfor seq, p := range o.pending {\n\t\t\tif seq < ss.FirstSeq {\n\t\t\t\tvar dseq uint64 = 1\n\t\t\t\tif p != nil {\n\t\t\t\t\tdseq = p.Sequence\n\t\t\t\t}\n\t\t\t\trdc := o.deliveryCount(seq)\n\t\t\t\ttoTerm = append(toTerm, seq, dseq, rdc)\n\t\t\t}\n\t\t}\n\t\to.mu.RUnlock()\n\n\t\tfor i := 0; i < len(toTerm); i += 3 {\n\t\t\tseq, dseq, rdc := toTerm[i], toTerm[i+1], toTerm[i+2]\n\t\t\to.processTerm(seq, dseq, rdc, ackTermLimitsReason, _EMPTY_)\n\t\t}\n\t}\n\n\t// Do one final check here.\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// If we are closed do not change anything and simply return.\n\tif o.closed {\n\t\treturn\n\t}\n\n\t// If we are here, and this should be rare, we still are off with our ack floor.\n\t// We will make sure we are not doing un-necessary work here if only off by a bit\n\t// since this could be normal for a high activity wq or stream.\n\t// We will set it explicitly to 1 behind our current lowest in pending, or if\n\t// pending is empty, to our current delivered -1.\n\tconst minOffThreshold = 50\n\tif ss.FirstSeq >= minOffThreshold && o.asflr < ss.FirstSeq-minOffThreshold {\n\t\tvar psseq, pdseq uint64\n\t\tfor seq, p := range o.pending {\n\t\t\tif psseq == 0 || seq < psseq {\n\t\t\t\tpsseq, pdseq = seq, p.Sequence\n\t\t\t}\n\t\t}\n\t\t// If we still have none, set to current delivered -1.\n\t\tif psseq == 0 {\n\t\t\tpsseq, pdseq = o.sseq-1, o.dseq-1\n\t\t\t// If still not adjusted.\n\t\t\tif psseq < ss.FirstSeq-1 {\n\t\t\t\tpsseq = ss.FirstSeq - 1\n\t\t\t}\n\t\t} else {\n\t\t\t// Since this was set via the pending, we should not include\n\t\t\t// it directly but set floors to -1.\n\t\t\tpsseq, pdseq = psseq-1, pdseq-1\n\t\t}\n\t\to.asflr, o.adflr = psseq, pdseq\n\t}\n}\n\nfunc (o *consumer) processInboundAcks(qch chan struct{}) {\n\t// Grab the server lock to watch for server quit.\n\to.mu.RLock()\n\ts, mset := o.srv, o.mset\n\thasInactiveThresh := o.cfg.InactiveThreshold > 0\n\n\to.mu.RUnlock()\n\n\tif s == nil || mset == nil {\n\t\treturn\n\t}\n\n\t// We will check this on entry and periodically.\n\to.checkAckFloor()\n\n\t// How often we will check for ack floor drift.\n\t// Spread these out for large numbers on a server restart.\n\tdelta := time.Duration(rand.Int63n(int64(time.Minute)))\n\tticker := time.NewTicker(time.Minute + delta)\n\tdefer ticker.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-o.ackMsgs.ch:\n\t\t\t// If we have an inactiveThreshold set, mark our activity.\n\t\t\t// Do this before processing acks, otherwise we might race if there are no pending messages\n\t\t\t// anymore and the inactivity threshold kicks in before we're able to mark activity.\n\t\t\tif hasInactiveThresh {\n\t\t\t\to.suppressDeletion()\n\t\t\t}\n\n\t\t\tacks := o.ackMsgs.pop()\n\t\t\tfor _, ack := range acks {\n\t\t\t\to.processAck(ack.subject, ack.reply, ack.hdr, ack.msg)\n\t\t\t\tack.returnToPool()\n\t\t\t}\n\t\t\to.ackMsgs.recycle(&acks)\n\t\tcase <-ticker.C:\n\t\t\to.checkAckFloor()\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Process inbound next message requests.\nfunc (o *consumer) processInboundNextMsgReqs(qch chan struct{}) {\n\t// Grab the server lock to watch for server quit.\n\to.mu.RLock()\n\ts := o.srv\n\to.mu.RUnlock()\n\n\tfor {\n\t\tselect {\n\t\tcase <-o.nextMsgReqs.ch:\n\t\t\treqs := o.nextMsgReqs.pop()\n\t\t\tfor _, req := range reqs {\n\t\t\t\to.processNextMsgRequest(req.reply, req.msg)\n\t\t\t\treq.returnToPool()\n\t\t\t}\n\t\t\to.nextMsgReqs.recycle(&reqs)\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Suppress auto cleanup on ack activity of any kind.\nfunc (o *consumer) suppressDeletion() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.closed {\n\t\treturn\n\t}\n\n\tif o.isPushMode() && o.dtmr != nil {\n\t\t// if dtmr is not nil we have started the countdown, simply reset to threshold.\n\t\to.dtmr.Reset(o.dthresh)\n\t} else if o.isPullMode() && o.waiting != nil {\n\t\t// Pull mode always has timer running, update last on waiting queue.\n\t\to.waiting.last = time.Now()\n\t\tif o.dtmr != nil {\n\t\t\to.dtmr.Reset(o.dthresh)\n\t\t}\n\t}\n}\n\n// loopAndGatherMsgs waits for messages for the consumer. qch is the quit channel,\n// upch is the unpause channel which fires when the PauseUntil deadline is reached.\nfunc (o *consumer) loopAndGatherMsgs(qch chan struct{}) {\n\t// On startup check to see if we are in a reply situation where replay policy is not instant.\n\tvar (\n\t\tlts int64 // last time stamp seen, used for replay.\n\t\tlseq uint64\n\t)\n\n\to.mu.RLock()\n\tmset := o.mset\n\tgetLSeq := o.replay\n\to.mu.RUnlock()\n\t// consumer is closed when mset is set to nil.\n\tif mset == nil {\n\t\treturn\n\t}\n\tif getLSeq {\n\t\tlseq = mset.state().LastSeq\n\t}\n\n\to.mu.Lock()\n\ts := o.srv\n\t// need to check again if consumer is closed\n\tif o.mset == nil {\n\t\to.mu.Unlock()\n\t\treturn\n\t}\n\t// For idle heartbeat support.\n\tvar hbc <-chan time.Time\n\thbd, hb := o.hbTimer()\n\tif hb != nil {\n\t\thbc = hb.C\n\t}\n\t// Interest changes.\n\tinch := o.inch\n\to.mu.Unlock()\n\n\t// Grab the stream's retention policy and name\n\tmset.cfgMu.RLock()\n\tstream, rp := mset.cfg.Name, mset.cfg.Retention\n\tmset.cfgMu.RUnlock()\n\n\tvar err error\n\n\t// Deliver all the msgs we have now, once done or on a condition, we wait for new ones.\n\tfor {\n\t\tvar (\n\t\t\tpmsg *jsPubMsg\n\t\t\tdc uint64\n\t\t\tdsubj string\n\t\t\tackReply string\n\t\t\tdelay time.Duration\n\t\t\tsz int\n\t\t\twrn, wrb int\n\t\t)\n\n\t\to.mu.Lock()\n\n\t\t// consumer is closed when mset is set to nil.\n\t\tif o.closed || o.mset == nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Clear last error.\n\t\terr = nil\n\n\t\t// If the consumer is paused then stop sending.\n\t\tif o.cfg.PauseUntil != nil && !o.cfg.PauseUntil.IsZero() && time.Now().Before(*o.cfg.PauseUntil) {\n\t\t\t// If the consumer is paused and we haven't reached the deadline yet then\n\t\t\t// go back to waiting.\n\t\t\tgoto waitForMsgs\n\t\t}\n\n\t\t// If we are in push mode and not active or under flowcontrol let's stop sending.\n\t\tif o.isPushMode() {\n\t\t\tif !o.active || (o.maxpb > 0 && o.pbytes > o.maxpb) {\n\t\t\t\tgoto waitForMsgs\n\t\t\t}\n\t\t} else if o.waiting.isEmpty() {\n\t\t\t// If we are in pull mode and no one is waiting already break and wait.\n\t\t\tgoto waitForMsgs\n\t\t}\n\n\t\t// Grab our next msg.\n\t\tpmsg, dc, err = o.getNextMsg()\n\n\t\t// We can release the lock now under getNextMsg so need to check this condition again here.\n\t\tif o.closed || o.mset == nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// On error either wait or return.\n\t\tif err != nil || pmsg == nil {\n\t\t\t// On EOF we can optionally fast sync num pending state.\n\t\t\tif err == ErrStoreEOF {\n\t\t\t\to.checkNumPendingOnEOF()\n\t\t\t}\n\t\t\tif err == ErrStoreMsgNotFound || err == errDeletedMsg || err == ErrStoreEOF || err == errMaxAckPending {\n\t\t\t\tgoto waitForMsgs\n\t\t\t} else {\n\t\t\t\tif pmsg != nil {\n\t\t\t\t\ts.Errorf(\"Received an error looking up message with sequence %d for consumer '%s > %s > %s': %v\",\n\t\t\t\t\t\tpmsg.seq, o.mset.acc, stream, o.cfg.Name, err)\n\t\t\t\t} else {\n\t\t\t\t\ts.Errorf(\"Received an error looking up message for consumer '%s > %s > %s': %v\",\n\t\t\t\t\t\to.mset.acc, stream, o.cfg.Name, err)\n\t\t\t\t}\n\t\t\t\tgoto waitForMsgs\n\t\t\t}\n\t\t}\n\n\t\t// Update our cached num pending here first.\n\t\tif dc == 1 {\n\t\t\to.npc--\n\t\t}\n\t\t// Pre-calculate ackReply\n\t\tackReply = o.ackReply(pmsg.seq, o.dseq, dc, pmsg.ts, o.numPending())\n\n\t\t// If headers only do not send msg payload.\n\t\t// Add in msg size itself as header.\n\t\tif o.cfg.HeadersOnly {\n\t\t\tconvertToHeadersOnly(pmsg)\n\t\t}\n\t\t// Calculate payload size. This can be calculated on client side.\n\t\t// We do not include transport subject here since not generally known on client.\n\t\tsz = len(pmsg.subj) + len(ackReply) + len(pmsg.hdr) + len(pmsg.msg)\n\n\t\tif o.isPushMode() {\n\t\t\tdsubj = o.dsubj\n\t\t} else if wr := o.nextWaiting(sz); wr != nil {\n\t\t\twrn, wrb = wr.n, wr.b\n\t\t\tdsubj = wr.reply\n\t\t\tif o.cfg.PriorityPolicy == PriorityPinnedClient {\n\t\t\t\t// FIXME(jrm): Can we make this prettier?\n\t\t\t\tif len(pmsg.hdr) == 0 {\n\t\t\t\t\tpmsg.hdr = genHeader(pmsg.hdr, JSPullRequestNatsPinId, o.currentPinId)\n\t\t\t\t\tpmsg.buf = append(pmsg.hdr, pmsg.msg...)\n\t\t\t\t} else {\n\t\t\t\t\tpmsg.hdr = genHeader(pmsg.hdr, JSPullRequestNatsPinId, o.currentPinId)\n\t\t\t\t\tbufLen := len(pmsg.hdr) + len(pmsg.msg)\n\t\t\t\t\tpmsg.buf = make([]byte, bufLen)\n\t\t\t\t\tpmsg.buf = append(pmsg.hdr, pmsg.msg...)\n\t\t\t\t}\n\n\t\t\t\tsz = len(pmsg.subj) + len(ackReply) + len(pmsg.hdr) + len(pmsg.msg)\n\n\t\t\t}\n\t\t\tif done := wr.recycleIfDone(); done && o.node != nil {\n\t\t\t\to.removeClusterPendingRequest(dsubj)\n\t\t\t} else if !done && wr.hb > 0 {\n\t\t\t\twr.hbt = time.Now().Add(wr.hb)\n\t\t\t}\n\t\t} else {\n\t\t\t// We will redo this one as long as this is not a redelivery.\n\t\t\t// Need to also test that this is not going backwards since if\n\t\t\t// we fail to deliver we can end up here from rdq but we do not\n\t\t\t// want to decrement o.sseq if that is the case.\n\t\t\tif dc == 1 && pmsg.seq == o.sseq-1 {\n\t\t\t\to.sseq--\n\t\t\t\to.npc++\n\t\t\t} else if !o.onRedeliverQueue(pmsg.seq) {\n\t\t\t\t// We are not on the rdq so decrement the delivery count\n\t\t\t\t// and add it back.\n\t\t\t\to.decDeliveryCount(pmsg.seq)\n\t\t\t\to.addToRedeliverQueue(pmsg.seq)\n\t\t\t}\n\t\t\tpmsg.returnToPool()\n\t\t\tpmsg = nil\n\t\t\tgoto waitForMsgs\n\t\t}\n\n\t\t// If we are in a replay scenario and have not caught up check if we need to delay here.\n\t\tif o.replay && lts > 0 {\n\t\t\tif delay = time.Duration(pmsg.ts - lts); delay > time.Millisecond {\n\t\t\t\to.mu.Unlock()\n\t\t\t\tselect {\n\t\t\t\tcase <-qch:\n\t\t\t\t\tpmsg.returnToPool()\n\t\t\t\t\tpmsg = nil\n\t\t\t\t\treturn\n\t\t\t\tcase <-time.After(delay):\n\t\t\t\t}\n\t\t\t\to.mu.Lock()\n\t\t\t}\n\t\t}\n\n\t\t// Track this regardless.\n\t\tlts = pmsg.ts\n\n\t\t// If we have a rate limit set make sure we check that here.\n\t\tif o.rlimit != nil {\n\t\t\tnow := time.Now()\n\t\t\tr := o.rlimit.ReserveN(now, sz)\n\t\t\tdelay := r.DelayFrom(now)\n\t\t\tif delay > 0 {\n\t\t\t\to.mu.Unlock()\n\t\t\t\tselect {\n\t\t\t\tcase <-qch:\n\t\t\t\t\tpmsg.returnToPool()\n\t\t\t\t\tpmsg = nil\n\t\t\t\t\treturn\n\t\t\t\tcase <-time.After(delay):\n\t\t\t\t}\n\t\t\t\to.mu.Lock()\n\t\t\t}\n\t\t}\n\n\t\t// Do actual delivery.\n\t\to.deliverMsg(dsubj, ackReply, pmsg, dc, rp)\n\n\t\t// If given request fulfilled batch size, but there are still pending bytes, send information about it.\n\t\tif wrn <= 0 && wrb > 0 {\n\t\t\tmsg := fmt.Appendf(nil, JsPullRequestRemainingBytesT, JSPullRequestPendingMsgs, wrn, JSPullRequestPendingBytes, wrb)\n\t\t\to.outq.send(newJSPubMsg(dsubj, _EMPTY_, _EMPTY_, msg, nil, nil, 0))\n\t\t}\n\t\t// Reset our idle heartbeat timer if set.\n\t\tif hb != nil {\n\t\t\thb.Reset(hbd)\n\t\t}\n\n\t\to.mu.Unlock()\n\t\tcontinue\n\n\twaitForMsgs:\n\t\t// If we were in a replay state check to see if we are caught up. If so clear.\n\t\tif o.replay && o.sseq > lseq {\n\t\t\to.replay = false\n\t\t}\n\n\t\t// Make sure to process any expired requests that are pending.\n\t\tvar wrExp <-chan time.Time\n\t\tif o.isPullMode() {\n\t\t\t// Dont expire oneshots if we are here because of max ack pending limit.\n\t\t\t_, _, _, fexp := o.processWaiting(err != errMaxAckPending)\n\t\t\tif !fexp.IsZero() {\n\t\t\t\texpires := time.Until(fexp)\n\t\t\t\tif expires <= 0 {\n\t\t\t\t\texpires = time.Millisecond\n\t\t\t\t}\n\t\t\t\twrExp = time.NewTimer(expires).C\n\t\t\t}\n\t\t}\n\n\t\t// We will wait here for new messages to arrive.\n\t\tmch, odsubj := o.mch, o.cfg.DeliverSubject\n\t\to.mu.Unlock()\n\n\t\tselect {\n\t\tcase <-mch:\n\t\t\t// Messages are waiting.\n\t\tcase interest := <-inch:\n\t\t\t// inch can be nil on pull-based, but then this will\n\t\t\t// just block and not fire.\n\t\t\to.updateDeliveryInterest(interest)\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-wrExp:\n\t\t\to.mu.Lock()\n\t\t\to.processWaiting(true)\n\t\t\to.mu.Unlock()\n\t\tcase <-hbc:\n\t\t\tif o.isActive() {\n\t\t\t\to.mu.RLock()\n\t\t\t\to.sendIdleHeartbeat(odsubj)\n\t\t\t\to.mu.RUnlock()\n\t\t\t}\n\t\t\t// Reset our idle heartbeat timer.\n\t\t\thb.Reset(hbd)\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) sendIdleHeartbeat(subj string) {\n\tconst t = \"NATS/1.0 100 Idle Heartbeat\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\"\n\tsseq, dseq := o.sseq-1, o.dseq-1\n\thdr := fmt.Appendf(nil, t, JSLastConsumerSeq, dseq, JSLastStreamSeq, sseq)\n\tif fcp := o.fcid; fcp != _EMPTY_ {\n\t\t// Add in that we are stalled on flow control here.\n\t\taddOn := fmt.Appendf(nil, \"%s: %s\\r\\n\\r\\n\", JSConsumerStalled, fcp)\n\t\thdr = append(hdr[:len(hdr)-LEN_CR_LF], []byte(addOn)...)\n\t}\n\to.outq.send(newJSPubMsg(subj, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n}\n\nfunc (o *consumer) ackReply(sseq, dseq, dc uint64, ts int64, pending uint64) string {\n\treturn fmt.Sprintf(o.ackReplyT, dc, sseq, dseq, ts, pending)\n}\n\n// Used mostly for testing. Sets max pending bytes for flow control setups.\nfunc (o *consumer) setMaxPendingBytes(limit int) {\n\to.pblimit = limit\n\to.maxpb = limit / 16\n\tif o.maxpb == 0 {\n\t\to.maxpb = 1\n\t}\n}\n\n// Does some sanity checks to see if we should re-calculate.\n// Since there is a race when decrementing when there is contention at the beginning of the stream.\n// The race is a getNextMsg skips a deleted msg, and then the decStreamPending call fires.\n// This does some quick sanity checks to see if we should re-calculate num pending.\n// Lock should be held.\nfunc (o *consumer) checkNumPending() (uint64, error) {\n\tif o.mset != nil && o.mset.store != nil {\n\t\tvar state StreamState\n\t\to.mset.store.FastState(&state)\n\t\tnpc := o.numPending()\n\t\t// Make sure we can't report more messages than there are.\n\t\t// TODO(nat): It's not great that this means consumer info has side effects,\n\t\t// since we can't know whether anyone will call it or not. The previous num\n\t\t// pending calculation that this replaces had the same problem though.\n\t\tif o.sseq > state.LastSeq {\n\t\t\to.npc = 0\n\t\t} else if npc > 0 {\n\t\t\to.npc = int64(min(npc, state.Msgs, state.LastSeq-o.sseq+1))\n\t\t}\n\t}\n\treturn o.numPending(), nil\n}\n\n// Lock should be held.\nfunc (o *consumer) numPending() uint64 {\n\tif o.npc < 0 {\n\t\treturn 0\n\t}\n\treturn uint64(o.npc)\n}\n\n// This will do a quick sanity check on num pending when we encounter\n// and EOF in the loop and gather.\n// Lock should be held.\nfunc (o *consumer) checkNumPendingOnEOF() {\n\tif o.mset == nil {\n\t\treturn\n\t}\n\tvar state StreamState\n\to.mset.store.FastState(&state)\n\tif o.sseq > state.LastSeq && o.npc != 0 {\n\t\t// We know here we can reset our running state for num pending.\n\t\to.npc, o.npf = 0, state.LastSeq\n\t}\n}\n\n// Call into streamNumPending after acquiring the consumer lock.\nfunc (o *consumer) streamNumPendingLocked() (uint64, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.streamNumPending()\n}\n\n// Will force a set from the stream store of num pending.\n// Depends on delivery policy, for last per subject we calculate differently.\n// Lock should be held.\nfunc (o *consumer) streamNumPending() (uint64, error) {\n\tif o.mset == nil || o.mset.store == nil {\n\t\to.npc, o.npf = 0, 0\n\t\treturn 0, nil\n\t}\n\tnpc, npf, err := o.calculateNumPending()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\to.npc, o.npf = int64(npc), npf\n\treturn o.numPending(), nil\n}\n\n// Will calculate num pending but only requires a read lock.\n// Depends on delivery policy, for last per subject we calculate differently.\n// At least RLock should be held.\nfunc (o *consumer) calculateNumPending() (npc, npf uint64, err error) {\n\tif o.mset == nil || o.mset.store == nil {\n\t\treturn 0, 0, nil\n\t}\n\n\tisLastPerSubject := o.cfg.DeliverPolicy == DeliverLastPerSubject\n\tfilters, subjf := o.filters, o.subjf\n\n\tif filters != nil {\n\t\treturn o.mset.store.NumPendingMulti(o.sseq, filters, isLastPerSubject)\n\t} else if len(subjf) > 0 {\n\t\tfilter := subjf[0].subject\n\t\treturn o.mset.store.NumPending(o.sseq, filter, isLastPerSubject)\n\t}\n\treturn o.mset.store.NumPending(o.sseq, _EMPTY_, isLastPerSubject)\n}\n\nfunc convertToHeadersOnly(pmsg *jsPubMsg) {\n\t// If headers only do not send msg payload.\n\t// Add in msg size itself as header.\n\thdr, msg := pmsg.hdr, pmsg.msg\n\tvar bb bytes.Buffer\n\tif len(hdr) == 0 {\n\t\tbb.WriteString(hdrLine)\n\t} else {\n\t\tbb.Write(hdr)\n\t\tbb.Truncate(len(hdr) - LEN_CR_LF)\n\t}\n\tbb.WriteString(JSMsgSize)\n\tbb.WriteString(\": \")\n\tbb.WriteString(strconv.FormatInt(int64(len(msg)), 10))\n\tbb.WriteString(CR_LF)\n\tbb.WriteString(CR_LF)\n\t// Replace underlying buf which we can use directly when we send.\n\t// TODO(dlc) - Probably just use directly when forming bytes.Buffer?\n\tpmsg.buf = pmsg.buf[:0]\n\tpmsg.buf = append(pmsg.buf, bb.Bytes()...)\n\t// Replace with new header.\n\tpmsg.hdr = pmsg.buf\n\t// Cancel msg payload\n\tpmsg.msg = nil\n}\n\n// Deliver a msg to the consumer.\n// Lock should be held and o.mset validated to be non-nil.\nfunc (o *consumer) deliverMsg(dsubj, ackReply string, pmsg *jsPubMsg, dc uint64, rp RetentionPolicy) {\n\tif o.mset == nil {\n\t\tpmsg.returnToPool()\n\t\treturn\n\t}\n\n\tdseq := o.dseq\n\to.dseq++\n\n\tpmsg.dsubj, pmsg.reply, pmsg.o = dsubj, ackReply, o\n\tpsz := pmsg.size()\n\n\tif o.maxpb > 0 {\n\t\to.pbytes += psz\n\t}\n\n\tmset := o.mset\n\tap := o.cfg.AckPolicy\n\n\t// Cant touch pmsg after this sending so capture what we need.\n\tseq, ts := pmsg.seq, pmsg.ts\n\n\t// Update delivered first.\n\to.updateDelivered(dseq, seq, dc, ts)\n\n\tif ap == AckExplicit || ap == AckAll {\n\t\to.trackPending(seq, dseq)\n\t} else if ap == AckNone {\n\t\to.adflr = dseq\n\t\to.asflr = seq\n\t}\n\n\t// Send message.\n\tif o.replicateDeliveries() {\n\t\to.addReplicatedQueuedMsg(pmsg)\n\t} else {\n\t\to.outq.send(pmsg)\n\t}\n\n\t// Flow control.\n\tif o.maxpb > 0 && o.needFlowControl(psz) {\n\t\to.sendFlowControl()\n\t}\n\n\t// If pull mode and we have inactivity threshold, signaled by dthresh, update last activity.\n\tif o.isPullMode() && o.dthresh > 0 {\n\t\to.waiting.last = time.Now()\n\t}\n\n\t// If we are ack none and mset is interest only we should make sure stream removes interest.\n\tif ap == AckNone && rp != LimitsPolicy {\n\t\tif mset != nil && mset.ackq != nil && (o.node == nil || o.cfg.Direct) {\n\t\t\tmset.ackq.push(seq)\n\t\t} else {\n\t\t\to.updateAcks(dseq, seq, _EMPTY_)\n\t\t}\n\t}\n}\n\n// replicateDeliveries returns whether deliveries should be replicated before sending them.\n// If we're replicated we MUST only send the message AFTER we've got quorum for updating\n// delivered state. Otherwise, we could be in an invalid state after a leader change.\n// We can send immediately if not replicated, not using acks, or using flow control (incompatible).\n// Lock should be held.\nfunc (o *consumer) replicateDeliveries() bool {\n\treturn o.node != nil && o.cfg.AckPolicy != AckNone && !o.cfg.FlowControl\n}\n\nfunc (o *consumer) needFlowControl(sz int) bool {\n\tif o.maxpb == 0 {\n\t\treturn false\n\t}\n\t// Decide whether to send a flow control message which we will need the user to respond.\n\t// We send when we are over 50% of our current window limit.\n\tif o.fcid == _EMPTY_ && o.pbytes > o.maxpb/2 {\n\t\treturn true\n\t}\n\t// If we have an existing outstanding FC, check to see if we need to expand the o.fcsz\n\tif o.fcid != _EMPTY_ && (o.pbytes-o.fcsz) >= o.maxpb {\n\t\to.fcsz += sz\n\t}\n\treturn false\n}\n\nfunc (o *consumer) processFlowControl(_ *subscription, c *client, _ *Account, subj, _ string, _ []byte) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Ignore if not the latest we have sent out.\n\tif subj != o.fcid {\n\t\treturn\n\t}\n\n\t// For slow starts and ramping up.\n\tif o.maxpb < o.pblimit {\n\t\to.maxpb *= 2\n\t\tif o.maxpb > o.pblimit {\n\t\t\to.maxpb = o.pblimit\n\t\t}\n\t}\n\n\t// Update accounting.\n\to.pbytes -= o.fcsz\n\tif o.pbytes < 0 {\n\t\to.pbytes = 0\n\t}\n\to.fcid, o.fcsz = _EMPTY_, 0\n\n\to.signalNewMessages()\n}\n\n// Lock should be held.\nfunc (o *consumer) fcReply() string {\n\tvar sb strings.Builder\n\tsb.WriteString(jsFlowControlPre)\n\tsb.WriteString(o.stream)\n\tsb.WriteByte(btsep)\n\tsb.WriteString(o.name)\n\tsb.WriteByte(btsep)\n\tvar b [4]byte\n\trn := rand.Int63()\n\tfor i, l := 0, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\tsb.Write(b[:])\n\treturn sb.String()\n}\n\n// sendFlowControl will send a flow control packet to the consumer.\n// Lock should be held.\nfunc (o *consumer) sendFlowControl() {\n\tif !o.isPushMode() {\n\t\treturn\n\t}\n\tsubj, rply := o.cfg.DeliverSubject, o.fcReply()\n\to.fcsz, o.fcid = o.pbytes, rply\n\thdr := []byte(\"NATS/1.0 100 FlowControl Request\\r\\n\\r\\n\")\n\to.outq.send(newJSPubMsg(subj, _EMPTY_, rply, hdr, nil, nil, 0))\n}\n\n// Tracks our outstanding pending acks. Only applicable to AckExplicit mode.\n// Lock should be held.\nfunc (o *consumer) trackPending(sseq, dseq uint64) {\n\tif o.pending == nil {\n\t\to.pending = make(map[uint64]*Pending)\n\t}\n\n\tnow := time.Now()\n\tif p, ok := o.pending[sseq]; ok {\n\t\t// Update timestamp but keep original consumer delivery sequence.\n\t\t// So do not update p.Sequence.\n\t\tp.Timestamp = now.UnixNano()\n\t} else {\n\t\to.pending[sseq] = &Pending{dseq, now.UnixNano()}\n\t}\n\n\t// We could have a backoff that set a timer higher than what we need for this message.\n\t// In that case, reset to lowest backoff required for a message redelivery.\n\tminDelay := o.ackWait(0)\n\tif l := len(o.cfg.BackOff); l > 0 {\n\t\tbi := int(o.rdc[sseq])\n\t\tif bi < 0 {\n\t\t\tbi = 0\n\t\t} else if bi >= l {\n\t\t\tbi = l - 1\n\t\t}\n\t\tminDelay = o.ackWait(o.cfg.BackOff[bi])\n\t}\n\tminDeadline := now.Add(minDelay)\n\tif o.ptmr == nil || o.ptmrEnd.After(minDeadline) {\n\t\to.resetPtmr(minDelay)\n\t}\n}\n\n// Credit back a failed delivery.\n// lock should be held.\nfunc (o *consumer) creditWaitingRequest(reply string) {\n\twq := o.waiting\n\tfor wr := wq.head; wr != nil; wr = wr.next {\n\t\tif wr.reply == reply {\n\t\t\twr.n++\n\t\t\twr.d--\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// didNotDeliver is called when a delivery for a consumer message failed.\n// Depending on our state, we will process the failure.\nfunc (o *consumer) didNotDeliver(seq uint64, subj string) {\n\to.mu.Lock()\n\tmset := o.mset\n\tif mset == nil {\n\t\to.mu.Unlock()\n\t\treturn\n\t}\n\t// Adjust back deliver count.\n\to.decDeliveryCount(seq)\n\n\tvar checkDeliveryInterest bool\n\tif o.isPushMode() {\n\t\to.active = false\n\t\tcheckDeliveryInterest = true\n\t} else if o.pending != nil {\n\t\t// Good chance we did not deliver because no interest so force a check.\n\t\to.processWaiting(false)\n\t\t// If it is still there credit it back.\n\t\to.creditWaitingRequest(subj)\n\t\t// pull mode and we have pending.\n\t\tif _, ok := o.pending[seq]; ok {\n\t\t\t// We found this messsage on pending, we need\n\t\t\t// to queue it up for immediate redelivery since\n\t\t\t// we know it was not delivered\n\t\t\tif !o.onRedeliverQueue(seq) {\n\t\t\t\to.addToRedeliverQueue(seq)\n\t\t\t\tif !o.waiting.isEmpty() {\n\t\t\t\t\to.signalNewMessages()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\to.mu.Unlock()\n\n\tif checkDeliveryInterest {\n\t\tlocalInterest := !o.hasNoLocalInterest()\n\t\to.updateDeliveryInterest(localInterest)\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) addToRedeliverQueue(seqs ...uint64) {\n\to.rdq = append(o.rdq, seqs...)\n\tfor _, seq := range seqs {\n\t\to.rdqi.Insert(seq)\n\t}\n}\n\n// Lock should be held.\nfunc (o *consumer) hasRedeliveries() bool {\n\treturn len(o.rdq) > 0\n}\n\nfunc (o *consumer) getNextToRedeliver() uint64 {\n\tif len(o.rdq) == 0 {\n\t\treturn 0\n\t}\n\tseq := o.rdq[0]\n\tif len(o.rdq) == 1 {\n\t\to.rdq = nil\n\t\to.rdqi.Empty()\n\t} else {\n\t\to.rdq = append(o.rdq[:0], o.rdq[1:]...)\n\t\to.rdqi.Delete(seq)\n\t}\n\treturn seq\n}\n\n// This checks if we already have this sequence queued for redelivery.\n// FIXME(dlc) - This is O(n) but should be fast with small redeliver size.\n// Lock should be held.\nfunc (o *consumer) onRedeliverQueue(seq uint64) bool {\n\treturn o.rdqi.Exists(seq)\n}\n\n// Remove a sequence from the redelivery queue.\n// Lock should be held.\nfunc (o *consumer) removeFromRedeliverQueue(seq uint64) bool {\n\tif !o.onRedeliverQueue(seq) {\n\t\treturn false\n\t}\n\tfor i, rseq := range o.rdq {\n\t\tif rseq == seq {\n\t\t\tif len(o.rdq) == 1 {\n\t\t\t\to.rdq = nil\n\t\t\t\to.rdqi.Empty()\n\t\t\t} else {\n\t\t\t\to.rdq = append(o.rdq[:i], o.rdq[i+1:]...)\n\t\t\t\to.rdqi.Delete(seq)\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Checks the pending messages.\nfunc (o *consumer) checkPending() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tmset := o.mset\n\t// On stop, mset and timer will be nil.\n\tif o.closed || mset == nil || o.ptmr == nil {\n\t\to.stopAndClearPtmr()\n\t\treturn\n\t}\n\n\tvar shouldUpdateState bool\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\tfseq := state.FirstSeq\n\n\tnow := time.Now().UnixNano()\n\tttl := int64(o.cfg.AckWait)\n\tnext := int64(o.ackWait(0))\n\t// However, if there is backoff, initializes with the largest backoff.\n\t// It will be adjusted as needed.\n\tif l := len(o.cfg.BackOff); l > 0 {\n\t\tnext = int64(o.cfg.BackOff[l-1])\n\t}\n\n\t// Since we can update timestamps, we have to review all pending.\n\t// We will now bail if we see an ack pending inbound to us via o.awl.\n\tvar expired []uint64\n\tcheck := len(o.pending) > 1024\n\tfor seq, p := range o.pending {\n\t\tif check && atomic.LoadInt64(&o.awl) > 0 {\n\t\t\to.resetPtmr(100 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\t// Check if these are no longer valid.\n\t\tif seq < fseq || seq <= o.asflr {\n\t\t\tdelete(o.pending, seq)\n\t\t\tdelete(o.rdc, seq)\n\t\t\to.removeFromRedeliverQueue(seq)\n\t\t\tshouldUpdateState = true\n\t\t\t// Check if we need to move ack floors.\n\t\t\tif seq > o.asflr {\n\t\t\t\to.asflr = seq\n\t\t\t}\n\t\t\tif p.Sequence > o.adflr {\n\t\t\t\to.adflr = p.Sequence\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\telapsed, deadline := now-p.Timestamp, ttl\n\t\tif len(o.cfg.BackOff) > 0 {\n\t\t\t// This is ok even if o.rdc is nil, we would get dc == 0, which is what we want.\n\t\t\tdc := int(o.rdc[seq])\n\t\t\tif dc < 0 {\n\t\t\t\t// Prevent consumer backoff from going backwards.\n\t\t\t\tdc = 0\n\t\t\t}\n\t\t\t// This will be the index for the next backoff, will set to last element if needed.\n\t\t\tnbi := dc + 1\n\t\t\tif dc+1 >= len(o.cfg.BackOff) {\n\t\t\t\tdc = len(o.cfg.BackOff) - 1\n\t\t\t\tnbi = dc\n\t\t\t}\n\t\t\tdeadline = int64(o.cfg.BackOff[dc])\n\t\t\t// Set `next` to the next backoff (if smaller than current `next` value).\n\t\t\tif nextBackoff := int64(o.cfg.BackOff[nbi]); nextBackoff < next {\n\t\t\t\tnext = nextBackoff\n\t\t\t}\n\t\t}\n\t\tif elapsed >= deadline {\n\t\t\t// We will check if we have hit our max deliveries. Previously we would do this on getNextMsg() which\n\t\t\t// worked well for push consumers, but with pull based consumers would require a new pull request to be\n\t\t\t// present to process and redelivered could be reported incorrectly.\n\t\t\tif !o.onRedeliverQueue(seq) && !o.hasMaxDeliveries(seq) {\n\t\t\t\texpired = append(expired, seq)\n\t\t\t}\n\t\t} else if deadline-elapsed < next {\n\t\t\t// Update when we should fire next.\n\t\t\tnext = deadline - elapsed\n\t\t}\n\t}\n\n\tif len(expired) > 0 {\n\t\t// We need to sort.\n\t\tslices.Sort(expired)\n\t\to.addToRedeliverQueue(expired...)\n\t\t// Now we should update the timestamp here since we are redelivering.\n\t\t// We will use an incrementing time to preserve order for any other redelivery.\n\t\toff := now - o.pending[expired[0]].Timestamp\n\t\tfor _, seq := range expired {\n\t\t\tif p, ok := o.pending[seq]; ok {\n\t\t\t\tp.Timestamp += off\n\t\t\t}\n\t\t}\n\t\to.signalNewMessages()\n\t}\n\n\tif len(o.pending) > 0 {\n\t\to.resetPtmr(time.Duration(next))\n\t} else {\n\t\t// Make sure to stop timer and clear out any re delivery queues\n\t\to.stopAndClearPtmr()\n\t\to.rdq = nil\n\t\to.rdqi.Empty()\n\t\to.pending = nil\n\t\t// Mimic behavior in processAckMsg when pending is empty.\n\t\to.adflr, o.asflr = o.dseq-1, o.sseq-1\n\t}\n\n\t// Update our state if needed.\n\tif shouldUpdateState {\n\t\tif err := o.writeStoreStateUnlocked(); err != nil && o.srv != nil && o.mset != nil && !o.closed {\n\t\t\ts, acc, mset, name := o.srv, o.acc, o.mset, o.name\n\t\t\ts.Warnf(\"Consumer '%s > %s > %s' error on write store state from check pending: %v\", acc, mset.getCfgName(), name, err)\n\t\t}\n\t}\n}\n\n// SeqFromReply will extract a sequence number from a reply subject.\nfunc (o *consumer) seqFromReply(reply string) uint64 {\n\t_, dseq, _ := ackReplyInfo(reply)\n\treturn dseq\n}\n\n// StreamSeqFromReply will extract the stream sequence from the reply subject.\nfunc (o *consumer) streamSeqFromReply(reply string) uint64 {\n\tsseq, _, _ := ackReplyInfo(reply)\n\treturn sseq\n}\n\n// Quick parser for positive numbers in ack reply encoding.\nfunc parseAckReplyNum(d string) (n int64) {\n\tif len(d) == 0 {\n\t\treturn -1\n\t}\n\tfor _, dec := range d {\n\t\tif dec < asciiZero || dec > asciiNine {\n\t\t\treturn -1\n\t\t}\n\t\tn = n*10 + (int64(dec) - asciiZero)\n\t}\n\treturn n\n}\n\nconst expectedNumReplyTokens = 9\n\n// Grab encoded information in the reply subject for a delivered message.\nfunc replyInfo(subject string) (sseq, dseq, dc uint64, ts int64, pending uint64) {\n\ttsa := [expectedNumReplyTokens]string{}\n\tstart, tokens := 0, tsa[:0]\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttokens = append(tokens, subject[start:])\n\tif len(tokens) != expectedNumReplyTokens || tokens[0] != \"$JS\" || tokens[1] != \"ACK\" {\n\t\treturn 0, 0, 0, 0, 0\n\t}\n\t// TODO(dlc) - Should we error if we do not match consumer name?\n\t// stream is tokens[2], consumer is 3.\n\tdc = uint64(parseAckReplyNum(tokens[4]))\n\tsseq, dseq = uint64(parseAckReplyNum(tokens[5])), uint64(parseAckReplyNum(tokens[6]))\n\tts = parseAckReplyNum(tokens[7])\n\tpending = uint64(parseAckReplyNum(tokens[8]))\n\n\treturn sseq, dseq, dc, ts, pending\n}\n\nfunc ackReplyInfo(subject string) (sseq, dseq, dc uint64) {\n\ttsa := [expectedNumReplyTokens]string{}\n\tstart, tokens := 0, tsa[:0]\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttokens = append(tokens, subject[start:])\n\tif len(tokens) != expectedNumReplyTokens || tokens[0] != \"$JS\" || tokens[1] != \"ACK\" {\n\t\treturn 0, 0, 0\n\t}\n\tdc = uint64(parseAckReplyNum(tokens[4]))\n\tsseq, dseq = uint64(parseAckReplyNum(tokens[5])), uint64(parseAckReplyNum(tokens[6]))\n\n\treturn sseq, dseq, dc\n}\n\n// NextSeq returns the next delivered sequence number for this consumer.\nfunc (o *consumer) nextSeq() uint64 {\n\to.mu.RLock()\n\tdseq := o.dseq\n\to.mu.RUnlock()\n\treturn dseq\n}\n\n// Used to hold skip list when deliver policy is last per subject.\ntype lastSeqSkipList struct {\n\tresume uint64\n\tseqs []uint64\n}\n\n// Let's us know we have a skip list, which is for deliver last per subject and we are just starting.\n// Lock should be held.\nfunc (o *consumer) hasSkipListPending() bool {\n\treturn o.lss != nil && len(o.lss.seqs) > 0\n}\n\n// reconcileStateWithStream reconciles consumer state when the stream has reverted\n// due to data loss (e.g., VM crash). This handles the case where consumer state\n// is ahead of the stream's last sequence.\n// Lock should be held.\nfunc (o *consumer) reconcileStateWithStream(streamLastSeq uint64) {\n\t// If an ack floor is higher than stream last sequence,\n\t// reset back down but keep the highest known sequences.\n\tif o.asflr > streamLastSeq {\n\t\to.asflr = streamLastSeq\n\t\t// Delivery floor is one below the delivered sequence,\n\t\t// but if it is zero somehow, ensure we don't underflow.\n\t\to.adflr = o.dseq\n\t\tif o.adflr > 0 {\n\t\t\to.adflr--\n\t\t}\n\t\to.pending = nil\n\t\to.rdc = nil\n\t}\n\n\t// Remove pending entries that are beyond the stream's last sequence\n\tif len(o.pending) > 0 {\n\t\tfor seq := range o.pending {\n\t\t\tif seq > streamLastSeq {\n\t\t\t\tdelete(o.pending, seq)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Remove redelivered entries that are beyond the stream's last sequence\n\tif len(o.rdc) > 0 {\n\t\tfor seq := range o.rdc {\n\t\t\tif seq > streamLastSeq {\n\t\t\t\tdelete(o.rdc, seq)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Update starting sequence and delivery sequence based on pending state\n\tif len(o.pending) == 0 {\n\t\to.sseq = o.asflr + 1\n\t\to.dseq = o.adflr + 1\n\t} else {\n\t\t// Find highest stream sequence in pending\n\t\tvar maxStreamSeq uint64\n\t\tvar maxConsumerSeq uint64\n\n\t\tfor streamSeq, p := range o.pending {\n\t\t\tif streamSeq > maxStreamSeq {\n\t\t\t\tmaxStreamSeq = streamSeq\n\t\t\t}\n\t\t\tif p.Sequence > maxConsumerSeq {\n\t\t\t\tmaxConsumerSeq = p.Sequence\n\t\t\t}\n\t\t}\n\n\t\t// Set next sequences based on highest pending\n\t\to.sseq = maxStreamSeq + 1\n\t\to.dseq = maxConsumerSeq + 1\n\t}\n}\n\n// Will select the starting sequence.\nfunc (o *consumer) selectStartingSeqNo() error {\n\tif o.mset == nil || o.mset.store == nil {\n\t\to.sseq = 1\n\t} else {\n\t\tvar state StreamState\n\t\to.mset.store.FastState(&state)\n\t\tif o.cfg.OptStartSeq == 0 {\n\t\t\tif o.cfg.DeliverPolicy == DeliverAll {\n\t\t\t\to.sseq = state.FirstSeq\n\t\t\t} else if o.cfg.DeliverPolicy == DeliverLast {\n\t\t\t\tif o.subjf == nil {\n\t\t\t\t\to.sseq = state.LastSeq\n\t\t\t\t} else {\n\t\t\t\t\t// If we are partitioned here this will be properly set when we become leader.\n\t\t\t\t\tfor _, filter := range o.subjf {\n\t\t\t\t\t\tss, err := o.mset.store.FilteredState(1, filter.subject)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif ss.Last > o.sseq {\n\t\t\t\t\t\t\to.sseq = ss.Last\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if o.cfg.DeliverPolicy == DeliverLastPerSubject {\n\t\t\t\t// If our parent stream is set to max msgs per subject of 1 this is just\n\t\t\t\t// a normal consumer at this point. We can avoid any heavy lifting.\n\t\t\t\to.mset.cfgMu.RLock()\n\t\t\t\tmmp := o.mset.cfg.MaxMsgsPer\n\t\t\t\to.mset.cfgMu.RUnlock()\n\t\t\t\tif mmp == 1 {\n\t\t\t\t\to.sseq = state.FirstSeq\n\t\t\t\t} else {\n\t\t\t\t\tfilters := make([]string, 0, len(o.subjf))\n\t\t\t\t\tif o.subjf == nil {\n\t\t\t\t\t\tfilters = append(filters, o.cfg.FilterSubject)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfor _, filter := range o.subjf {\n\t\t\t\t\t\t\tfilters = append(filters, filter.subject)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tlss := &lastSeqSkipList{resume: state.LastSeq}\n\t\t\t\t\tlss.seqs, _ = o.mset.store.MultiLastSeqs(filters, 0, 0)\n\n\t\t\t\t\tif len(lss.seqs) == 0 {\n\t\t\t\t\t\to.sseq = state.LastSeq\n\t\t\t\t\t} else {\n\t\t\t\t\t\to.sseq = lss.seqs[0]\n\t\t\t\t\t}\n\t\t\t\t\t// Assign skip list.\n\t\t\t\t\to.lss = lss\n\t\t\t\t}\n\t\t\t} else if o.cfg.OptStartTime != nil {\n\t\t\t\t// If we are here we are time based.\n\t\t\t\t// TODO(dlc) - Once clustered can't rely on this.\n\t\t\t\to.sseq = o.mset.store.GetSeqFromTime(*o.cfg.OptStartTime)\n\t\t\t\t// Here we want to see if we are filtered, and if so possibly close the gap\n\t\t\t\t// to the nearest first given our starting sequence from time. This is so we do\n\t\t\t\t// not force the system to do a linear walk between o.sseq and the real first.\n\t\t\t\tif len(o.subjf) > 0 {\n\t\t\t\t\tnseq := state.LastSeq\n\t\t\t\t\tfor _, filter := range o.subjf {\n\t\t\t\t\t\t// Use first sequence since this is more optimized atm.\n\t\t\t\t\t\tss, err := o.mset.store.FilteredState(state.FirstSeq, filter.subject)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif ss.First >= o.sseq && ss.First < nseq {\n\t\t\t\t\t\t\tnseq = ss.First\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Skip ahead if possible.\n\t\t\t\t\tif nseq > o.sseq && nseq < state.LastSeq {\n\t\t\t\t\t\to.sseq = nseq\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// DeliverNew\n\t\t\t\to.sseq = state.LastSeq + 1\n\t\t\t}\n\t\t} else {\n\t\t\to.sseq = o.cfg.OptStartSeq\n\t\t}\n\n\t\tif state.FirstSeq == 0 && (o.cfg.Direct || o.cfg.OptStartSeq == 0) {\n\t\t\t// If the stream is empty, deliver only new.\n\t\t\t// But only if mirroring/sourcing, or start seq is unset, otherwise need to respect provided value.\n\t\t\to.sseq = 1\n\t\t} else if o.sseq > state.LastSeq && (o.cfg.Direct || o.cfg.OptStartSeq == 0) {\n\t\t\t// If selected sequence is in the future, clamp back down.\n\t\t\t// But only if mirroring/sourcing, or start seq is unset, otherwise need to respect provided value.\n\t\t\to.sseq = state.LastSeq + 1\n\t\t} else if o.sseq < state.FirstSeq {\n\t\t\t// If the first sequence is further ahead than the starting sequence,\n\t\t\t// there are no messages there anymore, so move the sequence up.\n\t\t\to.sseq = state.FirstSeq\n\t\t}\n\t}\n\n\t// Always set delivery sequence to 1.\n\to.dseq = 1\n\t// Set ack delivery floor to delivery-1\n\to.adflr = o.dseq - 1\n\t// Set ack store floor to store-1\n\to.asflr = o.sseq - 1\n\t// Set our starting sequence state.\n\t// But only if we're not clustered, if clustered we propose upon becoming leader.\n\tif o.store != nil && o.sseq > 0 && o.cfg.replicas(&o.mset.cfg) == 1 {\n\t\tif err := o.store.SetStarting(o.sseq - 1); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Test whether a config represents a durable subscriber.\nfunc isDurableConsumer(config *ConsumerConfig) bool {\n\treturn config != nil && config.Durable != _EMPTY_\n}\n\nfunc (o *consumer) isDurable() bool {\n\treturn o.cfg.Durable != _EMPTY_\n}\n\n// Are we in push mode, delivery subject, etc.\nfunc (o *consumer) isPushMode() bool {\n\treturn o.cfg.DeliverSubject != _EMPTY_\n}\n\nfunc (o *consumer) isPullMode() bool {\n\treturn o.cfg.DeliverSubject == _EMPTY_\n}\n\n// Name returns the name of this consumer.\nfunc (o *consumer) String() string {\n\to.mu.RLock()\n\tn := o.name\n\to.mu.RUnlock()\n\treturn n\n}\n\nfunc createConsumerName() string {\n\treturn getHash(nuid.Next())\n}\n\n// deleteConsumer will delete the consumer from this stream.\nfunc (mset *stream) deleteConsumer(o *consumer) error {\n\treturn o.delete()\n}\n\nfunc (o *consumer) getStream() *stream {\n\to.mu.RLock()\n\tmset := o.mset\n\to.mu.RUnlock()\n\treturn mset\n}\n\nfunc (o *consumer) streamName() string {\n\to.mu.RLock()\n\tmset := o.mset\n\to.mu.RUnlock()\n\tif mset != nil {\n\t\treturn mset.name()\n\t}\n\treturn _EMPTY_\n}\n\n// Active indicates if this consumer is still active.\nfunc (o *consumer) isActive() bool {\n\to.mu.RLock()\n\tactive := o.active && o.mset != nil\n\to.mu.RUnlock()\n\treturn active\n}\n\n// hasNoLocalInterest return true if we have no local interest.\nfunc (o *consumer) hasNoLocalInterest() bool {\n\to.mu.RLock()\n\tinterest := o.acc.sl.HasInterest(o.cfg.DeliverSubject)\n\to.mu.RUnlock()\n\treturn !interest\n}\n\n// This is when the underlying stream has been purged.\n// sseq is the new first seq for the stream after purge.\n// Consumer lock should NOT be held but the parent stream\n// lock MUST be held.\nfunc (o *consumer) purge(sseq uint64, slseq uint64, isWider bool) {\n\t// Do not update our state unless we know we are the leader.\n\tif !o.isLeader() {\n\t\treturn\n\t}\n\t// Signals all have been purged for this consumer.\n\tif sseq == 0 && !isWider {\n\t\tsseq = slseq + 1\n\t}\n\n\tvar store StreamStore\n\tif isWider {\n\t\to.mu.RLock()\n\t\tif o.mset != nil {\n\t\t\tstore = o.mset.store\n\t\t}\n\t\to.mu.RUnlock()\n\t}\n\n\to.mu.Lock()\n\t// Do not go backwards\n\tif o.sseq < sseq {\n\t\to.sseq = sseq\n\t}\n\n\tif o.asflr < sseq {\n\t\to.asflr = sseq - 1\n\t\t// We need to remove those no longer relevant from pending.\n\t\tfor seq, p := range o.pending {\n\t\t\tif seq <= o.asflr {\n\t\t\t\tif p.Sequence > o.adflr {\n\t\t\t\t\to.adflr = p.Sequence\n\t\t\t\t\tif o.adflr > o.dseq {\n\t\t\t\t\t\to.dseq = o.adflr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tdelete(o.pending, seq)\n\t\t\t\tdelete(o.rdc, seq)\n\t\t\t\t// rdq handled below.\n\t\t\t}\n\t\t\tif isWider && store != nil {\n\t\t\t\t// Our filtered subject, which could be all, is wider than the underlying purge.\n\t\t\t\t// We need to check if the pending items left are still valid.\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tif _, err := store.LoadMsg(seq, &smv); err == errDeletedMsg || err == ErrStoreMsgNotFound {\n\t\t\t\t\tif p.Sequence > o.adflr {\n\t\t\t\t\t\to.adflr = p.Sequence\n\t\t\t\t\t\tif o.adflr > o.dseq {\n\t\t\t\t\t\t\to.dseq = o.adflr\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tdelete(o.pending, seq)\n\t\t\t\t\tdelete(o.rdc, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// This means we can reset everything at this point.\n\tif len(o.pending) == 0 {\n\t\to.pending, o.rdc = nil, nil\n\t\to.adflr, o.asflr = o.dseq-1, o.sseq-1\n\t}\n\n\t// We need to remove all those being queued for redelivery under o.rdq\n\tif len(o.rdq) > 0 {\n\t\trdq := o.rdq\n\t\to.rdq = nil\n\t\to.rdqi.Empty()\n\t\tfor _, sseq := range rdq {\n\t\t\tif sseq >= o.sseq {\n\t\t\t\to.addToRedeliverQueue(sseq)\n\t\t\t}\n\t\t}\n\t}\n\t// Grab some info in case of error below.\n\ts, acc, mset, name := o.srv, o.acc, o.mset, o.name\n\to.mu.Unlock()\n\n\tif err := o.writeStoreState(); err != nil && s != nil && mset != nil {\n\t\ts.Warnf(\"Consumer '%s > %s > %s' error on write store state from purge: %v\", acc, mset.nameLocked(false), name, err)\n\t}\n}\n\nfunc stopAndClearTimer(tp **time.Timer) {\n\tif *tp == nil {\n\t\treturn\n\t}\n\t// Will get drained in normal course, do not try to\n\t// drain here.\n\t(*tp).Stop()\n\t*tp = nil\n}\n\n// Stop will shutdown the consumer for the associated stream.\nfunc (o *consumer) stop() error {\n\treturn o.stopWithFlags(false, false, true, false)\n}\n\nfunc (o *consumer) deleteWithoutAdvisory() error {\n\treturn o.stopWithFlags(true, false, true, false)\n}\n\n// Delete will delete the consumer for the associated stream and send advisories.\nfunc (o *consumer) delete() error {\n\treturn o.stopWithFlags(true, false, true, true)\n}\n\n// To test for closed state.\nfunc (o *consumer) isClosed() bool {\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.closed\n}\n\nfunc (o *consumer) stopWithFlags(dflag, sdflag, doSignal, advisory bool) error {\n\t// If dflag is true determine if we are still assigned.\n\tvar isAssigned bool\n\tif dflag {\n\t\to.mu.RLock()\n\t\tacc, stream, consumer := o.acc, o.stream, o.name\n\t\tisClustered := o.js != nil && o.js.isClustered()\n\t\to.mu.RUnlock()\n\t\tif isClustered {\n\t\t\t// Grab jsa to check assignment.\n\t\t\tvar jsa *jsAccount\n\t\t\tif acc != nil {\n\t\t\t\t// Need lock here to avoid data race.\n\t\t\t\tacc.mu.RLock()\n\t\t\t\tjsa = acc.js\n\t\t\t\tacc.mu.RUnlock()\n\t\t\t}\n\t\t\tif jsa != nil {\n\t\t\t\tisAssigned = jsa.consumerAssigned(stream, consumer)\n\t\t\t}\n\t\t}\n\t}\n\n\to.mu.Lock()\n\tif o.closed {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\to.closed = true\n\n\t// Signal to the monitor loop.\n\t// Can't use only qch here, since that's used when stepping down as a leader.\n\tif o.mqch != nil {\n\t\tclose(o.mqch)\n\t\to.mqch = nil\n\t}\n\n\t// Check if we are the leader and are being deleted (as a node).\n\tif dflag && o.isLeader() {\n\t\t// If we are clustered and node leader (probable from above), stepdown.\n\t\tif node := o.node; node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\n\t\t// dflag does not necessarily mean that the consumer is being deleted,\n\t\t// just that the consumer node is being removed from this peer, so we\n\t\t// send delete advisories only if we are no longer assigned at the meta layer,\n\t\t// or we are not clustered.\n\t\tif !isAssigned && advisory {\n\t\t\to.sendDeleteAdvisoryLocked()\n\t\t}\n\t\tif o.isPullMode() {\n\t\t\t// Release any pending.\n\t\t\to.releaseAnyPendingRequests(isAssigned)\n\t\t}\n\t}\n\n\tif o.qch != nil {\n\t\tclose(o.qch)\n\t\to.qch = nil\n\t}\n\n\ta := o.acc\n\tstore := o.store\n\tmset := o.mset\n\to.mset = nil\n\to.active = false\n\to.unsubscribe(o.ackSub)\n\to.unsubscribe(o.reqSub)\n\to.unsubscribe(o.resetSub)\n\to.unsubscribe(o.fcSub)\n\to.ackSub = nil\n\to.reqSub = nil\n\to.resetSub = nil\n\to.fcSub = nil\n\tif o.infoSub != nil {\n\t\to.srv.sysUnsubscribe(o.infoSub)\n\t\to.infoSub = nil\n\t}\n\tc := o.client\n\to.client = nil\n\tsysc := o.sysc\n\to.sysc = nil\n\to.stopAndClearPtmr()\n\tstopAndClearTimer(&o.dtmr)\n\tstopAndClearTimer(&o.gwdtmr)\n\tdelivery := o.cfg.DeliverSubject\n\to.waiting = nil\n\t// Break us out of the readLoop.\n\tif doSignal {\n\t\to.signalNewMessages()\n\t}\n\tn := o.node\n\tqgroup := o.cfg.DeliverGroup\n\to.ackMsgs.unregister()\n\tif o.nextMsgReqs != nil {\n\t\to.nextMsgReqs.unregister()\n\t}\n\n\t// For cleaning up the node assignment.\n\tvar ca *consumerAssignment\n\tif dflag {\n\t\tca = o.ca\n\t}\n\tjs := o.js\n\to.mu.Unlock()\n\n\tif c != nil {\n\t\tc.closeConnection(ClientClosed)\n\t}\n\tif sysc != nil {\n\t\tsysc.closeConnection(ClientClosed)\n\t}\n\n\tif delivery != _EMPTY_ {\n\t\ta.sl.clearNotification(delivery, qgroup, o.inch)\n\t}\n\n\tvar rp RetentionPolicy\n\tif mset != nil {\n\t\tmset.mu.Lock()\n\t\tmset.removeConsumer(o)\n\t\t// No need for cfgMu's lock since mset.mu.Lock superseeds it.\n\t\trp = mset.cfg.Retention\n\t\tmset.mu.Unlock()\n\t}\n\n\t// Cleanup messages that lost interest.\n\tif dflag && rp == InterestPolicy {\n\t\to.cleanupNoInterestMessages(mset, true)\n\t}\n\n\t// Cluster cleanup.\n\tif n != nil {\n\t\tif dflag {\n\t\t\tn.Delete()\n\t\t} else {\n\t\t\tn.Stop()\n\t\t}\n\t}\n\n\tif ca != nil {\n\t\tjs.mu.Lock()\n\t\tif ca.Group != nil {\n\t\t\tca.Group.node = nil\n\t\t}\n\t\tjs.mu.Unlock()\n\t}\n\n\t// Clean up our store.\n\tvar err error\n\tif store != nil {\n\t\tif dflag {\n\t\t\tif sdflag {\n\t\t\t\terr = store.StreamDelete()\n\t\t\t} else {\n\t\t\t\terr = store.Delete()\n\t\t\t}\n\t\t} else {\n\t\t\terr = store.Stop()\n\t\t}\n\t} else if dflag {\n\t\t// If there's no store (for example, when it's offline), manually delete the directories.\n\t\to.mu.RLock()\n\t\tstream, consumer := o.stream, o.name\n\t\to.mu.RUnlock()\n\t\taccDir := filepath.Join(js.config.StoreDir, a.GetName())\n\t\tconsumersDir := filepath.Join(accDir, streamsDir, stream, consumerDir)\n\t\tos.RemoveAll(filepath.Join(consumersDir, consumer))\n\t}\n\n\treturn err\n}\n\n// We need to optionally remove all messages since we are interest based retention.\n// We will do this consistently on all replicas. Note that if in clustered mode the non-leader\n// consumers will need to restore state first.\n// ignoreInterest marks whether the consumer should be ignored when determining interest.\n// No lock held on entry.\nfunc (o *consumer) cleanupNoInterestMessages(mset *stream, ignoreInterest bool) {\n\to.mu.Lock()\n\tif !o.isLeader() {\n\t\to.readStoredState()\n\t}\n\tstart := o.asflr\n\to.mu.Unlock()\n\n\t// Make sure we start at worst with first sequence in the stream.\n\tstate := mset.state()\n\tif start < state.FirstSeq {\n\t\tstart = state.FirstSeq\n\t}\n\tstop := state.LastSeq\n\n\t// Consumer's interests are ignored by default. If we should not ignore interest, unset.\n\tco := o\n\tif !ignoreInterest {\n\t\tco = nil\n\t}\n\n\tvar rmseqs []uint64\n\tmset.mu.RLock()\n\n\t// If over this amount of messages to check, optimistically call to checkInterestState().\n\t// It will not always do the right thing in removing messages that lost interest, but ensures\n\t// we don't degrade performance by doing a linear scan through the whole stream.\n\t// Messages might need to expire based on limits to be cleaned up.\n\t// TODO(dlc) - Better way?\n\tconst bailThresh = 100_000\n\n\t// Check if we would be spending too much time here and defer to separate go routine.\n\tif len(mset.consumers) == 0 {\n\t\tmset.mu.RUnlock()\n\t\tmset.mu.Lock()\n\t\tdefer mset.mu.Unlock()\n\t\tmset.store.Purge()\n\t\tvar state StreamState\n\t\tmset.store.FastState(&state)\n\t\tmset.lseq = state.LastSeq\n\t\t// Also make sure we clear any pending acks.\n\t\tmset.clearAllPreAcksBelowFloor(state.FirstSeq)\n\t\treturn\n\t} else if stop-start > bailThresh {\n\t\tmset.mu.RUnlock()\n\t\tgo mset.checkInterestState()\n\t\treturn\n\t}\n\n\tmset.mu.RUnlock()\n\tmset.mu.Lock()\n\tfor seq := start; seq <= stop; seq++ {\n\t\tif mset.noInterest(seq, co) {\n\t\t\trmseqs = append(rmseqs, seq)\n\t\t}\n\t}\n\tmset.mu.Unlock()\n\n\t// These can be removed.\n\tfor _, seq := range rmseqs {\n\t\tmset.store.RemoveMsg(seq)\n\t}\n}\n\n// Check that we do not form a cycle by delivering to a delivery subject\n// that is part of the interest group.\nfunc deliveryFormsCycle(cfg *StreamConfig, deliverySubject string) bool {\n\tfor _, subject := range cfg.Subjects {\n\t\tif subjectIsSubsetMatch(deliverySubject, subject) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// switchToEphemeral is called on startup when recovering ephemerals.\nfunc (o *consumer) switchToEphemeral() {\n\to.mu.Lock()\n\to.cfg.Durable = _EMPTY_\n\tstore, ok := o.store.(*consumerFileStore)\n\tinterest := o.acc.sl.HasInterest(o.cfg.DeliverSubject)\n\t// Setup dthresh.\n\to.updateInactiveThreshold(&o.cfg)\n\to.updatePauseState(&o.cfg)\n\to.mu.Unlock()\n\n\t// Update interest\n\to.updateDeliveryInterest(interest)\n\t// Write out new config\n\tif ok {\n\t\tstore.updateConfig(o.cfg)\n\t}\n}\n\n// RequestNextMsgSubject returns the subject to request the next message when in pull or worker mode.\n// Returns empty otherwise.\nfunc (o *consumer) requestNextMsgSubject() string {\n\treturn o.nextMsgSubj\n}\n\nfunc (o *consumer) decStreamPending(sseq uint64, subj string) {\n\to.mu.Lock()\n\n\t// Update our cached num pending only if we think deliverMsg has not done so.\n\tif sseq >= o.sseq && o.isFilteredMatch(subj) {\n\t\to.npc--\n\t}\n\n\t// Check if this message was pending.\n\tp, wasPending := o.pending[sseq]\n\tvar rdc uint64\n\tif wasPending {\n\t\trdc = o.deliveryCount(sseq)\n\t}\n\n\to.mu.Unlock()\n\n\t// If it was pending process it like an ack.\n\tif wasPending {\n\t\t// We could have the lock for the stream so do this in a go routine.\n\t\t// TODO(dlc) - We should do this with ipq vs naked go routines.\n\t\tgo o.processTerm(sseq, p.Sequence, rdc, ackTermUnackedLimitsReason, _EMPTY_)\n\t}\n}\n\nfunc (o *consumer) account() *Account {\n\to.mu.RLock()\n\ta := o.acc\n\to.mu.RUnlock()\n\treturn a\n}\n\n// Creates a sublist for consumer.\n// All subjects share the same callback.\nfunc (o *consumer) signalSubs() []string {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.sigSubs != nil {\n\t\treturn o.sigSubs\n\t}\n\n\tif len(o.subjf) == 0 {\n\t\tsubs := []string{fwcs}\n\t\to.sigSubs = subs\n\t\treturn subs\n\t}\n\n\tsubs := make([]string, 0, len(o.subjf))\n\tfor _, filter := range o.subjf {\n\t\tsubs = append(subs, filter.subject)\n\t}\n\to.sigSubs = subs\n\treturn subs\n}\n\n// This is what will be called when our parent stream wants to kick us regarding a new message.\n// We know that this subject matches us by how the parent handles registering us with the signaling sublist,\n// but we must check if we are leader.\n// We do need the sequence of the message however and we use the msg as the encoded seq.\nfunc (o *consumer) processStreamSignal(seq uint64) {\n\t// We can get called here now when not leader, so bail fast\n\t// and without acquiring any locks.\n\tif !o.leader.Load() {\n\t\treturn\n\t}\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\tif o.mset == nil {\n\t\treturn\n\t}\n\tif seq > o.npf {\n\t\to.npc++\n\t}\n\tif seq < o.sseq {\n\t\treturn\n\t}\n\tif o.isPushMode() && o.active || o.isPullMode() && !o.waiting.isEmpty() {\n\t\to.signalNewMessages()\n\t}\n}\n\n// Used to compare if two multiple filtered subject lists are equal.\nfunc subjectSliceEqual(slice1 []string, slice2 []string) bool {\n\tif len(slice1) != len(slice2) {\n\t\treturn false\n\t}\n\tset2 := make(map[string]struct{}, len(slice2))\n\tfor _, val := range slice2 {\n\t\tset2[val] = struct{}{}\n\t}\n\tfor _, val := range slice1 {\n\t\tif _, ok := set2[val]; !ok {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Utility for simpler if conditions in Consumer config checks.\n// In future iteration, we can immediately create `o.subjf` and\n// use it to validate things.\nfunc gatherSubjectFilters(filter string, filters []string) []string {\n\tif filter != _EMPTY_ {\n\t\tfilters = append(filters, filter)\n\t}\n\t// list of filters should never contain non-empty filter.\n\treturn filters\n}\n\n// shouldStartMonitor will return true if we should start a monitor\n// goroutine or will return false if one is already running.\nfunc (o *consumer) shouldStartMonitor() bool {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.inMonitor {\n\t\treturn false\n\t}\n\to.monitorWg.Add(1)\n\to.inMonitor = true\n\treturn true\n}\n\n// Clear the monitor running state. The monitor goroutine should\n// call this in a defer to clean up on exit.\nfunc (o *consumer) clearMonitorRunning() {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.inMonitor {\n\t\to.monitorWg.Done()\n\t\to.inMonitor = false\n\t}\n}\n\n// Test whether we are in the monitor routine.\nfunc (o *consumer) isMonitorRunning() bool {\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.inMonitor\n}\n\n// If we detect that our ackfloor is higher than the stream's last sequence, return this error.\nvar errAckFloorHigherThanLastSeq = errors.New(\"consumer ack floor is higher than streams last sequence\")\nvar errAckFloorInvalid = errors.New(\"consumer ack floor is invalid\")\n\n// If we are a consumer of an interest or workqueue policy stream, process that state and make sure consistent.\nfunc (o *consumer) checkStateForInterestStream(ss *StreamState) error {\n\to.mu.RLock()\n\t// See if we need to process this update if our parent stream is not a limits policy stream.\n\tmset := o.mset\n\tshouldProcessState := mset != nil && o.retention != LimitsPolicy\n\tif o.closed || !shouldProcessState || o.store == nil || ss == nil {\n\t\to.mu.RUnlock()\n\t\treturn nil\n\t}\n\tstore := mset.store\n\tstate, err := o.store.State()\n\n\tfilters, subjf, filter := o.filters, o.subjf, _EMPTY_\n\tvar wc bool\n\tif filters == nil && subjf != nil {\n\t\tfilter, wc = subjf[0].subject, subjf[0].hasWildcard\n\t}\n\tchkfloor := o.chkflr\n\to.mu.RUnlock()\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tasflr := state.AckFloor.Stream\n\t// Protect ourselves against rolling backwards.\n\tif asflr&(1<<63) != 0 {\n\t\treturn errAckFloorInvalid\n\t}\n\tdflr := asflr\n\tif len(state.Pending) > 0 && state.Delivered.Stream > dflr {\n\t\tdflr = state.Delivered.Stream\n\t}\n\n\t// Check if the underlying stream's last sequence is less than our floor.\n\t// This can happen if the stream has been reset and has not caught up yet.\n\tif asflr > ss.LastSeq {\n\t\treturn errAckFloorHigherThanLastSeq\n\t}\n\n\tvar smv StoreMsg\n\tvar seq, nseq uint64\n\t// Start at first stream seq or a previous check floor, whichever is higher.\n\t// Note this will really help for interest retention, with WQ the loadNextMsg\n\t// gets us a long way already since it will skip deleted msgs not for our filter.\n\tfseq := ss.FirstSeq\n\tif chkfloor > fseq {\n\t\tfseq = chkfloor\n\t}\n\n\tvar retryAsflr uint64\n\tfor seq = fseq; dflr > 0 && seq <= dflr; seq++ {\n\t\tif filters != nil {\n\t\t\t_, nseq, err = store.LoadNextMsgMulti(filters, seq, &smv)\n\t\t} else {\n\t\t\t_, nseq, err = store.LoadNextMsg(filter, wc, seq, &smv)\n\t\t}\n\t\t// if we advanced sequence update our seq. This can be on no error and EOF.\n\t\tif nseq > seq {\n\t\t\tseq = nseq\n\t\t}\n\t\tif err == nil {\n\t\t\t// Only ack though if no error and seq <= ack floor.\n\t\t\tif seq <= asflr {\n\t\t\t\tdidRemove := mset.ackMsg(o, seq)\n\t\t\t\t// Removing the message could fail. For example if clustered since we need to propose it.\n\t\t\t\t// Overwrite retry floor (only the first time) to allow us to check next time if the removal was successful.\n\t\t\t\tif didRemove && retryAsflr == 0 {\n\t\t\t\t\tretryAsflr = seq\n\t\t\t\t}\n\t\t\t} else if seq <= dflr {\n\t\t\t\t// Store the first entry above our ack floor, so we don't need to look it up again on retryAsflr=0.\n\t\t\t\tif retryAsflr == 0 {\n\t\t\t\t\tretryAsflr = seq\n\t\t\t\t}\n\t\t\t\t// If we have pending, we will need to walk through to delivered in case we missed any of those acks as well.\n\t\t\t\tif _, ok := state.Pending[seq]; !ok {\n\t\t\t\t\t// The filters are already taken into account,\n\t\t\t\t\tmset.ackMsg(o, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if err == ErrStoreEOF {\n\t\t\tbreak\n\t\t}\n\t}\n\t// If retry floor was not overwritten, set to ack floor+1, we don't need to account for any retries below it.\n\t// However, our ack floor may be lower than the next message we can receive, so we correct it upward if needed.\n\tif retryAsflr == 0 {\n\t\tif filters != nil {\n\t\t\t_, nseq, err = store.LoadNextMsgMulti(filters, asflr+1, &smv)\n\t\t} else {\n\t\t\t_, nseq, err = store.LoadNextMsg(filter, wc, asflr+1, &smv)\n\t\t}\n\t\tif err == nil {\n\t\t\tretryAsflr = max(asflr+1, nseq)\n\t\t} else if err == ErrStoreEOF {\n\t\t\tretryAsflr = ss.LastSeq + 1\n\t\t}\n\t}\n\n\to.mu.Lock()\n\t// Update our check floor.\n\t// Check floor must never be greater than ack floor+1, otherwise subsequent calls to this function would skip work.\n\tif retryAsflr > o.chkflr {\n\t\to.chkflr = retryAsflr\n\t}\n\to.mu.Unlock()\n\treturn nil\n}\n\nfunc (o *consumer) resetPtmr(delay time.Duration) {\n\tif o.ptmr == nil {\n\t\to.ptmr = time.AfterFunc(delay, o.checkPending)\n\t} else {\n\t\to.ptmr.Reset(delay)\n\t}\n\to.ptmrEnd = time.Now().Add(delay)\n}\n\nfunc (o *consumer) stopAndClearPtmr() {\n\tstopAndClearTimer(&o.ptmr)\n\to.ptmrEnd = time.Time{}\n}\n\nfunc (o *consumer) resetPendingDeliveries() {\n\tfor _, pmsg := range o.pendingDeliveries {\n\t\tpmsg.returnToPool()\n\t}\n\to.pendingDeliveries = nil\n\tfor _, wd := range o.waitingDeliveries {\n\t\twd.recycle()\n\t}\n\to.waitingDeliveries = nil\n}\n" }, { "path": "server/core_benchmarks_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc BenchmarkCoreRequestReply(b *testing.B) {\n\tconst (\n\t\tsubject = \"test-subject\"\n\t)\n\n\tmessageSizes := []int64{\n\t\t1024, // 1kb\n\t\t4096, // 4kb\n\t\t40960, // 40kb\n\t\t409600, // 400kb\n\t}\n\n\tfor _, messageSize := range messageSizes {\n\t\tb.Run(fmt.Sprintf(\"msgSz=%db\", messageSize), func(b *testing.B) {\n\n\t\t\t// Start server\n\t\t\tserverOpts := DefaultOptions()\n\t\t\tserver := RunServer(serverOpts)\n\t\t\tdefer server.Shutdown()\n\n\t\t\tclientUrl := server.ClientURL()\n\n\t\t\t// Create \"echo\" subscriber\n\t\t\tncSub, err := nats.Connect(clientUrl)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tdefer ncSub.Close()\n\t\t\tsub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) {\n\t\t\t\t// Responder echoes the request payload as-is\n\t\t\t\tmsg.Respond(msg.Data)\n\t\t\t})\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\t// Create publisher\n\t\t\tncPub, err := nats.Connect(clientUrl)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tdefer ncPub.Close()\n\n\t\t\tvar errors = 0\n\n\t\t\t// Create message\n\t\t\tmessageData := make([]byte, messageSize)\n\t\t\trand.New(rand.NewSource(12345)).Read(messageData)\n\n\t\t\tb.SetBytes(messageSize)\n\n\t\t\t// Benchmark\n\t\t\tb.ResetTimer()\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tfastRandomMutation(messageData, 10)\n\n\t\t\t\t_, err := ncPub.Request(subject, messageData, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrors++\n\t\t\t\t}\n\t\t\t}\n\t\t\tb.StopTimer()\n\n\t\t\tb.ReportMetric(float64(errors), \"errors\")\n\t\t})\n\t}\n}\n\nfunc BenchmarkCoreTLSFanOut(b *testing.B) {\n\tconst (\n\t\tsubject = \"test-subject\"\n\t\tconfigsBasePath = \"./configs/tls\"\n\t\tmaxPendingMessages = 25\n\t\tmaxPendingBytes = 15 * 1024 * 1024 // 15MiB\n\t)\n\n\tkeyTypeCases := []string{\n\t\t\"none\",\n\t\t\"ed25519\",\n\t\t\"rsa-1024\",\n\t\t\"rsa-2048\",\n\t\t\"rsa-4096\",\n\t}\n\tmessageSizeCases := []int64{\n\t\t512 * 1024, // 512Kib\n\t}\n\tnumSubsCases := []int{\n\t\t5,\n\t}\n\n\t// Custom error handler that ignores ErrSlowConsumer.\n\t// Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher\n\t// than what the server can relay to subscribers.\n\tignoreSlowConsumerErrorHandler := func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\tif errors.Is(err, nats.ErrSlowConsumer) {\n\t\t\t// Swallow this error\n\t\t} else {\n\t\t\t_, _ = fmt.Fprintf(os.Stderr, \"Warning: %s\\n\", err)\n\t\t}\n\t}\n\n\tfor _, keyType := range keyTypeCases {\n\n\t\tb.Run(\n\t\t\tfmt.Sprintf(\"keyType=%s\", keyType),\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, messageSize := range messageSizeCases {\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tfmt.Sprintf(\"msgSz=%db\", messageSize),\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tfor _, numSubs := range numSubsCases {\n\t\t\t\t\t\t\t\tb.Run(\n\t\t\t\t\t\t\t\t\tfmt.Sprintf(\"subs=%d\", numSubs),\n\t\t\t\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t\t\t\t// Start server\n\t\t\t\t\t\t\t\t\t\tconfigPath := fmt.Sprintf(\"%s/tls-%s.conf\", configsBasePath, keyType)\n\t\t\t\t\t\t\t\t\t\tserver, _ := RunServerWithConfig(configPath)\n\t\t\t\t\t\t\t\t\t\tdefer server.Shutdown()\n\n\t\t\t\t\t\t\t\t\t\topts := []nats.Option{\n\t\t\t\t\t\t\t\t\t\t\tnats.MaxReconnects(-1),\n\t\t\t\t\t\t\t\t\t\t\tnats.ReconnectWait(0),\n\t\t\t\t\t\t\t\t\t\t\tnats.ErrorHandler(ignoreSlowConsumerErrorHandler),\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\tif keyType != \"none\" {\n\t\t\t\t\t\t\t\t\t\t\topts = append(opts, nats.Secure(&tls.Config{\n\t\t\t\t\t\t\t\t\t\t\t\tInsecureSkipVerify: true,\n\t\t\t\t\t\t\t\t\t\t\t}))\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\tclientUrl := server.ClientURL()\n\n\t\t\t\t\t\t\t\t\t\t// Count of messages received for by each subscriber\n\t\t\t\t\t\t\t\t\t\tcounters := make([]int, numSubs)\n\n\t\t\t\t\t\t\t\t\t\t// Wait group for subscribers to signal they received b.N messages\n\t\t\t\t\t\t\t\t\t\tvar wg sync.WaitGroup\n\t\t\t\t\t\t\t\t\t\twg.Add(numSubs)\n\n\t\t\t\t\t\t\t\t\t\t// Create subscribers\n\t\t\t\t\t\t\t\t\t\tfor i := 0; i < numSubs; i++ {\n\t\t\t\t\t\t\t\t\t\t\tsubIndex := i\n\t\t\t\t\t\t\t\t\t\t\tncSub, err := nats.Connect(clientUrl, opts...)\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\tdefer ncSub.Close()\n\t\t\t\t\t\t\t\t\t\t\tsub, err := ncSub.Subscribe(subject, func(msg *nats.Msg) {\n\t\t\t\t\t\t\t\t\t\t\t\tcounters[subIndex] += 1\n\t\t\t\t\t\t\t\t\t\t\t\tif counters[subIndex] == b.N {\n\t\t\t\t\t\t\t\t\t\t\t\t\twg.Done()\n\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"failed to subscribe: %s\", err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\terr = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes)\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"failed to set pending limits: %s\", err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\tdefer sub.Unsubscribe()\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// publisher\n\t\t\t\t\t\t\t\t\t\tncPub, err := nats.Connect(clientUrl, opts...)\n\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\tdefer ncPub.Close()\n\n\t\t\t\t\t\t\t\t\t\tvar errorCount = 0\n\n\t\t\t\t\t\t\t\t\t\t// random bytes as payload\n\t\t\t\t\t\t\t\t\t\tmessageData := make([]byte, messageSize)\n\t\t\t\t\t\t\t\t\t\trand.New(rand.NewSource(12345)).Read(messageData)\n\n\t\t\t\t\t\t\t\t\t\tquitCh := make(chan bool, 1)\n\n\t\t\t\t\t\t\t\t\t\tpublish := func() {\n\t\t\t\t\t\t\t\t\t\t\tfor {\n\t\t\t\t\t\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\t\t\t\t\t\tcase <-quitCh:\n\t\t\t\t\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\t\t\t\t\t// continue publishing\n\t\t\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t\t\tfastRandomMutation(messageData, 10)\n\t\t\t\t\t\t\t\t\t\t\t\terr := ncPub.Publish(subject, messageData)\n\t\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\t\terrorCount += 1\n\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Set bytes per operation\n\t\t\t\t\t\t\t\t\t\tb.SetBytes(messageSize)\n\t\t\t\t\t\t\t\t\t\t// Start the clock\n\t\t\t\t\t\t\t\t\t\tb.ResetTimer()\n\t\t\t\t\t\t\t\t\t\t// Start publishing as fast as the server allows\n\t\t\t\t\t\t\t\t\t\tgo publish()\n\t\t\t\t\t\t\t\t\t\t// Wait for all subscribers to have delivered b.N messages\n\t\t\t\t\t\t\t\t\t\twg.Wait()\n\t\t\t\t\t\t\t\t\t\t// Stop the clock\n\t\t\t\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\t\t\t\t// Stop publisher\n\t\t\t\t\t\t\t\t\t\tquitCh <- true\n\n\t\t\t\t\t\t\t\t\t\tb.ReportMetric(float64(errorCount), \"errors\")\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkCoreFanOut(b *testing.B) {\n\tconst (\n\t\tsubject = \"test-subject\"\n\t\tmaxPendingMessages = 25\n\t\tmaxPendingBytes = 15 * 1024 * 1024 // 15MiB\n\t)\n\n\tmessageSizeCases := []int64{\n\t\t100, // 100B\n\t\t1024, // 1KiB\n\t\t10240, // 10KiB\n\t\t512 * 1024, // 512KiB\n\t}\n\tnumSubsCases := []int{\n\t\t3,\n\t\t5,\n\t\t10,\n\t}\n\n\t// Custom error handler that ignores ErrSlowConsumer.\n\t// Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher\n\t// than what the server can relay to subscribers.\n\tignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif errors.Is(err, nats.ErrSlowConsumer) {\n\t\t\t// Swallow this error\n\t\t} else {\n\t\t\t_, _ = fmt.Fprintf(os.Stderr, \"Warning: %s\\n\", err)\n\t\t}\n\t}\n\n\tfor _, messageSize := range messageSizeCases {\n\t\tb.Run(\n\t\t\tfmt.Sprintf(\"msgSz=%db\", messageSize),\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, numSubs := range numSubsCases {\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tfmt.Sprintf(\"subs=%d\", numSubs),\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t// Start server\n\t\t\t\t\t\t\tdefaultOpts := DefaultOptions()\n\t\t\t\t\t\t\tserver := RunServer(defaultOpts)\n\t\t\t\t\t\t\tdefer server.Shutdown()\n\n\t\t\t\t\t\t\topts := []nats.Option{\n\t\t\t\t\t\t\t\tnats.MaxReconnects(-1),\n\t\t\t\t\t\t\t\tnats.ReconnectWait(0),\n\t\t\t\t\t\t\t\tnats.ErrorHandler(ignoreSlowConsumerErrorHandler),\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tclientUrl := server.ClientURL()\n\n\t\t\t\t\t\t\t// Count of messages received for by each subscriber\n\t\t\t\t\t\t\tcounters := make([]int, numSubs)\n\n\t\t\t\t\t\t\t// Wait group for subscribers to signal they received b.N messages\n\t\t\t\t\t\t\tvar wg sync.WaitGroup\n\t\t\t\t\t\t\twg.Add(numSubs)\n\n\t\t\t\t\t\t\t// Create subscribers\n\t\t\t\t\t\t\tfor i := 0; i < numSubs; i++ {\n\t\t\t\t\t\t\t\tsubIndex := i\n\t\t\t\t\t\t\t\tncSub, err := nats.Connect(clientUrl, opts...)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tdefer ncSub.Close()\n\t\t\t\t\t\t\t\tsub, err := ncSub.Subscribe(subject, func(_ *nats.Msg) {\n\t\t\t\t\t\t\t\t\tcounters[subIndex] += 1\n\t\t\t\t\t\t\t\t\tif counters[subIndex] == b.N {\n\t\t\t\t\t\t\t\t\t\twg.Done()\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"failed to subscribe: %s\", err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\terr = sub.SetPendingLimits(maxPendingMessages, maxPendingBytes)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"failed to set pending limits: %s\", err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tdefer sub.Unsubscribe()\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// publisher\n\t\t\t\t\t\t\tncPub, err := nats.Connect(clientUrl, opts...)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tdefer ncPub.Close()\n\n\t\t\t\t\t\t\tvar errorCount = 0\n\n\t\t\t\t\t\t\t// random bytes as payload\n\t\t\t\t\t\t\tmessageData := make([]byte, messageSize)\n\t\t\t\t\t\t\trand.New(rand.NewSource(123456)).Read(messageData)\n\n\t\t\t\t\t\t\tquitCh := make(chan bool, 1)\n\n\t\t\t\t\t\t\tpublish := func() {\n\t\t\t\t\t\t\t\tfor {\n\t\t\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\t\t\tcase <-quitCh:\n\t\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\t\t// continue publishing\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tfastRandomMutation(messageData, 10)\n\t\t\t\t\t\t\t\t\terr := ncPub.Publish(subject, messageData)\n\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\terrorCount += 1\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Set bytes per operation\n\t\t\t\t\t\t\tb.SetBytes(messageSize)\n\t\t\t\t\t\t\t// Start the clock\n\t\t\t\t\t\t\tb.ResetTimer()\n\t\t\t\t\t\t\t// Start publishing as fast as the server allows\n\t\t\t\t\t\t\tgo publish()\n\t\t\t\t\t\t\t// Wait for all subscribers to have delivered b.N messages\n\t\t\t\t\t\t\twg.Wait()\n\t\t\t\t\t\t\t// Stop the clock\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\t// Stop publisher\n\t\t\t\t\t\t\tquitCh <- true\n\n\t\t\t\t\t\t\tb.ReportMetric(100*float64(errorCount)/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkCoreFanIn(b *testing.B) {\n\n\ttype BenchPublisher struct {\n\t\t// nats connection for this publisher\n\t\tconn *nats.Conn\n\t\t// number of publishing errors encountered\n\t\tpublishErrors int\n\t\t// number of messages published\n\t\tpublishCounter int\n\t\t// quit channel which will terminate publishing\n\t\tquitCh chan bool\n\t\t// message data buffer\n\t\tmessageData []byte\n\t}\n\n\tconst subjectBaseName = \"test-subject\"\n\n\tmessageSizeCases := []int64{\n\t\t100, // 100B\n\t\t1024, // 1KiB\n\t\t10240, // 10KiB\n\t\t512 * 1024, // 512KiB\n\t}\n\tnumPubsCases := []int{\n\t\t3,\n\t\t5,\n\t\t10,\n\t}\n\n\t// Custom error handler that ignores ErrSlowConsumer.\n\t// Lots of them are expected in this benchmark which indiscriminately publishes at a rate higher\n\t// than what the server can relay to subscribers.\n\tignoreSlowConsumerErrorHandler := func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif errors.Is(err, nats.ErrSlowConsumer) {\n\t\t\t// Swallow this error\n\t\t} else {\n\t\t\t_, _ = fmt.Fprintf(os.Stderr, \"Warning: %s\\n\", err)\n\t\t}\n\t}\n\n\tworkload := func(b *testing.B, clientUrl string, numPubs int, messageSize int64) {\n\n\t\t// connection options\n\t\topts := []nats.Option{\n\t\t\tnats.MaxReconnects(-1),\n\t\t\tnats.ReconnectWait(0),\n\t\t\tnats.ErrorHandler(ignoreSlowConsumerErrorHandler),\n\t\t}\n\n\t\t// waits for all publishers sub-routines and for main thread to be ready\n\t\tvar publishersReadyWg sync.WaitGroup\n\t\tpublishersReadyWg.Add(numPubs + 1)\n\n\t\t// wait group to ensure all publishers have been torn down\n\t\tvar finishedPublishersWg sync.WaitGroup\n\t\tfinishedPublishersWg.Add(numPubs)\n\n\t\tpublishers := make([]BenchPublisher, numPubs)\n\t\t// create N publishers\n\t\tfor i := range publishers {\n\t\t\t// create publisher connection\n\t\t\tncPub, err := nats.Connect(clientUrl, opts...)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tdefer ncPub.Close()\n\n\t\t\t// create bench publisher object\n\t\t\tpublisher := BenchPublisher{\n\t\t\t\tconn: ncPub,\n\t\t\t\tpublishErrors: 0,\n\t\t\t\tpublishCounter: 0,\n\t\t\t\tquitCh: make(chan bool, 1),\n\t\t\t\tmessageData: make([]byte, messageSize),\n\t\t\t}\n\t\t\trand.New(rand.NewSource(int64(i))).Read(publisher.messageData)\n\t\t\tpublishers[i] = publisher\n\t\t}\n\n\t\t// total number of publishers that have published b.N to the subscriber successfully\n\t\tcompletedPublishersCount := 0\n\n\t\t// wait group blocks main thread until publish workload is completed, it is decremented after subscriber receives b.N messages from all publishers\n\t\tvar benchCompleteWg sync.WaitGroup\n\t\tbenchCompleteWg.Add(1)\n\n\t\t// start subscriber\n\t\tncSub, err := nats.Connect(clientUrl, opts...)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tdefer ncSub.Close()\n\n\t\t// subscriber\n\t\tncSub.Subscribe(fmt.Sprintf(\"%s.*\", subjectBaseName), func(msg *nats.Msg) {\n\t\t\t// get the publisher id from subject\n\t\t\tpubIdx, err := strconv.Atoi(msg.Subject[len(subjectBaseName)+1:])\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\t// message successfully received from publisher\n\t\t\tpublishers[pubIdx].publishCounter += 1\n\n\t\t\t// subscriber has received a total of b.N messages from this publisher\n\t\t\tif publishers[pubIdx].publishCounter == b.N {\n\t\t\t\tcompletedPublishersCount++\n\t\t\t\t// every publisher has successfully sent b.N messages to subscriber\n\t\t\t\tif completedPublishersCount == numPubs {\n\t\t\t\t\tbenchCompleteWg.Done()\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\n\t\t// start publisher sub-routines\n\t\tfor i := range publishers {\n\t\t\tgo func(pubId int) {\n\n\t\t\t\t// publisher sub-routine initialized\n\t\t\t\tpublishersReadyWg.Done()\n\n\t\t\t\tpublisher := publishers[pubId]\n\t\t\t\tsubject := fmt.Sprintf(\"%s.%d\", subjectBaseName, pubId)\n\n\t\t\t\t// signal that this publisher has been torn down\n\t\t\t\tdefer finishedPublishersWg.Done()\n\n\t\t\t\t// wait till all other publishers are ready to start workload\n\t\t\t\tpublishersReadyWg.Wait()\n\n\t\t\t\t// publish until quitCh is closed\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-publisher.quitCh:\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t\t// continue publishing\n\t\t\t\t\t}\n\t\t\t\t\tfastRandomMutation(publisher.messageData, 10)\n\t\t\t\t\terr := publisher.conn.Publish(subject, publisher.messageData)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tpublisher.publishErrors += 1\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(i)\n\t\t}\n\n\t\t// set bytes per operation\n\t\tb.SetBytes(messageSize)\n\t\t// main thread is ready\n\t\tpublishersReadyWg.Done()\n\t\t// wait till publishers are ready\n\t\tpublishersReadyWg.Wait()\n\n\t\t// start the clock\n\t\tb.ResetTimer()\n\t\t// wait till termination cond reached\n\t\tbenchCompleteWg.Wait()\n\t\t// stop the clock\n\t\tb.StopTimer()\n\n\t\t// send quit signal to all publishers\n\t\tfor i := range publishers {\n\t\t\tpublishers[i].quitCh <- true\n\t\t}\n\t\t// wait for all publishers to shutdown\n\t\tfinishedPublishersWg.Wait()\n\n\t\t// sum errors from all publishers\n\t\ttotalErrors := 0\n\t\tfor _, publisher := range publishers {\n\t\t\ttotalErrors += publisher.publishErrors\n\t\t}\n\t\t// sum total messages sent from all publishers\n\t\ttotalMessages := 0\n\t\tfor _, publisher := range publishers {\n\t\t\ttotalMessages += publisher.publishCounter\n\t\t}\n\t\terrorRate := 100 * float64(totalErrors) / float64(totalMessages)\n\n\t\t// report error rate\n\t\tb.ReportMetric(errorRate, \"%error\")\n\n\t}\n\n\t// benchmark case matrix\n\tfor _, messageSize := range messageSizeCases {\n\t\tb.Run(\n\t\t\tfmt.Sprintf(\"msgSz=%db\", messageSize),\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, numPubs := range numPubsCases {\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tfmt.Sprintf(\"pubs=%d\", numPubs),\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t// start server\n\t\t\t\t\t\t\tdefaultOpts := DefaultOptions()\n\t\t\t\t\t\t\tserver := RunServer(defaultOpts)\n\t\t\t\t\t\t\tdefer server.Shutdown()\n\n\t\t\t\t\t\t\t// get connection string\n\t\t\t\t\t\t\tclientUrl := server.ClientURL()\n\n\t\t\t\t\t\t\t// run fan-in workload\n\t\t\t\t\t\t\tworkload(b, clientUrl, numPubs, messageSize)\n\t\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t})\n\t}\n}\n\n// fastRandomMutation performs a minor in-place mutation to the given buffer.\n// This is useful in benchmark to avoid sending the same payload every time (which could result in some optimizations\n// we do not want to measure), while not slowing down the benchmark with a full payload generated for each operation.\nfunc fastRandomMutation(data []byte, mutations int) {\n\tfor i := 0; i < mutations; i++ {\n\t\tdata[fastrand.Uint32n(uint32(len(data)))] = byte(fastrand.Uint32() % math.MaxUint8)\n\t}\n}\n" }, { "path": "server/cron.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Based on code from https://github.com/robfig/cron\n// Copyright (C) 2012 Rob Figueiredo\n// All Rights Reserved.\n//\n// MIT LICENSE\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of\n// this software and associated documentation files (the \"Software\"), to deal in\n// the Software without restriction, including without limitation the rights to\n// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n// the Software, and to permit persons to whom the Software is furnished to do so,\n// subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all\n// copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// parseCron parses the given cron pattern and returns the next time it will fire based on the provided ts.\nfunc parseCron(pattern string, tz string, ts int64) (time.Time, error) {\n\tfields := strings.Fields(pattern)\n\tif len(fields) != 6 {\n\t\treturn time.Time{}, fmt.Errorf(\"pattern requires 6 fields, got %d\", len(fields))\n\t}\n\n\t// Load the time zone.\n\tloc, err := time.LoadLocation(tz)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\t// Parse each field.\n\tvar second, minute, hour, dayOfMonth, month, dayOfWeek uint64\n\tif second, err = getField(fields[0], seconds); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\tif minute, err = getField(fields[1], minutes); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\tif hour, err = getField(fields[2], hours); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\tif dayOfMonth, err = getField(fields[3], dom); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\tif month, err = getField(fields[4], months); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\tif dayOfWeek, err = getField(fields[5], dow); err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\t// General approach\n\t//\n\t// For Month, Day, Hour, Minute, Second:\n\t// Check if the time value matches. If yes, continue to the next field.\n\t// If the field doesn't match the schedule, then increment the field until it matches.\n\t// While incrementing the field, a wrap-around brings it back to the beginning\n\t// of the field list (since it is necessary to re-verify previous field values)\n\tnext := time.Unix(0, ts).In(loc)\n\n\t// Start at the earliest possible time (the upcoming second).\n\tnext = next.Truncate(time.Second).Add(time.Second)\n\n\t// This flag indicates whether a field has been truncated at one point.\n\ttruncated := false\n\n\t// If no time is found within five years, return error.\n\tyearLimit := next.Year() + 5\n\nWRAP:\n\tif next.Year() > yearLimit {\n\t\treturn time.Time{}, errors.New(\"pattern exceeds maximum range\")\n\t}\n\tfor 1< 1 {\n\t\t\textra = 0\n\t\t}\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"too many slashes: %s\", expr)\n\t}\n\n\tif start < r.min {\n\t\treturn 0, fmt.Errorf(\"beginning of range (%d) below minimum (%d): %s\", start, r.min, expr)\n\t}\n\tif end > r.max {\n\t\treturn 0, fmt.Errorf(\"end of range (%d) above maximum (%d): %s\", end, r.max, expr)\n\t}\n\tif start > end {\n\t\treturn 0, fmt.Errorf(\"beginning of range (%d) beyond end of range (%d): %s\", start, end, expr)\n\t}\n\tif step == 0 {\n\t\treturn 0, fmt.Errorf(\"step of range should be a positive number: %s\", expr)\n\t}\n\treturn getBits(start, end, step) | extra, nil\n}\n\n// parseIntOrName returns the (possibly-named) integer contained in expr.\nfunc parseIntOrName(expr string, names map[string]uint) (uint, error) {\n\tif names != nil {\n\t\tif namedInt, ok := names[strings.ToLower(expr)]; ok {\n\t\t\treturn namedInt, nil\n\t\t}\n\t}\n\treturn mustParseInt(expr)\n}\n\n// mustParseInt parses the given expression as an int or returns an error.\nfunc mustParseInt(expr string) (uint, error) {\n\tnum, err := strconv.Atoi(expr)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to parse int from %s: %s\", expr, err)\n\t}\n\tif num < 0 {\n\t\treturn 0, fmt.Errorf(\"negative number (%d) not allowed: %s\", num, expr)\n\t}\n\treturn uint(num), nil\n}\n\n// getBits sets all bits in the range [min, max], modulo the given step size.\nfunc getBits(min, max, step uint) uint64 {\n\tvar bits uint64\n\n\t// If step is 1, use shifts.\n\tif step == 1 {\n\t\treturn ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)\n\t}\n\n\t// Else, use a simple loop.\n\tfor i := min; i <= max; i += step {\n\t\tbits |= 1 << i\n\t}\n\treturn bits\n}\n\n// bounds provides a range of acceptable values (plus a map of name to value).\ntype bounds struct {\n\tmin, max uint\n\tnames map[string]uint\n}\n\n// The bounds for each field.\nvar (\n\tseconds = bounds{0, 59, nil}\n\tminutes = bounds{0, 59, nil}\n\thours = bounds{0, 23, nil}\n\tdom = bounds{1, 31, nil}\n\tmonths = bounds{1, 12, map[string]uint{\n\t\t\"jan\": 1,\n\t\t\"feb\": 2,\n\t\t\"mar\": 3,\n\t\t\"apr\": 4,\n\t\t\"may\": 5,\n\t\t\"jun\": 6,\n\t\t\"jul\": 7,\n\t\t\"aug\": 8,\n\t\t\"sep\": 9,\n\t\t\"oct\": 10,\n\t\t\"nov\": 11,\n\t\t\"dec\": 12,\n\t}}\n\tdow = bounds{0, 6, map[string]uint{\n\t\t\"sun\": 0,\n\t\t\"mon\": 1,\n\t\t\"tue\": 2,\n\t\t\"wed\": 3,\n\t\t\"thu\": 4,\n\t\t\"fri\": 5,\n\t\t\"sat\": 6,\n\t}}\n)\n\nconst (\n\t// Set the top bit if a star was included in the expression.\n\tstarBit = 1 << 63\n)\n\n// dayMatches returns true if the schedule's day-of-week and day-of-month\n// restrictions are satisfied by the given time.\nfunc dayMatches(dayOfMonth, dayOfWeek uint64, t time.Time) bool {\n\tvar (\n\t\tdomMatch = 1< 0\n\t\tdowMatch = 1< 0\n\t)\n\tif dayOfMonth&starBit > 0 || dayOfWeek&starBit > 0 {\n\t\treturn domMatch && dowMatch\n\t}\n\treturn domMatch || dowMatch\n}\n" }, { "path": "server/dirstore.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"container/heap\"\n\t\"container/list\"\n\t\"crypto/sha256\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/nats-io/nkeys\"\n\n\t\"github.com/nats-io/jwt/v2\" // only used to decode, not for storage\n)\n\nconst (\n\tfileExtension = \".jwt\"\n)\n\n// validatePathExists checks that the provided path exists and is a dir if requested\nfunc validatePathExists(path string, dir bool) (string, error) {\n\tif path == _EMPTY_ {\n\t\treturn _EMPTY_, errors.New(\"path is not specified\")\n\t}\n\n\tabs, err := filepath.Abs(path)\n\tif err != nil {\n\t\treturn _EMPTY_, fmt.Errorf(\"error parsing path [%s]: %v\", abs, err)\n\t}\n\n\tvar finfo os.FileInfo\n\tif finfo, err = os.Stat(abs); os.IsNotExist(err) {\n\t\treturn _EMPTY_, fmt.Errorf(\"the path [%s] doesn't exist\", abs)\n\t}\n\n\tmode := finfo.Mode()\n\tif dir && mode.IsRegular() {\n\t\treturn _EMPTY_, fmt.Errorf(\"the path [%s] is not a directory\", abs)\n\t}\n\n\tif !dir && mode.IsDir() {\n\t\treturn _EMPTY_, fmt.Errorf(\"the path [%s] is not a file\", abs)\n\t}\n\n\treturn abs, nil\n}\n\n// ValidateDirPath checks that the provided path exists and is a dir\nfunc validateDirPath(path string) (string, error) {\n\treturn validatePathExists(path, true)\n}\n\n// JWTChanged functions are called when the store file watcher notices a JWT changed\ntype JWTChanged func(publicKey string)\n\n// DirJWTStore implements the JWT Store interface, keeping JWTs in an optionally sharded\n// directory structure\ntype DirJWTStore struct {\n\tsync.Mutex\n\tdirectory string\n\tshard bool\n\treadonly bool\n\tdeleteType deleteType\n\toperator map[string]struct{}\n\texpiration *expirationTracker\n\tchanged JWTChanged\n\tdeleted JWTChanged\n}\n\nfunc newDir(dirPath string, create bool) (string, error) {\n\tfullPath, err := validateDirPath(dirPath)\n\tif err != nil {\n\t\tif !create {\n\t\t\treturn _EMPTY_, err\n\t\t}\n\t\tif err = os.MkdirAll(dirPath, defaultDirPerms); err != nil {\n\t\t\treturn _EMPTY_, err\n\t\t}\n\t\tif fullPath, err = validateDirPath(dirPath); err != nil {\n\t\t\treturn _EMPTY_, err\n\t\t}\n\t}\n\treturn fullPath, nil\n}\n\n// future proofing in case new options will be added\ntype dirJWTStoreOption any\n\n// Creates a directory based jwt store.\n// Reads files only, does NOT watch directories and files.\nfunc NewImmutableDirJWTStore(dirPath string, shard bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) {\n\ttheStore, err := NewDirJWTStore(dirPath, shard, false, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttheStore.readonly = true\n\treturn theStore, nil\n}\n\n// Creates a directory based jwt store.\n// Operates on files only, does NOT watch directories and files.\nfunc NewDirJWTStore(dirPath string, shard bool, create bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) {\n\tfullPath, err := newDir(dirPath, create)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttheStore := &DirJWTStore{\n\t\tdirectory: fullPath,\n\t\tshard: shard,\n\t}\n\treturn theStore, nil\n}\n\ntype deleteType int\n\nconst (\n\tNoDelete deleteType = iota\n\tRenameDeleted\n\tHardDelete\n)\n\n// Creates a directory based jwt store.\n//\n// When ttl is set deletion of file is based on it and not on the jwt expiration\n// To completely disable expiration (including expiration in jwt) set ttl to max duration time.Duration(math.MaxInt64)\n//\n// limit defines how many files are allowed at any given time. Set to math.MaxInt64 to disable.\n// evictOnLimit determines the behavior once limit is reached.\n// * true - Evict based on lru strategy\n// * false - return an error\nfunc NewExpiringDirJWTStore(dirPath string, shard bool, create bool, delete deleteType, expireCheck time.Duration, limit int64,\n\tevictOnLimit bool, ttl time.Duration, changeNotification JWTChanged, _ ...dirJWTStoreOption) (*DirJWTStore, error) {\n\tfullPath, err := newDir(dirPath, create)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttheStore := &DirJWTStore{\n\t\tdirectory: fullPath,\n\t\tshard: shard,\n\t\tdeleteType: delete,\n\t\tchanged: changeNotification,\n\t}\n\tif expireCheck <= 0 {\n\t\tif ttl != 0 {\n\t\t\texpireCheck = ttl / 2\n\t\t}\n\t\tif expireCheck == 0 || expireCheck > time.Minute {\n\t\t\texpireCheck = time.Minute\n\t\t}\n\t}\n\tif limit <= 0 {\n\t\tlimit = math.MaxInt64\n\t}\n\ttheStore.startExpiring(expireCheck, limit, evictOnLimit, ttl)\n\ttheStore.Lock()\n\terr = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {\n\t\tif strings.HasSuffix(path, fileExtension) {\n\t\t\tif theJwt, err := os.ReadFile(path); err == nil {\n\t\t\t\thash := sha256.Sum256(theJwt)\n\t\t\t\t_, file := filepath.Split(path)\n\t\t\t\ttheStore.expiration.track(strings.TrimSuffix(file, fileExtension), &hash, string(theJwt))\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\ttheStore.Unlock()\n\tif err != nil {\n\t\ttheStore.Close()\n\t\treturn nil, err\n\t}\n\treturn theStore, err\n}\n\nfunc (store *DirJWTStore) IsReadOnly() bool {\n\treturn store.readonly\n}\n\nfunc (store *DirJWTStore) LoadAcc(publicKey string) (string, error) {\n\treturn store.load(publicKey)\n}\n\nfunc (store *DirJWTStore) SaveAcc(publicKey string, theJWT string) error {\n\treturn store.save(publicKey, theJWT)\n}\n\nfunc (store *DirJWTStore) LoadAct(hash string) (string, error) {\n\treturn store.load(hash)\n}\n\nfunc (store *DirJWTStore) SaveAct(hash string, theJWT string) error {\n\treturn store.save(hash, theJWT)\n}\n\nfunc (store *DirJWTStore) Close() {\n\tstore.Lock()\n\tdefer store.Unlock()\n\tif store.expiration != nil {\n\t\tstore.expiration.close()\n\t\tstore.expiration = nil\n\t}\n}\n\n// Pack up to maxJWTs into a package\nfunc (store *DirJWTStore) Pack(maxJWTs int) (string, error) {\n\tcount := 0\n\tvar pack []string\n\tif maxJWTs > 0 {\n\t\tpack = make([]string, 0, maxJWTs)\n\t} else {\n\t\tpack = []string{}\n\t}\n\tstore.Lock()\n\terr := filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error {\n\t\tif !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT\n\t\t\tif count == maxJWTs { // won't match negative\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tpubKey := strings.TrimSuffix(filepath.Base(path), fileExtension)\n\t\t\tif store.expiration != nil {\n\t\t\t\tif _, ok := store.expiration.idx[pubKey]; !ok {\n\t\t\t\t\treturn nil // only include indexed files\n\t\t\t\t}\n\t\t\t}\n\t\t\tjwtBytes, err := os.ReadFile(path)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif store.expiration != nil {\n\t\t\t\tclaim, err := jwt.DecodeGeneric(string(jwtBytes))\n\t\t\t\tif err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\tpack = append(pack, fmt.Sprintf(\"%s|%s\", pubKey, string(jwtBytes)))\n\t\t\tcount++\n\t\t}\n\t\treturn nil\n\t})\n\tstore.Unlock()\n\tif err != nil {\n\t\treturn _EMPTY_, err\n\t} else {\n\t\treturn strings.Join(pack, \"\\n\"), nil\n\t}\n}\n\n// Pack up to maxJWTs into a message and invoke callback with it\nfunc (store *DirJWTStore) PackWalk(maxJWTs int, cb func(partialPackMsg string)) error {\n\tif maxJWTs <= 0 || cb == nil {\n\t\treturn errors.New(\"bad arguments to PackWalk\")\n\t}\n\tvar packMsg []string\n\tstore.Lock()\n\tdir := store.directory\n\texp := store.expiration\n\tstore.Unlock()\n\terr := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {\n\t\tif info != nil && !info.IsDir() && strings.HasSuffix(path, fileExtension) { // this is a JWT\n\t\t\tpubKey := strings.TrimSuffix(filepath.Base(path), fileExtension)\n\t\t\tstore.Lock()\n\t\t\tif exp != nil {\n\t\t\t\tif _, ok := exp.idx[pubKey]; !ok {\n\t\t\t\t\tstore.Unlock()\n\t\t\t\t\treturn nil // only include indexed files\n\t\t\t\t}\n\t\t\t}\n\t\t\tstore.Unlock()\n\t\t\tjwtBytes, err := os.ReadFile(path)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif len(jwtBytes) == 0 {\n\t\t\t\t// Skip if no contents in the JWT.\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif exp != nil {\n\t\t\t\tclaim, err := jwt.DecodeGeneric(string(jwtBytes))\n\t\t\t\tif err == nil && claim.Expires > 0 && claim.Expires < time.Now().Unix() {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\tpackMsg = append(packMsg, fmt.Sprintf(\"%s|%s\", pubKey, string(jwtBytes)))\n\t\t\tif len(packMsg) == maxJWTs { // won't match negative\n\t\t\t\tcb(strings.Join(packMsg, \"\\n\"))\n\t\t\t\tpackMsg = nil\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tif packMsg != nil {\n\t\tcb(strings.Join(packMsg, \"\\n\"))\n\t}\n\treturn err\n}\n\n// Merge takes the JWTs from package and adds them to the store\n// Merge is destructive in the sense that it doesn't check if the JWT\n// is newer or anything like that.\nfunc (store *DirJWTStore) Merge(pack string) error {\n\tnewJWTs := strings.Split(pack, \"\\n\")\n\tfor _, line := range newJWTs {\n\t\tif line == _EMPTY_ { // ignore blank lines\n\t\t\tcontinue\n\t\t}\n\t\tsplit := strings.Split(line, \"|\")\n\t\tif len(split) != 2 {\n\t\t\treturn fmt.Errorf(\"line in package didn't contain 2 entries: %q\", line)\n\t\t}\n\t\tpubKey := split[0]\n\t\tif !nkeys.IsValidPublicAccountKey(pubKey) {\n\t\t\treturn fmt.Errorf(\"key to merge is not a valid public account key\")\n\t\t}\n\t\tif err := store.saveIfNewer(pubKey, split[1]); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (store *DirJWTStore) Reload() error {\n\tstore.Lock()\n\texp := store.expiration\n\tif exp == nil || store.readonly {\n\t\tstore.Unlock()\n\t\treturn nil\n\t}\n\tidx := exp.idx\n\tchanged := store.changed\n\tisCache := store.expiration.evictOnLimit\n\t// clear out indexing data structures\n\texp.heap = make([]*jwtItem, 0, len(exp.heap))\n\texp.idx = make(map[string]*list.Element)\n\texp.lru = list.New()\n\texp.hash = [sha256.Size]byte{}\n\tstore.Unlock()\n\treturn filepath.Walk(store.directory, func(path string, info os.FileInfo, err error) error {\n\t\tif strings.HasSuffix(path, fileExtension) {\n\t\t\tif theJwt, err := os.ReadFile(path); err == nil {\n\t\t\t\thash := sha256.Sum256(theJwt)\n\t\t\t\t_, file := filepath.Split(path)\n\t\t\t\tpkey := strings.TrimSuffix(file, fileExtension)\n\t\t\t\tnotify := isCache // for cache, issue cb even when file not present (may have been evicted)\n\t\t\t\tif i, ok := idx[pkey]; ok {\n\t\t\t\t\tnotify = !bytes.Equal(i.Value.(*jwtItem).hash[:], hash[:])\n\t\t\t\t}\n\t\t\t\tstore.Lock()\n\t\t\t\texp.track(pkey, &hash, string(theJwt))\n\t\t\t\tstore.Unlock()\n\t\t\t\tif notify && changed != nil {\n\t\t\t\t\tchanged(pkey)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (store *DirJWTStore) pathForKey(publicKey string) string {\n\tif len(publicKey) < 2 {\n\t\treturn _EMPTY_\n\t}\n\tif !nkeys.IsValidPublicKey(publicKey) {\n\t\treturn _EMPTY_\n\t}\n\tfileName := fmt.Sprintf(\"%s%s\", publicKey, fileExtension)\n\tif store.shard {\n\t\tlast := publicKey[len(publicKey)-2:]\n\t\treturn filepath.Join(store.directory, last, fileName)\n\t} else {\n\t\treturn filepath.Join(store.directory, fileName)\n\t}\n}\n\n// Load checks the memory store and returns the matching JWT or an error\n// Assumes lock is NOT held\nfunc (store *DirJWTStore) load(publicKey string) (string, error) {\n\tstore.Lock()\n\tdefer store.Unlock()\n\tif path := store.pathForKey(publicKey); path == _EMPTY_ {\n\t\treturn _EMPTY_, fmt.Errorf(\"invalid public key\")\n\t} else if data, err := os.ReadFile(path); err != nil {\n\t\treturn _EMPTY_, err\n\t} else {\n\t\tif store.expiration != nil {\n\t\t\tstore.expiration.updateTrack(publicKey)\n\t\t}\n\t\treturn string(data), nil\n\t}\n}\n\n// write that keeps hash of all jwt in sync\n// Assumes the lock is held. Does return true or an error never both.\nfunc (store *DirJWTStore) write(path string, publicKey string, theJWT string) (bool, error) {\n\tif len(theJWT) == 0 {\n\t\treturn false, fmt.Errorf(\"invalid JWT\")\n\t}\n\tvar newHash *[sha256.Size]byte\n\tif store.expiration != nil {\n\t\th := sha256.Sum256([]byte(theJWT))\n\t\tnewHash = &h\n\t\tif v, ok := store.expiration.idx[publicKey]; ok {\n\t\t\tstore.expiration.updateTrack(publicKey)\n\t\t\t// this write is an update, move to back\n\t\t\tit := v.Value.(*jwtItem)\n\t\t\toldHash := it.hash[:]\n\t\t\tif bytes.Equal(oldHash, newHash[:]) {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t} else if int64(store.expiration.Len()) >= store.expiration.limit {\n\t\t\tif !store.expiration.evictOnLimit {\n\t\t\t\treturn false, errors.New(\"jwt store is full\")\n\t\t\t}\n\t\t\t// this write is an add, pick the least recently used value for removal\n\t\t\ti := store.expiration.lru.Front().Value.(*jwtItem)\n\t\t\tif err := os.Remove(store.pathForKey(i.publicKey)); err != nil {\n\t\t\t\treturn false, err\n\t\t\t} else {\n\t\t\t\tstore.expiration.unTrack(i.publicKey)\n\t\t\t}\n\t\t}\n\t}\n\tif err := os.WriteFile(path, []byte(theJWT), defaultFilePerms); err != nil {\n\t\treturn false, err\n\t} else if store.expiration != nil {\n\t\tstore.expiration.track(publicKey, newHash, theJWT)\n\t}\n\treturn true, nil\n}\n\nfunc (store *DirJWTStore) delete(publicKey string) error {\n\tif store.readonly {\n\t\treturn fmt.Errorf(\"store is read-only\")\n\t} else if store.deleteType == NoDelete {\n\t\treturn fmt.Errorf(\"store is not set up to for delete\")\n\t}\n\tstore.Lock()\n\tdefer store.Unlock()\n\tname := store.pathForKey(publicKey)\n\tif store.deleteType == RenameDeleted {\n\t\tif err := os.Rename(name, name+\".deleted\"); err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t} else if err := os.Remove(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\tstore.expiration.unTrack(publicKey)\n\tstore.deleted(publicKey)\n\treturn nil\n}\n\n// Save puts the JWT in a map by public key and performs update callbacks\n// Assumes lock is NOT held\nfunc (store *DirJWTStore) save(publicKey string, theJWT string) error {\n\tif store.readonly {\n\t\treturn fmt.Errorf(\"store is read-only\")\n\t}\n\tstore.Lock()\n\tpath := store.pathForKey(publicKey)\n\tif path == _EMPTY_ {\n\t\tstore.Unlock()\n\t\treturn fmt.Errorf(\"invalid public key\")\n\t}\n\tdirPath := filepath.Dir(path)\n\tif _, err := validateDirPath(dirPath); err != nil {\n\t\tif err := os.MkdirAll(dirPath, defaultDirPerms); err != nil {\n\t\t\tstore.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\tchanged, err := store.write(path, publicKey, theJWT)\n\tcb := store.changed\n\tstore.Unlock()\n\tif changed && cb != nil {\n\t\tcb(publicKey)\n\t}\n\treturn err\n}\n\n// Assumes the lock is NOT held, and only updates if the jwt is new, or the one on disk is older\n// When changed, invokes jwt changed callback\nfunc (store *DirJWTStore) saveIfNewer(publicKey string, theJWT string) error {\n\tif store.readonly {\n\t\treturn fmt.Errorf(\"store is read-only\")\n\t}\n\tpath := store.pathForKey(publicKey)\n\tif path == _EMPTY_ {\n\t\treturn fmt.Errorf(\"invalid public key\")\n\t}\n\tdirPath := filepath.Dir(path)\n\tif _, err := validateDirPath(dirPath); err != nil {\n\t\tif err := os.MkdirAll(dirPath, defaultDirPerms); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif _, err := os.Stat(path); err == nil {\n\t\tif newJWT, err := jwt.DecodeGeneric(theJWT); err != nil {\n\t\t\treturn err\n\t\t} else if existing, err := os.ReadFile(path); err != nil {\n\t\t\treturn err\n\t\t} else if existingJWT, err := jwt.DecodeGeneric(string(existing)); err != nil {\n\t\t\t// skip if it can't be decoded\n\t\t} else if existingJWT.ID == newJWT.ID {\n\t\t\treturn nil\n\t\t} else if existingJWT.IssuedAt > newJWT.IssuedAt {\n\t\t\treturn nil\n\t\t} else if newJWT.Subject != publicKey {\n\t\t\treturn fmt.Errorf(\"jwt subject nkey and provided nkey do not match\")\n\t\t} else if existingJWT.Subject != newJWT.Subject {\n\t\t\treturn fmt.Errorf(\"subject of existing and new jwt do not match\")\n\t\t}\n\t}\n\tstore.Lock()\n\tcb := store.changed\n\tchanged, err := store.write(path, publicKey, theJWT)\n\tstore.Unlock()\n\tif err != nil {\n\t\treturn err\n\t} else if changed && cb != nil {\n\t\tcb(publicKey)\n\t}\n\treturn nil\n}\n\nfunc xorAssign(lVal *[sha256.Size]byte, rVal [sha256.Size]byte) {\n\tfor i := range rVal {\n\t\t(*lVal)[i] ^= rVal[i]\n\t}\n}\n\n// returns a hash representing all indexed jwt\nfunc (store *DirJWTStore) Hash() [sha256.Size]byte {\n\tstore.Lock()\n\tdefer store.Unlock()\n\tif store.expiration == nil {\n\t\treturn [sha256.Size]byte{}\n\t} else {\n\t\treturn store.expiration.hash\n\t}\n}\n\n// An jwtItem is something managed by the priority queue\ntype jwtItem struct {\n\tindex int\n\tpublicKey string\n\texpiration int64 // consists of unix time of expiration (ttl when set or jwt expiration) in seconds\n\thash [sha256.Size]byte\n}\n\n// A expirationTracker implements heap.Interface and holds Items.\ntype expirationTracker struct {\n\theap []*jwtItem // sorted by jwtItem.expiration\n\tidx map[string]*list.Element\n\tlru *list.List // keep which jwt are least used\n\tlimit int64 // limit how many jwt are being tracked\n\tevictOnLimit bool // when limit is hit, error or evict using lru\n\tttl time.Duration\n\thash [sha256.Size]byte // xor of all jwtItem.hash in idx\n\tquit chan struct{}\n\twg sync.WaitGroup\n}\n\nfunc (q *expirationTracker) Len() int { return len(q.heap) }\n\nfunc (q *expirationTracker) Less(i, j int) bool {\n\tpq := q.heap\n\treturn pq[i].expiration < pq[j].expiration\n}\n\nfunc (q *expirationTracker) Swap(i, j int) {\n\tpq := q.heap\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].index = i\n\tpq[j].index = j\n}\n\nfunc (q *expirationTracker) Push(x any) {\n\tn := len(q.heap)\n\titem := x.(*jwtItem)\n\titem.index = n\n\tq.heap = append(q.heap, item)\n\tq.idx[item.publicKey] = q.lru.PushBack(item)\n}\n\nfunc (q *expirationTracker) Pop() any {\n\told := q.heap\n\tn := len(old)\n\titem := old[n-1]\n\told[n-1] = nil // avoid memory leak\n\titem.index = -1\n\tq.heap = old[0 : n-1]\n\tq.lru.Remove(q.idx[item.publicKey])\n\tdelete(q.idx, item.publicKey)\n\treturn item\n}\n\nfunc (pq *expirationTracker) updateTrack(publicKey string) {\n\tif e, ok := pq.idx[publicKey]; ok {\n\t\ti := e.Value.(*jwtItem)\n\t\tif pq.ttl != 0 {\n\t\t\t// only update expiration when set\n\t\t\ti.expiration = time.Now().Add(pq.ttl).UnixNano()\n\t\t\theap.Fix(pq, i.index)\n\t\t}\n\t\tif pq.evictOnLimit {\n\t\t\tpq.lru.MoveToBack(e)\n\t\t}\n\t}\n}\n\nfunc (pq *expirationTracker) unTrack(publicKey string) {\n\tif it, ok := pq.idx[publicKey]; ok {\n\t\txorAssign(&pq.hash, it.Value.(*jwtItem).hash)\n\t\theap.Remove(pq, it.Value.(*jwtItem).index)\n\t\tdelete(pq.idx, publicKey)\n\t}\n}\n\nfunc (pq *expirationTracker) track(publicKey string, hash *[sha256.Size]byte, theJWT string) {\n\tvar exp int64\n\t// prioritize ttl over expiration\n\tif pq.ttl != 0 {\n\t\tif pq.ttl == time.Duration(math.MaxInt64) {\n\t\t\texp = math.MaxInt64\n\t\t} else {\n\t\t\texp = time.Now().Add(pq.ttl).UnixNano()\n\t\t}\n\t} else {\n\t\tif g, err := jwt.DecodeGeneric(theJWT); err == nil {\n\t\t\texp = time.Unix(g.Expires, 0).UnixNano()\n\t\t}\n\t\tif exp == 0 {\n\t\t\texp = math.MaxInt64 // default to indefinite\n\t\t}\n\t}\n\tif e, ok := pq.idx[publicKey]; ok {\n\t\ti := e.Value.(*jwtItem)\n\t\txorAssign(&pq.hash, i.hash) // remove old hash\n\t\ti.expiration = exp\n\t\ti.hash = *hash\n\t\theap.Fix(pq, i.index)\n\t} else {\n\t\theap.Push(pq, &jwtItem{-1, publicKey, exp, *hash})\n\t}\n\txorAssign(&pq.hash, *hash) // add in new hash\n}\n\nfunc (pq *expirationTracker) close() {\n\tif pq == nil || pq.quit == nil {\n\t\treturn\n\t}\n\tclose(pq.quit)\n\tpq.quit = nil\n}\n\nfunc (store *DirJWTStore) startExpiring(reCheck time.Duration, limit int64, evictOnLimit bool, ttl time.Duration) {\n\tstore.Lock()\n\tdefer store.Unlock()\n\tquit := make(chan struct{})\n\tpq := &expirationTracker{\n\t\tmake([]*jwtItem, 0, 10),\n\t\tmake(map[string]*list.Element),\n\t\tlist.New(),\n\t\tlimit,\n\t\tevictOnLimit,\n\t\tttl,\n\t\t[sha256.Size]byte{},\n\t\tquit,\n\t\tsync.WaitGroup{},\n\t}\n\tstore.expiration = pq\n\tpq.wg.Add(1)\n\tgo func() {\n\t\tt := time.NewTicker(reCheck)\n\t\tdefer t.Stop()\n\t\tdefer pq.wg.Done()\n\t\tfor {\n\t\t\tnow := time.Now().UnixNano()\n\t\t\tstore.Lock()\n\t\t\tif pq.Len() > 0 {\n\t\t\t\tif it := pq.heap[0]; it.expiration <= now {\n\t\t\t\t\tpath := store.pathForKey(it.publicKey)\n\t\t\t\t\tif err := os.Remove(path); err == nil {\n\t\t\t\t\t\theap.Pop(pq)\n\t\t\t\t\t\tpq.unTrack(it.publicKey)\n\t\t\t\t\t\txorAssign(&pq.hash, it.hash)\n\t\t\t\t\t\tstore.Unlock()\n\t\t\t\t\t\tcontinue // we removed an entry, check next one right away\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tstore.Unlock()\n\t\t\tselect {\n\t\t\tcase <-t.C:\n\t\t\tcase <-quit:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n}\n" }, { "path": "server/dirstore_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nvar (\n\tone, two, three, four = \"\", \"\", \"\", \"\"\n\tjwt1, jwt2, jwt3, jwt4 = \"\", \"\", \"\", \"\"\n\top nkeys.KeyPair\n)\n\nfunc init() {\n\top, _ = nkeys.CreateOperator()\n\n\tnkone, _ := nkeys.CreateAccount()\n\tpub, _ := nkone.PublicKey()\n\tone = pub\n\tac := jwt.NewAccountClaims(pub)\n\tjwt1, _ = ac.Encode(op)\n\n\tnktwo, _ := nkeys.CreateAccount()\n\tpub, _ = nktwo.PublicKey()\n\ttwo = pub\n\tac = jwt.NewAccountClaims(pub)\n\tjwt2, _ = ac.Encode(op)\n\n\tnkthree, _ := nkeys.CreateAccount()\n\tpub, _ = nkthree.PublicKey()\n\tthree = pub\n\tac = jwt.NewAccountClaims(pub)\n\tjwt3, _ = ac.Encode(op)\n\n\tnkfour, _ := nkeys.CreateAccount()\n\tpub, _ = nkfour.PublicKey()\n\tfour = pub\n\tac = jwt.NewAccountClaims(pub)\n\tjwt4, _ = ac.Encode(op)\n}\n\nfunc TestShardedDirStoreWriteAndReadonly(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tstore, err := NewDirJWTStore(dir, true, false)\n\trequire_NoError(t, err)\n\n\texpected := map[string]string{\n\t\tone: \"alpha\",\n\t\ttwo: \"beta\",\n\t\tthree: \"gamma\",\n\t\tfour: \"delta\",\n\t}\n\n\tfor k, v := range expected {\n\t\tstore.SaveAcc(k, v)\n\t}\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err := store.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\tgot, err = store.LoadAcc(\"\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\terr = store.SaveAcc(\"\", \"onetwothree\")\n\trequire_Error(t, err)\n\tstore.Close()\n\n\t// re-use the folder for readonly mode\n\tstore, err = NewImmutableDirJWTStore(dir, true)\n\trequire_NoError(t, err)\n\n\trequire_True(t, store.IsReadOnly())\n\n\terr = store.SaveAcc(\"five\", \"omega\")\n\trequire_Error(t, err)\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\tstore.Close()\n}\n\nfunc TestUnshardedDirStoreWriteAndReadonly(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tstore, err := NewDirJWTStore(dir, false, false)\n\trequire_NoError(t, err)\n\n\texpected := map[string]string{\n\t\tone: \"alpha\",\n\t\ttwo: \"beta\",\n\t\tthree: \"gamma\",\n\t\tfour: \"delta\",\n\t}\n\n\trequire_False(t, store.IsReadOnly())\n\n\tfor k, v := range expected {\n\t\tstore.SaveAcc(k, v)\n\t}\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err := store.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\tgot, err = store.LoadAcc(\"\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\terr = store.SaveAcc(\"\", \"onetwothree\")\n\trequire_Error(t, err)\n\tstore.Close()\n\n\t// re-use the folder for readonly mode\n\tstore, err = NewImmutableDirJWTStore(dir, false)\n\trequire_NoError(t, err)\n\n\trequire_True(t, store.IsReadOnly())\n\n\terr = store.SaveAcc(\"five\", \"omega\")\n\trequire_Error(t, err)\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\tstore.Close()\n}\n\nfunc TestNoCreateRequiresDir(t *testing.T) {\n\tt.Parallel()\n\t_, err := NewDirJWTStore(\"/a/b/c\", true, false)\n\trequire_Error(t, err)\n}\n\nfunc TestCreateMakesDir(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tfullPath := filepath.Join(dir, \"a/b\")\n\n\t_, err := os.Stat(fullPath)\n\trequire_Error(t, err)\n\trequire_True(t, os.IsNotExist(err))\n\n\ts, err := NewDirJWTStore(fullPath, false, true)\n\trequire_NoError(t, err)\n\ts.Close()\n\n\t_, err = os.Stat(fullPath)\n\trequire_NoError(t, err)\n}\n\nfunc TestShardedDirStorePackMerge(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\tdir2 := t.TempDir()\n\tdir3 := t.TempDir()\n\n\tstore, err := NewDirJWTStore(dir, true, false)\n\trequire_NoError(t, err)\n\n\texpected := map[string]string{\n\t\tone: \"alpha\",\n\t\ttwo: \"beta\",\n\t\tthree: \"gamma\",\n\t\tfour: \"delta\",\n\t}\n\n\trequire_False(t, store.IsReadOnly())\n\n\tfor k, v := range expected {\n\t\tstore.SaveAcc(k, v)\n\t}\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err := store.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\tpack, err := store.Pack(-1)\n\trequire_NoError(t, err)\n\n\tinc, err := NewDirJWTStore(dir2, true, false)\n\trequire_NoError(t, err)\n\n\tinc.Merge(pack)\n\n\tfor k, v := range expected {\n\t\tgot, err := inc.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err = inc.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\tlimitedPack, err := inc.Pack(1)\n\trequire_NoError(t, err)\n\n\tlimited, err := NewDirJWTStore(dir3, true, false)\n\n\trequire_NoError(t, err)\n\n\tlimited.Merge(limitedPack)\n\n\tcount := 0\n\tfor k, v := range expected {\n\t\tgot, err := limited.LoadAcc(k)\n\t\tif err == nil {\n\t\t\tcount++\n\t\t\trequire_Equal(t, v, got)\n\t\t}\n\t}\n\n\trequire_Len(t, 1, count)\n\n\tgot, err = inc.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n}\n\nfunc TestShardedToUnsharedDirStorePackMerge(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\tdir2 := t.TempDir()\n\n\tstore, err := NewDirJWTStore(dir, true, false)\n\trequire_NoError(t, err)\n\n\texpected := map[string]string{\n\t\tone: \"alpha\",\n\t\ttwo: \"beta\",\n\t\tthree: \"gamma\",\n\t\tfour: \"delta\",\n\t}\n\n\trequire_False(t, store.IsReadOnly())\n\n\tfor k, v := range expected {\n\t\tstore.SaveAcc(k, v)\n\t}\n\n\tfor k, v := range expected {\n\t\tgot, err := store.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err := store.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\tpack, err := store.Pack(-1)\n\trequire_NoError(t, err)\n\n\tinc, err := NewDirJWTStore(dir2, false, false)\n\trequire_NoError(t, err)\n\n\tinc.Merge(pack)\n\n\tfor k, v := range expected {\n\t\tgot, err := inc.LoadAcc(k)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, v, got)\n\t}\n\n\tgot, err = inc.LoadAcc(\"random\")\n\trequire_Error(t, err)\n\trequire_Equal(t, \"\", got)\n\n\terr = store.Merge(\"foo\")\n\trequire_Error(t, err)\n\n\terr = store.Merge(\"\") // will skip it\n\trequire_NoError(t, err)\n\n\terr = store.Merge(\"a|something\") // should fail on a for sharding\n\trequire_Error(t, err)\n}\n\nfunc TestMergeOnlyOnNewer(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewDirJWTStore(dir, true, false)\n\trequire_NoError(t, err)\n\n\taccountKey, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\n\tpubKey, err := accountKey.PublicKey()\n\trequire_NoError(t, err)\n\n\taccount := jwt.NewAccountClaims(pubKey)\n\taccount.Name = \"old\"\n\tolderJWT, err := account.Encode(accountKey)\n\trequire_NoError(t, err)\n\n\ttime.Sleep(2 * time.Second)\n\n\taccount.Name = \"new\"\n\tnewerJWT, err := account.Encode(accountKey)\n\trequire_NoError(t, err)\n\n\t// Should work\n\terr = dirStore.SaveAcc(pubKey, olderJWT)\n\trequire_NoError(t, err)\n\tfromStore, err := dirStore.LoadAcc(pubKey)\n\trequire_NoError(t, err)\n\trequire_Equal(t, olderJWT, fromStore)\n\n\t// should replace\n\terr = dirStore.saveIfNewer(pubKey, newerJWT)\n\trequire_NoError(t, err)\n\tfromStore, err = dirStore.LoadAcc(pubKey)\n\trequire_NoError(t, err)\n\trequire_Equal(t, newerJWT, fromStore)\n\n\t// should fail\n\terr = dirStore.saveIfNewer(pubKey, olderJWT)\n\trequire_NoError(t, err)\n\tfromStore, err = dirStore.LoadAcc(pubKey)\n\trequire_NoError(t, err)\n\trequire_Equal(t, newerJWT, fromStore)\n}\n\nfunc createTestAccount(t *testing.T, dirStore *DirJWTStore, expSec int, accKey nkeys.KeyPair) string {\n\tt.Helper()\n\tpubKey, err := accKey.PublicKey()\n\trequire_NoError(t, err)\n\taccount := jwt.NewAccountClaims(pubKey)\n\tif expSec > 0 {\n\t\taccount.Expires = time.Now().Round(time.Second).Add(time.Second * time.Duration(expSec)).Unix()\n\t}\n\tjwt, err := account.Encode(accKey)\n\trequire_NoError(t, err)\n\terr = dirStore.SaveAcc(pubKey, jwt)\n\trequire_NoError(t, err)\n\treturn jwt\n}\n\nfunc assertStoreSize(t *testing.T, dirStore *DirJWTStore, length int) {\n\tt.Helper()\n\tf, err := os.ReadDir(dirStore.directory)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(f), length)\n\tdirStore.Lock()\n\trequire_Len(t, len(dirStore.expiration.idx), length)\n\trequire_Len(t, dirStore.expiration.lru.Len(), length)\n\trequire_Len(t, len(dirStore.expiration.heap), length)\n\tdirStore.Unlock()\n}\n\nfunc TestExpiration(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccount := func(expSec int) {\n\t\taccountKey, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tcreateTestAccount(t, dirStore, expSec, accountKey)\n\t}\n\n\thBegin := dirStore.Hash()\n\taccount(100)\n\thNoExp := dirStore.Hash()\n\trequire_NotEqual(t, hBegin, hNoExp)\n\taccount(1)\n\tnh2 := dirStore.Hash()\n\trequire_NotEqual(t, hNoExp, nh2)\n\tassertStoreSize(t, dirStore, 2)\n\n\tfailAt := time.Now().Add(4 * time.Second)\n\tfor time.Now().Before(failAt) {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tf, err := os.ReadDir(dir)\n\t\trequire_NoError(t, err)\n\t\tif len(f) == 1 {\n\t\t\tlh := dirStore.Hash()\n\t\t\trequire_Equal(t, string(hNoExp[:]), string(lh[:]))\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatalf(\"Waited more than 4 seconds for the file with expiration 1 second to expire\")\n}\n\nfunc TestLimit(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 5, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccount := func(expSec int) {\n\t\taccountKey, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tcreateTestAccount(t, dirStore, expSec, accountKey)\n\t}\n\n\th := dirStore.Hash()\n\n\taccountKey, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\t// update first account\n\tfor i := 0; i < 10; i++ {\n\t\tcreateTestAccount(t, dirStore, 50, accountKey)\n\t\tassertStoreSize(t, dirStore, 1)\n\t}\n\t// new accounts\n\tfor i := 0; i < 10; i++ {\n\t\taccount(i)\n\t\tnh := dirStore.Hash()\n\t\trequire_NotEqual(t, h, nh)\n\t\th = nh\n\t}\n\t// first account should be gone now accountKey.PublicKey()\n\tkey, _ := accountKey.PublicKey()\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, key))\n\trequire_True(t, os.IsNotExist(err))\n\n\t// update first account\n\tfor i := 0; i < 10; i++ {\n\t\tcreateTestAccount(t, dirStore, 50, accountKey)\n\t\tassertStoreSize(t, dirStore, 5)\n\t}\n}\n\nfunc TestLimitNoEvict(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, false, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccountKey1, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey1, err := accountKey1.PublicKey()\n\trequire_NoError(t, err)\n\taccountKey2, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taccountKey3, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey3, err := accountKey3.PublicKey()\n\trequire_NoError(t, err)\n\n\tcreateTestAccount(t, dirStore, 100, accountKey1)\n\tassertStoreSize(t, dirStore, 1)\n\tcreateTestAccount(t, dirStore, 1, accountKey2)\n\tassertStoreSize(t, dirStore, 2)\n\n\thBefore := dirStore.Hash()\n\t// 2 jwt are already stored. third must result in an error\n\tpubKey, err := accountKey3.PublicKey()\n\trequire_NoError(t, err)\n\taccount := jwt.NewAccountClaims(pubKey)\n\tjwt, err := account.Encode(accountKey3)\n\trequire_NoError(t, err)\n\terr = dirStore.SaveAcc(pubKey, jwt)\n\trequire_Error(t, err)\n\tassertStoreSize(t, dirStore, 2)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey1))\n\trequire_NoError(t, err)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey3))\n\trequire_True(t, os.IsNotExist(err))\n\t// check that the hash did not change\n\thAfter := dirStore.Hash()\n\trequire_True(t, bytes.Equal(hBefore[:], hAfter[:]))\n\t// wait for expiration of account2\n\ttime.Sleep(2200 * time.Millisecond)\n\terr = dirStore.SaveAcc(pubKey, jwt)\n\trequire_NoError(t, err)\n\tassertStoreSize(t, dirStore, 2)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey1))\n\trequire_NoError(t, err)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey3))\n\trequire_NoError(t, err)\n}\n\nfunc TestLruLoad(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccountKey1, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey1, err := accountKey1.PublicKey()\n\trequire_NoError(t, err)\n\taccountKey2, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taccountKey3, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey3, err := accountKey3.PublicKey()\n\trequire_NoError(t, err)\n\n\tcreateTestAccount(t, dirStore, 10, accountKey1)\n\tassertStoreSize(t, dirStore, 1)\n\tcreateTestAccount(t, dirStore, 10, accountKey2)\n\tassertStoreSize(t, dirStore, 2)\n\tdirStore.LoadAcc(pKey1) // will reorder 1/2\n\tcreateTestAccount(t, dirStore, 10, accountKey3)\n\tassertStoreSize(t, dirStore, 2)\n\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey1))\n\trequire_NoError(t, err)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey3))\n\trequire_NoError(t, err)\n}\n\nfunc TestLruVolume(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\treplaceCnt := 500 // needs to be bigger than 2 due to loop unrolling\n\tkeys := make([]string, replaceCnt)\n\n\tkey, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tkeys[0], err = key.PublicKey()\n\trequire_NoError(t, err)\n\tcreateTestAccount(t, dirStore, 10000, key) // not intended to expire\n\tassertStoreSize(t, dirStore, 1)\n\n\tkey, err = nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tkeys[1], err = key.PublicKey()\n\trequire_NoError(t, err)\n\tcreateTestAccount(t, dirStore, 10000, key)\n\tassertStoreSize(t, dirStore, 2)\n\n\tfor i := 2; i < replaceCnt; i++ {\n\t\tk, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tkeys[i], err = k.PublicKey()\n\t\trequire_NoError(t, err)\n\n\t\tcreateTestAccount(t, dirStore, 10000+rand.Intn(10000), k) // not intended to expire\n\t\tassertStoreSize(t, dirStore, 2)\n\t\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, keys[i-2]))\n\t\trequire_Error(t, err)\n\t\trequire_True(t, os.IsNotExist(err))\n\t\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, keys[i-1]))\n\t\trequire_NoError(t, err)\n\t\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, keys[i]))\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestLru(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 2, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccountKey1, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey1, err := accountKey1.PublicKey()\n\trequire_NoError(t, err)\n\taccountKey2, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taccountKey3, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tpKey3, err := accountKey3.PublicKey()\n\trequire_NoError(t, err)\n\n\tcreateTestAccount(t, dirStore, 1000, accountKey1)\n\tassertStoreSize(t, dirStore, 1)\n\tcreateTestAccount(t, dirStore, 1000, accountKey2)\n\tassertStoreSize(t, dirStore, 2)\n\tcreateTestAccount(t, dirStore, 1000, accountKey3)\n\tassertStoreSize(t, dirStore, 2)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey1))\n\trequire_Error(t, err)\n\trequire_True(t, os.IsNotExist(err))\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey3))\n\trequire_NoError(t, err)\n\n\t// update -> will change this keys position for eviction\n\tcreateTestAccount(t, dirStore, 1000, accountKey2)\n\tassertStoreSize(t, dirStore, 2)\n\t// recreate -> will evict 3\n\tcreateTestAccount(t, dirStore, 1, accountKey1)\n\tassertStoreSize(t, dirStore, 2)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey3))\n\trequire_True(t, os.IsNotExist(err))\n\t// let key1 expire. sleep expSec=1 + 1 for rounding\n\ttime.Sleep(2200 * time.Millisecond)\n\tassertStoreSize(t, dirStore, 1)\n\t_, err = os.Stat(fmt.Sprintf(\"%s/%s.jwt\", dir, pKey1))\n\trequire_True(t, os.IsNotExist(err))\n\t// recreate key3 - no eviction\n\tcreateTestAccount(t, dirStore, 1000, accountKey3)\n\tassertStoreSize(t, dirStore, 2)\n}\n\nfunc TestReload(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\tnotificationChan := make(chan struct{}, 5)\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, func(publicKey string) {\n\t\tnotificationChan <- struct{}{}\n\t})\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\tnewAccount := func() string {\n\t\tt.Helper()\n\t\taccKey, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tpKey, err := accKey.PublicKey()\n\t\trequire_NoError(t, err)\n\t\tpubKey, err := accKey.PublicKey()\n\t\trequire_NoError(t, err)\n\t\taccount := jwt.NewAccountClaims(pubKey)\n\t\tjwt, err := account.Encode(accKey)\n\t\trequire_NoError(t, err)\n\t\tfile := fmt.Sprintf(\"%s/%s.jwt\", dir, pKey)\n\t\terr = os.WriteFile(file, []byte(jwt), 0644)\n\t\trequire_NoError(t, err)\n\t\treturn file\n\t}\n\tfiles := make(map[string]struct{})\n\tassertStoreSize(t, dirStore, 0)\n\thash := dirStore.Hash()\n\temptyHash := [sha256.Size]byte{}\n\trequire_True(t, bytes.Equal(hash[:], emptyHash[:]))\n\tfor i := 0; i < 5; i++ {\n\t\tfiles[newAccount()] = struct{}{}\n\t\terr = dirStore.Reload()\n\t\trequire_NoError(t, err)\n\t\t<-notificationChan\n\t\tassertStoreSize(t, dirStore, i+1)\n\t\thash = dirStore.Hash()\n\t\trequire_False(t, bytes.Equal(hash[:], emptyHash[:]))\n\t\tmsg, err := dirStore.Pack(-1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(strings.Split(msg, \"\\n\")), len(files))\n\t}\n\tfor k := range files {\n\t\thash = dirStore.Hash()\n\t\trequire_False(t, bytes.Equal(hash[:], emptyHash[:]))\n\t\tremoveFile(t, k)\n\t\terr = dirStore.Reload()\n\t\trequire_NoError(t, err)\n\t\tassertStoreSize(t, dirStore, len(files)-1)\n\t\tdelete(files, k)\n\t\tmsg, err := dirStore.Pack(-1)\n\t\trequire_NoError(t, err)\n\t\tif len(files) != 0 { // when len is 0, we have an empty line\n\t\t\trequire_Len(t, len(strings.Split(msg, \"\\n\")), len(files))\n\t\t}\n\t}\n\trequire_True(t, len(notificationChan) == 0)\n\thash = dirStore.Hash()\n\trequire_True(t, bytes.Equal(hash[:], emptyHash[:]))\n}\n\nfunc TestExpirationUpdate(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\n\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*50, 10, true, 0, nil)\n\trequire_NoError(t, err)\n\tdefer dirStore.Close()\n\n\taccountKey, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\n\th := dirStore.Hash()\n\n\tcreateTestAccount(t, dirStore, 0, accountKey)\n\tnh := dirStore.Hash()\n\trequire_NotEqual(t, h, nh)\n\th = nh\n\n\ttime.Sleep(1500 * time.Millisecond)\n\tf, err := os.ReadDir(dir)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(f), 1)\n\n\tcreateTestAccount(t, dirStore, 2, accountKey)\n\tnh = dirStore.Hash()\n\trequire_NotEqual(t, h, nh)\n\th = nh\n\n\ttime.Sleep(1500 * time.Millisecond)\n\tf, err = os.ReadDir(dir)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(f), 1)\n\n\tcreateTestAccount(t, dirStore, 0, accountKey)\n\tnh = dirStore.Hash()\n\trequire_NotEqual(t, h, nh)\n\th = nh\n\n\ttime.Sleep(1500 * time.Millisecond)\n\tf, err = os.ReadDir(dir)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(f), 1)\n\n\tcreateTestAccount(t, dirStore, 1, accountKey)\n\tnh = dirStore.Hash()\n\trequire_NotEqual(t, h, nh)\n\n\ttime.Sleep(1500 * time.Millisecond)\n\tf, err = os.ReadDir(dir)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(f), 0)\n\n\tempty := [32]byte{}\n\th = dirStore.Hash()\n\trequire_Equal(t, string(h[:]), string(empty[:]))\n}\n\nfunc TestTTL(t *testing.T) {\n\tt.Parallel()\n\tdir := t.TempDir()\n\trequire_OneJWT := func() {\n\t\tt.Helper()\n\t\tf, err := os.ReadDir(dir)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(f), 1)\n\t}\n\ttest := func(op func(store *DirJWTStore, accountKey nkeys.KeyPair, accountPubKey string, jwt string)) {\n\t\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, 50*time.Millisecond, 10, true, 200*time.Millisecond, nil)\n\t\trequire_NoError(t, err)\n\t\tdefer dirStore.Close()\n\n\t\taccountKey, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tpubKey, err := accountKey.PublicKey()\n\t\trequire_NoError(t, err)\n\t\tjwt := createTestAccount(t, dirStore, 0, accountKey)\n\t\trequire_OneJWT()\n\t\t// observe non expiration due to activity\n\t\tfor i := 0; i < 4; i++ {\n\t\t\ttime.Sleep(110 * time.Millisecond)\n\t\t\top(dirStore, accountKey, pubKey, jwt)\n\t\t\trequire_OneJWT()\n\t\t}\n\t\t// observe expiration\n\t\tfor i := 0; i < 40; i++ {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tf, err := os.ReadDir(dir)\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(f) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tt.Fatalf(\"jwt should have expired by now\")\n\t}\n\tt.Run(\"no expiration due to load\", func(t *testing.T) {\n\t\ttest(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) {\n\t\t\tstore.LoadAcc(pubKey)\n\t\t})\n\t})\n\tt.Run(\"no expiration due to store\", func(t *testing.T) {\n\t\ttest(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) {\n\t\t\tstore.SaveAcc(pubKey, jwt)\n\t\t})\n\t})\n\tt.Run(\"no expiration due to overwrite\", func(t *testing.T) {\n\t\ttest(func(store *DirJWTStore, accountKey nkeys.KeyPair, pubKey string, jwt string) {\n\t\t\tcreateTestAccount(t, store, 0, accountKey)\n\t\t})\n\t})\n}\n\nfunc TestRemove(t *testing.T) {\n\tfor deleteType, test := range map[deleteType]struct {\n\t\texpected int\n\t\tmoved int\n\t}{\n\t\tHardDelete: {0, 0},\n\t\tRenameDeleted: {0, 1},\n\t\tNoDelete: {1, 0},\n\t} {\n\t\tdeleteType, test := deleteType, test // fixes govet capturing loop variables\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\tdir := t.TempDir()\n\t\t\trequire_OneJWT := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tf, err := os.ReadDir(dir)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(f), 1)\n\t\t\t}\n\t\t\tdirStore, err := NewExpiringDirJWTStore(dir, false, false, deleteType, 0, 10, true, 0, nil)\n\t\t\tdelPubKey := \"\"\n\t\t\tdirStore.deleted = func(publicKey string) {\n\t\t\t\tdelPubKey = publicKey\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dirStore.Close()\n\t\t\taccountKey, err := nkeys.CreateAccount()\n\t\t\trequire_NoError(t, err)\n\t\t\tpubKey, err := accountKey.PublicKey()\n\t\t\trequire_NoError(t, err)\n\t\t\tcreateTestAccount(t, dirStore, 0, accountKey)\n\t\t\trequire_OneJWT()\n\t\t\tdirStore.delete(pubKey)\n\t\t\tif deleteType == NoDelete {\n\t\t\t\trequire_True(t, delPubKey == \"\")\n\t\t\t} else {\n\t\t\t\trequire_True(t, delPubKey == pubKey)\n\t\t\t}\n\t\t\tf, err := filepath.Glob(dir + string(os.PathSeparator) + \"/*.jwt\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(f), test.expected)\n\t\t\tf, err = filepath.Glob(dir + string(os.PathSeparator) + \"/*.jwt.deleted\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(f), test.moved)\n\t\t})\n\t}\n}\n\nconst infDur = time.Duration(math.MaxInt64)\n\nfunc TestNotificationOnPack(t *testing.T) {\n\tt.Parallel()\n\tjwts := map[string]string{\n\t\tone: jwt1,\n\t\ttwo: jwt2,\n\t\tthree: jwt3,\n\t\tfour: jwt4,\n\t}\n\tnotificationChan := make(chan struct{}, len(jwts)) // set to same len so all extra will block\n\tnotification := func(pubKey string) {\n\t\tif _, ok := jwts[pubKey]; !ok {\n\t\t\tt.Fatalf(\"Key not found: %s\", pubKey)\n\t\t}\n\t\tnotificationChan <- struct{}{}\n\t}\n\tdirPack := t.TempDir()\n\tpackStore, err := NewExpiringDirJWTStore(dirPack, false, false, NoDelete, infDur, 0, true, 0, notification)\n\trequire_NoError(t, err)\n\t// prefill the store with data\n\tfor k, v := range jwts {\n\t\trequire_NoError(t, packStore.SaveAcc(k, v))\n\t}\n\tfor i := 0; i < len(jwts); i++ {\n\t\t<-notificationChan\n\t}\n\tmsg, err := packStore.Pack(-1)\n\trequire_NoError(t, err)\n\tpackStore.Close()\n\thash := packStore.Hash()\n\tfor _, shard := range []bool{true, false, true, false} {\n\t\tdirMerge := t.TempDir()\n\t\tmergeStore, err := NewExpiringDirJWTStore(dirMerge, shard, false, NoDelete, infDur, 0, true, 0, notification)\n\t\trequire_NoError(t, err)\n\t\t// set\n\t\terr = mergeStore.Merge(msg)\n\t\trequire_NoError(t, err)\n\t\tassertStoreSize(t, mergeStore, len(jwts))\n\t\thash1 := packStore.Hash()\n\t\trequire_True(t, bytes.Equal(hash[:], hash1[:]))\n\t\tfor i := 0; i < len(jwts); i++ {\n\t\t\t<-notificationChan\n\t\t}\n\t\t// overwrite - assure\n\t\terr = mergeStore.Merge(msg)\n\t\trequire_NoError(t, err)\n\t\tassertStoreSize(t, mergeStore, len(jwts))\n\t\thash2 := packStore.Hash()\n\t\trequire_True(t, bytes.Equal(hash1[:], hash2[:]))\n\n\t\thash = hash1\n\t\tmsg, err = mergeStore.Pack(-1)\n\t\trequire_NoError(t, err)\n\t\tmergeStore.Close()\n\t\trequire_True(t, len(notificationChan) == 0)\n\n\t\tfor k, v := range jwts {\n\t\t\tj, err := packStore.LoadAcc(k)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, j, v)\n\t\t}\n\t}\n}\n\nfunc TestNotificationOnPackWalk(t *testing.T) {\n\tt.Parallel()\n\tconst storeCnt = 5\n\tconst keyCnt = 50\n\tconst iterCnt = 8\n\tstore := [storeCnt]*DirJWTStore{}\n\tfor i := 0; i < storeCnt; i++ {\n\t\tdirMerge := t.TempDir()\n\t\tmergeStore, err := NewExpiringDirJWTStore(dirMerge, true, false, NoDelete, infDur, 0, true, 0, nil)\n\t\trequire_NoError(t, err)\n\t\tstore[i] = mergeStore\n\t}\n\tfor i := 0; i < iterCnt; i++ { //iterations\n\t\tjwts := make(map[string]string)\n\t\tfor j := 0; j < keyCnt; j++ {\n\t\t\tkp, _ := nkeys.CreateAccount()\n\t\t\tkey, _ := kp.PublicKey()\n\t\t\tac := jwt.NewAccountClaims(key)\n\t\t\tjwts[key], _ = ac.Encode(op)\n\t\t\trequire_NoError(t, store[0].SaveAcc(key, jwts[key]))\n\t\t}\n\t\tfor j := 0; j < storeCnt-1; j++ { // stores\n\t\t\terr := store[j].PackWalk(3, func(partialPackMsg string) {\n\t\t\t\terr := store[j+1].Merge(partialPackMsg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < storeCnt-1; i++ {\n\t\t\th1 := store[i].Hash()\n\t\t\th2 := store[i+1].Hash()\n\t\t\trequire_True(t, bytes.Equal(h1[:], h2[:]))\n\t\t}\n\t}\n\tfor i := 0; i < storeCnt; i++ {\n\t\tstore[i].Close()\n\t}\n}\n" }, { "path": "server/disk_avail.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows && !openbsd && !netbsd && !wasm && !illumos && !solaris\n\npackage server\n\nimport (\n\t\"os\"\n\t\"syscall\"\n)\n\nfunc diskAvailable(storeDir string) int64 {\n\tvar ba int64\n\tif _, err := os.Stat(storeDir); os.IsNotExist(err) {\n\t\tos.MkdirAll(storeDir, defaultDirPerms)\n\t}\n\tvar fs syscall.Statfs_t\n\tif err := syscall.Statfs(storeDir, &fs); err == nil {\n\t\t// Estimate 75% of available storage.\n\t\tba = int64(uint64(fs.Bavail) * uint64(fs.Bsize) / 4 * 3)\n\t} else {\n\t\t// Used 1TB default as a guess if all else fails.\n\t\tba = JetStreamMaxStoreDefault\n\t}\n\treturn ba\n}\n" }, { "path": "server/disk_avail_netbsd.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build netbsd\n\npackage server\n\n// TODO - See if there is a version of this for NetBSD.\nfunc diskAvailable(storeDir string) int64 {\n\treturn JetStreamMaxStoreDefault\n}\n" }, { "path": "server/disk_avail_openbsd.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build openbsd\n\npackage server\n\nimport (\n\t\"os\"\n\t\"syscall\"\n)\n\nfunc diskAvailable(storeDir string) int64 {\n\tvar ba int64\n\tif _, err := os.Stat(storeDir); os.IsNotExist(err) {\n\t\tos.MkdirAll(storeDir, defaultDirPerms)\n\t}\n\tvar fs syscall.Statfs_t\n\tif err := syscall.Statfs(storeDir, &fs); err == nil {\n\t\t// Estimate 75% of available storage.\n\t\tba = int64(uint64(fs.F_bavail) * uint64(fs.F_bsize) / 4 * 3)\n\t} else {\n\t\t// Used 1TB default as a guess if all else fails.\n\t\tba = JetStreamMaxStoreDefault\n\t}\n\treturn ba\n}\n" }, { "path": "server/disk_avail_solaris.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build illumos || solaris\n\npackage server\n\nimport (\n\t\"os\"\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc diskAvailable(storeDir string) int64 {\n\tvar ba int64\n\tif _, err := os.Stat(storeDir); os.IsNotExist(err) {\n\t\tos.MkdirAll(storeDir, defaultDirPerms)\n\t}\n\tvar fs unix.Statvfs_t\n\tif err := unix.Statvfs(storeDir, &fs); err == nil {\n\t\t// Estimate 75% of available storage.\n\t\tba = int64(uint64(fs.Frsize) * uint64(fs.Bavail) / 4 * 3)\n\t} else {\n\t\t// Used 1TB default as a guess if all else fails.\n\t\tba = JetStreamMaxStoreDefault\n\t}\n\treturn ba\n}\n\n" }, { "path": "server/disk_avail_wasm.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build wasm\n\npackage server\n\nfunc diskAvailable(storeDir string) int64 {\n\treturn JetStreamMaxStoreDefault\n}\n" }, { "path": "server/disk_avail_windows.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage server\n\n// TODO(dlc) - See if there is a version of this for windows.\nfunc diskAvailable(storeDir string) int64 {\n\treturn JetStreamMaxStoreDefault\n}\n" }, { "path": "server/elastic/elastic.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage elastic\n\nimport (\n\t\"weak\"\n)\n\nfunc Make[T any](ptr *T) *Pointer[T] {\n\treturn &Pointer[T]{\n\t\tweak: weak.Make(ptr),\n\t}\n}\n\ntype Pointer[T any] struct {\n\tweak weak.Pointer[T]\n\tstrong *T\n}\n\nfunc (e *Pointer[T]) Set(ptr *T) {\n\te.weak = weak.Make(ptr)\n\tif e.strong != nil {\n\t\te.strong = ptr\n\t}\n}\n\nfunc (e *Pointer[T]) Strengthen() {\n\tif e == nil || e.strong != nil {\n\t\treturn\n\t}\n\te.strong = e.weak.Value()\n}\n\nfunc (e *Pointer[T]) Weaken() {\n\tif e == nil || e.strong == nil {\n\t\treturn\n\t}\n\te.strong = nil\n}\n\nfunc (e *Pointer[T]) Value() *T {\n\tif e == nil {\n\t\treturn nil\n\t}\n\tif e.strong != nil {\n\t\treturn e.strong\n\t}\n\treturn e.weak.Value()\n}\n" }, { "path": "server/errors.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n)\n\nvar (\n\t// ErrConnectionClosed represents an error condition on a closed connection.\n\tErrConnectionClosed = errors.New(\"connection closed\")\n\n\t// ErrAuthentication represents an error condition on failed authentication.\n\tErrAuthentication = errors.New(\"authentication error\")\n\n\t// ErrAuthTimeout represents an error condition on failed authorization due to timeout.\n\tErrAuthTimeout = errors.New(\"authentication timeout\")\n\n\t// ErrAuthExpired represents an expired authorization due to timeout.\n\tErrAuthExpired = errors.New(\"authentication expired\")\n\n\t// ErrAuthProxyNotTrusted represents an error condition on failed authentication\n\t// due to a connection from a proxy not in the list of trusted proxies.\n\tErrAuthProxyNotTrusted = errors.New(\"proxy is not trusted\")\n\n\t// ErrAuthProxyRequired represents an error condition on failed authentication\n\t// due to a connection not coming from a proxy.\n\tErrAuthProxyRequired = errors.New(\"proxy connection required\")\n\n\t// ErrMaxPayload represents an error condition when the payload is too big.\n\tErrMaxPayload = errors.New(\"maximum payload exceeded\")\n\n\t// ErrMaxControlLine represents an error condition when the control line is too big.\n\tErrMaxControlLine = errors.New(\"maximum control line exceeded\")\n\n\t// ErrReservedPublishSubject represents an error condition when sending to a reserved subject, e.g. _SYS.>\n\tErrReservedPublishSubject = errors.New(\"reserved internal subject\")\n\n\t// ErrBadPublishSubject represents an error condition for an invalid publish subject.\n\tErrBadPublishSubject = errors.New(\"invalid publish subject\")\n\n\t// ErrBadSubject represents an error condition for an invalid subject.\n\tErrBadSubject = errors.New(\"invalid subject\")\n\n\t// ErrBadQualifier is used to error on a bad qualifier for a transform.\n\tErrBadQualifier = errors.New(\"bad qualifier\")\n\n\t// ErrBadClientProtocol signals a client requested an invalid client protocol.\n\tErrBadClientProtocol = errors.New(\"invalid client protocol\")\n\n\t// ErrTooManyConnections signals a client that the maximum number of connections supported by the\n\t// server has been reached.\n\tErrTooManyConnections = errors.New(\"maximum connections exceeded\")\n\n\t// ErrTooManyAccountConnections signals that an account has reached its maximum number of active\n\t// connections.\n\tErrTooManyAccountConnections = errors.New(\"maximum account active connections exceeded\")\n\n\t// ErrLeafNodeLoop signals a leafnode is trying to register for a cluster we already have registered.\n\tErrLeafNodeLoop = errors.New(\"leafnode loop detected\")\n\n\t// ErrTooManySubs signals a client that the maximum number of subscriptions per connection\n\t// has been reached.\n\tErrTooManySubs = errors.New(\"maximum subscriptions exceeded\")\n\n\t// ErrTooManySubTokens signals a client that the subject has too many tokens.\n\tErrTooManySubTokens = errors.New(\"subject has exceeded number of tokens limit\")\n\n\t// ErrClientConnectedToRoutePort represents an error condition when a client\n\t// attempted to connect to the route listen port.\n\tErrClientConnectedToRoutePort = errors.New(\"attempted to connect to route port\")\n\n\t// ErrClientConnectedToLeafNodePort represents an error condition when a client\n\t// attempted to connect to the leaf node listen port.\n\tErrClientConnectedToLeafNodePort = errors.New(\"attempted to connect to leaf node port\")\n\n\t// ErrLeafNodeHasSameClusterName represents an error condition when a leafnode is a cluster\n\t// and it has the same cluster name as the hub cluster.\n\tErrLeafNodeHasSameClusterName = errors.New(\"remote leafnode has same cluster name\")\n\n\t// ErrLeafNodeDisabled is when we disable leafnodes.\n\tErrLeafNodeDisabled = errors.New(\"leafnodes disabled\")\n\n\t// ErrConnectedToWrongPort represents an error condition when a connection is attempted\n\t// to the wrong listen port (for instance a LeafNode to a client port, etc...)\n\tErrConnectedToWrongPort = errors.New(\"attempted to connect to wrong port\")\n\n\t// ErrAccountExists is returned when an account is attempted to be registered\n\t// but already exists.\n\tErrAccountExists = errors.New(\"account exists\")\n\n\t// ErrBadAccount represents a malformed or incorrect account.\n\tErrBadAccount = errors.New(\"bad account\")\n\n\t// ErrReservedAccount represents a reserved account that can not be created.\n\tErrReservedAccount = errors.New(\"reserved account\")\n\n\t// ErrMissingAccount is returned when an account does not exist.\n\tErrMissingAccount = errors.New(\"account missing\")\n\n\t// ErrMissingService is returned when an account does not have an exported service.\n\tErrMissingService = errors.New(\"service missing\")\n\n\t// ErrBadServiceType is returned when latency tracking is being applied to non-singleton response types.\n\tErrBadServiceType = errors.New(\"bad service response type\")\n\n\t// ErrBadSampling is returned when the sampling for latency tracking is not 1 >= sample <= 100.\n\tErrBadSampling = errors.New(\"bad sampling percentage, should be 1-100\")\n\n\t// ErrAccountValidation is returned when an account has failed validation.\n\tErrAccountValidation = errors.New(\"account validation failed\")\n\n\t// ErrAccountExpired is returned when an account has expired.\n\tErrAccountExpired = errors.New(\"account expired\")\n\n\t// ErrNoAccountResolver is returned when we attempt an update but do not have an account resolver.\n\tErrNoAccountResolver = errors.New(\"account resolver missing\")\n\n\t// ErrAccountResolverUpdateTooSoon is returned when we attempt an update too soon to last request.\n\tErrAccountResolverUpdateTooSoon = errors.New(\"account resolver update too soon\")\n\n\t// ErrAccountResolverSameClaims is returned when same claims have been fetched.\n\tErrAccountResolverSameClaims = errors.New(\"account resolver no new claims\")\n\n\t// ErrStreamImportAuthorization is returned when a stream import is not authorized.\n\tErrStreamImportAuthorization = errors.New(\"stream import not authorized\")\n\n\t// ErrStreamImportBadPrefix is returned when a stream import prefix contains wildcards.\n\tErrStreamImportBadPrefix = errors.New(\"stream import prefix can not contain wildcard tokens\")\n\n\t// ErrStreamImportDuplicate is returned when a stream import is a duplicate of one that already exists.\n\tErrStreamImportDuplicate = errors.New(\"stream import already exists\")\n\n\t// ErrServiceImportAuthorization is returned when a service import is not authorized.\n\tErrServiceImportAuthorization = errors.New(\"service import not authorized\")\n\n\t// ErrImportFormsCycle is returned when an import would form a cycle.\n\tErrImportFormsCycle = errors.New(\"import forms a cycle\")\n\n\t// ErrCycleSearchDepth is returned when we have exceeded our maximum search depth..\n\tErrCycleSearchDepth = errors.New(\"search cycle depth exhausted\")\n\n\t// ErrClientOrRouteConnectedToGatewayPort represents an error condition when\n\t// a client or route attempted to connect to the Gateway port.\n\tErrClientOrRouteConnectedToGatewayPort = errors.New(\"attempted to connect to gateway port\")\n\n\t// ErrWrongGateway represents an error condition when a server receives a connect\n\t// request from a remote Gateway with a destination name that does not match the server's\n\t// Gateway's name.\n\tErrWrongGateway = errors.New(\"wrong gateway\")\n\n\t// ErrGatewayNameHasSpaces signals that the gateway name contains spaces, which is not allowed.\n\tErrGatewayNameHasSpaces = errors.New(\"gateway name cannot contain spaces\")\n\n\t// ErrNoSysAccount is returned when an attempt to publish or subscribe is made\n\t// when there is no internal system account defined.\n\tErrNoSysAccount = errors.New(\"system account not setup\")\n\n\t// ErrRevocation is returned when a credential has been revoked.\n\tErrRevocation = errors.New(\"credentials have been revoked\")\n\n\t// ErrServerNotRunning is used to signal an error that a server is not running.\n\tErrServerNotRunning = errors.New(\"server is not running\")\n\n\t// ErrServerNameHasSpaces signals that the server name contains spaces, which is not allowed.\n\tErrServerNameHasSpaces = errors.New(\"server name cannot contain spaces\")\n\n\t// ErrBadMsgHeader signals the parser detected a bad message header\n\tErrBadMsgHeader = errors.New(\"bad message header detected\")\n\n\t// ErrMsgHeadersNotSupported signals the parser detected a message header\n\t// but they are not supported on this server.\n\tErrMsgHeadersNotSupported = errors.New(\"message headers not supported\")\n\n\t// ErrNoRespondersRequiresHeaders signals that a client needs to have headers\n\t// on if they want no responders behavior.\n\tErrNoRespondersRequiresHeaders = errors.New(\"no responders requires headers support\")\n\n\t// ErrClusterNameConfigConflict signals that the options for cluster name in cluster and gateway are in conflict.\n\tErrClusterNameConfigConflict = errors.New(\"cluster name conflicts between cluster and gateway definitions\")\n\n\t// ErrClusterNameRemoteConflict signals that a remote server has a different cluster name.\n\tErrClusterNameRemoteConflict = errors.New(\"cluster name from remote server conflicts\")\n\n\t// ErrClusterNameHasSpaces signals that the cluster name contains spaces, which is not allowed.\n\tErrClusterNameHasSpaces = errors.New(\"cluster name cannot contain spaces\")\n\n\t// ErrMalformedSubject is returned when a subscription is made with a subject that does not conform to subject rules.\n\tErrMalformedSubject = errors.New(\"malformed subject\")\n\n\t// ErrSubscribePermissionViolation is returned when processing of a subscription fails due to permissions.\n\tErrSubscribePermissionViolation = errors.New(\"subscribe permission violation\")\n\n\t// ErrNoTransforms signals no subject transforms are available to map this subject.\n\tErrNoTransforms = errors.New(\"no matching transforms available\")\n\n\t// ErrCertNotPinned is returned when pinned certs are set and the certificate is not in it\n\tErrCertNotPinned = errors.New(\"certificate not pinned\")\n\n\t// ErrDuplicateServerName is returned when processing a server remote connection and\n\t// the server reports that this server name is already used in the cluster.\n\tErrDuplicateServerName = errors.New(\"duplicate server name\")\n\n\t// ErrMinimumVersionRequired is returned when a connection is not at the minimum version required.\n\tErrMinimumVersionRequired = errors.New(\"minimum version required\")\n\t// ErrLeafNodeMinVersionRejected is the leafnode protocol error prefix used\n\t// when rejecting a remote due to leafnodes.min_version.\n\tErrLeafNodeMinVersionRejected = errors.New(\"connection rejected since minimum version required is\")\n\n\t// ErrInvalidMappingDestination is used for all subject mapping destination errors\n\tErrInvalidMappingDestination = errors.New(\"invalid mapping destination\")\n\n\t// ErrInvalidMappingDestinationSubject is used to error on a bad transform destination mapping\n\tErrInvalidMappingDestinationSubject = fmt.Errorf(\"%w: invalid transform\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationNotUsingAllWildcards is used to error on a transform destination not using all of the token wildcards\n\tErrMappingDestinationNotUsingAllWildcards = fmt.Errorf(\"%w: not using all of the token wildcard(s)\", ErrInvalidMappingDestination)\n\n\t// ErrUnknownMappingDestinationFunction is returned when a subject mapping destination contains an unknown mustache-escaped mapping function.\n\tErrUnknownMappingDestinationFunction = fmt.Errorf(\"%w: unknown function\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationIndexOutOfRange is returned when the mapping destination function is passed an out of range wildcard index value for one of it's arguments\n\tErrMappingDestinationIndexOutOfRange = fmt.Errorf(\"%w: wildcard index out of range\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationNotEnoughArgs is returned when the mapping destination function is not passed enough arguments\n\tErrMappingDestinationNotEnoughArgs = fmt.Errorf(\"%w: not enough arguments passed to the function\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationInvalidArg is returned when the mapping destination function is passed and invalid argument\n\tErrMappingDestinationInvalidArg = fmt.Errorf(\"%w: function argument is invalid or in the wrong format\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationTooManyArgs is returned when the mapping destination function is passed too many arguments\n\tErrMappingDestinationTooManyArgs = fmt.Errorf(\"%w: too many arguments passed to the function\", ErrInvalidMappingDestination)\n\n\t// ErrMappingDestinationNotSupportedForImport is returned when you try to use a mapping function other than wildcard in a transform that needs to be reversible (i.e. an import)\n\tErrMappingDestinationNotSupportedForImport = fmt.Errorf(\"%w: the only mapping function allowed for import transforms is {{Wildcard()}}\", ErrInvalidMappingDestination)\n)\n\n// mappingDestinationErr is a type of subject mapping destination error\ntype mappingDestinationErr struct {\n\ttoken string\n\terr error\n}\n\nfunc (e *mappingDestinationErr) Error() string {\n\tif e.token == _EMPTY_ {\n\t\treturn e.err.Error()\n\t}\n\treturn fmt.Sprintf(\"%s in %s\", e.err, e.token)\n}\n\nfunc (e *mappingDestinationErr) Is(target error) bool {\n\treturn target == ErrInvalidMappingDestination\n}\n\n// configErr is a configuration error.\ntype configErr struct {\n\ttoken token\n\treason string\n}\n\n// Source reports the location of a configuration error.\nfunc (e *configErr) Source() string {\n\treturn fmt.Sprintf(\"%s:%d:%d\", e.token.SourceFile(), e.token.Line(), e.token.Position())\n}\n\n// Error reports the location and reason from a configuration error.\nfunc (e *configErr) Error() string {\n\tif e.token != nil {\n\t\treturn fmt.Sprintf(\"%s: %s\", e.Source(), e.reason)\n\t}\n\treturn e.reason\n}\n\n// unknownConfigFieldErr is an error reported in pedantic mode.\ntype unknownConfigFieldErr struct {\n\tconfigErr\n\tfield string\n}\n\n// Error reports that an unknown field was in the configuration.\nfunc (e *unknownConfigFieldErr) Error() string {\n\treturn fmt.Sprintf(\"%s: unknown field %q\", e.Source(), e.field)\n}\n\n// configWarningErr is an error reported in pedantic mode.\ntype configWarningErr struct {\n\tconfigErr\n\tfield string\n}\n\n// Error reports a configuration warning.\nfunc (e *configWarningErr) Error() string {\n\treturn fmt.Sprintf(\"%s: invalid use of field %q: %s\", e.Source(), e.field, e.reason)\n}\n\n// processConfigErr is the result of processing the configuration from the server.\ntype processConfigErr struct {\n\terrors []error\n\twarnings []error\n}\n\n// Error returns the collection of errors separated by new lines,\n// warnings appear first then hard errors.\nfunc (e *processConfigErr) Error() string {\n\tvar msg string\n\tfor _, err := range e.Warnings() {\n\t\tmsg += err.Error() + \"\\n\"\n\t}\n\tfor _, err := range e.Errors() {\n\t\tmsg += err.Error() + \"\\n\"\n\t}\n\treturn msg\n}\n\n// Warnings returns the list of warnings.\nfunc (e *processConfigErr) Warnings() []error {\n\treturn e.warnings\n}\n\n// Errors returns the list of errors.\nfunc (e *processConfigErr) Errors() []error {\n\treturn e.errors\n}\n\n// errCtx wraps an error and stores additional ctx information for tracing.\n// Does not print or return it unless explicitly requested.\ntype errCtx struct {\n\terror\n\tctx string\n}\n\nfunc NewErrorCtx(err error, format string, args ...any) error {\n\treturn &errCtx{err, fmt.Sprintf(format, args...)}\n}\n\n// Unwrap implement to work with errors.Is and errors.As\nfunc (e *errCtx) Unwrap() error {\n\tif e == nil {\n\t\treturn nil\n\t}\n\treturn e.error\n}\n\n// Context for error\nfunc (e *errCtx) Context() string {\n\tif e == nil {\n\t\treturn \"\"\n\t}\n\treturn e.ctx\n}\n\n// UnpackIfErrorCtx return Error or, if type is right error and context\nfunc UnpackIfErrorCtx(err error) string {\n\tif e, ok := err.(*errCtx); ok {\n\t\tif _, ok := e.error.(*errCtx); ok {\n\t\t\treturn fmt.Sprint(UnpackIfErrorCtx(e.error), \": \", e.Context())\n\t\t}\n\t\treturn fmt.Sprint(e.Error(), \": \", e.Context())\n\t}\n\treturn err.Error()\n}\n\n// implements: go 1.13 errors.Unwrap(err error) error\n// TODO replace with native code once we no longer support go1.12\nfunc errorsUnwrap(err error) error {\n\tu, ok := err.(interface {\n\t\tUnwrap() error\n\t})\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn u.Unwrap()\n}\n\n// ErrorIs implements: go 1.13 errors.Is(err, target error) bool\n// TODO replace with native code once we no longer support go1.12\nfunc ErrorIs(err, target error) bool {\n\t// this is an outright copy of go 1.13 errors.Is(err, target error) bool\n\t// removed isComparable\n\tif err == nil || target == nil {\n\t\treturn err == target\n\t}\n\n\tfor {\n\t\tif err == target {\n\t\t\treturn true\n\t\t}\n\t\tif x, ok := err.(interface{ Is(error) bool }); ok && x.Is(target) {\n\t\t\treturn true\n\t\t}\n\t\t// TODO: consider supporing target.Is(err). This would allow\n\t\t// user-definable predicates, but also may allow for coping with sloppy\n\t\t// APIs, thereby making it easier to get away with them.\n\t\tif err = errorsUnwrap(err); err == nil {\n\t\t\treturn false\n\t\t}\n\t}\n}\n" }, { "path": "server/errors.json", "content": "[\n {\n \"constant\": \"JSClusterPeerNotMemberErr\",\n \"code\": 400,\n \"error_code\": 10040,\n \"description\": \"peer not a member\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerEphemeralWithDurableInSubjectErr\",\n \"code\": 400,\n \"error_code\": 10019,\n \"description\": \"consumer expected to be ephemeral but detected a durable name set in subject\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamExternalDelPrefixOverlapsErrF\",\n \"code\": 400,\n \"error_code\": 10022,\n \"description\": \"stream external delivery prefix {prefix} overlaps with stream subject {subject}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAccountResourcesExceededErr\",\n \"code\": 400,\n \"error_code\": 10002,\n \"description\": \"resource limits exceeded for account\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterNotAvailErr\",\n \"code\": 503,\n \"error_code\": 10008,\n \"description\": \"JetStream system temporarily unavailable\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamSubjectOverlapErr\",\n \"code\": 400,\n \"error_code\": 10065,\n \"description\": \"subjects overlap with an existing stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamWrongLastSequenceErrF\",\n \"code\": 400,\n \"error_code\": 10071,\n \"description\": \"wrong last sequence: {seq}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSTemplateNameNotMatchSubjectErr\",\n \"code\": 400,\n \"error_code\": 10073,\n \"description\": \"template name in subject does not match request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterNoPeersErrF\",\n \"code\": 400,\n \"error_code\": 10005,\n \"description\": \"{err}\",\n \"comment\": \"Error causing no peers to be available\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerEphemeralWithDurableNameErr\",\n \"code\": 400,\n \"error_code\": 10020,\n \"description\": \"consumer expected to be ephemeral but a durable name was set in request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSInsufficientResourcesErr\",\n \"code\": 503,\n \"error_code\": 10023,\n \"description\": \"insufficient resources\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamResourcesExceeded\"\n },\n {\n \"constant\": \"JSMirrorMaxMessageSizeTooBigErr\",\n \"code\": 400,\n \"error_code\": 10030,\n \"description\": \"stream mirror must have max message size \\u003e= source\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTemplateDeleteErrF\",\n \"code\": 500,\n \"error_code\": 10067,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream template deletion failed error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBadRequestErr\",\n \"code\": 400,\n \"error_code\": 10003,\n \"description\": \"bad request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterUnSupportFeatureErr\",\n \"code\": 503,\n \"error_code\": 10036,\n \"description\": \"not currently supported in clustered mode\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerNotFoundErr\",\n \"code\": 404,\n \"error_code\": 10014,\n \"description\": \"consumer not found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceMaxMessageSizeTooBigErr\",\n \"code\": 400,\n \"error_code\": 10046,\n \"description\": \"stream source must have max message size \\u003e= target\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamAssignmentErrF\",\n \"code\": 500,\n \"error_code\": 10048,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream assignment error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMessageExceedsMaximumErr\",\n \"code\": 400,\n \"error_code\": 10054,\n \"description\": \"message size exceeds maximum allowed\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTemplateCreateErrF\",\n \"code\": 500,\n \"error_code\": 10066,\n \"description\": \"{err}\",\n \"comment\": \"Generic template creation failed string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSInvalidJSONErr\",\n \"code\": 400,\n \"error_code\": 10025,\n \"description\": \"invalid JSON: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamInvalidExternalDeliverySubjErrF\",\n \"code\": 400,\n \"error_code\": 10024,\n \"description\": \"stream external delivery prefix {prefix} must not contain wildcards\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamRestoreErrF\",\n \"code\": 500,\n \"error_code\": 10062,\n \"description\": \"restore failed: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterIncompleteErr\",\n \"code\": 503,\n \"error_code\": 10004,\n \"description\": \"incomplete results\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSNoAccountErr\",\n \"code\": 503,\n \"error_code\": 10035,\n \"description\": \"account not found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSRaftGeneralErrF\",\n \"code\": 500,\n \"error_code\": 10041,\n \"description\": \"{err}\",\n \"comment\": \"General RAFT error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSRestoreSubscribeFailedErrF\",\n \"code\": 500,\n \"error_code\": 10042,\n \"description\": \"JetStream unable to subscribe to restore snapshot {subject}: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamDeleteErrF\",\n \"code\": 500,\n \"error_code\": 10050,\n \"description\": \"{err}\",\n \"comment\": \"General stream deletion error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamExternalApiOverlapErrF\",\n \"code\": 400,\n \"error_code\": 10021,\n \"description\": \"stream external api prefix {prefix} must not overlap with {subject}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithSubjectsErr\",\n \"code\": 400,\n \"error_code\": 10034,\n \"description\": \"stream mirrors can not contain subjects\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithFirstSeqErr\",\n \"code\": 400,\n \"error_code\": 10143,\n \"description\": \"stream mirrors can not have first sequence configured\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSNotEnabledErr\",\n \"code\": 503,\n \"error_code\": 10076,\n \"description\": \"JetStream not enabled\",\n \"comment\": \"\",\n \"help\": \"This error indicates that JetStream is not enabled at a global level\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamNotEnabled\"\n },\n {\n \"constant\": \"JSNotEnabledForAccountErr\",\n \"code\": 503,\n \"error_code\": 10039,\n \"description\": \"JetStream not enabled for account\",\n \"comment\": \"\",\n \"help\": \"This error indicates that JetStream is not enabled for an account account level\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSequenceNotFoundErrF\",\n \"code\": 400,\n \"error_code\": 10043,\n \"description\": \"sequence {seq} not found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMirrorNotUpdatableErr\",\n \"code\": 400,\n \"error_code\": 10055,\n \"description\": \"stream mirror configuration can not be updated\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamSequenceNotMatchErr\",\n \"code\": 503,\n \"error_code\": 10063,\n \"description\": \"expected stream sequence does not match\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamWrongLastMsgIDErrF\",\n \"code\": 400,\n \"error_code\": 10070,\n \"description\": \"wrong last msg ID: {id}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSTempStorageFailedErr\",\n \"code\": 500,\n \"error_code\": 10072,\n \"description\": \"JetStream unable to open temp storage for restore\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStorageResourcesExceededErr\",\n \"code\": 500,\n \"error_code\": 10047,\n \"description\": \"insufficient storage resources available\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrStorageResourcesExceeded\"\n },\n {\n \"constant\": \"JSStreamMismatchErr\",\n \"code\": 400,\n \"error_code\": 10056,\n \"description\": \"stream name in subject does not match request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamNotMatchErr\",\n \"code\": 400,\n \"error_code\": 10060,\n \"description\": \"expected stream does not match\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorConsumerSetupFailedErrF\",\n \"code\": 500,\n \"error_code\": 10029,\n \"description\": \"{err}\",\n \"comment\": \"generic mirror consumer setup failure string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSNotEmptyRequestErr\",\n \"code\": 400,\n \"error_code\": 10038,\n \"description\": \"expected an empty request payload\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamNameExistErr\",\n \"code\": 400,\n \"error_code\": 10058,\n \"description\": \"stream name already in use with a different configuration\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamStreamAlreadyUsed\"\n },\n {\n \"constant\": \"JSClusterTagsErr\",\n \"code\": 400,\n \"error_code\": 10011,\n \"description\": \"tags placement not supported for operation\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMaximumConsumersLimitErr\",\n \"code\": 400,\n \"error_code\": 10026,\n \"description\": \"maximum consumers limit reached\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceConsumerSetupFailedErrF\",\n \"code\": 500,\n \"error_code\": 10045,\n \"description\": \"{err}\",\n \"comment\": \"General source consumer setup failure string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerCreateErrF\",\n \"code\": 500,\n \"error_code\": 10012,\n \"description\": \"{err}\",\n \"comment\": \"General consumer creation failure string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDurableNameNotInSubjectErr\",\n \"code\": 400,\n \"error_code\": 10016,\n \"description\": \"consumer expected to be durable but no durable name set in subject\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamLimitsErrF\",\n \"code\": 500,\n \"error_code\": 10053,\n \"description\": \"{err}\",\n \"comment\": \"General stream limits exceeded error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamReplicasNotUpdatableErr\",\n \"code\": 400,\n \"error_code\": 10061,\n \"description\": \"Replicas configuration can not be updated\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTemplateNotFoundErr\",\n \"code\": 404,\n \"error_code\": 10068,\n \"description\": \"template not found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterNotAssignedErr\",\n \"code\": 500,\n \"error_code\": 10007,\n \"description\": \"JetStream cluster not assigned to this server\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamNotAssigned\"\n },\n {\n \"constant\": \"JSClusterNotLeaderErr\",\n \"code\": 500,\n \"error_code\": 10009,\n \"description\": \"JetStream cluster can not handle request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamNotLeader\"\n },\n {\n \"constant\": \"JSConsumerNameExistErr\",\n \"code\": 400,\n \"error_code\": 10013,\n \"description\": \"consumer name already in use\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamConsumerAlreadyUsed\"\n },\n {\n \"constant\": \"JSMirrorWithSourcesErr\",\n \"code\": 400,\n \"error_code\": 10031,\n \"description\": \"stream mirrors can not also contain other sources\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamNotFoundErr\",\n \"code\": 404,\n \"error_code\": 10059,\n \"description\": \"stream not found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamStreamNotFound\"\n },\n {\n \"constant\": \"JSClusterRequiredErr\",\n \"code\": 503,\n \"error_code\": 10010,\n \"description\": \"JetStream clustering support required\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDurableNameNotSetErr\",\n \"code\": 400,\n \"error_code\": 10018,\n \"description\": \"consumer expected to be durable but a durable name was not set\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMaximumStreamsLimitErr\",\n \"code\": 400,\n \"error_code\": 10027,\n \"description\": \"maximum number of streams reached\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithStartSeqAndTimeErr\",\n \"code\": 400,\n \"error_code\": 10032,\n \"description\": \"stream mirrors can not have both start seq and start time configured\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamSnapshotErrF\",\n \"code\": 500,\n \"error_code\": 10064,\n \"description\": \"snapshot failed: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamUpdateErrF\",\n \"code\": 500,\n \"error_code\": 10069,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream update error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterNotActiveErr\",\n \"code\": 500,\n \"error_code\": 10006,\n \"description\": \"JetStream not in clustered mode\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrJetStreamNotClustered\"\n },\n {\n \"constant\": \"JSConsumerDurableNameNotMatchSubjectErr\",\n \"code\": 400,\n \"error_code\": 10017,\n \"description\": \"consumer name in subject does not match durable name in request\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMemoryResourcesExceededErr\",\n \"code\": 500,\n \"error_code\": 10028,\n \"description\": \"insufficient memory resources available\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrMemoryResourcesExceeded\"\n },\n {\n \"constant\": \"JSMirrorWithSubjectFiltersErr\",\n \"code\": 400,\n \"error_code\": 10033,\n \"description\": \"stream mirrors can not contain filtered subjects\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamCreateErrF\",\n \"code\": 500,\n \"error_code\": 10049,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream creation error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterServerNotMemberErr\",\n \"code\": 400,\n \"error_code\": 10044,\n \"description\": \"server is not a member of the cluster\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSNoMessageFoundErr\",\n \"code\": 404,\n \"error_code\": 10037,\n \"description\": \"no message found\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSnapshotDeliverSubjectInvalidErr\",\n \"code\": 400,\n \"error_code\": 10015,\n \"description\": \"deliver subject not valid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamGeneralErrorF\",\n \"code\": 500,\n \"error_code\": 10051,\n \"description\": \"{err}\",\n \"comment\": \"General stream failure string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamInvalidConfigF\",\n \"code\": 500,\n \"error_code\": 10052,\n \"description\": \"{err}\",\n \"comment\": \"Stream configuration validation error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamReplicasNotSupportedErr\",\n \"code\": 500,\n \"error_code\": 10074,\n \"description\": \"replicas \\u003e 1 not supported in non-clustered mode\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"ErrReplicasNotSupported\"\n },\n {\n \"constant\": \"JSStreamMsgDeleteFailedF\",\n \"code\": 500,\n \"error_code\": 10057,\n \"description\": \"{err}\",\n \"comment\": \"Generic message deletion failure error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSPeerRemapErr\",\n \"code\": 503,\n \"error_code\": 10075,\n \"description\": \"peer remap failed\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamStoreFailedF\",\n \"code\": 503,\n \"error_code\": 10077,\n \"description\": \"{err}\",\n \"comment\": \"Generic error when storing a message failed\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerConfigRequiredErr\",\n \"code\": 400,\n \"error_code\": 10078,\n \"description\": \"consumer config required\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDeliverToWildcardsErr\",\n \"code\": 400,\n \"error_code\": 10079,\n \"description\": \"consumer deliver subject has wildcards\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPushMaxWaitingErr\",\n \"code\": 400,\n \"error_code\": 10080,\n \"description\": \"consumer in push mode can not set max waiting\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDeliverCycleErr\",\n \"code\": 400,\n \"error_code\": 10081,\n \"description\": \"consumer deliver subject forms a cycle\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxPendingAckPolicyRequiredErr\",\n \"code\": 400,\n \"error_code\": 10082,\n \"description\": \"consumer requires ack policy for max ack pending\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerSmallHeartbeatErr\",\n \"code\": 400,\n \"error_code\": 10083,\n \"description\": \"consumer idle heartbeat needs to be \\u003e= 100ms\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPullRequiresAckErr\",\n \"code\": 400,\n \"error_code\": 10084,\n \"description\": \"consumer in pull mode requires explicit ack policy on workqueue stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPullNotDurableErr\",\n \"code\": 400,\n \"error_code\": 10085,\n \"description\": \"consumer in pull mode requires a durable name\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPullWithRateLimitErr\",\n \"code\": 400,\n \"error_code\": 10086,\n \"description\": \"consumer in pull mode can not have rate limit set\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxWaitingNegativeErr\",\n \"code\": 400,\n \"error_code\": 10087,\n \"description\": \"consumer max waiting needs to be positive\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerHBRequiresPushErr\",\n \"code\": 400,\n \"error_code\": 10088,\n \"description\": \"consumer idle heartbeat requires a push based consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerFCRequiresPushErr\",\n \"code\": 400,\n \"error_code\": 10089,\n \"description\": \"consumer flow control requires a push based consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDirectRequiresPushErr\",\n \"code\": 400,\n \"error_code\": 10090,\n \"description\": \"consumer direct requires a push based consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDirectRequiresEphemeralErr\",\n \"code\": 400,\n \"error_code\": 10091,\n \"description\": \"consumer direct requires an ephemeral consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerOnMappedErr\",\n \"code\": 400,\n \"error_code\": 10092,\n \"description\": \"consumer direct on a mapped consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerFilterNotSubsetErr\",\n \"code\": 400,\n \"error_code\": 10093,\n \"description\": \"consumer filter subject is not a valid subset of the interest subjects\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidPolicyErrF\",\n \"code\": 400,\n \"error_code\": 10094,\n \"description\": \"{err}\",\n \"comment\": \"Generic delivery policy error\",\n \"help\": \"Error returned for impossible deliver policies when combined with start sequences etc\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidSamplingErrF\",\n \"code\": 400,\n \"error_code\": 10095,\n \"description\": \"failed to parse consumer sampling configuration: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamInvalidErr\",\n \"code\": 500,\n \"error_code\": 10096,\n \"description\": \"stream not valid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerWQRequiresExplicitAckErr\",\n \"code\": 400,\n \"error_code\": 10098,\n \"description\": \"workqueue stream requires explicit ack\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerWQMultipleUnfilteredErr\",\n \"code\": 400,\n \"error_code\": 10099,\n \"description\": \"multiple non-filtered consumers not allowed on workqueue stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerWQConsumerNotUniqueErr\",\n \"code\": 400,\n \"error_code\": 10100,\n \"description\": \"filtered consumer not unique on workqueue stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerWQConsumerNotDeliverAllErr\",\n \"code\": 400,\n \"error_code\": 10101,\n \"description\": \"consumer must be deliver all on workqueue stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerNameTooLongErrF\",\n \"code\": 400,\n \"error_code\": 10102,\n \"description\": \"consumer name is too long, maximum allowed is {max}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerBadDurableNameErr\",\n \"code\": 400,\n \"error_code\": 10103,\n \"description\": \"durable name can not contain '.', '*', '\\u003e'\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerStoreFailedErrF\",\n \"code\": 500,\n \"error_code\": 10104,\n \"description\": \"error creating store for consumer: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerExistingActiveErr\",\n \"code\": 400,\n \"error_code\": 10105,\n \"description\": \"consumer already exists and is still active\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerReplacementWithDifferentNameErr\",\n \"code\": 400,\n \"error_code\": 10106,\n \"description\": \"consumer replacement durable config not the same\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDescriptionTooLongErrF\",\n \"code\": 400,\n \"error_code\": 10107,\n \"description\": \"consumer description is too long, maximum allowed is {max}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamHeaderExceedsMaximumErr\",\n \"code\": 400,\n \"error_code\": 10097,\n \"description\": \"header size exceeds maximum allowed of 64k\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerWithFlowControlNeedsHeartbeats\",\n \"code\": 400,\n \"error_code\": 10108,\n \"description\": \"consumer with flow control also needs heartbeats\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamSealedErr\",\n \"code\": 400,\n \"error_code\": 10109,\n \"description\": \"invalid operation on sealed stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamPurgeFailedF\",\n \"code\": 500,\n \"error_code\": 10110,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream purge failure error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamRollupFailedF\",\n \"code\": 500,\n \"error_code\": 10111,\n \"description\": \"{err}\",\n \"comment\": \"Generic stream rollup failure error string\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidDeliverSubject\",\n \"code\": 400,\n \"error_code\": 10112,\n \"description\": \"invalid push consumer deliver subject\",\n \"comment\": \"\",\n \"help\": \"Returned when the delivery subject on a Push Consumer is not a valid NATS Subject\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMaxBytesRequired\",\n \"code\": 400,\n \"error_code\": 10113,\n \"description\": \"account requires a stream config to have max bytes set\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxRequestBatchNegativeErr\",\n \"code\": 400,\n \"error_code\": 10114,\n \"description\": \"consumer max request batch needs to be \\u003e 0\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxRequestExpiresTooSmall\",\n \"code\": 400,\n \"error_code\": 10115,\n \"description\": \"consumer max request expires needs to be \\u003e= 1ms\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxDeliverBackoffErr\",\n \"code\": 400,\n \"error_code\": 10116,\n \"description\": \"max deliver is required to be \\u003e length of backoff values\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamInfoMaxSubjectsErr\",\n \"code\": 500,\n \"error_code\": 10117,\n \"description\": \"subject details would exceed maximum allowed\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamOfflineErr\",\n \"code\": 500,\n \"error_code\": 10118,\n \"description\": \"stream is offline\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerOfflineErr\",\n \"code\": 500,\n \"error_code\": 10119,\n \"description\": \"consumer is offline\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSNoLimitsErr\",\n \"code\": 400,\n \"error_code\": 10120,\n \"description\": \"no JetStream default or applicable tiered limit present\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxPendingAckExcessErrF\",\n \"code\": 400,\n \"error_code\": 10121,\n \"description\": \"consumer max ack pending exceeds system limit of {limit}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMaxStreamBytesExceeded\",\n \"code\": 400,\n \"error_code\": 10122,\n \"description\": \"stream max bytes exceeds account limit max stream bytes\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMoveAndScaleErr\",\n \"code\": 400,\n \"error_code\": 10123,\n \"description\": \"can not move and scale a stream in a single update\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMoveInProgressF\",\n \"code\": 400,\n \"error_code\": 10124,\n \"description\": \"stream move already in progress: {msg}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMaxRequestBatchExceededF\",\n \"code\": 400,\n \"error_code\": 10125,\n \"description\": \"consumer max request batch exceeds server limit of {limit}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerReplicasExceedsStream\",\n \"code\": 400,\n \"error_code\": 10126,\n \"description\": \"consumer config replica count exceeds parent stream\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerNameContainsPathSeparatorsErr\",\n \"code\": 400,\n \"error_code\": 10127,\n \"description\": \"Consumer name can not contain path separators\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamNameContainsPathSeparatorsErr\",\n \"code\": 400,\n \"error_code\": 10128,\n \"description\": \"Stream name can not contain path separators\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMoveNotInProgress\",\n \"code\": 400,\n \"error_code\": 10129,\n \"description\": \"stream move not in progress\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamNameExistRestoreFailedErr\",\n \"code\": 400,\n \"error_code\": 10130,\n \"description\": \"stream name already in use, cannot restore\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerCreateFilterSubjectMismatchErr\",\n \"code\": 400,\n \"error_code\": 10131,\n \"description\": \"Consumer create request did not match filtered subject from create subject\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerCreateDurableAndNameMismatch\",\n \"code\": 400,\n \"error_code\": 10132,\n \"description\": \"Consumer Durable and Name have to be equal if both are provided\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSReplicasCountCannotBeNegative\",\n \"code\": 400,\n \"error_code\": 10133,\n \"description\": \"replicas count cannot be negative\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerReplicasShouldMatchStream\",\n \"code\": 400,\n \"error_code\": 10134,\n \"description\": \"consumer config replicas must match interest retention stream's replicas\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMetadataLengthErrF\",\n \"code\": 400,\n \"error_code\": 10135,\n \"description\": \"consumer metadata exceeds maximum size of {limit}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDuplicateFilterSubjects\",\n \"code\": 400,\n \"error_code\": 10136,\n \"description\": \"consumer cannot have both FilterSubject and FilterSubjects specified\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerMultipleFiltersNotAllowed\",\n \"code\": 400,\n \"error_code\": 10137,\n \"description\": \"consumer with multiple subject filters cannot use subject based API\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerOverlappingSubjectFilters\",\n \"code\": 400,\n \"error_code\": 10138,\n \"description\": \"consumer subject filters cannot overlap\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerEmptyFilter\",\n \"code\": 400,\n \"error_code\": 10139,\n \"description\": \"consumer filter in FilterSubjects cannot be empty\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceDuplicateDetected\",\n \"code\": 400,\n \"error_code\": 10140,\n \"description\": \"duplicate source configuration detected\",\n \"comment\": \"source stream, filter and transform (plus external if present) must form a unique combination\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceInvalidStreamName\",\n \"code\": 400,\n \"error_code\": 10141,\n \"description\": \"sourced stream name is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorInvalidStreamName\",\n \"code\": 400,\n \"error_code\": 10142,\n \"description\": \"mirrored stream name is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceMultipleFiltersNotAllowed\",\n \"code\": 400,\n \"error_code\": 10144,\n \"description\": \"source with multiple subject transforms cannot also have a single subject filter\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceInvalidSubjectFilter\",\n \"code\": 400,\n \"error_code\": 10145,\n \"description\": \"source transform source: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceInvalidTransformDestination\",\n \"code\": 400,\n \"error_code\": 10146,\n \"description\": \"source transform: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceOverlappingSubjectFilters\",\n \"code\": 400,\n \"error_code\": 10147,\n \"description\": \"source filters can not overlap\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerAlreadyExists\",\n \"code\": 400,\n \"error_code\": 10148,\n \"description\": \"consumer already exists\",\n \"comment\": \"action CREATE is used for a existing consumer with a different config\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerDoesNotExist\",\n \"code\": 400,\n \"error_code\": 10149,\n \"description\": \"consumer does not exist\",\n \"comment\": \"action UPDATE is used for a nonexisting consumer\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorMultipleFiltersNotAllowed\",\n \"code\": 400,\n \"error_code\": 10150,\n \"description\": \"mirror with multiple subject transforms cannot also have a single subject filter\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorInvalidSubjectFilter\",\n \"code\": 400,\n \"error_code\": 10151,\n \"description\": \"mirror transform source: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorOverlappingSubjectFilters\",\n \"code\": 400,\n \"error_code\": 10152,\n \"description\": \"mirror subject filters can not overlap\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInactiveThresholdExcess\",\n \"code\": 400,\n \"error_code\": 10153,\n \"description\": \"consumer inactive threshold exceeds system limit of {limit}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorInvalidTransformDestination\",\n \"code\": 400,\n \"error_code\": 10154,\n \"description\": \"mirror transform: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTransformInvalidSource\",\n \"code\": 400,\n \"error_code\": 10155,\n \"description\": \"stream transform source: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTransformInvalidDestination\",\n \"code\": 400,\n \"error_code\": 10156,\n \"description\": \"stream transform: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSPedanticErrF\",\n \"code\": 400,\n \"error_code\": 10157,\n \"description\": \"pedantic mode: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamDuplicateMessageConflict\",\n \"code\": 409,\n \"error_code\": 10158,\n \"description\": \"duplicate message id is in process\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPriorityPolicyWithoutGroup\",\n \"code\": 400,\n \"error_code\": 10159,\n \"description\": \"Setting PriorityPolicy requires at least one PriorityGroup to be set\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidPriorityGroupErr\",\n \"code\": 400,\n \"error_code\": 10160,\n \"description\": \"Provided priority group does not exist for this consumer\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerEmptyGroupName\",\n \"code\": 400,\n \"error_code\": 10161,\n \"description\": \"Group name cannot be an empty string\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidGroupNameErr\",\n \"code\": 400,\n \"error_code\": 10162,\n \"description\": \"Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamExpectedLastSeqPerSubjectNotReady\",\n \"code\": 503,\n \"error_code\": 10163,\n \"description\": \"expected last sequence per subject temporarily unavailable\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamWrongLastSequenceConstantErr\",\n \"code\": 400,\n \"error_code\": 10164,\n \"description\": \"wrong last sequence\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageTTLInvalidErr\",\n \"code\": 400,\n \"error_code\": 10165,\n \"description\": \"invalid per-message TTL\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageTTLDisabledErr\",\n \"code\": 400,\n \"error_code\": 10166,\n \"description\": \"per-message TTL is disabled\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamTooManyRequests\",\n \"code\": 429,\n \"error_code\": 10167,\n \"description\": \"too many requests\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageIncrDisabledErr\",\n \"code\": 400,\n \"error_code\": 10168,\n \"description\": \"message counters is disabled\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageIncrMissingErr\",\n \"code\": 400,\n \"error_code\": 10169,\n \"description\": \"message counter increment is missing\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageIncrPayloadErr\",\n \"code\": 400,\n \"error_code\": 10170,\n \"description\": \"message counter has payload\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageIncrInvalidErr\",\n \"code\": 400,\n \"error_code\": 10171,\n \"description\": \"message counter increment is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageCounterBrokenErr\",\n \"code\": 400,\n \"error_code\": 10172,\n \"description\": \"message counter is broken\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithCountersErr\",\n \"code\": 400,\n \"error_code\": 10173,\n \"description\": \"stream mirrors can not also calculate counters\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishDisabledErr\",\n \"code\": 400,\n \"error_code\": 10174,\n \"description\": \"atomic publish is disabled\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishMissingSeqErr\",\n \"code\": 400,\n \"error_code\": 10175,\n \"description\": \"atomic publish sequence is missing\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishIncompleteBatchErr\",\n \"code\": 400,\n \"error_code\": 10176,\n \"description\": \"atomic publish batch is incomplete\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishUnsupportedHeaderBatchErr\",\n \"code\": 400,\n \"error_code\": 10177,\n \"description\": \"atomic publish unsupported header used: {header}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPushWithPriorityGroupErr\",\n \"code\": 400,\n \"error_code\": 10178,\n \"description\": \"priority groups can not be used with push consumers\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishInvalidBatchIDErr\",\n \"code\": 400,\n \"error_code\": 10179,\n \"description\": \"atomic publish batch ID is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamMinLastSeqErr\",\n \"code\": 412,\n \"error_code\": 10180,\n \"description\": \"min last sequence\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerAckPolicyInvalidErr\",\n \"code\": 400,\n \"error_code\": 10181,\n \"description\": \"consumer ack policy invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerReplayPolicyInvalidErr\",\n \"code\": 400,\n \"error_code\": 10182,\n \"description\": \"consumer replay policy invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerAckWaitNegativeErr\",\n \"code\": 400,\n \"error_code\": 10183,\n \"description\": \"consumer ack wait needs to be positive\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerBackOffNegativeErr\",\n \"code\": 400,\n \"error_code\": 10184,\n \"description\": \"consumer backoff needs to be positive\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSRequiredApiLevelErr\",\n \"code\": 412,\n \"error_code\": 10185,\n \"description\": \"JetStream minimum api level required\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithMsgSchedulesErr\",\n \"code\": 400,\n \"error_code\": 10186,\n \"description\": \"stream mirrors can not also schedule messages\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSSourceWithMsgSchedulesErr\",\n \"code\": 400,\n \"error_code\": 10187,\n \"description\": \"stream source can not also schedule messages\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesDisabledErr\",\n \"code\": 400,\n \"error_code\": 10188,\n \"description\": \"message schedules is disabled\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesPatternInvalidErr\",\n \"code\": 400,\n \"error_code\": 10189,\n \"description\": \"message schedules pattern is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesTargetInvalidErr\",\n \"code\": 400,\n \"error_code\": 10190,\n \"description\": \"message schedules target is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesTTLInvalidErr\",\n \"code\": 400,\n \"error_code\": 10191,\n \"description\": \"message schedules invalid per-message TTL\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesRollupInvalidErr\",\n \"code\": 400,\n \"error_code\": 10192,\n \"description\": \"message schedules invalid rollup\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamExpectedLastSeqPerSubjectInvalid\",\n \"code\": 400,\n \"error_code\": 10193,\n \"description\": \"missing sequence for expected last sequence per subject\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSStreamOfflineReasonErrF\",\n \"code\": 500,\n \"error_code\": 10194,\n \"description\": \"stream is offline: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerOfflineReasonErrF\",\n \"code\": 500,\n \"error_code\": 10195,\n \"description\": \"consumer is offline: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPriorityGroupWithPolicyNone\",\n \"code\": 400,\n \"error_code\": 10196,\n \"description\": \"consumer can not have priority groups when policy is none\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerPinnedTTLWithoutPriorityPolicyNone\",\n \"code\": 400,\n \"error_code\": 10197,\n \"description\": \"PinnedTTL cannot be set when PriorityPolicy is none\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithAtomicPublishErr\",\n \"code\": 400,\n \"error_code\": 10198,\n \"description\": \"stream mirrors can not also use atomic publishing\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishTooLargeBatchErrF\",\n \"code\": 400,\n \"error_code\": 10199,\n \"description\": \"atomic publish batch is too large: {size}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishInvalidBatchCommitErr\",\n \"code\": 400,\n \"error_code\": 10200,\n \"description\": \"atomic publish batch commit is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishContainsDuplicateMessageErr\",\n \"code\": 400,\n \"error_code\": 10201,\n \"description\": \"atomic publish batch contains duplicate message id\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSClusterServerMemberChangeInflightErr\",\n \"code\": 400,\n \"error_code\": 10202,\n \"description\": \"cluster member change is in progress\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMessageSchedulesSourceInvalidErr\",\n \"code\": 400,\n \"error_code\": 10203,\n \"description\": \"message schedules source is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSConsumerInvalidResetErr\",\n \"code\": 400,\n \"error_code\": 10204,\n \"description\": \"invalid reset: {err}\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBatchPublishDisabledErr\",\n \"code\": 400,\n \"error_code\": 10205,\n \"description\": \"batch publish is disabled\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBatchPublishInvalidPatternErr\",\n \"code\": 400,\n \"error_code\": 10206,\n \"description\": \"batch publish pattern is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBatchPublishInvalidBatchIDErr\",\n \"code\": 400,\n \"error_code\": 10207,\n \"description\": \"batch publish ID is invalid\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBatchPublishUnknownBatchIDErr\",\n \"code\": 400,\n \"error_code\": 10208,\n \"description\": \"batch publish ID unknown\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSMirrorWithBatchPublishErr\",\n \"code\": 400,\n \"error_code\": 10209,\n \"description\": \"stream mirrors can not also use batch publishing\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSAtomicPublishTooManyInflight\",\n \"code\": 429,\n \"error_code\": 10210,\n \"description\": \"atomic publish too many inflight\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n },\n {\n \"constant\": \"JSBatchPublishTooManyInflight\",\n \"code\": 429,\n \"error_code\": 10211,\n \"description\": \"batch publish too many inflight\",\n \"comment\": \"\",\n \"help\": \"\",\n \"url\": \"\",\n \"deprecates\": \"\"\n }\n]" }, { "path": "server/errors_gen.go", "content": "//go:build ignore\n\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"text/template\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nvar tagRe = regexp.MustCompile(\"\\\\{(.+?)}\")\n\nvar templ = `\n// Generated code, do not edit. See errors.json and run go generate to update\n\npackage server\n\nimport \"strings\"\n\nconst (\n{{- range $i, $error := . }}\n{{- if .Comment }}\n\t// {{ .Constant }} {{ .Comment }} ({{ .Description | print }})\n{{- else }}\n\t// {{ .Constant }} {{ .Description | print }}\n{{- end }}\n\t{{ .Constant }} ErrorIdentifier = {{ .ErrCode }}\n{{ end }}\n)\n\nvar (\n\tApiErrors = map[ErrorIdentifier]*ApiError{\n{{- range $i, $error := . }}\n\t\t{{ .Constant }}: {Code: {{ .Code }},ErrCode: {{ .ErrCode }},Description: {{ .Description | printf \"%q\" }}},{{- end }}\n\t}\n\n{{- range $i, $error := . }}\n{{- if .Deprecates }}\n// {{ .Deprecates }} Deprecated by {{ .Constant }} ApiError, use IsNatsError() for comparisons\n{{ .Deprecates }} = ApiErrors[{{ .Constant }}]\n{{- end }}\n{{- end }}\n)\n\n{{- range $i, $error := . }}\n// {{ .Constant | funcNameForConstant }} creates a new {{ .Constant }} error: {{ .Description | printf \"%q\" }}\nfunc {{ .Constant | funcNameForConstant }}({{ .Description | funcArgsForTags }}) *ApiError {\n eopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n \t}\n{{ if .Description | hasTags }}\n\te:=ApiErrors[{{.Constant}}]\n\targs:=e.toReplacerArgs([]interface{}{ {{.Description | replacerArgsForTags }} })\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n{{- else }}\n\treturn ApiErrors[{{.Constant}}]\n{{- end }}\n}\n\n{{- end }}\n`\n\nfunc panicIfErr(err error) {\n\tif err == nil {\n\t\treturn\n\t}\n\tpanic(err)\n}\n\nfunc goFmt(file string) error {\n\tc := exec.Command(\"go\", \"fmt\", file)\n\tout, err := c.CombinedOutput()\n\tif err != nil {\n\t\tlog.Printf(\"go fmt failed: %s\", string(out))\n\t}\n\n\treturn err\n}\n\nfunc checkIncrements(errs []server.ErrorsData) error {\n\tsort.Slice(errs, func(i, j int) bool {\n\t\treturn errs[i].ErrCode < errs[j].ErrCode\n\t})\n\n\tlast := errs[0].ErrCode\n\tgaps := []uint16{}\n\n\tfor i := 1; i < len(errs); i++ {\n\t\tif errs[i].ErrCode != last+1 {\n\t\t\tgaps = append(gaps, last)\n\t\t}\n\t\tlast = errs[i].ErrCode\n\t}\n\n\tif len(gaps) > 0 {\n\t\treturn fmt.Errorf(\"gaps found in sequences: %v\", gaps)\n\t}\n\n\treturn nil\n}\n\nfunc checkDupes(errs []server.ErrorsData) error {\n\tcodes := []uint16{}\n\thighest := uint16(0)\n\tfor _, err := range errs {\n\t\tcodes = append(codes, err.ErrCode)\n\t\tif highest < err.ErrCode {\n\t\t\thighest = err.ErrCode\n\t\t}\n\t}\n\n\tcodeKeys := make(map[uint16]bool)\n\tconstKeys := make(map[string]bool)\n\n\tfor _, entry := range errs {\n\t\tif _, found := codeKeys[entry.ErrCode]; found {\n\t\t\treturn fmt.Errorf(\"duplicate error code %+v, highest code is %d\", entry, highest)\n\t\t}\n\n\t\tif _, found := constKeys[entry.Constant]; found {\n\t\t\treturn fmt.Errorf(\"duplicate error constant %+v\", entry)\n\t\t}\n\n\t\tcodeKeys[entry.ErrCode] = true\n\t\tconstKeys[entry.Constant] = true\n\t}\n\n\treturn nil\n}\n\nfunc findTags(d string) []string {\n\ttags := []string{}\n\tfor _, tag := range tagRe.FindAllStringSubmatch(d, -1) {\n\t\tif len(tag) != 2 {\n\t\t\tcontinue\n\t\t}\n\n\t\ttags = append(tags, tag[1])\n\t}\n\n\tsort.Strings(tags)\n\n\treturn tags\n}\n\nfunc main() {\n\tej, err := os.ReadFile(\"server/errors.json\")\n\tpanicIfErr(err)\n\n\terrs := []server.ErrorsData{}\n\tpanicIfErr(json.Unmarshal(ej, &errs))\n\n\tpanicIfErr(checkDupes(errs))\n\tpanicIfErr(checkIncrements(errs))\n\n\tsort.Slice(errs, func(i, j int) bool {\n\t\treturn errs[i].Constant < errs[j].Constant\n\t})\n\n\tt := template.New(\"errors\").Funcs(\n\t\ttemplate.FuncMap{\n\t\t\t\"inc\": func(i int) int { return i + 1 },\n\t\t\t\"hasTags\": func(d string) bool {\n\t\t\t\treturn strings.Contains(d, \"{\") && strings.Contains(d, \"}\")\n\t\t\t},\n\t\t\t\"replacerArgsForTags\": func(d string) string {\n\t\t\t\tres := []string{}\n\t\t\t\tfor _, tag := range findTags(d) {\n\t\t\t\t\tres = append(res, `\"{`+tag+`}\"`)\n\t\t\t\t\tres = append(res, tag)\n\t\t\t\t}\n\n\t\t\t\treturn strings.Join(res, \", \")\n\t\t\t},\n\t\t\t\"funcArgsForTags\": func(d string) string {\n\t\t\t\tres := []string{}\n\t\t\t\tfor _, tag := range findTags(d) {\n\t\t\t\t\tif tag == \"err\" {\n\t\t\t\t\t\tres = append(res, \"err error\")\n\t\t\t\t\t} else if tag == \"seq\" {\n\t\t\t\t\t\tres = append(res, \"seq uint64\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tres = append(res, fmt.Sprintf(\"%s interface{}\", tag))\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tres = append(res, \"opts ...ErrorOption\")\n\n\t\t\t\treturn strings.Join(res, \", \")\n\t\t\t},\n\t\t\t\"funcNameForConstant\": func(c string) string {\n\t\t\t\tres := \"\"\n\n\t\t\t\tswitch {\n\t\t\t\tcase strings.HasSuffix(c, \"ErrF\"):\n\t\t\t\t\tres = fmt.Sprintf(\"New%sError\", strings.TrimSuffix(c, \"ErrF\"))\n\t\t\t\tcase strings.HasSuffix(c, \"Err\"):\n\t\t\t\t\tres = fmt.Sprintf(\"New%sError\", strings.TrimSuffix(c, \"Err\"))\n\t\t\t\tcase strings.HasSuffix(c, \"ErrorF\"):\n\t\t\t\t\tres = fmt.Sprintf(\"New%s\", strings.TrimSuffix(c, \"F\"))\n\t\t\t\tcase strings.HasSuffix(c, \"F\"):\n\t\t\t\t\tres = fmt.Sprintf(\"New%sError\", strings.TrimSuffix(c, \"F\"))\n\t\t\t\tdefault:\n\t\t\t\t\tres = fmt.Sprintf(\"New%s\", c)\n\t\t\t\t}\n\n\t\t\t\tif !strings.HasSuffix(res, \"Error\") {\n\t\t\t\t\tres = fmt.Sprintf(\"%sError\", res)\n\t\t\t\t}\n\n\t\t\t\treturn res\n\t\t\t},\n\t\t})\n\tp, err := t.Parse(templ)\n\tpanicIfErr(err)\n\n\ttf, err := os.CreateTemp(\"\", \"\")\n\tpanicIfErr(err)\n\tdefer tf.Close()\n\n\tpanicIfErr(p.Execute(tf, errs))\n\n\tpanicIfErr(os.Rename(tf.Name(), \"server/jetstream_errors_generated.go\"))\n\tpanicIfErr(goFmt(\"server/jetstream_errors_generated.go\"))\n}\n" }, { "path": "server/errors_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestErrCtx(t *testing.T) {\n\tctx := \"Extra context information\"\n\te := NewErrorCtx(ErrWrongGateway, \"%s\", ctx)\n\n\tif e.Error() != ErrWrongGateway.Error() {\n\t\tt.Fatalf(\"%v and %v are supposed to be identical\", e, ErrWrongGateway)\n\t}\n\tif e == ErrWrongGateway {\n\t\tt.Fatalf(\"%v and %v can't be compared this way\", e, ErrWrongGateway)\n\t}\n\tif !ErrorIs(e, ErrWrongGateway) {\n\t\tt.Fatalf(\"%s and %s \", e, ErrWrongGateway)\n\t}\n\tif UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() {\n\t\tt.Fatalf(\"Error of different type should be processed unchanged\")\n\t}\n\ttrace := UnpackIfErrorCtx(e)\n\tif !strings.HasPrefix(trace, ErrWrongGateway.Error()) {\n\t\tt.Fatalf(\"original error needs to remain\")\n\t}\n\tif !strings.HasSuffix(trace, ctx) {\n\t\tt.Fatalf(\"ctx needs to be added\")\n\t}\n}\n\nfunc TestErrCtxWrapped(t *testing.T) {\n\tctxO := \"Original Ctx\"\n\teO := NewErrorCtx(ErrWrongGateway, \"%s\", ctxO)\n\tctx := \"Extra context information\"\n\te := NewErrorCtx(eO, \"%s\", ctx)\n\n\tif e.Error() != ErrWrongGateway.Error() {\n\t\tt.Fatalf(\"%v and %v are supposed to be identical\", e, ErrWrongGateway)\n\t}\n\tif e == ErrWrongGateway {\n\t\tt.Fatalf(\"%v and %v can't be compared this way\", e, ErrWrongGateway)\n\t}\n\tif !ErrorIs(e, ErrWrongGateway) {\n\t\tt.Fatalf(\"%s and %s \", e, ErrWrongGateway)\n\t}\n\tif UnpackIfErrorCtx(ErrWrongGateway) != ErrWrongGateway.Error() {\n\t\tt.Fatalf(\"Error of different type should be processed unchanged\")\n\t}\n\ttrace := UnpackIfErrorCtx(e)\n\tif !strings.HasPrefix(trace, ErrWrongGateway.Error()) {\n\t\tt.Fatalf(\"original error needs to remain\")\n\t}\n\tif !strings.HasSuffix(trace, ctx) {\n\t\tt.Fatalf(\"ctx needs to be added\")\n\t}\n\tif !strings.Contains(trace, ctxO) {\n\t\tt.Fatalf(\"Needs to contain every context\")\n\t}\n}\n" }, { "path": "server/events.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"crypto/sha256\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/server/certidp\"\n\t\"github.com/nats-io/nats-server/v2/server/pse\"\n)\n\nconst (\n\taccLookupReqTokens = 6\n\taccLookupReqSubj = \"$SYS.REQ.ACCOUNT.%s.CLAIMS.LOOKUP\"\n\taccPackReqSubj = \"$SYS.REQ.CLAIMS.PACK\"\n\taccListReqSubj = \"$SYS.REQ.CLAIMS.LIST\"\n\taccClaimsReqSubj = \"$SYS.REQ.CLAIMS.UPDATE\"\n\taccDeleteReqSubj = \"$SYS.REQ.CLAIMS.DELETE\"\n\n\tconnectEventSubj = \"$SYS.ACCOUNT.%s.CONNECT\"\n\tdisconnectEventSubj = \"$SYS.ACCOUNT.%s.DISCONNECT\"\n\taccDirectReqSubj = \"$SYS.REQ.ACCOUNT.%s.%s\"\n\taccPingReqSubj = \"$SYS.REQ.ACCOUNT.PING.%s\" // atm. only used for STATZ and CONNZ import from system account\n\t// kept for backward compatibility when using http resolver\n\t// this overlaps with the names for events but you'd have to have the operator private key in order to succeed.\n\taccUpdateEventSubjOld = \"$SYS.ACCOUNT.%s.CLAIMS.UPDATE\"\n\taccUpdateEventSubjNew = \"$SYS.REQ.ACCOUNT.%s.CLAIMS.UPDATE\"\n\tconnsRespSubj = \"$SYS._INBOX_.%s\"\n\taccConnsEventSubjNew = \"$SYS.ACCOUNT.%s.SERVER.CONNS\"\n\taccConnsEventSubjOld = \"$SYS.SERVER.ACCOUNT.%s.CONNS\" // kept for backward compatibility\n\tlameDuckEventSubj = \"$SYS.SERVER.%s.LAMEDUCK\"\n\tshutdownEventSubj = \"$SYS.SERVER.%s.SHUTDOWN\"\n\tclientKickReqSubj = \"$SYS.REQ.SERVER.%s.KICK\"\n\tclientLDMReqSubj = \"$SYS.REQ.SERVER.%s.LDM\"\n\tauthErrorEventSubj = \"$SYS.SERVER.%s.CLIENT.AUTH.ERR\"\n\tauthErrorAccountEventSubj = \"$SYS.ACCOUNT.CLIENT.AUTH.ERR\"\n\tserverStatsSubj = \"$SYS.SERVER.%s.STATSZ\"\n\tserverDirectReqSubj = \"$SYS.REQ.SERVER.%s.%s\"\n\tserverPingReqSubj = \"$SYS.REQ.SERVER.PING.%s\"\n\tserverStatsPingReqSubj = \"$SYS.REQ.SERVER.PING\" // use $SYS.REQ.SERVER.PING.STATSZ instead\n\tserverReloadReqSubj = \"$SYS.REQ.SERVER.%s.RELOAD\" // with server ID\n\tleafNodeConnectEventSubj = \"$SYS.ACCOUNT.%s.LEAFNODE.CONNECT\" // for internal use only\n\tremoteLatencyEventSubj = \"$SYS.LATENCY.M2.%s\"\n\tinboxRespSubj = \"$SYS._INBOX.%s.%s\"\n\n\t// Used to return information to a user on bound account and user permissions.\n\tuserDirectInfoSubj = \"$SYS.REQ.USER.INFO\"\n\tuserDirectReqSubj = \"$SYS.REQ.USER.%s.INFO\"\n\n\t// FIXME(dlc) - Should account scope, even with wc for now, but later on\n\t// we can then shard as needed.\n\taccNumSubsReqSubj = \"$SYS.REQ.ACCOUNT.NSUBS\"\n\n\t// These are for exported debug services. These are local to this server only.\n\taccSubsSubj = \"$SYS.DEBUG.SUBSCRIBERS\"\n\n\tshutdownEventTokens = 4\n\tserverSubjectIndex = 2\n\taccUpdateTokensNew = 6\n\taccUpdateTokensOld = 5\n\taccUpdateAccIdxOld = 2\n\n\taccReqTokens = 5\n\taccReqAccIndex = 3\n\n\tocspPeerRejectEventSubj = \"$SYS.SERVER.%s.OCSP.PEER.CONN.REJECT\"\n\tocspPeerChainlinkInvalidEventSubj = \"$SYS.SERVER.%s.OCSP.PEER.LINK.INVALID\"\n)\n\n// FIXME(dlc) - make configurable.\nvar eventsHBInterval = 30 * time.Second\nvar statsHBInterval = 10 * time.Second\n\n// Default minimum wait time for sending statsz\nconst defaultStatszRateLimit = 1 * time.Second\n\n// Variable version so we can set in tests.\nvar statszRateLimit = defaultStatszRateLimit\n\ntype sysMsgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, hdr, msg []byte)\n\n// Used if we have to queue things internally to avoid the route/gw path.\ntype inSysMsg struct {\n\tsub *subscription\n\tc *client\n\tacc *Account\n\tsubj string\n\trply string\n\thdr []byte\n\tmsg []byte\n\tcb sysMsgHandler\n}\n\n// Used to send and receive messages from inside the server.\ntype internal struct {\n\taccount *Account\n\tclient *client\n\tseq uint64\n\tsid int\n\tservers map[string]*serverUpdate\n\tsweeper *time.Timer\n\tstmr *time.Timer\n\treplies map[string]msgHandler\n\tsendq *ipQueue[*pubMsg]\n\trecvq *ipQueue[*inSysMsg]\n\trecvqp *ipQueue[*inSysMsg] // For STATSZ/Pings\n\tresetCh chan struct{}\n\twg sync.WaitGroup\n\tsq *sendq\n\torphMax time.Duration\n\tchkOrph time.Duration\n\tstatsz time.Duration\n\tcstatsz time.Duration\n\tshash string\n\tinboxPre string\n\tremoteStatsSub *subscription\n\tlastStatsz time.Time\n}\n\n// ServerStatsMsg is sent periodically with stats updates.\ntype ServerStatsMsg struct {\n\tServer ServerInfo `json:\"server\"`\n\tStats ServerStats `json:\"statsz\"`\n}\n\n// ConnectEventMsg is sent when a new connection is made that is part of an account.\ntype ConnectEventMsg struct {\n\tTypedEvent\n\tServer ServerInfo `json:\"server\"`\n\tClient ClientInfo `json:\"client\"`\n}\n\n// ConnectEventMsgType is the schema type for ConnectEventMsg\nconst ConnectEventMsgType = \"io.nats.server.advisory.v1.client_connect\"\n\n// DisconnectEventMsg is sent when a new connection previously defined from a\n// ConnectEventMsg is closed.\ntype DisconnectEventMsg struct {\n\tTypedEvent\n\tServer ServerInfo `json:\"server\"`\n\tClient ClientInfo `json:\"client\"`\n\tSent DataStats `json:\"sent\"`\n\tReceived DataStats `json:\"received\"`\n\tReason string `json:\"reason\"`\n}\n\n// DisconnectEventMsgType is the schema type for DisconnectEventMsg\nconst DisconnectEventMsgType = \"io.nats.server.advisory.v1.client_disconnect\"\n\n// OCSPPeerRejectEventMsg is sent when a peer TLS handshake is ultimately rejected due to OCSP invalidation.\n// A \"peer\" can be an inbound client connection or a leaf connection to a remote server. Peer in event payload\n// is always the peer's (TLS) leaf cert, which may or may be the invalid cert (See also OCSPPeerChainlinkInvalidEventMsg)\ntype OCSPPeerRejectEventMsg struct {\n\tTypedEvent\n\tKind string `json:\"kind\"`\n\tPeer certidp.CertInfo `json:\"peer\"`\n\tServer ServerInfo `json:\"server\"`\n\tReason string `json:\"reason\"`\n}\n\n// OCSPPeerRejectEventMsgType is the schema type for OCSPPeerRejectEventMsg\nconst OCSPPeerRejectEventMsgType = \"io.nats.server.advisory.v1.ocsp_peer_reject\"\n\n// OCSPPeerChainlinkInvalidEventMsg is sent when a certificate (link) in a valid TLS chain is found to be OCSP invalid\n// during a peer TLS handshake. A \"peer\" can be an inbound client connection or a leaf connection to a remote server.\n// Peer and Link may be the same if the invalid cert was the peer's leaf cert\ntype OCSPPeerChainlinkInvalidEventMsg struct {\n\tTypedEvent\n\tLink certidp.CertInfo `json:\"link\"`\n\tPeer certidp.CertInfo `json:\"peer\"`\n\tServer ServerInfo `json:\"server\"`\n\tReason string `json:\"reason\"`\n}\n\n// OCSPPeerChainlinkInvalidEventMsgType is the schema type for OCSPPeerChainlinkInvalidEventMsg\nconst OCSPPeerChainlinkInvalidEventMsgType = \"io.nats.server.advisory.v1.ocsp_peer_link_invalid\"\n\n// AccountNumConns is an event that will be sent from a server that is tracking\n// a given account when the number of connections changes. It will also HB\n// updates in the absence of any changes.\ntype AccountNumConns struct {\n\tTypedEvent\n\tServer ServerInfo `json:\"server\"`\n\tAccountStat\n}\n\n// AccountStat contains the data common between AccountNumConns and AccountStatz\ntype AccountStat struct {\n\tAccount string `json:\"acc\"`\n\tName string `json:\"name\"`\n\tConns int `json:\"conns\"`\n\tLeafNodes int `json:\"leafnodes\"`\n\tTotalConns int `json:\"total_conns\"`\n\tNumSubs uint32 `json:\"num_subscriptions\"`\n\tSent DataStats `json:\"sent\"`\n\tReceived DataStats `json:\"received\"`\n\tSlowConsumers int64 `json:\"slow_consumers\"`\n}\n\nconst AccountNumConnsMsgType = \"io.nats.server.advisory.v1.account_connections\"\n\n// accNumConnsReq is sent when we are starting to track an account for the first\n// time. We will request others send info to us about their local state.\ntype accNumConnsReq struct {\n\tServer ServerInfo `json:\"server\"`\n\tAccount string `json:\"acc\"`\n}\n\n// ServerID is basic static info for a server.\ntype ServerID struct {\n\tName string `json:\"name\"`\n\tHost string `json:\"host\"`\n\tID string `json:\"id\"`\n}\n\n// Type for our server capabilities.\ntype ServerCapability uint64\n\n// ServerInfo identifies remote servers.\ntype ServerInfo struct {\n\tName string `json:\"name\"`\n\tHost string `json:\"host\"`\n\tID string `json:\"id\"`\n\tCluster string `json:\"cluster,omitempty\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tVersion string `json:\"ver\"`\n\tTags []string `json:\"tags,omitempty\"`\n\tMetadata map[string]string `json:\"metadata,omitempty\"`\n\t// Whether JetStream is enabled (deprecated in favor of the `ServerCapability`).\n\tJetStream bool `json:\"jetstream\"`\n\t// Generic capability flags\n\tFlags ServerCapability `json:\"flags\"`\n\t// Sequence and Time from the remote server for this message.\n\tSeq uint64 `json:\"seq\"`\n\tTime time.Time `json:\"time\"`\n}\n\nconst (\n\tJetStreamEnabled ServerCapability = 1 << iota // Server had JetStream enabled.\n\tBinaryStreamSnapshot // New stream snapshot capability.\n\tAccountNRG // Move NRG traffic out of system account.\n)\n\n// Set JetStream capability.\nfunc (si *ServerInfo) SetJetStreamEnabled() {\n\tsi.Flags |= JetStreamEnabled\n\t// Still set old version.\n\tsi.JetStream = true\n}\n\n// JetStreamEnabled indicates whether or not we have JetStream enabled.\nfunc (si *ServerInfo) JetStreamEnabled() bool {\n\t// Take into account old version.\n\treturn si.Flags&JetStreamEnabled != 0 || si.JetStream\n}\n\n// Set binary stream snapshot capability.\nfunc (si *ServerInfo) SetBinaryStreamSnapshot() {\n\tsi.Flags |= BinaryStreamSnapshot\n}\n\n// JetStreamEnabled indicates whether or not we have binary stream snapshot capbilities.\nfunc (si *ServerInfo) BinaryStreamSnapshot() bool {\n\treturn si.Flags&BinaryStreamSnapshot != 0\n}\n\n// Set account NRG capability.\nfunc (si *ServerInfo) SetAccountNRG() {\n\tsi.Flags |= AccountNRG\n}\n\n// AccountNRG indicates whether or not we support moving the NRG traffic out of the\n// system account and into the asset account.\nfunc (si *ServerInfo) AccountNRG() bool {\n\treturn si.Flags&AccountNRG != 0\n}\n\n// ClientInfo is detailed information about the client forming a connection.\ntype ClientInfo struct {\n\tStart *time.Time `json:\"start,omitempty\"`\n\tHost string `json:\"host,omitempty\"`\n\tID uint64 `json:\"id,omitempty\"`\n\tAccount string `json:\"acc,omitempty\"`\n\tService string `json:\"svc,omitempty\"`\n\tUser string `json:\"user,omitempty\"`\n\tName string `json:\"name,omitempty\"`\n\tLang string `json:\"lang,omitempty\"`\n\tVersion string `json:\"ver,omitempty\"`\n\tRTT time.Duration `json:\"rtt,omitempty\"`\n\tServer string `json:\"server,omitempty\"`\n\tCluster string `json:\"cluster,omitempty\"`\n\tAlternates []string `json:\"alts,omitempty\"`\n\tStop *time.Time `json:\"stop,omitempty\"`\n\tJwt string `json:\"jwt,omitempty\"`\n\tIssuerKey string `json:\"issuer_key,omitempty\"`\n\tNameTag string `json:\"name_tag,omitempty\"`\n\tTags jwt.TagList `json:\"tags,omitempty\"`\n\tKind string `json:\"kind,omitempty\"`\n\tClientType string `json:\"client_type,omitempty\"`\n\tMQTTClient string `json:\"client_id,omitempty\"` // This is the MQTT client ID\n\tNonce string `json:\"nonce,omitempty\"`\n\tReply string `json:\"reply,omitempty\"` // Original reply subject after a service import (only when needed).\n}\n\n// forAssignmentSnap returns the minimum amount of ClientInfo we need for assignment snapshots.\nfunc (ci *ClientInfo) forAssignmentSnap() *ClientInfo {\n\treturn &ClientInfo{\n\t\tAccount: ci.Account,\n\t\tService: ci.Service,\n\t\tCluster: ci.Cluster,\n\t}\n}\n\n// forProposal returns the minimum amount of ClientInfo we need for assignment proposals.\nfunc (ci *ClientInfo) forProposal() *ClientInfo {\n\tif ci == nil {\n\t\treturn nil\n\t}\n\tcci := *ci\n\tcci.Jwt = _EMPTY_\n\tcci.IssuerKey = _EMPTY_\n\treturn &cci\n}\n\n// forAdvisory returns the minimum amount of ClientInfo we need for JS advisory events.\nfunc (ci *ClientInfo) forAdvisory() *ClientInfo {\n\tif ci == nil {\n\t\treturn nil\n\t}\n\tcci := *ci\n\tcci.Jwt = _EMPTY_\n\tcci.Alternates = nil\n\treturn &cci\n}\n\n// ServerStats hold various statistics that we will periodically send out.\ntype ServerStats struct {\n\tStart time.Time `json:\"start\"`\n\tMem int64 `json:\"mem\"`\n\tCores int `json:\"cores\"`\n\tCPU float64 `json:\"cpu\"`\n\tConnections int `json:\"connections\"`\n\tTotalConnections uint64 `json:\"total_connections\"`\n\tActiveAccounts int `json:\"active_accounts\"`\n\tNumSubs uint32 `json:\"subscriptions\"`\n\tSent DataStats `json:\"sent\"`\n\tReceived DataStats `json:\"received\"`\n\tSlowConsumers int64 `json:\"slow_consumers\"`\n\tSlowConsumersStats *SlowConsumersStats `json:\"slow_consumer_stats,omitempty\"`\n\tStaleConnections int64 `json:\"stale_connections,omitempty\"`\n\tStaleConnectionStats *StaleConnectionStats `json:\"stale_connection_stats,omitempty\"`\n\tStalledClients int64 `json:\"stalled_clients,omitempty\"`\n\tRoutes []*RouteStat `json:\"routes,omitempty\"`\n\tGateways []*GatewayStat `json:\"gateways,omitempty\"`\n\tActiveServers int `json:\"active_servers,omitempty\"`\n\tJetStream *JetStreamVarz `json:\"jetstream,omitempty\"`\n\tMemLimit int64 `json:\"gomemlimit,omitempty\"`\n\tMaxProcs int `json:\"gomaxprocs,omitempty\"`\n}\n\n// RouteStat holds route statistics.\ntype RouteStat struct {\n\tID uint64 `json:\"rid\"`\n\tName string `json:\"name,omitempty\"`\n\tSent DataStats `json:\"sent\"`\n\tReceived DataStats `json:\"received\"`\n\tPending int `json:\"pending\"`\n}\n\n// GatewayStat holds gateway statistics.\ntype GatewayStat struct {\n\tID uint64 `json:\"gwid\"`\n\tName string `json:\"name\"`\n\tSent DataStats `json:\"sent\"`\n\tReceived DataStats `json:\"received\"`\n\tNumInbound int `json:\"inbound_connections\"`\n}\n\ntype MsgBytes struct {\n\tMsgs int64 `json:\"msgs\"`\n\tBytes int64 `json:\"bytes\"`\n}\n\n// DataStats reports how may msg and bytes. Applicable for both sent and received.\ntype DataStats struct {\n\tMsgBytes\n\tGateways *MsgBytes `json:\"gateways,omitempty\"`\n\tRoutes *MsgBytes `json:\"routes,omitempty\"`\n\tLeafs *MsgBytes `json:\"leafs,omitempty\"`\n}\n\n// Used for internally queueing up messages that the server wants to send.\ntype pubMsg struct {\n\tc *client\n\tsub string\n\trply string\n\tsi *ServerInfo\n\thdr []byte\n\tmsg any\n\toct compressionType\n\techo bool\n\tlast bool\n}\n\nvar pubMsgPool sync.Pool\n\nfunc newPubMsg(c *client, sub, rply string, si *ServerInfo, hdr []byte,\n\tmsg any, oct compressionType, echo, last bool) *pubMsg {\n\n\tvar m *pubMsg\n\tpm := pubMsgPool.Get()\n\tif pm != nil {\n\t\tm = pm.(*pubMsg)\n\t} else {\n\t\tm = &pubMsg{}\n\t}\n\t// When getting something from a pool it is critical that all fields are\n\t// initialized. Doing this way guarantees that if someone adds a field to\n\t// the structure, the compiler will fail the build if this line is not updated.\n\t(*m) = pubMsg{c, sub, rply, si, hdr, msg, oct, echo, last}\n\treturn m\n}\n\nfunc (pm *pubMsg) returnToPool() {\n\tif pm == nil {\n\t\treturn\n\t}\n\tpm.c, pm.sub, pm.rply, pm.si, pm.hdr, pm.msg = nil, _EMPTY_, _EMPTY_, nil, nil, nil\n\tpubMsgPool.Put(pm)\n}\n\n// Used to track server updates.\ntype serverUpdate struct {\n\tseq uint64\n\tltime time.Time\n}\n\n// TypedEvent is a event or advisory sent by the server that has nats type hints\n// typically used for events that might be consumed by 3rd party event systems\ntype TypedEvent struct {\n\tType string `json:\"type\"`\n\tID string `json:\"id\"`\n\tTime time.Time `json:\"timestamp\"`\n}\n\n// internalReceiveLoop will be responsible for dispatching all messages that\n// a server receives and needs to internally process, e.g. internal subs.\nfunc (s *Server) internalReceiveLoop(recvq *ipQueue[*inSysMsg]) {\n\tfor s.eventsRunning() {\n\t\tselect {\n\t\tcase <-recvq.ch:\n\t\t\tmsgs := recvq.pop()\n\t\t\tfor _, m := range msgs {\n\t\t\t\tif m.cb != nil {\n\t\t\t\t\tm.cb(m.sub, m.c, m.acc, m.subj, m.rply, m.hdr, m.msg)\n\t\t\t\t}\n\t\t\t}\n\t\t\trecvq.recycle(&msgs)\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// internalSendLoop will be responsible for serializing all messages that\n// a server wants to send.\nfunc (s *Server) internalSendLoop(wg *sync.WaitGroup) {\n\tdefer wg.Done()\n\nRESET:\n\ts.mu.RLock()\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\tsysc := s.sys.client\n\tresetCh := s.sys.resetCh\n\tsendq := s.sys.sendq\n\tid := s.info.ID\n\thost := s.info.Host\n\tservername := s.info.Name\n\tdomain := s.info.Domain\n\tseqp := &s.sys.seq\n\tjs := s.info.JetStream\n\tcluster := s.info.Cluster\n\tif s.gateway.enabled {\n\t\tcluster = s.getGatewayName()\n\t}\n\ts.mu.RUnlock()\n\n\t// Grab tags and metadata.\n\topts := s.getOpts()\n\ttags, metadata := opts.Tags, opts.Metadata\n\n\tfor s.eventsRunning() {\n\t\tselect {\n\t\tcase <-sendq.ch:\n\t\t\tmsgs := sendq.pop()\n\t\t\tfor _, pm := range msgs {\n\t\t\t\tif si := pm.si; si != nil {\n\t\t\t\t\tsi.Name = servername\n\t\t\t\t\tsi.Domain = domain\n\t\t\t\t\tsi.Host = host\n\t\t\t\t\tsi.Cluster = cluster\n\t\t\t\t\tsi.ID = id\n\t\t\t\t\tsi.Seq = atomic.AddUint64(seqp, 1)\n\t\t\t\t\tsi.Version = VERSION\n\t\t\t\t\tsi.Time = time.Now().UTC()\n\t\t\t\t\tsi.Tags = tags\n\t\t\t\t\tsi.Metadata = metadata\n\t\t\t\t\tsi.Flags = 0\n\t\t\t\t\tif js {\n\t\t\t\t\t\t// New capability based flags.\n\t\t\t\t\t\tsi.SetJetStreamEnabled()\n\t\t\t\t\t\tsi.SetBinaryStreamSnapshot()\n\t\t\t\t\t\tif s.accountNRGAllowed.Load() {\n\t\t\t\t\t\t\tsi.SetAccountNRG()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tvar b []byte\n\t\t\t\tif pm.msg != nil {\n\t\t\t\t\tswitch v := pm.msg.(type) {\n\t\t\t\t\tcase string:\n\t\t\t\t\t\tb = []byte(v)\n\t\t\t\t\tcase []byte:\n\t\t\t\t\t\tb = v\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tb, _ = json.Marshal(pm.msg)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Setup our client. If the user wants to use a non-system account use our internal\n\t\t\t\t// account scoped here so that we are not changing out accounts for the system client.\n\t\t\t\tvar c *client\n\t\t\t\tif pm.c != nil {\n\t\t\t\t\tc = pm.c\n\t\t\t\t} else {\n\t\t\t\t\tc = sysc\n\t\t\t\t}\n\n\t\t\t\t// Grab client lock.\n\t\t\t\tc.mu.Lock()\n\n\t\t\t\t// Prep internal structures needed to send message.\n\t\t\t\tc.pa.subject, c.pa.reply = []byte(pm.sub), []byte(pm.rply)\n\t\t\t\tc.pa.size, c.pa.szb = len(b), []byte(strconv.FormatInt(int64(len(b)), 10))\n\t\t\t\tc.pa.hdr, c.pa.hdb = -1, nil\n\t\t\t\ttrace := c.trace\n\n\t\t\t\t// Now check for optional compression.\n\t\t\t\tvar contentHeader string\n\t\t\t\tvar bb bytes.Buffer\n\n\t\t\t\tif len(b) > 0 {\n\t\t\t\t\tswitch pm.oct {\n\t\t\t\t\tcase gzipCompression:\n\t\t\t\t\t\tzw := gzip.NewWriter(&bb)\n\t\t\t\t\t\tzw.Write(b)\n\t\t\t\t\t\tzw.Close()\n\t\t\t\t\t\tb = bb.Bytes()\n\t\t\t\t\t\tcontentHeader = \"gzip\"\n\t\t\t\t\tcase snappyCompression:\n\t\t\t\t\t\tsw := s2.NewWriter(&bb, s2.WriterSnappyCompat())\n\t\t\t\t\t\tsw.Write(b)\n\t\t\t\t\t\tsw.Close()\n\t\t\t\t\t\tb = bb.Bytes()\n\t\t\t\t\t\tcontentHeader = \"snappy\"\n\t\t\t\t\tcase unsupportedCompression:\n\t\t\t\t\t\tcontentHeader = \"identity\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Optional Echo\n\t\t\t\treplaceEcho := c.echo != pm.echo\n\t\t\t\tif replaceEcho {\n\t\t\t\t\tc.echo = !c.echo\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\n\t\t\t\t// Add in NL\n\t\t\t\tb = append(b, _CRLF_...)\n\n\t\t\t\t// Optional raw header addition.\n\t\t\t\tif pm.hdr != nil {\n\t\t\t\t\tb = append(pm.hdr, b...)\n\t\t\t\t\tnhdr := len(pm.hdr)\n\t\t\t\t\tnsize := len(b) - LEN_CR_LF\n\t\t\t\t\t// MQTT producers don't have CRLF, so add it back.\n\t\t\t\t\tif c.isMqtt() {\n\t\t\t\t\t\tnsize += LEN_CR_LF\n\t\t\t\t\t}\n\t\t\t\t\t// Update pubArgs\n\t\t\t\t\t// If others will use this later we need to save and restore original.\n\t\t\t\t\tc.pa.hdr = nhdr\n\t\t\t\t\tc.pa.size = nsize\n\t\t\t\t\tc.pa.hdb = []byte(strconv.Itoa(nhdr))\n\t\t\t\t\tc.pa.szb = []byte(strconv.Itoa(nsize))\n\t\t\t\t}\n\n\t\t\t\t// Check if we should set content-encoding\n\t\t\t\tif contentHeader != _EMPTY_ {\n\t\t\t\t\tb = c.setHeader(contentEncodingHeader, contentHeader, b)\n\t\t\t\t}\n\n\t\t\t\t// Tracing\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(fmt.Sprintf(\"PUB %s %s %d\", c.pa.subject, c.pa.reply, c.pa.size), nil)\n\t\t\t\t\tc.traceMsg(b)\n\t\t\t\t}\n\n\t\t\t\t// Process like a normal inbound msg.\n\t\t\t\tc.processInboundClientMsg(b)\n\n\t\t\t\t// Put echo back if needed.\n\t\t\t\tif replaceEcho {\n\t\t\t\t\tc.mu.Lock()\n\t\t\t\t\tc.echo = !c.echo\n\t\t\t\t\tc.mu.Unlock()\n\t\t\t\t}\n\n\t\t\t\t// See if we are doing graceful shutdown.\n\t\t\t\tif !pm.last {\n\t\t\t\t\tc.flushClients(0) // Never spend time in place.\n\t\t\t\t} else {\n\t\t\t\t\t// For the Shutdown event, we need to send in place otherwise\n\t\t\t\t\t// there is a chance that the process will exit before the\n\t\t\t\t\t// writeLoop has a chance to send it.\n\t\t\t\t\tc.flushClients(time.Second)\n\t\t\t\t\tsendq.recycle(&msgs)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tpm.returnToPool()\n\t\t\t}\n\t\t\tsendq.recycle(&msgs)\n\t\tcase <-resetCh:\n\t\t\tgoto RESET\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Will send a shutdown message for lame-duck. Unlike sendShutdownEvent, this will\n// not close off the send queue or reply handler, as we may still have a workload\n// that needs migrating off.\n// Lock should be held.\nfunc (s *Server) sendLDMShutdownEventLocked() {\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\treturn\n\t}\n\tsubj := fmt.Sprintf(lameDuckEventSubj, s.info.ID)\n\tsi := &ServerInfo{}\n\ts.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true))\n}\n\n// Will send a shutdown message.\nfunc (s *Server) sendShutdownEvent() {\n\ts.mu.Lock()\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\tsubj := fmt.Sprintf(shutdownEventSubj, s.info.ID)\n\tsendq := s.sys.sendq\n\t// Stop any more messages from queueing up.\n\ts.sys.sendq = nil\n\t// Unhook all msgHandlers. Normal client cleanup will deal with subs, etc.\n\ts.sys.replies = nil\n\t// Send to the internal queue and mark as last.\n\tsi := &ServerInfo{}\n\tsendq.push(newPubMsg(nil, subj, _EMPTY_, si, nil, si, noCompression, false, true))\n\ts.mu.Unlock()\n}\n\n// Used to send an internal message to an arbitrary account.\nfunc (s *Server) sendInternalAccountMsg(a *Account, subject string, msg any) error {\n\treturn s.sendInternalAccountMsgWithReply(a, subject, _EMPTY_, nil, msg, false)\n}\n\n// Used to send an internal message with an optional reply to an arbitrary account.\nfunc (s *Server) sendInternalAccountMsgWithReply(a *Account, subject, reply string, hdr []byte, msg any, echo bool) error {\n\ts.mu.RLock()\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\ts.mu.RUnlock()\n\t\tif s.isShuttingDown() {\n\t\t\t// Skip in case this was called at the end phase during shut down\n\t\t\t// to avoid too many entries in the logs.\n\t\t\treturn nil\n\t\t}\n\t\treturn ErrNoSysAccount\n\t}\n\tc := s.sys.client\n\t// Replace our client with the account's internal client.\n\tif a != nil {\n\t\ta.mu.Lock()\n\t\tc = a.internalClient()\n\t\ta.mu.Unlock()\n\t}\n\ts.sys.sendq.push(newPubMsg(c, subject, reply, nil, hdr, msg, noCompression, echo, false))\n\ts.mu.RUnlock()\n\treturn nil\n}\n\n// Send system style message to an account scope.\nfunc (s *Server) sendInternalAccountSysMsg(a *Account, subj string, si *ServerInfo, msg any, ct compressionType) {\n\ts.mu.RLock()\n\tif s.sys == nil || s.sys.sendq == nil || a == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\tsendq := s.sys.sendq\n\ts.mu.RUnlock()\n\n\ta.mu.Lock()\n\tc := a.internalClient()\n\ta.mu.Unlock()\n\n\tsendq.push(newPubMsg(c, subj, _EMPTY_, si, nil, msg, ct, false, false))\n}\n\n// This will queue up a message to be sent.\n// Lock should not be held.\nfunc (s *Server) sendInternalMsgLocked(subj, rply string, si *ServerInfo, msg any) {\n\ts.mu.RLock()\n\ts.sendInternalMsg(subj, rply, si, msg)\n\ts.mu.RUnlock()\n}\n\n// This will queue up a message to be sent.\n// Assumes lock is held on entry.\nfunc (s *Server) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) {\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\treturn\n\t}\n\ts.sys.sendq.push(newPubMsg(nil, subj, rply, si, nil, msg, noCompression, false, false))\n}\n\n// Will send an api response.\nfunc (s *Server) sendInternalResponse(subj string, response *ServerAPIResponse) {\n\ts.mu.RLock()\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\ts.sys.sendq.push(newPubMsg(nil, subj, _EMPTY_, response.Server, nil, response, response.compress, false, false))\n\ts.mu.RUnlock()\n}\n\n// Used to send internal messages from other system clients to avoid no echo issues.\nfunc (c *client) sendInternalMsg(subj, rply string, si *ServerInfo, msg any) {\n\tif c == nil {\n\t\treturn\n\t}\n\ts := c.srv\n\tif s == nil {\n\t\treturn\n\t}\n\ts.mu.RLock()\n\tif s.sys == nil || s.sys.sendq == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\ts.sys.sendq.push(newPubMsg(c, subj, rply, si, nil, msg, noCompression, false, false))\n\ts.mu.RUnlock()\n}\n\n// Locked version of checking if events system running. Also checks server.\nfunc (s *Server) eventsRunning() bool {\n\tif s == nil {\n\t\treturn false\n\t}\n\ts.mu.RLock()\n\ter := s.isRunning() && s.eventsEnabled()\n\ts.mu.RUnlock()\n\treturn er\n}\n\n// EventsEnabled will report if the server has internal events enabled via\n// a defined system account.\nfunc (s *Server) EventsEnabled() bool {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.eventsEnabled()\n}\n\n// eventsEnabled will report if events are enabled.\n// Lock should be held.\nfunc (s *Server) eventsEnabled() bool {\n\treturn s.sys != nil && s.sys.client != nil && s.sys.account != nil\n}\n\n// TrackedRemoteServers returns how many remote servers we are tracking\n// from a system events perspective.\nfunc (s *Server) TrackedRemoteServers() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif !s.isRunning() || !s.eventsEnabled() {\n\t\treturn -1\n\t}\n\treturn len(s.sys.servers)\n}\n\n// Check for orphan servers who may have gone away without notification.\n// This should be wrapChk() to setup common locking.\nfunc (s *Server) checkRemoteServers() {\n\tnow := time.Now()\n\tfor sid, su := range s.sys.servers {\n\t\tif now.Sub(su.ltime) > s.sys.orphMax {\n\t\t\ts.Debugf(\"Detected orphan remote server: %q\", sid)\n\t\t\t// Simulate it going away.\n\t\t\ts.processRemoteServerShutdown(sid)\n\t\t}\n\t}\n\tif s.sys.sweeper != nil {\n\t\ts.sys.sweeper.Reset(s.sys.chkOrph)\n\t}\n}\n\n// Grab RSS and PCPU\n// Server lock will be held but released.\nfunc (s *Server) updateServerUsage(v *ServerStats) {\n\tvar vss int64\n\tpse.ProcUsage(&v.CPU, &v.Mem, &vss)\n\tv.Cores = runtime.NumCPU()\n\tv.MaxProcs = runtime.GOMAXPROCS(-1)\n\tif mm := debug.SetMemoryLimit(-1); mm < math.MaxInt64 {\n\t\tv.MemLimit = mm\n\t}\n}\n\n// Generate a route stat for our statz update.\nfunc routeStat(r *client) *RouteStat {\n\tif r == nil {\n\t\treturn nil\n\t}\n\tr.mu.Lock()\n\t// Note: *client.out[Msgs|Bytes] are not set using atomics,\n\t// unlike in[Msgs|Bytes].\n\trs := &RouteStat{\n\t\tID: r.cid,\n\t\tSent: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: r.outMsgs,\n\t\t\t\tBytes: r.outBytes,\n\t\t\t},\n\t\t},\n\t\tReceived: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: atomic.LoadInt64(&r.inMsgs),\n\t\t\t\tBytes: atomic.LoadInt64(&r.inBytes),\n\t\t\t},\n\t\t},\n\t\tPending: int(r.out.pb),\n\t}\n\tif r.route != nil {\n\t\trs.Name = r.route.remoteName\n\t}\n\tr.mu.Unlock()\n\treturn rs\n}\n\n// Actual send method for statz updates.\n// Lock should be held.\nfunc (s *Server) sendStatsz(subj string) {\n\tvar m ServerStatsMsg\n\ts.updateServerUsage(&m.Stats)\n\n\tif s.limitStatsz(subj) {\n\t\treturn\n\t}\n\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\n\t// Check that we have a system account, etc.\n\tif s.sys == nil || s.sys.account == nil {\n\t\treturn\n\t}\n\n\tshouldCheckInterest := func() bool {\n\t\topts := s.getOpts()\n\t\tif opts.Cluster.Port != 0 || opts.Gateway.Port != 0 || opts.LeafNode.Port != 0 {\n\t\t\treturn false\n\t\t}\n\t\t// If we are here we have no clustering or gateways and are not a leafnode hub.\n\t\t// Check for leafnode remotes that connect the system account.\n\t\tif len(opts.LeafNode.Remotes) > 0 {\n\t\t\tsysAcc := s.sys.account.GetName()\n\t\t\tfor _, r := range opts.LeafNode.Remotes {\n\t\t\t\tif r.LocalAccount == sysAcc {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}\n\n\t// if we are running standalone, check for interest.\n\tif shouldCheckInterest() {\n\t\t// Check if we even have interest in this subject.\n\t\tsacc := s.sys.account\n\t\trr := sacc.sl.Match(subj)\n\t\ttotalSubs := len(rr.psubs) + len(rr.qsubs)\n\t\tif totalSubs == 0 {\n\t\t\treturn\n\t\t} else if totalSubs == 1 && len(rr.psubs) == 1 {\n\t\t\t// For the broadcast subject we listen to that ourselves with no echo for remote updates.\n\t\t\t// If we are the only ones listening do not send either.\n\t\t\tif rr.psubs[0] == s.sys.remoteStatsSub {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tm.Stats.Start = s.start\n\tm.Stats.Connections = len(s.clients)\n\tm.Stats.TotalConnections = s.totalClients\n\tm.Stats.ActiveAccounts = int(atomic.LoadInt32(&s.activeAccounts))\n\tm.Stats.Received.Msgs = atomic.LoadInt64(&s.inMsgs)\n\tm.Stats.Received.Bytes = atomic.LoadInt64(&s.inBytes)\n\tm.Stats.Sent.Msgs = atomic.LoadInt64(&s.outMsgs)\n\tm.Stats.Sent.Bytes = atomic.LoadInt64(&s.outBytes)\n\tm.Stats.SlowConsumers = atomic.LoadInt64(&s.slowConsumers)\n\t// Evaluate the slow consumer stats, but set it only if one of the value is not 0.\n\tscs := &SlowConsumersStats{\n\t\tClients: s.NumSlowConsumersClients(),\n\t\tRoutes: s.NumSlowConsumersRoutes(),\n\t\tGateways: s.NumSlowConsumersGateways(),\n\t\tLeafs: s.NumSlowConsumersLeafs(),\n\t}\n\tif scs.Clients != 0 || scs.Routes != 0 || scs.Gateways != 0 || scs.Leafs != 0 {\n\t\tm.Stats.SlowConsumersStats = scs\n\t}\n\tm.Stats.StaleConnections = atomic.LoadInt64(&s.staleConnections)\n\tm.Stats.StalledClients = atomic.LoadInt64(&s.stalls)\n\tstcs := &StaleConnectionStats{\n\t\tClients: s.NumStaleConnectionsClients(),\n\t\tRoutes: s.NumStaleConnectionsRoutes(),\n\t\tGateways: s.NumStaleConnectionsGateways(),\n\t\tLeafs: s.NumStaleConnectionsLeafs(),\n\t}\n\tif stcs.Clients != 0 || stcs.Routes != 0 || stcs.Gateways != 0 || stcs.Leafs != 0 {\n\t\tm.Stats.StaleConnectionStats = stcs\n\t}\n\tm.Stats.NumSubs = s.numSubscriptions()\n\t// Routes\n\ts.forEachRoute(func(r *client) {\n\t\tm.Stats.Routes = append(m.Stats.Routes, routeStat(r))\n\t})\n\t// Gateways\n\tif s.gateway.enabled {\n\t\tgw := s.gateway\n\t\tgw.RLock()\n\t\tfor name, c := range gw.out {\n\t\t\tgs := &GatewayStat{Name: name}\n\t\t\tc.mu.Lock()\n\t\t\tgs.ID = c.cid\n\t\t\t// Note that *client.out[Msgs|Bytes] are not set using atomic,\n\t\t\t// unlike the in[Msgs|bytes].\n\t\t\tgs.Sent = DataStats{\n\t\t\t\tMsgBytes: MsgBytes{\n\t\t\t\t\tMsgs: c.outMsgs,\n\t\t\t\t\tBytes: c.outBytes,\n\t\t\t\t},\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\t// Gather matching inbound connections\n\t\t\tgs.Received = DataStats{}\n\t\t\tfor _, c := range gw.in {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tif c.gw.name == name {\n\t\t\t\t\tgs.Received.Msgs += atomic.LoadInt64(&c.inMsgs)\n\t\t\t\t\tgs.Received.Bytes += atomic.LoadInt64(&c.inBytes)\n\t\t\t\t\tgs.NumInbound++\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\t\t\t}\n\t\t\tm.Stats.Gateways = append(m.Stats.Gateways, gs)\n\t\t}\n\t\tgw.RUnlock()\n\t}\n\t// Active Servers\n\tm.Stats.ActiveServers = len(s.sys.servers) + 1\n\n\t// JetStream\n\tif js := s.js.Load(); js != nil {\n\t\tjStat := &JetStreamVarz{}\n\t\ts.mu.RUnlock()\n\t\tjs.mu.RLock()\n\t\tc := js.config\n\t\tc.StoreDir = _EMPTY_\n\t\tjStat.Config = &c\n\t\tjs.mu.RUnlock()\n\t\tjStat.Stats = js.usageStats()\n\t\t// Update our own usage since we do not echo so we will not hear ourselves.\n\t\tourNode := getHash(s.serverName())\n\t\tif v, ok := s.nodeToInfo.Load(ourNode); ok && v != nil {\n\t\t\tni := v.(nodeInfo)\n\t\t\tni.stats = jStat.Stats\n\t\t\tni.cfg = jStat.Config\n\t\t\ts.optsMu.RLock()\n\t\t\tni.tags = copyStrings(s.opts.Tags)\n\t\t\ts.optsMu.RUnlock()\n\t\t\ts.nodeToInfo.Store(ourNode, ni)\n\t\t}\n\t\t// Metagroup info.\n\t\tif mg := js.getMetaGroup(); mg != nil {\n\t\t\tif mg.Leader() {\n\t\t\t\tif ci := s.raftNodeToClusterInfo(mg); ci != nil {\n\t\t\t\t\tjStat.Meta = &MetaClusterInfo{\n\t\t\t\t\t\tName: ci.Name,\n\t\t\t\t\t\tLeader: ci.Leader,\n\t\t\t\t\t\tPeer: getHash(ci.Leader),\n\t\t\t\t\t\tReplicas: ci.Replicas,\n\t\t\t\t\t\tSize: mg.ClusterSize(),\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// non leader only include a shortened version without peers\n\t\t\t\tleader := s.serverNameForNode(mg.GroupLeader())\n\t\t\t\tjStat.Meta = &MetaClusterInfo{\n\t\t\t\t\tName: mg.Group(),\n\t\t\t\t\tLeader: leader,\n\t\t\t\t\tPeer: getHash(leader),\n\t\t\t\t\tSize: mg.ClusterSize(),\n\t\t\t\t}\n\t\t\t}\n\t\t\tif jStat.Meta != nil {\n\t\t\t\tif ipq := s.jsAPIRoutedReqs; ipq != nil {\n\t\t\t\t\tjStat.Meta.PendingRequests = ipq.len()\n\t\t\t\t}\n\t\t\t\tif ipq := s.jsAPIRoutedInfoReqs; ipq != nil {\n\t\t\t\t\tjStat.Meta.PendingInfos = ipq.len()\n\t\t\t\t}\n\t\t\t\tjStat.Meta.Pending = jStat.Meta.PendingRequests + jStat.Meta.PendingInfos\n\t\t\t\tjStat.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg)\n\t\t\t}\n\t\t}\n\t\tjStat.Limits = &s.getOpts().JetStreamLimits\n\t\tm.Stats.JetStream = jStat\n\t\ts.mu.RLock()\n\t}\n\t// Send message.\n\ts.sendInternalMsg(subj, _EMPTY_, &m.Server, &m)\n}\n\n// Limit updates to the heartbeat interval, max one second by default.\nfunc (s *Server) limitStatsz(subj string) bool {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\tif s.sys == nil {\n\t\treturn true\n\t}\n\n\t// Only limit the normal broadcast subject.\n\tif subj != fmt.Sprintf(serverStatsSubj, s.ID()) {\n\t\treturn false\n\t}\n\n\tinterval := statszRateLimit\n\tif s.sys.cstatsz < interval {\n\t\tinterval = s.sys.cstatsz\n\t}\n\tif time.Since(s.sys.lastStatsz) < interval {\n\t\t// Reschedule heartbeat for the next interval.\n\t\tif s.sys.stmr != nil {\n\t\t\ts.sys.stmr.Reset(time.Until(s.sys.lastStatsz.Add(interval)))\n\t\t}\n\t\treturn true\n\t}\n\ts.sys.lastStatsz = time.Now()\n\treturn false\n}\n\n// Send out our statz update.\n// This should be wrapChk() to setup common locking.\nfunc (s *Server) heartbeatStatsz() {\n\tif s.sys.stmr != nil {\n\t\t// Increase after startup to our max.\n\t\tif s.sys.cstatsz < s.sys.statsz {\n\t\t\ts.sys.cstatsz *= 2\n\t\t\tif s.sys.cstatsz > s.sys.statsz {\n\t\t\t\ts.sys.cstatsz = s.sys.statsz\n\t\t\t}\n\t\t}\n\t\ts.sys.stmr.Reset(s.sys.cstatsz)\n\t}\n\t// Do in separate Go routine.\n\tgo s.sendStatszUpdate()\n}\n\n// Reset statsz rate limit for the next broadcast.\n// This should be wrapChk() to setup common locking.\nfunc (s *Server) resetLastStatsz() {\n\ts.sys.lastStatsz = time.Time{}\n}\n\nfunc (s *Server) sendStatszUpdate() {\n\ts.sendStatsz(fmt.Sprintf(serverStatsSubj, s.ID()))\n}\n\n// This should be wrapChk() to setup common locking.\nfunc (s *Server) startStatszTimer() {\n\t// We will start by sending out more of these and trail off to the statsz being the max.\n\ts.sys.cstatsz = 250 * time.Millisecond\n\t// Send out the first one quickly, we will slowly back off.\n\ts.sys.stmr = time.AfterFunc(s.sys.cstatsz, s.wrapChk(s.heartbeatStatsz))\n}\n\n// Start a ticker that will fire periodically and check for orphaned servers.\n// This should be wrapChk() to setup common locking.\nfunc (s *Server) startRemoteServerSweepTimer() {\n\ts.sys.sweeper = time.AfterFunc(s.sys.chkOrph, s.wrapChk(s.checkRemoteServers))\n}\n\n// Length of our system hash used for server targeted messages.\nconst sysHashLen = 8\n\n// Computes a hash of 8 characters for the name.\nfunc getHash(name string) string {\n\treturn getHashSize(name, sysHashLen)\n}\n\n// Computes a hash for the given `name`. The result will be `size` characters long.\nfunc getHashSize(name string, size int) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(name))\n\tb := sha.Sum(nil)\n\tfor i := 0; i < size; i++ {\n\t\tb[i] = digits[int(b[i]%base)]\n\t}\n\treturn string(b[:size])\n}\n\n// Returns the node name for this server which is a hash of the server name.\nfunc (s *Server) Node() string {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif s.sys != nil {\n\t\treturn s.sys.shash\n\t}\n\treturn _EMPTY_\n}\n\n// This will setup our system wide tracking subs.\n// For now we will setup one wildcard subscription to\n// monitor all accounts for changes in number of connections.\n// We can make this on a per account tracking basis if needed.\n// Tradeoff is subscription and interest graph events vs connect and\n// disconnect events, etc.\nfunc (s *Server) initEventTracking() {\n\t// Capture sys in case we are shutdown while setting up.\n\ts.mu.RLock()\n\tsys := s.sys\n\ts.mu.RUnlock()\n\n\tif sys == nil || sys.client == nil || sys.account == nil {\n\t\treturn\n\t}\n\t// Create a system hash which we use for other servers to target us specifically.\n\tsys.shash = getHash(s.info.Name)\n\n\t// This will be for all inbox responses.\n\tsubject := fmt.Sprintf(inboxRespSubj, sys.shash, \"*\")\n\tif _, err := s.sysSubscribe(subject, s.inboxReply); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\tsys.inboxPre = subject\n\t// This is for remote updates for connection accounting.\n\tsubject = fmt.Sprintf(accConnsEventSubjOld, \"*\")\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking for %s: %v\", subject, err)\n\t\treturn\n\t}\n\t// This will be for responses for account info that we send out.\n\tsubject = fmt.Sprintf(connsRespSubj, s.info.ID)\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteConnsUpdate)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\t// Listen for broad requests to respond with number of subscriptions for a given subject.\n\tif _, err := s.sysSubscribe(accNumSubsReqSubj, s.noInlineCallback(s.nsubsRequest)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\t// Listen for statsz from others.\n\tsubject = fmt.Sprintf(serverStatsSubj, \"*\")\n\tif sub, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerUpdate)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t} else {\n\t\t// Keep track of this one.\n\t\tsys.remoteStatsSub = sub\n\t}\n\n\t// Listen for all server shutdowns.\n\tsubject = fmt.Sprintf(shutdownEventSubj, \"*\")\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\t// Listen for servers entering lame-duck mode.\n\t// NOTE: This currently is handled in the same way as a server shutdown, but has\n\t// a different subject in case we need to handle differently in future.\n\tsubject = fmt.Sprintf(lameDuckEventSubj, \"*\")\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteServerShutdown)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\t// Listen for account claims updates.\n\tsubscribeToUpdate := true\n\tif s.accResolver != nil {\n\t\tsubscribeToUpdate = !s.accResolver.IsTrackingUpdate()\n\t}\n\tif subscribeToUpdate {\n\t\tfor _, sub := range []string{accUpdateEventSubjOld, accUpdateEventSubjNew} {\n\t\t\tif _, err := s.sysSubscribe(fmt.Sprintf(sub, \"*\"), s.noInlineCallback(s.accountClaimUpdate)); err != nil {\n\t\t\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\t// Listen for ping messages that will be sent to all servers for statsz.\n\t// This subscription is kept for backwards compatibility. Got replaced by ...PING.STATZ from below\n\tif _, err := s.sysSubscribe(serverStatsPingReqSubj, s.noInlineCallbackStatsz(s.statszReq)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\tmonSrvc := map[string]sysMsgHandler{\n\t\t\"IDZ\": s.idzReq,\n\t\t\"STATSZ\": s.statszReq,\n\t\t\"VARZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &VarzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Varz(&optz.VarzOptions) })\n\t\t},\n\t\t\"SUBSZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &SubszEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Subsz(&optz.SubszOptions) })\n\t\t},\n\t\t\"CONNZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &ConnzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Connz(&optz.ConnzOptions) })\n\t\t},\n\t\t\"ROUTEZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &RoutezEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Routez(&optz.RoutezOptions) })\n\t\t},\n\t\t\"GATEWAYZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &GatewayzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Gatewayz(&optz.GatewayzOptions) })\n\t\t},\n\t\t\"LEAFZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &LeafzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Leafz(&optz.LeafzOptions) })\n\t\t},\n\t\t\"ACCOUNTZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &AccountzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Accountz(&optz.AccountzOptions) })\n\t\t},\n\t\t\"JSZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &JszEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Jsz(&optz.JSzOptions) })\n\t\t},\n\t\t\"HEALTHZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &HealthzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.healthz(&optz.HealthzOptions), nil })\n\t\t},\n\t\t\"PROFILEZ\": nil, // Special case, see below\n\t\t\"EXPVARZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &ExpvarzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.expvarz(optz), nil })\n\t\t},\n\t\t\"IPQUEUESZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &IpqueueszEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Ipqueuesz(&optz.IpqueueszOptions), nil })\n\t\t},\n\t\t\"RAFTZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &RaftzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) { return s.Raftz(&optz.RaftzOptions), nil })\n\t\t},\n\t}\n\tprofilez := func(_ *subscription, c *client, _ *Account, _, rply string, rmsg []byte) {\n\t\thdr, msg := c.msgParts(rmsg)\n\t\t// Need to copy since we are passing those to the go routine below.\n\t\thdr, msg = copyBytes(hdr), copyBytes(msg)\n\t\t// Execute in its own go routine because CPU profiling, for instance,\n\t\t// could take several seconds to complete.\n\t\tgo func() {\n\t\t\toptz := &ProfilezEventOptions{}\n\t\t\ts.zReq(c, rply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\treturn s.profilez(&optz.ProfilezOptions), nil\n\t\t\t})\n\t\t}()\n\t}\n\tfor name, req := range monSrvc {\n\t\tvar h msgHandler\n\t\tswitch name {\n\t\tcase \"PROFILEZ\":\n\t\t\th = profilez\n\t\tcase \"STATSZ\":\n\t\t\th = s.noInlineCallbackStatsz(req)\n\t\tdefault:\n\t\t\th = s.noInlineCallback(req)\n\t\t}\n\t\tsubject = fmt.Sprintf(serverDirectReqSubj, s.info.ID, name)\n\t\tif _, err := s.sysSubscribe(subject, h); err != nil {\n\t\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tsubject = fmt.Sprintf(serverPingReqSubj, name)\n\t\tif _, err := s.sysSubscribe(subject, h); err != nil {\n\t\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\textractAccount := func(subject string) (string, error) {\n\t\tif tk := strings.Split(subject, tsep); len(tk) != accReqTokens {\n\t\t\treturn _EMPTY_, fmt.Errorf(\"subject %q is malformed\", subject)\n\t\t} else {\n\t\t\treturn tk[accReqAccIndex], nil\n\t\t}\n\t}\n\tmonAccSrvc := map[string]sysMsgHandler{\n\t\t\"SUBSZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &SubszEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else {\n\t\t\t\t\toptz.SubszOptions.Subscriptions = true\n\t\t\t\t\toptz.SubszOptions.Account = acc\n\t\t\t\t\treturn s.Subsz(&optz.SubszOptions)\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t\"CONNZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &ConnzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else {\n\t\t\t\t\toptz.ConnzOptions.Account = acc\n\t\t\t\t\treturn s.Connz(&optz.ConnzOptions)\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t\"LEAFZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &LeafzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else {\n\t\t\t\t\toptz.LeafzOptions.Account = acc\n\t\t\t\t\treturn s.Leafz(&optz.LeafzOptions)\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t\"JSZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &JszEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else {\n\t\t\t\t\toptz.Account = acc\n\t\t\t\t\treturn s.JszAccount(&optz.JSzOptions)\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t\"INFO\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &AccInfoEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else {\n\t\t\t\t\treturn s.accountInfo(acc)\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t// STATZ is essentially a duplicate of CONNS with an envelope identical to the others.\n\t\t// For historical reasons CONNS is the odd one out.\n\t\t// STATZ is also less heavy weight than INFO\n\t\t\"STATZ\": func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &AccountStatzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif acc, err := extractAccount(subject); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else if acc == \"PING\" { // Filter PING subject. Happens for server as well. But wildcards are not used\n\t\t\t\t\treturn nil, errSkipZreq\n\t\t\t\t} else {\n\t\t\t\t\toptz.Accounts = []string{acc}\n\t\t\t\t\tif stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil {\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t} else if len(stz.Accounts) == 0 && !optz.IncludeUnused {\n\t\t\t\t\t\treturn nil, errSkipZreq\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn stz, nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t})\n\t\t},\n\t\t\"CONNS\": s.connsRequest,\n\t}\n\tfor name, req := range monAccSrvc {\n\t\tif _, err := s.sysSubscribe(fmt.Sprintf(accDirectReqSubj, \"*\", name), s.noInlineCallback(req)); err != nil {\n\t\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\n\t// User info. Do not propagate interest so that we know the local server to the connection\n\t// is the only one that will answer the requests.\n\tif _, err := s.sysSubscribeInternal(fmt.Sprintf(userDirectReqSubj, \"*\"), s.userInfoReq); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\n\t// For now only the STATZ subject has an account specific ping equivalent.\n\tif _, err := s.sysSubscribe(fmt.Sprintf(accPingReqSubj, \"STATZ\"),\n\t\ts.noInlineCallback(func(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t\t\toptz := &AccountStatzEventOptions{}\n\t\t\ts.zReq(c, reply, hdr, msg, &optz.EventFilterOptions, optz, func() (any, error) {\n\t\t\t\tif stz, err := s.AccountStatz(&optz.AccountStatzOptions); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t} else if len(stz.Accounts) == 0 && !optz.IncludeUnused {\n\t\t\t\t\treturn nil, errSkipZreq\n\t\t\t\t} else {\n\t\t\t\t\treturn stz, nil\n\t\t\t\t}\n\t\t\t})\n\t\t})); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\n\t// Listen for updates when leaf nodes connect for a given account. This will\n\t// force any gateway connections to move to `modeInterestOnly`\n\tsubject = fmt.Sprintf(leafNodeConnectEventSubj, \"*\")\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.leafNodeConnected)); err != nil {\n\t\ts.Errorf(\"Error setting up internal tracking: %v\", err)\n\t\treturn\n\t}\n\t// For tracking remote latency measurements.\n\tsubject = fmt.Sprintf(remoteLatencyEventSubj, sys.shash)\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.remoteLatencyUpdate)); err != nil {\n\t\ts.Errorf(\"Error setting up internal latency tracking: %v\", err)\n\t\treturn\n\t}\n\t// This is for simple debugging of number of subscribers that exist in the system.\n\tif _, err := s.sysSubscribeInternal(accSubsSubj, s.noInlineCallback(s.debugSubscribers)); err != nil {\n\t\ts.Errorf(\"Error setting up internal debug service for subscribers: %v\", err)\n\t\treturn\n\t}\n\n\t// Listen for requests to reload the server configuration.\n\tsubject = fmt.Sprintf(serverReloadReqSubj, s.info.ID)\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.reloadConfig)); err != nil {\n\t\ts.Errorf(\"Error setting up server reload handler: %v\", err)\n\t\treturn\n\t}\n\n\t// Client connection kick\n\tsubject = fmt.Sprintf(clientKickReqSubj, s.info.ID)\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.kickClient)); err != nil {\n\t\ts.Errorf(\"Error setting up client kick service: %v\", err)\n\t\treturn\n\t}\n\t// Client connection LDM\n\tsubject = fmt.Sprintf(clientLDMReqSubj, s.info.ID)\n\tif _, err := s.sysSubscribe(subject, s.noInlineCallback(s.ldmClient)); err != nil {\n\t\ts.Errorf(\"Error setting up client LDM service: %v\", err)\n\t\treturn\n\t}\n}\n\n// UserInfo returns basic information to a user about bound account and user permissions.\n// For account information they will need to ping that separately, and this allows security\n// controls on each subsystem if desired, e.g. account info, jetstream account info, etc.\ntype UserInfo struct {\n\tUserID string `json:\"user\"`\n\tAccount string `json:\"account\"`\n\tAccountName string `json:\"account_name,omitempty\"`\n\tUserName string `json:\"user_name,omitempty\"`\n\tPermissions *Permissions `json:\"permissions,omitempty\"`\n\tExpires time.Duration `json:\"expires,omitempty\"`\n}\n\n// Process a user info request.\nfunc (s *Server) userInfoReq(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tif !s.EventsEnabled() || reply == _EMPTY_ {\n\t\treturn\n\t}\n\n\tresponse := &ServerAPIResponse{Server: &ServerInfo{}}\n\n\tci, _, _, _, err := s.getRequestInfo(c, msg)\n\tif err != nil {\n\t\tresponse.Error = &ApiError{Code: http.StatusBadRequest}\n\t\ts.sendInternalResponse(reply, response)\n\t\treturn\n\t}\n\n\t// Look up the requester's account directly from ci.Account rather than\n\t// using the acc returned by getRequestInfo, which may resolve to the\n\t// service account (ci.Service) when the request arrives via a chained\n\t// service import.\n\tvar accountName string\n\tif ci.Account != _EMPTY_ {\n\t\tif reqAcc, _ := s.LookupAccount(ci.Account); reqAcc != nil {\n\t\t\taccountName = reqAcc.getNameTag()\n\t\t}\n\t}\n\n\tresponse.Data = &UserInfo{\n\t\tUserID: ci.User,\n\t\tAccount: ci.Account,\n\t\tAccountName: accountName,\n\t\tUserName: ci.NameTag,\n\t\tPermissions: c.publicPermissions(),\n\t\tExpires: c.claimExpiration(),\n\t}\n\ts.sendInternalResponse(reply, response)\n}\n\n// register existing accounts with any system exports.\nfunc (s *Server) registerSystemImportsForExisting() {\n\tvar accounts []*Account\n\n\ts.mu.RLock()\n\tif s.sys == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\tsacc := s.sys.account\n\ts.accounts.Range(func(k, v any) bool {\n\t\ta := v.(*Account)\n\t\tif a != sacc {\n\t\t\taccounts = append(accounts, a)\n\t\t}\n\t\treturn true\n\t})\n\ts.mu.RUnlock()\n\n\tfor _, a := range accounts {\n\t\ts.registerSystemImports(a)\n\t}\n}\n\n// add all exports a system account will need\nfunc (s *Server) addSystemAccountExports(sacc *Account) {\n\tif !s.EventsEnabled() {\n\t\treturn\n\t}\n\taccConnzSubj := fmt.Sprintf(accDirectReqSubj, \"*\", \"CONNZ\")\n\t// prioritize not automatically added exports\n\tif !sacc.hasServiceExportMatching(accConnzSubj) {\n\t\t// pick export type that clamps importing account id into subject\n\t\tif err := sacc.addServiceExportWithResponseAndAccountPos(accConnzSubj, Streamed, nil, 4); err != nil {\n\t\t\t//if err := sacc.AddServiceExportWithResponse(accConnzSubj, Streamed, nil); err != nil {\n\t\t\ts.Errorf(\"Error adding system service export for %q: %v\", accConnzSubj, err)\n\t\t}\n\t}\n\t// prioritize not automatically added exports\n\taccStatzSubj := fmt.Sprintf(accDirectReqSubj, \"*\", \"STATZ\")\n\tif !sacc.hasServiceExportMatching(accStatzSubj) {\n\t\t// pick export type that clamps importing account id into subject\n\t\tif err := sacc.addServiceExportWithResponseAndAccountPos(accStatzSubj, Streamed, nil, 4); err != nil {\n\t\t\ts.Errorf(\"Error adding system service export for %q: %v\", accStatzSubj, err)\n\t\t}\n\t}\n\t// FIXME(dlc) - Old experiment, Remove?\n\tif !sacc.hasServiceExportMatching(accSubsSubj) {\n\t\tif err := sacc.AddServiceExport(accSubsSubj, nil); err != nil {\n\t\t\ts.Errorf(\"Error adding system service export for %q: %v\", accSubsSubj, err)\n\t\t}\n\t}\n\n\t// User info export.\n\tuserInfoSubj := fmt.Sprintf(userDirectReqSubj, \"*\")\n\tif !sacc.hasServiceExportMatching(userInfoSubj) {\n\t\tif err := sacc.AddServiceExport(userInfoSubj, nil); err != nil {\n\t\t\ts.Errorf(\"Error adding system service export for %q: %v\", userInfoSubj, err)\n\t\t}\n\t\tmappedSubj := fmt.Sprintf(userDirectReqSubj, sacc.GetName())\n\t\tif err := sacc.AddServiceImport(sacc, userDirectInfoSubj, mappedSubj); err != nil {\n\t\t\ts.Errorf(\"Error setting up system service import %s: %v\", mappedSubj, err)\n\t\t}\n\t\t// Make sure to share details.\n\t\tsacc.setServiceImportSharing(sacc, mappedSubj, false, true)\n\t}\n\n\t// Register any accounts that existed prior.\n\ts.registerSystemImportsForExisting()\n\n\t// in case of a mixed mode setup, enable js exports anyway\n\tif s.JetStreamEnabled() || !s.standAloneMode() {\n\t\ts.checkJetStreamExports()\n\t}\n}\n\n// accountClaimUpdate will receive claim updates for accounts.\nfunc (s *Server) accountClaimUpdate(sub *subscription, c *client, _ *Account, subject, resp string, hdr, msg []byte) {\n\tif !s.EventsEnabled() {\n\t\treturn\n\t}\n\tvar pubKey string\n\ttoks := strings.Split(subject, tsep)\n\tif len(toks) == accUpdateTokensNew {\n\t\tpubKey = toks[accReqAccIndex]\n\t} else if len(toks) == accUpdateTokensOld {\n\t\tpubKey = toks[accUpdateAccIdxOld]\n\t} else {\n\t\ts.Debugf(\"Received account claims update on bad subject %q\", subject)\n\t\treturn\n\t}\n\tif len(msg) == 0 {\n\t\terr := errors.New(\"request body is empty\")\n\t\trespondToUpdate(s, resp, pubKey, \"jwt update error\", err)\n\t} else if claim, err := jwt.DecodeAccountClaims(string(msg)); err != nil {\n\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t} else if claim.Subject != pubKey {\n\t\terr := errors.New(\"subject does not match jwt content\")\n\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t} else if v, ok := s.accounts.Load(pubKey); !ok {\n\t\trespondToUpdate(s, resp, pubKey, \"jwt update skipped\", nil)\n\t} else if err := s.updateAccountWithClaimJWT(v.(*Account), string(msg)); err != nil {\n\t\trespondToUpdate(s, resp, pubKey, \"jwt update resulted in error\", err)\n\t} else {\n\t\trespondToUpdate(s, resp, pubKey, \"jwt updated\", nil)\n\t}\n}\n\n// processRemoteServerShutdown will update any affected accounts.\n// Will update the remote count for clients.\n// Lock assume held.\nfunc (s *Server) processRemoteServerShutdown(sid string) {\n\ts.accounts.Range(func(k, v any) bool {\n\t\tv.(*Account).removeRemoteServer(sid)\n\t\treturn true\n\t})\n\t// Update any state in nodeInfo.\n\ts.nodeToInfo.Range(func(k, v any) bool {\n\t\tni := v.(nodeInfo)\n\t\tif ni.id == sid {\n\t\t\tni.offline = true\n\t\t\ts.nodeToInfo.Store(k, ni)\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t})\n\tdelete(s.sys.servers, sid)\n}\n\nfunc (s *Server) sameDomain(domain string) bool {\n\treturn domain == _EMPTY_ || s.info.Domain == _EMPTY_ || domain == s.info.Domain\n}\n\n// remoteServerShutdown is called when we get an event from another server shutting down.\nfunc (s *Server) remoteServerShutdown(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif !s.eventsEnabled() {\n\t\treturn\n\t}\n\ttoks := strings.Split(subject, tsep)\n\tif len(toks) < shutdownEventTokens {\n\t\ts.Debugf(\"Received remote server shutdown on bad subject %q\", subject)\n\t\treturn\n\t}\n\n\tif len(msg) == 0 {\n\t\ts.Errorf(\"Remote server sent invalid (empty) shutdown message to %q\", subject)\n\t\treturn\n\t}\n\n\t// We have an optional serverInfo here, remove from nodeToX lookups.\n\tvar si ServerInfo\n\tif err := json.Unmarshal(msg, &si); err != nil {\n\t\ts.Debugf(\"Received bad server info for remote server shutdown\")\n\t\treturn\n\t}\n\n\t// JetStream node updates if applicable.\n\tnode := getHash(si.Name)\n\tif v, ok := s.nodeToInfo.Load(node); ok && v != nil {\n\t\tni := v.(nodeInfo)\n\t\tni.offline = true\n\t\ts.nodeToInfo.Store(node, ni)\n\t}\n\n\tsid := toks[serverSubjectIndex]\n\tif su := s.sys.servers[sid]; su != nil {\n\t\ts.processRemoteServerShutdown(sid)\n\t}\n}\n\n// remoteServerUpdate listens for statsz updates from other servers.\nfunc (s *Server) remoteServerUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tvar ssm ServerStatsMsg\n\tif len(msg) == 0 {\n\t\ts.Debugf(\"Received empty server info for remote server update\")\n\t\treturn\n\t} else if err := json.Unmarshal(msg, &ssm); err != nil {\n\t\ts.Debugf(\"Received bad server info for remote server update\")\n\t\treturn\n\t}\n\tsi := ssm.Server\n\n\t// Should do normal updates before bailing if wrong domain.\n\ts.mu.Lock()\n\tif s.isRunning() && s.eventsEnabled() && ssm.Server.ID != s.info.ID {\n\t\ts.updateRemoteServer(&si)\n\t}\n\ts.mu.Unlock()\n\n\t// JetStream node updates.\n\tif !s.sameDomain(si.Domain) {\n\t\treturn\n\t}\n\n\tvar cfg *JetStreamConfig\n\tvar stats *JetStreamStats\n\n\tif ssm.Stats.JetStream != nil {\n\t\tcfg = ssm.Stats.JetStream.Config\n\t\tstats = ssm.Stats.JetStream.Stats\n\t}\n\n\tnode := getHash(si.Name)\n\taccountNRG := si.AccountNRG()\n\toldInfo, _ := s.nodeToInfo.Swap(node, nodeInfo{\n\t\tname: si.Name,\n\t\tversion: si.Version,\n\t\tcluster: si.Cluster,\n\t\tdomain: si.Domain,\n\t\tid: si.ID,\n\t\ttags: si.Tags,\n\t\tcfg: cfg,\n\t\tstats: stats,\n\t\toffline: false,\n\t\tjs: si.JetStreamEnabled(),\n\t\tbinarySnapshots: si.BinaryStreamSnapshot(),\n\t\taccountNRG: accountNRG,\n\t})\n\tif oldInfo == nil || accountNRG != oldInfo.(nodeInfo).accountNRG {\n\t\t// One of the servers we received statsz from changed its mind about\n\t\t// whether or not it supports in-account NRG, so update the groups\n\t\t// with this information.\n\t\ts.updateNRGAccountStatus()\n\t}\n}\n\n// updateRemoteServer is called when we have an update from a remote server.\n// This allows us to track remote servers, respond to shutdown messages properly,\n// make sure that messages are ordered, and allow us to prune dead servers.\n// Lock should be held upon entry.\nfunc (s *Server) updateRemoteServer(si *ServerInfo) {\n\tsu := s.sys.servers[si.ID]\n\tif su == nil {\n\t\ts.sys.servers[si.ID] = &serverUpdate{si.Seq, time.Now()}\n\t\ts.processNewServer(si)\n\t} else {\n\t\t// Should always be going up.\n\t\tif si.Seq <= su.seq {\n\t\t\ts.Errorf(\"Received out of order remote server update from: %q\", si.ID)\n\t\t\treturn\n\t\t}\n\t\tsu.seq = si.Seq\n\t\tsu.ltime = time.Now()\n\t}\n}\n\n// processNewServer will hold any logic we want to use when we discover a new server.\n// Lock should be held upon entry.\nfunc (s *Server) processNewServer(si *ServerInfo) {\n\t// Right now we only check if we have leafnode servers and if so send another\n\t// connect update to make sure they switch this account to interest only mode.\n\ts.ensureGWsInterestOnlyForLeafNodes()\n\n\t// Add to our nodeToName\n\tif s.sameDomain(si.Domain) {\n\t\tnode := getHash(si.Name)\n\t\t// Only update if non-existent\n\t\tif _, ok := s.nodeToInfo.Load(node); !ok {\n\t\t\ts.nodeToInfo.Store(node, nodeInfo{\n\t\t\t\tname: si.Name,\n\t\t\t\tversion: si.Version,\n\t\t\t\tcluster: si.Cluster,\n\t\t\t\tdomain: si.Domain,\n\t\t\t\tid: si.ID,\n\t\t\t\ttags: si.Tags,\n\t\t\t\tcfg: nil,\n\t\t\t\tstats: nil,\n\t\t\t\toffline: false,\n\t\t\t\tjs: si.JetStreamEnabled(),\n\t\t\t\tbinarySnapshots: si.BinaryStreamSnapshot(),\n\t\t\t\taccountNRG: si.AccountNRG(),\n\t\t\t})\n\t\t}\n\t}\n\tgo s.updateNRGAccountStatus()\n\t// Announce ourselves..\n\t// Do this in a separate Go routine.\n\tgo s.sendStatszUpdate()\n}\n\n// Works out whether all nodes support moving the NRG traffic into\n// the account and moves it appropriately.\n// Server lock MUST NOT be held on entry.\nfunc (s *Server) updateNRGAccountStatus() {\n\ts.rnMu.RLock()\n\traftNodes := make([]RaftNode, 0, len(s.raftNodes))\n\tfor _, n := range s.raftNodes {\n\t\traftNodes = append(raftNodes, n)\n\t}\n\ts.rnMu.RUnlock()\n\tfor _, n := range raftNodes {\n\t\t// In the event that the node is happy that all nodes that\n\t\t// it cares about haven't changed, this will be a no-op.\n\t\tif err := n.RecreateInternalSubs(); err != nil {\n\t\t\tn.Stop()\n\t\t}\n\t}\n}\n\n// If GW is enabled on this server and there are any leaf node connections,\n// this function will send a LeafNode connect system event to the super cluster\n// to ensure that the GWs are in interest-only mode for this account.\n// Lock should be held upon entry.\n// TODO(dlc) - this will cause this account to be loaded on all servers. Need a better\n// way with GW2.\nfunc (s *Server) ensureGWsInterestOnlyForLeafNodes() {\n\tif !s.gateway.enabled || len(s.leafs) == 0 {\n\t\treturn\n\t}\n\tsent := make(map[*Account]bool, len(s.leafs))\n\tfor _, c := range s.leafs {\n\t\tif !sent[c.acc] {\n\t\t\ts.sendLeafNodeConnectMsg(c.acc.Name)\n\t\t\tsent[c.acc] = true\n\t\t}\n\t}\n}\n\n// shutdownEventing will clean up all eventing state.\nfunc (s *Server) shutdownEventing() {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\n\ts.mu.Lock()\n\tclearTimer(&s.sys.sweeper)\n\tclearTimer(&s.sys.stmr)\n\trc := s.sys.resetCh\n\ts.sys.resetCh = nil\n\twg := &s.sys.wg\n\ts.mu.Unlock()\n\n\t// We will queue up a shutdown event and wait for the\n\t// internal send loop to exit.\n\ts.sendShutdownEvent()\n\twg.Wait()\n\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\t// Whip through all accounts.\n\ts.accounts.Range(func(k, v any) bool {\n\t\tv.(*Account).clearEventing()\n\t\treturn true\n\t})\n\t// Turn everything off here.\n\ts.sys = nil\n\t// Make sure this is done after s.sys = nil, so that we don't\n\t// get sends to closed channels on badly-timed config reloads.\n\tclose(rc)\n}\n\n// Request for our local connection count.\nfunc (s *Server) connsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\ttk := strings.Split(subject, tsep)\n\tif len(tk) != accReqTokens {\n\t\ts.sys.client.Errorf(\"Bad subject account connections request message\")\n\t\treturn\n\t}\n\ta := tk[accReqAccIndex]\n\tm := accNumConnsReq{Account: a}\n\tif len(msg) > 0 {\n\t\tif err := json.Unmarshal(msg, &m); err != nil {\n\t\t\ts.sys.client.Errorf(\"Error unmarshalling account connections request message: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\tif m.Account != a {\n\t\ts.sys.client.Errorf(\"Error unmarshalled account does not match subject\")\n\t\treturn\n\t}\n\t// Here we really only want to lookup the account if its local. We do not want to fetch this\n\t// account if we have no interest in it.\n\tvar acc *Account\n\tif v, ok := s.accounts.Load(m.Account); ok {\n\t\tacc = v.(*Account)\n\t}\n\tif acc == nil {\n\t\treturn\n\t}\n\t// We know this is a local connection.\n\tif nlc := acc.NumLocalConnections(); nlc > 0 {\n\t\ts.mu.Lock()\n\t\ts.sendAccConnsUpdate(acc, reply)\n\t\ts.mu.Unlock()\n\t}\n}\n\n// leafNodeConnected is an event we will receive when a leaf node for a given account connects.\nfunc (s *Server) leafNodeConnected(sub *subscription, _ *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tm := accNumConnsReq{}\n\tif err := json.Unmarshal(msg, &m); err != nil {\n\t\ts.sys.client.Errorf(\"Error unmarshalling account connections request message: %v\", err)\n\t\treturn\n\t}\n\n\ts.mu.RLock()\n\tna := m.Account == _EMPTY_ || !s.eventsEnabled() || !s.gateway.enabled\n\ts.mu.RUnlock()\n\n\tif na {\n\t\treturn\n\t}\n\n\tif acc, _ := s.lookupAccount(m.Account); acc != nil {\n\t\ts.switchAccountToInterestMode(acc.Name)\n\t}\n}\n\n// Common filter options for system requests STATSZ VARZ SUBSZ CONNZ ROUTEZ GATEWAYZ LEAFZ\ntype EventFilterOptions struct {\n\tName string `json:\"server_name,omitempty\"` // filter by server name\n\tCluster string `json:\"cluster,omitempty\"` // filter by cluster name\n\tHost string `json:\"host,omitempty\"` // filter by host name\n\tExactMatch bool `json:\"exact_match,omitempty\"` // if the above filters should use exact matching or only \"contains\"\n\tTags []string `json:\"tags,omitempty\"` // filter by tags (must match all tags)\n\tDomain string `json:\"domain,omitempty\"` // filter by JS domain\n}\n\n// StatszEventOptions are options passed to Statsz\ntype StatszEventOptions struct {\n\t// No actual options yet\n\tEventFilterOptions\n}\n\n// Options for account Info\ntype AccInfoEventOptions struct {\n\t// No actual options yet\n\tEventFilterOptions\n}\n\n// In the context of system events, ConnzEventOptions are options passed to Connz\ntype ConnzEventOptions struct {\n\tConnzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, RoutezEventOptions are options passed to Routez\ntype RoutezEventOptions struct {\n\tRoutezOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, SubzEventOptions are options passed to Subz\ntype SubszEventOptions struct {\n\tSubszOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, VarzEventOptions are options passed to Varz\ntype VarzEventOptions struct {\n\tVarzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, GatewayzEventOptions are options passed to Gatewayz\ntype GatewayzEventOptions struct {\n\tGatewayzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, LeafzEventOptions are options passed to Leafz\ntype LeafzEventOptions struct {\n\tLeafzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, AccountzEventOptions are options passed to Accountz\ntype AccountzEventOptions struct {\n\tAccountzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, AccountzEventOptions are options passed to Accountz\ntype AccountStatzEventOptions struct {\n\tAccountStatzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, JszEventOptions are options passed to Jsz\ntype JszEventOptions struct {\n\tJSzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, HealthzEventOptions are options passed to Healthz\ntype HealthzEventOptions struct {\n\tHealthzOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, ProfilezEventOptions are options passed to Profilez\ntype ProfilezEventOptions struct {\n\tProfilezOptions\n\tEventFilterOptions\n}\n\n// In the context of system events, ExpvarzEventOptions are options passed to Expvarz\ntype ExpvarzEventOptions struct {\n\tEventFilterOptions\n}\n\n// In the context of system events, IpqueueszEventOptions are options passed to Ipqueuesz\ntype IpqueueszEventOptions struct {\n\tEventFilterOptions\n\tIpqueueszOptions\n}\n\n// In the context of system events, RaftzEventOptions are options passed to Raftz\ntype RaftzEventOptions struct {\n\tEventFilterOptions\n\tRaftzOptions\n}\n\n// returns true if the request does NOT apply to this server and can be ignored.\n// DO NOT hold the server lock when calling this.\nfunc (s *Server) filterRequest(fOpts *EventFilterOptions) bool {\n\tif fOpts == nil {\n\t\treturn false\n\t}\n\tif fOpts.ExactMatch {\n\t\tif (fOpts.Name != _EMPTY_ && fOpts.Name != s.info.Name) ||\n\t\t\t(fOpts.Host != _EMPTY_ && fOpts.Host != s.info.Host) ||\n\t\t\t(fOpts.Cluster != _EMPTY_ && fOpts.Cluster != s.ClusterName()) {\n\t\t\treturn true\n\t\t}\n\t} else if (fOpts.Name != _EMPTY_ && !strings.Contains(s.info.Name, fOpts.Name)) ||\n\t\t(fOpts.Host != _EMPTY_ && !strings.Contains(s.info.Host, fOpts.Host)) ||\n\t\t(fOpts.Cluster != _EMPTY_ && !strings.Contains(s.ClusterName(), fOpts.Cluster)) {\n\t\treturn true\n\t}\n\tif len(fOpts.Tags) > 0 {\n\t\topts := s.getOpts()\n\t\tfor _, t := range fOpts.Tags {\n\t\t\tif !opts.Tags.Contains(t) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\tif fOpts.Domain != _EMPTY_ && s.getOpts().JetStreamDomain != fOpts.Domain {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Encoding support (compression)\ntype compressionType int8\n\nconst (\n\tnoCompression = compressionType(iota)\n\tgzipCompression\n\tsnappyCompression\n\tunsupportedCompression\n)\n\n// ServerAPIResponse is the response type for the server API like varz, connz etc.\ntype ServerAPIResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData any `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n\n\t// Private to indicate compression if any.\n\tcompress compressionType\n}\n\n// Specialized response types for unmarshalling. These structures are not\n// used in the server code and only there for users of the Z endpoints to\n// unmarshal the data without having to create these structs in their code\n\n// ServerAPIConnzResponse is the response type connz\ntype ServerAPIConnzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Connz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIRoutezResponse is the response type for routez\ntype ServerAPIRoutezResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Routez `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIGatewayzResponse is the response type for gatewayz\ntype ServerAPIGatewayzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Gatewayz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIJszResponse is the response type for jsz\ntype ServerAPIJszResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *JSInfo `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIHealthzResponse is the response type for healthz\ntype ServerAPIHealthzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *HealthStatus `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIVarzResponse is the response type for varz\ntype ServerAPIVarzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Varz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPISubszResponse is the response type for subsz\ntype ServerAPISubszResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Subsz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPILeafzResponse is the response type for leafz\ntype ServerAPILeafzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Leafz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIAccountzResponse is the response type for accountz\ntype ServerAPIAccountzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *Accountz `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIExpvarzResponse is the response type for expvarz\ntype ServerAPIExpvarzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *ExpvarzStatus `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIpqueueszResponse is the response type for ipqueuesz\ntype ServerAPIpqueueszResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *IpqueueszStatus `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// ServerAPIRaftzResponse is the response type for raftz\ntype ServerAPIRaftzResponse struct {\n\tServer *ServerInfo `json:\"server\"`\n\tData *RaftzStatus `json:\"data,omitempty\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\n// statszReq is a request for us to respond with current statsz.\nfunc (s *Server) statszReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.EventsEnabled() {\n\t\treturn\n\t}\n\n\t// No reply is a signal that we should use our normal broadcast subject.\n\tif reply == _EMPTY_ {\n\t\treply = fmt.Sprintf(serverStatsSubj, s.info.ID)\n\t\ts.wrapChk(s.resetLastStatsz)\n\t}\n\n\topts := StatszEventOptions{}\n\tif len(msg) != 0 {\n\t\tif err := json.Unmarshal(msg, &opts); err != nil {\n\t\t\tresponse := &ServerAPIResponse{\n\t\t\t\tServer: &ServerInfo{},\n\t\t\t\tError: &ApiError{Code: http.StatusBadRequest, Description: err.Error()},\n\t\t\t}\n\t\t\ts.sendInternalMsgLocked(reply, _EMPTY_, response.Server, response)\n\t\t\treturn\n\t\t} else if ignore := s.filterRequest(&opts.EventFilterOptions); ignore {\n\t\t\treturn\n\t\t}\n\t}\n\ts.sendStatsz(reply)\n}\n\n// idzReq is for a request for basic static server info.\n// Try to not hold the write lock or dynamically create data.\nfunc (s *Server) idzReq(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tid := &ServerID{\n\t\tName: s.info.Name,\n\t\tHost: s.info.Host,\n\t\tID: s.info.ID,\n\t}\n\ts.sendInternalMsg(reply, _EMPTY_, nil, &id)\n}\n\nvar errSkipZreq = errors.New(\"filtered response\")\n\nconst (\n\tacceptEncodingHeader = \"Accept-Encoding\"\n\tcontentEncodingHeader = \"Content-Encoding\"\n)\n\n// This is not as formal as it could be. We see if anything has s2 or snappy first, then gzip.\nfunc getAcceptEncoding(hdr []byte) compressionType {\n\tae := strings.ToLower(string(getHeader(acceptEncodingHeader, hdr)))\n\tif ae == _EMPTY_ {\n\t\treturn noCompression\n\t}\n\tif strings.Contains(ae, \"snappy\") || strings.Contains(ae, \"s2\") {\n\t\treturn snappyCompression\n\t}\n\tif strings.Contains(ae, \"gzip\") {\n\t\treturn gzipCompression\n\t}\n\treturn unsupportedCompression\n}\n\nfunc (s *Server) zReq(_ *client, reply string, hdr, msg []byte, fOpts *EventFilterOptions, optz any, respf func() (any, error)) {\n\tif !s.EventsEnabled() || reply == _EMPTY_ {\n\t\treturn\n\t}\n\tresponse := &ServerAPIResponse{Server: &ServerInfo{}}\n\tvar err error\n\tstatus := 0\n\tif len(msg) != 0 {\n\t\tif err = json.Unmarshal(msg, optz); err != nil {\n\t\t\tstatus = http.StatusBadRequest // status is only included on error, so record how far execution got\n\t\t} else if s.filterRequest(fOpts) {\n\t\t\treturn\n\t\t}\n\t}\n\tif err == nil {\n\t\tresponse.Data, err = respf()\n\t\tif errors.Is(err, errSkipZreq) {\n\t\t\treturn\n\t\t} else if err != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\t\t}\n\t}\n\tif err != nil {\n\t\tresponse.Error = &ApiError{Code: status, Description: err.Error()}\n\t} else if len(hdr) > 0 {\n\t\tresponse.compress = getAcceptEncoding(hdr)\n\t}\n\ts.sendInternalResponse(reply, response)\n}\n\n// remoteConnsUpdate gets called when we receive a remote update from another server.\nfunc (s *Server) remoteConnsUpdate(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\tvar m AccountNumConns\n\tif len(msg) == 0 {\n\t\ts.sys.client.Errorf(\"No message body provided\")\n\t\treturn\n\t} else if err := json.Unmarshal(msg, &m); err != nil {\n\t\ts.sys.client.Errorf(\"Error unmarshalling account connection event message: %v\", err)\n\t\treturn\n\t}\n\n\t// See if we have the account registered, if not drop it.\n\t// Make sure this does not force us to load this account here.\n\tvar acc *Account\n\tif v, ok := s.accounts.Load(m.Account); ok {\n\t\tacc = v.(*Account)\n\t}\n\t// Silently ignore these if we do not have local interest in the account.\n\tif acc == nil {\n\t\treturn\n\t}\n\n\ts.mu.Lock()\n\n\t// check again here if we have been shutdown.\n\tif !s.isRunning() || !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\t// Double check that this is not us, should never happen, so error if it does.\n\tif m.Server.ID == s.info.ID {\n\t\ts.sys.client.Errorf(\"Processing our own account connection event message: ignored\")\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\t// If we are here we have interest in tracking this account. Update our accounting.\n\tclients := acc.updateRemoteServer(&m)\n\ts.updateRemoteServer(&m.Server)\n\ts.mu.Unlock()\n\t// Need to close clients outside of server lock\n\tfor _, c := range clients {\n\t\tc.maxAccountConnExceeded()\n\t}\n}\n\n// This will import any system level exports.\nfunc (s *Server) registerSystemImports(a *Account) {\n\tif a == nil || !s.EventsEnabled() {\n\t\treturn\n\t}\n\tsacc := s.SystemAccount()\n\tif sacc == nil || sacc == a {\n\t\treturn\n\t}\n\tdstAccName := sacc.Name\n\t// FIXME(dlc) - make a shared list between sys exports etc.\n\n\timportSrvc := func(subj, mappedSubj string) {\n\t\tif !a.serviceImportExists(dstAccName, subj) {\n\t\t\tif err := a.addServiceImportWithClaim(sacc, subj, mappedSubj, nil, true); err != nil {\n\t\t\t\ts.Errorf(\"Error setting up system service import %s -> %s for account: %v\",\n\t\t\t\t\tsubj, mappedSubj, err)\n\t\t\t}\n\t\t}\n\t}\n\t// Add in this to the account in 2 places.\n\t// \"$SYS.REQ.SERVER.PING.CONNZ\" and \"$SYS.REQ.ACCOUNT.PING.CONNZ\"\n\tmappedConnzSubj := fmt.Sprintf(accDirectReqSubj, a.Name, \"CONNZ\")\n\timportSrvc(fmt.Sprintf(accPingReqSubj, \"CONNZ\"), mappedConnzSubj)\n\timportSrvc(fmt.Sprintf(serverPingReqSubj, \"CONNZ\"), mappedConnzSubj)\n\timportSrvc(fmt.Sprintf(accPingReqSubj, \"STATZ\"), fmt.Sprintf(accDirectReqSubj, a.Name, \"STATZ\"))\n\n\t// This is for user's looking up their own info.\n\tmappedSubject := fmt.Sprintf(userDirectReqSubj, a.Name)\n\timportSrvc(userDirectInfoSubj, mappedSubject)\n\t// Make sure to share details.\n\ta.setServiceImportSharing(sacc, mappedSubject, false, true)\n}\n\n// Setup tracking for this account. This allows us to track global account activity.\n// Lock should be held on entry.\nfunc (s *Server) enableAccountTracking(a *Account) {\n\tif a == nil || !s.eventsEnabled() {\n\t\treturn\n\t}\n\n\t// TODO(ik): Generate payload although message may not be sent.\n\t// May need to ensure we do so only if there is a known interest.\n\t// This can get complicated with gateways.\n\n\tsubj := fmt.Sprintf(accDirectReqSubj, a.Name, \"CONNS\")\n\treply := fmt.Sprintf(connsRespSubj, s.info.ID)\n\tm := accNumConnsReq{Account: a.Name}\n\ts.sendInternalMsg(subj, reply, &m.Server, &m)\n}\n\n// Event on leaf node connect.\n// Lock should NOT be held on entry.\nfunc (s *Server) sendLeafNodeConnect(a *Account) {\n\ts.mu.Lock()\n\t// If we are not in operator mode, or do not have any gateways defined, this should also be a no-op.\n\tif a == nil || !s.eventsEnabled() || !s.gateway.enabled {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\ts.sendLeafNodeConnectMsg(a.Name)\n\ts.mu.Unlock()\n\n\ts.switchAccountToInterestMode(a.Name)\n}\n\n// Send the leafnode connect message.\n// Lock should be held.\nfunc (s *Server) sendLeafNodeConnectMsg(accName string) {\n\tsubj := fmt.Sprintf(leafNodeConnectEventSubj, accName)\n\tm := accNumConnsReq{Account: accName}\n\ts.sendInternalMsg(subj, _EMPTY_, &m.Server, &m)\n}\n\n// sendAccConnsUpdate is called to send out our information on the\n// account's local connections.\n// Lock should be held on entry.\nfunc (s *Server) sendAccConnsUpdate(a *Account, subj ...string) {\n\tif !s.eventsEnabled() || a == nil {\n\t\treturn\n\t}\n\tsendQ := s.sys.sendq\n\tif sendQ == nil {\n\t\treturn\n\t}\n\t// Build event with account name and number of local clients and leafnodes.\n\teid := s.nextEventID()\n\ta.mu.Lock()\n\tstat := a.statz()\n\tm := AccountNumConns{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: AccountNumConnsMsgType,\n\t\t\tID: eid,\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tAccountStat: *stat,\n\t}\n\t// Set timer to fire again unless we are at zero.\n\tif m.TotalConns == 0 {\n\t\tclearTimer(&a.ctmr)\n\t} else {\n\t\t// Check to see if we have an HB running and update.\n\t\tif a.ctmr == nil {\n\t\t\ta.ctmr = time.AfterFunc(eventsHBInterval, func() { s.accConnsUpdate(a) })\n\t\t} else {\n\t\t\ta.ctmr.Reset(eventsHBInterval)\n\t\t}\n\t}\n\tfor _, sub := range subj {\n\t\tmsg := newPubMsg(nil, sub, _EMPTY_, &m.Server, nil, &m, noCompression, false, false)\n\t\tsendQ.push(msg)\n\t}\n\ta.mu.Unlock()\n}\n\n// Lock should be held on entry.\nfunc (a *Account) statz() *AccountStat {\n\tlocalConns := a.numLocalConnections()\n\tleafConns := a.numLocalLeafNodes()\n\n\ta.stats.Lock()\n\treceived := DataStats{\n\t\tMsgBytes: MsgBytes{\n\t\t\tMsgs: a.stats.inMsgs,\n\t\t\tBytes: a.stats.inBytes,\n\t\t},\n\t\tGateways: &MsgBytes{\n\t\t\tMsgs: a.stats.gw.inMsgs,\n\t\t\tBytes: a.stats.gw.inBytes,\n\t\t},\n\t\tRoutes: &MsgBytes{\n\t\t\tMsgs: a.stats.rt.inMsgs,\n\t\t\tBytes: a.stats.rt.inBytes,\n\t\t},\n\t\tLeafs: &MsgBytes{\n\t\t\tMsgs: a.stats.ln.inMsgs,\n\t\t\tBytes: a.stats.ln.inBytes,\n\t\t},\n\t}\n\tsent := DataStats{\n\t\tMsgBytes: MsgBytes{\n\t\t\tMsgs: a.stats.outMsgs,\n\t\t\tBytes: a.stats.outBytes,\n\t\t},\n\t\tGateways: &MsgBytes{\n\t\t\tMsgs: a.stats.gw.outMsgs,\n\t\t\tBytes: a.stats.gw.outBytes,\n\t\t},\n\t\tRoutes: &MsgBytes{\n\t\t\tMsgs: a.stats.rt.outMsgs,\n\t\t\tBytes: a.stats.rt.outBytes,\n\t\t},\n\t\tLeafs: &MsgBytes{\n\t\t\tMsgs: a.stats.ln.outMsgs,\n\t\t\tBytes: a.stats.ln.outBytes,\n\t\t},\n\t}\n\tslowConsumers := a.stats.slowConsumers\n\ta.stats.Unlock()\n\n\treturn &AccountStat{\n\t\tAccount: a.Name,\n\t\tName: a.getNameTagLocked(),\n\t\tConns: localConns,\n\t\tLeafNodes: leafConns,\n\t\tTotalConns: localConns + leafConns,\n\t\tNumSubs: a.sl.Count(),\n\t\tReceived: received,\n\t\tSent: sent,\n\t\tSlowConsumers: slowConsumers,\n\t}\n}\n\n// accConnsUpdate is called whenever there is a change to the account's\n// number of active connections, or during a heartbeat.\n// We will not send for $G.\nfunc (s *Server) accConnsUpdate(a *Account) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif !s.eventsEnabled() || a == nil || a == s.gacc {\n\t\treturn\n\t}\n\ts.sendAccConnsUpdate(a, fmt.Sprintf(accConnsEventSubjOld, a.Name), fmt.Sprintf(accConnsEventSubjNew, a.Name))\n}\n\n// server lock should be held\nfunc (s *Server) nextEventID() string {\n\treturn s.eventIds.Next()\n}\n\n// accountConnectEvent will send an account client connect event if there is interest.\n// This is a billing event.\nfunc (s *Server) accountConnectEvent(c *client) {\n\ts.mu.Lock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\tif c.acc == nil {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\tm := ConnectEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: ConnectEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tClient: ClientInfo{\n\t\t\tStart: &c.start,\n\t\t\tHost: c.host,\n\t\t\tID: c.cid,\n\t\t\tAccount: accForClient(c),\n\t\t\tUser: c.getRawAuthUser(),\n\t\t\tName: c.opts.Name,\n\t\t\tLang: c.opts.Lang,\n\t\t\tVersion: c.opts.Version,\n\t\t\tJwt: c.opts.JWT,\n\t\t\tIssuerKey: issuerForClient(c),\n\t\t\tTags: c.tags,\n\t\t\tNameTag: c.acc.getNameTag(),\n\t\t\tKind: c.kindString(),\n\t\t\tClientType: c.clientTypeString(),\n\t\t\tMQTTClient: c.getMQTTClientID(),\n\t\t},\n\t}\n\tsubj := fmt.Sprintf(connectEventSubj, c.acc.Name)\n\tc.mu.Unlock()\n\n\ts.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m)\n}\n\n// accountDisconnectEvent will send an account client disconnect event if there is interest.\n// This is a billing event.\nfunc (s *Server) accountDisconnectEvent(c *client, now time.Time, reason string) {\n\ts.mu.Lock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\n\tif c.acc == nil {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tm := DisconnectEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: DisconnectEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: now,\n\t\t},\n\t\tClient: ClientInfo{\n\t\t\tStart: &c.start,\n\t\t\tStop: &now,\n\t\t\tHost: c.host,\n\t\t\tID: c.cid,\n\t\t\tAccount: accForClient(c),\n\t\t\tUser: c.getRawAuthUser(),\n\t\t\tName: c.opts.Name,\n\t\t\tLang: c.opts.Lang,\n\t\t\tVersion: c.opts.Version,\n\t\t\tRTT: c.getRTT(),\n\t\t\tJwt: c.opts.JWT,\n\t\t\tIssuerKey: issuerForClient(c),\n\t\t\tTags: c.tags,\n\t\t\tNameTag: c.acc.getNameTag(),\n\t\t\tKind: c.kindString(),\n\t\t\tClientType: c.clientTypeString(),\n\t\t\tMQTTClient: c.getMQTTClientID(),\n\t\t},\n\t\tSent: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: atomic.LoadInt64(&c.inMsgs),\n\t\t\t\tBytes: atomic.LoadInt64(&c.inBytes),\n\t\t\t},\n\t\t},\n\t\tReceived: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: c.outMsgs,\n\t\t\t\tBytes: c.outBytes,\n\t\t\t},\n\t\t},\n\t\tReason: reason,\n\t}\n\taccName := c.acc.Name\n\tc.mu.Unlock()\n\n\tsubj := fmt.Sprintf(disconnectEventSubj, accName)\n\ts.sendInternalMsgLocked(subj, _EMPTY_, &m.Server, &m)\n}\n\n// This is the system level event sent to the system account for operators.\nfunc (s *Server) sendAuthErrorEvent(c *client, reason string) {\n\ts.mu.Lock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\ts.mu.Unlock()\n\n\tnow := time.Now().UTC()\n\tc.mu.Lock()\n\tm := DisconnectEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: DisconnectEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: now,\n\t\t},\n\t\tClient: ClientInfo{\n\t\t\tStart: &c.start,\n\t\t\tStop: &now,\n\t\t\tHost: c.host,\n\t\t\tID: c.cid,\n\t\t\tAccount: accForClient(c),\n\t\t\tUser: c.getRawAuthUser(),\n\t\t\tName: c.opts.Name,\n\t\t\tLang: c.opts.Lang,\n\t\t\tVersion: c.opts.Version,\n\t\t\tRTT: c.getRTT(),\n\t\t\tJwt: c.opts.JWT,\n\t\t\tIssuerKey: issuerForClient(c),\n\t\t\tTags: c.tags,\n\t\t\tNameTag: c.acc.getNameTag(),\n\t\t\tKind: c.kindString(),\n\t\t\tClientType: c.clientTypeString(),\n\t\t\tMQTTClient: c.getMQTTClientID(),\n\t\t},\n\t\tSent: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: c.inMsgs,\n\t\t\t\tBytes: c.inBytes,\n\t\t\t},\n\t\t},\n\t\tReceived: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: c.outMsgs,\n\t\t\t\tBytes: c.outBytes,\n\t\t\t},\n\t\t},\n\t\tReason: reason,\n\t}\n\tc.mu.Unlock()\n\n\ts.mu.Lock()\n\tsubj := fmt.Sprintf(authErrorEventSubj, s.info.ID)\n\ts.sendInternalMsg(subj, _EMPTY_, &m.Server, &m)\n\ts.mu.Unlock()\n}\n\n// This is the account level event sent to the origin account for account owners.\nfunc (s *Server) sendAccountAuthErrorEvent(c *client, acc *Account, reason string) {\n\tif acc == nil {\n\t\treturn\n\t}\n\ts.mu.Lock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\ts.mu.Unlock()\n\n\tnow := time.Now().UTC()\n\tc.mu.Lock()\n\tm := DisconnectEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: DisconnectEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: now,\n\t\t},\n\t\tClient: ClientInfo{\n\t\t\tStart: &c.start,\n\t\t\tStop: &now,\n\t\t\tHost: c.host,\n\t\t\tID: c.cid,\n\t\t\tAccount: acc.Name,\n\t\t\tUser: c.getRawAuthUser(),\n\t\t\tName: c.opts.Name,\n\t\t\tLang: c.opts.Lang,\n\t\t\tVersion: c.opts.Version,\n\t\t\tRTT: c.getRTT(),\n\t\t\tJwt: c.opts.JWT,\n\t\t\tIssuerKey: issuerForClient(c),\n\t\t\tTags: c.tags,\n\t\t\tNameTag: c.acc.getNameTag(),\n\t\t\tKind: c.kindString(),\n\t\t\tClientType: c.clientTypeString(),\n\t\t\tMQTTClient: c.getMQTTClientID(),\n\t\t},\n\t\tSent: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: c.inMsgs,\n\t\t\t\tBytes: c.inBytes,\n\t\t\t},\n\t\t},\n\t\tReceived: DataStats{\n\t\t\tMsgBytes: MsgBytes{\n\t\t\t\tMsgs: c.outMsgs,\n\t\t\t\tBytes: c.outBytes,\n\t\t\t},\n\t\t},\n\t\tReason: reason,\n\t}\n\tc.mu.Unlock()\n\n\ts.sendInternalAccountSysMsg(acc, authErrorAccountEventSubj, &m.Server, &m, noCompression)\n}\n\n// Internal message callback.\n// If the msg is needed past the callback it is required to be copied.\n// rmsg contains header and the message. use client.msgParts(rmsg) to split them apart\ntype msgHandler func(sub *subscription, client *client, acc *Account, subject, reply string, rmsg []byte)\n\nconst (\n\trecvQMuxed = 1\n\trecvQStatsz = 2\n)\n\n// Create a wrapped callback handler for the subscription that will move it to an\n// internal recvQ for processing not inline with routes etc.\nfunc (s *Server) noInlineCallback(cb sysMsgHandler) msgHandler {\n\treturn s.noInlineCallbackRecvQSelect(cb, recvQMuxed)\n}\n\n// Create a wrapped callback handler for the subscription that will move it to an\n// internal recvQ for Statsz/Pings for processing not inline with routes etc.\nfunc (s *Server) noInlineCallbackStatsz(cb sysMsgHandler) msgHandler {\n\treturn s.noInlineCallbackRecvQSelect(cb, recvQStatsz)\n}\n\n// Create a wrapped callback handler for the subscription that will move it to an\n// internal IPQueue for processing not inline with routes etc.\nfunc (s *Server) noInlineCallbackRecvQSelect(cb sysMsgHandler, recvQSelect int) msgHandler {\n\ts.mu.RLock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.RUnlock()\n\t\treturn nil\n\t}\n\t// Capture here for direct reference to avoid any unnecessary blocking inline with routes, gateways etc.\n\tvar recvq *ipQueue[*inSysMsg]\n\tswitch recvQSelect {\n\tcase recvQStatsz:\n\t\trecvq = s.sys.recvqp\n\tdefault:\n\t\trecvq = s.sys.recvq\n\t}\n\ts.mu.RUnlock()\n\n\treturn func(sub *subscription, c *client, acc *Account, subj, rply string, rmsg []byte) {\n\t\t// Need to copy and split here.\n\t\thdr, msg := c.msgParts(rmsg)\n\t\trecvq.push(&inSysMsg{sub, c, acc, subj, rply, copyBytes(hdr), copyBytes(msg), cb})\n\t}\n}\n\n// Create an internal subscription. sysSubscribeQ for queue groups.\nfunc (s *Server) sysSubscribe(subject string, cb msgHandler) (*subscription, error) {\n\treturn s.systemSubscribe(subject, _EMPTY_, false, nil, cb)\n}\n\n// Create an internal subscription with queue\nfunc (s *Server) sysSubscribeQ(subject, queue string, cb msgHandler) (*subscription, error) {\n\treturn s.systemSubscribe(subject, queue, false, nil, cb)\n}\n\n// Create an internal subscription but do not forward interest.\nfunc (s *Server) sysSubscribeInternal(subject string, cb msgHandler) (*subscription, error) {\n\treturn s.systemSubscribe(subject, _EMPTY_, true, nil, cb)\n}\n\nfunc (s *Server) systemSubscribe(subject, queue string, internalOnly bool, c *client, cb msgHandler) (*subscription, error) {\n\ts.mu.Lock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.Unlock()\n\t\treturn nil, ErrNoSysAccount\n\t}\n\tif cb == nil {\n\t\ts.mu.Unlock()\n\t\treturn nil, fmt.Errorf(\"undefined message handler\")\n\t}\n\tif c == nil {\n\t\tc = s.sys.client\n\t}\n\ttrace := c.trace\n\ts.sys.sid++\n\tsid := strconv.Itoa(s.sys.sid)\n\ts.mu.Unlock()\n\n\t// Now create the subscription\n\tif trace {\n\t\tc.traceInOp(\"SUB\", []byte(subject+\" \"+queue+\" \"+sid))\n\t}\n\n\tvar q []byte\n\tif queue != _EMPTY_ {\n\t\tq = []byte(queue)\n\t}\n\n\t// Now create the subscription\n\treturn c.processSub([]byte(subject), q, []byte(sid), cb, internalOnly)\n}\n\nfunc (s *Server) sysUnsubscribe(sub *subscription) {\n\tif sub == nil {\n\t\treturn\n\t}\n\ts.mu.RLock()\n\tif !s.eventsEnabled() {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\tc := sub.client\n\ts.mu.RUnlock()\n\n\tif c != nil {\n\t\tc.processUnsub(sub.sid)\n\t}\n}\n\n// This will generate the tracking subject for remote latency from the response subject.\nfunc remoteLatencySubjectForResponse(subject []byte) string {\n\tif !isTrackedReply(subject) {\n\t\treturn \"\"\n\t}\n\ttoks := bytes.Split(subject, []byte(tsep))\n\t// FIXME(dlc) - Sprintf may become a performance concern at some point.\n\treturn fmt.Sprintf(remoteLatencyEventSubj, toks[len(toks)-2])\n}\n\n// remoteLatencyUpdate is used to track remote latency measurements for tracking on exported services.\nfunc (s *Server) remoteLatencyUpdate(sub *subscription, _ *client, _ *Account, subject, _ string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\tvar rl remoteLatency\n\tif err := json.Unmarshal(msg, &rl); err != nil {\n\t\ts.Errorf(\"Error unmarshalling remote latency measurement: %v\", err)\n\t\treturn\n\t}\n\t// Now we need to look up the responseServiceImport associated with this measurement.\n\tacc, err := s.LookupAccount(rl.Account)\n\tif err != nil {\n\t\ts.Warnf(\"Could not lookup account %q for latency measurement\", rl.Account)\n\t\treturn\n\t}\n\t// Now get the request id / reply. We need to see if we have a GW prefix and if so strip that off.\n\treply := rl.ReqId\n\tif gwPrefix, old := isGWRoutedSubjectAndIsOldPrefix([]byte(reply)); gwPrefix {\n\t\treply = string(getSubjectFromGWRoutedReply([]byte(reply), old))\n\t}\n\tacc.mu.RLock()\n\tsi := acc.exports.responses[reply]\n\tif si == nil {\n\t\tacc.mu.RUnlock()\n\t\treturn\n\t}\n\tlsub := si.latency.subject\n\tacc.mu.RUnlock()\n\n\tsi.acc.mu.Lock()\n\tm1 := si.m1\n\tm2 := rl.M2\n\n\t// So we have not processed the response tracking measurement yet.\n\tif m1 == nil {\n\t\t// Store our value there for them to pick up.\n\t\tsi.m1 = &m2\n\t}\n\tsi.acc.mu.Unlock()\n\n\tif m1 == nil {\n\t\treturn\n\t}\n\n\t// Calculate the correct latencies given M1 and M2.\n\tm1.merge(&m2)\n\n\t// Clear the requesting client since we send the result here.\n\tacc.mu.Lock()\n\tsi.rc = nil\n\tacc.mu.Unlock()\n\n\t// Send the metrics\n\ts.sendInternalAccountMsg(acc, lsub, m1)\n}\n\n// This is used for all inbox replies so that we do not send supercluster wide interest\n// updates for every request. Same trick used in modern NATS clients.\nfunc (s *Server) inboxReply(sub *subscription, c *client, acc *Account, subject, reply string, msg []byte) {\n\ts.mu.RLock()\n\tif !s.eventsEnabled() || s.sys.replies == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\tcb, ok := s.sys.replies[subject]\n\ts.mu.RUnlock()\n\n\tif ok && cb != nil {\n\t\tcb(sub, c, acc, subject, reply, msg)\n\t}\n}\n\n// Copied from go client.\n// We could use serviceReply here instead to save some code.\n// I prefer these semantics for the moment, when tracing you know what this is.\nconst (\n\tInboxPrefix = \"$SYS._INBOX.\"\n\tinboxPrefixLen = len(InboxPrefix)\n\trespInboxPrefixLen = inboxPrefixLen + sysHashLen + 1\n\treplySuffixLen = 8 // Gives us 62^8\n)\n\n// Creates an internal inbox used for replies that will be processed by the global wc handler.\nfunc (s *Server) newRespInbox() string {\n\tvar b [respInboxPrefixLen + replySuffixLen]byte\n\tpres := b[:respInboxPrefixLen]\n\tcopy(pres, s.sys.inboxPre)\n\trn := rand.Int63()\n\tfor i, l := respInboxPrefixLen, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\treturn string(b[:])\n}\n\n// accNumSubsReq is sent when we need to gather remote info on subs.\ntype accNumSubsReq struct {\n\tAccount string `json:\"acc\"`\n\tSubject string `json:\"subject\"`\n\tQueue []byte `json:\"queue,omitempty\"`\n}\n\n// helper function to total information from results to count subs.\nfunc totalSubs(rr *SublistResult, qg []byte) (nsubs int32) {\n\tif rr == nil {\n\t\treturn\n\t}\n\tcheckSub := func(sub *subscription) {\n\t\t// TODO(dlc) - This could be smarter.\n\t\tif qg != nil && !bytes.Equal(qg, sub.queue) {\n\t\t\treturn\n\t\t}\n\t\tif sub.client.kind == CLIENT || sub.client.isHubLeafNode() {\n\t\t\tnsubs++\n\t\t}\n\t}\n\tif qg == nil {\n\t\tfor _, sub := range rr.psubs {\n\t\t\tcheckSub(sub)\n\t\t}\n\t}\n\tfor _, qsub := range rr.qsubs {\n\t\tfor _, sub := range qsub {\n\t\t\tcheckSub(sub)\n\t\t}\n\t}\n\treturn\n}\n\n// Allows users of large systems to debug active subscribers for a given subject.\n// Payload should be the subject of interest.\nfunc (s *Server) debugSubscribers(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\t// Even though this is an internal only subscription, meaning interest was not forwarded, we could\n\t// get one here from a GW in optimistic mode. Ignore for now.\n\t// FIXME(dlc) - Should we send no interest here back to the GW?\n\tif c.kind != CLIENT {\n\t\treturn\n\t}\n\n\tvar ci ClientInfo\n\tif len(hdr) > 0 {\n\t\tif err := json.Unmarshal(getHeader(ClientInfoHdr, hdr), &ci); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\tvar acc *Account\n\tif ci.Service != _EMPTY_ {\n\t\tacc, _ = s.LookupAccount(ci.Service)\n\t} else if ci.Account != _EMPTY_ {\n\t\tacc, _ = s.LookupAccount(ci.Account)\n\t} else {\n\t\t// Direct $SYS access.\n\t\tacc = c.acc\n\t\tif acc == nil {\n\t\t\tacc = s.SystemAccount()\n\t\t}\n\t}\n\tif acc == nil {\n\t\treturn\n\t}\n\n\t// We could have a single subject or we could have a subject and a wildcard separated by whitespace.\n\targs := strings.Split(strings.TrimSpace(string(msg)), \" \")\n\tif len(args) == 0 {\n\t\ts.sendInternalAccountMsg(acc, reply, 0)\n\t\treturn\n\t}\n\n\ttsubj := args[0]\n\tvar qgroup []byte\n\tif len(args) > 1 {\n\t\tqgroup = []byte(args[1])\n\t}\n\n\tvar nsubs int32\n\n\tif subjectIsLiteral(tsubj) {\n\t\t// We will look up subscribers locally first then determine if we need to solicit other servers.\n\t\trr := acc.sl.Match(tsubj)\n\t\tnsubs = totalSubs(rr, qgroup)\n\t} else {\n\t\t// We have a wildcard, so this is a bit slower path.\n\t\tvar _subs [32]*subscription\n\t\tsubs := _subs[:0]\n\t\tacc.sl.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tif subjectIsSubsetMatch(string(sub.subject), tsubj) {\n\t\t\t\tif qgroup != nil && !bytes.Equal(qgroup, sub.queue) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif sub.client.kind == CLIENT || sub.client.isHubLeafNode() {\n\t\t\t\t\tnsubs++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// We should have an idea of how many responses to expect from remote servers.\n\tvar expected = acc.expectedRemoteResponses()\n\n\t// If we are only local, go ahead and return.\n\tif expected == 0 {\n\t\ts.sendInternalAccountMsg(nil, reply, nsubs)\n\t\treturn\n\t}\n\n\t// We need to solicit from others.\n\t// To track status.\n\tresponses := int32(0)\n\tdone := make(chan (bool))\n\n\ts.mu.Lock()\n\t// Create direct reply inbox that we multiplex under the WC replies.\n\treplySubj := s.newRespInbox()\n\t// Store our handler.\n\ts.sys.replies[replySubj] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {\n\t\tif n, err := strconv.ParseInt(string(msg), 10, 32); err == nil {\n\t\t\tatomic.AddInt32(&nsubs, int32(n))\n\t\t}\n\t\tif atomic.AddInt32(&responses, 1) >= expected {\n\t\t\tselect {\n\t\t\tcase done <- true:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}\n\t// Send the request to the other servers.\n\trequest := &accNumSubsReq{\n\t\tAccount: acc.Name,\n\t\tSubject: tsubj,\n\t\tQueue: qgroup,\n\t}\n\ts.sendInternalMsg(accNumSubsReqSubj, replySubj, nil, request)\n\ts.mu.Unlock()\n\n\t// FIXME(dlc) - We should rate limit here instead of blind Go routine.\n\tgo func() {\n\t\tselect {\n\t\tcase <-done:\n\t\tcase <-time.After(500 * time.Millisecond):\n\t\t}\n\t\t// Cleanup the WC entry.\n\t\tvar sendResponse bool\n\t\ts.mu.Lock()\n\t\tif s.sys != nil && s.sys.replies != nil {\n\t\t\tdelete(s.sys.replies, replySubj)\n\t\t\tsendResponse = true\n\t\t}\n\t\ts.mu.Unlock()\n\t\tif sendResponse {\n\t\t\t// Send the response.\n\t\t\ts.sendInternalAccountMsg(nil, reply, atomic.LoadInt32(&nsubs))\n\t\t}\n\t}()\n}\n\n// Request for our local subscription count. This will come from a remote origin server\n// that received the initial request.\nfunc (s *Server) nsubsRequest(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\tm := accNumSubsReq{}\n\tif len(msg) == 0 {\n\t\ts.sys.client.Errorf(\"request requires a body\")\n\t\treturn\n\t} else if err := json.Unmarshal(msg, &m); err != nil {\n\t\ts.sys.client.Errorf(\"Error unmarshalling account nsubs request message: %v\", err)\n\t\treturn\n\t}\n\t// Grab account.\n\tacc, _ := s.lookupAccount(m.Account)\n\tif acc == nil || acc.numLocalAndLeafConnections() == 0 {\n\t\treturn\n\t}\n\t// We will look up subscribers locally first then determine if we need to solicit other servers.\n\tvar nsubs int32\n\tif subjectIsLiteral(m.Subject) {\n\t\trr := acc.sl.Match(m.Subject)\n\t\tnsubs = totalSubs(rr, m.Queue)\n\t} else {\n\t\t// We have a wildcard, so this is a bit slower path.\n\t\tvar _subs [32]*subscription\n\t\tsubs := _subs[:0]\n\t\tacc.sl.All(&subs)\n\t\tfor _, sub := range subs {\n\t\t\tif (sub.client.kind == CLIENT || sub.client.isHubLeafNode()) && subjectIsSubsetMatch(string(sub.subject), m.Subject) {\n\t\t\t\tif m.Queue != nil && !bytes.Equal(m.Queue, sub.queue) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tnsubs++\n\t\t\t}\n\t\t}\n\t}\n\ts.sendInternalMsgLocked(reply, _EMPTY_, nil, nsubs)\n}\n\nfunc (s *Server) reloadConfig(sub *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\n\toptz := &EventFilterOptions{}\n\ts.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) {\n\t\t// Reload the server config, as requested.\n\t\treturn nil, s.Reload()\n\t})\n}\n\ntype KickClientReq struct {\n\tCID uint64 `json:\"cid\"`\n}\n\ntype LDMClientReq struct {\n\tCID uint64 `json:\"cid\"`\n}\n\nfunc (s *Server) kickClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\n\tvar req KickClientReq\n\tif err := json.Unmarshal(msg, &req); err != nil {\n\t\ts.sys.client.Errorf(\"Error unmarshalling kick client request: %v\", err)\n\t\treturn\n\t}\n\n\toptz := &EventFilterOptions{}\n\ts.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) {\n\t\treturn nil, s.DisconnectClientByID(req.CID)\n\t})\n\n}\n\nfunc (s *Server) ldmClient(_ *subscription, c *client, _ *Account, subject, reply string, hdr, msg []byte) {\n\tif !s.eventsRunning() {\n\t\treturn\n\t}\n\n\tvar req LDMClientReq\n\tif err := json.Unmarshal(msg, &req); err != nil {\n\t\ts.sys.client.Errorf(\"Error unmarshalling kick client request: %v\", err)\n\t\treturn\n\t}\n\n\toptz := &EventFilterOptions{}\n\ts.zReq(c, reply, hdr, msg, optz, optz, func() (any, error) {\n\t\treturn nil, s.LDMClientByID(req.CID)\n\t})\n}\n\n// Helper to grab account name for a client.\nfunc accForClient(c *client) string {\n\tif c.acc != nil {\n\t\treturn c.acc.Name\n\t}\n\treturn \"N/A\"\n}\n\n// Helper to grab issuer for a client.\nfunc issuerForClient(c *client) (issuerKey string) {\n\tif c == nil || c.user == nil {\n\t\treturn\n\t}\n\tissuerKey = c.user.SigningKey\n\tif issuerKey == _EMPTY_ && c.user.Account != nil {\n\t\tissuerKey = c.user.Account.Name\n\t}\n\treturn\n}\n\n// Helper to clear timers.\nfunc clearTimer(tp **time.Timer) {\n\tif t := *tp; t != nil {\n\t\tt.Stop()\n\t\t*tp = nil\n\t}\n}\n\n// Helper function to wrap functions with common test\n// to lock server and return if events not enabled.\nfunc (s *Server) wrapChk(f func()) func() {\n\treturn func() {\n\t\ts.mu.Lock()\n\t\tif !s.eventsEnabled() {\n\t\t\ts.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t\tf()\n\t\ts.mu.Unlock()\n\t}\n}\n\n// sendOCSPPeerRejectEvent sends a system level event to system account when a peer connection is\n// rejected due to OCSP invalid status of its trust chain(s).\nfunc (s *Server) sendOCSPPeerRejectEvent(kind string, peer *x509.Certificate, reason string) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif !s.eventsEnabled() {\n\t\treturn\n\t}\n\tif peer == nil {\n\t\ts.Errorf(certidp.ErrPeerEmptyNoEvent)\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\tnow := time.Now().UTC()\n\tm := OCSPPeerRejectEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: OCSPPeerRejectEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: now,\n\t\t},\n\t\tKind: kind,\n\t\tPeer: certidp.CertInfo{\n\t\t\tSubject: certidp.GetSubjectDNForm(peer),\n\t\t\tIssuer: certidp.GetIssuerDNForm(peer),\n\t\t\tFingerprint: certidp.GenerateFingerprint(peer),\n\t\t\tRaw: peer.Raw,\n\t\t},\n\t\tReason: reason,\n\t}\n\tsubj := fmt.Sprintf(ocspPeerRejectEventSubj, s.info.ID)\n\ts.sendInternalMsg(subj, _EMPTY_, &m.Server, &m)\n}\n\n// sendOCSPPeerChainlinkInvalidEvent sends a system level event to system account when a link in a peer's trust chain\n// is OCSP invalid.\nfunc (s *Server) sendOCSPPeerChainlinkInvalidEvent(peer *x509.Certificate, link *x509.Certificate, reason string) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif !s.eventsEnabled() {\n\t\treturn\n\t}\n\tif peer == nil || link == nil {\n\t\ts.Errorf(certidp.ErrPeerEmptyNoEvent)\n\t\treturn\n\t}\n\teid := s.nextEventID()\n\tnow := time.Now().UTC()\n\tm := OCSPPeerChainlinkInvalidEventMsg{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: OCSPPeerChainlinkInvalidEventMsgType,\n\t\t\tID: eid,\n\t\t\tTime: now,\n\t\t},\n\t\tLink: certidp.CertInfo{\n\t\t\tSubject: certidp.GetSubjectDNForm(link),\n\t\t\tIssuer: certidp.GetIssuerDNForm(link),\n\t\t\tFingerprint: certidp.GenerateFingerprint(link),\n\t\t\tRaw: link.Raw,\n\t\t},\n\t\tPeer: certidp.CertInfo{\n\t\t\tSubject: certidp.GetSubjectDNForm(peer),\n\t\t\tIssuer: certidp.GetIssuerDNForm(peer),\n\t\t\tFingerprint: certidp.GenerateFingerprint(peer),\n\t\t\tRaw: peer.Raw,\n\t\t},\n\t\tReason: reason,\n\t}\n\tsubj := fmt.Sprintf(ocspPeerChainlinkInvalidEventSubj, s.info.ID)\n\ts.sendInternalMsg(subj, _EMPTY_, &m.Server, &m)\n}\n" }, { "path": "server/events_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc createAccount(s *Server) (*Account, nkeys.KeyPair) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tjwt, _ := nac.Encode(okp)\n\taddAccountToMemResolver(s, pub, jwt)\n\tacc, err := s.LookupAccount(pub)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn acc, akp\n}\n\nfunc createUserCredsEx(t *testing.T, nuc *jwt.UserClaims, akp nkeys.KeyPair) nats.Option {\n\tt.Helper()\n\tkp, _ := nkeys.CreateUser()\n\tnuc.Subject, _ = kp.PublicKey()\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tuserCB := func() (string, error) {\n\t\treturn ujwt, nil\n\t}\n\tsigCB := func(nonce []byte) ([]byte, error) {\n\t\tsig, _ := kp.Sign(nonce)\n\t\treturn sig, nil\n\t}\n\treturn nats.UserJWT(userCB, sigCB)\n}\n\nfunc createUserCreds(t *testing.T, _ *Server, akp nkeys.KeyPair) nats.Option {\n\treturn createUserCredsEx(t, jwt.NewUserClaims(\"test\"), akp)\n}\n\nfunc runTrustedServer(t *testing.T) (*Server, *Options) {\n\tt.Helper()\n\topts := DefaultOptions()\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\topts.TrustedKeys = []string{pub}\n\topts.AccountResolver = &MemAccResolver{}\n\ts := RunServer(opts)\n\treturn s, opts\n}\n\nfunc runTrustedCluster(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) {\n\tt.Helper()\n\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\n\tmr := &MemAccResolver{}\n\n\t// Now create a system account.\n\t// NOTE: This can NOT be shared directly between servers.\n\t// Set via server options.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tjwt, _ := nac.Encode(okp)\n\n\tmr.Store(apub, jwt)\n\n\toptsA := DefaultOptions()\n\toptsA.Cluster.Name = \"TEST_CLUSTER_22\"\n\toptsA.Cluster.Host = \"127.0.0.1\"\n\toptsA.TrustedKeys = []string{pub}\n\toptsA.AccountResolver = mr\n\toptsA.SystemAccount = apub\n\toptsA.ServerName = \"A_SRV\"\n\t// Add in dummy gateway\n\toptsA.Gateway.Name = \"TEST_CLUSTER_22\"\n\toptsA.Gateway.Host = \"127.0.0.1\"\n\toptsA.Gateway.Port = -1\n\toptsA.gatewaysSolicitDelay = 30 * time.Second\n\n\tsa := RunServer(optsA)\n\n\toptsB := nextServerOpts(optsA)\n\toptsB.ServerName = \"B_SRV\"\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\tsb := RunServer(optsB)\n\n\tcheckClusterFormed(t, sa, sb)\n\n\treturn sa, optsA, sb, optsB, akp\n}\n\nfunc runTrustedGateways(t *testing.T) (*Server, *Options, *Server, *Options, nkeys.KeyPair) {\n\tt.Helper()\n\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\n\tmr := &MemAccResolver{}\n\n\t// Now create a system account.\n\t// NOTE: This can NOT be shared directly between servers.\n\t// Set via server options.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tjwt, _ := nac.Encode(okp)\n\n\tmr.Store(apub, jwt)\n\n\toptsA := testDefaultOptionsForGateway(\"A\")\n\toptsA.Cluster.Name = \"A\"\n\toptsA.Cluster.Host = \"127.0.0.1\"\n\toptsA.TrustedKeys = []string{pub}\n\toptsA.AccountResolver = mr\n\toptsA.SystemAccount = apub\n\n\tsa := RunServer(optsA)\n\n\toptsB := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\toptsB.Cluster.Name = \"B\"\n\toptsB.TrustedKeys = []string{pub}\n\toptsB.AccountResolver = mr\n\toptsB.SystemAccount = apub\n\n\tsb := RunServer(optsB)\n\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\n\treturn sa, optsA, sb, optsB, akp\n}\n\nfunc TestSystemAccount(t *testing.T) {\n\ts, _ := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, _ := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\tif s.sys == nil || s.sys.account == nil {\n\t\tt.Fatalf(\"Expected sys.account to be non-nil\")\n\t}\n\tif s.sys.client == nil {\n\t\tt.Fatalf(\"Expected sys.client to be non-nil\")\n\t}\n\n\ts.sys.client.mu.Lock()\n\tdefer s.sys.client.mu.Unlock()\n\tif s.sys.client.echo {\n\t\tt.Fatalf(\"Internal clients should always have echo false\")\n\t}\n}\n\nfunc TestSystemAccountNewConnection(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\t// We may not be able to hear ourselves (if the event is processed\n\t// before we create the sub), so we need to create a second client to\n\t// trigger the connect/disconnect events.\n\tacc2, akp2 := createAccount(s)\n\n\t// Be explicit to only receive the event for acc2.\n\tsub, _ := ncs.SubscribeSync(fmt.Sprintf(\"$SYS.ACCOUNT.%s.>\", acc2.Name))\n\tdefer sub.Unsubscribe()\n\tncs.Flush()\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name(\"TEST EVENTS\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tmsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tconnsMsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif strings.HasPrefix(msg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.SERVER.CONNS\", acc2.Name)) {\n\t\tmsg, connsMsg = connsMsg, msg\n\t}\n\tif !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.SERVER.CONNS\", acc2.Name)) {\n\t\tt.Fatalf(\"Expected subject to start with %q, got %q\", \"$SYS.ACCOUNT..CONNECT\", msg.Subject)\n\t}\n\tconns := AccountNumConns{}\n\tif err := json.Unmarshal(connsMsg.Data, &conns); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling conns event message: %v\", err)\n\t} else if conns.Account != acc2.Name {\n\t\tt.Fatalf(\"Wrong account in conns message: %v\", conns)\n\t} else if conns.Conns != 1 || conns.TotalConns != 1 || conns.LeafNodes != 0 {\n\t\tt.Fatalf(\"Wrong counts in conns message: %v\", conns)\n\t}\n\tif !strings.HasPrefix(msg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.CONNECT\", acc2.Name)) {\n\t\tt.Fatalf(\"Expected subject to start with %q, got %q\", \"$SYS.ACCOUNT..CONNECT\", msg.Subject)\n\t}\n\ttokens := strings.Split(msg.Subject, \".\")\n\tif len(tokens) < 4 {\n\t\tt.Fatalf(\"Expected 4 tokens, got %d\", len(tokens))\n\t}\n\taccount := tokens[2]\n\tif account != acc2.Name {\n\t\tt.Fatalf(\"Expected %q for account, got %q\", acc2.Name, account)\n\t}\n\n\tcem := ConnectEventMsg{}\n\tif err := json.Unmarshal(msg.Data, &cem); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling connect event message: %v\", err)\n\t}\n\tif cem.Type != ConnectEventMsgType {\n\t\tt.Fatalf(\"Incorrect schema in connect event: %s\", cem.Type)\n\t}\n\tif cem.Time.IsZero() {\n\t\tt.Fatalf(\"Event time is not set\")\n\t}\n\tif len(cem.ID) != 22 {\n\t\tt.Fatalf(\"Event ID is incorrectly set to len %d\", len(cem.ID))\n\t}\n\tif cem.Server.ID != s.ID() {\n\t\tt.Fatalf(\"Expected server to be %q, got %q\", s.ID(), cem.Server.ID)\n\t}\n\tif cem.Server.Seq == 0 {\n\t\tt.Fatalf(\"Expected sequence to be non-zero\")\n\t}\n\tif cem.Client.Name != \"TEST EVENTS\" {\n\t\tt.Fatalf(\"Expected client name to be %q, got %q\", \"TEST EVENTS\", cem.Client.Name)\n\t}\n\tif cem.Client.Lang != \"go\" {\n\t\tt.Fatalf(\"Expected client lang to be \\\"go\\\", got %q\", cem.Client.Lang)\n\t}\n\n\t// Now close the other client. Should fire a disconnect event.\n\t// First send and receive some messages.\n\tsub2, _ := nc.SubscribeSync(\"foo\")\n\tdefer sub2.Unsubscribe()\n\tsub3, _ := nc.SubscribeSync(\"*\")\n\tdefer sub3.Unsubscribe()\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\t}\n\tnc.Flush()\n\tnc.Close()\n\n\tmsg, err = sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tconnsMsg, err = sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif strings.HasPrefix(msg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.SERVER.CONNS\", acc2.Name)) {\n\t\tmsg, connsMsg = connsMsg, msg\n\t}\n\tif !strings.HasPrefix(connsMsg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.SERVER.CONNS\", acc2.Name)) {\n\t\tt.Fatalf(\"Expected subject to start with %q, got %q\", \"$SYS.ACCOUNT..CONNECT\", msg.Subject)\n\t} else if !strings.Contains(string(connsMsg.Data), `\"total_conns\":0`) {\n\t\tt.Fatalf(\"Expected event to reflect created connection, got: %s\", string(connsMsg.Data))\n\t}\n\tconns = AccountNumConns{}\n\tif err := json.Unmarshal(connsMsg.Data, &conns); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling conns event message: %v\", err)\n\t} else if conns.Account != acc2.Name {\n\t\tt.Fatalf(\"Wrong account in conns message: %v\", conns)\n\t} else if conns.Conns != 0 || conns.TotalConns != 0 || conns.LeafNodes != 0 {\n\t\tt.Fatalf(\"Wrong counts in conns message: %v\", conns)\n\t}\n\tif !strings.HasPrefix(msg.Subject, fmt.Sprintf(\"$SYS.ACCOUNT.%s.DISCONNECT\", acc2.Name)) {\n\t\tt.Fatalf(\"Expected subject to start with %q, got %q\", \"$SYS.ACCOUNT..DISCONNECT\", msg.Subject)\n\t}\n\ttokens = strings.Split(msg.Subject, \".\")\n\tif len(tokens) < 4 {\n\t\tt.Fatalf(\"Expected 4 tokens, got %d\", len(tokens))\n\t}\n\taccount = tokens[2]\n\tif account != acc2.Name {\n\t\tt.Fatalf(\"Expected %q for account, got %q\", acc2.Name, account)\n\t}\n\n\tdem := DisconnectEventMsg{}\n\tif err := json.Unmarshal(msg.Data, &dem); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling disconnect event message: %v\", err)\n\t}\n\tif dem.Type != DisconnectEventMsgType {\n\t\tt.Fatalf(\"Incorrect schema in connect event: %s\", cem.Type)\n\t}\n\tif dem.Time.IsZero() {\n\t\tt.Fatalf(\"Event time is not set\")\n\t}\n\tif len(dem.ID) != 22 {\n\t\tt.Fatalf(\"Event ID is incorrectly set to len %d\", len(cem.ID))\n\t}\n\tif dem.Server.ID != s.ID() {\n\t\tt.Fatalf(\"Expected server to be %q, got %q\", s.ID(), dem.Server.ID)\n\t}\n\tif dem.Server.Seq == 0 {\n\t\tt.Fatalf(\"Expected sequence to be non-zero\")\n\t}\n\tif dem.Server.Seq <= cem.Server.Seq {\n\t\tt.Fatalf(\"Expected sequence to be increasing\")\n\t}\n\n\tif cem.Client.Name != \"TEST EVENTS\" {\n\t\tt.Fatalf(\"Expected client name to be %q, got %q\", \"TEST EVENTS\", dem.Client.Name)\n\t}\n\tif dem.Client.Lang != \"go\" {\n\t\tt.Fatalf(\"Expected client lang to be \\\"go\\\", got %q\", dem.Client.Lang)\n\t}\n\n\tif dem.Sent.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs sent, got %d\", dem.Sent.Msgs)\n\t}\n\tif dem.Sent.Bytes != 110 {\n\t\tt.Fatalf(\"Expected 110 bytes sent, got %d\", dem.Sent.Bytes)\n\t}\n\tif dem.Received.Msgs != 20 {\n\t\tt.Fatalf(\"Expected 20 msgs received, got %d\", dem.Sent.Msgs)\n\t}\n\tif dem.Received.Bytes != 220 {\n\t\tt.Fatalf(\"Expected 220 bytes sent, got %d\", dem.Sent.Bytes)\n\t}\n}\n\nfunc runTrustedLeafServer(t *testing.T) (*Server, *Options) {\n\tt.Helper()\n\topts := DefaultOptions()\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\topts.TrustedKeys = []string{pub}\n\topts.AccountResolver = &MemAccResolver{}\n\topts.LeafNode.Port = -1\n\ts := RunServer(opts)\n\treturn s, opts\n}\n\nfunc genCredsFile(t *testing.T, jwt string, seed []byte) string {\n\tcreds := `\n\t\t-----BEGIN NATS USER JWT-----\n\t\t%s\n\t\t------END NATS USER JWT------\n\n\t\t************************* IMPORTANT *************************\n\t\tNKEY Seed printed below can be used to sign and prove identity.\n\t\tNKEYs are sensitive and should be treated as secrets.\n\n\t\t-----BEGIN USER NKEY SEED-----\n\t\t%s\n\t\t------END USER NKEY SEED------\n\n\t\t*************************************************************\n\t\t`\n\treturn createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), \"\\t\\t\", \"\", -1)))\n}\n\nfunc runSolicitWithCredentials(t *testing.T, opts *Options, creds string) (*Server, *Options, string) {\n\tcontent := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content, opts.LeafNode.Port, creds)\n\tconf := createConfFile(t, []byte(config))\n\ts, opts := RunServerWithConfig(conf)\n\treturn s, opts, conf\n}\n\n// Helper function to check that a leaf node has connected to our server.\nfunc checkLeafNodeConnected(t testing.TB, s *Server) {\n\tt.Helper()\n\tcheckLeafNodeConnectedCount(t, s, 1)\n}\n\n// Helper function to check that a leaf node has connected to n server.\nfunc checkLeafNodeConnectedCount(t testing.TB, s *Server, lnCons int) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 15*time.Millisecond, func() error {\n\t\tif nln := s.NumLeafNodes(); nln != lnCons {\n\t\t\treturn fmt.Errorf(\"Expected %d connected leafnode(s) for server %v, got %d\",\n\t\t\t\tlnCons, s, nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestSystemAccountingWithLeafNodes(t *testing.T) {\n\ts, opts := runTrustedLeafServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\tacc2, akp2 := createAccount(s)\n\n\t// Be explicit to only receive the event for acc2 account.\n\tsub, _ := ncs.SubscribeSync(fmt.Sprintf(\"$SYS.ACCOUNT.%s.DISCONNECT\", acc2.Name))\n\tdefer sub.Unsubscribe()\n\tncs.Flush()\n\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\t// Create a server that solicits a leafnode connection.\n\tsl, slopts, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Compute the expected number of subs on \"sl\" based on number\n\t// of existing subs before creating the sub on \"s\".\n\texpected := int(sl.NumSubscriptions() + 1)\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name(\"TEST LEAFNODE EVENTS\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tfooSub := natsSubSync(t, nc, \"foo\")\n\tnatsFlush(t, nc)\n\n\tcheckExpectedSubs(t, expected, sl)\n\n\tsurl := fmt.Sprintf(\"nats://%s:%d\", slopts.Host, slopts.Port)\n\tnc2, err := nats.Connect(surl, nats.Name(\"TEST LEAFNODE EVENTS\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Compute the expected number of subs on \"s\" based on number\n\t// of existing subs before creating the sub on \"sl\".\n\texpected = int(s.NumSubscriptions() + 1)\n\n\tm := []byte(\"HELLO WORLD\")\n\n\t// Now generate some traffic\n\tstarSub := natsSubSync(t, nc2, \"*\")\n\tfor i := 0; i < 10; i++ {\n\t\tnc2.Publish(\"foo\", m)\n\t\tnc2.Publish(\"bar\", m)\n\t}\n\tnatsFlush(t, nc2)\n\n\tcheckExpectedSubs(t, expected, s)\n\n\t// Now send some from the cluster side too.\n\tfor i := 0; i < 10; i++ {\n\t\tnc.Publish(\"foo\", m)\n\t\tnc.Publish(\"bar\", m)\n\t}\n\tnc.Flush()\n\n\t// Make sure all messages are received\n\tfor i := 0; i < 20; i++ {\n\t\tif _, err := fooSub.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not get message: %v\", err)\n\t\t}\n\t}\n\tfor i := 0; i < 40; i++ {\n\t\tif _, err := starSub.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not get message: %v\", err)\n\t\t}\n\t}\n\n\t// Now shutdown the leafnode server since this is where the event tracking should\n\t// happen. Right now we do not track local clients to the leafnode server that\n\t// solicited to the cluster, but we should track usage once the leafnode connection stops.\n\tsl.Shutdown()\n\n\t// Make sure we get disconnect event and that tracking is correct.\n\tmsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\n\tdem := DisconnectEventMsg{}\n\tif err := json.Unmarshal(msg.Data, &dem); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling disconnect event message: %v\", err)\n\t}\n\tif dem.Sent.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs sent, got %d\", dem.Sent.Msgs)\n\t}\n\tif dem.Sent.Bytes != 110 {\n\t\tt.Fatalf(\"Expected 110 bytes sent, got %d\", dem.Sent.Bytes)\n\t}\n\tif dem.Received.Msgs != 20 {\n\t\tt.Fatalf(\"Expected 20 msgs received, got %d\", dem.Received.Msgs)\n\t}\n\tif dem.Received.Bytes != 220 {\n\t\tt.Fatalf(\"Expected 220 bytes sent, got %d\", dem.Received.Bytes)\n\t}\n}\n\nfunc TestSystemAccountDisconnectBadLogin(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\t// Listen for auth error events though.\n\tasub, _ := ncs.SubscribeSync(\"$SYS.SERVER.*.CLIENT.AUTH.ERR\")\n\tdefer asub.Unsubscribe()\n\n\tncs.Flush()\n\n\tnats.Connect(url, nats.Name(\"TEST BAD LOGIN\"))\n\n\tm, err := asub.NextMsg(100 * time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Should have heard an auth error event\")\n\t}\n\tdem := DisconnectEventMsg{}\n\tif err := json.Unmarshal(m.Data, &dem); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling disconnect event message: %v\", err)\n\t}\n\tif dem.Reason != \"Authentication Failure\" {\n\t\tt.Fatalf(\"Expected auth error, got %q\", dem.Reason)\n\t}\n}\n\nfunc TestSysSubscribeRace(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tdone := make(chan struct{})\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tnc.Publish(\"foo\", []byte(\"hello\"))\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(10 * time.Millisecond)\n\n\treceived := make(chan struct{})\n\t// Create message callback handler.\n\tcb := func(sub *subscription, producer *client, _ *Account, subject, reply string, msg []byte) {\n\t\tselect {\n\t\tcase received <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n\t// Now create an internal subscription\n\tsub, err := s.sysSubscribe(\"foo\", cb)\n\tif sub == nil || err != nil {\n\t\tt.Fatalf(\"Expected to subscribe, got %v\", err)\n\t}\n\tselect {\n\tcase <-received:\n\t\tclose(done)\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive the message\")\n\t}\n\twg.Wait()\n}\n\nfunc TestSystemAccountInternalSubscriptions(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tsub, err := s.sysSubscribe(\"foo\", nil)\n\tif sub != nil || err != ErrNoSysAccount {\n\t\tt.Fatalf(\"Expected to get proper error, got %v\", err)\n\t}\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsub, err = s.sysSubscribe(\"foo\", nil)\n\tif sub != nil || err == nil {\n\t\tt.Fatalf(\"Expected to get error for no handler, got %v\", err)\n\t}\n\n\treceived := make(chan *nats.Msg)\n\t// Create message callback handler.\n\tcb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) {\n\t\tcopy := append([]byte(nil), msg...)\n\t\treceived <- &nats.Msg{Subject: subject, Reply: reply, Data: copy}\n\t}\n\n\t// Now create an internal subscription\n\tsub, err = s.sysSubscribe(\"foo\", cb)\n\tif sub == nil || err != nil {\n\t\tt.Fatalf(\"Expected to subscribe, got %v\", err)\n\t}\n\t// Now send out a message from our normal client.\n\tnc.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\n\tvar msg *nats.Msg\n\n\tselect {\n\tcase msg = <-received:\n\t\tif msg.Subject != \"foo\" {\n\t\t\tt.Fatalf(\"Expected \\\"foo\\\" as subject, got %q\", msg.Subject)\n\t\t}\n\t\tif msg.Reply != \"\" {\n\t\t\tt.Fatalf(\"Expected no reply, got %q\", msg.Reply)\n\t\t}\n\t\tif !bytes.Equal(msg.Data, []byte(\"HELLO WORLD\")) {\n\t\t\tt.Fatalf(\"Got the wrong msg payload: %q\", msg.Data)\n\t\t}\n\t\tbreak\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive the message\")\n\t}\n\ts.sysUnsubscribe(sub)\n\n\t// Now send out a message from our normal client.\n\t// We should not see this one.\n\tnc.Publish(\"foo\", []byte(\"You There?\"))\n\n\tselect {\n\tcase <-received:\n\t\tt.Fatalf(\"Received a message when we should not have\")\n\tcase <-time.After(100 * time.Millisecond):\n\t\tbreak\n\t}\n\n\t// Now make sure we do not hear ourselves. We optimize this for internally\n\t// generated messages.\n\ts.mu.Lock()\n\ts.sendInternalMsg(\"foo\", \"\", nil, msg.Data)\n\ts.mu.Unlock()\n\n\tselect {\n\tcase <-received:\n\t\tt.Fatalf(\"Received a message when we should not have\")\n\tcase <-time.After(100 * time.Millisecond):\n\t\tbreak\n\t}\n}\n\nfunc TestSystemAccountConnectionUpdatesStopAfterNoLocal(t *testing.T) {\n\tsa, _, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// Normal Account\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 4 // Limit to 4 connections.\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\n\t// Listen for updates to the new account connection activity.\n\treceived := make(chan *nats.Msg, 10)\n\tcb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) {\n\t\tcopy := append([]byte(nil), msg...)\n\t\treceived <- &nats.Msg{Subject: subject, Reply: reply, Data: copy}\n\t}\n\tsubj := fmt.Sprintf(accConnsEventSubjNew, pub)\n\tsub, err := sa.sysSubscribe(subj, cb)\n\tif sub == nil || err != nil {\n\t\tt.Fatalf(\"Expected to subscribe, got %v\", err)\n\t}\n\tdefer sa.sysUnsubscribe(sub)\n\n\t// Create a few users on the new account.\n\tclients := []*nats.Conn{}\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tfor i := 0; i < 4; i++ {\n\t\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tclients = append(clients, nc)\n\t}\n\n\t// Wait for all 4 notifications.\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif len(received) == 4 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not enough messages, %d vs 4\", len(received))\n\t})\n\n\t// Now lookup the account doing the events on sb.\n\tacc, _ := sb.LookupAccount(pub)\n\t// Make sure we have the timer running.\n\tacc.mu.RLock()\n\tctmr := acc.ctmr\n\tacc.mu.RUnlock()\n\tif ctmr == nil {\n\t\tt.Fatalf(\"Expected event timer for acc conns to be running\")\n\t}\n\n\t// Now close all of the connections.\n\tfor _, nc := range clients {\n\t\tnc.Close()\n\t}\n\n\t// Wait for the 4 new notifications, 8 total (4 for connect, 4 for disconnect)\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif len(received) == 8 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not enough messages, %d vs 4\", len(received))\n\t})\n\t// Drain the messages.\n\tfor i := 0; i < 7; i++ {\n\t\t<-received\n\t}\n\t// Check last one.\n\tmsg := <-received\n\tm := AccountNumConns{}\n\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling account connections request message: %v\", err)\n\t}\n\tif m.Conns != 0 {\n\t\tt.Fatalf(\"Expected Conns to be 0, got %d\", m.Conns)\n\t}\n\n\t// Should not receive any more messages..\n\tselect {\n\tcase <-received:\n\t\tt.Fatalf(\"Did not expect a message here\")\n\tcase <-time.After(50 * time.Millisecond):\n\t\tbreak\n\t}\n\n\t// Make sure we have the timer is NOT running.\n\tacc.mu.RLock()\n\tctmr = acc.ctmr\n\tacc.mu.RUnlock()\n\tif ctmr != nil {\n\t\tt.Fatalf(\"Expected event timer for acc conns to NOT be running after reaching zero local clients\")\n\t}\n}\n\nfunc TestSystemAccountConnectionLimits(t *testing.T) {\n\tsa, optsA, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// We want to test that we are limited to a certain number of active connections\n\t// across multiple servers.\n\n\t// Let's create a user account.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 4 // Limit to 4 connections.\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\n\t// Create a user on each server. Break on first failure.\n\tfor {\n\t\tnca1, err := nats.Connect(urlA, createUserCreds(t, sa, akp))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tdefer nca1.Close()\n\t\tncb1, err := nats.Connect(urlB, createUserCreds(t, sb, akp))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tdefer ncb1.Close()\n\t}\n\n\tcheckFor(t, 5*time.Second, 50*time.Millisecond, func() error {\n\t\ttotal := sa.NumClients() + sb.NumClients()\n\t\tif total > int(nac.Limits.Conn) {\n\t\t\treturn fmt.Errorf(\"Expected only %d connections, was allowed to connect %d\", nac.Limits.Conn, total)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestBadAccountUpdate(t *testing.T) {\n\tsa, _ := runTrustedServer(t)\n\tdefer sa.Shutdown()\n\takp1, _ := nkeys.CreateAccount()\n\tpub, _ := akp1.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tajwt1, err := nac.Encode(oKp)\n\trequire_NoError(t, err)\n\taddAccountToMemResolver(sa, pub, ajwt1)\n\takp2, _ := nkeys.CreateAccount()\n\tpub2, _ := akp2.PublicKey()\n\tnac.Subject = pub2 // maliciously use a different subject but pretend to remain pub\n\tajwt2, err := nac.Encode(oKp)\n\trequire_NoError(t, err)\n\tacc, err := sa.fetchAccount(pub)\n\trequire_NoError(t, err)\n\tif err := sa.updateAccountWithClaimJWT(acc, ajwt2); err != ErrAccountValidation {\n\t\tt.Fatalf(\"expected %v but got %v\", ErrAccountValidation, err)\n\t}\n}\n\n// Make sure connection limits apply to the system account itself.\nfunc TestSystemAccountSystemConnectionLimitsHonored(t *testing.T) {\n\tsa, optsA, sb, optsB, sakp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\t// Update system account to have 10 connections\n\tpub, _ := sakp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 10\n\tajwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, ajwt)\n\taddAccountToMemResolver(sb, pub, ajwt)\n\n\t// Update the accounts on each server with new claims to force update.\n\tsysAccA := sa.SystemAccount()\n\tsa.updateAccountWithClaimJWT(sysAccA, ajwt)\n\tsysAccB := sb.SystemAccount()\n\tsb.updateAccountWithClaimJWT(sysAccB, ajwt)\n\n\t// Check system here first, with no external it should be zero.\n\tsacc := sa.SystemAccount()\n\tif nlc := sacc.NumLocalConnections(); nlc != 0 {\n\t\tt.Fatalf(\"Expected no local connections, got %d\", nlc)\n\t}\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\n\t// Create a user on each server. Break on first failure.\n\ttc := 0\n\tfor {\n\t\tnca1, err := nats.Connect(urlA, createUserCreds(t, sa, sakp))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tdefer nca1.Close()\n\t\ttc++\n\n\t\tncb1, err := nats.Connect(urlB, createUserCreds(t, sb, sakp))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tdefer ncb1.Close()\n\t\ttc++\n\n\t\t// The account's connection count is exchanged between servers\n\t\t// so that the local count on each server reflects the total count.\n\t\t// Pause a bit to give a chance to each server to process the update.\n\t\ttime.Sleep(15 * time.Millisecond)\n\t}\n\tif tc != 10 {\n\t\tt.Fatalf(\"Expected to get 10 external connections, got %d\", tc)\n\t}\n\n\tcheckFor(t, 1*time.Second, 50*time.Millisecond, func() error {\n\t\ttotal := sa.NumClients() + sb.NumClients()\n\t\tif total > int(nac.Limits.Conn) {\n\t\t\treturn fmt.Errorf(\"Expected only %d connections, was allowed to connect %d\", nac.Limits.Conn, total)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Test that the remote accounting works when a server is started some time later.\nfunc TestSystemAccountConnectionLimitsServersStaggered(t *testing.T) {\n\tsa, optsA, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tsb.Shutdown()\n\n\t// Let's create a user account.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 4 // Limit to 4 connections.\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\t// Create max connections on sa.\n\tfor i := 0; i < int(nac.Limits.Conn); i++ {\n\t\tnc, err := nats.Connect(urlA, createUserCreds(t, sa, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on #%d try: %v\", i+1, err)\n\t\t}\n\t\tdefer nc.Close()\n\t}\n\n\t// Restart server B.\n\toptsB.AccountResolver = sa.AccountResolver()\n\toptsB.SystemAccount = sa.SystemAccount().Name\n\tsb = RunServer(optsB)\n\tdefer sb.Shutdown()\n\tcheckClusterFormed(t, sa, sb)\n\n\t// Trigger a load of the user account on the new server\n\t// NOTE: If we do not load the user, the user can be the first\n\t// to request this account, hence the connection will succeed.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif acc, err := sb.LookupAccount(pub); acc == nil || err != nil {\n\t\t\treturn fmt.Errorf(\"LookupAccount did not return account or failed, err=%v\", err)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Expect this to fail.\n\turlB := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tif _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil {\n\t\tt.Fatalf(\"Expected connection to fail due to max limit\")\n\t}\n}\n\n// Test that the remote accounting works when a server is shutdown.\nfunc TestSystemAccountConnectionLimitsServerShutdownGraceful(t *testing.T) {\n\tsa, optsA, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// Let's create a user account.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 10 // Limit to 10 connections.\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\taddAccountToMemResolver(sb, pub, jwt)\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\n\tfor i := 0; i < 5; i++ {\n\t\tnc, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tnc, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t}\n\n\t// We are at capacity so both of these should fail.\n\tif _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil {\n\t\tt.Fatalf(\"Expected connection to fail due to max limit\")\n\t}\n\tif _, err := nats.Connect(urlB, createUserCreds(t, sb, akp)); err == nil {\n\t\tt.Fatalf(\"Expected connection to fail due to max limit\")\n\t}\n\n\t// Now shutdown Server B.\n\tsb.Shutdown()\n\n\t// Now we should be able to create more on A now.\n\tfor i := 0; i < 5; i++ {\n\t\tnc, err := nats.Connect(urlA, createUserCreds(t, sa, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to connect on %d, got %v\", i, err)\n\t\t}\n\t\tdefer nc.Close()\n\t}\n}\n\n// Test that the remote accounting works when a server goes away.\nfunc TestSystemAccountConnectionLimitsServerShutdownForced(t *testing.T) {\n\tsa, optsA, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\n\t// Let's create a user account.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 20 // Limit to 20 connections.\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\taddAccountToMemResolver(sb, pub, jwt)\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\n\tfor i := 0; i < 10; i++ {\n\t\tc, err := nats.Connect(urlA, nats.NoReconnect(), createUserCreds(t, sa, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t\t}\n\t\tdefer c.Close()\n\t\tc, err = nats.Connect(urlB, nats.NoReconnect(), createUserCreds(t, sb, akp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t\t}\n\t\tdefer c.Close()\n\t}\n\n\t// Now shutdown Server B. Do so such that no communications go out.\n\tsb.mu.Lock()\n\tsb.sys = nil\n\tsb.mu.Unlock()\n\tsb.Shutdown()\n\n\tif _, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err == nil {\n\t\tt.Fatalf(\"Expected connection to fail due to max limit\")\n\t}\n\n\t// Let's speed up the checking process.\n\tsa.mu.Lock()\n\tsa.sys.chkOrph = 10 * time.Millisecond\n\tsa.sys.orphMax = 30 * time.Millisecond\n\tsa.sys.sweeper.Reset(sa.sys.chkOrph)\n\tsa.mu.Unlock()\n\n\t// We should eventually be able to connect.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tif c, err := nats.Connect(urlA, createUserCreds(t, sa, akp)); err != nil {\n\t\t\treturn err\n\t\t} else {\n\t\t\tc.Close()\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestSystemAccountFromConfig(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\topub, _ := kp.PublicKey()\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tdefer ts.Close()\n\n\tconfTemplate := `\n\t\tlisten: -1\n\t\ttrusted: %s\n\t\tsystem_account: %s\n\t\tresolver: URL(\"%s/jwt/v1/accounts/\")\n `\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, opub, apub, ts.URL)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif acc := s.SystemAccount(); acc == nil || acc.Name != apub {\n\t\tt.Fatalf(\"System Account not properly set\")\n\t}\n}\n\nfunc TestAccountClaimsUpdates(t *testing.T) {\n\ttest := func(subj string) {\n\t\ts, opts := runTrustedServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tsacc, sakp := createAccount(s)\n\t\ts.setSystemAccount(sacc)\n\n\t\t// Let's create a normal account with limits we can update.\n\t\tokp, _ := nkeys.FromSeed(oSeed)\n\t\takp, _ := nkeys.CreateAccount()\n\t\tpub, _ := akp.PublicKey()\n\t\tnac := jwt.NewAccountClaims(pub)\n\t\tnac.Limits.Conn = 4\n\t\tajwt, _ := nac.Encode(okp)\n\n\t\taddAccountToMemResolver(s, pub, ajwt)\n\n\t\tacc, _ := s.LookupAccount(pub)\n\t\tif acc.MaxActiveConnections() != 4 {\n\t\t\tt.Fatalf(\"Expected to see a limit of 4 connections\")\n\t\t}\n\n\t\t// Simulate a systems publisher so we can do an account claims update.\n\t\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t\tnc, err := nats.Connect(url, createUserCreds(t, s, sakp))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\n\t\t// Update the account\n\t\tnac = jwt.NewAccountClaims(pub)\n\t\tnac.Limits.Conn = 8\n\t\tissAt := time.Now().Add(-30 * time.Second).Unix()\n\t\tnac.IssuedAt = issAt\n\t\texpires := time.Now().Add(2 * time.Second).Unix()\n\t\tnac.Expires = expires\n\t\tajwt, _ = nac.Encode(okp)\n\n\t\t// Publish to the system update subject.\n\t\tclaimUpdateSubj := fmt.Sprintf(subj, pub)\n\t\tnc.Publish(claimUpdateSubj, []byte(ajwt))\n\t\tnc.Flush()\n\t\ttime.Sleep(200 * time.Millisecond)\n\n\t\tacc, _ = s.LookupAccount(pub)\n\t\tif acc.MaxActiveConnections() != 8 {\n\t\t\tt.Fatalf(\"Account was not updated\")\n\t\t}\n\t}\n\tt.Run(\"new\", func(t *testing.T) {\n\t\ttest(accUpdateEventSubjNew)\n\t})\n\tt.Run(\"old\", func(t *testing.T) {\n\t\ttest(accUpdateEventSubjOld)\n\t})\n}\n\nfunc TestAccountReqMonitoring(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\tsacc, sakp := createAccount(s)\n\ts.setSystemAccount(sacc)\n\ts.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()})\n\tunusedAcc, _ := createAccount(s)\n\tacc, akp := createAccount(s)\n\tacc.EnableJetStream(nil, nil)\n\tsubsz := fmt.Sprintf(accDirectReqSubj, acc.Name, \"SUBSZ\")\n\tconnz := fmt.Sprintf(accDirectReqSubj, acc.Name, \"CONNZ\")\n\tjsz := fmt.Sprintf(accDirectReqSubj, acc.Name, \"JSZ\")\n\n\tpStatz := fmt.Sprintf(accPingReqSubj, \"STATZ\")\n\tstatz := func(name string) string { return fmt.Sprintf(accDirectReqSubj, name, \"STATZ\") }\n\t// Create system account connection to query\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncSys, err := nats.Connect(url, createUserCreds(t, s, sakp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncSys.Close()\n\t// Create a connection that we can query\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\t// query SUBSZ for account\n\tresp, err := ncSys.Request(subsz, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), `\"num_subscriptions\":5,`)\n\t// create a subscription\n\tsub, err := nc.Subscribe(\"foo\", func(msg *nats.Msg) {})\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\trequire_NoError(t, nc.Flush())\n\t// query SUBSZ for account\n\tresp, err = ncSys.Request(subsz, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), `\"num_subscriptions\":6,`, `\"subject\":\"foo\"`)\n\t// query connections for account\n\tresp, err = ncSys.Request(connz, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), `\"num_connections\":1,`, `\"total\":1,`)\n\t// query connections for js account\n\tresp, err = ncSys.Request(jsz, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), `\"memory\":0,`, `\"storage\":0,`)\n\t// query statz/conns for account\n\tresp, err = ncSys.Request(statz(acc.Name), nil, time.Second)\n\trequire_NoError(t, err)\n\trespContentAcc := []string{`\"conns\":1,`, `\"total_conns\":1`, `\"slow_consumers\":0`, `\"sent\":{\"msgs\":0,\"bytes\":0`,\n\t\t`\"received\":{\"msgs\":0,\"bytes\":0`, `\"num_subscriptions\":`, fmt.Sprintf(`\"acc\":\"%s\"`, acc.Name)}\n\trequire_Contains(t, string(resp.Data), respContentAcc...)\n\n\trIb := ncSys.NewRespInbox()\n\trSub, err := ncSys.SubscribeSync(rIb)\n\trequire_NoError(t, err)\n\trequire_NoError(t, ncSys.PublishRequest(pStatz, rIb, nil))\n\tminRespContentForBothAcc := []string{`\"conns\":1,`, `\"total_conns\":1`, `\"slow_consumers\":0`, `\"acc\":\"`, `\"num_subscriptions\":`}\n\tresp, err = rSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), minRespContentForBothAcc...)\n\t// expect one entry per account\n\trequire_Contains(t, string(resp.Data), fmt.Sprintf(`\"acc\":\"%s\"`, acc.Name), fmt.Sprintf(`\"acc\":\"%s\"`, sacc.Name))\n\n\t// Test ping with filter by account name\n\trequire_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{\"accounts\":[\"%s\"]}`, sacc.Name))))\n\tm, err := rSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(m.Data), minRespContentForBothAcc...)\n\n\trequire_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{\"accounts\":[\"%s\"]}`, acc.Name))))\n\tm, err = rSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(m.Data), respContentAcc...)\n\n\t// Test include unused for statz and ping of statz\n\tunusedContent := []string{`\"conns\":0,`, `\"total_conns\":0`, `\"slow_consumers\":0`,\n\t\tfmt.Sprintf(`\"acc\":\"%s\"`, unusedAcc.Name)}\n\n\tresp, err = ncSys.Request(statz(unusedAcc.Name),\n\t\t[]byte(fmt.Sprintf(`{\"accounts\":[\"%s\"], \"include_unused\":true}`, unusedAcc.Name)),\n\t\ttime.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), unusedContent...)\n\n\trequire_NoError(t, ncSys.PublishRequest(pStatz, rIb,\n\t\t[]byte(fmt.Sprintf(`{\"accounts\":[\"%s\"], \"include_unused\":true}`, unusedAcc.Name))))\n\tresp, err = rSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Contains(t, string(resp.Data), unusedContent...)\n\n\trequire_NoError(t, ncSys.PublishRequest(pStatz, rIb, []byte(fmt.Sprintf(`{\"accounts\":[\"%s\"]}`, unusedAcc.Name))))\n\t_, err = rSub.NextMsg(200 * time.Millisecond)\n\trequire_Error(t, err)\n\n\t// Test ping from within account, send extra message to check counters.\n\trequire_NoError(t, nc.Publish(\"foo\", nil))\n\tib := nc.NewRespInbox()\n\trSub, err = nc.SubscribeSync(ib)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.PublishRequest(pStatz, ib, nil))\n\trequire_NoError(t, nc.Flush())\n\tresp, err = rSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\t// Since we now have processed our own message, sent msgs will be at least 1.\n\tpayload := string(resp.Data)\n\trespContentAcc = []string{`\"conns\":1,`, `\"total_conns\":1`, `\"slow_consumers\":0`, `\"sent\":{\"msgs\":1,\"bytes\":0`, fmt.Sprintf(`\"acc\":\"%s\"`, acc.Name)}\n\trequire_Contains(t, payload, respContentAcc...)\n\n\t// Depending on timing, statz message could be accounted too.\n\treceivedOK := strings.Contains(payload, `\"received\":{\"msgs\":1,\"bytes\":0`) || strings.Contains(payload, `\"received\":{\"msgs\":2,\"bytes\":0`)\n\trequire_True(t, receivedOK)\n\t_, err = rSub.NextMsg(200 * time.Millisecond)\n\trequire_Error(t, err)\n}\n\nfunc TestAccountReqInfo(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\tsacc, sakp := createAccount(s)\n\ts.setSystemAccount(sacc)\n\t// Let's create an account with service export.\n\takp, _ := nkeys.CreateAccount()\n\tpub1, _ := akp.PublicKey()\n\tnac1 := jwt.NewAccountClaims(pub1)\n\tnac1.Exports.Add(&jwt.Export{Subject: \"req.*\", Type: jwt.Service})\n\tajwt1, _ := nac1.Encode(oKp)\n\taddAccountToMemResolver(s, pub1, ajwt1)\n\ts.LookupAccount(pub1)\n\tinfo1 := fmt.Sprintf(accDirectReqSubj, pub1, \"INFO\")\n\t// Now add an account with service imports.\n\takp2, _ := nkeys.CreateAccount()\n\tpub2, _ := akp2.PublicKey()\n\tnac2 := jwt.NewAccountClaims(pub2)\n\tnac2.Imports.Add(&jwt.Import{Account: pub1, Subject: \"req.1\", Type: jwt.Service})\n\tajwt2, _ := nac2.Encode(oKp)\n\taddAccountToMemResolver(s, pub2, ajwt2)\n\ts.LookupAccount(pub2)\n\tinfo2 := fmt.Sprintf(accDirectReqSubj, pub2, \"INFO\")\n\t// Create system account connection to query\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncSys, err := nats.Connect(url, createUserCreds(t, s, sakp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncSys.Close()\n\tcheckCommon := func(info *AccountInfo, srv *ServerInfo, pub, jwt string) {\n\t\tif info.Complete != true {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.Complete)\n\t\t} else if info.Expired != false {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.Expired)\n\t\t} else if info.JetStream != false {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.JetStream)\n\t\t} else if info.ClientCnt != 0 {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.ClientCnt)\n\t\t} else if info.AccountName != pub {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.AccountName)\n\t\t} else if info.LeafCnt != 0 {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.LeafCnt)\n\t\t} else if info.Jwt != jwt {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", info.Jwt)\n\t\t} else if srv.Cluster != \"abc\" {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", srv.Cluster)\n\t\t} else if srv.Name != s.Name() {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", srv.Name)\n\t\t} else if srv.Host != opts.Host {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", srv.Host)\n\t\t} else if srv.Seq < 1 {\n\t\t\tt.Fatalf(\"Unexpected value: %v\", srv.Seq)\n\t\t}\n\t}\n\tinfo := AccountInfo{}\n\tsrv := ServerInfo{}\n\tmsg := struct {\n\t\tData *AccountInfo `json:\"data\"`\n\t\tSrv *ServerInfo `json:\"server\"`\n\t}{\n\t\t&info,\n\t\t&srv,\n\t}\n\tif resp, err := ncSys.Request(info1, nil, time.Second); err != nil {\n\t\tt.Fatalf(\"Error on request: %v\", err)\n\t} else if err := json.Unmarshal(resp.Data, &msg); err != nil {\n\t\tt.Fatalf(\"Unmarshalling failed: %v\", err)\n\t} else if len(info.Exports) != 1 {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.Exports)\n\t} else if len(info.Imports) != 4 {\n\t\tt.Fatalf(\"Unexpected value: %+v\", info.Imports)\n\t} else if info.Exports[0].Subject != \"req.*\" {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.Exports)\n\t} else if info.Exports[0].Type != jwt.Service {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.Exports)\n\t} else if info.Exports[0].ResponseType != jwt.ResponseTypeSingleton {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.Exports)\n\t} else if info.SubCnt != 4 {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.SubCnt)\n\t} else {\n\t\tcheckCommon(&info, &srv, pub1, ajwt1)\n\t}\n\tinfo = AccountInfo{}\n\tsrv = ServerInfo{}\n\tif resp, err := ncSys.Request(info2, nil, time.Second); err != nil {\n\t\tt.Fatalf(\"Error on request: %v\", err)\n\t} else if err := json.Unmarshal(resp.Data, &msg); err != nil {\n\t\tt.Fatalf(\"Unmarshalling failed: %v\", err)\n\t} else if len(info.Exports) != 0 {\n\t\tt.Fatalf(\"Unexpected value: %v\", info.Exports)\n\t} else if len(info.Imports) != 5 {\n\t\tt.Fatalf(\"Unexpected value: %+v\", info.Imports)\n\t}\n\t// Here we need to find our import\n\tvar si *ExtImport\n\tfor _, im := range info.Imports {\n\t\tif im.Subject == \"req.1\" {\n\t\t\tsi = &im\n\t\t\tbreak\n\t\t}\n\t}\n\tif si == nil {\n\t\tt.Fatalf(\"Could not find our import\")\n\t}\n\tif si.Type != jwt.Service {\n\t\tt.Fatalf(\"Unexpected value: %+v\", si)\n\t} else if si.Account != pub1 {\n\t\tt.Fatalf(\"Unexpected value: %+v\", si)\n\t} else if info.SubCnt != 5 {\n\t\tt.Fatalf(\"Unexpected value: %+v\", si)\n\t} else {\n\t\tcheckCommon(&info, &srv, pub2, ajwt2)\n\t}\n}\n\nfunc TestAccountClaimsUpdatesWithServiceImports(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tsacc, sakp := createAccount(s)\n\ts.setSystemAccount(sacc)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Let's create an account with service export.\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Exports.Add(&jwt.Export{Subject: \"req.*\", Type: jwt.Service})\n\tajwt, _ := nac.Encode(okp)\n\taddAccountToMemResolver(s, pub, ajwt)\n\ts.LookupAccount(pub)\n\n\t// Now add an account with multiple service imports.\n\takp2, _ := nkeys.CreateAccount()\n\tpub2, _ := akp2.PublicKey()\n\tnac2 := jwt.NewAccountClaims(pub2)\n\tnac2.Imports.Add(&jwt.Import{Account: pub, Subject: \"req.1\", Type: jwt.Service})\n\tajwt2, _ := nac2.Encode(okp)\n\n\taddAccountToMemResolver(s, pub2, ajwt2)\n\ts.LookupAccount(pub2)\n\n\tstartSubs := s.NumSubscriptions()\n\n\t// Simulate a systems publisher so we can do an account claims update.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, sakp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Update the account several times\n\tfor i := 1; i <= 10; i++ {\n\t\tnac2 = jwt.NewAccountClaims(pub2)\n\t\tnac2.Limits.Conn = int64(i)\n\t\tnac2.Imports.Add(&jwt.Import{Account: pub, Subject: \"req.1\", Type: jwt.Service})\n\t\tajwt2, _ = nac2.Encode(okp)\n\n\t\t// Publish to the system update subject.\n\t\tclaimUpdateSubj := fmt.Sprintf(accUpdateEventSubjNew, pub2)\n\t\tnc.Publish(claimUpdateSubj, []byte(ajwt2))\n\t}\n\tnc.Flush()\n\ttime.Sleep(50 * time.Millisecond)\n\n\tif startSubs < s.NumSubscriptions() {\n\t\tt.Fatalf(\"Subscriptions leaked: %d vs %d\", startSubs, s.NumSubscriptions())\n\t}\n}\n\nfunc TestAccountConnsLimitExceededAfterUpdate(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tsacc, _ := createAccount(s)\n\ts.setSystemAccount(sacc)\n\n\t// Let's create a normal account with limits we can update.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 10\n\tajwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(s, pub, ajwt)\n\tacc, _ := s.LookupAccount(pub)\n\n\t// Now create the max connections.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tfor {\n\t\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tdefer nc.Close()\n\t}\n\n\t// We should have max here.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tif total := s.NumClients(); total != acc.MaxActiveConnections() {\n\t\t\treturn fmt.Errorf(\"Expected %d connections, got %d\", acc.MaxActiveConnections(), total)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now change limits to make current connections over the limit.\n\tnac = jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 2\n\tajwt, _ = nac.Encode(okp)\n\n\ts.updateAccountWithClaimJWT(acc, ajwt)\n\tif acc.MaxActiveConnections() != 2 {\n\t\tt.Fatalf(\"Expected max connections to be set to 2, got %d\", acc.MaxActiveConnections())\n\t}\n\t// We should have closed the excess connections.\n\tcheckClientsCount(t, s, acc.MaxActiveConnections())\n}\n\nfunc TestAccountConnsLimitExceededAfterUpdateDisconnectNewOnly(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tsacc, _ := createAccount(s)\n\ts.setSystemAccount(sacc)\n\n\t// Let's create a normal account with limits we can update.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 10\n\tajwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(s, pub, ajwt)\n\tacc, _ := s.LookupAccount(pub)\n\n\t// Now create the max connections.\n\t// We create half then we will wait and then create the rest.\n\t// Will test that we disconnect the newest ones.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tfor i := 0; i < 5; i++ {\n\t\tnc, err := nats.Connect(url, nats.Name(\"OLD\"), nats.NoReconnect(), createUserCreds(t, s, akp))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tfor i := 0; i < 5; i++ {\n\t\tnc, err := nats.Connect(url, nats.Name(\"NEW\"), nats.NoReconnect(), createUserCreds(t, s, akp))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t}\n\n\t// We should have max here.\n\tcheckClientsCount(t, s, acc.MaxActiveConnections())\n\n\t// Now change limits to make current connections over the limit.\n\tnac = jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 5\n\tajwt, _ = nac.Encode(okp)\n\n\ts.updateAccountWithClaimJWT(acc, ajwt)\n\tif acc.MaxActiveConnections() != 5 {\n\t\tt.Fatalf(\"Expected max connections to be set to 2, got %d\", acc.MaxActiveConnections())\n\t}\n\t// We should have closed the excess connections.\n\tcheckClientsCount(t, s, acc.MaxActiveConnections())\n\n\tconnz, err := s.Connz(nil)\n\trequire_NoError(t, err)\n\n\t// There should only be OLD connections.\n\tfor _, c := range connz.Conns {\n\t\trequire_Equal(t, c.Name, \"OLD\")\n\t}\n}\n\nfunc TestSystemAccountWithBadRemoteLatencyUpdate(t *testing.T) {\n\ts, _ := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, _ := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\trl := remoteLatency{\n\t\tAccount: \"NONSENSE\",\n\t\tReqId: \"_INBOX.22\",\n\t}\n\tb, _ := json.Marshal(&rl)\n\ts.remoteLatencyUpdate(nil, nil, nil, \"foo\", _EMPTY_, nil, b)\n}\n\nfunc TestSystemAccountWithGateways(t *testing.T) {\n\tsa, oa, sb, ob, akp := runTrustedGateways(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// Create a client on A that will subscribe on $SYS.ACCOUNT.>\n\turla := fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port)\n\tnca := natsConnect(t, urla, createUserCreds(t, sa, akp), nats.Name(\"SYS\"))\n\tdefer nca.Close()\n\tnca.Flush()\n\n\tsub, _ := nca.SubscribeSync(\"$SYS.ACCOUNT.>\")\n\tdefer sub.Unsubscribe()\n\tnca.Flush()\n\n\t// If this tests fails with wrong number after 10 seconds we may have\n\t// added a new initial subscription for the eventing system.\n\tcheckExpectedSubs(t, 62, sa)\n\n\t// Create a client on B and see if we receive the event\n\turlb := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\tncb := natsConnect(t, urlb, createUserCreds(t, sb, akp), nats.Name(\"TEST EVENTS\"))\n\tdefer ncb.Close()\n\n\t// space for .CONNECT and .CONNS from SYS and $G as well as one extra message\n\tmsgs := [4]*nats.Msg{}\n\tvar err error\n\tmsgs[0], err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tmsgs[1], err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\t// TODO: There is a race currently that can cause the server to process the\n\t// system event *after* the subscription on \"A\" has been registered, and so\n\t// the \"nca\" client would receive its own CONNECT message.\n\tmsgs[2], _ = sub.NextMsg(250 * time.Millisecond)\n\n\tfindMsgs := func(sub string) []*nats.Msg {\n\t\trMsgs := []*nats.Msg{}\n\t\tfor _, m := range msgs {\n\t\t\tif m == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif m.Subject == sub {\n\t\t\t\trMsgs = append(rMsgs, m)\n\t\t\t}\n\t\t}\n\t\treturn rMsgs\n\t}\n\n\tmsg := findMsgs(fmt.Sprintf(\"$SYS.ACCOUNT.%s.CONNECT\", sa.SystemAccount().Name))\n\tvar bMsg *nats.Msg\n\tif len(msg) < 1 {\n\t\tt.Fatal(\"Expected at least one message\")\n\t}\n\tbMsg = msg[len(msg)-1]\n\n\trequire_Contains(t, string(bMsg.Data), sb.ID())\n\trequire_Contains(t, string(bMsg.Data), `\"cluster\":\"B\"`)\n\trequire_Contains(t, string(bMsg.Data), `\"name\":\"TEST EVENTS\"`)\n\n\tconnsMsgA := findMsgs(fmt.Sprintf(\"$SYS.ACCOUNT.%s.SERVER.CONNS\", sa.SystemAccount().Name))\n\tif len(connsMsgA) != 1 {\n\t\tt.Fatal(\"Expected a message\")\n\t}\n}\n\nfunc TestSystemAccountNoAuthUser(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\t$SYS {\n\t\t\t\tusers [{user: \"admin\", password: \"pwd\"}]\n\t\t\t}\n\t\t}\n\t`))\n\tdefer os.Remove(conf)\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tusrInfo string\n\t\tok bool\n\t\taccount string\n\t}{\n\t\t{\"valid user/pwd\", \"admin:pwd@\", true, \"$SYS\"},\n\t\t{\"invalid pwd\", \"admin:wrong@\", false, _EMPTY_},\n\t\t{\"some token\", \"sometoken@\", false, _EMPTY_},\n\t\t{\"user used without pwd\", \"admin@\", false, _EMPTY_}, // will be treated as a token\n\t\t{\"user with empty password\", \"admin:@\", false, _EMPTY_},\n\t\t{\"no user means global account\", _EMPTY_, true, globalAccountName},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\turl := fmt.Sprintf(\"nats://%s127.0.0.1:%d\", test.usrInfo, o.Port)\n\t\t\tnc, err := nats.Connect(url)\n\t\t\tif err != nil {\n\t\t\t\tif test.ok {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else if !test.ok {\n\t\t\t\tnc.Close()\n\t\t\t\tt.Fatalf(\"Should have failed, did not\")\n\t\t\t}\n\t\t\tvar accName string\n\t\t\ts.mu.Lock()\n\t\t\tfor _, c := range s.clients {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tif c.acc != nil {\n\t\t\t\t\taccName = c.acc.Name\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ts.mu.Unlock()\n\t\t\tnc.Close()\n\t\t\tcheckClientsCount(t, s, 0)\n\t\t\tif accName != test.account {\n\t\t\t\tt.Fatalf(\"The account should have been %q, got %q\", test.account, accName)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestServerAccountConns(t *testing.T) {\n\t// speed up hb\n\torgHBInterval := eventsHBInterval\n\teventsHBInterval = time.Millisecond * 100\n\tdefer func() { eventsHBInterval = orgHBInterval }()\n\tconf := createConfFile(t, []byte(`\n\t host: 127.0.0.1\n\t port: -1\n\t system_account: SYS\n\t accounts: {\n\t\t\t SYS: {users: [{user: s, password: s}]}\n\t\t\t ACC: {users: [{user: a, password: a}]}\n\t }`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"a\"))\n\tdefer nc.Close()\n\n\tsubOut, err := nc.SubscribeSync(\"foo\")\n\trequire_NoError(t, err)\n\thw := \"HELLO WORLD\"\n\tnc.Publish(\"foo\", []byte(hw))\n\tnc.Publish(\"bar\", []byte(hw)) // will only count towards received\n\tnc.Flush()\n\tm, err := subOut.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, string(m.Data), hw)\n\n\tncs := natsConnect(t, s.ClientURL(), nats.UserInfo(\"s\", \"s\"))\n\tdefer ncs.Close()\n\tsubs, err := ncs.SubscribeSync(\"$SYS.ACCOUNT.ACC.SERVER.CONNS\")\n\trequire_NoError(t, err)\n\n\tm, err = subs.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\taccConns := &AccountNumConns{}\n\terr = json.Unmarshal(m.Data, accConns)\n\trequire_NoError(t, err)\n\n\trequire_True(t, accConns.Received.Msgs == 2)\n\trequire_True(t, accConns.Received.Bytes == 2*int64(len(hw)))\n\trequire_True(t, accConns.Sent.Msgs == 1)\n\trequire_True(t, accConns.Sent.Bytes == int64(len(hw)))\n}\n\nfunc TestServerEventsStatsZ(t *testing.T) {\n\tserverStatsReqSubj := \"$SYS.REQ.SERVER.%s.STATSZ\"\n\tpreStart := time.Now().UTC()\n\t// Add little bit of delay to make sure that time check\n\t// between pre-start and actual start does not fail.\n\ttime.Sleep(5 * time.Millisecond)\n\tsa, optsA, sb, _, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\t// Same between actual start and post start.\n\ttime.Sleep(5 * time.Millisecond)\n\tpostStart := time.Now().UTC()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, sa, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\tsubj := fmt.Sprintf(serverStatsSubj, sa.ID())\n\tsub, _ := ncs.SubscribeSync(subj)\n\tdefer sub.Unsubscribe()\n\tncs.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\tncs.Flush()\n\n\t// Let's speed up the checking process.\n\tsa.mu.Lock()\n\tsa.sys.statsz = 10 * time.Millisecond\n\tsa.sys.stmr.Reset(sa.sys.statsz)\n\tsa.mu.Unlock()\n\n\t_, err = sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\t// Get it the second time so we can check some stats\n\tmsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tm := ServerStatsMsg{}\n\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t}\n\tif m.Server.ID != sa.ID() {\n\t\tt.Fatalf(\"Did not match IDs\")\n\t}\n\tif m.Server.Cluster != \"TEST_CLUSTER_22\" {\n\t\tt.Fatalf(\"Did not match cluster name\")\n\t}\n\tif m.Server.Version != VERSION {\n\t\tt.Fatalf(\"Did not match server version\")\n\t}\n\tif !m.Stats.Start.After(preStart) && m.Stats.Start.Before(postStart) {\n\t\tt.Fatalf(\"Got a wrong start time for the server %v\", m.Stats.Start)\n\t}\n\tif m.Stats.Connections != 1 {\n\t\tt.Fatalf(\"Did not match connections of 1, got %d\", m.Stats.Connections)\n\t}\n\tif m.Stats.ActiveAccounts != 1 {\n\t\tt.Fatalf(\"Did not match active accounts of 1, got %d\", m.Stats.ActiveAccounts)\n\t}\n\tif m.Stats.Sent.Msgs < 1 {\n\t\tt.Fatalf(\"Did not match sent msgs of >=1, got %d\", m.Stats.Sent.Msgs)\n\t}\n\tif m.Stats.Received.Msgs < 1 {\n\t\tt.Fatalf(\"Did not match received msgs of >=1, got %d\", m.Stats.Received.Msgs)\n\t}\n\t// Default pool size + 1 for system account\n\texpectedRoutes := DEFAULT_ROUTE_POOL_SIZE + 1\n\tif lr := len(m.Stats.Routes); lr != expectedRoutes {\n\t\tt.Fatalf(\"Expected %d routes, but got %d\", expectedRoutes, lr)\n\t}\n\n\t// Now let's prompt this server to send us the statsz\n\tsubj = fmt.Sprintf(serverStatsReqSubj, sa.ID())\n\tmsg, err = ncs.Request(subj, nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error trying to request statsz: %v\", err)\n\t}\n\tm2 := ServerStatsMsg{}\n\tif err := json.Unmarshal(msg.Data, &m2); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t}\n\tif m2.Server.ID != sa.ID() {\n\t\tt.Fatalf(\"Did not match IDs\")\n\t}\n\tif m2.Stats.Connections != 1 {\n\t\tt.Fatalf(\"Did not match connections of 1, got %d\", m2.Stats.Connections)\n\t}\n\tif m2.Stats.ActiveAccounts != 1 {\n\t\tt.Fatalf(\"Did not match active accounts of 1, got %d\", m2.Stats.ActiveAccounts)\n\t}\n\tif m2.Stats.Sent.Msgs < 3 {\n\t\tt.Fatalf(\"Did not match sent msgs of >= 3, got %d\", m2.Stats.Sent.Msgs)\n\t}\n\tif m2.Stats.Received.Msgs < 1 {\n\t\tt.Fatalf(\"Did not match received msgs of >= 1, got %d\", m2.Stats.Received.Msgs)\n\t}\n\tif lr := len(m2.Stats.Routes); lr != expectedRoutes {\n\t\tt.Fatalf(\"Expected %d routes, but got %d\", expectedRoutes, lr)\n\t}\n\n\tmsg, err = ncs.Request(subj, nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error trying to request statsz: %v\", err)\n\t}\n\tm3 := ServerStatsMsg{}\n\tif err := json.Unmarshal(msg.Data, &m3); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t}\n\tif m3.Server.ID != sa.ID() {\n\t\tt.Fatalf(\"Did not match IDs\")\n\t}\n\tif m3.Stats.Connections != 1 {\n\t\tt.Fatalf(\"Did not match connections of 1, got %d\", m3.Stats.Connections)\n\t}\n\tif m3.Stats.ActiveAccounts != 1 {\n\t\tt.Fatalf(\"Did not match active accounts of 1, got %d\", m3.Stats.ActiveAccounts)\n\t}\n\tif m3.Stats.Sent.Msgs < 4 {\n\t\tt.Fatalf(\"Did not match sent msgs of >= 4, got %d\", m3.Stats.Sent.Msgs)\n\t}\n\tif m3.Stats.Received.Msgs < 2 {\n\t\tt.Fatalf(\"Did not match received msgs of >= 2, got %d\", m3.Stats.Received.Msgs)\n\t}\n\tif lr := len(m3.Stats.Routes); lr != expectedRoutes {\n\t\tt.Fatalf(\"Expected %d routes, but got %d\", expectedRoutes, lr)\n\t}\n\tfor _, sr := range m3.Stats.Routes {\n\t\tif sr.Name != \"B_SRV\" {\n\t\t\tt.Fatalf(\"Expected server A's route to B to have Name set to %q, got %q\", \"B\", sr.Name)\n\t\t}\n\t}\n\t// Increment stalls to confirm they are reported.\n\tatomic.AddInt64(&sb.stalls, 3)\n\n\t// Now query B and check that route's name is \"A\"\n\tsubj = fmt.Sprintf(serverStatsReqSubj, sb.ID())\n\tncs.SubscribeSync(subj)\n\tmsg, err = ncs.Request(subj, nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error trying to request statsz: %v\", err)\n\t}\n\tm = ServerStatsMsg{}\n\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t}\n\tif lr := len(m.Stats.Routes); lr != expectedRoutes {\n\t\tt.Fatalf(\"Expected %d routes, but got %d\", expectedRoutes, lr)\n\t}\n\tfor _, sr := range m.Stats.Routes {\n\t\tif sr.Name != \"A_SRV\" {\n\t\t\tt.Fatalf(\"Expected server B's route to A to have Name set to %q, got %q\", \"A_SRV\", sr.Name)\n\t\t}\n\t}\n\trequire_Equal(t, m.Stats.StalledClients, 3)\n}\n\nfunc TestServerEventsHealthZSingleServer(t *testing.T) {\n\ttype healthzResp struct {\n\t\tHealthz HealthStatus `json:\"data\"`\n\t\tServer ServerInfo `json:\"server\"`\n\t}\n\tcfg := fmt.Sprintf(`listen: 127.0.0.1:-1\n\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tno_auth_user: one\n\n\taccounts {\n\t\tONE { users = [ { user: \"one\", pass: \"p\" } ]; jetstream: enabled }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}`, t.TempDir())\n\n\tserverHealthzReqSubj := \"$SYS.REQ.SERVER.%s.HEALTHZ\"\n\ts, _ := RunServerWithConfig(createConfFile(t, []byte(cfg)))\n\tdefer s.Shutdown()\n\n\tncs, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to cluster: %v\", err)\n\t}\n\n\tdefer ncs.Close()\n\tncAcc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncAcc.Close()\n\tjs, err := ncAcc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating JetStream context: %v\", err)\n\t}\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\tName: \"cons\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\n\tsubj := fmt.Sprintf(serverHealthzReqSubj, s.ID())\n\n\ttests := []struct {\n\t\tname string\n\t\treq *HealthzEventOptions\n\t\texpected HealthStatus\n\t}{\n\t\t{\n\t\t\tname: \"no parameters\",\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with js enabled only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tJSEnabledOnly: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with server only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tJSServerOnly: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with js meta only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tJSMetaOnly: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name and stream\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name, stream and consumer\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"cons\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with stream only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tError: `\"account\" must not be empty when checking stream health`,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"with stream only, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tDetails: true,\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorBadRequest,\n\t\t\t\t\t\tError: `\"account\" must not be empty when checking stream health`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"with account and consumer\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tConsumer: \"cons\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tError: `\"stream\" must not be empty when checking consumer health`,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"with account and consumer, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tConsumer: \"cons\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorBadRequest,\n\t\t\t\t\t\tError: `\"stream\" must not be empty when checking consumer health`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"account not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream account \"abc\" not found`,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"account not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t\t\tError: `JetStream account \"abc\" not found`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stream not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream stream \"abc\" not found on account \"ONE\"`,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stream not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\t\tStream: \"abc\",\n\t\t\t\t\t\tError: `JetStream stream \"abc\" not found on account \"ONE\"`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"consumer not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream consumer \"abc\" not found for stream \"test\" on account \"ONE\"`,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"consumer not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorConsumer,\n\t\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\t\tStream: \"test\",\n\t\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t\t\tError: `JetStream consumer \"abc\" not found for stream \"test\" on account \"ONE\"`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar body []byte\n\t\t\tvar err error\n\t\t\tif test.req != nil {\n\t\t\t\tbody, err = json.Marshal(test.req)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error marshaling request body: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tmsg, err := ncs.Request(subj, body, 1*time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error trying to request healthz: %v\", err)\n\t\t\t}\n\t\t\tvar health healthzResp\n\t\t\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(health.Healthz, test.expected) {\n\t\t\t\tt.Errorf(\"Invalid healthz status; want: %+v; got: %+v\", test.expected, health.Healthz)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestServerEventsHealthZClustered(t *testing.T) {\n\ttype healthzResp struct {\n\t\tHealthz HealthStatus `json:\"data\"`\n\t\tServer ServerInfo `json:\"server\"`\n\t}\n\tserverHealthzReqSubj := \"$SYS.REQ.SERVER.%s.HEALTHZ\"\n\tc := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to cluster: %v\", err)\n\t}\n\n\tdefer ncs.Close()\n\tncAcc, err := nats.Connect(c.randomServer().ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncAcc.Close()\n\tjs, err := ncAcc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating JetStream context: %v\", err)\n\t}\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\tName: \"cons\",\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\n\tc.waitOnStreamLeader(\"ONE\", \"test\")\n\tc.waitOnConsumerLeader(\"ONE\", \"test\", \"cons\")\n\tc.waitOnAllCurrent()\n\n\tsubj := fmt.Sprintf(serverHealthzReqSubj, c.servers[0].ID())\n\tpingSubj := fmt.Sprintf(serverHealthzReqSubj, \"PING\")\n\n\ttests := []struct {\n\t\tname string\n\t\treq *HealthzEventOptions\n\t\texpected HealthStatus\n\t\texpectedError string\n\t}{\n\t\t{\n\t\t\tname: \"no parameters\",\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with js enabled only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tJSEnabledOnly: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with server only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tJSServerOnly: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name and stream\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with account name, stream and consumer\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"cons\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{Status: \"ok\", StatusCode: 200},\n\t\t},\n\t\t{\n\t\t\tname: \"with stream only\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tError: `\"account\" must not be empty when checking stream health`,\n\t\t\t},\n\t\t\texpectedError: \"Bad request:\",\n\t\t},\n\t\t{\n\t\t\tname: \"with stream only, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tDetails: true,\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 400,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorBadRequest,\n\t\t\t\t\t\tError: `\"account\" must not be empty when checking stream health`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"account not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream account \"abc\" not found`,\n\t\t\t},\n\t\t\texpectedError: `account \"abc\" not found`,\n\t\t},\n\t\t{\n\t\t\tname: \"account not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\t\t\tAccount: \"abc\",\n\t\t\t\t\t\tError: `JetStream account \"abc\" not found`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stream not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream stream \"abc\" not found on account \"ONE\"`,\n\t\t\t},\n\t\t\texpectedError: `stream \"abc\" not found`,\n\t\t},\n\t\t{\n\t\t\tname: \"stream not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\t\tStream: \"abc\",\n\t\t\t\t\t\tError: `JetStream stream \"abc\" not found on account \"ONE\"`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"consumer not found\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"unavailable\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tError: `JetStream consumer \"abc\" not found for stream \"test\" on account \"ONE\"`,\n\t\t\t},\n\t\t\texpectedError: `consumer \"abc\" not found for stream \"test\"`,\n\t\t},\n\t\t{\n\t\t\tname: \"consumer not found, detailed\",\n\t\t\treq: &HealthzEventOptions{\n\t\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\tStream: \"test\",\n\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t\tDetails: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: HealthStatus{\n\t\t\t\tStatus: \"error\",\n\t\t\t\tStatusCode: 404,\n\t\t\t\tErrors: []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorConsumer,\n\t\t\t\t\t\tAccount: \"ONE\",\n\t\t\t\t\t\tStream: \"test\",\n\t\t\t\t\t\tConsumer: \"abc\",\n\t\t\t\t\t\tError: `JetStream consumer \"abc\" not found for stream \"test\" on account \"ONE\"`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar body []byte\n\t\t\tvar err error\n\t\t\tif test.req != nil {\n\t\t\t\tbody, err = json.Marshal(test.req)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error marshaling request body: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tmsg, err := ncs.Request(subj, body, 1*time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error trying to request healthz: %v\", err)\n\t\t\t}\n\t\t\tvar health healthzResp\n\t\t\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(health.Healthz, test.expected) {\n\t\t\t\tt.Errorf(\"Invalid healthz status; want: %+v; got: %+v\", test.expected, health.Healthz)\n\t\t\t}\n\n\t\t\treply := ncs.NewRespInbox()\n\t\t\tsub, err := ncs.SubscribeSync(reply)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating subscription: %v\", err)\n\t\t\t}\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// now PING all servers\n\t\t\tif err := ncs.PublishRequest(pingSubj, reply, body); err != nil {\n\t\t\t\tt.Fatalf(\"Publish error: %v\", err)\n\t\t\t}\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tmsg, err := sub.NextMsg(1 * time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error fetching healthz PING response: %v\", err)\n\t\t\t\t}\n\t\t\t\tvar health healthzResp\n\t\t\t\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\t\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(health.Healthz, test.expected) {\n\t\t\t\t\tt.Errorf(\"Invalid healthz status; want: %+v; got: %+v\", test.expected, health.Healthz)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) {\n\t\t\t\tt.Fatalf(\"Expected timeout error; got: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestServerEventsHealthZClustered_NoReplicas(t *testing.T) {\n\ttype healthzResp struct {\n\t\tHealthz HealthStatus `json:\"data\"`\n\t\tServer ServerInfo `json:\"server\"`\n\t}\n\tserverHealthzReqSubj := \"$SYS.REQ.SERVER.%s.HEALTHZ\"\n\tc := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tncs, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to cluster: %v\", err)\n\t}\n\n\tdefer ncs.Close()\n\tncAcc, err := nats.Connect(c.randomServer().ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncAcc.Close()\n\tjs, err := ncAcc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating JetStream context: %v\", err)\n\t}\n\n\tpingSubj := fmt.Sprintf(serverHealthzReqSubj, \"PING\")\n\n\tt.Run(\"non-replicated stream\", func(t *testing.T) {\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"test\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t\t}\n\t\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"cons\",\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t\t}\n\t\tbody, err := json.Marshal(HealthzEventOptions{\n\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\tAccount: \"ONE\",\n\t\t\t\tStream: \"test\",\n\t\t\t},\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error marshaling request body: %v\", err)\n\t\t}\n\n\t\treply := ncs.NewRespInbox()\n\t\tsub, err := ncs.SubscribeSync(reply)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating subscription: %v\", err)\n\t\t}\n\t\tdefer sub.Unsubscribe()\n\n\t\t// now PING all servers\n\t\tif err := ncs.PublishRequest(pingSubj, reply, body); err != nil {\n\t\t\tt.Fatalf(\"Publish error: %v\", err)\n\t\t}\n\t\tvar healthy int\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tmsg, err := sub.NextMsg(1 * time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error fetching healthz PING response: %v\", err)\n\t\t\t}\n\t\t\tvar health healthzResp\n\t\t\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t\t}\n\t\t\tif health.Healthz.Status == \"ok\" {\n\t\t\t\thealthy++\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !strings.Contains(health.Healthz.Error, `stream \"test\" not found`) {\n\t\t\t\tt.Errorf(\"Expected error to contain: %q, got: %s\", `stream \"test\" not found`, health.Healthz.Error)\n\t\t\t}\n\t\t}\n\t\tif healthy != 1 {\n\t\t\tt.Fatalf(\"Expected 1 healthy server; got: %d\", healthy)\n\t\t}\n\t\tif _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) {\n\t\t\tt.Fatalf(\"Expected timeout error; got: %v\", err)\n\t\t}\n\t})\n\n\tt.Run(\"non-replicated consumer\", func(t *testing.T) {\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"test-repl\",\n\t\t\tSubjects: []string{\"bar\"},\n\t\t\tReplicas: 3,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t\t}\n\t\t_, err = js.AddConsumer(\"test-repl\", &nats.ConsumerConfig{\n\t\t\tName: \"cons-single\",\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t\t}\n\t\tbody, err := json.Marshal(HealthzEventOptions{\n\t\t\tHealthzOptions: HealthzOptions{\n\t\t\t\tAccount: \"ONE\",\n\t\t\t\tStream: \"test-repl\",\n\t\t\t\tConsumer: \"cons-single\",\n\t\t\t},\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error marshaling request body: %v\", err)\n\t\t}\n\n\t\treply := ncs.NewRespInbox()\n\t\tsub, err := ncs.SubscribeSync(reply)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating subscription: %v\", err)\n\t\t}\n\t\tdefer sub.Unsubscribe()\n\n\t\t// now PING all servers\n\t\tif err := ncs.PublishRequest(pingSubj, reply, body); err != nil {\n\t\t\tt.Fatalf(\"Publish error: %v\", err)\n\t\t}\n\t\tvar healthy int\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tmsg, err := sub.NextMsg(1 * time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error fetching healthz PING response: %v\", err)\n\t\t\t}\n\t\t\tvar health healthzResp\n\t\t\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t\t}\n\t\t\tif health.Healthz.Status == \"ok\" {\n\t\t\t\thealthy++\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !strings.Contains(health.Healthz.Error, `consumer \"cons-single\" not found`) {\n\t\t\t\tt.Errorf(\"Expected error to contain: %q, got: %s\", `consumer \"cons-single\" not found`, health.Healthz.Error)\n\t\t\t}\n\t\t}\n\t\tif healthy != 1 {\n\t\t\tt.Fatalf(\"Expected 1 healthy server; got: %d\", healthy)\n\t\t}\n\t\tif _, err := sub.NextMsg(50 * time.Millisecond); !errors.Is(err, nats.ErrTimeout) {\n\t\t\tt.Fatalf(\"Expected timeout error; got: %v\", err)\n\t\t}\n\t})\n\n}\n\nfunc TestServerEventsHealthZJetStreamNotEnabled(t *testing.T) {\n\ttype healthzResp struct {\n\t\tHealthz HealthStatus `json:\"data\"`\n\t\tServer ServerInfo `json:\"server\"`\n\t}\n\tcfg := `listen: 127.0.0.1:-1\n\n\taccounts {\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}`\n\n\tserverHealthzReqSubj := \"$SYS.REQ.SERVER.%s.HEALTHZ\"\n\ts, _ := RunServerWithConfig(createConfFile(t, []byte(cfg)))\n\tdefer s.Shutdown()\n\n\tncs, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to cluster: %v\", err)\n\t}\n\n\tdefer ncs.Close()\n\n\tsubj := fmt.Sprintf(serverHealthzReqSubj, s.ID())\n\n\tmsg, err := ncs.Request(subj, nil, 1*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error trying to request healthz: %v\", err)\n\t}\n\tvar health healthzResp\n\tif err := json.Unmarshal(msg.Data, &health); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t}\n\tif health.Healthz.Status != \"ok\" {\n\t\tt.Errorf(\"Invalid healthz status; want: %q; got: %q\", \"ok\", health.Healthz.Status)\n\t}\n\tif health.Healthz.Error != \"\" {\n\t\tt.Errorf(\"HealthZ error: %s\", health.Healthz.Error)\n\t}\n}\n\nfunc TestServerEventsPingStatsZ(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\ttest := func(req []byte) {\n\t\treply := nc.NewRespInbox()\n\t\tsub, _ := nc.SubscribeSync(reply)\n\t\tnc.PublishRequest(serverStatsPingReqSubj, reply, req)\n\t\t// Make sure its a statsz\n\t\tm := ServerStatsMsg{}\n\t\t// Receive both manually.\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t\t}\n\t\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t}\n\t\tmsg, err = sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t\t}\n\t\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\t\tt.Fatalf(\"Error unmarshalling the statz json: %v\", err)\n\t\t}\n\t}\n\tstrRequestTbl := []string{\n\t\t`{\"cluster\":\"TEST\"}`,\n\t\t`{\"cluster\":\"CLUSTER\"}`,\n\t\t`{\"server_name\":\"SRV\"}`,\n\t\t`{\"server_name\":\"_\"}`,\n\t\tfmt.Sprintf(`{\"host\":\"%s\"}`, optsB.Host),\n\t\tfmt.Sprintf(`{\"host\":\"%s\", \"cluster\":\"CLUSTER\", \"name\":\"SRV\"}`, optsB.Host),\n\t}\n\tfor i, opt := range strRequestTbl {\n\t\tt.Run(fmt.Sprintf(\"%s-%d\", t.Name(), i), func(t *testing.T) {\n\t\t\ttest([]byte(opt))\n\t\t})\n\t}\n\trequestTbl := []StatszEventOptions{\n\t\t{EventFilterOptions: EventFilterOptions{Cluster: \"TEST\"}},\n\t\t{EventFilterOptions: EventFilterOptions{Cluster: \"CLUSTER\"}},\n\t\t{EventFilterOptions: EventFilterOptions{Name: \"SRV\"}},\n\t\t{EventFilterOptions: EventFilterOptions{Name: \"_\"}},\n\t\t{EventFilterOptions: EventFilterOptions{Host: optsB.Host}},\n\t\t{EventFilterOptions: EventFilterOptions{Host: optsB.Host, Cluster: \"CLUSTER\", Name: \"SRV\"}},\n\t}\n\tfor i, opt := range requestTbl {\n\t\tt.Run(fmt.Sprintf(\"%s-%d\", t.Name(), i), func(t *testing.T) {\n\t\t\tmsg, _ := json.MarshalIndent(&opt, \"\", \" \")\n\t\t\ttest(msg)\n\t\t})\n\t}\n}\n\nfunc TestServerEventsPingStatsZDedicatedRecvQ(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\t// We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS\n\t// event to be pushed in the mux'ed queue.\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttestReq := func(t *testing.T, subj string, expectTwo bool) {\n\t\tfor _, s := range []*Server{sa, sb} {\n\t\t\ts.mu.RLock()\n\t\t\trecvq := s.sys.recvq\n\t\t\ts.mu.RUnlock()\n\t\t\trecvq.Lock()\n\t\t\tdefer recvq.Unlock()\n\t\t}\n\t\treply := nc.NewRespInbox()\n\t\tsub := natsSubSync(t, nc, reply)\n\t\tnc.PublishRequest(subj, reply, nil)\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tif len(msg.Data) == 0 {\n\t\t\tt.Fatal(\"Unexpected empty response\")\n\t\t}\n\t\t// Make sure its a statsz\n\t\tm := ServerStatsMsg{}\n\t\terr := json.Unmarshal(msg.Data, &m)\n\t\trequire_NoError(t, err)\n\t\trequire_False(t, m.Stats.Start.IsZero())\n\t\tif expectTwo {\n\t\t\tmsg = natsNexMsg(t, sub, time.Second)\n\t\t\terr = json.Unmarshal(msg.Data, &m)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_False(t, m.Stats.Start.IsZero())\n\t\t}\n\t}\n\tconst statsz = \"STATSZ\"\n\tfor _, test := range []struct {\n\t\tname string\n\t\tf func() string\n\t\texpectTwo bool\n\t}{\n\t\t{\"server stats ping request subject\", func() string { return serverStatsPingReqSubj }, true},\n\t\t{\"server ping request subject\", func() string { return fmt.Sprintf(serverPingReqSubj, statsz) }, true},\n\t\t{\"server a direct request subject\", func() string { return fmt.Sprintf(serverDirectReqSubj, sa.ID(), statsz) }, false},\n\t\t{\"server b direct request subject\", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), statsz) }, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttestReq(t, test.f(), test.expectTwo)\n\t\t})\n\t}\n}\n\nfunc TestServerEventsPingStatsZFilter(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\trequestTbl := []string{\n\t\t`{\"cluster\":\"DOESNOTEXIST\"}`,\n\t\t`{\"host\":\"DOESNOTEXIST\"}`,\n\t\t`{\"server_name\":\"DOESNOTEXIST\"}`,\n\t}\n\tfor i, msg := range requestTbl {\n\t\tt.Run(fmt.Sprintf(\"%s-%d\", t.Name(), i), func(t *testing.T) {\n\t\t\t// Receive both manually.\n\t\t\tif _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Error, expected timeout: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n\trequestObjTbl := []EventFilterOptions{\n\t\t{Cluster: \"DOESNOTEXIST\"},\n\t\t{Host: \"DOESNOTEXIST\"},\n\t\t{Name: \"DOESNOTEXIST\"},\n\t}\n\tfor i, opt := range requestObjTbl {\n\t\tt.Run(fmt.Sprintf(\"%s-%d\", t.Name(), i), func(t *testing.T) {\n\t\t\tmsg, _ := json.MarshalIndent(&opt, \"\", \" \")\n\t\t\t// Receive both manually.\n\t\t\tif _, err := nc.Request(serverStatsPingReqSubj, []byte(msg), time.Second/4); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Error, expected timeout: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestServerEventsPingStatsZFailFilter(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Receive both manually.\n\tif msg, err := nc.Request(serverStatsPingReqSubj, []byte(`{MALFORMEDJSON`), time.Second/4); err != nil {\n\t\tt.Fatalf(\"Error: %v\", err)\n\t} else {\n\t\tresp := make(map[string]map[string]any)\n\t\tif err := json.Unmarshal(msg.Data, &resp); err != nil {\n\t\t\tt.Fatalf(\"Error unmarshalling the response json: %v\", err)\n\t\t}\n\t\tif resp[\"error\"][\"code\"].(float64) != http.StatusBadRequest {\n\t\t\tt.Fatal(\"bad error code\")\n\t\t}\n\t}\n}\n\nfunc TestServerEventsPingMonitorz(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\tsysAcc, _ := akp.PublicKey()\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc.Flush()\n\n\ttests := []struct {\n\t\tendpoint string\n\t\topt any\n\t\tresp any\n\t\trespField []string\n\t}{\n\t\t{\"VARZ\", nil, &Varz{},\n\t\t\t[]string{\"now\", \"cpu\", \"system_account\"}},\n\t\t{\"SUBSZ\", nil, &Subsz{},\n\t\t\t[]string{\"num_subscriptions\", \"num_cache\"}},\n\t\t{\"CONNZ\", nil, &Connz{},\n\t\t\t[]string{\"now\", \"connections\"}},\n\t\t{\"ROUTEZ\", nil, &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"GATEWAYZ\", nil, &Gatewayz{},\n\t\t\t[]string{\"now\", \"outbound_gateways\", \"inbound_gateways\"}},\n\t\t{\"LEAFZ\", nil, &Leafz{},\n\t\t\t[]string{\"now\", \"leafs\"}},\n\n\t\t{\"SUBSZ\", &SubszOptions{}, &Subsz{},\n\t\t\t[]string{\"num_subscriptions\", \"num_cache\"}},\n\t\t{\"CONNZ\", &ConnzOptions{}, &Connz{},\n\t\t\t[]string{\"now\", \"connections\"}},\n\t\t{\"ROUTEZ\", &RoutezOptions{}, &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"GATEWAYZ\", &GatewayzOptions{}, &Gatewayz{},\n\t\t\t[]string{\"now\", \"outbound_gateways\", \"inbound_gateways\"}},\n\t\t{\"LEAFZ\", &LeafzOptions{}, &Leafz{},\n\t\t\t[]string{\"now\", \"leafs\"}},\n\t\t{\"ACCOUNTZ\", &AccountzOptions{}, &Accountz{},\n\t\t\t[]string{\"now\", \"accounts\"}},\n\n\t\t{\"SUBSZ\", &SubszOptions{Limit: 5}, &Subsz{},\n\t\t\t[]string{\"num_subscriptions\", \"num_cache\"}},\n\t\t{\"CONNZ\", &ConnzOptions{Limit: 5}, &Connz{},\n\t\t\t[]string{\"now\", \"connections\"}},\n\t\t{\"ROUTEZ\", &RoutezOptions{SubscriptionsDetail: true}, &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"GATEWAYZ\", &GatewayzOptions{Accounts: true}, &Gatewayz{},\n\t\t\t[]string{\"now\", \"outbound_gateways\", \"inbound_gateways\"}},\n\t\t{\"LEAFZ\", &LeafzOptions{Subscriptions: true}, &Leafz{},\n\t\t\t[]string{\"now\", \"leafs\"}},\n\t\t{\"ACCOUNTZ\", &AccountzOptions{Account: sysAcc}, &Accountz{},\n\t\t\t[]string{\"now\", \"account_detail\"}},\n\t\t{\"LEAFZ\", &LeafzOptions{Account: sysAcc}, &Leafz{},\n\t\t\t[]string{\"now\", \"leafs\"}},\n\n\t\t{\"ROUTEZ\", json.RawMessage(`{\"cluster\":\"\"}`), &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"ROUTEZ\", json.RawMessage(`{\"name\":\"\"}`), &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"ROUTEZ\", json.RawMessage(`{\"cluster\":\"TEST_CLUSTER_22\"}`), &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"ROUTEZ\", json.RawMessage(`{\"cluster\":\"CLUSTER\"}`), &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\t\t{\"ROUTEZ\", json.RawMessage(`{\"cluster\":\"TEST_CLUSTER_22\", \"subscriptions\":true}`), &Routez{},\n\t\t\t[]string{\"now\", \"routes\"}},\n\n\t\t{\"JSZ\", nil, &JSzOptions{}, []string{\"now\", \"disabled\"}},\n\n\t\t{\"HEALTHZ\", nil, &JSzOptions{}, []string{\"status\"}},\n\t\t{\"HEALTHZ\", &HealthzOptions{JSEnabledOnly: true}, &JSzOptions{}, []string{\"status\"}},\n\t\t{\"HEALTHZ\", &HealthzOptions{JSServerOnly: true}, &JSzOptions{}, []string{\"status\"}},\n\t\t{\"HEALTHZ\", &HealthzOptions{JSMetaOnly: true}, &JSzOptions{}, []string{\"status\"}},\n\t\t{\"EXPVARZ\", nil, &ExpvarzStatus{}, []string{\"memstats\", \"cmdline\"}},\n\t}\n\n\tfor i, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%s-%d\", test.endpoint, i), func(t *testing.T) {\n\t\t\tvar opt []byte\n\t\t\tif test.opt != nil {\n\t\t\t\topt, err = json.Marshal(test.opt)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error marshaling opts: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treply := nc.NewRespInbox()\n\t\t\treplySubj, _ := nc.SubscribeSync(reply)\n\n\t\t\t// set a header to make sure request parsing knows to ignore them\n\t\t\tnc.PublishMsg(&nats.Msg{\n\t\t\t\tSubject: fmt.Sprintf(\"%s.%s\", serverStatsPingReqSubj, test.endpoint),\n\t\t\t\tReply: reply,\n\t\t\t\tHeader: nats.Header{\"header\": []string{\"for header sake\"}},\n\t\t\t\tData: opt,\n\t\t\t})\n\n\t\t\t// Receive both manually.\n\t\t\tmsg, err := replySubj.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t\t\t}\n\t\t\tresponse1 := make(map[string]map[string]any)\n\n\t\t\tif err := json.Unmarshal(msg.Data, &response1); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling response1 json: %v\", err)\n\t\t\t}\n\n\t\t\tserverName := \"\"\n\t\t\tif response1[\"server\"][\"name\"] == \"A_SRV\" {\n\t\t\t\tserverName = \"B_SRV\"\n\t\t\t} else if response1[\"server\"][\"name\"] == \"B_SRV\" {\n\t\t\t\tserverName = \"A_SRV\"\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Error finding server in %s\", string(msg.Data))\n\t\t\t}\n\t\t\tif resp, ok := response1[\"data\"]; !ok {\n\t\t\t\tt.Fatalf(\"Error finding: %s in %s\",\n\t\t\t\t\tstrings.ToLower(test.endpoint), string(msg.Data))\n\t\t\t} else {\n\t\t\t\tfor _, respField := range test.respField {\n\t\t\t\t\tif _, ok := resp[respField]; !ok {\n\t\t\t\t\t\tt.Fatalf(\"Error finding: %s in %s\", respField, resp)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tmsg, err = replySubj.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t\t\t}\n\t\t\tresponse2 := make(map[string]map[string]any)\n\t\t\tif err := json.Unmarshal(msg.Data, &response2); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling the response2 json: %v\", err)\n\t\t\t}\n\t\t\tif response2[\"server\"][\"name\"] != serverName {\n\t\t\t\tt.Fatalf(\"Error finding server %s in %s\", serverName, string(msg.Data))\n\t\t\t}\n\t\t\tif resp, ok := response2[\"data\"]; !ok {\n\t\t\t\tt.Fatalf(\"Error finding: %s in %s\",\n\t\t\t\t\tstrings.ToLower(test.endpoint), string(msg.Data))\n\t\t\t} else {\n\t\t\t\tfor _, respField := range test.respField {\n\t\t\t\t\tif val, ok := resp[respField]; !ok {\n\t\t\t\t\t\tt.Fatalf(\"Error finding: %s in %s\", respField, resp)\n\t\t\t\t\t} else if val == nil {\n\t\t\t\t\t\tt.Fatalf(\"Nil value found: %s in %s\", respField, resp)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestGatewayNameClientInfo(t *testing.T) {\n\tsa, _, sb, _, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\tc, _, l := newClientForServer(sa)\n\tdefer c.close()\n\n\tvar info Info\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Cluster != \"TEST_CLUSTER_22\" {\n\t\tt.Fatalf(\"Expected a cluster name of 'TEST_CLUSTER_22', got %q\", info.Cluster)\n\t}\n}\n\ntype slowAccResolver struct {\n\tsync.Mutex\n\tAccountResolver\n\tacc string\n}\n\nfunc (sr *slowAccResolver) Fetch(name string) (string, error) {\n\tsr.Lock()\n\tdelay := sr.acc == name\n\tsr.Unlock()\n\tif delay {\n\t\ttime.Sleep(200 * time.Millisecond)\n\t}\n\treturn sr.AccountResolver.Fetch(name)\n}\n\nfunc TestConnectionUpdatesTimerProperlySet(t *testing.T) {\n\torigEventsHBInterval := eventsHBInterval\n\teventsHBInterval = 50 * time.Millisecond\n\tdefer func() { eventsHBInterval = origEventsHBInterval }()\n\n\tsa, _, sb, optsB, _ := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// Normal Account\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tnac.Limits.Conn = 10 // set any limit...\n\tjwt, _ := nac.Encode(okp)\n\n\taddAccountToMemResolver(sa, pub, jwt)\n\n\t// Listen for HB updates...\n\tcount := int32(0)\n\tcb := func(sub *subscription, _ *client, _ *Account, subject, reply string, msg []byte) {\n\t\tatomic.AddInt32(&count, 1)\n\t}\n\tsubj := fmt.Sprintf(accConnsEventSubjNew, pub)\n\tsub, err := sa.sysSubscribe(subj, cb)\n\tif sub == nil || err != nil {\n\t\tt.Fatalf(\"Expected to subscribe, got %v\", err)\n\t}\n\tdefer sa.sysUnsubscribe(sub)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc := natsConnect(t, url, createUserCreds(t, sb, akp))\n\tdefer nc.Close()\n\n\ttime.Sleep(500 * time.Millisecond)\n\t// After waiting 500ms with HB interval of 50ms, we should get\n\t// about 10 updates, no much more\n\tif n := atomic.LoadInt32(&count); n > 15 {\n\t\tt.Fatalf(\"Expected about 10 updates, got %v\", n)\n\t}\n\n\t// Now lookup the account doing the events on sb.\n\tacc, _ := sb.LookupAccount(pub)\n\t// Make sure we have the timer running.\n\tacc.mu.RLock()\n\tctmr := acc.ctmr\n\tacc.mu.RUnlock()\n\tif ctmr == nil {\n\t\tt.Fatalf(\"Expected event timer for acc conns to be running\")\n\t}\n\n\tnc.Close()\n\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t// Make sure we have the timer is NOT running.\n\t\tacc.mu.RLock()\n\t\tctmr = acc.ctmr\n\t\tacc.mu.RUnlock()\n\t\tif ctmr != nil {\n\t\t\treturn fmt.Errorf(\"Expected event timer for acc conns to NOT be running after reaching zero local clients\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestServerEventsReceivedByQSubs(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\t// Listen for auth error events.\n\tqsub, _ := ncs.QueueSubscribeSync(\"$SYS.SERVER.*.CLIENT.AUTH.ERR\", \"queue\")\n\tdefer qsub.Unsubscribe()\n\n\tncs.Flush()\n\n\tnats.Connect(url, nats.Name(\"TEST BAD LOGIN\"))\n\n\tm, err := qsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Should have heard an auth error event\")\n\t}\n\tdem := DisconnectEventMsg{}\n\tif err := json.Unmarshal(m.Data, &dem); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling disconnect event message: %v\", err)\n\t}\n\tif dem.Reason != \"Authentication Failure\" {\n\t\tt.Fatalf(\"Expected auth error, got %q\", dem.Reason)\n\t}\n}\n\nfunc TestServerEventsFilteredByTag(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: -1\n\t\tserver_name: srv-A\n\t\tserver_tags: [\"foo\", \"bar\"]\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: -1\n\t\t\tno_advertise: true\n\t\t}\n\t\tsystem_account: SYS\n\t\taccounts: {\n\t\t\tSYS: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: b, password: b}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tno_auth_user: b\n `))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\tserver_name: srv-B\n\t\tserver_tags: [\"bar\", \"baz\"]\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: -1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n\t\tsystem_account: SYS\n\t\taccounts: {\n\t\t\tSYS: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: b, password: b}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tno_auth_user: b\n `, sA.opts.Cluster.Port)))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\tcheckClusterFormed(t, sA, sB)\n\tnc := natsConnect(t, sA.ClientURL())\n\tdefer nc.Close()\n\n\tib := nats.NewInbox()\n\treq := func(tags ...string) {\n\t\tt.Helper()\n\t\tr, err := json.Marshal(VarzEventOptions{EventFilterOptions: EventFilterOptions{Tags: tags}})\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(fmt.Sprintf(serverPingReqSubj, \"VARZ\"), ib, r)\n\t\trequire_NoError(t, err)\n\t}\n\n\tmsgs := make(chan *nats.Msg, 10)\n\tdefer close(msgs)\n\t_, err := nc.ChanSubscribe(ib, msgs)\n\trequire_NoError(t, err)\n\treq(\"none\")\n\tselect {\n\tcase <-msgs:\n\t\tt.Fatalf(\"no message expected\")\n\tcase <-time.After(200 * time.Millisecond):\n\t}\n\treq(\"foo\")\n\tm := <-msgs\n\trequire_Contains(t, string(m.Data), \"srv-A\", \"foo\", \"bar\")\n\treq(\"foo\", \"bar\")\n\tm = <-msgs\n\trequire_Contains(t, string(m.Data), \"srv-A\", \"foo\", \"bar\")\n\treq(\"baz\")\n\tm = <-msgs\n\trequire_Contains(t, string(m.Data), \"srv-B\", \"bar\", \"baz\")\n\treq(\"bar\")\n\tm1 := <-msgs\n\tm2 := <-msgs\n\trequire_Contains(t, string(m1.Data)+string(m2.Data), \"srv-A\", \"srv-B\", \"foo\", \"bar\", \"baz\")\n\trequire_Len(t, len(msgs), 0)\n}\n\nfunc TestServerUnstableEventFilterMatch(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: -1\n\t\tserver_name: srv1\n\t\tserver_tags: [\"foo\", \"bar\"]\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: -1\n\t\t\tno_advertise: true\n\t\t}\n\t\tsystem_account: SYS\n\t\taccounts: {\n\t\t\tSYS: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: b, password: b}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tno_auth_user: b\n `))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\tserver_name: srv10\n\t\tserver_tags: [\"bar\", \"baz\"]\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: -1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n\t\tsystem_account: SYS\n\t\taccounts: {\n\t\t\tSYS: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: b, password: b}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tno_auth_user: b\n `, sA.opts.Cluster.Port)))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\tcheckClusterFormed(t, sA, sB)\n\n\ttester := func(t *testing.T, nc *nats.Conn, name string, count int) {\n\t\tt.Helper()\n\n\t\tr, err := json.Marshal(VarzEventOptions{EventFilterOptions: EventFilterOptions{Name: name, ExactMatch: true}})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < count; i++ {\n\t\t\tres, err := nc.Request(fmt.Sprintf(serverPingReqSubj, \"VARZ\"), r, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar vz ServerAPIVarzResponse\n\t\t\terr = json.Unmarshal(res.Data, &vz)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif vz.Server.Name != name {\n\t\t\t\tt.Fatalf(\"Expected server name to be %q, got %q\", name, vz.Server.Name)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Connects to srv1 so it's most likely to respond first while we are asking srv10 to respond.\n\tnc := natsConnect(t, sA.ClientURL())\n\tdefer nc.Close()\n\ttester(t, nc, \"srv10\", 10)\n\n\t// Connects to srv10 so it's most likely to respond first while we are asking srv1 to respond.\n\tnc.Close()\n\tnc = natsConnect(t, sB.ClientURL())\n\tdefer nc.Close()\n\ttester(t, nc, \"srv1\", 10)\n}\n\n// https://github.com/nats-io/nats-server/issues/3177\nfunc TestServerEventsAndDQSubscribers(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterAccountsTempl, \"DDQ\", 3)\n\tdefer c.shutdown()\n\n\tnc, err := nats.Connect(c.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.QueueSubscribeSync(\"$SYS.ACCOUNT.*.DISCONNECT\", \"qq\")\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\t// Create and disconnect 10 random connections.\n\tfor i := 0; i < 10; i++ {\n\t\tnc, err := nats.Connect(c.randomServer().ClientURL())\n\t\trequire_NoError(t, err)\n\t\tnc.Close()\n\t}\n\n\tcheckSubsPending(t, sub, 10)\n}\n\nfunc TestServerEventsStatszSingleServer(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts { $SYS { users [{user: \"admin\", password: \"p1d\"}]} }\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Grab internal system client.\n\ts.mu.RLock()\n\tsysc := s.sys.client\n\twait := s.sys.cstatsz + 25*time.Millisecond\n\ts.mu.RUnlock()\n\n\t// Wait for when first statsz would have gone out..\n\ttime.Sleep(wait)\n\n\tsysc.mu.Lock()\n\toutMsgs := sysc.stats.outMsgs\n\tsysc.mu.Unlock()\n\n\trequire_True(t, outMsgs == 0)\n\n\t// Connect as a system user and make sure if there is\n\t// subscription interest that we will receive updates.\n\tnc, _ := jsClientConnect(t, s, nats.UserInfo(\"admin\", \"p1d\"))\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(serverStatsSubj, \"*\"))\n\trequire_NoError(t, err)\n\n\tcheckSubsPending(t, sub, 1)\n}\n\nfunc TestServerEventsReload(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts: {\n\t\t\t$SYS { users [{user: \"admin\", password: \"p1d\"}]}\n\t\t\ttest { users [{user: \"foo\", password: \"bar\"}]}\n\t\t}\n\t\tping_interval: \"100ms\"\n\t`))\n\topts := LoadConfig(conf)\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\tsubject := fmt.Sprintf(serverReloadReqSubj, s.info.ID)\n\n\t// Connect as a test user and make sure the reload endpoint is not\n\t// accessible.\n\tncTest, _ := jsClientConnect(t, s, nats.UserInfo(\"foo\", \"bar\"))\n\tdefer ncTest.Close()\n\ttestReply := ncTest.NewRespInbox()\n\tsub, err := ncTest.SubscribeSync(testReply)\n\trequire_NoError(t, err)\n\terr = ncTest.PublishRequest(subject, testReply, nil)\n\trequire_NoError(t, err)\n\t_, err = sub.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\trequire_True(t, s.getOpts().PingInterval == 100*time.Millisecond)\n\n\t// Connect as a system user.\n\tnc, _ := jsClientConnect(t, s, nats.UserInfo(\"admin\", \"p1d\"))\n\tdefer nc.Close()\n\n\t// rewrite the config file with a different ping interval\n\terr = os.WriteFile(conf, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts: {\n\t\t\t$SYS { users [{user: \"admin\", password: \"p1d\"}]}\n\t\t\ttest { users [{user: \"foo\", password: \"bar\"}]}\n\t\t}\n\t\tping_interval: \"200ms\"\n\t`), 0666)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(subject, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar apiResp = ServerAPIResponse{}\n\terr = json.Unmarshal(msg.Data, &apiResp)\n\trequire_NoError(t, err)\n\n\trequire_True(t, apiResp.Data == nil)\n\trequire_True(t, apiResp.Error == nil)\n\n\t// See that the ping interval has changed.\n\trequire_True(t, s.getOpts().PingInterval == 200*time.Millisecond)\n\n\t// rewrite the config file with a different ping interval\n\terr = os.WriteFile(conf, []byte(`garbage and nonsense`), 0666)\n\trequire_NoError(t, err)\n\n\t// Request the server to reload and wait for the response.\n\tmsg, err = nc.Request(subject, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tapiResp = ServerAPIResponse{}\n\terr = json.Unmarshal(msg.Data, &apiResp)\n\trequire_NoError(t, err)\n\n\trequire_True(t, apiResp.Data == nil)\n\trequire_Error(t, apiResp.Error, fmt.Errorf(\"Parse error on line 1: 'Expected a top-level value to end with a new line, comment or EOF, but got 'n' instead.'\"))\n\n\t// See that the ping interval has not changed.\n\trequire_True(t, s.getOpts().PingInterval == 200*time.Millisecond)\n}\n\nfunc TestServerEventsLDMKick(t *testing.T) {\n\tldmed := make(chan bool, 1)\n\tdisconnected := make(chan bool, 1)\n\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncs.Close()\n\n\t_, akp2 := createAccount(s)\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name(\"TEST EVENTS LDM+KICK\"), nats.LameDuckModeHandler(func(_ *nats.Conn) { ldmed <- true }))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { disconnected <- true })\n\n\tcid, err := nc.GetClientID()\n\tif err != nil {\n\t\tt.Fatalf(\"Error on getting the CID: %v\", err)\n\t}\n\n\treqldm := LDMClientReq{CID: cid}\n\treqldmpayload, _ := json.Marshal(reqldm)\n\treqkick := KickClientReq{CID: cid}\n\treqkickpayload, _ := json.Marshal(reqkick)\n\n\t// Test for data races when getting the client by ID\n\tuc := createUserCreds(t, s, akp2)\n\ttotalClients := 100\n\tsomeClients := make([]*nats.Conn, 0, totalClients)\n\tfor i := 0; i < totalClients; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL(), uc)\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t\tsomeClients = append(someClients, nc)\n\t}\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; i < totalClients; i++ {\n\t\t\tsomeClients[i].Close()\n\t\t}\n\t}()\n\tdefer wg.Wait()\n\n\t_, err = ncs.Request(fmt.Sprintf(\"$SYS.REQ.SERVER.%s.LDM\", s.ID()), reqldmpayload, time.Second)\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase <-ldmed:\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"timeout waiting for the connection to receive the LDM signal\")\n\t}\n\n\t_, err = ncs.Request(fmt.Sprintf(\"$SYS.REQ.SERVER.%s.KICK\", s.ID()), reqkickpayload, time.Second)\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"timeout waiting for the client to get disconnected\")\n\t}\n}\n\nfunc Benchmark_GetHash(b *testing.B) {\n\tb.StopTimer()\n\t// Get 100 random names\n\tnames := make([]string, 0, 100)\n\tfor i := 0; i < 100; i++ {\n\t\tnames = append(names, nuid.Next())\n\t}\n\thashes := make([]string, 0, 100)\n\tfor j := 0; j < 100; j++ {\n\t\tsha := sha256.New()\n\t\tsha.Write([]byte(names[j]))\n\t\tb := sha.Sum(nil)\n\t\tfor i := 0; i < 8; i++ {\n\t\t\tb[i] = digits[int(b[i]%base)]\n\t\t}\n\t\thashes = append(hashes, string(b[:8]))\n\t}\n\twg := sync.WaitGroup{}\n\twg.Add(8)\n\terrCh := make(chan error, 8)\n\tb.StartTimer()\n\tfor i := 0; i < 8; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tidx := rand.Intn(100)\n\t\t\t\tif h := getHash(names[idx]); h != hashes[idx] {\n\t\t\t\t\terrCh <- fmt.Errorf(\"Hash for name %q was %q, but should be %q\", names[idx], h, hashes[idx])\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tselect {\n\tcase err := <-errCh:\n\t\tb.Fatal(err.Error())\n\tdefault:\n\t}\n}\n\nfunc TestClusterSetupMsgs(t *testing.T) {\n\t// Tests will set this general faster, but here we want default for production.\n\toriginal := statszRateLimit\n\tstatszRateLimit = defaultStatszRateLimit\n\tdefer func() { statszRateLimit = original }()\n\n\tnumServers := 10\n\tc := createClusterEx(t, false, 0, false, \"cluster\", numServers)\n\tdefer shutdownCluster(c)\n\n\tcheckFor(t, 3*time.Second, 500*time.Millisecond, func() error {\n\t\tvar totalOut int\n\t\tfor _, server := range c.servers {\n\t\t\ttotalOut += int(atomic.LoadInt64(&server.outMsgs))\n\t\t}\n\t\ttotalExpected := numServers * numServers\n\t\tif totalOut >= totalExpected {\n\t\t\treturn fmt.Errorf(\"Total outMsgs is %d, expected < %d\\n\", totalOut, totalExpected)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestServerEventsProfileZNotBlockingRecvQ(t *testing.T) {\n\tsa, _, sb, optsB, akp := runTrustedCluster(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\t// For this test, we will run a single server because the profiling\n\t// would fail for the second server since it would detect that\n\t// one profiling is already running (2 servers, but same process).\n\tsa.Shutdown()\n\turl := fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, sb, akp))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\t// We need to wait a little bit for the $SYS.SERVER.ACCOUNT.%s.CONNS\n\t// event to be pushed in the mux'ed queue.\n\ttime.Sleep(300 * time.Millisecond)\n\n\tpo := ProfilezOptions{Name: \"cpu\", Duration: 1 * time.Second}\n\treq, err := json.Marshal(po)\n\trequire_NoError(t, err)\n\n\ttestReq := func(t *testing.T, subj string) {\n\t\t// Block the recvQ by locking it for the duration of this test.\n\t\tsb.mu.RLock()\n\t\trecvq := sb.sys.recvq\n\t\tsb.mu.RUnlock()\n\t\trecvq.Lock()\n\t\tdefer recvq.Unlock()\n\n\t\t// Send the profilez request on the given subject.\n\t\treply := nc.NewRespInbox()\n\t\tsub := natsSubSync(t, nc, reply)\n\t\tnc.PublishRequest(subj, reply, req)\n\t\tmsg := natsNexMsg(t, sub, 10*time.Second)\n\t\tif len(msg.Data) == 0 {\n\t\t\tt.Fatal(\"Unexpected empty response\")\n\t\t}\n\t\t// Make sure its a ServerAPIResponse\n\t\tresp := ServerAPIResponse{Data: &ProfilezStatus{}}\n\t\terr := json.Unmarshal(msg.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\t// Check profile status to make sure that we got something.\n\t\tps := resp.Data.(*ProfilezStatus)\n\t\tif ps.Error != _EMPTY_ {\n\t\t\tt.Fatalf(\"%s\", ps.Error)\n\t\t}\n\t\trequire_True(t, len(ps.Profile) > 0)\n\t}\n\tconst profilez = \"PROFILEZ\"\n\tfor _, test := range []struct {\n\t\tname string\n\t\tf func() string\n\t}{\n\t\t{\"server profilez request subject\", func() string { return fmt.Sprintf(serverPingReqSubj, profilez) }},\n\t\t{\"server direct request subject\", func() string { return fmt.Sprintf(serverDirectReqSubj, sb.ID(), profilez) }},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttestReq(t, test.f())\n\t\t})\n\t}\n}\n\nfunc TestServerEventsStatsZJetStreamApiLevel(t *testing.T) {\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\ts.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()})\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\trequire_NoError(t, err)\n\tdefer ncs.Close()\n\n\tmsg, err := ncs.Request(\"$SYS.REQ.SERVER.PING.STATSZ\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar stats ServerStatsMsg\n\terr = json.Unmarshal(msg.Data, &stats)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, stats.Stats.JetStream.Stats.API.Level, JSApiLevel)\n}\n\nfunc TestServerEventsPingStatsSlowConsumersStats(t *testing.T) {\n\ts, _ := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\tncs, err := nats.Connect(s.ClientURL(), createUserCreds(t, s, akp))\n\trequire_NoError(t, err)\n\tdefer ncs.Close()\n\n\tconst statsz = \"STATSZ\"\n\tfor _, test := range []struct {\n\t\tname string\n\t\tf func() string\n\t\texpectTwo bool\n\t}{\n\t\t{\"server stats ping request subject\", func() string { return serverStatsPingReqSubj }, true},\n\t\t{\"server ping request subject\", func() string { return fmt.Sprintf(serverPingReqSubj, statsz) }, true},\n\t\t{\"server direct request subject\", func() string { return fmt.Sprintf(serverDirectReqSubj, s.ID(), statsz) }, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Clear all slow consumers values\n\t\t\ts.scStats.clients.Store(0)\n\t\t\ts.scStats.routes.Store(0)\n\t\t\ts.scStats.gateways.Store(0)\n\t\t\ts.scStats.leafs.Store(0)\n\n\t\t\tmsg, err := ncs.Request(test.f(), nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar ssm ServerStatsMsg\n\t\t\terr = json.Unmarshal(msg.Data, &ssm)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// No slow consumer stats, so should be nil\n\t\t\trequire_True(t, ssm.Stats.SlowConsumersStats == nil)\n\n\t\t\t// Now set some values\n\t\t\ts.scStats.clients.Store(1)\n\t\t\ts.scStats.routes.Store(2)\n\t\t\ts.scStats.gateways.Store(3)\n\t\t\ts.scStats.leafs.Store(4)\n\n\t\t\tmsg, err = ncs.Request(test.f(), nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tssm = ServerStatsMsg{}\n\t\t\terr = json.Unmarshal(msg.Data, &ssm)\n\t\t\trequire_NoError(t, err)\n\n\t\t\trequire_NotNil(t, ssm.Stats.SlowConsumersStats)\n\t\t\tscs := ssm.Stats.SlowConsumersStats\n\t\t\trequire_Equal(t, scs.Clients, 1)\n\t\t\trequire_Equal(t, scs.Routes, 2)\n\t\t\trequire_Equal(t, scs.Gateways, 3)\n\t\t\trequire_Equal(t, scs.Leafs, 4)\n\t\t})\n\t}\n}\n\nfunc TestServerEventsPingStatsStaleConnectionStats(t *testing.T) {\n\ttempl := `\n listen: \"127.0.0.1:-1\"\n system_account = sys\n accounts {\n a {\n users = [{ user: a, pass: a }]\n }\n b {\n users = [{ user: b, pass: b }]\n }\n sys {\n users = [{ user: sys, pass: sys }]\n }\n }\n `\n\tconf := createConfFile(t, []byte(templ))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// 3 different ways to get statz:\n\tconst statsz = \"STATSZ\"\n\n\t// $SYS.REQ.SERVER.NCBT6MBA7Q7ZF4R4WVXTSZTCZDPPXX2ALLN3XM75VCBCNMPIKQFPFLKV.STATSZ\n\tdirReqSubject := func(s *Server) string {\n\t\treturn fmt.Sprintf(serverDirectReqSubj, s.ID(), statsz)\n\t}\n\t// $SYS.REQ.SERVER.PING.STATSZ\n\tpingReqSubject := func(s *Server) string {\n\t\treturn fmt.Sprintf(serverPingReqSubj, statsz)\n\t}\n\t// $SYS.REQ.SERVER.PING\n\tstatsPingSubject := func(s *Server) string {\n\t\treturn serverStatsPingReqSubj\n\t}\n\tsubjects := []string{dirReqSubject(s), pingReqSubject(s), statsPingSubject(s)}\n\n\tncs, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"sys\", \"sys\"))\n\trequire_NoError(t, err)\n\n\tfor _, subject := range subjects {\n\t\tmsg, err := ncs.Request(subject, nil, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar ssm ServerStatsMsg\n\t\terr = json.Unmarshal(msg.Data, &ssm)\n\t\trequire_NoError(t, err)\n\n\t\t// No stale connection stats.\n\t\trequire_True(t, ssm.Stats.StaleConnectionStats == nil)\n\t\trequire_Equal(t, s.NumStaleConnections(), int64(0))\n\t\trequire_Equal(t, s.NumStaleConnectionsClients(), uint64(0))\n\t\trequire_Equal(t, s.NumStaleConnectionsRoutes(), uint64(0))\n\t\trequire_Equal(t, s.NumStaleConnectionsGateways(), uint64(0))\n\t\trequire_Equal(t, s.NumStaleConnectionsLeafs(), uint64(0))\n\t}\n\n\t// Set some values and confirm.\n\ts.staleStats.clients.Store(1)\n\ts.staleStats.routes.Store(2)\n\ts.staleStats.gateways.Store(3)\n\ts.staleStats.leafs.Store(4)\n\tatomic.StoreInt64(&s.staleConnections, 10)\n\trequire_Equal(t, s.NumStaleConnections(), int64(10))\n\trequire_Equal(t, s.NumStaleConnectionsClients(), uint64(1))\n\trequire_Equal(t, s.NumStaleConnectionsRoutes(), uint64(2))\n\trequire_Equal(t, s.NumStaleConnectionsGateways(), uint64(3))\n\trequire_Equal(t, s.NumStaleConnectionsLeafs(), uint64(4))\n\n\tfor _, subject := range subjects {\n\t\tmsg, err := ncs.Request(subject, nil, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tssm := ServerStatsMsg{}\n\t\terr = json.Unmarshal(msg.Data, &ssm)\n\t\trequire_NoError(t, err)\n\n\t\trequire_NotNil(t, ssm.Stats.StaleConnectionStats)\n\t\tstcs := ssm.Stats.StaleConnectionStats\n\t\trequire_Equal(t, stcs.Clients, 1)\n\t\trequire_Equal(t, stcs.Routes, 2)\n\t\trequire_Equal(t, stcs.Gateways, 3)\n\t\trequire_Equal(t, stcs.Leafs, 4)\n\t\trequire_Equal(t, ssm.Stats.StaleConnections, int64(10))\n\t}\n}\n\nfunc TestServerEventsStatszMaxProcsMemLimit(t *testing.T) {\n\t// We want to prove that our set values are reflected in STATSZ,\n\t// so we can't use constants that might match the system that\n\t// the test is run on.\n\tomp, omm := runtime.GOMAXPROCS(-1), debug.SetMemoryLimit(-1)\n\tmp, mm := runtime.GOMAXPROCS(omp*2)*2, debug.SetMemoryLimit(omm/2)/2\n\n\t// When we're done, put everything back.\n\tdefer runtime.GOMAXPROCS(omp)\n\tdefer debug.SetMemoryLimit(omm)\n\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(s)\n\ts.setSystemAccount(acc)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, createUserCreds(t, s, akp))\n\trequire_NoError(t, err)\n\tdefer ncs.Close()\n\n\tmsg, err := ncs.Request(\"$SYS.REQ.SERVER.PING.STATSZ\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar stats ServerStatsMsg\n\trequire_NoError(t, json.Unmarshal(msg.Data, &stats))\n\trequire_Equal(t, stats.Stats.MaxProcs, mp)\n\trequire_Equal(t, stats.Stats.MemLimit, mm)\n}\n\nfunc TestSubszPagination(t *testing.T) {\n\ttype subszResp struct {\n\t\tSubsz Subsz `json:\"data\"`\n\t\tServer ServerInfo `json:\"server\"`\n\t}\n\ts, opts := runTrustedServer(t)\n\tdefer s.Shutdown()\n\n\tsysAcc, sysAkp := createAccount(s)\n\ts.setSystemAccount(sysAcc)\n\n\tacc, akp := createAccount(s)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncSys, err := nats.Connect(url, createUserCreds(t, s, sysAkp))\n\trequire_NoError(t, err)\n\tdefer ncSys.Close()\n\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Create 100 subscriptions.\n\tfor i := range 100 {\n\t\tnc.Subscribe(fmt.Sprintf(\"foo.%d\", i), func(_ *nats.Msg) {})\n\t}\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\treqSubject := fmt.Sprintf(accDirectReqSubj, acc.Name, \"SUBSZ\")\n\n\t// Request the first page.\n\tsubszReq := SubszOptions{Subscriptions: true, Limit: 10}\n\treq, _ := json.Marshal(subszReq)\n\tmsg, err := ncSys.Request(reqSubject, req, time.Second)\n\trequire_NoError(t, err)\n\n\tvar subsz subszResp\n\trequire_NoError(t, json.Unmarshal(msg.Data, &subsz))\n\trequire_Equal(t, len(subsz.Subsz.Subs), 10)\n\n\t// we cannot check for equality since we have to account for the monitoring subscriptions\n\tif subsz.Subsz.Total < 100 || subsz.Subsz.Total > 110 {\n\t\tt.Fatalf(\"Expected total subscriptions to be more than 100 and less than 110, got %d\", subsz.Subsz.Total)\n\t}\n\n\t// Now test with a sub filter\n\n\t// create 10 subs on \"bar.*\"\n\tfor range 10 {\n\t\tnc.Subscribe(\"bar.*\", func(_ *nats.Msg) {})\n\t}\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tsubszReq = SubszOptions{Subscriptions: true, Limit: 5, Test: \"bar.A\"}\n\treq, _ = json.Marshal(subszReq)\n\tmsg, err = ncSys.Request(reqSubject, req, time.Second)\n\trequire_NoError(t, err)\n\n\tvar subszFiltered subszResp\n\trequire_NoError(t, json.Unmarshal(msg.Data, &subszFiltered))\n\trequire_Equal(t, len(subszFiltered.Subsz.Subs), 5)\n\trequire_Equal(t, subszFiltered.Subsz.Total, 10)\n}\n\nfunc TestServerEventsConnectDisconnectForGlobalAcc(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\t$SYS {\n\t\t\t\tusers [{user: \"admin\", password: \"pwd\"}]\n\t\t\t}\n\t\t}\n\t`))\n\tdefer os.Remove(conf)\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tncs, err := nats.Connect(url, nats.UserInfo(\"admin\", \"pwd\"))\n\trequire_NoError(t, err)\n\tdefer ncs.Close()\n\n\ts1, err := ncs.SubscribeSync(fmt.Sprintf(connectEventSubj, globalAccountName))\n\trequire_NoError(t, err)\n\ts2, err := ncs.SubscribeSync(fmt.Sprintf(disconnectEventSubj, globalAccountName))\n\trequire_NoError(t, err)\n\n\t// Flush to make sure subscriptions are established\n\trequire_NoError(t, ncs.Flush())\n\n\t// Connect to global account\n\tncg, err := nats.Connect(url, nats.UserInfo(\"\", \"\"))\n\trequire_NoError(t, err)\n\n\t// System account should get a connect event\n\tmsg, err := s1.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Subject, fmt.Sprintf(connectEventSubj, globalAccountName))\n\n\t// Disconnect from global account\n\tncg.Close()\n\n\t// System account should get a disconnect event\n\tmsg, err = s2.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Subject, fmt.Sprintf(disconnectEventSubj, globalAccountName))\n}\n" }, { "path": "server/filestore.go", "content": "// Copyright 2019-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"archive/tar\"\n\t\"bytes\"\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"math\"\n\tmrand \"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"slices\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/antithesishq/antithesis-sdk-go/assert\"\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/minio/highwayhash\"\n\t\"github.com/nats-io/nats-server/v2/server/ats\"\n\t\"github.com/nats-io/nats-server/v2/server/avl\"\n\t\"github.com/nats-io/nats-server/v2/server/elastic\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nats-server/v2/server/stree\"\n\t\"github.com/nats-io/nats-server/v2/server/thw\"\n\t\"golang.org/x/crypto/chacha20\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n)\n\ntype FileStoreConfig struct {\n\t// Where the parent directory for all storage will be located.\n\tStoreDir string\n\t// BlockSize is the file block size. This also represents the maximum overhead size.\n\tBlockSize uint64\n\t// CacheExpire is how long with no activity until we expire the cache.\n\tCacheExpire time.Duration\n\t// SubjectStateExpire is how long with no activity until we expire a msg block's subject state.\n\tSubjectStateExpire time.Duration\n\t// SyncInterval is how often we sync to disk in the background.\n\tSyncInterval time.Duration\n\t// SyncAlways is when the stream should sync all data writes.\n\tSyncAlways bool\n\t// AsyncFlush allows async flush to batch write operations.\n\tAsyncFlush bool\n\t// Cipher is the cipher to use when encrypting.\n\tCipher StoreCipher\n\t// Compression is the algorithm to use when compressing.\n\tCompression StoreCompression\n\n\t// Internal reference to our server.\n\tsrv *Server\n}\n\n// FileStreamInfo allows us to remember created time.\ntype FileStreamInfo struct {\n\tCreated time.Time\n\tStreamConfig\n}\n\ntype StoreCipher int\n\nconst (\n\tChaCha StoreCipher = iota\n\tAES\n\tNoCipher\n)\n\nfunc (cipher StoreCipher) String() string {\n\tswitch cipher {\n\tcase ChaCha:\n\t\treturn \"ChaCha20-Poly1305\"\n\tcase AES:\n\t\treturn \"AES-GCM\"\n\tcase NoCipher:\n\t\treturn \"None\"\n\tdefault:\n\t\treturn \"Unknown StoreCipher\"\n\t}\n}\n\ntype StoreCompression uint8\n\nconst (\n\tNoCompression StoreCompression = iota\n\tS2Compression\n)\n\nfunc (alg StoreCompression) String() string {\n\tswitch alg {\n\tcase NoCompression:\n\t\treturn \"None\"\n\tcase S2Compression:\n\t\treturn \"S2\"\n\tdefault:\n\t\treturn \"Unknown StoreCompression\"\n\t}\n}\n\nfunc (alg StoreCompression) MarshalJSON() ([]byte, error) {\n\tvar str string\n\tswitch alg {\n\tcase S2Compression:\n\t\tstr = \"s2\"\n\tcase NoCompression:\n\t\tstr = \"none\"\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown compression algorithm\")\n\t}\n\treturn json.Marshal(str)\n}\n\nfunc (alg *StoreCompression) UnmarshalJSON(b []byte) error {\n\tvar str string\n\tif err := json.Unmarshal(b, &str); err != nil {\n\t\treturn err\n\t}\n\tswitch str {\n\tcase \"s2\":\n\t\t*alg = S2Compression\n\tcase \"none\":\n\t\t*alg = NoCompression\n\tdefault:\n\t\treturn fmt.Errorf(\"unknown compression algorithm\")\n\t}\n\treturn nil\n}\n\n// File ConsumerInfo is used for creating consumer stores.\ntype FileConsumerInfo struct {\n\tCreated time.Time\n\tName string\n\tConsumerConfig\n}\n\n// Default file and directory permissions.\nconst (\n\tdefaultDirPerms = os.FileMode(0700)\n\tdefaultFilePerms = os.FileMode(0600)\n)\n\ntype psi struct {\n\ttotal uint64\n\tfblk uint32\n\tlblk uint32\n}\n\ntype fileStore struct {\n\tsrv *Server\n\tmu sync.RWMutex\n\tstate StreamState\n\ttombs []uint64\n\tld *LostStreamData\n\tscb StorageUpdateHandler\n\trmcb StorageRemoveMsgHandler\n\tpmsgcb ProcessJetStreamMsgHandler\n\tageChk *time.Timer // Timer to expire messages.\n\tageChkRun bool // Whether message expiration is currently running.\n\tageChkTime int64 // When the message expiration is scheduled to run.\n\tsyncTmr *time.Timer\n\tcfg FileStreamInfo\n\tfcfg FileStoreConfig\n\tprf keyGen\n\toldprf keyGen\n\taek cipher.AEAD\n\tlmb *msgBlock\n\tblks []*msgBlock\n\tbim map[uint32]*msgBlock\n\tpsim *stree.SubjectTree[psi]\n\ttsl int\n\twfsmu sync.Mutex // Only one writeFullState at a time to protect from overwrites.\n\twfsrun atomic.Int64 // Is writeFullState already running? For timer check only\n\twfsadml int // writeFullState average dmap length, protected by wfsmu.\n\thh *highwayhash.Digest64\n\tqch chan struct{}\n\tfsld chan struct{}\n\tcmu sync.RWMutex\n\tcfs []ConsumerStore\n\twerr error\n\tsips int\n\tdirty int\n\tclosing bool\n\tclosed atomic.Bool // Atomic to reduce contention on ConsumerStores.\n\tfip bool\n\treceivedAny bool\n\tfirstMoved bool\n\tttls *thw.HashWheel\n\tscheduling *MsgScheduling\n\tsdm *SDMMeta\n\tlpex time.Time // Last PurgeEx call.\n}\n\n// Represents a message store block and its data.\ntype msgBlock struct {\n\t// Here for 32bit systems and atomic.\n\tfirst msgId\n\tlast msgId\n\tmu sync.RWMutex\n\tfs *fileStore\n\taek cipher.AEAD\n\tbek cipher.Stream\n\tseed []byte\n\tnonce []byte\n\tmfn string\n\tmfd *os.File\n\tcmp StoreCompression // Effective compression at the time of loading the block\n\tliwsz int64\n\tindex uint32\n\tbytes uint64 // User visible bytes count.\n\trbytes uint64 // Total bytes (raw) including deleted. Used for rolling to new blk.\n\tcbytes uint64 // Bytes count after last compaction. 0 if no compaction happened yet.\n\tmsgs uint64 // User visible message count.\n\tfss *stree.SubjectTree[SimpleState]\n\tkfn string\n\tlwts int64\n\tllts int64\n\tlrts int64\n\tlsts int64\n\tllseq uint64\n\thh *highwayhash.Digest64\n\tecache elastic.Pointer[cache]\n\tcache *cache\n\tcloads uint64\n\tcexp time.Duration\n\tfexp time.Duration\n\tctmr *time.Timer\n\twerr error\n\tdmap avl.SequenceSet\n\tfch chan struct{}\n\tqch chan struct{}\n\tlchk [8]byte\n\tloading bool\n\tflusher bool\n\tnoTrack bool\n\tneedSync bool\n\tsyncAlways bool\n\tnoCompact bool\n\tclosed bool\n\tttls uint64 // How many msgs have TTLs?\n\tschedules uint64 // How many msgs have schedules?\n\n\t// Used to mock write failures.\n\tmockWriteErr bool\n}\n\n// Write through caching layer that is also used on loading messages.\ntype cache struct {\n\tbuf []byte\n\twp int\n\tidx []uint32\n\tfseq uint64\n\tnra bool\n}\n\ntype msgId struct {\n\tseq uint64\n\tts int64\n}\n\nconst (\n\t// Magic is used to identify the file store files.\n\tmagic = uint8(22)\n\t// Version\n\tversion = uint8(1)\n\t// New IndexInfo Version\n\tnewVersion = uint8(2)\n\t// hdrLen\n\thdrLen = 2\n\t// This is where we keep the streams.\n\tstreamsDir = \"streams\"\n\t// This is where we keep inflight batches for streams.\n\tbatchesDir = \"batches\"\n\t// This is where we keep the message store blocks.\n\tmsgDir = \"msgs\"\n\t// This is where we temporarily move the messages dir.\n\tpurgeDir = \"__msgs__\"\n\t// This is where we temporarily move the new message block during purge.\n\tnewMsgDir = \"__new_msgs__\"\n\t// used to scan blk file names.\n\tblkScan = \"%d.blk\"\n\t// suffix of a block file\n\tblkSuffix = \".blk\"\n\t// used for compacted blocks that are staged.\n\tnewScan = \"%d.new\"\n\t// used to scan index file names.\n\tindexScan = \"%d.idx\"\n\t// used to store our block encryption key.\n\tkeyScan = \"%d.key\"\n\t// to look for orphans\n\tkeyScanAll = \"*.key\"\n\t// This is where we keep state on consumers.\n\tconsumerDir = \"obs\"\n\t// Index file for a consumer.\n\tconsumerState = \"o.dat\"\n\t// The suffix that will be given to a new temporary block for compression or when rewriting the full file.\n\tblkTmpSuffix = \".tmp\"\n\t// default cache buffer expiration\n\tdefaultCacheBufferExpiration = 10 * time.Second\n\t// default sync interval\n\tdefaultSyncInterval = 2 * time.Minute\n\t// default idle timeout to close FDs.\n\tcloseFDsIdle = 30 * time.Second\n\t// default expiration time for mb.fss when idle.\n\tdefaultFssExpiration = 2 * time.Minute\n\t// coalesceMinimum\n\tcoalesceMinimum = 16 * 1024\n\t// maxFlushWait is maximum we will wait to gather messages to flush.\n\tmaxFlushWait = 8 * time.Millisecond\n\n\t// Metafiles for streams and consumers.\n\tJetStreamMetaFile = \"meta.inf\"\n\tJetStreamMetaFileSum = \"meta.sum\"\n\tJetStreamMetaFileKey = \"meta.key\"\n\n\t// This is the full snapshotted state for the stream.\n\tstreamStreamStateFile = \"index.db\"\n\n\t// This is the encoded time hash wheel for TTLs.\n\tttlStreamStateFile = \"thw.db\"\n\n\t// This is the encoded message scheduling file.\n\tmsgSchedulingStreamStateFile = \"sched.db\"\n\n\t// AEK key sizes\n\tminMetaKeySize = 64\n\tminBlkKeySize = 64\n\n\t// Default stream block size.\n\tdefaultLargeBlockSize = 8 * 1024 * 1024 // 8MB\n\t// Default for workqueue or interest based.\n\tdefaultMediumBlockSize = 4 * 1024 * 1024 // 4MB\n\t// For smaller reuse buffers. Usually being generated during contention on the lead write buffer.\n\t// E.g. mirrors/sources etc.\n\tdefaultSmallBlockSize = 1 * 1024 * 1024 // 1MB\n\t// NOT an actual block size, but used for the sync.Pools, so that we don't allocate huge buffers\n\t// unnecessarily until there are enough writes to justify it.\n\tdefaultTinyBlockSize = 1 * 1024 * 256 // 256KB\n\t// Maximum size for the encrypted head block.\n\tmaximumEncryptedBlockSize = 2 * 1024 * 1024 // 2MB\n\t// Default for KV based\n\tdefaultKVBlockSize = defaultMediumBlockSize\n\t// max block size for now.\n\tmaxBlockSize = defaultLargeBlockSize\n\t// Compact minimum threshold.\n\tcompactMinimum = 2 * 1024 * 1024 // 2MB\n\t// FileStoreMinBlkSize is minimum size we will do for a blk size.\n\tFileStoreMinBlkSize = 32 * 1000 // 32kib\n\t// FileStoreMaxBlkSize is maximum size we will do for a blk size.\n\tFileStoreMaxBlkSize = maxBlockSize\n\t// Check for bad record length value due to corrupt data.\n\trlBadThresh = 32 * 1024 * 1024\n\t// Checksum size for hash for msg records.\n\trecordHashSize = 8\n)\n\nfunc newFileStore(fcfg FileStoreConfig, cfg StreamConfig) (*fileStore, error) {\n\treturn newFileStoreWithCreated(fcfg, cfg, time.Now().UTC(), nil, nil)\n}\n\nfunc newFileStoreWithCreated(fcfg FileStoreConfig, cfg StreamConfig, created time.Time, prf, oldprf keyGen) (fs *fileStore, err error) {\n\tif cfg.Name == _EMPTY_ {\n\t\treturn nil, fmt.Errorf(\"name required\")\n\t}\n\tif cfg.Storage != FileStorage {\n\t\treturn nil, fmt.Errorf(\"fileStore requires file storage type in config\")\n\t}\n\t// Default values.\n\tif fcfg.BlockSize == 0 {\n\t\tfcfg.BlockSize = dynBlkSize(cfg.Retention, cfg.MaxBytes, prf != nil)\n\t}\n\tif fcfg.BlockSize > maxBlockSize {\n\t\treturn nil, fmt.Errorf(\"filestore max block size is %s\", friendlyBytes(maxBlockSize))\n\t}\n\tif fcfg.CacheExpire == 0 {\n\t\tfcfg.CacheExpire = defaultCacheBufferExpiration\n\t}\n\tif fcfg.SubjectStateExpire == 0 {\n\t\tfcfg.SubjectStateExpire = defaultFssExpiration\n\t}\n\tif fcfg.SyncInterval == 0 {\n\t\tfcfg.SyncInterval = defaultSyncInterval\n\t}\n\n\t// Check the directory\n\tif stat, err := os.Stat(fcfg.StoreDir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(fcfg.StoreDir, defaultDirPerms); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not create storage directory - %v\", err)\n\t\t}\n\t} else if stat == nil || !stat.IsDir() {\n\t\treturn nil, fmt.Errorf(\"storage directory is not a directory\")\n\t}\n\ttmpfile, err := os.CreateTemp(fcfg.StoreDir, \"_test_\")\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"storage directory is not writable\")\n\t}\n\n\ttmpfile.Close()\n\t<-dios\n\tos.Remove(tmpfile.Name())\n\tdios <- struct{}{}\n\n\tfs = &fileStore{\n\t\tfcfg: fcfg,\n\t\tpsim: stree.NewSubjectTree[psi](),\n\t\tbim: make(map[uint32]*msgBlock),\n\t\tcfg: FileStreamInfo{Created: created, StreamConfig: cfg},\n\t\tprf: prf,\n\t\toldprf: oldprf,\n\t\tqch: make(chan struct{}),\n\t\tfsld: make(chan struct{}),\n\t\tsrv: fcfg.srv,\n\t}\n\n\t// Register with access time service.\n\tats.Register()\n\n\t// If we error before completion make sure to cleanup.\n\tdefer func() {\n\t\tif err != nil {\n\t\t\tats.Unregister()\n\t\t}\n\t}()\n\n\t// Only create a THW if we're going to allow TTLs.\n\tif cfg.AllowMsgTTL {\n\t\tfs.ttls = thw.NewHashWheel()\n\t}\n\t// Only create scheduling data structure if we're going to allow message schedules.\n\tif cfg.AllowMsgSchedules {\n\t\tfs.scheduling = newMsgScheduling(fs.runMsgScheduling)\n\t}\n\n\t// Set flush in place to AsyncFlush which by default is false.\n\tfs.fip = !fcfg.AsyncFlush\n\n\t// Check if this is a new setup.\n\tmdir := filepath.Join(fcfg.StoreDir, msgDir)\n\todir := filepath.Join(fcfg.StoreDir, consumerDir)\n\tif err := os.MkdirAll(mdir, defaultDirPerms); err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create message storage directory - %v\", err)\n\t}\n\tif err := os.MkdirAll(odir, defaultDirPerms); err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create consumer storage directory - %v\", err)\n\t}\n\n\t// Create highway hash for message blocks. Use sha256 of directory as key.\n\tkey := sha256.Sum256([]byte(cfg.Name))\n\tfs.hh, err = highwayhash.NewDigest64(key[:])\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create hash: %v\", err)\n\t}\n\n\tkeyFile := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey)\n\t_, err = os.Stat(keyFile)\n\t// Either the file should exist (err=nil), or it shouldn't. Any other error is reported.\n\tif err != nil && !os.IsNotExist(err) {\n\t\treturn nil, err\n\t}\n\t// Make sure we do not have an encrypted store underneath of us but no main key.\n\tif fs.prf == nil && err == nil {\n\t\treturn nil, errNoMainKey\n\t} else if fs.prf != nil && err == nil {\n\t\t// If encryption is configured and the key file exists, recover our keys.\n\t\tif err = fs.recoverAEK(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Attempt to recover our state.\n\terr = fs.recoverFullState()\n\tif err != nil {\n\t\tif !os.IsNotExist(err) {\n\t\t\tfs.warn(\"Recovering stream state from index errored: %v\", err)\n\t\t}\n\t\t// Hold onto state\n\t\tprior := fs.state\n\t\t// Reset anything that could have been set from above.\n\t\tfs.state = StreamState{}\n\t\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\t\tfs.bim = make(map[uint32]*msgBlock)\n\t\tfs.blks = nil\n\t\tfs.tombs = nil\n\n\t\t// Recover our message state the old way\n\t\tif err := fs.recoverMsgs(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tfs.mu.Lock()\n\t\t// Check if our prior state remembers a last sequence past where we can see.\n\t\t// Unless we're async flushing, in which case this can happen if some blocks weren't flushed.\n\t\tif prior.LastSeq > fs.state.LastSeq && !fs.fcfg.AsyncFlush {\n\t\t\tif mb, err := fs.newMsgBlockForWrite(); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn nil, err\n\t\t\t} else if err = mb.writeTombstone(prior.LastSeq, prior.LastTime.UnixNano()); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tfs.state.LastSeq, fs.state.LastTime = prior.LastSeq, prior.LastTime\n\t\t\tif fs.state.Msgs == 0 {\n\t\t\t\tfs.state.FirstSeq = fs.state.LastSeq + 1\n\t\t\t\tfs.state.FirstTime = time.Time{}\n\t\t\t}\n\t\t}\n\t\t// Since we recovered here, make sure to kick ourselves to write out our stream state.\n\t\tfs.dirty++\n\t\tfs.mu.Unlock()\n\t}\n\n\t// Lock during the remainder of the recovery.\n\tfs.mu.Lock()\n\t// Use defer to ensure the lock is released if any of the enforcement operations\n\t// run into issues to avoid potential deadlocks on exit.\n\tunlocked := false\n\tdefer func() {\n\t\tif !unlocked {\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}()\n\n\t// See if we can bring back our TTL timed hash wheel state from disk.\n\tif cfg.AllowMsgTTL {\n\t\tif err = fs.recoverTTLState(); err != nil && !os.IsNotExist(err) {\n\t\t\tfs.warn(\"Recovering TTL state from index errored: %v\", err)\n\t\t}\n\t}\n\n\t// See if we can bring back our message scheduling state from disk.\n\tif cfg.AllowMsgSchedules {\n\t\tif err = fs.recoverMsgSchedulingState(); err != nil && !os.IsNotExist(err) {\n\t\t\tfs.warn(\"Recovering message scheduling state from index errored: %v\", err)\n\t\t}\n\t}\n\n\t// Also make sure we get rid of old idx and fss files on return.\n\t// Do this in separate go routine vs inline and at end of processing.\n\tdefer func() {\n\t\tif fs != nil {\n\t\t\tgo fs.cleanupOldMeta()\n\t\t}\n\t}()\n\n\t// Check if we have any left over tombstones to process.\n\tif len(fs.tombs) > 0 {\n\t\tfor _, seq := range fs.tombs {\n\t\t\t_, err = fs.removeMsg(seq, false, true, false)\n\t\t\tif err != nil && err != ErrStoreEOF && err != ErrStoreMsgNotFound {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tfs.removeFromLostData(seq)\n\t\t}\n\t\t// Not needed after this phase.\n\t\tfs.tombs = nil\n\t}\n\n\t// Limits checks and enforcement.\n\tif err = fs.enforceMsgLimit(); err != nil {\n\t\treturn nil, err\n\t}\n\tif err = fs.enforceBytesLimit(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Do age checks too, make sure to call in place.\n\tif fs.cfg.MaxAge != 0 {\n\t\tif err = fs.expireMsgsOnRecover(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfs.startAgeChk()\n\t}\n\n\t// If we have max msgs per subject make sure the is also enforced.\n\tif fs.cfg.MaxMsgsPer > 0 {\n\t\tif err = fs.enforceMsgPerSubjectLimit(false); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Grab first sequence for check below while we have lock.\n\tfirstSeq := fs.state.FirstSeq\n\tfs.mu.Unlock()\n\tunlocked = true\n\n\t// If the stream has an initial sequence number then make sure we\n\t// have purged up until that point. We will do this only if the\n\t// recovered first sequence number is before our configured first\n\t// sequence. Need to do this locked as by now the age check timer\n\t// has started.\n\tif cfg.FirstSeq > 0 && firstSeq < cfg.FirstSeq {\n\t\tif _, err := fs.purge(cfg.FirstSeq); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Write our meta data if it does not exist or is zero'd out.\n\tmeta := filepath.Join(fcfg.StoreDir, JetStreamMetaFile)\n\tfi, err := os.Stat(meta)\n\tif err != nil && os.IsNotExist(err) || fi != nil && fi.Size() == 0 {\n\t\tif err := fs.writeStreamMeta(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// If we expect to be encrypted check that what we are restoring is not plaintext.\n\t// This can happen on snapshot restores or conversions.\n\tif fs.prf != nil {\n\t\tif _, err := os.Stat(keyFile); err != nil && os.IsNotExist(err) {\n\t\t\tif err := fs.writeStreamMeta(); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Setup our sync timer.\n\tfs.setSyncTimer()\n\n\t// Spin up the go routine that will write out our full state stream index.\n\tgo fs.flushStreamStateLoop(fs.qch, fs.fsld)\n\n\treturn fs, nil\n}\n\n// Lock all existing message blocks.\n// Lock held on entry.\nfunc (fs *fileStore) lockAllMsgBlocks() {\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t}\n}\n\n// Unlock all existing message blocks.\n// Lock held on entry.\nfunc (fs *fileStore) unlockAllMsgBlocks() {\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Unlock()\n\t}\n}\n\nfunc (fs *fileStore) UpdateConfig(cfg *StreamConfig) error {\n\tstart := time.Now()\n\tdefer func() {\n\t\tif took := time.Since(start); took > time.Minute {\n\t\t\tfs.warn(\"UpdateConfig took %v\", took.Round(time.Millisecond))\n\t\t}\n\t}()\n\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\tif cfg.Name == _EMPTY_ {\n\t\treturn fmt.Errorf(\"name required\")\n\t}\n\tif cfg.Storage != FileStorage {\n\t\treturn fmt.Errorf(\"fileStore requires file storage type in config\")\n\t}\n\tif cfg.MaxMsgsPer < -1 {\n\t\tcfg.MaxMsgsPer = -1\n\t}\n\n\tfs.mu.Lock()\n\tnew_cfg := FileStreamInfo{Created: fs.cfg.Created, StreamConfig: *cfg}\n\told_cfg := fs.cfg\n\t// The reference story has changed here, so this full msg block lock\n\t// may not be needed.\n\tfs.lockAllMsgBlocks()\n\tfs.cfg = new_cfg\n\tfs.unlockAllMsgBlocks()\n\tif err := fs.writeStreamMeta(); err != nil {\n\t\tfs.lockAllMsgBlocks()\n\t\tfs.cfg = old_cfg\n\t\tfs.unlockAllMsgBlocks()\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\n\t// Create or delete the THW if needed.\n\tif cfg.AllowMsgTTL && fs.ttls == nil {\n\t\tif err := fs.recoverTTLState(); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t} else if !cfg.AllowMsgTTL && fs.ttls != nil {\n\t\tfs.ttls = nil\n\t}\n\t// Create or delete the message scheduling state if needed.\n\tif cfg.AllowMsgSchedules && fs.scheduling == nil {\n\t\tif err := fs.recoverMsgSchedulingState(); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t} else if !cfg.AllowMsgSchedules && fs.scheduling != nil {\n\t\tfs.scheduling = nil\n\t}\n\n\t// Limits checks and enforcement.\n\tif err := fs.enforceMsgLimit(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\tif err := fs.enforceBytesLimit(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\n\t// Do age timers.\n\tif fs.ageChk == nil && fs.cfg.MaxAge != 0 {\n\t\tfs.startAgeChk()\n\t}\n\tif fs.ageChk != nil && fs.cfg.MaxAge == 0 {\n\t\tfs.ageChk.Stop()\n\t\tfs.ageChk = nil\n\t\tfs.ageChkTime = 0\n\t}\n\n\tif fs.cfg.MaxMsgsPer > 0 && (old_cfg.MaxMsgsPer == 0 || fs.cfg.MaxMsgsPer < old_cfg.MaxMsgsPer) {\n\t\tif err := fs.enforceMsgPerSubjectLimit(true); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif lmb := fs.lmb; lmb != nil {\n\t\t// Enable/disable async flush depending on if it's supported and already initialized.\n\t\tsupportsAsyncFlush := !fs.fcfg.SyncAlways && cfg.Replicas > 1\n\n\t\t// Async persist mode opts in to async flushing,\n\t\t// sync always would also be disabled if it was configured.\n\t\tif cfg.PersistMode == AsyncPersistMode {\n\t\t\tsupportsAsyncFlush = true\n\t\t\tfs.fcfg.SyncAlways = false\n\t\t\tlmb.syncAlways = false\n\t\t}\n\n\t\tif supportsAsyncFlush && !fs.fcfg.AsyncFlush {\n\t\t\tfs.fcfg.AsyncFlush = true\n\t\t\tlmb.spinUpFlushLoop()\n\t\t} else if !supportsAsyncFlush && fs.fcfg.AsyncFlush {\n\t\t\tfs.fcfg.AsyncFlush = false\n\t\t\tlmb.mu.Lock()\n\t\t\t// Quit the flush loop.\n\t\t\tif lmb.qch != nil {\n\t\t\t\tclose(lmb.qch)\n\t\t\t\tlmb.qch = nil\n\t\t\t}\n\t\t\t_, err := lmb.flushPendingMsgsLocked()\n\t\t\tlmb.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\t// Set flush in place to AsyncFlush which by default is false.\n\t\tfs.fip = !fs.fcfg.AsyncFlush\n\t}\n\tfs.mu.Unlock()\n\n\tif cfg.MaxAge != 0 || cfg.AllowMsgTTL {\n\t\tfs.expireMsgs()\n\t}\n\tif cfg.AllowMsgSchedules {\n\t\tfs.runMsgScheduling()\n\t}\n\treturn nil\n}\n\nfunc dynBlkSize(retention RetentionPolicy, maxBytes int64, encrypted bool) uint64 {\n\tif maxBytes > 0 {\n\t\tblkSize := (maxBytes / 4) + 1 // (25% overhead)\n\t\t// Round up to nearest 100\n\t\tif m := blkSize % 100; m != 0 {\n\t\t\tblkSize += 100 - m\n\t\t}\n\t\tif blkSize <= FileStoreMinBlkSize {\n\t\t\tblkSize = FileStoreMinBlkSize\n\t\t} else if blkSize >= FileStoreMaxBlkSize {\n\t\t\tblkSize = FileStoreMaxBlkSize\n\t\t} else {\n\t\t\tblkSize = defaultMediumBlockSize\n\t\t}\n\t\tif encrypted && blkSize > maximumEncryptedBlockSize {\n\t\t\t// Notes on this below.\n\t\t\tblkSize = maximumEncryptedBlockSize\n\t\t}\n\t\treturn uint64(blkSize)\n\t}\n\n\tswitch {\n\tcase encrypted:\n\t\t// In the case of encrypted stores, large blocks can result in worsened perf\n\t\t// since many writes on disk involve re-encrypting the entire block. For now,\n\t\t// we will enforce a cap on the block size when encryption is enabled to avoid\n\t\t// this.\n\t\treturn maximumEncryptedBlockSize\n\tcase retention == LimitsPolicy:\n\t\t// TODO(dlc) - Make the blocksize relative to this if set.\n\t\treturn defaultLargeBlockSize\n\tdefault:\n\t\t// TODO(dlc) - Make the blocksize relative to this if set.\n\t\treturn defaultMediumBlockSize\n\t}\n}\n\nfunc genEncryptionKey(sc StoreCipher, seed []byte) (ek cipher.AEAD, err error) {\n\tif sc == ChaCha {\n\t\tek, err = chacha20poly1305.NewX(seed)\n\t} else if sc == AES {\n\t\tblock, e := aes.NewCipher(seed)\n\t\tif e != nil {\n\t\t\treturn nil, e\n\t\t}\n\t\tek, err = cipher.NewGCMWithNonceSize(block, block.BlockSize())\n\t} else {\n\t\terr = errUnknownCipher\n\t}\n\treturn ek, err\n}\n\n// Generate an asset encryption key from the context and server PRF.\nfunc (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek cipher.Stream, seed, encrypted []byte, err error) {\n\tif fs.prf == nil {\n\t\treturn nil, nil, nil, nil, errNoEncryption\n\t}\n\t// Generate key encryption key.\n\trb, err := fs.prf([]byte(context))\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\tsc := fs.fcfg.Cipher\n\n\tkek, err := genEncryptionKey(sc, rb)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t}\n\t// Generate random asset encryption key seed.\n\n\tconst seedSize = 32\n\tseed = make([]byte, seedSize)\n\tif n, err := rand.Read(seed); err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t} else if n != seedSize {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"not enough seed bytes read (%d != %d\", n, seedSize)\n\t}\n\n\taek, err = genEncryptionKey(sc, seed)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\t// Generate our nonce. Use same buffer to hold encrypted seed.\n\tnonce := make([]byte, kek.NonceSize(), kek.NonceSize()+len(seed)+kek.Overhead())\n\tif n, err := rand.Read(nonce); err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t} else if n != len(nonce) {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"not enough nonce bytes read (%d != %d)\", n, len(nonce))\n\t}\n\n\tbek, err = genBlockEncryptionKey(sc, seed[:], nonce)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\treturn aek, bek, seed, kek.Seal(nonce, nonce, seed, nil), nil\n}\n\n// Will generate the block encryption key.\nfunc genBlockEncryptionKey(sc StoreCipher, seed, nonce []byte) (cipher.Stream, error) {\n\tif sc == ChaCha {\n\t\treturn chacha20.NewUnauthenticatedCipher(seed, nonce)\n\t} else if sc == AES {\n\t\tblock, err := aes.NewCipher(seed)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn cipher.NewCTR(block, nonce), nil\n\t}\n\treturn nil, errUnknownCipher\n}\n\n// Lock should be held.\nfunc (fs *fileStore) recoverAEK() error {\n\tif fs.prf != nil && fs.aek == nil {\n\t\tekey, err := os.ReadFile(filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trb, err := fs.prf([]byte(fs.cfg.Name))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tkek, err := genEncryptionKey(fs.fcfg.Cipher, rb)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tns := kek.NonceSize()\n\t\tseed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\taek, err := genEncryptionKey(fs.fcfg.Cipher, seed)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfs.aek = aek\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) setupAEK() error {\n\tif fs.prf != nil && fs.aek == nil {\n\t\tkey, _, _, encrypted, err := fs.genEncryptionKeys(fs.cfg.Name)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tkeyFile := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey)\n\t\tif _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) {\n\t\t\treturn err\n\t\t}\n\t\terr = fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Set our aek.\n\t\tfs.aek = key\n\t}\n\treturn nil\n}\n\n// Write out meta and the checksum.\n// Lock should be held.\nfunc (fs *fileStore) writeStreamMeta() error {\n\tif err := fs.setupAEK(); err != nil {\n\t\treturn err\n\t}\n\n\tmeta := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFile)\n\tif _, err := os.Stat(meta); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\tb, err := json.Marshal(fs.cfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Encrypt if needed.\n\tif fs.aek != nil {\n\t\tnonce := make([]byte, fs.aek.NonceSize(), fs.aek.NonceSize()+len(b)+fs.aek.Overhead())\n\t\tif n, err := rand.Read(nonce); err != nil {\n\t\t\treturn err\n\t\t} else if n != len(nonce) {\n\t\t\treturn fmt.Errorf(\"not enough nonce bytes read (%d != %d)\", n, len(nonce))\n\t\t}\n\t\tb = fs.aek.Seal(nonce, nonce, b, nil)\n\t}\n\n\terr = fs.writeFileWithOptionalSync(meta, b, defaultFilePerms)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfs.hh.Reset()\n\tfs.hh.Write(b)\n\tvar hb [highwayhash.Size64]byte\n\tchecksum := hex.EncodeToString(fs.hh.Sum(hb[:0]))\n\tsum := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileSum)\n\terr = fs.writeFileWithOptionalSync(sum, []byte(checksum), defaultFilePerms)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// Pools to recycle the blocks to help with memory pressure.\nvar blkPoolTiny = &sync.Pool{\n\tNew: func() any {\n\t\tb := [defaultTinyBlockSize]byte{}\n\t\treturn &b\n\t},\n}\nvar blkPoolSmall = &sync.Pool{\n\tNew: func() any {\n\t\tb := [defaultSmallBlockSize]byte{}\n\t\treturn &b\n\t},\n}\nvar blkPoolMedium = &sync.Pool{\n\tNew: func() any {\n\t\tb := [defaultMediumBlockSize]byte{}\n\t\treturn &b\n\t},\n}\nvar blkPoolBig = &sync.Pool{\n\tNew: func() any {\n\t\tb := [defaultLargeBlockSize]byte{}\n\t\treturn &b\n\t},\n}\n\n// Get a new msg block based on sz estimate.\nfunc getMsgBlockBuf(sz int) (buf []byte) {\n\tswitch {\n\tcase sz <= defaultTinyBlockSize:\n\t\treturn blkPoolTiny.Get().(*[defaultTinyBlockSize]byte)[:0]\n\tcase sz <= defaultSmallBlockSize:\n\t\treturn blkPoolSmall.Get().(*[defaultSmallBlockSize]byte)[:0]\n\tcase sz <= defaultMediumBlockSize:\n\t\treturn blkPoolMedium.Get().(*[defaultMediumBlockSize]byte)[:0]\n\tcase sz <= defaultLargeBlockSize:\n\t\treturn blkPoolBig.Get().(*[defaultLargeBlockSize]byte)[:0]\n\tdefault:\n\t\t// Ideally this should not happen, once we return a buffer that's\n\t\t// larger than defaultLargeBlockSize then we will refuse to recycle\n\t\t// it to stop the pools from bloating excessively.\n\t\treturn make([]byte, 0, sz)\n\t}\n}\n\n// Recycle the msg block.\nfunc recycleMsgBlockBuf(buf []byte) {\n\tswitch cap(buf) {\n\tcase defaultTinyBlockSize:\n\t\tb := (*[defaultTinyBlockSize]byte)(buf[0:defaultTinyBlockSize])\n\t\tblkPoolTiny.Put(b)\n\tcase defaultSmallBlockSize:\n\t\tb := (*[defaultSmallBlockSize]byte)(buf[0:defaultSmallBlockSize])\n\t\tblkPoolSmall.Put(b)\n\tcase defaultMediumBlockSize:\n\t\tb := (*[defaultMediumBlockSize]byte)(buf[0:defaultMediumBlockSize])\n\t\tblkPoolMedium.Put(b)\n\tcase defaultLargeBlockSize:\n\t\tb := (*[defaultLargeBlockSize]byte)(buf[0:defaultLargeBlockSize])\n\t\tblkPoolBig.Put(b)\n\tdefault:\n\t\t// Too large, let the GC collect it instead.\n\t}\n}\n\nconst (\n\tmsgHdrSize = 22\n\tchecksumSize = 8\n\temptyRecordLen = msgHdrSize + checksumSize\n)\n\n// Lock should be held.\nfunc (fs *fileStore) noTrackSubjects() bool {\n\treturn !(fs.psim.Size() > 0 || len(fs.cfg.Subjects) > 0 || fs.cfg.Mirror != nil || len(fs.cfg.Sources) > 0)\n}\n\n// Will init the basics for a message block.\nfunc (fs *fileStore) initMsgBlock(index uint32) *msgBlock {\n\tmb := &msgBlock{\n\t\tfs: fs,\n\t\tindex: index,\n\t\tcexp: fs.fcfg.CacheExpire,\n\t\tfexp: fs.fcfg.SubjectStateExpire,\n\t\tnoTrack: fs.noTrackSubjects(),\n\t\tsyncAlways: fs.fcfg.SyncAlways,\n\t}\n\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tmb.mfn = filepath.Join(mdir, fmt.Sprintf(blkScan, index))\n\n\tif mb.hh == nil {\n\t\tkey := sha256.Sum256(fs.hashKeyForBlock(index))\n\t\tmb.hh, _ = highwayhash.NewDigest64(key[:])\n\t}\n\treturn mb\n}\n\n// Check for encryption, we do not load keys on startup anymore so might need to load them here.\n// Lock for fs should be held.\nfunc (mb *msgBlock) checkAndLoadEncryption() error {\n\tif mb.fs != nil && mb.fs.prf != nil && (mb.aek == nil || mb.bek == nil) {\n\t\tif err := mb.fs.loadEncryptionForMsgBlock(mb); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Lock for fs should be held.\nfunc (fs *fileStore) loadEncryptionForMsgBlock(mb *msgBlock) error {\n\tif fs.prf == nil {\n\t\treturn nil\n\t}\n\n\tvar createdKeys bool\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tekey, err := os.ReadFile(filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index)))\n\tif err != nil {\n\t\t// We do not seem to have keys even though we should. Could be a plaintext conversion.\n\t\t// Create the keys and we will double check below.\n\t\tif err := fs.genEncryptionKeysForBlock(mb); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcreatedKeys = true\n\t} else {\n\t\tif len(ekey) < minBlkKeySize {\n\t\t\treturn errBadKeySize\n\t\t}\n\t\t// Recover key encryption key.\n\t\trb, err := fs.prf([]byte(fmt.Sprintf(\"%s:%d\", fs.cfg.Name, mb.index)))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tsc := fs.fcfg.Cipher\n\t\tkek, err := genEncryptionKey(sc, rb)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tns := kek.NonceSize()\n\t\tseed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)\n\t\tif err != nil {\n\t\t\t// We may be here on a cipher conversion, so attempt to convert.\n\t\t\tif err = mb.convertCipher(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tmb.seed, mb.nonce = seed, ekey[:ns]\n\t\t}\n\t\tmb.aek, err = genEncryptionKey(sc, mb.seed)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif mb.bek, err = genBlockEncryptionKey(sc, mb.seed, mb.nonce); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// If we created keys here, let's check the data and if it is plaintext convert here.\n\tif createdKeys {\n\t\tif err := mb.convertToEncrypted(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Load a last checksum if needed from the block file.\n// Lock should be held.\nfunc (mb *msgBlock) ensureLastChecksumLoaded() {\n\tvar empty [8]byte\n\tif mb.lchk != empty {\n\t\treturn\n\t}\n\tcopy(mb.lchk[0:], mb.lastChecksum())\n}\n\n// Lock held on entry\nfunc (fs *fileStore) recoverMsgBlock(index uint32) (*msgBlock, error) {\n\tmb := fs.initMsgBlock(index)\n\t// Open up the message file, but we will try to recover from the index file.\n\t// We will check that the last checksums match.\n\tfile, err := mb.openBlock()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer file.Close()\n\n\tif fi, err := file.Stat(); fi != nil {\n\t\tmb.rbytes = uint64(fi.Size())\n\t} else {\n\t\treturn nil, err\n\t}\n\n\t// Make sure encryption loaded if needed.\n\tif err = fs.loadEncryptionForMsgBlock(mb); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Grab last checksum from main block file.\n\tvar lchk [8]byte\n\tif mb.rbytes >= checksumSize {\n\t\tif mb.bek != nil {\n\t\t\t// We pass nil, so get a buf from the block pool, we'll need to recycle it afterward.\n\t\t\tbuf, _ := mb.loadBlock(nil)\n\t\t\tif len(buf) >= checksumSize {\n\t\t\t\tmb.bek.XORKeyStream(buf, buf)\n\t\t\t\tcopy(lchk[0:], buf[len(buf)-checksumSize:])\n\t\t\t}\n\t\t\t// We can recycle it now.\n\t\t\trecycleMsgBlockBuf(buf)\n\t\t} else if _, err = file.ReadAt(lchk[:], int64(mb.rbytes)-checksumSize); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif err = file.Close(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Read our index file. Use this as source of truth if possible.\n\t// This not applicable in >= 2.10 servers. Here for upgrade paths from < 2.10.\n\tif err := mb.readIndexInfo(); err == nil {\n\t\t// Quick sanity check here.\n\t\t// Note this only checks that the message blk file is not newer then this file, or is empty and we expect empty.\n\t\tif (mb.rbytes == 0 && mb.msgs == 0) || bytes.Equal(lchk[:], mb.lchk[:]) {\n\t\t\tif mb.msgs > 0 && !mb.noTrack && fs.psim != nil {\n\t\t\t\tif err = fs.populateGlobalPerSubjectInfo(mb); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\t// Try to dump any state we needed on recovery.\n\t\t\t\tmb.tryForceExpireCacheLocked()\n\t\t\t}\n\t\t\tfs.addMsgBlock(mb)\n\t\t\treturn mb, nil\n\t\t}\n\t}\n\n\t// If we get data loss rebuilding the message block state record that with the fs itself.\n\tld, tombs, err := mb.rebuildState()\n\tif err != nil {\n\t\treturn nil, err\n\t} else if ld != nil {\n\t\tfs.addLostData(ld)\n\t}\n\t// Collect all tombstones.\n\tif len(tombs) > 0 {\n\t\tfs.tombs = append(fs.tombs, tombs...)\n\t}\n\n\tif mb.msgs > 0 && !mb.noTrack && fs.psim != nil {\n\t\tif err = fs.populateGlobalPerSubjectInfo(mb); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// Try to dump any state we needed on recovery.\n\t\tmb.tryForceExpireCacheLocked()\n\t}\n\n\tif err = mb.closeFDs(); err != nil {\n\t\treturn nil, err\n\t}\n\tfs.addMsgBlock(mb)\n\n\treturn mb, nil\n}\n\nfunc (fs *fileStore) lostData() *LostStreamData {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\tif fs.ld == nil {\n\t\treturn nil\n\t}\n\tnld := *fs.ld\n\treturn &nld\n}\n\n// Lock should be held.\nfunc (fs *fileStore) addLostData(ld *LostStreamData) {\n\tif ld == nil {\n\t\treturn\n\t}\n\tif fs.ld != nil {\n\t\tvar added bool\n\t\tfor _, seq := range ld.Msgs {\n\t\t\tif _, found := fs.ld.exists(seq); !found {\n\t\t\t\tfs.ld.Msgs = append(fs.ld.Msgs, seq)\n\t\t\t\tadded = true\n\t\t\t}\n\t\t}\n\t\tif added {\n\t\t\tmsgs := fs.ld.Msgs\n\t\t\tslices.Sort(msgs)\n\t\t\tfs.ld.Bytes += ld.Bytes\n\t\t}\n\t} else {\n\t\tfs.ld = ld\n\t}\n}\n\n// Helper to see if we already have this sequence reported in our lost data.\nfunc (ld *LostStreamData) exists(seq uint64) (int, bool) {\n\ti := slices.IndexFunc(ld.Msgs, func(i uint64) bool {\n\t\treturn i == seq\n\t})\n\treturn i, i > -1\n}\n\nfunc (fs *fileStore) removeFromLostData(seq uint64) {\n\tif fs.ld == nil {\n\t\treturn\n\t}\n\tif i, found := fs.ld.exists(seq); found {\n\t\tfs.ld.Msgs = append(fs.ld.Msgs[:i], fs.ld.Msgs[i+1:]...)\n\t\tif len(fs.ld.Msgs) == 0 {\n\t\t\tfs.ld = nil\n\t\t}\n\t}\n}\n\nfunc (fs *fileStore) rebuildState(ld *LostStreamData) {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\tfs.rebuildStateLocked(ld)\n}\n\n// Lock should be held.\nfunc (fs *fileStore) rebuildStateLocked(ld *LostStreamData) {\n\tfs.addLostData(ld)\n\n\tfs.state.Msgs, fs.state.Bytes = 0, 0\n\tfs.state.FirstSeq, fs.state.LastSeq = 0, 0\n\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tfs.state.Msgs += mb.msgs\n\t\tfs.state.Bytes += mb.bytes\n\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\tif fs.state.FirstSeq == 0 || (fseq < fs.state.FirstSeq && mb.first.ts != 0) {\n\t\t\tfs.state.FirstSeq = fseq\n\t\t\tif mb.first.ts == 0 {\n\t\t\t\tfs.state.FirstTime = time.Time{}\n\t\t\t} else {\n\t\t\t\tfs.state.FirstTime = time.Unix(0, mb.first.ts).UTC()\n\t\t\t}\n\t\t}\n\t\t// Preserve last time, could have erased the last message in one block, and then\n\t\t// have a tombstone with the proper timestamp afterward in another block\n\t\tif lseq := atomic.LoadUint64(&mb.last.seq); lseq >= fs.state.LastSeq {\n\t\t\tfs.state.LastSeq = lseq\n\t\t\tif mb.last.ts == 0 {\n\t\t\t\tfs.state.LastTime = time.Time{}\n\t\t\t} else {\n\t\t\t\tfs.state.LastTime = time.Unix(0, mb.last.ts).UTC()\n\t\t\t}\n\t\t}\n\t\tmb.mu.RUnlock()\n\t}\n}\n\n// Attempt to convert the cipher used for this message block.\nfunc (mb *msgBlock) convertCipher() error {\n\tfs := mb.fs\n\tsc := fs.fcfg.Cipher\n\n\tvar osc StoreCipher\n\tswitch sc {\n\tcase ChaCha:\n\t\tosc = AES\n\tcase AES:\n\t\tosc = ChaCha\n\t}\n\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tekey, err := os.ReadFile(filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index)))\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(ekey) < minBlkKeySize {\n\t\treturn errBadKeySize\n\t}\n\ttype prfWithCipher struct {\n\t\tkeyGen\n\t\tStoreCipher\n\t}\n\tvar prfs []prfWithCipher\n\tif fs.prf != nil {\n\t\tprfs = append(prfs, prfWithCipher{fs.prf, sc})\n\t\tprfs = append(prfs, prfWithCipher{fs.prf, osc})\n\t}\n\tif fs.oldprf != nil {\n\t\tprfs = append(prfs, prfWithCipher{fs.oldprf, sc})\n\t\tprfs = append(prfs, prfWithCipher{fs.oldprf, osc})\n\t}\n\n\tfor _, prf := range prfs {\n\t\t// Recover key encryption key.\n\t\trb, err := prf.keyGen([]byte(fmt.Sprintf(\"%s:%d\", fs.cfg.Name, mb.index)))\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tkek, err := genEncryptionKey(prf.StoreCipher, rb)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tns := kek.NonceSize()\n\t\tseed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tnonce := ekey[:ns]\n\t\tbek, err := genBlockEncryptionKey(prf.StoreCipher, seed, nonce)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tbuf, _ := mb.loadBlock(nil)\n\t\tbek.XORKeyStream(buf, buf)\n\t\t// Check for compression, and make sure we can parse with old cipher and key file.\n\t\tif nbuf, err := mb.decompressIfNeeded(buf); err != nil {\n\t\t\treturn err\n\t\t} else if _, _, err = mb.rebuildStateFromBufLocked(nbuf, false); err != nil {\n\t\t\treturn err\n\t\t} else if err = mb.indexCacheBuf(nbuf); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Reset the cache since we just read everything in.\n\t\tmb.cache = nil\n\n\t\t// Generate new keys. If we error for some reason then we will put\n\t\t// the old keyfile back.\n\t\tif err := fs.genEncryptionKeysForBlock(mb); err != nil {\n\t\t\tkeyFile := filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index))\n\t\t\tfs.writeFileWithOptionalSync(keyFile, ekey, defaultFilePerms)\n\t\t\treturn err\n\t\t}\n\t\tmb.bek.XORKeyStream(buf, buf)\n\t\t<-dios\n\t\terr = os.WriteFile(mb.mfn, buf, defaultFilePerms)\n\t\tdios <- struct{}{}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\treturn fmt.Errorf(\"unable to recover keys\")\n}\n\n// Convert a plaintext block to encrypted.\nfunc (mb *msgBlock) convertToEncrypted() error {\n\tif mb.bek == nil {\n\t\treturn nil\n\t}\n\tbuf, err := mb.loadBlock(nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Check for compression.\n\tif buf, err = mb.decompressIfNeeded(buf); err != nil {\n\t\treturn err\n\t} else if _, _, err = mb.rebuildStateFromBufLocked(buf, false); err != nil {\n\t\treturn err\n\t} else if err = mb.indexCacheBuf(buf); err != nil {\n\t\t// This likely indicates this was already encrypted or corrupt.\n\t\tmb.cache = nil\n\t\treturn err\n\t}\n\t// Undo cache from above for later.\n\tmb.cache = nil\n\tmb.bek.XORKeyStream(buf, buf)\n\t<-dios\n\terr = os.WriteFile(mb.mfn, buf, defaultFilePerms)\n\tdios <- struct{}{}\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// Return the mb's index.\nfunc (mb *msgBlock) getIndex() uint32 {\n\tmb.mu.RLock()\n\tdefer mb.mu.RUnlock()\n\treturn mb.index\n}\n\n// Rebuild the state of the blk based on what we have on disk in the N.blk file.\n// We will return any lost data, and we will return any delete tombstones we encountered.\nfunc (mb *msgBlock) rebuildState() (*LostStreamData, []uint64, error) {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.rebuildStateLocked()\n}\n\n// Rebuild the state of the blk based on what we have on disk in the N.blk file.\n// Lock should be held.\nfunc (mb *msgBlock) rebuildStateLocked() (*LostStreamData, []uint64, error) {\n\t// Remove the .fss file and clear any cache we have set.\n\tmb.clearCacheAndOffset()\n\n\tbuf, err := mb.loadBlock(nil)\n\tdefer recycleMsgBlockBuf(buf)\n\n\tif err != nil || len(buf) == 0 {\n\t\t// Only allow continuing to mark lost data if the file itself doesn't exist, or was empty.\n\t\tif err != nil && err != errNoBlkData {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tvar ld *LostStreamData\n\t\t// No data to rebuild from here.\n\t\tif mb.msgs > 0 {\n\t\t\t// We need to declare lost data here.\n\t\t\tld = &LostStreamData{Msgs: make([]uint64, 0, mb.msgs), Bytes: mb.bytes}\n\t\t\tfirstSeq, lastSeq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\t\t\tfor seq := firstSeq; seq <= lastSeq; seq++ {\n\t\t\t\tif !mb.dmap.Exists(seq) {\n\t\t\t\t\tld.Msgs = append(ld.Msgs, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Clear invalid state. We will let this blk be added in here.\n\t\t\tmb.msgs, mb.bytes, mb.rbytes, mb.fss = 0, 0, 0, nil\n\t\t\tmb.dmap.Empty()\n\t\t\tatomic.StoreUint64(&mb.first.seq, atomic.LoadUint64(&mb.last.seq)+1)\n\t\t}\n\t\treturn ld, nil, nil\n\t}\n\n\t// Check if we need to decrypt.\n\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\treturn nil, nil, err\n\t}\n\t// Check for compression.\n\tif buf, err = mb.decompressIfNeeded(buf); err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn mb.rebuildStateFromBufLocked(buf, true)\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) rebuildStateFromBufLocked(buf []byte, allowTruncate bool) (*LostStreamData, []uint64, error) {\n\tvar err error\n\tstartLastSeq := atomic.LoadUint64(&mb.last.seq)\n\n\t// Clear state we need to rebuild.\n\tmb.msgs, mb.bytes, mb.rbytes, mb.fss = 0, 0, 0, nil\n\tatomic.StoreUint64(&mb.last.seq, 0)\n\tmb.last.ts = 0\n\tfirstNeedsSet := true\n\n\tmb.rbytes = uint64(len(buf))\n\n\taddToDmap := func(seq uint64) {\n\t\tif seq == 0 {\n\t\t\treturn\n\t\t}\n\t\tmb.dmap.Insert(seq)\n\t}\n\n\t// For tombstones that we find and collect.\n\tvar (\n\t\ttombstones []uint64\n\t\tmaxTombstoneSeq uint64\n\t\tmaxTombstoneTs int64\n\t)\n\n\tdefer func() {\n\t\t// For empty msg blocks make sure we recover last seq correctly based off of first.\n\t\t// Or if we seem to have no messages but had a tombstone, which we use to remember\n\t\t// sequences and timestamps now, use that to properly setup the first and last.\n\t\tif mb.msgs == 0 {\n\t\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t\tif fseq > 0 {\n\t\t\t\tatomic.StoreUint64(&mb.last.seq, fseq-1)\n\t\t\t} else if fseq == 0 && maxTombstoneSeq > 0 {\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, maxTombstoneSeq+1)\n\t\t\t\tmb.first.ts = 0\n\t\t\t\tif mb.last.seq == 0 {\n\t\t\t\t\tatomic.StoreUint64(&mb.last.seq, maxTombstoneSeq)\n\t\t\t\t\tmb.last.ts = maxTombstoneTs\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tvar le = binary.LittleEndian\n\n\t// There are cases where we're not allowed to truncate, like for an encrypted or compressed\n\t// block since the index will be the decrypted and decompressed index.\n\ttruncate := func(index uint32) error {\n\t\tvar fd *os.File\n\t\tif mb.mfd != nil {\n\t\t\tfd = mb.mfd\n\t\t} else {\n\t\t\t<-dios\n\t\t\tfd, err = os.OpenFile(mb.mfn, os.O_RDWR, defaultFilePerms)\n\t\t\tdios <- struct{}{}\n\t\t\tif err == nil {\n\t\t\t\tdefer fd.Close()\n\t\t\t}\n\t\t}\n\t\tif fd == nil {\n\t\t\treturn nil\n\t\t}\n\t\tif err := fd.Truncate(int64(index)); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Update our checksum.\n\t\tif index >= 8 {\n\t\t\tvar lchk [8]byte\n\t\t\tif _, err = fd.ReadAt(lchk[:], int64(index-8)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tcopy(mb.lchk[0:], lchk[:])\n\t\t}\n\t\tif err = fd.Sync(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\tgatherLost := func(lb uint32) *LostStreamData {\n\t\tvar ld LostStreamData\n\t\tfor seq := atomic.LoadUint64(&mb.last.seq) + 1; seq <= startLastSeq; seq++ {\n\t\t\tld.Msgs = append(ld.Msgs, seq)\n\t\t}\n\t\tld.Bytes = uint64(lb)\n\t\treturn &ld\n\t}\n\n\t// To detect gaps from compaction, and to ensure the sequence keeps moving up.\n\tvar last uint64\n\tvar hb [highwayhash.Size64]byte\n\n\tupdateLast := func(seq uint64, ts int64) {\n\t\t// The sequence needs to only ever move up.\n\t\tif seq <= last {\n\t\t\treturn\n\t\t}\n\n\t\t// Check for any gaps from compaction, meaning no ebit entry.\n\t\tif last > 0 && seq != last+1 && mb.msgs != 0 {\n\t\t\tfor dseq := last + 1; dseq < seq; dseq++ {\n\t\t\t\taddToDmap(dseq)\n\t\t\t}\n\t\t}\n\t\tlast = seq\n\t\tatomic.StoreUint64(&mb.last.seq, last)\n\t\tmb.last.ts = ts\n\t}\n\n\tfor index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; {\n\t\tif index+msgHdrSize > lbuf {\n\t\t\terr = errBadMsg{mb.mfn, fmt.Sprintf(\"message overrun (index %d lbuf %d)\", index, lbuf)}\n\t\t\tif allowTruncate {\n\t\t\t\tif err = truncate(index); err != nil {\n\t\t\t\t\treturn nil, nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn gatherLost(lbuf - index), tombstones, err\n\t\t}\n\n\t\thdr := buf[index : index+msgHdrSize]\n\t\trl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:]))\n\n\t\thasHeaders := rl&hbit != 0\n\t\t// Clear any headers bit that could be set.\n\t\trl &^= hbit\n\t\tshlen := slen\n\t\tif hasHeaders {\n\t\t\tshlen += 4\n\t\t}\n\t\tdlen := int(rl) - msgHdrSize\n\t\t// Do some quick sanity checks here.\n\t\tif dlen < 0 || shlen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh {\n\t\t\terr = errBadMsg{mb.mfn, fmt.Sprintf(\"sanity check failed (dlen %d slen %d rl %d index %d lbuf %d)\", dlen, slen, rl, index, lbuf)}\n\t\t\tif allowTruncate {\n\t\t\t\tif err = truncate(index); err != nil {\n\t\t\t\t\treturn nil, nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn gatherLost(lbuf - index), tombstones, err\n\t\t}\n\n\t\t// Check for checksum failures before additional processing.\n\t\tdata := buf[index+msgHdrSize : index+rl]\n\t\tsubj := data[:slen]\n\t\tif hh := mb.hh; hh != nil {\n\t\t\thh.Reset()\n\t\t\thh.Write(hdr[4:20])\n\t\t\thh.Write(subj)\n\t\t\tif hasHeaders {\n\t\t\t\thh.Write(data[slen+4 : dlen-recordHashSize])\n\t\t\t} else {\n\t\t\t\thh.Write(data[slen : dlen-recordHashSize])\n\t\t\t}\n\t\t\tchecksum := hh.Sum(hb[:0])\n\t\t\tif !bytes.Equal(checksum, data[len(data)-recordHashSize:]) {\n\t\t\t\terr = errBadMsg{mb.mfn, \"invalid checksum\"}\n\t\t\t\tif allowTruncate {\n\t\t\t\t\tif err = truncate(index); err != nil {\n\t\t\t\t\t\treturn nil, nil, err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn gatherLost(lbuf - index), tombstones, err\n\t\t\t}\n\t\t\tcopy(mb.lchk[0:], checksum)\n\t\t}\n\n\t\t// Grab our sequence and timestamp.\n\t\tseq := le.Uint64(hdr[4:])\n\t\tts := int64(le.Uint64(hdr[12:]))\n\n\t\t// Check if this is a delete tombstone.\n\t\tif seq&tbit != 0 {\n\t\t\tseq = seq &^ tbit\n\t\t\t// Need to process this here and make sure we have accounted for this properly.\n\t\t\ttombstones = append(tombstones, seq)\n\t\t\tif maxTombstoneSeq == 0 || seq > maxTombstoneSeq {\n\t\t\t\tmaxTombstoneSeq, maxTombstoneTs = seq, ts\n\t\t\t}\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\n\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t// This is an old erased message, or a new one that we can track.\n\t\tif seq == 0 || seq&ebit != 0 || seq < fseq {\n\t\t\tseq = seq &^ ebit\n\t\t\tif seq >= fseq {\n\t\t\t\tupdateLast(seq, ts)\n\t\t\t\tif mb.msgs == 0 {\n\t\t\t\t\tatomic.StoreUint64(&mb.first.seq, seq+1)\n\t\t\t\t\tmb.first.ts = 0\n\t\t\t\t} else if seq != 0 {\n\t\t\t\t\t// Only add to dmap if past recorded first seq and non-zero.\n\t\t\t\t\taddToDmap(seq)\n\t\t\t\t}\n\t\t\t}\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\n\t\t// This is for when we have index info that adjusts for deleted messages\n\t\t// at the head. So the first.seq will be already set here. If this is larger\n\t\t// replace what we have with this seq.\n\t\tif firstNeedsSet && seq >= fseq {\n\t\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\t\tfirstNeedsSet, mb.first.ts = false, ts\n\t\t}\n\n\t\t// The sequence needs to only ever move up.\n\t\tif seq <= last {\n\t\t\t// Advance to next record.\n\t\t\t// We've already accounted for this sequence and marked it as deleted.\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\t\tif !mb.dmap.Exists(seq) {\n\t\t\tmb.msgs++\n\t\t\tmb.bytes += uint64(rl)\n\t\t}\n\n\t\tupdateLast(seq, ts)\n\n\t\t// Advance to next record.\n\t\tindex += rl\n\t}\n\n\t// For empty msg blocks make sure we recover last seq correctly based off of first.\n\t// Or if we seem to have no messages but had a tombstone, which we use to remember\n\t// sequences and timestamps now, use that to properly setup the first and last.\n\tif mb.msgs == 0 {\n\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\tif fseq > 0 {\n\t\t\tatomic.StoreUint64(&mb.last.seq, fseq-1)\n\t\t} else if fseq == 0 && maxTombstoneSeq > 0 {\n\t\t\tatomic.StoreUint64(&mb.first.seq, maxTombstoneSeq+1)\n\t\t\tmb.first.ts = 0\n\t\t\tif lseq := atomic.LoadUint64(&mb.last.seq); lseq == 0 {\n\t\t\t\tatomic.StoreUint64(&mb.last.seq, maxTombstoneSeq)\n\t\t\t\tmb.last.ts = maxTombstoneTs\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, tombstones, nil\n}\n\n// For doing warn logging.\n// Lock should be held.\nfunc (fs *fileStore) warn(format string, args ...any) {\n\t// No-op if no server configured.\n\tif fs.srv == nil {\n\t\treturn\n\t}\n\tfs.srv.Warnf(fmt.Sprintf(\"Filestore [%s] %s\", fs.cfg.Name, format), args...)\n}\n\n// For doing error logging.\n// Lock should be held.\nfunc (fs *fileStore) error(format string, args ...any) {\n\t// No-op if no server configured.\n\tif fs.srv == nil {\n\t\treturn\n\t}\n\tfs.srv.Errorf(fmt.Sprintf(\"Filestore [%s] %s\", fs.cfg.Name, format), args...)\n}\n\n// For doing debug logging.\n// Lock should be held.\nfunc (fs *fileStore) debug(format string, args ...any) {\n\t// No-op if no server configured.\n\tif fs.srv == nil {\n\t\treturn\n\t}\n\tfs.srv.Debugf(fmt.Sprintf(\"Filestore [%s] %s\", fs.cfg.Name, format), args...)\n}\n\n// Track local state but ignore timestamps here.\nfunc updateTrackingState(state *StreamState, mb *msgBlock) {\n\tfirst := atomic.LoadUint64(&mb.first.seq)\n\tlast := atomic.LoadUint64(&mb.last.seq)\n\tif state.FirstSeq == 0 {\n\t\tstate.FirstSeq = first\n\t} else if first < state.FirstSeq && mb.first.ts != 0 {\n\t\tstate.FirstSeq = first\n\t}\n\tif last > state.LastSeq {\n\t\tstate.LastSeq = last\n\t}\n\tstate.Msgs += mb.msgs\n\tstate.Bytes += mb.bytes\n}\n\n// Determine if our tracking states are the same.\nfunc trackingStatesEqual(fs, mb *StreamState) bool {\n\t// When a fs is brand new the fs state will have first seq of 0, but tracking mb may have 1.\n\t// If either has a first sequence that is not 0 or 1 we will check if they are the same, otherwise skip.\n\tif (fs.FirstSeq > 1 && mb.FirstSeq > 1) || mb.FirstSeq > 1 {\n\t\treturn fs.Msgs == mb.Msgs && fs.FirstSeq == mb.FirstSeq && fs.LastSeq == mb.LastSeq && fs.Bytes == mb.Bytes\n\t}\n\treturn fs.Msgs == mb.Msgs && fs.LastSeq == mb.LastSeq && fs.Bytes == mb.Bytes\n}\n\n// recoverFullState will attempt to receover our last full state and re-process any state changes\n// that happened afterwards.\nfunc (fs *fileStore) recoverFullState() (rerr error) {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Check for any left over purged messages.\n\t<-dios\n\tif err := fs.recoverPartialPurge(); err != nil {\n\t\tdios <- struct{}{}\n\t\treturn err\n\t}\n\t// Grab our stream state file and load it in.\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\tbuf, err := os.ReadFile(fn)\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\tif !os.IsNotExist(err) {\n\t\t\tfs.warn(\"Could not read stream state file: %v\", err)\n\t\t}\n\t\treturn err\n\t}\n\n\tconst minLen = 32\n\tif len(buf) < minLen {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state too short (%d bytes)\", len(buf))\n\t\treturn errCorruptState\n\t}\n\n\t// The highwayhash will be on the end. Check that it still matches.\n\th := buf[len(buf)-highwayhash.Size64:]\n\tbuf = buf[:len(buf)-highwayhash.Size64]\n\tfs.hh.Reset()\n\tfs.hh.Write(buf)\n\tvar hb [highwayhash.Size64]byte\n\tif !bytes.Equal(h, fs.hh.Sum(hb[:0])) {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state checksum did not match\")\n\t\treturn errCorruptState\n\t}\n\n\t// Decrypt if needed.\n\t// We can be setup for encryption but if this is a snapshot restore we will be missing the keyfile\n\t// since snapshots strip encryption.\n\tif fs.prf != nil && fs.aek != nil {\n\t\tns := fs.aek.NonceSize()\n\t\tbuf, err = fs.aek.Open(nil, buf[:ns], buf[ns:], nil)\n\t\tif err != nil {\n\t\t\tfs.warn(\"Stream state error reading encryption key: %v\", err)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tversion := buf[1]\n\tif buf[0] != fullStateMagic || version < fullStateMinVersion || version > fullStateVersion {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state magic and version mismatch\")\n\t\treturn errCorruptState\n\t}\n\n\tbi := hdrLen\n\n\treadU64 := func() uint64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tv, n := binary.Uvarint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn 0\n\t\t}\n\t\tbi += n\n\t\treturn v\n\t}\n\treadI64 := func() int64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tv, n := binary.Varint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn -1\n\t\t}\n\t\tbi += n\n\t\treturn v\n\t}\n\n\tsetTime := func(t *time.Time, ts int64) {\n\t\tif ts == 0 {\n\t\t\t*t = time.Time{}\n\t\t} else {\n\t\t\t*t = time.Unix(0, ts).UTC()\n\t\t}\n\t}\n\n\tvar state StreamState\n\tstate.Msgs = readU64()\n\tstate.Bytes = readU64()\n\tstate.FirstSeq = readU64()\n\tbaseTime := readI64()\n\tsetTime(&state.FirstTime, baseTime)\n\tstate.LastSeq = readU64()\n\tsetTime(&state.LastTime, readI64())\n\n\t// Check for per subject info.\n\tif numSubjects := int(readU64()); numSubjects > 0 {\n\t\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\t\tfor i := 0; i < numSubjects; i++ {\n\t\t\tif lsubj := int(readU64()); lsubj > 0 {\n\t\t\t\tif bi+lsubj > len(buf) {\n\t\t\t\t\t_ = os.Remove(fn)\n\t\t\t\t\tfs.warn(\"Stream state bad subject len (%d)\", lsubj)\n\t\t\t\t\treturn errCorruptState\n\t\t\t\t}\n\t\t\t\t// If we have lots of subjects this will alloc for each one.\n\t\t\t\t// We could reference the underlying buffer, but we could guess wrong if\n\t\t\t\t// number of blocks is large and subjects is low, since we would reference buf.\n\t\t\t\tsubj := buf[bi : bi+lsubj]\n\t\t\t\t// We had a bug that could cause memory corruption in the PSIM that could have gotten stored to disk.\n\t\t\t\t// Only would affect subjects, so do quick check.\n\t\t\t\tif !isValidSubject(bytesToString(subj), true) {\n\t\t\t\t\t_ = os.Remove(fn)\n\t\t\t\t\tfs.warn(\"Stream state corrupt subject detected\")\n\t\t\t\t\treturn errCorruptState\n\t\t\t\t}\n\t\t\t\tbi += lsubj\n\t\t\t\tpsi := psi{total: readU64(), fblk: uint32(readU64())}\n\t\t\t\tif psi.total > 1 {\n\t\t\t\t\tpsi.lblk = uint32(readU64())\n\t\t\t\t} else {\n\t\t\t\t\tpsi.lblk = psi.fblk\n\t\t\t\t}\n\t\t\t\tfs.psim.Insert(subj, psi)\n\t\t\t\tfs.tsl += lsubj\n\t\t\t}\n\t\t}\n\t}\n\n\t// Track the state as represented by the blocks themselves.\n\tvar mstate StreamState\n\n\tif numBlocks := readU64(); numBlocks > 0 {\n\t\tlastIndex := int(numBlocks - 1)\n\t\tfs.blks = make([]*msgBlock, 0, numBlocks)\n\t\tfor i := 0; i < int(numBlocks); i++ {\n\t\t\tindex, nbytes, fseq, fts, lseq, lts, numDeleted := uint32(readU64()), readU64(), readU64(), readI64(), readU64(), readI64(), readU64()\n\t\t\tvar ttls uint64\n\t\t\tif version >= 2 {\n\t\t\t\tttls = readU64()\n\t\t\t}\n\t\t\tvar schedules uint64\n\t\t\tif version >= 3 {\n\t\t\t\tschedules = readU64()\n\t\t\t}\n\t\t\tif bi < 0 {\n\t\t\t\t_ = os.Remove(fn)\n\t\t\t\treturn errCorruptState\n\t\t\t}\n\t\t\tmb := fs.initMsgBlock(index)\n\t\t\tatomic.StoreUint64(&mb.first.seq, fseq)\n\t\t\tatomic.StoreUint64(&mb.last.seq, lseq)\n\t\t\tmb.msgs, mb.bytes = lseq-fseq+1, nbytes\n\t\t\tmb.first.ts, mb.last.ts = fts+baseTime, lts+baseTime\n\t\t\tmb.ttls = ttls\n\t\t\tmb.schedules = schedules\n\t\t\tif numDeleted > 0 {\n\t\t\t\tdmap, n, err := avl.Decode(buf[bi:])\n\t\t\t\tif err != nil {\n\t\t\t\t\t_ = os.Remove(fn)\n\t\t\t\t\tfs.warn(\"Stream state error decoding avl dmap: %v\", err)\n\t\t\t\t\treturn errCorruptState\n\t\t\t\t}\n\t\t\t\tmb.dmap = *dmap\n\t\t\t\tif mb.msgs > numDeleted {\n\t\t\t\t\tmb.msgs -= numDeleted\n\t\t\t\t} else {\n\t\t\t\t\tmb.msgs = 0\n\t\t\t\t}\n\t\t\t\tbi += n\n\t\t\t}\n\n\t\t\t// Pre-emptively mark block as closed, we'll confirm this block\n\t\t\t// still exists on disk and report it as lost if not.\n\t\t\tmb.closed = true\n\n\t\t\t// Only add in if not empty or the lmb.\n\t\t\tif mb.msgs > 0 || i == lastIndex {\n\t\t\t\tfs.addMsgBlock(mb)\n\t\t\t\tupdateTrackingState(&mstate, mb)\n\t\t\t} else {\n\t\t\t\t// Mark dirty to cleanup.\n\t\t\t\tfs.dirty++\n\t\t\t}\n\t\t}\n\t}\n\n\t// Pull in last block index for the block that had last checksum when we wrote the full state.\n\tblkIndex := uint32(readU64())\n\tvar lchk [8]byte\n\tif bi+len(lchk) > len(buf) {\n\t\tbi = -1\n\t} else {\n\t\tcopy(lchk[0:], buf[bi:bi+len(lchk)])\n\t}\n\n\t// Check if we had any errors.\n\tif bi < 0 {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state has no checksum present\")\n\t\treturn errCorruptState\n\t}\n\n\t// Move into place our state, msgBlks and subject info.\n\tfs.state = state\n\n\t// First let's check the happy path, open the blk file that was the lmb when we created the full state.\n\t// See if we have the last block available.\n\tvar matched bool\n\tmb := fs.lmb\n\tif mb == nil || mb.index != blkIndex {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state block does not exist or index mismatch\")\n\t\treturn errCorruptState\n\t}\n\tif _, err := os.Stat(mb.mfn); err != nil && os.IsNotExist(err) {\n\t\t// If our saved state is past what we see on disk, fallback and rebuild.\n\t\tif ld, _, _ := mb.rebuildState(); ld != nil {\n\t\t\tfs.addLostData(ld)\n\t\t}\n\t\tfs.warn(\"Stream state detected prior state, could not locate msg block %d\", blkIndex)\n\t\treturn errPriorState\n\t}\n\tif matched = bytes.Equal(mb.lastChecksum(), lchk[:]); !matched {\n\t\t// Detected a stale index.db, we didn't write it upon shutdown so can't rely on it being correct.\n\t\tfs.warn(\"Stream state outdated, last block has additional entries, will rebuild\")\n\t\treturn errPriorState\n\t}\n\n\t// We need to see if any blocks exist after our last one even though we matched the last record exactly.\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tvar dirs []os.DirEntry\n\n\t<-dios\n\tif f, err := os.Open(mdir); err == nil {\n\t\tdirs, _ = f.ReadDir(-1)\n\t\tf.Close()\n\t}\n\tdios <- struct{}{}\n\n\tvar index uint32\n\tfor _, fi := range dirs {\n\t\t// Ensure it's actually a block file, otherwise fmt.Sscanf also matches %d.blk.tmp\n\t\tif !strings.HasSuffix(fi.Name(), blkSuffix) {\n\t\t\tcontinue\n\t\t}\n\t\tif n, err := fmt.Sscanf(fi.Name(), blkScan, &index); err == nil && n == 1 {\n\t\t\tif index > blkIndex {\n\t\t\t\tfs.warn(\"Stream state outdated, found extra blocks, will rebuild\")\n\t\t\t\treturn errPriorState\n\t\t\t} else if mb, ok := fs.bim[index]; ok {\n\t\t\t\tmb.closed = false\n\t\t\t}\n\t\t}\n\t}\n\n\tvar rebuild bool\n\tfor _, mb := range fs.blks {\n\t\tif mb.closed {\n\t\t\trebuild = true\n\t\t\tif ld, _, _ := mb.rebuildState(); ld != nil {\n\t\t\t\tfs.addLostData(ld)\n\t\t\t}\n\t\t\tfs.warn(\"Stream state detected prior state, could not locate msg block %d\", mb.index)\n\t\t}\n\t}\n\tif rebuild {\n\t\treturn errPriorState\n\t}\n\n\t// We check first and last seq and number of msgs and bytes. If there is a difference,\n\t// return and error so we rebuild from the message block state on disk.\n\tif !trackingStatesEqual(&fs.state, &mstate) {\n\t\t_ = os.Remove(fn)\n\t\tfs.warn(\"Stream state encountered internal inconsistency on recover\")\n\t\treturn errCorruptState\n\t}\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) recoverTTLState() error {\n\t// See if we have a timed hash wheel for TTLs.\n\t<-dios\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, ttlStreamStateFile)\n\tbuf, err := os.ReadFile(fn)\n\tdios <- struct{}{}\n\n\tif err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\n\tfs.ttls = thw.NewHashWheel()\n\n\tvar ttlseq uint64\n\tif err == nil {\n\t\tttlseq, err = fs.ttls.Decode(buf)\n\t\tif err != nil {\n\t\t\tfs.warn(\"Error decoding TTL state: %s\", err)\n\t\t\t_ = os.Remove(fn)\n\t\t}\n\t}\n\n\tif ttlseq < fs.state.FirstSeq {\n\t\tttlseq = fs.state.FirstSeq\n\t}\n\n\tdefer fs.resetAgeChk(0)\n\tif fs.state.Msgs > 0 && ttlseq <= fs.state.LastSeq {\n\t\tfs.warn(\"TTL state is outdated; attempting to recover using linear scan (seq %d to %d)\", ttlseq, fs.state.LastSeq)\n\t\tvar (\n\t\t\tmb *msgBlock\n\t\t\tsm StoreMsg\n\t\t\tmblseq uint64\n\t\t)\n\t\tfor seq := ttlseq; seq <= fs.state.LastSeq; seq++ {\n\t\tretry:\n\t\t\tif mb == nil {\n\t\t\t\tif mb = fs.selectMsgBlock(seq); mb == nil {\n\t\t\t\t\t// Selecting the message block should return a block that contains this sequence,\n\t\t\t\t\t// or a later block if it can't be found.\n\t\t\t\t\t// It's an error if we can't find any block within the bounds of first and last seq.\n\t\t\t\t\tfs.warn(\"Error loading msg block with seq %d for recovering TTL\", seq)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tseq = atomic.LoadUint64(&mb.first.seq)\n\t\t\t\tmblseq = atomic.LoadUint64(&mb.last.seq)\n\t\t\t}\n\t\t\tif mb.ttls == 0 {\n\t\t\t\t// None of the messages in the block have message TTLs so don't\n\t\t\t\t// bother doing anything further with this block, skip to the end.\n\t\t\t\tseq = atomic.LoadUint64(&mb.last.seq) + 1\n\t\t\t}\n\t\t\tif seq > mblseq {\n\t\t\t\t// We've reached the end of the loaded block, so let's go back to the\n\t\t\t\t// beginning and process the next block.\n\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\tmb = nil\n\t\t\t\tif seq <= fs.state.LastSeq {\n\t\t\t\t\tgoto retry\n\t\t\t\t}\n\t\t\t\t// Done.\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tmsg, _, err := mb.fetchMsgNoCopy(seq, &sm)\n\t\t\tif err != nil {\n\t\t\t\tfs.warn(\"Error loading msg seq %d for recovering TTL: %s\", seq, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(msg.hdr) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif ttl, _ := getMessageTTL(msg.hdr); ttl > 0 {\n\t\t\t\texpires := time.Duration(msg.ts) + (time.Second * time.Duration(ttl))\n\t\t\t\tfs.ttls.Add(seq, int64(expires))\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) recoverMsgSchedulingState() error {\n\t// See if we have a timed hash wheel for TTLs.\n\t<-dios\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, msgSchedulingStreamStateFile)\n\tbuf, err := os.ReadFile(fn)\n\tdios <- struct{}{}\n\n\tif err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\n\tfs.scheduling = newMsgScheduling(fs.runMsgScheduling)\n\n\tvar schedSeq uint64\n\tif err == nil {\n\t\tschedSeq, err = fs.scheduling.decode(buf)\n\t\tif err != nil {\n\t\t\tfs.warn(\"Error decoding message scheduling state: %s\", err)\n\t\t\t_ = os.Remove(fn)\n\t\t}\n\t}\n\n\tif schedSeq < fs.state.FirstSeq {\n\t\tschedSeq = fs.state.FirstSeq\n\t}\n\n\tdefer fs.scheduling.resetTimer()\n\tif fs.state.Msgs > 0 && schedSeq <= fs.state.LastSeq {\n\t\tfs.warn(\"Message scheduling state is outdated; attempting to recover using linear scan (seq %d to %d)\", schedSeq, fs.state.LastSeq)\n\t\tvar (\n\t\t\tmb *msgBlock\n\t\t\tsm StoreMsg\n\t\t\tmblseq uint64\n\t\t)\n\t\tfor seq := schedSeq; seq <= fs.state.LastSeq; seq++ {\n\t\tretry:\n\t\t\tif mb == nil {\n\t\t\t\tif mb = fs.selectMsgBlock(seq); mb == nil {\n\t\t\t\t\t// Selecting the message block should return a block that contains this sequence,\n\t\t\t\t\t// or a later block if it can't be found.\n\t\t\t\t\t// It's an error if we can't find any block within the bounds of first and last seq.\n\t\t\t\t\tfs.warn(\"Error loading msg block with seq %d for recovering message schedules\", seq)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tseq = atomic.LoadUint64(&mb.first.seq)\n\t\t\t\tmblseq = atomic.LoadUint64(&mb.last.seq)\n\t\t\t}\n\t\t\tif mb.schedules == 0 {\n\t\t\t\t// None of the messages in the block have message schedules, so don't\n\t\t\t\t// bother doing anything further with this block, skip to the end.\n\t\t\t\tseq = atomic.LoadUint64(&mb.last.seq) + 1\n\t\t\t}\n\t\t\tif seq > mblseq {\n\t\t\t\t// We've reached the end of the loaded block, so let's go back to the\n\t\t\t\t// beginning and process the next block.\n\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\tmb = nil\n\t\t\t\tif seq <= fs.state.LastSeq {\n\t\t\t\t\tgoto retry\n\t\t\t\t}\n\t\t\t\t// Done.\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tmsg, _, err := mb.fetchMsgNoCopy(seq, &sm)\n\t\t\tif err != nil {\n\t\t\t\tfs.warn(\"Error loading msg seq %d for recovering message schedules: %s\", seq, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(msg.hdr) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif schedule, ok := nextMessageSchedule(sm.hdr, sm.ts); ok && !schedule.IsZero() {\n\t\t\t\tfs.scheduling.init(seq, sm.subj, schedule.UnixNano())\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Grabs last checksum for the named block file.\n// Takes into account encryption etc.\nfunc (mb *msgBlock) lastChecksum() []byte {\n\tf, err := mb.openBlock()\n\tif err != nil {\n\t\treturn nil\n\t}\n\tdefer f.Close()\n\n\tvar lchk [8]byte\n\tif fi, _ := f.Stat(); fi != nil {\n\t\tmb.rbytes = uint64(fi.Size())\n\t}\n\tif mb.rbytes < checksumSize {\n\t\treturn lchk[:]\n\t}\n\t// Encrypted?\n\tif err = mb.checkAndLoadEncryption(); err != nil {\n\t\treturn nil\n\t}\n\tif mb.bek != nil {\n\t\tif buf, _ := mb.loadBlock(nil); len(buf) >= checksumSize {\n\t\t\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tcopy(lchk[0:], buf[len(buf)-checksumSize:])\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t} else if _, err = f.ReadAt(lchk[:], int64(mb.rbytes)-checksumSize); err != nil {\n\t\treturn nil\n\t}\n\treturn lchk[:]\n}\n\n// This will make sure we clean up old idx and fss files.\nfunc (fs *fileStore) cleanupOldMeta() {\n\tfs.mu.RLock()\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tfs.mu.RUnlock()\n\n\t<-dios\n\tf, err := os.Open(mdir)\n\tdios <- struct{}{}\n\tif err != nil {\n\t\treturn\n\t}\n\n\tdirs, _ := f.ReadDir(-1)\n\tf.Close()\n\n\tconst (\n\t\tminLen = 4\n\t\tidxSuffix = \".idx\"\n\t\tfssSuffix = \".fss\"\n\t)\n\tfor _, fi := range dirs {\n\t\tif name := fi.Name(); strings.HasSuffix(name, idxSuffix) || strings.HasSuffix(name, fssSuffix) {\n\t\t\tos.Remove(filepath.Join(mdir, name))\n\t\t}\n\t}\n}\n\nfunc (fs *fileStore) recoverMsgs() error {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Check for any left over purged messages.\n\t<-dios\n\tif err := fs.recoverPartialPurge(); err != nil {\n\t\tdios <- struct{}{}\n\t\treturn err\n\t}\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tf, err := os.Open(mdir)\n\tif err != nil {\n\t\tdios <- struct{}{}\n\t\treturn errNotReadable\n\t}\n\tdirs, err := f.ReadDir(-1)\n\tf.Close()\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\treturn errNotReadable\n\t}\n\n\tindices := make(sort.IntSlice, 0, len(dirs))\n\tvar index int\n\tfor _, fi := range dirs {\n\t\t// Ensure it's actually a block file, otherwise fmt.Sscanf also matches %d.blk.tmp\n\t\tif !strings.HasSuffix(fi.Name(), blkSuffix) {\n\t\t\tcontinue\n\t\t}\n\t\tif n, err := fmt.Sscanf(fi.Name(), blkScan, &index); err == nil && n == 1 {\n\t\t\tindices = append(indices, index)\n\t\t}\n\t}\n\tindices.Sort()\n\n\t// Recover all of the msg blocks.\n\t// We now guarantee they are coming in order.\n\tfor _, index := range indices {\n\t\tif mb, err := fs.recoverMsgBlock(uint32(index)); err == nil && mb != nil {\n\t\t\t// This is a truncate block with possibly no index. If the OS got shutdown\n\t\t\t// out from underneath of us this is possible.\n\t\t\tmb.mu.Lock()\n\t\t\tif atomic.LoadUint64(&mb.first.seq) == 0 {\n\t\t\t\tif err := mb.dirtyCloseWithRemove(true); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tfs.removeMsgBlockFromList(mb)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If the stream is empty, reset the first/last sequences so these can\n\t\t\t// properly move up based purely on tombstones spread over multiple blocks.\n\t\t\tif fs.state.Msgs == 0 {\n\t\t\t\tfs.state.FirstSeq, fs.state.LastSeq = 0, 0\n\t\t\t\tfs.state.FirstTime, fs.state.LastTime = time.Time{}, time.Time{}\n\t\t\t}\n\t\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t\tif fs.state.FirstSeq == 0 || (fseq < fs.state.FirstSeq && mb.first.ts != 0) {\n\t\t\t\tfs.state.FirstSeq = fseq\n\t\t\t\tif mb.first.ts == 0 {\n\t\t\t\t\tfs.state.FirstTime = time.Time{}\n\t\t\t\t} else {\n\t\t\t\t\tfs.state.FirstTime = time.Unix(0, mb.first.ts).UTC()\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Preserve last time, could have erased the last message in one block, and then\n\t\t\t// have a tombstone with the proper timestamp afterward in another block\n\t\t\tif lseq := atomic.LoadUint64(&mb.last.seq); lseq >= fs.state.LastSeq {\n\t\t\t\tfs.state.LastSeq = lseq\n\t\t\t\tif mb.last.ts == 0 {\n\t\t\t\t\tfs.state.LastTime = time.Time{}\n\t\t\t\t} else {\n\t\t\t\t\tfs.state.LastTime = time.Unix(0, mb.last.ts).UTC()\n\t\t\t\t}\n\t\t\t}\n\t\t\tfs.state.Msgs += mb.msgs\n\t\t\tfs.state.Bytes += mb.bytes\n\t\t\t// If the block is empty, correct the sequences to be aligned with the current filestore state.\n\t\t\tif mb.msgs == 0 {\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, fs.state.LastSeq+1)\n\t\t\t\tmb.first.ts = 0\n\t\t\t\tatomic.StoreUint64(&mb.last.seq, fs.state.LastSeq)\n\t\t\t\tmb.last.ts = fs.state.LastTime.UnixNano()\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif len(fs.blks) > 0 {\n\t\tfs.lmb = fs.blks[len(fs.blks)-1]\n\t} else if _, err = fs.newMsgBlockForWrite(); err != nil {\n\t\treturn err\n\t}\n\n\t// Check if we encountered any lost data.\n\tif fs.ld != nil {\n\t\tvar emptyBlks []*msgBlock\n\t\tfor _, mb := range fs.blks {\n\t\t\tif mb.msgs == 0 && mb.rbytes == 0 && mb != fs.lmb {\n\t\t\t\temptyBlks = append(emptyBlks, mb)\n\t\t\t}\n\t\t}\n\t\tfor _, mb := range emptyBlks {\n\t\t\t// Need the mb lock here.\n\t\t\tmb.mu.Lock()\n\t\t\terr = fs.forceRemoveMsgBlock(mb)\n\t\t\tmb.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check for keyfiles orphans.\n\tif kms, err := filepath.Glob(filepath.Join(mdir, keyScanAll)); err == nil && len(kms) > 0 {\n\t\tvalid := make(map[uint32]bool)\n\t\tfor _, mb := range fs.blks {\n\t\t\tvalid[mb.index] = true\n\t\t}\n\t\tfor _, fn := range kms {\n\t\t\tvar index uint32\n\t\t\tshouldRemove := true\n\t\t\tif n, err := fmt.Sscanf(filepath.Base(fn), keyScan, &index); err == nil && n == 1 && valid[index] {\n\t\t\t\tshouldRemove = false\n\t\t\t}\n\t\t\tif shouldRemove {\n\t\t\t\tos.Remove(fn)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Will expire msgs that have aged out on restart.\n// We will treat this differently in case we have a recovery\n// that will expire alot of messages on startup.\n// Should only be called on startup.\nfunc (fs *fileStore) expireMsgsOnRecover() error {\n\tif fs.state.Msgs == 0 {\n\t\treturn nil\n\t}\n\n\t// If subject delete markers is configured, can't expire on recover.\n\t// When clustered we need to go through proposals.\n\tif fs.cfg.SubjectDeleteMarkerTTL > 0 {\n\t\treturn nil\n\t}\n\n\tvar minAge = time.Now().UnixNano() - int64(fs.cfg.MaxAge)\n\tvar purged, bytes uint64\n\tvar deleted int\n\tvar nts int64\n\n\t// If we expire all make sure to write out a tombstone. Need to be done by hand here,\n\t// usually taken care of by fs.removeMsgBlock() but we do not call that here.\n\tvar last msgId\n\n\tdeleteEmptyBlock := func(mb *msgBlock) error {\n\t\t// If we are the last keep state to remember first/last sequence.\n\t\t// Do this part by hand since not deleting one by one.\n\t\tif mb == fs.lmb {\n\t\t\tlast.seq = atomic.LoadUint64(&mb.last.seq)\n\t\t\tlast.ts = mb.last.ts\n\t\t}\n\t\t// Make sure we do subject cleanup as well.\n\t\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmb.fss.IterOrdered(func(bsubj []byte, ss *SimpleState) bool {\n\t\t\tsubj := bytesToString(bsubj)\n\t\t\tfor i := uint64(0); i < ss.Msgs; i++ {\n\t\t\t\tfs.removePerSubject(subj)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif err := mb.dirtyCloseWithRemove(true); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdeleted++\n\t\treturn nil\n\t}\n\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tif minAge < mb.first.ts {\n\t\t\tnts = mb.first.ts\n\t\t\tmb.mu.Unlock()\n\t\t\tbreak\n\t\t}\n\t\t// Can we remove whole block here?\n\t\tif mb.last.ts <= minAge {\n\t\t\tpurged += mb.msgs\n\t\t\tbytes += mb.bytes\n\t\t\terr := deleteEmptyBlock(mb)\n\t\t\tmb.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// If we are here we have to process the interior messages of this blk.\n\t\t// This will load fss as well.\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\n\t\tvar smv StoreMsg\n\t\tvar needNextFirst bool\n\n\t\t// Walk messages and remove if expired.\n\t\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\t\tfor seq := fseq; seq <= lseq; seq++ {\n\t\t\tsm, err := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t// Process interior deleted msgs.\n\t\t\tif err == errDeletedMsg {\n\t\t\t\t// Update dmap.\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\tmb.dmap.Delete(seq)\n\t\t\t\t}\n\t\t\t\t// Keep this updated just in case since we are removing dmap entries.\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\t\t\tneedNextFirst = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Break on other errors.\n\t\t\tif err != nil || sm == nil {\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\t\t\tneedNextFirst = true\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\t// No error and sm != nil from here onward.\n\n\t\t\t// Check for done.\n\t\t\tif minAge < sm.ts {\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, sm.seq)\n\t\t\t\tmb.first.ts = sm.ts\n\t\t\t\tneedNextFirst = false\n\t\t\t\tnts = sm.ts\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\t// Delete the message here.\n\t\t\tif mb.msgs > 0 {\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\t\t\tneedNextFirst = true\n\t\t\t\tsz := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t\t\tif sz > mb.bytes {\n\t\t\t\t\tsz = mb.bytes\n\t\t\t\t}\n\t\t\t\tmb.bytes -= sz\n\t\t\t\tbytes += sz\n\t\t\t\tmb.msgs--\n\t\t\t\tpurged++\n\t\t\t}\n\t\t\t// Update fss\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif _, err = mb.removeSeqPerSubject(sm.subj, seq); err != nil {\n\t\t\t\tmb.finishedWithCache()\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfs.removePerSubject(sm.subj)\n\t\t}\n\t\t// Make sure we have a proper next first sequence.\n\t\tif needNextFirst {\n\t\t\tmb.selectNextFirst()\n\t\t}\n\t\t// Check if empty after processing, could happen if tail of messages are all deleted.\n\t\tvar err error\n\t\tif mb.msgs == 0 {\n\t\t\terr = deleteEmptyBlock(mb)\n\t\t}\n\t\tmb.finishedWithCache()\n\t\tmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tbreak\n\t}\n\n\tif nts > 0 {\n\t\t// Make sure to set age check based on this value.\n\t\tfs.resetAgeChk(nts - minAge)\n\t}\n\n\tif deleted > 0 {\n\t\t// Update block map.\n\t\tif fs.bim != nil {\n\t\t\tfor _, mb := range fs.blks[:deleted] {\n\t\t\t\tdelete(fs.bim, mb.index)\n\t\t\t}\n\t\t}\n\t\t// Update blks slice.\n\t\tfs.blks = copyMsgBlocks(fs.blks[deleted:])\n\t\tif lb := len(fs.blks); lb == 0 {\n\t\t\tfs.lmb = nil\n\t\t} else {\n\t\t\tfs.lmb = fs.blks[lb-1]\n\t\t}\n\t}\n\t// Update top level accounting.\n\tif purged < fs.state.Msgs {\n\t\tfs.state.Msgs -= purged\n\t} else {\n\t\tfs.state.Msgs = 0\n\t}\n\tif bytes < fs.state.Bytes {\n\t\tfs.state.Bytes -= bytes\n\t} else {\n\t\tfs.state.Bytes = 0\n\t}\n\t// Make sure to we properly set the fs first sequence and timestamp.\n\tif err := fs.selectNextFirst(); err != nil {\n\t\treturn err\n\t}\n\n\t// Check if we have no messages and blocks left.\n\n\tif fs.lmb == nil && last.seq != 0 {\n\t\tif lmb, err := fs.newMsgBlockForWrite(); err != nil || lmb == nil {\n\t\t\tif err == nil {\n\t\t\t\terr = errors.New(\"lmb missing\")\n\t\t\t}\n\t\t\treturn err\n\t\t} else if err = fs.writeTombstone(last.seq, last.ts); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Clear any global subject state.\n\t\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\t}\n\n\t// If we purged anything, make sure we kick flush state loop.\n\tif purged > 0 {\n\t\tfs.dirty++\n\t}\n\treturn nil\n}\n\nfunc copyMsgBlocks(src []*msgBlock) []*msgBlock {\n\tif src == nil {\n\t\treturn nil\n\t}\n\tdst := make([]*msgBlock, len(src))\n\tcopy(dst, src)\n\treturn dst\n}\n\n// GetSeqFromTime looks for the first sequence number that has\n// the message with >= timestamp.\nfunc (fs *fileStore) GetSeqFromTime(t time.Time) uint64 {\n\tfs.mu.RLock()\n\tlastSeq := fs.state.LastSeq\n\tclosed := fs.isClosed()\n\tfs.mu.RUnlock()\n\n\tif closed {\n\t\treturn 0\n\t}\n\n\tmb := fs.selectMsgBlockForStart(t)\n\tif mb == nil {\n\t\treturn lastSeq + 1\n\t}\n\n\tfseq := atomic.LoadUint64(&mb.first.seq)\n\tlseq := atomic.LoadUint64(&mb.last.seq)\n\n\tvar (\n\t\tsmv StoreMsg\n\t\tcts int64\n\t\tcseq uint64\n\t\toff uint64\n\t)\n\tts := t.UnixNano()\n\n\t// Using a binary search, but need to be aware of interior deletes in the block.\n\tseq := lseq + 1\nloop:\n\tfor fseq <= lseq {\n\t\tmid := fseq + (lseq-fseq)/2\n\t\toff = 0\n\t\t// Potentially skip over gaps. We keep the original middle but keep track of a\n\t\t// potential delete range with an offset.\n\t\tfor {\n\t\t\tsm, _, err := mb.fetchMsgNoCopy(mid+off, &smv)\n\t\t\tif err != nil || sm == nil {\n\t\t\t\toff++\n\t\t\t\tif mid+off <= lseq {\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\t// Continue search to the left. Purposely ignore the skipped deletes here.\n\t\t\t\t\tlseq = mid - 1\n\t\t\t\t\tcontinue loop\n\t\t\t\t}\n\t\t\t}\n\t\t\tcts = sm.ts\n\t\t\tcseq = sm.seq\n\t\t\tbreak\n\t\t}\n\t\tif cts >= ts {\n\t\t\tseq = cseq\n\t\t\tif mid == fseq {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// Continue search to the left.\n\t\t\tlseq = mid - 1\n\t\t} else {\n\t\t\t// Continue search to the right (potentially skipping over interior deletes).\n\t\t\tfseq = mid + off + 1\n\t\t}\n\t}\n\treturn seq\n}\n\n// Find the first matching message against a sublist.\nfunc (mb *msgBlock) firstMatchingMulti(sl *gsl.SimpleSublist, start uint64, sm *StoreMsg) (*StoreMsg, bool, error) {\n\tmb.mu.Lock()\n\tvar didLoad bool\n\tvar updateLLTS bool\n\tdefer func() {\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.finishedWithCache()\n\t\tmb.mu.Unlock()\n\t}()\n\n\tif mb.fssNotLoaded() {\n\t\t// Make sure we have fss loaded.\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t\tdidLoad = true\n\t}\n\t// Mark fss activity.\n\tmb.lsts = ats.AccessTime()\n\n\t// Make sure to start at mb.first.seq if fseq < mb.first.seq\n\tif seq := atomic.LoadUint64(&mb.first.seq); seq > start {\n\t\tstart = seq\n\t}\n\tlseq := atomic.LoadUint64(&mb.last.seq)\n\n\t// If the FSS state has fewer entries than sequences in the linear scan,\n\t// then use intersection instead as likely going to be cheaper. This will\n\t// often be the case with high numbers of deletes, as well as a smaller\n\t// number of subjects in the block.\n\tif uint64(mb.fss.Size()) < lseq-start {\n\t\t// If there are no subject matches then this is effectively no-op.\n\t\thseq := uint64(math.MaxUint64)\n\t\tvar ierr error\n\t\tstree.IntersectGSL(mb.fss, sl, func(subj []byte, ss *SimpleState) {\n\t\t\tif ierr != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\t// mb is already loaded into the cache so should be fast-ish.\n\t\t\t\tif ierr = mb.recalculateForSubj(bytesToString(subj), ss); ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tfirst := max(start, ss.First)\n\t\t\tif first > ss.Last || first >= hseq {\n\t\t\t\t// The start cutoff is after the last sequence for this subject,\n\t\t\t\t// or we think we already know of a subject with an earlier msg\n\t\t\t\t// than our first seq for this subject.\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Need messages loaded from here on out.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif ierr = mb.loadMsgsWithLock(); ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdidLoad = true\n\t\t\t}\n\t\t\tif sm == nil {\n\t\t\t\tsm = new(StoreMsg)\n\t\t\t}\n\t\t\tif first == ss.First {\n\t\t\t\t// If the start floor is below where this subject starts then we can\n\t\t\t\t// short-circuit, avoiding needing to scan for the next message.\n\t\t\t\tif fsm, err := mb.cacheLookup(ss.First, sm); err == nil {\n\t\t\t\t\tsm = fsm\n\t\t\t\t\thseq = ss.First\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor seq := first; seq <= ss.Last; seq++ {\n\t\t\t\t// Otherwise we have a start floor that intersects where this subject\n\t\t\t\t// has messages in the block, so we need to walk up until we find a\n\t\t\t\t// message matching the subject.\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\t// Instead we will update it only once in a defer.\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tllseq := mb.llseq\n\t\t\t\tfsm, err := mb.cacheLookup(seq, sm)\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\tif sl.HasInterest(fsm.subj) {\n\t\t\t\t\thseq = seq\n\t\t\t\t\tsm = fsm\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// If we are here we did not match, so put the llseq back.\n\t\t\t\tmb.llseq = llseq\n\t\t\t}\n\t\t})\n\t\tif ierr != nil {\n\t\t\treturn nil, false, ierr\n\t\t}\n\t\tif hseq < uint64(math.MaxUint64) && sm != nil {\n\t\t\treturn sm, didLoad && start == lseq, nil\n\t\t}\n\t} else {\n\t\t// Need messages loaded from here on out.\n\t\tif mb.cacheNotLoaded() {\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\treturn nil, false, err\n\t\t\t}\n\t\t\tdidLoad = true\n\t\t}\n\t\tif sm == nil {\n\t\t\tsm = new(StoreMsg)\n\t\t}\n\n\t\tfor seq := start; seq <= lseq; seq++ {\n\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t// Instead we will update it only once in a defer.\n\t\t\t\tupdateLLTS = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tllseq := mb.llseq\n\t\t\tfsm, err := mb.cacheLookup(seq, sm)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\texpireOk := seq == lseq && mb.llseq != llseq && mb.llseq == seq\n\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\tif sl.HasInterest(fsm.subj) {\n\t\t\t\treturn fsm, expireOk, nil\n\t\t\t}\n\t\t\t// If we are here we did not match, so put the llseq back.\n\t\t\tmb.llseq = llseq\n\t\t}\n\t}\n\n\treturn nil, didLoad, ErrStoreMsgNotFound\n}\n\n// Find the first matching message.\n// fs lock should be held.\nfunc (mb *msgBlock) firstMatching(filter string, wc bool, start uint64, sm *StoreMsg) (*StoreMsg, bool, error) {\n\tmb.mu.Lock()\n\tvar updateLLTS bool\n\tdefer func() {\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.finishedWithCache()\n\t\tmb.mu.Unlock()\n\t}()\n\n\tfseq, isAll := start, filter == _EMPTY_ || filter == fwcs\n\n\tvar didLoad bool\n\tif mb.fssNotLoaded() {\n\t\t// Make sure we have fss loaded.\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t\tdidLoad = true\n\t}\n\t// Mark fss activity.\n\tmb.lsts = ats.AccessTime()\n\n\tif filter == _EMPTY_ {\n\t\tfilter = fwcs\n\t\twc = true\n\t}\n\n\t// If we only have 1 subject currently and it matches our filter we can also set isAll.\n\tif !isAll && mb.fss.Size() == 1 {\n\t\tif !wc {\n\t\t\t_, isAll = mb.fss.Find(stringToBytes(filter))\n\t\t} else {\n\t\t\t// Since mb.fss.Find won't work if filter is a wildcard, need to use Match instead.\n\t\t\tmb.fss.Match(stringToBytes(filter), func(subject []byte, _ *SimpleState) {\n\t\t\t\tisAll = true\n\t\t\t})\n\t\t}\n\t\t// If the only subject in this block isn't our filter, can simply short-circuit.\n\t\tif !isAll {\n\t\t\treturn nil, didLoad, ErrStoreMsgNotFound\n\t\t}\n\t}\n\t// Make sure to start at mb.first.seq if fseq < mb.first.seq\n\tfseq = max(fseq, atomic.LoadUint64(&mb.first.seq))\n\tlseq := atomic.LoadUint64(&mb.last.seq)\n\n\t// Optionally build the isMatch for wildcard filters.\n\tvar isMatch func(subj string) bool\n\t// Decide to build.\n\tif wc {\n\t\t_tsa, _fsa := [32]string{}, [32]string{}\n\t\ttsa, fsa := _tsa[:0], tokenizeSubjectIntoSlice(_fsa[:0], filter)\n\t\tisMatch = func(subj string) bool {\n\t\t\ttsa = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\t\treturn isSubsetMatchTokenized(tsa, fsa)\n\t\t}\n\t}\n\n\tsubjs := mb.fs.cfg.Subjects\n\t// If isAll or our single filter matches the filter arg do linear scan.\n\tdoLinearScan := isAll || (wc && len(subjs) == 1 && subjs[0] == filter)\n\t// If we do not think we should do a linear scan check how many fss we\n\t// would need to scan vs the full range of the linear walk. Optimize for\n\t// 25th quantile of a match in a linear walk. Filter should be a wildcard.\n\t// We should consult fss if our cache is not loaded and we only have fss loaded.\n\tif !doLinearScan && wc && mb.cacheAlreadyLoaded() {\n\t\tdoLinearScan = mb.fss.Size()*4 > int(lseq-fseq)\n\t}\n\n\tif !doLinearScan {\n\t\t// If we have a wildcard match against all tracked subjects we know about.\n\t\tfseq = lseq + 1\n\t\tif bfilter := stringToBytes(filter); wc {\n\t\t\tvar ierr error\n\t\t\tmb.fss.Match(bfilter, func(bsubj []byte, ss *SimpleState) {\n\t\t\t\tif ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\t\tif ierr = mb.recalculateForSubj(bytesToString(bsubj), ss); ierr != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif start <= ss.Last {\n\t\t\t\t\tfseq = min(fseq, max(start, ss.First))\n\t\t\t\t}\n\t\t\t})\n\t\t\tif ierr != nil {\n\t\t\t\treturn nil, false, ierr\n\t\t\t}\n\t\t} else if ss, _ := mb.fss.Find(bfilter); ss != nil {\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tif err := mb.recalculateForSubj(filter, ss); err != nil {\n\t\t\t\t\treturn nil, false, err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif start <= ss.Last {\n\t\t\t\tfseq = min(fseq, max(start, ss.First))\n\t\t\t}\n\t\t}\n\t}\n\n\tif fseq > lseq {\n\t\treturn nil, didLoad, ErrStoreMsgNotFound\n\t}\n\n\t// Need messages loaded from here on out.\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t\tdidLoad = true\n\t}\n\n\tif sm == nil {\n\t\tsm = new(StoreMsg)\n\t}\n\n\tfor seq := fseq; seq <= lseq; seq++ {\n\t\tif mb.dmap.Exists(seq) {\n\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t// Instead we will update it only once in a defer.\n\t\t\tupdateLLTS = true\n\t\t\tcontinue\n\t\t}\n\t\tllseq := mb.llseq\n\t\tfsm, err := mb.cacheLookup(seq, sm)\n\t\tif err != nil {\n\t\t\tif err == errPartialCache || err == errNoCache {\n\t\t\t\treturn nil, false, err\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\texpireOk := seq == lseq && mb.llseq != llseq && mb.llseq == seq\n\t\tif isAll {\n\t\t\treturn fsm, expireOk, nil\n\t\t}\n\t\tif wc && isMatch(sm.subj) {\n\t\t\treturn fsm, expireOk, nil\n\t\t} else if !wc && fsm.subj == filter {\n\t\t\treturn fsm, expireOk, nil\n\t\t}\n\t\t// If we are here we did not match, so put the llseq back.\n\t\tmb.llseq = llseq\n\t}\n\n\treturn nil, didLoad, ErrStoreMsgNotFound\n}\n\n// Find the previous matching message against a sublist, working BACKWARDS from start.\nfunc (mb *msgBlock) prevMatchingMulti(sl *gsl.SimpleSublist, start uint64, sm *StoreMsg) (*StoreMsg, bool, error) {\n\tmb.mu.Lock()\n\tvar didLoad bool\n\tvar updateLLTS bool\n\tdefer func() {\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.finishedWithCache()\n\t\tmb.mu.Unlock()\n\t}()\n\n\t// Need messages loaded from here on out.\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t\tdidLoad = true\n\t}\n\n\t// Make sure to start at mb.last.seq if lseq < mb.last.seq\n\tif seq := atomic.LoadUint64(&mb.last.seq); start > seq {\n\t\tstart = seq\n\t}\n\tlseq := atomic.LoadUint64(&mb.first.seq)\n\n\tif sm == nil {\n\t\tsm = new(StoreMsg)\n\t}\n\n\t// If the FSS state has fewer entries than sequences in the linear scan,\n\t// then use intersection instead as likely going to be cheaper. This will\n\t// often be the case with high numbers of deletes, as well as a smaller\n\t// number of subjects in the block.\n\tif uint64(mb.fss.Size()) < start-lseq {\n\t\t// If there are no subject matches then this is effectively no-op.\n\t\thseq := uint64(0)\n\t\tvar ierr error\n\t\tstree.IntersectGSL(mb.fss, sl, func(subj []byte, ss *SimpleState) {\n\t\t\tif ierr != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\t// mb is already loaded into the cache so should be fast-ish.\n\t\t\t\tif ierr = mb.recalculateForSubj(bytesToString(subj), ss); ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tfirst := min(start, ss.Last)\n\t\t\t// Skip if cutoff is before this subject's first, or if we already\n\t\t\t// have a higher-or-equal candidate (hseq holds the highest found).\n\t\t\tif first < ss.First || first <= hseq {\n\t\t\t\t// The start cutoff is before the first sequence for this subject,\n\t\t\t\t// or we already know of a subject with a later-or-equal msg.\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif first == ss.Last {\n\t\t\t\t// If the start floor is above where this subject starts then we can\n\t\t\t\t// short-circuit, avoiding needing to scan for the next message.\n\t\t\t\tif fsm, err := mb.cacheLookup(ss.Last, sm); err == nil {\n\t\t\t\t\tsm = fsm\n\t\t\t\t\thseq = ss.Last\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor seq := first; seq >= ss.First; seq-- {\n\t\t\t\t// Otherwise we have a start floor that intersects where this subject\n\t\t\t\t// has messages in the block, so we need to walk up until we find a\n\t\t\t\t// message matching the subject.\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\t// Instead we will update it only once in a defer.\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tllseq := mb.llseq\n\t\t\t\tfsm, err := mb.cacheLookup(seq, sm)\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\tif sl.HasInterest(fsm.subj) {\n\t\t\t\t\thseq = seq\n\t\t\t\t\tsm = fsm\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// If we are here we did not match, so put the llseq back.\n\t\t\t\tmb.llseq = llseq\n\t\t\t}\n\t\t})\n\t\tif ierr != nil {\n\t\t\treturn nil, false, ierr\n\t\t}\n\t\tif hseq > 0 && sm != nil {\n\t\t\treturn sm, didLoad && start == lseq, nil\n\t\t}\n\t} else {\n\t\tfor seq := start; seq >= lseq; seq-- {\n\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t// Instead we will update it only once in a defer.\n\t\t\t\tupdateLLTS = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tllseq := mb.llseq\n\t\t\tfsm, err := mb.cacheLookup(seq, sm)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\texpireOk := seq == lseq && mb.llseq != llseq && mb.llseq == seq\n\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\tif sl.HasInterest(fsm.subj) {\n\t\t\t\treturn fsm, expireOk, nil\n\t\t\t}\n\t\t\t// If we are here we did not match, so put the llseq back.\n\t\t\tmb.llseq = llseq\n\t\t}\n\t}\n\n\treturn nil, didLoad, ErrStoreMsgNotFound\n}\n\n// This will traverse a message block and generate the filtered pending.\nfunc (mb *msgBlock) filteredPending(subj string, wc bool, seq uint64) (total, first, last uint64, err error) {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.filteredPendingLocked(subj, wc, seq)\n}\n\n// This will traverse a message block and generate the filtered pending.\n// Lock should be held.\nfunc (mb *msgBlock) filteredPendingLocked(filter string, wc bool, sseq uint64) (total, first, last uint64, err error) {\n\tisAll := filter == _EMPTY_ || filter == fwcs\n\n\t// First check if we can optimize this part.\n\t// This means we want all and the starting sequence was before this block.\n\tif isAll {\n\t\tif fseq := atomic.LoadUint64(&mb.first.seq); sseq <= fseq {\n\t\t\treturn mb.msgs, fseq, atomic.LoadUint64(&mb.last.seq), nil\n\t\t}\n\t}\n\n\tif filter == _EMPTY_ {\n\t\tfilter, wc = fwcs, true\n\t}\n\n\tupdate := func(ss *SimpleState) {\n\t\ttotal += ss.Msgs\n\t\tif first == 0 || ss.First < first {\n\t\t\tfirst = ss.First\n\t\t}\n\t\tif ss.Last > last {\n\t\t\tlast = ss.Last\n\t\t}\n\t}\n\n\t// Make sure we have fss loaded.\n\tif err = mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\n\tvar havePartial bool\n\n\t// If we are not a wildcard just use Find() here. Avoids allocations.\n\tif !wc {\n\t\tif ss, ok := mb.fss.Find(stringToBytes(filter)); ok && ss != nil {\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tif err = mb.recalculateForSubj(filter, ss); err != nil {\n\t\t\t\t\treturn 0, 0, 0, err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif sseq <= ss.First {\n\t\t\t\tupdate(ss)\n\t\t\t} else if sseq <= ss.Last {\n\t\t\t\t// We matched but its a partial.\n\t\t\t\thavePartial = true\n\t\t\t}\n\t\t}\n\t} else {\n\t\tmb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) {\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif havePartial {\n\t\t\t\t// If we already found a partial then don't do anything else.\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tif err = mb.recalculateForSubj(bytesToString(bsubj), ss); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tif sseq <= ss.First {\n\t\t\t\tupdate(ss)\n\t\t\t} else if sseq <= ss.Last {\n\t\t\t\t// We matched but its a partial.\n\t\t\t\thavePartial = true\n\t\t\t}\n\t\t})\n\t\tif err != nil {\n\t\t\treturn 0, 0, 0, err\n\t\t}\n\t}\n\n\t// If we did not encounter any partials we can return here.\n\tif !havePartial {\n\t\treturn total, first, last, nil\n\t}\n\n\t// If we are here we need to scan the msgs.\n\t// Clear what we had.\n\ttotal, first, last = 0, 0, 0\n\n\t// If we load the cache for a linear scan we want to expire that cache upon exit.\n\tvar shouldExpire bool\n\tif mb.cacheNotLoaded() {\n\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn 0, 0, 0, err\n\t\t}\n\t\tshouldExpire = true\n\t}\n\tdefer mb.finishedWithCache()\n\n\t_tsa, _fsa := [32]string{}, [32]string{}\n\ttsa, fsa := _tsa[:0], _fsa[:0]\n\tvar isMatch func(subj string) bool\n\n\tif !wc {\n\t\tisMatch = func(subj string) bool { return subj == filter }\n\t} else {\n\t\tfsa = tokenizeSubjectIntoSlice(fsa[:0], filter)\n\t\tisMatch = func(subj string) bool {\n\t\t\ttsa = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\t\treturn isSubsetMatchTokenized(tsa, fsa)\n\t\t}\n\t}\n\n\t// 1. See if we match any subs from fss.\n\t// 2. If we match and the sseq is past ss.Last then we can use meta only.\n\t// 3. If we match and we need to do a partial, break and clear any totals and do a full scan like num pending.\n\n\tvar smv StoreMsg\n\tfor seq, lseq := sseq, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ {\n\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\tif sm == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif isAll || isMatch(sm.subj) {\n\t\t\ttotal++\n\t\t\tif first == 0 || seq < first {\n\t\t\t\tfirst = seq\n\t\t\t}\n\t\t\tif seq > last {\n\t\t\t\tlast = seq\n\t\t\t}\n\t\t}\n\t}\n\t// If we loaded this block for this operation go ahead and expire it here.\n\tif shouldExpire {\n\t\tmb.tryForceExpireCacheLocked()\n\t}\n\n\treturn total, first, last, nil\n}\n\n// FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence.\nfunc (fs *fileStore) FilteredState(sseq uint64, subj string) (SimpleState, error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tlseq := fs.state.LastSeq\n\tif sseq < fs.state.FirstSeq {\n\t\tsseq = fs.state.FirstSeq\n\t}\n\n\t// Returned state.\n\tvar ss SimpleState\n\n\t// If past the end no results.\n\tif sseq > lseq {\n\t\t// Make sure we track sequences\n\t\tss.First = fs.state.FirstSeq\n\t\tss.Last = fs.state.LastSeq\n\t\treturn ss, nil\n\t}\n\n\t// If we want all msgs that match we can shortcircuit.\n\t// TODO(dlc) - This can be extended for all cases but would\n\t// need to be careful on total msgs calculations etc.\n\tif sseq == fs.state.FirstSeq {\n\t\tif err := fs.numFilteredPending(subj, &ss); err != nil {\n\t\t\treturn ss, err\n\t\t}\n\t} else {\n\t\twc := subjectHasWildcard(subj)\n\t\t// Tracking subject state.\n\t\t// TODO(dlc) - Optimize for 2.10 with avl tree and no atomics per block.\n\t\tfor _, mb := range fs.blks {\n\t\t\t// Skip blocks that are less than our starting sequence.\n\t\t\tif sseq > atomic.LoadUint64(&mb.last.seq) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tt, f, l, err := mb.filteredPending(subj, wc, sseq)\n\t\t\tif err != nil {\n\t\t\t\treturn ss, err\n\t\t\t}\n\t\t\tss.Msgs += t\n\t\t\tif ss.First == 0 || (f > 0 && f < ss.First) {\n\t\t\t\tss.First = f\n\t\t\t}\n\t\t\tif l > ss.Last {\n\t\t\t\tss.Last = l\n\t\t\t}\n\t\t}\n\t}\n\n\treturn ss, nil\n}\n\n// This is used to see if we can selectively jump start blocks based on filter subject and a starting block index.\n// Will return -1 and ErrStoreEOF if no matches at all or no more from where we are.\nfunc (fs *fileStore) checkSkipFirstBlock(filter string, wc bool, bi int) (int, error) {\n\t// If we match everything, just move to next blk.\n\tif filter == _EMPTY_ || filter == fwcs {\n\t\treturn bi + 1, nil\n\t}\n\t// Move through psim to gather start and stop bounds.\n\tstart, stop := uint32(math.MaxUint32), uint32(0)\n\tif wc {\n\t\tfs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) {\n\t\t\tif psi.fblk < start {\n\t\t\t\tstart = psi.fblk\n\t\t\t}\n\t\t\tif psi.lblk > stop {\n\t\t\t\tstop = psi.lblk\n\t\t\t}\n\t\t})\n\t} else if psi, ok := fs.psim.Find(stringToBytes(filter)); ok {\n\t\tstart, stop = psi.fblk, psi.lblk\n\t}\n\t// Nothing was found.\n\tif start == uint32(math.MaxUint32) {\n\t\treturn -1, ErrStoreEOF\n\t}\n\treturn fs.selectSkipFirstBlock(bi, start, stop)\n}\n\n// This is used to see if we can selectively jump start blocks based on filter subjects and a starting block index.\n// Will return -1 and ErrStoreEOF if no matches at all or no more from where we are.\nfunc (fs *fileStore) checkSkipFirstBlockMulti(sl *gsl.SimpleSublist, bi int) (int, error) {\n\t// Move through psim to gather start and stop bounds.\n\tstart, stop := uint32(math.MaxUint32), uint32(0)\n\tstree.IntersectGSL(fs.psim, sl, func(subj []byte, psi *psi) {\n\t\tif psi.fblk < start {\n\t\t\tstart = psi.fblk\n\t\t}\n\t\tif psi.lblk > stop {\n\t\t\tstop = psi.lblk\n\t\t}\n\t})\n\t// Nothing was found.\n\tif start == uint32(math.MaxUint32) {\n\t\treturn -1, ErrStoreEOF\n\t}\n\treturn fs.selectSkipFirstBlock(bi, start, stop)\n}\n\nfunc (fs *fileStore) selectSkipFirstBlock(bi int, start, stop uint32) (int, error) {\n\t// Can not be nil so ok to inline dereference.\n\tmbi := fs.blks[bi].getIndex()\n\t// All matching msgs are behind us.\n\t// Less than AND equal is important because we were called because we missed searching bi.\n\tif stop <= mbi {\n\t\treturn -1, ErrStoreEOF\n\t}\n\t// If start is > index return dereference of fs.blks index.\n\tif start > mbi {\n\t\tif mb := fs.bim[start]; mb != nil {\n\t\t\tni, _ := fs.selectMsgBlockWithIndex(atomic.LoadUint64(&mb.last.seq))\n\t\t\treturn ni, nil\n\t\t}\n\t}\n\t// Otherwise just bump to the next one.\n\treturn bi + 1, nil\n}\n\n// Optimized way for getting all num pending matching a filter subject.\n// Lock should be held.\nfunc (fs *fileStore) numFilteredPending(filter string, ss *SimpleState) error {\n\treturn fs.numFilteredPendingWithLast(filter, true, ss)\n}\n\n// Optimized way for getting all num pending matching a filter subject and first sequence only.\n// Lock should be held.\nfunc (fs *fileStore) numFilteredPendingNoLast(filter string, ss *SimpleState) error {\n\treturn fs.numFilteredPendingWithLast(filter, false, ss)\n}\n\n// Optimized way for getting all num pending matching a filter subject.\n// Optionally look up last sequence. Sometimes do not need last and this avoids cost.\n// Read lock should be held.\nfunc (fs *fileStore) numFilteredPendingWithLast(filter string, last bool, ss *SimpleState) error {\n\tisAll := filter == _EMPTY_ || filter == fwcs\n\n\t// If isAll we do not need to do anything special to calculate the first and last and total.\n\tif isAll {\n\t\tss.First = fs.state.FirstSeq\n\t\tss.Last = fs.state.LastSeq\n\t\tss.Msgs = fs.state.Msgs\n\t\treturn nil\n\t}\n\t// Always reset.\n\tss.First, ss.Last, ss.Msgs = 0, 0, 0\n\n\t// We do need to figure out the first and last sequences.\n\twc := subjectHasWildcard(filter)\n\tstart, stop := uint32(math.MaxUint32), uint32(0)\n\n\tif wc {\n\t\tfs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) {\n\t\t\tss.Msgs += psi.total\n\t\t\t// Keep track of start and stop indexes for this subject.\n\t\t\tif psi.fblk < start {\n\t\t\t\tstart = psi.fblk\n\t\t\t}\n\t\t\tif psi.lblk > stop {\n\t\t\t\tstop = psi.lblk\n\t\t\t}\n\t\t})\n\t} else if psi, ok := fs.psim.Find(stringToBytes(filter)); ok {\n\t\tss.Msgs += psi.total\n\t\tstart, stop = psi.fblk, psi.lblk\n\t}\n\n\t// Did not find anything.\n\tif stop == 0 {\n\t\treturn nil\n\t}\n\n\t// Do start\n\tmb := fs.bim[start]\n\tif mb != nil {\n\t\t_, f, _, err := mb.filteredPending(filter, wc, 0)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tss.First = f\n\t}\n\n\tif ss.First == 0 {\n\t\t// This is a miss. This can happen since psi.fblk is lazy.\n\t\t// We will make sure to update fblk.\n\n\t\t// Hold this outside loop for psim fblk updates when done.\n\t\ti := start + 1\n\t\tfor ; i <= stop; i++ {\n\t\t\tmb := fs.bim[i]\n\t\t\tif mb == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif _, f, _, err := mb.filteredPending(filter, wc, 0); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if f > 0 {\n\t\t\t\tss.First = f\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// Update fblk since fblk was outdated.\n\t\t// We only require read lock here as that is desirable,\n\t\t// so we need to do this in a go routine to acquire write lock.\n\t\tgo func() {\n\t\t\tfs.mu.Lock()\n\t\t\tdefer fs.mu.Unlock()\n\t\t\tif !wc {\n\t\t\t\tif info, ok := fs.psim.Find(stringToBytes(filter)); ok {\n\t\t\t\t\tif i > info.fblk {\n\t\t\t\t\t\tinfo.fblk = i\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) {\n\t\t\t\t\tif i > psi.fblk {\n\t\t\t\t\t\tpsi.fblk = i\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t}()\n\t}\n\t// Now gather last sequence if asked to do so.\n\tif last {\n\t\tif mb = fs.bim[stop]; mb != nil {\n\t\t\t_, _, l, err := mb.filteredPending(filter, wc, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tss.Last = l\n\t\t}\n\t}\n\treturn nil\n}\n\n// SubjectsState returns a map of SimpleState for all matching subjects.\nfunc (fs *fileStore) SubjectsState(subject string) map[string]SimpleState {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || fs.noTrackSubjects() {\n\t\treturn nil\n\t}\n\n\tif subject == _EMPTY_ {\n\t\tsubject = fwcs\n\t}\n\n\tstart, stop := fs.blks[0], fs.lmb\n\t// We can short circuit if not a wildcard using psim for start and stop.\n\tif !subjectHasWildcard(subject) {\n\t\tinfo, ok := fs.psim.Find(stringToBytes(subject))\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\tif f := fs.bim[info.fblk]; f != nil {\n\t\t\tstart = f\n\t\t}\n\t\tif l := fs.bim[info.lblk]; l != nil {\n\t\t\tstop = l\n\t\t}\n\t}\n\n\t// Aggregate fss.\n\tfss := make(map[string]SimpleState)\n\tvar startFound bool\n\n\tfor _, mb := range fs.blks {\n\t\tif !startFound {\n\t\t\tif mb != start {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tstartFound = true\n\t\t}\n\n\t\tmb.mu.Lock()\n\t\tvar shouldExpire bool\n\t\tif mb.fssNotLoaded() {\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tshouldExpire = true\n\t\t}\n\t\t// Mark fss activity.\n\t\tmb.lsts = ats.AccessTime()\n\t\tvar ierr error\n\t\tmb.fss.Match(stringToBytes(subject), func(bsubj []byte, ss *SimpleState) {\n\t\t\tif ierr != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tsubj := string(bsubj)\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tif ierr = mb.recalculateForSubj(subj, ss); ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\toss := fss[subj]\n\t\t\tif oss.First == 0 { // New\n\t\t\t\tfss[subj] = *ss\n\t\t\t} else {\n\t\t\t\t// Merge here.\n\t\t\t\toss.Last, oss.Msgs = ss.Last, oss.Msgs+ss.Msgs\n\t\t\t\tfss[subj] = oss\n\t\t\t}\n\t\t})\n\t\tif shouldExpire {\n\t\t\t// Expire this cache before moving on.\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tmb.mu.Unlock()\n\t\tif ierr != nil {\n\t\t\treturn nil\n\t\t}\n\t\tif mb == stop {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn fss\n}\n\n// AllLastSeqs will return a sorted list of last sequences for all subjects.\nfunc (fs *fileStore) AllLastSeqs() ([]uint64, error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\treturn fs.allLastSeqsLocked()\n}\n\n// allLastSeqsLocked will return a sorted list of last sequences for all\n// subjects, but won't take the lock to do it, to avoid the issue of compounding\n// read locks causing a deadlock with a write lock.\nfunc (fs *fileStore) allLastSeqsLocked() ([]uint64, error) {\n\tif fs.state.Msgs == 0 || fs.noTrackSubjects() {\n\t\treturn nil, nil\n\t}\n\n\tnumSubjects := fs.psim.Size()\n\tseqs := make([]uint64, 0, numSubjects)\n\tsubs := make(map[string]struct{}, numSubjects)\n\n\tfor i := len(fs.blks) - 1; i >= 0; i-- {\n\t\tif len(subs) == numSubjects {\n\t\t\tbreak\n\t\t}\n\t\tmb := fs.blks[i]\n\t\tmb.mu.Lock()\n\n\t\tvar shouldExpire bool\n\t\tif mb.fssNotLoaded() {\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tshouldExpire = true\n\t\t}\n\n\t\tvar ierr error\n\t\tmb.fss.IterFast(func(bsubj []byte, ss *SimpleState) bool {\n\t\t\t// Check if already been processed and accounted.\n\t\t\tif _, ok := subs[string(bsubj)]; !ok {\n\t\t\t\t// Check if we need to recalculate. We only care about the last sequence.\n\t\t\t\tif ss.lastNeedsUpdate {\n\t\t\t\t\t// mb is already loaded into the cache so should be fast-ish.\n\t\t\t\t\tif ierr = mb.recalculateForSubj(bytesToString(bsubj), ss); ierr != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tseqs = append(seqs, ss.Last)\n\t\t\t\tsubs[string(bsubj)] = struct{}{}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif shouldExpire {\n\t\t\t// Expire this cache before moving on.\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t}\n\t\tmb.finishedWithCache()\n\t\tmb.mu.Unlock()\n\t\tif ierr != nil {\n\t\t\treturn nil, ierr\n\t\t}\n\t}\n\n\tslices.Sort(seqs)\n\treturn seqs, nil\n}\n\n// Helper to determine if the filter(s) represent all the subjects.\n// Most clients send in subjects even if they match the stream's ingest subjects.\n// Lock should be held.\nfunc (fs *fileStore) filterIsAll(filters []string) bool {\n\tif len(filters) != len(fs.cfg.Subjects) {\n\t\treturn false\n\t}\n\t// Sort so we can compare.\n\tslices.Sort(filters)\n\tslices.Sort(fs.cfg.Subjects)\n\tfor i, subj := range filters {\n\t\tif !subjectIsSubsetMatch(fs.cfg.Subjects[i], subj) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// MultiLastSeqs will return a sorted list of sequences that match all subjects presented in filters.\n// We will not exceed the maxSeq, which if 0 becomes the store's last sequence.\nfunc (fs *fileStore) MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || fs.noTrackSubjects() {\n\t\treturn nil, nil\n\t}\n\n\t// See if we can short circuit if we think they are asking for all last sequences and have no maxSeq or maxAllowed set.\n\tif maxSeq == 0 && maxAllowed <= 0 && fs.filterIsAll(filters) {\n\t\treturn fs.allLastSeqsLocked()\n\t}\n\n\tlastBlkIndex := len(fs.blks) - 1\n\tlastMB := fs.blks[lastBlkIndex]\n\n\t// Implied last sequence.\n\tif maxSeq == 0 {\n\t\tmaxSeq = fs.state.LastSeq\n\t} else {\n\t\t// Udate last mb index if not last seq.\n\t\tlastBlkIndex, lastMB = fs.selectMsgBlockWithIndex(maxSeq)\n\t}\n\t// Make sure non-nil\n\tif lastMB == nil {\n\t\treturn nil, nil\n\t}\n\n\t// Grab our last mb index (not same as blk index).\n\tlastMB.mu.RLock()\n\tlastMBIndex := lastMB.index\n\tlastMB.mu.RUnlock()\n\n\tsubs := make(map[string]*psi)\n\tvar numLess int\n\tvar maxBlk uint32\n\n\tfor _, filter := range filters {\n\t\tfs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) {\n\t\t\tsubs[string(subj)] = psi\n\t\t\tif psi.lblk < lastMBIndex {\n\t\t\t\tnumLess++\n\t\t\t\tif psi.lblk > maxBlk {\n\t\t\t\t\tmaxBlk = psi.lblk\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\t// If all subjects have a lower last index, select the largest for our walk backwards.\n\tif numLess == len(subs) {\n\t\tlastMB = fs.bim[maxBlk]\n\t}\n\n\t// Collect all sequences needed.\n\tseqs := make([]uint64, 0, len(subs))\n\tfor i, lnf := lastBlkIndex, false; i >= 0; i-- {\n\t\tif len(subs) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tmb := fs.blks[i]\n\t\tif !lnf {\n\t\t\tif mb != lastMB {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlnf = true\n\t\t}\n\t\t// We can start properly looking here.\n\t\tmb.mu.Lock()\n\t\tvar ierr error\n\t\tif ierr = mb.ensurePerSubjectInfoLoaded(); ierr != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn nil, ierr\n\t\t}\n\n\t\t// Iterate the fss and check against our subs. We will delete from subs as we add.\n\t\t// Once len(subs) == 0 we are done.\n\t\tmb.fss.IterFast(func(bsubj []byte, ss *SimpleState) bool {\n\t\t\t// Already been processed and accounted for was not matched in the first place.\n\t\t\tif subs[string(bsubj)] == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\t// Check if we need to recalculate. We only care about the last sequence.\n\t\t\tif ss.lastNeedsUpdate {\n\t\t\t\t// mb is already loaded into the cache so should be fast-ish.\n\t\t\t\tif ierr = mb.recalculateForSubj(bytesToString(bsubj), ss); ierr != nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If we are equal or below just add to seqs slice.\n\t\t\tif ss.Last <= maxSeq {\n\t\t\t\tseqs = append(seqs, ss.Last)\n\t\t\t\tdelete(subs, bytesToString(bsubj))\n\t\t\t} else {\n\t\t\t\t// Need to search for the real last since recorded last is > maxSeq.\n\t\t\t\tvar didLoad bool\n\t\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\t\tif ierr = mb.loadMsgsWithLock(); ierr != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t\tdidLoad = true\n\t\t\t\t}\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t\t\tlseq := min(atomic.LoadUint64(&mb.last.seq), maxSeq)\n\t\t\t\tssubj := bytesToString(bsubj)\n\t\t\t\tfor seq := lseq; seq >= fseq; seq-- {\n\t\t\t\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t\t\tif sm == nil || sm.subj != ssubj {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tseqs = append(seqs, sm.seq)\n\t\t\t\t\tdelete(subs, ssubj)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif didLoad {\n\t\t\t\t\tmb.finishedWithCache()\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tmb.mu.Unlock()\n\t\tif ierr != nil {\n\t\t\treturn nil, ierr\n\t\t}\n\n\t\t// If maxAllowed was sepcified check that we will not exceed that.\n\t\tif maxAllowed > 0 && len(seqs) > maxAllowed {\n\t\t\treturn nil, ErrTooManyResults\n\t\t}\n\t}\n\tif len(seqs) == 0 {\n\t\treturn nil, nil\n\t}\n\tslices.Sort(seqs)\n\treturn seqs, nil\n}\n\n// NumPending will return the number of pending messages matching the filter subject starting at sequence.\n// Optimized for stream num pending calculations for consumers.\nfunc (fs *fileStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\t// This can always be last for these purposes.\n\tvalidThrough = fs.state.LastSeq\n\n\tif fs.state.Msgs == 0 || sseq > fs.state.LastSeq {\n\t\treturn 0, validThrough, nil\n\t}\n\n\t// If sseq is less then our first set to first.\n\tif sseq < fs.state.FirstSeq {\n\t\tsseq = fs.state.FirstSeq\n\t}\n\t// Track starting for both block for the sseq and staring block that matches any subject.\n\tvar seqStart int\n\t// See if we need to figure out starting block per sseq.\n\tif sseq > fs.state.FirstSeq {\n\t\t// This should not, but can return -1, so make sure we check to avoid panic below.\n\t\tif seqStart, _ = fs.selectMsgBlockWithIndex(sseq); seqStart < 0 {\n\t\t\tseqStart = 0\n\t\t}\n\t}\n\n\tisAll := filter == _EMPTY_ || filter == fwcs\n\tif isAll && filter == _EMPTY_ {\n\t\tfilter = fwcs\n\t}\n\twc := subjectHasWildcard(filter)\n\n\t// See if filter was provided but its the only subject.\n\tif !isAll && !wc && fs.psim.Size() == 1 {\n\t\t_, isAll = fs.psim.Find(stringToBytes(filter))\n\t}\n\t// If we are isAll and have no deleted we can do a simpler calculation.\n\tif !lastPerSubject && isAll && (fs.state.LastSeq-fs.state.FirstSeq+1) == fs.state.Msgs {\n\t\tif sseq == 0 {\n\t\t\treturn fs.state.Msgs, validThrough, nil\n\t\t}\n\t\treturn fs.state.LastSeq - sseq + 1, validThrough, nil\n\t}\n\n\t_tsa, _fsa := [32]string{}, [32]string{}\n\ttsa, fsa := _tsa[:0], _fsa[:0]\n\tif wc {\n\t\tfsa = tokenizeSubjectIntoSlice(fsa[:0], filter)\n\t}\n\n\tisMatch := func(subj string) bool {\n\t\tif isAll {\n\t\t\treturn true\n\t\t}\n\t\tif !wc {\n\t\t\treturn subj == filter\n\t\t}\n\t\ttsa = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\treturn isSubsetMatchTokenized(tsa, fsa)\n\t}\n\n\t// Handle last by subject a bit differently.\n\t// We will scan PSIM since we accurately track the last block we have seen the subject in. This\n\t// allows us to only need to load at most one block now.\n\t// For the last block, we need to track the subjects that we know are in that block, and track seen\n\t// while in the block itself, but complexity there worth it.\n\tif lastPerSubject {\n\t\t// If we want all and our start sequence is equal or less than first return number of subjects.\n\t\tif isAll && sseq <= fs.state.FirstSeq {\n\t\t\treturn uint64(fs.psim.Size()), validThrough, nil\n\t\t}\n\t\t// If we are here we need to scan. We are going to scan the PSIM looking for lblks that are >= seqStart.\n\t\t// This will build up a list of all subjects from the selected block onward.\n\t\tlbm := make(map[string]bool)\n\t\tmb := fs.blks[seqStart]\n\t\tbi := mb.index\n\n\t\tfs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) {\n\t\t\t// If the select blk start is greater than entry's last blk skip.\n\t\t\tif bi > psi.lblk {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttotal++\n\t\t\t// We will track the subjects that are an exact match to the last block.\n\t\t\t// This is needed for last block processing.\n\t\t\tif psi.lblk == bi {\n\t\t\t\tlbm[string(subj)] = true\n\t\t\t}\n\t\t})\n\n\t\t// Now check if we need to inspect the seqStart block.\n\t\t// Grab write lock in case we need to load in msgs.\n\t\tmb.mu.Lock()\n\t\tvar updateLLTS bool\n\t\tvar shouldExpire bool\n\t\t// We need to walk this block to correct accounting from above.\n\t\tif sseq > mb.first.seq {\n\t\t\t// Track the ones we add back in case more than one.\n\t\t\tseen := make(map[string]bool)\n\t\t\t// We need to discount the total by subjects seen before sseq, but also add them right back in if they are >= sseq for this blk.\n\t\t\t// This only should be subjects we know have the last blk in this block.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\tvar smv StoreMsg\n\t\t\tfor seq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ {\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t\tif sm == nil || sm.subj == _EMPTY_ || !lbm[sm.subj] {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\tif isMatch(sm.subj) {\n\t\t\t\t\t// If less than sseq adjust off of total as long as this subject matched the last block.\n\t\t\t\t\tif seq < sseq {\n\t\t\t\t\t\tif !seen[sm.subj] {\n\t\t\t\t\t\t\ttotal--\n\t\t\t\t\t\t\tseen[sm.subj] = true\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if seen[sm.subj] {\n\t\t\t\t\t\t// This is equal or more than sseq, so add back in.\n\t\t\t\t\t\ttotal++\n\t\t\t\t\t\t// Make sure to not process anymore.\n\t\t\t\t\t\tdelete(seen, sm.subj)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If we loaded the block try to force expire.\n\t\tif shouldExpire {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.mu.Unlock()\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we would need to scan more from the beginning, revert back to calculating directly here.\n\t// TODO(dlc) - Redo properly with sublists etc for subject-based filtering.\n\tif seqStart >= (len(fs.blks) / 2) {\n\t\tfor i := seqStart; i < len(fs.blks); i++ {\n\t\t\tvar shouldExpire bool\n\t\t\tmb := fs.blks[i]\n\t\t\t// Hold write lock in case we need to load cache.\n\t\t\tmb.mu.Lock()\n\t\t\tif isAll && sseq <= atomic.LoadUint64(&mb.first.seq) {\n\t\t\t\ttotal += mb.msgs\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If we are here we need to at least scan the subject fss.\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif mb.fssNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\t// Mark fss activity.\n\t\t\tmb.lsts = ats.AccessTime()\n\n\t\t\tvar t uint64\n\t\t\tvar ierr error\n\t\t\tvar havePartial bool\n\t\t\tmb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) {\n\t\t\t\tif ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif havePartial {\n\t\t\t\t\t// If we already found a partial then don't do anything else.\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tsubj := bytesToString(bsubj)\n\t\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\t\tif ierr = mb.recalculateForSubj(subj, ss); ierr != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif sseq <= ss.First {\n\t\t\t\t\tt += ss.Msgs\n\t\t\t\t} else if sseq <= ss.Last {\n\t\t\t\t\t// We matched but its a partial.\n\t\t\t\t\thavePartial = true\n\t\t\t\t}\n\t\t\t})\n\t\t\tif ierr != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn 0, 0, ierr\n\t\t\t}\n\n\t\t\t// See if we need to scan msgs here.\n\t\t\tif havePartial {\n\t\t\t\t// Make sure we have the cache loaded.\n\t\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\treturn 0, 0, err\n\t\t\t\t\t}\n\t\t\t\t\tshouldExpire = true\n\t\t\t\t}\n\t\t\t\t// Clear on partial.\n\t\t\t\tt = 0\n\t\t\t\tstart := sseq\n\t\t\t\tif fseq := atomic.LoadUint64(&mb.first.seq); fseq > start {\n\t\t\t\t\tstart = fseq\n\t\t\t\t}\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tfor seq, lseq := start, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ {\n\t\t\t\t\tif sm, _ := mb.cacheLookupNoCopy(seq, &smv); sm != nil && isMatch(sm.subj) {\n\t\t\t\t\t\tt++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If we loaded this block for this operation go ahead and expire it here.\n\t\t\tif shouldExpire {\n\t\t\t\tmb.tryForceExpireCacheLocked()\n\t\t\t} else {\n\t\t\t\tmb.finishedWithCache()\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t\ttotal += t\n\t\t}\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we are here it's better to calculate totals from psim and adjust downward by scanning less blocks.\n\t// TODO(dlc) - Eventually when sublist uses generics, make this sublist driven instead.\n\tstart := uint32(math.MaxUint32)\n\tfs.psim.Match(stringToBytes(filter), func(_ []byte, psi *psi) {\n\t\ttotal += psi.total\n\t\t// Keep track of start index for this subject.\n\t\tif psi.fblk < start {\n\t\t\tstart = psi.fblk\n\t\t}\n\t})\n\t// See if we were asked for all, if so we are done.\n\tif sseq <= fs.state.FirstSeq {\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we are here we need to calculate partials for the first blocks.\n\tfirstSubjBlk := fs.bim[start]\n\tvar firstSubjBlkFound bool\n\t// Adjust in case not found.\n\tif firstSubjBlk == nil {\n\t\tfirstSubjBlkFound = true\n\t}\n\n\t// Track how many we need to adjust against the total.\n\tvar adjust uint64\n\tfor i := 0; i <= seqStart; i++ {\n\t\tmb := fs.blks[i]\n\t\t// We can skip blks if we know they are below the first one that has any subject matches.\n\t\tif !firstSubjBlkFound {\n\t\t\tif firstSubjBlkFound = (mb == firstSubjBlk); !firstSubjBlkFound {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// We need to scan this block.\n\t\tvar shouldExpire bool\n\t\tvar updateLLTS bool\n\t\tmb.mu.Lock()\n\t\t// Check if we should include all of this block in adjusting. If so work with metadata.\n\t\tif sseq > atomic.LoadUint64(&mb.last.seq) {\n\t\t\tif isAll {\n\t\t\t\tadjust += mb.msgs\n\t\t\t} else {\n\t\t\t\t// We need to adjust for all matches in this block.\n\t\t\t\t// Make sure we have fss loaded. This loads whole block now.\n\t\t\t\tif mb.fssNotLoaded() {\n\t\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\treturn 0, 0, err\n\t\t\t\t\t}\n\t\t\t\t\tshouldExpire = true\n\t\t\t\t}\n\t\t\t\t// Mark fss activity.\n\t\t\t\tmb.lsts = ats.AccessTime()\n\n\t\t\t\tmb.fss.Match(stringToBytes(filter), func(bsubj []byte, ss *SimpleState) {\n\t\t\t\t\tadjust += ss.Msgs\n\t\t\t\t})\n\t\t\t}\n\t\t} else {\n\t\t\t// This is the last block. We need to scan per message here.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\tvar last = atomic.LoadUint64(&mb.last.seq)\n\t\t\tif sseq < last {\n\t\t\t\tlast = sseq\n\t\t\t}\n\t\t\t// We need to walk all messages in this block\n\t\t\tvar smv StoreMsg\n\t\t\tfor seq := atomic.LoadUint64(&mb.first.seq); seq < last; seq++ {\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t\tif sm == nil || sm.subj == _EMPTY_ {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\t// Check if it matches our filter.\n\t\t\t\tif sm.seq < sseq && isMatch(sm.subj) {\n\t\t\t\t\tadjust++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If we loaded the block try to force expire.\n\t\tif shouldExpire {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.mu.Unlock()\n\t}\n\t// Make final adjustment.\n\tif adjust > total {\n\t\ttotal = 0\n\t} else {\n\t\ttotal -= adjust\n\t}\n\n\treturn total, validThrough, nil\n}\n\n// NumPending will return the number of pending messages matching any subject in the sublist starting at sequence.\n// Optimized for stream num pending calculations for consumers with lots of filtered subjects.\n// Subjects should not overlap, this property is held when doing multi-filtered consumers.\nfunc (fs *fileStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\t// This can always be last for these purposes.\n\tvalidThrough = fs.state.LastSeq\n\n\tif fs.state.Msgs == 0 || sseq > fs.state.LastSeq {\n\t\treturn 0, validThrough, nil\n\t}\n\n\t// If sseq is less then our first set to first.\n\tif sseq < fs.state.FirstSeq {\n\t\tsseq = fs.state.FirstSeq\n\t}\n\t// Track starting for both block for the sseq and staring block that matches any subject.\n\tvar seqStart int\n\t// See if we need to figure out starting block per sseq.\n\tif sseq > fs.state.FirstSeq {\n\t\t// This should not, but can return -1, so make sure we check to avoid panic below.\n\t\tif seqStart, _ = fs.selectMsgBlockWithIndex(sseq); seqStart < 0 {\n\t\t\tseqStart = 0\n\t\t}\n\t}\n\n\tisAll := sl == nil\n\n\t// See if filter was provided but its the only subject.\n\tif !isAll && fs.psim.Size() == 1 {\n\t\tfs.psim.IterFast(func(subject []byte, _ *psi) bool {\n\t\t\tisAll = sl.HasInterest(bytesToString(subject))\n\t\t\treturn true\n\t\t})\n\t}\n\t// If we are isAll and have no deleted we can do a simpler calculation.\n\tif !lastPerSubject && isAll && (fs.state.LastSeq-fs.state.FirstSeq+1) == fs.state.Msgs {\n\t\tif sseq == 0 {\n\t\t\treturn fs.state.Msgs, validThrough, nil\n\t\t}\n\t\treturn fs.state.LastSeq - sseq + 1, validThrough, nil\n\t}\n\t// Setup the isMatch function.\n\tisMatch := func(subj string) bool {\n\t\tif isAll {\n\t\t\treturn true\n\t\t}\n\t\treturn sl.HasInterest(subj)\n\t}\n\n\t// Handle last by subject a bit differently.\n\t// We will scan PSIM since we accurately track the last block we have seen the subject in. This\n\t// allows us to only need to load at most one block now.\n\t// For the last block, we need to track the subjects that we know are in that block, and track seen\n\t// while in the block itself, but complexity there worth it.\n\tif lastPerSubject {\n\t\t// If we want all and our start sequence is equal or less than first return number of subjects.\n\t\tif isAll && sseq <= fs.state.FirstSeq {\n\t\t\treturn uint64(fs.psim.Size()), validThrough, nil\n\t\t}\n\t\t// If we are here we need to scan. We are going to scan the PSIM looking for lblks that are >= seqStart.\n\t\t// This will build up a list of all subjects from the selected block onward.\n\t\tlbm := make(map[string]bool)\n\t\tmb := fs.blks[seqStart]\n\t\tbi := mb.index\n\n\t\tstree.IntersectGSL(fs.psim, sl, func(subj []byte, psi *psi) {\n\t\t\t// If the select blk start is greater than entry's last blk skip.\n\t\t\tif bi > psi.lblk {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttotal++\n\t\t\t// We will track the subjects that are an exact match to the last block.\n\t\t\t// This is needed for last block processing.\n\t\t\tif psi.lblk == bi {\n\t\t\t\tlbm[string(subj)] = true\n\t\t\t}\n\t\t})\n\n\t\t// Now check if we need to inspect the seqStart block.\n\t\t// Grab write lock in case we need to load in msgs.\n\t\tmb.mu.Lock()\n\t\tvar shouldExpire bool\n\t\tvar updateLLTS bool\n\t\t// We need to walk this block to correct accounting from above.\n\t\tif sseq > atomic.LoadUint64(&mb.first.seq) {\n\t\t\t// Track the ones we add back in case more than one.\n\t\t\tseen := make(map[string]bool)\n\t\t\t// We need to discount the total by subjects seen before sseq, but also add them right back in if they are >= sseq for this blk.\n\t\t\t// This only should be subjects we know have the last blk in this block.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\tvar smv StoreMsg\n\t\t\tfor seq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ {\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t\tif sm == nil || sm.subj == _EMPTY_ || !lbm[sm.subj] {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\tif isMatch(sm.subj) {\n\t\t\t\t\t// If less than sseq adjust off of total as long as this subject matched the last block.\n\t\t\t\t\tif seq < sseq {\n\t\t\t\t\t\tif !seen[sm.subj] {\n\t\t\t\t\t\t\ttotal--\n\t\t\t\t\t\t\tseen[sm.subj] = true\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if seen[sm.subj] {\n\t\t\t\t\t\t// This is equal or more than sseq, so add back in.\n\t\t\t\t\t\ttotal++\n\t\t\t\t\t\t// Make sure to not process anymore.\n\t\t\t\t\t\tdelete(seen, sm.subj)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If we loaded the block try to force expire.\n\t\tif shouldExpire {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.mu.Unlock()\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we would need to scan more from the beginning, revert back to calculating directly here.\n\tif seqStart >= (len(fs.blks) / 2) {\n\t\tfor i := seqStart; i < len(fs.blks); i++ {\n\t\t\tvar shouldExpire bool\n\t\t\tmb := fs.blks[i]\n\t\t\t// Hold write lock in case we need to load cache.\n\t\t\tmb.mu.Lock()\n\t\t\tif isAll && sseq <= atomic.LoadUint64(&mb.first.seq) {\n\t\t\t\ttotal += mb.msgs\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If we are here we need to at least scan the subject fss.\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif mb.fssNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\t// Mark fss activity.\n\t\t\tmb.lsts = ats.AccessTime()\n\n\t\t\tvar t uint64\n\t\t\tvar ierr error\n\t\t\tvar havePartial bool\n\t\t\tvar updateLLTS bool\n\t\t\tstree.IntersectGSL[SimpleState](mb.fss, sl, func(bsubj []byte, ss *SimpleState) {\n\t\t\t\tif ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tsubj := bytesToString(bsubj)\n\t\t\t\tif havePartial {\n\t\t\t\t\t// If we already found a partial then don't do anything else.\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\t\tif ierr = mb.recalculateForSubj(subj, ss); ierr != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif sseq <= ss.First {\n\t\t\t\t\tt += ss.Msgs\n\t\t\t\t} else if sseq <= ss.Last {\n\t\t\t\t\t// We matched but its a partial.\n\t\t\t\t\thavePartial = true\n\t\t\t\t}\n\t\t\t})\n\t\t\tif ierr != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn 0, 0, ierr\n\t\t\t}\n\n\t\t\t// See if we need to scan msgs here.\n\t\t\tif havePartial {\n\t\t\t\t// Make sure we have the cache loaded.\n\t\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\treturn 0, 0, err\n\t\t\t\t\t}\n\t\t\t\t\tshouldExpire = true\n\t\t\t\t}\n\t\t\t\t// Clear on partial.\n\t\t\t\tt = 0\n\t\t\t\tstart := sseq\n\t\t\t\tif fseq := atomic.LoadUint64(&mb.first.seq); fseq > start {\n\t\t\t\t\tstart = fseq\n\t\t\t\t}\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tfor seq, lseq := start, atomic.LoadUint64(&mb.last.seq); seq <= lseq; seq++ {\n\t\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif sm, _ := mb.cacheLookupNoCopy(seq, &smv); sm != nil && isMatch(sm.subj) {\n\t\t\t\t\t\tt++\n\t\t\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If we loaded this block for this operation go ahead and expire it here.\n\t\t\tif shouldExpire {\n\t\t\t\tmb.tryForceExpireCacheLocked()\n\t\t\t} else {\n\t\t\t\tmb.finishedWithCache()\n\t\t\t}\n\t\t\tif updateLLTS {\n\t\t\t\tmb.llts = ats.AccessTime()\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t\ttotal += t\n\t\t}\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we are here it's better to calculate totals from psim and adjust downward by scanning less blocks.\n\tstart := uint32(math.MaxUint32)\n\tstree.IntersectGSL(fs.psim, sl, func(subj []byte, psi *psi) {\n\t\ttotal += psi.total\n\t\t// Keep track of start index for this subject.\n\t\tif psi.fblk < start {\n\t\t\tstart = psi.fblk\n\t\t}\n\t})\n\n\t// See if we were asked for all, if so we are done.\n\tif sseq <= fs.state.FirstSeq {\n\t\treturn total, validThrough, nil\n\t}\n\n\t// If we are here we need to calculate partials for the first blocks.\n\tfirstSubjBlk := fs.bim[start]\n\tvar firstSubjBlkFound bool\n\t// Adjust in case not found.\n\tif firstSubjBlk == nil {\n\t\tfirstSubjBlkFound = true\n\t}\n\n\t// Track how many we need to adjust against the total.\n\tvar adjust uint64\n\tfor i := 0; i <= seqStart; i++ {\n\t\tmb := fs.blks[i]\n\t\t// We can skip blks if we know they are below the first one that has any subject matches.\n\t\tif !firstSubjBlkFound {\n\t\t\tif firstSubjBlkFound = (mb == firstSubjBlk); !firstSubjBlkFound {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// We need to scan this block.\n\t\tvar shouldExpire bool\n\t\tvar updateLLTS bool\n\t\tmb.mu.Lock()\n\t\t// Check if we should include all of this block in adjusting. If so work with metadata.\n\t\tif sseq > atomic.LoadUint64(&mb.last.seq) {\n\t\t\tif isAll {\n\t\t\t\tadjust += mb.msgs\n\t\t\t} else {\n\t\t\t\t// We need to adjust for all matches in this block.\n\t\t\t\t// Make sure we have fss loaded. This loads whole block now.\n\t\t\t\tif mb.fssNotLoaded() {\n\t\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\treturn 0, 0, err\n\t\t\t\t\t}\n\t\t\t\t\tshouldExpire = true\n\t\t\t\t}\n\t\t\t\t// Mark fss activity.\n\t\t\t\tmb.lsts = ats.AccessTime()\n\t\t\t\tstree.IntersectGSL(mb.fss, sl, func(bsubj []byte, ss *SimpleState) {\n\t\t\t\t\tadjust += ss.Msgs\n\t\t\t\t})\n\t\t\t}\n\t\t} else {\n\t\t\t// This is the last block. We need to scan per message here.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err = mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn 0, 0, err\n\t\t\t\t}\n\t\t\t\tshouldExpire = true\n\t\t\t}\n\t\t\tvar last = atomic.LoadUint64(&mb.last.seq)\n\t\t\tif sseq < last {\n\t\t\t\tlast = sseq\n\t\t\t}\n\t\t\t// We need to walk all messages in this block\n\t\t\tvar smv StoreMsg\n\t\t\tfor seq := atomic.LoadUint64(&mb.first.seq); seq < last; seq++ {\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t\t\tupdateLLTS = true\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\t\tif sm == nil || sm.subj == _EMPTY_ {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tupdateLLTS = false // cacheLookup already updated it.\n\t\t\t\t// Check if it matches our filter.\n\t\t\t\tif sm.seq < sseq && isMatch(sm.subj) {\n\t\t\t\t\tadjust++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If we loaded the block try to force expire.\n\t\tif shouldExpire {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tif updateLLTS {\n\t\t\tmb.llts = ats.AccessTime()\n\t\t}\n\t\tmb.mu.Unlock()\n\t}\n\t// Make final adjustment.\n\tif adjust > total {\n\t\ttotal = 0\n\t} else {\n\t\ttotal -= adjust\n\t}\n\n\treturn total, validThrough, nil\n}\n\n// SubjectsTotals return message totals per subject.\nfunc (fs *fileStore) SubjectsTotals(filter string) map[string]uint64 {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\treturn fs.subjectsTotalsLocked(filter)\n}\n\n// Lock should be held.\nfunc (fs *fileStore) subjectsTotalsLocked(filter string) map[string]uint64 {\n\tif fs.psim.Size() == 0 {\n\t\treturn nil\n\t}\n\t// Match all if no filter given.\n\tif filter == _EMPTY_ {\n\t\tfilter = fwcs\n\t}\n\tfst := make(map[string]uint64)\n\tfs.psim.Match(stringToBytes(filter), func(subj []byte, psi *psi) {\n\t\tfst[string(subj)] = psi.total\n\t})\n\treturn fst\n}\n\n// RegisterStorageUpdates registers a callback for updates to storage changes.\n// It will present number of messages and bytes as a signed integer and an\n// optional sequence number of the message if a single.\nfunc (fs *fileStore) RegisterStorageUpdates(cb StorageUpdateHandler) {\n\tfs.mu.Lock()\n\tfs.scb = cb\n\tbsz := fs.state.Bytes\n\tfs.mu.Unlock()\n\tif cb != nil && bsz > 0 {\n\t\tcb(0, int64(bsz), 0, _EMPTY_)\n\t}\n}\n\n// RegisterStorageRemoveMsg registers a callback to remove messages.\n// Replicated streams should propose removals, R1 can remove inline.\nfunc (fs *fileStore) RegisterStorageRemoveMsg(cb StorageRemoveMsgHandler) {\n\tfs.mu.Lock()\n\tfs.rmcb = cb\n\tfs.mu.Unlock()\n}\n\n// RegisterProcessJetStreamMsg registers a callback to process new JetStream messages.\nfunc (fs *fileStore) RegisterProcessJetStreamMsg(cb ProcessJetStreamMsgHandler) {\n\tfs.mu.Lock()\n\tfs.pmsgcb = cb\n\tfs.mu.Unlock()\n}\n\n// Helper to get hash key for specific message block.\n// Lock should be held\nfunc (fs *fileStore) hashKeyForBlock(index uint32) []byte {\n\treturn []byte(fmt.Sprintf(\"%s-%d\", fs.cfg.Name, index))\n}\n\nfunc (mb *msgBlock) setupWriteCache(buf []byte) error {\n\t// Not safe to use mb.cacheAlreadyLoaded() here as the heuristic\n\t// is wrong for writing to the beginning of an empty block cache.\n\tif mb.cache != nil {\n\t\treturn nil\n\t}\n\tif mb.cache = mb.ecache.Value(); mb.cache != nil {\n\t\treturn nil\n\t}\n\n\t// If there's an existing block on disk then we'll load it in.\n\tvar fi os.FileInfo\n\tif mb.mfd != nil {\n\t\tfi, _ = mb.mfd.Stat()\n\t} else if mb.mfn != _EMPTY_ {\n\t\tfi, _ = os.Stat(mb.mfn)\n\t}\n\tif fi != nil {\n\t\treturn mb.loadMsgsWithLock()\n\t}\n\n\t// Looks like there isn't an existing file on disk, mint a new cache.\n\tmb.cache = &cache{buf: buf}\n\tmb.ecache.Set(mb.cache)\n\tmb.llts = ats.AccessTime()\n\tmb.startCacheExpireTimer()\n\treturn nil\n}\n\n// finishedWithCache removes the strong reference to the cache, instead\n// returning it to the elastic pointer. Note that the elastic pointer can\n// be either weak or strong, they are strengthened when there are pending\n// writes and weak otherwise.\n// Lock must be held.\nfunc (mb *msgBlock) finishedWithCache() {\n\tif mb.cache != nil && mb.pendingWriteSizeLocked() == 0 {\n\t\tmb.cache = nil\n\t}\n}\n\n// This rolls to a new append msg block.\n// Lock should be held.\nfunc (fs *fileStore) newMsgBlockForWrite() (*msgBlock, error) {\n\tindex := uint32(1)\n\tvar rbuf []byte\n\n\tif lmb := fs.lmb; lmb != nil {\n\t\tlmb.mu.Lock()\n\t\tindex = lmb.index + 1\n\n\t\t// Quit our loops.\n\t\tif lmb.qch != nil {\n\t\t\tclose(lmb.qch)\n\t\t\tlmb.qch = nil\n\t\t}\n\t\t// If we had a write error before, don't allow continuing into a new block.\n\t\tif err := lmb.werr; err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// Flush any pending messages.\n\t\tif _, err := lmb.flushPendingMsgsLocked(); err != nil {\n\t\t\tlmb.mu.Unlock()\n\t\t\treturn nil, err\n\t\t}\n\t\t// Determine if we can reclaim any resources here.\n\t\tlmb.closeFDsLockedNoCheck()\n\t\tif lmb.cache != nil {\n\t\t\t// Reset write timestamp and see if we can expire this cache.\n\t\t\trbuf = lmb.tryExpireWriteCache()\n\t\t}\n\t\tlmb.mu.Unlock()\n\n\t\tif fs.fcfg.Compression != NoCompression {\n\t\t\t// We've now reached the end of this message block, if we want\n\t\t\t// to compress blocks then now's the time to do it.\n\t\t\tgo func() {\n\t\t\t\tlmb.mu.Lock()\n\t\t\t\tdefer lmb.mu.Unlock()\n\t\t\t\t// Might error, but we can't handle it here anyway.\n\t\t\t\t_ = lmb.recompressOnDiskIfNeeded()\n\t\t\t}()\n\t\t}\n\t}\n\n\tmb := fs.initMsgBlock(index)\n\t// Lock should be held to quiet race detector.\n\tmb.mu.Lock()\n\tif err := mb.setupWriteCache(rbuf); err != nil {\n\t\tmb.mu.Unlock()\n\t\treturn nil, err\n\t}\n\tmb.fss = stree.NewSubjectTree[SimpleState]()\n\n\t// Set cache time to creation time to start.\n\tmb.llts, mb.lwts = 0, ats.AccessTime()\n\t// Remember our last sequence number.\n\tatomic.StoreUint64(&mb.first.seq, fs.state.LastSeq+1)\n\tatomic.StoreUint64(&mb.last.seq, fs.state.LastSeq)\n\tmb.mu.Unlock()\n\n\t// Now do local hash.\n\tkey := sha256.Sum256(fs.hashKeyForBlock(index))\n\thh, err := highwayhash.NewDigest64(key[:])\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create hash: %v\", err)\n\t}\n\tmb.hh = hh\n\n\t<-dios\n\tmfd, err := os.OpenFile(mb.mfn, os.O_CREATE|os.O_RDWR, defaultFilePerms)\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\tif isPermissionError(err) {\n\t\t\treturn nil, err\n\t\t}\n\t\t_ = mb.dirtyCloseWithRemove(true)\n\t\treturn nil, fmt.Errorf(\"Error creating msg block file: %v\", err)\n\t}\n\tmb.mfd = mfd\n\n\t// Check if encryption is enabled.\n\tif fs.prf != nil {\n\t\tif err := fs.genEncryptionKeysForBlock(mb); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// If we know we will need this so go ahead and spin up.\n\tif !fs.fip {\n\t\tmb.spinUpFlushLoop()\n\t}\n\n\t// Add to our list of blocks and mark as last.\n\tfs.addMsgBlock(mb)\n\n\treturn mb, nil\n}\n\n// Generate the keys for this message block and write them out.\nfunc (fs *fileStore) genEncryptionKeysForBlock(mb *msgBlock) error {\n\tif mb == nil {\n\t\treturn nil\n\t}\n\tkey, bek, seed, encrypted, err := fs.genEncryptionKeys(fmt.Sprintf(\"%s:%d\", fs.cfg.Name, mb.index))\n\tif err != nil {\n\t\treturn err\n\t}\n\tmb.aek, mb.bek, mb.seed, mb.nonce = key, bek, seed, encrypted[:key.NonceSize()]\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tkeyFile := filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index))\n\tif _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\terr = fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms)\n\tif err != nil {\n\t\treturn err\n\t}\n\tmb.kfn = keyFile\n\treturn nil\n}\n\n// Stores a raw message with expected sequence number and timestamp.\n// Lock should be held.\nfunc (fs *fileStore) storeRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) (err error) {\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\n\t// Per subject max check needed.\n\tmmp := uint64(fs.cfg.MaxMsgsPer)\n\tvar psmc uint64\n\tpsmax := mmp > 0 && len(subj) > 0\n\tif psmax {\n\t\tif info, ok := fs.psim.Find(stringToBytes(subj)); ok {\n\t\t\tpsmc = info.total\n\t\t}\n\t}\n\n\tvar fseq uint64\n\t// Check if we are discarding new messages when we reach the limit.\n\t// If we are clustered, we do the enforcement above and should not disqualify\n\t// the message here since it could cause replicas to drift.\n\tif discardNewCheck && fs.cfg.Discard == DiscardNew {\n\t\tvar asl bool\n\t\tif psmax && psmc >= mmp {\n\t\t\t// If we are instructed to discard new per subject, this is an error.\n\t\t\tif fs.cfg.DiscardNewPer {\n\t\t\t\treturn ErrMaxMsgsPerSubject\n\t\t\t}\n\t\t\tif fseq, err = fs.firstSeqForSubj(subj); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tasl = true\n\t\t}\n\t\tif fs.cfg.MaxMsgs > 0 && fs.state.Msgs >= uint64(fs.cfg.MaxMsgs) && !asl {\n\t\t\treturn ErrMaxMsgs\n\t\t}\n\t\tif fs.cfg.MaxBytes > 0 && fs.state.Bytes+fileStoreMsgSize(subj, hdr, msg) > uint64(fs.cfg.MaxBytes) {\n\t\t\tif !asl || fs.sizeForSeq(fseq) < int(fileStoreMsgSize(subj, hdr, msg)) {\n\t\t\t\treturn ErrMaxBytes\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check sequence.\n\tif seq != fs.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn ErrSequenceMismatch\n\t\t}\n\t\tseq = fs.state.LastSeq + 1\n\t}\n\n\t// Return previous write errors immediately.\n\tif fs.werr != nil {\n\t\treturn fs.werr\n\t}\n\t// Persist any returned errors to be used in the future.\n\tdefer func() {\n\t\tif err != nil {\n\t\t\tfs.setWriteErr(err)\n\t\t}\n\t}()\n\n\t// Write msg record.\n\t// Add expiry bit to sequence if needed. This is so that if we need to\n\t// rebuild, we know which messages to look at more quickly.\n\tn, err := fs.writeMsgRecord(seq, ts, subj, hdr, msg)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Adjust top level tracking of per subject msg counts.\n\tif len(subj) > 0 && fs.psim != nil {\n\t\tindex := fs.lmb.index\n\t\tif info, ok := fs.psim.Find(stringToBytes(subj)); ok {\n\t\t\tinfo.total++\n\t\t\tif index > info.lblk {\n\t\t\t\tinfo.lblk = index\n\t\t\t}\n\t\t} else {\n\t\t\tfs.psim.Insert(stringToBytes(subj), psi{total: 1, fblk: index, lblk: index})\n\t\t\tfs.tsl += len(subj)\n\t\t}\n\t}\n\n\t// Adjust first if needed.\n\tnow := time.Unix(0, ts).UTC()\n\tif fs.state.Msgs == 0 {\n\t\tfs.state.FirstSeq = seq\n\t\tfs.state.FirstTime = now\n\t}\n\n\tfs.state.Msgs++\n\tfs.state.Bytes += n\n\tfs.state.LastSeq = seq\n\tfs.state.LastTime = now\n\n\t// Enforce per message limits.\n\t// We snapshotted psmc before our actual write, so >= comparison needed.\n\tif psmax && psmc >= mmp {\n\t\t// We may have done this above.\n\t\tif fseq == 0 {\n\t\t\tfseq, err = fs.firstSeqForSubj(subj)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif ok, err := fs.removeMsgViaLimits(fseq); err != nil {\n\t\t\treturn err\n\t\t} else if ok {\n\t\t\t// Make sure we are below the limit.\n\t\t\tif psmc--; psmc >= mmp {\n\t\t\t\tbsubj := stringToBytes(subj)\n\t\t\t\tfor info, ok := fs.psim.Find(bsubj); ok && info.total > mmp; info, ok = fs.psim.Find(bsubj) {\n\t\t\t\t\tif seq, err := fs.firstSeqForSubj(subj); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t} else if seq == 0 {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t} else if ok, err = fs.removeMsgViaLimits(seq); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t} else if !ok {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if mb := fs.selectMsgBlock(fseq); mb != nil {\n\t\t\t// If we are here we could not remove fseq from above, so rebuild.\n\t\t\tvar ld *LostStreamData\n\t\t\tif ld, _, err = mb.rebuildState(); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if ld != nil {\n\t\t\t\tfs.rebuildStateLocked(ld)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Limits checks and enforcement.\n\t// If they do any deletions they will update the\n\t// byte count on their own, so no need to compensate.\n\tif err = fs.enforceMsgLimit(); err != nil {\n\t\treturn err\n\t}\n\tif err = fs.enforceBytesLimit(); err != nil {\n\t\treturn err\n\t}\n\n\t// Per-message TTL.\n\tif ttl > 0 {\n\t\tif fs.ttls != nil {\n\t\t\texpires := time.Duration(ts) + (time.Second * time.Duration(ttl))\n\t\t\tfs.ttls.Add(seq, int64(expires))\n\t\t}\n\t\t// Need to count these regardless as we might want to enable TTLs\n\t\t// later via UpdateConfig.\n\t\tfs.lmb.ttls++\n\t}\n\n\t// Check if we have and need the age expiration timer running.\n\tswitch {\n\tcase fs.ttls != nil && ttl > 0:\n\t\tfs.resetAgeChk(0)\n\tcase fs.ageChk == nil && (fs.cfg.MaxAge > 0 || fs.ttls != nil):\n\t\tfs.startAgeChk()\n\t}\n\n\t// Message scheduling.\n\tif fs.scheduling != nil {\n\t\tif schedule, ok := nextMessageSchedule(hdr, ts); ok && !schedule.IsZero() {\n\t\t\tfs.scheduling.add(seq, subj, schedule.UnixNano())\n\t\t\tfs.lmb.schedules++\n\t\t} else {\n\t\t\tfs.scheduling.removeSubject(subj)\n\t\t}\n\n\t\t// Check for a repeating schedule and update such that it triggers again.\n\t\tif scheduleNext := bytesToString(sliceHeader(JSScheduleNext, hdr)); scheduleNext != _EMPTY_ && scheduleNext != JSScheduleNextPurge {\n\t\t\tscheduler := getMessageScheduler(hdr)\n\t\t\tif next, err := time.Parse(time.RFC3339Nano, scheduleNext); err == nil && scheduler != _EMPTY_ {\n\t\t\t\tfs.scheduling.update(scheduler, next.UnixNano())\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) setWriteErr(err error) {\n\tif fs.werr != nil {\n\t\treturn\n\t}\n\t// Ignore non-write errors.\n\tif err == ErrStoreClosed {\n\t\treturn\n\t}\n\t// If this is a not found report but do not disable.\n\tif os.IsNotExist(err) {\n\t\tfs.warn(\"Resource not found: %v\", err)\n\t\treturn\n\t}\n\tfs.error(\"Critical write error: %v\", err)\n\tfs.werr = err\n\tassert.Unreachable(\"Filestore encountered write error\", map[string]any{\n\t\t\"name\": fs.cfg.Name,\n\t\t\"err\": err,\n\t\t\"stack\": string(debug.Stack()),\n\t})\n}\n\n// StoreRawMsg stores a raw message with expected sequence number and timestamp.\nfunc (fs *fileStore) StoreRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) error {\n\tfs.mu.Lock()\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\terr := fs.storeRawMsg(subj, hdr, msg, seq, ts, ttl, discardNewCheck)\n\tcb := fs.scb\n\t// Check if first message timestamp requires expiry\n\t// sooner than initial replica expiry timer set to MaxAge when initializing.\n\tif !fs.receivedAny && fs.cfg.MaxAge != 0 && ts > 0 {\n\t\tfs.receivedAny = true\n\t\tfs.resetAgeChk(0)\n\t}\n\tfs.mu.Unlock()\n\n\tif err == nil && cb != nil {\n\t\tcb(1, int64(fileStoreMsgSize(subj, hdr, msg)), seq, subj)\n\t}\n\n\treturn err\n}\n\n// Store stores a message. We hold the main filestore lock for any write operation.\nfunc (fs *fileStore) StoreMsg(subj string, hdr, msg []byte, ttl int64) (uint64, int64, error) {\n\tfs.mu.Lock()\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn 0, 0, err\n\t}\n\tseq, ts := fs.state.LastSeq+1, time.Now().UnixNano()\n\t// This is called for a R1 with no expected sequence number, so perform DiscardNew checks on the store-level.\n\terr := fs.storeRawMsg(subj, hdr, msg, seq, ts, ttl, true)\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\tif err != nil {\n\t\tseq, ts = 0, 0\n\t} else if cb != nil {\n\t\tcb(1, int64(fileStoreMsgSize(subj, hdr, msg)), seq, subj)\n\t}\n\n\treturn seq, ts, err\n}\n\n// skipMsg will update this message block for a skipped message.\n// If we do not have any messages, just update the metadata, otherwise\n// we will place an empty record marking the sequence as used. The\n// sequence will be marked erased.\n// fs lock should be held.\nfunc (mb *msgBlock) skipMsg(seq uint64, now int64) error {\n\tif mb == nil {\n\t\treturn nil\n\t}\n\tvar needsRecord bool\n\n\tmb.mu.Lock()\n\tif err := mb.werr; err != nil {\n\t\tmb.mu.Unlock()\n\t\treturn err\n\t}\n\t// If we are empty can just do meta.\n\tif mb.msgs == 0 {\n\t\tatomic.StoreUint64(&mb.last.seq, seq)\n\t\tmb.last.ts = now\n\t\tatomic.StoreUint64(&mb.first.seq, seq+1)\n\t\tmb.first.ts = 0\n\t\tneedsRecord = mb == mb.fs.lmb\n\t} else {\n\t\tneedsRecord = true\n\t\tmb.dmap.Insert(seq)\n\t}\n\tif needsRecord {\n\t\tif err := mb.writeMsgRecordLocked(emptyRecordLen, seq|ebit, _EMPTY_, nil, nil, now, true, true); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\tmb.mu.Unlock()\n\tif !needsRecord {\n\t\tmb.kickFlusher()\n\t}\n\treturn nil\n}\n\n// SkipMsg will use the next sequence number but not store anything.\nfunc (fs *fileStore) SkipMsg(seq uint64) (uint64, error) {\n\t// Grab time.\n\tnow := ats.AccessTime()\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\treturn 0, err\n\t}\n\n\t// Check sequence matches our last sequence.\n\tif seq != fs.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn 0, ErrSequenceMismatch\n\t\t}\n\t\tseq = fs.state.LastSeq + 1\n\t}\n\n\t// Grab our current last message block.\n\tmb, err := fs.checkLastBlock(emptyRecordLen)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\t// Write skip msg.\n\tif err = mb.skipMsg(seq, now); err != nil {\n\t\tfs.setWriteErr(err)\n\t\treturn 0, err\n\t}\n\n\t// Update fs state.\n\tfs.state.LastSeq, fs.state.LastTime = seq, time.Unix(0, now).UTC()\n\tif fs.state.Msgs == 0 {\n\t\tfs.state.FirstSeq, fs.state.FirstTime = seq, time.Time{}\n\t}\n\tif seq == fs.state.FirstSeq {\n\t\tfs.state.FirstSeq, fs.state.FirstTime = seq+1, time.Time{}\n\t}\n\t// Mark as dirty for stream state.\n\tfs.dirty++\n\n\treturn seq, nil\n}\n\n// SkipMsgs skips multiple msgs. We will determine if we can fit into current lmb or we need to create a new block.\nfunc (fs *fileStore) SkipMsgs(seq uint64, num uint64) error {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\treturn err\n\t}\n\n\t// Check sequence matches our last sequence.\n\tif seq != fs.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn ErrSequenceMismatch\n\t\t}\n\t\tseq = fs.state.LastSeq + 1\n\t}\n\n\t// Limit number of dmap entries\n\tconst maxDeletes = 64 * 1024\n\tmb := fs.lmb\n\n\tvar msgs uint64\n\tnumDeletes := int(num)\n\tif mb != nil {\n\t\tmb.mu.RLock()\n\t\tnumDeletes += mb.dmap.Size()\n\t\tmsgs = mb.msgs\n\t\tmb.mu.RUnlock()\n\t}\n\tif mb == nil || numDeletes > maxDeletes && msgs > 0 || msgs > 0 && mb.blkSize()+emptyRecordLen > fs.fcfg.BlockSize {\n\t\tvar err error\n\t\tif mb, err = fs.newMsgBlockForWrite(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Insert into dmap all entries and place last as marker.\n\tnow := ats.AccessTime()\n\tlseq := seq + num - 1\n\n\tmb.mu.Lock()\n\tif err := mb.werr; err != nil {\n\t\tmb.mu.Unlock()\n\t\treturn err\n\t}\n\t// If we are empty update meta directly.\n\tif mb.msgs == 0 {\n\t\tatomic.StoreUint64(&mb.last.seq, lseq)\n\t\tmb.last.ts = now\n\t\tatomic.StoreUint64(&mb.first.seq, lseq+1)\n\t\tmb.first.ts = 0\n\t} else {\n\t\tfor ; seq <= lseq; seq++ {\n\t\t\tmb.dmap.Insert(seq)\n\t\t}\n\t}\n\t// Write out our placeholder.\n\terr := mb.writeMsgRecordLocked(emptyRecordLen, lseq|ebit, _EMPTY_, nil, nil, now, true, true)\n\tmb.mu.Unlock()\n\tif err != nil {\n\t\tfs.setWriteErr(err)\n\t\treturn err\n\t}\n\n\t// Now update FS accounting.\n\t// Update fs state.\n\tfs.state.LastSeq, fs.state.LastTime = lseq, time.Unix(0, now).UTC()\n\tif fs.state.Msgs == 0 {\n\t\tfs.state.FirstSeq, fs.state.FirstTime = lseq+1, time.Time{}\n\t}\n\n\t// Mark as dirty for stream state.\n\tfs.dirty++\n\n\treturn nil\n}\n\n// FlushAllPending flushes all data that was still pending to be written.\nfunc (fs *fileStore) FlushAllPending() error {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\t// Return previous write errors immediately.\n\tif fs.werr != nil {\n\t\treturn fs.werr\n\t}\n\treturn fs.checkAndFlushLastBlock()\n}\n\n// Lock should be held.\nfunc (fs *fileStore) rebuildFirst() error {\n\tif len(fs.blks) == 0 {\n\t\treturn nil\n\t}\n\tfmb := fs.blks[0]\n\tif fmb == nil {\n\t\treturn nil\n\t}\n\n\tld, _, _ := fmb.rebuildState()\n\tfmb.mu.RLock()\n\tisEmpty := fmb.msgs == 0\n\tfmb.mu.RUnlock()\n\tif isEmpty {\n\t\tfmb.mu.Lock()\n\t\terr := fs.forceRemoveMsgBlock(fmb)\n\t\tfmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := fs.selectNextFirst(); err != nil {\n\t\treturn err\n\t}\n\tfs.rebuildStateLocked(ld)\n\treturn nil\n}\n\n// Optimized helper function to return first sequence.\n// subj will always be publish subject here, meaning non-wildcard.\n// We assume a fast check that this subj even exists already happened.\n// Write lock should be held.\nfunc (fs *fileStore) firstSeqForSubj(subj string) (uint64, error) {\n\tif len(fs.blks) == 0 {\n\t\treturn 0, nil\n\t}\n\n\t// See if we can optimize where we start.\n\tstart, stop := fs.blks[0].index, fs.lmb.index\n\tif info, ok := fs.psim.Find(stringToBytes(subj)); ok {\n\t\tstart, stop = info.fblk, info.lblk\n\t}\n\n\tfor i := start; i <= stop; i++ {\n\t\tmb := fs.bim[i]\n\t\tif mb == nil {\n\t\t\tcontinue\n\t\t}\n\t\t// If we need to load msgs here and we need to walk multiple blocks this\n\t\t// could tie up the upper fs lock, so release while dealing with the block.\n\t\tfs.mu.Unlock()\n\n\t\tmb.mu.Lock()\n\t\tvar shouldExpire bool\n\t\tif mb.fssNotLoaded() {\n\t\t\t// Make sure we have fss loaded.\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\t// Re-acquire fs lock\n\t\t\t\tfs.mu.Lock()\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\tshouldExpire = true\n\t\t}\n\t\t// Mark fss activity.\n\t\tmb.lsts = ats.AccessTime()\n\n\t\tbsubj := stringToBytes(subj)\n\t\tif ss, ok := mb.fss.Find(bsubj); ok && ss != nil {\n\t\t\tvar err error\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\terr = mb.recalculateForSubj(subj, ss)\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t\t// Re-acquire fs lock\n\t\t\tfs.mu.Lock()\n\t\t\t// Adjust first if it was not where we thought it should be.\n\t\t\tif i != start {\n\t\t\t\tif info, ok := fs.psim.Find(bsubj); ok {\n\t\t\t\t\tinfo.fblk = i\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\treturn ss.First, nil\n\t\t}\n\t\t// If we did not find it and we loaded this msgBlock try to expire as long as not the last.\n\t\tif shouldExpire {\n\t\t\t// Expire this cache before moving on.\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tmb.mu.Unlock()\n\t\t// Re-acquire fs lock\n\t\tfs.mu.Lock()\n\t}\n\treturn 0, nil\n}\n\n// Will check the msg limit and drop firstSeq msg if needed.\n// Lock should be held.\nfunc (fs *fileStore) enforceMsgLimit() error {\n\tif fs.cfg.Discard != DiscardOld {\n\t\treturn nil\n\t}\n\tif fs.cfg.MaxMsgs <= 0 || fs.state.Msgs <= uint64(fs.cfg.MaxMsgs) {\n\t\treturn nil\n\t}\n\tfor nmsgs := fs.state.Msgs; nmsgs > uint64(fs.cfg.MaxMsgs); nmsgs = fs.state.Msgs {\n\t\t// If the first block can be removed fully, purge it entirely without needing to walk sequences.\n\t\tif len(fs.blks) > 0 {\n\t\t\tfmb := fs.blks[0]\n\t\t\tfmb.mu.RLock()\n\t\t\tmsgs := fmb.msgs\n\t\t\tfmb.mu.RUnlock()\n\t\t\tif nmsgs-msgs > uint64(fs.cfg.MaxMsgs) {\n\t\t\t\tif err := fs.purgeMsgBlock(fmb); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif removed, err := fs.deleteFirstMsg(); err != nil {\n\t\t\treturn err\n\t\t} else if !removed {\n\t\t\treturn fs.rebuildFirst()\n\t\t}\n\t}\n\treturn nil\n}\n\n// Will check the bytes limit and drop msgs if needed.\n// Lock should be held.\nfunc (fs *fileStore) enforceBytesLimit() error {\n\tif fs.cfg.Discard != DiscardOld {\n\t\treturn nil\n\t}\n\tif fs.cfg.MaxBytes <= 0 || fs.state.Bytes <= uint64(fs.cfg.MaxBytes) {\n\t\treturn nil\n\t}\n\tfor bs := fs.state.Bytes; bs > uint64(fs.cfg.MaxBytes); bs = fs.state.Bytes {\n\t\t// If the first block can be removed fully, purge it entirely without needing to walk sequences.\n\t\tif len(fs.blks) > 0 {\n\t\t\tfmb := fs.blks[0]\n\t\t\tfmb.mu.RLock()\n\t\t\tbytes := fmb.bytes\n\t\t\tfmb.mu.RUnlock()\n\t\t\tif bs-bytes > uint64(fs.cfg.MaxBytes) {\n\t\t\t\tif err := fs.purgeMsgBlock(fmb); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif removed, err := fs.deleteFirstMsg(); err != nil {\n\t\t\treturn err\n\t\t} else if !removed {\n\t\t\treturn fs.rebuildFirst()\n\t\t}\n\t}\n\treturn nil\n}\n\n// Will make sure we have limits honored for max msgs per subject on recovery or config update.\n// We will make sure to go through all msg blocks etc. but in practice this\n// will most likely only be the last one, so can take a more conservative approach.\n// Lock should be held.\nfunc (fs *fileStore) enforceMsgPerSubjectLimit(fireCallback bool) error {\n\tstart := time.Now()\n\tdefer func() {\n\t\tif took := time.Since(start); took > time.Minute {\n\t\t\tfs.warn(\"enforceMsgPerSubjectLimit took %v\", took.Round(time.Millisecond))\n\t\t}\n\t}()\n\n\tmaxMsgsPer := uint64(fs.cfg.MaxMsgsPer)\n\n\t// We may want to suppress callbacks from remove during this process\n\t// since these should have already been deleted and accounted for.\n\tif !fireCallback {\n\t\tcb := fs.scb\n\t\tfs.scb = nil\n\t\tdefer func() { fs.scb = cb }()\n\t}\n\n\tvar numMsgs uint64\n\n\t// collect all that are not correct.\n\tneedAttention := stree.NewSubjectTree[uint64]()\n\tfblk, lblk := uint32(math.MaxUint32), uint32(0)\n\tfs.psim.IterFast(func(subj []byte, psi *psi) bool {\n\t\tnumMsgs += psi.total\n\t\tif psi.total > maxMsgsPer {\n\t\t\tneedAttention.Insert(subj, psi.total)\n\t\t\tif psi.fblk < fblk {\n\t\t\t\tfblk = psi.fblk\n\t\t\t}\n\t\t\tif psi.lblk > lblk {\n\t\t\t\tlblk = psi.lblk\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\t// We had an issue with a use case where psim (and hence fss) were correct but idx was not and was not properly being caught.\n\t// So do a quick sanity check here. If we detect a skew do a rebuild then re-check.\n\tif numMsgs != fs.state.Msgs {\n\t\tfs.warn(\"Detected skew in subject-based total (%d) vs raw total (%d), rebuilding\", numMsgs, fs.state.Msgs)\n\t\t// Clear any global subject state.\n\t\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\t\tfor _, mb := range fs.blks {\n\t\t\tld, _, err := mb.rebuildState()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t} else if ld != nil {\n\t\t\t\tfs.addLostData(ld)\n\t\t\t}\n\t\t\tif err = fs.populateGlobalPerSubjectInfo(mb); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\t// Rebuild fs state too.\n\t\tfs.rebuildStateLocked(nil)\n\t\t// Need to redo blocks that need attention.\n\t\tneedAttention.Empty()\n\t\tfblk, lblk = uint32(math.MaxUint32), uint32(0)\n\t\tfs.psim.IterFast(func(subj []byte, psi *psi) bool {\n\t\t\tif psi.total > maxMsgsPer {\n\t\t\t\tneedAttention.Insert(subj, psi.total)\n\t\t\t\tif psi.fblk < fblk {\n\t\t\t\t\tfblk = psi.fblk\n\t\t\t\t}\n\t\t\t\tif psi.lblk > lblk {\n\t\t\t\t\tlblk = psi.lblk\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n\n\t// If nothing to do then stop.\n\tif fblk == math.MaxUint32 {\n\t\treturn nil\n\t}\n\n\t// Collect all the msgBlks we alter.\n\tblks := make(map[*msgBlock]struct{})\n\n\t// For re-use below.\n\tvar sm StoreMsg\n\tvar fss *stree.SubjectTree[*SimpleState]\n\tfor i := fblk; i <= lblk; i++ {\n\t\tmb := fs.bim[i]\n\t\tif mb == nil {\n\t\t\tcontinue\n\t\t}\n\t\tmb.mu.Lock()\n\t\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t\t// It isn't safe to intersect mb.fss directly, because removeMsgViaLimits modifies it\n\t\t// during the iteration, which can cause us to miss keys. We won't copy the entire\n\t\t// SimpleState structs though but rather just take pointers for speed.\n\t\tfss = fss.Empty()\n\t\tmb.fss.IterFast(func(subject []byte, val *SimpleState) bool {\n\t\t\tfss.Insert(subject, val)\n\t\t\treturn true\n\t\t})\n\t\tmb.mu.Unlock()\n\t\tvar ierr error\n\t\tstree.LazyIntersect(needAttention, fss, func(subj []byte, total *uint64, ssptr **SimpleState) {\n\t\t\tif ssptr == nil || total == nil || ierr != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tss := *ssptr\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tmb.mu.Lock()\n\t\t\t\tierr = mb.recalculateForSubj(bytesToString(subj), ss)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tif ierr != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor first := ss.First; *total > maxMsgsPer && first <= ss.Last; {\n\t\t\t\tm, _, err := mb.firstMatching(bytesToString(subj), false, first, &sm)\n\t\t\t\tif err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tfirst = m.seq + 1\n\t\t\t\tif removed, _ := fs.removeMsgViaLimits(m.seq); removed {\n\t\t\t\t\tblks[mb] = struct{}{}\n\t\t\t\t\t*total--\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t\tif ierr != nil {\n\t\t\treturn ierr\n\t\t}\n\t}\n\n\t// Expire the cache if we can.\n\tfor mb := range blks {\n\t\tmb.mu.Lock()\n\t\tif mb.msgs > 0 {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t}\n\t\tmb.mu.Unlock()\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) deleteFirstMsg() (bool, error) {\n\treturn fs.removeMsgViaLimits(fs.state.FirstSeq)\n}\n\n// If we remove via limits that can always be recovered on a restart we\n// do not force the system to update the index file.\n// Lock should be held.\nfunc (fs *fileStore) removeMsgViaLimits(seq uint64) (bool, error) {\n\treturn fs.removeMsg(seq, false, true, false)\n}\n\n// RemoveMsg will remove the message from this store.\n// Will return the number of bytes removed.\nfunc (fs *fileStore) RemoveMsg(seq uint64) (bool, error) {\n\treturn fs.removeMsg(seq, false, false, true)\n}\n\nfunc (fs *fileStore) EraseMsg(seq uint64) (bool, error) {\n\treturn fs.removeMsg(seq, true, false, true)\n}\n\n// Convenience function to remove per subject tracking at the filestore level.\n// Lock should be held.\nfunc (fs *fileStore) removePerSubject(subj string) uint64 {\n\tif len(subj) == 0 || fs.psim == nil {\n\t\treturn 0\n\t}\n\t// We do not update sense of fblk here but will do so when we resolve during lookup.\n\tbsubj := stringToBytes(subj)\n\tif info, ok := fs.psim.Find(bsubj); ok {\n\t\tinfo.total--\n\t\tif info.total == 1 {\n\t\t\tinfo.fblk = info.lblk\n\t\t} else if info.total == 0 {\n\t\t\tif _, ok = fs.psim.Delete(bsubj); ok {\n\t\t\t\tfs.tsl -= len(subj)\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t\treturn info.total\n\t}\n\treturn 0\n}\n\n// Remove a message, optionally rewriting the mb file.\nfunc (fs *fileStore) removeMsg(seq uint64, secure, viaLimits, needFSLock bool) (bool, error) {\n\tif seq == 0 {\n\t\treturn false, ErrStoreMsgNotFound\n\t}\n\tfsLock := func() {\n\t\tif needFSLock {\n\t\t\tfs.mu.Lock()\n\t\t}\n\t}\n\tfsUnlock := func() {\n\t\tif needFSLock {\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}\n\n\tfsLock()\n\tdefer fsUnlock()\n\n\tif fs.isClosed() {\n\t\treturn false, ErrStoreClosed\n\t}\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\treturn false, err\n\t}\n\t// If in encrypted mode negate secure rewrite here.\n\tif secure && fs.prf != nil {\n\t\tsecure = false\n\t}\n\n\tmb := fs.selectMsgBlock(seq)\n\tif mb == nil {\n\t\tvar err = ErrStoreEOF\n\t\tif seq <= fs.state.LastSeq {\n\t\t\terr = ErrStoreMsgNotFound\n\t\t}\n\t\treturn false, err\n\t}\n\n\treturn fs.removeMsgFromBlock(mb, seq, secure, viaLimits)\n}\n\n// Remove a message from the given block, optionally rewriting the mb file.\n// fs lock should be held.\nfunc (fs *fileStore) removeMsgFromBlock(mb *msgBlock, seq uint64, secure, viaLimits bool) (removed bool, rerr error) {\n\tmb.mu.Lock()\n\n\t// See if we are closed or the sequence number is still relevant or if we know its deleted.\n\tif mb.closed || seq < atomic.LoadUint64(&mb.first.seq) || mb.dmap.Exists(seq) {\n\t\tmb.mu.Unlock()\n\t\treturn false, nil\n\t}\n\n\t// Persist any write errors.\n\tdefer func() {\n\t\tif rerr != nil {\n\t\t\tfs.setWriteErr(rerr)\n\t\t}\n\t}()\n\n\tfifo := seq == atomic.LoadUint64(&mb.first.seq)\n\tisLastBlock := mb == fs.lmb\n\tisEmpty := mb.msgs == 1 // ... about to be zero though.\n\n\t// We used to not have to load in the messages except with callbacks or the filtered subject state (which is now always on).\n\t// Now just load regardless.\n\t// TODO(dlc) - Figure out a way not to have to load it in, we need subject tracking outside main data block.\n\tvar didLoad bool\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t\tdidLoad = true\n\t}\n\tfinishedWithCache := func() {\n\t\tif didLoad {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t}\n\n\tvar (\n\t\tsmv StoreMsg\n\t\tsubj string\n\t\tts int64\n\t\tlhdr, lmsg int\n\t\tttl int64\n\t)\n\t// We don't use a copy as long as that's possible. When unlocking mb or erasing, we'll copy the subject.\n\tsm, err := mb.cacheLookupNoCopy(seq, &smv)\n\tif err != nil {\n\t\tfinishedWithCache()\n\t\tmb.mu.Unlock()\n\t\t// Mimic err behavior from above check to dmap. No error returned if already removed.\n\t\tif err == ErrStoreMsgNotFound || err == errDeletedMsg {\n\t\t\terr = nil\n\t\t}\n\t\treturn false, err\n\t} else if sm != nil {\n\t\tsubj = sm.subj\n\t\tts = sm.ts\n\t\tlhdr = len(sm.hdr)\n\t\tlmsg = len(sm.msg)\n\t\tttl, _ = getMessageTTL(sm.hdr)\n\t}\n\n\t// Check if we need to write a deleted record tombstone.\n\t// This is for user initiated removes or to hold the first seq\n\t// when the last block is empty.\n\t// If not via limits and not empty (empty writes tombstone below if last) write tombstone.\n\tif !viaLimits && !isEmpty && sm != nil {\n\t\t// Need to copy the subject since we unlock and re-acquire, and the cache could change.\n\t\tsubj = copyString(subj)\n\t\tmb.mu.Unlock() // Only safe way to checkLastBlock is to unlock here...\n\t\tlmb, err := fs.checkLastBlock(emptyRecordLen)\n\t\tif err != nil {\n\t\t\tmb.mu.Lock()\n\t\t\tfinishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t\tif err := lmb.writeTombstone(seq, ts); err != nil {\n\t\t\tmb.mu.Lock()\n\t\t\tfinishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t\tmb.mu.Lock() // We'll need the lock back to carry on safely.\n\t}\n\n\t// Grab size\n\tmsz := fileStoreMsgSizeRaw(len(subj), lhdr, lmsg)\n\n\t// Set cache timestamp for last remove.\n\tmb.lrts = ats.AccessTime()\n\n\t// Must always perform the erase, even if the block is empty as it could contain tombstones.\n\tif secure {\n\t\t// Grab record info, but use the pre-computed record length.\n\t\tri, _, _, err := mb.slotInfo(int(seq - mb.cache.fseq))\n\t\tif err != nil {\n\t\t\tfinishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t\t// Need to copy the subject, as eraseMsg will overwrite the cache and we won't\n\t\t// be able to access sm.subj anymore later on.\n\t\tsubj = copyString(subj)\n\t\tif err := mb.eraseMsg(seq, int(ri), int(msz), isLastBlock); err != nil {\n\t\t\tfinishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t}\n\n\t// Global stats\n\tif fs.state.Msgs > 0 {\n\t\tfs.state.Msgs--\n\t}\n\tif msz < fs.state.Bytes {\n\t\tfs.state.Bytes -= msz\n\t} else {\n\t\tfs.state.Bytes = 0\n\t}\n\n\t// Now local mb updates.\n\tif mb.msgs > 0 {\n\t\tmb.msgs--\n\t}\n\tif msz < mb.bytes {\n\t\tmb.bytes -= msz\n\t} else {\n\t\tmb.bytes = 0\n\t}\n\n\t// Allow us to check compaction again.\n\tmb.noCompact = false\n\n\t// Mark as dirty for stream state.\n\tfs.dirty++\n\n\t// If we are tracking subjects here make sure we update that accounting.\n\tif err = mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\tfinishedWithCache()\n\t\tmb.mu.Unlock()\n\t\treturn false, err\n\t}\n\n\t// If we are tracking multiple subjects here make sure we update that accounting.\n\tif _, err = mb.removeSeqPerSubject(subj, seq); err != nil {\n\t\tfinishedWithCache()\n\t\tmb.mu.Unlock()\n\t\treturn false, err\n\t}\n\tfs.removePerSubject(subj)\n\tif fs.ttls != nil && ttl > 0 {\n\t\texpires := time.Duration(ts) + (time.Second * time.Duration(ttl))\n\t\tfs.ttls.Remove(seq, int64(expires))\n\t}\n\n\tif fifo {\n\t\tmb.selectNextFirst()\n\t\tif !isEmpty {\n\t\t\t// Can update this one in place.\n\t\t\tif seq == fs.state.FirstSeq {\n\t\t\t\tfs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq) // new one.\n\t\t\t\tif mb.first.ts == 0 {\n\t\t\t\t\tfs.state.FirstTime = time.Time{}\n\t\t\t\t} else {\n\t\t\t\t\tfs.state.FirstTime = time.Unix(0, mb.first.ts).UTC()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else if !isEmpty {\n\t\t// Out of order delete.\n\t\tmb.dmap.Insert(seq)\n\t\t// Make simple check here similar to Compact(). If we can save 50% and over a certain threshold do inline.\n\t\t// All other more thorough cleanup will happen in syncBlocks logic.\n\t\t// Note that we do not have to store empty records for the deleted, so don't use to calculate.\n\t\t// TODO(dlc) - This should not be inline, should kick the sync routine.\n\t\tif !isLastBlock && mb.shouldCompactInline() {\n\t\t\tif err = mb.compact(); err != nil {\n\t\t\t\tfinishedWithCache()\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif secure {\n\t\tif ld, _ := mb.flushPendingMsgsLocked(); ld != nil {\n\t\t\t// We have the mb lock here, this needs the mb locks so do in its own go routine.\n\t\t\tgo fs.rebuildState(ld)\n\t\t}\n\t}\n\n\t// If empty remove this block and check if we need to update first sequence.\n\t// We will write a tombstone at the end.\n\tvar firstSeqNeedsUpdate bool\n\tif isEmpty {\n\t\t// This writes tombstone iff mb == lmb, so no need to do above.\n\t\tif err = fs.removeMsgBlock(mb); err != nil {\n\t\t\tfinishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t\treturn false, err\n\t\t}\n\t\tfirstSeqNeedsUpdate = seq == fs.state.FirstSeq\n\t}\n\tfinishedWithCache()\n\tmb.mu.Unlock()\n\n\t// If we emptied the current message block and the seq was state.FirstSeq\n\t// then we need to jump message blocks. We will also write the index so\n\t// we don't lose track of the first sequence.\n\tif firstSeqNeedsUpdate {\n\t\tif err = fs.selectNextFirst(); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t}\n\n\tif cb := fs.scb; cb != nil {\n\t\t// If we have a callback registered we need to release lock regardless since cb might need it to lookup msg, etc.\n\t\tfs.mu.Unlock()\n\t\t// Storage updates.\n\t\tdelta := int64(msz)\n\t\tcb(-1, -delta, seq, subj)\n\n\t\tfs.mu.Lock()\n\t}\n\n\treturn true, nil\n}\n\n// Remove all messages in the range [first, last]\n// Lock should be held.\nfunc (fs *fileStore) removeMsgsInRange(first, last uint64, viaLimits bool) error {\n\tlast = min(last, fs.state.LastSeq)\n\tif first > last {\n\t\treturn nil\n\t}\n\n\tfirstBlock := sort.Search(len(fs.blks), func(i int) bool {\n\t\treturn atomic.LoadUint64(&fs.blks[i].last.seq) >= first\n\t})\n\tif firstBlock >= len(fs.blks) {\n\t\treturn nil\n\t}\n\n\tfor i := firstBlock; i < len(fs.blks); {\n\t\tmb := fs.blks[i]\n\t\tmbFirstSeq := atomic.LoadUint64(&mb.first.seq)\n\t\tmbLastSeq := atomic.LoadUint64(&mb.last.seq)\n\t\tif mbFirstSeq > last {\n\t\t\tbreak\n\t\t}\n\t\tif mbFirstSeq >= first && mbLastSeq <= last && mb.numPriorTombs() == 0 {\n\t\t\t// If this block stores no tombstones for previous blocks,\n\t\t\t// and its sequences are within the range to be removed,\n\t\t\t// we can get rid of the block entirely. To do that we use\n\t\t\t// purgeMgsBlock, which also removes the block from fs.blks.\n\t\t\t// After purgeMsgBlock, i will be the index of the following\n\t\t\t// msgBlock, if any. Therefore, continue without incrementing i.\n\t\t\tif err := fs.purgeMsgBlock(mb); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tfrom := max(first, mbFirstSeq)\n\t\t\tto := min(last, mbLastSeq)\n\t\t\tfor seq := from; seq <= to; seq++ {\n\t\t\t\tif _, err := fs.removeMsgFromBlock(mb, seq, false, viaLimits); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\ti++\n\t\t}\n\t}\n\treturn nil\n}\n\n// Tests whether we should try to compact this block while inline removing msgs.\n// We will want rbytes to be over the minimum and have a 2x potential savings.\n// If we compacted before but rbytes didn't improve much, guard against constantly compacting.\n// Lock should be held.\nfunc (mb *msgBlock) shouldCompactInline() bool {\n\treturn mb.rbytes > compactMinimum && mb.bytes*2 < mb.rbytes && (mb.cbytes == 0 || mb.bytes*2 < mb.cbytes)\n}\n\n// Tests whether we should try to compact this block while running periodic sync.\n// We will want rbytes to be over the minimum and have a 2x potential savings.\n// Ignores 2MB minimum.\n// Lock should be held.\nfunc (mb *msgBlock) shouldCompactSync() bool {\n\treturn mb.bytes*2 < mb.rbytes && !mb.noCompact\n}\n\n// This will compact and rewrite this block. This version will not process any tombstone cleanup.\n// Write lock needs to be held.\nfunc (mb *msgBlock) compact() error {\n\treturn mb.compactWithFloor(0, nil)\n}\n\n// This will compact and rewrite this block. This should only be called when we know we want to rewrite this block.\n// This should not be called on the lmb since we will prune tail deleted messages which could cause issues with\n// writing new messages. We will silently bail on any issues with the underlying block and let someone else detect.\n// if fseq > 0 we will attempt to cleanup stale tombstones.\n// Write lock needs to be held.\nfunc (mb *msgBlock) compactWithFloor(floor uint64, fsDmap *avl.SequenceSet) error {\n\twasLoaded := mb.cache != nil && mb.cacheAlreadyLoaded()\n\tif !wasLoaded {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tdefer mb.finishedWithCache()\n\n\tbuf := mb.cache.buf\n\tnbuf := getMsgBlockBuf(len(buf))\n\t// Recycle our nbuf when we are done.\n\tdefer recycleMsgBlockBuf(nbuf)\n\n\tvar le = binary.LittleEndian\n\tvar firstSet bool\n\tvar last uint64\n\tvar msgs uint64\n\n\tfseq := atomic.LoadUint64(&mb.first.seq)\n\tlseq := atomic.LoadUint64(&mb.last.seq)\n\tisDeleted := func(seq uint64) bool {\n\t\treturn seq == 0 || seq&ebit != 0 || mb.dmap.Exists(seq) || seq < fseq\n\t}\n\n\tfor index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; {\n\t\tif index+msgHdrSize > lbuf {\n\t\t\treturn fmt.Errorf(\"message overrun\")\n\t\t}\n\t\thdr := buf[index : index+msgHdrSize]\n\t\trl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:]))\n\t\thasHeaders := rl&hbit != 0\n\t\t// Clear any headers bit that could be set.\n\t\trl &^= hbit\n\t\tshlen := slen\n\t\tif hasHeaders {\n\t\t\tshlen += 4\n\t\t}\n\t\tdlen := int(rl) - msgHdrSize\n\t\t// Do some quick sanity checks here.\n\t\tif dlen < 0 || shlen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh {\n\t\t\treturn fmt.Errorf(\"sanity check failed\")\n\t\t}\n\t\t// Only need to process non-deleted messages.\n\t\tseq := le.Uint64(hdr[4:])\n\t\tts := int64(le.Uint64(hdr[12:]))\n\n\t\tif !isDeleted(seq) {\n\t\t\t// Check for tombstones.\n\t\t\tif seq&tbit != 0 {\n\t\t\t\tseq = seq &^ tbit\n\t\t\t\t// If this entry is for a lower seq than ours then keep around.\n\t\t\t\t// We also check that it is greater than our floor. Floor is zero on normal\n\t\t\t\t// calls to compact.\n\t\t\t\t// If the global delete map is set, check if a tombstone is still\n\t\t\t\t// referencing a message in another block. If not, it can be removed.\n\t\t\t\tif seq < fseq && seq >= floor && (fsDmap == nil || fsDmap.Exists(seq)) {\n\t\t\t\t\tnbuf = append(nbuf, buf[index:index+rl]...)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Normal message here.\n\t\t\t\tmsgs++\n\t\t\t\tnbuf = append(nbuf, buf[index:index+rl]...)\n\t\t\t\tif !firstSet {\n\t\t\t\t\tfirstSet = true\n\t\t\t\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\t\t\t}\n\t\t\t\tif seq >= last {\n\t\t\t\t\tlast = seq\n\t\t\t\t\tatomic.StoreUint64(&mb.last.seq, last)\n\t\t\t\t\tmb.last.ts = ts\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Advance to next record.\n\t\tindex += rl\n\t}\n\n\t// Handle compression\n\tif mb.cmp != NoCompression && len(nbuf) > 0 {\n\t\toriginalSize := len(nbuf)\n\t\tvar err error\n\t\tif nbuf, err = mb.cmp.Compress(nbuf); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmeta := &CompressionInfo{\n\t\t\tAlgorithm: mb.cmp,\n\t\t\tOriginalSize: uint64(originalSize),\n\t\t}\n\t\tnbuf = append(meta.MarshalMetadata(), nbuf...)\n\t}\n\n\t// Check for encryption.\n\tif mb.bek != nil && len(nbuf) > 0 {\n\t\t// Recreate to reset counter.\n\t\trbek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trbek.XORKeyStream(nbuf, nbuf)\n\t}\n\n\t// Close FDs first.\n\tmb.closeFDsLocked()\n\n\t// We will write to a new file and mv/rename it in case of failure.\n\tmfn := filepath.Join(mb.fs.fcfg.StoreDir, msgDir, fmt.Sprintf(newScan, mb.index))\n\t<-dios\n\terr := os.WriteFile(mfn, nbuf, defaultFilePerms)\n\tdios <- struct{}{}\n\tif err != nil {\n\t\t_ = os.Remove(mfn)\n\t\treturn err\n\t}\n\tif err := os.Rename(mfn, mb.mfn); err != nil {\n\t\t_ = os.Remove(mfn)\n\t\treturn err\n\t}\n\n\t// Make sure to sync\n\tmb.needSync = true\n\n\t// Capture the updated rbytes.\n\tif rbytes := uint64(len(nbuf)); rbytes == mb.rbytes {\n\t\t// No change, so set our noCompact bool here to avoid attempting to continually compress in syncBlocks.\n\t\tmb.noCompact = true\n\t} else {\n\t\tmb.rbytes = rbytes\n\t}\n\tmb.cbytes = mb.bytes\n\n\t// Remove any seqs from the beginning of the blk.\n\tfor seq, nfseq := fseq, atomic.LoadUint64(&mb.first.seq); seq < nfseq; seq++ {\n\t\tmb.dmap.Delete(seq)\n\t}\n\t// Remove any seqs from the ending of the blk.\n\tfor seq, nlseq := lseq, atomic.LoadUint64(&mb.last.seq); seq > nlseq; seq-- {\n\t\tmb.dmap.Delete(seq)\n\t}\n\t// If the block itself has no messages anymore (could still contain tombstones though),\n\t// then we need to account for that by resetting the last sequence and timestamp.\n\tif msgs == 0 {\n\t\tatomic.StoreUint64(&mb.last.seq, fseq-1)\n\t\tmb.last.ts = 0\n\t\tmb.dmap.Empty()\n\t}\n\t// Make sure we clear the cache since no longer valid.\n\tmb.clearCacheAndOffset()\n\t// If we entered with the msgs loaded make sure to reload them.\n\tif wasLoaded {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Grab info from a slot.\n// Lock should be held.\nfunc (mb *msgBlock) slotInfo(slot int) (uint32, uint32, bool, error) {\n\tswitch {\n\tcase mb.cache == nil: // Shouldn't be possible, but check it anyway.\n\t\treturn 0, 0, false, errNoCache\n\tcase slot < 0:\n\t\tmb.fs.warn(\"Partial cache: offset slot index %d is less zero\", slot)\n\t\treturn 0, 0, false, errPartialCache\n\tcase slot >= len(mb.cache.idx):\n\t\tmb.fs.warn(\"Partial cache: offset slot index %d is greater than index len %d\", slot, len(mb.cache.idx))\n\t\treturn 0, 0, false, errPartialCache\n\t}\n\n\tbi := mb.cache.idx[slot]\n\tri, hashChecked := (bi &^ cbit), (bi&cbit) != 0\n\n\t// If this is a deleted slot return here.\n\tif bi == dbit {\n\t\treturn 0, 0, false, errDeletedMsg\n\t}\n\n\t// Determine record length\n\tvar rl uint32\n\t// Need to account for dbit markers in idx.\n\t// So we will walk until we find valid idx slot to calculate rl.\n\tfor i := 1; slot+i < len(mb.cache.idx); i++ {\n\t\tni := mb.cache.idx[slot+i] &^ cbit\n\t\tif ni == dbit {\n\t\t\tcontinue\n\t\t}\n\t\trl = ni - ri\n\t\tbreak\n\t}\n\t// check if we had all trailing dbits.\n\t// If so use len of cache buf minus ri.\n\tif rl == 0 {\n\t\trl = uint32(len(mb.cache.buf)) - ri\n\t}\n\tif rl < msgHdrSize {\n\t\treturn 0, 0, false, errBadMsg{mb.mfn, fmt.Sprintf(\"length too short for slot %d\", slot)}\n\t}\n\treturn uint32(ri), rl, hashChecked, nil\n}\n\nfunc (fs *fileStore) isClosed() bool {\n\treturn fs.closed.Load()\n}\n\n// Will spin up our flush loop.\nfunc (mb *msgBlock) spinUpFlushLoop() {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\tmb.spinUpFlushLoopLocked()\n}\n\n// Will spin up our flush loop.\n// Lock should be held.\nfunc (mb *msgBlock) spinUpFlushLoopLocked() {\n\t// Are we already running or closed?\n\tif mb.flusher || mb.closed {\n\t\treturn\n\t}\n\tmb.flusher = true\n\tmb.fch = make(chan struct{}, 1)\n\tmb.qch = make(chan struct{})\n\tfch, qch := mb.fch, mb.qch\n\n\tgo mb.flushLoop(fch, qch)\n}\n\n// Raw low level kicker for flush loops.\nfunc kickFlusher(fch chan struct{}) {\n\tif fch != nil {\n\t\tselect {\n\t\tcase fch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n}\n\n// Kick flusher for this message block.\nfunc (mb *msgBlock) kickFlusher() {\n\tmb.mu.RLock()\n\tdefer mb.mu.RUnlock()\n\tkickFlusher(mb.fch)\n}\n\nfunc (mb *msgBlock) setInFlusher() {\n\tmb.mu.Lock()\n\tmb.flusher = true\n\tmb.mu.Unlock()\n}\n\nfunc (mb *msgBlock) clearInFlusher() {\n\tmb.mu.Lock()\n\tmb.flusher = false\n\tif mb.qch != nil {\n\t\tclose(mb.qch)\n\t\tmb.qch = nil\n\t}\n\tif mb.fch != nil {\n\t\tclose(mb.fch)\n\t\tmb.fch = nil\n\t}\n\tmb.mu.Unlock()\n}\n\n// flushLoop watches for messages, index info, or recently closed msg block updates.\nfunc (mb *msgBlock) flushLoop(fch, qch chan struct{}) {\n\tmb.setInFlusher()\n\tdefer mb.clearInFlusher()\n\n\tfor {\n\t\tselect {\n\t\tcase <-fch:\n\t\t\t// If we have pending messages process them first.\n\t\t\tif waiting := mb.pendingWriteSize(); waiting != 0 {\n\t\t\t\tts := 1 * time.Millisecond\n\t\t\t\tvar waited time.Duration\n\n\t\t\t\tfor waiting < coalesceMinimum {\n\t\t\t\t\ttime.Sleep(ts)\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-qch:\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t\tnewWaiting := mb.pendingWriteSize()\n\t\t\t\t\tif waited = waited + ts; waited > maxFlushWait || newWaiting <= waiting {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\twaiting = newWaiting\n\t\t\t\t\tts *= 2\n\t\t\t\t}\n\n\t\t\t\t// Ignore error here, the error is persisted as mb.werr and will be bubbled up later.\n\t\t\t\t_ = mb.flushPendingMsgs()\n\t\t\t}\n\n\t\t\t// Check if we are no longer the last message block. If we are\n\t\t\t// not we can close FDs and exit.\n\t\t\tmb.fs.mu.RLock()\n\t\t\tnotLast := mb != mb.fs.lmb\n\t\t\tmb.fs.mu.RUnlock()\n\n\t\t\tif notLast {\n\t\t\t\tif err := mb.closeFDs(); err == nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-qch:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) eraseMsg(seq uint64, ri, rl int, isLastBlock bool) error {\n\tvar le = binary.LittleEndian\n\tvar hdr [msgHdrSize]byte\n\n\tle.PutUint32(hdr[0:], uint32(rl))\n\tle.PutUint64(hdr[4:], seq|ebit)\n\tle.PutUint64(hdr[12:], 0)\n\tle.PutUint16(hdr[20:], 0)\n\n\t// Randomize record\n\tdata := make([]byte, rl-emptyRecordLen)\n\tif n, err := rand.Read(data); err != nil {\n\t\treturn err\n\t} else if n != len(data) {\n\t\treturn fmt.Errorf(\"not enough overwrite bytes read (%d != %d)\", n, len(data))\n\t}\n\n\t// Now write to underlying buffer.\n\tvar b bytes.Buffer\n\tb.Write(hdr[:])\n\tb.Write(data)\n\n\t// Calculate hash.\n\tmb.hh.Reset()\n\tmb.hh.Write(hdr[4:20])\n\tmb.hh.Write(data)\n\tvar hb [highwayhash.Size64]byte\n\tchecksum := mb.hh.Sum(hb[:0])\n\t// Write to msg record.\n\tb.Write(checksum)\n\n\t// Update both cache and disk.\n\tnbytes := b.Bytes()\n\n\t// Cache\n\tif ri+rl <= len(mb.cache.buf) {\n\t\tcopy(mb.cache.buf[ri:ri+rl], nbytes)\n\t}\n\n\t// Disk\n\tif mb.cache.wp > ri {\n\t\tif err := mb.atomicOverwriteFile(mb.cache.buf, !isLastBlock); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Truncate this message block to the tseq and ts.\n// Lock should be held.\nfunc (mb *msgBlock) truncate(tseq uint64, ts int64) (nmsgs, nbytes uint64, err error) {\n\t// Make sure we are loaded to process messages etc.\n\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\treturn 0, 0, err\n\t}\n\tdefer mb.finishedWithCache()\n\n\t// Calculate new eof using slot info from our new last sm.\n\tri, rl, _, err := mb.slotInfo(int(tseq - mb.cache.fseq))\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\t// Calculate new eof.\n\teof := int64(ri + rl)\n\n\t// FIXME(dlc) - We could be smarter here.\n\tif buf, _ := mb.bytesPending(); len(buf) > 0 {\n\t\tld, err := mb.flushPendingMsgsLocked()\n\t\tif ld != nil && mb.fs != nil {\n\t\t\t// We do not know if fs is locked or not at this point.\n\t\t\t// This should be an exceptional condition so do so in Go routine.\n\t\t\tgo mb.fs.rebuildState(ld)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t}\n\n\tvar purged, bytes uint64\n\n\tcheckDmap := mb.dmap.Size() > 0\n\tvar smv StoreMsg\n\n\tfor seq := atomic.LoadUint64(&mb.last.seq); seq > tseq; seq-- {\n\t\tif checkDmap {\n\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t// Delete and skip to next.\n\t\t\t\tmb.dmap.Delete(seq)\n\t\t\t\tcheckDmap = !mb.dmap.IsEmpty()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// We should have a valid msg to calculate removal stats.\n\t\tif m, err := mb.cacheLookupNoCopy(seq, &smv); err == nil {\n\t\t\tif mb.msgs > 0 {\n\t\t\t\trl := fileStoreMsgSize(m.subj, m.hdr, m.msg)\n\t\t\t\tmb.msgs--\n\t\t\t\tif rl > mb.bytes {\n\t\t\t\t\trl = mb.bytes\n\t\t\t\t}\n\t\t\t\tmb.bytes -= rl\n\t\t\t\tmb.rbytes -= rl\n\t\t\t\t// For return accounting.\n\t\t\t\tpurged++\n\t\t\t\tbytes += uint64(rl)\n\t\t\t}\n\t\t}\n\t}\n\n\t// If the block is compressed/encrypted then we have to load it into memory\n\t// and decompress/decrypt it, truncate it and then write it back out.\n\t// Otherwise, truncate the file itself and close the descriptor.\n\tif mb.cmp != NoCompression || mb.bek != nil {\n\t\tbuf, err := mb.loadBlock(nil)\n\t\tif err != nil {\n\t\t\treturn 0, 0, fmt.Errorf(\"failed to load block from disk: %w\", err)\n\t\t}\n\t\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t\tif buf, err = mb.decompressIfNeeded(buf); err != nil {\n\t\t\treturn 0, 0, fmt.Errorf(\"failed to decompress block: %w\", err)\n\t\t}\n\t\tbuf = buf[:eof]\n\t\tcopy(mb.lchk[0:], buf[len(buf)-checksumSize:])\n\t\t// We did decompress but don't recompress the truncated buffer here since we're the last block\n\t\t// and would otherwise have compressed data and allow to write uncompressed data in the same block.\n\t\tif err = mb.atomicOverwriteFile(buf, false); err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t} else if mb.mfd != nil {\n\t\tif err = mb.mfd.Truncate(eof); err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t\tif err = mb.mfd.Sync(); err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t\t// Update our checksum.\n\t\tvar lchk [8]byte\n\t\tif _, err = mb.mfd.ReadAt(lchk[:], eof-8); err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t\tcopy(mb.lchk[0:], lchk[:])\n\t} else {\n\t\treturn 0, 0, fmt.Errorf(\"failed to truncate msg block %d, file not open\", mb.index)\n\t}\n\n\t// Update our last msg.\n\tatomic.StoreUint64(&mb.last.seq, tseq)\n\tmb.last.ts = ts\n\n\t// Clear our cache.\n\tmb.clearCacheAndOffset()\n\n\t// Redo per subject info for this block.\n\tif err = mb.resetPerSubjectInfo(); err != nil {\n\t\treturn purged, bytes, err\n\t}\n\n\t// Load msgs again.\n\treturn purged, bytes, mb.loadMsgsWithLock()\n}\n\n// Helper to determine if the mb is empty.\nfunc (mb *msgBlock) isEmpty() bool {\n\treturn atomic.LoadUint64(&mb.first.seq) > atomic.LoadUint64(&mb.last.seq)\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) selectNextFirst() {\n\tvar seq uint64\n\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\tfor seq = fseq + 1; seq <= lseq; seq++ {\n\t\tif mb.dmap.Exists(seq) {\n\t\t\t// We will move past this so we can delete the entry.\n\t\t\tmb.dmap.Delete(seq)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\t// Set new first sequence.\n\tatomic.StoreUint64(&mb.first.seq, seq)\n\n\t// Check if we are empty..\n\tif seq > lseq {\n\t\tmb.first.ts = 0\n\t\treturn\n\t}\n\n\t// Need to get the timestamp.\n\t// We will try the cache direct and fallback if needed.\n\tvar smv StoreMsg\n\tsm, _ := mb.cacheLookupNoCopy(seq, &smv)\n\tif sm == nil {\n\t\t// Slow path, need to unlock.\n\t\tmb.mu.Unlock()\n\t\tsm, _, _ = mb.fetchMsgNoCopy(seq, &smv)\n\t\tmb.mu.Lock()\n\t}\n\tif sm != nil {\n\t\tmb.first.ts = sm.ts\n\t} else {\n\t\tmb.first.ts = 0\n\t}\n}\n\n// Select the next FirstSeq\n// Also cleans up empty blocks at the start only containing tombstones.\n// Lock should be held.\nfunc (fs *fileStore) selectNextFirst() error {\n\tif len(fs.blks) > 0 {\n\t\tfor len(fs.blks) > 1 {\n\t\t\tmb := fs.blks[0]\n\t\t\tmb.mu.Lock()\n\t\t\tempty := mb.msgs == 0\n\t\t\tif !empty {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\terr := fs.forceRemoveMsgBlock(mb)\n\t\t\tmb.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tmb := fs.blks[0]\n\t\tmb.mu.RLock()\n\t\tfs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq)\n\t\tif mb.first.ts == 0 {\n\t\t\tfs.state.FirstTime = time.Time{}\n\t\t} else {\n\t\t\tfs.state.FirstTime = time.Unix(0, mb.first.ts).UTC()\n\t\t}\n\t\tmb.mu.RUnlock()\n\t} else {\n\t\t// Could not find anything, so treat like purge\n\t\tfs.state.FirstSeq = fs.state.LastSeq + 1\n\t\tfs.state.FirstTime = time.Time{}\n\t}\n\t// Mark first as moved. Plays into tombstone cleanup for syncBlocks.\n\tfs.firstMoved = true\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) resetCacheExpireTimer(td time.Duration) {\n\tif td == 0 {\n\t\ttd = mb.cexp + 100*time.Millisecond\n\t}\n\tif mb.ctmr == nil {\n\t\tmb.ctmr = time.AfterFunc(td, mb.expireCache)\n\t} else {\n\t\tmb.ctmr.Reset(td)\n\t}\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) startCacheExpireTimer() {\n\tmb.resetCacheExpireTimer(0)\n}\n\n// Used when we load in a message block.\n// Lock should be held.\nfunc (mb *msgBlock) clearCacheAndOffset() {\n\t// Reset linear scan tracker.\n\tmb.llseq = 0\n\tmb.clearCache()\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) clearCache() {\n\tif mb.ctmr != nil {\n\t\ttsla := mb.sinceLastActivity()\n\t\tif mb.fss == nil || tsla > mb.fexp {\n\t\t\t// Force\n\t\t\tmb.fss = nil\n\t\t\tmb.ctmr.Stop()\n\t\t\tmb.ctmr = nil\n\t\t} else {\n\t\t\tmb.resetCacheExpireTimer(mb.fexp - tsla)\n\t\t}\n\t}\n\n\tmbcache := mb.cache\n\tif mbcache == nil {\n\t\tmbcache = mb.ecache.Value()\n\t}\n\tif mbcache == nil {\n\t\treturn\n\t}\n\n\tbuf := mbcache.buf\n\tmb.cache = nil\n\tmb.ecache.Set(nil)\n\trecycleMsgBlockBuf(buf)\n}\n\n// Called to possibly expire a message block cache.\nfunc (mb *msgBlock) expireCache() {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\tmb.expireCacheLocked()\n}\n\nfunc (mb *msgBlock) tryForceExpireCache() {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\tmb.tryForceExpireCacheLocked()\n}\n\n// We will attempt to force expire this by temporarily clearing the last load time.\nfunc (mb *msgBlock) tryForceExpireCacheLocked() {\n\tllts, lwts := mb.llts, mb.lwts\n\tmb.llts, mb.lwts = 0, 0\n\tmb.expireCacheLocked()\n\tmb.llts, mb.lwts = llts, lwts\n}\n\n// This is for expiration of the write cache, which will be partial with fip.\n// So we want to bypass the Pools here.\n// Lock should be held.\nfunc (mb *msgBlock) tryExpireWriteCache() []byte {\n\tif mb.cache == nil {\n\t\treturn nil\n\t}\n\tlwts, buf, llts, nra := mb.lwts, mb.cache.buf, mb.llts, mb.cache.nra\n\tmb.lwts, mb.cache.nra = 0, true\n\tmb.expireCacheLocked()\n\tmb.lwts = lwts\n\tif mb.cache != nil {\n\t\tmb.cache.nra = nra\n\t}\n\t// We could check for a certain time since last load, but to be safe just reuse if no loads at all.\n\tif llts == 0 && (mb.cache == nil || mb.cache.buf == nil) {\n\t\t// Clear last write time since we now are about to move on to a new lmb.\n\t\tmb.lwts = 0\n\t\treturn buf[:0]\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) expireCacheLocked() {\n\tvar strengthened bool\n\tif mb.cache == nil {\n\t\tmb.cache = mb.ecache.Value()\n\t\tstrengthened = true\n\t}\n\tif mb.cache == nil && mb.fss == nil {\n\t\tif mb.ctmr != nil {\n\t\t\tmb.ctmr.Stop()\n\t\t\tmb.ctmr = nil\n\t\t}\n\t\treturn\n\t}\n\n\t// Can't expire if we still have pending.\n\tif mb.cache != nil && len(mb.cache.buf)-int(mb.cache.wp) > 0 {\n\t\tmb.resetCacheExpireTimer(mb.cexp)\n\t\tif strengthened {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\treturn\n\t}\n\n\t// Grab timestamp to compare.\n\ttns := ats.AccessTime()\n\n\t// For the core buffer of messages, we care about reads and writes, but not removes.\n\tbufts := mb.llts\n\tif mb.lwts > bufts {\n\t\tbufts = mb.lwts\n\t}\n\n\t// Check for activity on the cache that would prevent us from expiring.\n\tif tns-bufts <= int64(mb.cexp) {\n\t\tmb.resetCacheExpireTimer(mb.cexp - time.Duration(tns-bufts))\n\t\tif strengthened {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\treturn\n\t}\n\n\t// If we are here we will at least expire the core msg buffer.\n\t// We need to capture offset in case we do a write next before a full load.\n\tif mb.cache != nil {\n\t\tif !mb.cache.nra {\n\t\t\trecycleMsgBlockBuf(mb.cache.buf)\n\t\t}\n\t\tmb.cache.buf = nil\n\t\tmb.cache.idx = mb.cache.idx[:0]\n\t\tmb.cache.wp = 0\n\t}\n\n\t// Check if we can clear out our idx unless under force expire.\n\t// fss we keep longer and expire under sync timer checks.\n\tmb.clearCache()\n}\n\nfunc (fs *fileStore) startAgeChk() {\n\tif fs.ageChk != nil {\n\t\treturn\n\t}\n\tif fs.cfg.MaxAge != 0 || fs.ttls != nil {\n\t\tfs.ageChk = time.AfterFunc(fs.cfg.MaxAge, fs.expireMsgs)\n\t}\n}\n\n// Lock should be held.\nfunc (fs *fileStore) resetAgeChk(delta int64) {\n\t// If we're already expiring messages, it will make sure to reset.\n\t// Don't trigger again, as that could result in many expire goroutines.\n\tif fs.ageChkRun {\n\t\treturn\n\t}\n\n\tvar next int64 = math.MaxInt64\n\tif fs.ttls != nil {\n\t\tnext = fs.ttls.GetNextExpiration(next)\n\t}\n\n\t// If there's no MaxAge and there's nothing waiting to be expired then\n\t// don't bother continuing. The next storeRawMsg() will wake us up if\n\t// needs be.\n\tif fs.cfg.MaxAge <= 0 && next == math.MaxInt64 {\n\t\tclearTimer(&fs.ageChk)\n\t\treturn\n\t}\n\n\t// Check to see if we should be firing sooner than MaxAge for an expiring TTL.\n\tfireIn := fs.cfg.MaxAge\n\n\t// If delta for next-to-expire message is unset, but we still have messages to remove.\n\t// Assume messages are removed through proposals, and we need to speed up subsequent age check.\n\tif delta == 0 && fs.state.Msgs > 0 {\n\t\tif until := 2 * time.Second; until < fireIn {\n\t\t\tfireIn = until\n\t\t}\n\t}\n\n\tif next < math.MaxInt64 {\n\t\t// Looks like there's a next expiration, use it either if there's no\n\t\t// MaxAge set or if it looks to be sooner than MaxAge is.\n\t\tif until := time.Until(time.Unix(0, next)); fireIn == 0 || until < fireIn {\n\t\t\tfireIn = until\n\t\t}\n\t}\n\n\t// If not then look at the delta provided (usually gap to next age expiry).\n\tif delta > 0 {\n\t\tif fireIn == 0 || time.Duration(delta) < fireIn {\n\t\t\tfireIn = time.Duration(delta)\n\t\t}\n\t}\n\n\t// Make sure we aren't firing too often either way, otherwise we can\n\t// negatively impact stream ingest performance.\n\tif fireIn < 250*time.Millisecond {\n\t\tfireIn = 250 * time.Millisecond\n\t}\n\n\t// If we want to kick the timer to run later than what was assigned before, don't reset it.\n\t// Otherwise, we could get in a situation where the timer is continuously reset, and it never runs.\n\texpires := ats.AccessTime() + fireIn.Nanoseconds()\n\tif fs.ageChkTime > 0 && expires > fs.ageChkTime {\n\t\treturn\n\t}\n\n\tfs.ageChkTime = expires\n\tif fs.ageChk != nil {\n\t\tfs.ageChk.Reset(fireIn)\n\t} else {\n\t\tfs.ageChk = time.AfterFunc(fireIn, fs.expireMsgs)\n\t}\n}\n\n// Lock should be held.\nfunc (fs *fileStore) cancelAgeChk() {\n\tif fs.ageChk != nil {\n\t\tfs.ageChk.Stop()\n\t\tfs.ageChk = nil\n\t\tfs.ageChkTime = 0\n\t}\n}\n\n// Will expire msgs that are too old.\nfunc (fs *fileStore) expireMsgs() {\n\t// We need to delete one by one here and can not optimize for the time being.\n\t// Reason is that we need more information to adjust ack pending in consumers.\n\tvar smv StoreMsg\n\tvar sm *StoreMsg\n\n\tfs.mu.Lock()\n\tmaxAge := int64(fs.cfg.MaxAge)\n\tminAge := ats.AccessTime() - maxAge\n\trmcb := fs.rmcb\n\tpmsgcb := fs.pmsgcb\n\tsdmTTL := int64(fs.cfg.SubjectDeleteMarkerTTL.Seconds())\n\tsdmEnabled := sdmTTL > 0\n\n\t// If SDM is enabled, but handlers aren't set up yet. Try again later.\n\tif sdmEnabled && (rmcb == nil || pmsgcb == nil) {\n\t\tfs.resetAgeChk(0)\n\t\tfs.mu.Unlock()\n\t\treturn\n\t}\n\tfs.ageChkRun = true\n\tfs.mu.Unlock()\n\n\tif maxAge > 0 {\n\t\tvar seq uint64\n\t\tfor sm, seq, _ = fs.LoadNextMsg(fwcs, true, 0, &smv); sm != nil && sm.ts <= minAge; sm, seq, _ = fs.LoadNextMsg(fwcs, true, seq+1, &smv) {\n\t\t\tif len(sm.hdr) > 0 {\n\t\t\t\tif ttl, err := getMessageTTL(sm.hdr); err == nil && ttl < 0 {\n\t\t\t\t\t// The message has a negative TTL, therefore it must \"never expire\".\n\t\t\t\t\tminAge = ats.AccessTime() - maxAge\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Remove the message and then, if LimitsTTL is enabled, try and work out\n\t\t\t// if it was the last message of that particular subject that we just deleted.\n\t\t\tif sdmEnabled {\n\t\t\t\tif last, ok := fs.shouldProcessSdm(seq, sm.subj); ok {\n\t\t\t\t\tsdm := last && !isSubjectDeleteMarker(sm.hdr)\n\t\t\t\t\tfs.handleRemovalOrSdm(seq, sm.subj, sdm, sdmTTL)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfs.mu.Lock()\n\t\t\t\tfs.removeMsgViaLimits(sm.seq)\n\t\t\t\tfs.mu.Unlock()\n\t\t\t}\n\t\t\t// Recalculate in case we are expiring a bunch.\n\t\t\tminAge = ats.AccessTime() - maxAge\n\t\t}\n\t}\n\tvar ageDelta int64\n\tif sm != nil {\n\t\tageDelta = sm.ts - minAge\n\t}\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// TODO: Not great that we're holding the lock here, but the timed hash wheel isn't thread-safe.\n\tnextTTL := int64(math.MaxInt64)\n\tvar rmSeqs []thw.HashWheelEntry\n\tif fs.ttls != nil {\n\t\tfs.ttls.ExpireTasks(func(seq uint64, ts int64) bool {\n\t\t\trmSeqs = append(rmSeqs, thw.HashWheelEntry{Seq: seq, Expires: ts})\n\t\t\t// We might need to remove messages out of band, those can fail, and we can be shutdown halfway\n\t\t\t// through so don't remove from THW just yet.\n\t\t\treturn false\n\t\t})\n\t\tnextTTL = fs.ttls.GetNextExpiration(math.MaxInt64)\n\t}\n\n\t// Remove messages collected by THW.\n\tif !sdmEnabled {\n\t\tfor _, rm := range rmSeqs {\n\t\t\tfs.removeMsg(rm.Seq, false, false, false)\n\t\t}\n\t} else {\n\t\t// THW is unordered, so must sort by sequence and must not be holding the lock.\n\t\tfs.mu.Unlock()\n\t\tslices.SortFunc(rmSeqs, func(a, b thw.HashWheelEntry) int {\n\t\t\tif a.Seq == b.Seq {\n\t\t\t\treturn 0\n\t\t\t} else if a.Seq < b.Seq {\n\t\t\t\treturn -1\n\t\t\t} else {\n\t\t\t\treturn 1\n\t\t\t}\n\t\t})\n\t\tfor _, rm := range rmSeqs {\n\t\t\t// Need to grab subject for the specified sequence if for SDM, and check\n\t\t\t// if the message hasn't been removed in the meantime.\n\t\t\t// We need to grab the message and check if we should process SDM while holding the lock,\n\t\t\t// otherwise we can race if a deletion of this message is in progress.\n\t\t\tfs.mu.Lock()\n\t\t\tsm, _ = fs.msgForSeqLocked(rm.Seq, &smv, false)\n\t\t\tif sm == nil {\n\t\t\t\tfs.ttls.Remove(rm.Seq, rm.Expires)\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlast, ok := fs.shouldProcessSdmLocked(rm.Seq, sm.subj)\n\t\t\tfs.mu.Unlock()\n\t\t\tif ok {\n\t\t\t\tsdm := last && !isSubjectDeleteMarker(sm.hdr)\n\t\t\t\tfs.handleRemovalOrSdm(rm.Seq, sm.subj, sdm, sdmTTL)\n\t\t\t}\n\t\t}\n\t\tfs.mu.Lock()\n\t}\n\n\t// Only cancel if no message left, not on potential lookup error that would result in sm == nil.\n\tfs.ageChkRun, fs.ageChkTime = false, 0\n\tif fs.state.Msgs == 0 && nextTTL == math.MaxInt64 {\n\t\tfs.cancelAgeChk()\n\t} else {\n\t\tfs.resetAgeChk(ageDelta)\n\t}\n}\n\nfunc (fs *fileStore) shouldProcessSdm(seq uint64, subj string) (bool, bool) {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\treturn fs.shouldProcessSdmLocked(seq, subj)\n}\n\n// Lock should be held.\nfunc (fs *fileStore) shouldProcessSdmLocked(seq uint64, subj string) (bool, bool) {\n\tif fs.sdm == nil {\n\t\tfs.sdm = newSDMMeta()\n\t}\n\n\tif p, ok := fs.sdm.pending[seq]; ok {\n\t\t// Don't allow more proposals for the same sequence if we already did recently.\n\t\tif time.Since(time.Unix(0, p.ts)) < 2*time.Second {\n\t\t\treturn p.last, false\n\t\t}\n\t\t// If we're about to use the cached value, and we knew it was last before,\n\t\t// quickly check that we don't have more remaining messages for the subject now.\n\t\t// Which means we are not the last anymore and must reset to not remove later data.\n\t\tif p.last {\n\t\t\tmsgs := fs.subjectsTotalsLocked(subj)[subj]\n\t\t\tnumPending := fs.sdm.totals[subj]\n\t\t\tif remaining := msgs - numPending; remaining > 0 {\n\t\t\t\tp.last = false\n\t\t\t}\n\t\t}\n\t\tfs.sdm.pending[seq] = SDMBySeq{p.last, time.Now().UnixNano()}\n\t\treturn p.last, true\n\t}\n\n\tmsgs := fs.subjectsTotalsLocked(subj)[subj]\n\tif msgs == 0 {\n\t\treturn false, true\n\t}\n\tnumPending := fs.sdm.totals[subj]\n\tremaining := msgs - numPending\n\treturn fs.sdm.trackPending(seq, subj, remaining == 1), true\n}\n\nfunc (fs *fileStore) handleRemovalOrSdm(seq uint64, subj string, sdm bool, sdmTTL int64) {\n\tif sdm {\n\t\tvar _hdr [128]byte\n\t\thdr := fmt.Appendf(\n\t\t\t_hdr[:0],\n\t\t\t\"NATS/1.0\\r\\n%s: %s\\r\\n%s: %s\\r\\n%s: %s\\r\\n\\r\\n\",\n\t\t\tJSMarkerReason, JSMarkerReasonMaxAge,\n\t\t\tJSMessageTTL, time.Duration(sdmTTL)*time.Second,\n\t\t\tJSMsgRollup, JSMsgRollupSubject,\n\t\t)\n\t\tmsg := &inMsg{\n\t\t\tsubj: subj,\n\t\t\thdr: hdr,\n\t\t}\n\t\tfs.pmsgcb(msg)\n\t} else {\n\t\tfs.rmcb(seq)\n\t}\n}\n\n// Will run through scheduled messages.\nfunc (fs *fileStore) runMsgScheduling() {\n\t// TODO: Not great that we're holding the lock here, but the timed hash wheel and message scheduling isn't thread-safe.\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// If scheduling is enabled, but handler isn't set up yet. Try again later.\n\tif fs.scheduling == nil {\n\t\treturn\n\t}\n\tif fs.pmsgcb == nil {\n\t\tfs.scheduling.resetTimer()\n\t\treturn\n\t}\n\tfs.scheduling.running = true\n\n\tscheduledMsgs := fs.scheduling.getScheduledMessages(\n\t\tfunc(seq uint64, smv *StoreMsg) *StoreMsg {\n\t\t\tsm, _ := fs.msgForSeqLocked(seq, smv, false)\n\t\t\treturn sm\n\t\t},\n\t\tfunc(subj string, smv *StoreMsg) *StoreMsg {\n\t\t\tsm, _ := fs.loadLastLocked(subj, smv)\n\t\t\treturn sm\n\t\t},\n\t)\n\tif len(scheduledMsgs) > 0 {\n\t\tfs.mu.Unlock()\n\t\tfor _, msg := range scheduledMsgs {\n\t\t\tfs.pmsgcb(msg)\n\t\t}\n\t\tfs.mu.Lock()\n\t}\n\n\tfs.scheduling.running, fs.scheduling.deadline = false, 0\n\tfs.scheduling.resetTimer()\n}\n\n// Lock should be held.\nfunc (fs *fileStore) checkAndFlushLastBlock() error {\n\tlmb := fs.lmb\n\tif lmb == nil {\n\t\treturn nil\n\t}\n\tlmb.mu.Lock()\n\tif err := lmb.werr; err != nil {\n\t\tlmb.mu.Unlock()\n\t\treturn err\n\t}\n\n\tif lmb.pendingWriteSizeLocked() == 0 {\n\t\tlmb.mu.Unlock()\n\t\treturn nil\n\t}\n\tld, err := lmb.flushPendingMsgsLocked()\n\tlmb.mu.Unlock()\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Since fs lock is held need to unlock the mb in case we need to rebuild state.\n\tif ld != nil {\n\t\tfs.rebuildStateLocked(ld)\n\t}\n\treturn nil\n}\n\n// This will check all the checksums on messages and report back any sequence numbers with errors.\nfunc (fs *fileStore) checkMsgs() *LostStreamData {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\tfs.checkAndFlushLastBlock()\n\n\t// Clear any global subject state.\n\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\n\tfor _, mb := range fs.blks {\n\t\t// Make sure encryption loaded if needed for the block.\n\t\tfs.loadEncryptionForMsgBlock(mb)\n\t\t// FIXME(dlc) - check tombstones here too?\n\t\tif ld, _, _ := mb.rebuildState(); ld != nil {\n\t\t\t// Rebuild fs state too.\n\t\t\tfs.rebuildStateLocked(ld)\n\t\t}\n\t\tfs.populateGlobalPerSubjectInfo(mb)\n\t}\n\n\treturn fs.ld\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) enableForWriting(fip bool) error {\n\tif mb == nil {\n\t\treturn errNoMsgBlk\n\t}\n\tif mb.mfd != nil {\n\t\treturn nil\n\t}\n\t<-dios\n\tmfd, err := os.OpenFile(mb.mfn, os.O_CREATE|os.O_RDWR, defaultFilePerms)\n\tdios <- struct{}{}\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error opening msg block file [%q]: %v\", mb.mfn, err)\n\t}\n\tmb.mfd = mfd\n\n\t// Spin up our flusher loop if needed.\n\tif !fip {\n\t\tmb.spinUpFlushLoopLocked()\n\t}\n\n\treturn nil\n}\n\n// Helper function to place a delete tombstone.\nfunc (mb *msgBlock) writeTombstone(seq uint64, ts int64) error {\n\treturn mb.writeMsgRecord(emptyRecordLen, seq|tbit, _EMPTY_, nil, nil, ts, true)\n}\n\n// Helper function to place a delete tombstone without flush.\n// Lock should not be held.\nfunc (mb *msgBlock) writeTombstoneNoFlush(seq uint64, ts int64) error {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.writeMsgRecordLocked(emptyRecordLen, seq|tbit, _EMPTY_, nil, nil, ts, false, false)\n}\n\n// Will write the message record to the underlying message block.\n// filestore lock will be held.\nfunc (mb *msgBlock) writeMsgRecord(rl, seq uint64, subj string, mhdr, msg []byte, ts int64, flush bool) error {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.writeMsgRecordLocked(rl, seq, subj, mhdr, msg, ts, flush, true)\n}\n\n// Will write the message record to the underlying message block.\n// filestore lock will be held.\n// mb lock should be held.\nfunc (mb *msgBlock) writeMsgRecordLocked(rl, seq uint64, subj string, mhdr, msg []byte, ts int64, flush, kick bool) (rerr error) {\n\t// Return previous write errors immediately.\n\tif mb.werr != nil {\n\t\treturn mb.werr\n\t}\n\t// Persist any returned errors to be used in the future.\n\tdefer func() {\n\t\tif rerr != nil && mb.werr == nil {\n\t\t\tmb.werr = rerr\n\t\t\tassert.Unreachable(\"Filestore msg block encountered write error\", map[string]any{\n\t\t\t\t\"name\": mb.fs.cfg.Name,\n\t\t\t\t\"mb.index\": mb.index,\n\t\t\t\t\"err\": rerr,\n\t\t\t\t\"stack\": string(debug.Stack()),\n\t\t\t})\n\t\t}\n\t}()\n\n\t// Enable for writing if our mfd is not open.\n\tif mb.mfd == nil {\n\t\tif err := mb.enableForWriting(flush && kick); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Check if we are tracking per subject for our simple state.\n\t// Do this before changing the cache that would trigger a flush pending msgs call\n\t// if we needed to regenerate the per subject info.\n\t// Note that tombstones have no subject so will not trigger here.\n\tif len(subj) > 0 && !mb.noTrack {\n\t\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Mark fss activity.\n\t\tmb.lsts = ats.AccessTime()\n\t\tif ss, ok := mb.fss.Find(stringToBytes(subj)); ok && ss != nil {\n\t\t\tss.Msgs++\n\t\t\tss.Last = seq\n\t\t\tss.lastNeedsUpdate = false\n\t\t} else {\n\t\t\tmb.fss.Insert(stringToBytes(subj), SimpleState{Msgs: 1, First: seq, Last: seq})\n\t\t}\n\t}\n\n\t// Make sure we have a cache setup. Do so after ensurePerSubjectInfoLoaded as it may\n\t// have already brought in a cache for us.\n\tif err := mb.setupWriteCache(nil); err != nil {\n\t\treturn err\n\t}\n\n\t// Make sure that the GC can't take away our writes by strengthening the elastic\n\t// reference. It will now stay strong until the flusher decides it is time to weaken.\n\tmb.ecache.Strengthen()\n\n\t// Make sure we have enough space to write into. If we don't then we can pull a buffer\n\t// from the next pool size up to save us from reallocating in append() below.\n\tif nsz := len(mb.cache.buf) + int(rl); cap(mb.cache.buf) < nsz {\n\t\tprev := mb.cache.buf\n\t\tmb.cache.buf = getMsgBlockBuf(nsz)\n\t\tif prev != nil {\n\t\t\tmb.cache.buf = mb.cache.buf[:copy(mb.cache.buf[:nsz], prev)]\n\t\t\trecycleMsgBlockBuf(prev)\n\t\t}\n\t}\n\n\t// Indexing\n\tindex := len(mb.cache.buf)\n\n\t// Formats\n\t// Format with no header\n\t// total_len(4) sequence(8) timestamp(8) subj_len(2) subj msg hash(8)\n\t// With headers, high bit on total length will be set.\n\t// total_len(4) sequence(8) timestamp(8) subj_len(2) subj hdr_len(4) hdr msg hash(8)\n\n\tvar le = binary.LittleEndian\n\n\tl := uint32(rl)\n\thasHeaders := len(mhdr) > 0\n\tif hasHeaders {\n\t\tl |= hbit\n\t}\n\n\t// Reserve space for the header on the underlying buffer.\n\tmb.cache.buf = append(mb.cache.buf, make([]byte, msgHdrSize)...)\n\thdr := mb.cache.buf[len(mb.cache.buf)-msgHdrSize : len(mb.cache.buf)]\n\tle.PutUint32(hdr[0:], l)\n\tle.PutUint64(hdr[4:], seq)\n\tle.PutUint64(hdr[12:], uint64(ts))\n\tle.PutUint16(hdr[20:], uint16(len(subj)))\n\n\t// Now write to underlying buffer.\n\tmb.cache.buf = append(mb.cache.buf, subj...)\n\n\tif hasHeaders {\n\t\tvar hlen [4]byte\n\t\tle.PutUint32(hlen[0:], uint32(len(mhdr)))\n\t\tmb.cache.buf = append(mb.cache.buf, hlen[:]...)\n\t\tmb.cache.buf = append(mb.cache.buf, mhdr...)\n\t}\n\tmb.cache.buf = append(mb.cache.buf, msg...)\n\n\t// Calculate hash.\n\tmb.hh.Reset()\n\tmb.hh.Write(hdr[4:20])\n\tmb.hh.Write(stringToBytes(subj))\n\tif hasHeaders {\n\t\tmb.hh.Write(mhdr)\n\t}\n\tmb.hh.Write(msg)\n\tchecksum := mb.hh.Sum(mb.lchk[:0:highwayhash.Size64])\n\tcopy(mb.lchk[0:], checksum)\n\n\t// Update write through cache.\n\t// Write to msg record.\n\tmb.cache.buf = append(mb.cache.buf, checksum...)\n\n\t// Set cache timestamp for last store.\n\tmb.lwts = ts\n\n\t// Only update index and do accounting if not a delete tombstone.\n\tif seq&tbit == 0 {\n\t\tlast := atomic.LoadUint64(&mb.last.seq)\n\t\t// Accounting, do this before stripping ebit, it is ebit aware.\n\t\tmb.updateAccounting(seq, ts, rl)\n\t\t// Strip ebit if set.\n\t\tseq = seq &^ ebit\n\t\t// If we have a hole due to skipping many messages, fill it.\n\t\tif len(mb.cache.idx) > 0 && last+1 < seq {\n\t\t\tfor dseq := last + 1; dseq < seq; dseq++ {\n\t\t\t\tmb.cache.idx = append(mb.cache.idx, dbit)\n\t\t\t}\n\t\t}\n\t\t// Write index\n\t\tif mb.cache.idx = append(mb.cache.idx, uint32(index)|cbit); len(mb.cache.idx) == 1 {\n\t\t\tmb.cache.fseq = seq\n\t\t}\n\t} else {\n\t\t// If the block is empty, still adjust the accounting accordingly.\n\t\ttseq := seq &^ tbit\n\t\tif mb.msgs == 0 && tseq > atomic.LoadUint64(&mb.last.seq) {\n\t\t\tatomic.StoreUint64(&mb.last.seq, tseq)\n\t\t\tmb.last.ts = ts\n\t\t\tatomic.StoreUint64(&mb.first.seq, tseq+1)\n\t\t\tmb.first.ts = 0\n\t\t}\n\t\t// Make sure to account for tombstones in rbytes.\n\t\tmb.rbytes += rl\n\t}\n\n\tfch, werr := mb.fch, mb.werr\n\tif werr != nil {\n\t\treturn werr\n\t}\n\n\t// If we should be flushing, or had a write error, do so here.\n\tif flush && mb.fs.fip {\n\t\tld, err := mb.flushPendingMsgsLocked()\n\t\tif ld != nil {\n\t\t\t// We have the mb lock here, this needs the mb locks so do in its own go routine.\n\t\t\tgo mb.fs.rebuildState(ld)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if kick {\n\t\t// Kick the flusher here.\n\t\tkickFlusher(fch)\n\t}\n\treturn nil\n}\n\n// How many bytes pending to be written for this message block.\nfunc (mb *msgBlock) pendingWriteSize() int {\n\tif mb == nil {\n\t\treturn 0\n\t}\n\tmb.mu.RLock()\n\tdefer mb.mu.RUnlock()\n\treturn mb.pendingWriteSizeLocked()\n}\n\n// How many bytes pending to be written for this message block.\nfunc (mb *msgBlock) pendingWriteSizeLocked() int {\n\tif mb == nil {\n\t\treturn 0\n\t}\n\tvar pending int\n\tif !mb.closed && mb.mfd != nil && mb.cache != nil {\n\t\tpending = len(mb.cache.buf) - int(mb.cache.wp)\n\t}\n\treturn pending\n}\n\n// Try to close our FDs if we can.\nfunc (mb *msgBlock) closeFDs() error {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.closeFDsLocked()\n}\n\nfunc (mb *msgBlock) closeFDsLocked() error {\n\tif buf, _ := mb.bytesPending(); len(buf) > 0 {\n\t\treturn errPendingData\n\t}\n\tmb.closeFDsLockedNoCheck()\n\treturn nil\n}\n\nfunc (mb *msgBlock) closeFDsLockedNoCheck() {\n\tif mb.mfd != nil {\n\t\t_ = mb.mfd.Close()\n\t\tmb.mfd = nil\n\t}\n}\n\n// bytesPending returns the buffer to be used for writing to the underlying file.\n// This marks we are in flush and will return nil if asked again until cleared.\n// Lock should be held.\nfunc (mb *msgBlock) bytesPending() ([]byte, error) {\n\tif mb == nil || mb.mfd == nil {\n\t\treturn nil, errNoPending\n\t}\n\tif mb.cache == nil {\n\t\treturn nil, errNoCache\n\t}\n\tif len(mb.cache.buf) <= mb.cache.wp {\n\t\treturn nil, errNoPending\n\t}\n\tbuf := mb.cache.buf[mb.cache.wp:]\n\tif len(buf) == 0 {\n\t\treturn nil, errNoPending\n\t}\n\treturn buf, nil\n}\n\n// Returns the current blkSize including deleted msgs etc.\nfunc (mb *msgBlock) blkSize() uint64 {\n\tif mb == nil {\n\t\treturn 0\n\t}\n\tmb.mu.RLock()\n\tnb := mb.rbytes\n\tmb.mu.RUnlock()\n\treturn nb\n}\n\n// Update accounting on a write msg.\n// Lock should be held.\nfunc (mb *msgBlock) updateAccounting(seq uint64, ts int64, rl uint64) {\n\tisDeleted := seq&ebit != 0\n\tif isDeleted {\n\t\tseq = seq &^ ebit\n\t}\n\n\tfseq := atomic.LoadUint64(&mb.first.seq)\n\tif (fseq == 0 || mb.first.ts == 0) && seq >= fseq {\n\t\tatomic.StoreUint64(&mb.first.seq, seq)\n\t\tmb.first.ts = ts\n\t}\n\t// Need atomics here for selectMsgBlock speed.\n\tatomic.StoreUint64(&mb.last.seq, seq)\n\tmb.last.ts = ts\n\tmb.rbytes += rl\n\tif !isDeleted {\n\t\tmb.bytes += rl\n\t\tmb.msgs++\n\t}\n}\n\n// Helper to check last msg block and create new one if too big.\n// Lock should be held.\nfunc (fs *fileStore) checkLastBlock(rl uint64) (lmb *msgBlock, err error) {\n\t// Grab our current last message block.\n\tlmb = fs.lmb\n\trbytes := lmb.blkSize()\n\tif lmb == nil || (rbytes > 0 && rbytes+rl > fs.fcfg.BlockSize) {\n\t\tif lmb, err = fs.newMsgBlockForWrite(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn lmb, nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) writeMsgRecord(seq uint64, ts int64, subj string, hdr, msg []byte) (uint64, error) {\n\t// Get size for this message.\n\trl := fileStoreMsgSize(subj, hdr, msg)\n\tif rl&hbit != 0 || rl > rlBadThresh {\n\t\treturn 0, ErrMsgTooLarge\n\t}\n\t// Grab our current last message block.\n\tmb, err := fs.checkLastBlock(rl)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\t// Mark as dirty for stream state.\n\tfs.dirty++\n\n\t// Ask msg block to store in write through cache.\n\terr = mb.writeMsgRecord(rl, seq, subj, hdr, msg, ts, fs.fip)\n\n\treturn rl, err\n}\n\n// For writing tombstones to our lmb. This version will enforce maximum block sizes.\n// Lock should be held.\nfunc (fs *fileStore) writeTombstone(seq uint64, ts int64) error {\n\t// Grab our current last message block.\n\tlmb, err := fs.checkLastBlock(emptyRecordLen)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn lmb.writeTombstone(seq, ts)\n}\n\n// For writing tombstones to our lmb. This version will enforce maximum block sizes.\n// This version does not flush contents.\n// Lock should be held.\nfunc (fs *fileStore) writeTombstoneNoFlush(seq uint64, ts int64) error {\n\tlmb, err := fs.checkLastBlock(emptyRecordLen)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Write tombstone without flush or kick.\n\treturn lmb.writeTombstoneNoFlush(seq, ts)\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) recompressOnDiskIfNeeded() error {\n\talg := mb.fs.fcfg.Compression\n\n\t// Open up the file block and read in the entire contents into memory.\n\t// One of two things will happen:\n\t// 1. The block will be compressed already and have a valid metadata\n\t// header, in which case we do nothing.\n\t// 2. The block will be uncompressed, in which case we will compress it\n\t// and then write it back out to disk, re-encrypting if necessary.\n\t<-dios\n\torigBuf, err := os.ReadFile(mb.mfn)\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to read original block from disk: %w\", err)\n\t}\n\n\t// If the block is encrypted then we will need to decrypt it before\n\t// doing anything. We always encrypt after compressing because then the\n\t// compression can be as efficient as possible on the raw data, whereas\n\t// the encrypted ciphertext will not compress anywhere near as well.\n\t// The block encryption also covers the optional compression metadata.\n\tif err = mb.encryptOrDecryptIfNeeded(origBuf); err != nil {\n\t\treturn err\n\t}\n\n\tmeta := &CompressionInfo{}\n\tif _, err := meta.UnmarshalMetadata(origBuf); err != nil {\n\t\t// An error is only returned here if there's a problem with parsing\n\t\t// the metadata. If the file has no metadata at all, no error is\n\t\t// returned and the algorithm defaults to no compression.\n\t\treturn fmt.Errorf(\"failed to read existing metadata header: %w\", err)\n\t}\n\tif meta.Algorithm == alg {\n\t\t// The block is already compressed with the chosen algorithm so there\n\t\t// is nothing else to do. This is not a common case, it is here only\n\t\t// to ensure we don't do unnecessary work in case something asked us\n\t\t// to recompress an already compressed block with the same algorithm.\n\t\treturn nil\n\t} else if alg != NoCompression {\n\t\t// The block is already compressed using some algorithm, so we need\n\t\t// to decompress the block using the existing algorithm before we can\n\t\t// recompress it with the new one.\n\t\tif origBuf, err = meta.Algorithm.Decompress(origBuf); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to decompress original block: %w\", err)\n\t\t}\n\t}\n\n\treturn mb.atomicOverwriteFile(origBuf, true)\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) atomicOverwriteFile(buf []byte, allowCompress bool) error {\n\tif mb.mfd != nil {\n\t\tmb.closeFDsLockedNoCheck()\n\t\tdefer mb.enableForWriting(mb.fs.fip)\n\t}\n\n\torigFN := mb.mfn // The original message block on disk.\n\ttmpFN := mb.mfn + blkTmpSuffix // The new block will be written here.\n\n\t// Rather than modifying the existing block on disk (which is a dangerous\n\t// operation if something goes wrong), create a new temporary file. We will\n\t// write out the new block here and then swap the files around afterwards\n\t// once everything else has succeeded correctly.\n\t<-dios\n\ttmpFD, err := os.OpenFile(tmpFN, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, defaultFilePerms)\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create temporary file: %w\", err)\n\t}\n\n\terrorCleanup := func(err error) error {\n\t\t_ = tmpFD.Close()\n\t\t_ = os.Remove(tmpFN)\n\t\treturn err\n\t}\n\n\talg := NoCompression\n\tif calg := mb.fs.fcfg.Compression; calg != NoCompression && allowCompress {\n\t\talg = calg\n\t\t// The original buffer at this point is uncompressed, so we will now compress\n\t\t// it if needed. Note that if the selected algorithm is NoCompression, the\n\t\t// Compress function will just return the input buffer unmodified.\n\t\toriginalSize := len(buf)\n\t\tif buf, err = alg.Compress(buf); err != nil {\n\t\t\treturn errorCleanup(fmt.Errorf(\"failed to compress block: %w\", err))\n\t\t}\n\n\t\t// We only need to write out the metadata header if compression is enabled.\n\t\t// If we're trying to uncompress the file on disk at this point, don't bother\n\t\t// writing metadata.\n\t\tmeta := &CompressionInfo{\n\t\t\tAlgorithm: alg,\n\t\t\tOriginalSize: uint64(originalSize),\n\t\t}\n\t\tbuf = append(meta.MarshalMetadata(), buf...)\n\t}\n\n\t// Re-encrypt the block if necessary.\n\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\treturn errorCleanup(err)\n\t}\n\n\t// Write the new block data (which might be compressed or encrypted) to the\n\t// temporary file.\n\tif n, err := tmpFD.Write(buf); err != nil {\n\t\treturn errorCleanup(fmt.Errorf(\"failed to write to temporary file: %w\", err))\n\t} else if n != len(buf) {\n\t\treturn errorCleanup(fmt.Errorf(\"short write to temporary file (%d != %d)\", n, len(buf)))\n\t}\n\tif err := tmpFD.Sync(); err != nil {\n\t\treturn errorCleanup(fmt.Errorf(\"failed to sync temporary file: %w\", err))\n\t}\n\tif err := tmpFD.Close(); err != nil {\n\t\treturn errorCleanup(fmt.Errorf(\"failed to close temporary file: %w\", err))\n\t}\n\n\t// Now replace the original file with the newly updated temp file.\n\tif err := os.Rename(tmpFN, origFN); err != nil {\n\t\treturn fmt.Errorf(\"failed to move temporary file into place: %w\", err)\n\t}\n\n\t// Since the message block might be retained in memory, make sure the\n\t// compression algorithm is up-to-date, since this will be needed when\n\t// compacting or truncating.\n\tmb.cmp = alg\n\n\t// Also update rbytes\n\tmb.rbytes = uint64(len(buf))\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) decompressIfNeeded(buf []byte) ([]byte, error) {\n\tvar meta CompressionInfo\n\tif n, err := meta.UnmarshalMetadata(buf); err != nil {\n\t\t// There was a problem parsing the metadata header of the block.\n\t\t// If there's no metadata header, an error isn't returned here,\n\t\t// we will instead just use default values of no compression.\n\t\treturn nil, err\n\t} else if n == 0 {\n\t\t// There were no metadata bytes, so we assume the block is not\n\t\t// compressed and return it as-is.\n\t\treturn buf, nil\n\t} else {\n\t\t// Metadata was present so it's quite likely the block contents\n\t\t// are compressed. If by any chance the metadata claims that the\n\t\t// block is uncompressed, then the input slice is just returned\n\t\t// unmodified.\n\t\treturn meta.Algorithm.Decompress(buf[n:])\n\t}\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) encryptOrDecryptIfNeeded(buf []byte) error {\n\tif mb.bek != nil && len(buf) > 0 {\n\t\tbek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmb.bek = bek\n\t\tmb.bek.XORKeyStream(buf, buf)\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) ensureRawBytesLoaded() error {\n\tif mb.rbytes > 0 {\n\t\treturn nil\n\t}\n\tf, err := mb.openBlock()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\tif fi, err := f.Stat(); fi != nil && err == nil {\n\t\tmb.rbytes = uint64(fi.Size())\n\t} else {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// Sync msg and index files as needed. This is called from a timer.\nfunc (fs *fileStore) syncBlocks() {\n\tif fs.isClosed() {\n\t\treturn\n\t}\n\tfs.mu.Lock()\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn\n\t}\n\tblks := append([]*msgBlock(nil), fs.blks...)\n\tlmb, firstMoved, firstSeq := fs.lmb, fs.firstMoved, fs.state.FirstSeq\n\t// Clear first moved.\n\tfs.firstMoved = false\n\tfs.mu.Unlock()\n\n\tstoreFsWerr := func(err error) {\n\t\tfs.mu.Lock()\n\t\tdefer fs.mu.Unlock()\n\t\tfs.setWriteErr(err)\n\t}\n\n\tvar fsDmapLoaded bool\n\tvar fsDmap avl.SequenceSet\n\n\tvar markDirty bool\n\tfor _, mb := range blks {\n\t\t// Do actual sync. Hold lock for consistency.\n\t\tmb.mu.Lock()\n\t\tif mb.closed {\n\t\t\tmb.mu.Unlock()\n\t\t\tcontinue\n\t\t}\n\t\t// Bubble up an individual block error into the broader filestore.\n\t\tif err := mb.werr; err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\tstoreFsWerr(err)\n\t\t\tcontinue\n\t\t}\n\t\t// See if we can close FDs due to being idle.\n\t\tif mb.mfd != nil && mb.sinceLastWriteActivity() > closeFDsIdle && mb.pendingWriteSizeLocked() == 0 {\n\t\t\tif err := mb.dirtyCloseWithRemove(false); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tstoreFsWerr(err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// If our first has moved and we are set to noCompact (which is from tombstones),\n\t\t// clear so that we might cleanup tombstones.\n\t\tif firstMoved && mb.noCompact {\n\t\t\tmb.noCompact = false\n\t\t}\n\t\t// Check if we should compact here as well.\n\t\t// Do not compact last mb.\n\t\tvar needsCompact bool\n\t\tif mb != lmb && mb.ensureRawBytesLoaded() == nil && mb.shouldCompactSync() {\n\t\t\tneedsCompact = true\n\t\t\tmarkDirty = true\n\t\t}\n\n\t\t// Flush anything that may be pending.\n\t\tif _, err := mb.flushPendingMsgsLocked(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\tstoreFsWerr(err)\n\t\t\tcontinue\n\t\t}\n\t\t// Check if we need to sync. We will not hold lock during actual sync.\n\t\tneedSync := mb.needSync\n\n\t\t// Reset. Because we let go of the lock, we could write new data to this mb which might or\n\t\t// might not be synced later if we would've reset after letting go of the lock.\n\t\tmb.needSync = false\n\t\tmb.mu.Unlock()\n\n\t\t// Check if we should compact here.\n\t\t// Need to hold fs lock in case we reference psim when loading in the mb and we may remove this block if truly empty.\n\t\tif needsCompact {\n\t\t\t// Load a delete map containing only interior deletes.\n\t\t\t// This is used when compacting to know if tombstones are still relevant,\n\t\t\t// and if not they can be compacted.\n\t\t\tif !fsDmapLoaded {\n\t\t\t\tfsDmapLoaded = true\n\t\t\t\tfsDmap = fs.deleteMap()\n\t\t\t}\n\t\t\tfs.mu.RLock()\n\t\t\tmb.mu.Lock()\n\t\t\t// If the block has already been removed in the meantime, we can simply skip.\n\t\t\tif _, ok := fs.bim[mb.index]; !ok {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tfs.mu.RUnlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\terr := mb.compactWithFloor(firstSeq, &fsDmap)\n\t\t\t// If this compact removed all raw bytes due to tombstone cleanup, schedule to remove.\n\t\t\tshouldRemove := mb.rbytes == 0\n\t\t\tmb.mu.Unlock()\n\t\t\tfs.mu.RUnlock()\n\t\t\tif err != nil {\n\t\t\t\tstoreFsWerr(err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Check if we should remove. This will not be common, so we will re-take fs write lock here vs changing\n\t\t\t// it above which we would prefer to be a readlock such that other lookups can occur while compacting this block.\n\t\t\tif shouldRemove {\n\t\t\t\tfs.mu.Lock()\n\t\t\t\tmb.mu.Lock()\n\t\t\t\terr = fs.removeMsgBlock(mb)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\tneedSync = false\n\t\t\t\tif err != nil {\n\t\t\t\t\tstoreFsWerr(err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Check if we need to sync this block.\n\t\tif needSync {\n\t\t\tmb.mu.Lock()\n\t\t\tvar fd *os.File\n\t\t\tvar err error\n\t\t\tvar didOpen bool\n\t\t\tif mb.mfd != nil {\n\t\t\t\tfd = mb.mfd\n\t\t\t} else {\n\t\t\t\t<-dios\n\t\t\t\tfd, err = os.OpenFile(mb.mfn, os.O_RDWR, defaultFilePerms)\n\t\t\t\tdios <- struct{}{}\n\t\t\t\tdidOpen = true\n\t\t\t\tif err != nil && !os.IsNotExist(err) {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\tstoreFsWerr(err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If we have an fd.\n\t\t\tif fd != nil {\n\t\t\t\tif err = fd.Sync(); err != nil {\n\t\t\t\t\t// Close fd if we opened it, but ignore its error since sync takes precedence.\n\t\t\t\t\tif didOpen {\n\t\t\t\t\t\t_ = fd.Close()\n\t\t\t\t\t}\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\tstoreFsWerr(err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// If we opened the file close the fd.\n\t\t\t\tif didOpen {\n\t\t\t\t\tif err = fd.Close(); err != nil {\n\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\tstoreFsWerr(err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t}\n\t}\n\n\tif fs.isClosed() {\n\t\treturn\n\t}\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\tfs.setSyncTimer()\n\tif markDirty {\n\t\tfs.dirty++\n\t}\n\n\t// Sync state file if we are not running with sync always.\n\tif !fs.fcfg.SyncAlways {\n\t\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tvar fd *os.File\n\t\tvar err error\n\t\t<-dios\n\t\tfd, err = os.OpenFile(fn, os.O_RDWR, defaultFilePerms)\n\t\tdios <- struct{}{}\n\t\tif err != nil && !os.IsNotExist(err) {\n\t\t\tfs.setWriteErr(err)\n\t\t\treturn\n\t\t}\n\t\tif fd != nil {\n\t\t\tif err = fd.Sync(); err != nil {\n\t\t\t\t// Close fd, but ignore its error since sync takes precedence.\n\t\t\t\t_ = fd.Close()\n\t\t\t\tfs.setWriteErr(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err = fd.Close(); err != nil {\n\t\t\t\tfs.setWriteErr(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Select the message block where this message should be found.\n// Return nil if not in the set.\n// Read lock should be held.\nfunc (fs *fileStore) selectMsgBlock(seq uint64) *msgBlock {\n\t_, mb := fs.selectMsgBlockWithIndex(seq)\n\treturn mb\n}\n\n// Lock should be held.\nfunc (fs *fileStore) selectMsgBlockWithIndex(seq uint64) (int, *msgBlock) {\n\t// Check for out of range.\n\tif seq < fs.state.FirstSeq || seq > fs.state.LastSeq || fs.state.Msgs == 0 {\n\t\treturn -1, nil\n\t}\n\n\tconst linearThresh = 32\n\tnb := len(fs.blks) - 1\n\n\tif nb < linearThresh {\n\t\tfor i, mb := range fs.blks {\n\t\t\tif seq <= atomic.LoadUint64(&mb.last.seq) {\n\t\t\t\treturn i, mb\n\t\t\t}\n\t\t}\n\t\treturn -1, nil\n\t}\n\n\t// Do traditional binary search here since we know the blocks are sorted by sequence first and last.\n\tfor low, high, mid := 0, nb, nb/2; low <= high; mid = (low + high) / 2 {\n\t\tmb := fs.blks[mid]\n\t\t// Right now these atomic loads do not factor in, so fine to leave. Was considering\n\t\t// uplifting these to fs scope to avoid atomic load but not needed.\n\t\tfirst, last := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\t\tif seq > last {\n\t\t\tlow = mid + 1\n\t\t} else if seq < first {\n\t\t\t// A message block's first sequence can change here meaning we could find a gap.\n\t\t\t// We want to behave like above, which if inclusive (we check at start) should\n\t\t\t// always return an index and a valid mb.\n\t\t\t// If we have a gap then our seq would be > fs.blks[mid-1].last.seq\n\t\t\tif mid == 0 || seq > atomic.LoadUint64(&fs.blks[mid-1].last.seq) {\n\t\t\t\treturn mid, mb\n\t\t\t}\n\t\t\thigh = mid - 1\n\t\t} else {\n\t\t\treturn mid, mb\n\t\t}\n\t}\n\n\treturn -1, nil\n}\n\n// Select the message block where this message should be found.\n// Return nil if not in the set.\nfunc (fs *fileStore) selectMsgBlockForStart(minTime time.Time) *msgBlock {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\t// Binary search for first block where last.ts >= t.\n\ti, _ := slices.BinarySearchFunc(fs.blks, minTime.UnixNano(), func(mb *msgBlock, target int64) int {\n\t\tmb.mu.RLock()\n\t\tlast := mb.last.ts\n\t\tmb.mu.RUnlock()\n\t\tswitch {\n\t\tcase last < target:\n\t\t\treturn -1\n\t\tcase last > target:\n\t\t\treturn 1\n\t\tdefault:\n\t\t\treturn 0\n\t\t}\n\t})\n\n\t// BinarySearchFunc returns an insertion point if not found.\n\t// Either way, i is the index of the first mb where mb.last.ts >= t.\n\tif i < len(fs.blks) {\n\t\treturn fs.blks[i]\n\t}\n\treturn nil\n}\n\n// Index a raw msg buffer.\n// Lock should be held.\nfunc (mb *msgBlock) indexCacheBuf(buf []byte) error {\n\tvar le = binary.LittleEndian\n\n\tvar alloc bool\n\tvar idx []uint32\n\tvar index uint32\n\n\tmbFirstSeq := atomic.LoadUint64(&mb.first.seq)\n\tmbLastSeq := atomic.LoadUint64(&mb.last.seq)\n\n\t// Sanity check here since we calculate size to allocate based on this.\n\tif mbFirstSeq > (mbLastSeq + 1) { // Purged state first == last + 1\n\t\tmb.fs.warn(\"indexCacheBuf corrupt state in %s: mb.first %d mb.last %d\", mb.mfn, mbFirstSeq, mbLastSeq)\n\t\t// This would cause idxSz to wrap.\n\t\treturn errCorruptState\n\t}\n\n\tidxSz := mbLastSeq - mbFirstSeq + 1\n\n\tif mb.cache == nil {\n\t\tmb.cache = mb.ecache.Value()\n\t}\n\tif mb.cache == nil {\n\t\tmb.cache = &cache{}\n\t\talloc = true\n\t} else {\n\t\t// The buf arg already came from the pool probably, so there's\n\t\t// no point in reusing mb.cache.buf's underlying capacity here.\n\t\t// Just recycle it for the next block load.\n\t\trecycleMsgBlockBuf(mb.cache.buf)\n\t}\n\tif idx = mb.cache.idx; uint64(cap(idx)) >= idxSz {\n\t\tidx = idx[:0]\n\t} else {\n\t\tidx = make([]uint32, 0, idxSz)\n\t}\n\tif !alloc {\n\t\t// We didn't allocate a new *cache so instead wipe the current\n\t\t// one, we already have reused idx if possible.\n\t\t*mb.cache = cache{}\n\t}\n\n\t// Create FSS if we should track.\n\tvar popFss bool\n\tif mb.fssNotLoaded() {\n\t\tmb.fss = stree.NewSubjectTree[SimpleState]()\n\t\tpopFss = true\n\t}\n\t// Mark fss activity.\n\tmb.lsts = ats.AccessTime()\n\tmb.ttls = 0\n\tmb.schedules = 0\n\n\tlbuf := uint32(len(buf))\n\tvar seq, ttls, schedules uint64\n\tvar sm StoreMsg // Used for finding headers\n\n\t// To ensure the sequence keeps moving up. As well as confirming our index\n\t// is aligned with the mb's first and last sequence.\n\tvar first uint64\n\tvar last uint64\n\n\tfor index < lbuf {\n\t\tif index+msgHdrSize > lbuf {\n\t\t\treturn errCorruptState\n\t\t}\n\t\thdr := buf[index : index+msgHdrSize]\n\t\trl, slen := le.Uint32(hdr[0:]), int(le.Uint16(hdr[20:]))\n\t\tseq = le.Uint64(hdr[4:])\n\n\t\thasHeaders := rl&hbit != 0\n\t\t// Clear any headers bit that could be set.\n\t\trl &^= hbit\n\t\tshlen := slen\n\t\tif hasHeaders {\n\t\t\tshlen += 4\n\t\t}\n\t\tdlen := int(rl) - msgHdrSize\n\t\t// Do some quick sanity checks here.\n\t\tif dlen < 0 || shlen > (dlen-recordHashSize) || dlen > int(rl) || index+rl > lbuf || rl > rlBadThresh {\n\t\t\tmb.fs.warn(\"indexCacheBuf corrupt record state in %s: dlen %d slen %d index %d rl %d lbuf %d\", mb.mfn, dlen, slen, index, rl, lbuf)\n\t\t\t// This means something is off.\n\t\t\t// TODO(dlc) - Add into bad list?\n\t\t\treturn errCorruptState\n\t\t}\n\n\t\t// Check for tombstones which we can skip in terms of indexing.\n\t\tif seq&tbit != 0 {\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\n\t\t// Clear any erase bits.\n\t\terased := seq&ebit != 0\n\t\tseq = seq &^ ebit\n\n\t\t// The sequence needs to only ever move up.\n\t\tif seq <= last {\n\t\t\t// Advance to next record.\n\t\t\t// We've already accounted for this sequence and marked it as deleted.\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\t\t// We defer checksum checks to individual msg cache lookups to amortorize costs and\n\t\t// not introduce latency for first message from a newly loaded block.\n\t\tif seq >= mbFirstSeq {\n\t\t\tlast = seq\n\n\t\t\t// If the first sequence doesn't align with what we had in-memory, we need to rebuild.\n\t\t\tif first == 0 {\n\t\t\t\tfirst = seq\n\t\t\t\tif mbFirstSeq != first {\n\t\t\t\t\treturn errCorruptState\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Track that we do not have holes.\n\t\t\tif slot := int(seq - mbFirstSeq); slot != len(idx) {\n\t\t\t\t// If we have a hole fill it.\n\t\t\t\tfor dseq := mbFirstSeq + uint64(len(idx)); dseq < seq; dseq++ {\n\t\t\t\t\tidx = append(idx, dbit)\n\t\t\t\t\tif dseq != 0 {\n\t\t\t\t\t\tmb.dmap.Insert(dseq)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Add to our index.\n\t\t\tidx = append(idx, index)\n\n\t\t\t// Make sure our dmap has this entry if it was erased.\n\t\t\t// If not, that means this erased message was not accounted for in our in-memory state.\n\t\t\tif erased && seq != 0 && !mb.dmap.Exists(seq) {\n\t\t\t\treturn errCorruptState\n\t\t\t}\n\n\t\t\t// Handle FSS inline here.\n\t\t\tif popFss && slen > 0 && !mb.noTrack && !erased && !mb.dmap.Exists(seq) {\n\t\t\t\tbsubj := buf[index+msgHdrSize : index+msgHdrSize+uint32(slen)]\n\t\t\t\tif ss, ok := mb.fss.Find(bsubj); ok && ss != nil {\n\t\t\t\t\tss.Msgs++\n\t\t\t\t\tss.Last = seq\n\t\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t\t} else {\n\t\t\t\t\tmb.fss.Insert(bsubj, SimpleState{\n\t\t\t\t\t\tMsgs: 1,\n\t\t\t\t\t\tFirst: seq,\n\t\t\t\t\t\tLast: seq,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Count how many TTLs/schedules we think are in this message block.\n\t\t\t// TODO(nat): Not terribly optimal...\n\t\t\tif hasHeaders {\n\t\t\t\tif fsm, err := mb.msgFromBufNoCopy(buf[index:], &sm, nil); err == nil && fsm != nil {\n\t\t\t\t\tif ttl := sliceHeader(JSMessageTTL, fsm.hdr); len(ttl) > 0 {\n\t\t\t\t\t\tttls++\n\t\t\t\t\t}\n\t\t\t\t\tif sched := sliceHeader(JSSchedulePattern, fsm.hdr); len(sched) > 0 {\n\t\t\t\t\t\tschedules++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tindex += rl\n\t}\n\n\t// If we ended up with a smaller or larger index, or the first/last sequence\n\t// doesn't align with what we had in-memory, we need to rebuild.\n\tif len(idx) != int(idxSz) || (first > 0 && mbFirstSeq != first) || (last > 0 && mbLastSeq != last) {\n\t\treturn errCorruptState\n\t}\n\n\tmb.cache.buf = buf\n\tmb.cache.idx = idx\n\tmb.cache.fseq = mbFirstSeq\n\tmb.cache.wp = int(lbuf)\n\tmb.ttls = ttls\n\tmb.schedules = schedules\n\n\treturn nil\n}\n\n// flushPendingMsgs writes out any messages for this message block.\nfunc (mb *msgBlock) flushPendingMsgs() error {\n\tmb.mu.Lock()\n\tfsLostData, err := mb.flushPendingMsgsLocked()\n\tfs := mb.fs\n\tmb.mu.Unlock()\n\n\t// Signals us that we need to rebuild filestore state.\n\tif fsLostData != nil && fs != nil {\n\t\t// Rebuild fs state too.\n\t\tfs.rebuildState(fsLostData)\n\t}\n\treturn err\n}\n\n// Write function for actual data.\n// mb.mfd should not be nil.\n// Lock should held.\nfunc (mb *msgBlock) writeAt(buf []byte, woff int64) (int, error) {\n\t// Used to mock write failures.\n\tif mb.mockWriteErr {\n\t\t// Reset on trip.\n\t\tmb.mockWriteErr = false\n\t\treturn 0, errors.New(\"mock write error\")\n\t}\n\t<-dios\n\tn, err := mb.mfd.WriteAt(buf, woff)\n\tdios <- struct{}{}\n\treturn n, err\n}\n\n// flushPendingMsgsLocked writes out any messages for this message block.\n// Lock should be held.\nfunc (mb *msgBlock) flushPendingMsgsLocked() (*LostStreamData, error) {\n\tvar weakenCache bool\n\tif mb.cache == nil {\n\t\tmb.cache = mb.ecache.Value()\n\t\tweakenCache = mb.cache != nil\n\t}\n\tif mb.cache == nil || mb.mfd == nil || mb.werr != nil {\n\t\treturn nil, mb.werr\n\t}\n\n\tbuf, err := mb.bytesPending()\n\t// If we got an error back return here.\n\tif err != nil {\n\t\t// No pending data to be written is not an error.\n\t\tif err == errNoPending || err == errNoCache {\n\t\t\terr = nil\n\t\t}\n\t\treturn nil, err\n\t}\n\n\twp := int64(mb.cache.wp)\n\tlob := len(buf)\n\n\t// TODO(dlc) - Normally we would not hold the lock across I/O so we can improve performance.\n\t// We will hold to stabilize the code base, as we have had a few anomalies with partial cache errors\n\t// under heavy load.\n\n\t// Check if we need to encrypt.\n\tif err := mb.checkAndLoadEncryption(); err != nil {\n\t\treturn nil, err\n\t}\n\tif mb.bek != nil && lob > 0 {\n\t\t// Need to leave original alone.\n\t\tvar dst []byte\n\t\tif lob <= defaultLargeBlockSize {\n\t\t\tdst = getMsgBlockBuf(lob)[:lob]\n\t\t\tdefer recycleMsgBlockBuf(dst)\n\t\t} else {\n\t\t\tdst = make([]byte, lob)\n\t\t}\n\t\tmb.bek.XORKeyStream(dst, buf)\n\t\tbuf = dst\n\t}\n\n\t// Append new data to the message block file.\n\tfor lbb := lob; lbb > 0; lbb = len(buf) {\n\t\tn, err := mb.writeAt(buf, wp)\n\t\tif err != nil {\n\t\t\t// Ignore the errors here, we'll try reloading just to figure out and return the lost data if we can.\n\t\t\t_ = mb.dirtyCloseWithRemove(false)\n\t\t\tld, _, _ := mb.rebuildStateLocked()\n\t\t\tmb.werr = err\n\t\t\tassert.Unreachable(\"Filestore msg block encountered flush error\", map[string]any{\n\t\t\t\t\"name\": mb.fs.cfg.Name,\n\t\t\t\t\"mb.index\": mb.index,\n\t\t\t\t\"err\": err,\n\t\t\t\t\"stack\": string(debug.Stack()),\n\t\t\t})\n\t\t\treturn ld, err\n\t\t}\n\t\t// Update our write offset.\n\t\twp += int64(n)\n\t\tbuf = buf[n:]\n\t}\n\n\t// Cache may be gone.\n\tif mb.cache == nil || mb.mfd == nil {\n\t\treturn nil, mb.werr\n\t}\n\n\t// Update write pointer.\n\tmb.cache.wp = int(wp)\n\n\t// Check if we are in sync always mode.\n\tif mb.syncAlways {\n\t\tif err = mb.mfd.Sync(); err != nil {\n\t\t\tmb.werr = err\n\t\t\tassert.Unreachable(\"Filestore msg block encountered sync error\", map[string]any{\n\t\t\t\t\"name\": mb.fs.cfg.Name,\n\t\t\t\t\"mb.index\": mb.index,\n\t\t\t\t\"err\": err,\n\t\t\t\t\"stack\": string(debug.Stack()),\n\t\t\t})\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tmb.needSync = true\n\t}\n\n\t// Check for additional writes while we were writing to the disk.\n\t// TODO(nat): This should not be possible at present since, as above, we are\n\t// not releasing the lock during I/O operation. Therefore this will always\n\t// return zero.\n\tif mb.pendingWriteSizeLocked() > 0 {\n\t\treturn nil, mb.werr\n\t}\n\n\t// Check last access time. If we think the block still has read interest\n\t// then we will weaken the pointer but otherwise try to hold onto it.\n\tif ts := ats.AccessTime(); ts < mb.llts || (ts-mb.llts) <= int64(mb.cexp) {\n\t\tif weakenCache {\n\t\t\tmb.cache = nil\n\t\t\tmb.ecache.Weaken()\n\t\t}\n\t\tmb.resetCacheExpireTimer(0)\n\t\treturn nil, mb.werr\n\t}\n\n\t// If not, we'll just drop the cache altogether & recycle the buffer.\n\tmb.cache.nra = false\n\tmb.expireCacheLocked()\n\treturn nil, mb.werr\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) clearLoading() {\n\tmb.loading = false\n}\n\n// Will load msgs from disk.\nfunc (mb *msgBlock) loadMsgs() error {\n\t// We hold the lock here the whole time by design.\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.loadMsgsWithLock()\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) cacheAlreadyLoaded() bool {\n\tif mb.cache == nil {\n\t\tmb.cache = mb.ecache.Value()\n\t}\n\tif mb.cache == nil || mb.cache.fseq == 0 || len(mb.cache.buf) == 0 {\n\t\treturn false\n\t}\n\tnumEntries := mb.msgs + uint64(mb.dmap.Size()) + (atomic.LoadUint64(&mb.first.seq) - mb.cache.fseq)\n\treturn numEntries == uint64(len(mb.cache.idx))\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) cacheNotLoaded() bool {\n\treturn !mb.cacheAlreadyLoaded()\n}\n\n// Report if our fss is not loaded.\n// Lock should be held.\nfunc (mb *msgBlock) fssNotLoaded() bool {\n\treturn mb.fss == nil && !mb.noTrack\n}\n\n// Wrap openBlock for the gated semaphore processing.\n// Lock should be held\nfunc (mb *msgBlock) openBlock() (*os.File, error) {\n\t// Gate with concurrent IO semaphore.\n\t<-dios\n\tf, err := os.Open(mb.mfn)\n\tdios <- struct{}{}\n\treturn f, err\n}\n\n// Used to load in the block contents.\n// Lock should be held and all conditionals satisfied prior.\nfunc (mb *msgBlock) loadBlock(buf []byte) ([]byte, error) {\n\tvar f *os.File\n\t// Re-use if we have mfd open.\n\tif mb.mfd != nil {\n\t\tf = mb.mfd\n\t\tif n, err := f.Seek(0, 0); n != 0 || err != nil {\n\t\t\tf = nil\n\t\t\tmb.closeFDsLockedNoCheck()\n\t\t}\n\t}\n\tif f == nil {\n\t\tvar err error\n\t\tf, err = mb.openBlock()\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\terr = errNoBlkData\n\t\t\t}\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer f.Close()\n\t}\n\n\tvar sz int\n\tif info, err := f.Stat(); err == nil {\n\t\tsz64 := info.Size()\n\t\tif int64(int(sz64)) == sz64 {\n\t\t\tsz = int(sz64)\n\t\t} else {\n\t\t\treturn nil, errMsgBlkTooBig\n\t\t}\n\t}\n\n\tif buf == nil || cap(buf) < sz {\n\t\t// getMsgBlockBuf will try to return a slice that fits from the\n\t\t// buffer pool, but if by chance `sz` is greater than the maximum\n\t\t// large block size, it'll return a new slice to fit regardless.\n\t\tbuf = getMsgBlockBuf(sz)\n\t}\n\n\t<-dios\n\tn, err := io.ReadFull(f, buf[:sz])\n\tdios <- struct{}{}\n\t// On success capture raw bytes size.\n\tif err == nil {\n\t\tmb.rbytes = uint64(n)\n\t}\n\treturn buf[:n], err\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) loadMsgsWithLock() error {\n\tif err := mb.checkAndLoadEncryption(); err != nil {\n\t\treturn err\n\t}\n\n\t// Check to see if we are loading already.\n\tif mb.loading {\n\t\treturn nil\n\t}\n\n\t// Set loading status.\n\tmb.loading = true\n\tdefer mb.clearLoading()\n\n\tvar nchecks int\n\ncheckCache:\n\tnchecks++\n\tif nchecks > 8 {\n\t\treturn errNoCache\n\t}\n\n\t// Check to see if we have a full cache.\n\tif mb.cacheAlreadyLoaded() {\n\t\treturn nil\n\t}\n\n\tmb.llts = ats.AccessTime()\n\n\t// FIXME(dlc) - We could be smarter here.\n\tif buf, _ := mb.bytesPending(); len(buf) > 0 {\n\t\tld, err := mb.flushPendingMsgsLocked()\n\t\tif ld != nil && mb.fs != nil {\n\t\t\t// We do not know if fs is locked or not at this point.\n\t\t\t// This should be an exceptional condition so do so in Go routine.\n\t\t\tgo mb.fs.rebuildState(ld)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tgoto checkCache\n\t}\n\n\t// Load in the whole block.\n\t// We want to hold the mb lock here to avoid any changes to state.\n\tbuf, err := mb.loadBlock(nil)\n\tif err != nil {\n\t\tmb.fs.warn(\"loadBlock error: %v\", err)\n\t\tif err == errNoBlkData {\n\t\t\tif ld, _, _ := mb.rebuildStateLocked(); ld != nil {\n\t\t\t\t// Rebuild fs state too.\n\t\t\t\tgo mb.fs.rebuildState(ld)\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}\n\n\t// Check if we need to decrypt.\n\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\treturn err\n\t}\n\t// Check for compression.\n\tif buf, err = mb.decompressIfNeeded(buf); err != nil {\n\t\treturn err\n\t}\n\n\tif err := mb.indexCacheBuf(buf); err != nil {\n\t\tif err == errCorruptState {\n\t\t\tvar ld *LostStreamData\n\t\t\tld, _, err = mb.rebuildStateLocked()\n\t\t\t// We do not know if fs is locked or not at this point.\n\t\t\t// This should be an exceptional condition so do so in Go routine.\n\t\t\t// Always rebuild the filestore's state if indexing fails, even if no data was lost,\n\t\t\t// our in-memory state was stale in that case.\n\t\t\tgo mb.fs.rebuildState(ld)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tgoto checkCache\n\t}\n\n\tif len(buf) > 0 {\n\t\tmb.cloads++\n\t\tmb.startCacheExpireTimer()\n\t}\n\n\t// Update the cache pointer since we cleared & rebuilt.\n\tmb.ecache.Set(mb.cache)\n\n\treturn nil\n}\n\n// Fetch a message from this block, possibly reading in and caching the messages.\n// We assume the block was selected and is correct, so we do not do range checks.\n// Lock should not be held.\nfunc (mb *msgBlock) fetchMsg(seq uint64, sm *StoreMsg) (*StoreMsg, bool, error) {\n\treturn mb.fetchMsgEx(seq, sm, true)\n}\n\n// Fetch a message from this block, possibly reading in and caching the messages.\n// We assume the block was selected and is correct, so we do not do range checks.\n// We will not copy the msg data.\n// Lock should not be held.\nfunc (mb *msgBlock) fetchMsgNoCopy(seq uint64, sm *StoreMsg) (*StoreMsg, bool, error) {\n\treturn mb.fetchMsgEx(seq, sm, false)\n}\n\n// Fetch a message from this block, possibly reading in and caching the messages.\n// We assume the block was selected and is correct, so we do not do range checks.\n// We will copy the msg data based on doCopy boolean.\n// Lock should not be held.\nfunc (mb *msgBlock) fetchMsgEx(seq uint64, sm *StoreMsg, doCopy bool) (*StoreMsg, bool, error) {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\n\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\tif seq < fseq || seq > lseq {\n\t\treturn nil, false, ErrStoreMsgNotFound\n\t}\n\n\t// See if we can short circuit if we already know msg deleted.\n\tif mb.dmap.Exists(seq) {\n\t\t// Update for scanning like cacheLookup would have.\n\t\tllseq := mb.llseq\n\t\tif mb.llseq == 0 || seq < mb.llseq || seq == mb.llseq+1 || seq == mb.llseq-1 {\n\t\t\tmb.llseq = seq\n\t\t}\n\t\texpireOk := (seq == lseq && llseq == seq-1) || (seq == fseq && llseq == seq+1)\n\t\treturn nil, expireOk, errDeletedMsg\n\t}\n\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t}\n\tdefer mb.finishedWithCache()\n\tllseq := mb.llseq\n\n\tfsm, err := mb.cacheLookupEx(seq, sm, doCopy)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\texpireOk := (seq == lseq && llseq == seq-1) || (seq == fseq && llseq == seq+1)\n\treturn fsm, expireOk, err\n}\n\nvar (\n\terrNoCache = errors.New(\"no message cache\")\n\terrDeletedMsg = errors.New(\"deleted message\")\n\terrPartialCache = errors.New(\"partial cache\")\n\terrNoPending = errors.New(\"message block does not have pending data\")\n\terrNotReadable = errors.New(\"storage directory not readable\")\n\terrCorruptState = errors.New(\"corrupt state file\")\n\terrPriorState = errors.New(\"prior state file\")\n\terrPendingData = errors.New(\"pending data still present\")\n\terrNoEncryption = errors.New(\"encryption not enabled\")\n\terrBadKeySize = errors.New(\"encryption bad key size\")\n\terrNoMsgBlk = errors.New(\"no message block\")\n\terrMsgBlkTooBig = errors.New(\"message block size exceeded int capacity\")\n\terrUnknownCipher = errors.New(\"unknown cipher\")\n\terrNoMainKey = errors.New(\"encrypted store encountered with no main key\")\n\terrNoBlkData = errors.New(\"message block data missing\")\n\terrStateTooBig = errors.New(\"store state too big for optional write\")\n)\n\ntype (\n\terrBadMsg struct{ fn, detail string }\n)\n\nfunc (e errBadMsg) Error() string {\n\tif e.detail != _EMPTY_ {\n\t\treturn fmt.Sprintf(\"malformed or corrupt message in %s: %s\", filepath.Base(e.fn), e.detail)\n\t}\n\treturn fmt.Sprintf(\"malformed or corrupt message in %s\", filepath.Base(e.fn))\n}\n\nconst (\n\t// \"Checksum bit\" is used in \"mb.cache.idx\" for marking messages that have had their checksums checked.\n\tcbit = 1 << 31\n\t// \"Delete bit\" is used in \"mb.cache.idx\" to mark an index as deleted and non-existent.\n\tdbit = 1 << 30\n\t// \"Header bit\" is used in \"rl\" to signal a message record with headers.\n\thbit = 1 << 31\n\t// \"Erase bit\" is used in \"seq\" for marking erased messages sequences.\n\tebit = 1 << 63\n\t// \"Tombstone bit\" is used in \"seq\" for marking tombstone sequences.\n\ttbit = 1 << 62\n)\n\n// Will do a lookup from cache.\n// This will copy the msg from the cache.\n// Lock should be held.\nfunc (mb *msgBlock) cacheLookup(seq uint64, sm *StoreMsg) (*StoreMsg, error) {\n\treturn mb.cacheLookupEx(seq, sm, true)\n}\n\n// Will do a lookup from cache.\n// This will NOT copy the msg from the cache.\n// Lock should be held.\nfunc (mb *msgBlock) cacheLookupNoCopy(seq uint64, sm *StoreMsg) (*StoreMsg, error) {\n\treturn mb.cacheLookupEx(seq, sm, false)\n}\n\n// Will do a lookup from cache.\n// Lock should be held.\nfunc (mb *msgBlock) cacheLookupEx(seq uint64, sm *StoreMsg, doCopy bool) (*StoreMsg, error) {\n\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\tswitch {\n\tcase lseq == fseq-1:\n\t\t// The block is empty, no messages have been written yet. This works because\n\t\t// newMsgBlockForWrite sets fseq=fs.State.LastSeq+1 and lseq=fs.State.LastSeq.\n\t\treturn nil, ErrStoreMsgNotFound\n\tcase seq < fseq || seq > lseq:\n\t\t// Sequence is out of range for this block.\n\t\treturn nil, ErrStoreMsgNotFound\n\t}\n\n\t// The llseq signals us when we can expire a cache at the end of a linear scan.\n\t// We want to only update when we know the last reads (multiple consumers) are sequential.\n\t// We want to account for forwards and backwards linear scans.\n\tif mb.llseq == 0 || seq < mb.llseq || seq == mb.llseq+1 || seq == mb.llseq-1 {\n\t\tmb.llseq = seq\n\t}\n\n\t// If we have a delete map check it.\n\tif mb.dmap.Exists(seq) {\n\t\tmb.llts = ats.AccessTime()\n\t\treturn nil, errDeletedMsg\n\t}\n\n\t// Detect no cache loaded.\n\tif mb.cache == nil {\n\t\tmb.cache = mb.ecache.Value()\n\t}\n\tif mb.cache == nil || mb.cache.fseq == 0 || len(mb.cache.idx) == 0 || len(mb.cache.buf) == 0 {\n\t\tvar reason string\n\t\tif mb.cache == nil {\n\t\t\treason = \"no cache\"\n\t\t} else if mb.cache.fseq == 0 {\n\t\t\treason = \"fseq is 0\"\n\t\t} else if len(mb.cache.idx) == 0 {\n\t\t\treason = \"no idx present\"\n\t\t} else {\n\t\t\treason = \"cache buf empty\"\n\t\t}\n\t\tmb.fs.warn(\"Cache lookup for sequence %d in block %d detected no cache: %s\", seq, mb.index, reason)\n\t\tif mb.cache != nil {\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t}\n\t\treturn nil, errNoCache\n\t}\n\t// Check partial cache status.\n\tif seq < mb.cache.fseq {\n\t\tmb.fs.warn(\"Partial cache: seq %d is less than cache fseq %d\", seq, mb.cache.fseq)\n\t\treturn nil, errPartialCache\n\t}\n\n\tbi, _, hashChecked, err := mb.slotInfo(int(seq - mb.cache.fseq))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Update cache activity.\n\tmb.llts = ats.AccessTime()\n\n\tli := int(bi)\n\tif li >= len(mb.cache.buf) {\n\t\tmb.fs.warn(\"Partial cache: slot index %d is less than cache buffer len %d\", li, len(mb.cache.buf))\n\t\treturn nil, errPartialCache\n\t}\n\tbuf := mb.cache.buf[li:]\n\n\t// We use the high bit to denote we have already checked the checksum.\n\tvar hh *highwayhash.Digest64\n\tif !hashChecked {\n\t\thh = mb.hh // This will force the hash check in msgFromBuf.\n\t}\n\n\t// Parse from the raw buffer.\n\tfsm, err := mb.msgFromBufEx(buf, sm, hh, doCopy)\n\tif err != nil || fsm == nil {\n\t\treturn nil, err\n\t}\n\n\t// Deleted messages that are decoded return a 0 for sequence.\n\tif fsm.seq == 0 {\n\t\treturn nil, errDeletedMsg\n\t}\n\n\tif seq != fsm.seq { // See TestFileStoreInvalidIndexesRebuilt.\n\t\tmb.tryForceExpireCacheLocked()\n\t\treturn nil, fmt.Errorf(\"sequence numbers for cache load did not match, %d vs %d\", seq, fsm.seq)\n\t}\n\n\t// Clear the check bit here after we know all is good.\n\tif !hashChecked {\n\t\tmb.cache.idx[seq-mb.cache.fseq] = (bi | cbit)\n\t}\n\n\treturn fsm, nil\n}\n\n// Used when we are checking if discarding a message due to max msgs per subject will give us\n// enough room for a max bytes condition.\n// Lock should be already held.\nfunc (fs *fileStore) sizeForSeq(seq uint64) int {\n\tif seq == 0 {\n\t\treturn 0\n\t}\n\tvar smv StoreMsg\n\tif mb := fs.selectMsgBlock(seq); mb != nil {\n\t\tif sm, _, _ := mb.fetchMsgNoCopy(seq, &smv); sm != nil {\n\t\t\treturn int(fileStoreMsgSize(sm.subj, sm.hdr, sm.msg))\n\t\t}\n\t}\n\treturn 0\n}\n\n// Will return message for the given sequence number.\n// This will be returned to external callers.\nfunc (fs *fileStore) msgForSeq(seq uint64, sm *StoreMsg) (*StoreMsg, error) {\n\treturn fs.msgForSeqLocked(seq, sm, true)\n}\n\n// Will return message for the given sequence number.\nfunc (fs *fileStore) msgForSeqLocked(seq uint64, sm *StoreMsg, needFSLock bool) (*StoreMsg, error) {\n\tif fs.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\t// TODO(dlc) - Since Store, Remove, Skip all hold the write lock on fs this will\n\t// be stalled. Need another lock if want to happen in parallel.\n\tif needFSLock {\n\t\tfs.mu.RLock()\n\t}\n\t// Indicates we want first msg.\n\tif seq == 0 {\n\t\tseq = fs.state.FirstSeq\n\t}\n\t// Make sure to snapshot here.\n\tmb, lseq := fs.selectMsgBlock(seq), fs.state.LastSeq\n\tif needFSLock {\n\t\tfs.mu.RUnlock()\n\t}\n\n\tif mb == nil {\n\t\tvar err = ErrStoreEOF\n\t\tif seq <= lseq {\n\t\t\terr = ErrStoreMsgNotFound\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tfsm, expireOk, err := mb.fetchMsg(seq, sm)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// We detected a linear scan and access to the last message.\n\t// If we are not the last message block we can try to expire the cache.\n\tif expireOk {\n\t\tmb.tryForceExpireCache()\n\t}\n\n\treturn fsm, nil\n}\n\n// Internal function to return msg parts from a raw buffer.\n// Raw buffer will be copied into sm.\n// Lock should be held.\nfunc (mb *msgBlock) msgFromBuf(buf []byte, sm *StoreMsg, hh *highwayhash.Digest64) (*StoreMsg, error) {\n\treturn mb.msgFromBufEx(buf, sm, hh, true)\n}\n\n// Internal function to return msg parts from a raw buffer.\n// Raw buffer will NOT be copied into sm.\n// Only use for internal use, any message that is passed to upper layers should use mb.msgFromBuf.\n// Lock should be held.\nfunc (mb *msgBlock) msgFromBufNoCopy(buf []byte, sm *StoreMsg, hh *highwayhash.Digest64) (*StoreMsg, error) {\n\treturn mb.msgFromBufEx(buf, sm, hh, false)\n}\n\n// Internal function to return msg parts from a raw buffer.\n// copy boolean will determine if we make a copy or not.\n// Lock should be held.\nfunc (mb *msgBlock) msgFromBufEx(buf []byte, sm *StoreMsg, hh *highwayhash.Digest64, doCopy bool) (*StoreMsg, error) {\n\tif len(buf) < emptyRecordLen {\n\t\treturn nil, errBadMsg{mb.mfn, \"record too short\"}\n\t}\n\tvar le = binary.LittleEndian\n\n\thdr := buf[:msgHdrSize]\n\trl := le.Uint32(hdr[0:])\n\thasHeaders := rl&hbit != 0\n\trl &^= hbit // clear header bit\n\tdlen := int(rl) - msgHdrSize\n\tslen := int(le.Uint16(hdr[20:]))\n\n\tshlen := slen\n\tif hasHeaders {\n\t\tshlen += 4\n\t}\n\t// Simple sanity check.\n\tif dlen < 0 || shlen > (dlen-recordHashSize) || dlen > int(rl) || int(rl) > len(buf) || rl > rlBadThresh {\n\t\treturn nil, errBadMsg{mb.mfn, fmt.Sprintf(\"sanity check failed (dlen %d slen %d rl %d buf %d)\", dlen, slen, rl, buf)}\n\t}\n\tdata := buf[msgHdrSize : msgHdrSize+dlen]\n\t// Do checksum tests here if requested.\n\tif hh != nil {\n\t\thh.Reset()\n\t\thh.Write(hdr[4:20])\n\t\thh.Write(data[:slen])\n\t\tif hasHeaders {\n\t\t\thh.Write(data[slen+4 : dlen-recordHashSize])\n\t\t} else {\n\t\t\thh.Write(data[slen : dlen-recordHashSize])\n\t\t}\n\t\tvar hb [highwayhash.Size64]byte\n\t\tif !bytes.Equal(hh.Sum(hb[:0]), data[len(data)-8:]) {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid checksum\"}\n\t\t}\n\t}\n\tseq := le.Uint64(hdr[4:])\n\tif seq&ebit != 0 {\n\t\tseq = 0\n\t}\n\tts := int64(le.Uint64(hdr[12:]))\n\n\t// Create a StoreMsg if needed.\n\tif sm == nil {\n\t\tsm = new(StoreMsg)\n\t} else {\n\t\tsm.clear()\n\t}\n\t// To recycle the large blocks we can never pass back a reference, so need to copy for the upper\n\t// layers and for us to be safe to expire, and recycle, the large msgBlocks.\n\tend := dlen - 8\n\tif len(data) < end {\n\t\treturn nil, errBadMsg{mb.mfn, \"invalid data length\"}\n\t}\n\n\tif hasHeaders {\n\t\tif slen+4 > len(data) {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid subject length greataer than data length\"}\n\t\t}\n\t\thl := le.Uint32(data[slen:])\n\t\tbi := slen + 4\n\t\tli := bi + int(hl)\n\t\tif bi > end {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid buffer index\"}\n\t\t}\n\t\tif doCopy {\n\t\t\tsm.buf = append(sm.buf, data[bi:end]...)\n\t\t} else {\n\t\t\tsm.buf = data[bi:end]\n\t\t}\n\t\tli, end = li-bi, end-bi\n\t\tif li > len(sm.buf) || end > len(sm.buf) {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid message length or end greater than buffer length\"}\n\t\t}\n\t\tsm.hdr = sm.buf[0:li:li]\n\t\tsm.msg = sm.buf[li:end]\n\t} else {\n\t\tif slen > end {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid subject length greater than end\"}\n\t\t}\n\t\tif doCopy {\n\t\t\tsm.buf = append(sm.buf, data[slen:end]...)\n\t\t} else {\n\t\t\tsm.buf = data[slen:end]\n\t\t}\n\t\tmlen := end - slen\n\t\tif mlen > len(sm.buf) {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid message length greater than buffer length\"}\n\t\t}\n\t\tsm.msg = sm.buf[0:mlen]\n\t}\n\tsm.seq, sm.ts = seq, ts\n\tif slen > 0 {\n\t\tif slen > len(data) {\n\t\t\treturn nil, errBadMsg{mb.mfn, \"invalid subject length greater than data length\"}\n\t\t}\n\t\tif doCopy {\n\t\t\t// Make a copy since sm.subj lifetime may last longer.\n\t\t\tsm.subj = string(data[:slen])\n\t\t} else {\n\t\t\tsm.subj = bytesToString(data[:slen])\n\t\t}\n\t}\n\n\treturn sm, nil\n}\n\n// SubjectForSeq will return what the subject is for this sequence if found.\nfunc (fs *fileStore) SubjectForSeq(seq uint64) (string, error) {\n\tfs.mu.RLock()\n\tif seq < fs.state.FirstSeq {\n\t\tfs.mu.RUnlock()\n\t\treturn _EMPTY_, ErrStoreMsgNotFound\n\t}\n\tvar smv StoreMsg\n\tmb := fs.selectMsgBlock(seq)\n\tfs.mu.RUnlock()\n\tif mb != nil {\n\t\tif sm, _, _ := mb.fetchMsgNoCopy(seq, &smv); sm != nil {\n\t\t\t// Copy the subject, as it's used elsewhere, and the backing cache could be reused in the meantime.\n\t\t\treturn copyString(sm.subj), nil\n\t\t}\n\t}\n\treturn _EMPTY_, ErrStoreMsgNotFound\n}\n\n// LoadMsg will lookup the message by sequence number and return it if found.\nfunc (fs *fileStore) LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) {\n\treturn fs.msgForSeq(seq, sm)\n}\n\n// loadLast will load the last message for a subject. Subject should be non empty and not \">\".\nfunc (fs *fileStore) loadLast(subj string, sm *StoreMsg) (lsm *StoreMsg, err error) {\n\tif fs.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\treturn fs.loadLastLocked(subj, sm)\n}\n\n// Lock should be held.\nfunc (fs *fileStore) loadLastLocked(subj string, sm *StoreMsg) (lsm *StoreMsg, err error) {\n\tif fs.isClosed() || fs.lmb == nil {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tif len(fs.blks) == 0 {\n\t\treturn nil, ErrStoreMsgNotFound\n\t}\n\n\twc := subjectHasWildcard(subj)\n\tvar start, stop uint32\n\n\t// If literal subject check for presence.\n\tif wc {\n\t\tstart = fs.lmb.index\n\t\tfs.psim.Match(stringToBytes(subj), func(_ []byte, psi *psi) {\n\t\t\t// Keep track of start and stop indexes for this subject.\n\t\t\tif psi.fblk < start {\n\t\t\t\tstart = psi.fblk\n\t\t\t}\n\t\t\tif psi.lblk > stop {\n\t\t\t\tstop = psi.lblk\n\t\t\t}\n\t\t})\n\t\t// None matched.\n\t\tif stop == 0 {\n\t\t\treturn nil, ErrStoreMsgNotFound\n\t\t}\n\t\t// These need to be swapped.\n\t\tstart, stop = stop, start\n\t} else if info, ok := fs.psim.Find(stringToBytes(subj)); ok {\n\t\tstart, stop = info.lblk, info.fblk\n\t} else {\n\t\treturn nil, ErrStoreMsgNotFound\n\t}\n\n\t// Walk blocks backwards.\n\tfor i := start; i >= stop; i-- {\n\t\tmb := fs.bim[i]\n\t\tif mb == nil {\n\t\t\tcontinue\n\t\t}\n\t\tmb.mu.Lock()\n\t\tif err = mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\treturn nil, err\n\t\t}\n\t\t// Mark fss activity.\n\t\tmb.lsts = ats.AccessTime()\n\n\t\tvar l uint64\n\t\t// Optimize if subject is not a wildcard.\n\t\tif !wc {\n\t\t\tif ss, ok := mb.fss.Find(stringToBytes(subj)); ok && ss != nil {\n\t\t\t\t// Check if we need to recalculate. We only care about the last sequence.\n\t\t\t\tif ss.lastNeedsUpdate {\n\t\t\t\t\t// mb is already loaded into the cache so should be fast-ish.\n\t\t\t\t\tif err = mb.recalculateForSubj(subj, ss); err != nil {\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tl = ss.Last\n\t\t\t}\n\t\t}\n\t\tif l == 0 {\n\t\t\t_, _, l, err = mb.filteredPendingLocked(subj, wc, atomic.LoadUint64(&mb.first.seq))\n\t\t\tif err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tvar didLoad bool\n\t\tif l > 0 {\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tdidLoad = true\n\t\t\t}\n\t\t\tlsm, err = mb.cacheLookup(l, sm)\n\t\t}\n\t\tif didLoad {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tmb.mu.Unlock()\n\t\tif l > 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn lsm, err\n}\n\n// LoadLastMsg will return the last message we have that matches a given subject.\n// The subject can be a wildcard.\nfunc (fs *fileStore) LoadLastMsg(subject string, smv *StoreMsg) (sm *StoreMsg, err error) {\n\tif subject == _EMPTY_ || subject == fwcs {\n\t\tsm, err = fs.msgForSeq(fs.lastSeq(), smv)\n\t} else {\n\t\tsm, err = fs.loadLast(subject, smv)\n\t}\n\tif sm == nil || (err != nil && err != ErrStoreClosed) {\n\t\terr = ErrStoreMsgNotFound\n\t}\n\treturn sm, err\n}\n\n// LoadNextMsgMulti will find the next message matching any entry in the sublist.\nfunc (fs *fileStore) LoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) {\n\tif fs.isClosed() {\n\t\treturn nil, 0, ErrStoreClosed\n\t}\n\tif sl == nil {\n\t\treturn fs.LoadNextMsg(_EMPTY_, false, start, smp)\n\t}\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || start > fs.state.LastSeq {\n\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t}\n\tif start < fs.state.FirstSeq {\n\t\tstart = fs.state.FirstSeq\n\t}\n\n\t// If start is less than or equal to beginning of our stream, meaning our first call,\n\t// let's check the psim to see if we can skip ahead.\n\tif start <= fs.state.FirstSeq {\n\t\tvar total uint64\n\t\tblkStart := uint32(math.MaxUint32)\n\t\tstree.IntersectGSL(fs.psim, sl, func(subj []byte, psi *psi) {\n\t\t\ttotal += psi.total\n\t\t\t// Keep track of start index for this subject.\n\t\t\tif psi.fblk < blkStart {\n\t\t\t\tblkStart = psi.fblk\n\t\t\t}\n\t\t})\n\t\t// Nothing available.\n\t\tif total == 0 {\n\t\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t\t}\n\t\t// We can skip ahead.\n\t\tif mb := fs.bim[blkStart]; mb != nil {\n\t\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t\tif fseq > start {\n\t\t\t\tstart = fseq\n\t\t\t}\n\t\t}\n\t}\n\n\tif bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 {\n\t\tfor i := bi; i < len(fs.blks); i++ {\n\t\t\tmb := fs.blks[i]\n\t\t\tif sm, expireOk, err := mb.firstMatchingMulti(sl, start, smp); err == nil {\n\t\t\t\tif expireOk {\n\t\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\t}\n\t\t\t\treturn sm, sm.seq, nil\n\t\t\t} else if err != ErrStoreMsgNotFound {\n\t\t\t\treturn nil, 0, err\n\t\t\t} else {\n\t\t\t\t// Nothing found in this block. We missed, if first block (bi) check psim.\n\t\t\t\t// Similar to above if start <= first seq.\n\t\t\t\t// TODO(dlc) - For v2 track these by filter subject since they will represent filtered consumers.\n\t\t\t\t// We should not do this at all if we are already on the last block.\n\t\t\t\tif i == bi && i < len(fs.blks)-1 {\n\t\t\t\t\tnbi, err := fs.checkSkipFirstBlockMulti(sl, bi)\n\t\t\t\t\t// Nothing available.\n\t\t\t\t\tif err == ErrStoreEOF {\n\t\t\t\t\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t\t\t\t\t}\n\t\t\t\t\t// See if we can jump ahead here.\n\t\t\t\t\t// Right now we can only spin on first, so if we have interior sparseness need to favor checking per block fss if loaded.\n\t\t\t\t\t// For v2 will track all blocks that have matches for psim.\n\t\t\t\t\tif nbi > i {\n\t\t\t\t\t\ti = nbi - 1 // For the iterator condition i++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Check if we can expire.\n\t\t\t\tif expireOk {\n\t\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\n}\n\nfunc (fs *fileStore) LoadNextMsg(filter string, wc bool, start uint64, sm *StoreMsg) (*StoreMsg, uint64, error) {\n\tif fs.isClosed() {\n\t\treturn nil, 0, ErrStoreClosed\n\t}\n\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || start > fs.state.LastSeq {\n\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t}\n\tif start < fs.state.FirstSeq {\n\t\tstart = fs.state.FirstSeq\n\t}\n\n\t// If start is less than or equal to beginning of our stream, meaning our first call,\n\t// let's check the psim to see if we can skip ahead.\n\tif start <= fs.state.FirstSeq {\n\t\tvar ss SimpleState\n\t\tif err := fs.numFilteredPendingNoLast(filter, &ss); err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\t// Nothing available.\n\t\tif ss.Msgs == 0 {\n\t\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t\t}\n\t\t// We can skip ahead.\n\t\tif ss.First > start {\n\t\t\tstart = ss.First\n\t\t}\n\t}\n\n\tif bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 {\n\t\tfor i := bi; i < len(fs.blks); i++ {\n\t\t\tmb := fs.blks[i]\n\t\t\tif sm, expireOk, err := mb.firstMatching(filter, wc, start, sm); err == nil {\n\t\t\t\tif expireOk {\n\t\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\t}\n\t\t\t\treturn sm, sm.seq, nil\n\t\t\t} else if err != ErrStoreMsgNotFound {\n\t\t\t\treturn nil, 0, err\n\t\t\t} else {\n\t\t\t\t// Nothing found in this block. We missed, if first block (bi) check psim.\n\t\t\t\t// Similar to above if start <= first seq.\n\t\t\t\t// TODO(dlc) - For v2 track these by filter subject since they will represent filtered consumers.\n\t\t\t\t// We should not do this at all if we are already on the last block.\n\t\t\t\t// Also if we are a wildcard do not check if large subject space.\n\t\t\t\tconst wcMaxSizeToCheck = 64 * 1024\n\t\t\t\tif i == bi && i < len(fs.blks)-1 && (!wc || fs.psim.Size() < wcMaxSizeToCheck) {\n\t\t\t\t\tnbi, err := fs.checkSkipFirstBlock(filter, wc, bi)\n\t\t\t\t\t// Nothing available.\n\t\t\t\t\tif err == ErrStoreEOF {\n\t\t\t\t\t\treturn nil, fs.state.LastSeq, ErrStoreEOF\n\t\t\t\t\t}\n\t\t\t\t\t// See if we can jump ahead here.\n\t\t\t\t\t// Right now we can only spin on first, so if we have interior sparseness need to favor checking per block fss if loaded.\n\t\t\t\t\t// For v2 will track all blocks that have matches for psim.\n\t\t\t\t\tif nbi > i {\n\t\t\t\t\t\ti = nbi - 1 // For the iterator condition i++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Check if we can expire.\n\t\t\t\tif expireOk {\n\t\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, fs.state.LastSeq, ErrStoreEOF\n}\n\n// Will load the next non-deleted msg starting at the start sequence and walking backwards.\nfunc (fs *fileStore) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error) {\n\tif fs.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || start < fs.state.FirstSeq {\n\t\treturn nil, ErrStoreEOF\n\t}\n\n\tif start > fs.state.LastSeq {\n\t\tstart = fs.state.LastSeq\n\t}\n\tif smp == nil {\n\t\tsmp = new(StoreMsg)\n\t}\n\n\tif bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 {\n\t\tfor i := bi; i >= 0; i-- {\n\t\t\tmb := fs.blks[i]\n\t\t\tmb.mu.Lock()\n\t\t\t// Need messages loaded from here on out.\n\t\t\tif mb.cacheNotLoaded() {\n\t\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlseq, fseq := atomic.LoadUint64(&mb.last.seq), atomic.LoadUint64(&mb.first.seq)\n\t\t\tif start > lseq {\n\t\t\t\tstart = lseq\n\t\t\t}\n\t\t\tfor seq := start; seq >= fseq; seq-- {\n\t\t\t\tif mb.dmap.Exists(seq) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif sm, err := mb.cacheLookup(seq, smp); err == nil {\n\t\t\t\t\tmb.finishedWithCache()\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\treturn sm, nil\n\t\t\t\t}\n\t\t\t}\n\t\t\tmb.finishedWithCache()\n\t\t\tmb.mu.Unlock()\n\t\t}\n\t}\n\n\treturn nil, ErrStoreEOF\n}\n\n// LoadPrevMsgMulti will find the previous message matching any entry in the sublist.\nfunc (fs *fileStore) LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) {\n\tif fs.isClosed() {\n\t\treturn nil, 0, ErrStoreClosed\n\t}\n\n\tif sl == nil {\n\t\tsm, err = fs.LoadPrevMsg(start, smp)\n\t\treturn\n\t}\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tif fs.state.Msgs == 0 || start < fs.state.FirstSeq {\n\t\treturn nil, fs.state.FirstSeq, ErrStoreEOF\n\t}\n\tif start > fs.state.LastSeq {\n\t\tstart = fs.state.LastSeq\n\t}\n\n\tif bi, _ := fs.selectMsgBlockWithIndex(start); bi >= 0 {\n\t\tfor i := bi; i >= 0; i-- {\n\t\t\tmb := fs.blks[i]\n\t\t\tif sm, expireOk, err := mb.prevMatchingMulti(sl, start, smp); err == nil {\n\t\t\t\tif expireOk {\n\t\t\t\t\tmb.tryForceExpireCache()\n\t\t\t\t}\n\t\t\t\treturn sm, sm.seq, nil\n\t\t\t} else if err != ErrStoreMsgNotFound {\n\t\t\t\treturn nil, 0, err\n\t\t\t} else if expireOk {\n\t\t\t\tmb.tryForceExpireCache()\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, fs.state.FirstSeq, ErrStoreEOF\n}\n\n// Type returns the type of the underlying store.\nfunc (fs *fileStore) Type() StorageType {\n\treturn FileStorage\n}\n\n// Returns number of subjects in this store.\n// Lock should be held.\nfunc (fs *fileStore) numSubjects() int {\n\treturn fs.psim.Size()\n}\n\n// numConsumers uses new lock.\nfunc (fs *fileStore) numConsumers() int {\n\tfs.cmu.RLock()\n\tdefer fs.cmu.RUnlock()\n\treturn len(fs.cfs)\n}\n\n// FastState will fill in state with only the following.\n// Msgs, Bytes, First and Last Sequence and Time and NumDeleted.\nfunc (fs *fileStore) FastState(state *StreamState) {\n\tfs.mu.RLock()\n\tstate.Msgs = fs.state.Msgs\n\tstate.Bytes = fs.state.Bytes\n\tstate.FirstSeq = fs.state.FirstSeq\n\tstate.FirstTime = fs.state.FirstTime\n\tstate.LastSeq = fs.state.LastSeq\n\tstate.LastTime = fs.state.LastTime\n\t// Make sure to reset if being re-used.\n\tstate.Deleted, state.NumDeleted = nil, 0\n\tif state.LastSeq > state.FirstSeq {\n\t\tstate.NumDeleted = int((state.LastSeq - state.FirstSeq + 1) - state.Msgs)\n\t\tif state.NumDeleted < 0 {\n\t\t\tstate.NumDeleted = 0\n\t\t}\n\t}\n\tstate.Consumers = fs.numConsumers()\n\tstate.NumSubjects = fs.numSubjects()\n\tfs.mu.RUnlock()\n}\n\n// State returns the current state of the stream.\nfunc (fs *fileStore) State() StreamState {\n\tfs.mu.RLock()\n\tstate := fs.state\n\tstate.Consumers = fs.numConsumers()\n\tstate.NumSubjects = fs.numSubjects()\n\tstate.Deleted = nil // make sure.\n\n\tif numDeleted := int((state.LastSeq - state.FirstSeq + 1) - state.Msgs); numDeleted > 0 {\n\t\tstate.Deleted = make([]uint64, 0, numDeleted)\n\t\tcur := fs.state.FirstSeq\n\n\t\tfor _, mb := range fs.blks {\n\t\t\tmb.mu.Lock()\n\t\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\t\t// Account for messages missing from the head.\n\t\t\tif fseq > cur {\n\t\t\t\tfor seq := cur; seq < fseq; seq++ {\n\t\t\t\t\tstate.Deleted = append(state.Deleted, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Only advance cur if we are increasing. We could have marker blocks with just tombstones.\n\t\t\tif last := atomic.LoadUint64(&mb.last.seq); last >= cur {\n\t\t\t\tcur = last + 1 // Expected next first.\n\t\t\t}\n\t\t\t// Add in deleted.\n\t\t\tmb.dmap.Range(func(seq uint64) bool {\n\t\t\t\tstate.Deleted = append(state.Deleted, seq)\n\t\t\t\treturn true\n\t\t\t})\n\t\t\tmb.mu.Unlock()\n\t\t}\n\t}\n\tfs.mu.RUnlock()\n\n\tstate.Lost = fs.lostData()\n\n\t// Can not be guaranteed to be sorted.\n\tif len(state.Deleted) > 0 {\n\t\tslices.Sort(state.Deleted)\n\t\tstate.NumDeleted = len(state.Deleted)\n\t}\n\treturn state\n}\n\nfunc (fs *fileStore) Utilization() (total, reported uint64, err error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\treported += mb.bytes\n\t\ttotal += mb.rbytes\n\t\tmb.mu.RUnlock()\n\t}\n\treturn total, reported, nil\n}\n\nfunc fileStoreMsgSizeRaw(slen, hlen, mlen int) uint64 {\n\tif hlen == 0 {\n\t\t// length of the message record (4bytes) + seq(8) + ts(8) + subj_len(2) + subj + msg + hash(8)\n\t\treturn uint64(22 + slen + mlen + 8)\n\t}\n\t// length of the message record (4bytes) + seq(8) + ts(8) + subj_len(2) + subj + hdr_len(4) + hdr + msg + hash(8)\n\treturn uint64(22 + slen + 4 + hlen + mlen + 8)\n}\n\nfunc fileStoreMsgSize(subj string, hdr, msg []byte) uint64 {\n\treturn fileStoreMsgSizeRaw(len(subj), len(hdr), len(msg))\n}\n\nfunc fileStoreMsgSizeEstimate(slen, maxPayload int) uint64 {\n\treturn uint64(emptyRecordLen + slen + 4 + maxPayload)\n}\n\n// ResetState resets any state that's temporary. For example when changing leaders.\nfunc (fs *fileStore) ResetState() {\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\tif fs.scheduling != nil {\n\t\tfs.scheduling.clearInflight()\n\t}\n}\n\n// Determine time since any last activity, read/load, write or remove.\nfunc (mb *msgBlock) sinceLastActivity() time.Duration {\n\tif mb.closed {\n\t\treturn 0\n\t}\n\tlast := mb.lwts\n\tif mb.lrts > last {\n\t\tlast = mb.lrts\n\t}\n\tif mb.llts > last {\n\t\tlast = mb.llts\n\t}\n\tif mb.lsts > last {\n\t\tlast = mb.lsts\n\t}\n\treturn time.Since(time.Unix(0, last).UTC())\n}\n\n// Determine time since last write or remove of a message.\n// Read lock should be held.\nfunc (mb *msgBlock) sinceLastWriteActivity() time.Duration {\n\tif mb.closed {\n\t\treturn 0\n\t}\n\tlast := mb.lwts\n\tif mb.lrts > last {\n\t\tlast = mb.lrts\n\t}\n\treturn time.Since(time.Unix(0, last).UTC())\n}\n\nfunc checkNewHeader(hdr []byte) error {\n\tif len(hdr) < 2 || hdr[0] != magic ||\n\t\t(hdr[1] != version && hdr[1] != newVersion) {\n\t\treturn errCorruptState\n\t}\n\treturn nil\n}\n\n// readIndexInfo will read in the index information for the message block.\nfunc (mb *msgBlock) readIndexInfo() error {\n\tifn := filepath.Join(mb.fs.fcfg.StoreDir, msgDir, fmt.Sprintf(indexScan, mb.index))\n\tbuf, err := os.ReadFile(ifn)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Set if first time.\n\tif mb.liwsz == 0 {\n\t\tmb.liwsz = int64(len(buf))\n\t}\n\n\t// Decrypt if needed.\n\tif mb.aek != nil {\n\t\tbuf, err = mb.aek.Open(buf[:0], mb.nonce, buf, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := checkNewHeader(buf); err != nil {\n\t\tdefer os.Remove(ifn)\n\t\treturn fmt.Errorf(\"bad index file\")\n\t}\n\n\tbi := hdrLen\n\n\t// Helpers, will set i to -1 on error.\n\treadSeq := func() uint64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tseq, n := binary.Uvarint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn 0\n\t\t}\n\t\tbi += n\n\t\treturn seq &^ ebit\n\t}\n\treadCount := readSeq\n\treadTimeStamp := func() int64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tts, n := binary.Varint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn -1\n\t\t}\n\t\tbi += n\n\t\treturn ts\n\t}\n\tmb.msgs = readCount()\n\tmb.bytes = readCount()\n\tatomic.StoreUint64(&mb.first.seq, readSeq())\n\tmb.first.ts = readTimeStamp()\n\tatomic.StoreUint64(&mb.last.seq, readSeq())\n\tmb.last.ts = readTimeStamp()\n\tdmapLen := readCount()\n\n\t// Check if this is a short write index file.\n\tif bi < 0 || bi+checksumSize > len(buf) {\n\t\tos.Remove(ifn)\n\t\treturn fmt.Errorf(\"short index file\")\n\t}\n\n\t// Check for consistency if accounting. If something is off bail and we will rebuild.\n\tif mb.msgs != (atomic.LoadUint64(&mb.last.seq)-atomic.LoadUint64(&mb.first.seq)+1)-dmapLen {\n\t\tos.Remove(ifn)\n\t\treturn fmt.Errorf(\"accounting inconsistent\")\n\t}\n\n\t// Checksum\n\tcopy(mb.lchk[0:], buf[bi:bi+checksumSize])\n\tbi += checksumSize\n\n\t// Now check for presence of a delete map\n\tif dmapLen > 0 {\n\t\t// New version is encoded avl seqset.\n\t\tif buf[1] == newVersion {\n\t\t\tdmap, _, err := avl.Decode(buf[bi:])\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"could not decode avl dmap: %v\", err)\n\t\t\t}\n\t\t\tmb.dmap = *dmap\n\t\t} else {\n\t\t\t// This is the old version.\n\t\t\tfor i, fseq := 0, atomic.LoadUint64(&mb.first.seq); i < int(dmapLen); i++ {\n\t\t\t\tseq := readSeq()\n\t\t\t\tif seq == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tmb.dmap.Insert(seq + fseq)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Will return total number of cache loads.\nfunc (fs *fileStore) cacheLoads() uint64 {\n\tvar tl uint64\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\ttl += mb.cloads\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\treturn tl\n}\n\n// Will return total number of cached bytes.\nfunc (fs *fileStore) cacheSize() uint64 {\n\tvar sz uint64\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tvar needsCleanup bool\n\t\tif mb.cache == nil {\n\t\t\tmb.cache = mb.ecache.Value()\n\t\t\tneedsCleanup = mb.cache != nil\n\t\t}\n\t\tif mb.cache != nil {\n\t\t\tsz += uint64(len(mb.cache.buf))\n\t\t}\n\t\tif needsCleanup {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\treturn sz\n}\n\n// Will return total number of dmapEntries for all msg blocks.\nfunc (fs *fileStore) dmapEntries() int {\n\tvar total int\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\ttotal += mb.dmap.Size()\n\t}\n\tfs.mu.RUnlock()\n\treturn total\n}\n\n// Fixed helper for iterating.\nfunc subjectsEqual(a, b string) bool {\n\treturn a == b\n}\n\nfunc subjectsAll(a, b string) bool {\n\treturn true\n}\n\nfunc compareFn(subject string) func(string, string) bool {\n\tif subject == _EMPTY_ || subject == fwcs {\n\t\treturn subjectsAll\n\t}\n\tif subjectHasWildcard(subject) {\n\t\treturn subjectIsSubsetMatch\n\t}\n\treturn subjectsEqual\n}\n\n// PurgeEx will remove messages based on subject filters, sequence and number of messages to keep.\n// Will return the number of purged messages.\nfunc (fs *fileStore) PurgeEx(subject string, sequence, keep uint64) (purged uint64, err error) {\n\tif subject == _EMPTY_ || subject == fwcs {\n\t\tif keep == 0 && sequence == 0 {\n\t\t\treturn fs.purge(0)\n\t\t}\n\t\tif sequence > 1 {\n\t\t\treturn fs.compact(sequence)\n\t\t}\n\t}\n\n\t// Persist any write errors.\n\tdefer func() {\n\t\tif err != nil {\n\t\t\tfs.mu.Lock()\n\t\t\tfs.setWriteErr(err)\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}()\n\n\t// Make sure to not leave subject if empty and we reach this spot.\n\tif subject == _EMPTY_ {\n\t\tsubject = fwcs\n\t}\n\n\teq, wc := compareFn(subject), subjectHasWildcard(subject)\n\tvar firstSeqNeedsUpdate bool\n\tvar bytes uint64\n\n\t// If we have a \"keep\" designation need to get full filtered state so we know how many to purge.\n\tvar maxp uint64\n\tif keep > 0 {\n\t\tss, err := fs.FilteredState(1, subject)\n\t\tif err != nil || keep >= ss.Msgs {\n\t\t\treturn 0, err\n\t\t}\n\t\tmaxp = ss.Msgs - keep\n\t}\n\n\tvar smv StoreMsg\n\tvar tombs []msgId\n\tvar lowSeq uint64\n\n\tif fs.isClosed() {\n\t\treturn purged, ErrStoreClosed\n\t}\n\tfs.mu.Lock()\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\t// We may remove blocks as we purge, so don't range directly on fs.blks\n\t// otherwise we may jump over some (see https://github.com/nats-io/nats-server/issues/3528)\n\tfor i := 0; i < len(fs.blks); i++ {\n\t\tmb := fs.blks[i]\n\t\tmb.mu.Lock()\n\n\t\t// If we do not have our fss, try to expire the cache if we have no items in this block.\n\t\tshouldExpire := mb.fssNotLoaded()\n\n\t\tt, f, l, err := mb.filteredPendingLocked(subject, wc, atomic.LoadUint64(&mb.first.seq))\n\t\tif err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t\tif t == 0 {\n\t\t\t// Expire if we were responsible for loading.\n\t\t\tif shouldExpire {\n\t\t\t\t// Expire this cache before moving on.\n\t\t\t\tmb.tryForceExpireCacheLocked()\n\t\t\t}\n\t\t\tmb.mu.Unlock()\n\t\t\tcontinue\n\t\t}\n\n\t\tif sequence > 1 && sequence <= l {\n\t\t\tl = sequence - 1\n\t\t}\n\n\t\tif mb.cacheNotLoaded() {\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\tshouldExpire = true\n\t\t}\n\n\t\tvar nrg uint64 // Number of remaining messages globally after removal from psim.\n\n\t\tfor seq, te := f, len(tombs); seq <= l; seq++ {\n\t\t\tif sm, _ := mb.cacheLookupNoCopy(seq, &smv); sm != nil && eq(sm.subj, subject) {\n\t\t\t\trl := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t\t\t// Do fast in place remove.\n\t\t\t\t// Stats\n\t\t\t\tif mb.msgs > 0 {\n\t\t\t\t\t// Msgs\n\t\t\t\t\tfs.state.Msgs--\n\t\t\t\t\tmb.msgs--\n\t\t\t\t\t// Bytes, make sure to not go negative.\n\t\t\t\t\tif rl > fs.state.Bytes {\n\t\t\t\t\t\trl = fs.state.Bytes\n\t\t\t\t\t}\n\t\t\t\t\tif rl > mb.bytes {\n\t\t\t\t\t\trl = mb.bytes\n\t\t\t\t\t}\n\t\t\t\t\tfs.state.Bytes -= rl\n\t\t\t\t\tmb.bytes -= rl\n\t\t\t\t\t// Totals\n\t\t\t\t\tpurged++\n\t\t\t\t\tbytes += rl\n\t\t\t\t}\n\t\t\t\t// PSIM and FSS updates.\n\t\t\t\tnr, err := mb.removeSeqPerSubject(sm.subj, seq)\n\t\t\t\tif err != nil {\n\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\tfs.mu.Unlock()\n\t\t\t\t\treturn purged, err\n\t\t\t\t}\n\t\t\t\tnrg = fs.removePerSubject(sm.subj)\n\n\t\t\t\t// Track tombstones we need to write.\n\t\t\t\ttombs = append(tombs, msgId{sm.seq, sm.ts})\n\t\t\t\tif sm.seq < lowSeq || lowSeq == 0 {\n\t\t\t\t\tlowSeq = sm.seq\n\t\t\t\t}\n\n\t\t\t\t// Check for first message.\n\t\t\t\tif seq == atomic.LoadUint64(&mb.first.seq) {\n\t\t\t\t\tmb.selectNextFirst()\n\t\t\t\t\tif mb.isEmpty() {\n\t\t\t\t\t\t// Since we are removing this block don't need to write tombstones.\n\t\t\t\t\t\ttombs = tombs[:te]\n\t\t\t\t\t\tif err = fs.removeMsgBlock(mb); err != nil {\n\t\t\t\t\t\t\tmb.mu.Unlock()\n\t\t\t\t\t\t\tfs.mu.Unlock()\n\t\t\t\t\t\t\treturn 0, err\n\t\t\t\t\t\t}\n\t\t\t\t\t\ti--\n\t\t\t\t\t\t// keep flag set, if set previously\n\t\t\t\t\t\tfirstSeqNeedsUpdate = firstSeqNeedsUpdate || seq == fs.state.FirstSeq\n\t\t\t\t\t} else if seq == fs.state.FirstSeq {\n\t\t\t\t\t\tfs.state.FirstSeq = atomic.LoadUint64(&mb.first.seq) // new one.\n\t\t\t\t\t\tif mb.first.ts == 0 {\n\t\t\t\t\t\t\tfs.state.FirstTime = time.Time{}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfs.state.FirstTime = time.Unix(0, mb.first.ts).UTC()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// Out of order delete.\n\t\t\t\t\tmb.dmap.Insert(seq)\n\t\t\t\t}\n\t\t\t\t// Break if we have emptied this block or if we set a maximum purge count.\n\t\t\t\tif mb.isEmpty() || (maxp > 0 && purged >= maxp) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// Also break if we know we have no more messages matching here.\n\t\t\t\t// This is only applicable for non-wildcarded filters.\n\t\t\t\tif !wc && nr == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Expire if we were responsible for loading and we do not seem to be doing successive purgeEx calls.\n\t\t// On successive calls - most likely from KV purge deletes, we want to keep the data loaded.\n\t\tif shouldExpire && time.Since(fs.lpex) > time.Second {\n\t\t\t// Expire this cache before moving on.\n\t\t\tmb.tryForceExpireCacheLocked()\n\t\t} else {\n\t\t\tmb.finishedWithCache()\n\t\t}\n\t\tmb.mu.Unlock()\n\n\t\t// Check if we should break out of top level too.\n\t\tif maxp > 0 && purged >= maxp {\n\t\t\tbreak\n\t\t}\n\t\t// Also check if not wildcarded and we have no remaining matches.\n\t\tif !wc && nrg == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\tif firstSeqNeedsUpdate {\n\t\tif err = fs.selectNextFirst(); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t}\n\n\t// Update the last purgeEx call time.\n\tdefer func() { fs.lpex = time.Now() }()\n\n\t// Write any tombstones as needed.\n\t// When writing multiple tombstones we will flush at the end.\n\tif len(tombs) > 0 {\n\t\tfor _, tomb := range tombs {\n\t\t\tif err = fs.writeTombstoneNoFlush(tomb.seq, tomb.ts); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t}\n\t\t// Flush any pending. If we change blocks the newMsgBlockForWrite() will flush any pending for us.\n\t\tif lmb := fs.lmb; lmb != nil {\n\t\t\tif err = lmb.flushPendingMsgs(); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t}\n\t}\n\n\tfs.dirty++\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\tif cb != nil {\n\t\tif purged == 1 {\n\t\t\tcb(-int64(purged), -int64(bytes), lowSeq, subject)\n\t\t} else {\n\t\t\t// FIXME(dlc) - Since we track lowSeq we could send to upper layer if they dealt with the condition properly.\n\t\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t\t}\n\t}\n\n\treturn purged, nil\n}\n\n// Purge will remove all messages from this store.\n// Will return the number of purged messages.\nfunc (fs *fileStore) Purge() (uint64, error) {\n\treturn fs.purge(0)\n}\n\nfunc (fs *fileStore) purge(fseq uint64) (purged uint64, rerr error) {\n\tif fs.isClosed() {\n\t\treturn 0, ErrStoreClosed\n\t}\n\n\t// Persist any write errors.\n\tdefer func() {\n\t\tif rerr != nil {\n\t\t\tfs.mu.Lock()\n\t\t\tfs.setWriteErr(rerr)\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}()\n\n\tfs.mu.Lock()\n\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn 0, err\n\t}\n\n\tpurged = fs.state.Msgs\n\trbytes := int64(fs.state.Bytes)\n\n\tfs.state.FirstSeq = fs.state.LastSeq + 1\n\tfs.state.FirstTime = time.Time{}\n\n\tfs.state.Bytes = 0\n\tfs.state.Msgs = 0\n\n\tfor _, mb := range fs.blks {\n\t\tmb.dirtyClose()\n\t}\n\n\t// Check if we need to set the first seq to a new number.\n\tif fseq > fs.state.FirstSeq {\n\t\tfs.state.FirstSeq = fseq\n\t\tfs.state.LastSeq = fseq - 1\n\t}\n\n\t// Make sure we have a lmb to write to.\n\tif _, err := fs.newMsgBlockForWrite(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\n\tlmb := fs.lmb\n\tatomic.StoreUint64(&lmb.first.seq, fs.state.FirstSeq)\n\tatomic.StoreUint64(&lmb.last.seq, fs.state.LastSeq)\n\tlmb.last.ts = fs.state.LastTime.UnixNano()\n\n\tif lseq := atomic.LoadUint64(&lmb.last.seq); lseq > 0 {\n\t\t// Leave a tombstone so we can remember our starting sequence in case\n\t\t// full state becomes corrupted.\n\t\tif err := fs.writeTombstone(lseq, lmb.last.ts); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t}\n\t// Close FDs since we'll move the file. We re-enable the FD after the purge is complete.\n\tif err := lmb.flushPendingMsgs(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\tif err := lmb.closeFDs(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\n\tfs.blks = nil\n\tfs.lmb = nil\n\tfs.bim = make(map[uint32]*msgBlock)\n\t// Clear any per subject tracking.\n\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\tfs.sdm.empty()\n\t// Mark dirty.\n\tfs.dirty++\n\tfs.addMsgBlock(lmb)\n\n\t// Move the msgs directory out of the way, will delete out of band.\n\t// FIXME(dlc) - These can error and we need to change api above to propagate?\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tndir := filepath.Join(fs.fcfg.StoreDir, newMsgDir)\n\tpdir := filepath.Join(fs.fcfg.StoreDir, purgeDir)\n\t<-dios\n\t// If purge directory still exists then we need to wait\n\t// in place and remove since rename would fail.\n\tif _, err := os.Stat(ndir); err == nil {\n\t\tif err = os.RemoveAll(ndir); err != nil {\n\t\t\tdios <- struct{}{}\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t} else if !os.IsNotExist(err) {\n\t\tdios <- struct{}{}\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\tif _, err := os.Stat(pdir); err == nil {\n\t\tif err = os.RemoveAll(pdir); err != nil {\n\t\t\tdios <- struct{}{}\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t} else if !os.IsNotExist(err) {\n\t\tdios <- struct{}{}\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\n\t// Create directory to move the new tombstone to.\n\tif err := os.MkdirAll(ndir, defaultDirPerms); err != nil {\n\t\tdios <- struct{}{}\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\t// Move out the block containing the tombstone. Also move the key file if encrypted.\n\t// The block file itself MUST be moved last to ensure we can assume the prior renames\n\t// were successful during recovery.\n\tfor _, mbf := range []string{fmt.Sprintf(keyScan, lmb.index), fmt.Sprintf(blkScan, lmb.index)} {\n\t\tb := filepath.Join(mdir, mbf)\n\t\ta := filepath.Join(ndir, mbf)\n\t\tif err := os.Rename(b, a); err != nil && !os.IsNotExist(err) {\n\t\t\tdios <- struct{}{}\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t}\n\t// Purge all remaining messages.\n\tif err := os.Rename(mdir, pdir); err != nil {\n\t\tdios <- struct{}{}\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\t// Rename the directory back to be left only with the tombstone.\n\tif err := os.Rename(ndir, mdir); err != nil {\n\t\tdios <- struct{}{}\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\tdios <- struct{}{}\n\n\t// Remove the purged messages directory asynchronously.\n\tgo func() {\n\t\t<-dios\n\t\t_ = os.RemoveAll(pdir)\n\t\tdios <- struct{}{}\n\t}()\n\n\t// Re-enable writing for the lmb.\n\tlmb.mu.Lock()\n\terr := lmb.enableForWriting(fs.fip)\n\tlmb.mu.Unlock()\n\tif err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn purged, err\n\t}\n\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\t// Force a new index.db to be written.\n\tif purged > 0 {\n\t\tfs.forceWriteFullState()\n\t}\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -rbytes, 0, _EMPTY_)\n\t}\n\n\treturn purged, nil\n}\n\n// Lock and dios should be held.\nfunc (fs *fileStore) recoverPartialPurge() error {\n\tndir := filepath.Join(fs.fcfg.StoreDir, newMsgDir)\n\tif entries, err := os.ReadDir(ndir); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t} else if err == nil {\n\t\t// If it's empty, that means we got hard killed before moving the tombstone, and we need to remove it.\n\t\tisEmpty := !slices.ContainsFunc(entries, func(e os.DirEntry) bool {\n\t\t\t// The directory is considered empty if it's missing a block file.\n\t\t\treturn strings.HasSuffix(e.Name(), blkSuffix)\n\t\t})\n\t\tif isEmpty {\n\t\t\t_ = os.RemoveAll(ndir)\n\t\t} else {\n\t\t\t// Otherwise, it contains the tombstone after purge, and the old messages need to be purged instead.\n\t\t\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\t\t\tif err = os.RemoveAll(mdir); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err = os.Rename(ndir, mdir); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\tpdir := filepath.Join(fs.fcfg.StoreDir, purgeDir)\n\tif _, err := os.Stat(pdir); err == nil {\n\t\t_ = os.RemoveAll(pdir)\n\t}\n\treturn nil\n}\n\n// Compact will remove all messages from this store up to\n// but not including the seq parameter.\n// Will return the number of purged messages.\nfunc (fs *fileStore) Compact(seq uint64) (uint64, error) {\n\treturn fs.compact(seq)\n}\n\nfunc (fs *fileStore) compact(seq uint64) (purged uint64, rerr error) {\n\tif fs.isClosed() {\n\t\treturn 0, ErrStoreClosed\n\t}\n\tif seq == 0 {\n\t\treturn fs.purge(seq)\n\t}\n\n\tfs.mu.Lock()\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn 0, err\n\t}\n\t// Same as purge all.\n\tif lseq := fs.state.LastSeq; seq > lseq {\n\t\tfs.mu.Unlock()\n\t\treturn fs.purge(seq)\n\t}\n\t// Short-circuit if the store was already compacted past this point.\n\tif fs.state.FirstSeq > seq {\n\t\tfs.mu.Unlock()\n\t\treturn purged, nil\n\t}\n\t// We have to delete interior messages.\n\tsmb := fs.selectMsgBlock(seq)\n\tif smb == nil {\n\t\tfs.mu.Unlock()\n\t\treturn 0, nil\n\t}\n\n\t// Persist any write errors.\n\tdefer func() {\n\t\tif rerr != nil {\n\t\t\tfs.mu.Lock()\n\t\t\tfs.setWriteErr(rerr)\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}()\n\n\tvar bytes uint64\n\n\t// All msgblocks up to this one can be thrown away.\n\tvar deleted int\n\tfor _, mb := range fs.blks {\n\t\tif mb == smb {\n\t\t\tbreak\n\t\t}\n\t\tmb.mu.Lock()\n\t\tpurged += mb.msgs\n\t\tbytes += mb.bytes\n\t\t// Make sure we do subject cleanup as well.\n\t\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\treturn 0, err\n\t\t}\n\t\tmb.fss.IterOrdered(func(bsubj []byte, ss *SimpleState) bool {\n\t\t\tsubj := bytesToString(bsubj)\n\t\t\tfor i := uint64(0); i < ss.Msgs; i++ {\n\t\t\t\tfs.removePerSubject(subj)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\t// Now close.\n\t\terr := mb.dirtyCloseWithRemove(true)\n\t\tmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t\tdeleted++\n\t}\n\n\tvar smv StoreMsg\n\tvar err error\n\tvar tombs []msgId\n\n\tsmb.mu.Lock()\n\tif atomic.LoadUint64(&smb.first.seq) == seq {\n\t\tfs.state.FirstSeq = atomic.LoadUint64(&smb.first.seq)\n\t\tfs.state.FirstTime = time.Unix(0, smb.first.ts).UTC()\n\t\tgoto SKIP\n\t}\n\n\t// Make sure we have the messages loaded.\n\tif smb.cacheNotLoaded() {\n\t\tif err = smb.loadMsgsWithLock(); err != nil {\n\t\t\tsmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\treturn purged, err\n\t\t}\n\t\tdefer func() {\n\t\t\t// The lock is released once we get here, so need to re-acquire.\n\t\t\tsmb.mu.Lock()\n\t\t\tsmb.finishedWithCache()\n\t\t\tsmb.mu.Unlock()\n\t\t}()\n\t}\n\tfor mseq := atomic.LoadUint64(&smb.first.seq); mseq < seq; mseq++ {\n\t\tsm, err := smb.cacheLookupNoCopy(mseq, &smv)\n\t\tif err == errDeletedMsg {\n\t\t\t// Update dmap.\n\t\t\tif !smb.dmap.IsEmpty() {\n\t\t\t\tsmb.dmap.Delete(mseq)\n\t\t\t}\n\t\t} else if sm != nil {\n\t\t\tsz := fileStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t\tif smb.msgs > 0 {\n\t\t\t\tsmb.msgs--\n\t\t\t\tif sz > smb.bytes {\n\t\t\t\t\tsz = smb.bytes\n\t\t\t\t}\n\t\t\t\tsmb.bytes -= sz\n\t\t\t\tbytes += sz\n\t\t\t\tpurged++\n\t\t\t}\n\t\t\t// Update fss\n\t\t\tif _, err := smb.removeSeqPerSubject(sm.subj, mseq); err != nil {\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t\tfs.removePerSubject(sm.subj)\n\t\t\ttombs = append(tombs, msgId{sm.seq, sm.ts})\n\t\t}\n\t}\n\n\t// Check if empty after processing, could happen if tail of messages are all deleted.\n\tif isEmpty := smb.msgs == 0; isEmpty {\n\t\t// Only remove if not the last block.\n\t\tif smb != fs.lmb {\n\t\t\tif err = smb.dirtyCloseWithRemove(true); err != nil {\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t\tdeleted++\n\t\t} else {\n\t\t\t// Make sure to sync changes.\n\t\t\tsmb.needSync = true\n\t\t}\n\t\t// Update fs first here as well.\n\t\tfs.state.FirstSeq = atomic.LoadUint64(&smb.last.seq) + 1\n\t\tfs.state.FirstTime = time.Time{}\n\n\t} else {\n\t\t// Make sure to sync changes.\n\t\tsmb.needSync = true\n\t\t// Just for start condition for selectNextFirst.\n\t\tif smb.first.seq < seq {\n\t\t\tatomic.StoreUint64(&smb.first.seq, seq-1)\n\t\t} else {\n\t\t\t// selectNextFirst always adds 1, so need to subtract 1 here.\n\t\t\tatomic.StoreUint64(&smb.first.seq, smb.first.seq-1)\n\t\t}\n\t\tsmb.selectNextFirst()\n\n\t\tfs.state.FirstSeq = atomic.LoadUint64(&smb.first.seq)\n\t\tfs.state.FirstTime = time.Unix(0, smb.first.ts).UTC()\n\n\t\t// Check if we should reclaim the head space from this block.\n\t\t// This will be optimistic only, so don't continue if we encounter any errors here.\n\t\tif smb.rbytes > compactMinimum && smb.bytes*2 < smb.rbytes {\n\t\t\tvar moff uint32\n\t\t\tmoff, _, _, err = smb.slotInfo(int(atomic.LoadUint64(&smb.first.seq) - smb.cache.fseq))\n\t\t\tif err != nil {\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t} else if moff >= uint32(len(smb.cache.buf)) {\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\tbuf := smb.cache.buf[moff:]\n\t\t\t// Don't reuse, copy to new recycled buf.\n\t\t\tnbuf := getMsgBlockBuf(len(buf))\n\t\t\tnbuf = append(nbuf, buf...)\n\t\t\tsmb.closeFDsLockedNoCheck()\n\t\t\t// Check for encryption.\n\t\t\tif smb.bek != nil && len(nbuf) > 0 {\n\t\t\t\t// Recreate to reset counter.\n\t\t\t\tbek, err := genBlockEncryptionKey(smb.fs.fcfg.Cipher, smb.seed, smb.nonce)\n\t\t\t\tif err != nil {\n\t\t\t\t\tsmb.mu.Unlock()\n\t\t\t\t\tfs.mu.Unlock()\n\t\t\t\t\treturn purged, err\n\t\t\t\t}\n\t\t\t\t// For future writes make sure to set smb.bek to keep counter correct.\n\t\t\t\tsmb.bek = bek\n\t\t\t\tsmb.bek.XORKeyStream(nbuf, nbuf)\n\t\t\t}\n\t\t\t// Recompress if necessary (smb.cmp contains the algorithm used when\n\t\t\t// the block was loaded from disk, or defaults to NoCompression if not)\n\t\t\tif nbuf, err = smb.cmp.Compress(nbuf); err != nil {\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\n\t\t\t// We will write to a new file and mv/rename it in case of failure.\n\t\t\tmfn := filepath.Join(smb.fs.fcfg.StoreDir, msgDir, fmt.Sprintf(newScan, smb.index))\n\t\t\t<-dios\n\t\t\terr = os.WriteFile(mfn, nbuf, defaultFilePerms)\n\t\t\tdios <- struct{}{}\n\t\t\tif err != nil {\n\t\t\t\t_ = os.Remove(mfn)\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t\tif err = os.Rename(mfn, smb.mfn); err != nil {\n\t\t\t\t_ = os.Remove(mfn)\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\n\t\t\t// Make sure to remove fss state.\n\t\t\tsmb.fss = nil\n\t\t\tsmb.clearCacheAndOffset()\n\t\t\tsmb.rbytes = uint64(len(nbuf))\n\t\t\t// Make sure we don't write any additional tombstones.\n\t\t\ttombs = nil\n\t\t}\n\t}\n\nSKIP:\n\tsmb.mu.Unlock()\n\n\t// Write any tombstones as needed.\n\t// When writing multiple tombstones we will flush at the end.\n\tif len(tombs) > 0 {\n\t\tfor _, tomb := range tombs {\n\t\t\tif err = fs.writeTombstoneNoFlush(tomb.seq, tomb.ts); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t}\n\t\t// Flush any pending. If we change blocks the newMsgBlockForWrite() will flush any pending for us.\n\t\tif lmb := fs.lmb; lmb != nil {\n\t\t\tif err = lmb.flushPendingMsgs(); err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn purged, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif deleted > 0 {\n\t\t// Update block map.\n\t\tif fs.bim != nil {\n\t\t\tfor _, mb := range fs.blks[:deleted] {\n\t\t\t\tdelete(fs.bim, mb.index)\n\t\t\t}\n\t\t}\n\t\t// Update blks slice.\n\t\tfs.blks = copyMsgBlocks(fs.blks[deleted:])\n\t\tif lb := len(fs.blks); lb == 0 {\n\t\t\tfs.lmb = nil\n\t\t} else {\n\t\t\tfs.lmb = fs.blks[lb-1]\n\t\t}\n\t}\n\n\t// Update top level accounting.\n\tif purged > fs.state.Msgs {\n\t\tpurged = fs.state.Msgs\n\t}\n\tfs.state.Msgs -= purged\n\tif fs.state.Msgs == 0 {\n\t\tfs.state.FirstSeq = fs.state.LastSeq + 1\n\t\tfs.state.FirstTime = time.Time{}\n\t}\n\n\tif bytes > fs.state.Bytes {\n\t\tbytes = fs.state.Bytes\n\t}\n\tfs.state.Bytes -= bytes\n\n\t// Any existing state file no longer applicable. We will force write a new one\n\t// after we release the lock.\n\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\tfs.dirty++\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\t// Force a new index.db to be written.\n\tif purged > 0 {\n\t\tfs.forceWriteFullState()\n\t}\n\n\tif cb != nil && purged > 0 {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn purged, err\n}\n\n// Will completely reset our store.\nfunc (fs *fileStore) reset() error {\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\n\tfs.mu.Lock()\n\n\tvar purged, bytes uint64\n\tcb := fs.scb\n\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tpurged += mb.msgs\n\t\tbytes += mb.bytes\n\t\terr := mb.dirtyCloseWithRemove(true)\n\t\tmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Reset\n\tfs.state.FirstSeq = 0\n\tfs.state.FirstTime = time.Time{}\n\tfs.state.LastSeq = 0\n\tfs.state.LastTime = time.Now().UTC()\n\t// Update msgs and bytes.\n\tfs.state.Msgs = 0\n\tfs.state.Bytes = 0\n\n\t// Reset blocks.\n\tfs.blks, fs.lmb = nil, nil\n\n\t// Reset subject mappings.\n\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\tfs.sdm.empty()\n\tfs.bim = make(map[uint32]*msgBlock)\n\n\t// If we purged anything, make sure we kick flush state loop.\n\tif purged > 0 {\n\t\tfs.dirty++\n\t}\n\n\tfs.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn nil\n}\n\n// Return all active tombstones in this msgBlock.\nfunc (mb *msgBlock) tombs() []msgId {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.tombsLocked()\n}\n\n// Return all active tombstones in this msgBlock.\n// Write lock should be held.\nfunc (mb *msgBlock) tombsLocked() []msgId {\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\tdefer mb.finishedWithCache()\n\n\tvar tombs []msgId\n\tvar le = binary.LittleEndian\n\tbuf := mb.cache.buf\n\n\tfor index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; {\n\t\tif index+msgHdrSize > lbuf {\n\t\t\treturn tombs\n\t\t}\n\t\thdr := buf[index : index+msgHdrSize]\n\t\trl, seq := le.Uint32(hdr[0:]), le.Uint64(hdr[4:])\n\t\t// Clear any headers bit that could be set.\n\t\trl &^= hbit\n\t\t// Check for tombstones.\n\t\tif seq&tbit != 0 {\n\t\t\tts := int64(le.Uint64(hdr[12:]))\n\t\t\ttombs = append(tombs, msgId{seq &^ tbit, ts})\n\t\t}\n\t\t// Advance to next record.\n\t\tindex += rl\n\t}\n\n\treturn tombs\n}\n\n// fs lock should be held.\nfunc (mb *msgBlock) numPriorTombs() int {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\treturn mb.numPriorTombsLocked()\n}\n\n// Return number of tombstones for messages prior to this msgBlock.\n// Both locks should be held.\n// Write lock should be held for block.\nfunc (mb *msgBlock) numPriorTombsLocked() int {\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn 0\n\t\t}\n\t}\n\tdefer mb.finishedWithCache()\n\n\tvar fseq uint64\n\tvar tombs int\n\tvar le = binary.LittleEndian\n\tbuf := mb.cache.buf\n\n\tfor index, lbuf := uint32(0), uint32(len(buf)); index < lbuf; {\n\t\tif index+msgHdrSize > lbuf {\n\t\t\treturn tombs\n\t\t}\n\t\thdr := buf[index : index+msgHdrSize]\n\t\trl, seq := le.Uint32(hdr[0:]), le.Uint64(hdr[4:])\n\t\t// Clear any headers bit that could be set.\n\t\trl &^= hbit\n\t\t// Check for tombstones.\n\t\tif seq&tbit != 0 {\n\t\t\tseq = seq &^ tbit\n\t\t\t// Tombstones below the global first seq are irrelevant.\n\t\t\t// And we only count tombstones below this block's first seq.\n\t\t\tif seq >= mb.fs.state.FirstSeq && (fseq == 0 || seq < fseq) {\n\t\t\t\ttombs++\n\t\t\t}\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\t\tif seq == 0 || seq&ebit != 0 {\n\t\t\tindex += rl\n\t\t\tcontinue\n\t\t}\n\t\t// Advance to next record.\n\t\tindex += rl\n\t\tif fseq == 0 {\n\t\t\tfseq = seq\n\t\t}\n\t}\n\n\treturn tombs\n}\n\n// Truncate will truncate a stream store up to seq. Sequence needs to be valid.\nfunc (fs *fileStore) Truncate(seq uint64) (rerr error) {\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\n\t// Check for request to reset.\n\tif seq == 0 {\n\t\treturn fs.reset()\n\t}\n\n\tfs.mu.Lock()\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\n\t// Persist any write errors.\n\tdefer func() {\n\t\tif rerr != nil {\n\t\t\tfs.mu.Lock()\n\t\t\tfs.setWriteErr(rerr)\n\t\t\tfs.mu.Unlock()\n\t\t}\n\t}()\n\n\t// Any existing state file will no longer be applicable. We will force write a new one\n\t// at the end, after we release the lock.\n\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\tvar err error\n\tvar lsm *StoreMsg\n\tsmb := fs.selectMsgBlock(seq)\n\tif smb != nil {\n\t\tlsm, _, err = smb.fetchMsgNoCopy(seq, nil)\n\t\tif err != nil && err != ErrStoreMsgNotFound && err != errDeletedMsg {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Reset last so new block doesn't contain truncated sequences/timestamps.\n\tvar lastTime int64\n\tif lsm != nil {\n\t\tlastTime = lsm.ts\n\t} else if smb != nil {\n\t\tlastTime = smb.last.ts\n\t} else {\n\t\tlastTime = fs.state.LastTime.UnixNano()\n\t}\n\tfs.state.LastSeq = seq\n\tfs.state.LastTime = time.Unix(0, lastTime).UTC()\n\n\t// Always create a new write block for any tombstones.\n\t// We'll truncate the selected message block as the last step, so can't write tombstones to it.\n\t// If we end up not needing to write tombstones, this block will be cleaned up at the end.\n\ttmb, err := fs.newMsgBlockForWrite()\n\tif err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\n\t// The selected message block needs to be removed if it needs to be fully truncated.\n\tvar removeSmb bool\n\tif smb != nil {\n\t\tremoveSmb = atomic.LoadUint64(&smb.first.seq) > seq\n\t}\n\n\t// If the selected block is not found or the message was deleted, we'll need to write a tombstone\n\t// at the truncated sequence so we don't roll backward on our last sequence and timestamp.\n\tif lsm == nil || removeSmb {\n\t\tif err = fs.writeTombstone(seq, lastTime); err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\tvar purged, bytes uint64\n\n\t// Remove any left over msg blocks.\n\tgetLastMsgBlock := func() *msgBlock {\n\t\t// Start at one before last, tmb will be the last most of the time\n\t\t// unless a new block gets added for tombstones.\n\t\tfor i := len(fs.blks) - 2; i >= 0; i-- {\n\t\t\tif mb := fs.blks[i]; mb.index < tmb.index {\n\t\t\t\treturn mb\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\tfor mb := getLastMsgBlock(); mb != nil && mb != smb; mb = getLastMsgBlock() {\n\t\tmb.mu.Lock()\n\t\tpurged += mb.msgs\n\t\tbytes += mb.bytes\n\n\t\t// We could have tombstones for messages before the truncated sequence.\n\t\t// Need to store those for blocks we're about to remove.\n\t\tif tombs := mb.tombsLocked(); len(tombs) > 0 {\n\t\t\t// Temporarily unlock while we write tombstones.\n\t\t\tmb.mu.Unlock()\n\t\t\tfor _, tomb := range tombs {\n\t\t\t\tif tomb.seq < seq {\n\t\t\t\t\tif err = fs.writeTombstone(tomb.seq, tomb.ts); err != nil {\n\t\t\t\t\t\tfs.mu.Unlock()\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tmb.mu.Lock()\n\t\t}\n\t\terr = fs.forceRemoveMsgBlock(mb)\n\t\tmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\thasWrittenTombstones := len(tmb.tombs()) > 0\n\tif smb != nil {\n\t\tsmb.mu.Lock()\n\t\tif removeSmb {\n\t\t\tpurged += smb.msgs\n\t\t\tbytes += smb.bytes\n\n\t\t\t// We could have tombstones for messages before the truncated sequence.\n\t\t\tif tombs := smb.tombsLocked(); len(tombs) > 0 {\n\t\t\t\t// Temporarily unlock while we write tombstones.\n\t\t\t\tsmb.mu.Unlock()\n\t\t\t\tfor _, tomb := range tombs {\n\t\t\t\t\tif tomb.seq < seq {\n\t\t\t\t\t\tif err = fs.writeTombstone(tomb.seq, tomb.ts); err != nil {\n\t\t\t\t\t\t\tfs.mu.Unlock()\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tsmb.mu.Lock()\n\t\t\t}\n\t\t\terr = fs.forceRemoveMsgBlock(smb)\n\t\t\tsmb.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto SKIP\n\t\t}\n\n\t\t// Make sure writeable.\n\t\tif err := smb.enableForWriting(fs.fip); err != nil {\n\t\t\tsmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\n\t\t// Truncate our selected message block.\n\t\tnmsgs, nbytes, err := smb.truncate(seq, lastTime)\n\t\tif err != nil {\n\t\t\tsmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\treturn fmt.Errorf(\"smb.truncate: %w\", err)\n\t\t}\n\t\t// Account for the truncated msgs and bytes.\n\t\tpurged += nmsgs\n\t\tbytes += nbytes\n\n\t\t// The selected message block is not the last anymore, need to close down resources.\n\t\tif hasWrittenTombstones {\n\t\t\t// Quit our loops.\n\t\t\tif smb.qch != nil {\n\t\t\t\tclose(smb.qch)\n\t\t\t\tsmb.qch = nil\n\t\t\t}\n\t\t\tsmb.closeFDsLockedNoCheck()\n\t\t\tsmb.recompressOnDiskIfNeeded()\n\t\t}\n\t\tsmb.mu.Unlock()\n\t}\n\nSKIP:\n\t// If no tombstones were written, we can remove the block and\n\t// purely rely on the selected block as the last block.\n\tif !hasWrittenTombstones {\n\t\tfs.lmb = smb\n\t\ttmb.mu.Lock()\n\t\terr = fs.forceRemoveMsgBlock(tmb)\n\t\ttmb.mu.Unlock()\n\t\tif err != nil {\n\t\t\tfs.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Reset last.\n\tfs.state.LastSeq = seq\n\tfs.state.LastTime = time.Unix(0, lastTime).UTC()\n\t// Update msgs and bytes.\n\tif purged > fs.state.Msgs {\n\t\tpurged = fs.state.Msgs\n\t}\n\tfs.state.Msgs -= purged\n\tif bytes > fs.state.Bytes {\n\t\tbytes = fs.state.Bytes\n\t}\n\tfs.state.Bytes -= bytes\n\n\t// Reset our subject lookup info.\n\tif err = fs.resetGlobalPerSubjectInfo(); err != nil {\n\t\tfs.mu.Unlock()\n\t\treturn err\n\t}\n\n\tfs.dirty++\n\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\t// Force a new index.db to be written.\n\tif purged > 0 {\n\t\tfs.forceWriteFullState()\n\t}\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn nil\n}\n\nfunc (fs *fileStore) lastSeq() uint64 {\n\tfs.mu.RLock()\n\tseq := fs.state.LastSeq\n\tfs.mu.RUnlock()\n\treturn seq\n}\n\n// Returns number of msg blks.\nfunc (fs *fileStore) numMsgBlocks() int {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\treturn len(fs.blks)\n}\n\n// Will add a new msgBlock.\n// Lock should be held.\nfunc (fs *fileStore) addMsgBlock(mb *msgBlock) {\n\tfs.blks = append(fs.blks, mb)\n\tfs.lmb = mb\n\tfs.bim[mb.index] = mb\n}\n\n// Remove from our list of blks.\n// Both locks should be held.\nfunc (fs *fileStore) removeMsgBlockFromList(mb *msgBlock) {\n\t// Remove from list.\n\tfor i, omb := range fs.blks {\n\t\tif mb == omb {\n\t\t\tfs.dirty++\n\t\t\tblks := append(fs.blks[:i], fs.blks[i+1:]...)\n\t\t\tfs.blks = copyMsgBlocks(blks)\n\t\t\tif fs.bim != nil {\n\t\t\t\tdelete(fs.bim, mb.index)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// Removes the msgBlock\n// Both locks should be held.\nfunc (fs *fileStore) removeMsgBlock(mb *msgBlock) error {\n\t// Check for us being last message block\n\tlseq, lts := atomic.LoadUint64(&mb.last.seq), mb.last.ts\n\tif mb == fs.lmb {\n\t\t// Creating a new message write block requires that the lmb lock is not held.\n\t\tmb.mu.Unlock()\n\t\t// Write the tombstone to remember since this was last block.\n\t\tif lmb, err := fs.newMsgBlockForWrite(); err != nil || lmb == nil {\n\t\t\tif err != nil {\n\t\t\t\terr = errors.New(\"lmb missing\")\n\t\t\t}\n\t\t\t// Re-acquire mb lock\n\t\t\tmb.mu.Lock()\n\t\t\treturn err\n\t\t} else if err = fs.writeTombstone(lseq, lts); err != nil {\n\t\t\t// Re-acquire mb lock\n\t\t\tmb.mu.Lock()\n\t\t\treturn err\n\t\t}\n\t\tmb.mu.Lock()\n\t} else if lseq == fs.state.LastSeq {\n\t\t// Need to write a tombstone for the last sequence if we're removing the block containing it.\n\t\tif err := fs.writeTombstone(lseq, lts); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Only delete message block after (potentially) writing a tombstone.\n\t// But only if it doesn't contain any tombstones for prior blocks.\n\tif mb.numPriorTombsLocked() > 0 {\n\t\treturn nil\n\t}\n\treturn fs.forceRemoveMsgBlock(mb)\n}\n\n// Removes the msgBlock, without writing tombstones to ensure the last sequence is preserved.\n// Both locks should be held.\nfunc (fs *fileStore) forceRemoveMsgBlock(mb *msgBlock) error {\n\tif err := mb.dirtyCloseWithRemove(true); err != nil {\n\t\treturn err\n\t}\n\tfs.removeMsgBlockFromList(mb)\n\treturn nil\n}\n\n// Purges and removes the msgBlock from the store.\n// Lock should be held.\nfunc (fs *fileStore) purgeMsgBlock(mb *msgBlock) error {\n\tmb.mu.Lock()\n\t// Adjust per-subject tracking if present.\n\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\tmb.mu.Unlock()\n\t\treturn err\n\t} else if mb.fss != nil {\n\t\tmb.fss.IterFast(func(bsubj []byte, ss *SimpleState) bool {\n\t\t\tsubj := bytesToString(bsubj)\n\t\t\tfor range ss.Msgs {\n\t\t\t\tfs.removePerSubject(subj)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n\t// Clean up scheduled message metadata if we know this block contained any.\n\tif fs.scheduling != nil && mb.schedules > 0 {\n\t\tif mb.cacheNotLoaded() {\n\t\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tvar smv StoreMsg\n\t\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\t\tfor seq := fseq; seq <= lseq; seq++ {\n\t\t\tsm, err := mb.cacheLookupNoCopy(seq, &smv)\n\t\t\tif err != nil && err != errDeletedMsg {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif sm == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif schedule, ok := getMessageSchedule(sm.hdr); ok && !schedule.IsZero() {\n\t\t\t\tfs.scheduling.remove(seq)\n\t\t\t}\n\t\t}\n\t}\n\t// Update top level accounting.\n\tmsgs, bytes := mb.msgs, mb.bytes\n\tif msgs > fs.state.Msgs {\n\t\tmsgs = fs.state.Msgs\n\t}\n\tif bytes > fs.state.Bytes {\n\t\tbytes = fs.state.Bytes\n\t}\n\tfs.state.Msgs -= msgs\n\tfs.state.Bytes -= bytes\n\tif err := fs.removeMsgBlock(mb); err != nil {\n\t\tmb.mu.Unlock()\n\t\treturn err\n\t}\n\tmb.tryForceExpireCacheLocked()\n\tmb.finishedWithCache()\n\tmb.mu.Unlock()\n\tif err := fs.selectNextFirst(); err != nil {\n\t\treturn err\n\t}\n\n\tif cb := fs.scb; cb != nil {\n\t\t// If we have a callback registered, we need to release lock regardless since consumers will recalculate pending.\n\t\tfs.mu.Unlock()\n\t\t// Storage updates.\n\t\tcb(-int64(msgs), -int64(bytes), 0, _EMPTY_)\n\t\tfs.mu.Lock()\n\t}\n\treturn nil\n}\n\n// Called by purge to simply get rid of the cache and close our fds.\n// Lock should not be held.\nfunc (mb *msgBlock) dirtyClose() {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\tmb.dirtyCloseWithRemove(false)\n}\n\n// Should be called with lock held.\nfunc (mb *msgBlock) dirtyCloseWithRemove(remove bool) error {\n\tif mb == nil {\n\t\treturn nil\n\t}\n\t// Stop cache expiration timer.\n\tif mb.ctmr != nil {\n\t\tmb.ctmr.Stop()\n\t\tmb.ctmr = nil\n\t}\n\t// Close cache\n\tmb.clearCacheAndOffset()\n\t// Quit our loops.\n\tif mb.qch != nil {\n\t\tclose(mb.qch)\n\t\tmb.qch = nil\n\t}\n\tif fd := mb.mfd; fd != nil {\n\t\tmb.mfd = nil\n\t\tif err := fd.Close(); err != nil && !os.IsNotExist(err) {\n\t\t\treturn err\n\t\t}\n\t}\n\tif remove {\n\t\t// Clear any tracking by subject if we are removing.\n\t\tmb.fss = nil\n\t\tif mb.mfn != _EMPTY_ {\n\t\t\tif err := os.Remove(mb.mfn); err != nil && !os.IsNotExist(err) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmb.mfn = _EMPTY_\n\t\t}\n\t\tif mb.kfn != _EMPTY_ {\n\t\t\tif err := os.Remove(mb.kfn); err != nil && !os.IsNotExist(err) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Remove a seq from the fss and select new first.\n// Lock should be held.\nfunc (mb *msgBlock) removeSeqPerSubject(subj string, seq uint64) (uint64, error) {\n\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\treturn 0, err\n\t}\n\tif mb.fss == nil {\n\t\treturn 0, nil\n\t}\n\tbsubj := stringToBytes(subj)\n\tss, ok := mb.fss.Find(bsubj)\n\tif !ok || ss == nil {\n\t\treturn 0, nil\n\t}\n\n\tmb.fs.sdm.removeSeqAndSubject(seq, subj)\n\tif ss.Msgs == 1 {\n\t\tmb.fss.Delete(bsubj)\n\t\treturn 0, nil\n\t}\n\n\tss.Msgs--\n\n\t// Only one left.\n\tif ss.Msgs == 1 {\n\t\tif !ss.lastNeedsUpdate && seq != ss.Last {\n\t\t\tss.First = ss.Last\n\t\t\tss.firstNeedsUpdate = false\n\t\t\treturn 1, nil\n\t\t}\n\t\tif !ss.firstNeedsUpdate && seq != ss.First {\n\t\t\tss.Last = ss.First\n\t\t\tss.lastNeedsUpdate = false\n\t\t\treturn 1, nil\n\t\t}\n\t}\n\n\t// We can lazily calculate the first/last sequence when needed.\n\tss.firstNeedsUpdate = seq == ss.First || ss.firstNeedsUpdate\n\tss.lastNeedsUpdate = seq == ss.Last || ss.lastNeedsUpdate\n\n\treturn ss.Msgs, nil\n}\n\n// Will recalculate the first and/or last sequence for this subject in this block.\n// Will avoid slower path message lookups and scan the cache directly instead.\nfunc (mb *msgBlock) recalculateForSubj(subj string, ss *SimpleState) error {\n\t// Need to make sure messages are loaded.\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer mb.finishedWithCache()\n\t}\n\n\tstartSlot := int(ss.First - mb.cache.fseq)\n\tif startSlot < 0 {\n\t\tstartSlot = 0\n\t}\n\tif startSlot >= len(mb.cache.idx) {\n\t\tss.First = ss.Last\n\t\tss.firstNeedsUpdate = false\n\t\tss.lastNeedsUpdate = false\n\t\treturn nil\n\t}\n\n\tendSlot := int(ss.Last - mb.cache.fseq)\n\tif endSlot < 0 {\n\t\tendSlot = 0\n\t}\n\tif endSlot >= len(mb.cache.idx) || startSlot > endSlot {\n\t\treturn nil\n\t}\n\n\tvar le = binary.LittleEndian\n\tif ss.firstNeedsUpdate {\n\t\t// Mark first as updated.\n\t\tss.firstNeedsUpdate = false\n\n\t\tfseq := ss.First + 1\n\t\tif mbFseq := atomic.LoadUint64(&mb.first.seq); fseq < mbFseq {\n\t\t\tfseq = mbFseq\n\t\t}\n\t\tfor slot := startSlot; slot < len(mb.cache.idx); slot++ {\n\t\t\tbi := mb.cache.idx[slot] &^ cbit\n\t\t\tif bi == dbit {\n\t\t\t\t// delete marker so skip.\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tli := int(bi)\n\t\t\tif li >= len(mb.cache.buf) {\n\t\t\t\tss.First = ss.Last\n\t\t\t\t// Only need to reset ss.lastNeedsUpdate, ss.firstNeedsUpdate is already reset above.\n\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tbuf := mb.cache.buf[li:]\n\t\t\thdr := buf[:msgHdrSize]\n\t\t\tslen := int(le.Uint16(hdr[20:]))\n\t\t\tif subj == bytesToString(buf[msgHdrSize:msgHdrSize+slen]) {\n\t\t\t\tseq := le.Uint64(hdr[4:])\n\t\t\t\tif seq < fseq || seq&ebit != 0 || mb.dmap.Exists(seq) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tss.First = seq\n\t\t\t\tif ss.Msgs == 1 {\n\t\t\t\t\tss.Last = seq\n\t\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\t// Skip the start slot ahead, if we need to recalculate last we can stop early.\n\t\t\t\tstartSlot = slot\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif ss.lastNeedsUpdate {\n\t\t// Mark last as updated.\n\t\tss.lastNeedsUpdate = false\n\n\t\tlseq := ss.Last - 1\n\t\tif mbLseq := atomic.LoadUint64(&mb.last.seq); lseq > mbLseq {\n\t\t\tlseq = mbLseq\n\t\t}\n\t\tfor slot := endSlot; slot >= startSlot; slot-- {\n\t\t\tbi := mb.cache.idx[slot] &^ cbit\n\t\t\tif bi == dbit {\n\t\t\t\t// delete marker so skip.\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tli := int(bi)\n\t\t\tif li >= len(mb.cache.buf) {\n\t\t\t\t// Can't overwrite ss.Last, just skip.\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tbuf := mb.cache.buf[li:]\n\t\t\thdr := buf[:msgHdrSize]\n\t\t\tslen := int(le.Uint16(hdr[20:]))\n\t\t\tif subj == bytesToString(buf[msgHdrSize:msgHdrSize+slen]) {\n\t\t\t\tseq := le.Uint64(hdr[4:])\n\t\t\t\tif seq > lseq || seq&ebit != 0 || mb.dmap.Exists(seq) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Sequence should never be lower, but guard against it nonetheless.\n\t\t\t\tif seq < ss.First {\n\t\t\t\t\tseq = ss.First\n\t\t\t\t}\n\t\t\t\tss.Last = seq\n\t\t\t\tif ss.Msgs == 1 {\n\t\t\t\t\tss.First = seq\n\t\t\t\t\tss.firstNeedsUpdate = false\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) resetGlobalPerSubjectInfo() error {\n\t// Clear any global subject state.\n\tfs.psim, fs.tsl = fs.psim.Empty(), 0\n\tif fs.noTrackSubjects() {\n\t\treturn nil\n\t}\n\tfor _, mb := range fs.blks {\n\t\tif err := fs.populateGlobalPerSubjectInfo(mb); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mb *msgBlock) resetPerSubjectInfo() error {\n\tmb.fss = nil\n\treturn mb.generatePerSubjectInfo()\n}\n\n// generatePerSubjectInfo will generate the per subject info via the raw msg block.\n// Lock should be held.\nfunc (mb *msgBlock) generatePerSubjectInfo() error {\n\t// Check if this mb is empty. This can happen when its the last one and we are holding onto it for seq and timestamp info.\n\tif mb.msgs == 0 {\n\t\treturn nil\n\t}\n\n\tif mb.cacheNotLoaded() {\n\t\tif err := mb.loadMsgsWithLock(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// indexCacheBuf can produce fss now, so if non-nil we are good.\n\t\tif mb.fss != nil {\n\t\t\treturn nil\n\t\t}\n\t\tdefer mb.finishedWithCache()\n\t}\n\n\t// Create new one regardless.\n\tmb.fss = mb.fss.Empty()\n\n\tvar smv StoreMsg\n\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\tfor seq := fseq; seq <= lseq; seq++ {\n\t\tif mb.dmap.Exists(seq) {\n\t\t\t// Optimisation to avoid calling cacheLookup which hits time.Now().\n\t\t\t// It gets set later on if the fss is non-empty anyway.\n\t\t\tcontinue\n\t\t}\n\t\tsm, err := mb.cacheLookupNoCopy(seq, &smv)\n\t\tif err != nil {\n\t\t\t// Since we are walking by sequence we can ignore some errors that are benign to rebuilding our state.\n\t\t\tif err == ErrStoreMsgNotFound || err == errDeletedMsg {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err == errNoCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif sm != nil && len(sm.subj) > 0 {\n\t\t\tif ss, ok := mb.fss.Find(stringToBytes(sm.subj)); ok && ss != nil {\n\t\t\t\tss.Msgs++\n\t\t\t\tss.Last = seq\n\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t} else {\n\t\t\t\tmb.fss.Insert(stringToBytes(sm.subj), SimpleState{Msgs: 1, First: seq, Last: seq})\n\t\t\t}\n\t\t}\n\t}\n\n\tif mb.fss.Size() > 0 {\n\t\t// Make sure we run the cache expire timer.\n\t\tmb.llts = ats.AccessTime()\n\t\t// Mark fss activity same as load time.\n\t\tmb.lsts = mb.llts\n\t\tmb.startCacheExpireTimer()\n\t}\n\treturn nil\n}\n\n// Helper to make sure fss loaded if we are tracking.\n// Lock should be held\nfunc (mb *msgBlock) ensurePerSubjectInfoLoaded() error {\n\tif mb.fss != nil || mb.noTrack {\n\t\tif mb.fss != nil {\n\t\t\t// Mark fss activity.\n\t\t\tmb.lsts = ats.AccessTime()\n\t\t}\n\t\treturn nil\n\t}\n\tif mb.msgs == 0 {\n\t\tmb.fss = stree.NewSubjectTree[SimpleState]()\n\t\treturn nil\n\t}\n\treturn mb.generatePerSubjectInfo()\n}\n\n// Called on recovery to populate the global psim state.\n// Lock should be held.\nfunc (fs *fileStore) populateGlobalPerSubjectInfo(mb *msgBlock) error {\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\n\tif err := mb.ensurePerSubjectInfoLoaded(); err != nil {\n\t\treturn err\n\t}\n\n\t// Now populate psim.\n\tmb.fss.IterFast(func(bsubj []byte, ss *SimpleState) bool {\n\t\tif len(bsubj) > 0 {\n\t\t\tif info, ok := fs.psim.Find(bsubj); ok {\n\t\t\t\tinfo.total += ss.Msgs\n\t\t\t\tif mb.index > info.lblk {\n\t\t\t\t\tinfo.lblk = mb.index\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfs.psim.Insert(bsubj, psi{total: ss.Msgs, fblk: mb.index, lblk: mb.index})\n\t\t\t\tfs.tsl += len(bsubj)\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\treturn nil\n}\n\n// Calls os.RemoveAll on the given `dir` directory, but if an error occurs,\n// retries up to one second. If that still fails, returns the last error\n// that os.RemoveAll returned.\nfunc removeAllWithRetry(dir string) error {\n\t<-dios\n\terr := os.RemoveAll(dir)\n\tdios <- struct{}{}\n\tif err == nil {\n\t\treturn nil\n\t}\n\tttl := time.Now().Add(time.Second)\n\tfor time.Now().Before(ttl) {\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\t<-dios\n\t\terr = os.RemoveAll(dir)\n\t\tdios <- struct{}{}\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn err\n}\n\n// Close the message block.\nfunc (mb *msgBlock) close(sync bool) {\n\tif mb == nil {\n\t\treturn\n\t}\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\n\tif mb.closed {\n\t\treturn\n\t}\n\n\t// Stop cache expiration timer.\n\tif mb.ctmr != nil {\n\t\tmb.ctmr.Stop()\n\t\tmb.ctmr = nil\n\t}\n\n\t// Clear fss.\n\tmb.fss = nil\n\n\t// Flush pending writes.\n\tmb.flushPendingMsgsLocked()\n\t// Close cache\n\tmb.clearCacheAndOffset()\n\t// Quit our loops.\n\tif mb.qch != nil {\n\t\tclose(mb.qch)\n\t\tmb.qch = nil\n\t}\n\tif mb.mfd != nil {\n\t\tif sync {\n\t\t\tmb.mfd.Sync()\n\t\t}\n\t\tmb.mfd.Close()\n\t}\n\tmb.mfd = nil\n\t// Mark as closed.\n\tmb.closed = true\n}\n\nfunc (fs *fileStore) closeAllMsgBlocks(sync bool) {\n\tfor _, mb := range fs.blks {\n\t\tmb.close(sync)\n\t}\n}\n\nfunc (fs *fileStore) Delete(inline bool) error {\n\tif fs.isClosed() {\n\t\t// Always attempt to remove since we could have been closed beforehand.\n\t\tos.RemoveAll(fs.fcfg.StoreDir)\n\t\t// Since we did remove, if we did have anything remaining make sure to\n\t\t// call into any storage updates that had been registered.\n\t\tfs.mu.Lock()\n\t\tcb, msgs, bytes := fs.scb, int64(fs.state.Msgs), int64(fs.state.Bytes)\n\t\t// Guard against double accounting if called twice.\n\t\tfs.state.Msgs, fs.state.Bytes = 0, 0\n\t\tfs.mu.Unlock()\n\t\tif msgs > 0 && cb != nil {\n\t\t\tcb(-msgs, -bytes, 0, _EMPTY_)\n\t\t}\n\t\treturn ErrStoreClosed\n\t}\n\n\tpdir := filepath.Join(fs.fcfg.StoreDir, purgeDir)\n\tif _, err := os.Stat(pdir); err == nil {\n\t\t_ = os.RemoveAll(pdir)\n\t}\n\n\t// Quickly close all blocks and simulate a purge w/o overhead an new write block.\n\tfs.mu.Lock()\n\tfor _, mb := range fs.blks {\n\t\tmb.dirtyClose()\n\t}\n\tdmsgs := fs.state.Msgs\n\tdbytes := int64(fs.state.Bytes)\n\tfs.state.Msgs, fs.state.Bytes = 0, 0\n\tfs.blks = nil\n\tcb := fs.scb\n\tfs.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(dmsgs), -dbytes, 0, _EMPTY_)\n\t}\n\n\tif err := fs.stop(true, false); err != nil {\n\t\treturn err\n\t}\n\n\t// Make sure we will not try to recover if killed before removal below completes.\n\tif err := os.Remove(filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFile)); err != nil {\n\t\treturn err\n\t}\n\t// Now move into different directory with \".\" prefix.\n\tndir := filepath.Join(filepath.Dir(fs.fcfg.StoreDir), tsep+filepath.Base(fs.fcfg.StoreDir))\n\tif err := os.Rename(fs.fcfg.StoreDir, ndir); err != nil {\n\t\treturn err\n\t}\n\t// Do this in separate Go routine in case lots of blocks.\n\t// Purge above protects us as does the removal of meta artifacts above.\n\tif inline {\n\t\tif err := removeAllWithRetry(ndir); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tgo removeAllWithRetry(ndir)\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (fs *fileStore) setSyncTimer() {\n\tif fs.syncTmr != nil {\n\t\tfs.syncTmr.Reset(fs.fcfg.SyncInterval)\n\t} else {\n\t\t// First time this fires will be between SyncInterval/2 and SyncInterval,\n\t\t// so that different stores are spread out, rather than having many of\n\t\t// them trying to all sync at once, causing blips and contending dios.\n\t\tstart := (fs.fcfg.SyncInterval / 2) + (time.Duration(mrand.Int63n(int64(fs.fcfg.SyncInterval / 2))))\n\t\tfs.syncTmr = time.AfterFunc(start, fs.syncBlocks)\n\t}\n}\n\n// Lock should be held.\nfunc (fs *fileStore) cancelSyncTimer() {\n\tif fs.syncTmr != nil {\n\t\tfs.syncTmr.Stop()\n\t\tfs.syncTmr = nil\n\t}\n}\n\n// The full state file is versioned.\n// - 0x1: original binary index.db format\n// - 0x2: adds support for TTL count field after num deleted\nconst (\n\tfullStateMagic = uint8(11)\n\tfullStateMinVersion = uint8(1) // What is the minimum version we know how to parse?\n\tfullStateVersion = uint8(3) // What is the current version written out to index.db?\n)\n\n// This go routine periodically writes out our full stream state index.\nfunc (fs *fileStore) flushStreamStateLoop(qch, done chan struct{}) {\n\t// Signal we are done on exit.\n\tdefer close(done)\n\n\t// Make sure we do not try to write these out too fast.\n\t// Spread these out for large numbers on a server restart.\n\tconst writeThreshold = 2 * time.Minute\n\twriteJitter := time.Duration(mrand.Int63n(int64(30 * time.Second)))\n\tt := time.NewTicker(writeThreshold + writeJitter)\n\tdefer t.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-t.C:\n\t\t\terr := fs.writeFullState()\n\t\t\tif isPermissionError(err) && fs.srv != nil {\n\t\t\t\tfs.warn(\"File system permission denied when flushing stream state, disabling JetStream: %v\", err)\n\t\t\t\t// messages in block cache could be lost in the worst case.\n\t\t\t\t// In the clustered mode it is very highly unlikely as a result of replication.\n\t\t\t\tfs.srv.DisableJetStream()\n\t\t\t\treturn\n\t\t\t}\n\n\t\tcase <-qch:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Helper since unixnano of zero time undefined.\nfunc timestampNormalized(t time.Time) int64 {\n\tif t.IsZero() {\n\t\treturn 0\n\t}\n\treturn t.UnixNano()\n}\n\n// writeFullState will proceed to write the full meta state iff not complex and time consuming.\n// Since this is for quick recovery it is optional and should not block/stall normal operations.\nfunc (fs *fileStore) writeFullState() error {\n\treturn fs._writeFullState(false)\n}\n\n// forceWriteFullState will proceed to write the full meta state.\nfunc (fs *fileStore) forceWriteFullState() error {\n\treturn fs._writeFullState(true)\n}\n\n// This will write the full binary state for the stream.\n// This plus everything new since last hash will be the total recovered state.\n// This state dump will have the following.\n// 1. Stream summary - Msgs, Bytes, First and Last (Sequence and Timestamp)\n// 2. PSIM - Per Subject Index Map - Tracks first and last blocks with subjects present.\n// 3. MBs - Index, Bytes, First and Last Sequence and Timestamps, and the deleted map (avl.seqset).\n// 4. Last block index and hash of record inclusive to this stream state.\nfunc (fs *fileStore) _writeFullState(force bool) error {\n\tif fs.isClosed() {\n\t\treturn nil\n\t}\n\n\t// If we aren't forcing an update then only queue this up if we aren't already\n\t// running. This means we can keep waiting on shutdown if needed but not build up\n\t// lots of waiting goroutines in a bad timer case.\n\tif fs.wfsrun.Add(1) > 1 && !force {\n\t\tfs.wfsrun.Add(-1)\n\t\treturn nil\n\t}\n\tdefer fs.wfsrun.Add(-1)\n\n\t// Only allow one _writeFullState to take place at a time, otherwise we can\n\t// have multiple goroutines trying to write the same file after we've released\n\t// the store lock.\n\tfs.wfsmu.Lock()\n\tdefer fs.wfsmu.Unlock()\n\n\tstart := time.Now()\n\tfs.mu.RLock()\n\tif fs.dirty == 0 {\n\t\tfs.mu.RUnlock()\n\t\treturn nil\n\t}\n\n\t// Configure encryption if needed.\n\tif fs.prf != nil {\n\t\t// Re-acquire temporarily as write lock to set up AEK.\n\t\tfs.mu.RUnlock()\n\t\tfs.mu.Lock()\n\t\terr := fs.setupAEK()\n\t\tfs.mu.Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfs.mu.RLock()\n\t}\n\n\t// For calculating size and checking time costs for non forced calls.\n\tnumSubjects := fs.numSubjects()\n\n\t// If we are not being forced to write out our state, check the complexity for time costs as to not\n\t// block or stall normal operations.\n\t// We will base off of number of subjects and interior deletes. A very large number of msg blocks could also\n\t// be used, but for next server version will redo all meta handling to be disk based. So this is temporary.\n\tif !force {\n\t\tconst numThreshold = 1_000_000\n\t\t// Calculate interior deletes.\n\t\tvar numDeleted int\n\t\tif fs.state.LastSeq > fs.state.FirstSeq {\n\t\t\tnumDeleted = int((fs.state.LastSeq - fs.state.FirstSeq + 1) - fs.state.Msgs)\n\t\t}\n\t\tif numSubjects > numThreshold || numDeleted > numThreshold {\n\t\t\tfs.mu.RUnlock()\n\t\t\treturn errStateTooBig\n\t\t}\n\t}\n\n\t// We track this through subsequent runs to get an avg per blk used for subsequent runs.\n\tavgDmapLen := fs.wfsadml\n\t// If first time through could be 0\n\tif avgDmapLen == 0 && ((fs.state.LastSeq-fs.state.FirstSeq+1)-fs.state.Msgs) > 0 {\n\t\tavgDmapLen = 1024\n\t}\n\n\t// Calculate and estimate of the uper bound on the size to avoid multiple allocations.\n\tsz := hdrLen + // Magic and Version\n\t\t(binary.MaxVarintLen64 * 6) + // FS data\n\t\tbinary.MaxVarintLen64 + fs.tsl + // NumSubjects + total subject length\n\t\tnumSubjects*(binary.MaxVarintLen64*4) + // psi record\n\t\tbinary.MaxVarintLen64 + // Num blocks.\n\t\tlen(fs.blks)*((binary.MaxVarintLen64*8)+avgDmapLen) + // msg blocks, avgDmapLen is est for dmaps\n\t\tbinary.MaxVarintLen64 + 8 + 8 // last index + record checksum + full state checksum\n\n\t// Do 4k on stack if possible.\n\tconst ssz = 4 * 1024\n\tvar buf []byte\n\n\tif sz <= ssz {\n\t\tvar _buf [ssz]byte\n\t\tbuf, sz = _buf[0:hdrLen:ssz], ssz\n\t} else {\n\t\tbuf = make([]byte, hdrLen, sz)\n\t}\n\n\tbuf[0], buf[1] = fullStateMagic, fullStateVersion\n\tbuf = binary.AppendUvarint(buf, fs.state.Msgs)\n\tbuf = binary.AppendUvarint(buf, fs.state.Bytes)\n\tbuf = binary.AppendUvarint(buf, fs.state.FirstSeq)\n\tbuf = binary.AppendVarint(buf, timestampNormalized(fs.state.FirstTime))\n\tbuf = binary.AppendUvarint(buf, fs.state.LastSeq)\n\tbuf = binary.AppendVarint(buf, timestampNormalized(fs.state.LastTime))\n\n\t// Do per subject information map if applicable.\n\tbuf = binary.AppendUvarint(buf, uint64(numSubjects))\n\tif numSubjects > 0 {\n\t\tfs.psim.Match([]byte(fwcs), func(subj []byte, psi *psi) {\n\t\t\tbuf = binary.AppendUvarint(buf, uint64(len(subj)))\n\t\t\tbuf = append(buf, subj...)\n\t\t\tbuf = binary.AppendUvarint(buf, psi.total)\n\t\t\tbuf = binary.AppendUvarint(buf, uint64(psi.fblk))\n\t\t\tif psi.total > 1 {\n\t\t\t\tbuf = binary.AppendUvarint(buf, uint64(psi.lblk))\n\t\t\t}\n\t\t})\n\t}\n\n\t// Now walk all blocks and write out first and last and optional dmap encoding.\n\tvar lbi uint32\n\tvar lchk [8]byte\n\n\tnb := len(fs.blks)\n\tbuf = binary.AppendUvarint(buf, uint64(nb))\n\n\t// Use basetime to save some space.\n\tbaseTime := timestampNormalized(fs.state.FirstTime)\n\tvar scratch [8 * 1024]byte\n\n\t// Track the state as represented by the mbs.\n\tvar mstate StreamState\n\n\tvar dmapTotalLen int\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tbuf = binary.AppendUvarint(buf, uint64(mb.index))\n\t\tbuf = binary.AppendUvarint(buf, mb.bytes)\n\t\tbuf = binary.AppendUvarint(buf, atomic.LoadUint64(&mb.first.seq))\n\t\tbuf = binary.AppendVarint(buf, mb.first.ts-baseTime)\n\t\tbuf = binary.AppendUvarint(buf, atomic.LoadUint64(&mb.last.seq))\n\t\tbuf = binary.AppendVarint(buf, mb.last.ts-baseTime)\n\n\t\tnumDeleted := mb.dmap.Size()\n\t\tbuf = binary.AppendUvarint(buf, uint64(numDeleted))\n\t\tbuf = binary.AppendUvarint(buf, mb.ttls) // Field is new in version 2\n\t\tbuf = binary.AppendUvarint(buf, mb.schedules) // Field is new in version 3\n\t\tif numDeleted > 0 {\n\t\t\tdmap := mb.dmap.Encode(scratch[:0])\n\t\t\tdmapTotalLen += len(dmap)\n\t\t\tbuf = append(buf, dmap...)\n\t\t}\n\t\t// If this is the last one grab the last checksum and the block index, e.g. 22.blk, 22 is the block index.\n\t\t// We use this to quickly open this file on recovery.\n\t\tif mb == fs.lmb {\n\t\t\tlbi = mb.index\n\t\t\tmb.ensureLastChecksumLoaded()\n\t\t\tcopy(lchk[0:], mb.lchk[:])\n\t\t}\n\t\tupdateTrackingState(&mstate, mb)\n\t\tmb.mu.RUnlock()\n\t}\n\tif dmapTotalLen > 0 {\n\t\tfs.wfsadml = dmapTotalLen / len(fs.blks)\n\t}\n\n\t// Place block index and hash onto the end.\n\tbuf = binary.AppendUvarint(buf, uint64(lbi))\n\tbuf = append(buf, lchk[:]...)\n\n\t// Encrypt if needed.\n\tif fs.prf != nil {\n\t\tnonce := make([]byte, fs.aek.NonceSize(), fs.aek.NonceSize()+len(buf)+fs.aek.Overhead())\n\t\tif n, err := rand.Read(nonce); err != nil {\n\t\t\tfs.mu.RUnlock()\n\t\t\treturn err\n\t\t} else if n != len(nonce) {\n\t\t\tfs.mu.RUnlock()\n\t\t\treturn fmt.Errorf(\"not enough nonce bytes read (%d != %d)\", n, len(nonce))\n\t\t}\n\t\tbuf = fs.aek.Seal(nonce, nonce, buf, nil)\n\t}\n\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\n\t// Need to have our own hasher here, as under a read lock we can't mutate the\n\t// fs.hh safely.\n\tkey := sha256.Sum256([]byte(fs.cfg.Name))\n\thh, _ := highwayhash.NewDigest64(key[:])\n\thh.Write(buf)\n\tbuf = hh.Sum(buf)\n\n\t// Snapshot prior dirty count.\n\tpriorDirty := fs.dirty\n\n\tstatesEqual := trackingStatesEqual(&fs.state, &mstate)\n\t// Release lock.\n\tfs.mu.RUnlock()\n\n\t// Check consistency here.\n\tif !statesEqual {\n\t\tfs.warn(\"Stream state encountered internal inconsistency on write\")\n\t\t// Rebuild our fs state from the mb state.\n\t\tfs.rebuildState(nil)\n\t\treturn errCorruptState\n\t}\n\n\tif cap(buf) > sz {\n\t\tfs.debug(\"WriteFullState reallocated from %d to %d\", sz, cap(buf))\n\t}\n\n\t// Only warn about construction time since file write not holding any locks.\n\tif took := time.Since(start); took > time.Minute {\n\t\tfs.warn(\"WriteFullState took %v (%d bytes)\", took.Round(time.Millisecond), len(buf))\n\t}\n\n\t// Write our update index.db\n\t// Protect with dios.\n\t<-dios\n\terr := os.WriteFile(fn, buf, defaultFilePerms)\n\t// if file system is not writable isPermissionError is set to true\n\tdios <- struct{}{}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Update dirty if successful.\n\tfs.mu.Lock()\n\tfs.dirty -= priorDirty\n\tfs.mu.Unlock()\n\n\t// Attempt to write both files, an error in one should not prevent the other from being written.\n\tttlErr := fs.writeTTLState()\n\tschedErr := fs.writeMsgSchedulingState()\n\tif ttlErr != nil {\n\t\treturn ttlErr\n\t} else if schedErr != nil {\n\t\treturn schedErr\n\t}\n\treturn nil\n}\n\nfunc (fs *fileStore) writeTTLState() error {\n\tfs.mu.RLock()\n\tif fs.ttls == nil {\n\t\tfs.mu.RUnlock()\n\t\treturn nil\n\t}\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, ttlStreamStateFile)\n\t// Must be lseq+1 to identify up to which sequence the TTLs are valid.\n\tbuf := fs.ttls.Encode(fs.state.LastSeq + 1)\n\tfs.mu.RUnlock()\n\n\treturn fs.writeFileWithOptionalSync(fn, buf, defaultFilePerms)\n}\n\nfunc (fs *fileStore) writeMsgSchedulingState() error {\n\tfs.mu.RLock()\n\tif fs.scheduling == nil {\n\t\tfs.mu.RUnlock()\n\t\treturn nil\n\t}\n\tfn := filepath.Join(fs.fcfg.StoreDir, msgDir, msgSchedulingStreamStateFile)\n\t// Must be lseq+1 to identify up to which sequence the schedules are valid.\n\tbuf := fs.scheduling.encode(fs.state.LastSeq + 1)\n\tfs.mu.RUnlock()\n\n\treturn fs.writeFileWithOptionalSync(fn, buf, defaultFilePerms)\n}\n\n// Stop the current filestore.\nfunc (fs *fileStore) Stop() error {\n\treturn fs.stop(false, true)\n}\n\n// Stop the current filestore.\nfunc (fs *fileStore) stop(delete, writeState bool) error {\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\n\tfs.mu.Lock()\n\tif fs.closing {\n\t\tfs.mu.Unlock()\n\t\treturn ErrStoreClosed\n\t}\n\n\t// Mark as closing. Do before releasing the lock to wait on the state flush loop\n\t// so we don't end up with this function running more than once.\n\tfs.closing = true\n\n\t// Release the state flusher loop.\n\tif fs.qch != nil {\n\t\tclose(fs.qch)\n\t\tfs.qch = nil\n\t}\n\n\tif writeState {\n\t\t// Wait for the state flush loop to exit.\n\t\tfsld := fs.fsld\n\t\tfs.mu.Unlock()\n\t\t<-fsld\n\t\tfs.mu.Lock()\n\n\t\tfs.checkAndFlushLastBlock()\n\t}\n\tfs.closeAllMsgBlocks(false)\n\n\tfs.cancelSyncTimer()\n\tfs.cancelAgeChk()\n\n\tif writeState {\n\t\t// Write full state if needed. If not dirty this is a no-op.\n\t\tfs.mu.Unlock()\n\t\tfs.forceWriteFullState()\n\t\tfs.mu.Lock()\n\t}\n\n\t// Mark as closed. Last message block needs to be cleared after\n\t// writeFullState has completed.\n\tfs.closed.Store(true)\n\tfs.lmb = nil\n\n\t// We should update the upper usage layer on a stop.\n\tcb, bytes := fs.scb, int64(fs.state.Bytes)\n\tfs.mu.Unlock()\n\n\tfs.cmu.Lock()\n\tvar _cfs [256]ConsumerStore\n\tcfs := append(_cfs[:0], fs.cfs...)\n\tfs.cfs = nil\n\tfs.cmu.Unlock()\n\n\tfor _, o := range cfs {\n\t\tif delete {\n\t\t\to.StreamDelete()\n\t\t} else {\n\t\t\to.Stop()\n\t\t}\n\t}\n\n\tif bytes > 0 && cb != nil {\n\t\tcb(0, -bytes, 0, _EMPTY_)\n\t}\n\n\t// Unregister from the access time service.\n\tats.Unregister()\n\n\treturn nil\n}\n\nconst errFile = \"errors.txt\"\n\n// Stream our snapshot through S2 compression and tar.\nfunc (fs *fileStore) streamSnapshot(w io.WriteCloser, includeConsumers bool, errCh chan string) {\n\tdefer close(errCh)\n\tdefer w.Close()\n\n\tenc := s2.NewWriter(w)\n\tdefer enc.Close()\n\n\ttw := tar.NewWriter(enc)\n\tdefer tw.Close()\n\n\tdefer func() {\n\t\tfs.mu.Lock()\n\t\tfs.sips--\n\t\tfs.mu.Unlock()\n\t}()\n\n\tmodTime := time.Now().UTC()\n\n\twriteFile := func(name string, buf []byte) error {\n\t\thdr := &tar.Header{\n\t\t\tName: name,\n\t\t\tMode: 0600,\n\t\t\tModTime: modTime,\n\t\t\tUname: \"nats\",\n\t\t\tGname: \"nats\",\n\t\t\tSize: int64(len(buf)),\n\t\t\tFormat: tar.FormatPAX,\n\t\t}\n\t\tif err := tw.WriteHeader(hdr); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := tw.Write(buf); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\twriteErr := func(err string) {\n\t\twriteFile(errFile, []byte(err))\n\t\terrCh <- err\n\t}\n\n\tfs.mu.Lock()\n\tblks := fs.blks\n\t// Grab our general meta data.\n\t// We do this now instead of pulling from files since they could be encrypted.\n\tmeta, err := json.Marshal(fs.cfg)\n\tif err != nil {\n\t\tfs.mu.Unlock()\n\t\twriteErr(fmt.Sprintf(\"Could not gather stream meta file: %v\", err))\n\t\treturn\n\t}\n\thh := fs.hh\n\thh.Reset()\n\thh.Write(meta)\n\tvar hb [highwayhash.Size64]byte\n\tsum := []byte(hex.EncodeToString(fs.hh.Sum(hb[:0])))\n\tfs.mu.Unlock()\n\n\t// Meta first.\n\tif writeFile(JetStreamMetaFile, meta) != nil {\n\t\treturn\n\t}\n\tif writeFile(JetStreamMetaFileSum, sum) != nil {\n\t\treturn\n\t}\n\n\t// Can't use join path here, tar only recognizes relative paths with forward slashes.\n\tmsgPre := msgDir + \"/\"\n\tvar bbuf []byte\n\n\t// Now do messages themselves.\n\tfor _, mb := range blks {\n\t\tif mb.pendingWriteSize() > 0 {\n\t\t\tmb.flushPendingMsgs()\n\t\t}\n\t\tmb.mu.Lock()\n\t\t// We could stream but don't want to hold the lock and prevent changes, so just read in and\n\t\t// release the lock for now.\n\t\tbbuf, err = mb.loadBlock(bbuf)\n\t\tif err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\twriteErr(fmt.Sprintf(\"Could not read message block [%d]: %v\", mb.index, err))\n\t\t\treturn\n\t\t}\n\t\t// Check for encryption.\n\t\tif mb.bek != nil && len(bbuf) > 0 {\n\t\t\trbek, err := genBlockEncryptionKey(fs.fcfg.Cipher, mb.seed, mb.nonce)\n\t\t\tif err != nil {\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\twriteErr(fmt.Sprintf(\"Could not create encryption key for message block [%d]: %v\", mb.index, err))\n\t\t\t\treturn\n\t\t\t}\n\t\t\trbek.XORKeyStream(bbuf, bbuf)\n\t\t}\n\t\t// Check for compression.\n\t\tif bbuf, err = mb.decompressIfNeeded(bbuf); err != nil {\n\t\t\tmb.mu.Unlock()\n\t\t\twriteErr(fmt.Sprintf(\"Could not decompress message block [%d]: %v\", mb.index, err))\n\t\t\treturn\n\t\t}\n\t\tmb.mu.Unlock()\n\n\t\t// Do this one unlocked.\n\t\tif writeFile(msgPre+fmt.Sprintf(blkScan, mb.index), bbuf) != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Do index.db last. We will force a write as well.\n\t// Write out full state as well before proceeding.\n\tif err := fs.forceWriteFullState(); err == nil {\n\t\tconst minLen = 32\n\t\tsfn := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tif buf, err := os.ReadFile(sfn); err == nil && len(buf) >= minLen {\n\t\t\tfs.mu.Lock()\n\t\t\tif fs.aek != nil {\n\t\t\t\tns := fs.aek.NonceSize()\n\t\t\t\tbuf, err = fs.aek.Open(nil, buf[:ns], buf[ns:len(buf)-highwayhash.Size64], nil)\n\t\t\t\tif err == nil {\n\t\t\t\t\t// Redo hash checksum at end on plaintext.\n\t\t\t\t\thh.Reset()\n\t\t\t\t\thh.Write(buf)\n\t\t\t\t\tbuf = fs.hh.Sum(buf)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfs.mu.Unlock()\n\t\t\tif err == nil && writeFile(msgPre+streamStreamStateFile, buf) != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// Bail if no consumers requested.\n\tif !includeConsumers {\n\t\treturn\n\t}\n\n\t// Do consumers' state last.\n\tfs.cmu.RLock()\n\tcfs := fs.cfs\n\tfs.cmu.RUnlock()\n\n\tfor _, cs := range cfs {\n\t\to, ok := cs.(*consumerFileStore)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\to.mu.Lock()\n\t\t// Grab our general meta data.\n\t\t// We do this now instead of pulling from files since they could be encrypted.\n\t\tmeta, err := json.Marshal(o.cfg)\n\t\tif err != nil {\n\t\t\to.mu.Unlock()\n\t\t\twriteErr(fmt.Sprintf(\"Could not gather consumer meta file for %q: %v\", o.name, err))\n\t\t\treturn\n\t\t}\n\t\to.hh.Reset()\n\t\to.hh.Write(meta)\n\t\tvar hb [highwayhash.Size64]byte\n\t\tsum := []byte(hex.EncodeToString(o.hh.Sum(hb[:0])))\n\n\t\t// We can have the running state directly encoded now.\n\t\tstate, err := o.encodeState()\n\t\tif err != nil {\n\t\t\to.mu.Unlock()\n\t\t\twriteErr(fmt.Sprintf(\"Could not encode consumer state for %q: %v\", o.name, err))\n\t\t\treturn\n\t\t}\n\t\todirPre := filepath.Join(consumerDir, o.name)\n\t\to.mu.Unlock()\n\n\t\t// Write all the consumer files.\n\t\tif writeFile(filepath.Join(odirPre, JetStreamMetaFile), meta) != nil {\n\t\t\treturn\n\t\t}\n\t\tif writeFile(filepath.Join(odirPre, JetStreamMetaFileSum), sum) != nil {\n\t\t\treturn\n\t\t}\n\t\twriteFile(filepath.Join(odirPre, consumerState), state)\n\t}\n}\n\n// Create a snapshot of this stream and its consumer's state along with messages.\nfunc (fs *fileStore) Snapshot(deadline time.Duration, checkMsgs, includeConsumers bool) (*SnapshotResult, error) {\n\tif fs.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tfs.mu.Lock()\n\n\t// Only allow one at a time.\n\tif fs.sips > 0 {\n\t\tfs.mu.Unlock()\n\t\treturn nil, ErrStoreSnapshotInProgress\n\t}\n\t// Mark us as snapshotting\n\tfs.sips += 1\n\tfs.mu.Unlock()\n\n\tif checkMsgs {\n\t\tld := fs.checkMsgs()\n\t\tif ld != nil && len(ld.Msgs) > 0 {\n\t\t\treturn nil, fmt.Errorf(\"snapshot check detected %d bad messages\", len(ld.Msgs))\n\t\t}\n\t}\n\n\tpr, pw := net.Pipe()\n\n\t// Set a write deadline here to protect ourselves.\n\tif deadline > 0 {\n\t\tpw.SetWriteDeadline(time.Now().Add(deadline))\n\t}\n\n\t// We can add to our stream while snapshotting but not \"user\" delete anything.\n\tvar state StreamState\n\tfs.FastState(&state)\n\n\t// Stream in separate Go routine.\n\terrCh := make(chan string, 1)\n\tgo fs.streamSnapshot(pw, includeConsumers, errCh)\n\n\treturn &SnapshotResult{pr, state, errCh}, nil\n}\n\n// Helper to return the config.\nfunc (fs *fileStore) fileStoreConfig() FileStoreConfig {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\treturn fs.fcfg\n}\n\n// Read lock all existing message blocks.\n// Lock held on entry.\nfunc (fs *fileStore) readLockAllMsgBlocks() {\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t}\n}\n\n// Read unlock all existing message blocks.\n// Lock held on entry.\nfunc (fs *fileStore) readUnlockAllMsgBlocks() {\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RUnlock()\n\t}\n}\n\n// Binary encoded state snapshot, >= v2.10 server.\nfunc (fs *fileStore) EncodedStreamState(failed uint64) ([]byte, error) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\t// Calculate deleted.\n\tvar numDeleted int64\n\tif fs.state.LastSeq > fs.state.FirstSeq {\n\t\tnumDeleted = int64(fs.state.LastSeq-fs.state.FirstSeq+1) - int64(fs.state.Msgs)\n\t\tif numDeleted < 0 {\n\t\t\tnumDeleted = 0\n\t\t}\n\t}\n\n\t// Encoded is Msgs, Bytes, FirstSeq, LastSeq, Failed, NumDeleted and optional DeletedBlocks\n\tvar buf [1024]byte\n\tbuf[0], buf[1] = streamStateMagic, streamStateVersion\n\tn := hdrLen\n\tn += binary.PutUvarint(buf[n:], fs.state.Msgs)\n\tn += binary.PutUvarint(buf[n:], fs.state.Bytes)\n\tn += binary.PutUvarint(buf[n:], fs.state.FirstSeq)\n\tn += binary.PutUvarint(buf[n:], fs.state.LastSeq)\n\tn += binary.PutUvarint(buf[n:], failed)\n\tn += binary.PutUvarint(buf[n:], uint64(numDeleted))\n\n\tb := buf[0:n]\n\n\tif numDeleted > 0 {\n\t\tvar scratch [4 * 1024]byte\n\n\t\tfs.readLockAllMsgBlocks()\n\t\tdefer fs.readUnlockAllMsgBlocks()\n\n\t\tfor _, db := range fs.deleteBlocks() {\n\t\t\tswitch db := db.(type) {\n\t\t\tcase *DeleteRange:\n\t\t\t\tfirst, _, num := db.State()\n\t\t\t\tscratch[0] = runLengthMagic\n\t\t\t\ti := 1\n\t\t\t\ti += binary.PutUvarint(scratch[i:], first)\n\t\t\t\ti += binary.PutUvarint(scratch[i:], num)\n\t\t\t\tb = append(b, scratch[0:i]...)\n\t\t\tcase *avl.SequenceSet:\n\t\t\t\tbuf := db.Encode(scratch[:0])\n\t\t\t\tb = append(b, buf...)\n\t\t\tdefault:\n\t\t\t\treturn nil, errors.New(\"no impl\")\n\t\t\t}\n\t\t}\n\t}\n\n\treturn b, nil\n}\n\n// deleteBlocks returns DeleteBlocks representing interior deletes\n// and gaps between blocks.\n// All blocks should be at least read locked.\nfunc (fs *fileStore) deleteBlocks() DeleteBlocks {\n\tvar dbs DeleteBlocks\n\tvar prevLast uint64\n\tvar prevRange *DeleteRange\n\tvar msgsSinceGap bool\n\n\tfor _, mb := range fs.blks {\n\t\t// Detect if we have a gap between these blocks.\n\t\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t\tif prevLast > 0 && prevLast+1 != fseq {\n\t\t\tgapSize := fseq - prevLast - 1\n\t\t\t// The previous DeleteRange can be extended\n\t\t\t// to include this gap, if there are no\n\t\t\t// blocks containing messages between the\n\t\t\t// two gaps.\n\t\t\tif prevRange != nil && !msgsSinceGap {\n\t\t\t\tprevRange.Num += gapSize\n\t\t\t} else {\n\t\t\t\tprevRange = &DeleteRange{\n\t\t\t\t\tFirst: prevLast + 1,\n\t\t\t\t\tNum: gapSize,\n\t\t\t\t}\n\t\t\t\tmsgsSinceGap = false\n\t\t\t\tdbs = append(dbs, prevRange)\n\t\t\t}\n\t\t}\n\t\tif mb.dmap.Size() > 0 {\n\t\t\tdbs = append(dbs, &mb.dmap)\n\t\t\tprevRange = nil\n\t\t}\n\t\tprevLast = atomic.LoadUint64(&mb.last.seq)\n\t\tmsgsSinceGap = msgsSinceGap || mb.msgs > 0\n\t}\n\treturn dbs\n}\n\n// deleteMap returns all interior deletes for each block based on the mb.dmap.\n// Specifically, this will not contain any deletes for blocks that have been removed.\n// This is useful to know whether a tombstone is still relevant and marked as deleted by an active block.\n// No locks should be held.\nfunc (fs *fileStore) deleteMap() (dmap avl.SequenceSet) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tfs.readLockAllMsgBlocks()\n\tdefer fs.readUnlockAllMsgBlocks()\n\n\tfor _, mb := range fs.blks {\n\t\tif mb.dmap.Size() > 0 {\n\t\t\tmb.dmap.Range(func(seq uint64) bool {\n\t\t\t\tdmap.Insert(seq)\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t}\n\treturn dmap\n}\n\n// SyncDeleted will make sure this stream has same deleted state as dbs.\n// This will only process deleted state within our current state.\nfunc (fs *fileStore) SyncDeleted(dbs DeleteBlocks) error {\n\tif fs.isClosed() {\n\t\treturn ErrStoreClosed\n\t}\n\n\tif len(dbs) == 0 {\n\t\treturn nil\n\t}\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Always return previous write errors.\n\tif err := fs.werr; err != nil {\n\t\treturn err\n\t}\n\n\tlseq := fs.state.LastSeq\n\tvar needsCheck DeleteBlocks\n\n\tfs.readLockAllMsgBlocks()\n\tmdbs := fs.deleteBlocks()\n\tfor i, db := range dbs {\n\t\tfirst, last, num := db.State()\n\t\t// If the block is same as what we have we can skip.\n\t\tif i < len(mdbs) {\n\t\t\teFirst, eLast, eNum := mdbs[i].State()\n\t\t\tif first == eFirst && last == eLast && num == eNum {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else if first > lseq {\n\t\t\t// Skip blocks not applicable to our current state.\n\t\t\tcontinue\n\t\t}\n\t\t// Need to insert these.\n\t\tneedsCheck = append(needsCheck, db)\n\t}\n\tfs.readUnlockAllMsgBlocks()\n\n\tfor _, db := range needsCheck {\n\t\tvar err error\n\t\tif dr, ok := db.(*DeleteRange); ok {\n\t\t\tfirst, last, _ := dr.State()\n\t\t\terr = fs.removeMsgsInRange(first, last, true)\n\t\t} else {\n\t\t\tdb.Range(func(dseq uint64) bool {\n\t\t\t\t_, err = fs.removeMsg(dseq, false, true, false)\n\t\t\t\t// Can continue safely if the message doesn't exist.\n\t\t\t\tif err == ErrStoreEOF || err == ErrStoreMsgNotFound {\n\t\t\t\t\terr = nil\n\t\t\t\t}\n\t\t\t\treturn err == nil\n\t\t\t})\n\t\t}\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Consumers\n////////////////////////////////////////////////////////////////////////////////\n\ntype consumerFileStore struct {\n\tmu sync.Mutex\n\tfs *fileStore\n\tcfg *FileConsumerInfo\n\tprf keyGen\n\taek cipher.AEAD\n\tname string\n\todir string\n\tifn string\n\thh *highwayhash.Digest64\n\tstate ConsumerState\n\tfch chan struct{}\n\tqch chan struct{}\n\tflusher bool\n\twriting bool\n\tdirty bool\n\tclosed bool\n}\n\nfunc (fs *fileStore) ConsumerStore(name string, created time.Time, cfg *ConsumerConfig) (ConsumerStore, error) {\n\tif fs == nil {\n\t\treturn nil, fmt.Errorf(\"filestore is nil\")\n\t}\n\tif fs.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\tif cfg == nil || name == _EMPTY_ {\n\t\treturn nil, fmt.Errorf(\"bad consumer config\")\n\t}\n\n\t// We now allow overrides from a stream being a filestore type and forcing a consumer to be memory store.\n\tif cfg.MemoryStorage {\n\t\t// Create directly here.\n\t\to := &consumerMemStore{ms: fs, cfg: *cfg}\n\t\tif err := fs.AddConsumer(o); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn o, nil\n\t}\n\n\todir := filepath.Join(fs.fcfg.StoreDir, consumerDir, name)\n\tif err := os.MkdirAll(odir, defaultDirPerms); err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create consumer directory - %v\", err)\n\t}\n\tcsi := &FileConsumerInfo{Name: name, Created: created, ConsumerConfig: *cfg}\n\to := &consumerFileStore{\n\t\tfs: fs,\n\t\tcfg: csi,\n\t\tprf: fs.prf,\n\t\tname: name,\n\t\todir: odir,\n\t\tifn: filepath.Join(odir, consumerState),\n\t}\n\tkey := sha256.Sum256([]byte(fs.cfg.Name + \"/\" + name))\n\thh, err := highwayhash.NewDigest64(key[:])\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create hash: %v\", err)\n\t}\n\to.hh = hh\n\n\t// Check for encryption.\n\tif o.prf != nil {\n\t\tif ekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey)); err == nil {\n\t\t\tif len(ekey) < minBlkKeySize {\n\t\t\t\treturn nil, errBadKeySize\n\t\t\t}\n\t\t\t// Recover key encryption key.\n\t\t\trb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tsc := fs.fcfg.Cipher\n\t\t\tkek, err := genEncryptionKey(sc, rb)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tns := kek.NonceSize()\n\t\t\tnonce := ekey[:ns]\n\t\t\tseed, err := kek.Open(nil, nonce, ekey[ns:], nil)\n\t\t\tif err != nil {\n\t\t\t\t// We may be here on a cipher conversion, so attempt to convert.\n\t\t\t\tif err = o.convertCipher(); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\to.aek, err = genEncryptionKey(sc, seed)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Track if we are creating the directory so that we can clean up if we encounter an error.\n\tvar didCreate bool\n\n\t// Write our meta data iff does not exist.\n\tmeta := filepath.Join(odir, JetStreamMetaFile)\n\tif _, err := os.Stat(meta); err != nil && os.IsNotExist(err) {\n\t\tdidCreate = true\n\t\tif err := o.writeConsumerMeta(); err != nil {\n\t\t\t_ = os.RemoveAll(odir)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// If we expect to be encrypted check that what we are restoring is not plaintext.\n\t// This can happen on snapshot restores or conversions.\n\tif o.prf != nil {\n\t\tkeyFile := filepath.Join(odir, JetStreamMetaFileKey)\n\t\tif _, err := os.Stat(keyFile); err != nil && os.IsNotExist(err) {\n\t\t\tif err := o.writeConsumerMeta(); err != nil {\n\t\t\t\tif didCreate {\n\t\t\t\t\t_ = os.RemoveAll(odir)\n\t\t\t\t}\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// Redo the state file as well here if we have one and we can tell it was plaintext.\n\t\t\tif buf, err := os.ReadFile(o.ifn); err == nil {\n\t\t\t\tif _, err := decodeConsumerState(buf); err == nil {\n\t\t\t\t\tstate, err := o.encryptState(buf)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t}\n\t\t\t\t\terr = fs.writeFileWithOptionalSync(o.ifn, state, defaultFilePerms)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif didCreate {\n\t\t\t\t\t\t\t_ = os.RemoveAll(odir)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Create channels to control our flush go routine.\n\to.fch = make(chan struct{}, 1)\n\to.qch = make(chan struct{})\n\n\t// Make sure to load in our state from disk if needed.\n\tif err = o.loadState(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Assign to filestore.\n\tif err = fs.AddConsumer(o); err != nil {\n\t\treturn nil, err\n\t}\n\n\tgo o.flushLoop(o.fch, o.qch)\n\treturn o, nil\n}\n\nfunc (o *consumerFileStore) convertCipher() error {\n\tfs := o.fs\n\todir := filepath.Join(fs.fcfg.StoreDir, consumerDir, o.name)\n\n\tekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey))\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(ekey) < minBlkKeySize {\n\t\treturn errBadKeySize\n\t}\n\t// Recover key encryption key.\n\trb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Do these in reverse since converting.\n\tsc := fs.fcfg.Cipher\n\tosc := AES\n\tif sc == AES {\n\t\tosc = ChaCha\n\t}\n\tkek, err := genEncryptionKey(osc, rb)\n\tif err != nil {\n\t\treturn err\n\t}\n\tns := kek.NonceSize()\n\tnonce := ekey[:ns]\n\tseed, err := kek.Open(nil, nonce, ekey[ns:], nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\taek, err := genEncryptionKey(osc, seed)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Now read in and decode our state using the old cipher.\n\tbuf, err := os.ReadFile(o.ifn)\n\tif err != nil {\n\t\treturn err\n\t}\n\tbuf, err = aek.Open(nil, buf[:ns], buf[ns:], nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Since we are here we recovered our old state.\n\t// Now write our meta, which will generate the new keys with the new cipher.\n\tif err := o.writeConsumerMeta(); err != nil {\n\t\treturn err\n\t}\n\n\t// Now write out or state with the new cipher.\n\treturn o.writeState(buf)\n}\n\n// Kick flusher for this consumer.\n// Lock should be held.\nfunc (o *consumerFileStore) kickFlusher() {\n\tif o.fch != nil {\n\t\tselect {\n\t\tcase o.fch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n\to.dirty = true\n}\n\n// Set in flusher status\nfunc (o *consumerFileStore) setInFlusher() {\n\to.mu.Lock()\n\to.flusher = true\n\to.mu.Unlock()\n}\n\n// Clear in flusher status\nfunc (o *consumerFileStore) clearInFlusher() {\n\to.mu.Lock()\n\to.flusher = false\n\to.mu.Unlock()\n}\n\n// Report in flusher status\nfunc (o *consumerFileStore) inFlusher() bool {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.flusher\n}\n\n// flushLoop watches for consumer updates and the quit channel.\nfunc (o *consumerFileStore) flushLoop(fch, qch chan struct{}) {\n\n\to.setInFlusher()\n\tdefer o.clearInFlusher()\n\n\t// Maintain approximately 10 updates per second per consumer under load.\n\tconst minTime = 100 * time.Millisecond\n\tvar lastWrite time.Time\n\tvar dt *time.Timer\n\n\tsetDelayTimer := func(addWait time.Duration) {\n\t\tif dt == nil {\n\t\t\tdt = time.NewTimer(addWait)\n\t\t\treturn\n\t\t}\n\t\tif !dt.Stop() {\n\t\t\tselect {\n\t\t\tcase <-dt.C:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\tdt.Reset(addWait)\n\t}\n\n\tfor {\n\t\tselect {\n\t\tcase <-fch:\n\t\t\tif ts := time.Since(lastWrite); ts < minTime {\n\t\t\t\tsetDelayTimer(minTime - ts)\n\t\t\t\tselect {\n\t\t\t\tcase <-dt.C:\n\t\t\t\tcase <-qch:\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\to.mu.Lock()\n\t\t\tif o.closed {\n\t\t\t\to.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tbuf, err := o.encodeState()\n\t\t\to.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// TODO(dlc) - if we error should start failing upwards.\n\t\t\tif err := o.writeState(buf); err == nil {\n\t\t\t\tlastWrite = time.Now()\n\t\t\t}\n\t\tcase <-qch:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// SetStarting sets our starting stream sequence.\nfunc (o *consumerFileStore) SetStarting(sseq uint64) error {\n\to.mu.Lock()\n\to.state.Delivered.Stream = sseq\n\to.state.AckFloor.Stream = sseq\n\tbuf, err := o.encodeState()\n\to.mu.Unlock()\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn o.writeState(buf)\n}\n\n// UpdateStarting updates our starting stream sequence.\nfunc (o *consumerFileStore) UpdateStarting(sseq uint64) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif sseq > o.state.Delivered.Stream {\n\t\to.state.Delivered.Stream = sseq\n\t\t// For AckNone just update delivered and ackfloor at the same time.\n\t\tif o.cfg.AckPolicy == AckNone {\n\t\t\to.state.AckFloor.Stream = sseq\n\t\t}\n\t}\n\t// Make sure we flush to disk.\n\to.kickFlusher()\n}\n\n// Reset all values in the store, and reset the starting sequence.\nfunc (o *consumerFileStore) Reset(sseq uint64) error {\n\to.mu.Lock()\n\to.state = ConsumerState{}\n\to.mu.Unlock()\n\treturn o.SetStarting(sseq)\n}\n\n// HasState returns if this store has a recorded state.\nfunc (o *consumerFileStore) HasState() bool {\n\to.mu.Lock()\n\t// We have a running state, or stored on disk but not yet initialized.\n\tif o.state.Delivered.Consumer != 0 || o.state.Delivered.Stream != 0 {\n\t\to.mu.Unlock()\n\t\treturn true\n\t}\n\t_, err := os.Stat(o.ifn)\n\to.mu.Unlock()\n\treturn err == nil\n}\n\n// UpdateDelivered is called whenever a new message has been delivered.\nfunc (o *consumerFileStore) UpdateDelivered(dseq, sseq, dc uint64, ts int64) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif dc != 1 && o.cfg.AckPolicy == AckNone {\n\t\treturn ErrNoAckPolicy\n\t}\n\n\t// On restarts the old leader may get a replay from the raft logs that are old.\n\tif dseq <= o.state.AckFloor.Consumer {\n\t\treturn nil\n\t}\n\n\t// See if we expect an ack for this.\n\tif o.cfg.AckPolicy != AckNone {\n\t\t// Need to create pending records here.\n\t\tif o.state.Pending == nil {\n\t\t\to.state.Pending = make(map[uint64]*Pending)\n\t\t}\n\t\tvar p *Pending\n\t\t// Check for an update to a message already delivered.\n\t\tif sseq <= o.state.Delivered.Stream {\n\t\t\tif p = o.state.Pending[sseq]; p != nil {\n\t\t\t\t// Do not update p.Sequence, that should be the original delivery sequence.\n\t\t\t\tp.Timestamp = ts\n\t\t\t}\n\t\t} else {\n\t\t\t// Add to pending.\n\t\t\to.state.Pending[sseq] = &Pending{dseq, ts}\n\t\t}\n\t\t// Update delivered as needed.\n\t\tif dseq > o.state.Delivered.Consumer {\n\t\t\to.state.Delivered.Consumer = dseq\n\t\t}\n\t\tif sseq > o.state.Delivered.Stream {\n\t\t\to.state.Delivered.Stream = sseq\n\t\t}\n\n\t\tif dc > 1 {\n\t\t\tif maxdc := uint64(o.cfg.MaxDeliver); maxdc > 0 && dc > maxdc {\n\t\t\t\t// Make sure to remove from pending.\n\t\t\t\tdelete(o.state.Pending, sseq)\n\t\t\t}\n\t\t\tif o.state.Redelivered == nil {\n\t\t\t\to.state.Redelivered = make(map[uint64]uint64)\n\t\t\t}\n\t\t\t// Only update if greater than what we already have.\n\t\t\tif o.state.Redelivered[sseq] < dc-1 {\n\t\t\t\to.state.Redelivered[sseq] = dc - 1\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// For AckNone just update delivered and ackfloor at the same time.\n\t\tif dseq > o.state.Delivered.Consumer {\n\t\t\to.state.Delivered.Consumer = dseq\n\t\t\to.state.AckFloor.Consumer = dseq\n\t\t}\n\t\tif sseq > o.state.Delivered.Stream {\n\t\t\to.state.Delivered.Stream = sseq\n\t\t\to.state.AckFloor.Stream = sseq\n\t\t}\n\t}\n\t// Make sure we flush to disk.\n\to.kickFlusher()\n\n\treturn nil\n}\n\n// UpdateAcks is called whenever a consumer with explicit ack or ack all acks a message.\nfunc (o *consumerFileStore) UpdateAcks(dseq, sseq uint64) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.cfg.AckPolicy == AckNone {\n\t\treturn ErrNoAckPolicy\n\t}\n\n\t// On restarts the old leader may get a replay from the raft logs that are old.\n\tif dseq <= o.state.AckFloor.Consumer {\n\t\treturn nil\n\t}\n\n\tif len(o.state.Pending) == 0 || o.state.Pending[sseq] == nil {\n\t\tdelete(o.state.Redelivered, sseq)\n\t\treturn ErrStoreMsgNotFound\n\t}\n\n\t// Check for AckAll here.\n\tif o.cfg.AckPolicy == AckAll {\n\t\tsgap := sseq - o.state.AckFloor.Stream\n\t\to.state.AckFloor.Consumer = dseq\n\t\to.state.AckFloor.Stream = sseq\n\t\tif sgap > uint64(len(o.state.Pending)) {\n\t\t\tfor seq := range o.state.Pending {\n\t\t\t\tif seq <= sseq {\n\t\t\t\t\tdelete(o.state.Pending, seq)\n\t\t\t\t\tdelete(o.state.Redelivered, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := sseq; seq > sseq-sgap && len(o.state.Pending) > 0; seq-- {\n\t\t\t\tdelete(o.state.Pending, seq)\n\t\t\t\tdelete(o.state.Redelivered, seq)\n\t\t\t}\n\t\t}\n\t\to.kickFlusher()\n\t\treturn nil\n\t}\n\n\t// AckExplicit\n\n\t// First delete from our pending state.\n\tif p, ok := o.state.Pending[sseq]; ok {\n\t\tdelete(o.state.Pending, sseq)\n\t\tif dseq > p.Sequence && p.Sequence > 0 {\n\t\t\tdseq = p.Sequence // Use the original.\n\t\t}\n\t}\n\tif len(o.state.Pending) == 0 {\n\t\to.state.AckFloor.Consumer = o.state.Delivered.Consumer\n\t\to.state.AckFloor.Stream = o.state.Delivered.Stream\n\t} else if dseq == o.state.AckFloor.Consumer+1 {\n\t\to.state.AckFloor.Consumer = dseq\n\t\to.state.AckFloor.Stream = sseq\n\n\t\tif o.state.Delivered.Consumer > dseq {\n\t\t\tfor ss := sseq + 1; ss <= o.state.Delivered.Stream; ss++ {\n\t\t\t\tif p, ok := o.state.Pending[ss]; ok {\n\t\t\t\t\tif p.Sequence > 0 {\n\t\t\t\t\t\to.state.AckFloor.Consumer = p.Sequence - 1\n\t\t\t\t\t\to.state.AckFloor.Stream = ss - 1\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// We do these regardless.\n\tdelete(o.state.Redelivered, sseq)\n\n\to.kickFlusher()\n\treturn nil\n}\n\nconst seqsHdrSize = 6*binary.MaxVarintLen64 + hdrLen\n\n// Encode our consumer state, version 2.\n// Lock should be held.\n\nfunc (o *consumerFileStore) EncodedState() ([]byte, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.encodeState()\n}\n\nfunc (o *consumerFileStore) encodeState() ([]byte, error) {\n\t// Grab reference to state, but make sure we load in if needed, so do not reference o.state directly.\n\tstate, err := o.stateWithCopyLocked(false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn encodeConsumerState(state), nil\n}\n\nfunc (o *consumerFileStore) UpdateConfig(cfg *ConsumerConfig) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// This is mostly unchecked here. We are assuming the upper layers have done sanity checking.\n\tcsi := o.cfg\n\tcsi.ConsumerConfig = *cfg\n\n\treturn o.writeConsumerMeta()\n}\n\nfunc (o *consumerFileStore) Update(state *ConsumerState) error {\n\t// Sanity checks.\n\tif state.AckFloor.Consumer > state.Delivered.Consumer {\n\t\treturn fmt.Errorf(\"bad ack floor for consumer\")\n\t}\n\tif state.AckFloor.Stream > state.Delivered.Stream {\n\t\treturn fmt.Errorf(\"bad ack floor for stream\")\n\t}\n\n\t// Copy to our state.\n\tvar pending map[uint64]*Pending\n\tvar redelivered map[uint64]uint64\n\tif len(state.Pending) > 0 {\n\t\tpending = make(map[uint64]*Pending, len(state.Pending))\n\t\tfor seq, p := range state.Pending {\n\t\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t\t\tif seq <= state.AckFloor.Stream || seq > state.Delivered.Stream {\n\t\t\t\treturn fmt.Errorf(\"bad pending entry, sequence [%d] out of range\", seq)\n\t\t\t}\n\t\t}\n\t}\n\tif len(state.Redelivered) > 0 {\n\t\tredelivered = make(map[uint64]uint64, len(state.Redelivered))\n\t\tfor seq, dc := range state.Redelivered {\n\t\t\tredelivered[seq] = dc\n\t\t}\n\t}\n\n\t// Replace our state.\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Check to see if this is an outdated update.\n\tif state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream {\n\t\treturn ErrStoreOldUpdate\n\t}\n\n\to.state.Delivered = state.Delivered\n\to.state.AckFloor = state.AckFloor\n\to.state.Pending = pending\n\to.state.Redelivered = redelivered\n\n\to.kickFlusher()\n\n\treturn nil\n}\n\n// ForceUpdate updates the consumer state without the backwards check.\n// This is used during recovery when we need to reset the consumer to an earlier sequence.\nfunc (o *consumerFileStore) ForceUpdate(state *ConsumerState) error {\n\t// Sanity checks.\n\tif state.AckFloor.Consumer > state.Delivered.Consumer {\n\t\treturn fmt.Errorf(\"bad ack floor for consumer\")\n\t}\n\tif state.AckFloor.Stream > state.Delivered.Stream {\n\t\treturn fmt.Errorf(\"bad ack floor for stream\")\n\t}\n\n\t// Copy to our state.\n\tvar pending map[uint64]*Pending\n\tvar redelivered map[uint64]uint64\n\tif len(state.Pending) > 0 {\n\t\tpending = make(map[uint64]*Pending, len(state.Pending))\n\t\tfor seq, p := range state.Pending {\n\t\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t\t\tif seq <= state.AckFloor.Stream || seq > state.Delivered.Stream {\n\t\t\t\treturn fmt.Errorf(\"bad pending entry, sequence [%d] out of range\", seq)\n\t\t\t}\n\t\t}\n\t}\n\tif len(state.Redelivered) > 0 {\n\t\tredelivered = make(map[uint64]uint64, len(state.Redelivered))\n\t\tfor seq, dc := range state.Redelivered {\n\t\t\tredelivered[seq] = dc\n\t\t}\n\t}\n\n\t// Replace our state.\n\to.mu.Lock()\n\to.state.Delivered = state.Delivered\n\to.state.AckFloor = state.AckFloor\n\to.state.Pending = pending\n\to.state.Redelivered = redelivered\n\tbuf, err := o.encodeState()\n\to.mu.Unlock()\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn o.writeState(buf)\n}\n\n// Will encrypt the state with our asset key. Will be a no-op if encryption not enabled.\n// Lock should be held.\nfunc (o *consumerFileStore) encryptState(buf []byte) ([]byte, error) {\n\tif o.aek == nil {\n\t\treturn buf, nil\n\t}\n\t// TODO(dlc) - Optimize on space usage a bit?\n\tnonce := make([]byte, o.aek.NonceSize(), o.aek.NonceSize()+len(buf)+o.aek.Overhead())\n\tif n, err := rand.Read(nonce); err != nil {\n\t\treturn nil, err\n\t} else if n != len(nonce) {\n\t\treturn nil, fmt.Errorf(\"not enough nonce bytes read (%d != %d)\", n, len(nonce))\n\t}\n\treturn o.aek.Seal(nonce, nonce, buf, nil), nil\n}\n\n// Used to limit number of disk IO calls in flight since they could all be blocking an OS thread.\n// https://github.com/nats-io/nats-server/issues/2742\nvar dios chan struct{}\n\n// Used to setup our simplistic counting semaphore using buffered channels.\n// golang.org's semaphore seemed a bit heavy.\nfunc init() {\n\t// Limit ourselves to a sensible number of blocking I/O calls. Range between\n\t// 4-16 concurrent disk I/Os based on CPU cores, or 50% of cores if greater\n\t// than 32 cores.\n\tmp := runtime.GOMAXPROCS(-1)\n\tnIO := min(16, max(4, mp))\n\tif mp > 32 {\n\t\t// If the system has more than 32 cores then limit dios to 50% of cores.\n\t\tnIO = max(16, min(mp, mp/2))\n\t}\n\tdios = make(chan struct{}, nIO)\n\t// Fill it up to start.\n\tfor i := 0; i < nIO; i++ {\n\t\tdios <- struct{}{}\n\t}\n}\n\nfunc (o *consumerFileStore) writeState(buf []byte) error {\n\t// Check if we have the index file open.\n\to.mu.Lock()\n\tif o.writing || len(buf) == 0 {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t// Check on encryption.\n\tif o.aek != nil {\n\t\tvar err error\n\t\tif buf, err = o.encryptState(buf); err != nil {\n\t\t\to.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t}\n\n\to.writing = true\n\to.dirty = false\n\tifn := o.ifn\n\to.mu.Unlock()\n\n\t// Lock not held here but we do limit number of outstanding calls that could block OS threads.\n\terr := o.fs.writeFileWithOptionalSync(ifn, buf, defaultFilePerms)\n\n\to.mu.Lock()\n\tif err != nil {\n\t\to.dirty = true\n\t}\n\to.writing = false\n\to.mu.Unlock()\n\n\treturn err\n}\n\n// Will upodate the config. Only used when recovering ephemerals.\nfunc (o *consumerFileStore) updateConfig(cfg ConsumerConfig) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\to.cfg = &FileConsumerInfo{ConsumerConfig: cfg}\n\treturn o.writeConsumerMeta()\n}\n\n// Write out the consumer meta data, i.e. state.\n// Lock should be held.\nfunc (cfs *consumerFileStore) writeConsumerMeta() error {\n\tmeta := filepath.Join(cfs.odir, JetStreamMetaFile)\n\tif _, err := os.Stat(meta); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\n\tif cfs.prf != nil && cfs.aek == nil {\n\t\tfs := cfs.fs\n\t\tkey, _, _, encrypted, err := fs.genEncryptionKeys(fs.cfg.Name + tsep + cfs.name)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcfs.aek = key\n\t\tkeyFile := filepath.Join(cfs.odir, JetStreamMetaFileKey)\n\t\tif _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) {\n\t\t\treturn err\n\t\t}\n\t\terr = cfs.fs.writeFileWithOptionalSync(keyFile, encrypted, defaultFilePerms)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tb, err := json.Marshal(cfs.cfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Encrypt if needed.\n\tif cfs.aek != nil {\n\t\tnonce := make([]byte, cfs.aek.NonceSize(), cfs.aek.NonceSize()+len(b)+cfs.aek.Overhead())\n\t\tif n, err := rand.Read(nonce); err != nil {\n\t\t\treturn err\n\t\t} else if n != len(nonce) {\n\t\t\treturn fmt.Errorf(\"not enough nonce bytes read (%d != %d)\", n, len(nonce))\n\t\t}\n\t\tb = cfs.aek.Seal(nonce, nonce, b, nil)\n\t}\n\n\terr = cfs.fs.writeFileWithOptionalSync(meta, b, defaultFilePerms)\n\tif err != nil {\n\t\treturn err\n\t}\n\tcfs.hh.Reset()\n\tcfs.hh.Write(b)\n\tvar hb [highwayhash.Size64]byte\n\tchecksum := hex.EncodeToString(cfs.hh.Sum(hb[:0]))\n\tsum := filepath.Join(cfs.odir, JetStreamMetaFileSum)\n\n\terr = cfs.fs.writeFileWithOptionalSync(sum, []byte(checksum), defaultFilePerms)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// Consumer version.\nfunc checkConsumerHeader(hdr []byte) (uint8, error) {\n\tif len(hdr) < 2 || hdr[0] != magic {\n\t\treturn 0, errCorruptState\n\t}\n\tversion := hdr[1]\n\tswitch version {\n\tcase 1, 2:\n\t\treturn version, nil\n\t}\n\treturn 0, fmt.Errorf(\"unsupported version: %d\", version)\n}\n\nfunc (o *consumerFileStore) copyPending() map[uint64]*Pending {\n\tpending := make(map[uint64]*Pending, len(o.state.Pending))\n\tfor seq, p := range o.state.Pending {\n\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t}\n\treturn pending\n}\n\nfunc (o *consumerFileStore) copyRedelivered() map[uint64]uint64 {\n\tredelivered := make(map[uint64]uint64, len(o.state.Redelivered))\n\tfor seq, dc := range o.state.Redelivered {\n\t\tredelivered[seq] = dc\n\t}\n\treturn redelivered\n}\n\n// Type returns the type of the underlying store.\nfunc (o *consumerFileStore) Type() StorageType { return FileStorage }\n\n// State retrieves the state from the state file.\n// This is not expected to be called in high performance code, only on startup.\nfunc (o *consumerFileStore) State() (*ConsumerState, error) {\n\treturn o.stateWithCopy(true)\n}\n\n// This will not copy pending or redelivered, so should only be done under the\n// consumer owner's lock.\nfunc (o *consumerFileStore) BorrowState() (*ConsumerState, error) {\n\treturn o.stateWithCopy(false)\n}\n\nfunc (o *consumerFileStore) stateWithCopy(doCopy bool) (*ConsumerState, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\treturn o.stateWithCopyLocked(doCopy)\n}\n\n// Lock should be held.\nfunc (o *consumerFileStore) stateWithCopyLocked(doCopy bool) (*ConsumerState, error) {\n\tif o.closed {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tstate := &ConsumerState{}\n\n\t// See if we have a running state or if we need to read in from disk.\n\tif o.state.Delivered.Consumer != 0 || o.state.Delivered.Stream != 0 {\n\t\tstate.Delivered = o.state.Delivered\n\t\tstate.AckFloor = o.state.AckFloor\n\t\tif len(o.state.Pending) > 0 {\n\t\t\tif doCopy {\n\t\t\t\tstate.Pending = o.copyPending()\n\t\t\t} else {\n\t\t\t\tstate.Pending = o.state.Pending\n\t\t\t}\n\t\t}\n\t\tif len(o.state.Redelivered) > 0 {\n\t\t\tif doCopy {\n\t\t\t\tstate.Redelivered = o.copyRedelivered()\n\t\t\t} else {\n\t\t\t\tstate.Redelivered = o.state.Redelivered\n\t\t\t}\n\t\t}\n\t\treturn state, nil\n\t}\n\n\t// Read the state in here from disk..\n\t<-dios\n\tbuf, err := os.ReadFile(o.ifn)\n\tdios <- struct{}{}\n\n\tif err != nil && !os.IsNotExist(err) {\n\t\treturn nil, err\n\t}\n\n\tif len(buf) == 0 {\n\t\treturn state, nil\n\t}\n\n\t// Check on encryption.\n\tif o.aek != nil {\n\t\tns := o.aek.NonceSize()\n\t\tbuf, err = o.aek.Open(nil, buf[:ns], buf[ns:], nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tstate, err = decodeConsumerState(buf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Copy this state into our own.\n\to.state.Delivered = state.Delivered\n\to.state.AckFloor = state.AckFloor\n\tif len(state.Pending) > 0 {\n\t\tif doCopy {\n\t\t\to.state.Pending = make(map[uint64]*Pending, len(state.Pending))\n\t\t\tfor seq, p := range state.Pending {\n\t\t\t\to.state.Pending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t\t\t}\n\t\t} else {\n\t\t\to.state.Pending = state.Pending\n\t\t}\n\t}\n\tif len(state.Redelivered) > 0 {\n\t\tif doCopy {\n\t\t\to.state.Redelivered = make(map[uint64]uint64, len(state.Redelivered))\n\t\t\tfor seq, dc := range state.Redelivered {\n\t\t\t\to.state.Redelivered[seq] = dc\n\t\t\t}\n\t\t} else {\n\t\t\to.state.Redelivered = state.Redelivered\n\t\t}\n\t}\n\n\treturn state, nil\n}\n\n// Lock should be held. Called at startup.\nfunc (o *consumerFileStore) loadState() error {\n\tif _, err := os.Stat(o.ifn); err == nil {\n\t\t// This will load our state in from disk.\n\t\t_, err = o.stateWithCopyLocked(false)\n\t\treturn err\n\t} else if os.IsNotExist(err) {\n\t\treturn nil\n\t} else {\n\t\treturn err\n\t}\n}\n\n// Decode consumer state.\nfunc decodeConsumerState(buf []byte) (*ConsumerState, error) {\n\tversion, err := checkConsumerHeader(buf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbi := hdrLen\n\t// Helpers, will set i to -1 on error.\n\treadSeq := func() uint64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tseq, n := binary.Uvarint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn 0\n\t\t}\n\t\tbi += n\n\t\treturn seq\n\t}\n\treadTimeStamp := func() int64 {\n\t\tif bi < 0 {\n\t\t\treturn 0\n\t\t}\n\t\tts, n := binary.Varint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn -1\n\t\t}\n\t\tbi += n\n\t\treturn ts\n\t}\n\t// Just for clarity below.\n\treadLen := readSeq\n\treadCount := readSeq\n\n\tstate := &ConsumerState{}\n\tstate.AckFloor.Consumer = readSeq()\n\tstate.AckFloor.Stream = readSeq()\n\tstate.Delivered.Consumer = readSeq()\n\tstate.Delivered.Stream = readSeq()\n\n\tif bi == -1 {\n\t\treturn nil, errCorruptState\n\t}\n\tif version == 1 {\n\t\t// Adjust back. Version 1 also stored delivered as next to be delivered,\n\t\t// so adjust that back down here.\n\t\tif state.AckFloor.Consumer > 1 {\n\t\t\tstate.Delivered.Consumer += state.AckFloor.Consumer - 1\n\t\t}\n\t\tif state.AckFloor.Stream > 1 {\n\t\t\tstate.Delivered.Stream += state.AckFloor.Stream - 1\n\t\t}\n\t}\n\n\t// Protect ourselves against rolling backwards.\n\tconst hbit = 1 << 63\n\tif state.AckFloor.Stream&hbit != 0 || state.Delivered.Stream&hbit != 0 {\n\t\treturn nil, errCorruptState\n\t}\n\n\t// We have additional stuff.\n\tif numPending := readLen(); numPending > 0 {\n\t\tmints := readTimeStamp()\n\t\tstate.Pending = make(map[uint64]*Pending, numPending)\n\t\tfor i := 0; i < int(numPending); i++ {\n\t\t\tsseq := readSeq()\n\t\t\tvar dseq uint64\n\t\t\tif version == 2 {\n\t\t\t\tdseq = readSeq()\n\t\t\t}\n\t\t\tts := readTimeStamp()\n\t\t\t// Check the state machine for corruption, not the value which could be -1.\n\t\t\tif bi == -1 {\n\t\t\t\treturn nil, errCorruptState\n\t\t\t}\n\t\t\t// Adjust seq back.\n\t\t\tsseq += state.AckFloor.Stream\n\t\t\tif sseq == 0 {\n\t\t\t\treturn nil, errCorruptState\n\t\t\t}\n\t\t\tif version == 2 {\n\t\t\t\tdseq += state.AckFloor.Consumer\n\t\t\t}\n\t\t\t// Adjust the timestamp back.\n\t\t\tif version == 1 {\n\t\t\t\tts = (ts + mints) * int64(time.Second)\n\t\t\t} else {\n\t\t\t\tts = (mints - ts) * int64(time.Second)\n\t\t\t}\n\t\t\t// Store in pending.\n\t\t\tstate.Pending[sseq] = &Pending{dseq, ts}\n\t\t}\n\t}\n\n\t// We have redelivered entries here.\n\tif numRedelivered := readLen(); numRedelivered > 0 {\n\t\tstate.Redelivered = make(map[uint64]uint64, numRedelivered)\n\t\tfor i := 0; i < int(numRedelivered); i++ {\n\t\t\tif seq, n := readSeq(), readCount(); seq > 0 && n > 0 {\n\t\t\t\t// Adjust seq back.\n\t\t\t\tseq += state.AckFloor.Stream\n\t\t\t\tstate.Redelivered[seq] = n\n\t\t\t}\n\t\t}\n\t}\n\n\treturn state, nil\n}\n\n// Stop the processing of the consumers's state.\nfunc (o *consumerFileStore) Stop() error {\n\to.mu.Lock()\n\tif o.closed {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\tif o.qch != nil {\n\t\tclose(o.qch)\n\t\to.qch = nil\n\t}\n\n\tvar err error\n\tvar buf []byte\n\n\tif o.dirty {\n\t\t// Make sure to write this out..\n\t\tif buf, err = o.encodeState(); err == nil && len(buf) > 0 {\n\t\t\tif o.aek != nil {\n\t\t\t\tif buf, err = o.encryptState(buf); err != nil {\n\t\t\t\t\to.mu.Unlock()\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\to.odir = _EMPTY_\n\to.closed = true\n\tifn, fs := o.ifn, o.fs\n\to.mu.Unlock()\n\n\tif err = fs.RemoveConsumer(o); err != nil {\n\t\treturn err\n\t}\n\n\tif len(buf) > 0 {\n\t\to.waitOnFlusher()\n\t\terr = o.fs.writeFileWithOptionalSync(ifn, buf, defaultFilePerms)\n\t}\n\treturn err\n}\n\nfunc (o *consumerFileStore) waitOnFlusher() {\n\tif !o.inFlusher() {\n\t\treturn\n\t}\n\n\ttimeout := time.Now().Add(100 * time.Millisecond)\n\tfor time.Now().Before(timeout) {\n\t\tif !o.inFlusher() {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n}\n\n// Delete the consumer.\nfunc (o *consumerFileStore) Delete() error {\n\treturn o.delete(false)\n}\n\nfunc (o *consumerFileStore) StreamDelete() error {\n\treturn o.delete(true)\n}\n\nfunc (o *consumerFileStore) delete(streamDeleted bool) error {\n\to.mu.Lock()\n\tif o.closed {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\tif o.qch != nil {\n\t\tclose(o.qch)\n\t\to.qch = nil\n\t}\n\n\todir := o.odir\n\to.odir = _EMPTY_\n\to.closed = true\n\tfs := o.fs\n\to.mu.Unlock()\n\n\t// If our stream was not deleted this will remove the directories.\n\tif odir != _EMPTY_ && !streamDeleted {\n\t\tif err := removeAllWithRetry(odir); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif !streamDeleted {\n\t\tif err := fs.RemoveConsumer(o); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (fs *fileStore) AddConsumer(o ConsumerStore) error {\n\tfs.cmu.Lock()\n\tdefer fs.cmu.Unlock()\n\tfs.cfs = append(fs.cfs, o)\n\treturn nil\n}\n\nfunc (fs *fileStore) RemoveConsumer(o ConsumerStore) error {\n\tfs.cmu.Lock()\n\tdefer fs.cmu.Unlock()\n\tfor i, cfs := range fs.cfs {\n\t\tif o == cfs {\n\t\t\tfs.cfs = append(fs.cfs[:i], fs.cfs[i+1:]...)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn nil\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Compression\n////////////////////////////////////////////////////////////////////////////////\n\ntype CompressionInfo struct {\n\tAlgorithm StoreCompression\n\tOriginalSize uint64\n}\n\nfunc (c *CompressionInfo) MarshalMetadata() []byte {\n\tb := make([]byte, 14) // 4 + potentially up to 10 for uint64\n\tb[0], b[1], b[2] = 'c', 'm', 'p'\n\tb[3] = byte(c.Algorithm)\n\tn := binary.PutUvarint(b[4:], c.OriginalSize)\n\treturn b[:4+n]\n}\n\nfunc (c *CompressionInfo) UnmarshalMetadata(b []byte) (int, error) {\n\tc.Algorithm = NoCompression\n\tc.OriginalSize = 0\n\tif len(b) < 5 { // 4 + min 1 for uvarint uint64\n\t\treturn 0, nil\n\t}\n\tif b[0] != 'c' || b[1] != 'm' || b[2] != 'p' {\n\t\treturn 0, nil\n\t}\n\tvar n int\n\tc.Algorithm = StoreCompression(b[3])\n\tc.OriginalSize, n = binary.Uvarint(b[4:])\n\tif n <= 0 {\n\t\treturn 0, fmt.Errorf(\"metadata incomplete\")\n\t}\n\treturn 4 + n, nil\n}\n\nfunc (alg StoreCompression) Compress(buf []byte) ([]byte, error) {\n\tif len(buf) < checksumSize {\n\t\treturn nil, fmt.Errorf(\"uncompressed buffer is too short\")\n\t}\n\tbodyLen := int64(len(buf) - checksumSize)\n\tvar output bytes.Buffer\n\tvar writer io.WriteCloser\n\tswitch alg {\n\tcase NoCompression:\n\t\treturn buf, nil\n\tcase S2Compression:\n\t\twriter = s2.NewWriter(&output)\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"compression algorithm not known\")\n\t}\n\n\tinput := bytes.NewReader(buf[:bodyLen])\n\tchecksum := buf[bodyLen:]\n\n\t// Compress the block content, but don't compress the checksum.\n\t// We will preserve it at the end of the block as-is.\n\tif n, err := io.CopyN(writer, input, bodyLen); err != nil {\n\t\treturn nil, fmt.Errorf(\"error writing to compression writer: %w\", err)\n\t} else if n != bodyLen {\n\t\treturn nil, fmt.Errorf(\"short write on body (%d != %d)\", n, bodyLen)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\treturn nil, fmt.Errorf(\"error closing compression writer: %w\", err)\n\t}\n\n\t// Now add the checksum back onto the end of the block.\n\tif n, err := output.Write(checksum); err != nil {\n\t\treturn nil, fmt.Errorf(\"error writing checksum: %w\", err)\n\t} else if n != checksumSize {\n\t\treturn nil, fmt.Errorf(\"short write on checksum (%d != %d)\", n, checksumSize)\n\t}\n\n\treturn output.Bytes(), nil\n}\n\nfunc (alg StoreCompression) Decompress(buf []byte) ([]byte, error) {\n\tif len(buf) < checksumSize {\n\t\treturn nil, fmt.Errorf(\"compressed buffer is too short\")\n\t}\n\tbodyLen := int64(len(buf) - checksumSize)\n\tinput := bytes.NewReader(buf[:bodyLen])\n\n\tvar reader io.ReadCloser\n\tswitch alg {\n\tcase NoCompression:\n\t\treturn buf, nil\n\tcase S2Compression:\n\t\treader = io.NopCloser(s2.NewReader(input))\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"compression algorithm not known\")\n\t}\n\n\t// Decompress the block content. The checksum isn't compressed so\n\t// we can preserve it from the end of the block as-is.\n\tchecksum := buf[bodyLen:]\n\toutput, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading compression reader: %w\", err)\n\t}\n\toutput = append(output, checksum...)\n\n\treturn output, reader.Close()\n}\n\n// writeFileWithOptionalSync is equivalent to os.WriteFile() but optionally\n// sets O_SYNC on the open file if SyncAlways is set. The dios semaphore is\n// handled automatically by this function, so don't wrap calls to it in dios.\nfunc (fs *fileStore) writeFileWithOptionalSync(name string, data []byte, perm fs.FileMode) error {\n\treturn writeAtomically(name, data, perm, fs.fcfg.SyncAlways)\n}\n\nfunc writeFileWithSync(name string, data []byte, perm fs.FileMode) error {\n\treturn writeAtomically(name, data, perm, true)\n}\n\n// Windows does not support fsyncing directory metadata, it results in a panic, so\n// we need to skip doing this there.\nconst canFsyncDirectories = runtime.GOOS != \"windows\"\n\nfunc writeAtomically(name string, data []byte, perm fs.FileMode, sync bool) error {\n\ttmp := name + \".tmp\"\n\tflags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC\n\tif sync {\n\t\tflags = flags | os.O_SYNC\n\t}\n\t<-dios\n\tdefer func() {\n\t\tdios <- struct{}{}\n\t}()\n\tf, err := os.OpenFile(tmp, flags, perm)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif _, err := f.Write(data); err != nil {\n\t\t// Close fd, but ignore its error since write takes precedence.\n\t\t_ = f.Close()\n\t\t_ = os.Remove(tmp)\n\t\treturn err\n\t}\n\tif err := f.Close(); err != nil {\n\t\t_ = os.Remove(tmp)\n\t\treturn err\n\t}\n\tif err := os.Rename(tmp, name); err != nil {\n\t\t_ = os.Remove(tmp)\n\t\treturn err\n\t}\n\tif sync && canFsyncDirectories {\n\t\t// To ensure that the file rename was persisted on all filesystems,\n\t\t// also try to flush the directory metadata.\n\t\tvar d *os.File\n\t\tif d, err = os.Open(filepath.Dir(name)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err = d.Sync(); err != nil {\n\t\t\t// Close fd, but ignore its error since sync takes precedence.\n\t\t\t_ = d.Close()\n\t\t\treturn err\n\t\t}\n\t\tif err = d.Close(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "server/filestore_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_store_tests\n\npackage server\n\nimport (\n\t\"archive/tar\"\n\t\"bytes\"\n\t\"crypto/hmac\"\n\tcrand \"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"math\"\n\t\"math/bits\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/nats-server/v2/server/ats\"\n\t\"github.com/nats-io/nats-server/v2/server/avl\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc testFileStoreAllPermutations(t *testing.T, fn func(t *testing.T, fcfg FileStoreConfig)) {\n\tfor _, fcfg := range []FileStoreConfig{\n\t\t{Cipher: NoCipher, Compression: NoCompression},\n\t\t{Cipher: NoCipher, Compression: S2Compression},\n\t\t{Cipher: AES, Compression: NoCompression},\n\t\t{Cipher: AES, Compression: S2Compression},\n\t\t{Cipher: ChaCha, Compression: NoCompression},\n\t\t{Cipher: ChaCha, Compression: S2Compression},\n\t} {\n\t\tsubtestName := fmt.Sprintf(\"%s-%s\", fcfg.Cipher, fcfg.Compression)\n\t\tt.Run(subtestName, func(t *testing.T) {\n\t\t\tfcfg.StoreDir = t.TempDir()\n\t\t\tfn(t, fcfg)\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t})\n\t}\n}\n\nfunc prf(fcfg *FileStoreConfig) func(context []byte) ([]byte, error) {\n\tif fcfg.Cipher == NoCipher {\n\t\treturn nil\n\t}\n\treturn func(context []byte) ([]byte, error) {\n\t\th := hmac.New(sha256.New, []byte(\"dlc22\"))\n\t\tif _, err := h.Write(context); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn h.Sum(nil), nil\n\t}\n}\n\nfunc TestFileStoreBasics(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 1; i <= 5; i++ {\n\t\t\tnow := time.Now().UnixNano()\n\t\t\tif seq, ts, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t} else if seq != uint64(i) {\n\t\t\t\tt.Fatalf(\"Expected sequence to be %d, got %d\", i, seq)\n\t\t\t} else if ts < now || ts > now+int64(time.Millisecond) {\n\t\t\t\tt.Fatalf(\"Expected timestamp to be current, got %v\", ts-now)\n\t\t\t}\n\t\t}\n\n\t\tstate := fs.State()\n\t\tif state.Msgs != 5 {\n\t\t\tt.Fatalf(\"Expected 5 msgs, got %d\", state.Msgs)\n\t\t}\n\t\texpectedSize := 5 * fileStoreMsgSize(subj, nil, msg)\n\t\tif state.Bytes != expectedSize {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", expectedSize, state.Bytes)\n\t\t}\n\n\t\tvar smv StoreMsg\n\t\tsm, err := fs.LoadMsg(2, &smv)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t\t}\n\t\tif sm.subj != subj {\n\t\t\tt.Fatalf(\"Subjects don't match, original %q vs %q\", subj, sm.subj)\n\t\t}\n\t\tif !bytes.Equal(sm.msg, msg) {\n\t\t\tt.Fatalf(\"Msgs don't match, original %q vs %q\", msg, sm.msg)\n\t\t}\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t\t}\n\n\t\tremove := func(seq, expectedMsgs uint64) {\n\t\t\tt.Helper()\n\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Got an error on remove of %d: %v\", seq, err)\n\t\t\t}\n\t\t\tif !removed {\n\t\t\t\tt.Fatalf(\"Expected remove to return true for %d\", seq)\n\t\t\t}\n\t\t\tif state := fs.State(); state.Msgs != expectedMsgs {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", expectedMsgs, state.Msgs)\n\t\t\t}\n\t\t}\n\n\t\t// Remove first\n\t\tremove(1, 4)\n\t\t// Remove last\n\t\tremove(5, 3)\n\t\t// Remove a middle\n\t\tremove(3, 2)\n\t})\n}\n\nfunc TestFileStoreMsgHeaders(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\tsubj, hdr, msg := \"foo\", []byte(\"name:derek\"), []byte(\"Hello World\")\n\t\telen := 22 + len(subj) + 4 + len(hdr) + len(msg) + 8\n\t\tif sz := int(fileStoreMsgSize(subj, hdr, msg)); sz != elen {\n\t\t\tt.Fatalf(\"Wrong size for stored msg with header\")\n\t\t}\n\t\tfs.StoreMsg(subj, hdr, msg, 0)\n\t\tvar smv StoreMsg\n\t\tsm, err := fs.LoadMsg(1, &smv)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t\t}\n\t\tif !bytes.Equal(msg, sm.msg) {\n\t\t\tt.Fatalf(\"Expected same msg, got %q vs %q\", sm.msg, msg)\n\t\t}\n\t\tif !bytes.Equal(hdr, sm.hdr) {\n\t\t\tt.Fatalf(\"Expected same hdr, got %q vs %q\", sm.hdr, hdr)\n\t\t}\n\t\tif removed, _ := fs.EraseMsg(1); !removed {\n\t\t\tt.Fatalf(\"Expected erase msg to return success\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreBasicWriteMsgsAndRestore(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tif _, err := newFileStore(fcfg, StreamConfig{Storage: MemoryStorage}); err == nil {\n\t\t\tt.Fatalf(\"Expected an error with wrong type\")\n\t\t}\n\t\tif _, err := newFileStore(fcfg, StreamConfig{Storage: FileStorage}); err == nil {\n\t\t\tt.Fatalf(\"Expected an error with no name\")\n\t\t}\n\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj := \"foo\"\n\n\t\t// Write 100 msgs\n\t\ttoStore := uint64(100)\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tmsg := []byte(fmt.Sprintf(\"[%08d] Hello World!\", i))\n\t\t\tif seq, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t} else if seq != uint64(i) {\n\t\t\t\tt.Fatalf(\"Expected sequence to be %d, got %d\", i, seq)\n\t\t\t}\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tmsg22 := []byte(fmt.Sprintf(\"[%08d] Hello World!\", 22))\n\t\texpectedSize := toStore * fileStoreMsgSize(subj, nil, msg22)\n\n\t\tif state.Bytes != expectedSize {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", expectedSize, state.Bytes)\n\t\t}\n\t\t// Stop will flush to disk.\n\t\tfs.Stop()\n\n\t\t// Make sure Store call after does not work.\n\t\tif _, _, err := fs.StoreMsg(subj, nil, []byte(\"no work\"), 0); err == nil {\n\t\t\tt.Fatalf(\"Expected an error for StoreMsg call after Stop, got none\")\n\t\t}\n\n\t\t// Restart\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != expectedSize {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", expectedSize, state.Bytes)\n\t\t}\n\n\t\t// Now write 100 more msgs\n\t\tfor i := uint64(101); i <= toStore*2; i++ {\n\t\t\tmsg := []byte(fmt.Sprintf(\"[%08d] Hello World!\", i))\n\t\t\tif seq, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t} else if seq != uint64(i) {\n\t\t\t\tt.Fatalf(\"Expected sequence to be %d, got %d\", i, seq)\n\t\t\t}\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != toStore*2 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore*2, state.Msgs)\n\t\t}\n\n\t\t// Now cycle again and make sure that last batch was stored.\n\t\t// Stop will flush to disk.\n\t\tfs.Stop()\n\n\t\t// Restart\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != toStore*2 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore*2, state.Msgs)\n\t\t}\n\t\tif state.Bytes != expectedSize*2 {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", expectedSize*2, state.Bytes)\n\t\t}\n\n\t\tfs.Purge()\n\t\tfs.Stop()\n\n\t\t// Restart\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != 0 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 0, state.Msgs)\n\t\t}\n\t\tif state.Bytes != 0 {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", 0, state.Bytes)\n\t\t}\n\n\t\tseq, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello\"), 0)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tfs.RemoveMsg(seq)\n\n\t\tfs.Stop()\n\n\t\t// Restart\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.FirstSeq, seq+1)\n\t})\n}\n\nfunc TestFileStoreSelectNextFirst(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 256\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tnumMsgs := 10\n\t\tsubj, msg := \"zzz\", []byte(\"Hello World\")\n\t\tfor i := 0; i < numMsgs; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != uint64(numMsgs) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", numMsgs, state.Msgs)\n\t\t}\n\n\t\t// Note the 256 block size is tied to the msg size below to give us 5 messages per block.\n\t\tif fmb := fs.selectMsgBlock(1); fmb.msgs != 5 {\n\t\t\tt.Fatalf(\"Expected 5 messages per block, but got %d\", fmb.msgs)\n\t\t}\n\n\t\t// Delete 2-7, this will cross message blocks.\n\t\tfor i := 2; i <= 7; i++ {\n\t\t\tfs.RemoveMsg(uint64(i))\n\t\t}\n\n\t\tif state := fs.State(); state.Msgs != 4 || state.FirstSeq != 1 {\n\t\t\tt.Fatalf(\"Expected 4 msgs, first seq of 11, got msgs of %d and first seq of %d\", state.Msgs, state.FirstSeq)\n\t\t}\n\t\t// Now close the gap which will force the system to jump underlying message blocks to find the right sequence.\n\t\tfs.RemoveMsg(1)\n\t\tif state := fs.State(); state.Msgs != 3 || state.FirstSeq != 8 {\n\t\t\tt.Fatalf(\"Expected 3 msgs, first seq of 8, got msgs of %d and first seq of %d\", state.Msgs, state.FirstSeq)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreSkipMsg(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 256\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tnumSkips := 10\n\t\tfor i := 0; i < numSkips; i++ {\n\t\t\tfs.SkipMsg(0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 0 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 0, state.Msgs)\n\t\t}\n\t\tif state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips) {\n\t\t\tt.Fatalf(\"Expected first to be %d and last to be %d. got first %d and last %d\", numSkips+1, numSkips, state.FirstSeq, state.LastSeq)\n\t\t}\n\n\t\tfs.StoreMsg(\"zzz\", nil, []byte(\"Hello World!\"), 0)\n\t\tfs.SkipMsg(0)\n\t\tfs.SkipMsg(0)\n\t\tfs.StoreMsg(\"zzz\", nil, []byte(\"Hello World!\"), 0)\n\t\tfs.SkipMsg(0)\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != 2 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 2, state.Msgs)\n\t\t}\n\t\tif state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips+5) {\n\t\t\tt.Fatalf(\"Expected first to be %d and last to be %d. got first %d and last %d\", numSkips+1, numSkips+5, state.FirstSeq, state.LastSeq)\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != 2 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 2, state.Msgs)\n\t\t}\n\t\tif state.FirstSeq != uint64(numSkips+1) || state.LastSeq != uint64(numSkips+5) {\n\t\t\tt.Fatalf(\"Expected first to be %d and last to be %d. got first %d and last %d\", numSkips+1, numSkips+5, state.FirstSeq, state.LastSeq)\n\t\t}\n\n\t\tvar smv StoreMsg\n\t\tsm, err := fs.LoadMsg(11, &smv)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up seq 11: %v\", err)\n\t\t}\n\t\tif sm.subj != \"zzz\" || string(sm.msg) != \"Hello World!\" {\n\t\t\tt.Fatalf(\"Message did not match\")\n\t\t}\n\n\t\tfs.SkipMsg(0)\n\t\tnseq, _, err := fs.StoreMsg(\"AAA\", nil, []byte(\"Skip?\"), 0)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up seq 11: %v\", err)\n\t\t}\n\t\tif nseq != 17 {\n\t\t\tt.Fatalf(\"Expected seq of %d but got %d\", 17, nseq)\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsm, err = fs.LoadMsg(nseq, &smv)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up seq %d: %v\", nseq, err)\n\t\t}\n\t\tif sm.subj != \"AAA\" || string(sm.msg) != \"Skip?\" {\n\t\t\tt.Fatalf(\"Message did not match\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreWriteExpireWrite(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcexp := 10 * time.Millisecond\n\t\tfcfg.CacheExpire = cexp\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttoSend := 10\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(\"zzz\", nil, []byte(\"Hello World!\"), 0)\n\t\t}\n\n\t\t// Wait for write cache portion to go to zero.\n\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\tif csz := fs.cacheSize(); csz != 0 {\n\t\t\t\treturn fmt.Errorf(\"cache size not 0, got %s\", friendlyBytes(int64(csz)))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(\"zzz\", nil, []byte(\"Hello World! - 22\"), 0)\n\t\t}\n\n\t\tif state := fs.State(); state.Msgs != uint64(toSend*2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toSend*2, state.Msgs)\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfcfg.CacheExpire = 0\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); state.Msgs != uint64(toSend*2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toSend*2, state.Msgs)\n\t\t}\n\n\t\t// Now load them in and check.\n\t\tvar smv StoreMsg\n\t\tfor i := 1; i <= toSend*2; i++ {\n\t\t\tsm, err := fs.LoadMsg(uint64(i), &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error looking up seq %d: %v\", i, err)\n\t\t\t}\n\t\t\tstr := \"Hello World!\"\n\t\t\tif i > toSend {\n\t\t\t\tstr = \"Hello World! - 22\"\n\t\t\t}\n\t\t\tif sm.subj != \"zzz\" || string(sm.msg) != str {\n\t\t\t\tt.Fatalf(\"Message did not match\")\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreMsgLimit(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxMsgs: 10}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 10, state.Msgs)\n\t\t}\n\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 10, state.Msgs)\n\t\t}\n\t\tif state.LastSeq != 11 {\n\t\t\tt.Fatalf(\"Expected the last sequence to be 11 now, but got %d\", state.LastSeq)\n\t\t}\n\t\tif state.FirstSeq != 2 {\n\t\t\tt.Fatalf(\"Expected the first sequence to be 2 now, but got %d\", state.FirstSeq)\n\t\t}\n\t\t// Make sure we can not lookup seq 1.\n\t\tif _, err := fs.LoadMsg(1, nil); err == nil {\n\t\t\tt.Fatalf(\"Expected error looking up seq 1 but got none\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreMsgLimitBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxMsgs: 1}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxMsgs: 1}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t})\n}\n\nfunc TestFileStoreBytesLimit(t *testing.T) {\n\tsubj, msg := \"foo\", make([]byte, 512)\n\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\ttoStore := uint64(1024)\n\tmaxBytes := storedMsgSize * toStore\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxBytes: int64(maxBytes)}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := uint64(0); i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != storedMsgSize*toStore {\n\t\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t\t}\n\n\t\t// Now send 10 more and check that bytes limit enforced.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t}\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != storedMsgSize*toStore {\n\t\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t\t}\n\t\tif state.FirstSeq != 11 {\n\t\t\tt.Fatalf(\"Expected first sequence to be 11, got %d\", state.FirstSeq)\n\t\t}\n\t\tif state.LastSeq != toStore+10 {\n\t\t\tt.Fatalf(\"Expected last sequence to be %d, got %d\", toStore+10, state.LastSeq)\n\t\t}\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/4771\nfunc TestFileStoreBytesLimitWithDiscardNew(t *testing.T) {\n\tsubj, msg := \"tiny\", make([]byte, 7)\n\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\ttoStore := uint64(2)\n\tmaxBytes := 100\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxBytes: int64(maxBytes), Discard: DiscardNew}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\tif i < int(toStore) {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t\t}\n\t\t\t} else if !errors.Is(err, ErrMaxBytes) {\n\t\t\t\tt.Fatalf(\"Storing msg should result in: %v\", ErrMaxBytes)\n\t\t\t}\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != storedMsgSize*toStore {\n\t\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreAgeLimit(t *testing.T) {\n\tmaxAge := 1 * time.Second\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tif fcfg.Compression != NoCompression || fcfg.Cipher != NoCipher {\n\t\t\t// TODO(nat): This test flakes at the moment with compression or encryption\n\t\t\t// because it takes too long to compress/encrypt the blocks in CI, by which\n\t\t\t// time the messages have expired. Need to think about a balanced age so that\n\t\t\t// the test doesn't take too long in these cases.\n\t\t\tt.SkipNow()\n\t\t}\n\n\t\tfcfg.BlockSize = 256\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxAge: maxAge}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store some messages. Does not really matter how many.\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 500\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tcheckExpired := func(t *testing.T) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 5*time.Second, maxAge, func() error {\n\t\t\t\tstate = fs.State()\n\t\t\t\tif state.Msgs != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t\t\t}\n\t\t\t\tif state.Bytes != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected no bytes, got %d\", state.Bytes)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\t// Let them expire\n\t\tcheckExpired(t)\n\n\t\t// Now add some more and make sure that timer will fire again.\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tfs.RemoveMsg(502)\n\t\tfs.RemoveMsg(602)\n\t\tfs.RemoveMsg(702)\n\t\tfs.RemoveMsg(802)\n\t\t// We will measure the time to make sure expires works with interior deletes.\n\t\tstart := time.Now()\n\t\tcheckExpired(t)\n\t\tif elapsed := time.Since(start); elapsed > 5*time.Second {\n\t\t\tt.Fatalf(\"Took too long to expire: %v\", elapsed)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreTimeStamps(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tlast := time.Now().UnixNano()\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 10; i++ {\n\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tvar smv StoreMsg\n\t\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error looking up msg [%d]: %v\", seq, err)\n\t\t\t}\n\t\t\t// These should be different\n\t\t\tif sm.ts <= last {\n\t\t\t\tt.Fatalf(\"Expected different timestamps, got last %v vs %v\", last, sm.ts)\n\t\t\t}\n\t\t\tlast = sm.ts\n\t\t}\n\t})\n}\n\nfunc TestFileStorePurge(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tblkSize := uint64(64 * 1024)\n\t\tfcfg.BlockSize = blkSize\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", make([]byte, 8*1024)\n\t\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t\ttoStore := uint64(1024)\n\t\tfor i := uint64(0); i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != storedMsgSize*toStore {\n\t\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t\t}\n\n\t\texpectedBlocks := int(storedMsgSize * toStore / blkSize)\n\t\tif numBlocks := fs.numMsgBlocks(); numBlocks <= expectedBlocks {\n\t\t\tt.Fatalf(\"Expected to have more then %d msg blocks, got %d\", blkSize, numBlocks)\n\t\t}\n\n\t\tfs.Purge()\n\n\t\tif numBlocks := fs.numMsgBlocks(); numBlocks != 1 {\n\t\t\tt.Fatalf(\"Expected to have exactly 1 empty msg block, got %d\", numBlocks)\n\t\t}\n\n\t\tcheckPurgeState := func(stored uint64) {\n\t\t\tt.Helper()\n\t\t\tstate = fs.State()\n\t\t\tif state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected 0 msgs after purge, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tif state.Bytes != 0 {\n\t\t\t\tt.Fatalf(\"Expected 0 bytes after purge, got %d\", state.Bytes)\n\t\t\t}\n\t\t\tif state.LastSeq != stored {\n\t\t\t\tt.Fatalf(\"Expected LastSeq to be %d., got %d\", toStore, state.LastSeq)\n\t\t\t}\n\t\t\tif state.FirstSeq != stored+1 {\n\t\t\t\tt.Fatalf(\"Expected FirstSeq to be %d., got %d\", toStore+1, state.FirstSeq)\n\t\t\t}\n\t\t}\n\t\tcheckPurgeState(toStore)\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif numBlocks := fs.numMsgBlocks(); numBlocks != 1 {\n\t\t\tt.Fatalf(\"Expected to have exactly 1 empty msg block, got %d\", numBlocks)\n\t\t}\n\n\t\tcheckPurgeState(toStore)\n\n\t\t// Now make sure we clean up any dangling purged messages.\n\t\tfor i := uint64(0); i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tif state.Bytes != storedMsgSize*toStore {\n\t\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t\t}\n\n\t\t// We will simulate crashing before the purge directory is cleared.\n\t\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\t\tpdir := filepath.Join(fs.fcfg.StoreDir, \"ptest\")\n\t\tos.Rename(mdir, pdir)\n\t\tos.MkdirAll(mdir, 0755)\n\n\t\tfs.Purge()\n\t\tcheckPurgeState(toStore * 2)\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tpurgeDir := filepath.Join(fs.fcfg.StoreDir, purgeDir)\n\t\tos.Rename(pdir, purgeDir)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif numBlocks := fs.numMsgBlocks(); numBlocks != 1 {\n\t\t\tt.Fatalf(\"Expected to have exactly 1 empty msg block, got %d\", numBlocks)\n\t\t}\n\n\t\tcheckPurgeState(toStore * 2)\n\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\tif _, err := os.Stat(purgeDir); err == nil {\n\t\t\t\treturn fmt.Errorf(\"purge directory still present\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc TestFileStoreCompact(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 350\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected 10 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tn, err := fs.Compact(6)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif n != 5 {\n\t\t\tt.Fatalf(\"Expected to have purged 5 msgs, got %d\", n)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 5 {\n\t\t\tt.Fatalf(\"Expected 5 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tif state.FirstSeq != 6 {\n\t\t\tt.Fatalf(\"Expected first seq of 6, got %d\", state.FirstSeq)\n\t\t}\n\t\t// Now test that compact will also reset first if seq > last\n\t\tn, err = fs.Compact(100)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif n != 5 {\n\t\t\tt.Fatalf(\"Expected to have purged 5 msgs, got %d\", n)\n\t\t}\n\t\tif state = fs.State(); state.FirstSeq != 100 {\n\t\t\tt.Fatalf(\"Expected first seq of 100, got %d\", state.FirstSeq)\n\t\t}\n\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state = fs.State(); state.FirstSeq != 100 {\n\t\t\tt.Fatalf(\"Expected first seq of 100, got %d\", state.FirstSeq)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreCompactLastPlusOne(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 8192\n\t\tfcfg.AsyncFlush = true\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", make([]byte, 10_000)\n\t\tfor i := 0; i < 10_000; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\t// The performance of this test is quite terrible with compression\n\t\t// if we have AsyncFlush = false, so we'll batch flushes instead.\n\t\tfs.mu.Lock()\n\t\tfs.checkAndFlushLastBlock()\n\t\tfs.mu.Unlock()\n\n\t\tif state := fs.State(); state.Msgs != 10_000 {\n\t\t\tt.Fatalf(\"Expected 1000000 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tif _, err := fs.Compact(10_001); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 0 {\n\t\t\tt.Fatalf(\"Expected no message but got %d\", state.Msgs)\n\t\t}\n\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\tstate = fs.State()\n\t\tif state.Msgs != 1 {\n\t\t\tt.Fatalf(\"Expected one message but got %d\", state.Msgs)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreCompactMsgCountBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected 10 msgs, got %d\", state.Msgs)\n\t\t}\n\t\t// Now delete 2,3,4.\n\t\tfs.EraseMsg(2)\n\t\tfs.EraseMsg(3)\n\t\tfs.EraseMsg(4)\n\n\t\t// Also delete 7,8, and 9.\n\t\tfs.RemoveMsg(7)\n\t\tfs.RemoveMsg(8)\n\t\tfs.RemoveMsg(9)\n\n\t\tn, err := fs.Compact(6)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// 1 & 5\n\t\tif n != 2 {\n\t\t\tt.Fatalf(\"Expected to have deleted 2 msgs, got %d\", n)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != 2 {\n\t\t\tt.Fatalf(\"Expected to have 2 remaining, got %d\", state.Msgs)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreCompactPerf(t *testing.T) {\n\tt.SkipNow()\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 8192\n\t\tfcfg.AsyncFlush = true\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 100_000; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != 100_000 {\n\t\t\tt.Fatalf(\"Expected 1000000 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tstart := time.Now()\n\t\tn, err := fs.Compact(90_001)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tt.Logf(\"Took %v to compact\\n\", time.Since(start))\n\n\t\tif n != 90_000 {\n\t\t\tt.Fatalf(\"Expected to have purged 90_000 msgs, got %d\", n)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 10_000 {\n\t\t\tt.Fatalf(\"Expected 10_000 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tif state.FirstSeq != 90_001 {\n\t\t\tt.Fatalf(\"Expected first seq of 90_001, got %d\", state.FirstSeq)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreStreamTruncate(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 350\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttseq := uint64(50)\n\n\t\tsubj, toStore := \"foo\", uint64(100)\n\t\tfor i := uint64(1); i < tseq; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"ok\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tsubj = \"bar\"\n\t\tfor i := tseq; i <= toStore; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"ok\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tif state := fs.State(); state.Msgs != toStore {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\n\t\tif err := fs.Truncate(tseq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif state := fs.State(); state.Msgs != tseq {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", tseq, state.Msgs)\n\t\t}\n\n\t\t// Now make sure we report properly if we have some deleted interior messages.\n\t\tfs.RemoveMsg(10)\n\t\tfs.RemoveMsg(20)\n\t\tfs.RemoveMsg(30)\n\t\tfs.RemoveMsg(40)\n\n\t\ttseq = uint64(25)\n\t\tif err := fs.Truncate(tseq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != tseq-2 {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", tseq-2, state.Msgs)\n\t\t}\n\t\texpected := []uint64{10, 20}\n\t\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t\t}\n\n\t\tbefore := state\n\n\t\t// Make sure we can recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\tmb := fs.getFirstBlock()\n\t\trequire_True(t, mb != nil)\n\t\trequire_NoError(t, mb.loadMsgs())\n\n\t\t// Also make sure we can recover properly with no index.db present.\n\t\t// We want to make sure we preserve tombstones from any blocks being deleted.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v without index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRemovePartialRecovery(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 100\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\n\t\t// Remove half\n\t\tfor i := 1; i <= toStore/2; i++ {\n\t\t\tfs.RemoveMsg(uint64(i))\n\t\t}\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != uint64(toStore/2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore/2, state.Msgs)\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate2 := fs.State()\n\t\tif !reflect.DeepEqual(state2, state) {\n\t\t\tt.Fatalf(\"Expected recovered state to be the same, got %+v vs %+v\\n\", state2, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRemoveOutOfOrderRecovery(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 100\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\n\t\t// Remove evens\n\t\tfor i := 2; i <= toStore; i += 2 {\n\t\t\tif removed, _ := fs.RemoveMsg(uint64(i)); !removed {\n\t\t\t\tt.Fatalf(\"Expected remove to return true\")\n\t\t\t}\n\t\t}\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != uint64(toStore/2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore/2, state.Msgs)\n\t\t}\n\n\t\tvar smv StoreMsg\n\t\tif _, err := fs.LoadMsg(1, &smv); err != nil {\n\t\t\tt.Fatalf(\"Expected to retrieve seq 1\")\n\t\t}\n\t\tfor i := 2; i <= toStore; i += 2 {\n\t\t\tif _, err := fs.LoadMsg(uint64(i), &smv); err == nil {\n\t\t\t\tt.Fatalf(\"Expected error looking up seq %d that should be deleted\", i)\n\t\t\t}\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate2 := fs.State()\n\t\tif !reflect.DeepEqual(state2, state) {\n\t\t\tt.Fatalf(\"Expected recovered states to be the same, got %+v vs %+v\\n\", state, state2)\n\t\t}\n\n\t\tif _, err := fs.LoadMsg(1, &smv); err != nil {\n\t\t\tt.Fatalf(\"Expected to retrieve seq 1\")\n\t\t}\n\t\tfor i := 2; i <= toStore; i += 2 {\n\t\t\tif _, err := fs.LoadMsg(uint64(i), nil); err == nil {\n\t\t\t\tt.Fatalf(\"Expected error looking up seq %d that should be deleted\", i)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreAgeLimitRecovery(t *testing.T) {\n\tmaxAge := 1 * time.Second\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.CacheExpire = 1 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage, MaxAge: maxAge}\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store some messages. Does not really matter how many.\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 100\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\t\tfs.Stop()\n\n\t\ttime.Sleep(maxAge)\n\n\t\tfcfg.CacheExpire = 0\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure they expire.\n\t\tcheckFor(t, time.Second, 2*maxAge, func() error {\n\t\t\tt.Helper()\n\t\t\tstate = fs.State()\n\t\t\tif state.Msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tif state.Bytes != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no bytes, got %d\", state.Bytes)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc TestFileStoreBitRot(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store some messages. Does not really matter how many.\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 100\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\n\t\tif ld := fs.checkMsgs(); ld != nil && len(ld.Msgs) > 0 {\n\t\t\tt.Fatalf(\"Expected to have no corrupt msgs, got %d\", len(ld.Msgs))\n\t\t}\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t// Now twiddle some bits.\n\t\t\tfs.mu.Lock()\n\t\t\tlmb := fs.lmb\n\t\t\tcontents, err := os.ReadFile(lmb.mfn)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(contents) > 0)\n\n\t\t\tvar index int\n\t\t\tfor {\n\t\t\t\tindex = rand.Intn(len(contents))\n\t\t\t\t// Reverse one byte anywhere.\n\t\t\t\tb := contents[index]\n\t\t\t\tcontents[index] = bits.Reverse8(b)\n\t\t\t\tif b != contents[index] {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tos.WriteFile(lmb.mfn, contents, 0644)\n\t\t\tfs.mu.Unlock()\n\n\t\t\tld := fs.checkMsgs()\n\t\t\tif len(ld.Msgs) > 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// If our bitrot caused us to not be able to recover any messages we can break as well.\n\t\t\tif state := fs.State(); state.Msgs == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// Fail the test if we have tried the 10 times and still did not\n\t\t\t// get any corruption report.\n\t\t\tif i == 9 {\n\t\t\t\tt.Fatalf(\"Expected to have corrupt msgs got none: changed [%d]\", index)\n\t\t\t}\n\t\t}\n\n\t\t// Make sure we can restore.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// checkMsgs will repair the underlying store, so checkMsgs should be clean now.\n\t\tif ld := fs.checkMsgs(); ld != nil {\n\t\t\t// If we have no msgs left this will report the head msgs as lost again.\n\t\t\tif state := fs.State(); state.Msgs > 0 {\n\t\t\t\tt.Fatalf(\"Expected no errors restoring checked and fixed filestore, got %+v\", ld)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreEraseMsg(t *testing.T) {\n\t// Just do no encryption, etc.\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir()}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfs.StoreMsg(subj, nil, msg, 0)\n\tfs.StoreMsg(subj, nil, msg, 0) // To keep block from being deleted.\n\tvar smv StoreMsg\n\tsm, err := fs.LoadMsg(1, &smv)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t}\n\tif !bytes.Equal(msg, sm.msg) {\n\t\tt.Fatalf(\"Expected same msg, got %q vs %q\", sm.msg, msg)\n\t}\n\tif removed, _ := fs.EraseMsg(1); !removed {\n\t\tt.Fatalf(\"Expected erase msg to return success\")\n\t}\n\tif sm2, _ := fs.msgForSeq(1, nil); sm2 != nil {\n\t\tt.Fatalf(\"Expected msg to be erased\")\n\t}\n\tfs.checkAndFlushLastBlock()\n\n\t// Now look on disk as well.\n\trl := fileStoreMsgSize(subj, nil, msg)\n\tbuf := make([]byte, rl)\n\tfp, err := os.Open(filepath.Join(fcfg.StoreDir, msgDir, fmt.Sprintf(blkScan, 1)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error opening msg block file: %v\", err)\n\t}\n\tdefer fp.Close()\n\n\tfp.ReadAt(buf, 0)\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\tmb.mu.Lock()\n\tsm, err = mb.msgFromBuf(buf, nil, nil)\n\tmb.mu.Unlock()\n\tif err != nil {\n\t\tt.Fatalf(\"error reading message from block: %v\", err)\n\t}\n\tif sm.subj == subj {\n\t\tt.Fatalf(\"Expected the subjects to be different\")\n\t}\n\tif sm.seq != 0 && sm.seq&ebit == 0 {\n\t\tt.Fatalf(\"Expected seq to be 0, marking as deleted, got %d\", sm.seq)\n\t}\n\tif sm.ts != 0 {\n\t\tt.Fatalf(\"Expected timestamp to be 0, got %d\", sm.ts)\n\t}\n\tif bytes.Equal(sm.msg, msg) {\n\t\tt.Fatalf(\"Expected message body to be randomized\")\n\t}\n}\n\nfunc TestFileStoreEraseAndNoIndexRecovery(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 100\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != uint64(toStore) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t\t}\n\n\t\t// Erase the even messages.\n\t\tfor i := 2; i <= toStore; i += 2 {\n\t\t\tif removed, _ := fs.EraseMsg(uint64(i)); !removed {\n\t\t\t\tt.Fatalf(\"Expected erase msg to return true\")\n\t\t\t}\n\t\t}\n\t\tstate = fs.State()\n\t\tif state.Msgs != uint64(toStore/2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore/2, state.Msgs)\n\t\t}\n\n\t\t// Stop and remove the optional index file.\n\t\tfs.Stop()\n\t\tifn := filepath.Join(fcfg.StoreDir, msgDir, fmt.Sprintf(indexScan, 1))\n\t\tos.Remove(ifn)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != uint64(toStore/2) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore/2, state.Msgs)\n\t\t}\n\n\t\tfor i := 2; i <= toStore; i += 2 {\n\t\t\tif _, err := fs.LoadMsg(uint64(i), nil); err == nil {\n\t\t\t\tt.Fatalf(\"Expected error looking up seq %d that should be erased\", i)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreMeta(t *testing.T) {\n\t// Just do no encryption, etc.\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir()}\n\tmconfig := StreamConfig{Name: \"ZZ-22-33\", Storage: FileStorage, Subjects: []string{\"foo.*\"}, Replicas: 22}\n\tfs, err := newFileStore(fcfg, mconfig)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmetafile := filepath.Join(fcfg.StoreDir, JetStreamMetaFile)\n\tmetasum := filepath.Join(fcfg.StoreDir, JetStreamMetaFileSum)\n\n\t// Test to make sure meta file and checksum are present.\n\tif _, err := os.Stat(metafile); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Expected metafile %q to exist\", metafile)\n\t}\n\tif _, err := os.Stat(metasum); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Expected metafile's checksum %q to exist\", metasum)\n\t}\n\n\tbuf, err := os.ReadFile(metafile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading metafile: %v\", err)\n\t}\n\tvar mconfig2 StreamConfig\n\tif err := json.Unmarshal(buf, &mconfig2); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling: %v\", err)\n\t}\n\tif !reflect.DeepEqual(mconfig, mconfig2) {\n\t\tt.Fatalf(\"Stream configs not equal, got %+v vs %+v\", mconfig2, mconfig)\n\t}\n\tchecksum, err := os.ReadFile(metasum)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading metafile checksum: %v\", err)\n\t}\n\n\tfs.mu.Lock()\n\tfs.hh.Reset()\n\tfs.hh.Write(buf)\n\tmychecksum := hex.EncodeToString(fs.hh.Sum(nil))\n\tfs.mu.Unlock()\n\n\tif mychecksum != string(checksum) {\n\t\tt.Fatalf(\"Checksums do not match, got %q vs %q\", mychecksum, checksum)\n\t}\n\n\t// Now create a consumer. Same deal for them.\n\toconfig := ConsumerConfig{\n\t\tDeliverSubject: \"d\",\n\t\tFilterSubject: \"foo\",\n\t\tAckPolicy: AckAll,\n\t}\n\toname := \"obs22\"\n\tobs, err := fs.ConsumerStore(oname, time.Time{}, &oconfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tometafile := filepath.Join(fcfg.StoreDir, consumerDir, oname, JetStreamMetaFile)\n\tometasum := filepath.Join(fcfg.StoreDir, consumerDir, oname, JetStreamMetaFileSum)\n\n\t// Test to make sure meta file and checksum are present.\n\tif _, err := os.Stat(ometafile); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Expected consumer metafile %q to exist\", ometafile)\n\t}\n\tif _, err := os.Stat(ometasum); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Expected consumer metafile's checksum %q to exist\", ometasum)\n\t}\n\n\tbuf, err = os.ReadFile(ometafile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading consumer metafile: %v\", err)\n\t}\n\n\tvar oconfig2 ConsumerConfig\n\tif err := json.Unmarshal(buf, &oconfig2); err != nil {\n\t\tt.Fatalf(\"Error unmarshalling: %v\", err)\n\t}\n\t// Since we set name we will get that back now.\n\toconfig.Name = oname\n\tif !reflect.DeepEqual(oconfig2, oconfig) {\n\t\tt.Fatalf(\"Consumer configs not equal, got %+v vs %+v\", oconfig2, oconfig)\n\t}\n\tchecksum, err = os.ReadFile(ometasum)\n\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading consumer metafile checksum: %v\", err)\n\t}\n\n\thh := obs.(*consumerFileStore).hh\n\thh.Reset()\n\thh.Write(buf)\n\tmychecksum = hex.EncodeToString(hh.Sum(nil))\n\tif mychecksum != string(checksum) {\n\t\tt.Fatalf(\"Checksums do not match, got %q vs %q\", mychecksum, checksum)\n\t}\n}\n\nfunc TestFileStoreWriteAndReadSameBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World!\")\n\n\t\tfor i := uint64(1); i <= 10; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t\tif _, err := fs.LoadMsg(i, nil); err != nil {\n\t\t\t\tt.Fatalf(\"Error loading %d: %v\", i, err)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreAndRetrieveMultiBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tsubj, msg := \"foo\", []byte(\"Hello World!\")\n\t\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t\tfcfg.BlockSize = 4 * storedMsgSize\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 20; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 20 {\n\t\t\tt.Fatalf(\"Expected 20 msgs, got %d\", state.Msgs)\n\t\t}\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar smv StoreMsg\n\t\tfor i := uint64(1); i <= 20; i++ {\n\t\t\tif _, err := fs.LoadMsg(i, &smv); err != nil {\n\t\t\t\tt.Fatalf(\"Error loading %d: %v\", i, err)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreCollapseDmap(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tsubj, msg := \"foo\", []byte(\"Hello World!\")\n\t\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t\tfcfg.BlockSize = 4 * storedMsgSize\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tstate := fs.State()\n\t\tif state.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected 10 msgs, got %d\", state.Msgs)\n\t\t}\n\n\t\tcheckDmapTotal := func(total int) {\n\t\t\tt.Helper()\n\t\t\tif nde := fs.dmapEntries(); nde != total {\n\t\t\t\tt.Fatalf(\"Expecting only %d entries, got %d\", total, nde)\n\t\t\t}\n\t\t}\n\n\t\tcheckFirstSeq := func(seq uint64) {\n\t\t\tt.Helper()\n\t\t\tstate := fs.State()\n\t\t\tif state.FirstSeq != seq {\n\t\t\t\tt.Fatalf(\"Expected first seq to be %d, got %d\", seq, state.FirstSeq)\n\t\t\t}\n\t\t}\n\n\t\t// Now remove some out of order, forming gaps and entries in dmaps.\n\t\tfs.RemoveMsg(2)\n\t\tcheckFirstSeq(1)\n\t\tfs.RemoveMsg(4)\n\t\tcheckFirstSeq(1)\n\t\tfs.RemoveMsg(8)\n\t\tcheckFirstSeq(1)\n\n\t\tstate = fs.State()\n\t\tif state.Msgs != 7 {\n\t\t\tt.Fatalf(\"Expected 7 msgs, got %d\", state.Msgs)\n\t\t}\n\n\t\tcheckDmapTotal(3)\n\n\t\t// Close gaps..\n\t\tfs.RemoveMsg(1)\n\t\tcheckDmapTotal(2)\n\t\tcheckFirstSeq(3)\n\n\t\tfs.RemoveMsg(3)\n\t\tcheckDmapTotal(1)\n\t\tcheckFirstSeq(5)\n\n\t\tfs.RemoveMsg(5)\n\t\tcheckDmapTotal(1)\n\t\tcheckFirstSeq(6)\n\n\t\tfs.RemoveMsg(7)\n\t\tcheckDmapTotal(2)\n\n\t\tfs.RemoveMsg(6)\n\t\tcheckDmapTotal(0)\n\t})\n}\n\nfunc TestFileStoreReadCache(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.CacheExpire = 100 * time.Millisecond\n\n\t\tsubj, msg := \"foo.bar\", make([]byte, 1024)\n\t\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttoStore := 500\n\t\ttotalBytes := uint64(toStore) * storedMsgSize\n\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\t// Wait for cache to go to zero.\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif csz := fs.cacheSize(); csz != 0 {\n\t\t\t\treturn fmt.Errorf(\"cache size not 0, got %s\", friendlyBytes(int64(csz)))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tfs.LoadMsg(1, nil)\n\t\tif csz := fs.cacheSize(); csz != totalBytes {\n\t\t\tt.Fatalf(\"Expected all messages to be cached, got %d vs %d\", csz, totalBytes)\n\t\t}\n\t\t// Should expire and be removed.\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif csz := fs.cacheSize(); csz != 0 {\n\t\t\t\treturn fmt.Errorf(\"cache size not 0, got %s\", friendlyBytes(int64(csz)))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif cls := fs.cacheLoads(); cls != 1 {\n\t\t\tt.Fatalf(\"Expected only 1 cache load, got %d\", cls)\n\t\t}\n\t\t// Now make sure we do not reload cache if there is activity.\n\t\tfs.LoadMsg(1, nil)\n\t\ttimeout := time.Now().Add(250 * time.Millisecond)\n\t\tfor time.Now().Before(timeout) {\n\t\t\tif cls := fs.cacheLoads(); cls != 2 {\n\t\t\t\tt.Fatalf(\"cache loads not 2, got %d\", cls)\n\t\t\t}\n\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\tfs.LoadMsg(1, nil) // register activity.\n\t\t}\n\t})\n}\n\nfunc TestFileStorePartialCacheExpiration(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcexp := 10 * time.Millisecond\n\t\tfcfg.CacheExpire = cexp\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.StoreMsg(\"foo\", nil, []byte(\"msg1\"), 0)\n\n\t\t// Should expire and be removed.\n\t\ttime.Sleep(2 * cexp)\n\t\tfs.StoreMsg(\"bar\", nil, []byte(\"msg2\"), 0)\n\n\t\t// Again wait for cache to expire.\n\t\ttime.Sleep(2 * cexp)\n\t\tif _, err := fs.LoadMsg(1, nil); err != nil {\n\t\t\tt.Fatalf(\"Error loading message 1: %v\", err)\n\t\t}\n\t})\n}\n\nfunc TestFileStorePartialIndexes(t *testing.T) {\n\t// TODO(nat): This test is no longer applicable as we no longer have positional\n\t// write caches but check before removing whether it proves anything else of value.\n\tt.SkipNow()\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcexp := 10 * time.Millisecond\n\t\tfcfg.CacheExpire = cexp\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttoSend := 5\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(\"foo\", nil, []byte(\"ok-1\"), 0)\n\t\t}\n\n\t\t// Now wait til the cache expires, including the index.\n\t\tfs.mu.Lock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.Unlock()\n\n\t\t// Force idx to expire by resetting last remove ts.\n\t\tmb.mu.Lock()\n\t\tmb.llts = mb.llts - int64(defaultCacheBufferExpiration*2)\n\t\tmb.mu.Unlock()\n\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tmb.mu.Lock()\n\t\t\tdefer mb.mu.Unlock()\n\t\t\tif mb.cache == nil || len(mb.cache.idx) == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Index not empty\")\n\t\t})\n\n\t\t// Create a partial cache by adding more msgs.\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(\"foo\", nil, []byte(\"ok-2\"), 0)\n\t\t}\n\t\t// If we now load in a message in second half if we do not\n\t\t// detect idx is a partial correctly this will panic.\n\t\tif _, err := fs.LoadMsg(8, nil); err != nil {\n\t\t\tt.Fatalf(\"Error loading %d: %v\", 1, err)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreInvalidIndexesRebuilt(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttoSend := 5\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(\"foo\", nil, []byte(\"ok-1\"), 0)\n\t\t}\n\t\tfs.FlushAllPending()\n\n\t\t// Now we're going to mangle the in-memory cache by changing\n\t\t// the sequence number of the first message. We also need to\n\t\t// trick the cache into believing the hash is already validated\n\t\t// so we don't fail on that. This is specifically testing the\n\t\t// seq != fsm.seq condition.\n\t\tmb := fs.selectMsgBlock(1)\n\t\trequire_NotNil(t, mb)\n\t\trequire_NoError(t, mb.loadMsgs())\n\t\trequire_True(t, mb.cacheAlreadyLoaded())\n\t\tri, rl, _, err := mb.slotInfo(0)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, mb.cache)\n\t\trequire_NotNil(t, mb.cache.buf)\n\t\tslot := mb.cache.buf[ri : ri+rl]\n\t\trequire_Equal(t, binary.LittleEndian.Uint64(slot[4:]), 1)\n\t\tbinary.LittleEndian.PutUint64(slot[4:], 12345)\n\t\tmb.cache.idx[0] = (mb.cache.idx[0] | cbit)\n\n\t\t// Expect an error on the first instance and for cacheLookupEx\n\t\t// to discard the cache.\n\t\t_, err = mb.cacheLookupEx(1, nil, false)\n\t\trequire_Error(t, err)\n\t\trequire_True(t, mb.ecache.Value() == nil)\n\n\t\t// Now fetchMsg should notice and rebuild the index with the\n\t\t// correct sequence from disk.\n\t\tsm, _, err := mb.fetchMsg(1, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.seq, 1)\n\t})\n}\n\nfunc TestFileStoreSnapshot(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tsubj, msg := \"foo\", []byte(\"Hello Snappy!\")\n\t\tscfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, scfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\ttoSend := 2233\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\t// Create a few consumers.\n\t\to1, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\to2, err := fs.ConsumerStore(\"o33\", time.Time{}, &ConsumerConfig{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tstate := &ConsumerState{}\n\t\tstate.Delivered.Consumer = 100\n\t\tstate.Delivered.Stream = 100\n\t\tstate.AckFloor.Consumer = 22\n\t\tstate.AckFloor.Stream = 22\n\n\t\tif err := o1.Update(state); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error updating state: %v\", err)\n\t\t}\n\t\tstate.AckFloor.Consumer = 33\n\t\tstate.AckFloor.Stream = 33\n\n\t\tif err := o2.Update(state); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error updating state: %v\", err)\n\t\t}\n\n\t\tsnapshot := func() []byte {\n\t\t\tt.Helper()\n\t\t\tr, err := fs.Snapshot(5*time.Second, true, true)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating snapshot\")\n\t\t\t}\n\t\t\tsnapshot, err := io.ReadAll(r.Reader)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading snapshot\")\n\t\t\t}\n\t\t\treturn snapshot\n\t\t}\n\n\t\t// This will unzip the snapshot and create a new filestore that will recover the state.\n\t\t// We will compare the states for this vs the original one.\n\t\tverifySnapshot := func(snap []byte) {\n\t\t\tt.Helper()\n\t\t\tr := bytes.NewReader(snap)\n\t\t\ttr := tar.NewReader(s2.NewReader(r))\n\n\t\t\trstoreDir := t.TempDir()\n\n\t\t\tfor {\n\t\t\t\thdr, err := tr.Next()\n\t\t\t\tif err == io.EOF {\n\t\t\t\t\tbreak // End of archive\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting next entry from snapshot: %v\", err)\n\t\t\t\t}\n\t\t\t\tfpath := filepath.Join(rstoreDir, filepath.Clean(hdr.Name))\n\t\t\t\tpdir := filepath.Dir(fpath)\n\t\t\t\tos.MkdirAll(pdir, 0755)\n\t\t\t\tfd, err := os.OpenFile(fpath, os.O_CREATE|os.O_RDWR, 0600)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error opening file[%s]: %v\", fpath, err)\n\t\t\t\t}\n\t\t\t\tif _, err := io.Copy(fd, tr); err != nil {\n\t\t\t\t\tt.Fatalf(\"Error writing file[%s]: %v\", fpath, err)\n\t\t\t\t}\n\t\t\t\tfd.Close()\n\t\t\t}\n\n\t\t\tfcfg.StoreDir = rstoreDir\n\t\t\tfsr, err := newFileStoreWithCreated(fcfg, scfg, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fsr.Stop()\n\t\t\tstate := fs.State()\n\t\t\trstate := fsr.State()\n\n\t\t\t// FIXME(dlc)\n\t\t\t// Right now the upper layers in JetStream recover the consumers and do not expect\n\t\t\t// the lower layers to do that. So for now blank that out of our original state.\n\t\t\t// Will have more exhaustive tests in jetstream_test.go.\n\t\t\tstate.Consumers = 0\n\n\t\t\t// Just check the state.\n\t\t\tif !reflect.DeepEqual(rstate, state) {\n\t\t\t\tt.Fatalf(\"Restored state does not match:\\n%+v\\n\\n%+v\", rstate, state)\n\t\t\t}\n\t\t}\n\n\t\t// Simple case first.\n\t\tsnap := snapshot()\n\t\tverifySnapshot(snap)\n\n\t\t// Remove first 100 messages.\n\t\tfor i := 1; i <= 100; i++ {\n\t\t\tfs.RemoveMsg(uint64(i))\n\t\t}\n\n\t\tsnap = snapshot()\n\t\tverifySnapshot(snap)\n\n\t\t// Now sporadic messages inside the stream.\n\t\ttotal := int64(toSend - 100)\n\t\t// Delete 50 random messages.\n\t\tfor i := 0; i < 50; i++ {\n\t\t\tseq := uint64(rand.Int63n(total) + 101)\n\t\t\tfs.RemoveMsg(seq)\n\t\t}\n\n\t\tsnap = snapshot()\n\t\tverifySnapshot(snap)\n\n\t\t// Make sure compaction works with snapshots.\n\t\tfs.mu.RLock()\n\t\tfor _, mb := range fs.blks {\n\t\t\t// Should not call compact on last msg block.\n\t\t\tif mb != fs.lmb {\n\t\t\t\tmb.mu.Lock()\n\t\t\t\terr = mb.compact()\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\t\tfs.mu.RUnlock()\n\n\t\tsnap = snapshot()\n\t\tverifySnapshot(snap)\n\n\t\t// Now check to make sure that we can still delete/erase messages.\n\t\tsr, err := fs.Snapshot(5*time.Second, false, true)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating snapshot\")\n\t\t}\n\t\t_, err = fs.RemoveMsg(122)\n\t\trequire_NoError(t, err)\n\n\t\t// Now make sure we can do these when we close the reader and release the snapshot condition.\n\t\tsr.Reader.Close()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tfs.mu.RLock()\n\t\t\tdefer fs.mu.RUnlock()\n\t\t\tif fs.sips != 0 {\n\t\t\t\treturn errors.New(\"snapshot is not finished\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Make sure if we do not read properly then it will close the writer and report an error.\n\t\tsr, err = fs.Snapshot(25*time.Millisecond, false, false)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating snapshot\")\n\t\t}\n\n\t\t// Cause snapshot to timeout.\n\t\ttime.Sleep(50 * time.Millisecond)\n\t\t// Read should fail\n\t\tvar buf [32]byte\n\t\tif _, err := sr.Reader.Read(buf[:]); err != io.EOF {\n\t\t\tt.Fatalf(\"Expected read to produce an error, got none\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreSnapshotAndSyncBlocks(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tscfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, scfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.cancelSyncTimer()\n\t\tfs.syncBlocks()\n\n\t\tfs.mu.Lock()\n\t\tif fs.syncTmr == nil {\n\t\t\tfs.mu.Unlock()\n\t\t\tt.Fatal(\"Expected sync timer to be set\")\n\t\t}\n\t\t// Simulate a snapshot being in progress. This should not prevent us syncing blocks.\n\t\tfs.sips++\n\t\tfs.mu.Unlock()\n\n\t\tfs.cancelSyncTimer()\n\t\tfs.syncBlocks()\n\n\t\tfs.mu.Lock()\n\t\tif fs.syncTmr == nil {\n\t\t\tfs.mu.Unlock()\n\t\t\tt.Fatal(\"Expected sync timer to be set\")\n\t\t}\n\t\tfs.mu.Unlock()\n\t})\n}\n\nfunc TestFileStoreConsumer(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\to, err := fs.ConsumerStore(\"obs22\", time.Time{}, &ConsumerConfig{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif state, err := o.State(); err != nil || state.Delivered.Consumer != 0 {\n\t\t\tt.Fatalf(\"Unexpected state or error: %v\", err)\n\t\t}\n\n\t\tstate := &ConsumerState{}\n\n\t\tupdateAndCheck := func() {\n\t\t\tt.Helper()\n\t\t\tif err := o.Update(state); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error updating state: %v\", err)\n\t\t\t}\n\t\t\ts2, err := o.State()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error getting state: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(state, s2) {\n\t\t\t\tt.Fatalf(\"State is not the same: wanted %+v got %+v\", state, s2)\n\t\t\t}\n\t\t}\n\n\t\tshouldFail := func() {\n\t\t\tt.Helper()\n\t\t\tif err := o.Update(state); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error and got none\")\n\t\t\t}\n\t\t}\n\n\t\tstate.Delivered.Consumer = 1\n\t\tstate.Delivered.Stream = 22\n\t\tupdateAndCheck()\n\n\t\tstate.Delivered.Consumer = 100\n\t\tstate.Delivered.Stream = 122\n\t\tstate.AckFloor.Consumer = 50\n\t\tstate.AckFloor.Stream = 123\n\t\t// This should fail, bad state.\n\t\tshouldFail()\n\t\t// So should this.\n\t\tstate.AckFloor.Consumer = 200\n\t\tstate.AckFloor.Stream = 100\n\t\tshouldFail()\n\n\t\t// Should succeed\n\t\tstate.AckFloor.Consumer = 50\n\t\tstate.AckFloor.Stream = 72\n\t\tupdateAndCheck()\n\n\t\ttn := time.Now().UnixNano()\n\n\t\t// We should sanity check pending here as well, so will check if a pending value is below\n\t\t// ack floor or above delivered.\n\t\tstate.Pending = map[uint64]*Pending{70: {70, tn}}\n\t\tshouldFail()\n\t\tstate.Pending = map[uint64]*Pending{140: {140, tn}}\n\t\tshouldFail()\n\t\tstate.Pending = map[uint64]*Pending{72: {72, tn}} // exact on floor should fail\n\t\tshouldFail()\n\n\t\t// Put timestamps a second apart.\n\t\t// We will downsample to second resolution to save space. So setup our times\n\t\t// to reflect that.\n\t\tago := time.Now().Add(-30 * time.Second).Truncate(time.Second)\n\t\tnt := func() *Pending {\n\t\t\tago = ago.Add(time.Second)\n\t\t\treturn &Pending{0, ago.UnixNano()}\n\t\t}\n\t\t// Should succeed.\n\t\tstate.Pending = map[uint64]*Pending{75: nt(), 80: nt(), 83: nt(), 90: nt(), 111: nt()}\n\t\tupdateAndCheck()\n\n\t\t// Now do redlivery, but first with no pending.\n\t\tstate.Pending = nil\n\t\tstate.Redelivered = map[uint64]uint64{22: 3, 44: 8}\n\t\tupdateAndCheck()\n\n\t\t// All together.\n\t\tstate.Pending = map[uint64]*Pending{75: nt(), 80: nt(), 83: nt(), 90: nt(), 111: nt()}\n\t\tupdateAndCheck()\n\n\t\t// Large one\n\t\tstate.Delivered.Consumer = 10000\n\t\tstate.Delivered.Stream = 10000\n\t\tstate.AckFloor.Consumer = 100\n\t\tstate.AckFloor.Stream = 100\n\t\t// Generate 8k pending.\n\t\tstate.Pending = make(map[uint64]*Pending)\n\t\tfor len(state.Pending) < 8192 {\n\t\t\tseq := uint64(rand.Intn(9890) + 101)\n\t\t\tif _, ok := state.Pending[seq]; !ok {\n\t\t\t\tstate.Pending[seq] = nt()\n\t\t\t}\n\t\t}\n\t\tupdateAndCheck()\n\n\t\tstate.Pending = nil\n\t\tstate.AckFloor.Consumer = 10000\n\t\tstate.AckFloor.Stream = 10000\n\t\tupdateAndCheck()\n\t})\n}\n\nfunc TestFileStoreConsumerEncodeDecodeRedelivered(t *testing.T) {\n\tstate := &ConsumerState{}\n\n\tstate.Delivered.Consumer = 100\n\tstate.Delivered.Stream = 100\n\tstate.AckFloor.Consumer = 50\n\tstate.AckFloor.Stream = 50\n\n\tstate.Redelivered = map[uint64]uint64{122: 3, 144: 8}\n\tbuf := encodeConsumerState(state)\n\n\trstate, err := decodeConsumerState(buf)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !reflect.DeepEqual(state, rstate) {\n\t\tt.Fatalf(\"States do not match: %+v vs %+v\", state, rstate)\n\t}\n}\n\nfunc TestFileStoreConsumerEncodeDecodePendingBelowStreamAckFloor(t *testing.T) {\n\tstate := &ConsumerState{}\n\n\tstate.Delivered.Consumer = 1192\n\tstate.Delivered.Stream = 10185\n\tstate.AckFloor.Consumer = 1189\n\tstate.AckFloor.Stream = 10815\n\n\tnow := time.Now().Round(time.Second).Add(-10 * time.Second).UnixNano()\n\tstate.Pending = map[uint64]*Pending{\n\t\t10782: {1190, now},\n\t\t10810: {1191, now + int64(time.Second)},\n\t\t10815: {1192, now + int64(2*time.Second)},\n\t}\n\tbuf := encodeConsumerState(state)\n\n\trstate, err := decodeConsumerState(buf)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif len(rstate.Pending) != 3 {\n\t\tt.Fatalf(\"Invalid pending: %v\", rstate.Pending)\n\t}\n\tfor k, v := range state.Pending {\n\t\trv, ok := rstate.Pending[k]\n\t\tif !ok {\n\t\t\tt.Fatalf(\"Did not find sseq=%v\", k)\n\t\t}\n\t\tif !reflect.DeepEqual(v, rv) {\n\t\t\tt.Fatalf(\"Pending for sseq=%v should be %+v, got %+v\", k, v, rv)\n\t\t}\n\t}\n\tstate.Pending, rstate.Pending = nil, nil\n\tif !reflect.DeepEqual(*state, *rstate) {\n\t\tt.Fatalf(\"States do not match: %+v vs %+v\", state, rstate)\n\t}\n}\n\nfunc TestFileStoreWriteFailures(t *testing.T) {\n\t// This test should be run inside an environment where this directory\n\t// has a limited size.\n\t// E.g. Docker\n\t// docker run -ti --tmpfs /jswf_test:rw,size=32k --rm -v ~/Development/go/src:/go/src -w /go/src/github.com/nats-io/nats-server/ golang:1.21 /bin/bash\n\ttdir := filepath.Join(\"/\", \"jswf_test\")\n\tif stat, err := os.Stat(tdir); err != nil || !stat.IsDir() {\n\t\tt.SkipNow()\n\t}\n\n\tstoreDir := filepath.Join(tdir, JetStreamStoreDir)\n\tos.MkdirAll(storeDir, 0755)\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.StoreDir = storeDir\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello Write Failures!\")\n\n\t\tvar lseq uint64\n\t\t// msz about will be ~54 bytes, so if limit is 32k trying to send 1000 will fail at some point.\n\t\tfor i := 1; i <= 1000; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tlseq = uint64(i)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif lseq == 0 {\n\t\t\tt.Fatalf(\"Expected to get a failure but did not\")\n\t\t}\n\n\t\tstate := fs.State()\n\n\t\tif state.LastSeq != lseq-1 {\n\t\t\tt.Fatalf(\"Expected last seq to be %d, got %d\\n\", lseq-1, state.LastSeq)\n\t\t}\n\t\tif state.Msgs != lseq-1 {\n\t\t\tt.Fatalf(\"Expected total msgs to be %d, got %d\\n\", lseq-1, state.Msgs)\n\t\t}\n\t\tif _, err := fs.LoadMsg(lseq, nil); err == nil {\n\t\t\tt.Fatalf(\"Expected error loading seq that failed, got none\")\n\t\t}\n\t\t// Loading should still work.\n\t\tif _, err := fs.LoadMsg(1, nil); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\t// Make sure we recover same state.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate2 := fs.State()\n\n\t\t// Ignore lost state.\n\t\tstate.Lost, state2.Lost = nil, nil\n\t\tif !reflect.DeepEqual(state2, state) {\n\t\t\tt.Fatalf(\"Expected recovered state to be the same\\n%+v\\nvs\\n%+v\\n\", state2, state)\n\t\t}\n\n\t\t// We should still fail here.\n\t\tfor i := 1; i <= 100; i++ {\n\t\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\trequire_Error(t, err)\n\n\t\tlseq = fs.State().LastSeq + 1\n\n\t\t// Purge should help.\n\t\tif _, err := fs.Purge(); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Wait for purge to complete its out of band processing.\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\t// Check we will fail again in same spot.\n\t\tfor i := 1; i <= 1000; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tif i != int(lseq) {\n\t\t\t\t\tt.Fatalf(\"Expected to fail after purge about the same spot, wanted %d got %d\", lseq, i)\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStorePerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.AsyncFlush = true\n\n\t\tsubj, msg := \"foo\", make([]byte, 1024-33)\n\t\tfor i := 0; i < len(msg); i++ {\n\t\t\tmsg[i] = 'D'\n\t\t}\n\t\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t\t// 5GB\n\t\ttoStore := 5 * 1024 * 1024 * 1024 / storedMsgSize\n\n\t\tfmt.Printf(\"storing %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstart := time.Now()\n\t\tfor i := 0; i < int(toStore); i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tfs.Stop()\n\n\t\ttt := time.Since(start)\n\t\tfmt.Printf(\"time to store is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\n\t\tfmt.Printf(\"Filesystem cache flush, paused 5 seconds.\\n\\n\")\n\t\ttime.Sleep(5 * time.Second)\n\n\t\tfmt.Printf(\"Restoring..\\n\")\n\t\tstart = time.Now()\n\t\tfcfg.AsyncFlush = false\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfmt.Printf(\"time to restore is %v\\n\\n\", time.Since(start))\n\n\t\tfmt.Printf(\"LOAD: reading %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tvar smv StoreMsg\n\t\tstart = time.Now()\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tif _, err := fs.LoadMsg(i, &smv); err != nil {\n\t\t\t\tt.Fatalf(\"Error loading %d: %v\", i, err)\n\t\t\t}\n\t\t}\n\n\t\ttt = time.Since(start)\n\t\tfmt.Printf(\"time to read all back messages is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\n\t\t// Do again to test skip for hash..\n\t\tfmt.Printf(\"\\nSKIP CHECKSUM: reading %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tstart = time.Now()\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tif _, err := fs.LoadMsg(i, &smv); err != nil {\n\t\t\t\tt.Fatalf(\"Error loading %d: %v\", i, err)\n\t\t\t}\n\t\t}\n\n\t\ttt = time.Since(start)\n\t\tfmt.Printf(\"time to read all back messages is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfmt.Printf(\"\\nremoving [in order] %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tstart = time.Now()\n\t\t// For reverse order.\n\t\t//for i := toStore; i > 0; i-- {\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tfs.RemoveMsg(i)\n\t\t}\n\t\tfs.Stop()\n\n\t\ttt = time.Since(start)\n\t\tfmt.Printf(\"time to remove all messages is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate := fs.State()\n\t\tif state.Msgs != 0 {\n\t\t\tt.Fatalf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t}\n\t\tif state.Bytes != 0 {\n\t\t\tt.Fatalf(\"Expected no bytes, got %d\", state.Bytes)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreReadBackMsgPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\tsubj := \"foo\"\n\tmsg := []byte(\"ABCDEFGH\") // Smaller shows problems more.\n\n\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t// Make sure we store 2 blocks.\n\ttoStore := defaultLargeBlockSize * 2 / storedMsgSize\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfmt.Printf(\"storing %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstart := time.Now()\n\t\tfor i := 0; i < int(toStore); i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\ttt := time.Since(start)\n\t\tfmt.Printf(\"time to store is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\n\t\t// We should not have cached here with no reads.\n\t\t// Pick something towards end of the block.\n\t\tindex := defaultLargeBlockSize/storedMsgSize - 22\n\t\tstart = time.Now()\n\t\tfs.LoadMsg(index, nil)\n\t\tfmt.Printf(\"Time to load first msg [%d] = %v\\n\", index, time.Since(start))\n\n\t\tstart = time.Now()\n\t\tfs.LoadMsg(index+2, nil)\n\t\tfmt.Printf(\"Time to load second msg [%d] = %v\\n\", index+2, time.Since(start))\n\t})\n}\n\n// This test is testing an upper level stream with a message or byte limit.\n// Even though this is 1, any limit would trigger same behavior once the limit was reached\n// and we were adding and removing.\nfunc TestFileStoreStoreLimitRemovePerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\tsubj, msg := \"foo\", make([]byte, 1024-33)\n\tfor i := 0; i < len(msg); i++ {\n\t\tmsg[i] = 'D'\n\t}\n\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t// 1GB\n\ttoStore := 1 * 1024 * 1024 * 1024 / storedMsgSize\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.RegisterStorageUpdates(func(md, bd int64, seq uint64, subj string) {})\n\n\t\tfmt.Printf(\"storing and removing (limit 1) %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tstart := time.Now()\n\t\tfor i := 0; i < int(toStore); i++ {\n\t\t\tseq, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error storing message: %v\", err)\n\t\t\t}\n\t\t\tif i > 0 {\n\t\t\t\tfs.RemoveMsg(seq - 1)\n\t\t\t}\n\t\t}\n\t\tfs.Stop()\n\n\t\ttt := time.Since(start)\n\t\tfmt.Printf(\"time to store and remove all messages is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\t})\n}\n\nfunc TestFileStorePubPerfWithSmallBlkSize(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\tsubj, msg := \"foo\", make([]byte, 1024-33)\n\tfor i := 0; i < len(msg); i++ {\n\t\tmsg[i] = 'D'\n\t}\n\tstoredMsgSize := fileStoreMsgSize(subj, nil, msg)\n\n\t// 1GB\n\ttoStore := 1 * 1024 * 1024 * 1024 / storedMsgSize\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfmt.Printf(\"storing %d msgs of %s each, totalling %s\\n\",\n\t\t\ttoStore,\n\t\t\tfriendlyBytes(int64(storedMsgSize)),\n\t\t\tfriendlyBytes(int64(toStore*storedMsgSize)),\n\t\t)\n\n\t\tfcfg.BlockSize = FileStoreMinBlkSize\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstart := time.Now()\n\t\tfor i := 0; i < int(toStore); i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tfs.Stop()\n\n\t\ttt := time.Since(start)\n\t\tfmt.Printf(\"time to store is %v\\n\", tt)\n\t\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n\t\tfmt.Printf(\"%s per sec\\n\", friendlyBytes(int64(float64(toStore*storedMsgSize)/tt.Seconds())))\n\t})\n}\n\n// Saw this manifest from a restart test with max delivered set for JetStream.\nfunc TestFileStoreConsumerRedeliveredLost(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tcfg := &ConsumerConfig{AckPolicy: AckExplicit}\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, cfg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\trestartConsumer := func() {\n\t\t\tt.Helper()\n\t\t\to.Stop()\n\t\t\ttime.Sleep(20 * time.Millisecond) // Wait for all things to settle.\n\t\t\to, err = fs.ConsumerStore(\"o22\", time.Time{}, cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Make sure we recovered Redelivered.\n\t\t\tstate, err := o.State()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif state == nil {\n\t\t\t\tt.Fatalf(\"Did not recover state\")\n\t\t\t}\n\t\t\tif len(state.Redelivered) == 0 {\n\t\t\t\tt.Fatalf(\"Did not recover redelivered\")\n\t\t\t}\n\t\t}\n\n\t\tts := time.Now().UnixNano()\n\t\to.UpdateDelivered(1, 1, 1, ts)\n\t\to.UpdateDelivered(2, 1, 2, ts)\n\t\to.UpdateDelivered(3, 1, 3, ts)\n\t\to.UpdateDelivered(4, 1, 4, ts)\n\t\to.UpdateDelivered(5, 2, 1, ts)\n\n\t\trestartConsumer()\n\n\t\to.UpdateDelivered(6, 2, 2, ts)\n\t\to.UpdateDelivered(7, 3, 1, ts)\n\n\t\trestartConsumer()\n\t\tif state, _ := o.State(); len(state.Pending) != 3 {\n\t\t\tt.Fatalf(\"Did not recover pending correctly\")\n\t\t}\n\n\t\to.UpdateAcks(7, 3)\n\t\to.UpdateAcks(6, 2)\n\n\t\trestartConsumer()\n\t\to.UpdateAcks(4, 1)\n\n\t\tstate, _ := o.State()\n\t\tif len(state.Pending) != 0 {\n\t\t\tt.Fatalf(\"Did not clear pending correctly\")\n\t\t}\n\t\tif len(state.Redelivered) != 0 {\n\t\t\tt.Fatalf(\"Did not clear redelivered correctly\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreConsumerFlusher(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Get the underlying impl.\n\t\toc := o.(*consumerFileStore)\n\t\t// Wait for flusher to be running.\n\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\tif !oc.inFlusher() {\n\t\t\t\treturn fmt.Errorf(\"Flusher not running\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Stop and make sure the flusher goes away\n\t\to.Stop()\n\t\t// Wait for flusher to stop.\n\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\tif oc.inFlusher() {\n\t\t\t\treturn fmt.Errorf(\"Flusher still running\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc TestFileStoreConsumerDeliveredUpdates(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Simple consumer, no ack policy configured.\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tdefer o.Stop()\n\n\t\ttestDelivered := func(dseq, sseq uint64) {\n\t\t\tt.Helper()\n\t\t\tts := time.Now().UnixNano()\n\t\t\tif err := o.UpdateDelivered(dseq, sseq, 1, ts); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tstate, err := o.State()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting state: %v\", err)\n\t\t\t}\n\t\t\tif state == nil {\n\t\t\t\tt.Fatalf(\"No state available\")\n\t\t\t}\n\t\t\texpected := SequencePair{dseq, sseq}\n\t\t\tif state.Delivered != expected {\n\t\t\t\tt.Fatalf(\"Unexpected state, wanted %+v, got %+v\", expected, state.Delivered)\n\t\t\t}\n\t\t\tif state.AckFloor != expected {\n\t\t\t\tt.Fatalf(\"Unexpected ack floor state, wanted %+v, got %+v\", expected, state.AckFloor)\n\t\t\t}\n\t\t\tif len(state.Pending) != 0 {\n\t\t\t\tt.Fatalf(\"Did not expect any pending, got %d pending\", len(state.Pending))\n\t\t\t}\n\t\t}\n\n\t\ttestDelivered(1, 100)\n\t\ttestDelivered(2, 110)\n\t\ttestDelivered(5, 130)\n\n\t\t// If we try to do an ack this should err since we are not configured with ack policy.\n\t\tif err := o.UpdateAcks(1, 100); err != ErrNoAckPolicy {\n\t\t\tt.Fatalf(\"Expected a no ack policy error on update acks, got %v\", err)\n\t\t}\n\t\t// Also if we do an update with a delivery count of anything but 1 here should also give same error.\n\t\tts := time.Now().UnixNano()\n\t\tif err := o.UpdateDelivered(5, 130, 2, ts); err != ErrNoAckPolicy {\n\t\t\tt.Fatalf(\"Expected a no ack policy error on update delivered with dc > 1, got %v\", err)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreConsumerDeliveredAndAckUpdates(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Simple consumer, no ack policy configured.\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tdefer o.Stop()\n\n\t\t// Track pending.\n\t\tvar pending int\n\n\t\ttestDelivered := func(dseq, sseq uint64) {\n\t\t\tt.Helper()\n\t\t\tts := time.Now().Round(time.Second).UnixNano()\n\t\t\tif err := o.UpdateDelivered(dseq, sseq, 1, ts); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tpending++\n\t\t\tstate, err := o.State()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting state: %v\", err)\n\t\t\t}\n\t\t\tif state == nil {\n\t\t\t\tt.Fatalf(\"No state available\")\n\t\t\t}\n\t\t\texpected := SequencePair{dseq, sseq}\n\t\t\tif state.Delivered != expected {\n\t\t\t\tt.Fatalf(\"Unexpected delivered state, wanted %+v, got %+v\", expected, state.Delivered)\n\t\t\t}\n\t\t\tif len(state.Pending) != pending {\n\t\t\t\tt.Fatalf(\"Expected %d pending, got %d pending\", pending, len(state.Pending))\n\t\t\t}\n\t\t}\n\n\t\ttestDelivered(1, 100)\n\t\ttestDelivered(2, 110)\n\t\ttestDelivered(3, 130)\n\t\ttestDelivered(4, 150)\n\t\ttestDelivered(5, 165)\n\n\t\ttestBadAck := func(dseq, sseq uint64) {\n\t\t\tt.Helper()\n\t\t\tif err := o.UpdateAcks(dseq, sseq); err == nil {\n\t\t\t\tt.Fatalf(\"Expected error but got none\")\n\t\t\t}\n\t\t}\n\t\ttestBadAck(3, 101)\n\t\ttestBadAck(1, 1)\n\n\t\ttestAck := func(dseq, sseq, dflr, sflr uint64) {\n\t\t\tt.Helper()\n\t\t\tif err := o.UpdateAcks(dseq, sseq); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tpending--\n\t\t\tstate, err := o.State()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting state: %v\", err)\n\t\t\t}\n\t\t\tif state == nil {\n\t\t\t\tt.Fatalf(\"No state available\")\n\t\t\t}\n\t\t\tif len(state.Pending) != pending {\n\t\t\t\tt.Fatalf(\"Expected %d pending, got %d pending\", pending, len(state.Pending))\n\t\t\t}\n\t\t\teflr := SequencePair{dflr, sflr}\n\t\t\tif state.AckFloor != eflr {\n\t\t\t\tt.Fatalf(\"Unexpected ack floor state, wanted %+v, got %+v\", eflr, state.AckFloor)\n\t\t\t}\n\t\t}\n\n\t\ttestAck(1, 100, 1, 109)\n\t\ttestAck(3, 130, 1, 109)\n\t\ttestAck(2, 110, 3, 149) // We do not track explicit state on previous stream floors, so we take last known -1\n\t\ttestAck(5, 165, 3, 149)\n\t\ttestAck(4, 150, 5, 165)\n\n\t\ttestDelivered(6, 170)\n\t\ttestDelivered(7, 171)\n\t\ttestDelivered(8, 172)\n\t\ttestDelivered(9, 173)\n\t\ttestDelivered(10, 200)\n\n\t\ttestAck(7, 171, 5, 165)\n\t\ttestAck(8, 172, 5, 165)\n\n\t\tstate, err := o.State()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting state: %v\", err)\n\t\t}\n\t\to.Stop()\n\n\t\to, err = fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tdefer o.Stop()\n\n\t\tnstate, err := o.State()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting state: %v\", err)\n\t\t}\n\t\tif !reflect.DeepEqual(nstate, state) {\n\t\t\tt.Fatalf(\"States don't match! NEW %+v OLD %+v\", nstate, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreStreamStateDeleted(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, toStore := \"foo\", uint64(10)\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tmsg := []byte(fmt.Sprintf(\"[%08d] Hello World!\", i))\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t}\n\t\t}\n\t\tstate := fs.State()\n\t\tif len(state.Deleted) != 0 {\n\t\t\tt.Fatalf(\"Expected deleted to be empty\")\n\t\t}\n\t\t// Now remove some interior messages.\n\t\tvar expected []uint64\n\t\tfor seq := uint64(2); seq < toStore; seq += 2 {\n\t\t\tfs.RemoveMsg(seq)\n\t\t\texpected = append(expected, seq)\n\t\t}\n\t\tstate = fs.State()\n\t\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t\t}\n\t\t// Now fill the gap by deleting 1 and 3\n\t\tfs.RemoveMsg(1)\n\t\tfs.RemoveMsg(3)\n\t\texpected = expected[2:]\n\t\tstate = fs.State()\n\t\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t\t}\n\t\tif state.FirstSeq != 5 {\n\t\t\tt.Fatalf(\"Expected first seq to be 5, got %d\", state.FirstSeq)\n\t\t}\n\t\tfs.Purge()\n\t\tif state = fs.State(); len(state.Deleted) != 0 {\n\t\t\tt.Fatalf(\"Expected no deleted after purge, got %+v\\n\", state.Deleted)\n\t\t}\n\t})\n}\n\n// We have reports that sometimes under load a stream could complain about a storage directory\n// not being empty.\nfunc TestFileStoreStreamDeleteDirNotEmpty(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, toStore := \"foo\", uint64(10)\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tmsg := []byte(fmt.Sprintf(\"[%08d] Hello World!\", i))\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\tready := make(chan bool)\n\t\tgo func() {\n\t\t\tg := filepath.Join(fcfg.StoreDir, \"g\")\n\t\t\tready <- true\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tos.WriteFile(g, []byte(\"OK\"), defaultFilePerms)\n\t\t\t}\n\t\t}()\n\n\t\t<-ready\n\t\tif err := fs.Delete(true); err != nil {\n\t\t\tt.Fatalf(\"Delete returned an error: %v\", err)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreConsumerPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Get the underlying impl.\n\t\toc := o.(*consumerFileStore)\n\t\t// Wait for flusher to br running\n\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\tif !oc.inFlusher() {\n\t\t\t\treturn fmt.Errorf(\"not in flusher\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Stop flusher for this benchmark since we will invoke directly.\n\t\toc.mu.Lock()\n\t\tqch := oc.qch\n\t\toc.qch = nil\n\t\toc.mu.Unlock()\n\t\tclose(qch)\n\n\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\tif oc.inFlusher() {\n\t\t\t\treturn fmt.Errorf(\"still in flusher\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\ttoStore := uint64(1_000_000)\n\n\t\tstart := time.Now()\n\n\t\tts := start.UnixNano()\n\n\t\tfor i := uint64(1); i <= toStore; i++ {\n\t\t\tif err := o.UpdateDelivered(i, i, 1, ts); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\t\ttt := time.Since(start)\n\t\tfmt.Printf(\"time to update %d is %v\\n\", toStore, tt)\n\t\tfmt.Printf(\"%.0f updates/sec\\n\", float64(toStore)/tt.Seconds())\n\n\t\tstart = time.Now()\n\t\toc.mu.Lock()\n\t\tbuf, err := oc.encodeState()\n\t\toc.mu.Unlock()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error encoding state: %v\", err)\n\t\t}\n\t\tfmt.Printf(\"time to encode %d bytes is %v\\n\", len(buf), time.Since(start))\n\t\tstart = time.Now()\n\t\toc.writeState(buf)\n\t\tfmt.Printf(\"time to write is %v\\n\", time.Since(start))\n\t})\n}\n\n// Reported by Ivan.\nfunc TestFileStoreStreamDeleteCacheBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.CacheExpire = 50 * time.Millisecond\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\n\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif _, err := fs.EraseMsg(1); err != nil {\n\t\t\tt.Fatalf(\"Got an error on remove of %d: %v\", 1, err)\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tif _, err := fs.LoadMsg(2, nil); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t\t}\n\t})\n}\n\n// rip\nfunc TestFileStoreStreamFailToRollBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 512\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxBytes: 300}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure we properly roll underlying blocks.\n\t\tn, msg := 200, bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(\"zzz\", nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\t// Grab some info for introspection.\n\t\tfs.mu.RLock()\n\t\tnumBlks := len(fs.blks)\n\t\tvar index uint32\n\t\tvar blkSize int64\n\t\tif numBlks > 0 {\n\t\t\tmb := fs.blks[0]\n\t\t\tmb.mu.RLock()\n\t\t\tindex = mb.index\n\t\t\tif fi, _ := os.Stat(mb.mfn); fi != nil {\n\t\t\t\tblkSize = fi.Size()\n\t\t\t}\n\t\t\tmb.mu.RUnlock()\n\t\t}\n\t\tfs.mu.RUnlock()\n\n\t\tif numBlks != 1 {\n\t\t\tt.Fatalf(\"Expected only one block, got %d\", numBlks)\n\t\t}\n\t\tif index < 60 {\n\t\t\tt.Fatalf(\"Expected a block index > 60, got %d\", index)\n\t\t}\n\t\tif blkSize > 512 {\n\t\t\tt.Fatalf(\"Expected block to be <= 512, got %d\", blkSize)\n\t\t}\n\t})\n}\n\n// We had a case where a consumer state had a redelivered record that had seq of 0.\n// This was causing the server to panic.\nfunc TestFileStoreBadConsumerState(t *testing.T) {\n\tbs := []byte(\"\\x16\\x02\\x01\\x01\\x03\\x02\\x01\\x98\\xf4\\x8a\\x8a\\f\\x01\\x03\\x86\\xfa\\n\\x01\\x00\\x01\")\n\tif cs, err := decodeConsumerState(bs); err != nil || cs == nil {\n\t\tt.Fatalf(\"Expected to not throw error, got %v and %+v\", err, cs)\n\t}\n}\n\nfunc TestFileStoreExpireMsgsOnStart(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tif fcfg.Compression != NoCompression || fcfg.Cipher != NoCipher {\n\t\t\t// TODO(nat): For some reason this test is very flaky in CI when using\n\t\t\t// encryption or compression but passes fine without.\n\t\t\tt.SkipNow()\n\t\t}\n\n\t\tfcfg.BlockSize = 8 * 1024\n\t\tttl := 250 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"ORDERS\", Subjects: []string{\"orders.*\"}, Storage: FileStorage, MaxAge: ttl}\n\t\tvar fs *fileStore\n\n\t\tstartFS := func() *fileStore {\n\t\t\tt.Helper()\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\treturn fs\n\t\t}\n\n\t\tnewFS := func() *fileStore {\n\t\t\tt.Helper()\n\t\t\tif fs != nil {\n\t\t\t\tfs.Stop()\n\t\t\t\tfs = nil\n\t\t\t}\n\t\t\tremoveDir(t, fcfg.StoreDir)\n\t\t\treturn startFS()\n\t\t}\n\n\t\trestartFS := func(delay time.Duration) *fileStore {\n\t\t\tif fs != nil {\n\t\t\t\tfs.Stop()\n\t\t\t\tfs = nil\n\t\t\t\ttime.Sleep(delay)\n\t\t\t}\n\t\t\tfs = startFS()\n\t\t\treturn fs\n\t\t}\n\n\t\tfs = newFS()\n\t\tdefer fs.Stop()\n\n\t\tmsg := bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\t\tloadMsgs := func(n int) {\n\t\t\tt.Helper()\n\t\t\tfor i := 1; i <= n; i++ {\n\t\t\t\tif _, _, err := fs.StoreMsg(fmt.Sprintf(\"orders.%d\", i%10), nil, msg, 0); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcheckState := func(msgs, first, last uint64) {\n\t\t\tt.Helper()\n\t\t\tif fs == nil {\n\t\t\t\tt.Fatalf(\"No fs\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tstate := fs.State()\n\t\t\tif state.Msgs != msgs {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", msgs, state.Msgs)\n\t\t\t}\n\t\t\tif state.FirstSeq != first {\n\t\t\t\tt.Fatalf(\"Expected %d as first, got %d\", first, state.FirstSeq)\n\t\t\t}\n\t\t\tif state.LastSeq != last {\n\t\t\t\tt.Fatalf(\"Expected %d as last, got %d\", last, state.LastSeq)\n\t\t\t}\n\t\t}\n\n\t\tcheckNumBlks := func(expected int) {\n\t\t\tt.Helper()\n\t\t\tfs.mu.RLock()\n\t\t\tn := len(fs.blks)\n\t\t\tfs.mu.RUnlock()\n\t\t\tif n != expected {\n\t\t\t\tt.Fatalf(\"Expected %d msg blks, got %d\", expected, n)\n\t\t\t}\n\t\t}\n\n\t\t// Check the filtered subject state and make sure that is tracked properly.\n\t\tcheckFiltered := func(subject string, ss SimpleState) {\n\t\t\tt.Helper()\n\t\t\tfss, err := fs.FilteredState(1, subject)\n\t\t\trequire_NoError(t, err)\n\t\t\tif fss != ss {\n\t\t\t\tt.Fatalf(\"Expected FilteredState of %+v, got %+v\", ss, fss)\n\t\t\t}\n\t\t}\n\n\t\t// Make sure state on disk matches (e.g. writeIndexInfo properly called)\n\t\tcheckBlkState := func(index int) {\n\t\t\tt.Helper()\n\t\t\tfs.mu.RLock()\n\t\t\tif index >= len(fs.blks) {\n\t\t\t\tt.Fatalf(\"Out of range, wanted %d but only %d blks\", index, len(fs.blks))\n\t\t\t}\n\t\t\tfs.mu.RUnlock()\n\t\t}\n\n\t\tlastSeqForBlk := func() uint64 {\n\t\t\tt.Helper()\n\t\t\tfs.mu.RLock()\n\t\t\tdefer fs.mu.RUnlock()\n\t\t\tif len(fs.blks) == 0 {\n\t\t\t\tt.Fatalf(\"No blocks?\")\n\t\t\t}\n\t\t\tmb := fs.blks[0]\n\t\t\tmb.mu.RLock()\n\t\t\tdefer mb.mu.RUnlock()\n\t\t\treturn mb.last.seq\n\t\t}\n\n\t\t// Actual testing here.\n\n\t\tloadMsgs(500)\n\t\trestartFS(ttl + 100*time.Millisecond)\n\t\tcheckState(0, 501, 500)\n\t\t// We actually hold onto the last one now to remember our starting sequence.\n\t\tcheckNumBlks(1)\n\n\t\t// Now check partial expires and the fss tracking state.\n\t\t// Small numbers is to keep them in one block.\n\t\tfs = newFS()\n\t\tloadMsgs(10)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tloadMsgs(10)\n\t\tcheckFiltered(\"orders.*\", SimpleState{Msgs: 20, First: 1, Last: 20})\n\n\t\trestartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half\n\t\tcheckState(10, 11, 20)\n\t\tcheckNumBlks(1)\n\t\tcheckFiltered(\"orders.*\", SimpleState{Msgs: 10, First: 11, Last: 20})\n\t\tcheckFiltered(\"orders.5\", SimpleState{Msgs: 1, First: 15, Last: 15})\n\t\tcheckBlkState(0)\n\n\t\tfs = newFS()\n\t\tloadMsgs(5)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tloadMsgs(15)\n\t\trestartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half\n\t\tcheckState(15, 6, 20)\n\t\tcheckFiltered(\"orders.*\", SimpleState{Msgs: 15, First: 6, Last: 20})\n\t\tcheckFiltered(\"orders.5\", SimpleState{Msgs: 2, First: 10, Last: 20})\n\n\t\t// Now we want to test that if the end of a msg block is all deletes msgs that we do the right thing.\n\t\tfs = newFS()\n\t\tloadMsgs(150)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tloadMsgs(100)\n\n\t\tcheckNumBlks(5)\n\n\t\t// Now delete 10 messages from the end of the first block which we will expire on restart.\n\t\t// We will expire up to seq 100, so delete 91-100.\n\t\tlseq := lastSeqForBlk()\n\t\tfor seq := lseq; seq > lseq-10; seq-- {\n\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\tif err != nil || !removed {\n\t\t\t\tt.Fatalf(\"Error removing message: %v\", err)\n\t\t\t}\n\t\t}\n\t\trestartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half\n\t\tcheckState(100, 151, 250)\n\t\tcheckNumBlks(3) // We should only have 3 blks left.\n\t\tcheckBlkState(0)\n\n\t\t// Now make sure that we properly clean up any internal dmap entries (sparse) when expiring.\n\t\tfs = newFS()\n\t\tloadMsgs(10)\n\t\t// Remove some in sparse fashion, adding to dmap.\n\t\tfs.RemoveMsg(2)\n\t\tfs.RemoveMsg(4)\n\t\tfs.RemoveMsg(6)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tloadMsgs(10)\n\t\trestartFS(ttl - 100*time.Millisecond + 25*time.Millisecond) // Just want half\n\t\tcheckState(10, 11, 20)\n\t\tcheckNumBlks(1)\n\t\tcheckBlkState(0)\n\n\t\t// Make sure expiring a block with tail deleted messages removes the message block etc.\n\t\tfs = newFS()\n\t\tloadMsgs(7)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tloadMsgs(3)\n\t\tfs.RemoveMsg(8)\n\t\tfs.RemoveMsg(9)\n\t\tfs.RemoveMsg(10)\n\t\trestartFS(ttl - 100*time.Millisecond + 25*time.Millisecond)\n\t\tcheckState(0, 11, 10)\n\n\t\tfs.Stop()\n\t\t// Not for start per se but since we have all the test tooling here check that Compact() does right thing as well.\n\t\tfs = newFS()\n\t\tdefer fs.Stop()\n\t\tloadMsgs(100)\n\t\tcheckFiltered(\"orders.*\", SimpleState{Msgs: 100, First: 1, Last: 100})\n\t\tcheckFiltered(\"orders.5\", SimpleState{Msgs: 10, First: 5, Last: 95})\n\t\t// Check that Compact keeps fss updated, does dmap etc.\n\t\tfs.Compact(51)\n\t\tcheckFiltered(\"orders.*\", SimpleState{Msgs: 50, First: 51, Last: 100})\n\t\tcheckFiltered(\"orders.5\", SimpleState{Msgs: 5, First: 55, Last: 95})\n\t\tcheckBlkState(0)\n\t})\n}\n\nfunc TestFileStoreSparseCompaction(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1024 * 1024\n\t\tcfg := StreamConfig{Name: \"KV\", Subjects: []string{\"kv.>\"}, Storage: FileStorage}\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tmsg := bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\t\tloadMsgs := func(n int) {\n\t\t\tt.Helper()\n\t\t\tfor i := 1; i <= n; i++ {\n\t\t\t\tif _, _, err := fs.StoreMsg(fmt.Sprintf(\"kv.%d\", i%10), nil, msg, 0); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcheckState := func(msgs, first, last uint64) {\n\t\t\tt.Helper()\n\t\t\tif fs == nil {\n\t\t\t\tt.Fatalf(\"No fs\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tstate := fs.State()\n\t\t\tif state.Msgs != msgs {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", msgs, state.Msgs)\n\t\t\t}\n\t\t\tif state.FirstSeq != first {\n\t\t\t\tt.Fatalf(\"Expected %d as first, got %d\", first, state.FirstSeq)\n\t\t\t}\n\t\t\tif state.LastSeq != last {\n\t\t\t\tt.Fatalf(\"Expected %d as last, got %d\", last, state.LastSeq)\n\t\t\t}\n\t\t}\n\n\t\tdeleteMsgs := func(seqs ...uint64) {\n\t\t\tt.Helper()\n\t\t\tfor _, seq := range seqs {\n\t\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\t\tif err != nil || !removed {\n\t\t\t\t\tt.Fatalf(\"Got an error on remove of %d: %v\", seq, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\teraseMsgs := func(seqs ...uint64) {\n\t\t\tt.Helper()\n\t\t\tfor _, seq := range seqs {\n\t\t\t\tremoved, err := fs.EraseMsg(seq)\n\t\t\t\tif err != nil || !removed {\n\t\t\t\t\tt.Fatalf(\"Got an error on erase of %d: %v\", seq, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcompact := func() {\n\t\t\tt.Helper()\n\t\t\tvar ssb, ssa StreamState\n\t\t\tfs.FastState(&ssb)\n\t\t\ttb, ub, _ := fs.Utilization()\n\n\t\t\tfs.mu.RLock()\n\t\t\tif len(fs.blks) < 2 {\n\t\t\t\tt.Fatalf(\"Not enough blocks?\")\n\t\t\t}\n\t\t\tmb := fs.blks[0]\n\t\t\tfs.mu.RUnlock()\n\n\t\t\tmb.mu.Lock()\n\t\t\terr = mb.compact()\n\t\t\tmb.mu.Unlock()\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfs.FastState(&ssa)\n\t\t\tif !reflect.DeepEqual(ssb, ssa) {\n\t\t\t\tt.Fatalf(\"States do not match\\n; %+v \\nvs %+v\", ssb, ssa)\n\t\t\t}\n\t\t\tta, ua, _ := fs.Utilization()\n\t\t\tif ub != ua {\n\t\t\t\tt.Fatalf(\"Expected used to be the same, got %d vs %d\", ub, ua)\n\t\t\t}\n\t\t\t// When using both encryption and compression, we're not always\n\t\t\t// guaranteed to have a smaller file after compaction.\n\t\t\tif ta >= tb && (fcfg.Cipher == NoCipher || fcfg.Compression == NoCompression) {\n\t\t\t\tt.Fatalf(\"Expected total after to be less then before, got %d vs %d\", tb, ta)\n\t\t\t}\n\t\t}\n\n\t\t// Actual testing here.\n\t\tloadMsgs(1000)\n\t\tcheckState(1000, 1, 1000)\n\n\t\t// Create a new lmb, since we'll compact the current one and that's not allowed on lmb.\n\t\tfs.mu.RLock()\n\t\tblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, blks, 1)\n\t\tstate := fs.State()\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, fs.writeTombstone(state.LastSeq, state.LastTime.UnixNano()))\n\n\t\t// Now delete a few messages.\n\t\tdeleteMsgs(1)\n\t\tcompact()\n\n\t\tdeleteMsgs(1000, 999, 998, 997)\n\t\tcompact()\n\n\t\teraseMsgs(500, 502, 504, 506, 508, 510)\n\t\tcompact()\n\t})\n}\n\nfunc TestFileStoreSparseCompactionWithInteriorDeletes(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"KV\", Subjects: []string{\"kv.>\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 1; i <= 1000; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(fmt.Sprintf(\"kv.%d\", i%10), nil, []byte(\"OK\"), 0); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\t// Now do interior deletes.\n\t\tfor _, seq := range []uint64{500, 600, 700, 800} {\n\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\tif err != nil || !removed {\n\t\t\t\tt.Fatalf(\"Got an error on remove of %d: %v\", seq, err)\n\t\t\t}\n\t\t}\n\n\t\t_, err = fs.LoadMsg(900, nil)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\t// Do compact by hand, make sure we can still access msgs past the interior deletes.\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tif err = lmb.dirtyCloseWithRemove(false); err != nil {\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tif err = lmb.compact(); err != nil {\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfs.mu.RUnlock()\n\n\t\tif _, err = fs.LoadMsg(900, nil); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t})\n}\n\n// When messages span multiple blocks and we want to purge but keep some amount, say 1, we would remove all.\n// This is because we would not break out of iterator across more message blocks.\n// Issue #2622\nfunc TestFileStorePurgeExKeepOneBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 128\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfill := bytes.Repeat([]byte(\"X\"), 128)\n\n\t\tfs.StoreMsg(\"A\", nil, []byte(\"META\"), 0)\n\t\tfs.StoreMsg(\"B\", nil, fill, 0)\n\t\tfs.StoreMsg(\"A\", nil, []byte(\"META\"), 0)\n\t\tfs.StoreMsg(\"B\", nil, fill, 0)\n\n\t\tfss, err := fs.FilteredState(1, \"A\")\n\t\trequire_NoError(t, err)\n\t\tif fss.Msgs != 2 {\n\t\t\tt.Fatalf(\"Expected to find 2 `A` msgs, got %d\", fss.Msgs)\n\t\t}\n\n\t\tn, err := fs.PurgeEx(\"A\", 0, 1)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif n != 1 {\n\t\t\tt.Fatalf(\"Expected PurgeEx to remove 1 `A` msgs, got %d\", n)\n\t\t}\n\t\tfss, err = fs.FilteredState(1, \"A\")\n\t\trequire_NoError(t, err)\n\t\tif fss.Msgs != 1 {\n\t\t\tt.Fatalf(\"Expected to find 1 `A` msgs, got %d\", fss.Msgs)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreFilteredPendingBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, []byte(\"msg\"), 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"bar\", nil, []byte(\"msg\"), 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"baz\", nil, []byte(\"msg\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\tfs.mu.Lock()\n\t\tmb := fs.lmb\n\t\tfs.mu.Unlock()\n\n\t\ttotal, f, l, err := mb.filteredPending(\"foo\", false, 3)\n\t\trequire_NoError(t, err)\n\t\tif total != 0 {\n\t\t\tt.Fatalf(\"Expected total of 0 but got %d\", total)\n\t\t}\n\t\tif f != 0 || l != 0 {\n\t\t\tt.Fatalf(\"Expected first and last to be 0 as well, but got %d %d\", f, l)\n\t\t}\n\t})\n}\n\n// Test to optimize the selectMsgBlock with lots of blocks.\nfunc TestFileStoreFetchPerf(t *testing.T) {\n\t// Comment out to run.\n\tt.SkipNow()\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 8192\n\t\tfcfg.AsyncFlush = true\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Will create 25k msg blocks.\n\t\tn, subj, msg := 100_000, \"zzz\", bytes.Repeat([]byte(\"ABC\"), 600)\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, _, err := fs.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\t// Time how long it takes us to load all messages.\n\t\tvar smv StoreMsg\n\t\tnow := time.Now()\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, err := fs.LoadMsg(uint64(i), &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error looking up seq %d: %v\", i, err)\n\t\t\t}\n\t\t}\n\t\tfmt.Printf(\"Elapsed to load all messages is %v\\n\", time.Since(now))\n\t})\n}\n\n// For things like raft log when we compact and have a message block that could reclaim > 50% of space for block we want to do that.\n// https://github.com/nats-io/nats-server/issues/2936\nfunc TestFileStoreCompactReclaimHeadSpace(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 4 * 1024 * 1024\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Create random bytes for payload to test for corruption vs repeated.\n\t\tmsg := make([]byte, 64*1024)\n\t\tcrand.Read(msg)\n\n\t\t// This gives us ~63 msgs in first and ~37 in second.\n\t\tn, subj := 100, \"z\"\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tcheckNumBlocks := func(n int) {\n\t\t\tt.Helper()\n\t\t\tfs.mu.RLock()\n\t\t\tdefer fs.mu.RUnlock()\n\t\t\tif len(fs.blks) != n {\n\t\t\t\tt.Fatalf(\"Expected to have %d blocks, got %d\", n, len(fs.blks))\n\t\t\t}\n\t\t}\n\n\t\tgetBlock := func(index int) *msgBlock {\n\t\t\tt.Helper()\n\t\t\tfs.mu.RLock()\n\t\t\tdefer fs.mu.RUnlock()\n\t\t\treturn fs.blks[index]\n\t\t}\n\n\t\t// Check that we did right thing and actually reclaimed since > 50%\n\t\tcheckBlock := func(mb *msgBlock) {\n\t\t\tt.Helper()\n\n\t\t\tmb.mu.RLock()\n\t\t\trbytes, mfn := mb.rbytes, mb.mfn\n\t\t\tfseq, lseq := mb.first.seq, mb.last.seq\n\t\t\tmb.mu.RUnlock()\n\n\t\t\t// Check that sizes match as long as we are not doing compression.\n\t\t\tif fcfg.Compression == NoCompression {\n\t\t\t\tfile, err := os.Open(mfn)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer file.Close()\n\t\t\t\tfi, err := file.Stat()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif rbytes != uint64(fi.Size()) {\n\t\t\t\t\tt.Fatalf(\"Expected to rbytes == fi.Size, got %d vs %d\", rbytes, fi.Size())\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Make sure we can pull messages and that they are ok.\n\t\t\tvar smv StoreMsg\n\t\t\tsm, err := fs.LoadMsg(fseq, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\tif !bytes.Equal(sm.msg, msg) {\n\t\t\t\tt.Fatalf(\"Msgs don't match, original %q vs %q\", msg, sm.msg)\n\t\t\t}\n\t\t\tsm, err = fs.LoadMsg(lseq, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\tif !bytes.Equal(sm.msg, msg) {\n\t\t\t\tt.Fatalf(\"Msgs don't match, original %q vs %q\", msg, sm.msg)\n\t\t\t}\n\t\t}\n\n\t\tcheckNumBlocks(2)\n\t\t_, err = fs.Compact(33)\n\t\trequire_NoError(t, err)\n\n\t\tcheckNumBlocks(2)\n\t\tcheckBlock(getBlock(0))\n\t\tcheckBlock(getBlock(1))\n\n\t\t_, err = fs.Compact(85)\n\t\trequire_NoError(t, err)\n\n\t\tcheckNumBlocks(1)\n\t\tcheckBlock(getBlock(0))\n\n\t\t// Make sure we can write.\n\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\tcheckNumBlocks(1)\n\t\tcheckBlock(getBlock(0))\n\n\t\t// Stop and start again.\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tcheckNumBlocks(1)\n\t\tcheckBlock(getBlock(0))\n\n\t\t// Make sure we can write.\n\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t})\n}\n\nfunc TestFileStoreRememberLastMsgTime(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tvar fs *fileStore\n\t\tcfg := StreamConfig{Name: \"TEST\", Storage: FileStorage, MaxAge: 1 * time.Second}\n\n\t\tgetFS := func() *fileStore {\n\t\t\tt.Helper()\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\treturn fs\n\t\t}\n\t\trestartFS := func() {\n\t\t\tt.Helper()\n\t\t\tfs.Stop()\n\t\t\tfs = getFS()\n\t\t}\n\n\t\tmsg := bytes.Repeat([]byte(\"X\"), 2*1024*1024)\n\n\t\t// Get first one.\n\t\tfs = getFS()\n\t\tdefer fs.Stop()\n\n\t\tseq, ts, err := fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t\t// We will test that last msg time survives from delete, purge and expires after restart.\n\t\tremoved, err := fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\n\t\tlt := time.Unix(0, ts).UTC()\n\t\trequire_True(t, lt == fs.State().LastTime)\n\n\t\t// Restart\n\t\trestartFS()\n\n\t\t// Test that last time survived.\n\t\trequire_True(t, lt == fs.State().LastTime)\n\n\t\tseq, ts, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\tvar smv StoreMsg\n\t\t_, err = fs.LoadMsg(seq, &smv)\n\t\trequire_NoError(t, err)\n\n\t\tfs.Purge()\n\n\t\t// Restart\n\t\trestartFS()\n\n\t\tlt = time.Unix(0, ts).UTC()\n\t\trequire_True(t, lt == fs.State().LastTime)\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t\tseq, ts, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\trequire_True(t, seq == 4)\n\n\t\t// Wait til messages expire.\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tstate := fs.State()\n\t\t\tif state.Msgs == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Still has %d msgs\", state.Msgs)\n\t\t})\n\n\t\t// Restart\n\t\trestartFS()\n\n\t\tlt = time.Unix(0, ts).UTC()\n\t\trequire_True(t, lt == fs.State().LastTime)\n\n\t\t// Now make sure we retain the true last seq.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t\tseq, ts, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\trequire_True(t, seq == 6)\n\t\tremoved, err = fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\n\t\tremoved, err = fs.RemoveMsg(seq - 1)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\n\t\t// Restart\n\t\trestartFS()\n\n\t\tlt = time.Unix(0, ts).UTC()\n\t\trequire_True(t, lt == fs.State().LastTime)\n\t\trequire_True(t, seq == 6)\n\t})\n}\n\nfunc (fs *fileStore) getFirstBlock() *msgBlock {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\tif len(fs.blks) == 0 {\n\t\treturn nil\n\t}\n\treturn fs.blks[0]\n}\n\nfunc TestFileStoreRebuildStateDmapAccountingBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1024 * 1024\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t// Delete 2-40.\n\t\tfor i := 2; i <= 40; i++ {\n\t\t\t_, err := fs.RemoveMsg(uint64(i))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tmb := fs.getFirstBlock()\n\t\trequire_True(t, mb != nil)\n\n\t\tcheck := func() {\n\t\t\tt.Helper()\n\t\t\tmb.mu.RLock()\n\t\t\tdefer mb.mu.RUnlock()\n\t\t\tdmapLen := uint64(mb.dmap.Size())\n\t\t\tif mb.msgs != (mb.last.seq-mb.first.seq+1)-dmapLen {\n\t\t\t\tt.Fatalf(\"Consistency check failed: %d != %d -> last %d first %d len(dmap) %d\",\n\t\t\t\t\tmb.msgs, (mb.last.seq-mb.first.seq+1)-dmapLen, mb.last.seq, mb.first.seq, dmapLen)\n\t\t\t}\n\t\t}\n\n\t\tcheck()\n\n\t\tmb.mu.Lock()\n\t\terr = mb.compact()\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\n\t\t// Now delete first.\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\n\t\tmb.mu.Lock()\n\t\t_, _, err = mb.rebuildStateLocked()\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\n\t\tcheck()\n\t})\n}\n\nfunc TestFileStorePurgeExWithSubject(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1000\n\t\tcfg := StreamConfig{Name: \"TEST\", Subjects: []string{\"foo.>\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tpayload := make([]byte, 20)\n\n\t\t_, _, err = fs.StoreMsg(\"foo.0\", nil, payload, 0)\n\t\trequire_NoError(t, err)\n\n\t\ttotal := 200\n\t\tfor i := 0; i < total; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo.1\", nil, payload, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, _, err = fs.StoreMsg(\"foo.2\", nil, []byte(\"xxxxxx\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Make sure we have our state file prior to Purge call.\n\t\trequire_NoError(t, fs.forceWriteFullState())\n\n\t\t// Capture the current index.db file.\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(buf) > 0)\n\n\t\t// This should purge all \"foo.1\"\n\t\tp, err := fs.PurgeEx(\"foo.1\", 1, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, p, uint64(total))\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\n\t\t// Make sure we can recover same state.\n\t\trequire_NoError(t, fs.Stop())\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tbefore := state\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\t// Also make sure we can recover properly with no index.db present.\n\t\t// We want to make sure we preserve any tombstones from the subject based purge.\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of\\n %+v, got\\n %+v without index.db state\", before, state)\n\t\t}\n\n\t\t// If we had an index.db from after PurgeEx but before Stop() would rewrite, make sure we\n\t\t// properly can recover with the old index file. This would be a crash after the PurgeEx() call.\n\t\trequire_NoError(t, fs.Stop())\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v with old index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStorePurgeExNoTombsOnBlockRemoval(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1000\n\t\tcfg := StreamConfig{Name: \"TEST\", Subjects: []string{\"foo.>\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tpayload := make([]byte, 20)\n\n\t\ttotal := 100\n\t\tfor i := 0; i < total; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo.1\", nil, payload, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, _, err = fs.StoreMsg(\"foo.2\", nil, payload, 0)\n\t\trequire_NoError(t, err)\n\n\t\trequire_Equal(t, fs.numMsgBlocks(), 6)\n\n\t\t// Make sure we have our state file prior to Purge call.\n\t\tfs.forceWriteFullState()\n\n\t\t// Capture the current index.db file if it exists.\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(buf) > 0)\n\n\t\t// This should purge all \"foo.1\". This will remove the blocks so we want to make sure\n\t\t// we do not write excessive tombstones here.\n\t\tp, err := fs.PurgeEx(\"foo.1\", 1, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, p, uint64(total))\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 1)\n\t\trequire_Equal(t, state.FirstSeq, 101)\n\n\t\t// Check that we only have 1 msg block.\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\t\t// Put the old index.db back. We want to make sure without the empty block tombstones that we\n\t\t// properly recover state.\n\t\tfs.Stop()\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 1)\n\t\trequire_Equal(t, state.FirstSeq, 101)\n\t})\n}\n\n// When the N.idx file is shorter than the previous write we could fail to recover the idx properly.\n// For instance, with encryption and an expiring stream that has no messages, when a restart happens the decrypt will fail\n// since their are extra bytes, and this could lead to a stream sequence reset to zero.\n//\n// NOTE: We do not use idx files anymore, but keeping test.\nfunc TestFileStoreShortIndexWriteBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t// Encrypted mode shows, but could effect non-encrypted mode.\n\t\tcfg := StreamConfig{Name: \"TEST\", Storage: FileStorage, MaxAge: time.Second}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t// Wait til messages all go away.\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif state := fs.State(); state.Msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tif state := fs.State(); state.FirstSeq != 101 {\n\t\t\tt.Fatalf(\"Expected first sequence of 101 vs %d\", state.FirstSeq)\n\t\t}\n\n\t\t// Now restart..\n\t\tfs.Stop()\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); state.FirstSeq != 101 || state.LastSeq != 100 {\n\t\t\tt.Fatalf(\"Expected first sequence of 101 vs %d\", state.FirstSeq)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreDoubleCompactWithWriteInBetweenEncryptedBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"ouch\")\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\t_, err = fs.Compact(5)\n\t\trequire_NoError(t, err)\n\n\t\tif state := fs.State(); state.LastSeq != 5 {\n\t\t\tt.Fatalf(\"Expected last sequence to be 5 but got %d\", state.LastSeq)\n\t\t}\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\t_, err = fs.Compact(10)\n\t\trequire_NoError(t, err)\n\n\t\tif state := fs.State(); state.LastSeq != 10 {\n\t\t\tt.Fatalf(\"Expected last sequence to be 10 but got %d\", state.LastSeq)\n\t\t}\n\t})\n}\n\n// When we kept the empty block for tracking sequence, we needed to reset the bek\n// counter when encrypted for subsequent writes to be correct. The bek in place could\n// possibly still have a non-zero counter from previous writes.\n// Happens when all messages expire and the are flushed and then subsequent writes occur.\nfunc TestFileStoreEncryptedKeepIndexNeedBekResetBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tttl := 1 * time.Second\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, MaxAge: ttl}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"ouch\")\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\t// Want to go to 0.\n\t\t// This will leave the marker.\n\t\tcheckFor(t, 5*time.Second, ttl, func() error {\n\t\t\tif state := fs.State(); state.Msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Now write additional messages.\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\t// Make sure the buffer is cleared.\n\t\tfs.mu.RLock()\n\t\tmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tmb.mu.Lock()\n\t\tmb.clearCacheAndOffset()\n\t\tmb.mu.Unlock()\n\n\t\t// Now make sure we can read.\n\t\tvar smv StoreMsg\n\t\t_, err = fs.LoadMsg(10, &smv)\n\t\trequire_NoError(t, err)\n\t})\n}\n\nfunc (fs *fileStore) reportMeta() (hasPSIM, hasAnyFSS bool) {\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\thasPSIM = fs.psim != nil\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\thasAnyFSS = hasAnyFSS || mb.fss != nil\n\t\tmb.mu.RUnlock()\n\t\tif hasAnyFSS {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn hasPSIM, hasAnyFSS\n}\n\nfunc TestFileStoreExpireSubjectMeta(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1024\n\t\tfcfg.CacheExpire = 500 * time.Millisecond\n\t\tfcfg.SubjectStateExpire = time.Second\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"kv.>\"}, Storage: FileStorage, MaxMsgsPer: 1}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tns := 100\n\t\tfor i := 1; i <= ns; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Test that on restart we do not have extensize metadata but do have correct number of subjects/keys.\n\t\t// Only thing really needed for store state / stream info.\n\t\tfs.Stop()\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\tif ss.NumSubjects != ns {\n\t\t\tt.Fatalf(\"Expected NumSubjects of %d, got %d\", ns, ss.NumSubjects)\n\t\t}\n\n\t\t// Make sure we clear mb fss meta\n\t\tcheckFor(t, fcfg.SubjectStateExpire*2, 500*time.Millisecond, func() error {\n\t\t\tif _, hasAnyFSS := fs.reportMeta(); hasAnyFSS {\n\t\t\t\treturn fmt.Errorf(\"Still have mb fss state\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// LoadLast, which is what KV uses, should load meta and succeed.\n\t\t_, err = fs.LoadLastMsg(\"kv.22\", nil)\n\t\trequire_NoError(t, err)\n\t\t// Make sure we clear mb fss meta\n\t\tcheckFor(t, fcfg.SubjectStateExpire*2, 500*time.Millisecond, func() error {\n\t\t\tif _, hasAnyFSS := fs.reportMeta(); hasAnyFSS {\n\t\t\t\treturn fmt.Errorf(\"Still have mb fss state\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc TestFileStoreMaxMsgsPerSubject(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 128\n\t\tfcfg.CacheExpire = time.Second\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"kv.>\"}, Storage: FileStorage, MaxMsgsPer: 1}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tns := 100\n\t\tfor i := 1; i <= ns; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor i := 1; i <= ns; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tif state := fs.State(); state.Msgs != 100 || state.FirstSeq != 101 || state.LastSeq != 200 || len(state.Deleted) != 0 {\n\t\t\tt.Fatalf(\"Bad state: %+v\", state)\n\t\t}\n\n\t\tif nb := fs.numMsgBlocks(); nb != 34 {\n\t\t\tt.Fatalf(\"Expected 34 blocks, got %d\", nb)\n\t\t}\n\t})\n}\n\n// Testing the case in https://github.com/nats-io/nats-server/issues/4247\nfunc TestFileStoreMaxMsgsAndMaxMsgsPerSubject(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 128\n\t\tfcfg.CacheExpire = time.Second\n\t\tcfg := StreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\tStorage: FileStorage,\n\t\t\tDiscard: DiscardNew, MaxMsgs: 100, // Total stream policy\n\t\t\tDiscardNewPer: true, MaxMsgsPer: 1, // Per-subject policy\n\t\t}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 1; i <= 101; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\tif i == 101 {\n\t\t\t\t// The 101th iteration should fail because MaxMsgs is set to\n\t\t\t\t// 100 and the policy is DiscardNew.\n\t\t\t\trequire_Error(t, err)\n\t\t\t} else {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\n\t\tfor i := 1; i <= 100; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\t// All of these iterations should fail because MaxMsgsPer is set\n\t\t\t// to 1 and DiscardNewPer is set to true, forcing us to reject\n\t\t\t// cases where there is already a message on this subject.\n\t\t\trequire_Error(t, err)\n\t\t}\n\n\t\tif state := fs.State(); state.Msgs != 100 || state.FirstSeq != 1 || state.LastSeq != 100 || len(state.Deleted) != 0 {\n\t\t\t// There should be 100 messages exactly, as the 101st subject\n\t\t\t// should have been rejected in the first loop, and any duplicates\n\t\t\t// on the other subjects should have been rejected in the second loop.\n\t\t\tt.Fatalf(\"Bad state: %+v\", state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreSubjectStateCacheExpiration(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 32\n\t\tfcfg.CacheExpire = time.Second\n\t\tfcfg.SubjectStateExpire = time.Second\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"kv.>\"}, Storage: FileStorage, MaxMsgsPer: 2}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 1; i <= 100; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.foo.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 1; i <= 100; i++ {\n\t\t\tsubj := fmt.Sprintf(\"kv.bar.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"value\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Make sure we clear mb fss meta before asking for SubjectState.\n\t\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif _, hasAnyFSS := fs.reportMeta(); hasAnyFSS {\n\t\t\t\treturn fmt.Errorf(\"Still have mb fss state\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tif fss := fs.SubjectsState(\"kv.bar.>\"); len(fss) != 100 {\n\t\t\tt.Fatalf(\"Expected 100 entries but got %d\", len(fss))\n\t\t}\n\n\t\tfss := fs.SubjectsState(\"kv.bar.99\")\n\t\tif len(fss) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 entry but got %d\", len(fss))\n\t\t}\n\t\texpected := SimpleState{Msgs: 1, First: 199, Last: 199}\n\t\tif ss := fss[\"kv.bar.99\"]; ss != expected {\n\t\t\tt.Fatalf(\"Bad subject state, expected %+v but got %+v\", expected, ss)\n\t\t}\n\n\t\t// Now add one to end and check as well for non-wildcard.\n\t\t_, _, err = fs.StoreMsg(\"kv.foo.1\", nil, []byte(\"value22\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\tif state := fs.State(); state.Msgs != 201 {\n\t\t\tt.Fatalf(\"Expected 201 msgs but got %+v\", state)\n\t\t}\n\n\t\tfss = fs.SubjectsState(\"kv.foo.1\")\n\t\tif len(fss) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 entry but got %d\", len(fss))\n\t\t}\n\t\texpected = SimpleState{Msgs: 2, First: 1, Last: 201}\n\t\tif ss := fss[\"kv.foo.1\"]; ss != expected {\n\t\t\tt.Fatalf(\"Bad subject state, expected %+v but got %+v\", expected, ss)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreEncrypted(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"aes ftw\")\n\t\tfor i := 0; i < 50; i++ {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{})\n\t\trequire_NoError(t, err)\n\n\t\tstate := &ConsumerState{}\n\t\tstate.Delivered.Consumer = 22\n\t\tstate.Delivered.Stream = 22\n\t\tstate.AckFloor.Consumer = 11\n\t\tstate.AckFloor.Stream = 11\n\t\terr = o.Update(state)\n\t\trequire_NoError(t, err)\n\n\t\to.Stop()\n\t\tfs.Stop()\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Now make sure we can read.\n\t\tvar smv StoreMsg\n\t\tsm, err := fs.LoadMsg(10, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, string(sm.msg) == \"aes ftw\")\n\n\t\to, err = fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{})\n\t\trequire_NoError(t, err)\n\t\trstate, err := o.State()\n\t\trequire_NoError(t, err)\n\n\t\tif rstate.Delivered != state.Delivered || rstate.AckFloor != state.AckFloor {\n\t\t\tt.Fatalf(\"Bad recovered consumer state, expected %+v got %+v\", state, rstate)\n\t\t}\n\t})\n}\n\n// Make sure we do not go through block loads when we know no subjects will exists, e.g. raft.\nfunc TestFileStoreNoFSSWhenNoSubjects(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tn, msg := 100, []byte(\"raft state\")\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, _, err := fs.StoreMsg(_EMPTY_, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tstate := fs.State()\n\t\trequire_True(t, state.Msgs == uint64(n))\n\n\t\tfs.Stop()\n\t\tfs, err = newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure we did not load the block trying to generate fss.\n\t\tfs.mu.RLock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.RUnlock()\n\n\t\tmb.mu.Lock()\n\t\tdefer mb.mu.Unlock()\n\n\t\tif mb.cloads > 0 {\n\t\t\tt.Fatalf(\"Expected no cache loads but got %d\", mb.cloads)\n\t\t}\n\t\tif mb.fss != nil {\n\t\t\tt.Fatalf(\"Expected fss to be nil\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreNoFSSBugAfterRemoveFirst(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 8 * 1024 * 1024\n\t\tfcfg.CacheExpire = 200 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.bar.*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tn, msg := 100, bytes.Repeat([]byte(\"ZZZ\"), 33) // ~100bytes\n\t\tfor i := 0; i < n; i++ {\n\t\t\tsubj := fmt.Sprintf(\"foo.bar.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tstate := fs.State()\n\t\trequire_True(t, state.Msgs == uint64(n))\n\n\t\t// Let fss expire.\n\t\ttime.Sleep(250 * time.Millisecond)\n\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\n\t\tsm, _, err := fs.LoadNextMsg(\"foo.>\", true, 1, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, sm.subj == \"foo.bar.1\")\n\n\t\t// Make sure mb.fss does not have the entry for foo.bar.0\n\t\tfs.mu.Lock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.Unlock()\n\t\tmb.mu.RLock()\n\t\tss, ok := mb.fss.Find([]byte(\"foo.bar.0\"))\n\t\tmb.mu.RUnlock()\n\n\t\tif ok && ss != nil {\n\t\t\tt.Fatalf(\"Expected no state for %q, but got %+v\\n\", \"foo.bar.0\", ss)\n\t\t}\n\t})\n}\n\n// NOTE: We do not use fss files anymore, but leaving test in place.\nfunc TestFileStoreNoFSSAfterRecover(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tn, msg := 100, []byte(\"no fss for you!\")\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, _, err := fs.StoreMsg(_EMPTY_, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tstate := fs.State()\n\t\trequire_True(t, state.Msgs == uint64(n))\n\n\t\tfs.Stop()\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure we did not load the block trying to generate fss.\n\t\tfs.mu.RLock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.RUnlock()\n\n\t\tmb.mu.Lock()\n\t\tdefer mb.mu.Unlock()\n\n\t\tif mb.fss != nil {\n\t\t\tt.Fatalf(\"Expected no fss post recover\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreFSSCloseAndKeepOnExpireOnRecoverBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tttl := 100 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage, MaxAge: ttl}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tfs.Stop()\n\t\ttime.Sleep(2 * ttl)\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); state.NumSubjects != 0 {\n\t\t\tt.Fatalf(\"Expected no subjects with no messages, got %d\", state.NumSubjects)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreExpireOnRecoverSubjectAccounting(t *testing.T) {\n\tconst msgLen = 19\n\tmsg := bytes.Repeat([]byte(\"A\"), msgLen)\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 100\n\t\tttl := 200 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxAge: ttl}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// These are in first block.\n\t\tfs.StoreMsg(\"A\", nil, msg, 0)\n\t\tfs.StoreMsg(\"B\", nil, msg, 0)\n\t\ttime.Sleep(ttl / 2)\n\t\t// This one in 2nd block.\n\t\tfs.StoreMsg(\"C\", nil, msg, 0)\n\n\t\tfs.Stop()\n\t\ttime.Sleep(ttl/2 + 10*time.Millisecond)\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure we take into account PSIM when throwing a whole block away.\n\t\tif state := fs.State(); state.NumSubjects != 1 {\n\t\t\tt.Fatalf(\"Expected 1 subject, got %d\", state.NumSubjects)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreFSSExpireNumPendingBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcexp := 100 * time.Millisecond\n\t\tfcfg.CacheExpire = cexp\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"KV.>\"}, MaxMsgsPer: 1, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Let FSS meta expire.\n\t\ttime.Sleep(2 * cexp)\n\n\t\t_, _, err = fs.StoreMsg(\"KV.X\", nil, []byte(\"Y\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\tfss, err := fs.FilteredState(1, \"KV.X\")\n\t\trequire_NoError(t, err)\n\t\tif fss.Msgs != 1 {\n\t\t\tt.Fatalf(\"Expected only 1 msg, got %d\", fss.Msgs)\n\t\t}\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/3484\nfunc TestFileStoreFilteredFirstMatchingBug(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.>\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"A\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"B\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"C\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\tfs.mu.RLock()\n\t\tmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\n\t\tmb.mu.Lock()\n\t\t// Simulate swapping out the fss state and reading it back in with only one subject\n\t\t// present in the block.\n\t\tif mb.fss != nil {\n\t\t\tmb.fss = nil\n\t\t}\n\t\t// Now load info back in.\n\t\tmb.generatePerSubjectInfo()\n\t\tmb.mu.Unlock()\n\n\t\t// Now add in a different subject.\n\t\t_, _, err = fs.StoreMsg(\"foo.bar\", nil, []byte(\"X\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Now see if a filtered load would incorrectly succeed.\n\t\tsm, _, err := fs.LoadNextMsg(\"foo.foo\", false, 4, nil)\n\t\tif err == nil || sm != nil {\n\t\t\tt.Fatalf(\"Loaded filtered message with wrong subject, wanted %q got %q\", \"foo.foo\", sm.subj)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreOutOfSpaceRebuildState(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, []byte(\"A\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, _, err = fs.StoreMsg(\"bar\", nil, []byte(\"B\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Grab state.\n\t\tstate := fs.State()\n\t\tss := fs.SubjectsState(\">\")\n\n\t\t// Set mock out of space error to trip.\n\t\tfs.mu.RLock()\n\t\tmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\n\t\tmb.mu.Lock()\n\t\tmb.mockWriteErr = true\n\t\tmb.mu.Unlock()\n\n\t\t_, _, err = fs.StoreMsg(\"baz\", nil, []byte(\"C\"), 0)\n\t\trequire_Error(t, err, errors.New(\"mock write error\"))\n\n\t\tnstate := fs.State()\n\t\tnss := fs.SubjectsState(\">\")\n\n\t\tif !reflect.DeepEqual(state, nstate) {\n\t\t\tt.Fatalf(\"State expected to be\\n %+v\\nvs\\n %+v\", state, nstate)\n\t\t}\n\n\t\tif !reflect.DeepEqual(ss, nss) {\n\t\t\tt.Fatalf(\"Subject state expected to be\\n %+v\\nvs\\n %+v\", ss, nss)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRebuildStateProperlyWithMaxMsgsPerSubject(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 4096\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\", \"bar\", \"baz\"}, Storage: FileStorage, MaxMsgsPer: 1}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Send one to baz at beginning.\n\t\t_, _, err = fs.StoreMsg(\"baz\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tns := 1000\n\t\tfor i := 1; i <= ns; i++ {\n\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\t_, _, err = fs.StoreMsg(\"bar\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\tif ss.NumSubjects != 3 {\n\t\t\tt.Fatalf(\"Expected NumSubjects of 3, got %d\", ss.NumSubjects)\n\t\t}\n\t\tif ss.Msgs != 3 {\n\t\t\tt.Fatalf(\"Expected NumMsgs of 3, got %d\", ss.Msgs)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreUpdateMaxMsgsPerSubject(t *testing.T) {\n\tcfg := StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxMsgsPer: 10,\n\t}\n\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure this is honored on an update.\n\t\tcfg.MaxMsgsPer = 50\n\t\terr = fs.UpdateConfig(&cfg)\n\t\trequire_NoError(t, err)\n\n\t\tnumStored := 22\n\t\tfor i := 0; i < numStored; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tss := fs.SubjectsState(\"foo\")[\"foo\"]\n\t\tif ss.Msgs != uint64(numStored) {\n\t\t\tt.Fatalf(\"Expected to have %d stored, got %d\", numStored, ss.Msgs)\n\t\t}\n\n\t\t// Now make sure we trunk if setting to lower value.\n\t\tcfg.MaxMsgsPer = 10\n\t\terr = fs.UpdateConfig(&cfg)\n\t\trequire_NoError(t, err)\n\n\t\tss = fs.SubjectsState(\"foo\")[\"foo\"]\n\t\tif ss.Msgs != 10 {\n\t\t\tt.Fatalf(\"Expected to have %d stored, got %d\", 10, ss.Msgs)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreBadFirstAndFailedExpireAfterRestart(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 256\n\t\tttl := time.Second\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage, MaxAge: ttl}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// With block size of 256 and subject and message below, seq 8 starts new block.\n\t\t// Will double check and fail test if not the case since test depends on this.\n\t\tsubj, msg := \"foo\", []byte(\"ZZ\")\n\t\t// These are all instant and will expire after 1 sec.\n\t\tstart := time.Now()\n\t\tfor i := 0; i < 7; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Put two more after a delay.\n\t\ttime.Sleep(1500 * time.Millisecond)\n\t\tseq, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Make sure that sequence 8 is first in second block, and break test if that is not true.\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.RLock()\n\t\tfirst := lmb.first.seq\n\t\tlmb.mu.RUnlock()\n\t\trequire_True(t, first == 8)\n\n\t\t// Instantly remove first one from second block.\n\t\t// On restart this will trigger expire on recover which will set fs.FirstSeq to the deleted one.\n\t\tfs.RemoveMsg(seq)\n\n\t\t// Stop the filstore and wait til first block expires.\n\t\tfs.Stop()\n\t\ttime.Sleep(ttl - time.Since(start) + (time.Second))\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Check that state is correct for first message which should be 9 and have a proper timestamp.\n\t\tvar state StreamState\n\t\tfs.FastState(&state)\n\t\tts := state.FirstTime\n\t\trequire_True(t, state.Msgs == 1)\n\t\trequire_True(t, state.FirstSeq == 9)\n\t\trequire_True(t, !state.FirstTime.IsZero())\n\n\t\t// Wait and make sure expire timer is still working properly.\n\t\ttime.Sleep(2 * ttl)\n\t\tfs.FastState(&state)\n\t\trequire_Equal(t, state.Msgs, 0)\n\t\trequire_Equal(t, state.FirstSeq, 10)\n\t\trequire_Equal(t, state.LastSeq, 9)\n\t\trequire_Equal(t, state.LastTime, ts)\n\t})\n}\n\nfunc TestFileStoreCompactAllWithDanglingLMB(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"ZZ\")\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.RemoveMsg(100)\n\t\tpurged, err := fs.Compact(100)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, purged == 99)\n\n\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t})\n}\n\nfunc TestFileStoreStateWithBlkFirstDeleted(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 4096\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 500\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Delete some messages from the beginning of an interior block.\n\t\tfs.mu.RLock()\n\t\trequire_True(t, len(fs.blks) > 2)\n\t\tfseq := fs.blks[1].first.seq\n\t\tfs.mu.RUnlock()\n\n\t\t// Now start from first seq of second blk and delete 10 msgs\n\t\tfor seq := fseq; seq < fseq+10; seq++ {\n\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\t\t}\n\n\t\t// This bug was in normal detailed state. But check fast state too.\n\t\tvar fstate StreamState\n\t\tfs.FastState(&fstate)\n\t\trequire_True(t, fstate.NumDeleted == 10)\n\t\tstate := fs.State()\n\t\trequire_True(t, state.NumDeleted == 10)\n\t})\n}\n\nfunc TestFileStoreMsgBlkFailOnKernelFaultLostDataReporting(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 4096\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 500\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// We want to make sure all of the scenarios report lost data properly.\n\t\t// Will run 3 scenarios, 1st block, last block, interior block.\n\t\t// The new system does not detect byzantine behavior by default on creating the store.\n\t\t// A LoadMsg() of checkMsgs() call will be needed now.\n\n\t\t// First block\n\t\tfs.mu.RLock()\n\t\trequire_True(t, len(fs.blks) > 0)\n\t\tmfn := fs.blks[0].mfn\n\t\tfs.mu.RUnlock()\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(mfn))\n\n\t\t// Restart.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, err = fs.LoadMsg(1, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t// Load will rebuild fs itself async..\n\t\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\t\tif state := fs.State(); state.Lost != nil {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn errors.New(\"no ld yet\")\n\t\t})\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.FirstSeq, 94)\n\t\trequire_True(t, state.Lost != nil)\n\t\trequire_Len(t, len(state.Lost.Msgs), 93)\n\n\t\t// Last block\n\t\tfs.mu.RLock()\n\t\trequire_True(t, len(fs.blks) > 0)\n\t\trequire_True(t, fs.lmb != nil)\n\t\tmfn = fs.lmb.mfn\n\t\tfs.mu.RUnlock()\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(mfn))\n\n\t\t// Restart.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.FirstSeq, 94)\n\t\trequire_Equal(t, state.LastSeq, 500) // Make sure we do not lose last seq.\n\t\trequire_Equal(t, state.NumDeleted, 35) // These are interiors\n\t\trequire_True(t, state.Lost != nil)\n\t\trequire_Len(t, len(state.Lost.Msgs), 35)\n\n\t\t// Interior block.\n\t\tfs.mu.RLock()\n\t\trequire_True(t, len(fs.blks) > 3)\n\t\tmfn = fs.blks[len(fs.blks)-3].mfn\n\t\tfs.mu.RUnlock()\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(mfn))\n\n\t\t// Restart.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Need checkMsgs to catch interior one.\n\t\trequire_True(t, fs.checkMsgs() != nil)\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.FirstSeq, 94)\n\t\trequire_Equal(t, state.LastSeq, 500) // Make sure we do not lose last seq.\n\t\trequire_Equal(t, state.NumDeleted, 128)\n\t\trequire_True(t, state.Lost != nil)\n\t\trequire_Len(t, len(state.Lost.Msgs), 93)\n\t})\n}\n\nfunc TestFileStoreAllFilteredStateWithDeleted(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1024\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tremove := func(seqs ...uint64) {\n\t\t\tfor _, seq := range seqs {\n\t\t\t\tok, err := fs.RemoveMsg(seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, ok)\n\t\t\t}\n\t\t}\n\n\t\tcheckFilteredState := func(start, msgs, first, last int) {\n\t\t\tfss, err := fs.FilteredState(uint64(start), _EMPTY_)\n\t\t\trequire_NoError(t, err)\n\t\t\tif fss.Msgs != uint64(msgs) {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", msgs, fss.Msgs)\n\t\t\t}\n\t\t\tif fss.First != uint64(first) {\n\t\t\t\tt.Fatalf(\"Expected %d to be first, got %d\", first, fss.First)\n\t\t\t}\n\t\t\tif fss.Last != uint64(last) {\n\t\t\t\tt.Fatalf(\"Expected %d to be last, got %d\", last, fss.Last)\n\t\t\t}\n\t\t}\n\n\t\tcheckFilteredState(1, 100, 1, 100)\n\t\tremove(2)\n\t\tcheckFilteredState(2, 98, 3, 100)\n\t\tremove(3, 4, 5)\n\t\tcheckFilteredState(2, 95, 6, 100)\n\t\tcheckFilteredState(6, 95, 6, 100)\n\t\tremove(8, 10, 12, 14, 16, 18)\n\t\tcheckFilteredState(7, 88, 7, 100)\n\n\t\t// Now check when purged that we return first and last sequences properly.\n\t\tfs.Purge()\n\t\tcheckFilteredState(0, 0, 101, 100)\n\t})\n}\n\nfunc TestFileStoreStreamTruncateResetMultiBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 128\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfs.syncBlocks()\n\t\trequire_True(t, fs.numMsgBlocks() == 500)\n\n\t\t// Reset everything\n\t\trequire_NoError(t, fs.Truncate(0))\n\t\trequire_True(t, fs.numMsgBlocks() == 0)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 0)\n\t\trequire_Equal(t, state.Bytes, 0)\n\t\trequire_Equal(t, state.FirstSeq, 0)\n\t\trequire_Equal(t, state.LastSeq, 0)\n\t\trequire_Equal(t, state.NumSubjects, 0)\n\t\trequire_Equal(t, state.NumDeleted, 0)\n\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfs.syncBlocks()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 1000)\n\t\trequire_Equal(t, state.Bytes, 44000)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 1000)\n\t\trequire_Equal(t, state.NumSubjects, 1)\n\t\trequire_Equal(t, state.NumDeleted, 0)\n\t})\n}\n\nfunc TestFileStoreStreamCompactMultiBlockSubjectInfo(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 128\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_True(t, fs.numMsgBlocks() == 500)\n\n\t\t// Compact such that we know we throw blocks away from the beginning.\n\t\tdeleted, err := fs.Compact(501)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, deleted == 500)\n\t\trequire_True(t, fs.numMsgBlocks() == 250)\n\n\t\t// Make sure we adjusted for subjects etc.\n\t\tstate := fs.State()\n\t\trequire_True(t, state.NumSubjects == 500)\n\t})\n}\n\nfunc TestFileStoreSubjectsTotals(t *testing.T) {\n\t// No need for all permutations here.\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{StoreDir: storeDir}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfmap := make(map[int]int)\n\tbmap := make(map[int]int)\n\n\tvar m map[int]int\n\tvar ft string\n\n\tfor i := 0; i < 10_000; i++ {\n\t\t// Flip coin for prefix\n\t\tif rand.Intn(2) == 0 {\n\t\t\tft, m = \"foo\", fmap\n\t\t} else {\n\t\t\tft, m = \"bar\", bmap\n\t\t}\n\t\tdt := rand.Intn(100)\n\t\tsubj := fmt.Sprintf(\"%s.%d\", ft, dt)\n\t\tm[dt]++\n\n\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now test SubjectsTotal\n\tfor dt, total := range fmap {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", dt)\n\t\tm := fs.SubjectsTotals(subj)\n\t\tif m[subj] != uint64(total) {\n\t\t\tt.Fatalf(\"Expected %q to have %d total, got %d\", subj, total, m[subj])\n\t\t}\n\t}\n\n\t// Check fmap.\n\tif st := fs.SubjectsTotals(\"foo.*\"); len(st) != len(fmap) {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", len(fmap), \"foo.*\", len(st))\n\t} else {\n\t\texpected := 0\n\t\tfor _, n := range fmap {\n\t\t\texpected += n\n\t\t}\n\t\treceived := uint64(0)\n\t\tfor _, n := range st {\n\t\t\treceived += n\n\t\t}\n\t\tif received != uint64(expected) {\n\t\t\tt.Fatalf(\"Expected %d total but got %d\", expected, received)\n\t\t}\n\t}\n\n\t// Check bmap.\n\tif st := fs.SubjectsTotals(\"bar.*\"); len(st) != len(bmap) {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", len(bmap), \"bar.*\", len(st))\n\t} else {\n\t\texpected := 0\n\t\tfor _, n := range bmap {\n\t\t\texpected += n\n\t\t}\n\t\treceived := uint64(0)\n\t\tfor _, n := range st {\n\t\t\treceived += n\n\t\t}\n\t\tif received != uint64(expected) {\n\t\t\tt.Fatalf(\"Expected %d total but got %d\", expected, received)\n\t\t}\n\t}\n\n\t// All with pwc match.\n\tif st, expected := fs.SubjectsTotals(\"*.*\"), len(bmap)+len(fmap); len(st) != expected {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", expected, \"*.*\", len(st))\n\t}\n}\n\nfunc TestFileStoreConsumerStoreEncodeAfterRestart(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tstate := &ConsumerState{}\n\n\t\tfunc() { // for defers\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.Stop()\n\n\t\t\tstate.Delivered.Consumer = 22\n\t\t\tstate.Delivered.Stream = 22\n\t\t\tstate.AckFloor.Consumer = 11\n\t\t\tstate.AckFloor.Stream = 11\n\t\t\terr = o.Update(state)\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\n\t\tfunc() { // for defers\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\to, err := fs.ConsumerStore(\"o22\", time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.Stop()\n\n\t\t\tif o.(*consumerFileStore).state.Delivered != state.Delivered {\n\t\t\t\tt.Fatalf(\"Consumer state is wrong %+v vs %+v\", o.(*consumerFileStore).state, state)\n\t\t\t}\n\t\t\tif o.(*consumerFileStore).state.AckFloor != state.AckFloor {\n\t\t\t\tt.Fatalf(\"Consumer state is wrong %+v vs %+v\", o.(*consumerFileStore).state, state)\n\t\t\t}\n\t\t}()\n\t})\n}\n\nfunc TestFileStoreNumPendingLargeNumBlks(t *testing.T) {\n\t// No need for all permutations here.\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{\n\t\tStoreDir: storeDir,\n\t\tBlockSize: 128, // Small on purpose to create alot of blks.\n\t}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"zzz\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"zzz\", bytes.Repeat([]byte(\"X\"), 100)\n\tnumMsgs := 10_000\n\n\tfor i := 0; i < numMsgs; i++ {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\tstart := time.Now()\n\ttotal, _, err := fs.NumPending(4000, \"zzz\", false)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(start), 15*time.Millisecond)\n\trequire_Equal(t, total, 6001)\n\n\tstart = time.Now()\n\ttotal, _, err = fs.NumPending(6000, \"zzz\", false)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(start), 25*time.Millisecond)\n\trequire_Equal(t, total, 4001)\n\n\t// Now delete a message in first half and second half.\n\tfs.RemoveMsg(1000)\n\tfs.RemoveMsg(9000)\n\n\tstart = time.Now()\n\ttotal, _, err = fs.NumPending(4000, \"zzz\", false)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(start), 50*time.Millisecond)\n\trequire_Equal(t, total, 6000)\n\n\tstart = time.Now()\n\ttotal, _, err = fs.NumPending(6000, \"zzz\", false)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(start), 50*time.Millisecond)\n\trequire_Equal(t, total, 4000)\n}\n\nfunc TestFileStoreSkipMsgAndNumBlocks(t *testing.T) {\n\t// No need for all permutations here.\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{\n\t\tStoreDir: storeDir,\n\t\tBlockSize: 128, // Small on purpose to create alot of blks.\n\t}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"zzz\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"zzz\", bytes.Repeat([]byte(\"X\"), 100)\n\n\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\trequire_NoError(t, err)\n\trequire_NoError(t, fs.SkipMsgs(0, 10_000))\n\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, fs.numMsgBlocks(), 3)\n}\n\nfunc TestFileStoreRestoreEncryptedWithNoKeyFuncFails(t *testing.T) {\n\t// No need for all permutations here.\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir(), Cipher: AES}\n\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"zzz\"}, Storage: FileStorage}\n\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"zzz\", bytes.Repeat([]byte(\"X\"), 100)\n\tnumMsgs := 100\n\tfor i := 0; i < numMsgs; i++ {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\tfs.Stop()\n\n\t// Make sure if we try to restore with no prf (key) that it fails.\n\t_, err = newFileStoreWithCreated(fcfg, cfg, time.Now(), nil, nil)\n\trequire_Error(t, err, errNoMainKey)\n}\n\nfunc TestFileStoreInitialFirstSeq(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, FirstSeq: 1000}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tseq, _, err := fs.StoreMsg(\"A\", nil, []byte(\"OK\"), 0)\n\t\trequire_NoError(t, err)\n\t\tif seq != 1000 {\n\t\t\tt.Fatalf(\"Message should have been sequence 1000 but was %d\", seq)\n\t\t}\n\n\t\tseq, _, err = fs.StoreMsg(\"B\", nil, []byte(\"OK\"), 0)\n\t\trequire_NoError(t, err)\n\t\tif seq != 1001 {\n\t\t\tt.Fatalf(\"Message should have been sequence 1001 but was %d\", seq)\n\t\t}\n\n\t\tvar state StreamState\n\t\tfs.FastState(&state)\n\t\tswitch {\n\t\tcase state.Msgs != 2:\n\t\t\tt.Fatalf(\"Expected 2 messages, got %d\", state.Msgs)\n\t\tcase state.FirstSeq != 1000:\n\t\t\tt.Fatalf(\"Expected first seq 1000, got %d\", state.FirstSeq)\n\t\tcase state.LastSeq != 1001:\n\t\t\tt.Fatalf(\"Expected last seq 1001, got %d\", state.LastSeq)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRecaluclateFirstForSubjBug(t *testing.T) {\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"foo\", nil, nil, 0) // 1\n\tfs.StoreMsg(\"bar\", nil, nil, 0) // 2\n\tfs.StoreMsg(\"foo\", nil, nil, 0) // 3\n\n\t// Now remove first 2..\n\tfs.RemoveMsg(1)\n\tfs.RemoveMsg(2)\n\n\t// Now grab first (and only) block.\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\n\t// Since we lazy update the first, simulate that we have not updated it as of yet.\n\tss := &SimpleState{Msgs: 1, First: 1, Last: 3, firstNeedsUpdate: true}\n\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\n\t// Flush the cache.\n\tmb.clearCacheAndOffset()\n\t// Now call with start sequence of 1, the old one\n\t// This will panic without the fix.\n\tmb.recalculateForSubj(\"foo\", ss)\n\t// Make sure it was update properly.\n\trequire_True(t, *ss == SimpleState{Msgs: 1, First: 3, Last: 3, firstNeedsUpdate: false})\n}\n\nfunc TestFileStoreKeepWithDeletedMsgsBug(t *testing.T) {\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := bytes.Repeat([]byte(\"A\"), 19)\n\tfor i := 0; i < 5; i++ {\n\t\tfs.StoreMsg(\"A\", nil, msg, 0)\n\t\tfs.StoreMsg(\"B\", nil, msg, 0)\n\t}\n\n\tn, err := fs.PurgeEx(\"A\", 0, 0)\n\trequire_NoError(t, err)\n\trequire_True(t, n == 5)\n\n\t// Purge with keep.\n\tn, err = fs.PurgeEx(_EMPTY_, 0, 2)\n\trequire_NoError(t, err)\n\trequire_True(t, n == 3)\n}\n\nfunc TestFileStoreRestartWithExpireAndLockingBug(t *testing.T) {\n\tsd := t.TempDir()\n\tscfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: sd}, scfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// 20 total\n\tmsg := []byte(\"HELLO WORLD\")\n\tfor i := 0; i < 10; i++ {\n\t\tfs.StoreMsg(\"A\", nil, msg, 0)\n\t\tfs.StoreMsg(\"B\", nil, msg, 0)\n\t}\n\tfs.Stop()\n\n\t// Now change config underneath of so we will do expires at startup.\n\tscfg.MaxMsgs = 15\n\tscfg.MaxMsgsPer = 2\n\tnewCfg := FileStreamInfo{Created: fs.cfg.Created, StreamConfig: scfg}\n\n\t// Replace\n\tfs.cfg = newCfg\n\trequire_NoError(t, fs.writeStreamMeta())\n\n\tfs, err = newFileStore(FileStoreConfig{StoreDir: sd}, scfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n}\n\n// Test that loads from lmb under lots of writes do not return errPartialCache.\nfunc TestFileStoreErrPartialLoad(t *testing.T) {\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tput := func(num int) {\n\t\tfor i := 0; i < num; i++ {\n\t\t\tfs.StoreMsg(\"Z\", nil, []byte(\"ZZZZZZZZZZZZZ\"), 0)\n\t\t}\n\t}\n\n\tput(100)\n\n\t// Dump cache of lmb.\n\tclearCache := func() {\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.Lock()\n\t\tlmb.clearCache()\n\t\tlmb.mu.Unlock()\n\t}\n\tclearCache()\n\n\tqch := make(chan struct{})\n\tdefer close(qch)\n\n\tfor i := 0; i < 10; i++ {\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-qch:\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t\tput(5)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tvar smv StoreMsg\n\tfor i := 0; i < 10_000; i++ {\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.Lock()\n\t\tfirst, last := fs.lmb.first.seq, fs.lmb.last.seq\n\t\tif i%100 == 0 {\n\t\t\tlmb.clearCache()\n\t\t}\n\t\tlmb.mu.Unlock()\n\n\t\tif spread := int(last - first); spread > 0 {\n\t\t\tseq := first + uint64(rand.Intn(spread))\n\t\t\t_, err = fs.LoadMsg(seq, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n}\n\nfunc TestFileStoreErrPartialLoadOnSyncClose(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 500},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// This yields an internal record length of 50 bytes. So 10 msgs per blk.\n\tmsgLen := 19\n\tmsg := bytes.Repeat([]byte(\"A\"), msgLen)\n\n\t// Load up half the block.\n\tfor _, subj := range []string{\"A\", \"B\", \"C\", \"D\", \"E\"} {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\t// Now simulate the sync timer closing the last block.\n\tfs.mu.RLock()\n\tlmb := fs.lmb\n\tfs.mu.RUnlock()\n\trequire_True(t, lmb != nil)\n\n\tlmb.mu.Lock()\n\tlmb.expireCacheLocked()\n\tlmb.dirtyCloseWithRemove(false)\n\tlmb.mu.Unlock()\n\n\tfs.StoreMsg(\"Z\", nil, msg, 0)\n\t_, err = fs.LoadMsg(1, nil)\n\trequire_NoError(t, err)\n}\n\nfunc TestFileStoreSyncIntervals(t *testing.T) {\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir(), SyncInterval: 250 * time.Millisecond}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckSyncFlag := func(expected bool) {\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.RLock()\n\t\tsyncNeeded := lmb.needSync\n\t\tlmb.mu.RUnlock()\n\t\tif syncNeeded != expected {\n\t\t\tt.Fatalf(\"Expected needSync to be %v\", expected)\n\t\t}\n\t}\n\n\tcheckSyncFlag(false)\n\tfs.StoreMsg(\"Z\", nil, []byte(\"hello\"), 0)\n\tcheckSyncFlag(true)\n\ttime.Sleep(400 * time.Millisecond)\n\tcheckSyncFlag(false)\n\tfs.Stop()\n\n\t// Now check always\n\tfcfg.SyncInterval = 10 * time.Second\n\tfcfg.SyncAlways = true\n\tfs, err = newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckSyncFlag(false)\n\tfs.StoreMsg(\"Z\", nil, []byte(\"hello\"), 0)\n\tcheckSyncFlag(false)\n}\n\n// https://github.com/nats-io/nats-server/issues/4529\n// Run this wuth --race and you will see the unlocked access that probably caused this.\nfunc TestFileStoreRecalcFirstSequenceBug(t *testing.T) {\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir()}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, MaxMsgsPer: 2, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := bytes.Repeat([]byte(\"A\"), 22)\n\n\tfor _, subj := range []string{\"A\", \"A\", \"B\", \"B\"} {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Make sure the buffer is cleared.\n\tclearLMBCache := func() {\n\t\tfs.mu.RLock()\n\t\tmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tmb.mu.Lock()\n\t\tmb.clearCacheAndOffset()\n\t\tmb.mu.Unlock()\n\t}\n\n\tclearLMBCache()\n\n\t// Do first here.\n\tfs.StoreMsg(\"A\", nil, msg, 0)\n\n\tvar wg sync.WaitGroup\n\tstart := make(chan bool)\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\t<-start\n\t\tfor i := 0; i < 1_000; i++ {\n\t\t\tfs.LoadLastMsg(\"A\", nil)\n\t\t\tclearLMBCache()\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\t<-start\n\t\tfor i := 0; i < 1_000; i++ {\n\t\t\tfs.StoreMsg(\"A\", nil, msg, 0)\n\t\t}\n\t}()\n\n\tclose(start)\n\twg.Wait()\n}\n\n///////////////////////////////////////////////////////////////////////////\n// New WAL based architecture tests\n///////////////////////////////////////////////////////////////////////////\n\nfunc TestFileStoreFullStateBasics(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 100\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tsubj, msgLen, recLen := \"A\", 19, uint64(50)\n\t\tmsgA := bytes.Repeat([]byte(\"A\"), msgLen)\n\t\tmsgZ := bytes.Repeat([]byte(\"Z\"), msgLen)\n\n\t\t// Send 2 msgs and stop, check for presence of our full state file.\n\t\t_, _, err = fs.StoreMsg(subj, nil, msgA, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(subj, nil, msgZ, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\t\t// Make sure there is a full state file after we do a stop.\n\t\trequire_NoError(t, fs.Stop())\n\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tif _, err := os.Stat(sfile); err != nil {\n\t\t\tt.Fatalf(\"Expected stream state file but got %v\", err)\n\t\t}\n\n\t\t// Read it in and make sure len > 0.\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(buf) > 0)\n\n\t\t// Now make sure we recover properly.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Make sure there are no old idx or fss files.\n\t\tmatches, err := filepath.Glob(filepath.Join(fcfg.StoreDir, msgDir, \"%d.fss\"))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(matches), 0)\n\t\tmatches, err = filepath.Glob(filepath.Join(fcfg.StoreDir, msgDir, \"%d.idx\"))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(matches), 0)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 2)\n\n\t\t// Now make sure we can read in values.\n\t\tvar smv StoreMsg\n\t\tsm, err := fs.LoadMsg(1, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, bytes.Equal(sm.msg, msgA))\n\n\t\tsm, err = fs.LoadMsg(2, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, bytes.Equal(sm.msg, msgZ))\n\n\t\t// Now add in 1 more here to split the lmb.\n\t\t_, _, err = fs.StoreMsg(subj, nil, msgZ, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Now stop the filestore and replace the old stream state and make sure we recover correctly.\n\t\trequire_NoError(t, fs.Stop())\n\n\t\t// Regrab the stream state\n\t\tbuf, err = os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Add in one more.\n\t\t_, _, err = fs.StoreMsg(subj, nil, msgZ, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, fs.Stop())\n\n\t\t// Put old stream state back with only 3.\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 4)\n\t\trequire_Equal(t, state.Bytes, 4*recLen)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 4)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\n\t\t// Make sure we are tracking subjects correctly.\n\t\tfs.mu.RLock()\n\t\tinfo, _ := fs.psim.Find(stringToBytes(subj))\n\t\tpsi := *info\n\t\tfs.mu.RUnlock()\n\n\t\trequire_Equal(t, psi.total, 4)\n\t\trequire_Equal(t, psi.fblk, 1)\n\t\trequire_Equal(t, psi.lblk, 2)\n\n\t\t// Store 1 more\n\t\t_, _, err = fs.StoreMsg(subj, nil, msgA, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, fs.Stop())\n\t\t// Put old stream state back with only 3.\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 5)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 5)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 3)\n\t\t// Make sure we are tracking subjects correctly.\n\t\tfs.mu.RLock()\n\t\tinfo, _ = fs.psim.Find(stringToBytes(subj))\n\t\tpsi = *info\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, psi.total, 5)\n\t\trequire_Equal(t, psi.fblk, 1)\n\t\trequire_Equal(t, psi.lblk, 3)\n\t})\n}\n\nfunc TestFileStoreFullStatePurge(t *testing.T) {\n\ttestFileStoreFullStatePurge(t, false)\n}\n\nfunc TestFileStoreFullStatePurgeFullRecovery(t *testing.T) {\n\ttestFileStoreFullStatePurge(t, true)\n}\n\nfunc testFileStoreFullStatePurge(t *testing.T, checkFullRecovery bool) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 132 // Leave room for tombstones.\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tsubj, msg := \"A\", bytes.Repeat([]byte(\"A\"), 19)\n\n\t\t// Should be 2 per block, so 5 blocks.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, _, err = fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tn, err := fs.Purge()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, n, 10)\n\t\tstate := fs.State()\n\t\trequire_NoError(t, fs.Stop())\n\t\tif checkFullRecovery {\n\t\t\trequire_NoError(t, os.Remove(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\t\t}\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\tif checkFullRecovery {\n\t\t\tfs.rebuildState(nil)\n\t\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\t\tstate, newState)\n\t\t\t}\n\t\t}\n\n\t\t// Add in more 10 more total, some B some C.\n\t\tfor i := 0; i < 5; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"B\", nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\t_, _, err = fs.StoreMsg(\"C\", nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tn, err = fs.PurgeEx(\"B\", 0, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, n, 5)\n\n\t\tstate = fs.State()\n\t\trequire_NoError(t, fs.Stop())\n\t\tif checkFullRecovery {\n\t\t\trequire_NoError(t, os.Remove(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\t\t}\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\tif checkFullRecovery {\n\t\t\tfs.rebuildState(nil)\n\t\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\t\tstate, newState)\n\t\t\t}\n\t\t}\n\n\t\t// Purge with keep.\n\t\tn, err = fs.PurgeEx(_EMPTY_, 0, 2)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, n, 3)\n\n\t\tstate = fs.State()\n\n\t\t// Do some quick checks here, keep had a bug.\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 18)\n\t\trequire_Equal(t, state.LastSeq, 20)\n\n\t\trequire_NoError(t, fs.Stop())\n\t\tif checkFullRecovery {\n\t\t\trequire_NoError(t, os.Remove(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\t\t}\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\tif checkFullRecovery {\n\t\t\tfs.rebuildState(nil)\n\t\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\t\tstate, newState)\n\t\t\t}\n\t\t}\n\n\t\t// Make sure we can survive a purge with no full stream state and have the correct first sequence.\n\t\t// This used to be provided by the idx file and is now tombstones and the full stream state snapshot.\n\t\tn, err = fs.Purge()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, n, 2)\n\t\tstate = fs.State()\n\t\trequire_NoError(t, fs.Stop())\n\t\tif checkFullRecovery {\n\t\t\trequire_NoError(t, os.Remove(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\t\t}\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\tif checkFullRecovery {\n\t\t\tfs.rebuildState(nil)\n\t\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\t\tstate, newState)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreFullStateTestUserRemoveWAL(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 132 // Leave room for tombstones.\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tmsgLen := 19\n\t\tmsgA := bytes.Repeat([]byte(\"A\"), msgLen)\n\t\tmsgZ := bytes.Repeat([]byte(\"Z\"), msgLen)\n\n\t\t// Store 2 msgs and delete first.\n\t\tfs.StoreMsg(\"A\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"Z\", nil, msgZ, 0)\n\t\tfs.RemoveMsg(1)\n\n\t\t// Check we can load things properly since the block will have a tombstone now for seq 1.\n\t\tsm, err := fs.LoadMsg(2, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, bytes.Equal(sm.msg, msgZ))\n\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tstate := fs.State()\n\t\tfs.Stop()\n\n\t\t// Grab the state from this stop.\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Check we can load things properly since the block will have a tombstone now for seq 1.\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.LoadMsg(1, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t\trequire_True(t, !state.FirstTime.IsZero())\n\n\t\t// Store 2 more msgs and delete 2 & 4.\n\t\tfs.StoreMsg(\"A\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"Z\", nil, msgZ, 0)\n\t\tfs.RemoveMsg(2)\n\t\tfs.RemoveMsg(4)\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, len(state.Deleted), state.NumDeleted)\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t\trequire_True(t, !state.FirstTime.IsZero())\n\n\t\t// Now close again and put back old stream state.\n\t\t// This will test that we can remember user deletes by placing tombstones in the lmb/wal.\n\t\tfs.Stop()\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tnewState := fs.State()\n\t\t// We will properly detect lost data for sequence #2 here.\n\t\trequire_True(t, newState.Lost != nil)\n\t\trequire_Equal(t, len(newState.Lost.Msgs), 1)\n\t\trequire_Equal(t, newState.Lost.Msgs[0], 2)\n\t\t// Clear for deep equal compare below.\n\t\tnewState.Lost = nil\n\n\t\tif !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t\trequire_True(t, !state.FirstTime.IsZero())\n\t})\n}\n\nfunc TestFileStoreFullStateTestSysRemovals(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 100\n\t\tcfg := StreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tSubjects: []string{\"*\"},\n\t\t\tMaxMsgs: 10,\n\t\t\tMaxMsgsPer: 1,\n\t\t\tStorage: FileStorage,\n\t\t}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tmsgLen := 19\n\t\tmsg := bytes.Repeat([]byte(\"A\"), msgLen)\n\n\t\tfor _, subj := range []string{\"A\", \"B\", \"A\", \"B\"} {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 3)\n\t\trequire_Equal(t, state.LastSeq, 4)\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\tfor _, subj := range []string{\"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"} {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 10)\n\t\trequire_Equal(t, state.FirstSeq, 3)\n\t\trequire_Equal(t, state.LastSeq, 12)\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\n\t\t// Goes over limit\n\t\tfs.StoreMsg(\"ZZZ\", nil, msg, 0)\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 10)\n\t\trequire_Equal(t, state.FirstSeq, 4)\n\t\trequire_Equal(t, state.LastSeq, 13)\n\t\tfs.Stop()\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state after purge does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreSelectBlockWithFirstSeqRemovals(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 100\n\t\tcfg := StreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tSubjects: []string{\"*\"},\n\t\t\tMaxMsgsPer: 1,\n\t\t\tStorage: FileStorage,\n\t\t}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tmsgLen := 19\n\t\tmsg := bytes.Repeat([]byte(\"A\"), msgLen)\n\n\t\tsubjects := \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+@$^\"\n\t\t// We need over 32 blocks to kick in binary search. So 32*2+1 (65) msgs to get 33 blocks.\n\t\tfor i := 0; i < 32*2+1; i++ {\n\t\t\tsubj := string(subjects[i])\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\trequire_Equal(t, fs.numMsgBlocks(), 33)\n\n\t\t// Now we want to delete the first msg of each block to move the first sequence.\n\t\t// Want to do this via system removes, not user initiated moves.\n\t\tfor i := 0; i < len(subjects); i += 2 {\n\t\t\tsubj := string(subjects[i])\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\n\t\t// We want to make sure that select always returns an index and a non-nil mb.\n\t\tfor seq := ss.FirstSeq; seq <= ss.LastSeq; seq++ {\n\t\t\tfs.mu.RLock()\n\t\t\tindex, mb := fs.selectMsgBlockWithIndex(seq)\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_True(t, index >= 0)\n\t\t\trequire_True(t, mb != nil)\n\t\t\trequire_Equal(t, (seq-1)/2, uint64(index))\n\t\t}\n\t})\n}\n\nfunc TestFileStoreMsgBlockHolesAndIndexing(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Grab the message block by hand and manipulate at that level.\n\tmb := fs.getFirstBlock()\n\twriteMsg := func(subj string, seq uint64) {\n\t\trl := fileStoreMsgSize(subj, nil, []byte(subj))\n\t\trequire_NoError(t, mb.writeMsgRecord(rl, seq, subj, nil, []byte(subj), time.Now().UnixNano(), true))\n\t\tfs.rebuildState(nil)\n\t}\n\treadMsg := func(seq uint64, expectedSubj string) {\n\t\t// Clear cache so we load back in from disk and need to properly process any holes.\n\t\tld, tombs, err := mb.rebuildState()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ld, nil)\n\t\trequire_Equal(t, len(tombs), 0)\n\t\tfs.rebuildState(nil)\n\t\tsm, _, err := mb.fetchMsg(seq, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, expectedSubj)\n\t\trequire_True(t, bytes.Equal(sm.buf[:len(expectedSubj)], []byte(expectedSubj)))\n\t}\n\n\twriteMsg(\"A\", 2)\n\trequire_Equal(t, mb.first.seq, 2)\n\trequire_Equal(t, mb.last.seq, 2)\n\n\twriteMsg(\"B\", 4)\n\trequire_Equal(t, mb.first.seq, 2)\n\trequire_Equal(t, mb.last.seq, 4)\n\n\twriteMsg(\"C\", 12)\n\n\treadMsg(4, \"B\")\n\trequire_True(t, mb.dmap.Exists(3))\n\n\treadMsg(12, \"C\")\n\treadMsg(2, \"A\")\n\n\t// Check that we get deleted for the right ones etc.\n\tcheckDeleted := func(seq uint64) {\n\t\t_, _, err = mb.fetchMsg(seq, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound, errDeletedMsg)\n\t\tmb.mu.RLock()\n\t\tshouldExist, exists := seq >= mb.first.seq, mb.dmap.Exists(seq)\n\t\tmb.mu.RUnlock()\n\t\tif shouldExist {\n\t\t\trequire_True(t, exists)\n\t\t}\n\t}\n\tcheckDeleted(1)\n\tcheckDeleted(3)\n\tfor seq := 5; seq < 12; seq++ {\n\t\tcheckDeleted(uint64(seq))\n\t}\n}\n\nfunc TestFileStoreMsgBlockCompactionAndHoles(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor _, subj := range []string{\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"} {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Leave first one but delete the rest.\n\tfor seq := uint64(2); seq < 10; seq++ {\n\t\tfs.RemoveMsg(seq)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\tmb := fs.getFirstBlock()\n\trequire_NotNil(t, mb)\n\n\t_, ub, _ := fs.Utilization()\n\n\t// Do compaction, should remove all excess now.\n\tmb.mu.Lock()\n\terr = mb.compact()\n\tmb.mu.Unlock()\n\trequire_NoError(t, err)\n\n\tta, ua, _ := fs.Utilization()\n\trequire_Equal(t, ub, ua)\n\trequire_Equal(t, ta, ua)\n}\n\nfunc TestFileStoreRemoveLastNoDoubleTombstones(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"A\", nil, []byte(\"hello\"), 0)\n\tfs.mu.Lock()\n\tfs.removeMsgViaLimits(1)\n\tfs.mu.Unlock()\n\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\tmb := fs.getFirstBlock()\n\trequire_NotNil(t, mb)\n\tmb.loadMsgs()\n\trbytes, _, err := fs.Utilization()\n\trequire_NoError(t, err)\n\trequire_Equal(t, rbytes, emptyRecordLen)\n}\n\nfunc TestFileStoreFullStateMultiBlockPastWAL(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 100\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tmsgLen := 19\n\t\tmsgA := bytes.Repeat([]byte(\"A\"), msgLen)\n\t\tmsgZ := bytes.Repeat([]byte(\"Z\"), msgLen)\n\n\t\t// Store 2 msgs\n\t\tfs.StoreMsg(\"A\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"B\", nil, msgZ, 0)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tfs.Stop()\n\n\t\t// Grab the state from this stop.\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store 6 more msgs.\n\t\tfs.StoreMsg(\"C\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"D\", nil, msgZ, 0)\n\t\tfs.StoreMsg(\"E\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"F\", nil, msgZ, 0)\n\t\tfs.StoreMsg(\"G\", nil, msgA, 0)\n\t\tfs.StoreMsg(\"H\", nil, msgZ, 0)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 4)\n\t\tstate := fs.State()\n\t\tfs.Stop()\n\n\t\t// Put back old stream state.\n\t\t// This will test that we properly walk multiple blocks past where we snapshotted state.\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t\trequire_True(t, !state.FirstTime.IsZero())\n\t})\n}\n\n// This tests we can successfully recover without having to rebuild the whole stream from a mid block index.db marker\n// when the updated block has a removed entry.\n// Make sure this does not cause a recover of the full state.\nfunc TestFileStoreFullStateMidBlockPastWAL(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxMsgsPer: 1}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 2 msgs per blk.\n\t\tmsg := bytes.Repeat([]byte(\"Z\"), 19)\n\n\t\t// Store 5 msgs\n\t\tfs.StoreMsg(\"A\", nil, msg, 0)\n\t\tfs.StoreMsg(\"B\", nil, msg, 0)\n\t\tfs.StoreMsg(\"C\", nil, msg, 0)\n\t\tfs.StoreMsg(\"D\", nil, msg, 0)\n\t\tfs.StoreMsg(\"E\", nil, msg, 0)\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tfs.Stop()\n\n\t\t// Grab the state from this stop.\n\t\tsfile := filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tbuf, err := os.ReadFile(sfile)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store 5 more messages, then remove seq 2, \"B\".\n\t\tfs.StoreMsg(\"F\", nil, msg, 0)\n\t\tfs.StoreMsg(\"G\", nil, msg, 0)\n\t\tfs.StoreMsg(\"H\", nil, msg, 0)\n\t\tfs.StoreMsg(\"I\", nil, msg, 0)\n\t\tfs.StoreMsg(\"J\", nil, msg, 0)\n\t\tfs.RemoveMsg(2)\n\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tstate := fs.State()\n\t\tfs.Stop()\n\n\t\t// Put back old stream state.\n\t\t// This will test that we properly walk multiple blocks past where we snapshotted state.\n\t\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\t\trequire_NoError(t, err)\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif newState := fs.State(); !reflect.DeepEqual(state, newState) {\n\t\t\tt.Fatalf(\"Restore state does not match:\\n%+v\\n%+v\",\n\t\t\t\tstate, newState)\n\t\t}\n\t\t// Check that index.db is still there. If we recover by raw data on a corrupt state we delete this.\n\t\t_, err = os.Stat(sfile)\n\t\trequire_NoError(t, err)\n\t})\n}\n\nfunc TestFileStoreCompactingBlocksOnSync(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 1000 // 20 msgs per block.\n\t\tfcfg.SyncInterval = 100 * time.Millisecond\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage, MaxMsgsPer: 1}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// This yields an internal record length of 50 bytes. So 20 msgs per blk.\n\t\tmsg := bytes.Repeat([]byte(\"Z\"), 19)\n\t\tsubjects := \"ABCDEFGHIJKLMNOPQRST\"\n\t\tfor _, subj := range subjects {\n\t\t\tfs.StoreMsg(string(subj), nil, msg, 0)\n\t\t}\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\ttotal, reported, err := fs.Utilization()\n\t\trequire_NoError(t, err)\n\n\t\trequire_Equal(t, total, reported)\n\n\t\t// Now start removing, since we are small this should not kick in any inline logic.\n\t\t// Remove all interior messages, leave 1 and 20. So write B-S\n\t\tfor i := 1; i < 19; i++ {\n\t\t\tfs.StoreMsg(string(subjects[i]), nil, msg, 0)\n\t\t}\n\t\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\n\t\tblkUtil := func() (uint64, uint64) {\n\t\t\tfs.mu.RLock()\n\t\t\tfmb := fs.blks[0]\n\t\t\tfs.mu.RUnlock()\n\t\t\tfmb.mu.RLock()\n\t\t\tdefer fmb.mu.RUnlock()\n\t\t\treturn fmb.rbytes, fmb.bytes\n\t\t}\n\n\t\ttotal, reported = blkUtil()\n\t\trequire_Equal(t, reported, 100)\n\t\t// Raw bytes will be 1000, but due to compression could be less.\n\t\tif fcfg.Compression != NoCompression {\n\t\t\trequire_True(t, total > reported)\n\t\t} else {\n\t\t\trequire_Equal(t, total, 1000)\n\t\t}\n\n\t\t// Make sure the sync interval when kicked in compacts down to rbytes == 100.\n\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\tif total, reported := blkUtil(); total <= reported {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Not compacted yet, raw %v vs reported %v\",\n\t\t\t\tfriendlyBytes(total), friendlyBytes(reported))\n\t\t})\n\t})\n}\n\n// Make sure a call to Compact() updates PSIM correctly.\nfunc TestFileStoreCompactAndPSIMWhenDeletingBlocks(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 512},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"A\", bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\n\t// Add in 10 As\n\tfor i := 0; i < 10; i++ {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 4)\n\n\t// Should leave 1.\n\tn, err := fs.Compact(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, n, 9)\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\tfs.mu.RLock()\n\tinfo, _ := fs.psim.Find(stringToBytes(subj))\n\tpsi := *info\n\tfs.mu.RUnlock()\n\n\trequire_Equal(t, psi.total, 1)\n\trequire_Equal(t, psi.fblk, psi.lblk)\n}\n\nfunc TestFileStoreTrackSubjLenForPSIM(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Place 1000 msgs with varying subjects.\n\t// Make sure we track the subject length properly.\n\tsmap := make(map[string]int, 1000)\n\tbuf := make([]byte, 10)\n\tfor i := 0; i < 1000; i++ {\n\t\tvar b strings.Builder\n\t\t// 1-6 tokens.\n\t\tnumTokens := rand.Intn(6) + 1\n\t\tfor i := 0; i < numTokens; i++ {\n\t\t\ttlen := rand.Intn(4) + 2\n\t\t\ttok := buf[:tlen]\n\t\t\tcrand.Read(tok)\n\t\t\tb.WriteString(hex.EncodeToString(tok))\n\t\t\tif i != numTokens-1 {\n\t\t\t\tb.WriteString(\".\")\n\t\t\t}\n\t\t}\n\t\tsubj := b.String()\n\t\t// Avoid dupes since will cause check to fail after we delete messages.\n\t\tif _, ok := smap[subj]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tsmap[subj] = len(subj)\n\t\tfs.StoreMsg(subj, nil, nil, 0)\n\t}\n\n\tcheck := func() {\n\t\tt.Helper()\n\t\tvar total int\n\t\tfor _, slen := range smap {\n\t\t\ttotal += slen\n\t\t}\n\t\tfs.mu.RLock()\n\t\ttsl := fs.tsl\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, tsl, total)\n\t}\n\n\tcheck()\n\n\t// Delete ~half\n\tvar smv StoreMsg\n\tfor i := 0; i < 500; i++ {\n\t\tseq := uint64(rand.Intn(1000) + 1)\n\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tfs.RemoveMsg(seq)\n\t\tdelete(smap, sm.subj)\n\t}\n\n\tcheck()\n\n\t// Make sure we can recover same after restart.\n\tfs.Stop()\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheck()\n\n\t// Drain the rest through purge.\n\tfs.Purge()\n\tsmap = nil\n\tcheck()\n}\n\n// This was used to make sure our estimate was correct, but not needed normally.\nfunc TestFileStoreLargeFullStatePSIM(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tbuf := make([]byte, 20)\n\tfor i := 0; i < 100_000; i++ {\n\t\tvar b strings.Builder\n\t\t// 1-6 tokens.\n\t\tnumTokens := rand.Intn(6) + 1\n\t\tfor i := 0; i < numTokens; i++ {\n\t\t\ttlen := rand.Intn(8) + 2\n\t\t\ttok := buf[:tlen]\n\t\t\tcrand.Read(tok)\n\t\t\tb.WriteString(hex.EncodeToString(tok))\n\t\t\tif i != numTokens-1 {\n\t\t\t\tb.WriteString(\".\")\n\t\t\t}\n\t\t}\n\t\tsubj := b.String()\n\t\tfs.StoreMsg(subj, nil, nil, 0)\n\t}\n\tfs.Stop()\n}\n\nfunc TestFileStoreLargeFullStateMetaCleanup(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"foo.bar.baz\", bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(subj, nil, nil, 0)\n\t}\n\tfs.Stop()\n\n\tmdir := filepath.Join(sd, msgDir)\n\tidxFile := filepath.Join(mdir, \"1.idx\")\n\tfssFile := filepath.Join(mdir, \"1.fss\")\n\trequire_NoError(t, os.WriteFile(idxFile, msg, defaultFilePerms))\n\trequire_NoError(t, os.WriteFile(fssFile, msg, defaultFilePerms))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif _, err := os.Stat(idxFile); err == nil {\n\t\t\treturn errors.New(\"idx file still exists\")\n\t\t}\n\t\tif _, err := os.Stat(fssFile); err == nil {\n\t\t\treturn errors.New(\"fss file still exists\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestFileStoreIndexDBExistsAfterShutdown(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj := \"foo.bar.baz\"\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(subj, nil, nil, 0)\n\t}\n\n\tidxFile := filepath.Join(sd, msgDir, streamStreamStateFile)\n\n\tfs.mu.Lock()\n\tfs.dirty = 1\n\tif err := os.Remove(idxFile); err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\tt.Fatal(err)\n\t}\n\tfs.mu.Unlock()\n\n\tfs.Stop()\n\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif _, err := os.Stat(idxFile); err != nil {\n\t\t\treturn fmt.Errorf(\"%q doesn't exist\", idxFile)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/4842\nfunc TestFileStoreSubjectCorruption(t *testing.T) {\n\tsd, blkSize := t.TempDir(), uint64(2*1024*1024)\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: blkSize},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tnumSubjects := 100\n\tmsgs := [][]byte{bytes.Repeat([]byte(\"ABC\"), 333), bytes.Repeat([]byte(\"ABC\"), 888), bytes.Repeat([]byte(\"ABC\"), 555)}\n\tfor i := 0; i < 10_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(numSubjects)+1)\n\t\tmsg := msgs[rand.Intn(len(msgs))]\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\tfs.Stop()\n\n\trequire_NoError(t, os.Remove(filepath.Join(sd, msgDir, streamStreamStateFile)))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: blkSize},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfor subj := range fs.SubjectsTotals(\">\") {\n\t\tvar n int\n\t\t_, err := fmt.Sscanf(subj, \"foo.%d\", &n)\n\t\trequire_NoError(t, err)\n\t}\n}\n\n// Since 2.10 we no longer have fss, and the approach for calculating NumPending would branch\n// based on the old fss metadata being present. This meant that calculating NumPending in >= 2.10.x\n// would load all blocks to complete. This test makes sure we do not do that anymore.\nfunc TestFileStoreNumPendingLastBySubject(t *testing.T) {\n\tsd, blkSize := t.TempDir(), uint64(1024)\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: blkSize},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tnumSubjects := 20\n\tmsg := bytes.Repeat([]byte(\"ABC\"), 25)\n\tfor i := 1; i <= 1000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d.%d\", rand.Intn(numSubjects)+1, i)\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Each block has ~8 msgs.\n\trequire_True(t, fs.numMsgBlocks() > 100)\n\n\tcalcCacheLoads := func() (cloads uint64) {\n\t\tfs.mu.RLock()\n\t\tdefer fs.mu.RUnlock()\n\t\tfor _, mb := range fs.blks {\n\t\t\tmb.mu.RLock()\n\t\t\tcloads += mb.cloads\n\t\t\tmb.mu.RUnlock()\n\t\t}\n\t\treturn cloads\n\t}\n\n\ttotal, _, err := fs.NumPending(0, \"foo.*.*\", true)\n\trequire_NoError(t, err)\n\trequire_Equal(t, total, 1000)\n\t// Make sure no blocks were loaded to calculate this as a new consumer.\n\trequire_Equal(t, calcCacheLoads(), 0)\n\n\tcheckResult := func(sseq, np uint64, filter string) {\n\t\tt.Helper()\n\t\tvar checkTotal uint64\n\t\tvar smv StoreMsg\n\t\tfor seq := sseq; seq <= 1000; seq++ {\n\t\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\tif subjectIsSubsetMatch(sm.subj, filter) {\n\t\t\t\tcheckTotal++\n\t\t\t}\n\t\t}\n\t\trequire_Equal(t, np, checkTotal)\n\t}\n\n\t// Make sure partials work properly.\n\tfor _, filter := range []string{\"foo.10.*\", \"*.22.*\", \"*.*.222\", \"foo.5.999\", \"*.2.*\"} {\n\t\tsseq := uint64(rand.Intn(250) + 200) // Between 200-450\n\t\ttotal, _, err = fs.NumPending(sseq, filter, true)\n\t\trequire_NoError(t, err)\n\t\tcheckResult(sseq, total, filter)\n\t}\n}\n\n// We had a bug that could cause internal memory corruption of the psim keys in memory\n// which could have been written to disk via index.db.\nfunc TestFileStoreCorruptPSIMOnDisk(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"foo.bar\", nil, []byte(\"ABC\"), 0)\n\tfs.StoreMsg(\"foo.baz\", nil, []byte(\"XYZ\"), 0)\n\n\t// Force bad subject.\n\tfs.mu.Lock()\n\tpsi, _ := fs.psim.Find(stringToBytes(\"foo.bar\"))\n\tbad := []rune(\"foo.bar\")\n\tbad[3] = utf8.RuneError\n\tfs.psim.Insert(stringToBytes(string(bad)), *psi)\n\tfs.psim.Delete(stringToBytes(\"foo.bar\"))\n\tfs.dirty++\n\tfs.mu.Unlock()\n\n\t// Restart\n\tfs.Stop()\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsm, err := fs.LoadLastMsg(\"foo.bar\", nil)\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(sm.msg, []byte(\"ABC\")))\n\n\tsm, err = fs.LoadLastMsg(\"foo.baz\", nil)\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(sm.msg, []byte(\"XYZ\")))\n}\n\nfunc TestFileStorePurgeExBufPool(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 1024},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(\"foo.foo\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\n\tp, err := fs.PurgeEx(\"foo.bar\", 1, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, p, 1000)\n\n\t// Now make sure we do not have all of the msg blocks cache's loaded.\n\tvar loaded int\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tif mb.cacheAlreadyLoaded() {\n\t\t\tloaded++\n\t\t}\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\trequire_Equal(t, loaded, 1)\n}\n\nfunc TestFileStoreFSSMeta(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 100, CacheExpire: 200 * time.Millisecond, SubjectStateExpire: time.Second},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// This yields an internal record length of 50 bytes. So 2 msgs per blk with subject len of 1, e.g. \"A\" or \"Z\".\n\tmsg := bytes.Repeat([]byte(\"Z\"), 19)\n\n\t// Should leave us with |A-Z| |Z-Z| |Z-Z| |Z-A|\n\tfs.StoreMsg(\"A\", nil, msg, 0)\n\tfor i := 0; i < 6; i++ {\n\t\tfs.StoreMsg(\"Z\", nil, msg, 0)\n\t}\n\tfs.StoreMsg(\"A\", nil, msg, 0)\n\n\t// Let cache's expire before PurgeEx which will load them back in.\n\ttime.Sleep(500 * time.Millisecond)\n\n\tp, err := fs.PurgeEx(\"A\", 1, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, p, 2)\n\n\t// Make sure cache is not loaded.\n\tvar stillHasCache bool\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tstillHasCache = stillHasCache || mb.cacheAlreadyLoaded()\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\n\trequire_False(t, stillHasCache)\n\n\t// Let fss expire via SubjectStateExpire.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\tvar noFSS bool\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tnoFSS = noFSS || mb.fssNotLoaded()\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\n\trequire_True(t, noFSS)\n}\n\nfunc TestFileStoreExpireCacheOnLinearWalk(t *testing.T) {\n\tsd := t.TempDir()\n\texpire := 250 * time.Millisecond\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, CacheExpire: expire},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// This yields an internal record length of 50 bytes.\n\tsubj, msg := \"Z\", bytes.Repeat([]byte(\"Z\"), 19)\n\n\t// Store 10 messages, so 5 blocks.\n\tfor i := 0; i < 10; i++ {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Let them all expire. This way we load as we walk and can test that we expire all blocks without\n\t// needing to worry about last write times blocking forced expiration.\n\ttime.Sleep(expire + ats.TickInterval)\n\n\tcheckNoCache := func() {\n\t\tt.Helper()\n\t\tfs.mu.RLock()\n\t\tvar stillHasCache bool\n\t\tfor _, mb := range fs.blks {\n\t\t\tmb.mu.RLock()\n\t\t\tstillHasCache = stillHasCache || mb.cacheAlreadyLoaded()\n\t\t\tmb.mu.RUnlock()\n\t\t}\n\t\tfs.mu.RUnlock()\n\t\trequire_False(t, stillHasCache)\n\t}\n\n\t// Walk forward.\n\tvar smv StoreMsg\n\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\t_, err := fs.LoadMsg(seq, &smv)\n\t\trequire_NoError(t, err)\n\t}\n\tcheckNoCache()\n\n\t// No test walking backwards. We have this scenario when we search for starting points for sourced streams.\n\t// Noticed some memory bloat when we have to search many blocks looking for a source that may be closer to the\n\t// beginning of the stream (infrequently updated sourced stream).\n\tfor seq := uint64(10); seq >= 1; seq-- {\n\t\t_, err := fs.LoadMsg(seq, &smv)\n\t\trequire_NoError(t, err)\n\t}\n\tcheckNoCache()\n\n\t// Now make sure still expires properly on linear scans with deleted msgs.\n\t// We want to make sure we track linear updates even if message deleted.\n\t_, err = fs.RemoveMsg(2)\n\trequire_NoError(t, err)\n\t_, err = fs.RemoveMsg(9)\n\trequire_NoError(t, err)\n\n\t// Walk forward.\n\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\t_, err := fs.LoadMsg(seq, &smv)\n\t\tif seq == 2 || seq == 9 {\n\t\t\trequire_Error(t, err, errDeletedMsg)\n\t\t} else {\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\tcheckNoCache()\n}\n\nfunc TestFileStoreSkipMsgs(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Test on empty FS first.\n\t// Make sure wrong starting sequence fails.\n\terr = fs.SkipMsgs(10, 100)\n\trequire_Error(t, err, ErrSequenceMismatch)\n\n\terr = fs.SkipMsgs(1, 100)\n\trequire_NoError(t, err)\n\n\tstate := fs.State()\n\trequire_Equal(t, state.FirstSeq, 101)\n\trequire_Equal(t, state.LastSeq, 100)\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\t// Now add alot.\n\terr = fs.SkipMsgs(101, 100_000)\n\trequire_NoError(t, err)\n\tstate = fs.State()\n\trequire_Equal(t, state.FirstSeq, 100_101)\n\trequire_Equal(t, state.LastSeq, 100_100)\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\t// Now add in a message, and then skip to check dmap.\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"foo\", nil, nil, 0)\n\terr = fs.SkipMsgs(2, 10)\n\trequire_NoError(t, err)\n\tstate = fs.State()\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 11)\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.NumDeleted, 10)\n\trequire_Equal(t, len(state.Deleted), 10)\n\n\t// Check Fast State too.\n\tstate.Deleted = nil\n\tfs.FastState(&state)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 11)\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.NumDeleted, 10)\n}\n\nfunc TestFileStoreOptimizeFirstLoadNextMsgWithSequenceZero(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 4096},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := bytes.Repeat([]byte(\"ZZZ\"), 33) // ~100bytes\n\n\tfor i := 0; i < 5000; i++ {\n\t\tfs.StoreMsg(\"foo.A\", nil, msg, 0)\n\t}\n\t// This will create alot of blocks, ~167.\n\t// Just used to check that we do not load these in when searching.\n\t// Now add in 10 for foo.bar at the end.\n\tfor i := 0; i < 10; i++ {\n\t\tfs.StoreMsg(\"foo.B\", nil, msg, 0)\n\t}\n\t// The bug would not be visible on running server per se since we would have had fss loaded\n\t// and that sticks around a bit longer, we would use that to skip over the early blocks. So stop\n\t// and restart the filestore.\n\tfs.Stop()\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 4096},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Now fetch the next message for foo.B but set starting sequence to 0.\n\t_, nseq, err := fs.LoadNextMsg(\"foo.B\", false, 0, nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, nseq, 5001)\n\t// Now check how many blks are loaded, should be only 1.\n\trequire_Equal(t, fs.cacheLoads(), 1)\n}\n\nfunc TestFileStoreWriteFullStateHighSubjectCardinality(t *testing.T) {\n\tt.Skip()\n\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 4096},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte{1, 2, 3}\n\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"subj_%d\", i)\n\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tstart := time.Now()\n\trequire_NoError(t, fs.writeFullState())\n\tt.Logf(\"Took %s to writeFullState\", time.Since(start))\n}\n\nfunc TestFileStoreEraseMsgWithDbitSlots(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"foo\", nil, []byte(\"abd\"), 0)\n\tfor i := 0; i < 10; i++ {\n\t\tfs.SkipMsg(0)\n\t}\n\tfs.StoreMsg(\"foo\", nil, []byte(\"abd\"), 0)\n\t// Now grab that first block and compact away the skips which will\n\t// introduce dbits into our idx.\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\t// Compact.\n\tmb.mu.Lock()\n\terr = mb.compact()\n\tmb.mu.Unlock()\n\trequire_NoError(t, err)\n\n\tremoved, err := fs.EraseMsg(1)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n}\n\nfunc TestFileStoreEraseMsgWithAllTrailingDbitSlots(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.StoreMsg(\"foo\", nil, []byte(\"abc\"), 0)\n\tfs.StoreMsg(\"foo\", nil, []byte(\"abcdefg\"), 0)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfs.SkipMsg(0)\n\t}\n\t// Now grab that first block and compact away the skips which will\n\t// introduce dbits into our idx.\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\t// Compact.\n\tmb.mu.Lock()\n\terr = mb.compact()\n\tmb.mu.Unlock()\n\trequire_NoError(t, err)\n\n\tremoved, err := fs.EraseMsg(2)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n}\n\nfunc TestFileStoreMultiLastSeqs(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 256}, // Make block size small to test multiblock selections with maxSeq\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\", \"bar.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 0; i < 33; i++ {\n\t\tfs.StoreMsg(\"foo.foo\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\t}\n\tfor i := 0; i < 33; i++ {\n\t\tfs.StoreMsg(\"bar.foo\", nil, msg, 0)\n\t\tfs.StoreMsg(\"bar.bar\", nil, msg, 0)\n\t\tfs.StoreMsg(\"bar.baz\", nil, msg, 0)\n\t}\n\n\tcheckResults := func(seqs, expected []uint64) {\n\t\tt.Helper()\n\t\tif len(seqs) != len(expected) {\n\t\t\tt.Fatalf(\"Expected %+v got %+v\", expected, seqs)\n\t\t}\n\t\tfor i := range seqs {\n\t\t\tif seqs[i] != expected[i] {\n\t\t\t\tt.Fatalf(\"Expected %+v got %+v\", expected, seqs)\n\t\t\t}\n\t\t}\n\t}\n\n\t// UpTo sequence 3. Tests block split.\n\tseqs, err := fs.MultiLastSeqs([]string{\"foo.*\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\t// Up to last sequence of the stream.\n\tseqs, err = fs.MultiLastSeqs([]string{\"foo.*\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n\t// Check for bar.* at the end.\n\tseqs, err = fs.MultiLastSeqs([]string{\"bar.*\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\t// This should find nothing.\n\tseqs, err = fs.MultiLastSeqs([]string{\"bar.*\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, nil)\n\n\t// Do multiple subjects explicitly.\n\tseqs, err = fs.MultiLastSeqs([]string{\"foo.foo\", \"foo.bar\", \"foo.baz\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\tseqs, err = fs.MultiLastSeqs([]string{\"foo.foo\", \"foo.bar\", \"foo.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n\tseqs, err = fs.MultiLastSeqs([]string{\"bar.foo\", \"bar.bar\", \"bar.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\tseqs, err = fs.MultiLastSeqs([]string{\"bar.foo\", \"bar.bar\", \"bar.baz\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, nil)\n\n\t// Check single works\n\tseqs, err = fs.MultiLastSeqs([]string{\"foo.foo\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1})\n\n\t// Now test that we properly de-duplicate between filters.\n\tseqs, err = fs.MultiLastSeqs([]string{\"foo.*\", \"foo.bar\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\tseqs, err = fs.MultiLastSeqs([]string{\"bar.>\", \"bar.bar\", \"bar.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\n\t// All\n\tseqs, err = fs.MultiLastSeqs([]string{\">\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99, 196, 197, 198})\n\tseqs, err = fs.MultiLastSeqs([]string{\">\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n}\n\nfunc TestFileStoreMultiLastSeqsMaxAllowed(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 100; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\t// Test that if we specify maxAllowed that we get the correct error.\n\tseqs, err := fs.MultiLastSeqs([]string{\"foo.*\"}, 0, 10)\n\trequire_True(t, seqs == nil)\n\trequire_Error(t, err, ErrTooManyResults)\n}\n\n// https://github.com/nats-io/nats-server/issues/5236\n// Unclear how the sequences get off here, this is just forcing the situation reported.\nfunc TestFileStoreMsgBlockFirstAndLastSeqCorrupt(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\tfs.Purge()\n\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\n\tmb.mu.Lock()\n\tmb.tryForceExpireCacheLocked()\n\tatomic.StoreUint64(&mb.last.seq, 9)\n\tmb.mu.Unlock()\n\n\t// We should rebuild here and return no error.\n\trequire_NoError(t, mb.loadMsgs())\n\tmb.mu.RLock()\n\tfseq, lseq := atomic.LoadUint64(&mb.first.seq), atomic.LoadUint64(&mb.last.seq)\n\tmb.mu.RUnlock()\n\trequire_Equal(t, fseq, 11)\n\trequire_Equal(t, lseq, 10)\n}\n\nfunc TestFileStoreWriteFullStateAfterPurgeEx(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\tfs.RemoveMsg(8)\n\tfs.RemoveMsg(9)\n\tfs.RemoveMsg(10)\n\n\tn, err := fs.PurgeEx(\">\", 8, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, n, 7)\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 11)\n\trequire_Equal(t, ss.LastSeq, 10)\n\n\t// Make sure this does not reset our state due to skew with msg blocks.\n\tfs.writeFullState()\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 11)\n\trequire_Equal(t, ss.LastSeq, 10)\n}\n\nfunc TestFileStoreFSSExpire(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 8192, CacheExpire: 1 * time.Second, SubjectStateExpire: time.Second},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, MaxMsgsPer: 1, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 1000; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\t// Flush fss by hand, cache should be flushed as well.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tmb.fss = nil\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\n\tfs.StoreMsg(\"foo.11\", nil, msg, 0)\n\ttime.Sleep(900 * time.Millisecond)\n\t// This should keep fss alive in the first block..\n\t// As well as cache itself due to remove activity.\n\tfs.StoreMsg(\"foo.22\", nil, msg, 0)\n\ttime.Sleep(300 * time.Millisecond)\n\t// Check that fss and the cache are still loaded.\n\tfs.mu.RLock()\n\tmb := fs.blks[0]\n\tfs.mu.RUnlock()\n\tmb.mu.RLock()\n\tcache, fss := mb.ecache.Value(), mb.fss\n\tmb.mu.RUnlock()\n\trequire_True(t, fss != nil)\n\trequire_True(t, cache != nil)\n}\n\nfunc TestFileStoreFSSExpireNumPending(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 8192, CacheExpire: 1 * time.Second, SubjectStateExpire: 2 * time.Second},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, MaxMsgsPer: 1, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 100_000; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.A.%d\", i), nil, msg, 0)\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.B.%d\", i), nil, msg, 0)\n\t}\n\t// Flush fss by hand, cache should be flushed as well.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tmb.fss = nil\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\n\tnb := fs.numMsgBlocks()\n\t// Now execute NumPending() such that we load lots of blocks and make sure fss do not expire.\n\tstart := time.Now()\n\tn, _, err := fs.NumPending(100_000, \"foo.A.*\", false)\n\trequire_NoError(t, err)\n\telapsed := time.Since(start)\n\n\trequire_Equal(t, n, 50_000)\n\t// Make sure we did not force expire the fss. We would have loaded first half of blocks.\n\tvar noFss bool\n\tlast := nb/2 - 1\n\tfs.mu.RLock()\n\tfor i, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tnoFss = mb.fss == nil\n\t\tmb.mu.RUnlock()\n\t\tif noFss || i == last {\n\t\t\tbreak\n\t\t}\n\t}\n\tfs.mu.RUnlock()\n\trequire_False(t, noFss)\n\n\t// Run again, make sure faster. This is consequence of fss being loaded now.\n\tstart = time.Now()\n\tfs.NumPending(100_000, \"foo.A.*\", false)\n\trequire_True(t, elapsed > 2*time.Since(start))\n\n\t// Now do with start past the mid-point.\n\tstart = time.Now()\n\tfs.NumPending(150_000, \"foo.B.*\", false)\n\telapsed = time.Since(start)\n\ttime.Sleep(time.Second)\n\tstart = time.Now()\n\tfs.NumPending(150_000, \"foo.B.*\", false)\n\trequire_True(t, elapsed > time.Since(start))\n\n\t// Sleep enough so that all mb.fss should expire, which is 2s above.\n\ttime.Sleep(4 * time.Second)\n\tfs.mu.RLock()\n\tfor i, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tfss := mb.fss\n\t\tmb.mu.RUnlock()\n\t\tif fss != nil {\n\t\t\tfs.mu.RUnlock()\n\t\t\tt.Fatalf(\"Detected loaded fss for mb %d (size %d)\", i, fss.Size())\n\t\t}\n\t}\n\tfs.mu.RUnlock()\n}\n\n// We want to ensure that recovery of deleted messages survives no index.db and compactions.\nfunc TestFileStoreRecoverWithRemovesAndNoIndexDB(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 250},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\tfs.RemoveMsg(1)\n\tfs.RemoveMsg(2)\n\tfs.RemoveMsg(8)\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 3)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 7)\n\n\t// Compact last block.\n\tfs.mu.RLock()\n\tlmb := fs.lmb\n\tfs.mu.RUnlock()\n\tlmb.mu.Lock()\n\terr = lmb.compact()\n\tlmb.mu.Unlock()\n\trequire_NoError(t, err)\n\t// Stop but remove index.db\n\tsfile := filepath.Join(sd, msgDir, streamStreamStateFile)\n\tfs.Stop()\n\tos.Remove(sfile)\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 3)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 7)\n}\n\nfunc TestFileStoreReloadAndLoseLastSequence(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfor i := 0; i < 22; i++ {\n\t\tfs.SkipMsg(0)\n\t}\n\n\t// Restart 5 times.\n\tfor i := 0; i < 5; i++ {\n\t\tfs.Stop()\n\t\tfs, err = newFileStore(\n\t\t\tFileStoreConfig{StoreDir: sd},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 23)\n\t\trequire_Equal(t, ss.LastSeq, 22)\n\t}\n}\n\nfunc TestFileStoreReloadAndLoseLastSequenceWithSkipMsgs(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Make sure same works with SkipMsgs which can kick in from delete blocks to replicas.\n\trequire_NoError(t, fs.SkipMsgs(0, 22))\n\n\t// Restart 5 times.\n\tfor i := 0; i < 5; i++ {\n\t\tfs.Stop()\n\t\tfs, err = newFileStore(\n\t\t\tFileStoreConfig{StoreDir: sd},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 23)\n\t\trequire_Equal(t, ss.LastSeq, 22)\n\t}\n}\n\nfunc TestFileStoreLoadLastWildcard(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 512},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(\"foo.11.foo\", nil, msg, 0)\n\t}\n\n\t// Make sure we remove fss since that would mask the problem that we walk\n\t// all the blocks.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tmb.fss = nil\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\n\t// Attempt to load the last msg using a wildcarded subject.\n\tsm, err := fs.LoadLastMsg(\"foo.22.*\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, sm.seq, 2)\n\n\t// Make sure that we took advantage of psim meta data and only load one block.\n\tvar cloads uint64\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tcloads += mb.cloads\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\trequire_Equal(t, cloads, 1)\n}\n\nfunc TestFileStoreLoadLastWildcardWithPresenceMultipleBlocks(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 64},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Make sure we have \"foo.222.bar\" in multiple blocks to show bug.\n\tfs.StoreMsg(\"foo.22.bar\", nil, []byte(\"hello\"), 0)\n\tfs.StoreMsg(\"foo.22.baz\", nil, []byte(\"ok\"), 0)\n\tfs.StoreMsg(\"foo.22.baz\", nil, []byte(\"ok\"), 0)\n\tfs.StoreMsg(\"foo.22.bar\", nil, []byte(\"hello22\"), 0)\n\trequire_True(t, fs.numMsgBlocks() > 1)\n\tsm, err := fs.LoadLastMsg(\"foo.*.bar\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, \"hello22\", string(sm.msg))\n}\n\n// We want to make sure that we update psim correctly on a miss.\nfunc TestFileStoreFilteredPendingPSIMFirstBlockUpdate(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 512},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// When PSIM detects msgs == 1 will catch up, so msgs needs to be > 1.\n\t// Then create a huge block gap.\n\tmsg := []byte(\"hello\")\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(\"foo.foo\", nil, msg, 0)\n\t}\n\t// Bookend with 2 more foo.baz\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\t// Now remove first one.\n\tremoved, err := fs.RemoveMsg(1)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\t// 84 blocks.\n\trequire_Equal(t, fs.numMsgBlocks(), 84)\n\tfs.mu.RLock()\n\tpsi, ok := fs.psim.Find([]byte(\"foo.baz\"))\n\tfs.mu.RUnlock()\n\trequire_True(t, ok)\n\trequire_Equal(t, psi.total, 2)\n\trequire_Equal(t, psi.fblk, 1)\n\trequire_Equal(t, psi.lblk, 84)\n\n\t// No make sure that a call to numFilterPending which will initially walk all blocks if starting from seq 1 updates psi.\n\tvar ss SimpleState\n\tfs.mu.RLock()\n\terr = fs.numFilteredPending(\"foo.baz\", &ss)\n\tfs.mu.RUnlock()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ss.Msgs, 2)\n\trequire_Equal(t, ss.First, 1002)\n\trequire_Equal(t, ss.Last, 1003)\n\n\t// Check psi was updated. This is done in separate go routine to acquire\n\t// the write lock now.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tfs.mu.RLock()\n\t\tpsi, ok = fs.psim.Find([]byte(\"foo.baz\"))\n\t\ttotal, fblk, lblk := psi.total, psi.fblk, psi.lblk\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, total, 2)\n\t\trequire_Equal(t, lblk, 84)\n\t\tif fblk != 84 {\n\t\t\treturn fmt.Errorf(\"fblk should be 84, still %d\", fblk)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestFileStoreWildcardFilteredPendingPSIMFirstBlockUpdate(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 512},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// When PSIM detects msgs == 1 will catch up, so msgs needs to be > 1.\n\t// Then create a huge block gap.\n\tmsg := []byte(\"hello\")\n\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(\"foo.1.foo\", nil, msg, 0)\n\t}\n\t// Bookend with 3 more, two foo.baz and two foo.bar.\n\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\n\t// Now remove first one for foo.bar and foo.baz.\n\tremoved, err := fs.RemoveMsg(1)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\tremoved, err = fs.RemoveMsg(2)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\n\t// 92 blocks.\n\trequire_Equal(t, fs.numMsgBlocks(), 92)\n\tfs.mu.RLock()\n\tpsi, ok := fs.psim.Find([]byte(\"foo.22.baz\"))\n\ttotal, fblk, lblk := psi.total, psi.fblk, psi.lblk\n\tfs.mu.RUnlock()\n\trequire_True(t, ok)\n\trequire_Equal(t, total, 2)\n\trequire_Equal(t, fblk, 1)\n\trequire_Equal(t, lblk, 92)\n\n\tfs.mu.RLock()\n\tpsi, ok = fs.psim.Find([]byte(\"foo.22.bar\"))\n\ttotal, fblk, lblk = psi.total, psi.fblk, psi.lblk\n\tfs.mu.RUnlock()\n\trequire_True(t, ok)\n\trequire_Equal(t, total, 2)\n\trequire_Equal(t, fblk, 1)\n\trequire_Equal(t, lblk, 92)\n\n\t// No make sure that a call to numFilterPending which will initially walk all blocks if starting from seq 1 updates psi.\n\tvar ss SimpleState\n\tfs.mu.RLock()\n\terr = fs.numFilteredPending(\"foo.22.*\", &ss)\n\tfs.mu.RUnlock()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ss.Msgs, 4)\n\trequire_Equal(t, ss.First, 1003)\n\trequire_Equal(t, ss.Last, 1006)\n\n\t// Check both psi were updated.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tfs.mu.RLock()\n\t\tpsi, ok = fs.psim.Find([]byte(\"foo.22.baz\"))\n\t\ttotal, fblk, lblk = psi.total, psi.fblk, psi.lblk\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, total, 2)\n\t\trequire_Equal(t, lblk, 92)\n\t\tif fblk != 92 {\n\t\t\treturn fmt.Errorf(\"fblk should be 92, still %d\", fblk)\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tfs.mu.RLock()\n\t\tpsi, ok = fs.psim.Find([]byte(\"foo.22.bar\"))\n\t\ttotal, fblk, lblk = psi.total, psi.fblk, psi.lblk\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, total, 2)\n\t\trequire_Equal(t, fblk, 92)\n\t\tif fblk != 92 {\n\t\t\treturn fmt.Errorf(\"fblk should be 92, still %d\", fblk)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Make sure if we only miss by one for fblk that we still update it.\nfunc TestFileStoreFilteredPendingPSIMFirstBlockUpdateNextBlock(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 128},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\t// Create 4 blocks, each block holds 2 msgs\n\tfor i := 0; i < 4; i++ {\n\t\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 4)\n\n\tfetch := func(subj string) *psi {\n\t\tt.Helper()\n\t\tfs.mu.RLock()\n\t\tvar info psi\n\t\tpsi, ok := fs.psim.Find([]byte(subj))\n\t\tif ok && psi != nil {\n\t\t\tinfo = *psi\n\t\t}\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, ok)\n\t\treturn &info\n\t}\n\n\tpsi := fetch(\"foo.22.bar\")\n\trequire_Equal(t, psi.total, 4)\n\trequire_Equal(t, psi.fblk, 1)\n\trequire_Equal(t, psi.lblk, 4)\n\n\t// Now remove first instance of \"foo.22.bar\"\n\tremoved, err := fs.RemoveMsg(1)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\n\t// Call into numFilterePending(), we want to make sure it updates fblk.\n\tvar ss SimpleState\n\tfs.mu.Lock()\n\terr = fs.numFilteredPending(\"foo.22.bar\", &ss)\n\tfs.mu.Unlock()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ss.Msgs, 3)\n\trequire_Equal(t, ss.First, 3)\n\trequire_Equal(t, ss.Last, 7)\n\n\t// Now make sure that we properly updated the psim entry.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tpsi = fetch(\"foo.22.bar\")\n\t\trequire_Equal(t, psi.total, 3)\n\t\trequire_Equal(t, psi.lblk, 4)\n\t\tif psi.fblk != 2 {\n\t\t\treturn fmt.Errorf(\"fblk should be 2, still %d\", psi.fblk)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now make sure wildcard calls into also update blks.\n\t// First remove first \"foo.22.baz\" which will remove first block.\n\tremoved, err = fs.RemoveMsg(2)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\t// Make sure 3 blks left.\n\trequire_Equal(t, fs.numMsgBlocks(), 3)\n\n\tpsi = fetch(\"foo.22.baz\")\n\trequire_Equal(t, psi.total, 3)\n\trequire_Equal(t, psi.fblk, 1)\n\trequire_Equal(t, psi.lblk, 4)\n\n\t// Now call wildcard version of numFilteredPending to make sure it clears.\n\tfs.mu.Lock()\n\terr = fs.numFilteredPending(\"foo.*.baz\", &ss)\n\tfs.mu.Unlock()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ss.Msgs, 3)\n\trequire_Equal(t, ss.First, 4)\n\trequire_Equal(t, ss.Last, 8)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tpsi = fetch(\"foo.22.baz\")\n\t\trequire_Equal(t, psi.total, 3)\n\t\trequire_Equal(t, psi.lblk, 4)\n\t\tif psi.fblk != 2 {\n\t\t\treturn fmt.Errorf(\"fblk should be 2, still %d\", psi.fblk)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestFileStoreLargeSparseMsgsDoNotLoadAfterLast(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 128},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\t// Create 2 blocks with each, each block holds 2 msgs\n\tfor i := 0; i < 2; i++ {\n\t\tfs.StoreMsg(\"foo.22.bar\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\t}\n\t// Now create 8 more blocks with just baz. So no matches for these 8 blocks\n\t// for \"foo.22.bar\".\n\tfor i := 0; i < 8; i++ {\n\t\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\t\tfs.StoreMsg(\"foo.22.baz\", nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 10)\n\n\t// Remove all blk cache and fss.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tmb.fss, mb.cache = nil, nil\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\n\t// \"foo.22.bar\" is at sequence 1 and 3.\n\t// Make sure if we do a LoadNextMsg() starting at 4 that we do not load\n\t// all the tail blocks.\n\t_, _, err = fs.LoadNextMsg(\"foo.*.bar\", true, 4, nil)\n\trequire_Error(t, err, ErrStoreEOF)\n\n\t// Now make sure we did not load fss and cache.\n\tvar loaded int\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tif mb.cache != nil || mb.fss != nil {\n\t\t\tloaded++\n\t\t}\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\t// We will load first block for starting seq 4, but no others should have loaded.\n\trequire_Equal(t, loaded, 1)\n}\n\nfunc TestFileStoreCheckSkipFirstBlockBug(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 128},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\n\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0)\n\tfor i := 0; i < 5; i++ {\n\t\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\t}\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0)\n\n\t// Should have created 4 blocks.\n\t// BB BB | AA BB | BB BB | BB BB | AA AA\n\trequire_Equal(t, fs.numMsgBlocks(), 5)\n\n\tfs.RemoveMsg(3)\n\tfs.RemoveMsg(4)\n\n\t// Second block should be gone now.\n\t// BB BB | -- -- | BB BB | BB BB | AA AA\n\trequire_Equal(t, fs.numMsgBlocks(), 4)\n\n\t_, _, err = fs.LoadNextMsg(\"foo.AA.bar\", false, 4, nil)\n\trequire_NoError(t, err)\n}\n\n// https://github.com/nats-io/nats-server/issues/5702\nfunc TestFileStoreTombstoneRbytes(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Block can hold 24 msgs.\n\t// So will fill one block and half of the other\n\tmsg := []byte(\"hello\")\n\tfor i := 0; i < 34; i++ {\n\t\tfs.StoreMsg(\"foo.22\", nil, msg, 0)\n\t}\n\trequire_True(t, fs.numMsgBlocks() > 1)\n\t// Now delete second half of first block which will place tombstones in second blk.\n\tfor seq := 11; seq <= 24; seq++ {\n\t\tfs.RemoveMsg(uint64(seq))\n\t}\n\t// Now check that rbytes has been properly accounted for in second block.\n\tfs.mu.RLock()\n\tblk := fs.blks[1]\n\tfs.mu.RUnlock()\n\n\tblk.mu.RLock()\n\tbytes, rbytes := blk.bytes, blk.rbytes\n\tblk.mu.RUnlock()\n\trequire_True(t, rbytes > bytes)\n}\n\nfunc TestFileStoreMsgBlockShouldCompact(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// 127 fit into a block.\n\tmsg := bytes.Repeat([]byte(\"Z\"), 64*1024)\n\tfor i := 0; i < 190; i++ {\n\t\tfs.StoreMsg(\"foo.22\", nil, msg, 0)\n\t}\n\trequire_True(t, fs.numMsgBlocks() > 1)\n\t// Now delete second half of first block which will place tombstones in second blk.\n\tfor seq := 64; seq <= 127; seq++ {\n\t\tfs.RemoveMsg(uint64(seq))\n\t}\n\tfs.mu.RLock()\n\tfblk := fs.blks[0]\n\tsblk := fs.blks[1]\n\tfs.mu.RUnlock()\n\n\tfblk.mu.RLock()\n\tbytes, rbytes := fblk.bytes, fblk.rbytes\n\tshouldCompact := fblk.shouldCompactInline()\n\tfblk.mu.RUnlock()\n\t// Should have tripped compaction already.\n\trequire_Equal(t, bytes, rbytes)\n\trequire_False(t, shouldCompact)\n\n\tsblk.mu.RLock()\n\tshouldCompact = sblk.shouldCompactInline()\n\tsblk.mu.RUnlock()\n\trequire_False(t, shouldCompact)\n}\n\nfunc TestFileStoreCheckSkipFirstBlockNotLoadOldBlocks(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 128},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\n\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0)\n\tfor i := 0; i < 6; i++ {\n\t\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\t}\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0) // Sequence 9\n\tfs.StoreMsg(\"foo.AA.bar\", nil, msg, 0) // Sequence 10\n\n\tfor i := 0; i < 4; i++ {\n\t\tfs.StoreMsg(\"foo.BB.bar\", nil, msg, 0)\n\t}\n\n\t// Should have created 7 blocks.\n\t// BB AA | BB BB | BB BB | BB BB | AA AA | BB BB | BB BB\n\trequire_Equal(t, fs.numMsgBlocks(), 7)\n\n\tfs.RemoveMsg(1)\n\tfs.RemoveMsg(2)\n\n\t// First block should be gone now.\n\t// -- -- | BB BB | BB BB | BB BB | AA AA | BB BB | BB BB\n\trequire_Equal(t, fs.numMsgBlocks(), 6)\n\n\t// Remove all blk cache and fss.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tmb.fss, mb.cache = nil, nil\n\t\tmb.mu.Unlock()\n\t}\n\tfs.mu.RUnlock()\n\n\t// But this means that the psim still points fblk to block 1.\n\t// So when we try to load AA from near the end (last AA sequence), it will not find anything and will then\n\t// check if we can skip ahead, but in the process reload blocks 2, 3, 4 amd 5..\n\t// This can trigger for an up to date consumer near the end of the stream that gets a new pull request that will pop it out of msgWait\n\t// and it will call LoadNextMsg() like we do here with starting sequence of 11.\n\t_, _, err = fs.LoadNextMsg(\"foo.AA.bar\", false, 11, nil)\n\trequire_Error(t, err, ErrStoreEOF)\n\n\t// Now make sure we did not load fss and cache.\n\tvar loaded int\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.RLock()\n\t\tif mb.cache != nil || mb.fss != nil {\n\t\t\tloaded++\n\t\t}\n\t\tmb.mu.RUnlock()\n\t}\n\tfs.mu.RUnlock()\n\t// We will load last block for starting seq 9, but no others should have loaded.\n\trequire_Equal(t, loaded, 1)\n}\n\nfunc TestFileStoreSyncCompressOnlyIfDirty(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 256, SyncInterval: 250 * time.Millisecond},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\n\t// 6 msgs per block.\n\t// Fill 2 blocks.\n\tfor i := 0; i < 12; i++ {\n\t\tfs.StoreMsg(\"foo.BB\", nil, msg, 0)\n\t}\n\t// Create third block with just one message in it.\n\tfs.StoreMsg(\"foo.BB\", nil, msg, 0)\n\n\t// Should have created 3 blocks.\n\trequire_Equal(t, fs.numMsgBlocks(), 3)\n\n\t// Now delete a bunch that will will fill up 3 block with tombstones.\n\tfor _, seq := range []uint64{2, 3, 4, 5, 8, 9, 10, 11} {\n\t\t_, err = fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t}\n\t// Now make sure we add 4/5th block so syncBlocks will try to compact.\n\tfor i := 0; i < 6; i++ {\n\t\tfs.StoreMsg(\"foo.BB\", nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 5)\n\n\t// All should have compact set.\n\tfs.mu.Lock()\n\t// Only check first 3 blocks.\n\tfor i := 0; i < 3; i++ {\n\t\tmb := fs.blks[i]\n\t\tmb.mu.Lock()\n\t\tshouldCompact := mb.shouldCompactSync()\n\t\tmb.mu.Unlock()\n\t\tif !shouldCompact {\n\t\t\tfs.mu.Unlock()\n\t\t\tt.Fatalf(\"Expected should compact to be true for %d, got false\", mb.getIndex())\n\t\t}\n\t}\n\tfs.mu.Unlock()\n\n\t// Let sync run.\n\ttime.Sleep(300 * time.Millisecond)\n\n\t// We want to make sure the last block, which is filled with tombstones and is not compactable, returns false now.\n\tfs.mu.Lock()\n\tfor _, mb := range fs.blks {\n\t\tmb.mu.Lock()\n\t\tshouldCompact := mb.shouldCompactSync()\n\t\tmb.mu.Unlock()\n\t\tif shouldCompact {\n\t\t\tfs.mu.Unlock()\n\t\t\tt.Fatalf(\"Expected should compact to be false for %d, got true\", mb.getIndex())\n\t\t}\n\t}\n\tfs.mu.Unlock()\n\n\t// Now remove some from block 3 and verify that compact is not suppressed.\n\t_, err = fs.RemoveMsg(13)\n\trequire_NoError(t, err)\n\n\tfs.mu.Lock()\n\tmb := fs.blks[2] // block 3.\n\tmb.mu.Lock()\n\tnoCompact := mb.noCompact\n\tmb.mu.Unlock()\n\tfs.mu.Unlock()\n\t// Verify that since we deleted a message we should be considered for compaction again in syncBlocks().\n\trequire_False(t, noCompact)\n}\n\n// This test is for deleted interior message tracking after compaction from limits based deletes, meaning no tombstones.\n// Bug was that dmap would not be properly be hydrated after the compact from rebuild. But we did so in populateGlobalInfo.\n// So this is just to fix a bug in rebuildState tracking gaps after a compact.\nfunc TestFileStoreDmapBlockRecoverAfterCompact(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 256},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, MaxMsgsPer: 1})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\n\t// 6 msgs per block.\n\t// Fill the first block.\n\tfor i := 1; i <= 6; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\n\t// Now create holes in the first block via the max msgs per subject of 1.\n\tfor i := 2; i < 6; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\t// Compact and rebuild the first blk. Do not have it call indexCacheBuf which will fix it up.\n\tmb := fs.getFirstBlock()\n\tmb.mu.Lock()\n\tif err = mb.compact(); err != nil {\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\t// Empty out dmap state.\n\tmb.dmap.Empty()\n\tld, tombs, err := mb.rebuildStateLocked()\n\tdmap := mb.dmap.Clone()\n\tmb.mu.Unlock()\n\n\trequire_NoError(t, err)\n\trequire_Equal(t, ld, nil)\n\trequire_Equal(t, len(tombs), 0)\n\trequire_Equal(t, dmap.Size(), 4)\n}\n\nfunc TestFileStoreRestoreIndexWithMatchButLeftOverBlocks(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 256},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, MaxMsgsPer: 1})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"hello\")\n\n\t// 6 msgs per block.\n\t// Fill the first 2 blocks.\n\tfor i := 1; i <= 12; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\n\t// We will now stop which will create the index.db file which will\n\t// match the last record exactly.\n\tsfile := filepath.Join(sd, msgDir, streamStreamStateFile)\n\tfs.Stop()\n\n\t// Grab it since we will put it back.\n\tbuf, err := os.ReadFile(sfile)\n\trequire_NoError(t, err)\n\trequire_True(t, len(buf) > 0)\n\n\t// Now do an additional block, but with the MaxMsgsPer this will remove the first block,\n\t// but leave the second so on recovery will match the checksum for the last msg in second block.\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 256},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, MaxMsgsPer: 1})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfor i := 1; i <= 6; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\n\t// Grab correct state, we will use it to make sure we do the right thing.\n\tvar state StreamState\n\tfs.FastState(&state)\n\n\trequire_Equal(t, state.Msgs, 12)\n\trequire_Equal(t, state.FirstSeq, 7)\n\trequire_Equal(t, state.LastSeq, 18)\n\t// This will be block 2 and 3.\n\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\n\tfs.Stop()\n\t// Put old stream state back.\n\trequire_NoError(t, os.WriteFile(sfile, buf, defaultFilePerms))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 256},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, MaxMsgsPer: 1})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.FastState(&state)\n\trequire_Equal(t, state.Msgs, 12)\n\trequire_Equal(t, state.FirstSeq, 7)\n\trequire_Equal(t, state.LastSeq, 18)\n}\n\nfunc TestFileStoreRestoreDeleteTombstonesExceedingMaxBlkSize(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 256\n\t\tfs, err := newFileStoreWithCreated(\n\t\t\tfcfg,\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage},\n\t\t\ttime.Now(),\n\t\t\tprf(&fcfg),\n\t\t\tnil,\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tn, err := fs.PurgeEx(_EMPTY_, 1_000_000_000, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, n, 0)\n\n\t\tmsg := []byte(\"hello\")\n\t\t// 6 msgs per block with blk size 256.\n\t\tfor i := 1; i <= 10_000; i++ {\n\t\t\tfs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t\t}\n\t\t// Now delete msgs which will write tombstones.\n\t\tfor seq := uint64(1_000_000_001); seq < 1_000_000_101; seq++ {\n\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\t\t}\n\n\t\t// Check last block and make sure the tombstones did not exceed blk size maximum.\n\t\t// Check to make sure no blocks exceed blk size.\n\t\tfs.mu.RLock()\n\t\tblks := append([]*msgBlock(nil), fs.blks...)\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\n\t\tvar emptyBlks []*msgBlock\n\t\tfor _, mb := range blks {\n\t\t\tmb.mu.RLock()\n\t\t\tbytes, rbytes := mb.bytes, mb.rbytes\n\t\t\tmb.mu.RUnlock()\n\t\t\trequire_True(t, bytes < 256)\n\t\t\trequire_True(t, rbytes < 256)\n\t\t\tif bytes == 0 && mb != lmb {\n\t\t\t\temptyBlks = append(emptyBlks, mb)\n\t\t\t}\n\t\t}\n\t\t// Check each block such that it signals it can be compacted but if we attempt compact here nothing should change.\n\t\tfor _, mb := range emptyBlks {\n\t\t\tmb.mu.Lock()\n\t\t\tmb.ensureRawBytesLoaded()\n\t\t\tbytes, rbytes, shouldCompact := mb.bytes, mb.rbytes, mb.shouldCompactSync()\n\t\t\t// Do the compact and make sure nothing changed.\n\t\t\terr = mb.compact()\n\t\t\tnbytes, nrbytes := mb.bytes, mb.rbytes\n\t\t\tmb.mu.Unlock()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, shouldCompact)\n\t\t\trequire_Equal(t, bytes, nbytes)\n\t\t\trequire_Equal(t, rbytes, nrbytes)\n\t\t}\n\n\t\t// Now remove first msg which will invalidate the tombstones since they will be < first sequence.\n\t\tremoved, err := fs.RemoveMsg(1_000_000_000)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\n\t\t// Now simulate a syncBlocks call and make sure it cleans up the tombstones that are no longer relevant.\n\t\tfs.syncBlocks()\n\t\tfor _, mb := range emptyBlks {\n\t\t\tmb.mu.Lock()\n\t\t\tmb.ensureRawBytesLoaded()\n\t\t\tindex, bytes, rbytes := mb.index, mb.bytes, mb.rbytes\n\t\t\tmb.mu.Unlock()\n\t\t\trequire_Equal(t, bytes, 0)\n\t\t\trequire_Equal(t, rbytes, 0)\n\t\t\t// Also make sure we removed these blks all together.\n\t\t\tfs.mu.RLock()\n\t\t\timb := fs.bim[index]\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_True(t, imb == nil)\n\t\t}\n\t})\n}\n\n///////////////////////////////////////////////////////////////////////////\n// Benchmarks\n///////////////////////////////////////////////////////////////////////////\n\nfunc Benchmark_FileStoreSelectMsgBlock(b *testing.B) {\n\t// We use small block size to create lots of blocks for this test.\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 128},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"A\", bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\n\t// Add in a bunch of blocks.\n\tfor i := 0; i < 1000; i++ {\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\tif fs.numMsgBlocks() < 1000 {\n\t\tb.Fatalf(\"Expected at least 1000 blocks, got %d\", fs.numMsgBlocks())\n\t}\n\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, mb := fs.selectMsgBlockWithIndex(1)\n\t\tif mb == nil {\n\t\t\tb.Fatalf(\"Expected a non-nil mb\")\n\t\t}\n\t}\n\tb.StopTimer()\n}\n\nfunc Benchmark_FileStoreLoadNextMsgSameFilterAsStream(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 100_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(1024))\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Needs to start at ~1 to show slowdown.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.*\", true, 10, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgLiteralSubject(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 100_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(1024))\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// This is the one we will try to match.\n\tfs.StoreMsg(\"foo.2222\", nil, msg, 0)\n\t// So not last and we think we are done linear scan.\n\tfs.StoreMsg(\"foo.3333\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _, err := fs.LoadNextMsg(\"foo.2222\", false, 10, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgNoMsgsFirstSeq(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// This should error with EOF\n\t\t_, _, err := fs.LoadNextMsg(\"foo.baz\", false, 1, &smv)\n\t\tif err != ErrStoreEOF {\n\t\t\tb.Fatalf(\"Wrong error, expected EOF got %v\", err)\n\t\t}\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgNoMsgsNotFirstSeq(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// This should error with EOF\n\t\t// Make sure the sequence is not first seq of 1.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.baz\", false, 10, &smv)\n\t\tif err != ErrStoreEOF {\n\t\t\tb.Fatalf(\"Wrong error, expected EOF got %v\", err)\n\t\t}\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgVerySparseMsgsFirstSeq(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\t// Make last msg one that would match.\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _, err := fs.LoadNextMsg(\"foo.baz\", false, 1, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgVerySparseMsgsNotFirstSeq(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\t// Make last msg one that would match.\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Make sure not first seq.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.baz\", false, 10, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgVerySparseMsgsInBetween(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir(), BlockSize: 8192},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Make first msg one that would match as well.\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\t// Add in a bunch of msgs\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tfs.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t}\n\t// Make last msg one that would match as well.\n\tfs.StoreMsg(\"foo.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Make sure not first seq.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.baz\", false, 2, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgVerySparseMsgsInBetweenWithWildcard(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Make first msg one that would match as well.\n\tfs.StoreMsg(\"foo.1.baz\", nil, msg, 0)\n\t// Add in a bunch of msgs.\n\t// We need to make sure we have a range of subjects that could kick in a linear scan.\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", rand.Intn(100_000)+2)\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Make last msg one that would match as well.\n\tfs.StoreMsg(\"foo.1.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Make sure not first seq.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.*.baz\", true, 2, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextManySubjectsWithWildcardNearLastBlock(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Make first msg one that would match as well.\n\tfs.StoreMsg(\"foo.1.baz\", nil, msg, 0)\n\t// Add in a bunch of msgs.\n\t// We need to make sure we have a range of subjects that could kick in a linear scan.\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", rand.Intn(100_000)+2)\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\t// Make last msg one that would match as well.\n\tfs.StoreMsg(\"foo.1.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Make sure not first seq.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.*.baz\", true, 999_990, &smv)\n\t\trequire_NoError(b, err)\n\t}\n}\n\nfunc Benchmark_FileStoreLoadNextMsgVerySparseMsgsLargeTail(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Small om purpose.\n\tmsg := []byte(\"ok\")\n\n\t// Make first msg one that would match as well.\n\tfs.StoreMsg(\"foo.1.baz\", nil, msg, 0)\n\t// Add in a bunch of msgs.\n\t// We need to make sure we have a range of subjects that could kick in a linear scan.\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", rand.Intn(64_000)+2)\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\tb.ResetTimer()\n\n\tvar smv StoreMsg\n\tfor i := 0; i < b.N; i++ {\n\t\t// Make sure not first seq.\n\t\t_, _, err := fs.LoadNextMsg(\"foo.*.baz\", true, 2, &smv)\n\t\trequire_Error(b, err, ErrStoreEOF)\n\t}\n}\n\nfunc Benchmark_FileStoreCreateConsumerStores(b *testing.B) {\n\tfor _, syncAlways := range []bool{true, false} {\n\t\tb.Run(fmt.Sprintf(\"%v\", syncAlways), func(b *testing.B) {\n\t\t\tfs, err := newFileStore(\n\t\t\t\tFileStoreConfig{StoreDir: b.TempDir(), SyncAlways: syncAlways},\n\t\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*.*\"}, Storage: FileStorage})\n\t\t\trequire_NoError(b, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\toconfig := ConsumerConfig{\n\t\t\t\tDeliverSubject: \"d\",\n\t\t\t\tFilterSubject: \"foo\",\n\t\t\t\tAckPolicy: AckAll,\n\t\t\t}\n\n\t\t\tb.ResetTimer()\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\toname := fmt.Sprintf(\"obs22_%d\", i)\n\t\t\t\tofs, err := fs.ConsumerStore(oname, time.Time{}, &oconfig)\n\t\t\t\trequire_NoError(b, err)\n\t\t\t\trequire_NoError(b, ofs.Stop())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Benchmark_FileStoreSubjectStateConsistencyOptimizationPerf(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: b.TempDir()},\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\t// Do R rounds of storing N messages.\n\t// MaxMsgsPer=1, so every unique subject that's placed only exists in the stream once.\n\t// If R=2, N=3 that means we'd place foo.0, foo.1, foo.2 in the first round, and the second\n\t// round we'd place foo.2, foo.1, foo.0, etc. This is intentional so that without any\n\t// optimizations we'd need to scan either 1 in the optimal case or N in the worst case.\n\t// Which is way more expensive than always knowing what the sequences are and it being O(1).\n\tr := max(2, b.N)\n\tn := 40_000\n\tb.ResetTimer()\n\tfor i := 0; i < r; i++ {\n\t\tfor j := 0; j < n; j++ {\n\t\t\td := j\n\t\t\tif i%2 == 0 {\n\t\t\t\td = n - j - 1\n\t\t\t}\n\t\t\tsubject := fmt.Sprintf(\"foo.%d\", d)\n\t\t\t_, _, err = fs.StoreMsg(subject, nil, nil, 0)\n\t\t\trequire_NoError(b, err)\n\t\t}\n\t}\n}\n\nfunc benchmarkFileStoreSyncDeletedFullBlocks(b *testing.B, msgSize int) {\n\tfs, _ := newFileStore(\n\t\tFileStoreConfig{\n\t\t\tStoreDir: b.TempDir(),\n\t\t\tBlockSize: defaultLargeBlockSize,\n\t\t},\n\t\tStreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tStorage: FileStorage,\n\t\t},\n\t)\n\tdefer fs.Stop()\n\n\tconst numBlocks = 50\n\tsubj, msg := \"foo\", make([]byte, msgSize)\n\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tb.StopTimer()\n\t\tfor len(fs.blks) < numBlocks {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\t\tdbs := DeleteBlocks{\n\t\t\t&DeleteRange{First: fs.state.FirstSeq, Num: fs.state.Msgs}}\n\n\t\tb.StartTimer()\n\t\tfs.SyncDeleted(dbs)\n\t}\n}\n\nfunc Benchmark_FileStoreSyncDeletedFullBlocks(b *testing.B) {\n\tsizes := []int{32, 64, 128, 256, 512, 1024}\n\tfor _, msgSize := range sizes {\n\t\tb.Run(fmt.Sprintf(\"MsgSize-%d\", msgSize),\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tbenchmarkFileStoreSyncDeletedFullBlocks(b, msgSize)\n\t\t\t})\n\t}\n}\n\nfunc benchmarkFileStoreSyncDeletedPartialBlocks(b *testing.B, msgSize int) {\n\tfs, _ := newFileStore(\n\t\tFileStoreConfig{\n\t\t\tStoreDir: b.TempDir(),\n\t\t\tBlockSize: defaultLargeBlockSize,\n\t\t},\n\t\tStreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tStorage: FileStorage,\n\t\t},\n\t)\n\tdefer fs.Stop()\n\n\tconst numBlocks = 100\n\tsubj, msg := \"foo\", make([]byte, msgSize)\n\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tb.StopTimer()\n\t\tif len(fs.blks) > 1 {\n\t\t\tfs.removeMsgsInRange(fs.state.FirstSeq, fs.blks[1].last.seq, true)\n\t\t}\n\t\tfor len(fs.blks) <= numBlocks {\n\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t}\n\n\t\tfirst := fs.state.FirstSeq + 1\n\t\tlast := fs.blks[1].last.seq\n\t\tdbs := DeleteBlocks{&DeleteRange{\n\t\t\tFirst: first,\n\t\t\tNum: last - first}}\n\n\t\tb.StartTimer()\n\t\tfs.SyncDeleted(dbs)\n\t}\n}\n\nfunc Benchmark_FileStoreSyncDeletedPartialBlocks(b *testing.B) {\n\tsizes := []int{16, 512}\n\tfor _, msgSize := range sizes {\n\t\tb.Run(fmt.Sprintf(\"MsgSize-%d\", msgSize),\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tbenchmarkFileStoreSyncDeletedPartialBlocks(b, msgSize)\n\t\t\t})\n\t}\n}\n\nfunc TestFileStoreWriteFullStateDetectCorruptState(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\t_, _, err = fs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Simulate a change in a message block not being reflected in the fs.\n\tmb := fs.selectMsgBlock(2)\n\tmb.mu.Lock()\n\tmb.msgs--\n\tmb.mu.Unlock()\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 10)\n\n\t// Make sure we detect the corrupt state and rebuild.\n\terr = fs.writeFullState()\n\trequire_Error(t, err, errCorruptState)\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 9)\n}\n\nfunc TestFileStoreRecoverFullStateDetectCorruptState(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\t_, _, err = fs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\terr = fs.writeFullState()\n\trequire_NoError(t, err)\n\n\tsfile := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\tbuf, err := os.ReadFile(sfile)\n\trequire_NoError(t, err)\n\t// Update to an incorrect message count.\n\tbinary.PutUvarint(buf[2:], 0)\n\t// Just append a corrected checksum to the end to make it pass the checks.\n\tfs.hh.Reset()\n\tfs.hh.Write(buf)\n\tbuf = fs.hh.Sum(buf)\n\terr = os.WriteFile(sfile, buf, defaultFilePerms)\n\trequire_NoError(t, err)\n\n\terr = fs.recoverFullState()\n\trequire_Error(t, err, errCorruptState)\n}\n\nfunc TestFileStoreResetConsumerToStreamState(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 30; i++ {\n\t\t_, _, err = fs.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\terr = fs.writeFullState()\n\trequire_NoError(t, err)\n\n\tobs, err := fs.ConsumerStore(\"c1\", time.Now(), &ConsumerConfig{\n\t\tDurable: \"c1\",\n\t\tFilterSubject: \"foo.*\",\n\t\tAckPolicy: AckNone,\n\t\tDeliverPolicy: DeliverAll,\n\t})\n\n\trequire_NoError(t, err)\n\tdefer obs.Stop()\n\n\tstate := &ConsumerState{}\n\tstate.Delivered = SequencePair{Consumer: 5, Stream: 5}\n\tstate.AckFloor = SequencePair{Consumer: 5, Stream: 5}\n\n\t// set to 5\n\terr = obs.Update(state)\n\trequire_NoError(t, err)\n\n\tcurrState, err := obs.State()\n\trequire_NoError(t, err)\n\n\tfsState := fs.State()\n\trequire_Equal(t, fsState.LastSeq, uint64(30))\n\trequire_Equal(t, fsState.FirstSeq, uint64(1))\n\trequire_Equal(t, currState.AckFloor, state.AckFloor)\n\trequire_Equal(t, currState.Delivered, state.Delivered)\n\trequire_Equal(t, len(currState.Redelivered), len(state.Redelivered))\n\trequire_Equal(t, len(currState.Pending), len(state.Pending))\n\n\tfs.mu.Lock()\n\tfs.state.FirstSeq = 0\n\tfs.state.LastSeq = 0\n\tfs.mu.Unlock()\n\n\t// set back to lower values\n\tnewState := &ConsumerState{\n\t\tDelivered: SequencePair{Consumer: 1, Stream: 4},\n\t\tAckFloor: SequencePair{Consumer: 1, Stream: 3},\n\t}\n\n\t// update should fail but force update should pass\n\terr = obs.Update(newState)\n\trequire_Error(t, err, ErrStoreOldUpdate)\n\n\terr = obs.ForceUpdate(newState)\n\trequire_NoError(t, err)\n\n\tcurrState, err = obs.State()\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, currState.AckFloor, newState.AckFloor)\n\trequire_Equal(t, currState.Delivered, newState.Delivered)\n\trequire_Equal(t, len(currState.Redelivered), len(newState.Redelivered))\n\trequire_Equal(t, len(currState.Pending), len(newState.Pending))\n}\n\nfunc TestFileStoreNumPendingMulti(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"ev.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\ttotalMsgs := 100_000\n\ttotalSubjects := 10_000\n\tnumFiltered := 5000\n\tstartSeq := uint64(5_000 + rand.Intn(90_000))\n\n\tsubjects := make([]string, 0, totalSubjects)\n\tfor i := 0; i < totalSubjects; i++ {\n\t\tsubjects = append(subjects, fmt.Sprintf(\"ev.%s\", nuid.Next()))\n\t}\n\n\t// Put in 100k msgs with random subjects.\n\tmsg := bytes.Repeat([]byte(\"ZZZ\"), 333)\n\tfor i := 0; i < totalMsgs; i++ {\n\t\t_, _, err = fs.StoreMsg(subjects[rand.Intn(totalSubjects)], nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now we want to do a calculate NumPendingMulti.\n\tfilters := gsl.NewSublist[struct{}]()\n\tfor filters.Count() < uint32(numFiltered) {\n\t\tfilter := subjects[rand.Intn(totalSubjects)]\n\t\tif !filters.HasInterest(filter) {\n\t\t\tfilters.Insert(filter, struct{}{})\n\t\t}\n\t}\n\n\t// Use new function.\n\ttotal, _, err := fs.NumPendingMulti(startSeq, filters, false)\n\trequire_NoError(t, err)\n\n\t// Check our results.\n\tvar checkTotal uint64\n\tvar smv StoreMsg\n\tfor seq := startSeq; seq <= uint64(totalMsgs); seq++ {\n\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\trequire_NoError(t, err)\n\t\tif filters.HasInterest(sm.subj) {\n\t\t\tcheckTotal++\n\t\t}\n\t}\n\trequire_Equal(t, total, checkTotal)\n}\n\nfunc TestFileStoreMessageTTL(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tttl := int64(1) // 1 second\n\n\tfor i := 1; i <= 10; i++ {\n\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, ttl)\n\t\trequire_NoError(t, err)\n\t}\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 10)\n\n\ttime.Sleep(time.Second * 2)\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 11)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 0)\n}\n\nfunc TestFileStoreMessageTTLRestart(t *testing.T) {\n\tdir := t.TempDir()\n\n\tt.Run(\"BeforeRestart\", func(t *testing.T) {\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tttl := int64(1) // 1 second\n\n\t\tfor i := 1; i <= 10; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, ttl)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 10)\n\t})\n\n\tt.Run(\"AfterRestart\", func(t *testing.T) {\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 10)\n\n\t\ttime.Sleep(time.Second * 2)\n\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 11)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 0)\n\t})\n}\n\nfunc TestFileStoreMessageTTLRecovered(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tt.Run(\"BeforeRestart\", func(t *testing.T) {\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tttl := int64(1) // 1 second\n\n\t\t// Make the first message with no ttl.\n\t\t// This exposes a bug we had in recovery.\n\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 9; i++ {\n\t\t\t// When the timed hash wheel state is deleted, the only way we can recover\n\t\t\t// the TTL is to look at the original message header, therefore the TTL\n\t\t\t// must be in the headers for this test to work.\n\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0\\r\\n%s: %d\\r\\n\", JSMessageTTL, ttl)\n\t\t\t_, _, err = fs.StoreMsg(\"test\", hdr, nil, ttl)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 10)\n\t})\n\n\tt.Run(\"AfterRestart\", func(t *testing.T) {\n\t\t// Delete the timed hash wheel state so that we are forced to do a linear scan\n\t\t// of message blocks containing TTL'd messages.\n\t\tfn := filepath.Join(dir, msgDir, ttlStreamStateFile)\n\t\trequire_NoError(t, os.RemoveAll(fn))\n\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tfs.mu.RLock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.RUnlock()\n\t\tmb.mu.RLock()\n\t\tttls := mb.ttls\n\t\tmb.mu.RUnlock()\n\n\t\trequire_Equal(t, ttls, 9)\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 10)\n\n\t\ttime.Sleep(time.Second * 2)\n\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 10)\n\t\trequire_Equal(t, ss.Msgs, 1)\n\t})\n}\n\nfunc TestFileStoreMessageTTLRecoveredSingleMessageWithoutStreamState(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tt.Run(\"BeforeRestart\", func(t *testing.T) {\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tttl := int64(1) // 1 second\n\t\thdr := fmt.Appendf(nil, \"NATS/1.0\\r\\n%s: %d\\r\\n\", JSMessageTTL, ttl)\n\t\t_, _, err = fs.StoreMsg(\"test\", hdr, nil, ttl)\n\t\trequire_NoError(t, err)\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t\trequire_Equal(t, ss.Msgs, 1)\n\t})\n\n\tt.Run(\"AfterRestart\", func(t *testing.T) {\n\t\t// Delete the stream state file so that we need to rebuild.\n\t\tfn := filepath.Join(dir, msgDir, streamStreamStateFile)\n\t\trequire_NoError(t, os.RemoveAll(fn))\n\t\t// Delete the timed hash wheel state so that we are forced to do a linear scan\n\t\t// of message blocks containing TTL'd messages.\n\t\tfn = filepath.Join(dir, msgDir, ttlStreamStateFile)\n\t\trequire_NoError(t, os.RemoveAll(fn))\n\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t\trequire_Equal(t, ss.Msgs, 1)\n\n\t\ttime.Sleep(time.Second * 2)\n\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 2)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t\trequire_Equal(t, ss.Msgs, 0)\n\t})\n}\n\nfunc TestFileStoreMessageTTLWriteTombstone(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tttl := int64(1)\n\n\t// When the timed hash wheel state is deleted, the only way we can recover\n\t// the TTL is to look at the original message header, therefore the TTL\n\t// must be in the headers for this test to work.\n\thdr := fmt.Appendf(nil, \"NATS/1.0\\r\\n%s: %d\\r\\n\", JSMessageTTL, ttl)\n\t_, _, err = fs.StoreMsg(\"test\", hdr, nil, ttl)\n\trequire_NoError(t, err)\n\n\t// Publish another message, but without TTL.\n\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\trequire_NoError(t, err)\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 2)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 2)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif fs.State().FirstSeq == 1 {\n\t\t\treturn errors.New(\"message not expired yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 2)\n\trequire_Equal(t, ss.LastSeq, 2)\n\n\tfs.Stop()\n\n\t// Delete the stream state file so that we need to rebuild.\n\t// Should have written a tombstone so we can properly recover.\n\tfn := filepath.Join(dir, msgDir, streamStreamStateFile)\n\trequire_NoError(t, os.RemoveAll(fn))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 2)\n\trequire_Equal(t, ss.LastSeq, 2)\n}\n\nfunc TestFileStoreMessageTTLRecoveredOffByOne(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tts := time.Now().UnixNano()\n\tttl := int64(120) // 2 minutes\n\texpires := time.Duration(ts) + (time.Second * time.Duration(ttl))\n\n\t// When the timed hash wheel state is deleted, the only way we can recover\n\t// the TTL is to look at the original message header, therefore the TTL\n\t// must be in the headers for this test to work.\n\thdr := fmt.Appendf(nil, \"NATS/1.0\\r\\n%s: %d\\r\\n\", JSMessageTTL, ttl)\n\trequire_NoError(t, fs.StoreRawMsg(\"test\", hdr, nil, 1, ts, ttl, false))\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 1)\n\n\tfs.mu.Lock()\n\tttlc := fs.ttls.Count()\n\t// Adding to the THW is idempotent, so we sneakily remove it here\n\t// so we can check it doesn't get re-added during restart.\n\terr = fs.ttls.Remove(1, int64(expires))\n\tfs.mu.Unlock()\n\trequire_Equal(t, ttlc, 1)\n\trequire_NoError(t, err)\n\n\tfs.Stop()\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 1)\n\n\tfs.mu.Lock()\n\tttlc = fs.ttls.Count()\n\tfs.mu.Unlock()\n\trequire_Equal(t, ttlc, 0)\n}\n\nfunc TestFileStoreDontSpamCompactWhenMostlyTombstones(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir(), BlockSize: defaultMediumBlockSize},\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Store a bunch of messages, ensuring we get enough blocks.\n\tmsg := bytes.Repeat([]byte(\"X\"), 100)\n\ttotalBlksAfterStore := 6\n\texpectedLseq := uint64(31_536*(totalBlksAfterStore-1) + 2)\n\tfor seq := uint64(1); seq <= expectedLseq; seq++ {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Confirms we have the required amount of blocks.\n\tfs.mu.RLock()\n\tlenBlks := len(fs.blks)\n\tlmb := fs.lmb\n\tfs.mu.RUnlock()\n\trequire_Len(t, lenBlks, totalBlksAfterStore)\n\n\t// Ensure the last block contains the last two messages.\n\tlmb.mu.RLock()\n\tfseq, lseq := lmb.first.seq, lmb.last.seq\n\tlmb.mu.RUnlock()\n\trequire_Equal(t, fseq, expectedLseq-1)\n\trequire_Equal(t, lseq, expectedLseq)\n\n\t// Remove all messages before the last block, will fill up last block with tombstones.\n\tfor seq := uint64(1); seq < fseq; seq++ {\n\t\tremoved, err := fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\t}\n\n\t// The last block will be filled with so many tombstones that\n\t// a new message block will be created to store the rest.\n\tfs.mu.RLock()\n\tlenBlks = len(fs.blks)\n\tfs.mu.RUnlock()\n\trequire_Len(t, lenBlks, 2)\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\tfmb := fs.blks[0]\n\tfmb.mu.Lock()\n\tdefer fmb.mu.Unlock()\n\n\t// We don't call fs.RemoveMsg as that would call into compact.\n\t// Instead, we mark as deleted and check compaction ourselves.\n\tfmb.dmap.Insert(expectedLseq)\n\tfmb.bytes /= 2\n\n\t// This message block takes up ~4MB but contains only one ~100 bytes message, and the rest is all tombstones.\n\t// We should allow trying to compact.\n\trequire_Equal(t, fmb.bytes, 133)\n\trequire_Equal(t, fmb.cbytes, 0)\n\trequire_True(t, fmb.shouldCompactInline())\n\n\t// Compact will be successful, but since it doesn't clean up tombstones it will be ineffective.\n\trequire_NoError(t, fmb.compact())\n\n\t// We should not allow compacting again as we're not removing tombstones inline.\n\t// Otherwise, we would spam compaction.\n\trequire_False(t, fmb.shouldCompactInline())\n\n\t// Just checking fmb.cbytes is tracking the previous value of fmb.bytes,\n\t// so it can block compaction until enough data has been removed.\n\trequire_Equal(t, fmb.bytes, 133)\n\trequire_Equal(t, fmb.cbytes, fmb.bytes)\n\n\t// Simulate having removed sufficient bytes and being allowed to compact again.\n\tfmb.bytes /= 2\n\trequire_True(t, fmb.shouldCompactInline())\n}\n\nfunc TestFileStoreSubjectDeleteMarkers(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{\n\t\t\tName: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage,\n\t\t\tMaxAge: time.Second, AllowMsgTTL: true,\n\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Capture subject delete marker proposals.\n\tch := make(chan *inMsg, 1)\n\tfs.rmcb = func(seq uint64) {\n\t\t_, err := fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t}\n\tfs.pmsgcb = func(im *inMsg) {\n\t\tch <- im\n\t}\n\n\t// Store three messages that will expire because of MaxAge.\n\tfor i := 0; i < 3; i++ {\n\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for MaxAge to pass.\n\ttime.Sleep(time.Second + time.Millisecond*500)\n\n\t// We should have placed a subject delete marker.\n\tim := require_ChanRead(t, ch, time.Second*5)\n\trequire_Equal(t, bytesToString(getHeader(JSMarkerReason, im.hdr)), JSMarkerReasonMaxAge)\n\trequire_Equal(t, bytesToString(getHeader(JSMessageTTL, im.hdr)), \"1s\")\n}\n\nfunc TestFileStoreStoreRawMessageThrowsPermissionErrorIfFSModeReadOnly(t *testing.T) {\n\t// Test fails in Buildkite environment. Skip it.\n\tskipIfBuildkite(t)\n\n\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"ev.1\"}, Storage: FileStorage}\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir(), BlockSize: 1024}, cfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tREADONLY_MODE := os.FileMode(0o555)\n\tORIGINAL_FILE_MODE, err := os.Stat(fs.fcfg.StoreDir)\n\trequire_NoError(t, err)\n\trequire_NoError(t, changeDirectoryPermission(fs.fcfg.StoreDir, READONLY_MODE))\n\tdefer func() {\n\t\trequire_NoError(t, changeDirectoryPermission(fs.fcfg.StoreDir, ORIGINAL_FILE_MODE.Mode()))\n\t}()\n\n\ttotalMsgs := 10000\n\tmsg := bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor i := 0; i < totalMsgs; i++ {\n\t\tif _, _, err = fs.StoreMsg(\"ev.1\", nil, msg, 0); err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_Error(t, err, os.ErrPermission)\n}\n\nfunc TestFileStoreWriteFullStateThrowsPermissionErrorIfFSModeReadOnly(t *testing.T) {\n\t// Test fails in Buildkite environment. Skip it.\n\tskipIfBuildkite(t)\n\n\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"ev.1\"}, Storage: FileStorage}\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tREADONLY_MODE := os.FileMode(0o555)\n\tORIGINAL_FILE_MODE, err := os.Stat(fs.fcfg.StoreDir)\n\trequire_NoError(t, err)\n\n\ttotalMsgs := 10000\n\tmsg := bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor i := 0; i < totalMsgs; i++ {\n\t\tif _, _, err := fs.StoreMsg(\"ev.1\", nil, msg, 0); err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\trequire_NoError(t, changeDirectoryPermission(fs.fcfg.StoreDir, READONLY_MODE))\n\trequire_Error(t, fs.writeFullState(), os.ErrPermission)\n\trequire_NoError(t, changeDirectoryPermission(fs.fcfg.StoreDir, ORIGINAL_FILE_MODE.Mode()))\n}\n\nfunc changeDirectoryPermission(directory string, mode fs.FileMode) error {\n\terr := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error accessing path %q: %w\", path, err)\n\t\t}\n\n\t\t// Check if the path is a directory or file and set permissions accordingly\n\t\tif info.IsDir() {\n\t\t\terr = os.Chmod(path, mode)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error changing directory permissions for %q: %w\", path, err)\n\t\t\t}\n\t\t} else {\n\t\t\terr = os.Chmod(path, mode)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error changing file permissions for %q: %w\", path, err)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn err\n}\n\nfunc TestFileStoreLeftoverSkipMsgInDmap(t *testing.T) {\n\tstoreDir := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: storeDir},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test.*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tgetLmbState := func(fs *fileStore) (uint64, uint64, int) {\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.RLock()\n\t\tfseq := atomic.LoadUint64(&lmb.first.seq)\n\t\tlseq := atomic.LoadUint64(&lmb.last.seq)\n\t\tdmaps := lmb.dmap.Size()\n\t\tlmb.mu.RUnlock()\n\t\treturn fseq, lseq, dmaps\n\t}\n\n\t// Only skip a message.\n\tfs.SkipMsg(0)\n\n\t// Confirm state.\n\tstate := fs.State()\n\trequire_Equal(t, state.FirstSeq, 2)\n\trequire_Equal(t, state.LastSeq, 1)\n\trequire_Equal(t, state.NumDeleted, 0)\n\tfseq, lseq, dmaps := getLmbState(fs)\n\trequire_Equal(t, fseq, 2)\n\trequire_Equal(t, lseq, 1)\n\trequire_Len(t, dmaps, 0)\n\n\t// Stop without writing index.db so we recover based on just the blk file.\n\trequire_NoError(t, fs.stop(false, false))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: storeDir},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test.*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Confirm the skipped message is not included in the deletes.\n\tstate = fs.State()\n\trequire_Equal(t, state.FirstSeq, 2)\n\trequire_Equal(t, state.LastSeq, 1)\n\trequire_Equal(t, state.NumDeleted, 0)\n\tfseq, lseq, dmaps = getLmbState(fs)\n\trequire_Equal(t, fseq, 2)\n\trequire_Equal(t, lseq, 1)\n\trequire_Len(t, dmaps, 0)\n}\n\nfunc TestFileStoreRecoverOnlyBlkFiles(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Confirm state as baseline.\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 1)\n\n\t\t// Restart should equal state.\n\t\trequire_NoError(t, fs.Stop())\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\t// Stream state should exist.\n\t\t_, err = os.Stat(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\t\trequire_NoError(t, err)\n\n\t\t// Stop and write some random files, but containing \".blk\", should be ignored.\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.WriteFile(filepath.Join(fs.fcfg.StoreDir, msgDir, \"10.blk.random\"), nil, defaultFilePerms))\n\t\trequire_NoError(t, os.WriteFile(filepath.Join(fs.fcfg.StoreDir, msgDir, fmt.Sprintf(\"10.blk.%s\", blkTmpSuffix)), nil, defaultFilePerms))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// The random files would previously result in stream state to be deleted.\n\t\t_, err = os.Stat(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\t\trequire_NoError(t, err)\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\t// Stop and remove stream state file.\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t// Recovering based on blocks should also ignore the random files.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRecoverAfterRemoveOperation(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\taction func(fs *fileStore)\n\t\tvalidate func(state StreamState)\n\t}{\n\t\t{\n\t\t\ttitle: \"None\",\n\t\t\taction: func(fs *fileStore) {},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 4)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"RemoveMsg\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\tremoved, err := fs.RemoveMsg(1)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 3)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 2)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"EraseMsg\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\terased, err := fs.EraseMsg(1)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, erased)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 3)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 2)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"Purge\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\tpurged, err := fs.Purge()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, purged, 4)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 0)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 5)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"PurgeEx-0\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\tpurged, err := fs.PurgeEx(\"foo.0\", 0, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, purged, 2)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 2)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t\trequire_Equal(t, state.NumDeleted, 1)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"PurgeEx-1\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\tpurged, err := fs.PurgeEx(\"foo.1\", 0, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, purged, 2)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t\trequire_Equal(t, state.NumDeleted, 2)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"Compact\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\tpurged, err := fs.Compact(3)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, purged, 2)\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 3)\n\t\t\t\trequire_Equal(t, state.LastSeq, 4)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"Truncate\",\n\t\t\taction: func(fs *fileStore) {\n\t\t\t\trequire_NoError(t, fs.Truncate(2))\n\t\t\t},\n\t\t\tvalidate: func(state StreamState) {\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, state.LastSeq, 2)\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}\n\t\t\t\tcreated := time.Now()\n\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\t\tsubject := fmt.Sprintf(\"foo.%d\", i%2)\n\t\t\t\t\t_, _, err = fs.StoreMsg(subject, nil, nil, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\n\t\t\t\ttest.action(fs)\n\n\t\t\t\t// Confirm state as baseline.\n\t\t\t\tbefore := fs.State()\n\t\t\t\ttest.validate(before)\n\n\t\t\t\t// Restart should equal state.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\n\t\t\t\t// Stop and remove stream state file.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t\t\t// Recovering based on blocks should result in the same state.\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\n\t\t\t\t// Rebuilding state must also result in the same state.\n\t\t\t\tfs.rebuildState(nil)\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestFileStoreRecoverAfterCompact(t *testing.T) {\n\t/*\n\t\tfs.Compact may rewrite the .blk file if it's large enough. In which case we don't\n\t\tneed to write tombstones. But if the rewrite isn't done, tombstones must be placed\n\t\tto ensure we can properly recover without the index.db file.\n\t*/\n\tfor _, test := range []struct {\n\t\tpayloadSize int\n\t\tusesTombstones bool\n\t}{\n\t\t{payloadSize: 1024, usesTombstones: true},\n\t\t{payloadSize: 1024 * 1024, usesTombstones: false},\n\t} {\n\t\tt.Run(strconv.Itoa(test.payloadSize), func(t *testing.T) {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\t\t\tcreated := time.Now()\n\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tsubject := \"foo\"\n\t\t\t\tpayload := make([]byte, test.payloadSize)\n\t\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\t\t_, _, err = fs.StoreMsg(subject, nil, payload, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\n\t\t\t\t// Confirm state before compacting.\n\t\t\t\tbefore := fs.State()\n\t\t\t\trequire_Equal(t, before.Msgs, 4)\n\t\t\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, before.LastSeq, 4)\n\t\t\t\trequire_Equal(t, before.Bytes, uint64(emptyRecordLen+len(subject)+test.payloadSize)*4)\n\n\t\t\t\tfs.mu.RLock()\n\t\t\t\tlmb := fs.lmb\n\t\t\t\tfs.mu.RUnlock()\n\n\t\t\t\t// Underlying message block should report the same.\n\t\t\t\tif fcfg.Cipher == NoCipher && fcfg.Compression == NoCompression {\n\t\t\t\t\tlmb.mu.RLock()\n\t\t\t\t\tbytes, rbytes := lmb.bytes, lmb.rbytes\n\t\t\t\t\tlmb.mu.RUnlock()\n\t\t\t\t\tsize := uint64(emptyRecordLen+len(subject)+test.payloadSize) * 4\n\t\t\t\t\trequire_Equal(t, bytes, size)\n\t\t\t\t\trequire_Equal(t, rbytes, size)\n\t\t\t\t}\n\n\t\t\t\t// Now compact.\n\t\t\t\tpurged, err := fs.Compact(4)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, purged, 3)\n\n\t\t\t\t// Confirm state after compacting.\n\t\t\t\t// Bytes should reflect only having a single message left.\n\t\t\t\tbefore = fs.State()\n\t\t\t\trequire_Equal(t, before.Msgs, 1)\n\t\t\t\trequire_Equal(t, before.FirstSeq, 4)\n\t\t\t\trequire_Equal(t, before.LastSeq, 4)\n\t\t\t\trequire_Equal(t, before.Bytes, uint64(emptyRecordLen+len(subject)+test.payloadSize))\n\n\t\t\t\t// Underlying message block should report the same.\n\t\t\t\t// Unless the compact didn't rewrite the block but used tombstones,\n\t\t\t\t// in which case we expect the raw bytes to include them.\n\t\t\t\tif fcfg.Cipher == NoCipher && fcfg.Compression == NoCompression {\n\t\t\t\t\tlmb.mu.RLock()\n\t\t\t\t\tbytes, rbytes := lmb.bytes, lmb.rbytes\n\t\t\t\t\tlmb.mu.RUnlock()\n\t\t\t\t\tsize := uint64(emptyRecordLen + len(subject) + test.payloadSize)\n\t\t\t\t\trequire_Equal(t, bytes, size)\n\t\t\t\t\tif test.usesTombstones {\n\t\t\t\t\t\t// 4 messages, 3 tombstones\n\t\t\t\t\t\tsize = uint64(emptyRecordLen+len(subject)+test.payloadSize)*4 + uint64(emptyRecordLen)*3\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, rbytes, size)\n\t\t\t\t}\n\n\t\t\t\t// Restart should equal state.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\n\t\t\t\t// Stop and remove stream state file.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t\t\t// Recovering based on blocks should result in the same state.\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\n\t\t\t\t// Rebuilding state must also result in the same state.\n\t\t\t\tfs.rebuildState(nil)\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestFileStoreRecoverWithEmptyMessageBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t// 4 messages with subject 'foo' and no payload.\n\t\tfcfg.BlockSize = 33 * 4\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// First message block contains 4 messages.\n\t\tfor i := 0; i < 4; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tlblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, lblks, 1)\n\n\t\t// Second (empty) message block only contains 2 tombstones.\n\t\tfor i := uint64(1); i <= 2; i++ {\n\t\t\tremoved, err := fs.RemoveMsg(i)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tlblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, lblks, 2)\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 2)\n\t\trequire_Equal(t, before.FirstSeq, 3)\n\t\trequire_Equal(t, before.LastSeq, 4)\n\n\t\t// Restart should equal state.\n\t\trequire_NoError(t, fs.Stop())\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t}\n\n\t\t// Stop and remove stream state file.\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t// Recovering based on blocks should result in the same state.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t}\n\n\t\t// Rebuilding state must also result in the same state.\n\t\tfs.rebuildState(nil)\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreRemoveMsgBlockFirst(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\trequire_NoError(t, err)\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 1)\n\n\tfs.Stop()\n\n\tfor _, f := range []string{streamStreamStateFile, \"1.blk\"} {\n\t\tfn := filepath.Join(dir, msgDir, f)\n\t\trequire_NoError(t, os.RemoveAll(fn))\n\t}\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// If the block is removed first, we have nothing to recover. So starting out empty would be expected.\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 0)\n\trequire_Equal(t, ss.FirstSeq, 0)\n\trequire_Equal(t, ss.LastSeq, 0)\n}\n\nfunc TestFileStoreRemoveMsgBlockLast(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\trequire_NoError(t, err)\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 1)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 1)\n\n\t// Copy first block so we can put it back later.\n\tofn := filepath.Join(dir, msgDir, \"1.blk\")\n\tnfn := filepath.Join(dir, msgDir, \"1.blk.cp\")\n\trequire_NoError(t, os.Rename(ofn, nfn))\n\n\t// Removing the last message will result in '2.blk' to be created, and '1.blk' to be removed.\n\t_, err = fs.RemoveMsg(1)\n\trequire_NoError(t, err)\n\t_, err = os.Stat(filepath.Join(dir, msgDir, \"2.blk\"))\n\trequire_NoError(t, err)\n\t_, err = os.Stat(ofn)\n\trequire_True(t, os.IsNotExist(err))\n\n\tfs.Stop()\n\n\t// Remove index.db so we need to recover based on blocks.\n\tfn := filepath.Join(dir, msgDir, streamStreamStateFile)\n\trequire_NoError(t, os.RemoveAll(fn))\n\n\t// Put back '1.blk' file to simulate being hard killed right\n\t// after creating '2.blk' but before cleaning up '1.blk'.\n\trequire_NoError(t, os.Rename(nfn, ofn))\n\t_, err = os.Stat(ofn)\n\trequire_False(t, os.IsNotExist(err))\n\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: FileStorage, AllowMsgTTL: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Should recognize correct state, and remove '1.blk'.\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 0)\n\trequire_Equal(t, ss.FirstSeq, 2)\n\trequire_Equal(t, ss.LastSeq, 1)\n\t_, err = os.Stat(ofn)\n\trequire_True(t, os.IsNotExist(err))\n}\n\nfunc TestFileStoreAllLastSeqs(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*.*\"}, MaxMsgsPer: 50, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubjs := []string{\"foo.foo\", \"foo.bar\", \"foo.baz\", \"bar.foo\", \"bar.bar\", \"bar.baz\"}\n\tmsg := []byte(\"abc\")\n\n\tfor i := 0; i < 100_000; i++ {\n\t\tsubj := subjs[rand.Intn(len(subjs))]\n\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\texpected := make([]uint64, 0, len(subjs))\n\tvar smv StoreMsg\n\tfor _, subj := range subjs {\n\t\tsm, err := fs.LoadLastMsg(subj, &smv)\n\t\trequire_NoError(t, err)\n\t\texpected = append(expected, sm.seq)\n\t}\n\tslices.Sort(expected)\n\n\tseqs, err := fs.AllLastSeqs()\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(seqs, expected))\n}\n\nfunc TestFileStoreRecoverDoesNotResetStreamState(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"ev.1\"}, Storage: FileStorage, MaxAge: 2 * time.Second, Retention: WorkQueuePolicy}\n\t\tcreated := time.Now()\n\t\tfcfg.BlockSize = 1024\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\t\ttoStore := 500\n\t\tfor i := 0; i < toStore; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfs.mu.RLock()\n\t\tblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, blks > 1)\n\n\t\t// Simulate a consumer consuming all messages, but this test\n\t\t// expires all messages from the stream instead.\n\t\ttime.Sleep(2500 * time.Millisecond)\n\n\t\t// Capture the state before stopping the store.\n\t\tfs.mu.RLock()\n\t\tblks = len(fs.blks)\n\t\tvar mfn string\n\t\tif blks > 0 {\n\t\t\tmfn = fs.blks[0].mfn\n\t\t}\n\t\tfs.mu.RUnlock()\n\n\t\t// Stream state should exist after shutting down.\n\t\trequire_NoError(t, fs.Stop())\n\t\t_, err = os.Stat(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\t\trequire_NoError(t, err)\n\n\t\t// A single block should have remained, but remove it so we need to restore from the index file.\n\t\t// The first/last sequences should be preserved and restored from the index.\n\t\trequire_Len(t, blks, 1)\n\t\trequire_NoError(t, os.Remove(mfn))\n\n\t\tfs, err = newFileStoreWithCreated(fs.fcfg, cfg, time.Now(), prf(&fs.fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tfs.mu.RLock()\n\t\tdefer fs.mu.RUnlock()\n\t\trequire_True(t, fs.state.FirstSeq|fs.state.LastSeq != 0)\n\t})\n}\n\nfunc TestFileStoreAccessTimeSpinUp(t *testing.T) {\n\t// In case running lots of tests.\n\ttime.Sleep(time.Second)\n\tngr := runtime.NumGoroutine()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tat := ats.AccessTime()\n\trequire_True(t, at != 0)\n\n\t// Now check we also cleanup.\n\tfs.Stop()\n\ttime.Sleep(2 * ats.TickInterval)\n\tngra := runtime.NumGoroutine()\n\trequire_Equal(t, ngr, ngra)\n}\n\nfunc TestFileStoreUpdateConfigTTLState(t *testing.T) {\n\tcfg := StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\">\"},\n\t\tStorage: FileStorage,\n\t}\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\trequire_Equal(t, fs.ttls, nil)\n\n\tcfg.AllowMsgTTL = true\n\trequire_NoError(t, fs.UpdateConfig(&cfg))\n\trequire_NotEqual(t, fs.ttls, nil)\n\n\tcfg.AllowMsgTTL = false\n\trequire_NoError(t, fs.UpdateConfig(&cfg))\n\trequire_Equal(t, fs.ttls, nil)\n}\n\nfunc TestFileStoreSubjectForSeq(t *testing.T) {\n\tcfg := StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tStorage: FileStorage,\n\t}\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tseq, _, err := fs.StoreMsg(\"foo.bar\", nil, nil, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, seq, 1)\n\n\t_, err = fs.SubjectForSeq(0)\n\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\tsubj, err := fs.SubjectForSeq(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, subj, \"foo.bar\")\n\n\t_, err = fs.SubjectForSeq(2)\n\trequire_Error(t, err, ErrStoreMsgNotFound)\n}\n\nfunc BenchmarkFileStoreSubjectAccesses(b *testing.B) {\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: b.TempDir()}, StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tStorage: FileStorage,\n\t})\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\tseq, _, err := fs.StoreMsg(\"foo.bar\", nil, []byte{1, 2, 3, 4, 5}, 0)\n\trequire_NoError(b, err)\n\trequire_Equal(b, seq, 1)\n\n\tb.Run(\"SubjectForSeq\", func(b *testing.B) {\n\t\tb.ReportAllocs()\n\t\tfor range b.N {\n\t\t\tsubj, err := fs.SubjectForSeq(1)\n\t\t\trequire_NoError(b, err)\n\t\t\trequire_Equal(b, subj, \"foo.bar\")\n\t\t}\n\t})\n\n\tb.Run(\"LoadMsg\", func(b *testing.B) {\n\t\tb.ReportAllocs()\n\t\tfor range b.N {\n\t\t\t// smv is deliberately inside the loop here because that's\n\t\t\t// effectively what is happening with needAck.\n\t\t\tvar smv StoreMsg\n\t\t\tsm, err := fs.LoadMsg(1, &smv)\n\t\t\trequire_NoError(b, err)\n\t\t\trequire_Equal(b, sm.subj, \"foo.bar\")\n\t\t}\n\t})\n}\n\nfunc TestFileStoreFirstMatchingMultiExpiry(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.>\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"A\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"B\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, _, err = fs.StoreMsg(\"foo.foo\", nil, []byte(\"C\"), 0)\n\t\trequire_NoError(t, err)\n\n\t\tfs.mu.RLock()\n\t\tmb := fs.lmb\n\t\tmb.expireCacheLocked()\n\t\tfs.mu.RUnlock()\n\n\t\tsl := gsl.NewSublist[struct{}]()\n\t\tsl.Insert(\"foo.foo\", struct{}{})\n\n\t\t_, didLoad, err := mb.firstMatchingMulti(sl, 1, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_False(t, didLoad)\n\n\t\t_, didLoad, err = mb.firstMatchingMulti(sl, 2, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_False(t, didLoad)\n\n\t\t_, didLoad, err = mb.firstMatchingMulti(sl, 3, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, didLoad) // last message, should expire\n\t})\n}\n\nfunc TestFileStoreNoPanicOnRecoverTTLWithCorruptBlocks(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, AllowMsgTTL: true}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\thdr := genHeader(nil, JSMessageTTL, \"1\")\n\t\tfor i := range 3 {\n\t\t\tif i > 0 {\n\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", hdr, []byte(\"A\"), 1)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.Lock()\n\t\tif blks := len(fs.blks); blks != 3 {\n\t\t\tfs.mu.Unlock()\n\t\t\tt.Fatalf(\"Expected 3 blocks, got %d\", blks)\n\t\t}\n\n\t\t// Manually corrupt the blocks by removing the second and changing the\n\t\t// sequence range for the last to that of the first.\n\t\tfmb := fs.blks[0]\n\t\tsmb := fs.blks[1]\n\t\tlmb := fs.lmb\n\t\tfseq, lseq := atomic.LoadUint64(&fmb.first.seq), atomic.LoadUint64(&fmb.last.seq)\n\t\tsmb.mu.Lock()\n\t\tfs.removeMsgBlock(smb)\n\t\tsmb.mu.Unlock()\n\t\tfs.mu.Unlock()\n\t\tatomic.StoreUint64(&lmb.first.seq, fseq)\n\t\tatomic.StoreUint64(&lmb.last.seq, lseq)\n\n\t\trequire_NoError(t, fs.recoverTTLState())\n\t})\n}\n\nfunc TestFileStoreAsyncTruncate(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 8192\n\t\tfcfg.AsyncFlush = true\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\trequire_NotNil(t, lmb)\n\n\t\t// Wait for flusher to be ready.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tlmb.mu.RLock()\n\t\t\tdefer lmb.mu.RUnlock()\n\t\t\tif !lmb.flusher {\n\t\t\t\treturn errors.New(\"flusher not active\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Now shutdown flusher and wait for it to be closed.\n\t\tlmb.mu.Lock()\n\t\tif lmb.qch != nil {\n\t\t\tclose(lmb.qch)\n\t\t\tlmb.qch = nil\n\t\t}\n\t\tlmb.mu.Unlock()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tlmb.mu.RLock()\n\t\t\tdefer lmb.mu.RUnlock()\n\t\t\tif lmb.flusher {\n\t\t\t\treturn errors.New(\"flusher still active\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Write some messages, none of them will have been flushed asynchronously.\n\t\tsubj, msg := \"foo\", make([]byte, 100)\n\t\tfor i := uint64(1); i <= 2; i++ {\n\t\t\tseq, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, seq, i)\n\t\t}\n\t\t// Truncate needs to flush if the data was not yet flushed asynchronously.\n\t\trequire_NoError(t, fs.Truncate(1))\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 1)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 1)\n\n\t\tfs.mu.RLock()\n\t\tfor _, mb := range fs.blks {\n\t\t\tif mb.pendingWriteSize() > 0 {\n\t\t\t\tfs.mu.RUnlock()\n\t\t\t\tt.Fatalf(\"Message block %d still has pending writes\", mb.index)\n\t\t\t}\n\t\t}\n\t\tfs.mu.RUnlock()\n\t})\n}\n\nfunc TestFileStoreAsyncFlushOnSkipMsgs(t *testing.T) {\n\tfor _, noFlushLoop := range []bool{false, true} {\n\t\tt.Run(fmt.Sprintf(\"NoFlushLoop=%v\", noFlushLoop), func(t *testing.T) {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tfcfg.BlockSize = 8192\n\t\t\t\tfcfg.AsyncFlush = true\n\n\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tfs.mu.RLock()\n\t\t\t\tfmb := fs.lmb\n\t\t\t\tfs.mu.RUnlock()\n\t\t\t\trequire_NotNil(t, fmb)\n\n\t\t\t\tif noFlushLoop {\n\t\t\t\t\t// Wait for flusher to be ready.\n\t\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\t\tfmb.mu.RLock()\n\t\t\t\t\t\tdefer fmb.mu.RUnlock()\n\t\t\t\t\t\tif !fmb.flusher {\n\t\t\t\t\t\t\treturn errors.New(\"flusher not active\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t\t// Now shutdown flusher and wait for it to be closed.\n\t\t\t\t\tfmb.mu.Lock()\n\t\t\t\t\tif fmb.qch != nil {\n\t\t\t\t\t\tclose(fmb.qch)\n\t\t\t\t\t\tfmb.qch = nil\n\t\t\t\t\t}\n\t\t\t\t\tfmb.mu.Unlock()\n\t\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\t\tfmb.mu.RLock()\n\t\t\t\t\t\tdefer fmb.mu.RUnlock()\n\t\t\t\t\t\tif fmb.flusher {\n\t\t\t\t\t\t\treturn errors.New(\"flusher still active\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t}\n\n\t\t\t\t// Confirm no pending writes.\n\t\t\t\trequire_Equal(t, fmb.pendingWriteSize(), 0)\n\n\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tif noFlushLoop {\n\t\t\t\t\t// Confirm above write is pending.\n\t\t\t\t\trequire_Equal(t, fmb.pendingWriteSize(), 33)\n\t\t\t\t}\n\n\t\t\t\trequire_NoError(t, fs.SkipMsgs(2, 100_000))\n\t\t\t\tfs.mu.RLock()\n\t\t\t\tif blks := len(fs.blks); blks != 2 {\n\t\t\t\t\tfs.mu.RUnlock()\n\t\t\t\t\tt.Fatalf(\"Expected 2 blocks, got %d\", blks)\n\t\t\t\t}\n\t\t\t\tlmb := fs.blks[1]\n\t\t\t\tfs.mu.RUnlock()\n\n\t\t\t\t// Should have immediately flushed the previous block.\n\t\t\t\trequire_Equal(t, fmb.pendingWriteSize(), 0)\n\n\t\t\t\t// Should eventually flush the last block.\n\t\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\tif p := lmb.pendingWriteSize(); p > 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected no pending writes, got %d\", p)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestFileStoreCompressionAfterTruncate(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\taction func(fs *fileStore, seq uint64)\n\t}{\n\t\t{\n\t\t\ttitle: \"RemoveMsg\",\n\t\t\taction: func(fs *fileStore, seq uint64) {\n\t\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"EraseMsg\",\n\t\t\taction: func(fs *fileStore, seq uint64) {\n\t\t\t\terased, err := fs.EraseMsg(seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, erased)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"Tombstone\",\n\t\t\taction: func(fs *fileStore, seq uint64) {\n\t\t\t\tremoved, err := fs.removeMsg(seq, false, false, true)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tfor _, recompress := range []bool{false, true} {\n\t\t\tt.Run(fmt.Sprintf(\"%s/Recompress=%v\", test.title, recompress), func(t *testing.T) {\n\t\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\t\t\t\tcreated := time.Now()\n\t\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer fs.Stop()\n\n\t\t\t\t\tfor range 2 {\n\t\t\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t}\n\n\t\t\t\t\tcheckCompressed := func(mb *msgBlock) (bool, error) {\n\t\t\t\t\t\tmb.mu.Lock()\n\t\t\t\t\t\tdefer mb.mu.Unlock()\n\t\t\t\t\t\tbuf, err := mb.loadBlock(nil)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn false, err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\t\t\t\t\t\treturn false, err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tvar meta CompressionInfo\n\t\t\t\t\t\tif n, err := meta.UnmarshalMetadata(buf); err != nil {\n\t\t\t\t\t\t\treturn false, err\n\t\t\t\t\t\t} else if n == 0 {\n\t\t\t\t\t\t\treturn false, nil\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\treturn meta.Algorithm != NoCompression, nil\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tsmb := fs.getFirstBlock()\n\t\t\t\t\trequire_NotNil(t, smb)\n\t\t\t\t\tcompressed, err := checkCompressed(smb)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_False(t, compressed)\n\n\t\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif fcfg.Compression != NoCompression {\n\t\t\t\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\t\t\tif compressed, err = checkCompressed(smb); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t} else if !compressed {\n\t\t\t\t\t\t\t\treturn errors.New(\"block not compressed yet\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t})\n\t\t\t\t\t} else {\n\t\t\t\t\t\tcompressed, err = checkCompressed(smb)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_False(t, compressed)\n\t\t\t\t\t}\n\n\t\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\ttest.action(fs, 2)\n\n\t\t\t\t\tstate := fs.State()\n\t\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\t\t\t\trequire_Equal(t, state.LastSeq, 3)\n\t\t\t\t\trequire_Equal(t, state.NumDeleted, 1)\n\n\t\t\t\t\trequire_NoError(t, fs.Truncate(2))\n\t\t\t\t\tstate = fs.State()\n\t\t\t\t\trequire_Equal(t, state.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\t\t\t\trequire_Equal(t, state.LastSeq, 2)\n\t\t\t\t\trequire_Equal(t, state.NumDeleted, 1)\n\n\t\t\t\t\tfs.mu.RLock()\n\t\t\t\t\tlmb := fs.lmb\n\t\t\t\t\tfs.mu.RUnlock()\n\t\t\t\t\tif smb == lmb {\n\t\t\t\t\t\tcompressed, err = checkCompressed(smb)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_False(t, compressed)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tcompressed, err = checkCompressed(lmb)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_False(t, compressed)\n\n\t\t\t\t\t\tif fcfg.Compression != NoCompression {\n\t\t\t\t\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\t\t\t\tif compressed, err = checkCompressed(smb); err != nil {\n\t\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t\t} else if !compressed {\n\t\t\t\t\t\t\t\t\treturn errors.New(\"block not compressed yet\")\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\trequire_NoError(t, fs.forceWriteFullState())\n\n\t\t\t\t\tseq, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Equal(t, seq, 3)\n\t\t\t\t\trequire_NoError(t, fs.forceWriteFullState())\n\n\t\t\t\t\tsmb.mu.Lock()\n\t\t\t\t\tsmb.clearCacheAndOffset()\n\t\t\t\t\tsmb.mu.Unlock()\n\n\t\t\t\t\trequire_NoError(t, smb.loadMsgsWithLock())\n\t\t\t\t\tcompressed, err = checkCompressed(smb)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tif smb == lmb {\n\t\t\t\t\t\trequire_False(t, compressed)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Equal(t, compressed, fcfg.Compression != NoCompression)\n\t\t\t\t\t}\n\n\t\t\t\t\tcompressed, err = checkCompressed(lmb)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_False(t, compressed)\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestFileStoreTruncateRemovedBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := range 3 {\n\t\t\tif i > 0 {\n\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, blks, 3)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 3)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 3)\n\t\trequire_Equal(t, state.NumDeleted, 0)\n\n\t\tremoved, err := fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, removed)\n\n\t\tfs.mu.RLock()\n\t\tblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, blks, 2)\n\n\t\tfs.mu.RLock()\n\t\tblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Len(t, blks, 2)\n\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 3)\n\t\trequire_Equal(t, state.NumDeleted, 1)\n\n\t\trequire_NoError(t, fs.Truncate(2))\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.Msgs, 1)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 2)\n\t\trequire_Equal(t, state.NumDeleted, 1)\n\t})\n}\n\nfunc TestFileStoreAtomicEraseMsg(t *testing.T) {\n\tfor _, lmb := range []bool{true, false} {\n\t\tt.Run(fmt.Sprintf(\"lmb=%v\", lmb), func(t *testing.T) {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\t\t\tcreated := time.Now()\n\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tfor range 3 {\n\t\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\n\t\t\t\tcheckCompressed := func(mb *msgBlock) (bool, error) {\n\t\t\t\t\tmb.mu.Lock()\n\t\t\t\t\tdefer mb.mu.Unlock()\n\t\t\t\t\tbuf, err := mb.loadBlock(nil)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn false, err\n\t\t\t\t\t}\n\t\t\t\t\tif err := mb.checkAndLoadEncryption(); err != nil {\n\t\t\t\t\t\treturn false, err\n\t\t\t\t\t}\n\t\t\t\t\tif err = mb.encryptOrDecryptIfNeeded(buf); err != nil {\n\t\t\t\t\t\treturn false, err\n\t\t\t\t\t}\n\t\t\t\t\tvar meta CompressionInfo\n\t\t\t\t\tif n, err := meta.UnmarshalMetadata(buf); err != nil {\n\t\t\t\t\t\treturn false, err\n\t\t\t\t\t} else if n == 0 {\n\t\t\t\t\t\treturn false, nil\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn meta.Algorithm != NoCompression, nil\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif !lmb {\n\t\t\t\t\tsmb := fs.getFirstBlock()\n\t\t\t\t\trequire_NotNil(t, smb)\n\n\t\t\t\t\tmb, err := fs.newMsgBlockForWrite()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tcompressed, err := checkCompressed(mb)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_False(t, compressed)\n\n\t\t\t\t\tif fcfg.Compression != NoCompression {\n\t\t\t\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\t\t\tif compressed, err = checkCompressed(smb); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t} else if !compressed {\n\t\t\t\t\t\t\t\treturn errors.New(\"block not compressed yet\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tbefore := fs.State()\n\t\t\t\trequire_Equal(t, before.Msgs, 3)\n\t\t\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, before.LastSeq, 3)\n\t\t\t\trequire_Equal(t, before.NumDeleted, 0)\n\n\t\t\t\tremoved, err := fs.EraseMsg(2)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\n\t\t\t\tbefore = fs.State()\n\t\t\t\trequire_Equal(t, before.Msgs, 2)\n\t\t\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\t\t\trequire_Equal(t, before.LastSeq, 3)\n\t\t\t\trequire_Equal(t, before.NumDeleted, 1)\n\n\t\t\t\tseq, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, seq, 4)\n\t\t\t\tbefore = fs.State()\n\n\t\t\t\tvalidateCompressed := func() {\n\t\t\t\t\tt.Helper()\n\t\t\t\t\tfs.mu.Lock()\n\t\t\t\t\tdefer fs.mu.Unlock()\n\t\t\t\t\tfor _, mb := range fs.blks {\n\t\t\t\t\t\tcompressed, err := checkCompressed(mb)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tif mb == fs.lmb {\n\t\t\t\t\t\t\trequire_False(t, compressed)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\trequire_Equal(t, compressed, fs.fcfg.Compression != NoCompression)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tvalidateCompressed()\n\n\t\t\t\t// Restart should equal before.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected before of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\t\t\t\tvalidateCompressed()\n\n\t\t\t\t// Stop and remove stream before file.\n\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t\t\t// Recovering based on blocks should result in the same before.\n\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected before of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\t\t\t\tvalidateCompressed()\n\n\t\t\t\t// Rebuilding before must also result in the same before.\n\t\t\t\tfs.rebuildState(nil)\n\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected before of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t}\n\t\t\t\tvalidateCompressed()\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestFileStoreRemoveBlockWithStaleStreamState(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := range 3 {\n\t\t\tif i > 0 {\n\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Get middle block.\n\t\tfs.mu.RLock()\n\t\trequire_Len(t, len(fs.blks), 3)\n\t\tmidfn := fs.blks[1].mfn\n\t\tfs.mu.RUnlock()\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(midfn))\n\n\t\t// Restart.\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := range 3 {\n\t\t\tseq := uint64(i + 1)\n\t\t\t_, err = fs.LoadMsg(seq, nil)\n\t\t\tif seq == 2 {\n\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t} else {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStoreMessageSchedule(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, AllowMsgSchedules: true})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Capture message schedule proposals.\n\tch := make(chan *inMsg, 1)\n\tfs.pmsgcb = func(im *inMsg) {\n\t\tch <- im\n\t}\n\n\t// Store a single message schedule.\n\tschedule := time.Now().Add(time.Second).Format(time.RFC3339Nano)\n\thdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf(\"@at %s\", schedule))\n\thdr = genHeader(hdr, JSScheduleTarget, \"foo.target\")\n\t_, _, err = fs.StoreMsg(\"foo.schedule\", hdr, nil, 0)\n\trequire_NoError(t, err)\n\n\t// We should have published a scheduled message.\n\tim := require_ChanRead(t, ch, time.Second*5)\n\trequire_Equal(t, im.subj, \"foo.target\")\n\trequire_Equal(t, bytesToString(getHeader(JSScheduler, im.hdr)), \"foo.schedule\")\n\trequire_Equal(t, bytesToString(getHeader(JSScheduleNext, im.hdr)), JSScheduleNextPurge)\n}\n\nfunc TestFileStoreMessageScheduleRecovered(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdir := t.TempDir()\n\tt.Run(\"BeforeRestart\", func(t *testing.T) {\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, AllowMsgSchedules: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tschedule := time.Now().Add(time.Second).Format(time.RFC3339Nano)\n\t\thdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf(\"@at %s\", schedule))\n\t\thdr = genHeader(hdr, JSScheduleTarget, \"foo.target\")\n\t\t_, _, err = fs.StoreMsg(\"foo.schedule\", hdr, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t\trequire_Equal(t, ss.Msgs, 1)\n\t})\n\n\tt.Run(\"AfterRestart\", func(t *testing.T) {\n\t\t// Delete the message scheduling state so that we are forced to do a linear scan\n\t\t// of message blocks containing message schedules.\n\t\tfn := filepath.Join(dir, msgDir, msgSchedulingStreamStateFile)\n\t\trequire_NoError(t, os.Remove(fn))\n\n\t\tfs, err := newFileStore(\n\t\t\tFileStoreConfig{StoreDir: dir, srv: s},\n\t\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: FileStorage, AllowMsgSchedules: true})\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\trequire_Equal(t, fs.numMsgBlocks(), 1)\n\t\tfs.mu.RLock()\n\t\tmb := fs.blks[0]\n\t\tfs.mu.RUnlock()\n\t\tmb.mu.RLock()\n\t\tschedules := mb.schedules\n\t\tmb.mu.RUnlock()\n\n\t\trequire_Equal(t, schedules, 1)\n\n\t\tvar ss StreamState\n\t\tfs.FastState(&ss)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t\trequire_Equal(t, ss.Msgs, 1)\n\t})\n}\n\nfunc TestFileStoreMessageScheduleEncodeDecode(t *testing.T) {\n\tms := newMsgScheduling(func() {})\n\tnow := time.Now()\n\n\t// Add many sequences.\n\tnumSequences := 100_000\n\tfor seq := 0; seq < numSequences; seq++ {\n\t\tts := now.Add(time.Duration(seq) * time.Second).UnixNano()\n\t\tsubj := fmt.Sprintf(\"foo.%d\", seq)\n\t\tms.add(uint64(seq), subj, ts)\n\t}\n\n\tb := ms.encode(12345)\n\trequire_True(t, len(b) > 17) // Bigger than just the header\n\n\tnms := newMsgScheduling(func() {})\n\tstamp, err := nms.decode(b)\n\trequire_NoError(t, err)\n\trequire_Equal(t, stamp, 12345)\n\trequire_Equal(t, ms.ttls.GetNextExpiration(math.MaxInt64), nms.ttls.GetNextExpiration(math.MaxInt64))\n\n\trequire_Len(t, len(ms.seqToSubj), len(nms.seqToSubj))\n\tfor seq, subj := range ms.seqToSubj {\n\t\trequire_Equal(t, subj, nms.seqToSubj[seq])\n\t}\n\n\trequire_Len(t, len(ms.schedules), len(nms.schedules))\n\tfor subj, sched := range ms.schedules {\n\t\tnsched := nms.schedules[subj]\n\t\trequire_NotNil(t, nsched)\n\t\trequire_Equal(t, sched.ts, nsched.ts)\n\t\trequire_Equal(t, sched.seq, nsched.seq)\n\t}\n}\n\nfunc TestFileStoreCorruptedNonOrderedSequences(t *testing.T) {\n\tfor _, test := range []struct {\n\t\ttitle string\n\t\tseqs []uint64\n\t\tmsgs uint64\n\t\tdeleted int\n\t}{\n\t\t{title: \"Unordered\", seqs: []uint64{1, 3, 2, 4}, msgs: 3, deleted: 1},\n\t\t{title: \"Duplicated\", seqs: []uint64{1, 2, 2, 3}, msgs: 3, deleted: 0},\n\t} {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\t\t\tcreated := time.Now()\n\t\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\n\t\t\t\tfor _, seq := range test.seqs {\n\t\t\t\t\t_, err = fs.writeMsgRecord(seq, 0, _EMPTY_, nil, nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\n\t\t\t\tfs.mu.RLock()\n\t\t\t\tlmb := fs.lmb\n\t\t\t\tfs.mu.RUnlock()\n\n\t\t\t\t// The filestore will not yet know that something was corrupt.\n\t\t\t\tlmb.mu.RLock()\n\t\t\t\tdefer lmb.mu.RUnlock()\n\t\t\t\trequire_Equal(t, lmb.msgs, 4)\n\t\t\t\trequire_Equal(t, lmb.dmap.Size(), 0)\n\n\t\t\t\t// Need to reset, otherwise the rebuild will be incorrect.\n\t\t\t\tatomic.StoreUint64(&lmb.first.seq, 0)\n\n\t\t\t\t// Upon rebuild it should realize and correct.\n\t\t\t\t_, _, err = lmb.rebuildStateLocked()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, lmb.msgs, test.msgs)\n\t\t\t\trequire_Equal(t, lmb.dmap.Size(), test.deleted)\n\n\t\t\t\t// Indexing should also realize and correct.\n\t\t\t\trequire_True(t, lmb.cacheNotLoaded())\n\t\t\t\tbuf, err := lmb.loadBlock(nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, lmb.encryptOrDecryptIfNeeded(buf))\n\t\t\t\tbuf, err = lmb.decompressIfNeeded(buf)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, lmb.indexCacheBuf(buf))\n\t\t\t\trequire_True(t, lmb.cacheAlreadyLoaded())\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc BenchmarkFileStoreGetSeqFromTime(b *testing.B) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{\n\t\t\tStoreDir: b.TempDir(),\n\t\t\tBlockSize: 16,\n\t\t},\n\t\tStreamConfig{\n\t\t\tName: \"foo\",\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\tStorage: FileStorage,\n\t\t},\n\t)\n\trequire_NoError(b, err)\n\tdefer fs.Stop()\n\n\tfor range 4096 {\n\t\t_, _, err := fs.StoreMsg(\"foo.bar\", nil, []byte{1, 2, 3, 4, 5}, 0)\n\t\trequire_NoError(b, err)\n\t}\n\n\tfs.mu.RLock()\n\tfs.blks[0].mu.RLock()\n\tfs.lmb.mu.RLock()\n\tstart := time.Unix(0, fs.blks[0].first.ts)\n\tmiddle := time.Unix(0, fs.blks[0].first.ts+(fs.lmb.last.ts-fs.blks[0].first.ts)/2)\n\tend := time.Unix(0, fs.lmb.last.ts)\n\tfs.blks[0].mu.RUnlock()\n\tfs.lmb.mu.RUnlock()\n\tfs.mu.RUnlock()\n\n\tb.Run(\"Start\", func(b *testing.B) {\n\t\tb.ReportAllocs()\n\t\tfor range b.N {\n\t\t\tfs.GetSeqFromTime(start)\n\t\t}\n\t})\n\n\tb.Run(\"Middle\", func(b *testing.B) {\n\t\tb.ReportAllocs()\n\t\tfor range b.N {\n\t\t\tfs.GetSeqFromTime(middle)\n\t\t}\n\t})\n\n\tb.Run(\"End\", func(b *testing.B) {\n\t\tb.ReportAllocs()\n\t\tfor range b.N {\n\t\t\tfs.GetSeqFromTime(end)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreCacheLookupOnEmptyBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\n\t\t// First make sure that we haven't got a strong reference to the cache.\n\t\trequire_NotNil(t, lmb)\n\t\tlmb.finishedWithCache()\n\t\trequire_True(t, lmb.cache == nil)\n\n\t\t// Specifically we want ErrStoreMsgNotFound, not errNoCache.\n\t\t_, err = lmb.cacheLookup(atomic.LoadUint64(&lmb.first.seq), nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t// Now make sure that we didn't strengthen the reference. This proves\n\t\t// that we short-circuited properly.\n\t\trequire_True(t, lmb.cache == nil)\n\t})\n}\n\nfunc TestFileStoreEraseMsgDoesNotLoseTombstones(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tsecret := []byte(\"secret!\")\n\t\t// The first message will remain throughout.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t// The second message wil be removed, so a tombstone will be placed.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t// The third message is secret and will be erased.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, secret, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Removing the second message places a tombstone.\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\n\t\t// A fourth message gets placed after the tombstone.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Now we erase the third message.\n\t\t// This erases this message and should not lose the tombstone that comes after it.\n\t\t_, err = fs.EraseMsg(3)\n\t\trequire_NoError(t, err)\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 2)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 4)\n\t\trequire_True(t, slices.Equal(before.Deleted, []uint64{2, 3}))\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t})\n}\n\nfunc TestFileStoreEraseMsgDoesNotLoseTombstonesInEmptyBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// The first message will remain throughout.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t// The second message wil be removed, so a tombstone will be placed.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tmb, err := fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\n\t\tsecret := []byte(\"secret!\")\n\t\t// The third message is secret and will be erased.\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, secret, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Removing the second message places a tombstone.\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\n\t\t// Now we erase the third message.\n\t\t// This erases this message and should not lose the tombstone that comes after it.\n\t\t// It should do the erase, even if the block would be empty afterward as it could contain tombstones.\n\t\t_, err = fs.EraseMsg(3)\n\t\trequire_NoError(t, err)\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 3)\n\t\trequire_True(t, slices.Equal(before.Deleted, []uint64{2, 3}))\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t// The message should be erased.\n\t\tbuf, err := mb.loadBlock(nil)\n\t\trequire_NoError(t, err)\n\t\trequire_False(t, bytes.Contains(buf, secret))\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t})\n}\n\nfunc TestFileStoreTombstonesNoFirstSeqRollback(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 10 * 33 // 10 messages per block.\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := 0; i < 20; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 20)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 20)\n\n\t\t// Expect 2 blocks with messages.\n\t\tfs.mu.RLock()\n\t\tlblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 2)\n\n\t\t// Write some tombstones for all messages, these will be in multiple blocks.\n\t\tfor seq := uint64(1); seq <= 20; seq++ {\n\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tbefore = fs.State()\n\t\trequire_Equal(t, before.Msgs, 0)\n\t\trequire_Equal(t, before.FirstSeq, 21)\n\t\trequire_Equal(t, before.LastSeq, 20)\n\n\t\t// Expect 1 block purely with tombstones.\n\t\tfs.mu.RLock()\n\t\tlblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 1)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreTombstonesSelectNextFirstCleanup(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 10 * 33 // 10 messages per block.\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Write a bunch of messages in multiple blocks.\n\t\tfor i := 0; i < 50; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor seq := uint64(2); seq <= 49; seq++ {\n\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\t\tfor i := 0; i < 50; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor seq := uint64(50); seq <= 100; seq++ {\n\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 100)\n\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\n\t\tbefore = fs.State()\n\t\trequire_Equal(t, before.Msgs, 0)\n\t\trequire_Equal(t, before.FirstSeq, 101)\n\t\trequire_Equal(t, before.LastSeq, 100)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreTombstonesSelectNextFirstCleanupOnRecovery(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 10 * 33 // 10 messages per block.\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Write a bunch of messages in multiple blocks.\n\t\tfor i := 0; i < 50; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor seq := uint64(2); seq <= 49; seq++ {\n\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\t\tfor i := 0; i < 50; i++ {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor seq := uint64(50); seq <= 100; seq++ {\n\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 100)\n\n\t\t// Explicitly write tombstone instead of calling fs.RemoveMsg,\n\t\t// so we need to recover from a hard kill.\n\t\trequire_NoError(t, fs.writeTombstone(1, 0))\n\t\tbefore = StreamState{FirstSeq: 101, FirstTime: time.Time{}, LastSeq: 100, LastTime: before.LastTime}\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreDetectDeleteGapWithLastSkipMsg(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Skip a message at a sequence such that a gap is created.\n\t\t// The gap should be detected later on as deleted messages.\n\t\trequire_NoError(t, fs.SkipMsgs(2, 3))\n\n\t\t// We should have 3 deletes, one is the skip msg, the other two is the gap.\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 4)\n\t\trequire_Equal(t, before.NumDeleted, 3)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.RLock()\n\t\tdefer mb.mu.RUnlock()\n\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 1)\n\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 4)\n\t\trequire_Len(t, mb.dmap.Size(), 3)\n\t})\n}\n\nfunc TestFileStoreDetectDeleteGapWithOnlySkipMsg(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Skip messages with a gap.\n\t\t_, err = fs.SkipMsg(1)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, fs.SkipMsgs(2, 3))\n\n\t\t// We should have no deletes, as the SkipMsgs only move the sequences up.\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 0)\n\t\trequire_Equal(t, before.FirstSeq, 5)\n\t\trequire_Equal(t, before.LastSeq, 4)\n\t\trequire_Equal(t, before.NumDeleted, 0)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state of %+v, got %+v\", before, state)\n\t\t}\n\n\t\t// The block should not register the deletes between the two SkipMsgs.\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.RLock()\n\t\tdefer mb.mu.RUnlock()\n\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 5)\n\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 4)\n\t\trequire_Len(t, mb.dmap.Size(), 0)\n\t})\n}\n\nfunc TestFileStoreEraseMsgErr(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.Lock()\n\t\tif mb.cache == nil {\n\t\t\tmb.mu.Unlock()\n\t\t\tt.Fatal(\"Expected cache to be initialized\")\n\t\t}\n\t\t// Set to a bogus value such that the file rename fails while performing the message erase.\n\t\tmb.mfn = _EMPTY_\n\t\tmb.mu.Unlock()\n\t\tfs.EraseMsg(2)\n\n\t\t// Cleanup \".tmp\" file if it was created due to the purposefully invalid file name above.\n\t\t_, err = os.Stat(blkTmpSuffix)\n\t\tif err == nil {\n\t\t\trequire_NoError(t, os.Remove(blkTmpSuffix))\n\t\t}\n\t})\n}\n\nfunc TestFileStorePurgeMsgBlock(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 10 * 33\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor range 20 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, blks, 2)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 20)\n\t\trequire_Equal(t, state.Msgs, 20)\n\t\trequire_Equal(t, state.Bytes, 20*33)\n\n\t\t// Purging the block should both remove the block and do the accounting.\n\t\tfmb := fs.getFirstBlock()\n\t\tfs.mu.Lock()\n\t\tfs.purgeMsgBlock(fmb)\n\t\tblks = len(fs.blks)\n\t\tfs.mu.Unlock()\n\n\t\trequire_Equal(t, blks, 1)\n\t\tstate = fs.State()\n\t\trequire_Equal(t, state.FirstSeq, 11)\n\t\trequire_Equal(t, state.LastSeq, 20)\n\t\trequire_Equal(t, state.Msgs, 10)\n\t\trequire_Equal(t, state.Bytes, 10*33)\n\t})\n}\n\nfunc TestFileStorePurgeMsgBlockUpdatesSubjects(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.BlockSize = 10 * 33\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor range 20 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfst := fs.SubjectsTotals(\"foo\")\n\t\trequire_Equal(t, fst[\"foo\"], uint64(20))\n\n\t\tfmb := fs.getFirstBlock()\n\t\tfs.mu.Lock()\n\t\tfs.purgeMsgBlock(fmb)\n\t\tfs.mu.Unlock()\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, uint64(10))\n\t\trequire_Equal(t, state.FirstSeq, uint64(11))\n\n\t\tfst = fs.SubjectsTotals(\"foo\")\n\t\trequire_Equal(t, fst[\"foo\"], uint64(10))\n\t})\n}\n\nfunc TestFileStorePurgeMsgBlockRemovesSchedules(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tschedule := time.Now().Add(time.Hour).Format(time.RFC3339Nano)\n\t\thdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf(\"@at %s\", schedule))\n\t\thdr = genHeader(hdr, JSScheduleTarget, \"foo.target.0\")\n\t\tmsgSize := fileStoreMsgSize(\"foo.sched.0\", hdr, []byte(\"x\"))\n\n\t\t// Force two blocks of 5 messages each.\n\t\tfcfg.BlockSize = uint64(msgSize * 5)\n\t\tcfg := StreamConfig{\n\t\t\tName: \"zzz\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgSchedules: true,\n\t\t}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := range 10 {\n\t\t\tsubj := fmt.Sprintf(\"foo.sched.%d\", i)\n\t\t\ttarget := fmt.Sprintf(\"foo.target.%d\", i)\n\t\t\thdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf(\"@at %s\", schedule))\n\t\t\thdr = genHeader(hdr, JSScheduleTarget, target)\n\t\t\t_, _, err = fs.StoreMsg(subj, hdr, []byte(\"x\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tblks := len(fs.blks)\n\t\tsts, msgs := len(fs.scheduling.seqToSubj), int(fs.state.Msgs)\n\t\tfs.mu.RUnlock()\n\t\trequire_True(t, blks >= 2)\n\t\trequire_Equal(t, sts, msgs)\n\n\t\tfmb := fs.getFirstBlock()\n\t\tfs.mu.Lock()\n\t\tfs.purgeMsgBlock(fmb)\n\t\tfs.mu.Unlock()\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, uint64(5))\n\n\t\tfs.mu.Lock()\n\t\tdefer fs.mu.Unlock()\n\t\trequire_Equal(t, len(fs.scheduling.seqToSubj), int(state.Msgs))\n\t\tfor seq := uint64(1); seq < state.FirstSeq; seq++ {\n\t\t\tif _, ok := fs.scheduling.seqToSubj[seq]; ok {\n\t\t\t\tt.Fatalf(\"expected schedule for seq %d to be removed\", seq)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestFileStorePurgeMsgBlockAccounting(t *testing.T) {\n\ttest := func(t *testing.T, update func(cfg *nats.StreamConfig)) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: nats.FileStorage,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsubj, data := \"foo\", make([]byte, 1024*1024)\n\t\tfor range 10 {\n\t\t\t_, err = js.Publish(subj, data)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tgacc := s.globalAccount()\n\t\tmset, err := gacc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tstate := mset.state()\n\t\tstats := gacc.JetStreamUsage()\n\t\trequire_Equal(t, state.Bytes, stats.JetStreamTier.Store)\n\n\t\tupdate(cfg)\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\tstate = mset.state()\n\t\tstats = gacc.JetStreamUsage()\n\t\trequire_Equal(t, state.Bytes, fileStoreMsgSizeRaw(len(subj), 0, len(data)))\n\t\trequire_Equal(t, state.Bytes, stats.JetStreamTier.Store)\n\t}\n\n\tt.Run(\"MaxMsgs\", func(t *testing.T) {\n\t\ttest(t, func(cfg *nats.StreamConfig) {\n\t\t\tcfg.MaxMsgs = 1\n\t\t})\n\t})\n\tt.Run(\"MaxBytes\", func(t *testing.T) {\n\t\ttest(t, func(cfg *nats.StreamConfig) {\n\t\t\tcfg.MaxBytes = int64(fileStoreMsgSizeRaw(3, 0, 1024*1024))\n\t\t})\n\t})\n}\n\nfunc TestFileStoreMissingDeletesAfterCompact(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Generate a block with 6 messages and then delete the first and last, as well as a larger gap in the middle.\n\t\tfor range 6 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(3)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(4)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(6)\n\t\trequire_NoError(t, err)\n\n\t\t// We'll compact the deletes later, but shouldn't be lmb.\n\t\tfmb := fs.getFirstBlock()\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\n\t\t// Confirm the block's state.\n\t\tfmb.mu.Lock()\n\t\tdefer fmb.mu.Unlock()\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.first.seq), 2)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.last.seq), 6)\n\t\trequire_Equal(t, fmb.msgs, 2)\n\t\trequire_Len(t, fmb.dmap.Size(), 3)\n\t\trequire_True(t, fmb.dmap.Exists(3))\n\t\trequire_True(t, fmb.dmap.Exists(4))\n\t\trequire_True(t, fmb.dmap.Exists(6))\n\n\t\t// Now compact and reload and the block should still have the correct deletes.\n\t\trequire_NoError(t, fmb.compact())\n\t\tfmb.clearCache()\n\t\tfmb.dmap.Empty()\n\t\trequire_NoError(t, fmb.loadMsgsWithLock())\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.first.seq), 2)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.last.seq), 5)\n\t\trequire_Equal(t, fmb.msgs, 2)\n\t\trequire_Len(t, fmb.dmap.Size(), 2)\n\t\trequire_True(t, fmb.dmap.Exists(3))\n\t\trequire_True(t, fmb.dmap.Exists(4))\n\n\t\t// Rebuilding should have the state remain the same.\n\t\t_, _, err = fmb.rebuildStateLocked()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.first.seq), 2)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.last.seq), 5)\n\t\trequire_Equal(t, fmb.msgs, 2)\n\n\t\t// Delete at sequence 5 such that the block can be compacted to a single message.\n\t\tfmb.mu.Unlock()\n\t\t_, err = fs.RemoveMsg(5)\n\t\tfmb.mu.Lock()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, fmb.compact())\n\t\tfmb.clearCache()\n\t\tfmb.dmap.Empty()\n\t\trequire_NoError(t, fmb.loadMsgsWithLock())\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.first.seq), 2)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.last.seq), 2)\n\t\trequire_Equal(t, fmb.msgs, 1)\n\t\trequire_Len(t, fmb.dmap.Size(), 0)\n\n\t\t// Rebuilding should have the state remain the same.\n\t\t_, _, err = fmb.rebuildStateLocked()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.first.seq), 2)\n\t\trequire_Equal(t, atomic.LoadUint64(&fmb.last.seq), 2)\n\t\trequire_Equal(t, fmb.msgs, 1)\n\t\trequire_Len(t, fmb.dmap.Size(), 0)\n\t})\n}\n\nfunc TestFileStoreIdxAccountingForSkipMsgs(t *testing.T) {\n\ttest := func(t *testing.T, skipMany bool) {\n\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\t\tcreated := time.Now()\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tif skipMany {\n\t\t\t\trequire_NoError(t, fs.SkipMsgs(2, 10))\n\t\t\t} else {\n\t\t\t\tfor i := range 10 {\n\t\t\t\t\t_, err = fs.SkipMsg(uint64(i + 2))\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfmb := fs.getFirstBlock()\n\t\t\tfmb.mu.Lock()\n\t\t\tdefer fmb.mu.Unlock()\n\n\t\t\tfor i := range 12 {\n\t\t\t\tseq := uint64(i + 1)\n\t\t\t\t_, err = fmb.cacheLookupNoCopy(seq, nil)\n\t\t\t\tif seq >= 2 && seq <= 11 {\n\t\t\t\t\trequire_Error(t, err, errDeletedMsg)\n\t\t\t\t} else {\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcache := fmb.cache\n\t\t\trequire_NotNil(t, cache)\n\t\t\trequire_Len(t, len(cache.idx), 12)\n\t\t})\n\t}\n\n\tt.Run(\"SkipMsg\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"SkipMsgs\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestFileStoreEmptyBlockContainsPriorTombstones(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// 1.blk\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// 2.blk\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(3) // Will create a new lmb with this as tombstone.\n\t\trequire_NoError(t, err)\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 1)\n\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\trequire_Equal(t, before.LastSeq, 3)\n\t\trequire_True(t, slices.Equal(before.Deleted, []uint64{2, 3}))\n\n\t\tfs.mu.RLock()\n\t\tlblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 3)\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t// Make sure we can recover properly with no index.db present.\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tlblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 3)\n\n\t\t_, err = fs.LoadMsg(2, nil)\n\t\trequire_Error(t, err, errDeletedMsg)\n\t\t_, err = fs.LoadMsg(3, nil)\n\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t// 3.blk\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\tfs.mu.RLock()\n\t\tlblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 3)\n\n\t\t// Removing the first message moves the first seq up.\n\t\t// Should also remove blocks without any messages and (invalidated) tombstones.\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\n\t\tfs.mu.RLock()\n\t\tlblks = len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 1)\n\t})\n}\n\nfunc TestFileStoreCompactTombstonesBelowFirstSeq(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// 1.blk\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// 2.blk\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(3)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 2)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 4)\n\t\trequire_True(t, slices.Equal(state.Deleted, []uint64{2, 3}))\n\n\t\tfs.mu.RLock()\n\t\tlblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\trequire_Equal(t, lblks, 2)\n\n\t\t// Block should report the two prior tombstones.\n\t\tfs.mu.Lock()\n\t\tlmb := fs.lmb\n\t\tlmb.mu.Lock()\n\t\tpriorTombs := lmb.numPriorTombsLocked()\n\t\tlmb.mu.Unlock()\n\t\tfs.mu.Unlock()\n\t\trequire_Equal(t, priorTombs, 2)\n\n\t\t// The first sequence moves up as a result of the removal.\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\n\t\t// Block should now report no prior tombstones, since they are now invalid.\n\t\tfs.mu.Lock()\n\t\tlmb.mu.Lock()\n\t\tpriorTombs = lmb.numPriorTombsLocked()\n\t\tlmb.mu.Unlock()\n\t\tfs.mu.Unlock()\n\t\trequire_Equal(t, priorTombs, 0)\n\n\t\t// Make sure we have a new last block such that we can compact.\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\n\t\tlmb.mu.RLock()\n\t\trbytes := lmb.rbytes\n\t\tshouldCompact := lmb.shouldCompactSync()\n\t\tlmb.mu.RUnlock()\n\t\trequire_True(t, shouldCompact)\n\t\tfs.syncBlocks()\n\n\t\tlmb.mu.RLock()\n\t\tdefer lmb.mu.RUnlock()\n\t\trequire_NotEqual(t, lmb.rbytes, rbytes)\n\t})\n}\n\nfunc TestFileStoreSyncBlocksFlushesAndSyncsMessages(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfcfg.AsyncFlush = true\n\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\trequire_NotNil(t, lmb)\n\n\t\t// Wait for flusher to be ready.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tlmb.mu.RLock()\n\t\t\tdefer lmb.mu.RUnlock()\n\t\t\tif !lmb.flusher {\n\t\t\t\treturn errors.New(\"flusher not active\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Now shutdown flusher and wait for it to be closed.\n\t\tlmb.mu.Lock()\n\t\tif lmb.qch != nil {\n\t\t\tclose(lmb.qch)\n\t\t\tlmb.qch = nil\n\t\t}\n\t\tlmb.mu.Unlock()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tlmb.mu.RLock()\n\t\t\tdefer lmb.mu.RUnlock()\n\t\t\tif lmb.flusher {\n\t\t\t\treturn errors.New(\"flusher still active\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tseq, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, seq, 1)\n\n\t\t// Update the last write timestamp to be in the past.\n\t\tlmb.mu.Lock()\n\t\tlwts := lmb.lwts\n\t\tlmb.lwts = 0\n\t\tlmb.mu.Unlock()\n\t\trequire_NotEqual(t, lwts, 0)\n\n\t\t// Syncing should write out the data.\n\t\tfs.syncBlocks()\n\n\t\t// Manually reset, sync should have written the data.\n\t\tlmb.clearCacheAndOffset()\n\n\t\tsm, err := fs.LoadMsg(1, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.seq, 1)\n\t\trequire_Equal(t, sm.subj, \"foo\")\n\t})\n}\n\nfunc TestJetStreamFileStoreSubjectsRemovedAfterSecureErase(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test.*\"},\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"test.1\", []byte(\"msg1\"))\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"test.2\", []byte(\"msg2\"))\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"test.3\", []byte(\"msg3\"))\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{SubjectsFilter: \">\"})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.NumSubjects, 3)\n\trequire_Len(t, len(si.State.Subjects), 3)\n\n\t// The bug happened here: the underlying eraseMsg() call in removeMsg() would\n\t// corrupt the sm.subj from the shallow cache lookup. We would then pass the\n\t// corrupted subject into removeSeqPerSubject() & removePerSubject(), resulting\n\t// in them being no-ops. This is now fixed.\n\trequire_NoError(t, js.SecureDeleteMsg(\"TEST\", 1))\n\n\tsi, err = js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{SubjectsFilter: \">\"})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.NumSubjects, 2)\n\trequire_Len(t, len(si.State.Subjects), 2)\n\n\t_, exists := si.State.Subjects[\"test.1\"]\n\trequire_False(t, exists)\n\trequire_Equal(t, si.State.Subjects[\"test.2\"], uint64(1))\n\trequire_Equal(t, si.State.Subjects[\"test.3\"], uint64(1))\n}\n\nfunc TestFileStorePreserveLastSeqAfterCompact(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = fs.Compact(2)\n\t\trequire_NoError(t, err)\n\n\t\tbefore := fs.State()\n\t\trequire_Equal(t, before.Msgs, 0)\n\t\trequire_Equal(t, before.FirstSeq, 2)\n\t\trequire_Equal(t, before.LastSeq, 1)\n\n\t\tfs.Stop()\n\t\tos.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile))\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t}\n\t})\n}\n\nfunc TestFileStoreSkipMsgAndCompactRequiresAppend(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// Store one very long message.\n\t\tmsg := make([]byte, 256*1024)\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// Skip and compact to that sequence, the block will be preserved and tombstones would be written.\n\t\t_, err = fs.SkipMsg(2)\n\t\trequire_NoError(t, err)\n\t\t_, err = fs.Compact(2)\n\t\trequire_NoError(t, err)\n\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 0)\n\t\trequire_Equal(t, state.FirstSeq, 3)\n\t\trequire_Equal(t, state.LastSeq, 2)\n\n\t\t// Skipping again would result in the bug. We tried to write to the start of the file,\n\t\t// but the old message data would still be there. Instead, now add the SkipMsg to the end.\n\t\t_, err = fs.SkipMsg(3)\n\t\trequire_NoError(t, err)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.Lock()\n\t\tdefer mb.mu.Unlock()\n\t\tmb.clearCacheAndOffset()\n\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\t})\n}\n\nfunc TestFileStoreCompactRewritesFileWithSwap(t *testing.T) {\n\tfcfg := FileStoreConfig{Cipher: NoCipher, Compression: NoCompression, StoreDir: t.TempDir()}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := make([]byte, 256*1024)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Compact should realize the block's data was largely removed, the file should be rewritten\n\t_, err = fs.Compact(20)\n\trequire_NoError(t, err)\n\n\tstate := fs.State()\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.FirstSeq, 20)\n\trequire_Equal(t, state.LastSeq, 20)\n\n\tmb := fs.getFirstBlock()\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\tmb.clearCacheAndOffset()\n\trequire_NoError(t, mb.loadMsgsWithLock())\n\n\tmbcache := mb.cache\n\trequire_NotNil(t, mbcache)\n\trequire_Len(t, len(mbcache.idx), 1)\n\trequire_Equal(t, mbcache.idx[0], 0)\n}\n\nfunc TestFileStoreIndexCacheBufIdxMismatch(t *testing.T) {\n\tconst (\n\t\tKindTruncateFull = iota\n\t\tKindTruncatePartial\n\t\tKindCompactHead\n\t\tKindCompactTail\n\t\tKindCompact\n\t)\n\ttest := func(t *testing.T, kind int) {\n\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tfor range 5 {\n\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tmb := fs.getFirstBlock()\n\t\t\tmb.mu.Lock()\n\t\t\tdefer mb.mu.Unlock()\n\n\t\t\tvar deleted int\n\t\t\tcompactHead := kind == KindCompactHead || kind == KindCompact\n\t\t\tcompactTail := kind == KindCompactTail || kind == KindCompact\n\t\t\tif compactHead || compactTail {\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 1)\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 5)\n\t\t\t\tif compactHead {\n\t\t\t\t\tatomic.StoreUint64(&mb.first.seq, 2)\n\t\t\t\t\tdeleted++\n\t\t\t\t}\n\t\t\t\tif compactTail {\n\t\t\t\t\tmb.dmap.Insert(5)\n\t\t\t\t\tdeleted++\n\t\t\t\t}\n\t\t\t\trequire_NoError(t, mb.compactWithFloor(0, nil))\n\n\t\t\t\t// Revert the state, upon reload we need to recognize this.\n\t\t\t\tatomic.StoreUint64(&mb.first.seq, 1)\n\t\t\t\tatomic.StoreUint64(&mb.last.seq, 5)\n\t\t\t}\n\n\t\t\tmb.clearCacheAndOffset()\n\t\t\tswitch kind {\n\t\t\tcase KindTruncateFull:\n\t\t\t\t// Truncate to be empty.\n\t\t\t\trequire_NoError(t, os.Truncate(mb.mfn, 0))\n\n\t\t\t\t// When loading messages, we should realize our in-memory state doesn't match what's on disk.\n\t\t\t\t// The block needs to be rebuilt and re-indexed.\n\t\t\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\t\t\t\trequire_Equal(t, mb.msgs, 0)\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 6)\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 5)\n\t\t\t\tfor i := range 5 {\n\t\t\t\t\tseq := uint64(i + 1)\n\t\t\t\t\t_, err = mb.cacheLookup(seq, nil)\n\t\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t\t}\n\t\t\tcase KindTruncatePartial:\n\t\t\t\t// Truncate to half of the original file.\n\t\t\t\tstat, err := os.Stat(mb.mfn)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, os.Truncate(mb.mfn, stat.Size()/2))\n\n\t\t\t\t// When loading messages, we should realize our in-memory state doesn't match what's on disk.\n\t\t\t\t// The block needs to be rebuilt and re-indexed.\n\t\t\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\t\t\t\trequire_Equal(t, mb.msgs, 2)\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 1)\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 2)\n\t\t\t\tfor i := range 5 {\n\t\t\t\t\tseq := uint64(i + 1)\n\t\t\t\t\t_, err = mb.cacheLookup(seq, nil)\n\t\t\t\t\tif seq <= 2 {\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase KindCompact, KindCompactHead, KindCompactTail:\n\t\t\t\t// Since we've reverted the state after compacting, our in-memory state doesn't match what's on disk.\n\t\t\t\t// The block needs to be rebuilt and re-indexed.\n\t\t\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\t\t\t\trequire_Equal(t, mb.msgs, uint64(5-deleted))\n\t\t\t\tif compactHead {\n\t\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 2)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), 1)\n\t\t\t\t}\n\t\t\t\tif compactTail {\n\t\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 4)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), 5)\n\t\t\t\t}\n\t\t\t\tfor i := range 5 {\n\t\t\t\t\tseq := uint64(i + 1)\n\t\t\t\t\t_, err = mb.cacheLookup(seq, nil)\n\t\t\t\t\tif seq == 1 && compactHead {\n\t\t\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t\t\t} else if seq == 5 && compactTail {\n\t\t\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tt.Run(\"TruncateFull\", func(t *testing.T) { test(t, KindTruncateFull) })\n\tt.Run(\"TruncatePartial\", func(t *testing.T) { test(t, KindTruncatePartial) })\n\tt.Run(\"CompactHead\", func(t *testing.T) { test(t, KindCompactHead) })\n\tt.Run(\"CompactTail\", func(t *testing.T) { test(t, KindCompactTail) })\n\tt.Run(\"Compact\", func(t *testing.T) { test(t, KindCompact) })\n}\n\nfunc TestFileStoreIndexCacheBufTombstoneMismatch(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor range 3 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.Lock()\n\t\tdefer mb.mu.Unlock()\n\n\t\t// Not only clear the cache, also clear the dmap.\n\t\t// indexCacheBuf should still properly index.\n\t\tmb.clearCacheAndOffset()\n\t\tmb.dmap.Empty()\n\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\n\t\tmbcache := mb.cache\n\t\trequire_NotNil(t, mbcache)\n\t\trequire_Len(t, len(mbcache.idx), 3)\n\t\trequire_Equal(t, mbcache.idx[0], 0)\n\t\trequire_Equal(t, mbcache.idx[1], 33)\n\t\trequire_Equal(t, mbcache.idx[2], 66)\n\t})\n}\n\nfunc TestFileStoreIndexCacheBufTombstoneMismatchAfterCompact(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor range 3 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, err = fs.RemoveMsg(2)\n\t\trequire_NoError(t, err)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.Lock()\n\t\tdefer mb.mu.Unlock()\n\n\t\trequire_NoError(t, mb.compactWithFloor(0, nil))\n\n\t\t// Not only clear the cache, also clear the dmap.\n\t\tmb.clearCacheAndOffset()\n\t\tmb.dmap.Empty()\n\t\trequire_NoError(t, mb.loadMsgsWithLock())\n\n\t\trequire_Equal(t, mb.dmap.Size(), 1)\n\t\trequire_True(t, mb.dmap.Exists(2))\n\t\tmbcache := mb.cache\n\t\trequire_NotNil(t, mbcache)\n\t\trequire_Len(t, len(mbcache.idx), 3)\n\t\trequire_Equal(t, mbcache.idx[0], 0)\n\t\t// Since the block was compacted, this will be both dbit and exist in the dmap which is checked above.\n\t\trequire_Equal(t, mbcache.idx[1], dbit)\n\t\t// Due to the compact, this message will be one entry earlier in the file than it was before.\n\t\trequire_Equal(t, mbcache.idx[2], 33)\n\t})\n}\n\nfunc TestFileStoreIndexCacheBufEraseMsgMismatch(t *testing.T) {\n\tfcfg := FileStoreConfig{Cipher: NoCipher, Compression: NoCompression, StoreDir: t.TempDir()}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfor range 3 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\t_, err = fs.EraseMsg(2)\n\trequire_NoError(t, err)\n\n\t// Revert the state in the filestore itself, as well as the block.\n\t// This simulates recovery based on a stale stream state file that should be recognized during runtime.\n\tbefore := fs.State()\n\tfs.mu.Lock()\n\tmsgs := fs.state.Msgs\n\tfs.state.Msgs = 3\n\tfs.mu.Unlock()\n\trequire_Equal(t, msgs, 2)\n\n\tmb := fs.getFirstBlock()\n\tmb.mu.Lock()\n\tdefer mb.mu.Unlock()\n\n\t// Not only clear the cache, also clear the dmap.\n\t// indexCacheBuf should still properly index.\n\tmb.clearCacheAndOffset()\n\tmb.dmap.Empty()\n\trequire_NoError(t, mb.loadMsgsWithLock())\n\n\tmbcache := mb.cache\n\trequire_NotNil(t, mbcache)\n\trequire_Len(t, len(mbcache.idx), 3)\n\trequire_Equal(t, mbcache.idx[0], 0)\n\trequire_Equal(t, mbcache.idx[1], 33)\n\trequire_Equal(t, mbcache.idx[2], 66)\n\n\t// This looks backwards but is intentional. The above asserts unlock based on the defer if they fail,\n\t// so need to do the reverse here while we inspect the filestore as a whole.\n\tmb.mu.Unlock()\n\tdefer mb.mu.Lock()\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\treturn fmt.Errorf(\"expected state\\n of %+v,\\n got %+v\", before, state)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestFileStoreCompactRestoresLastSeq(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar tss []int64\n\t\tfor range 4 {\n\t\t\t_, ts, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\ttss = append(tss, ts)\n\t\t}\n\n\t\tcheckMbState := func(fseq, lseq, msgs uint64) {\n\t\t\tt.Helper()\n\t\t\tmb := fs.getFirstBlock()\n\t\t\tmb.mu.Lock()\n\t\t\tdefer mb.mu.Unlock()\n\n\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), fseq)\n\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), lseq)\n\t\t\trequire_Equal(t, mb.first.ts, tss[fseq-1])\n\t\t\trequire_Equal(t, mb.last.ts, tss[lseq-1])\n\t\t\trequire_Equal(t, mb.msgs, msgs)\n\t\t\tdeletes := lseq - fseq + 1 - msgs\n\t\t\trequire_Equal(t, mb.dmap.Size(), int(deletes))\n\t\t}\n\t\tcheckMbState(1, 4, 4)\n\n\t\t_, err = fs.RemoveMsg(1)\n\t\trequire_NoError(t, err)\n\t\tcheckMbState(2, 4, 3)\n\n\t\t_, err = fs.RemoveMsg(4)\n\t\trequire_NoError(t, err)\n\t\tcheckMbState(2, 4, 2)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.Lock()\n\t\terr = mb.compactWithFloor(0, nil)\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t\tcheckMbState(2, 3, 2)\n\t})\n}\n\nfunc TestFileStoreCompactFullyResetsFirstAndLastSeq(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar tss []int64\n\t\tfor range 2 {\n\t\t\t_, ts, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\ttss = append(tss, ts)\n\t\t}\n\n\t\tcheckMbState := func(fseq, lseq, msgs uint64) {\n\t\t\tt.Helper()\n\t\t\tmb := fs.getFirstBlock()\n\t\t\tmb.mu.Lock()\n\t\t\tdefer mb.mu.Unlock()\n\n\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.first.seq), fseq)\n\t\t\trequire_Equal(t, mb.first.ts, tss[fseq-1])\n\t\t\tif lseq == 0 {\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), fseq-1)\n\t\t\t\trequire_Equal(t, mb.last.ts, 0)\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, atomic.LoadUint64(&mb.last.seq), lseq)\n\t\t\t\trequire_Equal(t, mb.last.ts, tss[lseq-1])\n\t\t\t}\n\t\t\trequire_Equal(t, mb.msgs, msgs)\n\t\t\tdeletes := lseq - fseq + 1 - msgs\n\t\t\trequire_Equal(t, mb.dmap.Size(), int(deletes))\n\t\t}\n\t\tcheckMbState(1, 2, 2)\n\n\t\tmb := fs.getFirstBlock()\n\t\t// Load the cache before compacting so that the compact itself needs to (re)load it.\n\t\trequire_NoError(t, mb.loadMsgs())\n\t\tmb.mu.Lock()\n\t\t// Manually 'delete' the whole block contents such that a call to rebuildState and indexCacheBuf\n\t\t// can't fix this. A call to compact needs to end with correctly resetting the first and last seq.\n\t\tmb.dmap.Insert(1)\n\t\tmb.dmap.Insert(2)\n\t\tmb.msgs = 0\n\t\terr = mb.compactWithFloor(0, nil)\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t\tcheckMbState(1, 0, 0)\n\t})\n}\n\nfunc TestFileStoreDoesntRebuildSubjectStateWithNoTrack(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\n\t\t// This implicitly was calling resetGlobalPerSubjectInfo and\n\t\t// populating \"foo\" back into the psim.\n\t\trequire_NoError(t, fs.Truncate(1))\n\n\t\tfs.mu.Lock()\n\t\tdefer fs.mu.Unlock()\n\t\trequire_Equal(t, fs.psim.Size(), 0)\n\t})\n}\n\nfunc TestFileStoreLoadNextMsgSkipAhead(t *testing.T) {\n\ttest := func(t *testing.T, headStart, singleSubjPerBlock bool) {\n\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}}\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tfor i := range 10 {\n\t\t\t\tsubj := \"a\"\n\t\t\t\tif !singleSubjPerBlock && i%2 == 0 {\n\t\t\t\t\tsubj = \"c\"\n\t\t\t\t}\n\t\t\t\t_, _, err = fs.StoreMsg(subj, nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif (i+1)%2 == 0 {\n\t\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"b\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfs.mu.Lock()\n\t\t\tfs.lockAllMsgBlocks()\n\t\t\tfor _, mb := range fs.blks {\n\t\t\t\tmb.lsts = 0\n\t\t\t}\n\t\t\tfs.unlockAllMsgBlocks()\n\t\t\tfs.mu.Unlock()\n\n\t\t\tvar start uint64\n\t\t\tif !headStart {\n\t\t\t\tstart = 3\n\t\t\t}\n\n\t\t\t// Check we properly skip over blocks that aren't relevant for our filter subject.\n\t\t\t// If we're starting with start=0, we should skip to the correct block immediately.\n\t\t\t// Otherwise, we should only need to access one block and then skip to the correct one.\n\t\t\tsm, nseq, err := fs.LoadNextMsg(\"b\", false, start, nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.seq, 11)\n\t\t\trequire_Equal(t, nseq, 11)\n\n\t\t\tfs.mu.Lock()\n\t\t\tdefer fs.mu.Unlock()\n\t\t\tfs.lockAllMsgBlocks()\n\t\t\tdefer fs.unlockAllMsgBlocks()\n\t\t\tfor _, mb := range fs.blks {\n\t\t\t\tt.Logf(\"mb %d: lsts=%d\", mb.index, mb.lsts)\n\t\t\t\tif mb.index == 6 || (mb.index == 2 && !headStart) {\n\t\t\t\t\trequire_NotEqual(t, mb.lsts, 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, mb.lsts, 0)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tfor _, singleSubjPerBlock := range []bool{true, false} {\n\t\ttitle := \"Single\"\n\t\tif !singleSubjPerBlock {\n\t\t\ttitle = \"Multi\"\n\t\t}\n\t\tt.Run(fmt.Sprintf(\"%s/Head\", title), func(t *testing.T) { test(t, true, singleSubjPerBlock) })\n\t\tt.Run(fmt.Sprintf(\"%s/Interior\", title), func(t *testing.T) { test(t, false, singleSubjPerBlock) })\n\t}\n}\n\nfunc TestFileStoreLoadNextMsgMultiSkipAhead(t *testing.T) {\n\ttest := func(t *testing.T, headStart, singleSubjPerBlock bool) {\n\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}}\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tfor i := range 10 {\n\t\t\t\tsubj := \"a\"\n\t\t\t\tif !singleSubjPerBlock && i%2 == 0 {\n\t\t\t\t\tsubj = \"c\"\n\t\t\t\t}\n\t\t\t\t_, _, err = fs.StoreMsg(subj, nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif (i+1)%2 == 0 {\n\t\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t_, _, err = fs.StoreMsg(\"b\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfs.mu.Lock()\n\t\t\tfs.lockAllMsgBlocks()\n\t\t\tfor _, mb := range fs.blks {\n\t\t\t\tmb.lsts = 0\n\t\t\t}\n\t\t\tfs.unlockAllMsgBlocks()\n\t\t\tfs.mu.Unlock()\n\n\t\t\tvar start uint64\n\t\t\tif !headStart {\n\t\t\t\tstart = 3\n\t\t\t}\n\n\t\t\t// Check we properly skip over blocks that aren't relevant for our filter subjects.\n\t\t\t// If we're starting with start=0, we should skip to the correct block immediately.\n\t\t\t// Otherwise, we should only need to access one block and then skip to the correct one.\n\t\t\tfilters := gsl.NewSublist[struct{}]()\n\t\t\trequire_NoError(t, filters.Insert(\"b\", struct{}{}))\n\t\t\trequire_NoError(t, filters.Insert(\"d\", struct{}{}))\n\t\t\tsm, nseq, err := fs.LoadNextMsgMulti(filters, start, nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.seq, 11)\n\t\t\trequire_Equal(t, nseq, 11)\n\n\t\t\tfs.mu.Lock()\n\t\t\tdefer fs.mu.Unlock()\n\t\t\tfs.lockAllMsgBlocks()\n\t\t\tdefer fs.unlockAllMsgBlocks()\n\t\t\tfor _, mb := range fs.blks {\n\t\t\t\tt.Logf(\"mb %d: lsts=%d\", mb.index, mb.lsts)\n\t\t\t\tif mb.index == 6 || (mb.index == 2 && !headStart) {\n\t\t\t\t\trequire_NotEqual(t, mb.lsts, 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, mb.lsts, 0)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tfor _, singleSubjPerBlock := range []bool{true, false} {\n\t\ttitle := \"Single\"\n\t\tif !singleSubjPerBlock {\n\t\t\ttitle = \"Multi\"\n\t\t}\n\t\tt.Run(fmt.Sprintf(\"%s/Head\", title), func(t *testing.T) { test(t, true, singleSubjPerBlock) })\n\t\tt.Run(fmt.Sprintf(\"%s/Interior\", title), func(t *testing.T) { test(t, false, singleSubjPerBlock) })\n\t}\n}\n\nfunc TestFileStoreDeleteRangeTwoGaps(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 256,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckDeleteBlocks := func(exp DeleteBlocks) {\n\t\tt.Helper()\n\t\tdBlocks := fs.deleteBlocks()\n\t\trequire_Equal(t, len(exp), len(dBlocks))\n\n\t\tfor i, found := range dBlocks {\n\t\t\tef, el, en := exp[i].State()\n\t\t\tff, fl, fn := found.State()\n\n\t\t\trequire_Equal(t, reflect.TypeOf(exp[i]), reflect.TypeOf(found))\n\t\t\trequire_Equal(t, ef, ff)\n\t\t\trequire_Equal(t, el, fl)\n\t\t\trequire_Equal(t, en, fn)\n\t\t}\n\t}\n\n\tmsg := make([]byte, 256)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Initially, we have sequences [1,20].\n\t// Each message fills one entire block\n\trequire_Equal(t, len(fs.blks), 20)\n\n\t// Make two gaps by removing sequences 10 and 15\n\t// [1,9] [11,14] [16,20]\n\tfs.RemoveMsg(10)\n\tfs.RemoveMsg(15)\n\n\t// If bug is present: deleteBlocks extends the first gap\n\t// to the second, even though non-deleted blocks are in\n\t// between the two gaps. Expect two distinct gaps.\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 10, Num: 1},\n\t\t&DeleteRange{First: 15, Num: 1},\n\t})\n}\n\nfunc TestFileStoreDeleteBlocksWithSingleMessageBlocks(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 256,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckDeleteBlocks := func(exp DeleteBlocks) {\n\t\tdBlocks := fs.deleteBlocks()\n\t\trequire_Equal(t, len(exp), len(dBlocks))\n\n\t\tfor i, found := range dBlocks {\n\t\t\tef, el, en := exp[i].State()\n\t\t\tff, fl, fn := found.State()\n\n\t\t\trequire_Equal(t, reflect.TypeOf(exp[i]), reflect.TypeOf(found))\n\t\t\trequire_Equal(t, ef, ff)\n\t\t\trequire_Equal(t, el, fl)\n\t\t\trequire_Equal(t, en, fn)\n\t\t}\n\t}\n\n\tmsg := make([]byte, 256)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Initially, we have sequences [1,20].\n\t// Each message fills one entire block\n\trequire_Equal(t, len(fs.blks), 20)\n\n\t// Removing sequence 10 leaves a gap\n\t// [1,9] [11,20]\n\tfs.RemoveMsg(10)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 10, Num: 1},\n\t})\n\n\t// Extend the gap by removing 11\n\t// [1,9] [12,20]\n\tfs.RemoveMsg(11)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 10, Num: 2},\n\t})\n\n\t// Make another gap by removing 5\n\t// [1,4] [6,9] [12,20]\n\tfs.RemoveMsg(5)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 1},\n\t\t&DeleteRange{First: 10, Num: 2},\n\t})\n\n\t// Remove blocks in the middle\n\t// [1,4] [12,20]\n\tfs.RemoveMsg(6)\n\tfs.RemoveMsg(7)\n\tfs.RemoveMsg(8)\n\tfs.RemoveMsg(9)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 7},\n\t})\n\n\t// Make a couple more gaps by removing 3 and 16,17,18\n\t// [1,2] [4,4] [12,15] [19,20]\n\tfs.RemoveMsg(3)\n\tfs.RemoveMsg(16)\n\tfs.RemoveMsg(17)\n\tfs.RemoveMsg(18)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 3, Num: 1},\n\t\t&DeleteRange{First: 5, Num: 7},\n\t\t&DeleteRange{First: 16, Num: 3},\n\t})\n\n\t// Removing the first sequence make no difference in gaps\n\t// [2,2] [4,4] [12,15] [19,20]\n\tfs.RemoveMsg(1)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 3, Num: 1},\n\t\t&DeleteRange{First: 5, Num: 7},\n\t\t&DeleteRange{First: 16, Num: 3},\n\t})\n\n\t// Removing the last sequence creates a gap\n\t// [2,2] [4,4] [12,15] [19,19] (empty block starting at 21)\n\tfs.RemoveMsg(20)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 3, Num: 1},\n\t\t&DeleteRange{First: 5, Num: 7},\n\t\t&DeleteRange{First: 16, Num: 3},\n\t\t&DeleteRange{First: 20, Num: 1},\n\t})\n\n\t// Extend the last gap by removing 19\n\t// [2,2] [4,4] [12,15] (empty block starting at 21)\n\tfs.RemoveMsg(19)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 3, Num: 1},\n\t\t&DeleteRange{First: 5, Num: 7},\n\t\t&DeleteRange{First: 16, Num: 5},\n\t})\n}\n\nfunc makeSequenceSet(seqs []uint64) *avl.SequenceSet {\n\tvar set avl.SequenceSet\n\tfor _, seq := range seqs {\n\t\tset.Insert(seq)\n\t}\n\treturn &set\n}\n\nfunc TestFileStoreDeleteBlocks(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 256,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckDeleteBlocks := func(exp DeleteBlocks) {\n\t\tdBlocks := fs.deleteBlocks()\n\t\trequire_Equal(t, len(exp), len(dBlocks))\n\n\t\tfor i, found := range dBlocks {\n\t\t\tef, el, en := exp[i].State()\n\t\t\tff, fl, fn := found.State()\n\n\t\t\trequire_Equal(t, reflect.TypeOf(exp[i]), reflect.TypeOf(found))\n\t\t\trequire_Equal(t, ef, ff)\n\t\t\trequire_Equal(t, el, fl)\n\t\t\trequire_Equal(t, en, fn)\n\t\t}\n\t}\n\n\tmsg := make([]byte, 20)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// We have 5 blocks, with 4 messages each.\n\t// block 1: [1,4]\n\t// block 2: [5,8]\n\t// block 3: [9,12]\n\t// block 4: [13,16]\n\t// block 5: [17,20]\n\trequire_Equal(t, len(fs.blks), 5)\n\n\t// Removing sequence 9, creates a gap between blocks 2 and 3\n\t// block 1: [1,4]\n\t// block 2: [5,8]\n\t// block 3: [10,12]\n\t// block 4: [13,16]\n\t// block 5: [17,20]\n\tfs.RemoveMsg(9)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 9, Num: 1},\n\t})\n\n\t// Extend the gap by removing 10\n\t// block 1: [1,4]\n\t// block 2: [5,8]\n\t// block 3: [11,12]\n\t// block 4: [13,16]\n\t// block 5: [17,20]\n\tfs.RemoveMsg(10)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 9, Num: 2},\n\t})\n\n\t// Extend the gap by removing 8, creates an interior delete\n\t// block 1: [1,4]\n\t// block 2: [5,7]\n\t// block 3: [11,12]\n\t// block 4: [13,16]\n\t// block 5: [17,20]\n\tfs.RemoveMsg(8)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\tmakeSequenceSet([]uint64{8}),\n\t\t&DeleteRange{First: 9, Num: 2},\n\t})\n\n\t// Make another gap by removing 17 and 18\n\t// block 1: [1,4]\n\t// block 2: [5,7]\n\t// block 3: [11,12]\n\t// block 4: [13,16]\n\t// block 5: [19,20]\n\tfs.RemoveMsg(18)\n\tfs.RemoveMsg(17)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\tmakeSequenceSet([]uint64{8}),\n\t\t&DeleteRange{First: 9, Num: 2},\n\t\t&DeleteRange{First: 17, Num: 2},\n\t})\n\n\t// Remove messages 5, 6, and 7 to create a fully deleted block\n\t// block 1: [1,4]\n\t// block 2: [11,12]\n\t// block 3: [13,16]\n\t// block 4: [19,20]\n\tfs.RemoveMsg(6)\n\tfs.RemoveMsg(5)\n\tfs.RemoveMsg(7)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 6},\n\t\t&DeleteRange{First: 17, Num: 2},\n\t})\n\n\t// Remove the last message\n\t// block 1: [1,4]\n\t// block 2: [11,12]\n\t// block 3: [13,16]\n\t// block 4: [19,19]\n\tfs.RemoveMsg(20)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 6},\n\t\t&DeleteRange{First: 17, Num: 2},\n\t\tmakeSequenceSet([]uint64{20}),\n\t})\n\n\t// Remove the last message\n\t// block 1: [1,4]\n\t// block 2: [11,12]\n\t// block 3: [13,16]\n\tfs.RemoveMsg(19)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 6},\n\t\t&DeleteRange{First: 17, Num: 4},\n\t})\n\n\t// Remove block in the middle\n\t// block 1: [1,4]\n\t// block 2: [13,16]\n\tfs.RemoveMsg(11)\n\tfs.RemoveMsg(12)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 5, Num: 8},\n\t\t&DeleteRange{First: 17, Num: 4},\n\t})\n\n\t// Remove the first block\n\t// block 1: [13,16]\n\tfs.RemoveMsg(1)\n\tfs.RemoveMsg(2)\n\tfs.RemoveMsg(3)\n\tfs.RemoveMsg(4)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 17, Num: 4},\n\t})\n}\n\nfunc TestFileStoreDeleteBlocksWithManyEmptyBlocks(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 33 * 6,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcheckDeleteBlocks := func(exp DeleteBlocks) {\n\t\tt.Helper()\n\t\tdBlocks := fs.deleteBlocks()\n\t\trequire_Equal(t, len(exp), len(dBlocks))\n\n\t\tfor i, found := range dBlocks {\n\t\t\tef, el, en := exp[i].State()\n\t\t\tff, fl, fn := found.State()\n\n\t\t\trequire_Equal(t, reflect.TypeOf(exp[i]), reflect.TypeOf(found))\n\t\t\trequire_Equal(t, ef, ff)\n\t\t\trequire_Equal(t, el, fl)\n\t\t\trequire_Equal(t, en, fn)\n\t\t}\n\t}\n\n\t// 1.blk, 6 msgs\n\tfor range 6 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\t// *.blk, tombstone for 1.blk + 2 msgs\n\t// every next block adds 2 tombstones for the previous block\n\tfor i := range 5 {\n\t\t_, err = fs.newMsgBlockForWrite()\n\t\trequire_NoError(t, err)\n\n\t\t// Every iteration will remove one message from 1.blk\n\t\t_, err = fs.RemoveMsg(uint64(i + 2))\n\t\trequire_NoError(t, err)\n\n\t\t// Every block after the first removes two messages from the previous block.\n\t\tif i > 0 {\n\t\t\tfor j := range 2 {\n\t\t\t\t_, err = fs.RemoveMsg(uint64(7 + (i-1)*2 + j))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\t\t// Every block except for the last stores two new messages for the next block to delete.\n\t\tif i < 4 {\n\t\t\tfor range 2 {\n\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\tmakeSequenceSet([]uint64{2, 3, 4, 5, 6}),\n\t\t&DeleteRange{First: 7, Num: 8},\n\t})\n\n\tfs.mu.Lock()\n\tfs.lockAllMsgBlocks()\n\tfor _, mb := range fs.blks {\n\t\tmb.compact()\n\t}\n\tfs.unlockAllMsgBlocks()\n\tfs.mu.Unlock()\n\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 2, Num: 13},\n\t})\n}\n\nfunc TestFileStoreTrailingSkipMsgsFromStreamStateFile(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}}\n\t\tcreated := time.Now()\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor range 10 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfs.mu.RLock()\n\t\tlmb := fs.lmb\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.RLock()\n\t\tmfn := lmb.mfn\n\t\tlmb.mu.RUnlock()\n\t\trequire_NotEqual(t, mfn, _EMPTY_)\n\n\t\t// Stop the store and truncate the block file.\n\t\t// Stopping should write out our stream state file, letting us \"remember\" the highest last sequence.\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Truncate(mfn, 33*5))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\t// We should recover with the right state, the last sequence coming from the stream state file,\n\t\t// and the messages from the recovered blocks.\n\t\tstate := fs.State()\n\t\trequire_Equal(t, state.Msgs, 5)\n\t\trequire_Equal(t, state.FirstSeq, 1)\n\t\trequire_Equal(t, state.LastSeq, 10)\n\n\t\t// Go through all messages, they should properly report deletes.\n\t\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\t\t_, err = fs.LoadMsg(seq, nil)\n\t\t\tif seq <= 5 {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t} else {\n\t\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\t\t\t}\n\t\t}\n\n\t\t// Also check a new block was created to represent this.\n\t\tfs.mu.RLock()\n\t\tlmb = fs.lmb\n\t\tlblks := len(fs.blks)\n\t\tfs.mu.RUnlock()\n\t\tlmb.mu.RLock()\n\t\tldmap := lmb.dmap.Size()\n\t\tlmb.mu.RUnlock()\n\t\trequire_Len(t, lblks, 2)\n\t\trequire_Len(t, ldmap, 0)\n\t\trequire_Equal(t, atomic.LoadUint64(&lmb.first.seq), 11)\n\t\trequire_Equal(t, atomic.LoadUint64(&lmb.last.seq), 10)\n\t})\n}\n\nfunc TestFileStoreSelectMsgBlockBinarySearch(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tfor i := range 33 {\n\t\t\tif i > 0 {\n\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tfor range 2 {\n\t\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tif i == 15 {\n\t\t\t\tfor _, seq := range []uint64{2, 5} {\n\t\t\t\t\t_, err = fs.newMsgBlockForWrite()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t_, err = fs.RemoveMsg(seq)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\ttest := func() {\n\t\t\t// Select a block containing a message we deleted.\n\t\t\ti, mb := fs.selectMsgBlockWithIndex(2)\n\t\t\trequire_Equal(t, i, 0)\n\t\t\trequire_Equal(t, mb.index, 1)\n\n\t\t\t// Once more for the other deleted message.\n\t\t\ti, mb = fs.selectMsgBlockWithIndex(5)\n\t\t\trequire_Equal(t, i, 2)\n\t\t\trequire_Equal(t, mb.index, 3)\n\n\t\t\t// Select the first block/message.\n\t\t\ti, mb = fs.selectMsgBlockWithIndex(1)\n\t\t\trequire_Equal(t, i, 0)\n\t\t\trequire_Equal(t, mb.index, 1)\n\n\t\t\t// Select a block before the delete block, but after the deleted sequences.\n\t\t\ti, mb = fs.selectMsgBlockWithIndex(10)\n\t\t\trequire_Equal(t, i, 4)\n\t\t\trequire_Equal(t, mb.index, 5)\n\n\t\t\t// Select a block after the delete block AND after the deleted sequences.\n\t\t\ti, mb = fs.selectMsgBlockWithIndex(64)\n\t\t\trequire_Equal(t, i, 33)\n\t\t\trequire_Equal(t, mb.index, 34)\n\t\t}\n\t\ttest()\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\ttest()\n\t})\n}\n\nfunc TestFileStoreRemoveMsgsInRange(t *testing.T) {\n\tfcfg := FileStoreConfig{Cipher: NoCipher, Compression: NoCompression, StoreDir: t.TempDir(), BlockSize: 256}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := make([]byte, 256)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\tcheckDeleteBlocks := func(exp DeleteBlocks) {\n\t\tdBlocks := fs.deleteBlocks()\n\t\trequire_Equal(t, len(exp), len(dBlocks))\n\n\t\tfor i, found := range dBlocks {\n\t\t\tef, el, en := exp[i].State()\n\t\t\tff, fl, fn := found.State()\n\n\t\t\trequire_Equal(t, reflect.TypeOf(exp[i]), reflect.TypeOf(found))\n\t\t\trequire_Equal(t, ef, ff)\n\t\t\trequire_Equal(t, el, fl)\n\t\t\trequire_Equal(t, en, fn)\n\t\t}\n\t}\n\n\trequire_Equal(t, len(fs.blks), 20)\n\n\t// Remove range [1,1]\n\tfs.removeMsgsInRange(1, 1, true)\n\trequire_Equal(t, len(fs.blks), 19)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 2)\n\n\t// Removing range [1,1] again is a noop\n\t// We are left with blocks [2,20]\n\tfs.removeMsgsInRange(1, 1, true)\n\trequire_Equal(t, len(fs.blks), 19)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 2)\n\n\t// Remove the range [1,2] should remove block with sequence 2\n\t// We are left with blocks [3,20]\n\tfs.removeMsgsInRange(1, 2, true)\n\trequire_Equal(t, len(fs.blks), 18)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 3)\n\n\t// Remove the first two blocks [3,4]\n\t// We are left with blocks [5,20]\n\tfs.removeMsgsInRange(3, 4, true)\n\trequire_Equal(t, len(fs.blks), 16)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 5)\n\n\t// Remove range [9, 13]\n\t// We are left with [5,8] [14,20]\n\tfs.removeMsgsInRange(9, 13, true)\n\trequire_Equal(t, len(fs.blks), 11)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 9, Num: 5},\n\t})\n\n\t// Make the gap larger by removing range [8 8]\n\t// We are left with [5,7] [14, 20]\n\tfs.removeMsgsInRange(8, 8, true)\n\trequire_Equal(t, len(fs.blks), 10)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 8, Num: 6},\n\t})\n\n\t// Make another gap by removing range [17, 17]\n\t// We are left with [5,7] [14,16] [18,20]\n\tfs.removeMsgsInRange(17, 17, true)\n\trequire_Equal(t, len(fs.blks), 9)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 8, Num: 6},\n\t\t&DeleteRange{First: 17, Num: 1},\n\t})\n\n\t// Remove the last block\n\t// We are left with [5,7] [14,16] [18,19] (empty block 21-20)\n\tfs.removeMsgsInRange(20, 20, true)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 8, Num: 6},\n\t\t&DeleteRange{First: 17, Num: 1},\n\t\t&DeleteRange{First: 20, Num: 1},\n\t})\n\n\t// Make a big gap removing range [7, 18]\n\t// We are left with [5,6] [19]\n\tfs.removeMsgsInRange(7, 18, true)\n\trequire_Equal(t, len(fs.blks), 4)\n\tcheckDeleteBlocks(DeleteBlocks{\n\t\t&DeleteRange{First: 7, Num: 12},\n\t\t&DeleteRange{First: 20, Num: 1},\n\t})\n\n\t// Remove everything\n\t// We are left with an empty block\n\tfs.removeMsgsInRange(1, 20, true)\n\trequire_Equal(t, len(fs.blks), 1)\n\trequire_Equal(t, fs.blks[0].msgs, 0)\n}\n\nfunc TestFileStoreRemoveMsgsInRangePartialBlocks(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 256,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := make([]byte, 16)\n\tfor range 20 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfs.mu.Lock()\n\tdefer fs.mu.Unlock()\n\n\t// Initially:\n\t// block 0 [ 1, 5]\n\t// block 1 [ 6,10]\n\t// block 2 [11,15]\n\t// block 3 [16,20]\n\trequire_Equal(t, len(fs.blks), 4)\n\n\t// Remove [5,8] leaves:\n\t// block 0 [ 1, 4]\n\t// block 1 [ 9,10]\n\t// block 2 [11,15]\n\t// block 3 [16,20]\n\tfs.removeMsgsInRange(5, 6, true)\n\tfs.blks[0].compact()\n\n\trequire_Equal(t, len(fs.blks), 4)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].last.seq), 4)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].first.seq), 7)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].last.seq), 10)\n\n\t// Remove [5,8] leaves:\n\t// block 0 [ 1, 3]\n\t// block 1 [12,15]\n\t// block 2 [16,20]\n\t// empty block 21 20\n\tfs.removeMsgsInRange(4, 11, true)\n\tfs.blks[0].compact()\n\n\trequire_Equal(t, len(fs.blks), 3)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].last.seq), 3)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].first.seq), 12)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].last.seq), 15)\n\n\t// Remove [13,14] leaves:\n\t// block 0 [ 1, 3]\n\t// block 1 12 and 15\n\t// block 2 [16,20]\n\tfs.removeMsgsInRange(13, 14, true)\n\tfs.blks[1].compact()\n\n\trequire_Equal(t, len(fs.blks), 3)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].last.seq), 3)\n\trequire_Equal(t, fs.blks[1].msgs, 2)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].first.seq), 12)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].last.seq), 15)\n\n\t// Remove [13,20] leaves:\n\t// block 0 [ 1, 3]\n\t// block 1 12\n\t// empty block 21 20\n\tfs.removeMsgsInRange(13, 20, true)\n\tfs.blks[1].compact()\n\n\trequire_Equal(t, len(fs.blks), 3)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].last.seq), 3)\n\trequire_Equal(t, fs.blks[1].msgs, 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].first.seq), 12)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[1].last.seq), 12)\n\n\t// Remove [10,20] leaves:\n\t// block 0 [ 1, 3]\n\t// empty block 21 20\n\tfs.removeMsgsInRange(10, 20, true)\n\trequire_Equal(t, len(fs.blks), 2)\n\n\t// Remove everything\n\t// empty block 21 20\n\tfs.removeMsgsInRange(1, 30, true)\n\n\trequire_Equal(t, len(fs.blks), 1)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].first.seq), 21)\n\trequire_Equal(t, atomic.LoadUint64(&fs.blks[0].last.seq), 20)\n}\n\nfunc TestFileStoreRemoveMsgsInRangeWithTombstones(t *testing.T) {\n\tfcfg := FileStoreConfig{\n\t\tCipher: NoCipher,\n\t\tCompression: NoCompression,\n\t\tStoreDir: t.TempDir(),\n\t\tBlockSize: 256,\n\t}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Block 1: [1,4]\n\tfor range 4 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Block 2: [5,8]\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\tfor range 4 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Block 3 [9,12] - tombs {4,3,2,10}\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\tfor range 4 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfs.RemoveMsg(4)\n\tfs.RemoveMsg(3)\n\tfs.RemoveMsg(2)\n\tfs.RemoveMsg(10)\n\n\t// Block 4 [13, 16] - tombs {14,15}\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\tfor range 4 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\tfs.RemoveMsg(14)\n\tfs.RemoveMsg(15)\n\n\t// Block 5 [17,17]\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\tfor range 4 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tcheckBlock := func(mb *msgBlock, fseq, lseq uint64, tombs []uint64) {\n\t\tt.Helper()\n\t\trequire_Equal(t, mb.first.seq, fseq)\n\t\trequire_Equal(t, mb.last.seq, lseq)\n\t\tmbTombs := make([]uint64, len(mb.tombs()))\n\t\tfor i, id := range mb.tombs() {\n\t\t\tmbTombs[i] = id.seq\n\t\t}\n\t\trequire_True(t, slices.Equal(tombs, mbTombs))\n\t}\n\n\trequire_Equal(t, len(fs.blks), 5)\n\tcheckBlock(fs.blks[0], 1, 4, nil)\n\tcheckBlock(fs.blks[1], 5, 8, nil)\n\tcheckBlock(fs.blks[2], 9, 12, []uint64{4, 3, 2, 10})\n\tcheckBlock(fs.blks[3], 13, 16, []uint64{14, 15})\n\tcheckBlock(fs.blks[4], 17, 20, nil)\n\n\t// for i, mb := range fs.blks {\n\t// \tmbFirstSeq := atomic.LoadUint64(&mb.first.seq)\n\t// \tmbLastSeq := atomic.LoadUint64(&mb.last.seq)\n\t// \tt.Log(i, mb.msgs, mbFirstSeq, mbLastSeq, mb.tombs())\n\t// }\n\n\t// Remove sequences 4-17\n\t// Block 1: removing the tail, noop because already deleted\n\t// Block 2: has no prior tombs, should be purged entirely\n\t// Block 3: has prior tombs, should be preserved\n\t// Block 4: has \"internal\" tombs, can be purged entirely\n\t// Block 5: updates mb first\n\tfs.removeMsgsInRange(4, 17, false)\n\n\tcheckBlock(fs.blks[0], 1, 4, nil)\n\tcheckBlock(fs.blks[1], 13, 12, []uint64{4, 3, 2, 10})\n\tcheckBlock(fs.blks[2], 18, 20, []uint64{9, 11, 17})\n\n\t// Remove everything (past the last sequence)\n\tfs.removeMsgsInRange(1, 100, false)\n\trequire_Equal(t, len(fs.blks), 1)\n\tcheckBlock(fs.blks[0], 21, 20, []uint64{20})\n\n\t// Attempt to remove the empty block\n\tfs.removeMsgsInRange(1, 100, false)\n\trequire_Equal(t, len(fs.blks), 1)\n\tcheckBlock(fs.blks[0], 21, 20, []uint64{20})\n}\n\nfunc TestFileStoreCorrectChecksumAfterTruncate(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tvar fchkCmp []byte\n\t\tvar fchk [8]byte\n\t\tfor i := range 5 {\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tif i == 2 {\n\t\t\t\tmb := fs.getFirstBlock()\n\t\t\t\tmb.mu.RLock()\n\t\t\t\tfchkCmp = mb.lastChecksum()\n\t\t\t\tcopy(fchk[:], mb.lchk[:])\n\t\t\t\tmb.mu.RUnlock()\n\t\t\t}\n\t\t}\n\t\trequire_True(t, bytes.Equal(fchkCmp, fchk[:]))\n\n\t\t// After truncating we should return to the checksum we stored above.\n\t\trequire_NoError(t, fs.Truncate(3))\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.RLock()\n\t\tlchkCmp := mb.lastChecksum()\n\t\tvar lchk [8]byte\n\t\tcopy(lchk[:], mb.lchk[:])\n\t\tmb.mu.RUnlock()\n\t\trequire_True(t, bytes.Equal(lchkCmp, lchk[:]))\n\t\trequire_True(t, bytes.Equal(fchkCmp, lchkCmp))\n\t})\n}\n\nfunc TestFileStoreRecoverTTLAndScheduleStateAndCounters(t *testing.T) {\n\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\tcreated := time.Now()\n\t\tcfg := StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}, AllowMsgTTL: true, AllowMsgSchedules: true}\n\t\tfs, err := newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tttl := int64(5)\n\t\tsched := time.Now().Add(5 * time.Second)\n\t\thdr := genHeader(nil, JSMessageTTL, strconv.FormatInt(ttl, 10))\n\t\thdr = genHeader(hdr, JSSchedulePattern, fmt.Sprintf(\"@at %s\", sched.Format(time.RFC3339)))\n\t\t_, _, err = fs.StoreMsg(\"foo\", hdr, nil, ttl)\n\t\trequire_NoError(t, err)\n\n\t\tmb := fs.getFirstBlock()\n\t\tmb.mu.RLock()\n\t\tttls := mb.ttls\n\t\tschedules := mb.schedules\n\t\tmb.mu.RUnlock()\n\t\trequire_Equal(t, ttls, 1)\n\t\trequire_Equal(t, schedules, 1)\n\n\t\trequire_NoError(t, fs.Stop())\n\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\n\t\tmb = fs.getFirstBlock()\n\t\tmb.mu.RLock()\n\t\tttls = mb.ttls\n\t\tschedules = mb.schedules\n\t\tmb.mu.RUnlock()\n\t\trequire_Equal(t, ttls, 1)\n\t\trequire_Equal(t, schedules, 1)\n\t})\n}\n\nfunc TestFileStoreCorruptionSetsHbitWithoutHeaders(t *testing.T) {\n\tconst (\n\t\tKindMsgFromBuf = iota\n\t\tKindIndexCacheBuf\n\t\tKindRebuildState\n\t)\n\ttest := func(t *testing.T, kind int) {\n\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage, Subjects: []string{\">\"}}, time.Now(), prf(&fcfg), nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tmb := fs.getFirstBlock()\n\t\t\tfs.mu.Lock()\n\t\t\tmb.mu.Lock()\n\t\t\terr = mb.writeMsgRecordLocked(emptyRecordLen|hbit, 1, _EMPTY_, nil, nil, 0, false, false)\n\t\t\tcache := mb.ecache.Value()\n\t\t\tmb.mu.Unlock()\n\t\t\tfs.mu.Unlock()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NotNil(t, cache)\n\n\t\t\tswitch kind {\n\t\t\tcase KindMsgFromBuf:\n\t\t\t\tmb.mu.Lock()\n\t\t\t\t_, err = mb.msgFromBufNoCopy(cache.buf, nil, mb.hh)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\trequire_True(t, strings.Contains(err.Error(), \"sanity check failed\"))\n\t\t\tcase KindIndexCacheBuf:\n\t\t\t\tmb.mu.Lock()\n\t\t\t\terr = mb.indexCacheBuf(cache.buf)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\trequire_Error(t, err, errCorruptState)\n\t\t\tcase KindRebuildState:\n\t\t\t\tmb.mu.Lock()\n\t\t\t\t_, _, err = mb.rebuildStateFromBufLocked(cache.buf, false)\n\t\t\t\tmb.mu.Unlock()\n\t\t\t\trequire_True(t, err != nil)\n\t\t\t\trequire_True(t, strings.Contains(err.Error(), \"sanity check failed\"))\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"unknown kind %d\", kind)\n\t\t\t}\n\t\t})\n\t}\n\tt.Run(\"msgFromBuf\", func(t *testing.T) { test(t, KindMsgFromBuf) })\n\tt.Run(\"indexCacheBuf\", func(t *testing.T) { test(t, KindIndexCacheBuf) })\n\tt.Run(\"rebuildState\", func(t *testing.T) { test(t, KindRebuildState) })\n}\n\nfunc TestFileStoreSyncBlocksNoErrorOnConcurrentRemovedBlock(t *testing.T) {\n\tfcfg := FileStoreConfig{Cipher: NoCipher, Compression: NoCompression, StoreDir: t.TempDir()}\n\tfs, err := newFileStoreWithCreated(fcfg, StreamConfig{Name: \"zzz\", Storage: FileStorage}, time.Now(), prf(&fcfg), nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfs.mu.Lock()\n\tif fs.syncTmr != nil {\n\t\tfs.syncTmr.Stop()\n\t\tfs.syncTmr = nil\n\t}\n\tfs.mu.Unlock()\n\n\t// 1.blk\n\tfor range 2 {\n\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// 2.blk\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\trequire_NoError(t, err)\n\t_, err = fs.RemoveMsg(2)\n\trequire_NoError(t, err)\n\n\t// 3.blk\n\t_, err = fs.newMsgBlockForWrite()\n\trequire_NoError(t, err)\n\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\trequire_NoError(t, err)\n\t_, err = fs.RemoveMsg(3)\n\trequire_NoError(t, err)\n\n\tfs.mu.RLock()\n\tmb := fs.blks[1]\n\tfs.mu.RUnlock()\n\trequire_Equal(t, mb.index, 2)\n\tmb.mu.RLock()\n\tshouldCompactSync := mb.shouldCompactSync()\n\tmb.mu.RUnlock()\n\trequire_True(t, shouldCompactSync)\n\n\tvar ready sync.WaitGroup\n\tvar wg sync.WaitGroup\n\tready.Add(1)\n\twg.Add(1)\n\tmb.mu.Lock()\n\tgo func() {\n\t\tready.Done()\n\t\tdefer wg.Done()\n\t\t// Wait some time for fs.syncBlocks to wait on the mb's lock.\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\t// Remove the block while fs.syncBlocks is still waiting.\n\t\tfs.mu.Lock()\n\t\terr = fs.forceRemoveMsgBlock(mb)\n\t\tfs.mu.Unlock()\n\t\tmb.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}()\n\n\tready.Wait()\n\tfs.syncBlocks()\n\twg.Wait()\n\n\tfs.mu.RLock()\n\tdefer fs.mu.RUnlock()\n\tmb.mu.RLock()\n\tdefer mb.mu.RUnlock()\n\trequire_True(t, mb.werr == nil)\n\trequire_True(t, fs.werr == nil)\n}\n\nfunc TestFileStoreNoDirectoryNotEmptyError(t *testing.T) {\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir()}\n\tmconfig := StreamConfig{Name: \"TEST_DELETE\", Storage: FileStorage, Subjects: []string{\"foo\"}, Replicas: 1}\n\tfs, err := newFileStore(fcfg, mconfig)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\toconfig := ConsumerConfig{\n\t\tDeliverSubject: \"d\",\n\t\tFilterSubject: \"foo\",\n\t\tAckPolicy: AckExplicit,\n\t}\n\n\tfor i := range 100 {\n\t\toname := fmt.Sprintf(\"delcons_%d\", i)\n\t\tobs, err := fs.ConsumerStore(oname, time.Time{}, &oconfig)\n\t\trequire_NoError(t, err)\n\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; ; i++ {\n\t\t\t\tif err := obs.SetStarting(uint64(i + 1)); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\n\t\ttime.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond)\n\n\t\terr = obs.Delete()\n\t\trequire_NoError(t, err)\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "server/gateway.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\t\"crypto/sha256\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nconst (\n\tdefaultSolicitGatewaysDelay = time.Second\n\tdefaultGatewayConnectDelay = time.Second\n\tdefaultGatewayConnectMaxDelay = 30 * time.Second\n\tdefaultGatewayReconnectDelay = time.Second\n\tdefaultGatewayRecentSubExpiration = 2 * time.Second\n\tdefaultGatewayMaxRUnsubBeforeSwitch = 1000\n\n\toldGWReplyPrefix = \"$GR.\"\n\toldGWReplyPrefixLen = len(oldGWReplyPrefix)\n\toldGWReplyStart = oldGWReplyPrefixLen + 5 // len of prefix above + len of hash (4) + \".\"\n\n\t// The new prefix is \"_GR_...\" where is 6 characters\n\t// hash of origin cluster name and is 6 characters hash of origin server pub key.\n\tgwReplyPrefix = \"_GR_.\"\n\tgwReplyPrefixLen = len(gwReplyPrefix)\n\tgwHashLen = 6\n\tgwClusterOffset = gwReplyPrefixLen\n\tgwServerOffset = gwClusterOffset + gwHashLen + 1\n\tgwSubjectOffset = gwServerOffset + gwHashLen + 1\n\n\t// Gateway connections send PINGs regardless of traffic. The interval is\n\t// either Options.PingInterval or this value, whichever is the smallest.\n\tgwMaxPingInterval = 15 * time.Second\n)\n\nvar (\n\tgatewayConnectDelay = defaultGatewayConnectDelay\n\tgatewayConnectMaxDelay = defaultGatewayConnectMaxDelay\n\tgatewayReconnectDelay = defaultGatewayReconnectDelay\n\tgatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch\n\tgatewaySolicitDelay = int64(defaultSolicitGatewaysDelay)\n\tgatewayMaxPingInterval = gwMaxPingInterval\n)\n\n// Warning when user configures gateway TLS insecure\nconst gatewayTLSInsecureWarning = \"TLS certificate chain and hostname of solicited gateways will not be verified. DO NOT USE IN PRODUCTION!\"\n\n// SetGatewaysSolicitDelay sets the initial delay before gateways\n// connections are initiated.\n// Used by tests.\nfunc SetGatewaysSolicitDelay(delay time.Duration) {\n\tatomic.StoreInt64(&gatewaySolicitDelay, int64(delay))\n}\n\n// ResetGatewaysSolicitDelay resets the initial delay before gateways\n// connections are initiated to its default values.\n// Used by tests.\nfunc ResetGatewaysSolicitDelay() {\n\tatomic.StoreInt64(&gatewaySolicitDelay, int64(defaultSolicitGatewaysDelay))\n}\n\nconst (\n\tgatewayCmdGossip byte = 1\n\tgatewayCmdAllSubsStart byte = 2\n\tgatewayCmdAllSubsComplete byte = 3\n)\n\n// GatewayInterestMode represents an account interest mode for a gateway connection\ntype GatewayInterestMode byte\n\n// GatewayInterestMode values\nconst (\n\t// optimistic is the default mode where a cluster will send\n\t// to a gateway unless it is been told that there is no interest\n\t// (this is for plain subscribers only).\n\tOptimistic GatewayInterestMode = iota\n\t// transitioning is when a gateway has to send too many\n\t// no interest on subjects to the remote and decides that it is\n\t// now time to move to modeInterestOnly (this is on a per account\n\t// basis).\n\tTransitioning\n\t// interestOnly means that a cluster sends all it subscriptions\n\t// interest to the gateway, which in return does not send a message\n\t// unless it knows that there is explicit interest.\n\tInterestOnly\n)\n\nfunc (im GatewayInterestMode) String() string {\n\tswitch im {\n\tcase Optimistic:\n\t\treturn \"Optimistic\"\n\tcase InterestOnly:\n\t\treturn \"Interest-Only\"\n\tcase Transitioning:\n\t\treturn \"Transitioning\"\n\tdefault:\n\t\treturn \"Unknown\"\n\t}\n}\n\nvar gwDoNotForceInterestOnlyMode bool\n\n// GatewayDoNotForceInterestOnlyMode is used ONLY in tests.\n// DO NOT USE in normal code or if you embed the NATS Server.\nfunc GatewayDoNotForceInterestOnlyMode(doNotForce bool) {\n\tgwDoNotForceInterestOnlyMode = doNotForce\n}\n\ntype srvGateway struct {\n\ttotalQSubs int64 //total number of queue subs in all remote gateways (used with atomic operations)\n\tsync.RWMutex\n\tenabled bool // Immutable, true if both a name and port are configured\n\tname string // Name of the Gateway on this server\n\tout map[string]*client // outbound gateways\n\touto []*client // outbound gateways maintained in an order suitable for sending msgs (currently based on RTT)\n\tin map[uint64]*client // inbound gateways\n\tremotes map[string]*gatewayCfg // Config of remote gateways\n\tURLs refCountedUrlSet // Set of all Gateway URLs in the cluster\n\tURL string // This server gateway URL (after possible random port is resolved)\n\tinfo *Info // Gateway Info protocol\n\tinfoJSON []byte // Marshal'ed Info protocol\n\trunknown bool // Rejects unknown (not configured) gateway connections\n\treplyPfx []byte // Will be \"$GNR.<1:reserved>.<8:cluster hash>.<8:server hash>.\"\n\n\t// For backward compatibility\n\toldReplyPfx []byte\n\toldHash []byte\n\n\t// We maintain the interest of subjects and queues per account.\n\t// For a given account, entries in the map could be something like this:\n\t// foo.bar {n: 3} \t\t\t// 3 subs on foo.bar\n\t// foo.> {n: 6}\t\t\t// 6 subs on foo.>\n\t// foo bar {n: 1, q: true} // 1 qsub on foo, queue bar\n\t// foo baz {n: 3, q: true} // 3 qsubs on foo, queue baz\n\tpasi struct {\n\t\t// Protect map since accessed from different go-routine and avoid\n\t\t// possible race resulting in RS+ being sent before RS- resulting\n\t\t// in incorrect interest suppression.\n\t\t// Will use while sending QSubs (on GW connection accept) and when\n\t\t// switching to the send-all-subs mode.\n\t\tsync.Mutex\n\t\tm map[string]map[string]*sitally\n\t}\n\n\t// This is to track recent subscriptions for a given account\n\trsubs sync.Map\n\n\tresolver netResolver // Used to resolve host name before calling net.Dial()\n\tsqbsz int // Max buffer size to send queue subs protocol. Used for testing.\n\trecSubExp time.Duration // For how long do we check if there is a subscription match for a message with reply\n\n\t// These are used for routing of mapped replies.\n\tsIDHash []byte // Server ID hash (6 bytes)\n\troutesIDByHash sync.Map // Route's server ID is hashed (6 bytes) and stored in this map.\n\n\t// If a server has its own configuration in the \"Gateways\" remotes configuration\n\t// we will keep track of the URLs that are defined in the config so they can\n\t// be reported in monitoring.\n\townCfgURLs []string\n}\n\n// Subject interest tally. Also indicates if the key in the map is a\n// queue or not.\ntype sitally struct {\n\tn int32 // number of subscriptions directly matching\n\tq bool // indicate that this is a queue\n}\n\ntype gatewayCfg struct {\n\tsync.RWMutex\n\t*RemoteGatewayOpts\n\thash []byte\n\toldHash []byte\n\turls map[string]*url.URL\n\tconnAttempts int\n\ttlsName string\n\timplicit bool\n\tvarzUpdateURLs bool // Tells monitoring code to update URLs when varz is inspected.\n}\n\n// Struct for client's gateway related fields\ntype gateway struct {\n\tname string\n\tcfg *gatewayCfg\n\tconnectURL *url.URL // Needed when sending CONNECT after receiving INFO from remote\n\toutsim *sync.Map // Per-account subject interest (or no-interest) (outbound conn)\n\tinsim map[string]*insie // Per-account subject no-interest sent or modeInterestOnly mode (inbound conn)\n\n\t// This is an outbound GW connection\n\toutbound bool\n\t// Set/check in readLoop without lock. This is to know that an inbound has sent the CONNECT protocol first\n\tconnected bool\n\t// Set to true if outbound is to a server that only knows about $GR, not $GNR\n\tuseOldPrefix bool\n\t// If true, it indicates that the inbound side will switch any account to\n\t// interest-only mode \"immediately\", so the outbound should disregard\n\t// the optimistic mode when checking for interest.\n\tinterestOnlyMode bool\n\t// Name of the remote server\n\tremoteName string\n}\n\n// Outbound subject interest entry.\ntype outsie struct {\n\tsync.RWMutex\n\t// Indicate that all subs should be stored. This is\n\t// set to true when receiving the command from the\n\t// remote that we are about to receive all its subs.\n\tmode GatewayInterestMode\n\t// If not nil, used for no-interest for plain subs.\n\t// If a subject is present in this map, it means that\n\t// the remote is not interested in that subject.\n\t// When we have received the command that says that\n\t// the remote has sent all its subs, this is set to nil.\n\tni map[string]struct{}\n\t// Contains queue subscriptions when in optimistic mode,\n\t// and all subs when pk is > 0.\n\tsl *Sublist\n\t// Number of queue subs\n\tqsubs int\n}\n\n// Inbound subject interest entry.\n// If `ni` is not nil, it stores the subjects for which an\n// RS- was sent to the remote gateway. When a subscription\n// is created, this is used to know if we need to send\n// an RS+ to clear the no-interest in the remote.\n// When an account is switched to modeInterestOnly (we send\n// all subs of an account to the remote), then `ni` is nil and\n// when all subs have been sent, mode is set to modeInterestOnly\ntype insie struct {\n\tni map[string]struct{} // Record if RS- was sent for given subject\n\tmode GatewayInterestMode\n}\n\ntype gwReplyMap struct {\n\tms string\n\texp int64\n}\n\ntype gwReplyMapping struct {\n\t// Indicate if we should check the map or not. Since checking the map is done\n\t// when processing inbound messages and requires the lock we want to\n\t// check only when needed. This is set/get using atomic, so needs to\n\t// be memory aligned.\n\tcheck int32\n\t// To keep track of gateway replies mapping\n\tmapping map[string]*gwReplyMap\n}\n\n// Returns the corresponding gw routed subject, and `true` to indicate that a\n// mapping was found. If no entry is found, the passed subject is returned\n// as-is and `false` is returned to indicate that no mapping was found.\n// Caller is responsible to ensure the locking.\nfunc (g *gwReplyMapping) get(subject []byte) ([]byte, bool) {\n\trm, ok := g.mapping[string(subject)]\n\tif !ok {\n\t\treturn subject, false\n\t}\n\tsubj := []byte(rm.ms)\n\treturn subj, true\n}\n\n// clone returns a deep copy of the RemoteGatewayOpts object\nfunc (r *RemoteGatewayOpts) clone() *RemoteGatewayOpts {\n\tif r == nil {\n\t\treturn nil\n\t}\n\tclone := &RemoteGatewayOpts{\n\t\tName: r.Name,\n\t\tURLs: deepCopyURLs(r.URLs),\n\t}\n\tif r.TLSConfig != nil {\n\t\tclone.TLSConfig = r.TLSConfig.Clone()\n\t\tclone.TLSTimeout = r.TLSTimeout\n\t}\n\treturn clone\n}\n\n// Ensure that gateway is properly configured.\nfunc validateGatewayOptions(o *Options) error {\n\tif o.Gateway.Name == _EMPTY_ && o.Gateway.Port == 0 {\n\t\treturn nil\n\t}\n\tif o.Gateway.Name == _EMPTY_ {\n\t\treturn errors.New(\"gateway has no name\")\n\t}\n\tif strings.Contains(o.Gateway.Name, \" \") {\n\t\treturn ErrGatewayNameHasSpaces\n\t}\n\tif o.Gateway.Port == 0 {\n\t\treturn fmt.Errorf(\"gateway %q has no port specified (select -1 for random port)\", o.Gateway.Name)\n\t}\n\tfor i, g := range o.Gateway.Gateways {\n\t\tif g.Name == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"gateway in the list %d has no name\", i)\n\t\t}\n\t\tif len(g.URLs) == 0 {\n\t\t\treturn fmt.Errorf(\"gateway %q has no URL\", g.Name)\n\t\t}\n\t}\n\tif err := validatePinnedCerts(o.Gateway.TLSPinnedCerts); err != nil {\n\t\treturn fmt.Errorf(\"gateway %q: %v\", o.Gateway.Name, err)\n\t}\n\treturn nil\n}\n\n// Computes a hash of 6 characters for the name.\n// This will be used for routing of replies.\nfunc getGWHash(name string) []byte {\n\treturn []byte(getHashSize(name, gwHashLen))\n}\n\nfunc getOldHash(name string) []byte {\n\tsha := sha256.New()\n\tsha.Write([]byte(name))\n\tfullHash := []byte(fmt.Sprintf(\"%x\", sha.Sum(nil)))\n\treturn fullHash[:4]\n}\n\n// Initialize the s.gateway structure. We do this even if the server\n// does not have a gateway configured. In some part of the code, the\n// server will check the number of outbound gateways, etc.. and so\n// we don't have to check if s.gateway is nil or not.\nfunc (s *Server) newGateway(opts *Options) error {\n\tgateway := &srvGateway{\n\t\tname: opts.Gateway.Name,\n\t\tout: make(map[string]*client),\n\t\touto: make([]*client, 0, 4),\n\t\tin: make(map[uint64]*client),\n\t\tremotes: make(map[string]*gatewayCfg),\n\t\tURLs: make(refCountedUrlSet),\n\t\tresolver: opts.Gateway.resolver,\n\t\trunknown: opts.Gateway.RejectUnknown,\n\t\toldHash: getOldHash(opts.Gateway.Name),\n\t}\n\tgateway.Lock()\n\tdefer gateway.Unlock()\n\n\tgateway.sIDHash = getGWHash(s.info.ID)\n\tclusterHash := getGWHash(opts.Gateway.Name)\n\tprefix := make([]byte, 0, gwSubjectOffset)\n\tprefix = append(prefix, gwReplyPrefix...)\n\tprefix = append(prefix, clusterHash...)\n\tprefix = append(prefix, '.')\n\tprefix = append(prefix, gateway.sIDHash...)\n\tprefix = append(prefix, '.')\n\tgateway.replyPfx = prefix\n\n\tprefix = make([]byte, 0, oldGWReplyStart)\n\tprefix = append(prefix, oldGWReplyPrefix...)\n\tprefix = append(prefix, gateway.oldHash...)\n\tprefix = append(prefix, '.')\n\tgateway.oldReplyPfx = prefix\n\n\tgateway.pasi.m = make(map[string]map[string]*sitally)\n\n\tif gateway.resolver == nil {\n\t\tgateway.resolver = netResolver(net.DefaultResolver)\n\t}\n\n\t// Create remote gateways\n\tfor _, rgo := range opts.Gateway.Gateways {\n\t\t// Ignore if there is a remote gateway with our name.\n\t\tif rgo.Name == gateway.name {\n\t\t\tgateway.ownCfgURLs = getURLsAsString(rgo.URLs)\n\t\t\tcontinue\n\t\t}\n\t\tcfg := &gatewayCfg{\n\t\t\tRemoteGatewayOpts: rgo.clone(),\n\t\t\thash: getGWHash(rgo.Name),\n\t\t\toldHash: getOldHash(rgo.Name),\n\t\t\turls: make(map[string]*url.URL, len(rgo.URLs)),\n\t\t}\n\t\tif opts.Gateway.TLSConfig != nil && cfg.TLSConfig == nil {\n\t\t\tcfg.TLSConfig = opts.Gateway.TLSConfig.Clone()\n\t\t}\n\t\tif cfg.TLSTimeout == 0 {\n\t\t\tcfg.TLSTimeout = opts.Gateway.TLSTimeout\n\t\t}\n\t\tfor _, u := range rgo.URLs {\n\t\t\t// For TLS, look for a hostname that we can use for TLSConfig.ServerName\n\t\t\tcfg.saveTLSHostname(u)\n\t\t\tcfg.urls[u.Host] = u\n\t\t}\n\t\tgateway.remotes[cfg.Name] = cfg\n\t}\n\n\tgateway.sqbsz = opts.Gateway.sendQSubsBufSize\n\tif gateway.sqbsz == 0 {\n\t\tgateway.sqbsz = maxBufSize\n\t}\n\tgateway.recSubExp = defaultGatewayRecentSubExpiration\n\n\tgateway.enabled = opts.Gateway.Name != \"\" && opts.Gateway.Port != 0\n\ts.gateway = gateway\n\treturn nil\n}\n\n// Update remote gateways TLS configurations after a config reload.\nfunc (g *srvGateway) updateRemotesTLSConfig(opts *Options) {\n\tg.Lock()\n\tdefer g.Unlock()\n\t// Instead of going over opts.Gateway.Gateways, which would include only\n\t// explicit remotes, we are going to go through g.remotes.\n\tfor name, cfg := range g.remotes {\n\t\tif name == g.name {\n\t\t\tcontinue\n\t\t}\n\t\tvar ro *RemoteGatewayOpts\n\t\t// We now need to go back and find the RemoteGatewayOpts but only if\n\t\t// this remote is explicit (otherwise it won't be found).\n\t\tif !cfg.isImplicit() {\n\t\t\tfor _, r := range opts.Gateway.Gateways {\n\t\t\t\tif r.Name == name {\n\t\t\t\t\tro = r\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcfg.Lock()\n\t\t// If we have an `ro` (that means an explicitly defined remote gateway)\n\t\t// and it has an explicit TLS config, use that one, otherwise (no explicit\n\t\t// TLS config in the remote, or implicit remote), use the TLS config from\n\t\t// the main block.\n\t\tif ro != nil && ro.TLSConfig != nil {\n\t\t\tcfg.TLSConfig = ro.TLSConfig.Clone()\n\t\t} else if opts.Gateway.TLSConfig != nil {\n\t\t\tcfg.TLSConfig = opts.Gateway.TLSConfig.Clone()\n\t\t}\n\t\t// Ensure that OCSP callbacks are always setup after a reload if needed.\n\t\tmustStaple := opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways\n\t\tif mustStaple && opts.Gateway.TLSConfig != nil {\n\t\t\tclientCB := opts.Gateway.TLSConfig.GetClientCertificate\n\t\t\tverifyCB := opts.Gateway.TLSConfig.VerifyConnection\n\t\t\tif mustStaple && cfg.TLSConfig != nil {\n\t\t\t\tif clientCB != nil && cfg.TLSConfig.GetClientCertificate == nil {\n\t\t\t\t\tcfg.TLSConfig.GetClientCertificate = clientCB\n\t\t\t\t}\n\t\t\t\tif verifyCB != nil && cfg.TLSConfig.VerifyConnection == nil {\n\t\t\t\t\tcfg.TLSConfig.VerifyConnection = verifyCB\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcfg.Unlock()\n\t}\n}\n\n// Returns if this server rejects connections from gateways that are not\n// explicitly configured.\nfunc (g *srvGateway) rejectUnknown() bool {\n\tg.RLock()\n\treject := g.runknown\n\tg.RUnlock()\n\treturn reject\n}\n\n// Starts the gateways accept loop and solicit explicit gateways\n// after an initial delay. This delay is meant to give a chance to\n// the cluster to form and this server gathers gateway URLs for this\n// cluster in order to send that as part of the connect/info process.\nfunc (s *Server) startGateways() {\n\ts.startGatewayAcceptLoop()\n\n\t// Delay start of creation of gateways to give a chance\n\t// to the local cluster to form.\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\n\t\tdur := s.getOpts().gatewaysSolicitDelay\n\t\tif dur == 0 {\n\t\t\tdur = time.Duration(atomic.LoadInt64(&gatewaySolicitDelay))\n\t\t}\n\n\t\tselect {\n\t\tcase <-time.After(dur):\n\t\t\ts.solicitGateways()\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t})\n}\n\n// This starts the gateway accept loop in a go routine, unless it\n// is detected that the server has already been shutdown.\nfunc (s *Server) startGatewayAcceptLoop() {\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tport := opts.Gateway.Port\n\tif port == -1 {\n\t\tport = 0\n\t}\n\n\ts.mu.Lock()\n\thp := net.JoinHostPort(opts.Gateway.Host, strconv.Itoa(port))\n\tl, e := natsListen(\"tcp\", hp)\n\ts.gatewayListenerErr = e\n\tif e != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Error listening on gateway port: %d - %v\", opts.Gateway.Port, e)\n\t\treturn\n\t}\n\ts.Noticef(\"Gateway name is %s\", s.getGatewayName())\n\ts.Noticef(\"Listening for gateways connections on %s\",\n\t\tnet.JoinHostPort(opts.Gateway.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port)))\n\n\ttlsReq := opts.Gateway.TLSConfig != nil\n\tauthRequired := opts.Gateway.Username != \"\"\n\tinfo := &Info{\n\t\tID: s.info.ID,\n\t\tName: opts.ServerName,\n\t\tVersion: s.info.Version,\n\t\tAuthRequired: authRequired,\n\t\tTLSRequired: tlsReq,\n\t\tTLSVerify: tlsReq,\n\t\tMaxPayload: s.info.MaxPayload,\n\t\tGateway: opts.Gateway.Name,\n\t\tGatewayNRP: true,\n\t\tHeaders: s.supportsHeaders(),\n\t\tProto: s.getServerProto(),\n\t}\n\t// Unless in some tests we want to keep the old behavior, we are now\n\t// (since v2.9.0) indicate that this server will switch all accounts\n\t// to InterestOnly mode when accepting an inbound or when a new\n\t// account is fetched.\n\tif !gwDoNotForceInterestOnlyMode {\n\t\tinfo.GatewayIOM = true\n\t}\n\n\t// If we have selected a random port...\n\tif port == 0 {\n\t\t// Write resolved port back to options.\n\t\topts.Gateway.Port = l.Addr().(*net.TCPAddr).Port\n\t}\n\t// Possibly override Host/Port based on Gateway.Advertise\n\tif err := s.setGatewayInfoHostPort(info, opts); err != nil {\n\t\ts.Fatalf(\"Error setting gateway INFO with Gateway.Advertise value of %s, err=%v\", opts.Gateway.Advertise, err)\n\t\tl.Close()\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\t// Setup state that can enable shutdown\n\ts.gatewayListener = l\n\n\t// Warn if insecure is configured in the main Gateway configuration\n\t// or any of the RemoteGateway's. This means that we need to check\n\t// remotes even if TLS would not be configured for the accept.\n\twarn := tlsReq && opts.Gateway.TLSConfig.InsecureSkipVerify\n\tif !warn {\n\t\tfor _, g := range opts.Gateway.Gateways {\n\t\t\tif g.TLSConfig != nil && g.TLSConfig.InsecureSkipVerify {\n\t\t\t\twarn = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif warn {\n\t\ts.Warnf(gatewayTLSInsecureWarning)\n\t}\n\tgo s.acceptConnections(l, \"Gateway\", func(conn net.Conn) { s.createGateway(nil, nil, conn) }, nil)\n\ts.mu.Unlock()\n}\n\n// Similar to setInfoHostPortAndGenerateJSON, but for gatewayInfo.\nfunc (s *Server) setGatewayInfoHostPort(info *Info, o *Options) error {\n\tgw := s.gateway\n\tgw.Lock()\n\tdefer gw.Unlock()\n\tgw.URLs.removeUrl(gw.URL)\n\tif o.Gateway.Advertise != \"\" {\n\t\tadvHost, advPort, err := parseHostPort(o.Gateway.Advertise, o.Gateway.Port)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tinfo.Host = advHost\n\t\tinfo.Port = advPort\n\t} else {\n\t\tinfo.Host = o.Gateway.Host\n\t\tinfo.Port = o.Gateway.Port\n\t\t// If the host is \"0.0.0.0\" or \"::\" we need to resolve to a public IP.\n\t\t// This will return at most 1 IP.\n\t\thostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(info.Host, false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif hostIsIPAny {\n\t\t\tif len(ips) == 0 {\n\t\t\t\t// TODO(ik): Should we fail here (prevent starting)? If not, we\n\t\t\t\t// are going to \"advertise\" the 0.0.0.0: url, which means\n\t\t\t\t// that remote are going to try to connect to 0.0.0.0:,\n\t\t\t\t// which means a connect to loopback address, which is going\n\t\t\t\t// to fail with either TLS error, conn refused if the remote\n\t\t\t\t// is using different gateway port than this one, or error\n\t\t\t\t// saying that it tried to connect to itself.\n\t\t\t\ts.Errorf(\"Could not find any non-local IP for gateway %q with listen specification %q\",\n\t\t\t\t\tgw.name, info.Host)\n\t\t\t} else {\n\t\t\t\t// Take the first from the list...\n\t\t\t\tinfo.Host = ips[0]\n\t\t\t}\n\t\t}\n\t}\n\tgw.URL = net.JoinHostPort(info.Host, strconv.Itoa(info.Port))\n\tif o.Gateway.Advertise != \"\" {\n\t\ts.Noticef(\"Advertise address for gateway %q is set to %s\", gw.name, gw.URL)\n\t} else {\n\t\ts.Noticef(\"Address for gateway %q is %s\", gw.name, gw.URL)\n\t}\n\tgw.URLs[gw.URL]++\n\tgw.info = info\n\tinfo.GatewayURL = gw.URL\n\t// (re)generate the gatewayInfoJSON byte array\n\tgw.generateInfoJSON()\n\treturn nil\n}\n\n// Generates the Gateway INFO protocol.\n// The gateway lock is held on entry\nfunc (g *srvGateway) generateInfoJSON() {\n\t// We could be here when processing a route INFO that has a gateway URL,\n\t// but this server is not configured for gateways, so simply ignore here.\n\t// The configuration mismatch is reported somewhere else.\n\tif !g.enabled || g.info == nil {\n\t\treturn\n\t}\n\tg.info.GatewayURLs = g.URLs.getAsStringSlice()\n\tb, err := json.Marshal(g.info)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tg.infoJSON = []byte(fmt.Sprintf(InfoProto, b))\n}\n\n// Goes through the list of registered gateways and try to connect to those.\n// The list (remotes) is initially containing the explicit remote gateways,\n// but the list is augmented with any implicit (discovered) gateway. Therefore,\n// this function only solicit explicit ones.\nfunc (s *Server) solicitGateways() {\n\tgw := s.gateway\n\tgw.RLock()\n\tdefer gw.RUnlock()\n\tfor _, cfg := range gw.remotes {\n\t\t// Since we delay the creation of gateways, it is\n\t\t// possible that server starts to receive inbound from\n\t\t// other clusters and in turn create outbounds. So here\n\t\t// we create only the ones that are configured.\n\t\tif !cfg.isImplicit() {\n\t\t\tcfg := cfg // Create new instance for the goroutine.\n\t\t\ts.startGoRoutine(func() {\n\t\t\t\ts.solicitGateway(cfg, true)\n\t\t\t\ts.grWG.Done()\n\t\t\t})\n\t\t}\n\t}\n}\n\n// Reconnect to the gateway after a little wait period. For explicit\n// gateways, we also wait for the default reconnect time.\nfunc (s *Server) reconnectGateway(cfg *gatewayCfg) {\n\tdefer s.grWG.Done()\n\n\tdelay := time.Duration(rand.Intn(100)) * time.Millisecond\n\tif !cfg.isImplicit() {\n\t\tdelay += gatewayReconnectDelay\n\t}\n\tselect {\n\tcase <-time.After(delay):\n\tcase <-s.quitCh:\n\t\treturn\n\t}\n\ts.solicitGateway(cfg, false)\n}\n\n// This function will loop trying to connect to any URL attached\n// to the given Gateway. It will return once a connection has been created.\nfunc (s *Server) solicitGateway(cfg *gatewayCfg, firstConnect bool) {\n\tvar (\n\t\topts = s.getOpts()\n\t\tisImplicit = cfg.isImplicit()\n\t\tattemptDelay = gatewayConnectDelay\n\t\tattempts int\n\t\ttypeStr string\n\t)\n\tif isImplicit {\n\t\ttypeStr = \"implicit\"\n\t} else {\n\t\ttypeStr = \"explicit\"\n\t}\n\n\tconst connFmt = \"Connecting to %s gateway %q (%s) at %s (attempt %v)\"\n\tconst connErrFmt = \"Error connecting to %s gateway %q (%s) at %s (attempt %v): %v\"\n\n\tfor s.isRunning() {\n\t\turls := cfg.getURLs()\n\t\tif len(urls) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tattempts++\n\t\treport := s.shouldReportConnectErr(firstConnect, attempts)\n\t\t// Iteration is random\n\t\tfor _, u := range urls {\n\t\t\taddress, err := s.getRandomIP(s.gateway.resolver, u.Host, nil)\n\t\t\tif err != nil {\n\t\t\t\ts.Errorf(\"Error getting IP for %s gateway %q (%s): %v\", typeStr, cfg.Name, u.Host, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif report {\n\t\t\t\ts.Noticef(connFmt, typeStr, cfg.Name, u.Host, address, attempts)\n\t\t\t} else {\n\t\t\t\ts.Debugf(connFmt, typeStr, cfg.Name, u.Host, address, attempts)\n\t\t\t}\n\t\t\tconn, err := natsDialTimeout(\"tcp\", address, DEFAULT_ROUTE_DIAL)\n\t\t\tif err == nil {\n\t\t\t\t// We could connect, create the gateway connection and return.\n\t\t\t\ts.createGateway(cfg, u, conn)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif report {\n\t\t\t\ts.Errorf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err)\n\t\t\t} else {\n\t\t\t\ts.Debugf(connErrFmt, typeStr, cfg.Name, u.Host, address, attempts, err)\n\t\t\t}\n\t\t\t// Break this loop if server is being shutdown...\n\t\t\tif !s.isRunning() {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif isImplicit {\n\t\t\tif opts.Gateway.ConnectRetries == 0 || attempts > opts.Gateway.ConnectRetries {\n\t\t\t\ts.gateway.Lock()\n\t\t\t\t// We could have just accepted an inbound for this remote gateway.\n\t\t\t\t// So if there is an inbound, let's try again to connect.\n\t\t\t\tif s.gateway.hasInbound(cfg.Name) {\n\t\t\t\t\ts.gateway.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tdelete(s.gateway.remotes, cfg.Name)\n\t\t\t\ts.gateway.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-time.After(attemptDelay):\n\t\t\tif opts.Gateway.ConnectBackoff {\n\t\t\t\t// Use exponential backoff for connection attempts.\n\t\t\t\tattemptDelay *= 2\n\t\t\t\tif attemptDelay > gatewayConnectMaxDelay {\n\t\t\t\t\tattemptDelay = gatewayConnectMaxDelay\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\n// Returns true if there is an inbound for the given `name`.\n// Lock held on entry.\nfunc (g *srvGateway) hasInbound(name string) bool {\n\tfor _, ig := range g.in {\n\t\tig.mu.Lock()\n\t\tigname := ig.gw.name\n\t\tig.mu.Unlock()\n\t\tif igname == name {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Called when a gateway connection is either accepted or solicited.\n// If accepted, the gateway is marked as inbound.\n// If solicited, the gateway is marked as outbound.\nfunc (s *Server) createGateway(cfg *gatewayCfg, url *url.URL, conn net.Conn) {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tnow := time.Now()\n\tc := &client{srv: s, nc: conn, start: now, last: now, kind: GATEWAY}\n\n\t// Are we creating the gateway based on the configuration\n\tsolicit := cfg != nil\n\tvar tlsRequired bool\n\n\ts.gateway.RLock()\n\tinfoJSON := s.gateway.infoJSON\n\ts.gateway.RUnlock()\n\n\t// Perform some initialization under the client lock\n\tc.mu.Lock()\n\tc.initClient()\n\tc.gw = &gateway{}\n\tif solicit {\n\t\t// This is an outbound gateway connection\n\t\tcfg.RLock()\n\t\ttlsRequired = cfg.TLSConfig != nil\n\t\tcfgName := cfg.Name\n\t\tcfg.RUnlock()\n\t\tc.gw.outbound = true\n\t\tc.gw.name = cfgName\n\t\tc.gw.cfg = cfg\n\t\tcfg.bumpConnAttempts()\n\t\t// Since we are delaying the connect until after receiving\n\t\t// the remote's INFO protocol, save the URL we need to connect to.\n\t\tc.gw.connectURL = url\n\n\t\tc.Noticef(\"Creating outbound gateway connection to %q\", cfgName)\n\t} else {\n\t\tc.flags.set(expectConnect)\n\t\t// Inbound gateway connection\n\t\tc.Noticef(\"Processing inbound gateway connection\")\n\t\t// Check if TLS is required for inbound GW connections.\n\t\ttlsRequired = opts.Gateway.TLSConfig != nil\n\t\t// We expect a CONNECT from the accepted connection.\n\t\tc.setAuthTimer(secondsToDuration(opts.Gateway.AuthTimeout))\n\t}\n\n\t// Check for TLS\n\tif tlsRequired {\n\t\tvar tlsConfig *tls.Config\n\t\tvar tlsName string\n\t\tvar timeout float64\n\n\t\tif solicit {\n\t\t\tvar (\n\t\t\t\tmustStaple = opts.OCSPConfig != nil && opts.OCSPConfig.Mode == OCSPModeAlways\n\t\t\t\tclientCB func(*tls.CertificateRequestInfo) (*tls.Certificate, error)\n\t\t\t\tverifyCB func(tls.ConnectionState) error\n\t\t\t)\n\t\t\t// Snapshot callbacks for OCSP outside an ongoing reload which might be happening.\n\t\t\tif mustStaple {\n\t\t\t\ts.reloadMu.RLock()\n\t\t\t\ts.optsMu.RLock()\n\t\t\t\tclientCB = s.opts.Gateway.TLSConfig.GetClientCertificate\n\t\t\t\tverifyCB = s.opts.Gateway.TLSConfig.VerifyConnection\n\t\t\t\ts.optsMu.RUnlock()\n\t\t\t\ts.reloadMu.RUnlock()\n\t\t\t}\n\n\t\t\tcfg.RLock()\n\t\t\ttlsName = cfg.tlsName\n\t\t\ttlsConfig = cfg.TLSConfig.Clone()\n\t\t\ttimeout = cfg.TLSTimeout\n\n\t\t\t// Ensure that OCSP callbacks are always setup on gateway reconnect when OCSP policy is set to always.\n\t\t\tif mustStaple {\n\t\t\t\tif clientCB != nil && tlsConfig.GetClientCertificate == nil {\n\t\t\t\t\ttlsConfig.GetClientCertificate = clientCB\n\t\t\t\t}\n\t\t\t\tif verifyCB != nil && tlsConfig.VerifyConnection == nil {\n\t\t\t\t\ttlsConfig.VerifyConnection = verifyCB\n\t\t\t\t}\n\t\t\t}\n\t\t\tcfg.RUnlock()\n\t\t} else {\n\t\t\ttlsConfig = opts.Gateway.TLSConfig\n\t\t\ttimeout = opts.Gateway.TLSTimeout\n\t\t}\n\n\t\t// Perform (either server or client side) TLS handshake.\n\t\tif resetTLSName, err := c.doTLSHandshake(\"gateway\", solicit, url, tlsConfig, tlsName, timeout, opts.Gateway.TLSPinnedCerts); err != nil {\n\t\t\tif resetTLSName {\n\t\t\t\tcfg.Lock()\n\t\t\t\tcfg.tlsName = _EMPTY_\n\t\t\t\tcfg.Unlock()\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Do final client initialization\n\tc.in.pacache = make(map[string]*perAccountCache)\n\tif solicit {\n\t\t// This is an outbound gateway connection\n\t\tc.gw.outsim = &sync.Map{}\n\t} else {\n\t\t// Inbound gateway connection\n\t\tc.gw.insim = make(map[string]*insie)\n\t}\n\n\t// Register in temp map for now until gateway properly registered\n\t// in out or in gateways.\n\tif !s.addToTempClients(c.cid, c) {\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(ServerShutdown)\n\t\treturn\n\t}\n\n\t// Only send if we accept a connection. Will send CONNECT+INFO as an\n\t// outbound only after processing peer's INFO protocol.\n\tif !solicit {\n\t\tc.enqueueProto(infoJSON)\n\t}\n\n\t// Spin up the read loop.\n\ts.startGoRoutine(func() { c.readLoop(nil) })\n\n\t// Spin up the write loop.\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\tif tlsRequired {\n\t\tc.Debugf(\"TLS handshake complete\")\n\t\tcs := c.nc.(*tls.Conn).ConnectionState()\n\t\tc.Debugf(\"TLS version %s, cipher suite %s\", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite))\n\t}\n\n\t// For outbound, we can't set the normal ping timer yet since the other\n\t// side would fail with a parse error should it receive anything but the\n\t// CONNECT protocol as the first protocol. We still want to make sure\n\t// that the connection is not stale until the first INFO from the remote\n\t// is received.\n\tif solicit {\n\t\tc.watchForStaleConnection(adjustPingInterval(GATEWAY, opts.PingInterval), opts.MaxPingsOut)\n\t}\n\n\tc.mu.Unlock()\n\n\t// Announce ourselves again to new connections.\n\tif solicit && s.EventsEnabled() {\n\t\ts.sendStatszUpdate()\n\t}\n}\n\n// Builds and sends the CONNECT protocol for a gateway.\n// Client lock held on entry.\nfunc (c *client) sendGatewayConnect(opts *Options) {\n\t// FIXME: This can race with updateRemotesTLSConfig\n\ttlsRequired := c.gw.cfg.TLSConfig != nil\n\turl := c.gw.connectURL\n\tc.gw.connectURL = nil\n\tvar user, pass string\n\tif userInfo := url.User; userInfo != nil {\n\t\tuser = userInfo.Username()\n\t\tpass, _ = userInfo.Password()\n\t} else if opts != nil {\n\t\tuser = opts.Gateway.Username\n\t\tpass = opts.Gateway.Password\n\t}\n\tcinfo := connectInfo{\n\t\tVerbose: false,\n\t\tPedantic: false,\n\t\tUser: user,\n\t\tPass: pass,\n\t\tTLS: tlsRequired,\n\t\tName: c.srv.info.ID,\n\t\tGateway: c.srv.gateway.name,\n\t}\n\tb, err := json.Marshal(cinfo)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tc.enqueueProto([]byte(fmt.Sprintf(ConProto, b)))\n}\n\n// Process the CONNECT protocol from a gateway connection.\n// Returns an error to the connection if the CONNECT is not from a gateway\n// (for instance a client or route connecting to the gateway port), or\n// if the destination does not match the gateway name of this server.\n//\n// \nfunc (c *client) processGatewayConnect(arg []byte) error {\n\tconnect := &connectInfo{}\n\tif err := json.Unmarshal(arg, connect); err != nil {\n\t\treturn err\n\t}\n\n\t// Coming from a client or a route, reject\n\tif connect.Gateway == \"\" {\n\t\tc.sendErrAndErr(ErrClientOrRouteConnectedToGatewayPort.Error())\n\t\tc.closeConnection(WrongPort)\n\t\treturn ErrClientOrRouteConnectedToGatewayPort\n\t}\n\n\tc.mu.Lock()\n\ts := c.srv\n\tc.mu.Unlock()\n\n\t// If we reject unknown gateways, make sure we have it configured,\n\t// otherwise return an error.\n\tif s.gateway.rejectUnknown() && s.getRemoteGateway(connect.Gateway) == nil {\n\t\tc.Errorf(\"Rejecting connection from gateway %q\", connect.Gateway)\n\t\tc.sendErr(fmt.Sprintf(\"Connection to gateway %q rejected\", s.getGatewayName()))\n\t\tc.closeConnection(WrongGateway)\n\t\treturn ErrWrongGateway\n\t}\n\n\tc.mu.Lock()\n\tc.gw.connected = true\n\t// Set the Ping timer after sending connect and info.\n\tc.setFirstPingTimer()\n\tc.mu.Unlock()\n\n\treturn nil\n}\n\n// Process the INFO protocol from a gateway connection.\n//\n// If the gateway connection is an outbound (this server initiated the connection),\n// this function checks that the incoming INFO contains the Gateway field. If empty,\n// it means that this is a response from an older server or that this server connected\n// to the wrong port.\n// The outbound gateway may also receive a gossip INFO protocol from the remote gateway,\n// indicating other gateways that the remote knows about. This server will try to connect\n// to those gateways (if not explicitly configured or already implicitly connected).\n// In both cases (explicit or implicit), the local cluster is notified about the existence\n// of this new gateway. This allows servers in the cluster to ensure that they have an\n// outbound connection to this gateway.\n//\n// For an inbound gateway, the gateway is simply registered and the info protocol\n// is saved to be used after processing the CONNECT.\n//\n// \nfunc (c *client) processGatewayInfo(info *Info) {\n\tvar (\n\t\tgwName string\n\t\tcfg *gatewayCfg\n\t)\n\tc.mu.Lock()\n\ts := c.srv\n\tcid := c.cid\n\n\t// Check if this is the first INFO. (this call sets the flag if not already set).\n\tisFirstINFO := c.flags.setIfNotSet(infoReceived)\n\n\tisOutbound := c.gw.outbound\n\tif isOutbound {\n\t\tgwName = c.gw.name\n\t\tcfg = c.gw.cfg\n\t} else if isFirstINFO {\n\t\tc.gw.name = info.Gateway\n\t}\n\tif isFirstINFO {\n\t\tc.opts.Name = info.ID\n\t\t// Get the protocol version from the INFO protocol. This will be checked\n\t\t// to see if this connection supports message tracing for instance.\n\t\tc.opts.Protocol = info.Proto\n\t\tc.gw.remoteName = info.Name\n\t}\n\tc.mu.Unlock()\n\n\t// For an outbound connection...\n\tif isOutbound {\n\t\t// Check content of INFO for fields indicating that it comes from a gateway.\n\t\t// If we incorrectly connect to the wrong port (client or route), we won't\n\t\t// have the Gateway field set.\n\t\tif info.Gateway == \"\" {\n\t\t\tc.sendErrAndErr(fmt.Sprintf(\"Attempt to connect to gateway %q using wrong port\", gwName))\n\t\t\tc.closeConnection(WrongPort)\n\t\t\treturn\n\t\t}\n\t\t// Check that the gateway name we got is what we expect\n\t\tif info.Gateway != gwName {\n\t\t\t// Unless this is the very first INFO, it may be ok if this is\n\t\t\t// a gossip request to connect to other gateways.\n\t\t\tif !isFirstINFO && info.GatewayCmd == gatewayCmdGossip {\n\t\t\t\t// If we are configured to reject unknown, do not attempt to\n\t\t\t\t// connect to one that we don't have configured.\n\t\t\t\tif s.gateway.rejectUnknown() && s.getRemoteGateway(info.Gateway) == nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\ts.processImplicitGateway(info)\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Otherwise, this is a failure...\n\t\t\t// We are reporting this error in the log...\n\t\t\tc.Errorf(\"Failing connection to gateway %q, remote gateway name is %q\",\n\t\t\t\tgwName, info.Gateway)\n\t\t\t// ...and sending this back to the remote so that the error\n\t\t\t// makes more sense in the remote server's log.\n\t\t\tc.sendErr(fmt.Sprintf(\"Connection from %q rejected, wanted to connect to %q, this is %q\",\n\t\t\t\ts.getGatewayName(), gwName, info.Gateway))\n\t\t\tc.closeConnection(WrongGateway)\n\t\t\treturn\n\t\t}\n\n\t\t// Check for duplicate server name with servers in our cluster\n\t\tif s.isDuplicateServerName(info.Name) {\n\t\t\tc.Errorf(\"Remote server has a duplicate name: %q\", info.Name)\n\t\t\tc.closeConnection(DuplicateServerName)\n\t\t\treturn\n\t\t}\n\n\t\t// Possibly add URLs that we get from the INFO protocol.\n\t\tif len(info.GatewayURLs) > 0 {\n\t\t\tcfg.updateURLs(info.GatewayURLs)\n\t\t}\n\n\t\t// If this is the first INFO, send our connect\n\t\tif isFirstINFO {\n\t\t\ts.gateway.RLock()\n\t\t\tinfoJSON := s.gateway.infoJSON\n\t\t\ts.gateway.RUnlock()\n\n\t\t\tsupportsHeaders := s.supportsHeaders()\n\t\t\topts := s.getOpts()\n\n\t\t\t// Note, if we want to support NKeys, then we would get the nonce\n\t\t\t// from this INFO protocol and can sign it in the CONNECT we are\n\t\t\t// going to send now.\n\t\t\tc.mu.Lock()\n\t\t\tc.gw.interestOnlyMode = info.GatewayIOM\n\t\t\tc.sendGatewayConnect(opts)\n\t\t\tc.Debugf(\"Gateway connect protocol sent to %q\", gwName)\n\t\t\t// Send INFO too\n\t\t\tc.enqueueProto(infoJSON)\n\t\t\tc.gw.useOldPrefix = !info.GatewayNRP\n\t\t\tc.headers = supportsHeaders && info.Headers\n\t\t\tc.mu.Unlock()\n\n\t\t\t// Register as an outbound gateway.. if we had a protocol to ack our connect,\n\t\t\t// then we should do that when process that ack.\n\t\t\tif s.registerOutboundGatewayConnection(gwName, c) {\n\t\t\t\tc.Noticef(\"Outbound gateway connection to %q (%s) registered\", gwName, info.ID)\n\t\t\t\t// Now that the outbound gateway is registered, we can remove from temp map.\n\t\t\t\ts.removeFromTempClients(cid)\n\t\t\t\t// Set the Ping timer after sending connect and info.\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.setFirstPingTimer()\n\t\t\t\tc.mu.Unlock()\n\t\t\t} else {\n\t\t\t\t// There was a bug that would cause a connection to possibly\n\t\t\t\t// be called twice resulting in reconnection of twice the\n\t\t\t\t// same outbound connection. The issue is fixed, but adding\n\t\t\t\t// defensive code above that if we did not register this connection\n\t\t\t\t// because we already have an outbound for this name, then\n\t\t\t\t// close this connection (and make sure it does not try to reconnect)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.flags.set(noReconnect)\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.closeConnection(WrongGateway)\n\t\t\t\treturn\n\t\t\t}\n\t\t} else if info.GatewayCmd > 0 {\n\t\t\tswitch info.GatewayCmd {\n\t\t\tcase gatewayCmdAllSubsStart:\n\t\t\t\tc.gatewayAllSubsReceiveStart(info)\n\t\t\t\treturn\n\t\t\tcase gatewayCmdAllSubsComplete:\n\t\t\t\tc.gatewayAllSubsReceiveComplete(info)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\ts.Warnf(\"Received unknown command %v from gateway %q\", info.GatewayCmd, gwName)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\t// Flood local cluster with information about this gateway.\n\t\t// Servers in this cluster will ensure that they have (or otherwise create)\n\t\t// an outbound connection to this gateway.\n\t\ts.forwardNewGatewayToLocalCluster(info)\n\n\t} else if isFirstINFO {\n\t\t// This is the first INFO of an inbound connection...\n\n\t\t// Check for duplicate server name with servers in our cluster\n\t\tif s.isDuplicateServerName(info.Name) {\n\t\t\tc.Errorf(\"Remote server has a duplicate name: %q\", info.Name)\n\t\t\tc.closeConnection(DuplicateServerName)\n\t\t\treturn\n\t\t}\n\n\t\ts.registerInboundGatewayConnection(cid, c)\n\t\tc.Noticef(\"Inbound gateway connection from %q (%s) registered\", info.Gateway, info.ID)\n\n\t\t// Now that it is registered, we can remove from temp map.\n\t\ts.removeFromTempClients(cid)\n\n\t\t// Send our QSubs.\n\t\ts.sendQueueSubsToGateway(c)\n\n\t\t// Initiate outbound connection. This function will behave correctly if\n\t\t// we have already one.\n\t\ts.processImplicitGateway(info)\n\n\t\t// Send back to the server that initiated this gateway connection the\n\t\t// list of all remote gateways known on this server.\n\t\ts.gossipGatewaysToInboundGateway(info.Gateway, c)\n\n\t\t// Now make sure if we have any knowledge of connected leafnodes that we resend the\n\t\t// connect events to switch those accounts into interest only mode.\n\t\ts.mu.Lock()\n\t\ts.ensureGWsInterestOnlyForLeafNodes()\n\t\ts.mu.Unlock()\n\t\tjs := s.js.Load()\n\n\t\t// If running in some tests, maintain the original behavior.\n\t\tif gwDoNotForceInterestOnlyMode && js != nil {\n\t\t\t// Switch JetStream accounts to interest-only mode.\n\t\t\tvar accounts []string\n\t\t\tjs.mu.Lock()\n\t\t\tif len(js.accounts) > 0 {\n\t\t\t\taccounts = make([]string, 0, len(js.accounts))\n\t\t\t\tfor accName := range js.accounts {\n\t\t\t\t\taccounts = append(accounts, accName)\n\t\t\t\t}\n\t\t\t}\n\t\t\tjs.mu.Unlock()\n\t\t\tfor _, accName := range accounts {\n\t\t\t\tif acc, err := s.LookupAccount(accName); err == nil && acc != nil {\n\t\t\t\t\tif acc.JetStreamEnabled() {\n\t\t\t\t\t\ts.switchAccountToInterestMode(acc.GetName())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if !gwDoNotForceInterestOnlyMode {\n\t\t\t// Starting 2.9.0, we are phasing out the optimistic mode, so change\n\t\t\t// all accounts to interest-only mode, unless instructed not to do so\n\t\t\t// in some tests.\n\t\t\ts.accounts.Range(func(_, v any) bool {\n\t\t\t\tacc := v.(*Account)\n\t\t\t\ts.switchAccountToInterestMode(acc.GetName())\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t}\n}\n\n// Sends to the given inbound gateway connection a gossip INFO protocol\n// for each gateway known by this server. This allows for a \"full mesh\"\n// of gateways.\nfunc (s *Server) gossipGatewaysToInboundGateway(gwName string, c *client) {\n\tgw := s.gateway\n\tgw.RLock()\n\tdefer gw.RUnlock()\n\tfor gwCfgName, cfg := range gw.remotes {\n\t\t// Skip the gateway that we just created\n\t\tif gwCfgName == gwName {\n\t\t\tcontinue\n\t\t}\n\t\tinfo := Info{\n\t\t\tID: s.info.ID,\n\t\t\tGatewayCmd: gatewayCmdGossip,\n\t\t}\n\t\turls := cfg.getURLsAsStrings()\n\t\tif len(urls) > 0 {\n\t\t\tinfo.Gateway = gwCfgName\n\t\t\tinfo.GatewayURLs = urls\n\t\t\tb, _ := json.Marshal(&info)\n\t\t\tc.mu.Lock()\n\t\t\tc.enqueueProto([]byte(fmt.Sprintf(InfoProto, b)))\n\t\t\tc.mu.Unlock()\n\t\t}\n\t}\n}\n\n// Sends the INFO protocol of a gateway to all routes known by this server.\nfunc (s *Server) forwardNewGatewayToLocalCluster(oinfo *Info) {\n\t// Need to protect s.routes here, so use server's lock\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\t// We don't really need the ID to be set, but, we need to make sure\n\t// that it is not set to the server ID so that if we were to connect\n\t// to an older server that does not expect a \"gateway\" INFO, it\n\t// would think that it needs to create an implicit route (since info.ID\n\t// would not match the route's remoteID), but will fail to do so because\n\t// the sent protocol will not have host/port defined.\n\tinfo := &Info{\n\t\tID: \"GW\" + s.info.ID,\n\t\tName: s.getOpts().ServerName,\n\t\tGateway: oinfo.Gateway,\n\t\tGatewayURLs: oinfo.GatewayURLs,\n\t\tGatewayCmd: gatewayCmdGossip,\n\t}\n\tb, _ := json.Marshal(info)\n\tinfoJSON := []byte(fmt.Sprintf(InfoProto, b))\n\n\ts.forEachRemote(func(r *client) {\n\t\tr.mu.Lock()\n\t\tr.enqueueProto(infoJSON)\n\t\tr.mu.Unlock()\n\t})\n}\n\n// Sends queue subscriptions interest to remote gateway.\n// This is sent from the inbound side, that is, the side that receives\n// messages from the remote's outbound connection. This side is\n// the one sending the subscription interest.\nfunc (s *Server) sendQueueSubsToGateway(c *client) {\n\ts.sendSubsToGateway(c, _EMPTY_)\n}\n\n// Sends all subscriptions for the given account to the remove gateway\n// This is sent from the inbound side, that is, the side that receives\n// messages from the remote's outbound connection. This side is\n// the one sending the subscription interest.\nfunc (s *Server) sendAccountSubsToGateway(c *client, accName string) {\n\ts.sendSubsToGateway(c, accName)\n}\n\nfunc gwBuildSubProto(buf *bytes.Buffer, accName string, acc map[string]*sitally, doQueues bool) {\n\tfor saq, si := range acc {\n\t\tif doQueues && si.q || !doQueues && !si.q {\n\t\t\tbuf.Write(rSubBytes)\n\t\t\tbuf.WriteString(accName)\n\t\t\tbuf.WriteByte(' ')\n\t\t\t// For queue subs (si.q is true), saq will be\n\t\t\t// subject + ' ' + queue, for plain subs, this is\n\t\t\t// just the subject.\n\t\t\tbuf.WriteString(saq)\n\t\t\tif doQueues {\n\t\t\t\tbuf.WriteString(\" 1\")\n\t\t\t}\n\t\t\tbuf.WriteString(CR_LF)\n\t\t}\n\t}\n}\n\n// Sends subscriptions to remote gateway.\nfunc (s *Server) sendSubsToGateway(c *client, accountName string) {\n\tvar (\n\t\tbufa = [32 * 1024]byte{}\n\t\tbbuf = bytes.NewBuffer(bufa[:0])\n\t)\n\n\tgw := s.gateway\n\n\t// This needs to run under this lock for the whole duration\n\tgw.pasi.Lock()\n\tdefer gw.pasi.Unlock()\n\n\t// If account is specified...\n\tif accountName != _EMPTY_ {\n\t\t// Simply send all plain subs (no queues) for this specific account\n\t\tgwBuildSubProto(bbuf, accountName, gw.pasi.m[accountName], false)\n\t\t// Instruct to send all subs (RS+/-) for this account from now on.\n\t\tc.mu.Lock()\n\t\te := c.gw.insim[accountName]\n\t\tif e == nil {\n\t\t\te = &insie{}\n\t\t\tc.gw.insim[accountName] = e\n\t\t}\n\t\te.mode = InterestOnly\n\t\tc.mu.Unlock()\n\t} else {\n\t\t// Send queues for all accounts\n\t\tfor accName, acc := range gw.pasi.m {\n\t\t\tgwBuildSubProto(bbuf, accName, acc, true)\n\t\t}\n\t}\n\n\tbuf := bbuf.Bytes()\n\n\t// Nothing to send.\n\tif len(buf) == 0 {\n\t\treturn\n\t}\n\tif len(buf) > cap(bufa) {\n\t\ts.Debugf(\"Sending subscriptions to %q, buffer size: %v\", c.gw.name, len(buf))\n\t}\n\t// Send\n\tc.mu.Lock()\n\tc.enqueueProto(buf)\n\tc.Debugf(\"Sent queue subscriptions to gateway\")\n\tc.mu.Unlock()\n}\n\n// This is invoked when getting an INFO protocol for gateway on the ROUTER port.\n// This function will then execute appropriate function based on the command\n// contained in the protocol.\n// \nfunc (s *Server) processGatewayInfoFromRoute(info *Info, routeSrvID string) {\n\tswitch info.GatewayCmd {\n\tcase gatewayCmdGossip:\n\t\ts.processImplicitGateway(info)\n\tdefault:\n\t\ts.Errorf(\"Unknown command %d from server %v\", info.GatewayCmd, routeSrvID)\n\t}\n}\n\n// Sends INFO protocols to the given route connection for each known Gateway.\n// These will be processed by the route and delegated to the gateway code to\n// invoke processImplicitGateway.\nfunc (s *Server) sendGatewayConfigsToRoute(route *client) {\n\tgw := s.gateway\n\tgw.RLock()\n\t// Send only to gateways for which we have actual outbound connection to.\n\tif len(gw.out) == 0 {\n\t\tgw.RUnlock()\n\t\treturn\n\t}\n\t// Collect gateway configs for which we have an outbound connection.\n\tgwCfgsa := [16]*gatewayCfg{}\n\tgwCfgs := gwCfgsa[:0]\n\tfor _, c := range gw.out {\n\t\tc.mu.Lock()\n\t\tif c.gw.cfg != nil {\n\t\t\tgwCfgs = append(gwCfgs, c.gw.cfg)\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n\tgw.RUnlock()\n\tif len(gwCfgs) == 0 {\n\t\treturn\n\t}\n\n\t// Check forwardNewGatewayToLocalCluster() as to why we set ID this way.\n\tinfo := Info{\n\t\tID: \"GW\" + s.info.ID,\n\t\tGatewayCmd: gatewayCmdGossip,\n\t}\n\tfor _, cfg := range gwCfgs {\n\t\turls := cfg.getURLsAsStrings()\n\t\tif len(urls) > 0 {\n\t\t\tinfo.Gateway = cfg.Name\n\t\t\tinfo.GatewayURLs = urls\n\t\t\tb, _ := json.Marshal(&info)\n\t\t\troute.mu.Lock()\n\t\t\troute.enqueueProto([]byte(fmt.Sprintf(InfoProto, b)))\n\t\t\troute.mu.Unlock()\n\t\t}\n\t}\n}\n\n// Initiates a gateway connection using the info contained in the INFO protocol.\n// If a gateway with the same name is already registered (either because explicitly\n// configured, or already implicitly connected), this function will augmment the\n// remote URLs with URLs present in the info protocol and return.\n// Otherwise, this function will register this remote (to prevent multiple connections\n// to the same remote) and call solicitGateway (which will run in a different go-routine).\nfunc (s *Server) processImplicitGateway(info *Info) {\n\ts.gateway.Lock()\n\tdefer s.gateway.Unlock()\n\t// Name of the gateway to connect to is the Info.Gateway field.\n\tgwName := info.Gateway\n\t// If this is our name, bail.\n\tif gwName == s.gateway.name {\n\t\treturn\n\t}\n\t// Check if we already have this config, and if so, we are done\n\tcfg := s.gateway.remotes[gwName]\n\tif cfg != nil {\n\t\t// However, possibly augment the list of URLs with the given\n\t\t// info.GatewayURLs content.\n\t\tcfg.Lock()\n\t\tcfg.addURLs(info.GatewayURLs)\n\t\tcfg.Unlock()\n\t\treturn\n\t}\n\topts := s.getOpts()\n\tcfg = &gatewayCfg{\n\t\tRemoteGatewayOpts: &RemoteGatewayOpts{Name: gwName},\n\t\thash: getGWHash(gwName),\n\t\toldHash: getOldHash(gwName),\n\t\turls: make(map[string]*url.URL, len(info.GatewayURLs)),\n\t\timplicit: true,\n\t}\n\tif opts.Gateway.TLSConfig != nil {\n\t\tcfg.TLSConfig = opts.Gateway.TLSConfig.Clone()\n\t\tcfg.TLSTimeout = opts.Gateway.TLSTimeout\n\t}\n\n\t// Since we know we don't have URLs (no config, so just based on what we\n\t// get from INFO), directly call addURLs(). We don't need locking since\n\t// we just created that structure and no one else has access to it yet.\n\tcfg.addURLs(info.GatewayURLs)\n\t// If there is no URL, we can't proceed.\n\tif len(cfg.urls) == 0 {\n\t\treturn\n\t}\n\ts.gateway.remotes[gwName] = cfg\n\ts.startGoRoutine(func() {\n\t\ts.solicitGateway(cfg, true)\n\t\ts.grWG.Done()\n\t})\n}\n\n// NumOutboundGateways is public here mostly for testing.\nfunc (s *Server) NumOutboundGateways() int {\n\treturn s.numOutboundGateways()\n}\n\n// Returns the number of outbound gateway connections\nfunc (s *Server) numOutboundGateways() int {\n\ts.gateway.RLock()\n\tn := len(s.gateway.out)\n\ts.gateway.RUnlock()\n\treturn n\n}\n\n// Returns the number of inbound gateway connections\nfunc (s *Server) numInboundGateways() int {\n\ts.gateway.RLock()\n\tn := len(s.gateway.in)\n\ts.gateway.RUnlock()\n\treturn n\n}\n\n// Returns the remoteGateway (if any) that has the given `name`\nfunc (s *Server) getRemoteGateway(name string) *gatewayCfg {\n\ts.gateway.RLock()\n\tcfg := s.gateway.remotes[name]\n\ts.gateway.RUnlock()\n\treturn cfg\n}\n\n// Used in tests\nfunc (g *gatewayCfg) bumpConnAttempts() {\n\tg.Lock()\n\tg.connAttempts++\n\tg.Unlock()\n}\n\n// Used in tests\nfunc (g *gatewayCfg) getConnAttempts() int {\n\tg.Lock()\n\tca := g.connAttempts\n\tg.Unlock()\n\treturn ca\n}\n\n// Used in tests\nfunc (g *gatewayCfg) resetConnAttempts() {\n\tg.Lock()\n\tg.connAttempts = 0\n\tg.Unlock()\n}\n\n// Returns if this remote gateway is implicit or not.\nfunc (g *gatewayCfg) isImplicit() bool {\n\tg.RLock()\n\tii := g.implicit\n\tg.RUnlock()\n\treturn ii\n}\n\n// getURLs returns an array of URLs in random order suitable for\n// an iteration to try to connect.\nfunc (g *gatewayCfg) getURLs() []*url.URL {\n\tg.RLock()\n\ta := make([]*url.URL, 0, len(g.urls))\n\tfor _, u := range g.urls {\n\t\ta = append(a, u)\n\t}\n\tg.RUnlock()\n\t// Map iteration is random, but not that good with small maps.\n\trand.Shuffle(len(a), func(i, j int) {\n\t\ta[i], a[j] = a[j], a[i]\n\t})\n\treturn a\n}\n\n// Similar to getURLs but returns the urls as an array of strings.\nfunc (g *gatewayCfg) getURLsAsStrings() []string {\n\tg.RLock()\n\ta := make([]string, 0, len(g.urls))\n\tfor _, u := range g.urls {\n\t\ta = append(a, u.Host)\n\t}\n\tg.RUnlock()\n\treturn a\n}\n\n// updateURLs creates the urls map with the content of the config's URLs array\n// and the given array that we get from the INFO protocol.\nfunc (g *gatewayCfg) updateURLs(infoURLs []string) {\n\tg.Lock()\n\t// Clear the map...\n\tg.urls = make(map[string]*url.URL, len(g.URLs)+len(infoURLs))\n\t// Add the urls from the config URLs array.\n\tfor _, u := range g.URLs {\n\t\tg.urls[u.Host] = u\n\t}\n\t// Then add the ones from the infoURLs array we got.\n\tg.addURLs(infoURLs)\n\t// The call above will set varzUpdateURLs only when finding ULRs in infoURLs\n\t// that are not present in the config. That does not cover the case where\n\t// previously \"discovered\" URLs are now gone. We could check \"before\" size\n\t// of g.urls and if bigger than current size, set the boolean to true.\n\t// Not worth it... simply set this to true to allow a refresh of gateway\n\t// URLs in varz.\n\tg.varzUpdateURLs = true\n\tg.Unlock()\n}\n\n// Saves the hostname of the given URL (if not already done).\n// This may be used as the ServerName of the TLSConfig when initiating a\n// TLS connection.\n// Write lock held on entry.\nfunc (g *gatewayCfg) saveTLSHostname(u *url.URL) {\n\tif g.TLSConfig != nil && g.tlsName == \"\" && net.ParseIP(u.Hostname()) == nil {\n\t\tg.tlsName = u.Hostname()\n\t}\n}\n\n// add URLs from the given array to the urls map only if not already present.\n// remoteGateway write lock is assumed to be held on entry.\n// Write lock is held on entry.\nfunc (g *gatewayCfg) addURLs(infoURLs []string) {\n\tvar scheme string\n\tif g.TLSConfig != nil {\n\t\tscheme = \"tls\"\n\t} else {\n\t\tscheme = \"nats\"\n\t}\n\tfor _, iu := range infoURLs {\n\t\tif _, present := g.urls[iu]; !present {\n\t\t\t// Urls in Info.GatewayURLs come without scheme. Add it to parse\n\t\t\t// the url (otherwise it fails).\n\t\t\tif u, err := url.Parse(fmt.Sprintf(\"%s://%s\", scheme, iu)); err == nil {\n\t\t\t\t// Also, if a tlsName has not been set yet and we are dealing\n\t\t\t\t// with a hostname and not a bare IP, save the hostname.\n\t\t\t\tg.saveTLSHostname(u)\n\t\t\t\t// Use u.Host for the key.\n\t\t\t\tg.urls[u.Host] = u\n\t\t\t\t// Signal that we have updated the list. Used by monitoring code.\n\t\t\t\tg.varzUpdateURLs = true\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Adds this URL to the set of Gateway URLs.\n// Returns true if the URL has been added, false otherwise.\n// Server lock held on entry\nfunc (s *Server) addGatewayURL(urlStr string) bool {\n\ts.gateway.Lock()\n\tadded := s.gateway.URLs.addUrl(urlStr)\n\tif added {\n\t\ts.gateway.generateInfoJSON()\n\t}\n\ts.gateway.Unlock()\n\treturn added\n}\n\n// Removes this URL from the set of gateway URLs.\n// Returns true if the URL has been removed, false otherwise.\n// Server lock held on entry\nfunc (s *Server) removeGatewayURL(urlStr string) bool {\n\tif s.isShuttingDown() {\n\t\treturn false\n\t}\n\ts.gateway.Lock()\n\tremoved := s.gateway.URLs.removeUrl(urlStr)\n\tif removed {\n\t\ts.gateway.generateInfoJSON()\n\t}\n\ts.gateway.Unlock()\n\treturn removed\n}\n\n// Sends a Gateway's INFO to all inbound GW connections.\n// Server lock is held on entry\nfunc (s *Server) sendAsyncGatewayInfo() {\n\ts.gateway.RLock()\n\tfor _, ig := range s.gateway.in {\n\t\tig.mu.Lock()\n\t\tig.enqueueProto(s.gateway.infoJSON)\n\t\tig.mu.Unlock()\n\t}\n\ts.gateway.RUnlock()\n}\n\n// This returns the URL of the Gateway listen spec, or empty string\n// if the server has no gateway configured.\nfunc (s *Server) getGatewayURL() string {\n\ts.gateway.RLock()\n\turl := s.gateway.URL\n\ts.gateway.RUnlock()\n\treturn url\n}\n\n// Returns this server gateway name.\n// Same than calling s.gateway.getName()\nfunc (s *Server) getGatewayName() string {\n\t// This is immutable\n\treturn s.gateway.name\n}\n\n// All gateway connections (outbound and inbound) are put in the given map.\nfunc (s *Server) getAllGatewayConnections(conns map[uint64]*client) {\n\tgw := s.gateway\n\tgw.RLock()\n\tfor _, c := range gw.out {\n\t\tc.mu.Lock()\n\t\tcid := c.cid\n\t\tc.mu.Unlock()\n\t\tconns[cid] = c\n\t}\n\tfor cid, c := range gw.in {\n\t\tconns[cid] = c\n\t}\n\tgw.RUnlock()\n}\n\n// Register the given gateway connection (*client) in the inbound gateways\n// map. The key is the connection ID (like for clients and routes).\nfunc (s *Server) registerInboundGatewayConnection(cid uint64, gwc *client) {\n\ts.gateway.Lock()\n\ts.gateway.in[cid] = gwc\n\ts.gateway.Unlock()\n}\n\n// Register the given gateway connection (*client) in the outbound gateways\n// map with the given name as the key.\nfunc (s *Server) registerOutboundGatewayConnection(name string, gwc *client) bool {\n\ts.gateway.Lock()\n\tif _, exist := s.gateway.out[name]; exist {\n\t\ts.gateway.Unlock()\n\t\treturn false\n\t}\n\ts.gateway.out[name] = gwc\n\ts.gateway.outo = append(s.gateway.outo, gwc)\n\ts.gateway.orderOutboundConnectionsLocked()\n\ts.gateway.Unlock()\n\treturn true\n}\n\n// Returns the outbound gateway connection (*client) with the given name,\n// or nil if not found\nfunc (s *Server) getOutboundGatewayConnection(name string) *client {\n\ts.gateway.RLock()\n\tgwc := s.gateway.out[name]\n\ts.gateway.RUnlock()\n\treturn gwc\n}\n\n// Returns all outbound gateway connections in the provided array.\n// The order of the gateways is suited for the sending of a message.\n// Current ordering is based on individual gateway's RTT value.\nfunc (s *Server) getOutboundGatewayConnections(a *[]*client) {\n\ts.gateway.RLock()\n\tfor i := 0; i < len(s.gateway.outo); i++ {\n\t\t*a = append(*a, s.gateway.outo[i])\n\t}\n\ts.gateway.RUnlock()\n}\n\n// Orders the array of outbound connections.\n// Current ordering is by lowest RTT.\n// Gateway write lock is held on entry\nfunc (g *srvGateway) orderOutboundConnectionsLocked() {\n\t// Order the gateways by lowest RTT\n\tslices.SortFunc(g.outo, func(i, j *client) int { return cmp.Compare(i.getRTTValue(), j.getRTTValue()) })\n}\n\n// Orders the array of outbound connections.\n// Current ordering is by lowest RTT.\nfunc (g *srvGateway) orderOutboundConnections() {\n\tg.Lock()\n\tg.orderOutboundConnectionsLocked()\n\tg.Unlock()\n}\n\n// Returns all inbound gateway connections in the provided array\nfunc (s *Server) getInboundGatewayConnections(a *[]*client) {\n\ts.gateway.RLock()\n\tfor _, gwc := range s.gateway.in {\n\t\t*a = append(*a, gwc)\n\t}\n\ts.gateway.RUnlock()\n}\n\n// This is invoked when a gateway connection is closed and the server\n// is removing this connection from its state.\nfunc (s *Server) removeRemoteGatewayConnection(c *client) {\n\tc.mu.Lock()\n\tcid := c.cid\n\tisOutbound := c.gw.outbound\n\tgwName := c.gw.name\n\tif isOutbound && c.gw.outsim != nil {\n\t\t// We do this to allow the GC to release this connection.\n\t\t// Since the map is used by the rest of the code without client lock,\n\t\t// we can't simply set it to nil, instead, just make sure we empty it.\n\t\tc.gw.outsim.Range(func(k, _ any) bool {\n\t\t\tc.gw.outsim.Delete(k)\n\t\t\treturn true\n\t\t})\n\t}\n\tc.mu.Unlock()\n\n\tgw := s.gateway\n\tgw.Lock()\n\tif isOutbound {\n\t\tdelete(gw.out, gwName)\n\t\tlouto := len(gw.outo)\n\t\treorder := false\n\t\tfor i := 0; i < len(gw.outo); i++ {\n\t\t\tif gw.outo[i] == c {\n\t\t\t\t// If last, simply remove and no need to reorder\n\t\t\t\tif i != louto-1 {\n\t\t\t\t\tgw.outo[i] = gw.outo[louto-1]\n\t\t\t\t\treorder = true\n\t\t\t\t}\n\t\t\t\tgw.outo = gw.outo[:louto-1]\n\t\t\t}\n\t\t}\n\t\tif reorder {\n\t\t\tgw.orderOutboundConnectionsLocked()\n\t\t}\n\t} else {\n\t\tdelete(gw.in, cid)\n\t}\n\tgw.Unlock()\n\ts.removeFromTempClients(cid)\n\n\tif isOutbound {\n\t\t// Update number of totalQSubs for this gateway\n\t\tqSubsRemoved := int64(0)\n\t\tc.mu.Lock()\n\t\tfor _, sub := range c.subs {\n\t\t\tif sub.queue != nil {\n\t\t\t\tqSubsRemoved++\n\t\t\t}\n\t\t}\n\t\tc.subs = nil\n\t\tc.mu.Unlock()\n\t\t// Update total count of qsubs in remote gateways.\n\t\tatomic.AddInt64(&c.srv.gateway.totalQSubs, -qSubsRemoved)\n\n\t} else {\n\t\tvar subsa [1024]*subscription\n\t\tvar subs = subsa[:0]\n\n\t\t// For inbound GW connection, if we have subs, those are\n\t\t// local subs on \"_R_.\" subjects.\n\t\tc.mu.Lock()\n\t\tfor _, sub := range c.subs {\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t\tc.subs = nil\n\t\tc.mu.Unlock()\n\t\tfor _, sub := range subs {\n\t\t\tc.removeReplySub(sub)\n\t\t}\n\t}\n}\n\n// GatewayAddr returns the net.Addr object for the gateway listener.\nfunc (s *Server) GatewayAddr() *net.TCPAddr {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif s.gatewayListener == nil {\n\t\treturn nil\n\t}\n\treturn s.gatewayListener.Addr().(*net.TCPAddr)\n}\n\n// A- protocol received from the remote after sending messages\n// on an account that it has no interest in. Mark this account\n// with a \"no interest\" marker to prevent further messages send.\n// \nfunc (c *client) processGatewayAccountUnsub(accName string) {\n\t// Just to indicate activity around \"subscriptions\" events.\n\tc.in.subs++\n\t// This account may have an entry because of queue subs.\n\t// If that's the case, we can reset the no-interest map,\n\t// but not set the entry to nil.\n\tsetToNil := true\n\tif ei, ok := c.gw.outsim.Load(accName); ei != nil {\n\t\te := ei.(*outsie)\n\t\te.Lock()\n\t\t// Reset the no-interest map if we have queue subs\n\t\t// and don't set the entry to nil.\n\t\tif e.qsubs > 0 {\n\t\t\te.ni = make(map[string]struct{})\n\t\t\tsetToNil = false\n\t\t}\n\t\te.Unlock()\n\t} else if ok {\n\t\t// Already set to nil, so skip\n\t\tsetToNil = false\n\t}\n\tif setToNil {\n\t\tc.gw.outsim.Store(accName, nil)\n\t}\n}\n\n// A+ protocol received from remote gateway if it had previously\n// sent an A-. Clear the \"no interest\" marker for this account.\n// \nfunc (c *client) processGatewayAccountSub(accName string) error {\n\t// Just to indicate activity around \"subscriptions\" events.\n\tc.in.subs++\n\t// If this account has an entry because of queue subs, we\n\t// can't delete the entry.\n\tremove := true\n\tif ei, ok := c.gw.outsim.Load(accName); ei != nil {\n\t\te := ei.(*outsie)\n\t\te.Lock()\n\t\tif e.qsubs > 0 {\n\t\t\tremove = false\n\t\t}\n\t\te.Unlock()\n\t} else if !ok {\n\t\t// There is no entry, so skip\n\t\tremove = false\n\t}\n\tif remove {\n\t\tc.gw.outsim.Delete(accName)\n\t}\n\treturn nil\n}\n\n// RS- protocol received from the remote after sending messages\n// on a subject that it has no interest in (but knows about the\n// account). Mark this subject with a \"no interest\" marker to\n// prevent further messages being sent.\n// If in modeInterestOnly or for a queue sub, remove from\n// the sublist if present.\n// \nfunc (c *client) processGatewayRUnsub(arg []byte) error {\n\t_, accName, subject, queue, err := c.parseUnsubProto(arg, true, false)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"processGatewaySubjectUnsub %s\", err.Error())\n\t}\n\n\tvar (\n\t\te *outsie\n\t\tuseSl bool\n\t\tnewe bool\n\t\tcallUpdate bool\n\t\tsrv *Server\n\t\tsub *subscription\n\t)\n\n\t// Possibly execute this on exit after all locks have been released.\n\t// If callUpdate is true, srv and sub will be not nil.\n\tdefer func() {\n\t\tif callUpdate {\n\t\t\tsrv.updateInterestForAccountOnGateway(accName, sub, -1)\n\t\t}\n\t}()\n\n\tc.mu.Lock()\n\tif c.gw.outsim == nil {\n\t\tc.Errorf(\"Received RS- from gateway on inbound connection\")\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn nil\n\t}\n\tdefer c.mu.Unlock()\n\t// If closed, c.subs map will be nil, so bail out.\n\tif c.isClosed() {\n\t\treturn nil\n\t}\n\n\tei, _ := c.gw.outsim.Load(accName)\n\tif ei != nil {\n\t\te = ei.(*outsie)\n\t\te.Lock()\n\t\tdefer e.Unlock()\n\t\t// If there is an entry, for plain sub we need\n\t\t// to know if we should store the sub\n\t\tuseSl = queue != nil || e.mode != Optimistic\n\t} else if queue != nil {\n\t\t// should not even happen...\n\t\tc.Debugf(\"Received RS- without prior RS+ for subject %q, queue %q\", subject, queue)\n\t\treturn nil\n\t} else {\n\t\t// Plain sub, assume optimistic sends, create entry.\n\t\te = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}\n\t\tnewe = true\n\t}\n\t// This is when a sub or queue sub is supposed to be in\n\t// the sublist. Look for it and remove.\n\tif useSl {\n\t\tvar ok bool\n\t\tkey := arg\n\t\t// m[string()] does not cause mem allocation\n\t\tsub, ok = c.subs[string(key)]\n\t\t// if RS- for a sub that we don't have, just ignore.\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\tif e.sl.Remove(sub) == nil {\n\t\t\tdelete(c.subs, bytesToString(key))\n\t\t\tif queue != nil {\n\t\t\t\te.qsubs--\n\t\t\t\tatomic.AddInt64(&c.srv.gateway.totalQSubs, -1)\n\t\t\t}\n\t\t\t// If last, we can remove the whole entry only\n\t\t\t// when in optimistic mode and there is no element\n\t\t\t// in the `ni` map.\n\t\t\tif e.sl.Count() == 0 && e.mode == Optimistic && len(e.ni) == 0 {\n\t\t\t\tc.gw.outsim.Delete(accName)\n\t\t\t}\n\t\t}\n\t\t// We are going to call updateInterestForAccountOnGateway on exit.\n\t\tsrv = c.srv\n\t\tcallUpdate = true\n\t} else {\n\t\te.ni[string(subject)] = struct{}{}\n\t\tif newe {\n\t\t\tc.gw.outsim.Store(accName, e)\n\t\t}\n\t}\n\treturn nil\n}\n\n// For plain subs, RS+ protocol received from remote gateway if it\n// had previously sent a RS-. Clear the \"no interest\" marker for\n// this subject (under this account).\n// For queue subs, or if in modeInterestOnly, register interest\n// from remote gateway.\n// \nfunc (c *client) processGatewayRSub(arg []byte) error {\n\t// Indicate activity.\n\tc.in.subs++\n\n\tvar (\n\t\tqueue []byte\n\t\tqw int32\n\t)\n\n\targs := splitArg(arg)\n\tswitch len(args) {\n\tcase 2:\n\tcase 4:\n\t\tqueue = args[2]\n\t\tqw = int32(parseSize(args[3]))\n\tdefault:\n\t\treturn fmt.Errorf(\"processGatewaySubjectSub Parse Error: '%s'\", arg)\n\t}\n\taccName := args[0]\n\tsubject := args[1]\n\n\tvar (\n\t\te *outsie\n\t\tuseSl bool\n\t\tnewe bool\n\t\tcallUpdate bool\n\t\tsrv *Server\n\t\tsub *subscription\n\t)\n\n\t// Possibly execute this on exit after all locks have been released.\n\t// If callUpdate is true, srv and sub will be not nil.\n\tdefer func() {\n\t\tif callUpdate {\n\t\t\tsrv.updateInterestForAccountOnGateway(string(accName), sub, 1)\n\t\t}\n\t}()\n\n\tc.mu.Lock()\n\tif c.gw.outsim == nil {\n\t\tc.Errorf(\"Received RS+ from gateway on inbound connection\")\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn nil\n\t}\n\tdefer c.mu.Unlock()\n\t// If closed, c.subs map will be nil, so bail out.\n\tif c.isClosed() {\n\t\treturn nil\n\t}\n\n\tei, _ := c.gw.outsim.Load(bytesToString(accName))\n\t// We should always have an existing entry for plain subs because\n\t// in optimistic mode we would have received RS- first, and\n\t// in full knowledge, we are receiving RS+ for an account after\n\t// getting many RS- from the remote..\n\tif ei != nil {\n\t\te = ei.(*outsie)\n\t\te.Lock()\n\t\tdefer e.Unlock()\n\t\tuseSl = queue != nil || e.mode != Optimistic\n\t} else if queue == nil {\n\t\treturn nil\n\t} else {\n\t\te = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}\n\t\tnewe = true\n\t\tuseSl = true\n\t}\n\tif useSl {\n\t\tvar key []byte\n\t\t// We store remote subs by account/subject[/queue].\n\t\t// For queue, remove the trailing weight\n\t\tif queue != nil {\n\t\t\tkey = arg[:len(arg)-len(args[3])-1]\n\t\t} else {\n\t\t\tkey = arg\n\t\t}\n\t\t// If RS+ for a sub that we already have, ignore.\n\t\t// (m[string()] does not allocate memory)\n\t\tif _, ok := c.subs[string(key)]; ok {\n\t\t\treturn nil\n\t\t}\n\t\t// new subscription. copy subject (and queue) to\n\t\t// not reference the underlying possibly big buffer.\n\t\tvar csubject []byte\n\t\tvar cqueue []byte\n\t\tif queue != nil {\n\t\t\t// make single allocation and use different slices\n\t\t\t// to point to subject and queue name.\n\t\t\tcbuf := make([]byte, len(subject)+1+len(queue))\n\t\t\tcopy(cbuf, key[len(accName)+1:])\n\t\t\tcsubject = cbuf[:len(subject)]\n\t\t\tcqueue = cbuf[len(subject)+1:]\n\t\t} else {\n\t\t\tcsubject = make([]byte, len(subject))\n\t\t\tcopy(csubject, subject)\n\t\t}\n\t\tsub = &subscription{client: c, subject: csubject, queue: cqueue, qw: qw}\n\t\t// If no error inserting in sublist...\n\t\tif e.sl.Insert(sub) == nil {\n\t\t\tc.subs[string(key)] = sub\n\t\t\tif queue != nil {\n\t\t\t\te.qsubs++\n\t\t\t\tatomic.AddInt64(&c.srv.gateway.totalQSubs, 1)\n\t\t\t}\n\t\t\tif newe {\n\t\t\t\tc.gw.outsim.Store(string(accName), e)\n\t\t\t}\n\t\t}\n\t\t// We are going to call updateInterestForAccountOnGateway on exit.\n\t\tsrv = c.srv\n\t\tcallUpdate = true\n\t} else {\n\t\tsubj := bytesToString(subject)\n\t\t// If this is an RS+ for a wc subject, then\n\t\t// remove from the no interest map all subjects\n\t\t// that are a subset of this wc subject.\n\t\tif subjectHasWildcard(subj) {\n\t\t\tfor k := range e.ni {\n\t\t\t\tif subjectIsSubsetMatch(k, subj) {\n\t\t\t\t\tdelete(e.ni, k)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tdelete(e.ni, subj)\n\t\t}\n\t}\n\treturn nil\n}\n\n// Returns true if this gateway has possible interest in the\n// given account/subject (which means, it does not have a registered\n// no-interest on the account and/or subject) and the sublist result\n// for queue subscriptions.\n// \nfunc (c *client) gatewayInterest(acc string, subj []byte) (bool, *SublistResult) {\n\tei, accountInMap := c.gw.outsim.Load(acc)\n\t// If there is an entry for this account and ei is nil,\n\t// it means that the remote is not interested at all in\n\t// this account and we could not possibly have queue subs.\n\tif accountInMap && ei == nil {\n\t\treturn false, nil\n\t}\n\t// Assume interest if account not in map, unless we support\n\t// only interest-only mode.\n\tpsi := !accountInMap && !c.gw.interestOnlyMode\n\tvar r *SublistResult\n\tif accountInMap {\n\t\t// If in map, check for subs interest with sublist.\n\t\te := ei.(*outsie)\n\t\te.RLock()\n\t\t// Unless each side has agreed on interest-only mode,\n\t\t// we may be in transition to modeInterestOnly\n\t\t// but until e.ni is nil, use it to know if we\n\t\t// should suppress interest or not.\n\t\tif !c.gw.interestOnlyMode && e.ni != nil {\n\t\t\tif _, inMap := e.ni[string(subj)]; !inMap {\n\t\t\t\tpsi = true\n\t\t\t}\n\t\t}\n\t\t// If we are in modeInterestOnly (e.ni will be nil)\n\t\t// or if we have queue subs, we also need to check sl.Match.\n\t\tif e.ni == nil || e.qsubs > 0 {\n\t\t\tr = e.sl.MatchBytes(subj)\n\t\t\tif len(r.psubs) > 0 {\n\t\t\t\tpsi = true\n\t\t\t}\n\t\t}\n\t\te.RUnlock()\n\t\t// Since callers may just check if the sublist result is nil or not,\n\t\t// make sure that if what is returned by sl.Match() is the emptyResult, then\n\t\t// we return nil to the caller.\n\t\tif r == emptyResult {\n\t\t\tr = nil\n\t\t}\n\t}\n\treturn psi, r\n}\n\n// switchAccountToInterestMode will switch an account over to interestMode.\n// Lock should NOT be held.\nfunc (s *Server) switchAccountToInterestMode(accName string) {\n\tgwsa := [16]*client{}\n\tgws := gwsa[:0]\n\ts.getInboundGatewayConnections(&gws)\n\n\tfor _, gin := range gws {\n\t\tvar e *insie\n\t\tvar ok bool\n\n\t\tgin.mu.Lock()\n\t\tif e, ok = gin.gw.insim[accName]; !ok || e == nil {\n\t\t\te = &insie{}\n\t\t\tgin.gw.insim[accName] = e\n\t\t}\n\t\t// Do it only if we are in Optimistic mode\n\t\tif e.mode == Optimistic {\n\t\t\tgin.gatewaySwitchAccountToSendAllSubs(e, accName)\n\t\t}\n\t\tgin.mu.Unlock()\n\t}\n}\n\n// This is invoked when registering (or unregistering) the first\n// (or last) subscription on a given account/subject. For each\n// GWs inbound connections, we will check if we need to send an RS+ or A+\n// protocol.\nfunc (s *Server) maybeSendSubOrUnsubToGateways(accName string, sub *subscription, added bool) {\n\tif sub.queue != nil {\n\t\treturn\n\t}\n\tgwsa := [16]*client{}\n\tgws := gwsa[:0]\n\ts.getInboundGatewayConnections(&gws)\n\tif len(gws) == 0 {\n\t\treturn\n\t}\n\tvar (\n\t\trsProtoa [512]byte\n\t\trsProto []byte\n\t\taccProtoa [256]byte\n\t\taccProto []byte\n\t\tproto []byte\n\t\tsubject = bytesToString(sub.subject)\n\t\thasWC = subjectHasWildcard(subject)\n\t)\n\tfor _, c := range gws {\n\t\tproto = nil\n\t\tc.mu.Lock()\n\t\te, inMap := c.gw.insim[accName]\n\t\t// If there is a inbound subject interest entry...\n\t\tif e != nil {\n\t\t\tsendProto := false\n\t\t\t// In optimistic mode, we care only about possibly sending RS+ (or A+)\n\t\t\t// so if e.ni is not nil we do things only when adding a new subscription.\n\t\t\tif e.ni != nil && added {\n\t\t\t\t// For wildcard subjects, we will remove from our no-interest\n\t\t\t\t// map, all subjects that are a subset of this wc subject, but we\n\t\t\t\t// still send the wc subject and let the remote do its own cleanup.\n\t\t\t\tif hasWC {\n\t\t\t\t\tfor enis := range e.ni {\n\t\t\t\t\t\tif subjectIsSubsetMatch(enis, subject) {\n\t\t\t\t\t\t\tdelete(e.ni, enis)\n\t\t\t\t\t\t\tsendProto = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else if _, noInterest := e.ni[subject]; noInterest {\n\t\t\t\t\tdelete(e.ni, subject)\n\t\t\t\t\tsendProto = true\n\t\t\t\t}\n\t\t\t} else if e.mode == InterestOnly {\n\t\t\t\t// We are in the mode where we always send RS+/- protocols.\n\t\t\t\tsendProto = true\n\t\t\t}\n\t\t\tif sendProto {\n\t\t\t\tif rsProto == nil {\n\t\t\t\t\t// Construct the RS+/- only once\n\t\t\t\t\tproto = rsProtoa[:0]\n\t\t\t\t\tif added {\n\t\t\t\t\t\tproto = append(proto, rSubBytes...)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tproto = append(proto, rUnsubBytes...)\n\t\t\t\t\t}\n\t\t\t\t\tproto = append(proto, accName...)\n\t\t\t\t\tproto = append(proto, ' ')\n\t\t\t\t\tproto = append(proto, sub.subject...)\n\t\t\t\t\tproto = append(proto, CR_LF...)\n\t\t\t\t\trsProto = proto\n\t\t\t\t} else {\n\t\t\t\t\t// Point to the already constructed RS+/-\n\t\t\t\t\tproto = rsProto\n\t\t\t\t}\n\t\t\t}\n\t\t} else if added && inMap {\n\t\t\t// Here, we have a `nil` entry for this account in\n\t\t\t// the map, which means that we have previously sent\n\t\t\t// an A-. We have a new subscription, so we need to\n\t\t\t// send an A+ and delete the entry from the map so\n\t\t\t// that we do this only once.\n\t\t\tdelete(c.gw.insim, accName)\n\t\t\tif accProto == nil {\n\t\t\t\t// Construct the A+ only once\n\t\t\t\tproto = accProtoa[:0]\n\t\t\t\tproto = append(proto, aSubBytes...)\n\t\t\t\tproto = append(proto, accName...)\n\t\t\t\tproto = append(proto, CR_LF...)\n\t\t\t\taccProto = proto\n\t\t\t} else {\n\t\t\t\t// Point to the already constructed A+\n\t\t\t\tproto = accProto\n\t\t\t}\n\t\t}\n\t\tif proto != nil {\n\t\t\tc.enqueueProto(proto)\n\t\t\tif c.trace {\n\t\t\t\tc.traceOutOp(\"\", proto[:len(proto)-LEN_CR_LF])\n\t\t\t}\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n}\n\n// This is invoked when the first (or last) queue subscription on a\n// given subject/group is registered (or unregistered). Sent to all\n// inbound gateways.\nfunc (s *Server) sendQueueSubOrUnsubToGateways(accName string, qsub *subscription, added bool) {\n\tif qsub.queue == nil {\n\t\treturn\n\t}\n\n\tgwsa := [16]*client{}\n\tgws := gwsa[:0]\n\ts.getInboundGatewayConnections(&gws)\n\tif len(gws) == 0 {\n\t\treturn\n\t}\n\n\tvar protoa [512]byte\n\tvar proto []byte\n\tfor _, c := range gws {\n\t\tif proto == nil {\n\t\t\tproto = protoa[:0]\n\t\t\tif added {\n\t\t\t\tproto = append(proto, rSubBytes...)\n\t\t\t} else {\n\t\t\t\tproto = append(proto, rUnsubBytes...)\n\t\t\t}\n\t\t\tproto = append(proto, accName...)\n\t\t\tproto = append(proto, ' ')\n\t\t\tproto = append(proto, qsub.subject...)\n\t\t\tproto = append(proto, ' ')\n\t\t\tproto = append(proto, qsub.queue...)\n\t\t\tif added {\n\t\t\t\t// For now, just use 1 for the weight\n\t\t\t\tproto = append(proto, ' ', '1')\n\t\t\t}\n\t\t\tproto = append(proto, CR_LF...)\n\t\t}\n\t\tc.mu.Lock()\n\t\t// If we add a queue sub, and we had previously sent an A-,\n\t\t// we don't need to send an A+ here, but we need to clear\n\t\t// the fact that we did sent the A- so that we don't send\n\t\t// an A+ when we will get the first non-queue sub registered.\n\t\tif added {\n\t\t\tif ei, ok := c.gw.insim[accName]; ok && ei == nil {\n\t\t\t\tdelete(c.gw.insim, accName)\n\t\t\t}\n\t\t}\n\t\tc.enqueueProto(proto)\n\t\tif c.trace {\n\t\t\tc.traceOutOp(\"\", proto[:len(proto)-LEN_CR_LF])\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n}\n\n// This is invoked when a subscription (plain or queue) is\n// added/removed locally or in our cluster. We use ref counting\n// to know when to update the inbound gateways.\n// \nfunc (s *Server) gatewayUpdateSubInterest(accName string, sub *subscription, change int32) {\n\tif sub.si {\n\t\treturn\n\t}\n\n\tvar (\n\t\tkeya [1024]byte\n\t\tkey = keya[:0]\n\t\tentry *sitally\n\t\tisNew bool\n\t)\n\n\ts.gateway.pasi.Lock()\n\tdefer s.gateway.pasi.Unlock()\n\n\taccMap := s.gateway.pasi.m\n\n\t// First see if we have the account\n\tst := accMap[accName]\n\tif st == nil {\n\t\t// Ignore remove of something we don't have\n\t\tif change < 0 {\n\t\t\treturn\n\t\t}\n\t\tst = make(map[string]*sitally)\n\t\taccMap[accName] = st\n\t\tisNew = true\n\t}\n\t// Lookup: build the key as subject[+' '+queue]\n\tkey = append(key, sub.subject...)\n\tif sub.queue != nil {\n\t\tkey = append(key, ' ')\n\t\tkey = append(key, sub.queue...)\n\t}\n\tif !isNew {\n\t\tentry = st[string(key)]\n\t}\n\tfirst := false\n\tlast := false\n\tif entry == nil {\n\t\t// Ignore remove of something we don't have\n\t\tif change < 0 {\n\t\t\treturn\n\t\t}\n\t\tentry = &sitally{n: change, q: sub.queue != nil}\n\t\tst[string(key)] = entry\n\t\tfirst = true\n\t} else {\n\t\tentry.n += change\n\t\tif entry.n <= 0 {\n\t\t\tdelete(st, bytesToString(key))\n\t\t\tlast = true\n\t\t\tif len(st) == 0 {\n\t\t\t\tdelete(accMap, accName)\n\t\t\t}\n\t\t}\n\t}\n\tif sub.client != nil {\n\t\trsubs := &s.gateway.rsubs\n\t\tacc := sub.client.acc\n\t\tsli, _ := rsubs.Load(acc)\n\t\tif change > 0 {\n\t\t\tvar sl *Sublist\n\t\t\tif sli == nil {\n\t\t\t\tsl = NewSublistNoCache()\n\t\t\t\trsubs.Store(acc, sl)\n\t\t\t} else {\n\t\t\t\tsl = sli.(*Sublist)\n\t\t\t}\n\t\t\tsl.Insert(sub)\n\t\t\ttime.AfterFunc(s.gateway.recSubExp, func() {\n\t\t\t\tsl.Remove(sub)\n\t\t\t})\n\t\t} else if sli != nil {\n\t\t\tsl := sli.(*Sublist)\n\t\t\tsl.Remove(sub)\n\t\t\tif sl.Count() == 0 {\n\t\t\t\trsubs.Delete(acc)\n\t\t\t}\n\t\t}\n\t}\n\tif first || last {\n\t\tif entry.q {\n\t\t\ts.sendQueueSubOrUnsubToGateways(accName, sub, first)\n\t\t} else {\n\t\t\ts.maybeSendSubOrUnsubToGateways(accName, sub, first)\n\t\t}\n\t}\n}\n\n// Returns true if the given subject is a GW routed reply subject,\n// that is, starts with $GNR and is long enough to contain cluster/server hash\n// and subject.\nfunc isGWRoutedReply(subj []byte) bool {\n\treturn len(subj) > gwSubjectOffset && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix\n}\n\n// Same than isGWRoutedReply but accepts the old prefix $GR and returns\n// a boolean indicating if this is the old prefix\nfunc isGWRoutedSubjectAndIsOldPrefix(subj []byte) (bool, bool) {\n\tif isGWRoutedReply(subj) {\n\t\treturn true, false\n\t}\n\tif len(subj) > oldGWReplyStart && bytesToString(subj[:oldGWReplyPrefixLen]) == oldGWReplyPrefix {\n\t\treturn true, true\n\t}\n\treturn false, false\n}\n\n// Returns true if subject starts with \"$GNR.\". This is to check that\n// clients can't publish on this subject.\nfunc hasGWRoutedReplyPrefix(subj []byte) bool {\n\treturn len(subj) > gwReplyPrefixLen && bytesToString(subj[:gwReplyPrefixLen]) == gwReplyPrefix\n}\n\n// Evaluates if the given reply should be mapped or not.\nfunc (g *srvGateway) shouldMapReplyForGatewaySend(acc *Account, reply []byte) bool {\n\t// If for this account there is a recent matching subscription interest\n\t// then we will map.\n\tsli, _ := g.rsubs.Load(acc)\n\tif sli == nil {\n\t\treturn false\n\t}\n\tsl := sli.(*Sublist)\n\tif sl.Count() > 0 {\n\t\tif sl.HasInterest(string(reply)) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nvar subPool = &sync.Pool{\n\tNew: func() any {\n\t\treturn &subscription{}\n\t},\n}\n\n// May send a message to all outbound gateways. It is possible\n// that the message is not sent to a given gateway if for instance\n// it is known that this gateway has no interest in the account or\n// subject, etc..\n// When invoked from a LEAF connection, `checkLeafQF` should be passed as `true`\n// so that we skip any queue subscription interest that is not part of the\n// `c.pa.queues` filter (similar to what we do in `processMsgResults`). However,\n// when processing service imports, then this boolean should be passes as `false`,\n// regardless if it is a LEAF connection or not.\n// \nfunc (c *client) sendMsgToGateways(acc *Account, msg, subject, reply []byte, qgroups [][]byte, checkLeafQF bool) bool {\n\t// We had some times when we were sending across a GW with no subject, and the other side would break\n\t// due to parser error. These need to be fixed upstream but also double check here.\n\tif len(subject) == 0 {\n\t\treturn false\n\t}\n\tgwsa := [16]*client{}\n\tgws := gwsa[:0]\n\t// This is in fast path, so avoid calling functions when possible.\n\t// Get the outbound connections in place instead of calling\n\t// getOutboundGatewayConnections().\n\tsrv := c.srv\n\tgw := srv.gateway\n\tgw.RLock()\n\tfor i := 0; i < len(gw.outo); i++ {\n\t\tgws = append(gws, gw.outo[i])\n\t}\n\tthisClusterReplyPrefix := gw.replyPfx\n\tthisClusterOldReplyPrefix := gw.oldReplyPfx\n\tgw.RUnlock()\n\tif len(gws) == 0 {\n\t\treturn false\n\t}\n\n\t// Copy off original pa in case it changes.\n\tpa := c.pa\n\n\tmt, _ := c.isMsgTraceEnabled()\n\tif mt != nil {\n\t\t// We are going to replace \"pa\" with our copy of c.pa, but to restore\n\t\t// to the original copy of c.pa, we need to save it again.\n\t\tcpa := c.pa\n\t\tmsg = mt.setOriginAccountHeaderIfNeeded(c, acc, msg)\n\t\tdefer func() { c.pa = cpa }()\n\t\t// Update pa with our current c.pa state.\n\t\tpa = c.pa\n\t}\n\n\tvar (\n\t\tqueuesa = [512]byte{}\n\t\tqueues = queuesa[:0]\n\t\taccName = acc.Name\n\t\tmreplya [256]byte\n\t\tmreply []byte\n\t\tdstHash []byte\n\t\tcheckReply = len(reply) > 0\n\t\tdidDeliver bool\n\t\tprodIsMQTT = c.isMqtt()\n\t\tdlvMsgs int64\n\t\tdlvExtraSz int64\n\t)\n\n\t// Get a subscription from the pool\n\tsub := subPool.Get().(*subscription)\n\n\t// Check if the subject is on the reply prefix, if so, we\n\t// need to send that message directly to the origin cluster.\n\tdirectSend, old := isGWRoutedSubjectAndIsOldPrefix(subject)\n\tif directSend {\n\t\tif old {\n\t\t\tdstHash = subject[oldGWReplyPrefixLen : oldGWReplyStart-1]\n\t\t} else {\n\t\t\tdstHash = subject[gwClusterOffset : gwClusterOffset+gwHashLen]\n\t\t}\n\t}\n\tfor i := 0; i < len(gws); i++ {\n\t\tgwc := gws[i]\n\t\tif directSend {\n\t\t\tgwc.mu.Lock()\n\t\t\tvar ok bool\n\t\t\tif gwc.gw.cfg != nil {\n\t\t\t\tif old {\n\t\t\t\t\tok = bytes.Equal(dstHash, gwc.gw.cfg.oldHash)\n\t\t\t\t} else {\n\t\t\t\t\tok = bytes.Equal(dstHash, gwc.gw.cfg.hash)\n\t\t\t\t}\n\t\t\t}\n\t\t\tgwc.mu.Unlock()\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else {\n\t\t\t// Plain sub interest and queue sub results for this account/subject\n\t\t\tpsi, qr := gwc.gatewayInterest(accName, subject)\n\t\t\tif !psi && qr == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tqueues = queuesa[:0]\n\t\t\tif qr != nil {\n\t\t\t\tfor i := 0; i < len(qr.qsubs); i++ {\n\t\t\t\t\tqsubs := qr.qsubs[i]\n\t\t\t\t\tif len(qsubs) > 0 {\n\t\t\t\t\t\tqueue := qsubs[0].queue\n\t\t\t\t\t\tif checkLeafQF {\n\t\t\t\t\t\t\t// Skip any queue that is not in the leaf's queue filter.\n\t\t\t\t\t\t\tskip := true\n\t\t\t\t\t\t\tfor _, qn := range c.pa.queues {\n\t\t\t\t\t\t\t\tif bytes.Equal(queue, qn) {\n\t\t\t\t\t\t\t\t\tskip = false\n\t\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif skip {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t// Now we still need to check that it was not delivered\n\t\t\t\t\t\t\t// locally by checking the given `qgroups`.\n\t\t\t\t\t\t}\n\t\t\t\t\t\tadd := true\n\t\t\t\t\t\tfor _, qn := range qgroups {\n\t\t\t\t\t\t\tif bytes.Equal(queue, qn) {\n\t\t\t\t\t\t\t\tadd = false\n\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif add {\n\t\t\t\t\t\t\tqgroups = append(qgroups, queue)\n\t\t\t\t\t\t\tqueues = append(queues, queue...)\n\t\t\t\t\t\t\tqueues = append(queues, ' ')\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !psi && len(queues) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif checkReply {\n\t\t\t// Check/map only once\n\t\t\tcheckReply = false\n\t\t\t// Assume we will use original\n\t\t\tmreply = reply\n\t\t\t// Decide if we should map.\n\t\t\tif gw.shouldMapReplyForGatewaySend(acc, reply) {\n\t\t\t\tmreply = mreplya[:0]\n\t\t\t\tgwc.mu.Lock()\n\t\t\t\tuseOldPrefix := gwc.gw.useOldPrefix\n\t\t\t\tgwc.mu.Unlock()\n\t\t\t\tif useOldPrefix {\n\t\t\t\t\tmreply = append(mreply, thisClusterOldReplyPrefix...)\n\t\t\t\t} else {\n\t\t\t\t\tmreply = append(mreply, thisClusterReplyPrefix...)\n\t\t\t\t}\n\t\t\t\tmreply = append(mreply, reply...)\n\t\t\t}\n\t\t}\n\n\t\t// Assume original message\n\t\tdmsg := msg\n\t\tif mt != nil {\n\t\t\t// If trace is enabled, we need to set the hop header per gateway.\n\t\t\tdmsg = mt.setHopHeader(c, dmsg)\n\t\t}\n\n\t\t// Setup the message header.\n\t\t// Make sure we are an 'R' proto by default\n\t\tc.msgb[0] = 'R'\n\t\tmh := c.msgb[:msgHeadProtoLen]\n\t\tmh = append(mh, accName...)\n\t\tmh = append(mh, ' ')\n\t\tmh = append(mh, subject...)\n\t\tmh = append(mh, ' ')\n\t\tif len(queues) > 0 {\n\t\t\tif len(reply) > 0 {\n\t\t\t\tmh = append(mh, \"+ \"...) // Signal that there is a reply.\n\t\t\t\tmh = append(mh, mreply...)\n\t\t\t\tmh = append(mh, ' ')\n\t\t\t} else {\n\t\t\t\tmh = append(mh, \"| \"...) // Only queues\n\t\t\t}\n\t\t\tmh = append(mh, queues...)\n\t\t} else if len(reply) > 0 {\n\t\t\tmh = append(mh, mreply...)\n\t\t\tmh = append(mh, ' ')\n\t\t}\n\t\t// Headers\n\t\thasHeader := c.pa.hdr > 0\n\t\tcanReceiveHeader := gwc.headers\n\n\t\tif hasHeader {\n\t\t\tif canReceiveHeader {\n\t\t\t\tmh[0] = 'H'\n\t\t\t\tmh = append(mh, c.pa.hdb...)\n\t\t\t\tmh = append(mh, ' ')\n\t\t\t\tmh = append(mh, c.pa.szb...)\n\t\t\t} else {\n\t\t\t\t// If we are here we need to truncate the payload size\n\t\t\t\tnsz := strconv.Itoa(c.pa.size - c.pa.hdr)\n\t\t\t\tmh = append(mh, nsz...)\n\t\t\t}\n\t\t} else {\n\t\t\tmh = append(mh, c.pa.szb...)\n\t\t}\n\n\t\tmh = append(mh, CR_LF...)\n\n\t\t// We reuse the subscription object that we pass to deliverMsg.\n\t\t// So set/reset important fields.\n\t\tsub.nm, sub.max = 0, 0\n\t\tsub.client = gwc\n\t\tsub.subject = subject\n\t\tif c.deliverMsg(prodIsMQTT, sub, acc, subject, mreply, mh, dmsg, false) {\n\t\t\t// We don't count internal deliveries so count only if sub.icb is nil\n\t\t\tif sub.icb == nil {\n\t\t\t\tdlvMsgs++\n\t\t\t\tdlvExtraSz += int64(len(dmsg) - len(msg))\n\t\t\t}\n\t\t\tdidDeliver = true\n\t\t}\n\n\t\t// If we set the header reset the origin pub args.\n\t\tif mt != nil {\n\t\t\tc.pa = pa\n\t\t}\n\t}\n\tif dlvMsgs > 0 {\n\t\ttotalBytes := dlvMsgs*int64(len(msg)) + dlvExtraSz\n\t\t// For non MQTT producers, remove the CR_LF * number of messages\n\t\tif !prodIsMQTT {\n\t\t\ttotalBytes -= dlvMsgs * int64(LEN_CR_LF)\n\t\t}\n\t\tif acc != nil {\n\t\t\tacc.stats.Lock()\n\t\t\tacc.stats.outMsgs += dlvMsgs\n\t\t\tacc.stats.outBytes += totalBytes\n\t\t\tacc.stats.gw.outMsgs += dlvMsgs\n\t\t\tacc.stats.gw.outBytes += totalBytes\n\t\t\tacc.stats.Unlock()\n\t\t}\n\t\tatomic.AddInt64(&srv.outMsgs, dlvMsgs)\n\t\tatomic.AddInt64(&srv.outBytes, totalBytes)\n\t}\n\t// Done with subscription, put back to pool. We don't need\n\t// to reset content since we explicitly set when using it.\n\t// However, make sure to not hold a reference to a connection.\n\tsub.client = nil\n\tsubPool.Put(sub)\n\treturn didDeliver\n}\n\n// Possibly sends an A- to the remote gateway `c`.\n// Invoked when processing an inbound message and the account is not found.\n// A check under a lock that protects processing of SUBs and UNSUBs is\n// done to make sure that we don't send the A- if a subscription has just\n// been created at the same time, which would otherwise results in the\n// remote never sending messages on this account until a new subscription\n// is created.\nfunc (s *Server) gatewayHandleAccountNoInterest(c *client, accName []byte) {\n\t// Check and possibly send the A- under this lock.\n\ts.gateway.pasi.Lock()\n\tdefer s.gateway.pasi.Unlock()\n\n\tsi, inMap := s.gateway.pasi.m[string(accName)]\n\tif inMap && si != nil && len(si) > 0 {\n\t\treturn\n\t}\n\tc.sendAccountUnsubToGateway(accName)\n}\n\n// Helper that sends an A- to this remote gateway if not already done.\n// This function should not be invoked directly but instead be invoked\n// by functions holding the gateway.pasi's Lock.\nfunc (c *client) sendAccountUnsubToGateway(accName []byte) {\n\t// Check if we have sent the A- or not.\n\tc.mu.Lock()\n\te, sent := c.gw.insim[string(accName)]\n\tif e != nil || !sent {\n\t\t// Add a nil value to indicate that we have sent an A-\n\t\t// so that we know to send A+ when needed.\n\t\tc.gw.insim[string(accName)] = nil\n\t\tvar protoa [256]byte\n\t\tproto := protoa[:0]\n\t\tproto = append(proto, aUnsubBytes...)\n\t\tproto = append(proto, accName...)\n\t\tproto = append(proto, CR_LF...)\n\t\tc.enqueueProto(proto)\n\t\tif c.trace {\n\t\t\tc.traceOutOp(\"\", proto[:len(proto)-LEN_CR_LF])\n\t\t}\n\t}\n\tc.mu.Unlock()\n}\n\n// Possibly sends an A- for this account or RS- for this subject.\n// Invoked when processing an inbound message and the account is found\n// but there is no interest on this subject.\n// A test is done under a lock that protects processing of SUBs and UNSUBs\n// and if there is no subscription at this time, we send an A-. If there\n// is at least a subscription, but no interest on this subject, we send\n// an RS- for this subject (if not already done).\nfunc (s *Server) gatewayHandleSubjectNoInterest(c *client, acc *Account, accName, subject []byte) {\n\ts.gateway.pasi.Lock()\n\tdefer s.gateway.pasi.Unlock()\n\n\t// If there is no subscription for this account, we would normally\n\t// send an A-, however, if this account has the internal subscription\n\t// for service reply, send a specific RS- for the subject instead.\n\t// Need to grab the lock here since sublist can change during reload.\n\tacc.mu.RLock()\n\thasSubs := acc.sl.Count() > 0 || acc.siReply != nil\n\tacc.mu.RUnlock()\n\n\t// If there is at least a subscription, possibly send RS-\n\tif hasSubs {\n\t\tsendProto := false\n\t\tc.mu.Lock()\n\t\t// Send an RS- protocol if not already done and only if\n\t\t// not in the modeInterestOnly.\n\t\te := c.gw.insim[string(accName)]\n\t\tif e == nil {\n\t\t\te = &insie{ni: make(map[string]struct{})}\n\t\t\te.ni[string(subject)] = struct{}{}\n\t\t\tc.gw.insim[string(accName)] = e\n\t\t\tsendProto = true\n\t\t} else if e.ni != nil {\n\t\t\t// If we are not in modeInterestOnly, check if we\n\t\t\t// have already sent an RS-\n\t\t\tif _, alreadySent := e.ni[string(subject)]; !alreadySent {\n\t\t\t\t// TODO(ik): pick some threshold as to when\n\t\t\t\t// we need to switch mode\n\t\t\t\tif len(e.ni) >= gatewayMaxRUnsubBeforeSwitch {\n\t\t\t\t\t// If too many RS-, switch to all-subs-mode.\n\t\t\t\t\tc.gatewaySwitchAccountToSendAllSubs(e, string(accName))\n\t\t\t\t} else {\n\t\t\t\t\te.ni[string(subject)] = struct{}{}\n\t\t\t\t\tsendProto = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif sendProto {\n\t\t\tvar (\n\t\t\t\tprotoa = [512]byte{}\n\t\t\t\tproto = protoa[:0]\n\t\t\t)\n\t\t\tproto = append(proto, rUnsubBytes...)\n\t\t\tproto = append(proto, accName...)\n\t\t\tproto = append(proto, ' ')\n\t\t\tproto = append(proto, subject...)\n\t\t\tproto = append(proto, CR_LF...)\n\t\t\tc.enqueueProto(proto)\n\t\t\tif c.trace {\n\t\t\t\tc.traceOutOp(\"\", proto[:len(proto)-LEN_CR_LF])\n\t\t\t}\n\t\t}\n\t\tc.mu.Unlock()\n\t} else {\n\t\t// There is not a single subscription, send an A- (if not already done).\n\t\tc.sendAccountUnsubToGateway([]byte(acc.Name))\n\t}\n}\n\n// Returns the cluster hash from the gateway reply prefix\nfunc (g *srvGateway) getClusterHash() []byte {\n\tg.RLock()\n\tclusterHash := g.replyPfx[gwClusterOffset : gwClusterOffset+gwHashLen]\n\tg.RUnlock()\n\treturn clusterHash\n}\n\n// Store this route in map with the key being the remote server's name hash\n// and the remote server's ID hash used by gateway replies mapping routing.\nfunc (s *Server) storeRouteByHash(srvIDHash string, c *client) {\n\tif !s.gateway.enabled {\n\t\treturn\n\t}\n\ts.gateway.routesIDByHash.Store(srvIDHash, c)\n}\n\n// Remove the route with the given keys from the map.\nfunc (s *Server) removeRouteByHash(srvIDHash string) {\n\tif !s.gateway.enabled {\n\t\treturn\n\t}\n\ts.gateway.routesIDByHash.Delete(srvIDHash)\n}\n\n// Returns the route with given hash or nil if not found.\n// This is for gateways only.\nfunc (s *Server) getRouteByHash(hash, accName []byte) (*client, bool) {\n\tid := bytesToString(hash)\n\tvar perAccount bool\n\tif v, ok := s.accRouteByHash.Load(bytesToString(accName)); ok {\n\t\tif v == nil {\n\t\t\tid += bytesToString(accName)\n\t\t\tperAccount = true\n\t\t} else {\n\t\t\tid += strconv.Itoa(v.(int))\n\t\t}\n\t}\n\tif v, ok := s.gateway.routesIDByHash.Load(id); ok {\n\t\treturn v.(*client), perAccount\n\t} else if !perAccount {\n\t\t// Check if we have a \"no pool\" connection at index 0.\n\t\tif v, ok := s.gateway.routesIDByHash.Load(bytesToString(hash) + \"0\"); ok {\n\t\t\tif r := v.(*client); r != nil {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tnoPool := r.route.noPool\n\t\t\t\tr.mu.Unlock()\n\t\t\t\tif noPool {\n\t\t\t\t\treturn r, false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil, perAccount\n}\n\n// Returns the subject from the routed reply\nfunc getSubjectFromGWRoutedReply(reply []byte, isOldPrefix bool) []byte {\n\tif isOldPrefix {\n\t\treturn reply[oldGWReplyStart:]\n\t}\n\treturn reply[gwSubjectOffset:]\n}\n\n// This should be invoked only from processInboundGatewayMsg() or\n// processInboundRoutedMsg() and is checking if the subject\n// (c.pa.subject) has the _GR_ prefix. If so, this is processed\n// as a GW reply and `true` is returned to indicate to the caller\n// that it should stop processing.\n// If gateway is not enabled on this server or if the subject\n// does not start with _GR_, `false` is returned and caller should\n// process message as usual.\nfunc (c *client) handleGatewayReply(msg []byte) (processed bool) {\n\t// Do not handle GW prefixed messages if this server does not have\n\t// gateway enabled or if the subject does not start with the previx.\n\tif !c.srv.gateway.enabled {\n\t\treturn false\n\t}\n\tisGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(c.pa.subject)\n\tif !isGWPrefix {\n\t\treturn false\n\t}\n\t// Save original subject (in case we have to forward)\n\torgSubject := c.pa.subject\n\n\tvar clusterHash []byte\n\tvar srvHash []byte\n\tvar subject []byte\n\n\tif oldPrefix {\n\t\tclusterHash = c.pa.subject[oldGWReplyPrefixLen : oldGWReplyStart-1]\n\t\t// Check if this reply is intended for our cluster.\n\t\tif !bytes.Equal(clusterHash, c.srv.gateway.oldHash) {\n\t\t\t// We could report, for now, just drop.\n\t\t\treturn true\n\t\t}\n\t\tsubject = c.pa.subject[oldGWReplyStart:]\n\t} else {\n\t\tclusterHash = c.pa.subject[gwClusterOffset : gwClusterOffset+gwHashLen]\n\t\t// Check if this reply is intended for our cluster.\n\t\tif !bytes.Equal(clusterHash, c.srv.gateway.getClusterHash()) {\n\t\t\t// We could report, for now, just drop.\n\t\t\treturn true\n\t\t}\n\t\tsrvHash = c.pa.subject[gwServerOffset : gwServerOffset+gwHashLen]\n\t\tsubject = c.pa.subject[gwSubjectOffset:]\n\t}\n\n\tvar route *client\n\tvar perAccount bool\n\n\t// If the origin is not this server, get the route this should be sent to.\n\tif c.kind == GATEWAY && srvHash != nil && !bytes.Equal(srvHash, c.srv.gateway.sIDHash) {\n\t\troute, perAccount = c.srv.getRouteByHash(srvHash, c.pa.account)\n\t\t// This will be possibly nil, and in this case we will try to process\n\t\t// the interest from this server.\n\t}\n\n\t// Adjust the subject\n\tc.pa.subject = subject\n\n\t// Use a stack buffer to rewrite c.pa.cache since we only need it for\n\t// getAccAndResultFromCache()\n\tvar _pacache [256]byte\n\tpacache := _pacache[:0]\n\t// For routes that are dedicated to an account, do not put the account\n\t// name in the pacache.\n\tif c.kind == GATEWAY || (c.kind == ROUTER && c.route != nil && len(c.route.accName) == 0) {\n\t\tpacache = append(pacache, c.pa.account...)\n\t\tpacache = append(pacache, ' ')\n\t}\n\tpacache = append(pacache, c.pa.subject...)\n\tc.pa.pacache = pacache\n\n\tacc, r := c.getAccAndResultFromCache()\n\tif acc == nil {\n\t\ttypeConn := \"routed\"\n\t\tif c.kind == GATEWAY {\n\t\t\ttypeConn = \"gateway\"\n\t\t}\n\t\tc.Debugf(\"Unknown account %q for %s message on subject: %q\", c.pa.account, typeConn, c.pa.subject)\n\t\tif c.kind == GATEWAY {\n\t\t\tc.srv.gatewayHandleAccountNoInterest(c, c.pa.account)\n\t\t}\n\t\treturn true\n\t}\n\n\t// If route is nil, we will process the incoming message locally.\n\tif route == nil {\n\t\t// Check if this is a service reply subject (_R_)\n\t\tisServiceReply := isServiceReply(c.pa.subject)\n\n\t\tvar queues [][]byte\n\t\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\t\tflags := pmrCollectQueueNames | pmrIgnoreEmptyQueueFilter\n\t\t\t// If this message came from a ROUTE, allow to pick queue subs\n\t\t\t// only if the message was directly sent by the \"gateway\" server\n\t\t\t// in our cluster that received it.\n\t\t\tif c.kind == ROUTER {\n\t\t\t\tflags |= pmrAllowSendFromRouteToRoute\n\t\t\t}\n\t\t\t_, queues = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flags)\n\t\t}\n\t\t// Since this was a reply that made it to the origin cluster,\n\t\t// we now need to send the message with the real subject to\n\t\t// gateways in case they have interest on that reply subject.\n\t\tif !isServiceReply {\n\t\t\tc.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, queues, false)\n\t\t}\n\t} else if c.kind == GATEWAY {\n\t\t// Only if we are a gateway connection should we try to route\n\t\t// to the server where the request originated.\n\t\tvar bufa [256]byte\n\t\tvar buf = bufa[:0]\n\t\tbuf = append(buf, msgHeadProto...)\n\t\tif !perAccount {\n\t\t\tbuf = append(buf, acc.Name...)\n\t\t\tbuf = append(buf, ' ')\n\t\t}\n\t\tbuf = append(buf, orgSubject...)\n\t\tbuf = append(buf, ' ')\n\t\tif len(c.pa.reply) > 0 {\n\t\t\tbuf = append(buf, c.pa.reply...)\n\t\t\tbuf = append(buf, ' ')\n\t\t}\n\t\tszb := c.pa.szb\n\t\tif c.pa.hdr >= 0 {\n\t\t\tif route.headers {\n\t\t\t\tbuf[0] = 'H'\n\t\t\t\tbuf = append(buf, c.pa.hdb...)\n\t\t\t\tbuf = append(buf, ' ')\n\t\t\t} else {\n\t\t\t\tszb = []byte(strconv.Itoa(c.pa.size - c.pa.hdr))\n\t\t\t\tmsg = msg[c.pa.hdr:]\n\t\t\t}\n\t\t}\n\t\tbuf = append(buf, szb...)\n\t\tmhEnd := len(buf)\n\t\tbuf = append(buf, _CRLF_...)\n\t\tbuf = append(buf, msg...)\n\n\t\troute.mu.Lock()\n\t\troute.enqueueProto(buf)\n\t\tif route.trace {\n\t\t\troute.traceOutOp(\"\", buf[:mhEnd])\n\t\t}\n\t\troute.mu.Unlock()\n\t}\n\treturn true\n}\n\n// Process a message coming from a remote gateway. Send to any sub/qsub\n// in our cluster that is matching. When receiving a message for an\n// account or subject for which there is no interest in this cluster\n// an A-/RS- protocol may be send back.\n// \nfunc (c *client) processInboundGatewayMsg(msg []byte) {\n\t// Update statistics\n\tc.in.msgs++\n\t// The msg includes the CR_LF, so pull back out for accounting.\n\tsize := len(msg) - LEN_CR_LF\n\tc.in.bytes += int32(size)\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\n\t// Mostly under testing scenarios.\n\tif c.srv == nil {\n\t\treturn\n\t}\n\n\t// If the subject (c.pa.subject) has the gateway prefix, this function will\n\t// handle it.\n\tif c.handleGatewayReply(msg) {\n\t\t// We are done here.\n\t\treturn\n\t}\n\n\tacc, r := c.getAccAndResultFromCache()\n\tif acc == nil {\n\t\tc.Debugf(\"Unknown account %q for gateway message on subject: %q\", c.pa.account, c.pa.subject)\n\t\tc.srv.gatewayHandleAccountNoInterest(c, c.pa.account)\n\t\treturn\n\t}\n\n\tacc.stats.Lock()\n\tacc.stats.inMsgs++\n\tacc.stats.inBytes += int64(size)\n\tacc.stats.gw.inMsgs++\n\tacc.stats.gw.inBytes += int64(size)\n\tacc.stats.Unlock()\n\n\t// Check if this is a service reply subject (_R_)\n\tnoInterest := len(r.psubs) == 0\n\tcheckNoInterest := true\n\tif acc.NumServiceImports() > 0 {\n\t\tif isServiceReply(c.pa.subject) {\n\t\t\tcheckNoInterest = false\n\t\t} else {\n\t\t\t// We need to eliminate the subject interest from the service imports here to\n\t\t\t// make sure we send the proper no interest if the service import is the only interest.\n\t\t\tnoInterest = true\n\t\t\tfor _, sub := range r.psubs {\n\t\t\t\t// sub.si indicates that this is a subscription for service import, and is immutable.\n\t\t\t\t// So sub.si is false, then this is a subscription for something else, so there is\n\t\t\t\t// actually proper interest.\n\t\t\t\tif !sub.si {\n\t\t\t\t\tnoInterest = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif checkNoInterest && noInterest {\n\t\t// If there is no interest on plain subs, possibly send an RS-,\n\t\t// even if there is qsubs interest.\n\t\tc.srv.gatewayHandleSubjectNoInterest(c, acc, c.pa.account, c.pa.subject)\n\n\t\t// If there is also no queue filter, then no point in continuing\n\t\t// (even if r.qsubs i > 0).\n\t\tif len(c.pa.queues) == 0 {\n\t\t\treturn\n\t\t}\n\t}\n\tc.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag)\n}\n\n// Indicates that the remote which we are sending messages to\n// has decided to send us all its subs interest so that we\n// stop doing optimistic sends.\n// \nfunc (c *client) gatewayAllSubsReceiveStart(info *Info) {\n\taccount := getAccountFromGatewayCommand(c, info, \"start\")\n\tif account == \"\" {\n\t\treturn\n\t}\n\n\tc.Debugf(\"Gateway %q: switching account %q to %s mode\",\n\t\tinfo.Gateway, account, InterestOnly)\n\n\t// Since the remote would send us this start command\n\t// only after sending us too many RS- for this account,\n\t// we should always have an entry here.\n\t// TODO(ik): Should we close connection with protocol violation\n\t// error if that happens?\n\tei, _ := c.gw.outsim.Load(account)\n\tif ei != nil {\n\t\te := ei.(*outsie)\n\t\te.Lock()\n\t\te.mode = Transitioning\n\t\te.Unlock()\n\t} else {\n\t\te := &outsie{sl: NewSublistWithCache()}\n\t\te.mode = Transitioning\n\t\tc.mu.Lock()\n\t\tc.gw.outsim.Store(account, e)\n\t\tc.mu.Unlock()\n\t}\n}\n\n// Indicates that the remote has finished sending all its\n// subscriptions and we should now not send unless we know\n// there is explicit interest.\n// \nfunc (c *client) gatewayAllSubsReceiveComplete(info *Info) {\n\taccount := getAccountFromGatewayCommand(c, info, \"complete\")\n\tif account == _EMPTY_ {\n\t\treturn\n\t}\n\t// Done receiving all subs from remote. Set the `ni`\n\t// map to nil so that gatewayInterest() no longer\n\t// uses it.\n\tei, _ := c.gw.outsim.Load(account)\n\tif ei != nil {\n\t\te := ei.(*outsie)\n\t\t// Needs locking here since `ni` is checked by\n\t\t// many go-routines calling gatewayInterest()\n\t\te.Lock()\n\t\te.ni = nil\n\t\te.mode = InterestOnly\n\t\te.Unlock()\n\n\t\tc.Debugf(\"Gateway %q: switching account %q to %s mode complete\",\n\t\t\tinfo.Gateway, account, InterestOnly)\n\t}\n}\n\n// small helper to get the account name from the INFO command.\nfunc getAccountFromGatewayCommand(c *client, info *Info, cmd string) string {\n\tif info.GatewayCmdPayload == nil {\n\t\tc.sendErrAndErr(fmt.Sprintf(\"Account absent from receive-all-subscriptions-%s command\", cmd))\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn _EMPTY_\n\t}\n\treturn string(info.GatewayCmdPayload)\n}\n\n// Switch to send-all-subs mode for the given gateway and account.\n// This is invoked when processing an inbound message and we\n// reach a point where we had to send a lot of RS- for this\n// account. We will send an INFO protocol to indicate that we\n// start sending all our subs (for this account), followed by\n// all subs (RS+) and finally an INFO to indicate the end of it.\n// The remote will then send messages only if it finds explicit\n// interest in the sublist created based on all RS+ that we just\n// sent.\n// The client's lock is held on entry.\n// \nfunc (c *client) gatewaySwitchAccountToSendAllSubs(e *insie, accName string) {\n\t// Set this map to nil so that the no-interest is no longer checked.\n\te.ni = nil\n\t// Switch mode to transitioning to prevent switchAccountToInterestMode\n\t// to possibly call this function multiple times.\n\te.mode = Transitioning\n\ts := c.srv\n\n\tremoteGWName := c.gw.name\n\tc.Debugf(\"Gateway %q: switching account %q to %s mode\",\n\t\tremoteGWName, accName, InterestOnly)\n\n\t// Function that will create an INFO protocol\n\t// and set proper command.\n\tsendCmd := func(cmd byte, useLock bool) {\n\t\t// Use bare server info and simply set the\n\t\t// gateway name and command\n\t\tinfo := Info{\n\t\t\tGateway: s.gateway.name,\n\t\t\tGatewayCmd: cmd,\n\t\t\tGatewayCmdPayload: stringToBytes(accName),\n\t\t}\n\n\t\tb, _ := json.Marshal(&info)\n\t\tinfoJSON := []byte(fmt.Sprintf(InfoProto, b))\n\t\tif useLock {\n\t\t\tc.mu.Lock()\n\t\t}\n\t\tc.enqueueProto(infoJSON)\n\t\tif useLock {\n\t\t\tc.mu.Unlock()\n\t\t}\n\t}\n\t// Send the start command. When remote receives this,\n\t// it may continue to send optimistic messages, but\n\t// it will start to register RS+/RS- in sublist instead\n\t// of noInterest map.\n\tsendCmd(gatewayCmdAllSubsStart, false)\n\n\t// Execute this in separate go-routine as to not block\n\t// the readLoop (which may cause the otherside to close\n\t// the connection due to slow consumer)\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\n\t\ts.sendAccountSubsToGateway(c, accName)\n\t\t// Send the complete command. When the remote receives\n\t\t// this, it will not send a message unless it has a\n\t\t// matching sub from us.\n\t\tsendCmd(gatewayCmdAllSubsComplete, true)\n\n\t\tc.Debugf(\"Gateway %q: switching account %q to %s mode complete\",\n\t\t\tremoteGWName, accName, InterestOnly)\n\t})\n}\n\n// Keeps track of the routed reply to be used when/if application sends back a\n// message on the reply without the prefix.\n// If `client` is not nil, it will be stored in the client gwReplyMapping structure,\n// and client lock is held on entry.\n// If `client` is nil, the mapping is stored in the client's account's gwReplyMapping\n// structure. Account lock will be explicitly acquired.\n// This is a server receiver because we use a timer interval that is avail in\n// Server.gateway object.\nfunc (s *Server) trackGWReply(c *client, acc *Account, reply, routedReply []byte) {\n\tvar l sync.Locker\n\tvar g *gwReplyMapping\n\tif acc != nil {\n\t\tacc.mu.Lock()\n\t\tdefer acc.mu.Unlock()\n\t\tg = &acc.gwReplyMapping\n\t\tl = &acc.mu\n\t} else {\n\t\tg = &c.gwReplyMapping\n\t\tl = &c.mu\n\t}\n\tttl := s.gateway.recSubExp\n\twasEmpty := len(g.mapping) == 0\n\tif g.mapping == nil {\n\t\tg.mapping = make(map[string]*gwReplyMap)\n\t}\n\t// The reason we pass both `reply` and `routedReply`, is that in some cases,\n\t// `routedReply` may have a deliver subject appended, something look like:\n\t// \"_GR_.xxx.yyy.$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0@$MQTT.msgs.foo\"\n\t// but `reply` has already been cleaned up (delivery subject removed from tail):\n\t// \"$JS.ACK.$MQTT_msgs.someid.1.1.1.1620086713306484000.0\"\n\t// So we will use that knowledge so we don't have to make any cleaning here.\n\troutedReply = routedReply[:gwSubjectOffset+len(reply)]\n\t// We need to make a copy so that we don't reference the underlying\n\t// read buffer.\n\tms := string(routedReply)\n\tgrm := &gwReplyMap{ms: ms, exp: time.Now().Add(ttl).UnixNano()}\n\t// If we are here with the same key but different mapped replies\n\t// (say $GNR._.A.srv1.bar and then $GNR._.B.srv2.bar), we need to\n\t// store it otherwise we would take the risk of the reply not\n\t// making it back.\n\tg.mapping[ms[gwSubjectOffset:]] = grm\n\tif wasEmpty {\n\t\tatomic.StoreInt32(&g.check, 1)\n\t\ts.gwrm.m.Store(g, l)\n\t\tif atomic.CompareAndSwapInt32(&s.gwrm.w, 0, 1) {\n\t\t\tselect {\n\t\t\tcase s.gwrm.ch <- ttl:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Starts a long lived go routine that is responsible to\n// remove GW reply mapping that have expired.\nfunc (s *Server) startGWReplyMapExpiration() {\n\ts.mu.Lock()\n\ts.gwrm.ch = make(chan time.Duration, 1)\n\ts.mu.Unlock()\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\n\t\tt := time.NewTimer(time.Hour)\n\t\tvar ttl time.Duration\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-t.C:\n\t\t\t\tif ttl == 0 {\n\t\t\t\t\tt.Reset(time.Hour)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tnow := time.Now().UnixNano()\n\t\t\t\tmapEmpty := true\n\t\t\t\ts.gwrm.m.Range(func(k, v any) bool {\n\t\t\t\t\tg := k.(*gwReplyMapping)\n\t\t\t\t\tl := v.(sync.Locker)\n\t\t\t\t\tl.Lock()\n\t\t\t\t\tfor k, grm := range g.mapping {\n\t\t\t\t\t\tif grm.exp <= now {\n\t\t\t\t\t\t\tdelete(g.mapping, k)\n\t\t\t\t\t\t\tif len(g.mapping) == 0 {\n\t\t\t\t\t\t\t\tatomic.StoreInt32(&g.check, 0)\n\t\t\t\t\t\t\t\ts.gwrm.m.Delete(g)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tl.Unlock()\n\t\t\t\t\tmapEmpty = false\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t\tif mapEmpty && atomic.CompareAndSwapInt32(&s.gwrm.w, 1, 0) {\n\t\t\t\t\tttl = 0\n\t\t\t\t\tt.Reset(time.Hour)\n\t\t\t\t} else {\n\t\t\t\t\tt.Reset(ttl)\n\t\t\t\t}\n\t\t\tcase cttl := <-s.gwrm.ch:\n\t\t\t\tttl = cttl\n\t\t\t\tif !t.Stop() {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-t.C:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tt.Reset(ttl)\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t})\n}\n" }, { "path": "server/gateway_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/logger\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\t\"golang.org/x/crypto/ocsp\"\n\n\t. \"github.com/nats-io/nats-server/v2/internal/ocsp\"\n)\n\nfunc init() {\n\tgatewayConnectDelay = 15 * time.Millisecond\n\tgatewayConnectMaxDelay = 15 * time.Millisecond\n\tgatewayReconnectDelay = 15 * time.Millisecond\n}\n\n// Wait for the expected number of outbound gateways, or fails.\nfunc waitForOutboundGateways(t *testing.T, s *Server, expected int, timeout time.Duration) {\n\tt.Helper()\n\tif timeout < 2*time.Second {\n\t\ttimeout = 2 * time.Second\n\t}\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tif n := s.numOutboundGateways(); n != expected {\n\t\t\treturn fmt.Errorf(\"Expected %v outbound gateway(s), got %v\", expected, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Wait for the expected number of inbound gateways, or fails.\nfunc waitForInboundGateways(t *testing.T, s *Server, expected int, timeout time.Duration) {\n\tt.Helper()\n\tif timeout < 2*time.Second {\n\t\ttimeout = 2 * time.Second\n\t}\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tif n := s.numInboundGateways(); n != expected {\n\t\t\treturn fmt.Errorf(\"Expected %v inbound gateway(s), got %v\", expected, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc waitForGatewayFailedConnect(t *testing.T, s *Server, gwName string, expectFailure bool, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tvar c int\n\t\tcfg := s.getRemoteGateway(gwName)\n\t\tif cfg != nil {\n\t\t\tc = cfg.getConnAttempts()\n\t\t}\n\t\tif expectFailure && c <= 1 {\n\t\t\treturn fmt.Errorf(\"Expected several attempts to connect, got %v\", c)\n\t\t} else if !expectFailure && c > 1 {\n\t\t\treturn fmt.Errorf(\"Expected single attempt to connect, got %v\", c)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkForRegisteredQSubInterest(t *testing.T, s *Server, gwName, acc, subj string, expected int, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tcount := 0\n\t\tc := s.getOutboundGatewayConnection(gwName)\n\t\tei, _ := c.gw.outsim.Load(acc)\n\t\tif ei != nil {\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tr := sl.Match(subj)\n\t\t\tfor _, qsubs := range r.qsubs {\n\t\t\t\tcount += len(qsubs)\n\t\t\t}\n\t\t}\n\t\tif count == expected {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Expected %v qsubs in sublist, got %v\", expected, count)\n\t})\n}\n\nfunc checkForSubjectNoInterest(t *testing.T, c *client, account, subject string, expectNoInterest bool, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tei, _ := c.gw.outsim.Load(account)\n\t\tif ei == nil {\n\t\t\treturn fmt.Errorf(\"Did not receive subject no-interest\")\n\t\t}\n\t\te := ei.(*outsie)\n\t\te.RLock()\n\t\t_, inMap := e.ni[subject]\n\t\te.RUnlock()\n\t\tif expectNoInterest {\n\t\t\tif inMap {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Did not receive subject no-interest on %q\", subject)\n\t\t}\n\t\tif inMap {\n\t\t\treturn fmt.Errorf(\"No-interest on subject %q was not cleared\", subject)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkForAccountNoInterest(t *testing.T, c *client, account string, expectedNoInterest bool, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tei, ok := c.gw.outsim.Load(account)\n\t\tif !ok && expectedNoInterest {\n\t\t\treturn fmt.Errorf(\"No-interest for account %q not yet registered\", account)\n\t\t} else if ok && !expectedNoInterest {\n\t\t\treturn fmt.Errorf(\"Account %q should not have a no-interest\", account)\n\t\t}\n\t\tif ei != nil {\n\t\t\treturn fmt.Errorf(\"Account %q should have a global no-interest, not subject no-interest\", account)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkGWInterestOnlyMode(t *testing.T, s *Server, outboundGWName, accName string) {\n\tt.Helper()\n\tcheckGWInterestOnlyModeOrNotPresent(t, s, outboundGWName, accName, false)\n}\n\nfunc checkGWInterestOnlyModeOrNotPresent(t *testing.T, s *Server, outboundGWName, accName string, notPresentOk bool) {\n\tt.Helper()\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tgwc := s.getOutboundGatewayConnection(outboundGWName)\n\t\tif gwc == nil {\n\t\t\treturn fmt.Errorf(\"No outbound gateway connection %q for server %v\", outboundGWName, s)\n\t\t}\n\t\tgwc.mu.Lock()\n\t\tdefer gwc.mu.Unlock()\n\t\tout, ok := gwc.gw.outsim.Load(accName)\n\t\tif !ok {\n\t\t\tif notPresentOk {\n\t\t\t\treturn nil\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"Server %v - outbound gateway connection %q: no account %q found in map\",\n\t\t\t\t\ts, outboundGWName, accName)\n\t\t\t}\n\t\t}\n\t\tif out == nil {\n\t\t\treturn fmt.Errorf(\"Server %v - outbound gateway connection %q: interest map not found for account %q\",\n\t\t\t\ts, outboundGWName, accName)\n\t\t}\n\t\te := out.(*outsie)\n\t\te.RLock()\n\t\tdefer e.RUnlock()\n\t\tif e.mode != InterestOnly {\n\t\t\treturn fmt.Errorf(\n\t\t\t\t\"Server %v - outbound gateway connection %q: account %q mode shoule be InterestOnly but is %v\",\n\t\t\t\ts, outboundGWName, accName, e.mode)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkGWInterestOnlyModeInterestOn(t *testing.T, s *Server, outboundGWName, accName, subject string) {\n\tt.Helper()\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tc := s.getOutboundGatewayConnection(outboundGWName)\n\t\toutsiei, _ := c.gw.outsim.Load(accName)\n\t\tif outsiei == nil {\n\t\t\treturn fmt.Errorf(\"Server %s - outbound gateway connection %q: no map entry found for account %q\",\n\t\t\t\ts, outboundGWName, accName)\n\t\t}\n\t\toutsie := outsiei.(*outsie)\n\t\tr := outsie.sl.Match(subject)\n\t\tif len(r.psubs) == 0 {\n\t\t\treturn fmt.Errorf(\"Server %s - outbound gateway connection %q - account %q: no subject interest for %q\",\n\t\t\t\ts, outboundGWName, accName, subject)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc waitCh(t *testing.T, ch chan bool, errTxt string) {\n\tt.Helper()\n\tselect {\n\tcase <-ch:\n\t\treturn\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(errTxt)\n\t}\n}\n\nvar noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {}\n\nfunc natsConnect(t testing.TB, url string, options ...nats.Option) *nats.Conn {\n\tt.Helper()\n\topts := nats.GetDefaultOptions()\n\tfor _, opt := range options {\n\t\tif err := opt(&opts); err != nil {\n\t\t\tt.Fatalf(\"Error applying client option: %v\", err)\n\t\t}\n\t}\n\tnc, err := nats.Connect(url, options...)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tif opts.AsyncErrorCB == nil {\n\t\t// Set this up to not pollute the logs when running tests.\n\t\tnc.SetErrorHandler(noOpErrHandler)\n\t}\n\n\treturn nc\n}\n\nfunc natsSub(t *testing.T, nc *nats.Conn, subj string, cb nats.MsgHandler) *nats.Subscription {\n\tt.Helper()\n\tsub, err := nc.Subscribe(subj, cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\treturn sub\n}\n\nfunc natsSubSync(t *testing.T, nc *nats.Conn, subj string) *nats.Subscription {\n\tt.Helper()\n\tsub, err := nc.SubscribeSync(subj)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\treturn sub\n}\n\nfunc natsNexMsg(t *testing.T, sub *nats.Subscription, timeout time.Duration) *nats.Msg {\n\tt.Helper()\n\tmsg, err := sub.NextMsg(timeout)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed getting next message: %v\", err)\n\t}\n\treturn msg\n}\n\nfunc natsQueueSub(t *testing.T, nc *nats.Conn, subj, queue string, cb nats.MsgHandler) *nats.Subscription {\n\tt.Helper()\n\tsub, err := nc.QueueSubscribe(subj, queue, cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\treturn sub\n}\n\nfunc natsQueueSubSync(t *testing.T, nc *nats.Conn, subj, queue string) *nats.Subscription {\n\tt.Helper()\n\tsub, err := nc.QueueSubscribeSync(subj, queue)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\treturn sub\n}\n\nfunc natsFlush(t *testing.T, nc *nats.Conn) {\n\tt.Helper()\n\tif err := nc.Flush(); err != nil {\n\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t}\n}\n\nfunc natsPub(t testing.TB, nc *nats.Conn, subj string, payload []byte) {\n\tt.Helper()\n\tif err := nc.Publish(subj, payload); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n}\n\nfunc natsPubReq(t *testing.T, nc *nats.Conn, subj, reply string, payload []byte) {\n\tt.Helper()\n\tif err := nc.PublishRequest(subj, reply, payload); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n}\n\nfunc natsUnsub(t *testing.T, sub *nats.Subscription) {\n\tt.Helper()\n\tif err := sub.Unsubscribe(); err != nil {\n\t\tt.Fatalf(\"Error on unsubscribe: %v\", err)\n\t}\n}\n\nfunc testDefaultOptionsForGateway(name string) *Options {\n\to := DefaultOptions()\n\to.NoSystemAccount = true\n\to.Cluster.Name = name\n\to.Gateway.Name = name\n\to.Gateway.Host = \"127.0.0.1\"\n\to.Gateway.Port = -1\n\to.gatewaysSolicitDelay = 15 * time.Millisecond\n\treturn o\n}\n\nfunc runGatewayServer(o *Options) *Server {\n\ts := RunServer(o)\n\ts.SetLogger(&DummyLogger{}, true, true)\n\treturn s\n}\n\nfunc testGatewayOptionsFromToWithServers(t *testing.T, org, dst string, servers ...*Server) *Options {\n\tt.Helper()\n\to := testDefaultOptionsForGateway(org)\n\tgw := &RemoteGatewayOpts{Name: dst}\n\tfor _, s := range servers {\n\t\tus := fmt.Sprintf(\"nats://127.0.0.1:%d\", s.GatewayAddr().Port)\n\t\tu, err := url.Parse(us)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t\t}\n\t\tgw.URLs = append(gw.URLs, u)\n\t}\n\to.Gateway.Gateways = append(o.Gateway.Gateways, gw)\n\treturn o\n}\n\nfunc testAddGatewayURLs(t *testing.T, o *Options, dst string, urls []string) {\n\tt.Helper()\n\tgw := &RemoteGatewayOpts{Name: dst}\n\tfor _, us := range urls {\n\t\tu, err := url.Parse(us)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t\t}\n\t\tgw.URLs = append(gw.URLs, u)\n\t}\n\to.Gateway.Gateways = append(o.Gateway.Gateways, gw)\n}\n\nfunc testGatewayOptionsFromToWithURLs(t *testing.T, org, dst string, urls []string) *Options {\n\to := testDefaultOptionsForGateway(org)\n\ttestAddGatewayURLs(t, o, dst, urls)\n\treturn o\n}\n\nfunc testGatewayOptionsWithTLS(t *testing.T, name string) *Options {\n\tt.Helper()\n\to := testDefaultOptionsForGateway(name)\n\tvar (\n\t\ttc = &TLSConfigOpts{}\n\t\terr error\n\t)\n\tif name == \"A\" {\n\t\ttc.CertFile = \"../test/configs/certs/srva-cert.pem\"\n\t\ttc.KeyFile = \"../test/configs/certs/srva-key.pem\"\n\t} else {\n\t\ttc.CertFile = \"../test/configs/certs/srvb-cert.pem\"\n\t\ttc.KeyFile = \"../test/configs/certs/srvb-key.pem\"\n\t}\n\ttc.CaFile = \"../test/configs/certs/ca.pem\"\n\to.Gateway.TLSConfig, err = GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t}\n\to.Gateway.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert\n\to.Gateway.TLSConfig.RootCAs = o.Gateway.TLSConfig.ClientCAs\n\to.Gateway.TLSTimeout = 2.0\n\treturn o\n}\n\nfunc testGatewayOptionsFromToWithTLS(t *testing.T, org, dst string, urls []string) *Options {\n\to := testGatewayOptionsWithTLS(t, org)\n\ttestAddGatewayURLs(t, o, dst, urls)\n\treturn o\n}\n\nfunc TestGatewayBasic(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.Gateway.ConnectRetries = 0\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\t// and an outbound too\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t// Stop s2 server\n\ts2.Shutdown()\n\n\t// gateway should go away\n\twaitForOutboundGateways(t, s1, 0, time.Second)\n\twaitForInboundGateways(t, s1, 0, time.Second)\n\n\t// Restart server\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// gateway should reconnect\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\t// Shutdown s1, remove the gateway from A to B and restart.\n\ts1.Shutdown()\n\t// When s2 detects the connection is closed, it will attempt\n\t// to reconnect once (even if the route is implicit).\n\t// We need to wait more than a dial timeout to make sure\n\t// s1 does not restart too quickly and s2 can actually reconnect.\n\ttime.Sleep(DEFAULT_ROUTE_DIAL + 250*time.Millisecond)\n\t// Restart s1 without gateway to B.\n\to1.Gateway.Gateways = nil\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should not have any outbound nor inbound\n\twaitForOutboundGateways(t, s1, 0, 2*time.Second)\n\twaitForInboundGateways(t, s1, 0, 2*time.Second)\n\n\t// Same for s2\n\twaitForOutboundGateways(t, s2, 0, 2*time.Second)\n\twaitForInboundGateways(t, s2, 0, 2*time.Second)\n\n\t// Verify that s2 no longer has A gateway in its list\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif s2.getRemoteGateway(\"A\") != nil {\n\t\t\treturn fmt.Errorf(\"Gateway A should have been removed from s2\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestGatewayIgnoreSelfReference(t *testing.T) {\n\to := testDefaultOptionsForGateway(\"A\")\n\t// To create a reference to itself before running the server\n\t// it means that we have to assign an explicit port\n\to.Gateway.Port = 5222\n\to.gatewaysSolicitDelay = 0\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", o.Gateway.Host, o.Gateway.Port))\n\tcfg := &RemoteGatewayOpts{\n\t\tName: \"A\",\n\t\tURLs: []*url.URL{u},\n\t}\n\to.Gateway.Gateways = append(o.Gateway.Gateways, cfg)\n\to.NoSystemAccount = true\n\ts := runGatewayServer(o)\n\tdefer s.Shutdown()\n\n\t// Wait a bit to make sure that there is no attempt to connect.\n\ttime.Sleep(20 * time.Millisecond)\n\n\t// No outbound connection expected, and no attempt to connect.\n\tif s.getRemoteGateway(\"A\") != nil {\n\t\tt.Fatalf(\"Should not have a remote gateway config for A\")\n\t}\n\tif s.getOutboundGatewayConnection(\"A\") != nil {\n\t\tt.Fatalf(\"Should not have a gateway connection to A\")\n\t}\n\ts.Shutdown()\n\n\t// Now try with config files and include\n\ts1, _ := RunServerWithConfig(\"configs/gwa.conf\")\n\tdefer s1.Shutdown()\n\n\ts2, _ := RunServerWithConfig(\"configs/gwb.conf\")\n\tdefer s2.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\tif s1.getRemoteGateway(\"A\") != nil {\n\t\tt.Fatalf(\"Should not have a remote gateway config for A\")\n\t}\n\tif s1.getOutboundGatewayConnection(\"A\") != nil {\n\t\tt.Fatalf(\"Should not have a gateway connection to A\")\n\t}\n\tif s2.getRemoteGateway(\"B\") != nil {\n\t\tt.Fatalf(\"Should not have a remote gateway config for B\")\n\t}\n\tif s2.getOutboundGatewayConnection(\"B\") != nil {\n\t\tt.Fatalf(\"Should not have a gateway connection to B\")\n\t}\n}\n\nfunc TestGatewayHeaderInfo(t *testing.T) {\n\to := testDefaultOptionsForGateway(\"A\")\n\ts := runGatewayServer(o)\n\tdefer s.Shutdown()\n\n\tgwconn, err := net.Dial(\"tcp\", net.JoinHostPort(o.Gateway.Host, fmt.Sprintf(\"%d\", o.Gateway.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error dialing server: %v\\n\", err)\n\t}\n\tdefer gwconn.Close()\n\tclient := bufio.NewReaderSize(gwconn, maxBufSize)\n\tl, err := client.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\tvar info serverInfo\n\tif err = json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif !info.Headers {\n\t\tt.Fatalf(\"Expected by default for header support to be enabled\")\n\t}\n\n\ts.Shutdown()\n\tgwconn.Close()\n\n\t// Now turn headers off.\n\to.NoHeaderSupport = true\n\ts = runGatewayServer(o)\n\tdefer s.Shutdown()\n\n\tgwconn, err = net.Dial(\"tcp\", net.JoinHostPort(o.Gateway.Host, fmt.Sprintf(\"%d\", o.Gateway.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error dialing server: %v\\n\", err)\n\t}\n\tdefer gwconn.Close()\n\tclient = bufio.NewReaderSize(gwconn, maxBufSize)\n\tl, err = client.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t}\n\tif err = json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Headers {\n\t\tt.Fatalf(\"Expected header support to be disabled\")\n\t}\n}\n\nfunc TestGatewayHeaderSupport(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.Gateway.ConnectRetries = 0\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// and an inbound too\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\t// and an outbound too\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tc, cr, _ := newClientForServer(s1)\n\tdefer c.close()\n\n\tconnect := \"CONNECT {\\\"headers\\\":true}\"\n\tsubOp := \"SUB foo 1\"\n\tpingOp := \"PING\\r\\n\"\n\tcmd := strings.Join([]string{connect, subOp, pingOp}, \"\\r\\n\")\n\tc.parseAsync(cmd)\n\tif _, err := cr.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\t// Wait for interest to be registered on s2\n\tcheckGWInterestOnlyModeInterestOn(t, s2, \"A\", globalAccountName, \"foo\")\n\n\tb, _, _ := newClientForServer(s2)\n\tdefer b.close()\n\n\tpubOp := \"HPUB foo 12 14\\r\\nName:Derek\\r\\nOK\\r\\n\"\n\tcmd = strings.Join([]string{connect, pubOp}, \"\\r\\n\")\n\tb.parseAsync(cmd)\n\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\tam := hmsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a match for %q\", l)\n\t}\n\tmatches := am[0]\n\tif len(matches) != 7 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 7)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[HDR_INDEX] != \"12\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[HDR_INDEX])\n\t}\n\tif matches[TLEN_INDEX] != \"14\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[TLEN_INDEX])\n\t}\n\tcheckPayload(cr, []byte(\"Name:Derek\\r\\nOK\\r\\n\"), t)\n}\n\nfunc TestGatewayHeaderDeliverStrippedMsg(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.Gateway.ConnectRetries = 0\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\to1.NoHeaderSupport = true\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// and an inbound too\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\t// and an outbound too\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tc, cr, _ := newClientForServer(s1)\n\tdefer c.close()\n\n\tconnect := \"CONNECT {\\\"headers\\\":true}\"\n\tsubOp := \"SUB foo 1\"\n\tpingOp := \"PING\\r\\n\"\n\tcmd := strings.Join([]string{connect, subOp, pingOp}, \"\\r\\n\")\n\tc.parseAsync(cmd)\n\tif _, err := cr.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\n\t// Wait for interest to be registered on s2\n\tcheckGWInterestOnlyModeInterestOn(t, s2, \"A\", globalAccountName, \"foo\")\n\n\tb, _, _ := newClientForServer(s2)\n\tdefer b.close()\n\n\tpubOp := \"HPUB foo 12 14\\r\\nName:Derek\\r\\nOK\\r\\n\"\n\tcmd = strings.Join([]string{connect, pubOp}, \"\\r\\n\")\n\tb.parseAsync(cmd)\n\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg from server: %v\\n\", err)\n\t}\n\tam := smsgPat.FindAllStringSubmatch(l, -1)\n\tif len(am) == 0 {\n\t\tt.Fatalf(\"Did not get a correct match for %q\", l)\n\t}\n\tmatches := am[0]\n\tif len(matches) != 6 {\n\t\tt.Fatalf(\"Did not get correct # matches: %d vs %d\\n\", len(matches), 6)\n\t}\n\tif matches[SUB_INDEX] != \"foo\" {\n\t\tt.Fatalf(\"Did not get correct subject: '%s'\\n\", matches[SUB_INDEX])\n\t}\n\tif matches[SID_INDEX] != \"1\" {\n\t\tt.Fatalf(\"Did not get correct sid: '%s'\\n\", matches[SID_INDEX])\n\t}\n\tif matches[LEN_INDEX] != \"2\" {\n\t\tt.Fatalf(\"Did not get correct msg length: '%s'\\n\", matches[LEN_INDEX])\n\t}\n\tcheckPayload(cr, []byte(\"OK\\r\\n\"), t)\n\tif cr.Buffered() != 0 {\n\t\tt.Fatalf(\"Expected no extra bytes to be buffered, got %d\", cr.Buffered())\n\t}\n}\n\nfunc TestGatewaySolicitDelay(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\t// Set the solicit delay to 0. This tests that server will use its\n\t// default value, currently set at 1 sec.\n\to1.gatewaysSolicitDelay = 0\n\tstart := time.Now()\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// After 500ms, check outbound gateway. Should not be there.\n\ttime.Sleep(500 * time.Millisecond)\n\tif time.Since(start) < defaultSolicitGatewaysDelay {\n\t\tif s1.numOutboundGateways() > 0 {\n\t\t\tt.Fatalf(\"The outbound gateway was initiated sooner than expected (%v)\", time.Since(start))\n\t\t}\n\t}\n\t// Ultimately, s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\ts1.Shutdown()\n\t// Make sure that server can be shutdown while waiting\n\t// for that initial solicit delay\n\to1.gatewaysSolicitDelay = 2 * time.Second\n\ts1 = runGatewayServer(o1)\n\tstart = time.Now()\n\ts1.Shutdown()\n\tif dur := time.Since(start); dur >= 2*time.Second {\n\t\tt.Fatalf(\"Looks like shutdown was delayed: %v\", dur)\n\t}\n}\n\nfunc TestGatewaySolicitDelayWithImplicitOutbounds(t *testing.T) {\n\t// Cause a situation where A connects to B, and because of\n\t// delay of solicit gateways set on B, we want to make sure\n\t// that B does not end-up with 2 connections to A.\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.gatewaysSolicitDelay = 500 * time.Millisecond\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound and inbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\t// Wait for more than s2 solicit delay\n\ttime.Sleep(750 * time.Millisecond)\n\t// The way we store outbound (map key'ed by gw name), we would\n\t// not know if we had created 2 (since the newer would replace\n\t// the older in the map). But if a second connection was made,\n\t// then s1 would have 2 inbounds. So check it has only 1.\n\twaitForInboundGateways(t, s1, 1, time.Second)\n}\n\ntype slowResolver struct {\n\tinLookupCh chan struct{}\n\treleaseCh chan struct{}\n}\n\nfunc (r *slowResolver) LookupHost(ctx context.Context, h string) ([]string, error) {\n\tif r.inLookupCh != nil {\n\t\tselect {\n\t\tcase r.inLookupCh <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t\t<-r.releaseCh\n\t} else {\n\t\ttime.Sleep(500 * time.Millisecond)\n\t}\n\treturn []string{h}, nil\n}\n\nfunc TestGatewaySolicitShutdown(t *testing.T) {\n\tvar urls []string\n\tfor i := 0; i < 5; i++ {\n\t\tu := fmt.Sprintf(\"nats://localhost:%d\", 1234+i)\n\t\turls = append(urls, u)\n\t}\n\to1 := testGatewayOptionsFromToWithURLs(t, \"A\", \"B\", urls)\n\to1.Gateway.resolver = &slowResolver{}\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\ttime.Sleep(o1.gatewaysSolicitDelay + 10*time.Millisecond)\n\n\tstart := time.Now()\n\ts1.Shutdown()\n\tif dur := time.Since(start); dur > 1200*time.Millisecond {\n\t\tt.Fatalf(\"Took too long to shutdown: %v\", dur)\n\t}\n}\n\nfunc testFatalErrorOnStart(t *testing.T, o *Options, errTxt string) {\n\tt.Helper()\n\ts := New(o)\n\tdefer s.Shutdown()\n\tl := &captureFatalLogger{fatalCh: make(chan string, 1)}\n\ts.SetLogger(l, false, false)\n\t// This does not block\n\ts.Start()\n\tselect {\n\tcase e := <-l.fatalCh:\n\t\tif !strings.Contains(e, errTxt) {\n\t\t\tt.Fatalf(\"Error should contain %q, got %s\", errTxt, e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Should have got a fatal error\")\n\t}\n\ts.Shutdown()\n\ts.WaitForShutdown()\n}\n\nfunc TestGatewayListenError(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testDefaultOptionsForGateway(\"A\")\n\to1.Gateway.Port = s2.GatewayAddr().Port\n\ttestFatalErrorOnStart(t, o1, \"listening on\")\n}\n\nfunc TestGatewayWithListenToAny(t *testing.T) {\n\tconfB1 := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t}\n\t`))\n\tsb1, ob1 := RunServerWithConfig(confB1)\n\tdefer sb1.Shutdown()\n\n\tconfB2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\troutes: [\"%s\"]\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t}\n\t`, fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))))\n\tsb2, ob2 := RunServerWithConfig(confB2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"B\"\n\t\t\t\t\turls: [\"%s\", \"%s\"]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Gateway.Port), fmt.Sprintf(\"nats://127.0.0.1:%d\", ob2.Gateway.Port))))\n\toa := LoadConfig(confA)\n\toa.gatewaysSolicitDelay = 15 * time.Millisecond\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\twaitForInboundGateways(t, sa, 2, time.Second)\n\n\tcheckAll := func(t *testing.T) {\n\t\tt.Helper()\n\t\tcheckURL := func(t *testing.T, s *Server) {\n\t\t\tt.Helper()\n\t\t\turl := s.getGatewayURL()\n\t\t\tif strings.HasPrefix(url, \"0.0.0.0\") {\n\t\t\t\tt.Fatalf(\"URL still references 0.0.0.0\")\n\t\t\t}\n\t\t\ts.gateway.RLock()\n\t\t\tfor url := range s.gateway.URLs {\n\t\t\t\tif strings.HasPrefix(url, \"0.0.0.0\") {\n\t\t\t\t\ts.gateway.RUnlock()\n\t\t\t\t\tt.Fatalf(\"URL still references 0.0.0.0\")\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.gateway.RUnlock()\n\n\t\t\tvar cfg *gatewayCfg\n\t\t\tif s.getGatewayName() == \"A\" {\n\t\t\t\tcfg = s.getRemoteGateway(\"B\")\n\t\t\t} else {\n\t\t\t\tcfg = s.getRemoteGateway(\"A\")\n\t\t\t}\n\t\t\turls := cfg.getURLs()\n\t\t\tfor _, url := range urls {\n\t\t\t\tif strings.HasPrefix(url.Host, \"0.0.0.0\") {\n\t\t\t\t\tt.Fatalf(\"URL still references 0.0.0.0\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcheckURL(t, sb1)\n\t\tcheckURL(t, sb2)\n\t\tcheckURL(t, sa)\n\t}\n\tcheckAll(t)\n\t// Perform a reload and ensure that nothing has changed\n\tservers := []*Server{sb1, sb2, sa}\n\tfor _, s := range servers {\n\t\tif err := s.Reload(); err != nil {\n\t\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t\t}\n\t\tcheckAll(t)\n\t}\n}\n\nfunc TestGatewayAdvertise(t *testing.T) {\n\to3 := testDefaultOptionsForGateway(\"C\")\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\t// Set the advertise so that this points to C\n\to1.Gateway.Advertise = fmt.Sprintf(\"127.0.0.1:%d\", s3.GatewayAddr().Port)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// We should have outbound from s1 to s2\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// But no inbound from s2\n\twaitForInboundGateways(t, s1, 0, time.Second)\n\n\t// And since B tries to connect to A but reaches C, it should fail to connect,\n\t// and without connect retries, stop trying. So no outbound for s2, and no\n\t// inbound/outbound for s3.\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 0, time.Second)\n\twaitForInboundGateways(t, s3, 0, time.Second)\n\twaitForOutboundGateways(t, s3, 0, time.Second)\n}\n\nfunc TestGatewayAdvertiseErr(t *testing.T) {\n\to1 := testDefaultOptionsForGateway(\"A\")\n\to1.Gateway.Advertise = \"wrong:address\"\n\ttestFatalErrorOnStart(t, o1, \"Gateway.Advertise\")\n}\n\nfunc TestGatewayAuth(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.Gateway.Username = \"me\"\n\to2.Gateway.Password = \"pwd\"\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithURLs(t, \"A\", \"B\", []string{fmt.Sprintf(\"nats://me:pwd@127.0.0.1:%d\", s2.GatewayAddr().Port)})\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\n\ts2.Shutdown()\n\ts1.Shutdown()\n\n\to2.Gateway.Username = \"me\"\n\to2.Gateway.Password = \"wrong\"\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Connection should fail...\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, 2*time.Second)\n\n\ts2.Shutdown()\n\ts1.Shutdown()\n\to2.Gateway.Username = \"wrong\"\n\to2.Gateway.Password = \"pwd\"\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Connection should fail...\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, 2*time.Second)\n}\n\nfunc TestGatewayTLS(t *testing.T) {\n\to2 := testGatewayOptionsWithTLS(t, \"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithTLS(t, \"A\", \"B\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.GatewayAddr().Port)})\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\t// and vice-versa\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\n\t// Stop s2 server\n\ts2.Shutdown()\n\n\t// gateway should go away\n\twaitForOutboundGateways(t, s1, 0, time.Second)\n\twaitForInboundGateways(t, s1, 0, time.Second)\n\twaitForOutboundGateways(t, s2, 0, time.Second)\n\twaitForInboundGateways(t, s2, 0, time.Second)\n\n\t// Restart server\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// gateway should reconnect\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\ts1.Shutdown()\n\t// Wait for s2 to lose connections to s1.\n\twaitForOutboundGateways(t, s2, 0, 2*time.Second)\n\twaitForInboundGateways(t, s2, 0, 2*time.Second)\n\n\t// Make an explicit TLS config for remote gateway config \"B\"\n\t// on cluster A.\n\to1.Gateway.Gateways[0].TLSConfig = o1.Gateway.TLSConfig.Clone()\n\tu, _ := url.Parse(fmt.Sprintf(\"tls://localhost:%d\", s2.GatewayAddr().Port))\n\to1.Gateway.Gateways[0].URLs = []*url.URL{u}\n\t// Make the TLSTimeout so small that it should fail to connect.\n\tsmallTimeout := 0.00000001\n\to1.Gateway.Gateways[0].TLSTimeout = smallTimeout\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Check that s1 reports connection failures\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, 2*time.Second)\n\n\t// Check that TLSConfig from s1's remote \"B\" is based on\n\t// what we have configured.\n\tcfg := s1.getRemoteGateway(\"B\")\n\tcfg.RLock()\n\ttlsName := cfg.tlsName\n\ttimeout := cfg.TLSTimeout\n\tcfg.RUnlock()\n\tif tlsName != \"localhost\" {\n\t\tt.Fatalf(\"Expected server name to be localhost, got %v\", tlsName)\n\t}\n\tif timeout != smallTimeout {\n\t\tt.Fatalf(\"Expected tls timeout to be %v, got %v\", smallTimeout, timeout)\n\t}\n\ts1.Shutdown()\n\t// Wait for s2 to lose connections to s1.\n\twaitForOutboundGateways(t, s2, 0, 2*time.Second)\n\twaitForInboundGateways(t, s2, 0, 2*time.Second)\n\n\t// Remove explicit TLSTimeout from gateway \"B\" and check that\n\t// we use the A's spec one.\n\to1.Gateway.Gateways[0].TLSTimeout = 0\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\tcfg = s1.getRemoteGateway(\"B\")\n\tcfg.RLock()\n\ttimeout = cfg.TLSTimeout\n\tcfg.RUnlock()\n\tif timeout != o1.Gateway.TLSTimeout {\n\t\tt.Fatalf(\"Expected tls timeout to be %v, got %v\", o1.Gateway.TLSTimeout, timeout)\n\t}\n}\n\nfunc TestGatewayTLSErrors(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithTLS(t, \"A\", \"B\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port)})\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Expect s1 to have a failed to connect count > 0\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, 2*time.Second)\n}\n\nfunc TestGatewayServerNameInTLSConfig(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\tvar (\n\t\ttc = &TLSConfigOpts{}\n\t\terr error\n\t)\n\ttc.CertFile = \"../test/configs/certs/server-noip.pem\"\n\ttc.KeyFile = \"../test/configs/certs/server-key-noip.pem\"\n\ttc.CaFile = \"../test/configs/certs/ca.pem\"\n\to2.Gateway.TLSConfig, err = GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t}\n\to2.Gateway.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert\n\to2.Gateway.TLSConfig.RootCAs = o2.Gateway.TLSConfig.ClientCAs\n\to2.Gateway.TLSTimeout = 2.0\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithTLS(t, \"A\", \"B\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.GatewayAddr().Port)})\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should fail to connect since we don't have proper expected hostname.\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, 2*time.Second)\n\n\t// Now set server name, and it should work.\n\ts1.Shutdown()\n\to1.Gateway.TLSConfig.ServerName = \"localhost\"\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n}\n\nfunc TestGatewayWrongDestination(t *testing.T) {\n\t// Start a server with a gateway named \"C\"\n\to2 := testDefaultOptionsForGateway(\"C\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// Configure a gateway to \"B\", but since we are connecting to \"C\"...\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// we should not be able to connect.\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, time.Second)\n\n\t// Shutdown s2 and fix the gateway name.\n\t// s1 should then connect ok and failed connect should be cleared.\n\ts2.Shutdown()\n\n\t// Reset the conn attempts\n\tcfg := s1.getRemoteGateway(\"B\")\n\tcfg.resetConnAttempts()\n\n\to2.Gateway.Name = \"B\"\n\to2.Cluster.Name = \"B\"\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// At some point, the number of failed connect count should be reset to 0.\n\twaitForGatewayFailedConnect(t, s1, \"B\", false, 2*time.Second)\n}\n\nfunc TestGatewayConnectToWrongPort(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// Configure a gateway to \"B\", but connect to the wrong port\n\turls := []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.Addr().(*net.TCPAddr).Port)}\n\to1 := testGatewayOptionsFromToWithURLs(t, \"A\", \"B\", urls)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// we should not be able to connect.\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, time.Second)\n\n\ts1.Shutdown()\n\n\t// Repeat with route port\n\turls = []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port)}\n\to1 = testGatewayOptionsFromToWithURLs(t, \"A\", \"B\", urls)\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// we should not be able to connect.\n\twaitForGatewayFailedConnect(t, s1, \"B\", true, time.Second)\n\n\ts1.Shutdown()\n\n\t// Now have a client connect to s2's gateway port.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.GatewayAddr().Port))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n}\n\nfunc TestGatewayCreateImplicit(t *testing.T) {\n\t// Create a regular cluster of 2 servers\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to3 := testDefaultOptionsForGateway(\"B\")\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port))\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s2, s3)\n\n\t// Now start s1 that creates a Gateway connection to s2 or s3\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2, s3)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// We should have an outbound gateway connection on ALL servers.\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 1, 2*time.Second)\n\n\t// Server s1 must have 2 inbound ones\n\twaitForInboundGateways(t, s1, 2, 2*time.Second)\n\n\t// However, s1 may have created the outbound to s2 or s3. It is possible that\n\t// either s2 or s3 does not an inbound connection.\n\ts2Inbound := s2.numInboundGateways()\n\ts3Inbound := s3.numInboundGateways()\n\tif (s2Inbound == 1 && s3Inbound != 0) || (s3Inbound == 1 && s2Inbound != 0) {\n\t\tt.Fatalf(\"Unexpected inbound for s2/s3: %v/%v\", s2Inbound, s3Inbound)\n\t}\n}\n\nfunc TestGatewayCreateImplicitOnNewRoute(t *testing.T) {\n\t// Start with only 2 clusters of 1 server each\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// Now start s1 that creates a Gateway connection to s2\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Check outbounds\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\n\t// Now add a server to cluster B\n\to3 := testDefaultOptionsForGateway(\"B\")\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port))\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\t// Wait for cluster between s2/s3 to form\n\tcheckClusterFormed(t, s2, s3)\n\n\t// s3 should have been notified about existence of A and create its gateway to A.\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 1, 2*time.Second)\n}\n\nfunc TestGatewayImplicitReconnect(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.Gateway.ConnectRetries = 5\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have an outbound gateway to s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t// s2 should have an inbound gateway\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\t// It will have also created an implicit outbound connection to s1.\n\t// We need to wait for that implicit outbound connection to be made\n\t// to show that it will try to reconnect when we stop/restart s1\n\t// (without config to connect to B).\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t// Shutdown s1, remove the gateway from A to B and restart.\n\ts1.Shutdown()\n\to1.Gateway.Gateways = o1.Gateway.Gateways[:0]\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// s1 should have both outbound and inbound to s2\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\n\t// Same for s2\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\n\t// Verify that s2 still has \"A\" in its gateway config\n\tif s2.getRemoteGateway(\"A\") == nil {\n\t\tt.Fatal(\"Gateway A should be in s2\")\n\t}\n}\n\nfunc TestGatewayImplicitReconnectRace(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tresolver := &slowResolver{\n\t\tinLookupCh: make(chan struct{}, 1),\n\t\treleaseCh: make(chan struct{}),\n\t}\n\tob.Gateway.resolver = resolver\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\t// Wait for the proper connections\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// On sb, change the URL to sa1 so that it is a name, instead of an IP,\n\t// so that we hit the slow resolver.\n\tcfg := sb.getRemoteGateway(\"A\")\n\tcfg.updateURLs([]string{fmt.Sprintf(\"localhost:%d\", sa1.GatewayAddr().Port)})\n\n\t// Shutdown sa1 now...\n\tsa1.Shutdown()\n\n\t// Wait to be notified that B has detected the connection close\n\t// and it is trying to resolve the host during the reconnect.\n\t<-resolver.inLookupCh\n\n\t// Start a new \"A\" server (sa2).\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\t// Make sure we have our outbound to sb registered on sa2 and inbound\n\t// from sa2 on sb before releasing the resolver.\n\twaitForOutboundGateways(t, sa2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\t// Now release the resolver and ensure we have all connections.\n\tclose(resolver.releaseCh)\n\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa2, 1, 2*time.Second)\n}\n\ntype gwReconnAttemptLogger struct {\n\tDummyLogger\n\terrCh chan string\n}\n\nfunc (l *gwReconnAttemptLogger) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, `Error connecting to implicit gateway \"A\"`) {\n\t\tselect {\n\t\tcase l.errCh <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestGatewayImplicitReconnectHonorConnectRetries(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tob.ReconnectErrorReports = 1\n\tob.Gateway.ConnectRetries = 2\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tl := &gwReconnAttemptLogger{errCh: make(chan string, 3)}\n\tsb.SetLogger(l, true, false)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\t// Wait for the proper connections\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Now have C connect to B.\n\toc := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", sb)\n\tsc := runGatewayServer(oc)\n\tdefer sc.Shutdown()\n\n\t// Wait for the proper connections\n\twaitForOutboundGateways(t, sa, 2, time.Second)\n\twaitForOutboundGateways(t, sb, 2, time.Second)\n\twaitForOutboundGateways(t, sc, 2, time.Second)\n\twaitForInboundGateways(t, sa, 2, time.Second)\n\twaitForInboundGateways(t, sb, 2, time.Second)\n\twaitForInboundGateways(t, sc, 2, time.Second)\n\n\t// Shutdown sa now...\n\tsa.Shutdown()\n\n\t// B will try to reconnect to A 3 times (we stop after attempts > ConnectRetries)\n\ttimeout := time.NewTimer(time.Second)\n\tfor i := 0; i < 3; i++ {\n\t\tselect {\n\t\tcase <-l.errCh:\n\t\t\t// OK\n\t\tcase <-timeout.C:\n\t\t\tt.Fatal(\"Did not get debug trace about reconnect\")\n\t\t}\n\t}\n\t// If we get 1 more, we have an issue!\n\tselect {\n\tcase e := <-l.errCh:\n\t\tt.Fatalf(\"Should not have attempted to reconnect: %q\", e)\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// OK!\n\t}\n\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sc, 1, 2*time.Second)\n\twaitForInboundGateways(t, sc, 1, 2*time.Second)\n}\n\nfunc TestGatewayReconnectExponentialBackoff(t *testing.T) {\n\toGatewayConnectDelay := gatewayConnectDelay\n\toGatewayConnectMaxDelay := gatewayConnectMaxDelay\n\tgatewayConnectDelay = 500 * time.Millisecond\n\tgatewayConnectMaxDelay = 2 * time.Second\n\tdefer func() {\n\t\tgatewayConnectDelay = oGatewayConnectDelay\n\t\tgatewayConnectMaxDelay = oGatewayConnectMaxDelay\n\t}()\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tob.ReconnectErrorReports = 1\n\tob.Gateway.ConnectRetries = 3\n\tob.Gateway.ConnectBackoff = true\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tl := &gwReconnAttemptLogger{errCh: make(chan string, 3)}\n\tsb.SetLogger(l, true, false)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\t// Wait for the proper connections\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Remove initial delay before reconnect, and allow for some skew.\n\tnow := time.Now().Add(gatewayReconnectDelay).Add(-100 * time.Millisecond)\n\tvar delay time.Duration\n\n\t// B will try to reconnect to A 3 times (we stop after attempts > ConnectRetries)\n\tsa.Shutdown()\n\tfor i := 0; i < ob.Gateway.ConnectRetries+1; i++ {\n\t\tselect {\n\t\tcase <-l.errCh:\n\t\t\tif since := time.Since(now); since < delay {\n\t\t\t\tt.Fatalf(\"Expected delay to take %v, took %v\", delay, since)\n\t\t\t}\n\t\t\tif delay == 0 {\n\t\t\t\tdelay = gatewayConnectDelay\n\t\t\t} else {\n\t\t\t\tdelay *= 2\n\t\t\t}\n\t\t\tif delay > gatewayConnectMaxDelay {\n\t\t\t\tdelay = gatewayConnectMaxDelay\n\t\t\t}\n\t\tcase <-time.After(gatewayConnectMaxDelay + time.Second):\n\t\t\tt.Fatal(\"Did not attempt to reconnect\")\n\t\t}\n\t}\n}\n\nfunc TestGatewayURLsFromClusterSentInINFO(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to3 := testDefaultOptionsForGateway(\"B\")\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port))\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s2, s3)\n\n\t// Now start s1 that creates a Gateway connection to s2\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Make sure we have proper outbound/inbound\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForOutboundGateways(t, s3, 1, time.Second)\n\n\t// Although s1 connected to s2 and knew only about s2, it should have\n\t// received the list of gateway URLs in the B cluster. So if we shutdown\n\t// server s2, it should be able to reconnect to s3.\n\ts2.Shutdown()\n\t// Wait for s3 to register that there s2 is gone.\n\tcheckNumRoutes(t, s3, 0)\n\t// s1 should have reconnected to s3 because it learned about it\n\t// when connecting earlier to s2.\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\t// Also make sure that the gateway's urls map has 2 urls.\n\tgw := s1.getRemoteGateway(\"B\")\n\tif gw == nil {\n\t\tt.Fatal(\"Did not find gateway B\")\n\t}\n\tgw.RLock()\n\tl := len(gw.urls)\n\tgw.RUnlock()\n\tif l != 2 {\n\t\tt.Fatalf(\"S1 should have 2 urls, got %v\", l)\n\t}\n}\n\nfunc TestGatewayUseUpdatedURLs(t *testing.T) {\n\t// For this test, we have cluster B with an explicit gateway to cluster A\n\t// on a given URL. Then we create cluster A with a gateway to B with server B's\n\t// GW url, and we expect server B to ultimately create an outbound GW connection\n\t// to server A (with the URL it will get from server A connecting to it).\n\n\tob := testGatewayOptionsFromToWithURLs(t, \"B\", \"A\", []string{\"nats://127.0.0.1:1234\"})\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\t// Add a delay before starting server A to make sure that server B start\n\t// initiating the connection to A on inexistant server at :1234.\n\ttime.Sleep(100 * time.Millisecond)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\t// sa should have no problem creating outbound connection to sb\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\n\t// Make sure that since sb learns about sa's GW URL, it can successfully\n\t// connect to it.\n\twaitForOutboundGateways(t, sb, 1, 3*time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n}\n\nfunc TestGatewayAutoDiscovery(t *testing.T) {\n\to4 := testDefaultOptionsForGateway(\"D\")\n\ts4 := runGatewayServer(o4)\n\tdefer s4.Shutdown()\n\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"D\", s4)\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\to2 := testGatewayOptionsFromToWithServers(t, \"B\", \"C\", s3)\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Each server should have 3 outbound gateway connections.\n\twaitForOutboundGateways(t, s1, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s4, 3, 2*time.Second)\n\n\ts1.Shutdown()\n\ts2.Shutdown()\n\ts3.Shutdown()\n\ts4.Shutdown()\n\n\to2 = testDefaultOptionsForGateway(\"B\")\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to4 = testGatewayOptionsFromToWithServers(t, \"D\", \"B\", s2)\n\ts4 = runGatewayServer(o4)\n\tdefer s4.Shutdown()\n\n\to3 = testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s2)\n\ts3 = runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\to1 = testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Each server should have 3 outbound gateway connections.\n\twaitForOutboundGateways(t, s1, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 3, 2*time.Second)\n\twaitForOutboundGateways(t, s4, 3, 2*time.Second)\n\n\ts1.Shutdown()\n\ts2.Shutdown()\n\ts3.Shutdown()\n\ts4.Shutdown()\n\n\to1 = testDefaultOptionsForGateway(\"A\")\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 = testDefaultOptionsForGateway(\"A\")\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s1.ClusterAddr().Port))\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to3 = testDefaultOptionsForGateway(\"A\")\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s1.ClusterAddr().Port))\n\ts3 = runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\to4 = testGatewayOptionsFromToWithServers(t, \"B\", \"A\", s1)\n\ts4 = runGatewayServer(o4)\n\tdefer s4.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s4, 1, 2*time.Second)\n\twaitForInboundGateways(t, s4, 3, 2*time.Second)\n\n\to5 := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s4)\n\ts5 := runGatewayServer(o5)\n\tdefer s5.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 2, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 2, 2*time.Second)\n\twaitForOutboundGateways(t, s4, 2, 2*time.Second)\n\twaitForInboundGateways(t, s4, 4, 2*time.Second)\n\twaitForOutboundGateways(t, s5, 2, 2*time.Second)\n\twaitForInboundGateways(t, s5, 4, 2*time.Second)\n}\n\nfunc TestGatewayRejectUnknown(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\t// Create a gateway from A to B, but configure B to reject non configured ones.\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\to1.Gateway.RejectUnknown = true\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Wait for outbound/inbound to be created.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\n\t// Create gateway C to B. B will tell C to connect to A,\n\t// which A should reject.\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s2)\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\t// s3 should have outbound to B, but not to A\n\twaitForOutboundGateways(t, s3, 1, time.Second)\n\t// s2 should have 2 inbounds (one from s1 one from s3)\n\twaitForInboundGateways(t, s2, 2, time.Second)\n\n\t// s1 should have single outbound/inbound with s2.\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\n\t// It should not have a registered remote gateway with C (s3)\n\tif s1.getOutboundGatewayConnection(\"C\") != nil {\n\t\tt.Fatalf(\"A should not have outbound gateway to C\")\n\t}\n\tif s1.getRemoteGateway(\"C\") != nil {\n\t\tt.Fatalf(\"A should not have a registered remote gateway to C\")\n\t}\n\n\t// Restart s1 and this time, B will tell A to connect to C.\n\t// But A will not even attempt that since it does not have\n\t// C configured.\n\ts1.Shutdown()\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\twaitForOutboundGateways(t, s2, 2, time.Second)\n\twaitForInboundGateways(t, s2, 2, time.Second)\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s3, 1, time.Second)\n\twaitForInboundGateways(t, s3, 1, time.Second)\n\t// It should not have a registered remote gateway with C (s3)\n\tif s1.getOutboundGatewayConnection(\"C\") != nil {\n\t\tt.Fatalf(\"A should not have outbound gateway to C\")\n\t}\n\tif s1.getRemoteGateway(\"C\") != nil {\n\t\tt.Fatalf(\"A should not have a registered remote gateway to C\")\n\t}\n}\n\nfunc TestGatewayNoReconnectOnClose(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t// Shutdown s1, and check that there is no attempt to reconnect.\n\ts1.Shutdown()\n\ttime.Sleep(250 * time.Millisecond)\n\twaitForOutboundGateways(t, s1, 0, time.Second)\n\twaitForOutboundGateways(t, s2, 0, time.Second)\n\twaitForInboundGateways(t, s2, 0, time.Second)\n}\n\nfunc TestGatewayDontSendSubInterest(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\ts2Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tsubnc := natsConnect(t, s2Url)\n\tdefer subnc.Close()\n\tnatsSub(t, subnc, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, subnc)\n\n\tcheckExpectedSubs(t, 1, s2)\n\t// Subscription should not be sent to s1\n\tcheckExpectedSubs(t, 0, s1)\n\n\t// Restart s1\n\ts1.Shutdown()\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tcheckExpectedSubs(t, 1, s2)\n\tcheckExpectedSubs(t, 0, s1)\n}\n\nfunc setAccountUserPassInOptions(o *Options, accName, username, password string) {\n\tacc := NewAccount(accName)\n\to.Accounts = append(o.Accounts, acc)\n\to.Users = append(o.Users, &User{Username: username, Password: password, Account: acc})\n}\n\nfunc TestGatewayAccountInterest(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\to2 := testDefaultOptionsForGateway(\"B\")\n\t// Add users to cause s2 to require auth. Will add an account with user later.\n\to2.Users = append([]*User(nil), &User{Username: \"test\", Password: \"pwd\"})\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\tsetAccountUserPassInOptions(o1, \"$foo\", \"ivan\", \"password\")\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Make this server initiate connection to A, so it is faster\n\t// when restarting it at the end of this test.\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"A\", s1)\n\tsetAccountUserPassInOptions(o3, \"$foo\", \"ivan\", \"password\")\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, time.Second)\n\twaitForOutboundGateways(t, s2, 2, time.Second)\n\twaitForOutboundGateways(t, s3, 2, time.Second)\n\n\ts1Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o1.Port)\n\tnc := natsConnect(t, s1Url)\n\tdefer nc.Close()\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\n\t// On first send, the message should be sent.\n\tcheckCount := func(t *testing.T, c *client, expected int) {\n\t\tt.Helper()\n\t\tc.mu.Lock()\n\t\tout := c.outMsgs\n\t\tc.mu.Unlock()\n\t\tif int(out) != expected {\n\t\t\tt.Fatalf(\"Expected %d message(s) to be sent over, got %v\", expected, out)\n\t\t}\n\t}\n\tgwcb := s1.getOutboundGatewayConnection(\"B\")\n\tcheckCount(t, gwcb, 1)\n\tgwcc := s1.getOutboundGatewayConnection(\"C\")\n\tcheckCount(t, gwcc, 1)\n\n\t// S2 and S3 should have sent a protocol indicating no account interest.\n\tcheckForAccountNoInterest(t, gwcb, \"$foo\", true, 2*time.Second)\n\tcheckForAccountNoInterest(t, gwcc, \"$foo\", true, 2*time.Second)\n\t// Second send should not go to B nor C.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 1)\n\tcheckCount(t, gwcc, 1)\n\n\t// Add account to S2 and a client, this should clear the no-interest\n\t// for that account.\n\ts2FooAcc, err := s2.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error registering account: %v\", err)\n\t}\n\ts2.mu.Lock()\n\ts2.users[\"ivan\"] = &User{Account: s2FooAcc, Username: \"ivan\", Password: \"password\"}\n\ts2.mu.Unlock()\n\ts2Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o2.Port)\n\tncS2 := natsConnect(t, s2Url)\n\tdefer ncS2.Close()\n\t// Any subscription should cause s2 to send an A+\n\tnatsSubSync(t, ncS2, \"asub\")\n\t// Wait for the A+\n\tcheckForAccountNoInterest(t, gwcb, \"$foo\", false, 2*time.Second)\n\n\t// Now publish a message that should go to B\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 2)\n\t// Still won't go to C since there is no sub interest\n\tcheckCount(t, gwcc, 1)\n\n\t// We should have received a subject no interest for foo\n\tcheckForSubjectNoInterest(t, gwcb, \"$foo\", \"foo\", true, 2*time.Second)\n\n\t// Now if we close the client, which removed the sole subscription,\n\t// and publish to a new subject, we should then get an A-\n\tncS2.Close()\n\t// Wait a bit...\n\ttime.Sleep(20 * time.Millisecond)\n\t// Publish on new subject\n\tnatsPub(t, nc, \"bar\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\t// It should go out to B...\n\tcheckCount(t, gwcb, 3)\n\t// But then we should get a A-\n\tcheckForAccountNoInterest(t, gwcb, \"$foo\", true, 2*time.Second)\n\n\t// Restart C and that should reset the no-interest\n\ts3.Shutdown()\n\ts3 = runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 2, 2*time.Second)\n\twaitForOutboundGateways(t, s3, 2, 2*time.Second)\n\n\t// First refresh gwcc\n\tgwcc = s1.getOutboundGatewayConnection(\"C\")\n\t// Verify that it's count is 0\n\tcheckCount(t, gwcc, 0)\n\t// Publish and now...\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\t// it should not go to B (no sub interest)\n\tcheckCount(t, gwcb, 3)\n\t// but will go to C\n\tcheckCount(t, gwcc, 1)\n}\n\nfunc TestGatewayAccountUnsub(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Connect on B\n\tncb := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port))\n\tdefer ncb.Close()\n\t// Create subscription\n\tnatsSub(t, ncb, \"foo\", func(m *nats.Msg) {\n\t\tncb.Publish(m.Reply, []byte(\"reply\"))\n\t})\n\tnatsFlush(t, ncb)\n\n\t// Connect on A\n\tnca := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\tdefer nca.Close()\n\t// Send a request\n\tif _, err := nca.Request(\"foo\", []byte(\"req\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Error getting reply: %v\", err)\n\t}\n\n\t// Now close connection on B\n\tncb.Close()\n\n\t// Publish lots of messages on \"foo\" from A.\n\t// We should receive an A- shortly and the number\n\t// of outbound messages from A to B should not be\n\t// close to the number of messages sent here.\n\ttotal := 5000\n\tfor i := 0; i < total; i++ {\n\t\tnatsPub(t, nca, \"foo\", []byte(\"hello\"))\n\t\t// Try to slow down things a bit to give a chance\n\t\t// to srvB to send the A- and to srvA to be able\n\t\t// to process it, which will then suppress the sends.\n\t\tif i%100 == 0 {\n\t\t\tnatsFlush(t, nca)\n\t\t}\n\t}\n\tnatsFlush(t, nca)\n\n\tc := sa.getOutboundGatewayConnection(\"B\")\n\tc.mu.Lock()\n\tout := c.outMsgs\n\tc.mu.Unlock()\n\n\tif out >= int64(80*total)/100 {\n\t\tt.Fatalf(\"Unexpected number of messages sent from A to B: %v\", out)\n\t}\n}\n\nfunc TestGatewaySubjectInterest(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\to1 := testDefaultOptionsForGateway(\"A\")\n\tsetAccountUserPassInOptions(o1, \"$foo\", \"ivan\", \"password\")\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", s1)\n\tsetAccountUserPassInOptions(o2, \"$foo\", \"ivan\", \"password\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t// We will create a subscription that we are not testing so\n\t// that we don't get an A- in this test.\n\ts2Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o2.Port)\n\tncb := natsConnect(t, s2Url)\n\tdefer ncb.Close()\n\tnatsSubSync(t, ncb, \"not.used\")\n\tcheckExpectedSubs(t, 1, s2)\n\n\ts1Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o1.Port)\n\tnc := natsConnect(t, s1Url)\n\tdefer nc.Close()\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\n\t// On first send, the message should be sent.\n\tcheckCount := func(t *testing.T, c *client, expected int) {\n\t\tt.Helper()\n\t\tc.mu.Lock()\n\t\tout := c.outMsgs\n\t\tc.mu.Unlock()\n\t\tif int(out) != expected {\n\t\t\tt.Fatalf(\"Expected %d message(s) to be sent over, got %v\", expected, out)\n\t\t}\n\t}\n\tgwcb := s1.getOutboundGatewayConnection(\"B\")\n\tcheckCount(t, gwcb, 1)\n\n\t// S2 should have sent a protocol indicating no subject interest.\n\tcheckNoInterest := func(t *testing.T, subject string, expectedNoInterest bool) {\n\t\tt.Helper()\n\t\tcheckForSubjectNoInterest(t, gwcb, \"$foo\", subject, expectedNoInterest, 2*time.Second)\n\t}\n\tcheckNoInterest(t, \"foo\", true)\n\t// Second send should not go through to B\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 1)\n\n\t// Now create subscription interest on B (s2)\n\tch := make(chan bool, 1)\n\tsub := natsSub(t, ncb, \"foo\", func(_ *nats.Msg) {\n\t\tch <- true\n\t})\n\tnatsFlush(t, ncb)\n\tcheckExpectedSubs(t, 2, s2)\n\tcheckExpectedSubs(t, 0, s1)\n\n\t// This should clear the no interest for this subject\n\tcheckNoInterest(t, \"foo\", false)\n\t// Third send should go to B\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 2)\n\n\t// Make sure message is received\n\twaitCh(t, ch, \"Did not get our message\")\n\t// Now unsubscribe, there won't be an UNSUB sent to the gateway.\n\tnatsUnsub(t, sub)\n\tnatsFlush(t, ncb)\n\tcheckExpectedSubs(t, 1, s2)\n\tcheckExpectedSubs(t, 0, s1)\n\n\t// So now sending a message should go over, but then we should get an RS-\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 3)\n\n\tcheckNoInterest(t, \"foo\", true)\n\n\t// Send one more time and now it should not go to B\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 3)\n\n\t// Send on bar, message should go over.\n\tnatsPub(t, nc, \"bar\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 4)\n\n\t// But now we should have receives an RS- on bar.\n\tcheckNoInterest(t, \"bar\", true)\n\n\t// Check that wildcards are supported. Create a subscription on '*' on B.\n\t// This should clear the no-interest on both \"foo\" and \"bar\"\n\tnatsSub(t, ncb, \"*\", func(_ *nats.Msg) {})\n\tnatsFlush(t, ncb)\n\tcheckExpectedSubs(t, 2, s2)\n\tcheckExpectedSubs(t, 0, s1)\n\tcheckNoInterest(t, \"foo\", false)\n\tcheckNoInterest(t, \"bar\", false)\n\t// Publish on message on foo and one on bar and they should go.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsPub(t, nc, \"bar\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 6)\n\n\t// Restart B and that should clear everything on A\n\tncb.Close()\n\ts2.Shutdown()\n\ts2 = runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tncb = natsConnect(t, s2Url)\n\tdefer ncb.Close()\n\tnatsSubSync(t, ncb, \"not.used\")\n\tcheckExpectedSubs(t, 1, s2)\n\n\tgwcb = s1.getOutboundGatewayConnection(\"B\")\n\tcheckCount(t, gwcb, 0)\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 1)\n\n\tcheckNoInterest(t, \"foo\", true)\n\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 1)\n\n\t// Add a node to B cluster and subscribe there.\n\t// We want to ensure that the no-interest is cleared\n\t// when s2 receives remote SUB from s2bis\n\to2bis := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", s1)\n\tsetAccountUserPassInOptions(o2bis, \"$foo\", \"ivan\", \"password\")\n\to2bis.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port))\n\ts2bis := runGatewayServer(o2bis)\n\tdefer s2bis.Shutdown()\n\n\tcheckClusterFormed(t, s2, s2bis)\n\n\t// Make sure all outbound gateway connections are setup\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForOutboundGateways(t, s2bis, 1, time.Second)\n\n\t// A should have 2 inbound\n\twaitForInboundGateways(t, s1, 2, time.Second)\n\n\t// Create sub on s2bis\n\tncb2bis := natsConnect(t, fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o2bis.Port))\n\tdefer ncb2bis.Close()\n\tnatsSub(t, ncb2bis, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, ncb2bis)\n\n\t// Wait for subscriptions to be registered locally on s2bis and remotely on s2\n\tcheckExpectedSubs(t, 2, s2, s2bis)\n\n\t// Check that subject no-interest on A was cleared.\n\tcheckNoInterest(t, \"foo\", false)\n\n\t// Now publish. Remember, s1 has outbound gateway to s2, and s2 does not\n\t// have a local subscription and has previously sent a no-interest on \"foo\".\n\t// We check that this has been cleared due to the interest on s2bis.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 2)\n}\n\nfunc TestGatewayDoesntSendBackToItself(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\ts2Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tnc2 := natsConnect(t, s2Url)\n\tdefer nc2.Close()\n\n\tcount := int32(0)\n\tcb := func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&count, 1)\n\t}\n\tnatsSub(t, nc2, \"foo\", cb)\n\tnatsFlush(t, nc2)\n\n\ts1Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port)\n\tnc1 := natsConnect(t, s1Url)\n\tdefer nc1.Close()\n\n\tnatsSub(t, nc1, \"foo\", cb)\n\tnatsFlush(t, nc1)\n\n\t// Now send 1 message. If there is a cycle, after few ms we\n\t// should have tons of messages...\n\tnatsPub(t, nc1, \"foo\", []byte(\"cycle\"))\n\tnatsFlush(t, nc1)\n\ttime.Sleep(100 * time.Millisecond)\n\tif c := atomic.LoadInt32(&count); c != 2 {\n\t\tt.Fatalf(\"Expected only 2 messages, got %v\", c)\n\t}\n}\n\nfunc TestGatewayOrderedOutbounds(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s2)\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, time.Second)\n\twaitForOutboundGateways(t, s2, 2, time.Second)\n\twaitForOutboundGateways(t, s3, 2, time.Second)\n\n\tgws := make([]*client, 0, 2)\n\ts2.getOutboundGatewayConnections(&gws)\n\n\t// RTTs are expected to be initially 0. So update RTT of first\n\t// in the array so that its value is no longer 0, this should\n\t// cause order to be flipped.\n\tc := gws[0]\n\tc.mu.Lock()\n\tc.sendPing()\n\tc.mu.Unlock()\n\n\t// Wait a tiny but\n\ttime.Sleep(15 * time.Millisecond)\n\t// Get the ordering again.\n\tgws = gws[:0]\n\ts2.getOutboundGatewayConnections(&gws)\n\t// Verify order is correct.\n\tfRTT := gws[0].getRTTValue()\n\tsRTT := gws[1].getRTTValue()\n\tif fRTT > sRTT {\n\t\tt.Fatalf(\"Wrong ordering: %v, %v\", fRTT, sRTT)\n\t}\n\n\t// What is the first in the array?\n\tgws[0].mu.Lock()\n\tgwName := gws[0].gw.name\n\tgws[0].mu.Unlock()\n\tif gwName == \"A\" {\n\t\ts1.Shutdown()\n\t} else {\n\t\ts3.Shutdown()\n\t}\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\tgws = gws[:0]\n\ts2.getOutboundGatewayConnections(&gws)\n\tif len(gws) != 1 {\n\t\tt.Fatalf(\"Expected size of outo to be 1, got %v\", len(gws))\n\t}\n\tgws[0].mu.Lock()\n\tname := gws[0].gw.name\n\tgws[0].mu.Unlock()\n\tif gwName == name {\n\t\tt.Fatalf(\"Gateway %q should have been removed\", gwName)\n\t}\n\t// Stop the remaining gateway\n\tif gwName == \"A\" {\n\t\ts3.Shutdown()\n\t} else {\n\t\ts1.Shutdown()\n\t}\n\twaitForOutboundGateways(t, s2, 0, time.Second)\n\tgws = gws[:0]\n\ts2.getOutboundGatewayConnections(&gws)\n\tif len(gws) != 0 {\n\t\tt.Fatalf(\"Expected size of outo to be 0, got %v\", len(gws))\n\t}\n}\n\nfunc TestGatewayQueueSub(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tsBUrl := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tncB := natsConnect(t, sBUrl)\n\tdefer ncB.Close()\n\n\tcount2 := int32(0)\n\tcb2 := func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&count2, 1)\n\t}\n\tqsubOnB := natsQueueSub(t, ncB, \"foo\", \"bar\", cb2)\n\tnatsFlush(t, ncB)\n\n\tsAUrl := fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port)\n\tncA := natsConnect(t, sAUrl)\n\tdefer ncA.Close()\n\n\tcount1 := int32(0)\n\tcb1 := func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&count1, 1)\n\t}\n\tqsubOnA := natsQueueSub(t, ncA, \"foo\", \"bar\", cb1)\n\tnatsFlush(t, ncA)\n\n\t// Make sure subs are registered on each server\n\tcheckExpectedSubs(t, 1, s1, s2)\n\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 1, time.Second)\n\tcheckForRegisteredQSubInterest(t, s2, \"A\", globalAccountName, \"foo\", 1, time.Second)\n\n\ttotal := 100\n\tsend := func(t *testing.T, nc *nats.Conn) {\n\t\tt.Helper()\n\t\tfor i := 0; i < total; i++ {\n\t\t\t// Alternate with adding a reply\n\t\t\tif i%2 == 0 {\n\t\t\t\tnatsPubReq(t, nc, \"foo\", \"reply\", []byte(\"msg\"))\n\t\t\t} else {\n\t\t\t\tnatsPub(t, nc, \"foo\", []byte(\"msg\"))\n\t\t\t}\n\t\t}\n\t\tnatsFlush(t, nc)\n\t}\n\t// Send from client connecting to S1 (cluster A)\n\tsend(t, ncA)\n\n\tcheck := func(t *testing.T, count *int32, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := int(atomic.LoadInt32(count)); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected to get %v messages, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Check that all messages stay on S1 (cluster A)\n\tcheck(t, &count1, total)\n\tcheck(t, &count2, 0)\n\n\t// Now send from the other side\n\tsend(t, ncB)\n\tcheck(t, &count1, total)\n\tcheck(t, &count2, total)\n\n\t// Reset counters\n\tatomic.StoreInt32(&count1, 0)\n\tatomic.StoreInt32(&count2, 0)\n\n\t// Add different queue group and make sure that messages are received\n\tcount3 := int32(0)\n\tcb3 := func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&count3, 1)\n\t}\n\tbatQSub := natsQueueSub(t, ncB, \"foo\", \"bat\", cb3)\n\tnatsFlush(t, ncB)\n\tcheckExpectedSubs(t, 2, s2)\n\n\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 2, time.Second)\n\n\tsend(t, ncA)\n\tcheck(t, &count1, total)\n\tcheck(t, &count2, 0)\n\tcheck(t, &count3, total)\n\n\t// Reset counters\n\tatomic.StoreInt32(&count1, 0)\n\tatomic.StoreInt32(&count2, 0)\n\tatomic.StoreInt32(&count3, 0)\n\n\tnatsUnsub(t, batQSub)\n\tnatsFlush(t, ncB)\n\tcheckExpectedSubs(t, 1, s2)\n\n\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 1, time.Second)\n\n\t// Stop qsub on A, and send messages to A, they should\n\t// be routed to B.\n\tqsubOnA.Unsubscribe()\n\tcheckExpectedSubs(t, 0, s1)\n\tsend(t, ncA)\n\tcheck(t, &count1, 0)\n\tcheck(t, &count2, total)\n\n\t// Reset counters\n\tatomic.StoreInt32(&count1, 0)\n\tatomic.StoreInt32(&count2, 0)\n\n\t// Create a C gateway now\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s2)\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, time.Second)\n\twaitForOutboundGateways(t, s2, 2, time.Second)\n\twaitForOutboundGateways(t, s3, 2, time.Second)\n\n\twaitForInboundGateways(t, s1, 2, time.Second)\n\twaitForInboundGateways(t, s2, 2, time.Second)\n\twaitForInboundGateways(t, s3, 2, time.Second)\n\n\t// Create another qsub \"bar\"\n\tsCUrl := fmt.Sprintf(\"nats://127.0.0.1:%d\", o3.Port)\n\tncC := natsConnect(t, sCUrl)\n\tdefer ncC.Close()\n\t// Associate this with count1 (since A qsub is no longer running)\n\tnatsQueueSub(t, ncC, \"foo\", \"bar\", cb1)\n\tnatsFlush(t, ncC)\n\tcheckExpectedSubs(t, 1, s3)\n\tcheckForRegisteredQSubInterest(t, s1, \"C\", globalAccountName, \"foo\", 1, time.Second)\n\tcheckForRegisteredQSubInterest(t, s2, \"C\", globalAccountName, \"foo\", 1, time.Second)\n\n\t// Artificially bump the RTT from A to C so that\n\t// the code should favor sending to B.\n\tgwcC := s1.getOutboundGatewayConnection(\"C\")\n\tgwcC.mu.Lock()\n\tgwcC.rtt = 10 * time.Second\n\tgwcC.mu.Unlock()\n\ts1.gateway.orderOutboundConnections()\n\n\tsend(t, ncA)\n\tcheck(t, &count1, 0)\n\tcheck(t, &count2, total)\n\n\t// Add a new group on s3 that should receive all messages\n\tnatsQueueSub(t, ncC, \"foo\", \"baz\", cb3)\n\tnatsFlush(t, ncC)\n\tcheckExpectedSubs(t, 2, s3)\n\tcheckForRegisteredQSubInterest(t, s1, \"C\", globalAccountName, \"foo\", 2, time.Second)\n\tcheckForRegisteredQSubInterest(t, s2, \"C\", globalAccountName, \"foo\", 2, time.Second)\n\n\t// Reset counters\n\tatomic.StoreInt32(&count1, 0)\n\tatomic.StoreInt32(&count2, 0)\n\n\t// Make the RTTs equal\n\tgwcC.mu.Lock()\n\tgwcC.rtt = time.Second\n\tgwcC.mu.Unlock()\n\n\tgwcB := s1.getOutboundGatewayConnection(\"B\")\n\tgwcB.mu.Lock()\n\tgwcB.rtt = time.Second\n\tgwcB.mu.Unlock()\n\n\ts1.gateway.Lock()\n\ts1.gateway.orderOutboundConnectionsLocked()\n\tdestName := s1.gateway.outo[0].gw.name\n\ts1.gateway.Unlock()\n\n\tsend(t, ncA)\n\t// Group baz should receive all messages\n\tcheck(t, &count3, total)\n\n\t// Ordering is normally re-evaluated when processing PONGs,\n\t// but rest of the time order will remain the same.\n\t// Since RTT are equal, messages will go to the first\n\t// GW in the array.\n\tif destName == \"B\" {\n\t\tcheck(t, &count2, total)\n\t} else if destName == \"C\" && int(atomic.LoadInt32(&count2)) != total {\n\t\tcheck(t, &count1, total)\n\t}\n\n\t// Unsubscribe qsub on B and C should receive\n\t// all messages on count1 and count3.\n\tqsubOnB.Unsubscribe()\n\tcheckExpectedSubs(t, 0, s2)\n\n\t// gwcB should have the qsubs interest map empty now.\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tei, _ := gwcB.gw.outsim.Load(globalAccountName)\n\t\tif ei != nil {\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tif sl.Count() == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Qsub interest for account should have been removed\")\n\t})\n\n\t// Reset counters\n\tatomic.StoreInt32(&count1, 0)\n\tatomic.StoreInt32(&count2, 0)\n\tatomic.StoreInt32(&count3, 0)\n\n\tsend(t, ncA)\n\tcheck(t, &count1, total)\n\tcheck(t, &count3, total)\n}\n\nfunc TestGatewayTotalQSubs(t *testing.T) {\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tsb1URL := fmt.Sprintf(\"nats://%s:%d\", ob1.Host, ob1.Port)\n\tncb1 := natsConnect(t, sb1URL, nats.ReconnectWait(50*time.Millisecond))\n\tdefer ncb1.Close()\n\n\tsb2URL := fmt.Sprintf(\"nats://%s:%d\", ob2.Host, ob2.Port)\n\tncb2 := natsConnect(t, sb2URL, nats.ReconnectWait(50*time.Millisecond))\n\tdefer ncb2.Close()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 1, 2*time.Second)\n\n\tcheckTotalQSubs := func(t *testing.T, s *Server, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := int(atomic.LoadInt64(&s.gateway.totalQSubs)); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected TotalQSubs to be %v, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcb := func(_ *nats.Msg) {}\n\n\tnatsQueueSub(t, ncb1, \"foo\", \"bar\", cb)\n\tcheckTotalQSubs(t, sa, 1)\n\tqsub2 := natsQueueSub(t, ncb1, \"foo\", \"baz\", cb)\n\tcheckTotalQSubs(t, sa, 2)\n\tqsub3 := natsQueueSub(t, ncb1, \"foo\", \"baz\", cb)\n\tcheckTotalQSubs(t, sa, 2)\n\n\t// Shutdown sb1, there should be a failover from clients\n\t// to sb2. sb2 will then send the queue subs to sa.\n\tsb1.Shutdown()\n\n\tcheckClientsCount(t, sb2, 2)\n\tcheckExpectedSubs(t, 3, sb2)\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb2, 1, 2*time.Second)\n\n\t// When sb1 is shutdown, the total qsubs on sa should fall\n\t// down to 0, but will be updated as soon as sa and sb2\n\t// connect to each other. So instead we will verify by\n\t// making sure that the count is 2 instead of 4 if there\n\t// was a bug.\n\t// (note that there are 2 qsubs on same group, so only\n\t// 1 RS+ would have been sent for that group)\n\tcheckTotalQSubs(t, sa, 2)\n\n\t// Restart sb1\n\tsb1 = runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 0, 2*time.Second)\n\twaitForInboundGateways(t, sb2, 1, 2*time.Second)\n\n\t// Now start unsubscribing. Start with one of the duplicate\n\t// and check that count stays same.\n\tnatsUnsub(t, qsub3)\n\tcheckTotalQSubs(t, sa, 2)\n\t// Now the other, which would cause an RS-\n\tnatsUnsub(t, qsub2)\n\tcheckTotalQSubs(t, sa, 1)\n\t// Now test that if connections are closed, things are updated\n\t// properly.\n\tncb1.Close()\n\tncb2.Close()\n\tcheckTotalQSubs(t, sa, 0)\n}\n\nfunc TestGatewaySendQSubsOnGatewayConnect(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\ts2Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tsubnc := natsConnect(t, s2Url)\n\tdefer subnc.Close()\n\n\tch := make(chan bool, 1)\n\tcb := func(_ *nats.Msg) {\n\t\tch <- true\n\t}\n\tnatsQueueSub(t, subnc, \"foo\", \"bar\", cb)\n\tnatsFlush(t, subnc)\n\n\t// Now start s1 that creates a gateway to s2\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 1, time.Second)\n\n\t// Publish from s1, message should be received on s2.\n\tpubnc := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port))\n\tdefer pubnc.Close()\n\t// Publish 1 message\n\tnatsPub(t, pubnc, \"foo\", []byte(\"hello\"))\n\twaitCh(t, ch, \"Did not get out message\")\n\tpubnc.Close()\n\n\ts1.Shutdown()\n\ts1 = runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 1, time.Second)\n\n\tpubnc = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port))\n\tdefer pubnc.Close()\n\t// Publish 1 message\n\tnatsPub(t, pubnc, \"foo\", []byte(\"hello\"))\n\twaitCh(t, ch, \"Did not get out message\")\n}\n\nfunc TestGatewaySendRemoteQSubs(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ob1.Cluster.Host, ob1.Cluster.Port))\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tsbURL := fmt.Sprintf(\"nats://127.0.0.1:%d\", ob2.Port)\n\tsubnc := natsConnect(t, sbURL)\n\tdefer subnc.Close()\n\n\tch := make(chan bool, 1)\n\tcb := func(_ *nats.Msg) {\n\t\tch <- true\n\t}\n\tqsub1 := natsQueueSub(t, subnc, \"foo\", \"bar\", cb)\n\tqsub2 := natsQueueSub(t, subnc, \"foo\", \"bar\", cb)\n\tnatsFlush(t, subnc)\n\n\t// There will be 2 local qsubs on the sb2 server where the client is connected\n\tcheckExpectedSubs(t, 2, sb2)\n\t// But only 1 remote on sb1\n\tcheckExpectedSubs(t, 1, sb1)\n\n\t// Now start s1 that creates a gateway to sb1 (the one that does not have the local QSub)\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"foo\", 1, time.Second)\n\n\t// Publish from s1, message should be received on s2.\n\tsaURL := fmt.Sprintf(\"nats://127.0.0.1:%d\", oa.Port)\n\tpubnc := natsConnect(t, saURL)\n\tdefer pubnc.Close()\n\t// Publish 1 message\n\tnatsPub(t, pubnc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, pubnc)\n\twaitCh(t, ch, \"Did not get out message\")\n\n\t// Note that since cluster B has no plain sub, an \"RS- $G foo\" will have been sent.\n\t// Wait for the no interest to be received by A\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tgw := sa.getOutboundGatewayConnection(\"B\").gw\n\t\tei, _ := gw.outsim.Load(globalAccountName)\n\t\tif ei != nil {\n\t\t\te := ei.(*outsie)\n\t\t\te.RLock()\n\t\t\tdefer e.RUnlock()\n\t\t\tif _, inMap := e.ni[\"foo\"]; inMap {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"No-interest still not registered\")\n\t})\n\n\t// Unsubscribe 1 qsub\n\tnatsUnsub(t, qsub1)\n\tnatsFlush(t, subnc)\n\t// There should be only 1 local qsub on sb2 now, and the remote should still exist on sb1\n\tcheckExpectedSubs(t, 1, sb1, sb2)\n\n\t// Publish 1 message\n\tnatsPub(t, pubnc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, pubnc)\n\twaitCh(t, ch, \"Did not get out message\")\n\n\t// Unsubscribe the remaining\n\tnatsUnsub(t, qsub2)\n\tnatsFlush(t, subnc)\n\n\t// No more subs now on both sb1 and sb2\n\tcheckExpectedSubs(t, 0, sb1, sb2)\n\n\t// Server sb1 should not have qsub in its sub interest map\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tvar entry *sitally\n\t\tvar err error\n\t\tsb1.gateway.pasi.Lock()\n\t\tasim := sb1.gateway.pasi.m[globalAccountName]\n\t\tif asim != nil {\n\t\t\tentry = asim[\"foo bar\"]\n\t\t}\n\t\tif entry != nil {\n\t\t\terr = fmt.Errorf(\"Map should not have an entry, got %#v\", entry)\n\t\t}\n\t\tsb1.gateway.pasi.Unlock()\n\t\treturn err\n\t})\n\n\t// Let's wait for A to receive the unsubscribe\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tgw := sa.getOutboundGatewayConnection(\"B\").gw\n\t\tei, _ := gw.outsim.Load(globalAccountName)\n\t\tif ei != nil {\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tif sl.Count() == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Interest still present\")\n\t})\n\n\t// Now send a message, it won't be sent because A received an RS-\n\t// on the first published message since there was no plain sub interest.\n\tnatsPub(t, pubnc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, pubnc)\n\n\t// Get the gateway connection from A (sa) to B (sb1)\n\tgw := sa.getOutboundGatewayConnection(\"B\")\n\tgw.mu.Lock()\n\tout := gw.outMsgs\n\tgw.mu.Unlock()\n\tif out != 2 {\n\t\tt.Fatalf(\"Expected 2 out messages, got %v\", out)\n\t}\n\n\t// Restart A\n\tpubnc.Close()\n\tsa.Shutdown()\n\tsa = runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\n\t// Check qsubs interest should be empty\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tgw := sa.getOutboundGatewayConnection(\"B\").gw\n\t\tif ei, _ := gw.outsim.Load(globalAccountName); ei == nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Interest still present\")\n\t})\n}\n\nfunc TestGatewayComplexSetup(t *testing.T) {\n\tdoLog := false\n\n\t// This test will have the following setup:\n\t// --- means route connection\n\t// === means gateway connection\n\t// [o] is outbound\n\t// [i] is inbound\n\t// Each server as an outbound connection to the other cluster.\n\t// It may have 0 or more inbound connection(s).\n\t//\n\t// Cluster A\t\tCluster B\n\t// sa1 [o]===========>[i]\n\t// | [i]<===========[o]\n\t// | \t\tsb1 ------- sb2\n\t// |\t\t\t\t\t[i]\t\t\t[o]\n\t// sa2 [o]=============^ \t\t ||\n\t//\t\t [i]<========================||\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\tif doLog {\n\t\tsb1.SetLogger(logger.NewTestLogger(\"[B1] - \", true), true, true)\n\t}\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\tif doLog {\n\t\tsa1.SetLogger(logger.NewTestLogger(\"[A1] - \", true), true, true)\n\t}\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sb1, 1, time.Second)\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sa1.ClusterAddr().Port))\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\tif doLog {\n\t\tsa2.SetLogger(logger.NewTestLogger(\"[A2] - \", true), true, true)\n\t}\n\n\tcheckClusterFormed(t, sa1, sa2)\n\n\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\twaitForInboundGateways(t, sb1, 2, time.Second)\n\n\tob2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa2)\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tvar sb2 *Server\n\tfor {\n\t\tsb2 = runGatewayServer(ob2)\n\t\tdefer sb2.Shutdown()\n\n\t\tcheckClusterFormed(t, sb1, sb2)\n\n\t\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\t\twaitForInboundGateways(t, sb2, 0, time.Second)\n\t\t// For this test, we want the outbound to be to sa2, so if we don't have that,\n\t\t// restart sb2 until we get lucky.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tif sa2.numInboundGateways() == 0 {\n\t\t\tsb2.Shutdown()\n\t\t\tsb2 = nil\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tif doLog {\n\t\tsb2.SetLogger(logger.NewTestLogger(\"[B2] - \", true), true, true)\n\t}\n\n\tch := make(chan bool, 1)\n\tcb := func(_ *nats.Msg) {\n\t\tch <- true\n\t}\n\n\t// Create a subscription on sa1 and sa2.\n\tncsa1 := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", oa1.Port))\n\tdefer ncsa1.Close()\n\tsub1 := natsSub(t, ncsa1, \"foo\", cb)\n\tnatsFlush(t, ncsa1)\n\n\tncsa2 := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", oa2.Port))\n\tdefer ncsa2.Close()\n\tsub2 := natsSub(t, ncsa2, \"foo\", cb)\n\tnatsFlush(t, ncsa2)\n\n\t// sa1 will have 1 local, one remote (from sa2), same for sa2.\n\tcheckExpectedSubs(t, 2, sa1, sa2)\n\n\t// Connect to sb2 and send 1 message\n\tncsb2 := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob2.Port))\n\tdefer ncsb2.Close()\n\tnatsPub(t, ncsb2, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, ncsb2)\n\n\tfor i := 0; i < 2; i++ {\n\t\twaitCh(t, ch, \"Did not get our message\")\n\t}\n\n\t// Unsubscribe sub2, and send 1, should still get it.\n\tnatsUnsub(t, sub2)\n\tnatsFlush(t, ncsa2)\n\tnatsPub(t, ncsb2, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, ncsb2)\n\twaitCh(t, ch, \"Did not get our message\")\n\n\t// Unsubscribe sub1, all server's sublist should be empty\n\tsub1.Unsubscribe()\n\tnatsFlush(t, ncsa1)\n\n\tcheckExpectedSubs(t, 0, sa1, sa2, sb1, sb2)\n\n\t// Create queue subs\n\ttotal := 100\n\tc1 := int32(0)\n\tc2 := int32(0)\n\tc3 := int32(0)\n\ttc := int32(0)\n\tnatsQueueSub(t, ncsa1, \"foo\", \"bar\", func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&c1, 1)\n\t\tif c := atomic.AddInt32(&tc, 1); int(c) == total {\n\t\t\tch <- true\n\t\t}\n\t})\n\tnatsFlush(t, ncsa1)\n\tnatsQueueSub(t, ncsa2, \"foo\", \"bar\", func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&c2, 1)\n\t\tif c := atomic.AddInt32(&tc, 1); int(c) == total {\n\t\t\tch <- true\n\t\t}\n\t})\n\tnatsFlush(t, ncsa2)\n\tcheckExpectedSubs(t, 2, sa1, sa2)\n\n\tqsubOnB2 := natsQueueSub(t, ncsb2, \"foo\", \"bar\", func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&c3, 1)\n\t\tif c := atomic.AddInt32(&tc, 1); int(c) == total {\n\t\t\tch <- true\n\t\t}\n\t})\n\tnatsFlush(t, ncsb2)\n\tcheckExpectedSubs(t, 1, sb2)\n\n\tcheckForRegisteredQSubInterest(t, sb1, \"A\", globalAccountName, \"foo\", 1, time.Second)\n\n\t// Publish all messages. The queue sub on cluster B should receive all\n\t// messages.\n\tfor i := 0; i < total; i++ {\n\t\tnatsPub(t, ncsb2, \"foo\", []byte(\"msg\"))\n\t}\n\tnatsFlush(t, ncsb2)\n\n\twaitCh(t, ch, \"Did not get all our queue messages\")\n\tif n := int(atomic.LoadInt32(&c1)); n != 0 {\n\t\tt.Fatalf(\"No message should have been received by qsub1, got %v\", n)\n\t}\n\tif n := int(atomic.LoadInt32(&c2)); n != 0 {\n\t\tt.Fatalf(\"No message should have been received by qsub2, got %v\", n)\n\t}\n\tif n := int(atomic.LoadInt32(&c3)); n != total {\n\t\tt.Fatalf(\"All messages should have been delivered to qsub on B, got %v\", n)\n\t}\n\n\t// Reset counters\n\tatomic.StoreInt32(&c1, 0)\n\tatomic.StoreInt32(&c2, 0)\n\tatomic.StoreInt32(&c3, 0)\n\tatomic.StoreInt32(&tc, 0)\n\n\t// Now send from cluster A, messages should be distributed to qsubs on A.\n\tfor i := 0; i < total; i++ {\n\t\tnatsPub(t, ncsa1, \"foo\", []byte(\"msg\"))\n\t}\n\tnatsFlush(t, ncsa1)\n\n\texpectedLow := int(float32(total/2) * 0.6)\n\texpectedHigh := int(float32(total/2) * 1.4)\n\tcheckCount := func(t *testing.T, count *int32) {\n\t\tt.Helper()\n\t\tc := int(atomic.LoadInt32(count))\n\t\tif c < expectedLow || c > expectedHigh {\n\t\t\tt.Fatalf(\"Expected value to be between %v/%v, got %v\", expectedLow, expectedHigh, c)\n\t\t}\n\t}\n\twaitCh(t, ch, \"Did not get all our queue messages\")\n\tcheckCount(t, &c1)\n\tcheckCount(t, &c2)\n\n\t// Now unsubscribe sub on B and reset counters\n\tnatsUnsub(t, qsubOnB2)\n\tcheckExpectedSubs(t, 0, sb2)\n\tatomic.StoreInt32(&c1, 0)\n\tatomic.StoreInt32(&c2, 0)\n\tatomic.StoreInt32(&c3, 0)\n\tatomic.StoreInt32(&tc, 0)\n\t// Publish from cluster B, messages should be delivered to cluster A.\n\tfor i := 0; i < total; i++ {\n\t\tnatsPub(t, ncsb2, \"foo\", []byte(\"msg\"))\n\t}\n\tnatsFlush(t, ncsb2)\n\n\twaitCh(t, ch, \"Did not get all our queue messages\")\n\tif n := int(atomic.LoadInt32(&c3)); n != 0 {\n\t\tt.Fatalf(\"There should not have been messages on unsubscribed sub, got %v\", n)\n\t}\n\tcheckCount(t, &c1)\n\tcheckCount(t, &c2)\n}\n\nfunc TestGatewayMsgSentOnlyOnce(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\ts2Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tnc2 := natsConnect(t, s2Url)\n\tdefer nc2.Close()\n\n\ts1Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port)\n\tnc1 := natsConnect(t, s1Url)\n\tdefer nc1.Close()\n\n\tch := make(chan bool, 1)\n\tcount := int32(0)\n\texpected := int32(4)\n\tcb := func(_ *nats.Msg) {\n\t\tif c := atomic.AddInt32(&count, 1); c == expected {\n\t\t\tch <- true\n\t\t}\n\t}\n\n\t// On s1, create 2 plain subs, 2 queue members for group\n\t// \"bar\" and 1 for group \"baz\".\n\tnatsSub(t, nc1, \">\", cb)\n\tnatsSub(t, nc1, \"foo\", cb)\n\tnatsQueueSub(t, nc1, \"foo\", \"bar\", cb)\n\tnatsQueueSub(t, nc1, \"foo\", \"bar\", cb)\n\tnatsQueueSub(t, nc1, \"foo\", \"baz\", cb)\n\tnatsFlush(t, nc1)\n\n\t// Ensure subs registered in S1\n\tcheckExpectedSubs(t, 5, s1)\n\n\t// Also need to wait for qsubs to be registered on s2.\n\tcheckForRegisteredQSubInterest(t, s2, \"A\", globalAccountName, \"foo\", 2, time.Second)\n\n\t// From s2, send 1 message, s1 should receive 1 only,\n\t// and total we should get the callback notified 4 times.\n\tnatsPub(t, nc2, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc2)\n\n\twaitCh(t, ch, \"Did not get our messages\")\n\t// Verifiy that count is still 4\n\tif c := atomic.LoadInt32(&count); c != expected {\n\t\tt.Fatalf(\"Expected %v messages, got %v\", expected, c)\n\t}\n\t// Check s2 outbound connection stats. It should say that it\n\t// sent only 1 message.\n\tc := s2.getOutboundGatewayConnection(\"A\")\n\tif c == nil {\n\t\tt.Fatalf(\"S2 outbound gateway not found\")\n\t}\n\tc.mu.Lock()\n\tout := c.outMsgs\n\tc.mu.Unlock()\n\tif out != 1 {\n\t\tt.Fatalf(\"Expected s2's outbound gateway to have sent a single message, got %v\", out)\n\t}\n\t// Now check s1's inbound gateway\n\ts1.gateway.RLock()\n\tc = nil\n\tfor _, ci := range s1.gateway.in {\n\t\tc = ci\n\t\tbreak\n\t}\n\ts1.gateway.RUnlock()\n\tif c == nil {\n\t\tt.Fatalf(\"S1 inbound gateway not found\")\n\t}\n\tif in := atomic.LoadInt64(&c.inMsgs); in != 1 {\n\t\tt.Fatalf(\"Expected s1's inbound gateway to have received a single message, got %v\", in)\n\t}\n}\n\ntype checkErrorLogger struct {\n\tDummyLogger\n\tcheckErrorStr string\n\tgotError bool\n}\n\nfunc (l *checkErrorLogger) Errorf(format string, args ...any) {\n\tl.DummyLogger.Errorf(format, args...)\n\tl.Lock()\n\tif strings.Contains(l.Msg, l.checkErrorStr) {\n\t\tl.gotError = true\n\t}\n\tl.Unlock()\n}\n\nfunc TestGatewayRoutedServerWithoutGatewayConfigured(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\to3 := DefaultOptions()\n\to3.NoSystemAccount = true\n\to3.Cluster.Name = \"B\"\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s2.ClusterAddr().Port))\n\ts3 := New(o3)\n\tdefer s3.Shutdown()\n\tl := &checkErrorLogger{checkErrorStr: \"not configured\"}\n\ts3.SetLogger(l, true, true)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\ts3.Start()\n\t\twg.Done()\n\t}()\n\n\tcheckClusterFormed(t, s2, s3)\n\n\t// Check that server s3 does not panic when being notified\n\t// about the A gateway, but report an error.\n\tdeadline := time.Now().Add(2 * time.Second)\n\tgotIt := false\n\tfor time.Now().Before(deadline) {\n\t\tl.Lock()\n\t\tgotIt = l.gotError\n\t\tl.Unlock()\n\t\tif gotIt {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(15 * time.Millisecond)\n\t}\n\tif !gotIt {\n\t\tt.Fatalf(\"Should have reported error about gateway not configured\")\n\t}\n\n\ts3.Shutdown()\n\twg.Wait()\n}\n\nfunc TestGatewaySendsToNonLocalSubs(t *testing.T) {\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sb1, 1, time.Second)\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sa1.ClusterAddr().Port))\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\tcheckClusterFormed(t, sa1, sa2)\n\n\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\twaitForInboundGateways(t, sb1, 2, time.Second)\n\n\tch := make(chan bool, 1)\n\t// Create an interest of sa2\n\tncSub := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", oa2.Port))\n\tdefer ncSub.Close()\n\tnatsSub(t, ncSub, \"foo\", func(_ *nats.Msg) { ch <- true })\n\tnatsFlush(t, ncSub)\n\tcheckExpectedSubs(t, 1, sa1, sa2)\n\n\t// Produce a message from sb1, make sure it can be received.\n\tncPub := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Port))\n\tdefer ncPub.Close()\n\tnatsPub(t, ncPub, \"foo\", []byte(\"hello\"))\n\twaitCh(t, ch, \"Did not get our message\")\n\n\tncSub.Close()\n\tncPub.Close()\n\tcheckExpectedSubs(t, 0, sa1, sa2)\n\n\t// Now create sb2 that has a route to sb1 and gateway connects to sa2.\n\tob2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa2)\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\n\tncSub = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", oa1.Port))\n\tdefer ncSub.Close()\n\tnatsSub(t, ncSub, \"foo\", func(_ *nats.Msg) { ch <- true })\n\tnatsFlush(t, ncSub)\n\tcheckExpectedSubs(t, 1, sa1, sa2)\n\n\tncPub = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob2.Port))\n\tdefer ncPub.Close()\n\tnatsPub(t, ncPub, \"foo\", []byte(\"hello\"))\n\twaitCh(t, ch, \"Did not get our message\")\n}\n\nfunc TestGatewayUnknownGatewayCommand(t *testing.T) {\n\to1 := testDefaultOptionsForGateway(\"A\")\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\tl := &checkErrorLogger{checkErrorStr: \"Unknown command\"}\n\ts1.SetLogger(l, true, true)\n\n\to2 := testDefaultOptionsForGateway(\"A\")\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", s1.ClusterAddr().Port))\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tvar route *client\n\ts2.mu.Lock()\n\tif r := getFirstRoute(s2); r != nil {\n\t\troute = r\n\t}\n\ts2.mu.Unlock()\n\n\troute.mu.Lock()\n\tinfo := &Info{\n\t\tGateway: \"B\",\n\t\tGatewayCmd: 255,\n\t}\n\tb, _ := json.Marshal(info)\n\troute.enqueueProto([]byte(fmt.Sprintf(InfoProto, b)))\n\troute.mu.Unlock()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tl.Lock()\n\t\tgotIt := l.gotError\n\t\tl.Unlock()\n\t\tif gotIt {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Did not get expected error\")\n\t})\n}\n\nfunc TestGatewayRandomIP(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithURLs(t, \"A\", \"B\",\n\t\t[]string{\n\t\t\t\"nats://noport\",\n\t\t\tfmt.Sprintf(\"nats://localhost:%d\", sb.GatewayAddr().Port),\n\t\t})\n\t// Create a dummy resolver that returns error since we\n\t// don't provide any IP. The code should then use the configured\n\t// url (localhost:port) and try with that, which in this case\n\t// should work.\n\toa.Gateway.resolver = &myDummyDNSResolver{}\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n}\n\nfunc TestGatewaySendQSubsBufSize(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tbufSize int\n\t}{\n\t\t{\n\t\t\tname: \"Bufsize 45, more than one at a time\",\n\t\t\tbufSize: 45,\n\t\t},\n\t\t{\n\t\t\tname: \"Bufsize 15, one at a time\",\n\t\t\tbufSize: 15,\n\t\t},\n\t\t{\n\t\t\tname: \"Bufsize 0, default to maxBufSize, all at once\",\n\t\t\tbufSize: 0,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\to2 := testDefaultOptionsForGateway(\"B\")\n\t\t\to2.Gateway.sendQSubsBufSize = test.bufSize\n\t\t\ts2 := runGatewayServer(o2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\ts2Url := fmt.Sprintf(\"nats://%s:%d\", o2.Host, o2.Port)\n\t\t\tnc := natsConnect(t, s2Url)\n\t\t\tdefer nc.Close()\n\t\t\tnatsQueueSub(t, nc, \"foo\", \"bar\", func(_ *nats.Msg) {})\n\t\t\tnatsQueueSub(t, nc, \"foo\", \"baz\", func(_ *nats.Msg) {})\n\t\t\tnatsQueueSub(t, nc, \"foo\", \"bat\", func(_ *nats.Msg) {})\n\t\t\tnatsQueueSub(t, nc, \"foo\", \"bax\", func(_ *nats.Msg) {})\n\n\t\t\tcheckExpectedSubs(t, 4, s2)\n\n\t\t\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\t\t\ts1 := runGatewayServer(o1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, s1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t\t\tcheckForRegisteredQSubInterest(t, s1, \"B\", globalAccountName, \"foo\", 4, time.Second)\n\n\t\t\t// Make sure we have the 4 we expected\n\t\t\tc := s1.getOutboundGatewayConnection(\"B\")\n\t\t\tei, _ := c.gw.outsim.Load(globalAccountName)\n\t\t\tif ei == nil {\n\t\t\t\tt.Fatalf(\"No interest found\")\n\t\t\t}\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tr := sl.Match(\"foo\")\n\t\t\tif len(r.qsubs) != 4 {\n\t\t\t\tt.Fatalf(\"Expected 4 groups, got %v\", len(r.qsubs))\n\t\t\t}\n\t\t\tvar gotBar, gotBaz, gotBat, gotBax bool\n\t\t\tfor _, qs := range r.qsubs {\n\t\t\t\tif len(qs) != 1 {\n\t\t\t\t\tt.Fatalf(\"Unexpected number of subs for group %s: %v\", qs[0].queue, len(qs))\n\t\t\t\t}\n\t\t\t\tq := qs[0].queue\n\t\t\t\tswitch string(q) {\n\t\t\t\tcase \"bar\":\n\t\t\t\t\tgotBar = true\n\t\t\t\tcase \"baz\":\n\t\t\t\t\tgotBaz = true\n\t\t\t\tcase \"bat\":\n\t\t\t\t\tgotBat = true\n\t\t\t\tcase \"bax\":\n\t\t\t\t\tgotBax = true\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected group name: %s\", q)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !gotBar || !gotBaz || !gotBat || !gotBax {\n\t\t\t\tt.Fatalf(\"Did not get all we wanted: bar=%v baz=%v bat=%v bax=%v\",\n\t\t\t\t\tgotBar, gotBaz, gotBat, gotBax)\n\t\t\t}\n\n\t\t\tnc.Close()\n\t\t\ts1.Shutdown()\n\t\t\ts2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, s1, 0, time.Second)\n\t\t\twaitForOutboundGateways(t, s2, 0, time.Second)\n\t\t})\n\t}\n}\n\nfunc TestGatewayRaceBetweenPubAndSub(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\ts2Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Port)\n\tnc2 := natsConnect(t, s2Url)\n\tdefer nc2.Close()\n\n\ts1Url := fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Port)\n\tvar ncaa [5]*nats.Conn\n\tvar nca = ncaa[:0]\n\tfor i := 0; i < 5; i++ {\n\t\tnc := natsConnect(t, s1Url)\n\t\tdefer nc.Close()\n\t\tnca = append(nca, nc)\n\t}\n\n\tch := make(chan bool, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(5)\n\tfor _, nc := range nca {\n\t\tnc := nc\n\t\tgo func(n *nats.Conn) {\n\t\t\tdefer wg.Done()\n\t\t\tfor {\n\t\t\t\tn.Publish(\"foo\", []byte(\"hello\"))\n\t\t\t\tselect {\n\t\t\t\tcase <-ch:\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\t\t}(nc)\n\t}\n\ttime.Sleep(100 * time.Millisecond)\n\tnatsQueueSub(t, nc2, \"foo\", \"bar\", func(m *nats.Msg) {\n\t\tnatsUnsub(t, m.Sub)\n\t\tclose(ch)\n\t})\n\twg.Wait()\n}\n\n// Returns the first (if any) of the inbound connections for this name.\nfunc getInboundGatewayConnection(s *Server, name string) *client {\n\tvar gwsa [4]*client\n\tvar gws = gwsa[:0]\n\ts.getInboundGatewayConnections(&gws)\n\tfor _, gw := range gws {\n\t\tgw.mu.Lock()\n\t\tok := gw.gw.name == name\n\t\tgw.mu.Unlock()\n\t\tif ok {\n\t\t\treturn gw\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc TestGatewaySendAllSubs(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tgatewayMaxRUnsubBeforeSwitch = 100\n\tdefer func() { gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch }()\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\toc := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", sb)\n\tsc := runGatewayServer(oc)\n\tdefer sc.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 2, time.Second)\n\twaitForOutboundGateways(t, sb, 2, time.Second)\n\twaitForOutboundGateways(t, sc, 2, time.Second)\n\twaitForInboundGateways(t, sa, 2, time.Second)\n\twaitForInboundGateways(t, sb, 2, time.Second)\n\twaitForInboundGateways(t, sc, 2, time.Second)\n\n\t// On A, create a sub to register some interest\n\taURL := fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port)\n\tncA := natsConnect(t, aURL)\n\tdefer ncA.Close()\n\tnatsSub(t, ncA, \"sub.on.a.*\", func(m *nats.Msg) {})\n\tnatsFlush(t, ncA)\n\tcheckExpectedSubs(t, 1, sa)\n\n\t// On C, have some sub activity while it receives\n\t// unwanted messages and switches to interestOnly mode.\n\tcURL := fmt.Sprintf(\"nats://%s:%d\", oc.Host, oc.Port)\n\tncC := natsConnect(t, cURL)\n\tdefer ncC.Close()\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\tdone := make(chan bool)\n\tconsCount := 0\n\taccsCount := 0\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; ; i++ {\n\t\t\t// Create subs and qsubs on same subject\n\t\t\tnatsSub(t, ncC, fmt.Sprintf(\"foo.%d\", i+1), func(_ *nats.Msg) {})\n\t\t\tnatsQueueSub(t, ncC, fmt.Sprintf(\"foo.%d\", i+1), fmt.Sprintf(\"bar.%d\", i+1), func(_ *nats.Msg) {})\n\t\t\t// Create psubs and qsubs on unique subjects\n\t\t\tnatsSub(t, ncC, fmt.Sprintf(\"foox.%d\", i+1), func(_ *nats.Msg) {})\n\t\t\tnatsQueueSub(t, ncC, fmt.Sprintf(\"fooy.%d\", i+1), fmt.Sprintf(\"bar.%d\", i+1), func(_ *nats.Msg) {})\n\t\t\tconsCount += 4\n\t\t\t// Register account\n\t\t\tsc.RegisterAccount(fmt.Sprintf(\"acc.%d\", i+1))\n\t\t\taccsCount++\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase <-time.After(15 * time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\t// From B publish on subjects for which C has an interest\n\tbURL := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\tncB := natsConnect(t, bURL)\n\tdefer ncB.Close()\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tfor {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tnatsPub(t, ncB, fmt.Sprintf(\"foo.%d\", i+1), []byte(\"hello\"))\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase <-time.After(5 * time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\t// From B, send a lot of messages that A is interested in,\n\t// but not C.\n\t// TODO(ik): May need to change that if we change the threshold\n\t// for when the switch happens.\n\ttotal := 300\n\tfor i := 0; i < total; i++ {\n\t\tif err := ncB.Publish(fmt.Sprintf(\"sub.on.a.%d\", i), []byte(\"hi\")); err != nil {\n\t\t\tt.Fatalf(\"Error waiting for reply: %v\", err)\n\t\t}\n\t}\n\tclose(done)\n\n\t// Normally, C would receive a message for each req inbox and\n\t// would send and RS- on that to B, making both have an unbounded\n\t// growth of the no-interest map. But after a certain amount\n\t// of RS-, C will send all its sub for the given account and\n\t// instruct B to send only if there is explicit interest.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t// Check C inbound connection from B\n\t\tc := getInboundGatewayConnection(sc, \"B\")\n\t\tc.mu.Lock()\n\t\tvar switchedMode bool\n\t\te := c.gw.insim[globalAccountName]\n\t\tif e != nil {\n\t\t\tswitchedMode = e.ni == nil && e.mode == InterestOnly\n\t\t}\n\t\tc.mu.Unlock()\n\t\tif !switchedMode {\n\t\t\treturn fmt.Errorf(\"C has still not switched mode\")\n\t\t}\n\t\treturn nil\n\t})\n\tcheckGWInterestOnlyMode(t, sb, \"C\", globalAccountName)\n\twg.Wait()\n\n\t// Check consCount and accsCount on C\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tsc.gateway.pasi.Lock()\n\t\tscount := len(sc.gateway.pasi.m[globalAccountName])\n\t\tsc.gateway.pasi.Unlock()\n\t\tif scount != consCount {\n\t\t\treturn fmt.Errorf(\"Expected %v consumers for global account, got %v\", consCount, scount)\n\t\t}\n\t\tacount := sc.numAccounts()\n\t\tif acount != accsCount+1 {\n\t\t\treturn fmt.Errorf(\"Expected %v accounts, got %v\", accsCount+1, acount)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Also, after all that, if a sub is created on C, it should\n\t// be sent to B (but not A). Check that this is the case.\n\t// So first send from A on the subject that we are going to\n\t// use for this new sub.\n\tnatsPub(t, ncA, \"newsub\", []byte(\"hello\"))\n\tnatsFlush(t, ncA)\n\taOutboundToC := sa.getOutboundGatewayConnection(\"C\")\n\tcheckForSubjectNoInterest(t, aOutboundToC, globalAccountName, \"newsub\", true, 2*time.Second)\n\n\tnewSubSub := natsSub(t, ncC, \"newsub\", func(_ *nats.Msg) {})\n\tnatsFlush(t, ncC)\n\tcheckExpectedSubs(t, consCount+1)\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tc := sb.getOutboundGatewayConnection(\"C\")\n\t\tei, _ := c.gw.outsim.Load(globalAccountName)\n\t\tif ei != nil {\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tr := sl.Match(\"newsub\")\n\t\t\tif len(r.psubs) == 1 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Newsub not registered on B\")\n\t})\n\tcheckForSubjectNoInterest(t, aOutboundToC, globalAccountName, \"newsub\", false, 2*time.Second)\n\n\tnatsUnsub(t, newSubSub)\n\tnatsFlush(t, ncC)\n\tcheckExpectedSubs(t, consCount)\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tc := sb.getOutboundGatewayConnection(\"C\")\n\t\tei, _ := c.gw.outsim.Load(globalAccountName)\n\t\tif ei != nil {\n\t\t\tsl := ei.(*outsie).sl\n\t\t\tr := sl.Match(\"newsub\")\n\t\t\tif len(r.psubs) == 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Newsub still registered on B\")\n\t})\n}\n\nfunc TestGatewaySendAllSubsBadProtocol(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// For this test, make sure to use inbound from A so\n\t// A will reconnect when we send bad proto that\n\t// causes connection to be closed.\n\tc := getInboundGatewayConnection(sa, \"B\")\n\t// Mock an invalid protocol (account name missing)\n\tinfo := &Info{\n\t\tGateway: \"B\",\n\t\tGatewayCmd: gatewayCmdAllSubsStart,\n\t}\n\tb, _ := json.Marshal(info)\n\tc.mu.Lock()\n\tc.enqueueProto([]byte(fmt.Sprintf(\"INFO %s\\r\\n\", b)))\n\tc.mu.Unlock()\n\n\torgConn := c\n\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\tcurConn := getInboundGatewayConnection(sa, \"B\")\n\t\tif orgConn == curConn {\n\t\t\treturn fmt.Errorf(\"Not reconnected\")\n\t\t}\n\t\treturn nil\n\t})\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\n\t// Refresh\n\tc = nil\n\tcheckFor(t, 3*time.Second, 15*time.Millisecond, func() error {\n\t\tc = getInboundGatewayConnection(sa, \"B\")\n\t\tif c == nil {\n\t\t\treturn fmt.Errorf(\"Did not reconnect\")\n\t\t}\n\t\treturn nil\n\t})\n\t// Do correct start\n\tinfo.GatewayCmdPayload = []byte(globalAccountName)\n\tb, _ = json.Marshal(info)\n\tc.mu.Lock()\n\tc.enqueueProto([]byte(fmt.Sprintf(\"INFO %s\\r\\n\", b)))\n\tc.mu.Unlock()\n\t// But incorrect end.\n\tinfo.GatewayCmd = gatewayCmdAllSubsComplete\n\tinfo.GatewayCmdPayload = nil\n\tb, _ = json.Marshal(info)\n\tc.mu.Lock()\n\tc.enqueueProto([]byte(fmt.Sprintf(\"INFO %s\\r\\n\", b)))\n\tc.mu.Unlock()\n\n\torgConn = c\n\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\tcurConn := getInboundGatewayConnection(sa, \"B\")\n\t\tif orgConn == curConn {\n\t\t\treturn fmt.Errorf(\"Not reconnected\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestGatewayRaceOnClose(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\tbURL := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\tncB := natsConnect(t, bURL, nats.NoReconnect())\n\tdefer ncB.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tcb := func(_ *nats.Msg) {}\n\t\tfor {\n\t\t\t// Expect failure at one point and just return.\n\t\t\tqsub, err := ncB.QueueSubscribe(\"foo\", \"bar\", cb)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err := qsub.Unsubscribe(); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\t// Wait a bit and kill B\n\ttime.Sleep(200 * time.Millisecond)\n\tsb.Shutdown()\n\twg.Wait()\n}\n\n// Similar to TestNewRoutesServiceImport but with 2 GW servers instead\n// of a cluster of 2 servers.\nfunc TestGatewayServiceImport(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\toa := testDefaultOptionsForGateway(\"A\")\n\tsetAccountUserPassInOptions(oa, \"$foo\", \"clientA\", \"password\")\n\tsetAccountUserPassInOptions(oa, \"$bar\", \"yyyyyyy\", \"password\")\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tob := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\tsetAccountUserPassInOptions(ob, \"$foo\", \"clientBFoo\", \"password\")\n\tsetAccountUserPassInOptions(ob, \"$bar\", \"clientB\", \"password\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Get accounts\n\tfooA, _ := sa.LookupAccount(\"$foo\")\n\tbarA, _ := sa.LookupAccount(\"$bar\")\n\tfooB, _ := sb.LookupAccount(\"$foo\")\n\tbarB, _ := sb.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooA.AddServiceExport(\"test.request\", nil)\n\tfooB.AddServiceExport(\"test.request\", nil)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Same on A.\n\tif err := barA.AddServiceImport(fooA, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint and responder.\n\taURL := fmt.Sprintf(\"nats://clientA:password@127.0.0.1:%d\", oa.Port)\n\tclientA := natsConnect(t, aURL)\n\tdefer clientA.Close()\n\n\tsubA := natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tbURL := fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob.Port)\n\tclientB := natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\tsubB := natsSubSync(t, clientB, \"reply\")\n\tnatsFlush(t, clientB)\n\n\tfor i := 1; i <= 2; i++ {\n\t\t// Send the request from clientB on foo.request,\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t\tnatsFlush(t, clientB)\n\n\t\t// Expect the request on A\n\t\tmsg, err := subA.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\tif msg.Reply == \"reply\" {\n\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t}\n\n\t\t// Check for duplicate message\n\t\tif msg, err := subA.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t}\n\n\t\t// Send reply\n\t\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\t\tnatsFlush(t, clientA)\n\n\t\tmsg, err = subB.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\n\t\t// Check for duplicate message\n\t\tif msg, err := subB.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t}\n\n\t\texpected := int64(i * 2)\n\t\tvz, _ := sa.Varz(nil)\n\t\tif vz.OutMsgs != expected {\n\t\t\tt.Fatalf(\"Expected %d outMsgs for A, got %v\", expected, vz.OutMsgs)\n\t\t}\n\n\t\t// For B, we expect it to send to gateway on the two subjects: test.request\n\t\t// and foo.request then send the reply to the client and optimistically\n\t\t// to the other gateway.\n\t\tif i == 1 {\n\t\t\texpected = 4\n\t\t} else {\n\t\t\t// The second time, one of the accounts will be suppressed and the reply going\n\t\t\t// back so we should only get 2 more messages.\n\t\t\texpected = 6\n\t\t}\n\t\tvz, _ = sb.Varz(nil)\n\t\tif vz.OutMsgs != expected {\n\t\t\tt.Fatalf(\"Expected %d outMsgs for B, got %v\", expected, vz.OutMsgs)\n\t\t}\n\t}\n\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 1 {\n\t\t\treturn fmt.Errorf(\"Expected one sub to be left on fooA, but got %d\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Speed up exiration\n\terr := fooA.SetServiceExportResponseThreshold(\"test.request\", 50*time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\t// Consume the requests, but don't reply to them...\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"subA did not receive request: %v\", err)\n\t\t}\n\t}\n\n\t// These reply subjects will be dangling off of $foo account on serverA.\n\t// Remove our service endpoint and wait for the dangling replies to go to zero.\n\tnatsUnsub(t, subA)\n\tnatsFlush(t, clientA)\n\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 0 {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d, should be zero\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Repeat similar test but without the small TTL and verify\n\t// that if B is shutdown, the dangling subs for replies are\n\t// cleared from the account sublist.\n\terr = fooA.SetServiceExportResponseThreshold(\"test.request\", 10*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\n\tsubA = natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\t// Consume the requests, but don't reply to them...\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"subA did not receive request: %v\", err)\n\t\t}\n\t}\n\n\t// Shutdown B\n\tclientB.Close()\n\tsb.Shutdown()\n\n\t// Close our last sub\n\tnatsUnsub(t, subA)\n\tnatsFlush(t, clientA)\n\n\t// Verify that they are gone before the 10 sec TTL\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 0 {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d, should be zero\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check that this all work in interest-only mode\n\tsb = runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tfooB, _ = sb.LookupAccount(\"$foo\")\n\tbarB, _ = sb.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooB.AddServiceExport(\"test.request\", nil)\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\t// We need at least a subscription on A otherwise when publishing\n\t// to subjects with no interest we would simply get an A-\n\tnatsSubSync(t, clientA, \"not.used\")\n\n\t// Create a client on B that will use account $foo\n\tbURL = fmt.Sprintf(\"nats://clientBFoo:password@127.0.0.1:%d\", ob.Port)\n\tclientB = natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\t// First flood with subjects that remote gw is not interested\n\t// so we switch to interest-only.\n\tfor i := 0; i < 1100; i++ {\n\t\tnatsPub(t, clientB, fmt.Sprintf(\"no.interest.%d\", i), []byte(\"hello\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\tcheckGWInterestOnlyMode(t, sb, \"A\", \"$foo\")\n\n\t// Go back to clientB on $bar.\n\tclientB.Close()\n\tbURL = fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob.Port)\n\tclientB = natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\tsubA = natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\tsubB = natsSubSync(t, clientB, \"reply\")\n\tnatsFlush(t, clientB)\n\n\t// Sine it is interest-only, B should receive an interest\n\t// on $foo test.request\n\tcheckGWInterestOnlyModeInterestOn(t, sb, \"A\", \"$foo\", \"test.request\")\n\n\t// Send the request from clientB on foo.request,\n\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\tnatsFlush(t, clientB)\n\n\t// Expect the request on A\n\tmsg, err := subA.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t}\n\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\tif msg.Reply == \"reply\" {\n\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n\n\t// Send reply\n\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\tnatsFlush(t, clientA)\n\n\tmsg, err = subB.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t}\n\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n}\n\nfunc TestGatewayServiceImportWithQueue(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\toa := testDefaultOptionsForGateway(\"A\")\n\tsetAccountUserPassInOptions(oa, \"$foo\", \"clientA\", \"password\")\n\tsetAccountUserPassInOptions(oa, \"$bar\", \"yyyyyyy\", \"password\")\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tob := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\tsetAccountUserPassInOptions(ob, \"$foo\", \"clientBFoo\", \"password\")\n\tsetAccountUserPassInOptions(ob, \"$bar\", \"clientB\", \"password\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Get accounts\n\tfooA, _ := sa.LookupAccount(\"$foo\")\n\tbarA, _ := sa.LookupAccount(\"$bar\")\n\tfooB, _ := sb.LookupAccount(\"$foo\")\n\tbarB, _ := sb.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooA.AddServiceExport(\"test.request\", nil)\n\tfooB.AddServiceExport(\"test.request\", nil)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Same on A.\n\tif err := barA.AddServiceImport(fooA, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint and responder.\n\taURL := fmt.Sprintf(\"nats://clientA:password@127.0.0.1:%d\", oa.Port)\n\tclientA := natsConnect(t, aURL)\n\tdefer clientA.Close()\n\n\tsubA := natsQueueSubSync(t, clientA, \"test.request\", \"queue\")\n\tnatsFlush(t, clientA)\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tbURL := fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob.Port)\n\tclientB := natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\tsubB := natsQueueSubSync(t, clientB, \"reply\", \"queue2\")\n\tnatsFlush(t, clientB)\n\n\t// Wait for queue interest on test.request from A to be registered\n\t// on server B.\n\tcheckForRegisteredQSubInterest(t, sb, \"A\", \"$foo\", \"test.request\", 1, time.Second)\n\n\tfor i := 0; i < 2; i++ {\n\t\t// Send the request from clientB on foo.request,\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t\tnatsFlush(t, clientB)\n\n\t\t// Expect the request on A\n\t\tmsg, err := subA.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\tif msg.Reply == \"reply\" {\n\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t}\n\t\t// Check for duplicate message\n\t\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t}\n\n\t\t// Send reply\n\t\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\t\tnatsFlush(t, clientA)\n\n\t\tmsg, err = subB.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\t// Check for duplicate message\n\t\tif msg, err := subB.NextMsg(250 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t}\n\n\t\texpected := int64((i + 1) * 2)\n\t\tvz, _ := sa.Varz(nil)\n\t\tif vz.OutMsgs != expected {\n\t\t\tt.Fatalf(\"Expected %d outMsgs for A, got %v\", expected, vz.OutMsgs)\n\t\t}\n\n\t\t// For B, we expect it to send to gateway on the two subjects: test.request\n\t\t// and foo.request then send the reply to the client and optimistically\n\t\t// to the other gateway.\n\t\tif i == 0 {\n\t\t\texpected = 4\n\t\t} else {\n\t\t\t// The second time, one of the accounts will be suppressed and the reply going\n\t\t\t// back so we should get only 2 more messages.\n\t\t\texpected = 6\n\t\t}\n\t\tvz, _ = sb.Varz(nil)\n\t\tif vz.OutMsgs != expected {\n\t\t\tt.Fatalf(\"Expected %d outMsgs for B, got %v\", expected, vz.OutMsgs)\n\t\t}\n\t}\n\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 1 {\n\t\t\treturn fmt.Errorf(\"Expected one sub to be left on fooA, but got %d\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Speed up exiration\n\terr := fooA.SetServiceExportResponseThreshold(\"test.request\", 10*time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\t// Consume the requests, but don't reply to them...\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"subA did not receive request: %v\", err)\n\t\t}\n\t}\n\n\t// These reply subjects will be dangling off of $foo account on serverA.\n\t// Remove our service endpoint and wait for the dangling replies to go to zero.\n\tnatsUnsub(t, subA)\n\tnatsFlush(t, clientA)\n\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 0 {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d, should be zero\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\tcheckForRegisteredQSubInterest(t, sb, \"A\", \"$foo\", \"test.request\", 0, time.Second)\n\n\t// Repeat similar test but without the small TTL and verify\n\t// that if B is shutdown, the dangling subs for replies are\n\t// cleared from the account sublist.\n\terr = fooA.SetServiceExportResponseThreshold(\"test.request\", 10*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\n\tsubA = natsQueueSubSync(t, clientA, \"test.request\", \"queue\")\n\tnatsFlush(t, clientA)\n\tcheckForRegisteredQSubInterest(t, sb, \"A\", \"$foo\", \"test.request\", 1, time.Second)\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\t// Consume the requests, but don't reply to them...\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"subA did not receive request %d: %v\", i+1, err)\n\t\t}\n\t}\n\n\t// Shutdown B\n\tclientB.Close()\n\tsb.Shutdown()\n\n\t// Close our last sub\n\tnatsUnsub(t, subA)\n\tnatsFlush(t, clientA)\n\n\t// Verify that they are gone before the 10 sec TTL\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 0 {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d, should be zero\", ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check that this all work in interest-only mode\n\tsb = runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tfooB, _ = sb.LookupAccount(\"$foo\")\n\tbarB, _ = sb.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooB.AddServiceExport(\"test.request\", nil)\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\t// We need at least a subscription on A otherwise when publishing\n\t// to subjects with no interest we would simply get an A-\n\tnatsSubSync(t, clientA, \"not.used\")\n\n\t// Create a client on B that will use account $foo\n\tbURL = fmt.Sprintf(\"nats://clientBFoo:password@127.0.0.1:%d\", ob.Port)\n\tclientB = natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\t// First flood with subjects that remote gw is not interested\n\t// so we switch to interest-only.\n\tfor i := 0; i < 1100; i++ {\n\t\tnatsPub(t, clientB, fmt.Sprintf(\"no.interest.%d\", i), []byte(\"hello\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\tcheckGWInterestOnlyMode(t, sb, \"A\", \"$foo\")\n\n\t// Go back to clientB on $bar.\n\tclientB.Close()\n\tbURL = fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob.Port)\n\tclientB = natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\tsubA = natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\tsubB = natsSubSync(t, clientB, \"reply\")\n\tnatsFlush(t, clientB)\n\n\t// Sine it is interest-only, B should receive an interest\n\t// on $foo test.request\n\tcheckGWInterestOnlyModeInterestOn(t, sb, \"A\", \"$foo\", \"test.request\")\n\n\t// Send the request from clientB on foo.request,\n\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\tnatsFlush(t, clientB)\n\n\t// Expect the request on A\n\tmsg, err := subA.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t}\n\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\tif msg.Reply == \"reply\" {\n\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n\n\t// Send reply\n\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\tnatsFlush(t, clientA)\n\n\tmsg, err = subB.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t}\n\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n}\n\nfunc ensureGWConnectTo(t *testing.T, s *Server, remoteGWName string, remoteGWServer *Server) {\n\tt.Helper()\n\tvar good bool\n\tfor i := 0; !good && (i < 3); i++ {\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tif s.numOutboundGateways() == 0 {\n\t\t\t\treturn fmt.Errorf(\"Still no gw outbound connection\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\togc := s.getOutboundGatewayConnection(remoteGWName)\n\t\togc.mu.Lock()\n\t\tname := ogc.opts.Name\n\t\tnc := ogc.nc\n\t\togc.mu.Unlock()\n\t\tif name != remoteGWServer.ID() {\n\t\t\trg := s.getRemoteGateway(remoteGWName)\n\t\t\tgoodURL := remoteGWServer.getGatewayURL()\n\t\t\trg.Lock()\n\t\t\tfor u := range rg.urls {\n\t\t\t\tif u != goodURL {\n\t\t\t\t\tdelete(rg.urls, u)\n\t\t\t\t}\n\t\t\t}\n\t\t\trg.Unlock()\n\t\t\tif nc != nil {\n\t\t\t\tnc.Close()\n\t\t\t}\n\t\t} else {\n\t\t\tgood = true\n\t\t}\n\t}\n\tif !good {\n\t\tt.Fatalf(\"Could not ensure that server connects to remote gateway %q at URL %q\",\n\t\t\tremoteGWName, remoteGWServer.getGatewayURL())\n\t}\n}\n\nfunc TestGatewayServiceImportComplexSetup(t *testing.T) {\n\t// This test will have following setup:\n\t//\n\t//\t\t\t\t\t\t |- responder (subs to \"$foo test.request\")\n\t// | (sends to \"$foo _R_.xxxx\")\n\t// route v\n\t// [A1]<----------------->[A2]\n\t// ^ |^ |\n\t// |gw| \\______gw________ gw|\n\t// | v \\ v\n\t// [B1]<----------------->[B2]\n\t// ^ route\n\t// |\n\t// |_ requestor (sends \"$bar foo.request reply\")\n\t//\n\n\t// Setup first A1 and B1 to ensure that they have GWs\n\t// connections as described above.\n\n\toa1 := testDefaultOptionsForGateway(\"A\")\n\tsetAccountUserPassInOptions(oa1, \"$foo\", \"clientA\", \"password\")\n\tsetAccountUserPassInOptions(oa1, \"$bar\", \"yyyyyyy\", \"password\")\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\tob1 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa1)\n\tsetAccountUserPassInOptions(ob1, \"$foo\", \"xxxxxxx\", \"password\")\n\tsetAccountUserPassInOptions(ob1, \"$bar\", \"clientB\", \"password\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sb1, 1, time.Second)\n\n\tob2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa1)\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tsetAccountUserPassInOptions(ob2, \"$foo\", \"clientBFoo\", \"password\")\n\tsetAccountUserPassInOptions(ob2, \"$bar\", \"clientB\", \"password\")\n\tob2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\n\twaitForInboundGateways(t, sa1, 2, time.Second)\n\twaitForInboundGateways(t, sb1, 1, time.Second)\n\twaitForInboundGateways(t, sb2, 0, time.Second)\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb2)\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sa1.ClusterAddr().Port))\n\tsetAccountUserPassInOptions(oa2, \"$foo\", \"clientA\", \"password\")\n\tsetAccountUserPassInOptions(oa2, \"$bar\", \"yyyyyyy\", \"password\")\n\toa2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\tensureGWConnectTo(t, sa2, \"B\", sb2)\n\n\tcheckClusterFormed(t, sa1, sa2)\n\tcheckClusterFormed(t, sb1, sb2)\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\twaitForOutboundGateways(t, sa2, 1, 2*time.Second)\n\n\twaitForInboundGateways(t, sa1, 2, time.Second)\n\twaitForInboundGateways(t, sb1, 1, time.Second)\n\twaitForInboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa2, 0, time.Second)\n\n\t// Verification that we have what we wanted\n\tc := sa2.getOutboundGatewayConnection(\"B\")\n\tif c == nil || c.opts.Name != sb2.ID() {\n\t\tt.Fatalf(\"A2 does not have outbound to B2\")\n\t}\n\tc = getInboundGatewayConnection(sa2, \"B\")\n\tif c != nil {\n\t\tt.Fatalf(\"Bad setup\")\n\t}\n\tc = sb2.getOutboundGatewayConnection(\"A\")\n\tif c == nil || c.opts.Name != sa1.ID() {\n\t\tt.Fatalf(\"B2 does not have outbound to A1\")\n\t}\n\tc = getInboundGatewayConnection(sb2, \"A\")\n\tif c == nil || c.opts.Name != sa2.ID() {\n\t\tt.Fatalf(\"Bad setup\")\n\t}\n\n\t// Ok, so now that we have proper setup, do actual test!\n\n\t// Get accounts\n\tfooA1, _ := sa1.LookupAccount(\"$foo\")\n\tbarA1, _ := sa1.LookupAccount(\"$bar\")\n\tfooA2, _ := sa2.LookupAccount(\"$foo\")\n\tbarA2, _ := sa2.LookupAccount(\"$bar\")\n\n\tfooB1, _ := sb1.LookupAccount(\"$foo\")\n\tbarB1, _ := sb1.LookupAccount(\"$bar\")\n\tfooB2, _ := sb2.LookupAccount(\"$foo\")\n\tbarB2, _ := sb2.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooA1.AddServiceExport(\"test.request\", nil)\n\tfooA2.AddServiceExport(\"test.request\", nil)\n\tfooB1.AddServiceExport(\"test.request\", nil)\n\tfooB2.AddServiceExport(\"test.request\", nil)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB1.AddServiceImport(fooB1, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\tif err := barB2.AddServiceImport(fooB2, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Same on A.\n\tif err := barA1.AddServiceImport(fooA1, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\tif err := barA2.AddServiceImport(fooA2, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to A2 and be the service endpoint and responder.\n\ta2URL := fmt.Sprintf(\"nats://clientA:password@127.0.0.1:%d\", oa2.Port)\n\tclientA := natsConnect(t, a2URL)\n\tdefer clientA.Close()\n\n\tsubA := natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\t// Now setup client B on B1 who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tb1URL := fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob1.Port)\n\tclientB := natsConnect(t, b1URL)\n\tdefer clientB.Close()\n\n\tsubB := natsSubSync(t, clientB, \"reply\")\n\tnatsFlush(t, clientB)\n\n\tvar msg *nats.Msg\n\tvar err error\n\tfor attempts := 1; attempts <= 2; attempts++ {\n\t\t// Send the request from clientB on foo.request,\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t\tnatsFlush(t, clientB)\n\n\t\t// Expect the request on A\n\t\tmsg, err = subA.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tif attempts == 1 {\n\t\t\t\t// Since we are in interestOnly mode, it is possible\n\t\t\t\t// that server B did not receive the subscription\n\t\t\t\t// interest yet, so try again.\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\tif msg.Reply == \"reply\" {\n\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t}\n\t}\n\t// Make sure we don't receive a second copy\n\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Received unexpected message: %v\", msg)\n\t}\n\n\t// Send reply\n\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\tnatsFlush(t, clientA)\n\n\tmsg, err = subB.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t}\n\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\t// Make sure we don't receive a second copy\n\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Received unexpected message: %v\", msg)\n\t}\n\n\tcheckSubs := func(t *testing.T, acc *Account, srvName string, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\tif ts := acc.TotalSubs(); ts != expected {\n\t\t\t\treturn fmt.Errorf(\"Number of subs is %d on acc=%s srv=%s, should be %v\", ts, acc.Name, srvName, expected)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckSubs(t, fooA1, \"A1\", 1)\n\tcheckSubs(t, barA1, \"A1\", 1)\n\tcheckSubs(t, fooA2, \"A2\", 1)\n\tcheckSubs(t, barA2, \"A2\", 1)\n\tcheckSubs(t, fooB1, \"B1\", 1)\n\tcheckSubs(t, barB1, \"B1\", 2)\n\tcheckSubs(t, fooB2, \"B2\", 1)\n\tcheckSubs(t, barB2, \"B2\", 2)\n\n\t// Speed up exiration\n\terr = fooA2.SetServiceExportResponseThreshold(\"test.request\", 10*time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\terr = fooB1.SetServiceExportResponseThreshold(\"test.request\", 10*time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t}\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t}\n\tnatsFlush(t, clientB)\n\n\t// Consume the requests, but don't reply to them...\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"subA did not receive request: %v\", err)\n\t\t}\n\t}\n\n\t// Unsubsribe all and ensure counts go to 0.\n\tnatsUnsub(t, subA)\n\tnatsFlush(t, clientA)\n\tnatsUnsub(t, subB)\n\tnatsFlush(t, clientB)\n\n\t// We should expire because ttl.\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif nr := len(fooA1.exports.responses); nr != 0 {\n\t\t\treturn fmt.Errorf(\"Number of responses is %d\", nr)\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckSubs(t, fooA1, \"A1\", 0)\n\tcheckSubs(t, fooA2, \"A2\", 0)\n\tcheckSubs(t, fooB1, \"B1\", 1)\n\tcheckSubs(t, fooB2, \"B2\", 1)\n\n\tcheckSubs(t, barA1, \"A1\", 1)\n\tcheckSubs(t, barA2, \"A2\", 1)\n\tcheckSubs(t, barB1, \"B1\", 1)\n\tcheckSubs(t, barB2, \"B2\", 1)\n\n\t// Check that this all work in interest-only mode.\n\n\t// We need at least a subscription on B2 otherwise when publishing\n\t// to subjects with no interest we would simply get an A-\n\tb2URL := fmt.Sprintf(\"nats://clientBFoo:password@127.0.0.1:%d\", ob2.Port)\n\tclientB2 := natsConnect(t, b2URL)\n\tdefer clientB2.Close()\n\tnatsSubSync(t, clientB2, \"not.used\")\n\tnatsFlush(t, clientB2)\n\n\t// Make A2 flood B2 with subjects that B2 is not interested in.\n\tfor i := 0; i < 1100; i++ {\n\t\tnatsPub(t, clientA, fmt.Sprintf(\"no.interest.%d\", i), []byte(\"hello\"))\n\t}\n\tnatsFlush(t, clientA)\n\t// Wait for B2 to switch to interest-only\n\tcheckGWInterestOnlyMode(t, sa2, \"B\", \"$foo\")\n\n\tsubA = natsSubSync(t, clientA, \"test.request\")\n\tnatsFlush(t, clientA)\n\n\tsubB = natsSubSync(t, clientB, \"reply\")\n\tnatsFlush(t, clientB)\n\n\tfor attempts := 1; attempts <= 2; attempts++ {\n\t\t// Send the request from clientB on foo.request,\n\t\tnatsPubReq(t, clientB, \"foo.request\", \"reply\", []byte(\"hi\"))\n\t\tnatsFlush(t, clientB)\n\n\t\t// Expect the request on A\n\t\tmsg, err = subA.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tif attempts == 1 {\n\t\t\t\t// Since we are in interestOnly mode, it is possible\n\t\t\t\t// that server B did not receive the subscription\n\t\t\t\t// interest yet, so try again.\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"test.request\" || string(msg.Data) != \"hi\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\tif msg.Reply == \"reply\" {\n\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t}\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n\n\t// Send reply\n\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\tnatsFlush(t, clientA)\n\n\tmsg, err = subB.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t}\n\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t}\n\n\t// Check for duplicate message\n\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t}\n}\n\nfunc TestGatewayServiceExportWithWildcards(t *testing.T) {\n\t// This test will have following setup:\n\t//\n\t//\t\t\t\t\t\t |- responder\n\t// |\n\t// route v\n\t// [A1]<----------------->[A2]\n\t// ^ |^ |\n\t// |gw| \\______gw________ gw|\n\t// | v \\ v\n\t// [B1]<----------------->[B2]\n\t// ^ route\n\t// |\n\t// |_ requestor\n\t//\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpublic bool\n\t}{\n\t\t{\n\t\t\tname: \"public\",\n\t\t\tpublic: true,\n\t\t},\n\t\t{\n\t\t\tname: \"private\",\n\t\t\tpublic: false,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\t// Setup first A1 and B1 to ensure that they have GWs\n\t\t\t// connections as described above.\n\n\t\t\toa1 := testDefaultOptionsForGateway(\"A\")\n\t\t\tsetAccountUserPassInOptions(oa1, \"$foo\", \"clientA\", \"password\")\n\t\t\tsetAccountUserPassInOptions(oa1, \"$bar\", \"yyyyyyy\", \"password\")\n\t\t\tsa1 := runGatewayServer(oa1)\n\t\t\tdefer sa1.Shutdown()\n\n\t\t\tob1 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa1)\n\t\t\tsetAccountUserPassInOptions(ob1, \"$foo\", \"xxxxxxx\", \"password\")\n\t\t\tsetAccountUserPassInOptions(ob1, \"$bar\", \"clientB\", \"password\")\n\t\t\tsb1 := runGatewayServer(ob1)\n\t\t\tdefer sb1.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\n\t\t\twaitForInboundGateways(t, sa1, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sb1, 1, time.Second)\n\n\t\t\tob2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa1)\n\t\t\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\t\t\tsetAccountUserPassInOptions(ob2, \"$foo\", \"clientBFoo\", \"password\")\n\t\t\tsetAccountUserPassInOptions(ob2, \"$bar\", \"clientB\", \"password\")\n\t\t\tob2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap\n\t\t\tsb2 := runGatewayServer(ob2)\n\t\t\tdefer sb2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\n\t\t\twaitForInboundGateways(t, sa1, 2, time.Second)\n\t\t\twaitForInboundGateways(t, sb1, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sb2, 0, time.Second)\n\n\t\t\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb2)\n\t\t\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sa1.ClusterAddr().Port))\n\t\t\tsetAccountUserPassInOptions(oa2, \"$foo\", \"clientA\", \"password\")\n\t\t\tsetAccountUserPassInOptions(oa2, \"$bar\", \"yyyyyyy\", \"password\")\n\t\t\toa2.gatewaysSolicitDelay = time.Nanosecond // 0 would be default, so nano to connect asap\n\t\t\tsa2 := runGatewayServer(oa2)\n\t\t\tdefer sa2.Shutdown()\n\n\t\t\tensureGWConnectTo(t, sa2, \"B\", sb2)\n\n\t\t\tcheckClusterFormed(t, sa1, sa2)\n\t\t\tcheckClusterFormed(t, sb1, sb2)\n\n\t\t\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sb1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sa2, 1, 2*time.Second)\n\n\t\t\twaitForInboundGateways(t, sa1, 2, time.Second)\n\t\t\twaitForInboundGateways(t, sb1, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sb2, 1, 2*time.Second)\n\t\t\twaitForInboundGateways(t, sa2, 0, time.Second)\n\n\t\t\t// Verification that we have what we wanted\n\t\t\tc := sa2.getOutboundGatewayConnection(\"B\")\n\t\t\tif c == nil || c.opts.Name != sb2.ID() {\n\t\t\t\tt.Fatalf(\"A2 does not have outbound to B2\")\n\t\t\t}\n\t\t\tc = getInboundGatewayConnection(sa2, \"B\")\n\t\t\tif c != nil {\n\t\t\t\tt.Fatalf(\"Bad setup\")\n\t\t\t}\n\t\t\tc = sb2.getOutboundGatewayConnection(\"A\")\n\t\t\tif c == nil || c.opts.Name != sa1.ID() {\n\t\t\t\tt.Fatalf(\"B2 does not have outbound to A1\")\n\t\t\t}\n\t\t\tc = getInboundGatewayConnection(sb2, \"A\")\n\t\t\tif c == nil || c.opts.Name != sa2.ID() {\n\t\t\t\tt.Fatalf(\"Bad setup\")\n\t\t\t}\n\n\t\t\t// Ok, so now that we have proper setup, do actual test!\n\n\t\t\t// Get accounts\n\t\t\tfooA1, _ := sa1.LookupAccount(\"$foo\")\n\t\t\tbarA1, _ := sa1.LookupAccount(\"$bar\")\n\t\t\tfooA2, _ := sa2.LookupAccount(\"$foo\")\n\t\t\tbarA2, _ := sa2.LookupAccount(\"$bar\")\n\n\t\t\tfooB1, _ := sb1.LookupAccount(\"$foo\")\n\t\t\tbarB1, _ := sb1.LookupAccount(\"$bar\")\n\t\t\tfooB2, _ := sb2.LookupAccount(\"$foo\")\n\t\t\tbarB2, _ := sb2.LookupAccount(\"$bar\")\n\n\t\t\tvar accs []*Account\n\t\t\t// Add in the service export for the requests.\n\t\t\tif !test.public {\n\t\t\t\taccs = []*Account{barA1}\n\t\t\t}\n\t\t\trequire_NoError(t, fooA1.AddServiceExport(\"ngs.update.*\", accs))\n\t\t\tif !test.public {\n\t\t\t\taccs = []*Account{barA2}\n\t\t\t}\n\t\t\trequire_NoError(t, fooA2.AddServiceExport(\"ngs.update.*\", accs))\n\t\t\tif !test.public {\n\t\t\t\taccs = []*Account{barB1}\n\t\t\t}\n\t\t\trequire_NoError(t, fooB1.AddServiceExport(\"ngs.update.*\", accs))\n\t\t\tif !test.public {\n\t\t\t\taccs = []*Account{barB2}\n\t\t\t}\n\t\t\trequire_NoError(t, fooB2.AddServiceExport(\"ngs.update.*\", accs))\n\n\t\t\t// Add import abilities to server B's bar account from foo.\n\t\t\tif err := barB1.AddServiceImport(fooB1, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\t\t\tif err := barB2.AddServiceImport(fooB2, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\t\t\t// Same on A.\n\t\t\tif err := barA1.AddServiceImport(fooA1, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\t\t\tif err := barA2.AddServiceImport(fooA2, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\n\t\t\t// clientA will be connected to A2 and be the service endpoint and responder.\n\t\t\ta2URL := fmt.Sprintf(\"nats://clientA:password@127.0.0.1:%d\", oa2.Port)\n\t\t\tclientA := natsConnect(t, a2URL)\n\t\t\tdefer clientA.Close()\n\n\t\t\tsubA := natsSubSync(t, clientA, \"ngs.update.$bar\")\n\t\t\tnatsFlush(t, clientA)\n\n\t\t\t// Now setup client B on B1 who will do a sub from account $bar\n\t\t\t// that should map account $foo's foo subject.\n\t\t\tb1URL := fmt.Sprintf(\"nats://clientB:password@127.0.0.1:%d\", ob1.Port)\n\t\t\tclientB := natsConnect(t, b1URL)\n\t\t\tdefer clientB.Close()\n\n\t\t\tsubB := natsSubSync(t, clientB, \"reply\")\n\t\t\tnatsFlush(t, clientB)\n\n\t\t\t// Ensure the subscription is known by the server we're connected to.\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\tvar msg *nats.Msg\n\t\t\tvar err error\n\t\t\tfor attempts := 1; attempts <= 2; attempts++ {\n\t\t\t\t// Send the request from clientB on foo.request,\n\t\t\t\tnatsPubReq(t, clientB, \"ngs.update\", \"reply\", []byte(\"hi\"))\n\t\t\t\tnatsFlush(t, clientB)\n\n\t\t\t\t// Expect the request on A\n\t\t\t\tmsg, err = subA.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif attempts == 1 {\n\t\t\t\t\t\t// Since we are in interestOnly mode, it is possible\n\t\t\t\t\t\t// that server B did not receive the subscription\n\t\t\t\t\t\t// interest yet, so try again.\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t\t\t}\n\t\t\t\tif msg.Subject != \"ngs.update.$bar\" || string(msg.Data) != \"hi\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t\t\t}\n\t\t\t\tif msg.Reply == \"reply\" {\n\t\t\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Make sure we don't receive a second copy\n\t\t\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Received unexpected message: %v\", msg)\n\t\t\t}\n\n\t\t\t// Send reply\n\t\t\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\t\t\tnatsFlush(t, clientA)\n\n\t\t\tmsg, err = subB.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t\t\t}\n\t\t\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t\t}\n\t\t\t// Make sure we don't receive a second copy\n\t\t\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Received unexpected message: %v\", msg)\n\t\t\t}\n\n\t\t\tcheckSubs := func(t *testing.T, acc *Account, srvName string, expected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t\tif ts := acc.TotalSubs(); ts != expected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Number of subs is %d on acc=%s srv=%s, should be %v\", ts, acc.Name, srvName, expected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tcheckSubs(t, fooA1, \"A1\", 1)\n\t\t\tcheckSubs(t, barA1, \"A1\", 1)\n\t\t\tcheckSubs(t, fooA2, \"A2\", 1)\n\t\t\tcheckSubs(t, barA2, \"A2\", 1)\n\t\t\tcheckSubs(t, fooB1, \"B1\", 1)\n\t\t\tcheckSubs(t, barB1, \"B1\", 2)\n\t\t\tcheckSubs(t, fooB2, \"B2\", 1)\n\t\t\tcheckSubs(t, barB2, \"B2\", 2)\n\n\t\t\t// Speed up expiration\n\t\t\terr = fooA1.SetServiceExportResponseThreshold(\"ngs.update.*\", 10*time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t\t\t}\n\t\t\terr = fooB1.SetServiceExportResponseThreshold(\"ngs.update.*\", 10*time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error setting response threshold: %v\", err)\n\t\t\t}\n\n\t\t\t// Send 100 requests from clientB on foo.request,\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tnatsPubReq(t, clientB, \"ngs.update\", \"reply\", []byte(\"hi\"))\n\t\t\t}\n\t\t\tnatsFlush(t, clientB)\n\n\t\t\t// Consume the requests, but don't reply to them...\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tif _, err := subA.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"subA did not receive request: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Unsubscribe all and ensure counts go to 0.\n\t\t\tnatsUnsub(t, subA)\n\t\t\tnatsFlush(t, clientA)\n\t\t\tnatsUnsub(t, subB)\n\t\t\tnatsFlush(t, clientB)\n\n\t\t\t// We should expire because ttl.\n\t\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tif nr := len(fooA1.exports.responses); nr != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Number of responses is %d\", nr)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tcheckSubs(t, fooA1, \"A1\", 0)\n\t\t\tcheckSubs(t, fooA2, \"A2\", 0)\n\t\t\tcheckSubs(t, fooB1, \"B1\", 1)\n\t\t\tcheckSubs(t, fooB2, \"B2\", 1)\n\n\t\t\tcheckSubs(t, barA1, \"A1\", 1)\n\t\t\tcheckSubs(t, barA2, \"A2\", 1)\n\t\t\tcheckSubs(t, barB1, \"B1\", 1)\n\t\t\tcheckSubs(t, barB2, \"B2\", 1)\n\n\t\t\t// Check that this all work in interest-only mode.\n\n\t\t\t// We need at least a subscription on B2 otherwise when publishing\n\t\t\t// to subjects with no interest we would simply get an A-\n\t\t\tb2URL := fmt.Sprintf(\"nats://clientBFoo:password@127.0.0.1:%d\", ob2.Port)\n\t\t\tclientB2 := natsConnect(t, b2URL)\n\t\t\tdefer clientB2.Close()\n\t\t\tnatsSubSync(t, clientB2, \"not.used\")\n\t\t\tnatsFlush(t, clientB2)\n\n\t\t\t// Make A2 flood B2 with subjects that B2 is not interested in.\n\t\t\tfor i := 0; i < 1100; i++ {\n\t\t\t\tnatsPub(t, clientA, fmt.Sprintf(\"no.interest.%d\", i), []byte(\"hello\"))\n\t\t\t}\n\t\t\tnatsFlush(t, clientA)\n\n\t\t\t// Wait for B2 to switch to interest-only\n\t\t\tcheckGWInterestOnlyMode(t, sa2, \"B\", \"$foo\")\n\n\t\t\tsubA = natsSubSync(t, clientA, \"ngs.update.*\")\n\t\t\tnatsFlush(t, clientA)\n\n\t\t\tsubB = natsSubSync(t, clientB, \"reply\")\n\t\t\tnatsFlush(t, clientB)\n\n\t\t\t// Ensure the subscription is known by the server we're connected to.\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\tfor attempts := 1; attempts <= 2; attempts++ {\n\t\t\t\t// Send the request from clientB on foo.request,\n\t\t\t\tnatsPubReq(t, clientB, \"ngs.update\", \"reply\", []byte(\"hi\"))\n\t\t\t\tnatsFlush(t, clientB)\n\n\t\t\t\t// Expect the request on A\n\t\t\t\tmsg, err = subA.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif attempts == 1 {\n\t\t\t\t\t\t// Since we are in interestOnly mode, it is possible\n\t\t\t\t\t\t// that server B did not receive the subscription\n\t\t\t\t\t\t// interest yet, so try again.\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tt.Fatalf(\"subA failed to get request: %v\", err)\n\t\t\t\t}\n\t\t\t\tif msg.Subject != \"ngs.update.$bar\" || string(msg.Data) != \"hi\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t\t\t}\n\t\t\t\tif msg.Reply == \"reply\" {\n\t\t\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check for duplicate message\n\t\t\tif msg, err := subA.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t\t}\n\n\t\t\t// Send reply\n\t\t\tnatsPub(t, clientA, msg.Reply, []byte(\"ok\"))\n\t\t\tnatsFlush(t, clientA)\n\n\t\t\tmsg, err = subB.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"subB failed to get reply: %v\", err)\n\t\t\t}\n\t\t\tif msg.Subject != \"reply\" || string(msg.Data) != \"ok\" {\n\t\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t\t}\n\n\t\t\t// Check for duplicate message\n\t\t\tif msg, err := subB.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// NOTE: if this fails for you and says only has <10 outbound, make sure ulimit for open files > 256.\nfunc TestGatewayMemUsage(t *testing.T) {\n\t// Try to clean up.\n\truntime.GC()\n\tvar m runtime.MemStats\n\truntime.ReadMemStats(&m)\n\tpta := m.TotalAlloc\n\n\to := testDefaultOptionsForGateway(\"A\")\n\ts := runGatewayServer(o)\n\tdefer s.Shutdown()\n\n\tvar servers []*Server\n\tservers = append(servers, s)\n\n\tnumServers := 10\n\tfor i := 0; i < numServers; i++ {\n\t\trn := fmt.Sprintf(\"RG_%d\", i+1)\n\t\to := testGatewayOptionsFromToWithServers(t, rn, \"A\", s)\n\t\ts := runGatewayServer(o)\n\t\tdefer s.Shutdown()\n\t\tservers = append(servers, s)\n\t}\n\n\t// Each server should have an outbound\n\tfor _, s := range servers {\n\t\twaitForOutboundGateways(t, s, numServers, 2*time.Second)\n\t}\n\t// The first started server should have numServers inbounds (since\n\t// they all connect to it).\n\twaitForInboundGateways(t, s, numServers, 2*time.Second)\n\n\t// Calculate in MB what we are using now.\n\tconst max = 50 * 1024 * 1024 // 50MB\n\truntime.ReadMemStats(&m)\n\tused := m.TotalAlloc - pta\n\tif used > max {\n\t\tt.Fatalf(\"Cluster using too much memory, expect < 50MB, got %dMB\", used/(1024*1024))\n\t}\n\n\tfor _, s := range servers {\n\t\ts.Shutdown()\n\t}\n}\n\nfunc TestGatewayMapReplyOnlyForRecentSub(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\t// Change s1's recent sub expiration default value\n\ts1.mu.Lock()\n\ts1.gateway.pasi.Lock()\n\ts1.gateway.recSubExp = 100 * time.Millisecond\n\ts1.gateway.pasi.Unlock()\n\ts1.mu.Unlock()\n\n\t// Setup a replier on s2\n\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", o2.Host, o2.Port))\n\tdefer nc2.Close()\n\terrCh := make(chan error, 1)\n\tnatsSub(t, nc2, \"foo\", func(m *nats.Msg) {\n\t\t// Send reply regardless..\n\t\tnc2.Publish(m.Reply, []byte(\"reply\"))\n\t\t// Check that reply given to application is not mapped.\n\t\tif !strings.HasPrefix(m.Reply, nats.InboxPrefix) {\n\t\t\terrCh <- fmt.Errorf(\"Reply expected to have normal inbox, got %v\", m.Reply)\n\t\t\treturn\n\t\t}\n\t\terrCh <- nil\n\t})\n\tnatsFlush(t, nc2)\n\tcheckExpectedSubs(t, 1, s2)\n\n\t// Create requestor on s1\n\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", o1.Host, o1.Port))\n\tdefer nc1.Close()\n\t// Send first request, reply should be mapped\n\tnc1.Request(\"foo\", []byte(\"msg1\"), time.Second)\n\t// Wait more than the recent sub expiration (that we have set to 100ms)\n\ttime.Sleep(200 * time.Millisecond)\n\t// Send second request (reply should not be mapped)\n\tnc1.Request(\"foo\", []byte(\"msg2\"), time.Second)\n\n\tselect {\n\tcase e := <-errCh:\n\t\tif e != nil {\n\t\t\tt.Fatal(e.Error())\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not get replies\")\n\t}\n}\n\ntype delayedWriteConn struct {\n\tsync.Mutex\n\tnet.Conn\n\tbytes [][]byte\n\tdelay bool\n\twg sync.WaitGroup\n}\n\nfunc (c *delayedWriteConn) Write(b []byte) (int, error) {\n\tc.Lock()\n\tdefer c.Unlock()\n\tif c.delay || len(c.bytes) > 0 {\n\t\tc.bytes = append(c.bytes, append([]byte(nil), b...))\n\t\tc.wg.Add(1)\n\t\tgo func() {\n\t\t\tdefer c.wg.Done()\n\t\t\tc.Lock()\n\t\t\tdefer c.Unlock()\n\t\t\tif c.delay {\n\t\t\t\tc.Unlock()\n\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\tc.Lock()\n\t\t\t}\n\t\t\tif len(c.bytes) > 0 {\n\t\t\t\tb = c.bytes[0]\n\t\t\t\tc.bytes = c.bytes[1:]\n\t\t\t\tc.Conn.Write(b)\n\t\t\t}\n\t\t}()\n\t\treturn len(b), nil\n\t}\n\treturn c.Conn.Write(b)\n}\n\n// This test uses a single account and makes sure that when\n// a reply subject is prefixed with $GR it comes back to\n// the origin cluster and delivered to proper reply subject\n// there, but also to subscribers on that reply subject\n// on the other cluster.\nfunc TestGatewaySendReplyAcrossGateways(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize int\n\t\tperacc bool\n\t}{\n\t\t{\"no pooling\", -1, false},\n\t\t{\"pooling\", 5, false},\n\t\t{\"per account\", 0, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tob := testDefaultOptionsForGateway(\"B\")\n\t\t\tob.Accounts = []*Account{NewAccount(\"ACC\")}\n\t\t\tob.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: ob.Accounts[0]}}\n\t\t\tsb := runGatewayServer(ob)\n\t\t\tdefer sb.Shutdown()\n\n\t\t\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\t\t\toa1.Cluster.PoolSize = test.poolSize\n\t\t\tif test.peracc {\n\t\t\t\toa1.Cluster.PinnedAccounts = []string{\"ACC\"}\n\t\t\t}\n\t\t\toa1.Accounts = []*Account{NewAccount(\"ACC\")}\n\t\t\toa1.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: oa1.Accounts[0]}}\n\t\t\tsa1 := runGatewayServer(oa1)\n\t\t\tdefer sa1.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, sb, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sb, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sa1, 1, time.Second)\n\n\t\t\t// Now start another server in cluster \"A\". This will allow us\n\t\t\t// to test the reply from cluster \"B\" coming back directly to\n\t\t\t// the server where the request originates, and indirectly through\n\t\t\t// route.\n\t\t\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\t\t\toa2.Cluster.PoolSize = test.poolSize\n\t\t\tif test.peracc {\n\t\t\t\toa2.Cluster.PinnedAccounts = []string{\"ACC\"}\n\t\t\t}\n\t\t\toa2.Accounts = []*Account{NewAccount(\"ACC\")}\n\t\t\toa2.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: oa2.Accounts[0]}}\n\t\t\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa1.Cluster.Host, oa1.Cluster.Port))\n\t\t\tsa2 := runGatewayServer(oa2)\n\t\t\tdefer sa2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\t\t\twaitForInboundGateways(t, sb, 2, time.Second)\n\t\t\tcheckClusterFormed(t, sa1, sa2)\n\n\t\t\treplySubj := \"bar\"\n\n\t\t\t// Setup a responder on sb\n\t\t\tncb := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@%s:%d\", ob.Host, ob.Port))\n\t\t\tdefer ncb.Close()\n\t\t\tnatsSub(t, ncb, \"foo\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"reply\"))\n\t\t\t})\n\t\t\t// Set a subscription on the reply subject on sb\n\t\t\tsubSB := natsSubSync(t, ncb, replySubj)\n\t\t\tnatsFlush(t, ncb)\n\t\t\tcheckExpectedSubs(t, 2, sb)\n\n\t\t\ttestReqReply := func(t *testing.T, host string, port int, createSubOnA bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tnca := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@%s:%d\", host, port))\n\t\t\t\tdefer nca.Close()\n\t\t\t\tif createSubOnA {\n\t\t\t\t\tsubSA := natsSubSync(t, nca, replySubj)\n\t\t\t\t\tnatsPubReq(t, nca, \"foo\", replySubj, []byte(\"hello\"))\n\t\t\t\t\tnatsNexMsg(t, subSA, time.Second)\n\t\t\t\t\t// Check for duplicates\n\t\t\t\t\tif _, err := subSA.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Received duplicate message on subSA!\")\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tnatsPubReq(t, nca, \"foo\", replySubj, []byte(\"hello\"))\n\t\t\t\t}\n\t\t\t\tnatsNexMsg(t, subSB, time.Second)\n\t\t\t\t// Check for duplicates\n\t\t\t\tif _, err := subSB.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\tt.Fatalf(\"Received duplicate message on subSB!\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Create requestor on sa1 to check for direct reply from GW:\n\t\t\ttestReqReply(t, oa1.Host, oa1.Port, true)\n\t\t\t// Wait for subscription to be gone...\n\t\t\tcheckExpectedSubs(t, 0, sa1)\n\t\t\t// Now create requestor on sa2, it will receive reply through sa1.\n\t\t\ttestReqReply(t, oa2.Host, oa2.Port, true)\n\t\t\tcheckExpectedSubs(t, 0, sa1)\n\t\t\tcheckExpectedSubs(t, 0, sa2)\n\n\t\t\t// Now issue requests but without any interest in the requestor's\n\t\t\t// origin cluster and make sure the other cluster gets the reply.\n\t\t\ttestReqReply(t, oa1.Host, oa1.Port, false)\n\t\t\ttestReqReply(t, oa2.Host, oa2.Port, false)\n\n\t\t\t// There is a possible race between sa2 sending the RS+ for the\n\t\t\t// subscription on the reply subject, and the GW reply making it\n\t\t\t// to sa1 before the RS+ is processed there.\n\t\t\t// We are going to force this race by making the route connection\n\t\t\t// block as needed.\n\n\t\t\tacc, _ := sa2.LookupAccount(\"ACC\")\n\t\t\tacc.mu.RLock()\n\t\t\tapi := acc.routePoolIdx\n\t\t\tacc.mu.RUnlock()\n\n\t\t\tvar route *client\n\t\t\tsa2.mu.Lock()\n\t\t\tif test.peracc {\n\t\t\t\tif conns, ok := sa2.accRoutes[\"ACC\"]; ok {\n\t\t\t\t\tfor _, r := range conns {\n\t\t\t\t\t\troute = r\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if test.poolSize > 0 {\n\t\t\t\tsa2.forEachRoute(func(r *client) {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tif r.route.poolIdx == api {\n\t\t\t\t\t\troute = r\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t})\n\t\t\t} else if r := getFirstRoute(sa2); r != nil {\n\t\t\t\troute = r\n\t\t\t}\n\t\t\tsa2.mu.Unlock()\n\t\t\troute.mu.Lock()\n\t\t\trouteConn := &delayedWriteConn{\n\t\t\t\tConn: route.nc,\n\t\t\t\twg: sync.WaitGroup{},\n\t\t\t}\n\t\t\troute.nc = routeConn\n\t\t\troute.mu.Unlock()\n\n\t\t\tdelayRoute := func() {\n\t\t\t\trouteConn.Lock()\n\t\t\t\trouteConn.delay = true\n\t\t\t\trouteConn.Unlock()\n\t\t\t}\n\t\t\tstopDelayRoute := func() {\n\t\t\t\trouteConn.Lock()\n\t\t\t\trouteConn.delay = false\n\t\t\t\twg := &routeConn.wg\n\t\t\t\trouteConn.Unlock()\n\t\t\t\twg.Wait()\n\t\t\t}\n\n\t\t\tdelayRoute()\n\t\t\ttestReqReply(t, oa2.Host, oa2.Port, true)\n\t\t\tstopDelayRoute()\n\n\t\t\t// Same test but now we have a local interest on the reply subject\n\t\t\t// on sa1 to make sure that interest there does not prevent sending\n\t\t\t// the RMSG to sa2, which is the origin of the request.\n\t\t\tcheckExpectedSubs(t, 0, sa1)\n\t\t\tcheckExpectedSubs(t, 0, sa2)\n\t\t\tnca1 := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@%s:%d\", oa1.Host, oa1.Port))\n\t\t\tdefer nca1.Close()\n\t\t\tsubSA1 := natsSubSync(t, nca1, replySubj)\n\t\t\tnatsFlush(t, nca1)\n\t\t\tcheckExpectedSubs(t, 1, sa1)\n\t\t\tcheckExpectedSubs(t, 1, sa2)\n\n\t\t\tdelayRoute()\n\t\t\ttestReqReply(t, oa2.Host, oa2.Port, true)\n\t\t\tstopDelayRoute()\n\n\t\t\tnatsNexMsg(t, subSA1, time.Second)\n\t\t})\n\t}\n}\n\n// This test will have a requestor on cluster A and responder\n// on cluster B, but when the responder sends the response,\n// it will also have a reply subject to receive a response\n// for the response.\nfunc TestGatewayPingPongReplyAcrossGateways(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tob.Accounts = []*Account{NewAccount(\"ACC\")}\n\tob.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: ob.Accounts[0]}}\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa1.Accounts = []*Account{NewAccount(\"ACC\")}\n\toa1.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: oa1.Accounts[0]}}\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\n\t// Now start another server in cluster \"A\". This will allow us\n\t// to test the reply from cluster \"B\" coming back directly to\n\t// the server where the request originates, and indirectly through\n\t// route.\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa2.Accounts = []*Account{NewAccount(\"ACC\")}\n\toa2.Users = []*User{{Username: \"user\", Password: \"pwd\", Account: oa2.Accounts[0]}}\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa1.Cluster.Host, oa1.Cluster.Port))\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\twaitForInboundGateways(t, sb, 2, time.Second)\n\tcheckClusterFormed(t, sa1, sa2)\n\n\t// Setup a responder on sb\n\tncb := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@%s:%d\", ob.Host, ob.Port))\n\tdefer ncb.Close()\n\tsbReplySubj := \"sbreply\"\n\tsubSB := natsSubSync(t, ncb, sbReplySubj)\n\tnatsSub(t, ncb, \"foo\", func(m *nats.Msg) {\n\t\tncb.PublishRequest(m.Reply, sbReplySubj, []byte(\"sb reply\"))\n\t})\n\tnatsFlush(t, ncb)\n\tcheckExpectedSubs(t, 2, sb)\n\n\ttestReqReply := func(t *testing.T, host string, port int) {\n\t\tt.Helper()\n\t\tnca := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@%s:%d\", host, port))\n\t\tdefer nca.Close()\n\t\tmsg, err := nca.Request(\"foo\", []byte(\"sa request\"), time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Did not get response: %v\", err)\n\t\t}\n\t\t// Check response from sb, it should have content \"sb reply\" and\n\t\t// reply subject should not have GW prefix\n\t\tif string(msg.Data) != \"sb reply\" || msg.Reply != sbReplySubj {\n\t\t\tt.Fatalf(\"Unexpected message from sb: %+v\", msg)\n\t\t}\n\t\t// Now send our own reply:\n\t\tnca.Publish(msg.Reply, []byte(\"sa reply\"))\n\t\t// And make sure that subS2 receives it...\n\t\tmsg = natsNexMsg(t, subSB, time.Second)\n\t\tif string(msg.Data) != \"sa reply\" || msg.Reply != _EMPTY_ {\n\t\t\tt.Fatalf(\"Unexpected message from sa: %v\", msg)\n\t\t}\n\t}\n\t// Create requestor on sa1 to check for direct reply from GW:\n\ttestReqReply(t, oa1.Host, oa1.Port)\n\t// Now from sa2 to see reply coming from route (sa1)\n\ttestReqReply(t, oa2.Host, oa2.Port)\n}\n\n// Similar to TestGatewaySendReplyAcrossGateways, but this time\n// with service import.\nfunc TestGatewaySendReplyAcrossGatewaysServiceImport(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsetAccountUserPassInOptions(ob, \"$foo\", \"clientBFoo\", \"password\")\n\tsetAccountUserPassInOptions(ob, \"$bar\", \"clientBBar\", \"password\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa1.Cluster.PoolSize = 1\n\tsetAccountUserPassInOptions(oa1, \"$foo\", \"clientAFoo\", \"password\")\n\tsetAccountUserPassInOptions(oa1, \"$bar\", \"clientABar\", \"password\")\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa2.Cluster.PoolSize = 1\n\tsetAccountUserPassInOptions(oa2, \"$foo\", \"clientAFoo\", \"password\")\n\tsetAccountUserPassInOptions(oa2, \"$bar\", \"clientABar\", \"password\")\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa1.Cluster.Host, oa1.Cluster.Port))\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\toa3 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa3.Cluster.PoolSize = 1\n\tsetAccountUserPassInOptions(oa3, \"$foo\", \"clientAFoo\", \"password\")\n\tsetAccountUserPassInOptions(oa3, \"$bar\", \"clientABar\", \"password\")\n\toa3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa1.Cluster.Host, oa1.Cluster.Port))\n\tsa3 := runGatewayServer(oa3)\n\tdefer sa3.Shutdown()\n\n\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\twaitForOutboundGateways(t, sa3, 1, time.Second)\n\twaitForInboundGateways(t, sb, 3, time.Second)\n\tcheckClusterFormed(t, sa1, sa2, sa3)\n\n\t// Setup account on B\n\tfooB, _ := sb.LookupAccount(\"$foo\")\n\t// Add in the service export for the requests. Make it public.\n\tfooB.AddServiceExport(\"foo.request\", nil)\n\n\t// Setup accounts on sa1, sa2 and sa3\n\tsetupAccsOnA := func(s *Server) {\n\t\t// Get accounts\n\t\tfooA, _ := s.LookupAccount(\"$foo\")\n\t\tbarA, _ := s.LookupAccount(\"$bar\")\n\t\t// Add in the service export for the requests. Make it public.\n\t\tfooA.AddServiceExport(\"foo.request\", nil)\n\t\t// Add import abilities to server A's bar account from foo.\n\t\tif err := barA.AddServiceImport(fooA, \"bar.request\", \"foo.request\"); err != nil {\n\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t}\n\t}\n\tsetupAccsOnA(sa1)\n\tsetupAccsOnA(sa2)\n\tsetupAccsOnA(sa3)\n\n\t// clientB will be connected to sb and be the service endpoint and responder.\n\tbURL := fmt.Sprintf(\"nats://clientBFoo:password@127.0.0.1:%d\", ob.Port)\n\tclientBFoo := natsConnect(t, bURL)\n\tdefer clientBFoo.Close()\n\tsubBFoo := natsSubSync(t, clientBFoo, \"foo.request\")\n\tnatsFlush(t, clientBFoo)\n\n\t// Create another client on B for account $bar that will listen to\n\t// the reply subject.\n\tbURL = fmt.Sprintf(\"nats://clientBBar:password@127.0.0.1:%d\", ob.Port)\n\tclientBBar := natsConnect(t, bURL)\n\tdefer clientBBar.Close()\n\treplySubj := \"reply\"\n\tsubBReply := natsSubSync(t, clientBBar, replySubj)\n\tnatsFlush(t, clientBBar)\n\n\ttestServiceImport := func(t *testing.T, host string, port int) {\n\t\tt.Helper()\n\t\tbURL := fmt.Sprintf(\"nats://clientABar:password@%s:%d\", host, port)\n\t\tclientABar := natsConnect(t, bURL)\n\t\tdefer clientABar.Close()\n\t\tsubAReply := natsSubSync(t, clientABar, replySubj)\n\t\tnatsFlush(t, clientABar)\n\n\t\t// Send the request from clientA on bar.request, which\n\t\t// will be translated to foo.request and sent over.\n\t\tnatsPubReq(t, clientABar, \"bar.request\", replySubj, []byte(\"hi\"))\n\t\tnatsFlush(t, clientABar)\n\n\t\t// Expect the request to be received on subAFoo\n\t\tmsg, err := subBFoo.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"subBFoo failed to get request: %v\", err)\n\t\t}\n\t\tif msg.Subject != \"foo.request\" || string(msg.Data) != \"hi\" {\n\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t}\n\t\tif msg.Reply == replySubj {\n\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t}\n\n\t\t// Check for duplicate message\n\t\tif msg, err := subBFoo.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected msg: %v\", msg)\n\t\t}\n\n\t\t// Send reply\n\t\tnatsPub(t, clientBFoo, msg.Reply, []byte(\"ok-42\"))\n\t\tnatsFlush(t, clientBFoo)\n\n\t\t// Now check that the subscription on the reply receives the message...\n\t\tcheckReply := func(t *testing.T, sub *nats.Subscription) {\n\t\t\tt.Helper()\n\t\t\tmsg, err = sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"sub failed to get reply: %v\", err)\n\t\t\t}\n\t\t\tif msg.Subject != replySubj || string(msg.Data) != \"ok-42\" {\n\t\t\t\tt.Fatalf(\"Unexpected message: %v\", msg)\n\t\t\t}\n\t\t}\n\t\t// Check subscription on A (where the request originated)\n\t\tcheckReply(t, subAReply)\n\t\t// And the subscription on B (where the responder is located)\n\t\tcheckReply(t, subBReply)\n\t}\n\n\t// We check the service import with GW working ok with either\n\t// direct connection between the responder's server to the\n\t// requestor's server and also through routes.\n\ttestServiceImport(t, oa1.Host, oa1.Port)\n\ttestServiceImport(t, oa2.Host, oa2.Port)\n\t// sa1 is the one receiving the reply from GW between B and A.\n\t// Check that the server routes directly to the server\n\t// with the interest.\n\tcheckRoute := func(t *testing.T, s *Server, expected int64) {\n\t\tt.Helper()\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\ts.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\tif r.route.remoteID != sa1.ID() {\n\t\t\t\tr.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tinMsgs := atomic.LoadInt64(&r.inMsgs)\n\t\t\tr.mu.Unlock()\n\t\t\tif inMsgs != expected {\n\t\t\t\tt.Fatalf(\"Expected %v incoming msgs, got %v\", expected, inMsgs)\n\t\t\t}\n\t\t})\n\t}\n\t// Wait a bit to make sure that we don't have a loop that\n\t// cause messages to be routed more than needed.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckRoute(t, sa2, 1)\n\tcheckRoute(t, sa3, 0)\n\n\ttestServiceImport(t, oa3.Host, oa3.Port)\n\t// Wait a bit to make sure that we don't have a loop that\n\t// cause messages to be routed more than needed.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckRoute(t, sa2, 1)\n\tcheckRoute(t, sa3, 1)\n}\n\nfunc TestGatewayClientsDontReceiveMsgsOnGWPrefix(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Setup a responder on sb\n\tncb := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port))\n\tdefer ncb.Close()\n\tnatsSub(t, ncb, \"foo\", func(m *nats.Msg) {\n\t\tif strings.HasPrefix(m.Reply, gwReplyPrefix) {\n\t\t\tm.Respond([]byte(fmt.Sprintf(\"-ERR: received request with mapped reply subject %q\", m.Reply)))\n\t\t} else {\n\t\t\tm.Respond([]byte(\"+OK: reply\"))\n\t\t}\n\t})\n\t// And create a sub on \">\" that should not get the $GR reply.\n\tsubSB := natsSubSync(t, ncb, \">\")\n\tnatsFlush(t, ncb)\n\tcheckExpectedSubs(t, 2, sb)\n\n\tnca := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\tdefer nca.Close()\n\tmsg, err := nca.Request(\"foo\", []byte(\"request\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Did not get response: %v\", err)\n\t}\n\tif string(msg.Data) != \"+OK: reply\" {\n\t\tt.Fatalf(\"Error from responder: %q\", msg.Data)\n\t}\n\n\t// subSB would have also received the request, so drop that one.\n\tmsg = natsNexMsg(t, subSB, time.Second)\n\tif string(msg.Data) != \"request\" {\n\t\tt.Fatalf(\"Wrong request: %q\", msg.Data)\n\t}\n\t// Once sa gets the direct reply, it should resend the reply\n\t// with normal subject. So subSB should get the message with\n\t// a subject that does not start with $GNR prefix.\n\tmsg = natsNexMsg(t, subSB, time.Second)\n\tif string(msg.Data) != \"+OK: reply\" || strings.HasPrefix(msg.Subject, gwReplyPrefix) {\n\t\tt.Fatalf(\"Unexpected message from sa: %v\", msg)\n\t}\n\t// Check no more message...\n\tif m, err := subSB.NextMsg(100 * time.Millisecond); m != nil || err == nil {\n\t\tt.Fatalf(\"Expected only 1 message, got %+v\", m)\n\t}\n}\n\nfunc TestGatewayNoAccInterestThenQSubThenRegularSub(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Connect on A and send a message\n\tncA := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\tdefer ncA.Close()\n\tnatsPub(t, ncA, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, ncA)\n\n\t// expect an A- on return\n\tgwb := sa.getOutboundGatewayConnection(\"B\")\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, true, time.Second)\n\n\t// Create a connection o B, and create a queue sub first\n\tncB := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port))\n\tdefer ncB.Close()\n\tqsub := natsQueueSubSync(t, ncB, \"bar\", \"queue\")\n\tnatsFlush(t, ncB)\n\n\t// A should have received a queue interest\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"bar\", 1, time.Second)\n\n\t// Now on B, create a regular sub\n\tsub := natsSubSync(t, ncB, \"baz\")\n\tnatsFlush(t, ncB)\n\n\t// From A now, produce a message on each subject and\n\t// expect both subs to receive their message.\n\tmsgForQSub := []byte(\"msg_qsub\")\n\tnatsPub(t, ncA, \"bar\", msgForQSub)\n\tnatsFlush(t, ncA)\n\n\tif msg := natsNexMsg(t, qsub, time.Second); !bytes.Equal(msgForQSub, msg.Data) {\n\t\tt.Fatalf(\"Expected msg for queue sub to be %q, got %q\", msgForQSub, msg.Data)\n\t}\n\n\t// Publish for the regular sub\n\tmsgForSub := []byte(\"msg_sub\")\n\tnatsPub(t, ncA, \"baz\", msgForSub)\n\tnatsFlush(t, ncA)\n\n\tif msg := natsNexMsg(t, sub, time.Second); !bytes.Equal(msgForSub, msg.Data) {\n\t\tt.Fatalf(\"Expected msg for sub to be %q, got %q\", msgForSub, msg.Data)\n\t}\n}\n\n// Similar to TestGatewayNoAccInterestThenQSubThenRegularSub but simulate\n// older incorrect behavior.\nfunc TestGatewayHandleUnexpectedASubUnsub(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Connect on A and send a message\n\tncA := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\tdefer ncA.Close()\n\tnatsPub(t, ncA, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, ncA)\n\n\t// expect an A- on return\n\tgwb := sa.getOutboundGatewayConnection(\"B\")\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, true, time.Second)\n\n\t// Create a connection o B, and create a queue sub first\n\tncB := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port))\n\tdefer ncB.Close()\n\tqsub := natsQueueSubSync(t, ncB, \"bar\", \"queue\")\n\tnatsFlush(t, ncB)\n\n\t// A should have received a queue interest\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"bar\", 1, time.Second)\n\n\t// Now on B, create a regular sub\n\tsub := natsSubSync(t, ncB, \"baz\")\n\tnatsFlush(t, ncB)\n\t// and reproduce old, wrong, behavior that would have resulted in sending an A-\n\tgwA := getInboundGatewayConnection(sb, \"A\")\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A- $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\n\t// From A now, produce a message on each subject and\n\t// expect both subs to receive their message.\n\tmsgForQSub := []byte(\"msg_qsub\")\n\tnatsPub(t, ncA, \"bar\", msgForQSub)\n\tnatsFlush(t, ncA)\n\n\tif msg := natsNexMsg(t, qsub, time.Second); !bytes.Equal(msgForQSub, msg.Data) {\n\t\tt.Fatalf(\"Expected msg for queue sub to be %q, got %q\", msgForQSub, msg.Data)\n\t}\n\n\t// Publish for the regular sub\n\tmsgForSub := []byte(\"msg_sub\")\n\tnatsPub(t, ncA, \"baz\", msgForSub)\n\tnatsFlush(t, ncA)\n\n\tif msg := natsNexMsg(t, sub, time.Second); !bytes.Equal(msgForSub, msg.Data) {\n\t\tt.Fatalf(\"Expected msg for sub to be %q, got %q\", msgForSub, msg.Data)\n\t}\n\n\t// Remove all subs on B.\n\tqsub.Unsubscribe()\n\tsub.Unsubscribe()\n\tncB.Flush()\n\n\t// Produce a message from A expect A-\n\tnatsPub(t, ncA, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, ncA)\n\n\t// expect an A- on return\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, true, time.Second)\n\n\t// Simulate B sending another A-, on A account no interest should remain same.\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A- $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, true, time.Second)\n\n\t// Create a queue sub on B\n\tqsub = natsQueueSubSync(t, ncB, \"bar\", \"queue\")\n\tnatsFlush(t, ncB)\n\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"bar\", 1, time.Second)\n\n\t// Make B send an A+ and verify that we sitll have the registered qsub interest\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A+ $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\n\t// Give a chance to A to possibly misbehave when receiving this proto\n\ttime.Sleep(250 * time.Millisecond)\n\t// Now check interest is still there\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"bar\", 1, time.Second)\n\n\tqsub.Unsubscribe()\n\tnatsFlush(t, ncB)\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", globalAccountName, \"bar\", 0, time.Second)\n\n\t// Send A-, server A should set entry to nil\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A- $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, true, time.Second)\n\n\t// Send A+ and entry should be removed since there is no longer reason to\n\t// keep the entry.\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A+ $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, false, time.Second)\n\n\t// Last A+ should not change because account already removed from map.\n\tgwA.mu.Lock()\n\tgwA.enqueueProto([]byte(\"A+ $G\\r\\n\"))\n\tgwA.mu.Unlock()\n\tcheckForAccountNoInterest(t, gwb, globalAccountName, false, time.Second)\n}\n\ntype captureGWInterestSwitchLogger struct {\n\tDummyLogger\n\timss []string\n}\n\nfunc (l *captureGWInterestSwitchLogger) Debugf(format string, args ...any) {\n\tl.Lock()\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, fmt.Sprintf(\"switching account %q to %s mode\", globalAccountName, InterestOnly)) ||\n\t\tstrings.Contains(msg, fmt.Sprintf(\"switching account %q to %s mode complete\", globalAccountName, InterestOnly)) {\n\t\tl.imss = append(l.imss, msg)\n\t}\n\tl.Unlock()\n}\n\nfunc TestGatewayLogAccountInterestModeSwitch(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tlogB := &captureGWInterestSwitchLogger{}\n\tsb.SetLogger(logB, true, true)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tlogA := &captureGWInterestSwitchLogger{}\n\tsa.SetLogger(logA, true, true)\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\tncB := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.Port))\n\tdefer ncB.Close()\n\tnatsSubSync(t, ncB, \"foo\")\n\tnatsFlush(t, ncB)\n\n\tncA := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", oa.Port))\n\tdefer ncA.Close()\n\tfor i := 0; i < gatewayMaxRUnsubBeforeSwitch+10; i++ {\n\t\tsubj := fmt.Sprintf(\"bar.%d\", i)\n\t\tnatsPub(t, ncA, subj, []byte(\"hello\"))\n\t}\n\tnatsFlush(t, ncA)\n\n\tgwA := getInboundGatewayConnection(sb, \"A\")\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tmode := Optimistic\n\t\tgwA.mu.Lock()\n\t\te := gwA.gw.insim[globalAccountName]\n\t\tif e != nil {\n\t\t\tmode = e.mode\n\t\t}\n\t\tgwA.mu.Unlock()\n\t\tif mode != InterestOnly {\n\t\t\treturn fmt.Errorf(\"not switched yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckGWInterestOnlyMode(t, sa, \"B\", globalAccountName)\n\n\tcheckLog := func(t *testing.T, l *captureGWInterestSwitchLogger) {\n\t\tt.Helper()\n\t\tl.Lock()\n\t\tlogs := append([]string(nil), l.imss...)\n\t\tl.Unlock()\n\n\t\tif len(logs) != 2 {\n\t\t\tt.Fatalf(\"Expected 2 logs about switching to interest-only, got %v\", logs)\n\t\t}\n\t\tif !strings.Contains(logs[0], \"switching account\") {\n\t\t\tt.Fatalf(\"First log statement should have been about switching, got %v\", logs[0])\n\t\t}\n\t\tif !strings.Contains(logs[1], \"complete\") {\n\t\t\tt.Fatalf(\"Second log statement should have been about having switched, got %v\", logs[1])\n\t\t}\n\t}\n\tcheckLog(t, logB)\n\tcheckLog(t, logA)\n\n\t// Clear log of server B\n\tlogB.Lock()\n\tlogB.imss = nil\n\tlogB.Unlock()\n\n\t// Force a switch on B to inbound gateway from A and make sure that it is\n\t// a no-op since this gateway connection has already been switched.\n\tsb.switchAccountToInterestMode(globalAccountName)\n\n\tlogB.Lock()\n\tdidSwitch := len(logB.imss) > 0\n\tlogB.Unlock()\n\tif didSwitch {\n\t\tt.Fatalf(\"Attempted to switch while it was already in interest mode only\")\n\t}\n}\n\nfunc TestGatewayAccountInterestModeSwitchOnlyOncePerAccount(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tlogB := &captureGWInterestSwitchLogger{}\n\tsb.SetLogger(logB, true, true)\n\n\tnc := natsConnect(t, sb.ClientURL())\n\tdefer nc.Close()\n\tnatsSubSync(t, nc, \"foo\")\n\tnatsQueueSubSync(t, nc, \"bar\", \"baz\")\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\twg := sync.WaitGroup{}\n\ttotal := 20\n\twg.Add(total)\n\tfor i := 0; i < total; i++ {\n\t\tgo func() {\n\t\t\tsb.switchAccountToInterestMode(globalAccountName)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\twg.Wait()\n\ttime.Sleep(50 * time.Millisecond)\n\tlogB.Lock()\n\tnl := len(logB.imss)\n\tlogB.Unlock()\n\t// There should be a trace for switching and when switch is complete\n\tif nl != 2 {\n\t\tt.Fatalf(\"Attempted to switch account too many times, number lines=%v\", nl)\n\t}\n}\n\nfunc TestGatewaySingleOutbound(t *testing.T) {\n\tl, err := natsListen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on listen: %v\", err)\n\t}\n\tdefer l.Close()\n\tport := l.Addr().(*net.TCPAddr).Port\n\n\toa := testGatewayOptionsFromToWithTLS(t, \"A\", \"B\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", port)})\n\toa.Gateway.TLSTimeout = 0.1\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\t// Wait a bit for reconnections\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Now prepare gateway B to take place of the bare listener.\n\tob := testGatewayOptionsWithTLS(t, \"B\")\n\t// There is a risk that when stopping the listener and starting\n\t// the actual server, that port is being reused by some other process.\n\tob.Gateway.Port = port\n\tl.Close()\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\t// To make sure that we don't fail, bump the TLSTimeout now.\n\tcfg := sa.getRemoteGateway(\"B\")\n\tcfg.Lock()\n\tcfg.TLSTimeout = 2.0\n\tcfg.Unlock()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\tsa.gateway.Lock()\n\tlm := len(sa.gateway.out)\n\tsa.gateway.Unlock()\n\tif lm != 1 {\n\t\tt.Fatalf(\"Expected 1 outbound, got %v\", lm)\n\t}\n}\n\nfunc TestGatewayReplyMapTracking(t *testing.T) {\n\t// Increase the recSubExp value on servers so we have time\n\t// to check the replies mapping structures.\n\tsubExp := 400 * time.Millisecond\n\tsetRecSub := func(s *Server) {\n\t\ts.gateway.pasi.Lock()\n\t\ts.gateway.recSubExp = subExp\n\t\ts.gateway.pasi.Unlock()\n\t}\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\tsetRecSub(sb)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\tsetRecSub(sa)\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\tncb := natsConnect(t, sb.ClientURL())\n\tdefer ncb.Close()\n\tcount := 0\n\ttotal := 100\n\tch := make(chan bool, 1)\n\tnatsSub(t, ncb, \"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"reply\"))\n\t\tif count++; count == total {\n\t\t\tch <- true\n\t\t}\n\t})\n\tnatsFlush(t, ncb)\n\n\tvar bc *client\n\tsb.mu.Lock()\n\tfor _, c := range sb.clients {\n\t\tbc = c\n\t\tbreak\n\t}\n\tsb.mu.Unlock()\n\n\tnca := natsConnect(t, sa.ClientURL())\n\tdefer nca.Close()\n\n\treplySub := natsSubSync(t, nca, \"bar.>\")\n\tfor i := 0; i < total; i++ {\n\t\tnca.PublishRequest(\"foo\", fmt.Sprintf(\"bar.%d\", i), []byte(\"request\"))\n\t}\n\n\twaitCh(t, ch, \"Did not receive all requests\")\n\n\tcheck := func(t *testing.T, expectedIndicator int32, expectLenMap int, expectedSrvMapEmpty bool) {\n\t\tt.Helper()\n\t\tbc.mu.Lock()\n\t\tmapIndicator := atomic.LoadInt32(&bc.gwReplyMapping.check)\n\t\tvar lenMap int\n\t\tif bc.gwReplyMapping.mapping != nil {\n\t\t\tlenMap = len(bc.gwReplyMapping.mapping)\n\t\t}\n\t\tbc.mu.Unlock()\n\t\tif mapIndicator != expectedIndicator {\n\t\t\tt.Fatalf(\"Client should map indicator should be %v, got %v\", expectedIndicator, mapIndicator)\n\t\t}\n\t\tif lenMap != expectLenMap {\n\t\t\tt.Fatalf(\"Client map should have %v entries, got %v\", expectLenMap, lenMap)\n\t\t}\n\t\tsrvMapEmpty := true\n\t\tsb.gwrm.m.Range(func(_, _ any) bool {\n\t\t\tsrvMapEmpty = false\n\t\t\treturn false\n\t\t})\n\t\tif srvMapEmpty != expectedSrvMapEmpty {\n\t\t\tt.Fatalf(\"Expected server map to be empty=%v, got %v\", expectedSrvMapEmpty, srvMapEmpty)\n\t\t}\n\t}\n\t// Check that indicator is set and that there \"total\" entries in the map\n\t// and that srv map is not empty\n\tcheck(t, 1, total, false)\n\n\t// Receive all replies\n\tfor i := 0; i < total; i++ {\n\t\tnatsNexMsg(t, replySub, time.Second)\n\t}\n\n\t// Wait until entries expire\n\ttime.Sleep(2*subExp + 100*time.Millisecond)\n\n\t// Now check again.\n\tcheck(t, 0, 0, true)\n}\n\nfunc TestGatewayNoAccountUnsubWhenServiceReplyInUse(t *testing.T) {\n\toa := testDefaultOptionsForGateway(\"A\")\n\tsetAccountUserPassInOptions(oa, \"$foo\", \"clientFoo\", \"password\")\n\tsetAccountUserPassInOptions(oa, \"$bar\", \"clientBar\", \"password\")\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tob := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\tsetAccountUserPassInOptions(ob, \"$foo\", \"clientFoo\", \"password\")\n\tsetAccountUserPassInOptions(ob, \"$bar\", \"clientBar\", \"password\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, time.Second)\n\twaitForOutboundGateways(t, sb, 1, time.Second)\n\twaitForInboundGateways(t, sa, 1, time.Second)\n\twaitForInboundGateways(t, sb, 1, time.Second)\n\n\t// Get accounts\n\tfooA, _ := sa.LookupAccount(\"$foo\")\n\tbarA, _ := sa.LookupAccount(\"$bar\")\n\tfooB, _ := sb.LookupAccount(\"$foo\")\n\tbarB, _ := sb.LookupAccount(\"$bar\")\n\n\t// Add in the service export for the requests. Make it public.\n\tfooA.AddServiceExport(\"test.request\", nil)\n\tfooB.AddServiceExport(\"test.request\", nil)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Same on A.\n\tif err := barA.AddServiceImport(fooA, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint and responder.\n\taURL := fmt.Sprintf(\"nats://clientFoo:password@127.0.0.1:%d\", oa.Port)\n\tclientA := natsConnect(t, aURL)\n\tdefer clientA.Close()\n\n\tnatsSub(t, clientA, \"test.request\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"reply\"))\n\t})\n\tnatsFlush(t, clientA)\n\n\t// Now setup client B on srvB who will send the requests.\n\tbURL := fmt.Sprintf(\"nats://clientBar:password@127.0.0.1:%d\", ob.Port)\n\tclientB := natsConnect(t, bURL)\n\tdefer clientB.Close()\n\n\tif _, err := clientB.Request(\"foo.request\", []byte(\"request\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Did not get the reply: %v\", err)\n\t}\n\n\tquitCh := make(chan bool, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-quitCh:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tclientA.Publish(\"any.subject\", []byte(\"any message\"))\n\t\t\t\ttime.Sleep(time.Millisecond)\n\t\t\t}\n\t\t}\n\t}()\n\tfor i := 0; i < 1000; i++ {\n\t\tif _, err := clientB.Request(\"foo.request\", []byte(\"request\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not get the reply: %v\", err)\n\t\t}\n\t}\n\tclose(quitCh)\n\twg.Wait()\n}\n\nfunc TestGatewayCloseTLSConnection(t *testing.T) {\n\toa := testGatewayOptionsWithTLS(t, \"A\")\n\toa.DisableShortFirstPing = true\n\toa.Gateway.TLSConfig.ClientAuth = tls.NoClientCert\n\toa.Gateway.TLSTimeout = 100\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tob1 := testGatewayOptionsFromToWithTLS(t, \"B\", \"A\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", sa.GatewayAddr().Port)})\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 1, 2*time.Second)\n\n\tendpoint := net.JoinHostPort(oa.Gateway.Host, fmt.Sprintf(\"%d\", oa.Gateway.Port))\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on dial: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\ttlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})\n\tdefer tlsConn.Close()\n\tif err := tlsConn.Handshake(); err != nil {\n\t\tt.Fatalf(\"Unexpected error during handshake: %v\", err)\n\t}\n\tconnectOp := []byte(\"CONNECT {\\\"name\\\":\\\"serverID\\\",\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"tls_required\\\":true,\\\"gateway\\\":\\\"B\\\"}\\r\\n\")\n\tif _, err := tlsConn.Write(connectOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tinfoOp := []byte(\"INFO {\\\"server_id\\\":\\\"serverID\\\",\\\"tls_required\\\":true,\\\"gateway\\\":\\\"B\\\",\\\"gateway_nrp\\\":true}\\r\\n\")\n\tif _, err := tlsConn.Write(infoOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tif _, err := tlsConn.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing PING: %v\", err)\n\t}\n\n\t// Get gw connection\n\tvar gw *client\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tsa.gateway.RLock()\n\t\tfor _, g := range sa.gateway.in {\n\t\t\tg.mu.Lock()\n\t\t\tif g.opts.Name == \"serverID\" {\n\t\t\t\tgw = g\n\t\t\t}\n\t\t\tg.mu.Unlock()\n\t\t\tbreak\n\t\t}\n\t\tsa.gateway.RUnlock()\n\t\tif gw == nil {\n\t\t\treturn fmt.Errorf(\"No gw registered yet\")\n\t\t}\n\t\treturn nil\n\t})\n\t// Fill the buffer. We want to timeout on write so that nc.Close()\n\t// would block due to a write that cannot complete.\n\tbuf := make([]byte, 64*1024)\n\tdone := false\n\tfor !done {\n\t\tgw.nc.SetWriteDeadline(time.Now().Add(time.Second))\n\t\tif _, err := gw.nc.Write(buf); err != nil {\n\t\t\tdone = true\n\t\t}\n\t\tgw.nc.SetWriteDeadline(time.Time{})\n\t}\n\tch := make(chan bool)\n\tgo func() {\n\t\tselect {\n\t\tcase <-ch:\n\t\t\treturn\n\t\tcase <-time.After(3 * time.Second):\n\t\t\tfmt.Println(\"!!!! closeConnection is blocked, test will hang !!!\")\n\t\t\treturn\n\t\t}\n\t}()\n\t// Close the gateway\n\tgw.closeConnection(SlowConsumerWriteDeadline)\n\tch <- true\n}\n\nfunc TestGatewayNoCrashOnInvalidSubject(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\tncB := natsConnect(t, sb.ClientURL())\n\tdefer ncB.Close()\n\n\tnatsSubSync(t, ncB, \"foo\")\n\tnatsFlush(t, ncB)\n\n\tncA := natsConnect(t, sa.ClientURL())\n\tdefer ncA.Close()\n\n\t// Send on an invalid subject. Since there is interest on B,\n\t// we will receive an RS- instead of A-\n\tnatsPub(t, ncA, \"bar..baz\", []byte(\"bad subject\"))\n\tnatsFlush(t, ncA)\n\n\t// Now create on B a sub on a wildcard subject\n\tsub := natsSubSync(t, ncB, \"bar.*\")\n\tnatsFlush(t, ncB)\n\n\t// Server should not have crashed...\n\tnatsPub(t, ncA, \"bar.baz\", []byte(\"valid subject\"))\n\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t}\n}\n\nfunc TestGatewayUpdateURLsFromRemoteCluster(t *testing.T) {\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := RunServer(ob1)\n\tdefer sb1.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\n\t// Add a server to cluster B.\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tsb2 := RunServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\n\tpmap := make(map[int]string)\n\tpmap[ob1.Gateway.Port] = \"B1\"\n\tpmap[ob2.Gateway.Port] = \"B2\"\n\n\tcheckURLs := func(eurls map[string]string) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\trg := sa.getRemoteGateway(\"B\")\n\t\t\turls := rg.getURLsAsStrings()\n\t\t\tfor _, u := range urls {\n\t\t\t\tif _, ok := eurls[u]; !ok {\n\t\t\t\t\t_, sport, _ := net.SplitHostPort(u)\n\t\t\t\t\tport, _ := strconv.Atoi(sport)\n\t\t\t\t\treturn fmt.Errorf(\"URL %q (%s) should not be in the list of urls (%q)\", u, pmap[port], eurls)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\texpected := make(map[string]string)\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob1.Gateway.Port)] = \"B1\"\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob2.Gateway.Port)] = \"B2\"\n\tcheckURLs(expected)\n\n\t// Add another in cluster B\n\tob3 := testDefaultOptionsForGateway(\"B\")\n\tob3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tsb3 := RunServer(ob3)\n\tdefer sb3.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2, sb3)\n\twaitForOutboundGateways(t, sb3, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 3, 2*time.Second)\n\n\tpmap[ob3.Gateway.Port] = \"B3\"\n\n\texpected = make(map[string]string)\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob1.Gateway.Port)] = \"B1\"\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob2.Gateway.Port)] = \"B2\"\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob3.Gateway.Port)] = \"B3\"\n\tcheckURLs(expected)\n\n\t// Now stop server SB2, which should cause SA to remove it from its list.\n\tsb2.Shutdown()\n\n\texpected = make(map[string]string)\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob1.Gateway.Port)] = \"B1\"\n\texpected[fmt.Sprintf(\"127.0.0.1:%d\", ob3.Gateway.Port)] = \"B3\"\n\tcheckURLs(expected)\n}\n\ntype capturePingConn struct {\n\tnet.Conn\n\tch chan struct{}\n}\n\nfunc (c *capturePingConn) Write(b []byte) (int, error) {\n\tif bytes.Contains(b, []byte(pingProto)) {\n\t\tselect {\n\t\tcase c.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n\treturn c.Conn.Write(b)\n}\n\nfunc TestGatewayPings(t *testing.T) {\n\tgatewayMaxPingInterval = 50 * time.Millisecond\n\tdefer func() { gatewayMaxPingInterval = gwMaxPingInterval }()\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\n\tc := sa.getOutboundGatewayConnection(\"B\")\n\tch := make(chan struct{}, 1)\n\tc.mu.Lock()\n\tc.nc = &capturePingConn{c.nc, ch}\n\tc.mu.Unlock()\n\n\tfor i := 0; i < 5; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\tt.Fatalf(\"Did not send PING\")\n\t\t}\n\t}\n}\n\nfunc TestGatewayTLSConfigReload(t *testing.T) {\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\t%s\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\toptsB := testGatewayOptionsFromToWithTLS(t, \"B\", \"A\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Gateway.Port)})\n\tsrvB := runGatewayServer(optsB)\n\tdefer srvB.Shutdown()\n\n\twaitForGatewayFailedConnect(t, srvB, \"A\", true, time.Second)\n\n\treloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file: \"../test/configs/certs/ca.pem\"`))\n\n\twaitForInboundGateways(t, srvA, 1, time.Second)\n\twaitForOutboundGateways(t, srvA, 1, time.Second)\n\twaitForInboundGateways(t, srvB, 1, time.Second)\n\twaitForOutboundGateways(t, srvB, 1, time.Second)\n}\n\nfunc TestGatewayTLSConfigReloadForRemote(t *testing.T) {\n\tSetGatewaysSolicitDelay(5 * time.Millisecond)\n\tdefer ResetGatewaysSolicitDelay()\n\n\toptsA := testGatewayOptionsWithTLS(t, \"A\")\n\tsrvA := runGatewayServer(optsA)\n\tdefer srvA.Shutdown()\n\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"A\"\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\t\t\t%s\n\t\t\t\t\t\ttimeout: 2\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.Gateway.Port, \"\")))\n\n\tsrvB, _ := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\twaitForGatewayFailedConnect(t, srvB, \"A\", true, time.Second)\n\n\treloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.Gateway.Port, `ca_file: \"../test/configs/certs/ca.pem\"`))\n\n\twaitForInboundGateways(t, srvA, 1, time.Second)\n\twaitForOutboundGateways(t, srvA, 1, time.Second)\n\twaitForInboundGateways(t, srvB, 1, time.Second)\n\twaitForOutboundGateways(t, srvB, 1, time.Second)\n}\n\nfunc TestGatewayTLSConfigReloadForImplicitRemote(t *testing.T) {\n\tSetGatewaysSolicitDelay(5 * time.Millisecond)\n\tdefer ResetGatewaysSolicitDelay()\n\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/srva-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/srva-key.pem\"\n\t\t\t\t%s\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tconfA := createConfFile(t, fmt.Appendf(nil, template, `ca_file: \"../test/configs/certs/ca.pem\"`))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\toptsB := testGatewayOptionsFromToWithTLS(t, \"B\", \"A\", []string{fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Gateway.Port)})\n\tsrvB := runGatewayServer(optsB)\n\tdefer srvB.Shutdown()\n\n\twaitForInboundGateways(t, srvA, 1, time.Second)\n\twaitForOutboundGateways(t, srvA, 1, time.Second)\n\twaitForInboundGateways(t, srvB, 1, time.Second)\n\twaitForOutboundGateways(t, srvB, 1, time.Second)\n\n\t// We will verify that the config reload of the tls{} block is applied to\n\t// the implicit remote (from A to B) by removing the ca_file.\n\treloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, \"\"))\n\n\t// Get the remote from A to B\n\tcfg := srvA.getRemoteGateway(\"B\")\n\trequire_NotNil(t, cfg)\n\tcfg.Lock()\n\ttc := cfg.TLSConfig\n\tcfg.Unlock()\n\trequire_NotNil(t, tc)\n\t// The CA should have been removed.\n\trequire_True(t, tc.ClientCAs == nil)\n\n\t// Reset the connection attempts, since we are going to close the connection\n\t// from A to B and make sure that connection keeps failing.\n\tcfg.resetConnAttempts()\n\n\t// Get the outbound connection and close it.\n\tc := srvA.getOutboundGatewayConnection(\"B\")\n\trequire_NotNil(t, c)\n\tc.mu.Lock()\n\tc.nc.Close()\n\tc.mu.Unlock()\n\n\t// Verify that we fail to connect from A to B now.\n\twaitForGatewayFailedConnect(t, srvA, \"B\", true, time.Second)\n}\n\nfunc TestGatewayAuthDiscovered(t *testing.T) {\n\tSetGatewaysSolicitDelay(5 * time.Millisecond)\n\tdefer ResetGatewaysSolicitDelay()\n\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tauthorization: { user: gwuser, password: changeme }\n\t\t}\n\t`))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tauthorization: { user: gwuser, password: changeme }\n\t\t\tgateways: [\n\t\t\t\t{ name: A, url: nats://gwuser:changeme@127.0.0.1:%d }\n\t\t\t]\n\t\t}\n\t`, optsA.Gateway.Port)))\n\tsrvB, _ := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\twaitForInboundGateways(t, srvA, 1, time.Second)\n\twaitForOutboundGateways(t, srvA, 1, time.Second)\n\twaitForInboundGateways(t, srvB, 1, time.Second)\n\twaitForOutboundGateways(t, srvB, 1, time.Second)\n}\n\nfunc TestGatewayTLSCertificateImplicitAllowPass(t *testing.T) {\n\ttestGatewayTLSCertificateImplicitAllow(t, true)\n}\n\nfunc TestGatewayTLSCertificateImplicitAllowFail(t *testing.T) {\n\ttestGatewayTLSCertificateImplicitAllow(t, false)\n}\n\nfunc testGatewayTLSCertificateImplicitAllow(t *testing.T, pass bool) {\n\t// Base config for the servers\n\tcfg := createTempFile(t, \"cfg\")\n\tcfg.WriteString(fmt.Sprintf(`\n\t\tgateway {\n\t\t tls {\n\t\t\tcert_file = \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\tkey_file = \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\tca_file = \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\tverify_cert_and_check_known_urls = true\n\t\t\tinsecure = %t\n\t\t\ttimeout = 1\n\t\t }\n\t\t}\n\t`, !pass)) // set insecure to skip verification on the outgoing end\n\tif err := cfg.Sync(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcfg.Close()\n\n\toptsA := LoadConfig(cfg.Name())\n\toptsB := LoadConfig(cfg.Name())\n\n\turlA := \"nats://localhost:9995\"\n\turlB := \"nats://localhost:9996\"\n\tif !pass {\n\t\turlA = \"nats://127.0.0.1:9995\"\n\t\turlB = \"nats://127.0.0.1:9996\"\n\t}\n\n\tgwA, err := url.Parse(urlA)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tgwB, err := url.Parse(urlB)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\toptsA.Host = \"127.0.0.1\"\n\toptsA.Port = -1\n\toptsA.Gateway.Name = \"A\"\n\toptsA.Gateway.Port = 9995\n\toptsA.Gateway.resolver = &localhostResolver{}\n\n\toptsB.Host = \"127.0.0.1\"\n\toptsB.Port = -1\n\toptsB.Gateway.Name = \"B\"\n\toptsB.Gateway.Port = 9996\n\toptsB.Gateway.resolver = &localhostResolver{}\n\n\tgateways := make([]*RemoteGatewayOpts, 2)\n\tgateways[0] = &RemoteGatewayOpts{\n\t\tName: optsA.Gateway.Name,\n\t\tURLs: []*url.URL{gwA},\n\t}\n\tgateways[1] = &RemoteGatewayOpts{\n\t\tName: optsB.Gateway.Name,\n\t\tURLs: []*url.URL{gwB},\n\t}\n\n\toptsA.Gateway.Gateways = gateways\n\toptsB.Gateway.Gateways = gateways\n\n\tSetGatewaysSolicitDelay(100 * time.Millisecond)\n\tdefer ResetGatewaysSolicitDelay()\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tif pass {\n\t\twaitForOutboundGateways(t, srvA, 1, 5*time.Second)\n\t\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\t} else {\n\t\ttime.Sleep(1 * time.Second) // the fail case uses the IP, so a short wait is sufficient\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tif srvA.NumOutboundGateways() != 0 || srvB.NumOutboundGateways() != 0 {\n\t\t\t\treturn fmt.Errorf(\"No outbound gateway connection expected\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n}\n\nfunc TestGatewayURLsNotRemovedOnDuplicateRoute(t *testing.T) {\n\t// For this test, we need to have servers in cluster B creating routes\n\t// to each other to help produce the \"duplicate route\" situation, so\n\t// we are forced to use deterministic ports.\n\tgetEphemeralPort := func() int {\n\t\tt.Helper()\n\t\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error getting a port: %v\", err)\n\t\t}\n\t\tp := l.Addr().(*net.TCPAddr).Port\n\t\tl.Close()\n\t\treturn p\n\t}\n\tp1 := getEphemeralPort()\n\tp2 := getEphemeralPort()\n\trouteURLs := fmt.Sprintf(\"nats://127.0.0.1:%d,nats://127.0.0.1:%d\", p1, p2)\n\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tob1.Cluster.Port = p1\n\tob1.Routes = RoutesFromStr(routeURLs)\n\tsb1 := RunServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Cluster.Port = p2\n\tob2.Routes = RoutesFromStr(routeURLs)\n\tsb2 := RunServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\n\tcheckURLs := func(s *Server) {\n\t\tt.Helper()\n\t\ts.mu.Lock()\n\t\turls := s.gateway.URLs.getAsStringSlice()\n\t\ts.mu.Unlock()\n\t\tif len(urls) != 2 {\n\t\t\tt.Fatalf(\"Expected 2 urls, got %v\", urls)\n\t\t}\n\t}\n\tcheckURLs(sb1)\n\tcheckURLs(sb2)\n\n\t// As for sa, we should have both sb1 and sb2 urls in its outbound urls map\n\tc := sa.getOutboundGatewayConnection(\"B\")\n\tif c == nil {\n\t\tt.Fatal(\"No outound connection found!\")\n\t}\n\tc.mu.Lock()\n\turls := c.gw.cfg.urls\n\tc.mu.Unlock()\n\tif len(urls) != 2 {\n\t\tt.Fatalf(\"Expected 2 urls to B, got %v\", urls)\n\t}\n}\n\nfunc TestGatewayDuplicateServerName(t *testing.T) {\n\t// We will have 2 servers per cluster names \"nats1\" and \"nats2\", and have\n\t// the servers in the second cluster with the same name, but we will make\n\t// sure to connect \"A/nats1\" to \"B/nats2\" and \"A/nats2\" to \"B/nats1\" and\n\t// verify that we still discover the duplicate names.\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tob1.ServerName = \"nats1\"\n\tsb1 := RunServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.ServerName = \"nats2\"\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tsb2 := RunServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb2)\n\toa1.ServerName = \"nats1\"\n\t// Needed later in the test\n\toa1.Gateway.RejectUnknown = true\n\tsa1 := RunServer(oa1)\n\tdefer sa1.Shutdown()\n\tsa1l := &captureErrorLogger{errCh: make(chan string, 100)}\n\tsa1.SetLogger(sa1l, false, false)\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\toa2.ServerName = \"nats2\"\n\t// Needed later in the test\n\toa2.Gateway.RejectUnknown = true\n\toa2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", oa1.Cluster.Port))\n\tsa2 := RunServer(oa2)\n\tdefer sa2.Shutdown()\n\tsa2l := &captureErrorLogger{errCh: make(chan string, 100)}\n\tsa2.SetLogger(sa2l, false, false)\n\n\tcheckClusterFormed(t, sa1, sa2)\n\n\tcheckForDupError := func(errCh chan string) {\n\t\tt.Helper()\n\t\ttimeout := time.NewTimer(time.Second)\n\t\tfor done := false; !done; {\n\t\t\tselect {\n\t\t\tcase err := <-errCh:\n\t\t\t\tif strings.Contains(err, \"server has a duplicate name\") {\n\t\t\t\t\tdone = true\n\t\t\t\t}\n\t\t\tcase <-timeout.C:\n\t\t\t\tt.Fatal(\"Did not get error about servers in super-cluster with same name\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// Since only servers from \"A\" have configured outbound to\n\t// cluster \"B\", only servers on \"A\" are expected to report error.\n\tfor _, errCh := range []chan string{sa1l.errCh, sa2l.errCh} {\n\t\tcheckForDupError(errCh)\n\t}\n\n\t// So now we are going to fix names and wait for the super cluster to form.\n\tsa2.Shutdown()\n\tsa1.Shutdown()\n\n\t// Drain the error channels\n\tfor _, errCh := range []chan string{sa1l.errCh, sa2l.errCh} {\n\t\tfor done := false; !done; {\n\t\t\tselect {\n\t\t\tcase <-errCh:\n\t\t\tdefault:\n\t\t\t\tdone = true\n\t\t\t}\n\t\t}\n\t}\n\n\toa1.ServerName = \"a_nats1\"\n\toa2.ServerName = \"a_nats2\"\n\tsa1 = RunServer(oa1)\n\tdefer sa1.Shutdown()\n\tsa2 = RunServer(oa2)\n\tdefer sa2.Shutdown()\n\n\tcheckClusterFormed(t, sa1, sa2)\n\n\twaitForOutboundGateways(t, sa1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sa2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\n\t// Now add a server on cluster B (that does not have outbound\n\t// gateway connections explicitly defined) and use the name\n\t// of one of the cluster A's server. We should get an error.\n\tob3 := testDefaultOptionsForGateway(\"B\")\n\tob3.ServerName = \"a_nats2\"\n\tob3.Accounts = []*Account{NewAccount(\"sys\")}\n\tob3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob2.Cluster.Port))\n\tsb3 := RunServer(ob3)\n\tdefer sb3.Shutdown()\n\tsb3l := &captureErrorLogger{errCh: make(chan string, 100)}\n\tsb3.SetLogger(sb3l, false, false)\n\n\tcheckClusterFormed(t, sb1, sb2, sb3)\n\n\t// It should report the error when trying to create the GW connection\n\tcheckForDupError(sb3l.errCh)\n\n\t// Stop this node\n\tsb3.Shutdown()\n\tcheckClusterFormed(t, sb1, sb2)\n\n\t// Now create a GW \"C\" with a server that uses the same name than one of\n\t// the server on \"A\", say \"a_nats2\".\n\t// This server will connect to \"B\", and \"B\" will gossip \"A\" back to \"C\"\n\t// and \"C\" will then try to connect to \"A\", but \"A\" rejects unknown, so\n\t// connection will be refused. However, we want to make sure that the\n\t// duplicate server name is still detected.\n\toc := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", sb1)\n\toc.ServerName = \"a_nats2\"\n\toc.Accounts = []*Account{NewAccount(\"sys\")}\n\tsc := RunServer(oc)\n\tdefer sc.Shutdown()\n\tscl := &captureErrorLogger{errCh: make(chan string, 100)}\n\tsc.SetLogger(scl, false, false)\n\n\t// It should report the error when trying to create the GW connection\n\t// to cluster \"A\"\n\tcheckForDupError(scl.errCh)\n}\n\nfunc TestGatewayNoPanicOnStartupWithMonitoring(t *testing.T) {\n\to := testDefaultOptionsForGateway(\"B\")\n\to.HTTPHost = \"127.0.0.1\"\n\to.HTTPPort = 8888\n\ts, err := NewServer(o)\n\trequire_NoError(t, err)\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\ttime.Sleep(50 * time.Millisecond)\n\t\ts.Start()\n\t\ts.WaitForShutdown()\n\t}()\n\n\tfor {\n\t\tg, err := s.Gatewayz(nil)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif g.Port != 0 && g.Port != s.GatewayAddr().Port {\n\t\t\tt.Fatalf(\"Unexpected port: %v vs %v\", g.Port, s.GatewayAddr().Port)\n\t\t}\n\t\tbreak\n\t}\n\ts.Shutdown()\n\twg.Wait()\n}\n\nfunc TestGatewaySwitchToInterestOnlyModeImmediately(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\t// Add users to cause s2 to require auth. Will add an account with user later.\n\to2.Users = append([]*User(nil), &User{Username: \"test\", Password: \"pwd\"})\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\tsetAccountUserPassInOptions(o1, \"$foo\", \"ivan\", \"password\")\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\ts1Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o1.Port)\n\tnc := natsConnect(t, s1Url)\n\tdefer nc.Close()\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\n\tcheckCount := func(t *testing.T, c *client, expected int) {\n\t\tt.Helper()\n\t\tc.mu.Lock()\n\t\tout := c.outMsgs\n\t\tc.mu.Unlock()\n\t\tif int(out) != expected {\n\t\t\tt.Fatalf(\"Expected %d message(s) to be sent over, got %v\", expected, out)\n\t\t}\n\t}\n\t// No message should be sent\n\tgwcb := s1.getOutboundGatewayConnection(\"B\")\n\tcheckCount(t, gwcb, 0)\n\n\t// Check that we are in interest-only mode, but in this case, since s2 does\n\t// have the account, we should have the account not even present in the map.\n\tcheckGWInterestOnlyModeOrNotPresent(t, s1, \"B\", \"$foo\", true)\n\n\t// Add account to S2 and a client.\n\ts2FooAcc, err := s2.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error registering account: %v\", err)\n\t}\n\ts2.mu.Lock()\n\ts2.users[\"ivan\"] = &User{Account: s2FooAcc, Username: \"ivan\", Password: \"password\"}\n\ts2.mu.Unlock()\n\ts2Url := fmt.Sprintf(\"nats://ivan:password@127.0.0.1:%d\", o2.Port)\n\tncS2 := natsConnect(t, s2Url)\n\tdefer ncS2.Close()\n\tnatsSubSync(t, ncS2, \"asub\")\n\t// This time we will have the account in the map and it will be interest-only\n\tcheckGWInterestOnlyMode(t, s1, \"B\", \"$foo\")\n\n\t// Now publish a message, still should not go because the sub is on \"asub\"\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 0)\n\n\tnatsSubSync(t, ncS2, \"foo\")\n\tnatsFlush(t, ncS2)\n\n\tcheckGWInterestOnlyModeInterestOn(t, s1, \"B\", \"$foo\", \"foo\")\n\n\t// Publish on foo\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\tcheckCount(t, gwcb, 1)\n}\n\nfunc TestGatewaySlowConsumer(t *testing.T) {\n\tgatewayMaxPingInterval = 50 * time.Millisecond\n\tdefer func() { gatewayMaxPingInterval = gwMaxPingInterval }()\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\n\tc := sa.getOutboundGatewayConnection(\"B\")\n\tc.mu.Lock()\n\tc.out.wdl = time.Nanosecond\n\tc.mu.Unlock()\n\n\t<-time.After(250 * time.Millisecond)\n\tgot := sa.NumSlowConsumersGateways()\n\texpected := uint64(1)\n\tif got != 1 {\n\t\tt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t}\n\tgot = sb.NumSlowConsumersGateways()\n\texpected = 0\n\tif got != expected {\n\t\tt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/5187\nfunc TestGatewayConnectEvents(t *testing.T) {\n\tcheckEvents := func(t *testing.T, name string, queue bool) {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tca := createClusterEx(t, true, 5*time.Millisecond, true, \"A\", 2)\n\t\t\tdefer shutdownCluster(ca)\n\n\t\t\tcb := createClusterEx(t, true, 5*time.Millisecond, true, \"B\", 2, ca)\n\t\t\tdefer shutdownCluster(cb)\n\n\t\t\tsysA, err := nats.Connect(ca.randomServer().ClientURL(), nats.UserInfo(\"sys\", \"pass\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sysA.Close()\n\n\t\t\tvar sub1 *nats.Subscription\n\t\t\tif queue {\n\t\t\t\tsub1, err = sysA.QueueSubscribeSync(\"$SYS.ACCOUNT.FOO.CONNECT\", \"myqueue\")\n\t\t\t} else {\n\t\t\t\tsub1, err = sysA.SubscribeSync(\"$SYS.ACCOUNT.FOO.CONNECT\")\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcA, err := nats.Connect(ca.randomServer().ClientURL(), nats.UserInfo(\"foo\", \"pass\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer cA.Close()\n\n\t\t\tmsg, err := sub1.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Subject, \"$SYS.ACCOUNT.FOO.CONNECT\")\n\n\t\t\tcB, err := nats.Connect(cb.randomServer().ClientURL(), nats.UserInfo(\"foo\", \"pass\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer cB.Close()\n\n\t\t\tmsg, err = sub1.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Subject, \"$SYS.ACCOUNT.FOO.CONNECT\")\n\t\t})\n\t}\n\n\tcheckEvents(t, \"Unqueued\", false)\n\tcheckEvents(t, \"Queued\", true)\n}\n\nfunc disconnectInboundGateways(s *Server) {\n\ts.gateway.RLock()\n\tin := s.gateway.in\n\ts.gateway.RUnlock()\n\n\ts.gateway.RLock()\n\tfor _, client := range in {\n\t\ts.gateway.RUnlock()\n\t\tclient.closeConnection(ClientClosed)\n\t\ts.gateway.RLock()\n\t}\n\ts.gateway.RUnlock()\n}\n\ntype testMissingOCSPStapleLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *testMissingOCSPStapleLogger) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"peer missing OCSP Staple\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestGatewayOCSPMissingPeerStapleIssue(t *testing.T) {\n\tconst (\n\t\tcaCert = \"../test/configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"../test/configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponderCustomTimeout(t, caCert, caKey, 10*time.Minute)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\n\t// Node A\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\n\t// Node B\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\n\t// Node C\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\n\t// Node A rotated certs\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"../test/configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\tocsp { mode = always }\n\n system_account = sys\n accounts {\n sys { users = [{ user: sys, pass: sys }]}\n guest { users = [{ user: guest, pass: guest }]}\n }\n no_auth_user = guest\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// Gateway B connects to Gateway A.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\tocsp { mode = always }\n\n system_account = sys\n accounts {\n sys { users = [{ user: sys, pass: sys }]}\n guest { users = [{ user: guest, pass: guest }]}\n }\n no_auth_user = guest\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: B\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\"\n\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t}]\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\n\t// Wait for connectivity between A and B.\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\t// Gateway C also connects to Gateway A.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\tocsp { mode = always }\n\n system_account = sys\n accounts {\n sys { users = [{ user: sys, pass: sys }]}\n guest { users = [{ user: guest, pass: guest }]}\n }\n no_auth_user = guest\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: C\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{name: \"A\", url: \"nats://127.0.0.1:%d\" }]\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// //\n\t// A and B are connected at this point and A is starting with certs that //\n\t// will be rotated.\n\t// //\n\t////////////////////////////////////////////////////////////////////////////\n\tcB, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\trequire_NoError(t, err)\n\tdefer cB.Close()\n\n\tcC, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsC.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\trequire_NoError(t, err)\n\tdefer cC.Close()\n\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\trequire_NoError(t, err)\n\n\tcA.Flush()\n\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\trequire_NoError(t, err)\n\tcB.Flush()\n\n\twaitForOutboundGateways(t, srvB, 1, 10*time.Second)\n\twaitForOutboundGateways(t, srvC, 2, 10*time.Second)\n\n\t/////////////////////////////////////////////////////////////////////////////////\n\t// //\n\t// Switch all the certs from server A, all OCSP monitors should be restarted //\n\t// so it should have new staples. //\n\t// //\n\t/////////////////////////////////////////////////////////////////////////////////\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\tocsp { mode = always }\n\n system_account = sys\n accounts {\n sys { users = [{ user: sys, pass: sys }]}\n guest { users = [{ user: guest, pass: guest }]}\n }\n no_auth_user = guest\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\n\t\t\t}\n\t\t}\n\t`\n\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\twaitForOutboundGateways(t, srvA, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvC, 2, 5*time.Second)\n\n\t// Now clients connect to C can communicate with B and A.\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\trequire_NoError(t, err)\n\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\trequire_NoError(t, err)\n\n\t// Reload and disconnect very fast trying to produce the race.\n\tctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)\n\tdefer cancel()\n\n\t// Swap logger from server to capture the missing peer log.\n\tlA := &testMissingOCSPStapleLogger{ch: make(chan string, 30)}\n\tsrvA.SetLogger(lA, false, false)\n\n\tlB := &testMissingOCSPStapleLogger{ch: make(chan string, 30)}\n\tsrvB.SetLogger(lB, false, false)\n\n\tlC := &testMissingOCSPStapleLogger{ch: make(chan string, 30)}\n\tsrvC.SetLogger(lC, false, false)\n\n\t// Start with a reload from the last server that connected directly to A.\n\terr = srvC.Reload()\n\trequire_NoError(t, err)\n\n\t// Stress reconnections and reloading servers without getting\n\t// missing OCSP peer staple errors.\n\tvar wg sync.WaitGroup\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(500 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tdisconnectInboundGateways(srvA)\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(500 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tdisconnectInboundGateways(srvB)\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(500 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tdisconnectInboundGateways(srvC)\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(700 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tsrvC.Reload()\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(800 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tsrvB.Reload()\n\t\t}\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tfor range time.NewTicker(900 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tsrvA.Reload()\n\t\t}\n\t}()\n\n\tselect {\n\tcase <-ctx.Done():\n\tcase msg := <-lA.ch:\n\t\tt.Fatalf(\"Server A: Got OCSP Staple error: %v\", msg)\n\tcase msg := <-lB.ch:\n\t\tt.Fatalf(\"Server B: Got OCSP Staple error: %v\", msg)\n\tcase msg := <-lC.ch:\n\t\tt.Fatalf(\"Server C: Got OCSP Staple error: %v\", msg)\n\t}\n\twaitForOutboundGateways(t, srvA, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvC, 2, 5*time.Second)\n\twg.Wait()\n}\n\nfunc TestGatewayOutboundDetectsStaleConnectionIfNoInfo(t *testing.T) {\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\trequire_NoError(t, err)\n\tdefer l.Close()\n\n\tch := make(chan struct{})\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tc, err := l.Accept()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tdefer c.Close()\n\t\t<-ch\n\t}()\n\n\turl := fmt.Sprintf(\"nats://%s\", l.Addr())\n\to := testGatewayOptionsFromToWithURLs(t, \"A\", \"B\", []string{url})\n\to.gatewaysSolicitDelay = time.Millisecond\n\to.DisableShortFirstPing = false\n\to.PingInterval = 50 * time.Millisecond\n\to.MaxPingsOut = 3\n\to.NoLog = false\n\ts, err := NewServer(o)\n\trequire_NoError(t, err)\n\tdefer s.Shutdown()\n\n\tlog := &captureDebugLogger{dbgCh: make(chan string, 100)}\n\ts.SetLogger(log, true, false)\n\ts.Start()\n\n\ttimeout := time.NewTimer(time.Second)\n\tdefer timeout.Stop()\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase dbg := <-log.dbgCh:\n\t\t\t// The server should not send PING because the accept side expects\n\t\t\t// the CONNECT as the first protocol (otherwise it would be a parse\n\t\t\t// error if that were to happen).\n\t\t\tif strings.Contains(dbg, \"Ping Timer\") {\n\t\t\t\tt.Fatalf(\"The server should not have sent a ping, got %q\", dbg)\n\t\t\t}\n\t\t\t// However, it should detect at one point that the connection is\n\t\t\t// stale and close it.\n\t\t\tif strings.Contains(dbg, \"Stale\") {\n\t\t\t\tdone = true\n\t\t\t}\n\t\tcase <-timeout.C:\n\t\t\tt.Fatalf(\"Did not capture the stale connection condition\")\n\t\t}\n\t}\n\n\ts.Shutdown()\n\tclose(ch)\n\twg.Wait()\n\ts.WaitForShutdown()\n}\n\nfunc TestGatewayConfigureWriteDeadline(t *testing.T) {\n\to1 := testDefaultOptionsForGateway(\"B\")\n\to1.Gateway.WriteDeadline = 5 * time.Second\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s1)\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\twaitForInboundGateways(t, s1, 1, time.Second)\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\n\ts1.mu.RLock()\n\tdefer s1.mu.RUnlock()\n\n\ts1.gateway.RLock()\n\tdefer s1.gateway.RUnlock()\n\n\tfor _, r := range s1.gateway.out {\n\t\tr.mu.Lock()\n\t\twdl := r.out.wdl\n\t\tr.mu.Unlock()\n\t\trequire_Equal(t, wdl, 5*time.Second)\n\t}\n\n\tfor _, r := range s1.gateway.in {\n\t\tr.mu.Lock()\n\t\twdl := r.out.wdl\n\t\tr.mu.Unlock()\n\t\trequire_Equal(t, wdl, 5*time.Second)\n\t}\n}\n\nfunc TestGatewayConfigureWriteTimeoutPolicy(t *testing.T) {\n\tfor name, policy := range map[string]WriteTimeoutPolicy{\n\t\t\"Default\": WriteTimeoutPolicyDefault,\n\t\t\"Retry\": WriteTimeoutPolicyRetry,\n\t\t\"Close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\to1 := testDefaultOptionsForGateway(\"B\")\n\t\t\to1.Gateway.WriteTimeout = policy\n\t\t\ts1 := runGatewayServer(o1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\to2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s1)\n\t\t\ts2 := runGatewayServer(o2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, s2, 1, time.Second)\n\t\t\twaitForInboundGateways(t, s1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, s1, 1, time.Second)\n\n\t\t\ts1.mu.RLock()\n\t\t\tdefer s1.mu.RUnlock()\n\n\t\t\ts1.gateway.RLock()\n\t\t\tdefer s1.gateway.RUnlock()\n\n\t\t\tfor _, r := range s1.gateway.out {\n\t\t\t\tif policy == WriteTimeoutPolicyDefault {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, WriteTimeoutPolicyRetry)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, policy)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, r := range s1.gateway.in {\n\t\t\t\tif policy == WriteTimeoutPolicyDefault {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, WriteTimeoutPolicyRetry)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, policy)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestGatewayProcessRSubNoBlockingAccountFetch(t *testing.T) {\n\tcreateAccountPubKey := func() string {\n\t\tkp, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tpubkey, err := kp.PublicKey()\n\t\trequire_NoError(t, err)\n\t\treturn pubkey\n\t}\n\tsysPub := createAccountPubKey()\n\taccPub := createAccountPubKey()\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: cache\n\t\t\tdir: '%s'\n\t\t\ttimeout: \"2s\"\n\t\t}\n\t\tgateway: {\n\t\t\tname: \"clust-B\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n `, ojwt, sysPub, dir)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Set up a mock gateway client.\n\tc := s.createInternalAccountClient()\n\tc.mu.Lock()\n\tc.gw = &gateway{}\n\tc.gw.outsim = &sync.Map{}\n\tc.nc = &net.IPConn{}\n\tc.mu.Unlock()\n\n\t// Receiving a R+ should not be blocking, since we're in the gateway's readLoop.\n\tstart := time.Now()\n\trequire_NoError(t, c.processGatewayRSub(fmt.Appendf(nil, \"%s subj queue 0\", accPub)))\n\tc.mu.Lock()\n\tsubs := len(c.subs)\n\tc.mu.Unlock()\n\trequire_Len(t, subs, 1)\n\trequire_LessThan(t, time.Since(start), 100*time.Millisecond)\n\n\t// Receiving a R- should not be blocking, since we're in the gateway's readLoop.\n\tstart = time.Now()\n\trequire_NoError(t, c.processGatewayRUnsub(fmt.Appendf(nil, \"%s subj queue\", accPub)))\n\tc.mu.Lock()\n\tsubs = len(c.subs)\n\tc.mu.Unlock()\n\trequire_Len(t, subs, 0)\n\trequire_LessThan(t, time.Since(start), 100*time.Millisecond)\n}\n" }, { "path": "server/gsl/gsl.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage gsl\n\nimport (\n\t\"errors\"\n\t\"strings\"\n\t\"sync\"\n)\n\n// Sublist is a routing mechanism to handle subject distribution and\n// provides a facility to match subjects from published messages to\n// interested subscribers. Subscribers can have wildcard subjects to\n// match multiple published subjects.\n\n// Common byte variables for wildcards and token separator.\nconst (\n\tpwc = '*'\n\tpwcs = \"*\"\n\tfwc = '>'\n\tfwcs = \">\"\n\ttsep = \".\"\n\tbtsep = '.'\n\t_EMPTY_ = \"\"\n)\n\n// Sublist related errors\nvar (\n\tErrInvalidSubject = errors.New(\"gsl: invalid subject\")\n\tErrNotFound = errors.New(\"gsl: no matches found\")\n\tErrNilChan = errors.New(\"gsl: nil channel\")\n\tErrAlreadyRegistered = errors.New(\"gsl: notification already registered\")\n)\n\n// SimpleSublist is an alias type for GenericSublist that takes\n// empty values, useful for tracking interest only without any\n// unnecessary allocations.\ntype SimpleSublist = GenericSublist[struct{}]\n\n// NewSimpleSublist will create a simple sublist.\nfunc NewSimpleSublist() *SimpleSublist {\n\treturn &GenericSublist[struct{}]{root: newLevel[struct{}]()}\n}\n\n// A GenericSublist stores and efficiently retrieves subscriptions.\ntype GenericSublist[T comparable] struct {\n\tsync.RWMutex\n\troot *level[T]\n\tcount uint32\n}\n\n// A node contains subscriptions and a pointer to the next level.\ntype node[T comparable] struct {\n\tnext *level[T]\n\tsubs map[T]string // value -> subject\n}\n\n// A level represents a group of nodes and special pointers to\n// wildcard nodes.\ntype level[T comparable] struct {\n\tnodes map[string]*node[T]\n\tpwc, fwc *node[T]\n}\n\n// Create a new default node.\nfunc newNode[T comparable]() *node[T] {\n\treturn &node[T]{subs: make(map[T]string)}\n}\n\n// Create a new default level.\nfunc newLevel[T comparable]() *level[T] {\n\treturn &level[T]{nodes: make(map[string]*node[T])}\n}\n\n// NewSublist will create a default sublist with caching enabled per the flag.\nfunc NewSublist[T comparable]() *GenericSublist[T] {\n\treturn &GenericSublist[T]{root: newLevel[T]()}\n}\n\n// Insert adds a subscription into the sublist\nfunc (s *GenericSublist[T]) Insert(subject string, value T) error {\n\ts.Lock()\n\n\tvar sfwc bool\n\tvar n *node[T]\n\tl := s.root\n\n\tfor t := range strings.SplitSeq(subject, tsep) {\n\t\tlt := len(t)\n\t\tif lt == 0 || sfwc {\n\t\t\ts.Unlock()\n\t\t\treturn ErrInvalidSubject\n\t\t}\n\n\t\tif lt > 1 {\n\t\t\tn = l.nodes[t]\n\t\t} else {\n\t\t\tswitch t[0] {\n\t\t\tcase pwc:\n\t\t\t\tn = l.pwc\n\t\t\tcase fwc:\n\t\t\t\tn = l.fwc\n\t\t\t\tsfwc = true\n\t\t\tdefault:\n\t\t\t\tn = l.nodes[t]\n\t\t\t}\n\t\t}\n\t\tif n == nil {\n\t\t\tn = newNode[T]()\n\t\t\tif lt > 1 {\n\t\t\t\tl.nodes[t] = n\n\t\t\t} else {\n\t\t\t\tswitch t[0] {\n\t\t\t\tcase pwc:\n\t\t\t\t\tl.pwc = n\n\t\t\t\tcase fwc:\n\t\t\t\t\tl.fwc = n\n\t\t\t\tdefault:\n\t\t\t\t\tl.nodes[t] = n\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif n.next == nil {\n\t\t\tn.next = newLevel[T]()\n\t\t}\n\t\tl = n.next\n\t}\n\n\tn.subs[value] = subject\n\n\ts.count++\n\ts.Unlock()\n\n\treturn nil\n}\n\n// Match will match all entries to the literal subject.\n// It will return a set of results for both normal and queue subscribers.\nfunc (s *GenericSublist[T]) Match(subject string, cb func(T)) {\n\ts.match(subject, cb, true)\n}\n\n// MatchBytes will match all entries to the literal subject.\n// It will return a set of results for both normal and queue subscribers.\nfunc (s *GenericSublist[T]) MatchBytes(subject []byte, cb func(T)) {\n\ts.match(string(subject), cb, true)\n}\n\n// HasInterest will return whether or not there is any interest in the subject.\n// In cases where more detail is not required, this may be faster than Match.\nfunc (s *GenericSublist[T]) HasInterest(subject string) bool {\n\treturn s.hasInterest(subject, true, nil)\n}\n\n// NumInterest will return the number of subs interested in the subject.\n// In cases where more detail is not required, this may be faster than Match.\nfunc (s *GenericSublist[T]) NumInterest(subject string) (np int) {\n\ts.hasInterest(subject, true, &np)\n\treturn\n}\n\nfunc (s *GenericSublist[T]) match(subject string, cb func(T), doLock bool) {\n\ttsa := [32]string{}\n\ttokens := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif i-start == 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\tif start >= len(subject) {\n\t\treturn\n\t}\n\ttokens = append(tokens, subject[start:])\n\n\tif doLock {\n\t\ts.RLock()\n\t\tdefer s.RUnlock()\n\t}\n\tmatchLevel(s.root, tokens, cb)\n}\n\nfunc (s *GenericSublist[T]) hasInterest(subject string, doLock bool, np *int) bool {\n\ttsa := [32]string{}\n\ttokens := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif i-start == 0 {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\tif start >= len(subject) {\n\t\treturn false\n\t}\n\ttokens = append(tokens, subject[start:])\n\n\tif doLock {\n\t\ts.RLock()\n\t\tdefer s.RUnlock()\n\t}\n\treturn matchLevelForAny(s.root, tokens, np)\n}\n\nfunc matchLevelForAny[T comparable](l *level[T], toks []string, np *int) bool {\n\tvar pwc, n *node[T]\n\tfor i, t := range toks {\n\t\tif l == nil {\n\t\t\treturn false\n\t\t}\n\t\tif l.fwc != nil {\n\t\t\tif np != nil {\n\t\t\t\t*np += len(l.fwc.subs)\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t\tif pwc = l.pwc; pwc != nil {\n\t\t\tif match := matchLevelForAny(pwc.next, toks[i+1:], np); match {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tn = l.nodes[t]\n\t\tif n != nil {\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\tif n != nil {\n\t\tif np != nil {\n\t\t\t*np += len(n.subs)\n\t\t}\n\t\tif len(n.subs) > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\tif pwc != nil {\n\t\tif np != nil {\n\t\t\t*np += len(pwc.subs)\n\t\t}\n\t\treturn len(pwc.subs) > 0\n\t}\n\treturn false\n}\n\n// callbacksForResults will make the necessary callbacks for each\n// result in this node.\nfunc callbacksForResults[T comparable](n *node[T], cb func(T)) {\n\tfor sub := range n.subs {\n\t\tcb(sub)\n\t}\n}\n\n// matchLevel is used to recursively descend into the trie.\nfunc matchLevel[T comparable](l *level[T], toks []string, cb func(T)) {\n\tvar pwc, n *node[T]\n\tfor i, t := range toks {\n\t\tif l == nil {\n\t\t\treturn\n\t\t}\n\t\tif l.fwc != nil {\n\t\t\tcallbacksForResults(l.fwc, cb)\n\t\t}\n\t\tif pwc = l.pwc; pwc != nil {\n\t\t\tmatchLevel(pwc.next, toks[i+1:], cb)\n\t\t}\n\t\tn = l.nodes[t]\n\t\tif n != nil {\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\tif n != nil {\n\t\tcallbacksForResults(n, cb)\n\t}\n\tif pwc != nil {\n\t\tcallbacksForResults(pwc, cb)\n\t}\n}\n\n// lnt is used to track descent into levels for a removal for pruning.\ntype lnt[T comparable] struct {\n\tl *level[T]\n\tn *node[T]\n\tt string\n}\n\n// Raw low level remove, can do batches with lock held outside.\nfunc (s *GenericSublist[T]) remove(subject string, value T, shouldLock bool) error {\n\tif shouldLock {\n\t\ts.Lock()\n\t\tdefer s.Unlock()\n\t}\n\n\tvar sfwc bool\n\tvar n *node[T]\n\tl := s.root\n\n\t// Track levels for pruning\n\tvar lnts [32]lnt[T]\n\tlevels := lnts[:0]\n\n\tfor t := range strings.SplitSeq(subject, tsep) {\n\t\tlt := len(t)\n\t\tif lt == 0 || sfwc {\n\t\t\treturn ErrInvalidSubject\n\t\t}\n\t\tif l == nil {\n\t\t\treturn ErrNotFound\n\t\t}\n\t\tif lt > 1 {\n\t\t\tn = l.nodes[t]\n\t\t} else {\n\t\t\tswitch t[0] {\n\t\t\tcase pwc:\n\t\t\t\tn = l.pwc\n\t\t\tcase fwc:\n\t\t\t\tn = l.fwc\n\t\t\t\tsfwc = true\n\t\t\tdefault:\n\t\t\t\tn = l.nodes[t]\n\t\t\t}\n\t\t}\n\t\tif n != nil {\n\t\t\tlevels = append(levels, lnt[T]{l, n, t})\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\n\tif !s.removeFromNode(n, value) {\n\t\treturn ErrNotFound\n\t}\n\n\ts.count--\n\n\tfor i := len(levels) - 1; i >= 0; i-- {\n\t\tl, n, t := levels[i].l, levels[i].n, levels[i].t\n\t\tif n.isEmpty() {\n\t\t\tl.pruneNode(n, t)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Remove will remove a subscription.\nfunc (s *GenericSublist[T]) Remove(subject string, value T) error {\n\treturn s.remove(subject, value, true)\n}\n\n// HasInterestStartingIn is a helper for subject tree intersection.\nfunc (s *GenericSublist[T]) HasInterestStartingIn(subj string) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\tvar _tokens [64]string\n\ttokens := tokenizeSubjectIntoSlice(_tokens[:0], subj)\n\treturn hasInterestStartingIn(s.root, tokens)\n}\n\nfunc hasInterestStartingIn[T comparable](l *level[T], tokens []string) bool {\n\tif l == nil {\n\t\treturn false\n\t}\n\tif len(tokens) == 0 {\n\t\treturn true\n\t}\n\ttoken := tokens[0]\n\tif l.fwc != nil {\n\t\treturn true\n\t}\n\tfound := false\n\tif pwc := l.pwc; pwc != nil {\n\t\tfound = found || hasInterestStartingIn(pwc.next, tokens[1:])\n\t}\n\tif n := l.nodes[token]; n != nil {\n\t\tfound = found || hasInterestStartingIn(n.next, tokens[1:])\n\t}\n\treturn found\n}\n\n// pruneNode is used to prune an empty node from the tree.\nfunc (l *level[T]) pruneNode(n *node[T], t string) {\n\tif n == nil {\n\t\treturn\n\t}\n\tif n == l.fwc {\n\t\tl.fwc = nil\n\t} else if n == l.pwc {\n\t\tl.pwc = nil\n\t} else {\n\t\tdelete(l.nodes, t)\n\t}\n}\n\n// isEmpty will test if the node has any entries. Used\n// in pruning.\nfunc (n *node[T]) isEmpty() bool {\n\treturn len(n.subs) == 0 && (n.next == nil || n.next.numNodes() == 0)\n}\n\n// Return the number of nodes for the given level.\nfunc (l *level[T]) numNodes() int {\n\tif l == nil {\n\t\treturn 0\n\t}\n\tnum := len(l.nodes)\n\tif l.pwc != nil {\n\t\tnum++\n\t}\n\tif l.fwc != nil {\n\t\tnum++\n\t}\n\treturn num\n}\n\n// Remove the sub for the given node.\nfunc (s *GenericSublist[T]) removeFromNode(n *node[T], value T) (found bool) {\n\tif n == nil {\n\t\treturn false\n\t}\n\tif _, found = n.subs[value]; found {\n\t\tdelete(n.subs, value)\n\t}\n\treturn found\n}\n\n// Count returns the number of subscriptions.\nfunc (s *GenericSublist[T]) Count() uint32 {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.count\n}\n\n// numLevels will return the maximum number of levels\n// contained in the Sublist tree.\nfunc (s *GenericSublist[T]) numLevels() int {\n\treturn visitLevel(s.root, 0)\n}\n\n// visitLevel is used to descend the Sublist tree structure\n// recursively.\nfunc visitLevel[T comparable](l *level[T], depth int) int {\n\tif l == nil || l.numNodes() == 0 {\n\t\treturn depth\n\t}\n\n\tdepth++\n\tmaxDepth := depth\n\n\tfor _, n := range l.nodes {\n\t\tif n == nil {\n\t\t\tcontinue\n\t\t}\n\t\tnewDepth := visitLevel(n.next, depth)\n\t\tif newDepth > maxDepth {\n\t\t\tmaxDepth = newDepth\n\t\t}\n\t}\n\tif l.pwc != nil {\n\t\tpwcDepth := visitLevel(l.pwc.next, depth)\n\t\tif pwcDepth > maxDepth {\n\t\t\tmaxDepth = pwcDepth\n\t\t}\n\t}\n\tif l.fwc != nil {\n\t\tfwcDepth := visitLevel(l.fwc.next, depth)\n\t\tif fwcDepth > maxDepth {\n\t\t\tmaxDepth = fwcDepth\n\t\t}\n\t}\n\treturn maxDepth\n}\n\n// use similar to append. meaning, the updated slice will be returned\nfunc tokenizeSubjectIntoSlice(tts []string, subject string) []string {\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttts = append(tts, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttts = append(tts, subject[start:])\n\treturn tts\n}\n" }, { "path": "server/gsl/gsl_test.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage gsl\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/antithesis\"\n)\n\nfunc TestGenericSublistInit(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_Equal(t, s.count, 0)\n\trequire_Equal(t, s.Count(), s.count)\n}\n\nfunc TestGenericSublistInsertCount(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"foo\", struct{}{}))\n\trequire_NoError(t, s.Insert(\"bar\", struct{}{}))\n\trequire_NoError(t, s.Insert(\"foo.bar\", struct{}{}))\n\trequire_Equal(t, s.Count(), 3)\n}\n\nfunc TestGenericSublistSimple(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"foo\", struct{}{}))\n\trequire_Matches(t, s, \"foo\", 1)\n}\n\nfunc TestGenericSublistSimpleMultiTokens(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"foo.bar.baz\", struct{}{}))\n\trequire_Matches(t, s, \"foo.bar.baz\", 1)\n}\n\nfunc TestGenericSublistPartialWildcard(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"a.b.c\", struct{}{}))\n\trequire_NoError(t, s.Insert(\"a.*.c\", struct{}{}))\n\trequire_Matches(t, s, \"a.b.c\", 2)\n}\n\nfunc TestGenericSublistPartialWildcardAtEnd(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"a.b.c\", struct{}{}))\n\trequire_NoError(t, s.Insert(\"a.b.*\", struct{}{}))\n\trequire_Matches(t, s, \"a.b.c\", 2)\n}\n\nfunc TestGenericSublistFullWildcard(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"a.b.c\", struct{}{}))\n\trequire_NoError(t, s.Insert(\"a.>\", struct{}{}))\n\trequire_Matches(t, s, \"a.b.c\", 2)\n\trequire_Matches(t, s, \"a.>\", 1)\n}\n\nfunc TestGenericSublistRemove(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\n\trequire_NoError(t, s.Insert(\"a.b.c.d\", struct{}{}))\n\trequire_Equal(t, s.Count(), 1)\n\trequire_Matches(t, s, \"a.b.c.d\", 1)\n\n\trequire_NoError(t, s.Remove(\"a.b.c.d\", struct{}{}))\n\trequire_Equal(t, s.Count(), 0)\n\trequire_Matches(t, s, \"a.b.c.d\", 0)\n}\n\nfunc TestGenericSublistRemoveWildcard(t *testing.T) {\n\ts := NewSublist[int]()\n\n\trequire_NoError(t, s.Insert(\"a.b.c.d\", 11))\n\trequire_NoError(t, s.Insert(\"a.b.*.d\", 22))\n\trequire_NoError(t, s.Insert(\"a.b.>\", 33))\n\trequire_Equal(t, s.Count(), 3)\n\trequire_Matches(t, s, \"a.b.c.d\", 3)\n\n\trequire_NoError(t, s.Remove(\"a.b.*.d\", 22))\n\trequire_Equal(t, s.Count(), 2)\n\trequire_Matches(t, s, \"a.b.c.d\", 2)\n\n\trequire_NoError(t, s.Remove(\"a.b.>\", 33))\n\trequire_Equal(t, s.Count(), 1)\n\trequire_Matches(t, s, \"a.b.c.d\", 1)\n\n\trequire_NoError(t, s.Remove(\"a.b.c.d\", 11))\n\trequire_Equal(t, s.Count(), 0)\n\trequire_Matches(t, s, \"a.b.c.d\", 0)\n}\n\nfunc TestGenericSublistRemoveCleanup(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_Equal(t, s.numLevels(), 0)\n\trequire_NoError(t, s.Insert(\"a.b.c.d.e.f\", struct{}{}))\n\trequire_Equal(t, s.numLevels(), 6)\n\trequire_NoError(t, s.Remove(\"a.b.c.d.e.f\", struct{}{}))\n\trequire_Equal(t, s.numLevels(), 0)\n}\n\nfunc TestGenericSublistRemoveCleanupWildcards(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_Equal(t, s.numLevels(), 0)\n\trequire_NoError(t, s.Insert(\"a.b.*.d.e.>\", struct{}{}))\n\trequire_Equal(t, s.numLevels(), 6)\n\trequire_NoError(t, s.Remove(\"a.b.*.d.e.>\", struct{}{}))\n\trequire_Equal(t, s.numLevels(), 0)\n}\n\nfunc TestGenericSublistInvalidSubjectsInsert(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\t// Insert, or subscriptions, can have wildcards, but not empty tokens,\n\t// and can not have a FWC that is not the terminal token.\n\trequire_Error(t, s.Insert(\".foo\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Insert(\"foo.\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Insert(\"foo..bar\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Insert(\"foo.bar..baz\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Insert(\"foo.>.baz\", struct{}{}), ErrInvalidSubject)\n}\n\nfunc TestGenericSublistBadSubjectOnRemove(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_Error(t, s.Insert(\"a.b..d\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Remove(\"a.b..d\", struct{}{}), ErrInvalidSubject)\n\trequire_Error(t, s.Remove(\"a.>.b\", struct{}{}), ErrInvalidSubject)\n}\n\nfunc TestGenericSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\"foo\", struct{}{}))\n\trequire_Matches(t, s, \"foo\", 1)\n\trequire_Matches(t, s, \"foo.bar\", 0)\n}\n\nfunc TestGenericSublistInsertWithWildcardsAsLiterals(t *testing.T) {\n\ts := NewSublist[int]()\n\tfor i, subject := range []string{\"foo.*-\", \"foo.>-\"} {\n\t\trequire_NoError(t, s.Insert(subject, i))\n\t\trequire_Matches(t, s, \"foo.bar\", 0)\n\t\trequire_Matches(t, s, subject, 1)\n\t}\n}\n\nfunc TestGenericSublistRemoveWithWildcardsAsLiterals(t *testing.T) {\n\ts := NewSublist[int]()\n\tfor i, subject := range []string{\"foo.*-\", \"foo.>-\"} {\n\t\trequire_NoError(t, s.Insert(subject, i))\n\t\trequire_Matches(t, s, \"foo.bar\", 0)\n\t\trequire_Matches(t, s, subject, 1)\n\t\trequire_Error(t, s.Remove(\"foo.bar\", i), ErrNotFound)\n\t\trequire_Equal(t, s.Count(), 1)\n\t\trequire_NoError(t, s.Remove(subject, i))\n\t\trequire_Equal(t, s.Count(), 0)\n\t}\n}\n\nfunc TestGenericSublistMatchWithEmptyTokens(t *testing.T) {\n\ts := NewSublist[struct{}]()\n\trequire_NoError(t, s.Insert(\">\", struct{}{}))\n\tfor _, subject := range []string{\".foo\", \"..foo\", \"foo..\", \"foo.\", \"foo..bar\", \"foo...bar\"} {\n\t\tt.Run(subject, func(t *testing.T) {\n\t\t\trequire_Matches(t, s, subject, 0)\n\t\t})\n\t}\n}\n\nfunc TestGenericSublistHasInterest(t *testing.T) {\n\ts := NewSublist[int]()\n\trequire_NoError(t, s.Insert(\"foo\", 11))\n\n\t// Expect to find that \"foo\" matches but \"bar\" doesn't.\n\t// At this point nothing should be in the cache.\n\trequire_True(t, s.HasInterest(\"foo\"))\n\trequire_False(t, s.HasInterest(\"bar\"))\n\n\t// Call Match on a subject we know there is no match.\n\trequire_Matches(t, s, \"bar\", 0)\n\trequire_False(t, s.HasInterest(\"bar\"))\n\n\t// Remove fooSub and check interest again\n\trequire_NoError(t, s.Remove(\"foo\", 11))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\n\t// Try with some wildcards\n\trequire_NoError(t, s.Insert(\"foo.*\", 22))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_True(t, s.HasInterest(\"foo.bar\"))\n\trequire_False(t, s.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove sub, there should be no interest\n\trequire_NoError(t, s.Remove(\"foo.*\", 22))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_False(t, s.HasInterest(\"foo.bar\"))\n\trequire_False(t, s.HasInterest(\"foo.bar.baz\"))\n\n\trequire_NoError(t, s.Insert(\"foo.>\", 33))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_True(t, s.HasInterest(\"foo.bar\"))\n\trequire_True(t, s.HasInterest(\"foo.bar.baz\"))\n\n\trequire_NoError(t, s.Remove(\"foo.>\", 33))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_False(t, s.HasInterest(\"foo.bar\"))\n\trequire_False(t, s.HasInterest(\"foo.bar.baz\"))\n\n\trequire_NoError(t, s.Insert(\"*.>\", 44))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_True(t, s.HasInterest(\"foo.bar\"))\n\trequire_True(t, s.HasInterest(\"foo.baz\"))\n\trequire_NoError(t, s.Remove(\"*.>\", 44))\n\n\trequire_NoError(t, s.Insert(\"*.bar\", 55))\n\trequire_False(t, s.HasInterest(\"foo\"))\n\trequire_True(t, s.HasInterest(\"foo.bar\"))\n\trequire_False(t, s.HasInterest(\"foo.baz\"))\n\trequire_NoError(t, s.Remove(\"*.bar\", 55))\n\n\trequire_NoError(t, s.Insert(\"*\", 66))\n\trequire_True(t, s.HasInterest(\"foo\"))\n\trequire_False(t, s.HasInterest(\"foo.bar\"))\n\trequire_NoError(t, s.Remove(\"*\", 66))\n}\n\nfunc TestGenericSublistHasInterestOverlapping(t *testing.T) {\n\ts := NewSublist[int]()\n\trequire_NoError(t, s.Insert(\"stream.A.child\", 11))\n\trequire_NoError(t, s.Insert(\"stream.*\", 11))\n\trequire_True(t, s.HasInterest(\"stream.A.child\"))\n\trequire_True(t, s.HasInterest(\"stream.A\"))\n}\n\n// TestGenericSublistHasInterestStartingInRace tests that HasInterestStartingIn\n// is safe to call concurrently with modifications to the sublist.\nfunc TestGenericSublistHasInterestStartingInRace(t *testing.T) {\n\ts := NewSublist[int]()\n\n\t// Pre-populate with some patterns\n\tfor i := 0; i < 10; i++ {\n\t\ts.Insert(\"foo.bar.baz\", i)\n\t\ts.Insert(\"foo.*.baz\", i+10)\n\t\ts.Insert(\"foo.>\", i+20)\n\t}\n\n\tdone := make(chan struct{})\n\tconst iterations = 1000\n\n\t// Goroutine 1: repeatedly call HasInterestStartingIn\n\tgo func() {\n\t\tfor i := 0; i < iterations; i++ {\n\t\t\ts.HasInterestStartingIn(\"foo\")\n\t\t\ts.HasInterestStartingIn(\"foo.bar\")\n\t\t\ts.HasInterestStartingIn(\"foo.bar.baz\")\n\t\t\ts.HasInterestStartingIn(\"other.subject\")\n\t\t}\n\t\tdone <- struct{}{}\n\t}()\n\n\t// Goroutine 2: repeatedly modify the sublist\n\tgo func() {\n\t\tfor i := 0; i < iterations; i++ {\n\t\t\tval := 1000 + i\n\t\t\ts.Insert(\"test.subject.\"+string(rune('a'+i%26)), val)\n\t\t\ts.Insert(\"foo.*.test\", val)\n\t\t\ts.Remove(\"test.subject.\"+string(rune('a'+i%26)), val)\n\t\t\ts.Remove(\"foo.*.test\", val)\n\t\t}\n\t\tdone <- struct{}{}\n\t}()\n\n\t// Goroutine 3: also call HasInterest (which does lock)\n\tgo func() {\n\t\tfor i := 0; i < iterations; i++ {\n\t\t\ts.HasInterest(\"foo.bar.baz\")\n\t\t\ts.HasInterest(\"foo.something.baz\")\n\t\t}\n\t\tdone <- struct{}{}\n\t}()\n\n\t// Wait for all goroutines\n\t<-done\n\t<-done\n\t<-done\n}\n\nfunc TestGenericSublistNumInterest(t *testing.T) {\n\ts := NewSublist[int]()\n\trequire_NoError(t, s.Insert(\"foo\", 11))\n\n\trequire_NumInterest := func(t *testing.T, subj string, wnp int) {\n\t\tt.Helper()\n\t\trequire_Matches(t, s, subj, wnp)\n\t\trequire_Equal(t, s.NumInterest(subj), wnp)\n\t}\n\n\t// Expect to find that \"foo\" matches but \"bar\" doesn't.\n\t// At this point nothing should be in the cache.\n\trequire_NumInterest(t, \"foo\", 1)\n\trequire_NumInterest(t, \"bar\", 0)\n\n\t// Remove fooSub and check interest again\n\trequire_NoError(t, s.Remove(\"foo\", 11))\n\trequire_NumInterest(t, \"foo\", 0)\n\n\t// Try with some wildcards\n\trequire_NoError(t, s.Insert(\"foo.*\", 22))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0)\n\n\t// Remove sub, there should be no interest\n\trequire_NoError(t, s.Remove(\"foo.*\", 22))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0)\n\n\trequire_NoError(t, s.Insert(\"foo.>\", 33))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 1)\n\n\trequire_NoError(t, s.Remove(\"foo.>\", 33))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0)\n\n\trequire_NoError(t, s.Insert(\"*.>\", 44))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 1)\n\trequire_NoError(t, s.Remove(\"*.>\", 44))\n\n\trequire_NoError(t, s.Insert(\"*.bar\", 55))\n\trequire_NumInterest(t, \"foo\", 0)\n\trequire_NumInterest(t, \"foo.bar\", 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0)\n\trequire_NoError(t, s.Remove(\"*.bar\", 55))\n\n\trequire_NoError(t, s.Insert(\"*\", 66))\n\trequire_NumInterest(t, \"foo\", 1)\n\trequire_NumInterest(t, \"foo.bar\", 0)\n\trequire_NoError(t, s.Remove(\"*\", 66))\n}\n\n// --- TEST HELPERS ---\n\nfunc require_Matches[T comparable](t *testing.T, s *GenericSublist[T], sub string, c int) {\n\tt.Helper()\n\tmatches := 0\n\ts.Match(sub, func(_ T) {\n\t\tmatches++\n\t})\n\trequire_Equal(t, matches, c)\n}\n\nfunc require_True(t testing.TB, b bool) {\n\tt.Helper()\n\tif !b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_True check\", nil)\n\t\tt.Fatalf(\"require true, but got false\")\n\t}\n}\n\nfunc require_False(t testing.TB, b bool) {\n\tt.Helper()\n\tif b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_False check\", nil)\n\t\tt.Fatalf(\"require false, but got true\")\n\t}\n}\n\nfunc require_NoError(t testing.TB, err error) {\n\tt.Helper()\n\tif err != nil {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_NoError check\", map[string]any{\n\t\t\t\"error\": err.Error(),\n\t\t})\n\t\tt.Fatalf(\"require no error, but got: %v\", err)\n\t}\n}\n\nfunc require_Error(t testing.TB, err error, expected ...error) {\n\tt.Helper()\n\tif err == nil {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Error check (nil error)\", nil)\n\t\tt.Fatalf(\"require error, but got none\")\n\t}\n\tif len(expected) == 0 {\n\t\treturn\n\t}\n\t// Try to strip nats prefix from Go library if present.\n\tconst natsErrPre = \"nats: \"\n\teStr := err.Error()\n\tif strings.HasPrefix(eStr, natsErrPre) {\n\t\teStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1)\n\t}\n\n\tfor _, e := range expected {\n\t\tif err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tantithesis.AssertUnreachable(t, \"Failed require_Error check (unexpected error)\", map[string]any{\n\t\t\"error\": err.Error(),\n\t})\n\tt.Fatalf(\"Expected one of %v, got '%v'\", expected, err)\n}\n\nfunc require_Equal[T comparable](t testing.TB, a, b T) {\n\tt.Helper()\n\tif a != b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Equal check\", nil)\n\t\tt.Fatalf(\"require %T equal, but got: %v != %v\", a, a, b)\n\t}\n}\n" }, { "path": "server/ipqueue.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\nconst ipQueueDefaultMaxRecycleSize = 4 * 1024\n\n// This is a generic intra-process queue.\ntype ipQueue[T any] struct {\n\tinprogress int64\n\tsync.Mutex\n\tch chan struct{}\n\telts []T\n\tpos int\n\tpool *sync.Pool\n\tsz uint64 // Calculated size (only if calc != nil)\n\tname string\n\tm *sync.Map\n\tipQueueOpts[T]\n}\n\ntype ipQueueOpts[T any] struct {\n\tmrs int // Max recycle size\n\tcalc func(e T) uint64 // Calc function for tracking size\n\tmsz uint64 // Limit by total calculated size\n\tmlen int // Limit by number of entries\n}\n\ntype ipQueueOpt[T any] func(*ipQueueOpts[T])\n\n// This option allows to set the maximum recycle size when attempting\n// to put back a slice to the pool.\nfunc ipqMaxRecycleSize[T any](max int) ipQueueOpt[T] {\n\treturn func(o *ipQueueOpts[T]) {\n\t\to.mrs = max\n\t}\n}\n\n// This option enables total queue size counting by passing in a function\n// that evaluates the size of each entry as it is pushed/popped. This option\n// enables the size() function.\nfunc ipqSizeCalculation[T any](calc func(e T) uint64) ipQueueOpt[T] {\n\treturn func(o *ipQueueOpts[T]) {\n\t\to.calc = calc\n\t}\n}\n\n// This option allows setting the maximum queue size. Once the limit is\n// reached, then push() will stop returning true and no more entries will\n// be stored until some more are popped. The ipQueue_SizeCalculation must\n// be provided for this to work.\nfunc ipqLimitBySize[T any](max uint64) ipQueueOpt[T] {\n\treturn func(o *ipQueueOpts[T]) {\n\t\to.msz = max\n\t}\n}\n\n// This option allows setting the maximum queue length. Once the limit is\n// reached, then push() will stop returning true and no more entries will\n// be stored until some more are popped.\nfunc ipqLimitByLen[T any](max int) ipQueueOpt[T] {\n\treturn func(o *ipQueueOpts[T]) {\n\t\to.mlen = max\n\t}\n}\n\nvar errIPQLenLimitReached = errors.New(\"IPQ len limit reached\")\nvar errIPQSizeLimitReached = errors.New(\"IPQ size limit reached\")\n\nfunc newIPQueue[T any](s *Server, name string, opts ...ipQueueOpt[T]) *ipQueue[T] {\n\tq := &ipQueue[T]{\n\t\tch: make(chan struct{}, 1),\n\t\tpool: &sync.Pool{\n\t\t\tNew: func() any {\n\t\t\t\t// Reason we use pointer to slice instead of slice is explained\n\t\t\t\t// here: https://staticcheck.io/docs/checks#SA6002\n\t\t\t\tres := make([]T, 0, 32)\n\t\t\t\treturn &res\n\t\t\t},\n\t\t},\n\t\tname: name,\n\t\tm: &s.ipQueues,\n\t\tipQueueOpts: ipQueueOpts[T]{\n\t\t\tmrs: ipQueueDefaultMaxRecycleSize,\n\t\t},\n\t}\n\tfor _, o := range opts {\n\t\to(&q.ipQueueOpts)\n\t}\n\ts.ipQueues.Store(name, q)\n\treturn q\n}\n\n// Add the element `e` to the queue, notifying the queue channel's `ch` if the\n// entry is the first to be added, and returns the length of the queue after\n// this element is added.\nfunc (q *ipQueue[T]) push(e T) (int, error) {\n\tq.Lock()\n\tl := len(q.elts) - q.pos\n\tif q.mlen > 0 && l == q.mlen {\n\t\tq.Unlock()\n\t\treturn l, errIPQLenLimitReached\n\t}\n\tif q.calc != nil {\n\t\tsz := q.calc(e)\n\t\tif q.msz > 0 && q.sz+sz > q.msz {\n\t\t\tq.Unlock()\n\t\t\treturn l, errIPQSizeLimitReached\n\t\t}\n\t\tq.sz += sz\n\t}\n\tif q.elts == nil {\n\t\t// What comes out of the pool is already of size 0, so no need for [:0].\n\t\tq.elts = *(q.pool.Get().(*[]T))\n\t}\n\tq.elts = append(q.elts, e)\n\tq.Unlock()\n\tif l == 0 {\n\t\tselect {\n\t\tcase q.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n\treturn l + 1, nil\n}\n\n// Returns the whole list of elements currently present in the queue,\n// emptying the queue. This should be called after receiving a notification\n// from the queue's `ch` notification channel that indicates that there\n// is something in the queue.\n// However, in cases where `drain()` may be called from another go\n// routine, it is possible that a routine is notified that there is\n// something, but by the time it calls `pop()`, the drain() would have\n// emptied the queue. So the caller should never assume that pop() will\n// return a slice of 1 or more, it could return `nil`.\nfunc (q *ipQueue[T]) pop() []T {\n\tif q == nil {\n\t\treturn nil\n\t}\n\tq.Lock()\n\tif len(q.elts)-q.pos == 0 {\n\t\tq.Unlock()\n\t\treturn nil\n\t}\n\tvar elts []T\n\tif q.pos == 0 {\n\t\telts = q.elts\n\t} else {\n\t\telts = q.elts[q.pos:]\n\t}\n\tq.elts, q.pos, q.sz = nil, 0, 0\n\tatomic.AddInt64(&q.inprogress, int64(len(elts)))\n\tq.Unlock()\n\treturn elts\n}\n\n// Returns the first element from the queue, if any. See comment above\n// regarding calling after being notified that there is something and\n// the use of drain(). In short, the caller should always check the\n// boolean return value to ensure that the value is genuine and not a\n// default empty value.\nfunc (q *ipQueue[T]) popOne() (T, bool) {\n\tq.Lock()\n\tl := len(q.elts) - q.pos\n\tif l == 0 {\n\t\tq.Unlock()\n\t\tvar empty T\n\t\treturn empty, false\n\t}\n\te := q.elts[q.pos]\n\tif l--; l > 0 {\n\t\tq.pos++\n\t\tif q.calc != nil {\n\t\t\tq.sz -= q.calc(e)\n\t\t}\n\t\t// We need to re-signal\n\t\tselect {\n\t\tcase q.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t} else {\n\t\t// We have just emptied the queue, so we can reuse unless it is too big.\n\t\tif cap(q.elts) <= q.mrs {\n\t\t\tq.elts = q.elts[:0]\n\t\t} else {\n\t\t\tq.elts = nil\n\t\t}\n\t\tq.pos, q.sz = 0, 0\n\t}\n\tq.Unlock()\n\treturn e, true\n}\n\n// After a pop(), the slice can be recycled for the next push() when\n// a first element is added to the queue.\n// This will also decrement the \"in progress\" count with the length\n// of the slice.\n// WARNING: The caller MUST never reuse `elts`.\nfunc (q *ipQueue[T]) recycle(elts *[]T) {\n\t// If invoked with a nil list, nothing to do.\n\tif elts == nil || *elts == nil {\n\t\treturn\n\t}\n\t// Update the in progress count.\n\tif len(*elts) > 0 {\n\t\tatomic.AddInt64(&q.inprogress, int64(-(len(*elts))))\n\t}\n\t// We also don't want to recycle huge slices, so check against the max.\n\t// q.mrs is normally immutable but can be changed, in a safe way, in some tests.\n\tif cap(*elts) > q.mrs {\n\t\treturn\n\t}\n\t(*elts) = (*elts)[:0]\n\tq.pool.Put(elts)\n}\n\n// Returns the current length of the queue.\nfunc (q *ipQueue[T]) len() int {\n\tq.Lock()\n\tdefer q.Unlock()\n\treturn len(q.elts) - q.pos\n}\n\n// Returns the calculated size of the queue (if ipQueue_SizeCalculation has been\n// passed in), otherwise returns zero.\nfunc (q *ipQueue[T]) size() uint64 {\n\tq.Lock()\n\tdefer q.Unlock()\n\treturn q.sz\n}\n\n// Empty the queue and consumes the notification signal if present.\n// Returns the number of items that were drained from the queue.\n// Note that this could cause a reader go routine that has been\n// notified that there is something in the queue (reading from queue's `ch`)\n// may then get nothing if `drain()` is invoked before the `pop()` or `popOne()`.\nfunc (q *ipQueue[T]) drain() int {\n\tif q == nil {\n\t\treturn 0\n\t}\n\tq.Lock()\n\tolen := len(q.elts) - q.pos\n\tq.elts, q.pos, q.sz = nil, 0, 0\n\t// Consume the signal if it was present to reduce the chance of a reader\n\t// routine to be think that there is something in the queue...\n\tselect {\n\tcase <-q.ch:\n\tdefault:\n\t}\n\tq.Unlock()\n\treturn olen\n}\n\n// Since the length of the queue goes to 0 after a pop(), it is good to\n// have an insight on how many elements are yet to be processed after a pop().\n// For that reason, the queue maintains a count of elements returned through\n// the pop() API. When the caller will call q.recycle(), this count will\n// be reduced by the size of the slice returned by pop().\nfunc (q *ipQueue[T]) inProgress() int64 {\n\treturn atomic.LoadInt64(&q.inprogress)\n}\n\n// Remove this queue from the server's map of ipQueues.\n// All ipQueue operations (such as push/pop/etc..) are still possible.\nfunc (q *ipQueue[T]) unregister() {\n\tif q == nil {\n\t\treturn\n\t}\n\tq.m.Delete(q.name)\n}\n" }, { "path": "server/ipqueue_test.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestIPQueueBasic(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\t// Check that pool has been created\n\tif q.pool == nil {\n\t\tt.Fatal(\"Expected pool to have been created\")\n\t}\n\t// Check for the default mrs\n\tif q.mrs != ipQueueDefaultMaxRecycleSize {\n\t\tt.Fatalf(\"Expected default max recycle size to be %v, got %v\",\n\t\t\tipQueueDefaultMaxRecycleSize, q.mrs)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\tt.Fatalf(\"Should not have been notified\")\n\tdefault:\n\t\t// OK!\n\t}\n\tif l := q.len(); l != 0 {\n\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t}\n\n\t// Try to change the max recycle size\n\tq2 := newIPQueue[int](s, \"test2\", ipqMaxRecycleSize[int](10))\n\tif q2.mrs != 10 {\n\t\tt.Fatalf(\"Expected max recycle size to be 10, got %v\", q2.mrs)\n\t}\n\n\t// Check that those 2 queues are registered\n\tvar gotFirst bool\n\tvar gotSecond bool\n\ts.ipQueues.Range(func(k, v any) bool {\n\t\tswitch k.(string) {\n\t\tcase \"test\":\n\t\t\tgotFirst = true\n\t\tcase \"test2\":\n\t\t\tgotSecond = true\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unknown queue: %q\", k.(string))\n\t\t}\n\t\treturn true\n\t})\n\tif !gotFirst {\n\t\tt.Fatalf(\"Did not find queue %q\", \"test\")\n\t}\n\tif !gotSecond {\n\t\tt.Fatalf(\"Did not find queue %q\", \"test2\")\n\t}\n\t// Unregister them\n\tq.unregister()\n\tq2.unregister()\n\t// They should have been removed from the map\n\ts.ipQueues.Range(func(k, v any) bool {\n\t\tt.Fatalf(\"Got queue %q\", k.(string))\n\t\treturn false\n\t})\n\t// But verify that we can still push/pop\n\tq.push(1)\n\telts := q.pop()\n\tif len(elts) != 1 {\n\t\tt.Fatalf(\"Should have gotten 1 element, got %v\", len(elts))\n\t}\n\tq2.push(2)\n\tif e, ok := q2.popOne(); !ok || e != 2 {\n\t\tt.Fatalf(\"popOne failed: %+v\", e)\n\t}\n}\n\nfunc TestIPQueuePush(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\tq.push(1)\n\tif l := q.len(); l != 1 {\n\t\tt.Fatalf(\"Expected len to be 1, got %v\", l)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\t// OK\n\tdefault:\n\t\tt.Fatalf(\"Should have been notified of addition\")\n\t}\n\t// Push a new element, we should not be notified.\n\tq.push(2)\n\tif l := q.len(); l != 2 {\n\t\tt.Fatalf(\"Expected len to be 2, got %v\", l)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\tt.Fatalf(\"Should not have been notified of addition\")\n\tdefault:\n\t\t// OK\n\t}\n}\n\nfunc TestIPQueuePop(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\tq.push(1)\n\t<-q.ch\n\telts := q.pop()\n\tif l := len(elts); l != 1 {\n\t\tt.Fatalf(\"Expected 1 elt, got %v\", l)\n\t}\n\tif l := q.len(); l != 0 {\n\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t}\n\t// The channel notification should be empty\n\tselect {\n\tcase <-q.ch:\n\t\tt.Fatalf(\"Should not have been notified of addition\")\n\tdefault:\n\t\t// OK\n\t}\n\t// Since pop() brings the number of pending to 0, we keep track of the\n\t// number of \"in progress\" elements. Check that the value is 1 here.\n\tif n := q.inProgress(); n != 1 {\n\t\tt.Fatalf(\"Expected count to be 1, got %v\", n)\n\t}\n\t// Recycling will bring it down to 0.\n\tq.recycle(&elts)\n\tif n := q.inProgress(); n != 0 {\n\t\tt.Fatalf(\"Expected count to be 0, got %v\", n)\n\t}\n\t// If we call pop() now, we should get nil.\n\tif elts = q.pop(); elts != nil {\n\t\tt.Fatalf(\"Expected nil, got %v\", elts)\n\t}\n\t// The in progress count should still be 0\n\tif n := q.inProgress(); n != 0 {\n\t\tt.Fatalf(\"Expected count to be 0, got %v\", n)\n\t}\n\t// Calling pop() on an unitialized queue will return nil\n\tvar q2 *ipQueue[int]\n\telts = q2.pop()\n\trequire_True(t, elts == nil)\n}\n\nfunc TestIPQueuePopOne(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\tq.push(1)\n\t<-q.ch\n\te, ok := q.popOne()\n\tif !ok {\n\t\tt.Fatal(\"Got nil\")\n\t}\n\tif i := e; i != 1 {\n\t\tt.Fatalf(\"Expected 1, got %v\", i)\n\t}\n\tif l := q.len(); l != 0 {\n\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t}\n\t// That does not affect the number of notProcessed\n\tif n := q.inProgress(); n != 0 {\n\t\tt.Fatalf(\"Expected count to be 0, got %v\", n)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\tt.Fatalf(\"Should not have been notified of addition\")\n\tdefault:\n\t\t// OK\n\t}\n\tq.push(2)\n\tq.push(3)\n\te, ok = q.popOne()\n\tif !ok {\n\t\tt.Fatal(\"Got nil\")\n\t}\n\tif i := e; i != 2 {\n\t\tt.Fatalf(\"Expected 2, got %v\", i)\n\t}\n\tif l := q.len(); l != 1 {\n\t\tt.Fatalf(\"Expected len to be 1, got %v\", l)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\t// OK\n\tdefault:\n\t\tt.Fatalf(\"Should have been notified that there is more\")\n\t}\n\te, ok = q.popOne()\n\tif !ok {\n\t\tt.Fatal(\"Got nil\")\n\t}\n\tif i := e; i != 3 {\n\t\tt.Fatalf(\"Expected 3, got %v\", i)\n\t}\n\tif l := q.len(); l != 0 {\n\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t}\n\tselect {\n\tcase <-q.ch:\n\t\tt.Fatalf(\"Should not have been notified that there is more\")\n\tdefault:\n\t\t// OK\n\t}\n\t// Calling it again now that we know there is nothing, we\n\t// should get nil.\n\tif e, ok = q.popOne(); ok {\n\t\tt.Fatalf(\"Expected nil, got %v\", e)\n\t}\n\n\tq = newIPQueue[int](s, \"test2\")\n\tq.push(1)\n\tq.push(2)\n\t// Capture current capacity\n\tq.Lock()\n\tc := cap(q.elts)\n\tq.Unlock()\n\te, ok = q.popOne()\n\tif !ok || e != 1 {\n\t\tt.Fatalf(\"Invalid value: %v\", e)\n\t}\n\tif l := q.len(); l != 1 {\n\t\tt.Fatalf(\"Expected len to be 1, got %v\", l)\n\t}\n\tvalues := q.pop()\n\tif len(values) != 1 || values[0] != 2 {\n\t\tt.Fatalf(\"Unexpected values: %v\", values)\n\t}\n\tif cap(values) != c-1 {\n\t\tt.Fatalf(\"Unexpected capacity: %v vs %v\", cap(values), c-1)\n\t}\n\tif l := q.len(); l != 0 {\n\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t}\n\t// Just make sure that this is ok...\n\tq.recycle(&values)\n}\n\nfunc TestIPQueueMultiProducers(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\n\twg := sync.WaitGroup{}\n\twg.Add(3)\n\tsend := func(start, end int) {\n\t\tdefer wg.Done()\n\n\t\tfor i := start; i <= end; i++ {\n\t\t\tq.push(i)\n\t\t}\n\t}\n\tgo send(1, 100)\n\tgo send(101, 200)\n\tgo send(201, 300)\n\n\ttm := time.NewTimer(2 * time.Second)\n\tm := make(map[int]struct{})\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase <-q.ch:\n\t\t\tvalues := q.pop()\n\t\t\tfor _, v := range values {\n\t\t\t\tm[v] = struct{}{}\n\t\t\t}\n\t\t\tq.recycle(&values)\n\t\t\tif n := q.inProgress(); n != 0 {\n\t\t\t\tt.Fatalf(\"Expected count to be 0, got %v\", n)\n\t\t\t}\n\t\t\tdone = len(m) == 300\n\t\tcase <-tm.C:\n\t\t\tt.Fatalf(\"Did not receive all elements: %v\", m)\n\t\t}\n\t}\n\twg.Wait()\n}\n\nfunc TestIPQueueRecycle(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\ttotal := 1000\n\tfor iter := 0; iter < 5; iter++ {\n\t\tvar sz int\n\t\tfor i := 0; i < total; i++ {\n\t\t\tsz, _ = q.push(i)\n\t\t}\n\t\tif sz != total {\n\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", total, sz)\n\t\t}\n\t\tvalues := q.pop()\n\t\tpreRecycleCap := cap(values)\n\t\tq.recycle(&values)\n\t\tsz, _ = q.push(1001)\n\t\tif sz != 1 {\n\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", 1, sz)\n\t\t}\n\t\tvalues = q.pop()\n\t\tif l := len(values); l != 1 {\n\t\t\tt.Fatalf(\"Len should be 1, got %v\", l)\n\t\t}\n\t\tif c := cap(values); c == preRecycleCap {\n\t\t\tbreak\n\t\t} else if iter == 4 {\n\t\t\t// We can't fail the test since there is no guarantee that the slice\n\t\t\t// is still present in the pool when we do a Get(), but let's log that\n\t\t\t// recycling did not occur even after all iterations..\n\t\t\tt.Logf(\"Seem like the previous slice was not recycled, old cap=%v new cap=%v\",\n\t\t\t\tpreRecycleCap, c)\n\t\t}\n\t}\n\n\tq = newIPQueue[int](s, \"test2\", ipqMaxRecycleSize[int](10))\n\tfor i := 0; i < 100; i++ {\n\t\tq.push(i)\n\t}\n\tvalues := q.pop()\n\tpreRecycleCap := cap(values)\n\tq.recycle(&values)\n\tq.push(1001)\n\tvalues = q.pop()\n\tif l := len(values); l != 1 {\n\t\tt.Fatalf(\"Len should be 1, got %v\", l)\n\t}\n\t// This time, we should not have recycled it, so the new cap should be whatever\n\t// the default is, but less than the preRecycleCap.\n\tif c := cap(values); c >= preRecycleCap {\n\t\tt.Fatalf(\"The slice should not have been put back in the pool, got cap of %v\", c)\n\t}\n\tq.recycle(&values)\n\n\t// Check that we don't crash when recycling a nil or empty slice\n\tvalues = q.pop()\n\tq.recycle(&values)\n\tq.recycle(nil)\n}\n\nfunc TestIPQueueDrain(t *testing.T) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\tfor iter, recycled := 0, false; iter < 5 && !recycled; iter++ {\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tq.push(i + 1)\n\t\t}\n\t\tq.drain()\n\t\t// Try to get something from the pool right away\n\t\ts := q.pool.Get()\n\t\trecycled := s != nil\n\t\tif !recycled {\n\t\t\t// We can't fail the test, since we have no guarantee it will be recycled\n\t\t\t// especially when running with `-race` flag...\n\t\t\tif iter == 4 {\n\t\t\t\tt.Log(\"nothing was recycled\")\n\t\t\t}\n\t\t}\n\t\t// Check that we have consumed the signal...\n\t\tselect {\n\t\tcase <-q.ch:\n\t\t\tt.Fatal(\"Signal should have been consumed by drain\")\n\t\tdefault:\n\t\t\t// OK!\n\t\t}\n\t\t// Check len\n\t\tif l := q.len(); l != 0 {\n\t\t\tt.Fatalf(\"Expected len to be 0, got %v\", l)\n\t\t}\n\t\tif recycled {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestIPQueueSizeCalculation(t *testing.T) {\n\ttype testType = [16]byte\n\tvar testValue testType\n\n\tcalc := ipqSizeCalculation[testType](func(e testType) uint64 {\n\t\treturn uint64(len(e))\n\t})\n\ts := &Server{}\n\tq := newIPQueue[testType](s, \"test\", calc)\n\n\tfor i := 0; i < 10; i++ {\n\t\tq.push(testValue)\n\t\trequire_Equal(t, q.len(), i+1)\n\t\trequire_Equal(t, q.size(), uint64(i+1)*uint64(len(testValue)))\n\t}\n\n\tfor i := 10; i > 5; i-- {\n\t\tq.popOne()\n\t\trequire_Equal(t, q.len(), i-1)\n\t\trequire_Equal(t, q.size(), uint64(i-1)*uint64(len(testValue)))\n\t}\n\n\tq.pop()\n\trequire_Equal(t, q.len(), 0)\n\trequire_Equal(t, q.size(), 0)\n}\n\nfunc TestIPQueueSizeCalculationWithLimits(t *testing.T) {\n\ttype testType = [16]byte\n\tvar testValue testType\n\n\tcalc := ipqSizeCalculation[testType](func(e testType) uint64 {\n\t\treturn uint64(len(e))\n\t})\n\ts := &Server{}\n\n\tt.Run(\"LimitByLen\", func(t *testing.T) {\n\t\tq := newIPQueue[testType](s, \"test\", calc, ipqLimitByLen[testType](5))\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tn, err := q.push(testValue)\n\t\t\tif i >= 5 {\n\t\t\t\trequire_Error(t, err, errIPQLenLimitReached)\n\t\t\t} else {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\trequire_LessThan(t, n, 6)\n\t\t}\n\t})\n\n\tt.Run(\"LimitBySize\", func(t *testing.T) {\n\t\tq := newIPQueue[testType](s, \"test\", calc, ipqLimitBySize[testType](16*5))\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tn, err := q.push(testValue)\n\t\t\tif i >= 5 {\n\t\t\t\trequire_Error(t, err, errIPQSizeLimitReached)\n\t\t\t} else {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\trequire_LessThan(t, n, 6)\n\t\t}\n\t})\n}\n\nfunc Benchmark_IPQueueSizeCalculation(b *testing.B) {\n\ttype testType = [16]byte\n\tvar testValue testType\n\n\ts := &Server{}\n\n\trunPopOne := func(b *testing.B, q *ipQueue[testType]) {\n\t\tb.SetBytes(16)\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\tq.push(testValue)\n\t\t}\n\t\tfor i := b.N; i > 0; i-- {\n\t\t\tq.popOne()\n\t\t}\n\t}\n\trunPopAll := func(b *testing.B, q *ipQueue[testType]) {\n\t\tb.SetBytes(16)\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\tq.push(testValue)\n\t\t}\n\t\telts := q.pop()\n\t\tfor _, v := range elts {\n\t\t\t_ = v\n\t\t}\n\t\tq.recycle(&elts)\n\t}\n\tfor _, test := range []struct {\n\t\tname string\n\t\trun func(b *testing.B, q *ipQueue[testType])\n\t}{\n\t\t{\"pop one\", runPopOne},\n\t\t{\"pop all\", runPopAll},\n\t} {\n\t\tb.Run(test.name, func(b *testing.B) {\n\t\t\t// Measures without calculation function overheads.\n\t\t\tb.Run(\"WithoutCalc\", func(b *testing.B) {\n\t\t\t\ttest.run(b, newIPQueue[testType](s, \"test\"))\n\t\t\t})\n\n\t\t\t// Measures the raw overhead of having a calculation function.\n\t\t\tb.Run(\"WithEmptyCalc\", func(b *testing.B) {\n\t\t\t\tcalc := ipqSizeCalculation[testType](func(e testType) uint64 {\n\t\t\t\t\treturn 0\n\t\t\t\t})\n\t\t\t\ttest.run(b, newIPQueue[testType](s, \"test\", calc))\n\t\t\t})\n\n\t\t\t// Measures the overhead of having a calculation function that\n\t\t\t// actually measures something useful.\n\t\t\tb.Run(\"WithLenCalc\", func(b *testing.B) {\n\t\t\t\tcalc := ipqSizeCalculation[testType](func(e testType) uint64 {\n\t\t\t\t\treturn uint64(len(e))\n\t\t\t\t})\n\t\t\t\ttest.run(b, newIPQueue[testType](s, \"test\", calc))\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc Benchmark_IPQueue1Prod1ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 1, 1, true, false)\n}\n\nfunc Benchmark_IPQueue5Prod1ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 5, 1, true, false)\n}\n\nfunc Benchmark_IPQueue1Prod5ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 1, 5, true, false)\n}\n\nfunc Benchmark_IPQueue5Prod5ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 5, 5, true, false)\n}\n\nfunc Benchmark_IPQueue1Prod1ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 1, 1, false, false)\n}\n\nfunc Benchmark_IPQueue5Prod1ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 5, 1, false, false)\n}\n\nfunc Benchmark_IPQueue1Prod5ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 1, 5, false, false)\n}\n\nfunc Benchmark_IPQueue5Prod5ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 5, 5, false, false)\n}\n\nfunc Benchmark_IPQueueWithLim1Prod1ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 1, 1, true, true)\n}\n\nfunc Benchmark_IPQueueWithLim5Prod1ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 5, 1, true, true)\n}\n\nfunc Benchmark_IPQueueWithLim1Prod5ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 1, 5, true, true)\n}\n\nfunc Benchmark_IPQueueWithLim5Prod5ConsPopOne(b *testing.B) {\n\tbenchIPQueue(b, 5, 5, true, true)\n}\n\nfunc Benchmark_IPQueueWithLim1Prod1ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 1, 1, false, true)\n}\n\nfunc Benchmark_IPQueueWithLim5Prod1ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 5, 1, false, true)\n}\n\nfunc Benchmark_IPQueueWithLim1Prod5ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 1, 5, false, true)\n}\n\nfunc Benchmark_IPQueueWithLim5Prod5ConsPopAll(b *testing.B) {\n\tbenchIPQueue(b, 5, 5, false, true)\n}\n\nfunc benchIPQueue(b *testing.B, numProd, numCons int, popOne, withLimits bool) {\n\tvar count atomic.Int64\n\tvar sum atomic.Int64\n\n\tperProd := (b.N / numProd) + 1\n\ttotal := int64(perProd * numProd)\n\n\tvar opts []ipQueueOpt[int]\n\tif withLimits {\n\t\t// Add len/size options but make sure they are big enough that we don't fail push() calls.\n\t\t// What we want is to activate code that is checking for limits.\n\t\topts = append(opts, ipqLimitByLen[int](int(total+100)))\n\t\topts = append(opts, ipqLimitBySize[int](uint64((total+100)*5)))\n\t\topts = append(opts, ipqSizeCalculation[int](func(e int) uint64 { return uint64(5) }))\n\t}\n\tqueue := newIPQueue[int](&Server{}, \"bench\", opts...)\n\n\tcdone := make(chan struct{}, numCons)\n\tfor i := 0; i < numCons; i++ {\n\t\tgo func() {\n\t\t\tfor range queue.ch {\n\t\t\t\tif popOne {\n\t\t\t\t\tv, ok := queue.popOne()\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tsum.Add(int64(v))\n\t\t\t\t\tif count.Add(1) >= total {\n\t\t\t\t\t\tcdone <- struct{}{}\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\telts := queue.pop()\n\t\t\t\t\tfor _, v := range elts {\n\t\t\t\t\t\tsum.Add(int64(v))\n\t\t\t\t\t\tif count.Add(1) >= total {\n\t\t\t\t\t\t\tqueue.recycle(&elts)\n\t\t\t\t\t\t\tcdone <- struct{}{}\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tqueue.recycle(&elts)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tpwg := sync.WaitGroup{}\n\tpwg.Add(numProd)\n\tvar expected atomic.Int64\n\tfor i := 0; i < numProd; i++ {\n\t\tgo func() {\n\t\t\tdefer pwg.Done()\n\t\t\tfor i := 0; i < perProd; i++ {\n\t\t\t\texpected.Add(int64(i + 1))\n\t\t\t\tqueue.push(i + 1)\n\t\t\t}\n\t\t}()\n\t}\n\n\tpwg.Wait()\n\t<-cdone\n\tb.StopTimer()\n\n\tfor i := 1; i < numCons; i++ {\n\t\tqueue.push(0)\n\t\t<-cdone\n\t}\n\tif sum.Load() != expected.Load() {\n\t\tb.Fatalf(\"Invalid sum, expected %v, got %v\", expected.Load(), sum.Load())\n\t}\n}\n\nfunc Benchmark_IPQueuePushAndPopOne(b *testing.B) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\tfor i := 0; i < b.N; i++ {\n\t\tq.push(i + 1)\n\t}\n\trequire_Equal[int](b, q.len(), b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tv, ok := q.popOne()\n\t\trequire_True(b, ok)\n\t\trequire_Equal[int](b, v, i+1)\n\t}\n\trequire_Equal[int](b, q.len(), 0)\n\trequire_Equal(b, q.inProgress(), 0)\n}\n\nfunc Benchmark_IPQueuePushAndPopAll(b *testing.B) {\n\ts := &Server{}\n\tq := newIPQueue[int](s, \"test\")\n\texpected := 0\n\tfor i := 0; i < b.N; i++ {\n\t\tq.push(i + 1)\n\t\texpected += i + 1\n\t}\n\trequire_Equal(b, q.len(), b.N)\n\t<-q.ch\n\n\telts := q.pop()\n\trequire_Equal(b, len(elts), b.N)\n\trequire_Equal(b, q.len(), 0)\n\trequire_Equal(b, q.inProgress(), int64(b.N))\n\tsum := 0\n\tfor _, v := range elts {\n\t\tsum += v\n\t}\n\tq.recycle(&elts)\n\trequire_Equal(b, expected, sum)\n\trequire_Equal(b, q.len(), 0)\n\trequire_Equal(b, q.inProgress(), 0)\n}\n" }, { "path": "server/jetstream.go", "content": "// Copyright 2019-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/hmac\"\n\t\"crypto/sha256\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime/debug\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/minio/highwayhash\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nats-server/v2/server/sysmem\"\n\t\"github.com/nats-io/nats-server/v2/server/tpm\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// JetStreamConfig determines this server's configuration.\n// MaxMemory and MaxStore are in bytes.\ntype JetStreamConfig struct {\n\tMaxMemory int64 `json:\"max_memory\"` // MaxMemory is the maximum size of memory type streams\n\tMaxStore int64 `json:\"max_storage\"` // MaxStore is the maximum size of file store type streams\n\tStoreDir string `json:\"store_dir,omitempty\"` // StoreDir is where storage files are stored\n\tSyncInterval time.Duration `json:\"sync_interval,omitempty\"` // SyncInterval is how frequently we sync to disk in the background by calling fsync\n\tSyncAlways bool `json:\"sync_always,omitempty\"` // SyncAlways indicates flushes are done after every write\n\tDomain string `json:\"domain,omitempty\"` // Domain is the JetStream domain\n\tCompressOK bool `json:\"compress_ok,omitempty\"` // CompressOK indicates if compression is supported\n\tUniqueTag string `json:\"unique_tag,omitempty\"` // UniqueTag is the unique tag assigned to this instance\n\tStrict bool `json:\"strict,omitempty\"` // Strict indicates if strict JSON parsing is performed\n}\n\n// Statistics about JetStream for this server.\ntype JetStreamStats struct {\n\tMemory uint64 `json:\"memory\"`\n\tStore uint64 `json:\"storage\"`\n\tReservedMemory uint64 `json:\"reserved_memory\"`\n\tReservedStore uint64 `json:\"reserved_storage\"`\n\tAccounts int `json:\"accounts\"`\n\tHAAssets int `json:\"ha_assets\"`\n\tAPI JetStreamAPIStats `json:\"api\"`\n}\n\ntype JetStreamAccountLimits struct {\n\tMaxMemory int64 `json:\"max_memory\"`\n\tMaxStore int64 `json:\"max_storage\"`\n\tMaxStreams int `json:\"max_streams\"`\n\tMaxConsumers int `json:\"max_consumers\"`\n\tMaxAckPending int `json:\"max_ack_pending\"`\n\tMemoryMaxStreamBytes int64 `json:\"memory_max_stream_bytes\"`\n\tStoreMaxStreamBytes int64 `json:\"storage_max_stream_bytes\"`\n\tMaxBytesRequired bool `json:\"max_bytes_required\"`\n}\n\ntype JetStreamTier struct {\n\tMemory uint64 `json:\"memory\"`\n\tStore uint64 `json:\"storage\"`\n\tReservedMemory uint64 `json:\"reserved_memory\"`\n\tReservedStore uint64 `json:\"reserved_storage\"`\n\tStreams int `json:\"streams\"`\n\tConsumers int `json:\"consumers\"`\n\tLimits JetStreamAccountLimits `json:\"limits\"`\n}\n\n// JetStreamAccountStats returns current statistics about the account's JetStream usage.\ntype JetStreamAccountStats struct {\n\tJetStreamTier // in case tiers are used, reflects totals with limits not set\n\tDomain string `json:\"domain,omitempty\"`\n\tAPI JetStreamAPIStats `json:\"api\"`\n\tTiers map[string]JetStreamTier `json:\"tiers,omitempty\"` // indexed by tier name\n}\n\n// JetStreamAPIStats holds stats about the API usage for this server\ntype JetStreamAPIStats struct {\n\tLevel int `json:\"level\"` // Level is the active API level this server implements\n\tTotal uint64 `json:\"total\"` // Total is the total API requests received since start\n\tErrors uint64 `json:\"errors\"` // Errors is the total API requests that resulted in error responses\n\tInflight uint64 `json:\"inflight,omitempty\"` // Inflight are the number of API requests currently being served\n}\n\n// This is for internal accounting for JetStream for this server.\ntype jetStream struct {\n\t// These are here first because of atomics on 32bit systems.\n\tapiInflight int64\n\tapiTotal int64\n\tapiErrors int64\n\tmemReserved int64\n\tstoreReserved int64\n\tmemUsed int64\n\tstoreUsed int64\n\tqueueLimit int64\n\tinfoQueueLimit int64\n\tclustered int32\n\tmu sync.RWMutex\n\tsrv *Server\n\tconfig JetStreamConfig\n\tcluster *jetStreamCluster\n\taccounts map[string]*jsAccount\n\tapiSubs *Sublist\n\tinfoSubs *gsl.SimpleSublist // Subjects for info-specific queue.\n\tstarted time.Time\n\n\t// System level request to purge a stream move\n\taccountPurge *subscription\n\n\t// Some bools regarding general state.\n\tmetaRecovering bool\n\tstandAlone bool\n\toos bool\n\tshuttingDown bool\n\n\t// Atomic versions\n\tdisabled atomic.Bool\n}\n\ntype remoteUsage struct {\n\ttiers map[string]*jsaUsage // indexed by tier name\n\tapi uint64\n\terr uint64\n}\n\ntype jsaStorage struct {\n\ttotal jsaUsage\n\tlocal jsaUsage\n}\n\n// This represents a jetstream enabled account.\n// Worth noting that we include the jetstream pointer, this is because\n// in general we want to be very efficient when receiving messages on\n// an internal sub for a stream, so we will direct link to the stream\n// and walk backwards as needed vs multiple hash lookups and locks, etc.\ntype jsAccount struct {\n\tmu sync.RWMutex\n\tjs *jetStream\n\taccount *Account\n\tstoreDir string\n\tinflight sync.Map\n\tstreams map[string]*stream\n\n\t// From server\n\tsendq *ipQueue[*pubMsg]\n\n\t// For limiting only running one checkAndSync at a time.\n\tsync atomic.Bool\n\n\t// Usage/limits related fields that will be protected by usageMu\n\tusageMu sync.RWMutex\n\tlimits map[string]JetStreamAccountLimits // indexed by tierName\n\tusage map[string]*jsaStorage // indexed by tierName\n\trusage map[string]*remoteUsage // indexed by node id\n\tapiTotal uint64\n\tapiErrors uint64\n\tusageApi uint64\n\tusageErr uint64\n\tupdatesPub string\n\tupdatesSub *subscription\n\tlupdate time.Time\n\tutimer *time.Timer\n}\n\n// Track general usage for this account.\ntype jsaUsage struct {\n\tmem int64\n\tstore int64\n}\n\n// EnableJetStream will enable JetStream support on this server with the given configuration.\n// A nil configuration will dynamically choose the limits and temporary file storage directory.\nfunc (s *Server) EnableJetStream(config *JetStreamConfig) error {\n\tif s.JetStreamEnabled() {\n\t\treturn fmt.Errorf(\"jetstream already enabled\")\n\t}\n\n\ts.Noticef(\"Starting JetStream\")\n\tstart := time.Now()\n\tdefer func() {\n\t\ts.Noticef(\"Took %s to start JetStream\", time.Since(start))\n\t}()\n\n\tif config == nil || config.MaxMemory <= 0 || config.MaxStore <= 0 {\n\t\tvar storeDir, domain, uniqueTag string\n\t\tvar maxStore, maxMem int64\n\t\tif config != nil {\n\t\t\tstoreDir, domain, uniqueTag = config.StoreDir, config.Domain, config.UniqueTag\n\t\t\tmaxStore, maxMem = config.MaxStore, config.MaxMemory\n\t\t}\n\t\tconfig = s.dynJetStreamConfig(storeDir, maxStore, maxMem)\n\t\tif maxMem > 0 {\n\t\t\tconfig.MaxMemory = maxMem\n\t\t}\n\t\tif domain != _EMPTY_ {\n\t\t\tconfig.Domain = domain\n\t\t}\n\t\tif uniqueTag != _EMPTY_ {\n\t\t\tconfig.UniqueTag = uniqueTag\n\t\t}\n\t\ts.Debugf(\"JetStream creating dynamic configuration - %s memory, %s disk\", friendlyBytes(config.MaxMemory), friendlyBytes(config.MaxStore))\n\t} else if config.StoreDir != _EMPTY_ {\n\t\tconfig.StoreDir = filepath.Join(config.StoreDir, JetStreamStoreDir)\n\t}\n\n\tcfg := *config\n\tif cfg.StoreDir == _EMPTY_ {\n\t\tcfg.StoreDir = filepath.Join(os.TempDir(), JetStreamStoreDir)\n\t\ts.Warnf(\"Temporary storage directory used, data could be lost on system reboot\")\n\t}\n\n\t// We will consistently place the 'jetstream' directory under the storedir that was handed to us. Prior to 2.2.3 though\n\t// we could have a directory on disk without the 'jetstream' directory. This will check and fix if needed.\n\tif err := s.checkStoreDir(&cfg); err != nil {\n\t\treturn err\n\t}\n\n\treturn s.enableJetStream(cfg)\n}\n\n// Function signature to generate a key encryption key.\ntype keyGen func(context []byte) ([]byte, error)\n\n// Return a key generation function or nil if encryption not enabled.\nfunc (s *Server) jsKeyGen(jsKey, info string) keyGen {\n\tif ek := jsKey; ek != _EMPTY_ {\n\t\treturn func(context []byte) ([]byte, error) {\n\t\t\th := hmac.New(sha256.New, []byte(ek))\n\t\t\tif _, err := h.Write([]byte(info)); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif _, err := h.Write(context); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn h.Sum(nil), nil\n\t\t}\n\t}\n\treturn nil\n}\n\n// Decode the encrypted metafile.\nfunc (s *Server) decryptMeta(sc StoreCipher, ekey, buf []byte, acc, context string) ([]byte, bool, error) {\n\tif len(ekey) < minMetaKeySize {\n\t\treturn nil, false, errBadKeySize\n\t}\n\tvar osc StoreCipher\n\tswitch sc {\n\tcase AES:\n\t\tosc = ChaCha\n\tcase ChaCha:\n\t\tosc = AES\n\t}\n\ttype prfWithCipher struct {\n\t\tkeyGen\n\t\tStoreCipher\n\t}\n\tvar prfs []prfWithCipher\n\tif prf := s.jsKeyGen(s.getOpts().JetStreamKey, acc); prf == nil {\n\t\treturn nil, false, errNoEncryption\n\t} else {\n\t\t// First of all, try our current encryption keys with both\n\t\t// store cipher algorithms.\n\t\tprfs = append(prfs, prfWithCipher{prf, sc})\n\t\tprfs = append(prfs, prfWithCipher{prf, osc})\n\t}\n\tif prf := s.jsKeyGen(s.getOpts().JetStreamOldKey, acc); prf != nil {\n\t\t// Then, if we have an old encryption key, try with also with\n\t\t// both store cipher algorithms.\n\t\tprfs = append(prfs, prfWithCipher{prf, sc})\n\t\tprfs = append(prfs, prfWithCipher{prf, osc})\n\t}\n\n\tfor i, prf := range prfs {\n\t\trb, err := prf.keyGen([]byte(context))\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tkek, err := genEncryptionKey(prf.StoreCipher, rb)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tns := kek.NonceSize()\n\t\tseed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\taek, err := genEncryptionKey(prf.StoreCipher, seed)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif aek.NonceSize() != kek.NonceSize() {\n\t\t\tcontinue\n\t\t}\n\t\tplain, err := aek.Open(nil, buf[:ns], buf[ns:], nil)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\treturn plain, i > 0, nil\n\t}\n\treturn nil, false, fmt.Errorf(\"unable to recover keys\")\n}\n\n// Check to make sure directory has the jetstream directory.\n// We will have it properly configured here now regardless, so need to look inside.\nfunc (s *Server) checkStoreDir(cfg *JetStreamConfig) error {\n\tfis, _ := os.ReadDir(cfg.StoreDir)\n\t// If we have nothing underneath us, could be just starting new, but if we see this we can check.\n\tif len(fis) != 0 {\n\t\treturn nil\n\t}\n\t// Let's check the directory above. If it has us 'jetstream' but also other stuff that we can\n\t// identify as accounts then we can fix.\n\tfis, _ = os.ReadDir(filepath.Dir(cfg.StoreDir))\n\t// If just one that is us 'jetstream' and all is ok.\n\tif len(fis) == 1 {\n\t\treturn nil\n\t}\n\n\thaveJetstreamDir := false\n\tfor _, fi := range fis {\n\t\tif fi.Name() == JetStreamStoreDir {\n\t\t\thaveJetstreamDir = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\tfor _, fi := range fis {\n\t\t// Skip the 'jetstream' directory.\n\t\tif fi.Name() == JetStreamStoreDir {\n\t\t\tcontinue\n\t\t}\n\t\t// Let's see if this is an account.\n\t\tif accName := fi.Name(); accName != _EMPTY_ {\n\t\t\t_, ok := s.accounts.Load(accName)\n\t\t\tif !ok && s.AccountResolver() != nil && nkeys.IsValidPublicAccountKey(accName) {\n\t\t\t\t// Account is not local but matches the NKEY account public key,\n\t\t\t\t// this is enough indication to move this directory, no need to\n\t\t\t\t// fetch the account.\n\t\t\t\tok = true\n\t\t\t}\n\t\t\t// If this seems to be an account go ahead and move the directory. This will include all assets\n\t\t\t// like streams and consumers.\n\t\t\tif ok {\n\t\t\t\tif !haveJetstreamDir {\n\t\t\t\t\terr := os.Mkdir(filepath.Join(filepath.Dir(cfg.StoreDir), JetStreamStoreDir), defaultDirPerms)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\thaveJetstreamDir = true\n\t\t\t\t}\n\t\t\t\told := filepath.Join(filepath.Dir(cfg.StoreDir), fi.Name())\n\t\t\t\tnew := filepath.Join(cfg.StoreDir, fi.Name())\n\t\t\t\ts.Noticef(\"JetStream relocated account %q to %q\", old, new)\n\t\t\t\tif err := os.Rename(old, new); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// This function sets/updates the jetstream encryption key and cipher based\n// on options. If the TPM options have been specified, a key is generated\n// and sealed by the TPM.\nfunc (s *Server) initJetStreamEncryption() (err error) {\n\topts := s.getOpts()\n\n\t// The TPM settings and other encryption settings are mutually exclusive.\n\tif opts.JetStreamKey != _EMPTY_ && opts.JetStreamTpm.KeysFile != _EMPTY_ {\n\t\treturn fmt.Errorf(\"JetStream encryption key may not be used with TPM options\")\n\t}\n\t// if we are using the standard method to set the encryption key just return and carry on.\n\tif opts.JetStreamKey != _EMPTY_ {\n\t\treturn nil\n\t}\n\t// if the tpm options are not used then no encryption has been configured and return.\n\tif opts.JetStreamTpm.KeysFile == _EMPTY_ {\n\t\treturn nil\n\t}\n\n\tif opts.JetStreamTpm.Pcr == 0 {\n\t\t// Default PCR to use in the TPM. Values can be 0-23, and most platforms\n\t\t// reserve values 0-12 for the OS, boot locker, disc encryption, etc.\n\t\t// 16 used for debugging. In sticking to NATS tradition, we'll use 22\n\t\t// as the default with the option being configurable.\n\t\topts.JetStreamTpm.Pcr = 22\n\t}\n\n\t// Using the TPM to generate or get the encryption key and update the encryption options.\n\topts.JetStreamKey, err = tpm.LoadJetStreamEncryptionKeyFromTPM(opts.JetStreamTpm.SrkPassword,\n\t\topts.JetStreamTpm.KeysFile, opts.JetStreamTpm.KeyPassword, opts.JetStreamTpm.Pcr)\n\n\treturn err\n}\n\n// enableJetStream will start up the JetStream subsystem.\nfunc (s *Server) enableJetStream(cfg JetStreamConfig) error {\n\tjs := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache(), infoSubs: gsl.NewSimpleSublist()}\n\ts.gcbMu.Lock()\n\tif s.gcbOutMax = s.getOpts().JetStreamMaxCatchup; s.gcbOutMax == 0 {\n\t\ts.gcbOutMax = defaultMaxTotalCatchupOutBytes\n\t}\n\ts.gcbMu.Unlock()\n\n\t// TODO: Not currently reloadable.\n\tatomic.StoreInt64(&js.queueLimit, s.getOpts().JetStreamRequestQueueLimit)\n\tatomic.StoreInt64(&js.infoQueueLimit, s.getOpts().JetStreamInfoQueueLimit)\n\n\ts.js.Store(js)\n\n\t// FIXME(dlc) - Allow memory only operation?\n\tif stat, err := os.Stat(cfg.StoreDir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(cfg.StoreDir, defaultDirPerms); err != nil {\n\t\t\treturn fmt.Errorf(\"could not create storage directory - %v\", err)\n\t\t}\n\t} else {\n\t\t// Make sure its a directory and that we can write to it.\n\t\tif stat == nil || !stat.IsDir() {\n\t\t\treturn fmt.Errorf(\"storage directory is not a directory\")\n\t\t}\n\t\ttmpfile, err := os.CreateTemp(cfg.StoreDir, \"_test_\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"storage directory is not writable\")\n\t\t}\n\t\ttmpfile.Close()\n\t\tos.Remove(tmpfile.Name())\n\t}\n\n\tif err := s.initJetStreamEncryption(); err != nil {\n\t\treturn err\n\t}\n\n\t// JetStream is an internal service so we need to make sure we have a system account.\n\t// This system account will export the JetStream service endpoints.\n\tif s.SystemAccount() == nil {\n\t\ts.SetDefaultSystemAccount()\n\t}\n\n\topts := s.getOpts()\n\tif !opts.DisableJetStreamBanner {\n\t\ts.Noticef(\" _ ___ _____ ___ _____ ___ ___ _ __ __\")\n\t\ts.Noticef(\" _ | | __|_ _/ __|_ _| _ \\\\ __| /_\\\\ | \\\\/ |\")\n\t\ts.Noticef(\"| || | _| | | \\\\__ \\\\ | | | / _| / _ \\\\| |\\\\/| |\")\n\t\ts.Noticef(\" \\\\__/|___| |_| |___/ |_| |_|_\\\\___/_/ \\\\_\\\\_| |_|\")\n\t\ts.Noticef(\"\")\n\t\ts.Noticef(\" https://docs.nats.io/jetstream\")\n\t\ts.Noticef(\"\")\n\t}\n\ts.Noticef(\"---------------- JETSTREAM ----------------\")\n\n\tif cfg.Strict {\n\t\ts.Noticef(\" Strict: %t\", cfg.Strict)\n\t}\n\n\ts.Noticef(\" Max Memory: %s\", friendlyBytes(cfg.MaxMemory))\n\ts.Noticef(\" Max Storage: %s\", friendlyBytes(cfg.MaxStore))\n\ts.Noticef(\" Store Directory: \\\"%s\\\"\", cfg.StoreDir)\n\tif cfg.Domain != _EMPTY_ {\n\t\ts.Noticef(\" Domain: %s\", cfg.Domain)\n\t}\n\n\tif ek := opts.JetStreamKey; ek != _EMPTY_ {\n\t\ts.Noticef(\" Encryption: %s\", opts.JetStreamCipher)\n\t}\n\tif opts.JetStreamTpm.KeysFile != _EMPTY_ {\n\t\ts.Noticef(\" TPM File: %q, Pcr: %d\", opts.JetStreamTpm.KeysFile,\n\t\t\topts.JetStreamTpm.Pcr)\n\t}\n\ts.Noticef(\" API Level: %d\", JSApiLevel)\n\ts.Noticef(\"-------------------------------------------\")\n\n\t// Setup our internal subscriptions.\n\tif err := s.setJetStreamExportSubs(); err != nil {\n\t\treturn fmt.Errorf(\"setting up internal jetstream subscriptions failed: %v\", err)\n\t}\n\n\t// Setup our internal system exports.\n\ts.Debugf(\" Exports:\")\n\ts.Debugf(\" %s\", jsAllAPI)\n\ts.setupJetStreamExports()\n\n\tstandAlone, canExtend := s.standAloneMode(), s.canExtendOtherDomain()\n\tif standAlone && canExtend && s.getOpts().JetStreamExtHint != jsWillExtend {\n\t\tcanExtend = false\n\t\ts.Noticef(\"Standalone server started in clustered mode do not support extending domains\")\n\t\ts.Noticef(`Manually disable standalone mode by setting the JetStream Option \"extension_hint: %s\"`, jsWillExtend)\n\t}\n\n\t// Indicate if we will be standalone for checking resource reservations, etc.\n\tjs.setJetStreamStandAlone(standAlone && !canExtend)\n\n\t// Enable accounts and restore state before starting clustering.\n\tif err := s.enableJetStreamAccounts(); err != nil {\n\t\treturn err\n\t}\n\n\t// If we are in clustered mode go ahead and start the meta controller.\n\tif !standAlone || canExtend {\n\t\tif err := s.enableJetStreamClustering(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Set our atomic bool to clustered.\n\t\ts.jsClustered.Store(true)\n\t}\n\n\t// Mark when we are up and running.\n\tjs.setStarted()\n\n\treturn nil\n}\n\nconst jsNoExtend = \"no_extend\"\nconst jsWillExtend = \"will_extend\"\n\n// This will check if we have a solicited leafnode that shares the system account\n// and extension is not manually disabled\nfunc (s *Server) canExtendOtherDomain() bool {\n\topts := s.getOpts()\n\tsysAcc := s.SystemAccount().GetName()\n\tfor _, r := range opts.LeafNode.Remotes {\n\t\tif r.LocalAccount == sysAcc {\n\t\t\tfor _, denySub := range r.DenyImports {\n\t\t\t\tif subjectIsSubsetMatch(denySub, raftAllSubj) {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (s *Server) updateJetStreamInfoStatus(enabled bool) {\n\ts.mu.Lock()\n\ts.info.JetStream = enabled\n\ts.mu.Unlock()\n}\n\n// restartJetStream will try to re-enable JetStream during a reload if it had been disabled during runtime.\nfunc (s *Server) restartJetStream() error {\n\topts := s.getOpts()\n\tcfg := JetStreamConfig{\n\t\tStoreDir: opts.StoreDir,\n\t\tSyncInterval: opts.SyncInterval,\n\t\tSyncAlways: opts.SyncAlways,\n\t\tMaxMemory: opts.JetStreamMaxMemory,\n\t\tMaxStore: opts.JetStreamMaxStore,\n\t\tDomain: opts.JetStreamDomain,\n\t\tStrict: !opts.NoJetStreamStrict,\n\t}\n\ts.Noticef(\"Restarting JetStream\")\n\terr := s.EnableJetStream(&cfg)\n\tif err != nil {\n\t\ts.Warnf(\"Can't start JetStream: %v\", err)\n\t\treturn s.DisableJetStream()\n\t}\n\ts.updateJetStreamInfoStatus(true)\n\treturn nil\n}\n\n// checkJetStreamExports will check if we have the JS exports setup\n// on the system account, and if not go ahead and set them up.\nfunc (s *Server) checkJetStreamExports() {\n\tif sacc := s.SystemAccount(); sacc != nil {\n\t\tsacc.mu.RLock()\n\t\tse := sacc.getServiceExport(jsAllAPI)\n\t\tsacc.mu.RUnlock()\n\t\tif se == nil {\n\t\t\ts.setupJetStreamExports()\n\t\t}\n\t}\n}\n\nfunc (s *Server) setupJetStreamExports() {\n\t// Setup our internal system export.\n\tif err := s.SystemAccount().AddServiceExport(jsAllAPI, nil); err != nil {\n\t\ts.Warnf(\"Error setting up jetstream service exports: %v\", err)\n\t}\n}\n\nfunc (s *Server) jetStreamOOSPending() (wasPending bool) {\n\tif js := s.getJetStream(); js != nil {\n\t\tjs.mu.Lock()\n\t\twasPending = js.oos\n\t\tjs.oos = true\n\t\tjs.mu.Unlock()\n\t}\n\treturn wasPending\n}\n\nfunc (s *Server) setJetStreamDisabled() {\n\tif js := s.getJetStream(); js != nil {\n\t\tjs.disabled.Store(true)\n\t}\n}\n\nfunc (s *Server) handleOutOfSpace(mset *stream) {\n\tif s.JetStreamEnabled() && !s.jetStreamOOSPending() {\n\t\tvar stream string\n\t\tif mset != nil {\n\t\t\tstream = mset.name()\n\t\t\ts.Errorf(\"JetStream out of %s resources, will be DISABLED\", mset.Store().Type())\n\t\t} else {\n\t\t\ts.Errorf(\"JetStream out of resources, will be DISABLED\")\n\t\t}\n\n\t\tgo s.DisableJetStream()\n\n\t\tadv := &JSServerOutOfSpaceAdvisory{\n\t\t\tTypedEvent: TypedEvent{\n\t\t\t\tType: JSServerOutOfStorageAdvisoryType,\n\t\t\t\tID: nuid.Next(),\n\t\t\t\tTime: time.Now().UTC(),\n\t\t\t},\n\t\t\tServer: s.Name(),\n\t\t\tServerID: s.ID(),\n\t\t\tStream: stream,\n\t\t\tCluster: s.cachedClusterName(),\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t}\n\t\ts.publishAdvisory(nil, JSAdvisoryServerOutOfStorage, adv)\n\t}\n}\n\n// DisableJetStream will turn off JetStream and signals in clustered mode\n// to have the metacontroller remove us from the peer list.\nfunc (s *Server) DisableJetStream() error {\n\tif !s.JetStreamEnabled() {\n\t\treturn nil\n\t}\n\n\ts.setJetStreamDisabled()\n\n\tif s.JetStreamIsClustered() {\n\t\tisLeader := s.JetStreamIsLeader()\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil {\n\t\t\ts.shutdownJetStream()\n\t\t\treturn nil\n\t\t}\n\t\tjs.mu.RLock()\n\t\tmeta := cc.meta\n\t\tjs.mu.RUnlock()\n\n\t\tif meta != nil {\n\t\t\tif isLeader {\n\t\t\t\ts.Warnf(\"JetStream initiating meta leader transfer\")\n\t\t\t\tmeta.StepDown()\n\t\t\t\tselect {\n\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\treturn nil\n\t\t\t\tcase <-time.After(2 * time.Second):\n\t\t\t\t}\n\t\t\t\tif !s.JetStreamIsCurrent() {\n\t\t\t\t\ts.Warnf(\"JetStream timeout waiting for meta leader transfer\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tmeta.Delete()\n\t\t}\n\t}\n\n\t// Update our info status.\n\ts.updateJetStreamInfoStatus(false)\n\n\t// Normal shutdown.\n\ts.shutdownJetStream()\n\n\t// Shut down the RAFT groups.\n\ts.shutdownRaftNodes()\n\n\treturn nil\n}\n\nfunc (s *Server) enableJetStreamAccounts() error {\n\t// Reuse the same task workers across all accounts, so that we don't explode\n\t// with a large number of goroutines on multi-account systems.\n\ttq := parallelTaskQueue(len(dios))\n\tdefer close(tq)\n\n\t// If we have no configured accounts setup then setup imports on global account.\n\tif s.globalAccountOnly() {\n\t\tgacc := s.GlobalAccount()\n\t\tgacc.mu.Lock()\n\t\tif len(gacc.jsLimits) == 0 {\n\t\t\tgacc.jsLimits = defaultJSAccountTiers\n\t\t}\n\t\tgacc.mu.Unlock()\n\t\tif err := s.configJetStream(gacc, tq); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if err := s.configAllJetStreamAccounts(tq); err != nil {\n\t\treturn fmt.Errorf(\"Error enabling jetstream on configured accounts: %v\", err)\n\t}\n\treturn nil\n}\n\n// enableAllJetStreamServiceImportsAndMappings turns on all service imports and mappings for jetstream for this account.\nfunc (a *Account) enableAllJetStreamServiceImportsAndMappings() error {\n\ta.mu.RLock()\n\ts := a.srv\n\ta.mu.RUnlock()\n\n\tif s == nil {\n\t\treturn fmt.Errorf(\"jetstream account not registered\")\n\t}\n\n\tvar dstAccName string\n\tif sacc := s.SystemAccount(); sacc != nil {\n\t\tdstAccName = sacc.Name\n\t}\n\n\tif !a.serviceImportExists(dstAccName, jsAllAPI) {\n\t\t// Capture si so we can turn on implicit sharing with JetStream layer.\n\t\t// Make sure to set \"to\" otherwise will incur performance slow down.\n\t\tsi, err := a.addServiceImport(s.SystemAccount(), jsAllAPI, jsAllAPI, nil)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Error setting up jetstream service imports for account: %v\", err)\n\t\t}\n\t\ta.mu.Lock()\n\t\tsi.share = true\n\t\ta.mu.Unlock()\n\t}\n\n\t// Check if we have a Domain specified.\n\t// If so add in a subject mapping that will allow local connected clients to reach us here as well.\n\tif opts := s.getOpts(); opts.JetStreamDomain != _EMPTY_ {\n\t\tmappings := generateJSMappingTable(opts.JetStreamDomain)\n\t\ta.mu.RLock()\n\t\tfor _, m := range a.mappings {\n\t\t\tdelete(mappings, m.src)\n\t\t}\n\t\ta.mu.RUnlock()\n\t\tfor src, dest := range mappings {\n\t\t\tif err := a.AddMapping(src, dest); err != nil {\n\t\t\t\ts.Errorf(\"Error adding JetStream domain mapping: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// enableJetStreamInfoServiceImportOnly will enable the single service import responder.\n// Should we do them all regardless?\nfunc (a *Account) enableJetStreamInfoServiceImportOnly() error {\n\t// Check if this import would be overshadowed. This can happen when accounts\n\t// are importing from another account for JS access.\n\tif a.serviceImportShadowed(JSApiAccountInfo) {\n\t\treturn nil\n\t}\n\n\treturn a.enableAllJetStreamServiceImportsAndMappings()\n}\n\nfunc (s *Server) configJetStream(acc *Account, tq chan<- func()) error {\n\tif acc == nil {\n\t\treturn nil\n\t}\n\tacc.mu.RLock()\n\tjsLimits := acc.jsLimits\n\tacc.mu.RUnlock()\n\tif jsLimits != nil {\n\t\t// Check if already enabled. This can be during a reload.\n\t\tif acc.JetStreamEnabled() {\n\t\t\tif err := acc.enableAllJetStreamServiceImportsAndMappings(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := acc.UpdateJetStreamLimits(jsLimits); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tif err := acc.EnableJetStream(jsLimits, tq); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif s.gateway.enabled {\n\t\t\t\ts.switchAccountToInterestMode(acc.GetName())\n\t\t\t}\n\t\t}\n\t} else if acc != s.SystemAccount() {\n\t\tif acc.JetStreamEnabled() {\n\t\t\tacc.DisableJetStream()\n\t\t}\n\t\t// We will setup basic service imports to respond to\n\t\t// requests if JS is enabled for this account.\n\t\tif err := acc.enableJetStreamInfoServiceImportOnly(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// configAllJetStreamAccounts walk all configured accounts and turn on jetstream if requested.\nfunc (s *Server) configAllJetStreamAccounts(tq chan<- func()) error {\n\t// Check to see if system account has been enabled. We could arrive here via reload and\n\t// a non-default system account.\n\ts.checkJetStreamExports()\n\n\t// Bail if server not enabled. If it was enabled and a reload turns it off\n\t// that will be handled elsewhere.\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn nil\n\t}\n\n\t// Snapshot into our own list. Might not be needed.\n\ts.mu.RLock()\n\tif s.sys != nil {\n\t\t// clustered stream removal will perform this cleanup as well\n\t\t// this is mainly for initial cleanup\n\t\tsaccName := s.sys.account.Name\n\t\taccStoreDirs, _ := os.ReadDir(js.config.StoreDir)\n\t\tfor _, acc := range accStoreDirs {\n\t\t\tif accName := acc.Name(); accName != saccName {\n\t\t\t\t// no op if not empty\n\t\t\t\taccDir := filepath.Join(js.config.StoreDir, accName)\n\t\t\t\tos.Remove(filepath.Join(accDir, streamsDir))\n\t\t\t\tos.Remove(accDir)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar jsAccounts []*Account\n\ts.accounts.Range(func(k, v any) bool {\n\t\tjsAccounts = append(jsAccounts, v.(*Account))\n\t\treturn true\n\t})\n\taccounts := &s.accounts\n\ts.mu.RUnlock()\n\n\t// Process any jetstream enabled accounts here. These will be accounts we are\n\t// already aware of at startup etc.\n\tfor _, acc := range jsAccounts {\n\t\tif err := s.configJetStream(acc, tq); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Now walk all the storage we have and resolve any accounts that we did not process already.\n\t// This is important in resolver/operator models.\n\tfis, _ := os.ReadDir(js.config.StoreDir)\n\tfor _, fi := range fis {\n\t\tif accName := fi.Name(); accName != _EMPTY_ {\n\t\t\t// Only load up ones not already loaded since they are processed above.\n\t\t\tif _, ok := accounts.Load(accName); !ok {\n\t\t\t\tif acc, err := s.lookupAccount(accName); err != nil && acc != nil {\n\t\t\t\t\tif err := s.configJetStream(acc, tq); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Mark our started time.\nfunc (js *jetStream) setStarted() {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tjs.started = time.Now()\n}\n\nfunc (js *jetStream) isEnabled() bool {\n\tif js == nil {\n\t\treturn false\n\t}\n\treturn !js.disabled.Load()\n}\n\n// Mark that we will be in standlone mode.\nfunc (js *jetStream) setJetStreamStandAlone(isStandAlone bool) {\n\tif js == nil {\n\t\treturn\n\t}\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tif js.standAlone = isStandAlone; js.standAlone {\n\t\t// Update our server atomic.\n\t\tjs.srv.isMetaLeader.Store(true)\n\t\tjs.accountPurge, _ = js.srv.systemSubscribe(JSApiAccountPurge, _EMPTY_, false, nil, js.srv.jsLeaderAccountPurgeRequest)\n\t} else if js.accountPurge != nil {\n\t\tjs.srv.sysUnsubscribe(js.accountPurge)\n\t}\n}\n\n// JetStreamEnabled reports if jetstream is enabled for this server.\nfunc (s *Server) JetStreamEnabled() bool {\n\treturn s.getJetStream().isEnabled()\n}\n\n// JetStreamEnabledForDomain will report if any servers have JetStream enabled within this domain.\nfunc (s *Server) JetStreamEnabledForDomain() bool {\n\tif s.JetStreamEnabled() {\n\t\treturn true\n\t}\n\n\tvar jsFound bool\n\t// If we are here we do not have JetStream enabled for ourselves, but we need to check all connected servers.\n\t// TODO(dlc) - Could optimize and memoize this.\n\ts.nodeToInfo.Range(func(k, v any) bool {\n\t\t// This should not be dependent on online status, so only check js.\n\t\tif v.(nodeInfo).js {\n\t\t\tjsFound = true\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t})\n\n\treturn jsFound\n}\n\n// Will signal that all pull requests for consumers on this server are now invalid.\nfunc (s *Server) signalPullConsumers() {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\n\t// In case we have stale pending requests.\n\tconst hdr = \"NATS/1.0 409 Server Shutdown\\r\\n\" + JSPullRequestPendingMsgs + \": %d\\r\\n\" + JSPullRequestPendingBytes + \": %d\\r\\n\\r\\n\"\n\tvar didSend bool\n\n\tfor _, jsa := range js.accounts {\n\t\tjsa.mu.RLock()\n\t\tfor _, stream := range jsa.streams {\n\t\t\tstream.mu.RLock()\n\t\t\tfor _, o := range stream.consumers {\n\t\t\t\to.mu.RLock()\n\t\t\t\t// Only signal on R1.\n\t\t\t\tif o.cfg.Replicas <= 1 {\n\t\t\t\t\tfor reply, wr := range o.pendingRequests() {\n\t\t\t\t\t\tshdr := fmt.Sprintf(hdr, wr.n, wr.b)\n\t\t\t\t\t\to.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, []byte(shdr), nil, nil, 0))\n\t\t\t\t\t\tdidSend = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\to.mu.RUnlock()\n\t\t\t}\n\t\t\tstream.mu.RUnlock()\n\t\t}\n\t\tjsa.mu.RUnlock()\n\t}\n\t// Give time for migration information to make it out of our server.\n\tif didSend {\n\t\ttime.Sleep(50 * time.Millisecond)\n\t}\n}\n\n// Helper for determining if we are shutting down.\nfunc (js *jetStream) isShuttingDown() bool {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.shuttingDown\n}\n\n// Shutdown jetstream for this server.\nfunc (s *Server) shutdownJetStream() {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn\n\t}\n\n\ts.Noticef(\"Initiating JetStream Shutdown...\")\n\tdefer s.Noticef(\"JetStream Shutdown\")\n\n\t// If we have folks blocked on sync requests, unblock.\n\t// Send 1 is enough, but use select in case they were all present.\n\tselect {\n\tcase s.syncOutSem <- struct{}{}:\n\tdefault:\n\t}\n\n\tvar _a [512]*Account\n\taccounts := _a[:0]\n\n\tjs.mu.Lock()\n\t// Collect accounts.\n\tfor _, jsa := range js.accounts {\n\t\tif a := jsa.acc(); a != nil {\n\t\t\taccounts = append(accounts, a)\n\t\t}\n\t}\n\taccPurgeSub := js.accountPurge\n\tjs.accountPurge = nil\n\t// Signal we are shutting down.\n\tjs.shuttingDown = true\n\tjs.mu.Unlock()\n\n\tif accPurgeSub != nil {\n\t\ts.sysUnsubscribe(accPurgeSub)\n\t}\n\n\tfor _, a := range accounts {\n\t\ta.removeJetStream()\n\t}\n\n\ts.js.Store(nil)\n\n\tjs.mu.Lock()\n\tjs.accounts = nil\n\n\tvar qch chan struct{}\n\tvar stopped chan struct{}\n\tif cc := js.cluster; cc != nil {\n\t\tif cc.qch != nil {\n\t\t\tqch, stopped = cc.qch, cc.stopped\n\t\t\tcc.qch, cc.stopped = nil, nil\n\t\t}\n\t\tjs.stopUpdatesSub()\n\t\tif cc.c != nil {\n\t\t\tcc.c.closeConnection(ClientClosed)\n\t\t\tcc.c = nil\n\t\t}\n\t\tcc.meta = nil\n\t\t// Set our atomic bool to false.\n\t\ts.jsClustered.Store(false)\n\t}\n\tjs.mu.Unlock()\n\n\t// If we were clustered signal the monitor cluster go routine.\n\t// We will wait for a bit for it to close.\n\t// Do this without the lock.\n\tif qch != nil {\n\t\tclose(qch) // Must be close() to signal *all* listeners\n\t\tselect {\n\t\tcase <-stopped:\n\t\tcase <-time.After(10 * time.Second):\n\t\t\ts.Warnf(\"Did not receive signal for successful shutdown of cluster routine\")\n\t\t}\n\t}\n}\n\n// JetStreamConfig will return the current config. Useful if the system\n// created a dynamic configuration. A copy is returned.\nfunc (s *Server) JetStreamConfig() *JetStreamConfig {\n\tvar c *JetStreamConfig\n\tif js := s.getJetStream(); js != nil {\n\t\tcopy := js.config\n\t\tc = &(copy)\n\t}\n\treturn c\n}\n\n// StoreDir returns the current JetStream directory.\nfunc (s *Server) StoreDir() string {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn _EMPTY_\n\t}\n\treturn js.config.StoreDir\n}\n\n// JetStreamNumAccounts returns the number of enabled accounts this server is tracking.\nfunc (s *Server) JetStreamNumAccounts() int {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn 0\n\t}\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\treturn len(js.accounts)\n}\n\n// JetStreamReservedResources returns the reserved resources if JetStream is enabled.\nfunc (s *Server) JetStreamReservedResources() (int64, int64, error) {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn -1, -1, NewJSNotEnabledForAccountError()\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.memReserved, js.storeReserved, nil\n}\n\nfunc (s *Server) getJetStream() *jetStream {\n\treturn s.js.Load()\n}\n\nfunc (a *Account) assignJetStreamLimits(limits map[string]JetStreamAccountLimits) {\n\ta.mu.Lock()\n\ta.jsLimits = limits\n\ta.mu.Unlock()\n}\n\n// EnableJetStream will enable JetStream on this account with the defined limits.\n// This is a helper for JetStreamEnableAccount.\nfunc (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq chan<- func()) error {\n\ta.mu.RLock()\n\ts := a.srv\n\ta.mu.RUnlock()\n\n\tif s == nil {\n\t\treturn fmt.Errorf(\"jetstream account not registered\")\n\t}\n\n\tif s.SystemAccount() == a {\n\t\treturn fmt.Errorf(\"jetstream can not be enabled on the system account\")\n\t}\n\n\ts.mu.RLock()\n\tif s.sys == nil {\n\t\ts.mu.RUnlock()\n\t\treturn ErrServerNotRunning\n\t}\n\tsendq := s.sys.sendq\n\ts.mu.RUnlock()\n\n\t// No limits means we dynamically set up limits.\n\t// We also place limits here so we know that the account is configured for JetStream.\n\tif len(limits) == 0 {\n\t\tlimits = defaultJSAccountTiers\n\t}\n\n\ta.assignJetStreamLimits(limits)\n\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\n\tjs.mu.Lock()\n\t// Accounts get reset to nil on shutdown, since we re-acquire the locks here, we need to check again.\n\tif js.accounts == nil {\n\t\tjs.mu.Unlock()\n\t\treturn NewJSNotEnabledError()\n\t}\n\n\tif jsa, ok := js.accounts[a.Name]; ok {\n\t\ta.mu.Lock()\n\t\ta.js = jsa\n\t\ta.mu.Unlock()\n\t\tjs.mu.Unlock()\n\t\treturn a.enableAllJetStreamServiceImportsAndMappings()\n\t}\n\n\t// Check the limits against existing reservations.\n\tif err := js.sufficientResources(limits); err != nil {\n\t\tjs.mu.Unlock()\n\t\treturn err\n\t}\n\n\tsysNode := s.Node()\n\n\tjsa := &jsAccount{js: js, account: a, limits: limits, streams: make(map[string]*stream), sendq: sendq, usage: make(map[string]*jsaStorage)}\n\tjsa.storeDir = filepath.Join(js.config.StoreDir, a.Name)\n\n\t// A single server does not need to do the account updates at this point.\n\tif js.cluster != nil || !s.standAloneMode() {\n\t\tjsa.usageMu.Lock()\n\t\tjsa.utimer = time.AfterFunc(usageTick, jsa.sendClusterUsageUpdateTimer)\n\t\t// Cluster mode updates to resource usage. System internal prevents echos.\n\t\tjsa.updatesPub = fmt.Sprintf(jsaUpdatesPubT, a.Name, sysNode)\n\t\tjsa.updatesSub, _ = s.sysSubscribe(fmt.Sprintf(jsaUpdatesSubT, a.Name), jsa.remoteUpdateUsage)\n\t\tjsa.usageMu.Unlock()\n\t}\n\n\tjs.accounts[a.Name] = jsa\n\t// Stamp inside account as well. Needs to be done under js's lock.\n\ta.mu.Lock()\n\ta.js = jsa\n\ta.mu.Unlock()\n\tjs.mu.Unlock()\n\n\t// Create the proper imports here.\n\tif err := a.enableAllJetStreamServiceImportsAndMappings(); err != nil {\n\t\treturn err\n\t}\n\n\ts.Debugf(\"Enabled JetStream for account %q\", a.Name)\n\tif l, ok := limits[_EMPTY_]; ok {\n\t\ts.Debugf(\" Max Memory: %s\", friendlyBytes(l.MaxMemory))\n\t\ts.Debugf(\" Max Storage: %s\", friendlyBytes(l.MaxStore))\n\t} else {\n\t\tfor t, l := range limits {\n\t\t\ts.Debugf(\" Tier: %s\", t)\n\t\t\ts.Debugf(\" Max Memory: %s\", friendlyBytes(l.MaxMemory))\n\t\t\ts.Debugf(\" Max Storage: %s\", friendlyBytes(l.MaxStore))\n\t\t}\n\t}\n\n\t// Clean up any old snapshots that were orphaned while staging.\n\tos.RemoveAll(filepath.Join(js.config.StoreDir, snapStagingDir))\n\n\tsdir := filepath.Join(jsa.storeDir, streamsDir)\n\tif _, err := os.Stat(sdir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(sdir, defaultDirPerms); err != nil {\n\t\t\treturn fmt.Errorf(\"could not create storage streams directory - %v\", err)\n\t\t}\n\t\t// Just need to make sure we can write to the directory.\n\t\t// Remove the directory will create later if needed.\n\t\tos.RemoveAll(sdir)\n\t\t// when empty remove parent directory, which may have been created as well\n\t\tos.Remove(jsa.storeDir)\n\t} else {\n\t\t// Restore any state here.\n\t\ts.Debugf(\"Recovering JetStream state for account %q\", a.Name)\n\t}\n\n\t// Remember if we should be encrypted and what cipher we think we should use.\n\tencrypted := s.getOpts().JetStreamKey != _EMPTY_\n\tsc := s.getOpts().JetStreamCipher\n\n\tdoConsumers := func(mset *stream, odir string) {\n\t\tofis, _ := os.ReadDir(odir)\n\t\tif len(ofis) > 0 {\n\t\t\ts.Noticef(\" Recovering %d consumers for stream - '%s > %s'\", len(ofis), mset.accName(), mset.name())\n\t\t}\n\t\tfor _, ofi := range ofis {\n\t\t\tmetafile := filepath.Join(odir, ofi.Name(), JetStreamMetaFile)\n\t\t\tmetasum := filepath.Join(odir, ofi.Name(), JetStreamMetaFileSum)\n\t\t\tif _, err := os.Stat(metafile); os.IsNotExist(err) {\n\t\t\t\ts.Warnf(\" Missing consumer metafile %q\", metafile)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbuf, err := os.ReadFile(metafile)\n\t\t\tif err != nil {\n\t\t\t\ts.Warnf(\" Error reading consumer metafile %q: %v\", metafile, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif _, err := os.Stat(metasum); os.IsNotExist(err) {\n\t\t\t\ts.Warnf(\" Missing consumer checksum for %q\", metasum)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Check if we are encrypted.\n\t\t\tif key, err := os.ReadFile(filepath.Join(odir, ofi.Name(), JetStreamMetaFileKey)); err == nil {\n\t\t\t\ts.Debugf(\" Consumer metafile is encrypted, reading encrypted keyfile\")\n\t\t\t\t// Decode the buffer before proceeding.\n\t\t\t\tctxName := mset.name() + tsep + ofi.Name()\n\t\t\t\tnbuf, _, err := s.decryptMeta(sc, key, buf, a.Name, ctxName)\n\t\t\t\tif err != nil {\n\t\t\t\t\ts.Warnf(\" Error decrypting our consumer metafile: %v\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tbuf = nbuf\n\t\t\t}\n\n\t\t\tvar cfg FileConsumerInfo\n\t\t\tdecoder := json.NewDecoder(bytes.NewReader(buf))\n\t\t\tdecoder.DisallowUnknownFields()\n\t\t\tstrictErr := decoder.Decode(&cfg)\n\t\t\tif strictErr != nil {\n\t\t\t\tcfg = FileConsumerInfo{}\n\t\t\t\tif err := json.Unmarshal(buf, &cfg); err != nil {\n\t\t\t\t\ts.Warnf(\" Error unmarshalling consumer metafile %q: %v\", metafile, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif supported := supportsRequiredApiLevel(cfg.Metadata); !supported || strictErr != nil {\n\t\t\t\tvar offlineReason string\n\t\t\t\tif !supported {\n\t\t\t\t\tapiLevel := getRequiredApiLevel(cfg.Metadata)\n\t\t\t\t\tif strictErr != nil {\n\t\t\t\t\t\tofflineReason = fmt.Sprintf(\"unsupported - config error: %s\", strings.TrimPrefix(strictErr.Error(), \"json: \"))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tofflineReason = fmt.Sprintf(\"unsupported - required API level: %s, current API level: %d\", apiLevel, JSApiLevel)\n\t\t\t\t\t}\n\t\t\t\t\ts.Warnf(\" Detected unsupported consumer '%s > %s > %s': %s\", a.Name, mset.name(), cfg.Name, offlineReason)\n\t\t\t\t} else {\n\t\t\t\t\tofflineReason = fmt.Sprintf(\"decoding error: %v\", strictErr)\n\t\t\t\t\ts.Warnf(\" Error unmarshalling consumer metafile %q: %v\", metafile, strictErr)\n\t\t\t\t}\n\t\t\t\tsingleServerMode := !s.JetStreamIsClustered() && s.standAloneMode()\n\t\t\t\tif singleServerMode {\n\t\t\t\t\tif !mset.closed.Load() {\n\t\t\t\t\t\ts.Warnf(\" Stopping unsupported stream '%s > %s'\", a.Name, mset.name())\n\t\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t\t\tmset.offlineReason = fmt.Sprintf(\"stopped - unsupported consumer %q\", cfg.Name)\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\tmset.stop(false, false)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Fake a consumer, so we can respond to API requests as single-server.\n\t\t\t\t\to := &consumer{\n\t\t\t\t\t\tmset: mset,\n\t\t\t\t\t\tjs: s.getJetStream(),\n\t\t\t\t\t\tacc: a,\n\t\t\t\t\t\tsrv: s,\n\t\t\t\t\t\tcfg: cfg.ConsumerConfig,\n\t\t\t\t\t\tactive: false,\n\t\t\t\t\t\tstream: mset.name(),\n\t\t\t\t\t\tname: cfg.Name,\n\t\t\t\t\t\tdseq: 1,\n\t\t\t\t\t\tsseq: 1,\n\t\t\t\t\t\tcreated: time.Now().UTC(),\n\t\t\t\t\t\tclosed: true,\n\t\t\t\t\t\tofflineReason: offlineReason,\n\t\t\t\t\t}\n\t\t\t\t\tif !cfg.Created.IsZero() {\n\t\t\t\t\t\to.created = cfg.Created\n\t\t\t\t\t}\n\n\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t\tmset.setConsumer(o)\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tisEphemeral := !isDurableConsumer(&cfg.ConsumerConfig)\n\t\t\tif isEphemeral {\n\t\t\t\t// This is an ephemeral consumer and this could fail on restart until\n\t\t\t\t// the consumer can reconnect. We will create it as a durable and switch it.\n\t\t\t\tcfg.ConsumerConfig.Durable = ofi.Name()\n\t\t\t}\n\t\t\tobs, err := mset.addConsumerWithAssignment(&cfg.ConsumerConfig, _EMPTY_, nil, true, ActionCreateOrUpdate, false)\n\t\t\tif err != nil {\n\t\t\t\ts.Warnf(\" Error adding consumer '%s > %s > %s': %v\", a.Name, mset.name(), cfg.Name, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif isEphemeral {\n\t\t\t\tobs.switchToEphemeral()\n\t\t\t}\n\t\t\tif !cfg.Created.IsZero() {\n\t\t\t\tobs.setCreatedTime(cfg.Created)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Now recover the streams.\n\tfis, _ := os.ReadDir(sdir)\n\tdoStream := func(fi os.DirEntry) error {\n\t\tplaintext := true\n\t\tmdir := filepath.Join(sdir, fi.Name())\n\t\t// Check for partially deleted streams. They are marked with \".\" prefix.\n\t\tif strings.HasPrefix(fi.Name(), tsep) {\n\t\t\tgo os.RemoveAll(mdir)\n\t\t\treturn nil\n\t\t}\n\t\tkey := sha256.Sum256([]byte(fi.Name()))\n\t\thh, err := highwayhash.NewDigest64(key[:])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmetafile := filepath.Join(mdir, JetStreamMetaFile)\n\t\tmetasum := filepath.Join(mdir, JetStreamMetaFileSum)\n\t\tif _, err := os.Stat(metafile); os.IsNotExist(err) {\n\t\t\ts.Warnf(\" Missing stream metafile for %q\", metafile)\n\t\t\treturn nil\n\t\t}\n\t\tbuf, err := os.ReadFile(metafile)\n\t\tif err != nil {\n\t\t\ts.Warnf(\" Error reading metafile %q: %v\", metafile, err)\n\t\t\treturn nil\n\t\t}\n\t\tif _, err := os.Stat(metasum); os.IsNotExist(err) {\n\t\t\ts.Warnf(\" Missing stream checksum file %q\", metasum)\n\t\t\treturn nil\n\t\t}\n\t\tsum, err := os.ReadFile(metasum)\n\t\tif err != nil {\n\t\t\ts.Warnf(\" Error reading Stream metafile checksum %q: %v\", metasum, err)\n\t\t\treturn nil\n\t\t}\n\t\thh.Write(buf)\n\t\tvar hb [highwayhash.Size64]byte\n\t\tchecksum := hex.EncodeToString(hh.Sum(hb[:0]))\n\t\tif checksum != string(sum) {\n\t\t\ts.Warnf(\" Stream metafile %q: checksums do not match %q vs %q\", metafile, sum, checksum)\n\t\t\treturn nil\n\t\t}\n\n\t\t// Track if we are converting ciphers.\n\t\tvar convertingCiphers bool\n\n\t\t// Check if we are encrypted.\n\t\tkeyFile := filepath.Join(mdir, JetStreamMetaFileKey)\n\t\tkeyBuf, err := os.ReadFile(keyFile)\n\t\tif err == nil {\n\t\t\ts.Debugf(\" Stream metafile is encrypted, reading encrypted keyfile\")\n\t\t\tif len(keyBuf) < minMetaKeySize {\n\t\t\t\ts.Warnf(\" Bad stream encryption key length of %d\", len(keyBuf))\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\t// Decode the buffer before proceeding.\n\t\t\tvar nbuf []byte\n\t\t\tnbuf, convertingCiphers, err = s.decryptMeta(sc, keyBuf, buf, a.Name, fi.Name())\n\t\t\tif err != nil {\n\t\t\t\ts.Warnf(\" Error decrypting our stream metafile: %v\", err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tbuf = nbuf\n\t\t\tplaintext = false\n\t\t}\n\n\t\tvar cfg FileStreamInfo\n\t\tdecoder := json.NewDecoder(bytes.NewReader(buf))\n\t\tdecoder.DisallowUnknownFields()\n\t\tstrictErr := decoder.Decode(&cfg)\n\t\tif strictErr != nil {\n\t\t\tcfg = FileStreamInfo{}\n\t\t\tif err := json.Unmarshal(buf, &cfg); err != nil {\n\t\t\t\ts.Warnf(\" Error unmarshalling stream metafile %q: %v\", metafile, err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tif supported := supportsRequiredApiLevel(cfg.Metadata); !supported || strictErr != nil {\n\t\t\tvar offlineReason string\n\t\t\tif !supported {\n\t\t\t\tapiLevel := getRequiredApiLevel(cfg.Metadata)\n\t\t\t\tif strictErr != nil {\n\t\t\t\t\tofflineReason = fmt.Sprintf(\"unsupported - config error: %s\", strings.TrimPrefix(strictErr.Error(), \"json: \"))\n\t\t\t\t} else {\n\t\t\t\t\tofflineReason = fmt.Sprintf(\"unsupported - required API level: %s, current API level: %d\", apiLevel, JSApiLevel)\n\t\t\t\t}\n\t\t\t\ts.Warnf(\" Detected unsupported stream '%s > %s': %s\", a.Name, cfg.StreamConfig.Name, offlineReason)\n\t\t\t} else {\n\t\t\t\tofflineReason = fmt.Sprintf(\"decoding error: %v\", strictErr)\n\t\t\t\ts.Warnf(\" Error unmarshalling stream metafile %q: %v\", metafile, strictErr)\n\t\t\t}\n\t\t\tsingleServerMode := !s.JetStreamIsClustered() && s.standAloneMode()\n\t\t\tif singleServerMode {\n\t\t\t\t// Fake a stream, so we can respond to API requests as single-server.\n\t\t\t\tmset := &stream{\n\t\t\t\t\tacc: a,\n\t\t\t\t\tjsa: jsa,\n\t\t\t\t\tcfg: cfg.StreamConfig,\n\t\t\t\t\tjs: js,\n\t\t\t\t\tsrv: s,\n\t\t\t\t\tstype: cfg.Storage,\n\t\t\t\t\tconsumers: make(map[string]*consumer),\n\t\t\t\t\tactive: false,\n\t\t\t\t\tcreated: time.Now().UTC(),\n\t\t\t\t\tofflineReason: offlineReason,\n\t\t\t\t}\n\t\t\t\tif !cfg.Created.IsZero() {\n\t\t\t\t\tmset.created = cfg.Created\n\t\t\t\t}\n\t\t\t\tmset.closed.Store(true)\n\n\t\t\t\tjsa.mu.Lock()\n\t\t\t\tjsa.streams[cfg.Name] = mset\n\t\t\t\tjsa.mu.Unlock()\n\n\t\t\t\t// Now do the consumers.\n\t\t\t\todir := filepath.Join(sdir, fi.Name(), consumerDir)\n\t\t\t\tdoConsumers(mset, odir)\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\n\t\t// We had a bug that set a default de dupe window on mirror, despite that being not a valid config\n\t\tfixCfgMirrorWithDedupWindow(&cfg.StreamConfig)\n\n\t\t// We had a bug that could allow subjects in that had prefix or suffix spaces. We check for that here\n\t\t// and will patch them on the fly for now. We will warn about them.\n\t\tvar hadSubjErr bool\n\t\tfor i, subj := range cfg.StreamConfig.Subjects {\n\t\t\tif !IsValidSubject(subj) {\n\t\t\t\ts.Warnf(\" Detected bad subject %q while adding stream %q, will attempt to repair\", subj, cfg.Name)\n\t\t\t\tif nsubj := strings.TrimSpace(subj); IsValidSubject(nsubj) {\n\t\t\t\t\ts.Warnf(\" Bad subject %q repaired to %q\", subj, nsubj)\n\t\t\t\t\tcfg.StreamConfig.Subjects[i] = nsubj\n\t\t\t\t} else {\n\t\t\t\t\ts.Warnf(\" Error recreating stream %q: %v\", cfg.Name, \"invalid subject\")\n\t\t\t\t\thadSubjErr = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif hadSubjErr {\n\t\t\treturn nil\n\t\t}\n\n\t\t// The other possible bug is assigning subjects to mirrors, so check for that and patch as well.\n\t\tif cfg.StreamConfig.Mirror != nil && len(cfg.StreamConfig.Subjects) > 0 {\n\t\t\ts.Warnf(\" Detected subjects on a mirrored stream %q, will remove\", cfg.Name)\n\t\t\tcfg.StreamConfig.Subjects = nil\n\t\t}\n\n\t\ts.Noticef(\" Starting restore for stream '%s > %s'\", a.Name, cfg.StreamConfig.Name)\n\t\trt := time.Now()\n\n\t\t// Log if we are converting from plaintext to encrypted.\n\t\tif encrypted {\n\t\t\tif plaintext {\n\t\t\t\ts.Noticef(\" Encrypting stream '%s > %s'\", a.Name, cfg.StreamConfig.Name)\n\t\t\t} else if convertingCiphers {\n\t\t\t\ts.Noticef(\" Converting to %s for stream '%s > %s'\", sc, a.Name, cfg.StreamConfig.Name)\n\t\t\t\t// Remove the key file to have system regenerate with the new cipher.\n\t\t\t\tos.Remove(keyFile)\n\t\t\t}\n\t\t}\n\n\t\t// Add in the stream.\n\t\tmset, err := a.recoverStream(&cfg.StreamConfig)\n\t\tif err != nil {\n\t\t\ts.Warnf(\" Error recreating stream %q: %v\", cfg.Name, err)\n\t\t\t// If we removed a keyfile from above make sure to put it back.\n\t\t\tif convertingCiphers {\n\t\t\t\terr := os.WriteFile(keyFile, keyBuf, defaultFilePerms)\n\t\t\t\tif err != nil {\n\t\t\t\t\ts.Warnf(\" Error replacing meta keyfile for stream %q: %v\", cfg.Name, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tif !cfg.Created.IsZero() {\n\t\t\tmset.setCreatedTime(cfg.Created)\n\t\t}\n\n\t\t// Might need to recover from a partial batch write, but only if a single replica stream.\n\t\tif cfg.AllowAtomicPublish && cfg.Replicas == 1 {\n\t\t\tvar (\n\t\t\t\tok bool\n\t\t\t\tsmv StoreMsg\n\t\t\t\tbatchId string\n\t\t\t\tbatchSeq uint64\n\t\t\t\tcommit bool\n\t\t\t\tcommitEob bool\n\t\t\t\tbatchStoreDir string\n\t\t\t\tstore StreamStore\n\t\t\t\tstate StreamState\n\t\t\t)\n\t\t\t// Check if the last message was part of a batch.\n\t\t\tsm, err := mset.store.LoadLastMsg(fwcs, &smv)\n\t\t\tif err != nil || sm == nil {\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\tbatchId = getBatchId(sm.hdr)\n\t\t\tbatchSeq, ok = getBatchSequence(sm.hdr)\n\t\t\tcommit = len(sliceHeader(JSBatchCommit, sm.hdr)) != 0\n\t\t\tif batchId == _EMPTY_ || !ok || commit {\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\t// We've observed a partial batch write. Write the remainder of the batch.\n\t\t\tbatchSeq++\n\t\t\t_, batchStoreDir = getBatchStoreDir(mset, batchId)\n\t\t\tif _, err = os.Stat(batchStoreDir); err != nil {\n\t\t\t\ts.Errorf(\" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v\",\n\t\t\t\t\tmset.accName(), mset.name(), batchSeq, err)\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\tstore, err = newBatchStore(mset, batchId)\n\t\t\tif err != nil {\n\t\t\t\ts.Errorf(\" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v\",\n\t\t\t\t\tmset.accName(), mset.name(), batchSeq, err)\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\tstore.FastState(&state)\n\t\t\tsm, err = store.LoadMsg(state.LastSeq, &smv)\n\t\t\tif err != nil || sm == nil {\n\t\t\t\ts.Errorf(\" Failed restoring partial batch write for stream '%s > %s' at sequence %d: last msg not found %d\",\n\t\t\t\t\tmset.accName(), mset.name(), batchSeq, state.LastSeq)\n\t\t\t\tgoto SKIP\n\t\t\t}\n\t\t\tcommitEob = bytes.Equal(sliceHeader(JSBatchCommit, sm.hdr), []byte(\"eob\"))\n\t\t\t// If the commit ends with an \"End Of Batch\" message, we don't store this.\n\t\t\tif commitEob {\n\t\t\t\tstate.LastSeq--\n\t\t\t}\n\t\t\ts.Noticef(\" Restoring partial batch write for stream '%s > %s' (seq %d to %d)\",\n\t\t\t\tmset.accName(), mset.name(), batchSeq, state.LastSeq)\n\t\t\t// Loop through items that weren't persisted yet.\n\t\t\tfor seq := batchSeq; seq <= state.LastSeq; seq++ {\n\t\t\t\tsm, err = store.LoadMsg(seq, &smv)\n\t\t\t\tif err != nil || sm == nil {\n\t\t\t\t\ts.Errorf(\" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v\",\n\t\t\t\t\t\tmset.accName(), mset.name(), seq, err)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\thdr := sm.hdr\n\t\t\t\t// If committed by EOB, the last message must get the normal commit header.\n\t\t\t\tif commitEob && seq == state.LastSeq {\n\t\t\t\t\thdr = genHeader(hdr, JSBatchCommit, \"1\")\n\t\t\t\t}\n\t\t\t\tmset.processJetStreamMsg(sm.subj, _EMPTY_, hdr, sm.msg, 0, 0, nil, false, true)\n\t\t\t}\n\t\t\tstore.Delete(true)\n\t\tSKIP:\n\t\t\tos.RemoveAll(filepath.Join(sdir, fi.Name(), batchesDir))\n\t\t}\n\n\t\tstate := mset.state()\n\t\ts.Noticef(\" Restored %s messages for stream '%s > %s' in %v\",\n\t\t\tcomma(int64(state.Msgs)), mset.accName(), mset.name(), time.Since(rt).Round(time.Millisecond))\n\n\t\t// Now do the consumers.\n\t\todir := filepath.Join(sdir, fi.Name(), consumerDir)\n\t\tdoConsumers(mset, odir)\n\n\t\t// Collect to check for dangling messages.\n\t\t// TODO(dlc) - Can be removed eventually.\n\t\tif cfg.StreamConfig.Retention == InterestPolicy {\n\t\t\tmset.checkForOrphanMsgs()\n\t\t\tmset.checkConsumerReplication()\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tif tq != nil {\n\t\t// If a parallelTaskQueue was provided then use that for concurrency.\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(len(fis))\n\t\tfor _, fi := range fis {\n\t\t\ttq <- func() {\n\t\t\t\tdoStream(fi)\n\t\t\t\twg.Done()\n\t\t\t}\n\t\t}\n\t\twg.Wait()\n\t} else {\n\t\t// No parallelTaskQueue provided, do inline as before.\n\t\tfor _, fi := range fis {\n\t\t\tdoStream(fi)\n\t\t}\n\t}\n\n\t// Make sure to cleanup any old remaining snapshots.\n\tos.RemoveAll(filepath.Join(jsa.storeDir, snapsDir))\n\n\ts.Debugf(\"JetStream state for account %q recovered\", a.Name)\n\n\treturn nil\n}\n\n// Return whether we require MaxBytes to be set and if > 0 an upper limit for stream size exists\n// Both limits are independent of each other.\nfunc (a *Account) maxBytesLimits(cfg *StreamConfig) (bool, int64) {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\tif jsa == nil {\n\t\treturn false, 0\n\t}\n\tjsa.usageMu.RLock()\n\tvar replicas int\n\tif cfg != nil {\n\t\treplicas = cfg.Replicas\n\t}\n\tselectedLimits, _, ok := jsa.selectLimits(replicas)\n\tjsa.usageMu.RUnlock()\n\tif !ok {\n\t\treturn false, 0\n\t}\n\tmaxStreamBytes := int64(0)\n\tif cfg.Storage == MemoryStorage {\n\t\tmaxStreamBytes = selectedLimits.MemoryMaxStreamBytes\n\t} else {\n\t\tmaxStreamBytes = selectedLimits.StoreMaxStreamBytes\n\t}\n\treturn selectedLimits.MaxBytesRequired, maxStreamBytes\n}\n\n// NumStreams will return how many streams we have.\nfunc (a *Account) numStreams() int {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\tif jsa == nil {\n\t\treturn 0\n\t}\n\tjsa.mu.Lock()\n\tn := len(jsa.streams)\n\tjsa.mu.Unlock()\n\treturn n\n}\n\n// Streams will return all known streams.\nfunc (a *Account) streams() []*stream {\n\treturn a.filteredStreams(_EMPTY_)\n}\n\nfunc (a *Account) filteredStreams(filter string) []*stream {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\n\tif jsa == nil {\n\t\treturn nil\n\t}\n\n\tjsa.mu.RLock()\n\tdefer jsa.mu.RUnlock()\n\n\tvar msets []*stream\n\tfor _, mset := range jsa.streams {\n\t\tif filter != _EMPTY_ {\n\t\t\tmset.cfgMu.RLock()\n\t\t\tfor _, subj := range mset.cfg.Subjects {\n\t\t\t\tif SubjectsCollide(filter, subj) {\n\t\t\t\t\tmsets = append(msets, mset)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.cfgMu.RUnlock()\n\t\t} else {\n\t\t\tmsets = append(msets, mset)\n\t\t}\n\t}\n\n\treturn msets\n}\n\n// lookupStream will lookup a stream by name.\nfunc (a *Account) lookupStream(name string) (*stream, error) {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\n\tif jsa == nil {\n\t\treturn nil, NewJSNotEnabledForAccountError()\n\t}\n\tjsa.mu.RLock()\n\tdefer jsa.mu.RUnlock()\n\n\tmset, ok := jsa.streams[name]\n\tif !ok {\n\t\treturn nil, NewJSStreamNotFoundError()\n\t}\n\treturn mset, nil\n}\n\n// UpdateJetStreamLimits will update the account limits for a JetStream enabled account.\nfunc (a *Account) UpdateJetStreamLimits(limits map[string]JetStreamAccountLimits) error {\n\ta.mu.RLock()\n\ts, jsa := a.srv, a.js\n\ta.mu.RUnlock()\n\n\tif s == nil {\n\t\treturn fmt.Errorf(\"jetstream account not registered\")\n\t}\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\tif jsa == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\n\tif len(limits) == 0 {\n\t\tlimits = defaultJSAccountTiers\n\t}\n\n\t// Calculate the delta between what we have and what we want.\n\tjsa.usageMu.RLock()\n\tdl := diffCheckedLimits(jsa.limits, limits)\n\tjsa.usageMu.RUnlock()\n\n\tjs.mu.Lock()\n\t// Check the limits against existing reservations.\n\tif err := js.sufficientResources(dl); err != nil {\n\t\tjs.mu.Unlock()\n\t\treturn err\n\t}\n\tjs.mu.Unlock()\n\n\t// Update\n\tjsa.usageMu.Lock()\n\tjsa.limits = limits\n\tjsa.usageMu.Unlock()\n\n\treturn nil\n}\n\nfunc diffCheckedLimits(a, b map[string]JetStreamAccountLimits) map[string]JetStreamAccountLimits {\n\tdiff := map[string]JetStreamAccountLimits{}\n\tfor t, la := range a {\n\t\t// in a, not in b will return 0\n\t\tlb := b[t]\n\t\tdiff[t] = JetStreamAccountLimits{\n\t\t\tMaxMemory: lb.MaxMemory - la.MaxMemory,\n\t\t\tMaxStore: lb.MaxStore - la.MaxStore,\n\t\t}\n\t}\n\tfor t, lb := range b {\n\t\tif la, ok := a[t]; !ok {\n\t\t\t// only in b not in a. (in a and b already covered)\n\t\t\tdiff[t] = JetStreamAccountLimits{\n\t\t\t\tMaxMemory: lb.MaxMemory - la.MaxMemory,\n\t\t\t\tMaxStore: lb.MaxStore - la.MaxStore,\n\t\t\t}\n\t\t}\n\t}\n\treturn diff\n}\n\n// Return reserved bytes for memory and file store streams for this account on this server.\n// Lock should be held.\nfunc (jsa *jsAccount) reservedStorage(tier string) (mem, store uint64) {\n\tfor _, mset := range jsa.streams {\n\t\tcfg := &mset.cfg\n\t\tif (tier == _EMPTY_ || tier == tierName(cfg.Replicas)) && cfg.MaxBytes > 0 {\n\t\t\tswitch cfg.Storage {\n\t\t\tcase FileStorage:\n\t\t\t\tstore += uint64(cfg.MaxBytes)\n\t\t\tcase MemoryStorage:\n\t\t\t\tmem += uint64(cfg.MaxBytes)\n\t\t\t}\n\t\t}\n\t}\n\treturn mem, store\n}\n\n// Return reserved bytes for memory and file store streams for this account in clustered mode.\n// js lock should be held.\nfunc reservedStorage(sas map[string]*streamAssignment, tier string) (mem, store uint64) {\n\tfor _, sa := range sas {\n\t\tcfg := sa.Config\n\t\tif (tier == _EMPTY_ || tier == tierName(cfg.Replicas)) && cfg.MaxBytes > 0 {\n\t\t\tswitch cfg.Storage {\n\t\t\tcase FileStorage:\n\t\t\t\tstore += uint64(cfg.MaxBytes)\n\t\t\tcase MemoryStorage:\n\t\t\t\tmem += uint64(cfg.MaxBytes)\n\t\t\t}\n\t\t}\n\t}\n\treturn mem, store\n}\n\n// JetStreamUsage reports on JetStream usage and limits for an account.\nfunc (a *Account) JetStreamUsage() JetStreamAccountStats {\n\ta.mu.RLock()\n\tjsa, aname := a.js, a.Name\n\taccJsLimits := a.jsLimits\n\ta.mu.RUnlock()\n\n\tvar stats JetStreamAccountStats\n\tif jsa != nil {\n\t\tjs := jsa.js\n\t\tjs.mu.RLock()\n\t\tcc := js.cluster\n\t\tsingleServer := cc == nil\n\t\tjsa.mu.RLock()\n\t\tjsa.usageMu.RLock()\n\t\tstats.Memory, stats.Store = jsa.storageTotals()\n\t\tstats.Domain = js.config.Domain\n\t\tstats.API = JetStreamAPIStats{\n\t\t\tLevel: JSApiLevel,\n\t\t\tTotal: jsa.apiTotal,\n\t\t\tErrors: jsa.apiErrors,\n\t\t}\n\t\tif singleServer {\n\t\t\tstats.ReservedMemory, stats.ReservedStore = jsa.reservedStorage(_EMPTY_)\n\t\t} else {\n\t\t\tstats.ReservedMemory, stats.ReservedStore = reservedStorage(cc.streams[aname], _EMPTY_)\n\t\t}\n\t\tl, defaultTier := jsa.limits[_EMPTY_]\n\t\tif defaultTier {\n\t\t\tstats.Limits = l\n\t\t} else {\n\t\t\tskipped := 0\n\t\t\tstats.Tiers = make(map[string]JetStreamTier)\n\t\t\tfor t, total := range jsa.usage {\n\t\t\t\tif _, ok := jsa.limits[t]; !ok && (*total) == (jsaStorage{}) {\n\t\t\t\t\t// skip tiers not present that don't contain a count\n\t\t\t\t\t// In case this shows an empty stream, that tier will be added when iterating over streams\n\t\t\t\t\tskipped++\n\t\t\t\t} else {\n\t\t\t\t\ttier := JetStreamTier{\n\t\t\t\t\t\tMemory: uint64(total.total.mem),\n\t\t\t\t\t\tStore: uint64(total.total.store),\n\t\t\t\t\t\tLimits: jsa.limits[t],\n\t\t\t\t\t}\n\t\t\t\t\tif singleServer {\n\t\t\t\t\t\ttier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t)\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t)\n\t\t\t\t\t}\n\t\t\t\t\tstats.Tiers[t] = tier\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(accJsLimits) != len(jsa.usage)-skipped {\n\t\t\t\t// insert unused limits\n\t\t\t\tfor t, lim := range accJsLimits {\n\t\t\t\t\tif _, ok := stats.Tiers[t]; !ok {\n\t\t\t\t\t\ttier := JetStreamTier{Limits: lim}\n\t\t\t\t\t\tif singleServer {\n\t\t\t\t\t\t\ttier.ReservedMemory, tier.ReservedStore = jsa.reservedStorage(t)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ttier.ReservedMemory, tier.ReservedStore = reservedStorage(cc.streams[aname], t)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tstats.Tiers[t] = tier\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tjsa.usageMu.RUnlock()\n\n\t\t// Clustered\n\t\tif cc := js.cluster; cc != nil {\n\t\t\tsas := cc.streams[aname]\n\t\t\tif defaultTier {\n\t\t\t\tstats.Streams = len(sas)\n\t\t\t\tstats.ReservedMemory, stats.ReservedStore = reservedStorage(sas, _EMPTY_)\n\t\t\t}\n\t\t\tfor _, sa := range sas {\n\t\t\t\tif defaultTier {\n\t\t\t\t\tstats.Consumers += len(sa.consumers)\n\t\t\t\t} else {\n\t\t\t\t\tstats.Streams++\n\t\t\t\t\tstreamTier := tierName(sa.Config.Replicas)\n\t\t\t\t\tsu, ok := stats.Tiers[streamTier]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tsu = JetStreamTier{}\n\t\t\t\t\t}\n\t\t\t\t\tsu.Streams++\n\t\t\t\t\tstats.Tiers[streamTier] = su\n\n\t\t\t\t\t// Now consumers, check each since could be different tiers.\n\t\t\t\t\tfor _, ca := range sa.consumers {\n\t\t\t\t\t\tstats.Consumers++\n\t\t\t\t\t\tconsumerTier := tierName(ca.Config.replicas(sa.Config))\n\t\t\t\t\t\tcu, ok := stats.Tiers[consumerTier]\n\t\t\t\t\t\tif !ok {\n\t\t\t\t\t\t\tcu = JetStreamTier{}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcu.Consumers++\n\t\t\t\t\t\tstats.Tiers[consumerTier] = cu\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tif defaultTier {\n\t\t\t\tstats.Streams = len(jsa.streams)\n\t\t\t}\n\t\t\tfor _, mset := range jsa.streams {\n\t\t\t\tconsCount := mset.numConsumers()\n\t\t\t\tstats.Consumers += consCount\n\t\t\t\tif !defaultTier {\n\t\t\t\t\tu, ok := stats.Tiers[mset.tier]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tu = JetStreamTier{}\n\t\t\t\t\t}\n\t\t\t\t\tu.Streams++\n\t\t\t\t\tstats.Streams++\n\t\t\t\t\tu.Consumers += consCount\n\t\t\t\t\tstats.Tiers[mset.tier] = u\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tjsa.mu.RUnlock()\n\t\tjs.mu.RUnlock()\n\t}\n\treturn stats\n}\n\n// DisableJetStream will disable JetStream for this account.\nfunc (a *Account) DisableJetStream() error {\n\treturn a.removeJetStream()\n}\n\n// removeJetStream is called when JetStream has been disabled for this account.\nfunc (a *Account) removeJetStream() error {\n\ta.mu.Lock()\n\ts := a.srv\n\ta.js = nil\n\ta.mu.Unlock()\n\n\tif s == nil {\n\t\treturn fmt.Errorf(\"jetstream account not registered\")\n\t}\n\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\n\treturn js.disableJetStream(js.lookupAccount(a))\n}\n\n// Disable JetStream for the account.\nfunc (js *jetStream) disableJetStream(jsa *jsAccount) error {\n\tif jsa == nil || jsa.account == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\n\tjs.mu.Lock()\n\tdelete(js.accounts, jsa.account.Name)\n\tjs.mu.Unlock()\n\n\tjsa.delete()\n\n\treturn nil\n}\n\n// jetStreamConfigured reports whether the account has JetStream configured, regardless of this\n// servers JetStream status.\nfunc (a *Account) jetStreamConfigured() bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn len(a.jsLimits) > 0\n}\n\n// JetStreamEnabled is a helper to determine if jetstream is enabled for an account.\nfunc (a *Account) JetStreamEnabled() bool {\n\tif a == nil {\n\t\treturn false\n\t}\n\ta.mu.RLock()\n\tenabled := a.js != nil\n\ta.mu.RUnlock()\n\treturn enabled\n}\n\nfunc (jsa *jsAccount) remoteUpdateUsage(sub *subscription, c *client, _ *Account, subject, _ string, msg []byte) {\n\t// jsa.js.srv is immutable and guaranteed to no be nil, so no lock needed.\n\ts := jsa.js.srv\n\n\tjsa.usageMu.Lock()\n\tdefer jsa.usageMu.Unlock()\n\n\tif len(msg) < minUsageUpdateLen {\n\t\ts.Warnf(\"Ignoring remote usage update with size too short\")\n\t\treturn\n\t}\n\tvar rnode string\n\tif li := strings.LastIndexByte(subject, btsep); li > 0 && li < len(subject) {\n\t\trnode = subject[li+1:]\n\t}\n\tif rnode == _EMPTY_ {\n\t\ts.Warnf(\"Received remote usage update with no remote node\")\n\t\treturn\n\t}\n\trUsage, ok := jsa.rusage[rnode]\n\tif !ok {\n\t\tif jsa.rusage == nil {\n\t\t\tjsa.rusage = make(map[string]*remoteUsage)\n\t\t}\n\t\trUsage = &remoteUsage{tiers: make(map[string]*jsaUsage)}\n\t\tjsa.rusage[rnode] = rUsage\n\t}\n\tupdateTotal := func(tierName string, memUsed, storeUsed int64) {\n\t\ttotal, ok := jsa.usage[tierName]\n\t\tif !ok {\n\t\t\ttotal = &jsaStorage{}\n\t\t\tjsa.usage[tierName] = total\n\t\t}\n\t\t// Update the usage for this remote.\n\t\tif usage := rUsage.tiers[tierName]; usage != nil {\n\t\t\t// Decrement our old values.\n\t\t\ttotal.total.mem -= usage.mem\n\t\t\ttotal.total.store -= usage.store\n\t\t\tusage.mem, usage.store = memUsed, storeUsed\n\t\t} else {\n\t\t\trUsage.tiers[tierName] = &jsaUsage{memUsed, storeUsed}\n\t\t}\n\t\ttotal.total.mem += memUsed\n\t\ttotal.total.store += storeUsed\n\t}\n\n\tvar le = binary.LittleEndian\n\tapiTotal, apiErrors := le.Uint64(msg[16:]), le.Uint64(msg[24:])\n\tmemUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:]))\n\n\t// We later extended the data structure to support multiple tiers\n\tvar excessRecordCnt uint32\n\tvar tierName string\n\n\tif len(msg) >= usageMultiTiersLen {\n\t\texcessRecordCnt = le.Uint32(msg[minUsageUpdateLen:])\n\t\tlength := le.Uint64(msg[minUsageUpdateLen+4:])\n\t\t// Need to protect past this point in case this is wrong.\n\t\tif uint64(len(msg)) < usageMultiTiersLen+length {\n\t\t\ts.Warnf(\"Received corrupt remote usage update\")\n\t\t\treturn\n\t\t}\n\t\ttierName = string(msg[usageMultiTiersLen : usageMultiTiersLen+length])\n\t\tmsg = msg[usageMultiTiersLen+length:]\n\t}\n\tupdateTotal(tierName, memUsed, storeUsed)\n\tfor ; excessRecordCnt > 0 && len(msg) >= usageRecordLen; excessRecordCnt-- {\n\t\tmemUsed, storeUsed := int64(le.Uint64(msg[0:])), int64(le.Uint64(msg[8:]))\n\t\tlength := le.Uint64(msg[16:])\n\t\tif uint64(len(msg)) < usageRecordLen+length {\n\t\t\ts.Warnf(\"Received corrupt remote usage update on excess record\")\n\t\t\treturn\n\t\t}\n\t\ttierName = string(msg[usageRecordLen : usageRecordLen+length])\n\t\tmsg = msg[usageRecordLen+length:]\n\t\tupdateTotal(tierName, memUsed, storeUsed)\n\t}\n\tjsa.apiTotal -= rUsage.api\n\tjsa.apiErrors -= rUsage.err\n\trUsage.api = apiTotal\n\trUsage.err = apiErrors\n\tjsa.apiTotal += apiTotal\n\tjsa.apiErrors += apiErrors\n}\n\n// When we detect a skew of some sort this will verify the usage reporting is correct.\n// No locks should be held.\nfunc (jsa *jsAccount) checkAndSyncUsage(tierName string, storeType StorageType) {\n\t// This will run in a separate go routine, so check that we are only running once.\n\tif !jsa.sync.CompareAndSwap(false, true) {\n\t\treturn\n\t}\n\tdefer jsa.sync.Store(false)\n\n\t// Hold the account read lock and the usage lock while we calculate.\n\t// We scope by tier and storage type, but if R3 File has 200 streams etc. could\n\t// show a pause. I did test with > 100 non-active streams and was 80-200ns or so.\n\t// Should be rare this gets called as well.\n\tjsa.mu.RLock()\n\tdefer jsa.mu.RUnlock()\n\tjs := jsa.js\n\tif js == nil {\n\t\treturn\n\t}\n\ts := js.srv\n\n\t// We need to collect the stream stores before we acquire the usage lock since in storeUpdates the\n\t// stream lock could be held if deletion are inline with storing a new message, e.g. via limits.\n\tvar stores []StreamStore\n\tfor _, mset := range jsa.streams {\n\t\tmset.mu.RLock()\n\t\tif mset.tier == tierName && mset.stype == storeType && mset.store != nil {\n\t\t\tstores = append(stores, mset.store)\n\t\t}\n\t\tmset.mu.RUnlock()\n\t}\n\n\t// Now range and qualify, hold usage lock to prevent updates.\n\tjsa.usageMu.Lock()\n\tdefer jsa.usageMu.Unlock()\n\n\tusage, ok := jsa.usage[tierName]\n\tif !ok {\n\t\treturn\n\t}\n\n\t// Collect current total for all stream stores that matched.\n\tvar total int64\n\tvar state StreamState\n\tfor _, store := range stores {\n\t\tstore.FastState(&state)\n\t\ttotal += int64(state.Bytes)\n\t}\n\n\tvar needClusterUpdate bool\n\t// If we do not match on our calculations compute delta and adjust.\n\tif storeType == MemoryStorage {\n\t\tif total != usage.local.mem {\n\t\t\ts.Warnf(\"MemStore usage drift of %v vs %v detected for account %q\",\n\t\t\t\tfriendlyBytes(total), friendlyBytes(usage.local.mem), jsa.account.GetName())\n\t\t\tdelta := total - usage.local.mem\n\t\t\tusage.local.mem += delta\n\t\t\tusage.total.mem += delta\n\t\t\tatomic.AddInt64(&js.memUsed, delta)\n\t\t\tneedClusterUpdate = true\n\t\t}\n\t} else {\n\t\tif total != usage.local.store {\n\t\t\ts.Warnf(\"FileStore usage drift of %v vs %v detected for account %q\",\n\t\t\t\tfriendlyBytes(total), friendlyBytes(usage.local.store), jsa.account.GetName())\n\t\t\tdelta := total - usage.local.store\n\t\t\tusage.local.store += delta\n\t\t\tusage.total.store += delta\n\t\t\tatomic.AddInt64(&js.storeUsed, delta)\n\t\t\tneedClusterUpdate = true\n\t\t}\n\t}\n\n\t// Publish our local updates if in clustered mode.\n\tif needClusterUpdate && js.isClusteredNoLock() {\n\t\tjsa.sendClusterUsageUpdate()\n\t}\n}\n\n// Updates accounting on in use memory and storage. This is called from locally\n// by the lower storage layers.\nfunc (jsa *jsAccount) updateUsage(tierName string, storeType StorageType, delta int64) {\n\t// jsa.js is immutable and cannot be nil, so ok w/o lock.\n\tjs := jsa.js\n\t// updateUsage() may be invoked under the mset's lock, so we can't get\n\t// the js' lock to check if clustered. So use this function that make\n\t// use of an atomic to do the check without having data race reports.\n\tisClustered := js.isClusteredNoLock()\n\n\tvar needsCheck bool\n\tjsa.usageMu.Lock()\n\ts, ok := jsa.usage[tierName]\n\tif !ok {\n\t\ts = &jsaStorage{}\n\t\tjsa.usage[tierName] = s\n\t}\n\tif storeType == MemoryStorage {\n\t\ts.local.mem += delta\n\t\ts.total.mem += delta\n\t\tatomic.AddInt64(&js.memUsed, delta)\n\t\tneedsCheck = s.local.mem < 0\n\t} else {\n\t\ts.local.store += delta\n\t\ts.total.store += delta\n\t\tatomic.AddInt64(&js.storeUsed, delta)\n\t\tneedsCheck = s.local.store < 0\n\t}\n\t// Publish our local updates if in clustered mode.\n\tif isClustered {\n\t\tjsa.sendClusterUsageUpdate()\n\t}\n\tjsa.usageMu.Unlock()\n\n\tif needsCheck {\n\t\t// We could be holding the stream lock from up in the stack, and this\n\t\t// will want the jsa lock, which would violate locking order.\n\t\t// So do this in a Go routine. The function will check if it is already running.\n\t\tgo jsa.checkAndSyncUsage(tierName, storeType)\n\t}\n}\n\nvar usageTick = 1500 * time.Millisecond\n\nfunc (jsa *jsAccount) sendClusterUsageUpdateTimer() {\n\tjsa.usageMu.Lock()\n\tdefer jsa.usageMu.Unlock()\n\tjsa.sendClusterUsageUpdate()\n\tif jsa.utimer != nil {\n\t\tjsa.utimer.Reset(usageTick)\n\t}\n}\n\n// For usage fields.\nconst (\n\tminUsageUpdateLen = 32\n\tstackUsageUpdate = 72\n\tusageRecordLen = 24\n\tusageMultiTiersLen = 44\n\tapiStatsAndNumTiers = 20\n\tminUsageUpdateWindow = 250 * time.Millisecond\n)\n\n// Send updates to our account usage for this server.\n// jsa.usageMu lock should be held.\nfunc (jsa *jsAccount) sendClusterUsageUpdate() {\n\t// These values are absolute so we can limit send rates.\n\tnow := time.Now()\n\tif now.Sub(jsa.lupdate) < minUsageUpdateWindow {\n\t\treturn\n\t}\n\tjsa.lupdate = now\n\n\tlenUsage := len(jsa.usage)\n\tif lenUsage == 0 {\n\t\treturn\n\t}\n\t// every base record contains mem/store/len(tier) as well as the tier name\n\tl := usageRecordLen * lenUsage\n\tfor tier := range jsa.usage {\n\t\tl += len(tier)\n\t}\n\t// first record contains api/usage errors as well as count for extra base records\n\tl += apiStatsAndNumTiers\n\n\tvar raw [stackUsageUpdate]byte\n\tvar b []byte\n\tif l > stackUsageUpdate {\n\t\tb = make([]byte, l)\n\t} else {\n\t\tb = raw[:l]\n\t}\n\n\tvar i int\n\tvar le = binary.LittleEndian\n\tfor tier, usage := range jsa.usage {\n\t\tle.PutUint64(b[i+0:], uint64(usage.local.mem))\n\t\tle.PutUint64(b[i+8:], uint64(usage.local.store))\n\t\tif i == 0 {\n\t\t\tle.PutUint64(b[16:], jsa.usageApi)\n\t\t\tle.PutUint64(b[24:], jsa.usageErr)\n\t\t\tle.PutUint32(b[32:], uint32(len(jsa.usage)-1))\n\t\t\tle.PutUint64(b[36:], uint64(len(tier)))\n\t\t\tcopy(b[usageMultiTiersLen:], tier)\n\t\t\ti = usageMultiTiersLen + len(tier)\n\t\t} else {\n\t\t\tle.PutUint64(b[i+16:], uint64(len(tier)))\n\t\t\tcopy(b[i+usageRecordLen:], tier)\n\t\t\ti += usageRecordLen + len(tier)\n\t\t}\n\t}\n\tjsa.sendq.push(newPubMsg(nil, jsa.updatesPub, _EMPTY_, nil, nil, b, noCompression, false, false))\n}\n\nfunc (js *jetStream) wouldExceedLimits(storeType StorageType, sz int) bool {\n\tvar (\n\t\ttotal *int64\n\t\tmax int64\n\t)\n\tif storeType == MemoryStorage {\n\t\ttotal, max = &js.memUsed, js.config.MaxMemory\n\t} else {\n\t\ttotal, max = &js.storeUsed, js.config.MaxStore\n\t}\n\treturn (atomic.LoadInt64(total) + int64(sz)) > max\n}\n\nfunc (js *jetStream) limitsExceeded(storeType StorageType) bool {\n\treturn js.wouldExceedLimits(storeType, 0)\n}\n\nfunc tierName(replicas int) string {\n\t// TODO (mh) this is where we could select based off a placement tag as well \"qos:tier\"\n\tif replicas == 0 {\n\t\treplicas = 1\n\t}\n\treturn fmt.Sprintf(\"R%d\", replicas)\n}\n\nfunc isSameTier(cfgA, cfgB *StreamConfig) bool {\n\ta := max(1, cfgA.Replicas)\n\tb := max(1, cfgB.Replicas)\n\t// TODO (mh) this is where we could select based off a placement tag as well \"qos:tier\"\n\treturn a == b\n}\n\nfunc (jsa *jsAccount) jetStreamAndClustered() (*jetStream, bool) {\n\tjsa.mu.RLock()\n\tjs := jsa.js\n\tjsa.mu.RUnlock()\n\treturn js, js.isClustered()\n}\n\n// jsa.usageMu read lock should be held.\nfunc (jsa *jsAccount) selectLimits(replicas int) (JetStreamAccountLimits, string, bool) {\n\tif selectedLimits, ok := jsa.limits[_EMPTY_]; ok {\n\t\treturn selectedLimits, _EMPTY_, true\n\t}\n\ttier := tierName(replicas)\n\tif selectedLimits, ok := jsa.limits[tier]; ok {\n\t\treturn selectedLimits, tier, true\n\t}\n\treturn JetStreamAccountLimits{}, _EMPTY_, false\n}\n\n// Lock should be held.\nfunc (jsa *jsAccount) countStreams(tier string, cfg *StreamConfig) (streams int) {\n\tfor _, sa := range jsa.streams {\n\t\t// Don't count the stream toward the limit if it already exists.\n\t\tif (tier == _EMPTY_ || isSameTier(&sa.cfg, cfg)) && sa.cfg.Name != cfg.Name {\n\t\t\tstreams++\n\t\t}\n\t}\n\treturn streams\n}\n\n// jsa.usageMu read lock (at least) should be held.\nfunc (jsa *jsAccount) storageTotals() (uint64, uint64) {\n\tmem := uint64(0)\n\tstore := uint64(0)\n\tfor _, sa := range jsa.usage {\n\t\tmem += uint64(sa.total.mem)\n\t\tstore += uint64(sa.total.store)\n\t}\n\treturn mem, store\n}\n\nfunc (jsa *jsAccount) limitsExceeded(storeType StorageType, tierName string, replicas int) (bool, *ApiError) {\n\treturn jsa.wouldExceedLimits(storeType, tierName, replicas, _EMPTY_, nil, nil)\n}\n\nfunc (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string, replicas int, subj string, hdr, msg []byte) (bool, *ApiError) {\n\tjsa.usageMu.RLock()\n\tdefer jsa.usageMu.RUnlock()\n\n\tselectedLimits, ok := jsa.limits[tierName]\n\tif !ok {\n\t\treturn true, NewJSNoLimitsError()\n\t}\n\tinUse := jsa.usage[tierName]\n\tif inUse == nil {\n\t\t// Imply totals of 0\n\t\treturn false, nil\n\t}\n\tr := int64(replicas)\n\t// Make sure replicas is correct.\n\tif r < 1 {\n\t\tr = 1\n\t}\n\t// This is for limits. If we have no tier, consider all to be flat, vs tiers like R3 where we want to scale limit by replication.\n\tlr := r\n\tif tierName == _EMPTY_ {\n\t\tlr = 1\n\t}\n\n\t// Since tiers are flat we need to scale limit up by replicas when checking.\n\tif storeType == MemoryStorage {\n\t\ttotalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r)\n\t\tif selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr {\n\t\t\treturn true, nil\n\t\t}\n\t} else {\n\t\ttotalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r)\n\t\tif selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\n\treturn false, nil\n}\n\n// Check account limits.\n// Read Lock should be held\nfunc (js *jetStream) checkAccountLimits(selected *JetStreamAccountLimits, config *StreamConfig, currentRes int64) error {\n\treturn js.checkLimits(selected, config, false, currentRes, 0)\n}\n\n// Check account and server limits.\n// Read Lock should be held\nfunc (js *jetStream) checkAllLimits(selected *JetStreamAccountLimits, config *StreamConfig, currentRes, maxBytesOffset int64) error {\n\treturn js.checkLimits(selected, config, true, currentRes, maxBytesOffset)\n}\n\n// Check if a new proposed msg set while exceed our account limits.\n// Lock should be held.\nfunc (js *jetStream) checkLimits(selected *JetStreamAccountLimits, config *StreamConfig, checkServer bool, currentRes, maxBytesOffset int64) error {\n\t// Check MaxConsumers\n\tif config.MaxConsumers > 0 && selected.MaxConsumers > 0 && config.MaxConsumers > selected.MaxConsumers {\n\t\treturn NewJSMaximumConsumersLimitError()\n\t}\n\t// stream limit is checked separately on stream create only!\n\t// Check storage, memory or disk.\n\treturn js.checkBytesLimits(selected, config.MaxBytes, config.Storage, checkServer, currentRes, maxBytesOffset)\n}\n\n// Check if additional bytes will exceed our account limits and optionally the server itself.\n// Read Lock should be held.\nfunc (js *jetStream) checkBytesLimits(selectedLimits *JetStreamAccountLimits, addBytes int64, storage StorageType, checkServer bool, currentRes, maxBytesOffset int64) error {\n\tif addBytes < 0 {\n\t\taddBytes = 1\n\t}\n\ttotalBytes := addBytes + maxBytesOffset\n\n\tswitch storage {\n\tcase MemoryStorage:\n\t\t// Account limits defined.\n\t\tif selectedLimits.MaxMemory >= 0 && currentRes+totalBytes > selectedLimits.MaxMemory {\n\t\t\treturn NewJSMemoryResourcesExceededError()\n\t\t}\n\t\t// Check if this server can handle request.\n\t\tif checkServer && js.memReserved+totalBytes > js.config.MaxMemory {\n\t\t\treturn NewJSMemoryResourcesExceededError()\n\t\t}\n\tcase FileStorage:\n\t\t// Account limits defined.\n\t\tif selectedLimits.MaxStore >= 0 && currentRes+totalBytes > selectedLimits.MaxStore {\n\t\t\treturn NewJSStorageResourcesExceededError()\n\t\t}\n\t\t// Check if this server can handle request.\n\t\tif checkServer && js.storeReserved+totalBytes > js.config.MaxStore {\n\t\t\treturn NewJSStorageResourcesExceededError()\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (jsa *jsAccount) acc() *Account {\n\treturn jsa.account\n}\n\n// Delete the JetStream resources.\nfunc (jsa *jsAccount) delete() {\n\tvar streams []*stream\n\n\tjsa.mu.Lock()\n\t// The update timer and subs need to be protected by usageMu lock\n\tjsa.usageMu.Lock()\n\tif jsa.utimer != nil {\n\t\tjsa.utimer.Stop()\n\t\tjsa.utimer = nil\n\t}\n\tif jsa.updatesSub != nil && jsa.js.srv != nil {\n\t\ts := jsa.js.srv\n\t\ts.sysUnsubscribe(jsa.updatesSub)\n\t\tjsa.updatesSub = nil\n\t}\n\tjsa.usageMu.Unlock()\n\n\tfor _, ms := range jsa.streams {\n\t\tstreams = append(streams, ms)\n\t}\n\tjsa.mu.Unlock()\n\n\tfor _, mset := range streams {\n\t\tmset.stop(false, false)\n\t}\n}\n\n// Lookup the jetstream account for a given account.\nfunc (js *jetStream) lookupAccount(a *Account) *jsAccount {\n\tif a == nil {\n\t\treturn nil\n\t}\n\tjs.mu.RLock()\n\tjsa := js.accounts[a.Name]\n\tjs.mu.RUnlock()\n\treturn jsa\n}\n\n// Report on JetStream stats and usage for this server.\nfunc (js *jetStream) usageStats() *JetStreamStats {\n\tvar stats JetStreamStats\n\tjs.mu.RLock()\n\tstats.Accounts = len(js.accounts)\n\tstats.ReservedMemory = uint64(js.memReserved)\n\tstats.ReservedStore = uint64(js.storeReserved)\n\ts := js.srv\n\tjs.mu.RUnlock()\n\tstats.API.Level = JSApiLevel\n\tstats.API.Total = uint64(atomic.LoadInt64(&js.apiTotal))\n\tstats.API.Errors = uint64(atomic.LoadInt64(&js.apiErrors))\n\tstats.API.Inflight = uint64(atomic.LoadInt64(&js.apiInflight))\n\t// Make sure we do not report negative.\n\tused := atomic.LoadInt64(&js.memUsed)\n\tif used < 0 {\n\t\tused = 0\n\t}\n\tstats.Memory = uint64(used)\n\tused = atomic.LoadInt64(&js.storeUsed)\n\tif used < 0 {\n\t\tused = 0\n\t}\n\tstats.Store = uint64(used)\n\tstats.HAAssets = s.numRaftNodes()\n\treturn &stats\n}\n\n// Check to see if we have enough system resources for this account.\n// Lock should be held.\nfunc (js *jetStream) sufficientResources(limits map[string]JetStreamAccountLimits) error {\n\t// If we are clustered we do not really know how many resources will be ultimately available.\n\t// This needs to be handled out of band.\n\t// If we are a single server, we can make decisions here.\n\tif limits == nil || !js.standAlone {\n\t\treturn nil\n\t}\n\n\ttotalMaxBytes := func(limits map[string]JetStreamAccountLimits) (int64, int64) {\n\t\ttotalMaxMemory := int64(0)\n\t\ttotalMaxStore := int64(0)\n\t\tfor _, l := range limits {\n\t\t\tif l.MaxMemory > 0 {\n\t\t\t\ttotalMaxMemory += l.MaxMemory\n\t\t\t}\n\t\t\tif l.MaxStore > 0 {\n\t\t\t\ttotalMaxStore += l.MaxStore\n\t\t\t}\n\t\t}\n\t\treturn totalMaxMemory, totalMaxStore\n\t}\n\n\ttotalMaxMemory, totalMaxStore := totalMaxBytes(limits)\n\n\t// Reserved is now specific to the MaxBytes for streams.\n\tif js.memReserved+totalMaxMemory > js.config.MaxMemory {\n\t\treturn NewJSMemoryResourcesExceededError()\n\t}\n\tif js.storeReserved+totalMaxStore > js.config.MaxStore {\n\t\treturn NewJSStorageResourcesExceededError()\n\t}\n\n\t// Since we know if we are here we are single server mode, check the account reservations.\n\tvar storeReserved, memReserved int64\n\tfor _, jsa := range js.accounts {\n\t\tif jsa.account.IsExpired() {\n\t\t\tcontinue\n\t\t}\n\t\tjsa.usageMu.RLock()\n\t\tmaxMemory, maxStore := totalMaxBytes(jsa.limits)\n\t\tjsa.usageMu.RUnlock()\n\t\tmemReserved += maxMemory\n\t\tstoreReserved += maxStore\n\t}\n\n\tif memReserved+totalMaxMemory > js.config.MaxMemory {\n\t\treturn NewJSMemoryResourcesExceededError()\n\t}\n\tif storeReserved+totalMaxStore > js.config.MaxStore {\n\t\treturn NewJSStorageResourcesExceededError()\n\t}\n\n\treturn nil\n}\n\n// This will reserve the stream resources requested.\n// This will spin off off of MaxBytes.\nfunc (js *jetStream) reserveStreamResources(cfg *StreamConfig) {\n\tif cfg == nil || cfg.MaxBytes <= 0 {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tswitch cfg.Storage {\n\tcase MemoryStorage:\n\t\tjs.memReserved += cfg.MaxBytes\n\tcase FileStorage:\n\t\tjs.storeReserved += cfg.MaxBytes\n\t}\n\ts, clustered := js.srv, !js.standAlone\n\tjs.mu.Unlock()\n\t// If clustered send an update to the system immediately.\n\tif clustered {\n\t\ts.sendStatszUpdate()\n\t}\n}\n\n// Release reserved resources held by a stream.\nfunc (js *jetStream) releaseStreamResources(cfg *StreamConfig) {\n\tif cfg == nil || cfg.MaxBytes <= 0 {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tswitch cfg.Storage {\n\tcase MemoryStorage:\n\t\tjs.memReserved -= cfg.MaxBytes\n\tcase FileStorage:\n\t\tjs.storeReserved -= cfg.MaxBytes\n\t}\n\ts, clustered := js.srv, !js.standAlone\n\tjs.mu.Unlock()\n\t// If clustered send an update to the system immediately.\n\tif clustered {\n\t\ts.sendStatszUpdate()\n\t}\n}\n\nconst (\n\t// JetStreamStoreDir is the prefix we use.\n\tJetStreamStoreDir = \"jetstream\"\n\t// JetStreamMaxStoreDefault is the default disk storage limit. 1TB\n\tJetStreamMaxStoreDefault = 1024 * 1024 * 1024 * 1024\n\t// JetStreamMaxMemDefault is only used when we can't determine system memory. 256MB\n\tJetStreamMaxMemDefault = 1024 * 1024 * 256\n\t// snapshot staging for restores.\n\tsnapStagingDir = \".snap-staging\"\n)\n\n// Dynamically create a config with a tmp based directory (repeatable) and 75% of system memory.\nfunc (s *Server) dynJetStreamConfig(storeDir string, maxStore, maxMem int64) *JetStreamConfig {\n\tjsc := &JetStreamConfig{}\n\tif storeDir != _EMPTY_ {\n\t\tjsc.StoreDir = filepath.Join(storeDir, JetStreamStoreDir)\n\t} else {\n\t\t// Create one in tmp directory, but make it consistent for restarts.\n\t\tjsc.StoreDir = filepath.Join(os.TempDir(), \"nats\", JetStreamStoreDir)\n\t\ts.Warnf(\"Temporary storage directory used, data could be lost on system reboot\")\n\t}\n\n\topts := s.getOpts()\n\n\t// Strict mode.\n\tjsc.Strict = !opts.NoJetStreamStrict\n\n\t// Sync options.\n\tjsc.SyncInterval = opts.SyncInterval\n\tjsc.SyncAlways = opts.SyncAlways\n\n\tif opts.maxStoreSet && maxStore >= 0 {\n\t\tjsc.MaxStore = maxStore\n\t} else {\n\t\tjsc.MaxStore = diskAvailable(jsc.StoreDir)\n\t}\n\n\tif opts.maxMemSet && maxMem >= 0 {\n\t\tjsc.MaxMemory = maxMem\n\t} else {\n\t\t// Estimate to 75% of total memory if we can determine system memory.\n\t\tif sysMem := sysmem.Memory(); sysMem > 0 {\n\t\t\t// Check if we have been limited with GOMEMLIMIT and if lower use that value.\n\t\t\tif gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 && gml < sysMem {\n\t\t\t\ts.Debugf(\"JetStream detected GOMEMLIMIT of %v\", friendlyBytes(gml))\n\t\t\t\tsysMem = gml\n\t\t\t}\n\t\t\tjsc.MaxMemory = sysMem / 4 * 3\n\t\t} else {\n\t\t\tjsc.MaxMemory = JetStreamMaxMemDefault\n\t\t}\n\t}\n\n\treturn jsc\n}\n\n// Helper function.\nfunc (a *Account) checkForJetStream() (*Server, *jsAccount, error) {\n\ta.mu.RLock()\n\ts := a.srv\n\tjsa := a.js\n\ta.mu.RUnlock()\n\n\tif s == nil || jsa == nil {\n\t\treturn nil, nil, NewJSNotEnabledForAccountError()\n\t}\n\n\treturn s, jsa, nil\n}\n\ntype Number interface {\n\tint | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64\n}\n\n// friendlyBytes returns a string with the given bytes int64\n// represented as a size, such as 1KB, 10MB, etc...\nfunc friendlyBytes[T Number](bytes T) string {\n\tfbytes := float64(bytes)\n\tbase := 1024\n\tpre := []string{\"K\", \"M\", \"G\", \"T\", \"P\", \"E\"}\n\tif fbytes < float64(base) {\n\t\treturn fmt.Sprintf(\"%v B\", fbytes)\n\t}\n\texp := int(math.Log(fbytes) / math.Log(float64(base)))\n\tindex := exp - 1\n\treturn fmt.Sprintf(\"%.2f %sB\", fbytes/math.Pow(float64(base), float64(exp)), pre[index])\n}\n\nfunc isValidName(name string) bool {\n\tif name == _EMPTY_ {\n\t\treturn false\n\t}\n\treturn !strings.ContainsAny(name, \" \\t\\r\\n\\f.*>\")\n}\n\n// To throttle the out of resources errors.\nfunc (s *Server) resourcesExceededError(storeType StorageType) {\n\tvar didAlert bool\n\n\ts.rerrMu.Lock()\n\tif now := time.Now(); now.Sub(s.rerrLast) > 10*time.Second {\n\t\ts.Errorf(\"JetStream %s resource limits exceeded for server\", strings.ToLower(storeType.String()))\n\t\ts.rerrLast = now\n\t\tdidAlert = true\n\t}\n\ts.rerrMu.Unlock()\n\n\t// If we are meta leader we should relinquish that here.\n\tif didAlert {\n\t\tif js := s.getJetStream(); js != nil {\n\t\t\tjs.mu.RLock()\n\t\t\tif cc := js.cluster; cc != nil && cc.meta != nil {\n\t\t\t\tcc.meta.StepDown()\n\t\t\t}\n\t\t\tjs.mu.RUnlock()\n\t\t}\n\t}\n}\n\n// For validating options.\nfunc validateJetStreamOptions(o *Options) error {\n\t// in non operator mode, the account names need to be configured\n\tif len(o.JsAccDefaultDomain) > 0 {\n\t\tif len(o.TrustedOperators) == 0 {\n\t\t\tfor a, domain := range o.JsAccDefaultDomain {\n\t\t\t\tfound := false\n\t\t\t\tif isReservedAccount(a) {\n\t\t\t\t\tfound = true\n\t\t\t\t} else {\n\t\t\t\t\tfor _, acc := range o.Accounts {\n\t\t\t\t\t\tif a == acc.GetName() {\n\t\t\t\t\t\t\tif len(acc.jsLimits) > 0 && domain != _EMPTY_ {\n\t\t\t\t\t\t\t\treturn fmt.Errorf(\"default_js_domain contains account name %q with enabled JetStream\", a)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tfound = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\treturn fmt.Errorf(\"in non operator mode, `default_js_domain` references non existing account %q\", a)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor a := range o.JsAccDefaultDomain {\n\t\t\t\tif !nkeys.IsValidPublicAccountKey(a) {\n\t\t\t\t\treturn fmt.Errorf(\"default_js_domain contains account name %q, which is not a valid public account nkey\", a)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor a, d := range o.JsAccDefaultDomain {\n\t\t\tsacc := DEFAULT_SYSTEM_ACCOUNT\n\t\t\tif o.SystemAccount != _EMPTY_ {\n\t\t\t\tsacc = o.SystemAccount\n\t\t\t}\n\t\t\tif a == sacc {\n\t\t\t\treturn fmt.Errorf(\"system account %q can not be in default_js_domain\", a)\n\t\t\t}\n\t\t\tif d == _EMPTY_ {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif sub := fmt.Sprintf(jsDomainAPI, d); !IsValidSubject(sub) {\n\t\t\t\treturn fmt.Errorf(\"default_js_domain contains account %q with invalid domain name %q\", a, d)\n\t\t\t}\n\t\t}\n\t}\n\tif o.JetStreamDomain != _EMPTY_ {\n\t\tif subj := fmt.Sprintf(jsDomainAPI, o.JetStreamDomain); !IsValidSubject(subj) {\n\t\t\treturn fmt.Errorf(\"invalid domain name: derived %q is not a valid subject\", subj)\n\t\t}\n\n\t\tif !isValidName(o.JetStreamDomain) {\n\t\t\treturn fmt.Errorf(\"invalid domain name: may not contain ., * or >\")\n\t\t}\n\t}\n\t// If not clustered no checks needed past here.\n\tif !o.JetStream || o.Cluster.Port == 0 {\n\t\treturn nil\n\t}\n\tif o.ServerName == _EMPTY_ {\n\t\treturn fmt.Errorf(\"jetstream cluster requires `server_name` to be set\")\n\t}\n\tif o.Cluster.Name == _EMPTY_ {\n\t\treturn fmt.Errorf(\"jetstream cluster requires `cluster.name` to be set\")\n\t}\n\n\th := strings.ToLower(o.JetStreamExtHint)\n\tswitch h {\n\tcase jsWillExtend, jsNoExtend, _EMPTY_:\n\t\to.JetStreamExtHint = h\n\tdefault:\n\t\treturn fmt.Errorf(\"expected 'no_extend' for string value, got '%s'\", h)\n\t}\n\n\tif o.JetStreamMaxCatchup < 0 {\n\t\treturn fmt.Errorf(\"jetstream max catchup cannot be negative\")\n\t}\n\treturn nil\n}\n\n// We had a bug that set a default de dupe window on mirror, despite that being not a valid config\nfunc fixCfgMirrorWithDedupWindow(cfg *StreamConfig) {\n\tif cfg == nil || cfg.Mirror == nil {\n\t\treturn\n\t}\n\tif cfg.Duplicates != 0 {\n\t\tcfg.Duplicates = 0\n\t}\n}\n\nfunc (s *Server) handleWritePermissionError() {\n\t//TODO Check if we should add s.jetStreamOOSPending in condition\n\tif s.JetStreamEnabled() {\n\t\ts.Errorf(\"File system permission denied while writing, disabling JetStream\")\n\n\t\tgo s.DisableJetStream()\n\n\t\t//TODO Send respective advisory if needed, same as in handleOutOfSpace\n\t}\n}\n" }, { "path": "server/jetstream_api.go", "content": "// Copyright 2020-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/nats-io/nuid\"\n)\n\n// Request API subjects for JetStream.\nconst (\n\t// All API endpoints.\n\tjsAllAPI = \"$JS.API.>\"\n\n\t// For constructing JetStream domain prefixes.\n\tjsDomainAPI = \"$JS.%s.API.>\"\n\n\tJSApiPrefix = \"$JS.API\"\n\n\t// JSApiAccountInfo is for obtaining general information about JetStream for this account.\n\t// Will return JSON response.\n\tJSApiAccountInfo = \"$JS.API.INFO\"\n\n\t// JSApiStreamCreate is the endpoint to create new streams.\n\t// Will return JSON response.\n\tJSApiStreamCreate = \"$JS.API.STREAM.CREATE.*\"\n\tJSApiStreamCreateT = \"$JS.API.STREAM.CREATE.%s\"\n\n\t// JSApiStreamUpdate is the endpoint to update existing streams.\n\t// Will return JSON response.\n\tJSApiStreamUpdate = \"$JS.API.STREAM.UPDATE.*\"\n\tJSApiStreamUpdateT = \"$JS.API.STREAM.UPDATE.%s\"\n\n\t// JSApiStreams is the endpoint to list all stream names for this account.\n\t// Will return JSON response.\n\tJSApiStreams = \"$JS.API.STREAM.NAMES\"\n\t// JSApiStreamList is the endpoint that will return all detailed stream information\n\tJSApiStreamList = \"$JS.API.STREAM.LIST\"\n\n\t// JSApiStreamInfo is for obtaining general information about a named stream.\n\t// Will return JSON response.\n\tJSApiStreamInfo = \"$JS.API.STREAM.INFO.*\"\n\tJSApiStreamInfoT = \"$JS.API.STREAM.INFO.%s\"\n\n\t// JSApiStreamDelete is the endpoint to delete streams.\n\t// Will return JSON response.\n\tJSApiStreamDelete = \"$JS.API.STREAM.DELETE.*\"\n\tJSApiStreamDeleteT = \"$JS.API.STREAM.DELETE.%s\"\n\n\t// JSApiStreamPurge is the endpoint to purge streams.\n\t// Will return JSON response.\n\tJSApiStreamPurge = \"$JS.API.STREAM.PURGE.*\"\n\tJSApiStreamPurgeT = \"$JS.API.STREAM.PURGE.%s\"\n\n\t// JSApiStreamSnapshot is the endpoint to snapshot streams.\n\t// Will return a stream of chunks with a nil chunk as EOF to\n\t// the deliver subject. Caller should respond to each chunk\n\t// with a nil body response for ack flow.\n\tJSApiStreamSnapshot = \"$JS.API.STREAM.SNAPSHOT.*\"\n\tJSApiStreamSnapshotT = \"$JS.API.STREAM.SNAPSHOT.%s\"\n\n\t// JSApiStreamRestore is the endpoint to restore a stream from a snapshot.\n\t// Caller should respond to each chunk with a nil body response.\n\tJSApiStreamRestore = \"$JS.API.STREAM.RESTORE.*\"\n\tJSApiStreamRestoreT = \"$JS.API.STREAM.RESTORE.%s\"\n\n\t// JSApiMsgDelete is the endpoint to delete messages from a stream.\n\t// Will return JSON response.\n\tJSApiMsgDelete = \"$JS.API.STREAM.MSG.DELETE.*\"\n\tJSApiMsgDeleteT = \"$JS.API.STREAM.MSG.DELETE.%s\"\n\n\t// JSApiMsgGet is the template for direct requests for a message by its stream sequence number.\n\t// Will return JSON response.\n\tJSApiMsgGet = \"$JS.API.STREAM.MSG.GET.*\"\n\tJSApiMsgGetT = \"$JS.API.STREAM.MSG.GET.%s\"\n\n\t// JSDirectMsgGet is the template for non-api layer direct requests for a message by its stream sequence number or last by subject.\n\t// Will return the message similar to how a consumer receives the message, no JSON processing.\n\t// If the message can not be found we will use a status header of 404. If the stream does not exist the client will get a no-responders or timeout.\n\tJSDirectMsgGet = \"$JS.API.DIRECT.GET.*\"\n\tJSDirectMsgGetT = \"$JS.API.DIRECT.GET.%s\"\n\n\t// This is a direct version of get last by subject, which will be the dominant pattern for KV access once 2.9 is released.\n\t// The stream and the key will be part of the subject to allow for no-marshal payloads and subject based security permissions.\n\tJSDirectGetLastBySubject = \"$JS.API.DIRECT.GET.*.>\"\n\tJSDirectGetLastBySubjectT = \"$JS.API.DIRECT.GET.%s.%s\"\n\n\t// jsDirectGetPre\n\tjsDirectGetPre = \"$JS.API.DIRECT.GET\"\n\n\t// JSApiConsumerCreate is the endpoint to create consumers for streams.\n\t// This was also the legacy endpoint for ephemeral consumers.\n\t// It now can take consumer name and optional filter subject, which when part of the subject controls access.\n\t// Will return JSON response.\n\tJSApiConsumerCreate = \"$JS.API.CONSUMER.CREATE.*\"\n\tJSApiConsumerCreateT = \"$JS.API.CONSUMER.CREATE.%s\"\n\tJSApiConsumerCreateEx = \"$JS.API.CONSUMER.CREATE.*.>\"\n\tJSApiConsumerCreateExT = \"$JS.API.CONSUMER.CREATE.%s.%s.%s\"\n\n\t// JSApiDurableCreate is the endpoint to create durable consumers for streams.\n\t// You need to include the stream and consumer name in the subject.\n\tJSApiDurableCreate = \"$JS.API.CONSUMER.DURABLE.CREATE.*.*\"\n\tJSApiDurableCreateT = \"$JS.API.CONSUMER.DURABLE.CREATE.%s.%s\"\n\n\t// JSApiConsumers is the endpoint to list all consumer names for the stream.\n\t// Will return JSON response.\n\tJSApiConsumers = \"$JS.API.CONSUMER.NAMES.*\"\n\tJSApiConsumersT = \"$JS.API.CONSUMER.NAMES.%s\"\n\n\t// JSApiConsumerList is the endpoint that will return all detailed consumer information\n\tJSApiConsumerList = \"$JS.API.CONSUMER.LIST.*\"\n\tJSApiConsumerListT = \"$JS.API.CONSUMER.LIST.%s\"\n\n\t// JSApiConsumerInfo is for obtaining general information about a consumer.\n\t// Will return JSON response.\n\tJSApiConsumerInfo = \"$JS.API.CONSUMER.INFO.*.*\"\n\tJSApiConsumerInfoT = \"$JS.API.CONSUMER.INFO.%s.%s\"\n\n\t// JSApiConsumerDelete is the endpoint to delete consumers.\n\t// Will return JSON response.\n\tJSApiConsumerDelete = \"$JS.API.CONSUMER.DELETE.*.*\"\n\tJSApiConsumerDeleteT = \"$JS.API.CONSUMER.DELETE.%s.%s\"\n\n\t// JSApiConsumerPause is the endpoint to pause or unpause consumers.\n\t// Will return JSON response.\n\tJSApiConsumerPause = \"$JS.API.CONSUMER.PAUSE.*.*\"\n\tJSApiConsumerPauseT = \"$JS.API.CONSUMER.PAUSE.%s.%s\"\n\n\t// JSApiRequestNextT is the prefix for the request next message(s) for a consumer in worker/pull mode.\n\tJSApiRequestNextT = \"$JS.API.CONSUMER.MSG.NEXT.%s.%s\"\n\n\t// JSApiConsumerResetT is the prefix for resetting a given consumer to a new starting sequence.\n\tJSApiConsumerResetT = \"$JS.API.CONSUMER.RESET.%s.%s\"\n\n\t// JSApiConsumerUnpinT is the prefix for unpinning subscription for a given consumer.\n\tJSApiConsumerUnpin = \"$JS.API.CONSUMER.UNPIN.*.*\"\n\tJSApiConsumerUnpinT = \"$JS.API.CONSUMER.UNPIN.%s.%s\"\n\n\t// jsRequestNextPre\n\tjsRequestNextPre = \"$JS.API.CONSUMER.MSG.NEXT.\"\n\n\t// For snapshots and restores. The ack will have additional tokens.\n\tjsSnapshotAckT = \"$JS.SNAPSHOT.ACK.%s.%s\"\n\tjsRestoreDeliverT = \"$JS.SNAPSHOT.RESTORE.%s.%s\"\n\n\t// JSApiStreamRemovePeer is the endpoint to remove a peer from a clustered stream and its consumers.\n\t// Will return JSON response.\n\tJSApiStreamRemovePeer = \"$JS.API.STREAM.PEER.REMOVE.*\"\n\tJSApiStreamRemovePeerT = \"$JS.API.STREAM.PEER.REMOVE.%s\"\n\n\t// JSApiStreamLeaderStepDown is the endpoint to have stream leader stepdown.\n\t// Will return JSON response.\n\tJSApiStreamLeaderStepDown = \"$JS.API.STREAM.LEADER.STEPDOWN.*\"\n\tJSApiStreamLeaderStepDownT = \"$JS.API.STREAM.LEADER.STEPDOWN.%s\"\n\n\t// JSApiConsumerLeaderStepDown is the endpoint to have consumer leader stepdown.\n\t// Will return JSON response.\n\tJSApiConsumerLeaderStepDown = \"$JS.API.CONSUMER.LEADER.STEPDOWN.*.*\"\n\tJSApiConsumerLeaderStepDownT = \"$JS.API.CONSUMER.LEADER.STEPDOWN.%s.%s\"\n\n\t// JSApiLeaderStepDown is the endpoint to have our metaleader stepdown.\n\t// Only works from system account.\n\t// Will return JSON response.\n\tJSApiLeaderStepDown = \"$JS.API.META.LEADER.STEPDOWN\"\n\n\t// JSApiRemoveServer is the endpoint to remove a peer server from the cluster.\n\t// Only works from system account.\n\t// Will return JSON response.\n\tJSApiRemoveServer = \"$JS.API.SERVER.REMOVE\"\n\n\t// JSApiAccountPurge is the endpoint to purge the js content of an account\n\t// Only works from system account.\n\t// Will return JSON response.\n\tJSApiAccountPurge = \"$JS.API.ACCOUNT.PURGE.*\"\n\tJSApiAccountPurgeT = \"$JS.API.ACCOUNT.PURGE.%s\"\n\n\t// JSApiServerStreamMove is the endpoint to move streams off a server\n\t// Only works from system account.\n\t// Will return JSON response.\n\tJSApiServerStreamMove = \"$JS.API.ACCOUNT.STREAM.MOVE.*.*\"\n\tJSApiServerStreamMoveT = \"$JS.API.ACCOUNT.STREAM.MOVE.%s.%s\"\n\n\t// JSApiServerStreamCancelMove is the endpoint to cancel a stream move\n\t// Only works from system account.\n\t// Will return JSON response.\n\tJSApiServerStreamCancelMove = \"$JS.API.ACCOUNT.STREAM.CANCEL_MOVE.*.*\"\n\tJSApiServerStreamCancelMoveT = \"$JS.API.ACCOUNT.STREAM.CANCEL_MOVE.%s.%s\"\n\n\t// The prefix for system level account API.\n\tjsAPIAccountPre = \"$JS.API.ACCOUNT.\"\n\n\t// jsAckT is the template for the ack message stream coming back from a consumer\n\t// when they ACK/NAK, etc a message.\n\tjsAckT = \"$JS.ACK.%s.%s\"\n\tjsAckPre = \"$JS.ACK.\"\n\tjsAckPreLen = len(jsAckPre)\n\n\t// jsFlowControl is for flow control subjects.\n\tjsFlowControlPre = \"$JS.FC.\"\n\t// jsFlowControl is for FC responses.\n\tjsFlowControl = \"$JS.FC.%s.%s.*\"\n\n\t// JSAdvisoryPrefix is a prefix for all JetStream advisories.\n\tJSAdvisoryPrefix = \"$JS.EVENT.ADVISORY\"\n\n\t// JSMetricPrefix is a prefix for all JetStream metrics.\n\tJSMetricPrefix = \"$JS.EVENT.METRIC\"\n\n\t// JSMetricConsumerAckPre is a metric containing ack latency.\n\tJSMetricConsumerAckPre = \"$JS.EVENT.METRIC.CONSUMER.ACK\"\n\n\t// JSAdvisoryConsumerMaxDeliveryExceedPre is a notification published when a message exceeds its delivery threshold.\n\tJSAdvisoryConsumerMaxDeliveryExceedPre = \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES\"\n\n\t// JSAdvisoryConsumerMsgNakPre is a notification published when a message has been naked\n\tJSAdvisoryConsumerMsgNakPre = \"$JS.EVENT.ADVISORY.CONSUMER.MSG_NAKED\"\n\n\t// JSAdvisoryConsumerMsgTerminatedPre is a notification published when a message has been terminated.\n\tJSAdvisoryConsumerMsgTerminatedPre = \"$JS.EVENT.ADVISORY.CONSUMER.MSG_TERMINATED\"\n\n\t// JSAdvisoryStreamCreatedPre notification that a stream was created.\n\tJSAdvisoryStreamCreatedPre = \"$JS.EVENT.ADVISORY.STREAM.CREATED\"\n\n\t// JSAdvisoryStreamDeletedPre notification that a stream was deleted.\n\tJSAdvisoryStreamDeletedPre = \"$JS.EVENT.ADVISORY.STREAM.DELETED\"\n\n\t// JSAdvisoryStreamUpdatedPre notification that a stream was updated.\n\tJSAdvisoryStreamUpdatedPre = \"$JS.EVENT.ADVISORY.STREAM.UPDATED\"\n\n\t// JSAdvisoryConsumerCreatedPre notification that a consumer was created.\n\tJSAdvisoryConsumerCreatedPre = \"$JS.EVENT.ADVISORY.CONSUMER.CREATED\"\n\n\t// JSAdvisoryConsumerDeletedPre notification that a consumer was deleted.\n\tJSAdvisoryConsumerDeletedPre = \"$JS.EVENT.ADVISORY.CONSUMER.DELETED\"\n\n\t// JSAdvisoryConsumerPausePre notification that a consumer paused/unpaused.\n\tJSAdvisoryConsumerPausePre = \"$JS.EVENT.ADVISORY.CONSUMER.PAUSE\"\n\n\t// JSAdvisoryConsumerPinnedPre notification that a consumer was pinned.\n\tJSAdvisoryConsumerPinnedPre = \"$JS.EVENT.ADVISORY.CONSUMER.PINNED\"\n\n\t// JSAdvisoryConsumerUnpinnedPre notification that a consumer was unpinned.\n\tJSAdvisoryConsumerUnpinnedPre = \"$JS.EVENT.ADVISORY.CONSUMER.UNPINNED\"\n\n\t// JSAdvisoryStreamSnapshotCreatePre notification that a snapshot was created.\n\tJSAdvisoryStreamSnapshotCreatePre = \"$JS.EVENT.ADVISORY.STREAM.SNAPSHOT_CREATE\"\n\n\t// JSAdvisoryStreamSnapshotCompletePre notification that a snapshot was completed.\n\tJSAdvisoryStreamSnapshotCompletePre = \"$JS.EVENT.ADVISORY.STREAM.SNAPSHOT_COMPLETE\"\n\n\t// JSAdvisoryStreamRestoreCreatePre notification that a restore was start.\n\tJSAdvisoryStreamRestoreCreatePre = \"$JS.EVENT.ADVISORY.STREAM.RESTORE_CREATE\"\n\n\t// JSAdvisoryStreamRestoreCompletePre notification that a restore was completed.\n\tJSAdvisoryStreamRestoreCompletePre = \"$JS.EVENT.ADVISORY.STREAM.RESTORE_COMPLETE\"\n\n\t// JSAdvisoryDomainLeaderElectedPre notification that a jetstream domain has elected a leader.\n\tJSAdvisoryDomainLeaderElected = \"$JS.EVENT.ADVISORY.DOMAIN.LEADER_ELECTED\"\n\n\t// JSAdvisoryStreamLeaderElectedPre notification that a replicated stream has elected a leader.\n\tJSAdvisoryStreamLeaderElectedPre = \"$JS.EVENT.ADVISORY.STREAM.LEADER_ELECTED\"\n\n\t// JSAdvisoryStreamQuorumLostPre notification that a stream and its consumers are stalled.\n\tJSAdvisoryStreamQuorumLostPre = \"$JS.EVENT.ADVISORY.STREAM.QUORUM_LOST\"\n\n\t// JSAdvisoryStreamBatchAbandonedPre notification that a stream's batch was abandoned.\n\tJSAdvisoryStreamBatchAbandonedPre = \"$JS.EVENT.ADVISORY.STREAM.BATCH_ABANDONED\"\n\n\t// JSAdvisoryConsumerLeaderElectedPre notification that a replicated consumer has elected a leader.\n\tJSAdvisoryConsumerLeaderElectedPre = \"$JS.EVENT.ADVISORY.CONSUMER.LEADER_ELECTED\"\n\n\t// JSAdvisoryConsumerQuorumLostPre notification that a consumer is stalled.\n\tJSAdvisoryConsumerQuorumLostPre = \"$JS.EVENT.ADVISORY.CONSUMER.QUORUM_LOST\"\n\n\t// JSAdvisoryServerOutOfStorage notification that a server has no more storage.\n\tJSAdvisoryServerOutOfStorage = \"$JS.EVENT.ADVISORY.SERVER.OUT_OF_STORAGE\"\n\n\t// JSAdvisoryServerRemoved notification that a server has been removed from the system.\n\tJSAdvisoryServerRemoved = \"$JS.EVENT.ADVISORY.SERVER.REMOVED\"\n\n\t// JSAdvisoryAPILimitReached notification that a server has reached the JS API hard limit.\n\tJSAdvisoryAPILimitReached = \"$JS.EVENT.ADVISORY.API.LIMIT_REACHED\"\n\n\t// JSAuditAdvisory is a notification about JetStream API access.\n\t// FIXME - Add in details about who..\n\tJSAuditAdvisory = \"$JS.EVENT.ADVISORY.API\"\n)\n\n// Headers used in $JS.API.> requests.\nconst (\n\t// JSRequiredApiLevel requires the API level of the responding server to have the specified minimum value.\n\tJSRequiredApiLevel = \"Nats-Required-Api-Level\"\n)\n\nvar denyAllClientJs = []string{jsAllAPI, \"$KV.>\", \"$OBJ.>\"}\nvar denyAllJs = []string{jscAllSubj, raftAllSubj, jsAllAPI, \"$KV.>\", \"$OBJ.>\"}\n\nfunc generateJSMappingTable(domain string) map[string]string {\n\tmappings := map[string]string{}\n\t// This set of mappings is very very very ugly.\n\t// It is a consequence of what we defined the domain prefix to be \"$JS.domain.API\" and it's mapping to \"$JS.API\"\n\t// For optics $KV and $OBJ where made to be independent subject spaces.\n\t// As materialized views of JS, they did not simply extend that subject space to say \"$JS.API.KV\" \"$JS.API.OBJ\"\n\t// This is very unfortunate!!!\n\t// Furthermore, it seemed bad to require different domain prefixes for JS/KV/OBJ.\n\t// Especially since the actual API for say KV, does use stream create from JS.\n\t// To avoid overlaps KV and OBJ views append the prefix to their API.\n\t// (Replacing $KV with the prefix allows users to create collisions with say the bucket name)\n\t// This mapping therefore needs to have extra token so that the mapping can properly discern between $JS, $KV, $OBJ\n\tfor srcMappingSuffix, to := range map[string]string{\n\t\t\"INFO\": JSApiAccountInfo,\n\t\t\"STREAM.>\": \"$JS.API.STREAM.>\",\n\t\t\"CONSUMER.>\": \"$JS.API.CONSUMER.>\",\n\t\t\"DIRECT.>\": \"$JS.API.DIRECT.>\",\n\t\t\"META.>\": \"$JS.API.META.>\",\n\t\t\"SERVER.>\": \"$JS.API.SERVER.>\",\n\t\t\"ACCOUNT.>\": \"$JS.API.ACCOUNT.>\",\n\t\t\"$KV.>\": \"$KV.>\",\n\t\t\"$OBJ.>\": \"$OBJ.>\",\n\t} {\n\t\tmappings[fmt.Sprintf(\"$JS.%s.API.%s\", domain, srcMappingSuffix)] = to\n\t}\n\treturn mappings\n}\n\n// JSMaxDescription is the maximum description length for streams and consumers.\nconst JSMaxDescriptionLen = 4 * 1024\n\n// JSMaxMetadataLen is the maximum length for streams and consumers metadata map.\n// It's calculated by summing length of all keys and values.\nconst JSMaxMetadataLen = 128 * 1024\n\n// JSMaxNameLen is the maximum name lengths for streams, consumers and templates.\n// Picked 255 as it seems to be a widely used file name limit\nconst JSMaxNameLen = 255\n\n// JSDefaultRequestQueueLimit is the default number of entries that we will\n// put on the global request queue before we react.\nconst JSDefaultRequestQueueLimit = 10_000\n\n// Responses for API calls.\n\n// ApiResponse is a standard response from the JetStream JSON API\ntype ApiResponse struct {\n\tType string `json:\"type\"`\n\tError *ApiError `json:\"error,omitempty\"`\n}\n\nconst JSApiSystemResponseType = \"io.nats.jetstream.api.v1.system_response\"\n\n// When passing back to the clients generalize store failures.\nvar (\n\terrStreamStoreFailed = errors.New(\"error creating store for stream\")\n\terrConsumerStoreFailed = errors.New(\"error creating store for consumer\")\n)\n\n// ToError checks if the response has a error and if it does converts it to an error avoiding\n// the pitfalls described by https://yourbasic.org/golang/gotcha-why-nil-error-not-equal-nil/\nfunc (r *ApiResponse) ToError() error {\n\tif r.Error == nil {\n\t\treturn nil\n\t}\n\n\treturn r.Error\n}\n\nconst JSApiOverloadedType = \"io.nats.jetstream.api.v1.system_overloaded\"\n\n// ApiPaged includes variables used to create paged responses from the JSON API\ntype ApiPaged struct {\n\tTotal int `json:\"total\"`\n\tOffset int `json:\"offset\"`\n\tLimit int `json:\"limit\"`\n}\n\n// ApiPagedRequest includes parameters allowing specific pages to be requests from APIs responding with ApiPaged\ntype ApiPagedRequest struct {\n\tOffset int `json:\"offset\"`\n}\n\n// JSApiAccountInfoResponse reports back information on jetstream for this account.\ntype JSApiAccountInfoResponse struct {\n\tApiResponse\n\t*JetStreamAccountStats\n}\n\nconst JSApiAccountInfoResponseType = \"io.nats.jetstream.api.v1.account_info_response\"\n\n// JSApiStreamCreateResponse stream creation.\ntype JSApiStreamCreateResponse struct {\n\tApiResponse\n\t*StreamInfo\n\tDidCreate bool `json:\"did_create,omitempty\"`\n}\n\nconst JSApiStreamCreateResponseType = \"io.nats.jetstream.api.v1.stream_create_response\"\n\n// JSApiStreamDeleteResponse stream removal.\ntype JSApiStreamDeleteResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiStreamDeleteResponseType = \"io.nats.jetstream.api.v1.stream_delete_response\"\n\n// JSMaxSubjectDetails The limit of the number of subject details we will send in a stream info response.\nconst JSMaxSubjectDetails = 100_000\n\ntype JSApiStreamInfoRequest struct {\n\tApiPagedRequest\n\tDeletedDetails bool `json:\"deleted_details,omitempty\"`\n\tSubjectsFilter string `json:\"subjects_filter,omitempty\"`\n}\n\ntype JSApiStreamInfoResponse struct {\n\tApiResponse\n\tApiPaged\n\t*StreamInfo\n}\n\nconst JSApiStreamInfoResponseType = \"io.nats.jetstream.api.v1.stream_info_response\"\n\n// JSApiNamesLimit is the maximum entries we will return for streams or consumers lists.\n// TODO(dlc) - with header or request support could request chunked response.\nconst JSApiNamesLimit = 1024\nconst JSApiListLimit = 256\n\ntype JSApiStreamNamesRequest struct {\n\tApiPagedRequest\n\t// These are filters that can be applied to the list.\n\tSubject string `json:\"subject,omitempty\"`\n}\n\n// JSApiStreamNamesResponse list of streams.\n// A nil request is valid and means all streams.\ntype JSApiStreamNamesResponse struct {\n\tApiResponse\n\tApiPaged\n\tStreams []string `json:\"streams\"`\n}\n\nconst JSApiStreamNamesResponseType = \"io.nats.jetstream.api.v1.stream_names_response\"\n\ntype JSApiStreamListRequest struct {\n\tApiPagedRequest\n\t// These are filters that can be applied to the list.\n\tSubject string `json:\"subject,omitempty\"`\n}\n\n// JSApiStreamListResponse list of detailed stream information.\n// A nil request is valid and means all streams.\ntype JSApiStreamListResponse struct {\n\tApiResponse\n\tApiPaged\n\tStreams []*StreamInfo `json:\"streams\"`\n\tMissing []string `json:\"missing,omitempty\"`\n\tOffline map[string]string `json:\"offline,omitempty\"`\n}\n\nconst JSApiStreamListResponseType = \"io.nats.jetstream.api.v1.stream_list_response\"\n\n// JSApiStreamPurgeRequest is optional request information to the purge API.\n// Subject will filter the purge request to only messages that match the subject, which can have wildcards.\n// Sequence will purge up to but not including this sequence and can be combined with subject filtering.\n// Keep will specify how many messages to keep. This can also be combined with subject filtering.\n// Note that Sequence and Keep are mutually exclusive, so both can not be set at the same time.\ntype JSApiStreamPurgeRequest struct {\n\t// Purge up to but not including sequence.\n\tSequence uint64 `json:\"seq,omitempty\"`\n\t// Subject to match against messages for the purge command.\n\tSubject string `json:\"filter,omitempty\"`\n\t// Number of messages to keep.\n\tKeep uint64 `json:\"keep,omitempty\"`\n}\n\ntype JSApiStreamPurgeResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n\tPurged uint64 `json:\"purged\"`\n}\n\nconst JSApiStreamPurgeResponseType = \"io.nats.jetstream.api.v1.stream_purge_response\"\n\ntype JSApiConsumerUnpinRequest struct {\n\tGroup string `json:\"group\"`\n}\n\ntype JSApiConsumerUnpinResponse struct {\n\tApiResponse\n}\n\nconst JSApiConsumerUnpinResponseType = \"io.nats.jetstream.api.v1.consumer_unpin_response\"\n\n// JSApiStreamUpdateResponse for updating a stream.\ntype JSApiStreamUpdateResponse struct {\n\tApiResponse\n\t*StreamInfo\n}\n\nconst JSApiStreamUpdateResponseType = \"io.nats.jetstream.api.v1.stream_update_response\"\n\n// JSApiMsgDeleteRequest delete message request.\ntype JSApiMsgDeleteRequest struct {\n\tSeq uint64 `json:\"seq\"`\n\tNoErase bool `json:\"no_erase,omitempty\"`\n}\n\ntype JSApiMsgDeleteResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiMsgDeleteResponseType = \"io.nats.jetstream.api.v1.stream_msg_delete_response\"\n\ntype JSApiStreamSnapshotRequest struct {\n\t// Subject to deliver the chunks to for the snapshot.\n\tDeliverSubject string `json:\"deliver_subject\"`\n\t// Do not include consumers in the snapshot.\n\tNoConsumers bool `json:\"no_consumers,omitempty\"`\n\t// Optional chunk size preference. Defaults to 128KB,\n\t// automatically clamped to within the range 1KB to 1MB.\n\t// A smaller chunk size means more in-flight messages\n\t// and more acks needed. Links with good throughput\n\t// but high latency may need to increase this.\n\tChunkSize int `json:\"chunk_size,omitempty\"`\n\t// Optional window size preference. Defaults to 8MB,\n\t// automatically clamped to within the range 1KB to 32MB.\n\t// very slow connections may need to reduce this to\n\t// avoid slow consumer issues.\n\tWindowSize int `json:\"window_size,omitempty\"`\n\t// Check all message's checksums prior to snapshot.\n\tCheckMsgs bool `json:\"jsck,omitempty\"`\n}\n\n// JSApiStreamSnapshotResponse is the direct response to the snapshot request.\ntype JSApiStreamSnapshotResponse struct {\n\tApiResponse\n\t// Configuration of the given stream.\n\tConfig *StreamConfig `json:\"config,omitempty\"`\n\t// Current State for the given stream.\n\tState *StreamState `json:\"state,omitempty\"`\n}\n\nconst JSApiStreamSnapshotResponseType = \"io.nats.jetstream.api.v1.stream_snapshot_response\"\n\n// JSApiStreamRestoreRequest is the required restore request.\ntype JSApiStreamRestoreRequest struct {\n\t// Configuration of the given stream.\n\tConfig StreamConfig `json:\"config\"`\n\t// Current State for the given stream.\n\tState StreamState `json:\"state\"`\n}\n\n// JSApiStreamRestoreResponse is the direct response to the restore request.\ntype JSApiStreamRestoreResponse struct {\n\tApiResponse\n\t// Subject to deliver the chunks to for the snapshot restore.\n\tDeliverSubject string `json:\"deliver_subject\"`\n}\n\nconst JSApiStreamRestoreResponseType = \"io.nats.jetstream.api.v1.stream_restore_response\"\n\n// JSApiStreamRemovePeerRequest is the required remove peer request.\ntype JSApiStreamRemovePeerRequest struct {\n\t// Server name or peer ID of the peer to be removed.\n\tPeer string `json:\"peer\"`\n}\n\n// JSApiStreamRemovePeerResponse is the response to a remove peer request.\ntype JSApiStreamRemovePeerResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiStreamRemovePeerResponseType = \"io.nats.jetstream.api.v1.stream_remove_peer_response\"\n\n// JSApiStreamLeaderStepDownResponse is the response to a leader stepdown request.\ntype JSApiStreamLeaderStepDownResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiStreamLeaderStepDownResponseType = \"io.nats.jetstream.api.v1.stream_leader_stepdown_response\"\n\n// JSApiConsumerLeaderStepDownResponse is the response to a consumer leader stepdown request.\ntype JSApiConsumerLeaderStepDownResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiConsumerLeaderStepDownResponseType = \"io.nats.jetstream.api.v1.consumer_leader_stepdown_response\"\n\n// JSApiLeaderStepdownRequest allows placement control over the meta leader placement.\ntype JSApiLeaderStepdownRequest struct {\n\tPlacement *Placement `json:\"placement,omitempty\"`\n}\n\n// JSApiLeaderStepDownResponse is the response to a meta leader stepdown request.\ntype JSApiLeaderStepDownResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiLeaderStepDownResponseType = \"io.nats.jetstream.api.v1.meta_leader_stepdown_response\"\n\n// JSApiMetaServerRemoveRequest will remove a peer from the meta group.\ntype JSApiMetaServerRemoveRequest struct {\n\t// Server name of the peer to be removed.\n\tServer string `json:\"peer\"`\n\t// Peer ID of the peer to be removed. If specified this is used\n\t// instead of the server name.\n\tPeer string `json:\"peer_id,omitempty\"`\n}\n\n// JSApiMetaServerRemoveResponse is the response to a peer removal request in the meta group.\ntype JSApiMetaServerRemoveResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiMetaServerRemoveResponseType = \"io.nats.jetstream.api.v1.meta_server_remove_response\"\n\n// JSApiMetaServerStreamMoveRequest will move a stream on a server to another\n// response to this will come as JSApiStreamUpdateResponse/JSApiStreamUpdateResponseType\ntype JSApiMetaServerStreamMoveRequest struct {\n\t// Server name of the peer to be evacuated.\n\tServer string `json:\"server,omitempty\"`\n\t// Cluster the server is in\n\tCluster string `json:\"cluster,omitempty\"`\n\t// Domain the sever is in\n\tDomain string `json:\"domain,omitempty\"`\n\t// Ephemeral placement tags for the move\n\tTags []string `json:\"tags,omitempty\"`\n}\n\nconst JSApiAccountPurgeResponseType = \"io.nats.jetstream.api.v1.account_purge_response\"\n\n// JSApiAccountPurgeResponse is the response to a purge request in the meta group.\ntype JSApiAccountPurgeResponse struct {\n\tApiResponse\n\tInitiated bool `json:\"initiated,omitempty\"`\n}\n\n// JSApiMsgGetRequest get a message request.\ntype JSApiMsgGetRequest struct {\n\tSeq uint64 `json:\"seq,omitempty\"`\n\tLastFor string `json:\"last_by_subj,omitempty\"`\n\tNextFor string `json:\"next_by_subj,omitempty\"`\n\n\t// Batch support. Used to request more than one msg at a time.\n\t// Can be used with simple starting seq, but also NextFor with wildcards.\n\tBatch int `json:\"batch,omitempty\"`\n\t// This will make sure we limit how much data we blast out. If not set we will\n\t// inherit the slow consumer default max setting of the server. Default is MAX_PENDING_SIZE.\n\tMaxBytes int `json:\"max_bytes,omitempty\"`\n\t// Return messages as of this start time.\n\tStartTime *time.Time `json:\"start_time,omitempty\"`\n\n\t// Multiple response support. Will get the last msgs matching the subjects. These can include wildcards.\n\tMultiLastFor []string `json:\"multi_last,omitempty\"`\n\t// Only return messages up to this sequence. If not set, will be last sequence for the stream.\n\tUpToSeq uint64 `json:\"up_to_seq,omitempty\"`\n\t// Only return messages up to this time.\n\tUpToTime *time.Time `json:\"up_to_time,omitempty\"`\n\t// Only return the message payload, excluding headers if present.\n\tNoHeaders bool `json:\"no_hdr,omitempty\"`\n}\n\ntype JSApiMsgGetResponse struct {\n\tApiResponse\n\tMessage *StoredMsg `json:\"message,omitempty\"`\n}\n\nconst JSApiMsgGetResponseType = \"io.nats.jetstream.api.v1.stream_msg_get_response\"\n\n// JSWaitQueueDefaultMax is the default max number of outstanding requests for pull consumers.\nconst JSWaitQueueDefaultMax = 512\n\ntype JSApiConsumerCreateResponse struct {\n\tApiResponse\n\t*ConsumerInfo\n}\n\nconst JSApiConsumerCreateResponseType = \"io.nats.jetstream.api.v1.consumer_create_response\"\n\ntype JSApiConsumerDeleteResponse struct {\n\tApiResponse\n\tSuccess bool `json:\"success,omitempty\"`\n}\n\nconst JSApiConsumerDeleteResponseType = \"io.nats.jetstream.api.v1.consumer_delete_response\"\n\ntype JSApiConsumerPauseRequest struct {\n\tPauseUntil time.Time `json:\"pause_until,omitempty\"`\n}\n\nconst JSApiConsumerPauseResponseType = \"io.nats.jetstream.api.v1.consumer_pause_response\"\n\ntype JSApiConsumerPauseResponse struct {\n\tApiResponse\n\tPaused bool `json:\"paused\"`\n\tPauseUntil time.Time `json:\"pause_until\"`\n\tPauseRemaining time.Duration `json:\"pause_remaining,omitempty\"`\n}\n\ntype JSApiConsumerInfoResponse struct {\n\tApiResponse\n\t*ConsumerInfo\n}\n\nconst JSApiConsumerInfoResponseType = \"io.nats.jetstream.api.v1.consumer_info_response\"\n\ntype JSApiConsumersRequest struct {\n\tApiPagedRequest\n}\n\ntype JSApiConsumerNamesResponse struct {\n\tApiResponse\n\tApiPaged\n\tConsumers []string `json:\"consumers\"`\n}\n\nconst JSApiConsumerNamesResponseType = \"io.nats.jetstream.api.v1.consumer_names_response\"\n\ntype JSApiConsumerListResponse struct {\n\tApiResponse\n\tApiPaged\n\tConsumers []*ConsumerInfo `json:\"consumers\"`\n\tMissing []string `json:\"missing,omitempty\"`\n\tOffline map[string]string `json:\"offline,omitempty\"`\n}\n\nconst JSApiConsumerListResponseType = \"io.nats.jetstream.api.v1.consumer_list_response\"\n\n// JSApiConsumerGetNextRequest is for getting next messages for pull based consumers.\ntype JSApiConsumerGetNextRequest struct {\n\tExpires time.Duration `json:\"expires,omitempty\"`\n\tBatch int `json:\"batch,omitempty\"`\n\tMaxBytes int `json:\"max_bytes,omitempty\"`\n\tNoWait bool `json:\"no_wait,omitempty\"`\n\tHeartbeat time.Duration `json:\"idle_heartbeat,omitempty\"`\n\tPriorityGroup\n}\n\n// JSApiConsumerResetRequest is for resetting a consumer to a specific sequence.\ntype JSApiConsumerResetRequest struct {\n\tSeq uint64 `json:\"seq,omitempty\"`\n}\n\ntype JSApiConsumerResetResponse struct {\n\tApiResponse\n\t*ConsumerInfo\n\tResetSeq uint64 `json:\"reset_seq\"`\n}\n\nconst JSApiConsumerResetResponseType = \"io.nats.jetstream.api.v1.consumer_reset_response\"\n\n// Structure that holds state for a JetStream API request that is processed\n// in a separate long-lived go routine. This is to avoid blocking connections.\ntype jsAPIRoutedReq struct {\n\tjsub *subscription\n\tsub *subscription\n\tacc *Account\n\tsubject string\n\treply string\n\tmsg []byte\n\tpa pubArg\n}\n\nfunc (js *jetStream) apiDispatch(sub *subscription, c *client, acc *Account, subject, reply string, rmsg []byte) {\n\t// Ignore system level directives meta stepdown and peer remove requests here.\n\tif subject == JSApiLeaderStepDown ||\n\t\tsubject == JSApiRemoveServer ||\n\t\tstrings.HasPrefix(subject, jsAPIAccountPre) {\n\t\treturn\n\t}\n\t// No lock needed, those are immutable.\n\ts, rr := js.srv, js.apiSubs.Match(subject)\n\n\thdr, msg := c.msgParts(rmsg)\n\tif len(sliceHeader(ClientInfoHdr, hdr)) == 0 {\n\t\t// Check if this is the system account. We will let these through for the account info only.\n\t\tsacc := s.SystemAccount()\n\t\tif sacc != acc {\n\t\t\treturn\n\t\t}\n\t\tif subject != JSApiAccountInfo {\n\t\t\t// Only respond from the initial server entry to the NATS system.\n\t\t\tif c.kind == CLIENT || c.kind == LEAF {\n\t\t\t\tvar resp = ApiResponse{\n\t\t\t\t\tType: JSApiSystemResponseType,\n\t\t\t\t\tError: NewJSNotEnabledForAccountError(),\n\t\t\t\t}\n\t\t\t\ts.sendAPIErrResponse(nil, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Short circuit for no interest.\n\tif len(rr.psubs)+len(rr.qsubs) == 0 {\n\t\tif (c.kind == CLIENT || c.kind == LEAF) && acc != s.SystemAccount() {\n\t\t\tci, acc, _, _, _ := s.getRequestInfo(c, rmsg)\n\t\t\tvar resp = ApiResponse{\n\t\t\t\tType: JSApiSystemResponseType,\n\t\t\t\tError: NewJSBadRequestError(),\n\t\t\t}\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\t// We should only have psubs and only 1 per result.\n\tif len(rr.psubs) != 1 {\n\t\ts.Warnf(\"Malformed JetStream API Request: [%s] %q\", subject, rmsg)\n\t\tif c.kind == CLIENT || c.kind == LEAF {\n\t\t\tci, acc, _, _, _ := s.getRequestInfo(c, rmsg)\n\t\t\tvar resp = ApiResponse{\n\t\t\t\tType: JSApiSystemResponseType,\n\t\t\t\tError: NewJSBadRequestError(),\n\t\t\t}\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tjsub := rr.psubs[0]\n\n\t// We need to make sure not to block. We will send the request to a long-lived\n\t// pool of go routines.\n\n\t// Increment inflight. Do this before queueing.\n\tatomic.AddInt64(&js.apiInflight, 1)\n\n\t// Copy the state. Note the JSAPI only uses the hdr index to piece apart the\n\t// header from the msg body. No other references are needed.\n\t// Check pending and warn if getting backed up.\n\tvar queue *ipQueue[*jsAPIRoutedReq]\n\tvar limit int64\n\tif js.infoSubs.HasInterest(subject) {\n\t\tqueue = s.jsAPIRoutedInfoReqs\n\t\tlimit = atomic.LoadInt64(&js.infoQueueLimit)\n\t} else {\n\t\tqueue = s.jsAPIRoutedReqs\n\t\tlimit = atomic.LoadInt64(&js.queueLimit)\n\t}\n\tpending, _ := queue.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa})\n\tif pending >= int(limit) {\n\t\ts.rateLimitFormatWarnf(\"%s limit reached, dropping %d requests\", queue.name, pending)\n\t\tdrained := int64(queue.drain())\n\t\tatomic.AddInt64(&js.apiInflight, -drained)\n\n\t\ts.publishAdvisory(nil, JSAdvisoryAPILimitReached, JSAPILimitReachedAdvisory{\n\t\t\tTypedEvent: TypedEvent{\n\t\t\t\tType: JSAPILimitReachedAdvisoryType,\n\t\t\t\tID: nuid.Next(),\n\t\t\t\tTime: time.Now().UTC(),\n\t\t\t},\n\t\t\tServer: s.Name(),\n\t\t\tDomain: js.config.Domain,\n\t\t\tDropped: drained,\n\t\t})\n\t}\n}\n\nfunc (s *Server) processJSAPIRoutedRequests() {\n\tdefer s.grWG.Done()\n\n\ts.mu.RLock()\n\tqueue, infoqueue := s.jsAPIRoutedReqs, s.jsAPIRoutedInfoReqs\n\tclient := &client{srv: s, kind: JETSTREAM}\n\ts.mu.RUnlock()\n\n\tjs := s.getJetStream()\n\n\tprocessFromQueue := func(ipq *ipQueue[*jsAPIRoutedReq]) {\n\t\t// Only pop one item at a time here, otherwise if the system is recovering\n\t\t// from queue buildup, then one worker will pull off all the tasks and the\n\t\t// others will be starved of work.\n\t\tif r, ok := ipq.popOne(); ok && r != nil {\n\t\t\tclient.pa = r.pa\n\t\t\tstart := time.Now()\n\t\t\tr.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg)\n\t\t\tif dur := time.Since(start); dur >= readLoopReportThreshold {\n\t\t\t\ts.Warnf(\"Internal subscription on %q took too long: %v\", r.subject, dur)\n\t\t\t}\n\t\t\tatomic.AddInt64(&js.apiInflight, -1)\n\t\t}\n\t}\n\n\tfor {\n\t\t// First select case is prioritizing queue, we will only fall through\n\t\t// to the second select case that considers infoqueue if queue is empty.\n\t\t// This effectively means infos are deprioritized.\n\t\tselect {\n\t\tcase <-queue.ch:\n\t\t\tprocessFromQueue(queue)\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tdefault:\n\t\t\tselect {\n\t\t\tcase <-infoqueue.ch:\n\t\t\t\tprocessFromQueue(infoqueue)\n\t\t\tcase <-queue.ch:\n\t\t\t\tprocessFromQueue(queue)\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (s *Server) setJetStreamExportSubs() error {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\n\t// Start the go routine that will process API requests received by the\n\t// subscription below when they are coming from routes, etc..\n\tconst maxProcs = 16\n\tmp := runtime.GOMAXPROCS(0)\n\t// Cap at 16 max for now on larger core setups.\n\tif mp > maxProcs {\n\t\tmp = maxProcs\n\t}\n\ts.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, \"JetStream API queue\")\n\ts.jsAPIRoutedInfoReqs = newIPQueue[*jsAPIRoutedReq](s, \"JetStream API info queue\")\n\tfor i := 0; i < mp; i++ {\n\t\ts.startGoRoutine(s.processJSAPIRoutedRequests)\n\t}\n\n\t// This is the catch all now for all JetStream API calls.\n\tif _, err := s.sysSubscribe(jsAllAPI, js.apiDispatch); err != nil {\n\t\treturn err\n\t}\n\n\tif err := s.SystemAccount().AddServiceExport(jsAllAPI, nil); err != nil {\n\t\ts.Warnf(\"Error setting up jetstream service exports: %v\", err)\n\t\treturn err\n\t}\n\n\t// API handles themselves.\n\t// infopairs are deprioritized compared to pairs in processJSAPIRoutedRequests.\n\tpairs := []struct {\n\t\tsubject string\n\t\thandler msgHandler\n\t}{\n\t\t{JSApiStreamCreate, s.jsStreamCreateRequest},\n\t\t{JSApiStreamUpdate, s.jsStreamUpdateRequest},\n\t\t{JSApiStreamDelete, s.jsStreamDeleteRequest},\n\t\t{JSApiStreamPurge, s.jsStreamPurgeRequest},\n\t\t{JSApiStreamSnapshot, s.jsStreamSnapshotRequest},\n\t\t{JSApiStreamRestore, s.jsStreamRestoreRequest},\n\t\t{JSApiStreamRemovePeer, s.jsStreamRemovePeerRequest},\n\t\t{JSApiStreamLeaderStepDown, s.jsStreamLeaderStepDownRequest},\n\t\t{JSApiConsumerLeaderStepDown, s.jsConsumerLeaderStepDownRequest},\n\t\t{JSApiMsgDelete, s.jsMsgDeleteRequest},\n\t\t{JSApiMsgGet, s.jsMsgGetRequest},\n\t\t{JSApiConsumerCreateEx, s.jsConsumerCreateRequest},\n\t\t{JSApiConsumerCreate, s.jsConsumerCreateRequest},\n\t\t{JSApiDurableCreate, s.jsConsumerCreateRequest},\n\t\t{JSApiConsumerDelete, s.jsConsumerDeleteRequest},\n\t\t{JSApiConsumerPause, s.jsConsumerPauseRequest},\n\t\t{JSApiConsumerUnpin, s.jsConsumerUnpinRequest},\n\t}\n\tinfopairs := []struct {\n\t\tsubject string\n\t\thandler msgHandler\n\t}{\n\t\t{JSApiAccountInfo, s.jsAccountInfoRequest},\n\t\t{JSApiStreams, s.jsStreamNamesRequest},\n\t\t{JSApiStreamList, s.jsStreamListRequest},\n\t\t{JSApiStreamInfo, s.jsStreamInfoRequest},\n\t\t{JSApiConsumers, s.jsConsumerNamesRequest},\n\t\t{JSApiConsumerList, s.jsConsumerListRequest},\n\t\t{JSApiConsumerInfo, s.jsConsumerInfoRequest},\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\t// As well as populating js.apiSubs for the dispatch function to use, we\n\t// will also populate js.infoSubs, so that the dispatch function can\n\t// decide quickly whether or not the request is an info request or not.\n\tfor _, p := range append(infopairs, pairs...) {\n\t\tsub := &subscription{subject: []byte(p.subject), icb: p.handler}\n\t\tif err := js.apiSubs.Insert(sub); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor _, p := range infopairs {\n\t\tif err := js.infoSubs.Insert(p.subject, struct{}{}); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (s *Server) sendAPIResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string) {\n\tacc.trackAPI()\n\tif reply != _EMPTY_ {\n\t\ts.sendInternalAccountMsg(nil, reply, response)\n\t}\n\ts.sendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response)\n}\n\nfunc (s *Server) sendAPIErrResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string) {\n\tacc.trackAPIErr()\n\tif reply != _EMPTY_ {\n\t\ts.sendInternalAccountMsg(nil, reply, response)\n\t}\n\ts.sendJetStreamAPIAuditAdvisory(ci, acc, subject, request, response)\n}\n\nconst errRespDelay = 500 * time.Millisecond\n\ntype delayedAPIResponse struct {\n\tci *ClientInfo\n\tacc *Account\n\tsubject string\n\treply string\n\trequest string\n\thdr []byte\n\tresponse string\n\trg *raftGroup\n\tdeadline time.Time\n\tnoJs bool\n\tnext *delayedAPIResponse\n}\n\n// Add `r` in the list that is maintained ordered by the `delayedAPIResponse.deadline` time.\nfunc addDelayedResponse(head, tail **delayedAPIResponse, r *delayedAPIResponse) {\n\t// Check if list empty.\n\tif *head == nil {\n\t\t*head, *tail = r, r\n\t\treturn\n\t}\n\t// Check if it should be added at the end, which is if after or equal to the tail.\n\tif r.deadline.After((*tail).deadline) || r.deadline.Equal((*tail).deadline) {\n\t\t(*tail).next, *tail = r, r\n\t\treturn\n\t}\n\t// Find its spot in the list.\n\tvar prev *delayedAPIResponse\n\tfor c := *head; c != nil; c = c.next {\n\t\t// We insert only if we are stricly before the current `c`.\n\t\tif r.deadline.Before(c.deadline) {\n\t\t\tr.next = c\n\t\t\tif prev != nil {\n\t\t\t\tprev.next = r\n\t\t\t} else {\n\t\t\t\t*head = r\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tprev = c\n\t}\n}\n\nfunc (s *Server) delayedAPIResponder() {\n\tdefer s.grWG.Done()\n\tvar (\n\t\thead, tail *delayedAPIResponse // Linked list.\n\t\tr *delayedAPIResponse // Updated by calling next().\n\t\trqch <-chan struct{} // Quit channel of the Raft group (if present).\n\t\ttm = time.NewTimer(time.Hour)\n\t)\n\tnext := func() {\n\t\tr, rqch = nil, nil\n\t\t// Check that JetStream is still on. Do not exit the go routine\n\t\t// since JS can be enabled/disabled. The go routine will exit\n\t\t// only if server is shutdown.\n\t\tjs := s.getJetStream()\n\t\tif js == nil {\n\t\t\t// Reset head and tail here. Also drain the ipQueue.\n\t\t\thead, tail = nil, nil\n\t\t\ts.delayedAPIResponses.drain()\n\t\t\t// Fall back into next \"if\" that resets timer.\n\t\t}\n\t\t// If there are no delayed messages then delay the timer for\n\t\t// a while.\n\t\tif head == nil {\n\t\t\ttm.Reset(time.Hour)\n\t\t\treturn\n\t\t}\n\t\t// Get the first expected message and then reset the timer.\n\t\tr = head\n\t\tjs.mu.RLock()\n\t\tif r.rg != nil && r.rg.node != nil {\n\t\t\t// If there's an attached Raft group to the delayed response\n\t\t\t// then pull out the quit channel, so that we don't bother\n\t\t\t// sending responses for entities which are now no longer\n\t\t\t// running.\n\t\t\trqch = r.rg.node.QuitC()\n\t\t}\n\t\tjs.mu.RUnlock()\n\t\ttm.Reset(time.Until(r.deadline))\n\t}\n\tpop := func() {\n\t\tif head == nil {\n\t\t\treturn\n\t\t}\n\t\thead = head.next\n\t\tif head == nil {\n\t\t\ttail = nil\n\t\t}\n\t}\n\tfor {\n\t\tselect {\n\t\tcase <-s.delayedAPIResponses.ch:\n\t\t\tv, ok := s.delayedAPIResponses.popOne()\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Add it to the list, and if ends up being the head, set things up.\n\t\t\taddDelayedResponse(&head, &tail, v)\n\t\t\tif v == head {\n\t\t\t\tnext()\n\t\t\t}\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-rqch:\n\t\t\t// If we were the head, drop and setup things for next.\n\t\t\tif r != nil && r == head {\n\t\t\t\tpop()\n\t\t\t}\n\t\t\tnext()\n\t\tcase <-tm.C:\n\t\t\tif r != nil {\n\t\t\t\t// If it's not a JS API error, send it as a raw response without additional API/audit tracking.\n\t\t\t\tif r.noJs {\n\t\t\t\t\ts.sendInternalAccountMsgWithReply(r.acc, r.subject, _EMPTY_, r.hdr, r.response, false)\n\t\t\t\t} else {\n\t\t\t\t\ts.sendAPIErrResponse(r.ci, r.acc, r.subject, r.reply, r.request, r.response)\n\t\t\t\t}\n\t\t\t\tpop()\n\t\t\t}\n\t\t\tnext()\n\t\t}\n\t}\n}\n\nfunc (s *Server) sendDelayedAPIErrResponse(ci *ClientInfo, acc *Account, subject, reply, request, response string, rg *raftGroup, duration time.Duration) {\n\ts.delayedAPIResponses.push(&delayedAPIResponse{\n\t\tci, acc, subject, reply, request, nil, response, rg, time.Now().Add(duration), false, nil,\n\t})\n}\n\nfunc (s *Server) sendDelayedErrResponse(acc *Account, subject string, hdr []byte, response string, duration time.Duration) {\n\ts.delayedAPIResponses.push(&delayedAPIResponse{\n\t\tnil, acc, subject, _EMPTY_, _EMPTY_, hdr, response, nil, time.Now().Add(duration), true, nil,\n\t})\n}\n\nfunc (s *Server) getRequestInfo(c *client, raw []byte) (pci *ClientInfo, acc *Account, hdr, msg []byte, err error) {\n\thdr, msg = c.msgParts(raw)\n\tvar ci ClientInfo\n\n\tif len(hdr) > 0 {\n\t\tif err := json.Unmarshal(sliceHeader(ClientInfoHdr, hdr), &ci); err != nil {\n\t\t\treturn nil, nil, nil, nil, err\n\t\t}\n\t}\n\n\tif ci.Service != _EMPTY_ {\n\t\tacc, _ = s.LookupAccount(ci.Service)\n\t} else if ci.Account != _EMPTY_ {\n\t\tacc, _ = s.LookupAccount(ci.Account)\n\t} else {\n\t\t// Direct $SYS access.\n\t\tacc = c.acc\n\t\tif acc == nil {\n\t\t\tacc = s.SystemAccount()\n\t\t}\n\t}\n\tif acc == nil {\n\t\treturn nil, nil, nil, nil, ErrMissingAccount\n\t}\n\treturn &ci, acc, hdr, msg, nil\n}\n\nfunc (s *Server) unmarshalRequest(c *client, acc *Account, subject string, msg []byte, v any) error {\n\tdecoder := json.NewDecoder(bytes.NewReader(msg))\n\tdecoder.DisallowUnknownFields()\n\n\tfor {\n\t\tif err := decoder.Decode(v); err != nil {\n\t\t\tif err == io.EOF {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tvar syntaxErr *json.SyntaxError\n\t\t\tif errors.As(err, &syntaxErr) {\n\t\t\t\terr = fmt.Errorf(\"%w at offset %d\", err, syntaxErr.Offset)\n\t\t\t}\n\n\t\t\tc.RateLimitWarnf(\"Invalid JetStream request '%s > %s': %s\", acc, subject, err)\n\n\t\t\tif s.JetStreamConfig().Strict {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\treturn json.Unmarshal(msg, v)\n\t\t}\n\t}\n}\n\nfunc (a *Account) trackAPI() {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\tif jsa != nil {\n\t\tjsa.usageMu.Lock()\n\t\tjsa.usageApi++\n\t\tjsa.apiTotal++\n\t\tjsa.sendClusterUsageUpdate()\n\t\tatomic.AddInt64(&jsa.js.apiTotal, 1)\n\t\tjsa.usageMu.Unlock()\n\t}\n}\n\nfunc (a *Account) trackAPIErr() {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\tif jsa != nil {\n\t\tjsa.usageMu.Lock()\n\t\tjsa.usageApi++\n\t\tjsa.apiTotal++\n\t\tjsa.usageErr++\n\t\tjsa.apiErrors++\n\t\tjsa.sendClusterUsageUpdate()\n\t\tatomic.AddInt64(&jsa.js.apiTotal, 1)\n\t\tatomic.AddInt64(&jsa.js.apiErrors, 1)\n\t\tjsa.usageMu.Unlock()\n\t}\n}\n\nconst badAPIRequestT = \"Malformed JetStream API Request: %q\"\n\n// Helper function to check on JetStream being enabled but also on status of leafnodes\n// If the local account is not enabled but does have leafnode connectivity we will not\n// want to error immediately and let the other side decide.\nfunc (a *Account) checkJetStream() (enabled, shouldError bool) {\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\treturn a.js != nil, a.nleafs+a.nrleafs == 0\n}\n\n// Request for current usage and limits for this account.\nfunc (s *Server) jsAccountInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiAccountInfoResponse{ApiResponse: ApiResponse{Type: JSApiAccountInfoResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif !doErr {\n\t\t\treturn\n\t\t}\n\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t} else {\n\t\tstats := acc.JetStreamUsage()\n\t\tresp.JetStreamAccountStats = &stats\n\t}\n\tb, err := json.Marshal(resp)\n\tif err != nil {\n\t\treturn\n\t}\n\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), string(b))\n}\n\n// Helpers for token extraction.\nfunc streamNameFromSubject(subject string) string {\n\treturn tokenAt(subject, 5)\n}\n\nfunc consumerNameFromSubject(subject string) string {\n\treturn tokenAt(subject, 6)\n}\n\nfunc (s *Server) jsonResponse(v any) string {\n\tb, err := json.Marshal(v)\n\tif err != nil {\n\t\ts.Warnf(\"Problem marshaling JSON for JetStream API:\", err)\n\t\treturn \"\"\n\t}\n\treturn string(b)\n}\n\n// Read lock must be held\nfunc (jsa *jsAccount) tieredReservation(tier string, cfg *StreamConfig) int64 {\n\tvar reservation int64\n\tfor _, sa := range jsa.streams {\n\t\t// Don't count the stream toward the limit if it already exists.\n\t\tif sa.cfg.Name == cfg.Name {\n\t\t\tcontinue\n\t\t}\n\t\tif (tier == _EMPTY_ || isSameTier(&sa.cfg, cfg)) && sa.cfg.MaxBytes > 0 && sa.cfg.Storage == cfg.Storage {\n\t\t\t// If tier is empty, all storage is flat and we should adjust for replicas.\n\t\t\t// Otherwise if tiered, storage replication already taken into consideration.\n\t\t\tif tier == _EMPTY_ && sa.cfg.Replicas > 1 {\n\t\t\t\treservation += (int64(sa.cfg.Replicas) * sa.cfg.MaxBytes)\n\t\t\t} else {\n\t\t\t\treservation += sa.cfg.MaxBytes\n\t\t\t}\n\t\t}\n\t}\n\treturn reservation\n}\n\n// Request to create a stream.\nfunc (s *Server) jsStreamCreateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar cfg StreamConfigRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &cfg); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Initialize asset version metadata.\n\tsetStaticStreamMetadata(&cfg.StreamConfig)\n\n\tstreamName := streamNameFromSubject(subject)\n\tif streamName != cfg.Name {\n\t\tresp.Error = NewJSStreamMismatchError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Check for path like separators in the name.\n\tif strings.ContainsAny(streamName, `\\/`) {\n\t\tresp.Error = NewJSStreamNameContainsPathSeparatorsError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Can't create a stream with a sealed state.\n\tif cfg.Sealed {\n\t\tresp.Error = NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for create can not be sealed\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are told to do mirror direct but are not mirroring, error.\n\tif cfg.MirrorDirect && cfg.Mirror == nil {\n\t\tresp.Error = NewJSStreamInvalidConfigError(fmt.Errorf(\"stream has no mirror but does have mirror direct\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Hand off to cluster for processing.\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredStreamRequest(ci, acc, subject, reply, rmsg, &cfg)\n\t\treturn\n\t}\n\n\tif err := acc.jsNonClusteredStreamLimitsCheck(&cfg.StreamConfig); err != nil {\n\t\tresp.Error = err\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.addStreamPedantic(&cfg.StreamConfig, cfg.Pedantic)\n\tif err != nil {\n\t\tif IsNatsErr(err, JSStreamStoreFailedF) {\n\t\t\ts.Warnf(\"Stream create failed for '%s > %s': %v\", acc, streamName, err)\n\t\t\terr = errStreamStoreFailed\n\t\t}\n\t\tresp.Error = NewJSStreamCreateError(err, Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tmsetCfg := mset.config()\n\tresp.StreamInfo = &StreamInfo{\n\t\tCreated: mset.createdTime(),\n\t\tState: mset.state(),\n\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\tTimeStamp: time.Now().UTC(),\n\t\tMirror: mset.mirrorInfo(),\n\t\tSources: mset.sourcesInfo(),\n\t}\n\tresp.DidCreate = true\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to update a stream.\nfunc (s *Server) jsStreamUpdateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tvar ncfg StreamConfigRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &ncfg); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tcfg, apiErr := s.checkStreamCfg(&ncfg.StreamConfig, acc, ncfg.Pedantic)\n\tif apiErr != nil {\n\t\tresp.Error = apiErr\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tstreamName := streamNameFromSubject(subject)\n\tif streamName != cfg.Name {\n\t\tresp.Error = NewJSStreamMismatchError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Handle clustered version here.\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredStreamUpdateRequest(ci, acc, subject, reply, copyBytes(rmsg), &cfg, nil, ncfg.Pedantic)\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(streamName)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.offlineReason != _EMPTY_ {\n\t\tresp.Error = NewJSStreamOfflineReasonError(errors.New(mset.offlineReason))\n\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\treturn\n\t}\n\n\t// Update asset version metadata.\n\tsetStaticStreamMetadata(&cfg)\n\n\tif err := mset.updatePedantic(&cfg, ncfg.Pedantic); err != nil {\n\t\tresp.Error = NewJSStreamUpdateError(err, Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmsetCfg := mset.config()\n\tresp.StreamInfo = &StreamInfo{\n\t\tCreated: mset.createdTime(),\n\t\tState: mset.state(),\n\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t\tMirror: mset.mirrorInfo(),\n\t\tSources: mset.sourcesInfo(),\n\t\tTimeStamp: time.Now().UTC(),\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request for the list of all stream names.\nfunc (s *Server) jsStreamNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamNamesResponse{ApiResponse: ApiResponse{Type: JSApiStreamNamesResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar offset int\n\tvar filter string\n\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiStreamNamesRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\toffset = req.Offset\n\t\tif req.Subject != _EMPTY_ {\n\t\t\tfilter = req.Subject\n\t\t}\n\t}\n\n\t// TODO(dlc) - Maybe hold these results for large results that we expect to be paged.\n\t// TODO(dlc) - If this list is long maybe do this in a Go routine?\n\tvar numStreams int\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\t// TODO(dlc) - Debug or Warn?\n\t\t\treturn\n\t\t}\n\t\tjs.mu.RLock()\n\t\tfor stream, sa := range cc.streams[acc.Name] {\n\t\t\tif IsNatsErr(sa.err, JSClusterNotAssignedErr) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif filter != _EMPTY_ {\n\t\t\t\t// These could not have subjects auto-filled in since they are raw and unprocessed.\n\t\t\t\tif len(sa.Config.Subjects) == 0 {\n\t\t\t\t\tif SubjectsCollide(filter, sa.Config.Name) {\n\t\t\t\t\t\tresp.Streams = append(resp.Streams, stream)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tfor _, subj := range sa.Config.Subjects {\n\t\t\t\t\t\tif SubjectsCollide(filter, subj) {\n\t\t\t\t\t\t\tresp.Streams = append(resp.Streams, stream)\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tresp.Streams = append(resp.Streams, stream)\n\t\t\t}\n\t\t}\n\t\tjs.mu.RUnlock()\n\t\tif len(resp.Streams) > 1 {\n\t\t\tslices.Sort(resp.Streams)\n\t\t}\n\t\tnumStreams = len(resp.Streams)\n\t\tif offset > numStreams {\n\t\t\toffset = numStreams\n\t\t}\n\t\tif offset > 0 {\n\t\t\tresp.Streams = resp.Streams[offset:]\n\t\t}\n\t\tif len(resp.Streams) > JSApiNamesLimit {\n\t\t\tresp.Streams = resp.Streams[:JSApiNamesLimit]\n\t\t}\n\t} else {\n\t\tmsets := acc.filteredStreams(filter)\n\t\t// Since we page results order matters.\n\t\tif len(msets) > 1 {\n\t\t\tslices.SortFunc(msets, func(i, j *stream) int { return cmp.Compare(i.cfg.Name, j.cfg.Name) })\n\t\t}\n\n\t\tnumStreams = len(msets)\n\t\tif offset > numStreams {\n\t\t\toffset = numStreams\n\t\t}\n\n\t\tfor _, mset := range msets[offset:] {\n\t\t\tresp.Streams = append(resp.Streams, mset.cfg.Name)\n\t\t\tif len(resp.Streams) >= JSApiNamesLimit {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tresp.Total = numStreams\n\tresp.Limit = JSApiNamesLimit\n\tresp.Offset = offset\n\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request for the list of all detailed stream info.\n// TODO(dlc) - combine with above long term\nfunc (s *Server) jsStreamListRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamListResponse{\n\t\tApiResponse: ApiResponse{Type: JSApiStreamListResponseType},\n\t\tStreams: []*StreamInfo{},\n\t}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar offset int\n\tvar filter string\n\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiStreamListRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\toffset = req.Offset\n\t\tif req.Subject != _EMPTY_ {\n\t\t\tfilter = req.Subject\n\t\t}\n\t}\n\n\t// Clustered mode will invoke a scatter and gather.\n\tif s.JetStreamIsClustered() {\n\t\t// Need to copy these off before sending.. don't move this inside startGoRoutine!!!\n\t\tmsg = copyBytes(msg)\n\t\ts.startGoRoutine(func() { s.jsClusteredStreamListRequest(acc, ci, filter, offset, subject, reply, msg) })\n\t\treturn\n\t}\n\n\t// TODO(dlc) - Maybe hold these results for large results that we expect to be paged.\n\t// TODO(dlc) - If this list is long maybe do this in a Go routine?\n\tvar msets []*stream\n\tif filter == _EMPTY_ {\n\t\tmsets = acc.streams()\n\t} else {\n\t\tmsets = acc.filteredStreams(filter)\n\t}\n\n\tslices.SortFunc(msets, func(i, j *stream) int { return cmp.Compare(i.cfg.Name, j.cfg.Name) })\n\n\tscnt := len(msets)\n\tif offset > scnt {\n\t\toffset = scnt\n\t}\n\n\tvar missingNames []string\n\tfor _, mset := range msets[offset:] {\n\t\tif mset.offlineReason != _EMPTY_ {\n\t\t\tif resp.Offline == nil {\n\t\t\t\tresp.Offline = make(map[string]string, 1)\n\t\t\t}\n\t\t\tresp.Offline[mset.getCfgName()] = mset.offlineReason\n\t\t\tmissingNames = append(missingNames, mset.getCfgName())\n\t\t\tcontinue\n\t\t}\n\n\t\tconfig := mset.config()\n\t\tresp.Streams = append(resp.Streams, &StreamInfo{\n\t\t\tCreated: mset.createdTime(),\n\t\t\tState: mset.state(),\n\t\t\tConfig: config,\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t\tMirror: mset.mirrorInfo(),\n\t\t\tSources: mset.sourcesInfo(),\n\t\t\tTimeStamp: time.Now().UTC(),\n\t\t})\n\t\tif len(resp.Streams) >= JSApiListLimit {\n\t\t\tbreak\n\t\t}\n\t}\n\tresp.Total = scnt\n\tresp.Limit = JSApiListLimit\n\tresp.Offset = offset\n\tresp.Missing = missingNames\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request for information about a stream.\nfunc (s *Server) jsStreamInfoRequest(sub *subscription, c *client, a *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstreamName := streamNameFromSubject(subject)\n\n\tvar resp = JSApiStreamInfoResponse{ApiResponse: ApiResponse{Type: JSApiStreamInfoResponseType}}\n\n\t// If someone creates a duplicate stream that is identical we will get this request forwarded to us.\n\t// Make sure the response type is for a create call.\n\tif rt := getHeader(JSResponseType, hdr); len(rt) > 0 && string(rt) == jsCreateResponse {\n\t\tresp.ApiResponse.Type = JSApiStreamCreateResponseType\n\t}\n\n\tvar clusterWideConsCount int\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil {\n\t\treturn\n\t}\n\t// If we are in clustered mode we need to be the stream leader to proceed.\n\tif cc != nil {\n\t\t// Check to make sure the stream is assigned.\n\t\tjs.mu.RLock()\n\t\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, streamName)\n\t\tvar offline bool\n\t\tif sa != nil {\n\t\t\tclusterWideConsCount = len(sa.consumers)\n\t\t\toffline = s.allPeersOffline(sa.Group)\n\t\t\tif sa.unsupported != nil && sa.Group != nil && cc.meta != nil && sa.Group.isMember(cc.meta.ID()) {\n\t\t\t\t// If we're a member for this stream, and it's not supported, report it as offline.\n\t\t\t\tresp.Error = NewJSStreamOfflineReasonError(errors.New(sa.unsupported.reason))\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t\tif isLeader && sa == nil {\n\t\t\t// We can't find the stream, so mimic what would be the errors below.\n\t\t\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\t\t\tif doErr {\n\t\t\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No stream present.\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else if sa == nil {\n\t\t\tif js.isLeaderless() {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\t// Delaying an error response gives the leader a chance to respond before us\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t}\n\t\t\treturn\n\t\t} else if isLeader && offline {\n\t\t\tresp.Error = NewJSStreamOfflineError()\n\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\treturn\n\t\t}\n\n\t\t// Check to see if we are a member of the group and if the group has no leader.\n\t\tisLeaderless := js.isGroupLeaderless(sa.Group)\n\n\t\t// We have the stream assigned and a leader, so only the stream leader should answer.\n\t\tif !acc.JetStreamIsStreamLeader(streamName) && !isLeaderless {\n\t\t\tif js.isLeaderless() {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\t// Delaying an error response gives the leader a chance to respond before us\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), sa.Group, errRespDelay)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// We may be in process of electing a leader, but if this is a scale up from 1 we will still be the state leader\n\t\t\t// while the new members work through the election and catchup process.\n\t\t\t// Double check for that instead of exiting here and being silent. e.g. nats stream update test --replicas=3\n\t\t\tjs.mu.RLock()\n\t\t\trg := sa.Group\n\t\t\tvar ourID string\n\t\t\tif cc.meta != nil {\n\t\t\t\tourID = cc.meta.ID()\n\t\t\t}\n\t\t\t// We have seen cases where rg is nil at this point,\n\t\t\t// so check explicitly and bail if that is the case.\n\t\t\tbail := rg == nil || !rg.isMember(ourID)\n\t\t\tif !bail {\n\t\t\t\t// We know we are a member here, if this group is new and we are preferred allow us to answer.\n\t\t\t\t// Also, we have seen cases where rg.node is nil at this point,\n\t\t\t\t// so check explicitly and bail if that is the case.\n\t\t\t\tbail = rg.Preferred != ourID || (rg.node != nil && time.Since(rg.node.Created()) > lostQuorumIntervalDefault)\n\t\t\t}\n\t\t\tjs.mu.RUnlock()\n\t\t\tif bail {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar details bool\n\tvar subjects string\n\tvar offset int\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiStreamInfoRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tdetails, subjects = req.DeletedDetails, req.SubjectsFilter\n\t\toffset = req.Offset\n\t}\n\n\tmset, err := acc.lookupStream(streamName)\n\t// Error is not to be expected at this point, but could happen if same stream trying to be created.\n\tif err != nil {\n\t\tif cc != nil {\n\t\t\t// This could be inflight, pause for a short bit and try again.\n\t\t\t// This will not be inline, so ok.\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tmset, err = acc.lookupStream(streamName)\n\t\t}\n\t\t// Check again.\n\t\tif err != nil {\n\t\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\tif mset.offlineReason != _EMPTY_ {\n\t\tresp.Error = NewJSStreamOfflineReasonError(errors.New(mset.offlineReason))\n\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\treturn\n\t}\n\n\tconfig := mset.config()\n\tresp.StreamInfo = &StreamInfo{\n\t\tCreated: mset.createdTime(),\n\t\tState: mset.stateWithDetail(details),\n\t\tConfig: *setDynamicStreamMetadata(&config),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t\tCluster: js.clusterInfo(mset.raftGroup()),\n\t\tMirror: mset.mirrorInfo(),\n\t\tSources: mset.sourcesInfo(),\n\t\tAlternates: js.streamAlternates(ci, config.Name),\n\t\tTimeStamp: time.Now().UTC(),\n\t}\n\tif clusterWideConsCount > 0 {\n\t\tresp.StreamInfo.State.Consumers = clusterWideConsCount\n\t}\n\n\t// Check if they have asked for subject details.\n\tif subjects != _EMPTY_ {\n\t\tst := mset.store.SubjectsTotals(subjects)\n\t\tif lst := len(st); lst > 0 {\n\t\t\t// Common for both cases.\n\t\t\tresp.Offset = offset\n\t\t\tresp.Limit = JSMaxSubjectDetails\n\t\t\tresp.Total = lst\n\n\t\t\tif offset == 0 && lst <= JSMaxSubjectDetails {\n\t\t\t\tresp.StreamInfo.State.Subjects = st\n\t\t\t} else {\n\t\t\t\t// Here we have to filter list due to offset or maximum constraints.\n\t\t\t\tsubjs := make([]string, 0, len(st))\n\t\t\t\tfor subj := range st {\n\t\t\t\t\tsubjs = append(subjs, subj)\n\t\t\t\t}\n\t\t\t\t// Sort it\n\t\t\t\tslices.Sort(subjs)\n\n\t\t\t\tif offset > len(subjs) {\n\t\t\t\t\toffset = len(subjs)\n\t\t\t\t}\n\n\t\t\t\tend := offset + JSMaxSubjectDetails\n\t\t\t\tif end > len(subjs) {\n\t\t\t\t\tend = len(subjs)\n\t\t\t\t}\n\t\t\t\tactualSize := end - offset\n\t\t\t\tvar sd map[string]uint64\n\n\t\t\t\tif actualSize > 0 {\n\t\t\t\t\tsd = make(map[string]uint64, actualSize)\n\t\t\t\t\tfor _, ss := range subjs[offset:end] {\n\t\t\t\t\t\tsd[ss] = st[ss]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tresp.StreamInfo.State.Subjects = sd\n\t\t\t}\n\t\t}\n\t}\n\t// Check for out of band catchups.\n\tif mset.hasCatchupPeers() {\n\t\tmset.checkClusterInfo(resp.StreamInfo.Cluster)\n\t}\n\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to have a stream leader stepdown.\nfunc (s *Server) jsStreamLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\t// Have extra token for this one.\n\tname := tokenAt(subject, 6)\n\n\tvar resp = JSApiStreamLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiStreamLeaderStepDownResponseType}}\n\n\t// If we are not in clustered mode this is a failed request.\n\tif !s.JetStreamIsClustered() {\n\t\tresp.Error = NewJSClusterRequiredError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are here we are clustered. See if we are the stream leader in order to proceed.\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\tif js.isLeaderless() {\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, name)\n\tjs.mu.RUnlock()\n\n\tif isLeader && sa == nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t} else if sa == nil {\n\t\treturn\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\t// Check to see if we are a member of the group and if the group has no leader.\n\tif js.isGroupLeaderless(sa.Group) {\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// We have the stream assigned and a leader, so only the stream leader should answer.\n\tif !acc.JetStreamIsStreamLeader(name) {\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(name)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif mset == nil {\n\t\tresp.Success = true\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\tnode := mset.raftNode()\n\tif node == nil {\n\t\tresp.Success = true\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\tvar preferredLeader string\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiLeaderStepdownRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif preferredLeader, resp.Error = s.getStepDownPreferredPlacement(node, req.Placement); resp.Error != nil {\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Call actual stepdown.\n\terr = node.StepDown(preferredLeader)\n\tif err != nil {\n\t\tresp.Error = NewJSRaftGeneralError(err, Unless(err))\n\t} else {\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to have a consumer leader stepdown.\nfunc (s *Server) jsConsumerLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiConsumerLeaderStepDownResponseType}}\n\n\t// If we are not in clustered mode this is a failed request.\n\tif !s.JetStreamIsClustered() {\n\t\tresp.Error = NewJSClusterRequiredError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are here we are clustered. See if we are the stream leader in order to proceed.\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\tif js.isLeaderless() {\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Have extra token for this one.\n\tstream := tokenAt(subject, 6)\n\tconsumer := tokenAt(subject, 7)\n\n\tjs.mu.RLock()\n\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)\n\tjs.mu.RUnlock()\n\n\tif isLeader && sa == nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t} else if sa == nil {\n\t\treturn\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar ca *consumerAssignment\n\tif sa.consumers != nil {\n\t\tca = sa.consumers[consumer]\n\t}\n\tif ca == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\t// Check to see if we are a member of the group and if the group has no leader.\n\tif js.isGroupLeaderless(ca.Group) {\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif !acc.JetStreamIsConsumerLeader(stream, consumer) {\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\to := mset.lookupConsumer(consumer)\n\tif o == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tn := o.raftNode()\n\tif n == nil {\n\t\tresp.Success = true\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\tvar preferredLeader string\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiLeaderStepdownRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif preferredLeader, resp.Error = s.getStepDownPreferredPlacement(n, req.Placement); resp.Error != nil {\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Call actual stepdown.\n\terr = n.StepDown(preferredLeader)\n\tif err != nil {\n\t\tresp.Error = NewJSRaftGeneralError(err, Unless(err))\n\t} else {\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to remove a peer from a clustered stream.\nfunc (s *Server) jsStreamRemovePeerRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\t// Have extra token for this one.\n\tname := tokenAt(subject, 6)\n\n\tvar resp = JSApiStreamRemovePeerResponse{ApiResponse: ApiResponse{Type: JSApiStreamRemovePeerResponseType}}\n\n\t// If we are not in clustered mode this is a failed request.\n\tif !s.JetStreamIsClustered() {\n\t\tresp.Error = NewJSClusterRequiredError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are here we are clustered. See if we are the stream leader in order to proceed.\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\tif js.isLeaderless() {\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tisLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, name)\n\tjs.mu.RUnlock()\n\n\t// Make sure we are meta leader.\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar req JSApiStreamRemovePeerRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif req.Peer == _EMPTY_ {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif sa == nil {\n\t\t// No stream present.\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\trg := sa.Group\n\n\t// Check to see if we are a member of the group.\n\t// Peer here is either a peer ID or a server name, convert to node name.\n\tnodeName := getHash(req.Peer)\n\tisMember := rg.isMember(nodeName)\n\tif !isMember {\n\t\tnodeName = req.Peer\n\t\tisMember = rg.isMember(nodeName)\n\t}\n\tjs.mu.RUnlock()\n\n\t// Make sure we are a member.\n\tif !isMember {\n\t\tresp.Error = NewJSClusterPeerNotMemberError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are here we have a valid peer member set for removal.\n\tif !js.removePeerFromStream(sa, nodeName) {\n\t\tresp.Error = NewJSPeerRemapError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tresp.Success = true\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to have the metaleader remove a peer from the system.\nfunc (s *Server) jsLeaderServerRemoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\tif acc != s.SystemAccount() {\n\t\treturn\n\t}\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tisLeader := cc.isLeader()\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\t// Extra checks here but only leader is listening.\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tvar resp = JSApiMetaServerRemoveResponse{ApiResponse: ApiResponse{Type: JSApiMetaServerRemoveResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar req JSApiMetaServerRemoveRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\t// Another peer-remove is already in progress, don't allow multiple concurrent changes.\n\tif cc.peerRemoveReply != nil {\n\t\tresp.Error = NewJSClusterServerMemberChangeInflightError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar found string\n\tfor _, p := range meta.Peers() {\n\t\t// If Peer is specified, it takes precedence\n\t\tif req.Peer != _EMPTY_ {\n\t\t\tif p.ID == req.Peer {\n\t\t\t\tfound = req.Peer\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\tif ok && si.(nodeInfo).name == req.Server {\n\t\t\tfound = p.ID\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif found == _EMPTY_ {\n\t\tresp.Error = NewJSClusterServerNotMemberError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif err := meta.ProposeRemovePeer(found); err != nil {\n\t\tif err == errMembershipChange {\n\t\t\tresp.Error = NewJSClusterServerMemberChangeInflightError()\n\t\t} else {\n\t\t\tresp.Error = NewJSRaftGeneralError(err)\n\t\t}\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif cc.peerRemoveReply == nil {\n\t\tcc.peerRemoveReply = make(map[string]peerRemoveInfo, 1)\n\t}\n\t// Only copy the request, the subject and reply are already copied.\n\tcc.peerRemoveReply[found] = peerRemoveInfo{ci: ci, subject: subject, reply: reply, request: string(msg)}\n}\n\nfunc (s *Server) peerSetToNames(ps []string) []string {\n\tnames := make([]string, len(ps))\n\tfor i := 0; i < len(ps); i++ {\n\t\tif si, ok := s.nodeToInfo.Load(ps[i]); !ok {\n\t\t\tnames[i] = ps[i]\n\t\t} else {\n\t\t\tnames[i] = si.(nodeInfo).name\n\t\t}\n\t}\n\treturn names\n}\n\n// looks up the peer id for a given server name. Cluster and domain name are optional filter criteria\nfunc (s *Server) nameToPeer(js *jetStream, serverName, clusterName, domainName string) string {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif cc := js.cluster; cc != nil {\n\t\tfor _, p := range cc.meta.Peers() {\n\t\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\t\tif ok && si.(nodeInfo).name == serverName {\n\t\t\t\tif clusterName == _EMPTY_ || clusterName == si.(nodeInfo).cluster {\n\t\t\t\t\tif domainName == _EMPTY_ || domainName == si.(nodeInfo).domain {\n\t\t\t\t\t\treturn p.ID\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn _EMPTY_\n}\n\n// Request to have the metaleader move a stream on a peer to another\nfunc (s *Server) jsLeaderServerStreamMoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\t// Extra checks here but only leader is listening.\n\tjs.mu.RLock()\n\tisLeader := cc.isLeader()\n\tjs.mu.RUnlock()\n\n\tif !isLeader {\n\t\treturn\n\t}\n\n\taccName := tokenAt(subject, 6)\n\tstreamName := tokenAt(subject, 7)\n\n\tif acc.GetName() != accName && acc != s.SystemAccount() {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar req JSApiMetaServerStreamMoveRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tsrcPeer := _EMPTY_\n\tif req.Server != _EMPTY_ {\n\t\tsrcPeer = s.nameToPeer(js, req.Server, req.Cluster, req.Domain)\n\t}\n\n\ttargetAcc, ok := s.accounts.Load(accName)\n\tif !ok {\n\t\tresp.Error = NewJSNoAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar streamFound bool\n\tcfg := StreamConfig{}\n\tcurrPeers := []string{}\n\tcurrCluster := _EMPTY_\n\tjs.mu.Lock()\n\tstreams, ok := cc.streams[accName]\n\tif ok {\n\t\tsa, ok := streams[streamName]\n\t\tif ok {\n\t\t\tcfg = *sa.Config.clone()\n\t\t\tstreamFound = true\n\t\t\tcurrPeers = sa.Group.Peers\n\t\t\tcurrCluster = sa.Group.Cluster\n\t\t}\n\t}\n\tjs.mu.Unlock()\n\n\tif !streamFound {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// if server was picked, make sure src peer exists and move it to first position.\n\t// removal will drop peers from the left\n\tif req.Server != _EMPTY_ {\n\t\tif srcPeer == _EMPTY_ {\n\t\t\tresp.Error = NewJSClusterServerNotMemberError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tvar peerFound bool\n\t\tfor i := 0; i < len(currPeers); i++ {\n\t\t\tif currPeers[i] == srcPeer {\n\t\t\t\tcopy(currPeers[1:], currPeers[:i])\n\t\t\t\tcurrPeers[0] = srcPeer\n\t\t\t\tpeerFound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !peerFound {\n\t\t\tresp.Error = NewJSClusterPeerNotMemberError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// make sure client is scoped to requested account\n\tciNew := *(ci)\n\tciNew.Account = accName\n\n\t// backup placement such that peers can be looked up with modified tag list\n\tvar origPlacement *Placement\n\tif cfg.Placement != nil {\n\t\ttmp := *cfg.Placement\n\t\torigPlacement = &tmp\n\t}\n\n\tif len(req.Tags) > 0 {\n\t\tif cfg.Placement == nil {\n\t\t\tcfg.Placement = &Placement{}\n\t\t}\n\t\tcfg.Placement.Tags = append(cfg.Placement.Tags, req.Tags...)\n\t}\n\n\tpeers, e := cc.selectPeerGroup(cfg.Replicas+1, currCluster, &cfg, currPeers, 1, nil)\n\tif len(peers) <= cfg.Replicas {\n\t\t// since expanding in the same cluster did not yield a result, try in different cluster\n\t\tpeers = nil\n\n\t\tclusters := map[string]struct{}{}\n\t\ts.nodeToInfo.Range(func(_, ni any) bool {\n\t\t\tif currCluster != ni.(nodeInfo).cluster {\n\t\t\t\tclusters[ni.(nodeInfo).cluster] = struct{}{}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\terrs := &selectPeerError{}\n\t\terrs.accumulate(e)\n\t\tfor cluster := range clusters {\n\t\t\tnewPeers, e := cc.selectPeerGroup(cfg.Replicas, cluster, &cfg, nil, 0, nil)\n\t\t\tif len(newPeers) >= cfg.Replicas {\n\t\t\t\tpeers = append([]string{}, currPeers...)\n\t\t\t\tpeers = append(peers, newPeers[:cfg.Replicas]...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\terrs.accumulate(e)\n\t\t}\n\t\tif peers == nil {\n\t\t\tresp.Error = NewJSClusterNoPeersError(errs)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\tcfg.Placement = origPlacement\n\n\ts.Noticef(\"Requested move for stream '%s > %s' R=%d from %+v to %+v\",\n\t\taccName, streamName, cfg.Replicas, s.peerSetToNames(currPeers), s.peerSetToNames(peers))\n\n\t// We will always have peers and therefore never do a callout, therefore it is safe to call inline\n\t// We should be fine ignoring pedantic mode here. as we do not touch configuration.\n\ts.jsClusteredStreamUpdateRequest(&ciNew, targetAcc.(*Account), subject, reply, rmsg, &cfg, peers, false)\n}\n\n// Request to have the metaleader move a stream on a peer to another\nfunc (s *Server) jsLeaderServerStreamCancelMoveRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\t// Extra checks here but only leader is listening.\n\tjs.mu.RLock()\n\tisLeader := cc.isLeader()\n\tjs.mu.RUnlock()\n\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\taccName := tokenAt(subject, 6)\n\tstreamName := tokenAt(subject, 7)\n\n\tif acc.GetName() != accName && acc != s.SystemAccount() {\n\t\treturn\n\t}\n\n\ttargetAcc, ok := s.accounts.Load(accName)\n\tif !ok {\n\t\tresp.Error = NewJSNoAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tstreamFound := false\n\tcfg := StreamConfig{}\n\tcurrPeers := []string{}\n\tjs.mu.Lock()\n\tstreams, ok := cc.streams[accName]\n\tif ok {\n\t\tsa, ok := streams[streamName]\n\t\tif ok {\n\t\t\tcfg = *sa.Config.clone()\n\t\t\tstreamFound = true\n\t\t\tcurrPeers = sa.Group.Peers\n\t\t}\n\t}\n\tjs.mu.Unlock()\n\n\tif !streamFound {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif len(currPeers) <= cfg.Replicas {\n\t\tresp.Error = NewJSStreamMoveNotInProgressError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// make sure client is scoped to requested account\n\tciNew := *(ci)\n\tciNew.Account = accName\n\n\tpeers := currPeers[:cfg.Replicas]\n\n\t// Remove placement in case tags don't match\n\t// This can happen if the move was initiated by modifying the tags.\n\t// This is an account operation.\n\t// This can NOT happen when the move was initiated by the system account.\n\t// There move honors the original tag list.\n\tif cfg.Placement != nil && len(cfg.Placement.Tags) != 0 {\n\tFOR_TAGCHECK:\n\t\tfor _, peer := range peers {\n\t\t\tsi, ok := s.nodeToInfo.Load(peer)\n\t\t\tif !ok {\n\t\t\t\t// can't verify tags, do the safe thing and error\n\t\t\t\tresp.Error = NewJSStreamGeneralError(\n\t\t\t\t\tfmt.Errorf(\"peer %s not present for tag validation\", peer))\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnodeTags := si.(nodeInfo).tags\n\t\t\tfor _, tag := range cfg.Placement.Tags {\n\t\t\t\tif !nodeTags.Contains(tag) {\n\t\t\t\t\t// clear placement as tags don't match\n\t\t\t\t\tcfg.Placement = nil\n\t\t\t\t\tbreak FOR_TAGCHECK\n\t\t\t\t}\n\t\t\t}\n\n\t\t}\n\t}\n\n\ts.Noticef(\"Requested cancel of move: R=%d '%s > %s' to peer set %+v and restore previous peer set %+v\",\n\t\tcfg.Replicas, accName, streamName, s.peerSetToNames(currPeers), s.peerSetToNames(peers))\n\n\t// We will always have peers and therefore never do a callout, therefore it is safe to call inline\n\ts.jsClusteredStreamUpdateRequest(&ciNew, targetAcc.(*Account), subject, reply, rmsg, &cfg, peers, false)\n}\n\n// Request to have an account purged\nfunc (s *Server) jsLeaderAccountPurgeRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\tif acc != s.SystemAccount() {\n\t\treturn\n\t}\n\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn\n\t}\n\n\taccName := tokenAt(subject, 5)\n\n\tvar resp = JSApiAccountPurgeResponse{ApiResponse: ApiResponse{Type: JSApiAccountPurgeResponseType}}\n\n\tif !s.JetStreamIsClustered() {\n\t\tvar streams []*stream\n\t\tvar ac *Account\n\t\tif ac, err = s.lookupAccount(accName); err == nil && ac != nil {\n\t\t\tstreams = ac.streams()\n\t\t}\n\n\t\ts.Noticef(\"Purge request for account %s (streams: %d, hasAccount: %t)\",\n\t\t\taccName, len(streams), ac != nil)\n\n\t\tfor _, mset := range streams {\n\t\t\terr := mset.delete()\n\t\t\tif err != nil {\n\t\t\t\tresp.Error = NewJSStreamDeleteError(err)\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tif err := os.RemoveAll(filepath.Join(js.config.StoreDir, accName)); err != nil {\n\t\t\tresp.Error = NewJSStreamGeneralError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tresp.Initiated = true\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t_, cc := s.getJetStreamCluster()\n\n\tjs.mu.RLock()\n\tisLeader := cc.isLeader()\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif js.isMetaRecovering() {\n\t\t// While in recovery mode, the data structures are not fully initialized\n\t\tresp.Error = NewJSClusterNotAvailError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tns, nc := 0, 0\n\tfor osa := range js.streamAssignmentsOrInflightSeq(accName) {\n\t\tfor oca := range js.consumerAssignmentsOrInflightSeq(accName, osa.Config.Name) {\n\t\t\tca := &consumerAssignment{Group: oca.Group, Stream: oca.Stream, Name: oca.Name, Config: oca.Config, Subject: subject, Client: oca.Client, Created: oca.Created}\n\t\t\tmeta.Propose(encodeDeleteConsumerAssignment(ca))\n\t\t\tcc.trackInflightConsumerProposal(accName, osa.Config.Name, ca, true)\n\t\t\tnc++\n\t\t}\n\t\tsa := &streamAssignment{Group: osa.Group, Config: osa.Config, Subject: subject, Client: osa.Client, Created: osa.Created}\n\t\tmeta.Propose(encodeDeleteStreamAssignment(sa))\n\t\tcc.trackInflightStreamProposal(accName, sa, true)\n\t\tns++\n\t}\n\tjs.mu.Unlock()\n\n\thasAccount := ns > 0\n\ts.Noticef(\"Purge request for account %s (streams: %d, consumer: %d, hasAccount: %t)\", accName, ns, nc, hasAccount)\n\n\tresp.Initiated = true\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n}\n\n// Request to have the meta leader stepdown.\n// These will only be received by the meta leader, so less checking needed.\nfunc (s *Server) jsLeaderStepDownRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\t// This should only be coming from the System Account.\n\tif acc != s.SystemAccount() {\n\t\ts.RateLimitWarnf(\"JetStream API stepdown request from non-system account: %q user: %q\", ci.serviceAccount(), ci.User)\n\t\treturn\n\t}\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\t// Extra checks here but only leader is listening.\n\tjs.mu.RLock()\n\tisLeader := cc.isLeader()\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tvar preferredLeader string\n\tvar resp = JSApiLeaderStepDownResponse{ApiResponse: ApiResponse{Type: JSApiLeaderStepDownResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiLeaderStepdownRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif preferredLeader, resp.Error = s.getStepDownPreferredPlacement(meta, req.Placement); resp.Error != nil {\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Call actual stepdown.\n\terr = meta.StepDown(preferredLeader)\n\tif err != nil {\n\t\tresp.Error = NewJSRaftGeneralError(err, Unless(err))\n\t} else {\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Check if given []bytes is a JSON Object or Array.\n// Technically, valid JSON can also be a plain string or number, but for our use case,\n// we care only for JSON objects or arrays which starts with `[` or `{`.\n// This function does not have to ensure valid JSON in its entirety. It is used merely\n// to hint the codepath if it should attempt to parse the request as JSON or not.\nfunc isJSONObjectOrArray(req []byte) bool {\n\t// Skip leading JSON whitespace (space, tab, newline, carriage return)\n\ti := 0\n\tfor i < len(req) && (req[i] == ' ' || req[i] == '\\t' || req[i] == '\\n' || req[i] == '\\r') {\n\t\ti++\n\t}\n\t// Check for empty input after trimming\n\tif i >= len(req) {\n\t\treturn false\n\t}\n\t// Check if the first non-whitespace character is '{' or '['\n\treturn req[i] == '{' || req[i] == '['\n}\n\nfunc isEmptyRequest(req []byte) bool {\n\tif len(req) == 0 {\n\t\treturn true\n\t}\n\tif bytes.Equal(req, []byte(\"{}\")) {\n\t\treturn true\n\t}\n\t// If we are here we didn't get our simple match, but still could be valid.\n\tvar v any\n\tif err := json.Unmarshal(req, &v); err != nil {\n\t\treturn false\n\t}\n\tvm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn false\n\t}\n\treturn len(vm) == 0\n}\n\n// getStepDownPreferredPlacement attempts to work out what the best placement is\n// for a stepdown request. The preferred server name always takes precedence, but\n// if not specified, the placement will be used to filter by cluster. The caller\n// should check for return API errors and return those to the requestor if needed.\nfunc (s *Server) getStepDownPreferredPlacement(group RaftNode, placement *Placement) (string, *ApiError) {\n\tif placement == nil {\n\t\treturn _EMPTY_, nil\n\t}\n\tvar preferredLeader string\n\tif placement.Preferred != _EMPTY_ {\n\t\tfor _, p := range group.Peers() {\n\t\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\t\tif !ok || si == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif si.(nodeInfo).name == placement.Preferred {\n\t\t\t\tpreferredLeader = p.ID\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif preferredLeader == group.ID() {\n\t\t\treturn _EMPTY_, NewJSClusterNoPeersError(fmt.Errorf(\"preferred server %q is already leader\", placement.Preferred))\n\t\t}\n\t\tif preferredLeader == _EMPTY_ {\n\t\t\treturn _EMPTY_, NewJSClusterNoPeersError(fmt.Errorf(\"preferred server %q not known\", placement.Preferred))\n\t\t}\n\t} else {\n\t\tpossiblePeers := make(map[*Peer]nodeInfo, len(group.Peers()))\n\t\tourID := group.ID()\n\t\tfor _, p := range group.Peers() {\n\t\t\tif p == nil {\n\t\t\t\tcontinue // ... shouldn't happen.\n\t\t\t}\n\t\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\t\tif !ok || si == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tni := si.(nodeInfo)\n\t\t\tif ni.offline || p.ID == ourID {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpossiblePeers[p] = ni\n\t\t}\n\t\t// If cluster is specified, filter out anything not matching the cluster name.\n\t\tif placement.Cluster != _EMPTY_ {\n\t\t\tfor p, si := range possiblePeers {\n\t\t\t\tif si.cluster != placement.Cluster {\n\t\t\t\t\tdelete(possiblePeers, p)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If tags are specified, filter out anything not matching all supplied tags.\n\t\tif len(placement.Tags) > 0 {\n\t\t\tfor p, si := range possiblePeers {\n\t\t\t\tmatchesAll := true\n\t\t\t\tfor _, tag := range placement.Tags {\n\t\t\t\t\tif matchesAll = matchesAll && si.tags.Contains(tag); !matchesAll {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !matchesAll {\n\t\t\t\t\tdelete(possiblePeers, p)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If there are no possible peers, return an error.\n\t\tif len(possiblePeers) == 0 {\n\t\t\treturn _EMPTY_, NewJSClusterNoPeersError(fmt.Errorf(\"no replacement peer connected\"))\n\t\t}\n\t\t// Take advantage of random map iteration order to select the preferred.\n\t\tfor p := range possiblePeers {\n\t\t\tpreferredLeader = p.ID\n\t\t\tbreak\n\t\t}\n\t}\n\treturn preferredLeader, nil\n}\n\n// Request to delete a stream.\nfunc (s *Server) jsStreamDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tif !isEmptyRequest(msg) {\n\t\tresp.Error = NewJSNotEmptyRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tstream := streamNameFromSubject(subject)\n\n\t// Clustered.\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredStreamDeleteRequest(ci, acc, stream, subject, reply, msg)\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif err := mset.delete(); err != nil {\n\t\tresp.Error = NewJSStreamDeleteError(err, Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tresp.Success = true\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to delete a message.\n// This expects a stream sequence number as the msg body.\nfunc (s *Server) jsMsgDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstream := tokenAt(subject, 6)\n\n\tvar resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}}\n\n\t// If we are in clustered mode we need to be the stream leader to proceed.\n\tif s.JetStreamIsClustered() {\n\t\t// Check to make sure the stream is assigned.\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\tjs.mu.RLock()\n\t\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)\n\t\tjs.mu.RUnlock()\n\n\t\tif isLeader && sa == nil {\n\t\t\t// We can't find the stream, so mimic what would be the errors below.\n\t\t\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\t\t\tif doErr {\n\t\t\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No stream present.\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else if sa == nil {\n\t\t\treturn\n\t\t}\n\n\t\t// Check to see if we are a member of the group and if the group has no leader.\n\t\tif js.isGroupLeaderless(sa.Group) {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\t// We have the stream assigned and a leader, so only the stream leader should answer.\n\t\tif !acc.JetStreamIsStreamLeader(stream) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tvar req JSApiMsgDeleteRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.cfg.Sealed {\n\t\tresp.Error = NewJSStreamSealedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.cfg.DenyDelete {\n\t\tresp.Error = NewJSStreamMsgDeleteFailedError(errors.New(\"message delete not permitted\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredMsgDeleteRequest(ci, acc, mset, stream, subject, reply, &req, rmsg)\n\t\treturn\n\t}\n\n\tvar removed bool\n\tif req.NoErase {\n\t\tremoved, err = mset.removeMsg(req.Seq)\n\t} else {\n\t\tremoved, err = mset.eraseMsg(req.Seq)\n\t}\n\tif err != nil {\n\t\tresp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err))\n\t} else if !removed {\n\t\tresp.Error = NewJSSequenceNotFoundError(req.Seq)\n\t} else {\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to get a raw stream message.\nfunc (s *Server) jsMsgGetRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstream := tokenAt(subject, 6)\n\n\tvar resp = JSApiMsgGetResponse{ApiResponse: ApiResponse{Type: JSApiMsgGetResponseType}}\n\n\t// If we are in clustered mode we need to be the stream leader to proceed.\n\tif s.JetStreamIsClustered() {\n\t\t// Check to make sure the stream is assigned.\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\tjs.mu.RLock()\n\t\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)\n\t\tjs.mu.RUnlock()\n\n\t\tif isLeader && sa == nil {\n\t\t\t// We can't find the stream, so mimic what would be the errors below.\n\t\t\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\t\t\tif doErr {\n\t\t\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No stream present.\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else if sa == nil {\n\t\t\treturn\n\t\t}\n\n\t\t// Check to see if we are a member of the group and if the group has no leader.\n\t\tif js.isGroupLeaderless(sa.Group) {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\t// We have the stream assigned and a leader, so only the stream leader should answer.\n\t\tif !acc.JetStreamIsStreamLeader(stream) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tvar req JSApiMsgGetRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// This version does not support batch.\n\tif req.Batch > 0 || req.MaxBytes > 0 {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Validate non-conflicting options. Seq, LastFor, and AsOfTime are mutually exclusive.\n\t// NextFor can be paired with Seq or AsOfTime indicating a filter subject.\n\tif (req.Seq > 0 && req.LastFor != _EMPTY_) ||\n\t\t(req.Seq == 0 && req.LastFor == _EMPTY_ && req.NextFor == _EMPTY_ && req.StartTime == nil) ||\n\t\t(req.Seq > 0 && req.StartTime != nil) ||\n\t\t(req.StartTime != nil && req.LastFor != _EMPTY_) ||\n\t\t(req.LastFor != _EMPTY_ && req.NextFor != _EMPTY_) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.offlineReason != _EMPTY_ {\n\t\t// Just let the request time out.\n\t\treturn\n\t}\n\n\tvar svp StoreMsg\n\tvar sm *StoreMsg\n\n\t// Ensure this read request is isolated and doesn't interleave with writes.\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\n\t// If AsOfTime is set, perform this first to get the sequence.\n\tvar seq uint64\n\tif req.StartTime != nil {\n\t\tseq = mset.store.GetSeqFromTime(*req.StartTime)\n\t} else {\n\t\tseq = req.Seq\n\t}\n\n\tif seq > 0 && req.NextFor == _EMPTY_ {\n\t\tsm, err = mset.store.LoadMsg(seq, &svp)\n\t} else if req.NextFor != _EMPTY_ {\n\t\tsm, _, err = mset.store.LoadNextMsg(req.NextFor, subjectHasWildcard(req.NextFor), seq, &svp)\n\t} else {\n\t\tsm, err = mset.store.LoadLastMsg(req.LastFor, &svp)\n\t}\n\tif err != nil {\n\t\tresp.Error = NewJSNoMessageFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tresp.Message = &StoredMsg{\n\t\tSubject: sm.subj,\n\t\tSequence: sm.seq,\n\t\tData: sm.msg,\n\t\tTime: time.Unix(0, sm.ts).UTC(),\n\t}\n\tif !req.NoHeaders {\n\t\tresp.Message.Header = sm.hdr\n\t}\n\n\t// Don't send response through API layer for this call.\n\ts.sendInternalAccountMsg(nil, reply, s.jsonResponse(resp))\n}\n\nfunc (s *Server) jsConsumerUnpinRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstream := streamNameFromSubject(subject)\n\tconsumer := consumerNameFromSubject(subject)\n\n\tvar resp = JSApiConsumerUnpinResponse{ApiResponse: ApiResponse{Type: JSApiConsumerUnpinResponseType}}\n\n\tif s.JetStreamIsClustered() {\n\t\t// Check to make sure the stream is assigned.\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\n\t\t// First check if the stream and consumer is there.\n\t\tjs.mu.RLock()\n\t\tsa := js.streamAssignment(acc.Name, stream)\n\t\tif sa == nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif sa.unsupported != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\t// Just let the request time out.\n\t\t\treturn\n\t\t}\n\n\t\tca, ok := sa.consumers[consumer]\n\t\tif !ok || ca == nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif ca.unsupported != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\t// Just let the request time out.\n\t\t\treturn\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t\t// Then check if we are the leader.\n\t\tmset, err := acc.lookupStream(stream)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\n\t\to := mset.lookupConsumer(consumer)\n\t\tif o == nil {\n\t\t\treturn\n\t\t}\n\t\tif !o.isLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar req JSApiConsumerUnpinRequest\n\tif err := json.Unmarshal(msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif req.Group == _EMPTY_ {\n\t\tresp.Error = NewJSInvalidJSONError(errors.New(\"consumer group not specified\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif !validGroupName.MatchString(req.Group) {\n\t\tresp.Error = NewJSConsumerInvalidGroupNameError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.offlineReason != _EMPTY_ {\n\t\t// Just let the request time out.\n\t\treturn\n\t}\n\to := mset.lookupConsumer(consumer)\n\tif o == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif o.offlineReason != _EMPTY_ {\n\t\t// Just let the request time out.\n\t\treturn\n\t}\n\n\tvar foundPriority bool\n\tfor _, group := range o.config().PriorityGroups {\n\t\tif group == req.Group {\n\t\t\tfoundPriority = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !foundPriority {\n\t\tresp.Error = NewJSConsumerInvalidPriorityGroupError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\to.mu.Lock()\n\to.unassignPinId()\n\to.sendUnpinnedAdvisoryLocked(req.Group, \"admin\")\n\to.mu.Unlock()\n\to.signalNewMessages()\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to purge a stream.\nfunc (s *Server) jsStreamPurgeRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstream := streamNameFromSubject(subject)\n\n\tvar resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}}\n\n\t// If we are in clustered mode we need to be the stream leader to proceed.\n\tif s.JetStreamIsClustered() {\n\t\t// Check to make sure the stream is assigned.\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\n\t\tjs.mu.RLock()\n\t\tisLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)\n\t\tjs.mu.RUnlock()\n\n\t\tif isLeader && sa == nil {\n\t\t\t// We can't find the stream, so mimic what would be the errors below.\n\t\t\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\t\t\tif doErr {\n\t\t\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No stream present.\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else if sa == nil {\n\t\t\tif js.isLeaderless() {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\t// Check to see if we are a member of the group and if the group has no leader.\n\t\tif js.isGroupLeaderless(sa.Group) {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\t// We have the stream assigned and a leader, so only the stream leader should answer.\n\t\tif !acc.JetStreamIsStreamLeader(stream) {\n\t\t\tif js.isLeaderless() {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar purgeRequest *JSApiStreamPurgeRequest\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiStreamPurgeRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif req.Sequence > 0 && req.Keep > 0 {\n\t\t\tresp.Error = NewJSBadRequestError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tpurgeRequest = &req\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.cfg.Sealed {\n\t\tresp.Error = NewJSStreamSealedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.cfg.DenyPurge {\n\t\tresp.Error = NewJSStreamPurgeFailedError(errors.New(\"stream purge not permitted\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredStreamPurgeRequest(ci, acc, mset, stream, subject, reply, rmsg, purgeRequest)\n\t\treturn\n\t}\n\n\tpurged, err := mset.purge(purgeRequest)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamGeneralError(err, Unless(err))\n\t} else {\n\t\tresp.Purged = purged\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\nfunc (acc *Account) jsNonClusteredStreamLimitsCheck(cfg *StreamConfig) *ApiError {\n\tvar replicas int\n\tif cfg != nil {\n\t\treplicas = cfg.Replicas\n\t}\n\tselectedLimits, tier, jsa, apiErr := acc.selectLimits(replicas)\n\tif apiErr != nil {\n\t\treturn apiErr\n\t}\n\tjsa.js.mu.RLock()\n\tdefer jsa.js.mu.RUnlock()\n\tjsa.mu.RLock()\n\tdefer jsa.mu.RUnlock()\n\tif selectedLimits.MaxStreams > 0 && jsa.countStreams(tier, cfg) >= selectedLimits.MaxStreams {\n\t\treturn NewJSMaximumStreamsLimitError()\n\t}\n\treserved := jsa.tieredReservation(tier, cfg)\n\tif err := jsa.js.checkAllLimits(selectedLimits, cfg, reserved, 0); err != nil {\n\t\treturn NewJSStreamLimitsError(err, Unless(err))\n\t}\n\treturn nil\n}\n\n// Request to restore a stream.\nfunc (s *Server) jsStreamRestoreRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamIsLeader() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif !acc.JetStreamEnabled() {\n\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar req JSApiStreamRestoreRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tstream := streamNameFromSubject(subject)\n\n\tif stream != req.Config.Name && req.Config.Name == _EMPTY_ {\n\t\treq.Config.Name = stream\n\t}\n\n\t// check stream config at the start of the restore process, not at the end\n\tcfg, apiErr := s.checkStreamCfg(&req.Config, acc, false)\n\tif apiErr != nil {\n\t\tresp.Error = apiErr\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredStreamRestoreRequest(ci, acc, &req, subject, reply, rmsg)\n\t\treturn\n\t}\n\n\tif err := acc.jsNonClusteredStreamLimitsCheck(&cfg); err != nil {\n\t\tresp.Error = err\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif _, err := acc.lookupStream(stream); err == nil {\n\t\tresp.Error = NewJSStreamNameExistRestoreFailedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\ts.processStreamRestore(ci, acc, &req.Config, subject, reply, string(msg))\n}\n\nfunc (s *Server) processStreamRestore(ci *ClientInfo, acc *Account, cfg *StreamConfig, subject, reply, msg string) <-chan error {\n\tjs := s.getJetStream()\n\n\tvar resp = JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}}\n\n\tsnapDir := filepath.Join(js.config.StoreDir, snapStagingDir)\n\tif _, err := os.Stat(snapDir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(snapDir, defaultDirPerms); err != nil {\n\t\t\tresp.Error = &ApiError{Code: 503, Description: \"JetStream unable to create temp storage for restore\"}\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn nil\n\t\t}\n\t}\n\n\ttfile, err := os.CreateTemp(snapDir, \"js-restore-\")\n\tif err != nil {\n\t\tresp.Error = NewJSTempStorageFailedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, msg, s.jsonResponse(&resp))\n\t\treturn nil\n\t}\n\n\tstreamName := cfg.Name\n\ts.Noticef(\"Starting restore for stream '%s > %s'\", acc.Name, streamName)\n\n\tstart := time.Now().UTC()\n\tdomain := s.getOpts().JetStreamDomain\n\ts.publishAdvisory(acc, JSAdvisoryStreamRestoreCreatePre+\".\"+streamName, &JSRestoreCreateAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSRestoreCreateAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: start,\n\t\t},\n\t\tStream: streamName,\n\t\tClient: ci.forAdvisory(),\n\t\tDomain: domain,\n\t})\n\n\t// Create our internal subscription to accept the snapshot.\n\trestoreSubj := fmt.Sprintf(jsRestoreDeliverT, streamName, nuid.Next())\n\n\ttype result struct {\n\t\terr error\n\t\treply string\n\t}\n\n\t// For signaling to upper layers.\n\tresultCh := make(chan result, 1)\n\tactiveQ := newIPQueue[int](s, fmt.Sprintf(\"[ACC:%s] stream '%s' restore\", acc.Name, streamName)) // of int\n\n\tvar total int\n\n\t// FIXME(dlc) - Probably take out of network path eventually due to disk I/O?\n\tprocessChunk := func(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\t\t// We require reply subjects to communicate back failures, flow etc. If they do not have one log and cancel.\n\t\tif reply == _EMPTY_ {\n\t\t\tsub.client.processUnsub(sub.sid)\n\t\t\tresultCh <- result{\n\t\t\t\tfmt.Errorf(\"restore for stream '%s > %s' requires reply subject for each chunk\", acc.Name, streamName),\n\t\t\t\treply,\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\t// Account client messages have \\r\\n on end. This is an error.\n\t\tif len(msg) < LEN_CR_LF {\n\t\t\tsub.client.processUnsub(sub.sid)\n\t\t\tresultCh <- result{\n\t\t\t\tfmt.Errorf(\"restore for stream '%s > %s' received short chunk\", acc.Name, streamName),\n\t\t\t\treply,\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\t// Adjust.\n\t\tmsg = msg[:len(msg)-LEN_CR_LF]\n\n\t\t// This means we are complete with our transfer from the client.\n\t\tif len(msg) == 0 {\n\t\t\ts.Debugf(\"Finished staging restore for stream '%s > %s'\", acc.Name, streamName)\n\t\t\tresultCh <- result{err, reply}\n\t\t\treturn\n\t\t}\n\n\t\t// We track total and check on server limits.\n\t\t// TODO(dlc) - We could check apriori and cancel initial request if we know it won't fit.\n\t\ttotal += len(msg)\n\t\tif js.wouldExceedLimits(FileStorage, total) {\n\t\t\ts.resourcesExceededError(FileStorage)\n\t\t\tresultCh <- result{NewJSInsufficientResourcesError(), reply}\n\t\t\treturn\n\t\t}\n\n\t\t// Append chunk to temp file. Mark as issue if we encounter an error.\n\t\tif n, err := tfile.Write(msg); n != len(msg) || err != nil {\n\t\t\tresultCh <- result{err, reply}\n\t\t\tif reply != _EMPTY_ {\n\t\t\t\ts.sendInternalAccountMsg(acc, reply, \"-ERR 'storage failure during restore'\")\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tactiveQ.push(len(msg))\n\n\t\ts.sendInternalAccountMsg(acc, reply, nil)\n\t}\n\n\tsub, err := acc.subscribeInternal(restoreSubj, processChunk)\n\tif err != nil {\n\t\ttfile.Close()\n\t\tos.Remove(tfile.Name())\n\t\tresp.Error = NewJSRestoreSubscribeFailedError(err, restoreSubj)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, msg, s.jsonResponse(&resp))\n\t\treturn nil\n\t}\n\n\t// Mark the subject so the end user knows where to send the snapshot chunks.\n\tresp.DeliverSubject = restoreSubj\n\ts.sendAPIResponse(ci, acc, subject, reply, msg, s.jsonResponse(resp))\n\n\tdoneCh := make(chan error, 1)\n\n\t// Monitor the progress from another Go routine.\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\t\tdefer func() {\n\t\t\ttfile.Close()\n\t\t\tos.Remove(tfile.Name())\n\t\t\tsub.client.processUnsub(sub.sid)\n\t\t\tactiveQ.unregister()\n\t\t}()\n\n\t\tconst activityInterval = 5 * time.Second\n\t\tnotActive := time.NewTimer(activityInterval)\n\t\tdefer notActive.Stop()\n\n\t\ttotal := 0\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase result := <-resultCh:\n\t\t\t\terr := result.err\n\t\t\t\tvar mset *stream\n\n\t\t\t\t// If we staged properly go ahead and do restore now.\n\t\t\t\tif err == nil {\n\t\t\t\t\ts.Debugf(\"Finalizing restore for stream '%s > %s'\", acc.Name, streamName)\n\t\t\t\t\ttfile.Seek(0, 0)\n\t\t\t\t\tmset, err = acc.RestoreStream(cfg, tfile)\n\t\t\t\t} else {\n\t\t\t\t\terrStr := err.Error()\n\t\t\t\t\ttmp := []rune(errStr)\n\t\t\t\t\ttmp[0] = unicode.ToUpper(tmp[0])\n\t\t\t\t\ts.Warnf(errStr)\n\t\t\t\t}\n\n\t\t\t\tend := time.Now().UTC()\n\n\t\t\t\t// TODO(rip) - Should this have the error code in it??\n\t\t\t\ts.publishAdvisory(acc, JSAdvisoryStreamRestoreCompletePre+\".\"+streamName, &JSRestoreCompleteAdvisory{\n\t\t\t\t\tTypedEvent: TypedEvent{\n\t\t\t\t\t\tType: JSRestoreCompleteAdvisoryType,\n\t\t\t\t\t\tID: nuid.Next(),\n\t\t\t\t\t\tTime: end,\n\t\t\t\t\t},\n\t\t\t\t\tStream: streamName,\n\t\t\t\t\tStart: start,\n\t\t\t\t\tEnd: end,\n\t\t\t\t\tBytes: int64(total),\n\t\t\t\t\tClient: ci.forAdvisory(),\n\t\t\t\t\tDomain: domain,\n\t\t\t\t})\n\n\t\t\t\tvar resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tresp.Error = NewJSStreamRestoreError(err, Unless(err))\n\t\t\t\t\ts.Warnf(\"Restore failed for %s for stream '%s > %s' in %v\",\n\t\t\t\t\t\tfriendlyBytes(int64(total)), acc.Name, streamName, end.Sub(start))\n\t\t\t\t} else {\n\t\t\t\t\tmsetCfg := mset.config()\n\t\t\t\t\tresp.StreamInfo = &StreamInfo{\n\t\t\t\t\t\tCreated: mset.createdTime(),\n\t\t\t\t\t\tState: mset.state(),\n\t\t\t\t\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\t\t\t\t\tTimeStamp: time.Now().UTC(),\n\t\t\t\t\t}\n\t\t\t\t\ts.Noticef(\"Completed restore of %s for stream '%s > %s' in %v\",\n\t\t\t\t\t\tfriendlyBytes(int64(total)), acc.Name, streamName, end.Sub(start).Round(time.Millisecond))\n\t\t\t\t}\n\n\t\t\t\t// On the last EOF, send back the stream info or error status.\n\t\t\t\ts.sendInternalAccountMsg(acc, result.reply, s.jsonResponse(&resp))\n\t\t\t\t// Signal to the upper layers.\n\t\t\t\tdoneCh <- err\n\t\t\t\treturn\n\t\t\tcase <-activeQ.ch:\n\t\t\t\tif n, ok := activeQ.popOne(); ok {\n\t\t\t\t\ttotal += n\n\t\t\t\t\tnotActive.Reset(activityInterval)\n\t\t\t\t}\n\t\t\tcase <-notActive.C:\n\t\t\t\terr := fmt.Errorf(\"restore for stream '%s > %s' is stalled\", acc, streamName)\n\t\t\t\tdoneCh <- err\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t})\n\n\treturn doneCh\n}\n\n// Process a snapshot request.\nfunc (s *Server) jsStreamSnapshotRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tsmsg := string(msg)\n\tstream := streamNameFromSubject(subject)\n\n\t// If we are in clustered mode we need to be the stream leader to proceed.\n\tif s.JetStreamIsClustered() && !acc.JetStreamIsStreamLeader(stream) {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamSnapshotResponse{ApiResponse: ApiResponse{Type: JSApiStreamSnapshotResponseType}}\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif !acc.JetStreamEnabled() {\n\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif isEmptyRequest(msg) {\n\t\tresp.Error = NewJSBadRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar req JSApiStreamSnapshotRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif !IsValidSubject(req.DeliverSubject) {\n\t\tresp.Error = NewJSSnapshotDeliverSubjectInvalidError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// We will do the snapshot in a go routine as well since check msgs may\n\t// stall this go routine.\n\tgo func() {\n\t\tif req.CheckMsgs {\n\t\t\ts.Noticef(\"Starting health check and snapshot for stream '%s > %s'\", mset.jsa.account.Name, mset.name())\n\t\t} else {\n\t\t\ts.Noticef(\"Starting snapshot for stream '%s > %s'\", mset.jsa.account.Name, mset.name())\n\t\t}\n\n\t\tstart := time.Now().UTC()\n\n\t\tsr, err := mset.snapshot(0, req.CheckMsgs, !req.NoConsumers)\n\t\tif err != nil {\n\t\t\ts.Warnf(\"Snapshot of stream '%s > %s' failed: %v\", mset.jsa.account.Name, mset.name(), err)\n\t\t\tresp.Error = NewJSStreamSnapshotError(err, Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, smsg, s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\tconfig := mset.config()\n\t\tresp.State = &sr.State\n\t\tresp.Config = &config\n\n\t\ts.sendAPIResponse(ci, acc, subject, reply, smsg, s.jsonResponse(resp))\n\n\t\ts.publishAdvisory(acc, JSAdvisoryStreamSnapshotCreatePre+\".\"+mset.name(), &JSSnapshotCreateAdvisory{\n\t\t\tTypedEvent: TypedEvent{\n\t\t\t\tType: JSSnapshotCreatedAdvisoryType,\n\t\t\t\tID: nuid.Next(),\n\t\t\t\tTime: time.Now().UTC(),\n\t\t\t},\n\t\t\tStream: mset.name(),\n\t\t\tState: sr.State,\n\t\t\tClient: ci.forAdvisory(),\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t})\n\n\t\t// Now do the real streaming.\n\t\ts.streamSnapshot(acc, mset, sr, &req)\n\n\t\tend := time.Now().UTC()\n\n\t\ts.publishAdvisory(acc, JSAdvisoryStreamSnapshotCompletePre+\".\"+mset.name(), &JSSnapshotCompleteAdvisory{\n\t\t\tTypedEvent: TypedEvent{\n\t\t\t\tType: JSSnapshotCompleteAdvisoryType,\n\t\t\t\tID: nuid.Next(),\n\t\t\t\tTime: end,\n\t\t\t},\n\t\t\tStream: mset.name(),\n\t\t\tStart: start,\n\t\t\tEnd: end,\n\t\t\tClient: ci.forAdvisory(),\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t})\n\n\t\ts.Noticef(\"Completed snapshot of %s for stream '%s > %s' in %v\",\n\t\t\tfriendlyBytes(int64(sr.State.Bytes)),\n\t\t\tmset.jsa.account.Name,\n\t\t\tmset.name(),\n\t\t\tend.Sub(start))\n\t}()\n}\n\n// Default chunk size for now.\nconst defaultSnapshotChunkSize = 128 * 1024 // 128KiB\nconst defaultSnapshotWindowSize = 8 * 1024 * 1024 // 8MiB\nconst defaultSnapshotAckTimeout = 5 * time.Second\n\nvar snapshotAckTimeout = defaultSnapshotAckTimeout\n\n// streamSnapshot will stream out our snapshot to the reply subject.\nfunc (s *Server) streamSnapshot(acc *Account, mset *stream, sr *SnapshotResult, req *JSApiStreamSnapshotRequest) {\n\tchunkSize, wndSize := req.ChunkSize, req.WindowSize\n\tif chunkSize == 0 {\n\t\tchunkSize = defaultSnapshotChunkSize\n\t}\n\tif wndSize == 0 {\n\t\twndSize = defaultSnapshotWindowSize\n\t}\n\tchunkSize = min(max(1024, chunkSize), 1024*1024) // Clamp within 1KiB to 1MiB\n\twndSize = min(max(1024, wndSize), 32*1024*1024) // Clamp within 1KiB to 32MiB\n\twndSize = max(wndSize, chunkSize) // Guarantee at least one chunk\n\tmaxInflight := wndSize / chunkSize // Between 1 and 32,768\n\n\t// Setup for the chunk stream.\n\treply := req.DeliverSubject\n\tr := sr.Reader\n\tdefer r.Close()\n\n\t// In case we run into an error, this allows subscription callbacks\n\t// to not sit and block endlessly.\n\tdone := make(chan struct{})\n\tdefer close(done)\n\n\t// Check interest for the snapshot deliver subject.\n\tinch := make(chan bool, 1)\n\tacc.sl.RegisterNotification(req.DeliverSubject, inch)\n\tdefer acc.sl.ClearNotification(req.DeliverSubject, inch)\n\thasInterest := <-inch\n\tif !hasInterest {\n\t\t// Allow 2 seconds or so for interest to show up.\n\t\tselect {\n\t\tcase <-inch:\n\t\tcase <-time.After(2 * time.Second):\n\t\t}\n\t}\n\n\t// One slot per chunk. Each chunk read takes a slot, each ack will\n\t// replace it. Smooths out in-flight number of chunks.\n\tslots := make(chan struct{}, maxInflight)\n\tfor range maxInflight {\n\t\tslots <- struct{}{}\n\t}\n\n\t// We will place sequence number and size of chunk sent in the reply.\n\tackSubj := fmt.Sprintf(jsSnapshotAckT, mset.name(), nuid.Next())\n\tackSub, _ := mset.subscribeInternal(ackSubj+\".>\", func(_ *subscription, _ *client, _ *Account, subject, _ string, _ []byte) {\n\t\tselect {\n\t\tcase slots <- struct{}{}:\n\t\tcase <-done:\n\t\t}\n\t})\n\tdefer mset.unsubscribe(ackSub)\n\n\tvar hdr []byte\n\tchunk := make([]byte, chunkSize)\n\tfor index := 1; ; index++ {\n\t\tselect {\n\t\tcase <-slots:\n\t\t\t// A slot has become available.\n\t\tcase <-inch:\n\t\t\t// The receiver appears to have gone away.\n\t\t\thdr = []byte(\"NATS/1.0 408 No Interest\\r\\n\\r\\n\")\n\t\t\tgoto done\n\t\tcase err := <-sr.errCh:\n\t\t\t// The snapshotting goroutine has failed for some reason.\n\t\t\thdr = []byte(fmt.Sprintf(\"NATS/1.0 500 %s\\r\\n\\r\\n\", err))\n\t\t\tgoto done\n\t\tcase <-time.After(snapshotAckTimeout):\n\t\t\t// It's taking a very long time for the receiver to send us acks,\n\t\t\t// they have probably stalled or there is high loss on the link.\n\t\t\thdr = []byte(\"NATS/1.0 408 No Flow Response\\r\\n\\r\\n\")\n\t\t\tgoto done\n\t\t}\n\t\tn, err := io.ReadFull(r, chunk)\n\t\tchunk := chunk[:n]\n\t\tif err != nil {\n\t\t\tif n > 0 {\n\t\t\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, chunk, nil, 0))\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tackReply := fmt.Sprintf(\"%s.%d.%d\", ackSubj, len(chunk), index)\n\t\tif hdr == nil {\n\t\t\thdr = []byte(\"NATS/1.0 204\\r\\n\\r\\n\")\n\t\t}\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, ackReply, nil, chunk, nil, 0))\n\t}\n\ndone:\n\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n}\n\n// For determining consumer request type.\ntype ccReqType uint8\n\nconst (\n\tccNew = iota\n\tccLegacyEphemeral\n\tccLegacyDurable\n)\n\n// Request to create a consumer where stream and optional consumer name are part of the subject, and optional\n// filtered subjects can be at the tail end.\n// Assumes stream and consumer names are single tokens.\nfunc (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}\n\n\tvar req CreateConsumerRequest\n\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tvar js *jetStream\n\tisClustered := s.JetStreamIsClustered()\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif isClustered {\n\t\tif req.Config.Direct {\n\t\t\t// Check to see if we have this stream and are the stream leader.\n\t\t\tif !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {\n\t\t\t\treturn\n\t\t\t}\n\t\t} else {\n\t\t\tvar cc *jetStreamCluster\n\t\t\tjs, cc = s.getJetStreamCluster()\n\t\t\tif js == nil || cc == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif js.isLeaderless() {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Make sure we are meta leader.\n\t\t\tif !s.JetStreamIsLeader() {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar streamName, consumerName, filteredSubject string\n\tvar rt ccReqType\n\n\tif n := numTokens(subject); n < 5 {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t} else if n == 5 {\n\t\t// Legacy ephemeral.\n\t\trt = ccLegacyEphemeral\n\t\tstreamName = streamNameFromSubject(subject)\n\t} else {\n\t\t// New style and durable legacy.\n\t\tif tokenAt(subject, 4) == \"DURABLE\" {\n\t\t\trt = ccLegacyDurable\n\t\t\tif n != 7 {\n\t\t\t\tresp.Error = NewJSConsumerDurableNameNotInSubjectError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\tstreamName = tokenAt(subject, 6)\n\t\t\tconsumerName = tokenAt(subject, 7)\n\t\t} else {\n\t\t\tstreamName = streamNameFromSubject(subject)\n\t\t\tconsumerName = consumerNameFromSubject(subject)\n\t\t\t// New has optional filtered subject as part of main subject..\n\t\t\tif n > 6 {\n\t\t\t\ttokens := strings.Split(subject, tsep)\n\t\t\t\tfilteredSubject = strings.Join(tokens[6:], tsep)\n\t\t\t}\n\t\t}\n\t}\n\n\tif streamName != req.Stream {\n\t\tresp.Error = NewJSStreamMismatchError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif consumerName != _EMPTY_ {\n\t\t// Check for path like separators in the name.\n\t\tif strings.ContainsAny(consumerName, `\\/`) {\n\t\t\tresp.Error = NewJSConsumerNameContainsPathSeparatorsError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Should we expect a durable name\n\tif rt == ccLegacyDurable {\n\t\tif numTokens(subject) < 7 {\n\t\t\tresp.Error = NewJSConsumerDurableNameNotInSubjectError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Now check on requirements for durable request.\n\t\tif req.Config.Durable == _EMPTY_ {\n\t\t\tresp.Error = NewJSConsumerDurableNameNotSetError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif consumerName != req.Config.Durable {\n\t\t\tresp.Error = NewJSConsumerDurableNameNotMatchSubjectError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Durable, so we need to honor the name.\n\t\treq.Config.Name = consumerName\n\t}\n\t// If new style and durable set make sure they match.\n\tif rt == ccNew {\n\t\tif req.Config.Durable != _EMPTY_ {\n\t\t\tif consumerName != req.Config.Durable {\n\t\t\t\tresp.Error = NewJSConsumerDurableNameNotMatchSubjectError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// New style ephemeral so we need to honor the name.\n\t\treq.Config.Name = consumerName\n\t}\n\t// Check for legacy ephemeral mis-configuration.\n\tif rt == ccLegacyEphemeral && req.Config.Durable != _EMPTY_ {\n\t\tresp.Error = NewJSConsumerEphemeralWithDurableNameError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// in case of multiple filters provided, error if new API is used.\n\tif filteredSubject != _EMPTY_ && len(req.Config.FilterSubjects) != 0 {\n\t\tresp.Error = NewJSConsumerMultipleFiltersNotAllowedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Check for a filter subject.\n\tif filteredSubject != _EMPTY_ && req.Config.FilterSubject != filteredSubject {\n\t\tresp.Error = NewJSConsumerCreateFilterSubjectMismatchError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif isClustered && !req.Config.Direct {\n\t\ts.jsClusteredConsumerRequest(ci, acc, subject, reply, rmsg, req.Stream, &req.Config, req.Action, req.Pedantic)\n\t\treturn\n\t}\n\n\t// If we are here we are single server mode.\n\tif req.Config.Replicas > 1 {\n\t\tresp.Error = NewJSStreamReplicasNotSupportedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tstream, err := acc.lookupStream(req.Stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif stream.offlineReason != _EMPTY_ {\n\t\tresp.Error = NewJSStreamOfflineReasonError(errors.New(stream.offlineReason))\n\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\treturn\n\t}\n\n\tif o := stream.lookupConsumer(consumerName); o != nil {\n\t\tif o.offlineReason != _EMPTY_ {\n\t\t\tresp.Error = NewJSConsumerOfflineReasonError(errors.New(o.offlineReason))\n\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\treturn\n\t\t}\n\t\t// If the consumer already exists then don't allow updating the PauseUntil, just set\n\t\t// it back to whatever the current configured value is.\n\t\to.mu.RLock()\n\t\treq.Config.PauseUntil = o.cfg.PauseUntil\n\t\to.mu.RUnlock()\n\t}\n\n\t// Initialize/update asset version metadata.\n\tsetStaticConsumerMetadata(&req.Config)\n\n\to, err := stream.addConsumerWithAction(&req.Config, req.Action, req.Pedantic)\n\n\tif err != nil {\n\t\tif IsNatsErr(err, JSConsumerStoreFailedErrF) {\n\t\t\tcname := req.Config.Durable // Will be empty if ephemeral.\n\t\t\ts.Warnf(\"Consumer create failed for '%s > %s > %s': %v\", acc, req.Stream, cname, err)\n\t\t\terr = errConsumerStoreFailed\n\t\t}\n\t\tresp.Error = NewJSConsumerCreateError(err, Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.initialInfo())\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n\n\to.mu.RLock()\n\tif o.cfg.PauseUntil != nil && !o.cfg.PauseUntil.IsZero() && time.Now().Before(*o.cfg.PauseUntil) {\n\t\to.sendPauseAdvisoryLocked(&o.cfg)\n\t}\n\to.mu.RUnlock()\n}\n\n// Request for the list of all consumer names.\nfunc (s *Server) jsConsumerNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerNamesResponse{\n\t\tApiResponse: ApiResponse{Type: JSApiConsumerNamesResponseType},\n\t\tConsumers: []string{},\n\t}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar offset int\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiConsumersRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\toffset = req.Offset\n\t}\n\n\tstreamName := streamNameFromSubject(subject)\n\tvar numConsumers int\n\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\t// TODO(dlc) - Debug or Warn?\n\t\t\treturn\n\t\t}\n\t\tjs.mu.RLock()\n\t\tsas := cc.streams[acc.Name]\n\t\tif sas == nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tsa := sas[streamName]\n\t\tif sa == nil || sa.err != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tfor consumer := range sa.consumers {\n\t\t\tresp.Consumers = append(resp.Consumers, consumer)\n\t\t}\n\t\tif len(resp.Consumers) > 1 {\n\t\t\tslices.Sort(resp.Consumers)\n\t\t}\n\t\tnumConsumers = len(resp.Consumers)\n\t\tif offset > numConsumers {\n\t\t\toffset = numConsumers\n\t\t}\n\t\tresp.Consumers = resp.Consumers[offset:]\n\t\tif len(resp.Consumers) > JSApiNamesLimit {\n\t\t\tresp.Consumers = resp.Consumers[:JSApiNamesLimit]\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t} else {\n\t\tmset, err := acc.lookupStream(streamName)\n\t\tif err != nil {\n\t\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\tobs := mset.getPublicConsumers()\n\t\tslices.SortFunc(obs, func(i, j *consumer) int { return cmp.Compare(i.name, j.name) })\n\n\t\tnumConsumers = len(obs)\n\t\tif offset > numConsumers {\n\t\t\toffset = numConsumers\n\t\t}\n\n\t\tfor _, o := range obs[offset:] {\n\t\t\tresp.Consumers = append(resp.Consumers, o.String())\n\t\t\tif len(resp.Consumers) >= JSApiNamesLimit {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tresp.Total = numConsumers\n\tresp.Limit = JSApiNamesLimit\n\tresp.Offset = offset\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request for the list of all detailed consumer information.\nfunc (s *Server) jsConsumerListRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerListResponse{\n\t\tApiResponse: ApiResponse{Type: JSApiConsumerListResponseType},\n\t\tConsumers: []*ConsumerInfo{},\n\t}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tvar offset int\n\tif isJSONObjectOrArray(msg) {\n\t\tvar req JSApiConsumersRequest\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\toffset = req.Offset\n\t}\n\n\tstreamName := streamNameFromSubject(subject)\n\n\t// Clustered mode will invoke a scatter and gather.\n\tif s.JetStreamIsClustered() {\n\t\t// Need to copy these off before sending.. don't move this inside startGoRoutine!!!\n\t\tmsg = copyBytes(msg)\n\t\ts.startGoRoutine(func() {\n\t\t\ts.jsClusteredConsumerListRequest(acc, ci, offset, streamName, subject, reply, msg)\n\t\t})\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(streamName)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tobs := mset.getPublicConsumers()\n\tslices.SortFunc(obs, func(i, j *consumer) int { return cmp.Compare(i.name, j.name) })\n\n\tocnt := len(obs)\n\tif offset > ocnt {\n\t\toffset = ocnt\n\t}\n\n\tvar missingNames []string\n\tfor _, o := range obs[offset:] {\n\t\tif o.offlineReason != _EMPTY_ {\n\t\t\tif resp.Offline == nil {\n\t\t\t\tresp.Offline = make(map[string]string, 1)\n\t\t\t}\n\t\t\tresp.Offline[o.name] = o.offlineReason\n\t\t\tmissingNames = append(missingNames, o.name)\n\t\t\tcontinue\n\t\t}\n\t\tif cinfo := o.info(); cinfo != nil {\n\t\t\tresp.Consumers = append(resp.Consumers, cinfo)\n\t\t}\n\t\tif len(resp.Consumers) >= JSApiListLimit {\n\t\t\tbreak\n\t\t}\n\t}\n\tresp.Total = ocnt\n\tresp.Limit = JSApiListLimit\n\tresp.Offset = offset\n\tresp.Missing = missingNames\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request for information about an consumer.\nfunc (s *Server) jsConsumerInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tstreamName := streamNameFromSubject(subject)\n\tconsumerName := consumerNameFromSubject(subject)\n\n\tvar resp = JSApiConsumerInfoResponse{ApiResponse: ApiResponse{Type: JSApiConsumerInfoResponseType}}\n\n\tif !isEmptyRequest(msg) {\n\t\tresp.Error = NewJSNotEmptyRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// If we are in clustered mode we need to be the consumer leader to proceed.\n\tif s.JetStreamIsClustered() {\n\t\t// Check to make sure the consumer is assigned.\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\n\t\tjs.mu.RLock()\n\t\tmeta := cc.meta\n\t\tjs.mu.RUnlock()\n\n\t\tif meta == nil {\n\t\t\treturn\n\t\t}\n\n\t\t// Since these could wait on the Raft group lock, don't do so under the JS lock.\n\t\tourID := meta.ID()\n\t\tgroupLeaderless := meta.Leaderless()\n\t\tgroupCreated := meta.Created()\n\n\t\tjs.mu.RLock()\n\t\tisLeader, sa, ca := cc.isLeader(), js.streamAssignment(acc.Name, streamName), js.consumerAssignment(acc.Name, streamName, consumerName)\n\t\tvar rg *raftGroup\n\t\tvar offline, isMember bool\n\t\tif ca != nil {\n\t\t\tif rg = ca.Group; rg != nil {\n\t\t\t\toffline = s.allPeersOffline(rg)\n\t\t\t\tisMember = rg.isMember(ourID)\n\t\t\t}\n\t\t\tif ca.unsupported != nil && isMember {\n\t\t\t\t// If we're a member for this consumer, and it's not supported, report it as offline.\n\t\t\t\tresp.Error = NewJSConsumerOfflineReasonError(errors.New(ca.unsupported.reason))\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// Capture consumer leader here.\n\t\tisConsumerLeader := cc.isConsumerLeader(acc.Name, streamName, consumerName)\n\t\t// Also capture if we think there is no meta leader.\n\t\tvar isLeaderLess bool\n\t\tif !isLeader {\n\t\t\tisLeaderLess = groupLeaderless && time.Since(groupCreated) > lostQuorumIntervalDefault\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t\tif isLeader && ca == nil {\n\t\t\t// We can't find the consumer, so mimic what would be the errors below.\n\t\t\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\t\t\tif doErr {\n\t\t\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif sa == nil {\n\t\t\t\tresp.Error = NewJSStreamNotFoundError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// If we are here the consumer is not present.\n\t\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else if ca == nil {\n\t\t\tif isLeaderLess {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\t// Delaying an error response gives the leader a chance to respond before us\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t}\n\t\t\treturn\n\t\t} else if isLeader && offline {\n\t\t\tresp.Error = NewJSConsumerOfflineError()\n\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\treturn\n\t\t}\n\n\t\t// Check to see if we are a member of the group and if the group has no leader.\n\t\tif isMember && js.isGroupLeaderless(ca.Group) {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\n\t\t// We have the consumer assigned and a leader, so only the consumer leader should answer.\n\t\tif !isConsumerLeader {\n\t\t\tif isLeaderLess {\n\t\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\t\t// Delaying an error response gives the leader a chance to respond before us\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), ca.Group, errRespDelay)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar node RaftNode\n\t\t\tvar leaderNotPartOfGroup bool\n\n\t\t\t// We have a consumer assignment.\n\t\t\tif isMember {\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tif rg != nil && rg.node != nil {\n\t\t\t\t\tnode = rg.node\n\t\t\t\t\tif gl := node.GroupLeader(); gl != _EMPTY_ && !rg.isMember(gl) {\n\t\t\t\t\t\tleaderNotPartOfGroup = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t}\n\n\t\t\t// Check if we should ignore all together.\n\t\t\tif node == nil {\n\t\t\t\t// We have been assigned but have not created a node yet. If we are a member return\n\t\t\t\t// our config and defaults for state and no cluster info.\n\t\t\t\tif isMember {\n\t\t\t\t\t// Since we access consumerAssignment, need js lock.\n\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\tresp.ConsumerInfo = &ConsumerInfo{\n\t\t\t\t\t\tStream: ca.Stream,\n\t\t\t\t\t\tName: ca.Name,\n\t\t\t\t\t\tCreated: ca.Created,\n\t\t\t\t\t\tConfig: setDynamicConsumerMetadata(ca.Config),\n\t\t\t\t\t\tTimeStamp: time.Now().UTC(),\n\t\t\t\t\t}\n\t\t\t\t\tb := s.jsonResponse(resp)\n\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), b)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// If we are a member and we have a group leader or we had a previous leader consider bailing out.\n\t\t\tif !node.Leaderless() || node.HadPreviousLeader() || (rg != nil && rg.Preferred != _EMPTY_ && rg.Preferred != ourID) {\n\t\t\t\tif leaderNotPartOfGroup {\n\t\t\t\t\tresp.Error = NewJSConsumerOfflineError()\n\t\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// If we are here we are a member and this is just a new consumer that does not have a (preferred) leader yet.\n\t\t\t// Will fall through and return what we have. All consumers can respond but this should be very rare\n\t\t\t// but makes more sense to clients when they try to create, get a consumer exists, and then do consumer info.\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif !acc.JetStreamEnabled() {\n\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(streamName)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tobs := mset.lookupConsumer(consumerName)\n\tif obs == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif obs.offlineReason != _EMPTY_ {\n\t\tresp.Error = NewJSConsumerOfflineReasonError(errors.New(obs.offlineReason))\n\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\treturn\n\t}\n\n\tif resp.ConsumerInfo = setDynamicConsumerInfoMetadata(obs.info()); resp.ConsumerInfo == nil {\n\t\t// This consumer returned nil which means it's closed. Respond with not found.\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to delete an Consumer.\nfunc (s *Server) jsConsumerDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tif s.JetStreamIsClustered() {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\tif !isEmptyRequest(msg) {\n\t\tresp.Error = NewJSNotEmptyRequestError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tstream := streamNameFromSubject(subject)\n\tconsumer := consumerNameFromSubject(subject)\n\n\tif s.JetStreamIsClustered() {\n\t\ts.jsClusteredConsumerDeleteRequest(ci, acc, stream, consumer, subject, reply, rmsg)\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tobs := mset.lookupConsumer(consumer)\n\tif obs == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif err := obs.delete(); err != nil {\n\t\tresp.Error = NewJSStreamGeneralError(err, Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tresp.Success = true\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// Request to pause or unpause a Consumer.\nfunc (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif c == nil || !s.JetStreamEnabled() {\n\t\treturn\n\t}\n\tci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)\n\tif err != nil {\n\t\ts.Warnf(badAPIRequestT, msg)\n\t\treturn\n\t}\n\n\tvar req JSApiConsumerPauseRequest\n\tvar resp = JSApiConsumerPauseResponse{ApiResponse: ApiResponse{Type: JSApiConsumerPauseResponseType}}\n\n\tif isJSONObjectOrArray(msg) {\n\t\tif err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Determine if we should proceed here when we are in clustered mode.\n\tisClustered := s.JetStreamIsClustered()\n\tjs, cc := s.getJetStreamCluster()\n\tif isClustered {\n\t\tif js == nil || cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif js.isLeaderless() {\n\t\t\tresp.Error = NewJSClusterNotAvailError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Make sure we are meta leader.\n\t\tif !s.JetStreamIsLeader() {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif errorOnRequiredApiLevel(hdr) {\n\t\tresp.Error = NewJSRequiredApiLevelError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif hasJS, doErr := acc.checkJetStream(); !hasJS {\n\t\tif doErr {\n\t\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t}\n\t\treturn\n\t}\n\n\tstream := streamNameFromSubject(subject)\n\tconsumer := consumerNameFromSubject(subject)\n\n\tif isClustered {\n\t\tjs.mu.RLock()\n\t\tsa := js.streamAssignment(acc.Name, stream)\n\t\tif sa == nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif sa.unsupported != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\t// Just let the request time out.\n\t\t\treturn\n\t\t}\n\n\t\tca, ok := sa.consumers[consumer]\n\t\tif !ok || ca == nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\tif ca.unsupported != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\t// Just let the request time out.\n\t\t\treturn\n\t\t}\n\n\t\tnca := *ca\n\t\tncfg := *ca.Config\n\t\tnca.Config = &ncfg\n\t\tmeta := cc.meta\n\t\tjs.mu.RUnlock()\n\t\tpauseUTC := req.PauseUntil.UTC()\n\t\tif !pauseUTC.IsZero() {\n\t\t\tnca.Config.PauseUntil = &pauseUTC\n\t\t} else {\n\t\t\tnca.Config.PauseUntil = nil\n\t\t}\n\n\t\t// Update asset version metadata due to updating pause/resume.\n\t\t// Only PauseUntil is updated above, so reuse config for both.\n\t\tsetStaticConsumerMetadata(nca.Config)\n\n\t\teca := encodeAddConsumerAssignment(&nca)\n\t\tmeta.Propose(eca)\n\n\t\tresp.PauseUntil = pauseUTC\n\t\tif resp.Paused = time.Now().Before(pauseUTC); resp.Paused {\n\t\t\tresp.PauseRemaining = time.Until(pauseUTC)\n\t\t}\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif mset.offlineReason != _EMPTY_ {\n\t\t// Just let the request time out.\n\t\treturn\n\t}\n\n\tobs := mset.lookupConsumer(consumer)\n\tif obs == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tif obs.offlineReason != _EMPTY_ {\n\t\t// Just let the request time out.\n\t\treturn\n\t}\n\n\tncfg := obs.cfg\n\tpauseUTC := req.PauseUntil.UTC()\n\tif !pauseUTC.IsZero() {\n\t\tncfg.PauseUntil = &pauseUTC\n\t} else {\n\t\tncfg.PauseUntil = nil\n\t}\n\n\t// Update asset version metadata due to updating pause/resume.\n\tsetStaticConsumerMetadata(&ncfg)\n\n\tif err := obs.updateConfig(&ncfg); err != nil {\n\t\t// The only type of error that should be returned here is from o.store,\n\t\t// so use a store failed error type.\n\t\tresp.Error = NewJSConsumerStoreFailedError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tresp.PauseUntil = pauseUTC\n\tif resp.Paused = time.Now().Before(pauseUTC); resp.Paused {\n\t\tresp.PauseRemaining = time.Until(pauseUTC)\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))\n}\n\n// sendJetStreamAPIAuditAdvisory will send the audit event for a given event.\nfunc (s *Server) sendJetStreamAPIAuditAdvisory(ci *ClientInfo, acc *Account, subject, request, response string) {\n\ts.publishAdvisory(acc, JSAuditAdvisory, JSAPIAudit{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSAPIAuditType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tServer: s.Name(),\n\t\tClient: ci.forAdvisory(),\n\t\tSubject: subject,\n\t\tRequest: request,\n\t\tResponse: response,\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t})\n}\n" }, { "path": "server/jetstream_batching.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/big\"\n\t\"path/filepath\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nvar (\n\t// Tracks the total inflight batches, across all streams and accounts that enable batching.\n\tglobalInflightAtomicBatches atomic.Int64\n\tglobalInflightFastBatches atomic.Int64\n)\n\ntype batching struct {\n\tmu sync.Mutex\n\tatomic map[string]*atomicBatch\n\tfast map[string]*fastBatch\n}\n\ntype atomicBatch struct {\n\ttimer *time.Timer // Inactivity timer for the batch.\n\tlseq uint64 // The highest sequence for this batch.\n\tstore StreamStore // Where the batch is staged before committing.\n}\n\ntype fastBatch struct {\n\ttimer *time.Timer // Inactivity timer for the batch.\n\tlseq uint64 // The highest sequence for this batch.\n\tsseq uint64 // Last persisted stream sequence.\n\tpseq uint64 // Last persisted batch sequence (is always lower or equal to lseq).\n\tfseq uint64 // Sequence of when we last sent a flow message (is always lower or equal to pseq).\n\tpending uint32 // Number of pending messages in the batch waiting to be persisted.\n\tackMessages uint16 // Ack will be sent every N messages.\n\tmaxAckMessages uint16 // Maximum ackMessages value the client allows.\n\treply string // The last reply subject seen when persisting a message.\n\tgapOk bool // Whether a gap is okay, if not, the batch would be rejected.\n\tcommit bool // If the batch is committed.\n}\n\n// newAtomicBatch creates an atomic batch publish object.\n// Lock should be held.\nfunc (batches *batching) newAtomicBatch(mset *stream, batchId string) (*atomicBatch, error) {\n\tstore, err := newBatchStore(mset, batchId)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tb := &atomicBatch{store: store}\n\tb.setupCleanupTimer(mset, batchId, batches)\n\treturn b, nil\n}\n\n// setupCleanupTimer sets up a timer to clean up the batch after a timeout.\nfunc (b *atomicBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {\n\t// Create a timer to clean up after timeout.\n\ttimeout := getCleanupTimeout(mset)\n\tb.timer = time.AfterFunc(timeout, func() {\n\t\tb.cleanup(batchId, batches)\n\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, BatchTimeout)\n\t})\n}\n\n// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.\n// Returns whether the timer was reset without it having expired before.\nfunc (b *atomicBatch) resetCleanupTimer(mset *stream) bool {\n\ttimeout := getCleanupTimeout(mset)\n\treturn b.timer.Reset(timeout)\n}\n\n// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.\nfunc (b *atomicBatch) cleanup(batchId string, batches *batching) {\n\tbatches.mu.Lock()\n\tdefer batches.mu.Unlock()\n\tb.cleanupLocked(batchId, batches)\n}\n\n// Lock should be held.\nfunc (b *atomicBatch) cleanupLocked(batchId string, batches *batching) {\n\tif b.timer == nil {\n\t\treturn\n\t}\n\tglobalInflightAtomicBatches.Add(-1)\n\tb.timer.Stop()\n\tb.store.Delete(true)\n\tdelete(batches.atomic, batchId)\n\t// Reset so that another invocation doesn't double-account.\n\tb.timer = nil\n}\n\n// Lock should be held.\nfunc (b *atomicBatch) stopLocked() {\n\tif b.timer == nil {\n\t\treturn\n\t}\n\tglobalInflightAtomicBatches.Add(-1)\n\tb.timer.Stop()\n\tb.store.Stop()\n\t// Reset so that another invocation doesn't double-account.\n\tb.timer = nil\n}\n\nfunc getBatchStoreDir(mset *stream, batchId string) (string, string) {\n\tmset.mu.RLock()\n\tjsa, name := mset.jsa, mset.cfg.Name\n\tmset.mu.RUnlock()\n\n\tjsa.mu.RLock()\n\tsd := jsa.storeDir\n\tjsa.mu.RUnlock()\n\n\tbname := getHash(batchId)\n\treturn bname, filepath.Join(sd, streamsDir, name, batchesDir, bname)\n}\n\nfunc newBatchStore(mset *stream, batchId string) (StreamStore, error) {\n\tmset.mu.RLock()\n\treplicas, storage := mset.cfg.Replicas, mset.cfg.Storage\n\tmset.mu.RUnlock()\n\n\tif replicas == 1 && storage == FileStorage {\n\t\tbname, storeDir := getBatchStoreDir(mset, batchId)\n\t\tfcfg := FileStoreConfig{AsyncFlush: true, BlockSize: defaultLargeBlockSize, StoreDir: storeDir}\n\t\ts := mset.srv\n\t\tprf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name)\n\t\tif prf != nil {\n\t\t\t// We are encrypted here, fill in correct cipher selection.\n\t\t\tfcfg.Cipher = s.getOpts().JetStreamCipher\n\t\t}\n\t\toldprf := s.jsKeyGen(s.getOpts().JetStreamOldKey, mset.acc.Name)\n\t\tcfg := StreamConfig{Name: bname, Storage: FileStorage}\n\t\treturn newFileStoreWithCreated(fcfg, cfg, time.Time{}, prf, oldprf)\n\t}\n\treturn newMemStore(&StreamConfig{Name: _EMPTY_, Storage: MemoryStorage})\n}\n\n// readyForCommit indicates the batch is ready to be committed.\n// If the timer has already cleaned up the batch, we can't commit.\n// Otherwise, we ensure the timer does not clean up the batch in the meantime.\n// Lock should be held.\nfunc (b *atomicBatch) readyForCommit() *BatchAbandonReason {\n\tif !b.timer.Stop() {\n\t\treturn &BatchTimeout\n\t}\n\tif b.store.FlushAllPending() != nil {\n\t\treturn &BatchIncomplete\n\t}\n\treturn nil\n}\n\n// newFastBatch creates a fast batch publish object.\n// Lock should be held.\nfunc (batches *batching) newFastBatch(mset *stream, batchId string, gapOk bool, maxAckMessages uint16) *fastBatch {\n\tb := &fastBatch{gapOk: gapOk, maxAckMessages: maxAckMessages}\n\tbatches.fastBatchInit(b)\n\tb.setupCleanupTimer(mset, batchId, batches)\n\treturn b\n}\n\n// fastBatchInit (re)initializes the ackMessages field for a fast batch.\n// Lock should be held.\nfunc (batches *batching) fastBatchInit(b *fastBatch) {\n\t// If it's the first batch, just allow what the client wants, otherwise we'll\n\t// need to coordinate and slowly ramp up this publisher.\n\t// TODO(mvv): fast ingest's initial flow value improvements?\n\tackMessages := min(500, b.maxAckMessages)\n\tif len(batches.fast) > 1 {\n\t\tackMessages = 1\n\t}\n\tb.ackMessages = ackMessages\n}\n\n// fastBatchReset resets the fast batch to an empty state and sends a flow control message.\n// Lock should be held.\nfunc (batches *batching) fastBatchReset(mset *stream, batchId string, b *fastBatch) {\n\t// If the timer already stopped before we could commit, we clean it up.\n\tif b.timer == nil || (!b.commit && !b.timer.Stop()) {\n\t\tb.cleanupLocked(batchId, batches)\n\t\treturn\n\t}\n\t// Otherwise, reset the state.\n\tbatches.fastBatchInit(b)\n\tb.timer.Reset(getCleanupTimeout(mset))\n\tb.commit = false\n\tb.pending = 0\n\tb.fseq, b.lseq = b.pseq, b.pseq\n\tb.sendFlowControl(b.fseq, mset, b.reply)\n}\n\n// fastBatchRegisterSequences registers the highest stored batch and stream sequence and returns\n// whether a PubAck should be sent if the batch has been committed.\n// If this is called on a follower, it only registers the highest stream and persisted batch sequences.\n// Lock should be held.\nfunc (batches *batching) fastBatchRegisterSequences(mset *stream, reply string, streamSeq uint64, isLeader bool, batch *FastBatch) bool {\n\tb, ok := batches.fast[batch.id]\n\tif !ok || !isLeader {\n\t\t// If this batch has committed, we can clean it up.\n\t\tif batch.commit {\n\t\t\tif b != nil {\n\t\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\t}\n\t\t\treturn false\n\t\t}\n\t\t// Otherwise, even as a follower, we record the latest state of this batch.\n\t\tif b == nil || !b.resetCleanupTimer(mset) {\n\t\t\tif b != nil {\n\t\t\t\t// The timer couldn't be reset, this means the timer already runs and is likely\n\t\t\t\t// waiting to acquire the lock. We reset the timer here so it doesn't clean up\n\t\t\t\t// this batch that we're about to overwrite.\n\t\t\t\tb.timer = nil\n\t\t\t} else {\n\t\t\t\t// If this is a new batch for us, even though we're a follower, we still need\n\t\t\t\t// to account toward the global inflight limit.\n\t\t\t\tglobalInflightFastBatches.Add(1)\n\t\t\t}\n\t\t\t// We'll need a copy as we'll use it as a key and later for cleanup.\n\t\t\tbatchId := copyString(batch.id)\n\t\t\tb = batches.newFastBatch(mset, batchId, batch.gapOk, batch.flow)\n\t\t\tif batches.fast == nil {\n\t\t\t\tbatches.fast = make(map[string]*fastBatch, 1)\n\t\t\t}\n\t\t\tbatches.fast[batchId] = b\n\t\t}\n\t\tb.sseq = streamSeq\n\t\tb.pseq, b.lseq = batch.seq, batch.seq\n\t\tb.reply = reply\n\t\treturn false\n\t}\n\tb.reply = reply\n\tif b.pending > 0 {\n\t\tb.pending--\n\t}\n\tb.sseq = streamSeq\n\t// Store last persisted batch sequence.\n\t// If we have no remaining pending writes, we might have had duplicate messages\n\t// and need to send additional flow control messages.\n\tvar skipped bool\n\tif b.pending == 0 {\n\t\tskipped = true\n\t\tb.pseq = b.lseq\n\t} else {\n\t\tb.pseq = batch.seq\n\t}\n\t// If the PubAck needs to be sent now as a result of a commit.\n\tif b.lseq == b.pseq && b.commit {\n\t\tb.cleanupLocked(batch.id, batches)\n\t\t// If we skipped ahead due to duplicate messages, send the PubAck with the highest sequence.\n\t\tif skipped {\n\t\t\tvar buf [256]byte\n\t\t\tpubAck := append(buf[:0], mset.pubAck...)\n\t\t\tresponse := append(pubAck, strconv.FormatUint(b.sseq, 10)...)\n\t\t\tresponse = append(response, fmt.Sprintf(\",\\\"batch\\\":%q,\\\"count\\\":%d}\", batch.id, b.lseq)...)\n\t\t\tif len(reply) > 0 {\n\t\t\t\tmset.outq.sendMsg(reply, response)\n\t\t\t}\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\tb.checkFlowControl(mset, reply, batches)\n\treturn false\n}\n\n// checkFlowControl checks whether a flow control message should be sent.\n// If so, it updates the flow values to speed up or slow down the publisher if needed.\n// Returns whether a flow control message was sent.\n// Lock should be held.\nfunc (b *fastBatch) checkFlowControl(mset *stream, reply string, batches *batching) bool {\n\tam := uint64(b.ackMessages)\n\tif b.pseq < b.fseq+am {\n\t\treturn false\n\t}\n\t// Instead of sending multiple flow control messages, skip ahead to only send the last.\n\tsteps := (b.pseq - b.fseq) / am\n\tb.fseq += steps * am\n\n\t// TODO(mvv): fast ingest's dynamic flow value improvements?\n\t// This is currently just a simple value to have a working version. Should take average\n\t// message sizes into account and compare how much this client is contributing to the\n\t// ingest IPQ total size and messages and have publishers share based on that.\n\tmaxAckMessages := uint16(500 / len(batches.fast))\n\tif maxAckMessages < 1 {\n\t\tmaxAckMessages = 1\n\t}\n\t// Limit to the client's allowed maximum.\n\tif maxAckMessages > b.maxAckMessages {\n\t\tmaxAckMessages = b.maxAckMessages\n\t}\n\n\tif b.ackMessages < maxAckMessages {\n\t\t// Ramp up.\n\t\tb.ackMessages *= 2\n\t\tif b.ackMessages > maxAckMessages {\n\t\t\tb.ackMessages = maxAckMessages\n\t\t}\n\t} else if b.ackMessages > maxAckMessages {\n\t\t// Slow down.\n\t\tb.ackMessages /= 2\n\t\tif b.ackMessages <= maxAckMessages {\n\t\t\tb.ackMessages = maxAckMessages\n\t\t}\n\t}\n\n\t// Finally, send the flow control message.\n\tb.sendFlowControl(b.fseq, mset, reply)\n\treturn true\n}\n\n// sendFlowControl sends a fast batch flow control message for the current highest sequence.\n// Lock should be held.\nfunc (b *fastBatch) sendFlowControl(batchSeq uint64, mset *stream, reply string) {\n\tif len(reply) == 0 {\n\t\treturn\n\t}\n\tresponse := BatchFlowAck{Sequence: batchSeq, Messages: b.ackMessages}.MarshalJSON()\n\tmset.outq.sendMsg(reply, response)\n}\n\n// fastBatchCommit ends the batch and commits the data up to that point. If all messages\n// have already been persisted, a PubAck is sent immediately. Otherwise, it will be sent\n// after the last message has been persisted.\n// Lock should be held.\nfunc (batches *batching) fastBatchCommit(b *fastBatch, batchId string, mset *stream, reply string) bool {\n\t// Either we commit now, or we clean up later, so stop the timer.\n\tif b.timer == nil || (!b.commit && !b.timer.Stop()) {\n\t\t// Shouldn't be possible for the timer to already be stopped if we haven't committed yet,\n\t\t// since we pre-check being able to reset the timer. But guard against it anyhow.\n\t\treturn true\n\t}\n\t// Mark that this batch commits.\n\tb.commit = true\n\t// If the whole batch has been persisted, we can respond with the PubAck now.\n\tif b.lseq == b.pseq {\n\t\tb.cleanupLocked(batchId, batches)\n\t\tvar buf [256]byte\n\t\tpubAck := append(buf[:0], mset.pubAck...)\n\t\tresponse := append(pubAck, strconv.FormatUint(b.sseq, 10)...)\n\t\tresponse = append(response, fmt.Sprintf(\",\\\"batch\\\":%q,\\\"count\\\":%d}\", batchId, b.lseq)...)\n\t\tif len(reply) > 0 {\n\t\t\tmset.outq.sendMsg(reply, response)\n\t\t}\n\t\treturn true\n\t}\n\t// Otherwise, we need to wait and the PubAck will be sent when the last message is persisted.\n\treturn false\n}\n\n// setupCleanupTimer sets up a timer to clean up the batch after a timeout.\nfunc (b *fastBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {\n\t// Create a timer to clean up after timeout.\n\ttimeout := getCleanupTimeout(mset)\n\tb.timer = time.AfterFunc(timeout, func() {\n\t\tb.cleanup(batchId, batches)\n\t\t// Only send the advisory if we're the leader. (Since we do the tracking on followers too)\n\t\tif mset.IsLeader() {\n\t\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, BatchTimeout)\n\t\t}\n\t})\n}\n\n// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.\n// Returns whether the timer was reset without it having expired before.\nfunc (b *fastBatch) resetCleanupTimer(mset *stream) bool {\n\tif b.commit {\n\t\treturn true\n\t}\n\tif b.timer == nil {\n\t\treturn false\n\t}\n\ttimeout := getCleanupTimeout(mset)\n\treturn b.timer.Reset(timeout)\n}\n\n// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.\nfunc (b *fastBatch) cleanup(batchId string, batches *batching) {\n\tbatches.mu.Lock()\n\tdefer batches.mu.Unlock()\n\tb.cleanupLocked(batchId, batches)\n}\n\n// Lock should be held.\nfunc (b *fastBatch) cleanupLocked(batchId string, batches *batching) {\n\t// If the timer is nil, it means this batch has been replaced with a new one.\n\t// This can happen on a follower depending on timing.\n\tif b.timer == nil {\n\t\treturn\n\t}\n\tglobalInflightFastBatches.Add(-1)\n\tb.timer.Stop()\n\tdelete(batches.fast, batchId)\n\t// Reset so that another invocation doesn't double-account.\n\tb.timer = nil\n}\n\n// getCleanupTimeout returns the timeout for the batch, taking into account the server's limits.\nfunc getCleanupTimeout(mset *stream) time.Duration {\n\ttimeout := streamMaxBatchTimeout\n\tif maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 {\n\t\ttimeout = maxBatchTimeout\n\t}\n\treturn timeout\n}\n\n// batchStagedDiff stages all changes for consistency checks until commit.\ntype batchStagedDiff struct {\n\tmsgIds map[string]struct{}\n\tcounter map[string]*msgCounterRunningTotal\n\tinflight map[string]*inflightSubjectRunningTotal\n\texpectedPerSubject map[string]*batchExpectedPerSubject\n}\n\ntype batchExpectedPerSubject struct {\n\tsseq uint64 // Stream sequence.\n\tclseq uint64 // Clustered proposal sequence.\n}\n\nfunc (diff *batchStagedDiff) commit(mset *stream) {\n\tif len(diff.msgIds) > 0 {\n\t\tts := time.Now().UnixNano()\n\t\tmset.ddMu.Lock()\n\t\tfor msgId := range diff.msgIds {\n\t\t\t// We stage with zero, and will update in processJetStreamMsg once we know the sequence.\n\t\t\tmset.storeMsgIdLocked(&ddentry{msgId, 0, ts})\n\t\t}\n\t\tmset.ddMu.Unlock()\n\t}\n\n\t// Store running totals for counters, we could have multiple counter increments proposed, but not applied yet.\n\tif len(diff.counter) > 0 {\n\t\tif mset.clusteredCounterTotal == nil {\n\t\t\tmset.clusteredCounterTotal = make(map[string]*msgCounterRunningTotal, len(diff.counter))\n\t\t}\n\t\tfor k, c := range diff.counter {\n\t\t\tmset.clusteredCounterTotal[k] = c\n\t\t}\n\t}\n\n\t// Track inflight.\n\tif len(diff.inflight) > 0 {\n\t\tif mset.inflight == nil {\n\t\t\tmset.inflight = make(map[string]*inflightSubjectRunningTotal, len(diff.inflight))\n\t\t}\n\t\tfor subj, i := range diff.inflight {\n\t\t\tif c, ok := mset.inflight[subj]; ok {\n\t\t\t\tc.bytes += i.bytes\n\t\t\t\tc.ops += i.ops\n\t\t\t} else {\n\t\t\t\tmset.inflight[subj] = i\n\t\t\t}\n\t\t}\n\t}\n\n\t// Track sequence and subject.\n\tif len(diff.expectedPerSubject) > 0 {\n\t\tif mset.expectedPerSubjectSequence == nil {\n\t\t\tmset.expectedPerSubjectSequence = make(map[uint64]string, len(diff.expectedPerSubject))\n\t\t}\n\t\tif mset.expectedPerSubjectInProcess == nil {\n\t\t\tmset.expectedPerSubjectInProcess = make(map[string]struct{}, len(diff.expectedPerSubject))\n\t\t}\n\t\tfor subj, e := range diff.expectedPerSubject {\n\t\t\tmset.expectedPerSubjectSequence[e.clseq] = subj\n\t\t\tmset.expectedPerSubjectInProcess[subj] = struct{}{}\n\t\t}\n\t}\n}\n\ntype batchApply struct {\n\tmu sync.Mutex\n\tid string // ID of the current batch.\n\tcount uint64 // Number of entries in the batch, for consistency checks.\n\tentries []*CommittedEntry // Previous entries that are part of this batch.\n\tentryStart int // The index into an entry indicating the first message of the batch.\n\tmaxApplied uint64 // Applied value before the entry containing the first message of the batch.\n}\n\n// clearBatchStateLocked clears in-memory apply-batch-related state.\n// batch.mu lock should be held.\nfunc (batch *batchApply) clearBatchStateLocked() {\n\tbatch.id = _EMPTY_\n\tbatch.count = 0\n\tbatch.entries = nil\n\tbatch.entryStart = 0\n\tbatch.maxApplied = 0\n}\n\n// rejectBatchStateLocked rejects the batch and clears in-memory apply-batch-related state.\n// Corrects mset.clfs to take the failed batch into account.\n// batch.mu lock should be held.\nfunc (batch *batchApply) rejectBatchStateLocked(mset *stream) {\n\tmset.clMu.Lock()\n\tmset.clfs += batch.count\n\tmset.clMu.Unlock()\n\t// We're rejecting the batch, so all entries need to be returned to the pool.\n\tfor _, bce := range batch.entries {\n\t\tbce.ReturnToPool()\n\t}\n\tbatch.clearBatchStateLocked()\n}\n\nfunc (batch *batchApply) rejectBatchState(mset *stream) {\n\tbatch.mu.Lock()\n\tdefer batch.mu.Unlock()\n\tbatch.rejectBatchStateLocked(mset)\n}\n\n// checkMsgHeadersPreClusteredProposal checks the message for expected/consistency headers.\n// mset.mu lock must NOT be held or used.\n// mset.clMu lock must be held.\nfunc checkMsgHeadersPreClusteredProposal(\n\tdiff *batchStagedDiff, mset *stream, subject string, hdr []byte, msg []byte, sourced bool, name string,\n\tjsa *jsAccount, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules bool,\n\tdiscard DiscardPolicy, discardNewPer bool, maxMsgSize int, maxMsgs int64, maxMsgsPer int64, maxBytes int64,\n) ([]byte, []byte, uint64, *ApiError, error) {\n\tvar incr *big.Int\n\n\t// Some header checks must be checked pre proposal.\n\tif len(hdr) > 0 {\n\t\t// Since we encode header len as u16 make sure we do not exceed.\n\t\t// Again this works if it goes through but better to be pre-emptive.\n\t\tif len(hdr) > math.MaxUint16 {\n\t\t\terr := fmt.Errorf(\"JetStream header size exceeds limits for '%s > %s'\", jsa.acc().Name, mset.cfg.Name)\n\t\t\treturn hdr, msg, 0, NewJSStreamHeaderExceedsMaximumError(), err\n\t\t}\n\t\t// Counter increments.\n\t\t// Only supported on counter streams, and payload must be empty (if not coming from a source).\n\t\tvar ok bool\n\t\tif incr, ok = getMessageIncr(hdr); !ok {\n\t\t\tapiErr := NewJSMessageIncrInvalidError()\n\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t} else if incr != nil && !sourced {\n\t\t\t// Only do checks if the message isn't sourced. Otherwise, we need to store verbatim.\n\t\t\tif !allowMsgCounter {\n\t\t\t\tapiErr := NewJSMessageIncrDisabledError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else if len(msg) > 0 {\n\t\t\t\tapiErr := NewJSMessageIncrPayloadError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else {\n\t\t\t\t// Check for incompatible headers.\n\t\t\t\tvar doErr bool\n\t\t\t\tif getRollup(hdr) != _EMPTY_ ||\n\t\t\t\t\tgetExpectedStream(hdr) != _EMPTY_ ||\n\t\t\t\t\tgetExpectedLastMsgId(hdr) != _EMPTY_ ||\n\t\t\t\t\tgetExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ {\n\t\t\t\t\tdoErr = true\n\t\t\t\t} else if _, ok = getExpectedLastSeq(hdr); ok {\n\t\t\t\t\tdoErr = true\n\t\t\t\t} else if _, ok = getExpectedLastSeqPerSubject(hdr); ok {\n\t\t\t\t\tdoErr = true\n\t\t\t\t}\n\n\t\t\t\tif doErr {\n\t\t\t\t\tapiErr := NewJSMessageIncrInvalidError()\n\t\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Expected stream name can also be pre-checked.\n\t\tif sname := getExpectedStream(hdr); sname != _EMPTY_ && sname != name {\n\t\t\treturn hdr, msg, 0, NewJSStreamNotMatchError(), errStreamMismatch\n\t\t}\n\t\t// TTL'd messages are rejected entirely if TTLs are not enabled on the stream, or if the TTL is invalid.\n\t\tif ttl, err := getMessageTTL(hdr); !sourced && (ttl != 0 || err != nil) {\n\t\t\tif !allowTTL {\n\t\t\t\treturn hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled\n\t\t\t} else if err != nil {\n\t\t\t\treturn hdr, msg, 0, NewJSMessageTTLInvalidError(), err\n\t\t\t}\n\t\t}\n\t\t// Check for MsgIds here at the cluster level to avoid excessive CLFS accounting.\n\t\t// Will help during restarts.\n\t\tif msgId := getMsgId(hdr); msgId != _EMPTY_ {\n\t\t\t// Dedupe if staged.\n\t\t\tif _, ok = diff.msgIds[msgId]; ok {\n\t\t\t\treturn hdr, msg, 0, NewJSAtomicPublishContainsDuplicateMessageError(), errMsgIdDuplicate\n\t\t\t}\n\t\t\tmset.ddMu.Lock()\n\t\t\tif dde := mset.checkMsgId(msgId); dde != nil {\n\t\t\t\tseq := dde.seq\n\t\t\t\tmset.ddMu.Unlock()\n\t\t\t\t// Should not return an invalid sequence, in that case error.\n\t\t\t\tif seq > 0 {\n\t\t\t\t\treturn hdr, msg, seq, NewJSAtomicPublishContainsDuplicateMessageError(), errMsgIdDuplicate\n\t\t\t\t} else {\n\t\t\t\t\treturn hdr, msg, 0, NewJSStreamDuplicateMessageConflictError(), errMsgIdDuplicate\n\t\t\t\t}\n\t\t\t}\n\t\t\tif diff.msgIds == nil {\n\t\t\t\tdiff.msgIds = map[string]struct{}{msgId: {}}\n\t\t\t} else {\n\t\t\t\tdiff.msgIds[msgId] = struct{}{}\n\t\t\t}\n\t\t\tmset.ddMu.Unlock()\n\t\t}\n\t}\n\n\t// Apply increment for counter.\n\t// But only if it's allowed for this stream. This can happen when we store verbatim for a sourced stream.\n\tif incr == nil && allowMsgCounter {\n\t\tapiErr := NewJSMessageIncrMissingError()\n\t\treturn hdr, msg, 0, apiErr, apiErr\n\t}\n\tif incr != nil && allowMsgCounter {\n\t\tvar initial big.Int\n\t\tvar sources CounterSources\n\n\t\t// If we've got a running total, update that, since we have inflight proposals updating the same counter.\n\t\tvar ok bool\n\t\tvar counter *msgCounterRunningTotal\n\t\tif counter, ok = diff.counter[subject]; ok {\n\t\t\tinitial = *counter.total\n\t\t\tsources = counter.sources\n\t\t} else if counter, ok = mset.clusteredCounterTotal[subject]; ok {\n\t\t\tinitial = *counter.total\n\t\t\tsources = counter.sources\n\t\t\t// Make an explicit copy to separate the staged data from what's committed.\n\t\t\t// Don't need to initialize all values, they'll be overwritten later.\n\t\t\tcounter = &msgCounterRunningTotal{ops: counter.ops}\n\t\t} else {\n\t\t\t// Load last message, and store as inflight running total.\n\t\t\tvar smv StoreMsg\n\t\t\tsm, err := mset.store.LoadLastMsg(subject, &smv)\n\t\t\tif err == nil && sm != nil {\n\t\t\t\tvar val CounterValue\n\t\t\t\t// Return an error if the counter is broken somehow.\n\t\t\t\tif json.Unmarshal(sm.msg, &val) != nil {\n\t\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t\t}\n\t\t\t\tif ncs := sliceHeader(JSMessageCounterSources, sm.hdr); len(ncs) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(ncs, &sources); err != nil {\n\t\t\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tinitial.SetString(val.Value, 10)\n\t\t\t}\n\t\t}\n\t\tsrchdr := sliceHeader(JSStreamSource, hdr)\n\t\tif len(srchdr) > 0 {\n\t\t\t// This is a sourced message, so we can't apply Nats-Incr but\n\t\t\t// instead should just update the source count header.\n\t\t\tfields := strings.Split(string(srchdr), \" \")\n\t\t\torigStream := fields[0]\n\t\t\torigSubj := subject\n\t\t\tif len(fields) >= 5 {\n\t\t\t\torigSubj = fields[4]\n\t\t\t}\n\t\t\tvar val CounterValue\n\t\t\tif json.Unmarshal(msg, &val) != nil {\n\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t}\n\t\t\tvar sourced big.Int\n\t\t\tsourced.SetString(val.Value, 10)\n\t\t\tif sources == nil {\n\t\t\t\tsources = map[string]map[string]string{}\n\t\t\t}\n\t\t\tif _, ok = sources[origStream]; !ok {\n\t\t\t\tsources[origStream] = map[string]string{}\n\t\t\t}\n\t\t\tprevVal := sources[origStream][origSubj]\n\t\t\tsources[origStream][origSubj] = sourced.String()\n\t\t\t// We will also replace the Nats-Incr header with the diff\n\t\t\t// between our last value from this source and this one, so\n\t\t\t// that the arithmetic is always correct.\n\t\t\tvar previous big.Int\n\t\t\tprevious.SetString(prevVal, 10)\n\t\t\tincr.Sub(&sourced, &previous)\n\t\t\thdr = setHeader(JSMessageIncr, incr.String(), hdr)\n\t\t}\n\t\t// Now make the change.\n\t\tinitial.Add(&initial, incr)\n\t\t// Generate the new payload.\n\t\tvar _msg [128]byte\n\t\tmsg = fmt.Appendf(_msg[:0], \"{%q:%q}\", \"val\", initial.String())\n\t\t// Write the updated source count headers.\n\t\tif len(sources) > 0 {\n\t\t\tnhdr, err := json.Marshal(sources)\n\t\t\tif err != nil {\n\t\t\t\treturn hdr, msg, 0, NewJSMessageCounterBrokenError(), err\n\t\t\t}\n\t\t\thdr = setHeader(JSMessageCounterSources, string(nhdr), hdr)\n\t\t}\n\n\t\t// Check to see if we are over the max msg size.\n\t\tmaxSize := int64(mset.srv.getOpts().MaxPayload)\n\t\tif maxMsgSize >= 0 && int64(maxMsgSize) < maxSize {\n\t\t\tmaxSize = int64(maxMsgSize)\n\t\t}\n\t\thdrLen, msgLen := int64(len(hdr)), int64(len(msg))\n\t\t// Subtract to prevent against overflows.\n\t\tif hdrLen > maxSize || msgLen > maxSize-hdrLen {\n\t\t\treturn hdr, msg, 0, NewJSStreamMessageExceedsMaximumError(), ErrMaxPayload\n\t\t}\n\n\t\t// Keep the in-memory counters up-to-date.\n\t\tif counter == nil {\n\t\t\tcounter = &msgCounterRunningTotal{}\n\t\t}\n\t\tcounter.total = &initial\n\t\tcounter.sources = sources\n\t\tcounter.ops++\n\t\tif diff.counter == nil {\n\t\t\tdiff.counter = map[string]*msgCounterRunningTotal{subject: counter}\n\t\t} else {\n\t\t\tdiff.counter[subject] = counter\n\t\t}\n\t}\n\n\tif len(hdr) > 0 {\n\t\t// Expected last sequence.\n\t\tif seq, exists := getExpectedLastSeq(hdr); exists && seq != mset.clseq-mset.clfs {\n\t\t\tmlseq := mset.clseq - mset.clfs\n\t\t\terr := fmt.Errorf(\"last sequence mismatch: %d vs %d\", seq, mlseq)\n\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceError(mlseq), err\n\t\t} else if exists && len(diff.inflight) > 0 {\n\t\t\t// Only the first message in a batch can contain an expected last sequence.\n\t\t\terr := fmt.Errorf(\"last sequence mismatch\")\n\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err\n\t\t}\n\n\t\t// Expected last sequence per subject.\n\t\tif seq, exists := getExpectedLastSeqPerSubject(hdr); exists {\n\t\t\t// Allow override of the subject used for the check.\n\t\t\tseqSubj := subject\n\t\t\tif optSubj := getExpectedLastSeqPerSubjectForSubject(hdr); optSubj != _EMPTY_ {\n\t\t\t\tseqSubj = optSubj\n\t\t\t}\n\n\t\t\t// The subject is already written to in this batch, we can't allow\n\t\t\t// expected checks since they would be incorrect.\n\t\t\tif _, ok := diff.inflight[seqSubj]; ok {\n\t\t\t\terr := errors.New(\"last sequence by subject mismatch\")\n\t\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err\n\t\t\t}\n\n\t\t\t// If the subject is already in process, block as otherwise we could have\n\t\t\t// multiple messages inflight with the same subject.\n\t\t\tif _, found := mset.expectedPerSubjectInProcess[seqSubj]; found {\n\t\t\t\terr := errors.New(\"last sequence by subject mismatch\")\n\t\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err\n\t\t\t}\n\n\t\t\t// If the subject is already in process but without expected headers, block as we would have\n\t\t\t// multiple messages inflight with the same subject.\n\t\t\tif _, ok := mset.inflight[seqSubj]; ok {\n\t\t\t\terr := errors.New(\"last sequence by subject mismatch\")\n\t\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceConstantError(), err\n\t\t\t}\n\n\t\t\t// If we've already done an expected-check on this subject, use the cached result.\n\t\t\tif e, ok := diff.expectedPerSubject[seqSubj]; ok {\n\t\t\t\tif e.sseq != seq {\n\t\t\t\t\terr := fmt.Errorf(\"last sequence by subject mismatch: %d vs %d\", seq, e.sseq)\n\t\t\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceError(e.sseq), err\n\t\t\t\t}\n\t\t\t\te.clseq = mset.clseq\n\t\t\t} else {\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tvar fseq uint64\n\t\t\t\tsm, err := mset.store.LoadLastMsg(seqSubj, &smv)\n\t\t\t\tif sm != nil {\n\t\t\t\t\tfseq = sm.seq\n\t\t\t\t}\n\t\t\t\tif err == ErrStoreMsgNotFound && seq == 0 {\n\t\t\t\t\tfseq, err = 0, nil\n\t\t\t\t}\n\t\t\t\tif err != nil || fseq != seq {\n\t\t\t\t\terr = fmt.Errorf(\"last sequence by subject mismatch: %d vs %d\", seq, fseq)\n\t\t\t\t\treturn hdr, msg, 0, NewJSStreamWrongLastSequenceError(fseq), err\n\t\t\t\t}\n\n\t\t\t\te = &batchExpectedPerSubject{sseq: fseq, clseq: mset.clseq}\n\t\t\t\tif diff.expectedPerSubject == nil {\n\t\t\t\t\tdiff.expectedPerSubject = map[string]*batchExpectedPerSubject{seqSubj: e}\n\t\t\t\t} else {\n\t\t\t\t\tdiff.expectedPerSubject[seqSubj] = e\n\t\t\t\t}\n\t\t\t}\n\t\t} else if getExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ {\n\t\t\tapiErr := NewJSStreamExpectedLastSeqPerSubjectInvalidError()\n\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t}\n\n\t\t// Message scheduling.\n\t\tif schedule, ok := getMessageSchedule(hdr); !ok {\n\t\t\tapiErr := NewJSMessageSchedulesPatternInvalidError()\n\t\t\tif !allowMsgSchedules {\n\t\t\t\tapiErr = NewJSMessageSchedulesDisabledError()\n\t\t\t}\n\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t} else if !schedule.IsZero() {\n\t\t\tif !allowMsgSchedules {\n\t\t\t\tapiErr := NewJSMessageSchedulesDisabledError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else if scheduleTtl, ok := getMessageScheduleTTL(hdr); !ok {\n\t\t\t\tapiErr := NewJSMessageSchedulesTTLInvalidError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else if scheduleTtl != _EMPTY_ && !allowTTL {\n\t\t\t\treturn hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled\n\t\t\t} else if scheduleTarget := getMessageScheduleTarget(hdr); scheduleTarget == _EMPTY_ ||\n\t\t\t\t!IsValidPublishSubject(scheduleTarget) || SubjectsCollide(scheduleTarget, subject) {\n\t\t\t\tapiErr := NewJSMessageSchedulesTargetInvalidError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else if scheduleSource := getMessageScheduleSource(hdr); scheduleSource != _EMPTY_ &&\n\t\t\t\t(scheduleSource == scheduleTarget || scheduleSource == subject || !IsValidPublishSubject(scheduleSource)) {\n\t\t\t\tapiErr := NewJSMessageSchedulesSourceInvalidError()\n\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t} else {\n\t\t\t\tmset.cfgMu.RLock()\n\t\t\t\tmatch := slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {\n\t\t\t\t\treturn SubjectsCollide(subj, scheduleTarget)\n\t\t\t\t})\n\t\t\t\tmset.cfgMu.RUnlock()\n\t\t\t\tif !match {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesTargetInvalidError()\n\t\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t\t}\n\n\t\t\t\t// Add a rollup sub header if it doesn't already exist.\n\t\t\t\t// Otherwise, it must exist already as a rollup on the subject.\n\t\t\t\tif rollup := getRollup(hdr); rollup == _EMPTY_ {\n\t\t\t\t\thdr = genHeader(hdr, JSMsgRollup, JSMsgRollupSubject)\n\t\t\t\t} else if rollup != JSMsgRollupSubject {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesRollupInvalidError()\n\t\t\t\t\treturn hdr, msg, 0, apiErr, apiErr\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Check for any rollups.\n\t\tif rollup := getRollup(hdr); rollup != _EMPTY_ {\n\t\t\tif !allowRollup || denyPurge {\n\t\t\t\terr := errors.New(\"rollup not permitted\")\n\t\t\t\treturn hdr, msg, 0, NewJSStreamRollupFailedError(err), err\n\t\t\t}\n\t\t\tswitch rollup {\n\t\t\tcase JSMsgRollupSubject:\n\t\t\t\t// Rolling up the subject is only allowed if the first occurrence of this subject in the batch.\n\t\t\t\tif _, ok := diff.inflight[subject]; ok {\n\t\t\t\t\terr := errors.New(\"batch rollup sub invalid\")\n\t\t\t\t\treturn hdr, msg, 0, NewJSStreamRollupFailedError(err), err\n\t\t\t\t}\n\t\t\tcase JSMsgRollupAll:\n\t\t\t\t// Rolling up the whole stream is only allowed if this is the first message of the batch.\n\t\t\t\tif len(diff.inflight) > 0 {\n\t\t\t\t\terr := errors.New(\"batch rollup all invalid\")\n\t\t\t\t\treturn hdr, msg, 0, NewJSStreamRollupFailedError(err), err\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\terr := fmt.Errorf(\"rollup value invalid: %q\", rollup)\n\t\t\t\treturn hdr, msg, 0, NewJSStreamRollupFailedError(err), err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Track inflight.\n\t// Store the subject to ensure other messages in this batch using\n\t// an expected check or rollup on the same subject fail.\n\tif diff.inflight == nil {\n\t\tdiff.inflight = make(map[string]*inflightSubjectRunningTotal, 1)\n\t}\n\tvar sz uint64\n\tif mset.store.Type() == FileStorage {\n\t\tsz = fileStoreMsgSizeRaw(len(subject), len(hdr), len(msg))\n\t} else {\n\t\tsz = memStoreMsgSizeRaw(len(subject), len(hdr), len(msg))\n\t}\n\tvar (\n\t\ti *inflightSubjectRunningTotal\n\t\tok bool\n\t\terr error\n\t)\n\tif i, ok = diff.inflight[subject]; ok {\n\t\ti.bytes += sz\n\t\ti.ops++\n\t} else {\n\t\ti = &inflightSubjectRunningTotal{bytes: sz, ops: 1}\n\t\tdiff.inflight[subject] = i\n\t}\n\n\t// Check if we have discard new with max msgs or bytes.\n\t// We need to deny here otherwise we'd need to bump CLFS, and it could succeed on some\n\t// peers and not others depending on consumer ack state (if interest policy).\n\t// So we deny here, if we allow that means we know it would succeed on every peer.\n\tif discard == DiscardNew {\n\t\tif maxMsgs > 0 || maxBytes > 0 {\n\t\t\t// Track usual max msgs/bytes thresholds for DiscardNew.\n\t\t\tvar state StreamState\n\t\t\tmset.store.FastState(&state)\n\n\t\t\ttotalMsgs := state.Msgs\n\t\t\ttotalBytes := state.Bytes\n\t\t\tfor _, i = range mset.inflight {\n\t\t\t\ttotalMsgs += i.ops\n\t\t\t\ttotalBytes += i.bytes\n\t\t\t}\n\t\t\tfor _, i = range diff.inflight {\n\t\t\t\ttotalMsgs += i.ops\n\t\t\t\ttotalBytes += i.bytes\n\t\t\t}\n\n\t\t\tif maxMsgs > 0 && totalMsgs > uint64(maxMsgs) {\n\t\t\t\terr = ErrMaxMsgs\n\t\t\t} else if maxBytes > 0 && totalBytes > uint64(maxBytes) {\n\t\t\t\terr = ErrMaxBytes\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err\n\t\t\t}\n\t\t}\n\n\t\t// Similarly, check DiscardNew per-subject threshold to not need to bump CLFS.\n\t\tif discardNewPer && maxMsgsPer > 0 {\n\t\t\t// Get the current total for this subject.\n\t\t\ttotalMsgsForSubject := mset.store.SubjectsTotals(subject)[subject]\n\t\t\t// Add inflight count in this batch and for this stream.\n\t\t\ttotalMsgsForSubject += i.ops\n\t\t\tif i, ok = mset.inflight[subject]; ok {\n\t\t\t\ttotalMsgsForSubject += i.ops\n\t\t\t}\n\t\t\tif totalMsgsForSubject > uint64(maxMsgsPer) {\n\t\t\t\terr = ErrMaxMsgsPerSubject\n\t\t\t\treturn hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn hdr, msg, 0, nil, nil\n}\n\n// recalculateClusteredSeq initializes or updates mset.clseq, for example after a leader change.\n// This is reused for normal clustered publishing into a stream, and for atomic and fast batch publishing.\n// mset.clMu lock must be held.\nfunc recalculateClusteredSeq(mset *stream) (lseq uint64) {\n\t// Need to unlock and re-acquire the locks in the proper order.\n\tmset.clMu.Unlock()\n\t// Locking order is stream -> batchMu -> clMu\n\tmset.mu.RLock()\n\tbatch := mset.batchApply\n\tvar batchCount uint64\n\tif batch != nil {\n\t\tbatch.mu.Lock()\n\t\tbatchCount = batch.count\n\t}\n\tmset.clMu.Lock()\n\t// Re-capture\n\tlseq = mset.lseq\n\tmset.clseq = lseq + mset.clfs + batchCount\n\t// Keep hold of the mset.clMu, but unlock the others.\n\tif batch != nil {\n\t\tbatch.mu.Unlock()\n\t}\n\tmset.mu.RUnlock()\n\treturn lseq\n}\n\n// commitSingleMsg commits and proposes a single message to the node.\n// This is reused both for normal publishing into a stream, and for fast batch publishing.\n// mset.clMu lock must be held.\nfunc commitSingleMsg(\n\tdiff *batchStagedDiff, mset *stream, subject string, reply string, hdr []byte, msg []byte, name string,\n\tjsa *jsAccount, mt *msgTrace, node RaftNode, replicas int, lseq uint64,\n) {\n\tdiff.commit(mset)\n\tesm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), false)\n\tvar mtKey uint64\n\tif mt != nil {\n\t\tmtKey = mset.clseq\n\t\tif mset.mt == nil {\n\t\t\tmset.mt = make(map[uint64]*msgTrace)\n\t\t}\n\t\tmset.mt[mtKey] = mt\n\t}\n\n\t// Do proposal.\n\t_ = node.Propose(esm)\n\t// The proposal can fail, but we always account for trying.\n\tmset.clseq++\n\tmset.trackReplicationTraffic(node, len(esm), replicas)\n\n\t// Check to see if we are being overrun.\n\t// TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured.\n\tif mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold {\n\t\tlerr := fmt.Errorf(\"JetStream stream '%s > %s' has high message lag\", jsa.acc().Name, name)\n\t\tmset.srv.RateLimitWarnf(\"%s\", lerr.Error())\n\t}\n}\n" }, { "path": "server/jetstream_batching_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/big\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestJetStreamAtomicBatchPublish(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\tstorage StorageType,\n\t\tretention RetentionPolicy,\n\t\treplicas int,\n\t) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar pubAck JSPubAckResponse\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tStorage: storage,\n\t\t\tRetention: retention,\n\t\t\tReplicas: replicas,\n\t\t}\n\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tm := nats.NewMsg(\"foo.0\")\n\t\tm.Data = []byte(\"foo.0\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\n\t\t// Publish with atomic publish disabled.\n\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_NotNil(t, pubAck.Error)\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishDisabledError())\n\n\t\t// Enable atomic publish.\n\t\tcfg.AllowAtomicPublish = true\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Publish without batch sequence errors.\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishMissingSeqError())\n\n\t\t// Publish a batch, misses start.\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishIncompleteBatchError())\n\n\t\t// Publish a \"batch\" which immediately commits.\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\n\t\t// Reset commit.\n\t\tm.Header.Del(\"Nats-Batch-Commit\")\n\n\t\t// Publish a \"batch\" which has gaps.\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"3\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishIncompleteBatchError())\n\n\t\t// Publish a batch of N messages.\n\t\tm.Header.Del(\"Nats-Batch-Commit\")\n\t\tfor seq, batch := uint64(1), uint64(5); seq <= batch; seq++ {\n\t\t\tm.Subject = fmt.Sprintf(\"foo.%d\", seq)\n\t\t\tm.Data = []byte(m.Subject)\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", strconv.FormatUint(seq, 10))\n\t\t\t// If not commit.\n\t\t\tif seq != batch {\n\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_True(t, pubAck.Error == nil)\n\t\t\trequire_Equal(t, pubAck.Sequence, 6)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 5)\n\t\t}\n\n\t\t// Validate stream contents.\n\t\tif retention != InterestPolicy {\n\t\t\tfor i := 0; i < 6; i++ {\n\t\t\t\trsm, err := js.GetMsg(\"TEST\", uint64(i+1))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\trequire_Equal(t, rsm.Subject, subj)\n\t\t\t\trequire_Equal(t, string(rsm.Data), subj)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, retention := range []RetentionPolicy{LimitsPolicy, InterestPolicy, WorkQueuePolicy} {\n\t\t\tfor _, replicas := range []int{1, 3} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/R%d\", storage, retention, replicas), func(t *testing.T) {\n\t\t\t\t\ttest(t, storage, retention, replicas)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishEmptyAck(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\tstorage StorageType,\n\t\tretention RetentionPolicy,\n\t\treplicas int,\n\t) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tStorage: storage,\n\t\t\tRetention: retention,\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Publish a batch of N messages.\n\t\tfor seq, batch := uint64(1), uint64(5); seq <= batch; seq++ {\n\t\t\tm := nats.NewMsg(fmt.Sprintf(\"foo.%d\", seq))\n\t\t\tm.Data = []byte(m.Subject)\n\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", strconv.FormatUint(seq, 10))\n\t\t\tcommit := seq == batch\n\t\t\tif commit {\n\t\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\t}\n\n\t\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\t// If not commit we should receive an empty ack for flow control.\n\t\t\tif !commit {\n\t\t\t\trequire_Len(t, len(rmsg.Data), 0)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, 5)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 5)\n\t\t}\n\t}\n\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, retention := range []RetentionPolicy{LimitsPolicy, InterestPolicy, WorkQueuePolicy} {\n\t\t\tfor _, replicas := range []int{1, 3} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/R%d\", storage, retention, replicas), func(t *testing.T) {\n\t\t\t\t\ttest(t, storage, retention, replicas)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishCommitEob(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\tstorage StorageType,\n\t\tretention RetentionPolicy,\n\t\treplicas int,\n\t) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar pubAck JSPubAckResponse\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: storage,\n\t\t\tRetention: retention,\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"3\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"eob\")\n\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.Sequence, 2)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 2)\n\n\t\t// Validate stream contents.\n\t\tif retention != InterestPolicy {\n\t\t\tfor seq := uint64(1); seq <= 2; seq++ {\n\t\t\t\trsm, err := js.GetMsg(\"TEST\", seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Id\"), \"uuid\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Sequence\"), strconv.FormatUint(seq, 10))\n\t\t\t\t// The last message should have the commit header set, even though the commit was done via EOB.\n\t\t\t\tif seq == 2 {\n\t\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Commit\"), \"1\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, retention := range []RetentionPolicy{LimitsPolicy, InterestPolicy, WorkQueuePolicy} {\n\t\t\tfor _, replicas := range []int{1, 3} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/R%d\", storage, retention, replicas), func(t *testing.T) {\n\t\t\t\t\ttest(t, storage, retention, replicas)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishLimits(t *testing.T) {\n\tstreamMaxAtomicBatchInflightPerStream = 1\n\tstreamMaxAtomicBatchInflightTotal = 1\n\tstreamMaxAtomicBatchSize = 2\n\tstreamMaxBatchTimeout = 500 * time.Millisecond\n\tdefer func() {\n\t\tstreamMaxAtomicBatchInflightPerStream = streamDefaultMaxAtomicBatchInflightPerStream\n\t\tstreamMaxAtomicBatchInflightTotal = streamDefaultMaxAtomicBatchInflightTotal\n\t\tstreamMaxAtomicBatchSize = streamDefaultMaxAtomicBatchSize\n\t\tstreamMaxBatchTimeout = streamDefaultMaxBatchTimeout\n\t}()\n\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar pubAck JSPubAckResponse\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"FOO\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t}\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// For testing total server-wide maximum inflight batches.\n\t\tcfg = &StreamConfig{\n\t\t\tName: \"BAR\",\n\t\t\tSubjects: []string{\"bar\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t}\n\t\t_, err = jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// A batch ID must not exceed the maximum length.\n\t\tfor _, length := range []int{64, 65} {\n\t\t\tlongBatchId := strings.Repeat(\"A\", length)\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Batch-Id\", longBatchId)\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\tif length <= 64 {\n\t\t\t\trequire_True(t, pubAck.Error == nil)\n\t\t\t} else {\n\t\t\t\trequire_NotNil(t, pubAck.Error)\n\t\t\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishInvalidBatchIDError())\n\t\t\t}\n\t\t}\n\n\t\t// One batch is inflight.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t// Another batch moves over the threshold and batch is denied.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"exceeds_threshold\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishTooManyInflightError())\n\n\t\t// Another batch on a different stream moves over the server-wide threshold and batch is denied.\n\t\tm = nats.NewMsg(\"bar\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"bar\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishTooManyInflightError())\n\n\t\t// The first batch should now time out.\n\t\tsl := c.streamLeader(globalAccountName, \"FOO\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"FOO\")\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset.mu.RLock()\n\t\t\tbatches := mset.batches\n\t\t\tmset.mu.RUnlock()\n\t\t\tif batches == nil {\n\t\t\t\treturn errors.New(\"batches not found\")\n\t\t\t}\n\t\t\tbatches.mu.Lock()\n\t\t\tgroups := len(batches.atomic)\n\t\t\tbatches.mu.Unlock()\n\t\t\tif groups != 0 {\n\t\t\t\treturn fmt.Errorf(\"expected 0 groups, got %d\", groups)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Publishing to the batch should also error since it timed out.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishIncompleteBatchError())\n\n\t\t// Batch should be rejected if going over the max batch size.\n\t\tfor _, size := range []int{streamMaxAtomicBatchSize, streamMaxAtomicBatchSize + 1} {\n\t\t\tfor i := range size {\n\t\t\t\tseq := i + 1\n\t\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"exceeds_threshold\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", strconv.Itoa(seq))\n\t\t\t\tcommit := seq == size\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tpubAck = JSPubAckResponse{}\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\t\tif size <= streamMaxAtomicBatchSize {\n\t\t\t\t\trequire_True(t, pubAck.Error == nil)\n\t\t\t\t\trequire_Equal(t, pubAck.BatchSize, size)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishTooLargeBatchError(streamMaxAtomicBatchSize))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAtomicBatchPublishDedupeNotAllowed(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\tnc *nats.Conn,\n\t\tjs nats.JetStreamContext,\n\t\tstorage StorageType,\n\t\tretention RetentionPolicy,\n\t\treplicas int,\n\t) {\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tRetention: retention,\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t\tStorage: storage,\n\t\t\tAllowAtomicPublish: true,\n\t\t}\n\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", nil, nats.MsgId(\"pre-existing\"))\n\t\trequire_NoError(t, err)\n\n\t\tvar pubAck JSPubAckResponse\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Msg-Id\", \"pre-existing\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_NotNil(t, pubAck.Error)\n\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishContainsDuplicateMessageError())\n\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\tm.Header.Set(\"Nats-Msg-Id\", \"msgId1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tpubAck = JSPubAckResponse{}\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\tm.Header.Set(\"Nats-Msg-Id\", \"msgId2\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.BatchSize, 2)\n\t\trequire_Equal(t, pubAck.Sequence, 3)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t}\n\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, retention := range []RetentionPolicy{LimitsPolicy, InterestPolicy, WorkQueuePolicy} {\n\t\t\tfor _, replicas := range []int{1, 3} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/R%d\", storage, retention, replicas), func(t *testing.T) {\n\t\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\t\tdefer c.shutdown()\n\n\t\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\ttest(t, nc, js, storage, retention, replicas)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishSourceAndMirror(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowAtomicPublish: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor seq := uint64(1); seq <= 3; seq++ {\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", strconv.FormatUint(seq, 10))\n\t\t\tcommit := seq == 3\n\t\t\tif !commit {\n\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\n\t\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar pubAck JSPubAckResponse\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, 3)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 3)\n\t\t}\n\n\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 2))\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Mirror can source batched messages but can't do atomic batching itself.\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"M-no-batch\",\n\t\t\tStorage: FileStorage,\n\t\t\tMirror: &StreamSource{Name: \"TEST\"},\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_Error(t, err, NewJSMirrorWithAtomicPublishError())\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"S\",\n\t\t\tStorage: FileStorage,\n\t\t\tSources: []*StreamSource{{Name: \"TEST\"}},\n\t\t\tReplicas: replicas,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, name := range []string{\"M\", \"S\"} {\n\t\t\t\tif si, err := js.StreamInfo(name); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if si.State.Msgs != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 2 messages for stream %q, got %d\", name, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Ensure the batching headers were removed when ingested into the source/mirror.\n\t\trsm, err := js.GetMsg(\"M\", 1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 0)\n\n\t\trsm, err = js.GetMsg(\"M\", 3)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 0)\n\n\t\trsm, err = js.GetMsg(\"S\", 1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 1)\n\t\trequire_Equal(t, rsm.Header.Get(JSStreamSource), \"TEST 1 > > foo\")\n\n\t\trsm, err = js.GetMsg(\"S\", 2)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 1)\n\t\trequire_Equal(t, rsm.Header.Get(JSStreamSource), \"TEST 3 > > foo\")\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAtomicBatchPublishCleanup(t *testing.T) {\n\tconst (\n\t\tDisable = iota\n\t\tStepDown\n\t\tDelete\n\t\tCommit\n\t)\n\n\ttest := func(t *testing.T, mode int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowAtomicPublish: false,\n\t\t\tReplicas: 3,\n\t\t}\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tmset.mu.RLock()\n\t\tbatches := mset.batches\n\t\tbatch := mset.batchApply\n\t\tmset.mu.RUnlock()\n\t\trequire_True(t, batches == nil)\n\t\trequire_True(t, batch == nil)\n\n\t\t// Enabling doesn't need to populate the batching state.\n\t\tcfg.AllowAtomicPublish = true\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\tmset.mu.RLock()\n\t\tbatches = mset.batches\n\t\tbatch = mset.batchApply\n\t\tmset.mu.RUnlock()\n\t\trequire_True(t, batches == nil)\n\t\trequire_True(t, batch == nil)\n\n\t\t// Publish a partial batch that needs to be cleaned up.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t// Publish another batch that commits, state should be cleaned up by default.\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"commit\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\n\t\tmset.mu.RLock()\n\t\tbatches = mset.batches\n\t\tbatch = mset.batchApply\n\t\tmset.mu.RUnlock()\n\t\trequire_NotNil(t, batches)\n\t\trequire_NotNil(t, batch)\n\t\tbatches.mu.Lock()\n\t\tgroups := len(batches.atomic)\n\t\tb := batches.atomic[\"uuid\"]\n\t\tbatches.mu.Unlock()\n\t\trequire_Len(t, groups, 1)\n\t\trequire_NotNil(t, b)\n\t\tstore := b.store\n\t\trequire_Equal(t, store.State().Msgs, 1)\n\t\tclfs := mset.getCLFS()\n\n\t\t// Should fully clean up the in-progress batch.\n\t\tswitch mode {\n\t\tcase Disable:\n\t\t\tcfg.AllowAtomicPublish = false\n\t\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\t\trequire_NoError(t, err)\n\t\tcase StepDown:\n\t\t\trequire_NoError(t, sl.JetStreamStepdownStream(globalAccountName, \"TEST\"))\n\t\tcase Delete:\n\t\t\trequire_NoError(t, mset.delete())\n\t\tcase Commit:\n\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset.mu.RLock()\n\t\t\tbatches = mset.batches\n\t\t\tmset.mu.RUnlock()\n\t\t\tif batches != nil {\n\t\t\t\tif mode != Commit {\n\t\t\t\t\treturn fmt.Errorf(\"expected no batches\")\n\t\t\t\t}\n\t\t\t\tbatches.mu.Lock()\n\t\t\t\tgroups = len(batches.atomic)\n\t\t\t\tbatches.mu.Unlock()\n\t\t\t\tif groups > 0 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 0 groups, got %d\", groups)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif msgs := store.State().Msgs; msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"expected 0 messages, got %d\", msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Should clean up the batch apply state.\n\t\tif mode == Disable || mode == Delete {\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tbatch = mset.batchApply\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\tnclfs := mset.getCLFS()\n\t\t\t\tif batch != nil {\n\t\t\t\t\treturn fmt.Errorf(\"expected no batch apply\")\n\t\t\t\t}\n\t\t\t\tif clfs != nclfs {\n\t\t\t\t\treturn fmt.Errorf(\"expected no change in CLFS\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t}\n\n\tt.Run(\"Disable\", func(t *testing.T) { test(t, Disable) })\n\tt.Run(\"StepDown\", func(t *testing.T) { test(t, StepDown) })\n\tt.Run(\"Delete\", func(t *testing.T) { test(t, Delete) })\n\tt.Run(\"Commit\", func(t *testing.T) { test(t, Commit) })\n}\n\nfunc TestJetStreamAtomicBatchPublishConfigOpts(t *testing.T) {\n\t// Defaults.\n\trequire_Equal(t, streamMaxBatchTimeout, 10*time.Second)\n\trequire_Equal(t, streamMaxAtomicBatchInflightPerStream, 50)\n\trequire_Equal(t, streamMaxAtomicBatchInflightTotal, 1000)\n\trequire_Equal(t, streamMaxAtomicBatchSize, 1000)\n\trequire_Equal(t, streamMaxFastBatchInflightPerStream, 1000)\n\trequire_Equal(t, streamMaxFastBatchInflightTotal, 50_000)\n\n\tbatchingConf := `\n\tlisten: 127.0.0.1:-1\n\tjetstream {\n\t\tstore_dir: %q\n\t\tlimits {\n\t\t\tbatch {\n\t\t\t\tmax_inflight_per_stream: %d\n\t\t\t\tmax_inflight_total: %d\n\t\t\t\tmax_msgs: %d\n\t\t\t\ttimeout: %s\n\t\t\t}\n\t\t}\n\t}`\n\n\tcf := createConfFile(t, []byte(fmt.Sprintf(batchingConf, t.TempDir(), 10, 20, 100, \"5s\")))\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\trequire_Equal(t, opts.JetStreamLimits.MaxBatchInflightPerStream, 10)\n\trequire_Equal(t, opts.JetStreamLimits.MaxBatchInflightTotal, 20)\n\trequire_Equal(t, opts.JetStreamLimits.MaxBatchSize, 100)\n\trequire_Equal(t, opts.JetStreamLimits.MaxBatchTimeout, 5*time.Second)\n\n\t// Reloading is not supported, that would potentially mean dropping random batches when lowering limits.\n\tchangeCurrentConfigContentWithNewContent(t, cf, []byte(fmt.Sprintf(batchingConf, t.TempDir(), 20, 40, 200, \"10s\")))\n\trequire_Error(t, s.Reload(), fmt.Errorf(\"config reload not supported for JetStreamLimits\"))\n}\n\nfunc TestJetStreamAtomicBatchPublishDenyHeaders(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowAtomicPublish: true,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// We might support these headers later on, but for now error.\n\t\tfor key, value := range map[string]string{\n\t\t\t\"Nats-Expected-Last-Msg-Id\": \"msgId\",\n\t\t} {\n\t\t\tt.Run(key, func(t *testing.T) {\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\t\t\tm.Header.Set(key, value)\n\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"eob\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSAtomicPublishUnsupportedHeaderBatchError(key))\n\t\t\t})\n\t\t}\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAtomicBatchPublishStageAndCommit(t *testing.T) {\n\ttype BatchItem struct {\n\t\tsubject string\n\t\theader nats.Header\n\t\tmsg []byte\n\t\terr error\n\t}\n\n\ttype BatchTest struct {\n\t\ttitle string\n\t\tallowRollup bool\n\t\tdenyPurge bool\n\t\tallowTTL bool\n\t\tallowMsgCounter bool\n\t\tallowMsgSchedules bool\n\t\tdiscardNew bool\n\t\tdiscardNewPerSubj bool\n\t\tinit func(mset *stream)\n\t\tbatch []BatchItem\n\t\tvalidate func(mset *stream, commit bool)\n\t}\n\n\ttests := []BatchTest{\n\t\t{\n\t\t\ttitle: \"dedupe-distinct\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgId: {\"foo\"}}},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSMsgId: {\"bar\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Equal(t, mset.checkMsgId(\"foo\") != nil, commit)\n\t\t\t\trequire_Equal(t, mset.checkMsgId(\"bar\") != nil, commit)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"dedupe\",\n\t\t\tinit: func(mset *stream) {\n\t\t\t\tmset.storeMsgId(&ddentry{id: \"foo\", seq: 0, ts: time.Now().UnixNano()})\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgId: {\"foo\"}}, err: errMsgIdDuplicate},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_NotNil(t, mset.checkMsgId(\"foo\"))\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"dedupe-staged\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgId: {\"foo\"}}},\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgId: {\"foo\"}}, err: errMsgIdDuplicate},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Equal(t, mset.checkMsgId(\"foo\") != nil, commit)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"counter-single\",\n\t\t\tallowMsgCounter: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMessageIncr: {\"1\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_True(t, mset.clusteredCounterTotal[\"foo\"] == nil)\n\t\t\t\t} else {\n\t\t\t\t\tcounter := mset.clusteredCounterTotal[\"foo\"]\n\t\t\t\t\trequire_NotNil(t, counter)\n\t\t\t\t\trequire_Equal(t, counter.ops, 1)\n\t\t\t\t\trequire_Equal(t, counter.total.String(), \"1\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"counter-multiple\",\n\t\t\tallowMsgCounter: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMessageIncr: {\"1\"}}},\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMessageIncr: {\"2\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_True(t, mset.clusteredCounterTotal[\"foo\"] == nil)\n\t\t\t\t} else {\n\t\t\t\t\tcounter := mset.clusteredCounterTotal[\"foo\"]\n\t\t\t\t\trequire_NotNil(t, counter)\n\t\t\t\t\trequire_Equal(t, counter.ops, 2)\n\t\t\t\t\trequire_Equal(t, counter.total.String(), \"3\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"counter-pre-init\",\n\t\t\tallowMsgCounter: true,\n\t\t\tinit: func(mset *stream) {\n\t\t\t\tmset.clusteredCounterTotal = map[string]*msgCounterRunningTotal{\n\t\t\t\t\t\"foo\": {total: big.NewInt(1), ops: 1},\n\t\t\t\t}\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMessageIncr: {\"2\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tcounter := mset.clusteredCounterTotal[\"foo\"]\n\t\t\t\trequire_NotNil(t, counter)\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Equal(t, counter.ops, 1)\n\t\t\t\t\trequire_Equal(t, counter.total.String(), \"1\")\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, counter.ops, 2)\n\t\t\t\t\trequire_Equal(t, counter.total.String(), \"3\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-disabled\",\n\t\t\tallowMsgSchedules: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"}},\n\t\t\t\t\terr: NewJSMessageSchedulesDisabledError(),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSSchedulePattern: {\"disabled\"}},\n\t\t\t\t\terr: NewJSMessageSchedulesDisabledError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-ttl-disabled\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTTL: {\"1s\"},\n\t\t\t\t\t},\n\t\t\t\t\terr: errMsgTTLDisabled,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-ttl-invalid\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tallowTTL: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTTL: {\"invalid\"},\n\t\t\t\t\t},\n\t\t\t\t\terr: NewJSMessageSchedulesTTLInvalidError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-invalid-schedule\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSSchedulePattern: {\"invalid\"}},\n\t\t\t\t\terr: NewJSMessageSchedulesPatternInvalidError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-target-mismatch\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"not.matching\"},\n\t\t\t\t\t},\n\t\t\t\t\terr: NewJSMessageSchedulesTargetInvalidError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-target-must-be-literal\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"foo.*\"},\n\t\t\t\t\t},\n\t\t\t\t\terr: NewJSMessageSchedulesTargetInvalidError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-target-must-be-unique\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"foo\"},\n\t\t\t\t\t},\n\t\t\t\t\terr: NewJSMessageSchedulesTargetInvalidError(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules-rollup-disabled\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"bar\"},\n\t\t\t\t\t\tJSMsgRollup: {JSMsgRollupSubject},\n\t\t\t\t\t},\n\t\t\t\t\terr: errors.New(\"rollup not permitted\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"msg-schedules\",\n\t\t\tallowMsgSchedules: true,\n\t\t\tallowRollup: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"baz\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tsubject: \"bar\",\n\t\t\t\t\theader: nats.Header{\n\t\t\t\t\t\tJSSchedulePattern: {\"@at 1970-01-01T00:00:00Z\"},\n\t\t\t\t\t\tJSScheduleTarget: {\"baz\"},\n\t\t\t\t\t\tJSMsgRollup: {JSMsgRollupSubject},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new\",\n\t\t\tdiscardNew: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo1\"},\n\t\t\t\t{subject: \"foo2\"},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 2)\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo1\"], inflightSubjectRunningTotal{bytes: 20, ops: 1})\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo2\"], inflightSubjectRunningTotal{bytes: 20, ops: 1})\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-msgs\",\n\t\t\tdiscardNew: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t{subject: \"foo\", err: ErrMaxMsgs},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 1)\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo\"], inflightSubjectRunningTotal{bytes: 19 * 3, ops: 3})\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-bytes\",\n\t\t\tdiscardNew: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo1\", msg: bytes.Repeat([]byte(\"A\"), 10)},\n\t\t\t\t{subject: \"foo2\", msg: bytes.Repeat([]byte(\"A\"), 11), err: ErrMaxBytes},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 2)\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo1\"], inflightSubjectRunningTotal{bytes: 30, ops: 1})\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo2\"], inflightSubjectRunningTotal{bytes: 31, ops: 1})\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-msgs-per-subj\",\n\t\t\tdiscardNew: true,\n\t\t\tdiscardNewPerSubj: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.inflight), 1)\n\t\t\t\t\trequire_Equal(t, *mset.inflight[\"foo\"], inflightSubjectRunningTotal{bytes: 19, ops: 1})\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-msgs-per-subj-duplicate\",\n\t\t\tdiscardNew: true,\n\t\t\tdiscardNewPerSubj: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t{subject: \"foo\", err: ErrMaxMsgsPerSubject},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-msgs-per-subj-inflight\",\n\t\t\tdiscardNew: true,\n\t\t\tdiscardNewPerSubj: true,\n\t\t\tinit: func(mset *stream) {\n\t\t\t\tmset.inflight = map[string]*inflightSubjectRunningTotal{\"foo\": {bytes: 123, ops: 1}}\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", err: ErrMaxMsgsPerSubject},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Len(t, len(mset.inflight), 1)\n\t\t\t\trequire_Equal(t, *mset.inflight[\"foo\"], inflightSubjectRunningTotal{bytes: 123, ops: 1})\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"discard-new-max-msgs-per-subj-pre-existing\",\n\t\t\tdiscardNew: true,\n\t\t\tdiscardNewPerSubj: true,\n\t\t\tinit: func(mset *stream) {\n\t\t\t\t_, _, err := mset.store.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", err: ErrMaxMsgsPerSubject},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Len(t, len(mset.inflight), 0)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-last-seq\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSeq: {\"0\"}}},\n\t\t\t\t{subject: \"bar\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-last-seq-not-first\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSExpectedLastSeq: {\"0\"}}, err: errors.New(\"last sequence mismatch\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-last-seq-invalid-first\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSeq: {\"1\"}}, err: errors.New(\"last sequence mismatch: 1 vs 0\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-last-seq-invalid\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSeq: {\"0\"}}},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSExpectedLastSeq: {\"0\"}}, err: errors.New(\"last sequence mismatch: 0 vs 1\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-simple\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSubjSeq: {\"0\"}}},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSExpectedLastSubjSeq: {\"0\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 2)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 2)\n\t\t\t\t\trequire_Equal(t, mset.expectedPerSubjectSequence[0], \"foo\")\n\t\t\t\t\trequire_Equal(t, mset.expectedPerSubjectSequence[1], \"bar\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-redundant-in-batch\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSExpectedLastSubjSeq: {\"0\"}},\n\t\t\t\t},\n\t\t\t\t// Would normally fail the batch, recognize in-process.\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSExpectedLastSubjSeq: {\"1\"}},\n\t\t\t\t\terr: errors.New(\"last sequence by subject mismatch: 1 vs 0\"),\n\t\t\t\t},\n\t\t\t\t// Redundant expected check results in an error. The subject 'foo' is also updated in the batch.\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSExpectedLastSubjSeq: {\"0\"}},\n\t\t\t\t\terr: errors.New(\"last sequence by subject mismatch\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 1)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 1)\n\t\t\t\t\trequire_Equal(t, mset.expectedPerSubjectSequence[0], \"foo\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-dupe-in-change\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSubjSeq: {\"0\"}, JSExpectedLastSubjSeqSubj: {\"baz\"}}},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSExpectedLastSubjSeq: {\"0\"}, JSExpectedLastSubjSeqSubj: {\"baz\"}}},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\tif !commit {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 0)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 1)\n\t\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 1)\n\t\t\t\t\trequire_Equal(t, mset.expectedPerSubjectSequence[1], \"baz\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-not-first\",\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t// Mismatch because only the first 'foo' can have the expected check.\n\t\t\t\t{\n\t\t\t\t\tsubject: \"foo\",\n\t\t\t\t\theader: nats.Header{JSExpectedLastSubjSeq: {\"0\"}},\n\t\t\t\t\terr: errors.New(\"last sequence by subject mismatch\"),\n\t\t\t\t},\n\t\t\t\t// Mismatch because only the first 'foo' can have the expected check.\n\t\t\t\t{\n\t\t\t\t\tsubject: \"bar\",\n\t\t\t\t\theader: nats.Header{JSExpectedLastSubjSeq: {\"0\"}, JSExpectedLastSubjSeqSubj: {\"foo\"}},\n\t\t\t\t\terr: errors.New(\"last sequence by subject mismatch\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-in-process\",\n\t\t\tinit: func(mset *stream) {\n\t\t\t\tmset.expectedPerSubjectInProcess = map[string]struct{}{\"foo\": {}}\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSubjSeq: {\"10\"}}, err: errors.New(\"last sequence by subject mismatch\")},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 1)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"expect-per-subj-inflight\",\n\t\t\tinit: func(mset *stream) {\n\t\t\t\tmset.inflight = map[string]*inflightSubjectRunningTotal{\"bar\": {bytes: 33, ops: 1}}\n\t\t\t},\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSExpectedLastSubjSeq: {\"10\"}, JSExpectedLastSubjSeqSubj: {\"bar\"}}, err: errors.New(\"last sequence by subject mismatch\")},\n\t\t\t},\n\t\t\tvalidate: func(mset *stream, commit bool) {\n\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\t\t\t\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 0)\n\t\t\t\trequire_Len(t, len(mset.inflight), 1)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-deny-purge\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: true,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {JSMsgRollupSubject}}, err: errors.New(\"rollup not permitted\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-invalid\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {\"invalid\"}}, err: fmt.Errorf(\"rollup value invalid: %q\", \"invalid\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-all-first\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {JSMsgRollupAll}}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-all-not-first\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\"},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSMsgRollup: {JSMsgRollupAll}}, err: errors.New(\"batch rollup all invalid\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-sub-unique\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {JSMsgRollupSubject}}},\n\t\t\t\t{subject: \"bar\", header: nats.Header{JSMsgRollup: {JSMsgRollupSubject}}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"rollup-sub-overlap\",\n\t\t\tallowRollup: true,\n\t\t\tdenyPurge: false,\n\t\t\tbatch: []BatchItem{\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {JSMsgRollupSubject}}},\n\t\t\t\t{subject: \"foo\", header: nats.Header{JSMsgRollup: {JSMsgRollupSubject}}, err: errors.New(\"batch rollup sub invalid\")},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tAllowAtomicPublish: true,\n\t\t\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.init != nil {\n\t\t\t\ttest.init(mset)\n\t\t\t}\n\n\t\t\tvar (\n\t\t\t\tdiscard DiscardPolicy\n\t\t\t\tdiscardNewPer bool\n\t\t\t\tmaxMsgs int64 = -1\n\t\t\t\tmaxMsgsPer int64 = -1\n\t\t\t\tmaxBytes int64 = -1\n\t\t\t)\n\t\t\tif test.discardNew {\n\t\t\t\tdiscard, maxMsgs, maxBytes = DiscardNew, 2, 60\n\t\t\t}\n\t\t\tif test.discardNewPerSubj {\n\t\t\t\trequire_True(t, test.discardNew)\n\t\t\t\tdiscardNewPer, maxMsgs, maxBytes, maxMsgsPer = true, -1, -1, 1\n\t\t\t}\n\n\t\t\tdiff := &batchStagedDiff{}\n\t\t\tmset.clMu.Lock()\n\t\t\tdefer mset.clMu.Unlock()\n\t\t\tvar hadError bool\n\t\t\tfor _, m := range test.batch {\n\t\t\t\tvar hdr []byte\n\t\t\t\tfor key, values := range m.header {\n\t\t\t\t\tfor _, value := range values {\n\t\t\t\t\t\thdr = genHeader(hdr, key, value)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t_, _, _, _, err = checkMsgHeadersPreClusteredProposal(diff, mset, m.subject, hdr, m.msg, false, \"TEST\", nil, test.allowRollup, test.denyPurge, test.allowTTL, test.allowMsgCounter, test.allowMsgSchedules, discard, discardNewPer, -1, maxMsgs, maxMsgsPer, maxBytes)\n\t\t\t\tif m.err != nil {\n\t\t\t\t\trequire_Error(t, err, m.err)\n\t\t\t\t} else if err != nil {\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\thadError = true\n\t\t\t\t}\n\t\t\t\tif test.validate != nil {\n\t\t\t\t\ttest.validate(mset, false)\n\t\t\t\t}\n\t\t\t\tmset.clseq++\n\t\t\t}\n\t\t\tif test.validate != nil && !hadError {\n\t\t\t\tdiff.commit(mset)\n\t\t\t\ttest.validate(mset, true)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishHighLevelRollback(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tStorage: FileStorage,\n\t\tRetention: InterestPolicy,\n\t\tMaxMsgs: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\trequireEmpty := func() {\n\t\tmset.ddMu.Lock()\n\t\tddarr, ddmap := len(mset.ddarr), len(mset.ddmap)\n\t\tmset.ddMu.Unlock()\n\t\trequire_Len(t, ddarr, 0)\n\t\trequire_Len(t, ddmap, 0)\n\n\t\tmset.clMu.Lock()\n\t\tinflight, subjSeq, subjInProcess := len(mset.inflight), len(mset.expectedPerSubjectSequence), len(mset.expectedPerSubjectInProcess)\n\t\tmset.clMu.Unlock()\n\t\trequire_Len(t, inflight, 0)\n\t\trequire_Len(t, subjSeq, 0)\n\t\trequire_Len(t, subjInProcess, 0)\n\t}\n\trequireEmpty()\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(\"Nats-Msg-Id\", \"dedupe\")\n\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"1\")\n\t_, err = js.PublishMsg(m)\n\trequire_Error(t, err, NewJSStreamWrongLastSequenceError(0))\n\trequireEmpty()\n}\n\nfunc TestJetStreamAtomicBatchPublishExpectedPerSubject(t *testing.T) {\n\ttype TestKind int\n\tconst (\n\t\tOnlyFirst TestKind = iota\n\t\tRedundant\n\t\tNotFirst\n\t)\n\n\ttest := func(t *testing.T, kind TestKind) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\tm := nats.NewMsg(\"foo\")\n\t\tif kind != NotFirst {\n\t\t\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"1\")\n\t\t}\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tvar pubAck JSPubAckResponse\n\n\t\t// Redundant expected headers are okay, as long as they reflect the state prior to the batch.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tif kind == Redundant || kind == NotFirst {\n\t\t\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"1\")\n\t\t}\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\tif kind == Redundant || kind == NotFirst {\n\t\t\trequire_NotNil(t, pubAck.Error)\n\t\t\trequire_Error(t, pubAck.Error, NewJSStreamWrongLastSequenceConstantError())\n\t\t\treturn\n\t\t}\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.Sequence, 3)\n\t\trequire_Equal(t, pubAck.BatchSize, 2)\n\n\t\t// The first message still contains the expected headers.\n\t\trsm, err := js.GetMsg(\"TEST\", 2)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Id\"), \"uuid\")\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Sequence\"), \"1\")\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Expected-Last-Subject-Sequence\"), \"1\")\n\n\t\t// The second message doesn't have the expected headers, as the condition was already checked\n\t\t// and seems inconsistent when getting the message afterward.\n\t\trsm, err = js.GetMsg(\"TEST\", 3)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Id\"), \"uuid\")\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Sequence\"), \"2\")\n\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Expected-Last-Subject-Sequence\"), _EMPTY_)\n\t}\n\n\tt.Run(\"single\", func(t *testing.T) { test(t, OnlyFirst) })\n\tt.Run(\"redundant\", func(t *testing.T) { test(t, Redundant) })\n\tt.Run(\"not-first\", func(t *testing.T) { test(t, NotFirst) })\n}\n\nfunc TestJetStreamAtomicBatchPublishSingleServerRecovery(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tRetention: LimitsPolicy,\n\t\tReplicas: 1,\n\t\tAllowAtomicPublish: true,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\t// Manually construct and store a batch, so it doesn't immediately commit.\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.Lock()\n\tbatches := &batching{}\n\tmset.batches = batches\n\tmset.mu.Unlock()\n\tbatches.mu.Lock()\n\tb, err := batches.newAtomicBatch(mset, \"uuid\")\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\thdr1 := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\thdr1 = genHeader(hdr1, \"Nats-Batch-Sequence\", \"1\")\n\t_, _, err = b.store.StoreMsg(\"foo\", hdr1, nil, 0)\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\n\thdr2 := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\thdr2 = genHeader(hdr2, \"Nats-Batch-Sequence\", \"2\")\n\thdr2 = genHeader(hdr2, \"Nats-Batch-Commit\", \"1\")\n\t_, _, err = b.store.StoreMsg(\"foo\", hdr2, nil, 0)\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\tabandonReason := b.readyForCommit()\n\tbatches.mu.Unlock()\n\trequire_True(t, abandonReason == nil)\n\n\t// Simulate the first message of the batch is committed.\n\terr = mset.processJetStreamMsg(\"foo\", _EMPTY_, hdr1, nil, 0, 0, nil, false, true)\n\trequire_NoError(t, err)\n\n\t// Simulate a hard kill, upon recovery the rest of the batch should be applied.\n\tport := s.opts.Port\n\tsd := s.StoreDir()\n\tnc.Close()\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tstate := mset.state()\n\trequire_Equal(t, state.Msgs, 2)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 2)\n\n\tfor seq := state.FirstSeq; seq <= state.LastSeq; seq++ {\n\t\tsm, err := mset.getMsg(seq)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Id\", sm.Header)), \"uuid\")\n\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Sequence\", sm.Header)), strconv.FormatUint(seq, 10))\n\t\t// The last message should have the commit header set.\n\t\tif seq == state.LastSeq {\n\t\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Commit\", sm.Header)), \"1\")\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishSingleServerRecoveryCommitEob(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tRetention: LimitsPolicy,\n\t\tReplicas: 1,\n\t\tAllowAtomicPublish: true,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\t// Manually construct and store a batch, so it doesn't immediately commit.\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.Lock()\n\tbatches := &batching{}\n\tmset.batches = batches\n\tmset.mu.Unlock()\n\tbatches.mu.Lock()\n\tb, err := batches.newAtomicBatch(mset, \"uuid\")\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\thdr1 := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\thdr1 = genHeader(hdr1, \"Nats-Batch-Sequence\", \"1\")\n\t_, _, err = b.store.StoreMsg(\"foo\", hdr1, nil, 0)\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\n\thdr2 := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\thdr2 = genHeader(hdr2, \"Nats-Batch-Sequence\", \"2\")\n\t_, _, err = b.store.StoreMsg(\"foo\", hdr2, nil, 0)\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\n\thdr3 := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\thdr3 = genHeader(hdr3, \"Nats-Batch-Sequence\", \"3\")\n\thdr3 = genHeader(hdr3, \"Nats-Batch-Commit\", \"eob\")\n\t_, _, err = b.store.StoreMsg(\"foo\", hdr3, nil, 0)\n\tif err != nil {\n\t\tbatches.mu.Unlock()\n\t\trequire_NoError(t, err)\n\t}\n\tabandonReason := b.readyForCommit()\n\tbatches.mu.Unlock()\n\trequire_True(t, abandonReason == nil)\n\n\t// Simulate the first message of the batch is committed.\n\terr = mset.processJetStreamMsg(\"foo\", _EMPTY_, hdr1, nil, 0, 0, nil, false, true)\n\trequire_NoError(t, err)\n\n\t// Simulate a hard kill, upon recovery the rest of the batch should be applied.\n\tport := s.opts.Port\n\tsd := s.StoreDir()\n\tnc.Close()\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tstate := mset.state()\n\trequire_Equal(t, state.Msgs, 2)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 2)\n\n\tfor seq := state.FirstSeq; seq <= state.LastSeq; seq++ {\n\t\tsm, err := mset.getMsg(seq)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Id\", sm.Header)), \"uuid\")\n\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Sequence\", sm.Header)), strconv.FormatUint(seq, 10))\n\t\t// The last message should have the commit header set, even though the commit was done via EOB.\n\t\tif seq == state.LastSeq {\n\t\t\trequire_Equal(t, string(getHeader(\"Nats-Batch-Commit\", sm.Header)), \"1\")\n\t\t}\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishEncode(t *testing.T) {\n\ttest := func(t *testing.T, commit bool, compress bool) {\n\t\tts := time.Now().UnixNano()\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\t\thdr = genHeader(hdr, \"Nats-Batch-Sequence\", \"1\")\n\t\tmsg := []byte(\"A\")\n\t\tif compress {\n\t\t\tmsg = bytes.Repeat(msg, compressThreshold)\n\t\t}\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", \"reply\", hdr, msg, 1, ts, false, \"uuid\", 1, commit)\n\n\t\tbuf, op := esm[1:], entryOp(esm[0])\n\t\tif commit {\n\t\t\trequire_Equal(t, op, batchCommitMsgOp)\n\t\t} else {\n\t\t\trequire_Equal(t, op, batchMsgOp)\n\t\t}\n\n\t\tbatchId, batchSeq, op, mbuf, err := decodeBatchMsg(buf)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, batchId, \"uuid\")\n\t\trequire_Equal(t, batchSeq, 1)\n\t\tif compress {\n\t\t\trequire_Equal(t, op, compressedStreamMsgOp)\n\t\t\tmbuf, err = s2.Decode(nil, mbuf)\n\t\t\trequire_NoError(t, err)\n\t\t} else {\n\t\t\trequire_Equal(t, op, streamMsgOp)\n\t\t}\n\n\t\tsubject, reply, dhdr, dmsg, lseq, dts, sourced, err := decodeStreamMsg(mbuf)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, subject, \"foo\")\n\t\trequire_Equal(t, reply, \"reply\")\n\t\trequire_True(t, bytes.Equal(dhdr, hdr))\n\t\trequire_True(t, bytes.Equal(dmsg, msg))\n\t\trequire_Equal(t, lseq, 1)\n\t\trequire_Equal(t, dts, ts)\n\t\trequire_False(t, sourced)\n\t}\n\n\tt.Run(\"normal\", func(t *testing.T) { test(t, false, false) })\n\tt.Run(\"normal-compress\", func(t *testing.T) { test(t, false, true) })\n\tt.Run(\"commit\", func(t *testing.T) { test(t, true, false) })\n\tt.Run(\"commit-compress\", func(t *testing.T) { test(t, true, true) })\n}\n\n// Test a batch within a single proposal, optionally combined with messages unrelated\n// to the batch but within the same proposal.\nfunc TestJetStreamAtomicBatchPublishProposeOne(t *testing.T) {\n\ttest := func(t *testing.T, combined bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tvar entries []*Entry\n\n\t\tmset.clMu.Lock()\n\t\tif combined {\n\t\t\tesm := encodeStreamMsg(\"foo\", _EMPTY_, nil, nil, mset.clseq, 0, false)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\n\t\tmsg := []byte(\"hello\")\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"1\", hdr)\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", 1, false)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"2\", hdr)\n\t\thdr = setHeader(\"Nats-Batch-Commit\", \"1\", hdr)\n\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", 2, true)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\n\t\tif combined {\n\t\t\tesm = encodeStreamMsg(\"foo\", _EMPTY_, nil, nil, mset.clseq, 0, false)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tmset.clMu.Unlock()\n\t\tn := mset.raftNode().(*raft)\n\t\tn.sendAppendEntry(entries)\n\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tif combined {\n\t\t\trequire_Equal(t, pubAck.Sequence, 6)\n\t\t} else {\n\t\t\trequire_Equal(t, pubAck.Sequence, 4)\n\t\t}\n\t}\n\n\tt.Run(\"single\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"combined\", func(t *testing.T) { test(t, true) })\n}\n\n// Test a batch spanning multiple proposals, optionally combined with messages unrelated\n// to the batch but within the same first/last proposal.\nfunc TestJetStreamAtomicBatchPublishProposeMultiple(t *testing.T) {\n\ttest := func(t *testing.T, partial bool, combined bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tvar entries []*Entry\n\t\tmset.clMu.Lock()\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\t\thdr = genHeader(hdr, \"Nats-Batch-Sequence\", \"1\")\n\t\tmsg := []byte(\"hello\")\n\t\tif combined {\n\t\t\tesm := encodeStreamMsg(\"foo\", _EMPTY_, nil, nil, mset.clseq, 0, false)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", 1, false)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\t\tmset.clMu.Unlock()\n\t\tn := mset.raftNode().(*raft)\n\t\tn.sendAppendEntry(entries)\n\n\t\tmset.clMu.Lock()\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"2\", hdr)\n\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", 2, false)\n\t\tmset.clseq++\n\t\tmset.clMu.Unlock()\n\t\tn.sendAppendEntry([]*Entry{newEntry(EntryNormal, esm)})\n\n\t\tentries = nil\n\t\tmset.clMu.Lock()\n\t\tif !partial {\n\t\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"3\", hdr)\n\t\t\thdr = genHeader(hdr, \"Nats-Batch-Commit\", \"1\")\n\t\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", 3, true)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tif combined {\n\t\t\tesm = encodeStreamMsg(\"foo\", _EMPTY_, nil, nil, mset.clseq, 0, false)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tmset.clMu.Unlock()\n\t\tn.sendAppendEntry(entries)\n\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\texpectedSeq := uint64(2)\n\t\tif !partial {\n\t\t\texpectedSeq += 3\n\t\t}\n\t\tif combined {\n\t\t\texpectedSeq += 2\n\t\t}\n\t\trequire_Equal(t, pubAck.Sequence, expectedSeq)\n\t}\n\n\tt.Run(\"partial\", func(t *testing.T) { test(t, true, false) })\n\tt.Run(\"partial-combined\", func(t *testing.T) { test(t, true, true) })\n\tt.Run(\"full\", func(t *testing.T) { test(t, false, false) })\n\tt.Run(\"full-combined\", func(t *testing.T) { test(t, false, true) })\n}\n\n// Test a batch that was only partially proposed.\nfunc TestJetStreamAtomicBatchPublishProposeOnePartialBatch(t *testing.T) {\n\tmaxEntries := 3\n\tfor i := range maxEntries + 1 {\n\t\tt.Run(fmt.Sprintf(\"I-%d\", i), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 3,\n\t\t\t\tAllowAtomicPublish: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t\tvar entries []*Entry\n\t\t\tmset.clMu.Lock()\n\t\t\tmsg := []byte(\"hello\")\n\t\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"uuid\")\n\t\t\tfor j := range 3 {\n\t\t\t\t// Skip if indices equal.\n\t\t\t\tif i == j {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tbseq := uint64(j + 1)\n\t\t\t\thdr = setHeader(\"Nats-Batch-Sequence\", strconv.FormatUint(bseq, 10), hdr)\n\t\t\t\tcommit := bseq == uint64(maxEntries)\n\t\t\t\tif commit {\n\t\t\t\t\thdr = setHeader(\"Nats-Batch-Commit\", \"1\", hdr)\n\t\t\t\t}\n\t\t\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, \"uuid\", bseq, commit)\n\t\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\t\tmset.clseq++\n\t\t\t}\n\t\t\tmset.clMu.Unlock()\n\t\t\tn := mset.raftNode().(*raft)\n\t\t\tn.sendAppendEntry(entries)\n\n\t\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t\texpectedSeq := uint64(2)\n\t\t\tif i >= maxEntries {\n\t\t\t\texpectedSeq += uint64(maxEntries)\n\t\t\t}\n\t\t\trequire_Equal(t, pubAck.Sequence, expectedSeq)\n\t\t})\n\t}\n}\n\n// Test multiple sequential batches, the first batch is partially proposed.\nfunc TestJetStreamAtomicBatchPublishProposeMultiplePartialBatches(t *testing.T) {\n\ttest := func(t *testing.T, batchSize int, retry bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tbatchId1, batchId2 := \"ID_1\", \"ID_2\"\n\t\tif retry {\n\t\t\tbatchId1, batchId2 = \"uuid\", \"uuid\"\n\t\t}\n\n\t\tvar entries []*Entry\n\t\tmset.clMu.Lock()\n\t\tmsg := []byte(\"hello\")\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", batchId1)\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"1\", hdr)\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, batchId1, 1, false)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\n\t\tfor j := range batchSize {\n\t\t\tbseq := uint64(j + 1)\n\t\t\thdr = genHeader(nil, \"Nats-Batch-Id\", batchId2)\n\t\t\thdr = setHeader(\"Nats-Batch-Sequence\", strconv.FormatUint(bseq, 10), hdr)\n\t\t\tcommit := bseq == uint64(batchSize)\n\t\t\tif commit {\n\t\t\t\thdr = setHeader(\"Nats-Batch-Commit\", \"1\", hdr)\n\t\t\t}\n\t\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, msg, mset.clseq, 0, false, batchId2, bseq, commit)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tmset.clMu.Unlock()\n\t\tn := mset.raftNode().(*raft)\n\t\tn.sendAppendEntry(entries)\n\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, uint64(2+batchSize))\n\n\t\t// Validate the stream only committed the full batch.\n\t\trsm, err := js.GetMsg(\"TEST\", 1)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, rsm.Header.Get(\"Nats-Batch-Id\") == _EMPTY_)\n\t\tfor j := range batchSize {\n\t\t\trsm, err = js.GetMsg(\"TEST\", uint64(2+j))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Id\"), batchId2)\n\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Batch-Sequence\"), strconv.Itoa(j+1))\n\t\t}\n\t\trsm, err = js.GetMsg(\"TEST\", uint64(2+batchSize))\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, rsm.Header.Get(\"Nats-Batch-Id\") == _EMPTY_)\n\n\t\tmset.clMu.Lock()\n\t\tclfs := mset.clfs\n\t\tmset.clMu.Unlock()\n\t\trequire_Equal(t, clfs, 1)\n\t}\n\tfor _, retry := range []bool{false, true} {\n\t\tfor i := range 2 {\n\t\t\tbatchSize := i + 1\n\t\t\ttitle := fmt.Sprintf(\"B-%d\", batchSize)\n\t\t\tif retry {\n\t\t\t\ttitle += \"/Retry\"\n\t\t\t} else {\n\t\t\t\ttitle += \"/NoRetry\"\n\t\t\t}\n\t\t\tt.Run(title, func(t *testing.T) {\n\t\t\t\ttest(t, batchSize, retry)\n\t\t\t})\n\t\t}\n\t}\n}\n\n// Test a continuous flow of batches spanning multiple append entries can still move applied up.\n// Also, test a server can become leader if the previous leader left it with a partial batch.\nfunc TestJetStreamAtomicBatchPublishContinuousBatchesStillMoveAppliedUp(t *testing.T) {\n\ttest := func(t *testing.T, largePayload bool) {\n\t\tvar payload []byte\n\t\tif largePayload {\n\t\t\t// Create a large payload that can be easily compressed.\n\t\t\tpayload = make([]byte, 2*compressThreshold)\n\t\t}\n\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tn := mset.raftNode().(*raft)\n\t\tindex, commit, applied := n.Progress()\n\n\t\t// The first batch spans two append entries, but commits.\n\t\tmset.clMu.Lock()\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"ID_1\")\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"1\", hdr)\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, payload, mset.clseq, 0, false, \"ID_1\", 1, false)\n\t\tmset.clseq++\n\t\tmset.clMu.Unlock()\n\t\tn.sendAppendEntry([]*Entry{newEntry(EntryNormal, esm)})\n\n\t\tvar entries []*Entry\n\t\tmset.clMu.Lock()\n\t\thdr = genHeader(nil, \"Nats-Batch-Id\", \"ID_1\")\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"2\", hdr)\n\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, payload, mset.clseq, 0, false, \"ID_1\", 2, false)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\n\t\thdr = genHeader(nil, \"Nats-Batch-Id\", \"ID_1\")\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"3\", hdr)\n\t\thdr = setHeader(\"Nats-Batch-Commit\", \"1\", hdr)\n\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, payload, mset.clseq, 0, false, \"ID_1\", 3, true)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clseq++\n\n\t\t// The second batch doesn't commit.\n\t\thdr = genHeader(nil, \"Nats-Batch-Id\", \"ID_2\")\n\t\thdr = setHeader(\"Nats-Batch-Sequence\", \"1\", hdr)\n\t\tesm = encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, payload, mset.clseq, 0, false, \"ID_2\", 1, false)\n\t\tmset.clseq++\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\tmset.clMu.Unlock()\n\t\tn.sendAppendEntry(entries)\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tn.RLock()\n\t\t\tnindex, ncommit, processed, napplied := n.pindex, n.commit, n.processed, n.applied\n\t\t\tn.RUnlock()\n\t\t\tif nindex == index {\n\t\t\t\treturn errors.New(\"index not updated\")\n\t\t\t} else if ncommit == commit {\n\t\t\t\treturn errors.New(\"commit not updated\")\n\t\t\t} else if napplied == applied {\n\t\t\t\treturn errors.New(\"applied not updated\")\n\t\t\t} else if napplied == ncommit {\n\t\t\t\treturn errors.New(\"applied should not be able to equal commit yet\")\n\t\t\t} else if processed != ncommit {\n\t\t\t\treturn errors.New(\"must have processed all commits\")\n\t\t\t}\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Followers are now stranded with a partial batch, one needs to become leader\n\t\t// and have the batch be rejected since it was partially proposed.\n\t\tsl.Shutdown()\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Confirm the last batch gets rejected, and we are still able to publish with quorum.\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 5)\n\n\t\tc.restartServer(sl)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Publish again, now with all servers online.\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 6)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\n\tt.Run(\"Normal\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"Compressed\", func(t *testing.T) { test(t, true) })\n}\n\n// Test a batch that's committed but only partially applied, and the server gets hard killed.\n// Upon recovery the server should skip the messages that were already applied but store the\n// messages that weren't applied yet.\nfunc TestJetStreamAtomicBatchPublishPartiallyAppliedBatchOnRecovery(t *testing.T) {\n\ttest := func(t *testing.T, largePayload bool) {\n\t\tvar payload []byte\n\t\texpectedOp := streamMsgOp\n\t\tif largePayload {\n\t\t\t// Create a large payload that can be easily compressed.\n\t\t\tpayload = make([]byte, 2*compressThreshold)\n\t\t\texpectedOp = compressedStreamMsgOp\n\t\t}\n\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Pause applies on all followers.\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, fmset.raftNode().PauseApply())\n\t\t}\n\n\t\t// Publish a batch.\n\t\tvar entries []*Entry\n\t\tmset.clMu.Lock()\n\t\tfor seq := uint64(1); seq <= 4; seq++ {\n\t\t\tbatchCommit := seq == 4\n\t\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"ID\")\n\t\t\thdr = genHeader(hdr, \"Nats-Batch-Sequence\", strconv.FormatUint(seq, 10))\n\t\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, hdr, payload, mset.clseq, 0, false, \"ID\", seq, batchCommit)\n\t\t\tif _, _, bop, _, err := decodeBatchMsg(esm[1:]); err != nil || bop != expectedOp {\n\t\t\t\tmset.clMu.Unlock()\n\t\t\t\tt.Fatalf(\"Unexpected batch op %v: %v\", bop, err)\n\t\t\t}\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tmset.clseq++\n\t\t}\n\t\tmset.clMu.Unlock()\n\n\t\t// Wait for the leader to have applied the batch, so the next\n\t\t// publish informs the followers that it can be committed.\n\t\tn := mset.raftNode().(*raft)\n\t\t_, _, applied := n.Progress()\n\t\tn.sendAppendEntry(entries)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t_, _, napplied := n.Progress()\n\t\t\tif applied == napplied {\n\t\t\t\treturn errors.New(\"batch hasn't been applied yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Publish so the commit moves up on all servers.\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 6)\n\n\t\t// Simulate this follower only partially applied the batch and then got hard killed.\n\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\tmset, err = rs.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tts := time.Now().UnixNano()\n\t\thdr := genHeader(nil, \"Nats-Batch-Id\", \"ID\")\n\t\thdr = genHeader(hdr, \"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, mset.store.StoreRawMsg(\"foo\", hdr, nil, 2, ts, 0, false))\n\n\t\thdr = genHeader(nil, \"Nats-Batch-Id\", \"ID\")\n\t\thdr = genHeader(hdr, \"Nats-Batch-Sequence\", \"2\")\n\t\trequire_NoError(t, mset.store.StoreRawMsg(\"foo\", hdr, nil, 3, ts, 0, false))\n\n\t\t// Unpause applies on the remaining follower.\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl || s == rs {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tfmset.raftNode().ResumeApply()\n\t\t}\n\n\t\t// Restart the \"hard killed\" follower, should apply the remainder of the batch upon recovery.\n\t\trs.Shutdown()\n\t\tc.restartServer(rs)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\n\tt.Run(\"Normal\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"Compressed\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamRollupIsolatedRead(t *testing.T) {\n\tconst (\n\t\tDirectGet = iota\n\t\tDirectBatchGet\n\t\tDirectMultiGet\n\t\tConsumer\n\t)\n\n\ttest := func(t *testing.T, mode int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tAllowRollup: true,\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\t// Reconnect to the selected server.\n\t\tnc.Close()\n\t\tnc, js = jsClientConnect(t, rs)\n\t\tdefer nc.Close()\n\n\t\tm := nats.NewMsg(\"foo.0\")\n\t\tpubAck, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// mset.processJetStreamMsg will first store the new message, then update dedupe state, and then do the rollup.\n\t\t// We block the replica such that it can store the new message but can't do the rollup yet.\n\t\t// A read should wait for the rollup to complete.\n\t\tmset.ddMu.Lock()\n\t\tm.Subject = \"foo.1\"\n\t\tm.Header.Set(JSMsgRollup, JSMsgRollupAll)\n\t\tm.Header.Set(JSMsgId, \"msgId\")\n\t\t_, _ = js.PublishMsg(m)\n\t\terr = checkForErr(2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tvar state StreamState\n\t\t\tmset.store.FastState(&state)\n\t\t\tif state.LastSeq != 2 {\n\t\t\t\treturn fmt.Errorf(\"expected last seq 2, got: %d\", state.LastSeq)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\tmset.ddMu.Unlock()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// We're now subscribing and going to do a request, while the write is still halfway.\n\t\tsub, err := nc.SubscribeSync(\"reply\")\n\t\tif err != nil {\n\t\t\tmset.ddMu.Unlock()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tdefer sub.Drain()\n\t\tif err = nc.Flush(); err != nil {\n\t\t\tmset.ddMu.Unlock()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Run two goroutines, one will unblock the first write after a short sleep,\n\t\t// the second will do the read/consumer request.\n\t\tvar (\n\t\t\tready sync.WaitGroup\n\t\t\trun sync.WaitGroup\n\t\t\tcleanup sync.WaitGroup\n\t\t)\n\t\tready.Add(2)\n\t\trun.Add(1)\n\t\tcleanup.Add(1)\n\t\tgo func() {\n\t\t\tready.Done()\n\t\t\tdefer cleanup.Done()\n\t\t\trun.Wait()\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\tmset.ddMu.Unlock()\n\t\t}()\n\t\tgo func() {\n\t\t\tready.Done()\n\t\t\trun.Wait()\n\t\t\tswitch mode {\n\t\t\tcase DirectGet:\n\t\t\t\tmset.getDirectRequest(&JSApiMsgGetRequest{LastFor: \"foo.0\"}, \"reply\")\n\t\t\tcase DirectBatchGet:\n\t\t\t\tmset.getDirectRequest(&JSApiMsgGetRequest{NextFor: \"foo.*\", Batch: 2}, \"reply\")\n\t\t\tcase DirectMultiGet:\n\t\t\t\tmset.getDirectRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"foo.*\"}, Batch: 2}, \"reply\")\n\t\t\tcase Consumer:\n\t\t\t\tmset.addConsumer(&ConsumerConfig{DeliverSubject: \"reply\", Direct: true})\n\t\t\t}\n\t\t}()\n\n\t\t// Wait for both to be ready, then run both of them.\n\t\tready.Wait()\n\t\trun.Done()\n\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t// Make sure cleanup has happened before validating.\n\t\tcleanup.Wait()\n\t\trequire_NoError(t, err)\n\t\tif mode == DirectGet {\n\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\t\t} else if mode != Consumer {\n\t\t\trequire_Equal(t, msg.Header.Get(JSNumPending), \"0\")\n\t\t\trequire_Equal(t, msg.Header.Get(JSSequence), \"2\")\n\t\t} else {\n\t\t\t// $JS.ACK.......\n\t\t\trequire_True(t, strings.HasPrefix(msg.Reply, jsAckPre))\n\t\t\ttokens := strings.Split(msg.Reply, \".\")\n\t\t\trequire_Len(t, len(tokens), 9)\n\t\t\trequire_Equal(t, tokens[8], \"0\") // pending\n\t\t\trequire_Equal(t, tokens[5], \"2\") // sseq\n\t\t}\n\t}\n\n\tt.Run(\"DirectGet\", func(t *testing.T) { test(t, DirectGet) })\n\tt.Run(\"DirectBatchGet\", func(t *testing.T) { test(t, DirectBatchGet) })\n\tt.Run(\"DirectMultiGet\", func(t *testing.T) { test(t, DirectMultiGet) })\n\tt.Run(\"Consumer\", func(t *testing.T) { test(t, Consumer) })\n}\n\nfunc TestJetStreamAtomicBatchPublishAdvisories(t *testing.T) {\n\ttempl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {\n\t\tmax_mem_store: 2GB\n\t\tmax_file_store: 2GB\n\t\tstore_dir: '%s'\n\t\tlimits {\n\t\t\tbatch {\n\t\t\t\ttimeout: 2s\n\t\t\t}\n\t\t}\n\t}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterWithTemplate(t, templ, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t\tStorage: FileStorage,\n\t\t\tAllowAtomicPublish: true,\n\t\t}\n\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.*\", JSAdvisoryStreamBatchAbandonedPre))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\t\trequire_NoError(t, nc.Flush())\n\n\t\t// Should receive an advisory if gaps are detected.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"3\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar advisory JSStreamBatchAbandonedAdvisory\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.BatchId, \"uuid\")\n\t\trequire_Equal(t, advisory.Reason, BatchIncomplete)\n\n\t\t// Should receive an advisory if the batch is too large.\n\t\tcount := 1002\n\t\tfor seq := 1; seq <= count; seq++ {\n\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\t\tm.Header.Set(\"Nats-Batch-Sequence\", strconv.Itoa(seq))\n\t\t\tif seq != count {\n\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar pubAck JSPubAckResponse\n\t\t\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\t\t\trmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_NotNil(t, pubAck.Error)\n\t\t\trequire_Error(t, pubAck.Error, NewJSAtomicPublishTooLargeBatchError(streamDefaultMaxAtomicBatchSize))\n\t\t}\n\n\t\tmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tadvisory = JSStreamBatchAbandonedAdvisory{}\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.BatchId, \"uuid\")\n\t\trequire_Equal(t, advisory.Reason, BatchLarge)\n\n\t\t// Should receive an advisory if the batch times out and is abandoned.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\tmsg, err = sub.NextMsg(3 * time.Second)\n\t\trequire_NoError(t, err)\n\t\tadvisory = JSStreamBatchAbandonedAdvisory{}\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.BatchId, \"uuid\")\n\t\trequire_Equal(t, advisory.Reason, BatchTimeout)\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\ttest(t, replicas)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAtomicBatchPublishExpectedSeq(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowAtomicPublish: true,\n\t})\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tclseqAndClfs := func() (uint64, uint64) {\n\t\tmset.clMu.Lock()\n\t\tdefer mset.clMu.Unlock()\n\t\treturn mset.clseq, mset.clfs\n\t}\n\n\tclseq, clfs := clseqAndClfs()\n\trequire_Equal(t, clseq, 0)\n\trequire_Equal(t, clfs, 0)\n\n\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n\n\tclseq, clfs = clseqAndClfs()\n\trequire_Equal(t, clseq, 0)\n\trequire_Equal(t, clfs, 0)\n\n\t// test expect last seq for standalone publish\n\t_, err = js.Publish(\"foo\", []byte(\"hello\"), nats.ExpectLastSequence(1))\n\trequire_NoError(t, err)\n\n\tclseq, clfs = clseqAndClfs()\n\trequire_Equal(t, clseq, 0)\n\trequire_Equal(t, clfs, 0)\n\n\t// now do a single msg batch with expect last seq\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(\"Nats-Expected-Last-Sequence\", \"2\")\n\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\tresp, err := nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\n\tvar pubAck JSPubAckResponse\n\trequire_NoError(t, json.Unmarshal(resp.Data, &pubAck))\n\tif pubAck.Error != nil {\n\t\tt.Fatalf(\"Commit error: %v\", pubAck.Error)\n\t}\n\trequire_Equal(t, pubAck.Sequence, 3)\n\trequire_Equal(t, pubAck.BatchSize, 1)\n\n\tclseq, clfs = clseqAndClfs()\n\trequire_Equal(t, clseq, 0)\n\trequire_Equal(t, clfs, 0)\n}\n\n// Test two different batches within the same append entry.\nfunc TestJetStreamAtomicBatchPublishPartialBatchInSharedAppendEntry(t *testing.T) {\n\ttest := func(t *testing.T, commit bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 3,\n\t\t\tAllowAtomicPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tjs := sl.getJetStream()\n\t\tesm1 := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, nil, []byte(\"b1\"), 0, 0, false, \"b1\", 1, false)\n\t\tesm2 := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, nil, []byte(\"b2\"), 1, 0, false, \"b2\", 1, commit)\n\t\tce := newCommittedEntry(2, []*Entry{\n\t\t\tnewEntry(EntryNormal, esm1),\n\t\t\tnewEntry(EntryNormal, esm2),\n\t\t})\n\t\t_, err = js.applyStreamEntries(mset, ce, false)\n\t\trequire_NoError(t, err)\n\n\t\tmset.mu.RLock()\n\t\tbatch := mset.batchApply\n\t\tmset.mu.RUnlock()\n\t\tbatch.mu.Lock()\n\t\tentryStart, maxApplied := batch.entryStart, batch.maxApplied\n\t\tbatch.mu.Unlock()\n\n\t\tif !commit {\n\t\t\trequire_Equal(t, entryStart, 1)\n\t\t\trequire_Equal(t, maxApplied, 1)\n\t\t\treturn\n\t\t}\n\t\trequire_Equal(t, entryStart, 0)\n\t\trequire_Equal(t, maxApplied, 0)\n\t\trequire_Equal(t, mset.lastSeq(), 1)\n\n\t\tvar smv StoreMsg\n\t\tsm, err := mset.store.LoadMsg(1, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, string(sm.buf), \"b2\")\n\t}\n\n\tt.Run(\"NoCommit\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"Commit\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamAtomicBatchPublishRejectPartialBatchOnLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc := clientConnectToServer(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tAllowAtomicPublish: true,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Propose a partial batch that doesn't commit.\n\tvar entries []*Entry\n\tfor i := range uint64(10) {\n\t\tesm := encodeStreamMsgAllowCompressAndBatch(\"foo\", _EMPTY_, nil, nil, i, 0, false, \"b\", i+1, false)\n\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t}\n\tn := mset.raftNode()\n\trequire_NoError(t, n.ProposeMulti(entries))\n\n\t// Wait for all servers to have received and populated the partial batch.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset2, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset2.mu.RLock()\n\t\t\tbatch := mset2.batchApply\n\t\t\tmset2.mu.RUnlock()\n\t\t\tif batch == nil {\n\t\t\t\treturn fmt.Errorf(\"expected batch to be set\")\n\t\t\t}\n\t\t\tbatch.mu.Lock()\n\t\t\tcount := batch.count\n\t\t\tbatch.mu.Unlock()\n\t\t\tif count != 10 {\n\t\t\t\treturn fmt.Errorf(\"expected batch count to be 10, got: %d\", count)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Trigger a leader change and ensure the CLFS gets accounted for.\n\trequire_NoError(t, n.StepDown())\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif clfs := mset.getCLFS(); clfs != 10 {\n\t\t\t\treturn fmt.Errorf(\"expected clfs to be 10, got: %d\", clfs)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamAtomicBatchPublishPersistModeAsync(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 1,\n\t\tAllowAtomicPublish: true,\n\t\tPersistMode: AsyncPersistMode,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is not supported with atomic batch publish\")))\n}\n\nfunc TestJetStreamAtomicBatchPublishExpectedLastSubjectSequence(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 1,\n\t\tAllowAtomicPublish: true,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo.A\", nil)\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo.B\", nil)\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo.A\")\n\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\tm.Header.Set(\"Nats-Expected-Last-Sequence\", \"2\")\n\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"1\")\n\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence-Subject\", \"foo.A\")\n\tmsg, err := nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msg.Data), 0) // Empty ack.\n\n\tm = nats.NewMsg(\"foo.B\")\n\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"2\")\n\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence-Subject\", \"foo.B\")\n\tmsg, err = nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSPubAckResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_True(t, resp.Error == nil)\n\trequire_Equal(t, resp.PubAck.Sequence, 4)\n\trequire_Equal(t, resp.PubAck.BatchId, \"uuid\")\n\trequire_Equal(t, resp.PubAck.BatchSize, 2)\n}\n\nfunc TestJetStreamAtomicBatchPublishCommitUnsupported(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t\tReplicas: 1,\n\t\tAllowAtomicPublish: true,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tvar resp JSPubAckResponse\n\tfor _, unsupportedCommit := range []string{\"\", \"unsupported\", \"0\"} {\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\t\t_, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tm.Header.Set(\"Nats-Batch-Sequence\", \"2\")\n\t\tm.Header.Set(\"Nats-Batch-Commit\", unsupportedCommit)\n\t\tmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tresp = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\trequire_True(t, resp.Error != nil)\n\t\trequire_Error(t, resp.Error, NewJSAtomicPublishInvalidBatchCommitError())\n\n\t\t// Confirm no batches are left.\n\t\tmset.mu.RLock()\n\t\tbatches := mset.batches\n\t\tmset.mu.RUnlock()\n\t\tbatches.mu.Lock()\n\t\tgroups := len(batches.atomic)\n\t\tbatches.mu.Unlock()\n\t\trequire_Len(t, groups, 0)\n\t}\n\n\t// The required API level should allow the batch to be rejected.\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(\"Nats-Batch-Id\", \"uuid\")\n\tm.Header.Set(\"Nats-Batch-Sequence\", \"1\")\n\tm.Header.Set(\"Nats-Batch-Commit\", \"1\")\n\tm.Header.Set(\"Nats-Required-Api-Level\", strconv.Itoa(math.MaxInt))\n\tmsg, err := nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\tresp = JSPubAckResponse{}\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_True(t, resp.Error != nil)\n\trequire_Error(t, resp.Error, NewJSRequiredApiLevelError())\n\n\t// If required API level check passes, the header should be stripped.\n\tm.Header.Set(\"Nats-Required-Api-Level\", \"0\")\n\tmsg, err = nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\tresp = JSPubAckResponse{}\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_True(t, resp.Error == nil)\n\trequire_Equal(t, resp.PubAck.Sequence, 1)\n\n\tsm, err := mset.getMsg(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(sliceHeader(JSRequiredApiLevel, sm.Header)), 0)\n}\n\nfunc generateFastBatchReply(inbox string, batchId string, batchSeq uint64, flow uint16, gap string, op int) string {\n\treturn fmt.Sprintf(\"%s.%s.%d.%s.%d.%d.$FI\", inbox, batchId, flow, gap, batchSeq, op)\n}\n\nfunc TestJetStreamFastBatchPublish(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\tstorage StorageType,\n\t\tretention RetentionPolicy,\n\t\treplicas int,\n\t) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tStorage: storage,\n\t\t\tRetention: retention,\n\t\t\tReplicas: replicas,\n\t\t}\n\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tm := nats.NewMsg(\"foo.0\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 0, 0, FastBatchGapFail, FastBatchOpStart)\n\t\tm.Data = []byte(\"foo.0\")\n\n\t\t// Publish with batch publish disabled.\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar pubAck JSPubAckResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_NotNil(t, pubAck.Error)\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishDisabledError())\n\n\t\t// Enable batch publish.\n\t\tcfg.AllowBatchPublish = true\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Publish with incorrect batch sequence errors.\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 0, 0, FastBatchGapFail, FastBatchOpStart)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishInvalidPatternError())\n\n\t\t// A batch ID must not exceed the maximum length.\n\t\tlongBatchId := strings.Repeat(\"A\", 65)\n\t\tm.Reply = generateFastBatchReply(inbox, longBatchId, 1, 0, FastBatchGapFail, FastBatchOpStart)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_NotNil(t, pubAck.Error)\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishInvalidBatchIDError())\n\n\t\t// Publish a batch, misses start.\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 0, FastBatchGapFail, FastBatchOpAppend)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishUnknownBatchIDError())\n\n\t\t// Publish a \"batch\" which immediately commits.\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t// The PubAck comes in immediately without an intermediate BatchFlowAck.\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\n\t\t// Publish a batch of N messages.\n\t\tfor seq, batch := uint64(1), uint64(5); seq <= batch; seq++ {\n\t\t\tm.Subject = fmt.Sprintf(\"foo.%d\", seq)\n\t\t\tm.Data = []byte(m.Subject)\n\t\t\tif seq == batch {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\t\t} else if seq == 1 {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpStart)\n\t\t\t} else {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpAppend)\n\t\t\t}\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t\t// Can already pre-check receiving the first flow control message.\n\t\t\tif seq == 1 {\n\t\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, strings.HasPrefix(string(rmsg.Data), \"{\\\"type\\\":\\\"ack\\\",\"))\n\t\t\t\tvar batchFlowAck BatchFlowAck\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\t\t\trequire_Equal(t, batchFlowAck.Messages, 10)\n\t\t\t\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\t\t\t}\n\t\t}\n\t\t// Should receive the PubAck upon commit.\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Equal(t, pubAck.Sequence, 6)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 5)\n\n\t\t// Validate stream contents.\n\t\tif retention != InterestPolicy {\n\t\t\tfor i := 0; i < 6; i++ {\n\t\t\t\trsm, err := js.GetMsg(\"TEST\", uint64(i+1))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\trequire_Equal(t, rsm.Subject, subj)\n\t\t\t\trequire_Equal(t, string(rsm.Data), subj)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, retention := range []RetentionPolicy{LimitsPolicy, InterestPolicy, WorkQueuePolicy} {\n\t\t\tfor _, replicas := range []int{1, 3} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/R%d\", storage, retention, replicas), func(t *testing.T) {\n\t\t\t\t\ttest(t, storage, retention, replicas)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishGapDetection(t *testing.T) {\n\ttest := func(\n\t\tt *testing.T,\n\t\treplicas int,\n\t\tgapMode string,\n\t) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, gapMode, FastBatchOpStart)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tif gapMode == \"unknown\" {\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_NotNil(t, pubAck.Error)\n\t\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishInvalidPatternError())\n\t\t\treturn\n\t\t}\n\n\t\tbatchFlowAck = BatchFlowAck{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\trequire_Equal(t, batchFlowAck.Messages, 10)\n\t\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\n\t\t// Now a message is missed and a gap should be detected.\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 3, 0, gapMode, FastBatchOpAppend)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\t// There will always be a flow control message with the missed sequences.\n\t\trequire_True(t, strings.HasPrefix(string(rmsg.Data), \"{\\\"type\\\":\\\"gap\\\",\"))\n\t\tvar batchFlowGap BatchFlowGap\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowGap))\n\t\trequire_Equal(t, batchFlowGap.ExpectedLastSequence, 1)\n\t\trequire_Equal(t, batchFlowGap.CurrentSequence, 3)\n\n\t\tswitch gapMode {\n\t\tcase FastBatchGapFail:\n\t\t\t// By default, if a gap is detected, the batch is rejected.\n\t\t\t// A PubAck is returned with the data that has been persisted up to that point.\n\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\t\tcase FastBatchGapOk:\n\t\t\t// If a gap is ok, the batch will continue to function.\n\t\t\t// An EOB commit should get us the PubAck for the third message.\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 4, 0, gapMode, FastBatchOpCommitEob)\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, 2)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 3)\n\t\tdefault:\n\t\t\tt.Fatalf(\"unexpected gap mode: %q\", gapMode)\n\t\t}\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, gapMode := range []string{\"fail\", \"ok\", \"unknown\"} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, gapMode), func(t *testing.T) {\n\t\t\t\ttest(t, replicas, gapMode)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishFlowControl(t *testing.T) {\n\ttempl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {\n\t\tmax_mem_store: 2GB\n\t\tmax_file_store: 2GB\n\t\tstore_dir: '%s'\n\t\tlimits {\n\t\t\tbatch {\n\t\t\t\ttimeout: 750ms\n\t\t\t}\n\t\t}\n\t}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterWithTemplate(t, templ, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tm := nats.NewMsg(\"foo\")\n\t\tlseq := uint64(5)\n\t\tfor seq := uint64(1); seq <= lseq; seq++ {\n\t\t\tif seq == lseq {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpCommit)\n\t\t\t} else if seq == 1 {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpStart)\n\t\t\t} else {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpAppend)\n\t\t\t}\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t\tif seq == 1 || seq%2 == 0 {\n\t\t\t\trmsg, err := sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tbatchFlowAck = BatchFlowAck{}\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\t\t\tif seq > 1 {\n\t\t\t\t\trequire_Equal(t, batchFlowAck.Sequence, seq)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\t\t\t\t}\n\t\t\t\trequire_Equal(t, batchFlowAck.Messages, 2)\n\t\t\t} else if seq == lseq {\n\t\t\t\trmsg, err := sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tpubAck = JSPubAckResponse{}\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 5)\n\t\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\t\trequire_Equal(t, pubAck.BatchSize, 5)\n\t\t\t}\n\n\t\t\t// Sleep between messages such that we'll go over the batch timeout.\n\t\t\t// New messages being received should receive the timer.\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t}\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\ttest(t, replicas)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishSourceAndMirror(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowBatchPublish: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor seq := uint64(1); seq <= 3; seq++ {\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tif seq == 1 {\n\t\t\t\tm.Reply = generateFastBatchReply(\"\", \"uuid\", seq, 10, FastBatchGapFail, FastBatchOpStart)\n\t\t\t} else {\n\t\t\t\tm.Reply = generateFastBatchReply(\"\", \"uuid\", seq, 10, FastBatchGapFail, FastBatchOpAppend)\n\t\t\t}\n\t\t\tcommit := seq == 3\n\t\t\tif !commit {\n\t\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tinbox := nats.NewInbox()\n\t\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Drain()\n\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 10, FastBatchGapFail, FastBatchOpCommit)\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t\trmsg, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar pubAck JSPubAckResponse\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, 3)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 3)\n\t\t}\n\n\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 2))\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Mirror can source batched messages but can't do fast batching itself.\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"M-no-batch\",\n\t\t\tStorage: FileStorage,\n\t\t\tMirror: &StreamSource{Name: \"TEST\"},\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_Error(t, err, NewJSMirrorWithBatchPublishError())\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"S\",\n\t\t\tStorage: FileStorage,\n\t\t\tSources: []*StreamSource{{Name: \"TEST\"}},\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, name := range []string{\"M\", \"S\"} {\n\t\t\t\tif si, err := js.StreamInfo(name); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if si.State.Msgs != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 2 messages for stream %q, got %d\", name, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Ensure the batching headers were removed when ingested into the source/mirror.\n\t\trsm, err := js.GetMsg(\"M\", 1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 0)\n\n\t\trsm, err = js.GetMsg(\"M\", 3)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 0)\n\n\t\trsm, err = js.GetMsg(\"S\", 1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 1)\n\t\trequire_Equal(t, rsm.Header.Get(JSStreamSource), \"TEST 1 > > foo\")\n\n\t\trsm, err = js.GetMsg(\"S\", 2)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(rsm.Header), 1)\n\t\trequire_Equal(t, rsm.Header.Get(JSStreamSource), \"TEST 3 > > foo\")\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamFastBatchPublishDuplicates(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\t// Simulate storing a duplicate message.\n\t\tmsgId := \"msgId\"\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\trequire_NotNil(t, sl)\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tdseq := uint64(1)\n\t\t// If replicated, we simulate it being proposed still.\n\t\tif replicas > 1 {\n\t\t\tdseq = 0\n\t\t}\n\t\tmset.storeMsgId(&ddentry{msgId, dseq, 0})\n\n\t\t// Publish a \"batch\" which immediately commits.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(JSMsgId, msgId)\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t// The PubAck comes in immediately without an intermediate BatchFlowAck.\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\tif replicas == 1 {\n\t\t\trequire_True(t, pubAck.Error == nil)\n\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\t\trequire_True(t, pubAck.Duplicate)\n\t\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\t\t} else {\n\t\t\trequire_True(t, pubAck.Error != nil)\n\t\t\trequire_Equal(t, pubAck.Error.Error(), NewJSStreamDuplicateMessageConflictError().Error())\n\t\t}\n\t\tmset.mu.RLock()\n\t\tbatches := mset.batches\n\t\tbatches.mu.Lock()\n\t\tfastBatches := len(mset.batches.fast)\n\t\tbatches.mu.Unlock()\n\t\tmset.mu.RUnlock()\n\t\trequire_Len(t, fastBatches, 0)\n\n\t\t// Flow setting so we get one flow control message below.\n\t\tflow := uint16(3)\n\n\t\t// Publish a batch of N messages that are all duplicates, we expect\n\t\t// to receive both a flow control message and PubAck.\n\t\tfor seq, batch := uint64(1), uint64(5); seq <= batch; seq++ {\n\t\t\tif seq == batch {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, flow, FastBatchGapFail, FastBatchOpCommit)\n\t\t\t} else if seq == 1 {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, flow, FastBatchGapFail, FastBatchOpStart)\n\t\t\t} else {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, flow, FastBatchGapFail, FastBatchOpAppend)\n\t\t\t}\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t\t// Can already pre-check receiving the first flow control message.\n\t\t\tif seq == 1 {\n\t\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tbatchFlowAck = BatchFlowAck{}\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\t\t\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\t\t\t\trequire_Equal(t, batchFlowAck.Messages, flow)\n\t\t\t}\n\n\t\t\t// Expect one flow control message for this batch.\n\t\t\tif seq%uint64(flow) == 0 {\n\t\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tbatchFlowAck = BatchFlowAck{}\n\t\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\t\t\trequire_Equal(t, batchFlowAck.Sequence, 3)\n\t\t\t\trequire_Equal(t, batchFlowAck.Messages, flow)\n\t\t\t}\n\t\t}\n\t\t// Should receive the PubAck upon commit.\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.Sequence, 0)\n\t\trequire_False(t, pubAck.Duplicate)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 5)\n\n\t\tmset.mu.RLock()\n\t\tbatches = mset.batches\n\t\tbatches.mu.Lock()\n\t\tfastBatches = len(mset.batches.fast)\n\t\tbatches.mu.Unlock()\n\t\tmset.mu.RUnlock()\n\t\trequire_Len(t, fastBatches, 0)\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\ttest(t, replicas)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishDuplicatesCluster(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tvar batchFlowAck BatchFlowAck\n\tvar pubAck JSPubAckResponse\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowBatchPublish: true,\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tmsgId := \"msgId\"\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(JSMsgId, msgId)\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 4, FastBatchGapFail, FastBatchOpStart)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 4)\n\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.Lock()\n\tbatches := mset.batches\n\tmset.mu.Unlock()\n\trequire_NotNil(t, batches)\n\n\tbatches.mu.Lock()\n\tb := batches.fast[\"uuid\"]\n\n\t// Simulate a pending message that will take some time to be processed due to replication.\n\tb.lseq++\n\tb.pending++\n\n\t// Simulate a bunch of messages being marked as duplicate and not being accounted for.\n\t// These two would trigger the first ack.\n\tb.lseq += 2\n\t// These would trigger the second ack.\n\tb.lseq += 4\n\n\t// Now simulate the second message we published is processed.\n\treply := generateFastBatchReply(inbox, \"uuid\", 2, 4, FastBatchGapFail, FastBatchOpAppend)\n\tbatches.fastBatchRegisterSequences(mset, reply, 2, true, &FastBatch{id: \"uuid\", seq: 2})\n\tbatches.mu.Unlock()\n\n\t// Normally we'd receive two flow control messages, but since both were triggered due to\n\t// the above process finishing, we only send one out to save bandwidth.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 4)\n\trequire_Equal(t, batchFlowAck.Sequence, 8)\n\n\t// Now commit the batch (but this message is also a duplicate).\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 9, 0, FastBatchGapFail, FastBatchOpCommit)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tpubAck = JSPubAckResponse{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\trequire_True(t, pubAck.Error == nil)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\trequire_False(t, pubAck.Duplicate)\n\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\trequire_Equal(t, pubAck.BatchSize, 9)\n}\n\nfunc TestJetStreamFastBatchPublishDuplicatesEobCommit(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgId := \"msgId\"\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(JSMsgId, msgId)\n\t\tfor seq := uint64(1); seq <= 2; seq++ {\n\t\t\tif seq == 1 {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpStart)\n\t\t\t} else {\n\t\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpAppend)\n\t\t\t}\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\t}\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tbatchFlowAck = BatchFlowAck{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\trequire_Equal(t, batchFlowAck.Messages, 10)\n\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 3, 0, FastBatchGapFail, FastBatchOpCommitEob)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 2)\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\ttest(t, replicas)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishHeaderCheckError(t *testing.T) {\n\ttest := func(t *testing.T, replicas int, gapMode string) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\t// Send the first message.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, gapMode, FastBatchOpStart)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t// Add a header that will require returning an error.\n\t\tm.Header.Set(JSExpectedLastSeq, \"100\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 0, gapMode, FastBatchOpAppend)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t// The first message triggered the initial flow control message.\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tbatchFlowAck = BatchFlowAck{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\trequire_Equal(t, batchFlowAck.Messages, 10)\n\n\t\t// The second message should always report the error based on the header.\n\t\t// But it should report about this error on that sequence.\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, strings.HasPrefix(string(rmsg.Data), \"{\\\"type\\\":\\\"err\\\",\"))\n\t\tvar batchFlowErr BatchFlowErr\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowErr))\n\t\trequire_Equal(t, batchFlowErr.Sequence, 2)\n\t\trequire_True(t, batchFlowErr.Error != nil)\n\t\trequire_Equal(t, batchFlowErr.Error.Error(), NewJSStreamWrongLastSequenceError(1).Error())\n\n\t\tswitch gapMode {\n\t\tcase FastBatchGapFail:\n\t\t\t// Nothing to do, we simply wait for the PubAck below.\n\t\t\tbreak\n\t\tcase FastBatchGapOk:\n\t\t\t// Commit the batch.\n\t\t\tm.Header.Del(JSExpectedLastSeq)\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 3, 0, gapMode, FastBatchOpCommitEob)\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected gap mode: %q\", gapMode)\n\t\t}\n\n\t\t// We always expect a PubAck containing the last persisted message data.\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\tswitch {\n\t\tcase gapMode == FastBatchGapFail:\n\t\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\t\tcase gapMode == FastBatchGapOk:\n\t\t\trequire_Equal(t, pubAck.BatchSize, 2)\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected gap mode: %q\", gapMode)\n\t\t}\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, gapMode := range []string{FastBatchGapFail, FastBatchGapOk} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, gapMode), func(t *testing.T) {\n\t\t\t\ttest(t, replicas, gapMode)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamFastBatchPublishPing(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tvar batchFlowAck BatchFlowAck\n\tvar pubAck JSPubAckResponse\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowBatchPublish: true,\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t// Send the first message.\n\tm := nats.NewMsg(\"foo\")\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 2, FastBatchGapOk, FastBatchOpStart)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\n\t// Send the second message that will trigger the flow control message.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 2, FastBatchGapOk, FastBatchOpAppend)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\t// A ping will resend the flow control message.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 2, FastBatchGapOk, FastBatchOpPing)\n\trequire_NoError(t, nc.PublishMsg(m))\n\tfor i := range 2 {\n\t\tif i > 0 {\n\t\t\trmsg, err = sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tbatchFlowAck = BatchFlowAck{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\trequire_Equal(t, batchFlowAck.Messages, 2)\n\t\trequire_Equal(t, batchFlowAck.Sequence, 2)\n\t}\n\n\t// Simulate losing one message and a ping making us aware of the gap.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 3, 2, FastBatchGapOk, FastBatchOpPing)\n\trequire_NoError(t, nc.PublishMsg(m))\n\t// First, we get the report of the detected gap.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar batchFlowGap BatchFlowGap\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowGap))\n\trequire_Equal(t, batchFlowGap.ExpectedLastSequence, 3)\n\trequire_Equal(t, batchFlowGap.CurrentSequence, 4)\n\t// Second, we get the last flow control message, since we might have missed that before.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 2)\n\n\t// Send another message.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 4, 2, FastBatchGapOk, FastBatchOpAppend)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 4)\n\n\t// Simulate losing many messages and a ping making us aware of the gap as well as updating flow control.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 100, 2, FastBatchGapOk, FastBatchOpPing)\n\trequire_NoError(t, nc.PublishMsg(m))\n\t// First, we get the report of the detected gap.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowGap = BatchFlowGap{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowGap))\n\trequire_Equal(t, batchFlowGap.ExpectedLastSequence, 5)\n\trequire_Equal(t, batchFlowGap.CurrentSequence, 101)\n\t// Second, we get the updated flow control message.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 100)\n\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 101, 2, FastBatchGapOk, FastBatchOpCommitEob)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tpubAck = JSPubAckResponse{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\trequire_True(t, pubAck.Error == nil)\n\trequire_Equal(t, pubAck.Sequence, 3)\n\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\trequire_Equal(t, pubAck.BatchSize, 100)\n}\n\nfunc TestJetStreamFastBatchSequentialDuplicateAndErrorPubAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowBatchPublish: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish the first message.\n\tmsgId := \"msgId\"\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(JSMsgId, msgId)\n\t_, err = nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t// Send the first message, which is a duplicate.\n\tm.Header.Set(JSMsgId, msgId)\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 2, FastBatchGapFail, FastBatchOpStart)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar batchFlowAck BatchFlowAck\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\n\t// Send the second message which triggers an error and terminates the batch.\n\tm.Header.Del(JSMsgId)\n\tm.Header.Set(JSExpectedLastSeq, \"100\")\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 2, FastBatchGapFail, FastBatchOpAppend)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar batchFlowErr BatchFlowErr\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowErr))\n\trequire_Equal(t, batchFlowErr.Sequence, 2)\n\trequire_True(t, batchFlowErr.Error != nil)\n\trequire_Equal(t, batchFlowErr.Error.Error(), NewJSStreamWrongLastSequenceError(1).Error())\n\n\t// Lastly, we expect the PubAck.\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar pubAck JSPubAckResponse\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\trequire_True(t, pubAck.Error == nil)\n\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\trequire_Equal(t, pubAck.BatchSize, 1)\n\t// The batch only had duplicate messages.\n\t// Technically, we shouldn't return a zero-sequence, but it's unavoidable.\n\trequire_Equal(t, pubAck.Sequence, 0)\n}\n\nfunc TestJetStreamFastBatchPublishAccImportExport(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 1MB, max_file_store: 1MB, store_dir: %q}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: a, password: a} ]\n\t\t\t\texports [\n\t\t\t\t\t{ service: \"foo\" }\n\t\t\t\t]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tusers: [ {user: b, password: b} ]\n\t\t\t\timports [\n\t\t\t\t\t{ service: { subject: \"foo\", account: A } }\n\t\t\t\t]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"a\", \"a\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowBatchPublish: true,\n\t}\n\t_, err = jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tcheckFastBatchPublish := func(user string) {\n\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, user))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar pubAck JSPubAckResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_True(t, pubAck.Error == nil)\n\t\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\t\trequire_Equal(t, pubAck.BatchSize, 1)\n\t}\n\tcheckFastBatchPublish(\"a\")\n\tcheckFastBatchPublish(\"b\")\n}\n\nfunc TestJetStreamFastBatchPublishFlowControlOnLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc := clientConnectToServer(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tAllowBatchPublish: true,\n\t}\n\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tm := nats.NewMsg(\"foo\")\n\tfor seq := uint64(1); seq <= 3; seq++ {\n\t\tif seq == 1 {\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpStart)\n\t\t} else {\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpAppend)\n\t\t}\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t}\n\trmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar batchFlowAck BatchFlowAck\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 2)\n\n\t// Should receive a flow ack after a leader change. This sends us a ping, but also informs us which\n\t// messages were persisted after the leader change.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sl)\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_NoError(t, mset.raftNode().StepDown())\n\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 3)\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif err = checkState(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset.mu.RLock()\n\t\t\tvar batches int\n\t\t\tif mset.batches != nil {\n\t\t\t\tmset.batches.mu.Lock()\n\t\t\t\tbatches = len(mset.batches.fast)\n\t\t\t\tmset.batches.mu.Unlock()\n\t\t\t}\n\t\t\tmset.mu.RUnlock()\n\t\t\tif batches != 1 {\n\t\t\t\treturn fmt.Errorf(\"expected 1 batch on %s, got %d\", s.Name(), batches)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// We now expect a new flow ack after two messages.\n\tfor seq := uint64(4); seq <= 5; seq++ {\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 2, FastBatchGapFail, FastBatchOpAppend)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t}\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tbatchFlowAck = BatchFlowAck{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 2)\n\trequire_Equal(t, batchFlowAck.Sequence, 5)\n\n\t// Commit the final message. We expect that the followers also clean up their state.\n\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 6, 2, FastBatchGapFail, FastBatchOpCommit)\n\trequire_NoError(t, nc.PublishMsg(m))\n\trmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar pubAck JSPubAckResponse\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\trequire_True(t, pubAck.Error == nil)\n\trequire_Equal(t, pubAck.BatchId, \"uuid\")\n\trequire_Equal(t, pubAck.BatchSize, 6)\n\trequire_Equal(t, pubAck.Sequence, 6)\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif err = checkState(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset.mu.RLock()\n\t\t\tvar batches int\n\t\t\tif mset.batches != nil {\n\t\t\t\tmset.batches.mu.Lock()\n\t\t\t\tbatches = len(mset.batches.fast)\n\t\t\t\tmset.batches.mu.Unlock()\n\t\t\t}\n\t\t\tmset.mu.RUnlock()\n\t\t\tif batches > 0 {\n\t\t\t\treturn fmt.Errorf(\"expected no batches on %s, got %d\", s.Name(), batches)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamFastBatchPublishFlowControlOnLeaderChangeAfterFailedCommitProposal(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc := clientConnectToServer(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tAllowBatchPublish: true,\n\t}\n\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tm := nats.NewMsg(\"foo\")\n\tfor seq := uint64(1); seq <= 3; seq++ {\n\t\tif seq == 1 {\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpStart)\n\t\t} else {\n\t\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", seq, 0, FastBatchGapFail, FastBatchOpAppend)\n\t\t}\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t}\n\trmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar batchFlowAck BatchFlowAck\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\trequire_Equal(t, batchFlowAck.Messages, 10)\n\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\n\t// Should receive a flow ack after a leader change. This sends us a ping, but also informs us which\n\t// messages were persisted after the leader change.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sl)\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tmset.mu.Lock()\n\tif mset.batches == nil {\n\t\tmset.mu.Unlock()\n\t\tt.Fatal(\"batches map is nil\")\n\t}\n\tmset.batches.mu.Lock()\n\tb := mset.batches.fast[\"uuid\"]\n\tif b == nil {\n\t\tmset.batches.mu.Unlock()\n\t\tmset.mu.Unlock()\n\t\tt.Fatal(\"batch is nil\")\n\t}\n\t// Simulate one more pending message that commits, but the proposal didn't make it due to the leader change.\n\tb.commit = true\n\tb.pending++\n\tmset.batches.mu.Unlock()\n\tmset.mu.Unlock()\n\n\t// Change to any other leader.\n\trequire_NoError(t, mset.raftNode().StepDown())\n\n\t// Change back to the previous leader.\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tnl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, nl)\n\trequire_NotEqual(t, nl, sl)\n\tnmset, err := nl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_NoError(t, nmset.raftNode().StepDown(mset.raftNode().ID()))\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\trequire_Equal(t, c.streamLeader(globalAccountName, \"TEST\"), sl)\n\n\t// The previous leader should now have reset the committed flag as the proposal failed.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\t\tif mset.batches == nil {\n\t\t\treturn errors.New(\"batches map is nil\")\n\t\t}\n\t\tmset.batches.mu.Lock()\n\t\tdefer mset.batches.mu.Unlock()\n\t\tb = mset.batches.fast[\"uuid\"]\n\t\tif b == nil {\n\t\t\treturn errors.New(\"batch is nil\")\n\t\t}\n\t\tif b.pending != 0 {\n\t\t\treturn errors.New(\"pending isn't reset\")\n\t\t}\n\t\tif b.commit {\n\t\t\treturn errors.New(\"commit isn't reset\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamFastBatchPublishLimits(t *testing.T) {\n\tstreamMaxFastBatchInflightPerStream = 1\n\tstreamMaxFastBatchInflightTotal = 1\n\tstreamMaxBatchTimeout = 500 * time.Millisecond\n\tdefer func() {\n\t\tstreamMaxFastBatchInflightPerStream = streamDefaultMaxFastBatchInflightPerStream\n\t\tstreamMaxFastBatchInflightTotal = streamDefaultMaxFastBatchInflightTotal\n\t\tstreamMaxBatchTimeout = streamDefaultMaxBatchTimeout\n\t}()\n\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar batchFlowAck BatchFlowAck\n\t\tvar pubAck JSPubAckResponse\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"FOO\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t}\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// For testing total server-wide maximum inflight batches.\n\t\tcfg = &StreamConfig{\n\t\t\tName: \"BAR\",\n\t\t\tSubjects: []string{\"bar\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tAllowBatchPublish: true,\n\t\t}\n\t\t_, err = jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tinbox := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(fmt.Sprintf(\"%s.>\", inbox))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\t// A batch ID must not exceed the maximum length.\n\t\tfor _, length := range []int{64, 65} {\n\t\t\tlongBatchId := strings.Repeat(\"A\", length)\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Reply = generateFastBatchReply(inbox, longBatchId, 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\t\trmsg, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tpubAck = JSPubAckResponse{}\n\t\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\t\tif length <= 64 {\n\t\t\t\trequire_True(t, pubAck.Error == nil)\n\t\t\t} else {\n\t\t\t\trequire_NotNil(t, pubAck.Error)\n\t\t\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishInvalidBatchIDError())\n\t\t\t}\n\t\t}\n\n\t\t// One batch is inflight.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 1, 0, FastBatchGapFail, FastBatchOpStart)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\trmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tbatchFlowAck = BatchFlowAck{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &batchFlowAck))\n\t\trequire_Equal(t, batchFlowAck.Messages, 10)\n\t\trequire_Equal(t, batchFlowAck.Sequence, 0)\n\n\t\t// Another batch moves over the threshold and batch is denied.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"exceeds_threshold\", 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishTooManyInflightError())\n\n\t\t// Another batch on a different stream moves over the server-wide threshold and batch is denied.\n\t\tm = nats.NewMsg(\"bar\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"bar\", 1, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishTooManyInflightError())\n\n\t\t// The first batch should now time out.\n\t\tsl := c.streamLeader(globalAccountName, \"FOO\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"FOO\")\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset.mu.RLock()\n\t\t\tbatches := mset.batches\n\t\t\tmset.mu.RUnlock()\n\t\t\tif batches == nil {\n\t\t\t\treturn errors.New(\"batches not found\")\n\t\t\t}\n\t\t\tbatches.mu.Lock()\n\t\t\tgroups := len(batches.fast)\n\t\t\tbatches.mu.Unlock()\n\t\t\tif groups != 0 {\n\t\t\t\treturn fmt.Errorf(\"expected 0 groups, got %d\", groups)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Publishing to the batch should also error since it timed out.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Reply = generateFastBatchReply(inbox, \"uuid\", 2, 0, FastBatchGapFail, FastBatchOpCommit)\n\t\trequire_NoError(t, nc.PublishMsg(m))\n\n\t\trmsg, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tpubAck = JSPubAckResponse{}\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &pubAck))\n\t\trequire_Error(t, pubAck.Error, NewJSBatchPublishUnknownBatchIDError())\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n" }, { "path": "server/jetstream_benchmark_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2\n\npackage server\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/bits\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc BenchmarkJetStreamConsume(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tstreamName = \"S\"\n\t\tsubject = \"s\"\n\t\tseed = 12345\n\t\tpublishTimeout = 30 * time.Second\n\t\tPublishBatchSize = 10000\n\t)\n\n\trunSyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string) (int, int, int) {\n\t\tconst nextMsgTimeout = 3 * time.Second\n\n\t\tsubOpts := []nats.SubOpt{\n\t\t\tnats.BindStream(streamName),\n\t\t}\n\t\tsub, err := js.SubscribeSync(_EMPTY_, subOpts...)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Failed to subscribe: %v\", err)\n\t\t}\n\t\tdefer sub.Unsubscribe()\n\n\t\tbitset := NewBitset(uint64(b.N))\n\t\tuniqueConsumed, duplicates, errors := 0, 0, 0\n\n\t\tb.ResetTimer()\n\n\t\tfor uniqueConsumed < b.N {\n\t\t\tmsg, err := sub.NextMsg(nextMsgTimeout)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"No more messages (received: %d/%d)\", uniqueConsumed, b.N)\n\t\t\t}\n\n\t\t\tmetadata, mdErr := msg.Metadata()\n\t\t\tif mdErr != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tackErr := msg.Ack()\n\t\t\tif ackErr != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tseq := metadata.Sequence.Stream\n\n\t\t\tindex := seq - 1\n\t\t\tif bitset.get(index) {\n\t\t\t\tduplicates++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tuniqueConsumed++\n\t\t\tbitset.set(index, true)\n\n\t\t\tif verbose && uniqueConsumed%1000 == 0 {\n\t\t\t\tb.Logf(\"Consumed: %d/%d\", bitset.count(), b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\n\t\treturn uniqueConsumed, duplicates, errors\n\t}\n\n\trunAsyncPushConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, ordered, durable bool) (int, int, int) {\n\t\tconst timeout = 3 * time.Minute\n\t\tbitset := NewBitset(uint64(b.N))\n\t\tdoneCh := make(chan bool, 1)\n\t\tuniqueConsumed, duplicates, errors := 0, 0, 0\n\n\t\thandleMsg := func(msg *nats.Msg) {\n\t\t\tmetadata, mdErr := msg.Metadata()\n\t\t\tif mdErr != nil {\n\t\t\t\t// fmt.Printf(\"Metadata error: %v\\n\", mdErr)\n\t\t\t\terrors++\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Ordered defaults to AckNone policy, don't try to ACK\n\t\t\tif !ordered {\n\t\t\t\tackErr := msg.Ack()\n\t\t\t\tif ackErr != nil {\n\t\t\t\t\t// fmt.Printf(\"Ack error: %v\\n\", ackErr)\n\t\t\t\t\terrors++\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tseq := metadata.Sequence.Stream\n\n\t\t\tindex := seq - 1\n\t\t\tif bitset.get(index) {\n\t\t\t\tduplicates++\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tuniqueConsumed++\n\t\t\tbitset.set(index, true)\n\n\t\t\tif uniqueConsumed == b.N {\n\t\t\t\tmsg.Sub.Unsubscribe()\n\t\t\t\tdoneCh <- true\n\t\t\t}\n\t\t\tif verbose && uniqueConsumed%1000 == 0 {\n\t\t\t\tb.Logf(\"Consumed %d/%d\", uniqueConsumed, b.N)\n\t\t\t}\n\t\t}\n\n\t\tsubOpts := []nats.SubOpt{\n\t\t\tnats.BindStream(streamName),\n\t\t}\n\n\t\tif ordered {\n\t\t\tsubOpts = append(subOpts, nats.OrderedConsumer())\n\t\t}\n\n\t\tif durable {\n\t\t\tsubOpts = append(subOpts, nats.Durable(\"c\"))\n\t\t}\n\n\t\tsub, err := js.Subscribe(_EMPTY_, handleMsg, subOpts...)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Failed to subscribe: %v\", err)\n\t\t}\n\t\tdefer sub.Unsubscribe()\n\n\t\tb.ResetTimer()\n\n\t\tselect {\n\t\tcase <-doneCh:\n\t\t\tb.StopTimer()\n\t\tcase <-time.After(timeout):\n\t\t\tb.Fatalf(\"Timeout, %d/%d received, %d errors\", uniqueConsumed, b.N, errors)\n\t\t}\n\n\t\treturn uniqueConsumed, duplicates, errors\n\t}\n\n\trunPullConsumer := func(b *testing.B, js nats.JetStreamContext, streamName string, durable bool) (int, int, int) {\n\t\tconst fetchMaxWait = nats.MaxWait(3 * time.Second)\n\t\tconst fetchMaxMessages = 1000\n\n\t\tbitset := NewBitset(uint64(b.N))\n\t\tuniqueConsumed, duplicates, errors := 0, 0, 0\n\n\t\tsubOpts := []nats.SubOpt{\n\t\t\tnats.BindStream(streamName),\n\t\t}\n\n\t\tconsumerName := _EMPTY_ // Default ephemeral\n\t\tif durable {\n\t\t\tconsumerName = \"c\" // Durable\n\t\t}\n\n\t\tsub, err := js.PullSubscribe(\"\", consumerName, subOpts...)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Failed to subscribe: %v\", err)\n\t\t}\n\t\tdefer sub.Unsubscribe()\n\n\t\tb.ResetTimer()\n\n\tfetchLoop:\n\t\tfor {\n\t\t\tmsgs, err := sub.Fetch(fetchMaxMessages, fetchMaxWait)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatalf(\"Failed to fetch: %v\", err)\n\t\t\t}\n\n\t\tprocessMsgsLoop:\n\t\t\tfor _, msg := range msgs {\n\t\t\t\tmetadata, mdErr := msg.Metadata()\n\t\t\t\tif mdErr != nil {\n\t\t\t\t\terrors++\n\t\t\t\t\tcontinue processMsgsLoop\n\t\t\t\t}\n\n\t\t\t\tackErr := msg.Ack()\n\t\t\t\tif ackErr != nil {\n\t\t\t\t\terrors++\n\t\t\t\t\tcontinue processMsgsLoop\n\t\t\t\t}\n\n\t\t\t\tseq := metadata.Sequence.Stream\n\n\t\t\t\tindex := seq - 1\n\t\t\t\tif bitset.get(index) {\n\t\t\t\t\tduplicates++\n\t\t\t\t\tcontinue processMsgsLoop\n\t\t\t\t}\n\n\t\t\t\tuniqueConsumed++\n\t\t\t\tbitset.set(index, true)\n\n\t\t\t\tif uniqueConsumed == b.N {\n\t\t\t\t\tmsg.Sub.Unsubscribe()\n\t\t\t\t\tbreak fetchLoop\n\t\t\t\t}\n\n\t\t\t\tif verbose && uniqueConsumed%1000 == 0 {\n\t\t\t\t\tb.Logf(\"Consumed %d/%d\", uniqueConsumed, b.N)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\n\t\treturn uniqueConsumed, duplicates, errors\n\t}\n\n\ttype ConsumerType string\n\tconst (\n\t\tPushSync ConsumerType = \"PUSH[Sync,Ephemeral]\"\n\t\tPushAsync ConsumerType = \"PUSH[Async,Ephemeral]\"\n\t\tPushAsyncOrdered ConsumerType = \"PUSH[Async,Ordered]\"\n\t\tPushAsyncDurable ConsumerType = \"PUSH[Async,Durable]\"\n\t\tPullDurable ConsumerType = \"PULL[Durable]\"\n\t\tPullEphemeral ConsumerType = \"PULL[Ephemeral]\"\n\t)\n\n\tbenchmarksCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t\tmessageSize int\n\t\tminMessages int\n\t}{\n\t\t{1, 1, 10, 100_000}, // Single node, 10B messages, ~1MiB minimum\n\t\t{1, 1, 1024, 1_000}, // Single node, 1KB messages, ~1MiB minimum\n\t\t{3, 3, 10, 100_000}, // Cluster, R3, 10B messages, ~1MiB minimum\n\t\t{3, 3, 1024, 1_000}, // Cluster, R3, 1KB messages, ~1MiB minimum\n\t}\n\n\t//Each of the cases above is run with each of the consumer types\n\tconsumerTypes := []ConsumerType{\n\t\tPushSync,\n\t\tPushAsync,\n\t\tPushAsyncOrdered,\n\t\tPushAsyncDurable,\n\t\tPullDurable,\n\t\tPullEphemeral,\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\n\t\tname := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,MsgSz=%db\",\n\t\t\tbc.clusterSize,\n\t\t\tbc.replicas,\n\t\t\tbc.messageSize,\n\t\t)\n\n\t\tb.Run(\n\t\t\tname,\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, ct := range consumerTypes {\n\t\t\t\t\tname := fmt.Sprintf(\n\t\t\t\t\t\t\"%v\",\n\t\t\t\t\t\tct,\n\t\t\t\t\t)\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tname,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t// Skip short runs, benchmark gets re-executed with a larger N\n\t\t\t\t\t\t\tif b.N < bc.minMessages {\n\t\t\t\t\t\t\t\tb.ResetTimer()\n\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Running %s with %d messages\", name, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Setting up %d nodes\", bc.clusterSize)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tcl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Creating stream with R=%d\", bc.replicas)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tstreamConfig := &nats.StreamConfig{\n\t\t\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\t\t\tSubjects: []string{subject},\n\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif _, err := js.AddStream(streamConfig); err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating stream: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\tif bc.replicas > 1 {\n\t\t\t\t\t\t\t\tconnectURL := cl.streamLeader(\"$G\", streamName).ClientURL()\n\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\t_, js = jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tmessage := make([]byte, bc.messageSize)\n\t\t\t\t\t\t\trand.New(rand.NewSource(int64(seed))).Read(message)\n\n\t\t\t\t\t\t\t// Publish b.N messages to the stream (in batches)\n\t\t\t\t\t\t\tfor i := 1; i <= b.N; i++ {\n\t\t\t\t\t\t\t\tfastRandomMutation(message, 10)\n\t\t\t\t\t\t\t\t_, err := js.PublishAsync(subject, message)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to publish: %s\", err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t// Limit outstanding published messages to PublishBatchSize\n\t\t\t\t\t\t\t\tif i%PublishBatchSize == 0 || i == b.N {\n\t\t\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Published %d/%d messages\", i, b.N)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tcase <-time.After(publishTimeout):\n\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Publish timed out\")\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Set size of each operation, for throughput calculation\n\t\t\t\t\t\t\tb.SetBytes(int64(bc.messageSize))\n\n\t\t\t\t\t\t\t// Discard time spent during setup\n\t\t\t\t\t\t\t// Consumer may reset again further in\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\tvar consumed, duplicates, errors int\n\n\t\t\t\t\t\t\tconst (\n\t\t\t\t\t\t\t\tordered = true\n\t\t\t\t\t\t\t\tunordered = false\n\t\t\t\t\t\t\t\tdurable = true\n\t\t\t\t\t\t\t\tephemeral = false\n\t\t\t\t\t\t\t)\n\n\t\t\t\t\t\t\tswitch ct {\n\t\t\t\t\t\t\tcase PushSync:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runSyncPushConsumer(b, js, streamName)\n\t\t\t\t\t\t\tcase PushAsync:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, ephemeral)\n\t\t\t\t\t\t\tcase PushAsyncOrdered:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, ordered, ephemeral)\n\t\t\t\t\t\t\tcase PushAsyncDurable:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runAsyncPushConsumer(b, js, streamName, unordered, durable)\n\t\t\t\t\t\t\tcase PullDurable:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runPullConsumer(b, js, streamName, durable)\n\t\t\t\t\t\t\tcase PullEphemeral:\n\t\t\t\t\t\t\t\tconsumed, duplicates, errors = runPullConsumer(b, js, streamName, ephemeral)\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tb.Fatalf(\"Unknown consumer type: %v\", ct)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Benchmark ends here, (consumer may have stopped earlier)\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tif consumed != b.N {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Something doesn't add up: %d != %d\", consumed, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(duplicates)*100/float64(b.N), \"%dupe\")\n\t\t\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\n// BenchmarkJetStreamConsumeFilteredContiguous verifies the fix in\n// https://github.com/nats-io/nats-server/pull/7015 and should\n// capture future regressions.\nfunc BenchmarkJetStreamConsumeFilteredContiguous(b *testing.B) {\n\tclusterSizeCases := []struct {\n\t\tclusterSize int // Single node or cluster\n\t\treplicas int // Stream replicas\n\t\tstorage nats.StorageType // Stream storage\n\t\tfilters int // How many subject filters?\n\t}{\n\t\t{1, 1, nats.MemoryStorage, 1},\n\t\t{1, 1, nats.MemoryStorage, 2},\n\t\t{3, 3, nats.MemoryStorage, 1},\n\t\t{3, 3, nats.MemoryStorage, 2},\n\t\t{1, 1, nats.FileStorage, 1},\n\t\t{1, 1, nats.FileStorage, 2},\n\t\t{3, 3, nats.FileStorage, 1},\n\t\t{3, 3, nats.FileStorage, 2},\n\t}\n\n\tfor _, cs := range clusterSizeCases {\n\t\tname := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,storage=%s\",\n\t\t\tcs.clusterSize,\n\t\t\tcs.replicas,\n\t\t\tcs.storage.String(),\n\t\t)\n\t\tif cs.filters != 2 { // historical default is 2\n\t\t\tname = name + \",SF\"\n\t\t}\n\t\tb.Run(name, func(b *testing.B) {\n\t\t\t_, _, shutdown, nc, js := startJSClusterAndConnect(b, cs.clusterSize)\n\t\t\tdefer shutdown()\n\t\t\tdefer nc.Close()\n\n\t\t\tvar msgs = b.N\n\t\t\tpayload := make([]byte, 1024)\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"test\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\tStorage: cs.storage,\n\t\t\t\tReplicas: cs.replicas,\n\t\t\t})\n\t\t\trequire_NoError(b, err)\n\n\t\t\tfor range msgs {\n\t\t\t\t_, err = js.Publish(\"foo\", payload)\n\t\t\t\trequire_NoError(b, err)\n\t\t\t}\n\n\t\t\t// Subject filters deliberately vary from the stream, ensures that we hit\n\t\t\t// the right paths in the filestore, rather than detecting 1:1 overlap.\n\t\t\tocfg := &nats.ConsumerConfig{\n\t\t\t\tName: \"test_consumer\",\n\t\t\t\tDeliverPolicy: nats.DeliverAllPolicy,\n\t\t\t\tAckPolicy: nats.AckNonePolicy,\n\t\t\t\tReplicas: cs.replicas,\n\t\t\t\tMemoryStorage: true,\n\t\t\t}\n\t\t\tswitch cs.filters {\n\t\t\tcase 1:\n\t\t\t\tocfg.FilterSubject = \"foo\"\n\t\t\tcase 2:\n\t\t\t\tocfg.FilterSubjects = []string{\"foo\", \"bar\"}\n\t\t\t}\n\t\t\t_, err = js.AddConsumer(\"test\", ocfg)\n\t\t\trequire_NoError(b, err)\n\n\t\t\tps, err := js.PullSubscribe(\"foo\", _EMPTY_, nats.Bind(\"test\", \"test_consumer\"))\n\t\t\trequire_NoError(b, err)\n\n\t\t\tb.SetBytes(int64(len(payload)))\n\t\t\tb.ReportAllocs()\n\t\t\tb.ResetTimer()\n\t\t\tfor range msgs {\n\t\t\t\tmsgs, err := ps.Fetch(1)\n\t\t\t\trequire_NoError(b, err)\n\t\t\t\trequire_Len(b, len(msgs), 1)\n\t\t\t}\n\t\t\tb.StopTimer()\n\t\t})\n\t}\n}\n\nfunc BenchmarkJetStreamConsumeWithFilters(b *testing.B) {\n\tconst (\n\t\tverbose = false\n\t\tstreamName = \"S\"\n\t\tsubjectPrefix = \"s\"\n\t\tseed = 123456\n\t\tmessageSize = 32\n\t\tconsumerReplicas = 1\n\t\tdomainNameLength = 36 // Length of domain portion of subject, must be an even number\n\t\tpublishBatchSize = 1000\n\t\tpublishTimeout = 10 * time.Second\n\t)\n\n\tclusterSizeCases := []struct {\n\t\tclusterSize int // Single node or cluster\n\t\treplicas int // Stream replicas\n\t\tstorage nats.StorageType // Stream storage\n\t}{\n\t\t{1, 1, nats.MemoryStorage},\n\t\t{3, 3, nats.MemoryStorage},\n\t\t{1, 1, nats.FileStorage},\n\t\t{3, 3, nats.FileStorage},\n\t}\n\n\tbenchmarksCases := []struct {\n\t\tdomains int // Number of distinct domains\n\t\tsubjectsPerDomain int // Number of distinct subjects within each domain\n\t\tfilters int // Number of filters (..>) per consumer\n\t\tconcurrentConsumers int // Number of consumer running\n\n\t}{\n\t\t{100, 10, 5, 12},\n\t\t{1000, 10, 25, 12},\n\t\t{10_000, 10, 50, 12},\n\t}\n\n\tfor _, cs := range clusterSizeCases {\n\t\tname := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,storage=%s\",\n\t\t\tcs.clusterSize,\n\t\t\tcs.replicas,\n\t\t\tcs.storage.String(),\n\t\t)\n\t\tb.Run(\n\t\t\tname,\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, bc := range benchmarksCases {\n\n\t\t\t\t\tname := fmt.Sprintf(\n\t\t\t\t\t\t\"D=%d,DS=%d,F=%d,C=%d\",\n\t\t\t\t\t\tbc.domains,\n\t\t\t\t\t\tbc.subjectsPerDomain,\n\t\t\t\t\t\tbc.filters,\n\t\t\t\t\t\tbc.concurrentConsumers,\n\t\t\t\t\t)\n\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tname,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tcl, s, shutdown, nc, js := startJSClusterAndConnect(b, cs.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Creating stream with R=%d\", cs.replicas)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tstreamConfig := &nats.StreamConfig{\n\t\t\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\t\t\tSubjects: []string{subjectPrefix + \".>\"},\n\t\t\t\t\t\t\t\tStorage: cs.storage,\n\t\t\t\t\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\t\t\t\t\tMaxAge: time.Hour,\n\t\t\t\t\t\t\t\tDuplicates: 10 * time.Second,\n\t\t\t\t\t\t\t\tDiscard: nats.DiscardOld,\n\t\t\t\t\t\t\t\tNoAck: false,\n\t\t\t\t\t\t\t\tMaxMsgs: -1,\n\t\t\t\t\t\t\t\tMaxBytes: -1,\n\t\t\t\t\t\t\t\tMaxConsumers: -1,\n\t\t\t\t\t\t\t\tReplicas: 1,\n\t\t\t\t\t\t\t\tMaxMsgsPerSubject: 1,\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif _, err := js.AddStream(streamConfig); err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating stream: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\tconnectURL := s.ClientURL()\n\t\t\t\t\t\t\tif cs.replicas > 1 {\n\t\t\t\t\t\t\t\tconnectURL = cl.streamLeader(\"$G\", streamName).ClientURL()\n\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\t_, js = jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\t\t\t\t\t\tmessage := make([]byte, messageSize)\n\t\t\t\t\t\t\tdomain := make([]byte, domainNameLength/2)\n\n\t\t\t\t\t\t\tdomains := make([]string, 0, bc.domains*bc.subjectsPerDomain)\n\n\t\t\t\t\t\t\t// Publish one message per subject for each domain\n\t\t\t\t\t\t\tpublished := 0\n\t\t\t\t\t\t\ttotalMessages := bc.domains * bc.subjectsPerDomain\n\t\t\t\t\t\t\tfor d := 1; d <= bc.domains; d++ {\n\t\t\t\t\t\t\t\trng.Read(domain)\n\t\t\t\t\t\t\t\tfor s := 1; s <= bc.subjectsPerDomain; s++ {\n\t\t\t\t\t\t\t\t\trng.Read(message)\n\t\t\t\t\t\t\t\t\tdomainString := fmt.Sprintf(\"%X\", domain)\n\t\t\t\t\t\t\t\t\tdomains = append(domains, domainString)\n\t\t\t\t\t\t\t\t\tsubject := fmt.Sprintf(\"%s.%s.%d\", subjectPrefix, domainString, s)\n\t\t\t\t\t\t\t\t\t_, err := js.PublishAsync(subject, message)\n\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"failed to publish: %s\", err)\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tpublished += 1\n\n\t\t\t\t\t\t\t\t\t// Wait for all pending to be published before trying to publish the next batch\n\t\t\t\t\t\t\t\t\tif published%publishBatchSize == 0 || published == totalMessages {\n\t\t\t\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\t\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Published %d/%d messages\", published, totalMessages)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\tcase <-time.After(publishTimeout):\n\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Publish timed out\")\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Number of messages that each new consumer expects to consume\n\t\t\t\t\t\t\tmessagesPerIteration := bc.filters * bc.subjectsPerDomain\n\n\t\t\t\t\t\t\t// Each call to 'subscribe_consume_unsubscribe' is one benchmark operation.\n\t\t\t\t\t\t\t// i.e. subscribe_consume_unsubscribe will be called a total of b.N times (split among C threads)\n\t\t\t\t\t\t\t// Each operation consists of:\n\t\t\t\t\t\t\t// - Create filter\n\t\t\t\t\t\t\t// - Create consumer / Subscribe\n\t\t\t\t\t\t\t// - Consume expected number of messages\n\t\t\t\t\t\t\t// - Unsubscribe\n\t\t\t\t\t\t\tsubscribeConsumeUnsubscribe := func(js nats.JetStreamContext, rng *rand.Rand) {\n\n\t\t\t\t\t\t\t\t// Select F unique domains to create F non-overlapping filters\n\t\t\t\t\t\t\t\tfilterDomains := make(map[string]bool, bc.filters)\n\t\t\t\t\t\t\t\tfilters := make([]string, 0, bc.filters)\n\t\t\t\t\t\t\t\tfor len(filterDomains) < bc.filters {\n\t\t\t\t\t\t\t\t\tdomain := domains[rng.Intn(len(domains))]\n\t\t\t\t\t\t\t\t\tif _, found := filterDomains[domain]; found {\n\t\t\t\t\t\t\t\t\t\t// Collision with existing filter, try again\n\t\t\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tfilterDomains[domain] = true\n\t\t\t\t\t\t\t\t\tfilters = append(filters, fmt.Sprintf(\"%s.%s.>\", subjectPrefix, domain))\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\tb.Logf(\"Subscribe with filters: %+v\", filters)\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t// Consumer callback\n\t\t\t\t\t\t\t\treceived := 0\n\t\t\t\t\t\t\t\tconsumeWg := sync.WaitGroup{}\n\t\t\t\t\t\t\t\tconsumeWg.Add(1)\n\t\t\t\t\t\t\t\tcb := func(msg *nats.Msg) {\n\t\t\t\t\t\t\t\t\treceived += 1\n\t\t\t\t\t\t\t\t\tif received == messagesPerIteration {\n\t\t\t\t\t\t\t\t\t\tconsumeWg.Done()\n\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Received %d/%d messages\", received, messagesPerIteration)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t// Create consumer\n\t\t\t\t\t\t\t\tsubOpts := []nats.SubOpt{\n\t\t\t\t\t\t\t\t\tnats.BindStream(streamName),\n\t\t\t\t\t\t\t\t\tnats.OrderedConsumer(),\n\t\t\t\t\t\t\t\t\tnats.ConsumerReplicas(consumerReplicas),\n\t\t\t\t\t\t\t\t\tnats.ConsumerFilterSubjects(filters...),\n\t\t\t\t\t\t\t\t\tnats.ConsumerMemoryStorage(),\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tvar sub *nats.Subscription\n\n\t\t\t\t\t\t\t\tsub, err := js.Subscribe(\"\", cb, subOpts...)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to subscribe: %s\", err)\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tdefer func(sub *nats.Subscription) {\n\t\t\t\t\t\t\t\t\terr := sub.Unsubscribe()\n\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\tb.Logf(\"Failed to unsubscribe: %s\", err)\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}(sub)\n\n\t\t\t\t\t\t\t\tconsumeWg.Wait()\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Wait for all consumer threads and main to be ready\n\t\t\t\t\t\t\twgReady := sync.WaitGroup{}\n\t\t\t\t\t\t\twgReady.Add(bc.concurrentConsumers + 1)\n\t\t\t\t\t\t\t// Wait until all consumer threads have completed\n\t\t\t\t\t\t\twgCompleted := sync.WaitGroup{}\n\t\t\t\t\t\t\twgCompleted.Add(bc.concurrentConsumers)\n\t\t\t\t\t\t\t// Operations left for consumer threads\n\t\t\t\t\t\t\topsCount := atomic.Int32{}\n\t\t\t\t\t\t\topsCount.Store(int32(b.N))\n\n\t\t\t\t\t\t\t// Start a pool of C goroutines, each one with a dedicated connection.\n\t\t\t\t\t\t\tfor i := 1; i <= bc.concurrentConsumers; i++ {\n\t\t\t\t\t\t\t\tgo func(consumerId int) {\n\n\t\t\t\t\t\t\t\t\t// Connect\n\t\t\t\t\t\t\t\t\tnc, js := jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t\t\t// Signal completion of work\n\t\t\t\t\t\t\t\t\tdefer wgCompleted.Done()\n\n\t\t\t\t\t\t\t\t\trng := rand.New(rand.NewSource(int64(seed + consumerId)))\n\n\t\t\t\t\t\t\t\t\t// Ready, wait for everyone else\n\t\t\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t\t\tcompleted := 0\n\t\t\t\t\t\t\t\t\tfor opsCount.Add(-1) >= 0 {\n\t\t\t\t\t\t\t\t\t\tsubscribeConsumeUnsubscribe(js, rng)\n\t\t\t\t\t\t\t\t\t\tcompleted += 1\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\tb.Logf(\"Consumer thread %d completed %d of %d operations\", consumerId, completed, b.N)\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}(i)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Wait for all consumers to be ready\n\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t// Start measuring time\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\t// Wait for consumers to have chewed through b.N operations\n\t\t\t\t\t\t\twgCompleted.Wait()\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\t// Throughput is not very important in this benchmark since each operation includes\n\t\t\t\t\t\t\t// subscribe, unsubscribe and retrieves just a few bytes\n\t\t\t\t\t\t\t//b.SetBytes(int64(messageSize * messagesPerIteration))\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamPublish(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tseed = 12345\n\t\tstreamName = \"S\"\n\t)\n\n\trunSyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string) (int, int) {\n\t\tpublished, errors := 0, 0\n\t\tmessage := make([]byte, messageSize)\n\t\trand.New(rand.NewSource(int64(seed))).Read(message)\n\n\t\tb.ResetTimer()\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tfastRandomMutation(message, 10)\n\t\t\tsubject := subjects[fastrand.Uint32n(uint32(len(subjects)))]\n\t\t\t_, pubErr := js.Publish(subject, message)\n\t\t\tif pubErr != nil {\n\t\t\t\terrors++\n\t\t\t} else {\n\t\t\t\tpublished++\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Published %d/%d, %d errors\", i, b.N, errors)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\n\t\treturn published, errors\n\t}\n\n\trunAsyncPublisher := func(b *testing.B, js nats.JetStreamContext, messageSize int, subjects []string, asyncWindow int) (int, int) {\n\t\tconst publishCompleteMaxWait = 30 * time.Second\n\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\tmessage := make([]byte, messageSize)\n\t\trng.Read(message)\n\n\t\tpublished, errors := 0, 0\n\n\t\tb.ResetTimer()\n\n\t\tfor published < b.N {\n\n\t\t\t// Normally publish a full batch (of size `asyncWindow`)\n\t\t\tpublishBatchSize := asyncWindow\n\t\t\t// Unless fewer are left to complete the benchmark\n\t\t\tif b.N-published < asyncWindow {\n\t\t\t\tpublishBatchSize = b.N - published\n\t\t\t}\n\n\t\t\tpending := make([]nats.PubAckFuture, 0, publishBatchSize)\n\n\t\t\tfor i := 0; i < publishBatchSize; i++ {\n\t\t\t\tfastRandomMutation(message, 10)\n\t\t\t\tsubject := subjects[rng.Intn(len(subjects))]\n\t\t\t\tpubAckFuture, err := js.PublishAsync(subject, message)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrors++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tpending = append(pending, pubAckFuture)\n\t\t\t}\n\n\t\t\t// All in this batch published, wait for completed\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(publishCompleteMaxWait):\n\t\t\t\tb.Fatalf(\"Publish timed out\")\n\t\t\t}\n\n\t\t\t// Verify one by one if they were published successfully\n\t\t\tfor _, pubAckFuture := range pending {\n\t\t\t\tselect {\n\t\t\t\tcase <-pubAckFuture.Ok():\n\t\t\t\t\tpublished++\n\t\t\t\tcase <-pubAckFuture.Err():\n\t\t\t\t\terrors++\n\t\t\t\tdefault:\n\t\t\t\t\tb.Fatalf(\"PubAck is still pending after publish completed\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif verbose {\n\t\t\t\tb.Logf(\"Published %d/%d\", published, b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\n\t\treturn published, errors\n\t}\n\n\ttype PublishType string\n\tconst (\n\t\tSync PublishType = \"Sync\"\n\t\tAsync PublishType = \"Async\"\n\t)\n\n\tbenchmarksCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t\tmessageSize int\n\t\tnumSubjects int\n\t}{\n\t\t{1, 1, 10, 1}, // Single node, 10B messages\n\t\t{1, 1, 1024, 1}, // Single node, 1KB messages\n\t\t{3, 3, 10, 1}, // 3-nodes cluster, R=3, 10B messages\n\t\t{3, 3, 1024, 1}, // 3-nodes cluster, R=3, 1KB messages\n\t\t{3, 3, 10, 1}, // 3-nodes cluster, R=3, 10B messages (async flush)\n\t\t{3, 3, 1024, 1}, // 3-nodes cluster, R=3, 1KB messages (async flush)\n\t}\n\n\t// All the cases above are run with each of the publisher cases below\n\tpublisherCases := []struct {\n\t\tpType PublishType\n\t\tasyncWindow int\n\t}{\n\t\t{Sync, -1},\n\t\t{Async, 1000},\n\t\t{Async, 4000},\n\t\t{Async, 8000},\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\t\tname := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,MsgSz=%db,Subjs=%d\",\n\t\t\tbc.clusterSize,\n\t\t\tbc.replicas,\n\t\t\tbc.messageSize,\n\t\t\tbc.numSubjects,\n\t\t)\n\t\tb.Run(\n\t\t\tname,\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, pc := range publisherCases {\n\t\t\t\t\tname := fmt.Sprintf(\"%v\", pc.pType)\n\t\t\t\t\tif pc.pType == Async && pc.asyncWindow > 0 {\n\t\t\t\t\t\tname = fmt.Sprintf(\"%s[W:%d]\", name, pc.asyncWindow)\n\t\t\t\t\t}\n\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tname,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tsubjects := make([]string, bc.numSubjects)\n\t\t\t\t\t\t\tfor i := 0; i < bc.numSubjects; i++ {\n\t\t\t\t\t\t\t\tsubjects[i] = fmt.Sprintf(\"s-%d\", i+1)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Running %s with %d ops\", name, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Setting up %d nodes\", bc.clusterSize)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tcl, _, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\tjsOpts := []nats.JSOpt{\n\t\t\t\t\t\t\t\tnats.MaxWait(10 * time.Second),\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif pc.asyncWindow > 0 && pc.pType == Async {\n\t\t\t\t\t\t\t\tjsOpts = append(jsOpts, nats.PublishAsyncMaxPending(pc.asyncWindow))\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tjs, err := nc.JetStream(jsOpts...)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Creating stream with R=%d and %d input subjects\", bc.replicas, bc.numSubjects)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t_, err = jsStreamCreate(b, nc, &StreamConfig{\n\t\t\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\t\t\tSubjects: subjects,\n\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating stream: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\tif bc.replicas > 1 {\n\t\t\t\t\t\t\t\tconnectURL := cl.streamLeader(\"$G\", streamName).ClientURL()\n\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\tnc, err = nats.Connect(connectURL)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to create client connection to stream leader: %v\", err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\tjs, err = nc.JetStream(jsOpts...)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Unexpected error getting JetStream context for stream leader: %v\", err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Running %v publisher with message size: %dB\", pc.pType, bc.messageSize)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tb.SetBytes(int64(bc.messageSize))\n\n\t\t\t\t\t\t\t// Benchmark starts here\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\tvar published, errors int\n\t\t\t\t\t\t\tswitch pc.pType {\n\t\t\t\t\t\t\tcase Sync:\n\t\t\t\t\t\t\t\tpublished, errors = runSyncPublisher(b, js, bc.messageSize, subjects)\n\t\t\t\t\t\t\tcase Async:\n\t\t\t\t\t\t\t\tpublished, errors = runAsyncPublisher(b, js, bc.messageSize, subjects, pc.asyncWindow)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Benchmark ends here\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tif published+errors != b.N {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Something doesn't add up: %d + %d != %d\", published, errors, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamMetaSnapshot(b *testing.B) {\n\tc := createJetStreamClusterExplicit(b, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsetup := func(reqLevel string) *jetStream {\n\t\tml := c.leader()\n\t\tacc, js := ml.globalAccount(), ml.getJetStream()\n\t\tn := js.getMetaGroup()\n\n\t\t// Create all streams and consumers.\n\t\tnumStreams := 200\n\t\tnumConsumers := 500\n\t\tci := &ClientInfo{Cluster: \"R3S\", Account: globalAccountName}\n\t\tjs.mu.Lock()\n\t\tmetadata := map[string]string{JSRequiredLevelMetadataKey: reqLevel}\n\t\tfor i := 0; i < numStreams; i++ {\n\t\t\tscfg := &StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"STREAM-%d\", i),\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"SUBJECT-%d\", i)},\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tMetadata: metadata,\n\t\t\t}\n\t\t\tcfg, _ := ml.checkStreamCfg(scfg, acc, false)\n\t\t\trg, _ := js.createGroupForStream(ci, &cfg)\n\t\t\tsa := &streamAssignment{Group: rg, Sync: syncSubjForStream(), Config: &cfg, Client: ci, Created: time.Now().UTC()}\n\t\t\tn.Propose(encodeAddStreamAssignment(sa))\n\n\t\t\tfor j := 0; j < numConsumers; j++ {\n\t\t\t\tccfg := &ConsumerConfig{\n\t\t\t\t\tDurable: fmt.Sprintf(\"CONSUMER-%d\", j),\n\t\t\t\t\tMemoryStorage: true,\n\t\t\t\t\tMetadata: metadata,\n\t\t\t\t}\n\t\t\t\tselectedLimits, _, _, _ := acc.selectLimits(ccfg.replicas(&cfg))\n\t\t\t\tsrvLim := &ml.getOpts().JetStreamLimits\n\t\t\t\tsetConsumerConfigDefaults(ccfg, &cfg, srvLim, selectedLimits, false)\n\t\t\t\trg = js.cluster.createGroupForConsumer(ccfg, sa)\n\t\t\t\tca := &consumerAssignment{Group: rg, Stream: cfg.Name, Name: ccfg.Durable, Config: ccfg, Client: ci, Created: time.Now().UTC()}\n\t\t\t\tn.Propose(encodeAddConsumerAssignment(ca))\n\t\t\t}\n\t\t}\n\t\tjs.mu.Unlock()\n\n\t\t// Wait for all servers to have created all assets.\n\t\tcheckFor(b, 20*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tsjs := s.getJetStream()\n\t\t\t\tsjs.mu.RLock()\n\t\t\t\tstreams := sjs.cluster.streams[globalAccountName]\n\t\t\t\tif len(streams) != numStreams {\n\t\t\t\t\tsjs.mu.RUnlock()\n\t\t\t\t\treturn fmt.Errorf(\"expected %d streams, got %d\", numStreams, len(streams))\n\t\t\t\t}\n\t\t\t\tfor _, sa := range streams {\n\t\t\t\t\tif nc := len(sa.consumers); nc != numConsumers {\n\t\t\t\t\t\tsjs.mu.RUnlock()\n\t\t\t\t\t\treturn fmt.Errorf(\"expected %d consumers, got %d\", numConsumers, nc)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tsjs.mu.RUnlock()\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn js\n\t}\n\n\tfor _, t := range []struct {\n\t\ttitle string\n\t\treqLevel string\n\t}{\n\t\t{title: \"Default\", reqLevel: \"0\"},\n\t\t{title: \"AllUnsupported\", reqLevel: strconv.Itoa(math.MaxInt)},\n\t} {\n\t\tb.Run(t.title, func(b *testing.B) {\n\t\t\tjs := setup(t.reqLevel)\n\t\t\tb.ResetTimer()\n\t\t\tfor range b.N {\n\t\t\t\tjs.metaSnapshot()\n\t\t\t}\n\t\t\tb.StopTimer()\n\t\t})\n\t}\n}\n\nfunc BenchmarkJetStreamCounters(b *testing.B) {\n\tconst (\n\t\tverbose = false\n\t\tseed = 12345\n\t\tstreamName = \"S\"\n\t)\n\n\t// We don't actually create real sourcing streams, we just populate\n\t// the Nats-Counter-Sources header to make it look like we have some,\n\t// so that we can see how the code performs for bringing forward the\n\t// latest headers each time.\n\tgenerateSources := func(t testing.TB, count int) string {\n\t\tt.Helper()\n\t\tsources := CounterSources{}\n\t\tfor i := range count {\n\t\t\tstreamName := fmt.Sprintf(\"STREAM_%d\", i%10)\n\t\t\tsubjectName := fmt.Sprintf(\"subject.%d\", i)\n\t\t\tif sources[streamName] == nil {\n\t\t\t\tsources[streamName] = map[string]string{}\n\t\t\t}\n\t\t\tsources[streamName][subjectName] = \"12345\"\n\t\t}\n\t\tj, err := json.Marshal(sources)\n\t\trequire_NoError(t, err)\n\t\treturn string(j)\n\t}\n\n\trunSyncPublisher := func(b *testing.B, js nats.JetStreamContext, subjects []string, sources int) (int, int) {\n\t\tpublished, errors := 0, 0\n\t\tmsg := &nats.Msg{\n\t\t\tHeader: nats.Header{},\n\t\t}\n\t\tmsg.Header.Set(JSMessageIncr, \"1\")\n\t\tif sources > 0 {\n\t\t\tmsg.Header.Set(JSMessageCounterSources, generateSources(b, sources))\n\t\t}\n\t\tb.ResetTimer()\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tmsg.Subject = subjects[fastrand.Uint32n(uint32(len(subjects)))]\n\t\t\tif _, pubErr := js.PublishMsg(msg); pubErr != nil {\n\t\t\t\terrors++\n\t\t\t} else {\n\t\t\t\tpublished++\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Published %d/%d, %d errors\", i, b.N, errors)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\t\treturn published, errors\n\t}\n\n\trunAsyncPublisher := func(b *testing.B, js nats.JetStreamContext, subjects []string, sources int, asyncWindow int) (int, int) {\n\t\tconst publishCompleteMaxWait = 30 * time.Second\n\t\tmsg := &nats.Msg{\n\t\t\tHeader: nats.Header{},\n\t\t}\n\t\tmsg.Header.Set(JSMessageIncr, \"1\")\n\t\tif sources > 0 {\n\t\t\tmsg.Header.Set(JSMessageCounterSources, generateSources(b, sources))\n\t\t}\n\t\tpublished, errors := 0, 0\n\t\tb.ResetTimer()\n\n\t\tfor published < b.N {\n\t\t\t// Normally publish a full batch (of size `asyncWindow`)\n\t\t\tpublishBatchSize := min(b.N-published, asyncWindow)\n\t\t\tpending := make([]nats.PubAckFuture, 0, publishBatchSize)\n\n\t\t\tfor range publishBatchSize {\n\t\t\t\tmsg.Subject = subjects[fastrand.Uint32n(uint32(len(subjects)))]\n\t\t\t\tpubAckFuture, err := js.PublishMsgAsync(msg)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrors++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tpending = append(pending, pubAckFuture)\n\t\t\t}\n\n\t\t\t// All in this batch published, wait for completed\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(publishCompleteMaxWait):\n\t\t\t\tb.Fatalf(\"Publish timed out\")\n\t\t\t}\n\n\t\t\t// Verify one by one if they were published successfully\n\t\t\tfor _, pubAckFuture := range pending {\n\t\t\t\tselect {\n\t\t\t\tcase <-pubAckFuture.Ok():\n\t\t\t\t\tpublished++\n\t\t\t\tcase <-pubAckFuture.Err():\n\t\t\t\t\terrors++\n\t\t\t\tdefault:\n\t\t\t\t\tb.Fatalf(\"PubAck is still pending after publish completed\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif verbose {\n\t\t\t\tb.Logf(\"Published %d/%d\", published, b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\t\treturn published, errors\n\t}\n\n\ttype PublishType string\n\tconst (\n\t\tSync PublishType = \"Sync\"\n\t\tAsync PublishType = \"Async\"\n\t)\n\n\ttype benchmarksCase struct {\n\t\tstorageType StorageType\n\t\tclusterSize int\n\t\treplicas int\n\t\tnumSubjects int\n\t\tsources int\n\t}\n\tvar benchmarksCases []benchmarksCase\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, replicas := range []int{1, 3} {\n\t\t\tfor _, numSubjects := range []int{1, 1000} {\n\t\t\t\tfor _, sources := range []int{0, 10, 25, 250} {\n\t\t\t\t\tbenchmarksCases = append(benchmarksCases, benchmarksCase{\n\t\t\t\t\t\tstorageType: storage,\n\t\t\t\t\t\tclusterSize: 3,\n\t\t\t\t\t\treplicas: replicas,\n\t\t\t\t\t\tnumSubjects: numSubjects,\n\t\t\t\t\t\tsources: sources,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// All the cases above are run with each of the publisher cases below\n\tpublisherCases := []struct {\n\t\tpType PublishType\n\t\tasyncWindow int\n\t}{\n\t\t{Sync, -1},\n\t\t{Async, 1000},\n\t\t{Async, 4000},\n\t\t{Async, 8000},\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\t\tname := fmt.Sprintf(\n\t\t\t\"S=%s,N=%d,R=%d,Subjs=%d,Srcs=%d\",\n\t\t\tbc.storageType,\n\t\t\tbc.clusterSize,\n\t\t\tbc.replicas,\n\t\t\tbc.numSubjects,\n\t\t\tbc.sources,\n\t\t)\n\n\t\tb.Run(name, func(b *testing.B) {\n\t\t\tfor _, pc := range publisherCases {\n\t\t\t\tname := fmt.Sprintf(\"%v\", pc.pType)\n\t\t\t\tif pc.pType == Async && pc.asyncWindow > 0 {\n\t\t\t\t\tname = fmt.Sprintf(\"%s[W:%d]\", name, pc.asyncWindow)\n\t\t\t\t}\n\n\t\t\t\tb.Run(name, func(b *testing.B) {\n\t\t\t\t\tsubjects := make([]string, bc.numSubjects)\n\t\t\t\t\tfor i := range bc.numSubjects {\n\t\t\t\t\t\tsubjects[i] = fmt.Sprintf(\"s-%d\", i+1)\n\t\t\t\t\t}\n\n\t\t\t\t\tif verbose {\n\t\t\t\t\t\tb.Logf(\"Running %s with %d ops\", name, b.N)\n\t\t\t\t\t}\n\n\t\t\t\t\tif verbose {\n\t\t\t\t\t\tb.Logf(\"Setting up %d nodes\", bc.clusterSize)\n\t\t\t\t\t}\n\n\t\t\t\t\tcl, _, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\tdefer shutdown()\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\tjsOpts := []nats.JSOpt{\n\t\t\t\t\t\tnats.MaxWait(10 * time.Second),\n\t\t\t\t\t}\n\n\t\t\t\t\tif pc.asyncWindow > 0 && pc.pType == Async {\n\t\t\t\t\t\tjsOpts = append(jsOpts, nats.PublishAsyncMaxPending(pc.asyncWindow))\n\t\t\t\t\t}\n\n\t\t\t\t\tjs, err := nc.JetStream(jsOpts...)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tb.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif verbose {\n\t\t\t\t\t\tb.Logf(\"Creating stream with R=%d and %d input subjects\", bc.replicas, bc.numSubjects)\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := jsStreamCreate(b, nc, &StreamConfig{\n\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\tStorage: bc.storageType,\n\t\t\t\t\t\tSubjects: subjects,\n\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\tAllowMsgCounter: true,\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tb.Fatalf(\"Error creating stream: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\tif bc.replicas > 1 {\n\t\t\t\t\t\tconnectURL := cl.streamLeader(\"$G\", streamName).ClientURL()\n\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\tnc, err = nats.Connect(connectURL)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tb.Fatalf(\"Failed to create client connection to stream leader: %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\tjs, err = nc.JetStream(jsOpts...)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tb.Fatalf(\"Unexpected error getting JetStream context for stream leader: %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tif verbose {\n\t\t\t\t\t\tb.Logf(\"Running %v publisher\", pc.pType)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Benchmark starts here\n\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\tvar published, errors int\n\t\t\t\t\tswitch pc.pType {\n\t\t\t\t\tcase Sync:\n\t\t\t\t\t\tpublished, errors = runSyncPublisher(b, js, subjects, bc.sources)\n\t\t\t\t\tcase Async:\n\t\t\t\t\t\tpublished, errors = runAsyncPublisher(b, js, subjects, bc.sources, pc.asyncWindow)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Benchmark ends here\n\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\tif published+errors != b.N {\n\t\t\t\t\t\tb.Fatalf(\"Something doesn't add up: %d + %d != %d\", published, errors, b.N)\n\t\t\t\t\t}\n\n\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkJetStreamInterestStreamWithLimit(b *testing.B) {\n\n\tconst (\n\t\tverbose = true\n\t\tseed = 12345\n\t\tpublishBatchSize = 100\n\t\tmessageSize = 256\n\t\tnumSubjects = 2500\n\t\tsubjectPrefix = \"S\"\n\t\tnumPublishers = 4\n\t\trandomData = true\n\t\twarmupMessages = 1\n\t)\n\n\tif verbose {\n\t\tb.Logf(\n\t\t\t\"BatchSize: %d, MsgSize: %d, Subjects: %d, Publishers: %d, Random Message: %v\",\n\t\t\tpublishBatchSize,\n\t\t\tmessageSize,\n\t\t\tnumSubjects,\n\t\t\tnumPublishers,\n\t\t\trandomData,\n\t\t)\n\t}\n\n\t// Benchmark parameters: sub-benchmarks are executed for every combination of the following 3 groups\n\t// Unless a more restrictive filter is specified, e.g.:\n\t// BenchmarkJetStreamInterestStreamWithLimit/.*R=3.*/Storage=Memory/unlimited\n\n\t// Parameter: Number of nodes and number of stream replicas\n\tclusterAndReplicasCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t}{\n\t\t{1, 1}, // Single node, R=1\n\t\t{3, 3}, // 3-nodes cluster, R=3\n\t}\n\n\t// Parameter: Stream storage type\n\tstorageTypeCases := []nats.StorageType{\n\t\tnats.MemoryStorage,\n\t\tnats.FileStorage,\n\t}\n\n\t// Parameter: Stream limit configuration\n\tlimitConfigCases := map[string]func(*nats.StreamConfig){\n\t\t\"unlimited\": func(config *nats.StreamConfig) {\n\t\t},\n\t\t\"MaxMsg=1000\": func(config *nats.StreamConfig) {\n\t\t\tconfig.MaxMsgs = 100\n\t\t},\n\t\t\"MaxMsg=10\": func(config *nats.StreamConfig) {\n\t\t\tconfig.MaxMsgs = 10\n\t\t},\n\t\t\"MaxPerSubject=10\": func(config *nats.StreamConfig) {\n\t\t\tconfig.MaxMsgsPerSubject = 10\n\t\t},\n\t\t\"MaxAge=1s\": func(config *nats.StreamConfig) {\n\t\t\tconfig.MaxAge = 1 * time.Second\n\t\t},\n\t\t\"MaxBytes=1MB\": func(config *nats.StreamConfig) {\n\t\t\tconfig.MaxBytes = 1024 * 1024\n\t\t},\n\t}\n\n\t// Context shared by publishers routines\n\ttype PublishersContext = struct {\n\t\treadyWg sync.WaitGroup\n\t\tcompletedWg sync.WaitGroup\n\t\tmessagesLeft int\n\t\tlock sync.Mutex\n\t\terrors int\n\t}\n\n\t// Helper: Publish synchronously as Goroutine\n\tpublish := func(publisherId int, ctx *PublishersContext, js nats.JetStreamContext) {\n\t\tdefer ctx.completedWg.Done()\n\t\terrors := 0\n\t\tmessageBuf := make([]byte, messageSize)\n\t\trand.New(rand.NewSource(int64(seed + publisherId))).Read(messageBuf)\n\n\t\t// Warm up: publish a few messages\n\t\tfor i := 0; i < warmupMessages; i++ {\n\t\t\tsubject := fmt.Sprintf(\"%s.%d\", subjectPrefix, fastrand.Uint32n(numSubjects))\n\t\t\tif randomData {\n\t\t\t\tfastRandomMutation(messageBuf, 10)\n\t\t\t}\n\t\t\t_, err := js.Publish(subject, messageBuf)\n\t\t\tif err != nil {\n\t\t\t\tb.Logf(\"Warning: failed to publish warmup message: %s\", err)\n\t\t\t}\n\t\t}\n\n\t\t// Signal this publisher is ready\n\t\tctx.readyWg.Done()\n\n\t\tfor {\n\t\t\t// Obtain a batch of messages to publish\n\t\t\tbatchSize := 0\n\t\t\t{\n\t\t\t\tctx.lock.Lock()\n\t\t\t\tif ctx.messagesLeft >= publishBatchSize {\n\t\t\t\t\tbatchSize = publishBatchSize\n\t\t\t\t} else if ctx.messagesLeft < publishBatchSize {\n\t\t\t\t\tbatchSize = ctx.messagesLeft\n\t\t\t\t}\n\t\t\t\tctx.messagesLeft -= batchSize\n\t\t\t\tctx.lock.Unlock()\n\t\t\t}\n\n\t\t\t// Nothing left to publish, terminate\n\t\t\tif batchSize == 0 {\n\t\t\t\tctx.lock.Lock()\n\t\t\t\tctx.errors += errors\n\t\t\t\tctx.lock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Publish a batch of messages\n\t\t\tfor i := 0; i < batchSize; i++ {\n\t\t\t\tsubject := fmt.Sprintf(\"%s.%d\", subjectPrefix, fastrand.Uint32n(numSubjects))\n\t\t\t\tif randomData {\n\t\t\t\t\tfastRandomMutation(messageBuf, 10)\n\t\t\t\t}\n\t\t\t\t_, err := js.Publish(subject, messageBuf)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrors += 1\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Benchmark matrix: (cluster and replicas) * (storage type) * (stream limit)\n\tfor _, benchmarkCase := range clusterAndReplicasCases {\n\t\tb.Run(\n\t\t\tfmt.Sprintf(\n\t\t\t\t\"N=%d,R=%d\",\n\t\t\t\tbenchmarkCase.clusterSize,\n\t\t\t\tbenchmarkCase.replicas,\n\t\t\t),\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, storageType := range storageTypeCases {\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tfmt.Sprintf(\"Storage=%v\", storageType),\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tfor limitDescription, limitConfigFunc := range limitConfigCases {\n\t\t\t\t\t\t\t\tb.Run(\n\t\t\t\t\t\t\t\t\tlimitDescription,\n\t\t\t\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\t\t\t\t// Print benchmark parameters\n\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\n\t\t\t\t\t\t\t\t\t\t\t\t\"Stream: %+v, Storage: [%v] Limit: [%s], Ops: %d\",\n\t\t\t\t\t\t\t\t\t\t\t\tbenchmarkCase,\n\t\t\t\t\t\t\t\t\t\t\t\tstorageType,\n\t\t\t\t\t\t\t\t\t\t\t\tlimitDescription,\n\t\t\t\t\t\t\t\t\t\t\t\tb.N,\n\t\t\t\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Setup server or cluster\n\t\t\t\t\t\t\t\t\t\tcl, ls, shutdown, nc, js := startJSClusterAndConnect(b, benchmarkCase.clusterSize)\n\t\t\t\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t\t\t\t// Common stream configuration\n\t\t\t\t\t\t\t\t\t\tstreamConfig := &nats.StreamConfig{\n\t\t\t\t\t\t\t\t\t\t\tName: \"S\",\n\t\t\t\t\t\t\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"%s.>\", subjectPrefix)},\n\t\t\t\t\t\t\t\t\t\t\tReplicas: benchmarkCase.replicas,\n\t\t\t\t\t\t\t\t\t\t\tStorage: storageType,\n\t\t\t\t\t\t\t\t\t\t\tDiscard: DiscardOld,\n\t\t\t\t\t\t\t\t\t\t\tRetention: DiscardOld,\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t// Configure stream limit\n\t\t\t\t\t\t\t\t\t\tlimitConfigFunc(streamConfig)\n\n\t\t\t\t\t\t\t\t\t\t// Create stream\n\t\t\t\t\t\t\t\t\t\tif _, err := js.AddStream(streamConfig); err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating stream: %v\", err)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Set up publishers shared context\n\t\t\t\t\t\t\t\t\t\tvar pubCtx PublishersContext\n\t\t\t\t\t\t\t\t\t\tpubCtx.readyWg.Add(numPublishers)\n\t\t\t\t\t\t\t\t\t\tpubCtx.completedWg.Add(numPublishers)\n\n\t\t\t\t\t\t\t\t\t\t// Hold this lock until all publishers are ready\n\t\t\t\t\t\t\t\t\t\tpubCtx.lock.Lock()\n\t\t\t\t\t\t\t\t\t\tpubCtx.messagesLeft = b.N\n\n\t\t\t\t\t\t\t\t\t\tconnectURL := ls.ClientURL()\n\t\t\t\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\t\t\t\tif benchmarkCase.replicas > 1 {\n\t\t\t\t\t\t\t\t\t\t\tconnectURL = cl.streamLeader(\"$G\", \"S\").ClientURL()\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Spawn publishers routines, each with its own connection and JS context\n\t\t\t\t\t\t\t\t\t\tfor i := 0; i < numPublishers; i++ {\n\t\t\t\t\t\t\t\t\t\t\tnc, err := nats.Connect(connectURL)\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\t\t\t\tjs, err := nc.JetStream()\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\tgo publish(i, &pubCtx, js)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Wait for all publishers to be ready\n\t\t\t\t\t\t\t\t\t\tpubCtx.readyWg.Wait()\n\n\t\t\t\t\t\t\t\t\t\t// Set size of each operation, for throughput calculation\n\t\t\t\t\t\t\t\t\t\tb.SetBytes(messageSize)\n\n\t\t\t\t\t\t\t\t\t\t// Benchmark starts here\n\t\t\t\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\t\t\t\t// Unblock the publishers\n\t\t\t\t\t\t\t\t\t\tpubCtx.lock.Unlock()\n\n\t\t\t\t\t\t\t\t\t\t// Wait for all publishers to complete\n\t\t\t\t\t\t\t\t\t\tpubCtx.completedWg.Wait()\n\n\t\t\t\t\t\t\t\t\t\t// Benchmark ends here\n\t\t\t\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\t\t\t\t// Sanity check, publishers may have died before completing\n\t\t\t\t\t\t\t\t\t\tif pubCtx.messagesLeft != 0 {\n\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Some messages left: %d\", pubCtx.messagesLeft)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\tb.ReportMetric(float64(pubCtx.errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamKV(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tkvName = \"BUCKET\"\n\t\tkeyPrefix = \"K_\"\n\t\tseed = 12345\n\t)\n\n\trunKVGet := func(b *testing.B, kv nats.KeyValue, keys []string) int {\n\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\terrors := 0\n\n\t\tb.ResetTimer()\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tkey := keys[rng.Intn(len(keys))]\n\t\t\t_, err := kv.Get(key)\n\t\t\tif err != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Completed %d/%d Get ops\", i, b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\t\treturn errors\n\t}\n\n\trunKVPut := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int {\n\n\t\tvalue := make([]byte, valueSize)\n\t\trand.New(rand.NewSource(int64(seed))).Read(value)\n\t\terrors := 0\n\n\t\tb.ResetTimer()\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tkey := keys[fastrand.Uint32n(uint32(len(keys)))]\n\t\t\tfastRandomMutation(value, 10)\n\t\t\t_, err := kv.Put(key, value)\n\t\t\tif err != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Completed %d/%d Put ops\", i, b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\t\treturn errors\n\t}\n\n\trunKVUpdate := func(b *testing.B, kv nats.KeyValue, keys []string, valueSize int) int {\n\t\tvalue := make([]byte, valueSize)\n\t\trand.New(rand.NewSource(int64(seed))).Read(value)\n\t\terrors := 0\n\n\t\tb.ResetTimer()\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tkey := keys[fastrand.Uint32n(uint32(len(keys)))]\n\n\t\t\tkve, getErr := kv.Get(key)\n\t\t\tif getErr != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tfastRandomMutation(value, 10)\n\t\t\t_, updateErr := kv.Update(key, value, kve.Revision())\n\t\t\tif updateErr != nil {\n\t\t\t\terrors++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Completed %d/%d Update ops\", i, b.N)\n\t\t\t}\n\t\t}\n\n\t\tb.StopTimer()\n\t\treturn errors\n\t}\n\n\ttype WorkloadType string\n\tconst (\n\t\tGet WorkloadType = \"GET\"\n\t\tPut WorkloadType = \"PUT\"\n\t\tUpdate WorkloadType = \"CAS\"\n\t)\n\n\tbenchmarksCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t\tnumKeys int\n\t\tvalueSize int\n\t}{\n\t\t{1, 1, 100, 100}, // 1 node with 100 keys, 100B values\n\t\t{1, 1, 1000, 100}, // 1 node with 1000 keys, 100B values\n\t\t{3, 3, 100, 100}, // 3 nodes with 100 keys, 100B values\n\t\t{3, 3, 1000, 100}, // 3 nodes with 1000 keys, 100B values\n\t\t{3, 3, 1000, 1024}, // 3 nodes with 1000 keys, 1KB values\n\t}\n\n\tworkloadCases := []WorkloadType{\n\t\tGet,\n\t\tPut,\n\t\tUpdate,\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\n\t\tbName := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,B=1,K=%d,ValSz=%db\",\n\t\t\tbc.clusterSize,\n\t\t\tbc.replicas,\n\t\t\tbc.numKeys,\n\t\t\tbc.valueSize,\n\t\t)\n\n\t\tb.Run(\n\t\t\tbName,\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, wc := range workloadCases {\n\t\t\t\t\twName := fmt.Sprintf(\"%v\", wc)\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\twName,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Running %s workload %s with %d messages\", wName, bName, b.N)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Setting up %d nodes\", bc.clusterSize)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Pre-generate all keys\n\t\t\t\t\t\t\tkeys := make([]string, 0, bc.numKeys)\n\t\t\t\t\t\t\tfor i := 1; i <= bc.numKeys; i++ {\n\t\t\t\t\t\t\t\tkey := fmt.Sprintf(\"%s%d\", keyPrefix, i)\n\t\t\t\t\t\t\t\tkeys = append(keys, key)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Setup server or cluster\n\t\t\t\t\t\t\tcl, _, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t// Create bucket\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\"Creating KV %s with R=%d\", kvName, bc.replicas)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tkvConfig := &nats.KeyValueConfig{\n\t\t\t\t\t\t\t\tBucket: kvName,\n\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tkv, err := js.CreateKeyValue(kvConfig)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating KV: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Initialize all keys\n\t\t\t\t\t\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\t\t\t\t\t\tvalue := make([]byte, bc.valueSize)\n\t\t\t\t\t\t\tfor _, key := range keys {\n\t\t\t\t\t\t\t\trng.Read(value)\n\t\t\t\t\t\t\t\t_, err := kv.Create(key, value)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to initialize %s/%s: %v\", kvName, key, err)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\tif bc.replicas > 1 {\n\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\tconnectURL := cl.streamLeader(\"$G\", fmt.Sprintf(\"KV_%s\", kvName)).ClientURL()\n\t\t\t\t\t\t\t\tnc, js = jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tkv, err = js.KeyValue(kv.Bucket())\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Error binding to KV: %v\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Set size of each operation, for throughput calculation\n\t\t\t\t\t\t\tb.SetBytes(int64(bc.valueSize))\n\n\t\t\t\t\t\t\t// Discard time spent during setup\n\t\t\t\t\t\t\t// May reset again further in\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\tvar errors int\n\n\t\t\t\t\t\t\tswitch wc {\n\t\t\t\t\t\t\tcase Get:\n\t\t\t\t\t\t\t\terrors = runKVGet(b, kv, keys)\n\t\t\t\t\t\t\tcase Put:\n\t\t\t\t\t\t\t\terrors = runKVPut(b, kv, keys, bc.valueSize)\n\t\t\t\t\t\t\tcase Update:\n\t\t\t\t\t\t\t\terrors = runKVUpdate(b, kv, keys, bc.valueSize)\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tb.Fatalf(\"Unknown workload type: %v\", wc)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Benchmark ends here, (may have stopped earlier)\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamObjStore(b *testing.B) {\n\tconst (\n\t\tverbose = false\n\t\tobjStoreName = \"B\"\n\t\tkeyPrefix = \"K_\"\n\t\tseed = 12345\n\t\tinitKeys = true\n\n\t\t// read/write ratios\n\t\tReadOnly = 1.0\n\t\tWriteOnly = 0.0\n\t)\n\n\t// rwRatio to string\n\trwRatioToString := func(rwRatio float64) string {\n\t\tswitch rwRatio {\n\t\tcase ReadOnly:\n\t\t\treturn \"readOnly\"\n\t\tcase WriteOnly:\n\t\t\treturn \"writeOnly\"\n\t\tdefault:\n\t\t\treturn fmt.Sprintf(\"%0.1f\", rwRatio)\n\t\t}\n\t}\n\n\t// benchmark for object store by performing read/write operations with data of random size\n\tRunObjStoreBenchmark := func(b *testing.B, objStore nats.ObjectStore, minObjSz int, maxObjSz int, numKeys int, rwRatio float64) (int, int, int) {\n\t\tvar (\n\t\t\terrors int\n\t\t\treads int\n\t\t\twrites int\n\t\t)\n\n\t\tdataBuf := make([]byte, maxObjSz)\n\t\trng := rand.New(rand.NewSource(int64(seed)))\n\t\trng.Read(dataBuf)\n\n\t\t// Each operation is processing a random amount of bytes within a size range which\n\t\t// will be either read from or written to an object store bucket. However, here we are\n\t\t// approximating the size of the processed data with a simple average of the range.\n\t\tb.SetBytes(int64((minObjSz + maxObjSz) / 2))\n\n\t\tfor i := 1; i <= b.N; i++ {\n\t\t\tkey := fmt.Sprintf(\"%s_%d\", keyPrefix, rng.Intn(numKeys))\n\t\t\tvar err error\n\n\t\t\trwOp := rng.Float64()\n\t\t\tswitch {\n\t\t\tcase rwOp <= rwRatio:\n\t\t\t\t// Read Op\n\t\t\t\t_, err = objStore.GetBytes(key)\n\t\t\t\treads++\n\t\t\tcase rwOp > rwRatio:\n\t\t\t\t// Write Op\n\t\t\t\t// dataSz is a random value between min-max object size and cannot be less than 1 byte\n\t\t\t\tdataSz := rng.Intn(maxObjSz-minObjSz+1) + minObjSz\n\t\t\t\tdata := dataBuf[:dataSz]\n\t\t\t\tfastRandomMutation(data, 10)\n\t\t\t\t_, err = objStore.PutBytes(key, data)\n\t\t\t\twrites++\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\terrors++\n\t\t\t}\n\n\t\t\tif verbose && i%1000 == 0 {\n\t\t\t\tb.Logf(\"Completed: %d reads, %d writes, %d errors. %d/%d total operations have been completed.\", reads, writes, errors, i, b.N)\n\t\t\t}\n\t\t}\n\t\treturn errors, reads, writes\n\t}\n\n\t// benchmark cases table\n\tbenchmarkCases := []struct {\n\t\tstorage nats.StorageType\n\t\tnumKeys int\n\t\tminObjSz int\n\t\tmaxObjSz int\n\t}{\n\t\t{nats.MemoryStorage, 100, 1024, 102400}, // mem storage, 100 objects sized (1KB-100KB)\n\t\t{nats.MemoryStorage, 100, 102400, 1048576}, // mem storage, 100 objects sized (100KB-1MB)\n\t\t{nats.MemoryStorage, 1000, 10240, 102400}, // mem storage, 1k objects of various size (10KB - 100KB)\n\t\t{nats.FileStorage, 100, 1024, 102400}, // file storage, 100 objects sized (1KB-100KB)\n\t\t{nats.FileStorage, 1000, 10240, 1048576}, // file storage, 1k objects of various size (10KB - 1MB)\n\t\t{nats.FileStorage, 100, 102400, 1048576}, // file storage, 100 objects sized (100KB-1MB)\n\t\t{nats.FileStorage, 100, 1048576, 10485760}, // file storage, 100 objects sized (1MB-10MB)\n\t\t{nats.FileStorage, 10, 10485760, 104857600}, // file storage, 10 objects sized (10MB-100MB)\n\t}\n\n\tvar (\n\t\tclusterSizeCases = []int{1, 3}\n\t\trwRatioCases = []float64{ReadOnly, WriteOnly, 0.8}\n\t)\n\n\t// Test with either single node or 3 node cluster\n\tfor _, clusterSize := range clusterSizeCases {\n\t\treplicas := clusterSize\n\t\tcName := fmt.Sprintf(\"N=%d,R=%d\", clusterSize, replicas)\n\t\tb.Run(\n\t\t\tcName,\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, rwRatio := range rwRatioCases {\n\t\t\t\t\trName := fmt.Sprintf(\"workload=%s\", rwRatioToString(rwRatio))\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\trName,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\t// Test all tabled benchmark cases\n\t\t\t\t\t\t\tfor _, bc := range benchmarkCases {\n\t\t\t\t\t\t\t\tbName := fmt.Sprintf(\"K=%d,storage=%s,minObjSz=%db,maxObjSz=%db\", bc.numKeys, bc.storage, bc.minObjSz, bc.maxObjSz)\n\t\t\t\t\t\t\t\tb.Run(\n\t\t\t\t\t\t\t\t\tbName,\n\t\t\t\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\t\t\t\t// Test setup\n\t\t\t\t\t\t\t\t\t\trng := rand.New(rand.NewSource(int64(seed)))\n\n\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Setting up %d nodes\", replicas)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Setup server or cluster\n\t\t\t\t\t\t\t\t\t\tcl, _, shutdown, nc, js := startJSClusterAndConnect(b, clusterSize)\n\t\t\t\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t\t\t\t// Initialize object store\n\t\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Creating ObjectStore %s with R=%d\", objStoreName, replicas)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\tobjStoreConfig := &nats.ObjectStoreConfig{\n\t\t\t\t\t\t\t\t\t\t\tBucket: objStoreName,\n\t\t\t\t\t\t\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\t\t\t\t\t\t\tStorage: bc.storage,\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\tobjStore, err := js.CreateObjectStore(objStoreConfig)\n\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Error creating ObjectStore: %v\", err)\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\t\t\t\tif clusterSize > 1 {\n\t\t\t\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\t\t\t\tconnectURL := cl.streamLeader(\"$G\", fmt.Sprintf(\"OBJ_%s\", objStoreName)).ClientURL()\n\t\t\t\t\t\t\t\t\t\t\tnc, js := jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\t\t\t\tobjStore, err = js.ObjectStore(objStoreName)\n\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Error binding to ObjectStore: %v\", err)\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// Initialize keys\n\t\t\t\t\t\t\t\t\t\tif initKeys {\n\t\t\t\t\t\t\t\t\t\t\tfor n := 0; n < bc.numKeys; n++ {\n\t\t\t\t\t\t\t\t\t\t\t\tkey := fmt.Sprintf(\"%s_%d\", keyPrefix, n)\n\t\t\t\t\t\t\t\t\t\t\t\tdataSz := rng.Intn(bc.maxObjSz-bc.minObjSz+1) + bc.minObjSz\n\t\t\t\t\t\t\t\t\t\t\t\tvalue := make([]byte, dataSz)\n\t\t\t\t\t\t\t\t\t\t\t\trng.Read(value)\n\t\t\t\t\t\t\t\t\t\t\t\t_, err := objStore.PutBytes(key, value)\n\t\t\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to initialize %s/%s: %v\", objStoreName, key, err)\n\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\t\t\t\t// Run benchmark\n\t\t\t\t\t\t\t\t\t\terrors, reads, writes := RunObjStoreBenchmark(b, objStore, bc.minObjSz, bc.maxObjSz, bc.numKeys, rwRatio)\n\n\t\t\t\t\t\t\t\t\t\t// Report metrics\n\t\t\t\t\t\t\t\t\t\tb.ReportMetric(float64(errors)*100/float64(b.N), \"%error\")\n\t\t\t\t\t\t\t\t\t\tb.ReportMetric(float64(reads), \"reads\")\n\t\t\t\t\t\t\t\t\t\tb.ReportMetric(float64(writes), \"writes\")\n\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamPublishConcurrent(b *testing.B) {\n\tconst (\n\t\tsubject = \"test-subject\"\n\t\tstreamName = \"test-stream\"\n\t)\n\n\ttype BenchPublisher struct {\n\t\t// nats connection for this publisher\n\t\tconn *nats.Conn\n\t\t// jetstream context\n\t\tjs nats.JetStreamContext\n\t\t// message buffer\n\t\tmessageData []byte\n\t\t// number of publish calls\n\t\tpublishCalls int\n\t\t// number of publish errors\n\t\tpublishErrors int\n\t}\n\n\tmessageSizeCases := []int64{\n\t\t10, // 10B\n\t\t1024, // 1KiB\n\t\t102400, // 100KiB\n\t}\n\tnumPubsCases := []int{\n\t\t12,\n\t}\n\n\treplicasCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t}{\n\t\t{1, 1},\n\t\t{3, 3},\n\t\t{3, 3},\n\t}\n\n\tworkload := func(b *testing.B, numPubs int, messageSize int64, clientUrl string) {\n\n\t\t// create N publishers\n\t\tpublishers := make([]BenchPublisher, numPubs)\n\t\tfor i := range publishers {\n\t\t\t// create publisher connection and jetstream context\n\t\t\tncPub, err := nats.Connect(clientUrl)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tdefer ncPub.Close()\n\t\t\tjsPub, err := ncPub.JetStream()\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\t// initialize publisher\n\t\t\tpublishers[i] = BenchPublisher{\n\t\t\t\tconn: ncPub,\n\t\t\t\tjs: jsPub,\n\t\t\t\tmessageData: make([]byte, messageSize),\n\t\t\t\tpublishCalls: 0,\n\t\t\t\tpublishErrors: 0,\n\t\t\t}\n\t\t\trand.New(rand.NewSource(int64(i))).Read(publishers[i].messageData)\n\t\t}\n\n\t\t// waits for all publishers sub-routines and for main thread to be ready\n\t\tvar workloadReadyWg sync.WaitGroup\n\t\tworkloadReadyWg.Add(1 + numPubs)\n\n\t\t// wait group blocks main thread until publish workload is completed, it is decremented after stream receives b.N messages from all publishers\n\t\tvar benchCompleteWg sync.WaitGroup\n\t\tbenchCompleteWg.Add(1)\n\n\t\t// wait group to ensure all publishers have been torn down\n\t\tvar finishedPublishersWg sync.WaitGroup\n\t\tfinishedPublishersWg.Add(numPubs)\n\n\t\t// start go routines for all publishers, wait till all publishers are initialized before starting publish workload\n\t\tfor i := range publishers {\n\n\t\t\tgo func(pubId int) {\n\t\t\t\t// signal that this publisher has been torn down\n\t\t\t\tdefer finishedPublishersWg.Done()\n\n\t\t\t\t// publisher sub-routine is ready\n\t\t\t\tworkloadReadyWg.Done()\n\n\t\t\t\t// start workload when main thread and all other publishers are ready\n\t\t\t\tworkloadReadyWg.Wait()\n\n\t\t\t\t// publish until stream receives b.N messages\n\t\t\t\tfor {\n\t\t\t\t\t// random bytes as payload\n\t\t\t\t\tfastRandomMutation(publishers[pubId].messageData, 10)\n\t\t\t\t\t// attempt to publish message\n\t\t\t\t\tpubAck, err := publishers[pubId].js.Publish(subject, publishers[pubId].messageData)\n\t\t\t\t\tpublishers[pubId].publishCalls += 1\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tpublishers[pubId].publishErrors += 1\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// all messages have been published to stream\n\t\t\t\t\tif pubAck.Sequence == uint64(b.N) {\n\t\t\t\t\t\tbenchCompleteWg.Done()\n\t\t\t\t\t}\n\t\t\t\t\t// a publisher has already published b.N messages, stop publishing\n\t\t\t\t\tif pubAck.Sequence >= uint64(b.N) {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(i)\n\t\t}\n\n\t\t// set bytes per operation\n\t\tb.SetBytes(messageSize)\n\n\t\t// main thread is ready\n\t\tworkloadReadyWg.Done()\n\t\t// start the clock\n\t\tb.ResetTimer()\n\n\t\t// wait till termination cond reached\n\t\tbenchCompleteWg.Wait()\n\t\t// stop the clock\n\t\tb.StopTimer()\n\n\t\t// wait for all publishers to shutdown\n\t\tfinishedPublishersWg.Wait()\n\n\t\t// sum up publish calls and errors\n\t\tpublishCalls := 0\n\t\tpublishErrors := 0\n\t\tfor _, pub := range publishers {\n\t\t\tpublishCalls += pub.publishCalls\n\t\t\tpublishErrors += pub.publishErrors\n\t\t}\n\n\t\t// report error rate\n\t\terrorRate := 100 * float64(publishErrors) / float64(publishCalls)\n\t\tb.ReportMetric(errorRate, \"%error\")\n\t}\n\n\t// benchmark case matrix\n\tfor _, replicasCase := range replicasCases {\n\t\ttitle := fmt.Sprintf(\"N=%d,R=%d\", replicasCase.clusterSize, replicasCase.replicas)\n\t\tb.Run(\n\t\t\ttitle,\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, messageSize := range messageSizeCases {\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tfmt.Sprintf(\"msgSz=%db\", messageSize),\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\tfor _, numPubs := range numPubsCases {\n\t\t\t\t\t\t\t\tb.Run(\n\t\t\t\t\t\t\t\t\tfmt.Sprintf(\"pubs=%d\", numPubs),\n\t\t\t\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\t\t\t\t// start jetstream cluster\n\t\t\t\t\t\t\t\t\t\tcl, ls, shutdown, nc, js := startJSClusterAndConnect(b, replicasCase.clusterSize)\n\t\t\t\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\t\t\tclientUrl := ls.ClientURL()\n\n\t\t\t\t\t\t\t\t\t\t// create stream\n\t\t\t\t\t\t\t\t\t\t_, err := jsStreamCreate(b, nc, &StreamConfig{\n\t\t\t\t\t\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\t\t\t\t\t\tSubjects: []string{subject},\n\t\t\t\t\t\t\t\t\t\t\tReplicas: replicasCase.replicas,\n\t\t\t\t\t\t\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t\t\t\t\t\t\t})\n\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\tdefer js.DeleteStream(streamName)\n\n\t\t\t\t\t\t\t\t\t\t// If replicated resource, connect to stream leader for lower variability\n\t\t\t\t\t\t\t\t\t\tif replicasCase.replicas > 1 {\n\t\t\t\t\t\t\t\t\t\t\tnc.Close()\n\t\t\t\t\t\t\t\t\t\t\tclientUrl = cl.streamLeader(\"$G\", streamName).ClientURL()\n\t\t\t\t\t\t\t\t\t\t\tnc, _ = jsClientConnectURL(b, clientUrl)\n\t\t\t\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t\t// run workload\n\t\t\t\t\t\t\t\t\t\tworkload(b, numPubs, messageSize, clientUrl)\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t})\n\t}\n}\n\nfunc BenchmarkJetStreamParallelStartup(b *testing.B) {\n\tomp := runtime.GOMAXPROCS(-1)\n\tstreams, msgs, cardinality := omp, 100_000, 10_000\n\n\t_, s, shutdown, nc, js := startJSClusterAndConnect(b, 1)\n\tdefer shutdown()\n\tjsc := *s.JetStreamConfig()\n\tsd := strings.TrimSuffix(jsc.StoreDir, \"/jetstream\")\n\n\tb.Logf(\"Building %d streams with %d messages, %d subjects...\", streams, msgs, cardinality)\n\tstart := time.Now()\n\tfor i := range streams {\n\t\tjsStreamCreate(b, nc, &StreamConfig{\n\t\t\tName: fmt.Sprintf(\"stream_%d\", i),\n\t\t\tSubjects: []string{fmt.Sprintf(\"%d.>\", i)},\n\t\t\tStorage: FileStorage,\n\t\t})\n\t\tfor n := range msgs {\n\t\t\tsubj := fmt.Sprintf(\"%d.%d\", i, n%cardinality)\n\t\t\t_, err := js.Publish(subj, nil)\n\t\t\trequire_NoError(b, err)\n\t\t}\n\t}\n\tb.Logf(\"Streams built in %s\", time.Since(start))\n\n\tbench := func(b *testing.B) {\n\t\ts.shutdownJetStream()\n\t\tjsc.StoreDir = sd\n\t\trequire_NoError(b, filepath.Walk(jsc.StoreDir, func(path string, info os.FileInfo, err error) error {\n\t\t\trequire_NoError(b, err)\n\t\t\tif info.Mode().IsRegular() && info.Name() == \"index.db\" {\n\t\t\t\treturn os.Truncate(path, 0)\n\t\t\t}\n\t\t\treturn nil\n\t\t}))\n\t\tb.ResetTimer()\n\t\ts.EnableJetStream(&jsc)\n\t}\n\n\t// Try to step down GOMAXPROCS in common CPU core counts.\n\tmp := 1 << (bits.Len(uint(omp)) - 1)\n\tif omp > mp {\n\t\tb.Run(fmt.Sprintf(\"GOMAXPROCS=%d\", omp), func(b *testing.B) {\n\t\t\tbench(b)\n\t\t})\n\t}\n\tfor ; mp >= 1; mp >>= 1 {\n\t\tb.Run(fmt.Sprintf(\"GOMAXPROCS=%d\", mp), func(b *testing.B) {\n\t\t\truntime.GOMAXPROCS(mp)\n\t\t\tdefer runtime.GOMAXPROCS(omp)\n\t\t\tbench(b)\n\t\t})\n\t}\n}\n\nfunc BenchmarkJetStreamScanForSources(b *testing.B) {\n\t_, s, shutdown, nc, js := startJSClusterAndConnect(b, 1)\n\tdefer shutdown()\n\n\tjsStreamCreate(b, nc, &StreamConfig{\n\t\tName: \"origin\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t})\n\n\tjsStreamCreate(b, nc, &StreamConfig{\n\t\tName: \"stream\",\n\t\tSubjects: []string{\"bar\"},\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{\n\t\t\t\tName: \"origin\",\n\t\t\t\tFilterSubject: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\t// Start by publishing some messages to the sourcing stream.\n\tfor range 1000 {\n\t\t_, err := js.Publish(\"bar\", nil)\n\t\trequire_NoError(b, err)\n\t}\n\n\t// Then publish a message to the origin stream.\n\t_, err := js.Publish(\"foo\", nil)\n\trequire_NoError(b, err)\n\n\t// Wait for the sourced message from the origin stream.\n\tcheckFor(b, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"stream\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1001 {\n\t\t\treturn fmt.Errorf(\"waiting for sourcing\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now publish a LOT more messages to the sourcing stream\n\t// which previously the reverse scan would have had to scan\n\t// over linearly.\n\tfor range 100_000 {\n\t\t_, err := js.Publish(\"bar\", nil)\n\t\trequire_NoError(b, err)\n\t}\n\n\tmset, err := s.globalAccount().lookupStream(\"stream\")\n\trequire_NoError(b, err)\n\n\tb.Run(\"StartingSequenceForSources\", func(b *testing.B) {\n\t\tmset.startingSequenceForSources()\n\t})\n}\n\n// Helper function to stand up a JS-enabled single server or cluster\nfunc startJSClusterAndConnect(b *testing.B, clusterSize int) (c *cluster, s *Server, shutdown func(), nc *nats.Conn, js nats.JetStreamContext) {\n\tb.Helper()\n\tvar err error\n\n\tif clusterSize == 1 {\n\t\ts = RunBasicJetStreamServer(b)\n\t\tshutdown = func() {\n\t\t\ts.Shutdown()\n\t\t}\n\t\ts.optsMu.Lock()\n\t\ts.opts.SyncInterval = 5 * time.Minute\n\t\ts.optsMu.Unlock()\n\t} else {\n\t\tc = createJetStreamClusterExplicit(b, \"BENCH_PUB\", clusterSize)\n\t\tc.waitOnClusterReadyWithNumPeers(clusterSize)\n\t\tc.waitOnLeader()\n\t\ts = c.leader()\n\t\tshutdown = func() {\n\t\t\tc.shutdown()\n\t\t}\n\t\tfor _, s := range c.servers {\n\t\t\ts.optsMu.Lock()\n\t\t\ts.opts.SyncInterval = 5 * time.Minute\n\t\t\ts.optsMu.Unlock()\n\t\t}\n\t}\n\n\tnc, err = nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"failed to connect: %s\", err)\n\t}\n\n\tjs, err = nc.JetStream()\n\tif err != nil {\n\t\tb.Fatalf(\"failed to init jetstream: %s\", err)\n\t}\n\n\treturn c, s, shutdown, nc, js\n}\n" }, { "path": "server/jetstream_cluster.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\tcrand \"crypto/rand\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"iter\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/antithesishq/antithesis-sdk-go/assert\"\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// jetStreamCluster holds information about the meta group and stream assignments.\ntype jetStreamCluster struct {\n\t// The metacontroller raftNode.\n\tmeta RaftNode\n\t// For stream and consumer assignments. All servers will have this be the same.\n\t// ACCOUNT -> STREAM -> Stream Assignment -> Consumers\n\tstreams map[string]map[string]*streamAssignment\n\t// These are inflight proposals and used to apply limits when there are\n\t// concurrent requests that would otherwise be accepted.\n\t// We also record the assignment for the stream/consumer. This is needed since if we have\n\t// concurrent requests for same account and stream/consumer we need to let it process to get\n\t// a response but they need to be same group, peers etc. and sync subjects.\n\tinflightStreams map[string]map[string]*inflightStreamInfo\n\tinflightConsumers map[string]map[string]map[string]*inflightConsumerInfo\n\t// Holds a map of a peer ID to the reply subject, to only respond after gaining\n\t// quorum on the peer-remove action.\n\tpeerRemoveReply map[string]peerRemoveInfo\n\t// Signals meta-leader should check the stream assignments.\n\tstreamsCheck bool\n\t// Server.\n\ts *Server\n\t// Internal client.\n\tc *client\n\t// Processing assignment results.\n\tstreamResults *subscription\n\tconsumerResults *subscription\n\t// System level request to have the leader stepdown.\n\tstepdown *subscription\n\t// System level requests to remove a peer.\n\tpeerRemove *subscription\n\t// System level request to move a stream\n\tpeerStreamMove *subscription\n\t// System level request to cancel a stream move\n\tpeerStreamCancelMove *subscription\n\t// To pop out the monitorCluster before the raft layer.\n\tqch chan struct{}\n\t// To notify others that monitorCluster has actually stopped.\n\tstopped chan struct{}\n\t// Track last meta snapshot time and duration for monitoring.\n\tlastMetaSnapTime int64 // Unix nanoseconds\n\tlastMetaSnapDuration int64 // Duration in nanoseconds\n}\n\n// Used to track inflight stream create/update/delete requests that have been proposed but not yet applied.\ntype inflightStreamInfo struct {\n\tops uint64 // Inflight operations, i.e. inflight stream creates/updates/deletes.\n\tdeleted bool // Whether the stream has been deleted.\n\t*streamAssignment\n}\n\n// Used to track inflight consumer create/update/delete requests that have been proposed but not yet applied.\ntype inflightConsumerInfo struct {\n\tops uint64 // Inflight operations, i.e. inflight consumer creates/updates/deletes.\n\tdeleted bool // Whether the consumer has been deleted.\n\t*consumerAssignment\n}\n\n// Used to track inflight peer-remove info to respond 'success' after quorum.\ntype peerRemoveInfo struct {\n\tci *ClientInfo\n\tsubject string\n\treply string\n\trequest string\n}\n\n// Used to guide placement of streams and meta controllers in clustered JetStream.\ntype Placement struct {\n\tCluster string `json:\"cluster,omitempty\"`\n\tTags []string `json:\"tags,omitempty\"`\n\tPreferred string `json:\"preferred,omitempty\"`\n}\n\n// Define types of the entry.\ntype entryOp uint8\n\n// ONLY ADD TO THE END, DO NOT INSERT IN BETWEEN WILL BREAK SERVER INTEROP.\nconst (\n\t// Meta ops.\n\tassignStreamOp entryOp = iota\n\tassignConsumerOp\n\tremoveStreamOp\n\tremoveConsumerOp\n\t// Stream ops.\n\tstreamMsgOp\n\tpurgeStreamOp\n\tdeleteMsgOp\n\t// Consumer ops.\n\tupdateDeliveredOp\n\tupdateAcksOp\n\t// Compressed consumer assignments.\n\tassignCompressedConsumerOp\n\t// Filtered Consumer skip.\n\tupdateSkipOp\n\t// Update Stream.\n\tupdateStreamOp\n\t// For updating information on pending pull requests.\n\taddPendingRequest\n\tremovePendingRequest\n\t// For sending compressed streams, either through RAFT or catchup.\n\tcompressedStreamMsgOp\n\t// For sending deleted gaps on catchups for replicas.\n\tdeleteRangeOp\n\t// Batch stream ops.\n\tbatchMsgOp\n\tbatchCommitMsgOp\n\t// Consumer rest to specific starting sequence.\n\tresetSeqOp\n)\n\n// raftGroups are controlled by the metagroup controller.\n// The raftGroups will house streams and consumers.\ntype raftGroup struct {\n\tName string `json:\"name\"`\n\tPeers []string `json:\"peers\"`\n\tStorage StorageType `json:\"store\"`\n\tCluster string `json:\"cluster,omitempty\"`\n\tPreferred string `json:\"preferred,omitempty\"`\n\tScaleUp bool `json:\"scale_up,omitempty\"`\n\t// Internal\n\tnode RaftNode\n}\n\n// streamAssignment is what the meta controller uses to assign streams to peers.\ntype streamAssignment struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tCreated time.Time `json:\"created\"`\n\tConfigJSON json.RawMessage `json:\"stream\"`\n\tConfig *StreamConfig `json:\"-\"`\n\tGroup *raftGroup `json:\"group\"`\n\tSync string `json:\"sync\"`\n\tSubject string `json:\"subject,omitempty\"`\n\tReply string `json:\"reply,omitempty\"`\n\tRestore *StreamState `json:\"restore_state,omitempty\"`\n\t// Internal\n\tconsumers map[string]*consumerAssignment\n\tresponded bool\n\trecovering bool\n\treassigning bool // i.e. due to placement issues, lack of resources, etc.\n\tresetting bool // i.e. there was an error, and we're stopping and starting the stream\n\terr error\n\tunsupported *unsupportedStreamAssignment\n}\n\ntype unsupportedStreamAssignment struct {\n\treason string\n\tinfo StreamInfo\n\tsysc *client\n\tinfoSub *subscription\n}\n\nfunc newUnsupportedStreamAssignment(s *Server, sa *streamAssignment, err error) *unsupportedStreamAssignment {\n\treason := \"stopped\"\n\tif err != nil {\n\t\tif errstr := err.Error(); strings.HasPrefix(errstr, \"json:\") {\n\t\t\treason = fmt.Sprintf(\"unsupported - config error: %s\", strings.TrimPrefix(err.Error(), \"json: \"))\n\t\t} else {\n\t\t\treason = fmt.Sprintf(\"stopped - %s\", errstr)\n\t\t}\n\t} else if sa.Config != nil && !supportsRequiredApiLevel(sa.Config.Metadata) {\n\t\tif req := getRequiredApiLevel(sa.Config.Metadata); req != _EMPTY_ {\n\t\t\treason = fmt.Sprintf(\"unsupported - required API level: %s, current API level: %d\", req, JSApiLevel)\n\t\t}\n\t}\n\treturn &unsupportedStreamAssignment{\n\t\treason: reason,\n\t\tinfo: StreamInfo{\n\t\t\tCreated: sa.Created,\n\t\t\tConfig: *setDynamicStreamMetadata(sa.Config),\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t\tTimeStamp: time.Now().UTC(),\n\t\t},\n\t}\n}\n\nfunc (usa *unsupportedStreamAssignment) setupInfoSub(s *Server, sa *streamAssignment) {\n\tif usa.infoSub != nil {\n\t\treturn\n\t}\n\n\t// Bind to the system account.\n\tic := s.createInternalJetStreamClient()\n\tic.registerWithAccount(s.SystemAccount())\n\tusa.sysc = ic\n\n\t// Note below the way we subscribe here is so that we can send requests to ourselves.\n\tisubj := fmt.Sprintf(clusterStreamInfoT, sa.Client.serviceAccount(), sa.Config.Name)\n\tusa.infoSub, _ = s.systemSubscribe(isubj, _EMPTY_, false, ic, usa.handleClusterStreamInfoRequest)\n}\n\nfunc (usa *unsupportedStreamAssignment) handleClusterStreamInfoRequest(_ *subscription, c *client, _ *Account, _, reply string, _ []byte) {\n\ts, acc := c.srv, c.acc\n\tinfo := streamInfoClusterResponse{OfflineReason: usa.reason, StreamInfo: usa.info}\n\ts.sendDelayedErrResponse(acc, reply, nil, s.jsonResponse(&info), errRespDelay)\n}\n\nfunc (usa *unsupportedStreamAssignment) closeInfoSub(s *Server) {\n\tif usa.infoSub != nil {\n\t\ts.sysUnsubscribe(usa.infoSub)\n\t\tusa.infoSub = nil\n\t}\n\tif usa.sysc != nil {\n\t\tusa.sysc.closeConnection(ClientClosed)\n\t\tusa.sysc = nil\n\t}\n}\n\n// consumerAssignment is what the meta controller uses to assign consumers to streams.\ntype consumerAssignment struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tCreated time.Time `json:\"created\"`\n\tName string `json:\"name\"`\n\tStream string `json:\"stream\"`\n\tConfigJSON json.RawMessage `json:\"consumer\"`\n\tConfig *ConsumerConfig `json:\"-\"`\n\tGroup *raftGroup `json:\"group\"`\n\tSubject string `json:\"subject,omitempty\"`\n\tReply string `json:\"reply,omitempty\"`\n\tState *ConsumerState `json:\"state,omitempty\"`\n\t// Internal\n\tresponded bool\n\trecovering bool\n\terr error\n\tunsupported *unsupportedConsumerAssignment\n}\n\ntype unsupportedConsumerAssignment struct {\n\treason string\n\tinfo ConsumerInfo\n\tsysc *client\n\tinfoSub *subscription\n}\n\nfunc newUnsupportedConsumerAssignment(ca *consumerAssignment, err error) *unsupportedConsumerAssignment {\n\treason := \"stopped\"\n\tif err != nil {\n\t\tif errstr := err.Error(); strings.HasPrefix(errstr, \"json:\") {\n\t\t\treason = fmt.Sprintf(\"unsupported - config error: %s\", strings.TrimPrefix(err.Error(), \"json: \"))\n\t\t} else {\n\t\t\treason = fmt.Sprintf(\"stopped - %s\", errstr)\n\t\t}\n\t} else if ca.Config != nil && !supportsRequiredApiLevel(ca.Config.Metadata) {\n\t\tif req := getRequiredApiLevel(ca.Config.Metadata); req != _EMPTY_ {\n\t\t\treason = fmt.Sprintf(\"unsupported - required API level: %s, current API level: %d\", getRequiredApiLevel(ca.Config.Metadata), JSApiLevel)\n\t\t}\n\t}\n\treturn &unsupportedConsumerAssignment{\n\t\treason: reason,\n\t\tinfo: ConsumerInfo{\n\t\t\tStream: ca.Stream,\n\t\t\tName: ca.Name,\n\t\t\tCreated: ca.Created,\n\t\t\tConfig: setDynamicConsumerMetadata(ca.Config),\n\t\t\tTimeStamp: time.Now().UTC(),\n\t\t},\n\t}\n}\n\nfunc (uca *unsupportedConsumerAssignment) setupInfoSub(s *Server, ca *consumerAssignment) {\n\tif uca.infoSub != nil {\n\t\treturn\n\t}\n\n\t// Bind to the system account.\n\tic := s.createInternalJetStreamClient()\n\tic.registerWithAccount(s.SystemAccount())\n\tuca.sysc = ic\n\n\t// Note below the way we subscribe here is so that we can send requests to ourselves.\n\tisubj := fmt.Sprintf(clusterConsumerInfoT, ca.Client.serviceAccount(), ca.Stream, ca.Name)\n\tuca.infoSub, _ = s.systemSubscribe(isubj, _EMPTY_, false, ic, uca.handleClusterConsumerInfoRequest)\n}\n\nfunc (uca *unsupportedConsumerAssignment) handleClusterConsumerInfoRequest(_ *subscription, c *client, _ *Account, _, reply string, _ []byte) {\n\ts, acc := c.srv, c.acc\n\tinfo := consumerInfoClusterResponse{OfflineReason: uca.reason, ConsumerInfo: uca.info}\n\ts.sendDelayedErrResponse(acc, reply, nil, s.jsonResponse(&info), errRespDelay)\n}\n\nfunc (uca *unsupportedConsumerAssignment) closeInfoSub(s *Server) {\n\tif uca.infoSub != nil {\n\t\ts.sysUnsubscribe(uca.infoSub)\n\t\tuca.infoSub = nil\n\t}\n\tif uca.sysc != nil {\n\t\tuca.sysc.closeConnection(ClientClosed)\n\t\tuca.sysc = nil\n\t}\n}\n\ntype writeableConsumerAssignment struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tCreated time.Time `json:\"created\"`\n\tName string `json:\"name\"`\n\tStream string `json:\"stream\"`\n\tConfigJSON json.RawMessage `json:\"consumer\"`\n\tGroup *raftGroup `json:\"group\"`\n}\n\n// streamPurge is what the stream leader will replicate when purging a stream.\ntype streamPurge struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tStream string `json:\"stream\"`\n\tLastSeq uint64 `json:\"last_seq\"`\n\tSubject string `json:\"subject\"`\n\tReply string `json:\"reply\"`\n\tRequest *JSApiStreamPurgeRequest `json:\"request,omitempty\"`\n}\n\n// streamMsgDelete is what the stream leader will replicate when deleting a message.\ntype streamMsgDelete struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tStream string `json:\"stream\"`\n\tSeq uint64 `json:\"seq\"`\n\tNoErase bool `json:\"no_erase,omitempty\"`\n\tSubject string `json:\"subject\"`\n\tReply string `json:\"reply\"`\n}\n\nconst (\n\tdefaultStoreDirName = \"_js_\"\n\tdefaultMetaGroupName = \"_meta_\"\n\tdefaultMetaFSBlkSize = 1024 * 1024\n\tjsExcludePlacement = \"!jetstream\"\n)\n\n// Returns information useful in mixed mode.\nfunc (s *Server) trackedJetStreamServers() (js, total int) {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif !s.isRunning() || !s.eventsEnabled() {\n\t\treturn -1, -1\n\t}\n\ts.nodeToInfo.Range(func(k, v any) bool {\n\t\tsi := v.(nodeInfo)\n\t\tif si.js {\n\t\t\tjs++\n\t\t}\n\t\ttotal++\n\t\treturn true\n\t})\n\treturn js, total\n}\n\nfunc (s *Server) getJetStreamCluster() (*jetStream, *jetStreamCluster) {\n\tif s.isShuttingDown() {\n\t\treturn nil, nil\n\t}\n\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn nil, nil\n\t}\n\n\t// Only set once, do not need a lock.\n\treturn js, js.cluster\n}\n\nfunc (s *Server) JetStreamIsClustered() bool {\n\treturn s.jsClustered.Load()\n}\n\nfunc (s *Server) JetStreamIsLeader() bool {\n\treturn s.isMetaLeader.Load()\n}\n\nfunc (s *Server) JetStreamIsCurrent() bool {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn false\n\t}\n\t// Grab what we need and release js lock.\n\tjs.mu.RLock()\n\tvar meta RaftNode\n\tcc := js.cluster\n\tif cc != nil {\n\t\tmeta = cc.meta\n\t}\n\tjs.mu.RUnlock()\n\n\tif cc == nil {\n\t\t// Non-clustered mode\n\t\treturn true\n\t}\n\treturn meta.Current()\n}\n\nfunc (s *Server) JetStreamSnapshotMeta() error {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\tjs.mu.RLock()\n\tcc := js.cluster\n\tisLeader := cc.isLeader()\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\tif !isLeader {\n\t\treturn errNotLeader\n\t}\n\n\tsnap, _, _, err := js.metaSnapshot()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn meta.InstallSnapshot(snap, false)\n}\n\nfunc (s *Server) JetStreamStepdownStream(account, stream string) error {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\tif cc == nil {\n\t\treturn NewJSClusterNotActiveError()\n\t}\n\t// Grab account\n\tacc, err := s.LookupAccount(account)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Grab stream\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif node := mset.raftNode(); node != nil {\n\t\tnode.StepDown()\n\t}\n\n\treturn nil\n}\n\nfunc (s *Server) JetStreamStepdownConsumer(account, stream, consumer string) error {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil {\n\t\treturn NewJSNotEnabledError()\n\t}\n\tif cc == nil {\n\t\treturn NewJSClusterNotActiveError()\n\t}\n\t// Grab account\n\tacc, err := s.LookupAccount(account)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Grab stream\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\to := mset.lookupConsumer(consumer)\n\tif o == nil {\n\t\treturn NewJSConsumerNotFoundError()\n\t}\n\n\tif node := o.raftNode(); node != nil {\n\t\tnode.StepDown()\n\t}\n\n\treturn nil\n}\n\nfunc (s *Server) JetStreamSnapshotStream(account, stream string) error {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\tif cc == nil {\n\t\treturn NewJSClusterNotActiveError()\n\t}\n\t// Grab account\n\tacc, err := s.LookupAccount(account)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Grab stream\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Hold lock when installing snapshot.\n\tmset.mu.Lock()\n\tif mset.node == nil {\n\t\tmset.mu.Unlock()\n\t\treturn nil\n\t}\n\terr = mset.node.InstallSnapshot(mset.stateSnapshotLocked(), false)\n\tmset.mu.Unlock()\n\n\treturn err\n}\n\nfunc (s *Server) JetStreamClusterPeers() []string {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn nil\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\n\tcc := js.cluster\n\tif !cc.isLeader() || cc.meta == nil {\n\t\treturn nil\n\t}\n\tpeers := cc.meta.Peers()\n\tvar nodes []string\n\tfor _, p := range peers {\n\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\tif !ok || si == nil {\n\t\t\tcontinue\n\t\t}\n\t\tni := si.(nodeInfo)\n\t\t// Ignore if offline, no JS, or no current stats have been received.\n\t\tif ni.offline || !ni.js || ni.stats == nil {\n\t\t\tcontinue\n\t\t}\n\t\tnodes = append(nodes, si.(nodeInfo).name)\n\t}\n\treturn nodes\n}\n\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isLeader() bool {\n\tif cc == nil {\n\t\t// Non-clustered mode\n\t\treturn true\n\t}\n\treturn cc.meta != nil && cc.meta.Leader()\n}\n\n// isStreamCurrent will determine if the stream is up to date.\n// For R1 it will make sure the stream is present on this server.\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isStreamCurrent(account, stream string) bool {\n\tif cc == nil {\n\t\t// Non-clustered mode\n\t\treturn true\n\t}\n\tas := cc.streams[account]\n\tif as == nil {\n\t\treturn false\n\t}\n\tsa := as[stream]\n\tif sa == nil {\n\t\treturn false\n\t}\n\trg := sa.Group\n\tif rg == nil {\n\t\treturn false\n\t}\n\n\tif rg.node == nil || rg.node.Current() {\n\t\t// Check if we are processing a snapshot and are catching up.\n\t\tacc, err := cc.s.LookupAccount(account)\n\t\tif err != nil {\n\t\t\treturn false\n\t\t}\n\t\tmset, err := acc.lookupStream(stream)\n\t\tif err != nil {\n\t\t\treturn false\n\t\t}\n\t\tif mset.isCatchingUp() {\n\t\t\treturn false\n\t\t}\n\t\t// Success.\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// isStreamHealthy will determine if the stream is up to date or very close.\n// For R1 it will make sure the stream is present on this server.\nfunc (js *jetStream) isStreamHealthy(acc *Account, sa *streamAssignment) error {\n\tjs.mu.RLock()\n\tif sa != nil && sa.unsupported != nil {\n\t\tjs.mu.RUnlock()\n\t\treturn nil\n\t}\n\ts, cc := js.srv, js.cluster\n\tif cc == nil {\n\t\t// Non-clustered mode\n\t\tjs.mu.RUnlock()\n\t\treturn nil\n\t}\n\tif sa == nil || sa.Group == nil {\n\t\tjs.mu.RUnlock()\n\t\treturn errors.New(\"stream assignment or group missing\")\n\t}\n\tstreamName := sa.Config.Name\n\tnode := sa.Group.node\n\tjs.mu.RUnlock()\n\n\t// First lookup stream and make sure its there.\n\tmset, err := acc.lookupStream(streamName)\n\tif err != nil {\n\t\treturn errors.New(\"stream not found\")\n\t}\n\n\tmsetNode := mset.raftNode()\n\tmset.cfgMu.RLock()\n\treplicas := mset.cfg.Replicas\n\tmset.cfgMu.RUnlock()\n\tvar nrgWerr error\n\tif node != nil {\n\t\tnrgWerr = node.GetWriteErr()\n\t}\n\tstreamWerr := mset.getWriteErr()\n\tswitch {\n\tcase replicas <= 1:\n\t\treturn nil // No further checks for R=1 streams\n\n\tcase node == nil:\n\t\treturn errors.New(\"group node missing\")\n\n\tcase msetNode == nil:\n\t\t// Can happen when the stream's node is not yet initialized.\n\t\treturn errors.New(\"stream node missing\")\n\n\tcase node != msetNode:\n\t\ts.Warnf(\"Detected stream cluster node skew '%s > %s'\", acc.GetName(), streamName)\n\t\treturn errors.New(\"cluster node skew detected\")\n\n\tcase nrgWerr != nil:\n\t\treturn fmt.Errorf(\"node write error: %v\", nrgWerr)\n\n\tcase streamWerr != nil:\n\t\treturn fmt.Errorf(\"stream write error: %v\", streamWerr)\n\n\tcase !mset.isMonitorRunning():\n\t\treturn errors.New(\"monitor goroutine not running\")\n\n\tcase mset.isCatchingUp():\n\t\treturn errors.New(\"stream catching up\")\n\n\tcase !node.Healthy():\n\t\treturn errors.New(\"group node unhealthy\")\n\n\tdefault:\n\t\treturn nil\n\t}\n}\n\n// isConsumerHealthy will determine if the consumer is up to date.\n// For R1 it will make sure the consunmer is present on this server.\nfunc (js *jetStream) isConsumerHealthy(mset *stream, consumer string, ca *consumerAssignment) error {\n\tjs.mu.RLock()\n\tif ca != nil && ca.unsupported != nil {\n\t\tjs.mu.RUnlock()\n\t\treturn nil\n\t}\n\tif mset == nil {\n\t\treturn errors.New(\"stream missing\")\n\t}\n\ts, cc := js.srv, js.cluster\n\tif cc == nil {\n\t\t// Non-clustered mode\n\t\tjs.mu.RUnlock()\n\t\treturn nil\n\t}\n\tif ca == nil || ca.Group == nil {\n\t\tjs.mu.RUnlock()\n\t\treturn errors.New(\"consumer assignment or group missing\")\n\t}\n\tcreated := ca.Created\n\tnode := ca.Group.node\n\tjs.mu.RUnlock()\n\n\t// Check if not running at all.\n\to := mset.lookupConsumer(consumer)\n\tif o == nil {\n\t\tif time.Since(created) < 5*time.Second {\n\t\t\t// No further checks, consumer is not available yet but should be soon.\n\t\t\t// We'll start erroring once we're sure this consumer is actually broken.\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.New(\"consumer not found\")\n\t}\n\n\toNode := o.raftNode()\n\trc, _ := o.replica()\n\tswitch {\n\tcase rc <= 1:\n\t\treturn nil // No further checks for R=1 consumers\n\n\tcase node == nil:\n\t\treturn errors.New(\"group node missing\")\n\n\tcase oNode == nil:\n\t\t// Can happen when the consumer's node is not yet initialized.\n\t\treturn errors.New(\"consumer node missing\")\n\n\tcase node != oNode:\n\t\tmset.mu.RLock()\n\t\taccName, streamName := mset.acc.GetName(), mset.cfg.Name\n\t\tmset.mu.RUnlock()\n\t\ts.Warnf(\"Detected consumer cluster node skew '%s > %s > %s'\", accName, streamName, consumer)\n\t\treturn errors.New(\"cluster node skew detected\")\n\n\tcase !o.isMonitorRunning():\n\t\treturn errors.New(\"monitor goroutine not running\")\n\n\tcase !node.Healthy():\n\t\treturn errors.New(\"group node unhealthy\")\n\n\tdefault:\n\t\treturn nil\n\t}\n}\n\n// subjectsOverlap checks all existing stream assignments for the account cross-cluster for subject overlap\n// Use only for clustered JetStream\n// Read lock should be held.\nfunc (js *jetStream) subjectsOverlap(acc string, subjects []string, osa *streamAssignment) bool {\n\tfor sa := range js.streamAssignmentsOrInflightSeq(acc) {\n\t\t// can't overlap yourself, assume osa pre-checked for deep equal if passed\n\t\tif osa != nil && sa.Config.Name == osa.Config.Name {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, subj := range sa.Config.Subjects {\n\t\t\tfor _, tsubj := range subjects {\n\t\t\t\tif SubjectsCollide(tsubj, subj) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *Account) getJetStreamFromAccount() (*Server, *jetStream, *jsAccount) {\n\ta.mu.RLock()\n\tjsa := a.js\n\ta.mu.RUnlock()\n\tif jsa == nil {\n\t\treturn nil, nil, nil\n\t}\n\tjsa.mu.RLock()\n\tjs := jsa.js\n\tjsa.mu.RUnlock()\n\tif js == nil {\n\t\treturn nil, nil, nil\n\t}\n\t// Lock not needed, set on creation.\n\ts := js.srv\n\treturn s, js, jsa\n}\n\nfunc (s *Server) JetStreamIsStreamLeader(account, stream string) bool {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn cc.isStreamLeader(account, stream)\n}\n\nfunc (a *Account) JetStreamIsStreamLeader(stream string) bool {\n\ts, js, jsa := a.getJetStreamFromAccount()\n\tif s == nil || js == nil || jsa == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.cluster.isStreamLeader(a.Name, stream)\n}\n\nfunc (s *Server) JetStreamIsStreamCurrent(account, stream string) bool {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn cc.isStreamCurrent(account, stream)\n}\n\nfunc (a *Account) JetStreamIsConsumerLeader(stream, consumer string) bool {\n\ts, js, jsa := a.getJetStreamFromAccount()\n\tif s == nil || js == nil || jsa == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.cluster.isConsumerLeader(a.Name, stream, consumer)\n}\n\nfunc (s *Server) JetStreamIsConsumerLeader(account, stream, consumer string) bool {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn cc.isConsumerLeader(account, stream, consumer)\n}\n\nfunc (s *Server) enableJetStreamClustering() error {\n\tif !s.isRunning() {\n\t\treturn nil\n\t}\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\t// Already set.\n\tif js.cluster != nil {\n\t\treturn nil\n\t}\n\n\ts.Noticef(\"Starting JetStream cluster\")\n\t// We need to determine if we have a stable cluster name and expected number of servers.\n\ts.Debugf(\"JetStream cluster checking for stable cluster name and peers\")\n\n\thasLeafNodeSystemShare := s.canExtendOtherDomain()\n\tif s.isClusterNameDynamic() && !hasLeafNodeSystemShare {\n\t\treturn errors.New(\"JetStream cluster requires cluster name\")\n\t}\n\tif s.configuredRoutes() == 0 && !hasLeafNodeSystemShare {\n\t\treturn errors.New(\"JetStream cluster requires configured routes or solicited leafnode for the system account\")\n\t}\n\n\treturn js.setupMetaGroup()\n}\n\n// isClustered returns if we are clustered.\n// Lock should not be held.\nfunc (js *jetStream) isClustered() bool {\n\t// This is only ever set, no need for lock here.\n\treturn js.cluster != nil\n}\n\n// isClusteredNoLock returns if we are clustered, but unlike isClustered() does\n// not use the jetstream's lock, instead, uses an atomic operation.\n// There are situations where some code wants to know if we are clustered but\n// can't use js.isClustered() without causing a lock inversion.\nfunc (js *jetStream) isClusteredNoLock() bool {\n\treturn atomic.LoadInt32(&js.clustered) == 1\n}\n\nfunc (js *jetStream) setupMetaGroup() error {\n\ts := js.srv\n\ts.Noticef(\"Creating JetStream metadata controller\")\n\n\t// Setup our WAL for the metagroup.\n\tsysAcc := s.SystemAccount()\n\tif sysAcc == nil {\n\t\treturn ErrNoSysAccount\n\t}\n\tstoreDir := filepath.Join(js.config.StoreDir, sysAcc.Name, defaultStoreDirName, defaultMetaGroupName)\n\n\tjs.srv.optsMu.RLock()\n\tsyncAlways := js.srv.opts.SyncAlways\n\tsyncInterval := js.srv.opts.SyncInterval\n\tjs.srv.optsMu.RUnlock()\n\tfs, err := newFileStoreWithCreated(\n\t\tFileStoreConfig{StoreDir: storeDir, BlockSize: defaultMetaFSBlkSize, AsyncFlush: false, SyncAlways: syncAlways, SyncInterval: syncInterval, srv: s},\n\t\tStreamConfig{Name: defaultMetaGroupName, Storage: FileStorage},\n\t\ttime.Now().UTC(),\n\t\ts.jsKeyGen(s.getOpts().JetStreamKey, defaultMetaGroupName),\n\t\ts.jsKeyGen(s.getOpts().JetStreamOldKey, defaultMetaGroupName),\n\t)\n\tif err != nil {\n\t\ts.Errorf(\"Error creating filestore: %v\", err)\n\t\treturn err\n\t}\n\n\tcfg := &RaftConfig{Name: defaultMetaGroupName, Store: storeDir, Log: fs, Recovering: true}\n\n\t// If we are soliciting leafnode connections and we are sharing a system account and do not disable it with a hint,\n\t// we want to move to observer mode so that we extend the solicited cluster or supercluster but do not form our own.\n\tcfg.Observer = s.canExtendOtherDomain() && s.getOpts().JetStreamExtHint != jsNoExtend\n\n\tvar bootstrap bool\n\tif ps, err := readPeerState(storeDir); err != nil {\n\t\ts.Noticef(\"JetStream cluster bootstrapping\")\n\t\tbootstrap = true\n\t\tpeers := s.ActivePeers()\n\t\ts.Debugf(\"JetStream cluster initial peers: %+v\", peers)\n\t\tif err := s.bootstrapRaftNode(cfg, peers, false); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif cfg.Observer {\n\t\t\ts.Noticef(\"Turning JetStream metadata controller Observer Mode on\")\n\t\t\ts.Noticef(\"In cases where the JetStream domain is not intended to be extended through a SYS account leaf node connection\")\n\t\t\ts.Noticef(\"and waiting for leader election until first contact is not acceptable,\")\n\t\t\ts.Noticef(`manually disable Observer Mode by setting the JetStream Option \"extension_hint: %s\"`, jsNoExtend)\n\t\t}\n\t} else {\n\t\ts.Noticef(\"JetStream cluster recovering state\")\n\t\t// correlate the value of observer with observations from a previous run.\n\t\tif cfg.Observer {\n\t\t\tswitch ps.domainExt {\n\t\t\tcase extExtended:\n\t\t\t\ts.Noticef(\"Keeping JetStream metadata controller Observer Mode on - due to previous contact\")\n\t\t\tcase extNotExtended:\n\t\t\t\ts.Noticef(\"Turning JetStream metadata controller Observer Mode off - due to previous contact\")\n\t\t\t\tcfg.Observer = false\n\t\t\tcase extUndetermined:\n\t\t\t\ts.Noticef(\"Turning JetStream metadata controller Observer Mode on - no previous contact\")\n\t\t\t\ts.Noticef(\"In cases where the JetStream domain is not intended to be extended through a SYS account leaf node connection\")\n\t\t\t\ts.Noticef(\"and waiting for leader election until first contact is not acceptable,\")\n\t\t\t\ts.Noticef(`manually disable Observer Mode by setting the JetStream Option \"extension_hint: %s\"`, jsNoExtend)\n\t\t\t}\n\t\t} else {\n\t\t\t// To track possible configuration changes, responsible for an altered value of cfg.Observer,\n\t\t\t// set extension state to undetermined.\n\t\t\tps.domainExt = extUndetermined\n\t\t\tif err := writePeerState(storeDir, ps); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Start up our meta node.\n\tn, err := s.startRaftNode(sysAcc.GetName(), cfg, pprofLabels{\n\t\t\"type\": \"metaleader\",\n\t\t\"account\": sysAcc.Name,\n\t})\n\tif err != nil {\n\t\ts.Warnf(\"Could not start metadata controller: %v\", err)\n\t\treturn err\n\t}\n\n\t// If we are bootstrapped with no state, start campaign early.\n\tif bootstrap {\n\t\tn.Campaign()\n\t}\n\n\tc := s.createInternalJetStreamClient()\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tjs.cluster = &jetStreamCluster{\n\t\tmeta: n,\n\t\tstreams: make(map[string]map[string]*streamAssignment),\n\t\ts: s,\n\t\tc: c,\n\t\tqch: make(chan struct{}),\n\t\tstopped: make(chan struct{}),\n\t}\n\tatomic.StoreInt32(&js.clustered, 1)\n\tc.registerWithAccount(sysAcc)\n\n\t// Set to true before we start.\n\tjs.metaRecovering = true\n\tjs.srv.startGoRoutine(\n\t\tjs.monitorCluster,\n\t\tpprofLabels{\n\t\t\t\"type\": \"metaleader\",\n\t\t\t\"account\": sysAcc.Name,\n\t\t},\n\t)\n\treturn nil\n}\n\nfunc (js *jetStream) getMetaGroup() RaftNode {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif js.cluster == nil {\n\t\treturn nil\n\t}\n\treturn js.cluster.meta\n}\n\nfunc (js *jetStream) server() *Server {\n\t// Lock not needed, only set once on creation.\n\treturn js.srv\n}\n\n// Will respond if we do not think we have a metacontroller leader.\nfunc (js *jetStream) isLeaderless() bool {\n\tjs.mu.RLock()\n\tcc := js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\tjs.mu.RUnlock()\n\t\treturn false\n\t}\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\t// If we don't have a leader.\n\t// Make sure we have been running for enough time.\n\tif meta.Leaderless() && time.Since(meta.Created()) > lostQuorumIntervalDefault {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Will respond iff we are a member and we know we have no leader.\nfunc (js *jetStream) isGroupLeaderless(rg *raftGroup) bool {\n\tif rg == nil || js == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tcc := js.cluster\n\tstarted := js.started\n\n\t// If we are not a member we can not say..\n\tif cc.meta == nil {\n\t\tjs.mu.RUnlock()\n\t\treturn false\n\t}\n\tif !rg.isMember(cc.meta.ID()) {\n\t\tjs.mu.RUnlock()\n\t\treturn false\n\t}\n\t// Single peer groups always have a leader if we are here.\n\tif rg.node == nil {\n\t\tjs.mu.RUnlock()\n\t\treturn false\n\t}\n\tnode := rg.node\n\tjs.mu.RUnlock()\n\t// If we don't have a leader.\n\tif node.Leaderless() {\n\t\t// Threshold for jetstream startup.\n\t\tconst startupThreshold = 10 * time.Second\n\n\t\tif node.HadPreviousLeader() {\n\t\t\t// Make sure we have been running long enough to intelligently determine this.\n\t\t\tif time.Since(started) > startupThreshold {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\t// Make sure we have been running for enough time.\n\t\tif time.Since(node.Created()) > lostQuorumIntervalDefault {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc (s *Server) JetStreamIsStreamAssigned(account, stream string) bool {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn false\n\t}\n\tacc, _ := s.LookupAccount(account)\n\tif acc == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tassigned := cc.isStreamAssigned(acc, stream)\n\tjs.mu.RUnlock()\n\treturn assigned\n}\n\n// streamAssigned informs us if this server has this stream assigned.\nfunc (jsa *jsAccount) streamAssigned(stream string) bool {\n\tjsa.mu.RLock()\n\tjs, acc := jsa.js, jsa.account\n\tjsa.mu.RUnlock()\n\n\tif js == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tassigned := js.cluster.isStreamAssigned(acc, stream)\n\tjs.mu.RUnlock()\n\treturn assigned\n}\n\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isStreamAssigned(a *Account, stream string) bool {\n\t// Non-clustered mode always return true.\n\tif cc == nil {\n\t\treturn true\n\t}\n\tif cc.meta == nil {\n\t\treturn false\n\t}\n\tas := cc.streams[a.Name]\n\tif as == nil {\n\t\treturn false\n\t}\n\tsa := as[stream]\n\tif sa == nil {\n\t\treturn false\n\t}\n\treturn sa.Group.isMember(cc.meta.ID())\n}\n\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isStreamLeader(account, stream string) bool {\n\t// Non-clustered mode always return true.\n\tif cc == nil {\n\t\treturn true\n\t}\n\tif cc.meta == nil {\n\t\treturn false\n\t}\n\n\tvar sa *streamAssignment\n\tif as := cc.streams[account]; as != nil {\n\t\tsa = as[stream]\n\t}\n\tif sa == nil {\n\t\treturn false\n\t}\n\trg := sa.Group\n\tif rg == nil {\n\t\treturn false\n\t}\n\t// Check if we are the leader of this raftGroup assigned to the stream.\n\tourID := cc.meta.ID()\n\tfor _, peer := range rg.Peers {\n\t\tif peer == ourID {\n\t\t\tif len(rg.Peers) == 1 || (rg.node != nil && rg.node.Leader()) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isConsumerLeader(account, stream, consumer string) bool {\n\t// Non-clustered mode always return true.\n\tif cc == nil {\n\t\treturn true\n\t}\n\tif cc.meta == nil {\n\t\treturn false\n\t}\n\n\tvar sa *streamAssignment\n\tif as := cc.streams[account]; as != nil {\n\t\tsa = as[stream]\n\t}\n\tif sa == nil {\n\t\treturn false\n\t}\n\t// Check if we are the leader of this raftGroup assigned to this consumer.\n\tca := sa.consumers[consumer]\n\tif ca == nil {\n\t\treturn false\n\t}\n\trg := ca.Group\n\tourID := cc.meta.ID()\n\tfor _, peer := range rg.Peers {\n\t\tif peer == ourID {\n\t\t\tif len(rg.Peers) == 1 || (rg.node != nil && rg.node.Leader()) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Track the stream for the account `accName` in the inflight proposals map.\n// This is done after proposing a stream change.\n// (Write) Lock held on entry.\nfunc (cc *jetStreamCluster) trackInflightStreamProposal(accName string, sa *streamAssignment, deleted bool) {\n\tif cc.inflightStreams == nil {\n\t\tcc.inflightStreams = make(map[string]map[string]*inflightStreamInfo)\n\t}\n\tstreams, ok := cc.inflightStreams[accName]\n\tif !ok {\n\t\tstreams = make(map[string]*inflightStreamInfo)\n\t\tcc.inflightStreams[accName] = streams\n\t}\n\tif inflight, ok := streams[sa.Config.Name]; ok {\n\t\tinflight.ops++\n\t\tinflight.deleted = deleted\n\t\tinflight.streamAssignment = sa\n\t} else {\n\t\tstreams[sa.Config.Name] = &inflightStreamInfo{1, deleted, sa}\n\t}\n}\n\n// Remove the stream `streamName` for the account `accName` from the inflight proposals map.\n// This is done after successful create/update/delete.\n// (Write) Lock held on entry.\nfunc (cc *jetStreamCluster) removeInflightStreamProposal(accName, streamName string) {\n\tif streams, ok := cc.inflightStreams[accName]; !ok {\n\t\treturn // No accounts.\n\t} else if inflight, ok := streams[streamName]; !ok {\n\t\treturn // No changes for this stream.\n\t} else if inflight.ops > 1 {\n\t\t// Decrement one pending operation.\n\t\tinflight.ops--\n\t} else {\n\t\t// No pending operations left, clean up.\n\t\tdelete(streams, streamName)\n\t\tif len(streams) == 0 {\n\t\t\tdelete(cc.inflightStreams, accName)\n\t\t}\n\t}\n}\n\n// Track the consumer for the `streamName` and account `accName` in the inflight proposals map.\n// This is done after proposing a consumer change.\n// (Write) Lock held on entry.\nfunc (cc *jetStreamCluster) trackInflightConsumerProposal(accName, streamName string, ca *consumerAssignment, deleted bool) {\n\tif cc.inflightConsumers == nil {\n\t\tcc.inflightConsumers = make(map[string]map[string]map[string]*inflightConsumerInfo)\n\t}\n\tstreams, ok := cc.inflightConsumers[accName]\n\tif !ok {\n\t\tstreams = make(map[string]map[string]*inflightConsumerInfo)\n\t\tcc.inflightConsumers[accName] = streams\n\t}\n\tconsumers, ok := streams[streamName]\n\tif !ok {\n\t\tconsumers = make(map[string]*inflightConsumerInfo)\n\t\tstreams[streamName] = consumers\n\t}\n\tif inflight, ok := consumers[ca.Name]; ok {\n\t\tinflight.ops++\n\t\tinflight.deleted = deleted\n\t\tinflight.consumerAssignment = ca\n\t} else {\n\t\tconsumers[ca.Name] = &inflightConsumerInfo{1, deleted, ca}\n\t}\n}\n\n// Remove the consumer `consumerName` for the `streamName` and account `accName` from the inflight proposals map.\n// This is done after successful create/update/delete.\n// (Write) Lock held on entry.\nfunc (cc *jetStreamCluster) removeInflightConsumerProposal(accName, streamName, consumerName string) {\n\tif streams, ok := cc.inflightConsumers[accName]; !ok {\n\t\treturn // No accounts.\n\t} else if consumers, ok := streams[streamName]; !ok {\n\t\treturn // No streams.\n\t} else if inflight, ok := consumers[consumerName]; !ok {\n\t\treturn // No changes for this consumer.\n\t} else if inflight.ops > 1 {\n\t\t// Decrement one pending operation.\n\t\tinflight.ops--\n\t} else {\n\t\t// No pending operations left, clean up.\n\t\tdelete(consumers, consumerName)\n\t\tif len(consumers) == 0 {\n\t\t\tdelete(streams, streamName)\n\t\t}\n\t\tif len(streams) == 0 {\n\t\t\tdelete(cc.inflightConsumers, accName)\n\t\t}\n\t}\n}\n\n// Return the cluster quit chan.\nfunc (js *jetStream) clusterQuitC() chan struct{} {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif js.cluster != nil {\n\t\treturn js.cluster.qch\n\t}\n\treturn nil\n}\n\n// Return the cluster stopped chan.\nfunc (js *jetStream) clusterStoppedC() chan struct{} {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif js.cluster != nil {\n\t\treturn js.cluster.stopped\n\t}\n\treturn nil\n}\n\n// Mark that the meta layer is recovering.\nfunc (js *jetStream) setMetaRecovering() {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tif js.cluster != nil {\n\t\t// metaRecovering\n\t\tjs.metaRecovering = true\n\t}\n}\n\n// Mark that the meta layer is no longer recovering.\nfunc (js *jetStream) clearMetaRecovering() {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tjs.metaRecovering = false\n}\n\n// Return whether the meta layer is recovering.\nfunc (js *jetStream) isMetaRecovering() bool {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.metaRecovering\n}\n\n// During recovery track any stream and consumer delete and update operations.\ntype recoveryUpdates struct {\n\tremoveStreams map[string]*streamAssignment\n\tremoveConsumers map[string]map[string]*consumerAssignment\n\taddStreams map[string]*streamAssignment\n\tupdateStreams map[string]*streamAssignment\n\tupdateConsumers map[string]map[string]*consumerAssignment\n}\n\nfunc (ru *recoveryUpdates) removeStream(sa *streamAssignment) {\n\tkey := sa.recoveryKey()\n\tru.removeStreams[key] = sa\n\tdelete(ru.addStreams, key)\n\tdelete(ru.updateStreams, key)\n\tdelete(ru.updateConsumers, key)\n\tdelete(ru.removeConsumers, key)\n}\n\nfunc (ru *recoveryUpdates) addStream(sa *streamAssignment) {\n\tkey := sa.recoveryKey()\n\tru.addStreams[key] = sa\n}\n\nfunc (ru *recoveryUpdates) updateStream(sa *streamAssignment) {\n\tkey := sa.recoveryKey()\n\tru.updateStreams[key] = sa\n}\n\nfunc (ru *recoveryUpdates) removeConsumer(ca *consumerAssignment) {\n\tkey := ca.recoveryKey()\n\tskey := ca.streamRecoveryKey()\n\tif _, ok := ru.removeConsumers[skey]; !ok {\n\t\tru.removeConsumers[skey] = map[string]*consumerAssignment{}\n\t}\n\tru.removeConsumers[skey][key] = ca\n\tif consumers, ok := ru.updateConsumers[skey]; ok {\n\t\tdelete(consumers, key)\n\t}\n}\n\nfunc (ru *recoveryUpdates) addOrUpdateConsumer(ca *consumerAssignment) {\n\tkey := ca.recoveryKey()\n\tskey := ca.streamRecoveryKey()\n\tif _, ok := ru.updateConsumers[skey]; !ok {\n\t\tru.updateConsumers[skey] = map[string]*consumerAssignment{}\n\t}\n\tru.updateConsumers[skey][key] = ca\n}\n\n// Called after recovery of the cluster on startup to check for any orphans.\n// Streams and consumers are recovered from disk, and the meta layer's mappings\n// should clean them up, but under crash scenarios there could be orphans.\nfunc (js *jetStream) checkForOrphans() {\n\t// Can not hold jetstream lock while trying to delete streams or consumers.\n\tjs.mu.Lock()\n\ts, cc := js.srv, js.cluster\n\ts.Debugf(\"JetStream cluster checking for orphans\")\n\n\t// We only want to cleanup any orphans if we know we are current with the meta-leader.\n\tmeta := cc.meta\n\tif meta == nil || meta.Leaderless() {\n\t\tjs.mu.Unlock()\n\t\ts.Debugf(\"JetStream cluster skipping check for orphans, no meta-leader\")\n\t\treturn\n\t}\n\tif !meta.Healthy() {\n\t\tjs.mu.Unlock()\n\t\ts.Debugf(\"JetStream cluster skipping check for orphans, not current with the meta-leader\")\n\t\treturn\n\t}\n\tstreams, consumers := js.getOrphans()\n\tjs.mu.Unlock()\n\n\tfor _, mset := range streams {\n\t\tmset.mu.RLock()\n\t\taccName, stream := mset.acc.Name, mset.cfg.Name\n\t\tmset.mu.RUnlock()\n\t\ts.Warnf(\"Detected orphaned stream '%s > %s', will cleanup\", accName, stream)\n\t\tif err := mset.delete(); err != nil {\n\t\t\ts.Warnf(\"Deleting stream encountered an error: %v\", err)\n\t\t}\n\t}\n\tfor _, o := range consumers {\n\t\to.mu.RLock()\n\t\taccName, mset, consumer := o.acc.Name, o.mset, o.name\n\t\to.mu.RUnlock()\n\t\tstream := \"N/A\"\n\t\tif mset != nil {\n\t\t\tmset.mu.RLock()\n\t\t\tstream = mset.cfg.Name\n\t\t\tmset.mu.RUnlock()\n\t\t}\n\t\tif o.isDurable() {\n\t\t\ts.Warnf(\"Detected orphaned durable consumer '%s > %s > %s', will cleanup\", accName, stream, consumer)\n\t\t} else {\n\t\t\ts.Debugf(\"Detected orphaned consumer '%s > %s > %s', will cleanup\", accName, stream, consumer)\n\t\t}\n\n\t\tif err := o.delete(); err != nil {\n\t\t\ts.Warnf(\"Deleting consumer encountered an error: %v\", err)\n\t\t}\n\t}\n}\n\n// Returns orphaned streams and consumers that were recovered from disk, but don't\n// exist as clustered stream/consumer assignments.\n// Lock should be held.\nfunc (js *jetStream) getOrphans() (streams []*stream, consumers []*consumer) {\n\tcc := js.cluster\n\tfor accName, jsa := range js.accounts {\n\t\tasa := cc.streams[accName]\n\t\tjsa.mu.RLock()\n\t\tfor stream, mset := range jsa.streams {\n\t\t\tif sa := asa[stream]; sa == nil {\n\t\t\t\tstreams = append(streams, mset)\n\t\t\t} else {\n\t\t\t\t// This one is good, check consumers now.\n\t\t\t\tfor _, o := range mset.getPublicConsumers() {\n\t\t\t\t\tif sa.consumers[o.String()] == nil {\n\t\t\t\t\t\tconsumers = append(consumers, o)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tjsa.mu.RUnlock()\n\t}\n\treturn streams, consumers\n}\n\nfunc (js *jetStream) monitorCluster() {\n\ts, n := js.server(), js.getMetaGroup()\n\tqch, stopped, rqch, lch, aq := js.clusterQuitC(), js.clusterStoppedC(), n.QuitC(), n.LeadChangeC(), n.ApplyQ()\n\n\tdefer s.grWG.Done()\n\tdefer close(stopped)\n\n\ts.Debugf(\"Starting metadata monitor\")\n\tdefer s.Debugf(\"Exiting metadata monitor\")\n\n\t// Make sure to stop the raft group on exit to prevent accidental memory bloat.\n\tdefer n.Stop()\n\tdefer s.isMetaLeader.Store(false)\n\n\tconst compactInterval = time.Minute\n\tconst compactMinInterval = 15 * time.Second\n\tt := time.NewTicker(compactInterval)\n\tdefer t.Stop()\n\n\t// Used to check cold boot cluster when possibly in mixed mode.\n\tconst leaderCheckInterval = time.Second\n\tlt := time.NewTicker(leaderCheckInterval)\n\tdefer lt.Stop()\n\n\t// Check the general health once an hour.\n\tconst healthCheckInterval = 1 * time.Hour\n\tht := time.NewTicker(healthCheckInterval)\n\tdefer ht.Stop()\n\n\t// Utility to check health.\n\tcheckHealth := func() {\n\t\tif hs := s.healthz(nil); hs.Error != _EMPTY_ {\n\t\t\ts.Warnf(\"%v\", hs.Error)\n\t\t}\n\t}\n\n\tvar (\n\t\tisLeader bool\n\t\tlastSnapTime time.Time\n\t\tcompactSizeMin = uint64(8 * 1024 * 1024) // 8MB\n\t\tminSnapDelta = 30 * time.Second\n\t)\n\n\t// Highwayhash key for generating hashes.\n\tkey := make([]byte, 32)\n\tcrand.Read(key)\n\n\t// Set to true to start.\n\tjs.setMetaRecovering()\n\trecovering := true\n\n\t// Snapshotting function.\n\tvar (\n\t\tsnapMu sync.Mutex\n\t\tsnapshotting bool\n\t\t// The first snapshot we do after recovery should clean up most of the log.\n\t\tfallbackSnapshot = true\n\t\t// Suppress non-forced snapshots after an error. Timer-based snapshots will become forced.\n\t\t// If failures continue, we force a snapshot even if we were catching up others\n\t\t// (otherwise we might indefinitely stall and grow log size).\n\t\tfailedSnapshots atomic.Uint32\n\t)\n\tdoSnapshot := func(force bool) {\n\t\t// Suppress during recovery.\n\t\t// If snapshots have failed, and we're not forced to, we'll wait for the timer since it'll now be forced.\n\t\tif recovering || (!force && failedSnapshots.Load() > 0) {\n\t\t\treturn\n\t\t}\n\t\t// Suppress if an async snapshot is already in progress.\n\t\tsnapMu.Lock()\n\t\tif snapshotting {\n\t\t\tsnapMu.Unlock()\n\t\t\treturn\n\t\t}\n\t\t// Look up what the threshold is for compaction. Re-reading from config here as it is reloadable.\n\t\tjs.srv.optsMu.RLock()\n\t\tethresh := js.srv.opts.JetStreamMetaCompact\n\t\tszthresh := js.srv.opts.JetStreamMetaCompactSize\n\t\t// Allows reverting to sync/blocking snapshots instead of the default async snapshots.\n\t\tsyncSnapshot := js.srv.opts.JetStreamMetaCompactSync\n\t\tjs.srv.optsMu.RUnlock()\n\t\t// Work out our criteria for snapshotting.\n\t\tbyEntries, bySize := ethresh > 0, szthresh > 0\n\t\tbyNeither := !byEntries && !bySize\n\t\t// For the meta layer we want to snapshot when over the above threshold (which could be 0 by default).\n\t\tne, nsz := n.Size()\n\t\tcreateSnapshot := force || byNeither || (byEntries && ne > ethresh) || (bySize && nsz > szthresh) || n.NeedSnapshot()\n\t\tif !createSnapshot {\n\t\t\tsnapMu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Start a checkpoint, we can either install the snapshot sync or async from here.\n\t\t// If we had a significant number of failed snapshots, start relaxing Raft-layer checks\n\t\t// to force it through. We might have been catching up a peer for a long period, and this\n\t\t// protects our log size from growing indefinitely.\n\t\tforceSnapshot := failedSnapshots.Load() > 4\n\t\tc, err := n.CreateSnapshotCheckpoint(forceSnapshot)\n\t\tif err != nil {\n\t\t\tif err != errNoSnapAvailable && err != errNodeClosed {\n\t\t\t\ts.Warnf(\"Error snapshotting JetStream cluster state: %v\", err)\n\t\t\t\t// If this is the first failure, reduce the interval of the snapshot timer.\n\t\t\t\t// This ensures we're not waiting too long for snapshotting to eventually become forced.\n\t\t\t\tif failedSnapshots.Load() == 0 {\n\t\t\t\t\tt.Reset(compactMinInterval)\n\t\t\t\t}\n\t\t\t\tfailedSnapshots.Add(1)\n\t\t\t}\n\t\t\tsnapMu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// If we're meant to install an async snapshot, check if the underlying log has grown way too large.\n\t\t// In that case, we fall back to a synchronous/blocking snapshot after all. This is a protective\n\t\t// measure to prevent the log from growing faster than we can asynchronously compact.\n\t\tif !fallbackSnapshot && !syncSnapshot {\n\t\t\tif szthresh == 0 {\n\t\t\t\tszthresh = compactSizeMin\n\t\t\t}\n\t\t\tif thresh := 10 * szthresh; nsz >= thresh {\n\t\t\t\ts.rateLimitFormatWarnf(\"JetStream cluster metalayer log size has exceeded async threshold (%s), will fall back to blocking snapshot\", friendlyBytes(thresh))\n\t\t\t\tfallbackSnapshot = true\n\t\t\t}\n\t\t}\n\n\t\t// Check if we should fall back to a blocking snapshot, either due to failures or extreme log size.\n\t\tif fallbackSnapshot || syncSnapshot {\n\t\t\tif !syncSnapshot {\n\t\t\t\ts.rateLimitFormatWarnf(\"Metalayer blocking snapshot starting\")\n\t\t\t}\n\t\t\tstart := time.Now()\n\t\t\tsnap, nsa, nca, err := js.metaSnapshot()\n\t\t\tif err != nil {\n\t\t\t\ts.Warnf(\"Error generating JetStream cluster snapshot: %v\", err)\n\t\t\t\tc.Abort()\n\t\t\t} else if csz, err := c.InstallSnapshot(snap); err == nil {\n\t\t\t\tlastSnapTime = time.Now()\n\t\t\t\t// If there was a failed snapshot before, we reduced the timer's interval.\n\t\t\t\t// Reset it back to the original interval now.\n\t\t\t\tif failedSnapshots.Load() > 0 {\n\t\t\t\t\tt.Reset(compactInterval)\n\t\t\t\t}\n\t\t\t\tfailedSnapshots.Store(0)\n\t\t\t\t// Fallback snapshot was successful, the next one can be async again.\n\t\t\t\tfallbackSnapshot = false\n\t\t\t\ttook := time.Since(start)\n\t\t\t\tif !syncSnapshot || took > 2*time.Second {\n\t\t\t\t\ts.rateLimitFormatWarnf(\"Metalayer blocking snapshot took %.3fs (streams: %d, consumers: %d, compacted: %s)\",\n\t\t\t\t\t\ttook.Seconds(), nsa, nca, friendlyBytes(csz))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tc.Abort()\n\t\t\t\tif err != errNoSnapAvailable && err != errNodeClosed && err != errSnapAborted {\n\t\t\t\t\ts.Warnf(\"Error snapshotting JetStream cluster state: %v\", err)\n\t\t\t\t\t// If this is the first failure, reduce the interval of the snapshot timer.\n\t\t\t\t\t// This ensures we're not waiting too long for snapshotting to eventually become forced.\n\t\t\t\t\tif failedSnapshots.Load() == 0 {\n\t\t\t\t\t\tt.Reset(compactMinInterval)\n\t\t\t\t\t}\n\t\t\t\t\tfailedSnapshots.Add(1)\n\t\t\t\t}\n\t\t\t}\n\t\t\tsnapMu.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Perform the snapshot asynchronously.\n\t\tstart := time.Now()\n\t\tsnapshotting = true\n\t\tsnapMu.Unlock()\n\t\ts.startGoRoutine(func() {\n\t\t\tdefer s.grWG.Done()\n\n\t\t\tabort := func(err error) {\n\t\t\t\tsnapMu.Lock()\n\t\t\t\tc.Abort()\n\t\t\t\tif err != errNoSnapAvailable && err != errNodeClosed && err != errSnapAborted {\n\t\t\t\t\ts.Warnf(\"Error snapshotting JetStream cluster state: %v, will fall back to blocking snapshot\", err)\n\t\t\t\t\tfallbackSnapshot = true\n\t\t\t\t\t// If this is the first failure, reduce the interval of the snapshot timer.\n\t\t\t\t\t// This ensures we're not waiting too long for snapshotting to eventually become forced.\n\t\t\t\t\tif failedSnapshots.Load() == 0 {\n\t\t\t\t\t\tt.Reset(compactMinInterval)\n\t\t\t\t\t}\n\t\t\t\t\tfailedSnapshots.Add(1)\n\t\t\t\t}\n\t\t\t\tsnapshotting = false\n\t\t\t\tsnapMu.Unlock()\n\t\t\t}\n\n\t\t\t// The strategy of asynchronous snapshotting:\n\t\t\t// - Not holding any locks. Only minimal for the Raft node itself, importantly not the JS lock.\n\t\t\t// - Load the last snapshot.\n\t\t\t// - Replay stream/consumer changes through the assignment state machine.\n\t\t\t// - Encode and install as the new snapshot.\n\t\t\tif data, err := c.LoadLastSnapshot(); err != nil && err != errNoSnapAvailable {\n\t\t\t\tabort(err)\n\t\t\t} else if streams, err := js.decodeMetaSnapshot(data); err != nil {\n\t\t\t\tabort(err)\n\t\t\t} else if err = js.collectStreamAndConsumerChanges(c, streams); err != nil {\n\t\t\t\tabort(err)\n\t\t\t} else if snap, nsa, nca, err := js.encodeMetaSnapshot(streams); err != nil {\n\t\t\t\tabort(err)\n\t\t\t} else if csz, err := c.InstallSnapshot(snap); err != nil {\n\t\t\t\tabort(err)\n\t\t\t} else {\n\t\t\t\t// Successful snapshot.\n\t\t\t\tlastSnapTime = time.Now()\n\t\t\t\t// If there was a failed snapshot before, we reduced the timer's interval.\n\t\t\t\t// Reset it back to the original interval now.\n\t\t\t\tif failedSnapshots.Load() > 0 {\n\t\t\t\t\tt.Reset(compactInterval)\n\t\t\t\t}\n\t\t\t\tfailedSnapshots.Store(0)\n\t\t\t\tif took := time.Since(start); took > 2*time.Second {\n\t\t\t\t\ts.rateLimitFormatWarnf(\"Metalayer async snapshot took %.3fs (streams: %d, consumers: %d, compacted: %s)\",\n\t\t\t\t\t\ttook.Seconds(), nsa, nca, friendlyBytes(csz))\n\t\t\t\t}\n\t\t\t\tsnapMu.Lock()\n\t\t\t\tsnapshotting = false\n\t\t\t\tsnapMu.Unlock()\n\t\t\t}\n\t\t})\n\t}\n\n\tvar ru *recoveryUpdates\n\n\t// Make sure to cancel any pending checkForOrphans calls if the\n\t// monitor goroutine exits.\n\tvar oc *time.Timer\n\tdefer stopAndClearTimer(&oc)\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\t// Server shutting down, but we might receive this before qch, so try to snapshot.\n\t\t\tsnapMu.Lock()\n\t\t\tfallbackSnapshot = true\n\t\t\tsnapMu.Unlock()\n\t\t\tdoSnapshot(false)\n\t\t\treturn\n\t\tcase <-rqch:\n\t\t\t// Raft node is closed, no use in trying to snapshot.\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\t// Clean signal from shutdown routine so do best effort attempt to snapshot meta layer.\n\t\t\tsnapMu.Lock()\n\t\t\tfallbackSnapshot = true\n\t\t\tsnapMu.Unlock()\n\t\t\tdoSnapshot(false)\n\t\t\treturn\n\t\tcase <-aq.ch:\n\t\t\tces := aq.pop()\n\t\t\tfor _, ce := range ces {\n\t\t\t\tif recovering && ru == nil {\n\t\t\t\t\tru = &recoveryUpdates{\n\t\t\t\t\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\t\t\t\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\t\t\t\t\taddStreams: make(map[string]*streamAssignment),\n\t\t\t\t\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\t\t\t\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif ce == nil {\n\t\t\t\t\tif ru != nil {\n\t\t\t\t\t\t// Process any removes that are still valid after recovery.\n\t\t\t\t\t\tfor _, cas := range ru.removeConsumers {\n\t\t\t\t\t\t\tfor _, ca := range cas {\n\t\t\t\t\t\t\t\tjs.processConsumerRemoval(ca)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfor _, sa := range ru.removeStreams {\n\t\t\t\t\t\t\tjs.processStreamRemoval(sa)\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Process stream additions.\n\t\t\t\t\t\tfor _, sa := range ru.addStreams {\n\t\t\t\t\t\t\tjs.processStreamAssignment(sa)\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Process pending updates.\n\t\t\t\t\t\tfor _, sa := range ru.updateStreams {\n\t\t\t\t\t\t\tjs.processUpdateStreamAssignment(sa)\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Now consumers.\n\t\t\t\t\t\tfor _, cas := range ru.updateConsumers {\n\t\t\t\t\t\t\tfor _, ca := range cas {\n\t\t\t\t\t\t\t\tjs.processConsumerAssignment(ca)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Signals we have replayed all of our metadata.\n\t\t\t\t\twasMetaRecovering := js.isMetaRecovering()\n\t\t\t\t\tjs.clearMetaRecovering()\n\t\t\t\t\trecovering = false\n\t\t\t\t\t// Clear.\n\t\t\t\t\tru = nil\n\t\t\t\t\ts.Debugf(\"Recovered JetStream cluster metadata\")\n\t\t\t\t\t// Snapshot now so we start with freshly compacted log.\n\t\t\t\t\tdoSnapshot(true)\n\t\t\t\t\tif wasMetaRecovering {\n\t\t\t\t\t\t// Reset, it could be we didn't need to install a snapshot. This ensures we don't degrade\n\t\t\t\t\t\t// to a blocking snapshot if we install our first snapshot during normal operations.\n\t\t\t\t\t\tsnapMu.Lock()\n\t\t\t\t\t\tfallbackSnapshot = false\n\t\t\t\t\t\tsnapMu.Unlock()\n\t\t\t\t\t\toc = time.AfterFunc(30*time.Second, js.checkForOrphans)\n\t\t\t\t\t\t// Do a health check here as well.\n\t\t\t\t\t\tgo checkHealth()\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif isRecovering, didSnap, err := js.applyMetaEntries(ce.Entries, ru); err == nil {\n\t\t\t\t\tvar nb uint64\n\t\t\t\t\t// Some entries can fail without an error when shutting down, don't move applied forward.\n\t\t\t\t\tif !js.isShuttingDown() {\n\t\t\t\t\t\t_, nb = n.Applied(ce.Index)\n\t\t\t\t\t}\n\t\t\t\t\tif js.hasPeerEntries(ce.Entries) || (didSnap && !isLeader) {\n\t\t\t\t\t\tdoSnapshot(true)\n\t\t\t\t\t} else if nb > compactSizeMin && time.Since(lastSnapTime) > minSnapDelta {\n\t\t\t\t\t\tdoSnapshot(false)\n\t\t\t\t\t}\n\t\t\t\t\trecovering = isRecovering\n\t\t\t\t} else {\n\t\t\t\t\ts.Warnf(\"Error applying JetStream cluster entries: %v\", err)\n\t\t\t\t}\n\t\t\t\tce.ReturnToPool()\n\t\t\t}\n\t\t\taq.recycle(&ces)\n\n\t\tcase isLeader = <-lch:\n\t\t\t// Process the change.\n\t\t\tjs.processLeaderChange(isLeader)\n\t\t\tif isLeader {\n\t\t\t\ts.sendInternalMsgLocked(serverStatsPingReqSubj, _EMPTY_, nil, nil)\n\t\t\t\t// Install a snapshot as we become leader.\n\t\t\t\tjs.checkClusterSize()\n\t\t\t\tdoSnapshot(false)\n\t\t\t}\n\n\t\tcase <-t.C:\n\t\t\t// Start forcing snapshots if they failed previously.\n\t\t\tforceIfFailed := failedSnapshots.Load() > 0\n\t\t\tdoSnapshot(forceIfFailed)\n\t\t\t// Periodically check the cluster size.\n\t\t\tif n.Leader() {\n\t\t\t\tjs.checkClusterSize()\n\t\t\t}\n\t\tcase <-ht.C:\n\t\t\t// Do this in a separate go routine.\n\t\t\tgo checkHealth()\n\n\t\tcase <-lt.C:\n\t\t\ts.Debugf(\"Checking JetStream cluster state\")\n\t\t\t// If we have a current leader or had one in the past we can cancel this here since the metaleader\n\t\t\t// will be in charge of all peer state changes.\n\t\t\t// For cold boot only.\n\t\t\tif !n.Leaderless() || n.HadPreviousLeader() {\n\t\t\t\tlt.Stop()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If we are here we do not have a leader and we did not have a previous one, so cold start.\n\t\t\t// Check to see if we can adjust our cluster size down iff we are in mixed mode and we have\n\t\t\t// seen a total that is what our original estimate was.\n\t\t\tcs := n.ClusterSize()\n\t\t\tif js, total := s.trackedJetStreamServers(); js < total && total >= cs && js != cs {\n\t\t\t\ts.Noticef(\"Adjusting JetStream expected peer set size to %d from original %d\", js, cs)\n\t\t\t\tn.AdjustBootClusterSize(js)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// This is called on first leader transition to double check the peers and cluster set size.\nfunc (js *jetStream) checkClusterSize() {\n\ts, n := js.server(), js.getMetaGroup()\n\tif n == nil {\n\t\treturn\n\t}\n\t// We will check that we have a correct cluster set size by checking for any non-js servers\n\t// which can happen in mixed mode.\n\tps := n.(*raft).currentPeerState()\n\tif len(ps.knownPeers) >= ps.clusterSize {\n\t\treturn\n\t}\n\n\t// Grab our active peers.\n\tpeers := s.ActivePeers()\n\n\t// If we have not registered all of our peers yet we can't do\n\t// any adjustments based on a mixed mode. We will periodically check back.\n\tif len(peers) < ps.clusterSize {\n\t\treturn\n\t}\n\n\ts.Debugf(\"Checking JetStream cluster size\")\n\n\t// If we are here our known set as the leader is not the same as the cluster size.\n\t// Check to see if we have a mixed mode setup.\n\tvar totalJS int\n\tfor _, p := range peers {\n\t\tif si, ok := s.nodeToInfo.Load(p); ok && si != nil {\n\t\t\tif si.(nodeInfo).js {\n\t\t\t\ttotalJS++\n\t\t\t}\n\t\t}\n\t}\n\t// If we have less then our cluster size adjust that here. Can not do individual peer removals since\n\t// they will not be in the tracked peers.\n\tif totalJS < ps.clusterSize {\n\t\ts.Debugf(\"Adjusting JetStream cluster size from %d to %d\", ps.clusterSize, totalJS)\n\t\tif err := n.AdjustClusterSize(totalJS); err != nil {\n\t\t\ts.Warnf(\"Error adjusting JetStream cluster size: %v\", err)\n\t\t}\n\t}\n}\n\n// Represents our stable meta state that we can write out.\ntype writeableStreamAssignment struct {\n\tClient *ClientInfo `json:\"client,omitempty\"`\n\tCreated time.Time `json:\"created\"`\n\tConfigJSON json.RawMessage `json:\"stream\"`\n\tGroup *raftGroup `json:\"group\"`\n\tSync string `json:\"sync\"`\n\tConsumers []*writeableConsumerAssignment\n}\n\nfunc (js *jetStream) clusterStreamConfig(accName, streamName string) (StreamConfig, bool) {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif sa, ok := js.cluster.streams[accName][streamName]; ok {\n\t\treturn *sa.Config, true\n\t}\n\treturn StreamConfig{}, false\n}\n\nfunc (js *jetStream) metaSnapshot() ([]byte, int, int, error) {\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tcc := js.cluster\n\treturn js.encodeMetaSnapshot(cc.streams)\n}\n\nfunc (js *jetStream) applyMetaSnapshot(buf []byte, ru *recoveryUpdates, isRecovering bool) error {\n\tstreams, err := js.decodeMetaSnapshot(buf)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tjs.mu.Lock()\n\tcc := js.cluster\n\n\tvar saAdd, saDel, saChk []*streamAssignment\n\t// Walk through the old list to generate the delete list.\n\tfor account, asa := range cc.streams {\n\t\tnasa := streams[account]\n\t\tfor sn, sa := range asa {\n\t\t\tif nsa := nasa[sn]; nsa == nil {\n\t\t\t\tsaDel = append(saDel, sa)\n\t\t\t} else {\n\t\t\t\tsaChk = append(saChk, nsa)\n\t\t\t}\n\t\t}\n\t}\n\t// Walk through the new list to generate the add list.\n\tfor account, nasa := range streams {\n\t\tasa := cc.streams[account]\n\t\tfor sn, sa := range nasa {\n\t\t\tif asa[sn] == nil {\n\t\t\t\tsaAdd = append(saAdd, sa)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Now walk the ones to check and process consumers.\n\tvar caAdd, caDel []*consumerAssignment\n\tfor _, sa := range saChk {\n\t\t// Make sure to add in all the new ones from sa.\n\t\tfor _, ca := range sa.consumers {\n\t\t\tcaAdd = append(caAdd, ca)\n\t\t}\n\t\tif osa := js.streamAssignment(sa.Client.serviceAccount(), sa.Config.Name); osa != nil {\n\t\t\tfor _, ca := range osa.consumers {\n\t\t\t\t// Consumer was either removed, or recreated with a different raft group.\n\t\t\t\tif nca := sa.consumers[ca.Name]; nca == nil {\n\t\t\t\t\tcaDel = append(caDel, ca)\n\t\t\t\t} else if nca.Group != nil && ca.Group != nil && nca.Group.Name != ca.Group.Name {\n\t\t\t\t\tcaDel = append(caDel, ca)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tjs.mu.Unlock()\n\n\t// Do removals first.\n\tfor _, sa := range saDel {\n\t\tjs.setStreamAssignmentRecovering(sa)\n\t\tif isRecovering {\n\t\t\tru.removeStream(sa)\n\t\t} else {\n\t\t\tjs.processStreamRemoval(sa)\n\t\t}\n\t}\n\t// Now do add for the streams. Also add in all consumers.\n\tfor _, sa := range saAdd {\n\t\tconsumers := sa.consumers\n\t\tjs.setStreamAssignmentRecovering(sa)\n\t\tif isRecovering {\n\t\t\t// Since we're recovering and storing up changes, we'll need to clear out these consumers.\n\t\t\t// Some might be removed, and we'll recover those later, must not be able to remember them.\n\t\t\tsa.consumers = nil\n\t\t\tru.addStream(sa)\n\t\t} else {\n\t\t\tjs.processStreamAssignment(sa)\n\t\t}\n\n\t\t// We can simply process the consumers.\n\t\tfor _, ca := range consumers {\n\t\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\t\tif isRecovering {\n\t\t\t\tru.addOrUpdateConsumer(ca)\n\t\t\t} else {\n\t\t\t\tjs.processConsumerAssignment(ca)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Perform updates on those in saChk. These were existing so make\n\t// sure to process any changes.\n\tfor _, sa := range saChk {\n\t\tjs.setStreamAssignmentRecovering(sa)\n\t\tif isRecovering {\n\t\t\tru.updateStream(sa)\n\t\t} else {\n\t\t\tjs.processUpdateStreamAssignment(sa)\n\t\t}\n\t}\n\n\t// Now do the deltas for existing stream's consumers.\n\tfor _, ca := range caDel {\n\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\tif isRecovering {\n\t\t\tru.removeConsumer(ca)\n\t\t} else {\n\t\t\tjs.processConsumerRemoval(ca)\n\t\t}\n\t}\n\tfor _, ca := range caAdd {\n\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\tif isRecovering {\n\t\t\tru.addOrUpdateConsumer(ca)\n\t\t} else {\n\t\t\tjs.processConsumerAssignment(ca)\n\t\t}\n\t}\n\n\t// If we're not recovering, we need to check if we have any left-over streams or consumers that aren't\n\t// tracked in our assignments. This could happen if we've restarted and recovered streams or consumers\n\t// from disk, but then got sent a snapshot to catch up from the meta-leader.\n\tif !isRecovering {\n\t\t// This logic is similar to that of checkForOrphans. But, this cleanup is focused on aligning with\n\t\t// a snapshot from a meta leader as a result of catchup. We silently delete them here, instead of\n\t\t// logging and sending out advisories.\n\t\tjs.mu.RLock()\n\t\tdeleteStreams, deleteConsumers := js.getOrphans()\n\t\tjs.mu.RUnlock()\n\t\tfor _, mset := range deleteStreams {\n\t\t\tmset.stop(true, false)\n\t\t}\n\t\tfor _, o := range deleteConsumers {\n\t\t\to.deleteWithoutAdvisory()\n\t\t}\n\t}\n\treturn nil\n}\n\n// Decode the meta snapshot from buf into the relevant stream and consumer assignments.\nfunc (js *jetStream) decodeMetaSnapshot(buf []byte) (map[string]map[string]*streamAssignment, error) {\n\tvar wsas []writeableStreamAssignment\n\tif len(buf) > 0 {\n\t\tjse, err := s2.Decode(nil, buf)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif err = json.Unmarshal(jse, &wsas); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Build our new version here outside of js.\n\tstreams := make(map[string]map[string]*streamAssignment)\n\tfor _, wsa := range wsas {\n\t\tas := streams[wsa.Client.serviceAccount()]\n\t\tif as == nil {\n\t\t\tas = make(map[string]*streamAssignment)\n\t\t\tstreams[wsa.Client.serviceAccount()] = as\n\t\t}\n\t\tsa := &streamAssignment{Client: wsa.Client, Created: wsa.Created, ConfigJSON: wsa.ConfigJSON, Group: wsa.Group, Sync: wsa.Sync}\n\t\tif err := decodeStreamAssignmentConfig(js.srv, sa); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(wsa.Consumers) > 0 {\n\t\t\tsa.consumers = make(map[string]*consumerAssignment)\n\t\t\tfor _, wca := range wsa.Consumers {\n\t\t\t\tif wca.Stream == _EMPTY_ {\n\t\t\t\t\twca.Stream = sa.Config.Name // Rehydrate from the stream name.\n\t\t\t\t}\n\t\t\t\tca := &consumerAssignment{Client: wca.Client, Created: wca.Created, Name: wca.Name, Stream: wca.Stream, ConfigJSON: wca.ConfigJSON, Group: wca.Group}\n\t\t\t\tif err := decodeConsumerAssignmentConfig(ca); err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tsa.consumers[ca.Name] = ca\n\t\t\t}\n\t\t}\n\t\tas[sa.Config.Name] = sa\n\t}\n\treturn streams, nil\n}\n\n// Encode the meta assignments into an encoded and compressed buffer.\n// Returns the snapshot itself, and the amount of streams and consumers.\nfunc (js *jetStream) encodeMetaSnapshot(streams map[string]map[string]*streamAssignment) ([]byte, int, int, error) {\n\tstart := time.Now()\n\tnsa := 0\n\tnca := 0\n\tfor _, asa := range streams {\n\t\tnsa += len(asa)\n\t}\n\tout := make([]writeableStreamAssignment, 0, nsa)\n\tfor _, asa := range streams {\n\t\tfor _, sa := range asa {\n\t\t\twsa := writeableStreamAssignment{\n\t\t\t\tClient: sa.Client.forAssignmentSnap(),\n\t\t\t\tCreated: sa.Created,\n\t\t\t\tConfigJSON: sa.ConfigJSON,\n\t\t\t\tGroup: sa.Group,\n\t\t\t\tSync: sa.Sync,\n\t\t\t\tConsumers: make([]*writeableConsumerAssignment, 0, len(sa.consumers)),\n\t\t\t}\n\t\t\tfor _, ca := range sa.consumers {\n\t\t\t\twca := writeableConsumerAssignment{\n\t\t\t\t\tClient: ca.Client.forAssignmentSnap(),\n\t\t\t\t\tCreated: ca.Created,\n\t\t\t\t\tName: ca.Name,\n\t\t\t\t\tStream: ca.Stream,\n\t\t\t\t\tConfigJSON: ca.ConfigJSON,\n\t\t\t\t\tGroup: ca.Group,\n\t\t\t\t}\n\t\t\t\twsa.Consumers = append(wsa.Consumers, &wca)\n\t\t\t\tnca++\n\t\t\t}\n\t\t\tout = append(out, wsa)\n\t\t}\n\t}\n\n\tif len(out) == 0 {\n\t\treturn nil, nsa, nca, nil\n\t}\n\n\t// Track how long it took to marshal the JSON\n\tmstart := time.Now()\n\tb, err := json.Marshal(out)\n\tmend := time.Since(mstart)\n\n\t// Must not be possible for a JSON marshaling error to result\n\t// in an empty snapshot.\n\tif err != nil {\n\t\treturn nil, nsa, nca, err\n\t}\n\n\t// Track how long it took to compress the JSON.\n\tcstart := time.Now()\n\tsnap := s2.Encode(nil, b)\n\tcend := time.Since(cstart)\n\ttook := time.Since(start)\n\n\tif took > 2*time.Second {\n\t\tjs.srv.rateLimitFormatWarnf(\"Metalayer snapshot generation took %.3fs (streams: %d, consumers: %d, marshal: %.3fs, s2: %.3fs, uncompressed: %s, compressed: %s)\",\n\t\t\ttook.Seconds(), nsa, nca, mend.Seconds(), cend.Seconds(), friendlyBytes(len(b)), friendlyBytes(len(snap)))\n\t}\n\n\t// Track in jsz monitoring as well.\n\tif cc := js.cluster; cc != nil {\n\t\tatomic.StoreInt64(&cc.lastMetaSnapTime, start.UnixNano())\n\t\tatomic.StoreInt64(&cc.lastMetaSnapDuration, int64(took))\n\t}\n\n\treturn snap, nsa, nca, nil\n}\n\n// Given a checkpoint, collects all relevant append entries that can be snapshotted.\nfunc (js *jetStream) collectStreamAndConsumerChanges(c RaftNodeCheckpoint, streams map[string]map[string]*streamAssignment) error {\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\tfor ae, err := range c.AppendEntriesSeq() {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, e := range ae.entries {\n\t\t\tif e.Type == EntryNormal {\n\t\t\t\tbuf := e.Data\n\t\t\t\top := entryOp(buf[0])\n\t\t\t\tswitch op {\n\t\t\t\tcase assignStreamOp, updateStreamOp, removeStreamOp:\n\t\t\t\t\tsa, err := decodeStreamAssignment(js.srv, buf[1:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode stream assignment: %q\", buf[1:])\n\t\t\t\t\t\tpanic(err)\n\t\t\t\t\t}\n\t\t\t\t\tif op == removeStreamOp {\n\t\t\t\t\t\tru.removeStream(sa)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tru.addStream(sa)\n\t\t\t\t\t}\n\t\t\t\tcase assignConsumerOp:\n\t\t\t\t\tca, err := decodeConsumerAssignment(buf[1:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode consumer assignment: %q\", buf[1:])\n\t\t\t\t\t\tpanic(err)\n\t\t\t\t\t}\n\t\t\t\t\tru.addOrUpdateConsumer(ca)\n\t\t\t\tcase assignCompressedConsumerOp:\n\t\t\t\t\tca, err := decodeConsumerAssignmentCompressed(buf[1:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode compressed consumer assignment: %q\", buf[1:])\n\t\t\t\t\t\tpanic(err)\n\t\t\t\t\t}\n\t\t\t\t\tru.addOrUpdateConsumer(ca)\n\t\t\t\tcase removeConsumerOp:\n\t\t\t\t\tca, err := decodeConsumerAssignment(buf[1:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode consumer assignment: %q\", buf[1:])\n\t\t\t\t\t\tpanic(err)\n\t\t\t\t\t}\n\t\t\t\t\tru.removeConsumer(ca)\n\t\t\t\tdefault:\n\t\t\t\t\tpanic(fmt.Sprintf(\"JetStream Cluster Unknown meta entry op type: %v\", entryOp(buf[0])))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// With our recovery structure, apply the stream/consumer diff.\n\tfor _, cas := range ru.removeConsumers {\n\t\tfor _, ca := range cas {\n\t\t\tif asa, ok := streams[ca.Client.serviceAccount()]; ok {\n\t\t\t\tif sa, ok := asa[ca.Stream]; ok {\n\t\t\t\t\tdelete(sa.consumers, ca.Name)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tfor _, sa := range ru.removeStreams {\n\t\tif asa, ok := streams[sa.Client.serviceAccount()]; ok {\n\t\t\tdelete(asa, sa.Config.Name)\n\t\t}\n\t}\n\tfor _, sa := range ru.addStreams {\n\t\tas := streams[sa.Client.serviceAccount()]\n\t\tif as == nil {\n\t\t\tas = make(map[string]*streamAssignment)\n\t\t\tstreams[sa.Client.serviceAccount()] = as\n\t\t}\n\t\t// Preserve consumers from the previous assignment.\n\t\tif osa := as[sa.Config.Name]; osa != nil {\n\t\t\tsa.consumers = osa.consumers\n\t\t}\n\t\tas[sa.Config.Name] = sa\n\t}\n\tfor _, cas := range ru.updateConsumers {\n\t\tfor _, ca := range cas {\n\t\t\tif asa, ok := streams[ca.Client.serviceAccount()]; ok {\n\t\t\t\tif sa, ok := asa[ca.Stream]; ok {\n\t\t\t\t\tif sa.consumers == nil {\n\t\t\t\t\t\tsa.consumers = make(map[string]*consumerAssignment)\n\t\t\t\t\t}\n\t\t\t\t\tsa.consumers[ca.Name] = ca\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Called on recovery to make sure we do not process like original.\nfunc (js *jetStream) setStreamAssignmentRecovering(sa *streamAssignment) {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tsa.responded = true\n\tsa.recovering = true\n\tsa.Restore = nil\n\tif sa.Group != nil {\n\t\tsa.Group.Preferred = _EMPTY_\n\t\tsa.Group.ScaleUp = false\n\t}\n}\n\n// Called on recovery to make sure we do not process like original.\nfunc (js *jetStream) setConsumerAssignmentRecovering(ca *consumerAssignment) {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tca.responded = true\n\tca.recovering = true\n\tif ca.Group != nil {\n\t\tca.Group.Preferred = _EMPTY_\n\t\tca.Group.ScaleUp = false\n\t}\n}\n\n// Just copies over and changes out the group so it can be encoded.\n// Lock should be held.\nfunc (sa *streamAssignment) copyGroup() *streamAssignment {\n\tcsa, cg := *sa, *sa.Group\n\tcsa.Group = &cg\n\tcsa.Group.Peers = copyStrings(sa.Group.Peers)\n\treturn &csa\n}\n\n// Just copies over and changes out the group so it can be encoded.\n// Lock should be held.\nfunc (ca *consumerAssignment) copyGroup() *consumerAssignment {\n\tcca, cg := *ca, *ca.Group\n\tcca.Group = &cg\n\tcca.Group.Peers = copyStrings(ca.Group.Peers)\n\treturn &cca\n}\n\n// Lock should be held.\nfunc (sa *streamAssignment) missingPeers() bool {\n\treturn len(sa.Group.Peers) < sa.Config.Replicas\n}\n\n// Called when we detect a new peer. Only the leader will process checking\n// for any streams, and consequently any consumers.\nfunc (js *jetStream) processAddPeer(peer string) {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\ts, cc := js.srv, js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\treturn\n\t}\n\tisLeader := cc.isLeader()\n\n\t// Now check if we are meta-leader. We will check for any re-assignments.\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tsir, ok := s.nodeToInfo.Load(peer)\n\tif !ok || sir == nil {\n\t\treturn\n\t}\n\tsi := sir.(nodeInfo)\n\n\tfor accName, sa := range js.streamAssignmentsOrInflightSeqAllAccounts() {\n\t\tif sa.unsupported != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif sa.missingPeers() {\n\t\t\t// Make sure the right cluster etc.\n\t\t\tif si.cluster != sa.Client.Cluster {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If we are here we can add in this peer.\n\t\t\tcsa := sa.copyGroup()\n\t\t\tcsa.Group.Peers = append(csa.Group.Peers, peer)\n\t\t\t// Send our proposal for this csa. Also use same group definition for all the consumers as well.\n\t\t\tcc.meta.Propose(encodeAddStreamAssignment(csa))\n\t\t\tcc.trackInflightStreamProposal(accName, csa, false)\n\t\t\tfor ca := range js.consumerAssignmentsOrInflightSeq(accName, csa.Config.Name) {\n\t\t\t\tif ca.unsupported != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Ephemerals are R=1, so only auto-remap durables, or R>1.\n\t\t\t\tif ca.Config.Durable != _EMPTY_ || len(ca.Group.Peers) > 1 {\n\t\t\t\t\tcca := ca.copyGroup()\n\t\t\t\t\tcca.Group.Peers = csa.Group.Peers\n\t\t\t\t\tcc.meta.Propose(encodeAddConsumerAssignment(cca))\n\t\t\t\t\tcc.trackInflightConsumerProposal(accName, csa.Config.Name, cca, false)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (js *jetStream) processRemovePeer(peer string) {\n\t// We may be already disabled.\n\tif js == nil || js.disabled.Load() {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\ts, cc := js.srv, js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\tisLeader := cc.isLeader()\n\t// All nodes will check if this is them.\n\tisUs := cc.meta.ID() == peer\n\tjs.mu.Unlock()\n\n\tif isUs {\n\t\ts.Errorf(\"JetStream being DISABLED, our server was removed from the cluster\")\n\t\tadv := &JSServerRemovedAdvisory{\n\t\t\tTypedEvent: TypedEvent{\n\t\t\t\tType: JSServerRemovedAdvisoryType,\n\t\t\t\tID: nuid.Next(),\n\t\t\t\tTime: time.Now().UTC(),\n\t\t\t},\n\t\t\tServer: s.Name(),\n\t\t\tServerID: s.ID(),\n\t\t\tCluster: s.cachedClusterName(),\n\t\t\tDomain: s.getOpts().JetStreamDomain,\n\t\t}\n\t\ts.publishAdvisory(nil, JSAdvisoryServerRemoved, adv)\n\n\t\tgo s.DisableJetStream()\n\t}\n\n\t// Now check if we are meta-leader. We will attempt re-assignment.\n\tif !isLeader {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tfor _, sa := range js.streamAssignmentsOrInflightSeqAllAccounts() {\n\t\tif sa.unsupported != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif rg := sa.Group; rg.isMember(peer) {\n\t\t\tjs.removePeerFromStreamLocked(sa, peer)\n\t\t}\n\t}\n}\n\n// Assumes all checks have already been done.\nfunc (js *jetStream) removePeerFromStream(sa *streamAssignment, peer string) bool {\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\treturn js.removePeerFromStreamLocked(sa, peer)\n}\n\n// Lock should be held.\nfunc (js *jetStream) removePeerFromStreamLocked(sa *streamAssignment, peer string) bool {\n\tif rg := sa.Group; !rg.isMember(peer) {\n\t\treturn false\n\t}\n\n\ts, cc, csa := js.srv, js.cluster, sa.copyGroup()\n\tif cc == nil || cc.meta == nil {\n\t\treturn false\n\t}\n\treplaced := cc.remapStreamAssignment(csa, peer)\n\taccName := sa.Client.serviceAccount()\n\tif !replaced {\n\t\ts.Warnf(\"JetStream cluster could not replace peer for stream '%s > %s'\", accName, sa.Config.Name)\n\t}\n\n\t// Send our proposal for this csa. Also use same group definition for all the consumers as well.\n\tcc.meta.Propose(encodeAddStreamAssignment(csa))\n\tcc.trackInflightStreamProposal(accName, csa, false)\n\trg := csa.Group\n\tfor ca := range js.consumerAssignmentsOrInflightSeq(accName, sa.Config.Name) {\n\t\tif ca.unsupported != nil {\n\t\t\tcontinue\n\t\t}\n\t\t// Ephemerals are R=1, so only auto-remap durables, or R>1.\n\t\tif ca.Config.Durable != _EMPTY_ {\n\t\t\tcca := ca.copyGroup()\n\t\t\tcca.Group.Peers, cca.Group.Preferred = rg.Peers, _EMPTY_\n\t\t\tcc.meta.Propose(encodeAddConsumerAssignment(cca))\n\t\t\tcc.trackInflightConsumerProposal(accName, csa.Config.Name, cca, false)\n\t\t} else if ca.Group.isMember(peer) {\n\t\t\t// These are ephemerals. Check to see if we deleted this peer.\n\t\t\tcc.meta.Propose(encodeDeleteConsumerAssignment(ca))\n\t\t\tcc.trackInflightConsumerProposal(accName, csa.Config.Name, ca, true)\n\t\t}\n\t}\n\treturn replaced\n}\n\n// Check if we have peer related entries.\nfunc (js *jetStream) hasPeerEntries(entries []*Entry) bool {\n\tfor _, e := range entries {\n\t\tif e.Type == EntryRemovePeer || e.Type == EntryAddPeer {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nconst ksep = \":\"\n\nfunc (sa *streamAssignment) recoveryKey() string {\n\tif sa == nil {\n\t\treturn _EMPTY_\n\t}\n\treturn sa.Client.serviceAccount() + ksep + sa.Config.Name\n}\n\nfunc (ca *consumerAssignment) streamRecoveryKey() string {\n\tif ca == nil {\n\t\treturn _EMPTY_\n\t}\n\treturn ca.Client.serviceAccount() + ksep + ca.Stream\n}\n\nfunc (ca *consumerAssignment) recoveryKey() string {\n\tif ca == nil {\n\t\treturn _EMPTY_\n\t}\n\treturn ca.Client.serviceAccount() + ksep + ca.Stream + ksep + ca.Name\n}\n\nfunc (js *jetStream) applyMetaEntries(entries []*Entry, ru *recoveryUpdates) (bool, bool, error) {\n\tvar didSnap bool\n\tisRecovering := ru != nil\n\n\tfor _, e := range entries {\n\t\t// If we received a lower-level catchup entry, mark that we're recovering.\n\t\t// We can optimize by staging all meta operations until we're caught up.\n\t\t// At that point we can apply the diff in one go.\n\t\tif e.Type == EntryCatchup {\n\t\t\tisRecovering = true\n\t\t\t// A catchup entry only contains this, so we can exit now and have the\n\t\t\t// recoveryUpdates struct be populated for the next invocation of applyMetaEntries.\n\t\t\treturn isRecovering, didSnap, nil\n\t\t}\n\n\t\tif e.Type == EntrySnapshot {\n\t\t\tif err := js.applyMetaSnapshot(e.Data, ru, isRecovering); err != nil {\n\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t}\n\t\t\tdidSnap = true\n\t\t} else if e.Type == EntryRemovePeer {\n\t\t\tif !js.isMetaRecovering() {\n\t\t\t\tpeer := string(e.Data)\n\t\t\t\tjs.processRemovePeer(peer)\n\n\t\t\t\t// The meta leader can now respond to the peer-removal,\n\t\t\t\t// since a quorum of nodes has this in their log.\n\t\t\t\ts := js.srv\n\t\t\t\tif s.JetStreamIsLeader() {\n\t\t\t\t\tvar (\n\t\t\t\t\t\tinfo peerRemoveInfo\n\t\t\t\t\t\tok bool\n\t\t\t\t\t)\n\t\t\t\t\tjs.mu.Lock()\n\t\t\t\t\tif cc := js.cluster; cc != nil && cc.peerRemoveReply != nil {\n\t\t\t\t\t\tif info, ok = cc.peerRemoveReply[peer]; ok {\n\t\t\t\t\t\t\tdelete(cc.peerRemoveReply, peer)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif len(cc.peerRemoveReply) == 0 {\n\t\t\t\t\t\t\tcc.peerRemoveReply = nil\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tjs.mu.Unlock()\n\n\t\t\t\t\tif info.reply != _EMPTY_ {\n\t\t\t\t\t\tsysAcc := s.SystemAccount()\n\t\t\t\t\t\tvar resp = JSApiMetaServerRemoveResponse{ApiResponse: ApiResponse{Type: JSApiMetaServerRemoveResponseType}}\n\t\t\t\t\t\tresp.Success = true\n\t\t\t\t\t\ts.sendAPIResponse(info.ci, sysAcc, info.subject, info.reply, info.request, s.jsonResponse(&resp))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if e.Type == EntryAddPeer {\n\t\t\tif !js.isMetaRecovering() {\n\t\t\t\tjs.processAddPeer(string(e.Data))\n\t\t\t}\n\t\t} else {\n\t\t\tbuf := e.Data\n\t\t\tswitch entryOp(buf[0]) {\n\t\t\tcase assignStreamOp:\n\t\t\t\tsa, err := decodeStreamAssignment(js.srv, buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode stream assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setStreamAssignmentRecovering(sa)\n\t\t\t\t\tru.addStream(sa)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processStreamAssignment(sa)\n\t\t\t\t}\n\t\t\tcase removeStreamOp:\n\t\t\t\tsa, err := decodeStreamAssignment(js.srv, buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode stream assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setStreamAssignmentRecovering(sa)\n\t\t\t\t\tru.removeStream(sa)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processStreamRemoval(sa)\n\t\t\t\t}\n\t\t\tcase assignConsumerOp:\n\t\t\t\tca, err := decodeConsumerAssignment(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode consumer assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\t\t\t\tru.addOrUpdateConsumer(ca)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processConsumerAssignment(ca)\n\t\t\t\t}\n\t\t\tcase assignCompressedConsumerOp:\n\t\t\t\tca, err := decodeConsumerAssignmentCompressed(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode compressed consumer assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\t\t\t\tru.addOrUpdateConsumer(ca)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processConsumerAssignment(ca)\n\t\t\t\t}\n\t\t\tcase removeConsumerOp:\n\t\t\t\tca, err := decodeConsumerAssignment(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode consumer assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\t\t\t\tru.removeConsumer(ca)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processConsumerRemoval(ca)\n\t\t\t\t}\n\t\t\tcase updateStreamOp:\n\t\t\t\tsa, err := decodeStreamAssignment(js.srv, buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tjs.srv.Errorf(\"JetStream cluster failed to decode stream assignment: %q\", buf[1:])\n\t\t\t\t\treturn isRecovering, didSnap, err\n\t\t\t\t}\n\t\t\t\tif isRecovering {\n\t\t\t\t\tjs.setStreamAssignmentRecovering(sa)\n\t\t\t\t\tru.updateStream(sa)\n\t\t\t\t} else {\n\t\t\t\t\tjs.processUpdateStreamAssignment(sa)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tpanic(fmt.Sprintf(\"JetStream Cluster Unknown meta entry op type: %v\", entryOp(buf[0])))\n\t\t\t}\n\t\t}\n\t}\n\treturn isRecovering, didSnap, nil\n}\n\nfunc (rg *raftGroup) isMember(id string) bool {\n\tif rg == nil {\n\t\treturn false\n\t}\n\tfor _, peer := range rg.Peers {\n\t\tif peer == id {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (rg *raftGroup) setPreferred(s *Server) {\n\tif rg == nil || len(rg.Peers) == 0 {\n\t\treturn\n\t}\n\tif len(rg.Peers) == 1 {\n\t\trg.Preferred = rg.Peers[0]\n\t} else {\n\t\tvar online []string\n\t\tfor _, p := range rg.Peers {\n\t\t\tsi, ok := s.nodeToInfo.Load(p)\n\t\t\tif !ok || si == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tni := si.(nodeInfo)\n\t\t\tif ni.offline {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tonline = append(online, p)\n\t\t}\n\n\t\tif len(online) == 0 {\n\t\t\t// No online servers, just randomly select a peer for the preferred.\n\t\t\tpi := rand.Int31n(int32(len(rg.Peers)))\n\t\t\trg.Preferred = rg.Peers[pi]\n\t\t} else if len(online) == 1 {\n\t\t\t// Only one online server.\n\t\t\trg.Preferred = online[0]\n\t\t} else {\n\t\t\t// Randomly select an online peer.\n\t\t\tpi := rand.Int31n(int32(len(online)))\n\t\t\trg.Preferred = online[pi]\n\t\t}\n\t}\n}\n\n// createRaftGroup is called to spin up this raft group if needed.\nfunc (js *jetStream) createRaftGroup(accName string, rg *raftGroup, recovering bool, storage StorageType, labels pprofLabels) (RaftNode, error) {\n\t// Must hold JS lock throughout, otherwise two parallel calls for the same raft group could result\n\t// in duplicate instances for the same identifier, if the current Raft node is shutting down.\n\t// We can release the lock temporarily while waiting for the Raft node to shut down.\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\ts, cc := js.srv, js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\treturn nil, NewJSClusterNotActiveError()\n\t}\n\n\t// If this is a single peer raft group or we are not a member return.\n\tif len(rg.Peers) <= 1 || !rg.isMember(cc.meta.ID()) {\n\t\t// Nothing to do here.\n\t\treturn nil, nil\n\t}\n\n\t// Check if we already have this assigned.\nretry:\n\tif node := s.lookupRaftNode(rg.Name); node != nil {\n\t\tif node.State() == Closed {\n\t\t\t// We're waiting for this node to finish shutting down before we replace it.\n\t\t\tjs.mu.Unlock()\n\t\t\tnode.WaitForStop()\n\t\t\tjs.mu.Lock()\n\t\t\tgoto retry\n\t\t}\n\t\ts.Debugf(\"JetStream cluster already has raft group %q assigned\", rg.Name)\n\t\t// Check and see if the group has the same peers. If not then we\n\t\t// will update the known peers, which will send a peerstate if leader.\n\t\tgroupPeerIDs := append([]string{}, rg.Peers...)\n\t\tvar samePeers bool\n\t\tif nodePeers := node.Peers(); len(rg.Peers) == len(nodePeers) {\n\t\t\tnodePeerIDs := make([]string, 0, len(nodePeers))\n\t\t\tfor _, n := range nodePeers {\n\t\t\t\tnodePeerIDs = append(nodePeerIDs, n.ID)\n\t\t\t}\n\t\t\tslices.Sort(groupPeerIDs)\n\t\t\tslices.Sort(nodePeerIDs)\n\t\t\tsamePeers = slices.Equal(groupPeerIDs, nodePeerIDs)\n\t\t}\n\t\tif !samePeers {\n\t\t\t// At this point we have no way of knowing:\n\t\t\t// 1. Whether the group has lost enough nodes to cause a quorum\n\t\t\t// loss, in which case a proposal may fail, therefore we will\n\t\t\t// force a peerstate write;\n\t\t\t// 2. Whether nodes in the group have other applies queued up\n\t\t\t// that could change the peerstate again, therefore the leader\n\t\t\t// should send out a new proposal anyway too just to make sure\n\t\t\t// that this change gets captured in the log.\n\t\t\tnode.UpdateKnownPeers(groupPeerIDs)\n\n\t\t\t// If the peers changed as a result of an update by the meta layer, we must reflect that in the log of\n\t\t\t// this group. Otherwise, a new peer would come up and instantly reset the peer state back to whatever is\n\t\t\t// in the log at that time, overwriting what the meta layer told it.\n\t\t\t// Will need to address this properly later on, by for example having the meta layer decide the new\n\t\t\t// placement, but have the leader of this group propose it through its own log instead.\n\t\t\tif node.Leader() {\n\t\t\t\tnode.ProposeKnownPeers(groupPeerIDs)\n\t\t\t}\n\t\t}\n\t\trg.node = node\n\t\treturn node, nil\n\t}\n\n\ts.Debugf(\"JetStream cluster creating raft group:%+v\", rg)\n\n\tsysAcc := s.SystemAccount()\n\tif sysAcc == nil {\n\t\ts.Debugf(\"JetStream cluster detected shutdown processing raft group: %+v\", rg)\n\t\treturn nil, errors.New(\"shutting down\")\n\t}\n\n\t// Check here to see if we have a max HA Assets limit set.\n\tif maxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets; maxHaAssets > 0 {\n\t\tif s.numRaftNodes() > maxHaAssets {\n\t\t\ts.Warnf(\"Maximum HA Assets limit reached: %d\", maxHaAssets)\n\t\t\t// Since the meta leader assigned this, send a statsz update to them to get them up to date.\n\t\t\tgo s.sendStatszUpdate()\n\t\t\treturn nil, errors.New(\"system limit reached\")\n\t\t}\n\t}\n\n\tstoreDir := filepath.Join(js.config.StoreDir, sysAcc.Name, defaultStoreDirName, rg.Name)\n\tvar store StreamStore\n\tif storage == FileStorage {\n\t\t// If the server is set to sync always, do the same for the Raft log.\n\t\tjs.srv.optsMu.RLock()\n\t\tsyncAlways := js.srv.opts.SyncAlways\n\t\tsyncInterval := js.srv.opts.SyncInterval\n\t\tjs.srv.optsMu.RUnlock()\n\t\tfs, err := newFileStoreWithCreated(\n\t\t\tFileStoreConfig{StoreDir: storeDir, BlockSize: defaultMediumBlockSize, AsyncFlush: false, SyncAlways: syncAlways, SyncInterval: syncInterval, srv: s},\n\t\t\tStreamConfig{Name: rg.Name, Storage: FileStorage, Metadata: labels},\n\t\t\ttime.Now().UTC(),\n\t\t\ts.jsKeyGen(s.getOpts().JetStreamKey, rg.Name),\n\t\t\ts.jsKeyGen(s.getOpts().JetStreamOldKey, rg.Name),\n\t\t)\n\t\tif err != nil {\n\t\t\ts.Errorf(\"Error creating filestore WAL: %v\", err)\n\t\t\treturn nil, err\n\t\t}\n\t\tstore = fs\n\t} else {\n\t\tms, err := newMemStore(&StreamConfig{Name: rg.Name, Storage: MemoryStorage})\n\t\tif err != nil {\n\t\t\ts.Errorf(\"Error creating memstore WAL: %v\", err)\n\t\t\treturn nil, err\n\t\t}\n\t\tstore = ms\n\t}\n\n\tcfg := &RaftConfig{Name: rg.Name, Store: storeDir, Log: store, Track: true, Recovering: recovering, ScaleUp: rg.ScaleUp}\n\n\tif _, err := readPeerState(storeDir); err != nil {\n\t\ts.bootstrapRaftNode(cfg, rg.Peers, true)\n\t}\n\n\tn, err := s.startRaftNode(accName, cfg, labels)\n\tif err != nil || n == nil {\n\t\ts.Debugf(\"Error creating raft group: %v\", err)\n\t\treturn nil, err\n\t}\n\t// Need JS lock to be held for the assignment to avoid data-race reports\n\trg.node = n\n\t// See if we are preferred and should start campaign immediately.\n\tif n.ID() == rg.Preferred && n.Term() == 0 {\n\t\tn.CampaignImmediately()\n\t}\n\treturn n, nil\n}\n\nfunc (mset *stream) raftGroup() *raftGroup {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\tif mset.sa == nil {\n\t\treturn nil\n\t}\n\treturn mset.sa.Group\n}\n\nfunc (mset *stream) raftNode() RaftNode {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.node\n}\n\nfunc (mset *stream) removeNode() {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tif n := mset.node; n != nil {\n\t\tn.Delete()\n\t\tmset.node = nil\n\t}\n}\n\n// Helper function to generate peer info.\n// lists and sets for old and new.\nfunc genPeerInfo(peers []string, split int) (newPeers, oldPeers []string, newPeerSet, oldPeerSet map[string]bool) {\n\tnewPeers = peers[split:]\n\toldPeers = peers[:split]\n\tnewPeerSet = make(map[string]bool, len(newPeers))\n\toldPeerSet = make(map[string]bool, len(oldPeers))\n\tfor i, peer := range peers {\n\t\tif i < split {\n\t\t\toldPeerSet[peer] = true\n\t\t} else {\n\t\t\tnewPeerSet[peer] = true\n\t\t}\n\t}\n\treturn\n}\n\n// This will wait for a period of time until all consumers are registered and have\n// their consumer assignments assigned.\n// Should only be called from monitorStream.\nfunc (mset *stream) waitOnConsumerAssignments() {\n\tmset.mu.RLock()\n\ts, js, acc, sa, name, replicas := mset.srv, mset.js, mset.acc, mset.sa, mset.cfg.Name, mset.cfg.Replicas\n\tmset.mu.RUnlock()\n\n\tif s == nil || js == nil || acc == nil || sa == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tnumExpectedConsumers := len(sa.consumers)\n\tjs.mu.RUnlock()\n\n\t// Max to wait.\n\tconst maxWaitTime = 10 * time.Second\n\tconst sleepTime = 500 * time.Millisecond\n\n\t// Wait up to 10s\n\ttimeout := time.Now().Add(maxWaitTime)\n\tfor time.Now().Before(timeout) {\n\t\tvar numReady int\n\t\tfor _, o := range mset.getConsumers() {\n\t\t\t// Make sure we are registered with our consumer assignment.\n\t\t\tif ca := o.consumerAssignment(); ca != nil {\n\t\t\t\tif replicas > 1 && !o.isMonitorRunning() {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tnumReady++\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// Check if we are good.\n\t\tif numReady >= numExpectedConsumers {\n\t\t\tbreak\n\t\t}\n\n\t\ts.Debugf(\"Waiting for consumers for interest based stream '%s > %s'\", acc.Name, name)\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-mset.monitorQuitC():\n\t\t\treturn\n\t\tcase <-time.After(sleepTime):\n\t\t}\n\t}\n\n\tif actual := mset.numConsumers(); actual < numExpectedConsumers {\n\t\ts.Warnf(\"All consumers not online for '%s > %s': expected %d but only have %d\", acc.Name, name, numExpectedConsumers, actual)\n\t}\n}\n\n// Monitor our stream node for this stream.\nfunc (js *jetStream) monitorStream(mset *stream, sa *streamAssignment, sendSnapshot bool) {\n\ts, cc := js.server(), js.cluster\n\tdefer s.grWG.Done()\n\tif mset != nil {\n\t\tdefer mset.monitorWg.Done()\n\t}\n\tjs.mu.RLock()\n\tn := sa.Group.node\n\tmeta := cc.meta\n\tjs.mu.RUnlock()\n\n\tif n == nil || meta == nil {\n\t\ts.Warnf(\"No RAFT group for '%s > %s'\", sa.Client.serviceAccount(), sa.Config.Name)\n\t\treturn\n\t}\n\n\t// Make sure only one is running.\n\tif mset != nil {\n\t\tif mset.checkInMonitor() {\n\t\t\treturn\n\t\t}\n\t\tdefer mset.clearMonitorRunning()\n\t}\n\n\t// Make sure to stop the raft group on exit to prevent accidental memory bloat.\n\t// This should be below the checkInMonitor call though to avoid stopping it out\n\t// from underneath the one that is running since it will be the same raft node.\n\tdefer func() {\n\t\t// We might be closing during shutdown, don't pre-emptively stop here since we'll still want to install snapshots.\n\t\tif mset != nil && !mset.closed.Load() {\n\t\t\tn.Stop()\n\t\t}\n\t}()\n\n\tqch, mqch, lch, aq, uch, ourPeerId := n.QuitC(), mset.monitorQuitC(), n.LeadChangeC(), n.ApplyQ(), mset.updateC(), meta.ID()\n\n\ts.Debugf(\"Starting stream monitor for '%s > %s' [%s]\", sa.Client.serviceAccount(), sa.Config.Name, n.Group())\n\tdefer s.Debugf(\"Exiting stream monitor for '%s > %s' [%s]\", sa.Client.serviceAccount(), sa.Config.Name, n.Group())\n\n\t// Make sure we do not leave the apply channel to fill up and block the raft layer.\n\tdefer func() {\n\t\tif n.State() == Closed {\n\t\t\treturn\n\t\t}\n\t\tn.StepDown()\n\t\t// Drain the commit queue...\n\t\taq.drain()\n\t}()\n\n\tconst (\n\t\tcompactInterval = 2 * time.Minute\n\t\tcompactMinInterval = 15 * time.Second\n\t\tcompactSizeMin = 8 * 1024 * 1024\n\t\tcompactNumMin = 65536\n\t)\n\n\t// Spread these out for large numbers on server restart.\n\trci := time.Duration(rand.Int63n(int64(time.Minute)))\n\tt := time.NewTicker(compactInterval + rci)\n\tdefer t.Stop()\n\n\tjs.mu.RLock()\n\tisLeader := cc.isStreamLeader(sa.Client.serviceAccount(), sa.Config.Name)\n\tisRestore := sa.Restore != nil\n\tjs.mu.RUnlock()\n\n\tacc, err := s.LookupAccount(sa.Client.serviceAccount())\n\tif err != nil {\n\t\ts.Warnf(\"Could not retrieve account for stream '%s > %s'\", sa.Client.serviceAccount(), sa.Config.Name)\n\t\treturn\n\t}\n\taccName := acc.GetName()\n\n\t// Don't allow the upper layer to install snapshots until we have\n\t// fully recovered from disk.\n\tisRecovering := true\n\n\tvar failedSnapshots int\n\tdoSnapshot := func(force bool) {\n\t\t// Suppress during recovery.\n\t\t// If snapshots have failed, and we're not forced to, we'll wait for the timer since it'll now be forced.\n\t\tif mset == nil || isRecovering || isRestore || (!force && failedSnapshots > 0) {\n\t\t\treturn\n\t\t}\n\n\t\t// Make sure all pending data is flushed before allowing snapshots.\n\t\tif err := mset.flushAllPending(); err != nil {\n\t\t\t// If the pending data couldn't be flushed, we have no safe way to continue.\n\t\t\ts.Errorf(\"Failed to flush pending data for '%s > %s' [%s]: %v\", accName, mset.name(), n.Group(), err)\n\t\t\tassert.Unreachable(\"Stream snapshot flush failed\", map[string]any{\n\t\t\t\t\"account\": accName,\n\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t\"group\": n.Group(),\n\t\t\t\t\"err\": err,\n\t\t\t})\n\t\t\tmset.setWriteErr(err)\n\t\t\tn.Stop()\n\t\t\treturn\n\t\t}\n\n\t\t// If we had a significant number of failed snapshots, start relaxing Raft-layer checks\n\t\t// to force it through. We might have been catching up a peer for a long period, and this\n\t\t// protects our log size from growing indefinitely.\n\t\tforceSnapshot := failedSnapshots > 4\n\t\tif err := n.InstallSnapshot(mset.stateSnapshot(), forceSnapshot); err == nil {\n\t\t\t// If there was a failed snapshot before, we reduced the timer's interval.\n\t\t\t// Reset it back to the original interval now.\n\t\t\tif failedSnapshots > 0 {\n\t\t\t\tt.Reset(compactInterval + rci)\n\t\t\t}\n\t\t\tfailedSnapshots = 0\n\t\t} else if err != errNoSnapAvailable && err != errNodeClosed && err != errCatchupsRunning {\n\t\t\ts.RateLimitWarnf(\"Failed to install snapshot for '%s > %s' [%s]: %v\", mset.acc.Name, mset.name(), n.Group(), err)\n\t\t\t// If this is the first failure, reduce the interval of the snapshot timer.\n\t\t\t// This ensures we're not waiting too long for snapshotting to eventually become forced.\n\t\t\tif failedSnapshots == 0 {\n\t\t\t\tt.Reset(compactMinInterval)\n\t\t\t}\n\t\t\tfailedSnapshots++\n\t\t}\n\t}\n\n\t// We will establish a restoreDoneCh no matter what. Will never be triggered unless\n\t// we replace with the restore chan.\n\trestoreDoneCh := make(<-chan error)\n\n\t// For migration tracking.\n\tvar mmt *time.Ticker\n\tvar mmtc <-chan time.Time\n\n\tstartMigrationMonitoring := func() {\n\t\tif mmt == nil {\n\t\t\tmmt = time.NewTicker(500 * time.Millisecond)\n\t\t\tmmtc = mmt.C\n\t\t}\n\t}\n\n\tstopMigrationMonitoring := func() {\n\t\tif mmt != nil {\n\t\t\tmmt.Stop()\n\t\t\tmmt, mmtc = nil, nil\n\t\t}\n\t}\n\tdefer stopMigrationMonitoring()\n\n\t// This is to optionally track when we are ready as a non-leader for direct access participation.\n\t// Either direct or if we are a direct mirror, or both.\n\tvar dat *time.Ticker\n\tvar datc <-chan time.Time\n\n\tstartDirectAccessMonitoring := func() {\n\t\tif dat == nil {\n\t\t\tdat = time.NewTicker(2 * time.Second)\n\t\t\tdatc = dat.C\n\t\t}\n\t}\n\n\tstopDirectMonitoring := func() {\n\t\tif dat != nil {\n\t\t\tdat.Stop()\n\t\t\tdat, datc = nil, nil\n\t\t}\n\t}\n\tdefer stopDirectMonitoring()\n\n\t// For checking interest state if applicable.\n\tvar cist *time.Ticker\n\tvar cistc <-chan time.Time\n\n\tcheckInterestInterval := checkInterestStateT + time.Duration(rand.Intn(checkInterestStateJ))*time.Second\n\n\tif mset != nil && mset.isInterestRetention() {\n\t\t// Wait on our consumers to be assigned and running before proceeding.\n\t\t// This can become important when a server has lots of assets\n\t\t// since we process streams first then consumers as an asset class.\n\t\tmset.waitOnConsumerAssignments()\n\t\t// Setup our periodic check here. We will check once we have restored right away.\n\t\tcist = time.NewTicker(checkInterestInterval)\n\t\tcistc = cist.C\n\t}\n\n\t// This is triggered during a scale up from R1 to clustered mode. We need the new followers to catchup,\n\t// similar to how we trigger the catchup mechanism post a backup/restore.\n\t// We can arrive here NOT being the leader, so we send the snapshot only if we are, and in this case\n\t// reset the notion that we need to send the snapshot. If we are not, then the first time the server\n\t// will switch to leader (in the loop below), we will send the snapshot.\n\tif sendSnapshot && isLeader && mset != nil && n != nil && !isRecovering {\n\t\tn.SendSnapshot(mset.stateSnapshot())\n\t\tsendSnapshot = false\n\t}\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\t// Server shutting down, but we might receive this before qch, so try to snapshot.\n\t\t\tdoSnapshot(false)\n\t\t\treturn\n\t\tcase <-mqch:\n\t\t\t// Clean signal from shutdown routine so do best effort attempt to snapshot.\n\t\t\t// Don't snapshot if not shutting down, monitor goroutine could be going away\n\t\t\t// on a scale down or a remove for example.\n\t\t\tif s.isShuttingDown() {\n\t\t\t\tdoSnapshot(false)\n\t\t\t}\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\t// Raft node is closed, no use in trying to snapshot.\n\t\t\treturn\n\t\tcase <-aq.ch:\n\t\t\tvar ne, nb uint64\n\t\t\t// If we bump clfs we will want to write out snapshot if within our time window.\n\t\t\tpclfs := mset.getCLFS()\n\n\t\t\tces := aq.pop()\n\t\t\tfor _, ce := range ces {\n\t\t\t\t// No special processing needed for when we are caught up on restart.\n\t\t\t\tif ce == nil {\n\t\t\t\t\tif !isRecovering {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tisRecovering = false\n\t\t\t\t\t// If we are interest based make sure to check consumers if interest retention policy.\n\t\t\t\t\t// This is to make sure we process any outstanding acks from all consumers.\n\t\t\t\t\tif mset != nil && mset.isInterestRetention() {\n\t\t\t\t\t\tfire := time.Duration(rand.Intn(5)+5) * time.Second\n\t\t\t\t\t\ttime.AfterFunc(fire, mset.checkInterestState)\n\t\t\t\t\t}\n\t\t\t\t\t// If we became leader during this time and we need to send a snapshot to our\n\t\t\t\t\t// followers, i.e. as a result of a scale-up from R1, do it now.\n\t\t\t\t\tif sendSnapshot && isLeader && mset != nil && n != nil {\n\t\t\t\t\t\tn.SendSnapshot(mset.stateSnapshot())\n\t\t\t\t\t\tsendSnapshot = false\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t} else if len(ce.Entries) == 0 {\n\t\t\t\t\t// If we have a partial batch, it needs to be rejected to ensure CLFS is correct.\n\t\t\t\t\tif mset != nil {\n\t\t\t\t\t\tmset.mu.RLock()\n\t\t\t\t\t\tbatch := mset.batchApply\n\t\t\t\t\t\tmset.mu.RUnlock()\n\t\t\t\t\t\tif batch != nil {\n\t\t\t\t\t\t\tbatch.rejectBatchState(mset)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Entry could be empty on a restore when mset is nil.\n\t\t\t\t\tne, nb = n.Applied(ce.Index)\n\t\t\t\t\tce.ReturnToPool()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Apply our entries.\n\t\t\t\tif maxApplied, err := js.applyStreamEntries(mset, ce, isRecovering); err == nil {\n\t\t\t\t\t// Update our applied.\n\t\t\t\t\tif maxApplied > 0 {\n\t\t\t\t\t\t// Indicate we've processed (but not applied) everything up to this point.\n\t\t\t\t\t\tne, nb = n.Processed(ce.Index, min(maxApplied, ce.Index))\n\t\t\t\t\t\t// Don't return entry to the pool, this is handled by the in-progress batch.\n\t\t\t\t\t} else {\n\t\t\t\t\t\tne, nb = n.Applied(ce.Index)\n\t\t\t\t\t\tce.ReturnToPool()\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// Make sure to clean up.\n\t\t\t\t\tce.ReturnToPool()\n\t\t\t\t\t// Our stream was closed out from underneath of us, simply return here.\n\t\t\t\t\tif err == errStreamClosed || err == errCatchupStreamStopped || err == ErrServerNotRunning {\n\t\t\t\t\t\taq.recycle(&ces)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\ts.Errorf(\"Error applying entries to '%s > %s': %v\", accName, sa.Config.Name, err)\n\t\t\t\t\tif isClusterResetErr(err) {\n\t\t\t\t\t\tif mset.isMirror() && mset.IsLeader() {\n\t\t\t\t\t\t\tmset.retryMirrorConsumer()\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// If the error signals we timed out of a snapshot, we should try to replay the snapshot\n\t\t\t\t\t\t// instead of fully resetting the state. Resetting the clustered state may result in\n\t\t\t\t\t\t// race conditions and should only be used as a last effort attempt.\n\t\t\t\t\t\tif errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries {\n\t\t\t\t\t\t\tif node := mset.raftNode(); node != nil && node.DrainAndReplaySnapshot() {\n\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// We will attempt to reset our cluster state.\n\t\t\t\t\t\tif mset.resetClusteredState(err) {\n\t\t\t\t\t\t\taq.recycle(&ces)\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if isOutOfSpaceErr(err) {\n\t\t\t\t\t\t// If applicable this will tear all of this down, but don't assume so and return.\n\t\t\t\t\t\ts.handleOutOfSpace(mset)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Encountered an unexpected error, can't continue.\n\t\t\t\t\t\tmset.setWriteErr(err)\n\t\t\t\t\t\taq.recycle(&ces)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\taq.recycle(&ces)\n\n\t\t\t// Check about snapshotting\n\t\t\t// If we have at least min entries to compact, go ahead and try to snapshot/compact.\n\t\t\tif ne >= compactNumMin || nb > compactSizeMin || mset.getCLFS() > pclfs {\n\t\t\t\tdoSnapshot(false)\n\t\t\t}\n\n\t\tcase isLeader = <-lch:\n\t\t\t// Process our leader change.\n\t\t\tjs.processStreamLeaderChange(mset, isLeader)\n\n\t\t\tif isLeader {\n\t\t\t\tif mset != nil && n != nil && sendSnapshot && !isRecovering {\n\t\t\t\t\t// If we *are* recovering at the time then this will get done when the apply queue\n\t\t\t\t\t// handles the nil guard to show the catchup ended.\n\t\t\t\t\tn.SendSnapshot(mset.stateSnapshot())\n\t\t\t\t\tsendSnapshot = false\n\t\t\t\t}\n\t\t\t\tif isRestore {\n\t\t\t\t\tacc, _ := s.LookupAccount(sa.Client.serviceAccount())\n\t\t\t\t\trestoreDoneCh = s.processStreamRestore(sa.Client, acc, sa.Config, _EMPTY_, sa.Reply, _EMPTY_)\n\t\t\t\t\tcontinue\n\t\t\t\t} else if n != nil && n.NeedSnapshot() {\n\t\t\t\t\tdoSnapshot(false)\n\t\t\t\t}\n\t\t\t\t// Always cancel if this was running.\n\t\t\t\tstopDirectMonitoring()\n\t\t\t} else if !n.Leaderless() {\n\t\t\t\tjs.setStreamAssignmentRecovering(sa)\n\t\t\t}\n\n\t\t\t// We may receive a leader change after the stream assignment which would cancel us\n\t\t\t// monitoring for this closely. So re-assess our state here as well.\n\t\t\t// Or the old leader is no longer part of the set and transferred leadership\n\t\t\t// for this leader to resume with removal\n\t\t\tmigrating := mset.isMigrating()\n\n\t\t\t// Check for migrations here. We set the state on the stream assignment update below.\n\t\t\tif isLeader && migrating {\n\t\t\t\tstartMigrationMonitoring()\n\t\t\t}\n\n\t\t\t// Here we are checking if we are not the leader but we have been asked to allow\n\t\t\t// direct access. We now allow non-leaders to participate in the queue group.\n\t\t\tif !isLeader && mset != nil {\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tad, md := mset.cfg.AllowDirect, mset.cfg.MirrorDirect\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\tif ad || md {\n\t\t\t\t\tstartDirectAccessMonitoring()\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase <-cistc:\n\t\t\tcist.Reset(checkInterestInterval)\n\t\t\t// We may be adjusting some things with consumers so do this in its own go routine.\n\t\t\tgo mset.checkInterestState()\n\n\t\tcase <-datc:\n\t\t\tif mset == nil || isRecovering {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// If we are leader we can stop, we know this is setup now.\n\t\t\tif isLeader {\n\t\t\t\tstopDirectMonitoring()\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tmset.mu.Lock()\n\t\t\tad, md, current := mset.cfg.AllowDirect, mset.cfg.MirrorDirect, mset.isCurrent()\n\t\t\tif !current {\n\t\t\t\tconst syncThreshold = 90.0\n\t\t\t\t// We are not current, but current means exactly caught up. Under heavy publish\n\t\t\t\t// loads we may never reach this, so check if we are within 90% caught up.\n\t\t\t\t_, c, a := mset.node.Progress()\n\t\t\t\tif c == 0 {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif p := float64(a) / float64(c) * 100.0; p < syncThreshold {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\ts.Debugf(\"Stream '%s > %s' enabling direct gets at %.0f%% synchronized\",\n\t\t\t\t\t\tsa.Client.serviceAccount(), sa.Config.Name, p)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We are current, cancel monitoring and create the direct subs as needed.\n\t\t\tif ad {\n\t\t\t\tmset.subscribeToDirect()\n\t\t\t}\n\t\t\tif md {\n\t\t\t\tmset.subscribeToMirrorDirect()\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t\t// Stop direct monitoring.\n\t\t\tstopDirectMonitoring()\n\n\t\tcase <-t.C:\n\t\t\t// Start forcing snapshots if they failed previously.\n\t\t\tforceIfFailed := failedSnapshots > 0\n\t\t\tdoSnapshot(forceIfFailed)\n\n\t\tcase <-uch:\n\t\t\t// keep stream assignment current\n\t\t\tsa = mset.streamAssignment()\n\n\t\t\t// We get this when we have a new stream assignment caused by an update.\n\t\t\t// We want to know if we are migrating.\n\t\t\tif migrating := mset.isMigrating(); migrating {\n\t\t\t\tif isLeader && mmtc == nil {\n\t\t\t\t\tstartMigrationMonitoring()\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t}\n\t\tcase <-mmtc:\n\t\t\tif !isLeader {\n\t\t\t\t// We are no longer leader, so not our job.\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Check to see where we are..\n\t\t\trg := mset.raftGroup()\n\n\t\t\t// Track the new peers and check the ones that are current.\n\t\t\tmset.mu.RLock()\n\t\t\treplicas := mset.cfg.Replicas\n\t\t\tmset.mu.RUnlock()\n\t\t\tif len(rg.Peers) <= replicas {\n\t\t\t\t// Migration no longer happening, so not our job anymore\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Make sure we have correct cluster information on the other peers.\n\t\t\tci := js.clusterInfo(rg)\n\t\t\tmset.checkClusterInfo(ci)\n\n\t\t\tnewPeers, _, newPeerSet, oldPeerSet := genPeerInfo(rg.Peers, len(rg.Peers)-replicas)\n\n\t\t\t// If we are part of the new peerset and we have been passed the baton.\n\t\t\t// We will handle scale down.\n\t\t\tif newPeerSet[ourPeerId] {\n\t\t\t\t// First need to check on any consumers and make sure they have moved properly before scaling down ourselves.\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tvar needToWait bool\n\t\t\t\tfor name, c := range sa.consumers {\n\t\t\t\t\tif c.unsupported != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tfor _, peer := range c.Group.Peers {\n\t\t\t\t\t\t// If we have peers still in the old set block.\n\t\t\t\t\t\tif oldPeerSet[peer] {\n\t\t\t\t\t\t\ts.Debugf(\"Scale down of '%s > %s' blocked by consumer '%s'\", accName, sa.Config.Name, name)\n\t\t\t\t\t\t\tneedToWait = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif needToWait {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\tif needToWait {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// We are good to go, can scale down here.\n\t\t\t\tn.ProposeKnownPeers(newPeers)\n\n\t\t\t\tcsa := sa.copyGroup()\n\t\t\t\tcsa.Group.Peers = newPeers\n\t\t\t\tcsa.Group.Preferred = ourPeerId\n\t\t\t\tcsa.Group.Cluster = s.cachedClusterName()\n\t\t\t\tcc.meta.ForwardProposal(encodeUpdateStreamAssignment(csa))\n\t\t\t\ts.Noticef(\"Scaling down '%s > %s' to %+v\", accName, sa.Config.Name, s.peerSetToNames(newPeers))\n\t\t\t} else {\n\t\t\t\t// We are the old leader here, from the original peer set.\n\t\t\t\t// We are simply waiting on the new peerset to be caught up so we can transfer leadership.\n\t\t\t\tvar newLeaderPeer, newLeader string\n\t\t\t\tneededCurrent, current := replicas/2+1, 0\n\n\t\t\t\tfor _, r := range ci.Replicas {\n\t\t\t\t\tif r.Current && newPeerSet[r.Peer] {\n\t\t\t\t\t\tcurrent++\n\t\t\t\t\t\tif newLeader == _EMPTY_ {\n\t\t\t\t\t\t\tnewLeaderPeer, newLeader = r.Peer, r.Name\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Check if we have a quorom.\n\t\t\t\tif current >= neededCurrent {\n\t\t\t\t\ts.Noticef(\"Transfer of stream leader for '%s > %s' to '%s'\", accName, sa.Config.Name, newLeader)\n\t\t\t\t\tn.ProposeKnownPeers(newPeers)\n\t\t\t\t\tn.StepDown(newLeaderPeer)\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase err := <-restoreDoneCh:\n\t\t\t// We have completed a restore from snapshot on this server. The stream assignment has\n\t\t\t// already been assigned but the replicas will need to catch up out of band. Consumers\n\t\t\t// will need to be assigned by forwarding the proposal and stamping the initial state.\n\t\t\ts.Debugf(\"Stream restore for '%s > %s' completed\", sa.Client.serviceAccount(), sa.Config.Name)\n\t\t\tif err != nil {\n\t\t\t\ts.Debugf(\"Stream restore failed: %v\", err)\n\t\t\t}\n\t\t\tisRestore = false\n\t\t\tsa.Restore = nil\n\t\t\t// If we were successful lookup up our stream now.\n\t\t\tif err == nil {\n\t\t\t\tif mset, err = acc.lookupStream(sa.Config.Name); mset != nil {\n\t\t\t\t\tmset.monitorWg.Add(1)\n\t\t\t\t\tdefer mset.monitorWg.Done()\n\t\t\t\t\tmset.checkInMonitor()\n\t\t\t\t\tmset.setStreamAssignment(sa)\n\t\t\t\t\t// Make sure to update our updateC which would have been nil.\n\t\t\t\t\tuch = mset.updateC()\n\t\t\t\t\t// Also update our mqch\n\t\t\t\t\tmqch = mset.monitorQuitC()\n\t\t\t\t\t// Setup a periodic check here if we are interest based as well.\n\t\t\t\t\tif mset.isInterestRetention() {\n\t\t\t\t\t\tcist = time.NewTicker(checkInterestInterval)\n\t\t\t\t\t\tcistc = cist.C\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tif mset != nil {\n\t\t\t\t\tmset.delete()\n\t\t\t\t}\n\t\t\t\tjs.mu.Lock()\n\t\t\t\tsa.err = err\n\t\t\t\tif n != nil {\n\t\t\t\t\tn.Delete()\n\t\t\t\t}\n\t\t\t\tresult := &streamAssignmentResult{\n\t\t\t\t\tAccount: sa.Client.serviceAccount(),\n\t\t\t\t\tStream: sa.Config.Name,\n\t\t\t\t\tRestore: &JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}},\n\t\t\t\t}\n\t\t\t\tresult.Restore.Error = NewJSStreamAssignmentError(err, Unless(err))\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\t// Send response to the metadata leader. They will forward to the user as needed.\n\t\t\t\ts.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif !isLeader {\n\t\t\t\tpanic(\"Finished restore but not leader\")\n\t\t\t}\n\t\t\t// Trigger the stream followers to catchup.\n\t\t\tif n = mset.raftNode(); n != nil {\n\t\t\t\tn.SendSnapshot(mset.stateSnapshot())\n\t\t\t}\n\t\t\tjs.processStreamLeaderChange(mset, isLeader)\n\n\t\t\t// Check to see if we have restored consumers here.\n\t\t\t// These are not currently assigned so we will need to do so here.\n\t\t\tif consumers := mset.getPublicConsumers(); len(consumers) > 0 {\n\t\t\t\tfor _, o := range consumers {\n\t\t\t\t\tname, cfg := o.String(), o.config()\n\t\t\t\t\trg := cc.createGroupForConsumer(&cfg, sa)\n\t\t\t\t\t// Pick a preferred leader.\n\t\t\t\t\trg.setPreferred(s)\n\n\t\t\t\t\t// Place our initial state here as well for assignment distribution.\n\t\t\t\t\tstate, _ := o.store.State()\n\t\t\t\t\tca := &consumerAssignment{\n\t\t\t\t\t\tGroup: rg,\n\t\t\t\t\t\tStream: sa.Config.Name,\n\t\t\t\t\t\tName: name,\n\t\t\t\t\t\tConfig: &cfg,\n\t\t\t\t\t\tClient: sa.Client,\n\t\t\t\t\t\tCreated: o.createdTime(),\n\t\t\t\t\t\tState: state,\n\t\t\t\t\t}\n\n\t\t\t\t\t// We make these compressed in case state is complex.\n\t\t\t\t\taddEntry := encodeAddConsumerAssignmentCompressed(ca)\n\t\t\t\t\tcc.meta.ForwardProposal(addEntry)\n\n\t\t\t\t\t// Check to make sure we see the assignment.\n\t\t\t\t\tgo func() {\n\t\t\t\t\t\tticker := time.NewTicker(time.Second)\n\t\t\t\t\t\tdefer ticker.Stop()\n\t\t\t\t\t\tfor range ticker.C {\n\t\t\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\t\t\tca, meta := js.consumerAssignment(ca.Client.serviceAccount(), sa.Config.Name, name), cc.meta\n\t\t\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\t\t\tif ca == nil {\n\t\t\t\t\t\t\t\ts.Warnf(\"Consumer assignment has not been assigned, retrying\")\n\t\t\t\t\t\t\t\tif meta != nil {\n\t\t\t\t\t\t\t\t\tmeta.ForwardProposal(addEntry)\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Determine if we are migrating\nfunc (mset *stream) isMigrating() bool {\n\tif mset == nil {\n\t\treturn false\n\t}\n\n\tmset.mu.RLock()\n\tjs, sa := mset.js, mset.sa\n\tmset.mu.RUnlock()\n\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\n\t// During migration we will always be R>1, even when we start R1.\n\t// So if we do not have a group or node we no we are not migrating.\n\tif sa == nil || sa.Group == nil || sa.Group.node == nil {\n\t\treturn false\n\t}\n\t// The sign of migration is if our group peer count != configured replica count.\n\tif sa.Config.Replicas == len(sa.Group.Peers) {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// resetClusteredState is called when a clustered stream had an error (e.g sequence mismatch, bad snapshot) and needs to be reset.\nfunc (mset *stream) resetClusteredState(err error) bool {\n\tmset.mu.RLock()\n\ts, js, jsa, sa, acc, node, name := mset.srv, mset.js, mset.jsa, mset.sa, mset.acc, mset.node, mset.nameLocked(false)\n\tstype, tierName, replicas := mset.cfg.Storage, mset.tier, mset.cfg.Replicas\n\tmset.mu.RUnlock()\n\n\t// The stream might already be deleted and not assigned to us anymore.\n\t// In any case, don't revive the stream if it's already closed.\n\tif mset.closed.Load() || (node != nil && node.IsDeleted()) {\n\t\ts.Warnf(\"Will not reset stream '%s > %s', stream is closed\", acc, mset.name())\n\t\t// Explicitly returning true here, we want the outside to break out of the monitoring loop as well.\n\t\treturn true\n\t}\n\n\tassert.Unreachable(\"Reset clustered state\", map[string]any{\n\t\t\"stream\": name,\n\t\t\"account\": acc.Name,\n\t\t\"err\": err,\n\t})\n\n\t// Stepdown regardless if we are the leader here.\n\tif node != nil {\n\t\tnode.StepDown()\n\t}\n\n\t// If we detect we are shutting down just return.\n\tif js != nil && js.isShuttingDown() {\n\t\ts.Debugf(\"Will not reset stream '%s > %s', JetStream shutting down\", acc, mset.name())\n\t\treturn false\n\t}\n\n\t// Server\n\tif js.limitsExceeded(stype) {\n\t\ts.Warnf(\"Will not reset stream '%s > %s', server resources exceeded\", acc, mset.name())\n\t\treturn false\n\t}\n\n\t// Account\n\tif exceeded, _ := jsa.limitsExceeded(stype, tierName, replicas); exceeded {\n\t\ts.Warnf(\"Stream '%s > %s' errored, account resources exceeded\", acc, mset.name())\n\t\treturn false\n\t}\n\n\tif node != nil {\n\t\tif errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries {\n\t\t\t// Don't delete all state, could've just been temporarily unable to reach the leader.\n\t\t\tnode.Stop()\n\t\t} else {\n\t\t\t// We delete our raft state. Will recreate.\n\t\t\tnode.Delete()\n\t\t}\n\t}\n\n\t// Preserve our current state and messages unless we have a first sequence mismatch.\n\tshouldDelete := err == errFirstSequenceMismatch\n\n\t// Need to do the rest in a separate Go routine.\n\tgo func() {\n\t\tmset.signalMonitorQuit()\n\t\tmset.monitorWg.Wait()\n\t\tmset.resetAndWaitOnConsumers()\n\t\t// Stop our stream.\n\t\tmset.stop(shouldDelete, false)\n\n\t\tif sa != nil {\n\t\t\tjs.mu.Lock()\n\t\t\tif js.shuttingDown {\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\ts.Warnf(\"Resetting stream cluster state for '%s > %s'\", sa.Client.serviceAccount(), sa.Config.Name)\n\t\t\t// Mark stream assignment as resetting, so we don't double-account reserved resources.\n\t\t\t// But only if we're not also releasing the resources as part of the delete.\n\t\t\tsa.resetting = !shouldDelete\n\t\t\t// Now wipe groups from assignments.\n\t\t\tsa.Group.node = nil\n\t\t\tvar consumers []*consumerAssignment\n\t\t\tif cc := js.cluster; cc != nil && cc.meta != nil {\n\t\t\t\tourID := cc.meta.ID()\n\t\t\t\tfor _, ca := range sa.consumers {\n\t\t\t\t\tif ca.unsupported != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif rg := ca.Group; rg != nil && rg.isMember(ourID) {\n\t\t\t\t\t\trg.node = nil // Erase group raft/node state.\n\t\t\t\t\t\tconsumers = append(consumers, ca)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tjs.mu.Unlock()\n\n\t\t\t// This will reset the stream and consumers.\n\t\t\t// Reset stream.\n\t\t\tjs.processClusterCreateStream(acc, sa)\n\t\t\t// Reset consumers.\n\t\t\tfor _, ca := range consumers {\n\t\t\t\tjs.processClusterCreateConsumer(nil, ca, nil, false)\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn true\n}\n\nfunc isControlHdr(hdr []byte) bool {\n\treturn bytes.HasPrefix(hdr, []byte(\"NATS/1.0 100 \"))\n}\n\n// Apply our stream entries.\n// Return maximum allowed applied value, if currently inside a batch, zero otherwise.\nfunc (js *jetStream) applyStreamEntries(mset *stream, ce *CommittedEntry, isRecovering bool) (uint64, error) {\n\tmset.mu.RLock()\n\tbatch := mset.batchApply\n\tmset.mu.RUnlock()\n\n\tfor i, e := range ce.Entries {\n\t\t// Ignore if lower-level catchup is started.\n\t\t// We don't need to optimize during this, all entries are handled as normal.\n\t\tif e.Type == EntryCatchup {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check if a batch is abandoned.\n\t\tif e.Type != EntryNormal && batch != nil && batch.id != _EMPTY_ {\n\t\t\tbatch.rejectBatchState(mset)\n\t\t}\n\n\t\tif e.Type == EntryNormal {\n\t\t\tbuf, op := e.Data, entryOp(e.Data[0])\n\t\t\tif op == batchMsgOp {\n\t\t\t\tbatchId, batchSeq, _, _, err := decodeBatchMsg(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\t\t// Initialize if unset.\n\t\t\t\tif batch == nil {\n\t\t\t\t\tbatch = &batchApply{}\n\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t\tmset.batchApply = batch\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t}\n\n\t\t\t\t// Need to grab the stream lock before the batch lock.\n\t\t\t\tif isRecovering {\n\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t}\n\t\t\t\tbatch.mu.Lock()\n\n\t\t\t\t// Previous batch (if any) was abandoned.\n\t\t\t\tif batch.id != _EMPTY_ && batchId != batch.id {\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t}\n\t\t\t\tif batchSeq == 1 {\n\t\t\t\t\t// If this is the first message in the batch, need to mark the start index.\n\t\t\t\t\t// We'll continue to check batch-completeness and try to find the commit.\n\t\t\t\t\t// At that point we'll commit the whole batch.\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t\tbatch.entryStart = i\n\t\t\t\t\tbatch.maxApplied = ce.Index - 1\n\t\t\t\t}\n\t\t\t\tbatch.id = batchId\n\n\t\t\t\t// While recovering, we could come up in the middle of a compacted batch that has already been applied.\n\t\t\t\t// This is possible if two batches are part of the same append entry, and the first batch was fully\n\t\t\t\t// applied but the second wasn't.\n\t\t\t\t// If we still see the first message of the batch, we don't skip any messages of the batch here.\n\t\t\t\tif isRecovering {\n\t\t\t\t\tif batchSeq > 1 && batch.count == 0 {\n\t\t\t\t\t\tif skip, err := mset.skipBatchIfRecovering(batch, buf); err != nil || skip {\n\t\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tpanic(err.Error())\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t}\n\n\t\t\t\tbatch.count++\n\t\t\t\t// If the sequence is not monotonically increasing/we identify gaps, the batch can't be accepted.\n\t\t\t\tif batchSeq != batch.count {\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tbatch.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t} else if op == batchCommitMsgOp {\n\t\t\t\tbatchId, batchSeq, _, _, err := decodeBatchMsg(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\n\t\t\t\t// Ensure the whole batch is fully isolated, and reads\n\t\t\t\t// can only happen after the full batch is committed.\n\t\t\t\tmset.mu.Lock()\n\n\t\t\t\t// Initialize if unset.\n\t\t\t\tif batch == nil {\n\t\t\t\t\tbatch = &batchApply{}\n\t\t\t\t\tmset.batchApply = batch\n\t\t\t\t}\n\t\t\t\tbatch.mu.Lock()\n\n\t\t\t\t// Previous batch (if any) was abandoned.\n\t\t\t\tif batch.id != _EMPTY_ && batchId != batch.id {\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t}\n\t\t\t\tif batchSeq == 1 {\n\t\t\t\t\t// If this is the first message in the batch, need to mark the start index.\n\t\t\t\t\t// This is a batch of size one that immediately commits.\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t\tbatch.entryStart = i\n\t\t\t\t\tbatch.maxApplied = ce.Index - 1\n\t\t\t\t}\n\t\t\t\tbatch.id = batchId\n\n\t\t\t\t// While recovering, we could come up in the middle of a compacted batch that has already been applied.\n\t\t\t\t// This is possible if two batches are part of the same append entry, and the first batch was fully\n\t\t\t\t// applied but the second wasn't.\n\t\t\t\t// If we still see the first message of the batch, we don't skip any messages of the batch here.\n\t\t\t\tif isRecovering && batchSeq > 1 && batch.count == 0 {\n\t\t\t\t\tif skip, err := mset.skipBatchIfRecovering(batch, buf); err != nil || skip {\n\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tpanic(err.Error())\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tbatch.count++\n\t\t\t\t// Detected a gap, reject the batch.\n\t\t\t\tif batchSeq != batch.count {\n\t\t\t\t\tbatch.rejectBatchStateLocked(mset)\n\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Process any entries that are part of this batch but prior to the current one.\n\t\t\t\tvar entries []*Entry\n\t\t\t\tfor j, bce := range batch.entries {\n\t\t\t\t\tif j == 0 {\n\t\t\t\t\t\t// The first needs only the entries when the batch is started.\n\t\t\t\t\t\tentries = bce.Entries[batch.entryStart:]\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Otherwise, all entries are used.\n\t\t\t\t\t\tentries = bce.Entries\n\t\t\t\t\t}\n\t\t\t\t\tfor _, entry := range entries {\n\t\t\t\t\t\t_, _, op, buf, err = decodeBatchMsg(entry.Data[1:])\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\t\tpanic(err.Error())\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif err = js.applyStreamMsgOp(mset, op, buf, isRecovering, false); err != nil {\n\t\t\t\t\t\t\t// Make sure to return remaining entries to the pool on an error.\n\t\t\t\t\t\t\tfor _, nce := range batch.entries[j:] {\n\t\t\t\t\t\t\t\tnce.ReturnToPool()\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t// Important to clear, otherwise we could return the entries to the pool multiple times.\n\t\t\t\t\t\t\tbatch.clearBatchStateLocked()\n\t\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\t\treturn 0, err\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Return the entry to the pool now.\n\t\t\t\t\tbce.ReturnToPool()\n\t\t\t\t}\n\t\t\t\tif len(batch.entries) == 0 {\n\t\t\t\t\t// Get within the same entry, but within the range of this batch.\n\t\t\t\t\tentries = ce.Entries[batch.entryStart : i+1]\n\t\t\t\t} else {\n\t\t\t\t\t// Get all entries up to and including the current one.\n\t\t\t\t\tentries = ce.Entries[:i+1]\n\t\t\t\t}\n\t\t\t\t// Process remaining entries in the current entry.\n\t\t\t\tfor _, entry := range entries {\n\t\t\t\t\t_, _, op, buf, err = decodeBatchMsg(entry.Data[1:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\tpanic(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\tif err = js.applyStreamMsgOp(mset, op, buf, isRecovering, false); err != nil {\n\t\t\t\t\t\t// Important to clear, otherwise we could return the entries to the pool multiple times.\n\t\t\t\t\t\tbatch.clearBatchStateLocked()\n\t\t\t\t\t\tbatch.mu.Unlock()\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\treturn 0, err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Clear state, batch was successful.\n\t\t\t\tbatch.clearBatchStateLocked()\n\t\t\t\tbatch.mu.Unlock()\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t} else if batch != nil && batch.id != _EMPTY_ {\n\t\t\t\t// If a batch is abandoned without a commit, reject it.\n\t\t\t\tbatch.rejectBatchState(mset)\n\t\t\t}\n\n\t\t\tswitch op {\n\t\t\tcase streamMsgOp, compressedStreamMsgOp:\n\t\t\t\tmbuf := buf[1:]\n\t\t\t\tif err := js.applyStreamMsgOp(mset, op, mbuf, isRecovering, true); err != nil {\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\n\t\t\tcase deleteMsgOp:\n\t\t\t\tmd, err := decodeMsgDelete(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tif node := mset.raftNode(); node != nil {\n\t\t\t\t\t\ts := js.srv\n\t\t\t\t\t\ts.Errorf(\"JetStream cluster could not decode delete msg for '%s > %s' [%s]\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), node.Group())\n\t\t\t\t\t}\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\t\ts := js.server()\n\n\t\t\t\tvar removed bool\n\t\t\t\tif md.NoErase {\n\t\t\t\t\tremoved, err = mset.removeMsg(md.Seq)\n\t\t\t\t} else {\n\t\t\t\t\tremoved, err = mset.eraseMsg(md.Seq)\n\t\t\t\t}\n\n\t\t\t\tvar isLeader bool\n\t\t\t\tif node := mset.raftNode(); node != nil && node.Leader() {\n\t\t\t\t\tisLeader = true\n\t\t\t\t}\n\n\t\t\t\tif err == ErrStoreEOF {\n\t\t\t\t\tif isLeader && !isRecovering {\n\t\t\t\t\t\tvar resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}}\n\t\t\t\t\t\tresp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err))\n\t\t\t\t\t\ts.sendAPIErrResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif err != nil && !isRecovering {\n\t\t\t\t\ts.Debugf(\"JetStream cluster failed to delete stream msg %d from '%s > %s': %v\",\n\t\t\t\t\t\tmd.Seq, md.Client.serviceAccount(), md.Stream, err)\n\t\t\t\t}\n\n\t\t\t\tif isLeader && !isRecovering {\n\t\t\t\t\tvar resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tresp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err))\n\t\t\t\t\t\ts.sendAPIErrResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t} else if !removed {\n\t\t\t\t\t\tresp.Error = NewJSSequenceNotFoundError(md.Seq)\n\t\t\t\t\t\ts.sendAPIErrResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tresp.Success = true\n\t\t\t\t\t\ts.sendAPIResponse(md.Client, mset.account(), md.Subject, md.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase purgeStreamOp:\n\t\t\t\tsp, err := decodeStreamPurge(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tif node := mset.raftNode(); node != nil {\n\t\t\t\t\t\ts := js.srv\n\t\t\t\t\t\ts.Errorf(\"JetStream cluster could not decode purge msg for '%s > %s' [%s]\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), node.Group())\n\t\t\t\t\t}\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\t\t// If no explicit request, fill in with leader stamped last sequence to protect ourselves on replay during server start.\n\t\t\t\tif sp.Request == nil || sp.Request.Sequence == 0 {\n\t\t\t\t\tpurgeSeq := sp.LastSeq + 1\n\t\t\t\t\tif sp.Request == nil {\n\t\t\t\t\t\tsp.Request = &JSApiStreamPurgeRequest{Sequence: purgeSeq}\n\t\t\t\t\t} else if sp.Request.Keep == 0 {\n\t\t\t\t\t\tsp.Request.Sequence = purgeSeq\n\t\t\t\t\t} else if isRecovering {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\ts := js.server()\n\t\t\t\tpurged, err := mset.purge(sp.Request)\n\t\t\t\tif err != nil {\n\t\t\t\t\ts.Warnf(\"JetStream cluster failed to purge stream %q for account %q: %v\", sp.Stream, sp.Client.serviceAccount(), err)\n\t\t\t\t}\n\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tisLeader := js.cluster.isStreamLeader(sp.Client.serviceAccount(), sp.Stream)\n\t\t\t\tjs.mu.RUnlock()\n\n\t\t\t\tif isLeader && !isRecovering {\n\t\t\t\t\tvar resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tresp.Error = NewJSStreamGeneralError(err, Unless(err))\n\t\t\t\t\t\ts.sendAPIErrResponse(sp.Client, mset.account(), sp.Subject, sp.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tresp.Purged = purged\n\t\t\t\t\t\tresp.Success = true\n\t\t\t\t\t\ts.sendAPIResponse(sp.Client, mset.account(), sp.Subject, sp.Reply, _EMPTY_, s.jsonResponse(resp))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tpanic(fmt.Sprintf(\"JetStream Cluster Unknown group entry op type: %v\", op))\n\t\t\t}\n\t\t} else if e.Type == EntrySnapshot {\n\t\t\t// Everything operates on new replicated state. Will convert legacy snapshots to this for processing.\n\t\t\tvar ss *StreamReplicatedState\n\n\t\t\tonBadState := func(err error) {\n\t\t\t\t// If we are the leader or recovering, meaning we own the snapshot,\n\t\t\t\t// we should stepdown and clear our raft state since our snapshot is bad.\n\t\t\t\tif isRecovering || mset.IsLeader() {\n\t\t\t\t\tmset.mu.RLock()\n\t\t\t\t\ts, accName, streamName := mset.srv, mset.acc.GetName(), mset.cfg.Name\n\t\t\t\t\tmset.mu.RUnlock()\n\t\t\t\t\ts.Warnf(\"Detected bad stream state, resetting '%s > %s'\", accName, streamName)\n\t\t\t\t\tmset.resetClusteredState(err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check if we are the new binary encoding.\n\t\t\tif IsEncodedStreamState(e.Data) {\n\t\t\t\tvar err error\n\t\t\t\tss, err = DecodeStreamState(e.Data)\n\t\t\t\tif err != nil {\n\t\t\t\t\tonBadState(err)\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tvar snap streamSnapshot\n\t\t\t\tif err := json.Unmarshal(e.Data, &snap); err != nil {\n\t\t\t\t\tonBadState(err)\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\t\t\t\t// Convert over to StreamReplicatedState\n\t\t\t\tss = &StreamReplicatedState{\n\t\t\t\t\tMsgs: snap.Msgs,\n\t\t\t\t\tBytes: snap.Bytes,\n\t\t\t\t\tFirstSeq: snap.FirstSeq,\n\t\t\t\t\tLastSeq: snap.LastSeq,\n\t\t\t\t\tFailed: snap.Failed,\n\t\t\t\t}\n\t\t\t\tif len(snap.Deleted) > 0 {\n\t\t\t\t\tss.Deleted = append(ss.Deleted, DeleteSlice(snap.Deleted))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif isRecovering || !mset.IsLeader() {\n\t\t\t\tif err := mset.processSnapshot(ss, ce.Index); err != nil && err != errAlreadyLeader {\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\t\t\t}\n\t\t} else if e.Type == EntryRemovePeer {\n\t\t\tjs.mu.RLock()\n\t\t\tvar ourID string\n\t\t\tif js.cluster != nil && js.cluster.meta != nil {\n\t\t\t\tourID = js.cluster.meta.ID()\n\t\t\t}\n\t\t\tjs.mu.RUnlock()\n\t\t\t// We only need to do processing if this is us.\n\t\t\tif peer := string(e.Data); peer == ourID && mset != nil {\n\t\t\t\t// Double check here with the registered stream assignment.\n\t\t\t\tshouldRemove := true\n\t\t\t\tif sa := mset.streamAssignment(); sa != nil && sa.Group != nil {\n\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\tshouldRemove = !sa.Group.isMember(ourID)\n\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t}\n\t\t\t\tif shouldRemove {\n\t\t\t\t\tmset.stop(true, false)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we're still actively processing a batch, must store the entry in-memory\n\t// to come back to it later once we find the commit.\n\tif batch != nil && batch.id != _EMPTY_ {\n\t\tbatch.mu.Lock()\n\t\tif batch.entries == nil {\n\t\t\tbatch.entries = []*CommittedEntry{ce}\n\t\t} else {\n\t\t\tbatch.entries = append(batch.entries, ce)\n\t\t}\n\t\tmaxApplied := batch.maxApplied\n\t\tbatch.mu.Unlock()\n\t\treturn maxApplied, nil\n\t}\n\treturn 0, nil\n}\n\n// skipBatchIfRecovering returns whether the batched message can be skipped because the batch was already fully applied.\n// Stream and batch.mu locks should be held.\nfunc (mset *stream) skipBatchIfRecovering(batch *batchApply, buf []byte) (bool, error) {\n\t_, _, op, mbuf, err := decodeBatchMsg(buf[1:])\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif op == compressedStreamMsgOp {\n\t\tif mbuf, err = s2.Decode(nil, mbuf); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t}\n\n\t_, _, _, _, lseq, _, _, err := decodeStreamMsg(mbuf)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\t// Grab last sequence and CLFS.\n\tlast, clfs := mset.lastSeqAndCLFS()\n\n\t// We can skip if we know this is less than what we already have.\n\tif lseq-clfs < last {\n\t\tmset.srv.Debugf(\"Apply stream entries for '%s > %s' skipping message with sequence %d with last of %d\",\n\t\t\tmset.accountLocked(false), mset.nameLocked(false), lseq+1-clfs, last)\n\t\t// Check for any preAcks in case we are interest based.\n\t\tmset.clearAllPreAcks(lseq + 1 - clfs)\n\t\tbatch.clearBatchStateLocked()\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc (js *jetStream) applyStreamMsgOp(mset *stream, op entryOp, mbuf []byte, isRecovering bool, needLock bool) error {\n\ts := js.srv\n\n\tif op == compressedStreamMsgOp {\n\t\tvar err error\n\t\tmbuf, err = s2.Decode(nil, mbuf)\n\t\tif err != nil {\n\t\t\tpanic(err.Error())\n\t\t}\n\t}\n\n\tsubject, reply, hdr, msg, lseq, ts, sourced, err := decodeStreamMsg(mbuf)\n\tif err != nil {\n\t\t// We're going to panic below, but if we're already holding the stream lock, we should let go now.\n\t\t// Otherwise we'll deadlock when trying to get the raft node.\n\t\tif !needLock {\n\t\t\tmset.mu.Unlock()\n\t\t}\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\ts.Errorf(\"JetStream cluster could not decode stream msg for '%s > %s' [%s]\",\n\t\t\t\tmset.account(), mset.name(), node.Group())\n\t\t}\n\t\tpanic(err.Error())\n\t}\n\n\t// Check for flowcontrol here.\n\tif len(msg) == 0 && len(hdr) > 0 && reply != _EMPTY_ && isControlHdr(hdr) {\n\t\tif !isRecovering {\n\t\t\tif needLock {\n\t\t\t\tmset.mu.RLock()\n\t\t\t}\n\t\t\tmset.sendFlowControlReply(reply)\n\t\t\tif needLock {\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tif needLock {\n\t\tmset.mu.RLock()\n\t}\n\t// Grab last sequence and CLFS.\n\tlast, clfs := mset.lastSeqAndCLFS()\n\tif needLock {\n\t\tmset.mu.RUnlock()\n\t}\n\n\t// We can skip if we know this is less than what we already have.\n\tif lseq-clfs < last {\n\t\ts.Debugf(\"Apply stream entries for '%s > %s' skipping message with sequence %d with last of %d\",\n\t\t\tmset.accountLocked(needLock), mset.nameLocked(needLock), lseq+1-clfs, last)\n\t\tif needLock {\n\t\t\tmset.mu.Lock()\n\t\t}\n\t\t// Check for any preAcks in case we are interest based.\n\t\tmset.clearAllPreAcks(lseq + 1 - clfs)\n\t\tif needLock {\n\t\t\tmset.mu.Unlock()\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Skip by hand here since first msg special case.\n\t// Reason is sequence is unsigned and for lseq being 0\n\t// the lseq under stream would have to be -1.\n\tif lseq == 0 && last != 0 {\n\t\treturn nil\n\t}\n\n\t// Messages to be skipped have no subject or timestamp or msg or hdr.\n\tif subject == _EMPTY_ && ts == 0 && len(msg) == 0 && len(hdr) == 0 {\n\t\t// Skip and update our lseq.\n\t\tlast, err := mset.store.SkipMsg(0)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif needLock {\n\t\t\tmset.mu.Lock()\n\t\t}\n\t\tmset.setLastSeq(last)\n\t\tmset.clearAllPreAcks(last)\n\t\tif needLock {\n\t\t\tmset.mu.Unlock()\n\t\t}\n\t\treturn nil\n\t}\n\n\tvar mt *msgTrace\n\t// If not recovering, see if we find a message trace object for this\n\t// sequence. Only the leader that has proposed this entry will have\n\t// stored the trace info.\n\tif !isRecovering {\n\t\tmt = mset.getAndDeleteMsgTrace(lseq)\n\t}\n\t// Process the actual message here.\n\terr = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts, mt, sourced, needLock)\n\n\t// If we have inflight make sure to clear after processing.\n\t// TODO(dlc) - technically check on inflight != nil could cause datarace.\n\t// But do not want to acquire lock since tracking this will be rare.\n\tif mset.inflight != nil {\n\t\tmset.clMu.Lock()\n\t\tif i, found := mset.inflight[subject]; found {\n\t\t\t// Decrement from pending operations. Once it reaches zero, it can be deleted.\n\t\t\tif i.ops > 0 {\n\t\t\t\tvar sz uint64\n\t\t\t\tif mset.store.Type() == FileStorage {\n\t\t\t\t\tsz = fileStoreMsgSizeRaw(len(subject), len(hdr), len(msg))\n\t\t\t\t} else {\n\t\t\t\t\tsz = memStoreMsgSizeRaw(len(subject), len(hdr), len(msg))\n\t\t\t\t}\n\t\t\t\tif i.bytes >= sz {\n\t\t\t\t\ti.bytes -= sz\n\t\t\t\t} else {\n\t\t\t\t\ti.bytes = 0\n\t\t\t\t}\n\t\t\t\ti.ops--\n\t\t\t}\n\t\t\tif i.ops == 0 {\n\t\t\t\tdelete(mset.inflight, subject)\n\t\t\t}\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\n\t// Update running total for counter.\n\tif mset.clusteredCounterTotal != nil {\n\t\tmset.clMu.Lock()\n\t\tif counter, found := mset.clusteredCounterTotal[subject]; found {\n\t\t\t// Decrement from pending operations. Once it reaches zero, it can be deleted.\n\t\t\tif counter.ops > 0 {\n\t\t\t\tcounter.ops--\n\t\t\t}\n\t\t\tif counter.ops == 0 {\n\t\t\t\tdelete(mset.clusteredCounterTotal, subject)\n\t\t\t}\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\n\t// Clear expected per subject state after processing.\n\tif mset.expectedPerSubjectSequence != nil {\n\t\tmset.clMu.Lock()\n\t\tif subj, found := mset.expectedPerSubjectSequence[lseq]; found {\n\t\t\tdelete(mset.expectedPerSubjectSequence, lseq)\n\t\t\tdelete(mset.expectedPerSubjectInProcess, subj)\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\n\tif err != nil {\n\t\tif err == errLastSeqMismatch {\n\n\t\t\tvar state StreamState\n\t\t\tmset.store.FastState(&state)\n\n\t\t\t// If we have no msgs and the other side is delivering us a sequence past where we\n\t\t\t// should be reset. This is possible if the other side has a stale snapshot and no longer\n\t\t\t// has those messages. So compact and retry to reset.\n\t\t\tif state.Msgs == 0 {\n\t\t\t\t_, err = mset.store.Compact(lseq + 1)\n\t\t\t\tif err == nil {\n\t\t\t\t\t// Retry\n\t\t\t\t\terr = mset.processJetStreamMsg(subject, reply, hdr, msg, lseq, ts, mt, sourced, needLock)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// FIXME(dlc) - We could just run a catchup with a request defining the span between what we expected\n\t\t\t// and what we got.\n\t\t}\n\n\t\t// Only return in place if we are going to reset our stream or we are out of space, or we are closed.\n\t\tif isClusterResetErr(err) || isOutOfSpaceErr(err) || err == errStreamClosed {\n\t\t\treturn err\n\t\t}\n\t\ts.Debugf(\"Apply stream entries for '%s > %s' got error processing message: %v\",\n\t\t\tmset.accountLocked(needLock), mset.nameLocked(needLock), err)\n\n\t\t// There are some errors that we can't recover from.\n\t\tif err != ErrMaxMsgs && err != ErrMaxBytes && err != ErrMaxMsgsPerSubject && err != ErrMsgTooLarge && err != ErrStoreClosed {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Returns the PeerInfo for all replicas of a raft node. This is different than node.Peers()\n// and is used for external facing advisories.\nfunc (s *Server) replicas(node RaftNode) []*PeerInfo {\n\tvar replicas []*PeerInfo\n\tfor _, rp := range node.Peers() {\n\t\tif sir, ok := s.nodeToInfo.Load(rp.ID); ok && sir != nil {\n\t\t\tsi := sir.(nodeInfo)\n\t\t\tpi := &PeerInfo{Peer: rp.ID, Name: si.name, Current: rp.Current, Offline: si.offline, Lag: rp.Lag}\n\t\t\tif !rp.Last.IsZero() {\n\t\t\t\tpi.Active = time.Since(rp.Last)\n\t\t\t}\n\t\t\treplicas = append(replicas, pi)\n\t\t}\n\t}\n\treturn replicas\n}\n\n// Process a leader change for the clustered stream.\nfunc (js *jetStream) processStreamLeaderChange(mset *stream, isLeader bool) {\n\tif mset == nil {\n\t\treturn\n\t}\n\tsa := mset.streamAssignment()\n\tif sa == nil {\n\t\treturn\n\t}\n\n\t// Clear inflight dedupe IDs, where seq=0.\n\tmset.ddMu.Lock()\n\tvar removed int\n\tfor i := len(mset.ddarr) - 1; i >= mset.ddindex; i-- {\n\t\tdde := mset.ddarr[i]\n\t\tif dde.seq != 0 {\n\t\t\tbreak\n\t\t}\n\t\tremoved++\n\t\tdelete(mset.ddmap, dde.id)\n\t}\n\tif removed > 0 {\n\t\tif len(mset.ddmap) > 0 {\n\t\t\tmset.ddarr = mset.ddarr[:len(mset.ddarr)-removed]\n\t\t} else {\n\t\t\tmset.ddmap = nil\n\t\t\tmset.ddarr = nil\n\t\t\tmset.ddindex = 0\n\t\t}\n\t}\n\tmset.ddMu.Unlock()\n\n\tmset.clMu.Lock()\n\t// Clear inflight if we have it.\n\tmset.inflight = nil\n\t// Clear running counter totals.\n\tmset.clusteredCounterTotal = nil\n\t// Clear expected per subject state.\n\tmset.expectedPerSubjectSequence = nil\n\tmset.expectedPerSubjectInProcess = nil\n\tmset.clMu.Unlock()\n\n\tjs.mu.Lock()\n\ts, account, err := js.srv, sa.Client.serviceAccount(), sa.err\n\tclient, subject, reply := sa.Client, sa.Subject, sa.Reply\n\thasResponded := sa.responded\n\tsa.responded = true\n\tjs.mu.Unlock()\n\n\tstreamName := mset.name()\n\n\tif isLeader {\n\t\ts.Noticef(\"JetStream cluster new stream leader for '%s > %s'\", account, streamName)\n\t\ts.sendStreamLeaderElectAdvisory(mset)\n\t} else {\n\t\t// We are stepping down.\n\t\t// Make sure if we are doing so because we have lost quorum that we send the appropriate advisories.\n\t\tif node := mset.raftNode(); node != nil && !node.Quorum() && time.Since(node.Created()) > 5*time.Second {\n\t\t\ts.sendStreamLostQuorumAdvisory(mset)\n\t\t}\n\n\t\t// Clear clseq. If we become leader again, it will be fixed up\n\t\t// automatically on the next mset.setLeader call.\n\t\tmset.clMu.Lock()\n\t\tif mset.clseq > 0 {\n\t\t\tmset.clseq = 0\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\n\t// Tell stream to switch leader status.\n\tmset.setLeader(isLeader)\n\n\tif !isLeader || hasResponded {\n\t\treturn\n\t}\n\n\tacc, _ := s.LookupAccount(account)\n\tif acc == nil {\n\t\treturn\n\t}\n\n\t// Send our response.\n\tvar resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}\n\tif err != nil {\n\t\tresp.Error = NewJSStreamCreateError(err, Unless(err))\n\t\ts.sendAPIErrResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t} else {\n\t\tmsetCfg := mset.config()\n\t\tresp.StreamInfo = &StreamInfo{\n\t\t\tCreated: mset.createdTime(),\n\t\t\tState: mset.state(),\n\t\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\t\tCluster: js.clusterInfo(mset.raftGroup()),\n\t\t\tSources: mset.sourcesInfo(),\n\t\t\tMirror: mset.mirrorInfo(),\n\t\t\tTimeStamp: time.Now().UTC(),\n\t\t}\n\t\tresp.DidCreate = true\n\t\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\tmset.sendCreateAdvisory()\n\t\t}\n\t}\n}\n\n// Fixed value ok for now.\nconst lostQuorumAdvInterval = 10 * time.Second\n\n// Determines if we should send lost quorum advisory. We throttle these after first one.\nfunc (mset *stream) shouldSendLostQuorum() bool {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tif time.Since(mset.lqsent) >= lostQuorumAdvInterval {\n\t\tmset.lqsent = time.Now()\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (s *Server) sendStreamLostQuorumAdvisory(mset *stream) {\n\tif mset == nil {\n\t\treturn\n\t}\n\tnode, stream, acc := mset.raftNode(), mset.name(), mset.account()\n\tif node == nil {\n\t\treturn\n\t}\n\tif !mset.shouldSendLostQuorum() {\n\t\treturn\n\t}\n\n\ts.Warnf(\"JetStream cluster stream '%s > %s' has NO quorum, stalled\", acc.GetName(), stream)\n\n\tsubj := JSAdvisoryStreamQuorumLostPre + \".\" + stream\n\tadv := &JSStreamQuorumLostAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamQuorumLostAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: stream,\n\t\tReplicas: s.replicas(node),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t}\n\n\t// Send to the user's account if not the system account.\n\tif acc != s.SystemAccount() {\n\t\ts.publishAdvisory(acc, subj, adv)\n\t}\n\t// Now do system level one. Place account info in adv, and nil account means system.\n\tadv.Account = acc.GetName()\n\ts.publishAdvisory(nil, subj, adv)\n}\n\nfunc (s *Server) sendStreamLeaderElectAdvisory(mset *stream) {\n\tif mset == nil {\n\t\treturn\n\t}\n\tnode, stream, acc := mset.raftNode(), mset.name(), mset.account()\n\tif node == nil {\n\t\treturn\n\t}\n\tsubj := JSAdvisoryStreamLeaderElectedPre + \".\" + stream\n\tadv := &JSStreamLeaderElectedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamLeaderElectedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: stream,\n\t\tLeader: s.serverNameForNode(node.GroupLeader()),\n\t\tReplicas: s.replicas(node),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t}\n\n\t// Send to the user's account if not the system account.\n\tif acc != s.SystemAccount() {\n\t\ts.publishAdvisory(acc, subj, adv)\n\t}\n\t// Now do system level one. Place account info in adv, and nil account means system.\n\tadv.Account = acc.GetName()\n\ts.publishAdvisory(nil, subj, adv)\n}\n\n// Will look up a stream assignment.\n// Lock should be held.\nfunc (js *jetStream) streamAssignment(account, stream string) (sa *streamAssignment) {\n\tcc := js.cluster\n\tif cc == nil {\n\t\treturn nil\n\t}\n\n\tif as := cc.streams[account]; as != nil {\n\t\tsa = as[stream]\n\t}\n\treturn sa\n}\n\n// Will look up a stream assignment, either an applied or inflight assignment.\n// Lock should be held.\nfunc (js *jetStream) streamAssignmentOrInflight(account, stream string) *streamAssignment {\n\tcc := js.cluster\n\tif cc == nil {\n\t\treturn nil\n\t}\n\tif streams, ok := cc.inflightStreams[account]; ok {\n\t\tif inflight, ok := streams[stream]; ok {\n\t\t\tif !inflight.deleted {\n\t\t\t\treturn inflight.streamAssignment\n\t\t\t} else {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\n\tif as := cc.streams[account]; as != nil {\n\t\treturn as[stream]\n\t}\n\treturn nil\n}\n\n// Will gather all stream assignments for a specific account, both applied and inflight assignments.\n// Lock should be held.\nfunc (js *jetStream) streamAssignmentsOrInflightSeq(account string) iter.Seq[*streamAssignment] {\n\treturn func(yield func(*streamAssignment) bool) {\n\t\tcc := js.cluster\n\t\tif cc == nil {\n\t\t\treturn\n\t\t}\n\t\tinflight := cc.inflightStreams[account]\n\t\tfor _, i := range inflight {\n\t\t\tif !i.deleted && !yield(i.streamAssignment) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tfor _, sa := range cc.streams[account] {\n\t\t\t// Skip if we already iterated over it as inflight.\n\t\t\tif _, ok := inflight[sa.Config.Name]; ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !yield(sa) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Will gather all stream assignments for all accounts, both applied and inflight assignments.\n// Lock should be held.\nfunc (js *jetStream) streamAssignmentsOrInflightSeqAllAccounts() iter.Seq2[string, *streamAssignment] {\n\treturn func(yield func(string, *streamAssignment) bool) {\n\t\tcc := js.cluster\n\t\tif cc == nil {\n\t\t\treturn\n\t\t}\n\t\tfor accName, inflight := range cc.inflightStreams {\n\t\t\tfor _, i := range inflight {\n\t\t\t\tif !i.deleted && !yield(accName, i.streamAssignment) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor accName, asa := range cc.streams {\n\t\t\tfor _, sa := range asa {\n\t\t\t\t// Skip if we already iterated over it as inflight.\n\t\t\t\tif inflight, ok := cc.inflightStreams[accName]; ok {\n\t\t\t\t\tif _, ok := inflight[sa.Config.Name]; ok {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !yield(accName, sa) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// processStreamAssignment is called when followers have replicated an assignment.\nfunc (js *jetStream) processStreamAssignment(sa *streamAssignment) {\n\tjs.mu.Lock()\n\ts, cc := js.srv, js.cluster\n\taccName, stream := sa.Client.serviceAccount(), sa.Config.Name\n\tnoMeta := cc == nil || cc.meta == nil\n\tvar ourID string\n\tif !noMeta {\n\t\tourID = cc.meta.ID()\n\t}\n\tvar isMember bool\n\tif sa.Group != nil && ourID != _EMPTY_ {\n\t\tisMember = sa.Group.isMember(ourID)\n\t}\n\n\tif s == nil || noMeta {\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\t// Remove this stream from the inflight proposals\n\tcc.removeInflightStreamProposal(accName, sa.Config.Name)\n\n\taccStreams := cc.streams[accName]\n\tif accStreams == nil {\n\t\taccStreams = make(map[string]*streamAssignment)\n\t} else if osa := accStreams[stream]; osa != nil {\n\t\tif osa != sa {\n\t\t\t// Copy over private existing state from former SA.\n\t\t\tif sa.Group != nil {\n\t\t\t\tsa.Group.node = osa.Group.node\n\t\t\t}\n\t\t\tsa.consumers = osa.consumers\n\t\t\tsa.responded = osa.responded\n\t\t\tsa.err = osa.err\n\t\t}\n\t\t// Unsubscribe if it was previously unsupported.\n\t\tif osa.unsupported != nil {\n\t\t\tosa.unsupported.closeInfoSub(js.srv)\n\t\t\t// If we've seen unsupported once, it remains for the lifetime of this server process.\n\t\t\tif sa.unsupported == nil {\n\t\t\t\tsa.unsupported = osa.unsupported\n\t\t\t}\n\t\t}\n\t}\n\n\t// Update our state.\n\taccStreams[stream] = sa\n\tcc.streams[accName] = accStreams\n\thasResponded := sa.responded\n\n\t// If unsupported, we can't register any further.\n\tif sa.unsupported != nil {\n\t\tsa.unsupported.setupInfoSub(s, sa)\n\t\ts.Warnf(\"Detected unsupported stream '%s > %s': %s\", accName, stream, sa.unsupported.reason)\n\t\tjs.mu.Unlock()\n\n\t\t// Need to stop the stream, we can't keep running with an old config.\n\t\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tmset, err := acc.lookupStream(stream)\n\t\tif err != nil || mset.closed.Load() {\n\t\t\treturn\n\t\t}\n\t\ts.Warnf(\"Stopping unsupported stream '%s > %s'\", accName, stream)\n\t\tmset.stop(false, false)\n\t\treturn\n\t}\n\tjs.mu.Unlock()\n\n\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\tif err != nil {\n\t\tll := fmt.Sprintf(\"Account [%s] lookup for stream create failed: %v\", accName, err)\n\t\tif isMember {\n\t\t\tif !hasResponded {\n\t\t\t\t// If we can not lookup the account and we are a member, send this result back to the metacontroller leader.\n\t\t\t\tresult := &streamAssignmentResult{\n\t\t\t\t\tAccount: accName,\n\t\t\t\t\tStream: stream,\n\t\t\t\t\tResponse: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}},\n\t\t\t\t}\n\t\t\t\tresult.Response.Error = NewJSNoAccountError()\n\t\t\t\ts.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result)\n\t\t\t}\n\t\t\ts.Warnf(ll)\n\t\t} else {\n\t\t\ts.Debugf(ll)\n\t\t}\n\t\treturn\n\t}\n\n\t// Check if this is for us..\n\tif isMember {\n\t\tjs.processClusterCreateStream(acc, sa)\n\t} else if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil {\n\t\t// We have one here even though we are not a member. This can happen on re-assignment.\n\t\ts.removeStream(mset, sa)\n\t}\n\n\t// If this stream assignment does not have a sync subject (bug) set that the meta-leader should check when elected.\n\tif sa.Sync == _EMPTY_ {\n\t\tjs.mu.Lock()\n\t\tcc.streamsCheck = true\n\t\tjs.mu.Unlock()\n\t}\n}\n\n// processUpdateStreamAssignment is called when followers have replicated an updated assignment.\nfunc (js *jetStream) processUpdateStreamAssignment(sa *streamAssignment) {\n\tjs.mu.RLock()\n\ts, cc := js.srv, js.cluster\n\tjs.mu.RUnlock()\n\tif s == nil || cc == nil {\n\t\t// TODO(dlc) - debug at least\n\t\treturn\n\t}\n\n\taccName := sa.Client.serviceAccount()\n\tstream := sa.Config.Name\n\n\tjs.mu.Lock()\n\tif cc.meta == nil {\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\t// Remove this stream from the inflight proposals\n\tcc.removeInflightStreamProposal(accName, sa.Config.Name)\n\n\tourID := cc.meta.ID()\n\n\tvar isMember bool\n\tif sa.Group != nil {\n\t\tisMember = sa.Group.isMember(ourID)\n\t}\n\n\taccStreams := cc.streams[accName]\n\tif accStreams == nil {\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\tosa := accStreams[stream]\n\tif osa == nil {\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\t// Copy over private existing state from former SA.\n\tif sa.Group != nil {\n\t\tsa.Group.node = osa.Group.node\n\t}\n\tsa.consumers = osa.consumers\n\tsa.err = osa.err\n\n\t// If we detect we are scaling down to 1, non-clustered, and we had a previous node, clear it here.\n\tif sa.Config.Replicas == 1 && sa.Group.node != nil {\n\t\tsa.Group.node = nil\n\t}\n\n\t// Update our state.\n\taccStreams[stream] = sa\n\tcc.streams[accName] = accStreams\n\n\t// Make sure we respond if we are a member.\n\tif isMember {\n\t\tsa.responded = false\n\t} else {\n\t\t// Make sure to clean up any old node in case this stream moves back here.\n\t\tif sa.Group != nil {\n\t\t\tsa.Group.node = nil\n\t\t}\n\t}\n\n\t// Unsubscribe if it was previously unsupported.\n\tif osa.unsupported != nil {\n\t\tosa.unsupported.closeInfoSub(js.srv)\n\t\t// If we've seen unsupported once, it remains for the lifetime of this server process.\n\t\tif sa.unsupported == nil {\n\t\t\tsa.unsupported = osa.unsupported\n\t\t}\n\t}\n\n\t// If unsupported, we can't register any further.\n\tif sa.unsupported != nil {\n\t\tsa.unsupported.setupInfoSub(s, sa)\n\t\ts.Warnf(\"Detected unsupported stream '%s > %s': %s\", accName, stream, sa.unsupported.reason)\n\t\tjs.mu.Unlock()\n\n\t\t// Need to stop the stream, we can't keep running with an old config.\n\t\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tmset, err := acc.lookupStream(stream)\n\t\tif err != nil || mset.closed.Load() {\n\t\t\treturn\n\t\t}\n\t\ts.Warnf(\"Stopping unsupported stream '%s > %s'\", accName, stream)\n\t\tmset.stop(false, false)\n\t\treturn\n\t}\n\tjs.mu.Unlock()\n\n\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\tif err != nil {\n\t\tll := fmt.Sprintf(\"Update Stream Account %s, error on lookup: %v\", accName, err)\n\t\tif isMember {\n\t\t\ts.Warnf(ll)\n\t\t} else {\n\t\t\ts.Debugf(ll)\n\t\t}\n\t\treturn\n\t}\n\n\t// Check if this is for us..\n\tif isMember {\n\t\tjs.processClusterUpdateStream(acc, osa, sa)\n\t} else if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil {\n\t\t// We have one here even though we are not a member. This can happen on re-assignment.\n\t\ts.removeStream(mset, sa)\n\t}\n}\n\n// Common function to remove ourselves from this server.\n// This can happen on re-assignment, move, etc\nfunc (s *Server) removeStream(mset *stream, nsa *streamAssignment) {\n\tif mset == nil {\n\t\treturn\n\t}\n\t// Make sure to use the new stream assignment, not our own.\n\ts.Debugf(\"JetStream removing stream '%s > %s' from this server\", nsa.Client.serviceAccount(), nsa.Config.Name)\n\tif node := mset.raftNode(); node != nil {\n\t\tnode.StepDown(nsa.Group.Preferred)\n\t\t// shutdown monitor by shutting down raft.\n\t\tnode.Delete()\n\t}\n\n\tvar isShuttingDown bool\n\t// Make sure this node is no longer attached to our stream assignment.\n\tif js, _ := s.getJetStreamCluster(); js != nil {\n\t\tjs.mu.Lock()\n\t\tnsa.Group.node = nil\n\t\tisShuttingDown = js.shuttingDown\n\t\tjs.mu.Unlock()\n\t}\n\n\tif !isShuttingDown {\n\t\t// wait for monitor to be shutdown.\n\t\tmset.signalMonitorQuit()\n\t\tmset.monitorWg.Wait()\n\t}\n\tmset.stop(true, false)\n}\n\n// processClusterUpdateStream is called when we have a stream assignment that\n// has been updated for an existing assignment and we are a member.\nfunc (js *jetStream) processClusterUpdateStream(acc *Account, osa, sa *streamAssignment) {\n\tif sa == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\ts, rg := js.srv, sa.Group\n\tclient, subject, reply := sa.Client, sa.Subject, sa.Reply\n\talreadyRunning, numReplicas := osa.Group.node != nil, len(rg.Peers)\n\tneedsNode := rg.node == nil\n\tstorage, cfg := sa.Config.Storage, sa.Config\n\thasResponded := sa.responded\n\tsa.responded = true\n\trecovering := sa.recovering\n\tjs.mu.Unlock()\n\n\tmset, err := acc.lookupStream(cfg.Name)\n\tif err == nil && mset != nil {\n\t\t// Make sure we have not had a new group assigned to us.\n\t\tif osa.Group.Name != sa.Group.Name {\n\t\t\ts.Warnf(\"JetStream cluster detected stream remapping for '%s > %s' from %q to %q\",\n\t\t\t\tacc, cfg.Name, osa.Group.Name, sa.Group.Name)\n\t\t\tmset.removeNode()\n\t\t\tmset.signalMonitorQuit()\n\t\t\tmset.monitorWg.Wait()\n\t\t\talreadyRunning, needsNode = false, true\n\t\t\t// Make sure to clear from original.\n\t\t\tjs.mu.Lock()\n\t\t\tosa.Group.node = nil\n\t\t\tjs.mu.Unlock()\n\t\t}\n\n\t\tif !alreadyRunning && numReplicas > 1 {\n\t\t\tif needsNode {\n\t\t\t\t// Since we are scaling up we want to make sure our sync subject\n\t\t\t\t// is registered before we start our raft node.\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.startClusterSubs()\n\t\t\t\tmset.mu.Unlock()\n\n\t\t\t\tjs.createRaftGroup(acc.GetName(), rg, recovering, storage, pprofLabels{\n\t\t\t\t\t\"type\": \"stream\",\n\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t})\n\t\t\t}\n\t\t\tmset.monitorWg.Add(1)\n\t\t\t// Start monitoring..\n\t\t\tstarted := s.startGoRoutine(\n\t\t\t\tfunc() { js.monitorStream(mset, sa, needsNode) },\n\t\t\t\tpprofLabels{\n\t\t\t\t\t\"type\": \"stream\",\n\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t},\n\t\t\t)\n\t\t\tif !started {\n\t\t\t\tmset.monitorWg.Done()\n\t\t\t}\n\t\t} else if numReplicas == 1 && alreadyRunning {\n\t\t\t// We downgraded to R1. Make sure we cleanup the raft node and the stream monitor.\n\t\t\tmset.removeNode()\n\t\t\tmset.signalMonitorQuit()\n\t\t\tmset.monitorWg.Wait()\n\t\t\t// In case we need to shutdown the cluster specific subs, etc.\n\t\t\tmset.mu.Lock()\n\t\t\t// Stop responding to sync requests.\n\t\t\tmset.stopClusterSubs()\n\t\t\t// Clear catchup state\n\t\t\tmset.clearAllCatchupPeers()\n\t\t\tmset.mu.Unlock()\n\t\t\t// Remove from meta layer.\n\t\t\tjs.mu.Lock()\n\t\t\trg.node = nil\n\t\t\tjs.mu.Unlock()\n\t\t}\n\t\t// Set the new stream assignment.\n\t\tmset.setStreamAssignment(sa)\n\n\t\t// Call update.\n\t\tif err = mset.updateWithAdvisory(cfg, !recovering, false); err != nil {\n\t\t\ts.Warnf(\"JetStream cluster error updating stream %q for account %q: %v\", cfg.Name, acc.Name, err)\n\t\t}\n\t}\n\n\t// If not found we must be expanding into this node since if we are here we know we are a member.\n\tif err == ErrJetStreamStreamNotFound {\n\t\tjs.processStreamAssignment(sa)\n\t\treturn\n\t}\n\n\tif err != nil {\n\t\tjs.mu.Lock()\n\t\tsa.err = err\n\t\tresult := &streamAssignmentResult{\n\t\t\tAccount: sa.Client.serviceAccount(),\n\t\t\tStream: sa.Config.Name,\n\t\t\tResponse: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}},\n\t\t\tUpdate: true,\n\t\t}\n\t\tresult.Response.Error = NewJSStreamGeneralError(err, Unless(err))\n\t\tjs.mu.Unlock()\n\n\t\t// Send response to the metadata leader. They will forward to the user as needed.\n\t\ts.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result)\n\t\treturn\n\t}\n\n\tisLeader := mset.IsLeader()\n\n\t// Check for missing syncSubject bug.\n\tif isLeader && osa != nil && osa.Sync == _EMPTY_ {\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn\n\t}\n\n\t// If we were a single node being promoted assume leadership role for purpose of responding.\n\tif !hasResponded && !isLeader && !alreadyRunning {\n\t\tisLeader = true\n\t}\n\n\t// Check if we should bail.\n\tif !isLeader || hasResponded || recovering {\n\t\treturn\n\t}\n\n\t// Send our response.\n\tvar resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}}\n\tmsetCfg := mset.config()\n\tresp.StreamInfo = &StreamInfo{\n\t\tCreated: mset.createdTime(),\n\t\tState: mset.state(),\n\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\tCluster: js.clusterInfo(mset.raftGroup()),\n\t\tMirror: mset.mirrorInfo(),\n\t\tSources: mset.sourcesInfo(),\n\t\tTimeStamp: time.Now().UTC(),\n\t}\n\n\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n}\n\n// processClusterCreateStream is called when we have a stream assignment that\n// has been committed and this server is a member of the peer group.\nfunc (js *jetStream) processClusterCreateStream(acc *Account, sa *streamAssignment) {\n\tif sa == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\ts, rg, created := js.srv, sa.Group, sa.Created\n\talreadyRunning := rg.node != nil\n\tstorage := sa.Config.Storage\n\trestore := sa.Restore\n\trecovering := sa.recovering\n\tjs.mu.RUnlock()\n\n\t// Process the raft group and make sure it's running if needed.\n\t_, err := js.createRaftGroup(acc.GetName(), rg, recovering, storage, pprofLabels{\n\t\t\"type\": \"stream\",\n\t\t\"account\": acc.Name,\n\t\t\"stream\": sa.Config.Name,\n\t})\n\n\t// If we are restoring, create the stream if we are R>1 and not the preferred who handles the\n\t// receipt of the snapshot itself.\n\tshouldCreate := true\n\tif restore != nil {\n\t\tif len(rg.Peers) == 1 || rg.node != nil && rg.node.ID() == rg.Preferred {\n\t\t\tshouldCreate = false\n\t\t} else {\n\t\t\tjs.mu.Lock()\n\t\t\tsa.Restore = nil\n\t\t\tjs.mu.Unlock()\n\t\t}\n\t}\n\n\t// Our stream.\n\tvar mset *stream\n\n\t// Process here if not restoring or not the leader.\n\tif shouldCreate && err == nil {\n\t\t// Go ahead and create or update the stream.\n\t\tmset, err = acc.lookupStream(sa.Config.Name)\n\t\tif err == nil && mset != nil {\n\t\t\tosa := mset.streamAssignment()\n\t\t\t// If we already have a stream assignment and they are the same exact config, short circuit here.\n\t\t\tif osa != nil {\n\t\t\t\tif reflect.DeepEqual(osa.Config, sa.Config) {\n\t\t\t\t\tif sa.Group.Name == osa.Group.Name && reflect.DeepEqual(sa.Group.Peers, osa.Group.Peers) {\n\t\t\t\t\t\t// Since this already exists we know it succeeded, just respond to this caller.\n\t\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\t\tclient, subject, reply, recovering := sa.Client, sa.Subject, sa.Reply, sa.recovering\n\t\t\t\t\t\tjs.mu.RUnlock()\n\n\t\t\t\t\t\tif !recovering {\n\t\t\t\t\t\t\tvar resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}\n\t\t\t\t\t\t\tmsetCfg := mset.config()\n\t\t\t\t\t\t\tresp.StreamInfo = &StreamInfo{\n\t\t\t\t\t\t\t\tCreated: mset.createdTime(),\n\t\t\t\t\t\t\t\tState: mset.state(),\n\t\t\t\t\t\t\t\tConfig: *setDynamicStreamMetadata(&msetCfg),\n\t\t\t\t\t\t\t\tCluster: js.clusterInfo(mset.raftGroup()),\n\t\t\t\t\t\t\t\tSources: mset.sourcesInfo(),\n\t\t\t\t\t\t\t\tMirror: mset.mirrorInfo(),\n\t\t\t\t\t\t\t\tTimeStamp: time.Now().UTC(),\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// We had a bug where we could have multiple assignments for the same\n\t\t\t\t\t\t// stream but with different group assignments, including multiple raft\n\t\t\t\t\t\t// groups. So check for that here. We can only bet on the last one being\n\t\t\t\t\t\t// consistent in the long run, so let it continue if we see this condition.\n\t\t\t\t\t\ts.Warnf(\"JetStream cluster detected duplicate assignment for stream %q for account %q\", sa.Config.Name, acc.Name)\n\t\t\t\t\t\tif osa.Group.node != nil && osa.Group.node != sa.Group.node {\n\t\t\t\t\t\t\tosa.Group.node.Delete()\n\t\t\t\t\t\t\tosa.Group.node = nil\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.setStreamAssignment(sa)\n\t\t\t// Check if our config has really been updated.\n\t\t\tcfg := mset.config()\n\t\t\tif !reflect.DeepEqual(&cfg, sa.Config) {\n\t\t\t\tif err = mset.updateWithAdvisory(sa.Config, false, false); err != nil {\n\t\t\t\t\ts.Warnf(\"JetStream cluster error updating stream %q for account %q: %v\", sa.Config.Name, acc.Name, err)\n\t\t\t\t\tif osa != nil {\n\t\t\t\t\t\t// Process the raft group and make sure it's running if needed.\n\t\t\t\t\t\tjs.createRaftGroup(acc.GetName(), osa.Group, osa.recovering, storage, pprofLabels{\n\t\t\t\t\t\t\t\"type\": \"stream\",\n\t\t\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t\t\t})\n\t\t\t\t\t\tmset.setStreamAssignment(osa)\n\t\t\t\t\t}\n\t\t\t\t\tif rg.node != nil {\n\t\t\t\t\t\trg.node.Delete()\n\t\t\t\t\t\trg.node = nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if err == NewJSStreamNotFoundError() {\n\t\t\t// Add in the stream here.\n\t\t\tmset, err = acc.addStreamWithAssignment(sa.Config, nil, sa, false, false)\n\t\t}\n\t\tif mset != nil {\n\t\t\tmset.setCreatedTime(created)\n\t\t}\n\t}\n\n\t// This is an error condition.\n\tif err != nil {\n\t\t// If we're shutting down we could get a variety of errors, for example:\n\t\t// 'JetStream not enabled for account' when looking up the stream.\n\t\t// Normally we can continue and delete state, but need to be careful when shutting down.\n\t\tif js.isShuttingDown() {\n\t\t\ts.Debugf(\"Could not create stream, JetStream shutting down\")\n\t\t\treturn\n\t\t}\n\n\t\tif IsNatsErr(err, JSStreamStoreFailedF) {\n\t\t\ts.Warnf(\"Stream create failed for '%s > %s': %v\", sa.Client.serviceAccount(), sa.Config.Name, err)\n\t\t\terr = errStreamStoreFailed\n\t\t}\n\t\tjs.mu.Lock()\n\n\t\tsa.err = err\n\t\thasResponded := sa.responded\n\n\t\t// If out of space do nothing for now.\n\t\tif isOutOfSpaceErr(err) {\n\t\t\thasResponded = true\n\t\t}\n\n\t\tif rg.node != nil {\n\t\t\trg.node.Delete()\n\t\t}\n\n\t\tvar result *streamAssignmentResult\n\t\tif !hasResponded {\n\t\t\tresult = &streamAssignmentResult{\n\t\t\t\tAccount: sa.Client.serviceAccount(),\n\t\t\t\tStream: sa.Config.Name,\n\t\t\t\tResponse: &JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}},\n\t\t\t}\n\t\t\tresult.Response.Error = NewJSStreamCreateError(err, Unless(err))\n\t\t}\n\t\tjs.mu.Unlock()\n\n\t\t// Send response to the metadata leader. They will forward to the user as needed.\n\t\tif result != nil {\n\t\t\ts.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, result)\n\t\t}\n\t\treturn\n\t}\n\n\t// Re-capture node.\n\tjs.mu.RLock()\n\tnode := rg.node\n\tjs.mu.RUnlock()\n\n\t// Start our monitoring routine.\n\tif node != nil {\n\t\tif !alreadyRunning {\n\t\t\tif mset != nil {\n\t\t\t\tmset.monitorWg.Add(1)\n\t\t\t}\n\t\t\tstarted := s.startGoRoutine(\n\t\t\t\tfunc() { js.monitorStream(mset, sa, false) },\n\t\t\t\tpprofLabels{\n\t\t\t\t\t\"type\": \"stream\",\n\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t},\n\t\t\t)\n\t\t\tif !started && mset != nil {\n\t\t\t\tmset.monitorWg.Done()\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// Single replica stream, process manually here.\n\t\t// If we are restoring, process that first.\n\t\tif sa.Restore != nil {\n\t\t\t// We are restoring a stream here.\n\t\t\trestoreDoneCh := s.processStreamRestore(sa.Client, acc, sa.Config, _EMPTY_, sa.Reply, _EMPTY_)\n\t\t\ts.startGoRoutine(func() {\n\t\t\t\tdefer s.grWG.Done()\n\t\t\t\tselect {\n\t\t\t\tcase err := <-restoreDoneCh:\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tmset, err = acc.lookupStream(sa.Config.Name)\n\t\t\t\t\t\tif mset != nil {\n\t\t\t\t\t\t\tmset.setStreamAssignment(sa)\n\t\t\t\t\t\t\tmset.setCreatedTime(created)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif mset != nil {\n\t\t\t\t\t\t\tmset.delete()\n\t\t\t\t\t\t}\n\t\t\t\t\t\tjs.mu.Lock()\n\t\t\t\t\t\tsa.err = err\n\t\t\t\t\t\tresult := &streamAssignmentResult{\n\t\t\t\t\t\t\tAccount: sa.Client.serviceAccount(),\n\t\t\t\t\t\t\tStream: sa.Config.Name,\n\t\t\t\t\t\t\tRestore: &JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}},\n\t\t\t\t\t\t}\n\t\t\t\t\t\tresult.Restore.Error = NewJSStreamRestoreError(err, Unless(err))\n\t\t\t\t\t\tjs.mu.Unlock()\n\t\t\t\t\t\t// Send response to the metadata leader. They will forward to the user as needed.\n\t\t\t\t\t\tb, _ := json.Marshal(result) // Avoids auto-processing and doing fancy json with newlines.\n\t\t\t\t\t\ts.sendInternalMsgLocked(streamAssignmentSubj, _EMPTY_, nil, b)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tjs.processStreamLeaderChange(mset, true)\n\n\t\t\t\t\t// Check to see if we have restored consumers here.\n\t\t\t\t\t// These are not currently assigned so we will need to do so here.\n\t\t\t\t\tif consumers := mset.getPublicConsumers(); len(consumers) > 0 {\n\t\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\t\tcc := js.cluster\n\t\t\t\t\t\tjs.mu.RUnlock()\n\n\t\t\t\t\t\tfor _, o := range consumers {\n\t\t\t\t\t\t\tname, cfg := o.String(), o.config()\n\t\t\t\t\t\t\trg := cc.createGroupForConsumer(&cfg, sa)\n\n\t\t\t\t\t\t\t// Place our initial state here as well for assignment distribution.\n\t\t\t\t\t\t\tca := &consumerAssignment{\n\t\t\t\t\t\t\t\tGroup: rg,\n\t\t\t\t\t\t\t\tStream: sa.Config.Name,\n\t\t\t\t\t\t\t\tName: name,\n\t\t\t\t\t\t\t\tConfig: &cfg,\n\t\t\t\t\t\t\t\tClient: sa.Client,\n\t\t\t\t\t\t\t\tCreated: o.createdTime(),\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\taddEntry := encodeAddConsumerAssignment(ca)\n\t\t\t\t\t\t\tcc.meta.ForwardProposal(addEntry)\n\n\t\t\t\t\t\t\t// Check to make sure we see the assignment.\n\t\t\t\t\t\t\tgo func() {\n\t\t\t\t\t\t\t\tticker := time.NewTicker(time.Second)\n\t\t\t\t\t\t\t\tdefer ticker.Stop()\n\t\t\t\t\t\t\t\tfor range ticker.C {\n\t\t\t\t\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\t\t\t\t\tca, meta := js.consumerAssignment(ca.Client.serviceAccount(), sa.Config.Name, name), cc.meta\n\t\t\t\t\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\t\t\t\t\tif ca == nil {\n\t\t\t\t\t\t\t\t\t\ts.Warnf(\"Consumer assignment has not been assigned, retrying\")\n\t\t\t\t\t\t\t\t\t\tif meta != nil {\n\t\t\t\t\t\t\t\t\t\t\tmeta.ForwardProposal(addEntry)\n\t\t\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t})\n\t\t} else {\n\t\t\tjs.processStreamLeaderChange(mset, true)\n\t\t}\n\t}\n}\n\n// processStreamRemoval is called when followers have replicated an assignment.\nfunc (js *jetStream) processStreamRemoval(sa *streamAssignment) {\n\tjs.mu.Lock()\n\ts, cc := js.srv, js.cluster\n\tif s == nil || cc == nil || cc.meta == nil {\n\t\t// TODO(dlc) - debug at least\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\taccName, stream, created := sa.Client.serviceAccount(), sa.Config.Name, sa.Created\n\tvar isMember bool\n\tif sa.Group != nil {\n\t\tisMember = sa.Group.isMember(cc.meta.ID())\n\t}\n\twasLeader := cc.isStreamLeader(accName, stream)\n\n\t// Check if we already have this assigned.\n\taccStreams := cc.streams[accName]\n\tneedDelete := accStreams != nil && accStreams[stream] != nil\n\tif needDelete {\n\t\tif osa := accStreams[stream]; osa != nil && osa.unsupported != nil {\n\t\t\tosa.unsupported.closeInfoSub(js.srv)\n\t\t\t// Remember we used to be unsupported, just so we can send a successful delete response.\n\t\t\tif sa.unsupported == nil {\n\t\t\t\tsa.unsupported = osa.unsupported\n\t\t\t}\n\t\t}\n\t\tdelete(accStreams, stream)\n\t\tif len(accStreams) == 0 {\n\t\t\tdelete(cc.streams, accName)\n\t\t}\n\t}\n\tcc.removeInflightStreamProposal(accName, sa.Config.Name)\n\tjs.mu.Unlock()\n\n\t// During initial/startup recovery we'll not have registered the stream assignment,\n\t// but might have recovered the stream from disk. We'll need to make sure that we only\n\t// delete the stream if it wasn't created after this delete.\n\tif !needDelete && !created.IsZero() {\n\t\tif acc, err := s.lookupOrFetchAccount(accName, isMember); err == nil {\n\t\t\tif mset, err := acc.lookupStream(stream); err == nil {\n\t\t\t\tneedDelete = !mset.createdTime().After(created)\n\t\t\t}\n\t\t}\n\t}\n\n\tif needDelete {\n\t\tjs.processClusterDeleteStream(sa, isMember, wasLeader)\n\t}\n}\n\nfunc (js *jetStream) processClusterDeleteStream(sa *streamAssignment, isMember, wasLeader bool) {\n\tif sa == nil {\n\t\treturn\n\t}\n\tjs.mu.RLock()\n\ts := js.srv\n\tnode := sa.Group.node\n\thadLeader := node == nil || !node.Leaderless()\n\toffline := s.allPeersOffline(sa.Group) || sa.unsupported != nil\n\tvar isMetaLeader bool\n\tif cc := js.cluster; cc != nil {\n\t\tisMetaLeader = cc.isLeader()\n\t}\n\trecovering := sa.recovering\n\tjs.mu.RUnlock()\n\n\tstopped := false\n\tvar resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}}\n\tvar err error\n\tvar acc *Account\n\n\t// Go ahead and delete the stream if we have it and the account here.\n\tif acc, _ = s.LookupAccount(sa.Client.serviceAccount()); acc != nil {\n\t\tif mset, _ := acc.lookupStream(sa.Config.Name); mset != nil {\n\t\t\t// shut down monitor by shutting down raft\n\t\t\tif n := mset.raftNode(); n != nil {\n\t\t\t\tn.Delete()\n\t\t\t}\n\t\t\t// wait for monitor to be shut down\n\t\t\tmset.signalMonitorQuit()\n\t\t\tmset.monitorWg.Wait()\n\t\t\terr = mset.stop(true, wasLeader)\n\t\t\tstopped = true\n\t\t} else if isMember {\n\t\t\ts.Warnf(\"JetStream failed to lookup running stream while removing stream '%s > %s' from this server\",\n\t\t\t\tsa.Client.serviceAccount(), sa.Config.Name)\n\t\t}\n\t} else if isMember {\n\t\ts.Warnf(\"JetStream failed to lookup account while removing stream '%s > %s' from this server\", sa.Client.serviceAccount(), sa.Config.Name)\n\t}\n\n\t// Always delete the node if present.\n\tif node != nil {\n\t\tnode.Delete()\n\t}\n\n\t// This is a stop gap cleanup in case\n\t// 1) the account or mset does not exist and/or\n\t// 2) node was nil (and couldn't be deleted)\n\tif !stopped || node == nil {\n\t\tif sacc := s.SystemAccount(); sacc != nil {\n\t\t\tsaccName := sacc.GetName()\n\t\t\tos.RemoveAll(filepath.Join(js.config.StoreDir, saccName, defaultStoreDirName, sa.Group.Name))\n\t\t\t// cleanup dependent consumer groups\n\t\t\tif !stopped {\n\t\t\t\tfor _, ca := range sa.consumers {\n\t\t\t\t\t// Make sure we cleanup any possible running nodes for the consumers.\n\t\t\t\t\tif isMember && ca.Group != nil && ca.Group.node != nil {\n\t\t\t\t\t\tca.Group.node.Delete()\n\t\t\t\t\t}\n\t\t\t\t\tos.RemoveAll(filepath.Join(js.config.StoreDir, saccName, defaultStoreDirName, ca.Group.Name))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\taccDir := filepath.Join(js.config.StoreDir, sa.Client.serviceAccount())\n\tstreamDir := filepath.Join(accDir, streamsDir)\n\tos.RemoveAll(filepath.Join(streamDir, sa.Config.Name))\n\n\t// no op if not empty\n\tos.Remove(streamDir)\n\tos.Remove(accDir)\n\n\t// Normally we want only the leader to respond here, but if we had no leader then all members will respond to make\n\t// sure we get feedback to the user.\n\tif !isMember || (hadLeader && !wasLeader) {\n\t\t// If all the peers are offline and we are the meta leader we will also respond, so suppress returning here.\n\t\tif !(offline && isMetaLeader) {\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Do not respond if the account does not exist any longer\n\tif acc == nil || recovering {\n\t\treturn\n\t}\n\n\tif err != nil {\n\t\tresp.Error = NewJSStreamGeneralError(err, Unless(err))\n\t\ts.sendAPIErrResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, s.jsonResponse(resp))\n\t} else {\n\t\tresp.Success = true\n\t\ts.sendAPIResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, s.jsonResponse(resp))\n\t}\n}\n\n// processConsumerAssignment is called when followers have replicated an assignment for a consumer.\nfunc (js *jetStream) processConsumerAssignment(ca *consumerAssignment) {\n\tjs.mu.RLock()\n\ts, cc := js.srv, js.cluster\n\taccName, stream, consumerName := ca.Client.serviceAccount(), ca.Stream, ca.Name\n\tnoMeta := cc == nil || cc.meta == nil\n\tshuttingDown := js.shuttingDown\n\tvar ourID string\n\tif !noMeta {\n\t\tourID = cc.meta.ID()\n\t}\n\tvar isMember bool\n\tif ca.Group != nil && ourID != _EMPTY_ {\n\t\tisMember = ca.Group.isMember(ourID)\n\t}\n\tjs.mu.RUnlock()\n\n\tif s == nil || noMeta || shuttingDown {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tsa := js.streamAssignment(accName, stream)\n\tif sa == nil {\n\t\tjs.mu.Unlock()\n\t\ts.Debugf(\"Consumer create failed, could not locate stream '%s > %s'\", accName, stream)\n\t\treturn\n\t}\n\n\t// Track if this existed already.\n\tvar wasExisting bool\n\n\t// Check if we have an existing consumer assignment.\n\tif sa.consumers == nil {\n\t\tsa.consumers = make(map[string]*consumerAssignment)\n\t}\n\toca := sa.consumers[ca.Name]\n\tif oca != nil {\n\t\twasExisting = true\n\t\t// Copy over private existing state from former CA.\n\t\tif ca.Group != nil {\n\t\t\tca.Group.node = oca.Group.node\n\t\t}\n\t\tca.responded = oca.responded\n\t\tca.err = oca.err\n\n\t\t// Unsubscribe if it was previously unsupported.\n\t\tif oca.unsupported != nil {\n\t\t\toca.unsupported.closeInfoSub(s)\n\t\t\t// If we've seen unsupported once, it remains for the lifetime of this server process.\n\t\t\tif ca.unsupported == nil {\n\t\t\t\tca.unsupported = oca.unsupported\n\t\t\t}\n\t\t}\n\t}\n\n\t// Capture the optional state. We will pass it along if we are a member to apply.\n\t// This is only applicable when restoring a stream with consumers.\n\tstate := ca.State\n\tca.State = nil\n\n\t// Place into our internal map under the stream assignment.\n\t// Ok to replace an existing one, we check on process call below.\n\tsa.consumers[ca.Name] = ca\n\tcc.removeInflightConsumerProposal(accName, stream, consumerName)\n\n\t// If unsupported, we can't register any further.\n\tif ca.unsupported != nil {\n\t\tca.unsupported.setupInfoSub(s, ca)\n\t\ts.Warnf(\"Detected unsupported consumer '%s > %s > %s': %s\", accName, stream, ca.Name, ca.unsupported.reason)\n\n\t\t// Mark stream as unsupported as well\n\t\tif sa.unsupported == nil {\n\t\t\tsa.unsupported = newUnsupportedStreamAssignment(s, sa, fmt.Errorf(\"unsupported consumer %q\", ca.Name))\n\t\t}\n\t\tsa.unsupported.setupInfoSub(s, sa)\n\t\tjs.mu.Unlock()\n\n\t\t// Be conservative by protecting the whole stream, even if just one consumer is unsupported.\n\t\t// This ensures it's safe, even with Interest-based retention where it would otherwise\n\t\t// continue accepting but dropping messages.\n\t\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tmset, err := acc.lookupStream(stream)\n\t\tif err != nil || mset.closed.Load() {\n\t\t\treturn\n\t\t}\n\t\ts.Warnf(\"Stopping unsupported stream '%s > %s'\", accName, stream)\n\t\tmset.stop(false, false)\n\t\treturn\n\t}\n\tjs.mu.Unlock()\n\n\tacc, err := s.lookupOrFetchAccount(accName, isMember)\n\tif err != nil {\n\t\tll := fmt.Sprintf(\"Account [%s] lookup for consumer create failed: %v\", accName, err)\n\t\tif isMember {\n\t\t\tif !js.isMetaRecovering() {\n\t\t\t\t// If we can not lookup the account and we are a member, send this result back to the metacontroller leader.\n\t\t\t\tresult := &consumerAssignmentResult{\n\t\t\t\t\tAccount: accName,\n\t\t\t\t\tStream: stream,\n\t\t\t\t\tConsumer: consumerName,\n\t\t\t\t\tResponse: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}},\n\t\t\t\t}\n\t\t\t\tresult.Response.Error = NewJSNoAccountError()\n\t\t\t\ts.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result)\n\t\t\t}\n\t\t\ts.Warnf(ll)\n\t\t} else {\n\t\t\ts.Debugf(ll)\n\t\t}\n\t\treturn\n\t}\n\n\t// Check if this is for us..\n\tif isMember {\n\t\tjs.processClusterCreateConsumer(oca, ca, state, wasExisting)\n\t} else if mset, _ := acc.lookupStream(sa.Config.Name); mset != nil {\n\t\tif o := mset.lookupConsumer(ca.Name); o != nil {\n\t\t\t// We have one here even though we are not a member. This can happen on re-assignment.\n\t\t\ts.removeConsumer(o, ca)\n\t\t}\n\t}\n}\n\n// Common function to remove ourselves from this server.\n// This can happen on re-assignment, move, etc\nfunc (s *Server) removeConsumer(o *consumer, nca *consumerAssignment) {\n\tif o == nil {\n\t\treturn\n\t}\n\t// Make sure to use the new stream assignment, not our own.\n\ts.Debugf(\"JetStream removing consumer '%s > %s > %s' from this server\", nca.Client.serviceAccount(), nca.Stream, nca.Name)\n\tif node := o.raftNode(); node != nil {\n\t\tnode.StepDown(nca.Group.Preferred)\n\t\t// shutdown monitor by shutting down raft.\n\t\tnode.Delete()\n\t}\n\n\tvar isShuttingDown bool\n\t// Make sure this node is no longer attached to our consumer assignment.\n\tif js, _ := s.getJetStreamCluster(); js != nil {\n\t\tjs.mu.Lock()\n\t\tnca.Group.node = nil\n\t\tnca.err = nil\n\t\tisShuttingDown = js.shuttingDown\n\t\tjs.mu.Unlock()\n\t}\n\n\tif !isShuttingDown {\n\t\t// wait for monitor to be shutdown.\n\t\to.signalMonitorQuit()\n\t\to.monitorWg.Wait()\n\t}\n\to.deleteWithoutAdvisory()\n}\n\nfunc (js *jetStream) processConsumerRemoval(ca *consumerAssignment) {\n\tjs.mu.Lock()\n\ts, cc := js.srv, js.cluster\n\tif s == nil || cc == nil || cc.meta == nil {\n\t\t// TODO(dlc) - debug at least\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\taccName, stream, name, created := ca.Client.serviceAccount(), ca.Stream, ca.Name, ca.Created\n\twasLeader := cc.isConsumerLeader(accName, stream, name)\n\n\t// Delete from our state.\n\tvar needDelete bool\n\tif accStreams := cc.streams[accName]; accStreams != nil {\n\t\tif sa := accStreams[ca.Stream]; sa != nil && sa.consumers != nil && sa.consumers[ca.Name] != nil {\n\t\t\toca := sa.consumers[ca.Name]\n\t\t\t// Make sure this removal is for what we have, otherwise ignore.\n\t\t\tif ca.Group != nil && oca.Group != nil && ca.Group.Name == oca.Group.Name {\n\t\t\t\tneedDelete = true\n\t\t\t\tdelete(sa.consumers, ca.Name)\n\t\t\t\t// Remember we used to be unsupported, just so we can send a successful delete response.\n\t\t\t\tif ca.unsupported == nil {\n\t\t\t\t\tca.unsupported = oca.unsupported\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tcc.removeInflightConsumerProposal(accName, stream, name)\n\tjs.mu.Unlock()\n\n\t// During initial/startup recovery we'll not have registered the consumer assignment,\n\t// but might have recovered the consumer from disk. We'll need to make sure that we only\n\t// delete the consumer if it wasn't created after this delete.\n\tif !needDelete && !created.IsZero() {\n\t\tif acc, err := s.LookupAccount(accName); err == nil {\n\t\t\tif mset, err := acc.lookupStream(stream); err == nil {\n\t\t\t\tif o := mset.lookupConsumer(name); o != nil {\n\t\t\t\t\tneedDelete = !o.createdTime().After(created)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif needDelete {\n\t\tjs.processClusterDeleteConsumer(ca, wasLeader)\n\t}\n}\n\ntype consumerAssignmentResult struct {\n\tAccount string `json:\"account\"`\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tResponse *JSApiConsumerCreateResponse `json:\"response,omitempty\"`\n}\n\n// processClusterCreateConsumer is when we are a member of the group and need to create the consumer.\nfunc (js *jetStream) processClusterCreateConsumer(oca, ca *consumerAssignment, state *ConsumerState, wasExisting bool) {\n\tif ca == nil {\n\t\treturn\n\t}\n\tjs.mu.RLock()\n\ts := js.srv\n\trg := ca.Group\n\talreadyRunning := rg != nil && rg.node != nil\n\taccName, stream, consumer := ca.Client.serviceAccount(), ca.Stream, ca.Name\n\trecovering := ca.recovering\n\tjs.mu.RUnlock()\n\n\tacc, err := s.LookupAccount(accName)\n\tif err != nil {\n\t\ts.Warnf(\"JetStream cluster failed to lookup account %q: %v\", accName, err)\n\t\treturn\n\t}\n\n\t// Go ahead and create or update the consumer.\n\tmset, err := acc.lookupStream(stream)\n\tif err != nil {\n\t\tif !js.isMetaRecovering() {\n\t\t\tjs.mu.Lock()\n\t\t\ts.Warnf(\"Consumer create failed, could not locate stream '%s > %s > %s'\", ca.Client.serviceAccount(), ca.Stream, ca.Name)\n\t\t\tca.err = NewJSStreamNotFoundError()\n\t\t\tresult := &consumerAssignmentResult{\n\t\t\t\tAccount: ca.Client.serviceAccount(),\n\t\t\t\tStream: ca.Stream,\n\t\t\t\tConsumer: ca.Name,\n\t\t\t\tResponse: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}},\n\t\t\t}\n\t\t\tresult.Response.Error = NewJSStreamNotFoundError()\n\t\t\ts.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result)\n\t\t\tjs.mu.Unlock()\n\t\t}\n\t\treturn\n\t}\n\n\t// Check if we already have this consumer running.\n\to := mset.lookupConsumer(consumer)\n\n\tif o != nil && oca != nil && oca.Group.Name != ca.Group.Name {\n\t\ts.Warnf(\"JetStream cluster detected consumer remapping for '%s > %s' from %q to %q\",\n\t\t\tacc, ca.Name, oca.Group.Name, ca.Group.Name)\n\t\to.clearNode()\n\t\to.signalMonitorQuit()\n\t\to.monitorWg.Wait()\n\t\talreadyRunning = false\n\t\t// Make sure to clear from original.\n\t\tjs.mu.Lock()\n\t\toca.Group.node = nil\n\t\tjs.mu.Unlock()\n\t}\n\n\t// Process the raft group and make sure it's running if needed.\n\tstorage := mset.config().Storage\n\tif ca.Config.MemoryStorage {\n\t\tstorage = MemoryStorage\n\t}\n\t// No-op if R1.\n\tjs.createRaftGroup(accName, rg, recovering, storage, pprofLabels{\n\t\t\"type\": \"consumer\",\n\t\t\"account\": mset.accName(),\n\t\t\"stream\": ca.Stream,\n\t\t\"consumer\": ca.Name,\n\t})\n\n\t// Check if we already have this consumer running.\n\tvar didCreate, isConfigUpdate, needsLocalResponse bool\n\tif o == nil {\n\t\t// Add in the consumer if needed.\n\t\tif o, err = mset.addConsumerWithAssignment(ca.Config, ca.Name, ca, js.isMetaRecovering(), ActionCreateOrUpdate, false); err == nil {\n\t\t\tdidCreate = true\n\t\t}\n\t} else {\n\t\t// This consumer exists.\n\t\t// Only update if config is really different.\n\t\tcfg := o.config()\n\t\tif isConfigUpdate = !reflect.DeepEqual(&cfg, ca.Config); isConfigUpdate {\n\t\t\t// Call into update, ignore consumer exists error here since this means an old deliver subject is bound\n\t\t\t// which can happen on restart etc.\n\t\t\t// JS lock needed as this can mutate the consumer assignments and race with updateInactivityThreshold.\n\t\t\tjs.mu.Lock()\n\t\t\terr := o.updateConfig(ca.Config)\n\t\t\tjs.mu.Unlock()\n\t\t\tif err != nil && err != NewJSConsumerNameExistError() {\n\t\t\t\t// This is essentially an update that has failed. Respond back to metaleader if we are not recovering.\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tif !js.metaRecovering {\n\t\t\t\t\tresult := &consumerAssignmentResult{\n\t\t\t\t\t\tAccount: accName,\n\t\t\t\t\t\tStream: stream,\n\t\t\t\t\t\tConsumer: consumer,\n\t\t\t\t\t\tResponse: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}},\n\t\t\t\t\t}\n\t\t\t\t\tresult.Response.Error = NewJSConsumerNameExistError()\n\t\t\t\t\ts.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, result)\n\t\t\t\t}\n\t\t\t\ts.Warnf(\"Consumer create failed during update for '%s > %s > %s': %v\", ca.Client.serviceAccount(), ca.Stream, ca.Name, err)\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tvar sendState bool\n\t\tjs.mu.RLock()\n\t\tn := rg.node\n\t\t// Check if we already had a consumer assignment and its still pending.\n\t\tcca, oca := ca, o.consumerAssignment()\n\t\tif oca != nil {\n\t\t\tif !oca.responded {\n\t\t\t\t// We can't override info for replying here otherwise leader once elected can not respond.\n\t\t\t\t// So copy over original client and the reply from the old ca.\n\t\t\t\tcac := *ca\n\t\t\t\tcac.Client = oca.Client\n\t\t\t\tcac.Reply = oca.Reply\n\t\t\t\tcca = &cac\n\t\t\t\tneedsLocalResponse = true\n\t\t\t}\n\t\t\t// If we look like we are scaling up, let's send our current state to the group.\n\t\t\tsendState = len(ca.Group.Peers) > len(oca.Group.Peers) && o.IsLeader() && n != nil\n\t\t\t// Signal that this is an update\n\t\t\tif ca.Reply != _EMPTY_ {\n\t\t\t\tisConfigUpdate = true\n\t\t\t}\n\t\t}\n\t\tjs.mu.RUnlock()\n\n\t\tif sendState {\n\t\t\tif snap, err := o.store.EncodedState(); err == nil {\n\t\t\t\tn.SendSnapshot(snap)\n\t\t\t}\n\t\t}\n\n\t\t// Set CA for our consumer.\n\t\to.setConsumerAssignment(cca)\n\t\ts.Debugf(\"JetStream cluster, consumer '%s > %s > %s' was already running\", ca.Client.serviceAccount(), ca.Stream, ca.Name)\n\t}\n\n\t// If we have an initial state set apply that now.\n\tif state != nil && o != nil {\n\t\to.mu.Lock()\n\t\terr = o.setStoreState(state)\n\t\to.mu.Unlock()\n\t}\n\n\tif err != nil {\n\t\t// If we're shutting down we could get a variety of errors.\n\t\t// Normally we can continue and delete state, but need to be careful when shutting down.\n\t\tif js.isShuttingDown() {\n\t\t\ts.Debugf(\"Could not create consumer, JetStream shutting down\")\n\t\t\treturn\n\t\t}\n\n\t\tif IsNatsErr(err, JSConsumerStoreFailedErrF) {\n\t\t\ts.Warnf(\"Consumer create failed for '%s > %s > %s': %v\", ca.Client.serviceAccount(), ca.Stream, ca.Name, err)\n\t\t\terr = errConsumerStoreFailed\n\t\t}\n\n\t\tjs.mu.Lock()\n\n\t\tca.err = err\n\t\thasResponded := ca.responded\n\n\t\t// If out of space do nothing for now.\n\t\tif isOutOfSpaceErr(err) {\n\t\t\thasResponded = true\n\t\t}\n\n\t\tif rg.node != nil {\n\t\t\trg.node.Delete()\n\t\t\t// Clear the node here.\n\t\t\trg.node = nil\n\t\t}\n\n\t\t// If we did seem to create a consumer make sure to stop it.\n\t\tif o != nil {\n\t\t\to.stop()\n\t\t}\n\n\t\tvar result *consumerAssignmentResult\n\t\tif !hasResponded && !js.metaRecovering {\n\t\t\tresult = &consumerAssignmentResult{\n\t\t\t\tAccount: ca.Client.serviceAccount(),\n\t\t\t\tStream: ca.Stream,\n\t\t\t\tConsumer: ca.Name,\n\t\t\t\tResponse: &JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}},\n\t\t\t}\n\t\t\tresult.Response.Error = NewJSConsumerCreateError(err, Unless(err))\n\t\t} else if err == errNoInterest {\n\t\t\t// This is a stranded ephemeral, let's clean this one up.\n\t\t\tsubject := fmt.Sprintf(JSApiConsumerDeleteT, ca.Stream, ca.Name)\n\t\t\tmset.outq.send(newJSPubMsg(subject, _EMPTY_, _EMPTY_, nil, nil, nil, 0))\n\t\t}\n\t\tjs.mu.Unlock()\n\n\t\tif result != nil {\n\t\t\t// Send response to the metadata leader. They will forward to the user as needed.\n\t\t\tb, _ := json.Marshal(result) // Avoids auto-processing and doing fancy json with newlines.\n\t\t\ts.sendInternalMsgLocked(consumerAssignmentSubj, _EMPTY_, nil, b)\n\t\t}\n\t} else {\n\t\tjs.mu.RLock()\n\t\tnode := rg.node\n\t\tjs.mu.RUnlock()\n\n\t\tif didCreate {\n\t\t\to.setCreatedTime(ca.Created)\n\t\t} else {\n\t\t\t// Check for scale down to 1..\n\t\t\tif node != nil && len(rg.Peers) == 1 {\n\t\t\t\to.clearNode()\n\t\t\t\to.signalMonitorQuit()\n\t\t\t\to.monitorWg.Wait()\n\t\t\t\t// Need to clear from rg too.\n\t\t\t\tjs.mu.Lock()\n\t\t\t\trg.node = nil\n\t\t\t\tclient, subject, reply := ca.Client, ca.Subject, ca.Reply\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\t// Perform the leader change in a goroutine, otherwise we could block meta operations.\n\t\t\t\tif o.shouldStartMonitor() {\n\t\t\t\t\tstarted := s.startGoRoutine(\n\t\t\t\t\t\tfunc() {\n\t\t\t\t\t\t\tdefer s.grWG.Done()\n\t\t\t\t\t\t\tdefer o.clearMonitorRunning()\n\t\t\t\t\t\t\to.setLeader(true)\n\t\t\t\t\t\t\tvar resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}\n\t\t\t\t\t\t\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.info())\n\t\t\t\t\t\t\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t\t\t\t\t\t},\n\t\t\t\t\t\tpprofLabels{\n\t\t\t\t\t\t\t\"type\": \"consumer\",\n\t\t\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t\t\t\t\"consumer\": ca.Name,\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t\tif !started {\n\t\t\t\t\t\to.clearMonitorRunning()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tif node == nil {\n\t\t\t// Wait for the previous routine to stop running.\n\t\t\to.signalMonitorQuit()\n\t\t\to.monitorWg.Wait()\n\t\t\t// Single replica consumer, process manually here.\n\t\t\tjs.mu.Lock()\n\t\t\t// Force response in case we think this is an update.\n\t\t\tif !js.metaRecovering && isConfigUpdate {\n\t\t\t\tca.responded = false\n\t\t\t}\n\t\t\tjs.mu.Unlock()\n\t\t\tcca := o.consumerAssignment()\n\t\t\t// Perform the leader change in a goroutine, otherwise we could block meta operations.\n\t\t\tif o.shouldStartMonitor() {\n\t\t\t\tstarted := s.startGoRoutine(\n\t\t\t\t\tfunc() {\n\t\t\t\t\t\tdefer s.grWG.Done()\n\t\t\t\t\t\tdefer o.clearMonitorRunning()\n\t\t\t\t\t\tjs.processConsumerLeaderChangeWithAssignment(o, cca, true)\n\t\t\t\t\t},\n\t\t\t\t\tpprofLabels{\n\t\t\t\t\t\t\"type\": \"consumer\",\n\t\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t\t\t\"consumer\": ca.Name,\n\t\t\t\t\t},\n\t\t\t\t)\n\t\t\t\tif !started {\n\t\t\t\t\to.clearMonitorRunning()\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// Clustered consumer.\n\t\t\t// Start our monitoring routine if needed.\n\t\t\tif !alreadyRunning {\n\t\t\t\t// Wait for the previous routine to stop running.\n\t\t\t\to.signalMonitorQuit()\n\t\t\t\to.monitorWg.Wait()\n\t\t\t\tif o.shouldStartMonitor() {\n\t\t\t\t\tstarted := s.startGoRoutine(\n\t\t\t\t\t\tfunc() { js.monitorConsumer(o, ca) },\n\t\t\t\t\t\tpprofLabels{\n\t\t\t\t\t\t\t\"type\": \"consumer\",\n\t\t\t\t\t\t\t\"account\": mset.accName(),\n\t\t\t\t\t\t\t\"stream\": mset.name(),\n\t\t\t\t\t\t\t\"consumer\": ca.Name,\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t\tif !started {\n\t\t\t\t\t\to.clearMonitorRunning()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// For existing consumer, only send response if not recovering.\n\t\t\tif wasExisting && !js.isMetaRecovering() {\n\t\t\t\tif o.IsLeader() || (!didCreate && needsLocalResponse) {\n\t\t\t\t\t// Process if existing as an update. Double check that this is not recovered.\n\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\tclient, subject, reply, recovering := ca.Client, ca.Subject, ca.Reply, ca.recovering\n\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\tif !recovering {\n\t\t\t\t\t\tvar resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}\n\t\t\t\t\t\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.info())\n\t\t\t\t\t\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (js *jetStream) processClusterDeleteConsumer(ca *consumerAssignment, wasLeader bool) {\n\tif ca == nil {\n\t\treturn\n\t}\n\tjs.mu.RLock()\n\ts := js.srv\n\tnode := ca.Group.node\n\toffline := s.allPeersOffline(ca.Group) || ca.unsupported != nil\n\tvar isMetaLeader bool\n\tif cc := js.cluster; cc != nil {\n\t\tisMetaLeader = cc.isLeader()\n\t}\n\trecovering := ca.recovering\n\tjs.mu.RUnlock()\n\n\tstopped := false\n\tvar resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}}\n\tvar err error\n\tvar acc *Account\n\n\t// Go ahead and delete the consumer if we have it and the account.\n\tif acc, _ = s.LookupAccount(ca.Client.serviceAccount()); acc != nil {\n\t\tif mset, _ := acc.lookupStream(ca.Stream); mset != nil {\n\t\t\tif o := mset.lookupConsumer(ca.Name); o != nil {\n\t\t\t\terr = o.stopWithFlags(true, false, true, wasLeader)\n\t\t\t\tstopped = true\n\t\t\t}\n\t\t}\n\t}\n\n\t// Always delete the node if present.\n\tif node != nil {\n\t\tnode.Delete()\n\t}\n\n\t// This is a stop gap cleanup in case\n\t// 1) the account, mset, or consumer does not exist and/or\n\t// 2) node was nil (and couldn't be deleted)\n\tif !stopped || node == nil {\n\t\tif sacc := s.SystemAccount(); sacc != nil {\n\t\t\tos.RemoveAll(filepath.Join(js.config.StoreDir, sacc.GetName(), defaultStoreDirName, ca.Group.Name))\n\t\t}\n\t}\n\n\taccDir := filepath.Join(js.config.StoreDir, ca.Client.serviceAccount())\n\tconsumersDir := filepath.Join(accDir, streamsDir, ca.Stream, consumerDir)\n\tos.RemoveAll(filepath.Join(consumersDir, ca.Name))\n\n\tif !wasLeader || ca.Reply == _EMPTY_ {\n\t\tif !(offline && isMetaLeader) {\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Do not respond if the account does not exist any longer or this is during recovery.\n\tif acc == nil || recovering {\n\t\treturn\n\t}\n\n\tif err != nil {\n\t\tresp.Error = NewJSConsumerNotFoundError(Unless(err))\n\t\ts.sendAPIErrResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(resp))\n\t} else {\n\t\tresp.Success = true\n\t\ts.sendAPIResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(resp))\n\t}\n}\n\n// Returns the consumer assignment, or nil if not present.\n// Lock should be held.\nfunc (js *jetStream) consumerAssignment(account, stream, consumer string) *consumerAssignment {\n\tif sa := js.streamAssignment(account, stream); sa != nil {\n\t\treturn sa.consumers[consumer]\n\t}\n\treturn nil\n}\n\n// Will look up a consumer assignment, either an applied or inflight assignment.\n// Lock should be held.\nfunc (js *jetStream) consumerAssignmentOrInflight(account, stream, consumer string) *consumerAssignment {\n\tcc := js.cluster\n\tif cc == nil {\n\t\treturn nil\n\t}\n\tif streams, ok := cc.inflightConsumers[account]; ok {\n\t\tif consumers, ok := streams[stream]; ok {\n\t\t\tif inflight, ok := consumers[consumer]; ok {\n\t\t\t\tif !inflight.deleted {\n\t\t\t\t\treturn inflight.consumerAssignment\n\t\t\t\t} else {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif sa := js.streamAssignment(account, stream); sa != nil {\n\t\treturn sa.consumers[consumer]\n\t}\n\treturn nil\n}\n\n// Will gather all consumer assignments for the specified account and stream, both applied and inflight assignments.\n// Lock should be held.\nfunc (js *jetStream) consumerAssignmentsOrInflightSeq(account, stream string) iter.Seq[*consumerAssignment] {\n\treturn func(yield func(*consumerAssignment) bool) {\n\t\tcc := js.cluster\n\t\tif cc == nil {\n\t\t\treturn\n\t\t}\n\n\t\tvar inflight map[string]*inflightConsumerInfo\n\t\tif streams, ok := cc.inflightConsumers[account]; ok {\n\t\t\tinflight = streams[stream]\n\t\t}\n\t\tfor _, i := range inflight {\n\t\t\tif !i.deleted && !yield(i.consumerAssignment) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tsa := js.streamAssignment(account, stream)\n\t\tif sa == nil {\n\t\t\treturn\n\t\t}\n\t\tfor _, ca := range sa.consumers {\n\t\t\t// Skip if we already iterated over it as inflight.\n\t\t\tif _, ok := inflight[ca.Name]; ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !yield(ca) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// consumerAssigned informs us if this server has this consumer assigned.\nfunc (jsa *jsAccount) consumerAssigned(stream, consumer string) bool {\n\tjsa.mu.RLock()\n\tjs, acc := jsa.js, jsa.account\n\tjsa.mu.RUnlock()\n\n\tif js == nil {\n\t\treturn false\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\treturn js.cluster.isConsumerAssigned(acc, stream, consumer)\n}\n\n// Read lock should be held.\nfunc (cc *jetStreamCluster) isConsumerAssigned(a *Account, stream, consumer string) bool {\n\t// Non-clustered mode always return true.\n\tif cc == nil {\n\t\treturn true\n\t}\n\tif cc.meta == nil {\n\t\treturn false\n\t}\n\tvar sa *streamAssignment\n\taccStreams := cc.streams[a.Name]\n\tif accStreams != nil {\n\t\tsa = accStreams[stream]\n\t}\n\tif sa == nil {\n\t\t// TODO(dlc) - This should not happen.\n\t\treturn false\n\t}\n\tca := sa.consumers[consumer]\n\tif ca == nil {\n\t\treturn false\n\t}\n\treturn ca.Group.isMember(cc.meta.ID())\n}\n\n// Returns our stream and underlying raft node.\nfunc (o *consumer) streamAndNode() (*stream, RaftNode) {\n\tif o == nil {\n\t\treturn nil, nil\n\t}\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.mset, o.node\n}\n\n// Return the replica count for this consumer. If the consumer has been\n// stopped, this will return an error.\nfunc (o *consumer) replica() (int, error) {\n\to.mu.RLock()\n\toCfg := o.cfg\n\tmset := o.mset\n\to.mu.RUnlock()\n\tif mset == nil {\n\t\treturn 0, errBadConsumer\n\t}\n\tsCfg := mset.config()\n\treturn oCfg.replicas(&sCfg), nil\n}\n\nfunc (o *consumer) raftGroup() *raftGroup {\n\tif o == nil {\n\t\treturn nil\n\t}\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\tif o.ca == nil {\n\t\treturn nil\n\t}\n\treturn o.ca.Group\n}\n\nfunc (o *consumer) raftNode() RaftNode {\n\tif o == nil {\n\t\treturn nil\n\t}\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\treturn o.node\n}\n\nfunc (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) {\n\ts, n, meta := js.server(), o.raftNode(), js.getMetaGroup()\n\tdefer s.grWG.Done()\n\n\tdefer o.clearMonitorRunning()\n\n\tif n == nil || meta == nil {\n\t\ts.Warnf(\"No RAFT group for '%s > %s > %s'\", o.acc.Name, ca.Stream, ca.Name)\n\t\treturn\n\t}\n\n\t// Make sure to stop the raft group on exit to prevent accidental memory bloat.\n\t// This should be below the checkInMonitor call though to avoid stopping it out\n\t// from underneath the one that is running since it will be the same raft node.\n\tdefer n.Stop()\n\n\tqch, mqch, lch, aq, uch, ourPeerId := n.QuitC(), o.monitorQuitC(), n.LeadChangeC(), n.ApplyQ(), o.updateC(), meta.ID()\n\n\ts.Debugf(\"Starting consumer monitor for '%s > %s > %s' [%s]\", o.acc.Name, ca.Stream, ca.Name, n.Group())\n\tdefer s.Debugf(\"Exiting consumer monitor for '%s > %s > %s' [%s]\", o.acc.Name, ca.Stream, ca.Name, n.Group())\n\n\tconst (\n\t\tcompactInterval = 2 * time.Minute\n\t\tcompactMinInterval = 15 * time.Second\n\t\tcompactSizeMin = 64 * 1024 // What is stored here is always small for consumers.\n\t\tcompactNumMin = 1024\n\t\tminSnapDelta = 10 * time.Second\n\t)\n\n\t// Spread these out for large numbers on server restart.\n\trci := time.Duration(rand.Int63n(int64(time.Minute)))\n\tt := time.NewTicker(compactInterval + rci)\n\tdefer t.Stop()\n\n\t// Highwayhash key for generating hashes.\n\tkey := make([]byte, 32)\n\tcrand.Read(key)\n\n\tvar lastSnapTime time.Time\n\n\t// Don't allow the upper layer to install snapshots until we have\n\t// fully recovered from disk.\n\trecovering := true\n\n\tvar failedSnapshots int\n\tdoSnapshot := func(force bool) {\n\t\t// Bail if trying too fast and not in a forced situation.\n\t\t// If snapshots have failed, and we're not forced to, we'll wait for the timer since it'll now be forced.\n\t\tif recovering || (!force && (time.Since(lastSnapTime) < minSnapDelta || failedSnapshots > 0)) {\n\t\t\treturn\n\t\t}\n\n\t\tif snap, err := o.store.EncodedState(); err == nil {\n\t\t\t// If we had a significant number of failed snapshots, start relaxing Raft-layer checks\n\t\t\t// to force it through. We might have been catching up a peer for a long period, and this\n\t\t\t// protects our log size from growing indefinitely.\n\t\t\tforceSnapshot := failedSnapshots > 4\n\t\t\tif err := n.InstallSnapshot(snap, forceSnapshot); err == nil {\n\t\t\t\tlastSnapTime = time.Now()\n\t\t\t\t// If there was a failed snapshot before, we reduced the timer's interval.\n\t\t\t\t// Reset it back to the original interval now.\n\t\t\t\tif failedSnapshots > 0 {\n\t\t\t\t\tt.Reset(compactInterval + rci)\n\t\t\t\t}\n\t\t\t\tfailedSnapshots = 0\n\t\t\t} else if err != errNoSnapAvailable && err != errNodeClosed && err != errCatchupsRunning {\n\t\t\t\ts.RateLimitWarnf(\"Failed to install snapshot for '%s > %s > %s' [%s]: %v\", o.acc.Name, ca.Stream, ca.Name, n.Group(), err)\n\t\t\t\t// If this is the first failure, reduce the interval of the snapshot timer.\n\t\t\t\t// This ensures we're not waiting too long for snapshotting to eventually become forced.\n\t\t\t\tif failedSnapshots == 0 {\n\t\t\t\t\tt.Reset(compactMinInterval)\n\t\t\t\t}\n\t\t\t\tfailedSnapshots++\n\t\t\t}\n\t\t} else {\n\t\t\ts.RateLimitWarnf(\"Failed to install snapshot for '%s > %s > %s' [%s]: %v\", o.acc.Name, ca.Stream, ca.Name, n.Group(), err)\n\t\t}\n\t}\n\n\t// For migration tracking.\n\tvar mmt *time.Ticker\n\tvar mmtc <-chan time.Time\n\n\tstartMigrationMonitoring := func() {\n\t\tif mmt == nil {\n\t\t\tmmt = time.NewTicker(500 * time.Millisecond)\n\t\t\tmmtc = mmt.C\n\t\t}\n\t}\n\n\tstopMigrationMonitoring := func() {\n\t\tif mmt != nil {\n\t\t\tmmt.Stop()\n\t\t\tmmt, mmtc = nil, nil\n\t\t}\n\t}\n\tdefer stopMigrationMonitoring()\n\n\t// Track if we are leader.\n\tvar isLeader bool\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\t// Server shutting down, but we might receive this before qch, so try to snapshot.\n\t\t\tdoSnapshot(false)\n\t\t\treturn\n\t\tcase <-mqch:\n\t\t\t// Clean signal from shutdown routine so do best effort attempt to snapshot.\n\t\t\t// Don't snapshot if not shutting down, monitor goroutine could be going away\n\t\t\t// on a scale down or a remove for example.\n\t\t\tif s.isShuttingDown() {\n\t\t\t\tdoSnapshot(false)\n\t\t\t}\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\t// Raft node is closed, no use in trying to snapshot.\n\t\t\treturn\n\t\tcase <-aq.ch:\n\t\t\tces := aq.pop()\n\t\t\tfor _, ce := range ces {\n\t\t\t\t// No special processing needed for when we are caught up on restart.\n\t\t\t\tif ce == nil {\n\t\t\t\t\tif !recovering {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\trecovering = false\n\t\t\t\t\tif n.NeedSnapshot() {\n\t\t\t\t\t\tdoSnapshot(true)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif err := js.applyConsumerEntries(o, ce, isLeader); err == nil {\n\t\t\t\t\tvar ne, nb uint64\n\t\t\t\t\t// We can't guarantee writes are flushed while we're shutting down. Just rely on replay during recovery.\n\t\t\t\t\tif !js.isShuttingDown() {\n\t\t\t\t\t\tne, nb = n.Applied(ce.Index)\n\t\t\t\t\t}\n\t\t\t\t\t// If we have at least min entries to compact, go ahead and snapshot/compact.\n\t\t\t\t\tif nb > 0 && ne >= compactNumMin || nb > compactSizeMin {\n\t\t\t\t\t\tdoSnapshot(false)\n\t\t\t\t\t}\n\t\t\t\t} else if err != errConsumerClosed {\n\t\t\t\t\ts.Warnf(\"Error applying consumer entries to '%s > %s'\", ca.Client.serviceAccount(), ca.Name)\n\t\t\t\t}\n\t\t\t\tce.ReturnToPool()\n\t\t\t}\n\t\t\taq.recycle(&ces)\n\n\t\tcase isLeader = <-lch:\n\t\t\tif recovering && !isLeader {\n\t\t\t\tjs.setConsumerAssignmentRecovering(ca)\n\t\t\t}\n\n\t\t\t// Process the change.\n\t\t\tif err := js.processConsumerLeaderChange(o, isLeader); err == nil {\n\t\t\t\t// Check our state if we are under an interest based stream.\n\t\t\t\tif mset := o.getStream(); mset != nil {\n\t\t\t\t\tvar ss StreamState\n\t\t\t\t\tmset.store.FastState(&ss)\n\t\t\t\t\to.checkStateForInterestStream(&ss)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// We may receive a leader change after the consumer assignment which would cancel us\n\t\t\t// monitoring for this closely. So re-assess our state here as well.\n\t\t\t// Or the old leader is no longer part of the set and transferred leadership\n\t\t\t// for this leader to resume with removal\n\t\t\trg := o.raftGroup()\n\n\t\t\t// Check for migrations (peer count and replica count differ) here.\n\t\t\t// We set the state on the stream assignment update below.\n\t\t\treplicas, err := o.replica()\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif isLeader && len(rg.Peers) != replicas {\n\t\t\t\tstartMigrationMonitoring()\n\t\t\t} else {\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t}\n\t\tcase <-uch:\n\t\t\t// keep consumer assignment current\n\t\t\tca = o.consumerAssignment()\n\t\t\t// We get this when we have a new consumer assignment caused by an update.\n\t\t\t// We want to know if we are migrating.\n\t\t\trg := o.raftGroup()\n\t\t\t// If we are migrating, monitor for the new peers to be caught up.\n\t\t\treplicas, err := o.replica()\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif isLeader && len(rg.Peers) != replicas {\n\t\t\t\tstartMigrationMonitoring()\n\t\t\t} else {\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t}\n\t\tcase <-mmtc:\n\t\t\tif !isLeader {\n\t\t\t\t// We are no longer leader, so not our job.\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trg := o.raftGroup()\n\t\t\tci := js.clusterInfo(rg)\n\t\t\treplicas, err := o.replica()\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(rg.Peers) <= replicas {\n\t\t\t\t// Migration no longer happening, so not our job anymore\n\t\t\t\tstopMigrationMonitoring()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tnewPeers, _, newPeerSet, _ := genPeerInfo(rg.Peers, len(rg.Peers)-replicas)\n\n\t\t\t// If we are part of the new peerset and we have been passed the baton.\n\t\t\t// We will handle scale down.\n\t\t\tif newPeerSet[ourPeerId] {\n\t\t\t\tn.ProposeKnownPeers(newPeers)\n\t\t\t\tcca := ca.copyGroup()\n\t\t\t\tcca.Group.Peers = newPeers\n\t\t\t\tcca.Group.Cluster = s.cachedClusterName()\n\t\t\t\tmeta.ForwardProposal(encodeAddConsumerAssignment(cca))\n\t\t\t\ts.Noticef(\"Scaling down '%s > %s > %s' to %+v\", ca.Client.serviceAccount(), ca.Stream, ca.Name, s.peerSetToNames(newPeers))\n\n\t\t\t} else {\n\t\t\t\tvar newLeaderPeer, newLeader, newCluster string\n\t\t\t\tneededCurrent, current := replicas/2+1, 0\n\t\t\t\tfor _, r := range ci.Replicas {\n\t\t\t\t\tif r.Current && newPeerSet[r.Peer] {\n\t\t\t\t\t\tcurrent++\n\t\t\t\t\t\tif newCluster == _EMPTY_ {\n\t\t\t\t\t\t\tnewLeaderPeer, newLeader, newCluster = r.Peer, r.Name, r.cluster\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Check if we have a quorom\n\t\t\t\tif current >= neededCurrent {\n\t\t\t\t\ts.Noticef(\"Transfer of consumer leader for '%s > %s > %s' to '%s'\", ca.Client.serviceAccount(), ca.Stream, ca.Name, newLeader)\n\t\t\t\t\tn.StepDown(newLeaderPeer)\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase <-t.C:\n\t\t\t// Start forcing snapshots if they failed previously.\n\t\t\tforceIfFailed := failedSnapshots > 0\n\t\t\tdoSnapshot(forceIfFailed)\n\t\t}\n\t}\n}\n\nfunc (js *jetStream) applyConsumerEntries(o *consumer, ce *CommittedEntry, isLeader bool) error {\n\tfor _, e := range ce.Entries {\n\t\t// Ignore if lower-level catchup is started.\n\t\t// We don't need to optimize during this, all entries are handled as normal.\n\t\tif e.Type == EntryCatchup {\n\t\t\tcontinue\n\t\t}\n\n\t\tif e.Type == EntrySnapshot {\n\t\t\tif !isLeader {\n\t\t\t\t// No-op needed?\n\t\t\t\tstate, err := decodeConsumerState(e.Data)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif mset, node := o.streamAndNode(); mset != nil && node != nil {\n\t\t\t\t\t\ts := js.srv\n\t\t\t\t\t\ts.Errorf(\"JetStream cluster could not decode consumer snapshot for '%s > %s > %s' [%s]\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), o, node.Group())\n\t\t\t\t\t}\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\n\t\t\t\tif err = o.store.Update(state); err != nil && err != ErrStoreOldUpdate {\n\t\t\t\t\to.mu.RLock()\n\t\t\t\t\ts, acc, mset, name := o.srv, o.acc, o.mset, o.name\n\t\t\t\t\to.mu.RUnlock()\n\t\t\t\t\tif s != nil && mset != nil {\n\t\t\t\t\t\ts.Warnf(\"Consumer '%s > %s > %s' error on store update from snapshot entry: %v\", acc, mset.name(), name, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Check our interest state if applicable.\n\t\t\t\tif mset := o.getStream(); mset != nil {\n\t\t\t\t\tvar ss StreamState\n\t\t\t\t\tmset.store.FastState(&ss)\n\t\t\t\t\t// We used to register preacks here if our ack floor was higher than the last sequence.\n\t\t\t\t\t// Now when streams catch up they properly call checkInterestState() and periodically run this as well.\n\t\t\t\t\t// If our states drift this could have allocated lots of pre-acks.\n\t\t\t\t\to.checkStateForInterestStream(&ss)\n\t\t\t\t}\n\t\t\t}\n\n\t\t} else if e.Type == EntryRemovePeer {\n\t\t\tjs.mu.RLock()\n\t\t\tvar ourID string\n\t\t\tif js.cluster != nil && js.cluster.meta != nil {\n\t\t\t\tourID = js.cluster.meta.ID()\n\t\t\t}\n\t\t\tjs.mu.RUnlock()\n\t\t\tif peer := string(e.Data); peer == ourID {\n\t\t\t\tshouldRemove := true\n\t\t\t\tif mset := o.getStream(); mset != nil {\n\t\t\t\t\tif sa := mset.streamAssignment(); sa != nil && sa.Group != nil {\n\t\t\t\t\t\tjs.mu.RLock()\n\t\t\t\t\t\tshouldRemove = !sa.Group.isMember(ourID)\n\t\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif shouldRemove {\n\t\t\t\t\to.stopWithFlags(true, false, false, false)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if e.Type == EntryAddPeer {\n\t\t\t// Ignore for now.\n\t\t} else {\n\t\t\tbuf := e.Data\n\t\t\tswitch entryOp(buf[0]) {\n\t\t\tcase updateDeliveredOp:\n\t\t\t\tdseq, sseq, dc, ts, err := decodeDeliveredUpdate(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tif mset, node := o.streamAndNode(); mset != nil && node != nil {\n\t\t\t\t\t\ts := js.srv\n\t\t\t\t\t\ts.Errorf(\"JetStream cluster could not decode consumer delivered update for '%s > %s > %s' [%s]\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), o, node.Group())\n\t\t\t\t\t}\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\t\t// Make sure to update delivered under the lock.\n\t\t\t\to.mu.Lock()\n\t\t\t\terr = o.store.UpdateDelivered(dseq, sseq, dc, ts)\n\t\t\t\to.ldt = time.Now()\n\t\t\t\t// Need to send message to the client, since we have quorum to do so now.\n\t\t\t\tif pmsg, ok := o.pendingDeliveries[sseq]; ok {\n\t\t\t\t\t// Copy delivery subject and sequence first, as the send returns it to the pool and clears it.\n\t\t\t\t\tdsubj, seq := pmsg.dsubj, pmsg.seq\n\t\t\t\t\to.outq.send(pmsg)\n\t\t\t\t\tdelete(o.pendingDeliveries, sseq)\n\n\t\t\t\t\t// Might need to send a request timeout after sending the last replicated delivery.\n\t\t\t\t\tif wd, ok := o.waitingDeliveries[dsubj]; ok && wd.seq == seq {\n\t\t\t\t\t\tif wd.pn > 0 || wd.pb > 0 {\n\t\t\t\t\t\t\thdr := fmt.Appendf(nil, \"NATS/1.0 408 Request Timeout\\r\\n%s: %d\\r\\n%s: %d\\r\\n\\r\\n\", JSPullRequestPendingMsgs, wd.pn, JSPullRequestPendingBytes, wd.pb)\n\t\t\t\t\t\t\to.outq.send(newJSPubMsg(dsubj, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\t\t\t\t}\n\t\t\t\t\t\twd.recycle()\n\t\t\t\t\t\tdelete(o.waitingDeliveries, dsubj)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\t\tif err != nil {\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\tcase updateAcksOp:\n\t\t\t\tdseq, sseq, err := decodeAckUpdate(buf[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tif mset, node := o.streamAndNode(); mset != nil && node != nil {\n\t\t\t\t\t\ts := js.srv\n\t\t\t\t\t\ts.Errorf(\"JetStream cluster could not decode consumer ack update for '%s > %s > %s' [%s]\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), o, node.Group())\n\t\t\t\t\t}\n\t\t\t\t\tpanic(err.Error())\n\t\t\t\t}\n\t\t\t\tif err := o.processReplicatedAck(dseq, sseq); err == errConsumerClosed {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase updateSkipOp:\n\t\t\t\to.mu.Lock()\n\t\t\t\tvar le = binary.LittleEndian\n\t\t\t\tsseq := le.Uint64(buf[1:])\n\t\t\t\tif !o.isLeader() && sseq > o.sseq {\n\t\t\t\t\to.sseq = sseq\n\t\t\t\t}\n\t\t\t\tif o.store != nil {\n\t\t\t\t\to.store.UpdateStarting(sseq - 1)\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\tcase resetSeqOp:\n\t\t\t\to.mu.Lock()\n\t\t\t\tvar le = binary.LittleEndian\n\t\t\t\tsseq := le.Uint64(buf[1:9])\n\t\t\t\treply := string(buf[9:])\n\t\t\t\to.resetLocalStartingSeq(sseq)\n\t\t\t\tif o.store != nil {\n\t\t\t\t\to.store.Reset(sseq - 1)\n\t\t\t\t}\n\t\t\t\t// Cleanup messages that lost interest.\n\t\t\t\tif o.retention == InterestPolicy {\n\t\t\t\t\tif mset := o.mset; mset != nil {\n\t\t\t\t\t\to.mu.Unlock()\n\t\t\t\t\t\tss := mset.state()\n\t\t\t\t\t\to.checkStateForInterestStream(&ss)\n\t\t\t\t\t\to.mu.Lock()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Recalculate pending, and re-trigger message delivery.\n\t\t\t\tif !o.isLeader() {\n\t\t\t\t\to.mu.Unlock()\n\t\t\t\t} else {\n\t\t\t\t\to.streamNumPending()\n\t\t\t\t\to.signalNewMessages()\n\t\t\t\t\ts, a := o.srv, o.acc\n\t\t\t\t\to.mu.Unlock()\n\t\t\t\t\tif reply != _EMPTY_ {\n\t\t\t\t\t\tvar resp = JSApiConsumerResetResponse{ApiResponse: ApiResponse{Type: JSApiConsumerResetResponseType}}\n\t\t\t\t\t\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.info())\n\t\t\t\t\t\tresp.ResetSeq = sseq\n\t\t\t\t\t\ts.sendInternalAccountMsg(a, reply, s.jsonResponse(&resp))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase addPendingRequest:\n\t\t\t\to.mu.Lock()\n\t\t\t\tif !o.isLeader() {\n\t\t\t\t\tif o.prm == nil {\n\t\t\t\t\t\to.prm = make(map[string]struct{})\n\t\t\t\t\t}\n\t\t\t\t\to.prm[string(buf[1:])] = struct{}{}\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\tcase removePendingRequest:\n\t\t\t\to.mu.Lock()\n\t\t\t\tif !o.isLeader() {\n\t\t\t\t\tif o.prm != nil {\n\t\t\t\t\t\tdelete(o.prm, string(buf[1:]))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\to.mu.Unlock()\n\t\t\tdefault:\n\t\t\t\tpanic(fmt.Sprintf(\"JetStream Cluster Unknown group entry op type: %v\", entryOp(buf[0])))\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nvar errConsumerClosed = errors.New(\"consumer closed\")\n\nfunc (o *consumer) processReplicatedAck(dseq, sseq uint64) error {\n\to.mu.Lock()\n\t// Update activity.\n\to.lat = time.Now()\n\n\tvar sagap uint64\n\tif o.cfg.AckPolicy == AckAll {\n\t\t// Always use the store state, as o.asflr is skipped ahead already.\n\t\t// Capture before updating store.\n\t\tstate, err := o.store.BorrowState()\n\t\tif err == nil {\n\t\t\tsagap = sseq - state.AckFloor.Stream\n\t\t}\n\t}\n\n\t// Do actual ack update to store.\n\t// Always do this to have it recorded.\n\to.store.UpdateAcks(dseq, sseq)\n\n\tmset := o.mset\n\tif o.closed || mset == nil {\n\t\to.mu.Unlock()\n\t\treturn errConsumerClosed\n\t}\n\tif mset.closed.Load() {\n\t\to.mu.Unlock()\n\t\treturn errStreamClosed\n\t}\n\n\t// Check if we have a reply that was requested.\n\tif reply := o.replies[sseq]; reply != _EMPTY_ {\n\t\to.outq.sendMsg(reply, nil)\n\t\tdelete(o.replies, sseq)\n\t}\n\n\tif o.retention == LimitsPolicy {\n\t\to.mu.Unlock()\n\t\treturn nil\n\t}\n\to.mu.Unlock()\n\n\tif sagap > 1 {\n\t\t// FIXME(dlc) - This is very inefficient, will need to fix.\n\t\tfor seq := sseq; seq > sseq-sagap; seq-- {\n\t\t\tmset.ackMsg(o, seq)\n\t\t}\n\t} else {\n\t\tmset.ackMsg(o, sseq)\n\t}\n\treturn nil\n}\n\nvar errBadAckUpdate = errors.New(\"jetstream cluster bad replicated ack update\")\nvar errBadDeliveredUpdate = errors.New(\"jetstream cluster bad replicated delivered update\")\n\nfunc decodeAckUpdate(buf []byte) (dseq, sseq uint64, err error) {\n\tvar bi, n int\n\tif dseq, n = binary.Uvarint(buf); n < 0 {\n\t\treturn 0, 0, errBadAckUpdate\n\t}\n\tbi += n\n\tif sseq, n = binary.Uvarint(buf[bi:]); n < 0 {\n\t\treturn 0, 0, errBadAckUpdate\n\t}\n\treturn dseq, sseq, nil\n}\n\nfunc decodeDeliveredUpdate(buf []byte) (dseq, sseq, dc uint64, ts int64, err error) {\n\tvar bi, n int\n\tif dseq, n = binary.Uvarint(buf); n < 0 {\n\t\treturn 0, 0, 0, 0, errBadDeliveredUpdate\n\t}\n\tbi += n\n\tif sseq, n = binary.Uvarint(buf[bi:]); n < 0 {\n\t\treturn 0, 0, 0, 0, errBadDeliveredUpdate\n\t}\n\tbi += n\n\tif dc, n = binary.Uvarint(buf[bi:]); n < 0 {\n\t\treturn 0, 0, 0, 0, errBadDeliveredUpdate\n\t}\n\tbi += n\n\tif ts, n = binary.Varint(buf[bi:]); n < 0 {\n\t\treturn 0, 0, 0, 0, errBadDeliveredUpdate\n\t}\n\treturn dseq, sseq, dc, ts, nil\n}\n\nfunc (js *jetStream) processConsumerLeaderChange(o *consumer, isLeader bool) error {\n\treturn js.processConsumerLeaderChangeWithAssignment(o, nil, isLeader)\n}\n\nfunc (js *jetStream) processConsumerLeaderChangeWithAssignment(o *consumer, ca *consumerAssignment, isLeader bool) error {\n\tstepDownIfLeader := func() error {\n\t\tif node := o.raftNode(); node != nil && isLeader {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn errors.New(\"failed to update consumer leader status\")\n\t}\n\n\tif o == nil || o.isClosed() {\n\t\treturn stepDownIfLeader()\n\t}\n\n\tif ca == nil {\n\t\tca = o.consumerAssignment()\n\t}\n\tif ca == nil {\n\t\treturn stepDownIfLeader()\n\t}\n\tjs.mu.Lock()\n\ts, account, err := js.srv, ca.Client.serviceAccount(), ca.err\n\tclient, subject, reply, streamName, consumerName := ca.Client, ca.Subject, ca.Reply, ca.Stream, ca.Name\n\thasResponded := ca.responded\n\tca.responded = true\n\tjs.mu.Unlock()\n\n\tacc, _ := s.LookupAccount(account)\n\tif acc == nil {\n\t\treturn stepDownIfLeader()\n\t}\n\n\tif isLeader {\n\t\t// Only log if the consumer is replicated and/or durable.\n\t\t// Logging about R1 ephemerals, like KV watchers, is mostly noise since the leader will always be known.\n\t\to.mu.RLock()\n\t\tisReplicated, durable := o.node != nil, o.isDurable()\n\t\to.mu.RUnlock()\n\t\tif isReplicated || durable {\n\t\t\ts.Noticef(\"JetStream cluster new consumer leader for '%s > %s > %s'\", ca.Client.serviceAccount(), streamName, consumerName)\n\t\t}\n\t\ts.sendConsumerLeaderElectAdvisory(o)\n\t} else {\n\t\t// We are stepping down.\n\t\t// Make sure if we are doing so because we have lost quorum that we send the appropriate advisories.\n\t\tif node := o.raftNode(); node != nil && !node.Quorum() && time.Since(node.Created()) > 5*time.Second {\n\t\t\ts.sendConsumerLostQuorumAdvisory(o)\n\t\t}\n\t}\n\n\t// Tell consumer to switch leader status.\n\to.setLeader(isLeader)\n\n\tif !isLeader || hasResponded {\n\t\tif isLeader {\n\t\t\to.clearInitialInfo()\n\t\t}\n\t\treturn nil\n\t}\n\n\tvar resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}\n\tif err != nil {\n\t\tresp.Error = NewJSConsumerCreateError(err, Unless(err))\n\t\ts.sendAPIErrResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t} else {\n\t\tresp.ConsumerInfo = setDynamicConsumerInfoMetadata(o.initialInfo())\n\t\ts.sendAPIResponse(client, acc, subject, reply, _EMPTY_, s.jsonResponse(&resp))\n\t\to.sendCreateAdvisory()\n\t}\n\n\t// Only send a pause advisory on consumer create if we're\n\t// actually paused. The timer would have been kicked by now\n\t// by the call to o.setLeader() above.\n\to.mu.RLock()\n\tif isLeader && o.cfg.PauseUntil != nil && !o.cfg.PauseUntil.IsZero() && time.Now().Before(*o.cfg.PauseUntil) {\n\t\to.sendPauseAdvisoryLocked(&o.cfg)\n\t}\n\to.mu.RUnlock()\n\n\treturn nil\n}\n\n// Determines if we should send lost quorum advisory. We throttle these after first one.\nfunc (o *consumer) shouldSendLostQuorum() bool {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\tif time.Since(o.lqsent) >= lostQuorumAdvInterval {\n\t\to.lqsent = time.Now()\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (s *Server) sendConsumerLostQuorumAdvisory(o *consumer) {\n\tif o == nil {\n\t\treturn\n\t}\n\tnode, stream, consumer, acc := o.raftNode(), o.streamName(), o.String(), o.account()\n\tif node == nil {\n\t\treturn\n\t}\n\tif !o.shouldSendLostQuorum() {\n\t\treturn\n\t}\n\n\ts.Warnf(\"JetStream cluster consumer '%s > %s > %s' has NO quorum, stalled.\", acc.GetName(), stream, consumer)\n\n\tsubj := JSAdvisoryConsumerQuorumLostPre + \".\" + stream + \".\" + consumer\n\tadv := &JSConsumerQuorumLostAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerQuorumLostAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: stream,\n\t\tConsumer: consumer,\n\t\tReplicas: s.replicas(node),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t}\n\n\t// Send to the user's account if not the system account.\n\tif acc != s.SystemAccount() {\n\t\ts.publishAdvisory(acc, subj, adv)\n\t}\n\t// Now do system level one. Place account info in adv, and nil account means system.\n\tadv.Account = acc.GetName()\n\ts.publishAdvisory(nil, subj, adv)\n}\n\nfunc (s *Server) sendConsumerLeaderElectAdvisory(o *consumer) {\n\tif o == nil {\n\t\treturn\n\t}\n\tnode, stream, consumer, acc := o.raftNode(), o.streamName(), o.String(), o.account()\n\tif node == nil {\n\t\treturn\n\t}\n\n\tsubj := JSAdvisoryConsumerLeaderElectedPre + \".\" + stream + \".\" + consumer\n\tadv := &JSConsumerLeaderElectedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSConsumerLeaderElectedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: stream,\n\t\tConsumer: consumer,\n\t\tLeader: s.serverNameForNode(node.GroupLeader()),\n\t\tReplicas: s.replicas(node),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t}\n\n\t// Send to the user's account if not the system account.\n\tif acc != s.SystemAccount() {\n\t\ts.publishAdvisory(acc, subj, adv)\n\t}\n\t// Now do system level one. Place account info in adv, and nil account means system.\n\tadv.Account = acc.GetName()\n\ts.publishAdvisory(nil, subj, adv)\n}\n\ntype streamAssignmentResult struct {\n\tAccount string `json:\"account\"`\n\tStream string `json:\"stream\"`\n\tResponse *JSApiStreamCreateResponse `json:\"create_response,omitempty\"`\n\tRestore *JSApiStreamRestoreResponse `json:\"restore_response,omitempty\"`\n\tUpdate bool `json:\"is_update,omitempty\"`\n}\n\n// Determine if this is an insufficient resources' error type.\nfunc isInsufficientResourcesErr(resp *JSApiStreamCreateResponse) bool {\n\treturn resp != nil && resp.Error != nil && IsNatsErr(resp.Error, JSInsufficientResourcesErr, JSMemoryResourcesExceededErr, JSStorageResourcesExceededErr)\n}\n\n// Process error results of stream and consumer assignments.\n// Success will be handled by stream leader.\nfunc (js *jetStream) processStreamAssignmentResults(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tvar result streamAssignmentResult\n\tif err := json.Unmarshal(msg, &result); err != nil {\n\t\t// TODO(dlc) - log\n\t\treturn\n\t}\n\tacc, _ := js.srv.LookupAccount(result.Account)\n\tif acc == nil {\n\t\t// TODO(dlc) - log\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\ts, cc := js.srv, js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\treturn\n\t}\n\n\tif sa := js.streamAssignmentOrInflight(result.Account, result.Stream); sa != nil && !sa.reassigning {\n\t\tcanDelete := !result.Update && time.Since(sa.Created) < 5*time.Second\n\n\t\t// See if we should retry in case this cluster is full but there are others.\n\t\tif cfg, ci := sa.Config, sa.Client; cfg != nil && ci != nil && isInsufficientResourcesErr(result.Response) && canDelete {\n\t\t\t// If cluster is defined we can not retry.\n\t\t\tif cfg.Placement == nil || cfg.Placement.Cluster == _EMPTY_ {\n\t\t\t\t// If we have additional clusters to try we can retry.\n\t\t\t\t// We have already verified that ci != nil.\n\t\t\t\tif len(ci.Alternates) > 0 {\n\t\t\t\t\tif rg, err := js.createGroupForStream(ci, cfg); err != nil {\n\t\t\t\t\t\ts.Warnf(\"Retrying cluster placement for stream '%s > %s' failed due to placement error: %+v\", result.Account, result.Stream, err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif org := sa.Group; org != nil && len(org.Peers) > 0 {\n\t\t\t\t\t\t\ts.Warnf(\"Retrying cluster placement for stream '%s > %s' due to insufficient resources in cluster %q\",\n\t\t\t\t\t\t\t\tresult.Account, result.Stream, s.clusterNameForNode(org.Peers[0]))\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ts.Warnf(\"Retrying cluster placement for stream '%s > %s' due to insufficient resources\", result.Account, result.Stream)\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Pick a new preferred leader.\n\t\t\t\t\t\trg.setPreferred(s)\n\t\t\t\t\t\t// Get rid of previous attempt.\n\t\t\t\t\t\tcc.meta.Propose(encodeDeleteStreamAssignment(sa))\n\t\t\t\t\t\tcc.trackInflightStreamProposal(result.Account, sa, true)\n\t\t\t\t\t\t// Propose new.\n\t\t\t\t\t\tsa.Group, sa.err = rg, nil\n\t\t\t\t\t\tcc.meta.Propose(encodeAddStreamAssignment(sa))\n\t\t\t\t\t\tcc.trackInflightStreamProposal(result.Account, sa, false)\n\t\t\t\t\t\t// When the new stream assignment is processed, sa.reassigning will be\n\t\t\t\t\t\t// automatically set back to false. Until then, don't process any more\n\t\t\t\t\t\t// assignment results.\n\t\t\t\t\t\tsa.reassigning = true\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Respond to the user here.\n\t\tvar resp string\n\t\tif result.Response != nil {\n\t\t\tresp = s.jsonResponse(result.Response)\n\t\t} else if result.Restore != nil {\n\t\t\tresp = s.jsonResponse(result.Restore)\n\t\t}\n\t\tif !sa.responded || result.Update {\n\t\t\tsa.responded = true\n\t\t\tjs.srv.sendAPIErrResponse(sa.Client, acc, sa.Subject, sa.Reply, _EMPTY_, resp)\n\t\t}\n\t\t// Remove this assignment if possible.\n\t\tif canDelete {\n\t\t\tsa.err = NewJSClusterNotAssignedError()\n\t\t\tcc.meta.Propose(encodeDeleteStreamAssignment(sa))\n\t\t\tcc.trackInflightStreamProposal(result.Account, sa, true)\n\t\t}\n\t}\n}\n\nfunc (js *jetStream) processConsumerAssignmentResults(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tvar result consumerAssignmentResult\n\tif err := json.Unmarshal(msg, &result); err != nil {\n\t\t// TODO(dlc) - log\n\t\treturn\n\t}\n\tacc, _ := js.srv.LookupAccount(result.Account)\n\tif acc == nil {\n\t\t// TODO(dlc) - log\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\ts, cc := js.srv, js.cluster\n\tif cc == nil || cc.meta == nil {\n\t\treturn\n\t}\n\n\tif sa := js.streamAssignment(result.Account, result.Stream); sa != nil && sa.consumers != nil {\n\t\tif ca := sa.consumers[result.Consumer]; ca != nil && !ca.responded {\n\t\t\tjs.srv.sendAPIErrResponse(ca.Client, acc, ca.Subject, ca.Reply, _EMPTY_, s.jsonResponse(result.Response))\n\t\t\tca.responded = true\n\n\t\t\t// Check if this failed.\n\t\t\t// TODO(dlc) - Could have mixed results, should track per peer.\n\t\t\t// Make sure this is recent response.\n\t\t\tif result.Response.Error != nil && result.Response.Error != NewJSConsumerNameExistError() && time.Since(ca.Created) < 2*time.Second {\n\t\t\t\t// Do not list in consumer names/lists.\n\t\t\t\tca.err = NewJSClusterNotAssignedError()\n\t\t\t}\n\t\t}\n\t}\n}\n\nconst (\n\tstreamAssignmentSubj = \"$SYS.JSC.STREAM.ASSIGNMENT.RESULT\"\n\tconsumerAssignmentSubj = \"$SYS.JSC.CONSUMER.ASSIGNMENT.RESULT\"\n)\n\n// Lock should be held.\nfunc (js *jetStream) startUpdatesSub() {\n\tcc, s, c := js.cluster, js.srv, js.cluster.c\n\tif cc.streamResults == nil {\n\t\tcc.streamResults, _ = s.systemSubscribe(streamAssignmentSubj, _EMPTY_, false, c, js.processStreamAssignmentResults)\n\t}\n\tif cc.consumerResults == nil {\n\t\tcc.consumerResults, _ = s.systemSubscribe(consumerAssignmentSubj, _EMPTY_, false, c, js.processConsumerAssignmentResults)\n\t}\n\tif cc.stepdown == nil {\n\t\tcc.stepdown, _ = s.systemSubscribe(JSApiLeaderStepDown, _EMPTY_, false, c, s.jsLeaderStepDownRequest)\n\t}\n\tif cc.peerRemove == nil {\n\t\tcc.peerRemove, _ = s.systemSubscribe(JSApiRemoveServer, _EMPTY_, false, c, s.jsLeaderServerRemoveRequest)\n\t}\n\tif cc.peerStreamMove == nil {\n\t\tcc.peerStreamMove, _ = s.systemSubscribe(JSApiServerStreamMove, _EMPTY_, false, c, s.jsLeaderServerStreamMoveRequest)\n\t}\n\tif cc.peerStreamCancelMove == nil {\n\t\tcc.peerStreamCancelMove, _ = s.systemSubscribe(JSApiServerStreamCancelMove, _EMPTY_, false, c, s.jsLeaderServerStreamCancelMoveRequest)\n\t}\n\tif js.accountPurge == nil {\n\t\tjs.accountPurge, _ = s.systemSubscribe(JSApiAccountPurge, _EMPTY_, false, c, s.jsLeaderAccountPurgeRequest)\n\t}\n}\n\n// Lock should be held.\nfunc (js *jetStream) stopUpdatesSub() {\n\tcc := js.cluster\n\tif cc.streamResults != nil {\n\t\tcc.s.sysUnsubscribe(cc.streamResults)\n\t\tcc.streamResults = nil\n\t}\n\tif cc.consumerResults != nil {\n\t\tcc.s.sysUnsubscribe(cc.consumerResults)\n\t\tcc.consumerResults = nil\n\t}\n\tif cc.stepdown != nil {\n\t\tcc.s.sysUnsubscribe(cc.stepdown)\n\t\tcc.stepdown = nil\n\t}\n\tif cc.peerRemove != nil {\n\t\tcc.s.sysUnsubscribe(cc.peerRemove)\n\t\tcc.peerRemove = nil\n\t}\n\tif cc.peerStreamMove != nil {\n\t\tcc.s.sysUnsubscribe(cc.peerStreamMove)\n\t\tcc.peerStreamMove = nil\n\t}\n\tif cc.peerStreamCancelMove != nil {\n\t\tcc.s.sysUnsubscribe(cc.peerStreamCancelMove)\n\t\tcc.peerStreamCancelMove = nil\n\t}\n\tif js.accountPurge != nil {\n\t\tcc.s.sysUnsubscribe(js.accountPurge)\n\t\tjs.accountPurge = nil\n\t}\n}\n\nfunc (s *Server) sendDomainLeaderElectAdvisory() {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\tnode := cc.meta\n\tjs.mu.RUnlock()\n\n\tif node == nil {\n\t\treturn\n\t}\n\n\tadv := &JSDomainLeaderElectedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSDomainLeaderElectedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tLeader: node.GroupLeader(),\n\t\tReplicas: s.replicas(node),\n\t\tCluster: s.cachedClusterName(),\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t}\n\n\ts.publishAdvisory(nil, JSAdvisoryDomainLeaderElected, adv)\n}\n\nfunc (js *jetStream) processLeaderChange(isLeader bool) {\n\tif js == nil {\n\t\treturn\n\t}\n\ts := js.srv\n\tif s == nil {\n\t\treturn\n\t}\n\t// Update our server atomic.\n\ts.isMetaLeader.Store(isLeader)\n\n\tif isLeader {\n\t\ts.Noticef(\"Self is new JetStream cluster metadata leader\")\n\t\ts.sendDomainLeaderElectAdvisory()\n\t} else {\n\t\tvar node string\n\t\tif meta := js.getMetaGroup(); meta != nil {\n\t\t\tnode = meta.GroupLeader()\n\t\t}\n\t\tif node == _EMPTY_ {\n\t\t\ts.Noticef(\"JetStream cluster no metadata leader\")\n\t\t} else if srv := js.srv.serverNameForNode(node); srv == _EMPTY_ {\n\t\t\ts.Noticef(\"JetStream cluster new remote metadata leader\")\n\t\t} else if clst := js.srv.clusterNameForNode(node); clst == _EMPTY_ {\n\t\t\ts.Noticef(\"JetStream cluster new metadata leader: %s\", srv)\n\t\t} else {\n\t\t\ts.Noticef(\"JetStream cluster new metadata leader: %s/%s\", srv, clst)\n\t\t}\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\t// Clear replies for peer-removes.\n\tjs.cluster.peerRemoveReply = nil\n\n\t// Clear inflight proposal tracking.\n\tjs.cluster.inflightStreams = nil\n\tjs.cluster.inflightConsumers = nil\n\n\tif isLeader {\n\t\tif meta := js.cluster.meta; meta != nil && meta.IsObserver() {\n\t\t\tmeta.StepDown()\n\t\t\treturn\n\t\t}\n\t}\n\n\tif isLeader {\n\t\tjs.startUpdatesSub()\n\t} else {\n\t\tjs.stopUpdatesSub()\n\t\t// TODO(dlc) - stepdown.\n\t}\n\n\t// If we have been signaled to check the streams, this is for a bug that left stream\n\t// assignments with no sync subject after an update and no way to sync/catchup outside of the RAFT layer.\n\tif isLeader && js.cluster.streamsCheck {\n\t\tcc := js.cluster\n\t\tfor acc, sa := range js.streamAssignmentsOrInflightSeqAllAccounts() {\n\t\t\tif sa.unsupported != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif sa.Sync == _EMPTY_ {\n\t\t\t\ts.Warnf(\"Stream assignment corrupt for stream '%s > %s'\", acc, sa.Config.Name)\n\t\t\t\tnsa := &streamAssignment{Group: sa.Group, Config: sa.Config, Subject: sa.Subject, Reply: sa.Reply, Client: sa.Client, Created: sa.Created}\n\t\t\t\tnsa.Sync = syncSubjForStream()\n\t\t\t\tcc.meta.Propose(encodeUpdateStreamAssignment(nsa))\n\t\t\t\tcc.trackInflightStreamProposal(acc, nsa, false)\n\t\t\t}\n\t\t}\n\t\t// Clear check.\n\t\tcc.streamsCheck = false\n\t}\n}\n\n// Lock should be held.\nfunc (cc *jetStreamCluster) remapStreamAssignment(sa *streamAssignment, removePeer string) bool {\n\t// Invoke placement algo passing RG peers that stay (existing) and the peer that is being removed (ignore)\n\tvar retain, ignore []string\n\tfor _, v := range sa.Group.Peers {\n\t\tif v == removePeer {\n\t\t\tignore = append(ignore, v)\n\t\t} else {\n\t\t\tretain = append(retain, v)\n\t\t}\n\t}\n\n\tnewPeers, placementError := cc.selectPeerGroup(len(sa.Group.Peers), sa.Group.Cluster, sa.Config, retain, 0, ignore)\n\n\tif placementError == nil {\n\t\tsa.Group.Peers = newPeers\n\t\t// Don't influence preferred leader.\n\t\tsa.Group.Preferred = _EMPTY_\n\t\treturn true\n\t}\n\n\t// If R1 just return to avoid bricking the stream.\n\tif sa.Group.node == nil || len(sa.Group.Peers) == 1 {\n\t\treturn false\n\t}\n\n\t// If we are here let's remove the peer at least, as long as we are R>1\n\tfor i, peer := range sa.Group.Peers {\n\t\tif peer == removePeer {\n\t\t\tsa.Group.Peers[i] = sa.Group.Peers[len(sa.Group.Peers)-1]\n\t\t\tsa.Group.Peers = sa.Group.Peers[:len(sa.Group.Peers)-1]\n\t\t\tbreak\n\t\t}\n\t}\n\treturn false\n}\n\ntype selectPeerError struct {\n\texcludeTag bool\n\toffline bool\n\tnoStorage bool\n\tuniqueTag bool\n\tmisc bool\n\tnoJsClust bool\n\tnoMatchTags map[string]struct{}\n\texcludeTags map[string]struct{}\n}\n\nfunc (e *selectPeerError) Error() string {\n\tb := strings.Builder{}\n\twriteBoolErrReason := func(hasErr bool, errMsg string) {\n\t\tif !hasErr {\n\t\t\treturn\n\t\t}\n\t\tb.WriteString(\", \")\n\t\tb.WriteString(errMsg)\n\t}\n\tb.WriteString(\"no suitable peers for placement\")\n\twriteBoolErrReason(e.offline, \"peer offline\")\n\twriteBoolErrReason(e.excludeTag, \"exclude tag set\")\n\twriteBoolErrReason(e.noStorage, \"insufficient storage\")\n\twriteBoolErrReason(e.uniqueTag, \"server tag not unique\")\n\twriteBoolErrReason(e.misc, \"miscellaneous issue\")\n\twriteBoolErrReason(e.noJsClust, \"jetstream not enabled in cluster\")\n\tif len(e.noMatchTags) != 0 {\n\t\tb.WriteString(\", tags not matched [\")\n\t\tvar firstTagWritten bool\n\t\tfor tag := range e.noMatchTags {\n\t\t\tif firstTagWritten {\n\t\t\t\tb.WriteString(\", \")\n\t\t\t}\n\t\t\tfirstTagWritten = true\n\t\t\tb.WriteRune('\\'')\n\t\t\tb.WriteString(tag)\n\t\t\tb.WriteRune('\\'')\n\t\t}\n\t\tb.WriteString(\"]\")\n\t}\n\tif len(e.excludeTags) != 0 {\n\t\tb.WriteString(\", tags excluded [\")\n\t\tvar firstTagWritten bool\n\t\tfor tag := range e.excludeTags {\n\t\t\tif firstTagWritten {\n\t\t\t\tb.WriteString(\", \")\n\t\t\t}\n\t\t\tfirstTagWritten = true\n\t\t\tb.WriteRune('\\'')\n\t\t\tb.WriteString(tag)\n\t\t\tb.WriteRune('\\'')\n\t\t}\n\t\tb.WriteString(\"]\")\n\t}\n\n\treturn b.String()\n}\n\nfunc (e *selectPeerError) addMissingTag(t string) {\n\tif e.noMatchTags == nil {\n\t\te.noMatchTags = map[string]struct{}{}\n\t}\n\te.noMatchTags[t] = struct{}{}\n}\n\nfunc (e *selectPeerError) addExcludeTag(t string) {\n\tif e.excludeTags == nil {\n\t\te.excludeTags = map[string]struct{}{}\n\t}\n\te.excludeTags[t] = struct{}{}\n}\n\nfunc (e *selectPeerError) accumulate(eAdd *selectPeerError) {\n\tif eAdd == nil {\n\t\treturn\n\t}\n\tacc := func(val *bool, valAdd bool) {\n\t\tif valAdd {\n\t\t\t*val = valAdd\n\t\t}\n\t}\n\tacc(&e.offline, eAdd.offline)\n\tacc(&e.excludeTag, eAdd.excludeTag)\n\tacc(&e.noStorage, eAdd.noStorage)\n\tacc(&e.uniqueTag, eAdd.uniqueTag)\n\tacc(&e.misc, eAdd.misc)\n\tacc(&e.noJsClust, eAdd.noJsClust)\n\tfor tag := range eAdd.noMatchTags {\n\t\te.addMissingTag(tag)\n\t}\n\tfor tag := range eAdd.excludeTags {\n\t\te.addExcludeTag(tag)\n\t}\n}\n\n// selectPeerGroup will select a group of peers to start a raft group.\n// when peers exist already the unique tag prefix check for the replaceFirstExisting will be skipped\n// js lock should be held.\nfunc (cc *jetStreamCluster) selectPeerGroup(r int, cluster string, cfg *StreamConfig, existing []string, replaceFirstExisting int, ignore []string) ([]string, *selectPeerError) {\n\tif cluster == _EMPTY_ || cfg == nil {\n\t\treturn nil, &selectPeerError{misc: true}\n\t}\n\n\tvar maxBytes uint64\n\tif cfg.MaxBytes > 0 {\n\t\tmaxBytes = uint64(cfg.MaxBytes)\n\t}\n\n\t// Check for tags.\n\ttype tagInfo struct {\n\t\ttag string\n\t\texclude bool\n\t}\n\tvar ti []tagInfo\n\tif cfg.Placement != nil {\n\t\tti = make([]tagInfo, 0, len(cfg.Placement.Tags))\n\t\tfor _, t := range cfg.Placement.Tags {\n\t\t\tti = append(ti, tagInfo{\n\t\t\t\ttag: strings.TrimPrefix(t, \"!\"),\n\t\t\t\texclude: strings.HasPrefix(t, \"!\"),\n\t\t\t})\n\t\t}\n\t}\n\n\t// Used for weighted sorting based on availability.\n\ttype wn struct {\n\t\tid string\n\t\tavail uint64\n\t\toff bool\n\t\tha int\n\t\tns int\n\t}\n\n\tvar nodes []wn\n\t// peers is a randomized list\n\ts, peers := cc.s, cc.meta.Peers()\n\n\tuniqueTagPrefix := s.getOpts().JetStreamUniqueTag\n\tif uniqueTagPrefix != _EMPTY_ {\n\t\tfor _, t := range ti {\n\t\t\tif strings.HasPrefix(t.tag, uniqueTagPrefix) {\n\t\t\t\t// disable uniqueness check if explicitly listed in tags\n\t\t\t\tuniqueTagPrefix = _EMPTY_\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tvar uniqueTags = make(map[string]*nodeInfo)\n\n\tcheckUniqueTag := func(ni *nodeInfo) (bool, *nodeInfo) {\n\t\tfor _, t := range ni.tags {\n\t\t\tif strings.HasPrefix(t, uniqueTagPrefix) {\n\t\t\t\tif n, ok := uniqueTags[t]; !ok {\n\t\t\t\t\tuniqueTags[t] = ni\n\t\t\t\t\treturn true, ni\n\t\t\t\t} else {\n\t\t\t\t\treturn false, n\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// default requires the unique prefix to be present\n\t\treturn false, nil\n\t}\n\n\t// Map existing.\n\tvar ep map[string]struct{}\n\tif le := len(existing); le > 0 {\n\t\tif le >= r {\n\t\t\treturn existing[:r], nil\n\t\t}\n\t\tep = make(map[string]struct{})\n\t\tfor i, p := range existing {\n\t\t\tep[p] = struct{}{}\n\t\t\tif uniqueTagPrefix == _EMPTY_ {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsi, ok := s.nodeToInfo.Load(p)\n\t\t\tif !ok || si == nil || i < replaceFirstExisting {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tni := si.(nodeInfo)\n\t\t\t// collect unique tags, but do not require them as this node is already part of the peerset\n\t\t\tcheckUniqueTag(&ni)\n\t\t}\n\t}\n\n\t// Map ignore\n\tvar ip map[string]struct{}\n\tif li := len(ignore); li > 0 {\n\t\tip = make(map[string]struct{})\n\t\tfor _, p := range ignore {\n\t\t\tip[p] = struct{}{}\n\t\t}\n\t}\n\n\t// Grab the number of streams and HA assets currently assigned to each peer.\n\t// HAAssets under usage is async, so calculate here in realtime based on assignments.\n\tpeerStreams := make(map[string]int, len(peers))\n\tpeerHA := make(map[string]int, len(peers))\n\tfor _, asa := range cc.streams {\n\t\tfor _, sa := range asa {\n\t\t\tif sa.unsupported != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tisHA := len(sa.Group.Peers) > 1\n\t\t\tfor _, peer := range sa.Group.Peers {\n\t\t\t\tpeerStreams[peer]++\n\t\t\t\tif isHA {\n\t\t\t\t\tpeerHA[peer]++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tmaxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets\n\n\t// An error is a result of multiple individual placement decisions.\n\t// Which is why we keep taps on how often which one happened.\n\terr := selectPeerError{}\n\n\tvar onlinePeers int\n\n\t// Shuffle them up.\n\trand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] })\n\tfor _, p := range peers {\n\t\tsi, ok := s.nodeToInfo.Load(p.ID)\n\t\tif !ok || si == nil {\n\t\t\terr.misc = true\n\t\t\tcontinue\n\t\t}\n\t\tni := si.(nodeInfo)\n\t\t// Only select from the designated named cluster.\n\t\tif ni.cluster != cluster {\n\t\t\ts.Debugf(\"Peer selection: discard %s@%s reason: not target cluster %s\", ni.name, ni.cluster, cluster)\n\t\t\tcontinue\n\t\t}\n\n\t\t// If we've never heard from a server, don't consider.\n\t\tif ni.cfg == nil || ni.stats == nil {\n\t\t\ts.Debugf(\"Peer selection: discard %s@%s reason: offline\", ni.name, ni.cluster)\n\t\t\terr.offline = true\n\t\t\tcontinue\n\t\t}\n\n\t\t// If ignore skip\n\t\tif _, ok := ip[p.ID]; ok {\n\t\t\tcontinue\n\t\t}\n\n\t\t// If existing also skip, we will add back in to front of the list when done.\n\t\tif _, ok := ep[p.ID]; ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tif ni.tags.Contains(jsExcludePlacement) {\n\t\t\ts.Debugf(\"Peer selection: discard %s@%s tags: %v reason: %s present\",\n\t\t\t\tni.name, ni.cluster, ni.tags, jsExcludePlacement)\n\t\t\terr.excludeTag = true\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(ti) > 0 {\n\t\t\tmatched := true\n\t\t\tfor _, t := range ti {\n\t\t\t\tcontains := ni.tags.Contains(t.tag)\n\t\t\t\tif t.exclude && contains {\n\t\t\t\t\tmatched = false\n\t\t\t\t\ts.Debugf(\"Peer selection: discard %s@%s tags: %v reason: excluded tag %s present\",\n\t\t\t\t\t\tni.name, ni.cluster, ni.tags, t)\n\t\t\t\t\terr.addExcludeTag(t.tag)\n\t\t\t\t\tbreak\n\t\t\t\t} else if !t.exclude && !contains {\n\t\t\t\t\tmatched = false\n\t\t\t\t\ts.Debugf(\"Peer selection: discard %s@%s tags: %v reason: mandatory tag %s not present\",\n\t\t\t\t\t\tni.name, ni.cluster, ni.tags, t)\n\t\t\t\t\terr.addMissingTag(t.tag)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !matched {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tvar available uint64\n\t\tif ni.stats != nil {\n\t\t\tswitch cfg.Storage {\n\t\t\tcase MemoryStorage:\n\t\t\t\tused := ni.stats.ReservedMemory\n\t\t\t\tif ni.stats.Memory > used {\n\t\t\t\t\tused = ni.stats.Memory\n\t\t\t\t}\n\t\t\t\tif ni.cfg.MaxMemory > int64(used) {\n\t\t\t\t\tavailable = uint64(ni.cfg.MaxMemory) - used\n\t\t\t\t}\n\t\t\tcase FileStorage:\n\t\t\t\tused := ni.stats.ReservedStore\n\t\t\t\tif ni.stats.Store > used {\n\t\t\t\t\tused = ni.stats.Store\n\t\t\t\t}\n\t\t\t\tif ni.cfg.MaxStore > int64(used) {\n\t\t\t\t\tavailable = uint64(ni.cfg.MaxStore) - used\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Otherwise check if we have enough room if maxBytes set.\n\t\tif maxBytes > 0 && maxBytes > available {\n\t\t\ts.Warnf(\"Peer selection: discard %s@%s (Max Bytes: %d) exceeds available %s storage of %d bytes\",\n\t\t\t\tni.name, ni.cluster, maxBytes, cfg.Storage.String(), available)\n\t\t\terr.noStorage = true\n\t\t\tcontinue\n\t\t}\n\t\t// HAAssets contain _meta_ which we want to ignore, hence > and not >=.\n\t\tif maxHaAssets > 0 && ni.stats != nil && ni.stats.HAAssets > maxHaAssets {\n\t\t\ts.Warnf(\"Peer selection: discard %s@%s (HA Asset Count: %d) exceeds max ha asset limit of %d for stream placement\",\n\t\t\t\tni.name, ni.cluster, ni.stats.HAAssets, maxHaAssets)\n\t\t\terr.misc = true\n\t\t\tcontinue\n\t\t}\n\n\t\tif uniqueTagPrefix != _EMPTY_ {\n\t\t\tif unique, owner := checkUniqueTag(&ni); !unique {\n\t\t\t\tif owner != nil {\n\t\t\t\t\ts.Debugf(\"Peer selection: discard %s@%s tags:%v reason: unique prefix %s owned by %s@%s\",\n\t\t\t\t\t\tni.name, ni.cluster, ni.tags, owner.name, owner.cluster)\n\t\t\t\t} else {\n\t\t\t\t\ts.Debugf(\"Peer selection: discard %s@%s tags:%v reason: unique prefix %s not present\",\n\t\t\t\t\t\tni.name, ni.cluster, ni.tags)\n\t\t\t\t}\n\t\t\t\terr.uniqueTag = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// Add to our list of potential nodes.\n\t\tnodes = append(nodes, wn{p.ID, available, ni.offline, peerHA[p.ID], peerStreams[p.ID]})\n\t\tif !ni.offline {\n\t\t\tonlinePeers++\n\t\t}\n\t}\n\n\t// If we could not select enough peers, fail.\n\tquorum := r/2 + 1\n\tmissingQuorum := onlinePeers+len(existing) < quorum\n\tif missingNodes := len(nodes) < (r - len(existing)); missingNodes || missingQuorum {\n\t\tif len(peers) == 0 {\n\t\t\terr.noJsClust = true\n\t\t} else if !missingNodes && missingQuorum {\n\t\t\terr.offline = true\n\t\t}\n\t\ts.Debugf(\"Peer selection: required %d nodes but found %d (cluster: %s replica: %d existing: %v/%d peers: %d result-peers: %d err: %+v)\",\n\t\t\tr-len(existing), len(nodes), cluster, r, existing, replaceFirstExisting, len(peers), len(nodes), err)\n\t\treturn nil, &err\n\t}\n\t// Sort based on available from most to least, breaking ties by number of total streams assigned to the peer.\n\tslices.SortFunc(nodes, func(i, j wn) int {\n\t\t// Prefer online servers to offline ones.\n\t\tif i.off != j.off {\n\t\t\tif i.off {\n\t\t\t\treturn 1\n\t\t\t} else {\n\t\t\t\treturn -1\n\t\t\t}\n\t\t}\n\t\tif i.avail == j.avail {\n\t\t\treturn cmp.Compare(i.ns, j.ns)\n\t\t}\n\t\treturn -cmp.Compare(i.avail, j.avail) // reverse\n\t})\n\t// If we are placing a replicated stream, let's sort based on HAAssets, as that is more important to balance.\n\tif cfg.Replicas > 1 {\n\t\tslices.SortStableFunc(nodes, func(i, j wn) int {\n\t\t\t// Prefer online servers to offline ones.\n\t\t\tif i.off != j.off {\n\t\t\t\tif i.off {\n\t\t\t\t\treturn 1\n\t\t\t\t} else {\n\t\t\t\t\treturn -1\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn cmp.Compare(i.ha, j.ha)\n\t\t})\n\t}\n\n\tvar results []string\n\tif len(existing) > 0 {\n\t\tresults = append(results, existing...)\n\t\tr -= len(existing)\n\t}\n\tfor _, r := range nodes[:r] {\n\t\tresults = append(results, r.id)\n\t}\n\treturn results, nil\n}\n\nfunc groupNameForStream(peers []string, storage StorageType) string {\n\treturn groupName(\"S\", peers, storage)\n}\n\nfunc groupNameForConsumer(peers []string, storage StorageType) string {\n\treturn groupName(\"C\", peers, storage)\n}\n\nfunc groupName(prefix string, peers []string, storage StorageType) string {\n\tgns := getHash(nuid.Next())\n\treturn fmt.Sprintf(\"%s-R%d%s-%s\", prefix, len(peers), storage.String()[:1], gns)\n}\n\n// returns stream count for this tier as well as applicable reservation size (not including cfg)\n// jetStream read lock should be held\nfunc (js *jetStream) tieredStreamAndReservationCount(accName, tier string, cfg *StreamConfig) (int, int64) {\n\tvar numStreams int\n\tvar reservation int64\n\tfor sa := range js.streamAssignmentsOrInflightSeq(accName) {\n\t\t// Don't count the stream toward the limit if it already exists.\n\t\tif sa.Config.Name == cfg.Name {\n\t\t\tcontinue\n\t\t}\n\t\tif tier == _EMPTY_ || isSameTier(sa.Config, cfg) {\n\t\t\tnumStreams++\n\t\t\tif sa.Config.MaxBytes > 0 && sa.Config.Storage == cfg.Storage {\n\t\t\t\t// If tier is empty, all storage is flat and we should adjust for replicas.\n\t\t\t\t// Otherwise if tiered, storage replication already taken into consideration.\n\t\t\t\tif tier == _EMPTY_ && sa.Config.Replicas > 1 {\n\t\t\t\t\treservation += sa.Config.MaxBytes * int64(sa.Config.Replicas)\n\t\t\t\t} else {\n\t\t\t\t\treservation += sa.Config.MaxBytes\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn numStreams, reservation\n}\n\n// createGroupForStream will create a group for assignment for the stream.\n// Lock should be held.\nfunc (js *jetStream) createGroupForStream(ci *ClientInfo, cfg *StreamConfig) (*raftGroup, *selectPeerError) {\n\treplicas := cfg.Replicas\n\tif replicas == 0 {\n\t\treplicas = 1\n\t}\n\n\t// Default connected cluster from the request origin.\n\tcc, cluster := js.cluster, ci.Cluster\n\t// If specified, override the default.\n\tclusterDefined := cfg.Placement != nil && cfg.Placement.Cluster != _EMPTY_\n\tif clusterDefined {\n\t\tcluster = cfg.Placement.Cluster\n\t}\n\tclusters := []string{cluster}\n\tif !clusterDefined {\n\t\tclusters = append(clusters, ci.Alternates...)\n\t}\n\n\t// Need to create a group here.\n\terrs := &selectPeerError{}\n\tfor _, cn := range clusters {\n\t\tpeers, err := cc.selectPeerGroup(replicas, cn, cfg, nil, 0, nil)\n\t\tif len(peers) < replicas {\n\t\t\terrs.accumulate(err)\n\t\t\tcontinue\n\t\t}\n\t\treturn &raftGroup{Name: groupNameForStream(peers, cfg.Storage), Storage: cfg.Storage, Peers: peers, Cluster: cn}, nil\n\t}\n\treturn nil, errs\n}\n\nfunc (acc *Account) selectLimits(replicas int) (*JetStreamAccountLimits, string, *jsAccount, *ApiError) {\n\t// Grab our jetstream account info.\n\tacc.mu.RLock()\n\tjsa := acc.js\n\tacc.mu.RUnlock()\n\n\tif jsa == nil {\n\t\treturn nil, _EMPTY_, nil, NewJSNotEnabledForAccountError()\n\t}\n\n\tjsa.usageMu.RLock()\n\tselectedLimits, tierName, ok := jsa.selectLimits(replicas)\n\tjsa.usageMu.RUnlock()\n\n\tif !ok {\n\t\treturn nil, _EMPTY_, nil, NewJSNoLimitsError()\n\t}\n\treturn &selectedLimits, tierName, jsa, nil\n}\n\n// Read lock needs to be held\nfunc (js *jetStream) jsClusteredStreamLimitsCheck(acc *Account, cfg *StreamConfig) *ApiError {\n\tvar replicas int\n\tif cfg != nil {\n\t\treplicas = cfg.Replicas\n\t}\n\tselectedLimits, tier, _, apiErr := acc.selectLimits(replicas)\n\tif apiErr != nil {\n\t\treturn apiErr\n\t}\n\n\tnumStreams, reservations := js.tieredStreamAndReservationCount(acc.Name, tier, cfg)\n\tif selectedLimits.MaxStreams > 0 && numStreams >= selectedLimits.MaxStreams {\n\t\treturn NewJSMaximumStreamsLimitError()\n\t}\n\t// Check for account limits here before proposing.\n\tif err := js.checkAccountLimits(selectedLimits, cfg, reservations); err != nil {\n\t\treturn NewJSStreamLimitsError(err, Unless(err))\n\t}\n\treturn nil\n}\n\nfunc (s *Server) jsClusteredStreamRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, config *StreamConfigRequest) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamCreateResponseType}}\n\n\tccfg, apiErr := s.checkStreamCfg(&config.StreamConfig, acc, config.Pedantic)\n\tif apiErr != nil {\n\t\tresp.Error = apiErr\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tcfg := &ccfg\n\n\t// Now process the request and proposal.\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tvar self *streamAssignment\n\tvar rg *raftGroup\n\tvar syncSubject string\n\n\t// Capture if we have existing/inflight assignment first.\n\tif osa := js.streamAssignmentOrInflight(acc.Name, cfg.Name); osa != nil {\n\t\tcopyStreamMetadata(cfg, osa.Config)\n\t\t// Set the index name on both to ensure the DeepEqual works\n\t\tcurrentIName := make(map[string]struct{})\n\t\tfor _, s := range osa.Config.Sources {\n\t\t\tcurrentIName[s.iname] = struct{}{}\n\t\t}\n\t\tfor _, s := range cfg.Sources {\n\t\t\ts.setIndexName()\n\t\t\tif _, ok := currentIName[s.iname]; !ok {\n\t\t\t\ts.iname = _EMPTY_\n\t\t\t}\n\t\t}\n\t\tif !reflect.DeepEqual(osa.Config, cfg) {\n\t\t\tresp.Error = NewJSStreamNameExistError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// This is an equal assignment.\n\t\tself, rg, syncSubject = osa, osa.Group, osa.Sync\n\t}\n\n\tif cfg.Sealed {\n\t\tresp.Error = NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for create can not be sealed\"))\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Check for subject collisions here.\n\tif js.subjectsOverlap(acc.Name, cfg.Subjects, self) {\n\t\tresp.Error = NewJSStreamSubjectOverlapError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tapiErr = js.jsClusteredStreamLimitsCheck(acc, cfg)\n\t// Check for stream limits here before proposing. These need to be tracked from meta layer, not jsa.\n\tif apiErr != nil {\n\t\tresp.Error = apiErr\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Create a new one here if needed.\n\tif rg == nil {\n\t\tnrg, err := js.createGroupForStream(ci, cfg)\n\t\tif err != nil {\n\t\t\tresp.Error = NewJSClusterNoPeersError(err)\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\trg = nrg\n\t\t// Pick a preferred leader.\n\t\trg.setPreferred(s)\n\t}\n\n\tif syncSubject == _EMPTY_ {\n\t\tsyncSubject = syncSubjForStream()\n\t}\n\t// Sync subject for post snapshot sync.\n\tsa := &streamAssignment{Group: rg, Sync: syncSubject, Config: cfg, Subject: subject, Reply: reply, Client: ci, Created: time.Now().UTC()}\n\tif err := cc.meta.Propose(encodeAddStreamAssignment(sa)); err == nil {\n\t\t// On success, add this as an inflight proposal so we can apply limits\n\t\t// on concurrent create requests while this stream assignment has\n\t\t// possibly not been processed yet.\n\t\tcc.trackInflightStreamProposal(acc.Name, sa, false)\n\t}\n}\n\nvar (\n\terrReqTimeout = errors.New(\"timeout while waiting for response\")\n\terrReqSrvExit = errors.New(\"server shutdown while waiting for response\")\n)\n\n// blocking utility call to perform requests on the system account\n// returns (synchronized) v or error\nfunc sysRequest[T any](s *Server, subjFormat string, args ...any) (*T, error) {\n\tisubj := fmt.Sprintf(subjFormat, args...)\n\n\ts.mu.Lock()\n\tif s.sys == nil {\n\t\ts.mu.Unlock()\n\t\treturn nil, ErrNoSysAccount\n\t}\n\tinbox := s.newRespInbox()\n\tresults := make(chan *T, 1)\n\ts.sys.replies[inbox] = func(_ *subscription, _ *client, _ *Account, _, _ string, msg []byte) {\n\t\tvar v T\n\t\tif err := json.Unmarshal(msg, &v); err != nil {\n\t\t\ts.Warnf(\"Error unmarshalling response for request '%s':%v\", isubj, err)\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase results <- &v:\n\t\tdefault:\n\t\t\ts.Warnf(\"Failed placing request response on internal channel\")\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\ts.sendInternalMsgLocked(isubj, inbox, nil, nil)\n\n\tdefer func() {\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\tif s.sys != nil && s.sys.replies != nil {\n\t\t\tdelete(s.sys.replies, inbox)\n\t\t}\n\t}()\n\n\tttl := time.NewTimer(2 * time.Second)\n\tdefer ttl.Stop()\n\n\tselect {\n\tcase <-s.quitCh:\n\t\treturn nil, errReqSrvExit\n\tcase <-ttl.C:\n\t\treturn nil, errReqTimeout\n\tcase data := <-results:\n\t\treturn data, nil\n\t}\n}\n\nfunc (s *Server) jsClusteredStreamUpdateRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, cfg *StreamConfig, peerSet []string, pedantic bool) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\t// Now process the request and proposal.\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\tmeta := cc.meta\n\tif meta == nil {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamUpdateResponse{ApiResponse: ApiResponse{Type: JSApiStreamUpdateResponseType}}\n\n\tosa := js.streamAssignmentOrInflight(acc.Name, cfg.Name)\n\tif osa == nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Don't allow updating if all peers are offline.\n\tif s.allPeersOffline(osa.Group) {\n\t\tresp.Error = NewJSStreamOfflineError()\n\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\treturn\n\t}\n\n\t// Update asset version metadata.\n\tsetStaticStreamMetadata(cfg)\n\n\tvar newCfg *StreamConfig\n\tif jsa := js.accounts[acc.Name]; jsa != nil {\n\t\tjs.mu.Unlock()\n\t\tncfg, err := jsa.configUpdateCheck(osa.Config, cfg, s, pedantic)\n\t\tjs.mu.Lock()\n\t\tif err != nil {\n\t\t\tresp.Error = NewJSStreamUpdateError(err, Unless(err))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t} else {\n\t\t\tnewCfg = ncfg\n\t\t}\n\t} else {\n\t\tresp.Error = NewJSNotEnabledForAccountError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\t// Check for mirror changes which are not allowed.\n\t// We will allow removing the mirror config to \"promote\" the mirror to a normal stream.\n\tif newCfg.Mirror != nil && !reflect.DeepEqual(newCfg.Mirror, osa.Config.Mirror) {\n\t\tresp.Error = NewJSStreamMirrorNotUpdatableError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Check for subject collisions here.\n\tif js.subjectsOverlap(acc.Name, cfg.Subjects, osa) {\n\t\tresp.Error = NewJSStreamSubjectOverlapError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Make copy so to not change original.\n\trg := osa.copyGroup().Group\n\n\t// Check for a move request.\n\tvar isMoveRequest, isMoveCancel bool\n\tif lPeerSet := len(peerSet); lPeerSet > 0 {\n\t\tisMoveRequest = true\n\t\t// check if this is a cancellation\n\t\tif lPeerSet == osa.Config.Replicas && lPeerSet <= len(rg.Peers) {\n\t\t\tisMoveCancel = true\n\t\t\t// can only be a cancellation if the peer sets overlap as expected\n\t\t\tfor i := 0; i < lPeerSet; i++ {\n\t\t\t\tif peerSet[i] != rg.Peers[i] {\n\t\t\t\t\tisMoveCancel = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tisMoveRequest = newCfg.Placement != nil && !reflect.DeepEqual(osa.Config.Placement, newCfg.Placement)\n\t}\n\n\t// Check for replica changes.\n\tisReplicaChange := newCfg.Replicas != osa.Config.Replicas\n\n\t// We stage consumer updates and do them after the stream update.\n\tvar consumers []*consumerAssignment\n\n\t// Check if this is a move request, but no cancellation, and we are already moving this stream.\n\tif isMoveRequest && !isMoveCancel && osa.Config.Replicas != len(rg.Peers) {\n\t\t// obtain stats to include in error message\n\t\tmsg := _EMPTY_\n\t\tif s.allPeersOffline(rg) {\n\t\t\tmsg = fmt.Sprintf(\"all %d peers offline\", len(rg.Peers))\n\t\t} else {\n\t\t\t// Need to release js lock.\n\t\t\tjs.mu.Unlock()\n\t\t\tif si, err := sysRequest[StreamInfo](s, clusterStreamInfoT, ci.serviceAccount(), cfg.Name); err != nil {\n\t\t\t\tmsg = fmt.Sprintf(\"error retrieving info: %s\", err.Error())\n\t\t\t} else if si != nil {\n\t\t\t\tcurrentCount := 0\n\t\t\t\tif si.Cluster.Leader != _EMPTY_ {\n\t\t\t\t\tcurrentCount++\n\t\t\t\t}\n\t\t\t\tcombinedLag := uint64(0)\n\t\t\t\tfor _, r := range si.Cluster.Replicas {\n\t\t\t\t\tif r.Current {\n\t\t\t\t\t\tcurrentCount++\n\t\t\t\t\t}\n\t\t\t\t\tcombinedLag += r.Lag\n\t\t\t\t}\n\t\t\t\tmsg = fmt.Sprintf(\"total peers: %d, current peers: %d, combined lag: %d\",\n\t\t\t\t\tlen(rg.Peers), currentCount, combinedLag)\n\t\t\t}\n\t\t\t// Re-acquire here.\n\t\t\tjs.mu.Lock()\n\t\t}\n\t\tresp.Error = NewJSStreamMoveInProgressError(msg)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Can not move and scale at same time.\n\tif isMoveRequest && isReplicaChange {\n\t\tresp.Error = NewJSStreamMoveAndScaleError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Reset notion of scaling up, if this was done in a previous update.\n\trg.ScaleUp = false\n\tif isReplicaChange {\n\t\tisScaleUp := newCfg.Replicas > len(rg.Peers)\n\t\t// We are adding new peers here.\n\t\tif isScaleUp {\n\t\t\t// Check that we have the allocation available.\n\t\t\tif err := js.jsClusteredStreamLimitsCheck(acc, newCfg); err != nil {\n\t\t\t\tresp.Error = err\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Check if we do not have a cluster assigned, and if we do not make sure we\n\t\t\t// try to pick one. This could happen with older streams that were assigned by\n\t\t\t// previous servers.\n\t\t\tif rg.Cluster == _EMPTY_ {\n\t\t\t\t// Prefer placement directives if we have them.\n\t\t\t\tif newCfg.Placement != nil && newCfg.Placement.Cluster != _EMPTY_ {\n\t\t\t\t\trg.Cluster = newCfg.Placement.Cluster\n\t\t\t\t} else {\n\t\t\t\t\t// Fall back to the cluster assignment from the client.\n\t\t\t\t\trg.Cluster = ci.Cluster\n\t\t\t\t}\n\t\t\t}\n\t\t\tpeers, err := cc.selectPeerGroup(newCfg.Replicas, rg.Cluster, newCfg, rg.Peers, 0, nil)\n\t\t\tif err != nil {\n\t\t\t\tresp.Error = NewJSClusterNoPeersError(err)\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\tif len(peers) == 1 || osa.Config.Replicas == 1 {\n\t\t\t\trg.Name = groupNameForStream(peers, rg.Storage)\n\t\t\t}\n\t\t\tif len(rg.Peers) == 1 {\n\t\t\t\t// This is scale up from being a singleton, set preferred to that singleton.\n\t\t\t\trg.Preferred = rg.Peers[0]\n\t\t\t}\n\t\t\trg.ScaleUp = true\n\t\t\trg.Peers = peers\n\t\t} else {\n\t\t\t// We are deleting nodes here. We want to do our best to preserve the current leader.\n\t\t\t// We have support now from above that guarantees we are in our own Go routine, so can\n\t\t\t// ask for stream info from the stream leader to make sure we keep the leader in the new list.\n\t\t\tvar curLeader string\n\t\t\tif !s.allPeersOffline(rg) {\n\t\t\t\t// Need to release js lock.\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\tif si, err := sysRequest[StreamInfo](s, clusterStreamInfoT, ci.serviceAccount(), cfg.Name); err != nil {\n\t\t\t\t\ts.Warnf(\"Did not receive stream info results for '%s > %s' due to: %s\", acc, cfg.Name, err)\n\t\t\t\t} else if si != nil {\n\t\t\t\t\tif cl := si.Cluster; cl != nil && cl.Leader != _EMPTY_ {\n\t\t\t\t\t\tcurLeader = getHash(cl.Leader)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Re-acquire here.\n\t\t\t\tjs.mu.Lock()\n\t\t\t}\n\t\t\t// If we identified a leader make sure its part of the new group.\n\t\t\tselected := make([]string, 0, newCfg.Replicas)\n\n\t\t\tif curLeader != _EMPTY_ {\n\t\t\t\tselected = append(selected, curLeader)\n\t\t\t}\n\t\t\tfor _, peer := range rg.Peers {\n\t\t\t\tif len(selected) == newCfg.Replicas {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif peer == curLeader {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif si, ok := s.nodeToInfo.Load(peer); ok && si != nil {\n\t\t\t\t\tif si.(nodeInfo).offline {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tselected = append(selected, peer)\n\t\t\t\t}\n\t\t\t}\n\t\t\trg.Peers = selected\n\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\t// MUST do this, otherwise a scaleup afterward could potentially lead to inconsistencies.\n\t\t\tif len(rg.Peers) == 1 {\n\t\t\t\trg.Name = groupNameForStream(rg.Peers, rg.Storage)\n\t\t\t}\n\t\t}\n\n\t\t// Need to remap any consumers.\n\t\tfor ca := range js.consumerAssignmentsOrInflightSeq(acc.Name, osa.Config.Name) {\n\t\t\t// Legacy ephemerals are R=1 but present as R=0, so only auto-remap named consumers, or if we are downsizing the consumer peers.\n\t\t\t// If stream is interest or workqueue policy always remaps since they require peer parity with stream.\n\t\t\tnumPeers := len(ca.Group.Peers)\n\t\t\tisAutoScale := ca.Config.Replicas == 0 && (ca.Config.Durable != _EMPTY_ || ca.Config.Name != _EMPTY_)\n\t\t\tif isAutoScale || numPeers > len(rg.Peers) || cfg.Retention != LimitsPolicy {\n\t\t\t\tcca := ca.copyGroup()\n\t\t\t\t// Adjust preferred as needed.\n\t\t\t\tif numPeers == 1 && isScaleUp {\n\t\t\t\t\tcca.Group.Preferred = ca.Group.Peers[0]\n\t\t\t\t} else {\n\t\t\t\t\tcca.Group.Preferred = _EMPTY_\n\t\t\t\t}\n\t\t\t\t// Assign new peers.\n\t\t\t\tcca.Group.Peers = rg.Peers\n\t\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\t\tif len(cca.Group.Peers) == 1 || numPeers == 1 {\n\t\t\t\t\tcca.Group.Name = groupNameForConsumer(cca.Group.Peers, cca.Group.Storage)\n\t\t\t\t}\n\t\t\t\t// If the replicas was not 0 make sure it matches here.\n\t\t\t\tif cca.Config.Replicas != 0 {\n\t\t\t\t\tcca.Config.Replicas = len(rg.Peers)\n\t\t\t\t}\n\t\t\t\t// We can not propose here before the stream itself so we collect them.\n\t\t\t\tconsumers = append(consumers, cca)\n\n\t\t\t} else if !isScaleUp {\n\t\t\t\t// We decided to leave this consumer's peer group alone but we are also scaling down.\n\t\t\t\t// We need to make sure we do not have any peers that are no longer part of the stream.\n\t\t\t\t// Note we handle down scaling of a consumer above if its number of peers were > new stream peers.\n\t\t\t\tvar needReplace []string\n\t\t\t\tfor _, rp := range ca.Group.Peers {\n\t\t\t\t\t// Check if we have an orphaned peer now for this consumer.\n\t\t\t\t\tif !rg.isMember(rp) {\n\t\t\t\t\t\tneedReplace = append(needReplace, rp)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(needReplace) > 0 {\n\t\t\t\t\tnewPeers := copyStrings(rg.Peers)\n\t\t\t\t\trand.Shuffle(len(newPeers), func(i, j int) { newPeers[i], newPeers[j] = newPeers[j], newPeers[i] })\n\t\t\t\t\t// If we had a small size then the peer set, restrict to the same number.\n\t\t\t\t\tif lp := len(ca.Group.Peers); lp < len(newPeers) {\n\t\t\t\t\t\tnewPeers = newPeers[:lp]\n\t\t\t\t\t}\n\t\t\t\t\tcca := ca.copyGroup()\n\t\t\t\t\t// Assign new peers.\n\t\t\t\t\tcca.Group.Peers = newPeers\n\t\t\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\t\t\tif len(cca.Group.Peers) == 1 || numPeers == 1 {\n\t\t\t\t\t\tcca.Group.Name = groupNameForConsumer(cca.Group.Peers, cca.Group.Storage)\n\t\t\t\t\t}\n\t\t\t\t\t// If the replicas was not 0 make sure it matches here.\n\t\t\t\t\tif cca.Config.Replicas != 0 {\n\t\t\t\t\t\tcca.Config.Replicas = len(newPeers)\n\t\t\t\t\t}\n\t\t\t\t\t// Check if all peers are invalid. This can happen with R1 under replicated streams that are being scaled down.\n\t\t\t\t\tif len(needReplace) == len(ca.Group.Peers) {\n\t\t\t\t\t\t// We have to transfer state to new peers.\n\t\t\t\t\t\t// we will grab our state and attach to the new assignment.\n\t\t\t\t\t\t// TODO(dlc) - In practice we would want to make sure the consumer is paused.\n\t\t\t\t\t\t// Need to release js lock.\n\t\t\t\t\t\tjs.mu.Unlock()\n\t\t\t\t\t\tif ci, err := sysRequest[ConsumerInfo](s, clusterConsumerInfoT, acc, osa.Config.Name, ca.Name); err != nil {\n\t\t\t\t\t\t\ts.Warnf(\"Did not receive consumer info results for '%s > %s > %s' due to: %s\", acc, osa.Config.Name, ca.Name, err)\n\t\t\t\t\t\t} else if ci != nil {\n\t\t\t\t\t\t\tcca.State = &ConsumerState{\n\t\t\t\t\t\t\t\tDelivered: SequencePair{\n\t\t\t\t\t\t\t\t\tConsumer: ci.Delivered.Consumer,\n\t\t\t\t\t\t\t\t\tStream: ci.Delivered.Stream,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tAckFloor: SequencePair{\n\t\t\t\t\t\t\t\t\tConsumer: ci.AckFloor.Consumer,\n\t\t\t\t\t\t\t\t\tStream: ci.AckFloor.Stream,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Re-acquire here.\n\t\t\t\t\t\tjs.mu.Lock()\n\t\t\t\t\t}\n\t\t\t\t\t// We can not propose here before the stream itself so we collect them.\n\t\t\t\t\tconsumers = append(consumers, cca)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t} else if isMoveRequest {\n\t\tif len(peerSet) == 0 {\n\t\t\tnrg, err := js.createGroupForStream(ci, newCfg)\n\t\t\tif err != nil {\n\t\t\t\tresp.Error = NewJSClusterNoPeersError(err)\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// filter peers present in both sets\n\t\t\tfor _, peer := range rg.Peers {\n\t\t\t\tif !slices.Contains(nrg.Peers, peer) {\n\t\t\t\t\tpeerSet = append(peerSet, peer)\n\t\t\t\t}\n\t\t\t}\n\t\t\tpeerSet = append(peerSet, nrg.Peers...)\n\t\t}\n\t\tif len(rg.Peers) == 1 {\n\t\t\trg.Preferred = peerSet[0]\n\t\t}\n\t\trg.Peers = peerSet\n\n\t\tfor ca := range js.consumerAssignmentsOrInflightSeq(acc.Name, osa.Config.Name) {\n\t\t\tcca := ca.copyGroup()\n\t\t\tr := cca.Config.replicas(osa.Config)\n\t\t\t// shuffle part of cluster peer set we will be keeping\n\t\t\trandPeerSet := copyStrings(peerSet[len(peerSet)-newCfg.Replicas:])\n\t\t\trand.Shuffle(newCfg.Replicas, func(i, j int) { randPeerSet[i], randPeerSet[j] = randPeerSet[j], randPeerSet[i] })\n\t\t\t// move overlapping peers at the end of randPeerSet and keep a tally of non overlapping peers\n\t\t\tdropPeerSet := make([]string, 0, len(cca.Group.Peers))\n\t\t\tfor _, p := range cca.Group.Peers {\n\t\t\t\tfound := false\n\t\t\t\tfor i, rp := range randPeerSet {\n\t\t\t\t\tif p == rp {\n\t\t\t\t\t\trandPeerSet[i] = randPeerSet[newCfg.Replicas-1]\n\t\t\t\t\t\trandPeerSet[newCfg.Replicas-1] = p\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tdropPeerSet = append(dropPeerSet, p)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcPeerSet := randPeerSet[newCfg.Replicas-r:]\n\t\t\t// In case of a set or cancel simply assign\n\t\t\tif len(peerSet) == newCfg.Replicas {\n\t\t\t\tcca.Group.Peers = cPeerSet\n\t\t\t} else {\n\t\t\t\tcca.Group.Peers = append(dropPeerSet, cPeerSet...)\n\t\t\t}\n\t\t\t// make sure it overlaps with peers and remove if not\n\t\t\tif cca.Group.Preferred != _EMPTY_ {\n\t\t\t\tif !slices.Contains(cca.Group.Peers, cca.Group.Preferred) {\n\t\t\t\t\tcca.Group.Preferred = _EMPTY_\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We can not propose here before the stream itself so we collect them.\n\t\t\tconsumers = append(consumers, cca)\n\t\t}\n\t} else {\n\t\t// All other updates make sure no preferred is set.\n\t\trg.Preferred = _EMPTY_\n\t}\n\n\tsyncSubject := osa.Sync\n\tif syncSubject == _EMPTY_ {\n\t\tsyncSubject = syncSubjForStream()\n\t}\n\tsa := &streamAssignment{Group: rg, Sync: syncSubject, Created: osa.Created, Config: newCfg, Subject: subject, Reply: reply, Client: ci}\n\tif err := meta.Propose(encodeUpdateStreamAssignment(sa)); err != nil {\n\t\treturn\n\t}\n\tcc.trackInflightStreamProposal(acc.Name, sa, false)\n\n\t// Process any staged consumers.\n\tfor _, ca := range consumers {\n\t\tmeta.Propose(encodeAddConsumerAssignment(ca))\n\t\tcc.trackInflightConsumerProposal(acc.Name, sa.Config.Name, ca, false)\n\t}\n}\n\nfunc (s *Server) jsClusteredStreamDeleteRequest(ci *ClientInfo, acc *Account, stream, subject, reply string, rmsg []byte) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tif cc.meta == nil {\n\t\treturn\n\t}\n\n\tosa := js.streamAssignmentOrInflight(acc.Name, stream)\n\tif osa == nil {\n\t\tvar resp = JSApiStreamDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamDeleteResponseType}}\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tsa := &streamAssignment{Group: osa.Group, Config: osa.Config, Subject: subject, Reply: reply, Client: ci, Created: osa.Created}\n\tif err := cc.meta.Propose(encodeDeleteStreamAssignment(sa)); err == nil {\n\t\tcc.trackInflightStreamProposal(acc.Name, sa, true)\n\t}\n}\n\n// Process a clustered purge request.\nfunc (s *Server) jsClusteredStreamPurgeRequest(\n\tci *ClientInfo,\n\tacc *Account,\n\tmset *stream,\n\tstream, subject, reply string,\n\trmsg []byte,\n\tpreq *JSApiStreamPurgeRequest,\n) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tsa := js.streamAssignment(acc.Name, stream)\n\tif sa == nil {\n\t\tresp := JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}}\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\tif n := sa.Group.node; n != nil {\n\t\tsp := &streamPurge{Stream: stream, LastSeq: mset.state().LastSeq, Subject: subject, Reply: reply, Client: ci, Request: preq}\n\t\tn.Propose(encodeStreamPurge(sp))\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\tjs.mu.Unlock()\n\n\tif mset == nil {\n\t\treturn\n\t}\n\n\tvar resp = JSApiStreamPurgeResponse{ApiResponse: ApiResponse{Type: JSApiStreamPurgeResponseType}}\n\tpurged, err := mset.purge(preq)\n\tif err != nil {\n\t\tresp.Error = NewJSStreamGeneralError(err, Unless(err))\n\t} else {\n\t\tresp.Purged = purged\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n}\n\nfunc (s *Server) jsClusteredStreamRestoreRequest(\n\tci *ClientInfo,\n\tacc *Account,\n\treq *JSApiStreamRestoreRequest,\n\tsubject, reply string, rmsg []byte) {\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tif cc.meta == nil {\n\t\treturn\n\t}\n\n\tcfg := &req.Config\n\tresp := JSApiStreamRestoreResponse{ApiResponse: ApiResponse{Type: JSApiStreamRestoreResponseType}}\n\n\tif err := js.jsClusteredStreamLimitsCheck(acc, cfg); err != nil {\n\t\tresp.Error = err\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif sa := js.streamAssignmentOrInflight(ci.serviceAccount(), cfg.Name); sa != nil {\n\t\tresp.Error = NewJSStreamNameExistRestoreFailedError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Raft group selection and placement.\n\trg, err := js.createGroupForStream(ci, cfg)\n\tif err != nil {\n\t\tresp.Error = NewJSClusterNoPeersError(err)\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\t// Pick a preferred leader.\n\trg.setPreferred(s)\n\tsa := &streamAssignment{Group: rg, Sync: syncSubjForStream(), Config: cfg, Subject: subject, Reply: reply, Client: ci, Created: time.Now().UTC()}\n\t// Now add in our restore state and pre-select a peer to handle the actual receipt of the snapshot.\n\tsa.Restore = &req.State\n\tif err := cc.meta.Propose(encodeAddStreamAssignment(sa)); err == nil {\n\t\tcc.trackInflightStreamProposal(ci.serviceAccount(), sa, false)\n\t}\n}\n\n// Determine if all peers for this group are offline.\nfunc (s *Server) allPeersOffline(rg *raftGroup) bool {\n\tif rg == nil {\n\t\treturn false\n\t}\n\t// Check to see if this stream has any servers online to respond.\n\tfor _, peer := range rg.Peers {\n\t\tif si, ok := s.nodeToInfo.Load(peer); ok && si != nil {\n\t\t\tif !si.(nodeInfo).offline {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// This will do a scatter and gather operation for all streams for this account. This is only called from metadata leader.\n// This will be running in a separate Go routine.\nfunc (s *Server) jsClusteredStreamListRequest(acc *Account, ci *ClientInfo, filter string, offset int, subject, reply string, rmsg []byte) {\n\tdefer s.grWG.Done()\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\n\tvar streams []*streamAssignment\n\tfor _, sa := range cc.streams[acc.Name] {\n\t\tif IsNatsErr(sa.err, JSClusterNotAssignedErr) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif filter != _EMPTY_ {\n\t\t\t// These could not have subjects auto-filled in since they are raw and unprocessed.\n\t\t\tif len(sa.Config.Subjects) == 0 {\n\t\t\t\tif SubjectsCollide(filter, sa.Config.Name) {\n\t\t\t\t\tstreams = append(streams, sa)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor _, subj := range sa.Config.Subjects {\n\t\t\t\t\tif SubjectsCollide(filter, subj) {\n\t\t\t\t\t\tstreams = append(streams, sa)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tstreams = append(streams, sa)\n\t\t}\n\t}\n\n\t// Needs to be sorted for offsets etc.\n\tif len(streams) > 1 {\n\t\tslices.SortFunc(streams, func(i, j *streamAssignment) int { return cmp.Compare(i.Config.Name, j.Config.Name) })\n\t}\n\n\tscnt := len(streams)\n\tif offset > scnt {\n\t\toffset = scnt\n\t}\n\tif offset > 0 {\n\t\tstreams = streams[offset:]\n\t}\n\tif len(streams) > JSApiListLimit {\n\t\tstreams = streams[:JSApiListLimit]\n\t}\n\n\tvar resp = JSApiStreamListResponse{\n\t\tApiResponse: ApiResponse{Type: JSApiStreamListResponseType},\n\t\tStreams: make([]*StreamInfo, 0, len(streams)),\n\t}\n\n\tjs.mu.RUnlock()\n\n\tif len(streams) == 0 {\n\t\tresp.Limit = JSApiListLimit\n\t\tresp.Offset = offset\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\t// Create an inbox for our responses and send out our requests.\n\ts.mu.Lock()\n\tinbox := s.newRespInbox()\n\trc := make(chan *streamInfoClusterResponse, len(streams))\n\n\t// Store our handler.\n\ts.sys.replies[inbox] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {\n\t\tvar si streamInfoClusterResponse\n\t\tif err := json.Unmarshal(msg, &si); err != nil {\n\t\t\ts.Warnf(\"Error unmarshalling clustered stream info response:%v\", err)\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase rc <- &si:\n\t\tdefault:\n\t\t\ts.Warnf(\"Failed placing remote stream info result on internal channel\")\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\t// Cleanup after.\n\tdefer func() {\n\t\ts.mu.Lock()\n\t\tif s.sys != nil && s.sys.replies != nil {\n\t\t\tdelete(s.sys.replies, inbox)\n\t\t}\n\t\ts.mu.Unlock()\n\t}()\n\n\tvar missingNames []string\n\tsent := map[string]int{}\n\n\t// Send out our requests here.\n\tjs.mu.RLock()\n\tfor _, sa := range streams {\n\t\tif s.allPeersOffline(sa.Group) {\n\t\t\t// Place offline onto our results by hand here.\n\t\t\tsi := &StreamInfo{\n\t\t\t\tConfig: *sa.Config,\n\t\t\t\tCreated: sa.Created,\n\t\t\t\tCluster: js.offlineClusterInfo(sa.Group),\n\t\t\t\tTimeStamp: time.Now().UTC(),\n\t\t\t}\n\t\t\tresp.Streams = append(resp.Streams, si)\n\t\t\tmissingNames = append(missingNames, sa.Config.Name)\n\t\t} else {\n\t\t\tisubj := fmt.Sprintf(clusterStreamInfoT, sa.Client.serviceAccount(), sa.Config.Name)\n\t\t\ts.sendInternalMsgLocked(isubj, inbox, nil, nil)\n\t\t\tsent[sa.Config.Name] = len(sa.consumers)\n\t\t}\n\t}\n\t// Don't hold lock.\n\tjs.mu.RUnlock()\n\n\tconst timeout = 4 * time.Second\n\tnotActive := time.NewTimer(timeout)\n\tdefer notActive.Stop()\n\nLOOP:\n\tfor len(sent) > 0 {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-notActive.C:\n\t\t\ts.Warnf(\"Did not receive all stream info results for %q\", acc)\n\t\t\tfor sName := range sent {\n\t\t\t\tmissingNames = append(missingNames, sName)\n\t\t\t}\n\t\t\tbreak LOOP\n\t\tcase si := <-rc:\n\t\t\tconsCount := sent[si.Config.Name]\n\t\t\tif consCount > 0 {\n\t\t\t\tsi.State.Consumers = consCount\n\t\t\t}\n\t\t\tdelete(sent, si.Config.Name)\n\t\t\tif si.OfflineReason == _EMPTY_ {\n\t\t\t\tresp.Streams = append(resp.Streams, &si.StreamInfo)\n\t\t\t} else if _, ok := resp.Offline[si.Config.Name]; !ok {\n\t\t\t\tif resp.Offline == nil {\n\t\t\t\t\tresp.Offline = make(map[string]string, 1)\n\t\t\t\t}\n\t\t\t\tresp.Offline[si.Config.Name] = si.OfflineReason\n\t\t\t\tmissingNames = append(missingNames, si.Config.Name)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Needs to be sorted as well.\n\tif len(resp.Streams) > 1 {\n\t\tslices.SortFunc(resp.Streams, func(i, j *StreamInfo) int { return cmp.Compare(i.Config.Name, j.Config.Name) })\n\t}\n\n\tresp.Total = scnt\n\tresp.Limit = JSApiListLimit\n\tresp.Offset = offset\n\tresp.Missing = missingNames\n\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n}\n\n// This will do a scatter and gather operation for all consumers for this stream and account.\n// This will be running in a separate Go routine.\nfunc (s *Server) jsClusteredConsumerListRequest(acc *Account, ci *ClientInfo, offset int, stream, subject, reply string, rmsg []byte) {\n\tdefer s.grWG.Done()\n\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.RLock()\n\n\tvar consumers []*consumerAssignment\n\tif sas := cc.streams[acc.Name]; sas != nil {\n\t\tif sa := sas[stream]; sa != nil {\n\t\t\t// Copy over since we need to sort etc.\n\t\t\tfor _, ca := range sa.consumers {\n\t\t\t\tconsumers = append(consumers, ca)\n\t\t\t}\n\t\t}\n\t}\n\t// Needs to be sorted.\n\tif len(consumers) > 1 {\n\t\tslices.SortFunc(consumers, func(i, j *consumerAssignment) int { return cmp.Compare(i.Config.Name, j.Config.Name) })\n\t}\n\n\tocnt := len(consumers)\n\tif offset > ocnt {\n\t\toffset = ocnt\n\t}\n\tif offset > 0 {\n\t\tconsumers = consumers[offset:]\n\t}\n\tif len(consumers) > JSApiListLimit {\n\t\tconsumers = consumers[:JSApiListLimit]\n\t}\n\n\t// Send out our requests here.\n\tvar resp = JSApiConsumerListResponse{\n\t\tApiResponse: ApiResponse{Type: JSApiConsumerListResponseType},\n\t\tConsumers: []*ConsumerInfo{},\n\t}\n\n\tjs.mu.RUnlock()\n\n\tif len(consumers) == 0 {\n\t\tresp.Limit = JSApiListLimit\n\t\tresp.Offset = offset\n\t\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n\t\treturn\n\t}\n\n\t// Create an inbox for our responses and send out requests.\n\ts.mu.Lock()\n\tinbox := s.newRespInbox()\n\trc := make(chan *consumerInfoClusterResponse, len(consumers))\n\n\t// Store our handler.\n\ts.sys.replies[inbox] = func(sub *subscription, _ *client, _ *Account, subject, _ string, msg []byte) {\n\t\tvar ci consumerInfoClusterResponse\n\t\tif err := json.Unmarshal(msg, &ci); err != nil {\n\t\t\ts.Warnf(\"Error unmarshaling clustered consumer info response:%v\", err)\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase rc <- &ci:\n\t\tdefault:\n\t\t\ts.Warnf(\"Failed placing consumer info result on internal chan\")\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\t// Cleanup after.\n\tdefer func() {\n\t\ts.mu.Lock()\n\t\tif s.sys != nil && s.sys.replies != nil {\n\t\t\tdelete(s.sys.replies, inbox)\n\t\t}\n\t\ts.mu.Unlock()\n\t}()\n\n\tvar missingNames []string\n\tsent := map[string]struct{}{}\n\n\t// Send out our requests here.\n\tjs.mu.RLock()\n\tfor _, ca := range consumers {\n\t\tif s.allPeersOffline(ca.Group) {\n\t\t\t// Place offline onto our results by hand here.\n\t\t\tci := &ConsumerInfo{\n\t\t\t\tConfig: ca.Config,\n\t\t\t\tCreated: ca.Created,\n\t\t\t\tCluster: js.offlineClusterInfo(ca.Group),\n\t\t\t\tTimeStamp: time.Now().UTC(),\n\t\t\t}\n\t\t\tresp.Consumers = append(resp.Consumers, ci)\n\t\t\tmissingNames = append(missingNames, ca.Name)\n\t\t} else {\n\t\t\tisubj := fmt.Sprintf(clusterConsumerInfoT, ca.Client.serviceAccount(), stream, ca.Name)\n\t\t\ts.sendInternalMsgLocked(isubj, inbox, nil, nil)\n\t\t\tsent[ca.Name] = struct{}{}\n\t\t}\n\t}\n\t// Don't hold lock.\n\tjs.mu.RUnlock()\n\n\tconst timeout = 4 * time.Second\n\tnotActive := time.NewTimer(timeout)\n\tdefer notActive.Stop()\n\nLOOP:\n\tfor len(sent) > 0 {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-notActive.C:\n\t\t\ts.Warnf(\"Did not receive all consumer info results for '%s > %s'\", acc, stream)\n\t\t\tfor cName := range sent {\n\t\t\t\tmissingNames = append(missingNames, cName)\n\t\t\t}\n\t\t\tbreak LOOP\n\t\tcase ci := <-rc:\n\t\t\tdelete(sent, ci.Name)\n\t\t\tif ci.OfflineReason == _EMPTY_ {\n\t\t\t\tresp.Consumers = append(resp.Consumers, &ci.ConsumerInfo)\n\t\t\t} else if _, ok := resp.Offline[ci.Name]; !ok {\n\t\t\t\tif resp.Offline == nil {\n\t\t\t\t\tresp.Offline = make(map[string]string, 1)\n\t\t\t\t}\n\t\t\t\tresp.Offline[ci.Name] = ci.OfflineReason\n\t\t\t\tmissingNames = append(missingNames, ci.Name)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Needs to be sorted as well.\n\tif len(resp.Consumers) > 1 {\n\t\tslices.SortFunc(resp.Consumers, func(i, j *ConsumerInfo) int { return cmp.Compare(i.Name, j.Name) })\n\t}\n\n\tresp.Total = ocnt\n\tresp.Limit = JSApiListLimit\n\tresp.Offset = offset\n\tresp.Missing = missingNames\n\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n}\n\nfunc encodeStreamPurge(sp *streamPurge) []byte {\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(purgeStreamOp))\n\tjson.NewEncoder(&bb).Encode(sp)\n\treturn bb.Bytes()\n}\n\nfunc decodeStreamPurge(buf []byte) (*streamPurge, error) {\n\tvar sp streamPurge\n\terr := json.Unmarshal(buf, &sp)\n\treturn &sp, err\n}\n\nfunc (s *Server) jsClusteredConsumerDeleteRequest(ci *ClientInfo, acc *Account, stream, consumer, subject, reply string, rmsg []byte) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tif cc.meta == nil {\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerDeleteResponse{ApiResponse: ApiResponse{Type: JSApiConsumerDeleteResponseType}}\n\n\tsa := js.streamAssignment(acc.Name, stream)\n\tif sa == nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\n\t}\n\tif sa.consumers == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\toca := sa.consumers[consumer]\n\tif oca == nil {\n\t\tresp.Error = NewJSConsumerNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tca := &consumerAssignment{Group: oca.Group, Stream: stream, Name: consumer, Config: oca.Config, Subject: subject, Reply: reply, Client: ci, Created: oca.Created}\n\tif err := cc.meta.Propose(encodeDeleteConsumerAssignment(ca)); err == nil {\n\t\tcc.trackInflightConsumerProposal(acc.Name, stream, ca, true)\n\t}\n}\n\nfunc encodeMsgDelete(md *streamMsgDelete) []byte {\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(deleteMsgOp))\n\tjson.NewEncoder(&bb).Encode(md)\n\treturn bb.Bytes()\n}\n\nfunc decodeMsgDelete(buf []byte) (*streamMsgDelete, error) {\n\tvar md streamMsgDelete\n\terr := json.Unmarshal(buf, &md)\n\treturn &md, err\n}\n\nfunc (s *Server) jsClusteredMsgDeleteRequest(ci *ClientInfo, acc *Account, mset *stream, stream, subject, reply string, req *JSApiMsgDeleteRequest, rmsg []byte) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tsa := js.streamAssignment(acc.Name, stream)\n\tif sa == nil {\n\t\ts.Debugf(\"Message delete failed, could not locate stream '%s > %s'\", acc.Name, stream)\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\n\t// Check for single replica items.\n\tif n := sa.Group.node; n != nil {\n\t\tmd := streamMsgDelete{Seq: req.Seq, NoErase: req.NoErase, Stream: stream, Subject: subject, Reply: reply, Client: ci}\n\t\tn.Propose(encodeMsgDelete(&md))\n\t\tjs.mu.Unlock()\n\t\treturn\n\t}\n\tjs.mu.Unlock()\n\n\tif mset == nil {\n\t\treturn\n\t}\n\n\tvar err error\n\tvar removed bool\n\tif req.NoErase {\n\t\tremoved, err = mset.removeMsg(req.Seq)\n\t} else {\n\t\tremoved, err = mset.eraseMsg(req.Seq)\n\t}\n\tvar resp = JSApiMsgDeleteResponse{ApiResponse: ApiResponse{Type: JSApiMsgDeleteResponseType}}\n\tif err != nil {\n\t\tresp.Error = NewJSStreamMsgDeleteFailedError(err, Unless(err))\n\t} else if !removed {\n\t\tresp.Error = NewJSSequenceNotFoundError(req.Seq)\n\t} else {\n\t\tresp.Success = true\n\t}\n\ts.sendAPIResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(resp))\n}\n\nfunc encodeAddStreamAssignment(sa *streamAssignment) []byte {\n\tcsa := *sa\n\tcsa.Client = csa.Client.forProposal()\n\tcsa.ConfigJSON, _ = json.Marshal(sa.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(assignStreamOp))\n\tjson.NewEncoder(&bb).Encode(csa)\n\treturn bb.Bytes()\n}\n\nfunc encodeUpdateStreamAssignment(sa *streamAssignment) []byte {\n\tcsa := *sa\n\tcsa.Client = csa.Client.forProposal()\n\tcsa.ConfigJSON, _ = json.Marshal(sa.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(updateStreamOp))\n\tjson.NewEncoder(&bb).Encode(csa)\n\treturn bb.Bytes()\n}\n\nfunc encodeDeleteStreamAssignment(sa *streamAssignment) []byte {\n\tcsa := *sa\n\tcsa.Client = csa.Client.forProposal()\n\tcsa.ConfigJSON, _ = json.Marshal(sa.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(removeStreamOp))\n\tjson.NewEncoder(&bb).Encode(csa)\n\treturn bb.Bytes()\n}\n\nfunc decodeStreamAssignment(s *Server, buf []byte) (*streamAssignment, error) {\n\tvar sa streamAssignment\n\tif err := json.Unmarshal(buf, &sa); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := decodeStreamAssignmentConfig(s, &sa); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &sa, nil\n}\n\nfunc decodeStreamAssignmentConfig(s *Server, sa *streamAssignment) error {\n\tvar unsupported bool\n\tvar cfg StreamConfig\n\tvar err error\n\tdecoder := json.NewDecoder(bytes.NewReader(sa.ConfigJSON))\n\tdecoder.DisallowUnknownFields()\n\tif err = decoder.Decode(&cfg); err != nil {\n\t\tunsupported = true\n\t\tcfg = StreamConfig{}\n\t\tif err2 := json.Unmarshal(sa.ConfigJSON, &cfg); err2 != nil {\n\t\t\treturn err2\n\t\t}\n\t}\n\tsa.Config = &cfg\n\tfixCfgMirrorWithDedupWindow(sa.Config)\n\n\tif unsupported || (sa.Config != nil && !supportsRequiredApiLevel(sa.Config.Metadata)) {\n\t\tsa.unsupported = newUnsupportedStreamAssignment(s, sa, err)\n\t}\n\treturn nil\n}\n\nfunc encodeDeleteRange(dr *DeleteRange) []byte {\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(deleteRangeOp))\n\tjson.NewEncoder(&bb).Encode(dr)\n\treturn bb.Bytes()\n}\n\nfunc decodeDeleteRange(buf []byte) (*DeleteRange, error) {\n\tvar dr DeleteRange\n\terr := json.Unmarshal(buf, &dr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &dr, err\n}\n\n// createGroupForConsumer will create a new group from same peer set as the stream.\nfunc (cc *jetStreamCluster) createGroupForConsumer(cfg *ConsumerConfig, sa *streamAssignment) *raftGroup {\n\tif len(sa.Group.Peers) == 0 || cfg.Replicas > len(sa.Group.Peers) {\n\t\treturn nil\n\t}\n\n\treplicas := cfg.replicas(sa.Config)\n\tpeers := copyStrings(sa.Group.Peers)\n\tvar _ss [5]string\n\tactive := _ss[:0]\n\n\t// Calculate all active peers.\n\tfor _, peer := range peers {\n\t\tif sir, ok := cc.s.nodeToInfo.Load(peer); ok && sir != nil {\n\t\t\tif !sir.(nodeInfo).offline {\n\t\t\t\tactive = append(active, peer)\n\t\t\t}\n\t\t}\n\t}\n\tif quorum := replicas/2 + 1; quorum > len(active) {\n\t\t// Not enough active to satisfy the request.\n\t\treturn nil\n\t}\n\n\t// If we want less then our parent stream, select from active.\n\tif replicas > 0 && replicas < len(peers) {\n\t\t// Pedantic in case stream is say R5 and consumer is R3 and 3 or more offline, etc.\n\t\tif len(active) < replicas {\n\t\t\treturn nil\n\t\t}\n\t\t// First shuffle the active peers and then select to account for replica = 1.\n\t\trand.Shuffle(len(active), func(i, j int) { active[i], active[j] = active[j], active[i] })\n\t\tpeers = active[:replicas]\n\t}\n\tstorage := sa.Config.Storage\n\tif cfg.MemoryStorage {\n\t\tstorage = MemoryStorage\n\t}\n\treturn &raftGroup{Name: groupNameForConsumer(peers, storage), Storage: storage, Peers: peers}\n}\n\n// jsClusteredConsumerRequest is first point of entry to create a consumer in clustered mode.\nfunc (s *Server) jsClusteredConsumerRequest(ci *ClientInfo, acc *Account, subject, reply string, rmsg []byte, stream string, cfg *ConsumerConfig, action ConsumerAction, pedantic bool) {\n\tjs, cc := s.getJetStreamCluster()\n\tif js == nil || cc == nil {\n\t\treturn\n\t}\n\n\tvar resp = JSApiConsumerCreateResponse{ApiResponse: ApiResponse{Type: JSApiConsumerCreateResponseType}}\n\n\tstreamCfg, ok := js.clusterStreamConfig(acc.Name, stream)\n\tif !ok {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tselectedLimits, _, _, apiErr := acc.selectLimits(cfg.replicas(&streamCfg))\n\tif apiErr != nil {\n\t\tresp.Error = apiErr\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\tsrvLim := &s.getOpts().JetStreamLimits\n\t// Make sure we have sane defaults\n\tif err := setConsumerConfigDefaults(cfg, &streamCfg, srvLim, selectedLimits, pedantic); err != nil {\n\t\tresp.Error = err\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tif err := checkConsumerCfg(cfg, srvLim, &streamCfg, acc, selectedLimits, false); err != nil {\n\t\tresp.Error = err\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\tjs.mu.Lock()\n\tdefer js.mu.Unlock()\n\n\tif cc.meta == nil {\n\t\treturn\n\t}\n\n\t// Lookup the stream assignment.\n\tsa := js.streamAssignmentOrInflight(acc.Name, stream)\n\tif sa == nil {\n\t\tresp.Error = NewJSStreamNotFoundError()\n\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\treturn\n\t}\n\n\t// Was a consumer name provided?\n\tvar oname string\n\tif isDurableConsumer(cfg) || cfg.Name != _EMPTY_ {\n\t\tif cfg.Name != _EMPTY_ {\n\t\t\toname = cfg.Name\n\t\t} else {\n\t\t\toname = cfg.Durable\n\t\t}\n\t}\n\n\t// Check for max consumers here to short circuit if possible.\n\t// Start with limit on a stream, but if one is defined at the level of the account\n\t// and is lower, use that limit.\n\tif action == ActionCreate || action == ActionCreateOrUpdate {\n\t\tmaxc := sa.Config.MaxConsumers\n\t\tif maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) {\n\t\t\tmaxc = selectedLimits.MaxConsumers\n\t\t}\n\t\tif maxc > 0 {\n\t\t\t// If the consumer name is specified and we think it already exists, then\n\t\t\t// we're likely updating an existing consumer, so don't count it. Otherwise\n\t\t\t// we will incorrectly return NewJSMaximumConsumersLimitError for an update.\n\t\t\tif oname == _EMPTY_ || js.consumerAssignmentOrInflight(acc.Name, stream, oname) == nil {\n\t\t\t\t// Don't count DIRECTS.\n\t\t\t\ttotal := 0\n\t\t\t\tfor ca := range js.consumerAssignmentsOrInflightSeq(acc.Name, stream) {\n\t\t\t\t\tif ca.unsupported != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif ca.Config != nil && !ca.Config.Direct {\n\t\t\t\t\t\ttotal++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif total >= maxc {\n\t\t\t\t\tresp.Error = NewJSMaximumConsumersLimitError()\n\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Also short circuit if DeliverLastPerSubject is set with no FilterSubject.\n\tif cfg.DeliverPolicy == DeliverLastPerSubject {\n\t\tif cfg.FilterSubject == _EMPTY_ && len(cfg.FilterSubjects) == 0 {\n\t\t\tresp.Error = NewJSConsumerInvalidPolicyError(fmt.Errorf(\"consumer delivery policy is deliver last per subject, but FilterSubject is not set\"))\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Setup proper default for ack wait if we are in explicit ack mode.\n\tif cfg.AckWait == 0 && (cfg.AckPolicy == AckExplicit || cfg.AckPolicy == AckAll) {\n\t\tcfg.AckWait = JsAckWaitDefault\n\t}\n\t// Setup default of -1, meaning no limit for MaxDeliver.\n\tif cfg.MaxDeliver == 0 {\n\t\tcfg.MaxDeliver = -1\n\t}\n\t// Set proper default for max ack pending if we are ack explicit and none has been set.\n\tif cfg.AckPolicy == AckExplicit && cfg.MaxAckPending == 0 {\n\t\tcfg.MaxAckPending = JsDefaultMaxAckPending\n\t}\n\n\tif cfg.PriorityPolicy == PriorityPinnedClient && cfg.PinnedTTL == 0 {\n\t\tcfg.PinnedTTL = JsDefaultPinnedTTL\n\t}\n\n\tvar ca *consumerAssignment\n\t// See if we have an existing one already under same durable name or\n\t// if name was set by the user.\n\tif oname != _EMPTY_ {\n\t\tif ca = js.consumerAssignmentOrInflight(acc.Name, stream, oname); ca != nil {\n\t\t\t// Provided config might miss metadata, copy from existing config.\n\t\t\tcopyConsumerMetadata(cfg, ca.Config)\n\n\t\t\tif action == ActionCreate && !reflect.DeepEqual(cfg, ca.Config) {\n\t\t\t\tresp.Error = NewJSConsumerAlreadyExistsError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Do quick sanity check on new cfg to prevent here if possible.\n\t\t\tif err := acc.checkNewConsumerConfig(ca.Config, cfg); err != nil {\n\t\t\t\tresp.Error = NewJSConsumerCreateError(err, Unless(err))\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Don't allow updating if all peers are offline.\n\t\t\tif s.allPeersOffline(ca.Group) {\n\t\t\t\tresp.Error = NewJSConsumerOfflineError()\n\t\t\t\ts.sendDelayedAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp), nil, errRespDelay)\n\t\t\t\treturn\n\t\t\t}\n\t\t} else {\n\t\t\t// Initialize/update asset version metadata.\n\t\t\t// First time creating this consumer, or updating.\n\t\t\tsetStaticConsumerMetadata(cfg)\n\t\t}\n\t}\n\n\t// Initialize/update asset version metadata.\n\t// But only if we're not creating, should only update it the first time\n\t// to be idempotent with versions where there's no versioning metadata.\n\tif action != ActionCreate {\n\t\tsetStaticConsumerMetadata(cfg)\n\t}\n\n\t// If this is new consumer.\n\tif ca == nil {\n\t\tif action == ActionUpdate {\n\t\t\tresp.Error = NewJSConsumerDoesNotExistError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\trg := cc.createGroupForConsumer(cfg, sa)\n\t\tif rg == nil {\n\t\t\tresp.Error = NewJSInsufficientResourcesError()\n\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\treturn\n\t\t}\n\t\t// Pick a preferred leader.\n\t\trg.setPreferred(s)\n\n\t\t// Inherit cluster from stream.\n\t\trg.Cluster = sa.Group.Cluster\n\n\t\t// We need to set the ephemeral here before replicating.\n\t\tif !isDurableConsumer(cfg) {\n\t\t\tif cfg.Name != _EMPTY_ {\n\t\t\t\toname = cfg.Name\n\t\t\t} else {\n\t\t\t\t// Make sure name is unique.\n\t\t\t\tfor {\n\t\t\t\t\toname = createConsumerName()\n\t\t\t\t\tif js.consumerAssignmentOrInflight(acc.Name, stream, oname) != nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(rg.Peers) > 1 {\n\t\t\tif maxHaAssets := s.getOpts().JetStreamLimits.MaxHAAssets; maxHaAssets != 0 {\n\t\t\t\tfor _, peer := range rg.Peers {\n\t\t\t\t\tif ni, ok := s.nodeToInfo.Load(peer); ok {\n\t\t\t\t\t\tni := ni.(nodeInfo)\n\t\t\t\t\t\tif stats := ni.stats; stats != nil && stats.HAAssets > maxHaAssets {\n\t\t\t\t\t\t\tresp.Error = NewJSInsufficientResourcesError()\n\t\t\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\t\t\t\ts.Warnf(\"%s@%s (HA Asset Count: %d) exceeds max ha asset limit of %d\"+\n\t\t\t\t\t\t\t\t\" for (durable) consumer %s placement on stream %s\",\n\t\t\t\t\t\t\t\tni.name, ni.cluster, ni.stats.HAAssets, maxHaAssets, oname, stream)\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Check if we are work queue policy.\n\t\t// We will do pre-checks here to avoid thrashing meta layer.\n\t\tif sa.Config.Retention == WorkQueuePolicy && !cfg.Direct {\n\t\t\tif cfg.AckPolicy != AckExplicit {\n\t\t\t\tresp.Error = NewJSConsumerWQRequiresExplicitAckError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\tsubjects := gatherSubjectFilters(cfg.FilterSubject, cfg.FilterSubjects)\n\t\t\tif len(subjects) == 0 && len(sa.consumers) > 0 {\n\t\t\t\tresp.Error = NewJSConsumerWQMultipleUnfilteredError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor oca := range js.consumerAssignmentsOrInflightSeq(acc.Name, stream) {\n\t\t\t\tif oca.Name == oname {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor _, psubj := range gatherSubjectFilters(oca.Config.FilterSubject, oca.Config.FilterSubjects) {\n\t\t\t\t\tfor _, subj := range subjects {\n\t\t\t\t\t\tif SubjectsCollide(subj, psubj) {\n\t\t\t\t\t\t\tresp.Error = NewJSConsumerWQConsumerNotUniqueError()\n\t\t\t\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tca = &consumerAssignment{\n\t\t\tGroup: rg,\n\t\t\tStream: stream,\n\t\t\tName: oname,\n\t\t\tConfig: cfg,\n\t\t\tSubject: subject,\n\t\t\tReply: reply,\n\t\t\tClient: ci,\n\t\t\tCreated: time.Now().UTC(),\n\t\t}\n\t} else {\n\t\t// If the consumer already exists then don't allow updating the PauseUntil, just set\n\t\t// it back to whatever the current configured value is.\n\t\tcfg.PauseUntil = ca.Config.PauseUntil\n\n\t\tnca := ca.copyGroup()\n\n\t\t// Reset notion of scaling up, if this was done in a previous update.\n\t\tnca.Group.ScaleUp = false\n\n\t\trBefore := nca.Config.replicas(sa.Config)\n\t\trAfter := cfg.replicas(sa.Config)\n\n\t\tvar curLeader string\n\t\tif rBefore != rAfter {\n\t\t\t// We are modifying nodes here. We want to do our best to preserve the current leader.\n\t\t\t// We have support now from above that guarantees we are in our own Go routine, so can\n\t\t\t// ask for stream info from the stream leader to make sure we keep the leader in the new list.\n\t\t\tif !s.allPeersOffline(ca.Group) {\n\t\t\t\t// Need to release js lock.\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\tif ci, err := sysRequest[ConsumerInfo](s, clusterConsumerInfoT, ci.serviceAccount(), sa.Config.Name, oname); err != nil {\n\t\t\t\t\ts.Warnf(\"Did not receive consumer info results for '%s > %s > %s' due to: %s\", acc, sa.Config.Name, oname, err)\n\t\t\t\t} else if ci != nil {\n\t\t\t\t\tif cl := ci.Cluster; cl != nil {\n\t\t\t\t\t\tcurLeader = getHash(cl.Leader)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Re-acquire here.\n\t\t\t\tjs.mu.Lock()\n\t\t\t}\n\t\t}\n\n\t\tif rBefore < rAfter {\n\t\t\tnewPeerSet := nca.Group.Peers\n\t\t\t// Scale up by adding new members from the stream peer set that are not yet in the consumer peer set.\n\t\t\tstreamPeerSet := copyStrings(sa.Group.Peers)\n\n\t\t\t// Respond with error when there is a config mismatch between the intended config and expected peer size.\n\t\t\tif len(streamPeerSet) < rAfter {\n\t\t\t\tresp.Error = NewJSConsumerReplicasExceedsStreamError()\n\t\t\t\ts.sendAPIErrResponse(ci, acc, subject, reply, string(rmsg), s.jsonResponse(&resp))\n\t\t\t\treturn\n\t\t\t}\n\t\t\trand.Shuffle(rAfter, func(i, j int) { streamPeerSet[i], streamPeerSet[j] = streamPeerSet[j], streamPeerSet[i] })\n\t\t\tfor _, p := range streamPeerSet {\n\t\t\t\tfound := false\n\t\t\t\tfor _, sp := range newPeerSet {\n\t\t\t\t\tif sp == p {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tnewPeerSet = append(newPeerSet, p)\n\t\t\t\t\tif len(newPeerSet) == rAfter {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\tif rBefore == 1 {\n\t\t\t\tnca.Group.Name = groupNameForConsumer(newPeerSet, nca.Group.Storage)\n\t\t\t}\n\t\t\tnca.Group.Peers = newPeerSet\n\t\t\tnca.Group.Preferred = curLeader\n\t\t\tnca.Group.ScaleUp = true\n\t\t} else if rBefore > rAfter {\n\t\t\tnewPeerSet := nca.Group.Peers\n\t\t\t// mark leader preferred and move it to end\n\t\t\tnca.Group.Preferred = curLeader\n\t\t\tif nca.Group.Preferred != _EMPTY_ {\n\t\t\t\tfor i, p := range newPeerSet {\n\t\t\t\t\tif nca.Group.Preferred == p {\n\t\t\t\t\t\tnewPeerSet[i] = newPeerSet[len(newPeerSet)-1]\n\t\t\t\t\t\tnewPeerSet[len(newPeerSet)-1] = p\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// scale down by removing peers from the end\n\t\t\tnewPeerSet = newPeerSet[len(newPeerSet)-rAfter:]\n\t\t\tnca.Group.Peers = newPeerSet\n\t\t\t// Single nodes are not recorded by the NRG layer so we can rename.\n\t\t\t// MUST do this, otherwise a scaleup afterward could potentially lead to inconsistencies.\n\t\t\tif len(nca.Group.Peers) == 1 {\n\t\t\t\tnca.Group.Name = groupNameForConsumer(nca.Group.Peers, nca.Group.Storage)\n\t\t\t}\n\t\t}\n\n\t\t// Update config and client info on copy of existing.\n\t\tnca.Config = cfg\n\t\tnca.Client = ci\n\t\tnca.Subject = subject\n\t\tnca.Reply = reply\n\t\tca = nca\n\t}\n\n\t// Do formal proposal.\n\tif err := cc.meta.Propose(encodeAddConsumerAssignment(ca)); err == nil {\n\t\tcc.trackInflightConsumerProposal(acc.Name, stream, ca, false)\n\t}\n}\n\nfunc encodeAddConsumerAssignment(ca *consumerAssignment) []byte {\n\tcca := *ca\n\tcca.Client = cca.Client.forProposal()\n\tcca.ConfigJSON, _ = json.Marshal(ca.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(assignConsumerOp))\n\tjson.NewEncoder(&bb).Encode(cca)\n\treturn bb.Bytes()\n}\n\nfunc encodeDeleteConsumerAssignment(ca *consumerAssignment) []byte {\n\tcca := *ca\n\tcca.Client = cca.Client.forProposal()\n\tcca.ConfigJSON, _ = json.Marshal(ca.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(removeConsumerOp))\n\tjson.NewEncoder(&bb).Encode(cca)\n\treturn bb.Bytes()\n}\n\nfunc decodeConsumerAssignment(buf []byte) (*consumerAssignment, error) {\n\tvar ca consumerAssignment\n\tif err := json.Unmarshal(buf, &ca); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := decodeConsumerAssignmentConfig(&ca); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &ca, nil\n}\n\nfunc decodeConsumerAssignmentConfig(ca *consumerAssignment) error {\n\tvar unsupported bool\n\tvar cfg ConsumerConfig\n\tvar err error\n\tdecoder := json.NewDecoder(bytes.NewReader(ca.ConfigJSON))\n\tdecoder.DisallowUnknownFields()\n\tif err = decoder.Decode(&cfg); err != nil {\n\t\tunsupported = true\n\t\tcfg = ConsumerConfig{}\n\t\tif err2 := json.Unmarshal(ca.ConfigJSON, &cfg); err2 != nil {\n\t\t\treturn err2\n\t\t}\n\t}\n\tca.Config = &cfg\n\tif unsupported || (ca.Config != nil && !supportsRequiredApiLevel(ca.Config.Metadata)) {\n\t\tca.unsupported = newUnsupportedConsumerAssignment(ca, err)\n\t}\n\treturn nil\n}\n\nfunc encodeAddConsumerAssignmentCompressed(ca *consumerAssignment) []byte {\n\tcca := *ca\n\tcca.Client = cca.Client.forProposal()\n\tcca.ConfigJSON, _ = json.Marshal(ca.Config)\n\tvar bb bytes.Buffer\n\tbb.WriteByte(byte(assignCompressedConsumerOp))\n\ts2e := s2.NewWriter(&bb)\n\tjson.NewEncoder(s2e).Encode(cca)\n\ts2e.Close()\n\treturn bb.Bytes()\n}\n\nfunc decodeConsumerAssignmentCompressed(buf []byte) (*consumerAssignment, error) {\n\tvar ca consumerAssignment\n\tbb := bytes.NewBuffer(buf)\n\ts2d := s2.NewReader(bb)\n\tdecoder := json.NewDecoder(s2d)\n\tif err := decoder.Decode(&ca); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := decodeConsumerAssignmentConfig(&ca); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &ca, nil\n}\n\nvar errBadStreamMsg = errors.New(\"jetstream cluster bad replicated stream msg\")\n\nfunc decodeStreamMsg(buf []byte) (subject, reply string, hdr, msg []byte, lseq uint64, ts int64, sourced bool, err error) {\n\tvar le = binary.LittleEndian\n\tif len(buf) < 26 {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\tlseq = le.Uint64(buf)\n\tbuf = buf[8:]\n\tts = int64(le.Uint64(buf))\n\tbuf = buf[8:]\n\tsl := int(le.Uint16(buf))\n\tbuf = buf[2:]\n\tif len(buf) < sl {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\tsubject = string(buf[:sl])\n\tbuf = buf[sl:]\n\tif len(buf) < 2 {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\trl := int(le.Uint16(buf))\n\tbuf = buf[2:]\n\tif len(buf) < rl {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\treply = string(buf[:rl])\n\tbuf = buf[rl:]\n\tif len(buf) < 2 {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\thl := int(le.Uint16(buf))\n\tbuf = buf[2:]\n\tif len(buf) < hl {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\tif hdr = buf[:hl]; len(hdr) == 0 {\n\t\thdr = nil\n\t}\n\tbuf = buf[hl:]\n\tif len(buf) < 4 {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\tml := int(le.Uint32(buf))\n\tbuf = buf[4:]\n\tif len(buf) < ml {\n\t\treturn _EMPTY_, _EMPTY_, nil, nil, 0, 0, false, errBadStreamMsg\n\t}\n\tif msg = buf[:ml]; len(msg) == 0 {\n\t\tmsg = nil\n\t}\n\tbuf = buf[ml:]\n\tif len(buf) > 0 {\n\t\tflags, _ := binary.Uvarint(buf)\n\t\tsourced = flags&msgFlagFromSourceOrMirror != 0\n\t}\n\treturn subject, reply, hdr, msg, lseq, ts, sourced, nil\n}\n\nfunc decodeBatchMsg(buf []byte) (batchId string, batchSeq uint64, op entryOp, mbuf []byte, err error) {\n\tvar le = binary.LittleEndian\n\tif len(buf) < 2 {\n\t\treturn _EMPTY_, 0, 0, nil, errBadStreamMsg\n\t}\n\tbl := int(le.Uint16(buf))\n\tbuf = buf[2:]\n\tif len(buf) < bl {\n\t\treturn _EMPTY_, 0, 0, nil, errBadStreamMsg\n\t}\n\tbatchId = string(buf[:bl])\n\tbuf = buf[bl:]\n\tvar n int\n\tbatchSeq, n = binary.Uvarint(buf)\n\tif n <= 0 {\n\t\treturn _EMPTY_, 0, 0, nil, errBadStreamMsg\n\t}\n\tbuf = buf[n:]\n\top = entryOp(buf[0])\n\tmbuf = buf[1:]\n\treturn batchId, batchSeq, op, mbuf, nil\n}\n\n// Flags for encodeStreamMsg/decodeStreamMsg.\nconst (\n\tmsgFlagFromSourceOrMirror uint64 = 1 << iota\n)\n\nfunc encodeStreamMsg(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, sourced bool) []byte {\n\treturn encodeStreamMsgAllowCompress(subject, reply, hdr, msg, lseq, ts, sourced)\n}\n\nfunc encodeStreamMsgAllowCompress(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, sourced bool) []byte {\n\treturn encodeStreamMsgAllowCompressAndBatch(subject, reply, hdr, msg, lseq, ts, sourced, _EMPTY_, 0, false)\n}\n\n// Threshold for compression.\n// TODO(dlc) - Eventually make configurable.\nconst compressThreshold = 8192 // 8k\n\n// If allowed and contents over the threshold we will compress.\nfunc encodeStreamMsgAllowCompressAndBatch(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, sourced bool, batchId string, batchSeq uint64, batchCommit bool) []byte {\n\t// Clip the subject, reply, header and msgs down. Operate on\n\t// uint64 lengths to avoid overflowing.\n\tslen := min(uint64(len(subject)), math.MaxUint16)\n\trlen := min(uint64(len(reply)), math.MaxUint16)\n\thlen := min(uint64(len(hdr)), math.MaxUint16)\n\tmlen := min(uint64(len(msg)), math.MaxUint32)\n\ttotal := slen + rlen + hlen + mlen\n\n\tshouldCompress := total > compressThreshold\n\telen := int(1 + 8 + 8 + total)\n\telen += (2 + 2 + 2 + 4 + 8) // Encoded lengths, 4bytes, flags are up to 8 bytes\n\n\tblen := min(uint64(len(batchId)), math.MaxUint16)\n\tif batchId != _EMPTY_ {\n\t\telen += int(2 + blen + 8) // length of batchId, batchId itself, batchSeq (up to 8 bytes)\n\t}\n\n\tvar flags uint64\n\tif sourced {\n\t\tflags |= msgFlagFromSourceOrMirror\n\t}\n\n\tvar le = binary.LittleEndian\n\tvar opIndex int\n\tbuf := make([]byte, 1, elen)\n\tif batchId != _EMPTY_ {\n\t\tif batchCommit {\n\t\t\tbuf[0] = byte(batchCommitMsgOp)\n\t\t} else {\n\t\t\tbuf[0] = byte(batchMsgOp)\n\t\t}\n\t\tbuf = le.AppendUint16(buf, uint16(blen))\n\t\tbuf = append(buf, batchId[:blen]...)\n\t\tbuf = binary.AppendUvarint(buf, batchSeq)\n\t\topIndex = len(buf)\n\t\tbuf = append(buf, byte(streamMsgOp))\n\t} else {\n\t\tbuf[opIndex] = byte(streamMsgOp)\n\t}\n\n\tbuf = le.AppendUint64(buf, lseq)\n\tbuf = le.AppendUint64(buf, uint64(ts))\n\tbuf = le.AppendUint16(buf, uint16(slen))\n\tbuf = append(buf, subject[:slen]...)\n\tbuf = le.AppendUint16(buf, uint16(rlen))\n\tbuf = append(buf, reply[:rlen]...)\n\tbuf = le.AppendUint16(buf, uint16(hlen))\n\tbuf = append(buf, hdr[:hlen]...)\n\tbuf = le.AppendUint32(buf, uint32(mlen))\n\tbuf = append(buf, msg[:mlen]...)\n\tbuf = binary.AppendUvarint(buf, flags)\n\n\t// Check if we should compress.\n\tif shouldCompress {\n\t\tnbuf := make([]byte, s2.MaxEncodedLen(elen))\n\t\tif opIndex > 0 {\n\t\t\tcopy(nbuf[:opIndex], buf[:opIndex])\n\t\t}\n\t\tnbuf[opIndex] = byte(compressedStreamMsgOp)\n\t\tebuf := s2.Encode(nbuf[opIndex+1:], buf[opIndex+1:])\n\t\t// Only pay the cost of decode on the other side if we compressed.\n\t\t// S2 will allow us to try without major penalty for non-compressable data.\n\t\tif len(ebuf) < len(buf) {\n\t\t\tbuf = nbuf[:len(ebuf)+opIndex+1]\n\t\t}\n\t}\n\n\treturn buf\n}\n\n// Determine if all peers in our set support the binary snapshot.\nfunc (mset *stream) supportsBinarySnapshot() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.supportsBinarySnapshotLocked()\n}\n\n// Determine if all peers in our set support the binary snapshot.\n// Lock should be held.\nfunc (mset *stream) supportsBinarySnapshotLocked() bool {\n\ts, n := mset.srv, mset.node\n\tif s == nil || n == nil {\n\t\treturn false\n\t}\n\t// Grab our peers and walk them to make sure we can all support binary stream snapshots.\n\tid, peers := n.ID(), n.Peers()\n\tfor _, p := range peers {\n\t\tif p.ID == id {\n\t\t\t// We know we support ourselves.\n\t\t\tcontinue\n\t\t}\n\t\t// Since release 2.10.16 only deny if we know the other node does not support.\n\t\tif sir, ok := s.nodeToInfo.Load(p.ID); ok && sir != nil && !sir.(nodeInfo).binarySnapshots {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// StreamSnapshot is used for snapshotting and out of band catch up in clustered mode.\n// Legacy, replace with binary stream snapshots.\ntype streamSnapshot struct {\n\tMsgs uint64 `json:\"messages\"`\n\tBytes uint64 `json:\"bytes\"`\n\tFirstSeq uint64 `json:\"first_seq\"`\n\tLastSeq uint64 `json:\"last_seq\"`\n\tFailed uint64 `json:\"clfs\"`\n\tDeleted []uint64 `json:\"deleted,omitempty\"`\n}\n\n// Grab a snapshot of a stream for clustered mode.\nfunc (mset *stream) stateSnapshot() []byte {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.stateSnapshotLocked()\n}\n\n// Grab a snapshot of a stream for clustered mode.\n// Lock should be held.\nfunc (mset *stream) stateSnapshotLocked() []byte {\n\t// Decide if we can support the new style of stream snapshots.\n\tif mset.supportsBinarySnapshotLocked() {\n\t\tsnap, err := mset.store.EncodedStreamState(mset.getCLFS())\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn snap\n\t}\n\n\t// Older v1 version with deleted as a sorted []uint64.\n\t// For a stream with millions or billions of interior deletes, this will be huge.\n\t// Now that all server versions 2.10.+ support binary snapshots, we should never fall back.\n\tassert.Unreachable(\"Legacy JSON stream snapshot used\", map[string]any{\n\t\t\"stream\": mset.cfg.Name,\n\t\t\"account\": mset.acc.Name,\n\t})\n\n\tstate := mset.store.State()\n\tsnap := &streamSnapshot{\n\t\tMsgs: state.Msgs,\n\t\tBytes: state.Bytes,\n\t\tFirstSeq: state.FirstSeq,\n\t\tLastSeq: state.LastSeq,\n\t\tFailed: mset.getCLFS(),\n\t\tDeleted: state.Deleted,\n\t}\n\tb, _ := json.Marshal(snap)\n\treturn b\n}\n\n// To warn when we are getting too far behind from what has been proposed vs what has been committed.\nconst streamLagWarnThreshold = 10_000\n\n// processClusteredInboundMsg will propose the inbound message to the underlying raft group.\nfunc (mset *stream) processClusteredInboundMsg(subject, reply string, hdr, msg []byte, mt *msgTrace, sourced bool) (retErr error) {\n\t// For possible error response.\n\tvar response []byte\n\n\tmset.mu.RLock()\n\tcanRespond := !mset.cfg.NoAck && len(reply) > 0\n\tname, stype := mset.cfg.Name, mset.cfg.Storage\n\tdiscard, discardNewPer, maxMsgs, maxMsgsPer, maxBytes := mset.cfg.Discard, mset.cfg.DiscardNewPer, mset.cfg.MaxMsgs, mset.cfg.MaxMsgsPer, mset.cfg.MaxBytes\n\ts, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node\n\tmaxMsgSize, lseq := int(mset.cfg.MaxMsgSize), mset.lseq\n\tisLeader, isSealed, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules := mset.isLeader(), mset.cfg.Sealed, mset.cfg.AllowRollup, mset.cfg.DenyPurge, mset.cfg.AllowMsgTTL, mset.cfg.AllowMsgCounter, mset.cfg.AllowMsgSchedules\n\tmset.mu.RUnlock()\n\n\t// This should not happen but possible now that we allow scale up, and scale down where this could trigger.\n\t//\n\t// We also invoke this in clustering mode for message tracing when not\n\t// performing message delivery.\n\tif node == nil || mt.traceOnly() {\n\t\treturn mset.processJetStreamMsg(subject, reply, hdr, msg, 0, 0, mt, sourced, true)\n\t}\n\n\t// If message tracing (with message delivery), we will need to send the\n\t// event on exit in case there was an error (if message was not proposed).\n\t// Otherwise, the event will be sent from processJetStreamMsg when\n\t// invoked by the leader (from applyStreamEntries).\n\tif mt != nil {\n\t\tdefer func() {\n\t\t\tif retErr != nil {\n\t\t\t\tmt.sendEventFromJetStream(retErr)\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Check that we are the leader. This can be false if we have scaled up from an R1 that had inbound queued messages.\n\tif !isLeader {\n\t\treturn NewJSClusterNotLeaderError()\n\t}\n\n\t// Bail here if sealed.\n\tif isSealed {\n\t\tvar resp = JSPubAckResponse{PubAck: &PubAck{Stream: mset.name()}, Error: NewJSStreamSealedError()}\n\t\tb, _ := json.Marshal(resp)\n\t\tmset.outq.sendMsg(reply, b)\n\t\treturn NewJSStreamSealedError()\n\t}\n\n\t// Check here pre-emptively if we have exceeded this server limits.\n\tif js.limitsExceeded(stype) {\n\t\ts.resourcesExceededError(stype)\n\t\tif canRespond {\n\t\t\tb, _ := json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: name}, Error: NewJSInsufficientResourcesError()})\n\t\t\toutq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, b, nil, 0))\n\t\t}\n\t\t// Stepdown regardless.\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn NewJSInsufficientResourcesError()\n\t}\n\n\t// Check here pre-emptively if we have exceeded our account limits.\n\tif exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded {\n\t\tif err == nil {\n\t\t\terr = NewJSAccountResourcesExceededError()\n\t\t}\n\t\ts.RateLimitWarnf(\"JetStream account limits exceeded for '%s': %s\", jsa.acc().GetName(), err.Error())\n\t\tif canRespond {\n\t\t\tvar resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}}\n\t\t\tresp.Error = err\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0))\n\t\t}\n\t\treturn err\n\t}\n\n\t// Check msgSize if we have a limit set there. Again this works if it goes through but better to be pre-emptive.\n\t// Subtract to prevent against overflows.\n\tif maxMsgSize >= 0 && (len(hdr) > maxMsgSize || len(msg) > maxMsgSize-len(hdr)) {\n\t\terr := fmt.Errorf(\"JetStream message size exceeds limits for '%s > %s'\", jsa.acc().Name, mset.cfg.Name)\n\t\ts.RateLimitWarnf(\"%s\", err.Error())\n\t\tif canRespond {\n\t\t\tvar resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}}\n\t\t\tresp.Error = NewJSStreamMessageExceedsMaximumError()\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0))\n\t\t}\n\t\treturn err\n\t}\n\n\t// Proceed with proposing this message.\n\n\t// We only use mset.clseq for clustering and in case we run ahead of actual commits.\n\t// Check if we need to set initial value here\n\tmset.clMu.Lock()\n\tif mset.clseq == 0 || mset.clseq < lseq+mset.clfs {\n\t\tlseq = recalculateClusteredSeq(mset)\n\t}\n\n\tvar (\n\t\tdseq uint64\n\t\tapiErr *ApiError\n\t\terr error\n\t)\n\tdiff := &batchStagedDiff{}\n\tif hdr, msg, dseq, apiErr, err = checkMsgHeadersPreClusteredProposal(diff, mset, subject, hdr, msg, sourced, name, jsa, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules, discard, discardNewPer, maxMsgSize, maxMsgs, maxMsgsPer, maxBytes); err != nil {\n\t\tmset.clMu.Unlock()\n\t\tif err == errMsgIdDuplicate && dseq > 0 {\n\t\t\tvar buf [256]byte\n\t\t\tpubAck := append(buf[:0], mset.pubAck...)\n\t\t\tresponse = append(pubAck, strconv.FormatUint(dseq, 10)...)\n\t\t\tresponse = append(response, \",\\\"duplicate\\\": true}\"...)\n\t\t\toutq.sendMsg(reply, response)\n\t\t\treturn err\n\t\t}\n\t\tif canRespond {\n\t\t\tvar resp = &JSPubAckResponse{PubAck: &PubAck{Stream: name}}\n\t\t\tresp.Error = apiErr\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\treturn err\n\t}\n\n\tcommitSingleMsg(diff, mset, subject, reply, hdr, msg, name, jsa, mt, node, r, lseq)\n\tmset.clMu.Unlock()\n\treturn nil\n}\n\nfunc (mset *stream) getAndDeleteMsgTrace(lseq uint64) *msgTrace {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\tmset.clMu.Lock()\n\tmt, ok := mset.mt[lseq]\n\tif ok {\n\t\tdelete(mset.mt, lseq)\n\t}\n\tmset.clMu.Unlock()\n\treturn mt\n}\n\n// For requesting messages post raft snapshot to catch up streams post server restart.\n// Any deleted msgs etc will be handled inline on catchup.\ntype streamSyncRequest struct {\n\tPeer string `json:\"peer,omitempty\"`\n\tFirstSeq uint64 `json:\"first_seq\"`\n\tLastSeq uint64 `json:\"last_seq\"`\n\tDeleteRangesOk bool `json:\"delete_ranges\"`\n\tMinApplied uint64 `json:\"min_applied\"`\n}\n\n// Given a stream state that represents a snapshot, calculate the sync request based on our current state.\n// Stream lock must be held.\nfunc (mset *stream) calculateSyncRequest(state *StreamState, snap *StreamReplicatedState, index uint64) *streamSyncRequest {\n\t// Shouldn't happen, but consequences are pretty bad if we have the lock held and\n\t// our caller tries to take the lock again on panic defer, as in processSnapshot.\n\tif state == nil || snap == nil || mset.node == nil {\n\t\treturn nil\n\t}\n\t// Quick check if we are already caught up.\n\tif state.LastSeq >= snap.LastSeq {\n\t\treturn nil\n\t}\n\treturn &streamSyncRequest{FirstSeq: state.LastSeq + 1, LastSeq: snap.LastSeq, Peer: mset.node.ID(), DeleteRangesOk: true, MinApplied: index}\n}\n\n// processSnapshotDeletes will update our current store based on the snapshot\n// but only processing deletes and new FirstSeq / purges.\nfunc (mset *stream) processSnapshotDeletes(snap *StreamReplicatedState) error {\n\tmset.mu.Lock()\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\t// Always adjust if FirstSeq has moved beyond our state.\n\tvar didReset bool\n\tif snap.FirstSeq > state.FirstSeq {\n\t\tif _, err := mset.store.Compact(snap.FirstSeq); err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t\tmset.store.FastState(&state)\n\t\tmset.lseq = state.LastSeq\n\t\tmset.clearAllPreAcksBelowFloor(state.FirstSeq)\n\t\tdidReset = true\n\t}\n\ts := mset.srv\n\tmset.mu.Unlock()\n\n\tif didReset {\n\t\ts.Warnf(\"Catchup for stream '%s > %s' resetting first sequence: %d on catchup request\",\n\t\t\tmset.account(), mset.name(), snap.FirstSeq)\n\t}\n\n\tif len(snap.Deleted) > 0 {\n\t\treturn mset.store.SyncDeleted(snap.Deleted)\n\t}\n\treturn nil\n}\n\nfunc (mset *stream) setCatchupPeer(peer string, lag uint64) {\n\tif peer == _EMPTY_ {\n\t\treturn\n\t}\n\tmset.mu.Lock()\n\tif mset.catchups == nil {\n\t\tmset.catchups = make(map[string]uint64)\n\t}\n\tmset.catchups[peer] = lag\n\tmset.mu.Unlock()\n}\n\n// Will decrement by one.\nfunc (mset *stream) updateCatchupPeer(peer string) {\n\tif peer == _EMPTY_ {\n\t\treturn\n\t}\n\tmset.mu.Lock()\n\tif lag := mset.catchups[peer]; lag > 0 {\n\t\tmset.catchups[peer] = lag - 1\n\t}\n\tmset.mu.Unlock()\n}\n\nfunc (mset *stream) decrementCatchupPeer(peer string, num uint64) {\n\tif peer == _EMPTY_ {\n\t\treturn\n\t}\n\tmset.mu.Lock()\n\tif lag := mset.catchups[peer]; lag > 0 {\n\t\tif lag >= num {\n\t\t\tlag -= num\n\t\t} else {\n\t\t\tlag = 0\n\t\t}\n\t\tmset.catchups[peer] = lag\n\t}\n\tmset.mu.Unlock()\n}\n\nfunc (mset *stream) clearCatchupPeer(peer string) {\n\tmset.mu.Lock()\n\tif mset.catchups != nil {\n\t\tdelete(mset.catchups, peer)\n\t}\n\tmset.mu.Unlock()\n}\n\n// Lock should be held.\nfunc (mset *stream) clearAllCatchupPeers() {\n\tif mset.catchups != nil {\n\t\tmset.catchups = nil\n\t}\n}\n\nfunc (mset *stream) lagForCatchupPeer(peer string) uint64 {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\tif mset.catchups == nil {\n\t\treturn 0\n\t}\n\treturn mset.catchups[peer]\n}\n\nfunc (mset *stream) hasCatchupPeers() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn len(mset.catchups) > 0\n}\n\nfunc (mset *stream) setCatchingUp() {\n\tmset.catchup.Store(true)\n}\n\nfunc (mset *stream) clearCatchingUp() {\n\tmset.catchup.Store(false)\n}\n\nfunc (mset *stream) isCatchingUp() bool {\n\treturn mset.catchup.Load()\n}\n\n// Determine if a non-leader is current.\n// Lock should be held.\nfunc (mset *stream) isCurrent() bool {\n\tif mset.node == nil {\n\t\treturn true\n\t}\n\treturn mset.node.Current() && !mset.catchup.Load()\n}\n\n// Maximum requests for the whole server that can be in flight at the same time.\nconst maxConcurrentSyncRequests = 32\n\nvar (\n\terrCatchupCorruptSnapshot = errors.New(\"corrupt stream snapshot detected\")\n\terrCatchupStalled = errors.New(\"catchup stalled\")\n\terrCatchupStreamStopped = errors.New(\"stream has been stopped\") // when a catchup is terminated due to the stream going away.\n\terrCatchupBadMsg = errors.New(\"bad catchup msg\")\n\terrCatchupWrongSeqForSkip = errors.New(\"wrong sequence for skipped msg\")\n\terrCatchupAbortedNoLeader = errors.New(\"catchup aborted, no leader\")\n\terrCatchupTooManyRetries = errors.New(\"catchup failed, too many retries\")\n)\n\n// Process a stream snapshot.\nfunc (mset *stream) processSnapshot(snap *StreamReplicatedState, index uint64) (e error) {\n\t// Update any deletes, etc.\n\tif err := mset.processSnapshotDeletes(snap); err != nil {\n\t\treturn err\n\t}\n\tmset.setCLFS(snap.Failed)\n\n\tmset.mu.Lock()\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\tsreq := mset.calculateSyncRequest(&state, snap, index)\n\n\tif mset.sa == nil || mset.node == nil {\n\t\tmset.mu.Unlock()\n\t\treturn errCatchupStreamStopped\n\t}\n\ts, js, subject, n, st := mset.srv, mset.js, mset.sa.Sync, mset.node, mset.cfg.Storage\n\tqname := fmt.Sprintf(\"[ACC:%s] stream '%s' snapshot\", mset.acc.Name, mset.cfg.Name)\n\tmset.mu.Unlock()\n\n\t// Always try to resume applies, we might be paused already if we timed out of processing the snapshot previously.\n\tdefer func() {\n\t\t// Don't bother resuming if server or stream is gone.\n\t\tif e != errCatchupStreamStopped && e != ErrServerNotRunning {\n\t\t\tn.ResumeApply()\n\t\t}\n\t}()\n\n\t// Bug that would cause this to be empty on stream update.\n\tif subject == _EMPTY_ {\n\t\treturn errCatchupCorruptSnapshot\n\t}\n\n\t// Just return if up to date or already exceeded limits.\n\tif sreq == nil || js.limitsExceeded(st) {\n\t\treturn nil\n\t}\n\n\t// Pause the apply channel for our raft group while we catch up.\n\tif err := n.PauseApply(); err != nil {\n\t\treturn err\n\t}\n\n\t// Set our catchup state.\n\tmset.setCatchingUp()\n\tdefer mset.clearCatchingUp()\n\n\tvar sub *subscription\n\tvar err error\n\n\tconst (\n\t\tstartInterval = 5 * time.Second\n\t\tactivityInterval = 30 * time.Second\n\t)\n\tnotActive := time.NewTimer(startInterval)\n\tdefer notActive.Stop()\n\n\tdefer func() {\n\t\tif sub != nil {\n\t\t\ts.sysUnsubscribe(sub)\n\t\t}\n\t\t// Make sure any consumers are updated for the pending amounts.\n\t\tmset.mu.Lock()\n\t\tfor _, o := range mset.consumers {\n\t\t\to.mu.Lock()\n\t\t\tif o.isLeader() {\n\t\t\t\to.streamNumPending()\n\t\t\t}\n\t\t\to.mu.Unlock()\n\t\t}\n\t\tmset.mu.Unlock()\n\n\t\t// If we are interest based make sure to check our ack floor state.\n\t\t// We will delay a bit to allow consumer states to also catchup.\n\t\tif mset.isInterestRetention() {\n\t\t\tfire := time.Duration(rand.Intn(10)+5) * time.Second\n\t\t\ttime.AfterFunc(fire, mset.checkInterestState)\n\t\t}\n\t}()\n\n\tvar releaseSem bool\n\treleaseSyncOutSem := func() {\n\t\tif !releaseSem {\n\t\t\treturn\n\t\t}\n\t\t// Need to use select for the server shutdown case.\n\t\tselect {\n\t\tcase s.syncOutSem <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t\treleaseSem = false\n\t}\n\t// On exit, we will release our semaphore if we acquired it.\n\tdefer releaseSyncOutSem()\n\n\t// Do not let this go on forever.\n\tstart := time.Now()\n\tconst maxRetries = 3\n\tvar numRetries int\n\nRETRY:\n\t// On retry, we need to release the semaphore we got. Call will be no-op\n\t// if releaseSem boolean has not been set to true on successfully getting\n\t// the semaphore.\n\treleaseSyncOutSem()\n\n\tif n.Leaderless() {\n\t\t// Prevent us from spinning if we've installed a snapshot from a leader but there's no leader online.\n\t\t// We wait a bit to check if a leader has come online in the meantime, if so we can continue.\n\t\tvar canContinue bool\n\t\tif numRetries == 0 {\n\t\t\ttime.Sleep(startInterval)\n\t\t\tcanContinue = !n.Leaderless()\n\t\t}\n\t\tif !canContinue {\n\t\t\treturn fmt.Errorf(\"%w for stream '%s > %s'\", errCatchupAbortedNoLeader, mset.account(), mset.name())\n\t\t}\n\t}\n\n\t// If we have a sub clear that here.\n\tif sub != nil {\n\t\ts.sysUnsubscribe(sub)\n\t\tsub = nil\n\t}\n\n\tif !s.isRunning() {\n\t\treturn ErrServerNotRunning\n\t}\n\n\tnumRetries++\n\tif numRetries > maxRetries {\n\t\t// Force a hard reset here.\n\t\treturn errCatchupTooManyRetries\n\t}\n\n\t// Block here if we have too many requests in flight.\n\t<-s.syncOutSem\n\treleaseSem = true\n\n\t// We may have been blocked for a bit, so the reset needs to ensure that we\n\t// consume the already fired timer.\n\tif !notActive.Stop() {\n\t\tselect {\n\t\tcase <-notActive.C:\n\t\tdefault:\n\t\t}\n\t}\n\tnotActive.Reset(startInterval)\n\n\t// Grab sync request again on failures.\n\tif sreq == nil {\n\t\tmset.mu.RLock()\n\t\tvar state StreamState\n\t\tmset.store.FastState(&state)\n\t\tsreq = mset.calculateSyncRequest(&state, snap, index)\n\t\tmset.mu.RUnlock()\n\t\tif sreq == nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t// Used to transfer message from the wire to another Go routine internally.\n\ttype im struct {\n\t\tmsg []byte\n\t\treply string\n\t}\n\t// This is used to notify the leader that it should stop the runCatchup\n\t// because we are either bailing out or going to retry due to an error.\n\tnotifyLeaderStopCatchup := func(mrec *im, err error) {\n\t\tif mrec.reply == _EMPTY_ {\n\t\t\treturn\n\t\t}\n\t\ts.sendInternalMsgLocked(mrec.reply, _EMPTY_, nil, err.Error())\n\t}\n\n\tmsgsQ := newIPQueue[*im](s, qname)\n\tdefer msgsQ.unregister()\n\n\t// Send our catchup request here.\n\treply := syncReplySubject()\n\tsub, err = s.sysSubscribe(reply, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {\n\t\t// Make copy since we are using a buffer from the inbound client/route.\n\t\tmsgsQ.push(&im{copyBytes(msg), reply})\n\t})\n\tif err != nil {\n\t\ts.Errorf(\"Could not subscribe to stream catchup: %v\", err)\n\t\tgoto RETRY\n\t}\n\n\t// Send our sync request.\n\tb, _ := json.Marshal(sreq)\n\ts.sendInternalMsgLocked(subject, reply, nil, b)\n\n\t// Remember when we sent this out to avoid loop spins on errors below.\n\treqSendTime := time.Now()\n\n\t// Clear our sync request.\n\tsreq = nil\n\n\t// Run our own select loop here.\n\tfor qch, lch := n.QuitC(), n.LeadChangeC(); ; {\n\t\tselect {\n\t\tcase <-msgsQ.ch:\n\t\t\tnotActive.Reset(activityInterval)\n\n\t\t\tmrecs := msgsQ.pop()\n\t\t\tfor _, mrec := range mrecs {\n\t\t\t\tmsg := mrec.msg\n\t\t\t\t// Check for eof signaling.\n\t\t\t\tif len(msg) == 0 {\n\t\t\t\t\tmsgsQ.recycle(&mrecs)\n\n\t\t\t\t\t// Sanity check that we've received all data expected by the snapshot.\n\t\t\t\t\tmset.mu.RLock()\n\t\t\t\t\tlseq := mset.lseq\n\t\t\t\t\tmset.mu.RUnlock()\n\t\t\t\t\tif lseq >= snap.LastSeq {\n\t\t\t\t\t\t// We MUST ensure all data is flushed up to this point, if the store hadn't already.\n\t\t\t\t\t\t// Because the snapshot needs to represent what has been persisted.\n\t\t\t\t\t\terr = mset.flushAllPending()\n\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\ts.Noticef(\"Catchup for stream '%s > %s' complete (took %v)\", mset.account(), mset.name(), time.Since(start).Round(time.Millisecond))\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ts.Noticef(\"Catchup for stream '%s > %s' errored: %v (took %v)\", mset.account(), mset.name(), err, time.Since(start).Round(time.Millisecond))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\n\t\t\t\t\t// Make sure we do not spin and make things worse.\n\t\t\t\t\tconst minRetryWait = 2 * time.Second\n\t\t\t\t\telapsed := time.Since(reqSendTime)\n\t\t\t\t\tif elapsed < minRetryWait {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\t\t\treturn ErrServerNotRunning\n\t\t\t\t\t\tcase <-qch:\n\t\t\t\t\t\t\treturn errCatchupStreamStopped\n\t\t\t\t\t\tcase <-time.After(minRetryWait - elapsed):\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tgoto RETRY\n\t\t\t\t}\n\t\t\t\tif _, err := mset.processCatchupMsg(msg); err == nil {\n\t\t\t\t\tif mrec.reply != _EMPTY_ {\n\t\t\t\t\t\ts.sendInternalMsgLocked(mrec.reply, _EMPTY_, nil, nil)\n\t\t\t\t\t}\n\t\t\t\t} else if isOutOfSpaceErr(err) {\n\t\t\t\t\tnotifyLeaderStopCatchup(mrec, err)\n\t\t\t\t\tmsgsQ.recycle(&mrecs)\n\t\t\t\t\treturn err\n\t\t\t\t} else if err == NewJSInsufficientResourcesError() {\n\t\t\t\t\tnotifyLeaderStopCatchup(mrec, err)\n\t\t\t\t\tif mset.js.limitsExceeded(mset.cfg.Storage) {\n\t\t\t\t\t\ts.resourcesExceededError(mset.cfg.Storage)\n\t\t\t\t\t} else {\n\t\t\t\t\t\ts.Warnf(\"Catchup for stream '%s > %s' errored, account resources exceeded: %v\", mset.account(), mset.name(), err)\n\t\t\t\t\t}\n\t\t\t\t\tmsgsQ.recycle(&mrecs)\n\t\t\t\t\treturn err\n\t\t\t\t} else {\n\t\t\t\t\tnotifyLeaderStopCatchup(mrec, err)\n\t\t\t\t\ts.Warnf(\"Catchup for stream '%s > %s' errored, will retry: %v\", mset.account(), mset.name(), err)\n\t\t\t\t\tmsgsQ.recycle(&mrecs)\n\n\t\t\t\t\t// Make sure we do not spin and make things worse.\n\t\t\t\t\tconst minRetryWait = 2 * time.Second\n\t\t\t\t\telapsed := time.Since(reqSendTime)\n\t\t\t\t\tif elapsed < minRetryWait {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\t\t\treturn ErrServerNotRunning\n\t\t\t\t\t\tcase <-qch:\n\t\t\t\t\t\t\treturn errCatchupStreamStopped\n\t\t\t\t\t\tcase <-time.After(minRetryWait - elapsed):\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tgoto RETRY\n\t\t\t\t}\n\t\t\t}\n\t\t\tnotActive.Reset(activityInterval)\n\t\t\tmsgsQ.recycle(&mrecs)\n\t\tcase <-notActive.C:\n\t\t\tif mrecs := msgsQ.pop(); len(mrecs) > 0 {\n\t\t\t\tmrec := mrecs[0]\n\t\t\t\tnotifyLeaderStopCatchup(mrec, errCatchupStalled)\n\t\t\t\tmsgsQ.recycle(&mrecs)\n\t\t\t}\n\t\t\ts.Warnf(\"Catchup for stream '%s > %s' stalled\", mset.account(), mset.name())\n\t\t\tgoto RETRY\n\t\tcase <-s.quitCh:\n\t\t\treturn ErrServerNotRunning\n\t\tcase <-qch:\n\t\t\treturn errCatchupStreamStopped\n\t\tcase isLeader := <-lch:\n\t\t\tif isLeader {\n\t\t\t\tn.StepDown()\n\t\t\t\tgoto RETRY\n\t\t\t}\n\t\t}\n\t}\n}\n\n// processCatchupMsg will be called to process out of band catchup msgs from a sync request.\nfunc (mset *stream) processCatchupMsg(msg []byte) (uint64, error) {\n\tif len(msg) == 0 {\n\t\treturn 0, errCatchupBadMsg\n\t}\n\top := entryOp(msg[0])\n\tif op != streamMsgOp && op != compressedStreamMsgOp && op != deleteRangeOp {\n\t\treturn 0, errCatchupBadMsg\n\t}\n\n\tmbuf := msg[1:]\n\tif op == deleteRangeOp {\n\t\tdr, err := decodeDeleteRange(mbuf)\n\t\tif err != nil {\n\t\t\treturn 0, errCatchupBadMsg\n\t\t}\n\t\t// Handle the delete range.\n\t\t// Make sure the sequences match up properly.\n\t\tmset.mu.Lock()\n\t\tif len(mset.preAcks) > 0 {\n\t\t\tfor seq := dr.First; seq < dr.First+dr.Num; seq++ {\n\t\t\t\tmset.clearAllPreAcks(seq)\n\t\t\t}\n\t\t}\n\t\tif err = mset.store.SkipMsgs(dr.First, dr.Num); err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn 0, errCatchupWrongSeqForSkip\n\t\t}\n\t\tmset.lseq = dr.First + dr.Num - 1\n\t\tlseq := mset.lseq\n\t\tmset.mu.Unlock()\n\t\treturn lseq, nil\n\t}\n\n\tif op == compressedStreamMsgOp {\n\t\tvar err error\n\t\tmbuf, err = s2.Decode(nil, mbuf)\n\t\tif err != nil {\n\t\t\tpanic(err.Error())\n\t\t}\n\t}\n\n\tsubj, _, hdr, msg, seq, ts, _, err := decodeStreamMsg(mbuf)\n\tif err != nil {\n\t\treturn 0, errCatchupBadMsg\n\t}\n\n\tmset.mu.Lock()\n\tst := mset.cfg.Storage\n\tif mset.hasAllPreAcks(seq, subj) {\n\t\tmset.clearAllPreAcks(seq)\n\t\t// Mark this to be skipped\n\t\tsubj, ts = _EMPTY_, 0\n\t}\n\tmset.mu.Unlock()\n\n\t// Since we're clustered we do not want to check limits based on tier here and possibly introduce skew.\n\tif mset.js.limitsExceeded(st) {\n\t\treturn 0, NewJSInsufficientResourcesError()\n\t}\n\n\t// Find the message TTL if any.\n\t// TODO(nat): If the TTL isn't valid by this stage then there isn't really a\n\t// lot we can do about it, as we'd break the catchup if we reject the message.\n\tttl, _ := getMessageTTL(hdr)\n\n\t// Put into our store\n\t// Messages to be skipped have no subject or timestamp.\n\t// TODO(dlc) - formalize with skipMsgOp\n\tif subj == _EMPTY_ && ts == 0 {\n\t\tif _, err = mset.store.SkipMsg(seq); err != nil {\n\t\t\treturn 0, errCatchupWrongSeqForSkip\n\t\t}\n\t} else if err := mset.store.StoreRawMsg(subj, hdr, msg, seq, ts, ttl, false); err != nil {\n\t\treturn 0, err\n\t}\n\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\t// Update our lseq.\n\tmset.setLastSeq(seq)\n\n\t// Check for MsgId and if we have one here make sure to update our internal map.\n\tif len(hdr) > 0 {\n\t\tif msgId := getMsgId(hdr); msgId != _EMPTY_ {\n\t\t\tmset.ddMu.Lock()\n\t\t\tmset.storeMsgIdLocked(&ddentry{msgId, seq, ts})\n\t\t\tmset.ddMu.Unlock()\n\t\t}\n\t}\n\n\treturn seq, nil\n}\n\n// flushAllPending will flush any pending writes as a result of installing a snapshot or performing catchup.\nfunc (mset *stream) flushAllPending() error {\n\treturn mset.store.FlushAllPending()\n}\n\nfunc (mset *stream) handleClusterSyncRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tvar sreq streamSyncRequest\n\tif err := json.Unmarshal(msg, &sreq); err != nil {\n\t\t// Log error.\n\t\treturn\n\t}\n\tmset.srv.startGoRoutine(func() { mset.runCatchup(reply, &sreq) })\n}\n\n// Lock should be held.\nfunc (js *jetStream) offlineClusterInfo(rg *raftGroup) *ClusterInfo {\n\ts := js.srv\n\n\tci := &ClusterInfo{Name: s.ClusterName(), RaftGroup: rg.Name}\n\tfor _, peer := range rg.Peers {\n\t\tif sir, ok := s.nodeToInfo.Load(peer); ok && sir != nil {\n\t\t\tsi := sir.(nodeInfo)\n\t\t\tpi := &PeerInfo{Peer: peer, Name: si.name, Current: false, Offline: true}\n\t\t\tci.Replicas = append(ci.Replicas, pi)\n\t\t}\n\t}\n\treturn ci\n}\n\n// clusterInfo will report on the status of the raft group.\nfunc (js *jetStream) clusterInfo(rg *raftGroup) *ClusterInfo {\n\tif js == nil {\n\t\treturn nil\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\n\ts := js.srv\n\tif rg == nil || rg.node == nil {\n\t\treturn &ClusterInfo{\n\t\t\tName: s.cachedClusterName(),\n\t\t\tLeader: s.Name(),\n\t\t}\n\t}\n\n\tn := rg.node\n\tci := &ClusterInfo{\n\t\tName: s.cachedClusterName(),\n\t\tLeader: s.serverNameForNode(n.GroupLeader()),\n\t\tLeaderSince: n.LeaderSince(),\n\t\tSystemAcc: n.IsSystemAccount(),\n\t\tTrafficAcc: n.GetTrafficAccountName(),\n\t\tRaftGroup: rg.Name,\n\t}\n\n\tnow := time.Now()\n\tid, peers := n.ID(), n.Peers()\n\n\t// If we are leaderless, do not suppress putting us in the peer list.\n\tif ci.Leader == _EMPTY_ {\n\t\tid = _EMPTY_\n\t}\n\n\tfor _, rp := range peers {\n\t\tif rp.ID != id && rg.isMember(rp.ID) {\n\t\t\tvar lastSeen time.Duration\n\t\t\tif now.After(rp.Last) && !rp.Last.IsZero() {\n\t\t\t\tlastSeen = now.Sub(rp.Last)\n\t\t\t}\n\t\t\tcurrent := rp.Current\n\t\t\tif current && lastSeen > lostQuorumInterval {\n\t\t\t\tcurrent = false\n\t\t\t}\n\t\t\t// Create a peer info with common settings if the peer has not been seen\n\t\t\t// yet (which can happen after the whole cluster is stopped and only some\n\t\t\t// of the nodes are restarted).\n\t\t\tpi := &PeerInfo{\n\t\t\t\tCurrent: current,\n\t\t\t\tOffline: true,\n\t\t\t\tActive: lastSeen,\n\t\t\t\tLag: rp.Lag,\n\t\t\t\tPeer: rp.ID,\n\t\t\t}\n\t\t\t// If node is found, complete/update the settings.\n\t\t\tif sir, ok := s.nodeToInfo.Load(rp.ID); ok && sir != nil {\n\t\t\t\tsi := sir.(nodeInfo)\n\t\t\t\tpi.Name, pi.Offline, pi.cluster = si.name, si.offline, si.cluster\n\t\t\t} else {\n\t\t\t\t// If not, then add a name that indicates that the server name\n\t\t\t\t// is unknown at this time, and clear the lag since it is misleading\n\t\t\t\t// (the node may not have that much lag).\n\t\t\t\t// Note: We return now the Peer ID in PeerInfo, so the \"(peerID: %s)\"\n\t\t\t\t// would technically not be required, but keeping it for now.\n\t\t\t\tpi.Name, pi.Lag = fmt.Sprintf(\"Server name unknown at this time (peerID: %s)\", rp.ID), 0\n\t\t\t}\n\t\t\tci.Replicas = append(ci.Replicas, pi)\n\t\t}\n\t}\n\t// Order the result based on the name so that we get something consistent\n\t// when doing repeated stream info in the CLI, etc...\n\tslices.SortFunc(ci.Replicas, func(i, j *PeerInfo) int { return cmp.Compare(i.Name, j.Name) })\n\treturn ci\n}\n\nfunc (mset *stream) checkClusterInfo(ci *ClusterInfo) {\n\tfor _, r := range ci.Replicas {\n\t\tpeer := getHash(r.Name)\n\t\tif lag := mset.lagForCatchupPeer(peer); lag > 0 {\n\t\t\tr.Current = false\n\t\t\tr.Lag = lag\n\t\t}\n\t}\n}\n\n// Return a list of alternates, ranked by preference order to the request, of stream mirrors.\n// This allows clients to select or get more information about read replicas that could be a\n// better option to connect to versus the original source.\nfunc (js *jetStream) streamAlternates(ci *ClientInfo, stream string) []StreamAlternate {\n\tif js == nil {\n\t\treturn nil\n\t}\n\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\n\ts, cc := js.srv, js.cluster\n\t// Track our domain.\n\tdomain := s.getOpts().JetStreamDomain\n\n\t// No clustering just return nil.\n\tif cc == nil {\n\t\treturn nil\n\t}\n\tacc, _ := s.LookupAccount(ci.serviceAccount())\n\tif acc == nil {\n\t\treturn nil\n\t}\n\n\t// Collect our ordering first for clusters.\n\tweights := make(map[string]int)\n\tall := []string{ci.Cluster}\n\tall = append(all, ci.Alternates...)\n\n\tfor i := 0; i < len(all); i++ {\n\t\tweights[all[i]] = len(all) - i\n\t}\n\n\tvar alts []StreamAlternate\n\tfor _, sa := range cc.streams[acc.Name] {\n\t\tif sa.unsupported != nil {\n\t\t\tcontinue\n\t\t}\n\t\t// Add in ourselves and any mirrors.\n\t\tif sa.Config.Name == stream || (sa.Config.Mirror != nil && sa.Config.Mirror.Name == stream) {\n\t\t\talts = append(alts, StreamAlternate{Name: sa.Config.Name, Domain: domain, Cluster: sa.Group.Cluster})\n\t\t}\n\t}\n\t// If just us don't fill in.\n\tif len(alts) == 1 {\n\t\treturn nil\n\t}\n\n\t// Sort based on our weights that originate from the request itself.\n\t// reverse sort\n\tslices.SortFunc(alts, func(i, j StreamAlternate) int { return -cmp.Compare(weights[i.Cluster], weights[j.Cluster]) })\n\n\treturn alts\n}\n\n// Internal request for stream info, this is coming on the wire so do not block here.\nfunc (mset *stream) handleClusterStreamInfoRequest(_ *subscription, c *client, _ *Account, subject, reply string, _ []byte) {\n\tgo mset.processClusterStreamInfoRequest(reply)\n}\n\nfunc (mset *stream) processClusterStreamInfoRequest(reply string) {\n\tmset.mu.RLock()\n\tsysc, js, sa, config := mset.sysc, mset.srv.js.Load(), mset.sa, mset.cfg\n\tisLeader := mset.isLeader()\n\tmset.mu.RUnlock()\n\n\t// By design all members will receive this. Normally we only want the leader answering.\n\t// But if we have stalled and lost quorom all can respond.\n\tif sa != nil && !js.isGroupLeaderless(sa.Group) && !isLeader {\n\t\treturn\n\t}\n\n\t// If we are not the leader let someone else possibly respond first.\n\tif !isLeader {\n\t\ttime.Sleep(500 * time.Millisecond)\n\t}\n\n\tsi := &StreamInfo{\n\t\tCreated: mset.createdTime(),\n\t\tState: mset.state(),\n\t\tConfig: config,\n\t\tCluster: js.clusterInfo(mset.raftGroup()),\n\t\tSources: mset.sourcesInfo(),\n\t\tMirror: mset.mirrorInfo(),\n\t\tTimeStamp: time.Now().UTC(),\n\t}\n\n\t// Check for out of band catchups.\n\tif mset.hasCatchupPeers() {\n\t\tmset.checkClusterInfo(si.Cluster)\n\t}\n\n\tsysc.sendInternalMsg(reply, _EMPTY_, nil, si)\n}\n\n// 64MB for now, for the total server. This is max we will blast out if asked to\n// do so to another server for purposes of catchups.\n// This number should be ok on 1Gbit interface.\nconst defaultMaxTotalCatchupOutBytes = int64(64 * 1024 * 1024)\n\n// Current total outstanding catchup bytes.\nfunc (s *Server) gcbTotal() int64 {\n\ts.gcbMu.RLock()\n\tdefer s.gcbMu.RUnlock()\n\treturn s.gcbOut\n}\n\n// Returns true if Current total outstanding catchup bytes is below\n// the maximum configured.\nfunc (s *Server) gcbBelowMax() bool {\n\ts.gcbMu.RLock()\n\tdefer s.gcbMu.RUnlock()\n\treturn s.gcbOut <= s.gcbOutMax\n}\n\n// Adds `sz` to the server's total outstanding catchup bytes and to `localsz`\n// under the gcbMu lock. The `localsz` points to the local outstanding catchup\n// bytes of the runCatchup go routine of a given stream.\nfunc (s *Server) gcbAdd(localsz *int64, sz int64) {\n\ts.gcbMu.Lock()\n\tatomic.AddInt64(localsz, sz)\n\ts.gcbOut += sz\n\tif s.gcbOut >= s.gcbOutMax && s.gcbKick == nil {\n\t\ts.gcbKick = make(chan struct{})\n\t}\n\ts.gcbMu.Unlock()\n}\n\n// Removes `sz` from the server's total outstanding catchup bytes and from\n// `localsz`, but only if `localsz` is non 0, which would signal that gcSubLast\n// has already been invoked. See that function for details.\n// Must be invoked under the gcbMu lock.\nfunc (s *Server) gcbSubLocked(localsz *int64, sz int64) {\n\tif atomic.LoadInt64(localsz) == 0 {\n\t\treturn\n\t}\n\tatomic.AddInt64(localsz, -sz)\n\ts.gcbOut -= sz\n\tif s.gcbKick != nil && s.gcbOut < s.gcbOutMax {\n\t\tclose(s.gcbKick)\n\t\ts.gcbKick = nil\n\t}\n}\n\n// Locked version of gcbSubLocked()\nfunc (s *Server) gcbSub(localsz *int64, sz int64) {\n\ts.gcbMu.Lock()\n\ts.gcbSubLocked(localsz, sz)\n\ts.gcbMu.Unlock()\n}\n\n// Similar to gcbSub() but reset `localsz` to 0 at the end under the gcbMu lock.\n// This will signal further calls to gcbSub() for this `localsz` pointer that\n// nothing should be done because runCatchup() has exited and any remaining\n// outstanding bytes value has already been decremented.\nfunc (s *Server) gcbSubLast(localsz *int64) {\n\ts.gcbMu.Lock()\n\ts.gcbSubLocked(localsz, *localsz)\n\t*localsz = 0\n\ts.gcbMu.Unlock()\n}\n\n// Returns our kick chan, or nil if it does not exist.\nfunc (s *Server) cbKickChan() <-chan struct{} {\n\ts.gcbMu.RLock()\n\tdefer s.gcbMu.RUnlock()\n\treturn s.gcbKick\n}\n\nfunc (mset *stream) runCatchup(sendSubject string, sreq *streamSyncRequest) {\n\ts := mset.srv\n\tdefer s.grWG.Done()\n\n\tconst maxOutBytes = int64(64 * 1024 * 1024) // 64MB for now, these are all internal, from server to server\n\tconst maxOutMsgs = int32(256 * 1024) // 256k in case we have lots of small messages or skip msgs.\n\toutb := int64(0)\n\toutm := int32(0)\n\n\t// On abnormal exit make sure to update global total.\n\tdefer s.gcbSubLast(&outb)\n\n\t// Flow control processing.\n\tackReplySize := func(subj string) int64 {\n\t\tif li := strings.LastIndexByte(subj, btsep); li > 0 && li < len(subj) {\n\t\t\treturn parseAckReplyNum(subj[li+1:])\n\t\t}\n\t\treturn 0\n\t}\n\n\tnextBatchC := make(chan struct{}, 4)\n\tnextBatchC <- struct{}{}\n\tremoteQuitCh := make(chan struct{})\n\n\tconst activityInterval = 30 * time.Second\n\tnotActive := time.NewTimer(activityInterval)\n\tdefer notActive.Stop()\n\n\t// Setup ackReply for flow control.\n\tackReply := syncAckSubject()\n\tackSub, _ := s.sysSubscribe(ackReply, func(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\t\tif len(msg) > 0 {\n\t\t\ts.Warnf(\"Catchup for stream '%s > %s' was aborted on the remote due to: %q\",\n\t\t\t\tmset.account(), mset.name(), msg)\n\t\t\ts.sysUnsubscribe(sub)\n\t\t\tclose(remoteQuitCh)\n\t\t\treturn\n\t\t}\n\t\tsz := ackReplySize(subject)\n\t\ts.gcbSub(&outb, sz)\n\t\tatomic.AddInt32(&outm, -1)\n\t\tmset.updateCatchupPeer(sreq.Peer)\n\t\t// Kick ourselves and anyone else who might have stalled on global state.\n\t\tselect {\n\t\tcase nextBatchC <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t\t// Reset our activity\n\t\tnotActive.Reset(activityInterval)\n\t})\n\tdefer s.sysUnsubscribe(ackSub)\n\tackReplyT := strings.ReplaceAll(ackReply, \".*\", \".%d\")\n\n\t// Grab our state.\n\tvar state StreamState\n\t// mset.store never changes after being set, don't need lock.\n\tmset.store.FastState(&state)\n\n\t// Setup sequences to walk through.\n\tseq, last := sreq.FirstSeq, sreq.LastSeq\n\n\t// The follower received a snapshot from another leader, and we've become leader since.\n\t// We have an up-to-date log but could be behind on applies. We must wait until we've reached the minimum required.\n\t// The follower will automatically retry after a timeout, so we can safely return here.\n\tif node := mset.raftNode(); node != nil {\n\t\tindex, _, applied := node.Progress()\n\t\t// Only skip if our log has enough entries, and they could be applied in the future.\n\t\tif index >= sreq.MinApplied && applied < sreq.MinApplied {\n\t\t\treturn\n\t\t}\n\t\t// We know here we've either applied enough entries, or our log doesn't have enough entries.\n\t\t// In the latter case the request expects us to have more. Just continue and value availability here.\n\t\t// This should only be possible if the logs have already desynced, and we shouldn't have become leader\n\t\t// in the first place. Not much we can do here in this (hypothetical) scenario.\n\n\t\t// Do another quick sanity check that we actually have enough data to satisfy the request.\n\t\t// If not, let's step down and hope a new leader can correct this.\n\t\tif state.LastSeq < last {\n\t\t\ts.Warnf(\"Catchup for stream '%s > %s' skipped, requested sequence %d was larger than current state: %+v\",\n\t\t\t\tmset.account(), mset.name(), seq, state)\n\t\t\tnode.StepDown()\n\t\t\treturn\n\t\t}\n\t}\n\n\tstart := time.Now()\n\tmset.setCatchupPeer(sreq.Peer, last-seq)\n\n\tvar spb int\n\tconst minWait = 5 * time.Second\n\n\tsendNextBatchAndContinue := func(qch chan struct{}) bool {\n\t\t// Check if we know we will not enter the loop because we are done.\n\t\tif seq > last {\n\t\t\ts.Noticef(\"Catchup for stream '%s > %s' complete (took %v)\", mset.account(), mset.name(), time.Since(start).Round(time.Millisecond))\n\t\t\t// EOF\n\t\t\ts.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil)\n\t\t\treturn false\n\t\t}\n\n\t\t// If we already sent a batch, we will try to make sure we can at least send a minimum\n\t\t// batch before sending the next batch.\n\t\tif spb > 0 {\n\t\t\t// Wait til we can send at least 4k\n\t\t\tconst minBatchWait = int32(4 * 1024)\n\t\t\tmw := time.NewTimer(minWait)\n\t\t\tfor done := maxOutMsgs-atomic.LoadInt32(&outm) > minBatchWait; !done; {\n\t\t\t\tselect {\n\t\t\t\tcase <-nextBatchC:\n\t\t\t\t\tdone = maxOutMsgs-atomic.LoadInt32(&outm) > minBatchWait\n\t\t\t\t\tif !done {\n\t\t\t\t\t\t// Wait for a small bit.\n\t\t\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// GC friendly.\n\t\t\t\t\t\tmw.Stop()\n\t\t\t\t\t}\n\t\t\t\tcase <-mw.C:\n\t\t\t\t\tdone = true\n\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\treturn false\n\t\t\t\tcase <-qch:\n\t\t\t\t\treturn false\n\t\t\t\tcase <-remoteQuitCh:\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\tspb = 0\n\t\t}\n\n\t\t// Send an encoded msg.\n\t\tsendEM := func(em []byte) {\n\t\t\t// Place size in reply subject for flow control.\n\t\t\tl := int64(len(em))\n\t\t\treply := fmt.Sprintf(ackReplyT, l)\n\t\t\ts.gcbAdd(&outb, l)\n\t\t\tatomic.AddInt32(&outm, 1)\n\t\t\ts.sendInternalMsgLocked(sendSubject, reply, nil, em)\n\t\t\tspb++\n\t\t}\n\n\t\t// If we support gap markers.\n\t\tvar dr DeleteRange\n\t\tdrOk := sreq.DeleteRangesOk\n\n\t\t// Will send our delete range.\n\t\t// Should already be checked for being valid.\n\t\tsendDR := func() {\n\t\t\tif dr.Num == 1 {\n\t\t\t\t// Send like a normal skip msg.\n\t\t\t\tsendEM(encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, dr.First, 0, false))\n\t\t\t} else {\n\t\t\t\t// We have a run, send a gap record. We send these without reply or tracking.\n\t\t\t\ts.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, encodeDeleteRange(&dr))\n\t\t\t\t// Clear out the pending for catchup.\n\t\t\t\tmset.decrementCatchupPeer(sreq.Peer, dr.Num)\n\t\t\t}\n\t\t\t// Reset always.\n\t\t\tdr.First, dr.Num = 0, 0\n\t\t}\n\n\t\t// See if we should use LoadNextMsg instead of walking sequence by sequence if we have an order magnitude more interior deletes.\n\t\t// Only makes sense with delete range capabilities.\n\t\tuseLoadNext := drOk && (uint64(state.NumDeleted) > 2*state.Msgs || state.NumDeleted > 1_000_000)\n\n\t\tvar smv StoreMsg\n\t\tfor ; seq <= last && atomic.LoadInt64(&outb) <= maxOutBytes && atomic.LoadInt32(&outm) <= maxOutMsgs && s.gcbBelowMax(); seq++ {\n\t\t\tvar sm *StoreMsg\n\t\t\tvar err error\n\t\t\t// If we should use load next do so here.\n\t\t\tif useLoadNext {\n\t\t\t\tvar nseq uint64\n\t\t\t\tsm, nseq, err = mset.store.LoadNextMsg(fwcs, true, seq, &smv)\n\t\t\t\tif err == nil && nseq > seq {\n\t\t\t\t\t// If we jumped over the requested last sequence, clamp it down.\n\t\t\t\t\t// Otherwise, we would send too much to the follower.\n\t\t\t\t\tif nseq > last {\n\t\t\t\t\t\tnseq = last\n\t\t\t\t\t\tsm = nil\n\t\t\t\t\t}\n\t\t\t\t\tdr.First, dr.Num = seq, nseq-seq\n\t\t\t\t\t// Jump ahead\n\t\t\t\t\tseq = nseq\n\t\t\t\t} else if err == ErrStoreEOF {\n\t\t\t\t\tdr.First, dr.Num = seq, last-seq\n\t\t\t\t\t// Clear EOF here for normal processing.\n\t\t\t\t\terr = nil\n\t\t\t\t\t// Jump ahead\n\t\t\t\t\tseq = last\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tsm, err = mset.store.LoadMsg(seq, &smv)\n\t\t\t}\n\n\t\t\t// if this is not a deleted msg, bail out.\n\t\t\tif err != nil && err != ErrStoreMsgNotFound && err != errDeletedMsg {\n\t\t\t\tif err == ErrStoreEOF {\n\t\t\t\t\tvar state StreamState\n\t\t\t\t\tmset.store.FastState(&state)\n\t\t\t\t\tif seq > state.LastSeq {\n\t\t\t\t\t\t// The snapshot has a larger last sequence then we have. This could be due to a truncation\n\t\t\t\t\t\t// when trying to recover after corruption, still not 100% sure. Could be off by 1 too somehow,\n\t\t\t\t\t\t// but tested a ton of those with no success.\n\t\t\t\t\t\ts.Warnf(\"Catchup for stream '%s > %s' completed (took %v), but requested sequence %d was larger than current state: %+v\",\n\t\t\t\t\t\t\tmset.account(), mset.name(), time.Since(start).Round(time.Millisecond), seq, state)\n\t\t\t\t\t\t// Try our best to redo our invalidated snapshot as well.\n\t\t\t\t\t\tif n := mset.raftNode(); n != nil {\n\t\t\t\t\t\t\tif snap := mset.stateSnapshot(); snap != nil {\n\t\t\t\t\t\t\t\tn.InstallSnapshot(snap, true)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// If we allow gap markers check if we have one pending.\n\t\t\t\t\t\tif drOk && dr.First > 0 {\n\t\t\t\t\t\t\tsendDR()\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Signal EOF\n\t\t\t\t\t\ts.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil)\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.Warnf(\"Error loading message for catchup '%s > %s': %v\", mset.account(), mset.name(), err)\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tif sm != nil {\n\t\t\t\t// If we allow gap markers check if we have one pending.\n\t\t\t\tif drOk && dr.First > 0 {\n\t\t\t\t\tsendDR()\n\t\t\t\t}\n\t\t\t\t// Send the normal message now.\n\t\t\t\tsendEM(encodeStreamMsgAllowCompress(sm.subj, _EMPTY_, sm.hdr, sm.msg, sm.seq, sm.ts, false))\n\t\t\t} else {\n\t\t\t\tif drOk {\n\t\t\t\t\tif dr.First == 0 {\n\t\t\t\t\t\tdr.First, dr.Num = seq, 1\n\t\t\t\t\t} else {\n\t\t\t\t\t\tdr.Num++\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// Skip record for deleted msg.\n\t\t\t\t\tsendEM(encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, seq, 0, false))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check if we are done.\n\t\t\tif seq == last {\n\t\t\t\t// Need to see if we have a pending delete range.\n\t\t\t\tif drOk && dr.First > 0 {\n\t\t\t\t\tsendDR()\n\t\t\t\t}\n\t\t\t\ts.Noticef(\"Catchup for stream '%s > %s' complete (took %v)\", mset.account(), mset.name(), time.Since(start).Round(time.Millisecond))\n\t\t\t\t// EOF\n\t\t\t\ts.sendInternalMsgLocked(sendSubject, _EMPTY_, nil, nil)\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-remoteQuitCh:\n\t\t\t\treturn false\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\tif drOk && dr.First > 0 {\n\t\t\tsendDR()\n\t\t}\n\t\treturn true\n\t}\n\n\t// Check is this stream got closed.\n\tmset.mu.RLock()\n\tqch := mset.qch\n\tmset.mu.RUnlock()\n\tif qch == nil {\n\t\treturn\n\t}\n\n\t// Run as long as we are still active and need catchup.\n\t// FIXME(dlc) - Purge event? Stream delete?\n\tfor {\n\t\t// Get this each time, will be non-nil if globally blocked and we will close to wake everyone up.\n\t\tcbKick := s.cbKickChan()\n\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-remoteQuitCh:\n\t\t\tmset.clearCatchupPeer(sreq.Peer)\n\t\t\treturn\n\t\tcase <-notActive.C:\n\t\t\ts.Warnf(\"Catchup for stream '%s > %s' stalled\", mset.account(), mset.name())\n\t\t\tmset.clearCatchupPeer(sreq.Peer)\n\t\t\treturn\n\t\tcase <-nextBatchC:\n\t\t\tif !sendNextBatchAndContinue(qch) {\n\t\t\t\tmset.clearCatchupPeer(sreq.Peer)\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-cbKick:\n\t\t\tif !sendNextBatchAndContinue(qch) {\n\t\t\t\tmset.clearCatchupPeer(sreq.Peer)\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-time.After(500 * time.Millisecond):\n\t\t\tif !sendNextBatchAndContinue(qch) {\n\t\t\t\tmset.clearCatchupPeer(sreq.Peer)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\nconst jscAllSubj = \"$JSC.>\"\n\nfunc syncSubjForStream() string {\n\treturn syncSubject(\"$JSC.SYNC\")\n}\n\nfunc syncReplySubject() string {\n\treturn syncSubject(\"$JSC.R\")\n}\n\nfunc infoReplySubject() string {\n\treturn syncSubject(\"$JSC.R\")\n}\n\nfunc syncAckSubject() string {\n\treturn syncSubject(\"$JSC.ACK\") + \".*\"\n}\n\nfunc syncSubject(pre string) string {\n\tvar sb strings.Builder\n\tsb.WriteString(pre)\n\tsb.WriteByte(btsep)\n\n\tvar b [replySuffixLen]byte\n\trn := rand.Int63()\n\tfor i, l := 0, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\n\tsb.Write(b[:])\n\treturn sb.String()\n}\n\nconst (\n\tclusterStreamInfoT = \"$JSC.SI.%s.%s\"\n\tclusterConsumerInfoT = \"$JSC.CI.%s.%s.%s\"\n\tjsaUpdatesSubT = \"$JSC.ARU.%s.*\"\n\tjsaUpdatesPubT = \"$JSC.ARU.%s.%s\"\n)\n" }, { "path": "server/jetstream_cluster_1_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\tcrand \"crypto/rand\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestJetStreamClusterConfig(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 16GB, max_file_store: 10TB, store_dir: '%s'}\n\t\tcluster { listen: 127.0.0.1:-1 }\n\t`))\n\n\tcheck := func(errStr string) {\n\t\tt.Helper()\n\t\topts, err := ProcessConfigFile(conf)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), errStr) {\n\t\t\tt.Fatalf(\"Expected an error of `%s`, got `%v`\", errStr, err)\n\t\t}\n\t}\n\n\tcheck(\"requires `server_name`\")\n\n\tconf = createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"TEST\"\n\t\tjetstream: {max_mem_store: 16GB, max_file_store: 10TB, store_dir: '%s'}\n\t\tcluster { listen: 127.0.0.1:-1 }\n\t`))\n\n\tcheck(\"requires `cluster.name`\")\n}\n\nfunc TestJetStreamClusterLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\t// Kill our current leader and force an election.\n\tc.leader().Shutdown()\n\tc.waitOnLeader()\n\n\t// Now killing our current leader should leave us leaderless.\n\tc.leader().Shutdown()\n\tc.expectNoLeader()\n}\n\nfunc TestJetStreamClusterExpand(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 2)\n\tdefer c.shutdown()\n\n\tc.addInNewServer()\n\tc.waitOnPeerCount(3)\n}\n\nfunc TestJetStreamClusterAccountInfo(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc := clientConnectToServer(t, c.randomServer())\n\tdefer nc.Close()\n\n\treply := nats.NewInbox()\n\tsub, _ := nc.SubscribeSync(reply)\n\n\tif err := nc.PublishRequest(JSApiAccountInfo, reply, nil); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, 1)\n\tresp, _ := sub.NextMsg(0)\n\n\tvar info JSApiAccountInfoResponse\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.JetStreamAccountStats == nil || info.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", info.Error)\n\t}\n\t// Make sure we only got 1 response.\n\t// Technically this will always work since its a singleton service export.\n\tif nmsgs, _, _ := sub.Pending(); nmsgs > 0 {\n\t\tt.Fatalf(\"Expected only a single response, got %d more\", nmsgs)\n\t}\n}\n\nfunc TestJetStreamClusterStreamLimitWithAccountDefaults(t *testing.T) {\n\t// 2MB memory, 8MB disk\n\tc := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, \"R3L\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 2,\n\t\tMaxBytes: 4 * 1024 * 1024,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tReplicas: 2,\n\t\tMaxBytes: 15 * 1024 * 1024,\n\t})\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"insufficient storage\")\n}\n\nfunc TestJetStreamClusterInfoRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R1S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tStorage: nats.FileStorage,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tnfoResp, err := nc.Request(\"$JS.API.STREAM.INFO.TEST\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar si StreamInfo\n\terr = json.Unmarshal(nfoResp.Data, &si)\n\trequire_NoError(t, err)\n\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected cluster info, got none\")\n\t}\n\n\tstream, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.Cluster.RaftGroup != stream.raftGroup().Name {\n\t\tt.Fatalf(\"Expected raft group %q to equal %q\", si.Cluster.RaftGroup, stream.raftGroup().Name)\n\t}\n\n\tvar sscfg StreamConfig\n\trCfgData, err := os.ReadFile(filepath.Join(s.opts.StoreDir, \"jetstream\", \"$SYS\", \"_js_\", stream.raftGroup().Name, \"meta.inf\"))\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(rCfgData, &sscfg)\n\trequire_NoError(t, err)\n\tif !reflect.DeepEqual(sscfg.Metadata, map[string]string{\"account\": \"$G\", \"stream\": \"TEST\", \"type\": \"stream\"}) {\n\t\tt.Fatalf(\"Invalid raft stream metadata: %v\", sscfg.Metadata)\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DURABLE\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\tconsumer := stream.lookupConsumer(\"DURABLE\")\n\n\tvar ci ConsumerInfo\n\tnfoResp, err = nc.Request(\"$JS.API.CONSUMER.INFO.TEST.DURABLE\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar cscfg ConsumerConfig\n\trCfgData, err = os.ReadFile(filepath.Join(s.opts.StoreDir, \"jetstream\", \"$SYS\", \"_js_\", consumer.raftGroup().Name, \"meta.inf\"))\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(rCfgData, &cscfg)\n\trequire_NoError(t, err)\n\tif !reflect.DeepEqual(cscfg.Metadata, map[string]string{\"account\": \"$G\", \"consumer\": \"DURABLE\", \"stream\": \"TEST\", \"type\": \"consumer\"}) {\n\t\tt.Fatalf(\"Invalid raft stream metadata: %v\", cscfg.Metadata)\n\t}\n\n\terr = json.Unmarshal(nfoResp.Data, &ci)\n\trequire_NoError(t, err)\n\n\tif ci.Cluster.RaftGroup != consumer.raftGroup().Name {\n\t\tt.Fatalf(\"Expected raft group %q to equal %q\", ci.Cluster.RaftGroup, consumer.raftGroup().Name)\n\t}\n}\n\nfunc TestJetStreamClusterSingleReplicaStreams(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R1S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected si to have cluster info\")\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\t// Now grab info for this stream.\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Now create a consumer. This should be pinned to same server that our stream was allocated to.\n\t// First do a normal sub.\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\n\t// Now create a consumer as well.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci == nil || ci.Name != \"dlc\" || ci.Stream != \"TEST\" {\n\t\tt.Fatalf(\"ConsumerInfo is not correct %+v\", ci)\n\t}\n\n\t// Now make sure that if we kill and restart the server that this stream and consumer return.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tsl.Shutdown()\n\tc.restartServer(sl)\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Now durable consumer.\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tif _, err = js.ConsumerInfo(\"TEST\", \"dlc\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterMultiReplicaStreams(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"RNS\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Now create a consumer. This should be affinitize to the same set of servers as the stream.\n\t// First do a normal sub.\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\n\t// Now create a consumer as well.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci == nil || ci.Name != \"dlc\" || ci.Stream != \"TEST\" || ci.NumPending != uint64(toSend) {\n\t\tt.Fatalf(\"ConsumerInfo is not correct %+v\", ci)\n\t}\n}\n\nfunc TestJetStreamClusterMultiReplicaStreamsDefaultFileMem(t *testing.T) {\n\tconst testConfig = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n`\n\tc := createJetStreamClusterWithTemplate(t, testConfig, \"RNS\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tMaxBytes: 1024,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Now create a consumer. This should be affinitize to the same set of servers as the stream.\n\t// First do a normal sub.\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\n\t// Now create a consumer as well.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci == nil || ci.Name != \"dlc\" || ci.Stream != \"TEST\" || ci.NumPending != uint64(toSend) {\n\t\tt.Fatalf(\"ConsumerInfo is not correct %+v\", ci)\n\t}\n}\n\nfunc TestJetStreamClusterMemoryStore(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3M\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tStorage: nats.MemoryStorage,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 100 messages.\n\tmsg, toSend := []byte(\"Hello MemoryStore\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\tif si.Cluster == nil || len(si.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Cluster info is incorrect: %+v\", si.Cluster)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Do a normal sub.\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n}\n\nfunc TestJetStreamClusterDelete(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"RNS\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"C22\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tReplicas: 2,\n\t\tStorage: nats.FileStorage,\n\t\tMaxMsgs: 100,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Error adding stream: %v\", err)\n\t}\n\n\t// Now create a consumer.\n\tif _, err := js.AddConsumer(\"C22\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error adding consumer: %v\", err)\n\t}\n\n\t// Now delete the consumer.\n\tif err := js.DeleteConsumer(\"C22\", \"dlc\"); err != nil {\n\t\tt.Fatalf(\"Error deleting consumer: %v\", err)\n\t}\n\n\t// Now delete the stream.\n\tif err := js.DeleteStream(\"C22\"); err != nil {\n\t\tt.Fatalf(\"Error deleting stream: %v\", err)\n\t}\n\n\t// This will get the current information about usage and limits for this account.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tinfo, err := js.AccountInfo()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif info.Streams != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no remaining streams, got %d\", info.Streams)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterStreamPurge(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\n\t// Now purge the stream.\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif si.State.Msgs != 0 || si.State.FirstSeq != uint64(toSend+1) {\n\t\tt.Fatalf(\"Expected no msgs, got: %+v\", si.State)\n\t}\n}\n\nfunc TestJetStreamClusterStreamUpdateSubjects(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t}\n\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure we can update subjects.\n\tcfg.Subjects = []string{\"bar\", \"baz\"}\n\n\tsi, err := js.UpdateStream(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil {\n\t\tt.Fatalf(\"Expected a stream info, got none\")\n\t}\n\tif !reflect.DeepEqual(si.Config.Subjects, cfg.Subjects) {\n\t\tt.Fatalf(\"Expected subjects to be updated: got %+v\", si.Config.Subjects)\n\t}\n\t// Make sure it registered\n\tjs2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err = js2.Publish(\"foo\", nil); err == nil {\n\t\tt.Fatalf(\"Expected this to fail\")\n\t}\n\tif _, err = js2.Publish(\"baz\", nil); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterBadStreamUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t}\n\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tmsg, toSend := []byte(\"Keep Me\"), 50\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Make sure a bad update will not remove our stream.\n\tcfg.Subjects = []string{\"foo..bar\"}\n\tif _, err := js.UpdateStream(cfg); err == nil || err == nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected error but got none or timeout\")\n\t}\n\n\t// Make sure we did not delete our original stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !reflect.DeepEqual(si.Config.Subjects, []string{\"foo\", \"bar\"}) {\n\t\tt.Fatalf(\"Expected subjects to be original ones, got %+v\", si.Config.Subjects)\n\t}\n}\n\nfunc TestJetStreamClusterConsumerRedeliveredInfo(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{Name: \"TEST\"}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.Publish(\"TEST\", []byte(\"CI\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tsub, _ := nc.SubscribeSync(\"R\")\n\tsub.AutoUnsubscribe(2)\n\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: \"R\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: 100 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, 2)\n\tsub.Unsubscribe()\n\n\tci, err = js.ConsumerInfo(\"TEST\", ci.Name)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci.NumRedelivered != 1 {\n\t\tt.Fatalf(\"Expected 1 redelivered, got %d\", ci.NumRedelivered)\n\t}\n}\n\nfunc TestJetStreamClusterConsumerState(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 5)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Make sure we are not connected to any of the stream servers so that we do not do client reconnect\n\t// when we take out the consumer leader.\n\tif s.JetStreamIsStreamAssigned(\"$G\", \"TEST\") {\n\t\tnc.Close()\n\t\tfor _, ns := range c.servers {\n\t\t\tif !ns.JetStreamIsStreamAssigned(\"$G\", \"TEST\") {\n\t\t\t\ts = ns\n\t\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Pull 5 messages and ack.\n\tfor _, m := range fetchMsgs(t, sub, 5, 5*time.Second) {\n\t\tm.AckSync()\n\t}\n\n\t// Let state propagate for exact comparison below.\n\ttime.Sleep(200 * time.Millisecond)\n\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\tif ci.AckFloor.Consumer != 5 {\n\t\tt.Fatalf(\"Expected ack floor of %d, got %d\", 5, ci.AckFloor.Consumer)\n\t}\n\n\tc.consumerLeader(\"$G\", \"TEST\", \"dlc\").Shutdown()\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\n\tnci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\t// nil out timestamp for better comparison\n\tnci.Delivered.Last, ci.Delivered.Last = nil, nil\n\tif nci.Delivered != ci.Delivered {\n\t\tt.Fatalf(\"Consumer delivered did not match after leader switch, wanted %+v, got %+v\", ci.Delivered, nci.Delivered)\n\t}\n\tnci.AckFloor.Last, ci.AckFloor.Last = nil, nil\n\tif nci.AckFloor != ci.AckFloor {\n\t\tt.Fatalf(\"Consumer ackfloor did not match after leader switch, wanted %+v, got %+v\", ci.AckFloor, nci.AckFloor)\n\t}\n\n\t// Now make sure we can receive new messages.\n\t// Pull last 5.\n\tfor _, m := range fetchMsgs(t, sub, 5, 5*time.Second) {\n\t\tm.AckSync()\n\t}\n\n\tnci, _ = sub.ConsumerInfo()\n\tif nci.Delivered.Consumer != 10 || nci.Delivered.Stream != 10 {\n\t\tt.Fatalf(\"Received bad delivered: %+v\", nci.Delivered)\n\t}\n\tif nci.AckFloor.Consumer != 10 || nci.AckFloor.Stream != 10 {\n\t\tt.Fatalf(\"Received bad ackfloor: %+v\", nci.AckFloor)\n\t}\n\tif nci.NumAckPending != 0 {\n\t\tt.Fatalf(\"Received bad ackpending: %+v\", nci.NumAckPending)\n\t}\n}\n\nfunc TestJetStreamClusterFullConsumerState(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfetchMsgs(t, sub, 1, 5*time.Second)\n\n\t// Now purge the stream.\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterMetaSnapshotsAndCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Shut one down.\n\trs := c.randomServer()\n\trs.Shutdown()\n\n\tc.waitOnLeader()\n\ts := c.leader()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tnumStreams := 4\n\t// Create 4 streams\n\t// FIXME(dlc) - R2 make sure we place properly.\n\tfor i := 0; i < numStreams; i++ {\n\t\tsn := fmt.Sprintf(\"T-%d\", i+1)\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: sn})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tc.leader().JetStreamSnapshotMeta()\n\n\trs = c.restartServer(rs)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(rs)\n\n\trs.Shutdown()\n\tc.waitOnLeader()\n\n\tfor i := 0; i < numStreams; i++ {\n\t\tsn := fmt.Sprintf(\"T-%d\", i+1)\n\t\terr := js.DeleteStream(sn)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\trs = c.restartServer(rs)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(rs)\n}\n\nfunc TestJetStreamClusterMetaSnapshotsMultiChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 2)\n\tdefer c.shutdown()\n\n\ts := c.leader()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Add in 2 streams with 1 consumer each.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"S1\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnStreamLeader(globalAccountName, \"S1\")\n\t_, err := js.AddConsumer(\"S1\", &nats.ConsumerConfig{Durable: \"S1C1\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnConsumerLeader(globalAccountName, \"S1\", \"S1C1\")\n\n\tif _, err = js.AddStream(&nats.StreamConfig{Name: \"S2\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnStreamLeader(globalAccountName, \"S2\")\n\t_, err = js.AddConsumer(\"S2\", &nats.ConsumerConfig{Durable: \"S2C1\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnConsumerLeader(globalAccountName, \"S2\", \"S2C1\")\n\n\t// Add in a new server to the group. This way we know we can delete the original streams and consumers.\n\trs := c.addInNewServer()\n\tc.waitOnServerCurrent(rs)\n\trsn := rs.Name()\n\n\t// Shut it down.\n\trs.Shutdown()\n\n\t// Wait for the peer to be removed.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, p := range s.JetStreamClusterPeers() {\n\t\t\tif p == rsn {\n\t\t\t\treturn fmt.Errorf(\"Old server still in peer set\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// We want to make changes here that test each delta scenario for the meta snapshots.\n\t// Add new stream and consumer.\n\tif _, err = js.AddStream(&nats.StreamConfig{Name: \"S3\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnStreamLeader(globalAccountName, \"S3\")\n\t_, err = js.AddConsumer(\"S3\", &nats.ConsumerConfig{Durable: \"S3C1\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnConsumerLeader(globalAccountName, \"S3\", \"S3C1\")\n\t// Delete stream S2\n\tresp, _ := nc.Request(fmt.Sprintf(JSApiStreamDeleteT, \"S2\"), nil, time.Second)\n\tvar dResp JSApiStreamDeleteResponse\n\tif err := json.Unmarshal(resp.Data, &dResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !dResp.Success || dResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", dResp.Error)\n\t}\n\t// Delete the consumer on S1 but add another.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerDeleteT, \"S1\", \"S1C1\"), nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, resp)\n\tvar cdResp JSApiConsumerDeleteResponse\n\tif err = json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !cdResp.Success || cdResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", cdResp)\n\t}\n\t// Add new consumer on S1\n\t_, err = js.AddConsumer(\"S1\", &nats.ConsumerConfig{Durable: \"S1C2\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnConsumerLeader(globalAccountName, \"S1\", \"S1C2\")\n\n\tcl := c.leader()\n\tcl.JetStreamSnapshotMeta()\n\tc.waitOnServerCurrent(cl)\n\n\trs = c.restartServer(rs)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(rs)\n}\n\nfunc TestJetStreamClusterStreamSynchedTimeStamps(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Storage: nats.MemoryStorage, Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err = js.Publish(\"foo\", []byte(\"TSS\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\t// Grab the message and timestamp from our current leader\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tm, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmeta, _ := m.Metadata()\n\n\tsub.Unsubscribe()\n\n\tsl := c.streamLeader(\"$G\", \"foo\")\n\n\tsl.Shutdown()\n\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"foo\")\n\n\tnc, js = jsClientConnect(t, c.leader())\n\tdefer nc.Close()\n\n\tsm, err := js.GetMsg(\"foo\", 1)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !sm.Time.Equal(meta.Timestamp) {\n\t\tt.Fatalf(\"Expected same timestamps, got %v vs %v\", sm.Time, meta.Timestamp)\n\t}\n}\n\n// Test to mimic what R.I. was seeing.\nfunc TestJetStreamClusterRestoreSingleConsumer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err = js.Publish(\"foo\", []byte(\"TSS\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif m, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t} else {\n\t\tm.AckSync()\n\t}\n\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"foo\")\n\n\ts = c.randomServer()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tvar names []string\n\tfor name := range js.StreamNames() {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(names))\n\t}\n\n\t// Now do detailed version.\n\tvar infos []*nats.StreamInfo\n\tfor info := range js.StreamsInfo() {\n\t\tinfos = append(infos, info)\n\t}\n\tif len(infos) != 1 {\n\t\tt.Fatalf(\"Expected 1 stream but got %d\", len(infos))\n\t}\n\tsi, err := js.StreamInfo(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"foo\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\n\t// Now check for consumer.\n\tnames = names[:0]\n\tfor name := range js.ConsumerNames(\"foo\") {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected 1 consumer but got %d\", len(names))\n\t}\n}\n\nfunc TestJetStreamClusterMaxBytesForStream(t *testing.T) {\n\t// Has max_file_store of 2GB\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tinfo, err := js.AccountInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Make sure we still are dynamic.\n\tif info.Limits.MaxStore != -1 || info.Limits.MaxMemory != -1 {\n\t\tt.Fatalf(\"Expected dynamic limits for the account, got %+v\\n\", info.Limits)\n\t}\n\t// Stream config.\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 2,\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024, // 2GB\n\t}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Make sure going over the single server limit though is enforced (for now).\n\tcfg.Name = \"TEST2\"\n\tcfg.MaxBytes *= 2\n\t_, err = js.AddStream(cfg)\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\")\n}\n\nfunc TestJetStreamClusterStreamPublishWithActiveConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err = js.Publish(\"foo\", []byte(\"TSS\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// FIXME(dlc) - Need to track this down.\n\tc.waitOnConsumerLeader(\"$G\", \"foo\", \"dlc\")\n\n\tif m, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t} else {\n\t\tm.AckSync()\n\t}\n\n\t// Send 10 messages.\n\tfor i := 1; i <= 10; i++ {\n\t\tpayload := []byte(fmt.Sprintf(\"MSG-%d\", i))\n\t\tif _, err = js.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tcheckSubsPending(t, sub, 10)\n\t// Sanity check for duplicate deliveries..\n\tif nmsgs, _, _ := sub.Pending(); nmsgs > 10 {\n\t\tt.Fatalf(\"Expected only %d responses, got %d more\", 10, nmsgs)\n\t}\n\tfor i := 1; i <= 10; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tpayload := []byte(fmt.Sprintf(\"MSG-%d\", i))\n\t\tif !bytes.Equal(m.Data, payload) {\n\t\t\tt.Fatalf(\"Did not get expected msg, expected %q, got %q\", payload, m.Data)\n\t\t}\n\t}\n\n\tvar ci *nats.ConsumerInfo\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tci, err = sub.ConsumerInfo()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error getting consumer info: %v\", err)\n\t\t}\n\t\tif ci.Delivered.Stream != 11 {\n\t\t\treturn fmt.Errorf(\"expected delivered stream sequence to be %d, got %d\", 11, ci.Delivered.Stream)\n\t\t}\n\t\treturn nil\n\t})\n\n\tc.consumerLeader(\"$G\", \"foo\", \"dlc\").Shutdown()\n\tc.waitOnConsumerLeader(\"$G\", \"foo\", \"dlc\")\n\n\tci2, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\n\tci.Cluster = nil\n\tci2.Cluster = nil\n\n\t// nil out timestamp for better comparison\n\tci.Delivered.Last, ci2.Delivered.Last = nil, nil\n\tci.AckFloor.Last, ci2.AckFloor.Last = nil, nil\n\tif !reflect.DeepEqual(ci, ci2) {\n\t\tt.Fatalf(\"Consumer info did not match: %+v vs %+v\", ci, ci2)\n\t}\n\n\t// In case the server above was also stream leader.\n\tc.waitOnStreamLeader(\"$G\", \"foo\")\n\n\t// Now send more..\n\t// Send 10 more messages.\n\tfor i := 11; i <= 20; i++ {\n\t\tpayload := []byte(fmt.Sprintf(\"MSG-%d\", i))\n\t\tif _, err = js.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcheckSubsPending(t, sub, 10)\n\t// Sanity check for duplicate deliveries..\n\tif nmsgs, _, _ := sub.Pending(); nmsgs > 10 {\n\t\tt.Fatalf(\"Expected only %d responses, got %d more\", 10, nmsgs)\n\t}\n\n\tfor i := 11; i <= 20; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tpayload := []byte(fmt.Sprintf(\"MSG-%d\", i))\n\t\tif !bytes.Equal(m.Data, payload) {\n\t\t\tt.Fatalf(\"Did not get expected msg, expected %q, got %q\", payload, m.Data)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterStreamOverlapSubjects(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST2\", Subjects: []string{\"foo\"}}); err == nil || err == nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected error but got none or timeout: %v\", err)\n\t}\n\n\t// Now grab list of streams and make sure the second is not there.\n\tvar names []string\n\tfor name := range js.StreamNames() {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(names))\n\t}\n\n\t// Now do a detailed version.\n\tvar infos []*nats.StreamInfo\n\tfor info := range js.StreamsInfo() {\n\t\tinfos = append(infos, info)\n\t}\n\tif len(infos) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(infos))\n\t}\n}\n\nfunc TestJetStreamClusterStreamInfoList(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcreateStream := func(name string) {\n\t\tt.Helper()\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tcreateStream(\"foo\")\n\tcreateStream(\"bar\")\n\tcreateStream(\"baz\")\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tsendBatch(\"foo\", 10)\n\tsendBatch(\"bar\", 22)\n\tsendBatch(\"baz\", 33)\n\n\t// Now get the stream list info.\n\tvar infos []*nats.StreamInfo\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tinfos = infos[:0]\n\t\tfor info := range js.StreamsInfo() {\n\t\t\tinfos = append(infos, info)\n\t\t}\n\t\tif len(infos) != 3 {\n\t\t\treturn fmt.Errorf(\"StreamInfo expected 3 results, got %d\", len(infos))\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor _, si := range infos {\n\t\tswitch si.Config.Name {\n\t\tcase \"foo\":\n\t\t\tif si.State.Msgs != 10 {\n\t\t\t\tt.Fatalf(\"Expected %d msgs but got %d\", 10, si.State.Msgs)\n\t\t\t}\n\t\tcase \"bar\":\n\t\t\tif si.State.Msgs != 22 {\n\t\t\t\tt.Fatalf(\"Expected %d msgs but got %d\", 22, si.State.Msgs)\n\t\t\t}\n\t\tcase \"baz\":\n\t\t\tif si.State.Msgs != 33 {\n\t\t\t\tt.Fatalf(\"Expected %d msgs but got %d\", 33, si.State.Msgs)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterConsumerInfoList(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Place messages so we can generate consumer state.\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcreateConsumer := func(name string) *nats.Subscription {\n\t\tt.Helper()\n\t\tsub, err := js.PullSubscribe(\"TEST\", name)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\treturn sub\n\t}\n\n\tsubFoo := createConsumer(\"foo\")\n\tsubBar := createConsumer(\"bar\")\n\tsubBaz := createConsumer(\"baz\")\n\n\t// Place consumers in various states.\n\tfor _, ss := range []struct {\n\t\tsub *nats.Subscription\n\t\tfetch int\n\t\tack int\n\t}{\n\t\t{subFoo, 4, 2},\n\t\t{subBar, 2, 0},\n\t\t{subBaz, 8, 6},\n\t} {\n\t\tmsgs := fetchMsgs(t, ss.sub, ss.fetch, 5*time.Second)\n\t\tfor i := 0; i < ss.ack; i++ {\n\t\t\tmsgs[i].AckSync()\n\t\t}\n\t}\n\n\t// Now get the consumer list info.\n\tvar infos []*nats.ConsumerInfo\n\tfor info := range js.ConsumersInfo(\"TEST\") {\n\t\tinfos = append(infos, info)\n\t}\n\tif len(infos) != 3 {\n\t\tt.Fatalf(\"ConsumerInfo expected 3 results, got %d\", len(infos))\n\t}\n\tfor _, ci := range infos {\n\t\tswitch ci.Name {\n\t\tcase \"foo\":\n\t\t\tif ci.Delivered.Consumer != 4 {\n\t\t\t\tt.Fatalf(\"Expected %d delivered but got %d\", 4, ci.Delivered.Consumer)\n\t\t\t}\n\t\t\tif ci.AckFloor.Consumer != 2 {\n\t\t\t\tt.Fatalf(\"Expected %d for ack floor but got %d\", 2, ci.AckFloor.Consumer)\n\t\t\t}\n\t\tcase \"bar\":\n\t\t\tif ci.Delivered.Consumer != 2 {\n\t\t\t\tt.Fatalf(\"Expected %d delivered but got %d\", 2, ci.Delivered.Consumer)\n\t\t\t}\n\t\t\tif ci.AckFloor.Consumer != 0 {\n\t\t\t\tt.Fatalf(\"Expected %d for ack floor but got %d\", 0, ci.AckFloor.Consumer)\n\t\t\t}\n\t\tcase \"baz\":\n\t\t\tif ci.Delivered.Consumer != 8 {\n\t\t\t\tt.Fatalf(\"Expected %d delivered but got %d\", 8, ci.Delivered.Consumer)\n\t\t\t}\n\t\t\tif ci.AckFloor.Consumer != 6 {\n\t\t\t\tt.Fatalf(\"Expected %d for ack floor but got %d\", 6, ci.AckFloor.Consumer)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterStreamUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsc := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tMaxMsgs: 10,\n\t\tDiscard: DiscardNew,\n\t}\n\n\tif _, err := js.AddStream(sc); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfor i := 1; i <= int(sc.MaxMsgs); i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", i))\n\t\tif _, err := js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Expect error here.\n\tif _, err := js.Publish(\"foo\", []byte(\"fail\")); err == nil {\n\t\tt.Fatalf(\"Expected publish to fail\")\n\t}\n\n\t// Now update MaxMsgs, select non-leader\n\ts = c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsc.MaxMsgs = 20\n\tsi, err := js.UpdateStream(sc)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Config.MaxMsgs != 20 {\n\t\tt.Fatalf(\"Expected to have config updated with max msgs of %d, got %d\", 20, si.Config.MaxMsgs)\n\t}\n\n\t// Do one that will fail. Wait and make sure we only are getting one response.\n\tsc.Name = \"TEST22\"\n\n\trsub, _ := nc.SubscribeSync(nats.NewInbox())\n\tdefer rsub.Unsubscribe()\n\tnc.Flush()\n\n\treq, _ := json.Marshal(sc)\n\tif err := nc.PublishRequest(fmt.Sprintf(JSApiStreamUpdateT, \"TEST\"), rsub.Subject, req); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Wait incase more than one reply sent.\n\ttime.Sleep(250 * time.Millisecond)\n\n\tif nmsgs, _, _ := rsub.Pending(); err != nil || nmsgs != 1 {\n\t\tt.Fatalf(\"Expected only one response, got %d\", nmsgs)\n\t}\n\n\tm, err := rsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t}\n\n\tvar scResp JSApiStreamCreateResponse\n\tif err := json.Unmarshal(m.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.StreamInfo != nil || scResp.Error == nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", scResp)\n\t}\n}\n\nfunc TestJetStreamClusterStreamExtendedUpdates(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tupdateStream := func() *nats.StreamInfo {\n\t\tsi, err := js.UpdateStream(cfg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\treturn si\n\t}\n\n\t// Subjects can be updated\n\tcfg.Subjects = []string{\"bar\", \"baz\"}\n\tif si := updateStream(); !reflect.DeepEqual(si.Config.Subjects, cfg.Subjects) {\n\t\tt.Fatalf(\"Did not get expected stream info: %+v\", si)\n\t}\n\t// Mirror changes are not supported for now\n\tcfg.Subjects = nil\n\tcfg.Mirror = &nats.StreamSource{Name: \"ORDERS\"}\n\t_, err := js.UpdateStream(cfg)\n\trequire_Error(t, err, NewJSStreamMirrorNotUpdatableError())\n}\n\nfunc TestJetStreamClusterDoubleAdd(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R32\", 2)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Streams should allow double add.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Check Consumers.\n\tcfg := &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy}\n\tif _, err := js.AddConsumer(\"TEST\", cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Check double add ok.\n\tif _, err := js.AddConsumer(\"TEST\", cfg); err != nil {\n\t\tt.Fatalf(\"Expected no error but got: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterDefaultMaxAckPending(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R32\", 2)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Do Consumers too.\n\tcfg := &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy}\n\tci, err := js.AddConsumer(\"TEST\", cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Check that we have a default set now for the max ack pending.\n\tif ci.Config.MaxAckPending != JsDefaultMaxAckPending {\n\t\tt.Fatalf(\"Expected a default for max ack pending of %d, got %d\", JsDefaultMaxAckPending, ci.Config.MaxAckPending)\n\t}\n}\n\nfunc TestJetStreamClusterStreamNormalCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", i))\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tsl.Shutdown()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Send 10 more while one replica offline.\n\tfor i := toSend; i <= toSend*2; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", i))\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Delete the first from the second batch.\n\tdreq := JSApiMsgDeleteRequest{Seq: uint64(toSend)}\n\tdreqj, err := json.Marshal(dreq)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp, _ := nc.Request(fmt.Sprintf(JSApiMsgDeleteT, \"TEST\"), dreqj, time.Second)\n\tvar delMsgResp JSApiMsgDeleteResponse\n\tif err = json.Unmarshal(resp.Data, &delMsgResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !delMsgResp.Success || delMsgResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", delMsgResp.Error)\n\t}\n\n\tsl = c.restartServer(sl)\n\tc.checkClusterFormed()\n\n\tc.waitOnServerCurrent(sl)\n\tc.waitOnStreamCurrent(sl, \"$G\", \"TEST\")\n}\n\nfunc TestJetStreamClusterStreamSnapshotCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tpseq := uint64(1)\n\tsendBatch := func(n int) {\n\t\tt.Helper()\n\t\t// Send a batch.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", pseq))\n\t\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t\tpseq++\n\t\t}\n\t}\n\n\tsendBatch(2)\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tsl.Shutdown()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tsendBatch(100)\n\n\tdeleteMsg := func(seq uint64) {\n\t\tif err := js.DeleteMsg(\"TEST\", seq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Delete the first from the second batch.\n\tdeleteMsg(pseq / 2)\n\t// Delete the next one too.\n\tdeleteMsg(pseq/2 + 1)\n\n\tnsl := c.streamLeader(\"$G\", \"TEST\")\n\tnsl.JetStreamSnapshotStream(\"$G\", \"TEST\")\n\n\t// Do some activity post snapshot as well.\n\t// Delete next to last.\n\tdeleteMsg(pseq - 2)\n\t// Send another batch.\n\tsendBatch(100)\n\n\tmset, err := nsl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tostate := mset.stateWithDetail(true)\n\n\tsl = c.restartServer(sl)\n\tc.checkClusterFormed()\n\n\tc.waitOnServerCurrent(sl)\n\tc.waitOnStreamCurrent(sl, \"$G\", \"TEST\")\n\n\tmset, err = sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif nstate := mset.stateWithDetail(true); !reflect.DeepEqual(ostate, nstate) {\n\t\t\treturn fmt.Errorf(\"States do not match after recovery: %+v vs %+v\", ostate, nstate)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterDeleteMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// R=1 make sure delete works.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", i))\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tdeleteMsg := func(seq uint64) {\n\t\tif err := js.DeleteMsg(\"TEST\", seq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tdeleteMsg(1)\n\n\t// Also make sure purge of R=1 works too.\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterDeleteMsgAndRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// R=1 make sure delete works.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"HELLO JSC-%d\", i))\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tdeleteMsg := func(seq uint64) {\n\t\tif err := js.DeleteMsg(\"TEST\", seq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tdeleteMsg(1)\n\n\tc.stopAll()\n\tc.restartAll()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n}\n\nfunc TestJetStreamClusterStreamSnapshotCatchupWithPurge(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\n\tsl.Shutdown()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tnsl := c.streamLeader(\"$G\", \"TEST\")\n\tif err := nsl.JetStreamSnapshotStream(\"$G\", \"TEST\"); err != nil {\n\t\tt.Fatalf(\"Error snapshotting stream: %v\", err)\n\t}\n\ttime.Sleep(250 * time.Millisecond)\n\n\tsl = c.restartServer(sl)\n\tc.checkClusterFormed()\n\n\t// Now purge the stream while we are recovering.\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\n\tc.waitOnServerCurrent(sl)\n\tc.waitOnStreamCurrent(sl, \"$G\", \"TEST\")\n\n\tnsl.Shutdown()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterExtendedStreamInfo(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 50\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tleader := c.streamLeader(\"$G\", \"TEST\").Name()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected cluster info\")\n\t}\n\tif si.Cluster.Name != c.name {\n\t\tt.Fatalf(\"Expected cluster name of %q, got %q\", c.name, si.Cluster.Name)\n\t}\n\n\tif si.Cluster.Leader != leader {\n\t\tt.Fatalf(\"Expected leader of %q, got %q\", leader, si.Cluster.Leader)\n\t}\n\tif len(si.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Expected %d replicas, got %d\", 2, len(si.Cluster.Replicas))\n\t}\n\n\t// Make sure that returned array is ordered\n\tfor i := 0; i < 50; i++ {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(si.Cluster.Replicas) == 2)\n\t\ts1 := si.Cluster.Replicas[0].Name\n\t\ts2 := si.Cluster.Replicas[1].Name\n\t\tif s1 > s2 {\n\t\t\tt.Fatalf(\"Expected replicas to be ordered, got %s then %s\", s1, s2)\n\t\t}\n\t}\n\n\t// We may need to wait a bit for peers to catch up.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$G\", \"TEST\")\n\t})\n\n\t// Shutdown the leader.\n\toldLeader := c.streamLeader(\"$G\", \"TEST\")\n\toldLeader.Shutdown()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Re-request.\n\tleader = c.streamLeader(\"$G\", \"TEST\").Name()\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected cluster info\")\n\t}\n\tif si.Cluster.Leader != leader {\n\t\tt.Fatalf(\"Expected leader of %q, got %q\", leader, si.Cluster.Leader)\n\t}\n\tif len(si.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Expected %d replicas, got %d\", 2, len(si.Cluster.Replicas))\n\t}\n\tfor _, peer := range si.Cluster.Replicas {\n\t\tif peer.Name == oldLeader.Name() {\n\t\t\tif peer.Current {\n\t\t\t\tt.Fatalf(\"Expected old leader to be reported as not current: %+v\", peer)\n\t\t\t}\n\t\t} else if !peer.Current {\n\t\t\tt.Fatalf(\"Expected replica to be current: %+v\", peer)\n\t\t}\n\t}\n\n\t// Now send a few more messages then restart the oldLeader.\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\toldLeader = c.restartServer(oldLeader)\n\tc.checkClusterFormed()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamCurrent(oldLeader, \"$G\", \"TEST\")\n\n\t// Re-request.\n\tleader = c.streamLeader(\"$G\", \"TEST\").Name()\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected cluster info\")\n\t}\n\tif si.Cluster.Leader != leader {\n\t\tt.Fatalf(\"Expected leader of %q, got %q\", leader, si.Cluster.Leader)\n\t}\n\tif len(si.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Expected %d replicas, got %d\", 2, len(si.Cluster.Replicas))\n\t}\n\n\t// We may need to wait a bit for peers to catch up.\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$G\", \"TEST\")\n\t})\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now do consumer.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\tfetchMsgs(t, sub, 10, 5*time.Second)\n\n\tleader = c.consumerLeader(\"$G\", \"TEST\", \"dlc\").Name()\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\n\tif ci.Cluster.Leader != leader {\n\t\tt.Fatalf(\"Expected leader of %q, got %q\", leader, ci.Cluster.Leader)\n\t}\n\tif len(ci.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Expected %d replicas, got %d\", 2, len(ci.Cluster.Replicas))\n\t}\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$G\", \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterExtendedStreamInfoSingleReplica(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 50\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tleader := c.streamLeader(\"$G\", \"TEST\").Name()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil {\n\t\tt.Fatalf(\"Expected cluster info\")\n\t}\n\tif si.Cluster.Name != c.name {\n\t\tt.Fatalf(\"Expected cluster name of %q, got %q\", c.name, si.Cluster.Name)\n\t}\n\tif si.Cluster.Leader != leader {\n\t\tt.Fatalf(\"Expected leader of %q, got %q\", leader, si.Cluster.Leader)\n\t}\n\tif len(si.Cluster.Replicas) != 0 {\n\t\tt.Fatalf(\"Expected no replicas but got %d\", len(si.Cluster.Replicas))\n\t}\n\n\t// Make sure we can grab consumer lists from any\n\tvar infos []*nats.ConsumerInfo\n\tfor info := range js.ConsumersInfo(\"TEST\") {\n\t\tinfos = append(infos, info)\n\t}\n\tif len(infos) != 0 {\n\t\tt.Fatalf(\"ConsumerInfo expected no paged results, got %d\", len(infos))\n\t}\n\n\t// Now add in a consumer.\n\tcfg := &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy}\n\tif _, err := js.AddConsumer(\"TEST\", cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tinfos = infos[:0]\n\tfor info := range js.ConsumersInfo(\"TEST\") {\n\t\tinfos = append(infos, info)\n\t}\n\tif len(infos) != 1 {\n\t\tt.Fatalf(\"ConsumerInfo expected 1 result, got %d\", len(infos))\n\t}\n\n\t// Now do direct names list as well.\n\tvar names []string\n\tfor name := range js.ConsumerNames(\"TEST\") {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 consumer but got %d\", len(names))\n\t}\n}\n\nfunc TestJetStreamClusterInterestRetention(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Retention: nats.InterestPolicy, Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"foo\")\n\tcl := c.consumerLeader(\"$G\", \"foo\", \"dlc\")\n\tif sl == cl {\n\t\t_, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"foo\"), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tc.waitOnStreamLeader(\"$G\", \"foo\")\n\t}\n\n\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tm, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting msg: %v\", err)\n\t}\n\tm.AckSync()\n\n\twaitForZero := func() {\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"foo\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got state: %+v\", si.State)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\twaitForZero()\n\n\t// Add in 50 messages.\n\tfor i := 0; i < 50; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"more\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tcheckSubsPending(t, sub, 50)\n\n\t// Now delete the consumer and make sure the stream goes to zero.\n\tif err := js.DeleteConsumer(\"foo\", \"dlc\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\twaitForZero()\n}\n\n// https://github.com/nats-io/nats-server/issues/2243\nfunc TestJetStreamClusterWorkQueueRetention(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"FOO\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 2,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo.test\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err = js.Publish(\"foo.test\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\tsi, err := js.StreamInfo(\"FOO\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 1 {\n\t\tt.Fatalf(\"Expected 1 msg, got state: %+v\", si.State)\n\t}\n\n\t// Fetch from our pull consumer and ack.\n\tfor _, m := range fetchMsgs(t, sub, 1, 5*time.Second) {\n\t\tm.AckSync()\n\t}\n\n\t// Make sure the messages are removed.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"FOO\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamClusterMirrorAndSourceWorkQueues(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"WQ\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"WQ22\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: \"WQ22\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tReplicas: 2,\n\t\tSources: []*nats.StreamSource{{Name: \"WQ22\"}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Allow direct sync consumers to connect.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\tif _, err = js.Publish(\"foo\", []byte(\"ok\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tif si, _ := js.StreamInfo(\"WQ22\"); si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no msgs for %q, got %d\", \"WQ22\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"M\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for %q, got %d\", \"M\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"S\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for %q, got %d\", \"S\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamClusterMirrorAndSourceInterestPolicyStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"WQ\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"IP22\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: \"IP22\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tReplicas: 2,\n\t\tSources: []*nats.StreamSource{{Name: \"IP22\"}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Allow sync consumers to connect.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\tif _, err = js.Publish(\"foo\", []byte(\"ok\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\t// This one will be 0 since no other interest exists.\n\t\tif si, _ := js.StreamInfo(\"IP22\"); si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no msgs for %q, got %d\", \"IP22\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"M\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for %q, got %d\", \"M\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"S\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for %q, got %d\", \"S\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create other interest on IP22.\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\t// Allow consumer state to propagate.\n\ttime.Sleep(500 * time.Millisecond)\n\n\tif _, err = js.Publish(\"foo\", []byte(\"ok\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\t// This one will be 0 since no other interest exists.\n\t\tif si, _ := js.StreamInfo(\"IP22\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for %q, got %d\", \"IP22\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"M\"); si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs for %q, got %d\", \"M\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js.StreamInfo(\"S\"); si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs for %q, got %d\", \"S\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterInterestRetentionWithFilteredConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"*\"}, Retention: nats.InterestPolicy, Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"d1\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer fsub.Unsubscribe()\n\n\tbsub, err := js.SubscribeSync(\"bar\", nats.Durable(\"d2\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer bsub.Unsubscribe()\n\n\tmsg := []byte(\"FILTERED\")\n\tsendMsg := func(subj string) {\n\t\tt.Helper()\n\t\tif _, err = js.Publish(subj, msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tgetAndAck := func(sub *nats.Subscription) {\n\t\tt.Helper()\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting msg: %v\", err)\n\t\t}\n\t\tm.AckSync()\n\t}\n\n\tjsq, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckState := func(expected uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tt.Helper()\n\t\t\tsi, err := jsq.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", expected, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsendMsg(\"foo\")\n\tcheckState(1)\n\tgetAndAck(fsub)\n\tcheckState(0)\n\tsendMsg(\"bar\")\n\tsendMsg(\"foo\")\n\tcheckState(2)\n\tgetAndAck(bsub)\n\tcheckState(1)\n\tgetAndAck(fsub)\n\tcheckState(0)\n\n\t// Now send a bunch of messages and then delete the consumer.\n\tfor i := 0; i < 10; i++ {\n\t\tsendMsg(\"foo\")\n\t\tsendMsg(\"bar\")\n\t}\n\tcheckState(20)\n\n\tif err := js.DeleteConsumer(\"TEST\", \"d1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := js.DeleteConsumer(\"TEST\", \"d2\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckState(0)\n\n\t// Now make sure pull based consumers work same.\n\tif _, err := js.PullSubscribe(\"foo\", \"dlc\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now send a bunch of messages and then delete the consumer.\n\tfor i := 0; i < 10; i++ {\n\t\tsendMsg(\"foo\")\n\t\tsendMsg(\"bar\")\n\t}\n\tcheckState(10)\n\n\tif err := js.DeleteConsumer(\"TEST\", \"dlc\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckState(0)\n}\n\nfunc TestJetStreamClusterEphemeralConsumerNoImmediateInterest(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// We want to relax the strict interest requirement.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{DeliverSubject: \"r\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcl := c.consumerLeader(\"$G\", \"TEST\", ci.Name)\n\tmset, err := cl.GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", \"TEST\")\n\t}\n\to := mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\to.setInActiveDeleteThreshold(500 * time.Millisecond)\n\n\t// Make sure the consumer goes away though eventually.\n\t// Should be 5 seconds wait.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif _, err := js.ConsumerInfo(\"TEST\", ci.Name); err != nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Consumer still present\")\n\t})\n}\n\nfunc TestJetStreamClusterEphemeralConsumerCleanup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 2})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsub, err := js.Subscribe(\"foo\", func(m *nats.Msg) {})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tci, _ := sub.ConsumerInfo()\n\tif ci == nil {\n\t\tt.Fatalf(\"Unexpected error: no consumer info\")\n\t}\n\n\t// We will look up by hand this consumer to set inactive threshold lower for this test.\n\tcl := c.consumerLeader(\"$G\", \"foo\", ci.Name)\n\tif cl == nil {\n\t\tt.Fatalf(\"Could not find consumer leader\")\n\t}\n\tmset, err := cl.GlobalAccount().lookupStream(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", \"foo\")\n\t}\n\to := mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\to.setInActiveDeleteThreshold(10 * time.Millisecond)\n\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tgetConsumers := func() []string {\n\t\tctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)\n\t\tdefer cancel()\n\n\t\tvar names []string\n\t\tfor name := range js.ConsumerNames(\"foo\", nats.Context(ctx)) {\n\t\t\tnames = append(names, name)\n\t\t}\n\t\treturn names\n\t}\n\n\tcheckConsumer := func(expected int) {\n\t\tconsumers := getConsumers()\n\t\tif len(consumers) != expected {\n\t\t\tt.Fatalf(\"Expected %d consumers but got %d\", expected, len(consumers))\n\t\t}\n\t}\n\n\tcheckConsumer(1)\n\n\t// Now Unsubscribe, since this is ephemeral this will make this go away.\n\tsub.Unsubscribe()\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif consumers := getConsumers(); len(consumers) == 0 {\n\t\t\treturn nil\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"Still %d consumers remaining\", len(consumers))\n\t\t}\n\t})\n}\n\nfunc TestJetStreamClusterEphemeralConsumersNotReplicated(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tci, _ := sub.ConsumerInfo()\n\tif ci == nil {\n\t\tt.Fatalf(\"Unexpected error: no consumer info\")\n\t}\n\n\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, 1)\n\tsub.NextMsg(0)\n\n\tif ci.Cluster == nil || len(ci.Cluster.Replicas) != 0 {\n\t\tt.Fatalf(\"Expected ephemeral to be R=1, got %+v\", ci.Cluster)\n\t}\n\tscl := c.serverByName(ci.Cluster.Leader)\n\tif scl == nil {\n\t\tt.Fatalf(\"Could not select server where ephemeral consumer is running\")\n\t}\n\n\t// Test migrations. If we are also metadata leader will not work so skip.\n\tif scl == c.leader() {\n\t\treturn\n\t}\n\n\tscl.Shutdown()\n\tc.waitOnStreamLeader(\"$G\", \"foo\")\n\n\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tif _, err := sub.NextMsg(500 * time.Millisecond); err != nil {\n\t\tt.Logf(\"Expected to see another message, but behavior is optimistic so can fail\")\n\t}\n}\n\nfunc TestJetStreamClusterUserSnapshotAndRestore(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 200\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Create consumer with no state.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"rip\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create another consumer as well and give it a non-simplistic state.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy, AckWait: 10 * time.Second})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tjsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Ack first 50.\n\tfor _, m := range fetchMsgs(t, jsub, 50, 5*time.Second) {\n\t\trequire_NoError(t, m.AckSync())\n\t}\n\t// Now ack every third message for next 50.\n\tfor i, m := range fetchMsgs(t, jsub, 50, 5*time.Second) {\n\t\tif i%3 == 0 {\n\t\t\trequire_NoError(t, m.AckSync())\n\t\t}\n\t}\n\n\t// Snapshot consumer info.\n\tci, err := jsub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\n\tsreq := &JSApiStreamSnapshotRequest{\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tChunkSize: 512,\n\t}\n\n\treq, _ := json.Marshal(sreq)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"TEST\"), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\n\tvar resp JSApiStreamSnapshotResponse\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\t// Grab state for comparison.\n\tstate := *resp.State\n\tconfig := *resp.Config\n\n\tvar snapshot []byte\n\tdone := make(chan bool)\n\n\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t// EOF\n\t\tif len(m.Data) == 0 {\n\t\t\tdone <- true\n\t\t\treturn\n\t\t}\n\t\t// Could be writing to a file here too.\n\t\tsnapshot = append(snapshot, m.Data...)\n\t\t// Flow ack\n\t\tm.Respond(nil)\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\n\tvar rresp JSApiStreamRestoreResponse\n\trreq := &JSApiStreamRestoreRequest{\n\t\tConfig: config,\n\t\tState: state,\n\t}\n\treq, _ = json.Marshal(rreq)\n\n\t// Make sure a restore to an existing stream fails.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, \"TEST\"), req, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif !IsNatsErr(rresp.Error, JSStreamNameExistRestoreFailedErr) {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", rresp.Error)\n\t}\n\n\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now make sure a restore will work.\n\t// Delete our stream first.\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.StreamInfo(\"TEST\"); err == nil || !strings.Contains(err.Error(), \"not found\") {\n\t\tt.Fatalf(\"Expected not found error: %v\", err)\n\t}\n\n\t// This should work properly.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, \"TEST\"), req, 5*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\tif rresp.DeliverSubject == _EMPTY_ {\n\t\tt.Fatalf(\"No deliver subject set on response: %+v\", rresp)\n\t}\n\t// Send our snapshot back in to restore the stream.\n\t// Can be any size message.\n\tvar chunk [1024]byte\n\tfor r := bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tnc.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t}\n\trmsg, err = nc.Request(rresp.DeliverSubject, nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" || si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\n\t// Make sure the replicas become current eventually. They will be doing catchup.\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$G\", \"TEST\")\n\t})\n\n\t// Wait on the system to elect a leader for the restored consumer.\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\n\t// Now check for the consumer being recreated.\n\tnci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// nil out timestamp for better comparison\n\tnci.Delivered.Last, ci.Delivered.Last = nil, nil\n\tif nci.Delivered != ci.Delivered {\n\t\tt.Fatalf(\"Delivered states do not match %+v vs %+v\", nci.Delivered, ci.Delivered)\n\t}\n\tnci.AckFloor.Last, ci.AckFloor.Last = nil, nil\n\tif nci.AckFloor != ci.AckFloor {\n\t\tt.Fatalf(\"Ack floors did not match %+v vs %+v\", nci.AckFloor, ci.AckFloor)\n\t}\n\n\t// Make sure consumer works.\n\t// It should pick up with the next delivery spot, so check for that as first message.\n\t// We should have all the messages for first delivery delivered.\n\twantSeq := 101\n\tfor _, m := range fetchMsgs(t, jsub, 100, 5*time.Second) {\n\t\tmeta, err := m.Metadata()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif meta.Sequence.Stream != uint64(wantSeq) {\n\t\t\tt.Fatalf(\"Expected stream sequence of %d, but got %d\", wantSeq, meta.Sequence.Stream)\n\t\t}\n\t\trequire_NoError(t, m.AckSync())\n\t\twantSeq++\n\t}\n\n\t// Check that redelivered come in now..\n\tredelivered := 50/3 + 1\n\tfetchMsgs(t, jsub, redelivered, 15*time.Second)\n\n\t// Now make sure the other server was properly caughtup.\n\t// Need to call this by hand for now.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar sdResp JSApiStreamLeaderStepDownResponse\n\tif err := json.Unmarshal(rmsg.Data, &sdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif sdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", sdResp.Error)\n\t}\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", err)\n\t}\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Unexpected stream info: %+v\", si)\n\t}\n\n\t// Check idle consumer\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"rip\")\n\n\t// Now check for the consumer being recreated.\n\tif _, err := js.ConsumerInfo(\"TEST\", \"rip\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterUserSnapshotAndRestoreConfigChanges(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// FIXME(dlc) - Do case with R=1\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t}\n\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tgetSnapshot := func() ([]byte, *StreamState) {\n\t\tt.Helper()\n\t\tsreq := &JSApiStreamSnapshotRequest{\n\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\tChunkSize: 1024,\n\t\t}\n\n\t\treq, _ := json.Marshal(sreq)\n\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"TEST\"), req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t\t}\n\n\t\tvar resp JSApiStreamSnapshotResponse\n\t\tjson.Unmarshal(rmsg.Data, &resp)\n\t\tif resp.Error != nil {\n\t\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t\t}\n\n\t\tvar snapshot []byte\n\t\tdone := make(chan bool)\n\n\t\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t\t// EOF\n\t\t\tif len(m.Data) == 0 {\n\t\t\t\tdone <- true\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Could be writing to a file here too.\n\t\t\tsnapshot = append(snapshot, m.Data...)\n\t\t\t// Flow ack\n\t\t\tm.Respond(nil)\n\t\t})\n\t\tdefer sub.Unsubscribe()\n\n\t\t// Wait to receive the snapshot.\n\t\tselect {\n\t\tcase <-done:\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t\t}\n\t\treturn snapshot, resp.State\n\t}\n\n\trestore := func(cfg *StreamConfig, state *StreamState, snap []byte) *nats.StreamInfo {\n\t\trreq := &JSApiStreamRestoreRequest{\n\t\t\tConfig: *cfg,\n\t\t\tState: *state,\n\t\t}\n\t\treq, err := json.Marshal(rreq)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, cfg.Name), req, 5*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar rresp JSApiStreamRestoreResponse\n\t\tjson.Unmarshal(rmsg.Data, &rresp)\n\t\tif rresp.Error != nil {\n\t\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t\t}\n\t\tif rresp.DeliverSubject == _EMPTY_ {\n\t\t\tt.Fatalf(\"No deliver subject set on response: %+v\", rresp)\n\t\t}\n\t\t// Send our snapshot back in to restore the stream.\n\t\t// Can be any size message.\n\t\tvar chunk [1024]byte\n\t\tfor r := bytes.NewReader(snap); ; {\n\t\t\tn, err := r.Read(chunk[:])\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tnc.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t\t}\n\t\trmsg, err = nc.Request(rresp.DeliverSubject, nil, 2*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\trresp.Error = nil\n\t\tjson.Unmarshal(rmsg.Data, &rresp)\n\t\tif rresp.Error != nil {\n\t\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t\t}\n\t\tsi, err := js.StreamInfo(cfg.Name)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\treturn si\n\t}\n\n\tsnap, state := getSnapshot()\n\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now change subjects.\n\tncfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"bar\", \"baz\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 2,\n\t}\n\tif si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) {\n\t\tt.Fatalf(\"Did not get expected stream info: %+v\", si)\n\t}\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Storage\n\tncfg.Storage = MemoryStorage\n\tif si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) {\n\t\tt.Fatalf(\"Did not get expected stream info: %+v\", si)\n\t}\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Now replicas\n\tncfg.Replicas = 3\n\tif si := restore(ncfg, state, snap); !reflect.DeepEqual(si.Config.Subjects, ncfg.Subjects) {\n\t\tt.Fatalf(\"Did not get expected stream info: %+v\", si)\n\t}\n}\n\nfunc TestJetStreamClusterAccountInfoAndLimits(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\t// Adjust our limits.\n\tc.updateLimits(\"$G\", map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 1024,\n\t\t\tMaxStore: 8000,\n\t\t\tMaxStreams: 3,\n\t\t\tMaxConsumers: 2,\n\t\t},\n\t})\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"baz\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create with same config is idempotent, and must not exceed max streams as it already exists.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"baz\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"JSC-OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tsendBatch(\"foo\", 25)\n\tsendBatch(\"bar\", 75)\n\tsendBatch(\"baz\", 10)\n\n\taccountStats := func() *nats.AccountInfo {\n\t\tt.Helper()\n\n\t\tinfo, err := js.AccountInfo()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\treturn info\n\t}\n\n\t// If subject is not 3 letters or payload not 2 this needs to change.\n\tconst msgSize = uint64(22 + 3 + 6 + 8)\n\n\tstats := accountStats()\n\tif stats.Streams != 3 {\n\t\tt.Fatalf(\"Should have been tracking 3 streams, found %d\", stats.Streams)\n\t}\n\texpectedSize := 25*msgSize + 75*msgSize*2 + 10*msgSize*3\n\t// This may lag.\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\tif stats.Store != expectedSize {\n\t\t\terr := fmt.Errorf(\"Expected store size to be %d, got %+v\\n\", expectedSize, stats)\n\t\t\tstats = accountStats()\n\t\t\treturn err\n\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check limit enforcement.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"fail\", Replicas: 3}); err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\n\t// We should be at 7995 at the moment with a limit of 8000, so any message will go over.\n\tif _, err := js.Publish(\"baz\", []byte(\"JSC-NOT-OK\")); err == nil {\n\t\tt.Fatalf(\"Expected publish error but got none\")\n\t}\n\n\t// Check consumers\n\t_, err := js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create (with explicit create API) for the same consumer must be idempotent, and not trigger limit.\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"foo\",\n\t\tConfig: ConsumerConfig{Durable: \"bar\"},\n\t\tAction: ActionCreate,\n\t}\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"foo\", \"bar\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", resp.Error)\n\t}\n\n\tmsg, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"foo\", \"bar\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp2 JSApiConsumerInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp2))\n\tif resp2.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", resp2.Error)\n\t}\n\n\t// This should fail.\n\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dlc22\", AckPolicy: nats.AckExplicitPolicy})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected error but got none\")\n\t}\n}\n\nfunc TestJetStreamClusterMaxStreamsReached(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomNonLeader())\n\tdefer nc.Close()\n\n\t// Adjust our limits.\n\tc.updateLimits(\"$G\", map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 1024,\n\t\t\tMaxStore: 1024,\n\t\t\tMaxStreams: 1,\n\t\t},\n\t})\n\n\t// Many stream creations in parallel for the same stream should not result in\n\t// maximum number of streams reached error. All should have a successful response.\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 15; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 1,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t// Adjust our limits.\n\tc.updateLimits(\"$G\", map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 1024,\n\t\t\tMaxStore: 1024,\n\t\t\tMaxStreams: 2,\n\t\t},\n\t})\n\n\t// Setup streams beforehand.\n\tfor d := 0; d < 2; d++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST-%d\", d),\n\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d\", d)},\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Many stream creations in parallel for streams that already exist should not result in\n\t// maximum number of streams reached error. All should have a successful response.\n\tfor i := 0; i < 15; i++ {\n\t\twg.Add(1)\n\t\td := i % 2\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"TEST-%d\", d),\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d\", d)},\n\t\t\t\tReplicas: 1,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc TestJetStreamClusterStreamLimits(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Check that large R will fail.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 5}); err == nil {\n\t\tt.Fatalf(\"Expected error but got none\")\n\t}\n\n\tmaxMsgs := 5\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tReplicas: 3,\n\t\tRetention: nats.LimitsPolicy,\n\t\tDiscard: DiscardNew,\n\t\tMaxMsgSize: 11,\n\t\tMaxMsgs: int64(maxMsgs),\n\t\tMaxAge: 250 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Large message should fail.\n\tif _, err := js.Publish(\"foo\", []byte(\"0123456789ZZZ\")); err == nil {\n\t\tt.Fatalf(\"Expected publish to fail\")\n\t}\n\n\tfor i := 0; i < maxMsgs; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"JSC-OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// These should fail.\n\tif _, err := js.Publish(\"foo\", []byte(\"JSC-OK\")); err == nil {\n\t\tt.Fatalf(\"Expected publish to fail\")\n\t}\n\n\t// Make sure when space frees up we can send more.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"foo\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tif _, err := js.Publish(\"foo\", []byte(\"ROUND2\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterStreamInterestOnlyPolicy(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\n\t// With no interest these should be no-ops.\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"JSC-OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsi, err := js.StreamInfo(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Expected no messages with no interest, got %d\", si.State.Msgs)\n\t}\n\n\t// Now create a consumer.\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"JSC-OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tcheckSubsPending(t, sub, toSend)\n\n\tsi, err = js.StreamInfo(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d messages with interest, got %d\", toSend, si.State.Msgs)\n\t}\n\tif si.State.FirstSeq != uint64(toSend+1) {\n\t\tt.Fatalf(\"Expected first sequence of %d, got %d\", toSend+1, si.State.FirstSeq)\n\t}\n\n\t// Now delete the consumer.\n\tsub.Unsubscribe()\n\t// That should make it go away.\n\tif _, err := js.ConsumerInfo(\"foo\", \"dlc\"); err == nil {\n\t\tt.Fatalf(\"Expected not found error, got none\")\n\t}\n\n\t// Wait for the messages to be purged.\n\tcheckFor(t, 5*time.Second, 20*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"foo\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Wanted 0 messages, got %d\", si.State.Msgs)\n\t})\n}\n\nfunc TestJetStreamClusterExtendedAccountInfo(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"JSC-OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Add in some streams with msgs and consumers.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-1\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-1\", 25)\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-2\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-2\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-2\", 50)\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-3\", Replicas: 3, Storage: nats.MemoryStorage}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-3\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-3\", 100)\n\n\t// Go client will lag so use direct for now.\n\tgetAccountInfo := func() *nats.AccountInfo {\n\t\tt.Helper()\n\n\t\tinfo, err := js.AccountInfo()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\treturn info\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tai := getAccountInfo()\n\t\tif ai.Streams != 3 || ai.Consumers != 3 {\n\t\t\treturn fmt.Errorf(\"AccountInfo not correct: %+v\", ai)\n\t\t}\n\t\tif ai.API.Total < 7 {\n\t\t\treturn fmt.Errorf(\"Expected at least 7 total API calls, got %d\", ai.API.Total)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do a failure to make sure we track API errors.\n\tjs.StreamInfo(\"NO-STREAM\")\n\tjs.ConsumerInfo(\"TEST-1\", \"NO-CONSUMER\")\n\tjs.ConsumerInfo(\"TEST-2\", \"NO-CONSUMER\")\n\tjs.ConsumerInfo(\"TEST-3\", \"NO-CONSUMER\")\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tai := getAccountInfo()\n\t\tif ai.API.Errors != 4 {\n\t\t\treturn fmt.Errorf(\"Expected 4 API calls to be errors, got %d\", ai.API.Errors)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPeerRemovalAPI(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tml := c.leader()\n\tnc, err := nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Expect error if unknown peer\n\treq := &JSApiMetaServerRemoveRequest{Server: \"S-9\"}\n\tjsreq, err := json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar resp JSApiMetaServerRemoveResponse\n\tif err := json.Unmarshal(rmsg.Data, &resp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\n\tsub, err := nc.SubscribeSync(JSAdvisoryServerRemoved)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\trs := c.randomNonLeader()\n\treq = &JSApiMetaServerRemoveRequest{Server: rs.Name()}\n\tjsreq, err = json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trmsg, err = nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp.Error = nil\n\tif err := json.Unmarshal(rmsg.Data, &resp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t}\n\tc.waitOnLeader()\n\tml = c.leader()\n\n\tcheckSubsPending(t, sub, 1)\n\tmadv, _ := sub.NextMsg(0)\n\tvar adv JSServerRemovedAdvisory\n\tif err := json.Unmarshal(madv.Data, &adv); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif adv.Server != rs.Name() {\n\t\tt.Fatalf(\"Expected advisory about %s being removed, got %+v\", rs.Name(), adv)\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tfor _, s := range ml.JetStreamClusterPeers() {\n\t\t\tif s == rs.Name() {\n\t\t\t\treturn fmt.Errorf(\"Still in the peer list\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPeerRemovalAndStreamReassignment(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Admin based API\n\tml := c.leader()\n\tnc, err = nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Select the non-leader server for the stream to remove.\n\tif len(si.Cluster.Replicas) < 2 {\n\t\tt.Fatalf(\"Not enough replicas found: %+v\", si.Cluster)\n\t}\n\ttoRemove, cl := si.Cluster.Replicas[0].Name, c.leader()\n\tif toRemove == cl.Name() {\n\t\ttoRemove = si.Cluster.Replicas[1].Name\n\t}\n\n\treq := &JSApiMetaServerRemoveRequest{Server: toRemove}\n\tjsreq, err := json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar resp JSApiMetaServerRemoveResponse\n\tif err := json.Unmarshal(rmsg.Data, &resp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t}\n\t// In case that server was also meta-leader.\n\tc.waitOnLeader()\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tfor _, s := range ml.JetStreamClusterPeers() {\n\t\t\tif s == toRemove {\n\t\t\t\treturn fmt.Errorf(\"Server still in the peer list\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now wait until the stream is now current.\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t}\n\t\t// We should not see the old server at all.\n\t\tfor _, p := range si.Cluster.Replicas {\n\t\t\tif p.Name == toRemove {\n\t\t\t\tt.Fatalf(\"Peer not removed yet: %+v\", toRemove)\n\t\t\t}\n\t\t\tif !p.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", p)\n\t\t\t}\n\t\t}\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPeerRemovalAndStreamReassignmentWithoutSpace(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Admin based API\n\tml := c.leader()\n\tnc, err = nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Select the non-leader server for the stream to remove.\n\tif len(si.Cluster.Replicas) < 2 {\n\t\tt.Fatalf(\"Not enough replicas found: %+v\", si.Cluster)\n\t}\n\ttoRemove, cl := si.Cluster.Replicas[0].Name, c.leader()\n\tif toRemove == cl.Name() {\n\t\ttoRemove = si.Cluster.Replicas[1].Name\n\t}\n\n\treq := &JSApiMetaServerRemoveRequest{Server: toRemove}\n\tjsreq, err := json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tvar resp JSApiMetaServerRemoveResponse\n\tif err := json.Unmarshal(rmsg.Data, &resp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t}\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tfor _, s := range ml.JetStreamClusterPeers() {\n\t\t\tif s == toRemove {\n\t\t\t\treturn fmt.Errorf(\"Server still in the peer list\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\t// Make sure only 2 peers at this point.\n\tc.waitOnPeerCount(2)\n\n\t// Now wait until the stream is now current.\n\tstreamCurrent := func(nr int) {\n\t\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t\t}\n\t\t\t// We should not see the old server at all.\n\t\t\tfor _, p := range si.Cluster.Replicas {\n\t\t\t\tif p.Name == toRemove {\n\t\t\t\t\treturn fmt.Errorf(\"Peer not removed yet: %+v\", toRemove)\n\t\t\t\t}\n\t\t\t\tif !p.Current {\n\t\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", p)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(si.Cluster.Replicas) != nr {\n\t\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", nr, len(si.Cluster.Replicas))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Make sure the peer was removed from the stream and that we did not fill the new spot.\n\tstreamCurrent(1)\n\n\t// Now add in a new server and make sure it gets added to our stream.\n\tc.addInNewServer()\n\tc.waitOnPeerCount(3)\n\n\tstreamCurrent(2)\n}\n\nfunc TestJetStreamClusterPeerRemovalAndServerBroughtBack(t *testing.T) {\n\t// Speed up for this test\n\tpeerRemoveTimeout = 2 * time.Second\n\tdefer func() {\n\t\tpeerRemoveTimeout = peerRemoveTimeoutDefault\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tml := c.leader()\n\tnc, err := nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tgetPeersCount := func() int {\n\t\tjs := ml.getJetStream()\n\t\tif js == nil {\n\t\t\treturn 0\n\t\t}\n\t\tjs.mu.RLock()\n\t\tdefer js.mu.RUnlock()\n\n\t\tcc := js.cluster\n\t\tif !cc.isLeader() || cc.meta == nil {\n\t\t\treturn 0\n\t\t}\n\t\treturn len(cc.meta.Peers())\n\t}\n\n\tcheckFor(t, 2*time.Second, 250*time.Millisecond, func() error {\n\t\tif l := getPeersCount(); l != 5 {\n\t\t\treturn fmt.Errorf(\"expected 5 peers, got %d\", l)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Shutdown server first.\n\trs := c.randomNonLeader()\n\trs.Shutdown()\n\n\t// Peers should still remain the same, even if one server is shut down.\n\tcheckFor(t, 2*time.Second, 250*time.Millisecond, func() error {\n\t\tif l := getPeersCount(); l != 5 {\n\t\t\treturn fmt.Errorf(\"expected 5 peers, got %d\", l)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Peer-remove after shutdown.\n\treq := &JSApiMetaServerRemoveRequest{Server: rs.Name()}\n\tjsreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiMetaServerRemoveResponse\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &resp))\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t}\n\n\t// Peer should be removed.\n\tcheckFor(t, 2*time.Second, 250*time.Millisecond, func() error {\n\t\tif l := getPeersCount(); l != 4 {\n\t\t\treturn fmt.Errorf(\"expected 4 peers, got %d\", l)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Bringing back the server should re-add to peers after peer-remove timeout.\n\tc.restartServer(rs)\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tif l := getPeersCount(); l != 5 {\n\t\t\treturn fmt.Errorf(\"expected 5 peers, got %d\", l)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPeerExclusionTag(t *testing.T) {\n\tc := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, \"C\", 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tswitch serverName {\n\t\t\tcase \"S-1\":\n\t\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [server:%s, intersect, %s]\", conf, serverName, jsExcludePlacement)\n\t\t\tcase \"S-2\":\n\t\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [server:%s, intersect]\", conf, serverName)\n\t\t\tdefault:\n\t\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [server:%s]\", conf, serverName)\n\t\t\t}\n\t\t})\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor i, c := range []nats.StreamConfig{\n\t\t{Replicas: 1, Placement: &nats.Placement{Tags: []string{\"server:S-1\"}}},\n\t\t{Replicas: 2, Placement: &nats.Placement{Tags: []string{\"intersect\"}}},\n\t\t{Replicas: 3}, // not enough server without !jetstream\n\t} {\n\t\tc.Name = fmt.Sprintf(\"TEST%d\", i)\n\t\tc.Subjects = []string{c.Name}\n\t\t_, err := js.AddStream(&c)\n\t\trequire_Error(t, err)\n\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"exclude tag set\")\n\t}\n\n\t// Test update failure\n\tcfg := &nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 2}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"exclude tag set\")\n\t// Test tag reload removing !jetstream tag, and allowing placement again\n\n\tsrv := c.serverByName(\"S-1\")\n\n\tv, err := srv.Varz(nil)\n\trequire_NoError(t, err)\n\trequire_True(t, v.Tags.Contains(jsExcludePlacement))\n\tcontent, err := os.ReadFile(srv.configFile)\n\trequire_NoError(t, err)\n\tnewContent := strings.ReplaceAll(string(content), fmt.Sprintf(\", %s]\", jsExcludePlacement), \"]\")\n\tchangeCurrentConfigContentWithNewContent(t, srv.configFile, []byte(newContent))\n\n\trequire_NoError(t, srv.Reload())\n\tsrv.sendStatszUpdate()\n\n\tv, err = srv.Varz(nil)\n\trequire_NoError(t, err)\n\trequire_True(t, !v.Tags.Contains(jsExcludePlacement))\n\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tvar err error\n\t\t\ts.nodeToInfo.Range(func(sn any, value any) bool {\n\t\t\t\tni := value.(nodeInfo)\n\t\t\t\tif ni.name != \"S-1\" {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\tif ni.tags.Contains(\"!jetstream\") {\n\t\t\t\t\terr = fmt.Errorf(\"%s still has jetstream exclude tag\", sn)\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterAccountPurge(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\taccKp, accpub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accpub)\n\taccClaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 * 5\n\taccClaim.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 * 5\n\taccJwt := encodeClaim(t, accClaim, accpub)\n\taccCreds := newUser(t, accKp)\n\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t`\n\tc := createJetStreamClusterWithTemplateAndModHook(t, tmlp, \"cluster\", 3,\n\t\tfunc(serverName, clustername, storeDir, conf string) string {\n\t\t\treturn conf + fmt.Sprintf(`\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: {\n\t\t\t\t\ttype: full\n\t\t\t\t\tdir: '%s/jwt'\n\t\t\t\t\ttimeout: \"10ms\"\n\t\t\t\t}`, ojwt, syspub, storeDir)\n\t\t})\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3)\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt, 3)\n\n\tc.waitOnAccount(accpub)\n\n\tcreateTestData := func(t *testing.T) {\n\t\tnc, js := jsClientConnect(t, c.randomNonLeader(), nats.UserCredentials(accCreds))\n\t\tdefer nc.Close()\n\n\t\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\t\tc.waitOnAccount(accpub)\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST1\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(accpub, \"TEST1\")\n\n\t\tci, err := js.AddConsumer(\"TEST1\",\n\t\t\t&nats.ConsumerConfig{Durable: \"DUR1\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ci.Config.Replicas == 0)\n\n\t\tci, err = js.AddConsumer(\"TEST1\",\n\t\t\t&nats.ConsumerConfig{Durable: \"DUR2\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\tReplicas: 1})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ci.Config.Replicas == 1)\n\n\t\ttoSend := uint64(1_000)\n\t\tfor i := uint64(0); i < toSend; i++ {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST2\",\n\t\t\tSubjects: []string{\"bar\"},\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tci, err = js.AddConsumer(\"TEST2\",\n\t\t\t&nats.ConsumerConfig{Durable: \"DUR1\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\tReplicas: 0})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ci.Config.Replicas == 0)\n\n\t\tfor i := uint64(0); i < toSend; i++ {\n\t\t\t_, err = js.Publish(\"bar\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tinspectDirs := func(t *testing.T, sysTotal, accTotal int) error {\n\t\tt.Helper()\n\t\tsysDirs := 0\n\t\taccDirs := 0\n\t\tfor _, s := range c.servers {\n\t\t\tfiles, err := os.ReadDir(filepath.Join(s.getOpts().StoreDir, \"jetstream\", syspub, \"_js_\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tsysDirs += len(files) - 1 // sub 1 for _meta_\n\t\t\tfiles, err = os.ReadDir(filepath.Join(s.getOpts().StoreDir, \"jetstream\", accpub, \"streams\"))\n\t\t\tif err == nil || err.(*os.PathError).Error() == \"no such file or directory\" {\n\t\t\t\taccDirs += len(files)\n\t\t\t}\n\t\t}\n\t\tif sysDirs != sysTotal || accDirs != accTotal {\n\t\t\treturn fmt.Errorf(\"expected directory count does not match %d == %d, %d == %d\",\n\t\t\t\tsysDirs, sysTotal, accDirs, accTotal)\n\t\t}\n\t\treturn nil\n\t}\n\n\tcheckForDirs := func(t *testing.T, sysTotal, accTotal int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 20*time.Second, 250*time.Millisecond, func() error {\n\t\t\treturn inspectDirs(t, sysTotal, accTotal)\n\t\t})\n\t}\n\n\tpurge := func(t *testing.T) {\n\t\tt.Helper()\n\t\tncsys, err := nats.Connect(c.randomServer().ClientURL(), nats.UserCredentials(sysCreds))\n\t\trequire_NoError(t, err)\n\t\tdefer ncsys.Close()\n\n\t\trequest := func() error {\n\t\t\tvar resp JSApiAccountPurgeResponse\n\t\t\tm, err := ncsys.Request(fmt.Sprintf(JSApiAccountPurgeT, accpub), nil, time.Second)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := json.Unmarshal(m.Data, &resp); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !resp.Initiated {\n\t\t\t\treturn fmt.Errorf(\"not started\")\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tcheckFor(t, 30*time.Second, 250*time.Millisecond, request)\n\t}\n\n\tt.Run(\"startup-cleanup\", func(t *testing.T) {\n\t\t_, newCleanupAcc1 := createKey(t)\n\t\t_, newCleanupAcc2 := createKey(t)\n\t\tfor _, s := range c.servers {\n\t\t\tos.MkdirAll(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc1, streamsDir), defaultDirPerms)\n\t\t\tos.MkdirAll(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc2), defaultDirPerms)\n\t\t}\n\t\tcreateTestData(t)\n\t\tcheckForDirs(t, 6, 4)\n\t\tc.stopAll()\n\t\tc.restartAll()\n\t\tfor _, s := range c.servers {\n\t\t\taccDir := filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc1)\n\t\t\t_, e := os.Stat(filepath.Join(accDir, streamsDir))\n\t\t\trequire_Error(t, e)\n\t\t\trequire_True(t, os.IsNotExist(e))\n\t\t\t_, e = os.Stat(accDir)\n\t\t\trequire_Error(t, e)\n\t\t\trequire_True(t, os.IsNotExist(e))\n\t\t\t_, e = os.Stat(filepath.Join(s.getOpts().StoreDir, JetStreamStoreDir, newCleanupAcc2))\n\t\t\trequire_Error(t, e)\n\t\t\trequire_True(t, os.IsNotExist(e))\n\t\t}\n\t\tcheckForDirs(t, 6, 4)\n\t\t// Make sure we have a leader for all assets before moving to the next test\n\t\tc.waitOnStreamLeader(accpub, \"TEST1\")\n\t\tc.waitOnConsumerLeader(accpub, \"TEST1\", \"DUR1\")\n\t\tc.waitOnConsumerLeader(accpub, \"TEST1\", \"DUR2\")\n\t\tc.waitOnStreamLeader(accpub, \"TEST2\")\n\t\tc.waitOnConsumerLeader(accpub, \"TEST2\", \"DUR1\")\n\t})\n\n\tt.Run(\"purge-with-restart\", func(t *testing.T) {\n\t\tcreateTestData(t)\n\t\tcheckForDirs(t, 6, 4)\n\t\tpurge(t)\n\t\tcheckForDirs(t, 0, 0)\n\t\tc.stopAll()\n\t\tc.restartAll()\n\t\tcheckForDirs(t, 0, 0)\n\t})\n\n\tt.Run(\"purge-with-reuse\", func(t *testing.T) {\n\t\tcreateTestData(t)\n\t\tcheckForDirs(t, 6, 4)\n\t\tpurge(t)\n\t\tcheckForDirs(t, 0, 0)\n\t\tcreateTestData(t)\n\t\tcheckForDirs(t, 6, 4)\n\t\tpurge(t)\n\t\tcheckForDirs(t, 0, 0)\n\t})\n\n\tt.Run(\"purge-deleted-account\", func(t *testing.T) {\n\t\tcreateTestData(t)\n\t\tcheckForDirs(t, 6, 4)\n\t\tc.stopAll()\n\t\tfor _, s := range c.servers {\n\t\t\trequire_NoError(t, os.Remove(s.getOpts().StoreDir+\"/jwt/\"+accpub+\".jwt\"))\n\t\t}\n\t\tc.restartAll()\n\t\tcheckForDirs(t, 6, 4)\n\t\tpurge(t)\n\t\tcheckForDirs(t, 0, 0)\n\t\tc.stopAll()\n\t\tc.restartAll()\n\t\tcheckForDirs(t, 0, 0)\n\t})\n}\n\nfunc TestJetStreamClusterScaleConsumer(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterTempl, \"C\", 3)\n\tdefer c.shutdown()\n\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tdurCfg := &nats.ConsumerConfig{Durable: \"DUR\", AckPolicy: nats.AckExplicitPolicy}\n\tci, err := js.AddConsumer(\"TEST\", durCfg)\n\trequire_NoError(t, err)\n\trequire_True(t, ci.Config.Replicas == 0)\n\n\ttoSend := uint64(1_000)\n\tfor i := uint64(0); i < toSend; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\ts, err := js.PullSubscribe(\"foo\", \"DUR\")\n\trequire_NoError(t, err)\n\n\tconsumeOne := func(expSeq uint64) error {\n\t\tif ci, err := js.ConsumerInfo(\"TEST\", \"DUR\"); err != nil {\n\t\t\treturn err\n\t\t} else if ci.Delivered.Stream != expSeq {\n\t\t\treturn fmt.Errorf(\"pre: not expected delivered stream %d, got %d\", expSeq, ci.Delivered.Stream)\n\t\t} else if ci.Delivered.Consumer != expSeq {\n\t\t\treturn fmt.Errorf(\"pre: not expected delivered consumer %d, got %d\", expSeq, ci.Delivered.Consumer)\n\t\t} else if ci.AckFloor.Stream != expSeq {\n\t\t\treturn fmt.Errorf(\"pre: not expected ack stream %d, got %d\", expSeq, ci.AckFloor.Stream)\n\t\t} else if ci.AckFloor.Consumer != expSeq {\n\t\t\treturn fmt.Errorf(\"pre: not expected ack consumer %d, got %d\", expSeq, ci.AckFloor.Consumer)\n\t\t}\n\t\tif m, err := s.Fetch(1); err != nil {\n\t\t\treturn err\n\t\t} else if err := m[0].AckSync(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\texpSeq = expSeq + 1\n\t\tif ci, err := js.ConsumerInfo(\"TEST\", \"DUR\"); err != nil {\n\t\t\treturn err\n\t\t} else if ci.Delivered.Stream != expSeq {\n\t\t\treturn fmt.Errorf(\"post: not expected delivered stream %d, got %d\", expSeq, ci.Delivered.Stream)\n\t\t} else if ci.Delivered.Consumer != expSeq {\n\t\t\treturn fmt.Errorf(\"post: not expected delivered consumer %d, got %d\", expSeq, ci.Delivered.Consumer)\n\t\t} else if ci.AckFloor.Stream != expSeq {\n\t\t\treturn fmt.Errorf(\"post: not expected ack stream %d, got %d\", expSeq, ci.AckFloor.Stream)\n\t\t} else if ci.AckFloor.Consumer != expSeq {\n\t\t\treturn fmt.Errorf(\"post: not expected ack consumer %d, got %d\", expSeq, ci.AckFloor.Consumer)\n\t\t}\n\t\treturn nil\n\t}\n\n\trequire_NoError(t, consumeOne(0))\n\n\t// scale down, up, down and up to default == 3 again\n\tfor i, r := range []int{1, 3, 1, 0} {\n\t\tdurCfg.Replicas = r\n\t\tif r == 0 {\n\t\t\tr = si.Config.Replicas\n\t\t}\n\t\tjs.UpdateConsumer(\"TEST\", durCfg)\n\n\t\tcheckFor(t, time.Second*30, time.Millisecond*250, func() error {\n\t\t\tif ci, err = js.ConsumerInfo(\"TEST\", \"DUR\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if ci.Cluster == nil {\n\t\t\t\treturn errors.New(\"no cluster info\")\n\t\t\t} else if ci.Cluster.Leader == _EMPTY_ {\n\t\t\t\treturn errors.New(\"no leader\")\n\t\t\t} else if len(ci.Cluster.Replicas) != r-1 {\n\t\t\t\treturn fmt.Errorf(\"not enough replica, got %d wanted %d\", len(ci.Cluster.Replicas), r-1)\n\t\t\t} else {\n\t\t\t\tfor _, r := range ci.Cluster.Replicas {\n\t\t\t\t\tif !r.Current || r.Offline || r.Lag != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"replica %s not current %t offline %t lag %d\", r.Name, r.Current, r.Offline, r.Lag)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\trequire_NoError(t, consumeOne(uint64(i+1)))\n\t}\n}\n\nfunc TestJetStreamClusterConsumerScaleUp(t *testing.T) {\n\tc := createJetStreamCluster(t, jsClusterTempl, \"HUB\", _EMPTY_, 3, 22020, true)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\tscfg := nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}\n\t_, err := js.AddStream(&scfg)\n\trequire_NoError(t, err)\n\tdefer js.DeleteStream(\"TEST\")\n\n\tdcfg := nats.ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 0}\n\t_, err = js.AddConsumer(\"TEST\", &dcfg)\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tscfg.Replicas = 2\n\t_, err = js.UpdateStream(&scfg)\n\trequire_NoError(t, err)\n\n\t// The scale up issue shows itself as permanent loss of consumer leadership\n\t// So give it some time for the change to propagate to new consumer peers and the quorum to disrupt\n\t// 2 seconds is a value arrived by experimentally, no sleep or a sleep of 1sec always had the test pass a lot.\n\ttime.Sleep(2 * time.Second)\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// There is also a timing component to the issue triggering.\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"DUR\")\n}\n\nfunc TestJetStreamClusterPeerOffline(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\trs := c.randomNonLeader()\n\n\tcheckPeer := func(ml, rs *Server, shouldBeOffline bool) {\n\t\tt.Helper()\n\n\t\tcheckFor(t, 5*time.Second, 50*time.Millisecond, func() error {\n\t\t\tvar found bool\n\t\t\tfor _, s := range ml.JetStreamClusterPeers() {\n\t\t\t\tif s == rs.Name() {\n\t\t\t\t\tfound = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !shouldBeOffline && !found {\n\t\t\t\treturn fmt.Errorf(\"Server %q not in the peers list\", rs.Name())\n\t\t\t} else if shouldBeOffline && found {\n\t\t\t\treturn fmt.Errorf(\"Server %q should not be in the peers list\", rs.Name())\n\t\t\t}\n\n\t\t\tvar ok bool\n\t\t\tml.nodeToInfo.Range(func(k, v any) bool {\n\t\t\t\tif si := v.(nodeInfo); si.name == rs.Name() {\n\t\t\t\t\tif shouldBeOffline && si.offline || !shouldBeOffline && !si.offline {\n\t\t\t\t\t\tok = true\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t\tif !ok {\n\t\t\t\tif shouldBeOffline {\n\t\t\t\t\treturn fmt.Errorf(\"Server %q should be marked as online\", rs.Name())\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Server %q is still marked as online\", rs.Name())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Shutdown the server and make sure that it is now showing as offline.\n\trs.Shutdown()\n\tcheckPeer(ml, rs, true)\n\n\t// Now restart that server and check that is no longer offline.\n\toldrs := rs\n\trs, _ = RunServerWithConfig(rs.getOpts().ConfigFile)\n\tdefer rs.Shutdown()\n\n\t// Replaced old with new server\n\tfor i := 0; i < len(c.servers); i++ {\n\t\tif c.servers[i] == oldrs {\n\t\t\tc.servers[i] = rs\n\t\t}\n\t}\n\n\t// Wait for cluster to be formed\n\tcheckClusterFormed(t, c.servers...)\n\n\t// Make sure that we have a leader (there can always be a re-election)\n\tc.waitOnLeader()\n\tml = c.leader()\n\n\t// Now check that rs is not offline\n\tcheckPeer(ml, rs, false)\n}\n\nfunc TestJetStreamClusterNoQuorumStepdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Setup subscription for leader elected.\n\tlesub, err := nc.SubscribeSync(JSAdvisoryStreamLeaderElectedPre + \".*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"NO-Q\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure we received our leader elected advisory.\n\tleadv, _ := lesub.NextMsg(0)\n\tif leadv == nil {\n\t\tt.Fatalf(\"Expected to receive a leader elected advisory\")\n\t}\n\tvar le JSStreamLeaderElectedAdvisory\n\tif err := json.Unmarshal(leadv.Data, &le); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ln := c.streamLeader(\"$G\", \"NO-Q\").Name(); le.Leader != ln {\n\t\tt.Fatalf(\"Expected to have leader %q in elect advisory, got %q\", ln, le.Leader)\n\t}\n\n\tpayload := []byte(\"Hello JSC\")\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := js.Publish(\"NO-Q\", payload); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Setup subscription for leader elected.\n\tclesub, err := nc.SubscribeSync(JSAdvisoryConsumerLeaderElectedPre + \".*.*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make durable to have R match Stream.\n\tsub, err := js.SubscribeSync(\"NO-Q\", nats.Durable(\"rr\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tci, err := sub.ConsumerInfo()\n\tif err != nil || ci == nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure we received our consumer leader elected advisory.\n\tleadv, _ = clesub.NextMsg(0)\n\tif leadv == nil {\n\t\tt.Fatalf(\"Expected to receive a consumer leader elected advisory\")\n\t}\n\n\t// Shutdown the non-leader.\n\tc.randomNonStreamLeader(\"$G\", \"NO-Q\").Shutdown()\n\n\t// This should eventually have us stepdown as leader since we would have lost quorum with R=2.\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\tif sl := c.streamLeader(\"$G\", \"NO-Q\"); sl == nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Still have leader for stream\")\n\t})\n\n\tnotAvailableErr := func(err error) bool {\n\t\treturn err != nil && (strings.Contains(err.Error(), \"unavailable\") || err == context.DeadlineExceeded)\n\t}\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif cl := c.consumerLeader(\"$G\", \"NO-Q\", ci.Name); cl == nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Still have leader for consumer\")\n\t})\n\n\tif _, err = js.ConsumerInfo(\"NO-Q\", ci.Name); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif _, err := sub.ConsumerInfo(); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\n\t// Now let's take out the other non meta-leader\n\t// We should get same error for general API calls.\n\tc.randomNonLeader().Shutdown()\n\tc.expectNoLeader()\n\n\t// Now make sure the general JS API responds with system unavailable.\n\tif _, err = js.AccountInfo(); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"NO-Q33\", Replicas: 2}); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif _, err := js.UpdateStream(&nats.StreamConfig{Name: \"NO-Q33\", Replicas: 2}); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif err := js.DeleteStream(\"NO-Q\"); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif err := js.PurgeStream(\"NO-Q\"); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif err := js.DeleteMsg(\"NO-Q\", 1); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\t// Consumer\n\tif _, err := js.AddConsumer(\"NO-Q\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy}); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif err := js.DeleteConsumer(\"NO-Q\", \"dlc\"); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\tif _, err := js.ConsumerInfo(\"NO-Q\", \"dlc\"); !notAvailableErr(err) {\n\t\tt.Fatalf(\"Expected an 'unavailable' error, got %v\", err)\n\t}\n\t// Listers\n\tfor info := range js.StreamsInfo() {\n\t\tt.Fatalf(\"Unexpected stream info, got %v\", info)\n\t}\n\tfor info := range js.ConsumersInfo(\"NO-Q\") {\n\t\tt.Fatalf(\"Unexpected consumer info, got %v\", info)\n\t}\n}\n\nfunc TestJetStreamClusterCreateResponseAdvisoriesHaveSubject(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.API\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST\", nats.Durable(\"DLC\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, 6)\n\n\tfor m, err := sub.NextMsg(0); err == nil; m, err = sub.NextMsg(0) {\n\t\tvar audit JSAPIAudit\n\t\tif err := json.Unmarshal(m.Data, &audit); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif audit.Subject == _EMPTY_ {\n\t\t\tt.Fatalf(\"Expected subject, got nothing\")\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterRestartAndRemoveAdvisories(t *testing.T) {\n\t// FIXME(dlc) - Flaky on Travis, skip for now.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.API\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\tcsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.*.CREATED.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer csub.Unsubscribe()\n\tnc.Flush()\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"JSC-OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Add in some streams with msgs and consumers.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-1\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-1\", nats.Durable(\"DC\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-1\", 25)\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-2\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-2\", nats.Durable(\"DC\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-2\", 50)\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST-3\", Replicas: 3, Storage: nats.MemoryStorage}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"TEST-3\", nats.Durable(\"DC\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"TEST-3\", 100)\n\n\tdrainSub := func(sub *nats.Subscription) {\n\t\tfor _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) {\n\t\t}\n\t}\n\n\t// Wait for the advisories for all streams and consumers.\n\tcheckSubsPending(t, sub, 12) // 3 streams, 3*2 consumers, 3 stream names lookups for creating consumers.\n\tdrainSub(sub)\n\n\t// Created audit events.\n\tcheckSubsPending(t, csub, 6)\n\tdrainSub(csub)\n\n\tusub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.*.UPDATED.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer usub.Unsubscribe()\n\tnc.Flush()\n\n\tcheckSubsPending(t, csub, 0)\n\tcheckSubsPending(t, sub, 0)\n\tcheckSubsPending(t, usub, 0)\n\n\t// Now restart the other two servers we are not connected to.\n\tfor _, cs := range c.servers {\n\t\tif cs != s {\n\t\t\tcs.Shutdown()\n\t\t\tc.restartServer(cs)\n\t\t}\n\t}\n\tc.waitOnAllCurrent()\n\n\tcheckSubsPending(t, csub, 0)\n\tcheckSubsPending(t, sub, 0)\n\tcheckSubsPending(t, usub, 0)\n\n\tdsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.*.DELETED.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer dsub.Unsubscribe()\n\tnc.Flush()\n\n\tc.waitOnConsumerLeader(\"$G\", \"TEST-1\", \"DC\")\n\tc.waitOnLeader()\n\n\t// Now check delete advisories as well.\n\tif err := js.DeleteConsumer(\"TEST-1\", \"DC\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, csub, 0)\n\tcheckSubsPending(t, dsub, 1)\n\tcheckSubsPending(t, sub, 1)\n\tcheckSubsPending(t, usub, 0)\n\tdrainSub(dsub)\n\n\tif err := js.DeleteStream(\"TEST-3\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, dsub, 2) // Stream and the consumer underneath.\n\tcheckSubsPending(t, sub, 2)\n}\n\nfunc TestJetStreamClusterNoDuplicateOnNodeRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"ND\", 2)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tif s == sl {\n\t\tnc.Close()\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tjs.Publish(\"foo\", []byte(\"msg1\"))\n\tif m, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t} else {\n\t\tm.AckSync()\n\t}\n\n\tsl.Shutdown()\n\tc.restartServer(sl)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\n\t// Send second msg\n\tjs.Publish(\"foo\", []byte(\"msg2\"))\n\tmsg, err := sub.NextMsg(5 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t}\n\tif string(msg.Data) != \"msg2\" {\n\t\tt.Fatalf(\"Unexpected message: %s\", msg.Data)\n\t}\n\tmsg.AckSync()\n\n\t// Make sure we don't get a duplicate.\n\tmsg, err = sub.NextMsg(250 * time.Millisecond)\n\tif err == nil {\n\t\tt.Fatalf(\"Should have gotten an error, got %s\", msg.Data)\n\t}\n}\n\nfunc TestJetStreamClusterNoDupePeerSelection(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NDP\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create 10 streams. Make sure none of them have a replica\n\t// that is the same as the leader.\n\tfor i := 1; i <= 10; i++ {\n\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST-%d\", i),\n\t\t\tReplicas: 3,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.Cluster == nil || si.Cluster.Leader == \"\" || len(si.Cluster.Replicas) != 2 {\n\t\t\tt.Fatalf(\"Unexpected cluster state for stream info: %+v\\n\", si.Cluster)\n\t\t}\n\t\t// Make sure that the replicas are not same as the leader.\n\t\tfor _, pi := range si.Cluster.Replicas {\n\t\t\tif pi.Name == si.Cluster.Leader {\n\t\t\t\tt.Fatalf(\"Found replica that is same as leader, meaning 2 nodes placed on same server\")\n\t\t\t}\n\t\t}\n\t\t// Now do a consumer and check same thing.\n\t\tsub, err := js.SubscribeSync(si.Config.Name)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tci, err := sub.ConsumerInfo()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t\t}\n\t\tfor _, pi := range ci.Cluster.Replicas {\n\t\t\tif pi.Name == ci.Cluster.Leader {\n\t\t\t\tt.Fatalf(\"Found replica that is same as leader, meaning 2 nodes placed on same server\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterStreamRemovePeer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"RNS\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.SubscribeSync(\"TEST\", nats.Durable(\"cat\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, toSend)\n\n\t// Do ephemeral too.\n\tesub, err := js.SubscribeSync(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, esub, toSend)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"pull\", AckPolicy: nats.AckNonePolicy})\n\trequire_NoError(t, err)\n\n\tpullSub, err := js.PullSubscribe(\"TEST\", \"pull\")\n\trequire_NoError(t, err)\n\n\t// First fetch the messages that are already there.\n\tmsgs, err := pullSub.Fetch(toSend, nats.MaxWait(500*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, toSend, len(msgs))\n\n\t// Now prepare a check to see if we get unwated `Consumer Deleted` error on peer remove.\n\tpullResults := make(chan error, 1)\n\tgo func() {\n\t\t_, err := pullSub.Fetch(1, nats.MaxWait(30*time.Second))\n\t\t// Let's check if we get unwted `Consumer Deleted` error on peer remove.\n\t\t// Everything else is fine (Leader Changed, Timeout, etc.)\n\t\tif err != nats.ErrConsumerDeleted {\n\t\t\tclose(pullResults)\n\t\t} else {\n\t\t\tpullResults <- err\n\t\t}\n\t}()\n\n\tci, err := esub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Could not fetch consumer info: %v\", err)\n\t}\n\t// Capture ephemeral's server and name.\n\tes, en := ci.Cluster.Leader, ci.Name\n\n\t// Grab stream info.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tpeers := []string{si.Cluster.Leader}\n\tfor _, p := range si.Cluster.Replicas {\n\t\tpeers = append(peers, p.Name)\n\t}\n\t// Pick a truly random server to remove.\n\trand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] })\n\ttoRemove := peers[0]\n\n\t// First test bad peer.\n\treq := &JSApiStreamRemovePeerRequest{Peer: \"NOT VALID\"}\n\tjsreq, err := json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Need to call this by hand for now.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"TEST\"), jsreq, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar rpResp JSApiStreamRemovePeerResponse\n\tif err := json.Unmarshal(resp.Data, &rpResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif rpResp.Error == nil || !strings.Contains(rpResp.Error.Description, \"peer not a member\") {\n\t\tt.Fatalf(\"Expected error for bad peer, got %+v\", rpResp.Error)\n\t}\n\trpResp.Error = nil\n\n\treq = &JSApiStreamRemovePeerRequest{Peer: toRemove}\n\tjsreq, err = json.Marshal(req)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"TEST\"), jsreq, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := json.Unmarshal(resp.Data, &rpResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif rpResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", rpResp.Error)\n\t}\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t}\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range si.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\tif si.Cluster.Leader == toRemove {\n\t\t\treturn fmt.Errorf(\"Peer not removed yet: %+v\", toRemove)\n\t\t}\n\t\tfor _, p := range si.Cluster.Replicas {\n\t\t\tif p.Name == toRemove {\n\t\t\t\treturn fmt.Errorf(\"Peer not removed yet: %+v\", toRemove)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"cat\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"pull\")\n\n\t// Now check consumer info as well.\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"cat\", nats.MaxWait(time.Second))\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch consumer info: %v\", err)\n\t\t}\n\t\tif len(ci.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(ci.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range ci.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\tif ci.Cluster.Leader == toRemove {\n\t\t\treturn fmt.Errorf(\"Peer not removed yet: %+v\", toRemove)\n\t\t}\n\t\tfor _, p := range ci.Cluster.Replicas {\n\t\t\tif p.Name == toRemove {\n\t\t\t\treturn fmt.Errorf(\"Peer not removed yet: %+v\", toRemove)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check if we got the `Consumer Deleted` error on the pull consumer.\n\tselect {\n\tcase err := <-pullResults:\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected timeout error or nil, got %v\", err)\n\t\t}\n\tdefault:\n\t}\n\n\t// Now check ephemeral consumer info.\n\t// Make sure we did not stamp same new group into the ephemeral where R=1.\n\tci, err = esub.ConsumerInfo()\n\t// If the leader was same as what we just removed, this should fail.\n\tif es == toRemove {\n\t\tif err != nats.ErrConsumerNotFound {\n\t\t\tt.Fatalf(\"Expected a not found error, got %v\", err)\n\t\t}\n\t\t// Also make sure this was removed all together.\n\t\t// We may proactively move things in the future.\n\t\tfor cn := range js.ConsumerNames(\"TEST\") {\n\t\t\tif cn == en {\n\t\t\t\tt.Fatalf(\"Expected ephemeral consumer to be deleted since we removed its only peer\")\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not fetch consumer info: %v\", err)\n\t\t}\n\t\tif len(ci.Cluster.Replicas) != 0 {\n\t\t\tt.Fatalf(\"Expected no replicas for ephemeral, got %d\", len(ci.Cluster.Replicas))\n\t\t}\n\t}\n\n\t// Confirm we can peer-remove by the peer ID instead of the server name too.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tnodeName := rs.Node()\n\trequire_Equal(t, nodeName, getHash(rs.Name()))\n\treq = &JSApiStreamRemovePeerRequest{Peer: nodeName}\n\tjsreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"TEST\"), jsreq, time.Second)\n\trequire_NoError(t, err)\n\trpResp = JSApiStreamRemovePeerResponse{}\n\trequire_NoError(t, json.Unmarshal(resp.Data, &rpResp))\n\trequire_True(t, rpResp.Success)\n}\n\nfunc TestJetStreamClusterStreamLeaderStepDown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"RNS\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.SubscribeSync(\"TEST\", nats.Durable(\"cat\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\toldLeader := c.streamLeader(\"$G\", \"TEST\").Name()\n\n\t// Need to call this by hand for now.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar sdResp JSApiStreamLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &sdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif sdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", sdResp.Error)\n\t}\n\n\t// Grab shorter timeout jetstream context.\n\tjs, err = nc.JetStream(nats.MaxWait(250 * time.Millisecond))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t}\n\t\tif si.Cluster.Leader == oldLeader {\n\t\t\treturn fmt.Errorf(\"Still have old leader\")\n\t\t}\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range si.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do consumer.\n\toldLeader = c.consumerLeader(\"$G\", \"TEST\", \"cat\").Name()\n\n\t// Need to call this by hand for now.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"cat\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar cdResp JSApiConsumerLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cdResp.Error)\n\t}\n\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"cat\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch consumer info: %v\", err)\n\t\t}\n\t\tif ci.Cluster.Leader == oldLeader {\n\t\t\treturn fmt.Errorf(\"Still have old leader\")\n\t\t}\n\t\tif len(ci.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(ci.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range ci.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterRemoveServer(t *testing.T) {\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"RNS\", 5)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tsub, err := js.SubscribeSync(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, toSend)\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcname := ci.Name\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tc.removeJetStream(sl)\n\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Faster timeout since we loop below checking for condition.\n\tjs, err = nc.JetStream(nats.MaxWait(250 * time.Millisecond))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Check the stream info is eventually correct.\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t}\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range si.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do consumer.\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", cname)\n\tcheckFor(t, 20*time.Second, 50*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", cname)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch consumer info: %v\", err)\n\t\t}\n\t\tif len(ci.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(ci.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range ci.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPurgeReplayAfterRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"P3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendBatch := func(n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tsendBatch(10)\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\tsendBatch(10)\n\n\tc.stopAll()\n\tc.restartAll()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\ts = c.randomServer()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs after restart, got %d\", si.State.Msgs)\n\t}\n}\n\nfunc TestJetStreamClusterStreamGetMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tmreq := &JSApiMsgGetRequest{Seq: 1}\n\treq, err := json.Marshal(mreq)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, \"TEST\"), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not retrieve stream message: %v\", err)\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"Could not retrieve stream message: %v\", err)\n\t}\n\n\tvar resp JSApiMsgGetResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t}\n\tif resp.Message == nil || resp.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", resp.Error)\n\t}\n}\n\nfunc TestJetStreamClusterStreamDirectGetMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t\tReplicas: 3,\n\t\tMaxMsgsPer: 1,\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\tsendStreamMsg(t, nc, \"foo\", \"bar\")\n\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"TEST\")\n\tgetMsg := func(req *JSApiMsgGetRequest) *nats.Msg {\n\t\tvar b []byte\n\t\tvar err error\n\t\tif req != nil {\n\t\t\tb, err = json.Marshal(req)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tm, err := nc.Request(getSubj, b, time.Second)\n\t\trequire_NoError(t, err)\n\t\treturn m\n\t}\n\n\tm := getMsg(&JSApiMsgGetRequest{LastFor: \"foo\"})\n\trequire_True(t, string(m.Data) == \"bar\")\n\trequire_True(t, m.Header.Get(JSStream) == \"TEST\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"1\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"foo\")\n\trequire_True(t, m.Subject != \"foo\")\n\trequire_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_)\n}\n\nfunc TestJetStreamClusterStreamPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnumConnections := 4\n\tvar conns []nats.JetStream\n\tfor i := 0; i < numConnections; i++ {\n\t\ts := c.randomServer()\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tconns = append(conns, js)\n\t}\n\n\ttoSend := 100000\n\tnumProducers := 8\n\n\tpayload := []byte(\"Hello JSC\")\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\tfor n := 0; n < numProducers; n++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tjs := conns[rand.Intn(numConnections)]\n\t\t\t<-startCh\n\t\t\tfor i := 0; i < int(toSend)/numProducers; i++ {\n\t\t\t\tif _, err = js.Publish(\"foo\", payload); err != nil {\n\t\t\t\t\tt.Errorf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Wait for Go routines.\n\ttime.Sleep(250 * time.Millisecond)\n\n\tstart := time.Now()\n\tclose(startCh)\n\twg.Wait()\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to send %d msgs with %d producers and R=3!\\n\", tt, toSend, numProducers)\n\tfmt.Printf(\"%.0f msgs/sec\\n\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamClusterConsumerPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ttoSend := 500000\n\tmsg := make([]byte, 64)\n\tcrand.Read(msg)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.Publish(\"TEST\", msg)\n\t}\n\tnc.Flush()\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected to have %d messages, got %d\", toSend, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\treceived := int32(0)\n\tdeliverTo := \"r\"\n\tdone := make(chan bool)\n\ttotal := int32(toSend)\n\tvar start time.Time\n\n\tnc.Subscribe(deliverTo, func(m *nats.Msg) {\n\t\tif r := atomic.AddInt32(&received, 1); r >= total {\n\t\t\tdone <- true\n\t\t} else if r == 1 {\n\t\t\tstart = time.Now()\n\t\t}\n\t})\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{DeliverSubject: deliverTo, Durable: \"gf\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tselect {\n\tcase <-done:\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Timed out?\")\n\t}\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, toSend)\n\tfmt.Printf(\"%.0f msgs/sec\\n\\n\", float64(toSend)/tt.Seconds())\n}\n\n// This test creates a queue consumer for the delivery subject,\n// and make sure it connects to the server that is not the leader\n// of the stream. A bug was not stripping the $JS.ACK reply subject\n// correctly, which means that ack sent on the reply subject was\n// dropped by the route.\nfunc TestJetStreamClusterQueueSubConsumer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R2S\", 2)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tReplicas: 1,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tinbox := nats.NewInbox()\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"ivan\",\n\t\t\tDeliverSubject: inbox,\n\t\t\tDeliverGroup: \"queue\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tAckWait: 100 * time.Millisecond,\n\t\t},\n\t}\n\treq, err := json.Marshal(obsReq)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"ivan\"), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error, got %+v\", ccResp.Error)\n\t}\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"ivan\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting consumer info: %v\", err)\n\t}\n\n\t// Now create a client that does NOT connect to the stream leader.\n\t// Start with url from first server in the cluster.\n\tu := c.servers[0].ClientURL()\n\t// If leader is \"S-1\", then use S-2 to connect to, which is at servers[1].\n\tif ci.Cluster.Leader == \"S-1\" {\n\t\tu = c.servers[1].ClientURL()\n\t}\n\tqsubnc, err := nats.Connect(u)\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer qsubnc.Close()\n\n\tch := make(chan struct{}, 2)\n\tif _, err := qsubnc.QueueSubscribe(inbox, \"queue\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t\tch <- struct{}{}\n\t}); err != nil {\n\t\tt.Fatalf(\"Error creating sub: %v\", err)\n\t}\n\n\t// Use the other connection to publish a message\n\tif _, err := js.Publish(\"foo.bar\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\n\t// Wait that we receive the message first.\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not receive message\")\n\t}\n\n\t// Message should be ack'ed and not redelivered.\n\tselect {\n\tcase <-ch:\n\t\tt.Fatal(\"Message redelivered!!!\")\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\nfunc TestJetStreamClusterLeaderStepdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\tcl := c.leader()\n\t// Now ask the system account to have the leader stepdown.\n\ts := c.randomNonLeader()\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(JSApiLeaderStepDown, nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on stepdown request: %v\", err)\n\t}\n\tvar sdr JSApiLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &sdr); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif sdr.Error != nil || !sdr.Success {\n\t\tt.Fatalf(\"Unexpected error for leader stepdown: %+v\", sdr.Error)\n\t}\n\n\tc.waitOnLeader()\n\tif cl == c.leader() {\n\t\tt.Fatalf(\"Expected a new metaleader, got same\")\n\t}\n}\n\nfunc TestJetStreamClusterSourcesFilteringAndUpdating(t *testing.T) {\n\towt := srcConsumerWaitTime\n\tsrcConsumerWaitTime = 2 * time.Second\n\tdefer func() { srcConsumerWaitTime = owt }()\n\n\tc := createJetStreamClusterExplicit(t, \"MSR\", 5)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckSync := func(msgsTest, msgsM uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 30*time.Second, time.Second, func() error {\n\t\t\tif tsi, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if msi, err := js.StreamInfo(\"M\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if tsi.State.Msgs != msgsTest {\n\t\t\t\treturn fmt.Errorf(\"received %d msgs from TEST, expected %d\", tsi.State.Msgs, msgsTest)\n\t\t\t} else if msi.State.Msgs != msgsM {\n\t\t\t\treturn fmt.Errorf(\"received %d msgs from M, expected %d\", msi.State.Msgs, msgsM)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\tdefer js.DeleteStream(\"TEST\")\n\n\t// Create M stream with a single source on \"foo\"\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\", FilterSubject: \"foo\"}},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\tdefer js.DeleteStream(\"M\")\n\n\t// check a message on \"bar\" doesn't get sourced\n\tsendBatch(\"bar\", 100)\n\tcheckSync(100, 0)\n\t// check a message on \"foo\" does get sourced\n\tsendBatch(\"foo\", 100)\n\tcheckSync(200, 100)\n\n\t// change remove the source on \"foo\" and add a new source on \"bar\"\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\", FilterSubject: \"bar\"}},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\t// As it is a new source (never been sourced before) it starts sourcing at the start of TEST\n\t// and therefore sources the message on \"bar\" that is in TEST\n\tcheckSync(200, 200)\n\n\t// new messages on \"foo\" are being filtered as it's not being currently sourced\n\tsendBatch(\"foo\", 100)\n\tcheckSync(300, 200)\n\t// new messages on \"bar\" are being sourced\n\tsendBatch(\"bar\", 100)\n\tcheckSync(400, 300)\n\n\t// re-add the source for \"foo\" keep the source on \"bar\"\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\", FilterSubject: \"bar\"}, {Name: \"TEST\", FilterSubject: \"foo\"}},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\t// check the 'backfill' of messages on \"foo\" that were published while the source was inactive\n\tcheckSync(400, 400)\n\n\t// causes startingSequenceForSources() to be called\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// check that it restarted the sources' consumers at the right place\n\tcheckSync(400, 400)\n\n\t// check both sources are still active\n\tsendBatch(\"bar\", 100)\n\tcheckSync(500, 500)\n\tsendBatch(\"foo\", 100)\n\tcheckSync(600, 600)\n\n\t// Check that purging the stream and does not cause the re-sourcing of the messages\n\tjs.PurgeStream(\"M\")\n\tcheckSync(600, 0)\n\n\t// Even after a leader change or restart\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\t// Allow direct sync consumers to connect.\n\t// This test could pass if we do not hook up and try to deliver the messages when we should not.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckSync(600, 0)\n}\n\nfunc TestJetStreamClusterSourcesUpdateOriginError(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSR\", 5)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsendBatch := func(subject string, n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckSync := func(msgsTest, msgsM uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif tsi, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if msi, err := js.StreamInfo(\"M\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if tsi.State.Msgs != msgsTest {\n\t\t\t\treturn fmt.Errorf(\"received %d msgs from TEST, expected %d\", tsi.State.Msgs, msgsTest)\n\t\t\t} else if msi.State.Msgs != msgsM {\n\t\t\t\treturn fmt.Errorf(\"received %d msgs from M, expected %d\", msi.State.Msgs, msgsM)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\", FilterSubject: \"foo\"}},\n\t\tReplicas: 2,\n\t})\n\n\trequire_NoError(t, err)\n\n\t// Send 100 msgs.\n\tsendBatch(\"foo\", 100)\n\tcheckSync(100, 100)\n\n\t// update makes source invalid\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\t// TODO check for downstream error propagation\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_Error(t, err)\n\n\tsendBatch(\"bar\", 100)\n\t// The source stream remains at 100 msgs as it still uses foo as it's filter\n\tcheckSync(200, 100)\n\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckSync(200, 100)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: no response from stream\")\n\n\tsendBatch(\"bar\", 100)\n\t// The source stream remains at 100 msgs as it still uses foo as it's filter\n\tcheckSync(300, 100)\n}\n\nfunc TestJetStreamClusterMirrorAndSourcesClusterRestart(t *testing.T) {\n\towt := srcConsumerWaitTime\n\tsrcConsumerWaitTime = 2 * time.Second\n\tdefer func() { srcConsumerWaitTime = owt }()\n\n\ttest := func(t *testing.T, mirror bool, filter bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"MSR\", 5)\n\t\tdefer c.shutdown()\n\n\t\t// Client for API requests.\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t// Origin\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\"},\n\t\t\tReplicas: 2,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfilterSubj := _EMPTY_\n\t\tif filter {\n\t\t\tfilterSubj = \"foo\"\n\t\t}\n\n\t\t// Create Mirror/Source now.\n\t\tif mirror {\n\t\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"M\",\n\t\t\t\tMirror: &nats.StreamSource{Name: \"TEST\", FilterSubject: filterSubj},\n\t\t\t\tReplicas: 2,\n\t\t\t})\n\t\t} else {\n\t\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"M\",\n\t\t\t\tSources: []*nats.StreamSource{{Name: \"TEST\", FilterSubject: filterSubj}},\n\t\t\t\tReplicas: 2,\n\t\t\t})\n\t\t}\n\t\trequire_NoError(t, err)\n\n\t\texpectedMsgCount := uint64(0)\n\n\t\tsendBatch := func(subject string, n int) {\n\t\t\tt.Helper()\n\t\t\tif subject == \"foo\" || !filter {\n\t\t\t\texpectedMsgCount += uint64(n)\n\t\t\t}\n\t\t\t// Send a batch to a given subject.\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tif _, err := js.Publish(subject, []byte(\"OK\")); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcheckSync := func(msgsTest, msgsM uint64) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 40*time.Second, time.Second, func() error {\n\t\t\t\tif tsi, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if msi, err := js.StreamInfo(\"M\"); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if tsi.State.Msgs != msgsTest {\n\t\t\t\t\treturn fmt.Errorf(\"received %d msgs from TEST, expected %d\", tsi.State.Msgs, msgsTest)\n\t\t\t\t} else if msi.State.Msgs != msgsM {\n\t\t\t\t\treturn fmt.Errorf(\"received %d msgs from M, expected %d\", msi.State.Msgs, msgsM)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\tsendBatch(\"foo\", 100)\n\t\tcheckSync(100, expectedMsgCount)\n\t\tsendBatch(\"bar\", 100)\n\t\tcheckSync(200, expectedMsgCount)\n\n\t\tnc.Close()\n\t\tc.stopAll()\n\t\tc.restartAll()\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\t\tnc, js = jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcheckSync(200, expectedMsgCount)\n\t\tsendBatch(\"foo\", 100)\n\t\tcheckSync(300, expectedMsgCount)\n\t\tsendBatch(\"bar\", 100)\n\t\tcheckSync(400, expectedMsgCount)\n\t}\n\tt.Run(\"mirror-filter\", func(t *testing.T) {\n\t\ttest(t, true, true)\n\t})\n\tt.Run(\"mirror-nofilter\", func(t *testing.T) {\n\t\ttest(t, true, false)\n\t})\n\tt.Run(\"source-filter\", func(t *testing.T) {\n\t\ttest(t, false, true)\n\t})\n\tt.Run(\"source-nofilter\", func(t *testing.T) {\n\t\ttest(t, false, false)\n\t})\n}\n\nfunc TestJetStreamClusterMirrorAndSourcesFilteredConsumers(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, \"MS5\", 5)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Create Mirror now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tdsubj := nats.NewInbox()\n\tnc.SubscribeSync(dsubj)\n\tnc.Flush()\n\n\tcreateConsumer := func(sn, fs string) {\n\t\tt.Helper()\n\t\t_, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\texpectFail := func(sn, fs string) {\n\t\tt.Helper()\n\t\t_, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs})\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"Expected error but got none\")\n\t\t}\n\t}\n\n\tcreateConsumer(\"M\", \"foo\")\n\tcreateConsumer(\"M\", \"bar\")\n\tcreateConsumer(\"M\", \"baz.foo\")\n\texpectFail(\"M\", \".\")\n\texpectFail(\"M\", \">.foo\")\n\n\t// Make sure wider scoped subjects work as well.\n\tcreateConsumer(\"M\", \"*\")\n\tcreateConsumer(\"M\", \">\")\n\n\t// Now do some sources.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"O1\", Subjects: []string{\"foo.*\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"O2\", Subjects: []string{\"bar.*\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create downstream now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{Name: \"O1\"}, {Name: \"O2\"}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcreateConsumer(\"S\", \"foo.1\")\n\tcreateConsumer(\"S\", \"bar.1\")\n\n\t// Now cross account stuff.\n\tnc2, js2 := jsClientConnect(t, s, nats.UserInfo(\"rip\", \"pass\"))\n\tdefer nc2.Close()\n\n\tif _, err := js2.AddStream(&nats.StreamConfig{Name: \"ORIGIN\", Subjects: []string{\"foo.*\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcfg := StreamConfig{\n\t\tName: \"SCA\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{{\n\t\t\tName: \"ORIGIN\",\n\t\t\tExternal: &ExternalStream{\n\t\t\t\tApiPrefix: \"RI.JS.API\",\n\t\t\t\tDeliverPrefix: \"RI.DELIVER.SYNC.SOURCES\",\n\t\t\t},\n\t\t}},\n\t}\n\treq, err := json.Marshal(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar scResp JSApiStreamCreateResponse\n\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.StreamInfo == nil || scResp.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", scResp.Error)\n\t}\n\n\t// Externals skip the checks for now.\n\tcreateConsumer(\"SCA\", \"foo.1\")\n\tcreateConsumer(\"SCA\", \"bar.1\")\n\tcreateConsumer(\"SCA\", \"baz\")\n}\n\nfunc TestJetStreamClusterCrossAccountMirrorsAndSources(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, \"C1\", 3)\n\tdefer c.shutdown()\n\n\t// Create source stream under RI account.\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"rip\", \"pass\"))\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// use large number to tease out FC issues\n\ttoSend := 3000\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tnc2, js2 := jsClientConnect(t, s)\n\tdefer nc2.Close()\n\n\t// Have to do this direct until we get Go client support.\n\t// Need to match jsClusterMirrorSourceImportsTempl imports.\n\t_, err := js2.AddStream(&nats.StreamConfig{\n\t\tName: \"MY_MIRROR_TEST\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\tAPIPrefix: \"RI.JS.API\",\n\t\t\t\tDeliverPrefix: \"RI.DELIVER.SYNC.MIRRORS\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MY_MIRROR_TEST\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not retrieve stream info: %s\", err)\n\t\t}\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", toSend, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do sources as well.\n\t_, err = js2.AddStream(&nats.StreamConfig{\n\t\tName: \"MY_SOURCE_TEST\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"RI.JS.API\",\n\t\t\t\t\tDeliverPrefix: \"RI.DELIVER.SYNC.SOURCES\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MY_SOURCE_TEST\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not retrieve stream info\")\n\t\t}\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", toSend, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamClusterFailMirrorsAndSources(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, \"C1\", 3)\n\tdefer c.shutdown()\n\n\t// Create source stream under RI account.\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"rip\", \"pass\"))\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2, Subjects: []string{\"test.>\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnc2, _ := jsClientConnect(t, s, nats.UserInfo(\"rip\", \"pass\"))\n\tdefer nc2.Close()\n\n\ttestPrefix := func(testName string, id ErrorIdentifier, cfg StreamConfig) {\n\t\tt.Run(testName, func(t *testing.T) {\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tresp, err := nc2.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar scResp JSApiStreamCreateResponse\n\t\t\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif scResp.Error == nil {\n\t\t\t\tt.Fatalf(\"Did expect an error but got none\")\n\t\t\t} else if !IsNatsErr(scResp.Error, id) {\n\t\t\t\tt.Fatalf(\"Expected different error: %s\", scResp.Error.Description)\n\t\t\t}\n\t\t})\n\t}\n\n\ttestPrefix(\"mirror-bad-apiprefix\", JSStreamExternalApiOverlapErrF, StreamConfig{\n\t\tName: \"MY_MIRROR_TEST\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &ExternalStream{\n\t\t\t\tApiPrefix: \"$JS.API\",\n\t\t\t\tDeliverPrefix: \"here\",\n\t\t\t},\n\t\t},\n\t})\n\ttestPrefix(\"source-bad-apiprefix\", JSStreamExternalApiOverlapErrF, StreamConfig{\n\t\tName: \"MY_SOURCE_TEST\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &ExternalStream{\n\t\t\t\tApiPrefix: \"$JS.API\",\n\t\t\t\tDeliverPrefix: \"here\",\n\t\t\t},\n\t\t},\n\t\t},\n\t})\n}\n\nfunc TestJetStreamClusterConsumerDeliveredSyncReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"mc\")\n\trequire_NoError(t, err)\n\n\t// Make us match first, but not next 10.\n\t_, err = js.Publish(\"foo.bar\", nil)\n\trequire_NoError(t, err)\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo.baz\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\topts := &JSzOptions{Accounts: true, Streams: true, Consumer: true}\n\tfor _, s := range c.servers {\n\t\tjsz, err := s.Jsz(opts)\n\t\trequire_NoError(t, err)\n\t\tci := jsz.AccountDetails[0].Streams[0].Consumer[0]\n\t\trequire_Equal(t, ci.Delivered.Consumer, 0)\n\t\trequire_Equal(t, ci.Delivered.Stream, 0)\n\t}\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 1)\n\tmeta, err := msgs[0].Metadata()\n\trequire_NoError(t, err)\n\trequire_Equal(t, meta.Sequence.Consumer, 1)\n\trequire_Equal(t, meta.Sequence.Stream, 1)\n\n\t// Allow some time for the state to propagate.\n\tmaxWait := 200 * time.Millisecond\n\ttime.Sleep(maxWait)\n\n\tfor _, s := range c.servers {\n\t\tjsz, err := s.Jsz(opts)\n\t\trequire_NoError(t, err)\n\t\tci := jsz.AccountDetails[0].Streams[0].Consumer[0]\n\t\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\t\trequire_Equal(t, ci.Delivered.Stream, 1)\n\t}\n\n\t// Now we want to make sure that jsz reporting will show the same\n\t// state, including delivered, which will have skipped to the end.\n\t// The skip can happen on several factors, but for here we just send\n\t// another pull request which we will let fail.\n\t_, err = sub.Fetch(1, nats.MaxWait(maxWait))\n\trequire_Error(t, err)\n\n\tfor _, s := range c.servers {\n\t\tjsz, err := s.Jsz(opts)\n\t\trequire_NoError(t, err)\n\t\tci := jsz.AccountDetails[0].Streams[0].Consumer[0]\n\t\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\t\trequire_Equal(t, ci.Delivered.Stream, 11)\n\t}\n}\n\n// This is to test follower ack fill logic when pending not empty.\n// There was a bug that would update p.Sequence in the stores (mem & file)\n// that would cause the logic to fail. Redeliveries were required to trigger.\nfunc TestJetStreamClusterConsumerAckSyncReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"mc\", nats.AckWait(250*time.Millisecond))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo.bar\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for all servers to converge on the same state. Stream and consumer leader could be different.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tmsgs, err := sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\t// Let redeliveries kick in.\n\ttime.Sleep(time.Second)\n\tmsgs, err = sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t// Randomize\n\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\n\tdontAck := uint64(7)\n\tvar skipped, last *nats.Msg\n\tfor _, m := range msgs {\n\t\tmeta, err := m.Metadata()\n\t\trequire_NoError(t, err)\n\t\tif meta.Sequence.Stream == dontAck {\n\t\t\tskipped = m\n\t\t\tcontinue\n\t\t}\n\t\tif meta.Sequence.Stream == 10 {\n\t\t\tlast = m\n\t\t\tcontinue\n\t\t}\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\trequire_NotNil(t, skipped)\n\trequire_NotNil(t, last)\n\n\tcheckAckFloor := func(consumer, stream uint64) {\n\t\topts := &JSzOptions{Accounts: true, Streams: true, Consumer: true}\n\t\tcheckFor(t, 3*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tjsz, err := s.Jsz(opts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tci := jsz.AccountDetails[0].Streams[0].Consumer[0]\n\t\t\t\tif ci.AckFloor.Consumer != consumer {\n\t\t\t\t\treturn fmt.Errorf(\"AckFloor.Consumer is not %d: %v\", consumer, ci.AckFloor.Consumer)\n\t\t\t\t}\n\t\t\t\tif ci.AckFloor.Stream != stream {\n\t\t\t\t\treturn fmt.Errorf(\"AckFloor.Stream is not %d: %v\", stream, ci.AckFloor.Stream)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Now we want to make sure that jsz reporting will show the same\n\t// state for ack floor.\n\tcheckAckFloor(dontAck-1, dontAck-1)\n\n\t// Now ack the skipped message\n\terr = skipped.AckSync()\n\trequire_NoError(t, err)\n\tcheckAckFloor(9, 9)\n\n\t// Now ack the last message\n\terr = last.AckSync()\n\trequire_NoError(t, err)\n\tcheckAckFloor(20, 10)\n}\n\nfunc TestJetStreamClusterConsumerDeleteInterestPolicyMultipleConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make the first sequence high. We already protect against it but for extra sanity.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100_000_000})\n\trequire_NoError(t, err)\n\n\t// Create 2 consumers.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C2\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\texpectedStreamMsgs := func(msgs uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != msgs {\n\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", si.State.Msgs, msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\texpectedStreamMsgs(100)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"C1\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(50)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 50)\n\tfor _, m := range msgs {\n\t\trequire_NoError(t, m.AckSync())\n\t}\n\n\t// Now delete second one and make sure accounting correct.\n\terr = js.DeleteConsumer(\"TEST\", \"C2\")\n\trequire_NoError(t, err)\n\n\texpectedStreamMsgs(50)\n}\n\nfunc TestJetStreamClusterConsumerAckNoneInterestPolicyShouldNotRetainAfterDelivery(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make the first sequence high. We already protect against it but for extra sanity.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100_000_000})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tAckPolicy: nats.AckNonePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 100)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"C1\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(100)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 100)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != 0\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterConsumerDeleteAckNoneInterestPolicyWithOthers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make the first sequence high. We already protect against it but for extra sanity.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100_000_000})\n\trequire_NoError(t, err)\n\n\t// Create 2 consumers.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C2\",\n\t\tAckPolicy: nats.AckNonePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\texpectedStreamMsgs := func(msgs uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != msgs {\n\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", si.State.Msgs, msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\texpectedStreamMsgs(100)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"C1\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(100)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 100)\n\tfor _, m := range msgs {\n\t\trequire_NoError(t, m.AckSync())\n\t}\n\t// AckNone will hold.\n\texpectedStreamMsgs(100)\n\n\t// Now delete second one and make sure accounting correct.\n\terr = js.DeleteConsumer(\"TEST\", \"C2\")\n\trequire_NoError(t, err)\n\n\texpectedStreamMsgs(0)\n}\n\n// Make sure to not allow non-system accounts to move meta leader.\nfunc TestJetStreamClusterMetaStepdownFromNonSysAccount(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\n\t// Client based API\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tml := c.leader()\n\n\t_, err := nc.Request(JSApiLeaderStepDown, nil, time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Make sure we did not move.\n\tc.waitOnLeader()\n\trequire_Equal(t, ml, c.leader())\n\n\t// System user can move it.\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\tresp, err := snc.Request(JSApiLeaderStepDown, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar sdr JSApiLeaderStepDownResponse\n\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\trequire_True(t, sdr.Success)\n\trequire_Equal(t, sdr.Error, nil)\n\n\t// Make sure we did move this time.\n\tc.waitOnLeader()\n\trequire_NotEqual(t, ml, c.leader())\n}\n\nfunc TestJetStreamClusterMaxDeliveriesOnInterestStreams(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub1, err := js.PullSubscribe(\"foo.*\", \"c1\", nats.AckWait(10*time.Millisecond), nats.MaxDeliver(1))\n\trequire_NoError(t, err)\n\n\tsub2, err := js.PullSubscribe(\"foo.*\", \"c2\", nats.AckWait(10*time.Millisecond), nats.MaxDeliver(1))\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo.bar\", []byte(\"HELLO\"))\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\n\tmsgs, err := sub1.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 1)\n\n\tmsgs, err = sub2.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 1)\n\n\t// Wait for redelivery to both consumers which will do nothing.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Now check that stream and consumer infos are correct.\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\t// Messages that are skipped due to max deliveries should NOT remove messages.\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.Consumers, 2)\n\n\tfor _, cname := range []string{\"c1\", \"c2\"} {\n\t\tci, err := js.ConsumerInfo(\"TEST\", cname)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\t\trequire_Equal(t, ci.Delivered.Stream, 1)\n\t\trequire_Equal(t, ci.AckFloor.Consumer, 0)\n\t\trequire_Equal(t, ci.AckFloor.Stream, 0)\n\t\trequire_Equal(t, ci.NumAckPending, 0)\n\t\trequire_Equal(t, ci.NumRedelivered, 0)\n\t\trequire_Equal(t, ci.NumPending, 0)\n\t}\n}\n\nfunc TestJetStreamClusterMetaRecoveryUpdatesDeletesConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tjs := c.leader().getJetStream()\n\n\tcreate := []*Entry{\n\t\t{EntryNormal, encodeAddStreamAssignment(&streamAssignment{\n\t\t\tConfig: &StreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\t})},\n\t\t{EntryNormal, encodeAddConsumerAssignment(&consumerAssignment{\n\t\t\tStream: \"TEST\",\n\t\t\tConfig: &ConsumerConfig{Name: \"consumer\"},\n\t\t})},\n\t}\n\n\tdelete := []*Entry{\n\t\t{EntryNormal, encodeDeleteStreamAssignment(&streamAssignment{\n\t\t\tConfig: &StreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\t})},\n\t}\n\n\t// Need to be recovering so that we accumulate recoveryUpdates.\n\tjs.setMetaRecovering()\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\n\t// Push recovery entries that create the stream & consumer.\n\t_, _, err := js.applyMetaEntries(create, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.updateConsumers), 1)\n\n\t// Now push another recovery entry that deletes the stream. The\n\t// entry that creates the consumer should now be gone.\n\t_, _, err = js.applyMetaEntries(delete, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.removeStreams), 1)\n\trequire_Len(t, len(ru.updateConsumers), 0)\n}\n\nfunc TestJetStreamClusterMetaRecoveryRecreateFileStreamAsMemory(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tjs := c.leader().getJetStream()\n\n\tcreateFileStream := []*Entry{\n\t\t{EntryNormal, encodeAddStreamAssignment(&streamAssignment{\n\t\t\tConfig: &StreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\t})},\n\t}\n\n\tdeleteFileStream := []*Entry{\n\t\t{EntryNormal, encodeDeleteStreamAssignment(&streamAssignment{\n\t\t\tConfig: &StreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\t})},\n\t}\n\n\tcreateMemoryStream := []*Entry{\n\t\t{EntryNormal, encodeAddStreamAssignment(&streamAssignment{\n\t\t\tConfig: &StreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\t})},\n\t}\n\n\tcreateConsumer := []*Entry{\n\t\t{EntryNormal, encodeAddConsumerAssignment(&consumerAssignment{\n\t\t\tStream: \"TEST\",\n\t\t\tConfig: &ConsumerConfig{Name: \"consumer\"},\n\t\t})},\n\t}\n\n\t// Need to be recovering so that we accumulate recoveryUpdates.\n\tjs.setMetaRecovering()\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\n\t// We created a file-based stream first, but deleted it shortly after.\n\t_, _, err := js.applyMetaEntries(createFileStream, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.removeStreams), 0)\n\n\t// Now push another recovery entry that deletes the stream.\n\t// The file-based stream should not have been created.\n\t_, _, err = js.applyMetaEntries(deleteFileStream, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 0)\n\trequire_Len(t, len(ru.removeStreams), 1)\n\n\t// Now stage a memory-based stream to be created.\n\t_, _, err = js.applyMetaEntries(createMemoryStream, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.removeStreams), 1)\n\trequire_Len(t, len(ru.updateConsumers), 0)\n\n\t// Also create a consumer on that memory-based stream.\n\t_, _, err = js.applyMetaEntries(createConsumer, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.removeStreams), 1)\n\trequire_Len(t, len(ru.updateConsumers), 1)\n}\n\nfunc TestJetStreamClusterMetaRecoveryConsumerCreateAndRemove(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\tencodeAddConsumerAssignment func(ca *consumerAssignment) []byte\n\t}{\n\t\t{title: \"simple\", encodeAddConsumerAssignment: encodeAddConsumerAssignment},\n\t\t{title: \"compressed\", encodeAddConsumerAssignment: encodeAddConsumerAssignmentCompressed},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tjs := c.leader().getJetStream()\n\n\t\t\tca := &consumerAssignment{Stream: \"TEST\", Name: \"consumer\"}\n\t\t\tcreateConsumer := []*Entry{{EntryNormal, test.encodeAddConsumerAssignment(ca)}}\n\t\t\tdeleteConsumer := []*Entry{{EntryNormal, encodeDeleteConsumerAssignment(ca)}}\n\n\t\t\t// Need to be recovering so that we accumulate recoveryUpdates.\n\t\t\tjs.setMetaRecovering()\n\t\t\tru := &recoveryUpdates{\n\t\t\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\t\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\t\t\taddStreams: make(map[string]*streamAssignment),\n\t\t\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\t\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t\t\t}\n\n\t\t\t// Creating the consumer should append to update consumers list.\n\t\t\t_, _, err := js.applyMetaEntries(createConsumer, ru)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(ru.updateConsumers[\":TEST\"]), 1)\n\t\t\trequire_Len(t, len(ru.removeConsumers), 0)\n\n\t\t\t// Deleting the consumer should append to remove consumers list and remove from update list.\n\t\t\t_, _, err = js.applyMetaEntries(deleteConsumer, ru)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(ru.removeConsumers[\":TEST\"]), 1)\n\t\t\trequire_Len(t, len(ru.updateConsumers[\":TEST\"]), 0)\n\n\t\t\t// When re-creating the consumer, add to update list.\n\t\t\t_, _, err = js.applyMetaEntries(createConsumer, ru)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(ru.removeConsumers[\":TEST\"]), 1)\n\t\t\trequire_Len(t, len(ru.updateConsumers[\":TEST\"]), 1)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterMetaRecoveryAddAndUpdateStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tjs := c.leader().getJetStream()\n\tci := &ClientInfo{Account: globalAccountName, Cluster: \"R3S\"}\n\tcfg := &StreamConfig{Name: \"TEST\", Storage: FileStorage, Replicas: 3}\n\trg, perr := js.createGroupForStream(ci, cfg)\n\tif perr != nil {\n\t\trequire_NoError(t, perr)\n\t}\n\tsa := &streamAssignment{Client: ci, Config: cfg, Group: rg}\n\n\t// Need to be recovering so that we accumulate recoveryUpdates.\n\tjs.setMetaRecovering()\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\n\t// We create a new stream, and we'll update it shortly after.\n\tentries := []*Entry{{EntryNormal, encodeAddStreamAssignment(sa)}}\n\t_, _, err := js.applyMetaEntries(entries, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.updateStreams), 0)\n\trequire_Len(t, len(ru.removeStreams), 0)\n\n\t// Now update the stream. The recovery updates should contain both the add and update.\n\t// If only the update would exist, the stream would not be created below.\n\tsa.Config.Subjects = []string{\"foo\"}\n\tentries = []*Entry{{EntryNormal, encodeUpdateStreamAssignment(sa)}}\n\t_, _, err = js.applyMetaEntries(entries, ru)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.updateStreams), 1)\n\trequire_Len(t, len(ru.removeStreams), 0)\n\n\t// Check the stream is properly added.\n\tfor _, sa := range ru.addStreams {\n\t\tjs.processStreamAssignment(sa)\n\t}\n\tsa = js.streamAssignment(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sa)\n\trequire_Len(t, len(sa.Config.Subjects), 0)\n\n\t// Check the stream is properly updated.\n\tfor _, sa := range ru.updateStreams {\n\t\tjs.processUpdateStreamAssignment(sa)\n\t}\n\tsa = js.streamAssignment(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sa)\n\trequire_Len(t, len(sa.Config.Subjects), 1)\n}\n\n// Make sure if we received acks that are out of bounds, meaning past our\n// last sequence or before our first that they are ignored and errored if applicable.\nfunc TestJetStreamClusterConsumerAckOutOfBounds(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo.bar\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo.*\", \"C\")\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 1)\n\tmsgs[0].AckSync()\n\n\t// Now ack way past the last sequence.\n\t_, err = nc.Request(\"$JS.ACK.TEST.C.1.10000000000.0.0.0\", nil, 250*time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Make sure that now changes happened to our state.\n\tci, err := js.ConsumerInfo(\"TEST\", \"C\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Consumer, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 1)\n\n\ts := c.consumerLeader(\"$G\", \"TEST\", \"C\")\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\tc.restartServer(s)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"C\")\n\n\t// Confirm new leader has same state for delivered and ack floor.\n\tci, err = js.ConsumerInfo(\"TEST\", \"C\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Consumer, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 1)\n}\n\nfunc TestJetStreamClusterCatchupLoadNextMsgTooManyDeletes(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\tcatchupRequest *streamSyncRequest\n\t\tsetup func(js nats.JetStreamContext)\n\t\tassert func(sub *nats.Subscription)\n\t}{\n\t\t{\n\t\t\ttitle: \"within-delete-gap\",\n\t\t\tsetup: func(js nats.JetStreamContext) {},\n\t\t},\n\t\t{\n\t\t\ttitle: \"EOF\",\n\t\t\tsetup: func(js nats.JetStreamContext) {\n\t\t\t\terr := js.DeleteMsg(\"TEST\", 100)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Starts and ends with subject \"foo\", we'll purge so there's a large gap of deletes in the middle.\n\t\t\t// This should force runCatchup to use LoadNextMsg instead of LoadMsg.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tsubject := \"bar\"\n\t\t\t\tif i == 0 || i == 99 {\n\t\t\t\t\tsubject = \"foo\"\n\t\t\t\t}\n\t\t\t\t_, err = js.Publish(subject, nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"bar\"})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Optionally run some extra setup.\n\t\t\ttest.setup(js)\n\n\t\t\t// Reconnect to stream leader.\n\t\t\tl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tnc.Close()\n\t\t\tnc, _ = jsClientConnect(t, l, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\t\tdefer nc.Close()\n\n\t\t\t// Setup wiretap and grab stream.\n\t\t\tsendSubject := \"test-wiretap\"\n\t\t\tsub, err := nc.SubscribeSync(sendSubject)\n\t\t\trequire_NoError(t, err)\n\t\t\terr = nc.Flush() // Must flush, otherwise our subscription could be too late.\n\t\t\trequire_NoError(t, err)\n\t\t\tacc, err := l.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Run custom catchup request and the test's asserts.\n\t\t\tsreq := &streamSyncRequest{Peer: \"peer\", FirstSeq: 5, LastSeq: 5, DeleteRangesOk: true}\n\t\t\trequire_True(t, mset.srv.startGoRoutine(func() { mset.runCatchup(sendSubject, sreq) }))\n\n\t\t\t// Our first message should be a skip msg.\n\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, entryOp(msg.Data[0]), streamMsgOp)\n\t\t\tsubj, _, _, _, seq, ts, _, err := decodeStreamMsg(msg.Data[1:])\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, seq, 5)\n\t\t\trequire_Equal(t, subj, _EMPTY_)\n\t\t\trequire_Equal(t, ts, 0)\n\n\t\t\t// And end with EOF.\n\t\t\tmsg, err = sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msg.Data), 0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterCatchupMustStallWhenBehindOnApplies(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Reconnect to stream leader.\n\tl := c.streamLeader(globalAccountName, \"TEST\")\n\tnc.Close()\n\tnc, _ = jsClientConnect(t, l, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t// Setup wiretap and grab stream.\n\tsendSubject := \"test-wiretap\"\n\tsub, err := nc.SubscribeSync(sendSubject)\n\trequire_NoError(t, err)\n\terr = nc.Flush() // Must flush, otherwise our subscription could be too late.\n\trequire_NoError(t, err)\n\tacc, err := l.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// We have a message at sequence 1, so expect a successful catchup.\n\tsreq1 := &streamSyncRequest{Peer: \"peer\", FirstSeq: 1, LastSeq: 1, DeleteRangesOk: true}\n\trequire_True(t, mset.srv.startGoRoutine(func() { mset.runCatchup(sendSubject, sreq1) }))\n\t// Expect the message at sequence 1.\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entryOp(msg.Data[0]), streamMsgOp)\n\tsubj, _, _, _, seq, _, _, err := decodeStreamMsg(msg.Data[1:])\n\trequire_NoError(t, err)\n\trequire_Equal(t, seq, 1)\n\trequire_Equal(t, subj, \"foo\")\n\t// And end with EOF.\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msg.Data), 0)\n\n\t// Add one additional entry into the log that's not applied yet.\n\tn := mset.node.(*raft)\n\tn.Lock()\n\tae := n.buildAppendEntry(nil)\n\terr = n.storeToWAL(ae)\n\tn.Unlock()\n\tindex, commit, applied := n.Progress()\n\trequire_NoError(t, err)\n\trequire_LessThan(t, applied, index)\n\trequire_Equal(t, commit, applied)\n\t// We have a message at sequence 1, but we haven't applied as many append entries.\n\t// We can't fulfill the request right now as we don't know yet if\n\t// that message will be deleted as part of upcoming append entries.\n\tsreq2 := &streamSyncRequest{Peer: \"peer\", FirstSeq: 1, LastSeq: 1, DeleteRangesOk: true, MinApplied: index}\n\trequire_True(t, mset.srv.startGoRoutine(func() { mset.runCatchup(sendSubject, sreq2) }))\n\t_, err = sub.NextMsg(time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// We have a message at sequence 1, but we haven't applied as many append entries.\n\t// Also, we seem to have a log that doesn't contain enough entries, even though we became leader.\n\t// Something has already gone wrong and got the logs to desync.\n\t// Value availability here and just fulfill the request.\n\tsreq3 := &streamSyncRequest{Peer: \"peer\", FirstSeq: 1, LastSeq: 1, DeleteRangesOk: true, MinApplied: 100}\n\trequire_True(t, mset.srv.startGoRoutine(func() { mset.runCatchup(sendSubject, sreq3) }))\n\t// Expect the message at sequence 1.\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entryOp(msg.Data[0]), streamMsgOp)\n\tsubj, _, _, _, seq, _, _, err = decodeStreamMsg(msg.Data[1:])\n\trequire_NoError(t, err)\n\trequire_Equal(t, seq, 1)\n\trequire_Equal(t, subj, \"foo\")\n\t// And end with EOF.\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msg.Data), 0)\n}\n\nfunc TestJetStreamClusterConsumerInfoAfterCreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnl := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, nl)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\tsi, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// We want to ensure the consumer can be created and be applied.\n\t// On the non-meta-leader server we'll pause applies, so need to make\n\t// sure the consumer is not created on that server.\n\tif si.Cluster.Leader == nl.Name() {\n\t\tml := c.leader()\n\t\tjreq, err := json.Marshal(&JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: ml.Name()}})\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), jreq, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar sdr JSApiLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\t}\n\n\t// Wait some time for the leader to settle.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Scale down to ensure the consumer gets created on this server.\n\tcfg.Replicas = 1\n\tsi, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequire_NotEqual(t, si.Cluster.Leader, nl.Name())\n\n\t// We pause applies for the server we're connected to.\n\t// This is fine for the RAFT log and allowing the consumer to be created,\n\t// but we will not be able to apply the consumer assignment for some time.\n\tmjs := nl.getJetStream()\n\trequire_NotNil(t, js)\n\tmg := mjs.getMetaGroup()\n\trequire_NotNil(t, mg)\n\terr = mg.(*raft).PauseApply()\n\trequire_NoError(t, err)\n\n\t// Add consumer.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\t// Consumer info should not fail, this server should not short-circuit because\n\t// it was not able to apply the consumer assignment.\n\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\n\t// Resume applies.\n\tmg.(*raft).ResumeApply()\n\n\t// Check consumer info still works.\n\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterStreamUpscalePeersAfterDownscale(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckPeerSet := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tpeers := mset.raftNode().Peers()\n\t\t\t\tif len(peers) != 5 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 5 peers, got %d\", len(peers))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckPeerSet()\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckPeerSet()\n}\n\nfunc TestJetStreamClusterClearAllPreAcksOnRemoveMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for all servers to converge on the same state.\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Register pre-acks on all servers.\n\t// Normally this can't happen as the stream leader will have the message that's acked available, just for testing.\n\tfor _, s := range c.servers {\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\n\t\t// Register pre-acks for the 3 messages.\n\t\tmset.registerPreAckLock(o, 1)\n\t\tmset.registerPreAckLock(o, 2)\n\t\tmset.registerPreAckLock(o, 3)\n\t}\n\n\t// Check there's an expected amount of pre-acks, and there are no pre-acks for the given sequence.\n\tcheckPreAcks := func(seq uint64, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tnumPreAcks := len(mset.preAcks)\n\t\t\t\tnumSeqPreAcks := len(mset.preAcks[seq])\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\tif numPreAcks != expected {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d pre-acks, got %d\", expected, numPreAcks)\n\t\t\t\t}\n\t\t\t\tif seq > 0 && numSeqPreAcks != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 0 pre-acks for seq %d, got %d\", seq, numSeqPreAcks)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Check all pre-acks were registered.\n\tcheckPreAcks(0, 3)\n\n\t// Deleting the message should clear the pre-ack.\n\terr = js.DeleteMsg(\"TEST\", 1)\n\trequire_NoError(t, err)\n\tcheckPreAcks(1, 2)\n\n\t// Erasing the message should clear the pre-ack.\n\terr = js.SecureDeleteMsg(\"TEST\", 2)\n\trequire_NoError(t, err)\n\tcheckPreAcks(2, 1)\n\n\t// Purging should clear all pre-acks below the purged floor.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 4})\n\trequire_NoError(t, err)\n\tcheckPreAcks(3, 0)\n}\n\nfunc TestJetStreamClusterStreamHealthCheckMustNotRecreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForStreamAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tjs := s.getJetStream()\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tsa := js.streamAssignment(globalAccountName, \"TEST\")\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\tif sa == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\twaitForNoStreamAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tjs := s.getJetStream()\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tsa := js.streamAssignment(globalAccountName, \"TEST\")\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\tif sa != nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment still available on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetStreamAssignment := func(rs *Server) (*jetStream, *Account, *streamAssignment, *stream) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tsa.Created = time.Time{}\n\t\treturn sjs, acc, sa, mset\n\t}\n\tcheckNodeIsClosed := func(sa *streamAssignment) {\n\t\tt.Helper()\n\t\trequire_True(t, sa.Group != nil)\n\t\trg := sa.Group\n\t\trequire_True(t, rg.node != nil)\n\t\tn := rg.node\n\t\trequire_Equal(t, n.State(), Closed)\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\twaitForStreamAssignments()\n\n\t// We manually stop the RAFT node and ensure it doesn't get restarted.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tsjs, acc, sa, mset := getStreamAssignment(rs)\n\trequire_True(t, sa.Group != nil)\n\trg := sa.Group\n\trequire_True(t, rg.node != nil)\n\tn := rg.node\n\tn.Stop()\n\tn.WaitForStop()\n\n\t// We wait for the monitor to exit, so we can set the flag back manually.\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\t\tif mset.inMonitor {\n\t\t\treturn errors.New(\"waiting for monitor to stop\")\n\t\t}\n\t\treturn nil\n\t})\n\tmset.mu.Lock()\n\tmset.inMonitor = true\n\tmset.mu.Unlock()\n\n\t// The RAFT node should be closed. Checking health must not change that.\n\t// Simulates a race condition where we're shutting down, but we're still in the stream monitor.\n\tcheckNodeIsClosed(sa)\n\tsjs.isStreamHealthy(acc, sa)\n\tcheckNodeIsClosed(sa)\n\n\terr = js.DeleteStream(\"TEST\")\n\trequire_NoError(t, err)\n\twaitForNoStreamAssignments()\n\n\t// Underlying layer would be aware the health check made a copy.\n\t// So we sneakily set these values back, which simulates a race condition where\n\t// the health check is called while the deletion is in progress. This could happen\n\t// depending on how the locks are used.\n\tsjs.mu.Lock()\n\tsjs.cluster.streams = make(map[string]map[string]*streamAssignment)\n\tsjs.cluster.streams[globalAccountName] = make(map[string]*streamAssignment)\n\tsjs.cluster.streams[globalAccountName][\"TEST\"] = sa\n\tsa.Group.node = n\n\tsjs.mu.Unlock()\n\n\t// The underlying stream has been deleted. Checking health must not recreate the stream.\n\tcheckNodeIsClosed(sa)\n\tsjs.isStreamHealthy(acc, sa)\n\tcheckNodeIsClosed(sa)\n}\n\nfunc TestJetStreamClusterStreamHealthCheckMustNotDeleteEarly(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForStreamAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tjs := s.getJetStream()\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tsa := js.streamAssignment(globalAccountName, \"TEST\")\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\tif sa == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetStreamAssignment := func(rs *Server) (*jetStream, *Account, *streamAssignment, *stream) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tsa.Created = time.Time{}\n\t\treturn sjs, acc, sa, mset\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\twaitForStreamAssignments()\n\n\t// We manually clear the node on the stream.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tsjs, acc, sa, mset := getStreamAssignment(rs)\n\tmset.mu.Lock()\n\tmset.node = nil\n\tmset.mu.Unlock()\n\tsjs.mu.Lock()\n\tgroup := sa.Group\n\tif group == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"sa.Group not initialized\")\n\t}\n\tnode := group.node\n\tif node == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"sa.Group.node not initialized\")\n\t}\n\tsjs.mu.Unlock()\n\n\t// The health check gets the Raft node of the assignment and checks it against the\n\t// Raft node of the stream. We simulate a race condition where the stream's Raft node\n\t// is not yet initialized. The health check MUST NOT delete the node.\n\tsjs.isStreamHealthy(acc, sa)\n\trequire_Equal(t, node.State(), Follower)\n}\n\nfunc TestJetStreamClusterStreamHealthCheckOnlyReportsSkew(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForStreamAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tjs := s.getJetStream()\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tsa := js.streamAssignment(globalAccountName, \"TEST\")\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\tif sa == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetStreamAssignment := func(rs *Server) (*jetStream, *Account, *streamAssignment, *stream) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tsa.Created = time.Time{}\n\t\treturn sjs, acc, sa, mset\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\twaitForStreamAssignments()\n\n\t// Confirm the stream and assignment Raft nodes are equal.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tsjs, acc, sa, mset := getStreamAssignment(rs)\n\tmset.mu.RLock()\n\tmsetNode := mset.node\n\tmset.mu.RUnlock()\n\tsjs.mu.Lock()\n\tgroup := sa.Group\n\tif group == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"sa.Group not initialized\")\n\t}\n\tnode := group.node\n\tif node == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"sa.Group.node not initialized\")\n\t}\n\tsjs.mu.Unlock()\n\trequire_Equal(t, msetNode, node)\n\n\t// Simulate stopping and restarting a new instance.\n\tnode.Stop()\n\tnode.WaitForStop()\n\tnode, err = sjs.createRaftGroup(globalAccountName, group, false, FileStorage, pprofLabels{})\n\trequire_NoError(t, err)\n\trequire_NotEqual(t, node.State(), Closed)\n\n\t// The health check gets the Raft node of the assignment and checks it against the\n\t// Raft node of the stream. We simulate a race condition where the assignment's Raft node\n\t// is re-newed, but the stream's node is still the old instance.\n\t// The health check MUST NOT delete the node.\n\trequire_Error(t, sjs.isStreamHealthy(acc, sa), errors.New(\"cluster node skew detected\"))\n\trequire_NotEqual(t, node.State(), Closed)\n}\n\nfunc TestJetStreamClusterStreamHealthCheckStreamCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tsjs := sl.getJetStream()\n\tacc := sl.globalAccount()\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tsjs.mu.Lock()\n\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\tsjs.mu.Unlock()\n\n\trequire_NoError(t, sjs.isStreamHealthy(acc, sa))\n\n\t// Check we can report unhealthy.\n\tn := mset.raftNode().(*raft)\n\tn.Lock()\n\tn.commit = 0\n\tn.Unlock()\n\trequire_Error(t, sjs.isStreamHealthy(acc, sa), errors.New(\"group node unhealthy\"))\n\n\t// Catching up should have precedence.\n\tmset.setCatchingUp()\n\trequire_True(t, mset.isCatchingUp())\n\trequire_Error(t, sjs.isStreamHealthy(acc, sa), errors.New(\"stream catching up\"))\n}\n\nfunc TestJetStreamClusterConsumerHealthCheckMustNotRecreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForConsumerAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s.getJetStream().consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\twaitForNoConsumerAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s.getJetStream().consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") != nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment still available on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetConsumerAssignment := func(rs *Server) (*jetStream, *streamAssignment, *consumerAssignment, *stream) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tca := sa.consumers[\"CONSUMER\"]\n\t\trequire_True(t, ca != nil)\n\t\tca.Created = time.Time{}\n\t\treturn sjs, sa, ca, mset\n\t}\n\tcheckNodeIsClosed := func(ca *consumerAssignment) {\n\t\tt.Helper()\n\t\trequire_True(t, ca.Group != nil)\n\t\trg := ca.Group\n\t\trequire_True(t, rg.node != nil)\n\t\tn := rg.node\n\t\trequire_Equal(t, n.State(), Closed)\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy, // Replicated consumers by default\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\twaitForConsumerAssignments()\n\n\t// We manually stop the RAFT node and ensure it doesn't get restarted.\n\trs := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tsjs, sa, ca, mset := getConsumerAssignment(rs)\n\trequire_True(t, ca.Group != nil)\n\trg := ca.Group\n\trequire_True(t, rg.node != nil)\n\tn := rg.node\n\tn.Stop()\n\tn.WaitForStop()\n\n\t// The RAFT node should be closed. Checking health must not change that.\n\t// Simulates a race condition where we're shutting down.\n\tcheckNodeIsClosed(ca)\n\trequire_Error(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca), errors.New(\"monitor goroutine not running\"))\n\tcheckNodeIsClosed(ca)\n\n\t// We create a new RAFT group, the health check should detect this skew.\n\t_, err = sjs.createRaftGroup(globalAccountName, ca.Group, false, MemoryStorage, pprofLabels{})\n\trequire_NoError(t, err)\n\tsjs.mu.Lock()\n\t// We set creating to now, since previously it would delete all data but NOT restart if created within <10s.\n\tca.Created = time.Now()\n\tsjs.mu.Unlock()\n\trequire_Error(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca), errors.New(\"cluster node skew detected\"))\n\n\terr = js.DeleteConsumer(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\twaitForNoConsumerAssignments()\n\n\t// Underlying layer would be aware the health check made a copy.\n\t// So we sneakily set these values back, which simulates a race condition where\n\t// the health check is called while the deletion is in progress. This could happen\n\t// depending on how the locks are used.\n\tsjs.mu.Lock()\n\tsjs.cluster.streams = make(map[string]map[string]*streamAssignment)\n\tsjs.cluster.streams[globalAccountName] = make(map[string]*streamAssignment)\n\tsjs.cluster.streams[globalAccountName][\"TEST\"] = sa\n\tca.Created = time.Time{}\n\tca.Group.node = n\n\tsjs.mu.Unlock()\n\n\t// The underlying consumer has been deleted. Checking health must not recreate the consumer.\n\tcheckNodeIsClosed(ca)\n\trequire_Error(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca), errors.New(\"consumer not found\"))\n\tcheckNodeIsClosed(ca)\n}\n\nfunc TestJetStreamClusterConsumerHealthCheckMustNotDeleteEarly(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForConsumerAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s.getJetStream().consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetConsumerAssignment := func(rs *Server) (*jetStream, *consumerAssignment, *stream, *consumer) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tca := sa.consumers[\"CONSUMER\"]\n\t\trequire_True(t, ca != nil)\n\t\tca.Created = time.Time{}\n\t\treturn sjs, ca, mset, o\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy, // Replicated consumers by default\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\twaitForConsumerAssignments()\n\n\t// We manually clear the node on the consumer.\n\trs := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tsjs, ca, mset, o := getConsumerAssignment(rs)\n\to.mu.Lock()\n\to.node = nil\n\to.mu.Unlock()\n\tsjs.mu.Lock()\n\tgroup := ca.Group\n\tif group == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"ca.Group not initialized\")\n\t}\n\tnode := group.node\n\tif node == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"ca.Group.node not initialized\")\n\t}\n\tsjs.mu.Unlock()\n\n\t// The health check gets the Raft node of the assignment and checks it against the\n\t// Raft node of the consumer. We simulate a race condition where the consumer's Raft node\n\t// is not yet initialized. The health check MUST NOT delete the node.\n\tsjs.isConsumerHealthy(mset, \"CONSUMER\", ca)\n\trequire_Equal(t, node.State(), Follower)\n}\n\nfunc TestJetStreamClusterConsumerHealthCheckOnlyReportsSkew(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\twaitForConsumerAssignments := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s.getJetStream().consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\t\t\treturn fmt.Errorf(\"stream assignment not found on %s\", s.Name())\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tgetConsumerAssignment := func(rs *Server) (*jetStream, *consumerAssignment, *stream, *consumer) {\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NotNil(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\n\t\tsjs := rs.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\n\t\tsas := sjs.cluster.streams[globalAccountName]\n\t\trequire_True(t, sas != nil)\n\t\tsa := sas[\"TEST\"]\n\t\trequire_True(t, sa != nil)\n\t\tca := sa.consumers[\"CONSUMER\"]\n\t\trequire_True(t, ca != nil)\n\t\tca.Created = time.Time{}\n\t\treturn sjs, ca, mset, o\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy, // Replicated consumers by default\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\twaitForConsumerAssignments()\n\n\t// Confirm the consumer and assignment Raft nodes are equal.\n\trs := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tsjs, ca, mset, o := getConsumerAssignment(rs)\n\to.mu.RLock()\n\toNode := o.node\n\to.mu.RUnlock()\n\tsjs.mu.Lock()\n\tgroup := ca.Group\n\tif group == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"ca.Group not initialized\")\n\t}\n\tnode := group.node\n\tif node == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"ca.Group.node not initialized\")\n\t}\n\tsjs.mu.Unlock()\n\trequire_Equal(t, oNode, node)\n\n\t// Simulate stopping and restarting a new instance.\n\tnode.Stop()\n\tnode.WaitForStop()\n\tnode, err = sjs.createRaftGroup(globalAccountName, group, false, FileStorage, pprofLabels{})\n\trequire_NoError(t, err)\n\trequire_NotEqual(t, node.State(), Closed)\n\n\t// The health check gets the Raft node of the assignment and checks it against the\n\t// Raft node of the consumer. We simulate a race condition where the assignment's Raft node\n\t// is re-newed, but the consumer's node is still the old instance.\n\t// The health check MUST NOT delete the node.\n\trequire_Error(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca), errors.New(\"cluster node skew detected\"))\n\trequire_NotEqual(t, node.State(), Closed)\n}\n\n// https://github.com/nats-io/nats-server/issues/7149\nfunc TestJetStreamClusterConsumerHealthCheckDeleted(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, cl)\n\tmset, err := cl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tsjs := cl.getJetStream()\n\tsjs.mu.Lock()\n\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\tif ca == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatal(\"ca not found\")\n\t}\n\t// Reset created time, simulating the consumer existed already for a while.\n\tca.Created = time.Time{}\n\tsjs.mu.Unlock()\n\n\t// The health check gathers all assignments and does checking after.\n\t// If the consumer was deleted in the meantime, it should not report an error.\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER\"))\n\trequire_Error(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca), errors.New(\"consumer not found\"))\n\n\t// The health check could run earlier than we're able to create the consumer.\n\t// In that case, wait before erroring.\n\tsjs.mu.Lock()\n\tca.Created = time.Now()\n\tsjs.mu.Unlock()\n\trequire_NoError(t, sjs.isConsumerHealthy(mset, \"CONSUMER\", ca))\n}\n\nfunc TestJetStreamClusterRespectConsumerStartSeq(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create replicated stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// We could have published messages into the stream that have not yet been applied on the follower.\n\t// If we create a consumer with a starting sequence in the future, we must respect it.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\tOptStartSeq: 20,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 19)\n\n\t// Same thing if the first sequence is not 0.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 10})\n\trequire_NoError(t, err)\n\n\t// Ensure all servers are up-to-date.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tci, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\tOptStartSeq: 20,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 19)\n\n\t// Only if we're requested to start at a sequence that's not available anymore\n\t// can we safely move it up. That data is gone already, so can't do anything else.\n\tci, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\tOptStartSeq: 5,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 9)\n}\n\nfunc TestJetStreamClusterPeerRemoveStreamConsumerDesync(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Must have at least one message in the stream.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tpeer := rs.Name()\n\trn := mset.raftNode()\n\tmset.mu.Lock()\n\tesm := encodeStreamMsgAllowCompress(\"foo\", _EMPTY_, nil, nil, mset.clseq, time.Now().UnixNano(), false)\n\tmset.clseq++\n\tmset.mu.Unlock()\n\t// Propose both remove peer and a normal entry within the same append entry.\n\terr = rn.ProposeMulti([]*Entry{\n\t\tnewEntry(EntryRemovePeer, []byte(peer)),\n\t\tnewEntry(EntryNormal, esm),\n\t})\n\trequire_NoError(t, err)\n\n\t// If the previous normal entry was skipped, we'd get a seq mismatch error here.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Now check the same but for a consumer.\n\t_, err = js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tacc, err = cl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err = acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\tupdateDeliveredBuffer := func() []byte {\n\t\tvar b [4*binary.MaxVarintLen64 + 1]byte\n\t\tb[0] = byte(updateDeliveredOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\tn += binary.PutUvarint(b[n:], 1)\n\t\tn += binary.PutVarint(b[n:], time.Now().UnixNano())\n\t\treturn b[:n]\n\t}\n\n\trs = c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tpeer = rs.Name()\n\trn = o.raftNode()\n\t// Propose both remove peer and a normal entry within the same append entry.\n\terr = rn.ProposeMulti([]*Entry{\n\t\tnewEntry(EntryRemovePeer, []byte(peer)),\n\t\tnewEntry(EntryNormal, updateDeliveredBuffer()),\n\t})\n\trequire_NoError(t, err)\n\n\t// Check the normal entry was applied.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\to.mu.Lock()\n\t\tdefer o.mu.Unlock()\n\t\tcs, err := o.store.State()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif cs.Delivered.Consumer != 100 || cs.Delivered.Stream != 100 {\n\t\t\treturn fmt.Errorf(\"expected sequence 100, got: %v\", cs.Delivered)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterStuckConsumerAfterLeaderChangeWithUnknownDeliveries(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish some messages into the stream.\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Ensure all servers are up-to-date.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// We only fetch 1 message here, since the condition is hard to trigger otherwise.\n\t// But, we're simulating fetching 3 messages and the consumer leader changing while\n\t// deliveries are happening. This will result in the new consumer leader not knowing\n\t// that the last two messages were also delivered (since we don't wait for quorum before delivering).\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\n\t// The client could send an acknowledgement, while the new consumer leader doesn't know about it\n\t// ever being delivered. It must NOT adjust any state and ignore the request to remain consistent.\n\t_, err = nc.Request(\"$JS.ACK.TEST.CONSUMER.1.3.3.0.0\", nil, time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Acknowledging a message that is known to be delivered is accepted still.\n\t_, err = nc.Request(\"$JS.ACK.TEST.CONSUMER.1.1.1.0.0\", nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Check for consistent consumer info.\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Consumer, 1)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Consumer, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 1)\n\n\t// Fetching for new messages MUST return the two messages the new consumer leader didn't\n\t// know were delivered before. If we wouldn't deliver these we'd have a stuck consumer.\n\tmsgs, err = sub.Fetch(2)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 2)\n\tfor _, msg := range msgs {\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\t// Check for consistent consumer info.\n\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Consumer, 3)\n\trequire_Equal(t, ci.Delivered.Stream, 3)\n\trequire_Equal(t, ci.AckFloor.Consumer, 3)\n\trequire_Equal(t, ci.AckFloor.Stream, 3)\n}\n\n// This is for when we are still using $SYS for NRG replication but we want to make sure\n// we track this in something visible to the end user.\nfunc TestJetStreamClusterAccountStatsForReplicatedStreams(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\t// Let's connect to stream leader to make sent messages predictable, otherwise we get those to come up based on routing to stream leader.\n\ts := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_True(t, s != nil)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// NRG traffic can be compressed, so make this unique so we can check stats correctly.\n\tmsg := make([]byte, 1024*1024)\n\tcrand.Read(msg)\n\n\t// Publish some messages into the stream.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Now grab the account stats for us and make sure we account for the replicated messages.\n\n\t// Opts to grab our account.\n\topts := &AccountStatzOptions{\n\t\tAccounts: []string{globalAccountName},\n\t}\n\tas, err := s.AccountStatz(opts)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(as.Accounts), 1)\n\n\taccStats := as.Accounts[0]\n\n\t// We need to account for possibility that the stream create was also on this server, hence the >= vs strict ==.\n\trequire_True(t, accStats.Received.Msgs >= 10)\n\trequire_True(t, accStats.Received.Bytes >= 1024*1024)\n\t// For sent, we will have 10 pub acks, and then should have 40 extra messages that are sent and accounted for\n\t// during the nrg propsal to the R5 peers.\n\trequire_True(t, accStats.Sent.Msgs >= 50)\n\trequire_True(t, accStats.Sent.Bytes >= accStats.Received.Bytes*4)\n}\n\nfunc TestJetStreamClusterRecreateConsumerFromMetaSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Initial setup.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\t// Wait for all servers to be fully up-to-date.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tif err = checkState(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, s := range c.servers {\n\t\t\tif acc, err := s.lookupAccount(globalAccountName); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if mset, err := acc.lookupStream(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if o := mset.lookupConsumer(\"CONSUMER\"); o == nil {\n\t\t\t\treturn errors.New(\"consumer doesn't exist\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Shutdown a random server.\n\trs := c.randomServer()\n\trs.Shutdown()\n\trs.WaitForShutdown()\n\n\t// Recreate connection, since we could have shutdown the server we were connected to.\n\tnc.Close()\n\tc.waitOnLeader()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Recreate consumer.\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER\"))\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\t// Wait for all servers (except for the one that's down) to have recreated the consumer.\n\tvar consumerRg string\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tconsumerRg = _EMPTY_\n\t\tfor _, s := range c.servers {\n\t\t\tif s == rs {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif acc, err := s.lookupAccount(globalAccountName); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if mset, err := acc.lookupStream(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if o := mset.lookupConsumer(\"CONSUMER\"); o == nil {\n\t\t\t\treturn errors.New(\"consumer doesn't exist\")\n\t\t\t} else if ccrg := o.raftNode().Group(); consumerRg == _EMPTY_ {\n\t\t\t\tconsumerRg = ccrg\n\t\t\t} else if consumerRg != ccrg {\n\t\t\t\treturn errors.New(\"consumer raft groups don't match\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Install snapshots on all remaining servers to \"hide\" the intermediate consumer recreate requests.\n\tfor _, s := range c.servers {\n\t\tif s != rs {\n\t\t\tsjs := s.getJetStream()\n\t\t\trequire_NotNil(t, sjs)\n\t\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\t\trequire_NoError(t, err)\n\t\t\tsjs.mu.RLock()\n\t\t\tmeta := sjs.cluster.meta\n\t\t\tsjs.mu.RUnlock()\n\t\t\trequire_NoError(t, meta.InstallSnapshot(snap, false))\n\t\t}\n\t}\n\n\t// Restart the server, it should receive a meta snapshot and recognize the consumer recreation.\n\trs = c.restartServer(rs)\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tconsumerRg = _EMPTY_\n\t\tfor _, s := range c.servers {\n\t\t\tif acc, err := s.lookupAccount(globalAccountName); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if mset, err := acc.lookupStream(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if o := mset.lookupConsumer(\"CONSUMER\"); o == nil {\n\t\t\t\treturn errors.New(\"consumer doesn't exist\")\n\t\t\t} else if rn := o.raftNode(); rn == nil {\n\t\t\t\treturn errors.New(\"consumer raft node doesn't exist\")\n\t\t\t} else if ccrg := rn.Group(); ccrg == _EMPTY_ {\n\t\t\t\treturn errors.New(\"consumer raft group doesn't exist\")\n\t\t\t} else if consumerRg == _EMPTY_ {\n\t\t\t\tconsumerRg = ccrg\n\t\t\t} else if consumerRg != ccrg {\n\t\t\t\treturn errors.New(\"consumer raft groups don't match\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterUpgradeStreamVersioning(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tml := c.leader()\n\tsjs := ml.getJetStream()\n\trn := sjs.getMetaGroup()\n\tacc, err := ml.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\n\t// Create stream config.\n\tcfg, apiErr := ml.checkStreamCfg(&StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}}, acc, false)\n\trequire_True(t, apiErr == nil)\n\n\t// Create and propose stream assignment.\n\tci := &ClientInfo{Cluster: \"R3S\", Account: globalAccountName}\n\trg, perr := sjs.createGroupForStream(ci, &cfg)\n\trequire_True(t, perr == nil)\n\tsa := &streamAssignment{Group: rg, Sync: syncSubjForStream(), Config: &cfg, Client: ci, Created: time.Now().UTC()}\n\trequire_NoError(t, rn.Propose(encodeAddStreamAssignment(sa)))\n\n\t// Wait for the stream assignment to have gone through.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\t\tif sjs.streamAssignment(globalAccountName, \"TEST\") == nil {\n\t\t\treturn errors.New(\"stream assignment does not exist yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor _, create := range []bool{true, false} {\n\t\ttitle := \"create\"\n\t\tif !create {\n\t\t\ttitle = \"update\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\tif create {\n\t\t\t\t// Create on 2.11+ should be idempotent with previous create on 2.10-.\n\t\t\t\tmcfg := &StreamConfig{}\n\t\t\t\tsetStaticStreamMetadata(mcfg)\n\t\t\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tMetadata: setDynamicStreamMetadata(mcfg).Metadata,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdeleteDynamicMetadata(si.Config.Metadata)\n\t\t\t\trequire_Len(t, len(si.Config.Metadata), 0)\n\t\t\t} else {\n\t\t\t\t// Update populates the versioning metadata.\n\t\t\t\tsi, err := js.UpdateStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(si.Config.Metadata), 3)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterUpgradeConsumerVersioning(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tml := c.leader()\n\tsjs := ml.getJetStream()\n\n\t// Wait for the stream assignment to have gone through.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\t\tif sjs.streamAssignment(globalAccountName, \"TEST\") == nil {\n\t\t\treturn errors.New(\"stream assignment does not exist yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\trn := sjs.getMetaGroup()\n\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sa)\n\tacc, err := ml.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\n\t// Create consumer config.\n\tcfg := &ConsumerConfig{Durable: \"CONSUMER\", Name: \"CONSUMER\"}\n\tstreamCfg, ok := sjs.clusterStreamConfig(acc.Name, \"TEST\")\n\tif !ok {\n\t\trequire_NoError(t, NewJSStreamNotFoundError())\n\t}\n\tselectedLimits, _, _, apiErr := acc.selectLimits(cfg.replicas(&streamCfg))\n\tif apiErr != nil {\n\t\trequire_NoError(t, apiErr)\n\t}\n\tsrvLim := &ml.getOpts().JetStreamLimits\n\tapiErr = setConsumerConfigDefaults(cfg, &streamCfg, srvLim, selectedLimits, false)\n\tif apiErr != nil {\n\t\trequire_NoError(t, apiErr)\n\t}\n\n\t// Create and propose consumer assignment.\n\tci := &ClientInfo{Cluster: \"R3S\", Account: globalAccountName}\n\trg := sjs.cluster.createGroupForConsumer(cfg, sa)\n\tca := &consumerAssignment{Group: rg, Stream: \"TEST\", Name: \"CONSUMER\", Config: cfg, Client: ci, Created: time.Now().UTC()}\n\trequire_NoError(t, rn.Propose(encodeAddConsumerAssignment(ca)))\n\n\t// Wait for the consumer assignment to have gone through.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\t\tif sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\treturn errors.New(\"consumer assignment does not exist yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor _, create := range []bool{true, false} {\n\t\ttitle := \"create\"\n\t\tif !create {\n\t\t\ttitle = \"update\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\tcreateConsumerRequest := func(obsReq CreateConsumerRequest) (*JSApiConsumerInfoResponse, error) {\n\t\t\t\treq, err := json.Marshal(obsReq)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"CONSUMER\"), req, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tvar resp JSApiConsumerInfoResponse\n\t\t\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\t\t\tif resp.Error != nil {\n\t\t\t\t\treturn nil, resp.Error\n\t\t\t\t}\n\t\t\t\treturn &resp, nil\n\t\t\t}\n\n\t\t\tif create {\n\t\t\t\t// Create on 2.11+ should be idempotent with previous create on 2.10-.\n\t\t\t\tncfg := &ConsumerConfig{Durable: \"CONSUMER\"}\n\t\t\t\tsetStaticConsumerMetadata(ncfg)\n\t\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\t\tStream: \"TEST\",\n\t\t\t\t\tConfig: *setDynamicConsumerMetadata(ncfg),\n\t\t\t\t\tAction: ActionCreate,\n\t\t\t\t}\n\t\t\t\tresp, err := createConsumerRequest(obsReq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdeleteDynamicMetadata(resp.Config.Metadata)\n\t\t\t\trequire_Len(t, len(resp.Config.Metadata), 0)\n\t\t\t} else {\n\t\t\t\t// Update populates the versioning metadata.\n\t\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\t\tStream: \"TEST\",\n\t\t\t\t\tConfig: ConsumerConfig{Durable: \"CONSUMER\"},\n\t\t\t\t\tAction: ActionUpdate,\n\t\t\t\t}\n\t\t\t\tresp, err := createConsumerRequest(obsReq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(resp.Config.Metadata), 3)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterInterestPolicyAckAll(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckAllPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"ok\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\texpectedStreamMsgs := func(msgs uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != msgs {\n\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", si.State.Msgs, msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\texpectedStreamMsgs(100)\n\n\tfor _, s := range c.servers {\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\t\to.mu.Lock()\n\t\t// Ensure o.checkStateForInterestStream can't hide that the issue happened.\n\t\to.chkflr = 1000\n\t\to.mu.Unlock()\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(50)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 50)\n\trequire_NoError(t, msgs[49].AckSync())\n\n\texpectedStreamMsgs(50)\n}\n\nfunc TestJetStreamClusterPreserveRedeliveredWithLaggingStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\", nats.AckWait(500*time.Millisecond))\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\trsf := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tacc, err := rsf.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_NoError(t, mset.raftNode().PauseApply())\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Move consumer leader to equal stream leader.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\treq := JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: sl.Name()}}\n\tdata, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"CONSUMER\"), data, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_Equal(t, c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\").Name(), sl.Name())\n\n\t// Get the message to be stored in redelivered state.\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmeta, err := msgs[0].Metadata()\n\trequire_NoError(t, err)\n\trequire_Equal(t, meta.NumDelivered, 1)\n\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmeta, err = msgs[0].Metadata()\n\trequire_NoError(t, err)\n\trequire_Equal(t, meta.NumDelivered, 2)\n\n\t// Move consumer leader to a different server.\n\treq = JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: rsf.Name()}}\n\tdata, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"CONSUMER\"), data, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_Equal(t, c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\").Name(), rsf.Name())\n\n\t// Confirming the stream contents of the consumer leader don't contain the message yet.\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_Error(t, err, nats.ErrTimeout)\n\trequire_Len(t, len(msgs), 0)\n\n\t// Now let it apply the missing message.\n\tmset.raftNode().ResumeApply()\n\n\t// Should preserve delivered state properly, otherwise the client would observe NumDelivered rolled back.\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmeta, err = msgs[0].Metadata()\n\trequire_NoError(t, err)\n\trequire_Equal(t, meta.NumDelivered, 3)\n}\n\nfunc TestJetStreamClusterInvalidJSACKOverRoute(t *testing.T) {\n\ttest := func(t *testing.T, stream, consumer, subject string) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: stream,\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\tRetention: nats.LimitsPolicy,\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(stream, &nats.ConsumerConfig{\n\t\t\tDurable: consumer,\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Reconnect to consumer follower.\n\t\tnc.Close()\n\t\trs := c.randomNonConsumerLeader(globalAccountName, stream, consumer)\n\t\tnc, js = jsClientConnect(t, rs)\n\t\tdefer nc.Close()\n\n\t\t_, err = js.Publish(subject, nil)\n\t\trequire_NoError(t, err)\n\n\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, stream)\n\t\t})\n\n\t\tsub, err := js.PullSubscribe(\"foo.>\", consumer)\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\n\t\t// The message went over a route and should have the correct subject, and working ACK.\n\t\tmsgs, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_Equal(t, msgs[0].Subject, subject)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\t}\n\n\tfor _, s := range []string{\"StreamNoAt\", \"StreamWith@At\"} {\n\t\tfor _, c := range []string{\"ConsumerNoAt\", \"ConsumerWith@At\"} {\n\t\t\tfor _, subject := range []string{\"foo.no.at\", \"foo.with@at\"} {\n\t\t\t\tt.Run(fmt.Sprintf(\"%s/%s/%s\", s, c, subject), func(t *testing.T) {\n\t\t\t\t\ttest(t, s, c, subject)\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterConsumerOnlyDeliverMsgAfterQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t\tAckWait: 2 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tacc, err := cl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\trn := o.raftNode().(*raft)\n\n\t// Force the leader to not be able to make proposals.\n\trn.Lock()\n\trn.werr = errors.New(\"block proposals\")\n\trn.Unlock()\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// We must only receive a message AFTER quorum was met for updating delivered state.\n\t// This should time out since proposals are blocked.\n\tmsgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))\n\trequire_Error(t, err, nats.ErrTimeout)\n\trequire_Len(t, len(msgs), 0)\n\n\t// Allow proposals to be made again.\n\trn.Lock()\n\trn.werr = nil\n\trn.Unlock()\n\n\t// Now it should pass.\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(2*time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n}\n\nfunc TestJetStreamClusterConsumerResetPendingDeliveriesOnMaxAckPendingUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t\tAckWait: 2 * time.Second,\n\t\tMaxAckPending: 100,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tacc, err := cl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\to.mu.Lock()\n\to.pendingDeliveries = map[uint64]*jsPubMsg{0: getJSPubMsgFromPool()}\n\to.mu.Unlock()\n\n\t// Increasing does not reset pending deliveries.\n\tcfg.MaxAckPending++\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\to.mu.Lock()\n\tl := len(o.pendingDeliveries)\n\to.mu.Unlock()\n\trequire_Equal(t, l, 1)\n\n\t// Decreasing does reset pending deliveries, so we can shrink the map.\n\tcfg.MaxAckPending--\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\to.mu.Lock()\n\tl = len(o.pendingDeliveries)\n\to.mu.Unlock()\n\trequire_Equal(t, l, 0)\n}\n\nfunc TestJetStreamClusterConsumerActiveAfterDidNotDeliverOverRoute(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tDeliverSubject: \"deliver_subject\",\n\t})\n\trequire_NoError(t, err)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_False(t, ci.PushBound)\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trs := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tncRs, jsRs := jsClientConnect(t, rs)\n\tdefer ncRs.Close()\n\n\t// Just noop callback.\n\tcb := func(msg *nats.Msg) {}\n\t_, err = jsRs.Subscribe(\"foo\", cb, nats.Bind(\"TEST\", \"CONSUMER\"))\n\trequire_NoError(t, err)\n\n\t// Eventually the sub should be known to the leader, and report active/bound.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ci.PushBound {\n\t\t\treturn errors.New(\"push not bound\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Simulate failed delivery over route connection.\n\tacc, err := cl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\to.didNotDeliver(1, \"foo\")\n\n\t// Consumer should still be active/bound.\n\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_True(t, ci.PushBound)\n}\n\nfunc TestJetStreamClusterOfflineR1StreamDenyUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Stop current R1 stream leader.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tsl.Shutdown()\n\tnc.Close()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Wait for meta leader, so we can send an update.\n\tc.waitOnLeader()\n\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_Error(t, err, NewJSStreamOfflineError())\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err, NewJSStreamOfflineError())\n}\n\nfunc TestJetStreamClusterOfflineR1ConsumerDenyUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 1,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\t// Stop current R1 consumer leader.\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tcl.Shutdown()\n\tnc.Close()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Wait for meta leader, so we can send an update.\n\tc.waitOnLeader()\n\n\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_Error(t, err, NewJSConsumerOfflineError())\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_Error(t, err, NewJSConsumerOfflineError())\n}\n\nfunc TestJetStreamClusterSnapshotStreamAssetOnShutdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tvar sds []string\n\tfor _, s := range c.servers {\n\t\tsds = append(sds, s.StoreDir())\n\t}\n\n\tfor _, sd := range sds {\n\t\tmatches, err := filepath.Glob(filepath.Join(sd, \"$SYS\", \"_js_\", \"*\", snapshotsDir, \"*\"))\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(matches) > 0)\n\t\tfor _, match := range matches {\n\t\t\trequire_NoError(t, os.RemoveAll(match))\n\t\t}\n\t}\n\n\t// Publish, so we have something new to snapshot.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Shutdown servers, and check if all made stream snapshots.\n\tfor _, s := range c.servers {\n\t\ts.Shutdown()\n\t}\n\tfor _, sd := range sds {\n\t\tmatches, err := filepath.Glob(filepath.Join(sd, \"$SYS\", \"_js_\", \"*\", snapshotsDir))\n\t\trequire_NoError(t, err)\n\t\t// Matches _meta_ and stream raft groups.\n\t\trequire_Len(t, len(matches), 2)\n\t\tvar foundStream bool\n\t\tfor _, match := range matches {\n\t\t\tif !strings.Contains(match, \"S-R3F\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfoundStream = true\n\t\t\tdirs, err := os.ReadDir(match)\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(dirs) != 1 {\n\t\t\t\tt.Errorf(\"Missing snapshot for %s\", match)\n\t\t\t}\n\t\t}\n\t\trequire_True(t, foundStream)\n\t}\n}\n\nfunc TestJetStreamClusterDontReviveRemovedStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Wait for stream to be assigned on all servers.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif !s.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\t\treturn fmt.Errorf(\"stream not yet assigned on %s\", s.Name())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Simulate the stream being scaled down, quickly after it was scaled up.\n\tcfg.Replicas = 1\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Wait for the stream to not be assigned on the servers (except the leader).\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif s.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\t\treturn fmt.Errorf(\"stream still assigned on %s\", s.Name())\n\t\t\t}\n\t\t}\n\t\tif !mset.closed.Load() {\n\t\t\treturn fmt.Errorf(\"stream not closed yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Simulating the stream was catching up and is resetting after timing out.\n\trequire_True(t, mset.resetClusteredState(nil))\n\n\t// Allow for some time here, mset.resetClusteredState might\n\t// spin up a goroutine if it's resetting the stream.\n\t// Confirm the stream is not revived.\n\ttime.Sleep(200 * time.Millisecond)\n\t_, err = rs.globalAccount().lookupStream(\"TEST\")\n\trequire_Error(t, err, ErrJetStreamStreamNotFound)\n}\n\nfunc TestJetStreamClusterCreateR3StreamWithOfflineNodes(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"FOO\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\t// Shutdown one server, we'll still have quorum.\n\trs1 := c.randomNonLeader()\n\trs1.Shutdown()\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"BAR\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\t// While we still have quorum, publish a few messages to the streams.\n\t// The offline server will need to both assign the stream and catchup when it comes back.\n\tfor _, subj := range []string{\"FOO\", \"BAR\"} {\n\t\t_, err = js.Publish(subj, nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Shutdown one more server, only one server left that can't reach quorum on its own.\n\trs2 := c.randomNonLeader()\n\trs2.Shutdown()\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"BAZ\", Replicas: 3})\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"peer offline\")\n\n\t// Restart offline servers and ensure all comes back up.\n\tc.restartServer(rs1)\n\tc.restartServer(rs2)\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"FOO\")\n\tc.waitOnStreamLeader(globalAccountName, \"BAR\")\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif err := checkState(t, c, globalAccountName, \"FOO\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := checkState(t, c, globalAccountName, \"BAR\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterCreateEphemeralConsumerWithOfflineNodes(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\t// Shutdown a random server.\n\tc.randomNonLeader().Shutdown()\n\n\tfor range 10 {\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{})\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestJetStreamClusterSetPreferredToOnlineNode(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Shutdown one random server.\n\tml := c.leader()\n\trs := c.randomNonLeader()\n\tjs := rs.getJetStream()\n\tcc := js.cluster\n\trequire_NotNil(t, cc)\n\tmeta := cc.meta\n\trequire_NotNil(t, meta)\n\trsid := meta.ID()\n\trs.Shutdown()\n\n\tcfg := &StreamConfig{Storage: FileStorage, Replicas: 3}\n\tjs = ml.getJetStream()\n\tcc = js.cluster\n\trequire_NotNil(t, cc)\n\tpeers, err := cc.selectPeerGroup(3, \"R3S\", cfg, nil, 0, nil)\n\t// Need explicit nil-check.\n\tif err != nil {\n\t\trequire_NoError(t, err)\n\t}\n\trequire_Len(t, len(peers), 3)\n\n\t// Should prefer online peers when selecting preferred.\n\trg := &raftGroup{Name: groupNameForStream(peers, cfg.Storage), Storage: cfg.Storage, Peers: peers, Cluster: \"R3S\"}\n\trg.setPreferred(ml)\n\trequire_NotEqual(t, rg.Preferred, rsid)\n}\n\nfunc TestJetStreamClusterAsyncFlushBasics(t *testing.T) {\n\ttest := func(t *testing.T, syncAlways bool) {\n\t\tsupportsAsync := !syncAlways\n\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tfor _, s := range c.servers {\n\t\t\ts.optsMu.Lock()\n\t\t\ts.opts.SyncAlways = syncAlways\n\t\t\ts.optsMu.Unlock()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar s *Server\n\t\tcheckStoreIsAsync := func(expectAsync bool) {\n\t\t\tt.Helper()\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tfs := mset.Store().(*fileStore)\n\t\t\tfs.mu.RLock()\n\t\t\tlmb := fs.lmb\n\t\t\tasyncFlush := fs.fcfg.AsyncFlush\n\t\t\tfip := fs.fip\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_Equal(t, asyncFlush, expectAsync)\n\t\t\trequire_Equal(t, fip, !expectAsync)\n\t\t\trequire_NotNil(t, lmb)\n\t\t\tlmb.mu.RLock()\n\t\t\tflusher := lmb.flusher\n\t\t\tlmb.mu.RUnlock()\n\t\t\tif expectAsync {\n\t\t\t\tif !flusher {\n\t\t\t\t\tt.Fatal(\"flusher not initialized\")\n\t\t\t\t} else if !asyncFlush {\n\t\t\t\t\tt.Fatal(\"async flush config not set\")\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif flusher {\n\t\t\t\t\tt.Fatal(\"flusher still initialized\")\n\t\t\t\t} else if asyncFlush {\n\t\t\t\t\tt.Fatal(\"async flush config not reset\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: 1,\n\t\t}\n\t\t// Test disabled async flush on create.\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\ts = c.streamLeader(globalAccountName, \"TEST\")\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckStoreIsAsync(false)\n\n\t\t// Enabling async flush.\n\t\tcfg.Replicas = 3\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckStoreIsAsync(supportsAsync)\n\n\t\t// Disabling async flush.\n\t\tcfg.Replicas = 1\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckStoreIsAsync(false)\n\n\t\t// Test async flush on create.\n\t\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t\tcfg.Replicas = 3\n\t\t_, err = jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\ts = c.streamLeader(globalAccountName, \"TEST\")\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckStoreIsAsync(supportsAsync)\n\t}\n\n\tt.Run(\"Default\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"SyncAlways\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterAsyncFlushFileStoreFlushOnSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now shutdown flusher and wait for it to be closed,\n\t// we'll not flush published messages automatically anymore.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfs := mset.Store().(*fileStore)\n\tfs.mu.RLock()\n\tlmb := fs.lmb\n\tfs.mu.RUnlock()\n\trequire_NotNil(t, lmb)\n\n\tlmb.mu.Lock()\n\tif lmb.qch != nil {\n\t\tclose(lmb.qch)\n\t\tlmb.qch = nil\n\t}\n\tlmb.mu.Unlock()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tlmb.mu.RLock()\n\t\tdefer lmb.mu.RUnlock()\n\t\tif lmb.flusher {\n\t\t\treturn errors.New(\"flusher still active\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Get the highest applied count from snapshot before below publish.\n\tn := mset.raftNode().(*raft)\n\t_, _, previousApplied := n.Progress()\n\n\t// Confirm no pending writes.\n\trequire_Equal(t, lmb.pendingWriteSize(), 0)\n\n\t// Publishing a message will still work, because writes are async.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Confirm above write is pending.\n\trequire_Equal(t, lmb.pendingWriteSize(), 33)\n\n\t// Make the upper layer snapshot by sending leader change signal.\n\t// It doesn't matter that we're already leader, it still gets handled.\n\t// Previously this used the mqch, but that now only snapshots on shutdown.\n\tn.leadc <- true\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tn.Lock()\n\t\tsnap, err := n.loadLastSnapshot()\n\t\tn.Unlock()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif snap.lastIndex <= previousApplied {\n\t\t\treturn fmt.Errorf(\"snapshot still at lastIndex=%d, expected=%d\", snap.lastIndex, previousApplied)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Confirm the write is flushed as a result of the snapshot.\n\trequire_Equal(t, lmb.pendingWriteSize(), 0)\n}\n\nfunc TestJetStreamClusterScheduledDelayedMessage(t *testing.T) {\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, storage), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: \"SchedulesDisabled\",\n\t\t\t\t\tSubjects: []string{\"disabled\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: false,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tschedulePattern := \"@at 1970-01-01T00:00:00Z\"\n\t\t\t\tm := nats.NewMsg(\"disabled\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern) // Needs to be valid, but is not used.\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesDisabledError())\n\n\t\t\t\tm = nats.NewMsg(\"disabled\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", \"disabled\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesDisabledError())\n\n\t\t\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: \"SchedulesEnabledNoTtl\",\n\t\t\t\t\tSubjects: []string{\"disabled.ttl\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: true,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tm = nats.NewMsg(\"disabled.ttl\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern) // Needs to be valid, but is not used.\n\t\t\t\tm.Header.Set(\"Nats-Schedule-TTL\", \"1s\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageTTLDisabledError())\n\n\t\t\t\tcfg := &StreamConfig{\n\t\t\t\t\tName: \"SchedulesEnabled\",\n\t\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: true,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t}\n\t\t\t\t_, err = jsStreamCreate(t, nc, cfg)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", \"invalid\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesPatternInvalidError())\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"not.matching\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesTargetInvalidError())\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.*\") // Must be literal.\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesTargetInvalidError())\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.invalid\") // Can't equal the publish subject.\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesTargetInvalidError())\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-TTL\", \"invalid\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesTTLInvalidError())\n\n\t\t\t\tm = nats.NewMsg(\"foo.invalid\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-TTL\", \"1s\")\n\t\t\t\tm.Header.Set(\"Nats-Rollup\", \"all\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesRollupInvalidError())\n\n\t\t\t\t// Schedule with a delay that takes a very long time.\n\t\t\t\tschedule := time.Now().Add(time.Hour).Format(time.RFC3339Nano)\n\t\t\t\tschedulePattern = fmt.Sprintf(\"@at %s\", schedule)\n\t\t\t\tm = nats.NewMsg(\"foo.schedule\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-TTL\", \"1s\")\n\t\t\t\tm.Header.Set(\"Nats-Rollup\", \"sub\")\n\t\t\t\tpubAck, err := js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t\t\t// Schedule where the rollup is missing, automatically gets the rollup added.\n\t\t\t\t// Also, will rollup the previous message for this schedule.\n\t\t\t\tschedule = time.Now().Add(time.Second).Format(time.RFC3339Nano)\n\t\t\t\tschedulePattern = fmt.Sprintf(\"@at %s\", schedule)\n\t\t\t\tm = nats.NewMsg(\"foo.schedule\")\n\t\t\t\tm.Data = []byte(\"hello\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", schedulePattern)\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-TTL\", \"1s\")\n\t\t\t\t// Add a bunch of headers that will be stripped on the scheduled message.\n\t\t\t\tm.Header.Set(\"Nats-Expected-Stream\", \"SchedulesEnabled\")\n\t\t\t\tm.Header.Set(\"Nats-Expected-Last-Sequence\", \"1\")\n\t\t\t\tm.Header.Set(\"Nats-Expected-Last-Subject-Sequence\", \"1\")\n\t\t\t\tm.Header.Set(\"Nats-Msg-Id\", \"X\")\n\t\t\t\tm.Header.Set(\"Nats-TTL\", \"60s\")\n\t\t\t\tm.Header.Set(\"Header\", \"Value\")\n\t\t\t\tpubAck, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t\t\t\tsl := c.streamLeader(globalAccountName, \"SchedulesEnabled\")\n\t\t\t\tmset, err := sl.globalAccount().lookupStream(\"SchedulesEnabled\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Only one schedule exists.\n\t\t\t\tstate := mset.state()\n\t\t\t\trequire_Equal(t, state.LastSeq, 2)\n\t\t\t\trequire_Equal(t, state.Msgs, 1)\n\n\t\t\t\t// Waiting for the delayed message to be published.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\tstate = mset.state()\n\t\t\t\t\tif state.LastSeq != 3 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected last seq 3, got %d\", state.LastSeq)\n\t\t\t\t\t} else if state.Msgs != 1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d\", state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\t// Confirm the scheduled message has the correct data.\n\t\t\t\trsm, err := js.GetLastMsg(\"SchedulesEnabled\", \"foo.publish\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, rsm.Sequence, 3)\n\t\t\t\trequire_True(t, bytes.Equal(rsm.Data, []byte(\"hello\")))\n\t\t\t\trequire_Len(t, len(rsm.Header), 4)\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Scheduler\"), \"foo.schedule\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Schedule-Next\"), \"purge\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-TTL\"), \"1s\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Header\"), \"Value\")\n\n\t\t\t\t// Waiting for the delayed message to age out due to its TTL.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\tstate = mset.state()\n\t\t\t\t\tif state.FirstSeq != 4 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected first seq 4, got %d\", state.FirstSeq)\n\t\t\t\t\t} else if state.LastSeq != 3 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected last seq 3, got %d\", state.LastSeq)\n\t\t\t\t\t} else if state.Msgs != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected no messages, got %d\", state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\t// Servers should be synced.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\treturn checkState(t, c, globalAccountName, \"SchedulesEnabled\")\n\t\t\t\t})\n\n\t\t\t\tcfg.AllowMsgSchedules = false\n\t\t\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\t\t\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"message schedules can not be disabled\")))\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterScheduledMessageSubjectSourcing(t *testing.T) {\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, storage), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tcfg := &StreamConfig{\n\t\t\t\t\tName: \"SchedulesEnabled\",\n\t\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: true,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t}\n\t\t\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tm := nats.NewMsg(\"foo.data\")\n\t\t\t\tm.Header.Set(\"Header\", \"Value\")\n\t\t\t\tm.Data = []byte(\"data\")\n\n\t\t\t\tpubAck, err := js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t\t\tm = nats.NewMsg(\"foo.schedule\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", \"@at 1970-01-01T00:00:00Z\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\n\t\t\t\t// Invalid sources include if the subject:\n\t\t\t\t// - matches the schedule/target subject\n\t\t\t\t// - contains wildcard/is not literal\n\t\t\t\tfor _, src := range []string{\"foo.schedule\", \"foo.publish\", \"foo.*\", \"foo.>\"} {\n\t\t\t\t\tm.Header.Set(\"Nats-Schedule-Source\", src)\n\t\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\t\trequire_Error(t, err, NewJSMessageSchedulesSourceInvalidError())\n\t\t\t\t}\n\n\t\t\t\t// Now publish using a correct source subject.\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Source\", \"foo.data\")\n\t\t\t\tpubAck, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t\t\t\tsl := c.streamLeader(globalAccountName, \"SchedulesEnabled\")\n\t\t\t\tmset, err := sl.globalAccount().lookupStream(\"SchedulesEnabled\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tstate := mset.state()\n\t\t\t\trequire_Equal(t, state.LastSeq, 2)\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\n\t\t\t\t// Waiting for the delayed message to be published.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\tstate = mset.state()\n\t\t\t\t\tif state.LastSeq != 3 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected last seq 3, got %d\", state.LastSeq)\n\t\t\t\t\t} else if state.Msgs != 2 {\n\t\t\t\t\t\t// One is the scheduled message, one is the sourced message.\n\t\t\t\t\t\treturn fmt.Errorf(\"expected 2 msgs, got %d\", state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\t// Confirm the scheduled message has the correct data.\n\t\t\t\trsm, err := js.GetLastMsg(\"SchedulesEnabled\", \"foo.publish\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, rsm.Sequence, 3)\n\t\t\t\trequire_True(t, bytes.Equal(rsm.Data, []byte(\"data\")))\n\t\t\t\trequire_Len(t, len(rsm.Header), 3)\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Scheduler\"), \"foo.schedule\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Nats-Schedule-Next\"), \"purge\")\n\t\t\t\trequire_Equal(t, rsm.Header.Get(\"Header\"), \"Value\")\n\n\t\t\t\t// Servers should be synced.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\treturn checkState(t, c, globalAccountName, \"SchedulesEnabled\")\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterScheduledDelayedMessageReversedHeaderOrder(t *testing.T) {\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, storage), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, _ := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tcfg := &StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: true,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t}\n\t\t\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\t\tgacc := sl.globalAccount()\n\t\t\t\t// The Nats-Schedule headers share a common prefix, so make sure if these are ordered differently\n\t\t\t\t// we can still properly schedule a message.\n\t\t\t\thdr := genHeader(nil, \"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\thdr = genHeader(hdr, \"Nats-Schedule\", \"@at 1970-01-01T00:00:00Z\")\n\t\t\t\trequire_NoError(t, sl.sendInternalAccountMsgWithReply(gacc, \"foo.schedule\", _EMPTY_, hdr, nil, false))\n\n\t\t\t\tmset, err := gacc.lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Waiting for the delayed message to be published.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\tstate := mset.state()\n\t\t\t\t\tif state.LastSeq != 2 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected last seq 2, got %d\", state.LastSeq)\n\t\t\t\t\t} else if state.Msgs != 1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d\", state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterScheduledIntervalMessage(t *testing.T) {\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, storage), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tcfg := &StreamConfig{\n\t\t\t\t\tName: \"SchedulesEnabled\",\n\t\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\t\tStorage: storage,\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tAllowMsgSchedules: true,\n\t\t\t\t}\n\t\t\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Schedule with an interval that triggers often.\n\t\t\t\tm := nats.NewMsg(\"foo.schedule\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule\", \"@every 1s\")\n\t\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.publish\")\n\t\t\t\tpubAck, err := js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t\t\tsl := c.streamLeader(globalAccountName, \"SchedulesEnabled\")\n\t\t\t\tmset, err := sl.globalAccount().lookupStream(\"SchedulesEnabled\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Waiting for the repeated message to be published enough times.\n\t\t\t\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\ttotal := mset.Store().SubjectsTotals(\"foo.publish\")[\"foo.publish\"]\n\t\t\t\t\tif total < 3 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected at least 3 messages, got %d\", total)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\t// Servers should be synced.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\treturn checkState(t, c, globalAccountName, \"SchedulesEnabled\")\n\t\t\t\t})\n\n\t\t\t\t// If we remove the schedule, the scheduling should stop.\n\t\t\t\tvar removed bool\n\t\t\t\ttotal := mset.Store().SubjectsTotals(\"foo.publish\")[\"foo.publish\"]\n\t\t\t\tfor i := 1; ; i++ {\n\t\t\t\t\tbefore := total\n\t\t\t\t\ttime.Sleep(1200 * time.Millisecond)\n\t\t\t\t\ttotal = mset.Store().SubjectsTotals(\"foo.publish\")[\"foo.publish\"]\n\t\t\t\t\tif before == total {\n\t\t\t\t\t\t// If the scheduling stopped before we removed, that's a problem.\n\t\t\t\t\t\trequire_True(t, removed)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tif !removed {\n\t\t\t\t\t\tremoved = true\n\t\t\t\t\t\trequire_NoError(t, js.DeleteMsg(\"SchedulesEnabled\", pubAck.Sequence))\n\t\t\t\t\t}\n\t\t\t\t\tif i > 2 {\n\t\t\t\t\t\tt.Fatal(\"Scheduling didn't stop\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Servers should be synced.\n\t\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\treturn checkState(t, c, globalAccountName, \"SchedulesEnabled\")\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterOfflineStreamAndConsumerAfterAssetCreateOrUpdate(t *testing.T) {\n\tclusterName := \"R3S\"\n\tc := createJetStreamClusterExplicit(t, clusterName, 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\n\tsjs := ml.getJetStream()\n\trequire_NotNil(t, sjs)\n\tsjs.mu.Lock()\n\tcc := sjs.cluster\n\tif cc == nil || cc.meta == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatalf(\"Expected cluster to be initialized\")\n\t}\n\n\trestart := func() {\n\t\tt.Helper()\n\t\tfor _, s := range c.servers {\n\t\t\tsjs = s.getJetStream()\n\t\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\t\trequire_NoError(t, err)\n\t\t\tmeta := sjs.getMetaGroup()\n\t\t\tmeta.InstallSnapshot(snap, false)\n\t\t}\n\n\t\tc.stopAll()\n\t\tc.restartAllSamePorts()\n\t\tc.waitOnLeader()\n\t\tml = c.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_NoError(t, nc.ForceReconnect())\n\n\t\tsjs = ml.getJetStream()\n\t\trequire_NotNil(t, sjs)\n\t\tsjs.mu.Lock()\n\t\tcc = sjs.cluster\n\t\tif cc == nil || cc.meta == nil {\n\t\t\tsjs.mu.Unlock()\n\t\t\tt.Fatalf(\"Expected cluster to be initialized\")\n\t\t}\n\t\tsjs.mu.Unlock()\n\t}\n\n\tgetValidMetaSnapshot := func() (wsas []writeableStreamAssignment) {\n\t\tt.Helper()\n\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(snap) > 0)\n\t\tdec, err := s2.Decode(nil, snap)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(dec, &wsas))\n\t\treturn wsas\n\t}\n\n\t// Create a stream that's unsupported.\n\tci := &ClientInfo{\n\t\tAccount: globalAccountName,\n\t\tCluster: clusterName,\n\t}\n\tscfg := &StreamConfig{\n\t\tName: \"DowngradeStreamTest\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tMetadata: map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt - 1)},\n\t}\n\trg, perr := sjs.createGroupForStream(ci, scfg)\n\tif perr != nil {\n\t\tsjs.mu.Unlock()\n\t\trequire_NoError(t, perr)\n\t}\n\tsa := &streamAssignment{\n\t\tConfig: scfg,\n\t\tSync: syncSubjForStream(),\n\t\tGroup: rg,\n\t\tCreated: time.Now().UTC(),\n\t\tClient: ci,\n\t}\n\terr := cc.meta.Propose(encodeAddStreamAssignment(sa))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\tunsupported := func(requiredApiLevel int) string {\n\t\treturn fmt.Sprintf(\"unsupported - required API level: %d, current API level: %d\", requiredApiLevel, JSApiLevel)\n\t}\n\texpectStreamInfo := func(offlineReason, streamName string) {\n\t\tvar msg *nats.Msg\n\t\tcheckFor(t, 3*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, streamName), nil, time.Second)\n\t\t\treturn err\n\t\t})\n\t\tvar si JSApiStreamInfoResponse\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &si))\n\t\trequire_NotNil(t, si.Error)\n\t\trequire_Error(t, si.Error, NewJSStreamOfflineReasonError(errors.New(offlineReason)))\n\n\t\tvar sn JSApiStreamNamesResponse\n\t\tmsg, err = nc.Request(JSApiStreams, nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &sn))\n\t\trequire_Len(t, len(sn.Streams), 1)\n\t\trequire_Equal(t, sn.Streams[0], streamName)\n\n\t\tvar sl JSApiStreamListResponse\n\t\tmsg, err = nc.Request(JSApiStreamList, nil, 2*time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &sl))\n\t\trequire_Len(t, len(sl.Streams), 0)\n\t\trequire_Len(t, len(sl.Missing), 1)\n\t\trequire_Equal(t, sl.Missing[0], streamName)\n\t\trequire_Len(t, len(sl.Offline), 1)\n\t\trequire_Equal(t, sl.Offline[streamName], offlineReason)\n\t}\n\n\t// Stream should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectStreamInfo(unsupported(math.MaxInt-1), \"DowngradeStreamTest\")\n\thealth := ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectStreamInfo(unsupported(math.MaxInt-1), \"DowngradeStreamTest\")\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas := getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa := &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeStreamTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt-1))\n\n\t// Update a stream that's unsupported.\n\tsjs.mu.Lock()\n\tscfg.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\terr = cc.meta.Propose(encodeUpdateStreamAssignment(sa))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\t// Stream should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectStreamInfo(unsupported(math.MaxInt), \"DowngradeStreamTest\")\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectStreamInfo(unsupported(math.MaxInt), \"DowngradeStreamTest\")\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeStreamTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt))\n\n\t// Deleting a stream should always work, even if it is unsupported.\n\trequire_NoError(t, js.DeleteStream(\"DowngradeStreamTest\"))\n\tsnap, _, _, err := sjs.metaSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, snap == nil)\n\n\t// Create a supported stream and consumer.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"DowngradeConsumerTest\", Replicas: 3})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"DowngradeConsumerTest\", &nats.ConsumerConfig{Name: \"consumer\"})\n\trequire_NoError(t, err)\n\n\t// Create a consumer that's unsupported.\n\tsjs.mu.Lock()\n\tccfg := &ConsumerConfig{\n\t\tName: \"DowngradeConsumerTest\",\n\t\tReplicas: 3,\n\t\tMetadata: map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt - 1)},\n\t}\n\trg = cc.createGroupForConsumer(ccfg, sa)\n\tca := &consumerAssignment{\n\t\tConfig: ccfg,\n\t\tGroup: rg,\n\t\tStream: \"DowngradeConsumerTest\",\n\t\tName: \"DowngradeConsumerTest\",\n\t\tCreated: time.Now().UTC(),\n\t\tClient: ci,\n\t}\n\terr = cc.meta.Propose(encodeAddConsumerAssignment(ca))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\texpectConsumerInfo := func(offlineReason string) {\n\t\tvar msg *nats.Msg\n\t\tcheckFor(t, 3*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumerInfoT, \"DowngradeConsumerTest\", \"DowngradeConsumerTest\"), nil, time.Second)\n\t\t\treturn err\n\t\t})\n\t\tvar ci JSApiConsumerInfoResponse\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &ci))\n\t\trequire_NotNil(t, ci.Error)\n\t\trequire_Error(t, ci.Error, NewJSConsumerOfflineReasonError(errors.New(offlineReason)))\n\n\t\tvar cn JSApiConsumerNamesResponse\n\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumersT, \"DowngradeConsumerTest\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &cn))\n\t\trequire_Len(t, len(cn.Consumers), 2)\n\t\trequire_Equal(t, cn.Consumers[0], \"DowngradeConsumerTest\")\n\t\tfor _, name := range cn.Consumers {\n\t\t\trequire_True(t, name == \"consumer\" || name == \"DowngradeConsumerTest\")\n\t\t}\n\n\t\tvar cl JSApiConsumerListResponse\n\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumerListT, \"DowngradeConsumerTest\"), nil, 5*time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &cl))\n\t\trequire_Len(t, len(cl.Consumers), 0)\n\t\trequire_Len(t, len(cl.Missing), 2)\n\t\tfor _, name := range cl.Missing {\n\t\t\trequire_True(t, name == \"consumer\" || name == \"DowngradeConsumerTest\")\n\t\t}\n\t\trequire_Len(t, len(cl.Offline), 1)\n\t\trequire_Equal(t, cl.Offline[\"DowngradeConsumerTest\"], offlineReason)\n\n\t\t// Stream should also be reported as offline.\n\t\t// Specifically, as \"stopped\" because it's still supported, but can't run due to the unsupported consumer.\n\t\texpectStreamInfo(\"stopped - unsupported consumer \\\"DowngradeConsumerTest\\\"\", \"DowngradeConsumerTest\")\n\t}\n\n\t// Consumer should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectConsumerInfo(unsupported(math.MaxInt - 1))\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectConsumerInfo(unsupported(math.MaxInt - 1))\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeConsumerTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], \"0\")\n\trequire_Len(t, len(wsas[0].Consumers), 2)\n\tfor _, wca := range wsas[0].Consumers {\n\t\tnca := &consumerAssignment{ConfigJSON: wca.ConfigJSON}\n\t\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\t\tif nca.Config.Name == \"DowngradeConsumerTest\" {\n\t\t\trequire_Equal(t, nca.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt-1))\n\t\t} else {\n\t\t\trequire_Equal(t, nca.Config.Name, \"consumer\")\n\t\t}\n\t}\n\n\t// Update a consumer (with compressed data) that's unsupported.\n\tccfg.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\tsjs.mu.Lock()\n\terr = cc.meta.Propose(encodeAddConsumerAssignmentCompressed(ca))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\t// Consumer should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectConsumerInfo(unsupported(math.MaxInt))\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectConsumerInfo(unsupported(math.MaxInt))\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeConsumerTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], \"0\")\n\trequire_Len(t, len(wsas[0].Consumers), 2)\n\tfor _, wca := range wsas[0].Consumers {\n\t\tnca := &consumerAssignment{ConfigJSON: wca.ConfigJSON}\n\t\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\t\tif nca.Config.Name == \"DowngradeConsumerTest\" {\n\t\t\trequire_Equal(t, nca.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt))\n\t\t} else {\n\t\t\trequire_Equal(t, nca.Config.Name, \"consumer\")\n\t\t}\n\t}\n\n\t// Deleting a consumer should always work, even if it is unsupported.\n\trequire_NoError(t, js.DeleteConsumer(\"DowngradeConsumerTest\", \"DowngradeConsumerTest\"))\n\tc.waitOnAllCurrent()\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeConsumerTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], \"0\")\n\trequire_Len(t, len(wsas[0].Consumers), 1)\n\tnca := &consumerAssignment{ConfigJSON: wsas[0].Consumers[0].ConfigJSON}\n\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\trequire_Equal(t, nca.Config.Name, \"consumer\")\n}\n\nfunc TestJetStreamClusterOfflineStreamAndConsumerAfterDowngrade(t *testing.T) {\n\tclusterName := \"R3S\"\n\tc := createJetStreamClusterExplicit(t, clusterName, 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\n\tsjs := ml.getJetStream()\n\trequire_NotNil(t, sjs)\n\tsjs.mu.Lock()\n\tcc := sjs.cluster\n\tif cc == nil || cc.meta == nil {\n\t\tsjs.mu.Unlock()\n\t\tt.Fatalf(\"Expected cluster to be initialized\")\n\t}\n\n\trestart := func() {\n\t\tt.Helper()\n\t\tfor _, s := range c.servers {\n\t\t\tsjs = s.getJetStream()\n\t\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\t\trequire_NoError(t, err)\n\t\t\tmeta := sjs.getMetaGroup()\n\t\t\tmeta.InstallSnapshot(snap, false)\n\t\t}\n\n\t\tc.stopAll()\n\t\tc.restartAllSamePorts()\n\t\tc.waitOnLeader()\n\t\tml = c.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_NoError(t, nc.ForceReconnect())\n\n\t\tsjs = ml.getJetStream()\n\t\trequire_NotNil(t, sjs)\n\t\tsjs.mu.Lock()\n\t\tcc = sjs.cluster\n\t\tif cc == nil || cc.meta == nil {\n\t\t\tsjs.mu.Unlock()\n\t\t\tt.Fatalf(\"Expected cluster to be initialized\")\n\t\t}\n\t\tsjs.mu.Unlock()\n\t}\n\n\tgetValidMetaSnapshot := func() (wsas []writeableStreamAssignment) {\n\t\tt.Helper()\n\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(snap) > 0)\n\t\tdec, err := s2.Decode(nil, snap)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(dec, &wsas))\n\t\treturn wsas\n\t}\n\n\t// Create a stream that's unsupported.\n\tci := &ClientInfo{\n\t\tAccount: globalAccountName,\n\t\tCluster: clusterName,\n\t}\n\tscfg := &StreamConfig{\n\t\tName: \"DowngradeStreamTest\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t}\n\trg, perr := sjs.createGroupForStream(ci, scfg)\n\tif perr != nil {\n\t\tsjs.mu.Unlock()\n\t\trequire_NoError(t, perr)\n\t}\n\tsa := &streamAssignment{\n\t\tConfig: scfg,\n\t\tSync: syncSubjForStream(),\n\t\tGroup: rg,\n\t\tCreated: time.Now().UTC(),\n\t\tClient: ci,\n\t}\n\terr := cc.meta.Propose(encodeAddStreamAssignment(sa))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"DowngradeStreamTest\")\n\n\texpectStreamInfo := func(offline bool) {\n\t\tif !offline {\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"DowngradeStreamTest\")\n\t\t}\n\t\tvar msg *nats.Msg\n\t\tcheckFor(t, 3*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"DowngradeStreamTest\"), nil, time.Second)\n\t\t\treturn err\n\t\t})\n\t\tvar si JSApiStreamInfoResponse\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &si))\n\t\tif !offline {\n\t\t\trequire_True(t, si.Error == nil)\n\t\t} else {\n\t\t\trequire_NotNil(t, si.Error)\n\t\t\trequire_Contains(t, si.Error.Error(), \"stream is offline\", \"unsupported\", \"required API level\")\n\t\t}\n\t}\n\n\t// Stream is still supported, so it should be available and healthz should report healthy.\n\texpectStreamInfo(false)\n\thealth := ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectStreamInfo(false)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas := getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa := &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeStreamTest\")\n\n\t// Update a stream to be unsupported.\n\tsjs.mu.Lock()\n\tscfg.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\terr = cc.meta.Propose(encodeUpdateStreamAssignment(sa))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\t// Stream should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectStreamInfo(true)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectStreamInfo(true)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeStreamTest\")\n\n\t// Deleting a stream should always work, even if it is unsupported.\n\trequire_NoError(t, js.DeleteStream(\"DowngradeStreamTest\"))\n\tsnap, _, _, err := sjs.metaSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, snap == nil)\n\n\t// Create a supported stream and consumer.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"DowngradeConsumerTest\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsjs.mu.Lock()\n\tccfg := &ConsumerConfig{\n\t\tName: \"DowngradeConsumerTest\",\n\t\tReplicas: 3,\n\t\tMaxWaiting: JSWaitQueueDefaultMax,\n\t}\n\trg = cc.createGroupForConsumer(ccfg, sa)\n\tca := &consumerAssignment{\n\t\tConfig: ccfg,\n\t\tGroup: rg,\n\t\tStream: \"DowngradeConsumerTest\",\n\t\tName: \"DowngradeConsumerTest\",\n\t\tCreated: time.Now().UTC(),\n\t\tClient: ci,\n\t}\n\terr = cc.meta.Propose(encodeAddConsumerAssignment(ca))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"DowngradeConsumerTest\", \"DowngradeConsumerTest\")\n\n\texpectConsumerInfo := func(offline bool) {\n\t\tif !offline {\n\t\t\tc.waitOnConsumerLeader(globalAccountName, \"DowngradeConsumerTest\", \"DowngradeConsumerTest\")\n\t\t}\n\t\tvar msg *nats.Msg\n\t\tcheckFor(t, 3*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumerInfoT, \"DowngradeConsumerTest\", \"DowngradeConsumerTest\"), nil, 2*time.Second)\n\t\t\treturn err\n\t\t})\n\t\tvar ci JSApiConsumerInfoResponse\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &ci))\n\t\tif !offline {\n\t\t\trequire_True(t, ci.Error == nil)\n\t\t} else {\n\t\t\trequire_NotNil(t, ci.Error)\n\t\t\trequire_Contains(t, ci.Error.Error(), \"consumer is offline\", \"unsupported\", \"required API level\")\n\t\t}\n\t}\n\n\t// Consumer is still supported, so it should be available and healthz should report healthy.\n\texpectConsumerInfo(false)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectConsumerInfo(false)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeConsumerTest\")\n\trequire_Len(t, len(wsas[0].Consumers), 1)\n\n\t// Update a consumer to be unsupported.\n\tccfg.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\tsjs.mu.Lock()\n\terr = cc.meta.Propose(encodeAddConsumerAssignment(ca))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\t// Consumer should be reported as offline, but healthz should report healthy to not block downgrades.\n\texpectConsumerInfo(true)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\trestart()\n\texpectConsumerInfo(true)\n\thealth = ml.healthz(&HealthzOptions{})\n\trequire_Equal(t, health.StatusCode, 200)\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeConsumerTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], \"0\")\n\trequire_Len(t, len(wsas[0].Consumers), 1)\n\tnca := &consumerAssignment{ConfigJSON: wsas[0].Consumers[0].ConfigJSON}\n\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\trequire_Equal(t, nca.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt))\n}\n\nfunc TestJetStreamClusterOfflineStreamAndConsumerUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"DowngradeTest\", Replicas: 3})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"DowngradeTest\", &nats.ConsumerConfig{Durable: \"D\", Replicas: 3})\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\tsjs := ml.getJetStream()\n\trequire_NotNil(t, sjs)\n\n\tvar sa *streamAssignment\n\tvar ca *consumerAssignment\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tsjs.mu.Lock()\n\t\tdefer sjs.mu.Unlock()\n\t\tsa = sjs.streamAssignment(globalAccountName, \"DowngradeTest\")\n\t\tif sa == nil {\n\t\t\treturn errors.New(\"stream assignment missing\")\n\t\t}\n\t\tca = sjs.consumerAssignment(globalAccountName, \"DowngradeTest\", \"D\")\n\t\tif ca == nil {\n\t\t\treturn errors.New(\"consumer assignment missing\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tgetValidMetaSnapshot := func() (wsas []writeableStreamAssignment) {\n\t\tt.Helper()\n\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(snap) > 0)\n\t\tdec, err := s2.Decode(nil, snap)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(dec, &wsas))\n\t\treturn wsas\n\t}\n\n\twsas := getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa := &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], \"0\")\n\trequire_Len(t, len(wsas[0].Consumers), 1)\n\tnca := &consumerAssignment{ConfigJSON: wsas[0].Consumers[0].ConfigJSON}\n\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\trequire_Equal(t, wsas[0].Consumers[0].Name, \"D\")\n\trequire_Equal(t, nca.Config.Metadata[\"_nats.req.level\"], \"0\")\n\n\t// Update a consumer to be unsupported.\n\tsjs.mu.Lock()\n\tca.Config.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\terr = sjs.cluster.meta.Propose(encodeAddConsumerAssignment(ca))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\n\t// Update the stream to be unsupported.\n\tsjs.mu.Lock()\n\tsa.Config.Metadata = map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)}\n\terr = sjs.cluster.meta.Propose(encodeUpdateStreamAssignment(sa))\n\tsjs.mu.Unlock()\n\trequire_NoError(t, err)\n\tc.waitOnAllCurrent()\n\n\twsas = getValidMetaSnapshot()\n\trequire_Len(t, len(wsas), 1)\n\tnsa = &streamAssignment{ConfigJSON: wsas[0].ConfigJSON}\n\trequire_NoError(t, decodeStreamAssignmentConfig(ml, nsa))\n\trequire_Equal(t, nsa.Config.Name, \"DowngradeTest\")\n\trequire_Equal(t, nsa.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt))\n\trequire_Len(t, len(wsas[0].Consumers), 1)\n\tnca = &consumerAssignment{ConfigJSON: wsas[0].Consumers[0].ConfigJSON}\n\trequire_NoError(t, decodeConsumerAssignmentConfig(nca))\n\trequire_Equal(t, wsas[0].Consumers[0].Name, \"D\")\n\trequire_Equal(t, nca.Config.Metadata[\"_nats.req.level\"], strconv.Itoa(math.MaxInt))\n}\n\nfunc TestJetStreamClusterOfflineStreamAndConsumerStrictDecoding(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tunsupportedJson := []byte(\"{\\\"unknown\\\": true}\")\n\tunsupportedStreamJson := []byte(fmt.Sprintf(\"{\\\"stream\\\":%s}\", unsupportedJson))\n\tunsupportedConsumerJson := []byte(fmt.Sprintf(\"{\\\"consumer\\\":%s}\", unsupportedJson))\n\n\tsa, err := decodeStreamAssignment(s, unsupportedStreamJson)\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(sa.ConfigJSON, unsupportedJson))\n\trequire_True(t, sa.unsupported != nil)\n\n\tca, err := decodeConsumerAssignment(unsupportedConsumerJson)\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(ca.ConfigJSON, unsupportedJson))\n\trequire_True(t, ca.unsupported != nil)\n\n\tvar bb bytes.Buffer\n\ts2e := s2.NewWriter(&bb)\n\t_, err = s2e.Write(unsupportedConsumerJson)\n\trequire_NoError(t, err)\n\trequire_NoError(t, s2e.Close())\n\tca, err = decodeConsumerAssignmentCompressed(bb.Bytes())\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(ca.ConfigJSON, unsupportedJson))\n\trequire_True(t, ca.unsupported != nil)\n}\n\nfunc TestJetStreamClusterStreamMonitorShutdownWithoutRaftNode(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif !s.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\t\treturn fmt.Errorf(\"stream not assigned on %s\", s.Name())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tvar nodes []RaftNode\n\tfor _, s := range c.servers {\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\t// Manually nil-out the node. This shouldn't happen normally,\n\t\t// but tests we can shut down purely with the monitor goroutine quit channel.\n\t\tmset.mu.Lock()\n\t\tn := mset.node\n\t\tmset.node = nil\n\t\tmset.mu.Unlock()\n\t\trequire_NotNil(t, n)\n\t\tnodes = append(nodes, n)\n\t}\n\tfor _, n := range nodes {\n\t\trequire_NotEqual(t, n.State(), Closed)\n\t}\n\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, n := range nodes {\n\t\t\tif state := n.State(); state != Closed {\n\t\t\t\treturn fmt.Errorf(\"node not closed on %s: %s\", n.ID(), state.String())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterConsumerMonitorShutdownWithoutRaftNode(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DURABLE\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\t_, cc := s.getJetStreamCluster()\n\t\t\tif !cc.isConsumerAssigned(s.globalAccount(), \"TEST\", \"DURABLE\") {\n\t\t\t\treturn fmt.Errorf(\"consumer not assigned on %s\", s.Name())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tvar nodes []RaftNode\n\tfor _, s := range c.servers {\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"DURABLE\")\n\t\trequire_NotNil(t, o)\n\t\t// Manually nil-out the node. This shouldn't happen normally,\n\t\t// but tests we can shut down purely with the monitor goroutine quit channel.\n\t\to.mu.Lock()\n\t\tn := o.node\n\t\to.node = nil\n\t\to.mu.Unlock()\n\t\trequire_NotNil(t, n)\n\t\tnodes = append(nodes, n)\n\t}\n\tfor _, n := range nodes {\n\t\trequire_NotEqual(t, n.State(), Closed)\n\t}\n\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, n := range nodes {\n\t\t\tif state := n.State(); state != Closed {\n\t\t\t\treturn fmt.Errorf(\"node not closed on %s: %s\", n.ID(), state.String())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterUnsetEmptyPlacement(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tPlacement: &nats.Placement{},\n\t}\n\tsi, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.Placement == nil)\n\n\tsi, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.Placement == nil)\n\n\t// Set a placement level\n\tcfg.Placement = &nats.Placement{Cluster: \"R3S\"}\n\tsi, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.Placement != nil)\n\trequire_Equal(t, si.Config.Placement.Cluster, \"R3S\")\n\n\t// And ensure it can be reset.\n\tcfg.Placement = &nats.Placement{}\n\tsi, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.Placement == nil)\n}\n\nfunc TestJetStreamClusterPersistModeAsync(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc := clientConnectToServer(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tPersistMode: AsyncPersistMode,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is not supported on replicated streams\")))\n}\n\nfunc TestJetStreamClusterDeleteMsgEOF(t *testing.T) {\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: replicas,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\trequire_Error(t, js.DeleteMsg(\"TEST\", 0), NewJSNoMessageFoundError())\n\t\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 1))\n\t\t\trequire_Error(t, js.DeleteMsg(\"TEST\", 1), NewJSNoMessageFoundError())\n\t\t\trequire_Error(t, js.DeleteMsg(\"TEST\", 2), NewJSStreamMsgDeleteFailedError(ErrStoreEOF))\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterCatchupSkipMsgDesync(t *testing.T) {\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storage,\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tsa := mset.streamAssignment()\n\n\t\t\t// Make sure this server can't become the leader.\n\t\t\tn := mset.raftNode().(*raft)\n\t\t\tn.SetObserver(true)\n\n\t\t\tsysNc, err := nats.Connect(rs.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sysNc.Close()\n\n\t\t\tsjs := rs.getJetStream()\n\t\t\tsjs.mu.RLock()\n\t\t\tsyncSubj := sa.Sync\n\t\t\tsjs.mu.RUnlock()\n\n\t\t\t// Respond to the catchup with an out-of-order SkipMsg.\n\t\t\tvar eof bool\n\t\t\tsub, err := sysNc.Subscribe(syncSubj, func(msg *nats.Msg) {\n\t\t\t\tif !eof {\n\t\t\t\t\tmsg.Respond(encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, 10, 0, false))\n\t\t\t\t\teof = true\n\t\t\t\t}\n\t\t\t\tmsg.Respond(nil)\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Drain()\n\t\t\trequire_NoError(t, sysNc.Flush()) // Must flush, otherwise our subscription could be too late.\n\n\t\t\terr = mset.processSnapshot(&StreamReplicatedState{FirstSeq: 1, LastSeq: 1}, 1)\n\t\t\trequire_Error(t, err, errCatchupTooManyRetries)\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterJszRaftLeaderReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DURABLE\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\t_, _, jsa := s.globalAccount().getJetStreamFromAccount()\n\t\t\tif !jsa.streamAssigned(\"TEST\") {\n\t\t\t\treturn fmt.Errorf(\"stream not assigned on %s\", s.Name())\n\t\t\t}\n\t\t\tif !jsa.consumerAssigned(\"TEST\", \"DURABLE\") {\n\t\t\t\treturn fmt.Errorf(\"consumer not assigned on %s\", s.Name())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"DURABLE\")\n\n\tfor _, s := range c.servers {\n\t\tjsi, err := s.Jsz(&JSzOptions{RaftGroups: true})\n\t\trequire_NoError(t, err)\n\t\tif s == sl {\n\t\t\trequire_Equal(t, jsi.StreamsLeader, 1)\n\t\t} else {\n\t\t\trequire_Equal(t, jsi.StreamsLeader, 0)\n\t\t}\n\t\tif s == cl {\n\t\t\trequire_Equal(t, jsi.ConsumersLeader, 1)\n\t\t} else {\n\t\t\trequire_Equal(t, jsi.ConsumersLeader, 0)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterNoInterestDesyncOnConsumerCreate(t *testing.T) {\n\ttest := func(t *testing.T, twoConsumers bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\tReplicas: 3,\n\t\t\tRetention: nats.InterestPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Pick a random server that will not know about the new consumer being created.\n\t\t// If servers determine \"no interest\" individually, these servers will desync.\n\t\trs := c.randomNonLeader()\n\t\tsjs := rs.getJetStream()\n\t\tmeta := sjs.getMetaGroup()\n\t\trequire_NoError(t, meta.PauseApply())\n\n\t\tsub, err := js.PullSubscribe(\"foo\", \"DURABLE\", nats.BindStream(\"TEST\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tcheckConsumersAssigned := func(expected int) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tvar count int\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tcount += mset.numConsumers()\n\t\t\t\t}\n\t\t\t\tif count != expected {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d, got %d\", expected, count)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\t// Confirm only two servers know about the consumer.\n\t\tcheckConsumersAssigned(2)\n\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"DURABLE\")\n\n\t\t// Publish a single message. All servers will receive this, but only two will store it.\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckLastSeq := func(lseq uint64) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif seq := mset.lastSeq(); seq != lseq {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected %d, got %d\", lseq, seq)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\tcheckLastSeq(1)\n\n\t\t// Resume the meta layer such that the consumer gets created on the remaining server.\n\t\tmeta.ResumeApply()\n\t\tcheckConsumersAssigned(3)\n\n\t\tif twoConsumers {\n\t\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DURABLE2\", FilterSubject: \"bar\"})\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckConsumersAssigned(6)\n\t\t}\n\n\t\t// All servers will now store another published message.\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckLastSeq(2)\n\n\t\t// Make sure the consumer leader is on the same server that didn't store the first message.\n\t\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"DURABLE\")\n\t\tif cl != rs {\n\t\t\tmset, err := cl.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\to := mset.lookupConsumer(\"DURABLE\")\n\t\t\trequire_NotNil(t, o)\n\t\t\tn := o.raftNode()\n\t\t\trequire_NoError(t, n.StepDown(rs.NodeName()))\n\t\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"DURABLE\")\n\t\t\tcl = c.consumerLeader(globalAccountName, \"TEST\", \"DURABLE\")\n\t\t\trequire_Equal(t, cl, rs)\n\t\t}\n\n\t\t// Since the consumer leader is the same as the server that didn't store the first message,\n\t\t// it can only receive and ack the second message.\n\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\tmetadata, err := msgs[0].Metadata()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, metadata.Sequence.Stream, 2)\n\t\trequire_Equal(t, metadata.NumPending, 0)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\tif twoConsumers {\n\t\t\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"DURABLE\"))\n\t\t}\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t// The servers will eventually be synced up again, but this relies on the interest state being checked.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == rs {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset.checkInterestState()\n\t\t\t}\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\tt.Run(\"OneConsumer\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"TwoConsumers\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterRaftCatchupSignalsMetaRecovery(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\trs := c.randomNonLeader()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif !rs.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\treturn errors.New(\"not assigned\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsjs := rs.getJetStream()\n\tsjs.mu.Lock()\n\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\tencodedDelete := encodeDeleteStreamAssignment(sa)\n\tsjs.mu.Unlock()\n\tmeta := sjs.getMetaGroup().(*raft)\n\tapply := meta.ApplyQ()\n\n\t// Should not panic if we receive a nil entry \"randomly\".\n\t_, err = apply.push(nil)\n\trequire_NoError(t, err)\n\n\t// Should put us in upper-layer recovery mode.\n\t// For this test using two large values so we remain in catchup.\n\tmeta.Lock()\n\tmeta.createCatchup(&appendEntry{pterm: 100, pindex: 100})\n\tmeta.sendCatchupSignal()\n\tmeta.Unlock()\n\n\t// Deleting a stream should be staged, not immediately performed.\n\t_, err = apply.push(newCommittedEntry(1, []*Entry{{EntryNormal, encodedDelete}}))\n\trequire_NoError(t, err)\n\ttime.Sleep(200 * time.Millisecond)\n\trequire_True(t, rs.JetStreamIsStreamAssigned(globalAccountName, \"TEST\"))\n\n\t// Canceling the catchup, because it's completed, should result in the staged changes to be applied.\n\tmeta.Lock()\n\tmeta.cancelCatchup()\n\tmeta.Unlock()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif rs.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\treturn errors.New(\"still assigned\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterRaftCatchupSignalsMetaRecoveryRecreateStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\trs := c.randomNonLeader()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif !rs.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\treturn errors.New(\"not assigned\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsjs := rs.getJetStream()\n\tsjs.mu.Lock()\n\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\tencodedDelete := encodeDeleteStreamAssignment(sa)\n\tsa.Config.Storage = MemoryStorage\n\tencodedAdd := encodeAddStreamAssignment(sa)\n\tencodedUpdate := encodeUpdateStreamAssignment(sa)\n\tsjs.mu.Unlock()\n\tmeta := sjs.getMetaGroup().(*raft)\n\tapply := meta.ApplyQ()\n\n\t// Should not panic if we receive a nil entry \"randomly\".\n\t_, err = apply.push(nil)\n\trequire_NoError(t, err)\n\n\t// Should put us in upper-layer recovery mode.\n\t// For this test using two large values so we remain in catchup.\n\tmeta.Lock()\n\tmeta.createCatchup(&appendEntry{pterm: 100, pindex: 100})\n\tmeta.Unlock()\n\n\t// Deleting a stream should be staged, not immediately performed.\n\t_, err = apply.push(newCommittedEntry(1, []*Entry{\n\t\t{EntryNormal, encodedDelete},\n\t\t{EntryNormal, encodedAdd},\n\t\t{EntryNormal, encodedUpdate},\n\t}))\n\trequire_NoError(t, err)\n\n\t// Canceling the catchup, because it's completed, should result in the staged changes to be applied.\n\tmeta.Lock()\n\tmeta.cancelCatchup()\n\tmeta.Unlock()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcfg := mset.config()\n\t\tif cfg.Storage != MemoryStorage {\n\t\t\treturn errors.New(\"still the old stream config\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterRaftCatchupSignalsMetaRecoveryRecreateConsumer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.ConsumerReplicas(3), nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\trs := c.randomNonLeader()\n\tsjs := rs.getJetStream()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\tif o == nil {\n\t\t\treturn errors.New(\"not found\")\n\t\t}\n\t\tstate, err := o.store.State()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif state.Delivered.Stream != 1 || state.AckFloor.Stream != 1 {\n\t\t\treturn errors.New(\"incorrect delivered/ack state\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsjs.mu.Lock()\n\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\tencodedDelete := encodeDeleteConsumerAssignment(ca)\n\tcca := ca.copyGroup()\n\tcca.Group.Peers = []string{rs.NodeName()}\n\tcca.Group.Name = groupNameForConsumer(cca.Group.Peers, FileStorage)\n\tencodedAdd := encodeAddConsumerAssignment(cca)\n\tsjs.mu.Unlock()\n\tmeta := sjs.getMetaGroup().(*raft)\n\tapply := meta.ApplyQ()\n\n\t// Should not panic if we receive a nil entry \"randomly\".\n\t_, err = apply.push(nil)\n\trequire_NoError(t, err)\n\n\t// Should put us in upper-layer recovery mode.\n\t// For this test using two large values so we remain in catchup.\n\tmeta.Lock()\n\tmeta.createCatchup(&appendEntry{pterm: 100, pindex: 100})\n\tmeta.Unlock()\n\n\t// Deleting a consumer should be staged, not immediately performed.\n\t_, err = apply.push(newCommittedEntry(1, []*Entry{\n\t\t{EntryNormal, encodedDelete},\n\t\t{EntryNormal, encodedAdd},\n\t}))\n\trequire_NoError(t, err)\n\n\t// Canceling the catchup, because it's completed, should result in the staged changes to be applied.\n\tmeta.Lock()\n\tmeta.cancelCatchup()\n\tmeta.Unlock()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\tif o == nil {\n\t\t\treturn errors.New(\"not found\")\n\t\t}\n\t\tstate, err := o.store.State()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif state.Delivered.Stream != 0 || state.AckFloor.Stream != 0 {\n\t\t\treturn errors.New(\"old delivered/ack state\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMetaRecoveryRecreateStream(t *testing.T) {\n\ttest := func(t *testing.T, newStream bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t// Create a stream containing 5 messages.\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tfor range 5 {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Select a server that's neither meta nor stream leader.\n\t\tvar rs *Server\n\t\tfor _, s := range c.servers {\n\t\t\tif s.JetStreamIsLeader() || s.JetStreamIsStreamLeader(globalAccountName, \"TEST\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trs = s\n\t\t}\n\n\t\t// When testing old stream state is cleaned up, pause the meta layer now.\n\t\t// Otherwise, we'll pause after creating the new stream.\n\t\tmeta := rs.getJetStream().getMetaGroup()\n\t\tif !newStream {\n\t\t\trequire_NoError(t, meta.PauseApply())\n\t\t}\n\n\t\t// Delete the stream.\n\t\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t\tif newStream {\n\t\t\t// Need to wait for the stream to be deleted on our selected server.\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tif rs.JetStreamIsStreamAssigned(globalAccountName, \"TEST\") {\n\t\t\t\t\treturn errors.New(\"still assigned\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\t// Recreate the stream with a single message.\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\tif newStream {\n\t\t\t// Wait for the state to be the new state, then pause the meta layer.\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\t\t\trequire_NoError(t, meta.PauseApply())\n\n\t\t\t// Also create a snapshot so that the stream can't \"cheat\" by replaying Raft entries.\n\t\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tstreamNode := mset.raftNode()\n\t\t\trequire_NoError(t, streamNode.InstallSnapshot(mset.stateSnapshot(), false))\n\t\t}\n\n\t\t// Although the meta node is paused, we add one more meta entry to move the commit up one more.\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"RANDOM\", Replicas: 3})\n\t\trequire_NoError(t, err)\n\n\t\t// We'll now restart this server.\n\t\tmeta.Stop()\n\t\tmeta.WaitForStop()\n\t\trs.Shutdown()\n\n\t\tif newStream {\n\t\t\t// If we're testing new state, make sure we can't recover from our peers.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == rs {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tn := mset.raftNode()\n\t\t\t\tn.SetObserver(true)\n\t\t\t\tn.StepDown()\n\t\t\t}\n\t\t}\n\t\trs = c.restartServer(rs)\n\t\tc.waitOnServerCurrent(rs)\n\n\t\t// We should arrive at the new state.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs == 1 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"incorrect amount of messages: %d\", state.Msgs)\n\t\t})\n\t}\n\n\tt.Run(\"OldStream\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"NewStream\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterMetaRecoveryRecreateConsumer(t *testing.T) {\n\ttest := func(t *testing.T, newConsumer bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t// Create a stream containing 5 messages.\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tfor range 5 {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Create a consumer and ack all 5 messages.\n\t\tcfg := &nats.ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tReplicas: 3,\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t}\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.Bind(\"TEST\", \"CONSUMER\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgs, err := sub.Fetch(5)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 5)\n\t\tfor _, msg := range msgs {\n\t\t\trequire_NoError(t, msg.AckSync())\n\t\t}\n\n\t\t// Select a server that's neither meta nor consumer leader.\n\t\tvar rs *Server\n\t\tfor _, s := range c.servers {\n\t\t\tif s.JetStreamIsLeader() || s.JetStreamIsConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trs = s\n\t\t}\n\n\t\t// When testing old consumer state is cleaned up, pause the meta layer now.\n\t\t// Otherwise, we'll pause after creating the new consumer.\n\t\tmeta := rs.getJetStream().getMetaGroup()\n\t\tif !newConsumer {\n\t\t\trequire_NoError(t, meta.PauseApply())\n\t\t}\n\n\t\t// Delete the consumer.\n\t\tsub.Drain()\n\t\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER\"))\n\n\t\tif newConsumer {\n\t\t\t// Need to wait for the consumer to be deleted on our selected server.\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tif rs.getJetStream().consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") != nil {\n\t\t\t\t\treturn errors.New(\"still assigned\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\t// Recreate the consumer and ack a single message.\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsub, err = js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.Bind(\"TEST\", \"CONSUMER\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgs, err = sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\tif newConsumer {\n\t\t\t// Wait for the state to be the new state, then pause the meta layer.\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\t\tif o == nil {\n\t\t\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t\t\t}\n\t\t\t\t\tstate, err := o.store.State()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif state.Delivered.Stream != 1 || state.AckFloor.Stream != 1 {\n\t\t\t\t\t\treturn errors.New(\"incorrect delivered/ack state\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\trequire_NoError(t, meta.PauseApply())\n\t\t}\n\n\t\t// Although the meta node is paused, we add one more meta entry to move the commit up one more.\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"RANDOM\", Replicas: 3})\n\t\trequire_NoError(t, err)\n\n\t\t// We'll now restart this server.\n\t\tsub.Drain()\n\t\tmeta.Stop()\n\t\tmeta.WaitForStop()\n\t\trs.Shutdown()\n\n\t\tif newConsumer {\n\t\t\t// If we're testing new state, make sure we can't recover from our peers.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == rs {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\trequire_NotNil(t, o)\n\t\t\t\tn := o.raftNode()\n\t\t\t\tn.SetObserver(true)\n\t\t\t\tn.StepDown()\n\t\t\t}\n\t\t}\n\t\trs = c.restartServer(rs)\n\t\tc.waitOnServerCurrent(rs)\n\n\t\t// We should arrive at the new state.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\tif o == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t\tstate, err := o.store.State()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif state.Delivered.Stream != 1 || state.AckFloor.Stream != 1 {\n\t\t\t\treturn errors.New(\"incorrect delivered/ack state\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tt.Run(\"OldConsumer\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"NewConsumer\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterMetaPeerRemoveResponseAfterQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, err := nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tvar (\n\t\ts1 *Server\n\t\ts2 *Server\n\t\ts3 *Server\n\t)\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t} else if s1 == nil {\n\t\t\ts1 = s\n\t\t} else if s2 == nil {\n\t\t\ts2 = s\n\t\t} else {\n\t\t\ts3 = s\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Shutdown a majority.\n\tremove1 := s3.Name()\n\tremove2 := s2.Name()\n\ts1.Shutdown()\n\ts2.Shutdown()\n\ts3.Shutdown()\n\n\treply := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tauditSub, err := nc.SubscribeSync(JSAuditAdvisory)\n\trequire_NoError(t, err)\n\tdefer auditSub.Drain()\n\n\t// Since a majority is down, we'll expect to time out since there's no quorum.\n\treq := &JSApiMetaServerRemoveRequest{Server: remove1}\n\tjsreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.PublishRequest(JSApiRemoveServer, reply, jsreq))\n\t_, err = sub.NextMsg(time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Retrying the request should fail, as the leader has already removed it as its peer.\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiMetaServerRemoveResponse\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerMemberChangeInflightError())\n\n\t// Audit should reflect the same.\n\trmsg, err = auditSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvar auditResp JSAPIAudit\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &auditResp))\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal([]byte(auditResp.Response), &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerMemberChangeInflightError())\n\n\t// Don't allow concurrent meta membership changes.\n\treq = &JSApiMetaServerRemoveRequest{Server: remove2}\n\tjsreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err = nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\trequire_NoError(t, err)\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerMemberChangeInflightError())\n\n\t// Audit should reflect the same.\n\trmsg, err = auditSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tauditResp = JSAPIAudit{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &auditResp))\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal([]byte(auditResp.Response), &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerMemberChangeInflightError())\n\n\t// Bring back one server so that the peer-remove can get quorum now.\n\t// The response should come in shortly after.\n\tc.restartServer(s1)\n\n\tc.waitOnLeader()\n\trequire_Equal(t, c.leader().Name(), ml.Name())\n\n\trmsg, err = sub.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &resp))\n\tif resp.Error != nil {\n\t\trequire_NoError(t, resp.Error)\n\t}\n\trequire_True(t, resp.Success)\n\n\t// Audit should reflect the same.\n\trmsg, err = auditSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tauditResp = JSAPIAudit{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &auditResp))\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal([]byte(auditResp.Response), &resp))\n\tif resp.Error != nil {\n\t\trequire_NoError(t, resp.Error)\n\t}\n\trequire_True(t, resp.Success)\n\n\t// A retry of the first peer-remove should return an error that the peer is already removed.\n\t// Both a \"success\" response and this error should be used to know the peer-remove was successful.\n\treq = &JSApiMetaServerRemoveRequest{Server: remove1}\n\tjsreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err = nc.Request(JSApiRemoveServer, jsreq, time.Second)\n\trequire_NoError(t, err)\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerNotMemberError())\n\n\t// Audit should reflect the same.\n\trmsg, err = auditSub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tauditResp = JSAPIAudit{}\n\trequire_NoError(t, json.Unmarshal(rmsg.Data, &auditResp))\n\tresp = JSApiMetaServerRemoveResponse{}\n\trequire_NoError(t, json.Unmarshal([]byte(auditResp.Response), &resp))\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\trequire_Error(t, resp.Error, NewJSClusterServerNotMemberError())\n}\n\n//\n// DO NOT ADD NEW TESTS IN THIS FILE (unless to balance test times)\n// Add at the end of jetstream_cluster__test.go, with being the highest value.\n//\n" }, { "path": "server/jetstream_cluster_2_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests_2\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\tcrand \"crypto/rand\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestJetStreamClusterJSAPIImport(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterImportsTempl, \"C1\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API - This will connect to the non-js account which imports JS.\n\t// Connect below does an AccountInfo call.\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Note if this was ephemeral we would need to setup export/import for that subject.\n\tsub, err := js.SubscribeSync(\"TEST\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Make sure we can look up both.\n\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := sub.ConsumerInfo(); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Names list..\n\tvar names []string\n\tfor name := range js.StreamNames() {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(names))\n\t}\n\n\t// Now send to stream.\n\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tsub, err = js.PullSubscribe(\"TEST\", \"tr\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmsgs := fetchMsgs(t, sub, 1, 5*time.Second)\n\n\tm := msgs[0]\n\tif m.Subject != \"TEST\" {\n\t\tt.Fatalf(\"Expected subject of %q, got %q\", \"TEST\", m.Subject)\n\t}\n\tif m.Header != nil {\n\t\tt.Fatalf(\"Expected no header on the message, got: %v\", m.Header)\n\t}\n\tmeta, err := m.Metadata()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif meta.Sequence.Consumer != 1 || meta.Sequence.Stream != 1 || meta.NumDelivered != 1 || meta.NumPending != 0 {\n\t\tt.Fatalf(\"Bad meta: %+v\", meta)\n\t}\n\n\tjs.Publish(\"TEST\", []byte(\"Second\"))\n\tjs.Publish(\"TEST\", []byte(\"Third\"))\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"tr\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Error getting consumer info: %v\", err)\n\t\t}\n\t\tif ci.NumPending != 2 {\n\t\t\treturn fmt.Errorf(\"NumPending still not 1: %v\", ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Ack across accounts.\n\tm, err = nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.TEST.tr\", []byte(\"+NXT\"), 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmeta, err = m.Metadata()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif meta.Sequence.Consumer != 2 || meta.Sequence.Stream != 2 || meta.NumDelivered != 1 || meta.NumPending != 1 {\n\t\tt.Fatalf(\"Bad meta: %+v\", meta)\n\t}\n\n\t// AckNext\n\t_, err = nc.Request(m.Reply, []byte(\"+NXT\"), 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterMultiRestartBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 10000 messages.\n\tmsg, toSend := make([]byte, 4*1024), 10000\n\tcrand.Read(msg)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected to have %d messages, got %d\", toSend, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// For this bug, we will stop and remove the complete state from one server.\n\ts := c.randomServer()\n\topts := s.getOpts()\n\ts.Shutdown()\n\tremoveDir(t, opts.StoreDir)\n\n\t// Then restart it.\n\tc.restartAll()\n\tc.waitOnAllCurrent()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\ts = c.serverByName(s.Name())\n\tc.waitOnStreamCurrent(s, \"$G\", \"TEST\")\n\n\t// Now restart them all..\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Make sure the replicas are current.\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$G\", \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterServerLimits(t *testing.T) {\n\t// 2MB memory, 8MB disk\n\tc := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, \"R3L\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tmsg, toSend := make([]byte, 4*1024), 5000\n\tcrand.Read(msg)\n\n\t// Memory first.\n\tmax_mem := uint64(2*1024*1024) + uint64(len(msg))\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TM\",\n\t\tReplicas: 3,\n\t\tStorage: nats.MemoryStorage,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Grab stream leader.\n\tsl := c.streamLeader(\"ONE\", \"TM\")\n\trequire_True(t, sl != nil)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TM\", msg); err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tif err == nil || !strings.HasPrefix(err.Error(), \"nats: insufficient resources\") {\n\t\tt.Fatalf(\"Expected a ErrJetStreamResourcesExceeded error, got %v\", err)\n\t}\n\n\t// Since we have run all servers out of resources, no leader will be elected to respond to stream info.\n\t// So check manually.\n\tacc, err := sl.LookupAccount(\"ONE\")\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TM\")\n\trequire_NoError(t, err)\n\tif state := mset.state(); state.Bytes > max_mem {\n\t\tt.Fatalf(\"Expected bytes of %v to not be greater then %v\",\n\t\t\tfriendlyBytes(int64(state.Bytes)),\n\t\t\tfriendlyBytes(int64(max_mem)),\n\t\t)\n\t}\n\n\tc.waitOnLeader()\n\n\t// Now disk.\n\tmax_disk := uint64(8*1024*1024) + uint64(len(msg))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TF\",\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Grab stream leader.\n\tsl = c.streamLeader(\"ONE\", \"TF\")\n\trequire_True(t, sl != nil)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TF\", msg); err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tif err == nil || !strings.HasPrefix(err.Error(), \"nats: insufficient resources\") {\n\t\tt.Fatalf(\"Expected a ErrJetStreamResourcesExceeded error, got %v\", err)\n\t}\n\n\t// Since we have run all servers out of resources, no leader will be elected to respond to stream info.\n\t// So check manually.\n\tacc, err = sl.LookupAccount(\"ONE\")\n\trequire_NoError(t, err)\n\tmset, err = acc.lookupStream(\"TF\")\n\trequire_NoError(t, err)\n\tif state := mset.state(); state.Bytes > max_disk {\n\t\tt.Fatalf(\"Expected bytes of %v to not be greater then %v\",\n\t\t\tfriendlyBytes(int64(state.Bytes)),\n\t\t\tfriendlyBytes(int64(max_disk)),\n\t\t)\n\t}\n}\n\nfunc TestJetStreamClusterAccountLoadFailure(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterLimitsTempl, \"R3L\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.leader())\n\tdefer nc.Close()\n\n\t// Remove the \"ONE\" account from non-leader\n\ts := c.randomNonLeader()\n\ts.mu.Lock()\n\ts.accounts.Delete(\"ONE\")\n\ts.mu.Unlock()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"F\", Replicas: 3})\n\tif err == nil || !strings.Contains(err.Error(), \"account not found\") {\n\t\tt.Fatalf(\"Expected an 'account not found' error but got %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterAckPendingWithExpired(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tMaxAge: 500 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 100 messages.\n\tmsg, toSend := make([]byte, 256), 100\n\tcrand.Read(msg)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci.NumAckPending != toSend {\n\t\tt.Fatalf(\"Expected %d to be pending, got %d\", toSend, ci.NumAckPending)\n\t}\n\n\t// Wait for messages to expire.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Once expired these messages can not be redelivered so should not be considered ack pending at this point.\n\t// Now ack..\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err = sub.ConsumerInfo()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ci.NumAckPending != 0 {\n\t\t\treturn fmt.Errorf(\"Expected nothing to be ack pending, got %d\", ci.NumAckPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterAckPendingWithMaxRedelivered(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 100 messages.\n\tmsg, toSend := []byte(\"HELLO\"), 100\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\",\n\t\tnats.MaxDeliver(2),\n\t\tnats.Durable(\"dlc\"),\n\t\tnats.AckWait(10*time.Millisecond),\n\t\tnats.MaxAckPending(50),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckSubsPending(t, sub, toSend*2)\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := sub.ConsumerInfo()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ci.NumAckPending != 0 {\n\t\t\treturn fmt.Errorf(\"Expected nothing to be ack pending, got %d\", ci.NumAckPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMixedMode(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\ttmpl string\n\t}{\n\t\t{\"multi-account\", jsClusterLimitsTempl},\n\t\t{\"global-account\", jsMixedModeGlobalAccountTempl},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\tc := createMixedModeCluster(t, test.tmpl, \"MM5\", _EMPTY_, 3, 2, true)\n\t\t\tdefer c.shutdown()\n\n\t\t\t// Client based API - Non-JS server.\n\t\t\tnc, js := jsClientConnect(t, c.serverByName(\"S-5\"))\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tml := c.leader()\n\t\t\tif ml == nil {\n\t\t\t\tt.Fatalf(\"No metaleader\")\n\t\t\t}\n\n\t\t\t// Make sure we are tracking only the JS peers.\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tpeers := ml.JetStreamClusterPeers()\n\t\t\t\tif len(peers) == 3 {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Not correct number of peers, expected %d, got %d\", 3, len(peers))\n\t\t\t})\n\n\t\t\t// Grab the underlying raft structure and make sure the system adjusts its cluster set size.\n\t\t\tmeta := ml.getJetStream().getMetaGroup().(*raft)\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tps := meta.currentPeerState()\n\t\t\t\tif len(ps.knownPeers) != 3 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected known peers to be 3, but got %+v\", ps.knownPeers)\n\t\t\t\t}\n\t\t\t\tif ps.clusterSize < 3 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected cluster size to be 3, but got %+v\", ps)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterLeafnodeSpokes(t *testing.T) {\n\tc := createJetStreamCluster(t, jsClusterTempl, \"HUB\", _EMPTY_, 3, 22020, false)\n\tdefer c.shutdown()\n\n\tlnc1 := c.createLeafNodesWithStartPortAndDomain(\"R1\", 3, 22110, _EMPTY_)\n\tdefer lnc1.shutdown()\n\n\tlnc2 := c.createLeafNodesWithStartPortAndDomain(\"R2\", 3, 22120, _EMPTY_)\n\tdefer lnc2.shutdown()\n\n\tlnc3 := c.createLeafNodesWithStartPortAndDomain(\"R3\", 3, 22130, _EMPTY_)\n\tdefer lnc3.shutdown()\n\n\t// Wait on all peers.\n\tc.waitOnPeerCount(12)\n\n\t// Make sure shrinking works.\n\tlnc3.shutdown()\n\tc.waitOnPeerCount(9)\n\n\tlnc3 = c.createLeafNodesWithStartPortAndDomain(\"LNC3\", 3, 22130, _EMPTY_)\n\tdefer lnc3.shutdown()\n\n\tc.waitOnPeerCount(12)\n}\n\nfunc TestJetStreamClusterLeafNodeDenyNoDupe(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 18033, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tln := c.createLeafNodeWithTemplate(\"LN-SPOKE\", tmpl)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, ln, 2)\n\n\t// Now disconnect our leafnode connections by restarting the server we are connected to..\n\tfor _, s := range c.servers {\n\t\tif s.ClusterName() != c.name {\n\t\t\tcontinue\n\t\t}\n\t\tif nln := s.NumLeafNodes(); nln > 0 {\n\t\t\ts.Shutdown()\n\t\t\tc.restartServer(s)\n\t\t}\n\t}\n\t// Make sure we are back connected.\n\tcheckLeafNodeConnectedCount(t, ln, 2)\n\n\t// Now grab leaf varz and make sure we have no dupe deny clauses.\n\tvz, err := ln.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Grab the correct remote.\n\tfor _, remote := range vz.LeafNode.Remotes {\n\t\tif remote.LocalAccount == ln.SystemAccount().Name {\n\t\t\tif remote.Deny != nil && len(remote.Deny.Exports) > 3 { // denyAll := []string{jscAllSubj, raftAllSubj, jsAllAPI}\n\t\t\t\tt.Fatalf(\"Dupe entries found: %+v\", remote.Deny)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// Multiple JS domains.\nfunc TestJetStreamClusterSingleLeafNodeWithoutSharedSystemAccount(t *testing.T) {\n\tc := createJetStreamCluster(t, strings.Replace(jsClusterAccountsTempl, \"store_dir\", \"domain: CORE, store_dir\", 1), \"HUB\", _EMPTY_, 3, 14333, true)\n\tdefer c.shutdown()\n\n\tln := c.createSingleLeafNodeNoSystemAccount()\n\tdefer ln.Shutdown()\n\n\t// The setup here has a single leafnode server with two accounts. One has JS, the other does not.\n\t// We want to test the following.\n\t// 1. For the account without JS, we simply will pass through to the HUB. Meaning since our local account\n\t// does not have it, we simply inherit the hub's by default.\n\t// 2. For the JS enabled account, we are isolated and use our local one only.\n\n\t// Check behavior of the account without JS.\n\t// Normally this should fail since our local account is not enabled. However, since we are bridging\n\t// via the leafnode we expect this to work here.\n\tnc, js := jsClientConnectEx(t, ln, []nats.JSOpt{nats.Domain(\"CORE\")}, nats.UserInfo(\"n\", \"p\"))\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil || si.Cluster.Name != \"HUB\" {\n\t\tt.Fatalf(\"Expected stream to be placed in %q\", \"HUB\")\n\t}\n\t// Do some other API calls.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"C1\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tseen := 0\n\tfor name := range js.StreamNames() {\n\t\tseen++\n\t\tif name != \"TEST\" {\n\t\t\tt.Fatalf(\"Expected only %q but got %q\", \"TEST\", name)\n\t\t}\n\t}\n\tif seen != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream, got %d\", seen)\n\t}\n\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\tif err := js.DeleteConsumer(\"TEST\", \"C1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.UpdateStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"bar\"}, Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now check the enabled account.\n\t// Check the enabled account only talks to its local JS domain by default.\n\tnc, js = jsClientConnect(t, ln, nats.UserInfo(\"y\", \"p\"))\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(JSAdvisoryStreamCreatedPre + \".>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster != nil {\n\t\tt.Fatalf(\"Expected no cluster designation for stream since created on single LN server\")\n\t}\n\n\t// Wait for a bit and make sure we only get one of these.\n\t// The HUB domain should be cut off by default.\n\ttime.Sleep(250 * time.Millisecond)\n\tcheckSubsPending(t, sub, 1)\n\t// Drain.\n\tfor _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) {\n\t}\n\n\t// Now try to talk to the HUB JS domain through a new context that uses a different mapped subject.\n\t// This is similar to how we let users cross JS domains between accounts as well.\n\tjs, err = nc.JetStream(nats.APIPrefix(\"$JS.HUB.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\t// This should fail here with jetstream not enabled.\n\tif _, err := js.AccountInfo(); err != nats.ErrJetStreamNotEnabled {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now add in a mapping to the connected account in the HUB.\n\t// This aligns with the APIPrefix context above and works across leafnodes.\n\t// TODO(dlc) - Should we have a mapping section for leafnode solicit?\n\tc.addSubjectMapping(\"ONE\", \"$JS.HUB.API.>\", \"$JS.API.>\")\n\n\t// Now it should work.\n\tif _, err := js.AccountInfo(); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure we can add a stream, etc.\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST22\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil || si.Cluster.Name != \"HUB\" {\n\t\tt.Fatalf(\"Expected stream to be placed in %q\", \"HUB\")\n\t}\n\n\tjsLocal, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\t// Create a mirror on the local leafnode for stream TEST22.\n\t_, err = jsLocal.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST22\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.HUB.API\"},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Publish a message to the HUB's TEST22 stream.\n\tif _, err := js.Publish(\"bar\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\t// Make sure the message arrives in our mirror.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsLocal.StreamInfo(\"M\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do the reverse and create a sourced stream in the HUB from our local stream on leafnode.\n\t// Inside the HUB we need to be able to find our local leafnode JetStream assets, so we need\n\t// a mapping in the LN server to allow this to work. Normally this will just be in server config.\n\tacc, err := ln.LookupAccount(\"JSY\")\n\tif err != nil {\n\t\tc.t.Fatalf(\"Unexpected error on %v: %v\", ln, err)\n\t}\n\tif err := acc.AddMapping(\"$JS.LN.API.>\", \"$JS.API.>\"); err != nil {\n\t\tc.t.Fatalf(\"Error adding mapping: %v\", err)\n\t}\n\n\t// js is the HUB JetStream context here.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"M\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.LN.API\"},\n\t\t}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure the message arrives in our sourced stream.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// JetStream Domains\nfunc TestJetStreamClusterDomains(t *testing.T) {\n\t// This adds in domain config option to template.\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 12232, true)\n\tdefer c.shutdown()\n\n\t// This leafnode is a single server with no domain but sharing the system account.\n\t// This extends the CORE domain through this leafnode.\n\tln := c.createLeafNodeWithTemplate(\"LN-SYS\",\n\t\tstrings.ReplaceAll(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"extension_hint: will_extend, domain: CORE, store_dir:\"))\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, ln, 2)\n\n\t// This shows we have extended this system.\n\tc.waitOnPeerCount(4)\n\tif ml := c.leader(); ml == ln {\n\t\tt.Fatalf(\"Detected a meta-leader in the leafnode: %s\", ml)\n\t}\n\n\t// Now create another LN but with a domain defined.\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tspoke := c.createLeafNodeWithTemplate(\"LN-SPOKE\", tmpl)\n\tdefer spoke.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, spoke, 2)\n\n\t// Should be the same, should not extend the CORE domain.\n\tc.waitOnPeerCount(4)\n\n\t// The domain signals to the system that we are our own JetStream domain and should not extend CORE.\n\t// We want to check to make sure we have all the deny properly setup.\n\tspoke.mu.Lock()\n\t// var hasDE, hasDI bool\n\tfor _, ln := range spoke.leafs {\n\t\tln.mu.Lock()\n\t\tremote := ln.leaf.remote\n\t\tln.mu.Unlock()\n\t\tremote.RLock()\n\t\tif remote.RemoteLeafOpts.LocalAccount == \"$SYS\" {\n\t\t\tfor _, s := range denyAllJs {\n\t\t\t\tif r := ln.perms.pub.deny.Match(s); len(r.psubs) != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected to have deny permission for %s\", s)\n\t\t\t\t}\n\t\t\t\tif r := ln.perms.sub.deny.Match(s); len(r.psubs) != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected to have deny permission for %s\", s)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tremote.RUnlock()\n\t}\n\tspoke.mu.Unlock()\n\n\t// Now do some operations.\n\t// Check the enabled account only talks to its local JS domain by default.\n\tnc, js := jsClientConnect(t, spoke)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster != nil {\n\t\tt.Fatalf(\"Expected no cluster designation for stream since created on single LN server\")\n\t}\n\n\t// Now try to talk to the CORE JS domain through a new context that uses a different mapped subject.\n\tjsCore, err := nc.JetStream(nats.APIPrefix(\"$JS.CORE.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\tif _, err := jsCore.AccountInfo(); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Make sure we can add a stream, etc.\n\tsi, err = jsCore.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST22\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster == nil || si.Cluster.Name != \"CORE\" {\n\t\tt.Fatalf(\"Expected stream to be placed in %q, got %q\", \"CORE\", si.Cluster.Name)\n\t}\n\n\tjsLocal, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\t// Create a mirror on our local leafnode for stream TEST22.\n\t_, err = jsLocal.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST22\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.CORE.API\"},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Publish a message to the CORE's TEST22 stream.\n\tif _, err := jsCore.Publish(\"bar\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\t// Make sure the message arrives in our mirror.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsLocal.StreamInfo(\"M\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// jsCore is the CORE JetStream domain.\n\t// Create a sourced stream in the CORE that is sourced from our mirror stream in our leafnode.\n\t_, err = jsCore.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"M\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.SPOKE.API\"},\n\t\t}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure the message arrives in our sourced stream.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsCore.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now connect directly to the CORE cluster and make sure we can operate there.\n\tnc, jsLocal = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create the js contexts again.\n\tjsSpoke, err := nc.JetStream(nats.APIPrefix(\"$JS.SPOKE.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\t// Publish a message to the CORE's TEST22 stream.\n\tif _, err := jsLocal.Publish(\"bar\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\t// Make sure the message arrives in our mirror.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsSpoke.StreamInfo(\"M\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure the message arrives in our sourced stream.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsLocal.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// We are connected to the CORE domain/system. Create a JetStream context referencing ourselves.\n\tjsCore, err = nc.JetStream(nats.APIPrefix(\"$JS.CORE.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\tsi, err = jsCore.StreamInfo(\"S\")\n\tif err != nil {\n\t\tt.Fatalf(\"Could not get stream info: %v\", err)\n\t}\n\tif si.State.Msgs != 2 {\n\t\tt.Fatalf(\"Expected 2 msgs, got state: %+v\", si.State)\n\t}\n}\n\nfunc TestJetStreamClusterDomainsWithNoJSHub(t *testing.T) {\n\t// Create our hub cluster with no JetStream defined.\n\tc := createMixedModeCluster(t, jsClusterAccountsTempl, \"NOJS5\", _EMPTY_, 0, 5, false)\n\tdefer c.shutdown()\n\n\tln := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStream()\n\tdefer ln.Shutdown()\n\n\tlnd := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(\"SPOKE\", \"nojs\")\n\tdefer lnd.Shutdown()\n\n\t// Client based API - Connected to the core cluster with no JS but account has JS.\n\ts := c.randomServer()\n\t// Make sure the JS interest from the LNs has made it to this server.\n\tcheckSubInterest(t, s, \"NOJS\", \"$JS.SPOKE.API.INFO\", time.Second)\n\tnc, _ := jsClientConnect(t, s, nats.UserInfo(\"nojs\", \"p\"))\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\treq, err := json.Marshal(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Do by hand to make sure we only get one response.\n\tsis := fmt.Sprintf(strings.ReplaceAll(JSApiStreamCreateT, JSApiPrefix, \"$JS.SPOKE.API\"), \"TEST\")\n\trs := nats.NewInbox()\n\tsub, _ := nc.SubscribeSync(rs)\n\tnc.PublishRequest(sis, rs, req)\n\t// Wait for response.\n\tcheckSubsPending(t, sub, 1)\n\t// Double check to make sure we only have 1.\n\tif nr, _, err := sub.Pending(); err != nil || nr != 1 {\n\t\tt.Fatalf(\"Expected 1 response, got %d and %v\", nr, err)\n\t}\n\tresp, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\t// This StreamInfo should *not* have a domain set.\n\t// Do by hand until this makes it to the Go client.\n\tvar si StreamInfo\n\tif err = json.Unmarshal(resp.Data, &si); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Domain != _EMPTY_ {\n\t\tt.Fatalf(\"Expected to have NO domain set but got %q\", si.Domain)\n\t}\n\n\t// Now let's create a stream specifically on the SPOKE domain.\n\tjs, err := nc.JetStream(nats.Domain(\"SPOKE\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST22\",\n\t\tSubjects: []string{\"bar\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now lookup by hand to check domain.\n\tresp, err = nc.Request(\"$JS.SPOKE.API.STREAM.INFO.TEST22\", nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err = json.Unmarshal(resp.Data, &si); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Domain != \"SPOKE\" {\n\t\tt.Fatalf(\"Expected to have domain set to %q but got %q\", \"SPOKE\", si.Domain)\n\t}\n}\n\n// Issue #2205\nfunc TestJetStreamClusterDomainsAndAPIResponses(t *testing.T) {\n\t// This adds in domain config option to template.\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 12232, true)\n\tdefer c.shutdown()\n\n\t// Now create spoke LN cluster.\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"SPOKE\", 5, 23913)\n\tdefer lnc.shutdown()\n\n\tlnc.waitOnClusterReady()\n\n\t// Make the physical connection to the CORE.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now create JS domain context and try to do same in LN cluster.\n\t// The issue referenced above details a bug where we can not receive a positive response.\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjsSpoke, err := nc.JetStream(nats.APIPrefix(\"$JS.SPOKE.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\tsi, err := jsSpoke.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != \"SPOKE\" {\n\t\tt.Fatalf(\"Expected %q as the cluster, got %q\", \"SPOKE\", si.Cluster.Name)\n\t}\n}\n\n// Issue #2202\nfunc TestJetStreamClusterDomainsAndSameNameSources(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 9323, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE-1, store_dir:\", 1)\n\tspoke1 := c.createLeafNodeWithTemplate(\"LN-SPOKE-1\", tmpl)\n\tdefer spoke1.Shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE-2, store_dir:\", 1)\n\tspoke2 := c.createLeafNodeWithTemplate(\"LN-SPOKE-2\", tmpl)\n\tdefer spoke2.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, spoke1, 2)\n\tcheckLeafNodeConnectedCount(t, spoke2, 2)\n\n\tsubjFor := func(s *Server) string {\n\t\tswitch s {\n\t\tcase spoke1:\n\t\t\treturn \"foo\"\n\t\tcase spoke2:\n\t\t\treturn \"bar\"\n\t\t}\n\t\treturn \"TEST\"\n\t}\n\n\t// Create the same name stream in both spoke domains.\n\tfor _, s := range []*Server{spoke1, spoke2} {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{subjFor(s)},\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tnc.Close()\n\t}\n\n\t// Now connect to the hub and create a sourced stream from both leafnode streams.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.SPOKE-1.API\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.SPOKE-2.API\"},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Publish a message to each spoke stream and we will check that our sourced stream gets both.\n\tfor _, s := range []*Server{spoke1, spoke2} {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tjs.Publish(subjFor(s), []byte(\"DOUBLE TROUBLE\"))\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\tt.Fatalf(\"Expected 1 msg, got %d\", si.State.Msgs)\n\t\t}\n\t\tnc.Close()\n\t}\n\n\t// Now make sure we have 2 msgs in our sourced stream.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure we can see our external information.\n\t// This not in the Go client yet so manual for now.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"S\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar ssi StreamInfo\n\tif err = json.Unmarshal(resp.Data, &ssi); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif len(ssi.Sources) != 2 {\n\t\tt.Fatalf(\"Expected 2 source streams, got %d\", len(ssi.Sources))\n\t}\n\tif ssi.Sources[0].External == nil {\n\t\tt.Fatalf(\"Expected a non-nil external designation\")\n\t}\n\tpre := ssi.Sources[0].External.ApiPrefix\n\tif pre != \"$JS.SPOKE-1.API\" && pre != \"$JS.SPOKE-2.API\" {\n\t\tt.Fatalf(\"Expected external api of %q, got %q\", \"$JS.SPOKE-[1|2].API\", ssi.Sources[0].External.ApiPrefix)\n\t}\n\n\t// Also create a mirror.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.SPOKE-1.API\"},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"M\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err = json.Unmarshal(resp.Data, &ssi); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ssi.Mirror == nil || ssi.Mirror.External == nil {\n\t\tt.Fatalf(\"Expected a non-nil external designation for our mirror\")\n\t}\n\tif ssi.Mirror.External.ApiPrefix != \"$JS.SPOKE-1.API\" {\n\t\tt.Fatalf(\"Expected external api of %q, got %q\", \"$JS.SPOKE-1.API\", ssi.Sources[0].External.ApiPrefix)\n\t}\n}\n\n// When a leafnode enables JS on an account that is not enabled on the remote cluster account this should fail\n// Accessing a jet stream in a different availability domain requires the client provide a damain name, or\n// the server having set up appropriate defaults (default_js_domain. tested in leafnode_test.go)\nfunc TestJetStreamClusterSingleLeafNodeEnablingJetStream(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 11322, true)\n\tdefer c.shutdown()\n\n\tln := c.createSingleLeafNodeNoSystemAccountAndEnablesJetStream()\n\tdefer ln.Shutdown()\n\n\t// Check that we have JS in the $G account on the leafnode.\n\tnc, js := jsClientConnect(t, ln)\n\tdefer nc.Close()\n\n\tif _, err := js.AccountInfo(); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Connect our client to the \"nojs\" account in the cluster but make sure JS works since its enabled via the leafnode.\n\ts := c.randomServer()\n\tnc, js = jsClientConnect(t, s, nats.UserInfo(\"nojs\", \"p\"))\n\tdefer nc.Close()\n\t_, err := js.AccountInfo()\n\t// error is context deadline exceeded as the local account has no js and can't reach the remote one\n\trequire_True(t, err == context.DeadlineExceeded)\n}\n\nfunc TestJetStreamClusterLeafNodesWithoutJS(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 11233, true)\n\tdefer c.shutdown()\n\n\ttestJS := func(s *Server, domain string, doDomainAPI bool) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tif doDomainAPI {\n\t\t\tvar err error\n\t\t\tapiPre := fmt.Sprintf(\"$JS.%s.API\", domain)\n\t\t\tif js, err = nc.JetStream(nats.APIPrefix(apiPre)); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t\t\t}\n\t\t}\n\t\tai, err := js.AccountInfo()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ai.Domain != domain {\n\t\t\tt.Fatalf(\"Expected domain of %q, got %q\", domain, ai.Domain)\n\t\t}\n\t}\n\n\tln := c.createLeafNodeWithTemplate(\"LN-SYS-S-NOJS\", jsClusterTemplWithSingleLeafNodeNoJS)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, ln, 2)\n\n\t// Check that we can access JS in the $G account on the cluster through the leafnode.\n\ttestJS(ln, \"HUB\", true)\n\tln.Shutdown()\n\n\t// Now create a leafnode cluster with No JS and make sure that works.\n\tlnc := c.createLeafNodesNoJS(\"LN-SYS-C-NOJS\", 3)\n\tdefer lnc.shutdown()\n\n\ttestJS(lnc.randomServer(), \"HUB\", true)\n\tlnc.shutdown()\n\n\t// Do mixed mode but with a JS config block that specifies domain and just sets it to disabled.\n\t// This is the preferred method for mixed mode, always define JS server config block just disable\n\t// in those you do not want it running.\n\t// e.g. jetstream: {domain: \"SPOKE\", enabled: false}\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tlncm := c.createLeafNodesWithTemplateMixedMode(tmpl, \"SPOKE\", 3, 2, true)\n\tdefer lncm.shutdown()\n\n\t// Now grab a non-JS server, last two are non-JS.\n\tsl := lncm.servers[0]\n\ttestJS(sl, \"SPOKE\", false)\n\n\t// Test that mappings work as well and we can access the hub.\n\ttestJS(sl, \"HUB\", true)\n}\n\nfunc TestJetStreamClusterLeafNodesWithSameDomainNames(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 11233, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"SPOKE\", 3, 11311)\n\tdefer lnc.shutdown()\n\n\tc.waitOnPeerCount(6)\n}\n\nfunc TestJetStreamClusterLeafDifferentAccounts(t *testing.T) {\n\tc := createJetStreamCluster(t, jsClusterAccountsTempl, \"HUB\", _EMPTY_, 2, 23133, false)\n\tdefer c.shutdown()\n\n\tln := c.createLeafNodesWithStartPortAndDomain(\"LN\", 2, 22110, _EMPTY_)\n\tdefer ln.shutdown()\n\n\t// Wait on all peers.\n\tc.waitOnPeerCount(4)\n\n\tnc, js := jsClientConnect(t, ln.randomServer())\n\tdefer nc.Close()\n\n\t// Make sure we can properly identify the right account when the leader received the request.\n\t// We need to map the client info header to the new account once received by the hub.\n\tif _, err := js.AccountInfo(); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterStreamInfoDeletedDetails(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R2\", 2)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"HELLO\"), 10\n\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Now remove some messages.\n\tdeleteMsg := func(seq uint64) {\n\t\tif err := js.DeleteMsg(\"TEST\", seq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\tdeleteMsg(2)\n\tdeleteMsg(4)\n\tdeleteMsg(6)\n\n\t// Need to do these via direct server request for now.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar si StreamInfo\n\tif err = json.Unmarshal(resp.Data, &si); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.NumDeleted != 3 {\n\t\tt.Fatalf(\"Expected %d deleted, got %d\", 3, si.State.NumDeleted)\n\t}\n\tif len(si.State.Deleted) != 0 {\n\t\tt.Fatalf(\"Expected not deleted details, but got %+v\", si.State.Deleted)\n\t}\n\n\t// Now request deleted details.\n\treq := JSApiStreamInfoRequest{DeletedDetails: true}\n\tb, _ := json.Marshal(req)\n\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), b, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif err = json.Unmarshal(resp.Data, &si); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif si.State.NumDeleted != 3 {\n\t\tt.Fatalf(\"Expected %d deleted, got %d\", 3, si.State.NumDeleted)\n\t}\n\tif len(si.State.Deleted) != 3 {\n\t\tt.Fatalf(\"Expected deleted details, but got %+v\", si.State.Deleted)\n\t}\n}\n\nfunc TestJetStreamClusterMirrorAndSourceExpiration(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSE\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Origin\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tbi := 1\n\tsendBatch := func(n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tmsg := fmt.Sprintf(\"ID: %d\", bi)\n\t\t\tbi++\n\t\t\tif _, err := js.PublishAsync(\"TEST\", []byte(msg)); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckStream := func(stream string, num uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 50*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != num {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", num, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckSource := func(num uint64) { t.Helper(); checkStream(\"S\", num) }\n\tcheckMirror := func(num uint64) { t.Helper(); checkStream(\"M\", num) }\n\tcheckTest := func(num uint64) { t.Helper(); checkStream(\"TEST\", num) }\n\n\tvar err error\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\tReplicas: 2,\n\t\tMaxAge: 500 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// We want this to not be same as TEST leader for this test.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tfor ss := sl; ss == sl; {\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"S\",\n\t\t\tSources: []*nats.StreamSource{{Name: \"TEST\"}},\n\t\t\tReplicas: 2,\n\t\t\tMaxAge: 500 * time.Millisecond,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ss = c.streamLeader(\"$G\", \"S\"); ss == sl {\n\t\t\t// Delete and retry.\n\t\t\tjs.DeleteStream(\"S\")\n\t\t}\n\t}\n\n\t// Allow direct sync consumers to connect.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\tsendBatch(100)\n\tcheckTest(100)\n\tcheckMirror(100)\n\tcheckSource(100)\n\n\t// Make sure they expire.\n\tcheckMirror(0)\n\tcheckSource(0)\n\n\t// Now stop the server housing the leader of the source stream.\n\tsl.Shutdown()\n\tc.restartServer(sl)\n\tcheckClusterFormed(t, c.servers...)\n\tc.waitOnStreamLeader(\"$G\", \"S\")\n\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\t// Make sure can process correctly after we have expired all of the messages.\n\tsendBatch(100)\n\t// Need to check both in parallel.\n\tscheck, mcheck := uint64(0), uint64(0)\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tif scheck != 100 {\n\t\t\tif si, _ := js.StreamInfo(\"S\"); si != nil {\n\t\t\t\tscheck = si.State.Msgs\n\t\t\t}\n\t\t}\n\t\tif mcheck != 100 {\n\t\t\tif si, _ := js.StreamInfo(\"M\"); si != nil {\n\t\t\t\tmcheck = si.State.Msgs\n\t\t\t}\n\t\t}\n\t\tif scheck == 100 && mcheck == 100 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Both not at 100 yet, S=%d, M=%d\", scheck, mcheck)\n\t})\n\n\tcheckTest(200)\n}\n\nfunc TestJetStreamClusterMirrorAndSourceSubLeaks(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSL\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tstartSubs := c.stableTotalSubs()\n\n\tvar ss []*nats.StreamSource\n\n\t// Create 10 origin streams\n\tfor i := 0; i < 10; i++ {\n\t\tsn := fmt.Sprintf(\"ORDERS-%d\", i+1)\n\t\tss = append(ss, &nats.StreamSource{Name: sn})\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: sn}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Create mux'd stream that sources all of the origin streams.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"MUX\",\n\t\tReplicas: 2,\n\t\tSources: ss,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now create a mirror of the mux stream.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: \"MUX\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Get stable subs count.\n\tafterSubs := c.stableTotalSubs()\n\n\tjs.DeleteStream(\"MIRROR\")\n\tjs.DeleteStream(\"MUX\")\n\n\tfor _, si := range ss {\n\t\tjs.DeleteStream(si.Name)\n\t}\n\n\t// Some subs take longer to settle out so we give ourselves a small buffer.\n\t// There will be 1 sub for client on each server (such as _INBOX.IvVJ2DOXUotn4RUSZZCFvp.*)\n\t// and 2 or 3 subs such as `_R_.xxxxx.>` on each server, so a total of 12 subs.\n\tif deleteSubs := c.stableTotalSubs(); deleteSubs > startSubs+12 {\n\t\tt.Fatalf(\"Expected subs to return to %d from a high of %d, but got %d\", startSubs, afterSubs, deleteSubs)\n\t}\n}\n\nfunc TestJetStreamClusterCreateConcurrentDurableConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSL\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create origin stream, must be R > 1\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"ORDERS\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.QueueSubscribeSync(\"ORDERS\", \"wq\", nats.Durable(\"shared\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now try to create durables concurrently.\n\tstart := make(chan struct{})\n\tvar wg sync.WaitGroup\n\tcreated := uint32(0)\n\terrs := make(chan error, 10)\n\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t<-start\n\t\t\t_, err := js.QueueSubscribeSync(\"ORDERS\", \"wq\", nats.Durable(\"shared\"))\n\t\t\tif err == nil {\n\t\t\t\tatomic.AddUint32(&created, 1)\n\t\t\t} else if !strings.Contains(err.Error(), \"consumer name already\") {\n\t\t\t\terrs <- err\n\t\t\t}\n\t\t}()\n\t}\n\n\tclose(start)\n\twg.Wait()\n\n\tif lc := atomic.LoadUint32(&created); lc != 10 {\n\t\tt.Fatalf(\"Expected all 10 to be created, got %d\", lc)\n\t}\n\tif len(errs) > 0 {\n\t\tt.Fatalf(\"Failed to create some sub: %v\", <-errs)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/2144\nfunc TestJetStreamClusterUpdateStreamToExisting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSL\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS1\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS2\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"bar\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"ORDERS2\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n}\n\nfunc TestJetStreamClusterCrossAccountInterop(t *testing.T) {\n\ttemplate := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: HUB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\tJS {\n\t\t\tjetstream: enabled\n\t\t\tusers = [ { user: \"rip\", pass: \"pass\" } ]\n\t\t\texports [\n\t\t\t\t{ service: \"$JS.API.CONSUMER.INFO.>\" }\n\t\t\t\t{ service: \"$JS.HUB.API.CONSUMER.>\", response: stream }\n\t\t\t\t{ stream: \"M.SYNC.>\" } # For the mirror\n\t\t\t]\n\t\t}\n\t\tIA {\n\t\t\tjetstream: enabled\n\t\t\tusers = [ { user: \"dlc\", pass: \"pass\" } ]\n\t\t\timports [\n\t\t\t\t{ service: { account: JS, subject: \"$JS.API.CONSUMER.INFO.TEST.DLC\"}, to: \"FROM.DLC\" }\n\t\t\t\t{ service: { account: JS, subject: \"$JS.HUB.API.CONSUMER.>\"}, to: \"js.xacc.API.CONSUMER.>\" }\n\t\t\t\t{ stream: { account: JS, subject: \"M.SYNC.>\"} }\n\t\t\t]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n\t`\n\n\tc := createJetStreamClusterWithTemplate(t, template, \"HUB\", 3)\n\tdefer c.shutdown()\n\n\t// Create the stream and the consumer under the JS/rip user.\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"rip\", \"pass\"))\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DLC\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Also create a stream via the domain qualified API.\n\tjs, err = nc.JetStream(nats.APIPrefix(\"$JS.HUB.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"ORDERS\", Replicas: 2}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now we want to access the consumer info from IA/dlc.\n\tnc2, js2 := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"dlc\", \"pass\"))\n\tdefer nc2.Close()\n\n\tif _, err := nc2.Request(\"FROM.DLC\", nil, time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure domain mappings etc work across accounts.\n\t// Setup a mirror.\n\t_, err = js2.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"ORDERS\",\n\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\tAPIPrefix: \"js.xacc.API\",\n\t\t\t\tDeliverPrefix: \"M.SYNC\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send 10 messages..\n\tmsg, toSend := []byte(\"Hello mapped domains\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"ORDERS\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MIRROR\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 10 {\n\t\t\treturn fmt.Errorf(\"Expected 10 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/2242\nfunc TestJetStreamClusterMsgIdDuplicateBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSL\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendMsgID := func(id string) (*nats.PubAck, error) {\n\t\tt.Helper()\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Add(JSMsgId, id)\n\t\tm.Data = []byte(\"HELLO WORLD\")\n\t\treturn js.PublishMsg(m)\n\t}\n\n\tif _, err := sendMsgID(\"1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// This should fail with duplicate detected.\n\tif pa, _ := sendMsgID(\"1\"); pa == nil || !pa.Duplicate {\n\t\tt.Fatalf(\"Expected duplicate but got none: %+v\", pa)\n\t}\n\t// This should be fine.\n\tif _, err := sendMsgID(\"2\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterNilMsgWithHeaderThroughSourcedStream(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 12232, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tspoke := c.createLeafNodeWithTemplate(\"SPOKE\", tmpl)\n\tdefer spoke.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, spoke, 2)\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, spoke)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tjsHub, err := nc.JetStream(nats.APIPrefix(\"$JS.HUB.API\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\n\t_, err = jsHub.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tReplicas: 2,\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.SPOKE.API\"},\n\t\t}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now send a message to the origin stream with nil body and a header.\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Add(\"X-Request-ID\", \"e9a639b4-cecb-4fbe-8376-1ef511ae1f8d\")\n\tm.Data = []byte(\"HELLO WORLD\")\n\n\tif _, err = jsHub.PublishMsg(m); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := jsHub.SubscribeSync(\"foo\", nats.BindStream(\"S\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif string(msg.Data) != \"HELLO WORLD\" {\n\t\tt.Fatalf(\"Message corrupt? Expecting %q got %q\", \"HELLO WORLD\", msg.Data)\n\t}\n}\n\n// Make sure varz reports the server usage not replicated usage etc.\nfunc TestJetStreamClusterVarzReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// ~100k per message.\n\tmsg := []byte(strings.Repeat(\"A\", 99_960))\n\tmsz := fileStoreMsgSize(\"TEST\", nil, msg)\n\ttotal := msz * 10\n\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\t// To show the bug we need this to allow remote usage to replicate.\n\ttime.Sleep(2 * usageTick)\n\n\tv, err := s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif v.JetStream.Stats.Store > total {\n\t\tt.Fatalf(\"Single server varz JetStream store usage should be <= %d, got %d\", total, v.JetStream.Stats.Store)\n\t}\n\n\tinfo, err := js.AccountInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Store < total*3 {\n\t\tt.Fatalf(\"Expected account information to show usage ~%d, got %d\", total*3, info.Store)\n\t}\n}\n\nfunc TestJetStreamClusterPurgeBySequence(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.*.*\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 2,\n\t\t\t\tMaxMsgsPer: 5,\n\t\t\t}\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tfor i := 0; i < 20; i++ {\n\t\t\t\tif _, err = js.Publish(\"kv.myapp.username\", []byte(fmt.Sprintf(\"value %d\", i))); err != nil {\n\t\t\t\t\tt.Fatalf(\"request failed: %s\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor i := 0; i < 20; i++ {\n\t\t\t\tif _, err = js.Publish(\"kv.myapp.password\", []byte(fmt.Sprintf(\"value %d\", i))); err != nil {\n\t\t\t\t\tt.Fatalf(\"request failed: %s\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\texpectSequences := func(t *testing.T, subject string, seq ...int) {\n\t\t\t\tsub, err := js.SubscribeSync(subject)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"sub failed: %s\", err)\n\t\t\t\t}\n\t\t\t\tdefer sub.Unsubscribe()\n\t\t\t\tfor _, i := range seq {\n\t\t\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatalf(\"didn't get message: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t\tmeta, err := msg.Metadata()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatalf(\"didn't get metadata: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t\tif meta.Sequence.Stream != uint64(i) {\n\t\t\t\t\t\tt.Fatalf(\"expected sequence %d got %d\", i, meta.Sequence.Stream)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\texpectSequences(t, \"kv.myapp.username\", 16, 17, 18, 19, 20)\n\t\t\texpectSequences(t, \"kv.myapp.password\", 36, 37, 38, 39, 40)\n\n\t\t\t// delete up to but not including 18 of username...\n\t\t\tjr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: \"kv.myapp.username\", Sequence: 18})\n\t\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"KV\"), jr, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"request failed: %s\", err)\n\t\t\t}\n\t\t\t// 18 should still be there\n\t\t\texpectSequences(t, \"kv.myapp.username\", 18, 19, 20)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterMaxConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"MAXC\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"in.maxc.>\"},\n\t\tMaxConsumers: 1,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsi, err := js.StreamInfo(\"MAXC\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Config.MaxConsumers != 1 {\n\t\tt.Fatalf(\"Expected max of 1, got %d\", si.Config.MaxConsumers)\n\t}\n\t// Make sure we get the right error.\n\t// This should succeed.\n\tif _, err := js.SubscribeSync(\"in.maxc.foo\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.SubscribeSync(\"in.maxc.bar\"); err == nil {\n\t\tt.Fatalf(\"Eexpected error but got none\")\n\t}\n}\n\nfunc TestJetStreamClusterMaxConsumersMultipleConcurrentRequests(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"MAXCC\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"in.maxcc.>\"},\n\t\tMaxConsumers: 1,\n\t\tReplicas: 3,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsi, err := js.StreamInfo(\"MAXCC\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Config.MaxConsumers != 1 {\n\t\tt.Fatalf(\"Expected max of 1, got %d\", si.Config.MaxConsumers)\n\t}\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\twg.Add(10)\n\tfor n := 0; n < 10; n++ {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\t\tgo func(js nats.JetStreamContext) {\n\t\t\tdefer wg.Done()\n\t\t\t<-startCh\n\t\t\tjs.SubscribeSync(\"in.maxcc.foo\")\n\t\t}(js)\n\t}\n\t// Wait for Go routines.\n\ttime.Sleep(250 * time.Millisecond)\n\n\tclose(startCh)\n\twg.Wait()\n\n\tvar names []string\n\tfor n := range js.ConsumerNames(\"MAXCC\") {\n\t\tnames = append(names, n)\n\t}\n\tif nc := len(names); nc > 1 {\n\t\tt.Fatalf(\"Expected only 1 consumer, got %d\", nc)\n\t}\n}\n\nfunc TestJetStreamClusterAccountMaxStreamsAndConsumersMultipleConcurrentRequests(t *testing.T) {\n\ttmpl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\tA {\n\t\t\tjetstream {\n\t\t\t\tmax_file: 9663676416\n\t\t\t\tmax_streams: 2\n\t\t\t\tmax_consumers: 1\n\t\t\t}\n\t\t\tusers = [ { user: \"a\", pass: \"pwd\" } ]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n\t`\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"MAXCC\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"in.maxcc.>\"},\n\t\tReplicas: 3,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsi, err := js.StreamInfo(\"MAXCC\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Config.MaxConsumers != -1 {\n\t\tt.Fatalf(\"Expected max of -1, got %d\", si.Config.MaxConsumers)\n\t}\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\twg.Add(10)\n\tfor n := 0; n < 10; n++ {\n\t\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"a\", \"pwd\"))\n\t\tdefer nc.Close()\n\t\tgo func(js nats.JetStreamContext, idx int) {\n\t\t\tdefer wg.Done()\n\t\t\t<-startCh\n\t\t\t// Test adding new streams\n\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"OTHER_%d\", idx),\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\t// Test adding consumers to MAXCC stream\n\t\t\tjs.SubscribeSync(\"in.maxcc.foo\", nats.BindStream(\"MAXCC\"))\n\t\t}(js, n)\n\t}\n\t// Wait for Go routines.\n\ttime.Sleep(250 * time.Millisecond)\n\n\tclose(startCh)\n\twg.Wait()\n\n\tvar names []string\n\tfor n := range js.StreamNames() {\n\t\tnames = append(names, n)\n\t}\n\tif nc := len(names); nc > 2 {\n\t\tt.Fatalf(\"Expected only 2 streams, got %d\", nc)\n\t}\n\tnames = names[:0]\n\tfor n := range js.ConsumerNames(\"MAXCC\") {\n\t\tnames = append(names, n)\n\t}\n\tif nc := len(names); nc > 1 {\n\t\tt.Fatalf(\"Expected only 1 consumer, got %d\", nc)\n\t}\n}\n\nfunc TestJetStreamClusterPanicDecodingConsumerState(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\trch := make(chan struct{}, 2)\n\tnc, js := jsClientConnect(t, c.randomServer(),\n\t\tnats.ReconnectWait(50*time.Millisecond),\n\t\tnats.MaxReconnects(-1),\n\t\tnats.ReconnectHandler(func(_ *nats.Conn) {\n\t\t\trch <- struct{}{}\n\t\t}),\n\t)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ORDERS.*\"},\n\t\tStorage: nats.FileStorage,\n\t\tReplicas: 3,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tDiscard: nats.DiscardNew,\n\t\tMaxMsgs: -1,\n\t\tMaxAge: time.Hour * 24 * 365,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"ORDERS.created\", \"durable\", nats.MaxAckPending(1000))\n\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating pull subscriber: %v\", err)\n\t}\n\n\tsendMsg := func(subject string) {\n\t\tt.Helper()\n\t\tif _, err := js.Publish(subject, []byte(\"msg\")); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendMsg(\"ORDERS.something\")\n\t\tsendMsg(\"ORDERS.created\")\n\t}\n\n\tfor total := 0; total != 100; {\n\t\tmsgs, err := sub.Fetch(100-total, nats.MaxWait(2*time.Second))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to fetch message: %v\", err)\n\t\t}\n\t\tfor _, m := range msgs {\n\t\t\tm.AckSync()\n\t\t\ttotal++\n\t\t}\n\t}\n\n\tc.stopAll()\n\tc.restartAllSamePorts()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"durable\")\n\n\tselect {\n\tcase <-rch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Did not reconnect\")\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendMsg(\"ORDERS.something\")\n\t\tsendMsg(\"ORDERS.created\")\n\t}\n\n\tfor total := 0; total != 100; {\n\t\tmsgs, err := sub.Fetch(100-total, nats.MaxWait(2*time.Second))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on fetch: %v\", err)\n\t\t}\n\t\tfor _, m := range msgs {\n\t\t\tm.AckSync()\n\t\t\ttotal++\n\t\t}\n\t}\n}\n\n// Had a report of leaked subs with pull subscribers.\nfunc TestJetStreamClusterPullConsumerLeakedSubs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"Domains.*\"},\n\t\tReplicas: 1,\n\t\tRetention: nats.InterestPolicy,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"Domains.Domain\", \"Domains-Api\", nats.MaxAckPending(20_000))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating pull subscriber: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\t// Load up a bunch of requests.\n\tnumRequests := 20\n\tfor i := 0; i < numRequests; i++ {\n\t\tjs.PublishAsync(\"Domains.Domain\", []byte(\"QUESTION\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\tnumSubs := c.stableTotalSubs()\n\n\t// With batch of 1 we do not see any issues, so set to 10.\n\t// Currently Go client uses auto unsub based on the batch size.\n\tfor i := 0; i < numRequests/10; i++ {\n\t\tmsgs, err := sub.Fetch(10)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tfor _, m := range msgs {\n\t\t\tm.AckSync()\n\t\t}\n\t}\n\n\t// Make sure the stream is empty..\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Stream should be empty, got %+v\", si)\n\t}\n\n\t// Make sure we did not leak any subs.\n\tif numSubsAfter := c.stableTotalSubs(); numSubsAfter != numSubs {\n\t\tt.Fatalf(\"Subs leaked: %d before, %d after\", numSubs, numSubsAfter)\n\t}\n}\n\nfunc TestJetStreamClusterPushConsumerQueueGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tjs.Publish(\"foo\", []byte(\"QG\"))\n\n\t// Do consumer by hand for now.\n\tinbox := nats.NewInbox()\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tDeliverSubject: inbox,\n\t\t\tDeliverGroup: \"22\",\n\t\t\tAckPolicy: AckNone,\n\t\t},\n\t}\n\treq, err := json.Marshal(obsReq)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"dlc\"), req, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error, got %+v\", ccResp.Error)\n\t}\n\n\tsub, _ := nc.SubscribeSync(inbox)\n\tif _, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Expected a timeout, we should not get messages here\")\n\t}\n\tqsub, _ := nc.QueueSubscribeSync(inbox, \"22\")\n\tcheckSubsPending(t, qsub, 1)\n\n\t// Test deleting the plain sub has not affect.\n\tsub.Unsubscribe()\n\tjs.Publish(\"foo\", []byte(\"QG\"))\n\tcheckSubsPending(t, qsub, 2)\n\n\tqsub.Unsubscribe()\n\tqsub2, _ := nc.QueueSubscribeSync(inbox, \"22\")\n\tjs.Publish(\"foo\", []byte(\"QG\"))\n\tcheckSubsPending(t, qsub2, 1)\n\n\t// Catch all sub.\n\tsub, _ = nc.SubscribeSync(inbox)\n\tqsub2.Unsubscribe() // Should be no more interest.\n\t// Send another, make sure we do not see the message flow here.\n\tjs.Publish(\"foo\", []byte(\"QG\"))\n\tif _, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Expected a timeout, we should not get messages here\")\n\t}\n}\n\nfunc TestJetStreamClusterConsumerLastActiveReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{Name: \"foo\", Replicas: 2}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendMsg := func() {\n\t\tt.Helper()\n\t\tif _, err := js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// TODO(dlc) - Do by hand for now until Go client has this.\n\tconsumerInfo := func(name string) *ConsumerInfo {\n\t\tt.Helper()\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerInfoT, \"foo\", name), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar cinfo JSApiConsumerInfoResponse\n\t\tif err := json.Unmarshal(resp.Data, &cinfo); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cinfo.ConsumerInfo == nil || cinfo.Error != nil {\n\t\t\tt.Fatalf(\"Got a bad response %+v\", cinfo)\n\t\t}\n\t\treturn cinfo.ConsumerInfo\n\t}\n\n\tif ci := consumerInfo(\"dlc\"); ci.Delivered.Last != nil || ci.AckFloor.Last != nil {\n\t\tt.Fatalf(\"Expected last to be nil by default, got %+v\", ci)\n\t}\n\n\tcheckTimeDiff := func(t1, t2 *time.Time) {\n\t\tt.Helper()\n\t\t// Compare on a seconds level\n\t\trt1, rt2 := t1.UTC().Round(time.Second), t2.UTC().Round(time.Second)\n\t\tif rt1 != rt2 {\n\t\t\td := rt1.Sub(rt2)\n\t\t\tif d > time.Second || d < -time.Second {\n\t\t\t\tt.Fatalf(\"Times differ too much, expected %v got %v\", rt1, rt2)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckDelivered := func(name string) {\n\t\tt.Helper()\n\t\tnow := time.Now()\n\t\tci := consumerInfo(name)\n\t\tif ci.Delivered.Last == nil {\n\t\t\tt.Fatalf(\"Expected delivered last to not be nil after activity, got %+v\", ci.Delivered)\n\t\t}\n\t\tcheckTimeDiff(&now, ci.Delivered.Last)\n\t}\n\n\tcheckLastAck := func(name string, m *nats.Msg) {\n\t\tt.Helper()\n\t\tnow := time.Now()\n\t\tif err := m.AckSync(); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tci := consumerInfo(name)\n\t\tif ci.AckFloor.Last == nil {\n\t\t\tt.Fatalf(\"Expected ack floor last to not be nil after ack, got %+v\", ci.AckFloor)\n\t\t}\n\t\t// Compare on a seconds level\n\t\tcheckTimeDiff(&now, ci.AckFloor.Last)\n\t}\n\n\tcheckAck := func(name string) {\n\t\tt.Helper()\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tcheckLastAck(name, m)\n\t}\n\n\t// Push\n\tsendMsg()\n\tcheckSubsPending(t, sub, 1)\n\tcheckDelivered(\"dlc\")\n\tcheckAck(\"dlc\")\n\n\t// Check pull.\n\tsub, err = js.PullSubscribe(\"foo\", \"rip\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendMsg()\n\t// Should still be nil since pull.\n\tif ci := consumerInfo(\"rip\"); ci.Delivered.Last != nil || ci.AckFloor.Last != nil {\n\t\tt.Fatalf(\"Expected last to be nil by default, got %+v\", ci)\n\t}\n\tmsgs, err := sub.Fetch(1)\n\tif err != nil || len(msgs) == 0 {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckDelivered(\"rip\")\n\tcheckLastAck(\"rip\", msgs[0])\n\n\t// Now test to make sure this state is held correctly across a cluster.\n\tci := consumerInfo(\"rip\")\n\tnc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"foo\", \"rip\"), nil, time.Second)\n\tc.waitOnConsumerLeader(\"$G\", \"foo\", \"rip\")\n\tnci := consumerInfo(\"rip\")\n\tif nci.Delivered.Last == nil {\n\t\tt.Fatalf(\"Expected delivered last to not be nil, got %+v\", nci.Delivered)\n\t}\n\tif nci.AckFloor.Last == nil {\n\t\tt.Fatalf(\"Expected ack floor last to not be nil, got %+v\", nci.AckFloor)\n\t}\n\n\tcheckTimeDiff(ci.Delivered.Last, nci.Delivered.Last)\n\tcheckTimeDiff(ci.AckFloor.Last, nci.AckFloor.Last)\n}\n\nfunc TestJetStreamClusterRaceOnRAFTCreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsrv := c.servers[0]\n\tnc, err := nats.Connect(srv.ClientURL())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tjs, err = nc.JetStream(nats.MaxWait(2 * time.Second))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsize := 10\n\twg := sync.WaitGroup{}\n\twg.Add(size)\n\tfor i := 0; i < size; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t// We don't care about possible failures here, we just want\n\t\t\t// parallel creation of a consumer.\n\t\t\tjs.PullSubscribe(\"foo\", \"shared\")\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc TestJetStreamClusterDeadlockOnVarz(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsrv := c.servers[0]\n\tnc, err := nats.Connect(srv.ClientURL())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsize := 10\n\twg := sync.WaitGroup{}\n\twg.Add(size)\n\tch := make(chan struct{})\n\tfor i := 0; i < size; i++ {\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\t\t\t<-ch\n\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"TEST%d\", i),\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t}(i)\n\t}\n\n\tclose(ch)\n\tfor i := 0; i < 10; i++ {\n\t\tsrv.Varz(nil)\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\twg.Wait()\n}\n\n// Issue #2397\nfunc TestJetStreamClusterStreamCatchupNoState(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R2S\", 2)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Hold onto servers.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tif sl == nil {\n\t\tt.Fatalf(\"Did not get a server\")\n\t}\n\tnsl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tif nsl == nil {\n\t\tt.Fatalf(\"Did not get a server\")\n\t}\n\t// Grab low level stream and raft node.\n\tmset, err := nsl.GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tnode := mset.raftNode()\n\tif node == nil {\n\t\tt.Fatalf(\"Could not get stream group name\")\n\t}\n\tgname := node.Group()\n\n\tnumRequests := 100\n\tfor i := 0; i < numRequests; i++ {\n\t\t// This will force a snapshot which will prune the normal log.\n\t\t// We will remove the snapshot to simulate the error condition.\n\t\tif i == 10 {\n\t\t\tif err := node.InstallSnapshot(mset.stateSnapshot(), false); err != nil {\n\t\t\t\tt.Fatalf(\"Error installing snapshot: %v\", err)\n\t\t\t}\n\t\t}\n\t\t_, err := js.Publish(\"foo.created\", []byte(\"REQ\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tconfig := nsl.JetStreamConfig()\n\tif config == nil {\n\t\tt.Fatalf(\"No config\")\n\t}\n\tlconfig := sl.JetStreamConfig()\n\tif lconfig == nil {\n\t\tt.Fatalf(\"No config\")\n\t}\n\n\tnc.Close()\n\tc.stopAll()\n\t// Remove all state by truncating for the non-leader.\n\t// For both make sure we have no raft snapshots.\n\tsnapDir := filepath.Join(lconfig.StoreDir, \"$SYS\", \"_js_\", gname, \"snapshots\")\n\trequire_NoError(t, os.RemoveAll(snapDir))\n\tmsgsDir := filepath.Join(lconfig.StoreDir, \"$SYS\", \"_js_\", gname, \"msgs\")\n\trequire_NoError(t, os.RemoveAll(msgsDir))\n\t// Remove all our raft state, we do not want to hold onto our term and index which\n\t// results in a coin toss for who becomes the leader.\n\traftDir := filepath.Join(config.StoreDir, \"$SYS\", \"_js_\", gname)\n\trequire_NoError(t, os.RemoveAll(raftDir))\n\n\t// Now restart.\n\tc.restartAll()\n\tfor _, cs := range c.servers {\n\t\tc.waitOnStreamCurrent(cs, \"$G\", \"TEST\")\n\t}\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t_, err = js.Publish(\"foo.created\", []byte(\"ZZZ\"))\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.State.LastSeq != 101 {\n\t\tt.Fatalf(\"bad state after restart: %+v\", si.State)\n\t}\n}\n\n// Issue #2525\nfunc TestJetStreamClusterLargeHeaders(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"add stream failed: %s\", err)\n\t}\n\n\t// We use u16 to encode msg header len. Make sure we do the right thing when > 65k.\n\tdata := make([]byte, 8*1024)\n\tcrand.Read(data)\n\tval := hex.EncodeToString(data)[:8*1024]\n\tm := nats.NewMsg(\"foo\")\n\tfor i := 1; i <= 10; i++ {\n\t\tm.Header.Add(fmt.Sprintf(\"LargeHeader-%d\", i), val)\n\t}\n\tm.Data = []byte(\"Hello Large Headers!\")\n\tif _, err = js.PublishMsg(m); err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n}\n\nfunc TestJetStreamClusterFlowControlRequiresHeartbeats(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tFlowControl: true,\n\t}); err == nil || IsNatsErr(err, JSConsumerWithFlowControlNeedsHeartbeats) {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamClusterMixedModeColdStartPrune(t *testing.T) {\n\t// Purposely make this unbalanced. Without changes this will never form a quorum to elect the meta-leader.\n\tc := createMixedModeCluster(t, jsMixedModeGlobalAccountTempl, \"MMCS5\", _EMPTY_, 3, 4, false)\n\tdefer c.shutdown()\n\n\t// Make sure we report cluster size.\n\tcheckClusterSize := func(s *Server) {\n\t\tt.Helper()\n\t\tjsi, err := s.Jsz(nil)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif jsi.Meta == nil {\n\t\t\tt.Fatalf(\"Expected a cluster info\")\n\t\t}\n\t\tif jsi.Meta.Size != 3 {\n\t\t\tt.Fatalf(\"Expected cluster size to be adjusted to %d, but got %d\", 3, jsi.Meta.Size)\n\t\t}\n\t}\n\n\tcheckClusterSize(c.leader())\n\tcheckClusterSize(c.randomNonLeader())\n}\n\nfunc TestJetStreamClusterMirrorAndSourceCrossNonNeighboringDomain(t *testing.T) {\n\tstoreDir1 := t.TempDir()\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain1, store_dir: '%s'}\n\t\taccounts {\n\t\t\tA:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n\t\t\tSYS:{ users:[ {user:s1,password:s1}]},\n\t\t}\n\t\tsystem_account = SYS\n\t\tno_auth_user: a1\n\t\tleafnodes: {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`, storeDir1)))\n\ts1, _ := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\tstoreDir2 := t.TempDir()\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain2, store_dir: '%s'}\n\t\taccounts {\n\t\t\tA:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n\t\t\tSYS:{ users:[ {user:s1,password:s1}]},\n\t\t}\n\t\tsystem_account = SYS\n\t\tno_auth_user: a1\n\t\tleafnodes:{\n\t\t\tremotes:[{ url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t\t\t\t { url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n\t\t}\n\t`, storeDir2, s1.opts.LeafNode.Port, s1.opts.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\tstoreDir3 := t.TempDir()\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 256MB, domain: domain3, store_dir: '%s'}\n\t\taccounts {\n\t\t\tA:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n\t\t\tSYS:{ users:[ {user:s1,password:s1}]},\n\t\t}\n\t\tsystem_account = SYS\n\t\tno_auth_user: a1\n\t\tleafnodes:{\n\t\t\tremotes:[{ url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t\t\t\t { url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n\t\t}\n\t`, storeDir3, s1.opts.LeafNode.Port, s1.opts.LeafNode.Port)))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, s1, 4)\n\tcheckLeafNodeConnectedCount(t, s2, 2)\n\tcheckLeafNodeConnectedCount(t, s3, 2)\n\n\tc2 := natsConnect(t, s2.ClientURL())\n\tdefer c2.Close()\n\tjs2, err := c2.JetStream(nats.Domain(\"domain2\"))\n\trequire_NoError(t, err)\n\tai2, err := js2.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ai2.Domain, \"domain2\")\n\t_, err = js2.AddStream(&nats.StreamConfig{\n\t\tName: \"disk\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"disk\"},\n\t})\n\trequire_NoError(t, err)\n\t_, err = js2.Publish(\"disk\", nil)\n\trequire_NoError(t, err)\n\tsi, err := js2.StreamInfo(\"disk\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 1)\n\n\tc3 := natsConnect(t, s3.ClientURL())\n\tdefer c3.Close()\n\tjs3, err := c3.JetStream(nats.Domain(\"domain3\"))\n\trequire_NoError(t, err)\n\tai3, err := js3.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ai3.Domain, \"domain3\")\n\n\t_, err = js3.AddStream(&nats.StreamConfig{\n\t\tName: \"stream-mirror\",\n\t\tStorage: nats.FileStorage,\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"disk\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.domain2.API\"},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js3.AddStream(&nats.StreamConfig{\n\t\tName: \"stream-source\",\n\t\tStorage: nats.FileStorage,\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"disk\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.domain2.API\"},\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tif si, _ := js3.StreamInfo(\"stream-mirror\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for mirror, got %d\", si.State.Msgs)\n\t\t}\n\t\tif si, _ := js3.StreamInfo(\"stream-source\"); si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg for source, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterSeal(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\t// Need to be done by hand until makes its way to Go client.\n\tcreateStream := func(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *JSApiStreamCreateResponse {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar scResp JSApiStreamCreateResponse\n\t\terr = json.Unmarshal(resp.Data, &scResp)\n\t\trequire_NoError(t, err)\n\t\treturn &scResp\n\t}\n\n\tupdateStream := func(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *JSApiStreamUpdateResponse {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar scResp JSApiStreamUpdateResponse\n\t\terr = json.Unmarshal(resp.Data, &scResp)\n\t\trequire_NoError(t, err)\n\t\treturn &scResp\n\t}\n\n\ttestSeal := func(t *testing.T, s *Server, replicas int) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\t// Should not be able to create a stream that starts sealed.\n\t\tscr := createStream(t, nc, &StreamConfig{Name: \"SEALED\", Replicas: replicas, Storage: MemoryStorage, Sealed: true})\n\t\tif scr.Error == nil {\n\t\t\tt.Fatalf(\"Expected an error but got none\")\n\t\t}\n\t\t// Create our stream.\n\t\tscr = createStream(t, nc, &StreamConfig{Name: \"SEALED\", Replicas: replicas, MaxAge: time.Minute, Storage: MemoryStorage})\n\t\tif scr.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", scr.Error)\n\t\t}\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tjs.Publish(\"SEALED\", []byte(\"OK\"))\n\t\t}\n\t\t// Update to sealed.\n\t\tsur := updateStream(t, nc, &StreamConfig{Name: \"SEALED\", Replicas: replicas, MaxAge: time.Minute, Storage: MemoryStorage, Sealed: true})\n\t\tif sur.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", sur.Error)\n\t\t}\n\n\t\t// Grab stream info and make sure its reflected as sealed.\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"SEALED\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar sir JSApiStreamInfoResponse\n\t\terr = json.Unmarshal(resp.Data, &sir)\n\t\trequire_NoError(t, err)\n\t\tif sir.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", sir.Error)\n\t\t}\n\t\tsi := sir.StreamInfo\n\t\tif !si.Config.Sealed {\n\t\t\tt.Fatalf(\"Expected the stream to be marked sealed, got %+v\\n\", si.Config)\n\t\t}\n\t\t// Make sure we also updated any max age and moved to discard new.\n\t\tif si.Config.MaxAge != 0 {\n\t\t\tt.Fatalf(\"Expected MaxAge to be cleared, got %v\", si.Config.MaxAge)\n\t\t}\n\t\tif si.Config.Discard != DiscardNew {\n\t\t\tt.Fatalf(\"Expected DiscardPolicy to be set to new, got %v\", si.Config.Discard)\n\t\t}\n\t\t// Also make sure we set denyDelete and denyPurge.\n\t\tif !si.Config.DenyDelete {\n\t\t\tt.Fatalf(\"Expected the stream to be marked as DenyDelete, got %+v\\n\", si.Config)\n\t\t}\n\t\tif !si.Config.DenyPurge {\n\t\t\tt.Fatalf(\"Expected the stream to be marked as DenyPurge, got %+v\\n\", si.Config)\n\t\t}\n\t\tif si.Config.AllowRollup {\n\t\t\tt.Fatalf(\"Expected the stream to be marked as not AllowRollup, got %+v\\n\", si.Config)\n\t\t}\n\n\t\t// Sealing is not reversible, so make sure we get an error trying to undo.\n\t\tsur = updateStream(t, nc, &StreamConfig{Name: \"SEALED\", Replicas: replicas, Storage: MemoryStorage, Sealed: false})\n\t\tif sur.Error == nil {\n\t\t\tt.Fatalf(\"Expected an error but got none\")\n\t\t}\n\n\t\t// Now test operations like publish a new msg, delete, purge etc all fail.\n\t\tif _, err := js.Publish(\"SEALED\", []byte(\"OK\")); err == nil {\n\t\t\tt.Fatalf(\"Expected a publish to fail\")\n\t\t}\n\t\tif err := js.DeleteMsg(\"SEALED\", 1); err == nil {\n\t\t\tt.Fatalf(\"Expected a delete to fail\")\n\t\t}\n\t\tif err := js.PurgeStream(\"SEALED\"); err == nil {\n\t\t\tt.Fatalf(\"Expected a purge to fail\")\n\t\t}\n\t\tif err := js.DeleteStream(\"SEALED\"); err != nil {\n\t\t\tt.Fatalf(\"Expected a delete to succeed, got %v\", err)\n\t\t}\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { testSeal(t, s, 1) })\n\tt.Run(\"Clustered\", func(t *testing.T) { testSeal(t, c.randomServer(), 3) })\n}\n\n// Issue #2568\nfunc TestJetStreamClusteredStreamCreateIdempotent(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"AUDIT\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tDenyDelete: true,\n\t\tDenyPurge: true,\n\t}\n\taddStream(t, nc, cfg)\n\taddStream(t, nc, cfg)\n}\n\nfunc TestJetStreamClusterRollupsRequirePurge(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"SENSORS\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"sensor.*.temp\"},\n\t\tMaxMsgsPer: 10,\n\t\tAllowRollup: true,\n\t\tDenyPurge: true,\n\t\tReplicas: 2,\n\t}\n\n\tj, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), j, time.Second)\n\trequire_NoError(t, err)\n\n\tvar cr JSApiStreamCreateResponse\n\terr = json.Unmarshal(resp.Data, &cr)\n\trequire_NoError(t, err)\n\tif cr.Error == nil || cr.Error.Description != \"roll-ups require the purge permission\" {\n\t\tt.Fatalf(\"unexpected error: %v\", cr.Error)\n\t}\n}\n\nfunc TestJetStreamClusterRollups(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"SENSORS\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"sensor.*.temp\"},\n\t\tMaxMsgsPer: 10,\n\t\tAllowRollup: true,\n\t\tReplicas: 2,\n\t}\n\taddStream(t, nc, cfg)\n\n\tvar bt [16]byte\n\tvar le = binary.LittleEndian\n\n\t// Generate 1000 random measurements for 10 sensors\n\tfor i := 0; i < 1000; i++ {\n\t\tid, temp := strconv.Itoa(rand.Intn(9)+1), rand.Int31n(42)+60 // 60-102 degrees.\n\t\tle.PutUint16(bt[0:], uint16(temp))\n\t\tjs.PublishAsync(fmt.Sprintf(\"sensor.%v.temp\", id), bt[:])\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Grab random sensor and do a rollup by averaging etc.\n\tsensor := fmt.Sprintf(\"sensor.%v.temp\", strconv.Itoa(rand.Intn(9)+1))\n\tsub, err := js.SubscribeSync(sensor)\n\trequire_NoError(t, err)\n\n\tvar total, samples int\n\tfor m, err := sub.NextMsg(time.Second); err == nil; m, err = sub.NextMsg(time.Second) {\n\t\ttotal += int(le.Uint16(m.Data))\n\t\tsamples++\n\t}\n\tsub.Unsubscribe()\n\tavg := uint16(total / samples)\n\tle.PutUint16(bt[0:], avg)\n\n\trollup := nats.NewMsg(sensor)\n\trollup.Data = bt[:]\n\trollup.Header.Set(JSMsgRollup, JSMsgRollupSubject)\n\t_, err = js.PublishMsg(rollup)\n\trequire_NoError(t, err)\n\tsub, err = js.SubscribeSync(sensor)\n\trequire_NoError(t, err)\n\t// Make sure only 1 left.\n\tcheckSubsPending(t, sub, 1)\n\tsub.Unsubscribe()\n\n\t// Now do all.\n\trollup.Header.Set(JSMsgRollup, JSMsgRollupAll)\n\t_, err = js.PublishMsg(rollup)\n\trequire_NoError(t, err)\n\t// Same thing as above should hold true.\n\tsub, err = js.SubscribeSync(sensor)\n\trequire_NoError(t, err)\n\t// Make sure only 1 left.\n\tcheckSubsPending(t, sub, 1)\n\tsub.Unsubscribe()\n\n\t// Also should only be 1 msgs in total stream left with JSMsgRollupAll\n\tsi, err := js.StreamInfo(\"SENSORS\")\n\trequire_NoError(t, err)\n\tif si.State.Msgs != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg left after rollup all, got %+v\", si.State)\n\t}\n}\n\nfunc TestJetStreamClusterRollupSubjectAndWatchers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"KVW\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"kv.*\"},\n\t\tMaxMsgsPer: 10,\n\t\tAllowRollup: true,\n\t\tReplicas: 2,\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := js.SubscribeSync(\"kv.*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsend := func(key, value string) {\n\t\tt.Helper()\n\t\t_, err := js.Publish(\"kv.\"+key, []byte(value))\n\t\trequire_NoError(t, err)\n\t}\n\n\trollup := func(key, value string) {\n\t\tt.Helper()\n\t\tm := nats.NewMsg(\"kv.\" + key)\n\t\tm.Data = []byte(value)\n\t\tm.Header.Set(JSMsgRollup, JSMsgRollupSubject)\n\t\t_, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t}\n\n\texpectUpdate := func(key, value string, seq uint64) {\n\t\tt.Helper()\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tif m.Subject != \"kv.\"+key {\n\t\t\tt.Fatalf(\"Keys don't match: %q vs %q\", m.Subject[3:], key)\n\t\t}\n\t\tif string(m.Data) != value {\n\t\t\tt.Fatalf(\"Values don't match: %q vs %q\", m.Data, value)\n\t\t}\n\t\tmeta, err := m.Metadata()\n\t\trequire_NoError(t, err)\n\t\tif meta.Sequence.Consumer != seq {\n\t\t\tt.Fatalf(\"Sequences don't match: %v vs %v\", meta.Sequence.Consumer, value)\n\t\t}\n\t}\n\n\trollup(\"name\", \"derek\")\n\texpectUpdate(\"name\", \"derek\", 1)\n\trollup(\"age\", \"22\")\n\texpectUpdate(\"age\", \"22\", 2)\n\n\tsend(\"name\", \"derek\")\n\texpectUpdate(\"name\", \"derek\", 3)\n\tsend(\"age\", \"22\")\n\texpectUpdate(\"age\", \"22\", 4)\n\tsend(\"age\", \"33\")\n\texpectUpdate(\"age\", \"33\", 5)\n\tsend(\"name\", \"ivan\")\n\texpectUpdate(\"name\", \"ivan\", 6)\n\tsend(\"name\", \"rip\")\n\texpectUpdate(\"name\", \"rip\", 7)\n\trollup(\"age\", \"50\")\n\texpectUpdate(\"age\", \"50\", 8)\n}\n\nfunc TestJetStreamClusterAppendOnly(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"AUDIT\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tDenyDelete: true,\n\t\tDenyPurge: true,\n\t}\n\tsi := addStream(t, nc, cfg)\n\tif !si.Config.DenyDelete || !si.Config.DenyPurge {\n\t\tt.Fatalf(\"Expected DenyDelete and DenyPurge to be set, got %+v\", si.Config)\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tjs.Publish(\"foo\", []byte(\"ok\"))\n\t}\n\t// Delete should not be allowed.\n\tif err := js.DeleteMsg(\"AUDIT\", 1); err == nil {\n\t\tt.Fatalf(\"Expected an error for delete but got none\")\n\t}\n\tif err := js.PurgeStream(\"AUDIT\"); err == nil {\n\t\tt.Fatalf(\"Expected an error for purge but got none\")\n\t}\n\n\tcfg.DenyDelete = false\n\tcfg.DenyPurge = false\n\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Error == nil {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n}\n\n// Related to #2642\nfunc TestJetStreamClusterStreamUpdateSyncBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t}\n\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tmsg, toSend := []byte(\"OK\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.PublishAsync(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcfg.Subjects = []string{\"foo\", \"bar\", \"baz\"}\n\tif _, err := js.UpdateStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Shutdown a server. The bug is that the update wiped the sync subject used to catchup a stream that has the RAFT layer snapshotted.\n\tnsl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tnsl.Shutdown()\n\t// make sure a leader exists\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tfor i := 0; i < toSend*4; i++ {\n\t\tif _, err := js.PublishAsync(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Throw in deletes as well.\n\tfor seq := uint64(200); seq < uint64(300); seq += 4 {\n\t\tif err := js.DeleteMsg(\"TEST\", seq); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// We need to snapshot to force upper layer catchup vs RAFT layer.\n\tc.waitOnAllCurrent()\n\tmset, err := c.streamLeader(\"$G\", \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", \"TEST\")\n\t}\n\tif err := mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tc.waitOnAllCurrent()\n\tnsl = c.restartServer(nsl)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamCurrent(nsl, \"$G\", \"TEST\")\n\n\tmset, _ = nsl.GlobalAccount().lookupStream(\"TEST\")\n\tcloneState := mset.state()\n\n\tmset, _ = c.streamLeader(\"$G\", \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\tleaderState := mset.state()\n\n\tif !reflect.DeepEqual(cloneState, leaderState) {\n\t\tt.Fatalf(\"States do not match: %+v vs %+v\", cloneState, leaderState)\n\t}\n}\n\n// Issue #2666\nfunc TestJetStreamClusterKVMultipleConcurrentCreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"TEST\", History: 1, TTL: 150 * time.Millisecond, Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\tfor n := 0; n < 5; n++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t<-startCh\n\t\t\tif r, err := kv.Create(\"name\", []byte(\"dlc\")); err == nil {\n\t\t\t\tif _, err = kv.Update(\"name\", []byte(\"rip\"), r); err != nil {\n\t\t\t\t\tt.Log(\"Unexpected Update error: \", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\t// Wait for Go routines to start.\n\ttime.Sleep(100 * time.Millisecond)\n\tclose(startCh)\n\twg.Wait()\n\t// Just make sure its there and picks up the phone.\n\tif _, err := js.StreamInfo(\"KV_TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now make sure we do ok when servers are restarted and we need to deal with dangling clfs state.\n\t// First non-leader.\n\trs := c.randomNonStreamLeader(\"$G\", \"KV_TEST\")\n\trs.Shutdown()\n\trs = c.restartServer(rs)\n\tc.waitOnStreamCurrent(rs, \"$G\", \"KV_TEST\")\n\n\tif _, err := kv.Put(\"name\", []byte(\"ik\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now the actual leader.\n\tsl := c.streamLeader(\"$G\", \"KV_TEST\")\n\tsl.Shutdown()\n\tsl = c.restartServer(sl)\n\tc.waitOnStreamLeader(\"$G\", \"KV_TEST\")\n\tc.waitOnStreamCurrent(sl, \"$G\", \"KV_TEST\")\n\n\tif _, err := kv.Put(\"name\", []byte(\"mh\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ttime.Sleep(time.Second)\n}\n\nfunc TestJetStreamClusterAccountInfoForSystemAccount(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t_, err := js.AccountInfo()\n\trequire_Error(t, err, nats.ErrJetStreamNotEnabledForAccount)\n}\n\nfunc TestJetStreamClusterListFilter(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ttestList := func(t *testing.T, srv *Server, r int) {\n\t\tnc, js := jsClientConnect(t, srv)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"ONE\",\n\t\t\tSubjects: []string{\"one.>\"},\n\t\t\tReplicas: r,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TWO\",\n\t\t\tSubjects: []string{\"two.>\"},\n\t\t\tReplicas: r,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(JSApiStreamList, []byte(\"{}\"), time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tlist := &JSApiStreamListResponse{}\n\t\terr = json.Unmarshal(resp.Data, list)\n\t\trequire_NoError(t, err)\n\n\t\tif len(list.Streams) != 2 {\n\t\t\tt.Fatalf(\"Expected 2 responses got %d\", len(list.Streams))\n\t\t}\n\n\t\tresp, err = nc.Request(JSApiStreamList, []byte(`{\"subject\":\"two.x\"}`), time.Second)\n\t\trequire_NoError(t, err)\n\t\tlist = &JSApiStreamListResponse{}\n\t\terr = json.Unmarshal(resp.Data, list)\n\t\trequire_NoError(t, err)\n\t\tif len(list.Streams) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 response got %d\", len(list.Streams))\n\t\t}\n\t\tif list.Streams[0].Config.Name != \"TWO\" {\n\t\t\tt.Fatalf(\"Expected stream TWO in result got %#v\", list.Streams[0])\n\t\t}\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { testList(t, s, 1) })\n\tt.Run(\"Clustered\", func(t *testing.T) { testList(t, c.randomServer(), 3) })\n}\n\nfunc TestJetStreamClusterConsumerUpdates(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 5)\n\tdefer c.shutdown()\n\n\ttestConsumerUpdate := func(t *testing.T, s *Server, replicas int) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t// Create a stream.\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t\t}\n\n\t\tcfg := &nats.ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tDescription: \"Update TEST\",\n\t\t\tFilterSubject: \"foo\",\n\t\t\tDeliverSubject: \"d.foo\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tAckWait: time.Minute,\n\t\t\tMaxDeliver: 5,\n\t\t\tMaxAckPending: 50,\n\t\t}\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Update delivery subject, which worked before, but upon review had issues unless replica count == clustered size.\n\t\tcfg.DeliverSubject = \"d.bar\"\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Bind deliver subject.\n\t\tsub, err := nc.SubscribeSync(\"d.bar\")\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\n\t\tncfg := *cfg\n\t\t// Deliver Subject\n\t\tncfg.DeliverSubject = \"d.baz\"\n\t\t// Description\n\t\tcfg.Description = \"New Description\"\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// MaxAckPending\n\t\tcheckSubsPending(t, sub, 50)\n\t\tcfg.MaxAckPending = 75\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t\tcheckSubsPending(t, sub, 75)\n\n\t\t// Drain sub, do not ack first ten though so we can test shortening AckWait.\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tif i >= 10 {\n\t\t\t\tm.AckSync()\n\t\t\t}\n\t\t}\n\n\t\t// AckWait\n\t\tcheckSubsPending(t, sub, 0)\n\t\tcfg.AckWait = 200 * time.Millisecond\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t\tcheckSubsPending(t, sub, 10)\n\n\t\t// Rate Limit\n\t\tcfg.RateLimit = 8 * 1024\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\tcfg.RateLimit = 0\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// These all should fail.\n\t\tncfg = *cfg\n\t\tncfg.DeliverPolicy = nats.DeliverLastPolicy\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tncfg.OptStartSeq = 22\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tnow := time.Now()\n\t\tncfg.OptStartTime = &now\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tncfg.AckPolicy = nats.AckAllPolicy\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tncfg.ReplayPolicy = nats.ReplayOriginalPolicy\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tncfg.Heartbeat = time.Second\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t\tncfg = *cfg\n\t\tncfg.FlowControl = true\n\t\t_, err = js.UpdateConsumer(\"TEST\", &ncfg)\n\t\trequire_Error(t, err)\n\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { testConsumerUpdate(t, s, 1) })\n\tt.Run(\"Clustered\", func(t *testing.T) { testConsumerUpdate(t, c.randomServer(), 2) })\n}\n\nfunc TestJetStreamClusterConsumerMaxDeliverUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tmaxDeliver := 2\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ard\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tMaxDeliver: maxDeliver,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"ard\")\n\trequire_NoError(t, err)\n\n\tcheckMaxDeliver := func() {\n\t\tt.Helper()\n\t\tfor i := 0; i <= maxDeliver; i++ {\n\t\t\tmsgs, err := sub.Fetch(2, nats.MaxWait(100*time.Millisecond))\n\t\t\tif i < maxDeliver {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, 1, len(msgs))\n\t\t\t\t_ = msgs[0].Nak()\n\t\t\t} else {\n\t\t\t\trequire_Error(t, err, nats.ErrTimeout)\n\t\t\t}\n\t\t}\n\t}\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\tcheckMaxDeliver()\n\n\t// update maxDeliver\n\tmaxDeliver++\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ard\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tMaxDeliver: maxDeliver,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\tcheckMaxDeliver()\n}\n\nfunc TestJetStreamClusterAccountReservations(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, \"C1\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\taccMax := 3\n\n\terrResources := errors.New(\"nats: insufficient storage resources available\")\n\n\ttest := func(t *testing.T, replica int) {\n\t\tmb := int64((1+accMax)-replica) * 1024 * 1024 * 1024 // GB, corrected for replication factor\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S1\", Subjects: []string{\"s1\"}, MaxBytes: mb, Replicas: replica})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S2\", Subjects: []string{\"s2\"}, MaxBytes: 1024, Replicas: replica})\n\t\trequire_Error(t, err, errResources)\n\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"S1\", Subjects: []string{\"s1\"}, MaxBytes: mb / 2, Replicas: replica})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S2\", Subjects: []string{\"s2\"}, MaxBytes: mb / 2, Replicas: replica})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S3\", Subjects: []string{\"s3\"}, MaxBytes: 1024, Replicas: replica})\n\t\trequire_Error(t, err, errResources)\n\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"S2\", Subjects: []string{\"s2\"}, MaxBytes: mb/2 + 1, Replicas: replica})\n\t\trequire_Error(t, err, errResources)\n\n\t\trequire_NoError(t, js.DeleteStream(\"S1\"))\n\t\trequire_NoError(t, js.DeleteStream(\"S2\"))\n\t}\n\ttest(t, 3)\n\ttest(t, 1)\n}\n\nfunc TestJetStreamClusterConcurrentAccountLimits(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\tvar swg sync.WaitGroup\n\tfailCount := int32(0)\n\n\tstart := func(name string) {\n\t\twg.Add(1)\n\t\tdefer wg.Done()\n\n\t\ts := c.randomServer()\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tswg.Done()\n\t\t<-startCh\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tReplicas: 3,\n\t\t\tMaxBytes: 1024 * 1024 * 1024,\n\t\t})\n\t\tif err != nil {\n\t\t\tatomic.AddInt32(&failCount, 1)\n\t\t\trequire_Equal(t, err.Error(), \"nats: insufficient storage resources available\")\n\t\t}\n\t}\n\n\tswg.Add(2)\n\tgo start(\"foo\")\n\tgo start(\"bar\")\n\tswg.Wait()\n\t// Now start both at same time.\n\tclose(startCh)\n\twg.Wait()\n\trequire_True(t, failCount == 1)\n}\n\nfunc TestJetStreamClusterBalancedPlacement(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesTempl, \"CB\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// We have 10GB (2GB X 5) available.\n\t// Use MaxBytes for ease of test (used works too) and place 5 1GB streams with R=2.\n\tfor i := 1; i <= 5; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"S-%d\", i),\n\t\t\tReplicas: 2,\n\t\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\t// Make sure the next one fails properly.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"FAIL\",\n\t\tReplicas: 2,\n\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t})\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"insufficient storage\")\n}\n\nfunc TestJetStreamClusterConsumerPendingBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tnc2, js2 := jsClientConnect(t, c.randomServer())\n\tdefer nc2.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\tstartCh, doneCh := make(chan bool), make(chan error)\n\tgo func() {\n\t\t<-startCh\n\t\t_, err := js2.AddConsumer(\"foo\", &nats.ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tFilterSubject: \"foo\",\n\t\t\tDeliverSubject: \"x\",\n\t\t})\n\t\tdoneCh <- err\n\t}()\n\n\tn := 10_000\n\tfor i := 0; i < n; i++ {\n\t\tnc.Publish(\"foo\", []byte(\"ok\"))\n\t\tif i == 222 {\n\t\t\tstartCh <- true\n\t\t}\n\t}\n\t// Wait for them to all be there.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"foo\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != uint64(n) {\n\t\t\treturn fmt.Errorf(\"Not received all messages\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tselect {\n\tcase err := <-doneCh:\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t\t}\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Timed out?\")\n\t}\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"foo\", \"dlc\")\n\t\trequire_NoError(t, err)\n\t\tif ci.NumPending != uint64(n) {\n\t\t\treturn fmt.Errorf(\"Expected NumPending to be %d, got %d\", n, ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPullPerf(t *testing.T) {\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjs.AddStream(&nats.StreamConfig{Name: \"f22\"})\n\tdefer js.DeleteStream(\"f22\")\n\n\tn, msg := 1_000_000, []byte(strings.Repeat(\"A\", 1000))\n\tfor i := 0; i < n; i++ {\n\t\tjs.PublishAsync(\"f22\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsi, err := js.StreamInfo(\"f22\")\n\trequire_NoError(t, err)\n\n\tfmt.Printf(\"msgs: %d, total_bytes: %v\\n\", si.State.Msgs, friendlyBytes(int64(si.State.Bytes)))\n\n\t// OrderedConsumer - fastest push based.\n\tstart := time.Now()\n\treceived, done := 0, make(chan bool)\n\t_, err = js.Subscribe(\"f22\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= n {\n\t\t\tdone <- true\n\t\t}\n\t}, nats.OrderedConsumer())\n\trequire_NoError(t, err)\n\n\t// Wait to receive all messages.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(30 * time.Second):\n\t\tt.Fatalf(\"Did not receive all of our messages\")\n\t}\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n\n\t// Now do pull based, this is custom for now.\n\t// Current nats.PullSubscribe maxes at about 1/2 the performance even with large batches.\n\t_, err = js.AddConsumer(\"f22\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckAllPolicy,\n\t\tMaxAckPending: 1000,\n\t})\n\trequire_NoError(t, err)\n\n\tr := 0\n\t_, err = nc.Subscribe(\"xx\", func(m *nats.Msg) {\n\t\tr++\n\t\tif r >= n {\n\t\t\tdone <- true\n\t\t}\n\t\tif r%750 == 0 {\n\t\t\tm.AckSync()\n\t\t}\n\t})\n\trequire_NoError(t, err)\n\n\t// Simulate an non-ending request.\n\treq := &JSApiConsumerGetNextRequest{Batch: n, Expires: 60 * time.Second}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\n\tstart = time.Now()\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"f22\", \"dlc\")\n\terr = nc.PublishRequest(rsubj, \"xx\", jreq)\n\trequire_NoError(t, err)\n\n\t// Wait to receive all messages.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(60 * time.Second):\n\t\tt.Fatalf(\"Did not receive all of our messages\")\n\t}\n\n\ttt = time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n}\n\n// Test that we get the right signaling when a consumer leader change occurs for any pending requests.\nfunc TestJetStreamClusterPullConsumerLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\tsub, err := nc.SubscribeSync(\"reply\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tdrainSub := func() {\n\t\tt.Helper()\n\t\tfor _, err := sub.NextMsg(0); err == nil; _, err = sub.NextMsg(0) {\n\t\t}\n\t\tcheckSubsPending(t, sub, 0)\n\t}\n\n\t// Queue up a request that can live for a bit.\n\treq := &JSApiConsumerGetNextRequest{Expires: 2 * time.Second}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\terr = nc.PublishRequest(rsubj, \"reply\", jreq)\n\trequire_NoError(t, err)\n\t// Make sure request is recorded and replicated.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckSubsPending(t, sub, 0)\n\n\t// Now have consumer leader change and make sure we get signaled that our request is not valid.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tcheckSubsPending(t, sub, 1)\n\tm, err := sub.NextMsg(0)\n\trequire_NoError(t, err)\n\t// Make sure this is an alert that tells us our request is no longer valid.\n\tif m.Header.Get(\"Status\") != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", m.Header.Get(\"Status\"))\n\t}\n\tcheckSubsPending(t, sub, 0)\n\n\t// Add a few messages to the stream to fulfill a request.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO\"))\n\t\trequire_NoError(t, err)\n\t}\n\treq = &JSApiConsumerGetNextRequest{Batch: 10, Expires: 10 * time.Second}\n\tjreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\terr = nc.PublishRequest(rsubj, \"reply\", jreq)\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, 10)\n\tdrainSub()\n\n\t// Now do a leader change again, make sure we do not get anything about that request.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckSubsPending(t, sub, 0)\n\n\t// Make sure we do not get anything if we expire, etc.\n\treq = &JSApiConsumerGetNextRequest{Batch: 10, Expires: 250 * time.Millisecond}\n\tjreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\terr = nc.PublishRequest(rsubj, \"reply\", jreq)\n\trequire_NoError(t, err)\n\t// Let it expire.\n\ttime.Sleep(350 * time.Millisecond)\n\tcheckSubsPending(t, sub, 1)\n\n\t// Now do a leader change again, make sure we do not get anything about that request.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tcheckSubsPending(t, sub, 1)\n}\n\nfunc TestJetStreamClusterEphemeralPullConsumerServerShutdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 2,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", ci.Name)\n\tsub, err := nc.SubscribeSync(\"reply\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Queue up a request that can live for a bit.\n\treq := &JSApiConsumerGetNextRequest{Expires: 2 * time.Second}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\terr = nc.PublishRequest(rsubj, \"reply\", jreq)\n\trequire_NoError(t, err)\n\t// Make sure request is recorded and replicated.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckSubsPending(t, sub, 0)\n\n\t// Now shutdown the server where this ephemeral lives.\n\tc.consumerLeader(\"$G\", \"TEST\", ci.Name).Shutdown()\n\tcheckSubsPending(t, sub, 1)\n\tm, err := sub.NextMsg(0)\n\trequire_NoError(t, err)\n\t// Make sure this is an alert that tells us our request is no longer valid.\n\tif m.Header.Get(\"Status\") != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", m.Header.Get(\"Status\"))\n\t}\n}\n\nfunc TestJetStreamClusterNAKBackoffs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 2,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"NAK\"))\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"dlc\"), nats.AckWait(5*time.Second), nats.ManualAck())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tcheckSubsPending(t, sub, 1)\n\tm, err := sub.NextMsg(0)\n\trequire_NoError(t, err)\n\n\t// Default nak will redeliver almost immediately.\n\t// We can now add a parse duration string after whitespace to the NAK proto.\n\tstart := time.Now()\n\tdnak := []byte(fmt.Sprintf(\"%s 200ms\", AckNak))\n\tm.Respond(dnak)\n\tcheckSubsPending(t, sub, 1)\n\telapsed := time.Since(start)\n\tif elapsed < 200*time.Millisecond {\n\t\tt.Fatalf(\"Took too short to redeliver, expected ~200ms but got %v\", elapsed)\n\t}\n\tif elapsed > time.Second {\n\t\tt.Fatalf(\"Took too long to redeliver, expected ~200ms but got %v\", elapsed)\n\t}\n\n\t// Now let's delay and make sure that is honored when a new consumer leader takes over.\n\tm, err = sub.NextMsg(0)\n\trequire_NoError(t, err)\n\tdnak = []byte(fmt.Sprintf(\"%s 1s\", AckNak))\n\tstart = time.Now()\n\tm.Respond(dnak)\n\t// Wait for NAK state to propagate.\n\ttime.Sleep(100 * time.Millisecond)\n\t// Ask leader to stepdown.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tcheckSubsPending(t, sub, 1)\n\telapsed = time.Since(start)\n\tif elapsed < time.Second {\n\t\tt.Fatalf(\"Took too short to redeliver, expected ~1s but got %v\", elapsed)\n\t}\n\tif elapsed > 3*time.Second {\n\t\tt.Fatalf(\"Took too long to redeliver, expected ~1s but got %v\", elapsed)\n\t}\n\n\t// Test json version.\n\tdelay, err := json.Marshal(&ConsumerNakOptions{Delay: 20 * time.Millisecond})\n\trequire_NoError(t, err)\n\tdnak = []byte(fmt.Sprintf(\"%s %s\", AckNak, delay))\n\tm, err = sub.NextMsg(0)\n\trequire_NoError(t, err)\n\tstart = time.Now()\n\tm.Respond(dnak)\n\tcheckSubsPending(t, sub, 1)\n\telapsed = time.Since(start)\n\tif elapsed < 20*time.Millisecond {\n\t\tt.Fatalf(\"Took too short to redeliver, expected ~20ms but got %v\", elapsed)\n\t}\n\tif elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Took too long to redeliver, expected ~20ms but got %v\", elapsed)\n\t}\n}\n\nfunc TestJetStreamClusterRedeliverBackoffs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 2,\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Produce some messages on bar so that when we create the consumer\n\t// on \"foo\", we don't have a 1:1 between consumer/stream sequence.\n\tfor i := 0; i < 10; i++ {\n\t\tjs.Publish(\"bar\", []byte(\"msg\"))\n\t}\n\n\t// Test when BackOff is configured and AckWait and MaxDeliver are as well.\n\t// Currently the BackOff will override AckWait, but we want MaxDeliver to be set to be at least len(BackOff)+1.\n\tccReq := &CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tFilterSubject: \"foo\",\n\t\t\tDeliverSubject: \"x\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tAckWait: 30 * time.Second,\n\t\t\tMaxDeliver: 2,\n\t\t\tBackOff: []time.Duration{25 * time.Millisecond, 100 * time.Millisecond, 250 * time.Millisecond},\n\t\t},\n\t}\n\treq, err := json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"dlc\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error == nil || ccResp.Error.ErrCode != 10116 {\n\t\tt.Fatalf(\"Expected an error when MaxDeliver is <= len(BackOff), got %+v\", ccResp.Error)\n\t}\n\n\t// Set MaxDeliver to 6.\n\tccReq.Config.MaxDeliver = 6\n\treq, err = json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"dlc\"), req, time.Second)\n\trequire_NoError(t, err)\n\tccResp.Error = nil\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", ccResp.Error)\n\t}\n\tif cfg := ccResp.ConsumerInfo.Config; cfg.AckWait != 25*time.Millisecond || cfg.MaxDeliver != 6 {\n\t\tt.Fatalf(\"Expected AckWait to be first BackOff (25ms) and MaxDeliver set to 6, got %+v\", cfg)\n\t}\n\n\tvar received []time.Time\n\tvar mu sync.Mutex\n\n\tsub, err := nc.Subscribe(\"x\", func(m *nats.Msg) {\n\t\tmu.Lock()\n\t\treceived = append(received, time.Now())\n\t\tmu.Unlock()\n\t})\n\trequire_NoError(t, err)\n\n\t// Send a message.\n\tstart := time.Now()\n\t_, err = js.Publish(\"foo\", []byte(\"m22\"))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\tmu.Lock()\n\t\tnr := len(received)\n\t\tmu.Unlock()\n\t\tif nr >= 6 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Only seen %d of 6\", nr)\n\t})\n\tsub.Unsubscribe()\n\n\texpected := ccReq.Config.BackOff\n\t// We expect the MaxDeliver to go until 6, so fill in two additional ones.\n\texpected = append(expected, 250*time.Millisecond, 250*time.Millisecond)\n\tfor i, tr := range received[1:] {\n\t\td := tr.Sub(start)\n\t\t// Adjust start for next calcs.\n\t\tstart = start.Add(d)\n\t\tif d < expected[i]-5*time.Millisecond || d > expected[i]*2+5*time.Millisecond {\n\t\t\tt.Fatalf(\"Timing is off for %d, expected ~%v, but got %v\", i, expected[i], d)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterConsumerUpgrade(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\ttestUpdate := func(t *testing.T, s *Server) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"X\"})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"X\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t\t// First create a consumer that is push based.\n\t\t_, err = js.AddConsumer(\"X\", &nats.ConsumerConfig{Durable: \"dlc\", DeliverSubject: \"Y\"})\n\t\trequire_NoError(t, err)\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { testUpdate(t, s) })\n\tt.Run(\"Clustered\", func(t *testing.T) { testUpdate(t, c.randomServer()) })\n}\n\nfunc TestJetStreamClusterAddConsumerWithInfo(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\ttestConsInfo := func(t *testing.T, s *Server) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tinbox := nats.NewInbox()\n\t\t\tsub := natsSubSync(t, nc, inbox)\n\n\t\t\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\t\tDeliverSubject: inbox,\n\t\t\t\tDeliverPolicy: nats.DeliverAllPolicy,\n\t\t\t\tFilterSubject: \"foo\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif ci.NumPending != 10 {\n\t\t\t\tt.Fatalf(\"Iter=%v - expected 10 messages pending on create, got %v\", i+1, ci.NumPending)\n\t\t\t}\n\t\t\tjs.DeleteConsumer(\"TEST\", ci.Name)\n\t\t\tsub.Unsubscribe()\n\t\t}\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { testConsInfo(t, s) })\n\tt.Run(\"Clustered\", func(t *testing.T) { testConsInfo(t, c.randomServer()) })\n}\n\nfunc TestJetStreamClusterStreamReplicaUpdates(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R7S\", 7)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Start out at R1\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tnumMsgs := 1000\n\tfor i := 0; i < numMsgs; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"HELLO WORLD\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tupdateReplicas := func(r int) {\n\t\tt.Helper()\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tleader := si.Cluster.Leader\n\n\t\tcfg.Replicas = r\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(si.Cluster.Replicas) != r-1 {\n\t\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", r-1, len(si.Cluster.Replicas))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Make sure we kept same leader.\n\t\tif si.Cluster.Leader != leader {\n\t\t\tt.Fatalf(\"Leader changed, expected %q got %q\", leader, si.Cluster.Leader)\n\t\t}\n\t\t// Make sure all are current.\n\t\tfor _, r := range si.Cluster.Replicas {\n\t\t\tc.waitOnStreamCurrent(c.serverByName(r.Name), \"$G\", \"TEST\")\n\t\t}\n\t\t// Check msgs.\n\t\tif si.State.Msgs != uint64(numMsgs) {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", numMsgs, si.State.Msgs)\n\t\t}\n\t\t// Make sure we have the right number of HA Assets running on the leader.\n\t\ts := c.serverByName(leader)\n\t\tjsi, err := s.Jsz(nil)\n\t\trequire_NoError(t, err)\n\t\tnha := 1 // meta always present.\n\t\tif len(si.Cluster.Replicas) > 0 {\n\t\t\tnha++\n\t\t}\n\t\tif nha != jsi.HAAssets {\n\t\t\tt.Fatalf(\"Expected %d HA asset(s), but got %d\", nha, jsi.HAAssets)\n\t\t}\n\t}\n\n\t// Update from 1-3\n\tupdateReplicas(3)\n\t// Update from 3-5\n\tupdateReplicas(5)\n\t// Update from 5-3\n\tupdateReplicas(3)\n\t// Update from 3-1\n\tupdateReplicas(1)\n}\n\nfunc TestJetStreamClusterStreamAndConsumerScaleUpAndDown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Start out at R3\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"cat\"))\n\trequire_NoError(t, err)\n\n\tnumMsgs := 10\n\tfor i := 0; i < numMsgs; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\t\trequire_NoError(t, err)\n\t}\n\tcheckSubsPending(t, sub, numMsgs)\n\n\t// Now ask leader to stepdown.\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar sdResp JSApiStreamLeaderStepDownResponse\n\terr = json.Unmarshal(rmsg.Data, &sdResp)\n\trequire_NoError(t, err)\n\n\tif sdResp.Error != nil || !sdResp.Success {\n\t\tt.Fatalf(\"Unexpected error: %+v\", sdResp.Error)\n\t}\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tupdateReplicas := func(r int) {\n\t\tt.Helper()\n\t\tcfg.Replicas = r\n\t\t_, err := js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"cat\")\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"cat\")\n\t\trequire_NoError(t, err)\n\t\tif ci.Cluster.Leader == _EMPTY_ {\n\t\t\tt.Fatalf(\"Expected a consumer leader but got none in consumer info\")\n\t\t}\n\t\tif len(ci.Cluster.Replicas)+1 != r {\n\t\t\tt.Fatalf(\"Expected consumer info to have %d peers, got %d\", r, len(ci.Cluster.Replicas)+1)\n\t\t}\n\t}\n\n\t// Capture leader, we want to make sure when we scale down this does not change.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\n\t// Scale down to 1.\n\tupdateReplicas(1)\n\n\tif sl != c.streamLeader(\"$G\", \"TEST\") {\n\t\tt.Fatalf(\"Expected same leader, but it changed\")\n\t}\n\n\t// Make sure we can still send to the stream.\n\tfor i := 0; i < numMsgs; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif si.State.Msgs != uint64(2*numMsgs) {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", 3*numMsgs, si.State.Msgs)\n\t}\n\n\tcheckSubsPending(t, sub, 2*numMsgs)\n\n\t// Now back up.\n\tupdateReplicas(3)\n\n\t// Send more.\n\tfor i := 0; i < numMsgs; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO WORLD\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif si.State.Msgs != uint64(3*numMsgs) {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", 3*numMsgs, si.State.Msgs)\n\t}\n\n\tcheckSubsPending(t, sub, 3*numMsgs)\n\n\t// Now check each individual stream on each server to make sure replication occurred.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(3*numMsgs) || state.FirstSeq != 1 || state.LastSeq != 30 || state.Bytes != 1320 {\n\t\t\t\treturn fmt.Errorf(\"Wrong state: %+v for server: %v\", state, s)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterInterestRetentionWithFilteredConsumersExtra(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsubjectNameZero := \"foo.bar\"\n\tsubjectNameOne := \"foo.baz\"\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Retention: nats.InterestPolicy, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tcheckState := func(expected uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif si.State.Msgs != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", expected, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsubZero, err := js.PullSubscribe(subjectNameZero, \"dlc-0\")\n\trequire_NoError(t, err)\n\n\tsubOne, err := js.PullSubscribe(subjectNameOne, \"dlc-1\")\n\trequire_NoError(t, err)\n\n\tmsg := []byte(\"FILTERED\")\n\t// Now send a bunch of messages\n\tfor i := 0; i < 1000; i++ {\n\t\t_, err = js.PublishAsync(subjectNameZero, msg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.PublishAsync(subjectNameOne, msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// should be 2000 in total\n\tcheckState(2000)\n\n\t// fetch and acknowledge, count records to ensure no errors acknowledging\n\tgetAndAckBatch := func(sub *nats.Subscription) {\n\t\tt.Helper()\n\t\tsuccessCounter := 0\n\t\tmsgs, err := sub.Fetch(1000)\n\t\trequire_NoError(t, err)\n\n\t\tfor _, m := range msgs {\n\t\t\terr = m.AckSync()\n\t\t\trequire_NoError(t, err)\n\t\t\tsuccessCounter++\n\t\t}\n\t\tif successCounter != 1000 {\n\t\t\tt.Fatalf(\"Unexpected number of acknowledges %d for subscription %v\", successCounter, sub)\n\t\t}\n\t}\n\n\t// fetch records subscription zero\n\tgetAndAckBatch(subZero)\n\t// fetch records for subscription one\n\tgetAndAckBatch(subOne)\n\t// Make sure stream is zero.\n\tcheckState(0)\n}\n\nfunc TestJetStreamClusterStreamConsumersCount(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsname := \"TEST_STREAM_CONS_COUNT\"\n\t_, err := js.AddStream(&nats.StreamConfig{Name: sname, Subjects: []string{\"foo\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\t// Create some R1 consumers\n\tfor i := 0; i < 10; i++ {\n\t\tinbox := nats.NewInbox()\n\t\tnatsSubSync(t, nc, inbox)\n\t\t_, err = js.AddConsumer(sname, &nats.ConsumerConfig{DeliverSubject: inbox})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now check that the consumer count in stream info/list is 10\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t// Check stream info\n\t\tsi, err := js.StreamInfo(sname)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Error getting stream info: %v\", err)\n\t\t}\n\t\tif n := si.State.Consumers; n != 10 {\n\t\t\treturn fmt.Errorf(\"From StreamInfo, expecting 10 consumers, got %v\", n)\n\t\t}\n\n\t\t// Now from stream list\n\t\tfor si := range js.StreamsInfo() {\n\t\t\tif n := si.State.Consumers; n != 10 {\n\t\t\t\treturn fmt.Errorf(\"From StreamsInfo, expecting 10 consumers, got %v\", n)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterFilteredAndIdleConsumerNRGGrowth(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsname := \"TEST\"\n\t_, err := js.AddStream(&nats.StreamConfig{Name: sname, Subjects: []string{\"foo.*\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo.baz\", nats.Durable(\"dlc\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10_000; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, 0)\n\n\t// Grab consumer's underlying info and make sure NRG log not running away do to no-op skips on filtered consumer.\n\t// Need a non-leader for the consumer, they are only ones getting skip ops to keep delivered updated.\n\tcl := c.consumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tvar s *Server\n\tfor _, s = range c.servers {\n\t\tif s != cl {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"dlc\")\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", \"dlc\")\n\t}\n\n\t// compactNumMin from monitorConsumer is 8192 atm.\n\tconst compactNumMin = 8192\n\tif entries, _ := o.raftNode().Size(); entries > compactNumMin {\n\t\tt.Fatalf(\"Expected <= %d entries, got %d\", compactNumMin, entries)\n\t}\n\n\t// Now make the consumer leader stepdown and make sure we have the proper snapshot.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar cdResp JSApiConsumerLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cdResp.Error)\n\t}\n\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n}\n\nfunc TestJetStreamClusterMirrorOrSourceNotActiveReporting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// We would previous calculate a large number if we actually never heard from the peer yet.\n\t// We want to make sure if we have never heard from the other side report -1 as Active.\n\t// It is possible if testing infra is slow that this could be legit, but should be pretty small.\n\tif si.Mirror.Active != -1 && si.Mirror.Active > 10*time.Millisecond {\n\t\tt.Fatalf(\"Expected an Active of -1, but got %v\", si.Mirror.Active)\n\t}\n}\n\nfunc TestJetStreamClusterStreamAdvisories(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tcheckAdv := func(t *testing.T, sub *nats.Subscription, expectedPrefixes ...string) {\n\t\tt.Helper()\n\t\tseen := make([]bool, len(expectedPrefixes))\n\t\tfor i := 0; i < len(expectedPrefixes); i++ {\n\t\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\t\tvar gotOne bool\n\t\t\tfor j, pfx := range expectedPrefixes {\n\t\t\t\tif !seen[j] && strings.HasPrefix(msg.Subject, pfx) {\n\t\t\t\t\tseen[j] = true\n\t\t\t\t\tgotOne = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !gotOne {\n\t\t\t\tt.Fatalf(\"Expected one of prefixes %q, got %q\", expectedPrefixes, msg.Subject)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Used to keep stream names pseudo unique. t.Name() has slashes in it which caused problems.\n\tvar testN int\n\n\tcheckAdvisories := func(t *testing.T, s *Server, replicas int) {\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\ttestN++\n\t\tstreamName := \"TEST_ADVISORIES_\" + fmt.Sprintf(\"%d\", testN)\n\n\t\tsub := natsSubSync(t, nc, \"$JS.EVENT.ADVISORY.STREAM.*.\"+streamName)\n\n\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: streamName,\n\t\t\tStorage: nats.FileStorage,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tadvisories := []string{JSAdvisoryStreamCreatedPre}\n\t\tif replicas > 1 {\n\t\t\tadvisories = append(advisories, JSAdvisoryStreamLeaderElectedPre)\n\t\t}\n\t\tcheckAdv(t, sub, advisories...)\n\n\t\tsi.Config.MaxMsgs = 1000\n\t\t_, err = js.UpdateStream(&si.Config)\n\t\trequire_NoError(t, err)\n\t\tcheckAdv(t, sub, JSAdvisoryStreamUpdatedPre)\n\n\t\tsnapreq := &JSApiStreamSnapshotRequest{\n\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\tChunkSize: 512,\n\t\t}\n\t\tvar snapshot []byte\n\t\tdone := make(chan bool)\n\t\tnc.Subscribe(snapreq.DeliverSubject, func(m *nats.Msg) {\n\t\t\t// EOF\n\t\t\tif len(m.Data) == 0 {\n\t\t\t\tdone <- true\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Could be writing to a file here too.\n\t\t\tsnapshot = append(snapshot, m.Data...)\n\t\t\t// Flow ack\n\t\t\tm.Respond(nil)\n\t\t})\n\n\t\treq, _ := json.Marshal(snapreq)\n\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, streamName), req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t\t}\n\n\t\tvar snapresp JSApiStreamSnapshotResponse\n\t\tjson.Unmarshal(rmsg.Data, &snapresp)\n\t\tif snapresp.Error != nil {\n\t\t\tt.Fatalf(\"Did not get correct error response: %+v\", snapresp.Error)\n\t\t}\n\n\t\t// Wait to receive the snapshot.\n\t\tselect {\n\t\tcase <-done:\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t\t}\n\n\t\tcheckAdv(t, sub, JSAdvisoryStreamSnapshotCreatePre)\n\t\tcheckAdv(t, sub, JSAdvisoryStreamSnapshotCompletePre)\n\n\t\terr = js.DeleteStream(streamName)\n\t\trequire_NoError(t, err)\n\t\tcheckAdv(t, sub, JSAdvisoryStreamDeletedPre)\n\n\t\tstate := *snapresp.State\n\t\tconfig := *snapresp.Config\n\t\tresreq := &JSApiStreamRestoreRequest{\n\t\t\tConfig: config,\n\t\t\tState: state,\n\t\t}\n\t\treq, _ = json.Marshal(resreq)\n\t\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, streamName), req, 5*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar resresp JSApiStreamRestoreResponse\n\t\tjson.Unmarshal(rmsg.Data, &resresp)\n\t\tif resresp.Error != nil {\n\t\t\tt.Fatalf(\"Got an unexpected error response: %+v\", resresp.Error)\n\t\t}\n\n\t\t// Send our snapshot back in to restore the stream.\n\t\t// Can be any size message.\n\t\tvar chunk [1024]byte\n\t\tfor r := bytes.NewReader(snapshot); ; {\n\t\t\tn, err := r.Read(chunk[:])\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tnc.Request(resresp.DeliverSubject, chunk[:n], time.Second)\n\t\t}\n\t\trmsg, err = nc.Request(resresp.DeliverSubject, nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tresresp.Error = nil\n\t\tjson.Unmarshal(rmsg.Data, &resresp)\n\t\tif resresp.Error != nil {\n\t\t\tt.Fatalf(\"Got an unexpected error response: %+v\", resresp.Error)\n\t\t}\n\n\t\tcheckAdv(t, sub, JSAdvisoryStreamRestoreCreatePre)\n\t\t// At this point, the stream_created advisory may be sent before\n\t\t// or after the restore_complete advisory because they are sent\n\t\t// using different \"send queues\". That is, the restore uses the\n\t\t// server's event queue while the stream_created is sent from\n\t\t// the stream's own send queue.\n\t\tadvisories = append(advisories, JSAdvisoryStreamRestoreCompletePre)\n\t\tcheckAdv(t, sub, advisories...)\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) { checkAdvisories(t, s, 1) })\n\tt.Run(\"Clustered_R1\", func(t *testing.T) { checkAdvisories(t, c.randomServer(), 1) })\n\tt.Run(\"Clustered_R3\", func(t *testing.T) { checkAdvisories(t, c.randomServer(), 3) })\n}\n\n// If the config files have duplicate routes this can have the metagroup estimate a size for the system\n// which prevents reaching quorum and electing a meta-leader.\nfunc TestJetStreamClusterDuplicateRoutesDisruptJetStreamMetaGroup(t *testing.T) {\n\ttmpl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: RR\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [\n\t\t\tnats-route://127.0.0.1:%d\n\t\t\tnats-route://127.0.0.1:%d\n\t\t\tnats-route://127.0.0.1:%d\n\t\t\t# These will be dupes\n\t\t\tnats-route://127.0.0.1:%d\n\t\t\tnats-route://127.0.0.1:%d\n\t\t\tnats-route://127.0.0.1:%d\n\t\t]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\n\tc := &cluster{servers: make([]*Server, 0, 3), opts: make([]*Options, 0, 3), name: \"RR\", t: t}\n\n\trports := []int{22208, 22209, 22210}\n\tfor i, p := range rports {\n\t\tsname, sd := fmt.Sprintf(\"S%d\", i+1), t.TempDir()\n\t\tcf := fmt.Sprintf(tmpl, sname, sd, p, rports[0], rports[1], rports[2], rports[0], rports[1], rports[2])\n\t\ts, o := RunServerWithConfig(createConfFile(t, []byte(cf)))\n\t\tc.servers, c.opts = append(c.servers, s), append(c.opts, o)\n\t}\n\tdefer c.shutdown()\n\n\tcheckClusterFormed(t, c.servers...)\n\tc.waitOnClusterReady()\n}\n\nfunc TestJetStreamClusterDuplicateMsgIdsOnCatchupAndLeaderTakeover(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Shutdown a non-leader.\n\tnc.Close()\n\tsr := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tsr.Shutdown()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tm := nats.NewMsg(\"TEST\")\n\tm.Data = []byte(\"OK\")\n\n\tn := 10\n\tfor i := 0; i < n; i++ {\n\t\tm.Header.Set(JSMsgId, strconv.Itoa(i))\n\t\t_, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t}\n\n\tm.Header.Set(JSMsgId, \"8\")\n\tpa, err := js.PublishMsg(m)\n\trequire_NoError(t, err)\n\tif !pa.Duplicate {\n\t\tt.Fatalf(\"Expected msg to be a duplicate\")\n\t}\n\n\t// Now force a snapshot, want to test catchup above RAFT layer.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tif node := mset.raftNode(); node == nil {\n\t\tt.Fatalf(\"Could not get stream group name\")\n\t} else if err := node.InstallSnapshot(mset.stateSnapshot(), false); err != nil {\n\t\tt.Fatalf(\"Error installing snapshot: %v\", err)\n\t}\n\n\t// Now restart\n\tsr = c.restartServer(sr)\n\tc.waitOnStreamCurrent(sr, \"$G\", \"TEST\")\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Now make them the leader.\n\tfor sr != c.streamLeader(\"$G\", \"TEST\") {\n\t\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t}\n\n\t// Make sure this gets rejected.\n\tpa, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\tif !pa.Duplicate {\n\t\tt.Fatalf(\"Expected msg to be a duplicate\")\n\t}\n}\n\nfunc TestJetStreamClusterConsumerLeaderChangeDeadlock(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create a stream and durable with ack explicit\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"test\", Subjects: []string{\"foo\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\tDurable: \"test\",\n\t\tDeliverSubject: \"bar\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: 250 * time.Millisecond,\n\t})\n\trequire_NoError(t, err)\n\n\t// Wait for a leader\n\tc.waitOnConsumerLeader(\"$G\", \"test\", \"test\")\n\tcl := c.consumerLeader(\"$G\", \"test\", \"test\")\n\n\t// Publish a message\n\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\trequire_NoError(t, err)\n\n\t// Create nats consumer on \"bar\" and don't ack it\n\tsub := natsSubSync(t, nc, \"bar\")\n\tnatsNexMsg(t, sub, time.Second)\n\t// Wait for redeliveries, to make sure it is in the redelivery map\n\tnatsNexMsg(t, sub, time.Second)\n\tnatsNexMsg(t, sub, time.Second)\n\n\tmset, err := cl.GlobalAccount().lookupStream(\"test\")\n\trequire_NoError(t, err)\n\trequire_True(t, mset != nil)\n\n\t// There are parts in the code (for instance when signaling to consumers\n\t// that there are new messages) where we get the mset lock and iterate\n\t// over the consumers and get consumer lock. We are going to do that\n\t// in a go routine while we send a consumer step down request from\n\t// another go routine. We will watch for possible deadlock and if\n\t// found report it.\n\tch := make(chan struct{})\n\tdoneCh := make(chan struct{})\n\tgo func() {\n\t\tdefer close(doneCh)\n\t\tfor {\n\t\t\tmset.mu.Lock()\n\t\t\tfor _, o := range mset.consumers {\n\t\t\t\to.mu.Lock()\n\t\t\t\ttime.Sleep(time.Duration(rand.Intn(10)) * time.Millisecond)\n\t\t\t\to.mu.Unlock()\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Now cause a leader changes\n\tfor i := 0; i < 5; i++ {\n\t\tm, err := nc.Request(\"$JS.API.CONSUMER.LEADER.STEPDOWN.test.test\", nil, 2*time.Second)\n\t\t// Ignore error here and check for deadlock below\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\t// if there is a message, check that it is success\n\t\tvar resp JSApiConsumerLeaderStepDownResponse\n\t\terr = json.Unmarshal(m.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, resp.Success)\n\t\tc.waitOnConsumerLeader(\"$G\", \"test\", \"test\")\n\t}\n\n\tclose(ch)\n\tselect {\n\tcase <-doneCh:\n\t\t// OK!\n\tcase <-time.After(2 * time.Second):\n\t\tbuf := make([]byte, 1000000)\n\t\tn := runtime.Stack(buf, true)\n\t\tt.Fatalf(\"Suspected deadlock, printing current stack. The test suite may timeout and will also dump the stack\\n%s\\n\", buf[:n])\n\t}\n}\n\n// We were compacting to keep the raft log manageable but not snapshotting, which meant that restarted\n// servers could complain about no snapshot and could not sync after that condition.\n// Changes also address https://github.com/nats-io/nats-server/issues/2936\nfunc TestJetStreamClusterMemoryConsumerCompactVsSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create a stream and durable with ack explicit\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tStorage: nats.MemoryStorage,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Bring a non-leader down.\n\ts := c.randomNonConsumerLeader(\"$G\", \"test\", \"d\")\n\ts.Shutdown()\n\n\t// In case that was also mete or stream leader.\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"test\")\n\t// In case we were connected there.\n\tnc.Close()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Generate some state.\n\tfor i := 0; i < 2000; i++ {\n\t\t_, err := js.PublishAsync(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsub, err := js.PullSubscribe(\"test\", \"d\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 2; i++ {\n\t\tfor _, m := range fetchMsgs(t, sub, 1000, 5*time.Second) {\n\t\t\tm.AckSync()\n\t\t}\n\t}\n\n\t// Restart our downed server.\n\ts = c.restartServer(s)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(s)\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"test\", \"d\")\n\t\trequire_NoError(t, err)\n\t\tfor _, r := range ci.Cluster.Replicas {\n\t\t\tif !r.Current || r.Lag != 0 {\n\t\t\t\treturn fmt.Errorf(\"Replica not current: %+v\", r)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMemoryConsumerInterestRetention(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tStorage: nats.MemoryStorage,\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"test\", nats.Durable(\"dlc\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_, err := js.PublishAsync(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\ttoAck := 100\n\tfor i := 0; i < toAck; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tm.AckSync()\n\t}\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"test\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif n := si.State.Msgs; n != 900 {\n\t\t\treturn fmt.Errorf(\"Waiting for msgs count to be 900, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsi, err := js.StreamInfo(\"test\")\n\trequire_NoError(t, err)\n\n\tci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\t// Make sure acks are not only replicated but processed in a way to remove messages from the replica streams.\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"test\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(\"$G\", \"test\")\n\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"test\", \"dlc\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(\"$G\", \"test\", \"dlc\")\n\n\tnsi, err := js.StreamInfo(\"test\")\n\trequire_NoError(t, err)\n\tif !reflect.DeepEqual(nsi.State, si.State) {\n\t\tt.Fatalf(\"Stream states do not match: %+v vs %+v\", si.State, nsi.State)\n\t}\n\n\tnci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\t// Last may be skewed a very small amount.\n\tci.AckFloor.Last, nci.AckFloor.Last = nil, nil\n\tif nci.AckFloor != ci.AckFloor {\n\t\tt.Fatalf(\"Consumer AckFloors are not the same: %+v vs %+v\", ci.AckFloor, nci.AckFloor)\n\t}\n}\n\nfunc TestJetStreamClusterDeleteAndRestoreAndRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"TEST\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"dlc\"))\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 22; i++ {\n\t\t_, err := js.Publish(\"TEST\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.SubscribeSync(\"TEST\", nats.Durable(\"dlc\"), nats.Description(\"SECOND\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 5; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tm.AckSync()\n\t}\n\n\t// Now restart.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tsl.Shutdown()\n\tsl = c.restartServer(sl)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 22 {\n\t\t\treturn fmt.Errorf(\"State is not correct after restart, expected 22 msgs, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\tif ci.AckFloor.Consumer != 5 {\n\t\tt.Fatalf(\"Bad ack floor: %+v\", ci.AckFloor)\n\t}\n\n\t// Now delete and make sure consumer does not come back.\n\t// First add a delete something else.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST2\"})\n\trequire_NoError(t, err)\n\trequire_NoError(t, js.DeleteStream(\"TEST2\"))\n\t// Now the consumer.\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"dlc\"))\n\n\tsl.Shutdown()\n\tc.restartServer(sl)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// In rare circumstances this could be recovered and then quickly deleted.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif _, err := js.ConsumerInfo(\"TEST\", \"dlc\"); err == nil {\n\t\t\treturn fmt.Errorf(\"Not cleaned up yet\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMirrorSourceLoop(t *testing.T) {\n\ttest := func(t *testing.T, s *Server, replicas int) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\t// Create a source/mirror loop\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"1\",\n\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\tReplicas: replicas,\n\t\t\tSources: []*nats.StreamSource{{Name: \"DECOY\"}, {Name: \"2\"}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"DECOY\",\n\t\t\tSubjects: []string{\"baz\"},\n\t\t\tReplicas: replicas,\n\t\t\tSources: []*nats.StreamSource{{Name: \"NOTTHERE\"}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"2\",\n\t\t\tReplicas: replicas,\n\t\t\tSources: []*nats.StreamSource{{Name: \"3\"}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"3\",\n\t\t\tReplicas: replicas,\n\t\t\tSources: []*nats.StreamSource{{Name: \"1\"}},\n\t\t})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: detected cycle\")\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\t\ttest(t, s, 1)\n\t})\n\tt.Run(\"Clustered\", func(t *testing.T) {\n\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 5)\n\t\tdefer c.shutdown()\n\t\ttest(t, c.randomServer(), 2)\n\t})\n}\n\nfunc TestJetStreamClusterMirrorDeDupWindow(t *testing.T) {\n\towt := srcConsumerWaitTime\n\tsrcConsumerWaitTime = 2 * time.Second\n\tdefer func() { srcConsumerWaitTime = owt }()\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Cluster != nil)\n\trequire_True(t, si.Config.Replicas == 3)\n\trequire_True(t, len(si.Cluster.Replicas) == 2)\n\n\tsend := func(count int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < count; i++ {\n\t\t\t_, err := js.Publish(\"foo\", []byte(\"msg\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\t// Send 100 messages\n\tsend(100)\n\n\t// Now create a valid one.\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 3,\n\t\tMirror: &nats.StreamSource{Name: \"S\"},\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Cluster != nil)\n\trequire_True(t, si.Config.Replicas == 3)\n\trequire_True(t, len(si.Cluster.Replicas) == 2)\n\n\tcheck := func(expected int) {\n\t\tt.Helper()\n\t\t// Wait for all messages to be in mirror\n\t\tcheckFor(t, 15*time.Second, 50*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"M\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := si.State.Msgs; int(n) != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %v msgs, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheck(100)\n\n\t// Restart cluster\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"S\")\n\tc.waitOnStreamLeader(globalAccountName, \"M\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsi, err = js.StreamInfo(\"M\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.Cluster != nil)\n\trequire_True(t, si.Config.Replicas == 3)\n\trequire_True(t, len(si.Cluster.Replicas) == 2)\n\n\t// Send 100 messages\n\tsend(100)\n\tcheck(200)\n}\n\nfunc TestJetStreamClusterNewHealthz(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"R1\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"R3\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create subscribers (durable and ephemeral for each)\n\tfsube, err := js.SubscribeSync(\"foo\")\n\trequire_NoError(t, err)\n\tfsubd, err := js.SubscribeSync(\"foo\", nats.Durable(\"d\"))\n\trequire_NoError(t, err)\n\n\t_, err = js.SubscribeSync(\"bar\")\n\trequire_NoError(t, err)\n\tbsubd, err := js.SubscribeSync(\"bar\", nats.Durable(\"d\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"foo\"))\n\t\trequire_NoError(t, err)\n\t}\n\tcheckSubsPending(t, fsube, 20)\n\tcheckSubsPending(t, fsubd, 20)\n\n\t// Select the server where we know the R1 stream is running.\n\tsl := c.streamLeader(\"$G\", \"R1\")\n\tsl.Shutdown()\n\n\t// Do same on R3 so that sl has to recover some things before healthz should be good.\n\tc.waitOnStreamLeader(\"$G\", \"R3\")\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"bar\", []byte(\"bar\"))\n\t\trequire_NoError(t, err)\n\t}\n\t// Ephemeral is skipped, might have been on the downed server.\n\tcheckSubsPending(t, bsubd, 10)\n\n\tsl = c.restartServer(sl)\n\tc.waitOnServerHealthz(sl)\n}\n\nfunc TestJetStreamClusterConsumerOverrides(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Test replica override.\n\t// Make sure we can not go \"wider\" than the parent stream.\n\tccReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"d\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tReplicas: 5,\n\t\t},\n\t}\n\treq, err := json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\n\tci, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"d\"), req, time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(ci.Data, &resp)\n\trequire_NoError(t, err)\n\n\tif resp.Error == nil || !IsNatsErr(resp.Error, JSConsumerReplicasExceedsStream) {\n\t\tt.Fatalf(\"Expected an error when replicas > parent stream, got %+v\", resp.Error)\n\t}\n\n\t// Durables inherit the replica count from the stream, so make sure we can override that.\n\tccReq.Config.Replicas = 1\n\treq, err = json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\n\tci, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"d\"), req, time.Second)\n\trequire_NoError(t, err)\n\n\tresp.Error = nil\n\terr = json.Unmarshal(ci.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error == nil)\n\n\tcheckCount := func(durable string, expected int) {\n\t\tt.Helper()\n\t\tcount := 0\n\t\tfor _, s := range c.servers {\n\t\t\tif mset, err := s.GlobalAccount().lookupStream(\"TEST\"); err == nil {\n\t\t\t\tif o := mset.lookupConsumer(durable); o != nil {\n\t\t\t\t\tcount++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif count != expected {\n\t\t\tt.Fatalf(\"Expected %d consumers in cluster, got %d\", expected, count)\n\t\t}\n\t}\n\tcheckCount(\"d\", 1)\n\n\t// Now override storage and force storage to memory based.\n\tccReq.Config.MemoryStorage = true\n\tccReq.Config.Durable = \"m\"\n\tccReq.Config.Replicas = 3\n\n\treq, err = json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\n\tci, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"m\"), req, time.Second)\n\trequire_NoError(t, err)\n\n\tresp.Error = nil\n\terr = json.Unmarshal(ci.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error == nil)\n\n\tcheckCount(\"m\", 3)\n\n\t// Make sure memory setting is for both consumer raft log and consumer store.\n\ts := c.consumerLeader(\"$G\", \"TEST\", \"m\")\n\trequire_True(t, s != nil)\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"m\")\n\trequire_True(t, o != nil)\n\n\tst := o.store.Type()\n\tn := o.raftNode()\n\trequire_True(t, n != nil)\n\trn := n.(*raft)\n\trn.RLock()\n\twal := rn.wal\n\trn.RUnlock()\n\trequire_Equal(t, wal.Type(), MemoryStorage)\n\trequire_Equal(t, st, MemoryStorage)\n\n\t// Now make sure we account properly for the consumers.\n\t// Add in normal here first.\n\t_, err = js.SubscribeSync(\"foo\", nats.Durable(\"d22\"))\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Consumers, 3)\n\n\terr = js.DeleteConsumer(\"TEST\", \"d\")\n\trequire_NoError(t, err)\n\n\t// Also make sure the stream leader direct store reports same with mixed and matched.\n\ts = c.streamLeader(\"$G\", \"TEST\")\n\trequire_True(t, s != nil)\n\tmset, err = s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tstate := mset.Store().State()\n\t\tif state.Consumers != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 consumers, got %d\", state.Consumers)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Fast state version as well.\n\tfstate := mset.stateWithDetail(false)\n\trequire_Equal(t, fstate.Consumers, 2)\n\n\t// Make sure delete accounting works too.\n\terr = js.DeleteConsumer(\"TEST\", \"m\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tstate := mset.Store().State()\n\t\tif state.Consumers != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 consumer, got %d\", state.Consumers)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Fast state version as well.\n\tfstate = mset.stateWithDetail(false)\n\trequire_Equal(t, fstate.Consumers, 1)\n}\n\nfunc TestJetStreamClusterStreamRepublish(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"RP\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tReplicas: 3,\n\t\tRePublish: &RePublish{\n\t\t\tSource: \">\",\n\t\t\tDestination: \"RP.>\",\n\t\t},\n\t}\n\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"RP.>\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := []byte(\"OK TO REPUBLISH?\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err = js.PublishAsync(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.PublishAsync(\"bar\", msg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.PublishAsync(\"baz\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, toSend*3)\n\n\tlseq := map[string]int{\n\t\t\"foo\": 0,\n\t\t\"bar\": 0,\n\t\t\"baz\": 0,\n\t}\n\n\tfor i := 1; i <= toSend; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\t// Grab info from Header\n\t\trequire_True(t, m.Header.Get(JSStream) == \"RP\")\n\t\t// Make sure sequence is correct.\n\t\tseq, err := strconv.Atoi(m.Header.Get(JSSequence))\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, seq == i)\n\t\t// Make sure timestamp is correct\n\t\tts, err := time.Parse(time.RFC3339Nano, m.Header.Get(JSTimeStamp))\n\t\trequire_NoError(t, err)\n\t\torigMsg, err := js.GetMsg(\"RP\", uint64(seq))\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ts == origMsg.Time)\n\t\t// Make sure last sequence matches last seq we received on this subject.\n\t\tlast, err := strconv.Atoi(m.Header.Get(JSLastSequence))\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, last == lseq[m.Subject])\n\t\tlseq[m.Subject] = seq\n\t}\n}\n\nfunc TestJetStreamClusterConsumerDeliverNewNotConsumingBeforeStepDownOrRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: inbox,\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\tcheckCount := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := int(ci.NumPending); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %v pending, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckCount(10)\n\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dur\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar cdResp JSApiConsumerLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cdResp.Error)\n\t}\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\tcheckCount(10)\n\n\t// Check also servers restart\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckCount(10)\n\n\t// Make sure messages can be consumed\n\tsub := natsSubSync(t, nc, inbox)\n\tfor i := 0; i < 10; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"i=%v next msg error: %v\", i, err)\n\t\t}\n\t\tmsg.AckSync()\n\t}\n\tcheckCount(0)\n}\n\nfunc TestJetStreamClusterConsumerDeliverNewMaxRedeliveriesAndServerRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: inbox,\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\tMaxDeliver: 3,\n\t\tAckWait: 250 * time.Millisecond,\n\t\tFilterSubject: \"foo.bar\",\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\n\tsendStreamMsg(t, nc, \"foo.bar\", \"msg\")\n\n\tsub := natsSubSync(t, nc, inbox)\n\tfor i := 0; i < 3; i++ {\n\t\tnatsNexMsg(t, sub, time.Second)\n\t}\n\t// Now check that there is no more redeliveries\n\tif msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout, got msg=%+v err=%v\", msg, err)\n\t}\n\n\t// Check server restart\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsub = natsSubSync(t, nc, inbox)\n\t// We should not have messages being redelivered.\n\tif msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout, got msg=%+v err=%v\", msg, err)\n\t}\n}\n\nfunc TestJetStreamClusterNoRestartAdvisories(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create 10 consumers\n\tfor i := 0; i < 10; i++ {\n\t\tdur := fmt.Sprintf(\"dlc-%d\", i)\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: dur, AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t}\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor i := 0; i < 1000; i++ {\n\t\tjs.PublishAsync(\"foo\", msg)\n\t\tjs.PublishAsync(\"bar\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Add state to a consumer.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc-2\")\n\trequire_NoError(t, err)\n\tfor _, m := range fetchMsgs(t, sub, 5, time.Second) {\n\t\tm.AckSync()\n\t}\n\tnc.Close()\n\n\t// Required to show the bug.\n\tc.leader().JetStreamSnapshotMeta()\n\n\tnc, _ = jsClientConnect(t, c.consumerLeader(\"$G\", \"TEST\", \"dlc-2\"))\n\tdefer nc.Close()\n\n\tsub, err = nc.SubscribeSync(\"$JS.EVENT.ADVISORY.API\")\n\trequire_NoError(t, err)\n\n\t// Shutdown and Restart.\n\ts := c.randomNonConsumerLeader(\"$G\", \"TEST\", \"dlc-2\")\n\ts.Shutdown()\n\ts = c.restartServer(s)\n\tc.waitOnServerHealthz(s)\n\n\tcheckSubsPending(t, sub, 0)\n\n\tnc, _ = jsClientConnect(t, c.randomNonStreamLeader(\"$G\", \"TEST\"))\n\tdefer nc.Close()\n\n\tsub, err = nc.SubscribeSync(\"$JS.EVENT.ADVISORY.STREAM.UPDATED.>\")\n\trequire_NoError(t, err)\n\n\ts = c.streamLeader(\"$G\", \"TEST2\")\n\ts.Shutdown()\n\ts = c.restartServer(s)\n\tc.waitOnServerHealthz(s)\n\n\tcheckSubsPending(t, sub, 0)\n}\n\nfunc TestJetStreamClusterR1StreamPlacementNoReservation(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsp := make(map[string]int)\n\tfor i := 0; i < 100; i++ {\n\t\tsname := fmt.Sprintf(\"T-%d\", i)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: sname,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tsp[c.streamLeader(\"$G\", sname).Name()]++\n\t}\n\n\tfor serverName, num := range sp {\n\t\tif num > 60 {\n\t\t\tt.Fatalf(\"Streams not distributed, expected ~30-35 but got %d for server %q\", num, serverName)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterConsumerAndStreamNamesWithPathSeparators(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"usr/bin\"})\n\trequire_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: `Documents\\readme.txt`})\n\trequire_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName)\n\n\t// Now consumers.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"T\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: \"a/b\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: `a\\b`, AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName)\n}\n\nfunc TestJetStreamClusterFilteredMirrors(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MSR\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 3)\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", msg)\n\t\tjs.PublishAsync(\"bar\", msg)\n\t\tjs.PublishAsync(\"baz\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Create Mirror now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\", FilterSubject: \"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 100 {\n\t\t\treturn fmt.Errorf(\"Expected 100 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsub, err := js.PullSubscribe(\"foo\", \"d\", nats.BindStream(\"M\"))\n\trequire_NoError(t, err)\n\n\t// Make sure we only have \"foo\" and that sequence numbers preserved.\n\tsseq, dseq := uint64(1), uint64(1)\n\tfor _, m := range fetchMsgs(t, sub, 100, 5*time.Second) {\n\t\trequire_True(t, m.Subject == \"foo\")\n\t\tmeta, err := m.Metadata()\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, meta.Sequence.Consumer == dseq)\n\t\tdseq++\n\t\trequire_True(t, meta.Sequence.Stream == sseq)\n\t\tsseq += 3\n\t}\n}\n\n// Test for making sure we error on same cluster name.\nfunc TestJetStreamClusterSameClusterLeafNodes(t *testing.T) {\n\tc := createJetStreamCluster(t, jsClusterAccountsTempl, \"SAME\", _EMPTY_, 3, 11233, true)\n\tdefer c.shutdown()\n\n\t// Do by hand since by default we check for connections.\n\ttmpl := c.createLeafSolicit(jsClusterTemplWithLeafNode)\n\tlc := createJetStreamCluster(t, tmpl, \"SAME\", \"S-\", 2, 22111, false)\n\tdefer lc.shutdown()\n\n\ttime.Sleep(200 * time.Millisecond)\n\n\t// Make sure no leafnodes are connected.\n\tfor _, s := range lc.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 0)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/3178\nfunc TestJetStreamClusterLeafNodeSPOFMigrateLeaders(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterTempl, \"store_dir:\", \"domain: REMOTE, store_dir:\", 1)\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"HUB\", 2)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"LNC\", 2, 22110)\n\tdefer lnc.shutdown()\n\n\tlnc.waitOnClusterReady()\n\n\t// Place JS assets in LN, and we will do a pull consumer from the HUB.\n\tnc, js := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Cluster.Name == \"LNC\")\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"HELLO\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Create the consumer.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"d\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tdsubj := \"$JS.CORE.API.CONSUMER.MSG.NEXT.TEST.d\"\n\t// Grab directly using domain based subject but from the HUB cluster.\n\t_, err = nc.Request(dsubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Now we will force the consumer leader's server to drop and stall leafnode connections.\n\tcl := lnc.consumerLeader(\"$G\", \"TEST\", \"d\")\n\tcl.setJetStreamMigrateOnRemoteLeaf()\n\tcl.closeAndDisableLeafnodes()\n\n\t// Now make sure we can eventually get a message again.\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t_, err = nc.Request(dsubj, nil, 500*time.Millisecond)\n\t\treturn err\n\t})\n\n\tnc, _ = jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\t// Now make sure the consumer, or any other asset, can not become a leader on this node while the leafnode\n\t// is disconnected.\n\tcsd := fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"d\")\n\tfor i := 0; i < 10; i++ {\n\t\tnc.Request(csd, nil, time.Second)\n\t\tlnc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"d\")\n\t\tif lnc.consumerLeader(globalAccountName, \"TEST\", \"d\") == cl {\n\t\t\tt.Fatalf(\"Consumer leader should not migrate to server without a leafnode connection\")\n\t\t}\n\t}\n\n\t// Now make sure once leafnode is back we can have leaders on this server.\n\tcl.reEnableLeafnodes()\n\tcheckLeafNodeConnectedCount(t, cl, 2)\n\n\t// Make sure we can migrate back to this server now that we are connected.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tnc.Request(csd, nil, time.Second)\n\t\tlnc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"d\")\n\t\tif lnc.consumerLeader(globalAccountName, \"TEST\", \"d\") == cl {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not this server yet\")\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/3178\nfunc TestJetStreamClusterLeafNodeSPOFMigrateLeadersWithMigrateDelay(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterTempl, \"store_dir:\", \"domain: REMOTE, store_dir:\", 1)\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"HUB\", 2)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"LNC\", 2, 22110)\n\tdefer lnc.shutdown()\n\n\tlnc.waitOnClusterReady()\n\n\t// Place JS assets in LN, and we will do a pull consumer from the HUB.\n\tnc, js := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Cluster.Name == \"LNC\")\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"HELLO\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Create the consumer.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"d\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tdsubj := \"$JS.CORE.API.CONSUMER.MSG.NEXT.TEST.d\"\n\t// Grab directly using domain based subject but from the HUB cluster.\n\t_, err = nc.Request(dsubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Now we will force the consumer leader's server to drop and stall leafnode connections.\n\tcl := lnc.consumerLeader(\"$G\", \"TEST\", \"d\")\n\tcl.setJetStreamMigrateOnRemoteLeafWithDelay(5 * time.Second)\n\tcl.closeAndDisableLeafnodes()\n\n\t// Now make sure we can eventually get a message again.\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t_, err = nc.Request(dsubj, nil, 500*time.Millisecond)\n\t\treturn err\n\t})\n\n\tnc, _ = jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\t// Now make sure the consumer, or any other asset, can not become a leader on this node while the leafnode\n\t// is disconnected.\n\tcsd := fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"d\")\n\tfor i := 0; i < 10; i++ {\n\t\tnc.Request(csd, nil, time.Second)\n\t\tlnc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"d\")\n\t\tif lnc.consumerLeader(globalAccountName, \"TEST\", \"d\") == cl {\n\t\t\tt.Fatalf(\"Consumer leader should not migrate to server without a leafnode connection\")\n\t\t}\n\t}\n\n\t// Now make sure once leafnode is back we can have leaders on this server.\n\tcl.reEnableLeafnodes()\n\tcheckLeafNodeConnectedCount(t, cl, 2)\n\tfor _, ln := range cl.leafRemoteCfgs {\n\t\trequire_True(t, ln.jsMigrateTimer == nil)\n\t}\n\n\t// Make sure we can migrate back to this server now that we are connected.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tnc.Request(csd, nil, time.Second)\n\t\tlnc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"d\")\n\t\tif lnc.consumerLeader(globalAccountName, \"TEST\", \"d\") == cl {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not this server yet\")\n\t})\n}\n\nfunc TestJetStreamClusterStreamCatchupWithTruncateAndPriorSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Shutdown a replica\n\trs := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\trs.Shutdown()\n\tif s == rs {\n\t\tnc.Close()\n\t\ts = c.randomServer()\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t}\n\n\tmsg, toSend := []byte(\"OK\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.PublishAsync(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Force snapshot\n\trequire_NoError(t, mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false))\n\n\t// Now truncate the store on purpose.\n\terr = mset.store.Truncate(50)\n\trequire_NoError(t, err)\n\n\t// Restart Server.\n\trs = c.restartServer(rs)\n\n\t// Make sure we can become current.\n\t// With bug we would fail here.\n\tc.waitOnStreamCurrent(rs, \"$G\", \"TEST\")\n}\n\nfunc TestJetStreamClusterNoOrphanedDueToNoConnection(t *testing.T) {\n\torgEventsHBInterval := eventsHBInterval\n\teventsHBInterval = 500 * time.Millisecond\n\tdefer func() { eventsHBInterval = orgEventsHBInterval }()\n\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckSysServers := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\ts.mu.RLock()\n\t\t\t\tnum := len(s.sys.servers)\n\t\t\t\ts.mu.RUnlock()\n\t\t\t\tif num != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected server %q to have 2 servers, got %v\", s, num)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckSysServers()\n\tnc.Close()\n\n\ts.mu.RLock()\n\tval := (s.sys.orphMax / eventsHBInterval) + 2\n\ts.mu.RUnlock()\n\ttime.Sleep(val * eventsHBInterval)\n\tcheckSysServers()\n}\n\nfunc TestJetStreamClusterStreamResetOnExpirationDuringPeerDownAndRestartWithLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tMaxAge: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tn := 100\n\tfor i := 0; i < n; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"NORESETPLS\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Shutdown a non-leader before expiration.\n\tnsl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tnsl.Shutdown()\n\n\t// Wait for all messages to expire.\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\tdefer cancel()\n\t\tsi, err := js.StreamInfo(\"TEST\", nats.Context(ctx))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Wanted 0 messages, got %d\", si.State.Msgs)\n\t})\n\n\t// Now restart the non-leader server, twice. First time clears raft,\n\t// second will not have any index state or raft to tell it what is first sequence.\n\tnsl = c.restartServer(nsl)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(nsl)\n\n\t// Now clear raft WAL.\n\tmset, err := nsl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\t// Snapshot could already be done during shutdown. If so, snapshotting again will not be available.\n\terr = mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false)\n\tif err != nil {\n\t\trequire_Error(t, err, errNoSnapAvailable)\n\t}\n\n\tnsl.Shutdown()\n\tnsl = c.restartServer(nsl)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(nsl)\n\n\t// We will now check this server directly.\n\tmset, err = nsl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif state := mset.state(); state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) {\n\t\tt.Fatalf(\"Expected first sequence of %d, got %d\", n+1, state.FirstSeq)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif state := si.State; state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) {\n\t\tt.Fatalf(\"Expected first sequence of %d, got %d\", n+1, state.FirstSeq)\n\t}\n\n\t// Now move the leader there and double check, but above test is sufficient.\n\tcheckFor(t, 30*time.Second, 250*time.Millisecond, func() error {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t\tif c.streamLeader(\"$G\", \"TEST\") == nsl {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"No correct leader yet\")\n\t})\n\n\tif state := mset.state(); state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) {\n\t\tt.Fatalf(\"Expected first sequence of %d, got %d\", n+1, state.FirstSeq)\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif state := si.State; state.FirstSeq != uint64(n+1) || state.LastSeq != uint64(n) {\n\t\tt.Fatalf(\"Expected first sequence of %d, got %d\", n+1, state.FirstSeq)\n\t}\n}\n\nfunc TestJetStreamClusterPullConsumerMaxWaiting(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"test.*\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxWaiting: 10,\n\t})\n\trequire_NoError(t, err)\n\n\t// Cannot be updated.\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxWaiting: 1,\n\t})\n\tif !strings.Contains(err.Error(), \"can not be updated\") {\n\t\tt.Fatalf(`expected \"cannot be updated\" error, got %s`, err)\n\t}\n}\n\nfunc TestJetStreamClusterEncryptedDoubleSnapshotBug(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterEncryptedTempl, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxAge: time.Second,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tnumMsgs := 50\n\tfor i := 0; i < numMsgs; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"SNAP\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Perform a snapshot on a follower.\n\tnl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tmset, err := nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\terr = mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"SNAP2\"))\n\trequire_NoError(t, err)\n\n\tfor _, seq := range []uint64{1, 11, 22, 51} {\n\t\tjs.DeleteMsg(\"TEST\", seq)\n\t}\n\n\terr = mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"SNAP3\"))\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterRePublishUpdateSupported(t *testing.T) {\n\ttest := func(t *testing.T, s *Server, stream string, replicas int) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: stream,\n\t\t\tStorage: nats.MemoryStorage,\n\t\t\tReplicas: replicas,\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\texpectUpdate := func() {\n\t\t\tt.Helper()\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar resp JSApiStreamCreateResponse\n\t\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\t\trequire_NoError(t, err)\n\t\t\tif resp.Type != JSApiStreamUpdateResponseType {\n\t\t\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamUpdateResponseType)\n\t\t\t}\n\t\t\tif IsNatsErr(resp.Error, JSStreamInvalidConfigF) {\n\t\t\t\tt.Fatalf(\"Expected no error regarding config error, got %+v\", resp.Error)\n\t\t\t}\n\t\t}\n\n\t\texpectRepublished := func(expectedRepub bool) {\n\t\t\tt.Helper()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Create a subscriber for foo.> so that we can see\n\t\t\t// our published message being echoed back to us.\n\t\t\tsf, err := nc.SubscribeSync(\"foo.>\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sf.Unsubscribe()\n\n\t\t\t// Create a subscriber for bar.> so that we can see\n\t\t\t// any potentially republished messages.\n\t\t\tsb, err := nc.SubscribeSync(\"bar.>\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sf.Unsubscribe()\n\n\t\t\t// Publish a message, it will hit the foo.> stream and\n\t\t\t// may potentially be republished to the bar.> stream.\n\t\t\t_, err = js.Publish(\"foo.\"+stream, []byte(\"HELLO!\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for a little while so that we have enough time\n\t\t\t// to determine whether it's going to arrive on one or\n\t\t\t// both streams.\n\t\t\tcheckSubsPending(t, sf, 1)\n\t\t\tif expectedRepub {\n\t\t\t\tcheckSubsPending(t, sb, 1)\n\t\t\t} else {\n\t\t\t\tcheckSubsPending(t, sb, 0)\n\t\t\t}\n\t\t}\n\n\t\t// At this point there's no republish config, so we should\n\t\t// only receive our published message on foo.>.\n\t\texpectRepublished(false)\n\n\t\t// Add a republish config so that everything on foo.> also\n\t\t// gets republished to bar.>.\n\t\tcfg.RePublish = &nats.RePublish{\n\t\t\tSource: \"foo.>\",\n\t\t\tDestination: \"bar.>\",\n\t\t}\n\t\texpectUpdate()\n\t\texpectRepublished(true)\n\n\t\t// Now take the republish config away again, so we should go\n\t\t// back to only getting them on foo.>.\n\t\tcfg.RePublish = nil\n\t\texpectUpdate()\n\t\texpectRepublished(false)\n\t}\n\n\tt.Run(\"Single\", func(t *testing.T) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\ttest(t, s, \"single\", 1)\n\t})\n\tt.Run(\"Clustered\", func(t *testing.T) {\n\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\tdefer c.shutdown()\n\n\t\ttest(t, c.randomNonLeader(), \"clustered\", 3)\n\t})\n}\n\nfunc TestJetStreamClusterDirectGetFromLeafnode(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: CORE, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 19022, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tln := c.createLeafNodeWithTemplate(\"LN-SPOKE\", tmpl)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, ln, 2)\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"KV\"})\n\trequire_NoError(t, err)\n\n\t_, err = kv.PutString(\"age\", \"22\")\n\trequire_NoError(t, err)\n\n\t// Now connect to the ln and make sure we can do a domain direct get.\n\tnc, _ = jsClientConnect(t, ln)\n\tdefer nc.Close()\n\n\tjs, err = nc.JetStream(nats.Domain(\"CORE\"))\n\trequire_NoError(t, err)\n\n\tkv, err = js.KeyValue(\"KV\")\n\trequire_NoError(t, err)\n\n\tentry, err := kv.Get(\"age\")\n\trequire_NoError(t, err)\n\trequire_True(t, string(entry.Value()) == \"22\")\n}\n\nfunc TestJetStreamClusterUnknownReplicaOnClusterRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tlname := c.streamLeader(globalAccountName, \"TEST\").Name()\n\tsendStreamMsg(t, nc, \"foo\", \"msg1\")\n\n\tnc.Close()\n\tc.stopAll()\n\t// Restart the leader...\n\tfor _, s := range c.servers {\n\t\tif s.Name() == lname {\n\t\t\tc.restartServer(s)\n\t\t}\n\t}\n\t// And one of the other servers\n\tfor _, s := range c.servers {\n\t\tif s.Name() != lname {\n\t\t\tc.restartServer(s)\n\t\t\tbreak\n\t\t}\n\t}\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\tsendStreamMsg(t, nc, \"foo\", \"msg2\")\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif len(si.Cluster.Replicas) != 2 {\n\t\tt.Fatalf(\"Leader is %s - expected 2 peers, got %+v\", si.Cluster.Leader, si.Cluster.Replicas[0])\n\t}\n\t// However, since the leader does not know the name of the server\n\t// we should report an \"unknown\" name.\n\tvar ok bool\n\tfor _, r := range si.Cluster.Replicas {\n\t\tif strings.Contains(r.Name, \"unknown\") {\n\t\t\t// Check that it has no lag reported, and the it is not current.\n\t\t\tif r.Current {\n\t\t\t\tt.Fatal(\"Expected non started node to be marked as not current\")\n\t\t\t}\n\t\t\tif r.Lag != 0 {\n\t\t\t\tt.Fatalf(\"Expected lag to not be set, was %v\", r.Lag)\n\t\t\t}\n\t\t\tif r.Active != 0 {\n\t\t\t\tt.Fatalf(\"Expected active to not be set, was: %v\", r.Active)\n\t\t\t}\n\t\t\tok = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !ok {\n\t\tt.Fatalf(\"Should have had an unknown server name, did not: %+v - %+v\", si.Cluster.Replicas[0], si.Cluster.Replicas[1])\n\t}\n}\n\nfunc TestJetStreamClusterSnapshotBeforePurgeAndCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxAge: 5 * time.Second,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tnl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\n\t// Make sure we do not get disconnected when shutting the non-leader down.\n\tnc, js = jsClientConnect(t, sl)\n\tdefer nc.Close()\n\n\tsend1k := func() {\n\t\tt.Helper()\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tjs.PublishAsync(\"foo\", []byte(\"SNAP\"))\n\t\t}\n\t\tselect {\n\t\tcase <-js.PublishAsyncComplete():\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t}\n\t}\n\n\t// Send first 1000 to everyone.\n\tsend1k()\n\n\t// Now shutdown a non-leader.\n\tc.waitOnStreamCurrent(nl, \"$G\", \"TEST\")\n\tnl.Shutdown()\n\n\t// Send another 1000.\n\tsend1k()\n\n\t// Force snapshot on the leader.\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\terr = mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\t// Purge\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Send another 1000.\n\tsend1k()\n\n\t// We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages.\n\tnc, _ = jsClientConnect(t, sl, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"$JSC.R.>\")\n\trequire_NoError(t, err)\n\n\t// Now restart non-leader.\n\tnl = c.restartServer(nl)\n\tc.waitOnStreamCurrent(nl, \"$G\", \"TEST\")\n\n\t// Grab state directly from non-leader.\n\tmset, err = nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif state := mset.state(); state.FirstSeq != 2001 || state.LastSeq != 3000 {\n\t\t\treturn fmt.Errorf(\"Incorrect state: %+v\", state)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure we only sent 2 sync catchup msgs.\n\t// This is for the delete range, and the EOF.\n\tnmsgs, _, _ := sub.Pending()\n\tif nmsgs != 2 {\n\t\tt.Fatalf(\"Expected only 2 sync catchup msgs to be sent signaling eof, but got %d\", nmsgs)\n\t}\n\n\tmsg, err := sub.NextMsg(0)\n\trequire_NoError(t, err)\n\tmbuf := msg.Data[1:]\n\tdr, err := decodeDeleteRange(mbuf)\n\trequire_NoError(t, err)\n\trequire_Equal(t, dr.First, 1001)\n\trequire_Equal(t, dr.Num, 1000)\n\n\tmsg, err = sub.NextMsg(0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msg.Data), 0)\n}\n\nfunc TestJetStreamClusterStreamResetWithLargeFirstSeq(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxAge: 5 * time.Second,\n\t\tReplicas: 1,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Fake a very large first seq.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tmset.mu.Lock()\n\tmset.store.Compact(1_000_000)\n\tmset.mu.Unlock()\n\t// Restart\n\tsl.Shutdown()\n\tsl = c.restartServer(sl)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Make sure we have the correct state after restart.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 1_000_000)\n\n\t// Now add in 10,000 messages.\n\tnum := 10_000\n\tfor i := 0; i < num; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"SNAP\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 1_000_000)\n\trequire_True(t, si.State.LastSeq == uint64(1_000_000+num-1))\n\n\t// We want to make sure we do not send unnecessary skip msgs when we know we do not have all of these messages.\n\tncs, _ := jsClientConnect(t, sl, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\tsub, err := ncs.SubscribeSync(\"$JSC.R.>\")\n\trequire_NoError(t, err)\n\n\t// Now scale up to R3.\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\tnl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamCurrent(nl, \"$G\", \"TEST\")\n\n\t// Make sure we only sent the number of catchup msgs we expected.\n\tcheckFor(t, 5*time.Second, 50*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); nmsgs != (cfg.Replicas-1)*(num+1) {\n\t\t\treturn fmt.Errorf(\"expected %d catchup msgs, but got %d\", (cfg.Replicas-1)*(num+1), nmsgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterStreamCatchupInteriorNilMsgs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tnum := 100\n\tfor l := 0; l < 5; l++ {\n\t\tfor i := 0; i < num-1; i++ {\n\t\t\tjs.PublishAsync(\"foo\", []byte(\"SNAP\"))\n\t\t}\n\t\t// Blank msg.\n\t\tjs.PublishAsync(\"foo\", nil)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Make sure we have the correct state after restart.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 500)\n\n\t// Now scale up to R3.\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\tnl := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnStreamCurrent(nl, \"$G\", \"TEST\")\n\n\tmset, err := nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tmset.mu.RLock()\n\tstate := mset.store.State()\n\tmset.mu.RUnlock()\n\trequire_True(t, state.Msgs == 500)\n}\n\ntype captureCatchupWarnLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *captureCatchupWarnLogger) Warnf(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"simulate error\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\ntype catchupMockStore struct {\n\tStreamStore\n\tch chan uint64\n}\n\nfunc (s catchupMockStore) LoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error) {\n\ts.ch <- seq\n\treturn s.StreamStore.LoadMsg(seq, sm)\n}\n\nfunc TestJetStreamClusterLeaderAbortsCatchupOnFollowerError(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tpayload := string(make([]byte, 1024))\n\ttotal := 100\n\tfor i := 0; i < total; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", payload)\n\t}\n\n\tc.waitOnAllCurrent()\n\n\t// Get the stream leader\n\tleader := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := leader.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tvar syncSubj string\n\tmset.mu.RLock()\n\tif mset.syncSub != nil {\n\t\tsyncSubj = string(mset.syncSub.subject)\n\t}\n\tmset.mu.RUnlock()\n\n\tif syncSubj == _EMPTY_ {\n\t\tt.Fatal(\"Did not find the sync request subject\")\n\t}\n\n\t// Setup the logger on the leader to make sure we capture the error and print\n\t// and also stop the runCatchup.\n\tl := &captureCatchupWarnLogger{ch: make(chan string, 10)}\n\tleader.SetLogger(l, false, false)\n\n\t// Set a fake message store that will allow us to verify\n\t// a few things.\n\tmset.mu.Lock()\n\torgMS := mset.store\n\tms := catchupMockStore{StreamStore: mset.store, ch: make(chan uint64)}\n\tmset.store = ms\n\tmset.mu.Unlock()\n\n\t// Need the system account to simulate the sync request that we are going to send.\n\tsysNC := natsConnect(t, c.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer sysNC.Close()\n\t// Setup a subscription to receive the messages sent by the leader.\n\tsub := natsSubSync(t, sysNC, nats.NewInbox())\n\treq := &streamSyncRequest{\n\t\tFirstSeq: 1,\n\t\tLastSeq: uint64(total),\n\t\tPeer: \"bozo\", // Should be one of the node name, but does not matter here\n\t}\n\tb, _ := json.Marshal(req)\n\t// Send the sync request and use our sub's subject for destination where leader\n\t// needs to send messages to.\n\tnatsPubReq(t, sysNC, syncSubj, sub.Subject, b)\n\n\t// The mock store is blocked loading the first message, so we need to consume\n\t// the sequence before being able to receive the message in our sub.\n\tselect {\n\tcase seq := <-ms.ch:\n\t\tif seq != 1 {\n\t\t\tt.Fatalf(\"Expected sequence to be 1, got %v\", seq)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get the expected error\")\n\t}\n\n\t// Now consume and the leader should report the error and terminate runCatchup\n\tmsg := natsNexMsg(t, sub, time.Second)\n\trequire_NoError(t, msg.Respond([]byte(\"simulate error\")))\n\n\tselect {\n\tcase <-l.ch:\n\t\t// OK\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get the expected error\")\n\t}\n\n\t// The mock store should be blocked in seq==2 now, but after consuming, it should\n\t// abort the runCatchup.\n\tselect {\n\tcase seq := <-ms.ch:\n\t\tif seq != 2 {\n\t\t\tt.Fatalf(\"Expected sequence to be 2, got %v\", seq)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get the expected error\")\n\t}\n\n\t// We may have some more messages loaded as a race between when the sub will\n\t// indicate that the catchup should stop and the part where we send messages\n\t// in the batch, but we should likely not have sent all messages.\n\tloaded := 0\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase <-ms.ch:\n\t\t\tloaded++\n\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\tdone = true\n\t\t}\n\t}\n\tif loaded > 10 {\n\t\tt.Fatalf(\"Too many messages were sent after detecting remote is done: %v\", loaded)\n\t}\n\n\tch := make(chan string, 1)\n\tmset.mu.Lock()\n\tmset.store = orgMS\n\tleader.sysUnsubscribe(mset.syncSub)\n\tmset.syncSub = nil\n\tleader.systemSubscribe(syncSubj, _EMPTY_, false, mset.sysc, func(_ *subscription, _ *client, _ *Account, _, reply string, msg []byte) {\n\t\tvar sreq streamSyncRequest\n\t\tif err := json.Unmarshal(msg, &sreq); err != nil {\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase ch <- reply:\n\t\tdefault:\n\t\t}\n\t})\n\tmset.mu.Unlock()\n\tsyncRepl := natsSubSync(t, sysNC, nats.NewInbox()+\".>\")\n\t// Make sure our sub is propagated\n\ttime.Sleep(250 * time.Millisecond)\n\n\tif v := leader.gcbTotal(); v != 0 {\n\t\tt.Fatalf(\"Expected gcbTotal to be 0, got %v\", v)\n\t}\n\n\tbuf := make([]byte, 1_000_000)\n\tn := runtime.Stack(buf, true)\n\tif bytes.Contains(buf[:n], []byte(\"runCatchup\")) {\n\t\tt.Fatalf(\"Looks like runCatchup is still running:\\n%s\", buf[:n])\n\t}\n\n\tmset.mu.Lock()\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\tsnapshot := &streamSnapshot{\n\t\tMsgs: state.Msgs,\n\t\tBytes: state.Bytes,\n\t\tFirstSeq: state.FirstSeq,\n\t\tLastSeq: state.LastSeq + 1,\n\t}\n\tb, _ = json.Marshal(snapshot)\n\tmset.node.SendSnapshot(b)\n\tmset.mu.Unlock()\n\n\tvar sreqSubj string\n\tselect {\n\tcase sreqSubj = <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not receive sync request\")\n\t}\n\n\t// Now send a message with a wrong sequence and expect to receive an error.\n\tem := encodeStreamMsg(\"foo\", _EMPTY_, nil, []byte(\"fail\"), 102, time.Now().UnixNano(), false)\n\tleader.sendInternalMsgLocked(sreqSubj, syncRepl.Subject, nil, em)\n\tmsg = natsNexMsg(t, syncRepl, time.Second)\n\tif len(msg.Data) == 0 {\n\t\tt.Fatal(\"Expected err response from the remote\")\n\t}\n}\n\nfunc TestJetStreamClusterStreamDirectGetNotTooSoon(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tMaxMsgsPer: 1,\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\tsendStreamMsg(t, nc, \"foo\", \"bar\")\n\n\tgetSubj := fmt.Sprintf(JSDirectGetLastBySubjectT, \"TEST\", \"foo\")\n\n\t// Make sure we get all direct subs.\n\tcheckForDirectSubs := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset.mu.RLock()\n\t\t\t\thasBoth := mset.directSub != nil && mset.lastBySub != nil\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\tif !hasBoth {\n\t\t\t\t\treturn fmt.Errorf(\"%v does not have both direct subs registered\", s)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t_, err := nc.Request(getSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tcheckForDirectSubs()\n\n\t// We want to make sure that when starting up we do not listen until we have a leader.\n\tnc.Close()\n\tc.stopAll()\n\n\t// Start just one..\n\ts, opts := RunServerWithConfig(c.opts[0].ConfigFile)\n\tc.servers[0] = s\n\tc.opts[0] = opts\n\n\tnc, _ = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err = nc.Request(getSubj, nil, time.Second)\n\trequire_Error(t, err, nats.ErrNoResponders)\n\n\t// Now start all and make sure they all eventually have subs for direct access.\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t_, err = nc.Request(getSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tcheckForDirectSubs()\n}\n\nfunc TestJetStreamClusterStaleReadsOnRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\n\tr1 := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tr1.Shutdown()\n\n\tnc.Close()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err = js.Publish(\"bar\", nil)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"baz\", nil)\n\trequire_NoError(t, err)\n\n\tr2 := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tr2.Shutdown()\n\n\tsl.Shutdown()\n\n\tc.restartServer(r2)\n\tc.restartServer(r1)\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tnc.Close()\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"bar\", nil)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"baz\", nil)\n\trequire_NoError(t, err)\n\n\tc.restartServer(sl)\n\tc.waitOnAllCurrent()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Grab expected from leader.\n\tvar state StreamState\n\tsl = c.streamLeader(\"$G\", \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.store.FastState(&state)\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif s.Running() {\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tvar fs StreamState\n\t\t\t\tmset.store.FastState(&fs)\n\t\t\t\tif !reflect.DeepEqual(fs, state) {\n\t\t\t\t\treturn fmt.Errorf(\"States do not match, expected %+v but got %+v\", state, fs)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterReplicasChangeStreamInfo(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tnumStreams := 1\n\tmsgsPerStream := 10\n\tfor i := 0; i < numStreams; i++ {\n\t\tsname := fmt.Sprintf(\"TEST_%v\", i)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: sname,\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tfor j := 0; j < msgsPerStream; j++ {\n\t\t\tsendStreamMsg(t, nc, sname, \"msg\")\n\t\t}\n\t}\n\n\tcheckStreamInfo := func(js nats.JetStreamContext) {\n\t\tt.Helper()\n\t\tcheckFor(t, 20*time.Second, 15*time.Millisecond, func() error {\n\t\t\tfor i := 0; i < numStreams; i++ {\n\t\t\t\tsi, err := js.StreamInfo(fmt.Sprintf(\"TEST_%v\", i))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != uint64(msgsPerStream) || si.State.FirstSeq != 1 || si.State.LastSeq != uint64(msgsPerStream) {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid stream info for %s: %+v\", si.Config.Name, si.State)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckStreamInfo(js)\n\n\t// Block the meta snapshots on all servers so they need to replay the log.\n\tfor _, rs := range c.servers {\n\t\tmeta := rs.getJetStream().getMetaGroup().(*raft)\n\t\tmeta.Lock()\n\t\tmeta.progress = make(map[string]*ipQueue[uint64])\n\t\tmeta.progress[\"blockSnapshots\"] = newIPQueue[uint64](meta.s, \"blockSnapshots\")\n\t\tmeta.Unlock()\n\t}\n\n\t// Update replicas down to 1\n\tfor i := 0; i < numStreams; i++ {\n\t\tsname := fmt.Sprintf(\"TEST_%v\", i)\n\t\t_, err := js.UpdateStream(&nats.StreamConfig{\n\t\t\tName: sname,\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\tcheckStreamInfo(js)\n\n\t// Back up to 3\n\tfor i := 0; i < numStreams; i++ {\n\t\tsname := fmt.Sprintf(\"TEST_%v\", i)\n\t\t_, err := js.UpdateStream(&nats.StreamConfig{\n\t\t\tName: sname,\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(globalAccountName, sname)\n\t\tfor _, s := range c.servers {\n\t\t\tc.waitOnStreamCurrent(s, globalAccountName, sname)\n\t\t}\n\t}\n\tcheckStreamInfo(js)\n\n\t// Now shutdown the cluster and restart it\n\tnc.Close()\n\tc.stopAll()\n\tc.restartAll()\n\n\tfor i := 0; i < numStreams; i++ {\n\t\tsname := fmt.Sprintf(\"TEST_%v\", i)\n\t\tc.waitOnStreamLeader(globalAccountName, sname)\n\t\tfor _, s := range c.servers {\n\t\t\tc.waitOnStreamCurrent(s, globalAccountName, sname)\n\t\t}\n\t}\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckStreamInfo(js)\n}\n\nfunc TestJetStreamClusterMaxOutstandingCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MCB\", 3)\n\tdefer c.shutdown()\n\n\tfor _, s := range c.servers {\n\t\ts.gcbMu.RLock()\n\t\tv := s.gcbOutMax\n\t\ts.gcbMu.RUnlock()\n\t\tif v != defaultMaxTotalCatchupOutBytes {\n\t\t\tt.Fatalf(\"Server %v, expected max_outstanding_catchup to be %v, got %v\", s, defaultMaxTotalCatchupOutBytes, v)\n\t\t}\n\t}\n\n\tc.shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { max_outstanding_catchup: 1KB, domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf: { listen: 127.0.0.1:-1 }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t`\n\tc = createJetStreamClusterWithTemplate(t, tmpl, \"MCB\", 3)\n\tdefer c.shutdown()\n\n\tfor _, s := range c.servers {\n\t\ts.gcbMu.RLock()\n\t\tv := s.gcbOutMax\n\t\ts.gcbMu.RUnlock()\n\t\tif v != 1024 {\n\t\t\tt.Fatalf(\"Server %v, expected max_outstanding_catchup to be 1KB, got %v\", s, v)\n\t\t}\n\t}\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Close client now and will create new one\n\tnc.Close()\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tfollower := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tfollower.Shutdown()\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Create new connection in case we would have been connected to follower.\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tpayload := string(make([]byte, 2048))\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", payload)\n\t}\n\n\t// Cause snapshots on leader\n\tmset, err := c.streamLeader(globalAccountName, \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\terr = mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\t// Resart server and it should be able to catchup\n\tfollower = c.restartServer(follower)\n\tc.waitOnStreamCurrent(follower, globalAccountName, \"TEST\")\n\n\t// Config reload not supported\n\ts := c.servers[0]\n\tcfile := s.getOpts().ConfigFile\n\tcontent, err := os.ReadFile(cfile)\n\trequire_NoError(t, err)\n\tconf := string(content)\n\tconf = strings.ReplaceAll(conf, \"max_outstanding_catchup: 1KB,\", \"max_outstanding_catchup: 1MB,\")\n\terr = os.WriteFile(cfile, []byte(conf), 0644)\n\trequire_NoError(t, err)\n\terr = s.Reload()\n\trequire_Error(t, err, fmt.Errorf(\"config reload not supported for JetStreamMaxCatchup: old=1024, new=1048576\"))\n}\n\nfunc TestJetStreamClusterCompressedStreamMessages(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trn := mset.raftNode().(*raft)\n\n\t// Block snapshots by marking as-if we're doing catchup.\n\tblockSnapshots := func() {\n\t\trn.Lock()\n\t\tdefer rn.Unlock()\n\t\trn.progress = make(map[string]*ipQueue[uint64])\n\t\trn.progress[\"blockSnapshots\"] = newIPQueue[uint64](rn.s, \"blockSnapshots\")\n\t}\n\tblockSnapshots()\n\n\t// 32k (compress threshold ~8k)\n\tpublishMessages := func() {\n\t\tt.Helper()\n\t\ttoSend, msg := 10_000, []byte(strings.Repeat(\"ABCD\", 8*1024))\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tjs.PublishAsync(\"foo\", msg)\n\t\t}\n\t\tselect {\n\t\tcase <-js.PublishAsyncComplete():\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t}\n\t}\n\tpublishMessages()\n\n\tcheckMessagesAreCompressed := func() {\n\t\tt.Helper()\n\t\trn.Lock()\n\t\tdefer rn.Unlock()\n\t\tvar ss StreamState\n\t\trn.wal.FastState(&ss)\n\t\trequire_NotEqual(t, ss.Msgs, 0)\n\t\tvar containsCompressed bool\n\t\tfor i := ss.FirstSeq; i <= ss.LastSeq; i++ {\n\t\t\tae, err := rn.loadEntry(i)\n\t\t\trequire_NoError(t, err)\n\t\t\tfor _, e := range ae.entries {\n\t\t\t\tif e.Type == EntryNormal && len(e.Data) > 0 {\n\t\t\t\t\tif entryOp(e.Data[0]) == streamMsgOp {\n\t\t\t\t\t\tt.Fatalf(\"Received non-compressed stream msg\")\n\t\t\t\t\t} else if entryOp(e.Data[0]) == compressedStreamMsgOp {\n\t\t\t\t\t\tcontainsCompressed = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tae.returnToPool()\n\t\t}\n\t\trequire_True(t, containsCompressed)\n\t}\n\tcheckMessagesAreCompressed()\n\n\tnc.Close()\n\tc.stopAll()\n\n\t// Only restart two servers, leaving one offline for the time being.\n\tc.restartServer(c.servers[0])\n\tc.restartServer(c.servers[1])\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tsl = c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err = sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err = acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trn = mset.raftNode().(*raft)\n\tblockSnapshots()\n\n\t// Now that the stream leader is elected, bring up the last server.\n\t// Must ensure compression is still used, even if the server came up a bit later.\n\tls := c.restartServer(c.servers[2])\n\tc.waitOnServerHealthz(ls)\n\n\t// Need to reconnect.\n\ts = c.streamLeader(globalAccountName, \"TEST\")\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tpublishMessages()\n\tcheckMessagesAreCompressed()\n}\n\n// https://github.com/nats-io/nats-server/issues/5612\nfunc TestJetStreamClusterWorkQueueLosingMessagesOnConsumerDelete(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := []byte(\"test alskdjalksdjalskdjaksdjlaksjdlkajsdlakjsdlakjsdlakjdlakjsdlaksjdlj\")\n\tfor _, subj := range []string{\"2\", \"5\", \"7\", \"9\"} {\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tjs.Publish(subj, msg)\n\t\t}\n\t}\n\n\tcfg := &nats.ConsumerConfig{\n\t\tName: \"test\",\n\t\tFilterSubjects: []string{\"6\", \"7\", \"8\", \"9\", \"10\"},\n\t\tDeliverSubject: \"bob\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: time.Minute,\n\t\tMaxAckPending: 1,\n\t}\n\n\t_, err = nc.SubscribeSync(\"bob\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 5; i++ {\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t\ttime.Sleep(time.Second)\n\t\tjs.DeleteConsumer(\"TEST\", \"test\")\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 40)\n}\n\nfunc TestJetStreamClusterR1ConsumerAdvisory(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.CONSUMER.CREATED.>\")\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"c1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckSubsPending(t, sub, 1)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"c2\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckSubsPending(t, sub, 2)\n}\n\nfunc TestJetStreamClusterMessageTTLCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R2S\", 2)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\taddStream(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 2,\n\t\tAllowMsgTTL: true,\n\t})\n\n\tmsg := &nats.Msg{\n\t\tSubject: \"foo.bar\",\n\t\tHeader: nats.Header{},\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tmsg.Header.Set(\"Nats-TTL\", \"60s\")\n\t\t_, err := js.PublishMsg(msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tnsl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sl, nsl)\n\n\t// Force the stream leader to take a snapshot, eliminating the Raft log,\n\t// so we can force an upper-layer catchup.\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\tgname := node.Group()\n\trequire_NotNil(t, node)\n\trequire_NoError(t, node.InstallSnapshot(mset.stateSnapshot(), false))\n\tvar state StreamState\n\tnode.(*raft).wal.FastState(&state)\n\trequire_Equal(t, state.Msgs, 0)\n\n\tconfig := nsl.JetStreamConfig()\n\tlconfig := sl.JetStreamConfig()\n\trequire_NotNil(t, config, lconfig)\n\n\tnc.Close()\n\tc.stopAll()\n\n\t// Throw away all of the non-leader's state. This will result in the\n\t// stream being recreated by the metaleader and then an upper layer\n\t// catchup will happen.\n\traftDir := filepath.Join(config.StoreDir, \"$SYS\", \"_js_\", gname)\n\trequire_NoError(t, os.RemoveAll(raftDir))\n\tmsgsDir := filepath.Join(config.StoreDir, globalAccountName, \"streams\", \"TEST\")\n\trequire_NoError(t, os.RemoveAll(msgsDir))\n\n\t// Start the servers again and wait for it to happen.\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tfor _, cs := range c.servers {\n\t\tc.waitOnStreamCurrent(cs, \"$G\", \"TEST\")\n\t}\n\n\tsl = c.streamLeader(globalAccountName, \"TEST\")\n\tnsl = c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sl, nsl)\n\n\t// The timed hash wheel on the follower should now be populated.\n\tmset, err = nsl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfs := mset.Store()\n\trequire_NotNil(t, fs)\n\trequire_NotEqual(t, fs.(*fileStore).ttls.GetNextExpiration(math.MaxInt64), math.MaxInt64)\n}\n\nfunc TestJetStreamClusterConsumerRedeliveryAfterUnexpectedReplicatedAck(t *testing.T) {\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t\tStorage: storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tgetConsumerRaftNode := func(s *Server) *raft {\n\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\trequire_NotNil(t, o)\n\t\t\t\treturn o.raftNode().(*raft)\n\t\t\t}\n\n\t\t\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\t\trn := getConsumerRaftNode(cl)\n\t\t\trn.RLock()\n\t\t\tbindex := rn.pindex\n\t\t\trn.RUnlock()\n\n\t\t\t// Simulate sending replicated ack.\n\t\t\tdseq, sseq := uint64(100), uint64(100)\n\t\t\tvar b [2*binary.MaxVarintLen64 + 1]byte\n\t\t\tb[0] = byte(updateAcksOp)\n\t\t\tn := 1\n\t\t\tn += binary.PutUvarint(b[n:], dseq)\n\t\t\tn += binary.PutUvarint(b[n:], sseq)\n\t\t\trequire_NoError(t, rn.Propose(b[:n]))\n\n\t\t\t// Wait for ack to be applied.\n\t\t\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\trn.RLock()\n\t\t\t\tdefer rn.RUnlock()\n\t\t\t\tif rn.applied <= bindex {\n\t\t\t\t\treturn errors.New(\"proposal not applied yet\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Ensure other servers can't become leader.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s != cl {\n\t\t\t\t\tcn := getConsumerRaftNode(s)\n\t\t\t\t\tcn.Lock()\n\t\t\t\t\tcn.paused = true\n\t\t\t\t\tcn.Unlock()\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Step down and campaign, should re-elect the same leader.\n\t\t\trequire_NoError(t, rn.StepDown())\n\t\t\trequire_NoError(t, rn.Campaign())\n\t\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\n\t\t\t// Replicated ack would move starting sequence ahead, which made it so we skipped messages.\n\t\t\t// It should have been ignored, and we should get the messages redelivered.\n\t\t\tmsgs, err := sub.Fetch(3, nats.MaxWait(2*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 3)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterConsumerResetStartingSequenceToAgreedState(t *testing.T) {\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t\tStorage: storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// Memory storage is less resilient in this case since we have no storage.\n\t\t\t// File storage can differentiate between no/empty state and written empty state.\n\t\t\t// Whereas memory only knows it's empty. We need one message to be delivered,\n\t\t\t// so we can recognize and revert to non-empty state.\n\t\t\tif storage == nats.MemoryStorage {\n\t\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(msgs), 1)\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\trequire_NoError(t, msg.AckSync())\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\n\t\t\t// Ensure other servers can't become leader.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s != cl {\n\t\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\t\trequire_NotNil(t, o)\n\t\t\t\t\tcn := o.raftNode().(*raft)\n\t\t\t\t\tcn.Lock()\n\t\t\t\t\tcn.paused = true\n\t\t\t\t\tcn.Unlock()\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tacc, err := cl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\trequire_NotNil(t, o)\n\t\t\trn := o.raftNode().(*raft)\n\n\t\t\t// Move consumer leader's state ahead to an unreasonable value.\n\t\t\t// Its WAL does not contain entries for this, so reset back to agreed state.\n\t\t\to.mu.Lock()\n\t\t\to.sseq = 100\n\t\t\tstore := o.store\n\t\t\to.mu.Unlock()\n\n\t\t\t// File storage is not fully resilient, as there is an initial race condition where it becomes\n\t\t\t// leader again before the flusher got to write the store state.\n\t\t\t// We need to wait here to confirm the state is written.\n\t\t\tif storage == nats.FileStorage {\n\t\t\t\tcfs := store.(*consumerFileStore)\n\t\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t\to.mu.Lock()\n\t\t\t\t\tdefer o.mu.Unlock()\n\t\t\t\t\tif _, err := os.Stat(cfs.ifn); err != nil {\n\t\t\t\t\t\treturn errors.New(\"store not written yet\")\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Step down and campaign, should re-elect the same leader.\n\t\t\trequire_NoError(t, rn.StepDown())\n\t\t\trequire_NoError(t, rn.Campaign())\n\t\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\n\t\t\t// Messages would not be redelivered if starting sequence was not reset back to agreed state.\n\t\t\tmsgs, err := sub.Fetch(3, nats.MaxWait(2*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 3)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkers(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: time.Second,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsub, err := js.SubscribeSync(\"test\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err = js.Publish(\"test\", nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, msg.AckSync())\n\t\t\t}\n\n\t\t\tmsg, err := sub.NextMsg(time.Second * 10)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Header.Get(JSMarkerReason), \"MaxAge\")\n\t\t\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkerClusteredProposal(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storageType,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: 3 * time.Second,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// First message is applied by all replicas.\n\t\t\t_, err = js.Publish(\"test\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait so MaxAge is applied and a subject delete marker is placed.\n\t\t\ttime.Sleep(4 * time.Second)\n\n\t\t\t// Second message should be successful and not be influenced by the prior subject delete marker.\n\t\t\t_, err = js.Publish(\"test\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersTTLRollupWithMaxAge(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"test\"},\n\t\tMaxAge: time.Second,\n\t\tAllowMsgTTL: true,\n\t\tAllowRollup: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\n\tsub, err := js.SubscribeSync(\"test\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\trh := nats.Header{}\n\trh.Set(JSMessageTTL, \"2s\") // MaxAge will get here first.\n\trh.Set(JSMsgRollup, JSMsgRollupSubject)\n\t_, err = js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: rh,\n\t})\n\trequire_NoError(t, err)\n\n\t// Expect to only have the rollup message here.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, 4)\n\n\tmsg, err := sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(JSMsgRollup), JSMsgRollupSubject)\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"2s\")\n\tmeta, err := msg.Metadata()\n\trequire_NoError(t, err)\n\trequire_NoError(t, msg.AckSync())\n\n\tmsg, err = sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Second, time.Since(meta.Timestamp))\n\trequire_Equal(t, msg.Header.Get(JSMarkerReason), \"MaxAge\")\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersTTLRollupWithoutMaxAge(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t\tAllowRollup: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"test\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\trh := nats.Header{}\n\trh.Set(JSMessageTTL, \"1s\")\n\trh.Set(JSMsgRollup, JSMsgRollupSubject)\n\t_, err = js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: rh,\n\t})\n\trequire_NoError(t, err)\n\n\t// Expect to only have the rollup message here.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, 4)\n\n\tmsg, err := sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(JSMsgRollup), JSMsgRollupSubject)\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n\trequire_NoError(t, msg.AckSync())\n\n\t// Wait for the rollup message to hit the TTL.\n\ttime.Sleep(2500 * time.Millisecond)\n\n\t// Now it should be gone, and it will have been replaced with a\n\t// subject delete marker (which is also gone by now).\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 6)\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersTimingWithMaxAge(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tAllowRollup: true,\n\t\t\t\tMaxAge: time.Minute,\n\t\t\t\tSubjectDeleteMarkerTTL: 1 * time.Second,\n\t\t\t})\n\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: nats.Header{},\n\t\t\t}\n\t\t\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\n\t\t\t// We expect each of these messages to age out properly and not for\n\t\t\t// the timer to stay stuck at the MaxAge interval.\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err := js.PublishMsg(msg)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// After TTL a subject delete marker will be placed.\n\t\t\t\ttime.Sleep(time.Second + 500*time.Millisecond)\n\t\t\t\t_, err = js.GetLastMsg(\"TEST\", \"test\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// It will be expired soon.\n\t\t\t\ttime.Sleep(time.Second)\n\t\t\t\t_, err = js.GetLastMsg(\"TEST\", \"test\")\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterDesyncAfterFailedScaleUp(t *testing.T) {\n\ttest := func(t *testing.T, noState bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 1,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Set up some initial state for the stream and consumer.\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgs, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\t// Scale up the stream and consumer.\n\t\tcfg.Replicas = 3\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\tcheckStreamAndConsumerState := func() error {\n\t\t\tt.Helper()\n\t\t\tsRef := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tcRef := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\t\t// If all state was removed, the leaders must equal.\n\t\t\t// Otherwise, the consumer leader may differ because the snapshot contains all required state.\n\t\t\tif sRef != cRef && noState {\n\t\t\t\treturn errors.New(\"stream and consumer leader don't equal\")\n\t\t\t} else if sRef == nil || cRef == nil {\n\t\t\t\treturn errors.New(\"stream and/or consumer leader missing\")\n\t\t\t}\n\n\t\t\tmset, err := sRef.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstreamState := mset.state()\n\n\t\t\tmset, err = cRef.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\tif o == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t\tconsumerState, err := o.store.State()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tacc := s.globalAccount()\n\t\t\t\tmset, err = acc.lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif state := mset.state(); !reflect.DeepEqual(streamState, state) {\n\t\t\t\t\treturn fmt.Errorf(\"stream states don't equal: %v vs %v\", streamState, state)\n\t\t\t\t}\n\t\t\t\to = mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\tif o == nil {\n\t\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t\t}\n\t\t\t\tif state, _ := o.store.State(); !reflect.DeepEqual(consumerState, state) {\n\t\t\t\t\treturn fmt.Errorf(\"consumer states don't equal: %v vs %v\", consumerState, state)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkStreamAndConsumerState()\n\t\t})\n\n\t\t// Install stream and consumer snapshots.\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, mset.raftNode().InstallSnapshot(mset.stateSnapshot(), false))\n\t\tstreamGroup := mset.raftNode().Group()\n\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\t\tstate, err := o.store.EncodedState()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, o.raftNode().InstallSnapshot(state, false))\n\t\tconsumerGroup := o.raftNode().Group()\n\n\t\t// Stop stream/consumer leader, and clear state on the followers except for meta.\n\t\tsl.Shutdown()\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsd := s.StoreDir()\n\t\t\ts.Shutdown()\n\t\t\trequire_NoError(t, os.RemoveAll(filepath.Join(sd, globalAccountName, streamsDir, \"TEST\")))\n\t\t\tif noState {\n\t\t\t\trequire_NoError(t, os.RemoveAll(filepath.Join(sd, DEFAULT_SYSTEM_ACCOUNT, defaultStoreDirName, streamGroup)))\n\t\t\t\trequire_NoError(t, os.RemoveAll(filepath.Join(sd, DEFAULT_SYSTEM_ACCOUNT, defaultStoreDirName, consumerGroup)))\n\t\t\t}\n\t\t}\n\n\t\t// Restart all servers except the leader.\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.restartServer(s)\n\t\t}\n\n\t\t// Allow some time for the restarted servers to try and become leader.\n\t\ttime.Sleep(2 * time.Second)\n\t\trequire_True(t, c.streamLeader(globalAccountName, \"TEST\") == nil)\n\t\tif noState {\n\t\t\trequire_True(t, c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\") == nil)\n\t\t}\n\n\t\t// Restart the old leader, now the state should converge.\n\t\tc.restartServer(sl)\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkStreamAndConsumerState()\n\t\t})\n\t}\n\n\tt.Run(\"NoState\", func(t *testing.T) { test(t, true) })\n\tt.Run(\"OnlySnapshot\", func(t *testing.T) { test(t, false) })\n}\n\nfunc TestJetStreamClusterScaleUpWithQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}\n\tsi, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Ensure the stream gets scaled up on the meta leader,\n\t// so we can pause applies on the other servers.\n\tml := c.leader()\n\tif si.Cluster.Leader != ml.Name() {\n\t\tncSys, err := nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\trequire_NoError(t, err)\n\t\tdefer ncSys.Close()\n\n\t\tjreq, err := json.Marshal(&JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: si.Cluster.Leader}})\n\t\trequire_NoError(t, err)\n\t\tresp, err := ncSys.Request(JSApiLeaderStepDown, jreq, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar sdr JSApiLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\t}\n\n\tc.waitOnLeader()\n\tml = c.leader()\n\trequire_Equal(t, si.Cluster.Leader, ml.Name())\n\n\t// Set up some initial state for the stream.\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t// Pause applies on one server.\n\t// The stream must be able to get quorum.\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\tn := s.getJetStream().getMetaGroup()\n\t\trequire_NotNil(t, n)\n\t\trequire_NoError(t, n.PauseApply())\n\t\tbreak\n\t}\n\n\t// Scale up the stream.\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\trequire_Equal(t, ml, c.streamLeader(globalAccountName, \"TEST\"))\n\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t// Now resume applies.\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\tn := s.getJetStream().getMetaGroup()\n\t\trequire_NotNil(t, n)\n\t\tn.ResumeApply()\n\t}\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterDesyncAfterDiskResetOne(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\toutdated := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\n\t// Reconnect to stream leader.\n\tnc.Close()\n\tnc, js = jsClientConnect(t, sl)\n\tdefer nc.Close()\n\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Make sure this server misses the next publish.\n\toutdated.Shutdown()\n\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t// Fully remove data directory for one server.\n\tvar reset *Server\n\tfor _, s := range c.servers {\n\t\tsd := s.StoreDir()\n\t\ts.Shutdown()\n\t\tif s != sl && s != outdated {\n\t\t\trequire_NoError(t, os.RemoveAll(sd))\n\t\t\treset = s\n\t\t}\n\t}\n\n\t// Restart only the reset and outdated servers.\n\tc.restartServer(reset)\n\tc.restartServer(outdated)\n\n\t// Allow some time for the restarted servers to try and become leader.\n\ttime.Sleep(2 * time.Second)\n\trequire_True(t, c.streamLeader(globalAccountName, \"TEST\") == nil)\n\n\t// Restart the old leader, now the state should converge.\n\tc.restartServer(sl)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterDesyncAfterDiskResetAllButOne(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\trs := c.randomServer()\n\n\t// Fully remove data directory for all but one server.\n\tfor _, s := range c.servers {\n\t\tsd := s.StoreDir()\n\t\ts.Shutdown()\n\t\tif s != rs {\n\t\t\trequire_NoError(t, os.RemoveAll(sd))\n\t\t}\n\t}\n\n\t// Restart all servers, except for the one that contains the data.\n\tfor _, s := range c.servers {\n\t\tif s != rs {\n\t\t\tc.restartServer(s)\n\t\t}\n\t}\n\n\t// Allow some time for the restarted servers to try and become leader.\n\ttime.Sleep(2 * time.Second)\n\trequire_True(t, c.leader() == nil)\n\trequire_True(t, c.streamLeader(globalAccountName, \"TEST\") == nil)\n\n\t// Restart the old leader, now the state should converge.\n\tc.restartServer(rs)\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\n//\n// DO NOT ADD NEW TESTS IN THIS FILE (unless to balance test times)\n// Add at the end of jetstream_cluster__test.go, with being the highest value.\n//\n" }, { "path": "server/jetstream_cluster_3_test.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests_3\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nats.go/jetstream\"\n)\n\nfunc TestJetStreamClusterRemovePeerByID(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Wait for a leader\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Get the name of the one that is not restarted\n\tsrvName := c.opts[2].ServerName\n\t// And its node ID\n\tpeerID := c.servers[2].Node()\n\n\tnc.Close()\n\t// Now stop the whole cluster\n\tc.stopAll()\n\t// Restart all but one\n\tfor i := 0; i < 2; i++ {\n\t\topts := c.opts[i]\n\t\ts, o := RunServerWithConfig(opts.ConfigFile)\n\t\tc.servers[i] = s\n\t\tc.opts[i] = o\n\t}\n\n\tc.waitOnClusterReadyWithNumPeers(2)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Now attempt to remove by name, this should fail because the cluster\n\t// was restarted and names are not persisted.\n\tml := c.leader()\n\tnc, err = nats.Connect(ml.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\treq := &JSApiMetaServerRemoveRequest{Server: srvName}\n\tjsreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(JSApiRemoveServer, jsreq, 2*time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiMetaServerRemoveResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error != nil)\n\trequire_True(t, IsNatsErr(resp.Error, JSClusterServerNotMemberErr))\n\n\t// Now try by ID, but first with an ID that does not match any peerID\n\treq.Peer = \"some_bad_id\"\n\tjsreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err = nc.Request(JSApiRemoveServer, jsreq, 2*time.Second)\n\trequire_NoError(t, err)\n\n\tresp = JSApiMetaServerRemoveResponse{}\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error != nil)\n\trequire_True(t, IsNatsErr(resp.Error, JSClusterServerNotMemberErr))\n\n\t// Now with the proper peer ID\n\treq.Peer = peerID\n\tjsreq, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\trmsg, err = nc.Request(JSApiRemoveServer, jsreq, 2*time.Second)\n\trequire_NoError(t, err)\n\n\tresp = JSApiMetaServerRemoveResponse{}\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error == nil)\n\trequire_True(t, resp.Success)\n}\n\nfunc TestJetStreamClusterDiscardNewAndMaxMsgsPerSubject(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tstorage StorageType\n\t\treplicas int\n\t}{\n\t\t{\"MEM-R1\", MemoryStorage, 1},\n\t\t{\"FILE-R1\", FileStorage, 1},\n\t\t{\"MEM-R3\", MemoryStorage, 3},\n\t\t{\"FILE-R3\", FileStorage, 3},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tjs.DeleteStream(\"KV\")\n\t\t\t// Make sure setting new without DiscardPolicy also being new is error.\n\t\t\tcfg := &StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"KV.>\"},\n\t\t\t\tStorage: test.storage,\n\t\t\t\tAllowDirect: true,\n\t\t\t\tDiscardNewPer: true,\n\t\t\t\tMaxMsgs: 10,\n\t\t\t\tReplicas: test.replicas,\n\t\t\t}\n\t\t\tif _, apiErr := addStreamWithError(t, nc, cfg); apiErr == nil {\n\t\t\t\tt.Fatalf(\"Expected API error but got none\")\n\t\t\t} else if apiErr.ErrCode != 10052 || !strings.Contains(apiErr.Description, \"discard new per subject requires discard new policy\") {\n\t\t\t\tt.Fatalf(\"Got wrong error: %+v\", apiErr)\n\t\t\t}\n\n\t\t\t// Set broad discard new policy to engage DiscardNewPer\n\t\t\tcfg.Discard = DiscardNew\n\t\t\t// We should also error here since we have not setup max msgs per subject.\n\t\t\tif _, apiErr := addStreamWithError(t, nc, cfg); apiErr == nil {\n\t\t\t\tt.Fatalf(\"Expected API error but got none\")\n\t\t\t} else if apiErr.ErrCode != 10052 || !strings.Contains(apiErr.Description, \"discard new per subject requires max msgs per subject > 0\") {\n\t\t\t\tt.Fatalf(\"Got wrong error: %+v\", apiErr)\n\t\t\t}\n\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t\taddStream(t, nc, cfg)\n\n\t\t\t// We want to test that we reject new messages on a per subject basis if the\n\t\t\t// max msgs per subject limit has been hit, even if other limits have not.\n\t\t\t_, err := js.Publish(\"KV.foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"KV.foo\", nil)\n\t\t\t// Go client does not have const for this one.\n\t\t\trequire_Error(t, err, errors.New(\"nats: maximum messages per subject exceeded\"))\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterCreateConsumerWithReplicaOneGetsResponse(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"C3\")\n\n\t// Update to scale down to R1, that should work (get a response)\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"C3\")\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"C3\")\n\trequire_NoError(t, err)\n\trequire_True(t, ci.Config.Replicas == 1)\n\trequire_True(t, len(ci.Cluster.Replicas) == 0)\n}\n\nfunc TestJetStreamClusterMetaRecoveryLogic(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Stream delete is answered by stream leader, stream add is answered by meta leader.\n\t// If meta leader is slower to delete, a quick add-after-delete would error with stream already exists.\n\twaitForDeleteStream := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\tml := c.leader()\n\t\t\tif ml == nil {\n\t\t\t\treturn errors.New(\"no meta leader\")\n\t\t\t}\n\t\t\tsjs := ml.getJetStream()\n\t\t\tsjs.mu.RLock()\n\t\t\tsa := sjs.streamAssignment(\"$G\", \"TEST\")\n\t\t\tsjs.mu.RUnlock()\n\t\t\tif sa != nil {\n\t\t\t\treturn errors.New(\"stream exists still\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\terr = js.DeleteStream(\"TEST\")\n\trequire_NoError(t, err)\n\twaitForDeleteStream()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\terr = js.DeleteStream(\"TEST\")\n\trequire_NoError(t, err)\n\twaitForDeleteStream()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tosi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tc.stopAll()\n\tc.restartAll()\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\ts := c.leader()\n\t\ths := s.healthz(&HealthzOptions{\n\t\t\tJSMetaOnly: true,\n\t\t})\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error)\n\t\t}\n\t\treturn nil\n\t})\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\ts = c.randomNonLeader()\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(&HealthzOptions{\n\t\t\tJSMetaOnly: true,\n\t\t})\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error)\n\t\t}\n\t\treturn nil\n\t})\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif !reflect.DeepEqual(si.Config, osi.Config) {\n\t\tt.Fatalf(\"Expected %+v, but got %+v\", osi.Config, si.Config)\n\t}\n}\n\nfunc TestJetStreamClusterDeleteConsumerWhileServerDown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomNonLeader())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DC\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\ts := c.randomNonConsumerLeader(\"$G\", \"TEST\", \"DC\")\n\ts.Shutdown()\n\n\tc.waitOnLeader() // In case that was metaleader.\n\tnc, js = jsClientConnect(t, c.randomNonLeader()) // In case we were connected there.\n\tdefer nc.Close()\n\n\terr = js.DeleteConsumer(\"TEST\", \"DC\")\n\trequire_NoError(t, err)\n\n\t// Restart.\n\ts = c.restartServer(s)\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(&HealthzOptions{\n\t\t\tJSEnabledOnly: false,\n\t\t\tJSServerOnly: false,\n\t\t})\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure we can not see it on the server that was down at the time of delete.\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif o := mset.lookupConsumer(\"DC\"); o != nil {\n\t\tt.Fatalf(\"Expected to not find consumer, but did\")\n\t}\n\n\t// Now repeat but force a meta snapshot.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DC\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Shut down the server but ensure it can't make a snapshot during shutdown.\n\ts = c.randomNonConsumerLeader(\"$G\", \"TEST\", \"DC\")\n\tmeta := s.getJetStream().getMetaGroup().(*raft)\n\tmeta.Lock()\n\tmeta.progress = make(map[string]*ipQueue[uint64])\n\tmeta.progress[\"blockSnapshots\"] = newIPQueue[uint64](meta.s, \"blockSnapshots\")\n\tmeta.Unlock()\n\ts.Shutdown()\n\n\tc.waitOnLeader() // In case that was metaleader.\n\tnc, js = jsClientConnect(t, c.randomNonLeader()) // In case we were connected there.\n\tdefer nc.Close()\n\n\terr = js.DeleteConsumer(\"TEST\", \"DC\")\n\trequire_NoError(t, err)\n\n\terr = c.leader().JetStreamSnapshotMeta()\n\trequire_NoError(t, err)\n\n\t// Restart.\n\ts = c.restartServer(s)\n\tcheckFor(t, time.Second*2, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(&HealthzOptions{\n\t\t\tJSEnabledOnly: false,\n\t\t\tJSServerOnly: false,\n\t\t})\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure we can not see it on the server that was down at the time of delete.\n\tmset, err = s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif o := mset.lookupConsumer(\"DC\"); o != nil {\n\t\tt.Fatalf(\"Expected to not find consumer, but did\")\n\t}\n}\n\nfunc TestJetStreamClusterNegativeReplicas(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ttestBadReplicas := func(t *testing.T, s *Server, name string) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tReplicas: -1,\n\t\t})\n\t\trequire_Error(t, err, NewJSReplicasCountCannotBeNegativeError())\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Check update now.\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tReplicas: -11,\n\t\t})\n\t\trequire_Error(t, err, NewJSReplicasCountCannotBeNegativeError())\n\n\t\t// Now same for consumers\n\t\tdurName := fmt.Sprintf(\"%s_dur\", name)\n\t\t_, err = js.AddConsumer(name, &nats.ConsumerConfig{\n\t\t\tDurable: durName,\n\t\t\tReplicas: -1,\n\t\t})\n\t\trequire_Error(t, err, NewJSReplicasCountCannotBeNegativeError())\n\n\t\t_, err = js.AddConsumer(name, &nats.ConsumerConfig{\n\t\t\tDurable: durName,\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Check update now\n\t\t_, err = js.UpdateConsumer(name, &nats.ConsumerConfig{\n\t\t\tDurable: durName,\n\t\t\tReplicas: -11,\n\t\t})\n\t\trequire_Error(t, err, NewJSReplicasCountCannotBeNegativeError())\n\t}\n\n\tt.Run(\"Standalone\", func(t *testing.T) { testBadReplicas(t, s, \"TEST1\") })\n\tt.Run(\"Clustered\", func(t *testing.T) { testBadReplicas(t, c.randomServer(), \"TEST2\") })\n}\n\nfunc TestJetStreamClusterUserGivenConsName(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ttest := func(t *testing.T, s *Server, stream string, replicas int, cons string) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: stream,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tcc := &CreateConsumerRequest{\n\t\t\tStream: stream,\n\t\t\tConfig: ConsumerConfig{\n\t\t\t\tName: cons,\n\t\t\t\tFilterSubject: stream,\n\t\t\t\tInactiveThreshold: 10 * time.Second,\n\t\t\t},\n\t\t}\n\t\tsubj := fmt.Sprintf(JSApiConsumerCreateExT, stream, cons, stream)\n\t\treq, err := json.Marshal(cc)\n\t\trequire_NoError(t, err)\n\n\t\treply, err := nc.Request(subj, req, 2*time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar cresp JSApiConsumerCreateResponse\n\t\tjson.Unmarshal(reply.Data, &cresp)\n\t\tif cresp.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", cresp.Error)\n\t\t}\n\t\trequire_Equal(t, cresp.Name, cons)\n\t\trequire_Equal(t, cresp.Config.Name, cons)\n\n\t\t// Resend the add request but before change something that the server\n\t\t// should reject since the consumer already exist and we don't support\n\t\t// the update of the consumer that way.\n\t\tcc.Config.DeliverPolicy = DeliverNew\n\t\treq, err = json.Marshal(cc)\n\t\trequire_NoError(t, err)\n\t\treply, err = nc.Request(subj, req, 2*time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tcresp = JSApiConsumerCreateResponse{}\n\t\tjson.Unmarshal(reply.Data, &cresp)\n\t\trequire_Error(t, cresp.Error, NewJSConsumerCreateError(errors.New(\"deliver policy can not be updated\")))\n\t}\n\n\tt.Run(\"Standalone\", func(t *testing.T) { test(t, s, \"TEST\", 1, \"cons\") })\n\tt.Run(\"Clustered R1\", func(t *testing.T) { test(t, c.randomServer(), \"TEST2\", 1, \"cons2\") })\n\tt.Run(\"Clustered R3\", func(t *testing.T) { test(t, c.randomServer(), \"TEST3\", 3, \"cons3\") })\n}\n\nfunc TestJetStreamClusterUserGivenConsNameWithLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\tconsName := \"myephemeral\"\n\tcc := &CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tName: consName,\n\t\t\tFilterSubject: \"foo\",\n\t\t\tInactiveThreshold: time.Hour,\n\t\t\tReplicas: 3,\n\t\t},\n\t}\n\tsubj := fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", consName, \"foo\")\n\treq, err := json.Marshal(cc)\n\trequire_NoError(t, err)\n\n\treply, err := nc.Request(subj, req, 2*time.Second)\n\trequire_NoError(t, err)\n\n\tvar cresp JSApiConsumerCreateResponse\n\tjson.Unmarshal(reply.Data, &cresp)\n\tif cresp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", cresp.Error)\n\t}\n\trequire_Equal(t, cresp.Name, consName)\n\trequire_Equal(t, cresp.Config.Name, consName)\n\n\t// Consumer leader name\n\tclname := cresp.ConsumerInfo.Cluster.Leader\n\n\tnreq := &JSApiConsumerGetNextRequest{Batch: 1, Expires: time.Second}\n\treq, err = json.Marshal(nreq)\n\trequire_NoError(t, err)\n\n\tsub := natsSubSync(t, nc, \"xxx\")\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", consName)\n\terr = nc.PublishRequest(rsubj, \"xxx\", req)\n\trequire_NoError(t, err)\n\n\tmsg := natsNexMsg(t, sub, time.Second)\n\trequire_Equal(t, string(msg.Data), \"msg\")\n\n\t// Shutdown the consumer leader\n\tcl := c.serverByName(clname)\n\tcl.Shutdown()\n\n\t// Wait for a bit to be sure that we lost leadership\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Wait for new leader\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", consName)\n\n\t// Make sure we can still consume.\n\tfor i := 0; i < 2; i++ {\n\t\terr = nc.PublishRequest(rsubj, \"xxx\", req)\n\t\trequire_NoError(t, err)\n\n\t\tmsg = natsNexMsg(t, sub, time.Second)\n\t\tif len(msg.Data) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\trequire_Equal(t, string(msg.Data), \"msg\")\n\t\treturn\n\t}\n\tt.Fatal(\"Did not receive message\")\n}\n\nfunc TestJetStreamClusterMirrorCrossDomainOnLeadnodeNoSystemShare(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CORE\", _EMPTY_, 3, 18033, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tln := c.createLeafNodeWithTemplateNoSystem(\"LN-SPOKE\", tmpl)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, ln, 1)\n\n\t// Create origin stream in hub.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxMsgsPerSubject: 10,\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now create the mirror on the leafnode.\n\tlnc, ljs := jsClientConnect(t, ln)\n\tdefer lnc.Close()\n\n\t_, err = ljs.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMaxMsgsPerSubject: 10,\n\t\tAllowDirect: true,\n\t\tMirrorDirect: true,\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\tAPIPrefix: \"$JS.HUB.API\",\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish to the hub stream and make sure the mirror gets those messages.\n\tfor i := 0; i < 20; i++ {\n\t\tjs.Publish(\"foo\", nil)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 10)\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := ljs.StreamInfo(\"M\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 10 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"State not current: %+v\", si.State)\n\t})\n}\n\nfunc TestJetStreamClusterFirstSeqMismatch(t *testing.T) {\n\tc := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, \"C\", 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\ttf := createTempFile(t, \"\")\n\t\t\tlogName := tf.Name()\n\t\t\ttf.Close()\n\t\t\treturn fmt.Sprintf(\"%s\\nlogfile: '%s'\", conf, logName)\n\t\t})\n\tdefer c.shutdown()\n\n\trs := c.randomServer()\n\tnc, js := jsClientConnect(t, rs)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tMaxAge: 2 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tmset, err := c.streamLeader(globalAccountName, \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tif rs == nl {\n\t\tnc.Close()\n\t\tfor _, s := range c.servers {\n\t\t\tif s != nl {\n\t\t\t\tnc, _ = jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tch := make(chan struct{})\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; ; i++ {\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(2500 * time.Millisecond)\n\tnl.Shutdown()\n\n\ttime.Sleep(500 * time.Millisecond)\n\tnode.InstallSnapshot(mset.stateSnapshot(), false)\n\ttime.Sleep(3500 * time.Millisecond)\n\n\tc.restartServer(nl)\n\tc.waitOnAllCurrent()\n\n\tclose(ch)\n\twg.Wait()\n\n\tlog := nl.getOpts().LogFile\n\tnl.Shutdown()\n\n\tcontent, err := os.ReadFile(log)\n\trequire_NoError(t, err)\n\tif bytes.Contains(content, []byte(errFirstSequenceMismatch.Error())) {\n\t\tt.Fatalf(\"First sequence mismatch occurred!\")\n\t}\n}\n\nfunc TestJetStreamClusterConsumerInactiveThreshold(t *testing.T) {\n\t// Create a standalone, a cluster, and a super cluster\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\ttest := func(t *testing.T, c *cluster, s *Server, replicas int) {\n\t\tif c != nil {\n\t\t\ts = c.randomServer()\n\t\t}\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tsname := fmt.Sprintf(\"TEST%d\", replicas)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: sname,\n\t\t\tSubjects: []string{sname},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tif c != nil {\n\t\t\tc.waitOnStreamLeader(globalAccountName, sname)\n\t\t}\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tjs.PublishAsync(sname, []byte(\"ok\"))\n\t\t}\n\t\tselect {\n\t\tcase <-js.PublishAsyncComplete():\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t}\n\n\t\twaitOnCleanup := func(ci *nats.ConsumerInfo) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\t\t_, err := js.ConsumerInfo(ci.Stream, ci.Name)\n\t\t\t\tif err == nil {\n\t\t\t\t\treturn fmt.Errorf(\"Consumer still present\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\t// Test to make sure inactive threshold is enforced for all types.\n\t\t// Ephemeral and Durable, both push and pull.\n\n\t\t// Ephemeral Push (no bind to deliver subject)\n\t\tci, err := js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDeliverSubject: \"_no_bind_\",\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\n\t\t// Ephemeral Pull\n\t\tci, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\n\t\t// Support InactiveThresholds for Durables as well.\n\n\t\t// Durable Push (no bind to deliver subject)\n\t\tci, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d1\",\n\t\t\tDeliverSubject: \"_no_bind_\",\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\n\t\t// Durable Push (no bind to deliver subject) with an activity\n\t\t// threshold set after creation\n\t\tci, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d2\",\n\t\t\tDeliverSubject: \"_no_bind_\",\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tif c != nil {\n\t\t\tc.waitOnConsumerLeader(globalAccountName, sname, \"d2\")\n\t\t}\n\t\t_, err = js.UpdateConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d2\",\n\t\t\tDeliverSubject: \"_no_bind_\",\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\n\t\t// Durable Pull\n\t\tci, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d3\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\n\t\t// Durable Pull with an inactivity threshold set after creation\n\t\tci, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d4\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tif c != nil {\n\t\t\tc.waitOnConsumerLeader(globalAccountName, sname, \"d4\")\n\t\t}\n\t\t_, err = js.UpdateConsumer(sname, &nats.ConsumerConfig{\n\t\t\tDurable: \"d4\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tInactiveThreshold: 50 * time.Millisecond,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\twaitOnCleanup(ci)\n\t}\n\n\tt.Run(\"standalone\", func(t *testing.T) { test(t, nil, s, 1) })\n\tt.Run(\"cluster-r1\", func(t *testing.T) { test(t, c, nil, 1) })\n\tt.Run(\"cluster-r3\", func(t *testing.T) { test(t, c, nil, 3) })\n\tt.Run(\"super-cluster-r1\", func(t *testing.T) { test(t, sc.randomCluster(), nil, 1) })\n\tt.Run(\"super-cluster-r3\", func(t *testing.T) { test(t, sc.randomCluster(), nil, 3) })\n}\n\n// To capture our false warnings for clustered stream lag.\ntype testStreamLagWarnLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *testStreamLagWarnLogger) Warnf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"has high message lag\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\n// False triggering warnings on stream lag because not offsetting by failures.\nfunc TestJetStreamClusterStreamLagWarning(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\n\tl := &testStreamLagWarnLogger{ch: make(chan string, 10)}\n\tsl.SetLogger(l, false, false)\n\n\t// We only need to trigger post RAFT propose failures that increment mset.clfs.\n\t// Dedupe with msgIDs is one, so we will use that.\n\tm := nats.NewMsg(\"foo\")\n\tm.Data = []byte(\"OK\")\n\tm.Header.Set(JSMsgId, \"zz\")\n\n\t// Make sure we know we will trip the warning threshold.\n\tfor i := 0; i < 2*streamLagWarnThreshold; i++ {\n\t\tjs.PublishMsgAsync(m)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tselect {\n\tcase msg := <-l.ch:\n\t\tt.Fatalf(\"Unexpected msg lag warning seen: %s\", msg)\n\tcase <-time.After(100 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/3603\nfunc TestJetStreamClusterSignalPullConsumersOnDelete(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create 2 pull consumers.\n\tsub1, err := js.PullSubscribe(\"foo\", \"d1\")\n\trequire_NoError(t, err)\n\n\tsub2, err := js.PullSubscribe(\"foo\", \"d2\")\n\trequire_NoError(t, err)\n\n\t// We want to make sure we get kicked out prior to the timeout\n\t// when consumers are being deleted or the parent stream is being deleted.\n\t// Note this should be lower case, Go client needs to be updated.\n\texpectedErr := errors.New(\"nats: consumer deleted\")\n\n\t// Queue up the delete for sub1\n\ttime.AfterFunc(250*time.Millisecond, func() { js.DeleteConsumer(\"TEST\", \"d1\") })\n\tstart := time.Now()\n\t_, err = sub1.Fetch(1, nats.MaxWait(10*time.Second))\n\trequire_Error(t, err, expectedErr)\n\n\t// Check that we bailed early.\n\tif time.Since(start) > time.Second {\n\t\tt.Fatalf(\"Took to long to bail out on consumer delete\")\n\t}\n\n\ttime.AfterFunc(250*time.Millisecond, func() { js.DeleteStream(\"TEST\") })\n\tstart = time.Now()\n\t_, err = sub2.Fetch(1, nats.MaxWait(10*time.Second))\n\trequire_Error(t, err, expectedErr)\n\tif time.Since(start) > time.Second {\n\t\tt.Fatalf(\"Took to long to bail out on stream delete\")\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/3559\nfunc TestJetStreamClusterSourceWithOptStartTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ttest := func(t *testing.T, c *cluster, s *Server) {\n\n\t\treplicas := 1\n\t\tif c != nil {\n\t\t\ts = c.randomServer()\n\t\t\treplicas = 3\n\t\t}\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tyesterday := time.Now().Add(-24 * time.Hour)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"SOURCE\",\n\t\t\tReplicas: replicas,\n\t\t\tSources: []*nats.StreamSource{{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tOptStartTime: &yesterday,\n\t\t\t}},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"MIRROR\",\n\t\t\tReplicas: replicas,\n\t\t\tMirror: &nats.StreamSource{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tOptStartTime: &yesterday,\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\ttotal := 10\n\t\tfor i := 0; i < total; i++ {\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"hello\")\n\t\t}\n\n\t\tcheckCount := func(sname string, expected int) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 10*time.Second, 50*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(sname)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif n := si.State.Msgs; n != uint64(expected) {\n\t\t\t\t\treturn fmt.Errorf(\"Expected stream %q to have %v messages, got %v\", sname, expected, n)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\tcheckCount(\"TEST\", 10)\n\t\tcheckCount(\"SOURCE\", 10)\n\t\tcheckCount(\"MIRROR\", 10)\n\n\t\terr = js.PurgeStream(\"SOURCE\")\n\t\trequire_NoError(t, err)\n\t\terr = js.PurgeStream(\"MIRROR\")\n\t\trequire_NoError(t, err)\n\n\t\tcheckCount(\"TEST\", 10)\n\t\tcheckCount(\"SOURCE\", 0)\n\t\tcheckCount(\"MIRROR\", 0)\n\n\t\tnc.Close()\n\t\tif c != nil {\n\t\t\tc.stopAll()\n\t\t\tc.restartAll()\n\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"SOURCE\")\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"MIRROR\")\n\n\t\t\ts = c.randomServer()\n\t\t} else {\n\t\t\tsd := s.JetStreamConfig().StoreDir\n\t\t\ts.Shutdown()\n\t\t\ts = RunJetStreamServerOnPort(-1, sd)\n\t\t\tdefer s.Shutdown()\n\t\t}\n\n\t\t// Wait a bit before checking because sync'ing (even with the defect)\n\t\t// would not happen right away. I tried with 1 sec and test would pass,\n\t\t// so need to be at least that much.\n\t\ttime.Sleep(2 * time.Second)\n\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tcheckCount(\"TEST\", 10)\n\t\tcheckCount(\"SOURCE\", 0)\n\t\tcheckCount(\"MIRROR\", 0)\n\t}\n\n\tt.Run(\"standalone\", func(t *testing.T) { test(t, nil, s) })\n\tt.Run(\"cluster\", func(t *testing.T) { test(t, c, nil) })\n}\n\ntype networkCableUnplugged struct {\n\tnet.Conn\n\tsync.Mutex\n\tunplugged bool\n\twb bytes.Buffer\n\twg sync.WaitGroup\n}\n\nfunc (c *networkCableUnplugged) Write(b []byte) (int, error) {\n\tc.Lock()\n\tif c.unplugged {\n\t\tc.wb.Write(b)\n\t\tc.Unlock()\n\t\treturn len(b), nil\n\t} else if c.wb.Len() > 0 {\n\t\tc.wb.Write(b)\n\t\tbuf := c.wb.Bytes()\n\t\tc.wb.Reset()\n\t\tc.Unlock()\n\t\tif _, err := c.Conn.Write(buf); err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\treturn len(b), nil\n\t}\n\tc.Unlock()\n\treturn c.Conn.Write(b)\n}\n\nfunc (c *networkCableUnplugged) Read(b []byte) (int, error) {\n\tc.Lock()\n\twait := c.unplugged\n\tc.Unlock()\n\tif wait {\n\t\tc.wg.Wait()\n\t}\n\treturn c.Conn.Read(b)\n}\n\nfunc TestJetStreamClusterScaleDownWhileNoQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\t// Let's have a server from this R2 stream be network partitionned.\n\t// We will take the leader, but doesn't have to be.\n\t// To simulate partition, we will replace all its routes with a\n\t// special connection that drops messages.\n\tsl := c.serverByName(si.Cluster.Leader)\n\tif s == sl {\n\t\tnc.Close()\n\t\tfor s = c.randomServer(); s != sl; s = c.randomServer() {\n\t\t}\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t}\n\n\tsl.mu.Lock()\n\tsl.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tncu := &networkCableUnplugged{Conn: r.nc, unplugged: true}\n\t\tncu.wg.Add(1)\n\t\tr.nc = ncu\n\t\tr.mu.Unlock()\n\t})\n\tsl.mu.Unlock()\n\n\t// Wait for the stream info to fail\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"stream still has a leader\")\n\t})\n\n\t// Make sure if meta leader was on same server as stream leader we make sure\n\t// it elects new leader to receive update request.\n\tc.waitOnLeader()\n\n\t// Now try to edit the stream by making it an R1. In some case we get\n\t// a context deadline error, in some no error. So don't check the returned error.\n\tjs.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}, nats.MaxWait(5*time.Second))\n\n\tsl.mu.Lock()\n\tsl.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tncu := r.nc.(*networkCableUnplugged)\n\t\tncu.Lock()\n\t\tncu.unplugged = false\n\t\tncu.wg.Done()\n\t\tncu.Unlock()\n\t\tr.mu.Unlock()\n\t})\n\tsl.mu.Unlock()\n\n\tcheckClusterFormed(t, c.servers...)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n}\n\n// We noticed that ha_assets enforcement seemed to not be upheld when assets created in a rapid fashion.\nfunc TestJetStreamClusterHAssetsEnforcement(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterTempl, \"store_dir:\", \"limits: {max_ha_assets: 2}, store_dir:\", 1)\n\tc := createJetStreamClusterWithTemplateAndModHook(t, tmpl, \"R3S\", 3, nil)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST-1\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST-2\",\n\t\tSubjects: []string{\"bar\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\texceededErrs := []error{errors.New(\"system limit reached\"), errors.New(\"no suitable peers\")}\n\n\t// Should fail.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST-3\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 3,\n\t})\n\trequire_Error(t, err, exceededErrs...)\n}\n\nfunc TestJetStreamClusterInterestStreamConsumer(t *testing.T) {\n\tcheckInterestStateT = 4 * time.Second\n\tcheckInterestStateJ = 1\n\tdefer func() {\n\t\tcheckInterestStateT = defaultCheckInterestStateT\n\t\tcheckInterestStateJ = defaultCheckInterestStateJ\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tvar subs []*nats.Subscription\n\tns := 5\n\n\tfor i := 0; i < ns; i++ {\n\t\tdn := fmt.Sprintf(\"d%d\", i)\n\t\tsub, err := js.PullSubscribe(\"foo\", dn)\n\t\trequire_NoError(t, err)\n\t\tsubs = append(subs, sub)\n\t}\n\n\t// Send 10 msgs\n\tn := 10\n\tfor i := 0; i < n; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\t// Collect all the messages.\n\tvar msgs []*nats.Msg\n\tfor _, sub := range subs {\n\t\tlmsgs := fetchMsgs(t, sub, n, time.Second)\n\t\tif len(lmsgs) != n {\n\t\t\tt.Fatalf(\"Did not receive all msgs: %d vs %d\", len(lmsgs), n)\n\t\t}\n\t\tmsgs = append(msgs, lmsgs...)\n\t}\n\n\t// Shuffle\n\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\tfor _, m := range msgs {\n\t\trequire_NoError(t, m.AckSync())\n\t}\n\n\t// Make sure replicated acks are processed.\n\tcheckFor(t, 20*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Should not have any messages left: %d of %d\", si.State.Msgs, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterNoPanicOnStreamInfoWhenNoLeaderYet(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc := natsConnect(t, c.randomServer().ClientURL())\n\tdefer nc.Close()\n\n\tjs, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tch := make(chan struct{})\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tfor {\n\t\t\tjs.StreamInfo(\"TEST\")\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tcase <-time.After(15 * time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Don't care if this succeeds or not (could get a context deadline\n\t// due to the low MaxWait() when creating the context).\n\tjs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\n\tclose(ch)\n\twg.Wait()\n}\n\nfunc TestJetStreamClusterNoTimeoutOnStreamInfoOnPreferredLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Simulate the preferred stream leader to not have initialized the raft node yet.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tsjs := sl.getJetStream()\n\trg := mset.raftGroup()\n\tsjs.mu.Lock()\n\trg.node = nil\n\tsjs.mu.Unlock()\n\n\t// Should not time out on the stream info during this condition.\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n}\n\n// Issue https://github.com/nats-io/nats-server/issues/3630\nfunc TestJetStreamClusterPullConsumerAcksExtendInactivityThreshold(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\n\tn := 10\n\tfor i := 0; i < n; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\t// Pull Consumer\n\tsub, err := js.PullSubscribe(\"foo\", \"d\", nats.InactiveThreshold(time.Second), nats.AckWait(time.Second))\n\trequire_NoError(t, err)\n\n\tfetchMsgs(t, sub, n/2, time.Second)\n\t// Will wait for .5s.\n\ttime.Sleep(500 * time.Millisecond)\n\tmsgs := fetchMsgs(t, sub, n/2, time.Second)\n\tif len(msgs) != n/2 {\n\t\tt.Fatalf(\"Did not receive msgs: %d vs %d\", len(msgs), n/2)\n\t}\n\n\t// Wait for .5s.\n\ttime.Sleep(500 * time.Millisecond)\n\tmsgs[0].Ack() // Ack\n\t// Wait another .5s.\n\ttime.Sleep(500 * time.Millisecond)\n\tmsgs[1].Nak() // Nak\n\t// Wait another .5s.\n\ttime.Sleep(500 * time.Millisecond)\n\tmsgs[2].Term() // Term\n\ttime.Sleep(500 * time.Millisecond)\n\tmsgs[3].InProgress() // WIP\n\n\t// The above should have kept the consumer alive.\n\t_, err = js.ConsumerInfo(\"TEST\", \"d\")\n\trequire_NoError(t, err)\n\n\t// Make sure it gets cleaned up.\n\ttime.Sleep(3500 * time.Millisecond)\n\t_, err = js.ConsumerInfo(\"TEST\", \"d\")\n\trequire_Error(t, err, nats.ErrConsumerNotFound)\n}\n\n// https://github.com/nats-io/nats-server/issues/3677\nfunc TestJetStreamClusterParallelStreamCreation(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnp := 100\n\n\tstartCh := make(chan bool)\n\terrCh := make(chan error, np)\n\n\twg := sync.WaitGroup{}\n\twg.Add(np)\n\n\tstart := sync.WaitGroup{}\n\tstart.Add(np)\n\n\tfor i := 0; i < np; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Individual connection\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\t\t\t// Signal we are ready\n\t\t\tstart.Done()\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"common.*.*\"},\n\t\t\t\tReplicas: 3,\n\t\t\t}); err != nil {\n\t\t\t\terrCh <- err\n\t\t\t}\n\t\t}()\n\t}\n\n\tstart.Wait()\n\tclose(startCh)\n\twg.Wait()\n\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n\n\t// We had a bug during parallel stream creation as well that would overwrite the sync subject used for catchups, etc.\n\t// Test that here as well by shutting down a non-leader, adding a whole bunch of messages, and making sure on restart\n\t// we properly recover.\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tnl.Shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 128)\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"common.foo.bar\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\t// We need to force the leader to do a snapshot so we kick in upper layer catchup which depends on syncSubject.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\trequire_NotNil(t, node)\n\tnode.InstallSnapshot(mset.stateSnapshot(), false)\n\n\tnl = c.restartServer(nl)\n\tc.waitOnStreamCurrent(nl, globalAccountName, \"TEST\")\n\n\tmset, err = nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Check state directly.\n\tmset.mu.Lock()\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\tmset.mu.Unlock()\n\n\trequire_Equal(t, state.Msgs, 100)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 100)\n}\n\n// In addition to test above, if streams were attempted to be created in parallel\n// it could be that multiple raft groups would be created for the same asset.\nfunc TestJetStreamClusterParallelStreamCreationDupeRaftGroups(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnp := 20\n\n\tstartCh := make(chan bool)\n\twg := sync.WaitGroup{}\n\twg.Add(np)\n\tfor i := 0; i < np; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Individual connection\n\t\t\tnc, _ := jsClientConnect(t, c.randomServer())\n\t\t\tjs, _ := nc.JetStream(nats.MaxWait(time.Second))\n\t\t\tdefer nc.Close()\n\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\t// Ignore errors in this test, care about raft group and metastate.\n\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"common.*.*\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t}()\n\t}\n\n\tclose(startCh)\n\twg.Wait()\n\n\t// Restart a server too.\n\ts := c.randomServer()\n\ts.Shutdown()\n\ts = c.restartServer(s)\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t// Check that this server has only two active raft nodes after restart.\n\tif nrn := s.numRaftNodes(); nrn != 2 {\n\t\tt.Fatalf(\"Expected only two active raft nodes, got %d\", nrn)\n\t}\n\n\t// Make sure we only have 2 unique raft groups for all servers.\n\t// One for meta, one for stream.\n\texpected := 2\n\trg := make(map[string]struct{})\n\tfor _, s := range c.servers {\n\t\ts.rnMu.RLock()\n\t\tfor _, ni := range s.raftNodes {\n\t\t\tn := ni.(*raft)\n\t\t\trg[n.Group()] = struct{}{}\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t}\n\tif len(rg) != expected {\n\t\tt.Fatalf(\"Expected only %d distinct raft groups for all servers, go %d\", expected, len(rg))\n\t}\n}\n\nfunc TestJetStreamClusterParallelConsumerCreation(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnectNewAPI(t, c.randomServer())\n\tdefer nc.Close()\n\n\tctx := context.Background()\n\n\t_, err := js.CreateStream(ctx, jetstream.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"common.*.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnp := 50\n\n\tstartCh := make(chan bool)\n\terrCh := make(chan error, np)\n\n\tcfg := jetstream.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tReplicas: 3,\n\t}\n\n\twg := sync.WaitGroup{}\n\tswg := sync.WaitGroup{}\n\twg.Add(np)\n\tswg.Add(np)\n\n\tfor i := 0; i < np; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Individual connection\n\t\t\tnc, js := jsClientConnectNewAPI(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tswg.Done()\n\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\tif _, err := js.CreateConsumer(ctx, \"TEST\", cfg); err != nil {\n\t\t\t\terrCh <- err\n\t\t\t}\n\t\t}()\n\t}\n\n\tswg.Wait()\n\tclose(startCh)\n\n\twg.Wait()\n\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n\n\t// Make sure we only have 3 unique raft groups for all servers.\n\t// One for meta, one for stream, one for consumer.\n\texpected := 3\n\trg := make(map[string]struct{})\n\tfor _, s := range c.servers {\n\t\ts.rnMu.RLock()\n\t\tfor _, ni := range s.raftNodes {\n\t\t\tn := ni.(*raft)\n\t\t\trg[n.Group()] = struct{}{}\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t}\n\tif len(rg) != expected {\n\t\tt.Fatalf(\"Expected only %d distinct raft groups for all servers, go %d\", expected, len(rg))\n\t}\n}\n\nfunc TestJetStreamClusterGhostEphemeralsAfterRestart(t *testing.T) {\n\tconsumerNotActiveStartInterval = time.Second\n\tconsumerNotActiveMaxInterval = time.Second\n\tt.Cleanup(func() {\n\t\tconsumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval\n\t\tconsumerNotActiveMaxInterval = defaultConsumerNotActiveMaxInterval\n\t})\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Add in 100 memory based ephemerals.\n\tfor i := 0; i < 100; i++ {\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tReplicas: 1,\n\t\t\tInactiveThreshold: time.Second,\n\t\t\tMemoryStorage: true,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Grab random server.\n\trs := c.randomServer()\n\t// Now shutdown cluster.\n\tc.stopAll()\n\n\t// Let the consumers all expire.\n\ttime.Sleep(2 * time.Second)\n\n\t// Restart first and wait so that we know it will try cleanup without a metaleader.\n\t// It will fail as there's no metaleader at that time, it should keep retrying on an interval.\n\tc.restartServer(rs)\n\ttime.Sleep(time.Second)\n\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsubj := fmt.Sprintf(JSApiConsumerListT, \"TEST\")\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\t// Request will take at most 4 seconds if some consumers can't be found.\n\t\tm, err := nc.Request(subj, nil, 5*time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar resp JSApiConsumerListResponse\n\t\terr = json.Unmarshal(m.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\tif len(resp.Consumers) != 0 {\n\t\t\treturn fmt.Errorf(\"Still have %d consumers\", len(resp.Consumers))\n\t\t}\n\t\tif len(resp.Missing) != 0 {\n\t\t\treturn fmt.Errorf(\"Still have %d missing consumers\", len(resp.Missing))\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterReplacementPolicyAfterPeerRemove(t *testing.T) {\n\t// R3 scenario where there is a redundant node in each unique cloud so removing a peer should result in\n\t// an immediate replacement also preserving cloud uniqueness.\n\n\tsc := createJetStreamClusterExplicit(t, \"PR9\", 9)\n\tsc.waitOnPeerCount(9)\n\n\treset := func(s *Server) {\n\t\ts.mu.Lock()\n\t\trch := s.sys.resetCh\n\t\ts.mu.Unlock()\n\t\tif rch != nil {\n\t\t\trch <- struct{}{}\n\t\t}\n\t\ts.sendStatszUpdate()\n\t}\n\n\ttags := []string{\"cloud:aws\", \"cloud:aws\", \"cloud:aws\", \"cloud:gcp\", \"cloud:gcp\", \"cloud:gcp\", \"cloud:az\", \"cloud:az\", \"cloud:az\"}\n\n\tvar serverUTags = make(map[string]string)\n\n\tfor i, s := range sc.servers {\n\t\ts.optsMu.Lock()\n\t\tserverUTags[s.Name()] = tags[i]\n\t\ts.opts.Tags.Add(tags[i])\n\t\ts.opts.JetStreamUniqueTag = \"cloud\"\n\t\ts.optsMu.Unlock()\n\t\treset(s)\n\t}\n\n\tml := sc.leader()\n\tjs := ml.getJetStream()\n\trequire_True(t, js != nil)\n\tjs.mu.RLock()\n\tcc := js.cluster\n\trequire_True(t, cc != nil)\n\n\t// Walk and make sure all tags are registered.\n\texpires := time.Now().Add(10 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tallOK := true\n\t\tfor _, p := range cc.meta.Peers() {\n\t\t\tsi, ok := ml.nodeToInfo.Load(p.ID)\n\t\t\trequire_True(t, ok)\n\t\t\tni := si.(nodeInfo)\n\t\t\tif len(ni.tags) == 0 {\n\t\t\t\tallOK = false\n\t\t\t\treset(sc.serverByName(ni.name))\n\t\t\t}\n\t\t}\n\t\tif allOK {\n\t\t\tbreak\n\t\t}\n\t}\n\tjs.mu.RUnlock()\n\tdefer sc.shutdown()\n\n\tsc.waitOnClusterReadyWithNumPeers(9)\n\n\ts := sc.leader()\n\tnc, jsc := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsc.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tosi, err := jsc.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Double check original placement honors unique_tag\n\tvar uTags = make(map[string]struct{})\n\n\tuTags[serverUTags[osi.Cluster.Leader]] = struct{}{}\n\tfor _, replica := range osi.Cluster.Replicas {\n\t\tevalTag := serverUTags[replica.Name]\n\t\tif _, exists := uTags[evalTag]; !exists {\n\t\t\tuTags[evalTag] = struct{}{}\n\t\t\tcontinue\n\t\t} else {\n\t\t\tt.Fatalf(\"expected initial placement to honor unique_tag\")\n\t\t}\n\t}\n\n\t// Remove a peer and select replacement 5 times to avoid false good\n\tfor i := 0; i < 5; i++ {\n\t\t// Remove 1 peer replica (this will be random cloud region as initial placement was randomized ordering)\n\t\t// After each successful iteration, osi will reflect the current RG peers\n\t\ttoRemove := osi.Cluster.Replicas[0].Name\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"TEST\"), []byte(`{\"peer\":\"`+toRemove+`\"}`), time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar rpResp JSApiStreamRemovePeerResponse\n\t\terr = json.Unmarshal(resp.Data, &rpResp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, rpResp.Success)\n\n\t\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\t\tosi, err = jsc.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(osi.Cluster.Replicas) != 2 {\n\t\t\t\treturn fmt.Errorf(\"expected R3, got R%d\", len(osi.Cluster.Replicas)+1)\n\t\t\t}\n\t\t\t// STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred by\n\t\t\t// checking that the toRemove peer is gone.\n\t\t\tfor _, replica := range osi.Cluster.Replicas {\n\t\t\t\tif replica.Name == toRemove {\n\t\t\t\t\treturn fmt.Errorf(\"expected replaced replica, old replica still present\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Validate that replacement with new peer still honors\n\t\tuTags = make(map[string]struct{}) //reset\n\n\t\tuTags[serverUTags[osi.Cluster.Leader]] = struct{}{}\n\t\tfor _, replica := range osi.Cluster.Replicas {\n\t\t\tevalTag := serverUTags[replica.Name]\n\t\t\tif _, exists := uTags[evalTag]; !exists {\n\t\t\t\tuTags[evalTag] = struct{}{}\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"expected new peer and revised placement to honor unique_tag\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterReplacementPolicyAfterPeerRemoveNoPlace(t *testing.T) {\n\t// R3 scenario where there are exactly three unique cloud nodes, so removing a peer should NOT\n\t// result in a new peer\n\n\tsc := createJetStreamClusterExplicit(t, \"threeup\", 3)\n\tsc.waitOnPeerCount(3)\n\n\treset := func(s *Server) {\n\t\ts.mu.Lock()\n\t\trch := s.sys.resetCh\n\t\ts.mu.Unlock()\n\t\tif rch != nil {\n\t\t\trch <- struct{}{}\n\t\t}\n\t\ts.sendStatszUpdate()\n\t}\n\n\ttags := []string{\"cloud:aws\", \"cloud:gcp\", \"cloud:az\"}\n\n\tvar serverUTags = make(map[string]string)\n\n\tfor i, s := range sc.servers {\n\t\ts.optsMu.Lock()\n\t\tserverUTags[s.Name()] = tags[i]\n\t\ts.opts.Tags.Add(tags[i])\n\t\ts.opts.JetStreamUniqueTag = \"cloud\"\n\t\ts.optsMu.Unlock()\n\t\treset(s)\n\t}\n\n\tml := sc.leader()\n\tjs := ml.getJetStream()\n\trequire_True(t, js != nil)\n\tjs.mu.RLock()\n\tcc := js.cluster\n\trequire_True(t, cc != nil)\n\n\t// Walk and make sure all tags are registered.\n\texpires := time.Now().Add(10 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tallOK := true\n\t\tfor _, p := range cc.meta.Peers() {\n\t\t\tsi, ok := ml.nodeToInfo.Load(p.ID)\n\t\t\trequire_True(t, ok)\n\t\t\tni := si.(nodeInfo)\n\t\t\tif len(ni.tags) == 0 {\n\t\t\t\tallOK = false\n\t\t\t\treset(sc.serverByName(ni.name))\n\t\t\t}\n\t\t}\n\t\tif allOK {\n\t\t\tbreak\n\t\t}\n\t}\n\tjs.mu.RUnlock()\n\tdefer sc.shutdown()\n\n\tsc.waitOnClusterReadyWithNumPeers(3)\n\n\ts := sc.leader()\n\tnc, jsc := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsc.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tosi, err := jsc.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Double check original placement honors unique_tag\n\tvar uTags = make(map[string]struct{})\n\n\tuTags[serverUTags[osi.Cluster.Leader]] = struct{}{}\n\tfor _, replica := range osi.Cluster.Replicas {\n\t\tevalTag := serverUTags[replica.Name]\n\t\tif _, exists := uTags[evalTag]; !exists {\n\t\t\tuTags[evalTag] = struct{}{}\n\t\t\tcontinue\n\t\t} else {\n\t\t\tt.Fatalf(\"expected initial placement to honor unique_tag\")\n\t\t}\n\t}\n\n\t// Remove 1 peer replica (this will be random cloud region as initial placement was randomized ordering)\n\t_, err = nc.Request(\"$JS.API.STREAM.PEER.REMOVE.TEST\", []byte(`{\"peer\":\"`+osi.Cluster.Replicas[0].Name+`\"}`), time.Second*10)\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Verify R2 since no eligible peer can replace the removed peer without braking unique constraint\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tosi, err = jsc.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif len(osi.Cluster.Replicas) != 1 {\n\t\t\treturn fmt.Errorf(\"expected R2, got R%d\", len(osi.Cluster.Replicas)+1)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Validate that remaining members still honor unique tags\n\tuTags = make(map[string]struct{}) //reset\n\n\tuTags[serverUTags[osi.Cluster.Leader]] = struct{}{}\n\tfor _, replica := range osi.Cluster.Replicas {\n\t\tevalTag := serverUTags[replica.Name]\n\t\tif _, exists := uTags[evalTag]; !exists {\n\t\t\tuTags[evalTag] = struct{}{}\n\t\t\tcontinue\n\t\t} else {\n\t\t\tt.Fatalf(\"expected revised placement to honor unique_tag\")\n\t\t}\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/3191\nfunc TestJetStreamClusterLeafnodeDuplicateConsumerMessages(t *testing.T) {\n\t// Cluster B\n\tc := createJetStreamCluster(t, jsClusterTempl, \"B\", _EMPTY_, 2, 22020, false)\n\tdefer c.shutdown()\n\n\t// Cluster A\n\t// Domain is \"A'\n\tlc := c.createLeafNodesWithStartPortAndDomain(\"A\", 2, 22110, \"A\")\n\tdefer lc.shutdown()\n\n\tlc.waitOnClusterReady()\n\n\t// We want A-S-1 connected to B-S-1 and A-S-2 connected to B-S-2\n\t// So adjust if needed.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor i, ls := range lc.servers {\n\t\t\tls.mu.RLock()\n\t\t\tvar remoteServer string\n\t\t\tfor _, rc := range ls.leafs {\n\t\t\t\trc.mu.Lock()\n\t\t\t\tremoteServer = rc.leaf.remoteServer\n\t\t\t\trc.mu.Unlock()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tls.mu.RUnlock()\n\n\t\t\twantedRemote := fmt.Sprintf(\"S-%d\", i+1)\n\t\t\tif remoteServer != wantedRemote {\n\t\t\t\tls.Shutdown()\n\t\t\t\tlc.restartServer(ls)\n\t\t\t\treturn fmt.Errorf(\"Leafnode server %d not connected to %q\", i+1, wantedRemote)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Wait on ready again.\n\tlc.waitOnClusterReady()\n\n\t// Create a stream and a durable pull consumer on cluster A.\n\tlnc, ljs := jsClientConnect(t, lc.randomServer())\n\tdefer lnc.Close()\n\n\t_, err := ljs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure stream leader is on S-1\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := ljs.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.Cluster.Leader == \"A-S-1\" {\n\t\t\treturn nil\n\t\t}\n\t\t_, err = lnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\treturn fmt.Errorf(\"Stream leader not placed on A-S-1\")\n\t})\n\n\t_, err = ljs.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t_, err = ljs.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tReplicas: 2,\n\t\tMaxDeliver: 1,\n\t\tAckPolicy: nats.AckNonePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure consumer leader is on S-2\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := ljs.ConsumerInfo(\"TEST\", \"dlc\")\n\t\trequire_NoError(t, err)\n\t\tif ci.Cluster.Leader == \"A-S-2\" {\n\t\t\treturn nil\n\t\t}\n\t\t_, err = lnc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"dlc\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\treturn fmt.Errorf(\"Stream leader not placed on A-S-1\")\n\t})\n\n\t_, err = ljs.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Send 2 messages.\n\tsendStreamMsg(t, lnc, \"foo\", \"M-1\")\n\tsendStreamMsg(t, lnc, \"foo\", \"M-2\")\n\n\t// Now bind apps to cluster B servers and bind to pull consumer.\n\tnc1, _ := jsClientConnect(t, c.servers[0])\n\tdefer nc1.Close()\n\tjs1, err := nc1.JetStream(nats.Domain(\"A\"))\n\trequire_NoError(t, err)\n\n\tsub1, err := js1.PullSubscribe(\"foo\", \"dlc\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\tdefer sub1.Unsubscribe()\n\n\tnc2, _ := jsClientConnect(t, c.servers[1])\n\tdefer nc2.Close()\n\tjs2, err := nc2.JetStream(nats.Domain(\"A\"))\n\trequire_NoError(t, err)\n\n\tsub2, err := js2.PullSubscribe(\"foo\", \"dlc\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\tdefer sub2.Unsubscribe()\n\n\t// Make sure we can properly get messages.\n\tmsgs, err := sub1.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\trequire_True(t, string(msgs[0].Data) == \"M-1\")\n\n\tmsgs, err = sub2.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\trequire_True(t, string(msgs[0].Data) == \"M-2\")\n\n\t// Make sure delivered state makes it to other server to not accidentally send M-2 again\n\t// and fail the test below.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Now let's introduce and event, where A-S-2 will now reconnect after a restart to B-S-2\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tls := lc.servers[1]\n\t\twantedRemote := \"S-1\"\n\t\tvar remoteServer string\n\n\t\tls.mu.RLock()\n\t\tfor _, rc := range ls.leafs {\n\t\t\trc.mu.Lock()\n\t\t\tremoteServer = rc.leaf.remoteServer\n\t\t\trc.mu.Unlock()\n\t\t\tbreak\n\t\t}\n\t\tls.mu.RUnlock()\n\n\t\tif remoteServer != wantedRemote {\n\t\t\tls.Shutdown()\n\t\t\tlc.restartServer(ls)\n\t\t\treturn fmt.Errorf(\"Leafnode server not connected to %q\", wantedRemote)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Wait on ready again.\n\tlc.waitOnClusterReady()\n\tlc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tlc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dlc\")\n\n\t// Send 2 more messages.\n\tsendStreamMsg(t, lnc, \"foo\", \"M-3\")\n\tsendStreamMsg(t, lnc, \"foo\", \"M-4\")\n\n\tmsgs, err = sub1.Fetch(2)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 2)\n\trequire_True(t, string(msgs[0].Data) == \"M-3\")\n\trequire_True(t, string(msgs[1].Data) == \"M-4\")\n\n\t// Send 2 more messages.\n\tsendStreamMsg(t, lnc, \"foo\", \"M-5\")\n\tsendStreamMsg(t, lnc, \"foo\", \"M-6\")\n\n\tmsgs, err = sub2.Fetch(2)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 2)\n\trequire_True(t, string(msgs[0].Data) == \"M-5\")\n\trequire_True(t, string(msgs[1].Data) == \"M-6\")\n}\n\nfunc snapRGSet(pFlag bool, banner string, osi *nats.StreamInfo) *map[string]struct{} {\n\tvar snapSet = make(map[string]struct{})\n\tif pFlag {\n\t\tfmt.Println(banner)\n\t}\n\tif osi == nil {\n\t\tif pFlag {\n\t\t\tfmt.Printf(\"bonkers!\\n\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tsnapSet[osi.Cluster.Leader] = struct{}{}\n\tif pFlag {\n\t\tfmt.Printf(\"Leader: %s\\n\", osi.Cluster.Leader)\n\t}\n\tfor _, replica := range osi.Cluster.Replicas {\n\t\tsnapSet[replica.Name] = struct{}{}\n\t\tif pFlag {\n\t\t\tfmt.Printf(\"Replica: %s\\n\", replica.Name)\n\t\t}\n\t}\n\n\treturn &snapSet\n}\n\nfunc TestJetStreamClusterAfterPeerRemoveZeroState(t *testing.T) {\n\t// R3 scenario (w/messages) in a 4-node cluster. Peer remove from RG and add back to same RG later.\n\t// Validate that original peer brought no memory or issues from its previous RG tour of duty, specifically\n\t// that the restored peer has the correct filestore usage bytes for the asset.\n\tvar err error\n\n\tsc := createJetStreamClusterExplicit(t, \"cl4\", 4)\n\tdefer sc.shutdown()\n\n\tsc.waitOnClusterReadyWithNumPeers(4)\n\n\ts := sc.leader()\n\tnc, jsc := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err = jsc.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(globalAccountName, \"foo\")\n\n\tosi, err := jsc.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\n\t// make sure 0 msgs\n\trequire_True(t, osi.State.Msgs == 0)\n\n\t// load up messages\n\ttoSend := 10000\n\t// storage bytes with JS message overhead\n\tassetStoreBytesExpected := uint64(460000)\n\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := []byte(\"Hello World\")\n\t\tif _, err = jsc.Publish(\"foo.a\", msg); err != nil {\n\t\t\tt.Fatalf(\"unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tosi, err = jsc.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\n\t// make sure 10000 msgs\n\trequire_True(t, osi.State.Msgs == uint64(toSend))\n\n\torigSet := *snapRGSet(false, \"== Orig RG Set ==\", osi)\n\n\t// remove 1 peer replica (1 of 2 non-leaders)\n\torigPeer := osi.Cluster.Replicas[0].Name\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"foo\"), []byte(`{\"peer\":\"`+origPeer+`\"}`), time.Second)\n\trequire_NoError(t, err)\n\tvar rpResp JSApiStreamRemovePeerResponse\n\terr = json.Unmarshal(resp.Data, &rpResp)\n\trequire_NoError(t, err)\n\trequire_True(t, rpResp.Success)\n\n\t// validate the origPeer is removed with a replacement newPeer\n\tsc.waitOnStreamLeader(globalAccountName, \"foo\")\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tosi, err = jsc.StreamInfo(\"foo\")\n\t\trequire_NoError(t, err)\n\t\tif len(osi.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"expected R3, got R%d\", len(osi.Cluster.Replicas)+1)\n\t\t}\n\t\t// STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred\n\t\tfor _, replica := range osi.Cluster.Replicas {\n\t\t\tif replica.Name == origPeer {\n\t\t\t\treturn fmt.Errorf(\"expected replaced replica, old replica still present\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// identify the new peer\n\tvar newPeer string\n\tosi, err = jsc.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\tnewSet := *snapRGSet(false, \"== New RG Set ==\", osi)\n\tfor peer := range newSet {\n\t\t_, ok := origSet[peer]\n\t\tif !ok {\n\t\t\tnewPeer = peer\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_True(t, newPeer != \"\")\n\n\t// kick out newPeer which will cause origPeer to be assigned to the RG again\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"foo\"), []byte(`{\"peer\":\"`+newPeer+`\"}`), time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &rpResp)\n\trequire_NoError(t, err)\n\trequire_True(t, rpResp.Success)\n\n\t// validate the newPeer is removed and R3 has reformed (with origPeer)\n\tsc.waitOnStreamLeader(globalAccountName, \"foo\")\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tosi, err = jsc.StreamInfo(\"foo\")\n\t\trequire_NoError(t, err)\n\t\tif len(osi.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"expected R3, got R%d\", len(osi.Cluster.Replicas)+1)\n\t\t}\n\t\t// STREAM.PEER.REMOVE is asynchronous command; make sure remove has occurred\n\t\tfor _, replica := range osi.Cluster.Replicas {\n\t\t\tif replica.Name == newPeer {\n\t\t\t\treturn fmt.Errorf(\"expected replaced replica, old replica still present\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tosi, err = jsc.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\n\t// make sure all msgs reported in stream at this point with original leader\n\trequire_True(t, osi.State.Msgs == uint64(toSend))\n\n\tsnapRGSet(false, \"== RG Set w/origPeer Back ==\", osi)\n\n\t// get a handle to original peer server\n\tvar origServer *Server = sc.serverByName(origPeer)\n\tif origServer == nil {\n\t\tt.Fatalf(\"expected to get a handle to original peer server by name\")\n\t}\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tjszResult, err := origServer.Jsz(nil)\n\t\trequire_NoError(t, err)\n\t\tif jszResult.Store != assetStoreBytesExpected {\n\t\t\treturn fmt.Errorf(\"expected %d storage on orig peer, got %d\", assetStoreBytesExpected, jszResult.Store)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMemLeaderRestart(t *testing.T) {\n\t// Test if R3 clustered mem store asset leader server restarted, that asset remains stable with final quorum\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, jsc := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t_, err := jsc.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// load up messages\n\ttoSend := 10000\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := []byte(\"Hello World\")\n\t\tif _, err = jsc.Publish(\"foo.a\", msg); err != nil {\n\t\t\tt.Fatalf(\"unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tosi, err := jsc.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\t// make sure 10000 msgs\n\trequire_True(t, osi.State.Msgs == uint64(toSend))\n\n\t// Shutdown the stream leader server\n\trs := c.serverByName(osi.Cluster.Leader)\n\trs.Shutdown()\n\n\t// Make sure that we have a META leader (there can always be a re-election)\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\n\t// Should still have quorum and a new leader\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tosi, err = jsc.StreamInfo(\"foo\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"expected healthy stream asset, got %s\", err.Error())\n\t\t}\n\t\tif osi.Cluster.Leader == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"expected healthy stream asset with new leader\")\n\t\t}\n\t\tif osi.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"expected healthy stream asset %d messages, got %d messages\", toSend, osi.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now restart the old leader peer (old stream state)\n\toldrs := rs\n\trs, _ = RunServerWithConfig(rs.getOpts().ConfigFile)\n\tdefer rs.Shutdown()\n\n\t// Replaced old with new server\n\tfor i := 0; i < len(c.servers); i++ {\n\t\tif c.servers[i] == oldrs {\n\t\t\tc.servers[i] = rs\n\t\t}\n\t}\n\n\t// Wait for cluster to be formed\n\tcheckClusterFormed(t, c.servers...)\n\n\t// Make sure that we have a leader (there can always be a re-election)\n\tc.waitOnLeader()\n\n\t// Can we get stream info after return\n\tosi, err = jsc.StreamInfo(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"expected stream asset info return, got %s\", err.Error())\n\t}\n\n\t// When asset leader came back did we re-form with quorum\n\tif osi.Cluster.Leader == \"\" {\n\t\tt.Fatalf(\"expected a current leader after old leader restarted\")\n\t}\n}\n\n// Customer reported R1 consumers that seemed to be ghosted after server restart.\nfunc TestJetStreamClusterLostConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"GHOST\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfor j := 0; j < 10; j++ {\n\t\t\t_, err := js.Publish(fmt.Sprintf(\"events.%d.%d\", i, j), []byte(\"test\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\ts := c.randomServer()\n\ts.Shutdown()\n\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcc := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tReplicas: 1,\n\t\t},\n\t}\n\treq, err := json.Marshal(cc)\n\trequire_NoError(t, err)\n\n\treqSubj := fmt.Sprintf(JSApiConsumerCreateT, \"TEST\")\n\n\t// Now create 50 consumers. Ensure they are successfully created, so they're included in our snapshot.\n\tfor i := 0; i < 50; i++ {\n\t\t_, err = nc.Request(reqSubj, req, time.Second)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Grab the meta leader.\n\tml := c.leader()\n\trequire_NoError(t, ml.JetStreamSnapshotMeta())\n\n\tnumConsumerAssignments := func(s *Server) int {\n\t\tt.Helper()\n\t\tjs := s.getJetStream()\n\t\tjs.mu.RLock()\n\t\tdefer js.mu.RUnlock()\n\t\tcc := js.cluster\n\t\tfor _, asa := range cc.streams {\n\t\t\tfor _, sa := range asa {\n\t\t\t\treturn len(sa.consumers)\n\t\t\t}\n\t\t}\n\t\treturn 0\n\t}\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tnum := numConsumerAssignments(ml)\n\t\tif num == 50 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Consumers is only %d\", num)\n\t})\n\n\t// Restart the server we shutdown. We snapshotted to the snapshot\n\t// has to fill in the new consumers.\n\t// The bug would fail to add them to the meta state since the stream\n\t// existed.\n\ts = c.restartServer(s)\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tnum := numConsumerAssignments(s)\n\t\tif num == 50 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Consumers is only %d\", num)\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/3636\nfunc TestJetStreamClusterScaleDownDuringServerOffline(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"hello\")\n\t}\n\n\ts := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\ts.Shutdown()\n\n\tc.waitOnLeader()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\ts = c.restartServer(s)\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(nil)\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Reported by a customer manually upgrading their streams to support direct gets.\n// Worked if single replica but not in clustered mode.\nfunc TestJetStreamClusterDirectGetStreamUpgrade(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"KV_TEST\",\n\t\tSubjects: []string{\"$KV.TEST.>\"},\n\t\tDiscard: nats.DiscardNew,\n\t\tMaxMsgsPerSubject: 1,\n\t\tDenyDelete: true,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tkv, err := js.KeyValue(\"TEST\")\n\trequire_NoError(t, err)\n\n\t_, err = kv.PutString(\"name\", \"derek\")\n\trequire_NoError(t, err)\n\n\tentry, err := kv.Get(\"name\")\n\trequire_NoError(t, err)\n\trequire_True(t, string(entry.Value()) == \"derek\")\n\n\t// Now simulate a update to the stream to support direct gets.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"KV_TEST\",\n\t\tSubjects: []string{\"$KV.TEST.>\"},\n\t\tDiscard: nats.DiscardNew,\n\t\tMaxMsgsPerSubject: 1,\n\t\tDenyDelete: true,\n\t\tAllowDirect: true,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Rebind to KV to make sure we DIRECT version of Get().\n\tkv, err = js.KeyValue(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Make sure direct get works.\n\tentry, err = kv.Get(\"name\")\n\trequire_NoError(t, err)\n\trequire_True(t, string(entry.Value()) == \"derek\")\n}\n\n// For interest (or workqueue) based streams its important to match the replication factor.\n// This was the case but now that more control over consumer creation is allowed its possible\n// to create a consumer where the replication factor does not match. This could cause\n// instability in the state between servers and cause problems on leader switches.\nfunc TestJetStreamClusterInterestPolicyStreamForConsumersToMatchRFactor(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"XX\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 1,\n\t})\n\n\trequire_Error(t, err, NewJSConsumerReplicasShouldMatchStreamError())\n}\n\n// https://github.com/nats-io/nats-server/issues/3791\nfunc TestJetStreamClusterKVWatchersWithServerDown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tkv.PutString(\"foo\", \"bar\")\n\tkv.PutString(\"foo\", \"baz\")\n\n\t// Shutdown a follower.\n\ts := c.randomNonStreamLeader(globalAccountName, \"KV_TEST\")\n\ts.Shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjs, err = nc.JetStream(nats.MaxWait(2 * time.Second))\n\trequire_NoError(t, err)\n\n\tkv, err = js.KeyValue(\"TEST\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tw, err := kv.Watch(\"foo\")\n\t\trequire_NoError(t, err)\n\t\tw.Stop()\n\t}\n}\n\n// TestJetStreamClusterCurrentVsHealth is designed to show the\n// difference between \"current\" and \"healthy\" when async publishes\n// outpace the rate at which they can be applied.\nfunc TestJetStreamClusterCurrentVsHealth(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\tserver := c.randomNonLeader()\n\n\tnc, js := jsClientConnect(t, server)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tserver = c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tstream, err := server.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\traft, ok := stream.raftGroup().node.(*raft)\n\trequire_True(t, ok)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_, err := js.PublishAsync(\"foo\", []byte(\"bar\"))\n\t\trequire_NoError(t, err)\n\n\t\traft.RLock()\n\t\tcommit := raft.commit\n\t\tapplied := raft.applied\n\t\traft.RUnlock()\n\n\t\tcurrent := raft.Current()\n\t\thealthy := raft.Healthy()\n\n\t\tif !current || !healthy || commit != applied {\n\t\t\tt.Logf(\n\t\t\t\t\"%d | Current %v, healthy %v, commit %d, applied %d, pending %d\",\n\t\t\t\ti, current, healthy, commit, applied, commit-applied,\n\t\t\t)\n\t\t}\n\t}\n}\n\n// Several users and customers use this setup, but many times across leafnodes.\n// This should be allowed in same account since we are really protecting against\n// multiple pub acks with cycle detection.\nfunc TestJetStreamClusterActiveActiveSourcedStreams(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"A\",\n\t\tSubjects: []string{\"A.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"B\",\n\t\tSubjects: []string{\"B.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"A\",\n\t\tSubjects: []string{\"A.>\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"B\",\n\t\t\tFilterSubject: \"B.>\",\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Before this would fail.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"B\",\n\t\tSubjects: []string{\"B.>\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"A\",\n\t\t\tFilterSubject: \"A.>\",\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterUpdateConsumerShouldNotForceDeleteOnRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R7S\", 7)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"D\",\n\t\tDeliverSubject: \"_no_bind_\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Shutdown a consumer follower.\n\tnc.Close()\n\ts := c.serverByName(ci.Cluster.Replicas[0].Name)\n\ts.Shutdown()\n\n\tc.waitOnLeader()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Change delivery subject.\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"D\",\n\t\tDeliverSubject: \"_d_\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Create interest in new and old deliver subject.\n\t_, err = nc.SubscribeSync(\"_d_\")\n\trequire_NoError(t, err)\n\t_, err = nc.SubscribeSync(\"_no_bind_\")\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\tc.restartServer(s)\n\tc.waitOnAllCurrent()\n\n\t// Wait on bad error that would cleanup consumer.\n\ttime.Sleep(time.Second)\n\n\t_, err = js.ConsumerInfo(\"TEST\", \"D\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterInterestPolicyEphemeral(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tfor _, test := range []struct {\n\t\ttestName string\n\t\tstream string\n\t\tsubject string\n\t\tdurable string\n\t\tname string\n\t\tpolicy nats.RetentionPolicy\n\t}{\n\t\t{testName: \"LimitsWithName\", name: \"eph\", subject: \"limeph\", stream: \"LIMIT_EPH\", policy: nats.LimitsPolicy},\n\t\t{testName: \"InterestWithDurable\", durable: \"eph\", subject: \"intdur\", stream: \"INT_DUR\", policy: nats.InterestPolicy},\n\t\t{testName: \"InterestWithName\", name: \"eph\", subject: \"inteph\", stream: \"INT_EPH\", policy: nats.InterestPolicy},\n\t} {\n\t\tt.Run(test.testName, func(t *testing.T) {\n\t\t\tvar err error\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: test.stream,\n\t\t\t\tSubjects: []string{test.subject},\n\t\t\t\tRetention: test.policy,\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tconst inactiveThreshold = time.Second\n\n\t\t\t_, err = js.AddConsumer(test.stream, &nats.ConsumerConfig{\n\t\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\tInactiveThreshold: inactiveThreshold,\n\t\t\t\tDurable: test.durable,\n\t\t\t\tName: test.name,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tname := test.durable\n\t\t\tif test.durable == _EMPTY_ {\n\t\t\t\tname = test.name\n\t\t\t}\n\n\t\t\tconst msgs = 5_000\n\t\t\tsub, err := js.Subscribe(_EMPTY_, func(msg *nats.Msg) {\n\t\t\t\trequire_NoError(t, msg.Ack())\n\t\t\t}, nats.Bind(test.stream, name), nats.ManualAck())\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// This happens only if we start publishing messages after consumer was created.\n\t\t\tpubErr := make(chan error, 1)\n\t\t\tpubDone := make(chan struct{})\n\t\t\tgo func(subject string) {\n\t\t\t\tdefer close(pubDone)\n\t\t\t\tfor i := 0; i < msgs; i++ {\n\t\t\t\t\t_, err := js.Publish(subject, []byte(\"DATA\"))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase pubErr <- err:\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(test.subject)\n\n\t\t\t// Wait for inactive threshold to expire and all messages to be published and received\n\t\t\t// Bug is we clean up active consumers when we should not.\n\t\t\ttime.Sleep(3 * inactiveThreshold / 2)\n\n\t\t\tselect {\n\t\t\tcase <-pubDone:\n\t\t\tcase <-time.After(10 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\tselect {\n\t\t\tcase err := <-pubErr:\n\t\t\t\tt.Fatalf(\"Publish error: %v\", err)\n\t\t\tdefault:\n\t\t\t}\n\n\t\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tinfo, err := js.ConsumerInfo(test.stream, name)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"Expected to be able to retrieve consumer: %v\", err)\n\t\t\t\t}\n\t\t\t\tif info.Delivered.Stream != msgs {\n\t\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", info.Delivered.Stream, msgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Stop the subscription and remove the interest.\n\t\t\terr = sub.Unsubscribe()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Now wait for interest inactivity threshold to kick in.\n\t\t\ttime.Sleep(3 * inactiveThreshold / 2)\n\n\t\t\t// Check if the consumer has been removed.\n\t\t\t_, err = js.ConsumerInfo(test.stream, name)\n\t\t\trequire_Error(t, err, nats.ErrConsumerNotFound)\n\t\t})\n\t}\n}\n\n// TestJetStreamClusterWALBuildupOnNoOpPull tests whether or not the consumer\n// RAFT log is being compacted when the stream is idle but we are performing\n// lots of fetches. Otherwise the disk usage just spirals out of control if\n// there are no other state changes to trigger a compaction.\nfunc TestJetStreamClusterWALBuildupOnNoOpPull(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\n\t\t\"foo\",\n\t\t\"durable\",\n\t\tnats.ConsumerReplicas(3),\n\t)\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10000; i++ {\n\t\t_, _ = sub.Fetch(1, nats.MaxWait(time.Microsecond))\n\t}\n\n\t// Needs to be at least 10 seconds, otherwise we won't hit the\n\t// minSnapDelta that prevents us from snapshotting too often\n\ttime.Sleep(time.Second * 11)\n\n\tfor i := 0; i < 1024; i++ {\n\t\t_, _ = sub.Fetch(1, nats.MaxWait(time.Microsecond))\n\t}\n\n\ttime.Sleep(time.Second)\n\n\tserver := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"durable\")\n\n\tstream, err := server.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tconsumer := stream.lookupConsumer(\"durable\")\n\trequire_NotNil(t, consumer)\n\n\tentries, bytes := consumer.raftNode().Size()\n\tt.Log(\"new entries:\", entries)\n\tt.Log(\"new bytes:\", bytes)\n\n\tif max := uint64(1024); entries > max {\n\t\tt.Fatalf(\"got %d entries, expected less than %d entries\", entries, max)\n\t}\n}\n\n// Found in https://github.com/nats-io/nats-server/issues/3848\n// When Max Age was specified and stream was scaled up, new replicas\n// were expiring messages much later than the leader.\nfunc TestJetStreamClusterStreamMaxAgeScaleUp(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tstorage nats.StorageType\n\t\tstream string\n\t\tpurge bool\n\t}{\n\t\t{name: \"file\", storage: nats.FileStorage, stream: \"A\", purge: false},\n\t\t{name: \"memory\", storage: nats.MemoryStorage, stream: \"B\", purge: false},\n\t\t{name: \"file with purge\", storage: nats.FileStorage, stream: \"C\", purge: true},\n\t\t{name: \"memory with purge\", storage: nats.MemoryStorage, stream: \"D\", purge: true},\n\t} {\n\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tttl := time.Second * 5\n\t\t\t// Add stream with one replica and short MaxAge.\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: test.stream,\n\t\t\t\tReplicas: 1,\n\t\t\t\tSubjects: []string{test.stream},\n\t\t\t\tMaxAge: ttl,\n\t\t\t\tStorage: test.storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Add some messages.\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsendStreamMsg(t, nc, test.stream, \"HELLO\")\n\t\t\t}\n\t\t\t// We need to also test if we properly set expiry\n\t\t\t// if first sequence is not 1.\n\t\t\tif test.purge {\n\t\t\t\terr = js.PurgeStream(test.stream)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// Add some messages.\n\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\tsendStreamMsg(t, nc, test.stream, \"HELLO\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Mark the time when all messages were published.\n\t\t\tstart := time.Now()\n\n\t\t\t// Sleep for half of the MaxAge time.\n\t\t\ttime.Sleep(ttl / 2)\n\n\t\t\t// Scale up the Stream to 3 replicas.\n\t\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\tName: test.stream,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjects: []string{test.stream},\n\t\t\t\tMaxAge: ttl,\n\t\t\t\tStorage: test.storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tc.waitOnStreamLeader(globalAccountName, test.stream)\n\n\t\t\t// All messages should still be there.\n\t\t\tinfo, err := js.StreamInfo(test.stream)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, info.State.Msgs, 10)\n\n\t\t\t// Wait until MaxAge is reached.\n\t\t\ttime.Sleep(ttl - time.Since(start) + (1 * time.Second))\n\n\t\t\t// Check if all messages are expired.\n\t\t\tinfo, err = js.StreamInfo(test.stream)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, info.State.Msgs, 0)\n\n\t\t\t// Now switch leader to one of replicas\n\t\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, test.stream), nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tc.waitOnStreamLeader(globalAccountName, test.stream)\n\n\t\t\t// and make sure that it also expired all messages\n\t\t\tinfo, err = js.StreamInfo(test.stream)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, info.State.Msgs, 0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterWorkQueueConsumerReplicatedAfterScaleUp(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 1,\n\t\tSubjects: []string{\"WQ\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create an ephemeral consumer.\n\tsub, err := js.SubscribeSync(\"WQ\")\n\trequire_NoError(t, err)\n\n\t// Scale up to R3.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"WQ\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\trequire_True(t, ci.Config.Replicas == 0 || ci.Config.Replicas == 3)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", ci.Name)\n\ts := c.consumerLeader(globalAccountName, \"TEST\", ci.Name)\n\trequire_NotNil(t, s)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\to := mset.lookupConsumer(ci.Name)\n\trequire_NotNil(t, o)\n\trequire_NotNil(t, o.raftNode())\n}\n\n// https://github.com/nats-io/nats-server/issues/3953\nfunc TestJetStreamClusterWorkQueueAfterScaleUp(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 1,\n\t\tSubjects: []string{\"WQ\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"d1\",\n\t\tDeliverSubject: \"d1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\twch := make(chan bool, 1)\n\t_, err = nc.Subscribe(\"d1\", func(msg *nats.Msg) {\n\t\tmsg.AckSync()\n\t\twch <- true\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"WQ\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tsendStreamMsg(t, nc, \"WQ\", \"SOME WORK\")\n\n\tselect {\n\tcase <-wch:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive ack signal\")\n\t}\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Still have %d msgs left\", si.State.Msgs)\n\t})\n}\n\nfunc TestJetStreamClusterInterestBasedStreamAndConsumerSnapshots(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Durable(\"d22\"))\n\trequire_NoError(t, err)\n\n\tnum := 200\n\tfor i := 0; i < num; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, num)\n\n\t// Shutdown one server.\n\ts := c.randomServer()\n\ts.Shutdown()\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now ack all messages while the other server is down.\n\tfor i := 0; i < num; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tm.AckSync()\n\t}\n\n\t// Wait for all message acks to be processed and all messages to be removed.\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Still have %d msgs left\", si.State.Msgs)\n\t})\n\n\t// Force a snapshot on the consumer leader before restarting the downed server.\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"d22\")\n\trequire_NotNil(t, cl)\n\n\tmset, err := cl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\to := mset.lookupConsumer(\"d22\")\n\trequire_NotNil(t, o)\n\n\tsnap, err := o.store.EncodedState()\n\trequire_NoError(t, err)\n\n\tn := o.raftNode()\n\trequire_NotNil(t, n)\n\trequire_NoError(t, n.InstallSnapshot(snap, false))\n\n\t// Now restart the downed server.\n\ts = c.restartServer(s)\n\n\t// Make the restarted server the eventual leader.\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tif sl := c.streamLeader(globalAccountName, \"TEST\"); sl != s {\n\t\t\tsl.JetStreamStepdownStream(globalAccountName, \"TEST\")\n\t\t\treturn fmt.Errorf(\"Server %s is not leader yet\", s)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 0)\n}\n\nfunc TestJetStreamClusterConsumerFollowerStoreStateAckFloorBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"C\", nats.BindStream(\"TEST\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\tnum := 100\n\tfor i := 0; i < num; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\t}\n\n\t// This one prevents the state for pending from reaching 0 and resetting, which would not show the bug.\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\n\t// Ack all but one and out of order and make sure all consumers have the same stored state.\n\tmsgs, err := sub.Fetch(num, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == num)\n\n\t_, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\n\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\tfor _, m := range msgs {\n\t\tif err := m.AckSync(); err != nil {\n\t\t\tt.Fatalf(\"Ack failed :%+v\", err)\n\t\t}\n\t}\n\n\tcheckConsumerState := func(delivered, ackFloor nats.SequenceInfo, numAckPending int) error {\n\t\texpectedDelivered := uint64(num) + 1\n\t\tif delivered.Stream != expectedDelivered || delivered.Consumer != expectedDelivered {\n\t\t\treturn fmt.Errorf(\"Wrong delivered, expected %d got %+v\", expectedDelivered, delivered)\n\t\t}\n\t\texpectedAck := uint64(num)\n\t\tif ackFloor.Stream != expectedAck || ackFloor.Consumer != expectedAck {\n\t\t\treturn fmt.Errorf(\"Wrong ackFloor, expected %d got %+v\", expectedAck, ackFloor)\n\t\t}\n\t\tif numAckPending != 1 {\n\t\t\treturn errors.New(\"Expected num ack pending to be 1\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"C\")\n\trequire_NoError(t, err)\n\trequire_NoError(t, checkConsumerState(ci.Delivered, ci.AckFloor, ci.NumAckPending))\n\n\t// Check each consumer on each server for it's store state and make sure it matches as well.\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif mset == nil {\n\t\t\t\treturn errors.New(\"Mset should not be nil\")\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"C\")\n\t\t\tif o == nil {\n\t\t\t\treturn errors.New(\"Consumer should not be nil\")\n\t\t\t}\n\n\t\t\tstate, err := o.store.State()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdelivered := nats.SequenceInfo{Stream: state.Delivered.Stream, Consumer: state.Delivered.Consumer}\n\t\t\tackFloor := nats.SequenceInfo{Stream: state.AckFloor.Stream, Consumer: state.AckFloor.Consumer}\n\t\t\tif err := checkConsumerState(delivered, ackFloor, len(state.Pending)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now stepdown the consumer and move its leader and check the state after transition.\n\t// Make the restarted server the eventual leader.\n\tseen := make(map[*Server]bool)\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"C\")\n\trequire_NotNil(t, cl)\n\tseen[cl] = true\n\n\tallSeen := func() bool {\n\t\tfor _, s := range c.servers {\n\t\t\tif !seen[s] {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}\n\n\tcheckAllLeaders := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"C\")\n\t\t\tif allSeen() {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"C\")\n\t\t\tseen[cl] = true\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"C\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := checkConsumerState(ci.Delivered, ci.AckFloor, ci.NumAckPending); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tcl.JetStreamStepdownConsumer(globalAccountName, \"TEST\", \"C\")\n\t\t\treturn fmt.Errorf(\"Not all servers have been consumer leader yet\")\n\t\t})\n\t}\n\n\tcheckAllLeaders()\n\n\t// No restart all servers and check again.\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tseen = make(map[*Server]bool)\n\tcheckAllLeaders()\n}\n\nfunc TestJetStreamClusterInterestLeakOnDisableJetStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.leader())\n\tdefer nc.Close()\n\n\tfor i := 1; i <= 5; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"test_%d\", i),\n\t\t\tSubjects: []string{fmt.Sprintf(\"test_%d\", i)},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tc.waitOnAllCurrent()\n\n\tserver := c.randomNonLeader()\n\taccount := server.SystemAccount()\n\n\tserver.DisableJetStream()\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tvar sublist []*subscription\n\t\taccount.sl.localSubs(&sublist, false)\n\n\t\tvar danglingJSC, danglingRaft int\n\t\tfor _, sub := range sublist {\n\t\t\tif strings.HasPrefix(string(sub.subject), \"$JSC.\") {\n\t\t\t\tdanglingJSC++\n\t\t\t} else if strings.HasPrefix(string(sub.subject), \"$NRG.\") {\n\t\t\t\tdanglingRaft++\n\t\t\t}\n\t\t}\n\t\tif danglingJSC > 0 || danglingRaft > 0 {\n\t\t\treturn fmt.Errorf(\"unexpected dangling interests for JetStream assets after shutdown (%d $JSC, %d $NRG)\", danglingJSC, danglingRaft)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterNoLeadersDuringLameDuck(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Grab the first server and set lameduck option directly.\n\ts := c.servers[0]\n\ts.optsMu.Lock()\n\ts.opts.LameDuckDuration = 5 * time.Second\n\ts.opts.LameDuckGracePeriod = -5 * time.Second\n\ts.optsMu.Unlock()\n\n\t// Connect to the third server.\n\tnc, js := jsClientConnect(t, c.servers[2])\n\tdefer nc.Close()\n\n\tallServersHaveLeaders := func() bool {\n\t\thaveLeader := make(map[*Server]bool)\n\t\tfor _, s := range c.servers {\n\t\t\ts.rnMu.RLock()\n\t\t\tfor _, n := range s.raftNodes {\n\t\t\t\tif n.Leader() {\n\t\t\t\t\thaveLeader[s] = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.rnMu.RUnlock()\n\t\t}\n\t\treturn len(haveLeader) == len(c.servers)\n\t}\n\n\t// Create streams until we have a leader on all the servers.\n\tvar index int\n\tcheckFor(t, 10*time.Second, time.Millisecond, func() error {\n\t\tif allServersHaveLeaders() {\n\t\t\treturn nil\n\t\t}\n\t\tindex++\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST_%d\", index),\n\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d\", index)},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\treturn fmt.Errorf(\"All servers do not have at least one leader\")\n\t})\n\n\t// Put our server into lameduck mode.\n\t// Need a client.\n\tdummy, _ := jsClientConnect(t, s)\n\tdefer dummy.Close()\n\tgo s.lameDuckMode()\n\n\t// Wait for all leaders to move off.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\ts.rnMu.RLock()\n\t\tdefer s.rnMu.RUnlock()\n\t\tfor _, n := range s.raftNodes {\n\t\t\tif n.Leader() {\n\t\t\t\treturn fmt.Errorf(\"Server still has a leader\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// All leader evacuated.\n\n\t// Create a go routine that will create streams constantly.\n\tqch := make(chan bool)\n\tgo func() {\n\t\tvar index int\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-time.After(time.Millisecond):\n\t\t\t\tindex++\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"NEW_TEST_%d\", index),\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"bar.%d\", index)},\n\t\t\t\t\tReplicas: 3,\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\tcase <-qch:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\tdefer close(qch)\n\n\t// Make sure we do not have any leaders placed on the lameduck server.\n\tfor s.isRunning() {\n\t\tvar hasLeader bool\n\t\ts.rnMu.RLock()\n\t\tfor _, n := range s.raftNodes {\n\t\t\thasLeader = hasLeader || n.Leader()\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t\tif hasLeader {\n\t\t\tt.Fatalf(\"Server had a leader when it should not due to lameduck mode\")\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterNoR1AssetsDuringLameDuck(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Grab the first server and set lameduck option directly.\n\ts := c.servers[0]\n\ts.optsMu.Lock()\n\ts.opts.LameDuckDuration = 5 * time.Second\n\ts.opts.LameDuckGracePeriod = -5 * time.Second\n\ts.optsMu.Unlock()\n\n\t// Connect to the server to keep it alive when we go into LDM.\n\tdummy, _ := jsClientConnect(t, s)\n\tdefer dummy.Close()\n\n\t// Connect to the third server.\n\tnc, js := jsClientConnect(t, c.servers[2])\n\tdefer nc.Close()\n\n\t// Now put the first server into lame duck mode.\n\tgo s.lameDuckMode()\n\n\t// Wait for news to arrive that the first server has gone into\n\t// lame duck mode and been marked offline.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tid := s.info.ID\n\t\ts := c.servers[2]\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\n\t\tvar isOffline bool\n\t\ts.nodeToInfo.Range(func(_, v any) bool {\n\t\t\tni := v.(nodeInfo)\n\t\t\tif ni.id == id {\n\t\t\t\tisOffline = ni.offline\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\n\t\tif !isOffline {\n\t\t\treturn fmt.Errorf(\"first node is still online unexpectedly\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Create a go routine that will create streams constantly.\n\tqch := make(chan bool)\n\tgo func() {\n\t\tvar index int\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-time.After(time.Millisecond * 25):\n\t\t\t\tindex++\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"NEW_TEST_%d\", index),\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"bar.%d\", index)},\n\t\t\t\t\tReplicas: 1,\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\tcase <-qch:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\tdefer close(qch)\n\n\tgacc := s.GlobalAccount()\n\tif gacc == nil {\n\t\tt.Fatalf(\"No global account\")\n\t}\n\t// Make sure we do not have any R1 assets placed on the lameduck server.\n\tfor s.isRunning() {\n\t\tif len(gacc.streams()) > 0 {\n\t\t\tt.Fatalf(\"Server had an R1 asset when it should not due to lameduck mode\")\n\t\t}\n\t\ttime.Sleep(15 * time.Millisecond)\n\t}\n\ts.WaitForShutdown()\n}\n\n// If a consumer has not been registered (possible in heavily loaded systems with lots of assets)\n// it could miss the signal of a message going away. If that message was pending and expires the\n// ack floor could fall below the stream first sequence. This test will force that condition and\n// make sure the system resolves itself.\nfunc TestJetStreamClusterConsumerAckFloorDrift(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t\tMaxAge: time.Second,\n\t\tMaxMsgs: 10,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"C\")\n\trequire_NoError(t, err)\n\n\t// Publish as many messages as the ack floor check threshold +5 (what we set ackfloor to later).\n\ttotalMessages := 55\n\tfor i := 0; i < totalMessages; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"HELLO\")\n\t}\n\n\t// No-op but will surface as delivered.\n\t_, err = sub.Fetch(10)\n\trequire_NoError(t, err)\n\n\t// We will initialize the state with delivered being 10 and ackfloor being 0 directly.\n\t// Fetch will asynchronously propagate this state, so can't reliably request this from the leader immediately.\n\tstate := &ConsumerState{Delivered: SequencePair{Consumer: 10, Stream: 10}}\n\n\t// Now let messages expire.\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"stream still has msgs\")\n\t})\n\n\t// Set state to ackfloor of 5 and no pending.\n\tstate.AckFloor.Consumer = 5\n\tstate.AckFloor.Stream = 5\n\tstate.Pending = nil\n\n\t// Now put back the state underneath of the consumers.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"C\")\n\t\trequire_NotNil(t, o)\n\t\to.mu.Lock()\n\t\to.applyState(state)\n\t\tcfs := o.store.(*consumerFileStore)\n\t\to.mu.Unlock()\n\t\t// The lower layer will ignore, so set more directly.\n\t\tcfs.mu.Lock()\n\t\tcfs.state = *state\n\t\tcfs.mu.Unlock()\n\t\t// Also snapshot to remove any raft entries that could affect it.\n\t\tsnap, err := o.store.EncodedState()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, o.raftNode().InstallSnapshot(snap, false))\n\t}\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"C\")\n\trequire_NotNil(t, cl)\n\terr = cl.JetStreamStepdownConsumer(globalAccountName, \"TEST\", \"C\")\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"C\")\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"C\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Replicated state should stay the same.\n\t\tif ci.AckFloor.Stream != 5 && ci.AckFloor.Consumer != 5 {\n\t\t\treturn fmt.Errorf(\"replicated AckFloor not correct, expected %d, got %+v\", 5, ci.AckFloor)\n\t\t}\n\n\t\tcl = c.consumerLeader(globalAccountName, \"TEST\", \"C\")\n\t\tmset, err := cl.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"C\")\n\t\trequire_NotNil(t, o)\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\n\t\t// Make sure we catch this and adjust.\n\t\tif o.asflr != uint64(totalMessages) && o.adflr != 10 {\n\t\t\treturn fmt.Errorf(\"leader AckFloor not correct, expected %d, got %+v\", 10, ci.AckFloor)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterInterestStreamFilteredConsumersWithNoInterest(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create three subscribers.\n\tackCb := func(m *nats.Msg) { m.Ack() }\n\n\t_, err = js.Subscribe(\"foo\", ackCb, nats.BindStream(\"TEST\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t_, err = js.Subscribe(\"bar\", ackCb, nats.BindStream(\"TEST\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t_, err = js.Subscribe(\"baz\", ackCb, nats.BindStream(\"TEST\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Now send 100 messages, randomly picking foo or bar, but never baz.\n\tfor i := 0; i < 100; i++ {\n\t\tif rand.Intn(2) > 0 {\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"HELLO\")\n\t\t} else {\n\t\t\tsendStreamMsg(t, nc, \"bar\", \"WORLD\")\n\t\t}\n\t}\n\n\t// Messages are expected to go to 0.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"stream still has msgs\")\n\t})\n}\n\nfunc TestJetStreamClusterChangeClusterAfterStreamCreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"HELLO\")\n\t}\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tc.stopAll()\n\n\tc.name = \"FOO\"\n\tfor _, o := range c.opts {\n\t\tbuf, err := os.ReadFile(o.ConfigFile)\n\t\trequire_NoError(t, err)\n\t\tnbuf := bytes.Replace(buf, []byte(\"name: NATS\"), []byte(\"name: FOO\"), 1)\n\t\terr = os.WriteFile(o.ConfigFile, nbuf, 0640)\n\t\trequire_NoError(t, err)\n\t}\n\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\t// This should fail with no suitable peers, since the asset was created under the NATS cluster which has no peers.\n\trequire_Error(t, err, errors.New(\"nats: no suitable peers for placement\"))\n\n\t// Make sure we can swap the cluster.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tPlacement: &nats.Placement{Cluster: \"FOO\"},\n\t})\n\trequire_NoError(t, err)\n}\n\n// The consumer info() call does not take into account whether a consumer\n// is a leader or not, so results would be very different when asking servers\n// that housed consumer followers vs leaders.\nfunc TestJetStreamClusterConsumerInfoForJszForFollowers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"HELLO\")\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"d\")\n\trequire_NoError(t, err)\n\n\tfetch, ack := 122, 22\n\tmsgs, err := sub.Fetch(fetch, nats.MaxWait(10*time.Second))\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == fetch)\n\tfor _, m := range msgs[:ack] {\n\t\tm.AckSync()\n\t}\n\t// Let acks propagate.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tfor _, s := range c.servers {\n\t\tjsz, err := s.Jsz(&JSzOptions{Accounts: true, Consumer: true})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(jsz.AccountDetails) == 1)\n\t\trequire_True(t, len(jsz.AccountDetails[0].Streams) == 1)\n\t\trequire_True(t, len(jsz.AccountDetails[0].Streams[0].Consumer) == 1)\n\t\tconsumer := jsz.AccountDetails[0].Streams[0].Consumer[0]\n\t\tif consumer.Delivered.Consumer != uint64(fetch) || consumer.Delivered.Stream != uint64(fetch) {\n\t\t\tt.Fatalf(\"Incorrect delivered for %v: %+v\", s, consumer.Delivered)\n\t\t}\n\t\tif consumer.AckFloor.Consumer != uint64(ack) || consumer.AckFloor.Stream != uint64(ack) {\n\t\t\tt.Fatalf(\"Incorrect ackfloor for %v: %+v\", s, consumer.AckFloor)\n\t\t}\n\t}\n}\n\n// Make sure that stopping a stream shutdowns down it's raft node.\nfunc TestJetStreamClusterStreamNodeShutdownBugOnStop(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"HELLO\")\n\t}\n\n\ts := c.randomServer()\n\tnumNodesStart := s.numRaftNodes()\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\trequire_NotNil(t, node)\n\tnode.InstallSnapshot(mset.stateSnapshot(), false)\n\t// Stop the stream\n\tmset.stop(false, false)\n\tnode.WaitForStop()\n\n\tif numNodes := s.numRaftNodes(); numNodes != numNodesStart-1 {\n\t\tt.Fatalf(\"RAFT nodes after stream stop incorrect: %d vs %d\", numNodesStart, numNodes)\n\t}\n}\n\nfunc TestJetStreamClusterStreamAccountingOnStoreError(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := strings.Repeat(\"Z\", 32*1024)\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", msg)\n\t}\n\ts := c.randomServer()\n\tacc, err := s.LookupAccount(\"$U\")\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.Lock()\n\tmset.store.Stop()\n\tsjs := mset.js\n\tmset.mu.Unlock()\n\n\t// Now delete the stream\n\tjs.DeleteStream(\"TEST\")\n\n\t// Wait for this to propgate.\n\t// The bug will have us not release reserved resources properly.\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tinfo, err := js.AccountInfo()\n\t\trequire_NoError(t, err)\n\t\t// Default tier\n\t\tif info.Store != 0 {\n\t\t\treturn fmt.Errorf(\"Expected store to be 0 but got %v\", friendlyBytes(info.Store))\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now check js from server directly regarding reserved.\n\tsjs.mu.RLock()\n\treserved := sjs.storeReserved\n\tsjs.mu.RUnlock()\n\t// Under bug will show 1GB\n\tif reserved != 0 {\n\t\tt.Fatalf(\"Expected store reserved to be 0 after stream delete, got %v\", friendlyBytes(reserved))\n\t}\n}\n\nfunc TestJetStreamClusterStreamAccountingDriftFixups(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterMaxBytesAccountLimitTempl, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxBytes: 2 * 1024 * 1024,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := strings.Repeat(\"Z\", 32*1024)\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", msg)\n\t}\n\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tinfo, err := js.AccountInfo()\n\t\trequire_NoError(t, err)\n\t\tif info.Store != 0 {\n\t\t\treturn fmt.Errorf(\"Store usage not 0: %d\", info.Store)\n\t\t}\n\t\treturn nil\n\t})\n\n\ts := c.leader()\n\tjsz, err := s.Jsz(nil)\n\trequire_NoError(t, err)\n\trequire_True(t, jsz.JetStreamStats.Store == 0)\n\n\tacc, err := s.LookupAccount(\"$U\")\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tjsa, tier, stype := mset.jsa, mset.tier, mset.stype\n\tmset.mu.RUnlock()\n\t// Drift the usage.\n\tjsa.updateUsage(tier, stype, -100)\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tinfo, err := js.AccountInfo()\n\t\trequire_NoError(t, err)\n\t\tif info.Store != 0 {\n\t\t\treturn fmt.Errorf(\"Store usage not 0: %d\", info.Store)\n\t\t}\n\t\treturn nil\n\t})\n\tjsz, err = s.Jsz(nil)\n\trequire_NoError(t, err)\n\trequire_True(t, jsz.JetStreamStats.Store == 0)\n}\n\n// Some older streams seem to have been created or exist with no explicit cluster setting.\n// For server <= 2.9.16 you could not scale the streams up since we could not place them in another cluster.\nfunc TestJetStreamClusterStreamScaleUpNoGroupCluster(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Manually going to grab stream assignment and update it to be without the group cluster.\n\ts := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tsa := mset.streamAssignment()\n\trequire_NotNil(t, sa)\n\t// Make copy to not change stream's\n\tsa = sa.copyGroup()\n\t// Remove cluster and preferred.\n\tsa.Group.Cluster = _EMPTY_\n\tsa.Group.Preferred = _EMPTY_\n\t// Insert into meta layer.\n\tif sjs := s.getJetStream(); sjs != nil {\n\t\tsjs.mu.RLock()\n\t\tmeta := sjs.cluster.meta\n\t\tsjs.mu.RUnlock()\n\t\tif meta != nil {\n\t\t\tmeta.ForwardProposal(encodeUpdateStreamAssignment(sa))\n\t\t}\n\t}\n\t// Make sure it got propagated..\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tsa := mset.streamAssignment().copyGroup()\n\t\trequire_NotNil(t, sa)\n\t\tif sa.Group.Cluster != _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"Cluster still not cleared\")\n\t\t}\n\t\treturn nil\n\t})\n\t// Now we know it has been nil'd out. Make sure we can scale up.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n}\n\n// https://github.com/nats-io/nats-server/issues/4162\nfunc TestJetStreamClusterStaleDirectGetOnRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = kv.PutString(\"foo\", \"bar\")\n\trequire_NoError(t, err)\n\n\t// Close client in case we were connected to server below.\n\t// We will recreate.\n\tnc.Close()\n\n\t// Shutdown a non-leader.\n\ts := c.randomNonStreamLeader(globalAccountName, \"KV_TEST\")\n\ts.Shutdown()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err = js.KeyValue(\"TEST\")\n\trequire_NoError(t, err)\n\n\t_, err = kv.PutString(\"foo\", \"baz\")\n\trequire_NoError(t, err)\n\n\terrCh := make(chan error, 100)\n\tdone := make(chan struct{})\n\n\tgo func() {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tkv, err := js.KeyValue(\"TEST\")\n\t\tif err != nil {\n\t\t\terrCh <- err\n\t\t\treturn\n\t\t}\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tentry, err := kv.Get(\"foo\")\n\t\t\t\tif err != nil {\n\t\t\t\t\terrCh <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif v := string(entry.Value()); v != \"baz\" {\n\t\t\t\t\terrCh <- fmt.Errorf(\"Got wrong value: %q\", v)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Restart\n\tc.restartServer(s)\n\t// Wait for a bit to make sure as this server participates in direct gets\n\t// it does not server stale reads.\n\ttime.Sleep(2 * time.Second)\n\tclose(done)\n\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors but got %v\", <-errCh)\n\t}\n}\n\n// This test mimics a user's setup where there is a cloud cluster/domain, and one for eu and ap that are leafnoded into the\n// cloud cluster, and one for cn that is leafnoded into the ap cluster.\n// We broke basic connectivity in 2.9.17 from publishing in eu for delivery in cn on same account which is daisy chained through ap.\n// We will also test cross account delivery in this test as well.\nfunc TestJetStreamClusterLeafnodePlusDaisyChainSetup(t *testing.T) {\n\tvar cloudTmpl = `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: CLOUD, store_dir: '%s'}\n\n\t\tleaf { listen: 127.0.0.1:-1 }\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts {\n\t\t\tF {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers = [ { user: \"F\", pass: \"pass\" } ]\n\t\t\t\texports [ { stream: \"F.>\" } ]\n\t\t\t}\n\t\t\tT {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers = [ { user: \"T\", pass: \"pass\" } ]\n\t\t\t\timports [ { stream: { account: F, subject: \"F.>\"} } ]\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}`\n\n\t// Now create the cloud and make sure we are connected.\n\t// Cloud\n\tc := createJetStreamCluster(t, cloudTmpl, \"CLOUD\", _EMPTY_, 3, 22020, false)\n\tdefer c.shutdown()\n\n\tvar lnTmpl = `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t{{leaf}}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts {\n\t\t\tF {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers = [ { user: \"F\", pass: \"pass\" } ]\n\t\t\t\texports [ { stream: \"F.>\" } ]\n\t\t\t}\n\t\t\tT {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers = [ { user: \"T\", pass: \"pass\" } ]\n\t\t\t\timports [ { stream: { account: F, subject: \"F.>\"} } ]\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}`\n\n\tvar leafFrag = `\n\t\t\tleaf {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tremotes [ { urls: [ %s ], account: \"T\" }, { urls: [ %s ], account: \"F\" } ]\n\t\t\t}`\n\n\tgenLeafTmpl := func(tmpl string, c *cluster) string {\n\t\tt.Helper()\n\t\t// Create our leafnode cluster template first.\n\t\tvar lnt, lnf []string\n\t\tfor _, s := range c.servers {\n\t\t\tif s.ClusterName() != c.name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tlnt = append(lnt, fmt.Sprintf(\"nats://T:pass@%s:%d\", ln.Host, ln.Port))\n\t\t\tlnf = append(lnf, fmt.Sprintf(\"nats://F:pass@%s:%d\", ln.Host, ln.Port))\n\t\t}\n\t\tlntc := strings.Join(lnt, \", \")\n\t\tlnfc := strings.Join(lnf, \", \")\n\t\treturn strings.Replace(tmpl, \"{{leaf}}\", fmt.Sprintf(leafFrag, lntc, lnfc), 1)\n\t}\n\n\t// Cluster EU\n\t// Domain is \"EU'\n\ttmpl := strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"EU\"), 1)\n\ttmpl = genLeafTmpl(tmpl, c)\n\tlceu := createJetStreamCluster(t, tmpl, \"EU\", \"EU-\", 3, 22110, false)\n\tlceu.waitOnClusterReady()\n\tdefer lceu.shutdown()\n\n\tfor _, s := range lceu.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 2)\n\t}\n\n\t// Cluster AP\n\t// Domain is \"AP'\n\ttmpl = strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"AP\"), 1)\n\ttmpl = genLeafTmpl(tmpl, c)\n\tlcap := createJetStreamCluster(t, tmpl, \"AP\", \"AP-\", 3, 22180, false)\n\tlcap.waitOnClusterReady()\n\tdefer lcap.shutdown()\n\n\tfor _, s := range lcap.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 2)\n\t}\n\n\t// Cluster CN\n\t// Domain is \"CN'\n\t// This one connects to AP, not the cloud hub.\n\ttmpl = strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"CN\"), 1)\n\ttmpl = genLeafTmpl(tmpl, lcap)\n\tlccn := createJetStreamCluster(t, tmpl, \"CN\", \"CN-\", 3, 22280, false)\n\tlccn.waitOnClusterReady()\n\tdefer lccn.shutdown()\n\n\tfor _, s := range lccn.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 2)\n\t}\n\n\t// Now connect to CN on account F and subscribe to data.\n\tnc, _ := jsClientConnect(t, lccn.randomServer(), nats.UserInfo(\"F\", \"pass\"))\n\tdefer nc.Close()\n\tfsub, err := nc.SubscribeSync(\"F.EU.>\")\n\trequire_NoError(t, err)\n\n\t// Same for account T where the import is.\n\tnc, _ = jsClientConnect(t, lccn.randomServer(), nats.UserInfo(\"T\", \"pass\"))\n\tdefer nc.Close()\n\ttsub, err := nc.SubscribeSync(\"F.EU.>\")\n\trequire_NoError(t, err)\n\n\t// Let sub propagate.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Now connect to EU on account F and generate data.\n\tnc, _ = jsClientConnect(t, lceu.randomServer(), nats.UserInfo(\"F\", \"pass\"))\n\tdefer nc.Close()\n\n\tnum := 10\n\tfor i := 0; i < num; i++ {\n\t\terr := nc.Publish(\"F.EU.DATA\", []byte(fmt.Sprintf(\"MSG-%d\", i)))\n\t\trequire_NoError(t, err)\n\t}\n\n\tcheckSubsPending(t, fsub, num)\n\t// Since we export and import in each cluster, we will receive 4x.\n\t// First hop from EU -> CLOUD is 1F and 1T\n\t// Second hop from CLOUD -> AP is 1F, 1T and another 1T\n\t// Third hop from AP -> CN is 1F, 1T, 1T and 1T\n\t// Each cluster hop that has the export/import mapping will add another T message copy.\n\tcheckSubsPending(t, tsub, num*4)\n\n\t// Create stream in cloud.\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"F\", \"pass\"))\n\tdefer nc.Close()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, fmt.Sprintf(\"TEST.%d\", i), \"OK\")\n\t}\n\n\t// Now connect to EU.\n\tnc, js = jsClientConnect(t, lceu.randomServer(), nats.UserInfo(\"F\", \"pass\"))\n\tdefer nc.Close()\n\n\t// Create a mirror.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tDomain: \"CLOUD\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 100 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"State not current: %+v\", si.State)\n\t})\n}\n\n// https://github.com/nats-io/nats-server/pull/4197\nfunc TestJetStreamClusterPurgeExReplayAfterRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"P3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"TEST.0\", \"OK\")\n\tsendStreamMsg(t, nc, \"TEST.1\", \"OK\")\n\tsendStreamMsg(t, nc, \"TEST.2\", \"OK\")\n\n\trunTest := func(f func(js nats.JetStreamManager)) *nats.StreamInfo {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t// install snapshot, then execute interior func, ensuring the purge will be recovered later\n\t\tfsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tfsl.JetStreamSnapshotStream(globalAccountName, \"TEST\")\n\n\t\tf(js)\n\t\ttime.Sleep(250 * time.Millisecond)\n\n\t\tfsl.Shutdown()\n\t\tfsl.WaitForShutdown()\n\t\tfsl = c.restartServer(fsl)\n\t\tc.waitOnServerCurrent(fsl)\n\n\t\tnc, js = jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\n\t\t// keep stepping down so the stream leader matches the initial leader\n\t\t// we need to check if it restored from the snapshot properly\n\t\tfor sl != fsl {\n\t\t\t_, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\t\tsl = c.streamLeader(globalAccountName, \"TEST\")\n\t\t}\n\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\treturn si\n\t}\n\tsi := runTest(func(js nats.JetStreamManager) {\n\t\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"TEST.0\"})\n\t\trequire_NoError(t, err)\n\t})\n\tif si.State.Msgs != 2 {\n\t\tt.Fatalf(\"Expected 2 msgs after restart, got %d\", si.State.Msgs)\n\t}\n\tif si.State.FirstSeq != 2 || si.State.LastSeq != 3 {\n\t\tt.Fatalf(\"Expected FirstSeq=2, LastSeq=3 after restart, got FirstSeq=%d, LastSeq=%d\",\n\t\t\tsi.State.FirstSeq, si.State.LastSeq)\n\t}\n\n\tsi = runTest(func(js nats.JetStreamManager) {\n\t\terr = js.PurgeStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\t// Send 2 more messages.\n\t\tsendStreamMsg(t, nc, \"TEST.1\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.2\", \"OK\")\n\t})\n\tif si.State.Msgs != 2 {\n\t\tt.Fatalf(\"Expected 2 msgs after restart, got %d\", si.State.Msgs)\n\t}\n\tif si.State.FirstSeq != 4 || si.State.LastSeq != 5 {\n\t\tt.Fatalf(\"Expected FirstSeq=4, LastSeq=5 after restart, got FirstSeq=%d, LastSeq=%d\",\n\t\t\tsi.State.FirstSeq, si.State.LastSeq)\n\t}\n\n\t// Now test a keep\n\tsi = runTest(func(js nats.JetStreamManager) {\n\t\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Keep: 1})\n\t\trequire_NoError(t, err)\n\t\t// Send 4 more messages.\n\t\tsendStreamMsg(t, nc, \"TEST.1\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.2\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.3\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.1\", \"OK\")\n\t})\n\tif si.State.Msgs != 5 {\n\t\tt.Fatalf(\"Expected 5 msgs after restart, got %d\", si.State.Msgs)\n\t}\n\tif si.State.FirstSeq != 5 || si.State.LastSeq != 9 {\n\t\tt.Fatalf(\"Expected FirstSeq=5, LastSeq=9 after restart, got FirstSeq=%d, LastSeq=%d\",\n\t\t\tsi.State.FirstSeq, si.State.LastSeq)\n\t}\n\n\t// Now test a keep on a subject\n\tsi = runTest(func(js nats.JetStreamManager) {\n\t\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"TEST.1\", Keep: 1})\n\t\trequire_NoError(t, err)\n\t\t// Send 3 more messages.\n\t\tsendStreamMsg(t, nc, \"TEST.1\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.2\", \"OK\")\n\t\tsendStreamMsg(t, nc, \"TEST.3\", \"OK\")\n\t})\n\tif si.State.Msgs != 7 {\n\t\tt.Fatalf(\"Expected 7 msgs after restart, got %d\", si.State.Msgs)\n\t}\n\tif si.State.FirstSeq != 5 || si.State.LastSeq != 12 {\n\t\tt.Fatalf(\"Expected FirstSeq=5, LastSeq=12 after restart, got FirstSeq=%d, LastSeq=%d\",\n\t\t\tsi.State.FirstSeq, si.State.LastSeq)\n\t}\n}\n\nfunc TestJetStreamClusterConsumerCleanupWithSameName(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"UPDATES\",\n\t\tSubjects: []string{\"DEVICE.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create a consumer that will be an R1 that we will auto-recreate but using the same name.\n\t// We want to make sure that the system does not continually try to cleanup the new one from the old one.\n\n\t// Track the sequence for restart etc.\n\tvar seq atomic.Uint64\n\n\tmsgCB := func(msg *nats.Msg) {\n\t\tmsg.AckSync()\n\t\tmeta, err := msg.Metadata()\n\t\trequire_NoError(t, err)\n\t\tseq.Store(meta.Sequence.Stream)\n\t}\n\n\twaitOnSeqDelivered := func(expected uint64) {\n\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\treceived := seq.Load()\n\t\t\tif received == expected {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Seq is %d, want %d\", received, expected)\n\t\t})\n\t}\n\n\tdoSub := func() {\n\t\t_, err = js.Subscribe(\n\t\t\t\"DEVICE.22\",\n\t\t\tmsgCB,\n\t\t\tnats.ConsumerName(\"dlc\"),\n\t\t\tnats.SkipConsumerLookup(),\n\t\t\tnats.StartSequence(seq.Load()+1),\n\t\t\tnats.MaxAckPending(1), // One at a time.\n\t\t\tnats.ManualAck(),\n\t\t\tnats.ConsumerReplicas(1),\n\t\t\tnats.ConsumerMemoryStorage(),\n\t\t\tnats.MaxDeliver(1),\n\t\t\tnats.InactiveThreshold(time.Second),\n\t\t\tnats.IdleHeartbeat(250*time.Millisecond),\n\t\t)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Track any errors for consumer not active so we can recreate the consumer.\n\terrCh := make(chan error, 10)\n\tnc.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {\n\t\tif errors.Is(err, nats.ErrConsumerNotActive) {\n\t\t\ts.Unsubscribe()\n\t\t\terrCh <- err\n\t\t\tdoSub()\n\t\t}\n\t})\n\n\tdoSub()\n\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-1\")\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-2\")\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-3\")\n\twaitOnSeqDelivered(3)\n\n\t// Shutdown the consumer's leader.\n\ts := c.consumerLeader(globalAccountName, \"UPDATES\", \"dlc\")\n\ts.Shutdown()\n\tc.waitOnStreamLeader(globalAccountName, \"UPDATES\")\n\n\t// In case our client connection was to the same server.\n\tnc, _ = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-4\")\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-5\")\n\tsendStreamMsg(t, nc, \"DEVICE.22\", \"update-6\")\n\n\t// Wait for the consumer not active error.\n\t<-errCh\n\t// Now restart server with the old consumer.\n\tc.restartServer(s)\n\t// Wait on all messages delivered.\n\twaitOnSeqDelivered(6)\n\t// Make sure no other errors showed up\n\trequire_True(t, len(errCh) == 0)\n}\nfunc TestJetStreamClusterConsumerActions(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tvar err error\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\tecSubj := fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"CONSUMER\", \"test\")\n\tcrReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverPolicy: DeliverLast,\n\t\t\tFilterSubject: \"test\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t},\n\t}\n\n\t// A new consumer. Should not be an error.\n\tcrReq.Action = ActionCreate\n\treq, err := json.Marshal(crReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", ccResp.Error)\n\t}\n\tccResp.Error = nil\n\n\t// Consumer exists, but config is the same, so should be ok\n\tresp, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected er response: %v\", ccResp.Error)\n\t}\n\tccResp.Error = nil\n\t// Consumer exists. Config is different, so should error\n\tcrReq.Config.Description = \"changed\"\n\treq, err = json.Marshal(crReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error == nil {\n\t\tt.Fatalf(\"Unexpected ok response\")\n\t}\n\n\tccResp.Error = nil\n\t// Consumer update, so update should be ok\n\tcrReq.Action = ActionUpdate\n\tcrReq.Config.Description = \"changed again\"\n\treq, err = json.Marshal(crReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error response: %v\", ccResp.Error)\n\t}\n\n\tecSubj = fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"NEW\", \"test\")\n\tccResp.Error = nil\n\t// Updating new consumer, so should error\n\tcrReq.Config.Name = \"NEW\"\n\treq, err = json.Marshal(crReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error == nil {\n\t\tt.Fatalf(\"Unexpected ok response\")\n\t}\n}\n\nfunc TestJetStreamClusterSnapshotAndRestoreWithHealthz(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend, msg := 1000, bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.PublishAsync(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsreq := &JSApiStreamSnapshotRequest{\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tChunkSize: 512,\n\t}\n\treq, _ := json.Marshal(sreq)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiStreamSnapshotResponse\n\tjson.Unmarshal(rmsg.Data, &resp)\n\trequire_True(t, resp.Error == nil)\n\n\tstate := *resp.State\n\tcfg := *resp.Config\n\n\tvar snapshot []byte\n\tdone := make(chan bool)\n\n\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t// EOF\n\t\tif len(m.Data) == 0 {\n\t\t\tdone <- true\n\t\t\treturn\n\t\t}\n\t\t// Could be writing to a file here too.\n\t\tsnapshot = append(snapshot, m.Data...)\n\t\t// Flow ack\n\t\tm.Respond(nil)\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\n\t// Delete before we try to restore.\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\tcheckHealth := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tstatus := s.healthz(nil)\n\t\t\t\tif status.Error != _EMPTY_ {\n\t\t\t\t\treturn fmt.Errorf(\"%s - %v\", s.Name(), status.Error)\n\t\t\t\t}\n\t\t\t\tif status.Status != \"ok\" {\n\t\t\t\t\treturn fmt.Errorf(\"%s - %v\", s.Name(), status.Status)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tvar rresp JSApiStreamRestoreResponse\n\trreq := &JSApiStreamRestoreRequest{\n\t\tConfig: cfg,\n\t\tState: state,\n\t}\n\treq, _ = json.Marshal(rreq)\n\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, \"TEST\"), req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\trequire_True(t, rresp.Error == nil)\n\n\tcheckHealth()\n\n\t// We will now chunk the snapshot responses (and EOF).\n\tvar chunk [1024]byte\n\tfor i, r := 0, bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tnc.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t\ti++\n\t\t// We will call healthz for all servers halfway through the restore.\n\t\tif i%100 == 0 {\n\t\t\tcheckHealth()\n\t\t}\n\t}\n\trmsg, err = nc.Request(rresp.DeliverSubject, nil, time.Second)\n\trequire_NoError(t, err)\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\trequire_True(t, rresp.Error == nil)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == uint64(toSend))\n\n\tcheckHealth()\n\n\t// Make sure stepdown works, this would fail before the fix.\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, 5*time.Second)\n\trequire_NoError(t, err)\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == uint64(toSend))\n\n\tcheckHealth()\n\n\t// Now make sure if we try to restore to a single server that the artifact is cleaned up and the server returns ok for healthz.\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, \"TEST\"), req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\trequire_True(t, rresp.Error == nil)\n\n\tfor i, r := 0, bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\t_, err = nc.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t\trequire_NoError(t, err)\n\t\ti++\n\t}\n\trmsg, err = nc.Request(rresp.DeliverSubject, nil, time.Second)\n\trequire_NoError(t, err)\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\n\trequire_True(t, rresp.Error != nil)\n\trequire_Equal(t, rresp.ApiResponse.Error.ErrCode, 10074)\n\n\tstatus := s.healthz(nil)\n\trequire_Equal(t, status.StatusCode, 200)\n}\n\nfunc TestJetStreamClusterBinaryStreamSnapshotCapability(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"NATS\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tmset, err := c.streamLeader(globalAccountName, \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif !mset.supportsBinarySnapshot() {\n\t\tt.Fatalf(\"Expected to signal that we could support binary stream snapshots\")\n\t}\n}\n\nfunc TestJetStreamClusterBadEncryptKey(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterEncryptedTempl, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create 10 streams.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST-%d\", i),\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Grab random server.\n\ts := c.randomServer()\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\n\tvar opts *Options\n\tfor i := 0; i < len(c.servers); i++ {\n\t\tif c.servers[i] == s {\n\t\t\topts = c.opts[i]\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_NotNil(t, opts)\n\n\t// Replace key with an empty key.\n\tbuf, err := os.ReadFile(opts.ConfigFile)\n\trequire_NoError(t, err)\n\tnbuf := bytes.Replace(buf, []byte(\"key: \\\"s3cr3t!\\\"\"), []byte(\"key: \\\"\\\"\"), 1)\n\terr = os.WriteFile(opts.ConfigFile, nbuf, 0640)\n\trequire_NoError(t, err)\n\n\t// Make sure trying to start the server now fails.\n\ts, err = NewServer(LoadConfig(opts.ConfigFile))\n\trequire_NoError(t, err)\n\trequire_NotNil(t, s)\n\ts.Start()\n\tif err := s.readyForConnections(1 * time.Second); err == nil {\n\t\tt.Fatalf(\"Expected server not to start\")\n\t}\n}\n\nfunc TestJetStreamClusterAccountUsageDrifts(t *testing.T) {\n\ttmpl := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\t\tleaf {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t`\n\topFrag := `\n\t\t\toperator: %s\n\t\t\tsystem_account: %s\n\t\t\tresolver: { type: MEM }\n\t\t\tresolver_preload = {\n\t\t\t\t%s : %s\n\t\t\t\t%s : %s\n\t\t\t}\n\t\t`\n\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: -1, Consumer: 1, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: -1, Consumer: 1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\n\ttemplate := tmpl + fmt.Sprintf(opFrag, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\tc := createJetStreamClusterWithTemplate(t, template, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST1\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t\tMaxMsgs: 1000,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"bar\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// These expected store values can come directly from stream info's state bytes.\n\t// We will *= 3 for R3\n\tcheckAccount := func(r1u, r3u uint64) {\n\t\tt.Helper()\n\t\tr3u *= 3\n\n\t\t// Remote usage updates can be delayed, so wait for a bit for values we want.\n\t\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\t\tinfo, err := js.AccountInfo()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(info.Tiers) >= 2)\n\t\t\t// These can move.\n\t\t\tif u := info.Tiers[\"R1\"].Store; u != r1u {\n\t\t\t\treturn fmt.Errorf(\"Expected R1 to be %v, got %v\", friendlyBytes(r1u), friendlyBytes(u))\n\t\t\t}\n\t\t\tif u := info.Tiers[\"R3\"].Store; u != r3u {\n\t\t\t\treturn fmt.Errorf(\"Expected R3 to be %v, got %v\", friendlyBytes(r3u), friendlyBytes(u))\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckAccount(0, 0)\n\n\t// Now add in some R3 data.\n\tmsg := bytes.Repeat([]byte(\"Z\"), 32*1024) // 32k\n\tsmallMsg := bytes.Repeat([]byte(\"Z\"), 4*1024) // 4k\n\n\tfor i := 0; i < 1000; i++ {\n\t\tjs.Publish(\"foo\", msg)\n\t}\n\tsir3, err := js.StreamInfo(\"TEST1\")\n\trequire_NoError(t, err)\n\n\tcheckAccount(0, sir3.State.Bytes)\n\n\t// Now add in some R1 data.\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(\"bar\", msg)\n\t}\n\n\tsir1, err := js.StreamInfo(\"TEST2\")\n\trequire_NoError(t, err)\n\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// We will now test a bunch of scenarios to see that we are doing accounting correctly.\n\n\t// Since our R3 has a limit of 1000 msgs, let's add in more msgs and drop older ones.\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(\"foo\", smallMsg)\n\t}\n\tsir3, err = js.StreamInfo(\"TEST1\")\n\trequire_NoError(t, err)\n\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// Move our R3 stream leader and make sure acounting is correct.\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST1\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(aExpPub, \"TEST1\")\n\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// Now scale down.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST1\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t\tMaxMsgs: 1000,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckAccount(sir1.State.Bytes+sir3.State.Bytes, 0)\n\n\t// Add in more msgs which will replace the older and bigger ones.\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(\"foo\", smallMsg)\n\t}\n\tsir3, err = js.StreamInfo(\"TEST1\")\n\trequire_NoError(t, err)\n\n\t// Now scale back up.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST1\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxBytes: 1 * 1024 * 1024 * 1024,\n\t\tMaxMsgs: 1000,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(aExpPub, \"TEST1\")\n\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// Test Purge.\n\terr = js.PurgeStream(\"TEST1\")\n\trequire_NoError(t, err)\n\n\tcheckAccount(sir1.State.Bytes, 0)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tjs.Publish(\"foo\", smallMsg)\n\t}\n\tsir3, err = js.StreamInfo(\"TEST1\")\n\trequire_NoError(t, err)\n\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// Need system user here to move the leader.\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(sysCreds))\n\tdefer snc.Close()\n\n\trequestLeaderStepDown := func() {\n\t\tml := c.leader()\n\t\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\t\tif cml := c.leader(); cml == ml {\n\t\t\t\tsnc.Request(JSApiLeaderStepDown, nil, time.Second)\n\t\t\t\treturn fmt.Errorf(\"Metaleader has not moved yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Test meta leader stepdowns.\n\tfor i := 0; i < len(c.servers); i++ {\n\t\trequestLeaderStepDown()\n\t\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\t}\n\n\t// Now test cluster reset operations where we internally reset the NRG and optionally the stream too.\n\t// Only applicable to TEST1 stream which is R3.\n\tnl := c.randomNonStreamLeader(aExpPub, \"TEST1\")\n\tacc, err := nl.LookupAccount(aExpPub)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST1\")\n\trequire_NoError(t, err)\n\t// NRG only\n\tmset.resetClusteredState(nil)\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\t// Need to re-lookup this stream since we will recreate from reset above.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err = acc.lookupStream(\"TEST1\")\n\t\treturn err\n\t})\n\t// Now NRG and Stream state itself.\n\tmset.resetClusteredState(errFirstSequenceMismatch)\n\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\n\t// Now test server restart\n\tfor _, s := range c.servers {\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t\ts = c.restartServer(s)\n\n\t\t// Wait on healthz and leader etc.\n\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif hs := s.healthz(nil); hs.Error != _EMPTY_ {\n\t\t\t\treturn errors.New(hs.Error)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tc.waitOnLeader()\n\t\tc.waitOnStreamLeader(aExpPub, \"TEST1\")\n\t\tc.waitOnStreamLeader(aExpPub, \"TEST2\")\n\n\t\t// Now check account again.\n\t\tcheckAccount(sir1.State.Bytes, sir3.State.Bytes)\n\t}\n}\n\nfunc TestJetStreamClusterStreamFailTracking(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Data = []byte(\"OK\")\n\n\tb, bsz := 0, 5\n\tsendBatch := func() {\n\t\tfor i := b * bsz; i < b*bsz+bsz; i++ {\n\t\t\tmsgId := fmt.Sprintf(\"ID:%d\", i)\n\t\t\tm.Header.Set(JSMsgId, msgId)\n\t\t\t// Send it twice on purpose.\n\t\t\tjs.PublishMsg(m)\n\t\t\tjs.PublishMsg(m)\n\t\t}\n\t\tb++\n\t}\n\n\tsendBatch()\n\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tsendBatch()\n\n\t// Now stop one and restart.\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tmset, err := nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\t// Reset raft\n\tmset.resetClusteredState(nil)\n\ttime.Sleep(100 * time.Millisecond)\n\n\tnl.Shutdown()\n\tnl.WaitForShutdown()\n\n\tsendBatch()\n\n\tnl = c.restartServer(nl)\n\n\tsendBatch()\n\n\tfor {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tif nl == c.streamLeader(globalAccountName, \"TEST\") {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tsendBatch()\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure all in order.\n\terrCh := make(chan error, 100)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\texpected, seen := b*bsz, 0\n\n\tsub, err := js.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\texpectedID := fmt.Sprintf(\"ID:%d\", seen)\n\t\tif v := msg.Header.Get(JSMsgId); v != expectedID {\n\t\t\terrCh <- err\n\t\t\twg.Done()\n\t\t\tmsg.Sub.Unsubscribe()\n\t\t\treturn\n\t\t}\n\t\tseen++\n\t\tif seen >= expected {\n\t\t\twg.Done()\n\t\t\tmsg.Sub.Unsubscribe()\n\t\t}\n\t})\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\twg.Wait()\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n}\n\nfunc TestJetStreamClusterStreamFailTrackingSnapshots(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Data = []byte(\"OK\")\n\n\t// Send 1000 a dupe every msgID.\n\tfor i := 0; i < 1000; i++ {\n\t\tmsgId := fmt.Sprintf(\"ID:%d\", i)\n\t\tm.Header.Set(JSMsgId, msgId)\n\t\t// Send it twice on purpose.\n\t\tjs.PublishMsg(m)\n\t\tjs.PublishMsg(m)\n\t}\n\n\t// Now stop one.\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tnl.Shutdown()\n\tnl.WaitForShutdown()\n\n\t// Now send more and make sure leader snapshots.\n\tfor i := 1000; i < 2000; i++ {\n\t\tmsgId := fmt.Sprintf(\"ID:%d\", i)\n\t\tm.Header.Set(JSMsgId, msgId)\n\t\t// Send it twice on purpose.\n\t\tjs.PublishMsg(m)\n\t\tjs.PublishMsg(m)\n\t}\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\trequire_NotNil(t, node)\n\tnode.InstallSnapshot(mset.stateSnapshot(), false)\n\n\t// Now restart nl\n\tnl = c.restartServer(nl)\n\tc.waitOnServerCurrent(nl)\n\n\t// Move leader to NL\n\tfor {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tif nl == c.streamLeader(globalAccountName, \"TEST\") {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure all in order.\n\terrCh := make(chan error, 100)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\texpected, seen := 2000, 0\n\n\tsub, err := js.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\texpectedID := fmt.Sprintf(\"ID:%d\", seen)\n\t\tif v := msg.Header.Get(JSMsgId); v != expectedID {\n\t\t\terrCh <- err\n\t\t\twg.Done()\n\t\t\tmsg.Sub.Unsubscribe()\n\t\t\treturn\n\t\t}\n\t\tseen++\n\t\tif seen >= expected {\n\t\t\twg.Done()\n\t\t\tmsg.Sub.Unsubscribe()\n\t\t}\n\t})\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\twg.Wait()\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n}\n\nfunc TestJetStreamClusterOrphanConsumerSubjects(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\", \"bar.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"consumer_foo\",\n\t\tDurable: \"consumer_foo\",\n\t\tFilterSubject: \"foo.something\",\n\t})\n\trequire_NoError(t, err)\n\n\tfor _, replicas := range []int{3, 1, 3} {\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"bar.>\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tc.waitOnAllCurrent()\n\t}\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"consumer_foo\")\n\n\tinfo, err := js.ConsumerInfo(\"TEST\", \"consumer_foo\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.Cluster != nil)\n\trequire_NotEqual(t, info.Cluster.Leader, \"\")\n\trequire_Equal(t, len(info.Cluster.Replicas), 2)\n}\n\nfunc TestJetStreamClusterDurableConsumerInactiveThresholdLeaderSwitch(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Queue a msg.\n\tsendStreamMsg(t, nc, \"foo\", \"ok\")\n\n\tthresh := 250 * time.Millisecond\n\n\t// This will start the timer.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.InactiveThreshold(thresh))\n\trequire_NoError(t, err)\n\n\t// Switch over leader.\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"dlc\")\n\tcl.JetStreamStepdownConsumer(globalAccountName, \"TEST\", \"dlc\")\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dlc\")\n\n\t// Create activity on this consumer.\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\n\t// This is consider activity as well. So we can watch now up to thresh to make sure consumer still active.\n\tmsgs[0].AckSync()\n\n\t// The consumer should not disappear for next `thresh` interval unless old leader does so.\n\ttimeout := time.Now().Add(thresh)\n\tfor time.Now().Before(timeout) {\n\t\t_, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\t\tif err == nats.ErrConsumerNotFound {\n\t\t\tt.Fatalf(\"Consumer deleted when it should not have been\")\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterConsumerMaxDeliveryNumAckPendingBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// send 50 msgs\n\tfor i := 0; i < 50; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"ok\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// File based.\n\tsub, err := js.PullSubscribe(\"foo\", \"file\",\n\t\tnats.ManualAck(),\n\t\tnats.MaxDeliver(1),\n\t\tnats.AckWait(time.Second),\n\t\tnats.MaxAckPending(10),\n\t)\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t// Let first batch expire.\n\ttime.Sleep(1200 * time.Millisecond)\n\n\tcia, err := js.ConsumerInfo(\"TEST\", \"file\")\n\trequire_NoError(t, err)\n\n\t// Make sure followers will have exact same state.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"file\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"file\")\n\n\tcib, err := js.ConsumerInfo(\"TEST\", \"file\")\n\trequire_NoError(t, err)\n\n\t// Want to compare sans cluster details which we know will change due to leader change.\n\t// Also last activity for delivered can be slightly off so nil out as well.\n\tcheckConsumerInfo := func(a, b *nats.ConsumerInfo, replicated bool) {\n\t\tt.Helper()\n\t\trequire_Equal(t, a.Delivered.Consumer, 10)\n\t\trequire_Equal(t, a.Delivered.Stream, 10)\n\t\t// Agreed upon state is always used. Otherwise, o.asflr and o.adflr would be skipped ahead.\n\t\trequire_Equal(t, a.AckFloor.Consumer, 0)\n\t\trequire_Equal(t, a.AckFloor.Stream, 0)\n\t\trequire_Equal(t, a.NumPending, 40)\n\t\trequire_Equal(t, a.NumRedelivered, 0)\n\t\ta.Cluster, b.Cluster = nil, nil\n\t\ta.Delivered.Last, b.Delivered.Last = nil, nil\n\t\tif !reflect.DeepEqual(a, b) {\n\t\t\tt.Fatalf(\"ConsumerInfo do not match\\n\\t%+v\\n\\t%+v\", a, b)\n\t\t}\n\t}\n\n\tcheckConsumerInfo(cia, cib, true)\n\n\t// Memory based.\n\tsub, err = js.PullSubscribe(\"foo\", \"mem\",\n\t\tnats.ManualAck(),\n\t\tnats.MaxDeliver(1),\n\t\tnats.AckWait(time.Second),\n\t\tnats.MaxAckPending(10),\n\t\tnats.ConsumerMemoryStorage(),\n\t)\n\trequire_NoError(t, err)\n\n\tmsgs, err = sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t// Let first batch retry and expire.\n\ttime.Sleep(1200 * time.Millisecond)\n\n\tcia, err = js.ConsumerInfo(\"TEST\", \"mem\")\n\trequire_NoError(t, err)\n\n\t// Make sure followers will have exact same state.\n\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"mem\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"mem\")\n\n\tcib, err = js.ConsumerInfo(\"TEST\", \"mem\")\n\trequire_NoError(t, err)\n\n\tcheckConsumerInfo(cia, cib, true)\n\n\t// Now file based but R1 and server restart.\n\tsub, err = js.PullSubscribe(\"foo\", \"r1\",\n\t\tnats.ManualAck(),\n\t\tnats.MaxDeliver(1),\n\t\tnats.AckWait(time.Second),\n\t\tnats.MaxAckPending(10),\n\t\tnats.ConsumerReplicas(1),\n\t)\n\trequire_NoError(t, err)\n\n\tmsgs, err = sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t// Let first batch retry and expire.\n\ttime.Sleep(1200 * time.Millisecond)\n\n\tcia, err = js.ConsumerInfo(\"TEST\", \"r1\")\n\trequire_NoError(t, err)\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"r1\")\n\tcl.Shutdown()\n\tcl.WaitForShutdown()\n\tcl = c.restartServer(cl)\n\tc.waitOnServerCurrent(cl)\n\n\tcib, err = js.ConsumerInfo(\"TEST\", \"r1\")\n\trequire_NoError(t, err)\n\n\t// Created can skew a small bit due to server restart, this is expected.\n\tnow := time.Now()\n\tcia.Created, cib.Created = now, now\n\tcheckConsumerInfo(cia, cib, false)\n}\n\nfunc TestJetStreamClusterConsumerDefaultsFromStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tstreamTmpl := &StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"test.*\"},\n\t\tStorage: MemoryStorage,\n\t\tConsumerLimits: StreamConsumerLimits{\n\t\t\tMaxAckPending: 0,\n\t\t\tInactiveThreshold: 0,\n\t\t},\n\t}\n\n\t// Since nats.go doesn't yet know about the consumer limits, craft\n\t// the stream configuration request by hand.\n\tstreamCreate := func(maxAckPending int, inactiveThreshold time.Duration) (*StreamConfig, error) {\n\t\tcfg := streamTmpl\n\t\tcfg.ConsumerLimits = StreamConsumerLimits{\n\t\t\tMaxAckPending: maxAckPending,\n\t\t\tInactiveThreshold: inactiveThreshold,\n\t\t}\n\t\tj, err := json.Marshal(cfg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, \"test\"), j, time.Second*3)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvar resp JSApiStreamCreateResponse\n\t\tif err := json.Unmarshal(msg.Data, &resp); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif resp.StreamInfo == nil {\n\t\t\treturn nil, resp.ApiResponse.ToError()\n\t\t}\n\t\treturn &resp.Config, resp.ApiResponse.ToError()\n\t}\n\tstreamUpdate := func(maxAckPending int, inactiveThreshold time.Duration) (*StreamConfig, error) {\n\t\tcfg := streamTmpl\n\t\tcfg.ConsumerLimits = StreamConsumerLimits{\n\t\t\tMaxAckPending: maxAckPending,\n\t\t\tInactiveThreshold: inactiveThreshold,\n\t\t}\n\t\tj, err := json.Marshal(cfg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, \"test\"), j, time.Second*3)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvar resp JSApiStreamUpdateResponse\n\t\tif err := json.Unmarshal(msg.Data, &resp); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif resp.StreamInfo == nil {\n\t\t\treturn nil, resp.ApiResponse.ToError()\n\t\t}\n\t\treturn &resp.Config, resp.ApiResponse.ToError()\n\t}\n\n\tif _, err := streamCreate(15, time.Second); err != nil {\n\t\tt.Fatalf(\"Failed to add stream: %v\", err)\n\t}\n\n\tt.Run(\"InheritDefaultsFromStream\", func(t *testing.T) {\n\t\tci, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"InheritDefaultsFromStream\",\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tswitch {\n\t\tcase ci.Config.InactiveThreshold != time.Second:\n\t\t\tt.Fatalf(\"InactiveThreshold should be 1s, got %s\", ci.Config.InactiveThreshold)\n\t\tcase ci.Config.MaxAckPending != 15:\n\t\t\tt.Fatalf(\"MaxAckPending should be 15, got %d\", ci.Config.MaxAckPending)\n\t\t}\n\t})\n\n\tt.Run(\"CreateConsumerErrorOnExceedMaxAckPending\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"CreateConsumerErrorOnExceedMaxAckPending\",\n\t\t\tMaxAckPending: 30,\n\t\t})\n\t\tswitch e := err.(type) {\n\t\tcase *nats.APIError:\n\t\t\tif ErrorIdentifier(e.ErrorCode) != JSConsumerMaxPendingAckExcessErrF {\n\t\t\t\tt.Fatalf(\"invalid error code, got %d, wanted %d\", e.ErrorCode, JSConsumerMaxPendingAckExcessErrF)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"should have returned API error, got %T\", e)\n\t\t}\n\t})\n\n\tt.Run(\"CreateConsumerErrorOnExceedInactiveThreshold\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"CreateConsumerErrorOnExceedInactiveThreshold\",\n\t\t\tInactiveThreshold: time.Second * 2,\n\t\t})\n\t\tswitch e := err.(type) {\n\t\tcase *nats.APIError:\n\t\t\tif ErrorIdentifier(e.ErrorCode) != JSConsumerInactiveThresholdExcess {\n\t\t\t\tt.Fatalf(\"invalid error code, got %d, wanted %d\", e.ErrorCode, JSConsumerInactiveThresholdExcess)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"should have returned API error, got %T\", e)\n\t\t}\n\t})\n\n\tt.Run(\"UpdateStreamErrorOnViolateConsumerMaxAckPending\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"UpdateStreamErrorOnViolateConsumerMaxAckPending\",\n\t\t\tMaxAckPending: 15,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tif _, err = streamUpdate(10, 0); err == nil {\n\t\t\tt.Fatalf(\"stream update should have errored but didn't\")\n\t\t}\n\t})\n\n\tt.Run(\"UpdateStreamErrorOnViolateConsumerInactiveThreshold\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"UpdateStreamErrorOnViolateConsumerInactiveThreshold\",\n\t\t\tInactiveThreshold: time.Second,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tif _, err = streamUpdate(0, time.Second/2); err == nil {\n\t\t\tt.Fatalf(\"stream update should have errored but didn't\")\n\t\t}\n\t})\n}\n\n// Discovered that we are not properly setting certain default filestore blkSizes.\nfunc TestJetStreamClusterCheckFileStoreBlkSizes(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Normal Stream\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// KV\n\t_, err = js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tblkSize := func(fs *fileStore) uint64 {\n\t\tfs.mu.RLock()\n\t\tdefer fs.mu.RUnlock()\n\t\treturn fs.fcfg.BlockSize\n\t}\n\n\t// We will check now the following filestores.\n\t// meta\n\t// TEST stream and NRG\n\t// C3 NRG\n\t// KV_TEST stream and NRG\n\tfor _, s := range c.servers {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\t// META\n\t\tjs.mu.RLock()\n\t\tmeta := cc.meta\n\t\tjs.mu.RUnlock()\n\t\trequire_True(t, meta != nil)\n\t\tfs := meta.(*raft).wal.(*fileStore)\n\t\trequire_True(t, blkSize(fs) == defaultMetaFSBlkSize)\n\n\t\t// TEST STREAM\n\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tmset.mu.RLock()\n\t\tfs = mset.store.(*fileStore)\n\t\tmset.mu.RUnlock()\n\t\trequire_True(t, blkSize(fs) == defaultLargeBlockSize)\n\n\t\t// KV STREAM\n\t\t// Now the KV which is different default size.\n\t\tkv, err := s.GlobalAccount().lookupStream(\"KV_TEST\")\n\t\trequire_NoError(t, err)\n\t\tkv.mu.RLock()\n\t\tfs = kv.store.(*fileStore)\n\t\tkv.mu.RUnlock()\n\t\trequire_True(t, blkSize(fs) == defaultKVBlockSize)\n\n\t\t// Now check NRGs\n\t\t// TEST Stream\n\t\tn := mset.raftNode()\n\t\trequire_True(t, n != nil)\n\t\tfs = n.(*raft).wal.(*fileStore)\n\t\trequire_True(t, blkSize(fs) == defaultMediumBlockSize)\n\t\t// KV TEST Stream\n\t\tn = kv.raftNode()\n\t\trequire_True(t, n != nil)\n\t\tfs = n.(*raft).wal.(*fileStore)\n\t\trequire_True(t, blkSize(fs) == defaultMediumBlockSize)\n\t\t// Consumer\n\t\to := mset.lookupConsumer(\"C3\")\n\t\trequire_True(t, o != nil)\n\t\tn = o.raftNode()\n\t\trequire_True(t, n != nil)\n\t\tfs = n.(*raft).wal.(*fileStore)\n\t\trequire_True(t, blkSize(fs) == defaultMediumBlockSize)\n\t}\n}\n\nfunc TestJetStreamClusterDetectOrphanNRGs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Normal Stream\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DC\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// We will force an orphan for a certain server.\n\ts := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tsgn := mset.raftNode().Group()\n\n\to := mset.lookupConsumer(\"DC\")\n\trequire_True(t, o != nil)\n\togn := o.raftNode().Group()\n\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t// Should only be meta NRG left.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tif rns := s.numRaftNodes(); rns != 1 {\n\t\t\treturn fmt.Errorf(\"expected only 1 (meta) RAFT node, got: %d\", rns)\n\t\t}\n\t\treturn nil\n\t})\n\n\ts.rnMu.RLock()\n\tdefer s.rnMu.RUnlock()\n\trequire_True(t, s.lookupRaftNode(sgn) == nil)\n\trequire_True(t, s.lookupRaftNode(ogn) == nil)\n}\n\n// https://github.com/nats-io/nats-server/issues/4732\nfunc TestJetStreamClusterStreamLimitsOnScaleUpAndMove(t *testing.T) {\n\ttmpl := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t`\n\topFrag := `\n\t\t\toperator: %s\n\t\t\tsystem_account: %s\n\t\t\tresolver: { type: MEM }\n\t\t\tresolver_preload = {\n\t\t\t\t%s : %s\n\t\t\t\t%s : %s\n\t\t\t}\n\t\t`\n\n\t_, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: -1, Consumer: -1, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 0, Consumer: -1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\n\ttemplate := tmpl + fmt.Sprintf(opFrag, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamCluster(t, template, \"CLOUD\", _EMPTY_, 3, 22020, true)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend, msg := 100, bytes.Repeat([]byte(\"Z\"), 1024)\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.PublishAsync(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Scale up should fail here since no R3 storage.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_Error(t, err, errors.New(\"insufficient storage resources\"))\n}\n\nfunc TestJetStreamClusterAPIAccessViaSystemAccount(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Connect to system account.\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\trequire_Error(t, err, NewJSNotEnabledForAccountError())\n\n\t// Make sure same behavior swith single server.\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, t.TempDir())))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\trequire_Error(t, err, NewJSNotEnabledForAccountError())\n}\n\nfunc TestJetStreamClusterStreamResetPreacks(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100_000_000})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Put 20 msgs in.\n\tfor i := 0; i < 20; i++ {\n\t\t_, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Consume and ack 10.\n\tmsgs, err := sub.Fetch(10, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\tfor _, msg := range msgs {\n\t\tmsg.AckSync()\n\t}\n\n\t// Now grab a non-leader server.\n\t// We will shut it down and remove the stream data.\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tmset, err := nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfs := mset.store.(*fileStore)\n\tmdir := filepath.Join(fs.fcfg.StoreDir, msgDir)\n\tnl.Shutdown()\n\t// In case that was the consumer leader.\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dlc\")\n\n\t// Now consume the remaining 10 and ack.\n\tmsgs, err = sub.Fetch(10, nats.MaxWait(10*time.Second))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\tfor _, msg := range msgs {\n\t\tmsg.AckSync()\n\t}\n\n\t// Now remove the stream manually.\n\trequire_NoError(t, os.RemoveAll(mdir))\n\tnl = c.restartServer(nl)\n\tc.waitOnAllCurrent()\n\n\tmset, err = nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tstate := mset.state()\n\t\tif state.Msgs != 0 || state.FirstSeq != 100_000_020 {\n\t\t\treturn fmt.Errorf(\"Not correct state yet: %+v\", state)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterDomainAdvisory(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: NGS, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"R3S\", _EMPTY_, 3, 18033, true)\n\tdefer c.shutdown()\n\n\t// Connect to system account.\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(JSAdvisoryDomainLeaderElected)\n\trequire_NoError(t, err)\n\n\t// Ask meta leader to move and make sure we get an advisory.\n\tnc.Request(JSApiLeaderStepDown, nil, time.Second)\n\tc.waitOnLeader()\n\n\tcheckSubsPending(t, sub, 1)\n\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tvar adv JSDomainLeaderElectedAdvisory\n\trequire_NoError(t, json.Unmarshal(m.Data, &adv))\n\n\tml := c.leader()\n\tjs, cc := ml.getJetStreamCluster()\n\tjs.mu.RLock()\n\tpeer := cc.meta.ID()\n\tjs.mu.RUnlock()\n\n\trequire_Equal(t, adv.Leader, peer)\n\trequire_Equal(t, adv.Domain, \"NGS\")\n\trequire_Equal(t, adv.Cluster, \"R3S\")\n\trequire_Equal(t, len(adv.Replicas), 3)\n}\n\nfunc TestJetStreamClusterLimitsBasedStreamFileStoreDesync(t *testing.T) {\n\tconf := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {\n\t\tstore_dir: '%s',\n\t}\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n system_account: sys\n no_auth_user: js\n\taccounts {\n\t sys {\n\t users = [\n\t { user: sys, pass: sys }\n\t ]\n\t }\n\t js {\n\t jetstream = { max_store = 3mb }\n\t users = [\n\t { user: js, pass: js }\n\t ]\n\t }\n\t}`\n\tc := createJetStreamClusterWithTemplate(t, conf, \"limits\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcnc, cjs := jsClientConnect(t, c.randomServer())\n\tdefer cnc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"LTEST\",\n\t\tSubjects: []string{\"messages.*\"},\n\t\tReplicas: 3,\n\t\tMaxAge: 10 * time.Minute,\n\t\tMaxMsgs: 100_000,\n\t})\n\trequire_NoError(t, err)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tpsub, err := cjs.PullSubscribe(\"messages.*\", \"consumer\")\n\trequire_NoError(t, err)\n\n\tvar (\n\t\twg sync.WaitGroup\n\t\treceived uint64\n\t\terrCh = make(chan error, 100_000)\n\t\treceivedMap = make(map[string]*nats.Msg)\n\t)\n\twg.Add(1)\n\tgo func() {\n\t\ttick := time.NewTicker(20 * time.Millisecond)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\tmsgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\treceived++\n\t\t\t\t\treceivedMap[msg.Subject] = msg\n\t\t\t\t\tif meta, _ := msg.Metadata(); meta.NumDelivered > 1 {\n\t\t\t\t\t\tt.Logf(\"GOT MSG: %s :: %+v :: %d\", msg.Subject, meta, len(msg.Data))\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Send 20_000 msgs at roughly 1 msg per msec\n\tshouldDrop := make(map[string]error)\n\twg.Add(1)\n\tgo func() {\n\t\tpayload := []byte(strings.Repeat(\"A\", 1024))\n\t\ttick := time.NewTicker(1 * time.Millisecond)\n\t\tfor i := 1; i < 100_000; {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\t// This should run into 3MB quota and get errors right away\n\t\t\t\t// before the max msgs limit does.\n\t\t\t\tsubject := fmt.Sprintf(\"messages.%d\", i)\n\t\t\t\t_, err := js.Publish(subject, payload, nats.RetryAttempts(0))\n\t\t\t\tif err != nil {\n\t\t\t\t\terrCh <- err\n\t\t\t\t}\n\t\t\t\ti++\n\n\t\t\t\t// Any message over this number should not be a success\n\t\t\t\t// since the stream should be full due to the quota.\n\t\t\t\t// Here we capture that the messages have failed to confirm.\n\t\t\t\tif err != nil && i > 1000 {\n\t\t\t\t\tshouldDrop[subject] = err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Collect enough errors to cause things to get out of sync.\n\tvar errCount int\nSetup:\n\tfor {\n\t\tselect {\n\t\tcase err = <-errCh:\n\t\t\terrCount++\n\t\t\tif errCount >= 20_000 {\n\t\t\t\t// Stop both producing and consuming.\n\t\t\t\tcancel()\n\t\t\t\tbreak Setup\n\t\t\t}\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Timed out waiting for limits error\")\n\t\t}\n\t}\n\n\t// Both goroutines should be exiting now..\n\twg.Wait()\n\n\t// Check messages that ought to have been dropped.\n\tfor subject := range receivedMap {\n\t\tfound, ok := shouldDrop[subject]\n\t\tif ok {\n\t\t\tt.Errorf(\"Should have dropped message published on %q since got error: %v\", subject, found)\n\t\t}\n\t}\n\n\tgetStreamDetails := func(t *testing.T, srv *Server) *StreamDetail {\n\t\tt.Helper()\n\t\tjsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true})\n\t\trequire_NoError(t, err)\n\t\tif len(jsz.AccountDetails) > 0 && len(jsz.AccountDetails[0].Streams) > 0 {\n\t\t\tdetails := jsz.AccountDetails[0]\n\t\t\tstream := details.Streams[0]\n\t\t\treturn &stream\n\t\t}\n\t\tt.Error(\"Could not find account details\")\n\t\treturn nil\n\t}\n\tcheckState := func(t *testing.T) error {\n\t\tt.Helper()\n\n\t\tleaderSrv := c.streamLeader(\"js\", \"LTEST\")\n\t\tstreamLeader := getStreamDetails(t, leaderSrv)\n\t\t// t.Logf(\"Stream Leader: %+v\", streamLeader.State)\n\t\terrs := make([]error, 0)\n\t\tfor _, srv := range c.servers {\n\t\t\tif srv == leaderSrv {\n\t\t\t\t// Skip self\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tstream := getStreamDetails(t, srv)\n\t\t\tif stream.State.Msgs != streamLeader.State.Msgs {\n\t\t\t\terr := fmt.Errorf(\"Leader %v has %d messages, Follower %v has %d messages\",\n\t\t\t\t\tstream.Cluster.Leader, streamLeader.State.Msgs,\n\t\t\t\t\tsrv.Name(), stream.State.Msgs,\n\t\t\t\t)\n\t\t\t\terrs = append(errs, err)\n\t\t\t}\n\t\t}\n\t\tif len(errs) > 0 {\n\t\t\treturn errors.Join(errs...)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Confirm state of the leader.\n\tleaderSrv := c.streamLeader(\"js\", \"LTEST\")\n\tstreamLeader := getStreamDetails(t, leaderSrv)\n\tif streamLeader.State.Msgs != received {\n\t\tt.Errorf(\"Leader %v has %d messages stored but %d messages were received (delta: %d)\",\n\t\t\tleaderSrv.Name(), streamLeader.State.Msgs, received, received-streamLeader.State.Msgs)\n\t}\n\tcinfo, err := psub.ConsumerInfo()\n\trequire_NoError(t, err)\n\tif received != cinfo.Delivered.Consumer {\n\t\tt.Errorf(\"Unexpected consumer sequence. Got: %v, expected: %v\",\n\t\t\tcinfo.Delivered.Consumer, received)\n\t}\n\n\t// Check whether there was a drift among the leader and followers.\n\tvar (\n\t\tlastErr error\n\t\tattempts int\n\t)\nCheck:\n\tfor range time.NewTicker(1 * time.Second).C {\n\t\tlastErr = checkState(t)\n\t\tif attempts > 5 {\n\t\t\tbreak Check\n\t\t}\n\t\tattempts++\n\t}\n\n\t// Read the stream\n\tpsub2, err := cjs.PullSubscribe(\"messages.*\", \"\")\n\trequire_NoError(t, err)\n\nConsume2:\n\tfor {\n\t\tmsgs, err := psub2.Fetch(100)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, msg := range msgs {\n\t\t\tmsg.Ack()\n\n\t\t\tmeta, _ := msg.Metadata()\n\t\t\tif meta.NumPending == 0 {\n\t\t\t\tbreak Consume2\n\t\t\t}\n\t\t}\n\t}\n\n\tcinfo2, err := psub2.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\ta := cinfo.Delivered.Consumer\n\tb := cinfo2.Delivered.Consumer\n\tif a != b {\n\t\tt.Errorf(\"Consumers to same stream are at different sequences: %d vs %d\", a, b)\n\t}\n\n\t// Test is done but replicas were in sync so can stop testing at this point.\n\tif lastErr == nil {\n\t\treturn\n\t}\n\n\t// Now we will cause a few step downs while out of sync to get different results.\n\tt.Errorf(\"Replicas are out of sync:\\n%v\", lastErr)\n\n\tstepDown := func() {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"LTEST\"), nil, time.Second)\n\t}\n\t// Check StreamInfo in this state then trigger a few step downs.\n\tvar prevLeaderMsgs uint64\n\tleaderSrv = c.streamLeader(\"js\", \"LTEST\")\n\tsinfo, err := js.StreamInfo(\"LTEST\")\n\tprevLeaderMsgs = sinfo.State.Msgs\n\tfor i := 0; i < 10; i++ {\n\t\tstepDown()\n\t\ttime.Sleep(2 * time.Second)\n\n\t\tleaderSrv = c.streamLeader(\"js\", \"LTEST\")\n\t\tsinfo, err = js.StreamInfo(\"LTEST\")\n\t\tif err != nil {\n\t\t\tt.Logf(\"Error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif leaderSrv != nil && sinfo != nil {\n\t\t\tt.Logf(\"When leader is %v, Messages: %d\", leaderSrv.Name(), sinfo.State.Msgs)\n\n\t\t\t// Leave the leader as the replica with less messages that was out of sync.\n\t\t\tif prevLeaderMsgs > sinfo.State.Msgs {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tt.Logf(\"Changed to use leader %v which has %d messages\", leaderSrv.Name(), sinfo.State.Msgs)\n\n\t// Read the stream again\n\tpsub3, err := cjs.PullSubscribe(\"messages.*\", \"\")\n\trequire_NoError(t, err)\n\nConsume3:\n\tfor {\n\t\tmsgs, err := psub3.Fetch(100)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, msg := range msgs {\n\t\t\tmsg.Ack()\n\n\t\t\tmeta, _ := msg.Metadata()\n\t\t\tif meta.NumPending == 0 {\n\t\t\t\tbreak Consume3\n\t\t\t}\n\t\t}\n\t}\n\n\tcinfo3, err := psub3.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\t// Compare against consumer that was created before resource limits error\n\t// with one created before the step down.\n\ta = cinfo.Delivered.Consumer\n\tb = cinfo2.Delivered.Consumer\n\tif a != b {\n\t\tt.Errorf(\"Consumers to same stream are at different sequences: %d vs %d\", a, b)\n\t}\n\n\t// Compare against consumer that was created before resource limits error\n\t// with one created AFTER the step down.\n\ta = cinfo.Delivered.Consumer\n\tb = cinfo3.Delivered.Consumer\n\tif a != b {\n\t\tt.Errorf(\"Consumers to same stream are at different sequences: %d vs %d\", a, b)\n\t}\n\n\t// Compare consumers created after the resource limits error.\n\ta = cinfo2.Delivered.Consumer\n\tb = cinfo3.Delivered.Consumer\n\tif a != b {\n\t\tt.Errorf(\"Consumers to same stream are at different sequences: %d vs %d\", a, b)\n\t}\n}\n\nfunc TestJetStreamClusterAccountFileStoreLimits(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"limits\", 3)\n\tdefer c.shutdown()\n\n\tlimits := map[string]JetStreamAccountLimits{\n\t\t\"R1\": {\n\t\t\tMaxMemory: 1 << 10,\n\t\t\tMaxStore: 1 << 10,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t\t\"R3\": {\n\t\t\tMaxMemory: 1 << 10,\n\t\t\tMaxStore: 1 << 10,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t}\n\n\t// Update the limits in all servers.\n\tfor _, s := range c.servers {\n\t\tacc := s.GlobalAccount()\n\t\tif err := acc.UpdateJetStreamLimits(limits); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t\t}\n\t}\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor _, replicas := range []int64{1, 3} {\n\t\tsname := fmt.Sprintf(\"test-stream:%d\", replicas)\n\t\tt.Run(sname, func(t *testing.T) {\n\t\t\tsconfig := &nats.StreamConfig{\n\t\t\t\tName: sname,\n\t\t\t\tReplicas: int(replicas),\n\t\t\t\tStorage: nats.FileStorage,\n\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t}\n\t\t\t_, err := js.AddStream(sconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error creating stream: %v\", err)\n\t\t\t}\n\n\t\t\tdata := []byte(strings.Repeat(\"A\", 1<<8))\n\t\t\tfor i := 0; i < 30; i++ {\n\t\t\t\tif _, err = js.Publish(sname, data); err != nil && !strings.Contains(err.Error(), \"resource limits exceeded for account\") {\n\t\t\t\t\tt.Errorf(\"Error publishing random data (iteration %d): %v\", i, err)\n\t\t\t\t}\n\n\t\t\t\tif err = nc.Flush(); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error flushing connection: %v\", err)\n\t\t\t\t}\n\n\t\t\t\t_, err = js.StreamInfo(sname)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsi, err := js.StreamInfo(sname)\n\t\t\trequire_NoError(t, err)\n\t\t\tst := si.State\n\t\t\tmaxStore := limits[fmt.Sprintf(\"R%d\", replicas)].MaxStore\n\t\t\tif int64(st.Bytes) > replicas*maxStore {\n\t\t\t\tt.Errorf(\"Unexpected size of stream: got %d, expected less than %d\\nstate: %#v\", st.Bytes, maxStore, st)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterTieredReservationConsistency(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, cjs := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tsubj := fmt.Sprintf(\"R%d\", replicas)\n\t\tmaxBytes := int64(1)\n\t\tif replicas > 1 {\n\t\t\tmaxBytes = 10\n\t\t}\n\t\t_, err := cjs.AddStream(&nats.StreamConfig{\n\t\t\tName: subj,\n\t\t\tReplicas: replicas,\n\t\t\tStorage: nats.FileStorage,\n\t\t\tRetention: nats.LimitsPolicy,\n\t\t\tMaxBytes: maxBytes,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"R3\")\n\t})\n\n\tsl := c.streamLeader(globalAccountName, \"R1\")\n\t_, js, jsa := sl.globalAccount().getJetStreamFromAccount()\n\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tjsa.mu.RLock()\n\tdefer jsa.mu.RUnlock()\n\n\tcfg := &StreamConfig{Storage: FileStorage}\n\n\t// No tier, R1: 1, R3: 10*R\n\ttier := _EMPTY_\n\trequire_Equal(t, jsa.tieredReservation(tier, cfg), 31)\n\tstreams, reservation := js.tieredStreamAndReservationCount(globalAccountName, tier, cfg)\n\trequire_Equal(t, streams, 2)\n\trequire_Equal(t, reservation, 31)\n\n\t// R1 tier, R1: 1\n\ttier, cfg.Replicas = \"R1\", 1\n\trequire_Equal(t, jsa.tieredReservation(tier, cfg), 1)\n\tstreams, reservation = js.tieredStreamAndReservationCount(globalAccountName, tier, cfg)\n\trequire_Equal(t, streams, 1)\n\trequire_Equal(t, reservation, 1)\n\n\t// R3 tier, R3: 10\n\ttier, cfg.Replicas = \"R3\", 3\n\trequire_Equal(t, jsa.tieredReservation(tier, cfg), 10)\n\tstreams, reservation = js.tieredStreamAndReservationCount(globalAccountName, tier, cfg)\n\trequire_Equal(t, streams, 1)\n\trequire_Equal(t, reservation, 10)\n}\n\nfunc TestJetStreamClusterCorruptMetaSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tnc.Close()\n\n\t// Restart the server so it generates a snapshot.\n\ts := c.randomServer()\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\ts = c.restartServer(s)\n\trequire_False(t, s.isShuttingDown())\n\n\t// Perform a couple leader elections to add a couple more entries to the Raft log.\n\t// Once we corrupt the snapshot, we shouldn't recover with only the log.\n\tfor range 2 {\n\t\tc.waitOnLeader()\n\t\tml := c.leader()\n\t\trequire_NotNil(t, ml)\n\t\tmeta := ml.getJetStream().getMetaGroup()\n\t\trequire_NoError(t, meta.StepDown())\n\t}\n\n\t// Stop the meta group of our selected server early, making sure it can't generate a new snapshot.\n\tmeta := s.getJetStream().getMetaGroup().(*raft)\n\tmeta.Stop()\n\tmeta.WaitForStop()\n\n\tmeta.RLock()\n\tsnapfile := meta.snapfile\n\tmeta.RUnlock()\n\tconfigFile := s.getOpts().ConfigFile\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\n\t// Truncate/corrupt the snapshot.\n\trequire_NoError(t, os.Truncate(snapfile, 0))\n\n\t// The server should not start up.\n\topts := LoadConfig(configFile)\n\ts, err = NewServer(opts)\n\trequire_NoError(t, err)\n\ts.Start()\n\trequire_Equal(t, s.numRaftNodes(), 0)\n}\n\nfunc TestJetStreamClusterProcessSnapshotPanicAfterStreamDelete(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tsa, node := mset.sa, mset.node\n\tmset.mu.RUnlock()\n\trequire_True(t, sa == nil)\n\trequire_True(t, node == nil)\n\trequire_Error(t, mset.processSnapshot(&StreamReplicatedState{}, 0), errCatchupStreamStopped)\n\n\tmset.setStreamAssignment(&streamAssignment{}) // If the stream assignment is set, but the node is nil.\n\tmset.mu.RLock()\n\tsa, node = mset.sa, mset.node\n\tmset.mu.RUnlock()\n\trequire_True(t, sa != nil)\n\trequire_True(t, node == nil)\n\trequire_Error(t, mset.processSnapshot(&StreamReplicatedState{}, 0), errCatchupStreamStopped)\n}\n\nfunc TestJetStreamClusterDiscardNewPerSubjectRejectsWithoutCLFSBump(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tDiscard: nats.DiscardNew,\n\t\tDiscardNewPerSubject: true,\n\t\tMaxMsgsPerSubject: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// First publish should succeed.\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t// The second should fail, since the limit is hit.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_Error(t, err, ErrMaxMsgsPerSubject)\n\n\t// Retry after deleting, it should succeed afterward.\n\trequire_NoError(t, js.DeleteMsg(\"TEST\", 1))\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// CLFS should NOT be bumped.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, mset.getCLFS(), 0)\n\t}\n}\n\nfunc TestJetStreamClusterStreamDesyncDuringSnapshot(t *testing.T) {\n\tconst (\n\t\tKindRemoveMsg = iota\n\t\tKindReset\n\t\tKindTruncate\n\t)\n\ttest := func(t *testing.T, kind int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t\tStorage: nats.FileStorage,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 2)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tfs := mset.store.(*fileStore)\n\t\tfs.mu.Lock()\n\t\tfs.sips++\n\t\tfs.mu.Unlock()\n\n\t\tswitch kind {\n\t\tcase KindRemoveMsg:\n\t\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 1))\n\t\tcase KindReset:\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, mset.store.Truncate(0))\n\t\t\t}\n\t\tcase KindTruncate:\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, mset.store.Truncate(1))\n\t\t\t}\n\t\t}\n\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\n\tt.Run(\"RemoveMsg\", func(t *testing.T) { test(t, KindRemoveMsg) })\n\tt.Run(\"Reset\", func(t *testing.T) { test(t, KindReset) })\n\tt.Run(\"Truncate\", func(t *testing.T) { test(t, KindTruncate) })\n}\n\nfunc TestJetStreamClusterDeletedNodeDoesNotReviveStreamAfterCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tfor _, s := range c.servers {\n\t\tif s == rs {\n\t\t\tcontinue\n\t\t}\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t}\n\n\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tsnap := mset.stateSnapshot()\n\n\t// Reset the entire store so we can catchup based on the above snapshot.\n\tfs := mset.store.(*fileStore)\n\trequire_NoError(t, fs.reset())\n\n\t// Mark the node as leaderless, and get the upper-layer to start a catchup from a snapshot.\n\tnode := mset.raftNode()\n\tnode.(*raft).hasleader.Store(false)\n\tnode.ApplyQ().push(newCommittedEntry(10, []*Entry{{EntrySnapshot, snap}}))\n\n\t// Since the node is leaderless, it will retry after some time. We wait a little here to ensure\n\t// it's waiting there as well, and then we delete the node outright.\n\ttime.Sleep(time.Second)\n\tnode.Delete()\n\n\t// The stream's goroutine should eventually be stopped. This will fail if the stream is revived.\n\tvar retries int\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err = rs.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\tretries = 0\n\t\t\treturn err\n\t\t}\n\t\tif mset.isMonitorRunning() {\n\t\t\tretries = 0\n\t\t\treturn errors.New(\"monitor still running\")\n\t\t}\n\t\tif state := mset.raftNode().State(); state != Closed {\n\t\t\tretries = 0\n\t\t\treturn errors.New(\"node not closed\")\n\t\t}\n\t\tretries++\n\t\tif retries < 3 {\n\t\t\treturn errors.New(\"still confirming stable state\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/7718\nfunc TestJetStreamClusterLeakedSubsWithStreamImportOverlappingJetStreamSubs(t *testing.T) {\n\ttmpl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t ACC: {\n\t\tjetstream: enabled\n\t\tusers: [{user: acc, password: acc}]\n\t\timports: [{stream: {account: zone, subject: \">\"}}]\n\t }\n\t zone: {\n\t\tjetstream: enabled\n\t\tusers: [{user: zone, password: zone}]\n\t\texports: [{stream: \">\"}]\n\t }\n\t}\n\tno_auth_user: acc\n`\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckExpectedSubs := func(expected uint32) (actual uint32) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\te := expected\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tsubs := s.NumSubscriptions()\n\t\t\t\tif e == 0 {\n\t\t\t\t\te = subs\n\t\t\t\t} else if e != subs {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d subs, got %d\", e, subs)\n\t\t\t\t}\n\t\t\t}\n\t\t\tactual = e\n\t\t\treturn nil\n\t\t})\n\t\treturn actual\n\t}\n\n\t// Track subscription counts between stream/consumer create/deletes.\n\tvar baseline, sc, cc uint32\n\n\t// Perform a couple iterations to check we get to predictable subscription counts.\n\tfor range 3 {\n\t\t// Zero means we don't know the expected count, but still ALL servers must equal.\n\t\tinitial := checkExpectedSubs(0)\n\n\t\t// If we've iterated once, we'll know the baseline. Each next iteration must be equal to this.\n\t\tif baseline != 0 {\n\t\t\trequire_Equal(t, baseline, initial)\n\t\t}\n\n\t\t// Add the stream.\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 1,\n\t\t\tStorage: nats.FileStorage,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tsl := c.streamLeader(\"ACC\", \"TEST\")\n\t\trequire_NotNil(t, sl)\n\t\tafterStreamCreate := checkExpectedSubs(sl.NumSubscriptions())\n\t\tif sc == 0 {\n\t\t\tsc = afterStreamCreate\n\t\t}\n\t\trequire_Equal(t, sc, afterStreamCreate)\n\n\t\t// Add the consumer.\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\t\trequire_NoError(t, err)\n\t\tafterConsumerCreate := checkExpectedSubs(sl.NumSubscriptions())\n\t\tif cc == 0 {\n\t\t\tcc = afterConsumerCreate\n\t\t}\n\t\trequire_Equal(t, cc, afterConsumerCreate)\n\n\t\t// Delete the consumer, the subscriptions should drop down to what they were after the stream was created.\n\t\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER\"))\n\t\tafterConsumerDelete := checkExpectedSubs(sl.NumSubscriptions())\n\t\trequire_Equal(t, afterStreamCreate, afterConsumerDelete)\n\n\t\t// Deleting the stream should drop the subscriptions back to the baseline.\n\t\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t\tafterStreamDelete := checkExpectedSubs(sl.NumSubscriptions())\n\t\tif baseline == 0 {\n\t\t\tbaseline = afterStreamDelete\n\t\t}\n\t\trequire_Equal(t, baseline, afterStreamDelete)\n\t}\n}\n\nfunc TestJetStreamClusterInterestStreamWithConsumerFilterUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create a consumer with a filter on 'foo.a'.\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tFilterSubjects: []string{\"foo.a\", \"foo.c\"},\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.Bind(\"TEST\", \"CONSUMER\"))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tcheckFilterSubject := func(expected string) {\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\tif o == nil {\n\t\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t\t}\n\t\t\t\tif !slices.Contains(o.config().FilterSubjects, expected) {\n\t\t\t\t\treturn fmt.Errorf(\"expected filter subject %q, got %q\", expected, o.config().FilterSubjects)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckFilterSubject(\"foo.a\")\n\n\t// Publishing a message to 'foo.a' should be persisted since it matches the consumer.\n\t_, err = js.Publish(\"foo.a\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif state, err := checkStateAndErr(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t} else if state.Msgs != 1 || state.FirstSeq != 1 || state.LastSeq != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d [%d:%d]\", state.Msgs, state.FirstSeq, state.LastSeq)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Fetch and ack the message with 'foo.a'.\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmsg := msgs[0]\n\trequire_Equal(t, msg.Subject, \"foo.a\")\n\trequire_NoError(t, msg.AckSync())\n\n\t// Update the consumer, removing the 'foo.a' filter, and adding 'foo.b'.\n\tcfg.FilterSubjects = []string{\"foo.b\", \"foo.c\"}\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\tcheckFilterSubject(\"foo.b\")\n\n\t// Publishing a message to 'foo.b' should be persisted since it matches the consumer.\n\t_, err = js.Publish(\"foo.b\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif state, err := checkStateAndErr(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t} else if state.Msgs != 1 || state.FirstSeq != 2 || state.LastSeq != 2 {\n\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d [%d:%d]\", state.Msgs, state.FirstSeq, state.LastSeq)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Fetch and ack the message with 'foo.b'.\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmsg = msgs[0]\n\trequire_Equal(t, msg.Subject, \"foo.b\")\n\trequire_NoError(t, msg.AckSync())\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif state, err := checkStateAndErr(t, c, globalAccountName, \"TEST\"); err != nil {\n\t\t\treturn err\n\t\t} else if state.Msgs != 0 || state.FirstSeq != 3 || state.LastSeq != 2 {\n\t\t\treturn fmt.Errorf(\"expected 0 msgs, got %d [%d:%d]\", state.Msgs, state.FirstSeq, state.LastSeq)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterStreamRecreateChangesRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\trs := c.randomServer()\n\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn := mset.raftNode()\n\told := n.Group()\n\n\t// Recreate the stream.\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tfor range 2 {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Expect the group to change.\n\tmset, err = rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn = mset.raftNode()\n\trequire_NotEqual(t, old, n.Group())\n}\n\nfunc TestJetStreamClusterStreamScaleDownChangesRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Publish a couple messages.\n\tfor range 2 {\n\t\t_, err = js.Publish(\"foo\", []byte(\"A\"))\n\t\trequire_NoError(t, err)\n\t}\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tvar pausedServer *Server\n\tvar streamLeader *Server\n\tfor _, s := range c.servers {\n\t\tif streamLeader != nil && pausedServer != nil {\n\t\t\tbreak\n\t\t}\n\t\tif s.JetStreamIsStreamLeader(globalAccountName, \"TEST\") {\n\t\t\tstreamLeader = s\n\t\t\tcontinue\n\t\t}\n\t\t// Select a server that's neither the stream leader nor the meta leader.\n\t\tif pausedServer == nil && !s.JetStreamIsLeader() {\n\t\t\tpausedServer = s\n\t\t\tcontinue\n\t\t}\n\t}\n\trequire_NotNil(t, pausedServer)\n\trequire_NotNil(t, streamLeader)\n\n\t// Pause the meta layer on one server, simulating slow meta changes.\n\t// The stream update will take a while to apply on this server.\n\tsjs := pausedServer.getJetStream()\n\tmeta := sjs.getMetaGroup()\n\trequire_NoError(t, meta.PauseApply())\n\tmset, err := pausedServer.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn := mset.raftNode()\n\told := n.Group()\n\n\t// Scale stream down and back up.\n\tcfg.Replicas = 1\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\t// Publish a couple more messages while it's R1.\n\tfor range 2 {\n\t\t_, err = js.Publish(\"foo\", []byte(\"B\"))\n\t\trequire_NoError(t, err)\n\t}\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Wait for some time to let the servers catch each other up. Can't use equality checks here.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Step down the current stream leader without selecting a new preferred leader.\n\tlmset, err := streamLeader.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tlmset.raftNode().(*raft).switchToFollower(noLeader)\n\n\t// The paused server is the first to campaign. If Raft groups are reused, this will be the leader.\n\trequire_NoError(t, n.CampaignImmediately())\n\ttime.Sleep(500 * time.Millisecond)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Publish a couple more messages.\n\tfor range 2 {\n\t\t_, err = js.Publish(\"foo\", []byte(\"C\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for some time to have the published messages be persisted. Can't use equality checks here.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Check that the messages are received in the right order.\n\t// The paused server should only contain a subset.\n\tfor _, s := range c.servers {\n\t\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tfor seq := uint64(1); seq <= 6; seq++ {\n\t\t\tsm, err := mset.store.LoadMsg(seq, nil)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif seq <= 2 {\n\t\t\t\trequire_Equal(t, \"A\", string(sm.buf))\n\t\t\t} else if seq <= 4 {\n\t\t\t\trequire_Equal(t, \"B\", string(sm.buf))\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, \"C\", string(sm.buf))\n\t\t\t}\n\t\t}\n\t}\n\n\t// Unpause the server and have it create the scaled up stream under a new Raft group.\n\tmeta.ResumeApply()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err = pausedServer.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tn = mset.raftNode()\n\t\tif ng := n.Group(); old == ng {\n\t\t\treturn fmt.Errorf(\"expected new group but got %q\", ng)\n\t\t}\n\t\treturn nil\n\t})\n\t// Now all servers should end up being synchronized.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterStreamRescaleCatchup(t *testing.T) {\n\ttest := func(t *testing.T, doSnapshot bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.leader())\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\tvar rs *Server\n\t\tfor _, s := range c.servers {\n\t\t\t// Select a server that's neither the stream leader nor the meta leader.\n\t\t\tif !s.JetStreamIsLeader() && !s.JetStreamIsStreamLeader(globalAccountName, \"TEST\") {\n\t\t\t\trs = s\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\trequire_NotNil(t, rs)\n\t\trs.Shutdown()\n\t\trs.WaitForShutdown()\n\n\t\t// Scale stream down and back up.\n\t\tcfg.Replicas = 1\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tcfg.Replicas = 3\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Wait for some time to let the servers catch each other up. Can't use equality checks here.\n\t\ttime.Sleep(500 * time.Millisecond)\n\t\tif doSnapshot {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == rs {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsjs := s.getJetStream()\n\t\t\t\tn := sjs.getMetaGroup()\n\t\t\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, n.InstallSnapshot(snap, false))\n\t\t\t}\n\t\t}\n\n\t\t// Publish another message, after restart all servers should become synchronized.\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tc.restartServer(rs)\n\n\t\t// Now all servers should end up being synchronized.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\tt.Run(\"Catchup\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"Snapshot\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterConsumerRecreateChangesRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\tcheckConsumerCount := func(expected int) {\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif consumers := mset.numConsumers(); consumers != expected {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d consumer, got %d\", expected, consumers)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckConsumerCount(1)\n\n\trs := c.randomServer()\n\tmset, err := rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn := mset.lookupConsumer(\"CONSUMER\").raftNode()\n\told := n.Group()\n\n\t// Recreate the consumer.\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER\"))\n\tcheckConsumerCount(0)\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\tcheckConsumerCount(1)\n\n\t// Expect the group to change.\n\tmset, err = rs.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn = mset.lookupConsumer(\"CONSUMER\").raftNode()\n\trequire_NotEqual(t, old, n.Group())\n}\n\nfunc TestJetStreamClusterConsumerScaleDownChangesRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\tcheckConsumerCount := func(expected int) {\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif consumers := mset.numConsumers(); consumers != expected {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d consumer, got %d\", expected, consumers)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckConsumerCount(1)\n\n\tvar pausedServer *Server\n\tfor _, s := range c.servers {\n\t\t// Select a server that's neither the consumer leader nor the meta leader.\n\t\tif !s.JetStreamIsLeader() && !s.JetStreamIsConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\") {\n\t\t\tpausedServer = s\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_NotNil(t, pausedServer)\n\n\t// Pause the meta layer on one server, simulating slow meta changes.\n\t// The consumer update will take a while to apply on this server.\n\tsjs := pausedServer.getJetStream()\n\tmeta := sjs.getMetaGroup()\n\trequire_NoError(t, meta.PauseApply())\n\tmset, err := pausedServer.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn := mset.lookupConsumer(\"CONSUMER\").raftNode()\n\told := n.Group()\n\n\t// Scale stream down and back up.\n\tcfg.Replicas = 1\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\tcfg.Replicas = 3\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\t// Wait for some time to let the servers catch each other up.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Our paused server should still have the old consumer.\n\tmset, err = pausedServer.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn = mset.lookupConsumer(\"CONSUMER\").raftNode()\n\trequire_Equal(t, old, n.Group())\n\n\t// Unpause the server and have it create the scaled up stream under a new Raft group.\n\tmeta.ResumeApply()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset, err = pausedServer.globalAccount().lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\tif o == nil {\n\t\t\treturn fmt.Errorf(\"consumer not found\")\n\t\t}\n\t\tn = o.raftNode()\n\t\tif n == nil {\n\t\t\treturn fmt.Errorf(\"raft node not found\")\n\t\t}\n\t\tif ng := n.Group(); old == ng {\n\t\t\treturn fmt.Errorf(\"expected new group but got %q\", ng)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterConsumerRescaleCatchup(t *testing.T) {\n\ttest := func(t *testing.T, doSnapshot bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.leader())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tcfg := &nats.ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tReplicas: 3,\n\t\t}\n\t\t_, err = js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif consumers := mset.numConsumers(); consumers != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 1 consumer, got %d\", consumers)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tvar rs *Server\n\t\tfor _, s := range c.servers {\n\t\t\t// Select a server that's neither the stream leader nor the meta leader.\n\t\t\tif !s.JetStreamIsLeader() && !s.JetStreamIsConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\") {\n\t\t\t\trs = s\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\trequire_NotNil(t, rs)\n\t\trs.Shutdown()\n\t\trs.WaitForShutdown()\n\n\t\t// Scale stream down and back up.\n\t\tcfg.Replicas = 1\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t\tcfg.Replicas = 3\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Wait for some time to let the servers catch each other up. Can't use equality checks here.\n\t\ttime.Sleep(500 * time.Millisecond)\n\t\tif doSnapshot {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == rs {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsjs := s.getJetStream()\n\t\t\t\tn := sjs.getMetaGroup()\n\t\t\t\tsnap, _, _, err := sjs.metaSnapshot()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, n.InstallSnapshot(snap, false))\n\t\t\t}\n\t\t}\n\n\t\t// After restart all servers should become synchronized.\n\t\tc.restartServer(rs)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tvar g string\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\t\tif o == nil {\n\t\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t\t}\n\t\t\t\tn := o.raftNode()\n\t\t\t\tif n == nil {\n\t\t\t\t\treturn errors.New(\"no raft node\")\n\t\t\t\t} else if ng := n.Group(); g == \"\" {\n\t\t\t\t\tg = ng\n\t\t\t\t} else if ng != g {\n\t\t\t\t\treturn fmt.Errorf(\"expected same group, got %q and %q\", g, ng)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tt.Run(\"Catchup\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"Snapshot\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestJetStreamClusterConcurrentStreamUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\ts.Shutdown()\n\t}\n\n\t// The stream update to seal the stream will be received and proposed by the meta leader,\n\t// but it will not be able to achieve quorum immediately.\n\tcfg.Sealed = true\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.Publish(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req))\n\n\t// We need to wait a bit to ensure the above request is handled first.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// A concurrent stream update should error immediately if the config update check fails.\n\tcfg.Sealed = false\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not unseal a sealed stream\")))\n\n\t// Confirm the meta leader actually tracked the inflight stream update that was sent first.\n\tsjs, cc := ml.getJetStreamCluster()\n\tsjs.mu.RLock()\n\ti := len(cc.inflightStreams)\n\tsjs.mu.RUnlock()\n\trequire_Equal(t, i, 1)\n\n\t// Restart the servers, eventually the stream should be reporting as sealed.\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\tc.restartServer(s)\n\t}\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tm, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, cfg.Name), nil, 200*time.Millisecond)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar resp JSApiStreamInfoResponse\n\t\tif err = json.Unmarshal(m.Data, &resp); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !resp.Config.Sealed {\n\t\t\treturn errors.New(\"stream isn't sealed yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// The inflight state should be cleared.\n\tfor _, s := range c.servers {\n\t\tsjs, cc = s.getJetStreamCluster()\n\t\tsjs.mu.RLock()\n\t\ti = len(cc.inflightStreams)\n\t\tsjs.mu.RUnlock()\n\t\trequire_Equal(t, i, 0)\n\t}\n}\n\nfunc TestJetStreamClusterConcurrentConsumerCreateWithMaxConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tMaxConsumers: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\ts.Shutdown()\n\t}\n\n\t// The consumer create will be received and proposed by the meta leader,\n\t// but it will not be able to achieve quorum.\n\tccr := CreateConsumerRequest{Stream: \"TEST\", Config: ConsumerConfig{Durable: \"C1\", Replicas: 1}}\n\treq, err := json.Marshal(ccr)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.Publish(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"C1\"), req))\n\n\t// We need to wait a bit to ensure the above request is handled first.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Another consumer create should error immediately since with the inflight consumer we're at limits.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"C2\", Replicas: 1})\n\trequire_Error(t, err, NewJSMaximumConsumersLimitError())\n\n\t// Confirm the meta leader actually tracked the inflight consumer create that was sent first.\n\tsjs, cc := ml.getJetStreamCluster()\n\tsjs.mu.RLock()\n\tiu := len(cc.inflightConsumers)\n\tsjs.mu.RUnlock()\n\trequire_Equal(t, iu, 1)\n\n\t// Restart the servers, eventually the consumer should exist.\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\tc.restartServer(s)\n\t}\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tvar found bool\n\t\tfor _, s := range c.servers {\n\t\t\tsjs = s.getJetStream()\n\t\t\tsjs.mu.RLock()\n\t\t\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"C1\")\n\t\t\tsjs.mu.RUnlock()\n\t\t\tif ca != nil {\n\t\t\t\tfound = true\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\treturn errors.New(\"consumer not found\")\n\t\t}\n\n\t\tsjs, cc = ml.getJetStreamCluster()\n\t\tsjs.mu.RLock()\n\t\tiu = len(cc.inflightConsumers)\n\t\tsjs.mu.RUnlock()\n\t\tif iu != 0 {\n\t\t\treturn fmt.Errorf(\"expected no inflight consumer updates, got %d\", iu)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterLostConsumerAfterInflightConsumerUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\tfor _, s := range c.servers {\n\t\tif s == ml {\n\t\t\tcontinue\n\t\t}\n\t\ts.Shutdown()\n\t}\n\n\t// The consumer update will be received and proposed by the meta leader,\n\t// but it will not be able to achieve quorum.\n\tccr := CreateConsumerRequest{Stream: \"TEST\", Config: ConsumerConfig{Durable: \"CONSUMER\"}}\n\treq, err := json.Marshal(ccr)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.Publish(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"CONSUMER\"), req))\n\n\t// We need to wait a bit to ensure the above request is handled.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Confirm the meta leader actually tracked the inflight consumer update that was sent.\n\tsjs, cc := ml.getJetStreamCluster()\n\tsjs.mu.RLock()\n\tiu := len(cc.inflightConsumers)\n\tsjs.mu.RUnlock()\n\trequire_Equal(t, iu, 1)\n\n\t// Snapshot meta, if it didn't separately track the inflight consumer update, it could lose the entire consumer.\n\trequire_NoError(t, ml.JetStreamSnapshotMeta())\n\tml.Shutdown()\n\n\t// Restart all servers.\n\tfor _, s := range c.servers {\n\t\tc.restartServer(s)\n\t}\n\t// Check the consumer still exists on all servers.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tsjs = s.getJetStream()\n\t\t\tsjs.mu.RLock()\n\t\t\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\t\tsjs.mu.RUnlock()\n\t\t\tif ca == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterStreamRaftGroupChangesWhenMovingToOrOffR1(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\trequireGroupPrefix := func(prefix string) {\n\t\tt.Helper()\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\trequire_NotNil(t, sl)\n\t\tsjs := sl.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\t\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\t\tif sa.Group == nil {\n\t\t\tt.Fatal(\"no group\")\n\t\t} else if !strings.HasPrefix(sa.Group.Name, prefix) {\n\t\t\tt.Fatalf(\"expected group prefix %q, got %q\", prefix, sa.Group.Name)\n\t\t}\n\t}\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"S-R1F-\")\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"S-R3F-\")\n\n\tcfg.Replicas = 1\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"S-R1F-\")\n\n\tcfg.Replicas = 5\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"S-R5F-\")\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\t// The group MUST remain the same as what it was for R5.\n\t// Changing it would violate replication safety.\n\trequireGroupPrefix(\"S-R5F-\")\n}\n\nfunc TestJetStreamClusterConsumerRaftGroupChangesWhenMovingToOrOffR1(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\trequireGroupPrefix := func(prefix string) {\n\t\tt.Helper()\n\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\trequire_NotNil(t, cl)\n\t\tsjs := cl.getJetStream()\n\t\tsjs.mu.RLock()\n\t\tdefer sjs.mu.RUnlock()\n\t\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\tif ca.Group == nil {\n\t\t\tt.Fatal(\"no group\")\n\t\t} else if !strings.HasPrefix(ca.Group.Name, prefix) {\n\t\t\tt.Fatalf(\"expected group prefix %q, got %q\", prefix, ca.Group.Name)\n\t\t}\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 1,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"C-R1F-\")\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"C-R3F-\")\n\n\tcfg.Replicas = 1\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"C-R1F-\")\n\n\tcfg.Replicas = 5\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\trequireGroupPrefix(\"C-R5F-\")\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\t// The group MUST remain the same as what it was for R5.\n\t// Changing it would violate replication safety.\n\trequireGroupPrefix(\"C-R5F-\")\n}\n\nfunc TestJetStreamClusterStreamUpdateMaxConsumersLimit(t *testing.T) {\n\ttest := func(t *testing.T, replicas int, remove bool) {\n\t\tvar (\n\t\t\tgetStreamLeader func() *Server\n\t\t\trestart func()\n\t\t\ts *Server\n\t\t)\n\t\trequire_NotEqual(t, replicas, 0)\n\t\tif replicas == 1 {\n\t\t\ttmpl := `\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\t\t`\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, t.TempDir())))\n\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tgetStreamLeader = func() *Server { return s }\n\t\t\trestart = func() {\n\t\t\t\ts.Shutdown()\n\t\t\t\ts.WaitForShutdown()\n\t\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\t\t// No need to defer shutdown here, that's already handled by the defer above.\n\t\t\t\tgetStreamLeader = func() *Server { return s }\n\t\t\t}\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\n\t\t\tgetStreamLeader = func() *Server { return c.streamLeader(globalAccountName, \"TEST\") }\n\t\t\trestart = func() {\n\t\t\t\tc.stopAll()\n\t\t\t\tc.restartAll()\n\t\t\t\tc.waitOnLeader()\n\t\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\t\t}\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Pre-create consumer configs to be reused.\n\t\tcc1 := &nats.ConsumerConfig{Durable: \"CONSUMER_1\", Replicas: replicas}\n\t\tcc2 := &nats.ConsumerConfig{Durable: \"CONSUMER_2\", Replicas: replicas}\n\t\tcc3 := &nats.ConsumerConfig{Durable: \"CONSUMER_3\", Replicas: replicas}\n\n\t\t// Create two consumers.\n\t\tfor _, cc := range []*nats.ConsumerConfig{cc1, cc2} {\n\t\t\t_, err = js.AddConsumer(\"TEST\", cc)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Check we have two consumers.\n\t\tsl := getStreamLeader()\n\t\trequire_NotNil(t, sl)\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(mset.getPublicConsumers()), 2)\n\n\t\t// Updating the max consumers limit should preserve the current consumers, even if they're over the limit.\n\t\tcfg.MaxConsumers = 1\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(mset.getPublicConsumers()), 2)\n\n\t\t// Adding a consumer shouldn't be allowed as we're already over the limit (2 > 1).\n\t\t_, err = js.AddConsumer(\"TEST\", cc3)\n\t\trequire_Error(t, err, NewJSMaximumConsumersLimitError())\n\n\t\t// We should still be allowed to update a pre-existing consumer.\n\t\tfor _, cc := range []*nats.ConsumerConfig{cc1, cc2} {\n\t\t\t_, err = js.UpdateConsumer(\"TEST\", cc)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// If we're testing removes, if we delete a consumer we're still at limit, so can't add another.\n\t\tif remove {\n\t\t\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"CONSUMER_2\"))\n\t\t\t_, err = js.AddConsumer(\"TEST\", cc3)\n\t\t\trequire_Error(t, err, NewJSMaximumConsumersLimitError())\n\t\t}\n\n\t\t// Restart, and if we didn't remove a consumer above, we should still come up with\n\t\t// all consumers even if it's over the limit.\n\t\trestart()\n\t\tsl = getStreamLeader()\n\t\trequire_NotNil(t, sl)\n\t\tmset, err = sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif remove {\n\t\t\trequire_Len(t, len(mset.getPublicConsumers()), 1)\n\t\t} else {\n\t\t\trequire_Len(t, len(mset.getPublicConsumers()), 2)\n\t\t}\n\n\t\t// Reconnect.\n\t\tnc.Close()\n\t\tnc, js = jsClientConnect(t, sl)\n\t\tdefer nc.Close()\n\n\t\t// Allow 'infinite' consumers again, and confirm all consumers can be created.\n\t\tcfg.MaxConsumers = -1\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tfor _, cc := range []*nats.ConsumerConfig{cc1, cc2, cc3} {\n\t\t\t_, err = js.AddConsumer(\"TEST\", cc)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_Len(t, len(mset.getPublicConsumers()), 3)\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, remove := range []bool{false, true} {\n\t\t\tdesc := \"Add\"\n\t\t\tif remove {\n\t\t\t\tdesc = \"Remove\"\n\t\t\t}\n\t\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, desc), func(t *testing.T) { test(t, replicas, remove) })\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterScaleDownWaitsForMonitorRoutineQuit(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create R3 stream and consumer.\n\tscfg := &nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3}\n\tccfg := &nats.ConsumerConfig{Name: \"CONSUMER\", Replicas: 3}\n\t_, err := js.AddStream(scfg)\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", ccfg)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tsjs := s.getJetStream()\n\t\t\tif sjs.streamAssignment(globalAccountName, \"TEST\") == nil {\n\t\t\t\treturn errors.New(\"stream not found\")\n\t\t\t}\n\t\t\tif sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tcf := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, cf)\n\tmset, err := cf.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Increment the wait group for this test to confirm the right ordering.\n\to.mu.Lock()\n\tinMonitor := o.inMonitor\n\twg := &o.monitorWg\n\twg.Add(1)\n\to.mu.Unlock()\n\trequire_True(t, inMonitor)\n\n\t// The monitor routine should stop.\n\tccfg.Replicas = 1\n\t_, err = js.UpdateConsumer(\"TEST\", ccfg)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\t\tif o.inMonitor {\n\t\t\treturn errors.New(\"consumer still in monitor\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// The consumer itself should still exist.\n\trequire_NotNil(t, mset.lookupConsumer(\"CONSUMER\"))\n\n\t// Simulate the monitor routine being done now and the consumer being removed.\n\twg.Done()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif mset.lookupConsumer(\"CONSUMER\") != nil {\n\t\t\treturn errors.New(\"consumer still exists\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsf := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sf)\n\tmset, err = sf.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Increment the wait group for this test to confirm the right ordering.\n\tmset.mu.Lock()\n\tinMonitor = mset.inMonitor\n\twg = &mset.monitorWg\n\twg.Add(1)\n\tmset.mu.Unlock()\n\trequire_True(t, inMonitor)\n\n\t// The monitor routine should stop.\n\tscfg.Replicas = 1\n\t_, err = js.UpdateStream(scfg)\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\t\tif mset.inMonitor {\n\t\t\treturn errors.New(\"stream still in monitor\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// The stream itself should still exist.\n\t_, err = sf.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Simulate the monitor routine being done now and the stream being removed.\n\twg.Done()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t_, err = sf.globalAccount().lookupStream(\"TEST\")\n\t\tif !errors.Is(err, NewJSStreamNotFoundError()) {\n\t\t\treturn errors.New(\"stream still exists\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterConsumerRemapWaitsForMonitorRoutineQuit(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create R3 stream and consumer.\n\tscfg := &nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3}\n\tccfg := &nats.ConsumerConfig{Name: \"CONSUMER\", Replicas: 3}\n\t_, err := js.AddStream(scfg)\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", ccfg)\n\trequire_NoError(t, err)\n\tml := c.leader()\n\tsjs, cc := ml.getJetStreamCluster()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif sjs.streamAssignment(globalAccountName, \"TEST\") == nil {\n\t\t\treturn errors.New(\"stream not found\")\n\t\t}\n\t\tif sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\") == nil {\n\t\t\treturn errors.New(\"consumer not found\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tmset, err := ml.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Increment the wait group for this test to confirm the right ordering.\n\to.mu.Lock()\n\tinMonitor := o.inMonitor\n\twg := &o.monitorWg\n\twg.Add(1)\n\trn := o.node\n\to.mu.Unlock()\n\trequire_True(t, inMonitor)\n\n\t// Simulate a consumer Raft group remapping that has been collapsed down into just a single update.\n\t// Instead of one update to R1 and then to R3, it's just one update straight to the new R3 group.\n\tca := sjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, ca)\n\tcca := ca.copyGroup()\n\tcca.Group.Name = groupNameForConsumer(cca.Group.Peers, cca.Group.Storage)\n\trequire_NoError(t, cc.meta.Propose(encodeAddConsumerAssignment(cca)))\n\n\t// The monitor routine should stop.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\t\tif o.inMonitor {\n\t\t\treturn errors.New(\"consumer still in monitor\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// The previous Raft node should be stopped.\n\trequire_Equal(t, rn.State(), Closed)\n\n\t// Simulate the monitor routine being done now and the new monitor routine being started.\n\twg.Done()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\t\tif o.node == nil {\n\t\t\treturn errors.New(\"consumer has no Raft node yet\")\n\t\t} else if !o.inMonitor {\n\t\t\treturn errors.New(\"consumer monitor not started\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterAccountStoreLimits(t *testing.T) {\n\ttest := func(t *testing.T, replicas int, storage nats.StorageType) {\n\t\tstoreLimit := fileStoreMsgSize(\"B\", nil, nil)\n\t\tmemLimit := memStoreMsgSize(\"B\", nil, nil)\n\t\tlimit := JetStreamAccountLimits{\n\t\t\tMaxMemory: int64(memLimit * 6),\n\t\t\tMemoryMaxStreamBytes: int64(memLimit * 3),\n\t\t\tMaxStore: int64(storeLimit * 6),\n\t\t\tStoreMaxStreamBytes: int64(storeLimit * 3),\n\t\t\tMaxBytesRequired: true,\n\t\t}\n\t\ttier := fmt.Sprintf(\"R%d\", replicas)\n\t\tlimits := map[string]JetStreamAccountLimits{tier: limit}\n\n\t\tvar s *Server\n\t\tvar c *cluster\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t\trequire_NoError(t, s.globalAccount().UpdateJetStreamLimits(limits))\n\t\t} else {\n\t\t\tc = createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\tfor _, cs := range c.servers {\n\t\t\t\trequire_NoError(t, cs.globalAccount().UpdateJetStreamLimits(limits))\n\t\t\t}\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tresourcesErr := NewJSStorageResourcesExceededError()\n\t\tmaxAcc, maxBytes := limit.MaxStore, limit.StoreMaxStreamBytes\n\t\tif storage == nats.MemoryStorage {\n\t\t\tresourcesErr = NewJSMemoryResourcesExceededError()\n\t\t\tmaxAcc, maxBytes = limit.MaxMemory, limit.MemoryMaxStreamBytes\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t// No MaxBytes errors because it's required.\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"A\", Replicas: replicas, Storage: storage})\n\t\trequire_Error(t, err, NewJSStreamMaxBytesRequiredError())\n\n\t\t// MaxBytes larger than account limit errors.\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"A\", Replicas: replicas, Storage: storage, MaxBytes: maxAcc + 1})\n\t\tif replicas == 1 {\n\t\t\trequire_Error(t, err, resourcesErr)\n\t\t} else {\n\t\t\trequire_Error(t, err, NewJSStreamMaxStreamBytesExceededError())\n\t\t}\n\n\t\t// MaxBytes larger than bytes limit errors.\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"A\", Replicas: replicas, Storage: storage, MaxBytes: maxBytes + 1})\n\t\trequire_Error(t, err, NewJSStreamMaxStreamBytesExceededError())\n\n\t\t// Create two streams that exactly fit the limit.\n\t\tfor _, subj := range []string{\"A\", \"B\"} {\n\t\t\t_, err = js.AddStream(&nats.StreamConfig{Name: subj, Replicas: replicas, Storage: storage, MaxBytes: maxBytes})\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Another stream over the limit errors.\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"C\", Replicas: replicas, Storage: storage, MaxBytes: 1})\n\t\trequire_Error(t, err, resourcesErr)\n\n\t\t// We can publish more than the maximum bytes into the stream as it is DiscardOld.\n\t\tfor range 10 {\n\t\t\t_, err = js.Publish(\"A\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Once the last stream fills up too close to the account limit, we eventually get an error.\n\t\tstart := time.Now()\n\t\tfor i := 0; ; i++ {\n\t\t\tif time.Since(start) > 5*time.Second {\n\t\t\t\tt.Fatalf(\"timed out waiting for error\")\n\t\t\t}\n\t\t\t_, err = js.Publish(\"B\", nil)\n\t\t\tif i < 3 {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t} else if replicas == 1 {\n\t\t\t\t// Expect an error immediately after we hit the limit if we're R1.\n\t\t\t\trequire_Error(t, err, NewJSAccountResourcesExceededError())\n\t\t\t\tbreak\n\t\t\t} else if err != nil {\n\t\t\t\t// For replicated this might take some time as servers report about their usage.\n\t\t\t\trequire_Error(t, err, NewJSAccountResourcesExceededError())\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t}\n\n\t\t// If clustered, confirm all is in sync still.\n\t\tif c != nil {\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"B\")\n\t\t\t})\n\t\t}\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\tt.Run(\"Memory\", func(t *testing.T) { test(t, replicas, nats.MemoryStorage) })\n\t\t\tt.Run(\"File\", func(t *testing.T) { test(t, replicas, nats.FileStorage) })\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterDontEncodeConsumerStateInMetaSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Add a replicated stream and a single replica consumer.\n\tscfg := &nats.StreamConfig{Name: \"TEST\", Replicas: 3}\n\t_, err := js.AddStream(scfg)\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Name: \"CONSUMER\", Replicas: 1})\n\trequire_NoError(t, err)\n\n\t// Ensure the stream and consumer leaders differ.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tif sl == cl {\n\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, mset.raftNode().StepDown())\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tsl = c.streamLeader(globalAccountName, \"TEST\")\n\t}\n\trequire_NotEqual(t, sl, cl)\n\n\t// Scale down the stream so the R1 consumer needs to be moved to a different server.\n\tscfg.Replicas = 1\n\t_, err = js.UpdateStream(scfg)\n\trequire_NoError(t, err)\n\n\t// Signal that the meta leader should create a snapshot. We need to do this indirectly\n\t// through a noop peer change, as we need the monitor goroutine to perform the snapshot.\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\tmeta := ml.getJetStream().getMetaGroup().(*raft)\n\tmeta.RLock()\n\tpapplied := meta.papplied\n\tmeta.RUnlock()\n\trequire_NoError(t, meta.ProposeAddPeer(meta.ID()))\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tmeta.RLock()\n\t\tdefer meta.RUnlock()\n\t\tif papplied == meta.papplied {\n\t\t\treturn errors.New(\"no snapshot yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Load the new snapshot and validate consumer state isn't preserved.\n\tsnap, err := meta.loadLastSnapshot()\n\trequire_NoError(t, err)\n\tsjs := ml.getJetStream()\n\taccStreams, err := sjs.decodeMetaSnapshot(snap.data)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(accStreams), 1)\n\tstreams := accStreams[globalAccountName]\n\trequire_Len(t, len(streams), 1)\n\tstream := streams[\"TEST\"]\n\trequire_NotNil(t, stream)\n\trequire_Len(t, len(stream.consumers), 1)\n\tconsumer := stream.consumers[\"CONSUMER\"]\n\trequire_NotNil(t, consumer)\n\trequire_True(t, consumer.State == nil)\n}\n\nfunc TestJetStreamClusteredStreamCreateIdempotentWithSources(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"source.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"SOURCED\",\n\t\tSubjects: []string{\"sourced.>\"},\n\t\tReplicas: 3,\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"SOURCE\",\n\t\t\t\tFilterSubject: \"source.>\",\n\t\t\t},\n\t\t},\n\t}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Step down the stream leader until it lands on the meta leader.\n\t// This ensures the meta leader's stored assignment has iname populated\n\t// via the shared StreamSource pointer.\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\tsl := c.streamLeader(globalAccountName, \"SOURCED\")\n\trequire_NotNil(t, sl)\n\tif sl != ml {\n\t\tmset, err := sl.globalAccount().lookupStream(\"SOURCED\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, mset.raftNode().StepDown(ml.Node()))\n\t\tc.waitOnStreamLeader(globalAccountName, \"SOURCED\")\n\t\tsl = c.streamLeader(globalAccountName, \"SOURCED\")\n\t}\n\trequire_Equal(t, ml, sl)\n\n\t// The second create should be idempotent, and succeed even though iname was set.\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterMetaSnapshotPreservesConsumersOnStreamUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Name: \"CONSUMER\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\ttriggerMetaSnapshot := func(t *testing.T, c *cluster) {\n\t\tt.Helper()\n\t\tml := c.leader()\n\t\trequire_NotNil(t, ml)\n\t\tmeta := ml.getJetStream().getMetaGroup().(*raft)\n\t\tmeta.RLock()\n\t\tpapplied := meta.papplied\n\t\tmeta.RUnlock()\n\t\trequire_NoError(t, meta.ProposeAddPeer(meta.ID()))\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmeta.RLock()\n\t\t\tdefer meta.RUnlock()\n\t\t\tif papplied == meta.papplied {\n\t\t\t\treturn errors.New(\"no snapshot yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tloadSnapshotStreams := func(t *testing.T, c *cluster) map[string]map[string]*streamAssignment {\n\t\tt.Helper()\n\t\tml := c.leader()\n\t\trequire_NotNil(t, ml)\n\t\tmeta := ml.getJetStream().getMetaGroup().(*raft)\n\t\tsnap, err := meta.loadLastSnapshot()\n\t\trequire_NoError(t, err)\n\t\tsjs := ml.getJetStream()\n\t\taccStreams, err := sjs.decodeMetaSnapshot(snap.data)\n\t\trequire_NoError(t, err)\n\t\treturn accStreams\n\t}\n\n\t// Create a baseline snapshot that includes consumers.\n\ttriggerMetaSnapshot(t, c)\n\taccStreams := loadSnapshotStreams(t, c)\n\tstream := accStreams[globalAccountName][\"TEST\"]\n\trequire_NotNil(t, stream)\n\trequire_NotNil(t, stream.consumers[\"CONSUMER\"])\n\n\t// Update stream config, then create a new snapshot from the previous checkpoint.\n\tcfg.Description = \"updated\"\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\ttriggerMetaSnapshot(t, c)\n\n\t// Updated snapshots must preserve existing stream consumers.\n\taccStreams = loadSnapshotStreams(t, c)\n\tstream = accStreams[globalAccountName][\"TEST\"]\n\trequire_NotNil(t, stream)\n\trequire_NotNil(t, stream.consumers[\"CONSUMER\"])\n}\n\nfunc TestJetStreamClusterCheckForOrphansDoesntDeleteDirectConsumers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"MIRROR\", Mirror: &nats.StreamSource{Name: \"TEST\"}})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sl)\n\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif c := mset.numDirectConsumers(); c != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 consumer, got %d\", c)\n\t\t}\n\t\treturn nil\n\t})\n\n\trequire_Equal(t, mset.numDirectConsumers(), 1)\n\tsl.getJetStream().checkForOrphans()\n\trequire_Equal(t, mset.numDirectConsumers(), 1)\n}\n\nfunc TestJetStreamClusterConsumerAssignmentsOrInflightSeqWithInflightStream(t *testing.T) {\n\tconst acc, stream, consumer = \"A\", \"S\", \"C\"\n\tjs := &jetStream{cluster: &jetStreamCluster{\n\t\tstreams: map[string]map[string]*streamAssignment{},\n\t\tinflightStreams: map[string]map[string]*inflightStreamInfo{\n\t\t\tacc: {stream: {streamAssignment: &streamAssignment{Config: &StreamConfig{Name: stream}}}},\n\t\t},\n\t\tinflightConsumers: map[string]map[string]map[string]*inflightConsumerInfo{\n\t\t\tacc: {stream: {consumer: {consumerAssignment: &consumerAssignment{Name: consumer}}}},\n\t\t},\n\t}}\n\n\tvar got []string\n\tfor ca := range js.consumerAssignmentsOrInflightSeq(acc, stream) {\n\t\tgot = append(got, ca.Name)\n\t}\n\tif len(got) != 1 || got[0] != consumer {\n\t\tt.Fatalf(\"Unexpected consumers: %+v\", got)\n\t}\n}\n" }, { "path": "server/jetstream_cluster_4_test.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests_4\n\npackage server\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/big\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc TestJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T) {\n\tt.Run(\"max msgs\", func(t *testing.T) {\n\t\ttestJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{\n\t\t\tName: \"WQTEST_MM\",\n\t\t\tSubjects: []string{\"messages.*\"},\n\t\t\tReplicas: 3,\n\t\t\tMaxAge: 10 * time.Minute,\n\t\t\tMaxMsgs: 100,\n\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\tDiscard: nats.DiscardNew,\n\t\t})\n\t})\n\tt.Run(\"max bytes\", func(t *testing.T) {\n\t\ttestJetStreamClusterWorkQueueStreamDiscardNewDesync(t, &nats.StreamConfig{\n\t\t\tName: \"WQTEST_MB\",\n\t\t\tSubjects: []string{\"messages.*\"},\n\t\t\tReplicas: 3,\n\t\t\tMaxAge: 10 * time.Minute,\n\t\t\tMaxBytes: 1 * 1024 * 1024,\n\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\tDiscard: nats.DiscardNew,\n\t\t})\n\t})\n}\n\nfunc testJetStreamClusterWorkQueueStreamDiscardNewDesync(t *testing.T, sc *nats.StreamConfig) {\n\tconf := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {\n\t\tstore_dir: '%s',\n\t}\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n system_account: sys\n no_auth_user: js\n\taccounts {\n\t sys {\n\t users = [\n\t { user: sys, pass: sys }\n\t ]\n\t }\n\t js {\n\t jetstream = enabled\n\t users = [\n\t { user: js, pass: js }\n\t ]\n\t }\n\t}`\n\tc := createJetStreamClusterWithTemplate(t, conf, sc.Name, 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcnc, cjs := jsClientConnect(t, c.randomServer())\n\tdefer cnc.Close()\n\n\t_, err := js.AddStream(sc)\n\trequire_NoError(t, err)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tpsub, err := cjs.PullSubscribe(\"messages.*\", \"consumer\")\n\trequire_NoError(t, err)\n\n\tstepDown := func() {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Name), nil, time.Second)\n\t}\n\n\t// Messages will be produced and consumed in parallel, then once there are\n\t// enough errors a leader election will be triggered.\n\tvar (\n\t\twg sync.WaitGroup\n\t\treceived uint64\n\t\terrCh = make(chan error, 100_000)\n\t\treceivedMap = make(map[string]*nats.Msg)\n\t)\n\twg.Add(1)\n\tgo func() {\n\t\ttick := time.NewTicker(20 * time.Millisecond)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\tmsgs, err := psub.Fetch(10, nats.MaxWait(200*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\t// The consumer will continue to timeout here eventually.\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\treceived++\n\t\t\t\t\treceivedMap[msg.Subject] = msg\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tshouldDrop := make(map[string]error)\n\twg.Add(1)\n\tgo func() {\n\t\tpayload := []byte(strings.Repeat(\"A\", 1024))\n\t\ttick := time.NewTicker(1 * time.Millisecond)\n\t\tfor i := 1; ; i++ {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\tsubject := fmt.Sprintf(\"messages.%d\", i)\n\t\t\t\t_, err := js.Publish(subject, payload, nats.RetryAttempts(0))\n\t\t\t\t// Capture the messages that have failed.\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Unless it was a timeout, we can't be sure if the message was received,\n\t\t\t\t\t// and we just didn't get a response.\n\t\t\t\t\tif errors.Is(err, nats.ErrTimeout) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\terrCh <- err\n\t\t\t\t\tshouldDrop[subject] = err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Collect enough errors to cause things to get out of sync.\n\tvar errCount int\nSetup:\n\tfor {\n\t\tselect {\n\t\tcase err = <-errCh:\n\t\t\terrCount++\n\t\t\tif errCount%500 == 0 {\n\t\t\t\tstepDown()\n\t\t\t} else if errCount >= 2000 {\n\t\t\t\t// Stop both producing and consuming.\n\t\t\t\tcancel()\n\t\t\t\tbreak Setup\n\t\t\t}\n\t\tcase <-time.After(5 * time.Second):\n\t\t\t// Unblock the test and continue.\n\t\t\tcancel()\n\t\t\tbreak Setup\n\t\t}\n\t}\n\n\t// Both goroutines should be exiting now..\n\twg.Wait()\n\n\t// Let acks propagate for stream checks.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Check messages that ought to have been dropped.\n\tfor subject := range receivedMap {\n\t\tfound, ok := shouldDrop[subject]\n\t\tif ok {\n\t\t\tt.Errorf(\"Should have dropped message published on %q since got error: %v\", subject, found)\n\t\t}\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/5071\nfunc TestJetStreamClusterStreamPlacementDistribution(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 5)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor i := 1; i <= 10; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST:%d\", i),\n\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d.*\", i)},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// 10 streams, 3 replicas div 5 servers.\n\texpectedStreams := 10 * 3 / 5\n\tfor _, s := range c.servers {\n\t\tjsz, err := s.Jsz(nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, jsz.Streams, expectedStreams)\n\t}\n}\n\nfunc TestJetStreamClusterSourceWorkingQueueWithLimit(t *testing.T) {\n\tconst (\n\t\ttotalMsgs = 300\n\t\tmaxMsgs = 100\n\t\tmaxBytes = maxMsgs * 100\n\t\tmsgPayloadFormat = \"%0100d\" // %0100d is 100 bytes. Must match payload value above.\n\t)\n\tc := createJetStreamClusterExplicit(t, \"WQ3\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"test\", Subjects: []string{\"test\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"wq\", MaxMsgs: maxMsgs, Discard: nats.DiscardNew, Retention: nats.WorkQueuePolicy,\n\t\tSources: []*nats.StreamSource{{Name: \"test\"}}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"wq2\", MaxBytes: maxBytes, Discard: nats.DiscardNew, Retention: nats.WorkQueuePolicy,\n\t\tSources: []*nats.StreamSource{{Name: \"test\"}}, Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsendBatch := func(subject string, n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, err = js.Publish(subject, []byte(fmt.Sprintf(msgPayloadFormat, i)))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tf := func(ss *nats.Subscription, done chan bool) {\n\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\tm, err := ss.Fetch(1, nats.MaxWait(3*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\tp, err := strconv.Atoi(string(m[0].Data))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, p, i)\n\t\t\ttime.Sleep(11 * time.Millisecond)\n\t\t\terr = m[0].Ack()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tdone <- true\n\t}\n\n\t// Populate the sourced stream.\n\tsendBatch(\"test\", totalMsgs)\n\n\tcheckFor(t, 3*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != maxMsgs {\n\t\t\treturn fmt.Errorf(\"expected %d msgs on stream wq, got state: %+v\", maxMsgs, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckFor(t, 3*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq2\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Bytes > maxBytes {\n\t\t\treturn fmt.Errorf(\"expected no more than %d bytes on stream wq2, got state: %+v\", maxBytes, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t_, err = js.AddConsumer(\"wq\", &nats.ConsumerConfig{Durable: \"wqc\", FilterSubject: \"test\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tss1, err := js.PullSubscribe(\"test\", \"wqc\", nats.Bind(\"wq\", \"wqc\"))\n\trequire_NoError(t, err)\n\n\tvar doneChan1 = make(chan bool)\n\tgo f(ss1, doneChan1)\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs > 0 && si.State.Msgs <= maxMsgs {\n\t\t\treturn fmt.Errorf(\"expected 0 msgs on stream wq, got: %d\", si.State.Msgs)\n\t\t} else if si.State.Msgs > maxMsgs {\n\t\t\tt.Fatalf(\"got more than our %d message limit on stream wq: %+v\", maxMsgs, si.State)\n\t\t}\n\n\t\treturn nil\n\t})\n\n\tselect {\n\tcase <-doneChan1:\n\t\tss1.Drain()\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t_, err = js.AddConsumer(\"wq2\", &nats.ConsumerConfig{Durable: \"wqc\", FilterSubject: \"test\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tss2, err := js.PullSubscribe(\"test\", \"wqc\", nats.Bind(\"wq2\", \"wqc\"))\n\trequire_NoError(t, err)\n\n\tvar doneChan2 = make(chan bool)\n\tgo f(ss2, doneChan2)\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq2\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Bytes > 0 && si.State.Bytes <= maxBytes {\n\t\t\treturn fmt.Errorf(\"expected 0 bytes on stream wq2, got: %+v\", si.State)\n\t\t} else if si.State.Bytes > maxBytes {\n\t\t\tt.Fatalf(\"got more than our %d bytes limit on stream wq2: %+v\", maxMsgs, si.State)\n\t\t}\n\n\t\treturn nil\n\t})\n\n\tselect {\n\tcase <-doneChan2:\n\t\tss2.Drain()\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n}\n\nfunc TestJetStreamClusterConsumerPauseViaConfig(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tReplicas: 3,\n\t})\n\n\tsub, err := js.PullSubscribe(\"foo\", \"\", nats.Bind(\"TEST\", \"my_consumer\"))\n\trequire_NoError(t, err)\n\n\tstepdown := func() {\n\t\tt.Helper()\n\t\t_, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"my_consumer\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\t}\n\n\tpublish := func(wait time.Duration) {\n\t\tt.Helper()\n\t\tfor i := 0; i < 5; i++ {\n\t\t\t_, err = js.Publish(\"foo\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err := sub.Fetch(5, nats.MaxWait(wait))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 5)\n\t}\n\n\t// This should be fast as there's no deadline.\n\tpublish(time.Second)\n\n\t// Now we're going to set the deadline.\n\tdeadline := jsTestPause_PauseConsumer(t, nc, \"TEST\", \"my_consumer\", time.Now().Add(time.Second*3))\n\tc.waitOnAllCurrent()\n\n\t// It will now take longer than 3 seconds.\n\tpublish(time.Second * 5)\n\trequire_True(t, time.Now().After(deadline))\n\n\t// The next set of publishes after the deadline should now be fast.\n\tpublish(time.Second)\n\n\t// We'll kick the leader, but since we're after the deadline, this\n\t// should still be fast.\n\tstepdown()\n\tpublish(time.Second)\n\n\t// Now we're going to do an update and then immediately kick the\n\t// leader. The pause should still be in effect afterwards.\n\tdeadline = jsTestPause_PauseConsumer(t, nc, \"TEST\", \"my_consumer\", time.Now().Add(time.Second*3))\n\tc.waitOnAllCurrent()\n\tpublish(time.Second * 5)\n\trequire_True(t, time.Now().After(deadline))\n\n\t// The next set of publishes after the deadline should now be fast.\n\tpublish(time.Second)\n}\n\nfunc TestJetStreamClusterConsumerPauseViaEndpoint(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"push\", \"pull\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tt.Run(\"PullConsumer\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tName: \"pull_consumer\",\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(\"pull\", \"\", nats.Bind(\"TEST\", \"pull_consumer\"))\n\t\trequire_NoError(t, err)\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err := sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\n\t\t// Now we'll pause the consumer for 3 seconds.\n\t\tdeadline := time.Now().Add(time.Second * 3)\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"pull_consumer\", deadline).Equal(deadline))\n\t\tc.waitOnAllCurrent()\n\n\t\t// This should fail as we'll wait for only half of the deadline.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, err = sub.Fetch(10, nats.MaxWait(time.Until(deadline)/2))\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t// This should succeed after a short wait, and when we're done,\n\t\t// we should be after the deadline.\n\t\tmsgs, err = sub.Fetch(10)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\t\trequire_True(t, time.Now().After(deadline))\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err = sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"pull_consumer\", time.Time{}).Equal(time.Time{}))\n\t\tc.waitOnAllCurrent()\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err = sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\t})\n\n\tt.Run(\"PushConsumer\", func(t *testing.T) {\n\t\tch := make(chan *nats.Msg, 100)\n\t\t_, err = js.ChanSubscribe(\"push\", ch, nats.BindStream(\"TEST\"), nats.ConsumerName(\"push_consumer\"))\n\t\trequire_NoError(t, err)\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\n\t\t// Now we'll pause the consumer for 3 seconds.\n\t\tdeadline := time.Now().Add(time.Second * 3)\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"push_consumer\", deadline).Equal(deadline))\n\t\tc.waitOnAllCurrent()\n\n\t\t// This should succeed after a short wait, and when we're done,\n\t\t// we should be after the deadline.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second*5)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t\trequire_True(t, time.Now().After(deadline))\n\t\t}\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"push_consumer\", time.Time{}).Equal(time.Time{}))\n\t\tc.waitOnAllCurrent()\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\t})\n}\n\nfunc TestJetStreamClusterConsumerPauseTimerFollowsLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Hour)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tReplicas: 3,\n\t})\n\n\tfor i := 0; i < 10; i++ {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\t\tc.waitOnAllCurrent()\n\n\t\tfor _, s := range c.servers {\n\t\t\tstream, err := s.gacc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tconsumer := stream.lookupConsumer(\"my_consumer\")\n\t\t\trequire_NotEqual(t, consumer, nil)\n\n\t\t\tisLeader := s.JetStreamIsConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\n\t\t\tconsumer.mu.RLock()\n\t\t\thasTimer := consumer.uptmr != nil\n\t\t\tconsumer.mu.RUnlock()\n\n\t\t\trequire_Equal(t, isLeader, hasTimer)\n\t\t}\n\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"my_consumer\"), nil, maxElectionTimeout)\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestJetStreamClusterConsumerPauseResumeViaEndpoint(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"CONSUMER\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tgetConsumerInfo := func() ConsumerInfo {\n\t\tvar ci ConsumerInfo\n\t\tinfoResp, err := nc.Request(\"$JS.API.CONSUMER.INFO.TEST.CONSUMER\", nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(infoResp.Data, &ci)\n\t\trequire_NoError(t, err)\n\t\treturn ci\n\t}\n\n\t// Ensure we are not paused\n\trequire_False(t, getConsumerInfo().Paused)\n\n\t// Use pause advisories to know when pause/resume is applied.\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(JSAdvisoryConsumerPausePre+\".TEST.CONSUMER\", ch)\n\trequire_NoError(t, err)\n\n\t// Now we'll pause the consumer for 30 seconds.\n\tdeadline := time.Now().Add(time.Second * 30)\n\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"CONSUMER\", deadline).Equal(deadline))\n\trequire_ChanRead(t, ch, time.Second*2)\n\trequire_Len(t, len(ch), 0)\n\n\t// Ensure the consumer reflects being paused\n\trequire_True(t, getConsumerInfo().Paused)\n\n\tsubj := fmt.Sprintf(\"$JS.API.CONSUMER.PAUSE.%s.%s\", \"TEST\", \"CONSUMER\")\n\t_, err = nc.Request(subj, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_ChanRead(t, ch, time.Second*2)\n\trequire_Len(t, len(ch), 0)\n\n\t// Ensure the consumer reflects being resumed\n\trequire_False(t, getConsumerInfo().Paused)\n}\n\nfunc TestJetStreamClusterConsumerPauseHeartbeats(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Hour)\n\tdsubj := \"deliver_subj\"\n\n\tci := jsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tHeartbeat: time.Millisecond * 100,\n\t\tDeliverSubject: dsubj,\n\t})\n\trequire_True(t, ci.Config.PauseUntil.Equal(deadline))\n\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(dsubj, ch)\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\tmsg := require_ChanRead(t, ch, time.Millisecond*200)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"100\")\n\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Idle Heartbeat\")\n\t}\n}\n\nfunc TestJetStreamClusterConsumerPauseAdvisories(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckAdvisory := func(msg *nats.Msg, shouldBePaused bool, deadline time.Time) {\n\t\tt.Helper()\n\t\tvar advisory JSConsumerPauseAdvisory\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.Stream, \"TEST\")\n\t\trequire_Equal(t, advisory.Consumer, \"my_consumer\")\n\t\trequire_Equal(t, advisory.Paused, shouldBePaused)\n\t\trequire_True(t, advisory.PauseUntil.Equal(deadline))\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(JSAdvisoryConsumerPausePre+\".TEST.my_consumer\", ch)\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Second)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tReplicas: 3,\n\t})\n\n\t// First advisory should tell us that the consumer was paused\n\t// on creation.\n\tmsg := require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// The second one for the unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Now we'll pause the consumer for a second using the API.\n\tdeadline = time.Now().Add(time.Second)\n\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"my_consumer\", deadline).Equal(deadline))\n\n\t// Third advisory should tell us about the pause via the API.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Finally that should unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Now we're going to set the deadline into the future so we can\n\t// see what happens when we kick leaders or restart.\n\tdeadline = time.Now().Add(time.Hour)\n\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"my_consumer\", deadline).Equal(deadline))\n\n\t// Setting the deadline should have generated an advisory.\n\tmsg = require_ChanRead(t, ch, time.Second)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Try to kick the consumer leader.\n\tsrv := c.consumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\tsrv.JetStreamStepdownConsumer(globalAccountName, \"TEST\", \"my_consumer\")\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\n\t// This shouldn't have generated an advisory.\n\trequire_NoChanRead(t, ch, time.Second)\n}\n\nfunc TestJetStreamClusterConsumerPauseSurvivesRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckTimer := func(s *Server) {\n\t\tstream, err := s.gacc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tconsumer := stream.lookupConsumer(\"my_consumer\")\n\t\trequire_NotEqual(t, consumer, nil)\n\n\t\tconsumer.mu.RLock()\n\t\ttimer := consumer.uptmr\n\t\tconsumer.mu.RUnlock()\n\t\trequire_True(t, timer != nil)\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Hour)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tReplicas: 3,\n\t})\n\n\t// First try with just restarting the consumer leader.\n\tsrv := c.consumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\tsrv.Shutdown()\n\tc.restartServer(srv)\n\tc.waitOnAllCurrent()\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\tleader := c.consumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\trequire_True(t, leader != nil)\n\tcheckTimer(leader)\n\n\t// Then try restarting the entire cluster.\n\tc.stopAll()\n\tc.restartAllSamePorts()\n\tc.waitOnAllCurrent()\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\tleader = c.consumerLeader(globalAccountName, \"TEST\", \"my_consumer\")\n\trequire_True(t, leader != nil)\n\tcheckTimer(leader)\n}\n\nfunc TestJetStreamClusterConsumerNRGCleanup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// First call is just to create the pull subscribers.\n\t_, err = js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"dlc\"))\n\n\t// Now delete the stream.\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t// Now make sure we cleaned up the NRG directories for the stream and consumer.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tvar numConsumers, numStreams int\n\t\tfor _, s := range c.servers {\n\t\t\tsd := s.JetStreamConfig().StoreDir\n\t\t\tnd := filepath.Join(sd, \"$SYS\", \"_js_\")\n\t\t\tf, err := os.Open(nd)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdirs, err := f.ReadDir(-1)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, fi := range dirs {\n\t\t\t\tif strings.HasPrefix(fi.Name(), \"C-\") {\n\t\t\t\t\tnumConsumers++\n\t\t\t\t} else if strings.HasPrefix(fi.Name(), \"S-\") {\n\t\t\t\t\tnumStreams++\n\t\t\t\t}\n\t\t\t}\n\t\t\tf.Close()\n\t\t}\n\n\t\tif numConsumers != 0 {\n\t\t\treturn fmt.Errorf(\"expected 0 consumers, got %d\", numConsumers)\n\t\t}\n\t\tif numStreams != 0 {\n\t\t\treturn fmt.Errorf(\"expected 0 streams, got %d\", numStreams)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/4878\nfunc TestClusteredInterestConsumerFilterEdit(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"INTEREST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"interest.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"INTEREST\", &nats.ConsumerConfig{\n\t\tDurable: \"C0\",\n\t\tFilterSubject: \"interest.>\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(fmt.Sprintf(\"interest.%d\", i), []byte(strconv.Itoa(i)))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// we check we got 10 messages\n\tnfo, err := js.StreamInfo(\"INTEREST\")\n\trequire_NoError(t, err)\n\tif nfo.State.Msgs != 10 {\n\t\tt.Fatalf(\"expected 10 messages got %d\", nfo.State.Msgs)\n\t}\n\n\t// now we lower the consumer interest from all subjects to 1,\n\t// then check the stream state and check if interest behavior still works\n\t_, err = js.UpdateConsumer(\"INTEREST\", &nats.ConsumerConfig{\n\t\tDurable: \"C0\",\n\t\tFilterSubject: \"interest.1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// we should now have only one message left\n\tnfo, err = js.StreamInfo(\"INTEREST\")\n\trequire_NoError(t, err)\n\tif nfo.State.Msgs != 1 {\n\t\tt.Fatalf(\"expected 1 message got %d\", nfo.State.Msgs)\n\t}\n}\n\nfunc TestJetStreamClusterDoubleAckRedelivery(t *testing.T) {\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {\n\t\t\tstore_dir: '%s',\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\tserver_tags: [\"test\"]\n\t\tsystem_account: sys\n\t\tno_auth_user: js\n\t\taccounts {\n\t\t\tsys { users = [ { user: sys, pass: sys } ] }\n\t\t\tjs {\n\t\t\t\tjetstream = enabled\n\t\t\t\tusers = [ { user: js, pass: js } ]\n\t\t }\n\t\t}`\n\tc := createJetStreamClusterWithTemplate(t, conf, \"R3F\", 3)\n\tdefer c.shutdown()\n\tfor _, s := range c.servers {\n\t\ts.optsMu.Lock()\n\t\ts.opts.LameDuckDuration = 15 * time.Second\n\t\ts.opts.LameDuckGracePeriod = -15 * time.Second\n\t\ts.optsMu.Unlock()\n\t}\n\ts := c.randomNonLeader()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsc, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"LIMITS\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tReplicas: 3,\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\tstepDown := func() {\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, sc.Config.Name), nil, time.Second)\n\t}\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tvar wg sync.WaitGroup\n\tproducer := func(name string) {\n\t\twg.Add(1)\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tdefer wg.Done()\n\n\t\ti := 0\n\t\tpayload := []byte(strings.Repeat(\"Z\", 1024))\n\t\tfor range time.NewTicker(1 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tmsgID := nats.MsgId(fmt.Sprintf(\"%s:%d\", name, i))\n\t\t\tjs.PublishAsync(\"foo.bar\", payload, msgID, nats.RetryAttempts(10))\n\t\t\ti++\n\t\t}\n\t}\n\tgo producer(\"A\")\n\tgo producer(\"B\")\n\tgo producer(\"C\")\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"ABC\", nats.AckWait(5*time.Second), nats.MaxAckPending(1000), nats.PullMaxWaiting(1000))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype ackResult struct {\n\t\tack *nats.Msg\n\t\toriginal *nats.Msg\n\t\tredelivered *nats.Msg\n\t}\n\treceived := make(map[string]int64)\n\tacked := make(map[string]*ackResult)\n\terrors := make(map[string]error)\n\textraRedeliveries := 0\n\n\twg.Add(1)\n\tgo func() {\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tdefer wg.Done()\n\n\t\tfetch := func(t *testing.T, batchSize int) {\n\t\t\tmsgs, err := sub.Fetch(batchSize, nats.MaxWait(500*time.Millisecond))\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, msg := range msgs {\n\t\t\t\tmeta, err := msg.Metadata()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tmsgID := msg.Header.Get(nats.MsgIdHdr)\n\t\t\t\tif err, ok := errors[msgID]; ok {\n\t\t\t\t\tt.Logf(\"Redelivery (num_delivered: %v) after failed Ack Sync: %+v - %+v - error: %v\", meta.NumDelivered, msg.Reply, msg.Header, err)\n\t\t\t\t}\n\t\t\t\tif resp, ok := acked[msgID]; ok {\n\t\t\t\t\tt.Errorf(\"Redelivery (num_delivered: %v) after successful Ack Sync: msgID:%v - redelivered:%v - original:%+v - ack:%+v\",\n\t\t\t\t\t\tmeta.NumDelivered, msgID, msg.Reply, resp.original.Reply, resp.ack)\n\t\t\t\t\tresp.redelivered = msg\n\t\t\t\t\textraRedeliveries++\n\t\t\t\t}\n\t\t\t\treceived[msgID]++\n\n\t\t\t\t// Retry quickly a few times after there is a failed ack.\n\t\t\tRetries:\n\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\tresp, err := nc.Request(msg.Reply, []byte(\"+ACK\"), 500*time.Millisecond)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Logf(\"Error: %v %v\", msgID, err)\n\t\t\t\t\t\terrors[msgID] = err\n\t\t\t\t\t} else {\n\t\t\t\t\t\tacked[msgID] = &ackResult{resp, msg, nil}\n\t\t\t\t\t\tbreak Retries\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tfetch(t, 1)\n\t\t\tfetch(t, 50)\n\t\t}\n\t}()\n\n\t// Cause a couple of step downs before the restarts as well.\n\ttime.AfterFunc(5*time.Second, func() { stepDown() })\n\ttime.AfterFunc(10*time.Second, func() { stepDown() })\n\n\t// Let messages be produced, and then restart the servers.\n\t<-time.After(15 * time.Second)\n\nNextServer:\n\tfor _, s := range c.servers {\n\t\ts.lameDuckMode()\n\t\ts.WaitForShutdown()\n\t\ts = c.restartServer(s)\n\n\t\thctx, hcancel := context.WithTimeout(ctx, 60*time.Second)\n\t\tdefer hcancel()\n\t\tfor range time.NewTicker(2 * time.Second).C {\n\t\t\tselect {\n\t\t\tcase <-hctx.Done():\n\t\t\t\tt.Logf(\"WRN: Timed out waiting for healthz from %s\", s)\n\t\t\t\tcontinue NextServer\n\t\t\tdefault:\n\t\t\t}\n\n\t\t\tstatus := s.healthz(nil)\n\t\t\tif status.StatusCode == 200 {\n\t\t\t\tcontinue NextServer\n\t\t\t}\n\t\t}\n\t\t// Pause in-between server restarts.\n\t\ttime.Sleep(10 * time.Second)\n\t}\n\n\t// Stop all producer and consumer goroutines to check results.\n\tcancel()\n\tselect {\n\tcase <-ctx.Done():\n\tcase <-time.After(10 * time.Second):\n\t}\n\twg.Wait()\n\tif extraRedeliveries > 0 {\n\t\tt.Fatalf(\"Received %v redeliveries after a successful ack\", extraRedeliveries)\n\t}\n}\n\nfunc TestJetStreamClusterBusyStreams(t *testing.T) {\n\tt.Skip(\"Too long for CI at the moment\")\n\ttype streamSetup struct {\n\t\tconfig *nats.StreamConfig\n\t\tconsumers []*nats.ConsumerConfig\n\t\tsubjects []string\n\t}\n\ttype job func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster)\n\ttype testParams struct {\n\t\tcluster string\n\t\tstreams []*streamSetup\n\t\tproducers int\n\t\tconsumers int\n\t\trestartAny bool\n\t\trestartWait time.Duration\n\t\tldmRestart bool\n\t\trolloutRestart bool\n\t\trestarts int\n\t\tcheckHealthz bool\n\t\tjobs []job\n\t\texpect job\n\t\tduration time.Duration\n\t\tproducerMsgs int\n\t\tproducerMsgSize int\n\t}\n\ttest := func(t *testing.T, test *testParams) {\n\t\tconf := `\n listen: 127.0.0.1:-1\n http: 127.0.0.1:-1\n server_name: %s\n jetstream: {\n domain: \"cloud\"\n store_dir: '%s',\n }\n cluster {\n name: %s\n listen: 127.0.0.1:%d\n routes = [%s]\n }\n server_tags: [\"test\"]\n system_account: sys\n\n no_auth_user: js\n accounts {\n sys { users = [ { user: sys, pass: sys } ] }\n\n js { jetstream = enabled\n users = [ { user: js, pass: js } ]\n }\n }`\n\t\tc := createJetStreamClusterWithTemplate(t, conf, test.cluster, 3)\n\t\tdefer c.shutdown()\n\t\tfor _, s := range c.servers {\n\t\t\ts.optsMu.Lock()\n\t\t\ts.opts.LameDuckDuration = 15 * time.Second\n\t\t\ts.opts.LameDuckGracePeriod = -15 * time.Second\n\t\t\ts.optsMu.Unlock()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar wg sync.WaitGroup\n\t\tfor _, stream := range test.streams {\n\t\t\tstream := stream\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\t_, err := js.AddStream(stream.config)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tfor _, consumer := range stream.consumers {\n\t\t\t\t\t_, err := js.AddConsumer(stream.config.Name, consumer)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t\twg.Wait()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), test.duration)\n\t\tdefer cancel()\n\t\tfor _, stream := range test.streams {\n\t\t\tpayload := []byte(strings.Repeat(\"A\", test.producerMsgSize))\n\t\t\tstream := stream\n\t\t\tsubjects := stream.subjects\n\n\t\t\t// Create publishers on different connections that sends messages\n\t\t\t// to all the consumers subjects.\n\t\t\tvar n atomic.Uint64\n\t\t\tfor i := 0; i < test.producers; i++ {\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\tfor range time.NewTicker(1 * time.Millisecond).C {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tfor _, subject := range subjects {\n\t\t\t\t\t\t\tmsgID := nats.MsgId(fmt.Sprintf(\"n:%d\", n.Load()))\n\t\t\t\t\t\t\t_, err := js.Publish(subject, payload, nats.AckWait(200*time.Millisecond), msgID)\n\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\tif nn := n.Add(1); int(nn) >= test.producerMsgs {\n\t\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\n\t\t\t// Create multiple parallel pull subscribers per consumer config.\n\t\t\tfor i := 0; i < test.consumers; i++ {\n\t\t\t\tfor _, consumer := range stream.consumers {\n\t\t\t\t\twg.Add(1)\n\n\t\t\t\t\tconsumer := consumer\n\t\t\t\t\tgo func() {\n\t\t\t\t\t\tdefer wg.Done()\n\n\t\t\t\t\t\tfor attempts := 0; attempts < 60; attempts++ {\n\t\t\t\t\t\t\t_, err := js.ConsumerInfo(stream.config.Name, consumer.Name)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tt.Logf(\"WRN: Failed creating pull subscriber: %v - %v - %v - %v\",\n\t\t\t\t\t\t\t\t\tconsumer.FilterSubject, stream.config.Name, consumer.Name, err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsub, err := js.PullSubscribe(consumer.FilterSubject, \"\", nats.Bind(stream.config.Name, consumer.Name))\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tt.Logf(\"WRN: Failed creating pull subscriber: %v - %v - %v - %v\",\n\t\t\t\t\t\t\t\tconsumer.FilterSubject, stream.config.Name, consumer.Name, err)\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\t\tfor range time.NewTicker(100 * time.Millisecond).C {\n\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond))\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\t\t\t\tmsg.Ack()\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tmsgs, err = sub.Fetch(100, nats.MaxWait(200*time.Millisecond))\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\t\t\t\tmsg.Ack()\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor _, job := range test.jobs {\n\t\t\tgo job(t, nc, js, c)\n\t\t}\n\t\tif test.restarts > 0 {\n\t\t\twg.Add(1)\n\t\t\ttime.AfterFunc(test.restartWait, func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tfor i := 0; i < test.restarts; i++ {\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase test.restartAny:\n\t\t\t\t\t\ts := c.servers[rand.Intn(len(c.servers))]\n\t\t\t\t\t\tif test.ldmRestart {\n\t\t\t\t\t\t\ts.lameDuckMode()\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\ts.Shutdown()\n\t\t\t\t\t\t}\n\t\t\t\t\t\ts.WaitForShutdown()\n\t\t\t\t\t\tc.restartServer(s)\n\t\t\t\t\tcase test.rolloutRestart:\n\t\t\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\t\t\tif test.ldmRestart {\n\t\t\t\t\t\t\t\ts.lameDuckMode()\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\ts.Shutdown()\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\ts.WaitForShutdown()\n\t\t\t\t\t\t\ts = c.restartServer(s)\n\n\t\t\t\t\t\t\tif test.checkHealthz {\n\t\t\t\t\t\t\t\thctx, hcancel := context.WithTimeout(ctx, 15*time.Second)\n\t\t\t\t\t\t\t\tdefer hcancel()\n\n\t\t\t\t\t\t\tHealthz:\n\t\t\t\t\t\t\t\tfor range time.NewTicker(2 * time.Second).C {\n\t\t\t\t\t\t\t\t\tselect {\n\t\t\t\t\t\t\t\t\tcase <-hctx.Done():\n\t\t\t\t\t\t\t\t\t\tbreak Healthz\n\t\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tstatus := s.healthz(nil)\n\t\t\t\t\t\t\t\t\tif status.StatusCode == 200 {\n\t\t\t\t\t\t\t\t\t\tbreak Healthz\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tc.waitOnClusterReady()\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t\ttest.expect(t, nc, js, c)\n\t\tcancel()\n\t\twg.Wait()\n\t}\n\tstepDown := func(nc *nats.Conn, streamName string) {\n\t\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, streamName), nil, time.Second)\n\t}\n\tgetStreamDetails := func(t *testing.T, c *cluster, accountName, streamName string) *StreamDetail {\n\t\tt.Helper()\n\t\tsrv := c.streamLeader(accountName, streamName)\n\t\tjsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true})\n\t\trequire_NoError(t, err)\n\t\tfor _, acc := range jsz.AccountDetails {\n\t\t\tif acc.Name == accountName {\n\t\t\t\tfor _, stream := range acc.Streams {\n\t\t\t\t\tif stream.Name == streamName {\n\t\t\t\t\t\treturn &stream\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tt.Error(\"Could not find account details\")\n\t\treturn nil\n\t}\n\tcheckMsgsEqual := func(t *testing.T, c *cluster, accountName, streamName string) {\n\t\tstate := getStreamDetails(t, c, accountName, streamName).State\n\t\tvar msets []*stream\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err := s.LookupAccount(accountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(streamName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmsets = append(msets, mset)\n\t\t}\n\t\tfor seq := state.FirstSeq; seq <= state.LastSeq; seq++ {\n\t\t\tvar msgId string\n\t\t\tvar smv StoreMsg\n\t\t\tfor _, mset := range msets {\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tsm, err := mset.store.LoadMsg(seq, &smv)\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif msgId == _EMPTY_ {\n\t\t\t\t\tmsgId = string(sm.hdr)\n\t\t\t\t} else if msgId != string(sm.hdr) {\n\t\t\t\t\tt.Fatalf(\"MsgIds do not match for seq %d: %q vs %q\", seq, msgId, sm.hdr)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tcheckConsumer := func(t *testing.T, c *cluster, accountName, streamName, consumerName string) {\n\t\tt.Helper()\n\t\tvar leader string\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, err := s.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true})\n\t\t\trequire_NoError(t, err)\n\t\t\tfor _, acc := range jsz.AccountDetails {\n\t\t\t\tif acc.Name == accountName {\n\t\t\t\t\tfor _, stream := range acc.Streams {\n\t\t\t\t\t\tif stream.Name == streamName {\n\t\t\t\t\t\t\tfor _, consumer := range stream.Consumer {\n\t\t\t\t\t\t\t\tif leader == \"\" {\n\t\t\t\t\t\t\t\t\tleader = consumer.Cluster.Leader\n\t\t\t\t\t\t\t\t} else if leader != consumer.Cluster.Leader {\n\t\t\t\t\t\t\t\t\tt.Errorf(\"There are two leaders for %s/%s: %s vs %s\",\n\t\t\t\t\t\t\t\t\t\tstream.Name, consumer.Name, leader, consumer.Cluster.Leader)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"R1F/rescale/R3F/sources:10/limits\", func(t *testing.T) {\n\t\ttestDuration := 3 * time.Minute\n\t\ttotalStreams := 10\n\t\tstreams := make([]*streamSetup, totalStreams)\n\t\tsources := make([]*nats.StreamSource, totalStreams)\n\t\tfor i := 0; i < totalStreams; i++ {\n\t\t\tname := fmt.Sprintf(\"test:%d\", i)\n\t\t\tst := &streamSetup{\n\t\t\t\tconfig: &nats.StreamConfig{\n\t\t\t\t\tName: name,\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"test.%d.*\", i)},\n\t\t\t\t\tReplicas: 1,\n\t\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\t},\n\t\t\t}\n\t\t\tst.subjects = append(st.subjects, fmt.Sprintf(\"test.%d.0\", i))\n\t\t\tsources[i] = &nats.StreamSource{Name: name}\n\t\t\tstreams[i] = st\n\t\t}\n\n\t\t// Create Source consumer.\n\t\tsourceSetup := &streamSetup{\n\t\t\tconfig: &nats.StreamConfig{\n\t\t\t\tName: \"source-test\",\n\t\t\t\tReplicas: 1,\n\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\tSources: sources,\n\t\t\t},\n\t\t\tconsumers: make([]*nats.ConsumerConfig, 0),\n\t\t}\n\t\tcc := &nats.ConsumerConfig{\n\t\t\tName: \"A\",\n\t\t\tDurable: \"A\",\n\t\t\tFilterSubject: \"test.>\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t}\n\t\tsourceSetup.consumers = append(sourceSetup.consumers, cc)\n\t\tstreams = append(streams, sourceSetup)\n\n\t\tscale := func(replicas int, wait time.Duration) job {\n\t\t\treturn func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) {\n\t\t\t\tconfig := sourceSetup.config\n\t\t\t\ttime.AfterFunc(wait, func() {\n\t\t\t\t\tconfig.Replicas = replicas\n\t\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\t\t_, err := js.UpdateStream(config)\n\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\n\t\texpect := func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) {\n\t\t\t// The source stream should not be stuck or be different from the other streams.\n\t\t\ttime.Sleep(testDuration + 1*time.Minute)\n\t\t\taccName := \"js\"\n\t\t\tstreamName := \"source-test\"\n\n\t\t\t// Check a few times to see if there are no changes in the number of messages.\n\t\t\tvar changed bool\n\t\t\tvar prevMsgs uint64\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsinfo, err := js.StreamInfo(streamName)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Error: %v\", err)\n\t\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tprevMsgs = sinfo.State.Msgs\n\t\t\t}\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsinfo, err := js.StreamInfo(streamName)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Error: %v\", err)\n\t\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tchanged = prevMsgs != sinfo.State.Msgs\n\t\t\t\tprevMsgs = sinfo.State.Msgs\n\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t}\n\t\t\tif !changed {\n\t\t\t\t// Doing a leader step down should not cause the messages to change.\n\t\t\t\tstepDown(nc, streamName)\n\n\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\tsinfo, err := js.StreamInfo(streamName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Logf(\"Error: %v\", err)\n\t\t\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tchanged = prevMsgs != sinfo.State.Msgs\n\t\t\t\t\tprevMsgs = sinfo.State.Msgs\n\t\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t\t}\n\t\t\t\tif changed {\n\t\t\t\t\tt.Error(\"Stream msgs changed after the step down\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t/////////////////////////////////////////////////////////////////////////////////////////\n\t\t\t// //\n\t\t\t// The number of messages sourced should match the count from all the other streams. //\n\t\t\t// //\n\t\t\t/////////////////////////////////////////////////////////////////////////////////////////\n\t\t\tvar expectedMsgs uint64\n\t\t\tfor i := 0; i < totalStreams; i++ {\n\t\t\t\tname := fmt.Sprintf(\"test:%d\", i)\n\t\t\t\tsinfo, err := js.StreamInfo(name)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\texpectedMsgs += sinfo.State.Msgs\n\t\t\t}\n\t\t\tsinfo, err := js.StreamInfo(streamName)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tgotMsgs := sinfo.State.Msgs\n\t\t\tif gotMsgs != expectedMsgs {\n\t\t\t\tt.Errorf(\"stream with sources has %v messages, but total sourced messages should be %v\", gotMsgs, expectedMsgs)\n\t\t\t}\n\t\t\tcheckConsumer(t, c, accName, streamName, \"A\")\n\t\t\tcheckMsgsEqual(t, c, accName, streamName)\n\t\t}\n\t\ttest(t, &testParams{\n\t\t\tcluster: t.Name(),\n\t\t\tstreams: streams,\n\t\t\tproducers: 10,\n\t\t\tconsumers: 10,\n\t\t\trestarts: 1,\n\t\t\trolloutRestart: true,\n\t\t\tldmRestart: true,\n\t\t\tcheckHealthz: true,\n\t\t\t// TODO(dlc) - If this overlaps with the scale jobs this test will fail.\n\t\t\t// Leaders will be elected with partial state.\n\t\t\trestartWait: 65 * time.Second,\n\t\t\tjobs: []job{\n\t\t\t\tscale(3, 15*time.Second),\n\t\t\t\tscale(1, 30*time.Second),\n\t\t\t\tscale(3, 60*time.Second),\n\t\t\t},\n\t\t\texpect: expect,\n\t\t\tduration: testDuration,\n\t\t\tproducerMsgSize: 1024,\n\t\t\tproducerMsgs: 100_000,\n\t\t})\n\t})\n\n\tt.Run(\"rollouts\", func(t *testing.T) {\n\t\tshared := func(t *testing.T, sc *nats.StreamConfig, tp *testParams) func(t *testing.T) {\n\t\t\treturn func(t *testing.T) {\n\t\t\t\ttestDuration := 3 * time.Minute\n\t\t\t\ttotalStreams := 30\n\t\t\t\tconsumersPerStream := 5\n\t\t\t\tstreams := make([]*streamSetup, totalStreams)\n\t\t\t\tfor i := 0; i < totalStreams; i++ {\n\t\t\t\t\tname := fmt.Sprintf(\"test:%d\", i)\n\t\t\t\t\tst := &streamSetup{\n\t\t\t\t\t\tconfig: &nats.StreamConfig{\n\t\t\t\t\t\t\tName: name,\n\t\t\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"test.%d.*\", i)},\n\t\t\t\t\t\t\tReplicas: 3,\n\t\t\t\t\t\t\tDiscard: sc.Discard,\n\t\t\t\t\t\t\tRetention: sc.Retention,\n\t\t\t\t\t\t\tStorage: sc.Storage,\n\t\t\t\t\t\t\tMaxMsgs: sc.MaxMsgs,\n\t\t\t\t\t\t\tMaxBytes: sc.MaxBytes,\n\t\t\t\t\t\t\tMaxAge: sc.MaxAge,\n\t\t\t\t\t\t},\n\t\t\t\t\t\tconsumers: make([]*nats.ConsumerConfig, 0),\n\t\t\t\t\t}\n\t\t\t\t\tfor j := 0; j < consumersPerStream; j++ {\n\t\t\t\t\t\tsubject := fmt.Sprintf(\"test.%d.%d\", i, j)\n\t\t\t\t\t\tname := fmt.Sprintf(\"A:%d:%d\", i, j)\n\t\t\t\t\t\tcc := &nats.ConsumerConfig{\n\t\t\t\t\t\t\tName: name,\n\t\t\t\t\t\t\tDurable: name,\n\t\t\t\t\t\t\tFilterSubject: subject,\n\t\t\t\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\t\t\t}\n\t\t\t\t\t\tst.consumers = append(st.consumers, cc)\n\t\t\t\t\t\tst.subjects = append(st.subjects, subject)\n\t\t\t\t\t}\n\t\t\t\t\tstreams[i] = st\n\t\t\t\t}\n\t\t\t\texpect := func(t *testing.T, nc *nats.Conn, js nats.JetStreamContext, c *cluster) {\n\t\t\t\t\ttime.Sleep(testDuration + 1*time.Minute)\n\t\t\t\t\taccName := \"js\"\n\t\t\t\t\tfor i := 0; i < totalStreams; i++ {\n\t\t\t\t\t\tstreamName := fmt.Sprintf(\"test:%d\", i)\n\t\t\t\t\t\tcheckMsgsEqual(t, c, accName, streamName)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttest(t, &testParams{\n\t\t\t\t\tcluster: t.Name(),\n\t\t\t\t\tstreams: streams,\n\t\t\t\t\tproducers: 10,\n\t\t\t\t\tconsumers: 10,\n\t\t\t\t\trestarts: tp.restarts,\n\t\t\t\t\trolloutRestart: tp.rolloutRestart,\n\t\t\t\t\tldmRestart: tp.ldmRestart,\n\t\t\t\t\tcheckHealthz: tp.checkHealthz,\n\t\t\t\t\trestartWait: tp.restartWait,\n\t\t\t\t\texpect: expect,\n\t\t\t\t\tduration: testDuration,\n\t\t\t\t\tproducerMsgSize: 1024,\n\t\t\t\t\tproducerMsgs: 100_000,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t\tfor prefix, st := range map[string]nats.StorageType{\"R3F\": nats.FileStorage, \"R3M\": nats.MemoryStorage} {\n\t\t\tt.Run(prefix, func(t *testing.T) {\n\t\t\t\tfor rolloutType, params := range map[string]*testParams{\n\t\t\t\t\t// Rollouts using graceful restarts and checking healthz.\n\t\t\t\t\t\"ldm\": {\n\t\t\t\t\t\trestarts: 1,\n\t\t\t\t\t\trolloutRestart: true,\n\t\t\t\t\t\tldmRestart: true,\n\t\t\t\t\t\tcheckHealthz: true,\n\t\t\t\t\t\trestartWait: 45 * time.Second,\n\t\t\t\t\t},\n\t\t\t\t\t// Non graceful restarts calling Shutdown, but using healthz on startup.\n\t\t\t\t\t\"term\": {\n\t\t\t\t\t\trestarts: 1,\n\t\t\t\t\t\trolloutRestart: true,\n\t\t\t\t\t\tldmRestart: false,\n\t\t\t\t\t\tcheckHealthz: true,\n\t\t\t\t\t\trestartWait: 45 * time.Second,\n\t\t\t\t\t},\n\t\t\t\t} {\n\t\t\t\t\tt.Run(rolloutType, func(t *testing.T) {\n\t\t\t\t\t\tt.Run(\"limits\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t\tt.Run(\"wq\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t\tt.Run(\"interest\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t\tt.Run(\"limits:dn:max-per-subject\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t\tMaxMsgsPerSubject: 1,\n\t\t\t\t\t\t\tDiscard: nats.DiscardNew,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t\tt.Run(\"wq:dn:max-msgs\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t\tMaxMsgs: 10_000,\n\t\t\t\t\t\t\tDiscard: nats.DiscardNew,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t\tt.Run(\"wq:dn-per-subject:max-msgs\", shared(t, &nats.StreamConfig{\n\t\t\t\t\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\t\t\t\t\tStorage: st,\n\t\t\t\t\t\t\tMaxMsgs: 10_000,\n\t\t\t\t\t\t\tMaxMsgsPerSubject: 100,\n\t\t\t\t\t\t\tDiscard: nats.DiscardNew,\n\t\t\t\t\t\t\tDiscardNewPerSubject: true,\n\t\t\t\t\t\t}, params))\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/5488\nfunc TestJetStreamClusterSingleMaxConsumerUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tMaxConsumers: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"test_consumer\",\n\t\tMaxAckPending: 1000,\n\t})\n\trequire_NoError(t, err)\n\n\t// This would previously return a \"nats: maximum consumers limit\n\t// reached\" (10026) error.\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"test_consumer\",\n\t\tMaxAckPending: 1001,\n\t})\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterStreamLastSequenceResetAfterStorageWipe(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// After bug was found, number of streams and wiping store directory really did not affect.\n\tnumStreams := 50\n\tvar wg sync.WaitGroup\n\twg.Add(numStreams)\n\n\tfor i := 1; i <= numStreams; i++ {\n\t\tgo func(n int) {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"TEST:%d\", n),\n\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d.*\", n)},\n\t\t\t\tReplicas: 3,\n\t\t\t}, nats.MaxWait(30*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", n)\n\t\t\tfor i := 0; i < 222; i++ {\n\t\t\t\tjs.Publish(subj, nil)\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\tfor i := 0; i < 5; i++ {\n\t\t// Walk the servers and shut each down, and wipe the storage directory.\n\t\tfor _, s := range c.servers {\n\t\t\tsd := s.JetStreamConfig().StoreDir\n\t\t\ts.Shutdown()\n\t\t\ts.WaitForShutdown()\n\t\t\tos.RemoveAll(sd)\n\t\t\ts = c.restartServer(s)\n\t\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\ths := s.healthz(nil)\n\t\t\t\tif hs.Error != _EMPTY_ {\n\t\t\t\t\treturn errors.New(hs.Error)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\n\t\tfor _, s := range c.servers {\n\t\t\tfor i := 1; i <= numStreams; i++ {\n\t\t\t\tstream := fmt.Sprintf(\"TEST:%d\", i)\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(stream)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tvar state StreamState\n\t\t\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\t\tmset.store.FastState(&state)\n\t\t\t\t\tif state.LastSeq != 222 {\n\t\t\t\t\t\treturn fmt.Errorf(\"%v Wrong last sequence %d for %q - State %+v\", s, state.LastSeq, stream, state)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterAckFloorBetweenLeaderAndFollowers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo.*\", \"consumer\")\n\trequire_NoError(t, err)\n\n\t// Move consumer leader to be on the same server as the stream leader.\n\t// Without this fetching messages right after getting a response to js.Publish could mean\n\t// some messages are not available to the consumer yet.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"consumer\")\n\tif cl.Name() != sl.Name() {\n\t\treq := JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: sl.Name()}}\n\t\tdata, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"TEST\", \"consumer\"), data, time.Second)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"consumer\")\n\t\tcl = c.consumerLeader(globalAccountName, \"TEST\", \"consumer\")\n\t\trequire_Equal(t, cl.Name(), sl.Name())\n\t}\n\n\t// Do 25 rounds.\n\tfor i := 1; i <= 25; i++ {\n\t\t// Send 50 msgs.\n\t\tfor x := 0; x < 50; x++ {\n\t\t\t_, err := js.Publish(\"foo.bar\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err := sub.Fetch(50)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 50)\n\t\t// Randomize\n\t\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\t\tfor _, m := range msgs {\n\t\t\trequire_NoError(t, m.AckSync())\n\t\t}\n\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tconsumer := mset.lookupConsumer(\"consumer\")\n\t\t\trequire_NotEqual(t, consumer, nil)\n\t\t\tinfo := consumer.info()\n\t\t\trequire_Equal(t, info.NumAckPending, 0)\n\t\t\trequire_Equal(t, info.AckFloor.Consumer, uint64(i*50))\n\t\t\trequire_Equal(t, info.AckFloor.Stream, uint64(i*50))\n\t\t}\n\t}\n}\n\n// https://github.com/nats-io/nats-server/pull/5600\nfunc TestJetStreamClusterConsumerLeak(t *testing.T) {\n\tN := 2000 // runs in under 10s, but significant enough to see the difference.\n\tNConcurrent := 100\n\n\tclusterConf := `\n\tlisten: 127.0.0.1:-1\n\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleafnodes {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\tONE { users = [ { user: \"one\", pass: \"p\" } ]; jetstream: enabled }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\n\tcl := createJetStreamClusterWithTemplate(t, clusterConf, \"Leak-test\", 3)\n\tdefer cl.shutdown()\n\tcl.waitOnLeader()\n\n\ts := cl.randomNonLeader()\n\n\t// Create the test stream.\n\tstreamName := \"LEAK_TEST_STREAM\"\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"one\", \"p\"))\n\tdefer nc.Close()\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: streamName,\n\t\tSubjects: []string{\"$SOMETHING.>\"},\n\t\tStorage: nats.FileStorage,\n\t\tRetention: nats.InterestPolicy,\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tconcurrent := make(chan struct{}, NConcurrent)\n\tfor i := 0; i < NConcurrent; i++ {\n\t\tconcurrent <- struct{}{}\n\t}\n\terrors := make(chan error, N)\n\n\twg := sync.WaitGroup{}\n\twg.Add(N)\n\n\t// Gather the stats for comparison.\n\tbefore := &runtime.MemStats{}\n\truntime.GC()\n\truntime.ReadMemStats(before)\n\n\tfor i := 0; i < N; {\n\t\t// wait for a slot to open up\n\t\t<-concurrent\n\t\ti++\n\t\tgo func() {\n\t\t\tdefer func() {\n\t\t\t\tconcurrent <- struct{}{}\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"one\", \"p\"))\n\t\t\tdefer nc.Close()\n\n\t\t\tconsumerName := \"sessid_\" + nuid.Next()\n\t\t\t_, err := js.AddConsumer(streamName, &nats.ConsumerConfig{\n\t\t\t\tDeliverSubject: \"inbox\",\n\t\t\t\tDurable: consumerName,\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\t\t\tFilterSubject: \"$SOMETHING.ELSE.subject\",\n\t\t\t\tAckWait: 30 * time.Second,\n\t\t\t\tMaxAckPending: 1024,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\terrors <- fmt.Errorf(\"Error on JetStream consumer creation: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tcl.waitOnAllCurrent()\n\n\t\t\terr = js.DeleteConsumer(streamName, consumerName)\n\t\t\tif err != nil {\n\t\t\t\terrors <- fmt.Errorf(\"Error on JetStream consumer deletion: %v\", err)\n\t\t\t}\n\t\t}()\n\t}\n\n\twg.Wait()\n\tif len(errors) > 0 {\n\t\tfor err := range errors {\n\t\t\tt.Fatalf(\"%v\", err)\n\t\t}\n\t}\n\n\tafter := &runtime.MemStats{}\n\truntime.GC()\n\truntime.ReadMemStats(after)\n\n\t// Before https://github.com/nats-io/nats-server/pull/5600 this test was\n\t// adding 180Mb+ to HeapInuse. Now it's under 40Mb (ran locally on a Mac)\n\tlimit := before.HeapInuse + 100*1024*1024 // 100MB\n\tif after.HeapInuse > before.HeapInuse+limit {\n\t\tt.Fatalf(\"Extra memory usage too high: %v\", after.HeapInuse-before.HeapInuse)\n\t}\n}\n\nfunc TestJetStreamClusterAccountNRG(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tleader := c.streamLeader(globalAccountName, \"TEST\")\n\tstream, err := leader.gacc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trg := stream.node.(*raft)\n\n\tt.Run(\"Disabled\", func(t *testing.T) {\n\t\t// Switch off account NRG on all servers in the cluster.\n\t\tfor _, s := range c.servers {\n\t\t\ts.accountNRGAllowed.Store(false)\n\t\t\ts.sendStatszUpdate()\n\t\t}\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tfor _, s := range c.servers {\n\t\t\ts.GlobalAccount().nrgAccount = \"\"\n\t\t\ts.updateNRGAccountStatus()\n\t\t}\n\n\t\t// Check account interest for the AppendEntry subject.\n\t\tcheckFor(t, time.Second, time.Millisecond*25, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif !s.sys.account.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"system account should have interest\")\n\t\t\t\t}\n\t\t\t\tif s.gacc.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"global account shouldn't have interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Check that the Raft traffic is in the system account, as we\n\t\t// haven't moved it elsewhere yet.\n\t\t{\n\t\t\tsub, err := snc.SubscribeSync(rg.asubj)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, sub.AutoUnsubscribe(1))\n\n\t\t\tmsg, err := sub.NextMsg(time.Second * 3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, msg != nil)\n\t\t}\n\t})\n\n\tt.Run(\"Mixed\", func(t *testing.T) {\n\t\t// Switch on account NRG on a single server in the cluster and\n\t\t// leave it off on the rest.\n\t\tfor i, s := range c.servers {\n\t\t\ts.accountNRGAllowed.Store(i == 0)\n\t\t\ts.sendStatszUpdate()\n\t\t}\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tfor i, s := range c.servers {\n\t\t\tif i == 0 {\n\t\t\t\ts.GlobalAccount().nrgAccount = globalAccountName\n\t\t\t} else {\n\t\t\t\ts.GlobalAccount().nrgAccount = \"\"\n\t\t\t}\n\t\t\ts.updateNRGAccountStatus()\n\t\t}\n\n\t\t// Check account interest for the AppendEntry subject.\n\t\tcheckFor(t, time.Second, time.Millisecond*25, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif !s.sys.account.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"system account should have interest\")\n\t\t\t\t}\n\t\t\t\tif s.gacc.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"global account shouldn't have interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Check that the Raft traffic is in the system account, as we\n\t\t// don't claim support for account NRG on all nodes in the group.\n\t\t{\n\t\t\tsub, err := snc.SubscribeSync(rg.asubj)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, sub.AutoUnsubscribe(1))\n\n\t\t\tmsg, err := sub.NextMsg(time.Second * 3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, msg != nil)\n\t\t}\n\t})\n\n\tt.Run(\"Enabled\", func(t *testing.T) {\n\t\t// Switch on account NRG on all servers in the cluster.\n\t\tfor _, s := range c.servers {\n\t\t\ts.accountNRGAllowed.Store(true)\n\t\t\ts.sendStatszUpdate()\n\t\t}\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tfor _, s := range c.servers {\n\t\t\ts.GlobalAccount().nrgAccount = globalAccountName\n\t\t\ts.updateNRGAccountStatus()\n\t\t}\n\n\t\t// Check account interest for the AppendEntry subject.\n\t\tcheckFor(t, time.Second, time.Millisecond*25, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s.sys.account.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"system account shouldn't have interest\")\n\t\t\t\t}\n\t\t\t\tif !s.gacc.sl.HasInterest(rg.asubj) {\n\t\t\t\t\treturn fmt.Errorf(\"global account should have interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Check that the traffic moved into the global account as\n\t\t// expected.\n\t\t{\n\t\t\tsub, err := nc.SubscribeSync(rg.asubj)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, sub.AutoUnsubscribe(1))\n\n\t\t\tmsg, err := sub.NextMsg(time.Second * 3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, msg != nil)\n\t\t}\n\t})\n}\n\nfunc TestJetStreamClusterAccountNRGConfigNoPanic(t *testing.T) {\n\tclusterConf := `\n\t\tlisten: 127.0.0.1:-1\n\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts {\n\t\t\tONE { jetstream: { cluster_traffic: system } }\n\t\t\tTWO { jetstream: { cluster_traffic: owner } }\n\t\t}\n\t`\n\n\tcl := createJetStreamClusterWithTemplate(t, clusterConf, \"test\", 3)\n\tdefer cl.shutdown()\n\n\tfor _, s := range cl.servers {\n\t\tacc, err := s.lookupAccount(\"ONE\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, acc.nrgAccount, _EMPTY_) // Empty for the system account\n\n\t\tacc, err = s.lookupAccount(\"TWO\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, acc.nrgAccount, \"TWO\")\n\t}\n}\n\nfunc TestJetStreamClusterWQRoundRobinSubjectRetention(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"wq_stream\",\n\t\tSubjects: []string{\"something.>\"},\n\t\tStorage: nats.FileStorage,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tn := (i % 5) + 1\n\t\t_, err := js.Publish(fmt.Sprintf(\"something.%d\", n), nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\n\t\t\"something.5\",\n\t\t\"wq_consumer_5\",\n\t\tnats.BindStream(\"wq_stream\"),\n\t\tnats.ConsumerReplicas(3),\n\t)\n\trequire_NoError(t, err)\n\n\tfor {\n\t\tmsgs, _ := sub.Fetch(5)\n\t\tif len(msgs) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfor _, msg := range msgs {\n\t\t\trequire_NoError(t, msg.AckSync())\n\t\t}\n\t}\n\n\tsi, err := js.StreamInfo(\"wq_stream\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 80)\n\trequire_Equal(t, si.State.NumDeleted, 20)\n\trequire_Equal(t, si.State.NumSubjects, 4)\n}\n\nfunc TestJetStreamClusterMetaSyncOrphanCleanup(t *testing.T) {\n\tc := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, \"R3S\", 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [server:%s]\", conf, serverName)\n\t\t})\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Create a bunch of streams on S1\n\tfor i := 0; i < 100; i++ {\n\t\tstream := fmt.Sprintf(\"TEST-%d\", i)\n\t\tsubject := fmt.Sprintf(\"TEST.%d\", i)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: stream,\n\t\t\tSubjects: []string{subject},\n\t\t\tStorage: nats.FileStorage,\n\t\t\tPlacement: &nats.Placement{Tags: []string{\"server:S-1\"}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t// Put in 10 msgs to each\n\t\tfor j := 0; j < 10; j++ {\n\t\t\t_, err := js.Publish(subject, nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\t// Now we will shutdown S1 and remove all of its meta-data to trip the condition.\n\ts := c.serverByName(\"S-1\")\n\trequire_True(t, s != nil)\n\n\tsd := s.JetStreamConfig().StoreDir\n\tnd := filepath.Join(sd, \"$SYS\", \"_js_\", \"_meta_\")\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\tos.RemoveAll(nd)\n\ts = c.restartServer(s)\n\tc.waitOnServerCurrent(s)\n\tjsz, err := s.Jsz(nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, jsz.Streams, 100)\n\n\t// These will be recreated by the meta layer, but if the orphan detection deleted them they will be empty,\n\t// so check all streams to make sure they still have data.\n\tacc := s.GlobalAccount()\n\tvar state StreamState\n\tfor i := 0; i < 100; i++ {\n\t\tmset, err := acc.lookupStream(fmt.Sprintf(\"TEST-%d\", i))\n\t\trequire_NoError(t, err)\n\t\tmset.store.FastState(&state)\n\t\trequire_Equal(t, state.Msgs, 10)\n\t}\n}\n\nfunc TestJetStreamClusterKeyValueDesyncAfterHardKill(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.serverByName(\"S-1\"))\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"inconsistency\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// First create should succeed.\n\trevision, err := kv.Create(\"key.exists\", []byte(\"1\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, revision, 1)\n\n\t// Second create will be rejected but bump CLFS.\n\t_, err = kv.Create(\"key.exists\", []byte(\"2\"))\n\trequire_Error(t, err)\n\n\t// Insert a new message, should only be applied once, even if we hard kill and replay afterward.\n\trevision, err = kv.Put(\"key.put\", []byte(\"3\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, revision, 2)\n\n\t// Restart a server\n\ts3 := c.serverByName(\"S-3\")\n\t// We will remove the index.db file after we shutdown.\n\tmset, err := s3.GlobalAccount().lookupStream(\"KV_inconsistency\")\n\trequire_NoError(t, err)\n\tfs := mset.store.(*fileStore)\n\tifile := filepath.Join(fs.fcfg.StoreDir, msgDir, \"index.db\")\n\n\ts3.Shutdown()\n\ts3.WaitForShutdown()\n\t// Remove the index.db file to simulate a hard kill where server can not write out the index.db file.\n\trequire_NoError(t, os.Remove(ifile))\n\n\tc.restartServer(s3)\n\tc.waitOnClusterReady()\n\tc.waitOnAllCurrent()\n\n\terr = checkState(t, c, \"$G\", \"KV_inconsistency\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterKeyValueLastSeqMismatch(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor _, r := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R=%d\", r), func(t *testing.T) {\n\t\t\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\t\t\tBucket: fmt.Sprintf(\"mismatch_%v\", r),\n\t\t\t\tReplicas: r,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\trevision, err := kv.Create(\"foo\", []byte(\"1\"))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, revision, 1)\n\n\t\t\trevision, err = kv.Create(\"bar\", []byte(\"2\"))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, revision, 2)\n\n\t\t\t// Now say we want to update baz but iff last was revision 1.\n\t\t\t_, err = kv.Update(\"baz\", []byte(\"3\"), uint64(1))\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Equal(t, err.Error(), `nats: wrong last sequence: 0`)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterPubAckSequenceDupe(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"TEST_CLUSTER\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\ttype client struct {\n\t\tnc *nats.Conn\n\t\tjs nats.JetStreamContext\n\t}\n\n\tclients := make([]client, len(c.servers))\n\tfor i, server := range c.servers {\n\t\tclients[i].nc, clients[i].js = jsClientConnect(t, server)\n\t\tdefer clients[i].nc.Close()\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST_STREAM\",\n\t\tSubjects: []string{\"TEST_SUBJECT.*\"},\n\t\tReplicas: 3,\n\t\tDuplicates: 1 * time.Minute,\n\t})\n\trequire_NoError(t, err)\n\n\tmsgData := []byte(\"...\")\n\n\tfor seq := uint64(1); seq < 10; seq++ {\n\n\t\tif seq%3 == 0 {\n\t\t\tc.restartAll()\n\t\t}\n\n\t\tmsgSubject := \"TEST_SUBJECT.\" + strconv.FormatUint(seq, 10)\n\t\tmsgIdOpt := nats.MsgId(nuid.Next())\n\n\t\tfirstPublisherClient := &clients[rand.Intn(len(clients))]\n\t\tsecondPublisherClient := &clients[rand.Intn(len(clients))]\n\n\t\tpubAck1, err := firstPublisherClient.js.Publish(msgSubject, msgData, msgIdOpt)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, seq, pubAck1.Sequence)\n\t\trequire_False(t, pubAck1.Duplicate)\n\n\t\tpubAck2, err := secondPublisherClient.js.Publish(msgSubject, msgData, msgIdOpt)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, seq, pubAck2.Sequence)\n\t\trequire_True(t, pubAck2.Duplicate)\n\n\t}\n\n}\n\nfunc TestJetStreamClusterPubAckSequenceDupeAsync(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"TEST_CLUSTER\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST_STREAM\",\n\t\tSubjects: []string{\"TEST_SUBJECT\"},\n\t\tReplicas: 3,\n\t\tDuplicates: 1 * time.Minute,\n\t})\n\trequire_NoError(t, err)\n\n\tmsgData := []byte(\"...\")\n\n\tfor seq := uint64(1); seq < 10; seq++ {\n\n\t\tmsgSubject := \"TEST_SUBJECT\"\n\t\tmsgIdOpt := nats.MsgId(nuid.Next())\n\t\tconflictErr := &nats.APIError{ErrorCode: nats.ErrorCode(JSStreamDuplicateMessageConflict)}\n\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(2)\n\n\t\t// Fire off 2 publish requests in parallel\n\t\t// The first one \"stages\" a duplicate entry before even proposing the message\n\t\t// The second one gets a pubAck with sequence zero by hitting the staged duplicated entry\n\n\t\tpubAcks := [2]*nats.PubAck{}\n\t\tfor i := 0; i < 2; i++ {\n\t\t\tgo func(i int) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tvar err error\n\t\t\t\tpubAcks[i], err = js.Publish(msgSubject, msgData, msgIdOpt)\n\t\t\t\t// Conflict on duplicate message, wait a bit before retrying to get the proper pubAck.\n\t\t\t\tif errors.Is(err, conflictErr) {\n\t\t\t\t\ttime.Sleep(time.Millisecond * 500)\n\t\t\t\t\tpubAcks[i], err = js.Publish(msgSubject, msgData, msgIdOpt)\n\t\t\t\t}\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}(i)\n\t\t}\n\n\t\twg.Wait()\n\t\trequire_Equal(t, pubAcks[0].Sequence, seq)\n\t\trequire_Equal(t, pubAcks[1].Sequence, seq)\n\n\t\t// Exactly one of the pubAck should be marked dupe\n\t\trequire_True(t, (pubAcks[0].Duplicate || pubAcks[1].Duplicate) && (pubAcks[0].Duplicate != pubAcks[1].Duplicate))\n\t}\n}\n\nfunc TestJetStreamClusterPubAckSequenceDupeResetAfterLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tDuplicates: 2 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Store one msg ID that needs to be preserved, and another that should be removed during leader change.\n\tmset.ddMu.Lock()\n\tmset.storeMsgIdLocked(&ddentry{\"genuine\", 1, time.Now().UnixNano()})\n\tmset.storeMsgIdLocked(&ddentry{\"msgId\", 0, time.Now().UnixNano()})\n\tmset.ddMu.Unlock()\n\n\t// Simulates the msg ID being in process.\n\t_, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_Error(t, err, NewJSStreamDuplicateMessageConflictError())\n\n\t// Move stream leader to a different server.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\treq := JSApiLeaderStepdownRequest{Placement: &Placement{Preferred: rs.Name()}}\n\tdata, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), data, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Move stream leader back.\n\treq.Placement.Preferred = sl.Name()\n\tdata, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), data, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnsl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_True(t, nsl == sl)\n\n\tmset.ddMu.Lock()\n\tlenDdmap, lenDdarr := len(mset.ddmap), len(mset.ddarr)\n\tmset.ddMu.Unlock()\n\trequire_Len(t, lenDdmap, 1)\n\trequire_Len(t, lenDdarr, 1)\n\n\t// Now the publish should pass.\n\t_, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterConsumeWithStartSequence(t *testing.T) {\n\n\tconst (\n\t\tNumMessages = 10\n\t\tChosenSeq = 5\n\t\tStreamName = \"TEST\"\n\t\tStreamSubject = \"ORDERS.*\"\n\t\tStreamSubjectPrefix = \"ORDERS.\"\n\t)\n\n\tfor _, ClusterSize := range []int{\n\t\t1, // Single server\n\t\t3, // 3-node cluster\n\t} {\n\t\tR := ClusterSize\n\t\tt.Run(\n\t\t\tfmt.Sprintf(\"Nodes:%d,Replicas:%d\", ClusterSize, R),\n\t\t\tfunc(t *testing.T) {\n\t\t\t\t// This is the success condition for all sub-tests below\n\t\t\t\tvar ExpectedMsgId = \"\"\n\t\t\t\tcheckMessage := func(t *testing.T, msg *nats.Msg) {\n\t\t\t\t\tt.Helper()\n\n\t\t\t\t\tmsgMeta, err := msg.Metadata()\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\t// Check sequence number\n\t\t\t\t\trequire_Equal(t, msgMeta.Sequence.Stream, ChosenSeq)\n\n\t\t\t\t\t// Check message id\n\t\t\t\t\trequire_NotEqual(t, ExpectedMsgId, \"\")\n\t\t\t\t\trequire_Equal(t, msg.Header.Get(nats.MsgIdHdr), ExpectedMsgId)\n\t\t\t\t}\n\n\t\t\t\tcheckRawMessage := func(t *testing.T, msg *nats.RawStreamMsg) {\n\t\t\t\t\tt.Helper()\n\n\t\t\t\t\t// Check sequence number\n\t\t\t\t\trequire_Equal(t, msg.Sequence, ChosenSeq)\n\n\t\t\t\t\t// Check message id\n\t\t\t\t\trequire_NotEqual(t, ExpectedMsgId, \"\")\n\t\t\t\t\trequire_Equal(t, msg.Header.Get(nats.MsgIdHdr), ExpectedMsgId)\n\t\t\t\t}\n\n\t\t\t\t// Setup: start server or cluster, connect client\n\t\t\t\tvar server *Server\n\t\t\t\tif ClusterSize == 1 {\n\t\t\t\t\tserver = RunBasicJetStreamServer(t)\n\t\t\t\t\tdefer server.Shutdown()\n\t\t\t\t} else {\n\t\t\t\t\tc := createJetStreamCluster(t, jsClusterTempl, \"HUB\", _EMPTY_, ClusterSize, 22020, true)\n\t\t\t\t\tdefer c.shutdown()\n\t\t\t\t\tserver = c.randomServer()\n\t\t\t\t}\n\n\t\t\t\t// Setup: connect\n\t\t\t\tvar nc *nats.Conn\n\t\t\t\tvar js nats.JetStreamContext\n\t\t\t\tnc, js = jsClientConnect(t, server)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t// Setup: create stream\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tReplicas: R,\n\t\t\t\t\tName: StreamName,\n\t\t\t\t\tSubjects: []string{StreamSubject},\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Setup: populate stream\n\t\t\t\tbuf := make([]byte, 100)\n\t\t\t\tfor i := uint64(1); i <= NumMessages; i++ {\n\t\t\t\t\tmsgId := nuid.Next()\n\t\t\t\t\tpubAck, err := js.Publish(StreamSubjectPrefix+strconv.Itoa(int(i)), buf, nats.MsgId(msgId))\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\t// Verify assumption made in tests below\n\t\t\t\t\trequire_Equal(t, pubAck.Sequence, i)\n\n\t\t\t\t\tif i == ChosenSeq {\n\t\t\t\t\t\t// Save the expected message id for the chosen message\n\t\t\t\t\t\tExpectedMsgId = msgId\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Setup: create subscriptions, needs to be after stream creation or OptStartSeq could be clipped\n\t\t\t\tvar preCreatedSub, preCreatedSubDurable *nats.Subscription\n\t\t\t\t{\n\t\t\t\t\tpreCreatedSub, err = js.PullSubscribe(\n\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\"\",\n\t\t\t\t\t\tnats.StartSequence(ChosenSeq),\n\t\t\t\t\t)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer func() {\n\t\t\t\t\t\trequire_NoError(t, preCreatedSub.Unsubscribe())\n\t\t\t\t\t}()\n\n\t\t\t\t\tconst Durable = \"dlc_pre_created\"\n\t\t\t\t\tc, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{\n\t\t\t\t\t\tDurable: Durable,\n\t\t\t\t\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\t\t\t\t\tOptStartSeq: ChosenSeq,\n\t\t\t\t\t\tReplicas: R,\n\t\t\t\t\t})\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer func() {\n\t\t\t\t\t\trequire_NoError(t, js.DeleteConsumer(c.Stream, c.Name))\n\t\t\t\t\t}()\n\n\t\t\t\t\tpreCreatedSubDurable, err = js.PullSubscribe(\n\t\t\t\t\t\t\"\",\n\t\t\t\t\t\t\"\",\n\t\t\t\t\t\tnats.Bind(StreamName, Durable),\n\t\t\t\t\t)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer func() {\n\t\t\t\t\t\trequire_NoError(t, preCreatedSubDurable.Unsubscribe())\n\t\t\t\t\t}()\n\t\t\t\t}\n\n\t\t\t\t// Tests various ways to consume the stream starting at the ChosenSeq sequence\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"DurableConsumer\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tconst Durable = \"dlc\"\n\t\t\t\t\t\tc, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{\n\t\t\t\t\t\t\tDurable: Durable,\n\t\t\t\t\t\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\t\t\t\t\t\tOptStartSeq: ChosenSeq,\n\t\t\t\t\t\t\tReplicas: R,\n\t\t\t\t\t\t})\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, js.DeleteConsumer(c.Stream, c.Name))\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tsub, err := js.PullSubscribe(\n\t\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\tDurable,\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t\tcheckMessage(t, msgs[0])\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"DurableConsumerWithBind\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tconst Durable = \"dlc_bind\"\n\t\t\t\t\t\tc, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{\n\t\t\t\t\t\t\tDurable: Durable,\n\t\t\t\t\t\t\tDeliverPolicy: nats.DeliverByStartSequencePolicy,\n\t\t\t\t\t\t\tOptStartSeq: ChosenSeq,\n\t\t\t\t\t\t\tReplicas: R,\n\t\t\t\t\t\t})\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, js.DeleteConsumer(c.Stream, c.Name))\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tsub, err := js.PullSubscribe(\n\t\t\t\t\t\t\t\"\",\n\t\t\t\t\t\t\t\"\",\n\t\t\t\t\t\t\tnats.Bind(StreamName, Durable),\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t\tcheckMessage(t, msgs[0])\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"PreCreatedDurableConsumerWithBind\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tmsgs, err := preCreatedSubDurable.Fetch(1)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t\tcheckMessage(t, msgs[0])\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"PullConsumer\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tsub, err := js.PullSubscribe(\n\t\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\t\"\",\n\t\t\t\t\t\t\tnats.StartSequence(ChosenSeq),\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t\tcheckMessage(t, msgs[0])\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"PreCreatedPullConsumer\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tmsgs, err := preCreatedSub.Fetch(1)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t\tcheckMessage(t, msgs[0])\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"SynchronousConsumer\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tsub, err := js.SubscribeSync(\n\t\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\tnats.StartSequence(ChosenSeq),\n\t\t\t\t\t\t)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tmsg, err := sub.NextMsg(1 * time.Second)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tcheckMessage(t, msg)\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"CallbackSubscribe\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tvar waitGroup sync.WaitGroup\n\t\t\t\t\t\twaitGroup.Add(1)\n\t\t\t\t\t\t// To be populated by callback\n\t\t\t\t\t\tvar receivedMsg *nats.Msg\n\n\t\t\t\t\t\tsub, err := js.Subscribe(\n\t\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\tfunc(msg *nats.Msg) {\n\t\t\t\t\t\t\t\t// Save first message received\n\t\t\t\t\t\t\t\tif receivedMsg == nil {\n\t\t\t\t\t\t\t\t\treceivedMsg = msg\n\t\t\t\t\t\t\t\t\twaitGroup.Done()\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tnats.StartSequence(ChosenSeq),\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\twaitGroup.Wait()\n\t\t\t\t\t\trequire_NotNil(t, receivedMsg)\n\t\t\t\t\t\tcheckMessage(t, receivedMsg)\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"ChannelSubscribe\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\tmsgChannel := make(chan *nats.Msg, 1)\n\t\t\t\t\t\tsub, err := js.ChanSubscribe(\n\t\t\t\t\t\t\tStreamSubject,\n\t\t\t\t\t\t\tmsgChannel,\n\t\t\t\t\t\t\tnats.StartSequence(ChosenSeq),\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t\t\t}()\n\n\t\t\t\t\t\tmsg := <-msgChannel\n\t\t\t\t\t\tcheckMessage(t, msg)\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"GetRawStreamMessage\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\trawMsg, err := js.GetMsg(StreamName, ChosenSeq)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tcheckRawMessage(t, rawMsg)\n\t\t\t\t\t},\n\t\t\t\t)\n\n\t\t\t\tt.Run(\n\t\t\t\t\t\"GetLastMessageBySubject\",\n\t\t\t\t\tfunc(t *testing.T) {\n\t\t\t\t\t\trawMsg, err := js.GetLastMsg(\n\t\t\t\t\t\t\tStreamName,\n\t\t\t\t\t\t\tfmt.Sprintf(\"ORDERS.%d\", ChosenSeq),\n\t\t\t\t\t\t)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tcheckRawMessage(t, rawMsg)\n\t\t\t\t\t},\n\t\t\t\t)\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc TestJetStreamClusterAckDeleted(t *testing.T) {\n\n\tconst (\n\t\tNumMessages = 10\n\t\tStreamName = \"TEST\"\n\t\tStreamSubject = \"ORDERS.*\"\n\t\tStreamSubjectPrefix = \"ORDERS.\"\n\t)\n\n\tfor _, ClusterSize := range []int{\n\t\t1, // Single server\n\t\t3, // 3-node cluster\n\t} {\n\t\tR := ClusterSize\n\t\tt.Run(\n\t\t\tfmt.Sprintf(\"Nodes:%d,Replicas:%d\", ClusterSize, R),\n\t\t\tfunc(t *testing.T) {\n\t\t\t\t// Setup: start server or cluster, connect client\n\t\t\t\tvar server *Server\n\t\t\t\tif ClusterSize == 1 {\n\t\t\t\t\tserver = RunBasicJetStreamServer(t)\n\t\t\t\t\tdefer server.Shutdown()\n\t\t\t\t} else {\n\t\t\t\t\tc := createJetStreamCluster(t, jsClusterTempl, \"HUB\", _EMPTY_, ClusterSize, 22020, true)\n\t\t\t\t\tdefer c.shutdown()\n\t\t\t\t\tserver = c.randomServer()\n\t\t\t\t}\n\n\t\t\t\t// Setup: connect\n\t\t\t\tvar nc *nats.Conn\n\t\t\t\tvar js nats.JetStreamContext\n\t\t\t\tnc, js = jsClientConnect(t, server)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t// Setup: create stream\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tReplicas: R,\n\t\t\t\t\tName: StreamName,\n\t\t\t\t\tSubjects: []string{StreamSubject},\n\t\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\t\tDiscard: nats.DiscardOld,\n\t\t\t\t\tMaxMsgs: 1, // Only keep the latest message\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Setup: create durable consumer and subscription\n\t\t\t\tconst Durable = \"dlc\"\n\t\t\t\tc, err := js.AddConsumer(StreamName, &nats.ConsumerConfig{\n\t\t\t\t\tDurable: Durable,\n\t\t\t\t\tReplicas: R,\n\t\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\t\tMaxAckPending: NumMessages,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer func() {\n\t\t\t\t\trequire_NoError(t, js.DeleteConsumer(c.Stream, c.Name))\n\t\t\t\t}()\n\n\t\t\t\t// Setup: create durable consumer subscription\n\t\t\t\tsub, err := js.PullSubscribe(\n\t\t\t\t\t\"\",\n\t\t\t\t\t\"\",\n\t\t\t\t\tnats.Bind(StreamName, Durable),\n\t\t\t\t)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer func() {\n\t\t\t\t\trequire_NoError(t, sub.Unsubscribe())\n\t\t\t\t}()\n\n\t\t\t\t// Collect received and non-ACKed messages\n\t\t\t\treceivedMessages := make([]*nats.Msg, 0, NumMessages)\n\n\t\t\t\tbuf := make([]byte, 100)\n\t\t\t\tfor i := uint64(1); i <= NumMessages; i++ {\n\t\t\t\t\t// Publish one message\n\t\t\t\t\tmsgId := nuid.Next()\n\t\t\t\t\tpubAck, err := js.Publish(\n\t\t\t\t\t\tStreamSubjectPrefix+strconv.Itoa(int(i)),\n\t\t\t\t\t\tbuf,\n\t\t\t\t\t\tnats.MsgId(msgId),\n\t\t\t\t\t)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Equal(t, pubAck.Sequence, i)\n\n\t\t\t\t\t// Consume message\n\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Equal(t, len(msgs), 1)\n\n\t\t\t\t\t// Validate message\n\t\t\t\t\tmsg := msgs[0]\n\t\t\t\t\trequire_Equal(t, msgs[0].Header.Get(nats.MsgIdHdr), msgId)\n\n\t\t\t\t\t// Validate message metadata\n\t\t\t\t\tmsgMeta, err := msg.Metadata()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t// Check sequence number\n\t\t\t\t\trequire_Equal(t, msgMeta.Sequence.Stream, i)\n\n\t\t\t\t\t// Save for ACK later\n\t\t\t\t\treceivedMessages = append(receivedMessages, msg)\n\t\t\t\t}\n\n\t\t\t\t// Verify stream state, expecting a single message due to limits\n\t\t\t\tstreamInfo, err := js.StreamInfo(StreamName)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, streamInfo.State.Msgs, 1)\n\n\t\t\t\t// Verify consumer state, expecting ack floor corresponding to messages dropped\n\t\t\t\tconsumerInfo, err := js.ConsumerInfo(StreamName, Durable)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, consumerInfo.NumAckPending, 1)\n\t\t\t\trequire_Equal(t, consumerInfo.AckFloor.Stream, 9)\n\t\t\t\trequire_Equal(t, consumerInfo.AckFloor.Consumer, 9)\n\n\t\t\t\t// ACK all messages (all except last have been dropped from the stream)\n\t\t\t\tfor _, message := range receivedMessages {\n\t\t\t\t\terr := message.AckSync()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\n\t\t\t\t// Verify consumer state, all messages ACKed\n\t\t\t\tconsumerInfo, err = js.ConsumerInfo(StreamName, Durable)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, consumerInfo.NumAckPending, 0)\n\t\t\t\trequire_Equal(t, consumerInfo.AckFloor.Stream, 10)\n\t\t\t\trequire_Equal(t, consumerInfo.AckFloor.Consumer, 10)\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc TestJetStreamClusterAPILimitDefault(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tfor _, s := range c.servers {\n\t\ts.optsMu.RLock()\n\t\tlim := s.opts.JetStreamRequestQueueLimit\n\t\tilim := s.opts.JetStreamInfoQueueLimit\n\t\ts.optsMu.RUnlock()\n\n\t\trequire_Equal(t, lim, JSDefaultRequestQueueLimit)\n\t\trequire_Equal(t, ilim, JSDefaultRequestQueueLimit)\n\t\trequire_Equal(t, atomic.LoadInt64(&s.getJetStream().queueLimit), JSDefaultRequestQueueLimit)\n\t\trequire_Equal(t, atomic.LoadInt64(&s.getJetStream().infoQueueLimit), JSDefaultRequestQueueLimit)\n\t}\n}\n\nfunc TestJetStreamClusterAPILimitAdvisory(t *testing.T) {\n\t// Hit the limit straight away.\n\tconst queueLimit = 1\n\n\tconfig := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {\n\t\t\tmax_mem_store: 256MB\n\t\t\tmax_file_store: 2GB\n\t\t\tstore_dir: '%s'\n\t\t\trequest_queue_limit: ` + fmt.Sprintf(\"%d\", queueLimit) + `\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n `\n\tc := createJetStreamClusterWithTemplate(t, config, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\ts := c.randomNonLeader()\n\n\tfor _, s := range c.servers {\n\t\tlim := atomic.LoadInt64(&s.getJetStream().queueLimit)\n\t\trequire_Equal(t, lim, queueLimit)\n\t}\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\tsub, err := snc.SubscribeSync(JSAdvisoryAPILimitReached)\n\trequire_NoError(t, err)\n\n\t// There's a very slim chance that a worker could pick up a request between\n\t// pushing to and draining the queue, so make sure we've sent enough of them\n\t// to reliably trigger a drain and advisory.\n\tinbox := nc.NewRespInbox()\n\ttotal := 100\n\tfor range total {\n\t\trequire_NoError(t, nc.PublishMsg(&nats.Msg{\n\t\t\tSubject: fmt.Sprintf(JSApiConsumerListT, \"TEST\"),\n\t\t\tReply: inbox,\n\t\t}))\n\t}\n\n\tfor range total {\n\t\t// Wait for the advisory to come in.\n\t\tmsg, err := sub.NextMsg(time.Second * 5)\n\t\trequire_NoError(t, err)\n\t\tvar advisory JSAPILimitReachedAdvisory\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.Domain, _EMPTY_) // No JetStream domain was set.\n\t\tif advisory.Dropped >= 1 {\n\t\t\t// We are done!\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatal(\"Did not get any advisory with dropped > 0\")\n}\n\nfunc TestJetStreamClusterPendingRequestsInJsz(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\tmetaleader := c.leader()\n\n\tsjs := metaleader.getJetStream()\n\tsjs.mu.Lock()\n\tsub := &subscription{\n\t\tsubject: []byte(\"$JS.API.VERY_SLOW\"),\n\t\ticb: func(sub *subscription, client *client, acc *Account, subject, reply string, rmsg []byte) {\n\t\t\tselect {\n\t\t\tcase <-client.srv.quitCh:\n\t\t\tcase <-time.After(time.Second * 3):\n\t\t\t}\n\t\t},\n\t}\n\terr := metaleader.getJetStream().apiSubs.Insert(sub)\n\tsjs.mu.Unlock()\n\n\trequire_NoError(t, err)\n\n\tnc, _ := jsClientConnect(t, c.randomNonLeader())\n\tdefer nc.Close()\n\n\tinbox := nc.NewRespInbox()\n\tmsg := &nats.Msg{\n\t\tSubject: \"$JS.API.VERY_SLOW\",\n\t\tReply: inbox,\n\t}\n\n\t// Fall short of hitting the API limit by a little bit,\n\t// otherwise the requests get drained away.\n\tfor i := 0; i < JSDefaultRequestQueueLimit-10; i++ {\n\t\trequire_NoError(t, nc.PublishMsg(msg))\n\t}\n\n\t// We could check before above published messages are received,\n\t// so allow some retries for pending messages to build up.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tjsz, err := metaleader.Jsz(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif jsz.Meta == nil {\n\t\t\treturn errors.New(\"jsz.Meta == nil\")\n\t\t}\n\t\tif jsz.Meta.Pending == 0 {\n\t\t\treturn errors.New(\"jsz.Meta.Pending == 0, expected pending requests\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\tch := make(chan *nats.Msg, 1)\n\tssub, err := snc.ChanSubscribe(fmt.Sprintf(serverStatsSubj, metaleader.ID()), ch)\n\trequire_NoError(t, err)\n\trequire_NoError(t, ssub.AutoUnsubscribe(1))\n\n\tmsg = require_ChanRead(t, ch, time.Second*5)\n\tvar m ServerStatsMsg\n\trequire_NoError(t, json.Unmarshal(msg.Data, &m))\n\trequire_True(t, m.Stats.JetStream != nil)\n\trequire_NotEqual(t, m.Stats.JetStream.Meta.Pending, 0)\n}\n\nfunc TestJetStreamClusterConsumerReplicasAfterScale(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomNonLeader())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 5,\n\t})\n\trequire_NoError(t, err)\n\n\t// Put some messages in to test consumer state transfer.\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Create four different consumers.\n\t// Normal where we inherit replicas from parent.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Config.Replicas, 0)\n\trequire_Equal(t, len(ci.Cluster.Replicas), 4)\n\n\t// Ephemeral\n\tci, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Config.Replicas, 0) // Legacy ephemeral is 0 here too.\n\trequire_Equal(t, len(ci.Cluster.Replicas), 0)\n\teName := ci.Name\n\n\t// R1\n\tci, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"r1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Config.Replicas, 1)\n\trequire_Equal(t, len(ci.Cluster.Replicas), 0)\n\n\t// R3\n\tci, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"r3\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Config.Replicas, 3)\n\trequire_Equal(t, len(ci.Cluster.Replicas), 2)\n\n\t// Now create some state on r1 consumer.\n\tsub, err := js.PullSubscribe(\"foo\", \"r1\")\n\trequire_NoError(t, err)\n\n\tfetch := rand.Intn(99) + 1 // Needs to be at least 1.\n\tmsgs, err := sub.Fetch(fetch, nats.MaxWait(10*time.Second))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), fetch)\n\tack := rand.Intn(fetch)\n\tfor i := 0; i <= ack; i++ {\n\t\tmsgs[i].AckSync()\n\t}\n\tr1ci, err := js.ConsumerInfo(\"TEST\", \"r1\")\n\trequire_NoError(t, err)\n\tr1ci.Delivered.Last, r1ci.AckFloor.Last = nil, nil\n\n\t// Now scale stream to R3.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tcheckConsumerReplicas := func(t *testing.T, stream, consumer string, cor, clr int) {\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif ci, err = js.ConsumerInfo(stream, consumer); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif ci.Config.Replicas != cor {\n\t\t\t\treturn fmt.Errorf(\"config replicas %d != %d\", ci.Config.Replicas, cor)\n\t\t\t}\n\t\t\tif ci.Cluster == nil {\n\t\t\t\treturn fmt.Errorf(\"cluster nil\")\n\t\t\t}\n\t\t\tif len(ci.Cluster.Replicas) != clr {\n\t\t\t\treturn fmt.Errorf(\"cluster replica peers %d != %d\", len(ci.Cluster.Replicas), clr)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Now check each.\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"dur\")\n\tcheckConsumerReplicas(t, \"TEST\", \"dur\", 0, 2)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", eName)\n\tcheckConsumerReplicas(t, \"TEST\", eName, 0, 0)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"r1\")\n\tcheckConsumerReplicas(t, \"TEST\", \"r1\", 1, 0)\n\n\t// Now check that state transferred correctly.\n\tci.Delivered.Last, ci.AckFloor.Last = nil, nil\n\tif ci.Delivered != r1ci.Delivered {\n\t\tt.Fatalf(\"Delivered state for R1 incorrect, wanted %+v got %+v\",\n\t\t\tr1ci.Delivered, ci.Delivered)\n\t}\n\tif ci.AckFloor != r1ci.AckFloor {\n\t\tt.Fatalf(\"AckFloor state for R1 incorrect, wanted %+v got %+v\",\n\t\t\tr1ci.AckFloor, ci.AckFloor)\n\t}\n\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"r3\")\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"r1\")\n\tcheckConsumerReplicas(t, \"TEST\", \"r3\", 3, 2)\n}\n\nfunc TestJetStreamClusterConsumerReplicasAfterScaleMoveConsumer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"), nats.ConsumerReplicas(1))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\t// The consumer will have received anc acked one message.\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 1)\n\tci.AckFloor.Last = nil\n\tconsumerLeader := ci.Cluster.Leader\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster.Leader != consumerLeader {\n\t\t\treturn nil\n\t\t}\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn errors.New(\"expected leader to not be equal\")\n\t})\n\n\t// Scaling the stream down to 1 replica means the R1 consumer will need to move to the current stream leader.\n\tcfg.Replicas = 1\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Check that the consumer state wasn't lost.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tci2, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tci2.AckFloor.Last = nil\n\t\tif !reflect.DeepEqual(ci.AckFloor, ci2.AckFloor) {\n\t\t\treturn fmt.Errorf(\"%+v vs %+v\", ci.AckFloor, ci2.AckFloor)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterDesyncAfterQuitDuringCatchup(t *testing.T) {\n\tfor title, test := range map[string]func(s *Server, rn RaftNode){\n\t\t\"RAFT\": func(s *Server, rn RaftNode) {\n\t\t\trn.Stop()\n\t\t\trn.WaitForStop()\n\t\t},\n\t\t\"server\": func(s *Server, rn RaftNode) {\n\t\t\ts.running.Store(false)\n\t\t},\n\t} {\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for all servers to have applied everything up to this point.\n\t\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\t_, _, applied := mset.raftNode().Progress()\n\t\t\t\t\tif applied != 1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"expected applied to be %d, got %d\", 1, applied)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\trn := mset.raftNode()\n\t\t\tsnap, err := json.Marshal(streamSnapshot{Msgs: 1, Bytes: 1, FirstSeq: 100, LastSeq: 100, Failed: 0, Deleted: nil})\n\t\t\trequire_NoError(t, err)\n\t\t\tesm := encodeStreamMsgAllowCompress(\"foo\", _EMPTY_, nil, nil, 0, 0, false)\n\n\t\t\t// Lock stream so that we can go into processSnapshot but must wait for this to unlock.\n\t\t\tmset.mu.Lock()\n\t\t\tvar unlocked bool\n\t\t\tdefer func() {\n\t\t\t\tif !unlocked {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\t_, err = rn.ApplyQ().push(newCommittedEntry(100, []*Entry{newEntry(EntrySnapshot, snap)}))\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = rn.ApplyQ().push(newCommittedEntry(101, []*Entry{newEntry(EntryNormal, esm)}))\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Waiting for the apply queue entry to be captured in monitorStream first.\n\t\t\ttime.Sleep(time.Second)\n\n\t\t\t// Set commit to a very high number, just so that we allow upping Applied()\n\t\t\tn := rn.(*raft)\n\t\t\tn.Lock()\n\t\t\tn.commit = 1000\n\t\t\tn.Unlock()\n\n\t\t\t// Now stop the underlying RAFT node/server so processSnapshot must exit because of it.\n\t\t\ttest(rs, rn)\n\t\t\tmset.mu.Unlock()\n\t\t\tunlocked = true\n\n\t\t\t// Allow some time for the applied number to be updated, in which case it's an error.\n\t\t\ttime.Sleep(time.Second)\n\t\t\t_, _, applied := mset.raftNode().Progress()\n\t\t\trequire_Equal(t, applied, 1)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterDesyncAfterErrorDuringCatchup(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\tonErrorCondition func(server *Server, mset *stream)\n\t}{\n\t\t{\n\t\t\ttitle: \"TooManyRetries\",\n\t\t\tonErrorCondition: func(server *Server, mset *stream) {\n\t\t\t\t// Too many retries while processing snapshot is considered a cluster reset.\n\t\t\t\t// If a leader is temporarily unavailable we shouldn't blow away our state.\n\t\t\t\trequire_True(t, isClusterResetErr(errCatchupTooManyRetries))\n\t\t\t\tmset.resetClusteredState(errCatchupTooManyRetries)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"AbortedNoLeader\",\n\t\t\tonErrorCondition: func(server *Server, mset *stream) {\n\t\t\t\tfor _, n := range server.raftNodes {\n\t\t\t\t\trn := n.(*raft)\n\t\t\t\t\tif rn.accName == \"$G\" {\n\t\t\t\t\t\trn.Lock()\n\t\t\t\t\t\trn.updateLeader(noLeader)\n\t\t\t\t\t\trn.Unlock()\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Processing a snapshot while there's no leader elected is considered a cluster reset.\n\t\t\t\t// If a leader is temporarily unavailable we shouldn't blow away our state.\n\t\t\t\tvar snap StreamReplicatedState\n\t\t\t\tsnap.LastSeq = 1_000 // ensure we can catchup based on the snapshot\n\t\t\t\tappliedIndex := uint64(0) // incorrect index, but doesn't matter for this test\n\t\t\t\terr := mset.processSnapshot(&snap, appliedIndex)\n\t\t\t\trequire_True(t, errors.Is(err, errCatchupAbortedNoLeader))\n\t\t\t\trequire_True(t, isClusterResetErr(err))\n\t\t\t\tmset.resetClusteredState(err)\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tstreamLeader := si.Cluster.Leader\n\t\t\tstreamLeaderServer := c.serverByName(streamLeader)\n\t\t\tnc.Close()\n\t\t\tnc, js = jsClientConnect(t, streamLeaderServer)\n\t\t\tdefer nc.Close()\n\n\t\t\tservers := slices.DeleteFunc([]string{\"S-1\", \"S-2\", \"S-3\"}, func(s string) bool {\n\t\t\t\treturn s == streamLeader\n\t\t\t})\n\n\t\t\t// Publish 10 messages.\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tpubAck, err := js.Publish(\"foo\", []byte(\"ok\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, uint64(i+1))\n\t\t\t}\n\n\t\t\toutdatedServerName := servers[0]\n\t\t\tclusterResetServerName := servers[1]\n\n\t\t\toutdatedServer := c.serverByName(outdatedServerName)\n\t\t\toutdatedServer.Shutdown()\n\t\t\toutdatedServer.WaitForShutdown()\n\n\t\t\t// Publish 10 more messages, one server will be behind.\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tpubAck, err := js.Publish(\"foo\", []byte(\"ok\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pubAck.Sequence, uint64(i+11))\n\t\t\t}\n\n\t\t\t// We will not need the client anymore.\n\t\t\tnc.Close()\n\n\t\t\t// Shutdown stream leader so one server remains.\n\t\t\tstreamLeaderServer.Shutdown()\n\t\t\tstreamLeaderServer.WaitForShutdown()\n\n\t\t\tclusterResetServer := c.serverByName(clusterResetServerName)\n\t\t\tacc, err := clusterResetServer.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Run error condition.\n\t\t\ttest.onErrorCondition(clusterResetServer, mset)\n\n\t\t\t// Stream leader stays offline, we only start the server with missing stream data.\n\t\t\t// We expect that the reset server must not allow the outdated server to become leader, as that would result in desync.\n\t\t\tc.restartServer(outdatedServer)\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t\t\t// Outdated server must NOT become the leader.\n\t\t\tnewStreamLeaderServer := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\trequire_Equal(t, newStreamLeaderServer.Name(), clusterResetServerName)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterConsumerDesyncAfterErrorDuringStreamCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tconsumerLeader := ci.Cluster.Leader\n\tconsumerLeaderServer := c.serverByName(consumerLeader)\n\tnc.Close()\n\tnc, js = jsClientConnect(t, consumerLeaderServer)\n\tdefer nc.Close()\n\n\tservers := slices.DeleteFunc([]string{\"S-1\", \"S-2\", \"S-3\"}, func(s string) bool {\n\t\treturn s == consumerLeader\n\t})\n\n\t// Publish 1 message, consume, and ack it.\n\tpubAck, err := js.Publish(\"foo\", []byte(\"ok\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\toutdatedServerName := servers[0]\n\tclusterResetServerName := servers[1]\n\n\toutdatedServer := c.serverByName(outdatedServerName)\n\toutdatedServer.Shutdown()\n\toutdatedServer.WaitForShutdown()\n\n\t// Publish and ack another message, one server will be behind.\n\tpubAck, err = js.Publish(\"foo\", []byte(\"ok\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\t// We will not need the client anymore.\n\tnc.Close()\n\n\t// Shutdown consumer leader so one server remains.\n\tconsumerLeaderServer.Shutdown()\n\tconsumerLeaderServer.WaitForShutdown()\n\n\tclusterResetServer := c.serverByName(clusterResetServerName)\n\tacc, err := clusterResetServer.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Run error condition.\n\tmset.resetClusteredState(nil)\n\n\t// Consumer leader stays offline, we only start the server with missing stream/consumer data.\n\t// We expect that the reset server must not allow the outdated server to become leader, as that would result in desync.\n\tc.restartServer(outdatedServer)\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\n\t// Outdated server must NOT become the leader.\n\tnewConsumerLeaderServer := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_Equal(t, newConsumerLeaderServer.Name(), clusterResetServerName)\n}\n\nfunc TestJetStreamClusterDesyncAfterEofFromOldStreamLeader(t *testing.T) {\n\ttest := func(t *testing.T, eof bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\t\tdefer c.shutdown()\n\n\t\tcs := c.randomServer()\n\t\tnc, js := jsClientConnect(t, cs)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: 5,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tvar rs *Server\n\t\tvar catchup *Server\n\t\tfor _, s := range c.servers {\n\t\t\tif s != sl && s != cs {\n\t\t\t\tif rs == nil {\n\t\t\t\t\trs = s\n\t\t\t\t} else {\n\t\t\t\t\tcatchup = s\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Shutdown server that needs to catch up, so it gets a snapshot from the leader after restart.\n\t\tcatchup.Shutdown()\n\n\t\t// One message is received and applied by all replicas.\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// Disable Raft and start cluster subs for server, simulating an old leader with an outdated log.\n\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tmset.startClusterSubs()\n\t\trn := mset.raftNode()\n\t\trn.Stop()\n\t\trn.WaitForStop()\n\n\t\t// Temporarily disable cluster subs for this test.\n\t\t// Normally due to multiple cluster subs responses will interleave, but this is simpler for this test.\n\t\tacc, err = sl.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err = acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tmset.stopClusterSubs()\n\n\t\t// Publish another message that the old leader will not get.\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, sl.JetStreamSnapshotStream(globalAccountName, \"TEST\"))\n\n\t\tsa := sl.getJetStream().streamAssignment(globalAccountName, \"TEST\")\n\t\trequire_NotNil(t, sa)\n\n\t\t// Send EOF immediately to requesting server, otherwise the server needs to time out and retry.\n\t\tif eof {\n\t\t\tsnc, err := nats.Connect(rs.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer snc.Close()\n\n\t\t\tsub, err := snc.Subscribe(sa.Sync, func(msg *nats.Msg) {\n\t\t\t\t// EOF\n\t\t\t\trs.sendInternalMsgLocked(msg.Reply, _EMPTY_, nil, nil)\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Drain()\n\t\t}\n\n\t\t// Restart server so it starts catching up.\n\t\tcatchup = c.restartServer(catchup)\n\n\t\t// Wait for server to start catching up.\n\t\t// This shouldn't be a problem. The server should retry catchup, recognizing it wasn't caught up fully.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif a, err := catchup.lookupAccount(globalAccountName); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if m, err := a.lookupStream(\"TEST\"); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if !m.isCatchingUp() {\n\t\t\t\treturn errors.New(\"stream not catching up\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Stop old leader, and re-enable cluster subs on proper leader.\n\t\trs.Shutdown()\n\t\tmset.startClusterSubs()\n\n\t\t// Server should automatically restart catchup and get the missing data.\n\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\n\tt.Run(\"eof\", func(t *testing.T) { test(t, true) })\n\tt.Run(\"retry\", func(t *testing.T) { test(t, false) })\n}\n\nfunc TestJetStreamClusterReservedResourcesAccountingAfterClusterReset(t *testing.T) {\n\tfor _, clusterResetErr := range []error{errLastSeqMismatch, errFirstSequenceMismatch} {\n\t\tt.Run(clusterResetErr.Error(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tmaxBytes := int64(1024 * 1024 * 1024)\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxBytes: maxBytes,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\n\t\t\tmem, store, err := sl.JetStreamReservedResources()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, mem, 0)\n\t\t\trequire_Equal(t, store, maxBytes)\n\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsjs := sl.getJetStream()\n\t\t\trn := mset.raftNode()\n\t\t\tsa := mset.streamAssignment()\n\t\t\tsjs.mu.RLock()\n\t\t\tsaGroupNode := sa.Group.node\n\t\t\tsjs.mu.RUnlock()\n\t\t\trequire_NotNil(t, sa)\n\t\t\trequire_Equal(t, rn, saGroupNode)\n\n\t\t\trequire_True(t, mset.resetClusteredState(clusterResetErr))\n\n\t\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\tsjs.mu.RLock()\n\t\t\t\tdefer sjs.mu.RUnlock()\n\t\t\t\tif sa.Group.node == nil || sa.Group.node == saGroupNode {\n\t\t\t\t\treturn errors.New(\"waiting for reset to complete\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tmem, store, err = sl.JetStreamReservedResources()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, mem, 0)\n\t\t\trequire_Equal(t, store, maxBytes)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterHardKillAfterStreamAdd(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Simulate being hard killed by:\n\t// 1. copy directories before shutdown\n\tcopyToSrcMap := make(map[string]string)\n\tfor _, s := range c.servers {\n\t\tsd := s.StoreDir()\n\t\tcopySd := path.Join(t.TempDir(), JetStreamStoreDir)\n\t\terr = copyDir(t, copySd, sd)\n\t\trequire_NoError(t, err)\n\t\tcopyToSrcMap[copySd] = sd\n\t}\n\n\t// 2. stop all\n\tnc.Close()\n\tc.stopAll()\n\n\t// 3. revert directories to before shutdown\n\tfor cp, dest := range copyToSrcMap {\n\t\terr = os.RemoveAll(dest)\n\t\trequire_NoError(t, err)\n\t\terr = copyDir(t, dest, cp)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// 4. restart\n\tc.restartAll()\n\tc.waitOnAllCurrent()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Stream should exist still and not be removed after hard killing all servers, so expect no error.\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterDesyncAfterPublishToLeaderWithoutQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tstreamLeader := si.Cluster.Leader\n\tstreamLeaderServer := c.serverByName(streamLeader)\n\tnc.Close()\n\tnc, js = jsClientConnect(t, streamLeaderServer)\n\tdefer nc.Close()\n\n\tservers := slices.DeleteFunc([]string{\"S-1\", \"S-2\", \"S-3\"}, func(s string) bool {\n\t\treturn s == streamLeader\n\t})\n\n\t// Stop followers so further publishes will not have quorum.\n\tfollowerName1 := servers[0]\n\tfollowerName2 := servers[1]\n\tfollowerServer1 := c.serverByName(followerName1)\n\tfollowerServer2 := c.serverByName(followerName2)\n\tfollowerServer1.Shutdown()\n\tfollowerServer2.Shutdown()\n\tfollowerServer1.WaitForShutdown()\n\tfollowerServer2.WaitForShutdown()\n\n\t// Although this request will time out, it will be added to the stream leader's WAL.\n\t_, err = js.Publish(\"foo\", []byte(\"first\"), nats.AckWait(time.Second))\n\trequire_NotNil(t, err)\n\trequire_Equal(t, err, nats.ErrTimeout)\n\n\t// Now shut down the leader as well.\n\tnc.Close()\n\tstreamLeaderServer.Shutdown()\n\tstreamLeaderServer.WaitForShutdown()\n\n\t// Only restart the (previous) followers.\n\tfollowerServer1 = c.restartServer(followerServer1)\n\tc.restartServer(followerServer2)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnc, js = jsClientConnect(t, followerServer1)\n\tdefer nc.Close()\n\n\t// Publishing a message will now have quorum.\n\tpubAck, err := js.Publish(\"foo\", []byte(\"first, this is a retry\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t// Bring up the previous stream leader.\n\tc.restartServer(streamLeaderServer)\n\tc.waitOnAllCurrent()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Check all servers ended up with the last published message, which had quorum.\n\tcheckFor(t, 3*time.Second, 250*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 1 || state.Bytes != 55 {\n\t\t\t\treturn fmt.Errorf(\"stream state didn't match, got %d messages with %d bytes\", state.Msgs, state.Bytes)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterPreserveWALDuringCatchupWithMatchingTerm(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcheckConsistency := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 3*time.Second, 250*time.Millisecond, func() error {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tstate := mset.state()\n\t\t\t\tif state.Msgs != 3 || state.Bytes != 99 {\n\t\t\t\t\treturn fmt.Errorf(\"stream state didn't match, got %d messages with %d bytes\", state.Msgs, state.Bytes)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\trequire_NoError(t, err)\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trn := mset.raftNode().(*raft)\n\tleaderId := rn.ID()\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tnc.Close()\n\tcheckConsistency()\n\n\t// Pick one server that will only store a part of the messages in its WAL.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tacc, err = rs.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err = acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\trn = mset.raftNode().(*raft)\n\tindex, commit, _ := rn.Progress()\n\trequire_Equal(t, index, 4)\n\trequire_Equal(t, index, commit)\n\n\t// We'll simulate as-if the last message was never received/stored.\n\t// Will need to truncate the stream, correct lseq (so the msg isn't skipped) and truncate the WAL.\n\t// This will simulate that the RAFT layer can restore it.\n\tmset.mu.Lock()\n\tmset.lseq--\n\terr = mset.store.Truncate(2)\n\tmset.mu.Unlock()\n\trequire_NoError(t, err)\n\trn.Lock()\n\trn.truncateWAL(rn.pterm, rn.pindex-1)\n\trn.Unlock()\n\n\t// Check all servers ended up with all published messages, which had quorum.\n\tcheckConsistency()\n\n\t// Check that all entries came from the expected leader.\n\tfor _, n := range rs.raftNodes {\n\t\trn := n.(*raft)\n\t\tif rn.accName == globalAccountName {\n\t\t\tae, err := rn.loadEntry(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, ae.leader, leaderId)\n\n\t\t\tae, err = rn.loadEntry(2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, ae.leader, leaderId)\n\n\t\t\tae, err = rn.loadEntry(3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, ae.leader, leaderId)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterDesyncAfterRestartReplacesLeaderSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Reconnect to the leader.\n\tleader := c.streamLeader(globalAccountName, \"TEST\")\n\tnc.Close()\n\tnc, js = jsClientConnect(t, leader)\n\tdefer nc.Close()\n\n\tlookupStream := func(s *Server) *stream {\n\t\tt.Helper()\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\treturn mset\n\t}\n\n\t// Stop one follower so it lags behind.\n\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tmset := lookupStream(rs)\n\tn := mset.node.(*raft)\n\tfollowerSnapshots := path.Join(n.sd, snapshotsDir)\n\trs.Shutdown()\n\trs.WaitForShutdown()\n\n\t// Move the stream forward so the follower requires a snapshot.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 10})\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Install a snapshot on the leader, ensuring RAFT entries are compacted and a snapshot remains.\n\tmset = lookupStream(leader)\n\tn = mset.node.(*raft)\n\terr = n.InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\tc.stopAll()\n\n\t// Replace follower snapshot with the leader's.\n\t// This simulates the follower coming online, getting a snapshot from the leader after which it goes offline.\n\tleaderSnapshots := path.Join(n.sd, snapshotsDir)\n\terr = os.RemoveAll(followerSnapshots)\n\trequire_NoError(t, err)\n\terr = copyDir(t, followerSnapshots, leaderSnapshots)\n\trequire_NoError(t, err)\n\n\t// Start the follower, it will load the snapshot from the leader.\n\trs = c.restartServer(rs)\n\n\t// Shutting down must check that the leader's snapshot is not overwritten.\n\trs.Shutdown()\n\trs.WaitForShutdown()\n\n\t// Now start all servers back up.\n\tc.restartAll()\n\tc.waitOnAllCurrent()\n\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterKeepRaftStateIfStreamCreationFailedDuringShutdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tnc.Close()\n\n\t// Capture RAFT storage directory and JetStream handle before shutdown.\n\ts := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tsd := mset.node.(*raft).sd\n\tjss := s.getJetStream()\n\n\t// Shutdown the server.\n\t// Normally there are no actions taken anymore after shutdown completes,\n\t// but still do so to simulate actions taken while shutdown is in progress.\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\n\t// Check RAFT state is kept.\n\tfiles, err := os.ReadDir(sd)\n\trequire_NoError(t, err)\n\trequire_True(t, len(files) > 0)\n\n\t// Simulate server shutting down, JetStream being disabled and a stream being created.\n\tsa := &streamAssignment{\n\t\tConfig: &StreamConfig{Name: \"TEST\"},\n\t\tGroup: &raftGroup{node: &raft{}},\n\t}\n\tjss.processClusterCreateStream(acc, sa)\n\n\t// Check RAFT state is not deleted due to failing stream creation.\n\tfiles, err = os.ReadDir(sd)\n\trequire_NoError(t, err)\n\trequire_True(t, len(files) > 0)\n}\n\nfunc TestJetStreamClusterMetaSnapshotReCreateConsistency(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tscfg := &nats.StreamConfig{Name: \"TEST\", Replicas: 3}\n\t_, err := js.AddStream(scfg)\n\trequire_NoError(t, err)\n\n\tccfg := &nats.ConsumerConfig{Name: \"consumer\", Replicas: 3}\n\t_, err = js.AddConsumer(\"TEST\", ccfg)\n\trequire_NoError(t, err)\n\n\tml := c.leader()\n\tmjs := ml.getJetStream()\n\tmjs.mu.Lock()\n\tsa := mjs.streamAssignment(globalAccountName, \"TEST\")\n\tca := mjs.consumerAssignment(globalAccountName, \"TEST\", \"consumer\")\n\n\toldStreamGroup := sa.Group.Name\n\toldConsumerGroup := ca.Group.Name\n\tstreamDelete := encodeDeleteStreamAssignment(sa)\n\n\tcsa := sa.copyGroup()\n\tcca := ca.copyGroup()\n\tcsa.Group.Name, csa.Config.Replicas = \"new-group\", 1\n\tcca.Group.Name, cca.Config.Replicas = \"new-group\", 1\n\tstreamAdd := encodeAddStreamAssignment(csa)\n\tconsumerAdd := encodeAddConsumerAssignment(cca)\n\tmjs.mu.Unlock()\n\n\t// Get the snapshot before removing the stream below so we can recover fresh.\n\tsnap, _, _, err := mjs.metaSnapshot()\n\trequire_NoError(t, err)\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\tnc.Close()\n\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\n\t// Simulate recovering:\n\t// - snapshot with a stream and consumer\n\t// - normal entry deleting the stream\n\t// - normal entry re-adding the stream and consumer under different configs\n\t// This should result in a consistent state.\n\tmjs.mu.Lock()\n\tmjs.metaRecovering = true\n\tmjs.mu.Unlock()\n\t_, _, err = mjs.applyMetaEntries([]*Entry{\n\t\tnewEntry(EntrySnapshot, snap),\n\t\tnewEntry(EntryNormal, streamDelete),\n\t\tnewEntry(EntryNormal, streamAdd),\n\t\tnewEntry(EntryNormal, consumerAdd),\n\t}, ru)\n\trequire_NoError(t, err)\n\n\t// Recovery should contain the stream and consumer create.\n\trequire_Len(t, len(ru.addStreams), 1)\n\trequire_Len(t, len(ru.updateConsumers), 1)\n\n\t// Process those updates.\n\tfor _, sa = range ru.addStreams {\n\t\tmjs.processStreamAssignment(sa)\n\t}\n\tfor _, cas := range ru.updateConsumers {\n\t\tfor _, ca = range cas {\n\t\t\tmjs.processConsumerAssignment(ca)\n\t\t}\n\t}\n\n\t// Should not have created old Raft nodes during recovery.\n\tn1 := ml.lookupRaftNode(oldStreamGroup)\n\tn2 := ml.lookupRaftNode(oldConsumerGroup)\n\trequire_True(t, n1 == nil)\n\trequire_True(t, n2 == nil)\n}\n\nfunc TestJetStreamClusterMetaSnapshotConsumerDeleteConsistency(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tscfg := &nats.StreamConfig{Name: \"TEST\", Replicas: 1}\n\t_, err := js.AddStream(scfg)\n\trequire_NoError(t, err)\n\n\tccfg := &nats.ConsumerConfig{Name: \"consumer\", Replicas: 1}\n\t_, err = js.AddConsumer(\"TEST\", ccfg)\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmjs := sl.getJetStream()\n\tmjs.mu.Lock()\n\tca := mjs.consumerAssignment(globalAccountName, \"TEST\", \"consumer\")\n\tca.Created = time.Time{} // Simulate this consumer existed for a while already.\n\tdeleteConsumer := encodeDeleteConsumerAssignment(ca)\n\tmjs.mu.Unlock()\n\n\t// Get the snapshot before removing the stream below so we can recover fresh.\n\tsnap, _, _, err := mjs.metaSnapshot()\n\trequire_NoError(t, err)\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\tnc.Close()\n\n\tru := &recoveryUpdates{\n\t\tremoveStreams: make(map[string]*streamAssignment),\n\t\tremoveConsumers: make(map[string]map[string]*consumerAssignment),\n\t\taddStreams: make(map[string]*streamAssignment),\n\t\tupdateStreams: make(map[string]*streamAssignment),\n\t\tupdateConsumers: make(map[string]map[string]*consumerAssignment),\n\t}\n\n\t// Simulate recovering:\n\t// - snapshot with a stream and consumer\n\t// - normal entry deleting the consumer\n\t// This should result in a consistent state.\n\tmjs.mu.Lock()\n\tmjs.metaRecovering = true\n\tmjs.mu.Unlock()\n\t_, _, err = mjs.applyMetaEntries([]*Entry{\n\t\tnewEntry(EntrySnapshot, snap),\n\t\tnewEntry(EntryNormal, deleteConsumer),\n\t}, ru)\n\trequire_NoError(t, err)\n\n\t// Recovery should contain the stream create and the consumer delete.\n\trequire_Len(t, len(ru.updateConsumers), 1)\n\trequire_Len(t, len(ru.removeConsumers), 1)\n\trequire_Len(t, len(ru.addStreams), 1)\n\n\t// Process those updates.\n\tfor _, cas := range ru.updateConsumers {\n\t\trequire_Len(t, len(cas), 0)\n\t}\n\tfor _, cas := range ru.removeConsumers {\n\t\tfor _, ca = range cas {\n\t\t\tmjs.processConsumerRemoval(ca)\n\t\t}\n\t}\n\tfor _, sa := range ru.addStreams {\n\t\tmjs.processStreamAssignment(sa)\n\t}\n\tmjs.clearMetaRecovering()\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\ths := sl.healthz(&HealthzOptions{})\n\t\tif hs.Error != _EMPTY_ {\n\t\t\treturn errors.New(hs.Error) // Would previously error with \"consumer not found\".\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMetaSnapshotDecodeError(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tjs := c.leader().getJetStream()\n\n\tisRecovering, didSnap, err := js.applyMetaEntries(\n\t\t[]*Entry{newEntry(EntrySnapshot, []byte(\"not a valid snapshot\"))}, nil)\n\n\trequire_Error(t, err)\n\trequire_False(t, isRecovering)\n\trequire_False(t, didSnap)\n}\n\nfunc TestJetStreamClusterConsumerDontSendSnapshotOnLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Add a message and let the consumer ack it, this moves the consumer's RAFT applied up to 1.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\terr = msgs[0].AckSync()\n\trequire_NoError(t, err)\n\n\t// We don't need the client anymore.\n\tnc.Close()\n\n\tlookupConsumer := func(s *Server) *consumer {\n\t\tt.Helper()\n\t\tmset, err := s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tacc, err := mset.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := acc.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\t\treturn o\n\t}\n\n\t// Grab current consumer leader before moving all into observer mode.\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tfor _, s := range c.servers {\n\t\t// Put all consumer's RAFT into observer mode, this will prevent all servers from trying to become leader.\n\t\to := lookupConsumer(s)\n\t\to.node.SetObserver(true)\n\t\tif s != cl {\n\t\t\t// For all followers, pause apply so they only store messages in WAL but not apply and possibly snapshot.\n\t\t\terr = o.node.PauseApply()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tupdateDeliveredBuffer := func() []byte {\n\t\tvar b [4*binary.MaxVarintLen64 + 1]byte\n\t\tb[0] = byte(updateDeliveredOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\tn += binary.PutUvarint(b[n:], 1)\n\t\tn += binary.PutVarint(b[n:], time.Now().UnixNano())\n\t\treturn b[:n]\n\t}\n\n\tupdateAcksBuffer := func() []byte {\n\t\tvar b [2*binary.MaxVarintLen64 + 1]byte\n\t\tb[0] = byte(updateAcksOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\tn += binary.PutUvarint(b[n:], 100)\n\t\treturn b[:n]\n\t}\n\n\t// Store an uncommitted entry into our WAL, which will be committed and applied later.\n\tco := lookupConsumer(cl)\n\trn := co.node.(*raft)\n\trn.Lock()\n\tentries := []*Entry{{EntryNormal, updateDeliveredBuffer()}, {EntryNormal, updateAcksBuffer()}}\n\tae := encode(t, rn.buildAppendEntry(entries))\n\terr = rn.storeToWAL(ae)\n\tminPindex := rn.pindex\n\trn.Unlock()\n\trequire_NoError(t, err)\n\n\t// Simulate leader change, we do this so we can check what happens in the upper layer logic.\n\trn.leadc <- true\n\trn.SetObserver(false)\n\n\t// Since upper layer is async, we don't know whether it will or will not act on the leader change.\n\t// Wait for some time to check if it does.\n\ttime.Sleep(2 * time.Second)\n\trn.RLock()\n\tmaxPindex := rn.pindex\n\trn.RUnlock()\n\n\tr := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\tro := lookupConsumer(r)\n\trn = ro.node.(*raft)\n\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\trn.RLock()\n\t\tdefer rn.RUnlock()\n\t\tif rn.pindex < maxPindex {\n\t\t\treturn fmt.Errorf(\"rn.pindex too low, expected %d, got %d\", maxPindex, rn.pindex)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// We should only have 'Normal' entries.\n\t// If we'd get a 'Snapshot' entry, that would mean it had incomplete state and would be reverting committed state.\n\tvar state StreamState\n\trn.wal.FastState(&state)\n\tfor seq := minPindex; seq <= maxPindex; seq++ {\n\t\tae, err = rn.loadEntry(seq)\n\t\trequire_NoError(t, err)\n\t\tfor _, entry := range ae.entries {\n\t\t\trequire_Equal(t, entry.Type, EntryNormal)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterDontInstallSnapshotWhenStoppingStream(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Wait for all servers to have applied everything.\n\tvar maxApplied uint64\n\tvar tries int\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, _, applied := mset.node.Progress(); applied > maxApplied {\n\t\t\t\tmaxApplied, tries = applied, 0\n\t\t\t\treturn fmt.Errorf(\"applied upped to %d\", maxApplied)\n\t\t\t} else if applied != maxApplied {\n\t\t\t\treturn fmt.Errorf(\"applied doesn't match, expected %d, got %d\", maxApplied, applied)\n\t\t\t}\n\t\t}\n\t\ttries++\n\t\tif tries < 3 {\n\t\t\treturn fmt.Errorf(\"retrying for applied %d (try %d)\", maxApplied, tries)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Install a snapshot on a follower.\n\ts := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\terr = mset.node.InstallSnapshot(mset.stateSnapshotLocked(), false)\n\trequire_NoError(t, err)\n\n\t// Validate the snapshot reflects applied.\n\tvalidateStreamState := func(snap *snapshot) {\n\t\tt.Helper()\n\t\trequire_Equal(t, snap.lastIndex, maxApplied)\n\t\tss, err := DecodeStreamState(snap.data)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\trequire_Equal(t, ss.LastSeq, 1)\n\t}\n\tsnap, err := mset.node.(*raft).loadLastSnapshot()\n\trequire_NoError(t, err)\n\tvalidateStreamState(snap)\n\n\t// Simulate a message being stored, but not calling Applied yet.\n\terr = mset.processJetStreamMsg(\"foo\", _EMPTY_, nil, nil, 1, time.Now().UnixNano(), nil, false, true)\n\trequire_NoError(t, err)\n\n\t// Simulate the stream being stopped before we're able to call Applied.\n\t// If we'd install a snapshot during this, which would be a race condition,\n\t// we'd store a snapshot with state that's ahead of applied.\n\terr = mset.stop(false, false)\n\trequire_NoError(t, err)\n\n\t// Validate the snapshot is the same as before.\n\tsnap, err = mset.node.(*raft).loadLastSnapshot()\n\trequire_NoError(t, err)\n\tvalidateStreamState(snap)\n}\n\nfunc TestJetStreamClusterDontInstallSnapshotWhenStoppingConsumer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Add a message and let the consumer ack it, this moves the consumer's RAFT applied up.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\terr = msgs[0].AckSync()\n\trequire_NoError(t, err)\n\n\t// Wait for all servers to have applied everything.\n\tvar maxApplied uint64\n\tvar tries int\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\tif o == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t\tif _, _, applied := o.node.Progress(); applied > maxApplied {\n\t\t\t\tmaxApplied, tries = applied, 0\n\t\t\t\treturn fmt.Errorf(\"applied upped to %d\", maxApplied)\n\t\t\t} else if applied != maxApplied {\n\t\t\t\treturn fmt.Errorf(\"applied doesn't match, expected %d, got %d\", maxApplied, applied)\n\t\t\t}\n\t\t}\n\t\ttries++\n\t\tif tries < 3 {\n\t\t\treturn fmt.Errorf(\"retrying for applied %d (try %d)\", maxApplied, tries)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Install a snapshot on a follower.\n\ts := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\tsnapBytes, err := o.store.EncodedState()\n\trequire_NoError(t, err)\n\terr = o.node.InstallSnapshot(snapBytes, false)\n\trequire_NoError(t, err)\n\n\t// Validate the snapshot reflects applied.\n\tvalidateConsumerState := func(snap *snapshot) {\n\t\tt.Helper()\n\t\trequire_Equal(t, snap.lastIndex, maxApplied)\n\t\tstate, err := decodeConsumerState(snap.data)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, state.Delivered.Consumer, 1)\n\t\trequire_Equal(t, state.Delivered.Stream, 1)\n\t}\n\tsnap, err := o.node.(*raft).loadLastSnapshot()\n\trequire_NoError(t, err)\n\tvalidateConsumerState(snap)\n\n\t// Simulate a message being delivered, but not calling Applied yet.\n\terr = o.store.UpdateDelivered(2, 2, 1, time.Now().UnixNano())\n\trequire_NoError(t, err)\n\n\t// Simulate the consumer being stopped before we're able to call Applied.\n\t// If we'd install a snapshot during this, which would be a race condition,\n\t// we'd store a snapshot with state that's ahead of applied.\n\terr = o.stop()\n\trequire_NoError(t, err)\n\n\t// Validate the snapshot is the same as before.\n\tsnap, err = o.node.(*raft).loadLastSnapshot()\n\trequire_NoError(t, err)\n\tvalidateConsumerState(snap)\n}\n\nfunc TestJetStreamClusterStreamConsumerStateResetAfterRecreate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\tstream := \"test:0\"\n\tconfig := &nats.StreamConfig{\n\t\tName: stream,\n\t\tSubjects: []string{\"test.0.*\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tMaxMsgs: 100_000,\n\t\tDiscard: nats.DiscardNew,\n\t\tDuplicates: 5 * time.Second,\n\t\tStorage: nats.MemoryStorage,\n\t}\n\tconsumer := \"A:0:0\"\n\tsubject := \"test.0.0\"\n\tvar (\n\t\tduration = 30 * time.Minute\n\t\tproducerMsgs = 200_000\n\t\tproducerMsgSize = 1024\n\t\tpayload = []byte(strings.Repeat(\"A\", producerMsgSize))\n\t\twg sync.WaitGroup\n\t\tn atomic.Uint64\n\t\tcanPublish atomic.Bool\n\t)\n\tcreateStream := func(t *testing.T) {\n\t\tt.Helper()\n\t\t_, err := js.AddStream(config)\n\t\trequire_NoError(t, err)\n\t\tconsumer := &nats.ConsumerConfig{\n\t\t\tDurable: consumer,\n\t\t\tReplicas: 3,\n\t\t\tMaxAckPending: 100_000,\n\t\t\tMaxWaiting: 100_000,\n\t\t\tFilterSubject: subject,\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t}\n\t\t_, err = js.AddConsumer(stream, consumer)\n\t\trequire_NoError(t, err)\n\t}\n\tdeleteStream := func(t *testing.T) {\n\t\terr := js.DeleteStream(stream)\n\t\trequire_NoError(t, err)\n\t}\n\tstopPublishing := func() {\n\t\tcanPublish.Store(false)\n\t}\n\tresumePublishing := func() {\n\t\tcanPublish.Store(true)\n\t}\n\t// Setup stream\n\tctx, cancel := context.WithTimeout(context.Background(), duration)\n\tdefer cancel()\n\tcreateStream(t)\n\t// Setup producer\n\tresumePublishing()\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\t\tfor range time.NewTicker(1 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tif !canPublish.Load() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, err := js.Publish(\"test.0.0\", payload, nats.AckWait(200*time.Millisecond))\n\t\t\tif err == nil {\n\t\t\t\tif nn := n.Add(1); int(nn) >= producerMsgs {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\t// Setup consumer\n\tacked := make(chan struct{}, 100)\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\tAttempts:\n\t\tfor attempts := 0; attempts < 10; attempts++ {\n\t\t\t_, err := js.ConsumerInfo(stream, consumer)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"WRN: Failed creating pull subscriber: %v - %v - %v - %v\",\n\t\t\t\t\tsubject, stream, consumer, err)\n\t\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbreak Attempts\n\t\t}\n\t\tsub, err := js.PullSubscribe(subject, \"\", nats.Bind(stream, consumer))\n\t\tif err != nil {\n\t\t\tt.Logf(\"WRN: Failed creating pull subscriber: %v - %v - %v - %v\",\n\t\t\t\tsubject, stream, consumer, err)\n\t\t\treturn\n\t\t}\n\t\trequire_NoError(t, err)\n\t\tfor range time.NewTicker(100 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond))\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, msg := range msgs {\n\t\t\t\ttime.AfterFunc(3*time.Second, func() {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t\tacked <- struct{}{}\n\t\t\t\t})\n\t\t\t}\n\t\t\tmsgs, err = sub.Fetch(10, nats.MaxWait(200*time.Millisecond))\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, msg := range msgs {\n\t\t\t\tmsg.Ack()\n\t\t\t}\n\t\t}\n\t}()\n\t// Let publish and consume to happen for a bit.\n\ttime.Sleep(2 * time.Second)\n\t// Recreate the stream\n\tdeleteStream(t)\n\tstopPublishing()\n\tcreateStream(t)\n\tfor i := 0; i < 3; i++ {\n\t\tjs.Publish(\"test.0.0\", payload, nats.AckWait(200*time.Millisecond))\n\t}\n\tselect {\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Timed out waiting for ack\")\n\tcase <-acked:\n\t\ttime.Sleep(2 * time.Second)\n\t}\n\tsinfo, err := js.StreamInfo(stream)\n\trequire_NoError(t, err)\n\tcinfo, err := js.ConsumerInfo(stream, consumer)\n\trequire_NoError(t, err)\n\tcancel()\n\tif cinfo.Delivered.Stream > sinfo.State.LastSeq {\n\t\tt.Fatalf(\"Consumer Stream sequence is ahead of Stream LastSeq: consumer=%d, stream=%d\", cinfo.Delivered.Stream, sinfo.State.LastSeq)\n\t}\n\twg.Wait()\n}\n\nfunc TestJetStreamClusterStreamAckMsgR1SignalsRemovedMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 1,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\ts := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Too high sequence, should register pre-ack and return true allowing for retries.\n\trequire_True(t, mset.ackMsg(o, 100))\n\n\tvar smv StoreMsg\n\tsm, err := mset.store.LoadMsg(1, &smv)\n\trequire_NoError(t, err)\n\trequire_Equal(t, sm.subj, \"foo\")\n\n\t// Should not remove if the message is still pending, even if we call ack.\n\trequire_False(t, mset.ackMsg(o, 1))\n\tsm, err = mset.store.LoadMsg(1, &smv)\n\trequire_NoError(t, err)\n\trequire_Equal(t, sm.subj, \"foo\")\n\n\t// Now do a proper ack, should immediately remove the message since it's R1.\n\to.mu.Lock()\n\to.sseq, o.dseq = 2, 2\n\to.asflr, o.adflr = 1, 1\n\to.mu.Unlock()\n\trequire_True(t, mset.ackMsg(o, 1))\n\t_, err = mset.store.LoadMsg(1, &smv)\n\trequire_Error(t, err, ErrStoreMsgNotFound)\n}\n\nfunc TestJetStreamClusterStreamAckMsgR3SignalsRemovedMsg(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Wait for all servers to know about the stream and consumer.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif mset.lookupConsumer(\"CONSUMER\") == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Also wait for the published message to be replicated.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tsf := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\n\tmsetL, err := sl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tol := msetL.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, ol)\n\tmsetF, err := sf.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tof := msetF.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, of)\n\n\t// Too high sequence, should register pre-ack and return true allowing for retries.\n\trequire_True(t, msetL.ackMsg(ol, 100))\n\trequire_True(t, msetF.ackMsg(of, 100))\n\n\t// We're bypassing the normal ack flow, so must set these values ourselves.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\n\t\tif o.IsLeader() {\n\t\t\to.mu.Lock()\n\t\t\to.sseq, o.dseq = 2, 2\n\t\t\to.asflr, o.adflr = 1, 1\n\t\t\to.mu.Unlock()\n\t\t} else {\n\t\t\trequire_NoError(t, o.store.Update(&ConsumerState{\n\t\t\t\tDelivered: SequencePair{1, 1},\n\t\t\t\tAckFloor: SequencePair{1, 1},\n\t\t\t}))\n\t\t}\n\t}\n\n\t// Ack message on follower, should not remove message as that's proposed by the leader.\n\t// But should still signal message removal.\n\trequire_True(t, msetF.ackMsg(of, 1))\n\n\t// Confirm all servers have the message.\n\tvar smv StoreMsg\n\tfor _, s := range c.servers {\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tsm, err := mset.store.LoadMsg(1, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, \"foo\")\n\t}\n\n\t// Now do a proper ack, should propose the message removal since it's R3.\n\trequire_True(t, msetL.ackMsg(ol, 1))\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, err = mset.store.LoadMsg(1, &smv)\n\t\t\tif err != ErrStoreMsgNotFound {\n\t\t\t\treturn fmt.Errorf(\"expected error, but got: %v\", err)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterExpectedPerSubjectConsistency(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\ts := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Block updates when subject already in process.\n\tmset.clMu.Lock()\n\tmset.expectedPerSubjectSequence = map[uint64]string{0: \"foo\"}\n\tmset.expectedPerSubjectInProcess = map[string]struct{}{\"foo\": {}}\n\tmset.clMu.Unlock()\n\t_, err = js.Publish(\"foo\", nil, nats.ExpectLastSequencePerSubject(0))\n\trequire_Error(t, err, NewJSStreamWrongLastSequenceConstantError())\n\n\t// Block updates when subject already inflight without expected headers.\n\tmset.clMu.Lock()\n\tmset.expectedPerSubjectSequence = nil\n\tmset.expectedPerSubjectInProcess = nil\n\tmset.inflight = map[string]*inflightSubjectRunningTotal{\"foo\": {bytes: 33, ops: 1}}\n\tmset.clMu.Unlock()\n\t_, err = js.Publish(\"foo\", nil, nats.ExpectLastSequencePerSubject(0))\n\trequire_Error(t, err, NewJSStreamWrongLastSequenceConstantError())\n\n\t// Allow updates when ready and subject not already in process.\n\tmset.clMu.Lock()\n\tmset.expectedPerSubjectSequence = nil\n\tmset.expectedPerSubjectInProcess = nil\n\tmset.inflight = nil\n\tmset.clMu.Unlock()\n\tpa, err := js.Publish(\"foo\", nil, nats.ExpectLastSequencePerSubject(0))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pa.Sequence, 1)\n\n\t// Should be cleaned up after publish.\n\tmset.clMu.Lock()\n\tdefer mset.clMu.Unlock()\n\trequire_Len(t, len(mset.expectedPerSubjectSequence), 0)\n\trequire_Len(t, len(mset.expectedPerSubjectInProcess), 0)\n}\n\nfunc TestJetStreamClusterMsgCounterRunningTotalConsistency(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tRetention: LimitsPolicy,\n\t\tReplicas: 3,\n\t\tAllowMsgCounter: true,\n\t})\n\trequire_NoError(t, err)\n\n\ts := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Running total should be kept up-to-date.\n\tmset.clMu.Lock()\n\tmset.clusteredCounterTotal = map[string]*msgCounterRunningTotal{\n\t\t\"foo\": {total: big.NewInt(10), ops: 1},\n\t}\n\tmset.clMu.Unlock()\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Set(\"Nats-Incr\", \"1\")\n\tpubAck, err := js.PublishMsg(m)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\trsm, err := js.GetLastMsg(\"TEST\", \"foo\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, rsm.Sequence, 1)\n\tvar count CounterValue\n\trequire_NoError(t, json.Unmarshal(rsm.Data, &count))\n\trequire_Equal(t, count.Value, \"11\")\n\n\t// Confirm running total has properly been mutated.\n\ttotal, ops := _EMPTY_, uint64(0)\n\tmset.clMu.Lock()\n\tif l := len(mset.clusteredCounterTotal); l != 1 {\n\t\tmset.clMu.Unlock()\n\t\trequire_Len(t, l, 1)\n\t}\n\tif counter, ok := mset.clusteredCounterTotal[\"foo\"]; !ok {\n\t\tmset.clMu.Unlock()\n\t\tt.Fatal(\"counter not found\")\n\t} else {\n\t\ttotal = counter.total.String()\n\t\tops = counter.ops\n\t}\n\tmset.clMu.Unlock()\n\trequire_Equal(t, total, \"11\")\n\trequire_Equal(t, ops, 1)\n\n\t// Reset. Running totals should be removed once all inflight counter operations are applied.\n\tmset.clMu.Lock()\n\tmset.clusteredCounterTotal = nil\n\tmset.clMu.Unlock()\n\tpubAck, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\n\trsm, err = js.GetLastMsg(\"TEST\", \"foo\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, rsm.Sequence, 2)\n\trequire_NoError(t, json.Unmarshal(rsm.Data, &count))\n\trequire_Equal(t, count.Value, \"12\")\n\n\t// Should be cleaned up after publish.\n\tmset.clMu.Lock()\n\tdefer mset.clMu.Unlock()\n\trequire_Len(t, len(mset.clusteredCounterTotal), 0)\n}\n\nfunc TestJetStreamClusterConsistencyAfterLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tacc, err := sl.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tn := mset.raftNode().(*raft)\n\tn.Lock()\n\t// Block snapshots from being made, preserving the full log so we can validate it later.\n\tn.progress = make(map[string]*ipQueue[uint64])\n\tn.progress[\"blockSnapshots\"] = newIPQueue[uint64](n.s, \"blockSnapshots\")\n\t// Put into observer so the RAFT code doesn't switch to candidate on its own.\n\tn.observer = true\n\tn.Unlock()\n\n\tnc.Close()\n\tnc, js = jsClientConnect(t, sl)\n\tdefer nc.Close()\n\n\t// Publish a message and confirm all servers are up-to-date.\n\t// This ensures the first message entry has lseq=0.\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\t// Shutdown all other servers so no changes get quorum.\n\tfor _, s := range c.servers {\n\t\tif s != sl {\n\t\t\ts.Shutdown()\n\t\t\ts.WaitForShutdown()\n\t\t}\n\t}\n\n\t// Publish an initial set of messages to be stored in the leader's WAL, but without quorum.\n\tvar ppindex uint64\n\tfor n.State() == Leader && ppindex <= 10 {\n\t\terr = nc.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\tn.RLock()\n\t\tppindex = n.pindex\n\t\tn.RUnlock()\n\t}\n\t// Only continue if we were actually able to persist messages.\n\trequire_LessThan(t, 10, ppindex)\n\n\t// Step down to follower, forcing a leader transition.\n\tn.stepdown(noLeader)\n\tn.RLock()\n\tppindex = n.pindex\n\tn.RUnlock()\n\n\t// We don't know when the RAFT run loop transitions, just wait for some time.\n\ttime.Sleep(time.Second)\n\n\t// Publish a second set of messages to be stored in the leader's WAL, but without quorum.\n\tvar npindex uint64\n\tn.switchToLeader()\n\tfor n.State() == Leader && npindex <= ppindex*2 {\n\t\terr = nc.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\tn.RLock()\n\t\tnpindex = n.pindex\n\t\tn.RUnlock()\n\t}\n\n\tn.RLock()\n\tvar ss StreamState\n\tn.wal.FastState(&ss)\n\tn.RUnlock()\n\n\t// Go through all messages and confirm the last seq that's\n\t// proposed to the NRG layer is monotonically increasing.\n\tvar clseq uint64\n\tfor seq := ss.FirstSeq; seq <= ss.LastSeq; seq++ {\n\t\tae, err := n.loadEntry(seq)\n\t\trequire_NoError(t, err)\n\t\tfor _, e := range ae.entries {\n\t\t\t_ = e\n\t\t\tif e.Type == EntryNormal && len(e.Data) > 0 && entryOp(e.Data[0]) == streamMsgOp {\n\t\t\t\tsubject, _, _, _, lseq, _, _, err := decodeStreamMsg(e.Data[1:])\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, subject, \"foo\")\n\t\t\t\t// Sequence must monotonically increase. If it wouldn't that would mean the new leader accepted\n\t\t\t\t// new messages before it was in the same state as the previous leader when it stepped down.\n\t\t\t\t// Or there are gaps, which would mean the JetStream layer would run into errLastSeqMismatch.\n\t\t\t\trequire_Equal(t, lseq, clseq)\n\t\t\t\tclseq++\n\t\t\t} else if e.Type != EntryPeerState {\n\t\t\t\tt.Fatalf(\"Received unhandled entry type: %s\\n\", e.Type)\n\t\t\t}\n\t\t}\n\t}\n\trequire_NotEqual(t, clseq, 0)\n\n\t// Remove observer flag so it can become leader on its own.\n\tn.Lock()\n\tn.observer = false\n\tn.Unlock()\n\n\t// Restart one other server, this should result in the previous leader\n\t// to become leader again, as it has the most up-to-date log.\n\t// If we'd bring all of them up it would be up to chance who'd become leader.\n\tfor _, s := range c.servers {\n\t\tif s != sl {\n\t\t\tc.restartServer(s)\n\t\t\tbreak\n\t\t}\n\t}\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Confirm the sequence still monotonically increases.\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, clseq+1)\n\n\tfor _, s := range c.servers {\n\t\tif !s.isRunning() {\n\t\t\tc.restartServer(s)\n\t\t}\n\t}\n\tc.waitOnAllCurrent()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Confirm the sequence still monotonically increases.\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, clseq+2)\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n}\n\nfunc TestJetStreamClusterMetaStepdownPreferred(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t// We know of the preferred server and will successfully hand over to it.\n\tt.Run(\"KnownPreferred\", func(t *testing.T) {\n\t\tleader := c.leader()\n\t\tvar preferred *Server\n\t\tfor _, s := range c.servers {\n\t\t\tif s == leader {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpreferred = s\n\t\t\tbreak\n\t\t}\n\n\t\tbody, err := json.Marshal(JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tPreferred: preferred.Name(),\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(JSApiLeaderStepDown, body, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar apiresp JSApiLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &apiresp))\n\t\trequire_True(t, apiresp.Success)\n\t\trequire_Equal(t, apiresp.Error, nil)\n\n\t\tc.waitOnLeader()\n\t\trequire_Equal(t, preferred, c.leader())\n\t})\n\n\t// We don't know of a server that matches that name so the stepdown fails.\n\tt.Run(\"UnknownPreferred\", func(t *testing.T) {\n\t\tbody, err := json.Marshal(JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tPreferred: \"i_dont_exist\",\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(JSApiLeaderStepDown, body, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar apiresp JSApiLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &apiresp))\n\t\trequire_False(t, apiresp.Success)\n\t\trequire_NotNil(t, apiresp.Error)\n\t\trequire_Equal(t, ErrorIdentifier(apiresp.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// The preferred server happens to already be the leader so the stepdown fails.\n\tt.Run(\"SamePreferred\", func(t *testing.T) {\n\t\tbody, err := json.Marshal(JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tPreferred: c.leader().Name(),\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(JSApiLeaderStepDown, body, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar apiresp ApiResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &apiresp))\n\t\trequire_NotNil(t, apiresp.Error)\n\t\trequire_Equal(t, ErrorIdentifier(apiresp.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n}\n\nfunc TestJetStreamClusterOnlyPublishAdvisoriesWhenInterest(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tsubj := \"$JS.ADVISORY.TEST\"\n\ts1 := c.servers[0]\n\ts2 := c.servers[1]\n\n\t// On the first server, see if we think the advisory will be published.\n\trequire_False(t, s1.publishAdvisory(s1.GlobalAccount(), subj, \"test\"))\n\n\t// On the second server, subscribe to the advisory subject.\n\tnc, _ := jsClientConnect(t, s2)\n\tdefer nc.Close()\n\n\t_, err := nc.Subscribe(subj, func(_ *nats.Msg) {})\n\trequire_NoError(t, err)\n\n\t// Wait for the interest to propagate to the first server.\n\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\tif !s1.GlobalAccount().sl.HasInterest(subj) {\n\t\t\treturn fmt.Errorf(\"expected interest in %q, not yet found\", subj)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// On the first server, try and publish the advisory again. THis time\n\t// it should succeed.\n\trequire_True(t, s1.publishAdvisory(s1.GlobalAccount(), subj, \"test\"))\n}\n\nfunc TestJetStreamClusterRoutedAPIRecoverPerformance(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomNonLeader())\n\tdefer nc.Close()\n\n\t// We only run 16 JetStream API workers.\n\tmp := runtime.GOMAXPROCS(0)\n\tif mp > 16 {\n\t\tmp = 16\n\t}\n\n\tleader := c.leader()\n\tljs := leader.js.Load()\n\n\t// Take the JS lock, which allows the JS API queue to build up.\n\tljs.mu.Lock()\n\tdefer ljs.mu.Unlock()\n\n\tcount := JSDefaultRequestQueueLimit - 1\n\tch := make(chan *nats.Msg, count)\n\n\tinbox := nc.NewRespInbox()\n\t_, err := nc.ChanSubscribe(inbox, ch)\n\trequire_NoError(t, err)\n\n\t// To ensure a fair starting line, we need to submit as many tasks as\n\t// there are JS workers whilst holding the JS lock. This will ensure that\n\t// each JS API worker is properly wedged.\n\tmsg := &nats.Msg{\n\t\tSubject: fmt.Sprintf(JSApiConsumerInfoT, \"Doesnt\", \"Exist\"),\n\t\tReply: \"no_one_here\",\n\t}\n\tfor i := 0; i < mp; i++ {\n\t\trequire_NoError(t, nc.PublishMsg(msg))\n\t}\n\n\t// Then we want to submit a fixed number of tasks, big enough to fill\n\t// the queue, so that we can measure them.\n\tmsg = &nats.Msg{\n\t\tSubject: fmt.Sprintf(JSApiConsumerInfoT, \"Doesnt\", \"Exist\"),\n\t\tReply: inbox,\n\t}\n\tfor i := 0; i < count; i++ {\n\t\trequire_NoError(t, nc.PublishMsg(msg))\n\t}\n\tcheckFor(t, 5*time.Second, 25*time.Millisecond, func() error {\n\t\tif queued := leader.jsAPIRoutedInfoReqs.len(); queued != count {\n\t\t\treturn fmt.Errorf(\"expected %d queued requests, got %d\", count, queued)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now we're going to release the lock and start timing. The workers\n\t// will now race to clear the queues and we'll wait to see how long\n\t// it takes for them all to respond.\n\tstart := time.Now()\n\tljs.mu.Unlock()\n\tfor i := 0; i < count; i++ {\n\t\t<-ch\n\t}\n\tljs.mu.Lock()\n\tt.Logf(\"Took %s to clear %d items\", time.Since(start), count)\n}\n\nfunc TestJetStreamClusterMessageTTLWhenSourcing(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"Origin\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t\tReplicas: 3,\n\t})\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TTLEnabled\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"Origin\"},\n\t\t},\n\t\tAllowMsgTTL: true,\n\t\tReplicas: 3,\n\t})\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TTLDisabled\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"Origin\"},\n\t\t},\n\t\tAllowMsgTTL: false,\n\t\tReplicas: 3,\n\t})\n\n\thdr := nats.Header{}\n\thdr.Add(JSMessageTTL, \"1s\")\n\n\t_, err := js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: hdr,\n\t})\n\trequire_NoError(t, err)\n\n\tfor _, stream := range []string{\"TTLEnabled\", \"TTLDisabled\"} {\n\t\tt.Run(stream, func(t *testing.T) {\n\t\t\tsc, err := js.PullSubscribe(\"test\", \"consumer\", nats.BindStream(stream))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmsgs, err := sc.Fetch(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 1)\n\t\t\trequire_Equal(t, msgs[0].Header.Get(JSMessageTTL), \"1s\")\n\n\t\t\ttime.Sleep(time.Second)\n\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\trequire_NoError(t, err)\n\t\t\tif stream == \"TTLDisabled\" {\n\t\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterMessageTTLWhenMirroring(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"Origin\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t\tReplicas: 3,\n\t})\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TTLEnabled\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{\n\t\t\tName: \"Origin\",\n\t\t},\n\t\tAllowMsgTTL: true,\n\t\tReplicas: 3,\n\t})\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TTLDisabled\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{\n\t\t\tName: \"Origin\",\n\t\t},\n\t\tAllowMsgTTL: false,\n\t\tReplicas: 3,\n\t})\n\n\thdr := nats.Header{}\n\thdr.Add(JSMessageTTL, \"1s\")\n\n\t_, err := js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: hdr,\n\t})\n\trequire_NoError(t, err)\n\n\tfor _, stream := range []string{\"TTLEnabled\", \"TTLDisabled\"} {\n\t\tt.Run(stream, func(t *testing.T) {\n\t\t\tsc, err := js.PullSubscribe(\"test\", \"consumer\", nats.BindStream(stream))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmsgs, err := sc.Fetch(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 1)\n\t\t\trequire_Equal(t, msgs[0].Header.Get(JSMessageTTL), \"1s\")\n\n\t\t\ttime.Sleep(time.Second)\n\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\trequire_NoError(t, err)\n\t\t\tif stream == \"TTLDisabled\" {\n\t\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterMessageTTLDisabled(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tReplicas: 3,\n\t})\n\n\tmsg := &nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: nats.Header{},\n\t}\n\n\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\t_, err := js.PublishMsg(msg)\n\trequire_Error(t, err)\n\n\t// In clustered mode we should have caught this before generating the\n\t// proposal, therefore the CLFS should not have been bumped by this.\n\tfor _, s := range c.servers {\n\t\tstream, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, stream.getCLFS(), 0)\n\t}\n}\n\nfunc TestJetStreamClusterCreateStreamPerf(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor i := 0; i < 10; i++ {\n\t\t// Check that creating a replicated asset doesn't take too long.\n\t\tstart := time.Now()\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"TEST-%d\", i),\n\t\t\tRetention: nats.LimitsPolicy,\n\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d\", i)},\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tif elapsed := time.Since(start); elapsed > 150*time.Millisecond {\n\t\t\tt.Skipf(\"Took too long to create a R3 stream: %v\", elapsed)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterTTLAndDedupe(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Add(JSMsgId, \"msgId\")\n\tm.Header.Add(JSMessageTTL, \"10s\")\n\t_, err = js.PublishMsg(m)\n\trequire_Error(t, err, NewJSMessageTTLDisabledError())\n\n\t// Retrying should get TTL disabled still, not any other error.\n\t_, err = js.PublishMsg(m)\n\trequire_Error(t, err, NewJSMessageTTLDisabledError())\n\n\t// Send without TTL should succeed.\n\tm.Header.Del(JSMessageTTL)\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterInvalidTTLAndDedupe(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tAllowMsgTTL: true,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Header.Add(JSMsgId, \"msgId\")\n\tm.Header.Add(JSMessageTTL, \"invalid!\")\n\t_, err = js.PublishMsg(m)\n\trequire_Error(t, err, NewJSMessageTTLInvalidError())\n\n\t// Retry with a negative TTL.\n\tm.Header.Set(JSMessageTTL, \"-10s\")\n\t_, err = js.PublishMsg(m)\n\trequire_Error(t, err, NewJSMessageTTLInvalidError())\n\n\t// Retrying with valid TTL should be successful, and not be marked as duplicate.\n\tm.Header.Set(JSMessageTTL, \"10s\")\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterServerPeerRemovePeersDrift(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R4S\", 4)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.LimitsPolicy,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tvar acc *Account\n\tvar mset *stream\n\n\t// Wait for 3 of the 4 servers to have created the stream.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tcount := 0\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err = s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err = acc.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\tif o == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcount++\n\t\t}\n\t\tif count != 3 {\n\t\t\treturn fmt.Errorf(\"expected 3 streams/consumers, got: %d\", count)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\n\t// Get a random server that:\n\t// - is not stream leader\n\t// - is not meta leader (peer-removing the meta leader has other issues)\n\t// - already hosts the stream so a peer-remove results in changing the stream peer set\n\tvar rs *Server\n\tfor _, s := range c.servers {\n\t\tacc, err = s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\t_, err = acc.lookupStream(\"TEST\")\n\t\tif s == sl || s.isMetaLeader.Load() || err != nil {\n\t\t\tcontinue\n\t\t}\n\t\trs = s\n\t\tbreak\n\t}\n\tif rs == nil {\n\t\tt.Fatal(\"No server found that's not either stream or meta leader.\")\n\t}\n\trs.Shutdown()\n\n\t// Peer-remove the selected server so the stream moves to the remaining empty server.\n\tsc, err := nats.Connect(sl.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\treq := &JSApiMetaServerRemoveRequest{Server: rs.Name()}\n\tjsreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\t_, err = sc.Request(JSApiRemoveServer, jsreq, time.Second)\n\trequire_NoError(t, err)\n\n\t// Eventually there should again be a R3 stream and everyone should agree on the peers.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tcount := 0\n\t\tvar streamPeers []string\n\t\tvar consumerPeers []string\n\t\tfor _, s := range c.servers {\n\t\t\tif s == rs {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tacc, err = s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err = acc.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\tif o == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer not found on %s\", s.Name())\n\t\t\t}\n\t\t\tmrn := mset.raftNode().(*raft)\n\t\t\tmrn.RLock()\n\t\t\tstreamPeerNames := mrn.peerNames()\n\t\t\tmrn.RUnlock()\n\n\t\t\torn := o.raftNode().(*raft)\n\t\t\torn.RLock()\n\t\t\tconsumerPeerNames := orn.peerNames()\n\t\t\torn.RUnlock()\n\n\t\t\tslices.Sort(streamPeerNames)\n\t\t\tslices.Sort(consumerPeerNames)\n\t\t\tif count == 0 {\n\t\t\t\tstreamPeers = streamPeerNames\n\t\t\t\tconsumerPeers = consumerPeerNames\n\t\t\t} else if !slices.Equal(streamPeers, streamPeerNames) {\n\t\t\t\trsid := rs.NodeName()\n\t\t\t\tcontainsOld := slices.Contains(streamPeers, rsid) || slices.Contains(streamPeerNames, rsid)\n\t\t\t\treturn fmt.Errorf(\"no equal stream peers, expected: %v, got: %v, contains old peer (%s): %v\", streamPeers, streamPeerNames, rsid, containsOld)\n\t\t\t} else if !slices.Equal(consumerPeers, consumerPeerNames) {\n\t\t\t\trsid := rs.NodeName()\n\t\t\t\tcontainsOld := slices.Contains(consumerPeers, rsid) || slices.Contains(consumerPeerNames, rsid)\n\t\t\t\treturn fmt.Errorf(\"no equal consumer peers, expected: %v, got: %v, contains old peer (%s): %v\", consumerPeers, consumerPeerNames, rsid, containsOld)\n\t\t\t}\n\t\t\tcount++\n\t\t}\n\t\tif count != 3 {\n\t\t\treturn fmt.Errorf(\"expected 3 servers hosting stream/consumer, got: %d\", count)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamStreamTagPlacement(t *testing.T) {\n\tc := createJetStreamClusterWithTemplateAndModHook(t, jsClusterTempl, \"R3S\", 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [%s]\", conf, serverName)\n\t\t})\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tfor i, c := range []nats.StreamConfig{\n\t\t{Replicas: 1, Placement: &nats.Placement{Tags: []string{\"!S-1\", \"!S-2\", \"!S-3\"}}}, // exclude all servers.\n\t\t{Replicas: 2, Placement: &nats.Placement{Tags: []string{\"!S-1\", \"!S-2\"}}}, // exclude two servers.\n\t\t{Replicas: 3, Placement: &nats.Placement{Tags: []string{\"!S-2\"}}}, // not enough server.\n\t} {\n\t\tc.Name = fmt.Sprintf(\"TEST%d\", i)\n\t\tc.Subjects = []string{c.Name}\n\t\t_, err := js.AddStream(&c)\n\t\trequire_Error(t, err)\n\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"tags excluded\", \"S-2\")\n\t}\n\n\t// Test adding excluded tags to an existing stream.\n\tcfg := &nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tcfg.Placement = &nats.Placement{Tags: []string{\"!S-1\"}}\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"tags excluded\", \"S-1\")\n\n\t// Test changing replicas to 2 and then add the excluded tag.\n\tcfg.Replicas = 2\n\tcfg.Placement = nil\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\tvar leaderName string\n\t// Wait until we have two replicas\n\tcheckFor(t, 6*time.Second, 1*time.Second, func() error {\n\t\ts, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif s.Cluster.Leader == \"\" {\n\t\t\treturn fmt.Errorf(\"no leader yet\")\n\t\t}\n\n\t\tif len(s.Cluster.Replicas) != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 replica, got %d\", len(s.Cluster.Replicas))\n\t\t}\n\t\tleaderName = s.Cluster.Leader\n\t\treturn nil\n\t})\n\n\t// Now that we only have two servers, we can exclude one.\n\tcfg.Replicas = 2\n\tcfg.Placement = &nats.Placement{Tags: []string{\"!\" + leaderName}}\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Check that the stream grab the correct servers.\n\t// This can take a bit as we need to first add the new replica and later remove the old one.\n\tcheckFor(t, 6*time.Second, 1*time.Second, func() error {\n\t\ts, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif len(s.Cluster.Replicas) != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 replica, got %d\", len(s.Cluster.Replicas))\n\t\t}\n\n\t\tif s.Cluster.Leader == leaderName {\n\t\t\treturn fmt.Errorf(\"expected leader to be different than %q\", leaderName)\n\t\t}\n\n\t\tif s.Cluster.Replicas[0].Name == leaderName {\n\t\t\treturn fmt.Errorf(\"expected replica to be different than %q\", leaderName)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterObserverNotElectedMetaLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\n\tgetMeta := func(s *Server) *raft {\n\t\tif js := s.getJetStream(); js != nil {\n\t\t\tif mg := js.getMetaGroup(); mg != nil {\n\t\t\t\treturn mg.(*raft)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tsetToObserverAndStepDown := func(s *Server) {\n\t\tif meta := getMeta(s); meta != nil {\n\t\t\tmeta.setObserver(true, extExtended)\n\t\t\tmeta.StepDown()\n\t\t}\n\t}\n\n\tvar wg sync.WaitGroup\n\n\tfor range 10 {\n\t\t// Pick what will be the new leader since we are going to switch\n\t\t// the 2 other servers to observer mode and make them step down.\n\t\tnewLeader := c.randomNonLeader()\n\t\tleader := c.leader()\n\n\t\tvar other *Server\n\t\tfor _, s := range c.servers {\n\t\t\tif s != newLeader && s != leader {\n\t\t\t\tother = s\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t// Add some random delay before changing state and stepping down.\n\t\t\ttime.Sleep(time.Duration(rand.Intn(25)) * time.Millisecond)\n\t\t\tsetToObserverAndStepDown(other)\n\t\t}()\n\t\tsetToObserverAndStepDown(leader)\n\t\twg.Wait()\n\n\t\t// Wait for the newLeader to really be elected.\n\t\tcheckFor(t, 10*time.Second, 50*time.Millisecond, func() error {\n\t\t\tif !newLeader.JetStreamIsLeader() {\n\t\t\t\treturn fmt.Errorf(\"Server %q is still not leader\", newLeader)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Change the observer back to false.\n\t\tfor _, s := range []*Server{leader, other} {\n\t\t\tif meta := getMeta(s); meta != nil {\n\t\t\t\tmeta.SetObserver(false)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterParallelCreateRaftGroup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tml := c.leader()\n\tsjs := ml.getJetStream()\n\tsa := sjs.streamAssignment(globalAccountName, \"TEST\")\n\trequire_NotNil(t, sa)\n\tacc, err := ml.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\trg, storage := sa.Group, sa.Group.Storage\n\trn := mset.raftNode().(*raft)\n\t// Do first half of Stop.\n\trequire_NotEqual(t, rn.state.Swap(int32(Closed)), int32(Closed))\n\n\tvar wg sync.WaitGroup\n\tvar finish sync.WaitGroup\n\twg.Add(2)\n\tfinish.Add(2)\n\n\tvar mu sync.Mutex\n\tvar nodes []RaftNode\n\n\t// Call createRaftGroup in parallel.\n\tfor i := 0; i < 2; i++ {\n\t\tgo func() {\n\t\t\twg.Done()\n\t\t\tdefer finish.Done()\n\t\t\tif n, rerr := sjs.createRaftGroup(acc.GetName(), rg, false, storage, pprofLabels{}); rerr == nil {\n\t\t\t\tmu.Lock()\n\t\t\t\tnodes = append(nodes, n)\n\t\t\t\tmu.Unlock()\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Wait for both goroutines, and allow some time for both to have entered createRaftGroup.\n\twg.Wait()\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Do second half of Stop while goroutines are in createRaftGroup.\n\trn.leaderState.Store(false)\n\trn.leaderSince.Store(nil)\n\tclose(rn.quit)\n\n\t// Wait for node and goroutines to stop.\n\trn.WaitForStop()\n\tfinish.Wait()\n\n\t// Should only create one new node instance.\n\trequire_Len(t, len(nodes), 2)\n\trequire_Equal(t, nodes[0], nodes[1])\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersMinimumTTL(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, replicas := range []int{1, 3} {\n\t\t\tt.Run(fmt.Sprintf(\"%s/R%d\", storageType, replicas), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tStorage: storageType,\n\t\t\t\t\tSubjectDeleteMarkerTTL: 3 * time.Second,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, \"1s\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// After the TTL expires it should still be there, because SubjectDeleteMarkerTTL is the minimum.\n\t\t\t\ttime.Sleep(1500 * time.Millisecond)\n\t\t\t\trsm, err := js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, rsm.Header.Get(JSMessageTTL), \"3\") // 3s from the SDM TTL.\n\n\t\t\t\t// Need to wait for the subject delete marker to be placed.\n\t\t\t\ttime.Sleep(2 * time.Second)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t\t// Expect a subject delete marker based on per-message TTL.\n\t\t\t\tsm, err := js.GetMsg(\"TEST\", 2)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\n\t\t\t\t// Since we have a subject delete marker now with a higher TTL, if this\n\t\t\t\t// message's lower TTL would be respected then we would not have a new\n\t\t\t\t// subject delete marker when this message expires.\n\t\t\t\tpa, err := js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pa.Sequence, 3)\n\n\t\t\t\t// After some time the first subject delete marker should be gone, and the new one should be placed.\n\t\t\t\ttime.Sleep(3500 * time.Millisecond)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 2)\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t\tsm, err = js.GetMsg(\"TEST\", 4)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersMinimumTTLExceptionMaxMsgsPer(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, replicas := range []int{1, 3} {\n\t\t\tt.Run(fmt.Sprintf(\"%s/R%d\", storageType, replicas), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tStorage: storageType,\n\t\t\t\t\tSubjectDeleteMarkerTTL: time.Hour,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t\tMaxMsgsPer: 1,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, \"1s\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// After the TTL expires the message should be gone, even though SubjectDeleteMarkerTTL is the minimum\n\t\t\t\t// normally. This works because MaxMsgsPer=1 is set.\n\t\t\t\ttime.Sleep(1500 * time.Millisecond)\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t\t// Expect a subject delete marker based on per-message TTL.\n\t\t\t\tsm, err := js.GetMsg(\"TEST\", 2)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\n\t\t\t\t// Since we have a subject delete marker now with a higher TTL, if this\n\t\t\t\t// message's lower TTL would be respected then we would not have a new\n\t\t\t\t// subject delete marker when this message expires. But with MaxMsgsPer=1\n\t\t\t\t// this doesn't apply, because the subject delete marker gets removed.\n\t\t\t\tpa, err := js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, pa.Sequence, 3)\n\n\t\t\t\t// After some time the first subject delete marker should be gone, and the new one should be placed.\n\t\t\t\ttime.Sleep(1500 * time.Millisecond)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 2)\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t\tsm, err = js.GetMsg(\"TEST\", 4)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterSubjectDeleteMarkersNoMsgTTLSet(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tfor _, replicas := range []int{1, 3} {\n\t\t\tt.Run(fmt.Sprintf(\"%s/R%d\", storageType, replicas), func(t *testing.T) {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\tdefer c.shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tStorage: storageType,\n\t\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t\t\tAllowMsgTTL: true,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t// Message should not expire based on SubjectDeleteMarkerTTL if no TTL is set.\n\t\t\t\ttime.Sleep(1500 * time.Millisecond)\n\t\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterSDMMaxAgeOnRecover(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tReplicas: 3,\n\t\tMaxAge: time.Second,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Wait for all servers to have applied the published message.\n\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\trs := c.randomServer()\n\tfor _, s := range c.servers {\n\t\ts.Shutdown()\n\t}\n\n\t// MaxAge would expire the message on recovery, ensure that's not done because we're clustered with SDM.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\trs = c.restartServer(rs)\n\tacc, err := rs.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\t_, err = mset.getMsg(1)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamClusterSDMMaxAgeRemoveMsgProposal(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: time.Second,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Hour,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for all servers to have applied the published message.\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode()\n\t\t\trequire_NoError(t, rn.PauseApply())\n\n\t\t\t// Check we can get the message.\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Let MaxAge expire the message, but this replica is paused so should not remove on its own.\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Resume and check all replicas agree on the state.\n\t\t\trn.ResumeApply()\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\t// Now the message should be gone.\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMMaxAgeRemoveMsgProposalLimitRetries(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: time.Second,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Hour,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for all servers to have applied the published message.\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode().(*raft)\n\n\t\t\t// Force the leader to not be able to know its remove proposals were accepted.\n\t\t\trn.Lock()\n\t\t\tpindex, pcommit := rn.pindex, rn.commit\n\t\t\trn.commit = 1_000 * pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Let MaxAge expire the message.\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\n\t\t\t// Spam a bunch of expiry calls, shouldn't spam message delete proposals.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\t\tfs.expireMsgs()\n\t\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\t\tms.expireMsgs()\n\t\t\t\t}\n\t\t\t\t// Spread them out, so that they can't be grouped into just one Raft proposal.\n\t\t\t\ttime.Sleep(2 * time.Millisecond)\n\t\t\t}\n\n\t\t\t// Put the original commit back, so they can now be committed/applied.\n\t\t\trn.Lock()\n\t\t\trn.commit = pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Ensures we can check the stream leader's full log has been applied.\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Expect two entries: one initial removal, one publish.\n\t\t\tnindex, _, _ := rn.Progress()\n\t\t\trequire_Equal(t, nindex, pindex+2)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMTTLRemoveMsgProposal(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjectDeleteMarkerTTL: 2 * time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(JSMessageTTL, \"2s\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for all servers to have applied the published message.\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\trs := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := rs.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode()\n\t\t\trequire_NoError(t, rn.PauseApply())\n\n\t\t\t// Check we can get the message.\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Let TTL expire the message, but this replica is paused so should not remove on its own.\n\t\t\ttime.Sleep(2500 * time.Millisecond)\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Resume and check all replicas agree on the state.\n\t\t\trn.ResumeApply()\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\t// Now the message should be gone.\n\t\t\t_, err = mset.getMsg(1)\n\t\t\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMInflightTTL(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjectDeleteMarkerTTL: 2 * time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, fmt.Sprintf(\"%dms\", 2000+500*i))\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode().(*raft)\n\n\t\t\t// Force the leader to not be able to know its remove proposals were accepted.\n\t\t\trn.Lock()\n\t\t\tpcommit := rn.commit\n\t\t\trn.commit = 1_000 * pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Check all messages are there.\n\t\t\tfor seq := uint64(1); seq <= 2; seq++ {\n\t\t\t\t_, err = js.GetMsg(\"TEST\", seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Let TTL expire the messages while we're not applying the removals.\n\t\t\ttime.Sleep(3 * time.Second)\n\n\t\t\t// Put the original commit back, so they can now be committed/applied.\n\t\t\trn.Lock()\n\t\t\trn.commit = pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Ensures we can check the stream leader's full log has been applied.\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Check all messages are removed.\n\t\t\tfor seq := uint64(1); seq <= 2; seq++ {\n\t\t\t\t_, err = js.GetMsg(\"TEST\", seq)\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\t\t\t}\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMTTLAndMaxMsgsPer(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjectDeleteMarkerTTL: 2 * time.Second,\n\t\t\t\t// Used for the second part of this test, checks proper accounting\n\t\t\t\t// with removals through MaxMsgsPer and TTL.\n\t\t\t\tMaxMsgsPer: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckNoPending := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\t\tfs.mu.RLock()\n\t\t\t\t\tpending := fs.sdm.pending\n\t\t\t\t\tfs.mu.RUnlock()\n\t\t\t\t\tif len(pending) > 0 {\n\t\t\t\t\t\tt.Fatalf(\"Expected no pending messages, but got: %v\", pending)\n\t\t\t\t\t}\n\t\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\t\tms.mu.RLock()\n\t\t\t\t\tpending := ms.sdm.pending\n\t\t\t\t\tms.mu.RUnlock()\n\t\t\t\t\tif len(pending) > 0 {\n\t\t\t\t\t\tt.Fatalf(\"Expected no pending messages, but got: %v\", pending)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Publish initial batch of messages with TTLs, but wait\n\t\t\t// in-between so they expire as we publish.\n\t\t\tfor i := 0; i < 6; i++ {\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, \"2s\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\t}\n\n\t\t\t// Wait for all messages to be expired.\n\t\t\ttime.Sleep(2 * time.Second)\n\n\t\t\t// Check all messages are removed. Must not have subject delete markers be placed in-between.\n\t\t\tfor seq := uint64(1); seq <= 6; seq++ {\n\t\t\t\tsm, err := js.GetMsg(\"TEST\", seq)\n\t\t\t\tif sm != nil && sm.Header.Get(JSMarkerReason) != _EMPTY_ {\n\t\t\t\t\tt.Fatalf(\"Got unexpected subject delete marker at sequence %d\", seq)\n\t\t\t\t}\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\t\t\t}\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 7)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\n\t\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 7))\n\t\t\tcheckNoPending()\n\n\t\t\t// Publish another batch of messages with TTLs, but only\n\t\t\t// wait initially and have MaxMsgsPer kick in during it.\n\t\t\tfor i := 0; i < 6; i++ {\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, \"2s\")\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif i < 3 {\n\t\t\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Wait for all messages to be expired.\n\t\t\ttime.Sleep(2500 * time.Millisecond)\n\n\t\t\t// Check all messages are removed. Must not have subject delete markers be placed in-between.\n\t\t\tfor seq := uint64(8); seq <= 13; seq++ {\n\t\t\t\tsm, err = js.GetMsg(\"TEST\", seq)\n\t\t\t\tif sm != nil && sm.Header.Get(JSMarkerReason) != _EMPTY_ {\n\t\t\t\t\tt.Fatalf(\"Got unexpected subject delete marker at sequence %d\", seq)\n\t\t\t\t}\n\t\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\t\t\t}\n\n\t\t\tcheckNoPending()\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err = js.GetMsg(\"TEST\", 14)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMMsgTTLReverseExpiry(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tSubjectDeleteMarkerTTL: 2 * time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\t\tm.Header.Set(JSMessageTTL, fmt.Sprintf(\"%ds\", 3-i))\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Wait for all servers to have applied the published message.\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode().(*raft)\n\n\t\t\t// Force the leader to not be able to know its remove proposals were accepted.\n\t\t\trn.Lock()\n\t\t\tpindex, pcommit := rn.pindex, rn.commit\n\t\t\trn.commit = 1_000 * pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Let TTL expire the messages, but the leader does not know the proposals will go through.\n\t\t\ttime.Sleep(3500 * time.Millisecond)\n\n\t\t\t// Put the original commit back, so they can now be committed/applied.\n\t\t\trn.Lock()\n\t\t\trn.commit = pcommit\n\t\t\trn.Unlock()\n\n\t\t\t// Eventually the message should be gone.\n\t\t\t// Ensures we can check the stream leader's full log has been applied.\n\t\t\t_, err = js.Publish(\"bar\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Expect a subject delete marker.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)\n\n\t\t\t// Expect three entries, one leader change, the subject delete marker rollup, and one published message.\n\t\t\tnindex, _, _ := rn.Progress()\n\t\t\trequire_Equal(t, nindex, pindex+3)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMResetLast(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: 2 * time.Second,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for all servers to have applied the published message.\n\t\t\tcheckFor(t, 500*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t\t})\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\n\t\t\tpauseApplies := func(pause bool) {\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tif s == sl {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trn := mset.raftNode().(*raft)\n\t\t\t\t\tif pause {\n\t\t\t\t\t\trequire_NoError(t, rn.PauseApply())\n\t\t\t\t\t} else {\n\t\t\t\t\t\trn.ResumeApply()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Pause applies on all replicas.\n\t\t\tpauseApplies(true)\n\n\t\t\t// Publish message after pause, so the replicas cache the first message as needing SDM.\n\t\t\t// Then once applies are resumed they should not place SDM.\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(JSMessageTTL, \"never\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trn := mset.raftNode().(*raft)\n\n\t\t\t// Force the leader to not be able to make proposals.\n\t\t\trn.Lock()\n\t\t\trn.werr = errors.New(\"block proposals\")\n\t\t\trn.Unlock()\n\n\t\t\t// Let MaxAge expire the message, but the replicas are paused.\n\t\t\ttime.Sleep(2500 * time.Millisecond)\n\n\t\t\t// Resume applies on all replicas.\n\t\t\tpauseApplies(false)\n\n\t\t\t// Shutdown and wait for new stream leader.\n\t\t\tsl.Shutdown()\n\t\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t\t\t// Wait for the new leader to have removed the first message and added the second message.\n\t\t\tcheckFor(t, 2500*time.Millisecond, 200*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != 1 || si.State.FirstSeq != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"incorrect state: %v\", si.State)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t_, err = js.GetMsg(\"TEST\", 1)\n\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\t\t\t// There should be no subject delete marker. Only the \"never\" TTL message.\n\t\t\tsm, err := js.GetMsg(\"TEST\", 2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.Subject, \"foo\")\n\t\t\trequire_Equal(t, sm.Header.Get(JSMessageTTL), \"never\")\n\t\t\trequire_Equal(t, sm.Header.Get(JSMarkerReason), _EMPTY_)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterSDMMaxAgeProposeExpiryShortRetry(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc := clientConnectToServer(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxAge: 2 * time.Second,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Hour,\n\t\t\t}\n\n\t\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\t\tacc, err := sl.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Manually store a message with a timestamp far in the past.\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\trequire_NoError(t, fs.StoreRawMsg(\"foo\", nil, nil, 1, 1, 0, false))\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\trequire_NoError(t, ms.StoreRawMsg(\"foo\", nil, nil, 1, 1, 0, false))\n\t\t\t}\n\t\t\t// We'll also need to manually adjust the upper-layer last sequence.\n\t\t\tmset.mu.Lock()\n\t\t\tmset.lseq = 1\n\t\t\tmset.mu.Unlock()\n\n\t\t\tcfg.MaxAge = time.Hour\n\t\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\tfs.mu.Lock()\n\t\t\t\tfs.resetAgeChk(0)\n\t\t\t\tfs.mu.Unlock()\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\tms.mu.Lock()\n\t\t\t\tms.resetAgeChk(0)\n\t\t\t\tms.mu.Unlock()\n\t\t\t}\n\n\t\t\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t_, err = mset.getMsg(1)\n\t\t\t\tif !errors.Is(err, ErrStoreMsgNotFound) {\n\t\t\t\t\treturn fmt.Errorf(\"expected msg not found error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterInvalidR1Config(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R1TEST\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.servers[0])\n\tdefer nc.Close()\n\n\tnc2, js2 := jsClientConnect(t, c.servers[2])\n\tdefer nc2.Close()\n\n\tcreateStreams := func(t *testing.T, js nats.JetStreamContext, n, replicas int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tsname := fmt.Sprintf(\"S:%d\", i)\n\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\tName: sname,\n\t\t\t\tMaxMsgsPerSubject: 5,\n\t\t\t\tReplicas: replicas,\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"A.%d.>\", i)},\n\t\t\t})\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tjs.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\t\tName: sname,\n\t\t\t\tDurable: sname,\n\t\t\t\tFilterSubject: \">\",\n\t\t\t})\n\t\t\tjs.Publish(fmt.Sprintf(\"A.%d.foo\", i), []byte(\"one\"))\n\t\t}\n\t}\n\n\t// Create 5 streams in parallel with different configs, then\n\t// check whether one of them is in an undefined state.\n\ttotalStreams := 5\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tcreateStreams(t, js, totalStreams, 1)\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tcreateStreams(t, js2, totalStreams, 2)\n\t}()\n\twg.Wait()\n\n\tfor i := 0; i < totalStreams; i++ {\n\t\tci, err := js.StreamInfo(fmt.Sprintf(\"S:%d\", i))\n\t\trequire_NoError(t, err)\n\n\t\t// Make sure that consumer scale up when peers are missing responds with error.\n\t\tif ci.Config.Replicas == 2 {\n\t\t\t// Starting with a single replica should still be valid.\n\t\t\tjs.AddConsumer(ci.Config.Name, &nats.ConsumerConfig{\n\t\t\t\tName: \"test\",\n\t\t\t\tReplicas: 1,\n\t\t\t})\n\t\t\t_, err = js.UpdateConsumer(ci.Config.Name, &nats.ConsumerConfig{\n\t\t\t\tName: \"test\",\n\t\t\t\tReplicas: 2,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\trequire_Equal(t, err.Error(), \"nats: consumer config replica count exceeds parent stream\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamClusterMultiLeaderR3Config(t *testing.T) {\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {\n\t\t\tstore_dir: '%s',\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\tserver_tags: [\"test\"]\n\t\tsystem_account: sys\n\t\tno_auth_user: js\n\t\taccounts {\n\t\t\tsys { users = [ { user: sys, pass: sys } ] }\n\t\t\tjs {\n\t\t\t\tjetstream = enabled\n\t\t\t\tusers = [ { user: js, pass: js } ]\n\t\t }\n\t\t}`\n\tc := createJetStreamClusterWithTemplate(t, conf, \"R3TEST\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.servers[0])\n\tdefer nc.Close()\n\n\tnc2, js2 := jsClientConnect(t, c.servers[2])\n\tdefer nc2.Close()\n\n\tcreateStreams := func(t *testing.T, js nats.JetStreamContext, n, replicas int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tsname := fmt.Sprintf(\"S:%d\", i)\n\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\tName: sname,\n\t\t\t\tMaxMsgsPerSubject: 5,\n\t\t\t\tReplicas: replicas,\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"A.%d.>\", i)},\n\t\t\t})\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tjs.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\t\tName: sname,\n\t\t\t\tDurable: sname,\n\t\t\t\tFilterSubject: \">\",\n\t\t\t})\n\t\t\tjs.Publish(fmt.Sprintf(\"A.%d.foo\", i), []byte(\"one\"))\n\t\t}\n\t}\n\n\t// Create streams in parallel with different configs, then\n\t// check whether one of them is in an undefined state.\n\ttotalStreams := 5\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tcreateStreams(t, js, totalStreams, 3)\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tcreateStreams(t, js2, totalStreams, 1)\n\t}()\n\twg.Wait()\n\n\tcheckMultiLeader := func(accountName, streamName string) error {\n\t\tleaders := make(map[string]bool)\n\t\tfor _, srv := range c.servers {\n\t\t\tjsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, acc := range jsz.AccountDetails {\n\t\t\t\tif acc.Name == accountName {\n\t\t\t\t\tfor _, stream := range acc.Streams {\n\t\t\t\t\t\tif stream.Name == streamName {\n\t\t\t\t\t\t\tleaders[stream.Cluster.Leader] = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(leaders) > 1 {\n\t\t\treturn fmt.Errorf(\"There are multiple leaders on %s stream: %+v\", streamName, leaders)\n\t\t}\n\t\treturn nil\n\t}\n\n\tvar invalidStream string\n\tfor i := 0; i < totalStreams; i++ {\n\t\tci, err := js.StreamInfo(fmt.Sprintf(\"S:%d\", i))\n\t\trequire_NoError(t, err)\n\t\tif ci.Config.Replicas == 1 {\n\t\t\tif ci.Cluster != nil {\n\t\t\t\tpeers := ci.Cluster.Replicas\n\t\t\t\tif len(peers) > 1 {\n\t\t\t\t\tinvalidStream = ci.Config.Name\n\t\t\t\t\tt.Errorf(\"Unexpected stream config drift, 1 replica expected but found %v peers\", len(peers))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif len(invalidStream) > 0 {\n\t\t_, err := js.StreamInfo(invalidStream)\n\t\trequire_NoError(t, err)\n\n\t\t// Restart server where first client is connected and almost all R1 replicas landed.\n\t\tsrv := c.servers[0]\n\t\tsrv.Shutdown()\n\t\tsrv.WaitForShutdown()\n\t\ttime.Sleep(2 * time.Second)\n\t\tc.restartServer(srv)\n\t\tc.waitOnClusterReady()\n\t\tcheckFor(t, 30*time.Second, 200*time.Millisecond, func() error {\n\t\t\treturn checkMultiLeader(\"js\", invalidStream)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterAccountMaxConnectionsReconnect(t *testing.T) {\n\tconf := `\n listen: 127.0.0.1:-1\n http: -1\n server_name: %s\n jetstream: {\n store_dir: '%s',\n }\n cluster {\n name: %s\n listen: 127.0.0.1:%d\n routes = [%s]\n }\n server_tags: [\"test\"]\n system_account: sys\n no_auth_user: js\n accounts {\n sys { users = [ { user: sys, pass: sys } ] }\n js {\n jetstream = enabled\n users = [ { user: js, pass: js } ]\n limits {\n max_connections: 5\n }\n }\n }\n `\n\tc := createJetStreamClusterWithTemplate(t, conf, \"R3CONNECT\", 3)\n\tdefer c.shutdown()\n\tvar conns []*nats.Conn\n\n\tdisconnects := make([]chan error, 0)\n\tfor i := 1; i <= 5; i++ {\n\t\tdisconnectCh := make(chan error)\n\t\tc, _ := jsClientConnect(t, c.servers[0], nats.UserInfo(\"js\", \"js\"), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {\n\t\t\tdisconnectCh <- err\n\t\t}))\n\t\tdefer c.Close()\n\t\tconns = append(conns, c)\n\t\tdisconnects = append(disconnects, disconnectCh)\n\t\t// Small delay to ensure distinct start times.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\ts := c.servers[0]\n\tacc, err := s.lookupAccount(\"js\")\n\trequire_NoError(t, err)\n\n\tcheckCounts := func(expectedNumConnections, expectedJSClients, expectedTotalClients int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tacc.mu.RLock()\n\t\t\tclients := acc.getClientsLocked()\n\t\t\tnumConnections := acc.NumConnections()\n\t\t\tjsClients := int(acc.sysclients)\n\t\t\ttotalClients := len(clients)\n\t\t\tacc.mu.RUnlock()\n\n\t\t\tif numConnections != expectedNumConnections {\n\t\t\t\treturn fmt.Errorf(\"Expected %d connections got %d\", expectedNumConnections, numConnections)\n\t\t\t}\n\t\t\tif jsClients != expectedJSClients {\n\t\t\t\treturn fmt.Errorf(\"Expected %d js clients got %d\", expectedJSClients, jsClients)\n\t\t\t}\n\t\t\tif totalClients != expectedTotalClients {\n\t\t\t\treturn fmt.Errorf(\"Expected %d total clients got %d\", expectedTotalClients, totalClients)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckCounts(5, 0, 5)\n\n\tnc := conns[0]\n\tjs, _ := nc.JetStream()\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"foo:%d\", i),\n\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d\", i)},\n\t\t\tReplicas: 3, // Must be R3 to stop flaking due to stream placement\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(fmt.Sprintf(\"foo.%d\", i), []byte(\"hello\"), nats.AckWait(5*time.Second))\n\t\trequire_NoError(t, err)\n\t}\n\n\tcheckCounts(5, 20, 25)\n\n\tcheckFor(t, 30*time.Second, 200*time.Millisecond, func() error {\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err := js.Publish(fmt.Sprintf(\"foo.%d\", i), []byte(\"hello\"), nats.AckWait(5*time.Second))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Force account update to trigger connection limit enforcement.\n\taccClaims := jwt.NewAccountClaims(acc.Name)\n\taccClaims.Limits.Conn = 1\n\taccClaims.Limits.MemoryStorage = -1\n\taccClaims.Limits.DiskStorage = -1\n\taccClaims.Limits.Streams = -1\n\taccClaims.Limits.Consumer = -1\n\n\t// Update server, before this would have disconnected JS internal clients with\n\t// 'JETSTREAM - maximum account active connections exceeded'.\n\ts.UpdateAccountClaims(acc, accClaims)\n\n\t// Allow some time for enforcement.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tacc, err = s.lookupAccount(\"js\")\n\trequire_NoError(t, err)\n\n\t// JETSTREAM internal clients should still linger after reducing connections.\n\tcheckCounts(5, 20, 20)\n\n\t// Wait for disconnections from the most recent client.\n\tdisconnectCh := disconnects[2]\n\tselect {\n\tcase <-disconnectCh:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected newest connection to disconnect!\")\n\t}\n\n\tcheckFor(t, 30*time.Second, 200*time.Millisecond, func() error {\n\t\tactiveConnections := 0\n\t\tfor _, conn := range conns {\n\t\t\tif !conn.IsClosed() {\n\t\t\t\tactiveConnections++\n\t\t\t}\n\t\t}\n\t\tif activeConnections < 5 {\n\t\t\treturn fmt.Errorf(\"Unexpected number of connections: %d\", activeConnections)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Force account update to trigger connection limit enforcement.\n\taccClaims = jwt.NewAccountClaims(acc.Name)\n\taccClaims.Limits.Conn = 10\n\taccClaims.Limits.MemoryStorage = -1\n\taccClaims.Limits.DiskStorage = -1\n\taccClaims.Limits.Streams = -1\n\taccClaims.Limits.Consumer = -1\n\n\t// Update all servers then confirm that internal JS clients should work\n\t// and clients have reconnected.\n\tfor _, s := range c.servers {\n\t\tacc, err := s.lookupAccount(\"js\")\n\t\trequire_NoError(t, err)\n\t\ts.UpdateAccountClaims(acc, accClaims)\n\t}\n\tcheckFor(t, 30*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, nc := range conns {\n\t\t\tjs, _ := nc.JetStream()\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tstream := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\t_, err := js.Publish(stream, []byte(\"hello\"), nats.AckWait(5*time.Second))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamClusterMetaCompactThreshold(t *testing.T) {\n\tfor _, thresh := range []uint64{0, 5, 10} {\n\t\tt.Run(fmt.Sprintf(\"%d\", thresh), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R1TEST\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\tfor _, s := range c.servers {\n\t\t\t\ts.optsMu.Lock()\n\t\t\t\ts.opts.JetStreamMetaCompact = thresh\n\t\t\t\ts.optsMu.Unlock()\n\t\t\t}\n\n\t\t\tnc, _ := jsClientConnect(t, c.servers[0])\n\t\t\tdefer nc.Close()\n\n\t\t\tleader := c.leader()\n\t\t\t_, cc := leader.getJetStreamCluster()\n\t\t\trg := cc.meta.(*raft)\n\n\t\t\t// We will get nowhere near math.MaxInt, as we will hit the\n\t\t\t// compaction threshold and return early, but keeps \"i\" moving up.\n\t\t\tfor i := range math.MaxInt {\n\t\t\t\trg.RLock()\n\t\t\t\tpapplied := rg.papplied\n\t\t\t\trg.RUnlock()\n\n\t\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"test_%d\", i),\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"test.%d\", i)},\n\t\t\t\t\tStorage: MemoryStorage,\n\t\t\t\t})\n\n\t\t\t\t// Kicking the leader change channel is the easiest way to\n\t\t\t\t// trick monitorCluster() into calling doSnapshot().\n\t\t\t\tentries, _ := cc.meta.Size()\n\t\t\t\tcc.meta.(*raft).leadc <- true\n\n\t\t\t\t// Should we have compacted on this iteration?\n\t\t\t\tif entries > thresh {\n\t\t\t\t\tcheckFor(t, time.Second, 5*time.Millisecond, func() error {\n\t\t\t\t\t\trg.RLock()\n\t\t\t\t\t\tnpapplied := rg.papplied\n\t\t\t\t\t\trg.RUnlock()\n\t\t\t\t\t\tif npapplied <= papplied {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"haven't snapshotted yet (%d <= %d)\", npapplied, papplied)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t\tentries, _ = cc.meta.Size()\n\t\t\t\t\trequire_Equal(t, entries, 0)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamClusterMetaCompactSizeThreshold(t *testing.T) {\n\tfor _, othresh := range []any{int64(1), \"4K\", \"32K\", \"1M\"} {\n\t\tt.Run(fmt.Sprintf(\"%v\", othresh), func(t *testing.T) {\n\t\t\tit, err := getStorageSize(othresh)\n\t\t\trequire_NoError(t, err)\n\t\t\tthresh := uint64(it)\n\n\t\t\tc := createJetStreamClusterExplicit(t, \"R1TEST\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\tfor _, s := range c.servers {\n\t\t\t\ts.optsMu.Lock()\n\t\t\t\ts.opts.JetStreamMetaCompactSize = thresh\n\t\t\t\ts.optsMu.Unlock()\n\t\t\t}\n\n\t\t\tnc, _ := jsClientConnect(t, c.servers[0])\n\t\t\tdefer nc.Close()\n\n\t\t\tleader := c.leader()\n\t\t\t_, cc := leader.getJetStreamCluster()\n\t\t\trg := cc.meta.(*raft)\n\n\t\t\t// We will get nowhere near math.MaxInt, as we will hit the\n\t\t\t// compaction threshold and return early, but keeps \"i\" moving up.\n\t\t\tfor i := range math.MaxInt {\n\t\t\t\trg.RLock()\n\t\t\t\tpapplied := rg.papplied\n\t\t\t\trg.RUnlock()\n\n\t\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"test_%d\", i),\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"test.%d\", i)},\n\t\t\t\t\tStorage: MemoryStorage,\n\t\t\t\t})\n\n\t\t\t\t// Kicking the leader change channel is the easiest way to\n\t\t\t\t// trick monitorCluster() into calling doSnapshot().\n\t\t\t\t_, size := cc.meta.Size()\n\t\t\t\tcc.meta.(*raft).leadc <- true\n\n\t\t\t\t// Should we have compacted on this iteration?\n\t\t\t\tif size > thresh {\n\t\t\t\t\tcheckFor(t, time.Second, 5*time.Millisecond, func() error {\n\t\t\t\t\t\trg.RLock()\n\t\t\t\t\t\tnpapplied := rg.papplied\n\t\t\t\t\t\trg.RUnlock()\n\t\t\t\t\t\tif npapplied <= papplied {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"haven't snapshotted yet (%d <= %d)\", npapplied, papplied)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t\t_, size = cc.meta.Size()\n\t\t\t\t\trequire_Equal(t, size, 0)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Test that setStoreState treats ErrStoreOldUpdate as a non-error.\n// This can happen during processClusterCreateConsumer when a consumer\n// assignment carries stale state (e.g. from a stream restore or meta\n// snapshot), but the consumer was already recovered from disk with\n// newer state. Without the fix, the error would propagate and cause\n// the consumer to be stopped and its Raft node deleted.\nfunc TestJetStreamClusterConsumerSetStoreStateOldUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish messages and ack them to advance the consumer state.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\t\trequire_NoError(t, err)\n\t}\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 10)\n\tfor _, m := range msgs {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for all consumers on all servers to be caught up to ack floor 10.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\ta, err := s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tms, err := a.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tco := ms.lookupConsumer(\"CONSUMER\")\n\t\t\tif co == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t\tinfo := co.info()\n\t\t\tif info.AckFloor.Consumer != 10 {\n\t\t\t\treturn fmt.Errorf(\"expected ack floor 10, got %d\", info.AckFloor.Consumer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Pick a follower server to test on. This simulates the scenario where\n\t// a consumer already exists with advanced state and then\n\t// processClusterCreateConsumer is called with stale state.\n\trs := c.randomNonConsumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, rs)\n\n\t// Grab the consumer's Raft node before the stale state is applied.\n\tacc, err := rs.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\to.mu.RLock()\n\traftNode := o.node\n\to.mu.RUnlock()\n\trequire_NotNil(t, raftNode)\n\n\t// Build a consumer assignment with stale state, as would happen during\n\t// a stream restore where the consumer info returned has older state.\n\trsjs := rs.getJetStream()\n\trsjs.mu.RLock()\n\tca := rsjs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\trsjs.mu.RUnlock()\n\trequire_NotNil(t, ca)\n\n\tstaleState := &ConsumerState{\n\t\tDelivered: SequencePair{Consumer: 5, Stream: 5},\n\t\tAckFloor: SequencePair{Consumer: 5, Stream: 5},\n\t}\n\n\t// Check that consumer is running and healthy.\n\tcurrent := mset.lookupConsumer(\"CONSUMER\")\n\tif current == nil {\n\t\tt.Errorf(\"Expected consumer to be present on restart\")\n\t}\n\tcinfo := current.info()\n\trequire_Equal(t, cinfo.AckFloor.Consumer, 10)\n\trequire_Equal(t, cinfo.AckFloor.Stream, 10)\n\n\t// Call processClusterCreateConsumer with the stale state, simulating\n\t// what happens during stream restore or meta snapshot recovery.\n\t// Without the fix, this would delete the consumer's Raft node and stop it.\n\trsjs.processClusterCreateConsumer(ca, ca, staleState, true)\n\n\t// Verify the consumer still exists on this server.\n\to = mset.lookupConsumer(\"CONSUMER\")\n\tif o == nil {\n\t\tt.Fatal(\"Expected consumer to still be present after consumer create with stale data\")\n\t}\n\n\t// Verify the consumer's Raft node was NOT deleted.\n\to.mu.RLock()\n\tnodeAfter := o.node\n\to.mu.RUnlock()\n\trequire_NotNil(t, nodeAfter)\n\trequire_Equal(t, raftNode, nodeAfter)\n\n\t// Verify the consumer retained its correct (newer) state, not the stale state.\n\tinfo := o.info()\n\trequire_Equal(t, info.AckFloor.Consumer, 10)\n\trequire_Equal(t, info.AckFloor.Stream, 10)\n\n\t// Verify the consumer is still fully functional by publishing and consuming more.\n\tfor i := 0; i < 5; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"after-stale\"))\n\t\trequire_NoError(t, err)\n\t}\n\tmsgs, err = sub.Fetch(5)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 5)\n\tfor _, m := range msgs {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestJetStreamClusterConsumerSetStoreStateOldUpdateRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tReplicas: 3,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish messages and ack them to advance the consumer state.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\t\trequire_NoError(t, err)\n\t}\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 10)\n\tfor _, m := range msgs {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for all consumers on all servers to be caught up to ack floor 10.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\ta, err := s.lookupAccount(globalAccountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tms, err := a.lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tco := ms.lookupConsumer(\"CONSUMER\")\n\t\t\tif co == nil {\n\t\t\t\treturn errors.New(\"consumer not found\")\n\t\t\t}\n\t\t\tinfo := co.info()\n\t\t\tif info.AckFloor.Consumer != 10 {\n\t\t\t\treturn fmt.Errorf(\"expected ack floor 10, got %d\", info.AckFloor.Consumer)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Pick a server that is not the meta leader. We will shut it down and\n\t// then propose a stale consumer assignment entry to meta while it is\n\t// down. When it restarts and catches up on the meta Raft, it will\n\t// replay the entry with the stale State.\n\tml := c.leader()\n\trequire_NotNil(t, ml)\n\tvar rs *Server\n\tfor _, s := range c.servers {\n\t\tif s != ml {\n\t\t\trs = s\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_NotNil(t, rs)\n\n\trs.Shutdown()\n\trs.WaitForShutdown()\n\n\t// Reconnect to a remaining server.\n\tnc.Close()\n\tnc, js = jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t// Propose a consumer assignment with stale State to the meta Raft.\n\t// This simulates what happens during a stream scale-down reassignment\n\t// where the consumer info returned has older state than what the\n\t// consumer already has on disk.\n\tmljs := ml.getJetStream()\n\tmljs.mu.RLock()\n\tcc := mljs.cluster\n\tca := mljs.consumerAssignment(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, ca)\n\n\tcca := ca.copyGroup()\n\tcca.State = &ConsumerState{\n\t\tDelivered: SequencePair{Consumer: 5, Stream: 5},\n\t\tAckFloor: SequencePair{Consumer: 5, Stream: 5},\n\t}\n\tconsumerAdd := encodeAddConsumerAssignment(cca)\n\tmljs.mu.RUnlock()\n\trequire_NoError(t, cc.meta.Propose(consumerAdd))\n\n\t// Wait for the meta leader to apply the entry.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Restart the server. During meta Raft catchup, it will replay the\n\t// consumer assignment entry that has State set to ack floor 5, but the\n\t// consumer's store on disk already has ack floor 10. Without the fix,\n\t// setStoreState returns ErrStoreOldUpdate which triggers\n\t// rg.node.Delete() and o.stop(), destroying the consumer.\n\trs = c.restartServer(rs)\n\tc.waitOnServerCurrent(rs)\n\n\t// Verify the consumer still exists on the restarted server and has\n\t// the correct (newer) state, not the stale state.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\ta, err := rs.lookupAccount(globalAccountName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tms, err := a.lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tco := ms.lookupConsumer(\"CONSUMER\")\n\t\tif co == nil {\n\t\t\treturn errors.New(\"consumer not found after restart - Raft node was likely deleted\")\n\t\t}\n\t\tinfo := co.info()\n\t\tif info.AckFloor.Consumer != 10 {\n\t\t\treturn fmt.Errorf(\"expected ack floor 10, got %d\", info.AckFloor.Consumer)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Verify the consumer is still fully functional by publishing and consuming more.\n\tfor i := 0; i < 5; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"after-restart\"))\n\t\trequire_NoError(t, err)\n\t}\n\tsub, err = js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tmsgs, err = sub.Fetch(5)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 5)\n\tfor _, m := range msgs {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestJetStreamClusterSourcingIntoDiscardNewPerSubject(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"A\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tDuplicates: 100 * time.Millisecond,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tsConfig := StreamConfig{\n\t\tName: \"B\",\n\t\tStorage: MemoryStorage,\n\t\tSources: []*StreamSource{{Name: \"A\"}},\n\t\tMaxMsgsPer: 1,\n\t\tDiscard: DiscardNew,\n\t\tDiscardNewPer: true,\n\t\tDuplicates: 10 * time.Second,\n\t\tReplicas: 3,\n\t}\n\n\treq, err := json.Marshal(&sConfig)\n\trequire_NoError(t, err)\n\n\tr, err := nc.Request(\"$JS.API.STREAM.CREATE.B\", req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(r.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_Equal(t, resp.Error, nil)\n\n\t_, err = js.Publish(\"foo.1\", []byte(\"1\"))\n\trequire_NoError(t, err)\n\n\t// This second publish to the same subject should not be sourced\n\t// due to discard new per subject with MaxMsgsPer=1.\n\t_, err = js.Publish(\"foo.1\", []byte(\"2\"))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 message, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check the message content.\n\tmsgp, err := js.GetMsg(\"B\", uint64(1))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.1\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n\n\t// Purge stream B so sourcing can continue past the per-subject limit.\n\trequire_NoError(t, js.PurgeStream(\"B\"))\n\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 message, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// After purge, the second message should now be sourced.\n\tmsgp, err = js.GetMsg(\"B\", uint64(2))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.1\")\n\trequire_Equal(t, string(msgp.Data), \"2\")\n\n\t// Test duplicate message ID handling: publish with explicit Nats-Msg-Id.\n\tmsg := nats.NewMsg(\"foo.2\")\n\tmsg.Data = []byte(\"1\")\n\tmsg.Header.Set(\"Nats-Msg-Id\", \"dup1\")\n\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// Must be able to source the message after the duplicate.\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tmsgp, err = js.GetMsg(\"B\", uint64(3))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.2\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n\n\t// Wait for the dedup window on stream A (100ms) to expire.\n\ttime.Sleep(200 * time.Millisecond)\n\n\t// Publish a message with the same Nats-Msg-Id but to a different subject.\n\t// Stream A's dedup window has expired, so A will accept it.\n\t// Stream B has a 10s dedup window, so it should detect this as a duplicate and skip it.\n\tmsg = nats.NewMsg(\"foo.3\")\n\tmsg.Data = []byte(\"1\")\n\tmsg.Header.Set(\"Nats-Msg-Id\", \"dup1\")\n\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// The duplicate should be skipped by B; message count should stay at 2.\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Sourcing must continue processing after the skipped duplicate.\n\t_, err = js.Publish(\"foo.4\", []byte(\"1\"))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 3 {\n\t\t\treturn fmt.Errorf(\"expected 3 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tmsgp, err = js.GetMsg(\"B\", uint64(4))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.4\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n}\n" }, { "path": "server/jetstream_cluster_long_test.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// This test file is skipped by default to avoid accidentally running (e.g. `go test ./server`)\n//go:build !skip_js_tests && include_js_long_tests\n\npackage server\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\n// TestLongKVPutWithServerRestarts overwrites values in a replicated KV bucket for a fixed amount of time.\n// Also restarts a random server at fixed interval.\n// The test fails if updates fail for a continuous interval of time, or if a server fails to restart and catch up.\nfunc TestLongKVPutWithServerRestarts(t *testing.T) {\n\t// RNG Seed\n\tconst Seed = 123456\n\t// Number of keys in bucket\n\tconst NumKeys = 1000\n\t// Size of (random) values\n\tconst ValueSize = 1024\n\t// Test duration\n\tconst Duration = 3 * time.Minute\n\t// If no updates successful for this interval, then fail the test\n\tconst MaxRetry = 5 * time.Second\n\t// Minimum time between put operations\n\tconst UpdatesInterval = 1 * time.Millisecond\n\t// Time between progress reports to console\n\tconst ProgressInterval = 10 * time.Second\n\t// Time between server restarts\n\tconst ServerRestartInterval = 5 * time.Second\n\n\ttype Parameters struct {\n\t\tnumServers int\n\t\treplicas int\n\t\tstorage nats.StorageType\n\t}\n\n\ttest := func(t *testing.T, p Parameters) {\n\t\trng := rand.New(rand.NewSource(Seed))\n\n\t\t// Create cluster\n\t\tclusterName := fmt.Sprintf(\"C_%d-%s\", p.numServers, p.storage)\n\t\tcluster := createJetStreamClusterExplicit(t, clusterName, p.numServers)\n\t\tdefer cluster.shutdown()\n\n\t\t// Connect to a random server but client will discover others too.\n\t\tnc, js := jsClientConnect(t, cluster.randomServer())\n\t\tdefer nc.Close()\n\n\t\t// Create bucket\n\t\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\t\tBucket: \"TEST\",\n\t\t\tReplicas: p.replicas,\n\t\t\tStorage: p.storage,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Initialize list of keys\n\t\tkeys := make([]string, NumKeys)\n\t\tfor i := 0; i < NumKeys; i++ {\n\t\t\tkeys[i] = fmt.Sprintf(\"key-%d\", i)\n\t\t}\n\n\t\t// Initialize keys in bucket with an empty value\n\t\tfor _, key := range keys {\n\t\t\t_, err := kv.Put(key, []byte{})\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Track statistics\n\t\tvar stats = struct {\n\t\t\tstart time.Time\n\t\t\tlastSuccessfulUpdate time.Time\n\t\t\tupdateOk uint64\n\t\t\tupdateErr uint64\n\t\t\trestarts uint64\n\t\t\trestartsMap map[string]int\n\t\t}{\n\t\t\tstart: time.Now(),\n\t\t\tlastSuccessfulUpdate: time.Now(),\n\t\t\trestartsMap: make(map[string]int, p.numServers),\n\t\t}\n\t\tfor _, server := range cluster.servers {\n\t\t\tstats.restartsMap[server.Name()] = 0\n\t\t}\n\n\t\t// Print statistics\n\t\tprintProgress := func() {\n\n\t\t\tt.Logf(\n\t\t\t\t\"[%s] %d updates %d errors, %d server restarts (%v)\",\n\t\t\t\ttime.Since(stats.start).Round(time.Second),\n\t\t\t\tstats.updateOk,\n\t\t\t\tstats.updateErr,\n\t\t\t\tstats.restarts,\n\t\t\t\tstats.restartsMap,\n\t\t\t)\n\t\t}\n\n\t\t// Print update on completion\n\t\tdefer printProgress()\n\n\t\t// Pick a random key and update it with a random value\n\t\tvalueBuffer := make([]byte, ValueSize)\n\t\tupdateRandomKey := func() error {\n\t\t\tkey := keys[rand.Intn(NumKeys)]\n\t\t\t_, err := rng.Read(valueBuffer)\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = kv.Put(key, valueBuffer)\n\t\t\treturn err\n\t\t}\n\n\t\t// Set up timers and tickers\n\t\tendTestTimer := time.After(Duration)\n\t\tnextUpdateTicker := time.NewTicker(UpdatesInterval)\n\t\tprogressTicker := time.NewTicker(ProgressInterval)\n\t\trestartServerTicker := time.NewTicker(ServerRestartInterval)\n\n\trunLoop:\n\t\tfor {\n\t\t\tselect {\n\n\t\t\tcase <-endTestTimer:\n\t\t\t\tbreak runLoop\n\n\t\t\tcase <-nextUpdateTicker.C:\n\t\t\t\terr := updateRandomKey()\n\t\t\t\tif err == nil {\n\t\t\t\t\tstats.updateOk++\n\t\t\t\t\tstats.lastSuccessfulUpdate = time.Now()\n\t\t\t\t} else {\n\t\t\t\t\tstats.updateErr++\n\t\t\t\t\tif time.Since(stats.lastSuccessfulUpdate) > MaxRetry {\n\t\t\t\t\t\tt.Fatalf(\"Could not successfully update for over %s\", MaxRetry)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\tcase <-restartServerTicker.C:\n\t\t\t\trandomServer := cluster.servers[rng.Intn(len(cluster.servers))]\n\t\t\t\trandomServer.Shutdown()\n\t\t\t\trandomServer.WaitForShutdown()\n\t\t\t\trestartedServer := cluster.restartServer(randomServer)\n\t\t\t\tcluster.waitOnClusterReady()\n\t\t\t\tcluster.waitOnServerHealthz(restartedServer)\n\t\t\t\tcluster.waitOnAllCurrent()\n\t\t\t\tstats.restarts++\n\t\t\t\tstats.restartsMap[randomServer.Name()]++\n\n\t\t\tcase <-progressTicker.C:\n\t\t\t\tprintProgress()\n\t\t\t}\n\t\t}\n\t}\n\n\ttestCases := []Parameters{\n\t\t{numServers: 5, replicas: 3, storage: nats.MemoryStorage},\n\t\t{numServers: 5, replicas: 3, storage: nats.FileStorage},\n\t}\n\n\tfor _, testCase := range testCases {\n\t\tname := fmt.Sprintf(\"N:%d,R:%d,%s\", testCase.numServers, testCase.replicas, testCase.storage)\n\t\tt.Run(\n\t\t\tname,\n\t\t\tfunc(t *testing.T) { test(t, testCase) },\n\t\t)\n\t}\n}\n\n// This is a RaftChainOfBlocks test that randomly starts and stops nodes to exercise recovery and snapshots.\nfunc TestLongNRGChainOfBlocks(t *testing.T) {\n\tconst (\n\t\tClusterSize = 3\n\t\tGroupSize = 3\n\t\tConvergenceTimeout = 30 * time.Second\n\t\tDuration = 10 * time.Minute\n\t\tPrintStateInterval = 3 * time.Second\n\t)\n\n\t// Create cluster\n\tc := createJetStreamClusterExplicit(t, \"Test\", ClusterSize)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"ChainOfBlocks\", GroupSize, newRaftChainStateMachine)\n\trg.waitOnLeader()\n\n\t// Available operations\n\ttype TestOperation string\n\tconst (\n\t\tStopOne TestOperation = \"Stop one active node\"\n\t\tStopAll = \"Stop all active nodes\"\n\t\tRestartOne = \"Restart one stopped node\"\n\t\tRestartAll = \"Restart all stopped nodes\"\n\t\tSnapshot = \"Snapshot one active node\"\n\t\tPropose = \"Propose a value via one active node\"\n\t\tProposeLeader = \"Propose a value via leader\"\n\t\tPause = \"Let things run undisturbed for a while\"\n\t\tCheck = \"Wait for nodes to converge\"\n\t)\n\n\t// Weighted distribution of operations, one is randomly chosen from this vector in each iteration\n\topsWeighted := []TestOperation{\n\t\tStopOne,\n\t\tStopOne,\n\t\tStopOne,\n\t\tStopAll,\n\t\tStopAll,\n\t\tRestartOne,\n\t\tRestartOne,\n\t\tRestartOne,\n\t\tRestartAll,\n\t\tRestartAll,\n\t\tRestartAll,\n\t\tSnapshot,\n\t\tSnapshot,\n\t\tPropose,\n\t\tPropose,\n\t\tPropose,\n\t\tPropose,\n\t\tProposeLeader,\n\t\tProposeLeader,\n\t\tProposeLeader,\n\t\tProposeLeader,\n\t\tPause,\n\t\tPause,\n\t\tPause,\n\t\tPause,\n\t\tCheck,\n\t\tCheck,\n\t\tCheck,\n\t\tCheck,\n\t}\n\n\trngSeed := time.Now().UnixNano()\n\trng := rand.New(rand.NewSource(rngSeed))\n\tt.Logf(\"Seed: %d\", rngSeed)\n\n\t// Chose a node from the list (and remove it)\n\tpickRandomNode := func(nodes []stateMachine) ([]stateMachine, stateMachine) {\n\t\tif len(nodes) == 0 {\n\t\t\t// Input list is empty\n\t\t\treturn nodes, nil\n\t\t}\n\t\t// Pick random node\n\t\ti := rng.Intn(len(nodes))\n\t\tnode := nodes[i]\n\t\t// Move last element in its place\n\t\tnodes[i] = nodes[len(nodes)-1]\n\t\t// Return slice excluding last element\n\t\treturn nodes[:len(nodes)-1], node\n\t}\n\n\t// Create summary status string for all replicas\n\tchainStatusString := func() string {\n\t\tb := strings.Builder{}\n\t\tfor _, sm := range rg {\n\t\t\tcsm := sm.(*RCOBStateMachine)\n\t\t\trunning, blocksCount, blockHash := csm.getCurrentHash()\n\t\t\tif running {\n\t\t\t\tb.WriteString(\n\t\t\t\t\tfmt.Sprintf(\n\t\t\t\t\t\t\" [%s (%s): %d blocks, hash=%s],\",\n\t\t\t\t\t\tcsm.server().Name(),\n\t\t\t\t\t\tcsm.node().ID(),\n\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\tblockHash,\n\t\t\t\t\t),\n\t\t\t\t)\n\t\t\t} else {\n\t\t\t\tb.WriteString(\n\t\t\t\t\tfmt.Sprintf(\n\t\t\t\t\t\t\" [%s (%s): STOPPED],\",\n\t\t\t\t\t\tcsm.server().Name(),\n\t\t\t\t\t\tcsm.node().ID(),\n\t\t\t\t\t),\n\t\t\t\t)\n\n\t\t\t}\n\t\t}\n\t\treturn b.String()\n\t}\n\n\t// Track the highest number of blocks applied by any of the replicas\n\thighestBlocksCount := uint64(0)\n\n\t// Track active and stopped nodes\n\tactiveNodes := make([]stateMachine, 0, GroupSize)\n\tstoppedNodes := make([]stateMachine, 0, GroupSize)\n\n\t// Initially all nodes are active\n\tactiveNodes = append(activeNodes, rg...)\n\n\tdefer func() {\n\t\tt.Logf(\"Final state: %s\", chainStatusString())\n\t}()\n\n\tprintStateTicker := time.NewTicker(PrintStateInterval)\n\ttestTimer := time.NewTimer(Duration)\n\tstart := time.Now()\n\titeration := 0\n\tstopCooldown := 0\n\n\tfor {\n\n\t\titeration++\n\t\tselect {\n\t\tcase <-printStateTicker.C:\n\t\t\tt.Logf(\n\t\t\t\t\"[%s] State: %s\",\n\t\t\t\ttime.Since(start).Round(time.Second),\n\t\t\t\tchainStatusString(),\n\t\t\t)\n\t\tcase <-testTimer.C:\n\t\t\t// Test completed\n\t\t\treturn\n\t\tdefault:\n\t\t\t// Continue\n\t\t}\n\n\t\t// Choose a random operation to perform in this iteration\n\t\tnextOperation := opsWeighted[rng.Intn(len(opsWeighted))]\n\n\t\t// If we're about to stop one or all nodes, do some sanity checks to ensure we don't\n\t\t// spam stops which would constantly interrupt the chance of making progress.\n\t\tif nextOperation == StopOne || nextOperation == StopAll {\n\t\t\tif stopCooldown > 0 {\n\t\t\t\tnextOperation = Propose\n\t\t\t} else {\n\t\t\t\tstopCooldown = 50\n\t\t\t}\n\t\t}\n\t\tif stopCooldown > 0 {\n\t\t\tstopCooldown--\n\t\t}\n\n\t\tif RCOBOptions.verbose {\n\t\t\tt.Logf(\"Iteration %d: %s\", iteration, nextOperation)\n\t\t}\n\n\t\tswitch nextOperation {\n\n\t\tcase StopOne:\n\t\t\t// Stop an active node (if any are left active)\n\t\t\tvar n stateMachine\n\t\t\tactiveNodes, n = pickRandomNode(activeNodes)\n\t\t\tif n != nil {\n\t\t\t\tn.stop()\n\t\t\t\tstoppedNodes = append(stoppedNodes, n)\n\t\t\t}\n\n\t\tcase StopAll:\n\t\t\t// Stop all active nodes (if any are active)\n\t\t\tfor _, node := range activeNodes {\n\t\t\t\tnode.stop()\n\t\t\t}\n\t\t\tstoppedNodes = append(stoppedNodes, activeNodes...)\n\t\t\tactiveNodes = make([]stateMachine, 0, GroupSize)\n\n\t\tcase RestartOne:\n\t\t\t// Restart a stopped node (if any are stopped)\n\t\t\tvar n stateMachine\n\t\t\tstoppedNodes, n = pickRandomNode(stoppedNodes)\n\t\t\tif n != nil {\n\t\t\t\tn.restart()\n\t\t\t\tactiveNodes = append(activeNodes, n)\n\t\t\t}\n\n\t\tcase RestartAll:\n\t\t\t// Restart all stopped nodes (if any)\n\t\t\tfor _, node := range stoppedNodes {\n\t\t\t\tnode.restart()\n\t\t\t}\n\t\t\tactiveNodes = append(activeNodes, stoppedNodes...)\n\t\t\tstoppedNodes = make([]stateMachine, 0, GroupSize)\n\n\t\tcase Snapshot:\n\t\t\t// Choose a random active node and tell it to create a snapshot\n\t\t\tif len(activeNodes) > 0 {\n\t\t\t\tn := activeNodes[rng.Intn(len(activeNodes))]\n\t\t\t\tn.(*RCOBStateMachine).createSnapshot()\n\t\t\t}\n\n\t\tcase Propose:\n\t\t\t// Make an active node propose the next block (if any nodes are active)\n\t\t\tif len(activeNodes) > 0 {\n\t\t\t\tn := activeNodes[rng.Intn(len(activeNodes))]\n\t\t\t\tn.(*RCOBStateMachine).proposeBlock()\n\t\t\t}\n\n\t\tcase ProposeLeader:\n\t\t\t// Make the leader propose the next block (if a leader is active)\n\t\t\tleader := rg.leader()\n\t\t\tif leader != nil {\n\t\t\t\tleader.(*RCOBStateMachine).proposeBlock()\n\t\t\t}\n\n\t\tcase Pause:\n\t\t\t// Noop, let things run undisturbed for a little bit\n\t\t\ttime.Sleep(time.Duration(rng.Intn(250)) * time.Millisecond)\n\n\t\tcase Check:\n\t\t\t// Restart any stopped node\n\t\t\tfor _, node := range stoppedNodes {\n\t\t\t\tnode.restart()\n\t\t\t}\n\t\t\tactiveNodes = append(activeNodes, stoppedNodes...)\n\t\t\tstoppedNodes = make([]stateMachine, 0, GroupSize)\n\n\t\t\t// Ensure all nodes (eventually) converge\n\t\t\tcheckFor(t, ConvergenceTimeout, 250*time.Millisecond,\n\t\t\t\tfunc() error {\n\t\t\t\t\treferenceNode := rg[0]\n\t\t\t\t\t// Save block count and hash of first node as reference\n\t\t\t\t\t_, referenceBlocksCount, referenceHash := referenceNode.(*RCOBStateMachine).getCurrentHash()\n\n\t\t\t\t\t// Compare each node against reference\n\t\t\t\t\tfor _, n := range rg {\n\t\t\t\t\t\tsm := n.(*RCOBStateMachine)\n\t\t\t\t\t\trunning, blocksCount, blockHash := sm.getCurrentHash()\n\t\t\t\t\t\tif !running {\n\t\t\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\t\t\"node not running: %s (%s)\",\n\t\t\t\t\t\t\t\tsm.server().Name(),\n\t\t\t\t\t\t\t\tsm.node().ID(),\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Track the highest block delivered by any node\n\t\t\t\t\t\tif blocksCount > highestBlocksCount {\n\t\t\t\t\t\t\tif RCOBOptions.verbose {\n\t\t\t\t\t\t\t\tt.Logf(\n\t\t\t\t\t\t\t\t\t\"New highest blocks count: %d (%s (%s))\",\n\t\t\t\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\t\t\t\tsm.s.Name(),\n\t\t\t\t\t\t\t\t\tsm.n.ID(),\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\thighestBlocksCount = blocksCount\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Each replica must match the reference node\n\n\t\t\t\t\t\tif blocksCount != referenceBlocksCount {\n\t\t\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\t\t\"different number of blocks %d (%s (%s) vs. %d (%s (%s))\",\n\t\t\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\t\t\tsm.server().Name(),\n\t\t\t\t\t\t\t\tsm.node().ID(),\n\t\t\t\t\t\t\t\treferenceBlocksCount,\n\t\t\t\t\t\t\t\treferenceNode.server().Name(),\n\t\t\t\t\t\t\t\treferenceNode.node().ID(),\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t} else if blockHash != referenceHash {\n\t\t\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\t\t\"different hash after %d blocks %s (%s (%s) vs. %s (%s (%s))\",\n\t\t\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\t\t\tblockHash,\n\t\t\t\t\t\t\t\tsm.server().Name(),\n\t\t\t\t\t\t\t\tsm.node().ID(),\n\t\t\t\t\t\t\t\treferenceHash,\n\t\t\t\t\t\t\t\treferenceNode.server().Name(),\n\t\t\t\t\t\t\t\treferenceNode.node().ID(),\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Verify consistency check was against the highest block known\n\t\t\t\t\tif referenceBlocksCount < highestBlocksCount {\n\t\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\t\"nodes converged below highest known block count: %d: %s\",\n\t\t\t\t\t\t\thighestBlocksCount,\n\t\t\t\t\t\t\tchainStatusString(),\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\n\t\t\t\t\t// All nodes reached the same state, check passed\n\t\t\t\t\treturn nil\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\t}\n}\n\nfunc TestLongClusterWorkQueueMessagesNotSkipped(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\titerations := 500_000\n\tstream := \"s1\"\n\tsubjf := \"subj.>\"\n\tconsumers := map[string]string{\n\t\t\"c1\": \"subj.c1\",\n\t\t\"c2\": \"subj.c2.*\",\n\t\t\"c3\": \"subj.c3\",\n\t}\n\n\tsig := make(chan *nats.Msg, 900)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: stream,\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{subjf},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tDuplicates: time.Minute * 2,\n\t\tReplicas: 3,\n\t\tMaxAge: time.Hour,\n\t})\n\trequire_NoError(t, err)\n\n\tfor name, subjf := range consumers {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err = js.AddConsumer(stream, &nats.ConsumerConfig{\n\t\t\tName: name,\n\t\t\tFilterSubject: subjf,\n\t\t\tReplicas: 3,\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tDeliverPolicy: nats.DeliverAllPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tps, err := js.PullSubscribe(subjf, \"\", nats.Bind(stream, name))\n\t\trequire_NoError(t, err)\n\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tmsgs, err := ps.FetchBatch(300)\n\t\t\t\tif errors.Is(err, nats.ErrTimeout) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrSubscriptionClosed) {\n\t\t\t\t\treturn // ... for when the test finishes\n\t\t\t\t}\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tfor msg := range msgs.Messages() {\n\t\t\t\t\tgo func() {\n\t\t\t\t\t\ttime.Sleep(time.Millisecond * time.Duration(rand.Int31n(100)))\n\t\t\t\t\t\trequire_NoError(t, msg.Ack())\n\t\t\t\t\t\tsig <- msg\n\t\t\t\t\t}()\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tgo func() {\n\t\thdrs := nats.Header{}\n\t\tfor i := 1; i <= iterations; i++ {\n\t\t\t// Pick a random consumer to hit this time (map iteration order is\n\t\t\t// non-deterministic, but break to do it just once).\n\t\t\tfor _, subj := range consumers {\n\t\t\t\thdrs.Set(\"Nats-Msg-Id\", fmt.Sprintf(\"msg-%d\", i))\n\t\t\t\tif strings.HasPrefix(subj, \"*\") {\n\t\t\t\t\tsubj = strings.Replace(subj, \"*\", fmt.Sprintf(\"%d\", i), 1)\n\t\t\t\t}\n\t\t\t\t_, err := js.PublishMsg(&nats.Msg{\n\t\t\t\t\tSubject: subj,\n\t\t\t\t\tHeader: hdrs,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 1; i <= iterations; i++ {\n\t\tif i%10000 == 0 {\n\t\t\tt.Logf(\"%d messages out of %d\", i, iterations)\n\t\t}\n\n\t\tselect {\n\t\tcase <-sig:\n\t\tcase <-time.After(time.Second * 2):\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\trequire_NoError(t, err)\n\t\t\tt.Logf(\"Stream info: %+v\", si.State)\n\n\t\t\tfor name := range consumers {\n\t\t\t\tci, err := js.ConsumerInfo(stream, name)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tt.Logf(\"Consumer %q info: %+v, %+v\", name, ci.AckFloor, ci.Delivered)\n\t\t\t}\n\n\t\t\tt.Fatalf(\"Didn't receive message %d\", i)\n\t\t}\n\t}\n}\n\nfunc TestLongClusterJetStreamKeyValueSync(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tfor _, s := range c.servers {\n\t\ts.optsMu.Lock()\n\t\ts.opts.LameDuckDuration = 15 * time.Second\n\t\ts.opts.LameDuckGracePeriod = -15 * time.Second\n\t\ts.optsMu.Unlock()\n\t}\n\ts := c.randomNonLeader()\n\tconnect := func(t *testing.T) (*nats.Conn, nats.JetStreamContext) {\n\t\treturn jsClientConnect(t, s)\n\t}\n\n\tconst accountName = \"$G\"\n\tconst letterBytes = \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\"\n\tcreateData := func(n int) []byte {\n\t\tb := make([]byte, n)\n\t\tfor i := range b {\n\t\t\tb[i] = letterBytes[rand.Intn(len(letterBytes))]\n\t\t}\n\t\treturn b\n\t}\n\tgetOrCreateKvStore := func(kvname string) (nats.KeyValue, error) {\n\t\t_, js := connect(t)\n\t\tkvExists := false\n\t\texistingKvnames := js.KeyValueStoreNames()\n\t\tfor existingKvname := range existingKvnames {\n\t\t\tif existingKvname == kvname {\n\t\t\t\tkvExists = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !kvExists {\n\t\t\treturn js.CreateKeyValue(&nats.KeyValueConfig{\n\t\t\t\tBucket: kvname,\n\t\t\t\tReplicas: 3,\n\t\t\t\tStorage: nats.FileStorage,\n\t\t\t})\n\t\t} else {\n\t\t\treturn js.KeyValue(kvname)\n\t\t}\n\t}\n\tabs := func(x int64) int64 {\n\t\tif x < 0 {\n\t\t\treturn -x\n\t\t}\n\t\treturn x\n\t}\n\tvar counter int64\n\tvar errorCounter int64\n\n\tcheckMsgsEqual := func(t *testing.T, accountName, streamName string) error {\n\t\t// Gather all the streams replicas and compare contents.\n\t\tmsets := make(map[*Server]*stream)\n\t\tfor _, s := range c.servers {\n\t\t\tacc, err := s.LookupAccount(accountName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmset, err := acc.lookupStream(streamName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmsets[s] = mset\n\t\t}\n\n\t\tstr := getStreamDetails(t, c, accountName, streamName)\n\t\tif str == nil {\n\t\t\treturn fmt.Errorf(\"could not get stream leader state\")\n\t\t}\n\t\tstate := str.State\n\t\tfor seq := state.FirstSeq; seq <= state.LastSeq; seq++ {\n\t\t\tvar msgId string\n\t\t\tvar smv StoreMsg\n\t\t\tfor replica, mset := range msets {\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tsm, err := mset.store.LoadMsg(seq, &smv)\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t\tif err != nil {\n\t\t\t\t\tif err == ErrStoreMsgNotFound || err == errDeletedMsg {\n\t\t\t\t\t\t// Skip these.\n\t\t\t\t\t} else {\n\t\t\t\t\t\tt.Logf(\"WRN: Error loading message (seq=%d) from stream %q on replica %q: %v\", seq, streamName, replica, err)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif msgId == _EMPTY_ {\n\t\t\t\t\tmsgId = string(sm.hdr)\n\t\t\t\t} else if msgId != string(sm.hdr) {\n\t\t\t\t\tt.Errorf(\"MsgIds do not match for seq %d on stream %q: %q vs %q\", seq, streamName, msgId, sm.hdr)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tkeyUpdater := func(ctx context.Context, cancel context.CancelFunc, kvname string, numKeys int) {\n\t\tkv, err := getOrCreateKvStore(kvname)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"[%s]:%v\", kvname, err)\n\t\t}\n\t\tfor i := 0; i < numKeys; i++ {\n\t\t\tkey := fmt.Sprintf(\"key-%d\", i)\n\t\t\tkv.Create(key, createData(160))\n\t\t}\n\t\tlastData := make(map[string][]byte)\n\t\trevisions := make(map[string]uint64)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tr := rand.Intn(numKeys)\n\t\t\tkey := fmt.Sprintf(\"key-%d\", r)\n\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\t_, err := kv.Get(key)\n\t\t\t\tif err != nil {\n\t\t\t\t\tatomic.AddInt64(&errorCounter, 1)\n\t\t\t\t\tif err == nats.ErrKeyNotFound {\n\t\t\t\t\t\tt.Logf(\"WRN: Key not found! [%s/%s] - [%s]\", kvname, key, err)\n\t\t\t\t\t\tcancel()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tk, err := kv.Get(key)\n\t\t\tif err != nil {\n\t\t\t\tatomic.AddInt64(&errorCounter, 1)\n\t\t\t} else {\n\t\t\t\tif revisions[key] != 0 && abs(int64(k.Revision())-int64(revisions[key])) < 2 {\n\t\t\t\t\tlastDataVal, ok := lastData[key]\n\t\t\t\t\tif ok && k.Revision() == revisions[key] && slices.Compare(lastDataVal, k.Value()) != 0 {\n\t\t\t\t\t\tt.Logf(\"data loss [%s/%s][rev:%d] expected:[%v] is:[%v]\", kvname, key, revisions[key], string(lastDataVal), string(k.Value()))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tnewData := createData(160)\n\t\t\t\trevisions[key], err = kv.Update(key, newData, k.Revision())\n\t\t\t\tif err != nil && err != nats.ErrTimeout {\n\t\t\t\t\tatomic.AddInt64(&errorCounter, 1)\n\t\t\t\t} else {\n\t\t\t\t\tlastData[key] = newData\n\t\t\t\t}\n\t\t\t\tatomic.AddInt64(&counter, 1)\n\t\t\t}\n\t\t}\n\t}\n\n\tstreamCount := 50\n\tkeysCount := 100\n\tstreamPrefix := \"IKV\"\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)\n\tdefer cancel()\n\n\t// The keyUpdaters will run for less time.\n\tkctx, kcancel := context.WithTimeout(context.Background(), 3*time.Minute)\n\tdefer kcancel()\n\n\tvar wg sync.WaitGroup\n\tvar streams []string\n\tfor i := 0; i < streamCount; i++ {\n\t\tstreamName := fmt.Sprintf(\"%s-%d\", streamPrefix, i)\n\t\tstreams = append(streams, \"KV_\"+streamName)\n\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\t\t\tkeyUpdater(kctx, cancel, streamName, keysCount)\n\t\t}(i)\n\t}\n\n\tdebug := false\n\tnc2, _ := jsClientConnect(t, s)\n\tif debug {\n\t\tgo func() {\n\t\t\tfor range time.NewTicker(5 * time.Second).C {\n\t\t\t\tselect {\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t\tfor _, str := range streams {\n\t\t\t\t\tleaderSrv := c.streamLeader(accountName, str)\n\t\t\t\t\tif leaderSrv == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tstreamLeader := getStreamDetails(t, c, accountName, str)\n\t\t\t\t\tif streamLeader == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tt.Logf(\"|------------------------------------------------------------------------------------------------------------------------|\")\n\t\t\t\t\tlstate := streamLeader.State\n\t\t\t\t\tt.Logf(\"| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d |\",\n\t\t\t\t\t\tstr, leaderSrv.String()+\"*\", lstate.Msgs, lstate.Bytes, lstate.NumDeleted, lstate.FirstSeq, lstate.LastSeq,\n\t\t\t\t\t)\n\t\t\t\t\tfor _, srv := range c.servers {\n\t\t\t\t\t\tif srv == leaderSrv {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tacc, err := srv.LookupAccount(accountName)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tstream, err := acc.lookupStream(str)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tt.Logf(\"Error looking up stream %s on %s replica\", str, srv)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tstate := stream.state()\n\n\t\t\t\t\t\tunsynced := lstate.Msgs != state.Msgs || lstate.Bytes != state.Bytes ||\n\t\t\t\t\t\t\tlstate.NumDeleted != state.NumDeleted || lstate.FirstSeq != state.FirstSeq || lstate.LastSeq != state.LastSeq\n\n\t\t\t\t\t\tvar result string\n\t\t\t\t\t\tif unsynced {\n\t\t\t\t\t\t\tresult = \"UNSYNCED\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\tt.Logf(\"| %-10s | %-10s | msgs:%-10d | bytes:%-10d | deleted:%-10d | first:%-10d | last:%-10d | %s\",\n\t\t\t\t\t\t\tstr, srv, state.Msgs, state.Bytes, state.NumDeleted, state.FirstSeq, state.LastSeq, result,\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tt.Logf(\"|------------------------------------------------------------------------------------------------------------------------| %v\", nc2.ConnectedUrl())\n\t\t\t}\n\t\t}()\n\t}\n\n\tcheckStreams := func(t *testing.T) {\n\t\tfor _, str := range streams {\n\t\t\tcheckFor(t, time.Minute, 500*time.Millisecond, func() error {\n\t\t\t\treturn checkState(t, c, accountName, str)\n\t\t\t})\n\t\t\tcheckFor(t, time.Minute, 500*time.Millisecond, func() error {\n\t\t\t\treturn checkMsgsEqual(t, accountName, str)\n\t\t\t})\n\t\t}\n\t}\n\nLoop:\n\tfor range time.NewTicker(30 * time.Second).C {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tbreak Loop\n\t\tdefault:\n\t\t}\n\t\trollout := func(t *testing.T) {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\t// For graceful mode\n\t\t\t\ts.lameDuckMode()\n\t\t\t\ts.WaitForShutdown()\n\t\t\t\ts = c.restartServer(s)\n\n\t\t\t\thctx, hcancel := context.WithTimeout(context.Background(), 15*time.Second)\n\t\t\t\tdefer hcancel()\n\n\t\t\tHealthz:\n\t\t\t\tfor range time.NewTicker(2 * time.Second).C {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-hctx.Done():\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\n\t\t\t\t\tstatus := s.healthz(nil)\n\t\t\t\t\tif status.StatusCode == 200 {\n\t\t\t\t\t\tbreak Healthz\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tc.waitOnClusterReady()\n\t\t\t\tcheckStreams(t)\n\t\t\t}\n\t\t}\n\t\trollout(t)\n\t\tcheckStreams(t)\n\t}\n\twg.Wait()\n\tcheckStreams(t)\n}\n\nfunc TestLongClusterCLFSOnDuplicates(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tnc2, js2 := jsClientConnect(t, c.randomServer())\n\tdefer nc2.Close()\n\n\tstreamName := \"TESTW\"\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: streamName,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tStorage: nats.FileStorage,\n\t\tMaxAge: 3 * time.Minute,\n\t\tDuplicates: 2 * time.Minute,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, streamName)\n\n\tvar wg sync.WaitGroup\n\n\t// The test will be successful if it runs for this long without dup issues.\n\tctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)\n\tdefer cancel()\n\n\tgo func() {\n\t\ttick := time.NewTicker(10 * time.Second)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\tc.streamLeader(globalAccountName, streamName).JetStreamStepdownStream(globalAccountName, streamName)\n\t\t\t}\n\t\t}\n\t}()\n\twg.Add(1)\n\n\tfor i := 0; i < 5; i++ {\n\t\tgo func(i int) {\n\t\t\tvar err error\n\t\t\tsub, err := js2.PullSubscribe(\"foo\", fmt.Sprintf(\"A:%d\", i))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\twg.Done()\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\n\t\t\t\tmsgs, err := sub.Fetch(100, nats.MaxWait(200*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}(i)\n\t\twg.Add(1)\n\t}\n\n\t// Sync producer that only does a couple of duplicates, cancel the test\n\t// if we get too many errors without responses.\n\terrCh := make(chan error, 10)\n\tgo func() {\n\t\t// Try sync publishes normally in this state and see if it times out.\n\t\tfor i := 0; ; i++ {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\n\t\t\tvar succeeded bool\n\t\t\tvar failures int\n\t\t\tfor n := 0; n < 10; n++ {\n\t\t\t\t_, err := js.Publish(\n\t\t\t\t\t\"foo\", []byte(\"test\"),\n\t\t\t\t\tnats.MsgId(fmt.Sprintf(\"sync:checking:%d\", i)),\n\t\t\t\t\tnats.RetryAttempts(30),\n\t\t\t\t\tnats.AckWait(500*time.Millisecond),\n\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\tfailures++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsucceeded = true\n\t\t\t}\n\t\t\tif !succeeded {\n\t\t\t\terrCh <- fmt.Errorf(\"Too many publishes failed with timeout: failures=%d, i=%d\", failures, i)\n\t\t\t}\n\t\t}\n\t}()\n\twg.Add(1)\n\nLoop:\n\tfor n := uint64(0); true; n++ {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tbreak Loop\n\t\tcase e := <-errCh:\n\t\t\tt.Error(e)\n\t\t\tbreak Loop\n\t\tdefault:\n\t\t}\n\t\t// Cause a lot of duplicates very fast until producer stalls.\n\t\tfor i := 0; i < 128; i++ {\n\t\t\tmsgID := nats.MsgId(fmt.Sprintf(\"id.%d.%d\", n, i))\n\t\t\tjs.PublishAsync(\"foo\", []byte(\"test\"), msgID, nats.RetryAttempts(10))\n\t\t}\n\t}\n\tcancel()\n\twg.Wait()\n}\n\nfunc TestLongClusterJetStreamRestartThenScaleStreamReplicas(t *testing.T) {\n\tt.Skip(\"This test takes too long, need to make shorter\")\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tnc2, producer := jsClientConnect(t, s)\n\tdefer nc2.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tend := time.Now().Add(2 * time.Second)\n\tfor time.Now().Before(end) {\n\t\tproducer.Publish(\"foo\", []byte(strings.Repeat(\"A\", 128)))\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 5; i++ {\n\t\tsub, err := js.PullSubscribe(\"foo\", fmt.Sprintf(\"C-%d\", i))\n\t\trequire_NoError(t, err)\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor range time.NewTicker(10 * time.Millisecond).C {\n\t\t\t\tselect {\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\n\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\tif err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, nats.ErrConnectionClosed) {\n\t\t\t\t\tt.Logf(\"Pull Error: %v\", err)\n\t\t\t\t}\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\tc.lameDuckRestartAll()\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t// Swap the logger to try to detect the condition after the restart.\n\tloggers := make([]*captureDebugLogger, 3)\n\tfor i, srv := range c.servers {\n\t\tl := &captureDebugLogger{dbgCh: make(chan string, 10)}\n\t\tloggers[i] = l\n\t\tsrv.SetLogger(l, true, false)\n\t}\n\tcondition := `Direct proposal ignored, not leader (state: CLOSED)`\n\terrCh := make(chan error, 10)\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase dl := <-loggers[0].dbgCh:\n\t\t\t\tif strings.Contains(dl, condition) {\n\t\t\t\t\terrCh <- errors.New(condition)\n\t\t\t\t}\n\t\t\tcase dl := <-loggers[1].dbgCh:\n\t\t\t\tif strings.Contains(dl, condition) {\n\t\t\t\t\terrCh <- errors.New(condition)\n\t\t\t\t}\n\t\t\tcase dl := <-loggers[2].dbgCh:\n\t\t\t\tif strings.Contains(dl, condition) {\n\t\t\t\t\terrCh <- errors.New(condition)\n\t\t\t\t}\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Start publishing again for a while.\n\tend = time.Now().Add(2 * time.Second)\n\tfor time.Now().Before(end) {\n\t\tproducer.Publish(\"foo\", []byte(strings.Repeat(\"A\", 128)))\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\n\t// Try to do a stream edit back to R=1 after doing all the upgrade.\n\tinfo, _ := js.StreamInfo(\"TEST\")\n\tsconfig := info.Config\n\tsconfig.Replicas = 1\n\t_, err = js.UpdateStream(&sconfig)\n\trequire_NoError(t, err)\n\n\t// Leave running for some time after the update.\n\ttime.Sleep(2 * time.Second)\n\n\tinfo, _ = js.StreamInfo(\"TEST\")\n\tsconfig = info.Config\n\tsconfig.Replicas = 3\n\t_, err = js.UpdateStream(&sconfig)\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatalf(\"Bad condition on raft node: %v\", e)\n\tcase <-time.After(2 * time.Second):\n\t\t// Done\n\t}\n\n\t// Stop goroutines and wait for them to exit.\n\tcancel()\n\twg.Wait()\n}\n\nfunc TestLongFileStoreEnforceMsgPerSubjectLimit(t *testing.T) {\n\ttd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: td, BlockSize: 1024},\n\t\tStreamConfig{\n\t\t\tName: \"zzz\", Subjects: []string{\"test.>\"}, Storage: FileStorage,\n\t\t},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tt.Logf(\"Starting publishes\")\n\tfor i := 0; i < 100_000; i++ {\n\t\t_, _, err := fs.StoreMsg(fmt.Sprintf(\"test.%06d\", i), nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\t// Now update some of them. Leave a bit of a mess with some big gaps.\n\tfor i := 0; i < 5_000_000; i++ {\n\t\tn := rand.Int31n(100_000)\n\t\tif n < 5000 {\n\t\t\tcontinue\n\t\t}\n\t\t_, _, err := fs.StoreMsg(fmt.Sprintf(\"test.%06d\", n), nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\tt.Logf(\"Publish complete\")\n\n\trequire_NoError(t, fs.Stop())\n\tfs, err = newFileStore(\n\t\tFileStoreConfig{StoreDir: td, BlockSize: 1024},\n\t\tStreamConfig{\n\t\t\tName: \"zzz\", Subjects: []string{\"test.>\"}, Storage: FileStorage,\n\t\t\tMaxMsgsPer: 1,\n\t\t},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Mangle the filestore state and then see how long it takes to enforce\n\t// the per-subject limit.\n\tfs.state.Msgs++\n\tstart := time.Now()\n\tfs.enforceMsgPerSubjectLimit(false)\n\trequire_LessThan(t, time.Since(start), time.Minute)\n\tt.Logf(\"Took %s\", time.Since(start))\n}\n" }, { "path": "server/jetstream_consumer_test.go", "content": "// Copyright 2022-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_consumer_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\tcrand \"crypto/rand\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\tos \"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go/jetstream\"\n\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc TestJetStreamConsumerMultipleFiltersRemoveFilters(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"one\", \"data\")\n\tsendStreamMsg(t, nc, \"two\", \"data\")\n\tsendStreamMsg(t, nc, \"three\", \"data\")\n\n\tconsumer, err := js.PullSubscribe(\"\", \"consumer\", nats.Bind(\"TEST\", \"consumer\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err := consumer.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tFilterSubjects: []string{},\n\t})\n\trequire_NoError(t, err)\n\n\tmsgs, err = consumer.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\n\tmsgs, err = consumer.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n}\n\nfunc TestJetStreamConsumerMultipleFiltersRace(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\tvar seqs []uint64\n\tvar mu sync.Mutex\n\n\ttotal := 10_000\n\tvar wg sync.WaitGroup\n\n\tsend := func(subj string) {\n\t\tdefer wg.Done()\n\t\tfor i := 0; i < total; i++ {\n\t\t\tsendStreamMsg(t, nc, subj, \"data\")\n\t\t}\n\t}\n\twg.Add(4)\n\tgo send(\"one\")\n\tgo send(\"two\")\n\tgo send(\"three\")\n\tgo send(\"four\")\n\twg.Wait()\n\n\tmset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tFilterSubjects: []string{\"one\", \"two\", \"three\"},\n\t\tAckPolicy: AckExplicit,\n\t})\n\n\tdone := make(chan struct{})\n\twg.Add(10)\n\tfor i := 0; i < 10; i++ {\n\t\tgo func(t *testing.T) {\n\t\t\tdefer wg.Done()\n\n\t\t\tc, err := js.PullSubscribe(_EMPTY_, \"consumer\", nats.Bind(\"TEST\", \"consumer\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-done:\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t\tmsgs, err := c.Fetch(10, nats.MaxWait(2*time.Second))\n\t\t\t\t// We don't want to stop before at expected number of messages, as we want\n\t\t\t\t// to also test against getting to many messages.\n\t\t\t\t// Because of that, we ignore timeout and connection closed errors.\n\t\t\t\tif err != nil && err != nats.ErrTimeout && err != nats.ErrConnectionClosed {\n\t\t\t\t\tt.Errorf(\"error while fetching messages: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\tinfo, err := msg.Metadata()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\tseqs = append(seqs, info.Sequence.Consumer)\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}(t)\n\t}\n\n\tcheckFor(t, 30*time.Second, 100*time.Millisecond, func() error {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tif len(seqs) != 3*total {\n\t\t\treturn fmt.Errorf(\"found %d messages instead of %d\", len(seqs), 3*total)\n\t\t}\n\t\tslices.Sort(seqs)\n\t\tfor i := 1; i < len(seqs); i++ {\n\t\t\tif seqs[i] != seqs[i-1]+1 {\n\t\t\t\tfmt.Printf(\"seqs: %+v\\n\", seqs)\n\t\t\t\treturn fmt.Errorf(\"sequence mismatch at %v\", i)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\tclose(done)\n\twg.Wait()\n}\n\nfunc TestJetStreamConsumerMultipleConsumersSingleFilter(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t// Setup few subjects with varying messages count.\n\tsubjects := []struct {\n\t\tsubject string\n\t\tmessages int\n\t\twc bool\n\t}{\n\t\t{subject: \"one\", messages: 5000},\n\t\t{subject: \"two\", messages: 7500},\n\t\t{subject: \"three\", messages: 2500},\n\t\t{subject: \"four\", messages: 1000},\n\t\t{subject: \"five.>\", messages: 3000, wc: true},\n\t}\n\n\ttotalMsgs := 0\n\tfor _, subject := range subjects {\n\t\ttotalMsgs += subject.messages\n\t}\n\n\t// Setup consumers, filtering some of the messages from the stream.\n\tconsumers := []*struct {\n\t\tname string\n\t\tsubjects []string\n\t\texpectedMsgs int\n\t\tdelivered atomic.Int32\n\t}{\n\t\t{name: \"C1\", subjects: []string{\"one\"}, expectedMsgs: 5000},\n\t\t{name: \"C2\", subjects: []string{\"two\"}, expectedMsgs: 7500},\n\t\t{name: \"C3\", subjects: []string{\"one\"}, expectedMsgs: 5000},\n\t\t{name: \"C4\", subjects: []string{\"one\"}, expectedMsgs: 5000},\n\t}\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\", \"five.>\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\tfor c, consumer := range consumers {\n\t\t_, err := mset.addConsumer(&ConsumerConfig{\n\t\t\tDurable: consumer.name,\n\t\t\tFilterSubjects: consumer.subjects,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tDeliverPolicy: DeliverAll,\n\t\t\tAckWait: time.Second * 30,\n\t\t\tDeliverSubject: nc.NewInbox(),\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tgo func(c int, name string) {\n\t\t\t_, err = js.Subscribe(\"\", func(m *nats.Msg) {\n\t\t\t\trequire_NoError(t, m.Ack())\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tconsumers[c].delivered.Add(1)\n\n\t\t\t}, nats.Bind(\"TEST\", name))\n\t\t\trequire_NoError(t, err)\n\t\t}(c, consumer.name)\n\t}\n\n\t// Publish with random intervals, while consumers are active.\n\tvar wg sync.WaitGroup\n\tfor _, subject := range subjects {\n\t\twg.Add(subject.messages)\n\t\tgo func(subject string, messages int, wc bool) {\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\ttime.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Millisecond)\n\t\t\tfor i := 0; i < messages; i++ {\n\t\t\t\ttime.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Microsecond)\n\t\t\t\t// If subject has wildcard, add random last subject token.\n\t\t\t\tpubSubject := subject\n\t\t\t\tif wc {\n\t\t\t\t\tpubSubject = fmt.Sprintf(\"%v.%v\", subject, rand.Int63n(10))\n\t\t\t\t}\n\t\t\t\t_, err := js.PublishAsync(pubSubject, []byte(\"data\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\twg.Done()\n\t\t\t}\n\t\t}(subject.subject, subject.messages, subject.wc)\n\t}\n\twg.Wait()\n\n\tcheckFor(t, time.Second*10, time.Millisecond*500, func() error {\n\t\tfor _, consumer := range consumers {\n\t\t\tinfo, err := js.ConsumerInfo(\"TEST\", consumer.name)\n\t\t\trequire_NoError(t, err)\n\t\t\tif info.Delivered.Consumer != uint64(consumer.expectedMsgs) {\n\t\t\t\treturn fmt.Errorf(\"%v:expected consumer delivered seq %v, got %v. actually delivered: %v\",\n\t\t\t\t\tconsumer.name, consumer.expectedMsgs, info.Delivered.Consumer, consumer.delivered.Load())\n\t\t\t}\n\t\t\tif info.AckFloor.Consumer != uint64(consumer.expectedMsgs) {\n\t\t\t\treturn fmt.Errorf(\"%v: expected consumer ack floor %v, got %v\", consumer.name, totalMsgs, info.AckFloor.Consumer)\n\t\t\t}\n\t\t\tif consumer.delivered.Load() != int32(consumer.expectedMsgs) {\n\n\t\t\t\treturn fmt.Errorf(\"%v: expected %v, got %v\", consumer.name, consumer.expectedMsgs, consumer.delivered.Load())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamConsumerMultipleConsumersMultipleFilters(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t// Setup few subjects with varying messages count.\n\tsubjects := []struct {\n\t\tsubject string\n\t\tmessages int\n\t\twc bool\n\t}{\n\t\t{subject: \"one\", messages: 50},\n\t\t{subject: \"two\", messages: 75},\n\t\t{subject: \"three\", messages: 250},\n\t\t{subject: \"four\", messages: 10},\n\t\t{subject: \"five.>\", messages: 300, wc: true},\n\t}\n\n\ttotalMsgs := 0\n\tfor _, subject := range subjects {\n\t\ttotalMsgs += subject.messages\n\t}\n\n\t// Setup consumers, filtering some of the messages from the stream.\n\tconsumers := []*struct {\n\t\tname string\n\t\tsubjects []string\n\t\texpectedMsgs int\n\t\tdelivered atomic.Int32\n\t}{\n\t\t{name: \"C1\", subjects: []string{\"one\", \"two\"}, expectedMsgs: 125},\n\t\t{name: \"C2\", subjects: []string{\"two\", \"three\"}, expectedMsgs: 325},\n\t\t{name: \"C3\", subjects: []string{\"one\", \"three\"}, expectedMsgs: 300},\n\t\t{name: \"C4\", subjects: []string{\"one\", \"five.>\"}, expectedMsgs: 350},\n\t}\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\", \"five.>\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\tfor c, consumer := range consumers {\n\t\t_, err := mset.addConsumer(&ConsumerConfig{\n\t\t\tDurable: consumer.name,\n\t\t\tFilterSubjects: consumer.subjects,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tDeliverPolicy: DeliverAll,\n\t\t\tAckWait: time.Second * 30,\n\t\t\tDeliverSubject: nc.NewInbox(),\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tgo func(c int, name string) {\n\t\t\t_, err = js.Subscribe(\"\", func(m *nats.Msg) {\n\t\t\t\trequire_NoError(t, m.Ack())\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tconsumers[c].delivered.Add(1)\n\n\t\t\t}, nats.Bind(\"TEST\", name))\n\t\t\trequire_NoError(t, err)\n\t\t}(c, consumer.name)\n\t}\n\n\t// Publish with random intervals, while consumers are active.\n\tvar wg sync.WaitGroup\n\tfor _, subject := range subjects {\n\t\twg.Add(subject.messages)\n\t\tgo func(subject string, messages int, wc bool) {\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\ttime.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Millisecond)\n\t\t\tfor i := 0; i < messages; i++ {\n\t\t\t\ttime.Sleep(time.Duration(rand.Int63n(1000)+1) * time.Microsecond)\n\t\t\t\t// If subject has wildcard, add random last subject token.\n\t\t\t\tpubSubject := subject\n\t\t\t\tif wc {\n\t\t\t\t\tpubSubject = fmt.Sprintf(\"%v.%v\", subject, rand.Int63n(10))\n\t\t\t\t}\n\t\t\t\tack, err := js.PublishAsync(pubSubject, []byte(\"data\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tgo func() {\n\t\t\t\t\tack.Ok()\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t}\n\t\t}(subject.subject, subject.messages, subject.wc)\n\t}\n\tdone := make(chan struct{})\n\tgo func() {\n\t\twg.Wait()\n\t\tclose(done)\n\t}()\n\n\tselect {\n\tcase <-time.After(time.Second * 15):\n\t\tt.Fatalf(\"Timed out waiting for acks\")\n\tcase <-done:\n\t}\n\twg.Wait()\n\n\tcheckFor(t, time.Second*15, time.Second*1, func() error {\n\t\tfor _, consumer := range consumers {\n\t\t\tinfo, err := js.ConsumerInfo(\"TEST\", consumer.name)\n\t\t\trequire_NoError(t, err)\n\t\t\tif info.Delivered.Consumer != uint64(consumer.expectedMsgs) {\n\t\t\t\treturn fmt.Errorf(\"%v:expected consumer delivered seq %v, got %v. actually delivered: %v\",\n\t\t\t\t\tconsumer.name, consumer.expectedMsgs, info.Delivered.Consumer, consumer.delivered.Load())\n\t\t\t}\n\t\t\tif info.AckFloor.Consumer != uint64(consumer.expectedMsgs) {\n\t\t\t\treturn fmt.Errorf(\"%v: expected consumer ack floor %v, got %v\", consumer.name, totalMsgs, info.AckFloor.Consumer)\n\t\t\t}\n\t\t\tif consumer.delivered.Load() != int32(consumer.expectedMsgs) {\n\n\t\t\t\treturn fmt.Errorf(\"%v: expected %v, got %v\", consumer.name, consumer.expectedMsgs, consumer.delivered.Load())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamConsumerMultipleFiltersSequence(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\", \"five.>\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t\tDeliverSubject: nc.NewInbox(),\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\tsendStreamMsg(t, nc, \"one\", fmt.Sprintf(\"%d\", i))\n\t}\n\tfor i := 20; i < 40; i++ {\n\t\tsendStreamMsg(t, nc, \"two\", fmt.Sprintf(\"%d\", i))\n\t}\n\tfor i := 40; i < 60; i++ {\n\t\tsendStreamMsg(t, nc, \"one\", fmt.Sprintf(\"%d\", i))\n\t}\n\n\tsub, err := js.SubscribeSync(\"\", nats.Bind(\"TEST\", \"DUR\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 60; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second * 1)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, string(msg.Data) == fmt.Sprintf(\"%d\", i))\n\t}\n}\n\nfunc TestJetStreamConsumerActions(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\", \"five.>\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create Consumer. No consumers existed before, so should be fine.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\t// Create consumer again. Should be ok if action is CREATE but config is exactly the same.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\t// Create consumer again. Should error if action is CREATE.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tFilterSubjects: []string{\"one\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionCreate, false)\n\trequire_Error(t, err)\n\n\t// Update existing consumer. Should be fine, as consumer exists.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"DUR\",\n\t\tFilterSubjects: []string{\"one\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionUpdate, false)\n\trequire_NoError(t, err)\n\n\t// Update consumer. Should error, as this consumer does not exist.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"NEW\",\n\t\tFilterSubjects: []string{\"one\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionUpdate, false)\n\trequire_Error(t, err)\n\n\t// Create new ephemeral. Should be fine as the consumer doesn't exist already\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tName: \"EPH\",\n\t\tFilterSubjects: []string{\"one\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\n\t// Trying to create it again right away. Should error as it already exists (and hasn't been cleaned up yet)\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tName: \"EPH\",\n\t\tFilterSubjects: []string{\"one\"},\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverPolicy: DeliverAll,\n\t\tAckWait: time.Second * 30,\n\t}, ActionCreate, false)\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamConsumerActionsOnWorkQueuePolicyStream(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: WorkQueuePolicy,\n\t\tSubjects: []string{\"one\", \"two\", \"three\", \"four\", \"five.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t\tAckPolicy: AckExplicit,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C2\",\n\t\tFilterSubjects: []string{\"three\", \"four\"},\n\t\tAckPolicy: AckExplicit,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tFilterSubjects: []string{\"five.*\"},\n\t\tAckPolicy: AckExplicit,\n\t}, ActionCreate, false)\n\trequire_NoError(t, err)\n\n\t// Updating a consumer by removing a previous subject filter.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tFilterSubjects: []string{\"one\"}, // Remove a subject.\n\t\tAckPolicy: AckExplicit,\n\t}, ActionUpdate, false)\n\trequire_NoError(t, err)\n\n\t// Updating a consumer without overlapping subjects.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C2\",\n\t\tFilterSubjects: []string{\"three\", \"four\", \"two\"}, // Add previously removed subject.\n\t\tAckPolicy: AckExplicit,\n\t}, ActionUpdate, false)\n\trequire_NoError(t, err)\n\n\t// Creating a consumer with overlapping subjects should return an error.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C4\",\n\t\tFilterSubjects: []string{\"one\", \"two\", \"three\", \"four\"},\n\t\tAckPolicy: AckExplicit,\n\t}, ActionCreate, false)\n\trequire_Error(t, err)\n\tif !IsNatsErr(err, JSConsumerWQConsumerNotUniqueErr) {\n\t\tt.Errorf(\"want error %q, got %q\", ApiErrors[JSConsumerWQConsumerNotUniqueErr], err)\n\t}\n\n\t// Updating a consumer with overlapping subjects should return an error.\n\t_, err = mset.addConsumerWithAction(&ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tFilterSubjects: []string{\"one\", \"two\", \"three\", \"four\"},\n\t\tAckPolicy: AckExplicit,\n\t}, ActionUpdate, false)\n\trequire_Error(t, err)\n\tif !IsNatsErr(err, JSConsumerWQConsumerNotUniqueErr) {\n\t\tt.Errorf(\"want error %q, got %q\", ApiErrors[JSConsumerWQConsumerNotUniqueErr], err)\n\t}\n}\n\nfunc TestJetStreamConsumerActionsViaAPI(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t_, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: LimitsPolicy,\n\t\tSubjects: []string{\"one\"},\n\t\tMaxAge: time.Second * 90,\n\t})\n\trequire_NoError(t, err)\n\n\t// Update non-existing consumer, which should fail.\n\trequest, err := json.Marshal(&CreateConsumerRequest{\n\t\tAction: ActionUpdate,\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"hello\",\n\t\t},\n\t\tStream: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\tresp, err := nc.Request(\"$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello\", []byte(request), time.Second*6)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\trequire_Error(t, ccResp.Error)\n\n\t// create non existing consumer - which should be fine.\n\tccResp.Error = nil\n\trequest, err = json.Marshal(&CreateConsumerRequest{\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"hello\",\n\t\t},\n\t\tStream: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\tresp, err = nc.Request(\"$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello\", []byte(request), time.Second*6)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"expected nil, got %v\", ccResp.Error)\n\t}\n\n\t// re-create existing consumer - which should be an error.\n\tccResp.Error = nil\n\trequest, err = json.Marshal(&CreateConsumerRequest{\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"hello\",\n\t\t\tFilterSubject: \"one\",\n\t\t},\n\t\tStream: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(\"$JS.API.CONSUMER.DURABLE.CREATE.TEST.hello\", []byte(request), time.Second*6)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error == nil {\n\t\tt.Fatalf(\"expected err, got nil\")\n\t}\n\n\t// create a named ephemeral consumer\n\tccResp.Error = nil\n\trequest, err = json.Marshal(&CreateConsumerRequest{\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tName: \"ephemeral\",\n\t\t\tFilterSubject: \"one\",\n\t\t},\n\t\tStream: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(\"$JS.API.CONSUMER.CREATE.TEST.ephemeral\", []byte(request), time.Second*6)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\n\t// re-create existing consumer - which should be an error.\n\tccResp.Error = nil\n\trequest, err = json.Marshal(&CreateConsumerRequest{\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tName: \"ephemeral\",\n\t\t\tFilterSubject: \"one\",\n\t\t},\n\t\tStream: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(\"$JS.API.CONSUMER.CREATE.TEST.ephemeral\", []byte(request), time.Second*6)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error == nil {\n\t\tt.Fatalf(\"expected err, got nil\")\n\t}\n}\n\nfunc TestJetStreamConsumerActionsUnmarshal(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tgiven []byte\n\t\texpected ConsumerAction\n\t\texpectErr bool\n\t}{\n\t\t{name: \"action create\", given: []byte(`{\"action\": \"create\"}`), expected: ActionCreate},\n\t\t{name: \"action update\", given: []byte(`{\"action\": \"update\"}`), expected: ActionUpdate},\n\t\t{name: \"no action\", given: []byte(\"{}\"), expected: ActionCreateOrUpdate},\n\t\t{name: \"unknown\", given: []byte(`{\"action\": \"unknown\"}`), expected: ActionCreateOrUpdate, expectErr: true},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\tvar request CreateConsumerRequest\n\t\t\terr := json.Unmarshal(test.given, &request)\n\t\t\tfmt.Printf(\"given: %v, expected: %v\\n\", test.expectErr, err)\n\t\t\tif !test.expectErr {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t} else {\n\t\t\t\trequire_Error(t, err)\n\t\t\t}\n\t\t\trequire_True(t, test.expected == request.Action)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerMultipleFiltersLastPerSubject(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, error := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"one\", \"two\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, error)\n\n\tsendStreamMsg(t, nc, \"one\", \"1\")\n\tsendStreamMsg(t, nc, \"one\", \"2\")\n\tsendStreamMsg(t, nc, \"one\", \"3\")\n\tsendStreamMsg(t, nc, \"two\", \"1\")\n\tsendStreamMsg(t, nc, \"two\", \"2\")\n\tsendStreamMsg(t, nc, \"two\", \"3\")\n\n\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"C\",\n\t\tFilterSubjects: []string{\"one\", \"two\"},\n\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\tReplicas: 3,\n\t\tDeliverSubject: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\tconsumer, err := js.SubscribeSync(\"\", nats.Bind(\"TEST\", \"C\"))\n\trequire_NoError(t, err)\n\n\t// expect last message for subject \"one\"\n\tmsg, err := consumer.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, \"3\", string(msg.Data))\n\trequire_Equal(t, \"one\", msg.Subject)\n\n\t// expect last message for subject \"two\"\n\tmsg, err = consumer.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, \"3\", string(msg.Data))\n\trequire_Equal(t, \"two\", msg.Subject)\n\n}\n\nfunc consumerWithFilterSubjects(filterSubjects []string) *consumer {\n\tc := consumer{}\n\tfor _, filter := range filterSubjects {\n\t\tsub := &subjectFilter{\n\t\t\tsubject: filter,\n\t\t\thasWildcard: subjectHasWildcard(filter),\n\t\t\ttokenizedSubject: tokenizeSubjectIntoSlice(nil, filter),\n\t\t}\n\t\tc.subjf = append(c.subjf, sub)\n\t}\n\n\treturn &c\n}\n\nfunc filterSubjects(n int) []string {\n\tfs := make([]string, 0, n)\n\tfor {\n\t\tliterals := []string{\"foo\", \"bar\", nuid.Next(), \"xyz\", \"abcdef\"}\n\t\tfs = append(fs, strings.Join(literals, \".\"))\n\t\tif len(fs) == n {\n\t\t\treturn fs\n\t\t}\n\t\t// Create more filterSubjects by going through the literals and replacing one with the '*' wildcard.\n\t\tl := len(literals)\n\t\tfor i := 0; i < l; i++ {\n\t\t\te := make([]string, l)\n\t\t\tfor j := 0; j < l; j++ {\n\t\t\t\tif j == i {\n\t\t\t\t\te[j] = \"*\"\n\t\t\t\t} else {\n\t\t\t\t\te[j] = literals[j]\n\t\t\t\t}\n\t\t\t}\n\t\t\tfs = append(fs, strings.Join(e, \".\"))\n\t\t\tif len(fs) == n {\n\t\t\t\treturn fs\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamConsumerIsFilteredMatch(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tfilterSubjects []string\n\t\tsubject string\n\t\tresult bool\n\t}{\n\t\t{\"no filter\", []string{}, \"foo.bar\", true},\n\t\t{\"literal match\", []string{\"foo.baz\", \"foo.bar\"}, \"foo.bar\", true},\n\t\t{\"literal mismatch\", []string{\"foo.baz\", \"foo.bar\"}, \"foo.ban\", false},\n\t\t{\"wildcard > match\", []string{\"bar.>\", \"foo.>\"}, \"foo.bar\", true},\n\t\t{\"wildcard > match\", []string{\"bar.>\", \"foo.>\"}, \"bar.foo\", true},\n\t\t{\"wildcard > mismatch\", []string{\"bar.>\", \"foo.>\"}, \"baz.foo\", false},\n\t\t{\"wildcard * match\", []string{\"bar.*\", \"foo.*\"}, \"foo.bar\", true},\n\t\t{\"wildcard * match\", []string{\"bar.*\", \"foo.*\"}, \"bar.foo\", true},\n\t\t{\"wildcard * mismatch\", []string{\"bar.*\", \"foo.*\"}, \"baz.foo\", false},\n\t\t{\"wildcard * match\", []string{\"foo.*.x\", \"foo.*.y\"}, \"foo.bar.x\", true},\n\t\t{\"wildcard * match\", []string{\"foo.*.x\", \"foo.*.y\", \"foo.*.z\"}, \"foo.bar.z\", true},\n\t\t{\"many mismatch\", filterSubjects(100), \"foo.bar.do.not.match.any.filter.subject\", false},\n\t\t{\"many match\", filterSubjects(100), \"foo.bar.12345.xyz.abcdef\", true}, // will be matched by \"foo.bar.*.xyz.abcdef\"\n\t} {\n\t\ttest := test\n\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\tc := consumerWithFilterSubjects(test.filterSubjects)\n\t\t\tif res := c.isFilteredMatch(test.subject); res != test.result {\n\t\t\t\tt.Fatalf(\"Subject %q filtered match of %v, should be %v, got %v\",\n\t\t\t\t\ttest.subject, test.filterSubjects, test.result, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerWorkQueuePolicyOverlap(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*.*\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ConsumerA\",\n\t\tFilterSubject: \"foo.bar.*\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ConsumerB\",\n\t\tFilterSubject: \"foo.*.bar\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(err.Error(), \"unique\"))\n}\n\nfunc TestJetStreamConsumerIsEqualOrSubsetMatch(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tfilterSubjects []string\n\t\tsubject string\n\t\tresult bool\n\t}{\n\t\t{\"no filter\", []string{}, \"foo.bar\", false},\n\t\t{\"literal match\", []string{\"foo.baz\", \"foo.bar\"}, \"foo.bar\", true},\n\t\t{\"literal mismatch\", []string{\"foo.baz\", \"foo.bar\"}, \"foo.ban\", false},\n\t\t{\"literal match\", []string{\"bar.>\", \"foo.>\"}, \"foo.>\", true},\n\t\t{\"subset match\", []string{\"bar.foo.>\", \"foo.bar.>\"}, \"bar.>\", true},\n\t\t{\"subset mismatch\", []string{\"bar.>\", \"foo.>\"}, \"baz.foo.>\", false},\n\t\t{\"literal match\", filterSubjects(100), \"foo.bar.*.xyz.abcdef\", true},\n\t\t{\"subset match\", filterSubjects(100), \"foo.bar.>\", true},\n\t} {\n\t\ttest := test\n\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\tc := consumerWithFilterSubjects(test.filterSubjects)\n\t\t\tif res := c.isEqualOrSubsetMatch(test.subject); res != test.result {\n\t\t\t\tt.Fatalf(\"Subject %q subset match of %v, should be %v, got %v\",\n\t\t\t\t\ttest.subject, test.filterSubjects, test.result, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerBackOff(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig nats.ConsumerConfig\n\t\tshouldErr bool\n\t}{\n\t\t{\n\t\t\tname: \"backoff_with_max_deliver\",\n\t\t\tconfig: nats.ConsumerConfig{\n\t\t\t\tMaxDeliver: 3,\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute},\n\t\t\t},\n\t\t\tshouldErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"backoff_with_max_deliver_equal\",\n\t\t\tconfig: nats.ConsumerConfig{\n\t\t\t\tMaxDeliver: 3,\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute, time.Hour},\n\t\t\t},\n\t\t\tshouldErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"backoff_with_max_deliver_equal_to_zero\",\n\t\t\tconfig: nats.ConsumerConfig{\n\t\t\t\tMaxDeliver: 0,\n\t\t\t\tBackOff: []time.Duration{},\n\t\t\t},\n\t\t\tshouldErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"backoff_with_max_deliver_smaller\",\n\t\t\tconfig: nats.ConsumerConfig{\n\t\t\t\tMaxDeliver: 2,\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute, time.Hour},\n\t\t\t},\n\t\t\tshouldErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"backoff_with_default_max_deliver\",\n\t\t\tconfig: nats.ConsumerConfig{\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute, time.Hour},\n\t\t\t},\n\t\t\tshouldErr: false,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: test.name,\n\t\t\t\tSubjects: []string{test.name},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.AddConsumer(test.name, &test.config)\n\t\t\trequire_True(t, test.shouldErr == (err != nil))\n\t\t\tif test.shouldErr {\n\t\t\t\trequire_True(t, strings.Contains(err.Error(), \"max deliver\"))\n\t\t\t}\n\n\t\t\t// test if updating consumers works too.\n\t\t\ttest.config.Durable = \"consumer\"\n\t\t\t_, err = js.AddConsumer(test.name, &nats.ConsumerConfig{\n\t\t\t\tDurable: test.config.Durable,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\ttest.config.Description = \"Updated\"\n\t\t\t_, err = js.UpdateConsumer(test.name, &test.config)\n\t\t\trequire_True(t, test.shouldErr == (err != nil))\n\t\t\tif test.shouldErr {\n\t\t\t\trequire_True(t, strings.Contains(err.Error(), \"max deliver\"))\n\t\t\t}\n\t\t})\n\n\t}\n}\n\nfunc TestJetStreamConsumerDelete(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\treplicas int\n\t}{\n\t\t{\"single server\", 1},\n\t\t{\"clustered\", 3},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar s *Server\n\t\t\tif test.replicas == 1 {\n\t\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\t\tdefer s.Shutdown()\n\t\t\t} else {\n\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", test.replicas)\n\t\t\t\tdefer c.shutdown()\n\t\t\t\ts = c.randomServer()\n\t\t\t}\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"events.>\"},\n\t\t\t\tMaxAge: time.Second * 90,\n\t\t\t\tReplicas: test.replicas,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\t\tDurable: \"consumer\",\n\t\t\t\tFilterSubject: \"events.>\",\n\t\t\t\tReplicas: test.replicas,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tjs.Publish(\"events.1\", []byte(\"hello\"))\n\n\t\t\tcr := JSApiConsumerGetNextRequest{\n\t\t\t\tBatch: 10,\n\t\t\t\tExpires: time.Second * 30,\n\t\t\t}\n\t\t\tcrBytes, err := json.Marshal(cr)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tinbox := nats.NewInbox()\n\t\t\tconsumerSub, err := nc.SubscribeSync(inbox)\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"consumer\"), inbox, crBytes)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmsg, err := consumerSub.NextMsg(time.Second * 30)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, \"hello\", string(msg.Data))\n\n\t\t\tjs.DeleteConsumer(\"TEST\", \"consumer\")\n\n\t\t\tmsg, err = consumerSub.NextMsg(time.Second * 30)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif !strings.Contains(string(msg.Header.Get(\"Description\")), \"Consumer Deleted\") {\n\t\t\t\tt.Fatalf(\"Expected exclusive consumer error, got %q\", msg.Header.Get(\"Description\"))\n\t\t\t}\n\t\t})\n\n\t}\n}\n\nfunc TestJetStreamConsumerFetchWithDrain(t *testing.T) {\n\tt.Skip()\n\n\ttest := func(t *testing.T, cc *nats.ConsumerConfig) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tRetention: nats.LimitsPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(\"TEST\", cc)\n\t\trequire_NoError(t, err)\n\n\t\tconst messages = 10_000\n\n\t\tfor i := 0; i < messages; i++ {\n\t\t\tsendStreamMsg(t, nc, \"foo\", fmt.Sprintf(\"%d\", i+1))\n\t\t}\n\n\t\tcr := JSApiConsumerGetNextRequest{\n\t\t\tBatch: 100_000,\n\t\t\tExpires: 10 * time.Second,\n\t\t}\n\t\tcrBytes, err := json.Marshal(cr)\n\t\trequire_NoError(t, err)\n\n\t\tmsgs := make(map[int]int)\n\n\t\tprocessMsg := func(t *testing.T, sub *nats.Subscription, msgs map[int]int) bool {\n\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tmetadata, err := msg.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, msg.Ack())\n\n\t\t\tv, err := strconv.Atoi(string(msg.Data))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, uint64(v), metadata.Sequence.Stream)\n\n\t\t\tif _, ok := msgs[int(metadata.Sequence.Stream-1)]; !ok && len(msgs) > 0 {\n\t\t\t\tt.Logf(\"Stream Sequence gap detected: current %d\", metadata.Sequence.Stream)\n\t\t\t}\n\t\t\tif _, ok := msgs[int(metadata.Sequence.Stream)]; ok {\n\t\t\t\tt.Fatalf(\"Message for seq %d has been seen before\", metadata.Sequence.Stream)\n\t\t\t}\n\t\t\t// We do not expect official redeliveries here so this should always be 1.\n\t\t\tif metadata.NumDelivered != 1 {\n\t\t\t\tt.Errorf(\"Expected NumDelivered of 1, got %d for seq %d\",\n\t\t\t\t\tmetadata.NumDelivered, metadata.Sequence.Stream)\n\t\t\t}\n\t\t\tmsgs[int(metadata.Sequence.Stream)] = int(metadata.NumDelivered)\n\t\t\treturn true\n\t\t}\n\n\t\tfor {\n\t\t\tinbox := nats.NewInbox()\n\t\t\tsub, err := nc.SubscribeSync(inbox)\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"C\"), inbox, crBytes)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Drain after first message processed.\n\t\t\tprocessMsg(t, sub, msgs)\n\t\t\tsub.Drain()\n\n\t\t\tfor {\n\t\t\t\tif !processMsg(t, sub, msgs) {\n\t\t\t\t\tif len(msgs) == messages {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"no-backoff\", func(t *testing.T) {\n\t\ttest(t, &nats.ConsumerConfig{\n\t\t\tDurable: \"C\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tAckWait: 20 * time.Second,\n\t\t})\n\t})\n\tt.Run(\"with-backoff\", func(t *testing.T) {\n\t\ttest(t, &nats.ConsumerConfig{\n\t\t\tDurable: \"C\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tAckWait: 20 * time.Second,\n\t\t\tBackOff: []time.Duration{25 * time.Millisecond, 100 * time.Millisecond, 250 * time.Millisecond},\n\t\t})\n\t})\n}\n\nfunc TestJetStreamConsumerLongSubjectHang(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\treadSubj := \"a1.\"\n\tpurgeSubj := \"a2.\"\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{readSubj + \">\", purgeSubj + \">\"},\n\t\tAllowRollup: true,\n\t})\n\trequire_NoError(t, err)\n\n\tprefix := strings.Repeat(\"a\", 22)\n\tfor i := 0; i < 2; i++ {\n\t\tsubj := readSubj + prefix + fmt.Sprintf(\"%d\", i)\n\t\t_, err = js.Publish(subj, []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t\tchunkSubj := purgeSubj + fmt.Sprintf(\"%d\", i)\n\t\t_, err = js.Publish(chunkSubj, []byte(\"contents\"))\n\t\trequire_NoError(t, err)\n\t}\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: purgeSubj + \">\"})\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\t// we should have 2 msgs left after purge\n\trequire_Equal(t, si.State.Msgs, 2)\n\n\tsub, err := js.SubscribeSync(readSubj+\">\", nats.OrderedConsumer(), nats.DeliverLastPerSubject())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tfor i := 0; i < 2; i++ {\n\t\tm, err := sub.NextMsg(500 * time.Millisecond)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, string(m.Data) == \"hello\")\n\t}\n}\n\nfunc TestJetStreamConsumerPedanticMode(t *testing.T) {\n\n\tsingleServerTemplate := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t\tlimits: {max_request_batch: 250}\n\t\t\t}\n\t\t\tno_auth_user: u\n\t\t\taccounts {\n\t\t\t\tONE {\n\t\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\t\tjetstream: enabled\n\t\t\t\t}\n\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t}`\n\n\tclusterTemplate := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t\tlimits: {max_request_batch: 250}\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t\t\tno_auth_user: u\n\t\t\taccounts {\n\t\t\t\tONE {\n\t\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\t\tjetstream: enabled\n\t\t\t\t}\n\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t}`\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\tgivenConfig ConsumerConfig\n\t\tgivenLimits nats.StreamConsumerLimits\n\t\tserverTemplateSingle string\n\t\tserverTemplateCluster string\n\t\tshouldError bool\n\t\tpedantic bool\n\t\treplicas int\n\t}{\n\t\t{\n\t\t\tname: \"default_non_pedantic\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t},\n\t\t\tgivenLimits: nats.StreamConsumerLimits{\n\t\t\t\tInactiveThreshold: time.Minute,\n\t\t\t\tMaxAckPending: 100,\n\t\t\t},\n\t\t\tshouldError: false,\n\t\t\tpedantic: false,\n\t\t},\n\t\t{\n\t\t\tname: \"default_pedantic_inactive_threshold\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t},\n\t\t\tgivenLimits: nats.StreamConsumerLimits{\n\t\t\t\tInactiveThreshold: time.Minute,\n\t\t\t},\n\t\t\tshouldError: true,\n\t\t\tpedantic: true,\n\t\t},\n\t\t{\n\t\t\tname: \"default_pedantic_max_ack_pending\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t},\n\t\t\tgivenLimits: nats.StreamConsumerLimits{\n\t\t\t\tMaxAckPending: 100,\n\t\t\t},\n\t\t\tshouldError: true,\n\t\t\tpedantic: true,\n\t\t},\n\t\t{\n\t\t\tname: \"pedantic_backoff_no_ack_wait\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute},\n\t\t\t},\n\t\t\tpedantic: true,\n\t\t\tshouldError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"backoff_no_ack_wait\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t\tBackOff: []time.Duration{time.Second, time.Minute},\n\t\t\t},\n\t\t\tpedantic: false,\n\t\t\tshouldError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"max_batch_requests\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t},\n\t\t\tserverTemplateSingle: singleServerTemplate,\n\t\t\tserverTemplateCluster: clusterTemplate,\n\t\t\tpedantic: false,\n\t\t\tshouldError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"pedantic_max_batch_requests\",\n\t\t\tgivenConfig: ConsumerConfig{\n\t\t\t\tDurable: \"durable\",\n\t\t\t},\n\t\t\tserverTemplateSingle: singleServerTemplate,\n\t\t\tserverTemplateCluster: clusterTemplate,\n\t\t\tpedantic: true,\n\t\t\tshouldError: true,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tfor _, mode := range []string{\"clustered\", \"single\"} {\n\t\t\tt.Run(fmt.Sprintf(\"%v_%v\", mode, test.name), func(t *testing.T) {\n\n\t\t\t\tvar s *Server\n\t\t\t\tif mode == \"single\" {\n\t\t\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\t\t\tdefer s.Shutdown()\n\t\t\t\t} else {\n\t\t\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\t\t\tdefer c.shutdown()\n\t\t\t\t\ts = c.randomServer()\n\t\t\t\t}\n\n\t\t\t\treplicas := 1\n\t\t\t\tif mode == \"clustered\" {\n\t\t\t\t\treplicas = 3\n\t\t\t\t}\n\n\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tjs.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: test.name,\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tReplicas: replicas,\n\t\t\t\t\tConsumerLimits: nats.StreamConsumerLimits{\n\t\t\t\t\t\tInactiveThreshold: time.Minute,\n\t\t\t\t\t\tMaxAckPending: 100,\n\t\t\t\t\t},\n\t\t\t\t})\n\n\t\t\t\t_, err := addConsumerWithError(t, nc, &CreateConsumerRequest{\n\t\t\t\t\tStream: test.name,\n\t\t\t\t\tConfig: test.givenConfig,\n\t\t\t\t\tAction: ActionCreateOrUpdate,\n\t\t\t\t\tPedantic: test.pedantic,\n\t\t\t\t})\n\t\t\t\trequire_True(t, (err != nil) == test.shouldError)\n\t\t\t\tif err != nil {\n\t\t\t\t\trequire_True(t, strings.Contains(err.Error(), \"pedantic\"))\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamConsumerStuckAckPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttype ActiveWorkItem struct {\n\t\tID int\n\t\tExpiry time.Time\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST_ACTIVE_WORK_ITEMS\",\n\t\tDiscard: nats.DiscardOld,\n\t\tMaxMsgsPerSubject: 1,\n\t\tSubjects: []string{\"TEST_ACTIVE_WORK_ITEMS.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST_ACTIVE_WORK_ITEMS\", &nats.ConsumerConfig{\n\t\tDurable: \"testactiveworkitemsconsumer\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: -1,\n\t\tMaxWaiting: 20000,\n\t\tAckWait: 15 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"TEST_ACTIVE_WORK_ITEMS.>\", \"testactiveworkitemsconsumer\", nats.BindStream(\"TEST_ACTIVE_WORK_ITEMS\"))\n\trequire_NoError(t, err)\n\n\terrs := make(chan error)\n\tgo func() {\n\t\tfor {\n\t\t\tmsgs, err := sub.Fetch(200)\n\t\t\tif err != nil {\n\t\t\t\t// test is done. stop the loop.\n\t\t\t\tif errors.Is(err, nats.ErrSubscriptionClosed) || errors.Is(err, nats.ErrConnectionClosed) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif !errors.Is(err, nats.ErrTimeout) {\n\t\t\t\t\terrs <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, msg := range msgs {\n\t\t\t\tmsg := msg\n\t\t\t\tvar workItem ActiveWorkItem\n\t\t\t\tif err := json.Unmarshal(msg.Data, &workItem); err != nil {\n\t\t\t\t\terrs <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tnow := time.Now()\n\t\t\t\t// If the work item has not expired, nak it with the respective delay.\n\t\t\t\tif workItem.Expiry.After(now) {\n\t\t\t\t\tmsg.NakWithDelay(workItem.Expiry.Sub(now))\n\t\t\t\t} else {\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < 25_000; i++ {\n\t\t// Publish item to TEST_ACTIVE_WORK_ITEMS stream with an expiry time.\n\t\tworkItem := ActiveWorkItem{ID: i, Expiry: time.Now().Add(30 * time.Second)}\n\t\tdata, err := json.Marshal(workItem)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(fmt.Sprintf(\"TEST_ACTIVE_WORK_ITEMS.%d\", i), data)\n\t\trequire_NoError(t, err)\n\n\t\t// Update expiry time and republish item to TEST_ACTIVE_WORK_ITEMS stream.\n\t\tworkItem.Expiry = time.Now().Add(3 * time.Second)\n\t\tdata, err = json.Marshal(workItem)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(fmt.Sprintf(\"TEST_ACTIVE_WORK_ITEMS.%d\", i), data)\n\t\trequire_NoError(t, err)\n\t}\n\tnoChange := false\n\tlastNumAckPending := 0\n\tcheckFor(t, 60*time.Second, 3*time.Second, func() error {\n\t\tselect {\n\t\tcase err := <-errs:\n\t\t\tt.Fatalf(\"consumer goroutine failed: %v\", err)\n\t\tdefault:\n\t\t}\n\t\tci, err := js.ConsumerInfo(\"TEST_ACTIVE_WORK_ITEMS\", \"testactiveworkitemsconsumer\")\n\t\trequire_NoError(t, err)\n\n\t\tif lastNumAckPending != 0 && lastNumAckPending == ci.NumAckPending {\n\t\t\tnoChange = true\n\t\t}\n\t\tlastNumAckPending = ci.NumAckPending\n\n\t\t// If we have no change since last check, we can fail the test before `totalWait` timeout.\n\t\tif ci.NumAckPending > 0 && ci.NumPending == 0 {\n\t\t\tif noChange {\n\t\t\t\t_, err := sub.Fetch(1)\n\t\t\t\tif err != nil && errors.Is(err, nats.ErrTimeout) {\n\n\t\t\t\t\tt.Fatalf(\"num ack pending: %d\\t num pending: %v\\n\", ci.NumAckPending, ci.NumPending)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"num ack pending: %d\\t num pending: %v\\n\", ci.NumAckPending, ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamConsumerPinned(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\", \"bar\", \"baz\"},\n\t\tRetention: LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo.>\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPinnedClient,\n\t\tAckPolicy: AckExplicit,\n\t\tPinnedTTL: 10 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := nats.NewMsg(fmt.Sprintf(\"foo.%d\", i))\n\t\tmsg.Data = []byte(fmt.Sprintf(\"msg-%d\", i))\n\t\t// Add headers to check if we properly serialize Nats-Pin-Id with and without headers.\n\t\tif i%2 == 0 {\n\t\t\tmsg.Header.Add(\"Some-Header\", \"Value\")\n\t\t}\n\t\t_, err = js.PublishMsg(msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\treq := JSApiConsumerGetNextRequest{Batch: 3, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\treqb, _ := json.Marshal(req)\n\treply := \"ONE\"\n\treplies, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb))\n\n\treply2 := \"TWO\"\n\treplies2, err := nc.SubscribeSync(reply2)\n\trequire_NoError(t, err)\n\tdefer replies2.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply2, reqb))\n\n\t// This is the first Pull Request, so it should become the pinned one.\n\tmsg, err := replies.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\t// Check if we are really pinned.\n\tpinned := msg.Header.Get(\"Nats-Pin-Id\")\n\tif pinned == \"\" {\n\t\tt.Fatalf(\"Expected pinned message, got none\")\n\t}\n\n\t// Here, we should have pull request that just idles, as it is not pinned.\n\t_, err = replies2.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\t// While the pinned one continues to get messages.\n\tmsg, err = replies.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\n\t// Just making sure that the other one does not get round-robined message.\n\t_, err = replies2.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\t// Now let's send a request with wrong pinned id.\n\treq = JSApiConsumerGetNextRequest{Batch: 3, Expires: 250 * time.Millisecond, PriorityGroup: PriorityGroup{\n\t\tId: \"WRONG\",\n\t\tGroup: \"A\",\n\t}}\n\treqBad, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\treplies3, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies3.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqBad))\n\n\t// and make sure we got error telling us it's wrong ID.\n\tmsg, err = replies3.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tif msg.Header.Get(\"Status\") != \"423\" {\n\t\tt.Fatalf(\"Expected 423, got %v\", msg.Header.Get(\"Status\"))\n\t}\n\t// Send a new request with a good pinned ID.\n\treq = JSApiConsumerGetNextRequest{Batch: 3, Expires: 250 * time.Millisecond, PriorityGroup: PriorityGroup{\n\t\tId: pinned,\n\t\tGroup: \"A\",\n\t}}\n\treqb, _ = json.Marshal(req)\n\treply = \"FOUR\"\n\treplies4, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies4.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb))\n\n\t// and check that we got a message.\n\tmsg, err = replies4.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\n\tadvisories, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.CONSUMER.*.TEST.C\")\n\trequire_NoError(t, err)\n\n\t// Send a new request without pin ID, which should work after the TTL.\n\treq = JSApiConsumerGetNextRequest{Batch: 3, Expires: 50 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\treqb, _ = json.Marshal(req)\n\treply = \"FIVE\"\n\treplies5, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies5.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb))\n\n\tcheckFor(t, 20*time.Second, 1*time.Second, func() error {\n\t\t_, err = replies5.NextMsg(500 * time.Millisecond)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t})\n\n\tadvisory, err := advisories.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, fmt.Sprintf(\"%s.TEST.C\", JSAdvisoryConsumerUnpinnedPre), advisory.Subject)\n\tadvisory, err = advisories.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, fmt.Sprintf(\"%s.TEST.C\", JSAdvisoryConsumerPinnedPre), advisory.Subject)\n\n\t// Manually unpin the current fetch request.\n\trequest := JSApiConsumerUnpinRequest{Group: \"A\"}\n\trequestData, err := json.Marshal(request)\n\trequire_NoError(t, err)\n\tmsg, err = nc.Request(\"$JS.API.CONSUMER.UNPIN.TEST.C\", requestData, time.Second*1)\n\trequire_NoError(t, err)\n\n\tvar response JSApiConsumerUnpinResponse\n\terr = json.Unmarshal(msg.Data, &response)\n\trequire_NoError(t, err)\n\trequire_True(t, response.Error == nil)\n\n\t// check if we got proper advisories.\n\tadvisory, err = advisories.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, fmt.Sprintf(\"%s.TEST.C\", JSAdvisoryConsumerUnpinnedPre), advisory.Subject)\n\n\treplies6, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies6.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqBad))\n\n\t_, err = replies6.NextMsg(time.Second * 5)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamConsumerPinnedUnsetsAfterAtMostPinnedTTL(t *testing.T) {\n\ttest := func(t *testing.T, publish bool) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tacc := s.GlobalAccount()\n\n\t\tmset, err := acc.addStream(&StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tRetention: LimitsPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tif publish {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\tDurable: \"C\",\n\t\t\tFilterSubject: \"foo\",\n\t\t\tPriorityGroups: []string{\"A\"},\n\t\t\tPriorityPolicy: PriorityPinnedClient,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tPinnedTTL: time.Second,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// We have a too large expiry with respect to the PinnedTTL, we can only keep the pin for 2xPinnedTTL at most.\n\t\treq := JSApiConsumerGetNextRequest{Batch: 2, Expires: 2 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\"}}\n\t\treqb, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\treply := \"ONE\"\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\t\trequire_NoError(t, nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"C\"), reply, reqb))\n\n\t\t// The pin should be held for PinnedTTL at most, especially since the expiry is way larger.\n\t\tvar pinId string\n\t\tstart := time.Now()\n\t\tfor time.Since(start) < 750*time.Millisecond {\n\t\t\to.mu.RLock()\n\t\t\tcurrentPinId := o.currentPinId\n\t\t\to.mu.RUnlock()\n\t\t\tif pinId == _EMPTY_ {\n\t\t\t\tpinId = currentPinId\n\t\t\t} else {\n\t\t\t\t// The pin ID shouldn't change.\n\t\t\t\trequire_Equal(t, pinId, currentPinId)\n\t\t\t}\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t}\n\n\t\t// If a message was published, we should be pinned when this message was delivered.\n\t\tif publish {\n\t\t\trequire_NotEqual(t, pinId, _EMPTY_)\n\n\t\t\t// Wait some time so we can check we're not unpinned. After PinnedTTL it should be reset,\n\t\t\t// even if the client's expiry was longer or its pull request is still there.\n\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t}\n\n\t\t// If we didn't get any messages delivered (or after timeout) the consumer should be unpinned.\n\t\to.mu.RLock()\n\t\tcurrentPinId := o.currentPinId\n\t\to.mu.RUnlock()\n\t\trequire_Equal(t, currentPinId, _EMPTY_)\n\n\t\t// Finally, check that:\n\t\t// - We received the published message first (if it was published).\n\t\t// - We receive the request timeout for the pull expiring.\n\t\tif publish {\n\t\t\tmsg, err := sub.NextMsg(200 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Header.Get(\"Nats-Pin-Id\"), pinId)\n\t\t\trequire_Equal(t, msg.Subject, \"foo\")\n\t\t}\n\t\tmsg, err := sub.NextMsg(3 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"408\")\n\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Request Timeout\")\n\t}\n\tt.Run(\"Publish\", func(t *testing.T) { test(t, true) })\n\tt.Run(\"NoMessages\", func(t *testing.T) { test(t, false) })\n}\n\nfunc TestJetStreamConsumerPinnedUnsubscribeOnPinned(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\", \"bar\", \"baz\"},\n\t\tRetention: LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo.>\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPinnedClient,\n\t\tAckPolicy: AckExplicit,\n\t\tPinnedTTL: time.Second,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo.1\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n\n\treq := JSApiConsumerGetNextRequest{Batch: 3, Expires: 5 * time.Second, Heartbeat: 250 * time.Millisecond, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\treqb, _ := json.Marshal(req)\n\treply := \"ONE\"\n\treplies, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb))\n\n\treply2 := \"TWO\"\n\treplies2, err := nc.SubscribeSync(reply2)\n\trequire_NoError(t, err)\n\tdefer replies2.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply2, reqb))\n\n\t// This is the first Pull Request, so it should become the pinned one.\n\tmsg, err := replies.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\t// Check if we are really pinned.\n\tpinned := msg.Header.Get(\"Nats-Pin-Id\")\n\tif pinned == \"\" {\n\t\tt.Fatalf(\"Expected pinned message, got none\")\n\t}\n\n\terr = replies.Unsubscribe()\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo.1\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n\n\t// Here, we should receive a heartbeat message.\n\tmsg, err = replies2.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"100\")\n\n\t// receive heartbeats until pinned ttl expires\n\t// and we should get a new message with new pin id\n\tfor {\n\t\tmsg, err = replies2.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\t\tif msg.Header.Get(\"Status\") == \"100\" {\n\t\t\tcontinue\n\t\t}\n\t\tif msg.Header.Get(\"Nats-Pin-Id\") == \"\" {\n\t\t\tt.Fatalf(\"Expected pinned message, got none\")\n\t\t}\n\t\tbreak\n\t}\n}\n\n// This tests if Unpin works correctly when there are no pending messages.\n// It checks if the next pinned client will be different than the first one\n// after new messages is published.\nfunc TestJetStreamConsumerUnpinNoMessages(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPinnedClient,\n\t\tAckPolicy: AckExplicit,\n\t\tPinnedTTL: 30 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\treq := JSApiConsumerGetNextRequest{Batch: 30, Expires: 60 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\treqb, _ := json.Marshal(req)\n\treply := \"ONE\"\n\treplies, err := nc.SubscribeSync(reply)\n\trequire_NoError(t, err)\n\tdefer replies.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb))\n\n\treply2 := \"TWO\"\n\treplies2, err := nc.SubscribeSync(reply2)\n\trequire_NoError(t, err)\n\tdefer replies2.Drain()\n\trequire_NoError(t, nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply2, reqb))\n\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\n\tmsg, err := replies.NextMsg(1 * time.Second)\n\tpinId := msg.Header.Get(\"Nats-Pin-Id\")\n\trequire_NotEqual(t, pinId, \"\")\n\trequire_NoError(t, err)\n\t_, err = replies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\n\t_, err = replies2.NextMsg(1 * time.Second)\n\trequire_Error(t, err)\n\n\tunpinRequest := func(t *testing.T, nc *nats.Conn, stream, consumer, group string) *ApiError {\n\t\tvar response JSApiConsumerUnpinResponse\n\t\trequest := JSApiConsumerUnpinRequest{Group: group}\n\t\trequestData, err := json.Marshal(request)\n\t\trequire_NoError(t, err)\n\t\tmsg, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.UNPIN.%s.%s\", stream, consumer), requestData, time.Second*1)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(msg.Data, &response)\n\t\trequire_NoError(t, err)\n\t\treturn response.Error\n\t}\n\n\tunpinError := unpinRequest(t, nc, \"TEST\", \"C\", \"A\")\n\trequire_True(t, unpinError == nil)\n\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\n\t// Old pinned client should get info that it is no longer pinned.\n\tmsg, err = replies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"423\")\n\n\t// While the new one should get the message and new pin.\n\tmsg, err = replies2.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, string(msg.Data), \"data\")\n\trequire_NotEqual(t, msg.Header.Get(\"Nats-Pin-Id\"), pinId)\n}\n\n// In some scenarios, if the next waiting request is the same as the old pinned, it could be picked as a new pin.\n// This test replicates that behavior and checks if the new pin is different than the old one.\nfunc TestJetStreamConsumerUnpinPickDifferentRequest(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPinnedClient,\n\t\tAckPolicy: AckExplicit,\n\t\tPinnedTTL: 30 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\n\treq := JSApiConsumerGetNextRequest{Batch: 5, Expires: 15 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\n\treqBytes, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\n\tfirstInbox := \"FIRST\"\n\tfirstReplies, err := nc.SubscribeSync(firstInbox)\n\trequire_NoError(t, err)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", firstInbox, reqBytes)\n\n\tmsg, err := firstReplies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\tpinId := msg.Header.Get(\"Nats-Pin-Id\")\n\trequire_NotEqual(t, pinId, \"\")\n\n\treqPinned := JSApiConsumerGetNextRequest{Batch: 5, Expires: 15 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t\tId: pinId,\n\t}}\n\t_, err = json.Marshal(reqPinned)\n\trequire_NoError(t, err)\n\n\tsecondInbox := \"SECOND\"\n\tsecondReplies, err := nc.SubscribeSync(secondInbox)\n\trequire_NoError(t, err)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", secondInbox, reqBytes)\n\n\t_, err = secondReplies.NextMsg(1 * time.Second)\n\trequire_Error(t, err)\n\n\tunpinRequest := func(t *testing.T, nc *nats.Conn, stream, consumer, group string) *ApiError {\n\t\tvar response JSApiConsumerUnpinResponse\n\t\trequest := JSApiConsumerUnpinRequest{Group: group}\n\t\trequestData, err := json.Marshal(request)\n\t\trequire_NoError(t, err)\n\t\tmsg, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.UNPIN.%s.%s\", stream, consumer), requestData, time.Second*1)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(msg.Data, &response)\n\t\trequire_NoError(t, err)\n\t\treturn response.Error\n\t}\n\n\tunpinRequest(t, nc, \"TEST\", \"C\", \"A\")\n\t_, err = firstReplies.NextMsg(1 * time.Second)\n\t// If there are no messages in the stream, do not expect unpin message to arrive.\n\t// Advisory will be sent immediately, but messages with headers - only when there is anything to be sent.\n\trequire_Error(t, err)\n\t// Send a new message to the stream.\n\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\t// Check if the old pinned will get the information about bad pin.\n\tmsg, err = firstReplies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"423\")\n\t// Make sure that the old pin is cleared.\n\trequire_Equal(t, msg.Header.Get(\"Nats-Pin-Id\"), \"\")\n\n\t// Try different wr.\n\tmsg, err = secondReplies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\t// Make sure that its pin is different than the old one and not empty.\n\trequire_NotEqual(t, msg.Header.Get(\"Nats-Pin-Id\"), pinId)\n\trequire_NotEqual(t, msg.Header.Get(\"Nats-Pin-Id\"), \"\")\n}\n\nfunc TestJetStreamConsumerPinnedTTL(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPinnedClient,\n\t\tAckPolicy: AckExplicit,\n\t\tPinnedTTL: 3 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"data\")\n\t}\n\n\treq := JSApiConsumerGetNextRequest{Batch: 1, Expires: 10 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\n\treqBytes, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\n\tfirstInbox := \"FIRST\"\n\tfirstReplies, err := nc.SubscribeSync(firstInbox)\n\trequire_NoError(t, err)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", firstInbox, reqBytes)\n\n\tmsg, err := firstReplies.NextMsg(1 * time.Second)\n\trequire_NoError(t, err)\n\tpinId := msg.Header.Get(\"Nats-Pin-Id\")\n\trequire_NotEqual(t, pinId, \"\")\n\n\tsecondInbox := \"SECOND\"\n\tsecondReplies, err := nc.SubscribeSync(secondInbox)\n\trequire_NoError(t, err)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", secondInbox, reqBytes)\n\n\t// Expect error, as first request should be still pinned.\n\t_, err = secondReplies.NextMsg(1 * time.Second)\n\trequire_Error(t, err)\n\n\t// During the 5 second window, the first Pin should time out and this request\n\t// should become the pinned one and get the message.\n\tmsg, err = secondReplies.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\tnewPinId := msg.Header.Get(\"Nats-Pin-Id\")\n\trequire_NotEqual(t, newPinId, pinId)\n\trequire_NotEqual(t, newPinId, \"\")\n\n\tthirdInbox := \"THIRD\"\n\tthirdReplies, err := nc.SubscribeSync(thirdInbox)\n\trequire_NoError(t, err)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", thirdInbox, reqBytes)\n\n\t// The same process as above, but tests different codepath - one where Pin\n\t// is set on existing waiting request.\n\tmsg, err = thirdReplies.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\trequire_NotEqual(t, msg.Header.Get(\"Nats-Pin-Id\"), pinId)\n\trequire_NotEqual(t, msg.Header.Get(\"Nats-Pin-Id\"), newPinId)\n\trequire_NotEqual(t, newPinId, \"\")\n\n}\n\nfunc TestJetStreamConsumerUnpin(t *testing.T) {\n\tsingle := RunBasicJetStreamServer(t)\n\tdefer single.Shutdown()\n\tnc, js := jsClientConnect(t, single)\n\tdefer nc.Close()\n\n\tcluster := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer cluster.shutdown()\n\tcnc, cjs := jsClientConnect(t, cluster.randomServer())\n\tdefer cnc.Close()\n\n\t// Create a stream and consumer for both single server and clustered mode.\n\tfor _, server := range []struct {\n\t\treplicas int\n\t\tjs nats.JetStreamContext\n\t\tnc *nats.Conn\n\t}{\n\t\t{1, js, nc},\n\t\t{3, cjs, cnc},\n\t} {\n\n\t\t_, err := server.js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.>\", \"bar\", \"baz\"},\n\t\t\tReplicas: server.replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tconsumerConfig := CreateConsumerRequest{\n\t\t\tStream: \"TEST\",\n\t\t\tAction: ActionCreate,\n\t\t\tConfig: ConsumerConfig{\n\t\t\t\tDurable: \"C\",\n\t\t\t\tFilterSubject: \"foo.>\",\n\t\t\t\tPriorityGroups: []string{\"A\"},\n\t\t\t\tPriorityPolicy: PriorityPinnedClient,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tPinnedTTL: 10 * time.Second,\n\t\t\t},\n\t\t}\n\t\treq, err := json.Marshal(consumerConfig)\n\t\trequire_NoError(t, err)\n\t\trmsg, err := server.nc.Request(fmt.Sprintf(JSApiDurableCreateT, consumerConfig.Stream, consumerConfig.Config.Durable), req, 5*time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar resp JSApiConsumerCreateResponse\n\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, resp.Error == nil)\n\n\t}\n\tcluster.waitOnStreamLeader(\"$G\", \"TEST\")\n\tcluster.waitOnConsumerLeader(\"$G\", \"TEST\", \"C\")\n\n\tunpinRequest := func(t *testing.T, nc *nats.Conn, stream, consumer, group string) *ApiError {\n\t\tvar response JSApiConsumerUnpinResponse\n\t\trequest := JSApiConsumerUnpinRequest{Group: group}\n\t\trequestData, err := json.Marshal(request)\n\t\trequire_NoError(t, err)\n\t\tmsg, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.UNPIN.%s.%s\", stream, consumer), requestData, time.Second*1)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(msg.Data, &response)\n\t\trequire_NoError(t, err)\n\t\treturn response.Error\n\t}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnc *nats.Conn\n\t\tstream string\n\t\tconsumer string\n\t\tgroup string\n\t\terr *ApiError\n\t}{\n\t\t{\"unpin non-existing group\", nc, \"TEST\", \"C\", \"B\", &ApiError{ErrCode: uint16(JSConsumerInvalidPriorityGroupErr)}},\n\t\t{\"unpin on missing stream\", nc, \"NOT_EXIST\", \"C\", \"A\", &ApiError{ErrCode: uint16(JSStreamNotFoundErr)}},\n\t\t{\"unpin on missing consumer\", nc, \"TEST\", \"NOT_EXIST\", \"A\", &ApiError{ErrCode: uint16(JSConsumerNotFoundErr)}},\n\t\t{\"unpin missing group\", nc, \"TEST\", \"C\", \"\", &ApiError{ErrCode: uint16(JSInvalidJSONErr)}},\n\t\t{\"unpin bad group name\", nc, \"TEST\", \"C\", \"group name\\r\\n\", &ApiError{ErrCode: uint16(JSConsumerInvalidGroupNameErr)}},\n\t\t{\"ok unpin\", nc, \"TEST\", \"C\", \"A\", nil},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\terr := unpinRequest(t, nc, test.stream, test.consumer, test.group)\n\t\t\tif test.err != nil {\n\t\t\t\trequire_True(t, err.ErrCode == test.err.ErrCode)\n\t\t\t} else {\n\t\t\t\trequire_True(t, err == nil)\n\t\t\t}\n\t\t})\n\t\tt.Run(fmt.Sprintf(\"%s clustered\", test.name), func(t *testing.T) {\n\t\t\terr := unpinRequest(t, cnc, test.stream, test.consumer, test.group)\n\t\t\tif test.err != nil {\n\t\t\t\trequire_True(t, err.ErrCode == test.err.ErrCode)\n\t\t\t} else {\n\t\t\t\trequire_True(t, err == nil)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerWithPriorityGroups(t *testing.T) {\n\tsingle := RunBasicJetStreamServer(t)\n\tdefer single.Shutdown()\n\tnc, js := jsClientConnect(t, single)\n\tdefer nc.Close()\n\n\tcluster := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer cluster.shutdown()\n\tcnc, cjs := jsClientConnect(t, cluster.randomServer())\n\tdefer cnc.Close()\n\n\t// Create a stream and consumer for both single server and clustered mode.\n\tfor _, server := range []struct {\n\t\treplicas int\n\t\tjs nats.JetStreamContext\n\t}{\n\t\t{1, js},\n\t\t{3, cjs},\n\t} {\n\n\t\t_, err := server.js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.>\", \"bar\", \"baz\"},\n\t\t\tReplicas: server.replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\tcluster.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnc *nats.Conn\n\t\tstream string\n\t\tconsumer string\n\t\tgroups []string\n\t\tmode PriorityPolicy\n\t\tdeliverSubject string\n\t\terr *ApiError\n\t}{\n\t\t{\"Pinned Consumer with Priority Group\", nc, \"TEST\", \"PINNED\", []string{\"A\"}, PriorityPinnedClient, \"\", nil},\n\t\t{\"Pinned Consumer with Priority Group, clustered\", cnc, \"TEST\", \"PINNED\", []string{\"A\"}, PriorityPinnedClient, \"\", nil},\n\t\t{\"Overflow Consumer with Priority Group\", nc, \"TEST\", \"OVERFLOW\", []string{\"A\"}, PriorityOverflow, \"\", nil},\n\t\t{\"Overflow Consumer with Priority Group, clustered\", cnc, \"TEST\", \"OVERFLOW\", []string{\"A\"}, PriorityOverflow, \"\", nil},\n\t\t{\"Pinned Consumer without Priority Group\", nc, \"TEST\", \"PINNED_NO_GROUP\", nil, PriorityPinnedClient, \"\", &ApiError{ErrCode: uint16(JSConsumerPriorityPolicyWithoutGroup)}},\n\t\t{\"Pinned Consumer without Priority Group, clustered\", cnc, \"TEST\", \"PINNED_NO_GROUP\", nil, PriorityPinnedClient, \"\", &ApiError{ErrCode: uint16(JSConsumerPriorityPolicyWithoutGroup)}},\n\t\t{\"Overflow Consumer without Priority Group\", nc, \"TEST\", \"PINNED_NO_GROUP\", nil, PriorityOverflow, \"\", &ApiError{ErrCode: uint16(JSConsumerPriorityPolicyWithoutGroup)}},\n\t\t{\"Overflow Consumer without Priority Group, clustered\", cnc, \"TEST\", \"PINNED_NO_GROUP\", nil, PriorityOverflow, \"\", &ApiError{ErrCode: uint16(JSConsumerPriorityPolicyWithoutGroup)}},\n\t\t{\"Pinned Consumer with empty Priority Group\", nc, \"TEST\", \"PINNED_NO_GROUP\", []string{\"\"}, PriorityPinnedClient, \"\", &ApiError{ErrCode: uint16(JSConsumerEmptyGroupName)}},\n\t\t{\"Pinned Consumer with empty Priority Group, clustered\", cnc, \"TEST\", \"PINNED_NO_GROUP\", []string{\"\"}, PriorityPinnedClient, \"\", &ApiError{ErrCode: uint16(JSConsumerEmptyGroupName)}},\n\t\t{\"Pinned Consumer with empty Priority Group\", nc, \"TEST\", \"PINNED_NO_GROUP\", []string{\"\"}, PriorityOverflow, \"\", &ApiError{ErrCode: uint16(JSConsumerEmptyGroupName)}},\n\t\t{\"Pinned Consumer with empty Priority Group, clustered\", cnc, \"TEST\", \"PINNED_NO_GROUP\", []string{\"\"}, PriorityOverflow, \"\", &ApiError{ErrCode: uint16(JSConsumerEmptyGroupName)}},\n\t\t{\"Consumer with `none` policy priority and no pinned TTL set\", nc, \"TEST\", \"NONE\", []string{}, PriorityNone, \"\", &ApiError{ErrCode: uint16(JSConsumerPinnedTTLWithoutPriorityPolicyNone)}},\n\t\t{\"Consumer with `none` policy priority and Priority Group set\", nc, \"TEST\", \"NONE_WITH_GROUPS\", []string{\"A\"}, PriorityNone, \"\", &ApiError{ErrCode: uint16(JSConsumerPriorityGroupWithPolicyNone)}},\n\t\t{\"Push consumer with Priority Group\", nc, \"TEST\", \"PUSH_WITH_POLICY\", []string{\"A\"}, PriorityOverflow, \"subject\", &ApiError{ErrCode: uint16(JSConsumerPushWithPriorityGroupErr)}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\tconsumerConfig := CreateConsumerRequest{\n\t\t\t\tStream: \"TEST\",\n\t\t\t\tAction: ActionCreate,\n\t\t\t\tConfig: ConsumerConfig{\n\t\t\t\t\tDurable: test.consumer,\n\t\t\t\t\tFilterSubject: \"foo.>\",\n\t\t\t\t\tPriorityGroups: test.groups,\n\t\t\t\t\tPriorityPolicy: test.mode,\n\t\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\t\tDeliverSubject: test.deliverSubject,\n\t\t\t\t\tPinnedTTL: 10 * time.Second,\n\t\t\t\t},\n\t\t\t}\n\t\t\treq, err := json.Marshal(consumerConfig)\n\t\t\trequire_NoError(t, err)\n\t\t\trmsg, err := test.nc.Request(fmt.Sprintf(JSApiDurableCreateT, consumerConfig.Stream, consumerConfig.Config.Durable), req, 5*time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar resp JSApiConsumerCreateResponse\n\t\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.err != nil {\n\t\t\t\trequire_True(t, resp.Error.ErrCode == test.err.ErrCode)\n\t\t\t} else {\n\t\t\t\trequire_True(t, resp.Error == nil)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerPriorityPullRequests(t *testing.T) {\n\tsingle := RunBasicJetStreamServer(t)\n\tdefer single.Shutdown()\n\tnc, js := jsClientConnect(t, single)\n\tdefer nc.Close()\n\n\tcluster := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer cluster.shutdown()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"pinned.>\", \"overflow.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"STANDARD\",\n\t})\n\trequire_NoError(t, err)\n\n\tconsumerConfig := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"PINNED\",\n\t\t\tFilterSubject: \"pinned.>\",\n\t\t\tPriorityGroups: []string{\"A\"},\n\t\t\tPriorityPolicy: PriorityPinnedClient,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tPinnedTTL: 10 * time.Second,\n\t\t},\n\t}\n\treq, err := json.Marshal(consumerConfig)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, consumerConfig.Stream, consumerConfig.Config.Durable), req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error == nil)\n\n\tconsumerConfig = CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tAction: ActionCreate,\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"OVERFLOW\",\n\t\t\tFilterSubject: \"overflow.>\",\n\t\t\tPriorityGroups: []string{\"A\"},\n\t\t\tPriorityPolicy: PriorityOverflow,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tPinnedTTL: 5 * time.Second,\n\t\t},\n\t}\n\treq, err = json.Marshal(consumerConfig)\n\trequire_NoError(t, err)\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, consumerConfig.Stream, consumerConfig.Config.Durable), req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_True(t, resp.Error == nil)\n\n\tfor i := 0; i < 50; i++ {\n\t\tsendStreamMsg(t, nc, \"pinned.1\", fmt.Sprintf(\"msg-%d\", i))\n\t\tsendStreamMsg(t, nc, \"overflow.1\", fmt.Sprintf(\"msg-%d\", i))\n\t}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnc *nats.Conn\n\t\tconsumer string\n\t\trequest JSApiConsumerGetNextRequest\n\t\tdescription string\n\t}{\n\t\t{\"Pinned Pull Request\", nc, \"PINNED\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\"}}, \"\"},\n\t\t{\"Pinned Pull Request, no group\", nc, \"PINNED\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{}}, \"Bad Request - Priority Group missing\"},\n\t\t{\"Pinned Pull Request, bad group\", nc, \"PINNED\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"Bad\"}}, \"Bad Request - Invalid Priority Group\"},\n\t\t{\"Pinned Pull Request, against Overflow\", nc, \"OVERFLOW\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", Id: \"PINNED-ID\"}}, \"Bad Request - Not a Pinned Client Priority consumer\"},\n\t\t{\"Pinned Pull Request, against standard consumer\", nc, \"STANDARD\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", Id: \"PINNED-ID\"}}, \"Bad Request - Not a Pinned Client Priority consumer\"},\n\t\t{\"Overflow Pull Request, overflow below threshold\", nc, \"OVERFLOW\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", MinPending: 1000}}, \"Request Timeout\"},\n\t\t{\"Overflow Pull Request, overflow above threshold\", nc, \"OVERFLOW\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", MinPending: 10}}, \"\"},\n\t\t{\"Overflow Pull Request, minPending OR minAckPending above threshold\", nc, \"OVERFLOW\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", MinPending: 10, MinAckPending: 5}}, \"\"},\n\t\t{\"Overflow Pull Request, against pinned\", nc, \"PINNED\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", MinPending: 10}}, \"Bad Request - Not a Overflow Priority consumer\"},\n\t\t{\"Overflow Pull Request, against standard consumer\", nc, \"STANDARD\", JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second, PriorityGroup: PriorityGroup{Group: \"A\", MinPending: 10}}, \"Bad Request - Not a Overflow Priority consumer\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tinbox := nats.NewInbox()\n\t\t\treplies, err := test.nc.SubscribeSync(inbox)\n\t\t\treqb, _ := json.Marshal(test.request)\n\t\t\trequire_NoError(t, err)\n\t\t\tnc.PublishRequest(fmt.Sprintf(\"$JS.API.CONSUMER.MSG.NEXT.TEST.%s\", test.consumer), inbox, reqb)\n\t\t\trequire_NoError(t, err)\n\t\t\tmsg, err := replies.NextMsg(10 * time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, test.description, msg.Header.Get(\"Description\"))\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerOverflow(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\", \"bar\", \"baz\"},\n\t\tRetention: LimitsPolicy,\n\t\tStorage: FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubject: \"foo.>\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityOverflow,\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo.1\", \"msg-1\")\n\n\t// nothing unacked, so should return nothing.\n\treq := JSApiConsumerGetNextRequest{Batch: 1, Expires: 90 * time.Second, PriorityGroup: PriorityGroup{\n\t\tMinAckPending: 1,\n\t\tGroup: \"A\",\n\t}}\n\tackPending1 := sendRequest(t, nc, \"ackPending\", req)\n\t_, err = ackPending1.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\t// one pending message, so should return it.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 90 * time.Second, PriorityGroup: PriorityGroup{\n\t\tMinPending: 1,\n\t\tGroup: \"A\",\n\t}}\n\tnumPending1 := sendRequest(t, nc, \"singleOverflow\", req)\n\tmsg, err := numPending1.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\n\tsendStreamMsg(t, nc, \"foo.1\", \"msg-2\")\n\tsendStreamMsg(t, nc, \"foo.1\", \"msg-3\")\n\n\t// overflow set to 10, so we should not get any messages, as there are only few pending.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 90 * time.Second, PriorityGroup: PriorityGroup{\n\t\tMinPending: 10,\n\t\tGroup: \"A\",\n\t}}\n\tnumPending10 := sendRequest(t, nc, \"overflow\", req)\n\t_, err = numPending10.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\t// without overflow, we should get messages.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 90 * time.Second}\n\tfetchNoOverflow := sendRequest(t, nc, \"without_overflow\", req)\n\tnoOverflowMsg, err := fetchNoOverflow.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, noOverflowMsg)\n\n\t// Now add more messages.\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo.1\", \"msg-1\")\n\t}\n\n\t// and previous batch should receive messages now.\n\tmsg, err = numPending10.NextMsg(time.Second * 5)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n\n\t// But one with max ack pending should get nothing.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 90 * time.Second, PriorityGroup: PriorityGroup{\n\t\tMinAckPending: 50,\n\t\tGroup: \"A\",\n\t}}\n\tmaxAckPending50 := sendRequest(t, nc, \"maxAckPending\", req)\n\t_, err = maxAckPending50.NextMsg(time.Second)\n\trequire_Error(t, err)\n\n\t// However, when we miss a lot of acks, we should get messages on overflow with max ack pending.\n\treq = JSApiConsumerGetNextRequest{Batch: 200, Expires: 90 * time.Second, PriorityGroup: PriorityGroup{\n\t\tGroup: \"A\",\n\t}}\n\tfetchNoOverflow = sendRequest(t, nc, \"without_overflow\", req)\n\tnoOverflowMsg, err = fetchNoOverflow.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, noOverflowMsg)\n\n\tmsg, err = maxAckPending50.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, msg)\n}\n\nfunc TestJetStreamConsumerMultipleFitersWithStartDate(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tpast := time.Now().Add(-90 * time.Second)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"events.foo\", \"msg-1\")\n\tsendStreamMsg(t, nc, \"events.bar\", \"msg-2\")\n\tsendStreamMsg(t, nc, \"events.baz\", \"msg-3\")\n\tsendStreamMsg(t, nc, \"events.biz\", \"msg-4\")\n\tsendStreamMsg(t, nc, \"events.faz\", \"msg-5\")\n\tsendStreamMsg(t, nc, \"events.foo\", \"msg-6\")\n\tsendStreamMsg(t, nc, \"events.biz\", \"msg-7\")\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tfilterSubjects []string\n\t\tstartTime time.Time\n\t\texpectedMessages uint64\n\t\texpectedStreamSequence uint64\n\t}{\n\t\t{\"Single-Filter-first-sequence\", []string{\"events.foo\"}, past, 2, 0},\n\t\t{\"Multiple-Filter-first-sequence\", []string{\"events.foo\", \"events.bar\", \"events.baz\"}, past, 4, 0},\n\t\t{\"Multiple-Filters-second-subject\", []string{\"events.bar\", \"events.baz\"}, past, 2, 1},\n\t\t{\"Multiple-Filters-first-last-subject\", []string{\"events.foo\", \"events.biz\"}, past, 4, 0},\n\t\t{\"Multiple-Filters-in-future\", []string{\"events.foo\", \"events.biz\"}, time.Now().Add(1 * time.Minute), 0, 7},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tinfo, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\t\tDurable: test.name,\n\t\t\t\tFilterSubjects: test.filterSubjects,\n\t\t\t\tDeliverPolicy: nats.DeliverByStartTimePolicy,\n\t\t\t\tOptStartTime: &test.startTime,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, test.expectedStreamSequence, info.Delivered.Stream)\n\t\t\trequire_Equal(t, test.expectedMessages, info.NumPending)\n\t\t})\n\t}\n\n}\n\nfunc TestPriorityGroupNameRegex(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tgroup string\n\t\tvalid bool\n\t}{\n\t\t{\"valid-short\", \"A\", true},\n\t\t{\"valid-with-accepted-special-chars\", \"group/consumer=A\", true},\n\t\t{\"empty\", \"\", false},\n\t\t{\"with-space\", \"A B\", false},\n\t\t{\"with-tab\", \"A B\", false},\n\t\t{\"too-long-name\", \"group-name-that-is-too-long\", false},\n\t\t{\"line-termination\", \"\\r\\n\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\trequire_Equal(t, test.valid, validGroupName.MatchString(test.group))\n\t\t})\n\t}\n}\n\nfunc sendRequest(t *testing.T, nc *nats.Conn, reply string, req JSApiConsumerGetNextRequest) *nats.Subscription {\n\treqb, _ := json.Marshal(req)\n\treplies, err := nc.SubscribeSync(reply)\n\tnc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.C\", reply, reqb)\n\trequire_NoError(t, err)\n\treturn replies\n}\n\nfunc Benchmark____JetStreamConsumerIsFilteredMatch(b *testing.B) {\n\tsubject := \"foo.bar.do.not.match.any.filter.subject\"\n\tfor n := 1; n <= 1024; n *= 2 {\n\t\tname := fmt.Sprintf(\"%d filter subjects\", int(n))\n\t\tc := consumerWithFilterSubjects(filterSubjects(int(n)))\n\t\tb.Run(name, func(b *testing.B) {\n\t\t\tc.isFilteredMatch(subject)\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/6085\nfunc TestJetStreamConsumerBackoffNotRespectedWithMultipleInflightRedeliveries(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmaxDeliver := 3\n\tbackoff := []time.Duration{2 * time.Second, 4 * time.Second}\n\tsub, err := js.SubscribeSync(\n\t\t\"events.>\",\n\t\tnats.MaxDeliver(maxDeliver),\n\t\tnats.BackOff(backoff),\n\t\tnats.AckExplicit(),\n\t)\n\trequire_NoError(t, err)\n\n\tcalculateExpectedBackoff := func(numDelivered int) time.Duration {\n\t\texpectedBackoff := 500 * time.Millisecond\n\t\tfor i := 0; i < numDelivered-1 && i < len(backoff); i++ {\n\t\t\texpectedBackoff += backoff[i]\n\t\t}\n\t\treturn expectedBackoff\n\t}\n\n\t// We get one message to be redelivered using the final backoff duration.\n\tfirstMsgSent := time.Now()\n\tsendStreamMsg(t, nc, \"events.first\", \"msg-1\")\n\t_, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(1))\n\t_, err = sub.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(2))\n\t// This message will be redelivered with the final/highest backoff below.\n\n\t// If we now send a new message, the pending timer should be reset to the first backoff.\n\t// Otherwise, if it remains at the final backoff duration we'll get this message redelivered too late.\n\tsendStreamMsg(t, nc, \"events.second\", \"msg-2\")\n\n\tfor {\n\t\tmsg, err := sub.NextMsg(5 * time.Second)\n\t\trequire_NoError(t, err)\n\t\tif msg.Subject == \"events.first\" {\n\t\t\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(3))\n\t\t\tcontinue\n\t\t}\n\n\t\t// We expect the second message to be redelivered using the specified backoff strategy.\n\t\t// Before, the first redelivery of the second message would be sent after the highest backoff duration.\n\t\tmetadata, err := msg.Metadata()\n\t\trequire_NoError(t, err)\n\t\tnumDelivered := int(metadata.NumDelivered)\n\t\texpectedBackoff := calculateExpectedBackoff(numDelivered)\n\t\trequire_LessThan(t, time.Since(metadata.Timestamp), expectedBackoff)\n\n\t\t// We've received all message, test passed.\n\t\tif numDelivered >= maxDeliver {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestJetStreamConsumerBackoffWhenBackoffLengthIsEqualToMaxDeliverConfig(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmaxDeliver := 3\n\tbackoff := []time.Duration{time.Second, 2 * time.Second, 3 * time.Second}\n\tsub, err := js.SubscribeSync(\n\t\t\"events.>\",\n\t\tnats.MaxDeliver(maxDeliver),\n\t\tnats.BackOff(backoff),\n\t\tnats.AckExplicit(),\n\t)\n\trequire_NoError(t, err)\n\n\tcalculateExpectedBackoff := func(numDelivered int) time.Duration {\n\t\treturn backoff[numDelivered-1] + 50*time.Millisecond // 50ms of margin to system overhead\n\t}\n\n\t// message to be redelivered using backoff duration.\n\tfirstMsgSent := time.Now()\n\tsendStreamMsg(t, nc, \"events.first\", \"msg-1\")\n\t_, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(1))\n\t_, err = sub.NextMsg(2 * time.Second)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(2))\n\t_, err = sub.NextMsg(3 * time.Second)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Since(firstMsgSent), calculateExpectedBackoff(3))\n}\n\nfunc TestJetStreamConsumerRetryAckAfterTimeout(t *testing.T) {\n\tfor _, ack := range []struct {\n\t\ttitle string\n\t\tpolicy nats.SubOpt\n\t}{\n\t\t{title: \"AckExplicit\", policy: nats.AckExplicit()},\n\t\t{title: \"AckAll\", policy: nats.AckAll()},\n\t} {\n\t\tt.Run(ack.title, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\", ack.policy)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 1)\n\n\t\t\tmsg := msgs[0]\n\t\t\t// Send core request so the client is unaware of the ack being sent.\n\t\t\t_, err = nc.Request(msg.Reply, nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// It could be we have already acked this specific message, but we haven't received the success response.\n\t\t\t// Retrying the ack should not time out and still signal success.\n\t\t\terr = msg.AckSync()\n\t\t\trequire_NoError(t, err)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerSwitchLeaderDuringInflightAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 2_000; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\n\t\t\"foo\",\n\t\t\"CONSUMER\",\n\t\tnats.MaxAckPending(2_000),\n\t\tnats.ManualAck(),\n\t\tnats.AckExplicit(),\n\t\tnats.AckWait(2*time.Second),\n\t)\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\tmsgs, err := sub.Fetch(2_000)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 2_000)\n\n\t// Simulate an ack being pushed, and o.setLeader(false) being called before the ack is processed and resets o.awl\n\tatomic.AddInt64(&o.awl, 1)\n\to.setLeader(false)\n\to.setLeader(true)\n\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(5*time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n}\n\nfunc TestJetStreamConsumerMessageDeletedDuringRedelivery(t *testing.T) {\n\tstorageTypes := []nats.StorageType{nats.MemoryStorage, nats.FileStorage}\n\tfor _, storageType := range storageTypes {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsub, err := js.PullSubscribe(\n\t\t\t\t\"foo\",\n\t\t\t\t\"CONSUMER\",\n\t\t\t\tnats.ManualAck(),\n\t\t\t\tnats.AckExplicit(),\n\t\t\t\tnats.AckWait(time.Second),\n\t\t\t)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\t\trequire_NotNil(t, o)\n\n\t\t\tmsgs, err := sub.Fetch(3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Len(t, len(msgs), 3)\n\n\t\t\terr = js.DeleteMsg(\"TEST\", 2)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for mset.storeUpdates to call into o.decStreamPending which runs\n\t\t\t// the o.processTerm goroutine, removing one message from pending.\n\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\to.mu.RLock()\n\t\t\t\tdefer o.mu.RUnlock()\n\t\t\t\tif len(o.pending) != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"expected 2 pending, but got %d\", len(o.pending))\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now empty the redelivery queue and reset the pending state.\n\t\t\to.mu.Lock()\n\t\t\tfor _, seq := range o.rdq {\n\t\t\t\to.removeFromRedeliverQueue(seq)\n\t\t\t}\n\t\t\to.pending = make(map[uint64]*Pending)\n\t\t\to.pending[2] = &Pending{}\n\t\t\to.addToRedeliverQueue(2)\n\n\t\t\t// Also reset delivery/ack floors to confirm they get corrected.\n\t\t\to.adflr, o.asflr = 0, 0\n\t\t\to.dseq, o.sseq = 11, 11\n\n\t\t\t// Getting the next message should skip seq 2, as that's deleted, but must not touch state.\n\t\t\t_, _, err = o.getNextMsg()\n\t\t\to.mu.Unlock()\n\t\t\trequire_Error(t, err, ErrStoreEOF)\n\t\t\trequire_Len(t, len(o.pending), 1)\n\n\t\t\t// Simulate the o.processTerm goroutine running after a call to o.getNextMsg.\n\t\t\t// Pending state and delivery/ack floors should be corrected.\n\t\t\to.processTerm(2, 2, 1, ackTermUnackedLimitsReason, _EMPTY_)\n\n\t\t\to.mu.RLock()\n\t\t\tdefer o.mu.RUnlock()\n\t\t\trequire_Len(t, len(o.pending), 0)\n\t\t\trequire_Equal(t, o.adflr, 10)\n\t\t\trequire_Equal(t, o.asflr, 10)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerDeliveryCount(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 2; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\n\t\t\"foo\",\n\t\t\"CONSUMER\",\n\t\tnats.ManualAck(),\n\t\tnats.AckExplicit(),\n\t\tnats.AckWait(time.Second),\n\t\tnats.MaxDeliver(1),\n\t)\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\tmsgs, err := sub.Fetch(2)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 2)\n\trequire_NoError(t, msgs[1].Nak())\n\n\trequire_Equal(t, o.deliveryCount(1), 1)\n\trequire_Equal(t, o.deliveryCount(2), 1)\n\n\t// max deliver 1 so this will fail\n\t_, err = sub.Fetch(1, nats.MaxWait(250*time.Millisecond))\n\trequire_Error(t, err)\n\n\t// This would previously report delivery count 0, because o.rdc!=nil\n\trequire_Equal(t, o.deliveryCount(1), 1)\n\trequire_Equal(t, o.deliveryCount(2), 1)\n}\n\nfunc TestJetStreamConsumerCreate(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"foo\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}, Retention: WorkQueuePolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"foo\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}, Retention: WorkQueuePolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Check for basic errors.\n\t\t\tif _, err := mset.addConsumer(nil); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error for no config\")\n\t\t\t}\n\n\t\t\t// No deliver subject, meaning its in pull mode, work queue mode means it is required to\n\t\t\t// do explicit ack.\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on work queue / pull mode without explicit ack mode\")\n\t\t\t}\n\n\t\t\t// Check for delivery subject errors.\n\n\t\t\t// Literal delivery subject required.\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: \"foo.*\"}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on bad delivery subject\")\n\t\t\t}\n\t\t\t// Check for cycles\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: \"foo\"}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on delivery subject that forms a cycle\")\n\t\t\t}\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: \"bar\"}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on delivery subject that forms a cycle\")\n\t\t\t}\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: \"*\"}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on delivery subject that forms a cycle\")\n\t\t\t}\n\n\t\t\t// StartPosition conflicts\n\t\t\tnow := time.Now().UTC()\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDeliverSubject: \"A\",\n\t\t\t\tOptStartSeq: 1,\n\t\t\t\tOptStartTime: &now,\n\t\t\t}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on start position conflicts\")\n\t\t\t}\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDeliverSubject: \"A\",\n\t\t\t\tOptStartTime: &now,\n\t\t\t}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on start position conflicts\")\n\t\t\t}\n\n\t\t\t// Non-Durables need to have subscription to delivery subject.\n\t\t\tdelivery := nats.NewInbox()\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\t\t\tsub, _ := nc.SubscribeSync(delivery)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, AckPolicy: AckExplicit})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\n\t\t\tif err := mset.deleteConsumer(o); err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error on delete, got %v\", err)\n\t\t\t}\n\n\t\t\t// Now let's check that durables can be created and a duplicate call to add will be ok.\n\t\t\tdcfg := &ConsumerConfig{\n\t\t\t\tDurable: \"ddd\",\n\t\t\t\tDeliverSubject: delivery,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t}\n\t\t\tif _, err = mset.addConsumer(dcfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error creating consumer: %v\", err)\n\t\t\t}\n\t\t\tif _, err = mset.addConsumer(dcfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error creating second identical consumer: %v\", err)\n\t\t\t}\n\t\t\t// Not test that we can change the delivery subject if that is only thing that has not\n\t\t\t// changed and we are not active.\n\t\t\tsub.Unsubscribe()\n\t\t\tsub, _ = nc.SubscribeSync(\"d.d.d\")\n\t\t\tnc.Flush()\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tdcfg.DeliverSubject = \"d.d.d\"\n\t\t\tif _, err = mset.addConsumer(dcfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error creating third consumer with just deliver subject changed: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerAndStreamDescriptions(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdescr := \"foo asset\"\n\tacc := s.GlobalAccount()\n\n\t// Check stream's first.\n\tmset, err := acc.addStream(&StreamConfig{Name: \"foo\", Description: descr})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tif cfg := mset.config(); cfg.Description != descr {\n\t\tt.Fatalf(\"Expected a description of %q, got %q\", descr, cfg.Description)\n\t}\n\n\t// Now consumer\n\tedescr := \"analytics\"\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDescription: edescr,\n\t\tDeliverSubject: \"to\",\n\t\tAckPolicy: AckNone})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\tif cfg := o.config(); cfg.Description != edescr {\n\t\tt.Fatalf(\"Expected a description of %q, got %q\", edescr, cfg.Description)\n\t}\n\n\t// Test max.\n\tdata := make([]byte, JSMaxDescriptionLen+1)\n\tcrand.Read(data)\n\tbigDescr := base64.StdEncoding.EncodeToString(data)\n\n\t_, err = acc.addStream(&StreamConfig{Name: \"bar\", Description: bigDescr})\n\tif err == nil || !strings.Contains(err.Error(), \"description is too long\") {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDescription: bigDescr,\n\t\tDeliverSubject: \"to\",\n\t\tAckPolicy: AckNone})\n\tif err == nil || !strings.Contains(err.Error(), \"description is too long\") {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n}\nfunc TestJetStreamConsumerWithNameAndDurable(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tdescr := \"foo asset\"\n\tname := \"name\"\n\tdurable := \"durable\"\n\tacc := s.GlobalAccount()\n\n\t// Check stream's first.\n\tmset, err := acc.addStream(&StreamConfig{Name: \"foo\", Description: descr})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tif cfg := mset.config(); cfg.Description != descr {\n\t\tt.Fatalf(\"Expected a description of %q, got %q\", descr, cfg.Description)\n\t}\n\n\t// it's ok to specify both durable and name, but they have to be the same.\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: \"to\",\n\t\tDurable: \"consumer\",\n\t\tName: \"consumer\",\n\t\tAckPolicy: AckNone})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\n\t// if they're not the same, expect error\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: \"to\",\n\t\tDurable: durable,\n\t\tName: name,\n\t\tAckPolicy: AckNone})\n\n\tif !strings.Contains(err.Error(), \"Consumer Durable and Name have to be equal\") {\n\t\tt.Fatalf(\"Wrong error while adding consumer with not matching Name and Durable: %v\", err)\n\t}\n}\n\nfunc TestJetStreamConsumerWithStartTime(t *testing.T) {\n\tsubj := \"my_stream\"\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: subj, Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: subj, Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tfsCfg := &FileStoreConfig{BlockSize: 100}\n\t\t\tmset, err := s.GlobalAccount().addStreamWithStore(c.mconfig, fsCfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 250\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, subj, fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tstartTime := time.Now().UTC()\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, subj, fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\n\t\t\tif msgs := mset.state().Msgs; msgs != uint64(toSend*2) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*2, msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d\",\n\t\t\t\tDeliverPolicy: DeliverByStartTime,\n\t\t\t\tOptStartTime: &startTime,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tmsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tsseq, dseq, _, _, _ := replyInfo(msg.Reply)\n\t\t\tif dseq != 1 {\n\t\t\t\tt.Fatalf(\"Expected delivered seq of 1, got %d\", dseq)\n\t\t\t}\n\t\t\tif sseq != uint64(toSend+1) {\n\t\t\t\tt.Fatalf(\"Expected to get store seq of %d, got %d\", toSend+1, sseq)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Test for https://github.com/nats-io/jetstream/issues/143\nfunc TestJetStreamConsumerWithMultipleStartOptions(t *testing.T) {\n\tsubj := \"my_stream\"\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: subj, Subjects: []string{\"foo.>\"}, Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: subj, Subjects: []string{\"foo.>\"}, Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\tStream: subj,\n\t\t\t\tConfig: ConsumerConfig{\n\t\t\t\t\tDurable: \"d\",\n\t\t\t\t\tDeliverPolicy: DeliverLast,\n\t\t\t\t\tFilterSubject: \"foo.22\",\n\t\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\t},\n\t\t\t}\n\t\t\treq, err := json.Marshal(obsReq)\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, subj), req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tnc.Close()\n\t\t\ts.Shutdown()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerMaxDeliveries(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Queue up our work item.\n\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\tmaxDeliver := 5\n\t\t\tackWait := 10 * time.Millisecond\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDeliverSubject: sub.Subject,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: ackWait,\n\t\t\t\tMaxDeliver: maxDeliver,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\t// Wait for redeliveries to pile up.\n\t\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs != maxDeliver {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, maxDeliver)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now wait a bit longer and make sure we do not have more than maxDeliveries.\n\t\t\ttime.Sleep(2 * ackWait)\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxDeliver {\n\t\t\t\tt.Fatalf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, maxDeliver)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerSingleTokenSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfilterSubject := \"foo\"\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{filterSubject},\n\t})\n\trequire_NoError(t, err)\n\n\treq, err := json.Marshal(&CreateConsumerRequest{Stream: \"TEST\", Config: ConsumerConfig{\n\t\tFilterSubject: filterSubject,\n\t\tName: \"name\",\n\t}})\n\n\tif err != nil {\n\t\tt.Fatalf(\"failed to marshal consumer create request: %v\", err)\n\t}\n\n\tresp, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.CREATE.%s.%s.%s\", \"TEST\", \"name\", \"not_filter_subject\"), req, time.Second*10)\n\n\tvar apiResp ApiResponse\n\tjson.Unmarshal(resp.Data, &apiResp)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to unmarshal response: %v\", err)\n\t}\n\tif apiResp.Error == nil {\n\t\tt.Fatalf(\"expected error, got nil\")\n\t}\n\tif apiResp.Error.ErrCode != 10131 {\n\t\tt.Fatalf(\"expected error 10131, got %v\", apiResp.Error)\n\t}\n}\n\nfunc TestJetStreamConsumerPullDelayedFirstPullWithReplayOriginal(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Queue up our work item.\n\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tReplayPolicy: ReplayOriginal,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\t// Force delay here which triggers the bug.\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\n\t\t\tif _, err = nc.Request(o.requestNextMsgSubject(), nil, time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerAckFloorFill(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MQ\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MQ\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfor i := 1; i <= 4; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, fmt.Sprintf(\"msg-%d\", i))\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d\",\n\t\t\t\tDeliverSubject: sub.Subject,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tvar first *nats.Msg\n\n\t\t\tfor i := 1; i <= 3; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error receiving message %d: %v\", i, err)\n\t\t\t\t}\n\t\t\t\t// Don't ack 1 or 4.\n\t\t\t\tif i == 1 {\n\t\t\t\t\tfirst = m\n\t\t\t\t} else if i == 2 || i == 3 {\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t\tif info := o.info(); info.AckFloor.Consumer != 0 {\n\t\t\t\tt.Fatalf(\"Expected the ack floor to be 0, got %d\", info.AckFloor.Consumer)\n\t\t\t}\n\t\t\t// Now ack first, should move ack floor to 3.\n\t\t\tfirst.Respond(nil)\n\t\t\tnc.Flush()\n\n\t\t\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\t\t\tif info := o.info(); info.AckFloor.Consumer != 3 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected the ack floor to be 3, got %d\", info.AckFloor.Consumer)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerAckAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"ACK-ACK\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"worker\", AckPolicy: AckExplicit})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\trqn := o.requestNextMsgSubject()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// 4 for number of ack protocols to test them all.\n\tfor i := 0; i < 4; i++ {\n\t\tsendStreamMsg(t, nc, mname, \"Hello World!\")\n\t}\n\n\ttestAck := func(ackType []byte) {\n\t\tm, err := nc.Request(rqn, nil, 10*time.Millisecond)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Send a request for the ack and make sure the server \"ack's\" the ack.\n\t\tif _, err := nc.Request(m.Reply, ackType, 10*time.Millisecond); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on ack/ack: %v\", err)\n\t\t}\n\t}\n\n\ttestAck(AckAck)\n\ttestAck(AckNak)\n\ttestAck(AckProgress)\n\ttestAck(AckTerm)\n}\n\nfunc TestJetStreamConsumerRateLimit(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"RATELIMIT\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tmsgSize := 128 * 1024\n\tmsg := make([]byte, msgSize)\n\tcrand.Read(msg)\n\n\t// 10MB\n\ttotalSize := 10 * 1024 * 1024\n\ttoSend := totalSize / msgSize\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.Publish(mname, msg)\n\t}\n\tnc.Flush()\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tstate := mset.state()\n\t\tif state.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// 100Mbit\n\trateLimit := uint64(100 * 1024 * 1024)\n\t// Make sure if you set a rate with a pull based consumer it errors.\n\t_, err = mset.addConsumer(&ConsumerConfig{Durable: \"to\", AckPolicy: AckExplicit, RateLimit: rateLimit})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\n\t// Now create one and measure the rate delivered.\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"rate\",\n\t\tDeliverSubject: \"to\",\n\t\tRateLimit: rateLimit,\n\t\tAckPolicy: AckNone})\n\trequire_NoError(t, err)\n\tdefer o.delete()\n\n\tvar received int\n\tdone := make(chan bool)\n\n\tstart := time.Now()\n\n\tnc.Subscribe(\"to\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= toSend {\n\t\t\tdone <- true\n\t\t}\n\t})\n\tnc.Flush()\n\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive all the messages in time\")\n\t}\n\n\ttt := time.Since(start)\n\trate := float64(8*toSend*msgSize) / tt.Seconds()\n\tif rate > float64(rateLimit)*1.25 {\n\t\tt.Fatalf(\"Exceeded desired rate of %d mbps, got %0.f mbps\", rateLimit/(1024*1024), rate/(1024*1024))\n\t}\n}\n\nfunc TestJetStreamConsumerEphemeralRecoveryAfterServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MYS\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\tdefer sub.Unsubscribe()\n\tnc.Flush()\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: sub.Subject,\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\tdefer o.delete()\n\n\t// Snapshot our name.\n\toname := o.String()\n\n\t// Send 100 messages\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, mname, \"Hello World!\")\n\t}\n\tif state := mset.state(); state.Msgs != 100 {\n\t\tt.Fatalf(\"Expected %d messages, got %d\", 100, state.Msgs)\n\t}\n\n\t// Read 6 messages\n\tfor i := 0; i <= 6; i++ {\n\t\tif m, err := sub.NextMsg(time.Second); err == nil {\n\t\t\tm.Respond(nil)\n\t\t} else {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Capture port since it was dynamic.\n\tu, _ := url.Parse(s.ClientURL())\n\tport, _ := strconv.Atoi(u.Port())\n\n\trestartServer := func() {\n\t\tt.Helper()\n\t\t// Stop current\n\t\tsd := s.JetStreamConfig().StoreDir\n\t\ts.Shutdown()\n\t\t// Restart.\n\t\ts = RunJetStreamServerOnPort(port, sd)\n\t}\n\n\t// Do twice\n\tfor i := 0; i < 2; i++ {\n\t\t// Restart.\n\t\trestartServer()\n\t\tdefer s.Shutdown()\n\n\t\tmset, err = s.GlobalAccount().lookupStream(mname)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t\t}\n\t\to = mset.lookupConsumer(oname)\n\t\tif o == nil {\n\t\t\tt.Fatalf(\"Error looking up consumer %q\", oname)\n\t\t}\n\t\t// Make sure config does not have durable.\n\t\tif cfg := o.config(); cfg.Durable != _EMPTY_ {\n\t\t\tt.Fatalf(\"Expected no durable to be set\")\n\t\t}\n\t\t// Wait for it to become active\n\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\tif !o.isActive() {\n\t\t\t\treturn fmt.Errorf(\"Consumer not active\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Now close the connection. Make sure this acts like an ephemeral and goes away.\n\to.setInActiveDeleteThreshold(10 * time.Millisecond)\n\tnc.Close()\n\n\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif o := mset.lookupConsumer(oname); o != nil {\n\t\t\treturn fmt.Errorf(\"Consumer still active\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamConsumerMaxDeliveryAndServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MYS\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tstreamCreated := mset.createdTime()\n\n\tdsubj := \"D.TO\"\n\tmax := 3\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"TO\",\n\t\tDeliverSubject: dsubj,\n\t\tAckPolicy: AckExplicit,\n\t\tAckWait: 100 * time.Millisecond,\n\t\tMaxDeliver: max,\n\t})\n\tdefer o.delete()\n\n\tconsumerCreated := o.createdTime()\n\t// For calculation of consumer created times below.\n\ttime.Sleep(5 * time.Millisecond)\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(dsubj)\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\t// Send one message.\n\tsendStreamMsg(t, nc, mname, \"order-1\")\n\n\tcheckSubPending := func(numExpected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif nmsgs, _, _ := sub.Pending(); nmsgs != numExpected {\n\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckNumMsgs := func(numExpected uint64) {\n\t\tt.Helper()\n\t\tmset, err = s.GlobalAccount().lookupStream(mname)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t\t}\n\t\tstate := mset.state()\n\t\tif state.Msgs != numExpected {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", numExpected, state.Msgs)\n\t\t}\n\t}\n\n\t// Wait til we know we have max queued up.\n\tcheckSubPending(max)\n\n\t// Once here we have gone over the limit for the 1st message for max deliveries.\n\t// Send second\n\tsendStreamMsg(t, nc, mname, \"order-2\")\n\n\t// Just wait for first delivery + one redelivery.\n\tcheckSubPending(max + 2)\n\n\t// Capture port since it was dynamic.\n\tu, _ := url.Parse(s.ClientURL())\n\tport, _ := strconv.Atoi(u.Port())\n\n\trestartServer := func() {\n\t\tt.Helper()\n\t\tsd := s.JetStreamConfig().StoreDir\n\t\t// Stop current\n\t\ts.Shutdown()\n\t\t// Restart.\n\t\ts = RunJetStreamServerOnPort(port, sd)\n\t}\n\n\twaitForClientReconnect := func() {\n\t\tt.Helper()\n\t\trequire_NoError(t, nc.ForceReconnect())\n\t\tcheckFor(t, 2500*time.Millisecond, 5*time.Millisecond, func() error {\n\t\t\tif !nc.IsConnected() {\n\t\t\t\treturn fmt.Errorf(\"Not connected\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Restart.\n\trestartServer()\n\tdefer s.Shutdown()\n\n\tcheckNumMsgs(2)\n\n\t// Wait for client to be reconnected.\n\twaitForClientReconnect()\n\n\t// Once we are here send third order.\n\tsendStreamMsg(t, nc, mname, \"order-3\")\n\tcheckNumMsgs(3)\n\n\t// Restart.\n\trestartServer()\n\tdefer s.Shutdown()\n\n\tcheckNumMsgs(3)\n\n\t// Wait for client to be reconnected.\n\twaitForClientReconnect()\n\n\t// Now we should have max times three on our sub.\n\tcheckSubPending(max * 3)\n\n\t// Now do some checks on created timestamps.\n\tmset, err = s.GlobalAccount().lookupStream(mname)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t}\n\tif mset.createdTime() != streamCreated {\n\t\tt.Fatalf(\"Stream creation time not restored, wanted %v, got %v\", streamCreated, mset.createdTime())\n\t}\n\to = mset.lookupConsumer(\"TO\")\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer: %v\", err)\n\t}\n\t// Consumer created times can have a very small skew.\n\tdelta := o.createdTime().Sub(consumerCreated)\n\tif delta > 5*time.Millisecond {\n\t\tt.Fatalf(\"Consumer creation time not restored, wanted %v, got %v\", consumerCreated, o.createdTime())\n\t}\n}\n\nfunc TestJetStreamConsumerDeleteAndServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsendSubj := \"MYQ\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sendSubj, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// Create basic work queue mode consumer.\n\toname := \"WQ\"\n\to, err := mset.addConsumer(workerModeConfig(oname))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\n\t// Now delete and then we will restart the\n\to.delete()\n\n\tif numo := mset.numConsumers(); numo != 0 {\n\t\tt.Fatalf(\"Expected to have zero consumers, got %d\", numo)\n\t}\n\n\t// Capture port since it was dynamic.\n\tu, _ := url.Parse(s.ClientURL())\n\tport, _ := strconv.Atoi(u.Port())\n\tsd := s.JetStreamConfig().StoreDir\n\n\t// Stop current\n\ts.Shutdown()\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\tmset, err = s.GlobalAccount().lookupStream(sendSubj)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", sendSubj)\n\t}\n\n\tif numo := mset.numConsumers(); numo != 0 {\n\t\tt.Fatalf(\"Expected to have zero consumers, got %d\", numo)\n\t}\n}\n\nfunc TestJetStreamConsumerDurableReconnectWithOnlyPending(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DT\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DT\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tdname := \"d22\"\n\t\t\tsubj1 := nats.NewInbox()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: dname,\n\t\t\t\tDeliverSubject: subj1,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 25 * time.Millisecond})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tsendMsg := func(payload string) {\n\t\t\t\tt.Helper()\n\t\t\t\tif err := nc.Publish(\"foo.22\", []byte(payload)); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsendMsg(\"1\")\n\n\t\t\tsub, _ := nc.SubscribeSync(subj1)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tcheckFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, 1)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now unsubscribe and wait to become inactive\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Consumer is still active\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Send the second message while delivery subscriber is not running\n\t\t\tsendMsg(\"2\")\n\n\t\t\t// Now we should be able to replace the delivery subject.\n\t\t\tsubj2 := nats.NewInbox()\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: dname,\n\t\t\t\tDeliverSubject: subj2,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 25 * time.Millisecond})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error trying to add a new durable consumer: %v\", err)\n\t\t\t}\n\t\t\tsub, _ = nc.SubscribeSync(subj2)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// We should get msg \"1\" and \"2\" delivered. They will be reversed.\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tmsg, err := sub.NextMsg(500 * time.Millisecond)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tsseq, _, dc, _, _ := replyInfo(msg.Reply)\n\t\t\t\tif sseq == 1 && dc == 1 {\n\t\t\t\t\tt.Fatalf(\"Expected a redelivery count greater then 1 for sseq 1, got %d\", dc)\n\t\t\t\t}\n\t\t\t\tif sseq != 1 && sseq != 2 {\n\t\t\t\t\tt.Fatalf(\"Expected stream sequence of 1 or 2 but got %d\", sseq)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerDurableFilteredSubjectReconnect(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DT\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DT\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsendMsgs := func(toSend int) {\n\t\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\t\tvar subj string\n\t\t\t\t\tif i%2 == 0 {\n\t\t\t\t\t\tsubj = \"foo.AA\"\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsubj = \"foo.ZZ\"\n\t\t\t\t\t}\n\t\t\t\t\t_, err := js.Publish(subj, []byte(\"OK!\"))\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Send 50 msgs\n\t\t\ttoSend := 50\n\t\t\tsendMsgs(toSend)\n\n\t\t\tdname := \"d33\"\n\t\t\tdsubj := nats.NewInbox()\n\n\t\t\t// Now create an consumer for foo.AA, only requesting the last one.\n\t\t\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: dname,\n\t\t\t\tDeliverSubject: dsubj,\n\t\t\t\tFilterSubject: \"foo.AA\",\n\t\t\t\tDeliverPolicy: DeliverLast,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 100 * time.Millisecond,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tsub, _ := nc.SubscribeSync(dsubj)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// Used to calculate difference between store seq and delivery seq.\n\t\t\tstoreBaseOff := 47\n\n\t\t\tgetMsg := func(seq int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tsseq := 2*seq + storeBaseOff\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\trsseq, roseq, dcount, _, _ := replyInfo(m.Reply)\n\t\t\t\tif roseq != uint64(seq) {\n\t\t\t\t\tt.Fatalf(\"Expected consumer sequence of %d , got %d\", seq, roseq)\n\t\t\t\t}\n\t\t\t\tif rsseq != uint64(sseq) {\n\t\t\t\t\tt.Fatalf(\"Expected stream sequence of %d , got %d\", sseq, rsseq)\n\t\t\t\t}\n\t\t\t\tif dcount != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected message to not be marked as redelivered\")\n\t\t\t\t}\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\tgetRedeliveredMsg := func(seq int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t_, roseq, dcount, _, _ := replyInfo(m.Reply)\n\t\t\t\tif roseq != uint64(seq) {\n\t\t\t\t\tt.Fatalf(\"Expected consumer sequence of %d , got %d\", seq, roseq)\n\t\t\t\t}\n\t\t\t\tif dcount < 2 {\n\t\t\t\t\tt.Fatalf(\"Expected message to be marked as redelivered\")\n\t\t\t\t}\n\t\t\t\t// Ack this message.\n\t\t\t\tm.Respond(nil)\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\t// All consumers start at 1 and always have increasing sequence numbers.\n\t\t\tm := getMsg(1)\n\t\t\tm.Respond(nil)\n\n\t\t\t// Now send 50 more, so 100 total, 26 (last + 50/2) for this consumer.\n\t\t\tsendMsgs(toSend)\n\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend*2) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*2, state.Msgs)\n\t\t\t}\n\n\t\t\t// For tracking next expected.\n\t\t\tnextSeq := 2\n\t\t\tnoAcks := 0\n\t\t\tfor i := 0; i < toSend/2; i++ {\n\t\t\t\tm := getMsg(nextSeq)\n\t\t\t\tif i%2 == 0 {\n\t\t\t\t\tm.Respond(nil) // Ack evens.\n\t\t\t\t} else {\n\t\t\t\t\tnoAcks++\n\t\t\t\t}\n\t\t\t\tnextSeq++\n\t\t\t}\n\n\t\t\t// We should now get those redelivered.\n\t\t\tfor i := 0; i < noAcks; i++ {\n\t\t\t\tgetRedeliveredMsg(nextSeq)\n\t\t\t\tnextSeq++\n\t\t\t}\n\n\t\t\t// Now send 50 more.\n\t\t\tsendMsgs(toSend)\n\n\t\t\tstoreBaseOff -= noAcks * 2\n\n\t\t\tfor i := 0; i < toSend/2; i++ {\n\t\t\t\tm := getMsg(nextSeq)\n\t\t\t\tm.Respond(nil)\n\t\t\t\tnextSeq++\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerInactiveNoDeadlock(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send lots of msgs and have them queued up.\n\t\t\tfor i := 0; i < 10000; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\n\t\t\tif state := mset.state(); state.Msgs != 10000 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 10000, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tsub.SetPendingLimits(-1, -1)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Force us to become inactive but we want to make sure we do not lock up\n\t\t\t// the internal sendq.\n\t\t\tsub.Unsubscribe()\n\t\t\tnc.Flush()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerReconnect(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"ET\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"ET\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// Capture the subscription.\n\t\t\tdelivery := sub.Subject\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, AckPolicy: AckExplicit})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif !o.isActive() {\n\t\t\t\tt.Fatalf(\"Expected the consumer to be considered active\")\n\t\t\t}\n\t\t\tif numo := mset.numConsumers(); numo != 1 {\n\t\t\t\tt.Fatalf(\"Expected number of consumers to be 1, got %d\", numo)\n\t\t\t}\n\n\t\t\t// We will simulate reconnect by unsubscribing on one connection and forming\n\t\t\t// the same on another. Once we have cluster tests we will do more testing on\n\t\t\t// reconnect scenarios.\n\t\t\tgetMsg := func(seqno int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error for %d: %v\", seqno, err)\n\t\t\t\t}\n\t\t\t\tif seq := o.seqFromReply(m.Reply); seq != uint64(seqno) {\n\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", seqno, seq)\n\t\t\t\t}\n\t\t\t\tm.Respond(nil)\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\tsendMsg := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tif err := nc.Publish(\"foo.22\", []byte(\"OK!\")); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheckForInActive := func() {\n\t\t\t\tcheckFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\t\tif o.isActive() {\n\t\t\t\t\t\treturn fmt.Errorf(\"Consumer is still active\")\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Send and Pull first message.\n\t\t\tsendMsg() // 1\n\t\t\tgetMsg(1)\n\t\t\t// Cancel first one.\n\t\t\tsub.Unsubscribe()\n\t\t\t// Re-establish new sub on same subject.\n\t\t\tsub, _ = nc.SubscribeSync(delivery)\n\t\t\tnc.Flush()\n\n\t\t\t// We should be getting 2 here.\n\t\t\tsendMsg() // 2\n\t\t\tgetMsg(2)\n\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckForInActive()\n\n\t\t\t// send 3-10\n\t\t\tfor i := 0; i <= 7; i++ {\n\t\t\t\tsendMsg()\n\t\t\t}\n\t\t\t// Make sure they are all queued up with no interest.\n\t\t\tnc.Flush()\n\n\t\t\t// Restablish again.\n\t\t\tsub, _ = nc.SubscribeSync(delivery)\n\t\t\tnc.Flush()\n\n\t\t\t// We should be getting 3-10 here.\n\t\t\tfor i := 3; i <= 10; i++ {\n\t\t\t\tgetMsg(i)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerDurableReconnect(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DT\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DT\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tdname := \"d22\"\n\t\t\tsubj1 := nats.NewInbox()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: dname,\n\t\t\t\tDeliverSubject: subj1,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 50 * time.Millisecond})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tsendMsg := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tif err := nc.Publish(\"foo.22\", []byte(\"OK!\")); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Send 10 msgs\n\t\t\ttoSend := 10\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendMsg()\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(subj1)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tcheckFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tgetMsg := func(seqno int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif seq := o.streamSeqFromReply(m.Reply); seq != uint64(seqno) {\n\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", seqno, seq)\n\t\t\t\t}\n\t\t\t\tm.Respond(nil)\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\t// Ack first half\n\t\t\tfor i := 1; i <= toSend/2; i++ {\n\t\t\t\tm := getMsg(i)\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\n\t\t\t// Now unsubscribe and wait to become inactive\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Consumer is still active\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now we should be able to replace the delivery subject.\n\t\t\tsubj2 := nats.NewInbox()\n\t\t\tsub, _ = nc.SubscribeSync(subj2)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: dname,\n\t\t\t\tDeliverSubject: subj2,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 50 * time.Millisecond})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error trying to add a new durable consumer: %v\", err)\n\t\t\t}\n\n\t\t\t// We should get the remaining messages here.\n\t\t\tfor i := toSend/2 + 1; i <= toSend; i++ {\n\t\t\t\tm := getMsg(i)\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerReplayRate(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 10 msgs\n\t\t\ttotalMsgs := 10\n\n\t\t\tvar gaps []time.Duration\n\t\t\tlst := time.Now()\n\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tgaps = append(gaps, time.Since(lst))\n\t\t\t\tlst = time.Now()\n\t\t\t\tnc.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t\t// Calculate a gap between messages.\n\t\t\t\tgap := 10*time.Millisecond + time.Duration(rand.Intn(20))*time.Millisecond\n\t\t\t\ttime.Sleep(gap)\n\t\t\t}\n\n\t\t\tif state := mset.state(); state.Msgs != uint64(totalMsgs) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", totalMsgs, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Firehose/instant which is default.\n\t\t\tlast := time.Now()\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tnow := time.Now()\n\t\t\t\t// Delivery from addConsumer starts in a go routine, so be\n\t\t\t\t// more tolerant for the first message.\n\t\t\t\tlimit := 5 * time.Millisecond\n\t\t\t\tif i == 0 {\n\t\t\t\t\tlimit = 10 * time.Millisecond\n\t\t\t\t}\n\t\t\t\tif now.Sub(last) > limit {\n\t\t\t\t\tt.Fatalf(\"Expected firehose/instant delivery, got message gap of %v\", now.Sub(last))\n\t\t\t\t}\n\t\t\t\tlast = now\n\t\t\t}\n\n\t\t\t// Now do replay rate to match original.\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, ReplayPolicy: ReplayOriginal})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Original rate messsages were received for push based consumer.\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tstart := time.Now()\n\t\t\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tgap := time.Since(start)\n\t\t\t\t// 15ms is high but on macs time.Sleep(delay) does not sleep only delay.\n\t\t\t\t// Also on travis if things get bogged down this could be delayed.\n\t\t\t\tgl, gh := gaps[i]-10*time.Millisecond, gaps[i]+15*time.Millisecond\n\t\t\t\tif gap < gl || gap > gh {\n\t\t\t\t\tt.Fatalf(\"Gap is off for %d, expected %v got %v\", i, gaps[i], gap)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Now create pull based.\n\t\t\toc := workerModeConfig(\"PM\")\n\t\t\toc.ReplayPolicy = ReplayOriginal\n\t\t\to, err = mset.addConsumer(oc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tstart := time.Now()\n\t\t\t\tif _, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tgap := time.Since(start)\n\t\t\t\t// 10ms is high but on macs time.Sleep(delay) does not sleep only delay.\n\t\t\t\tgl, gh := gaps[i]-5*time.Millisecond, gaps[i]+10*time.Millisecond\n\t\t\t\tif gap < gl || gap > gh {\n\t\t\t\t\tt.Fatalf(\"Gap is incorrect for %d, expected %v got %v\", i, gaps[i], gap)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerReplayRateNoAck(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 10 msgs\n\t\t\ttotalMsgs := 10\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tnc.Request(\"DC\", []byte(\"Hello World\"), time.Second)\n\t\t\t\ttime.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != uint64(totalMsgs) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", totalMsgs, state.Msgs)\n\t\t\t}\n\t\t\tsubj := \"d.dc\"\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"derek\",\n\t\t\t\tDeliverSubject: subj,\n\t\t\t\tAckPolicy: AckNone,\n\t\t\t\tReplayPolicy: ReplayOriginal,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\t\t\t// Sleep a random amount of time.\n\t\t\ttime.Sleep(time.Duration(rand.Intn(20)) * time.Millisecond)\n\n\t\t\tsub, _ := nc.SubscribeSync(subj)\n\t\t\tnc.Flush()\n\n\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != totalMsgs {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, totalMsgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerReplayQuit(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 2 msgs\n\t\t\tnc.Request(\"DC\", []byte(\"OK!\"), time.Second)\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\tnc.Request(\"DC\", []byte(\"OK!\"), time.Second)\n\n\t\t\tif state := mset.state(); state.Msgs != 2 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 2, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// Now do replay rate to match original.\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, ReplayPolicy: ReplayOriginal})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\t// Allow loop and deliver / replay go routine to spin up\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tbase := runtime.NumGoroutine()\n\t\t\to.delete()\n\n\t\t\tcheckFor(t, 100*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif runtime.NumGoroutine() >= base {\n\t\t\t\t\treturn fmt.Errorf(\"Consumer go routines still running\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tmsetConfig := StreamConfig{\n\t\tName: \"sr22\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\n\tmset, err := acc.addStream(&msetConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tpayload := []byte(\"Hello World\")\n\n\ttoStore := 2000000\n\tfor i := 0; i < toStore; i++ {\n\t\tnc.Publish(\"foo\", payload)\n\t}\n\tnc.Flush()\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tDeliverSubject: \"d\",\n\t\tAckPolicy: AckNone,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\n\tvar received int\n\tdone := make(chan bool)\n\n\tnc.Subscribe(\"d\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= toStore {\n\t\t\tdone <- true\n\t\t}\n\t})\n\tstart := time.Now()\n\tnc.Flush()\n\n\t<-done\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n}\n\nfunc TestJetStreamConsumerAckFileStorePerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tmsetConfig := StreamConfig{\n\t\tName: \"sr22\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\n\tmset, err := acc.addStream(&msetConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tpayload := []byte(\"Hello World\")\n\n\ttoStore := uint64(200000)\n\tfor i := uint64(0); i < toStore; i++ {\n\t\tnc.Publish(\"foo\", payload)\n\t}\n\tnc.Flush()\n\n\tif msgs := mset.state().Msgs; msgs != uint64(toStore) {\n\t\tt.Fatalf(\"Expected %d messages, got %d\", toStore, msgs)\n\t}\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tDeliverSubject: \"d\",\n\t\tAckPolicy: AckExplicit,\n\t\tAckWait: 10 * time.Minute,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\tdefer o.stop()\n\n\tvar received uint64\n\tdone := make(chan bool)\n\n\tsub, _ := nc.Subscribe(\"d\", func(m *nats.Msg) {\n\t\tm.Respond(nil) // Ack\n\t\treceived++\n\t\tif received >= toStore {\n\t\t\tdone <- true\n\t\t}\n\t})\n\tsub.SetPendingLimits(-1, -1)\n\n\tstart := time.Now()\n\tnc.Flush()\n\n\t<-done\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toStore)/tt.Seconds())\n}\n\n// This test is in support fo clients that want to match on subject, they\n// can set the filter subject always. We always store the subject so that\n// should the stream later be edited to expand into more subjects the consumer\n// still gets what was actually requested\nfunc TestJetStreamConsumerFilterSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsc := &StreamConfig{Name: \"MY_STREAM\", Subjects: []string{\"foo\"}}\n\tmset, err := s.GlobalAccount().addStream(sc)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tcfg := &ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tDeliverSubject: \"A\",\n\t\tAckPolicy: AckExplicit,\n\t\tFilterSubject: \"foo\",\n\t}\n\n\to, err := mset.addConsumer(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\tdefer o.delete()\n\n\tif o.info().Config.FilterSubject != \"foo\" {\n\t\tt.Fatalf(\"Expected the filter to be stored\")\n\t}\n\n\t// Now use the original cfg with updated delivery subject and make sure that works ok.\n\tcfg = &ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tDeliverSubject: \"B\",\n\t\tAckPolicy: AckExplicit,\n\t\tFilterSubject: \"foo\",\n\t}\n\n\to, err = mset.addConsumer(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\tdefer o.delete()\n}\n\nfunc TestJetStreamConsumerUpdateRedelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: MemoryStorage,\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\tRetention: InterestPolicy,\n\t\t}},\n\t\t{\"FileStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\tRetention: InterestPolicy,\n\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Create a durable consumer.\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dur22\",\n\t\t\t\tDeliverSubject: sub.Subject,\n\t\t\t\tFilterSubject: \"foo.bar\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 100 * time.Millisecond,\n\t\t\t\tMaxDeliver: 3,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Send 20 messages\n\t\t\ttoSend := 20\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", fmt.Sprintf(\"msg-%v\", i))\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %v messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\t// Receive the messages and ack only every 4th\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t\t\t\t}\n\t\t\t\tseq, _, _, _, _ := replyInfo(m.Reply)\n\t\t\t\t// 4, 8, 12, 16, 20\n\t\t\t\tif seq%4 == 0 {\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Now close the sub and open a new one and update the consumer.\n\t\t\tsub.Unsubscribe()\n\n\t\t\t// Wait for it to become inactive\n\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Consumer still active\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Send 20 more messages.\n\t\t\tfor i := toSend; i < toSend*2; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", fmt.Sprintf(\"msg-%v\", i))\n\t\t\t}\n\n\t\t\t// Create new subscription.\n\t\t\tsub, _ = nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dur22\",\n\t\t\t\tDeliverSubject: sub.Subject,\n\t\t\t\tFilterSubject: \"foo.bar\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 100 * time.Millisecond,\n\t\t\t\tMaxDeliver: 3,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\texpect := toSend + toSend - 5 // mod 4 acks\n\t\t\tcheckFor(t, time.Second, 5*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expect {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, expect)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tfor i, eseq := 0, uint64(1); i < expect; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Skip the ones we ack'd from above. We should not get them back here.\n\t\t\t\tif eseq <= uint64(toSend) && eseq%4 == 0 {\n\t\t\t\t\teseq++\n\t\t\t\t}\n\t\t\t\tseq, _, dc, _, _ := replyInfo(m.Reply)\n\t\t\t\tif seq != eseq {\n\t\t\t\t\tt.Fatalf(\"Expected stream sequence of %d, got %d\", eseq, seq)\n\t\t\t\t}\n\t\t\t\tif seq <= uint64(toSend) && dc != 2 {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count of 2 for sequence of %d, got %d\", seq, dc)\n\t\t\t\t}\n\t\t\t\tif seq > uint64(toSend) && dc != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count of 1 for sequence of %d, got %d\", seq, dc)\n\t\t\t\t}\n\t\t\t\tif seq > uint64(toSend) {\n\t\t\t\t\tm.Respond(nil) // Ack\n\t\t\t\t}\n\t\t\t\teseq++\n\t\t\t}\n\n\t\t\t// We should get the second half back since we did not ack those from above.\n\t\t\texpect = toSend - 5\n\t\t\tcheckFor(t, time.Second, 5*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expect {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, expect)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tfor i, eseq := 0, uint64(1); i < expect; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Skip the ones we ack'd from above. We should not get them back here.\n\t\t\t\tif eseq <= uint64(toSend) && eseq%4 == 0 {\n\t\t\t\t\teseq++\n\t\t\t\t}\n\t\t\t\tseq, _, dc, _, _ := replyInfo(m.Reply)\n\t\t\t\tif seq != eseq {\n\t\t\t\t\tt.Fatalf(\"Expected stream sequence of %d, got %d\", eseq, seq)\n\t\t\t\t}\n\t\t\t\tif dc != 3 {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count of 3 for sequence of %d, got %d\", seq, dc)\n\t\t\t\t}\n\t\t\t\teseq++\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerMaxAckPending(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: MemoryStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t\t{\"FileStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Do error scenarios.\n\t\t\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d22\",\n\t\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\t\tAckPolicy: AckNone,\n\t\t\t\tMaxAckPending: 1,\n\t\t\t})\n\t\t\tif err == nil {\n\t\t\t\tt.Fatalf(\"Expected error, MaxAckPending only applicable to ack != AckNone\")\n\t\t\t}\n\n\t\t\t// Queue up 100 messages.\n\t\t\ttoSend := 100\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\n\t\t\t// Limit to 33\n\t\t\tmaxAckPending := 33\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d22\",\n\t\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tMaxAckPending: maxAckPending,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdefer o.delete()\n\n\t\t\tsub, _ := nc.SubscribeSync(o.info().Config.DeliverSubject)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tcheckSubPending := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tcheckSubPending(maxAckPending)\n\t\t\t// We hit the limit, double check we stayed there.\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending {\n\t\t\t\tt.Fatalf(\"Too many messages received: %d vs %d\", nmsgs, maxAckPending)\n\t\t\t}\n\n\t\t\t// Now ack them all.\n\t\t\tfor i := 0; i < maxAckPending; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error receiving message %d: %v\", i, err)\n\t\t\t\t}\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\t\t\tcheckSubPending(maxAckPending)\n\n\t\t\to.stop()\n\t\t\tmset.purge(nil)\n\n\t\t\t// Now test a consumer that is live while we publish messages to the stream.\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d33\",\n\t\t\t\tDeliverSubject: nats.NewInbox(),\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tMaxAckPending: maxAckPending,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdefer o.delete()\n\n\t\t\tsub, _ = nc.SubscribeSync(o.info().Config.DeliverSubject)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\tcheckSubPending(0)\n\n\t\t\t// Now stream more then maxAckPending.\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.baz\", fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\t\t\tcheckSubPending(maxAckPending)\n\t\t\t// We hit the limit, double check we stayed there.\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending {\n\t\t\t\tt.Fatalf(\"Too many messages received: %d vs %d\", nmsgs, maxAckPending)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerPullMaxAckPending(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: MemoryStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t\t{\"FileStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Queue up 100 messages.\n\t\t\ttoSend := 100\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\n\t\t\t// Limit to 33\n\t\t\tmaxAckPending := 33\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d22\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tMaxAckPending: maxAckPending,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdefer o.delete()\n\n\t\t\tgetSubj := o.requestNextMsgSubject()\n\n\t\t\tvar toAck []*nats.Msg\n\n\t\t\tfor i := 0; i < maxAckPending; i++ {\n\t\t\t\tif m, err := nc.Request(getSubj, nil, time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t} else {\n\t\t\t\t\ttoAck = append(toAck, m)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Now ack them all.\n\t\t\tfor _, m := range toAck {\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\n\t\t\t// Now do batch above the max.\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tcheckSubPending := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\treq := &JSApiConsumerGetNextRequest{Batch: maxAckPending}\n\t\t\tjreq, _ := json.Marshal(req)\n\t\t\tnc.PublishRequest(getSubj, sub.Subject, jreq)\n\n\t\t\tcheckSubPending(maxAckPending)\n\t\t\t// We hit the limit, double check we stayed there.\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxAckPending {\n\t\t\t\tt.Fatalf(\"Too many messages received: %d vs %d\", nmsgs, maxAckPending)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerPullMaxAckPendingRedeliveries(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: MemoryStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t\t{\"FileStore\", &StreamConfig{\n\t\t\tName: \"MY_STREAM\",\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Queue up 10 messages.\n\t\t\ttoSend := 10\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\n\t\t\t// Limit to 1\n\t\t\tmaxAckPending := 1\n\t\t\tackWait := 20 * time.Millisecond\n\t\t\texpSeq := uint64(4)\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"d22\",\n\t\t\t\tDeliverPolicy: DeliverByStartSequence,\n\t\t\t\tOptStartSeq: expSeq,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: ackWait,\n\t\t\t\tMaxAckPending: maxAckPending,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdefer o.delete()\n\n\t\t\tgetSubj := o.requestNextMsgSubject()\n\t\t\tdelivery := uint64(1)\n\n\t\t\tgetNext := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := nc.Request(getSubj, nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tsseq, dseq, dcount, _, pending := replyInfo(m.Reply)\n\t\t\t\tif sseq != expSeq {\n\t\t\t\t\tt.Fatalf(\"Expected stream sequence of %d, got %d\", expSeq, sseq)\n\t\t\t\t}\n\t\t\t\tif dseq != delivery {\n\t\t\t\t\tt.Fatalf(\"Expected consumer sequence of %d, got %d\", delivery, dseq)\n\t\t\t\t}\n\t\t\t\tif dcount != delivery {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count of %d, got %d\", delivery, dcount)\n\t\t\t\t}\n\t\t\t\tif pending != uint64(toSend)-expSeq {\n\t\t\t\t\tt.Fatalf(\"Expected pending to be %d, got %d\", uint64(toSend)-expSeq, pending)\n\t\t\t\t}\n\t\t\t\tdelivery++\n\t\t\t}\n\n\t\t\tgetNext()\n\t\t\tgetNext()\n\t\t\tgetNext()\n\t\t\tgetNext()\n\t\t\tgetNext()\n\t\t})\n\t}\n}\n\n// User report of bug.\nfunc TestJetStreamConsumerBadNumPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS\",\n\t\tSubjects: []string{\"orders.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tnewOrders := func(n int) {\n\t\t// Queue up new orders.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tjs.Publish(\"orders.created\", []byte(\"NEW\"))\n\t\t}\n\t}\n\n\tnewOrders(10)\n\n\t// Create to subscribers.\n\tprocess := func(m *nats.Msg) {\n\t\tjs.Publish(\"orders.approved\", []byte(\"APPROVED\"))\n\t}\n\n\top, err := js.Subscribe(\"orders.created\", process)\n\trequire_NoError(t, err)\n\n\tdefer op.Unsubscribe()\n\n\tmon, err := js.SubscribeSync(\"orders.*\")\n\trequire_NoError(t, err)\n\n\tdefer mon.Unsubscribe()\n\n\twaitForMsgs := func(n uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"ORDERS\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != n {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", n, si.State)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckForNoPending := func(sub *nats.Subscription) {\n\t\tt.Helper()\n\t\tif ci, err := sub.ConsumerInfo(); err != nil || ci == nil || ci.NumPending != 0 {\n\t\t\tif ci != nil && ci.NumPending != 0 {\n\t\t\t\tt.Fatalf(\"Bad consumer NumPending, expected 0 but got %d\", ci.NumPending)\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Bad consumer info: %+v\", ci)\n\t\t\t}\n\t\t}\n\t}\n\n\twaitForMsgs(20)\n\tcheckForNoPending(op)\n\tcheckForNoPending(mon)\n\n\tnewOrders(10)\n\n\twaitForMsgs(40)\n\tcheckForNoPending(op)\n\tcheckForNoPending(mon)\n}\n\n// We had a report of a consumer delete crashing the server when in interest retention mode.\n// This I believe is only really possible in clustered mode, but we will force the issue here.\nfunc TestJetStreamConsumerCleanupWithRetentionPolicy(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS\",\n\t\tSubjects: []string{\"orders.*\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"orders.*\")\n\trequire_NoError(t, err)\n\n\tpayload := []byte(\"Hello World\")\n\tfor i := 0; i < 10; i++ {\n\t\tsubj := fmt.Sprintf(\"orders.%d\", i+1)\n\t\tjs.Publish(subj, payload)\n\t}\n\n\tcheckSubsPending(t, sub, 10)\n\n\tfor i := 0; i < 10; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tm.AckSync()\n\t}\n\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting consumer info: %v\", err)\n\t}\n\n\tacc := s.GlobalAccount()\n\tmset, err := acc.lookupStream(\"ORDERS\")\n\trequire_NoError(t, err)\n\n\to := mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\tlseq := mset.lastSeq()\n\to.mu.Lock()\n\t// Force boundary condition here.\n\to.asflr = lseq + 2\n\to.mu.Unlock()\n\tsub.Unsubscribe()\n\n\t// Make sure server still available.\n\tif _, err := js.StreamInfo(\"ORDERS\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamConsumerPendingBugWithKV(t *testing.T) {\n\tmsc := StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"kv.>\"},\n\t\tStorage: MemoryStorage,\n\t\tMaxMsgsPer: 1,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Client based API\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Not in Go client under server yet.\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tjs.Publish(\"kv.1\", []byte(\"1\"))\n\t\t\tjs.Publish(\"kv.2\", []byte(\"2\"))\n\t\t\tjs.Publish(\"kv.3\", []byte(\"3\"))\n\t\t\tjs.Publish(\"kv.1\", []byte(\"4\"))\n\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != 3 {\n\t\t\t\tt.Fatalf(\"Expected 3 total msgs, got %d\", si.State.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dlc\",\n\t\t\t\tDeliverSubject: \"xxx\",\n\t\t\t\tDeliverPolicy: DeliverLastPerSubject,\n\t\t\t\tFilterSubject: \">\",\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif ci := o.info(); ci.NumPending != 3 {\n\t\t\t\tt.Fatalf(\"Expected pending of 3, got %d\", ci.NumPending)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Issue #2423\nfunc TestJetStreamConsumerBadCreateErr(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: nats.MemoryStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t// When adding a consumer with both deliver subject and max wait (push vs pull),\n\t// we got the wrong err about deliver subject having a wildcard.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"nowcerr\",\n\t\tDeliverSubject: \"X\",\n\t\tMaxWaiting: 100,\n\t})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\tif !strings.Contains(err.Error(), \"push mode can not set max waiting\") {\n\t\tt.Fatalf(\"Incorrect error returned: %v\", err)\n\t}\n}\n\nfunc TestJetStreamConsumerPushBound(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// We want to test extended consumer info for push based consumers.\n\t// We need to do these by hand for now.\n\tcreateConsumer := func(name, deliver string) {\n\t\tt.Helper()\n\t\tcreq := CreateConsumerRequest{\n\t\t\tStream: \"TEST\",\n\t\t\tConfig: ConsumerConfig{\n\t\t\t\tDurable: name,\n\t\t\t\tDeliverSubject: deliver,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t},\n\t\t}\n\t\treq, err := json.Marshal(creq)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", name), req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar ccResp JSApiConsumerCreateResponse\n\t\tif err := json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ccResp.ConsumerInfo == nil || ccResp.Error != nil {\n\t\t\tt.Fatalf(\"Got a bad response %+v\", ccResp)\n\t\t}\n\t}\n\n\tconsumerInfo := func(name string) *ConsumerInfo {\n\t\tt.Helper()\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerInfoT, \"TEST\", name), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar cinfo JSApiConsumerInfoResponse\n\t\tif err := json.Unmarshal(resp.Data, &cinfo); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cinfo.ConsumerInfo == nil || cinfo.Error != nil {\n\t\t\tt.Fatalf(\"Got a bad response %+v\", cinfo)\n\t\t}\n\t\treturn cinfo.ConsumerInfo\n\t}\n\n\t// First create a durable push and make sure we show now active status.\n\tcreateConsumer(\"dlc\", \"d.X\")\n\tif ci := consumerInfo(\"dlc\"); ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be false\")\n\t}\n\t// Now bind the deliver subject.\n\tsub, _ := nc.SubscribeSync(\"d.X\")\n\tnc.Flush() // Make sure it registers.\n\t// Check that its reported.\n\tif ci := consumerInfo(\"dlc\"); !ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be set\")\n\t}\n\tsub.Unsubscribe()\n\tnc.Flush() // Make sure it registers.\n\tif ci := consumerInfo(\"dlc\"); ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be false\")\n\t}\n\n\t// Now make sure we have queue groups indictated as needed.\n\tcreateConsumer(\"ik\", \"d.Z\")\n\t// Now bind the deliver subject with a queue group.\n\tsub, _ = nc.QueueSubscribeSync(\"d.Z\", \"g22\")\n\tdefer sub.Unsubscribe()\n\tnc.Flush() // Make sure it registers.\n\t// Check that queue group is not reported.\n\tif ci := consumerInfo(\"ik\"); ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be false\")\n\t}\n\tsub.Unsubscribe()\n\tnc.Flush() // Make sure it registers.\n\tif ci := consumerInfo(\"ik\"); ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be false\")\n\t}\n\n\t// Make sure pull consumers report PushBound as false by default.\n\tcreateConsumer(\"rip\", _EMPTY_)\n\tif ci := consumerInfo(\"rip\"); ci.PushBound {\n\t\tt.Fatalf(\"Expected push bound to be false\")\n\t}\n}\n\n// Got a report of memory leaking, tracked it to internal clients for consumers.\nfunc TestJetStreamConsumerInternalClientLeak(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.MemoryStorage,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tga, sa := s.GlobalAccount(), s.SystemAccount()\n\tncb, nscb := ga.NumConnections(), sa.NumConnections()\n\n\t// Create 10 consumers\n\tfor i := 0; i < 10; i++ {\n\t\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{DeliverSubject: \"x\"})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Accelerate ephemeral cleanup.\n\t\tmset, err := ga.lookupStream(\"TEST\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to find a stream for %q\", \"TEST\")\n\t\t}\n\t\to := mset.lookupConsumer(ci.Name)\n\t\tif o == nil {\n\t\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t\t}\n\t\to.setInActiveDeleteThreshold(500 * time.Millisecond)\n\t}\n\n\t// Wait for them to all go away.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Consumers == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Consumers still present\")\n\t})\n\t// Make sure we are not leaking clients/connections.\n\t// Server does not see these so need to look at account.\n\tif nca := ga.NumConnections(); nca != ncb {\n\t\tt.Fatalf(\"Leaked clients in global account: %d vs %d\", ncb, nca)\n\t}\n\tif nsca := sa.NumConnections(); nsca != nscb {\n\t\tt.Fatalf(\"Leaked clients in system account: %d vs %d\", nscb, nsca)\n\t}\n}\n\nfunc TestJetStreamConsumerEventingRaceOnShutdown(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.NoReconnect())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.MemoryStorage,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tfor {\n\t\t\tif _, err := js.SubscribeSync(\"foo\", nats.BindStream(\"TEST\")); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(50 * time.Millisecond)\n\ts.Shutdown()\n\n\twg.Wait()\n}\n\nfunc TestJetStreamConsumerNoMsgPayload(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S\"})\n\trequire_NoError(t, err)\n\n\tmsg := nats.NewMsg(\"S\")\n\tmsg.Header.Set(\"name\", \"derek\")\n\tmsg.Data = bytes.Repeat([]byte(\"A\"), 128)\n\tfor i := 0; i < 10; i++ {\n\t\tmsg.Reply = _EMPTY_ // Fixed in Go client but not in embdedded on yet.\n\t\t_, err = js.PublishMsgAsync(msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tmset, err := s.GlobalAccount().lookupStream(\"S\")\n\trequire_NoError(t, err)\n\n\t// Now create our consumer with no payload option.\n\t_, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: \"_d_\", Durable: \"d22\", HeadersOnly: true})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"S\", nats.Durable(\"d22\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tif len(m.Data) > 0 {\n\t\t\tt.Fatalf(\"Expected no payload\")\n\t\t}\n\t\tif ms := m.Header.Get(JSMsgSize); ms != \"128\" {\n\t\t\tt.Fatalf(\"Expected a header with msg size, got %q\", ms)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamConsumerUpdateSurvival(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"X\"})\n\trequire_NoError(t, err)\n\n\t// First create a consumer with max ack pending.\n\t_, err = js.AddConsumer(\"X\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: 1024,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now do same name but pull. This will update the MaxAcKPending\n\tci, err := js.UpdateConsumer(\"X\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: 22,\n\t})\n\trequire_NoError(t, err)\n\n\tif ci.Config.MaxAckPending != 22 {\n\t\tt.Fatalf(\"Expected MaxAckPending to be 22, got %d\", ci.Config.MaxAckPending)\n\t}\n\n\t// Make sure this survives across a restart.\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tci, err = js.ConsumerInfo(\"X\", \"dlc\")\n\trequire_NoError(t, err)\n\n\tif ci.Config.MaxAckPending != 22 {\n\t\tt.Fatalf(\"Expected MaxAckPending to be 22, got %d\", ci.Config.MaxAckPending)\n\t}\n}\n\nfunc TestJetStreamConsumerPendingCountWithRedeliveries(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"test\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: 50 * time.Millisecond,\n\t\tMaxDeliver: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish 1st message\n\t_, err = js.Publish(\"foo\", []byte(\"msg1\"))\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"test\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\tfor _, m := range msgs {\n\t\trequire_Equal(t, string(m.Data), \"msg1\")\n\t\t// Do not ack the message\n\t}\n\t// Check consumer info, pending should be 0\n\tci, err := js.ConsumerInfo(\"TEST\", \"test\")\n\trequire_NoError(t, err)\n\tif ci.NumPending != 0 {\n\t\tt.Fatalf(\"Expected consumer info pending count to be 0, got %v\", ci.NumPending)\n\t}\n\n\t// Wait for more than expiration\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Publish 2nd message\n\t_, err = js.Publish(\"foo\", []byte(\"msg2\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\tfor _, m := range msgs {\n\t\trequire_Equal(t, string(m.Data), \"msg2\")\n\t\t// Its deliver count should be 1\n\t\tmeta, err := m.Metadata()\n\t\trequire_NoError(t, err)\n\t\tif meta.NumDelivered != 1 {\n\t\t\tt.Fatalf(\"Expected message's deliver count to be 1, got %v\", meta.NumDelivered)\n\t\t}\n\t}\n\t// Check consumer info, pending should be 0\n\tci, err = js.ConsumerInfo(\"TEST\", \"test\")\n\trequire_NoError(t, err)\n\tif ci.NumPending != 0 {\n\t\tt.Fatalf(\"Expected consumer info pending count to be 0, got %v\", ci.NumPending)\n\t}\n}\n\nfunc TestJetStreamConsumerPullHeartBeats(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"T\", Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"T\", \"dlc\")\n\n\ttype tsMsg struct {\n\t\treceived time.Time\n\t\tmsg *nats.Msg\n\t}\n\n\tdoReq := func(batch int, hb, expires time.Duration, expected int) []*tsMsg {\n\t\tt.Helper()\n\t\treq := &JSApiConsumerGetNextRequest{Batch: batch, Expires: expires, Heartbeat: hb}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\treply := nats.NewInbox()\n\t\tvar msgs []*tsMsg\n\t\tvar mu sync.Mutex\n\n\t\tsub, err := nc.Subscribe(reply, func(m *nats.Msg) {\n\t\t\tmu.Lock()\n\t\t\tmsgs = append(msgs, &tsMsg{time.Now(), m})\n\t\t\tmu.Unlock()\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\terr = nc.PublishRequest(rsubj, reply, jreq)\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\t\tmu.Lock()\n\t\t\tnr := len(msgs)\n\t\t\tmu.Unlock()\n\t\t\tif nr >= expected {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Only have seen %d of %d responses\", nr, expected)\n\t\t})\n\t\tsub.Unsubscribe()\n\t\treturn msgs\n\t}\n\n\treqBad := nats.Header{\"Status\": []string{\"400\"}, \"Description\": []string{\"Bad Request - heartbeat value too large\"}}\n\texpectErr := func(msgs []*tsMsg) {\n\t\tt.Helper()\n\t\tif len(msgs) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 msg, got %d\", len(msgs))\n\t\t}\n\t\tif !reflect.DeepEqual(msgs[0].msg.Header, reqBad) {\n\t\t\tt.Fatalf(\"Expected %+v hdr, got %+v\", reqBad, msgs[0].msg.Header)\n\t\t}\n\t}\n\n\t// Test errors first.\n\t// Setting HB with no expires.\n\texpectErr(doReq(1, 100*time.Millisecond, 0, 1))\n\t// If HB larger than 50% of expires..\n\texpectErr(doReq(1, 75*time.Millisecond, 100*time.Millisecond, 1))\n\n\texpectHBs := func(start time.Time, msgs []*tsMsg, expected int, hbi time.Duration) {\n\t\tt.Helper()\n\t\tif len(msgs) != expected {\n\t\t\tt.Fatalf(\"Expected %d but got %d\", expected, len(msgs))\n\t\t}\n\t\t// expected -1 should be all HBs.\n\t\tfor i, ts := 0, start; i < expected-1; i++ {\n\t\t\ttr, m := msgs[i].received, msgs[i].msg\n\t\t\tif m.Header.Get(\"Status\") != \"100\" {\n\t\t\t\tt.Fatalf(\"Expected a 100 status code, got %q\", m.Header.Get(\"Status\"))\n\t\t\t}\n\t\t\tif m.Header.Get(\"Description\") != \"Idle Heartbeat\" {\n\t\t\t\tt.Fatalf(\"Wrong description, got %q\", m.Header.Get(\"Description\"))\n\t\t\t}\n\t\t\tts = ts.Add(hbi)\n\t\t\tif tr.Before(ts) {\n\t\t\t\tt.Fatalf(\"Received at wrong time: %v vs %v\", tr, ts)\n\t\t\t}\n\t\t}\n\t\t// Last msg should be timeout.\n\t\tlm := msgs[len(msgs)-1].msg\n\t\tif key := lm.Header.Get(\"Status\"); key != \"408\" {\n\t\t\tt.Fatalf(\"Expected 408 Request Timeout, got %s\", key)\n\t\t}\n\t}\n\n\t// These should work. Test idle first.\n\tstart, msgs := time.Now(), doReq(1, 50*time.Millisecond, 250*time.Millisecond, 5)\n\texpectHBs(start, msgs, 5, 50*time.Millisecond)\n\n\t// Now test that we do not send heartbeats while we receive traffic.\n\tgo func() {\n\t\tfor i := 0; i < 5; i++ {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tjs.Publish(\"T\", nil)\n\t\t}\n\t}()\n\n\tmsgs = doReq(10, 75*time.Millisecond, 350*time.Millisecond, 6)\n\t// The first 5 should be msgs, no HBs.\n\tfor i := 0; i < 5; i++ {\n\t\tif m := msgs[i].msg; len(m.Header) > 0 {\n\t\t\tt.Fatalf(\"Got a potential heartbeat msg when we should not have: %+v\", m.Header)\n\t\t}\n\t}\n\t// Last should be timeout.\n\tlm := msgs[len(msgs)-1].msg\n\tif key := lm.Header.Get(\"Status\"); key != \"408\" {\n\t\tt.Fatalf(\"Expected 408 Request Timeout, got %s\", key)\n\t}\n}\n\nfunc TestJetStreamConsumerAckSampling(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tSampleFrequency: \"100%\",\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\n\tmsub, err := nc.SubscribeSync(\"$JS.EVENT.METRIC.>\")\n\trequire_NoError(t, err)\n\n\tfor _, m := range fetchMsgs(t, sub, 1, time.Second) {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\tm, err := msub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tvar am JSConsumerAckMetric\n\terr = json.Unmarshal(m.Data, &am)\n\trequire_NoError(t, err)\n\n\tif am.Stream != \"TEST\" || am.Consumer != \"dlc\" || am.ConsumerSeq != 1 {\n\t\tt.Fatalf(\"Not a proper ack metric: %+v\", am)\n\t}\n\n\t// Do less than 100%\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"alps\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tSampleFrequency: \"50%\",\n\t})\n\trequire_NoError(t, err)\n\n\tasub, err := js.PullSubscribe(\"foo\", \"alps\")\n\trequire_NoError(t, err)\n\n\ttotal := 500\n\tfor i := 0; i < total; i++ {\n\t\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tmp := 0\n\tfor _, m := range fetchMsgs(t, asub, total, time.Second) {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t\tmp++\n\t}\n\tnc.Flush()\n\n\tif mp != total {\n\t\tt.Fatalf(\"Got only %d msgs out of %d\", mp, total)\n\t}\n\n\tnmsgs, _, err := msub.Pending()\n\trequire_NoError(t, err)\n\n\t// Should be ~250\n\tif nmsgs < 200 || nmsgs > 300 {\n\t\tt.Fatalf(\"Expected about 250, got %d\", nmsgs)\n\t}\n}\n\nfunc TestJetStreamConsumerAckSamplingSpecifiedUsingUpdateConsumer(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tSampleFrequency: \"100%\",\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\n\tmsub, err := nc.SubscribeSync(\"$JS.EVENT.METRIC.>\")\n\trequire_NoError(t, err)\n\n\tfor _, m := range fetchMsgs(t, sub, 1, time.Second) {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\tm, err := msub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tvar am JSConsumerAckMetric\n\terr = json.Unmarshal(m.Data, &am)\n\trequire_NoError(t, err)\n\n\tif am.Stream != \"TEST\" || am.Consumer != \"dlc\" || am.ConsumerSeq != 1 {\n\t\tt.Fatalf(\"Not a proper ack metric: %+v\", am)\n\t}\n}\n\nfunc TestJetStreamConsumerMaxDeliverUpdate(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\tmaxDeliver := 2\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ard\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tMaxDeliver: maxDeliver,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"ard\")\n\trequire_NoError(t, err)\n\n\tcheckMaxDeliver := func() {\n\t\tt.Helper()\n\t\tfor i := 0; i <= maxDeliver; i++ {\n\t\t\tmsgs, err := sub.Fetch(2, nats.MaxWait(100*time.Millisecond))\n\t\t\tif i < maxDeliver {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, 1, len(msgs))\n\t\t\t\t_ = msgs[0].Nak()\n\t\t\t} else {\n\t\t\t\trequire_Error(t, err, nats.ErrTimeout)\n\t\t\t}\n\t\t}\n\t}\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\tcheckMaxDeliver()\n\n\t// update maxDeliver\n\tmaxDeliver++\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"ard\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tMaxDeliver: maxDeliver,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\tcheckMaxDeliver()\n}\n\nfunc TestJetStreamConsumerStreamUpdate(t *testing.T) {\n\ttest := func(t *testing.T, s *Server, replica int) {\n\t\tnc := natsConnect(t, s.ClientURL())\n\t\tdefer nc.Close()\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"foo\", Duplicates: 1 * time.Minute, Replicas: replica})\n\t\tdefer js.DeleteStream(\"foo\")\n\t\trequire_NoError(t, err)\n\t\t// Update with no change\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"foo\", Duplicates: 1 * time.Minute, Replicas: replica})\n\t\trequire_NoError(t, err)\n\t\t// Update with change\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Description: \"stream\", Name: \"foo\", Duplicates: 1 * time.Minute, Replicas: replica})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t\t// Update with no change\n\t\t_, err = js.UpdateConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t\t// Update with change\n\t\t_, err = js.UpdateConsumer(\"foo\", &nats.ConsumerConfig{Description: \"consumer\", Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t}\n\tt.Run(\"clustered\", func(t *testing.T) {\n\t\tc := createJetStreamClusterWithTemplate(t, `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t\t\tno_auth_user: u\n\t\t\taccounts {\n\t\t\t\tONE {\n\t\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\t\tjetstream: enabled\n\t\t\t\t}\n\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t}`, \"clust\", 3)\n\t\tdefer c.shutdown()\n\t\ts := c.randomServer()\n\t\tt.Run(\"r3\", func(t *testing.T) {\n\t\t\ttest(t, s, 3)\n\t\t})\n\t\tt.Run(\"r1\", func(t *testing.T) {\n\t\t\ttest(t, s, 1)\n\t\t})\n\t})\n\tt.Run(\"single\", func(t *testing.T) {\n\t\tstoreDir := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}`,\n\t\t\tstoreDir)))\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\t\ttest(t, s, 1)\n\t})\n}\n\nfunc TestJetStreamConsumerDeliverNewNotConsumingBeforeRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: inbox,\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\tcheckCount := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := int(ci.NumPending); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %v pending, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckCount(10)\n\n\ttime.Sleep(300 * time.Millisecond)\n\n\t// Check server restart\n\tnc.Close()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcheckCount(10)\n\n\t// Make sure messages can be consumed\n\tsub := natsSubSync(t, nc, inbox)\n\tfor i := 0; i < 10; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"i=%v next msg error: %v\", i, err)\n\t\t}\n\t\tmsg.AckSync()\n\t}\n\tcheckCount(0)\n}\n\nfunc TestJetStreamConsumerNumPendingWithMaxPerSubjectGreaterThanOne(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttest := func(t *testing.T, st nats.StorageType) {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"KV.*.*\"},\n\t\t\tMaxMsgsPerSubject: 2,\n\t\t\tStorage: st,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// If we allow more than one msg per subject, consumer's num pending can be off (bug in store level).\n\t\t// This requires a filtered state, simple states work ok.\n\t\t// Since we now rely on stream's filtered state when asked directly for consumer info in >=2.8.3.\n\t\tjs.PublishAsync(\"KV.plans.foo\", []byte(\"OK\"))\n\t\tjs.PublishAsync(\"KV.plans.bar\", []byte(\"OK\"))\n\t\tjs.PublishAsync(\"KV.plans.baz\", []byte(\"OK\"))\n\t\t// These are required, the consumer needs to filter these out to see the bug.\n\t\tjs.PublishAsync(\"KV.config.foo\", []byte(\"OK\"))\n\t\tjs.PublishAsync(\"KV.config.bar\", []byte(\"OK\"))\n\t\tjs.PublishAsync(\"KV.config.baz\", []byte(\"OK\"))\n\n\t\t// Double up some now.\n\t\tjs.PublishAsync(\"KV.plans.bar\", []byte(\"OK\"))\n\t\tjs.PublishAsync(\"KV.plans.baz\", []byte(\"OK\"))\n\n\t\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"d\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\t\tFilterSubject: \"KV.plans.*\",\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\terr = js.DeleteStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tif ci.NumPending != 3 {\n\t\t\tt.Fatalf(\"Expected 3 NumPending, but got %d\", ci.NumPending)\n\t\t}\n\t}\n\n\tt.Run(\"MemoryStore\", func(t *testing.T) { test(t, nats.MemoryStorage) })\n\tt.Run(\"FileStore\", func(t *testing.T) { test(t, nats.FileStorage) })\n}\n\nfunc TestJetStreamConsumerAndStreamNamesWithPathSeparators(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"usr/bin\"})\n\trequire_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: `Documents\\readme.txt`})\n\trequire_Error(t, err, NewJSStreamNameContainsPathSeparatorsError(), nats.ErrInvalidStreamName)\n\n\t// Now consumers.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"T\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: \"a/b\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: `a\\b`, AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err, NewJSConsumerNameContainsPathSeparatorsError(), nats.ErrInvalidConsumerName)\n}\n\nfunc TestJetStreamConsumerUpdateFilterSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"T\", Subjects: []string{\"foo\", \"bar\", \"baz\"}})\n\trequire_NoError(t, err)\n\n\t// 10 foo\n\tfor i := 0; i < 10; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t// 20 bar\n\tfor i := 0; i < 20; i++ {\n\t\tjs.PublishAsync(\"bar\", []byte(\"OK\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"d\")\n\trequire_NoError(t, err)\n\n\t// Consume 5 msgs\n\tmsgs, err := sub.Fetch(5)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 5)\n\n\t// Now update to different filter subject.\n\t_, err = js.UpdateConsumer(\"T\", &nats.ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tFilterSubject: \"bar\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err = js.PullSubscribe(\"bar\", \"d\")\n\trequire_NoError(t, err)\n\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\n\t// Make sure meta and pending etc are all correct.\n\tm := msgs[0]\n\tmeta, err := m.Metadata()\n\trequire_NoError(t, err)\n\n\tif meta.Sequence.Consumer != 6 || meta.Sequence.Stream != 11 {\n\t\tt.Fatalf(\"Sequence incorrect %+v\", meta.Sequence)\n\t}\n\tif meta.NumDelivered != 1 {\n\t\tt.Fatalf(\"Expected NumDelivered to be 1, got %d\", meta.NumDelivered)\n\t}\n\tif meta.NumPending != 19 {\n\t\tt.Fatalf(\"Expected NumPending to be 19, got %d\", meta.NumPending)\n\t}\n}\n\n// Originally pull consumers were FIFO with respect to the request, not delivery of messages.\n// We have changed to have the behavior be FIFO but on an individual message basis.\n// So after a message is delivered, the request, if still outstanding, effectively\n// goes to the end of the queue of requests pending.\nfunc TestJetStreamConsumerPullConsumerFIFO(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"T\"})\n\trequire_NoError(t, err)\n\n\t// Create pull consumer.\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: \"d\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// Simulate 10 pull requests each asking for 10 messages.\n\tvar subs []*nats.Subscription\n\tfor i := 0; i < 10; i++ {\n\t\tinbox := nats.NewInbox()\n\t\tsub := natsSubSync(t, nc, inbox)\n\t\tsubs = append(subs, sub)\n\t\treq := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 60 * time.Second}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(fmt.Sprintf(JSApiRequestNextT, \"T\", \"d\"), inbox, jreq)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Not all core publishes could have made it to the leader yet, wait for some time.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Now send 100 messages.\n\tfor i := 0; i < 100; i++ {\n\t\t_, err = js.Publish(\"T\", []byte(\"FIFO FTW!\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for messages\n\tfor _, sub := range subs {\n\t\tcheckSubsPending(t, sub, 10)\n\t}\n\n\t// Confirm FIFO\n\tfor index, sub := range subs {\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tmeta, err := m.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\t// We expect these to be FIFO per message. E.g. sub #1 = [1, 11, 21, 31, ..]\n\t\t\tif sseq := meta.Sequence.Stream; sseq != uint64(index+1+(10*i)) {\n\t\t\t\tt.Fatalf(\"Expected message #%d for sub #%d to be %d, but got %d\", i+1, index+1, index+1+(10*i), sseq)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Make sure that when we reach an ack limit that we follow one shot semantics.\nfunc TestJetStreamConsumerPullConsumerOneShotOnMaxAckLimit(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"T\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tjs.Publish(\"T\", []byte(\"OK\"))\n\t}\n\n\tsub, err := js.PullSubscribe(\"T\", \"d\", nats.MaxAckPending(5))\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\tmsgs, err := sub.Fetch(10, nats.MaxWait(2*time.Second))\n\trequire_NoError(t, err)\n\n\tif elapsed := time.Since(start); elapsed >= 2*time.Second {\n\t\tt.Fatalf(\"Took too long, not one shot behavior: %v\", elapsed)\n\t}\n\n\tif len(msgs) != 5 {\n\t\tt.Fatalf(\"Expected 5 msgs, got %d\", len(msgs))\n\t}\n}\n\nfunc TestJetStreamConsumerDeliverNewMaxRedeliveriesAndServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tinbox := nats.NewInbox()\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: inbox,\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\tMaxDeliver: 3,\n\t\tAckWait: 250 * time.Millisecond,\n\t\tFilterSubject: \"foo.bar\",\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo.bar\", \"msg\")\n\n\tsub := natsSubSync(t, nc, inbox)\n\tfor i := 0; i < 3; i++ {\n\t\tnatsNexMsg(t, sub, time.Second)\n\t}\n\t// Now check that there is no more redeliveries\n\tif msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout, got msg=%+v err=%v\", msg, err)\n\t}\n\n\t// Give a chance to things to be persisted\n\ttime.Sleep(300 * time.Millisecond)\n\n\t// Check server restart\n\tnc.Close()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, _ = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub = natsSubSync(t, nc, inbox)\n\t// We should not have messages being redelivered.\n\tif msg, err := sub.NextMsg(300 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout, got msg=%+v err=%v\", msg, err)\n\t}\n}\n\nfunc TestJetStreamConsumerPendingLowerThanStreamFirstSeq(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\tinbox := nats.NewInbox()\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDeliverSubject: inbox,\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverAllPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub := natsSubSync(t, nc, inbox)\n\tfor i := 0; i < 10; i++ {\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"dur\")\n\trequire_True(t, o != nil)\n\to.stop()\n\tmset.store.Compact(1_000_000)\n\tnc.Close()\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 1_000_000)\n\trequire_True(t, si.State.LastSeq == 999_999)\n\n\tacc, err = s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err = acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to = mset.lookupConsumer(\"dur\")\n\trequire_True(t, o != nil)\n\n\tnatsSubSync(t, nc, inbox)\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ci.NumAckPending != 0 {\n\t\t\treturn fmt.Errorf(\"NumAckPending should be 0, got %v\", ci.NumAckPending)\n\t\t}\n\t\t// Delivered stays the same because it reflects what was actually delivered.\n\t\t// And must not be influenced by purges/compacts.\n\t\tif ci.Delivered.Stream != 10 {\n\t\t\treturn fmt.Errorf(\"Delivered.Stream should be 10, got %v\", ci.Delivered.Stream)\n\t\t}\n\n\t\t// Starting sequence should be skipped ahead, respecting the compact.\n\t\to.mu.RLock()\n\t\tsseq := o.sseq\n\t\to.mu.RUnlock()\n\t\tif sseq != 1_000_000 {\n\t\t\treturn fmt.Errorf(\"o.sseq should be 1,000,000, got %v\", sseq)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Bug when stream's consumer config does not force filestore to track per subject information.\nfunc TestJetStreamConsumerEOFBugNewFileStore(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.bar.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\tdsubj := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(dsubj)\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\t// Filter needs to be a wildcard. Need to bind to the\n\t_, err = js.AddConsumer(\"M\", &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: \"foo.>\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\t_, err := js.PublishAsync(\"foo.bar.baz\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tm.Respond(nil)\n\t}\n\n\t// Now force an expiration.\n\tmset, err := s.GlobalAccount().lookupStream(\"M\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tstore := mset.store.(*fileStore)\n\tmset.mu.RUnlock()\n\tstore.mu.RLock()\n\tmb := store.blks[0]\n\tstore.mu.RUnlock()\n\tmb.mu.Lock()\n\tmb.fss = nil\n\tmb.mu.Unlock()\n\n\t// Now send another message.\n\t_, err = js.PublishAsync(\"foo.bar.baz\", []byte(\"OK\"))\n\trequire_NoError(t, err)\n\n\t// This will fail with the bug.\n\t_, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamConsumerMultipleSubjectsLast(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tdurable := \"durable\"\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tName: \"name\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"error while creating stream\")\n\t}\n\n\tsendStreamMsg(t, nc, \"events\", \"1\")\n\tsendStreamMsg(t, nc, \"data\", \"2\")\n\tsendStreamMsg(t, nc, \"other\", \"3\")\n\tsendStreamMsg(t, nc, \"events\", \"4\")\n\tsendStreamMsg(t, nc, \"data\", \"5\")\n\tsendStreamMsg(t, nc, \"data\", \"6\")\n\tsendStreamMsg(t, nc, \"other\", \"7\")\n\tsendStreamMsg(t, nc, \"other\", \"8\")\n\n\t// if they're not the same, expect error\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverPolicy: DeliverLast,\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tDurable: durable,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(_EMPTY_, nats.Bind(\"name\", durable))\n\trequire_NoError(t, err)\n\n\tmsg, err := sub.NextMsg(time.Millisecond * 500)\n\trequire_NoError(t, err)\n\n\tj, err := strconv.Atoi(string(msg.Data))\n\trequire_NoError(t, err)\n\texpectedStreamSeq := 6\n\tif j != expectedStreamSeq {\n\t\tt.Fatalf(\"wrong sequence, expected %v got %v\", expectedStreamSeq, j)\n\t}\n\n\trequire_NoError(t, msg.AckSync())\n\n\t// check if we don't get more than we wanted\n\tmsg, err = sub.NextMsg(time.Millisecond * 500)\n\tif msg != nil || err == nil {\n\t\tt.Fatalf(\"should not get more messages\")\n\t}\n\n\tinfo, err := js.ConsumerInfo(\"name\", durable)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, info.NumAckPending, 0)\n\trequire_Equal(t, info.AckFloor.Stream, 6)\n\trequire_Equal(t, info.AckFloor.Consumer, 1)\n\trequire_Equal(t, info.NumPending, 0)\n}\n\nfunc TestJetStreamConsumerMultipleSubjectsLastPerSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tdurable := \"durable\"\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events.*\", \"data.>\", \"other\"},\n\t\tName: \"name\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"error while creating stream\")\n\t}\n\n\tsendStreamMsg(t, nc, \"events.1\", \"bad\")\n\tsendStreamMsg(t, nc, \"events.1\", \"events.1\")\n\n\tsendStreamMsg(t, nc, \"data.1\", \"bad\")\n\tsendStreamMsg(t, nc, \"data.1\", \"bad\")\n\tsendStreamMsg(t, nc, \"data.1\", \"bad\")\n\tsendStreamMsg(t, nc, \"data.1\", \"bad\")\n\tsendStreamMsg(t, nc, \"data.1\", \"data.1\")\n\n\tsendStreamMsg(t, nc, \"events.2\", \"bad\")\n\tsendStreamMsg(t, nc, \"events.2\", \"bad\")\n\t// this is last proper sequence,\n\tsendStreamMsg(t, nc, \"events.2\", \"events.2\")\n\n\tsendStreamMsg(t, nc, \"other\", \"bad\")\n\tsendStreamMsg(t, nc, \"other\", \"bad\")\n\n\t// if they're not the same, expect error\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverPolicy: DeliverLastPerSubject,\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubjects: []string{\"events.*\", \"data.>\"},\n\t\tDurable: durable,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"\", nats.Bind(\"name\", durable))\n\trequire_NoError(t, err)\n\n\tcheckMessage := func(t *testing.T, subject string, payload string, ack bool) {\n\t\tmsg, err := sub.NextMsg(time.Millisecond * 500)\n\t\trequire_NoError(t, err)\n\n\t\tif string(msg.Data) != payload {\n\t\t\tt.Fatalf(\"expected %v paylaod, got %v\", payload, string(msg.Data))\n\t\t}\n\t\tif subject != msg.Subject {\n\t\t\tt.Fatalf(\"expected %v subject, got %v\", subject, msg.Subject)\n\t\t}\n\t\tif ack {\n\t\t\tmsg.AckSync()\n\t\t}\n\t}\n\n\tcheckMessage(t, \"events.1\", \"events.1\", true)\n\tcheckMessage(t, \"data.1\", \"data.1\", true)\n\tcheckMessage(t, \"events.2\", \"events.2\", false)\n\n\tinfo, err := js.ConsumerInfo(\"name\", durable)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, info.AckFloor.Consumer, 2)\n\trequire_Equal(t, info.AckFloor.Stream, 9)\n\trequire_Equal(t, info.Delivered.Stream, 10)\n\trequire_Equal(t, info.Delivered.Consumer, 3)\n\n\trequire_NoError(t, err)\n\n}\nfunc TestJetStreamConsumerMultipleSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tdurable := \"durable\"\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{\n\t\tSubjects: []string{\"events.>\", \"data.>\"},\n\t\tName: \"name\",\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i += 2 {\n\t\tsendStreamMsg(t, nc, \"events.created\", fmt.Sprintf(\"created %v\", i))\n\t\tsendStreamMsg(t, nc, \"data.processed\", fmt.Sprintf(\"processed %v\", i+1))\n\t}\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: durable,\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubjects: []string{\"events.created\", \"data.processed\"},\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"\", nats.Bind(\"name\", durable))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\tmsg, err := sub.NextMsg(time.Millisecond * 500)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\tinfo, err := js.ConsumerInfo(\"name\", durable)\n\trequire_NoError(t, err)\n\trequire_True(t, info.NumAckPending == 0)\n\trequire_True(t, info.NumPending == 0)\n\trequire_True(t, info.AckFloor.Consumer == 20)\n\trequire_True(t, info.AckFloor.Stream == 20)\n\n}\n\nfunc TestJetStreamConsumerMultipleSubjectsWithEmpty(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tdurable := \"durable\"\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tSubjects: []string{\"events.>\"},\n\t\tName: \"name\",\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"events.created\", fmt.Sprintf(\"%v\", i))\n\t}\n\n\t// if they're not the same, expect error\n\t_, err = js.AddConsumer(\"name\", &nats.ConsumerConfig{\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubject: \"\",\n\t\tDurable: durable,\n\t\tAckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"\", nats.Bind(\"name\", durable))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 9; i++ {\n\t\tmsg, err := sub.NextMsg(time.Millisecond * 500)\n\t\trequire_NoError(t, err)\n\t\tj, err := strconv.Atoi(string(msg.Data))\n\t\trequire_NoError(t, err)\n\t\tif j != i {\n\t\t\tt.Fatalf(\"wrong sequence, expected %v got %v\", i, j)\n\t\t}\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\tinfo, err := js.ConsumerInfo(\"name\", durable)\n\trequire_NoError(t, err)\n\trequire_True(t, info.Delivered.Stream == 10)\n\trequire_True(t, info.Delivered.Consumer == 10)\n\trequire_True(t, info.AckFloor.Stream == 9)\n\trequire_True(t, info.AckFloor.Consumer == 9)\n\trequire_True(t, info.NumAckPending == 1)\n\n\tresp := createConsumer(t, nc, \"name\", ConsumerConfig{\n\t\tFilterSubjects: []string{\"\"},\n\t\tDeliverSubject: \"multiple\",\n\t\tDurable: \"multiple\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_True(t, resp.Error.ErrCode == 10139)\n}\n\nfunc TestJetStreamConsumerOverlappingSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t_, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events.>\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\tresp := createConsumer(t, nc, \"deliver\", ConsumerConfig{\n\t\tFilterSubjects: []string{\"events.one\", \"events.*\"},\n\t\tDurable: \"name\",\n\t})\n\n\tif resp.Error.ErrCode != 10138 {\n\t\tt.Fatalf(\"this should error as we have overlapping subjects, got %+v\", resp.Error)\n\t}\n}\n\nfunc TestJetStreamConsumerMultipleSubjectsAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tDurable: \"name\",\n\t\tAckPolicy: AckExplicit,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"events\", \"1\")\n\tsendStreamMsg(t, nc, \"data\", \"2\")\n\tsendStreamMsg(t, nc, \"data\", \"3\")\n\tsendStreamMsg(t, nc, \"data\", \"4\")\n\tsendStreamMsg(t, nc, \"events\", \"5\")\n\tsendStreamMsg(t, nc, \"data\", \"6\")\n\tsendStreamMsg(t, nc, \"data\", \"7\")\n\n\tconsumer, err := js.PullSubscribe(\"\", \"name\", nats.Bind(\"deliver\", \"name\"))\n\trequire_NoError(t, err)\n\n\tmsg, err := consumer.Fetch(3)\n\trequire_NoError(t, err)\n\n\trequire_Len(t, len(msg), 3)\n\n\trequire_NoError(t, msg[0].AckSync())\n\trequire_NoError(t, msg[1].AckSync())\n\n\t// Due to consumer signaling it might not immediately be reflected.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tinfo, err := js.ConsumerInfo(\"deliver\", \"name\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif info.AckFloor.Consumer != 2 {\n\t\t\treturn fmt.Errorf(\"bad consumer sequence. expected %v, got %v\", 2, info.AckFloor.Consumer)\n\t\t}\n\t\tif info.AckFloor.Stream != 2 {\n\t\t\treturn fmt.Errorf(\"bad stream sequence. expected %v, got %v\", 2, info.AckFloor.Stream)\n\t\t}\n\t\tif info.NumPending != 4 {\n\t\t\treturn fmt.Errorf(\"bad num pending. Expected %v, got %v\", 4, info.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamConsumerMultipleSubjectAndNewAPI(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t_, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"data\", \"events\"},\n\t\tName: \"deliver\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"error while creating stream\")\n\t}\n\n\treq, err := json.Marshal(&CreateConsumerRequest{Stream: \"deliver\", Config: ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tName: \"name\",\n\t\tDurable: \"name\",\n\t}})\n\trequire_NoError(t, err)\n\n\tresp, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.CREATE.%s.%s.%s\", \"deliver\", \"name\", \"data.>\"), req, time.Second*10)\n\n\tvar apiResp JSApiConsumerCreateResponse\n\tjson.Unmarshal(resp.Data, &apiResp)\n\trequire_NoError(t, err)\n\n\tif apiResp.Error.ErrCode != 10137 {\n\t\tt.Fatal(\"this should error as multiple subject filters is incompatible with new API and didn't\")\n\t}\n\n}\n\nfunc TestJetStreamConsumerMultipleSubjectsWithAddedMessages(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tdurable := \"durable\"\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events.>\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\t// if they're not the same, expect error\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubjects: []string{\"events.created\", \"events.processed\"},\n\t\tDurable: durable,\n\t\tAckPolicy: AckExplicit,\n\t})\n\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"events.created\", \"0\")\n\tsendStreamMsg(t, nc, \"events.created\", \"1\")\n\tsendStreamMsg(t, nc, \"events.created\", \"2\")\n\tsendStreamMsg(t, nc, \"events.created\", \"3\")\n\tsendStreamMsg(t, nc, \"events.other\", \"BAD\")\n\tsendStreamMsg(t, nc, \"events.processed\", \"4\")\n\tsendStreamMsg(t, nc, \"events.processed\", \"5\")\n\tsendStreamMsg(t, nc, \"events.processed\", \"6\")\n\tsendStreamMsg(t, nc, \"events.other\", \"BAD\")\n\tsendStreamMsg(t, nc, \"events.processed\", \"7\")\n\tsendStreamMsg(t, nc, \"events.processed\", \"8\")\n\n\tsub, err := js.SubscribeSync(\"\", nats.Bind(\"deliver\", durable))\n\tif err != nil {\n\t\tt.Fatalf(\"error while subscribing to Consumer: %v\", err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tif i == 5 {\n\t\t\tsendStreamMsg(t, nc, \"events.created\", \"9\")\n\t\t}\n\t\tif i == 9 {\n\t\t\tsendStreamMsg(t, nc, \"events.other\", \"BAD\")\n\t\t\tsendStreamMsg(t, nc, \"events.created\", \"11\")\n\t\t}\n\t\tif i == 7 {\n\t\t\tsendStreamMsg(t, nc, \"events.processed\", \"10\")\n\t\t}\n\n\t\tmsg, err := sub.NextMsg(time.Second * 1)\n\t\trequire_NoError(t, err)\n\t\tj, err := strconv.Atoi(string(msg.Data))\n\t\trequire_NoError(t, err)\n\t\tif j != i {\n\t\t\tt.Fatalf(\"wrong sequence, expected %v got %v\", i, j)\n\t\t}\n\t\tif err := msg.AckSync(); err != nil {\n\t\t\tt.Fatalf(\"error while acking the message :%v\", err)\n\t\t}\n\n\t}\n\n\tinfo, err := js.ConsumerInfo(\"deliver\", durable)\n\trequire_NoError(t, err)\n\n\trequire_True(t, info.Delivered.Consumer == 12)\n\trequire_True(t, info.Delivered.Stream == 15)\n\trequire_True(t, info.AckFloor.Stream == 12)\n\trequire_True(t, info.AckFloor.Consumer == 10)\n}\n\nfunc TestJetStreamConsumerThreeFilters(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events\", \"data\", \"other\", \"ignored\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"ignored\", \"100\")\n\tsendStreamMsg(t, nc, \"events\", \"0\")\n\tsendStreamMsg(t, nc, \"events\", \"1\")\n\n\tsendStreamMsg(t, nc, \"data\", \"2\")\n\tsendStreamMsg(t, nc, \"ignored\", \"100\")\n\tsendStreamMsg(t, nc, \"data\", \"3\")\n\n\tsendStreamMsg(t, nc, \"other\", \"4\")\n\tsendStreamMsg(t, nc, \"data\", \"5\")\n\tsendStreamMsg(t, nc, \"other\", \"6\")\n\tsendStreamMsg(t, nc, \"data\", \"7\")\n\tsendStreamMsg(t, nc, \"ignored\", \"100\")\n\n\tmset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tDurable: \"multi\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\n\tconsumer, err := js.PullSubscribe(\"\", \"multi\", nats.Bind(\"TEST\", \"multi\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err := consumer.Fetch(6)\n\trequire_NoError(t, err)\n\tfor i, msg := range msgs {\n\t\trequire_Equal(t, string(msg.Data), fmt.Sprintf(\"%d\", i))\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\tinfo, err := js.ConsumerInfo(\"TEST\", \"multi\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.Delivered.Stream == 8)\n\trequire_True(t, info.Delivered.Consumer == 6)\n\trequire_True(t, info.NumPending == 2)\n\trequire_True(t, info.NumAckPending == 0)\n\trequire_True(t, info.AckFloor.Consumer == 6)\n\trequire_True(t, info.AckFloor.Stream == 8)\n}\n\nfunc TestJetStreamConsumerUpdateFilterSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events\", \"data\", \"other\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"other\", \"100\")\n\tsendStreamMsg(t, nc, \"events\", \"0\")\n\tsendStreamMsg(t, nc, \"events\", \"1\")\n\tsendStreamMsg(t, nc, \"data\", \"2\")\n\tsendStreamMsg(t, nc, \"data\", \"3\")\n\tsendStreamMsg(t, nc, \"other\", \"4\")\n\tsendStreamMsg(t, nc, \"data\", \"5\")\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tDurable: \"multi\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_NoError(t, err)\n\n\tconsumer, err := js.PullSubscribe(\"\", \"multi\", nats.Bind(\"TEST\", \"multi\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err := consumer.Fetch(3)\n\trequire_NoError(t, err)\n\tfor i, msg := range msgs {\n\t\trequire_Equal(t, string(msg.Data), fmt.Sprintf(\"%d\", i))\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tDurable: \"multi\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_NoError(t, err)\n\n\tupdatedConsumer, err := js.PullSubscribe(\"\", \"multi\", nats.Bind(\"TEST\", \"multi\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err = updatedConsumer.Fetch(3)\n\trequire_NoError(t, err)\n\tfor i, msg := range msgs {\n\t\trequire_Equal(t, string(msg.Data), fmt.Sprintf(\"%d\", i+3))\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n}\n\nfunc TestJetStreamConsumerAndStreamMetadata(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmetadata := map[string]string{\"key\": \"value\", \"_nats_created_version\": \"2.9.11\"}\n\tacc := s.GlobalAccount()\n\n\t// Check stream's first.\n\tmset, err := acc.addStream(&StreamConfig{Name: \"foo\", Metadata: metadata})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tif cfg := mset.config(); !reflect.DeepEqual(metadata, cfg.Metadata) {\n\t\tt.Fatalf(\"Expected a metadata of %q, got %q\", metadata, cfg.Metadata)\n\t}\n\n\t// Now consumer\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tMetadata: metadata,\n\t\tDeliverSubject: \"to\",\n\t\tAckPolicy: AckNone})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\tif cfg := o.config(); !reflect.DeepEqual(metadata, cfg.Metadata) {\n\t\tt.Fatalf(\"Expected a metadata of %q, got %q\", metadata, cfg.Metadata)\n\t}\n\n\t// Test max.\n\tdata := make([]byte, JSMaxMetadataLen/100)\n\tcrand.Read(data)\n\tbigValue := base64.StdEncoding.EncodeToString(data)\n\n\tbigMetadata := make(map[string]string, 101)\n\tfor i := 0; i < 101; i++ {\n\t\tbigMetadata[fmt.Sprintf(\"key%d\", i)] = bigValue\n\t}\n\n\t_, err = acc.addStream(&StreamConfig{Name: \"bar\", Metadata: bigMetadata})\n\tif err == nil || !strings.Contains(err.Error(), \"stream metadata exceeds\") {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tMetadata: bigMetadata,\n\t\tDeliverSubject: \"to\",\n\t\tAckPolicy: AckNone})\n\tif err == nil || !strings.Contains(err.Error(), \"consumer metadata exceeds\") {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n}\n\nfunc TestJetStreamConsumerPurge(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"test.1\", \"hello\")\n\tsendStreamMsg(t, nc, \"test.2\", \"hello\")\n\n\tsub, err := js.PullSubscribe(\"test.>\", \"consumer\")\n\trequire_NoError(t, err)\n\n\t// Purge one of the subjects.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"test.2\"})\n\trequire_NoError(t, err)\n\n\tinfo, err := js.ConsumerInfo(\"TEST\", \"consumer\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.NumPending == 1)\n\n\t// Expect to get message from not purged subject.\n\t_, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"OTHER\",\n\t\tSubjects: []string{\"other.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish two items to two subjects.\n\tsendStreamMsg(t, nc, \"other.1\", \"hello\")\n\tsendStreamMsg(t, nc, \"other.2\", \"hello\")\n\n\tsub, err = js.PullSubscribe(\"other.>\", \"other_consumer\")\n\trequire_NoError(t, err)\n\n\t// Purge whole stream.\n\terr = js.PurgeStream(\"OTHER\", &nats.StreamPurgeRequest{})\n\trequire_NoError(t, err)\n\n\tinfo, err = js.ConsumerInfo(\"OTHER\", \"other_consumer\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.NumPending == 0)\n\n\t// This time expect error, as we purged whole stream,\n\t_, err = sub.Fetch(1)\n\trequire_Error(t, err)\n\n}\n\nfunc TestJetStreamConsumerFilterUpdate(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\", \"bar.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\tsendStreamMsg(t, nc, \"foo.data\", \"OK\")\n\t}\n\n\tsub, err := nc.SubscribeSync(\"deliver\")\n\trequire_NoError(t, err)\n\n\tjs.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubject: \"foo.data\",\n\t})\n\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubject: \"foo.>\",\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo.other\", \"data\")\n\n\t// This will timeout if filters were not properly updated.\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckNumFilter := func(expected int) {\n\t\tt.Helper()\n\t\tmset.mu.RLock()\n\t\tnf := mset.numFilter\n\t\tmset.mu.RUnlock()\n\t\tif nf != expected {\n\t\t\tt.Fatalf(\"Expected stream's numFilter to be %d, got %d\", expected, nf)\n\t\t}\n\t}\n\n\tcheckNumFilter(1)\n\n\t// Update consumer once again, now not having any filters\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"consumer\",\n\t\tDeliverSubject: \"deliver\",\n\t\tFilterSubject: _EMPTY_,\n\t})\n\trequire_NoError(t, err)\n\n\t// and expect that numFilter reports correctly.\n\tcheckNumFilter(0)\n}\n\nfunc TestJetStreamConsumerAckFloorWithExpired(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxAge: 2 * time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tnmsgs := 100\n\tfor i := 0; i < nmsgs; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.AckWait(time.Second))\n\trequire_NoError(t, err)\n\n\t// Queue up all for ack pending.\n\tmsgs, err := sub.Fetch(nmsgs)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == nmsgs)\n\n\t// Let all messages expire.\n\ttime.Sleep(3 * time.Second)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 0)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\trequire_True(t, ci.Delivered.Consumer == uint64(nmsgs))\n\trequire_True(t, ci.Delivered.Stream == uint64(nmsgs))\n\trequire_True(t, ci.AckFloor.Consumer == uint64(nmsgs))\n\trequire_True(t, ci.AckFloor.Stream == uint64(nmsgs))\n\trequire_True(t, ci.NumAckPending == 0)\n\trequire_True(t, ci.NumPending == 0)\n\trequire_True(t, ci.NumRedelivered == 0)\n}\n\nfunc TestJetStreamConsumerIsFiltered(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tacc := s.GlobalAccount()\n\n\ttests := []struct {\n\t\tname string\n\t\tstreamSubjects []string\n\t\tfilters []string\n\t\tisFiltered bool\n\t}{\n\t\t{\n\t\t\tname: \"single_subject\",\n\t\t\tstreamSubjects: []string{\"one\"},\n\t\t\tfilters: []string{\"one\"},\n\t\t\tisFiltered: false,\n\t\t},\n\t\t{\n\t\t\tname: \"single_subject_filtered\",\n\t\t\tstreamSubjects: []string{\"one.>\"},\n\t\t\tfilters: []string{\"one.filter\"},\n\t\t\tisFiltered: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multi_subject_non_filtered\",\n\t\t\tstreamSubjects: []string{\"multi\", \"foo\", \"bar.>\"},\n\t\t\tfilters: []string{\"multi\", \"bar.>\", \"foo\"},\n\t\t\tisFiltered: false,\n\t\t},\n\t\t{\n\t\t\tname: \"multi_subject_filtered_wc\",\n\t\t\tstreamSubjects: []string{\"events\", \"data\"},\n\t\t\tfilters: []string{\"data\"},\n\t\t\tisFiltered: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multi_subject_filtered\",\n\t\t\tstreamSubjects: []string{\"machines\", \"floors\"},\n\t\t\tfilters: []string{\"machines\"},\n\t\t\tisFiltered: true,\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmset, err := acc.addStream(&StreamConfig{\n\t\t\t\tName: test.name,\n\t\t\t\tSubjects: test.streamSubjects,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tFilterSubjects: test.filters,\n\t\t\t\tDurable: test.name,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\trequire_True(t, o.isFiltered() == test.isFiltered)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerWithFormattingSymbol(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"Test%123\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\n\t_, err = js.AddConsumer(\"Test%123\", &nats.ConsumerConfig{\n\t\tDurable: \"Test%123\",\n\t\tFilterSubject: \"foo\",\n\t\tDeliverSubject: \"bar\",\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo\", nats.Bind(\"Test%123\", \"Test%123\"))\n\trequire_NoError(t, err)\n\n\t_, err = sub.NextMsg(time.Second * 5)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamConsumerDefaultsFromStream(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\tif _, err := acc.addStream(&StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"test.*\"},\n\t\tConsumerLimits: StreamConsumerLimits{\n\t\t\tMaxAckPending: 15,\n\t\t\tInactiveThreshold: time.Second,\n\t\t},\n\t}); err != nil {\n\t\tt.Fatalf(\"Failed to add stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to connect to JetStream: %v\", err)\n\t}\n\n\tt.Run(\"InheritDefaultsFromStream\", func(t *testing.T) {\n\t\tci, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"InheritDefaultsFromStream\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tswitch {\n\t\tcase ci.Config.InactiveThreshold != time.Second:\n\t\t\tt.Fatalf(\"InactiveThreshold should be 1s, got %s\", ci.Config.InactiveThreshold)\n\t\tcase ci.Config.MaxAckPending != 15:\n\t\t\tt.Fatalf(\"MaxAckPending should be 15, got %d\", ci.Config.MaxAckPending)\n\t\t}\n\t})\n\n\tt.Run(\"CreateConsumerErrorOnExceedMaxAckPending\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"CreateConsumerErrorOnExceedMaxAckPending\",\n\t\t\tMaxAckPending: 30,\n\t\t})\n\t\tswitch e := err.(type) {\n\t\tcase *nats.APIError:\n\t\t\tif ErrorIdentifier(e.ErrorCode) != JSConsumerMaxPendingAckExcessErrF {\n\t\t\t\tt.Fatalf(\"invalid error code, got %d, wanted %d\", e.ErrorCode, JSConsumerMaxPendingAckExcessErrF)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"should have returned API error, got %T\", e)\n\t\t}\n\t})\n\n\tt.Run(\"CreateConsumerErrorOnExceedInactiveThreshold\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"CreateConsumerErrorOnExceedInactiveThreshold\",\n\t\t\tInactiveThreshold: time.Second * 2,\n\t\t})\n\t\tswitch e := err.(type) {\n\t\tcase *nats.APIError:\n\t\t\tif ErrorIdentifier(e.ErrorCode) != JSConsumerInactiveThresholdExcess {\n\t\t\t\tt.Fatalf(\"invalid error code, got %d, wanted %d\", e.ErrorCode, JSConsumerInactiveThresholdExcess)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"should have returned API error, got %T\", e)\n\t\t}\n\t})\n\n\tt.Run(\"UpdateStreamErrorOnViolateConsumerMaxAckPending\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"UpdateStreamErrorOnViolateConsumerMaxAckPending\",\n\t\t\tMaxAckPending: 15,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tstream, err := acc.lookupStream(\"test\")\n\t\trequire_NoError(t, err)\n\n\t\terr = stream.update(&StreamConfig{\n\t\t\tName: \"test\",\n\t\t\tSubjects: []string{\"test.*\"},\n\t\t\tConsumerLimits: StreamConsumerLimits{\n\t\t\t\tMaxAckPending: 10,\n\t\t\t},\n\t\t})\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"stream update should have errored but didn't\")\n\t\t}\n\t})\n\n\tt.Run(\"UpdateStreamErrorOnViolateConsumerInactiveThreshold\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\t\tName: \"UpdateStreamErrorOnViolateConsumerInactiveThreshold\",\n\t\t\tInactiveThreshold: time.Second,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tstream, err := acc.lookupStream(\"test\")\n\t\trequire_NoError(t, err)\n\n\t\terr = stream.update(&StreamConfig{\n\t\t\tName: \"test\",\n\t\t\tSubjects: []string{\"test.*\"},\n\t\t\tConsumerLimits: StreamConsumerLimits{\n\t\t\t\tInactiveThreshold: time.Second / 2,\n\t\t\t},\n\t\t})\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"stream update should have errored but didn't\")\n\t\t}\n\t})\n}\n\n// Server issue 4685\nfunc TestJetStreamConsumerPendingForKV(t *testing.T) {\n\tfor _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"test.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tMaxMsgsPerSubject: 10,\n\t\t\t\tDiscard: nats.DiscardNew,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"test.1\", []byte(\"x\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar msg *nats.Msg\n\n\t\t\t// this is the detail that triggers the off by one, remove this code and all tests pass\n\t\t\tmsg = nats.NewMsg(\"test.1\")\n\t\t\tmsg.Data = []byte(\"y\")\n\t\t\tmsg.Header.Add(nats.ExpectedLastSeqHdr, \"1\")\n\t\t\t_, err = js.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"test.2\", []byte(\"x\"))\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = js.Publish(\"test.3\", []byte(\"x\"))\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = js.Publish(\"test.4\", []byte(\"x\"))\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = js.Publish(\"test.5\", []byte(\"x\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tnfo, err := js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{SubjectsFilter: \">\"})\n\t\t\trequire_NoError(t, err)\n\n\t\t\trequire_Equal(t, len(nfo.State.Subjects), 5)\n\n\t\t\tsub, err := js.SubscribeSync(\"test.>\", nats.DeliverLastPerSubject())\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmsg, err = sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tmeta, err := msg.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, meta.NumPending, 4)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerNakThenAckFloorMove(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 11; i++ {\n\t\tjs.Publish(\"foo\", []byte(\"OK\"))\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.AckWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(11)\n\trequire_NoError(t, err)\n\n\t// Nak first one\n\tmsgs[0].Nak()\n\n\t// Ack 2-10\n\tfor i := 1; i < 10; i++ {\n\t\tmsgs[i].AckSync()\n\t}\n\t// Hold onto last.\n\tlastMsg := msgs[10]\n\n\tci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, ci.AckFloor.Consumer, 0)\n\trequire_Equal(t, ci.AckFloor.Stream, 0)\n\trequire_Equal(t, ci.NumAckPending, 2)\n\n\t// Grab first messsage again and ack this time.\n\tmsgs, err = sub.Fetch(1)\n\trequire_NoError(t, err)\n\tmsgs[0].AckSync()\n\n\tci, err = sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, ci.Delivered.Consumer, 12)\n\trequire_Equal(t, ci.Delivered.Stream, 11)\n\trequire_Equal(t, ci.AckFloor.Consumer, 10)\n\trequire_Equal(t, ci.AckFloor.Stream, 10)\n\trequire_Equal(t, ci.NumAckPending, 1)\n\n\t// Make sure when we ack last one we collapse the AckFloor.Consumer\n\t// with the higher delivered due to re-deliveries.\n\tlastMsg.AckSync()\n\tci, err = sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, ci.Delivered.Consumer, 12)\n\trequire_Equal(t, ci.Delivered.Stream, 11)\n\trequire_Equal(t, ci.AckFloor.Consumer, 12)\n\trequire_Equal(t, ci.AckFloor.Stream, 11)\n\trequire_Equal(t, ci.NumAckPending, 0)\n}\n\nfunc TestJetStreamConsumerPauseViaConfig(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tt.Run(\"CreateShouldSucceed\", func(t *testing.T) {\n\t\tdeadline := time.Now().Add(time.Hour)\n\t\tci := jsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\t\tName: \"my_consumer_1\",\n\t\t\tPauseUntil: &deadline,\n\t\t})\n\t\trequire_True(t, ci != nil)\n\t\trequire_True(t, ci.Config != nil)\n\t\trequire_True(t, ci.Config.PauseUntil != nil)\n\t\trequire_True(t, ci.Config.PauseUntil.Equal(deadline))\n\t})\n\n\tt.Run(\"UpdateShouldFail\", func(t *testing.T) {\n\t\tdeadline := time.Now().Add(time.Hour)\n\t\tci := jsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\t\tName: \"my_consumer_2\",\n\t\t})\n\t\trequire_True(t, ci != nil)\n\t\trequire_True(t, ci.Config != nil)\n\t\trequire_True(t, ci.Config.PauseUntil == nil || ci.Config.PauseUntil.IsZero())\n\n\t\tvar cc ConsumerConfig\n\t\tj, err := json.Marshal(ci.Config)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(j, &cc))\n\n\t\tpauseUntil := time.Now().Add(time.Hour)\n\t\tcc.PauseUntil = &pauseUntil\n\t\tci2 := jsTestPause_CreateOrUpdateConsumer(t, nc, ActionUpdate, \"TEST\", cc)\n\t\trequire_False(t, ci2.Config.PauseUntil != nil && ci2.Config.PauseUntil.Equal(deadline))\n\t\trequire_True(t, ci2.Config.PauseUntil == nil || ci2.Config.PauseUntil.Equal(time.Time{}))\n\t})\n}\n\nfunc TestJetStreamConsumerPauseViaEndpoint(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"push\", \"pull\"},\n\t})\n\trequire_NoError(t, err)\n\n\tt.Run(\"PullConsumer\", func(t *testing.T) {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tName: \"pull_consumer\",\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(\"pull\", \"\", nats.Bind(\"TEST\", \"pull_consumer\"))\n\t\trequire_NoError(t, err)\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err := sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\n\t\t// Now we'll pause the consumer for 3 seconds.\n\t\tdeadline := time.Now().Add(time.Second * 3)\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"pull_consumer\", deadline).Equal(deadline))\n\n\t\t// This should fail as we'll wait for only half of the deadline.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, err = sub.Fetch(10, nats.MaxWait(time.Until(deadline)/2))\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t// This should succeed after a short wait, and when we're done,\n\t\t// we should be after the deadline.\n\t\tmsgs, err = sub.Fetch(10)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\t\trequire_True(t, time.Now().After(deadline))\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err = sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"pull_consumer\", time.Time{}).Equal(time.Time{}))\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"pull\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tmsgs, err = sub.Fetch(10, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), 10)\n\t})\n\n\tt.Run(\"PushConsumer\", func(t *testing.T) {\n\t\tch := make(chan *nats.Msg, 100)\n\t\t_, err = js.ChanSubscribe(\"push\", ch, nats.BindStream(\"TEST\"), nats.ConsumerName(\"push_consumer\"))\n\t\trequire_NoError(t, err)\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\n\t\t// Now we'll pause the consumer for 3 seconds.\n\t\tdeadline := time.Now().Add(time.Second * 3)\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"push_consumer\", deadline).Equal(deadline))\n\n\t\t// This should succeed after a short wait, and when we're done,\n\t\t// we should be after the deadline.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second*5)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t\trequire_True(t, time.Now().After(deadline))\n\t\t}\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\n\t\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"push_consumer\", time.Time{}).Equal(time.Time{}))\n\n\t\t// This should succeed as there's no pause, so it definitely\n\t\t// shouldn't take more than a second.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"push\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tmsg := require_ChanRead(t, ch, time.Second)\n\t\t\trequire_NotEqual(t, msg, nil)\n\t\t}\n\t})\n}\n\nfunc TestJetStreamConsumerPauseResumeViaEndpoint(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"CONSUMER\",\n\t})\n\trequire_NoError(t, err)\n\n\tgetConsumerInfo := func() ConsumerInfo {\n\t\tvar ci ConsumerInfo\n\t\tinfoResp, err := nc.Request(\"$JS.API.CONSUMER.INFO.TEST.CONSUMER\", nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(infoResp.Data, &ci)\n\t\trequire_NoError(t, err)\n\t\treturn ci\n\t}\n\n\t// Ensure we are not paused\n\trequire_False(t, getConsumerInfo().Paused)\n\n\t// Now we'll pause the consumer for 30 seconds.\n\tdeadline := time.Now().Add(time.Second * 30)\n\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"CONSUMER\", deadline).Equal(deadline))\n\n\t// Ensure the consumer reflects being paused\n\trequire_True(t, getConsumerInfo().Paused)\n\n\tsubj := fmt.Sprintf(\"$JS.API.CONSUMER.PAUSE.%s.%s\", \"TEST\", \"CONSUMER\")\n\t_, err = nc.Request(subj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Ensure the consumer reflects being resumed\n\trequire_False(t, getConsumerInfo().Paused)\n}\n\nfunc TestJetStreamConsumerPauseHeartbeats(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Hour)\n\tdsubj := \"deliver_subj\"\n\n\tci := jsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tHeartbeat: time.Millisecond * 100,\n\t\tDeliverSubject: dsubj,\n\t})\n\trequire_True(t, ci.Config.PauseUntil.Equal(deadline))\n\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(dsubj, ch)\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\tmsg := require_ChanRead(t, ch, time.Millisecond*200)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"100\")\n\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Idle Heartbeat\")\n\t}\n}\n\nfunc TestJetStreamConsumerPauseAdvisories(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcheckAdvisory := func(msg *nats.Msg, shouldBePaused bool, deadline time.Time) {\n\t\tt.Helper()\n\t\tvar advisory JSConsumerPauseAdvisory\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.Stream, \"TEST\")\n\t\trequire_Equal(t, advisory.Consumer, \"my_consumer\")\n\t\trequire_Equal(t, advisory.Paused, shouldBePaused)\n\t\trequire_True(t, advisory.PauseUntil.Equal(deadline))\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(JSAdvisoryConsumerPausePre+\".TEST.my_consumer\", ch)\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Second)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t})\n\n\t// First advisory should tell us that the consumer was paused\n\t// on creation.\n\tmsg := require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\n\t// The second one for the unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n\n\t// Now we'll pause the consumer using the API.\n\tdeadline = time.Now().Add(time.Second)\n\trequire_True(t, jsTestPause_PauseConsumer(t, nc, \"TEST\", \"my_consumer\", deadline).Equal(deadline))\n\n\t// Third advisory should tell us about the pause via the API.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\n\t// Finally that should unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n}\n\nfunc TestJetStreamConsumerSurvivesRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Hour)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t})\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tstream, err := s.gacc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tconsumer := stream.lookupConsumer(\"my_consumer\")\n\trequire_NotEqual(t, consumer, nil)\n\n\tconsumer.mu.RLock()\n\ttimer := consumer.uptmr\n\tconsumer.mu.RUnlock()\n\trequire_True(t, timer != nil)\n}\n\nfunc TestJetStreamConsumerInfoNumPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"LIMITS\",\n\t\tSubjects: []string{\"js.in.limits\"},\n\t\tMaxMsgs: 100,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"LIMITS\", &nats.ConsumerConfig{\n\t\tName: \"PULL\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tjs.Publish(\"js.in.limits\", []byte(\"x\"))\n\t}\n\n\tci, err := js.ConsumerInfo(\"LIMITS\", \"PULL\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 100)\n\n\t// Now restart the server.\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tci, err = js.ConsumerInfo(\"LIMITS\", \"PULL\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 100)\n}\n\nfunc TestJetStreamConsumerDontDecrementPendingCountOnSkippedMsg(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\n\trequireExpected := func(expected int64) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\to.mu.RLock()\n\t\t\tnpc := o.npc\n\t\t\to.mu.RUnlock()\n\t\t\tif npc != expected {\n\t\t\t\treturn fmt.Errorf(\"expected npc=%d, got %d\", expected, npc)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Should initially report no messages available.\n\trequireExpected(0)\n\n\t// A new message is available, should report in pending.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequireExpected(1)\n\n\t// Pending count should decrease when the message is deleted.\n\terr = js.DeleteMsg(\"TEST\", 1)\n\trequire_NoError(t, err)\n\trequireExpected(0)\n\n\t// Make more messages available, should report in pending.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequireExpected(2)\n\n\t// Simulate getNextMsg being called and the starting sequence to skip over a deleted message.\n\t// Also simulate one pending message.\n\to.mu.Lock()\n\to.sseq = 100\n\to.npc--\n\to.pending = make(map[uint64]*Pending)\n\to.pending[2] = &Pending{}\n\to.mu.Unlock()\n\n\t// Since this message is pending we should not decrement pending count as we've done so already.\n\to.decStreamPending(2, \"foo\")\n\trequireExpected(1)\n\n\t// This is the deleted message that was skipped, we've hit the race condition and are not able to\n\t// fix it at this point. If we decrement then we could have decremented it twice if the message\n\t// was removed as a result of an Ack with Interest or WorkQueue retention, instead of due to contention.\n\to.decStreamPending(3, \"foo\")\n\trequireExpected(1)\n}\n\nfunc TestJetStreamConsumerPendingCountAfterMsgAckAboveFloor(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 2; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\n\trequireExpected := func(expected int64) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\to.mu.RLock()\n\t\t\tnpc := o.npc\n\t\t\to.mu.RUnlock()\n\t\t\tif npc != expected {\n\t\t\t\treturn fmt.Errorf(\"expected npc=%d, got %d\", expected, npc)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Expect 2 messages pending.\n\trequireExpected(2)\n\n\t// Fetch 2 messages and ack the last.\n\tmsgs, err := sub.Fetch(2)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 2)\n\tmsg := msgs[1]\n\terr = msg.AckSync()\n\trequire_NoError(t, err)\n\n\t// We've fetched 2 message so should report 0 pending.\n\trequireExpected(0)\n}\n\nfunc TestJetStreamConsumerPullRemoveInterest(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MYS-PULL\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\twcfg := &ConsumerConfig{Durable: \"worker\", AckPolicy: AckExplicit}\n\to, err := mset.addConsumer(wcfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\trqn := o.requestNextMsgSubject()\n\tdefer o.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Ask for a message even though one is not there. This will queue us up for waiting.\n\tif _, err := nc.Request(rqn, nil, 10*time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\n\t// This is using new style request mechanism. so drop the connection itself to get rid of interest.\n\tnc.Close()\n\n\t// Wait for client cleanup\n\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif n := s.NumClients(); err != nil || n != 0 {\n\t\t\treturn fmt.Errorf(\"Still have %d clients\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\tnc = clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Send a message\n\tsendStreamMsg(t, nc, mname, \"Hello World!\")\n\n\tmsg, err := nc.Request(rqn, nil, time.Second)\n\trequire_NoError(t, err)\n\t_, dseq, dc, _, _ := replyInfo(msg.Reply)\n\tif dseq != 1 {\n\t\tt.Fatalf(\"Expected consumer sequence of 1, got %d\", dseq)\n\t}\n\tif dc != 1 {\n\t\tt.Fatalf(\"Expected delivery count of 1, got %d\", dc)\n\t}\n\n\t// Now do old school request style and more than one waiting.\n\tnc = clientConnectWithOldRequest(t, s)\n\tdefer nc.Close()\n\n\t// Now queue up 10 waiting via failed requests.\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := nc.Request(rqn, nil, 1*time.Millisecond); err == nil {\n\t\t\tt.Fatalf(\"Expected an error, got none\")\n\t\t}\n\t}\n\n\t// Send a second message\n\tsendStreamMsg(t, nc, mname, \"Hello World!\")\n\n\tmsg, err = nc.Request(rqn, nil, time.Second)\n\trequire_NoError(t, err)\n\t_, dseq, dc, _, _ = replyInfo(msg.Reply)\n\tif dseq != 2 {\n\t\tt.Fatalf(\"Expected consumer sequence of 2, got %d\", dseq)\n\t}\n\tif dc != 1 {\n\t\tt.Fatalf(\"Expected delivery count of 1, got %d\", dc)\n\t}\n}\n\nfunc TestJetStreamConsumerPullMaxWaitingOfOne(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"TEST.A\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tMaxWaiting: 1,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// First check that a request can timeout (we had an issue where this was\n\t// not the case for MaxWaiting of 1).\n\treq := JSApiConsumerGetNextRequest{Batch: 1, Expires: 250 * time.Millisecond}\n\treqb, _ := json.Marshal(req)\n\tmsg, err := nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", reqb, 13000*time.Millisecond)\n\trequire_NoError(t, err)\n\tif v := msg.Header.Get(\"Status\"); v != \"408\" {\n\t\tt.Fatalf(\"Expected 408, got: %s\", v)\n\t}\n\n\t// Now have a request waiting...\n\treq = JSApiConsumerGetNextRequest{Batch: 1}\n\treqb, _ = json.Marshal(req)\n\t// Send the request, but do not block since we want then to send an extra\n\t// request that should be rejected.\n\tsub := natsSubSync(t, nc, nats.NewInbox())\n\terr = nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", sub.Subject, reqb)\n\trequire_NoError(t, err)\n\n\t// Send a new request, this should be rejected as a 409.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 250 * time.Millisecond}\n\treqb, _ = json.Marshal(req)\n\tmsg, err = nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", reqb, 300*time.Millisecond)\n\trequire_NoError(t, err)\n\tif v := msg.Header.Get(\"Status\"); v != \"409\" {\n\t\tt.Fatalf(\"Expected 409, got: %s\", v)\n\t}\n\tif v := msg.Header.Get(\"Description\"); v != \"Exceeded MaxWaiting\" {\n\t\tt.Fatalf(\"Expected error about exceeded max waiting, got: %s\", v)\n\t}\n}\n\nfunc TestJetStreamConsumerPullMaxWaitingOfOneWithHeartbeatInterval(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"TEST.A\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tMaxWaiting: 1,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// First check that a request can timeout (we had an issue where this was\n\t// not the case for MaxWaiting of 1).\n\treq := JSApiConsumerGetNextRequest{Batch: 1, Expires: 250 * time.Millisecond}\n\treqb, _ := json.Marshal(req)\n\tmsg, err := nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", reqb, 13000*time.Millisecond)\n\trequire_NoError(t, err)\n\tif v := msg.Header.Get(\"Status\"); v != \"408\" {\n\t\tt.Fatalf(\"Expected 408, got: %s\", v)\n\t}\n\n\t// Now have a request waiting...\n\treq = JSApiConsumerGetNextRequest{Batch: 1}\n\treqb, _ = json.Marshal(req)\n\t// Send the request, but do not block since we want then to send an extra\n\t// request that should be rejected.\n\tsub := natsSubSync(t, nc, nats.NewInbox())\n\terr = nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", sub.Subject, reqb)\n\trequire_NoError(t, err)\n\n\t// Send a new request, this should not respond since we specified an idle heartbeat,\n\t// therefore the client will just expect to miss those instead.\n\treq = JSApiConsumerGetNextRequest{Batch: 1, Expires: 500 * time.Millisecond, Heartbeat: 250 * time.Millisecond}\n\treqb, _ = json.Marshal(req)\n\t_, err = nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", reqb, 300*time.Millisecond)\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamConsumerPullMaxWaiting(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"test.*\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxWaiting: 10,\n\t})\n\trequire_NoError(t, err)\n\n\t// Cannot be updated.\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxWaiting: 1,\n\t})\n\tif !strings.Contains(err.Error(), \"can not be updated\") {\n\t\tt.Fatalf(`expected \"cannot be updated\" error, got %s`, err)\n\t}\n}\n\nfunc TestJetStreamConsumerPullRequestCleanup(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"T\", Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"T\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\treq := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 100 * time.Millisecond}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\n\t// Need interest otherwise the requests will be recycled based on that.\n\t_, err = nc.SubscribeSync(\"xx\")\n\trequire_NoError(t, err)\n\n\t// Queue up 100 requests.\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"T\", \"dlc\")\n\tfor i := 0; i < 100; i++ {\n\t\terr = nc.PublishRequest(rsubj, \"xx\", jreq)\n\t\trequire_NoError(t, err)\n\t}\n\t// Wait to expire\n\ttime.Sleep(200 * time.Millisecond)\n\n\tci, err := js.ConsumerInfo(\"T\", \"dlc\")\n\trequire_NoError(t, err)\n\n\tif ci.NumWaiting != 0 {\n\t\tt.Fatalf(\"Expected to see no waiting requests, got %d\", ci.NumWaiting)\n\t}\n}\n\nfunc TestJetStreamConsumerPullRequestMaximums(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Need to do this via server for now.\n\tacc := s.GlobalAccount()\n\tmset, err := acc.addStream(&StreamConfig{Name: \"TEST\", Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tMaxRequestBatch: 10,\n\t\tMaxRequestMaxBytes: 10_000,\n\t\tMaxRequestExpires: time.Second,\n\t\tAckPolicy: AckExplicit,\n\t})\n\trequire_NoError(t, err)\n\n\tgenReq := func(b, mb int, e time.Duration) []byte {\n\t\treq := &JSApiConsumerGetNextRequest{Batch: b, Expires: e, MaxBytes: mb}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\treturn jreq\n\t}\n\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\n\t// Exceeds max batch size.\n\tresp, err := nc.Request(rsubj, genReq(11, 0, 100*time.Millisecond), time.Second)\n\trequire_NoError(t, err)\n\tif status := resp.Header.Get(\"Status\"); status != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", status)\n\t}\n\n\t// Exceeds max expires.\n\tresp, err = nc.Request(rsubj, genReq(1, 0, 10*time.Minute), time.Second)\n\trequire_NoError(t, err)\n\tif status := resp.Header.Get(\"Status\"); status != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", status)\n\t}\n\n\t// Exceeds max bytes.\n\tresp, err = nc.Request(rsubj, genReq(10, 10_000*2, 10*time.Minute), time.Second)\n\trequire_NoError(t, err)\n\tif status := resp.Header.Get(\"Status\"); status != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", status)\n\t}\n}\n\nfunc TestJetStreamConsumerPullCrossAccountExpires(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: dlc, password: foo} ]\n\t\t\t\texports [ { service: \"$JS.API.CONSUMER.MSG.NEXT.>\", response: stream } ]\n\t\t\t},\n\t\t\tIU: {\n\t\t\t\tusers: [ {user: mh, password: bar} ]\n\t\t\t\timports [ { service: { subject: \"$JS.API.CONSUMER.MSG.NEXT.*.*\", account: JS } }]\n\t\t\t\t# Re-export for dasiy chain test.\n\t\t\t\texports [ { service: \"$JS.API.CONSUMER.MSG.NEXT.>\", response: stream } ]\n\t\t\t},\n\t\t\tIU2: {\n\t\t\t\tusers: [ {user: ik, password: bar} ]\n\t\t\t\timports [ { service: { subject: \"$JS.API.CONSUMER.MSG.NEXT.*.*\", account: IU } } ]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Connect to JS account and create stream, put some messages into it.\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"dlc\", \"foo\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"PC\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\ttoSend := 50\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now create pull consumer.\n\t_, err = js.AddConsumer(\"PC\", &nats.ConsumerConfig{Durable: \"PC\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// Now access from the importing account.\n\tnc2, _ := jsClientConnect(t, s, nats.UserInfo(\"mh\", \"bar\"))\n\tdefer nc2.Close()\n\n\t// Make sure batch request works properly with stream response.\n\treq := &JSApiConsumerGetNextRequest{Batch: 10}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"PC\", \"PC\")\n\t// Make sure we can get a batch correctly etc.\n\t// This requires response stream above in the export definition.\n\tsub, err := nc2.SubscribeSync(\"xx\")\n\trequire_NoError(t, err)\n\terr = nc2.PublishRequest(rsubj, \"xx\", jreq)\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, 10)\n\n\t// Now let's queue up a bunch of requests and then delete interest to make sure the system\n\t// removes those requests.\n\n\t// Purge stream\n\terr = js.PurgeStream(\"PC\")\n\trequire_NoError(t, err)\n\n\t// Queue up 10 requests\n\tfor i := 0; i < 10; i++ {\n\t\terr = nc2.PublishRequest(rsubj, \"xx\", jreq)\n\t\trequire_NoError(t, err)\n\t}\n\t// Since using different connection, flush to make sure processed.\n\tnc2.Flush()\n\n\tci, err := js.ConsumerInfo(\"PC\", \"PC\")\n\trequire_NoError(t, err)\n\tif ci.NumWaiting != 10 {\n\t\tt.Fatalf(\"Expected to see 10 waiting requests, got %d\", ci.NumWaiting)\n\t}\n\n\t// Now remove interest and make sure requests are removed.\n\tsub.Unsubscribe()\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"PC\", \"PC\")\n\t\trequire_NoError(t, err)\n\t\tif ci.NumWaiting != 0 {\n\t\t\treturn fmt.Errorf(\"Requests still present\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now let's test that ephemerals will go away as well when interest etc is no longer around.\n\tci, err = js.AddConsumer(\"PC\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// Set the inactivity threshold by hand for now.\n\tjsacc, err := s.LookupAccount(\"JS\")\n\trequire_NoError(t, err)\n\tmset, err := jsacc.lookupStream(\"PC\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\terr = o.setInActiveDeleteThreshold(50 * time.Millisecond)\n\trequire_NoError(t, err)\n\n\trsubj = fmt.Sprintf(JSApiRequestNextT, \"PC\", ci.Name)\n\tsub, err = nc2.SubscribeSync(\"zz\")\n\trequire_NoError(t, err)\n\terr = nc2.PublishRequest(rsubj, \"zz\", jreq)\n\trequire_NoError(t, err)\n\n\t// Wait past inactive threshold.\n\ttime.Sleep(100 * time.Millisecond)\n\t// Make sure it is still there..\n\tci, err = js.ConsumerInfo(\"PC\", ci.Name)\n\trequire_NoError(t, err)\n\tif ci.NumWaiting != 1 {\n\t\tt.Fatalf(\"Expected to see 1 waiting request, got %d\", ci.NumWaiting)\n\t}\n\n\t// Now release interest.\n\tsub.Unsubscribe()\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\t_, err := js.ConsumerInfo(\"PC\", ci.Name)\n\t\tif err == nil {\n\t\t\treturn fmt.Errorf(\"Consumer still present\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now test daisy chained.\n\ttoSend = 10\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tci, err = js.AddConsumer(\"PC\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// Set the inactivity threshold by hand for now.\n\to = mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\t// Make this one longer so we test request purge and ephemerals in same test.\n\terr = o.setInActiveDeleteThreshold(500 * time.Millisecond)\n\trequire_NoError(t, err)\n\n\t// Now access from the importing account.\n\tnc3, _ := jsClientConnect(t, s, nats.UserInfo(\"ik\", \"bar\"))\n\tdefer nc3.Close()\n\n\tsub, err = nc3.SubscribeSync(\"yy\")\n\trequire_NoError(t, err)\n\n\trsubj = fmt.Sprintf(JSApiRequestNextT, \"PC\", ci.Name)\n\terr = nc3.PublishRequest(rsubj, \"yy\", jreq)\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, 10)\n\n\t// Purge stream\n\terr = js.PurgeStream(\"PC\")\n\trequire_NoError(t, err)\n\n\t// Queue up 10 requests\n\tfor i := 0; i < 10; i++ {\n\t\terr = nc3.PublishRequest(rsubj, \"yy\", jreq)\n\t\trequire_NoError(t, err)\n\t}\n\t// Since using different connection, flush to make sure processed.\n\tnc3.Flush()\n\n\tci, err = js.ConsumerInfo(\"PC\", ci.Name)\n\trequire_NoError(t, err)\n\tif ci.NumWaiting != 10 {\n\t\tt.Fatalf(\"Expected to see 10 waiting requests, got %d\", ci.NumWaiting)\n\t}\n\n\t// Now remove interest and make sure requests are removed.\n\tsub.Unsubscribe()\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"PC\", ci.Name)\n\t\trequire_NoError(t, err)\n\t\tif ci.NumWaiting != 0 {\n\t\t\treturn fmt.Errorf(\"Requests still present\")\n\t\t}\n\t\treturn nil\n\t})\n\t// Now make sure the ephemeral goes away too.\n\t// Ephemerals have jitter by default of up to 1s.\n\tcheckFor(t, 6*time.Second, 10*time.Millisecond, func() error {\n\t\t_, err := js.ConsumerInfo(\"PC\", ci.Name)\n\t\tif err == nil {\n\t\t\treturn fmt.Errorf(\"Consumer still present\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamConsumerPullCrossAccountExpiresNoDataRace(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: dlc, password: foo} ]\n\t\t\t\texports [ { service: \"$JS.API.CONSUMER.MSG.NEXT.>\", response: stream } ]\n\t\t\t},\n\t\t\tIU: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: ik, password: bar} ]\n\t\t\t\timports [ { service: { subject: \"$JS.API.CONSUMER.MSG.NEXT.*.*\", account: JS } }]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ttest := func() {\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\n\t\t// Connect to JS account and create stream, put some messages into it.\n\t\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"dlc\", \"foo\"))\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"PC\", Subjects: []string{\"foo\"}})\n\t\trequire_NoError(t, err)\n\n\t\ttoSend := 100\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\t_, err := js.Publish(\"foo\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Create pull consumer.\n\t\t_, err = js.AddConsumer(\"PC\", &nats.ConsumerConfig{Durable: \"PC\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\n\t\t// Now access from the importing account.\n\t\tnc2, _ := jsClientConnect(t, s, nats.UserInfo(\"ik\", \"bar\"))\n\t\tdefer nc2.Close()\n\n\t\treq := &JSApiConsumerGetNextRequest{Batch: 1}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\trsubj := fmt.Sprintf(JSApiRequestNextT, \"PC\", \"PC\")\n\t\tsub, err := nc2.SubscribeSync(\"xx\")\n\t\trequire_NoError(t, err)\n\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\tsub.Unsubscribe()\n\t\t\twg.Done()\n\t\t}()\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tnc2.PublishRequest(rsubj, \"xx\", jreq)\n\t\t}\n\t\twg.Wait()\n\t}\n\t// Need to rerun this test several times to get the race (which then would possible be panic\n\t// such as: \"fatal error: concurrent map read and map write\"\n\tfor iter := 0; iter < 10; iter++ {\n\t\ttest()\n\t}\n}\n\n// This tests account export/import replies across a LN connection with account import/export\n// on both sides of the LN.\nfunc TestJetStreamConsumerPullCrossAccountsAndLeafNodes(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: SJS\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 4GB, max_file_store: 1TB, domain: JSD, store_dir: %q }\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: dlc, password: foo} ]\n\t\t\t\texports [ { service: \"$JS.API.CONSUMER.MSG.NEXT.>\", response: stream } ]\n\t\t\t},\n\t\t\tIU: {\n\t\t\t\tusers: [ {user: mh, password: bar} ]\n\t\t\t\timports [ { service: { subject: \"$JS.API.CONSUMER.MSG.NEXT.*.*\", account: JS } }]\n\t\t\t},\n\t\t}\n\t\tleaf { listen: \"127.0.0.1:-1\" }\n\t`, t.TempDir())))\n\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: SLN\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tusers: [ {user: l, password: p} ]\n\t\t\t\texports [ { service: \"$JS.JSD.API.CONSUMER.MSG.NEXT.>\", response: stream } ]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tusers: [ {user: m, password: p} ]\n\t\t\t\timports [ { service: { subject: \"$JS.JSD.API.CONSUMER.MSG.NEXT.*.*\", account: A } }]\n\t\t\t},\n\t\t}\n\t\t# bind local A to IU account on other side of LN.\n\t\tleaf { remotes [ { url: nats://mh:bar@127.0.0.1:%d; account: A } ] }\n\t`, o.LeafNode.Port)))\n\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, s, 1)\n\n\t// Connect to JS account, create stream and consumer and put in some messages.\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"dlc\", \"foo\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"PC\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now create durable pull consumer.\n\t_, err = js.AddConsumer(\"PC\", &nats.ConsumerConfig{Durable: \"PC\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// Now access from the account on the leafnode, so importing on both sides and crossing a leafnode connection.\n\tnc2, _ := jsClientConnect(t, s2, nats.UserInfo(\"m\", \"p\"))\n\tdefer nc2.Close()\n\n\treq := &JSApiConsumerGetNextRequest{Batch: toSend, Expires: 500 * time.Millisecond}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\n\t// Make sure we can get a batch correctly etc.\n\t// This requires response stream above in the export definition.\n\tsub, err := nc2.SubscribeSync(\"xx\")\n\trequire_NoError(t, err)\n\n\trsubj := \"$JS.JSD.API.CONSUMER.MSG.NEXT.PC.PC\"\n\terr = nc2.PublishRequest(rsubj, \"xx\", jreq)\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, 10)\n\n\t// Queue up a bunch of requests.\n\tfor i := 0; i < 10; i++ {\n\t\terr = nc2.PublishRequest(rsubj, \"xx\", jreq)\n\t\trequire_NoError(t, err)\n\t}\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"PC\", \"PC\")\n\t\trequire_NoError(t, err)\n\t\tif ci.NumWaiting != 10 {\n\t\t\treturn fmt.Errorf(\"Expected to see 10 waiting requests, got %d\", ci.NumWaiting)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Remove interest.\n\tsub.Unsubscribe()\n\t// Make sure requests go away eventually after they expire.\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"PC\", \"PC\")\n\t\trequire_NoError(t, err)\n\t\tif ci.NumWaiting != 0 {\n\t\t\treturn fmt.Errorf(\"Expected to see no waiting requests, got %d\", ci.NumWaiting)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// This test is to explicitly test for all combinations of pull consumer behavior.\n// 1. Long poll, will be used to emulate push. A request is only invalidated when batch is filled, it expires, or we lose interest.\n// 2. Batch 1, will return no messages or a message. Works today.\n// 3. Conditional wait, or one shot. This is what the clients do when the do a fetch().\n// They expect to wait up to a given time for any messages but will return once they have any to deliver, so parital fills.\n// 4. Try, which never waits at all ever.\nfunc TestJetStreamConsumerPullOneShotBehavior(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\t// We will do low level requests by hand for this test as to not depend on any client impl.\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\n\tgetNext := func(batch int, expires time.Duration, noWait bool) (numMsgs int, elapsed time.Duration, hdr *nats.Header) {\n\t\tt.Helper()\n\t\treq := &JSApiConsumerGetNextRequest{Batch: batch, Expires: expires, NoWait: noWait}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\t// Create listener.\n\t\treply, msgs := nats.NewInbox(), make(chan *nats.Msg, batch)\n\t\tsub, err := nc.ChanSubscribe(reply, msgs)\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\n\t\t// Send request.\n\t\tstart := time.Now()\n\t\terr = nc.PublishRequest(rsubj, reply, jreq)\n\t\trequire_NoError(t, err)\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase m := <-msgs:\n\t\t\t\tif len(m.Data) == 0 && m.Header != nil {\n\t\t\t\t\treturn numMsgs, time.Since(start), &m.Header\n\t\t\t\t}\n\t\t\t\tnumMsgs++\n\t\t\t\tif numMsgs >= batch {\n\t\t\t\t\treturn numMsgs, time.Since(start), nil\n\t\t\t\t}\n\t\t\tcase <-time.After(expires + 250*time.Millisecond):\n\t\t\t\tt.Fatalf(\"Did not receive all the msgs in time\")\n\t\t\t}\n\t\t}\n\t}\n\n\texpect := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header, lt time.Duration, gt time.Duration) {\n\t\tt.Helper()\n\t\tn, e, h := getNext(batch, expires, noWait)\n\t\tif n != ne {\n\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", ne, n)\n\t\t}\n\t\tif !reflect.DeepEqual(h, he) {\n\t\t\tt.Fatalf(\"Expected %+v hdr, got %+v\", he, h)\n\t\t}\n\t\tif lt > 0 && e > lt {\n\t\t\tt.Fatalf(\"Expected elapsed of %v to be less than %v\", e, lt)\n\t\t}\n\t\tif gt > 0 && e < gt {\n\t\t\tt.Fatalf(\"Expected elapsed of %v to be greater than %v\", e, gt)\n\t\t}\n\t}\n\texpectAfter := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header, gt time.Duration) {\n\t\tt.Helper()\n\t\texpect(batch, expires, noWait, ne, he, 0, gt)\n\t}\n\texpectInstant := func(batch int, expires time.Duration, noWait bool, ne int, he *nats.Header) {\n\t\tt.Helper()\n\t\texpect(batch, expires, noWait, ne, he, 5*time.Millisecond, 0)\n\t}\n\texpectOK := func(batch int, expires time.Duration, noWait bool, ne int) {\n\t\tt.Helper()\n\t\texpectInstant(batch, expires, noWait, ne, nil)\n\t}\n\n\tnoMsgs := &nats.Header{\"Status\": []string{\"404\"}, \"Description\": []string{\"No Messages\"}}\n\treqTimeout := &nats.Header{\"Status\": []string{\"408\"}, \"Description\": []string{\"Request Timeout\"}, \"Nats-Pending-Bytes\": []string{\"0\"}, \"Nats-Pending-Messages\": []string{\"1\"}}\n\n\t// We are empty here, meaning no messages available.\n\t// Do not wait, should get noMsgs.\n\texpectInstant(1, 0, true, 0, noMsgs)\n\t// We should wait here for the full second.\n\texpectAfter(1, 250*time.Millisecond, false, 0, reqTimeout, 250*time.Millisecond)\n\t// This should also wait since no messages are available. This is the one shot scenario, or wait for at least a message if none are there.\n\texpectAfter(1, 500*time.Millisecond, true, 0, reqTimeout, 500*time.Millisecond)\n\n\t// Now let's put some messages into the system.\n\tfor i := 0; i < 20; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now run same 3 scenarios.\n\texpectOK(1, 0, true, 1)\n\texpectOK(5, 500*time.Millisecond, false, 5)\n\texpectOK(5, 500*time.Millisecond, true, 5)\n}\n\nfunc TestJetStreamConsumerPullMultipleRequestsExpireOutOfOrder(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\t// We will now queue up 4 requests. All should expire but they will do so out of order.\n\t// We want to make sure we get them in correct order.\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\tsub, err := nc.SubscribeSync(\"i.*\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tfor _, expires := range []time.Duration{200, 100, 25, 75} {\n\t\treply := fmt.Sprintf(\"i.%d\", expires)\n\t\treq := &JSApiConsumerGetNextRequest{Expires: expires * time.Millisecond}\n\t\tjreq, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(rsubj, reply, jreq)\n\t\trequire_NoError(t, err)\n\t}\n\tstart := time.Now()\n\tcheckSubsPending(t, sub, 4)\n\telapsed := time.Since(start)\n\n\tif elapsed < 200*time.Millisecond || elapsed > 500*time.Millisecond {\n\t\tt.Fatalf(\"Expected elapsed to be close to %v, but got %v\", 200*time.Millisecond, elapsed)\n\t}\n\n\tvar rs []string\n\tfor i := 0; i < 4; i++ {\n\t\tm, err := sub.NextMsg(0)\n\t\trequire_NoError(t, err)\n\t\trs = append(rs, m.Subject)\n\t}\n\tif expected := []string{\"i.25\", \"i.75\", \"i.100\", \"i.200\"}; !reflect.DeepEqual(rs, expected) {\n\t\tt.Fatalf(\"Received in wrong order, wanted %+v, got %+v\", expected, rs)\n\t}\n}\n\nfunc TestJetStreamConsumerPullNoAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ORDERS.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckNonePolicy,\n\t})\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamConsumerPullLastPerSubjectRedeliveries(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 20; i++ {\n\t\tsendStreamMsg(t, nc, fmt.Sprintf(\"foo.%v\", i), \"msg\")\n\t}\n\n\t// Create a pull sub with a maxackpending that is <= of the number of\n\t// messages in the stream and as much as we are going to Fetch() below.\n\tsub, err := js.PullSubscribe(\">\", \"dur\",\n\t\tnats.AckExplicit(),\n\t\tnats.BindStream(\"TEST\"),\n\t\tnats.DeliverLastPerSubject(),\n\t\tnats.MaxAckPending(10),\n\t\tnats.MaxRequestBatch(10),\n\t\tnats.AckWait(250*time.Millisecond))\n\trequire_NoError(t, err)\n\n\t// Fetch the max number of message we can get, and don't ack them.\n\t_, err = sub.Fetch(10, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\n\t// Wait for more than redelivery time.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Fetch again, make sure we can get those 10 messages.\n\tmsgs, err := sub.Fetch(10, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 10)\n\t// Make sure those were the first 10 messages\n\tfor i, m := range msgs {\n\t\tif m.Subject != fmt.Sprintf(\"foo.%v\", i) {\n\t\t\tt.Fatalf(\"Expected message for subject foo.%v, got %v\", i, m.Subject)\n\t\t}\n\t\tm.Ack()\n\t}\n}\n\nfunc TestJetStreamConsumerPullTimeoutHeaders(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tnc.Publish(\"foo.foo\", []byte(\"foo\"))\n\tnc.Publish(\"foo.bar\", []byte(\"bar\"))\n\tnc.Publish(\"foo.else\", []byte(\"baz\"))\n\tnc.Flush()\n\n\t// We will do low level requests by hand for this test as to not depend on any client impl.\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\n\tmaxBytes := 1024\n\tbatch := 50\n\treq := &JSApiConsumerGetNextRequest{Batch: batch, Expires: 100 * time.Millisecond, NoWait: false, MaxBytes: maxBytes}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\t// Create listener.\n\treply, msgs := nats.NewInbox(), make(chan *nats.Msg, batch)\n\tsub, err := nc.ChanSubscribe(reply, msgs)\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Send request.\n\terr = nc.PublishRequest(rsubj, reply, jreq)\n\trequire_NoError(t, err)\n\n\tbytesReceived := 0\n\tmessagesReceived := 0\n\n\tcheckHeaders := func(expectedStatus, expectedDesc string, m *nats.Msg) {\n\t\tt.Helper()\n\t\tif value := m.Header.Get(\"Status\"); value != expectedStatus {\n\t\t\tt.Fatalf(\"Expected status %q, got %q\", expectedStatus, value)\n\t\t}\n\t\tif value := m.Header.Get(\"Description\"); value != expectedDesc {\n\t\t\tt.Fatalf(\"Expected description %q, got %q\", expectedDesc, value)\n\t\t}\n\t\tif value := m.Header.Get(JSPullRequestPendingMsgs); value != fmt.Sprint(batch-messagesReceived) {\n\t\t\tt.Fatalf(\"Expected %d messages, got %s\", batch-messagesReceived, value)\n\t\t}\n\t\tif value := m.Header.Get(JSPullRequestPendingBytes); value != fmt.Sprint(maxBytes-bytesReceived) {\n\t\t\tt.Fatalf(\"Expected %d bytes, got %s\", maxBytes-bytesReceived, value)\n\t\t}\n\t}\n\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase m := <-msgs:\n\t\t\tif len(m.Data) == 0 && m.Header != nil {\n\t\t\t\tcheckHeaders(\"408\", \"Request Timeout\", m)\n\t\t\t\tdone = true\n\t\t\t} else {\n\t\t\t\tmessagesReceived += 1\n\t\t\t\tbytesReceived += (len(m.Data) + len(m.Header) + len(m.Reply) + len(m.Subject))\n\t\t\t}\n\t\tcase <-time.After(100 + 250*time.Millisecond):\n\t\t\tt.Fatalf(\"Did not receive all the msgs in time\")\n\t\t}\n\t}\n\n\t// Now resend the request but then shutdown the server and\n\t// make sure we have the same info.\n\terr = nc.PublishRequest(rsubj, reply, jreq)\n\trequire_NoError(t, err)\n\tnatsFlush(t, nc)\n\n\ts.Shutdown()\n\n\t// It is possible that the client did not receive, so let's not fail\n\t// on that. But if the 409 indicating the server is shutdown\n\t// is received, then it should have the new headers.\n\tmessagesReceived, bytesReceived = 0, 0\n\tselect {\n\tcase m := <-msgs:\n\t\tcheckHeaders(\"409\", \"Server Shutdown\", m)\n\tcase <-time.After(500 * time.Millisecond):\n\t\t// we can't fail for that.\n\t\tt.Logf(\"Subscription did not receive the pull request response on server shutdown\")\n\t}\n}\n\nfunc TestJetStreamConsumerPullBatchCompleted(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmsgSize := 128\n\tmsg := make([]byte, msgSize)\n\tcrand.Read(msg)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\treq := JSApiConsumerGetNextRequest{Batch: 0, MaxBytes: 1024, Expires: 250 * time.Millisecond}\n\n\treqb, _ := json.Marshal(req)\n\tsub := natsSubSync(t, nc, nats.NewInbox())\n\terr = nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.dur\", sub.Subject, reqb)\n\trequire_NoError(t, err)\n\n\t// Expect first message to arrive normally.\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\n\t// Second message should be info that batch is complete, but there were pending bytes.\n\tpullMsg, err := sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\n\tif v := pullMsg.Header.Get(\"Status\"); v != \"409\" {\n\t\tt.Fatalf(\"Expected 409, got: %s\", v)\n\t}\n\tif v := pullMsg.Header.Get(\"Description\"); v != \"Batch Completed\" {\n\t\tt.Fatalf(\"Expected Batch Completed, got: %s\", v)\n\t}\n}\n\nfunc TestJetStreamConsumerPullLargeBatchExpired(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"add stream failed: %s\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.PullMaxWaiting(10), nats.MaxAckPending(10*50_000_000))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating pull subscriber: %v\", err)\n\t}\n\n\t// Queue up 10 batch requests with timeout.\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\treq := &JSApiConsumerGetNextRequest{Batch: 50_000_000, Expires: 100 * time.Millisecond}\n\tjreq, _ := json.Marshal(req)\n\tfor i := 0; i < 10; i++ {\n\t\tnc.PublishRequest(rsubj, \"bar\", jreq)\n\t}\n\tnc.Flush()\n\n\t// Let them all expire.\n\ttime.Sleep(150 * time.Millisecond)\n\n\t// Now do another and measure how long to timeout and shutdown the server.\n\tstart := time.Now()\n\tsub.Fetch(1, nats.MaxWait(100*time.Millisecond))\n\ts.Shutdown()\n\n\tif delta := time.Since(start); delta > 200*time.Millisecond {\n\t\tt.Fatalf(\"Took too long to expire: %v\", delta)\n\t}\n}\n\nfunc TestJetStreamConsumerPullTimeout(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"pr\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tconst numMessages = 1000\n\t// Send messages in small intervals.\n\tgo func() {\n\t\tfor i := 0; i < numMessages; i++ {\n\t\t\ttime.Sleep(time.Millisecond * 10)\n\t\t\tsendStreamMsg(t, nc, \"TEST\", \"data\")\n\t\t}\n\t}()\n\n\t// Prepare manual Pull Request.\n\treq := &JSApiConsumerGetNextRequest{Batch: 200, NoWait: false, Expires: time.Millisecond * 100}\n\tjreq, _ := json.Marshal(req)\n\n\tsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"pr\")\n\treply := \"_pr_\"\n\tvar got atomic.Int32\n\tnc.PublishRequest(subj, reply, jreq)\n\n\t// Manually subscribe to inbox subject and send new request only if we get `408 Request Timeout`.\n\tsub, _ := nc.Subscribe(reply, func(msg *nats.Msg) {\n\t\tif msg.Header.Get(\"Status\") == \"408\" && msg.Header.Get(\"Description\") == \"Request Timeout\" {\n\t\t\tnc.PublishRequest(subj, reply, jreq)\n\t\t\tnc.Flush()\n\t\t} else {\n\t\t\tgot.Add(1)\n\t\t\tmsg.Ack()\n\t\t}\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Check if we're not stuck.\n\tcheckFor(t, time.Second*30, time.Second*1, func() error {\n\t\tif got.Load() < int32(numMessages) {\n\t\t\treturn fmt.Errorf(\"expected %d messages\", numMessages)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamConsumerPullMaxBytes(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Put in ~2MB, each ~100k\n\tmsz, dsz := 100_000, 99_950\n\ttotal, msg := 20, []byte(strings.Repeat(\"Z\", dsz))\n\n\tfor i := 0; i < total; i++ {\n\t\tif _, err := js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"pr\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\treq := &JSApiConsumerGetNextRequest{MaxBytes: 100, NoWait: true}\n\tjreq, _ := json.Marshal(req)\n\n\tsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"pr\")\n\treply := \"_pr_\"\n\tsub, _ := nc.SubscribeSync(reply)\n\tdefer sub.Unsubscribe()\n\n\tcheckHeader := func(m *nats.Msg, expected *nats.Header) {\n\t\tt.Helper()\n\t\tif len(m.Data) != 0 {\n\t\t\tt.Fatalf(\"Did not expect data, got %d bytes\", len(m.Data))\n\t\t}\n\t\texpectedStatus, givenStatus := expected.Get(\"Status\"), m.Header.Get(\"Status\")\n\t\texpectedDesc, givenDesc := expected.Get(\"Description\"), m.Header.Get(\"Description\")\n\t\tif expectedStatus != givenStatus || expectedDesc != givenDesc {\n\t\t\tt.Fatalf(\"expected %s %s, got %s %s\", expectedStatus, expectedDesc, givenStatus, givenDesc)\n\t\t}\n\t}\n\n\t// If we ask for less MaxBytes then a single message make sure we get an error.\n\tbadReq := &nats.Header{\"Status\": []string{\"409\"}, \"Description\": []string{\"Message Size Exceeds MaxBytes\"}}\n\n\tnc.PublishRequest(subj, reply, jreq)\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, 0)\n\tcheckHeader(m, badReq)\n\n\t// If we request a ton of max bytes make sure batch size overrides.\n\treq = &JSApiConsumerGetNextRequest{Batch: 1, MaxBytes: 10_000_000, NoWait: true}\n\tjreq, _ = json.Marshal(req)\n\tnc.PublishRequest(subj, reply, jreq)\n\t// we expect two messages, as the second one should be `Batch Completed` status.\n\tcheckSubsPending(t, sub, 2)\n\n\t// first one is message from the stream.\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_True(t, len(m.Data) == dsz)\n\trequire_True(t, len(m.Header) == 0)\n\t// second one is the status.\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tif v := m.Header.Get(\"Description\"); v != \"Batch Completed\" {\n\t\tt.Fatalf(\"Expected Batch Completed, got: %s\", v)\n\t}\n\tcheckSubsPending(t, sub, 0)\n\n\t// Same but with batch > 1\n\treq = &JSApiConsumerGetNextRequest{Batch: 5, MaxBytes: 10_000_000, NoWait: true}\n\tjreq, _ = json.Marshal(req)\n\tnc.PublishRequest(subj, reply, jreq)\n\t// 6, not 5, as 6th is the status.\n\tcheckSubsPending(t, sub, 6)\n\tfor i := 0; i < 5; i++ {\n\t\tm, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(m.Data) == dsz)\n\t\trequire_True(t, len(m.Header) == 0)\n\t}\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tif v := m.Header.Get(\"Description\"); v != \"Batch Completed\" {\n\t\tt.Fatalf(\"Expected Batch Completed, got: %s\", v)\n\t}\n\tcheckSubsPending(t, sub, 0)\n\n\t// Now ask for large batch but make sure we are limited by batch size.\n\treq = &JSApiConsumerGetNextRequest{Batch: 1_000, MaxBytes: msz * 4, NoWait: true}\n\tjreq, _ = json.Marshal(req)\n\tnc.PublishRequest(subj, reply, jreq)\n\t// Receive 4 messages + the 409\n\tcheckSubsPending(t, sub, 5)\n\tfor i := 0; i < 4; i++ {\n\t\tm, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(m.Data) == dsz)\n\t\trequire_True(t, len(m.Header) == 0)\n\t}\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tcheckHeader(m, badReq)\n\tcheckSubsPending(t, sub, 0)\n\n\treq = &JSApiConsumerGetNextRequest{Batch: 1_000, MaxBytes: msz, NoWait: true}\n\tjreq, _ = json.Marshal(req)\n\tnc.PublishRequest(subj, reply, jreq)\n\t// Receive 1 message + 409\n\tcheckSubsPending(t, sub, 2)\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_True(t, len(m.Data) == dsz)\n\trequire_True(t, len(m.Header) == 0)\n\tm, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tcheckHeader(m, badReq)\n\tcheckSubsPending(t, sub, 0)\n}\n\n// https://github.com/nats-io/nats-server/issues/6824\nfunc TestJetStreamConsumerDeliverAllOverlappingFilterSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnectNewAPI(t, s)\n\tdefer nc.Close()\n\n\tctx := context.Background()\n\t_, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"stream.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tpublishMessageCount := 10\n\tfor i := 0; i < publishMessageCount; i++ {\n\t\t_, err = js.Publish(ctx, \"stream.A\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Create consumer\n\tconsumer, err := js.CreateOrUpdateConsumer(ctx, \"TEST\", jetstream.ConsumerConfig{\n\t\tDeliverPolicy: jetstream.DeliverAllPolicy,\n\t\tFilterSubjects: []string{\n\t\t\t\"stream.A\",\n\t\t\t\"stream.A.>\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tmessages := make(chan jetstream.Msg)\n\tcc, err := consumer.Consume(func(msg jetstream.Msg) {\n\t\tmessages <- msg\n\t\tmsg.Ack()\n\t})\n\trequire_NoError(t, err)\n\tdefer cc.Drain()\n\n\tvar count = 0\n\tfor {\n\t\tif count == publishMessageCount {\n\t\t\t// All messages received.\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase <-messages:\n\t\t\tcount++\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Errorf(\"Timeout reached, %d messages received. Exiting.\", count)\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/6844\nfunc TestJetStreamConsumerDeliverAllNonOverlappingFilterSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnectNewAPI(t, s)\n\tdefer nc.Close()\n\n\tctx := context.Background()\n\t_, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"stream.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tpublishMessageCount := 10\n\tfor i := 0; i < publishMessageCount; i++ {\n\t\t_, err = js.Publish(ctx, \"stream.subject\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Create consumer\n\tconsumer, err := js.CreateOrUpdateConsumer(ctx, \"TEST\", jetstream.ConsumerConfig{\n\t\tDeliverPolicy: jetstream.DeliverAllPolicy,\n\t\tFilterSubjects: []string{\n\t\t\t\"stream.subject.A\",\n\t\t\t\"stream.subject.A.>\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\ti, err := consumer.Info(ctx)\n\trequire_NoError(t, err)\n\trequire_Equal(t, i.NumPending, 0)\n}\n\n// https://github.com/nats-io/nats-server/issues/7336\nfunc TestJetStreamConsumerDeliverPartialOverlappingFilterSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnectNewAPI(t, s)\n\tdefer nc.Close()\n\n\tctx := context.Background()\n\t_, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"stream.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tpublishMessageCount := 10\n\tfor i := 0; i < publishMessageCount; i++ {\n\t\t_, err = js.Publish(ctx, \"stream.A\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Create consumer\n\tconsumer, err := js.CreateOrUpdateConsumer(ctx, \"TEST\", jetstream.ConsumerConfig{\n\t\tDeliverPolicy: jetstream.DeliverAllPolicy,\n\t\tFilterSubjects: []string{\n\t\t\t\"stream.A\",\n\t\t\t\"stream.*.A\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tmessages := make(chan jetstream.Msg)\n\tcc, err := consumer.Consume(func(msg jetstream.Msg) {\n\t\tmessages <- msg\n\t\tmsg.Ack()\n\t})\n\trequire_NoError(t, err)\n\tdefer cc.Drain()\n\n\tvar count = 0\n\tfor {\n\t\tif count == publishMessageCount {\n\t\t\t// All messages received.\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase <-messages:\n\t\t\tcount++\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Errorf(\"Timeout reached, %d messages received. Exiting.\", count)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestJetStreamConsumerStateAlwaysFromStore(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckWait: 2 * time.Second,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo.bar\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish two messages, one the consumer is interested in.\n\t_, err = js.Publish(\"foo.bar\", nil)\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo.other\", nil)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t// Consumer info should start empty.\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 0)\n\trequire_Equal(t, ci.AckFloor.Stream, 0)\n\n\t// Fetch more messages than match our filter.\n\tmsgs, err := sub.Fetch(2, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\n\t// We have received, but not acknowledged, consumer info must reflect that.\n\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 0)\n\n\t// Now we acknowledge the message and expect our delivered/ackfloor to be correct.\n\trequire_NoError(t, msgs[0].AckSync())\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.Delivered.Stream, 1)\n\trequire_Equal(t, ci.AckFloor.Stream, 1)\n}\n\nfunc TestJetStreamConsumerPullNoWaitBatchLargerThanPending(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"C\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tFilterSubject: \"foo\",\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\treq := JSApiConsumerGetNextRequest{Batch: 10, NoWait: true}\n\n\t\tfor range 5 {\n\t\t\t_, err := js.Publish(\"foo\", []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tsub := sendRequest(t, nc, \"rply\", req)\n\t\tdefer sub.Unsubscribe()\n\n\t\t// Should get all 5 messages.\n\t\tfor range 5 {\n\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(msg.Data) == 0 && msg.Header != nil {\n\t\t\t\tt.Fatalf(\"Expected data, got: %s\", msg.Header.Get(\"Description\"))\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamConsumerNotInactiveDuringAckWait(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tInactiveThreshold: 500 * time.Millisecond, // Pull mode adds up to 1 second randomly.\n\t\t\tAckWait: time.Minute,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgs, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\n\t\t// AckWait is still active, so must not delete the consumer while waiting for an ack.\n\t\ttime.Sleep(1750 * time.Millisecond)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\t// Not waiting on AckWait anymore, consumer is deleted after the inactivity threshold.\n\t\ttime.Sleep(1750 * time.Millisecond)\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_Error(t, err, nats.ErrConsumerNotFound)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamConsumerNotInactiveDuringAckWaitBackoff(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tInactiveThreshold: 500 * time.Millisecond, // Pull mode adds up to 1 second randomly.\n\t\t\tBackOff: []time.Duration{\n\t\t\t\t2 * time.Second,\n\t\t\t\t4 * time.Second,\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tmsgs, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\n\t\t// AckWait is still active, so must not delete the consumer while waiting for an ack.\n\t\ttime.Sleep(1750 * time.Millisecond)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msgs[0].Nak())\n\n\t\tmsgs, err = sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\n\t\t// AckWait is still active, now based on backoff, so must not delete the consumer while waiting for an ack.\n\t\t// We've confirmed can wait 2s AckWait + InactiveThreshold, now check we can also wait for the backoff.\n\t\ttime.Sleep(3750 * time.Millisecond)\n\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\t// Not waiting on AckWait anymore, consumer is deleted after the inactivity threshold.\n\t\ttime.Sleep(1750 * time.Millisecond)\n\t\t_, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_Error(t, err, nats.ErrConsumerNotFound)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestSortingConsumerPullRequests(t *testing.T) {\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\trequests []waitingRequest\n\t\texpected []waitingRequest\n\t}{\n\t\t{\n\t\t\tname: \"sort\",\n\t\t\trequests: []waitingRequest{\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}},\n\t\t\t},\n\t\t\texpected: []waitingRequest{\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test if sort is stable\",\n\t\t\trequests: []waitingRequest{\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1a\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2a\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1b\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2b\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1c\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}, reply: \"3a\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2c\"},\n\t\t\t},\n\t\t\texpected: []waitingRequest{\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1a\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1b\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1c\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2a\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2b\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2c\"},\n\t\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}, reply: \"3a\"},\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Helper()\n\n\t\t\twq := newWaitQueue(100)\n\n\t\t\tfor _, r := range test.requests {\n\t\t\t\terr := wq.addPrioritized(&r)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tif wq.n != len(test.expected) {\n\t\t\t\tt.Fatalf(\"Expected %d requests, got %d\", len(test.expected), wq.n)\n\t\t\t}\n\n\t\t\t// Verify order\n\t\t\tfor i, expected := range test.expected {\n\t\t\t\tcurrent := wq.peek()\n\t\t\t\tif current == nil {\n\t\t\t\t\tt.Fatalf(\"Expected request at position %d, but queue was empty\", i)\n\t\t\t\t}\n\t\t\t\tif current.priorityGroup.Priority != expected.priorityGroup.Priority {\n\t\t\t\t\tt.Fatalf(\"At position %d: expected priority %d, got %d\",\n\t\t\t\t\t\ti, expected.priorityGroup.Priority, current.priorityGroup.Priority)\n\t\t\t\t}\n\t\t\t\tif current.reply != expected.reply {\n\t\t\t\t\tt.Fatalf(\"At position %d: expected reply %q, got %q\", i, expected.reply, current.reply)\n\t\t\t\t}\n\t\t\t\twq.removeCurrent()\n\t\t\t}\n\n\t\t})\n\t}\n}\n\nfunc TestWaitQueuePopAndRequeue(t *testing.T) {\n\tt.Run(\"basic requeue with batches\", func(t *testing.T) {\n\t\twq := newWaitQueue(100)\n\n\t\t// Add elements with different priorities\n\t\treqs := []waitingRequest{\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1a\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1b\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1c\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2a\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2b\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2c\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}, reply: \"3a\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}, reply: \"3b\", n: 3},\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 3}, reply: \"3c\", n: 3},\n\t\t}\n\n\t\tfor i := range reqs {\n\t\t\terr := wq.addPrioritized(&reqs[i])\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\ti := 0\n\t\tj := 0\n\t\tfor {\n\t\t\twr := wq.popAndRequeue()\n\t\t\trequire_Equal(t, wr.reply, fmt.Sprintf(\"%da\", j+1))\n\t\t\twr = wq.popAndRequeue()\n\t\t\trequire_Equal(t, wr.reply, fmt.Sprintf(\"%db\", j+1))\n\t\t\twr = wq.popAndRequeue()\n\t\t\trequire_Equal(t, wr.reply, fmt.Sprintf(\"%dc\", j+1))\n\n\t\t\ti++\n\t\t\tif i%3 == 0 {\n\t\t\t\tj++\n\t\t\t}\n\t\t\trequire_Equal(t, wq.n, 9-(j*3))\n\t\t\tif j == 2 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t})\n\n\tt.Run(\"request removal when fully served\", func(t *testing.T) {\n\t\twq := newWaitQueue(100)\n\n\t\treqs := []waitingRequest{\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1a\", n: 2}, // Will be removed after 2 pops\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"1b\", n: 1}, // Will be removed after 1 pop\n\t\t\t{priorityGroup: &PriorityGroup{Priority: 2}, reply: \"2a\", n: 3}, // Will remain\n\t\t}\n\n\t\tfor i := range reqs {\n\t\t\terr := wq.addPrioritized(&reqs[i])\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tinitialCount := wq.n\n\n\t\t// Pop 1a first time (n=2 -> n=1, should requeue)\n\t\twr := wq.popAndRequeue()\n\t\tif wr == nil || wr.reply != \"1a\" || wr.n != 1 {\n\t\t\tt.Fatalf(\"Expected 1a with n=1, got %v with n=%d\", wr, wr.n)\n\t\t}\n\t\tif wq.n != initialCount {\n\t\t\tt.Fatalf(\"Queue size should remain %d, got %d\", initialCount, wq.n)\n\t\t}\n\n\t\t// Pop 1b (n=1 -> n=0, should be removed)\n\t\twr = wq.popAndRequeue()\n\t\tif wr == nil || wr.reply != \"1b\" || wr.n != 0 {\n\t\t\tt.Fatalf(\"Expected 1b with n=0, got %v with n=%d\", wr, wr.n)\n\t\t}\n\t\tif wq.n != initialCount-1 {\n\t\t\tt.Fatalf(\"Queue size should be %d after 1b removal, got %d\", initialCount-1, wq.n)\n\t\t}\n\n\t\t// Pop 1a second time (n=1 -> n=0, should be removed)\n\t\twr = wq.popAndRequeue()\n\t\tif wr == nil || wr.reply != \"1a\" || wr.n != 0 {\n\t\t\tt.Fatalf(\"Expected 1a with n=0, got %v with n=%d\", wr, wr.n)\n\t\t}\n\t\tif wq.n != initialCount-2 {\n\t\t\tt.Fatalf(\"Queue size should be %d after 1a removal, got %d\", initialCount-2, wq.n)\n\t\t}\n\n\t\t// Only 2a should remain\n\t\tnext := wq.peek()\n\t\tif next == nil || next.reply != \"2a\" || next.n != 3 {\n\t\t\tt.Fatalf(\"Expected only 2a with n=3 to remain, got %v with n=%d\", next, next.n)\n\t\t}\n\t})\n\n\tt.Run(\"single element lifecycle\", func(t *testing.T) {\n\t\twq := newWaitQueue(10)\n\t\treq := waitingRequest{priorityGroup: &PriorityGroup{Priority: 1}, reply: \"single\", n: 2}\n\t\twq.add(&req)\n\n\t\t// First pop (n=2 -> n=1, should stay)\n\t\tresult := wq.popAndRequeue()\n\t\tif result == nil || result.reply != \"single\" || result.n != 1 {\n\t\t\tt.Fatalf(\"Expected single with n=1, got %v with n=%d\", result, result.n)\n\t\t}\n\t\tif wq.n != 1 {\n\t\t\tt.Fatalf(\"Queue should still have 1 element, got %d\", wq.n)\n\t\t}\n\n\t\t// Second pop (n=1 -> n=0, should be removed)\n\t\tresult = wq.popAndRequeue()\n\t\tif result == nil || result.reply != \"single\" || result.n != 0 {\n\t\t\tt.Fatalf(\"Expected single with n=0, got %v with n=%d\", result, result.n)\n\t\t}\n\t\tif wq.n != 0 {\n\t\t\tt.Fatalf(\"Queue should be empty after removal, got %d elements\", wq.n)\n\t\t}\n\t\tif wq.head != nil {\n\t\t\tt.Fatalf(\"Queue head should be nil after removal, got %v\", wq.head)\n\t\t}\n\t})\n}\n\nfunc TestJetStreamConsumerPrioritized(t *testing.T) {\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: AckNone,\n\t\tFilterSubject: \"foo\",\n\t\tPriorityGroups: []string{\"A\"},\n\t\tPriorityPolicy: PriorityPrioritized,\n\t})\n\trequire_NoError(t, err)\n\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Helper function to send pull request and get response\n\tsendPullRequest := func(t *testing.T, inbox string, priority int, batch int) *nats.Subscription {\n\t\tsub, err := nc.SubscribeSync(inbox)\n\t\trequire_NoError(t, err)\n\n\t\treq := JSApiConsumerGetNextRequest{\n\t\t\tBatch: batch,\n\t\t\tExpires: 10 * time.Second,\n\t\t\tPriorityGroup: PriorityGroup{\n\t\t\t\tGroup: \"A\",\n\t\t\t\tPriority: priority,\n\t\t\t},\n\t\t}\n\t\treqb, _ := json.Marshal(req)\n\t\terr = nc.PublishRequest(\"$JS.API.CONSUMER.MSG.NEXT.TEST.CONSUMER\", inbox, reqb)\n\t\trequire_NoError(t, err)\n\n\t\treturn sub\n\t}\n\n\tt.Run(\"invalid priority number\", func(t *testing.T) {\n\n\t\tsub := sendPullRequest(t, \"invalid_priority\", 10, 1)\n\n\t\tmsg, err := sub.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"400\")\n\n\t\tsub2 := sendPullRequest(t, \"invalid_priority\", -5, 1)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"400\")\n\n\t\tmsg, err = sub2.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"400\")\n\n\t})\n\n\tt.Run(\"priority ordering\", func(t *testing.T) {\n\t\tpriority3 := sendPullRequest(t, \"priority3\", 3, 1) // Priority 3\n\t\tpriority1 := sendPullRequest(t, \"priority1\", 1, 1) // Priority 1 (should be served first)\n\t\tpriority2 := sendPullRequest(t, \"priority2\", 2, 2) // Priority 2\n\n\t\t// Small delay to ensure requests are processed\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"message\"))\n\t\trequire_NoError(t, err)\n\n\t\t// Priority 3 should time out\n\t\t_, err := priority3.NextMsg(200 * time.Millisecond)\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t// Priority 2 should time out\n\t\t_, err = priority2.NextMsg(200 * time.Millisecond)\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t// Priority 1 should get message first\n\t\tmsg, err := priority1.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"message\"))\n\t\trequire_NoError(t, err)\n\n\t\t// Priority 3 should time out.\n\t\t_, err = priority3.NextMsg(200 * time.Millisecond)\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t// Priority 2 should get message next\n\t\tmsg, err = priority2.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"message\"))\n\t\trequire_NoError(t, err)\n\n\t\t// Priority 2 should get message next\n\t\tmsg, err = priority2.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t// Priority 3 should time out\n\t\t_, err = priority3.NextMsg(2 * time.Second)\n\t\trequire_Error(t, err, nats.ErrTimeout)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"message\"))\n\t\trequire_NoError(t, err)\n\n\t\t// Priority 3 should get message next\n\t\tmsg, err = priority3.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\t})\n\n\tt.Run(\"dynamic priority interruption\", func(t *testing.T) {\n\n\t\tinbox3 := nats.NewInbox()\n\t\tsub3 := sendPullRequest(t, inbox3, 3, 3)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"msg\"))\n\t\trequire_NoError(t, err)\n\n\t\tmsg, err := sub3.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t// Now, despite priority 3 still having pending messages, a new pull request\n\t\t// with a lower priority should be able to take over the delivery.\n\t\tinbox2 := nats.NewInbox()\n\t\tsub2 := sendPullRequest(t, inbox2, 2, 2)\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"msg\"))\n\t\trequire_NoError(t, err)\n\n\t\tmsg, err = sub2.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t// The same should happen with priority 1.\n\t\tinbox1 := nats.NewInbox()\n\t\tsub1 := sendPullRequest(t, inbox1, 1, 1) // Priority 1, batch 3\n\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"msg\"))\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"msg\"))\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"foo\", fmt.Appendf(nil, \"msg\"))\n\t\trequire_NoError(t, err)\n\n\t\tmsg, err = sub1.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t// Now priority 2 should take over for last message.\n\t\tmsg, err = sub2.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\n\t\t// Finally, priority 3 should get its last message.\n\t\tmsg, err = sub3.NextMsg(2 * time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NotNil(t, msg)\n\t})\n}\n\nfunc TestJetStreamConsumerMaxDeliverUnderflow(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{Durable: \"CONSUMER\", MaxDeliver: -1}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Infinite MaxDeliver should be zero.\n\to.mu.RLock()\n\tmaxdc := o.maxdc\n\to.mu.RUnlock()\n\trequire_Equal(t, maxdc, 0)\n\n\t// Finite MaxDeliver should be reported the same.\n\tcfg.MaxDeliver = 1\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\to.mu.RLock()\n\tmaxdc = o.maxdc\n\to.mu.RUnlock()\n\trequire_Equal(t, maxdc, 1)\n\n\t// Infinite MaxDeliver should be zero.\n\tcfg.MaxDeliver = -1\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\to.mu.RLock()\n\tmaxdc = o.maxdc\n\to.mu.RUnlock()\n\trequire_Equal(t, maxdc, 0)\n}\n\n// https://github.com/nats-io/nats-server/issues/7457\nfunc TestJetStreamConsumerNoWaitNoMessagesOnEos(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\tsub, err := nc.SubscribeSync(\"reply\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// Fiddle with the pending count such that the NoWait request will go through,\n\t// and the \"404 No Messages\" will be sent when hitting the end of the stream.\n\to.mu.Lock()\n\to.npc++\n\to.mu.Unlock()\n\n\treq := &JSApiConsumerGetNextRequest{NoWait: true}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\to.processNextMsgRequest(\"reply\", jreq)\n\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"No Messages\")\n}\n\n// https://github.com/nats-io/nats-server/issues/5373\nfunc TestJetStreamConsumerNoWaitNoMessagesOnEosWithDeliveredMsgs(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\trequire_NoError(t, err)\n\n\tsub, err := nc.SubscribeSync(\"reply\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\treq := &JSApiConsumerGetNextRequest{NoWait: true, Batch: 2}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\to.processNextMsgRequest(\"reply\", jreq)\n\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Subject, \"foo\")\n\trequire_Equal(t, string(msg.Data), \"msg\")\n\n\t// We requested two messages but the stream only contained 1.\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"No Messages\")\n}\n\nfunc TestJetStreamConsumerEfficientInterestStateCheck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: InterestPolicy,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t_, _, err = mset.store.StoreMsg(\"foo\", nil, nil, 0)\n\trequire_NoError(t, err)\n\t_, _, err = mset.store.StoreMsg(\"foo\", nil, nil, 0)\n\trequire_NoError(t, err)\n\trequire_NoError(t, mset.store.SkipMsgs(3, 10_000_000))\n\t_, _, err = mset.store.StoreMsg(\"foo\", nil, nil, 0)\n\trequire_NoError(t, err)\n\n\t// Only acknowledge the first message, but the last two will have been marked as delivered.\n\t// This will create a very large gap between the ack floor and the delivered sequence.\n\tmsgs, err := sub.Fetch(3)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 3)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\n\t// We used to do a linear scan, which was terribly slow for this case. Instead,\n\t// we should efficiently skip over deletes or messages that don't match our filter.\n\tvar ss StreamState\n\tmset.store.FastState(&ss)\n\tstart := time.Now()\n\trequire_NoError(t, o.checkStateForInterestStream(&ss))\n\telapsed := time.Since(start)\n\trequire_LessThan(t, elapsed, 50*time.Millisecond)\n}\n\nfunc TestJetStreamConsumerWithCorruptStateIsDeleted(t *testing.T) {\n\tstoreDir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {store_dir: %q}\n\t`, storeDir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DURABLE\"})\n\trequire_NoError(t, err)\n\n\t// Consumer should be returned in names and info requests.\n\tvar names []string\n\tfor name := range js.ConsumerNames(\"TEST\") {\n\t\tnames = append(names, name)\n\t}\n\trequire_Len(t, len(names), 1)\n\trequire_Equal(t, names[0], \"DURABLE\")\n\n\t_, err = js.ConsumerInfo(\"TEST\", \"DURABLE\")\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"DURABLE\")\n\trequire_NotNil(t, o)\n\tfs := o.store.(*consumerFileStore)\n\tfs.mu.Lock()\n\tifn := fs.ifn\n\tfs.mu.Unlock()\n\n\t// Stop the server and zero the consumer's data file.\n\ts.Shutdown()\n\tstat, err := os.Stat(ifn)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, 0, stat.Size())\n\tbuf := bytes.Repeat([]byte{0}, int(stat.Size()))\n\trequire_NoError(t, os.WriteFile(ifn, buf, defaultFilePerms))\n\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Reconnect.\n\tnc.Close()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// The consumer should be recognized to have corrupt state and be deleted automatically,\n\t// so it doesn't show up in requests.\n\tnames = make([]string, 0)\n\tfor name := range js.ConsumerNames(\"TEST\") {\n\t\tnames = append(names, name)\n\t}\n\trequire_Len(t, len(names), 0)\n\n\t_, err = js.ConsumerInfo(\"TEST\", \"DURABLE\")\n\trequire_Error(t, err, nats.ErrConsumerNotFound)\n}\n\nfunc TestJetStreamConsumerNoDeleteAfterConcurrentShutdownAndLeaderChange(t *testing.T) {\n\tstoreDir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {store_dir: %q}\n\t`, storeDir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tmset, err := s.globalAccount().addStream(&StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"CONSUMER\", AckPolicy: AckExplicit})\n\trequire_NoError(t, err)\n\n\tfs := o.store.(*consumerFileStore)\n\tfs.mu.Lock()\n\todir := fs.odir\n\tfs.mu.Unlock()\n\n\t// Simulate a server shutting down at the same time that a replicated consumer becomes leader.\n\t// Previously the consumer would be deleted due to not being able to create the required subscriptions.\n\ts.Shutdown()\n\n\t// We'll revert required settings to ensure we can set it as leader.\n\to.mu.Lock()\n\to.mset = mset\n\to.closed = false\n\to.leader.Store(false)\n\tfs.mu.Lock()\n\tfs.closed = false\n\tfs.odir = odir\n\tfs.mu.Unlock()\n\to.mu.Unlock()\n\to.setLeader(true)\n\n\t// Check the consumer was not deleted.\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tmset, err = s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to = mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n}\n\nfunc TestJetStreamConsumerOnlyRecalculatePendingIfFilterSubjectUpdated(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{Durable: \"DURABLE\"}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\t// Publishing a message will adjust the pending count for the consumer.\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"DURABLE\")\n\trequire_NotNil(t, o)\n\to.mu.RLock()\n\tnpc, npf := o.npc, o.npf\n\to.mu.RUnlock()\n\trequire_Equal(t, npc, 1)\n\trequire_Equal(t, npf, 0)\n\n\t// Updating a consumer with the same config should NOT recalculate pending.\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\to.mu.RLock()\n\tnpc, npf = o.npc, o.npf\n\to.mu.RUnlock()\n\trequire_Equal(t, npc, 1)\n\trequire_Equal(t, npf, 0)\n\n\t// Updating the filter subject should recalculate pending.\n\tcfg.FilterSubject = \"foo\"\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\to.mu.RLock()\n\tnpc, npf = o.npc, o.npf\n\to.mu.RUnlock()\n\trequire_Equal(t, npc, 1)\n\trequire_Equal(t, npf, 1)\n}\n\nfunc TestJetStreamConsumerCheckNumPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{Durable: \"DURABLE\"}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\t// Publish 5 messages.\n\tfor range 5 {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"DURABLE\")\n\trequire_NotNil(t, o)\n\n\t// Initial state should point to the first message, with all 5 pending.\n\to.mu.Lock()\n\tsseq, np := o.sseq, o.numPending()\n\to.mu.Unlock()\n\trequire_Equal(t, sseq, 1)\n\trequire_Equal(t, np, 5)\n\n\t// We've sent 2 messages, the next message is seq 3, should return 3 pending.\n\to.mu.Lock()\n\to.sseq = 3\n\tnp, _ = o.checkNumPending()\n\to.mu.Unlock()\n\trequire_Equal(t, np, 3)\n\n\t// An excessively large sequence should not panic the num pending correction to 0.\n\to.mu.Lock()\n\to.sseq = 100\n\tnp, _ = o.checkNumPending()\n\to.mu.Unlock()\n\trequire_Equal(t, np, 0)\n\n\t// Delete the tail of the stream.\n\tfor i := range 4 {\n\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", uint64(i+2)))\n\t}\n\t// We've sent a single message, but that is the only message in the stream.\n\t// We don't fully recalculate the pending count, but should at least return something that's reasonable.\n\t// There's only a single message in the stream, so our pending can't be larger than that.\n\to.mu.Lock()\n\to.sseq = 2\n\to.npc = 4\n\tnp, _ = o.checkNumPending()\n\to.mu.Unlock()\n\trequire_Equal(t, np, 1)\n}\n\n// https://github.com/nats-io/nats-server/issues/7779\nfunc TestJetStreamConsumerAllowOverlappingSubjectsIfNotSubset(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"event.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tparts := []string{\"foo\", \"bar\", \"baz\", \"oth\"}\n\tfor _, start := range parts {\n\t\tfor _, end := range parts {\n\t\t\tsubj := fmt.Sprintf(\"event.%s.%s\", start, end)\n\t\t\t_, err = js.Publish(subj, nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\t// First consumer isn't allowed, since 'event.*.foo' is a subset of 'event.>'.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"OVERLAP\",\n\t\tFilterSubjects: []string{\n\t\t\t\"event.>\",\n\t\t\t\"event.*.foo\",\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSConsumerOverlappingSubjectFiltersError())\n\n\t// The second consumer's subjects do overlap, but they are separate sets (that partially overlap).\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"DURABLE\",\n\t\tFilterSubjects: []string{\n\t\t\t\"event.foo.*\",\n\t\t\t\"event.*.foo\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"DURABLE\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t// Confirm the consumer gets all messages in the expected order without duplicates.\n\tbatch := len(parts) * len(parts)\n\tmsgs, err := sub.Fetch(batch, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\tvar count int\n\tfor _, start := range parts {\n\t\tfor _, end := range parts {\n\t\t\tif start != \"foo\" && end != \"foo\" {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcount++\n\t\t\trequire_True(t, len(msgs) >= count)\n\t\t\tmsg := msgs[count-1]\n\t\t\tsubj := fmt.Sprintf(\"event.%s.%s\", start, end)\n\t\t\trequire_Equal(t, msg.Subject, subj)\n\t\t}\n\t}\n\trequire_Equal(t, count, 7)\n\trequire_Len(t, len(msgs), count)\n}\n\nfunc TestJetStreamConsumerResetToSequence(t *testing.T) {\n\ttest := func(replicas int) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err := js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\",\n\t\t\tnats.BindStream(\"TEST\"),\n\t\t\tnats.MaxAckPending(1),\n\t\t\tnats.AckWait(time.Second),\n\t\t\tnats.ConsumerReplicas(replicas),\n\t\t)\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\tfor i := range 4 {\n\t\t\t_, err = js.Publish(\"foo\", []byte(fmt.Sprintf(\"msg%d\", i+1)))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t// Fetch and ack the first message.\n\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_Equal(t, string(msgs[0].Data), \"msg1\")\n\t\trequire_NoError(t, msgs[0].AckSync())\n\n\t\t// Now switch the stream to interest now that the messages and consumer are all available.\n\t\tcfg.Retention = nats.InterestPolicy\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.FirstSeq != 2 {\n\t\t\t\treturn fmt.Errorf(\"expected first seq to be 2, got %v\", state)\n\t\t\t}\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\ts := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\to := mset.lookupConsumer(\"CONSUMER\")\n\t\trequire_NotNil(t, o)\n\n\t\ttype Expected struct{ numPending, numAckPending, dseq, adflr, sseq, asflr uint64 }\n\t\tcheckConsumerInfo := func(e Expected) {\n\t\t\tt.Helper()\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\t\trequire_NoError(t, err)\n\t\t\to.mu.RLock()\n\t\t\tsseq, dseq, adflr, asflr, pending := o.sseq, o.dseq, o.adflr, o.asflr, len(o.pending)\n\t\t\to.mu.RUnlock()\n\t\t\trequire_Equal(t, ci.NumPending, e.numPending)\n\t\t\t// NumAckPending needs to match both in the store and running state.\n\t\t\trequire_Equal(t, ci.NumAckPending, int(e.numAckPending))\n\t\t\trequire_Equal(t, pending, int(e.numAckPending))\n\t\t\t// Delivered.Consumer needs to match both in the store and running state.\n\t\t\trequire_Equal(t, ci.Delivered.Consumer, e.dseq)\n\t\t\trequire_Equal(t, dseq-1, e.dseq)\n\t\t\t// AckFloor.Consumer needs to match both in the store and running state.\n\t\t\trequire_Equal(t, ci.AckFloor.Consumer, e.adflr)\n\t\t\trequire_Equal(t, adflr, e.adflr)\n\t\t\t// Delivered.Stream needs to match both in the store and running state.\n\t\t\trequire_Equal(t, ci.Delivered.Stream, e.sseq)\n\t\t\trequire_Equal(t, sseq-1, e.sseq)\n\t\t\t// AckFloor.Stream needs to match both in the store and running state.\n\t\t\trequire_Equal(t, ci.AckFloor.Stream, e.asflr)\n\t\t\trequire_Equal(t, asflr, e.asflr)\n\t\t}\n\t\t// A single message was acked, 3 are ready to be delivered.\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 3, numAckPending: 0,\n\t\t\tdseq: 1, adflr: 1,\n\t\t\tsseq: 1, asflr: 1,\n\t\t})\n\n\t\tmsgs, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_Equal(t, string(msgs[0].Data), \"msg2\")\n\t\t// Don't ack.\n\n\t\t// A message has been delivered and needs to be acked still.\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 2, numAckPending: 1,\n\t\t\tdseq: 2, adflr: 1,\n\t\t\tsseq: 2, asflr: 1,\n\t\t})\n\n\t\t// Resetting the consumer with an empty request results in a reset back to the ack floor.\n\t\tvar resp JSApiConsumerResetResponse\n\t\tmsg, err := nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\t\trequire_Equal(t, resp.ResetSeq, 2)\n\t\trequire_Equal(t, resp.Delivered.Stream, 1)\n\t\trequire_Equal(t, resp.Delivered.Last, nil)\n\t\trequire_Equal(t, resp.AckFloor.Stream, 1)\n\t\trequire_Equal(t, resp.AckFloor.Last, nil)\n\n\t\t// Should be back.\n\t\trequire_Equal(t, resp.NumPending, 3)\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 3, numAckPending: 0,\n\t\t\tdseq: 0, adflr: 0,\n\t\t\tsseq: 1, asflr: 1,\n\t\t})\n\n\t\t// Trying to reset to zero also resets back to the ack floor.\n\t\treq := JSApiConsumerResetRequest{Seq: 0}\n\t\tdata, err := json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\tresp = JSApiConsumerResetResponse{}\n\t\tmsg, err = nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\t\trequire_Equal(t, resp.ResetSeq, 2)\n\t\trequire_Equal(t, resp.Delivered.Stream, 1)\n\t\trequire_Equal(t, resp.Delivered.Last, nil)\n\t\trequire_Equal(t, resp.AckFloor.Stream, 1)\n\t\trequire_Equal(t, resp.AckFloor.Last, nil)\n\n\t\trequire_Equal(t, resp.NumPending, 3)\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 3, numAckPending: 0,\n\t\t\tdseq: 0, adflr: 0,\n\t\t\tsseq: 1, asflr: 1,\n\t\t})\n\n\t\t// Resetting the consumer to the last message's sequence so it can be delivered still.\n\t\treq = JSApiConsumerResetRequest{Seq: 4}\n\t\tdata, err = json.Marshal(req)\n\t\trequire_NoError(t, err)\n\t\tresp = JSApiConsumerResetResponse{}\n\t\tmsg, err = nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\t\trequire_Equal(t, resp.ResetSeq, 4)\n\t\trequire_Equal(t, resp.Delivered.Stream, 3)\n\t\trequire_Equal(t, resp.Delivered.Last, nil)\n\t\trequire_Equal(t, resp.AckFloor.Stream, 3)\n\t\trequire_Equal(t, resp.AckFloor.Last, nil)\n\n\t\t// One message left as pending, the delivered counts are now zero,\n\t\t// but the sequence and AckFloor are moved up.\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 1, numAckPending: 0,\n\t\t\tdseq: 0, adflr: 0,\n\t\t\tsseq: 3, asflr: 3,\n\t\t})\n\n\t\t// As a result of moving the starting sequence up, some messages\n\t\t// have now lost interest and need to be removed.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.FirstSeq != 4 {\n\t\t\t\treturn fmt.Errorf(\"expected first seq to be 4, got %v\", state)\n\t\t\t}\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\n\t\t// We fetch the last message.\n\t\tmsgs, err = sub.Fetch(1, nats.MaxWait(2*time.Second))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_Equal(t, string(msgs[0].Data), \"msg4\")\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 0, numAckPending: 1,\n\t\t\tdseq: 1, adflr: 0,\n\t\t\tsseq: 4, asflr: 3,\n\t\t})\n\n\t\t// After acking, the delivered count should only show 1, but the AckFloor is higher.\n\t\trequire_NoError(t, msgs[0].AckSync())\n\t\tcheckConsumerInfo(Expected{\n\t\t\tnumPending: 0, numAckPending: 0,\n\t\t\tdseq: 1, adflr: 1,\n\t\t\tsseq: 4, asflr: 4,\n\t\t})\n\n\t\t// The stream should now be empty.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tmset, err := sl.globalAccount().lookupStream(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 0 || state.FirstSeq != 5 {\n\t\t\t\treturn fmt.Errorf(\"stream is not empty: %v\", state)\n\t\t\t}\n\t\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t\t})\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", replicas), func(t *testing.T) {\n\t\t\ttest(replicas)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamConsumerResetToSequenceConstraintOnStartSeq(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\",\n\t\tnats.BindStream(\"TEST\"),\n\t\tnats.StartSequence(3),\n\t)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tfor i := range 4 {\n\t\t_, err = js.Publish(\"foo\", []byte(fmt.Sprintf(\"msg%d\", i+1)))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Fetch and ack the first message.\n\tmsgs, err := sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_Equal(t, string(msgs[0].Data), \"msg3\")\n\trequire_NoError(t, msgs[0].AckSync())\n\n\t// Trying to reset below the configured starting sequence fails.\n\treq := JSApiConsumerResetRequest{Seq: 2}\n\tdata, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerResetResponse\n\tmsg, err := nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\trequire_NotNil(t, resp.Error)\n\trequire_Error(t, resp.Error, NewJSConsumerInvalidResetError(errors.New(\"below start seq\")))\n\n\t// Resetting above or equal to the configured starting sequence succeeds.\n\treq = JSApiConsumerResetRequest{Seq: 3}\n\tdata, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\tresp = JSApiConsumerResetResponse{}\n\tmsg, err = nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\trequire_Equal(t, resp.ResetSeq, 3)\n\trequire_Equal(t, resp.Delivered.Stream, 2)\n\trequire_Equal(t, resp.Delivered.Last, nil)\n\trequire_Equal(t, resp.AckFloor.Stream, 2)\n\trequire_Equal(t, resp.AckFloor.Last, nil)\n\n\t// Fetching should now get the same message.\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_Equal(t, string(msgs[0].Data), \"msg3\")\n\trequire_NoError(t, msgs[0].AckSync())\n}\n\nfunc TestJetStreamConsumerResetToSequenceConstraintOnStartTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tfor i := range 2 {\n\t\t_, err = js.Publish(\"foo\", []byte(fmt.Sprintf(\"msg%d\", i+1)))\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\",\n\t\tnats.BindStream(\"TEST\"),\n\t\tnats.StartTime(time.Now()),\n\t)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tfor i := range 2 {\n\t\t_, err = js.Publish(\"foo\", []byte(fmt.Sprintf(\"msg%d\", i+3)))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Fetch and ack the first message.\n\tmsgs, err := sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_Equal(t, string(msgs[0].Data), \"msg3\")\n\trequire_NoError(t, msgs[0].AckSync())\n\n\t// Trying to reset below the configured starting time fails.\n\treq := JSApiConsumerResetRequest{Seq: 2}\n\tdata, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerResetResponse\n\tmsg, err := nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\trequire_NotNil(t, resp.Error)\n\trequire_Error(t, resp.Error, NewJSConsumerInvalidResetError(errors.New(\"below start time\")))\n\n\t// Resetting to a sequence that has a starting time above or equal to the configured starting time succeeds.\n\treq = JSApiConsumerResetRequest{Seq: 3}\n\tdata, err = json.Marshal(req)\n\trequire_NoError(t, err)\n\tresp = JSApiConsumerResetResponse{}\n\tmsg, err = nc.Request(\"$JS.API.CONSUMER.RESET.TEST.CONSUMER\", data, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_Equal(t, resp.Type, JSApiConsumerResetResponseType)\n\trequire_Equal(t, resp.ResetSeq, 3)\n\trequire_Equal(t, resp.Delivered.Stream, 2)\n\trequire_Equal(t, resp.Delivered.Last, nil)\n\trequire_Equal(t, resp.AckFloor.Stream, 2)\n\trequire_Equal(t, resp.AckFloor.Last, nil)\n\n\t// Fetching should now get the same message.\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_Equal(t, string(msgs[0].Data), \"msg3\")\n\trequire_NoError(t, msgs[0].AckSync())\n}\n\n// https://github.com/nats-io/nats-server/issues/7847\nfunc TestJetStreamConsumerLegacyDurableCreateSetsConsumerName(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tcc := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"CONSUMER\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tReplicas: 1,\n\t\t},\n\t\tAction: ActionCreate,\n\t}\n\treq, err := json.Marshal(cc)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(\"$JS.API.CONSUMER.CREATE.TEST.CONSUMER\", req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerCreateResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_True(t, resp.Error == nil)\n\trequire_Equal(t, resp.Config.Durable, \"CONSUMER\")\n\trequire_Equal(t, resp.Config.Name, \"CONSUMER\")\n\n\tmsg, err = nc.Request(\"$JS.API.CONSUMER.DURABLE.CREATE.TEST.CONSUMER\", req, time.Second)\n\trequire_NoError(t, err)\n\tresp = JSApiConsumerCreateResponse{}\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\trequire_True(t, resp.Error == nil)\n\trequire_Equal(t, resp.Config.Durable, \"CONSUMER\")\n\trequire_Equal(t, resp.Config.Name, \"CONSUMER\")\n}\n\n// https://github.com/nats-io/nats-server/issues/7852\nfunc TestJetStreamConsumerSingleFilterSubjectInFilterSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tFilterSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\to.mu.RLock()\n\tdefer o.mu.RUnlock()\n\n\t// Should not initialize the sublist, as that will make us use LoadNextMsgMulti versus LoadNextMsg.\n\trequire_Len(t, len(o.subjf), 1)\n\trequire_True(t, o.filters == nil)\n}\n\nfunc TestJetStreamConsumerReconcileConsumerAfterStreamDataLoss(t *testing.T) {\n\ttest := func(t *testing.T, totalMsgs int) {\n\t\tstoreDir := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {store_dir: %q}\n\t`, storeDir)))\n\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Publish a few messages.\n\t\tfor range totalMsgs {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: \"DURABLE\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\t// Consume all available messages.\n\t\tmsgs, err := sub.Fetch(totalMsgs, nats.MaxWait(200*time.Millisecond))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), totalMsgs)\n\t\tfor _, msg := range msgs {\n\t\t\trequire_NoError(t, msg.AckSync())\n\t\t}\n\n\t\t// Confirm the consumer info reports all messages as delivered and acked.\n\t\tlseq := uint64(totalMsgs)\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ci.NumPending, 0)\n\t\trequire_Equal(t, ci.NumAckPending, 0)\n\t\trequire_Equal(t, ci.Delivered.Stream, lseq)\n\t\trequire_Equal(t, ci.AckFloor.Stream, lseq)\n\t\trequire_Equal(t, ci.Delivered.Consumer, lseq)\n\n\t\t// Shut down the server and manually remove or truncate the message blocks, simulating data loss.\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tfs := mset.store.(*fileStore)\n\t\tblk := filepath.Join(fs.fcfg.StoreDir, msgDir, \"1.blk\")\n\t\tindex := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\tnc.Close()\n\t\ts.Shutdown()\n\t\tif totalMsgs > 1 {\n\t\t\tstat, err := os.Stat(blk)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, os.Truncate(blk, stat.Size()/2+1))\n\t\t} else {\n\t\t\trequire_NoError(t, os.Remove(blk))\n\t\t}\n\t\trequire_NoError(t, os.Remove(index))\n\n\t\t// Restart the server and reconnect.\n\t\ts, _ = RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\t\tnc, js = jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t// Publish another message. Due to the simulated data loss, the stream sequence should continue\n\t\t// counting after truncating the corrupted data.\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, lseq)\n\n\t\tsub, err = js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Drain()\n\n\t\t// The consumer should be able to consume above message.\n\t\t// Previously the consumer state would not be reconciled and would not be able to consume the message.\n\t\tmsgs, err = sub.Fetch(1, nats.MaxWait(200*time.Millisecond))\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\tmsg := msgs[0]\n\t\tmeta, err := msg.Metadata()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, meta.Sequence.Stream, lseq)\n\t\trequire_NoError(t, msg.AckSync())\n\n\t\t// Confirm the consumer info reports all messages as delivered and acked.\n\t\t// But the delivered sequence shouldn't be reset and still move monotonically.\n\t\tci, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ci.NumPending, 0)\n\t\trequire_Equal(t, ci.NumAckPending, 0)\n\t\trequire_Equal(t, ci.Delivered.Stream, lseq)\n\t\trequire_Equal(t, ci.AckFloor.Stream, lseq)\n\t\trequire_Equal(t, ci.Delivered.Consumer, lseq+1)\n\t}\n\n\tfor _, totalMsgs := range []int{1, 2} {\n\t\tt.Run(fmt.Sprint(totalMsgs), func(t *testing.T) { test(t, totalMsgs) })\n\t}\n}\n" }, { "path": "server/jetstream_errors.go", "content": "package server\n\nimport (\n\t\"fmt\"\n)\n\ntype errOpts struct {\n\terr error\n}\n\n// ErrorOption configures a NATS Error helper\ntype ErrorOption func(*errOpts)\n\n// Unless ensures that if err is a ApiErr that err will be returned rather than the one being created via the helper\nfunc Unless(err error) ErrorOption {\n\treturn func(opts *errOpts) {\n\t\topts.err = err\n\t}\n}\n\nfunc parseOpts(opts []ErrorOption) *errOpts {\n\teopts := &errOpts{}\n\tfor _, opt := range opts {\n\t\topt(eopts)\n\t}\n\treturn eopts\n}\n\ntype ErrorIdentifier uint16\n\n// IsNatsErr determines if an error matches ID, if multiple IDs are given if the error matches any of these the function will be true\nfunc IsNatsErr(err error, ids ...ErrorIdentifier) bool {\n\tif err == nil {\n\t\treturn false\n\t}\n\n\tce, ok := err.(*ApiError)\n\tif !ok || ce == nil {\n\t\treturn false\n\t}\n\n\tfor _, id := range ids {\n\t\tae, ok := ApiErrors[id]\n\t\tif !ok || ae == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif ce.ErrCode == ae.ErrCode {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// ApiError is included in all responses if there was an error.\ntype ApiError struct {\n\tCode int `json:\"code\"`\n\tErrCode uint16 `json:\"err_code,omitempty\"`\n\tDescription string `json:\"description,omitempty\"`\n}\n\n// ErrorsData is the source data for generated errors as found in errors.json\ntype ErrorsData struct {\n\tConstant string `json:\"constant\"`\n\tCode int `json:\"code\"`\n\tErrCode uint16 `json:\"error_code\"`\n\tDescription string `json:\"description\"`\n\tComment string `json:\"comment\"`\n\tHelp string `json:\"help\"`\n\tURL string `json:\"url\"`\n\tDeprecates string `json:\"deprecates\"`\n}\n\nfunc (e *ApiError) Error() string {\n\treturn fmt.Sprintf(\"%s (%d)\", e.Description, e.ErrCode)\n}\n\nfunc (e *ApiError) toReplacerArgs(replacements []any) []string {\n\tvar (\n\t\tra []string\n\t\tkey string\n\t)\n\n\tfor i, replacement := range replacements {\n\t\tif i%2 == 0 {\n\t\t\tkey = replacement.(string)\n\t\t\tcontinue\n\t\t}\n\n\t\tswitch v := replacement.(type) {\n\t\tcase string:\n\t\t\tra = append(ra, key, v)\n\t\tcase error:\n\t\t\tra = append(ra, key, v.Error())\n\t\tdefault:\n\t\t\tra = append(ra, key, fmt.Sprintf(\"%v\", v))\n\t\t}\n\t}\n\n\treturn ra\n}\n" }, { "path": "server/jetstream_errors_generated.go", "content": "// Generated code, do not edit. See errors.json and run go generate to update\n\npackage server\n\nimport \"strings\"\n\nconst (\n\t// JSAccountResourcesExceededErr resource limits exceeded for account\n\tJSAccountResourcesExceededErr ErrorIdentifier = 10002\n\n\t// JSAtomicPublishContainsDuplicateMessageErr atomic publish batch contains duplicate message id\n\tJSAtomicPublishContainsDuplicateMessageErr ErrorIdentifier = 10201\n\n\t// JSAtomicPublishDisabledErr atomic publish is disabled\n\tJSAtomicPublishDisabledErr ErrorIdentifier = 10174\n\n\t// JSAtomicPublishIncompleteBatchErr atomic publish batch is incomplete\n\tJSAtomicPublishIncompleteBatchErr ErrorIdentifier = 10176\n\n\t// JSAtomicPublishInvalidBatchCommitErr atomic publish batch commit is invalid\n\tJSAtomicPublishInvalidBatchCommitErr ErrorIdentifier = 10200\n\n\t// JSAtomicPublishInvalidBatchIDErr atomic publish batch ID is invalid\n\tJSAtomicPublishInvalidBatchIDErr ErrorIdentifier = 10179\n\n\t// JSAtomicPublishMissingSeqErr atomic publish sequence is missing\n\tJSAtomicPublishMissingSeqErr ErrorIdentifier = 10175\n\n\t// JSAtomicPublishTooLargeBatchErrF atomic publish batch is too large: {size}\n\tJSAtomicPublishTooLargeBatchErrF ErrorIdentifier = 10199\n\n\t// JSAtomicPublishTooManyInflight atomic publish too many inflight\n\tJSAtomicPublishTooManyInflight ErrorIdentifier = 10210\n\n\t// JSAtomicPublishUnsupportedHeaderBatchErr atomic publish unsupported header used: {header}\n\tJSAtomicPublishUnsupportedHeaderBatchErr ErrorIdentifier = 10177\n\n\t// JSBadRequestErr bad request\n\tJSBadRequestErr ErrorIdentifier = 10003\n\n\t// JSBatchPublishDisabledErr batch publish is disabled\n\tJSBatchPublishDisabledErr ErrorIdentifier = 10205\n\n\t// JSBatchPublishInvalidBatchIDErr batch publish ID is invalid\n\tJSBatchPublishInvalidBatchIDErr ErrorIdentifier = 10207\n\n\t// JSBatchPublishInvalidPatternErr batch publish pattern is invalid\n\tJSBatchPublishInvalidPatternErr ErrorIdentifier = 10206\n\n\t// JSBatchPublishTooManyInflight batch publish too many inflight\n\tJSBatchPublishTooManyInflight ErrorIdentifier = 10211\n\n\t// JSBatchPublishUnknownBatchIDErr batch publish ID unknown\n\tJSBatchPublishUnknownBatchIDErr ErrorIdentifier = 10208\n\n\t// JSClusterIncompleteErr incomplete results\n\tJSClusterIncompleteErr ErrorIdentifier = 10004\n\n\t// JSClusterNoPeersErrF Error causing no peers to be available ({err})\n\tJSClusterNoPeersErrF ErrorIdentifier = 10005\n\n\t// JSClusterNotActiveErr JetStream not in clustered mode\n\tJSClusterNotActiveErr ErrorIdentifier = 10006\n\n\t// JSClusterNotAssignedErr JetStream cluster not assigned to this server\n\tJSClusterNotAssignedErr ErrorIdentifier = 10007\n\n\t// JSClusterNotAvailErr JetStream system temporarily unavailable\n\tJSClusterNotAvailErr ErrorIdentifier = 10008\n\n\t// JSClusterNotLeaderErr JetStream cluster can not handle request\n\tJSClusterNotLeaderErr ErrorIdentifier = 10009\n\n\t// JSClusterPeerNotMemberErr peer not a member\n\tJSClusterPeerNotMemberErr ErrorIdentifier = 10040\n\n\t// JSClusterRequiredErr JetStream clustering support required\n\tJSClusterRequiredErr ErrorIdentifier = 10010\n\n\t// JSClusterServerMemberChangeInflightErr cluster member change is in progress\n\tJSClusterServerMemberChangeInflightErr ErrorIdentifier = 10202\n\n\t// JSClusterServerNotMemberErr server is not a member of the cluster\n\tJSClusterServerNotMemberErr ErrorIdentifier = 10044\n\n\t// JSClusterTagsErr tags placement not supported for operation\n\tJSClusterTagsErr ErrorIdentifier = 10011\n\n\t// JSClusterUnSupportFeatureErr not currently supported in clustered mode\n\tJSClusterUnSupportFeatureErr ErrorIdentifier = 10036\n\n\t// JSConsumerAckPolicyInvalidErr consumer ack policy invalid\n\tJSConsumerAckPolicyInvalidErr ErrorIdentifier = 10181\n\n\t// JSConsumerAckWaitNegativeErr consumer ack wait needs to be positive\n\tJSConsumerAckWaitNegativeErr ErrorIdentifier = 10183\n\n\t// JSConsumerAlreadyExists action CREATE is used for a existing consumer with a different config (consumer already exists)\n\tJSConsumerAlreadyExists ErrorIdentifier = 10148\n\n\t// JSConsumerBackOffNegativeErr consumer backoff needs to be positive\n\tJSConsumerBackOffNegativeErr ErrorIdentifier = 10184\n\n\t// JSConsumerBadDurableNameErr durable name can not contain '.', '*', '>'\n\tJSConsumerBadDurableNameErr ErrorIdentifier = 10103\n\n\t// JSConsumerConfigRequiredErr consumer config required\n\tJSConsumerConfigRequiredErr ErrorIdentifier = 10078\n\n\t// JSConsumerCreateDurableAndNameMismatch Consumer Durable and Name have to be equal if both are provided\n\tJSConsumerCreateDurableAndNameMismatch ErrorIdentifier = 10132\n\n\t// JSConsumerCreateErrF General consumer creation failure string ({err})\n\tJSConsumerCreateErrF ErrorIdentifier = 10012\n\n\t// JSConsumerCreateFilterSubjectMismatchErr Consumer create request did not match filtered subject from create subject\n\tJSConsumerCreateFilterSubjectMismatchErr ErrorIdentifier = 10131\n\n\t// JSConsumerDeliverCycleErr consumer deliver subject forms a cycle\n\tJSConsumerDeliverCycleErr ErrorIdentifier = 10081\n\n\t// JSConsumerDeliverToWildcardsErr consumer deliver subject has wildcards\n\tJSConsumerDeliverToWildcardsErr ErrorIdentifier = 10079\n\n\t// JSConsumerDescriptionTooLongErrF consumer description is too long, maximum allowed is {max}\n\tJSConsumerDescriptionTooLongErrF ErrorIdentifier = 10107\n\n\t// JSConsumerDirectRequiresEphemeralErr consumer direct requires an ephemeral consumer\n\tJSConsumerDirectRequiresEphemeralErr ErrorIdentifier = 10091\n\n\t// JSConsumerDirectRequiresPushErr consumer direct requires a push based consumer\n\tJSConsumerDirectRequiresPushErr ErrorIdentifier = 10090\n\n\t// JSConsumerDoesNotExist action UPDATE is used for a nonexisting consumer (consumer does not exist)\n\tJSConsumerDoesNotExist ErrorIdentifier = 10149\n\n\t// JSConsumerDuplicateFilterSubjects consumer cannot have both FilterSubject and FilterSubjects specified\n\tJSConsumerDuplicateFilterSubjects ErrorIdentifier = 10136\n\n\t// JSConsumerDurableNameNotInSubjectErr consumer expected to be durable but no durable name set in subject\n\tJSConsumerDurableNameNotInSubjectErr ErrorIdentifier = 10016\n\n\t// JSConsumerDurableNameNotMatchSubjectErr consumer name in subject does not match durable name in request\n\tJSConsumerDurableNameNotMatchSubjectErr ErrorIdentifier = 10017\n\n\t// JSConsumerDurableNameNotSetErr consumer expected to be durable but a durable name was not set\n\tJSConsumerDurableNameNotSetErr ErrorIdentifier = 10018\n\n\t// JSConsumerEmptyFilter consumer filter in FilterSubjects cannot be empty\n\tJSConsumerEmptyFilter ErrorIdentifier = 10139\n\n\t// JSConsumerEmptyGroupName Group name cannot be an empty string\n\tJSConsumerEmptyGroupName ErrorIdentifier = 10161\n\n\t// JSConsumerEphemeralWithDurableInSubjectErr consumer expected to be ephemeral but detected a durable name set in subject\n\tJSConsumerEphemeralWithDurableInSubjectErr ErrorIdentifier = 10019\n\n\t// JSConsumerEphemeralWithDurableNameErr consumer expected to be ephemeral but a durable name was set in request\n\tJSConsumerEphemeralWithDurableNameErr ErrorIdentifier = 10020\n\n\t// JSConsumerExistingActiveErr consumer already exists and is still active\n\tJSConsumerExistingActiveErr ErrorIdentifier = 10105\n\n\t// JSConsumerFCRequiresPushErr consumer flow control requires a push based consumer\n\tJSConsumerFCRequiresPushErr ErrorIdentifier = 10089\n\n\t// JSConsumerFilterNotSubsetErr consumer filter subject is not a valid subset of the interest subjects\n\tJSConsumerFilterNotSubsetErr ErrorIdentifier = 10093\n\n\t// JSConsumerHBRequiresPushErr consumer idle heartbeat requires a push based consumer\n\tJSConsumerHBRequiresPushErr ErrorIdentifier = 10088\n\n\t// JSConsumerInactiveThresholdExcess consumer inactive threshold exceeds system limit of {limit}\n\tJSConsumerInactiveThresholdExcess ErrorIdentifier = 10153\n\n\t// JSConsumerInvalidDeliverSubject invalid push consumer deliver subject\n\tJSConsumerInvalidDeliverSubject ErrorIdentifier = 10112\n\n\t// JSConsumerInvalidGroupNameErr Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters\n\tJSConsumerInvalidGroupNameErr ErrorIdentifier = 10162\n\n\t// JSConsumerInvalidPolicyErrF Generic delivery policy error ({err})\n\tJSConsumerInvalidPolicyErrF ErrorIdentifier = 10094\n\n\t// JSConsumerInvalidPriorityGroupErr Provided priority group does not exist for this consumer\n\tJSConsumerInvalidPriorityGroupErr ErrorIdentifier = 10160\n\n\t// JSConsumerInvalidResetErr invalid reset: {err}\n\tJSConsumerInvalidResetErr ErrorIdentifier = 10204\n\n\t// JSConsumerInvalidSamplingErrF failed to parse consumer sampling configuration: {err}\n\tJSConsumerInvalidSamplingErrF ErrorIdentifier = 10095\n\n\t// JSConsumerMaxDeliverBackoffErr max deliver is required to be > length of backoff values\n\tJSConsumerMaxDeliverBackoffErr ErrorIdentifier = 10116\n\n\t// JSConsumerMaxPendingAckExcessErrF consumer max ack pending exceeds system limit of {limit}\n\tJSConsumerMaxPendingAckExcessErrF ErrorIdentifier = 10121\n\n\t// JSConsumerMaxPendingAckPolicyRequiredErr consumer requires ack policy for max ack pending\n\tJSConsumerMaxPendingAckPolicyRequiredErr ErrorIdentifier = 10082\n\n\t// JSConsumerMaxRequestBatchExceededF consumer max request batch exceeds server limit of {limit}\n\tJSConsumerMaxRequestBatchExceededF ErrorIdentifier = 10125\n\n\t// JSConsumerMaxRequestBatchNegativeErr consumer max request batch needs to be > 0\n\tJSConsumerMaxRequestBatchNegativeErr ErrorIdentifier = 10114\n\n\t// JSConsumerMaxRequestExpiresTooSmall consumer max request expires needs to be >= 1ms\n\tJSConsumerMaxRequestExpiresTooSmall ErrorIdentifier = 10115\n\n\t// JSConsumerMaxWaitingNegativeErr consumer max waiting needs to be positive\n\tJSConsumerMaxWaitingNegativeErr ErrorIdentifier = 10087\n\n\t// JSConsumerMetadataLengthErrF consumer metadata exceeds maximum size of {limit}\n\tJSConsumerMetadataLengthErrF ErrorIdentifier = 10135\n\n\t// JSConsumerMultipleFiltersNotAllowed consumer with multiple subject filters cannot use subject based API\n\tJSConsumerMultipleFiltersNotAllowed ErrorIdentifier = 10137\n\n\t// JSConsumerNameContainsPathSeparatorsErr Consumer name can not contain path separators\n\tJSConsumerNameContainsPathSeparatorsErr ErrorIdentifier = 10127\n\n\t// JSConsumerNameExistErr consumer name already in use\n\tJSConsumerNameExistErr ErrorIdentifier = 10013\n\n\t// JSConsumerNameTooLongErrF consumer name is too long, maximum allowed is {max}\n\tJSConsumerNameTooLongErrF ErrorIdentifier = 10102\n\n\t// JSConsumerNotFoundErr consumer not found\n\tJSConsumerNotFoundErr ErrorIdentifier = 10014\n\n\t// JSConsumerOfflineErr consumer is offline\n\tJSConsumerOfflineErr ErrorIdentifier = 10119\n\n\t// JSConsumerOfflineReasonErrF consumer is offline: {err}\n\tJSConsumerOfflineReasonErrF ErrorIdentifier = 10195\n\n\t// JSConsumerOnMappedErr consumer direct on a mapped consumer\n\tJSConsumerOnMappedErr ErrorIdentifier = 10092\n\n\t// JSConsumerOverlappingSubjectFilters consumer subject filters cannot overlap\n\tJSConsumerOverlappingSubjectFilters ErrorIdentifier = 10138\n\n\t// JSConsumerPinnedTTLWithoutPriorityPolicyNone PinnedTTL cannot be set when PriorityPolicy is none\n\tJSConsumerPinnedTTLWithoutPriorityPolicyNone ErrorIdentifier = 10197\n\n\t// JSConsumerPriorityGroupWithPolicyNone consumer can not have priority groups when policy is none\n\tJSConsumerPriorityGroupWithPolicyNone ErrorIdentifier = 10196\n\n\t// JSConsumerPriorityPolicyWithoutGroup Setting PriorityPolicy requires at least one PriorityGroup to be set\n\tJSConsumerPriorityPolicyWithoutGroup ErrorIdentifier = 10159\n\n\t// JSConsumerPullNotDurableErr consumer in pull mode requires a durable name\n\tJSConsumerPullNotDurableErr ErrorIdentifier = 10085\n\n\t// JSConsumerPullRequiresAckErr consumer in pull mode requires explicit ack policy on workqueue stream\n\tJSConsumerPullRequiresAckErr ErrorIdentifier = 10084\n\n\t// JSConsumerPullWithRateLimitErr consumer in pull mode can not have rate limit set\n\tJSConsumerPullWithRateLimitErr ErrorIdentifier = 10086\n\n\t// JSConsumerPushMaxWaitingErr consumer in push mode can not set max waiting\n\tJSConsumerPushMaxWaitingErr ErrorIdentifier = 10080\n\n\t// JSConsumerPushWithPriorityGroupErr priority groups can not be used with push consumers\n\tJSConsumerPushWithPriorityGroupErr ErrorIdentifier = 10178\n\n\t// JSConsumerReplacementWithDifferentNameErr consumer replacement durable config not the same\n\tJSConsumerReplacementWithDifferentNameErr ErrorIdentifier = 10106\n\n\t// JSConsumerReplayPolicyInvalidErr consumer replay policy invalid\n\tJSConsumerReplayPolicyInvalidErr ErrorIdentifier = 10182\n\n\t// JSConsumerReplicasExceedsStream consumer config replica count exceeds parent stream\n\tJSConsumerReplicasExceedsStream ErrorIdentifier = 10126\n\n\t// JSConsumerReplicasShouldMatchStream consumer config replicas must match interest retention stream's replicas\n\tJSConsumerReplicasShouldMatchStream ErrorIdentifier = 10134\n\n\t// JSConsumerSmallHeartbeatErr consumer idle heartbeat needs to be >= 100ms\n\tJSConsumerSmallHeartbeatErr ErrorIdentifier = 10083\n\n\t// JSConsumerStoreFailedErrF error creating store for consumer: {err}\n\tJSConsumerStoreFailedErrF ErrorIdentifier = 10104\n\n\t// JSConsumerWQConsumerNotDeliverAllErr consumer must be deliver all on workqueue stream\n\tJSConsumerWQConsumerNotDeliverAllErr ErrorIdentifier = 10101\n\n\t// JSConsumerWQConsumerNotUniqueErr filtered consumer not unique on workqueue stream\n\tJSConsumerWQConsumerNotUniqueErr ErrorIdentifier = 10100\n\n\t// JSConsumerWQMultipleUnfilteredErr multiple non-filtered consumers not allowed on workqueue stream\n\tJSConsumerWQMultipleUnfilteredErr ErrorIdentifier = 10099\n\n\t// JSConsumerWQRequiresExplicitAckErr workqueue stream requires explicit ack\n\tJSConsumerWQRequiresExplicitAckErr ErrorIdentifier = 10098\n\n\t// JSConsumerWithFlowControlNeedsHeartbeats consumer with flow control also needs heartbeats\n\tJSConsumerWithFlowControlNeedsHeartbeats ErrorIdentifier = 10108\n\n\t// JSInsufficientResourcesErr insufficient resources\n\tJSInsufficientResourcesErr ErrorIdentifier = 10023\n\n\t// JSInvalidJSONErr invalid JSON: {err}\n\tJSInvalidJSONErr ErrorIdentifier = 10025\n\n\t// JSMaximumConsumersLimitErr maximum consumers limit reached\n\tJSMaximumConsumersLimitErr ErrorIdentifier = 10026\n\n\t// JSMaximumStreamsLimitErr maximum number of streams reached\n\tJSMaximumStreamsLimitErr ErrorIdentifier = 10027\n\n\t// JSMemoryResourcesExceededErr insufficient memory resources available\n\tJSMemoryResourcesExceededErr ErrorIdentifier = 10028\n\n\t// JSMessageCounterBrokenErr message counter is broken\n\tJSMessageCounterBrokenErr ErrorIdentifier = 10172\n\n\t// JSMessageIncrDisabledErr message counters is disabled\n\tJSMessageIncrDisabledErr ErrorIdentifier = 10168\n\n\t// JSMessageIncrInvalidErr message counter increment is invalid\n\tJSMessageIncrInvalidErr ErrorIdentifier = 10171\n\n\t// JSMessageIncrMissingErr message counter increment is missing\n\tJSMessageIncrMissingErr ErrorIdentifier = 10169\n\n\t// JSMessageIncrPayloadErr message counter has payload\n\tJSMessageIncrPayloadErr ErrorIdentifier = 10170\n\n\t// JSMessageSchedulesDisabledErr message schedules is disabled\n\tJSMessageSchedulesDisabledErr ErrorIdentifier = 10188\n\n\t// JSMessageSchedulesPatternInvalidErr message schedules pattern is invalid\n\tJSMessageSchedulesPatternInvalidErr ErrorIdentifier = 10189\n\n\t// JSMessageSchedulesRollupInvalidErr message schedules invalid rollup\n\tJSMessageSchedulesRollupInvalidErr ErrorIdentifier = 10192\n\n\t// JSMessageSchedulesSourceInvalidErr message schedules source is invalid\n\tJSMessageSchedulesSourceInvalidErr ErrorIdentifier = 10203\n\n\t// JSMessageSchedulesTTLInvalidErr message schedules invalid per-message TTL\n\tJSMessageSchedulesTTLInvalidErr ErrorIdentifier = 10191\n\n\t// JSMessageSchedulesTargetInvalidErr message schedules target is invalid\n\tJSMessageSchedulesTargetInvalidErr ErrorIdentifier = 10190\n\n\t// JSMessageTTLDisabledErr per-message TTL is disabled\n\tJSMessageTTLDisabledErr ErrorIdentifier = 10166\n\n\t// JSMessageTTLInvalidErr invalid per-message TTL\n\tJSMessageTTLInvalidErr ErrorIdentifier = 10165\n\n\t// JSMirrorConsumerSetupFailedErrF generic mirror consumer setup failure string ({err})\n\tJSMirrorConsumerSetupFailedErrF ErrorIdentifier = 10029\n\n\t// JSMirrorInvalidStreamName mirrored stream name is invalid\n\tJSMirrorInvalidStreamName ErrorIdentifier = 10142\n\n\t// JSMirrorInvalidSubjectFilter mirror transform source: {err}\n\tJSMirrorInvalidSubjectFilter ErrorIdentifier = 10151\n\n\t// JSMirrorInvalidTransformDestination mirror transform: {err}\n\tJSMirrorInvalidTransformDestination ErrorIdentifier = 10154\n\n\t// JSMirrorMaxMessageSizeTooBigErr stream mirror must have max message size >= source\n\tJSMirrorMaxMessageSizeTooBigErr ErrorIdentifier = 10030\n\n\t// JSMirrorMultipleFiltersNotAllowed mirror with multiple subject transforms cannot also have a single subject filter\n\tJSMirrorMultipleFiltersNotAllowed ErrorIdentifier = 10150\n\n\t// JSMirrorOverlappingSubjectFilters mirror subject filters can not overlap\n\tJSMirrorOverlappingSubjectFilters ErrorIdentifier = 10152\n\n\t// JSMirrorWithAtomicPublishErr stream mirrors can not also use atomic publishing\n\tJSMirrorWithAtomicPublishErr ErrorIdentifier = 10198\n\n\t// JSMirrorWithBatchPublishErr stream mirrors can not also use batch publishing\n\tJSMirrorWithBatchPublishErr ErrorIdentifier = 10209\n\n\t// JSMirrorWithCountersErr stream mirrors can not also calculate counters\n\tJSMirrorWithCountersErr ErrorIdentifier = 10173\n\n\t// JSMirrorWithFirstSeqErr stream mirrors can not have first sequence configured\n\tJSMirrorWithFirstSeqErr ErrorIdentifier = 10143\n\n\t// JSMirrorWithMsgSchedulesErr stream mirrors can not also schedule messages\n\tJSMirrorWithMsgSchedulesErr ErrorIdentifier = 10186\n\n\t// JSMirrorWithSourcesErr stream mirrors can not also contain other sources\n\tJSMirrorWithSourcesErr ErrorIdentifier = 10031\n\n\t// JSMirrorWithStartSeqAndTimeErr stream mirrors can not have both start seq and start time configured\n\tJSMirrorWithStartSeqAndTimeErr ErrorIdentifier = 10032\n\n\t// JSMirrorWithSubjectFiltersErr stream mirrors can not contain filtered subjects\n\tJSMirrorWithSubjectFiltersErr ErrorIdentifier = 10033\n\n\t// JSMirrorWithSubjectsErr stream mirrors can not contain subjects\n\tJSMirrorWithSubjectsErr ErrorIdentifier = 10034\n\n\t// JSNoAccountErr account not found\n\tJSNoAccountErr ErrorIdentifier = 10035\n\n\t// JSNoLimitsErr no JetStream default or applicable tiered limit present\n\tJSNoLimitsErr ErrorIdentifier = 10120\n\n\t// JSNoMessageFoundErr no message found\n\tJSNoMessageFoundErr ErrorIdentifier = 10037\n\n\t// JSNotEmptyRequestErr expected an empty request payload\n\tJSNotEmptyRequestErr ErrorIdentifier = 10038\n\n\t// JSNotEnabledErr JetStream not enabled\n\tJSNotEnabledErr ErrorIdentifier = 10076\n\n\t// JSNotEnabledForAccountErr JetStream not enabled for account\n\tJSNotEnabledForAccountErr ErrorIdentifier = 10039\n\n\t// JSPedanticErrF pedantic mode: {err}\n\tJSPedanticErrF ErrorIdentifier = 10157\n\n\t// JSPeerRemapErr peer remap failed\n\tJSPeerRemapErr ErrorIdentifier = 10075\n\n\t// JSRaftGeneralErrF General RAFT error string ({err})\n\tJSRaftGeneralErrF ErrorIdentifier = 10041\n\n\t// JSReplicasCountCannotBeNegative replicas count cannot be negative\n\tJSReplicasCountCannotBeNegative ErrorIdentifier = 10133\n\n\t// JSRequiredApiLevelErr JetStream minimum api level required\n\tJSRequiredApiLevelErr ErrorIdentifier = 10185\n\n\t// JSRestoreSubscribeFailedErrF JetStream unable to subscribe to restore snapshot {subject}: {err}\n\tJSRestoreSubscribeFailedErrF ErrorIdentifier = 10042\n\n\t// JSSequenceNotFoundErrF sequence {seq} not found\n\tJSSequenceNotFoundErrF ErrorIdentifier = 10043\n\n\t// JSSnapshotDeliverSubjectInvalidErr deliver subject not valid\n\tJSSnapshotDeliverSubjectInvalidErr ErrorIdentifier = 10015\n\n\t// JSSourceConsumerSetupFailedErrF General source consumer setup failure string ({err})\n\tJSSourceConsumerSetupFailedErrF ErrorIdentifier = 10045\n\n\t// JSSourceDuplicateDetected source stream, filter and transform (plus external if present) must form a unique combination (duplicate source configuration detected)\n\tJSSourceDuplicateDetected ErrorIdentifier = 10140\n\n\t// JSSourceInvalidStreamName sourced stream name is invalid\n\tJSSourceInvalidStreamName ErrorIdentifier = 10141\n\n\t// JSSourceInvalidSubjectFilter source transform source: {err}\n\tJSSourceInvalidSubjectFilter ErrorIdentifier = 10145\n\n\t// JSSourceInvalidTransformDestination source transform: {err}\n\tJSSourceInvalidTransformDestination ErrorIdentifier = 10146\n\n\t// JSSourceMaxMessageSizeTooBigErr stream source must have max message size >= target\n\tJSSourceMaxMessageSizeTooBigErr ErrorIdentifier = 10046\n\n\t// JSSourceMultipleFiltersNotAllowed source with multiple subject transforms cannot also have a single subject filter\n\tJSSourceMultipleFiltersNotAllowed ErrorIdentifier = 10144\n\n\t// JSSourceOverlappingSubjectFilters source filters can not overlap\n\tJSSourceOverlappingSubjectFilters ErrorIdentifier = 10147\n\n\t// JSSourceWithMsgSchedulesErr stream source can not also schedule messages\n\tJSSourceWithMsgSchedulesErr ErrorIdentifier = 10187\n\n\t// JSStorageResourcesExceededErr insufficient storage resources available\n\tJSStorageResourcesExceededErr ErrorIdentifier = 10047\n\n\t// JSStreamAssignmentErrF Generic stream assignment error string ({err})\n\tJSStreamAssignmentErrF ErrorIdentifier = 10048\n\n\t// JSStreamCreateErrF Generic stream creation error string ({err})\n\tJSStreamCreateErrF ErrorIdentifier = 10049\n\n\t// JSStreamDeleteErrF General stream deletion error string ({err})\n\tJSStreamDeleteErrF ErrorIdentifier = 10050\n\n\t// JSStreamDuplicateMessageConflict duplicate message id is in process\n\tJSStreamDuplicateMessageConflict ErrorIdentifier = 10158\n\n\t// JSStreamExpectedLastSeqPerSubjectInvalid missing sequence for expected last sequence per subject\n\tJSStreamExpectedLastSeqPerSubjectInvalid ErrorIdentifier = 10193\n\n\t// JSStreamExpectedLastSeqPerSubjectNotReady expected last sequence per subject temporarily unavailable\n\tJSStreamExpectedLastSeqPerSubjectNotReady ErrorIdentifier = 10163\n\n\t// JSStreamExternalApiOverlapErrF stream external api prefix {prefix} must not overlap with {subject}\n\tJSStreamExternalApiOverlapErrF ErrorIdentifier = 10021\n\n\t// JSStreamExternalDelPrefixOverlapsErrF stream external delivery prefix {prefix} overlaps with stream subject {subject}\n\tJSStreamExternalDelPrefixOverlapsErrF ErrorIdentifier = 10022\n\n\t// JSStreamGeneralErrorF General stream failure string ({err})\n\tJSStreamGeneralErrorF ErrorIdentifier = 10051\n\n\t// JSStreamHeaderExceedsMaximumErr header size exceeds maximum allowed of 64k\n\tJSStreamHeaderExceedsMaximumErr ErrorIdentifier = 10097\n\n\t// JSStreamInfoMaxSubjectsErr subject details would exceed maximum allowed\n\tJSStreamInfoMaxSubjectsErr ErrorIdentifier = 10117\n\n\t// JSStreamInvalidConfigF Stream configuration validation error string ({err})\n\tJSStreamInvalidConfigF ErrorIdentifier = 10052\n\n\t// JSStreamInvalidErr stream not valid\n\tJSStreamInvalidErr ErrorIdentifier = 10096\n\n\t// JSStreamInvalidExternalDeliverySubjErrF stream external delivery prefix {prefix} must not contain wildcards\n\tJSStreamInvalidExternalDeliverySubjErrF ErrorIdentifier = 10024\n\n\t// JSStreamLimitsErrF General stream limits exceeded error string ({err})\n\tJSStreamLimitsErrF ErrorIdentifier = 10053\n\n\t// JSStreamMaxBytesRequired account requires a stream config to have max bytes set\n\tJSStreamMaxBytesRequired ErrorIdentifier = 10113\n\n\t// JSStreamMaxStreamBytesExceeded stream max bytes exceeds account limit max stream bytes\n\tJSStreamMaxStreamBytesExceeded ErrorIdentifier = 10122\n\n\t// JSStreamMessageExceedsMaximumErr message size exceeds maximum allowed\n\tJSStreamMessageExceedsMaximumErr ErrorIdentifier = 10054\n\n\t// JSStreamMinLastSeqErr min last sequence\n\tJSStreamMinLastSeqErr ErrorIdentifier = 10180\n\n\t// JSStreamMirrorNotUpdatableErr stream mirror configuration can not be updated\n\tJSStreamMirrorNotUpdatableErr ErrorIdentifier = 10055\n\n\t// JSStreamMismatchErr stream name in subject does not match request\n\tJSStreamMismatchErr ErrorIdentifier = 10056\n\n\t// JSStreamMoveAndScaleErr can not move and scale a stream in a single update\n\tJSStreamMoveAndScaleErr ErrorIdentifier = 10123\n\n\t// JSStreamMoveInProgressF stream move already in progress: {msg}\n\tJSStreamMoveInProgressF ErrorIdentifier = 10124\n\n\t// JSStreamMoveNotInProgress stream move not in progress\n\tJSStreamMoveNotInProgress ErrorIdentifier = 10129\n\n\t// JSStreamMsgDeleteFailedF Generic message deletion failure error string ({err})\n\tJSStreamMsgDeleteFailedF ErrorIdentifier = 10057\n\n\t// JSStreamNameContainsPathSeparatorsErr Stream name can not contain path separators\n\tJSStreamNameContainsPathSeparatorsErr ErrorIdentifier = 10128\n\n\t// JSStreamNameExistErr stream name already in use with a different configuration\n\tJSStreamNameExistErr ErrorIdentifier = 10058\n\n\t// JSStreamNameExistRestoreFailedErr stream name already in use, cannot restore\n\tJSStreamNameExistRestoreFailedErr ErrorIdentifier = 10130\n\n\t// JSStreamNotFoundErr stream not found\n\tJSStreamNotFoundErr ErrorIdentifier = 10059\n\n\t// JSStreamNotMatchErr expected stream does not match\n\tJSStreamNotMatchErr ErrorIdentifier = 10060\n\n\t// JSStreamOfflineErr stream is offline\n\tJSStreamOfflineErr ErrorIdentifier = 10118\n\n\t// JSStreamOfflineReasonErrF stream is offline: {err}\n\tJSStreamOfflineReasonErrF ErrorIdentifier = 10194\n\n\t// JSStreamPurgeFailedF Generic stream purge failure error string ({err})\n\tJSStreamPurgeFailedF ErrorIdentifier = 10110\n\n\t// JSStreamReplicasNotSupportedErr replicas > 1 not supported in non-clustered mode\n\tJSStreamReplicasNotSupportedErr ErrorIdentifier = 10074\n\n\t// JSStreamReplicasNotUpdatableErr Replicas configuration can not be updated\n\tJSStreamReplicasNotUpdatableErr ErrorIdentifier = 10061\n\n\t// JSStreamRestoreErrF restore failed: {err}\n\tJSStreamRestoreErrF ErrorIdentifier = 10062\n\n\t// JSStreamRollupFailedF Generic stream rollup failure error string ({err})\n\tJSStreamRollupFailedF ErrorIdentifier = 10111\n\n\t// JSStreamSealedErr invalid operation on sealed stream\n\tJSStreamSealedErr ErrorIdentifier = 10109\n\n\t// JSStreamSequenceNotMatchErr expected stream sequence does not match\n\tJSStreamSequenceNotMatchErr ErrorIdentifier = 10063\n\n\t// JSStreamSnapshotErrF snapshot failed: {err}\n\tJSStreamSnapshotErrF ErrorIdentifier = 10064\n\n\t// JSStreamStoreFailedF Generic error when storing a message failed ({err})\n\tJSStreamStoreFailedF ErrorIdentifier = 10077\n\n\t// JSStreamSubjectOverlapErr subjects overlap with an existing stream\n\tJSStreamSubjectOverlapErr ErrorIdentifier = 10065\n\n\t// JSStreamTemplateCreateErrF Generic template creation failed string ({err})\n\tJSStreamTemplateCreateErrF ErrorIdentifier = 10066\n\n\t// JSStreamTemplateDeleteErrF Generic stream template deletion failed error string ({err})\n\tJSStreamTemplateDeleteErrF ErrorIdentifier = 10067\n\n\t// JSStreamTemplateNotFoundErr template not found\n\tJSStreamTemplateNotFoundErr ErrorIdentifier = 10068\n\n\t// JSStreamTooManyRequests too many requests\n\tJSStreamTooManyRequests ErrorIdentifier = 10167\n\n\t// JSStreamTransformInvalidDestination stream transform: {err}\n\tJSStreamTransformInvalidDestination ErrorIdentifier = 10156\n\n\t// JSStreamTransformInvalidSource stream transform source: {err}\n\tJSStreamTransformInvalidSource ErrorIdentifier = 10155\n\n\t// JSStreamUpdateErrF Generic stream update error string ({err})\n\tJSStreamUpdateErrF ErrorIdentifier = 10069\n\n\t// JSStreamWrongLastMsgIDErrF wrong last msg ID: {id}\n\tJSStreamWrongLastMsgIDErrF ErrorIdentifier = 10070\n\n\t// JSStreamWrongLastSequenceConstantErr wrong last sequence\n\tJSStreamWrongLastSequenceConstantErr ErrorIdentifier = 10164\n\n\t// JSStreamWrongLastSequenceErrF wrong last sequence: {seq}\n\tJSStreamWrongLastSequenceErrF ErrorIdentifier = 10071\n\n\t// JSTempStorageFailedErr JetStream unable to open temp storage for restore\n\tJSTempStorageFailedErr ErrorIdentifier = 10072\n\n\t// JSTemplateNameNotMatchSubjectErr template name in subject does not match request\n\tJSTemplateNameNotMatchSubjectErr ErrorIdentifier = 10073\n)\n\nvar (\n\tApiErrors = map[ErrorIdentifier]*ApiError{\n\t\tJSAccountResourcesExceededErr: {Code: 400, ErrCode: 10002, Description: \"resource limits exceeded for account\"},\n\t\tJSAtomicPublishContainsDuplicateMessageErr: {Code: 400, ErrCode: 10201, Description: \"atomic publish batch contains duplicate message id\"},\n\t\tJSAtomicPublishDisabledErr: {Code: 400, ErrCode: 10174, Description: \"atomic publish is disabled\"},\n\t\tJSAtomicPublishIncompleteBatchErr: {Code: 400, ErrCode: 10176, Description: \"atomic publish batch is incomplete\"},\n\t\tJSAtomicPublishInvalidBatchCommitErr: {Code: 400, ErrCode: 10200, Description: \"atomic publish batch commit is invalid\"},\n\t\tJSAtomicPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10179, Description: \"atomic publish batch ID is invalid\"},\n\t\tJSAtomicPublishMissingSeqErr: {Code: 400, ErrCode: 10175, Description: \"atomic publish sequence is missing\"},\n\t\tJSAtomicPublishTooLargeBatchErrF: {Code: 400, ErrCode: 10199, Description: \"atomic publish batch is too large: {size}\"},\n\t\tJSAtomicPublishTooManyInflight: {Code: 429, ErrCode: 10210, Description: \"atomic publish too many inflight\"},\n\t\tJSAtomicPublishUnsupportedHeaderBatchErr: {Code: 400, ErrCode: 10177, Description: \"atomic publish unsupported header used: {header}\"},\n\t\tJSBadRequestErr: {Code: 400, ErrCode: 10003, Description: \"bad request\"},\n\t\tJSBatchPublishDisabledErr: {Code: 400, ErrCode: 10205, Description: \"batch publish is disabled\"},\n\t\tJSBatchPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10207, Description: \"batch publish ID is invalid\"},\n\t\tJSBatchPublishInvalidPatternErr: {Code: 400, ErrCode: 10206, Description: \"batch publish pattern is invalid\"},\n\t\tJSBatchPublishTooManyInflight: {Code: 429, ErrCode: 10211, Description: \"batch publish too many inflight\"},\n\t\tJSBatchPublishUnknownBatchIDErr: {Code: 400, ErrCode: 10208, Description: \"batch publish ID unknown\"},\n\t\tJSClusterIncompleteErr: {Code: 503, ErrCode: 10004, Description: \"incomplete results\"},\n\t\tJSClusterNoPeersErrF: {Code: 400, ErrCode: 10005, Description: \"{err}\"},\n\t\tJSClusterNotActiveErr: {Code: 500, ErrCode: 10006, Description: \"JetStream not in clustered mode\"},\n\t\tJSClusterNotAssignedErr: {Code: 500, ErrCode: 10007, Description: \"JetStream cluster not assigned to this server\"},\n\t\tJSClusterNotAvailErr: {Code: 503, ErrCode: 10008, Description: \"JetStream system temporarily unavailable\"},\n\t\tJSClusterNotLeaderErr: {Code: 500, ErrCode: 10009, Description: \"JetStream cluster can not handle request\"},\n\t\tJSClusterPeerNotMemberErr: {Code: 400, ErrCode: 10040, Description: \"peer not a member\"},\n\t\tJSClusterRequiredErr: {Code: 503, ErrCode: 10010, Description: \"JetStream clustering support required\"},\n\t\tJSClusterServerMemberChangeInflightErr: {Code: 400, ErrCode: 10202, Description: \"cluster member change is in progress\"},\n\t\tJSClusterServerNotMemberErr: {Code: 400, ErrCode: 10044, Description: \"server is not a member of the cluster\"},\n\t\tJSClusterTagsErr: {Code: 400, ErrCode: 10011, Description: \"tags placement not supported for operation\"},\n\t\tJSClusterUnSupportFeatureErr: {Code: 503, ErrCode: 10036, Description: \"not currently supported in clustered mode\"},\n\t\tJSConsumerAckPolicyInvalidErr: {Code: 400, ErrCode: 10181, Description: \"consumer ack policy invalid\"},\n\t\tJSConsumerAckWaitNegativeErr: {Code: 400, ErrCode: 10183, Description: \"consumer ack wait needs to be positive\"},\n\t\tJSConsumerAlreadyExists: {Code: 400, ErrCode: 10148, Description: \"consumer already exists\"},\n\t\tJSConsumerBackOffNegativeErr: {Code: 400, ErrCode: 10184, Description: \"consumer backoff needs to be positive\"},\n\t\tJSConsumerBadDurableNameErr: {Code: 400, ErrCode: 10103, Description: \"durable name can not contain '.', '*', '>'\"},\n\t\tJSConsumerConfigRequiredErr: {Code: 400, ErrCode: 10078, Description: \"consumer config required\"},\n\t\tJSConsumerCreateDurableAndNameMismatch: {Code: 400, ErrCode: 10132, Description: \"Consumer Durable and Name have to be equal if both are provided\"},\n\t\tJSConsumerCreateErrF: {Code: 500, ErrCode: 10012, Description: \"{err}\"},\n\t\tJSConsumerCreateFilterSubjectMismatchErr: {Code: 400, ErrCode: 10131, Description: \"Consumer create request did not match filtered subject from create subject\"},\n\t\tJSConsumerDeliverCycleErr: {Code: 400, ErrCode: 10081, Description: \"consumer deliver subject forms a cycle\"},\n\t\tJSConsumerDeliverToWildcardsErr: {Code: 400, ErrCode: 10079, Description: \"consumer deliver subject has wildcards\"},\n\t\tJSConsumerDescriptionTooLongErrF: {Code: 400, ErrCode: 10107, Description: \"consumer description is too long, maximum allowed is {max}\"},\n\t\tJSConsumerDirectRequiresEphemeralErr: {Code: 400, ErrCode: 10091, Description: \"consumer direct requires an ephemeral consumer\"},\n\t\tJSConsumerDirectRequiresPushErr: {Code: 400, ErrCode: 10090, Description: \"consumer direct requires a push based consumer\"},\n\t\tJSConsumerDoesNotExist: {Code: 400, ErrCode: 10149, Description: \"consumer does not exist\"},\n\t\tJSConsumerDuplicateFilterSubjects: {Code: 400, ErrCode: 10136, Description: \"consumer cannot have both FilterSubject and FilterSubjects specified\"},\n\t\tJSConsumerDurableNameNotInSubjectErr: {Code: 400, ErrCode: 10016, Description: \"consumer expected to be durable but no durable name set in subject\"},\n\t\tJSConsumerDurableNameNotMatchSubjectErr: {Code: 400, ErrCode: 10017, Description: \"consumer name in subject does not match durable name in request\"},\n\t\tJSConsumerDurableNameNotSetErr: {Code: 400, ErrCode: 10018, Description: \"consumer expected to be durable but a durable name was not set\"},\n\t\tJSConsumerEmptyFilter: {Code: 400, ErrCode: 10139, Description: \"consumer filter in FilterSubjects cannot be empty\"},\n\t\tJSConsumerEmptyGroupName: {Code: 400, ErrCode: 10161, Description: \"Group name cannot be an empty string\"},\n\t\tJSConsumerEphemeralWithDurableInSubjectErr: {Code: 400, ErrCode: 10019, Description: \"consumer expected to be ephemeral but detected a durable name set in subject\"},\n\t\tJSConsumerEphemeralWithDurableNameErr: {Code: 400, ErrCode: 10020, Description: \"consumer expected to be ephemeral but a durable name was set in request\"},\n\t\tJSConsumerExistingActiveErr: {Code: 400, ErrCode: 10105, Description: \"consumer already exists and is still active\"},\n\t\tJSConsumerFCRequiresPushErr: {Code: 400, ErrCode: 10089, Description: \"consumer flow control requires a push based consumer\"},\n\t\tJSConsumerFilterNotSubsetErr: {Code: 400, ErrCode: 10093, Description: \"consumer filter subject is not a valid subset of the interest subjects\"},\n\t\tJSConsumerHBRequiresPushErr: {Code: 400, ErrCode: 10088, Description: \"consumer idle heartbeat requires a push based consumer\"},\n\t\tJSConsumerInactiveThresholdExcess: {Code: 400, ErrCode: 10153, Description: \"consumer inactive threshold exceeds system limit of {limit}\"},\n\t\tJSConsumerInvalidDeliverSubject: {Code: 400, ErrCode: 10112, Description: \"invalid push consumer deliver subject\"},\n\t\tJSConsumerInvalidGroupNameErr: {Code: 400, ErrCode: 10162, Description: \"Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters\"},\n\t\tJSConsumerInvalidPolicyErrF: {Code: 400, ErrCode: 10094, Description: \"{err}\"},\n\t\tJSConsumerInvalidPriorityGroupErr: {Code: 400, ErrCode: 10160, Description: \"Provided priority group does not exist for this consumer\"},\n\t\tJSConsumerInvalidResetErr: {Code: 400, ErrCode: 10204, Description: \"invalid reset: {err}\"},\n\t\tJSConsumerInvalidSamplingErrF: {Code: 400, ErrCode: 10095, Description: \"failed to parse consumer sampling configuration: {err}\"},\n\t\tJSConsumerMaxDeliverBackoffErr: {Code: 400, ErrCode: 10116, Description: \"max deliver is required to be > length of backoff values\"},\n\t\tJSConsumerMaxPendingAckExcessErrF: {Code: 400, ErrCode: 10121, Description: \"consumer max ack pending exceeds system limit of {limit}\"},\n\t\tJSConsumerMaxPendingAckPolicyRequiredErr: {Code: 400, ErrCode: 10082, Description: \"consumer requires ack policy for max ack pending\"},\n\t\tJSConsumerMaxRequestBatchExceededF: {Code: 400, ErrCode: 10125, Description: \"consumer max request batch exceeds server limit of {limit}\"},\n\t\tJSConsumerMaxRequestBatchNegativeErr: {Code: 400, ErrCode: 10114, Description: \"consumer max request batch needs to be > 0\"},\n\t\tJSConsumerMaxRequestExpiresTooSmall: {Code: 400, ErrCode: 10115, Description: \"consumer max request expires needs to be >= 1ms\"},\n\t\tJSConsumerMaxWaitingNegativeErr: {Code: 400, ErrCode: 10087, Description: \"consumer max waiting needs to be positive\"},\n\t\tJSConsumerMetadataLengthErrF: {Code: 400, ErrCode: 10135, Description: \"consumer metadata exceeds maximum size of {limit}\"},\n\t\tJSConsumerMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10137, Description: \"consumer with multiple subject filters cannot use subject based API\"},\n\t\tJSConsumerNameContainsPathSeparatorsErr: {Code: 400, ErrCode: 10127, Description: \"Consumer name can not contain path separators\"},\n\t\tJSConsumerNameExistErr: {Code: 400, ErrCode: 10013, Description: \"consumer name already in use\"},\n\t\tJSConsumerNameTooLongErrF: {Code: 400, ErrCode: 10102, Description: \"consumer name is too long, maximum allowed is {max}\"},\n\t\tJSConsumerNotFoundErr: {Code: 404, ErrCode: 10014, Description: \"consumer not found\"},\n\t\tJSConsumerOfflineErr: {Code: 500, ErrCode: 10119, Description: \"consumer is offline\"},\n\t\tJSConsumerOfflineReasonErrF: {Code: 500, ErrCode: 10195, Description: \"consumer is offline: {err}\"},\n\t\tJSConsumerOnMappedErr: {Code: 400, ErrCode: 10092, Description: \"consumer direct on a mapped consumer\"},\n\t\tJSConsumerOverlappingSubjectFilters: {Code: 400, ErrCode: 10138, Description: \"consumer subject filters cannot overlap\"},\n\t\tJSConsumerPinnedTTLWithoutPriorityPolicyNone: {Code: 400, ErrCode: 10197, Description: \"PinnedTTL cannot be set when PriorityPolicy is none\"},\n\t\tJSConsumerPriorityGroupWithPolicyNone: {Code: 400, ErrCode: 10196, Description: \"consumer can not have priority groups when policy is none\"},\n\t\tJSConsumerPriorityPolicyWithoutGroup: {Code: 400, ErrCode: 10159, Description: \"Setting PriorityPolicy requires at least one PriorityGroup to be set\"},\n\t\tJSConsumerPullNotDurableErr: {Code: 400, ErrCode: 10085, Description: \"consumer in pull mode requires a durable name\"},\n\t\tJSConsumerPullRequiresAckErr: {Code: 400, ErrCode: 10084, Description: \"consumer in pull mode requires explicit ack policy on workqueue stream\"},\n\t\tJSConsumerPullWithRateLimitErr: {Code: 400, ErrCode: 10086, Description: \"consumer in pull mode can not have rate limit set\"},\n\t\tJSConsumerPushMaxWaitingErr: {Code: 400, ErrCode: 10080, Description: \"consumer in push mode can not set max waiting\"},\n\t\tJSConsumerPushWithPriorityGroupErr: {Code: 400, ErrCode: 10178, Description: \"priority groups can not be used with push consumers\"},\n\t\tJSConsumerReplacementWithDifferentNameErr: {Code: 400, ErrCode: 10106, Description: \"consumer replacement durable config not the same\"},\n\t\tJSConsumerReplayPolicyInvalidErr: {Code: 400, ErrCode: 10182, Description: \"consumer replay policy invalid\"},\n\t\tJSConsumerReplicasExceedsStream: {Code: 400, ErrCode: 10126, Description: \"consumer config replica count exceeds parent stream\"},\n\t\tJSConsumerReplicasShouldMatchStream: {Code: 400, ErrCode: 10134, Description: \"consumer config replicas must match interest retention stream's replicas\"},\n\t\tJSConsumerSmallHeartbeatErr: {Code: 400, ErrCode: 10083, Description: \"consumer idle heartbeat needs to be >= 100ms\"},\n\t\tJSConsumerStoreFailedErrF: {Code: 500, ErrCode: 10104, Description: \"error creating store for consumer: {err}\"},\n\t\tJSConsumerWQConsumerNotDeliverAllErr: {Code: 400, ErrCode: 10101, Description: \"consumer must be deliver all on workqueue stream\"},\n\t\tJSConsumerWQConsumerNotUniqueErr: {Code: 400, ErrCode: 10100, Description: \"filtered consumer not unique on workqueue stream\"},\n\t\tJSConsumerWQMultipleUnfilteredErr: {Code: 400, ErrCode: 10099, Description: \"multiple non-filtered consumers not allowed on workqueue stream\"},\n\t\tJSConsumerWQRequiresExplicitAckErr: {Code: 400, ErrCode: 10098, Description: \"workqueue stream requires explicit ack\"},\n\t\tJSConsumerWithFlowControlNeedsHeartbeats: {Code: 400, ErrCode: 10108, Description: \"consumer with flow control also needs heartbeats\"},\n\t\tJSInsufficientResourcesErr: {Code: 503, ErrCode: 10023, Description: \"insufficient resources\"},\n\t\tJSInvalidJSONErr: {Code: 400, ErrCode: 10025, Description: \"invalid JSON: {err}\"},\n\t\tJSMaximumConsumersLimitErr: {Code: 400, ErrCode: 10026, Description: \"maximum consumers limit reached\"},\n\t\tJSMaximumStreamsLimitErr: {Code: 400, ErrCode: 10027, Description: \"maximum number of streams reached\"},\n\t\tJSMemoryResourcesExceededErr: {Code: 500, ErrCode: 10028, Description: \"insufficient memory resources available\"},\n\t\tJSMessageCounterBrokenErr: {Code: 400, ErrCode: 10172, Description: \"message counter is broken\"},\n\t\tJSMessageIncrDisabledErr: {Code: 400, ErrCode: 10168, Description: \"message counters is disabled\"},\n\t\tJSMessageIncrInvalidErr: {Code: 400, ErrCode: 10171, Description: \"message counter increment is invalid\"},\n\t\tJSMessageIncrMissingErr: {Code: 400, ErrCode: 10169, Description: \"message counter increment is missing\"},\n\t\tJSMessageIncrPayloadErr: {Code: 400, ErrCode: 10170, Description: \"message counter has payload\"},\n\t\tJSMessageSchedulesDisabledErr: {Code: 400, ErrCode: 10188, Description: \"message schedules is disabled\"},\n\t\tJSMessageSchedulesPatternInvalidErr: {Code: 400, ErrCode: 10189, Description: \"message schedules pattern is invalid\"},\n\t\tJSMessageSchedulesRollupInvalidErr: {Code: 400, ErrCode: 10192, Description: \"message schedules invalid rollup\"},\n\t\tJSMessageSchedulesSourceInvalidErr: {Code: 400, ErrCode: 10203, Description: \"message schedules source is invalid\"},\n\t\tJSMessageSchedulesTTLInvalidErr: {Code: 400, ErrCode: 10191, Description: \"message schedules invalid per-message TTL\"},\n\t\tJSMessageSchedulesTargetInvalidErr: {Code: 400, ErrCode: 10190, Description: \"message schedules target is invalid\"},\n\t\tJSMessageTTLDisabledErr: {Code: 400, ErrCode: 10166, Description: \"per-message TTL is disabled\"},\n\t\tJSMessageTTLInvalidErr: {Code: 400, ErrCode: 10165, Description: \"invalid per-message TTL\"},\n\t\tJSMirrorConsumerSetupFailedErrF: {Code: 500, ErrCode: 10029, Description: \"{err}\"},\n\t\tJSMirrorInvalidStreamName: {Code: 400, ErrCode: 10142, Description: \"mirrored stream name is invalid\"},\n\t\tJSMirrorInvalidSubjectFilter: {Code: 400, ErrCode: 10151, Description: \"mirror transform source: {err}\"},\n\t\tJSMirrorInvalidTransformDestination: {Code: 400, ErrCode: 10154, Description: \"mirror transform: {err}\"},\n\t\tJSMirrorMaxMessageSizeTooBigErr: {Code: 400, ErrCode: 10030, Description: \"stream mirror must have max message size >= source\"},\n\t\tJSMirrorMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10150, Description: \"mirror with multiple subject transforms cannot also have a single subject filter\"},\n\t\tJSMirrorOverlappingSubjectFilters: {Code: 400, ErrCode: 10152, Description: \"mirror subject filters can not overlap\"},\n\t\tJSMirrorWithAtomicPublishErr: {Code: 400, ErrCode: 10198, Description: \"stream mirrors can not also use atomic publishing\"},\n\t\tJSMirrorWithBatchPublishErr: {Code: 400, ErrCode: 10209, Description: \"stream mirrors can not also use batch publishing\"},\n\t\tJSMirrorWithCountersErr: {Code: 400, ErrCode: 10173, Description: \"stream mirrors can not also calculate counters\"},\n\t\tJSMirrorWithFirstSeqErr: {Code: 400, ErrCode: 10143, Description: \"stream mirrors can not have first sequence configured\"},\n\t\tJSMirrorWithMsgSchedulesErr: {Code: 400, ErrCode: 10186, Description: \"stream mirrors can not also schedule messages\"},\n\t\tJSMirrorWithSourcesErr: {Code: 400, ErrCode: 10031, Description: \"stream mirrors can not also contain other sources\"},\n\t\tJSMirrorWithStartSeqAndTimeErr: {Code: 400, ErrCode: 10032, Description: \"stream mirrors can not have both start seq and start time configured\"},\n\t\tJSMirrorWithSubjectFiltersErr: {Code: 400, ErrCode: 10033, Description: \"stream mirrors can not contain filtered subjects\"},\n\t\tJSMirrorWithSubjectsErr: {Code: 400, ErrCode: 10034, Description: \"stream mirrors can not contain subjects\"},\n\t\tJSNoAccountErr: {Code: 503, ErrCode: 10035, Description: \"account not found\"},\n\t\tJSNoLimitsErr: {Code: 400, ErrCode: 10120, Description: \"no JetStream default or applicable tiered limit present\"},\n\t\tJSNoMessageFoundErr: {Code: 404, ErrCode: 10037, Description: \"no message found\"},\n\t\tJSNotEmptyRequestErr: {Code: 400, ErrCode: 10038, Description: \"expected an empty request payload\"},\n\t\tJSNotEnabledErr: {Code: 503, ErrCode: 10076, Description: \"JetStream not enabled\"},\n\t\tJSNotEnabledForAccountErr: {Code: 503, ErrCode: 10039, Description: \"JetStream not enabled for account\"},\n\t\tJSPedanticErrF: {Code: 400, ErrCode: 10157, Description: \"pedantic mode: {err}\"},\n\t\tJSPeerRemapErr: {Code: 503, ErrCode: 10075, Description: \"peer remap failed\"},\n\t\tJSRaftGeneralErrF: {Code: 500, ErrCode: 10041, Description: \"{err}\"},\n\t\tJSReplicasCountCannotBeNegative: {Code: 400, ErrCode: 10133, Description: \"replicas count cannot be negative\"},\n\t\tJSRequiredApiLevelErr: {Code: 412, ErrCode: 10185, Description: \"JetStream minimum api level required\"},\n\t\tJSRestoreSubscribeFailedErrF: {Code: 500, ErrCode: 10042, Description: \"JetStream unable to subscribe to restore snapshot {subject}: {err}\"},\n\t\tJSSequenceNotFoundErrF: {Code: 400, ErrCode: 10043, Description: \"sequence {seq} not found\"},\n\t\tJSSnapshotDeliverSubjectInvalidErr: {Code: 400, ErrCode: 10015, Description: \"deliver subject not valid\"},\n\t\tJSSourceConsumerSetupFailedErrF: {Code: 500, ErrCode: 10045, Description: \"{err}\"},\n\t\tJSSourceDuplicateDetected: {Code: 400, ErrCode: 10140, Description: \"duplicate source configuration detected\"},\n\t\tJSSourceInvalidStreamName: {Code: 400, ErrCode: 10141, Description: \"sourced stream name is invalid\"},\n\t\tJSSourceInvalidSubjectFilter: {Code: 400, ErrCode: 10145, Description: \"source transform source: {err}\"},\n\t\tJSSourceInvalidTransformDestination: {Code: 400, ErrCode: 10146, Description: \"source transform: {err}\"},\n\t\tJSSourceMaxMessageSizeTooBigErr: {Code: 400, ErrCode: 10046, Description: \"stream source must have max message size >= target\"},\n\t\tJSSourceMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10144, Description: \"source with multiple subject transforms cannot also have a single subject filter\"},\n\t\tJSSourceOverlappingSubjectFilters: {Code: 400, ErrCode: 10147, Description: \"source filters can not overlap\"},\n\t\tJSSourceWithMsgSchedulesErr: {Code: 400, ErrCode: 10187, Description: \"stream source can not also schedule messages\"},\n\t\tJSStorageResourcesExceededErr: {Code: 500, ErrCode: 10047, Description: \"insufficient storage resources available\"},\n\t\tJSStreamAssignmentErrF: {Code: 500, ErrCode: 10048, Description: \"{err}\"},\n\t\tJSStreamCreateErrF: {Code: 500, ErrCode: 10049, Description: \"{err}\"},\n\t\tJSStreamDeleteErrF: {Code: 500, ErrCode: 10050, Description: \"{err}\"},\n\t\tJSStreamDuplicateMessageConflict: {Code: 409, ErrCode: 10158, Description: \"duplicate message id is in process\"},\n\t\tJSStreamExpectedLastSeqPerSubjectInvalid: {Code: 400, ErrCode: 10193, Description: \"missing sequence for expected last sequence per subject\"},\n\t\tJSStreamExpectedLastSeqPerSubjectNotReady: {Code: 503, ErrCode: 10163, Description: \"expected last sequence per subject temporarily unavailable\"},\n\t\tJSStreamExternalApiOverlapErrF: {Code: 400, ErrCode: 10021, Description: \"stream external api prefix {prefix} must not overlap with {subject}\"},\n\t\tJSStreamExternalDelPrefixOverlapsErrF: {Code: 400, ErrCode: 10022, Description: \"stream external delivery prefix {prefix} overlaps with stream subject {subject}\"},\n\t\tJSStreamGeneralErrorF: {Code: 500, ErrCode: 10051, Description: \"{err}\"},\n\t\tJSStreamHeaderExceedsMaximumErr: {Code: 400, ErrCode: 10097, Description: \"header size exceeds maximum allowed of 64k\"},\n\t\tJSStreamInfoMaxSubjectsErr: {Code: 500, ErrCode: 10117, Description: \"subject details would exceed maximum allowed\"},\n\t\tJSStreamInvalidConfigF: {Code: 500, ErrCode: 10052, Description: \"{err}\"},\n\t\tJSStreamInvalidErr: {Code: 500, ErrCode: 10096, Description: \"stream not valid\"},\n\t\tJSStreamInvalidExternalDeliverySubjErrF: {Code: 400, ErrCode: 10024, Description: \"stream external delivery prefix {prefix} must not contain wildcards\"},\n\t\tJSStreamLimitsErrF: {Code: 500, ErrCode: 10053, Description: \"{err}\"},\n\t\tJSStreamMaxBytesRequired: {Code: 400, ErrCode: 10113, Description: \"account requires a stream config to have max bytes set\"},\n\t\tJSStreamMaxStreamBytesExceeded: {Code: 400, ErrCode: 10122, Description: \"stream max bytes exceeds account limit max stream bytes\"},\n\t\tJSStreamMessageExceedsMaximumErr: {Code: 400, ErrCode: 10054, Description: \"message size exceeds maximum allowed\"},\n\t\tJSStreamMinLastSeqErr: {Code: 412, ErrCode: 10180, Description: \"min last sequence\"},\n\t\tJSStreamMirrorNotUpdatableErr: {Code: 400, ErrCode: 10055, Description: \"stream mirror configuration can not be updated\"},\n\t\tJSStreamMismatchErr: {Code: 400, ErrCode: 10056, Description: \"stream name in subject does not match request\"},\n\t\tJSStreamMoveAndScaleErr: {Code: 400, ErrCode: 10123, Description: \"can not move and scale a stream in a single update\"},\n\t\tJSStreamMoveInProgressF: {Code: 400, ErrCode: 10124, Description: \"stream move already in progress: {msg}\"},\n\t\tJSStreamMoveNotInProgress: {Code: 400, ErrCode: 10129, Description: \"stream move not in progress\"},\n\t\tJSStreamMsgDeleteFailedF: {Code: 500, ErrCode: 10057, Description: \"{err}\"},\n\t\tJSStreamNameContainsPathSeparatorsErr: {Code: 400, ErrCode: 10128, Description: \"Stream name can not contain path separators\"},\n\t\tJSStreamNameExistErr: {Code: 400, ErrCode: 10058, Description: \"stream name already in use with a different configuration\"},\n\t\tJSStreamNameExistRestoreFailedErr: {Code: 400, ErrCode: 10130, Description: \"stream name already in use, cannot restore\"},\n\t\tJSStreamNotFoundErr: {Code: 404, ErrCode: 10059, Description: \"stream not found\"},\n\t\tJSStreamNotMatchErr: {Code: 400, ErrCode: 10060, Description: \"expected stream does not match\"},\n\t\tJSStreamOfflineErr: {Code: 500, ErrCode: 10118, Description: \"stream is offline\"},\n\t\tJSStreamOfflineReasonErrF: {Code: 500, ErrCode: 10194, Description: \"stream is offline: {err}\"},\n\t\tJSStreamPurgeFailedF: {Code: 500, ErrCode: 10110, Description: \"{err}\"},\n\t\tJSStreamReplicasNotSupportedErr: {Code: 500, ErrCode: 10074, Description: \"replicas > 1 not supported in non-clustered mode\"},\n\t\tJSStreamReplicasNotUpdatableErr: {Code: 400, ErrCode: 10061, Description: \"Replicas configuration can not be updated\"},\n\t\tJSStreamRestoreErrF: {Code: 500, ErrCode: 10062, Description: \"restore failed: {err}\"},\n\t\tJSStreamRollupFailedF: {Code: 500, ErrCode: 10111, Description: \"{err}\"},\n\t\tJSStreamSealedErr: {Code: 400, ErrCode: 10109, Description: \"invalid operation on sealed stream\"},\n\t\tJSStreamSequenceNotMatchErr: {Code: 503, ErrCode: 10063, Description: \"expected stream sequence does not match\"},\n\t\tJSStreamSnapshotErrF: {Code: 500, ErrCode: 10064, Description: \"snapshot failed: {err}\"},\n\t\tJSStreamStoreFailedF: {Code: 503, ErrCode: 10077, Description: \"{err}\"},\n\t\tJSStreamSubjectOverlapErr: {Code: 400, ErrCode: 10065, Description: \"subjects overlap with an existing stream\"},\n\t\tJSStreamTemplateCreateErrF: {Code: 500, ErrCode: 10066, Description: \"{err}\"},\n\t\tJSStreamTemplateDeleteErrF: {Code: 500, ErrCode: 10067, Description: \"{err}\"},\n\t\tJSStreamTemplateNotFoundErr: {Code: 404, ErrCode: 10068, Description: \"template not found\"},\n\t\tJSStreamTooManyRequests: {Code: 429, ErrCode: 10167, Description: \"too many requests\"},\n\t\tJSStreamTransformInvalidDestination: {Code: 400, ErrCode: 10156, Description: \"stream transform: {err}\"},\n\t\tJSStreamTransformInvalidSource: {Code: 400, ErrCode: 10155, Description: \"stream transform source: {err}\"},\n\t\tJSStreamUpdateErrF: {Code: 500, ErrCode: 10069, Description: \"{err}\"},\n\t\tJSStreamWrongLastMsgIDErrF: {Code: 400, ErrCode: 10070, Description: \"wrong last msg ID: {id}\"},\n\t\tJSStreamWrongLastSequenceConstantErr: {Code: 400, ErrCode: 10164, Description: \"wrong last sequence\"},\n\t\tJSStreamWrongLastSequenceErrF: {Code: 400, ErrCode: 10071, Description: \"wrong last sequence: {seq}\"},\n\t\tJSTempStorageFailedErr: {Code: 500, ErrCode: 10072, Description: \"JetStream unable to open temp storage for restore\"},\n\t\tJSTemplateNameNotMatchSubjectErr: {Code: 400, ErrCode: 10073, Description: \"template name in subject does not match request\"},\n\t}\n\t// ErrJetStreamNotClustered Deprecated by JSClusterNotActiveErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamNotClustered = ApiErrors[JSClusterNotActiveErr]\n\t// ErrJetStreamNotAssigned Deprecated by JSClusterNotAssignedErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamNotAssigned = ApiErrors[JSClusterNotAssignedErr]\n\t// ErrJetStreamNotLeader Deprecated by JSClusterNotLeaderErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamNotLeader = ApiErrors[JSClusterNotLeaderErr]\n\t// ErrJetStreamConsumerAlreadyUsed Deprecated by JSConsumerNameExistErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamConsumerAlreadyUsed = ApiErrors[JSConsumerNameExistErr]\n\t// ErrJetStreamResourcesExceeded Deprecated by JSInsufficientResourcesErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamResourcesExceeded = ApiErrors[JSInsufficientResourcesErr]\n\t// ErrMemoryResourcesExceeded Deprecated by JSMemoryResourcesExceededErr ApiError, use IsNatsError() for comparisons\n\tErrMemoryResourcesExceeded = ApiErrors[JSMemoryResourcesExceededErr]\n\t// ErrJetStreamNotEnabled Deprecated by JSNotEnabledErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamNotEnabled = ApiErrors[JSNotEnabledErr]\n\t// ErrStorageResourcesExceeded Deprecated by JSStorageResourcesExceededErr ApiError, use IsNatsError() for comparisons\n\tErrStorageResourcesExceeded = ApiErrors[JSStorageResourcesExceededErr]\n\t// ErrJetStreamStreamAlreadyUsed Deprecated by JSStreamNameExistErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamStreamAlreadyUsed = ApiErrors[JSStreamNameExistErr]\n\t// ErrJetStreamStreamNotFound Deprecated by JSStreamNotFoundErr ApiError, use IsNatsError() for comparisons\n\tErrJetStreamStreamNotFound = ApiErrors[JSStreamNotFoundErr]\n\t// ErrReplicasNotSupported Deprecated by JSStreamReplicasNotSupportedErr ApiError, use IsNatsError() for comparisons\n\tErrReplicasNotSupported = ApiErrors[JSStreamReplicasNotSupportedErr]\n)\n\n// NewJSAccountResourcesExceededError creates a new JSAccountResourcesExceededErr error: \"resource limits exceeded for account\"\nfunc NewJSAccountResourcesExceededError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAccountResourcesExceededErr]\n}\n\n// NewJSAtomicPublishContainsDuplicateMessageError creates a new JSAtomicPublishContainsDuplicateMessageErr error: \"atomic publish batch contains duplicate message id\"\nfunc NewJSAtomicPublishContainsDuplicateMessageError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishContainsDuplicateMessageErr]\n}\n\n// NewJSAtomicPublishDisabledError creates a new JSAtomicPublishDisabledErr error: \"atomic publish is disabled\"\nfunc NewJSAtomicPublishDisabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishDisabledErr]\n}\n\n// NewJSAtomicPublishIncompleteBatchError creates a new JSAtomicPublishIncompleteBatchErr error: \"atomic publish batch is incomplete\"\nfunc NewJSAtomicPublishIncompleteBatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishIncompleteBatchErr]\n}\n\n// NewJSAtomicPublishInvalidBatchCommitError creates a new JSAtomicPublishInvalidBatchCommitErr error: \"atomic publish batch commit is invalid\"\nfunc NewJSAtomicPublishInvalidBatchCommitError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishInvalidBatchCommitErr]\n}\n\n// NewJSAtomicPublishInvalidBatchIDError creates a new JSAtomicPublishInvalidBatchIDErr error: \"atomic publish batch ID is invalid\"\nfunc NewJSAtomicPublishInvalidBatchIDError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishInvalidBatchIDErr]\n}\n\n// NewJSAtomicPublishMissingSeqError creates a new JSAtomicPublishMissingSeqErr error: \"atomic publish sequence is missing\"\nfunc NewJSAtomicPublishMissingSeqError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishMissingSeqErr]\n}\n\n// NewJSAtomicPublishTooLargeBatchError creates a new JSAtomicPublishTooLargeBatchErrF error: \"atomic publish batch is too large: {size}\"\nfunc NewJSAtomicPublishTooLargeBatchError(size interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSAtomicPublishTooLargeBatchErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{size}\", size})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSAtomicPublishTooManyInflightError creates a new JSAtomicPublishTooManyInflight error: \"atomic publish too many inflight\"\nfunc NewJSAtomicPublishTooManyInflightError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSAtomicPublishTooManyInflight]\n}\n\n// NewJSAtomicPublishUnsupportedHeaderBatchError creates a new JSAtomicPublishUnsupportedHeaderBatchErr error: \"atomic publish unsupported header used: {header}\"\nfunc NewJSAtomicPublishUnsupportedHeaderBatchError(header interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSAtomicPublishUnsupportedHeaderBatchErr]\n\targs := e.toReplacerArgs([]interface{}{\"{header}\", header})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSBadRequestError creates a new JSBadRequestErr error: \"bad request\"\nfunc NewJSBadRequestError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBadRequestErr]\n}\n\n// NewJSBatchPublishDisabledError creates a new JSBatchPublishDisabledErr error: \"batch publish is disabled\"\nfunc NewJSBatchPublishDisabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBatchPublishDisabledErr]\n}\n\n// NewJSBatchPublishInvalidBatchIDError creates a new JSBatchPublishInvalidBatchIDErr error: \"batch publish ID is invalid\"\nfunc NewJSBatchPublishInvalidBatchIDError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBatchPublishInvalidBatchIDErr]\n}\n\n// NewJSBatchPublishInvalidPatternError creates a new JSBatchPublishInvalidPatternErr error: \"batch publish pattern is invalid\"\nfunc NewJSBatchPublishInvalidPatternError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBatchPublishInvalidPatternErr]\n}\n\n// NewJSBatchPublishTooManyInflightError creates a new JSBatchPublishTooManyInflight error: \"batch publish too many inflight\"\nfunc NewJSBatchPublishTooManyInflightError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBatchPublishTooManyInflight]\n}\n\n// NewJSBatchPublishUnknownBatchIDError creates a new JSBatchPublishUnknownBatchIDErr error: \"batch publish ID unknown\"\nfunc NewJSBatchPublishUnknownBatchIDError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSBatchPublishUnknownBatchIDErr]\n}\n\n// NewJSClusterIncompleteError creates a new JSClusterIncompleteErr error: \"incomplete results\"\nfunc NewJSClusterIncompleteError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterIncompleteErr]\n}\n\n// NewJSClusterNoPeersError creates a new JSClusterNoPeersErrF error: \"{err}\"\nfunc NewJSClusterNoPeersError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSClusterNoPeersErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSClusterNotActiveError creates a new JSClusterNotActiveErr error: \"JetStream not in clustered mode\"\nfunc NewJSClusterNotActiveError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterNotActiveErr]\n}\n\n// NewJSClusterNotAssignedError creates a new JSClusterNotAssignedErr error: \"JetStream cluster not assigned to this server\"\nfunc NewJSClusterNotAssignedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterNotAssignedErr]\n}\n\n// NewJSClusterNotAvailError creates a new JSClusterNotAvailErr error: \"JetStream system temporarily unavailable\"\nfunc NewJSClusterNotAvailError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterNotAvailErr]\n}\n\n// NewJSClusterNotLeaderError creates a new JSClusterNotLeaderErr error: \"JetStream cluster can not handle request\"\nfunc NewJSClusterNotLeaderError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterNotLeaderErr]\n}\n\n// NewJSClusterPeerNotMemberError creates a new JSClusterPeerNotMemberErr error: \"peer not a member\"\nfunc NewJSClusterPeerNotMemberError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterPeerNotMemberErr]\n}\n\n// NewJSClusterRequiredError creates a new JSClusterRequiredErr error: \"JetStream clustering support required\"\nfunc NewJSClusterRequiredError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterRequiredErr]\n}\n\n// NewJSClusterServerMemberChangeInflightError creates a new JSClusterServerMemberChangeInflightErr error: \"cluster member change is in progress\"\nfunc NewJSClusterServerMemberChangeInflightError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterServerMemberChangeInflightErr]\n}\n\n// NewJSClusterServerNotMemberError creates a new JSClusterServerNotMemberErr error: \"server is not a member of the cluster\"\nfunc NewJSClusterServerNotMemberError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterServerNotMemberErr]\n}\n\n// NewJSClusterTagsError creates a new JSClusterTagsErr error: \"tags placement not supported for operation\"\nfunc NewJSClusterTagsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterTagsErr]\n}\n\n// NewJSClusterUnSupportFeatureError creates a new JSClusterUnSupportFeatureErr error: \"not currently supported in clustered mode\"\nfunc NewJSClusterUnSupportFeatureError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSClusterUnSupportFeatureErr]\n}\n\n// NewJSConsumerAckPolicyInvalidError creates a new JSConsumerAckPolicyInvalidErr error: \"consumer ack policy invalid\"\nfunc NewJSConsumerAckPolicyInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerAckPolicyInvalidErr]\n}\n\n// NewJSConsumerAckWaitNegativeError creates a new JSConsumerAckWaitNegativeErr error: \"consumer ack wait needs to be positive\"\nfunc NewJSConsumerAckWaitNegativeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerAckWaitNegativeErr]\n}\n\n// NewJSConsumerAlreadyExistsError creates a new JSConsumerAlreadyExists error: \"consumer already exists\"\nfunc NewJSConsumerAlreadyExistsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerAlreadyExists]\n}\n\n// NewJSConsumerBackOffNegativeError creates a new JSConsumerBackOffNegativeErr error: \"consumer backoff needs to be positive\"\nfunc NewJSConsumerBackOffNegativeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerBackOffNegativeErr]\n}\n\n// NewJSConsumerBadDurableNameError creates a new JSConsumerBadDurableNameErr error: \"durable name can not contain '.', '*', '>'\"\nfunc NewJSConsumerBadDurableNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerBadDurableNameErr]\n}\n\n// NewJSConsumerConfigRequiredError creates a new JSConsumerConfigRequiredErr error: \"consumer config required\"\nfunc NewJSConsumerConfigRequiredError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerConfigRequiredErr]\n}\n\n// NewJSConsumerCreateDurableAndNameMismatchError creates a new JSConsumerCreateDurableAndNameMismatch error: \"Consumer Durable and Name have to be equal if both are provided\"\nfunc NewJSConsumerCreateDurableAndNameMismatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerCreateDurableAndNameMismatch]\n}\n\n// NewJSConsumerCreateError creates a new JSConsumerCreateErrF error: \"{err}\"\nfunc NewJSConsumerCreateError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerCreateErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerCreateFilterSubjectMismatchError creates a new JSConsumerCreateFilterSubjectMismatchErr error: \"Consumer create request did not match filtered subject from create subject\"\nfunc NewJSConsumerCreateFilterSubjectMismatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerCreateFilterSubjectMismatchErr]\n}\n\n// NewJSConsumerDeliverCycleError creates a new JSConsumerDeliverCycleErr error: \"consumer deliver subject forms a cycle\"\nfunc NewJSConsumerDeliverCycleError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDeliverCycleErr]\n}\n\n// NewJSConsumerDeliverToWildcardsError creates a new JSConsumerDeliverToWildcardsErr error: \"consumer deliver subject has wildcards\"\nfunc NewJSConsumerDeliverToWildcardsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDeliverToWildcardsErr]\n}\n\n// NewJSConsumerDescriptionTooLongError creates a new JSConsumerDescriptionTooLongErrF error: \"consumer description is too long, maximum allowed is {max}\"\nfunc NewJSConsumerDescriptionTooLongError(max interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerDescriptionTooLongErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{max}\", max})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerDirectRequiresEphemeralError creates a new JSConsumerDirectRequiresEphemeralErr error: \"consumer direct requires an ephemeral consumer\"\nfunc NewJSConsumerDirectRequiresEphemeralError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDirectRequiresEphemeralErr]\n}\n\n// NewJSConsumerDirectRequiresPushError creates a new JSConsumerDirectRequiresPushErr error: \"consumer direct requires a push based consumer\"\nfunc NewJSConsumerDirectRequiresPushError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDirectRequiresPushErr]\n}\n\n// NewJSConsumerDoesNotExistError creates a new JSConsumerDoesNotExist error: \"consumer does not exist\"\nfunc NewJSConsumerDoesNotExistError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDoesNotExist]\n}\n\n// NewJSConsumerDuplicateFilterSubjectsError creates a new JSConsumerDuplicateFilterSubjects error: \"consumer cannot have both FilterSubject and FilterSubjects specified\"\nfunc NewJSConsumerDuplicateFilterSubjectsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDuplicateFilterSubjects]\n}\n\n// NewJSConsumerDurableNameNotInSubjectError creates a new JSConsumerDurableNameNotInSubjectErr error: \"consumer expected to be durable but no durable name set in subject\"\nfunc NewJSConsumerDurableNameNotInSubjectError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDurableNameNotInSubjectErr]\n}\n\n// NewJSConsumerDurableNameNotMatchSubjectError creates a new JSConsumerDurableNameNotMatchSubjectErr error: \"consumer name in subject does not match durable name in request\"\nfunc NewJSConsumerDurableNameNotMatchSubjectError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDurableNameNotMatchSubjectErr]\n}\n\n// NewJSConsumerDurableNameNotSetError creates a new JSConsumerDurableNameNotSetErr error: \"consumer expected to be durable but a durable name was not set\"\nfunc NewJSConsumerDurableNameNotSetError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerDurableNameNotSetErr]\n}\n\n// NewJSConsumerEmptyFilterError creates a new JSConsumerEmptyFilter error: \"consumer filter in FilterSubjects cannot be empty\"\nfunc NewJSConsumerEmptyFilterError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerEmptyFilter]\n}\n\n// NewJSConsumerEmptyGroupNameError creates a new JSConsumerEmptyGroupName error: \"Group name cannot be an empty string\"\nfunc NewJSConsumerEmptyGroupNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerEmptyGroupName]\n}\n\n// NewJSConsumerEphemeralWithDurableInSubjectError creates a new JSConsumerEphemeralWithDurableInSubjectErr error: \"consumer expected to be ephemeral but detected a durable name set in subject\"\nfunc NewJSConsumerEphemeralWithDurableInSubjectError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerEphemeralWithDurableInSubjectErr]\n}\n\n// NewJSConsumerEphemeralWithDurableNameError creates a new JSConsumerEphemeralWithDurableNameErr error: \"consumer expected to be ephemeral but a durable name was set in request\"\nfunc NewJSConsumerEphemeralWithDurableNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerEphemeralWithDurableNameErr]\n}\n\n// NewJSConsumerExistingActiveError creates a new JSConsumerExistingActiveErr error: \"consumer already exists and is still active\"\nfunc NewJSConsumerExistingActiveError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerExistingActiveErr]\n}\n\n// NewJSConsumerFCRequiresPushError creates a new JSConsumerFCRequiresPushErr error: \"consumer flow control requires a push based consumer\"\nfunc NewJSConsumerFCRequiresPushError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerFCRequiresPushErr]\n}\n\n// NewJSConsumerFilterNotSubsetError creates a new JSConsumerFilterNotSubsetErr error: \"consumer filter subject is not a valid subset of the interest subjects\"\nfunc NewJSConsumerFilterNotSubsetError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerFilterNotSubsetErr]\n}\n\n// NewJSConsumerHBRequiresPushError creates a new JSConsumerHBRequiresPushErr error: \"consumer idle heartbeat requires a push based consumer\"\nfunc NewJSConsumerHBRequiresPushError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerHBRequiresPushErr]\n}\n\n// NewJSConsumerInactiveThresholdExcessError creates a new JSConsumerInactiveThresholdExcess error: \"consumer inactive threshold exceeds system limit of {limit}\"\nfunc NewJSConsumerInactiveThresholdExcessError(limit interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerInactiveThresholdExcess]\n\targs := e.toReplacerArgs([]interface{}{\"{limit}\", limit})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerInvalidDeliverSubjectError creates a new JSConsumerInvalidDeliverSubject error: \"invalid push consumer deliver subject\"\nfunc NewJSConsumerInvalidDeliverSubjectError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerInvalidDeliverSubject]\n}\n\n// NewJSConsumerInvalidGroupNameError creates a new JSConsumerInvalidGroupNameErr error: \"Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters\"\nfunc NewJSConsumerInvalidGroupNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerInvalidGroupNameErr]\n}\n\n// NewJSConsumerInvalidPolicyError creates a new JSConsumerInvalidPolicyErrF error: \"{err}\"\nfunc NewJSConsumerInvalidPolicyError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerInvalidPolicyErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerInvalidPriorityGroupError creates a new JSConsumerInvalidPriorityGroupErr error: \"Provided priority group does not exist for this consumer\"\nfunc NewJSConsumerInvalidPriorityGroupError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerInvalidPriorityGroupErr]\n}\n\n// NewJSConsumerInvalidResetError creates a new JSConsumerInvalidResetErr error: \"invalid reset: {err}\"\nfunc NewJSConsumerInvalidResetError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerInvalidResetErr]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerInvalidSamplingError creates a new JSConsumerInvalidSamplingErrF error: \"failed to parse consumer sampling configuration: {err}\"\nfunc NewJSConsumerInvalidSamplingError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerInvalidSamplingErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerMaxDeliverBackoffError creates a new JSConsumerMaxDeliverBackoffErr error: \"max deliver is required to be > length of backoff values\"\nfunc NewJSConsumerMaxDeliverBackoffError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMaxDeliverBackoffErr]\n}\n\n// NewJSConsumerMaxPendingAckExcessError creates a new JSConsumerMaxPendingAckExcessErrF error: \"consumer max ack pending exceeds system limit of {limit}\"\nfunc NewJSConsumerMaxPendingAckExcessError(limit interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerMaxPendingAckExcessErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{limit}\", limit})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerMaxPendingAckPolicyRequiredError creates a new JSConsumerMaxPendingAckPolicyRequiredErr error: \"consumer requires ack policy for max ack pending\"\nfunc NewJSConsumerMaxPendingAckPolicyRequiredError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMaxPendingAckPolicyRequiredErr]\n}\n\n// NewJSConsumerMaxRequestBatchExceededError creates a new JSConsumerMaxRequestBatchExceededF error: \"consumer max request batch exceeds server limit of {limit}\"\nfunc NewJSConsumerMaxRequestBatchExceededError(limit interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerMaxRequestBatchExceededF]\n\targs := e.toReplacerArgs([]interface{}{\"{limit}\", limit})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerMaxRequestBatchNegativeError creates a new JSConsumerMaxRequestBatchNegativeErr error: \"consumer max request batch needs to be > 0\"\nfunc NewJSConsumerMaxRequestBatchNegativeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMaxRequestBatchNegativeErr]\n}\n\n// NewJSConsumerMaxRequestExpiresTooSmallError creates a new JSConsumerMaxRequestExpiresTooSmall error: \"consumer max request expires needs to be >= 1ms\"\nfunc NewJSConsumerMaxRequestExpiresTooSmallError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMaxRequestExpiresTooSmall]\n}\n\n// NewJSConsumerMaxWaitingNegativeError creates a new JSConsumerMaxWaitingNegativeErr error: \"consumer max waiting needs to be positive\"\nfunc NewJSConsumerMaxWaitingNegativeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMaxWaitingNegativeErr]\n}\n\n// NewJSConsumerMetadataLengthError creates a new JSConsumerMetadataLengthErrF error: \"consumer metadata exceeds maximum size of {limit}\"\nfunc NewJSConsumerMetadataLengthError(limit interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerMetadataLengthErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{limit}\", limit})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerMultipleFiltersNotAllowedError creates a new JSConsumerMultipleFiltersNotAllowed error: \"consumer with multiple subject filters cannot use subject based API\"\nfunc NewJSConsumerMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerMultipleFiltersNotAllowed]\n}\n\n// NewJSConsumerNameContainsPathSeparatorsError creates a new JSConsumerNameContainsPathSeparatorsErr error: \"Consumer name can not contain path separators\"\nfunc NewJSConsumerNameContainsPathSeparatorsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerNameContainsPathSeparatorsErr]\n}\n\n// NewJSConsumerNameExistError creates a new JSConsumerNameExistErr error: \"consumer name already in use\"\nfunc NewJSConsumerNameExistError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerNameExistErr]\n}\n\n// NewJSConsumerNameTooLongError creates a new JSConsumerNameTooLongErrF error: \"consumer name is too long, maximum allowed is {max}\"\nfunc NewJSConsumerNameTooLongError(max interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerNameTooLongErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{max}\", max})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerNotFoundError creates a new JSConsumerNotFoundErr error: \"consumer not found\"\nfunc NewJSConsumerNotFoundError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerNotFoundErr]\n}\n\n// NewJSConsumerOfflineError creates a new JSConsumerOfflineErr error: \"consumer is offline\"\nfunc NewJSConsumerOfflineError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerOfflineErr]\n}\n\n// NewJSConsumerOfflineReasonError creates a new JSConsumerOfflineReasonErrF error: \"consumer is offline: {err}\"\nfunc NewJSConsumerOfflineReasonError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerOfflineReasonErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerOnMappedError creates a new JSConsumerOnMappedErr error: \"consumer direct on a mapped consumer\"\nfunc NewJSConsumerOnMappedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerOnMappedErr]\n}\n\n// NewJSConsumerOverlappingSubjectFiltersError creates a new JSConsumerOverlappingSubjectFilters error: \"consumer subject filters cannot overlap\"\nfunc NewJSConsumerOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerOverlappingSubjectFilters]\n}\n\n// NewJSConsumerPinnedTTLWithoutPriorityPolicyNoneError creates a new JSConsumerPinnedTTLWithoutPriorityPolicyNone error: \"PinnedTTL cannot be set when PriorityPolicy is none\"\nfunc NewJSConsumerPinnedTTLWithoutPriorityPolicyNoneError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPinnedTTLWithoutPriorityPolicyNone]\n}\n\n// NewJSConsumerPriorityGroupWithPolicyNoneError creates a new JSConsumerPriorityGroupWithPolicyNone error: \"consumer can not have priority groups when policy is none\"\nfunc NewJSConsumerPriorityGroupWithPolicyNoneError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPriorityGroupWithPolicyNone]\n}\n\n// NewJSConsumerPriorityPolicyWithoutGroupError creates a new JSConsumerPriorityPolicyWithoutGroup error: \"Setting PriorityPolicy requires at least one PriorityGroup to be set\"\nfunc NewJSConsumerPriorityPolicyWithoutGroupError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPriorityPolicyWithoutGroup]\n}\n\n// NewJSConsumerPullNotDurableError creates a new JSConsumerPullNotDurableErr error: \"consumer in pull mode requires a durable name\"\nfunc NewJSConsumerPullNotDurableError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPullNotDurableErr]\n}\n\n// NewJSConsumerPullRequiresAckError creates a new JSConsumerPullRequiresAckErr error: \"consumer in pull mode requires explicit ack policy on workqueue stream\"\nfunc NewJSConsumerPullRequiresAckError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPullRequiresAckErr]\n}\n\n// NewJSConsumerPullWithRateLimitError creates a new JSConsumerPullWithRateLimitErr error: \"consumer in pull mode can not have rate limit set\"\nfunc NewJSConsumerPullWithRateLimitError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPullWithRateLimitErr]\n}\n\n// NewJSConsumerPushMaxWaitingError creates a new JSConsumerPushMaxWaitingErr error: \"consumer in push mode can not set max waiting\"\nfunc NewJSConsumerPushMaxWaitingError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPushMaxWaitingErr]\n}\n\n// NewJSConsumerPushWithPriorityGroupError creates a new JSConsumerPushWithPriorityGroupErr error: \"priority groups can not be used with push consumers\"\nfunc NewJSConsumerPushWithPriorityGroupError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerPushWithPriorityGroupErr]\n}\n\n// NewJSConsumerReplacementWithDifferentNameError creates a new JSConsumerReplacementWithDifferentNameErr error: \"consumer replacement durable config not the same\"\nfunc NewJSConsumerReplacementWithDifferentNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerReplacementWithDifferentNameErr]\n}\n\n// NewJSConsumerReplayPolicyInvalidError creates a new JSConsumerReplayPolicyInvalidErr error: \"consumer replay policy invalid\"\nfunc NewJSConsumerReplayPolicyInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerReplayPolicyInvalidErr]\n}\n\n// NewJSConsumerReplicasExceedsStreamError creates a new JSConsumerReplicasExceedsStream error: \"consumer config replica count exceeds parent stream\"\nfunc NewJSConsumerReplicasExceedsStreamError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerReplicasExceedsStream]\n}\n\n// NewJSConsumerReplicasShouldMatchStreamError creates a new JSConsumerReplicasShouldMatchStream error: \"consumer config replicas must match interest retention stream's replicas\"\nfunc NewJSConsumerReplicasShouldMatchStreamError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerReplicasShouldMatchStream]\n}\n\n// NewJSConsumerSmallHeartbeatError creates a new JSConsumerSmallHeartbeatErr error: \"consumer idle heartbeat needs to be >= 100ms\"\nfunc NewJSConsumerSmallHeartbeatError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerSmallHeartbeatErr]\n}\n\n// NewJSConsumerStoreFailedError creates a new JSConsumerStoreFailedErrF error: \"error creating store for consumer: {err}\"\nfunc NewJSConsumerStoreFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSConsumerStoreFailedErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSConsumerWQConsumerNotDeliverAllError creates a new JSConsumerWQConsumerNotDeliverAllErr error: \"consumer must be deliver all on workqueue stream\"\nfunc NewJSConsumerWQConsumerNotDeliverAllError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerWQConsumerNotDeliverAllErr]\n}\n\n// NewJSConsumerWQConsumerNotUniqueError creates a new JSConsumerWQConsumerNotUniqueErr error: \"filtered consumer not unique on workqueue stream\"\nfunc NewJSConsumerWQConsumerNotUniqueError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerWQConsumerNotUniqueErr]\n}\n\n// NewJSConsumerWQMultipleUnfilteredError creates a new JSConsumerWQMultipleUnfilteredErr error: \"multiple non-filtered consumers not allowed on workqueue stream\"\nfunc NewJSConsumerWQMultipleUnfilteredError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerWQMultipleUnfilteredErr]\n}\n\n// NewJSConsumerWQRequiresExplicitAckError creates a new JSConsumerWQRequiresExplicitAckErr error: \"workqueue stream requires explicit ack\"\nfunc NewJSConsumerWQRequiresExplicitAckError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerWQRequiresExplicitAckErr]\n}\n\n// NewJSConsumerWithFlowControlNeedsHeartbeatsError creates a new JSConsumerWithFlowControlNeedsHeartbeats error: \"consumer with flow control also needs heartbeats\"\nfunc NewJSConsumerWithFlowControlNeedsHeartbeatsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSConsumerWithFlowControlNeedsHeartbeats]\n}\n\n// NewJSInsufficientResourcesError creates a new JSInsufficientResourcesErr error: \"insufficient resources\"\nfunc NewJSInsufficientResourcesError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSInsufficientResourcesErr]\n}\n\n// NewJSInvalidJSONError creates a new JSInvalidJSONErr error: \"invalid JSON: {err}\"\nfunc NewJSInvalidJSONError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSInvalidJSONErr]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSMaximumConsumersLimitError creates a new JSMaximumConsumersLimitErr error: \"maximum consumers limit reached\"\nfunc NewJSMaximumConsumersLimitError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMaximumConsumersLimitErr]\n}\n\n// NewJSMaximumStreamsLimitError creates a new JSMaximumStreamsLimitErr error: \"maximum number of streams reached\"\nfunc NewJSMaximumStreamsLimitError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMaximumStreamsLimitErr]\n}\n\n// NewJSMemoryResourcesExceededError creates a new JSMemoryResourcesExceededErr error: \"insufficient memory resources available\"\nfunc NewJSMemoryResourcesExceededError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMemoryResourcesExceededErr]\n}\n\n// NewJSMessageCounterBrokenError creates a new JSMessageCounterBrokenErr error: \"message counter is broken\"\nfunc NewJSMessageCounterBrokenError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageCounterBrokenErr]\n}\n\n// NewJSMessageIncrDisabledError creates a new JSMessageIncrDisabledErr error: \"message counters is disabled\"\nfunc NewJSMessageIncrDisabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageIncrDisabledErr]\n}\n\n// NewJSMessageIncrInvalidError creates a new JSMessageIncrInvalidErr error: \"message counter increment is invalid\"\nfunc NewJSMessageIncrInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageIncrInvalidErr]\n}\n\n// NewJSMessageIncrMissingError creates a new JSMessageIncrMissingErr error: \"message counter increment is missing\"\nfunc NewJSMessageIncrMissingError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageIncrMissingErr]\n}\n\n// NewJSMessageIncrPayloadError creates a new JSMessageIncrPayloadErr error: \"message counter has payload\"\nfunc NewJSMessageIncrPayloadError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageIncrPayloadErr]\n}\n\n// NewJSMessageSchedulesDisabledError creates a new JSMessageSchedulesDisabledErr error: \"message schedules is disabled\"\nfunc NewJSMessageSchedulesDisabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesDisabledErr]\n}\n\n// NewJSMessageSchedulesPatternInvalidError creates a new JSMessageSchedulesPatternInvalidErr error: \"message schedules pattern is invalid\"\nfunc NewJSMessageSchedulesPatternInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesPatternInvalidErr]\n}\n\n// NewJSMessageSchedulesRollupInvalidError creates a new JSMessageSchedulesRollupInvalidErr error: \"message schedules invalid rollup\"\nfunc NewJSMessageSchedulesRollupInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesRollupInvalidErr]\n}\n\n// NewJSMessageSchedulesSourceInvalidError creates a new JSMessageSchedulesSourceInvalidErr error: \"message schedules source is invalid\"\nfunc NewJSMessageSchedulesSourceInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesSourceInvalidErr]\n}\n\n// NewJSMessageSchedulesTTLInvalidError creates a new JSMessageSchedulesTTLInvalidErr error: \"message schedules invalid per-message TTL\"\nfunc NewJSMessageSchedulesTTLInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesTTLInvalidErr]\n}\n\n// NewJSMessageSchedulesTargetInvalidError creates a new JSMessageSchedulesTargetInvalidErr error: \"message schedules target is invalid\"\nfunc NewJSMessageSchedulesTargetInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageSchedulesTargetInvalidErr]\n}\n\n// NewJSMessageTTLDisabledError creates a new JSMessageTTLDisabledErr error: \"per-message TTL is disabled\"\nfunc NewJSMessageTTLDisabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageTTLDisabledErr]\n}\n\n// NewJSMessageTTLInvalidError creates a new JSMessageTTLInvalidErr error: \"invalid per-message TTL\"\nfunc NewJSMessageTTLInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMessageTTLInvalidErr]\n}\n\n// NewJSMirrorConsumerSetupFailedError creates a new JSMirrorConsumerSetupFailedErrF error: \"{err}\"\nfunc NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSMirrorConsumerSetupFailedErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSMirrorInvalidStreamNameError creates a new JSMirrorInvalidStreamName error: \"mirrored stream name is invalid\"\nfunc NewJSMirrorInvalidStreamNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorInvalidStreamName]\n}\n\n// NewJSMirrorInvalidSubjectFilterError creates a new JSMirrorInvalidSubjectFilter error: \"mirror transform source: {err}\"\nfunc NewJSMirrorInvalidSubjectFilterError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSMirrorInvalidSubjectFilter]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSMirrorInvalidTransformDestinationError creates a new JSMirrorInvalidTransformDestination error: \"mirror transform: {err}\"\nfunc NewJSMirrorInvalidTransformDestinationError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSMirrorInvalidTransformDestination]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSMirrorMaxMessageSizeTooBigError creates a new JSMirrorMaxMessageSizeTooBigErr error: \"stream mirror must have max message size >= source\"\nfunc NewJSMirrorMaxMessageSizeTooBigError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorMaxMessageSizeTooBigErr]\n}\n\n// NewJSMirrorMultipleFiltersNotAllowedError creates a new JSMirrorMultipleFiltersNotAllowed error: \"mirror with multiple subject transforms cannot also have a single subject filter\"\nfunc NewJSMirrorMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorMultipleFiltersNotAllowed]\n}\n\n// NewJSMirrorOverlappingSubjectFiltersError creates a new JSMirrorOverlappingSubjectFilters error: \"mirror subject filters can not overlap\"\nfunc NewJSMirrorOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorOverlappingSubjectFilters]\n}\n\n// NewJSMirrorWithAtomicPublishError creates a new JSMirrorWithAtomicPublishErr error: \"stream mirrors can not also use atomic publishing\"\nfunc NewJSMirrorWithAtomicPublishError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithAtomicPublishErr]\n}\n\n// NewJSMirrorWithBatchPublishError creates a new JSMirrorWithBatchPublishErr error: \"stream mirrors can not also use batch publishing\"\nfunc NewJSMirrorWithBatchPublishError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithBatchPublishErr]\n}\n\n// NewJSMirrorWithCountersError creates a new JSMirrorWithCountersErr error: \"stream mirrors can not also calculate counters\"\nfunc NewJSMirrorWithCountersError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithCountersErr]\n}\n\n// NewJSMirrorWithFirstSeqError creates a new JSMirrorWithFirstSeqErr error: \"stream mirrors can not have first sequence configured\"\nfunc NewJSMirrorWithFirstSeqError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithFirstSeqErr]\n}\n\n// NewJSMirrorWithMsgSchedulesError creates a new JSMirrorWithMsgSchedulesErr error: \"stream mirrors can not also schedule messages\"\nfunc NewJSMirrorWithMsgSchedulesError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithMsgSchedulesErr]\n}\n\n// NewJSMirrorWithSourcesError creates a new JSMirrorWithSourcesErr error: \"stream mirrors can not also contain other sources\"\nfunc NewJSMirrorWithSourcesError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithSourcesErr]\n}\n\n// NewJSMirrorWithStartSeqAndTimeError creates a new JSMirrorWithStartSeqAndTimeErr error: \"stream mirrors can not have both start seq and start time configured\"\nfunc NewJSMirrorWithStartSeqAndTimeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithStartSeqAndTimeErr]\n}\n\n// NewJSMirrorWithSubjectFiltersError creates a new JSMirrorWithSubjectFiltersErr error: \"stream mirrors can not contain filtered subjects\"\nfunc NewJSMirrorWithSubjectFiltersError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithSubjectFiltersErr]\n}\n\n// NewJSMirrorWithSubjectsError creates a new JSMirrorWithSubjectsErr error: \"stream mirrors can not contain subjects\"\nfunc NewJSMirrorWithSubjectsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSMirrorWithSubjectsErr]\n}\n\n// NewJSNoAccountError creates a new JSNoAccountErr error: \"account not found\"\nfunc NewJSNoAccountError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNoAccountErr]\n}\n\n// NewJSNoLimitsError creates a new JSNoLimitsErr error: \"no JetStream default or applicable tiered limit present\"\nfunc NewJSNoLimitsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNoLimitsErr]\n}\n\n// NewJSNoMessageFoundError creates a new JSNoMessageFoundErr error: \"no message found\"\nfunc NewJSNoMessageFoundError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNoMessageFoundErr]\n}\n\n// NewJSNotEmptyRequestError creates a new JSNotEmptyRequestErr error: \"expected an empty request payload\"\nfunc NewJSNotEmptyRequestError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNotEmptyRequestErr]\n}\n\n// NewJSNotEnabledError creates a new JSNotEnabledErr error: \"JetStream not enabled\"\nfunc NewJSNotEnabledError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNotEnabledErr]\n}\n\n// NewJSNotEnabledForAccountError creates a new JSNotEnabledForAccountErr error: \"JetStream not enabled for account\"\nfunc NewJSNotEnabledForAccountError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSNotEnabledForAccountErr]\n}\n\n// NewJSPedanticError creates a new JSPedanticErrF error: \"pedantic mode: {err}\"\nfunc NewJSPedanticError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSPedanticErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSPeerRemapError creates a new JSPeerRemapErr error: \"peer remap failed\"\nfunc NewJSPeerRemapError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSPeerRemapErr]\n}\n\n// NewJSRaftGeneralError creates a new JSRaftGeneralErrF error: \"{err}\"\nfunc NewJSRaftGeneralError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSRaftGeneralErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSReplicasCountCannotBeNegativeError creates a new JSReplicasCountCannotBeNegative error: \"replicas count cannot be negative\"\nfunc NewJSReplicasCountCannotBeNegativeError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSReplicasCountCannotBeNegative]\n}\n\n// NewJSRequiredApiLevelError creates a new JSRequiredApiLevelErr error: \"JetStream minimum api level required\"\nfunc NewJSRequiredApiLevelError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSRequiredApiLevelErr]\n}\n\n// NewJSRestoreSubscribeFailedError creates a new JSRestoreSubscribeFailedErrF error: \"JetStream unable to subscribe to restore snapshot {subject}: {err}\"\nfunc NewJSRestoreSubscribeFailedError(err error, subject interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSRestoreSubscribeFailedErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err, \"{subject}\", subject})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSSequenceNotFoundError creates a new JSSequenceNotFoundErrF error: \"sequence {seq} not found\"\nfunc NewJSSequenceNotFoundError(seq uint64, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSSequenceNotFoundErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{seq}\", seq})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSSnapshotDeliverSubjectInvalidError creates a new JSSnapshotDeliverSubjectInvalidErr error: \"deliver subject not valid\"\nfunc NewJSSnapshotDeliverSubjectInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSnapshotDeliverSubjectInvalidErr]\n}\n\n// NewJSSourceConsumerSetupFailedError creates a new JSSourceConsumerSetupFailedErrF error: \"{err}\"\nfunc NewJSSourceConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSSourceConsumerSetupFailedErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSSourceDuplicateDetectedError creates a new JSSourceDuplicateDetected error: \"duplicate source configuration detected\"\nfunc NewJSSourceDuplicateDetectedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceDuplicateDetected]\n}\n\n// NewJSSourceInvalidStreamNameError creates a new JSSourceInvalidStreamName error: \"sourced stream name is invalid\"\nfunc NewJSSourceInvalidStreamNameError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceInvalidStreamName]\n}\n\n// NewJSSourceInvalidSubjectFilterError creates a new JSSourceInvalidSubjectFilter error: \"source transform source: {err}\"\nfunc NewJSSourceInvalidSubjectFilterError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSSourceInvalidSubjectFilter]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSSourceInvalidTransformDestinationError creates a new JSSourceInvalidTransformDestination error: \"source transform: {err}\"\nfunc NewJSSourceInvalidTransformDestinationError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSSourceInvalidTransformDestination]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSSourceMaxMessageSizeTooBigError creates a new JSSourceMaxMessageSizeTooBigErr error: \"stream source must have max message size >= target\"\nfunc NewJSSourceMaxMessageSizeTooBigError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceMaxMessageSizeTooBigErr]\n}\n\n// NewJSSourceMultipleFiltersNotAllowedError creates a new JSSourceMultipleFiltersNotAllowed error: \"source with multiple subject transforms cannot also have a single subject filter\"\nfunc NewJSSourceMultipleFiltersNotAllowedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceMultipleFiltersNotAllowed]\n}\n\n// NewJSSourceOverlappingSubjectFiltersError creates a new JSSourceOverlappingSubjectFilters error: \"source filters can not overlap\"\nfunc NewJSSourceOverlappingSubjectFiltersError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceOverlappingSubjectFilters]\n}\n\n// NewJSSourceWithMsgSchedulesError creates a new JSSourceWithMsgSchedulesErr error: \"stream source can not also schedule messages\"\nfunc NewJSSourceWithMsgSchedulesError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSSourceWithMsgSchedulesErr]\n}\n\n// NewJSStorageResourcesExceededError creates a new JSStorageResourcesExceededErr error: \"insufficient storage resources available\"\nfunc NewJSStorageResourcesExceededError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStorageResourcesExceededErr]\n}\n\n// NewJSStreamAssignmentError creates a new JSStreamAssignmentErrF error: \"{err}\"\nfunc NewJSStreamAssignmentError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamAssignmentErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamCreateError creates a new JSStreamCreateErrF error: \"{err}\"\nfunc NewJSStreamCreateError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamCreateErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamDeleteError creates a new JSStreamDeleteErrF error: \"{err}\"\nfunc NewJSStreamDeleteError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamDeleteErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamDuplicateMessageConflictError creates a new JSStreamDuplicateMessageConflict error: \"duplicate message id is in process\"\nfunc NewJSStreamDuplicateMessageConflictError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamDuplicateMessageConflict]\n}\n\n// NewJSStreamExpectedLastSeqPerSubjectInvalidError creates a new JSStreamExpectedLastSeqPerSubjectInvalid error: \"missing sequence for expected last sequence per subject\"\nfunc NewJSStreamExpectedLastSeqPerSubjectInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamExpectedLastSeqPerSubjectInvalid]\n}\n\n// NewJSStreamExpectedLastSeqPerSubjectNotReadyError creates a new JSStreamExpectedLastSeqPerSubjectNotReady error: \"expected last sequence per subject temporarily unavailable\"\nfunc NewJSStreamExpectedLastSeqPerSubjectNotReadyError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamExpectedLastSeqPerSubjectNotReady]\n}\n\n// NewJSStreamExternalApiOverlapError creates a new JSStreamExternalApiOverlapErrF error: \"stream external api prefix {prefix} must not overlap with {subject}\"\nfunc NewJSStreamExternalApiOverlapError(prefix interface{}, subject interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamExternalApiOverlapErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{prefix}\", prefix, \"{subject}\", subject})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamExternalDelPrefixOverlapsError creates a new JSStreamExternalDelPrefixOverlapsErrF error: \"stream external delivery prefix {prefix} overlaps with stream subject {subject}\"\nfunc NewJSStreamExternalDelPrefixOverlapsError(prefix interface{}, subject interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamExternalDelPrefixOverlapsErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{prefix}\", prefix, \"{subject}\", subject})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamGeneralError creates a new JSStreamGeneralErrorF error: \"{err}\"\nfunc NewJSStreamGeneralError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamGeneralErrorF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamHeaderExceedsMaximumError creates a new JSStreamHeaderExceedsMaximumErr error: \"header size exceeds maximum allowed of 64k\"\nfunc NewJSStreamHeaderExceedsMaximumError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamHeaderExceedsMaximumErr]\n}\n\n// NewJSStreamInfoMaxSubjectsError creates a new JSStreamInfoMaxSubjectsErr error: \"subject details would exceed maximum allowed\"\nfunc NewJSStreamInfoMaxSubjectsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamInfoMaxSubjectsErr]\n}\n\n// NewJSStreamInvalidConfigError creates a new JSStreamInvalidConfigF error: \"{err}\"\nfunc NewJSStreamInvalidConfigError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamInvalidConfigF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamInvalidError creates a new JSStreamInvalidErr error: \"stream not valid\"\nfunc NewJSStreamInvalidError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamInvalidErr]\n}\n\n// NewJSStreamInvalidExternalDeliverySubjError creates a new JSStreamInvalidExternalDeliverySubjErrF error: \"stream external delivery prefix {prefix} must not contain wildcards\"\nfunc NewJSStreamInvalidExternalDeliverySubjError(prefix interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamInvalidExternalDeliverySubjErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{prefix}\", prefix})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamLimitsError creates a new JSStreamLimitsErrF error: \"{err}\"\nfunc NewJSStreamLimitsError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamLimitsErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamMaxBytesRequiredError creates a new JSStreamMaxBytesRequired error: \"account requires a stream config to have max bytes set\"\nfunc NewJSStreamMaxBytesRequiredError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMaxBytesRequired]\n}\n\n// NewJSStreamMaxStreamBytesExceededError creates a new JSStreamMaxStreamBytesExceeded error: \"stream max bytes exceeds account limit max stream bytes\"\nfunc NewJSStreamMaxStreamBytesExceededError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMaxStreamBytesExceeded]\n}\n\n// NewJSStreamMessageExceedsMaximumError creates a new JSStreamMessageExceedsMaximumErr error: \"message size exceeds maximum allowed\"\nfunc NewJSStreamMessageExceedsMaximumError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMessageExceedsMaximumErr]\n}\n\n// NewJSStreamMinLastSeqError creates a new JSStreamMinLastSeqErr error: \"min last sequence\"\nfunc NewJSStreamMinLastSeqError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMinLastSeqErr]\n}\n\n// NewJSStreamMirrorNotUpdatableError creates a new JSStreamMirrorNotUpdatableErr error: \"stream mirror configuration can not be updated\"\nfunc NewJSStreamMirrorNotUpdatableError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMirrorNotUpdatableErr]\n}\n\n// NewJSStreamMismatchError creates a new JSStreamMismatchErr error: \"stream name in subject does not match request\"\nfunc NewJSStreamMismatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMismatchErr]\n}\n\n// NewJSStreamMoveAndScaleError creates a new JSStreamMoveAndScaleErr error: \"can not move and scale a stream in a single update\"\nfunc NewJSStreamMoveAndScaleError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMoveAndScaleErr]\n}\n\n// NewJSStreamMoveInProgressError creates a new JSStreamMoveInProgressF error: \"stream move already in progress: {msg}\"\nfunc NewJSStreamMoveInProgressError(msg interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamMoveInProgressF]\n\targs := e.toReplacerArgs([]interface{}{\"{msg}\", msg})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamMoveNotInProgressError creates a new JSStreamMoveNotInProgress error: \"stream move not in progress\"\nfunc NewJSStreamMoveNotInProgressError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamMoveNotInProgress]\n}\n\n// NewJSStreamMsgDeleteFailedError creates a new JSStreamMsgDeleteFailedF error: \"{err}\"\nfunc NewJSStreamMsgDeleteFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamMsgDeleteFailedF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamNameContainsPathSeparatorsError creates a new JSStreamNameContainsPathSeparatorsErr error: \"Stream name can not contain path separators\"\nfunc NewJSStreamNameContainsPathSeparatorsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamNameContainsPathSeparatorsErr]\n}\n\n// NewJSStreamNameExistError creates a new JSStreamNameExistErr error: \"stream name already in use with a different configuration\"\nfunc NewJSStreamNameExistError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamNameExistErr]\n}\n\n// NewJSStreamNameExistRestoreFailedError creates a new JSStreamNameExistRestoreFailedErr error: \"stream name already in use, cannot restore\"\nfunc NewJSStreamNameExistRestoreFailedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamNameExistRestoreFailedErr]\n}\n\n// NewJSStreamNotFoundError creates a new JSStreamNotFoundErr error: \"stream not found\"\nfunc NewJSStreamNotFoundError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamNotFoundErr]\n}\n\n// NewJSStreamNotMatchError creates a new JSStreamNotMatchErr error: \"expected stream does not match\"\nfunc NewJSStreamNotMatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamNotMatchErr]\n}\n\n// NewJSStreamOfflineError creates a new JSStreamOfflineErr error: \"stream is offline\"\nfunc NewJSStreamOfflineError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamOfflineErr]\n}\n\n// NewJSStreamOfflineReasonError creates a new JSStreamOfflineReasonErrF error: \"stream is offline: {err}\"\nfunc NewJSStreamOfflineReasonError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamOfflineReasonErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamPurgeFailedError creates a new JSStreamPurgeFailedF error: \"{err}\"\nfunc NewJSStreamPurgeFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamPurgeFailedF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamReplicasNotSupportedError creates a new JSStreamReplicasNotSupportedErr error: \"replicas > 1 not supported in non-clustered mode\"\nfunc NewJSStreamReplicasNotSupportedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamReplicasNotSupportedErr]\n}\n\n// NewJSStreamReplicasNotUpdatableError creates a new JSStreamReplicasNotUpdatableErr error: \"Replicas configuration can not be updated\"\nfunc NewJSStreamReplicasNotUpdatableError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamReplicasNotUpdatableErr]\n}\n\n// NewJSStreamRestoreError creates a new JSStreamRestoreErrF error: \"restore failed: {err}\"\nfunc NewJSStreamRestoreError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamRestoreErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamRollupFailedError creates a new JSStreamRollupFailedF error: \"{err}\"\nfunc NewJSStreamRollupFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamRollupFailedF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamSealedError creates a new JSStreamSealedErr error: \"invalid operation on sealed stream\"\nfunc NewJSStreamSealedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamSealedErr]\n}\n\n// NewJSStreamSequenceNotMatchError creates a new JSStreamSequenceNotMatchErr error: \"expected stream sequence does not match\"\nfunc NewJSStreamSequenceNotMatchError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamSequenceNotMatchErr]\n}\n\n// NewJSStreamSnapshotError creates a new JSStreamSnapshotErrF error: \"snapshot failed: {err}\"\nfunc NewJSStreamSnapshotError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamSnapshotErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamStoreFailedError creates a new JSStreamStoreFailedF error: \"{err}\"\nfunc NewJSStreamStoreFailedError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamStoreFailedF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamSubjectOverlapError creates a new JSStreamSubjectOverlapErr error: \"subjects overlap with an existing stream\"\nfunc NewJSStreamSubjectOverlapError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamSubjectOverlapErr]\n}\n\n// NewJSStreamTemplateCreateError creates a new JSStreamTemplateCreateErrF error: \"{err}\"\nfunc NewJSStreamTemplateCreateError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamTemplateCreateErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamTemplateDeleteError creates a new JSStreamTemplateDeleteErrF error: \"{err}\"\nfunc NewJSStreamTemplateDeleteError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamTemplateDeleteErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamTemplateNotFoundError creates a new JSStreamTemplateNotFoundErr error: \"template not found\"\nfunc NewJSStreamTemplateNotFoundError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamTemplateNotFoundErr]\n}\n\n// NewJSStreamTooManyRequestsError creates a new JSStreamTooManyRequests error: \"too many requests\"\nfunc NewJSStreamTooManyRequestsError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamTooManyRequests]\n}\n\n// NewJSStreamTransformInvalidDestinationError creates a new JSStreamTransformInvalidDestination error: \"stream transform: {err}\"\nfunc NewJSStreamTransformInvalidDestinationError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamTransformInvalidDestination]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamTransformInvalidSourceError creates a new JSStreamTransformInvalidSource error: \"stream transform source: {err}\"\nfunc NewJSStreamTransformInvalidSourceError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamTransformInvalidSource]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamUpdateError creates a new JSStreamUpdateErrF error: \"{err}\"\nfunc NewJSStreamUpdateError(err error, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamUpdateErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{err}\", err})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamWrongLastMsgIDError creates a new JSStreamWrongLastMsgIDErrF error: \"wrong last msg ID: {id}\"\nfunc NewJSStreamWrongLastMsgIDError(id interface{}, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamWrongLastMsgIDErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{id}\", id})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSStreamWrongLastSequenceConstantError creates a new JSStreamWrongLastSequenceConstantErr error: \"wrong last sequence\"\nfunc NewJSStreamWrongLastSequenceConstantError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSStreamWrongLastSequenceConstantErr]\n}\n\n// NewJSStreamWrongLastSequenceError creates a new JSStreamWrongLastSequenceErrF error: \"wrong last sequence: {seq}\"\nfunc NewJSStreamWrongLastSequenceError(seq uint64, opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\te := ApiErrors[JSStreamWrongLastSequenceErrF]\n\targs := e.toReplacerArgs([]interface{}{\"{seq}\", seq})\n\treturn &ApiError{\n\t\tCode: e.Code,\n\t\tErrCode: e.ErrCode,\n\t\tDescription: strings.NewReplacer(args...).Replace(e.Description),\n\t}\n}\n\n// NewJSTempStorageFailedError creates a new JSTempStorageFailedErr error: \"JetStream unable to open temp storage for restore\"\nfunc NewJSTempStorageFailedError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSTempStorageFailedErr]\n}\n\n// NewJSTemplateNameNotMatchSubjectError creates a new JSTemplateNameNotMatchSubjectErr error: \"template name in subject does not match request\"\nfunc NewJSTemplateNameNotMatchSubjectError(opts ...ErrorOption) *ApiError {\n\teopts := parseOpts(opts)\n\tif ae, ok := eopts.err.(*ApiError); ok {\n\t\treturn ae\n\t}\n\n\treturn ApiErrors[JSTemplateNameNotMatchSubjectErr]\n}\n" }, { "path": "server/jetstream_errors_test.go", "content": "package server\n\nimport (\n\t\"errors\"\n\t\"testing\"\n)\n\nfunc TestIsNatsErr(t *testing.T) {\n\tif !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSNotEnabledForAccountErr) {\n\t\tt.Fatalf(\"Expected error match\")\n\t}\n\n\tif IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr) {\n\t\tt.Fatalf(\"Expected error mismatch\")\n\t}\n\n\tif IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSClusterNotAvailErr) {\n\t\tt.Fatalf(\"Expected error mismatch\")\n\t}\n\n\tif !IsNatsErr(ApiErrors[JSNotEnabledForAccountErr], JSClusterNotActiveErr, JSNotEnabledForAccountErr) {\n\t\tt.Fatalf(\"Expected error match\")\n\t}\n\n\tif !IsNatsErr(&ApiError{ErrCode: 10039}, 1, JSClusterNotActiveErr, JSNotEnabledForAccountErr) {\n\t\tt.Fatalf(\"Expected error match\")\n\t}\n\n\tif IsNatsErr(&ApiError{ErrCode: 10039}, 1, 2, JSClusterNotActiveErr) {\n\t\tt.Fatalf(\"Expected error mismatch\")\n\t}\n\n\tif IsNatsErr(nil, JSClusterNotActiveErr) {\n\t\tt.Fatalf(\"Expected error mismatch\")\n\t}\n\n\tif IsNatsErr(errors.New(\"x\"), JSClusterNotActiveErr) {\n\t\tt.Fatalf(\"Expected error mismatch\")\n\t}\n}\n\nfunc TestApiError_Error(t *testing.T) {\n\tif es := ApiErrors[JSClusterNotActiveErr].Error(); es != \"JetStream not in clustered mode (10006)\" {\n\t\tt.Fatalf(\"Expected 'JetStream not in clustered mode (10006)', got %q\", es)\n\t}\n}\n\nfunc TestApiError_NewWithTags(t *testing.T) {\n\tne := NewJSRestoreSubscribeFailedError(errors.New(\"failed error\"), \"the.subject\")\n\tif ne.Description != \"JetStream unable to subscribe to restore snapshot the.subject: failed error\" {\n\t\tt.Fatalf(\"Expected 'JetStream unable to subscribe to restore snapshot the.subject: failed error' got %q\", ne.Description)\n\t}\n\n\tif ne == ApiErrors[JSRestoreSubscribeFailedErrF] {\n\t\tt.Fatalf(\"Expected a new instance\")\n\t}\n}\n\nfunc TestApiError_NewWithUnless(t *testing.T) {\n\tif ne := NewJSStreamRestoreError(errors.New(\"failed error\"), Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) {\n\t\tt.Fatalf(\"Expected JSNotEnabledForAccountErr got %s\", ne)\n\t}\n\n\tif ne := NewJSStreamRestoreError(errors.New(\"failed error\")); !IsNatsErr(ne, JSStreamRestoreErrF) {\n\t\tt.Fatalf(\"Expected JSStreamRestoreErrF got %s\", ne)\n\t}\n\n\tif ne := NewJSStreamRestoreError(errors.New(\"failed error\"), Unless(errors.New(\"other error\"))); !IsNatsErr(ne, JSStreamRestoreErrF) {\n\t\tt.Fatalf(\"Expected JSStreamRestoreErrF got %s\", ne)\n\t}\n\n\tif ne := NewJSPeerRemapError(Unless(ApiErrors[JSNotEnabledForAccountErr])); !IsNatsErr(ne, JSNotEnabledForAccountErr) {\n\t\tt.Fatalf(\"Expected JSNotEnabledForAccountErr got %s\", ne)\n\t}\n\n\tif ne := NewJSPeerRemapError(Unless(nil)); !IsNatsErr(ne, JSPeerRemapErr) {\n\t\tt.Fatalf(\"Expected JSPeerRemapErr got %s\", ne)\n\t}\n\n\tif ne := NewJSPeerRemapError(Unless(errors.New(\"other error\"))); !IsNatsErr(ne, JSPeerRemapErr) {\n\t\tt.Fatalf(\"Expected JSPeerRemapErr got %s\", ne)\n\t}\n}\n" }, { "path": "server/jetstream_events.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\n// publishAdvisory sends the given advisory into the account. Returns true if\n// it was sent, false if not (i.e. due to lack of interest or a marshal error).\nfunc (s *Server) publishAdvisory(acc *Account, subject string, adv any) bool {\n\tif acc == nil {\n\t\tacc = s.SystemAccount()\n\t\tif acc == nil {\n\t\t\treturn false\n\t\t}\n\t}\n\n\t// If there is no one listening for this advisory then save ourselves the effort\n\t// and don't bother encoding the JSON or sending it.\n\tif sl := acc.sl; (sl != nil && !sl.HasInterest(subject)) && !s.hasGatewayInterest(acc.Name, subject) {\n\t\treturn false\n\t}\n\n\tej, err := json.Marshal(adv)\n\tif err == nil {\n\t\terr = s.sendInternalAccountMsg(acc, subject, ej)\n\t\tif err != nil {\n\t\t\ts.Warnf(\"Advisory could not be sent for account %q: %v\", acc.Name, err)\n\t\t}\n\t} else {\n\t\ts.Warnf(\"Advisory could not be serialized for account %q: %v\", acc.Name, err)\n\t}\n\treturn err == nil\n}\n\n// JSAPIAudit is an advisory about administrative actions taken on JetStream\ntype JSAPIAudit struct {\n\tTypedEvent\n\tServer string `json:\"server\"`\n\tClient *ClientInfo `json:\"client\"`\n\tSubject string `json:\"subject\"`\n\tRequest string `json:\"request,omitempty\"`\n\tResponse string `json:\"response\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\nconst JSAPIAuditType = \"io.nats.jetstream.advisory.v1.api_audit\"\n\n// ActionAdvisoryType indicates which action against a stream, consumer or template triggered an advisory\ntype ActionAdvisoryType string\n\nconst (\n\tCreateEvent ActionAdvisoryType = \"create\"\n\tDeleteEvent ActionAdvisoryType = \"delete\"\n\tModifyEvent ActionAdvisoryType = \"modify\"\n)\n\n// JSStreamActionAdvisory indicates that a stream was created, edited or deleted\ntype JSStreamActionAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tAction ActionAdvisoryType `json:\"action\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\nconst JSStreamActionAdvisoryType = \"io.nats.jetstream.advisory.v1.stream_action\"\n\n// JSConsumerActionAdvisory indicates that a consumer was created or deleted\ntype JSConsumerActionAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tAction ActionAdvisoryType `json:\"action\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\nconst JSConsumerActionAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_action\"\n\n// JSConsumerPauseAdvisory indicates that a consumer was paused or unpaused\ntype JSConsumerPauseAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tPaused bool `json:\"paused\"`\n\tPauseUntil time.Time `json:\"pause_until,omitempty\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\nconst JSConsumerPauseAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_pause\"\n\n// JSConsumerAckMetric is a metric published when a user acknowledges a message, the\n// number of these that will be published is dependent on SampleFrequency\ntype JSConsumerAckMetric struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tConsumerSeq uint64 `json:\"consumer_seq\"`\n\tStreamSeq uint64 `json:\"stream_seq\"`\n\tDelay int64 `json:\"ack_time\"`\n\tDeliveries uint64 `json:\"deliveries\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSConsumerAckMetricType is the schema type for JSConsumerAckMetricType\nconst JSConsumerAckMetricType = \"io.nats.jetstream.metric.v1.consumer_ack\"\n\n// JSConsumerDeliveryExceededAdvisory is an advisory informing that a message hit\n// its MaxDeliver threshold and so might be a candidate for DLQ handling\ntype JSConsumerDeliveryExceededAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tStreamSeq uint64 `json:\"stream_seq\"`\n\tDeliveries uint64 `json:\"deliveries\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSConsumerDeliveryExceededAdvisoryType is the schema type for JSConsumerDeliveryExceededAdvisory\nconst JSConsumerDeliveryExceededAdvisoryType = \"io.nats.jetstream.advisory.v1.max_deliver\"\n\n// JSConsumerDeliveryNakAdvisory is an advisory informing that a message was\n// naked by the consumer\ntype JSConsumerDeliveryNakAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tConsumerSeq uint64 `json:\"consumer_seq\"`\n\tStreamSeq uint64 `json:\"stream_seq\"`\n\tDeliveries uint64 `json:\"deliveries\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSConsumerDeliveryNakAdvisoryType is the schema type for JSConsumerDeliveryNakAdvisory\nconst JSConsumerDeliveryNakAdvisoryType = \"io.nats.jetstream.advisory.v1.nak\"\n\n// JSConsumerDeliveryTerminatedAdvisory is an advisory informing that a message was\n// terminated by the consumer, so might be a candidate for DLQ handling\ntype JSConsumerDeliveryTerminatedAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tConsumerSeq uint64 `json:\"consumer_seq\"`\n\tStreamSeq uint64 `json:\"stream_seq\"`\n\tDeliveries uint64 `json:\"deliveries\"`\n\tReason string `json:\"reason,omitempty\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSConsumerDeliveryTerminatedAdvisoryType is the schema type for JSConsumerDeliveryTerminatedAdvisory\nconst JSConsumerDeliveryTerminatedAdvisoryType = \"io.nats.jetstream.advisory.v1.terminated\"\n\n// JSSnapshotCreateAdvisory is an advisory sent after a snapshot is successfully started\ntype JSSnapshotCreateAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tState StreamState `json:\"state\"`\n\tClient *ClientInfo `json:\"client\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSSnapshotCreatedAdvisoryType is the schema type for JSSnapshotCreateAdvisory\nconst JSSnapshotCreatedAdvisoryType = \"io.nats.jetstream.advisory.v1.snapshot_create\"\n\n// JSSnapshotCompleteAdvisory is an advisory sent after a snapshot is successfully started\ntype JSSnapshotCompleteAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tStart time.Time `json:\"start\"`\n\tEnd time.Time `json:\"end\"`\n\tClient *ClientInfo `json:\"client\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSSnapshotCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory\nconst JSSnapshotCompleteAdvisoryType = \"io.nats.jetstream.advisory.v1.snapshot_complete\"\n\n// JSRestoreCreateAdvisory is an advisory sent after a snapshot is successfully started\ntype JSRestoreCreateAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tClient *ClientInfo `json:\"client\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSRestoreCreateAdvisoryType is the schema type for JSSnapshotCreateAdvisory\nconst JSRestoreCreateAdvisoryType = \"io.nats.jetstream.advisory.v1.restore_create\"\n\n// JSRestoreCompleteAdvisory is an advisory sent after a snapshot is successfully started\ntype JSRestoreCompleteAdvisory struct {\n\tTypedEvent\n\tStream string `json:\"stream\"`\n\tStart time.Time `json:\"start\"`\n\tEnd time.Time `json:\"end\"`\n\tBytes int64 `json:\"bytes\"`\n\tClient *ClientInfo `json:\"client\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSRestoreCompleteAdvisoryType is the schema type for JSSnapshotCreateAdvisory\nconst JSRestoreCompleteAdvisoryType = \"io.nats.jetstream.advisory.v1.restore_complete\"\n\n// Clustering specific.\n\n// JSClusterLeaderElectedAdvisoryType is sent when the system elects a new meta leader.\nconst JSDomainLeaderElectedAdvisoryType = \"io.nats.jetstream.advisory.v1.domain_leader_elected\"\n\n// JSClusterLeaderElectedAdvisory indicates that a domain has elected a new leader.\ntype JSDomainLeaderElectedAdvisory struct {\n\tTypedEvent\n\tLeader string `json:\"leader\"`\n\tReplicas []*PeerInfo `json:\"replicas\"`\n\tCluster string `json:\"cluster\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSStreamLeaderElectedAdvisoryType is sent when the system elects a new leader for a stream.\nconst JSStreamLeaderElectedAdvisoryType = \"io.nats.jetstream.advisory.v1.stream_leader_elected\"\n\n// JSStreamLeaderElectedAdvisory indicates that a stream has elected a new leader.\ntype JSStreamLeaderElectedAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tLeader string `json:\"leader\"`\n\tReplicas []*PeerInfo `json:\"replicas\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSStreamQuorumLostAdvisoryType is sent when the system detects a clustered stream and\n// its consumers are stalled and unable to make progress.\nconst JSStreamQuorumLostAdvisoryType = \"io.nats.jetstream.advisory.v1.stream_quorum_lost\"\n\n// JSStreamQuorumLostAdvisory indicates that a stream has lost quorum and is stalled.\ntype JSStreamQuorumLostAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tReplicas []*PeerInfo `json:\"replicas\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSStreamBatchAbandonedAdvisoryType is sent when a stream's atomic batch is abandoned.\nconst JSStreamBatchAbandonedAdvisoryType = \"io.nats.jetstream.advisory.v1.stream_batch_abandoned\"\n\n// JSStreamBatchAbandonedAdvisory indicates that a stream's batch was abandoned.\ntype JSStreamBatchAbandonedAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tBatchId string `json:\"batch\"`\n\tReason BatchAbandonReason `json:\"reason\"`\n}\n\ntype BatchAbandonReason string\n\nvar (\n\tBatchTimeout BatchAbandonReason = \"timeout\"\n\tBatchLarge BatchAbandonReason = \"large\"\n\tBatchIncomplete BatchAbandonReason = \"incomplete\"\n)\n\n// JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer.\nconst JSConsumerLeaderElectedAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_leader_elected\"\n\n// JSConsumerLeaderElectedAdvisory indicates that a consumer has elected a new leader.\ntype JSConsumerLeaderElectedAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tLeader string `json:\"leader\"`\n\tReplicas []*PeerInfo `json:\"replicas\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSConsumerQuorumLostAdvisoryType is sent when the system detects a clustered consumer and\n// is stalled and unable to make progress.\nconst JSConsumerQuorumLostAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_quorum_lost\"\n\n// JSConsumerQuorumLostAdvisory indicates that a consumer has lost quorum and is stalled.\ntype JSConsumerQuorumLostAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tReplicas []*PeerInfo `json:\"replicas\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\nconst JSConsumerGroupPinnedAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_group_pinned\"\n\n// JSConsumerGroupPinnedAdvisory that a group switched to a new pinned client\ntype JSConsumerGroupPinnedAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tGroup string `json:\"group\"`\n\tPinnedClientId string `json:\"pinned_id\"`\n}\n\nconst JSConsumerGroupUnpinnedAdvisoryType = \"io.nats.jetstream.advisory.v1.consumer_group_unpinned\"\n\n// JSConsumerGroupUnpinnedAdvisory indicates that a pin was lost\ntype JSConsumerGroupUnpinnedAdvisory struct {\n\tTypedEvent\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream\"`\n\tConsumer string `json:\"consumer\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tGroup string `json:\"group\"`\n\t// one of \"admin\" or \"timeout\", could be an enum up to the implementor to decide\n\tReason string `json:\"reason\"`\n}\n\n// JSServerOutOfStorageAdvisoryType is sent when the server is out of storage space.\nconst JSServerOutOfStorageAdvisoryType = \"io.nats.jetstream.advisory.v1.server_out_of_space\"\n\n// JSServerOutOfSpaceAdvisory indicates that a stream has lost quorum and is stalled.\ntype JSServerOutOfSpaceAdvisory struct {\n\tTypedEvent\n\tServer string `json:\"server\"`\n\tServerID string `json:\"server_id\"`\n\tStream string `json:\"stream,omitempty\"`\n\tCluster string `json:\"cluster\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSServerRemovedAdvisoryType is sent when the server has been removed and JS disabled.\nconst JSServerRemovedAdvisoryType = \"io.nats.jetstream.advisory.v1.server_removed\"\n\n// JSServerRemovedAdvisory indicates that a stream has lost quorum and is stalled.\ntype JSServerRemovedAdvisory struct {\n\tTypedEvent\n\tServer string `json:\"server\"`\n\tServerID string `json:\"server_id\"`\n\tCluster string `json:\"cluster\"`\n\tDomain string `json:\"domain,omitempty\"`\n}\n\n// JSAPILimitReachedAdvisoryType is sent when the JS API request queue limit is reached.\nconst JSAPILimitReachedAdvisoryType = \"io.nats.jetstream.advisory.v1.api_limit_reached\"\n\n// JSAPILimitReachedAdvisory is a advisory published when JetStream hits the queue length limit.\ntype JSAPILimitReachedAdvisory struct {\n\tTypedEvent\n\tServer string `json:\"server\"` // Server that created the event, name or ID\n\tDomain string `json:\"domain,omitempty\"` // Domain the server belongs to\n\tDropped int64 `json:\"dropped\"` // How many messages did we drop from the queue\n}\n" }, { "path": "server/jetstream_helpers_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Do not exlude this file with the !skip_js_tests since those helpers\n// are also used by MQTT.\n\n//lint:file-ignore U1000 Avoid detecting as unused code\n\npackage server\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nats.go/jetstream\"\n\t\"golang.org/x/time/rate\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/antithesis\"\n)\n\n// Support functions\n\nfunc init() {\n\t// Speed up raft for tests.\n\thbInterval = 50 * time.Millisecond\n\tminElectionTimeout = 1500 * time.Millisecond\n\tmaxElectionTimeout = 3500 * time.Millisecond\n\tlostQuorumInterval = 2 * time.Second\n\tlostQuorumCheck = 4 * hbInterval\n\n\t// For statz and jetstream placement speedups as well.\n\tstatszRateLimit = 0\n}\n\n// Used to setup clusters of clusters for tests.\ntype cluster struct {\n\tservers []*Server\n\topts []*Options\n\tname string\n\tt testing.TB\n\tnproxies []*netProxy\n}\n\n// Used to setup superclusters for tests.\ntype supercluster struct {\n\tt *testing.T\n\tclusters []*cluster\n\tnproxies []*netProxy\n}\n\nfunc (sc *supercluster) shutdown() {\n\tif sc == nil {\n\t\treturn\n\t}\n\tfor _, np := range sc.nproxies {\n\t\tnp.stop()\n\t}\n\tfor _, c := range sc.clusters {\n\t\tshutdownCluster(c)\n\t}\n}\n\nfunc (sc *supercluster) randomServer() *Server {\n\treturn sc.randomCluster().randomServer()\n}\n\nfunc (sc *supercluster) serverByName(sname string) *Server {\n\tfor _, c := range sc.clusters {\n\t\tif s := c.serverByName(sname); s != nil {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (sc *supercluster) waitOnStreamLeader(account, stream string) {\n\tsc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfor _, c := range sc.clusters {\n\t\t\tif leader := c.streamLeader(account, stream); leader != nil {\n\t\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(sc.t, \"Timeout in supercluster.waitOnStreamLeader\", map[string]any{\n\t\t\"account\": account,\n\t\t\"stream\": stream,\n\t})\n\tsc.t.Fatalf(\"Expected a stream leader for %q %q, got none\", account, stream)\n}\n\nvar jsClusterAccountsTempl = `\n\tlisten: 127.0.0.1:-1\n\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\twebsocket {\n\t\tlisten: 127.0.0.1:-1\n\t\tcompression: true\n\t\thandshake_timeout: \"5s\"\n\t\tno_tls: true\n\t}\n\n\tno_auth_user: one\n\n\taccounts {\n\t\tONE { users = [ { user: \"one\", pass: \"p\" } ]; jetstream: enabled }\n\t\tTWO { users = [ { user: \"two\", pass: \"p\" } ]; jetstream: enabled }\n\t\tNOJS { users = [ { user: \"nojs\", pass: \"p\" } ] }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsClusterTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterEncryptedTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s', key: \"s3cr3t!\"}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterMaxBytesTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: u\n\n\taccounts {\n\t\t$U {\n\t\t\tusers = [ { user: \"u\", pass: \"p\" } ]\n\t\t\tjetstream: {\n\t\t\t\tmax_mem: 128MB\n\t\t\t\tmax_file: 18GB\n\t\t\t\tmax_bytes: true // Forces streams to indicate max_bytes.\n\t\t\t}\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsClusterMaxBytesAccountLimitTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 4GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: u\n\n\taccounts {\n\t\t$U {\n\t\t\tusers = [ { user: \"u\", pass: \"p\" } ]\n\t\t\tjetstream: {\n\t\t\t\tmax_mem: 128MB\n\t\t\t\tmax_file: 3GB\n\t\t\t\tmax_bytes: true // Forces streams to indicate max_bytes.\n\t\t\t}\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsSuperClusterTempl = `\n\t%s\n\tgateway {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\tgateways = [%s\n\t\t]\n\t}\n\n\tsystem_account: \"$SYS\"\n`\n\nvar jsClusterLimitsTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: u\n\n\taccounts {\n\t\tONE {\n\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\tjetstream: enabled\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsMixedModeGlobalAccountTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsGWTempl = `%s{name: %s, urls: [%s]}`\n\nvar jsClusterAccountLimitsTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: js\n\n\taccounts {\n\t\t$JS { users = [ { user: \"js\", pass: \"p\" } ]; jetstream: {max_store: 1MB, max_mem: 0} }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nfunc createJetStreamTaggedSuperCluster(t *testing.T) *supercluster {\n\treturn createJetStreamTaggedSuperClusterWithGWProxy(t, nil)\n}\n\nfunc createJetStreamTaggedSuperClusterWithGWProxy(t *testing.T, gwm gwProxyMap) *supercluster {\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 3, 3, nil, gwm)\n\tsc.waitOnPeerCount(9)\n\n\treset := func(s *Server) {\n\t\ts.mu.Lock()\n\t\trch := s.sys.resetCh\n\t\ts.mu.Unlock()\n\t\tif rch != nil {\n\t\t\trch <- struct{}{}\n\t\t}\n\t\ts.sendStatszUpdate()\n\t}\n\n\t// Make first cluster AWS, US country code.\n\tfor i, s := range sc.clusterForName(\"C1\").servers {\n\t\ts.optsMu.Lock()\n\t\ts.opts.Tags.Add(\"cloud:aws\")\n\t\ts.opts.Tags.Add(\"country:us\")\n\t\ts.opts.Tags.Add(fmt.Sprintf(\"node:%d\", i+1))\n\t\ts.optsMu.Unlock()\n\t\treset(s)\n\t}\n\t// Make second cluster GCP, UK country code.\n\tfor i, s := range sc.clusterForName(\"C2\").servers {\n\t\ts.optsMu.Lock()\n\t\ts.opts.Tags.Add(\"cloud:gcp\")\n\t\ts.opts.Tags.Add(\"country:uk\")\n\t\ts.opts.Tags.Add(fmt.Sprintf(\"node:%d\", i+1))\n\t\ts.optsMu.Unlock()\n\t\treset(s)\n\t}\n\t// Make third cluster AZ, JP country code.\n\tfor i, s := range sc.clusterForName(\"C3\").servers {\n\t\ts.optsMu.Lock()\n\t\ts.opts.Tags.Add(\"cloud:az\")\n\t\ts.opts.Tags.Add(\"country:jp\")\n\t\ts.opts.Tags.Add(fmt.Sprintf(\"node:%d\", i+1))\n\t\ts.optsMu.Unlock()\n\t\treset(s)\n\t}\n\n\tsc.waitOnLeader()\n\tml := sc.leader()\n\trequire_True(t, ml != nil)\n\tjs := ml.getJetStream()\n\trequire_True(t, js != nil)\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tcc := js.cluster\n\trequire_True(t, cc != nil)\n\n\t// Walk and make sure all tags are registered.\n\texpires := time.Now().Add(10 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tallOK := true\n\t\tfor _, p := range cc.meta.Peers() {\n\t\t\tsi, ok := ml.nodeToInfo.Load(p.ID)\n\t\t\trequire_True(t, ok)\n\t\t\tni := si.(nodeInfo)\n\t\t\tif len(ni.tags) == 0 {\n\t\t\t\tallOK = false\n\t\t\t\treset(sc.serverByName(ni.name))\n\t\t\t}\n\t\t}\n\t\tif allOK {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn sc\n}\n\nfunc createJetStreamSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster {\n\treturn createJetStreamSuperClusterWithTemplate(t, jsClusterTempl, numServersPer, numClusters)\n}\n\nfunc createJetStreamSuperClusterWithTemplate(t *testing.T, tmpl string, numServersPer, numClusters int) *supercluster {\n\treturn createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, numServersPer, numClusters, nil, nil)\n}\n\n// For doing proxyies in GWs.\ntype gwProxy struct {\n\trtt time.Duration\n\tup int\n\tdown int\n}\n\n// For use in normal clusters.\ntype clusterProxy = gwProxy\n\n// Maps cluster names to proxy settings.\ntype gwProxyMap map[string]*gwProxy\n\nfunc createJetStreamSuperClusterWithTemplateAndModHook(t *testing.T, tmpl string, numServersPer, numClusters int, modify modifyCb, gwm gwProxyMap) *supercluster {\n\tt.Helper()\n\tif numServersPer < 1 {\n\t\tt.Fatalf(\"Number of servers must be >= 1\")\n\t}\n\tif numClusters <= 1 {\n\t\tt.Fatalf(\"Number of clusters must be > 1\")\n\t}\n\n\tstartClusterPorts := []int{20_022, 22_022, 24_022}\n\tstartGatewayPorts := []int{20_122, 22_122, 24_122}\n\tstartClusterPort := startClusterPorts[rand.Intn(len(startClusterPorts))]\n\tstartGWPort := startGatewayPorts[rand.Intn(len(startGatewayPorts))]\n\n\t// Make the GWs form faster for the tests.\n\tSetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer ResetGatewaysSolicitDelay()\n\n\tcp, gp := startClusterPort, startGWPort\n\tvar clusters []*cluster\n\tvar nproxies []*netProxy\n\tvar gws []string\n\n\t// Build GWs first, will be same for all servers.\n\tfor i, port := 1, gp; i <= numClusters; i++ {\n\t\tcn := fmt.Sprintf(\"C%d\", i)\n\t\tvar gwp *gwProxy\n\t\tif len(gwm) > 0 {\n\t\t\tgwp = gwm[cn]\n\t\t}\n\t\tvar urls []string\n\t\tfor n := 0; n < numServersPer; n++ {\n\t\t\trouteURL := fmt.Sprintf(\"nats-route://127.0.0.1:%d\", port)\n\t\t\tif gwp != nil {\n\t\t\t\tnp := createNetProxy(gwp.rtt, gwp.up, gwp.down, routeURL, false)\n\t\t\t\tnproxies = append(nproxies, np)\n\t\t\t\trouteURL = np.routeURL()\n\t\t\t}\n\t\t\turls = append(urls, routeURL)\n\t\t\tport++\n\t\t}\n\t\tgws = append(gws, fmt.Sprintf(jsGWTempl, \"\\n\\t\\t\\t\", cn, strings.Join(urls, \",\")))\n\t}\n\tgwconf := strings.Join(gws, _EMPTY_)\n\n\tfor i := 1; i <= numClusters; i++ {\n\t\tcn := fmt.Sprintf(\"C%d\", i)\n\t\t// Go ahead and build configurations.\n\t\tc := &cluster{servers: make([]*Server, 0, numServersPer), opts: make([]*Options, 0, numServersPer), name: cn}\n\n\t\t// Build out the routes that will be shared with all configs.\n\t\tvar routes []string\n\t\tfor port := cp; port < cp+numServersPer; port++ {\n\t\t\troutes = append(routes, fmt.Sprintf(\"nats-route://127.0.0.1:%d\", port))\n\t\t}\n\t\trouteConfig := strings.Join(routes, \",\")\n\n\t\tfor si := 0; si < numServersPer; si++ {\n\t\t\tstoreDir := t.TempDir()\n\t\t\tsn := fmt.Sprintf(\"%s-S%d\", cn, si+1)\n\t\t\tbconf := fmt.Sprintf(tmpl, sn, storeDir, cn, cp+si, routeConfig)\n\t\t\tconf := fmt.Sprintf(jsSuperClusterTempl, bconf, cn, gp, gwconf)\n\t\t\tgp++\n\t\t\tif modify != nil {\n\t\t\t\tconf = modify(sn, cn, storeDir, conf)\n\t\t\t}\n\t\t\ts, o := RunServerWithConfig(createConfFile(t, []byte(conf)))\n\t\t\tc.servers = append(c.servers, s)\n\t\t\tc.opts = append(c.opts, o)\n\t\t}\n\t\tcheckClusterFormed(t, c.servers...)\n\t\tclusters = append(clusters, c)\n\t\tcp += numServersPer\n\t\tc.t = t\n\t}\n\n\t// Start any proxies.\n\tfor _, np := range nproxies {\n\t\tnp.start()\n\t}\n\n\t// Wait for the supercluster to be formed.\n\tegws := numClusters - 1\n\tfor _, c := range clusters {\n\t\tfor _, s := range c.servers {\n\t\t\twaitForOutboundGateways(t, s, egws, 10*time.Second)\n\t\t}\n\t}\n\n\tsc := &supercluster{t, clusters, nproxies}\n\tsc.waitOnLeader()\n\tsc.waitOnAllCurrent()\n\n\t// Wait for all the peer nodes to be registered.\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tvar peers []string\n\t\tif ml := sc.leader(); ml != nil {\n\t\t\tpeers = ml.ActivePeers()\n\t\t\tif len(peers) == numClusters*numServersPer {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Not correct number of peers, expected %d, got %d\", numClusters*numServersPer, len(peers))\n\t})\n\n\tif sc.leader() == nil {\n\t\tsc.t.Fatalf(\"Expected a cluster leader, got none\")\n\t}\n\n\treturn sc\n}\n\nfunc (sc *supercluster) createLeafNodes(clusterName string, numServers int) *cluster {\n\tsc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\treturn sc.createLeafNodesWithDomain(clusterName, numServers, \"\")\n}\n\nfunc (sc *supercluster) createLeafNodesWithDomain(clusterName string, numServers int, domain string) *cluster {\n\tsc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\treturn sc.randomCluster().createLeafNodes(clusterName, numServers, domain)\n}\n\nfunc (sc *supercluster) createSingleLeafNode(extend bool) *Server {\n\tsc.t.Helper()\n\n\treturn sc.randomCluster().createLeafNode(extend)\n}\n\nfunc (sc *supercluster) leader() *Server {\n\tfor _, c := range sc.clusters {\n\t\tif leader := c.leader(); leader != nil {\n\t\t\treturn leader\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (sc *supercluster) waitOnLeader() {\n\tsc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfor _, c := range sc.clusters {\n\t\t\tif leader := c.leader(); leader != nil {\n\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(25 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(sc.t, \"Timeout in supercluster.waitOnStreamLeader\", nil)\n\tsc.t.Fatalf(\"Expected a cluster leader, got none\")\n}\n\nfunc (sc *supercluster) waitOnAccount(account string) {\n\tsc.t.Helper()\n\texpires := time.Now().Add(40 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfound := true\n\t\tfor _, c := range sc.clusters {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tacc, err := s.fetchAccount(account)\n\t\t\t\tfound = found && err == nil && acc != nil\n\t\t\t}\n\t\t}\n\t\tif found {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tcontinue\n\t}\n\n\tsc.t.Fatalf(\"Expected account %q to exist but didn't\", account)\n}\n\nfunc (sc *supercluster) waitOnAllCurrent() {\n\tsc.t.Helper()\n\tfor _, c := range sc.clusters {\n\t\tc.waitOnAllCurrent()\n\t}\n}\n\nfunc (sc *supercluster) clusterForName(name string) *cluster {\n\tfor _, c := range sc.clusters {\n\t\tif c.name == name {\n\t\t\treturn c\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (sc *supercluster) randomCluster() *cluster {\n\tclusters := append(sc.clusters[:0:0], sc.clusters...)\n\trand.Shuffle(len(clusters), func(i, j int) { clusters[i], clusters[j] = clusters[j], clusters[i] })\n\treturn clusters[0]\n}\n\nfunc (sc *supercluster) waitOnPeerCount(n int) {\n\tsc.t.Helper()\n\tsc.waitOnLeader()\n\tleader := sc.leader()\n\texpires := time.Now().Add(30 * time.Second)\n\t// Make sure we have all peers, and take into account the meta leader could still change.\n\tfor time.Now().Before(expires) {\n\t\tif leader = sc.leader(); leader == nil {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tcontinue\n\t\t}\n\t\tif len(leader.JetStreamClusterPeers()) == n {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(sc.t, \"Timeout in supercluster.waitOnPeerCount\", nil)\n\tsc.t.Fatalf(\"Expected a super cluster peer count of %d, got %d\", n, len(leader.JetStreamClusterPeers()))\n}\n\nvar jsClusterMirrorSourceImportsTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: dlc\n\n\taccounts {\n\t\tJS {\n\t\t\tjetstream: enabled\n\t\t\tusers = [ { user: \"rip\", pass: \"pass\" } ]\n\t\t\texports [\n\t\t\t\t{ service: \"$JS.API.CONSUMER.>\" } # To create internal consumers to mirror/source.\n\t\t\t\t{ stream: \"RI.DELIVER.SYNC.>\" } # For the mirror/source consumers sending to IA via delivery subject.\n\t\t\t\t{ service: \"$JS.FC.>\" }\n\t\t\t]\n\t\t}\n\t\tIA {\n\t\t\tjetstream: enabled\n\t\t\tusers = [ { user: \"dlc\", pass: \"pass\" } ]\n\t\t\timports [\n\t\t\t\t{ service: { account: JS, subject: \"$JS.API.CONSUMER.>\"}, to: \"RI.JS.API.CONSUMER.>\" }\n\t\t\t\t{ stream: { account: JS, subject: \"RI.DELIVER.SYNC.>\"} }\n\t\t\t\t{ service: {account: JS, subject: \"$JS.FC.>\" }}\n\t\t\t]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsClusterImportsTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tno_auth_user: dlc\n\n\taccounts {\n\t\tJS {\n\t\t\tjetstream: enabled\n\t\t\tusers = [ { user: \"rip\", pass: \"pass\" } ]\n\t\t\texports [\n\t\t\t\t{ service: \"$JS.API.>\", response: stream }\n\t\t\t\t{ service: \"TEST\" } # For publishing to the stream.\n\t\t\t\t{ service: \"$JS.ACK.TEST.*.>\" }\n\t\t\t]\n\t\t}\n\t\tIA {\n\t\t\tusers = [ { user: \"dlc\", pass: \"pass\" } ]\n\t\t\timports [\n\t\t\t\t{ service: { subject: \"$JS.API.>\", account: JS }}\n\t\t\t\t{ service: { subject: \"TEST\", account: JS }}\n\t\t\t\t{ service: { subject: \"$JS.ACK.TEST.*.>\", account: JS }}\n\t\t\t]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nfunc createMixedModeCluster(t testing.TB, tmpl string, clusterName, snPre string, numJsServers, numNonServers int, doJSConfig bool) *cluster {\n\tt.Helper()\n\n\tif clusterName == _EMPTY_ || numJsServers < 0 || numNonServers < 1 {\n\t\tt.Fatalf(\"Bad params\")\n\t}\n\n\tnumServers := numJsServers + numNonServers\n\tconst startClusterPort = 23232\n\n\t// Build out the routes that will be shared with all configs.\n\tvar routes []string\n\tfor cp := startClusterPort; cp < startClusterPort+numServers; cp++ {\n\t\troutes = append(routes, fmt.Sprintf(\"nats-route://127.0.0.1:%d\", cp))\n\t}\n\trouteConfig := strings.Join(routes, \",\")\n\n\t// Go ahead and build configurations and start servers.\n\tc := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName}\n\n\tfor cp := startClusterPort; cp < startClusterPort+numServers; cp++ {\n\t\tstoreDir := t.TempDir()\n\n\t\tsn := fmt.Sprintf(\"%sS-%d\", snPre, cp-startClusterPort+1)\n\t\tconf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig)\n\n\t\t// Disable JS here.\n\t\tif cp-startClusterPort >= numJsServers {\n\t\t\t// We can disable by commmenting it out, meaning no JS config, or can set the config up and just set disabled.\n\t\t\t// e.g. jetstream: {domain: \"SPOKE\", enabled: false}\n\t\t\tif doJSConfig {\n\t\t\t\tconf = strings.Replace(conf, \"jetstream: {\", \"jetstream: { enabled: false, \", 1)\n\t\t\t} else {\n\t\t\t\tconf = strings.Replace(conf, \"jetstream: \", \"# jetstream: \", 1)\n\t\t\t}\n\t\t}\n\n\t\ts, o := RunServerWithConfig(createConfFile(t, []byte(conf)))\n\t\tc.servers = append(c.servers, s)\n\t\tc.opts = append(c.opts, o)\n\t}\n\tc.t = t\n\n\t// Wait til we are formed and have a leader.\n\tc.checkClusterFormed()\n\tif numJsServers > 0 {\n\t\tc.waitOnPeerCount(numJsServers)\n\t}\n\n\treturn c\n}\n\n// This will create a cluster that is explicitly configured for the routes, etc.\n// and also has a defined clustername. All configs for routes and cluster name will be the same.\nfunc createJetStreamClusterExplicit(t testing.TB, clusterName string, numServers int) *cluster {\n\treturn createJetStreamClusterWithTemplate(t, jsClusterTempl, clusterName, numServers)\n}\n\nfunc createJetStreamClusterWithTemplate(t testing.TB, tmpl string, clusterName string, numServers int) *cluster {\n\treturn createJetStreamClusterWithTemplateAndModHook(t, tmpl, clusterName, numServers, nil)\n}\n\nfunc createJetStreamClusterWithTemplateAndModHook(t testing.TB, tmpl string, clusterName string, numServers int, modify modifyCb) *cluster {\n\tstartPorts := []int{7_022, 9_022, 11_022, 15_022}\n\tport := startPorts[rand.Intn(len(startPorts))]\n\treturn createJetStreamClusterAndModHook(t, tmpl, clusterName, _EMPTY_, numServers, port, true, modify)\n}\n\nfunc createJetStreamCluster(t testing.TB, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool) *cluster {\n\treturn createJetStreamClusterAndModHook(t, tmpl, clusterName, snPre, numServers, portStart, waitOnReady, nil)\n}\n\ntype modifyCb func(serverName, clusterName, storeDir, conf string) string\n\nfunc createJetStreamClusterAndModHook(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, waitOnReady bool, modify modifyCb) *cluster {\n\treturn createJetStreamClusterEx(t, tmpl, cName, snPre, numServers, portStart, waitOnReady, modify, nil)\n}\n\nfunc createJetStreamClusterWithNetProxy(t testing.TB, cName string, numServers int, cnp *clusterProxy) *cluster {\n\tstartPorts := []int{7_122, 9_122, 11_122, 15_122}\n\tport := startPorts[rand.Intn(len(startPorts))]\n\treturn createJetStreamClusterEx(t, jsClusterTempl, cName, _EMPTY_, numServers, port, true, nil, cnp)\n}\n\nfunc createJetStreamClusterEx(t testing.TB, tmpl, cName, snPre string, numServers int, portStart int, wait bool, modify modifyCb, cnp *clusterProxy) *cluster {\n\tt.Helper()\n\tif cName == _EMPTY_ || numServers < 1 {\n\t\tt.Fatalf(\"Bad params\")\n\t}\n\n\t// Flaky test prevention:\n\t// Binding a socket to IP stack port 0 will bind an ephemeral port from an OS-specific range.\n\t// If someone passes in to us a port spec which would cover that range, the test would be flaky.\n\t// Adjust these ports to be the most inclusive across the port runner OSes.\n\t// Linux: /proc/sys/net/ipv4/ip_local_port_range : 32768:60999\n\t// is useful, and shows there's no safe available range without OS-specific tuning.\n\t// Our tests are usually run on Linux. Folks who care about other OSes: if you can't tune your test-runner OS to match, please\n\t// propose a viable alternative.\n\tconst prohibitedPortFirst = 32768\n\tconst prohibitedPortLast = 60999\n\tif (portStart >= prohibitedPortFirst && portStart <= prohibitedPortLast) ||\n\t\t(portStart+numServers-1 >= prohibitedPortFirst && portStart+numServers-1 <= prohibitedPortLast) {\n\t\tt.Fatalf(\"test setup failure: may not specify a cluster port range which falls within %d:%d\", prohibitedPortFirst, prohibitedPortLast)\n\t}\n\n\t// Build out the routes that will be shared with all configs.\n\tvar routes []string\n\tvar nproxies []*netProxy\n\tfor cp := portStart; cp < portStart+numServers; cp++ {\n\t\trouteURL := fmt.Sprintf(\"nats-route://127.0.0.1:%d\", cp)\n\t\tif cnp != nil {\n\t\t\tnp := createNetProxy(cnp.rtt, cnp.up, cnp.down, routeURL, false)\n\t\t\tnproxies = append(nproxies, np)\n\t\t\trouteURL = np.routeURL()\n\t\t}\n\t\troutes = append(routes, routeURL)\n\t}\n\trouteConfig := strings.Join(routes, \",\")\n\n\t// Go ahead and build configurations and start servers.\n\tc := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: cName, nproxies: nproxies}\n\n\t// Start any proxies.\n\tfor _, np := range nproxies {\n\t\tnp.start()\n\t}\n\n\tfor cp := portStart; cp < portStart+numServers; cp++ {\n\t\tstoreDir := t.TempDir()\n\t\tsn := fmt.Sprintf(\"%sS-%d\", snPre, cp-portStart+1)\n\t\tconf := fmt.Sprintf(tmpl, sn, storeDir, cName, cp, routeConfig)\n\t\tif modify != nil {\n\t\t\tconf = modify(sn, cName, storeDir, conf)\n\t\t}\n\t\ts, o := RunServerWithConfig(createConfFile(t, []byte(conf)))\n\t\tc.servers = append(c.servers, s)\n\t\tc.opts = append(c.opts, o)\n\t}\n\tc.t = t\n\n\t// Wait til we are formed and have a leader.\n\tc.checkClusterFormed()\n\tif wait {\n\t\tc.waitOnClusterReady()\n\t}\n\n\treturn c\n}\n\nfunc (c *cluster) addInNewServer() *Server {\n\tc.t.Helper()\n\tsn := fmt.Sprintf(\"S-%d\", len(c.servers)+1)\n\tstoreDir := c.t.TempDir()\n\tseedRoute := fmt.Sprintf(\"nats-route://127.0.0.1:%d\", c.opts[0].Cluster.Port)\n\tconf := fmt.Sprintf(jsClusterTempl, sn, storeDir, c.name, -1, seedRoute)\n\ts, o := RunServerWithConfig(createConfFile(c.t, []byte(conf)))\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\tc.checkClusterFormed()\n\treturn s\n}\n\n// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here.\nfunc (c *cluster) createSingleLeafNodeNoSystemAccount() *Server {\n\tas := c.randomServer()\n\tlno := as.getOpts().LeafNode\n\tln1 := fmt.Sprintf(\"nats://one:p@%s:%d\", lno.Host, lno.Port)\n\tln2 := fmt.Sprintf(\"nats://two:p@%s:%d\", lno.Host, lno.Port)\n\tconf := fmt.Sprintf(jsClusterSingleLeafNodeTempl, c.t.TempDir(), ln1, ln2)\n\ts, o := RunServerWithConfig(createConfFile(c.t, []byte(conf)))\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\n\tcheckLeafNodeConnectedCount(c.t, as, 2)\n\n\treturn s\n}\n\n// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here.\nfunc (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStream() *Server {\n\treturn c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(_EMPTY_, \"nojs\")\n}\n\nfunc (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(domain, user string) *Server {\n\ttmpl := jsClusterSingleLeafNodeLikeNGSTempl\n\tif domain != _EMPTY_ {\n\t\tnsc := fmt.Sprintf(\"domain: %s, store_dir:\", domain)\n\t\ttmpl = strings.Replace(jsClusterSingleLeafNodeLikeNGSTempl, \"store_dir:\", nsc, 1)\n\t}\n\tas := c.randomServer()\n\tlno := as.getOpts().LeafNode\n\tln := fmt.Sprintf(\"nats://%s:p@%s:%d\", user, lno.Host, lno.Port)\n\tconf := fmt.Sprintf(tmpl, c.t.TempDir(), ln)\n\ts, o := RunServerWithConfig(createConfFile(c.t, []byte(conf)))\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\n\tcheckLeafNodeConnectedCount(c.t, as, 1)\n\n\treturn s\n}\n\nvar jsClusterSingleLeafNodeLikeNGSTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: LNJS\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf { remotes [ { urls: [ %s ] } ] }\n`\n\nvar jsClusterSingleLeafNodeTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: LNJS\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf { remotes [\n\t\t{ urls: [ %s ], account: \"JSY\" }\n\t\t{ urls: [ %s ], account: \"JSN\" } ]\n\t}\n\n\taccounts {\n\t\tJSY { users = [ { user: \"y\", pass: \"p\" } ]; jetstream: true }\n\t\tJSN { users = [ { user: \"n\", pass: \"p\" } ] }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\nvar jsClusterTemplWithLeafNode = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\t{{leaf}}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterTemplWithLeafNodeNoJS = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\n\t# Need to keep below since it fills in the store dir by default so just comment out.\n\t# jetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\t{{leaf}}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterTemplWithSingleLeafNode = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\t{{leaf}}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterTemplWithSingleFleetLeafNode = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tcluster: { name: fleet }\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\t{{leaf}}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsClusterTemplWithSingleLeafNodeNoJS = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\n\t# jetstream: {store_dir: '%s'}\n\n\t{{leaf}}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\nvar jsLeafFrag = `\n\tleaf {\n\t\tremotes [\n\t\t\t{ urls: [ %s ] }\n\t\t\t{ urls: [ %s ], account: \"$SYS\" }\n\t\t]\n\t}\n`\nvar jsLeafNoSysFrag = `\n\tleaf {\n\t\tremotes [\n\t\t\t{ urls: [ %s ] }\n\t\t]\n\t}\n`\n\nfunc (c *cluster) createLeafNodes(clusterName string, numServers int, domain string) *cluster {\n\tc.t.Helper()\n\treturn c.createLeafNodesWithStartPortAndDomain(clusterName, numServers, 22111, domain)\n}\n\nfunc (c *cluster) createLeafNodesNoJS(clusterName string, numServers int) *cluster {\n\tc.t.Helper()\n\treturn c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNodeNoJS, clusterName, numServers, 21333)\n}\n\nfunc (c *cluster) createLeafNodesWithStartPortAndDomain(clusterName string, numServers int, portStart int, domain string) *cluster {\n\tc.t.Helper()\n\tif domain == _EMPTY_ {\n\t\treturn c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNode, clusterName, numServers, portStart)\n\t}\n\ttmpl := strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, domain), 1)\n\treturn c.createLeafNodesWithTemplateAndStartPort(tmpl, clusterName, numServers, portStart)\n}\n\nfunc (c *cluster) createLeafNode(extend bool) *Server {\n\tc.t.Helper()\n\tif extend {\n\t\treturn c.createLeafNodeWithTemplate(\"LNS\",\n\t\t\tstrings.ReplaceAll(jsClusterTemplWithSingleLeafNode, \"store_dir:\", \" extension_hint: will_extend, store_dir:\"))\n\t} else {\n\t\treturn c.createLeafNodeWithTemplate(\"LNS\", jsClusterTemplWithSingleLeafNode)\n\t}\n}\n\nfunc (c *cluster) createLeafNodeWithTemplate(name, template string) *Server {\n\tc.t.Helper()\n\ttmpl := c.createLeafSolicit(template)\n\tconf := fmt.Sprintf(tmpl, name, c.t.TempDir())\n\ts, o := RunServerWithConfig(createConfFile(c.t, []byte(conf)))\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\treturn s\n}\n\nfunc (c *cluster) createLeafNodeWithTemplateNoSystem(name, template string) *Server {\n\treturn c.createLeafNodeWithTemplateNoSystemWithProto(name, template, \"nats\")\n}\n\nfunc (c *cluster) createLeafNodeWithTemplateNoSystemWithProto(name, template, proto string) *Server {\n\tc.t.Helper()\n\ttmpl := c.createLeafSolicitNoSystemWithProto(template, proto)\n\tconf := fmt.Sprintf(tmpl, name, c.t.TempDir())\n\ts, o := RunServerWithConfig(createConfFile(c.t, []byte(conf)))\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\treturn s\n}\n\n// Helper to generate the leaf solicit configs.\nfunc (c *cluster) createLeafSolicit(tmpl string) string {\n\treturn c.createLeafSolicitWithProto(tmpl, \"nats\")\n}\n\nfunc (c *cluster) createLeafSolicitWithProto(tmpl, proto string) string {\n\tc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\tvar lns, lnss []string\n\tfor _, s := range c.servers {\n\t\tif s.ClusterName() != c.name {\n\t\t\tcontinue\n\t\t}\n\t\tln := s.getOpts().LeafNode\n\t\tlns = append(lns, fmt.Sprintf(\"%s://%s:%d\", proto, ln.Host, ln.Port))\n\t\tlnss = append(lnss, fmt.Sprintf(\"%s://admin:s3cr3t!@%s:%d\", proto, ln.Host, ln.Port))\n\t}\n\tlnc := strings.Join(lns, \", \")\n\tlnsc := strings.Join(lnss, \", \")\n\tlconf := fmt.Sprintf(jsLeafFrag, lnc, lnsc)\n\treturn strings.Replace(tmpl, \"{{leaf}}\", lconf, 1)\n}\n\nfunc (c *cluster) createLeafSolicitNoSystemWithProto(tmpl, proto string) string {\n\tc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\tvar lns []string\n\tfor _, s := range c.servers {\n\t\tif s.ClusterName() != c.name {\n\t\t\tcontinue\n\t\t}\n\t\tswitch proto {\n\t\tcase \"nats\", \"tls\":\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tlns = append(lns, fmt.Sprintf(\"%s://%s:%d\", proto, ln.Host, ln.Port))\n\t\tcase \"ws\", \"wss\":\n\t\t\tln := s.getOpts().Websocket\n\t\t\tlns = append(lns, fmt.Sprintf(\"%s://%s:%d\", proto, ln.Host, ln.Port))\n\t\t}\n\t}\n\tlnc := strings.Join(lns, \", \")\n\treturn strings.Replace(tmpl, \"{{leaf}}\", fmt.Sprintf(jsLeafNoSysFrag, lnc), 1)\n}\n\nfunc (c *cluster) createLeafNodesWithTemplateMixedMode(template, clusterName string, numJsServers, numNonServers int, doJSConfig bool) *cluster {\n\tc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\ttmpl := c.createLeafSolicit(template)\n\tpre := clusterName + \"-\"\n\n\tlc := createMixedModeCluster(c.t, tmpl, clusterName, pre, numJsServers, numNonServers, doJSConfig)\n\tfor _, s := range lc.servers {\n\t\tcheckLeafNodeConnectedCount(c.t, s, 2)\n\t}\n\tlc.waitOnClusterReadyWithNumPeers(numJsServers)\n\n\treturn lc\n}\n\nfunc (c *cluster) createLeafNodesWithTemplateAndStartPort(template, clusterName string, numServers int, portStart int) *cluster {\n\tc.t.Helper()\n\n\t// Create our leafnode cluster template first.\n\ttmpl := c.createLeafSolicit(template)\n\tpre := clusterName + \"-\"\n\tlc := createJetStreamCluster(c.t, tmpl, clusterName, pre, numServers, portStart, false)\n\tfor _, s := range lc.servers {\n\t\tcheckLeafNodeConnectedCount(c.t, s, 2)\n\t}\n\treturn lc\n}\n\n// Helper function to close and disable leafnodes.\nfunc (s *Server) closeAndDisableLeafnodes() {\n\tvar leafs []*client\n\ts.mu.Lock()\n\tfor _, ln := range s.leafs {\n\t\tleafs = append(leafs, ln)\n\t}\n\t// Disable leafnodes for now.\n\ts.leafDisableConnect = true\n\ts.mu.Unlock()\n\n\tfor _, ln := range leafs {\n\t\tln.closeConnection(Revocation)\n\t}\n}\n\n// Helper function to re-enable leafnode connections.\nfunc (s *Server) reEnableLeafnodes() {\n\ts.mu.Lock()\n\t// Re-enable leafnodes.\n\ts.leafDisableConnect = false\n\ts.mu.Unlock()\n}\n\n// Helper to set the remote migrate feature.\nfunc (s *Server) setJetStreamMigrateOnRemoteLeaf() {\n\ts.mu.Lock()\n\tfor _, cfg := range s.leafRemoteCfgs {\n\t\tcfg.JetStreamClusterMigrate = true\n\t}\n\ts.mu.Unlock()\n}\n\n// Helper to set the remote migrate feature.\nfunc (s *Server) setJetStreamMigrateOnRemoteLeafWithDelay(delay time.Duration) {\n\ts.mu.Lock()\n\tfor _, cfg := range s.leafRemoteCfgs {\n\t\tcfg.JetStreamClusterMigrate = true\n\t\tcfg.JetStreamClusterMigrateDelay = delay\n\t}\n\ts.mu.Unlock()\n}\n\n// Will add in the mapping for the account to each server.\nfunc (c *cluster) addSubjectMapping(account, src, dest string) {\n\tc.t.Helper()\n\n\tfor _, s := range c.servers {\n\t\tif s.ClusterName() != c.name {\n\t\t\tcontinue\n\t\t}\n\t\tacc, err := s.LookupAccount(account)\n\t\tif err != nil {\n\t\t\tc.t.Fatalf(\"Unexpected error on %v: %v\", s, err)\n\t\t}\n\t\tif err := acc.AddMapping(src, dest); err != nil {\n\t\t\tc.t.Fatalf(\"Error adding mapping: %v\", err)\n\t\t}\n\t}\n\t// Make sure interest propagates.\n\ttime.Sleep(200 * time.Millisecond)\n}\n\n// Adjust limits for the given account.\nfunc (c *cluster) updateLimits(account string, newLimits map[string]JetStreamAccountLimits) {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\tacc, err := s.LookupAccount(account)\n\t\tif err != nil {\n\t\t\tc.t.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif err := acc.UpdateJetStreamLimits(newLimits); err != nil {\n\t\t\tc.t.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n}\n\n// Hack for staticcheck\nvar skip = func(t *testing.T) {\n\tt.SkipNow()\n}\n\nfunc jsClientConnect(t testing.TB, s *Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) {\n\tt.Helper()\n\tnc, err := nats.Connect(s.ClientURL(), opts...)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tjs, err := nc.JetStream(nats.MaxWait(10 * time.Second))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\treturn nc, js\n}\n\nfunc jsClientConnectEx(t testing.TB, s *Server, jsOpts []nats.JSOpt, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) {\n\tt.Helper()\n\tnc, err := nats.Connect(s.ClientURL(), opts...)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tjo := []nats.JSOpt{nats.MaxWait(10 * time.Second)}\n\tif len(jsOpts) > 0 {\n\t\tjo = append(jo, jsOpts...)\n\t}\n\tjs, err := nc.JetStream(jo...)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\treturn nc, js\n}\n\nfunc jsClientConnectNewAPI(t testing.TB, s *Server, opts ...nats.Option) (*nats.Conn, jetstream.JetStream) {\n\tt.Helper()\n\tnc, err := nats.Connect(s.ClientURL(), opts...)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tjs, err := jetstream.New(nc, jetstream.WithDefaultTimeout(10*time.Second))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\treturn nc, js\n}\n\nfunc jsClientConnectURL(t testing.TB, url string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) {\n\tt.Helper()\n\n\tnc, err := nats.Connect(url, opts...)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tjs, err := nc.JetStream(nats.MaxWait(10 * time.Second))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\treturn nc, js\n}\n\n// jsStreamCreate is for sending a stream create for fields that nats.go does not know about yet.\nfunc jsStreamCreate(t testing.TB, nc *nats.Conn, cfg *StreamConfig) (*StreamConfig, error) {\n\tt.Helper()\n\n\tj, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), j, time.Second*3)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiStreamUpdateResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\n\tif resp.Error != nil {\n\t\treturn nil, resp.Error\n\t}\n\n\trequire_NotNil(t, resp.StreamInfo)\n\treturn &resp.Config, nil\n}\n\n// jsStreamUpdate is for sending a stream create for fields that nats.go does not know about yet.\nfunc jsStreamUpdate(t testing.TB, nc *nats.Conn, cfg *StreamConfig) (*StreamConfig, error) {\n\tt.Helper()\n\n\tj, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), j, time.Second*3)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiStreamUpdateResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\n\tif resp.Error != nil {\n\t\treturn nil, resp.Error\n\t}\n\n\trequire_NotNil(t, resp.StreamInfo)\n\treturn &resp.Config, nil\n}\n\nfunc checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) {\n\tt.Helper()\n\tcheckFor(t, 10*time.Second, 20*time.Millisecond, func() error {\n\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc fetchMsgs(t *testing.T, sub *nats.Subscription, numExpected int, totalWait time.Duration) []*nats.Msg {\n\tt.Helper()\n\tresult := make([]*nats.Msg, 0, numExpected)\n\tfor start, count, wait := time.Now(), numExpected, totalWait; len(result) != numExpected; {\n\t\tmsgs, err := sub.Fetch(count, nats.MaxWait(wait))\n\t\tif err != nil {\n\t\t\tantithesis.AssertUnreachable(t, \"Fetch error\", map[string]any{\n\t\t\t\t\"error\": err.Error(),\n\t\t\t})\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tresult = append(result, msgs...)\n\t\tcount -= len(msgs)\n\t\tif wait = totalWait - time.Since(start); wait < 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(result) != numExpected {\n\t\tantithesis.AssertUnreachable(t, \"Unexpected fetch messages count\", map[string]any{\n\t\t\t\"expected\": numExpected,\n\t\t\t\"actual\": len(result),\n\t\t})\n\t\tt.Fatalf(\"Unexpected msg count, got %d, want %d\", len(result), numExpected)\n\t}\n\treturn result\n}\n\nfunc (c *cluster) restartServer(rs *Server) *Server {\n\tc.t.Helper()\n\tindex := -1\n\tvar opts *Options\n\tfor i, s := range c.servers {\n\t\tif s == rs {\n\t\t\tindex = i\n\t\t\tbreak\n\t\t}\n\t}\n\tif index < 0 {\n\t\tc.t.Fatalf(\"Could not find server %v to restart\", rs)\n\t}\n\topts = c.opts[index]\n\ts, o := RunServerWithConfig(opts.ConfigFile)\n\tc.servers[index] = s\n\tc.opts[index] = o\n\treturn s\n}\n\nfunc (c *cluster) checkClusterFormed() {\n\tc.t.Helper()\n\tcheckClusterFormed(c.t, c.servers...)\n}\n\nfunc (c *cluster) waitOnPeerCount(n int) {\n\tc.t.Helper()\n\tc.waitOnLeader()\n\tleader := c.leader()\n\texpires := time.Now().Add(30 * time.Second)\n\t// Make sure we have all peers, and take into account the meta leader could still change.\n\tfor time.Now().Before(expires) {\n\t\tif leader = c.leader(); leader == nil {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tcontinue\n\t\t}\n\t\tif len(leader.JetStreamClusterPeers()) == n {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnPeerCount\", map[string]any{\n\t\t\"cluster\": c.name,\n\t})\n\tc.t.Fatalf(\"Expected a cluster peer count of %d, got %d\", n, len(leader.JetStreamClusterPeers()))\n}\n\nfunc (c *cluster) waitOnConsumerLeader(account, stream, consumer string) {\n\tc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tif leader := c.consumerLeader(account, stream, consumer); leader != nil {\n\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnConsumerLeader\", map[string]any{\n\t\t\"cluster\": c.name,\n\t\t\"account\": account,\n\t\t\"stream\": stream,\n\t\t\"consumer\": consumer,\n\t})\n\tc.t.Fatalf(\"Expected a consumer leader for %q %q %q, got none\", account, stream, consumer)\n}\n\nfunc (c *cluster) consumerLeader(account, stream, consumer string) *Server {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\tif s.JetStreamIsConsumerLeader(account, stream, consumer) {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) randomNonConsumerLeader(account, stream, consumer string) *Server {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\tif !s.JetStreamIsConsumerLeader(account, stream, consumer) {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) waitOnStreamLeader(account, stream string) {\n\tc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tif leader := c.streamLeader(account, stream); leader != nil {\n\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnStreamLeader\", map[string]any{\n\t\t\"cluster\": c.name,\n\t\t\"account\": account,\n\t\t\"stream\": stream,\n\t})\n\tc.t.Fatalf(\"Expected a stream leader for %q %q, got none\", account, stream)\n}\n\nfunc (c *cluster) randomNonStreamLeader(account, stream string) *Server {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\tif s.JetStreamIsStreamAssigned(account, stream) && !s.JetStreamIsStreamLeader(account, stream) {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) streamLeader(account, stream string) *Server {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\tif s.JetStreamIsStreamLeader(account, stream) {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) waitOnStreamCurrent(s *Server, account, stream string) {\n\tc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tif s.JetStreamIsStreamCurrent(account, stream) {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnStreamCurrent\", map[string]any{\n\t\t\"cluster\": c.name,\n\t\t\"account\": account,\n\t\t\"stream\": stream,\n\t})\n\tc.t.Fatalf(\"Expected server %q to eventually be current for stream %q\", s, stream)\n}\n\nfunc (c *cluster) waitOnServerHealthz(s *Server) {\n\tc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\ths := s.healthz(nil)\n\t\tif hs.Status == \"ok\" && hs.Error == _EMPTY_ {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnServerHealthz\", map[string]any{\n\t\t\"cluster\": c.name,\n\t\t\"server\": s.Name(),\n\t})\n\tc.t.Fatalf(\"Expected server %q to eventually return healthz 'ok', but got %q\", s, s.healthz(nil).Error)\n}\n\nfunc (c *cluster) waitOnServerCurrent(s *Server) {\n\tc.t.Helper()\n\texpires := time.Now().Add(30 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tif !s.JetStreamEnabled() || s.JetStreamIsCurrent() {\n\t\t\treturn\n\t\t}\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnServerCurrent\", map[string]any{\n\t\t\"cluster\": c.name,\n\t\t\"server\": s.Name(),\n\t})\n\tc.t.Fatalf(\"Expected server %q to eventually be current\", s)\n}\n\nfunc (c *cluster) waitOnAllCurrent() {\n\tc.t.Helper()\n\tfor _, cs := range c.servers {\n\t\tc.waitOnServerCurrent(cs)\n\t}\n}\n\nfunc (c *cluster) serverByName(sname string) *Server {\n\tfor _, s := range c.servers {\n\t\tif s.Name() == sname {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) randomNonLeader() *Server {\n\t// range should randomize.. but..\n\tfor _, s := range c.servers {\n\t\tif s.Running() && !s.JetStreamIsLeader() {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) leader() *Server {\n\tfor _, s := range c.servers {\n\t\tif s.JetStreamIsLeader() {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *cluster) expectNoLeader() {\n\tc.t.Helper()\n\texpires := time.Now().Add(maxElectionTimeout)\n\tfor time.Now().Before(expires) {\n\t\tif c.leader() == nil {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(20 * time.Millisecond)\n\t}\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.expectNoLeader\", map[string]any{\n\t\t\"cluster\": c.name,\n\t})\n\tc.t.Fatalf(\"Expected no leader but have one\")\n}\n\nfunc (c *cluster) waitOnLeader() {\n\tc.t.Helper()\n\texpires := time.Now().Add(40 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tif leader := c.leader(); leader != nil {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnLeader\", map[string]any{\n\t\t\"cluster\": c.name,\n\t})\n\tc.t.Fatalf(\"Expected a cluster leader, got none\")\n}\n\nfunc (c *cluster) waitOnAccount(account string) {\n\tc.t.Helper()\n\texpires := time.Now().Add(40 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfound := true\n\t\tfor _, s := range c.servers {\n\t\t\ts.optsMu.RLock()\n\t\t\twantJS := s.opts.JetStream\n\t\t\ts.optsMu.RUnlock()\n\t\t\tacc, err := s.fetchAccount(account)\n\t\t\tif found = found && err == nil && acc != nil; wantJS {\n\t\t\t\tfound = found && acc.JetStreamEnabled()\n\t\t\t}\n\t\t}\n\t\tif found {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tcontinue\n\t}\n\n\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnAccount\", map[string]any{\n\t\t\"account\": account,\n\t})\n\tc.t.Fatalf(\"Expected account %q to exist but didn't\", account)\n}\n\n// Helper function to check that a cluster is formed\nfunc (c *cluster) waitOnClusterReady() {\n\tc.t.Helper()\n\tc.waitOnClusterReadyWithNumPeers(len(c.servers))\n\tc.waitOnLeader()\n}\n\nfunc (c *cluster) waitOnClusterReadyWithNumPeers(numPeersExpected int) {\n\tc.t.Helper()\n\tvar leader *Server\n\texpires := time.Now().Add(40 * time.Second)\n\t// Make sure we have all peers, and take into account the meta leader could still change.\n\tfor time.Now().Before(expires) {\n\t\tif leader = c.leader(); leader == nil {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tcontinue\n\t\t}\n\t\tif len(leader.JetStreamClusterPeers()) == numPeersExpected {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n\n\tif leader == nil {\n\t\tc.shutdown()\n\t\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnClusterReadyWithNumPeers (1)\", map[string]any{\n\t\t\t\"cluster\": c.name,\n\t\t})\n\t\tc.t.Fatalf(\"Failed to elect a meta-leader\")\n\t}\n\n\tpeersSeen := len(leader.JetStreamClusterPeers())\n\tc.shutdown()\n\n\tif leader == nil {\n\t\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnClusterReadyWithNumPeers (2)\", map[string]any{\n\t\t\t\"cluster\": c.name,\n\t\t})\n\t\tc.t.Fatalf(\"Expected a cluster leader and fully formed cluster, no leader\")\n\t} else {\n\t\tantithesis.AssertUnreachable(c.t, \"Timeout in cluster.waitOnClusterReadyWithNumPeers (3)\", map[string]any{\n\t\t\t\"cluster\": c.name,\n\t\t\t\"peers_expected\": numPeersExpected,\n\t\t\t\"peers_seen\": peersSeen,\n\t\t})\n\t\tc.t.Fatalf(\"Expected a fully formed cluster, only %d of %d peers seen\", peersSeen, numPeersExpected)\n\t}\n}\n\n// Helper function to remove JetStream from a server.\nfunc (c *cluster) removeJetStream(s *Server) {\n\tc.t.Helper()\n\tindex := -1\n\tfor i, cs := range c.servers {\n\t\tif cs == s {\n\t\t\tindex = i\n\t\t\tbreak\n\t\t}\n\t}\n\tcf := c.opts[index].ConfigFile\n\tcb, _ := os.ReadFile(cf)\n\tvar sb strings.Builder\n\tfor _, l := range strings.Split(string(cb), \"\\n\") {\n\t\tif !strings.HasPrefix(strings.TrimSpace(l), \"jetstream\") {\n\t\t\tsb.WriteString(l + \"\\n\")\n\t\t}\n\t}\n\tif err := os.WriteFile(cf, []byte(sb.String()), 0644); err != nil {\n\t\tc.t.Fatalf(\"Error writing updated config file: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tc.t.Fatalf(\"Error on server reload: %v\", err)\n\t}\n\ttime.Sleep(100 * time.Millisecond)\n}\n\nfunc (c *cluster) stopAll() {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\ts.Shutdown()\n\t}\n\tfor _, s := range c.servers {\n\t\ts.WaitForShutdown()\n\t}\n}\n\nfunc (c *cluster) restartAll() {\n\tc.t.Helper()\n\tfor i, s := range c.servers {\n\t\tif !s.Running() {\n\t\t\topts := c.opts[i]\n\t\t\ts, o := RunServerWithConfig(opts.ConfigFile)\n\t\t\tc.servers[i] = s\n\t\t\tc.opts[i] = o\n\t\t}\n\t}\n\tc.waitOnClusterReady()\n}\n\nfunc (c *cluster) lameDuckRestartAll() {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\ts.lameDuckMode()\n\t}\n\tfor i, s := range c.servers {\n\t\ts.WaitForShutdown()\n\t\tif !s.Running() {\n\t\t\topts := c.opts[i]\n\t\t\ts, o := RunServerWithConfig(opts.ConfigFile)\n\t\t\tc.servers[i] = s\n\t\t\tc.opts[i] = o\n\t\t}\n\t}\n\tc.waitOnClusterReady()\n}\n\nfunc (c *cluster) restartAllSamePorts() {\n\tc.t.Helper()\n\tfor i, s := range c.servers {\n\t\tif !s.Running() {\n\t\t\topts := c.opts[i]\n\t\t\ts := RunServer(opts)\n\t\t\tc.servers[i] = s\n\t\t}\n\t}\n\tc.waitOnClusterReady()\n}\n\nfunc (c *cluster) totalSubs() (total int) {\n\tc.t.Helper()\n\tfor _, s := range c.servers {\n\t\ttotal += int(s.NumSubscriptions())\n\t}\n\treturn total\n}\n\nfunc (c *cluster) stableTotalSubs() (total int) {\n\tnsubs := -1\n\tcheckFor(c.t, 2*time.Second, 250*time.Millisecond, func() error {\n\t\tsubs := c.totalSubs()\n\t\tif subs == nsubs {\n\t\t\treturn nil\n\t\t}\n\t\tnsubs = subs\n\t\treturn fmt.Errorf(\"Still stabilizing\")\n\t})\n\treturn nsubs\n\n}\n\nfunc addStreamPedanticWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfigRequest) (*StreamInfo, *ApiError) {\n\tt.Helper()\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, 5*time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Type != JSApiStreamCreateResponseType {\n\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamCreateResponseType)\n\t}\n\treturn resp.StreamInfo, resp.Error\n}\n\nfunc updateStreamPedanticWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfigRequest) (*StreamInfo, *ApiError) {\n\tt.Helper()\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Type != JSApiStreamUpdateResponseType {\n\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamUpdateResponseType)\n\t}\n\treturn resp.StreamInfo, resp.Error\n}\n\nfunc addStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo {\n\tt.Helper()\n\tsi, err := addStreamWithError(t, nc, cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", err)\n\t}\n\treturn si\n}\n\nfunc addStreamWithError(t *testing.T, nc *nats.Conn, cfg *StreamConfig) (*StreamInfo, *ApiError) {\n\tt.Helper()\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, 5*time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Type != JSApiStreamCreateResponseType {\n\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamCreateResponseType)\n\t}\n\treturn resp.StreamInfo, resp.Error\n}\n\nfunc updateStream(t *testing.T, nc *nats.Conn, cfg *StreamConfig) *StreamInfo {\n\tt.Helper()\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Type != JSApiStreamUpdateResponseType {\n\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamUpdateResponseType)\n\t}\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t}\n\treturn resp.StreamInfo\n}\n\n// setInActiveDeleteThreshold sets the delete threshold for how long to wait\n// before deleting an inactive consumer.\nfunc (o *consumer) setInActiveDeleteThreshold(dthresh time.Duration) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tdeleteWasRunning := o.dtmr != nil\n\tstopAndClearTimer(&o.dtmr)\n\t// Do not add jitter if set via here.\n\to.dthresh = dthresh\n\tif deleteWasRunning {\n\t\to.dtmr = time.AfterFunc(o.dthresh, func() { o.deleteNotActive() })\n\t}\n\treturn nil\n}\n\n// Net Proxy - For introducing RTT and BW constraints.\ntype netProxy struct {\n\tsync.RWMutex\n\tlistener net.Listener\n\tconns []net.Conn\n\trtt time.Duration\n\tup int\n\tdown int\n\turl string\n\tsurl string\n\tport int\n}\n\nfunc newNetProxy(rtt time.Duration, upRate, downRate int, serverURL string) *netProxy {\n\treturn createNetProxy(rtt, upRate, downRate, serverURL, true)\n}\n\nfunc createNetProxy(rtt time.Duration, upRate, downRate int, serverURL string, start bool) *netProxy {\n\thp := net.JoinHostPort(\"127.0.0.1\", \"0\")\n\tl, e := net.Listen(\"tcp\", hp)\n\tif e != nil {\n\t\tpanic(fmt.Sprintf(\"Error listening on port: %s, %q\", hp, e))\n\t}\n\tu, err := url.Parse(serverURL)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Could not parse server URL: %v\", err))\n\t}\n\n\tvar clientURL string\n\tport := l.Addr().(*net.TCPAddr).Port\n\tif u.User != nil {\n\t\tclientURL = fmt.Sprintf(\"nats://%v@127.0.0.1:%d\", u.User, port)\n\t} else {\n\t\tclientURL = fmt.Sprintf(\"nats://127.0.0.1:%d\", port)\n\t}\n\n\tproxy := &netProxy{\n\t\tlistener: l,\n\t\trtt: rtt,\n\t\tup: upRate,\n\t\tdown: downRate,\n\t\turl: clientURL,\n\t\tsurl: serverURL,\n\t\tport: port,\n\t}\n\tif start {\n\t\tproxy.start()\n\t}\n\treturn proxy\n}\n\nfunc (np *netProxy) start() {\n\tu, err := url.Parse(np.surl)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Could not parse server URL: %v\", err))\n\t}\n\thost := u.Host\n\n\t// Check if this is restart.\n\t// We nil out listener on stop()\n\tif np.listener == nil && np.port != 0 {\n\t\thp := net.JoinHostPort(\"127.0.0.1\", fmt.Sprintf(\"%d\", np.port))\n\t\tnp.listener, err = net.Listen(\"tcp\", hp)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Sprintf(\"Error listening on port: %s, %q\", hp, err))\n\t\t}\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tclient, err := np.listener.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tserver, err := net.DialTimeout(\"tcp\", host, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tnp.conns = append(np.conns, client, server)\n\t\t\tgo np.loop(np.up, client, server)\n\t\t\tgo np.loop(np.down, server, client)\n\t\t}\n\t}()\n}\n\nfunc (np *netProxy) clientURL() string {\n\treturn np.url\n}\n\nfunc (np *netProxy) routeURL() string {\n\treturn strings.Replace(np.url, \"nats\", \"nats-route\", 1)\n}\n\nfunc (np *netProxy) leafURL() string {\n\treturn strings.Replace(np.url, \"nats\", \"nats-leaf\", 1)\n}\n\nfunc (np *netProxy) loop(tbw int, r, w net.Conn) {\n\tconst rbl = 8192\n\tvar buf [rbl]byte\n\tctx := context.Background()\n\n\trl := rate.NewLimiter(rate.Limit(tbw), rbl)\n\n\tfor {\n\t\tn, err := r.Read(buf[:])\n\t\tif err != nil {\n\t\t\tw.Close()\n\t\t\treturn\n\t\t}\n\t\t// RTT delays\n\t\tnp.RLock()\n\t\tdelay := np.rtt / 2\n\t\tnp.RUnlock()\n\t\tif delay > 0 {\n\t\t\ttime.Sleep(delay)\n\t\t}\n\t\tif err := rl.WaitN(ctx, n); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif _, err = w.Write(buf[:n]); err != nil {\n\t\t\tr.Close()\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (np *netProxy) updateRTT(rtt time.Duration) {\n\tnp.Lock()\n\tnp.rtt = rtt\n\tnp.Unlock()\n}\n\nfunc (np *netProxy) stop() {\n\tif np.listener != nil {\n\t\tnp.listener.Close()\n\t\tnp.listener = nil\n\t\tfor _, c := range np.conns {\n\t\t\tc.Close()\n\t\t}\n\t}\n}\n\n// Bitset, aka bitvector, allows tracking of large number of bits efficiently\ntype bitset struct {\n\t// Bit map storage\n\tbitmap []uint8\n\t// Number of bits currently set to 1\n\tcurrentCount uint64\n\t// Number of bits stored\n\tsize uint64\n}\n\nfunc NewBitset(size uint64) *bitset {\n\tbyteSize := (size + 7) / 8 //Round up to the nearest byte\n\n\treturn &bitset{\n\t\tbitmap: make([]uint8, int(byteSize)),\n\t\tsize: size,\n\t\tcurrentCount: 0,\n\t}\n}\n\nfunc (b *bitset) get(index uint64) bool {\n\tif index >= b.size {\n\t\tpanic(fmt.Sprintf(\"Index %d out of bounds, size %d\", index, b.size))\n\t}\n\tbyteIndex := index / 8\n\tbitIndex := uint(index % 8)\n\tbit := (b.bitmap[byteIndex] & (uint8(1) << bitIndex))\n\treturn bit != 0\n}\n\nfunc (b *bitset) set(index uint64, value bool) {\n\tif index >= b.size {\n\t\tpanic(fmt.Sprintf(\"Index %d out of bounds, size %d\", index, b.size))\n\t}\n\tbyteIndex := index / 8\n\tbitIndex := uint(index % 8)\n\tbyteMask := uint8(1) << bitIndex\n\tisSet := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) != 0\n\tif value {\n\t\tb.bitmap[byteIndex] |= byteMask\n\t\tif !isSet {\n\t\t\tb.currentCount += 1\n\t\t}\n\t} else {\n\t\tb.bitmap[byteIndex] &= ^byteMask\n\t\tif isSet {\n\t\t\tb.currentCount -= 1\n\t\t}\n\t}\n}\n\nfunc (b *bitset) count() uint64 {\n\treturn b.currentCount\n}\n\nfunc (b *bitset) String() string {\n\tconst block = 8 // 8 bytes, 64 bits per line\n\tsb := strings.Builder{}\n\n\tsb.WriteString(fmt.Sprintf(\"Bits set: %d/%d\\n\", b.currentCount, b.size))\n\tfor i := 0; i < len(b.bitmap); i++ {\n\t\tif i%block == 0 {\n\t\t\tif i > 0 {\n\t\t\t\tsb.WriteString(\"\\n\")\n\t\t\t}\n\t\t\tsb.WriteString(fmt.Sprintf(\"[%4d] \", i*8))\n\t\t}\n\t\tfor j := uint8(0); j < 8; j++ {\n\t\t\tif b.bitmap[i]&(1< 0 {\n\t\t\t\tsb.WriteString(\"1\")\n\t\t\t} else {\n\t\t\t\tsb.WriteString(\"0\")\n\t\t\t}\n\t\t}\n\t}\n\tsb.WriteString(\"\\n\")\n\treturn sb.String()\n}\n\nfunc copyDir(t *testing.T, dst, src string) error {\n\tt.Helper()\n\tsrcFS := os.DirFS(src)\n\treturn fs.WalkDir(srcFS, \".\", func(p string, d os.DirEntry, err error) error {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tnewPath := path.Join(dst, p)\n\t\tif d.IsDir() {\n\t\t\treturn os.MkdirAll(newPath, defaultDirPerms)\n\t\t}\n\t\tr, err := srcFS.Open(p)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer r.Close()\n\n\t\tw, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, defaultFilePerms)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer w.Close()\n\t\t_, err = io.Copy(w, r)\n\t\treturn err\n\t})\n}\n\nfunc getStreamDetails(t *testing.T, c *cluster, accountName, streamName string) *StreamDetail {\n\tt.Helper()\n\tsrv := c.streamLeader(accountName, streamName)\n\tif srv == nil {\n\t\treturn nil\n\t}\n\tjsz, err := srv.Jsz(&JSzOptions{Accounts: true, Streams: true, Consumer: true})\n\trequire_NoError(t, err)\n\tfor _, acc := range jsz.AccountDetails {\n\t\tif acc.Name == accountName {\n\t\t\tfor _, stream := range acc.Streams {\n\t\t\t\tif stream.Name == streamName {\n\t\t\t\t\treturn &stream\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tt.Error(\"Could not find account details\")\n\treturn nil\n}\n\nfunc checkState(t *testing.T, c *cluster, accountName, streamName string) error {\n\tt.Helper()\n\t_, err := checkStateAndErr(t, c, accountName, streamName)\n\treturn err\n}\n\nfunc checkStateAndErr(t *testing.T, c *cluster, accountName, streamName string) (StreamState, error) {\n\tt.Helper()\n\n\tleaderSrv := c.streamLeader(accountName, streamName)\n\tif leaderSrv == nil {\n\t\treturn StreamState{}, fmt.Errorf(\"no leader server found for stream %q\", streamName)\n\t}\n\tstreamLeader := getStreamDetails(t, c, accountName, streamName)\n\tif streamLeader == nil {\n\t\treturn StreamState{}, fmt.Errorf(\"no leader found for stream %q\", streamName)\n\t}\n\n\tacc, err := leaderSrv.LookupAccount(accountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(streamName)\n\trequire_NoError(t, err)\n\n\tcfgReplicas := mset.cfg.Replicas\n\tfoundReplicas := 1 // We already know the leader.\n\tvar errs []error\n\tfor _, srv := range c.servers {\n\t\tif srv == leaderSrv {\n\t\t\t// Skip self\n\t\t\tcontinue\n\t\t}\n\t\tif srv.isShuttingDown() {\n\t\t\t// Skip if shutdown.\n\t\t\tfor _, replica := range streamLeader.Cluster.Replicas {\n\t\t\t\tif replica.Name == srv.Name() {\n\t\t\t\t\tfoundReplicas++\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tacc, err = srv.LookupAccount(accountName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tmset, err = acc.lookupStream(streamName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tfoundReplicas++\n\t\tstate := mset.state()\n\n\t\tif state.Msgs != streamLeader.State.Msgs {\n\t\t\terr := fmt.Errorf(\"[%s] Leader %v has %d messages, Follower %v has %d messages\",\n\t\t\t\tstreamName, leaderSrv, streamLeader.State.Msgs,\n\t\t\t\tsrv, state.Msgs,\n\t\t\t)\n\t\t\terrs = append(errs, err)\n\t\t}\n\t\tif state.FirstSeq != streamLeader.State.FirstSeq {\n\t\t\terr := fmt.Errorf(\"[%s] Leader %v FirstSeq is %d, Follower %v is at %d\",\n\t\t\t\tstreamName, leaderSrv, streamLeader.State.FirstSeq,\n\t\t\t\tsrv, state.FirstSeq,\n\t\t\t)\n\t\t\terrs = append(errs, err)\n\t\t}\n\t\tif state.LastSeq != streamLeader.State.LastSeq {\n\t\t\terr := fmt.Errorf(\"[%s] Leader %v LastSeq is %d, Follower %v is at %d\",\n\t\t\t\tstreamName, leaderSrv, streamLeader.State.LastSeq,\n\t\t\t\tsrv, state.LastSeq,\n\t\t\t)\n\t\t\terrs = append(errs, err)\n\t\t}\n\t\tif state.NumDeleted != streamLeader.State.NumDeleted {\n\t\t\terr := fmt.Errorf(\"[%s] Leader %v NumDeleted is %d, Follower %v is at %d\\nSTATE_A: %+v\\nSTATE_B: %+v\\n\",\n\t\t\t\tstreamName, leaderSrv, streamLeader.State.NumDeleted,\n\t\t\t\tsrv, state.NumDeleted, streamLeader.State, state,\n\t\t\t)\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif cfgReplicas != foundReplicas {\n\t\terr := fmt.Errorf(\"[%s] Expected %d Replicas, got %d\\n\", streamName, cfgReplicas, foundReplicas)\n\t\terrs = append(errs, err)\n\t}\n\tif len(errs) > 0 {\n\t\treturn StreamState{}, errors.Join(errs...)\n\t}\n\treturn streamLeader.State, nil\n}\n" }, { "path": "server/jetstream_jwt_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests\n\npackage server\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\n\tjwt \"github.com/nats-io/jwt/v2\"\n)\n\nfunc TestJetStreamJWTLimits(t *testing.T) {\n\tupdateJwt := func(url string, creds string, pubKey string, jwt string) {\n\t\tt.Helper()\n\t\tc := natsConnect(t, url, nats.UserCredentials(creds))\n\t\tdefer c.Close()\n\t\tif msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t} else {\n\t\t\tcontent := make(map[string]any)\n\t\t\tif err := json.Unmarshal(msg.Data, &content); err != nil {\n\t\t\t\tt.Fatalf(\"%v\", err)\n\t\t\t} else if _, ok := content[\"data\"]; !ok {\n\t\t\t\tt.Fatalf(\"did not get an ok response got: %v\", content)\n\t\t\t}\n\t\t}\n\t}\n\trequire_IdenticalLimits := func(infoLim JetStreamAccountLimits, lim jwt.JetStreamLimits) {\n\t\tt.Helper()\n\t\tif int64(infoLim.MaxConsumers) != lim.Consumer || int64(infoLim.MaxStreams) != lim.Streams ||\n\t\t\tinfoLim.MaxMemory != lim.MemoryStorage || infoLim.MaxStore != lim.DiskStorage {\n\t\t\tt.Fatalf(\"limits do not match %v != %v\", infoLim, lim)\n\t\t}\n\t}\n\texpect_JSDisabledForAccount := func(c *nats.Conn) {\n\t\tt.Helper()\n\t\tif _, err := c.Request(\"$JS.API.INFO\", nil, time.Second); err != nats.ErrTimeout && err != nats.ErrNoResponders {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\texpect_InfoError := func(c *nats.Conn) {\n\t\tt.Helper()\n\t\tvar info JSApiAccountInfoResponse\n\t\tif resp, err := c.Request(\"$JS.API.INFO\", nil, time.Second); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t} else if err = json.Unmarshal(resp.Data, &info); err != nil {\n\t\t\tt.Fatalf(\"response1 %v got error %v\", string(resp.Data), err)\n\t\t} else if info.Error == nil {\n\t\t\tt.Fatalf(\"expected error\")\n\t\t}\n\t}\n\tvalidate_limits := func(c *nats.Conn, expectedLimits jwt.JetStreamLimits) {\n\t\tt.Helper()\n\t\tvar info JSApiAccountInfoResponse\n\t\tif resp, err := c.Request(\"$JS.API.INFO\", nil, time.Second); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t} else if err = json.Unmarshal(resp.Data, &info); err != nil {\n\t\t\tt.Fatalf(\"response1 %v got error %v\", string(resp.Data), err)\n\t\t} else {\n\t\t\trequire_IdenticalLimits(info.Limits, expectedLimits)\n\t\t}\n\t}\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysUKp, _ := nkeys.CreateUser()\n\tsysUSeed, _ := sysUKp.Seed()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject, _ = sysUKp.PublicKey()\n\tsysUserJwt, err := uclaim.Encode(sysKp)\n\trequire_NoError(t, err)\n\tsysKp.Seed()\n\tsysCreds := genCredsFile(t, sysUserJwt, sysUSeed)\n\t// limits to apply and check\n\tlimits1 := jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1, Consumer: 2, MaxBytesRequired: true}\n\t// has valid limits that would fail when incorrectly applied twice\n\tlimits2 := jwt.JetStreamLimits{MemoryStorage: 4096 * 1024, DiskStorage: 8192 * 1024, Streams: 3, Consumer: 4}\n\t// limits exceeding actual configured value of DiskStorage\n\tlimitsExceeded := jwt.JetStreamLimits{MemoryStorage: 8192 * 1024, DiskStorage: 16384 * 1024, Streams: 5, Consumer: 6}\n\t// create account using jetstream with both limits\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\tclaim.Limits.JetStreamLimits = limits1\n\taJwt1, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tclaim.Limits.JetStreamLimits = limits2\n\taJwt2, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tclaim.Limits.JetStreamLimits = limitsExceeded\n\taJwtLimitsExceeded, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{} // disabled\n\taJwt4, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\t// account user\n\tuKp, _ := nkeys.CreateUser()\n\tuSeed, _ := uKp.Seed()\n\tuclaim = newJWTTestUserClaims()\n\tuclaim.Subject, _ = uKp.PublicKey()\n\tuserJwt, err := uclaim.Encode(akp)\n\trequire_NoError(t, err)\n\tuserCreds := genCredsFile(t, userJwt, uSeed)\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: \"%s\"}\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, dir, ojwt, dir, sysPub)))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tport := opts.Port\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt1)\n\tc := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCreds), nats.ReconnectWait(200*time.Millisecond))\n\tdefer c.Close()\n\tvalidate_limits(c, limits1)\n\t// keep using the same connection\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt2)\n\tvalidate_limits(c, limits2)\n\t// keep using the same connection but do NOT CHANGE anything.\n\t// This tests if the jwt is applied a second time (would fail)\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt2)\n\tvalidate_limits(c, limits2)\n\t// keep using the same connection. This update EXCEEDS LIMITS\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded)\n\tvalidate_limits(c, limits2)\n\t// disable test after failure\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt4)\n\texpect_InfoError(c)\n\t// re enable, again testing with a value that can't be applied twice\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt2)\n\tvalidate_limits(c, limits2)\n\t// disable test no prior failure\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt4)\n\texpect_InfoError(c)\n\t// Wrong limits form start\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded)\n\texpect_JSDisabledForAccount(c)\n\t// enable js but exceed limits. Followed by fix via restart\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt2)\n\tvalidate_limits(c, limits2)\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded)\n\tvalidate_limits(c, limits2)\n\ts.Shutdown()\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:%d\n\t\tjetstream: {max_mem_store: 20Mb, max_file_store: 20Mb, store_dir: \"%s\"}\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, port, dir, ojwt, dir, sysPub)))\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tc.Flush() // force client to discover the disconnect\n\tcheckClientsCount(t, s, 1)\n\tvalidate_limits(c, limitsExceeded)\n\ts.Shutdown()\n\t// disable jetstream test\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:%d\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, port, ojwt, dir, sysPub)))\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tc.Flush() // force client to discover the disconnect\n\tcheckClientsCount(t, s, 1)\n\texpect_JSDisabledForAccount(c)\n\t// test that it stays disabled\n\tupdateJwt(s.ClientURL(), sysCreds, aPub, aJwt2)\n\texpect_JSDisabledForAccount(c)\n\tc.Close()\n}\n\nfunc TestJetStreamJWTDisallowBearer(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taccIdPub, err := accKp.PublicKey()\n\trequire_NoError(t, err)\n\taClaim := jwt.NewAccountClaims(accIdPub)\n\taccJwt1, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\taClaim.Limits.DisallowBearer = true\n\taccJwt2, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tuc := jwt.NewUserClaims(\"dummy\")\n\tuc.BearerToken = true\n\tuOpt1 := createUserCredsEx(t, uc, accKp)\n\tuc.BearerToken = false\n\tuOpt2 := createUserCredsEx(t, uc, accKp)\n\n\tdir := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s/jwt'\n\t\t}\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t}\n\t\t`, ojwt, syspub, dir, syspub, sysJwt)))\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\tdisconnectErrCh := make(chan error, 10)\n\tdefer close(disconnectErrCh)\n\tnc1, err := nats.Connect(s.ClientURL(), uOpt1,\n\t\tnats.NoReconnect(),\n\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\tdisconnectErrCh <- err\n\t\t}))\n\trequire_NoError(t, err)\n\tdefer nc1.Close()\n\n\t// update jwt and observe bearer token get disconnected\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1)\n\tselect {\n\tcase err := <-disconnectErrCh:\n\t\trequire_Contains(t, err.Error(), \"authorization violation\")\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"expected error on disconnect\")\n\t}\n\n\t// assure bearer token is not allowed to connect\n\t_, err = nats.Connect(s.ClientURL(), uOpt1)\n\trequire_Error(t, err)\n\n\t// assure non bearer token can connect\n\tnc2, err := nats.Connect(s.ClientURL(), uOpt2)\n\trequire_NoError(t, err)\n\tdefer nc2.Close()\n}\n\nfunc TestJetStreamJWTMove(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\n\ttest := func(t *testing.T, replicas int, accClaim *jwt.AccountClaims) {\n\t\taccClaim.Name = \"acc\"\n\t\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\t\taccCreds := newUser(t, accKp)\n\n\t\ttmlp := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\t\tleaf {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t\t`\n\t\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 5, 2,\n\t\t\tfunc(serverName, clustername, storeDir, conf string) string {\n\t\t\t\tswitch sname := serverName[strings.Index(serverName, \"-\")+1:]; sname {\n\t\t\t\tcase \"S1\", \"S2\":\n\t\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream\", \"#jetstream\")\n\t\t\t\t}\n\t\t\t\treturn conf + fmt.Sprintf(`\n\t\t\t\t\tserver_tags: [cloud:%s-tag]\n\t\t\t\t\toperator: %s\n\t\t\t\t\tsystem_account: %s\n\t\t\t\t\tresolver: {\n\t\t\t\t\t\ttype: full\n\t\t\t\t\t\tdir: '%s/jwt'\n\t\t\t\t\t}\n\t\t\t\t\tresolver_preload = {\n\t\t\t\t\t\t%s : %s\n\t\t\t\t\t}\n\t\t\t\t`, clustername, ojwt, syspub, storeDir, syspub, sysJwt)\n\t\t\t}, nil)\n\t\tdefer sc.shutdown()\n\n\t\ts := sc.serverByName(\"C1-S1\")\n\t\trequire_False(t, s.JetStreamEnabled())\n\t\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt, 10)\n\n\t\tsc.waitOnAccount(aExpPub)\n\n\t\ts = sc.serverByName(\"C2-S1\")\n\t\trequire_False(t, s.JetStreamEnabled())\n\n\t\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\t\tdefer nc.Close()\n\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\n\t\tci, err := js.AddStream(&nats.StreamConfig{Name: \"MOVE-ME\", Replicas: replicas,\n\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:C1-tag\"}}})\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ci.Cluster.Name, \"C1\")\n\n\t\t_, err = js.AddConsumer(\"MOVE-ME\", &nats.ConsumerConfig{Durable: \"dur\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"MOVE-ME\", []byte(\"hello world\"))\n\t\trequire_NoError(t, err)\n\n\t\t// Perform actual move\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"MOVE-ME\", Replicas: replicas,\n\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:C2-tag\"}}})\n\t\trequire_NoError(t, err)\n\n\t\tsc.clusterForName(\"C2\").waitOnStreamLeader(aExpPub, \"MOVE-ME\")\n\n\t\tcheckFor(t, 30*time.Second, 250*time.Millisecond, func() error {\n\t\t\tif si, err := js.StreamInfo(\"MOVE-ME\"); err != nil {\n\t\t\t\treturn fmt.Errorf(\"stream: %v\", err)\n\t\t\t} else if si.Cluster.Name != \"C2\" {\n\t\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t\t} else if !strings.HasPrefix(si.Cluster.Leader, \"C2-\") {\n\t\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t\t} else if len(si.Cluster.Replicas) != replicas-1 {\n\t\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", replicas-1, len(si.Cluster.Replicas))\n\t\t\t} else if si.State.Msgs != 1 {\n\t\t\t\treturn fmt.Errorf(\"expected one message\")\n\t\t\t}\n\t\t\t// Now make sure consumer has leader etc..\n\t\t\tif ci, err := js.ConsumerInfo(\"MOVE-ME\", \"dur\"); err != nil {\n\t\t\t\treturn fmt.Errorf(\"stream: %v\", err)\n\t\t\t} else if ci.Cluster.Name != \"C2\" {\n\t\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", ci.Cluster.Name)\n\t\t\t} else if ci.Cluster.Leader == _EMPTY_ {\n\t\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tsub, err := js.PullSubscribe(\"\", \"dur\", nats.BindStream(\"MOVE-ME\"))\n\t\trequire_NoError(t, err)\n\t\tm, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, m[0].AckSync())\n\t}\n\n\tt.Run(\"tiered\", func(t *testing.T) {\n\t\taccClaim := jwt.NewAccountClaims(aExpPub)\n\t\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\t\tDiskStorage: 1100, Consumer: 1, Streams: 1}\n\t\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\t\tDiskStorage: 3300, Consumer: 1, Streams: 1}\n\n\t\tt.Run(\"R3\", func(t *testing.T) {\n\t\t\ttest(t, 3, accClaim)\n\t\t})\n\t\tt.Run(\"R1\", func(t *testing.T) {\n\t\t\ttest(t, 1, accClaim)\n\t\t})\n\t})\n\n\tt.Run(\"non-tiered\", func(t *testing.T) {\n\t\taccClaim := jwt.NewAccountClaims(aExpPub)\n\t\taccClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{\n\t\t\tDiskStorage: 4400, Consumer: 2, Streams: 2}\n\n\t\tt.Run(\"R3\", func(t *testing.T) {\n\t\t\ttest(t, 3, accClaim)\n\t\t})\n\t\tt.Run(\"R1\", func(t *testing.T) {\n\t\t\ttest(t, 1, accClaim)\n\t\t})\n\t})\n}\n\nfunc TestJetStreamJWTClusteredTiers(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tnewUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, Consumer: 2, Streams: 2}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, Consumer: 1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t// Test absent tiers\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR2\", Replicas: 2, Subjects: []string{\"testR2\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: no JetStream default or applicable tiered limit present\")\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR5\", Replicas: 5, Subjects: []string{\"testR5\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: no JetStream default or applicable tiered limit present\")\n\n\t// Test tiers up to stream limits\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-1\", Replicas: 1, Subjects: []string{\"testR1-1\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR3-1\", Replicas: 3, Subjects: []string{\"testR3-1\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-2\", Replicas: 1, Subjects: []string{\"testR1-2\"}})\n\trequire_NoError(t, err)\n\n\t// Test exceeding tiered stream limit\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-3\", Replicas: 1, Subjects: []string{\"testR1-3\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum number of streams reached\")\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR3-3\", Replicas: 3, Subjects: []string{\"testR3-3\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum number of streams reached\")\n\n\t// Test tiers up to consumer limits\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"testR3-1\", &nats.ConsumerConfig{Durable: \"dur2\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur3\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// test exceeding tiered consumer limits\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur4\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum consumers limit reached\")\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur5\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum consumers limit reached\")\n\n\t// test tiered storage limit\n\tmsg := [512]byte{}\n\t_, err = js.Publish(\"testR1-1\", msg[:])\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"testR3-1\", msg[:])\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"testR3-1\", msg[:])\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"testR1-2\", msg[:])\n\trequire_NoError(t, err)\n\n\ttime.Sleep(2000 * time.Millisecond) // wait for update timer to synchronize totals\n\n\t// test exceeding tiered storage limit\n\t_, err = js.Publish(\"testR1-1\", []byte(\"1\"))\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\t_, err = js.Publish(\"testR3-1\", []byte(\"fail this message!\"))\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\n\t// retrieve limits\n\tvar info JSApiAccountInfoResponse\n\tm, err := nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(m.Data, &info)\n\trequire_NoError(t, err)\n\n\trequire_True(t, info.Memory == 0)\n\t// R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit\n\t// Alternative to checking both values is, prior to the info request, wait for another update\n\trequire_True(t, info.Store == 4400 || info.Store == 4439)\n\trequire_True(t, info.Streams == 3)\n\trequire_True(t, info.Consumers == 3)\n\trequire_True(t, info.Limits == JetStreamAccountLimits{})\n\tr1 := info.Tiers[\"R1\"]\n\trequire_True(t, r1.Streams == 2)\n\trequire_True(t, r1.Consumers == 2)\n\t// R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit\n\t// Alternative to checking both values is, prior to the info request, wait for another update\n\trequire_True(t, r1.Store == 1100 || r1.Store == 1139)\n\trequire_True(t, r1.Memory == 0)\n\trequire_True(t, r1.Limits == JetStreamAccountLimits{\n\t\tMaxMemory: 0,\n\t\tMaxStore: 1100,\n\t\tMaxStreams: 2,\n\t\tMaxConsumers: 2,\n\t\tMaxAckPending: -1,\n\t\tMemoryMaxStreamBytes: -1,\n\t\tStoreMaxStreamBytes: -1,\n\t\tMaxBytesRequired: false,\n\t})\n\tr3 := info.Tiers[\"R3\"]\n\trequire_True(t, r3.Streams == 1)\n\trequire_True(t, r3.Consumers == 1)\n\trequire_True(t, r3.Store == 3300)\n\trequire_True(t, r3.Memory == 0)\n\trequire_True(t, r3.Limits == JetStreamAccountLimits{\n\t\tMaxMemory: 0,\n\t\tMaxStore: 1100,\n\t\tMaxStreams: 1,\n\t\tMaxConsumers: 1,\n\t\tMaxAckPending: -1,\n\t\tMemoryMaxStreamBytes: -1,\n\t\tStoreMaxStreamBytes: -1,\n\t\tMaxBytesRequired: false,\n\t})\n}\n\nfunc TestJetStreamJWTClusteredTiersChange(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 500, MemoryStorage: 0, Consumer: 1, Streams: 1}\n\taccJwt1 := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\tstart := time.Now()\n\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t`\n\tc := createJetStreamClusterWithTemplateAndModHook(t, tmlp, \"cluster\", 3,\n\t\tfunc(serverName, clustername, storeDir, conf string) string {\n\t\t\treturn conf + fmt.Sprintf(`\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: {\n\t\t\t\t\ttype: full\n\t\t\t\t\tdir: '%s/jwt'\n\t\t\t\t}`, ojwt, syspub, storeDir)\n\t\t})\n\tdefer c.shutdown()\n\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3)\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3)\n\n\tc.waitOnAccount(aExpPub)\n\n\tnc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t// Test tiers up to stream limits\n\tcfg := &nats.StreamConfig{Name: \"testR1-1\", Replicas: 1, Subjects: []string{\"testR1-1\"}, MaxBytes: 1000}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err, errors.New(\"nats: insufficient storage resources available\"))\n\n\ttime.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1}\n\taccJwt2 := encodeClaim(t, accClaim, aExpPub)\n\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3)\n\n\tvar rBefore, rAfter JSApiAccountInfoResponse\n\tm, err := nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(m.Data, &rBefore)\n\trequire_NoError(t, err)\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\tm, err = nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(m.Data, &rAfter)\n\trequire_NoError(t, err)\n\trequire_True(t, rBefore.Tiers[\"R1\"].Streams == 1)\n\trequire_True(t, rBefore.Tiers[\"R1\"].Streams == rAfter.Tiers[\"R3\"].Streams)\n\trequire_True(t, rBefore.Tiers[\"R3\"].Streams == 0)\n\trequire_True(t, rAfter.Tiers[\"R1\"].Streams == 0)\n}\n\nfunc TestJetStreamJWTClusteredDeleteTierWithStreamAndMove(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1}\n\taccJwt1 := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\tstart := time.Now()\n\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t`\n\tc := createJetStreamClusterWithTemplateAndModHook(t, tmlp, \"cluster\", 3,\n\t\tfunc(serverName, clustername, storeDir, conf string) string {\n\t\t\treturn conf + fmt.Sprintf(`\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: {\n\t\t\t\t\ttype: full\n\t\t\t\t\tdir: '%s/jwt'\n\t\t\t\t}`, ojwt, syspub, storeDir)\n\t\t})\n\tdefer c.shutdown()\n\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3)\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3)\n\n\tc.waitOnAccount(aExpPub)\n\n\tnc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t// Test tiers up to stream limits\n\tcfg := &nats.StreamConfig{Name: \"testR1-1\", Replicas: 1, Subjects: []string{\"testR1-1\"}, MaxBytes: 1000}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tsl := c.streamLeader(aExpPub, \"testR1-1\")\n\trequire_NotNil(t, sl)\n\tacc, err := sl.lookupAccount(aExpPub)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"testR1-1\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, mset.lastSeq(), 0)\n\n\t_, err = js.Publish(\"testR1-1\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, mset.lastSeq(), 1)\n\n\ttime.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes\n\tdelete(accClaim.Limits.JetStreamTieredLimits, \"R1\")\n\taccJwt2 := encodeClaim(t, accClaim, aExpPub)\n\tupdateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3)\n\n\tvar respBefore JSApiAccountInfoResponse\n\tm, err := nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(m.Data, &respBefore)\n\trequire_NoError(t, err)\n\n\trequire_True(t, respBefore.JetStreamAccountStats.Tiers[\"R3\"].Streams == 0)\n\trequire_True(t, respBefore.JetStreamAccountStats.Tiers[\"R1\"].Streams == 1)\n\n\t_, err = js.Publish(\"testR1-1\", nil)\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: no JetStream default or applicable tiered limit present\")\n\trequire_Equal(t, mset.lastSeq(), 1)\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// I noticed this taking > 5 seconds\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\t_, err = js.Publish(\"testR1-1\", nil)\n\t\treturn err\n\t})\n\n\tvar respAfter JSApiAccountInfoResponse\n\tm, err = nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(m.Data, &respAfter)\n\trequire_NoError(t, err)\n\n\trequire_True(t, respAfter.JetStreamAccountStats.Tiers[\"R3\"].Streams == 1)\n\trequire_True(t, respAfter.JetStreamAccountStats.Tiers[\"R3\"].Store > 0)\n\n\t_, ok := respAfter.JetStreamAccountStats.Tiers[\"R1\"]\n\trequire_True(t, !ok)\n}\n\nfunc TestJetStreamJWTSysAccUpdateMixedMode(t *testing.T) {\n\tskp, spub := createKey(t)\n\tsUsr := createUserCreds(t, nil, skp)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"SYS\"\n\tsysJwt := encodeClaim(t, sysClaim, spub)\n\tencodeJwt1Time := time.Now()\n\n\takp, apub := createKey(t)\n\taUsr := createUserCreds(t, nil, akp)\n\tclaim := jwt.NewAccountClaims(apub)\n\tclaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024\n\tclaim.Limits.JetStreamLimits.Streams = 1\n\tjwt1 := encodeClaim(t, claim, apub)\n\n\tbasePath := \"/ngs/v1/accounts/jwt/\"\n\treqCount := int32(0)\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == basePath {\n\t\t\tw.Write([]byte(\"ok\"))\n\t\t} else if strings.HasSuffix(r.URL.Path, spub) {\n\t\t\tw.Write([]byte(sysJwt))\n\t\t} else if strings.HasSuffix(r.URL.Path, apub) {\n\t\t\tw.Write([]byte(jwt1))\n\t\t} else {\n\t\t\t// only count requests that could be filled\n\t\t\treturn\n\t\t}\n\t\tatomic.AddInt32(&reqCount, 1)\n\t}))\n\tdefer ts.Close()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n `\n\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\t// create an ngs like setup, with connection and non connection server\n\t\t\tif clusterName == \"C1\" {\n\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream\", \"#jetstream\")\n\t\t\t}\n\t\t\treturn fmt.Sprintf(`%s\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: URL(\"%s%s\")`, conf, ojwt, spub, ts.URL, basePath)\n\t\t}, nil)\n\tdefer sc.shutdown()\n\tdisconnectChan := make(chan struct{}, 100)\n\tdefer close(disconnectChan)\n\tdisconnectCb := nats.DisconnectErrHandler(func(conn *nats.Conn, err error) {\n\t\tdisconnectChan <- struct{}{}\n\t})\n\n\ts := sc.clusterForName(\"C1\").randomServer()\n\n\tsysNc := natsConnect(t, s.ClientURL(), sUsr, disconnectCb, nats.NoCallbacksAfterClientClose())\n\tdefer sysNc.Close()\n\taNc := natsConnect(t, s.ClientURL(), aUsr, disconnectCb, nats.NoCallbacksAfterClientClose())\n\tdefer aNc.Close()\n\n\tjs, err := aNc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tsc.waitOnAccount(apub)\n\n\tsi, err := js.AddStream(&nats.StreamConfig{Name: \"bar\", Subjects: []string{\"bar\"}, Replicas: 3})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"C2\")\n\t_, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\n\tr, err := sysNc.Request(fmt.Sprintf(serverPingReqSubj, \"ACCOUNTZ\"),\n\t\t[]byte(fmt.Sprintf(`{\"account\":\"%s\"}`, spub)), time.Second)\n\trequire_NoError(t, err)\n\trespb := ServerAPIResponse{Data: &Accountz{}}\n\trequire_NoError(t, json.Unmarshal(r.Data, &respb))\n\n\thasJSExp := func(resp *ServerAPIResponse) bool {\n\t\tfound := false\n\t\tfor _, e := range resp.Data.(*Accountz).Account.Exports {\n\t\t\tif e.Subject == jsAllAPI {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\treturn found\n\t}\n\trequire_True(t, hasJSExp(&respb))\n\n\t// make sure jti increased\n\ttime.Sleep(time.Second - time.Since(encodeJwt1Time))\n\tsysJwt2 := encodeClaim(t, sysClaim, spub)\n\n\toldRcount := atomic.LoadInt32(&reqCount)\n\t_, err = sysNc.Request(fmt.Sprintf(accUpdateEventSubjNew, spub), []byte(sysJwt2), time.Second)\n\trequire_NoError(t, err)\n\t// test to make sure connected client (aNc) was not kicked\n\ttime.Sleep(200 * time.Millisecond)\n\trequire_True(t, len(disconnectChan) == 0)\n\n\t// ensure nothing new has happened, lookup for account not found is skipped during inc\n\trequire_True(t, atomic.LoadInt32(&reqCount) == oldRcount)\n\t// no responders\n\t_, err = aNc.Request(\"foo\", nil, time.Second)\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: no responders available for request\")\n\n\tnc2, js2 := jsClientConnect(t, sc.clusterForName(\"C2\").randomServer(), aUsr)\n\tdefer nc2.Close()\n\t_, err = js2.AccountInfo()\n\trequire_NoError(t, err)\n\n\tr, err = sysNc.Request(fmt.Sprintf(serverPingReqSubj, \"ACCOUNTZ\"),\n\t\t[]byte(fmt.Sprintf(`{\"account\":\"%s\"}`, spub)), time.Second)\n\trequire_NoError(t, err)\n\trespa := ServerAPIResponse{Data: &Accountz{}}\n\trequire_NoError(t, json.Unmarshal(r.Data, &respa))\n\trequire_True(t, hasJSExp(&respa))\n\n\t_, err = js.AccountInfo()\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamJWTExpiredAccountNotCountedTowardLimits(t *testing.T) {\n\top, _ := nkeys.CreateOperator()\n\topPk, _ := op.PublicKey()\n\tsk, _ := nkeys.CreateOperator()\n\tskPk, _ := sk.PublicKey()\n\topClaim := jwt.NewOperatorClaims(opPk)\n\topClaim.SigningKeys.Add(skPk)\n\topJwt, err := opClaim.Encode(op)\n\trequire_NoError(t, err)\n\tcreateAccountAndUser := func(pubKey, jwt1, creds1 *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10}\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(sk)\n\t\trequire_NoError(t, err)\n\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\n\t\tujwt1, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\t}\n\tgenerateRequest := func(accs []string, kp nkeys.KeyPair) []byte {\n\t\tt.Helper()\n\t\topk, _ := kp.PublicKey()\n\t\tc := jwt.NewGenericClaims(opk)\n\t\tc.Data[\"accounts\"] = accs\n\t\tcJwt, err := c.Encode(kp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected no error %v\", err)\n\t\t}\n\t\treturn []byte(cJwt)\n\t}\n\n\tvar syspub, sysjwt, sysCreds string\n\tcreateAccountAndUser(&syspub, &sysjwt, &sysCreds)\n\n\tdirSrv := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: \"%s\"}\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tallow_delete: true\n\t\t\tdir: '%s'\n\t\t\ttimeout: \"500ms\"\n\t\t}\n `, opJwt, dirSrv, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// update system account jwt\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1)\n\n\tvar apub, ajwt1, aCreds1 string\n\tcreateAccountAndUser(&apub, &ajwt1, &aCreds1)\n\t// push jwt (for full resolver)\n\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1)\n\n\tncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1))\n\tdefer ncA.Close()\n\n\tai, err := jsA.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, ai.Limits.MaxMemory == 7*1024*1024)\n\tncA.Close()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds))\n\tdefer nc.Close()\n\tresp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second)\n\trequire_NoError(t, err)\n\trequire_True(t, strings.Contains(string(resp.Data), `\"message\":\"deleted 1 accounts\"`))\n\n\tvar apub2, ajwt2, aCreds2 string\n\tcreateAccountAndUser(&apub2, &ajwt2, &aCreds2)\n\t// push jwt (for full resolver)\n\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt2, 1)\n\n\tncB, jsB := jsClientConnect(t, s, nats.UserCredentials(aCreds2))\n\tdefer ncB.Close()\n\n\tai, err = jsB.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, ai.Limits.MaxMemory == 7*1024*1024)\n}\n\nfunc TestJetStreamJWTExpiredAccountWorksAfterExpirationUpdated(t *testing.T) {\n\tsysKp, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\tsysCreds := newUser(t, sysKp)\n\n\takp, apub := createKey(t)\n\trequire_NoError(t, err)\n\taccClaim := jwt.NewAccountClaims(apub)\n\taccClaim.Name = \"TEST\"\n\taccClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1}\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdirSrv := t.TempDir()\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: \"%s\"}\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: {\n\t\t\t\t\ttype: full\n\t\t\t\t\tallow_delete: true\n\t\t\t\t\tdir: '%s'\n\t\t\t\t}\n\t\t\t`, dirSrv, ojwt, spub, dir)))\n\tdefer removeFile(t, conf)\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)\n\n\tuserCreds := newUser(t, akp)\n\tnc, js := jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect(),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\t// Default handler would print to stderr, silence it\n\t\t}))\n\tdefer nc.Close()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tif _, err := js.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t}\n\t}()\n\n\t// Update the account so it expires in 2 seconds.\n\texpires := time.Now().Add(2 * time.Second)\n\taccClaim.Expires = expires.Unix()\n\taccJwt = encodeClaim(t, accClaim, apub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)\n\n\t// Wait for the publishing routine to fail.\n\twg.Wait()\n\n\t// Close this client, we will create a new one.\n\tnc.Close()\n\n\t// Verify that we can't connect anymore. Because of rounding, we may have\n\t// to try to avoid flapping.\n\tfor range 5 {\n\t\tnc, err = nats.Connect(s.ClientURL(), nats.UserCredentials(userCreds), nats.NoReconnect(),\n\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\t\t// Default handler would print to stderr, silence it\n\t\t\t}))\n\t\tif err != nil {\n\t\t\t// ok!\n\t\t\tbreak\n\t\t}\n\t\tnc.Close()\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\t// Check that the user account is marked as expired.\n\taz, err := s.Accountz(&AccountzOptions{Account: apub})\n\trequire_NoError(t, err)\n\trequire_True(t, az.Account != nil)\n\trequire_True(t, az.Account.Expired)\n\t// But should still be true\n\trequire_True(t, az.Account.JetStream)\n\n\t// Update the expiration to 1 hour\n\texpires = time.Now().Add(time.Hour)\n\taccClaim.Expires = expires.Unix()\n\taccJwt = encodeClaim(t, accClaim, apub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)\n\n\t// Check that its is no longer expired and has still JetStream enabled.\n\taz, err = s.Accountz(&AccountzOptions{Account: apub})\n\trequire_NoError(t, err)\n\trequire_True(t, az.Account != nil)\n\trequire_False(t, az.Account.Expired)\n\trequire_True(t, az.Account.JetStream)\n\n\t// Create a connection and ensure we connect ok and can send a message.\n\tnc, js = jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect())\n\tdefer nc.Close()\n\n\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamJWTDeletedAccountDoesNotLeakSubscriptions(t *testing.T) {\n\top, _ := nkeys.CreateOperator()\n\topPk, _ := op.PublicKey()\n\tsk, _ := nkeys.CreateOperator()\n\tskPk, _ := sk.PublicKey()\n\topClaim := jwt.NewOperatorClaims(opPk)\n\topClaim.SigningKeys.Add(skPk)\n\topJwt, err := opClaim.Encode(op)\n\trequire_NoError(t, err)\n\tcreateAccountAndUser := func(pubKey, jwt1, creds1 *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10}\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(sk)\n\t\trequire_NoError(t, err)\n\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\n\t\tujwt1, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\t}\n\tgenerateRequest := func(accs []string, kp nkeys.KeyPair) []byte {\n\t\tt.Helper()\n\t\topk, _ := kp.PublicKey()\n\t\tc := jwt.NewGenericClaims(opk)\n\t\tc.Data[\"accounts\"] = accs\n\t\tcJwt, err := c.Encode(kp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected no error %v\", err)\n\t\t}\n\t\treturn []byte(cJwt)\n\t}\n\n\tvar syspub, sysjwt, sysCreds string\n\tcreateAccountAndUser(&syspub, &sysjwt, &sysCreds)\n\n\tdirSrv := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: %v}\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tallow_delete: true\n\t\t\tdir: '%s'\n\t\t\ttimeout: \"500ms\"\n\t\t}\n\t`, opJwt, dirSrv, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tcheckNumSubs := func(expected uint32) uint32 {\n\t\tt.Helper()\n\t\t// Wait a bit before capturing number of subs...\n\t\ttime.Sleep(250 * time.Millisecond)\n\n\t\tvar ns uint32\n\t\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\t\tsubsz, err := s.Subsz(nil)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tns = subsz.NumSubs\n\t\t\tif expected > 0 && ns > expected {\n\t\t\t\treturn fmt.Errorf(\"Expected num subs to be back at %v, got %v\",\n\t\t\t\t\texpected, ns)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn ns\n\t}\n\tbeforeCreate := checkNumSubs(0)\n\n\t// update system account jwt\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1)\n\n\tcreateAndDelete := func() {\n\t\tt.Helper()\n\n\t\tvar apub, ajwt1, aCreds1 string\n\t\tcreateAccountAndUser(&apub, &ajwt1, &aCreds1)\n\t\t// push jwt (for full resolver)\n\t\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1)\n\n\t\tncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1))\n\t\tdefer ncA.Close()\n\n\t\tai, err := jsA.AccountInfo()\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ai.Limits.MaxMemory == 7*1024*1024)\n\t\tncA.Close()\n\n\t\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds))\n\t\tdefer nc.Close()\n\n\t\tresp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, strings.Contains(string(resp.Data), `\"message\":\"deleted 1 accounts\"`))\n\t}\n\n\t// Create and delete multiple accounts\n\tfor i := 0; i < 10; i++ {\n\t\tcreateAndDelete()\n\t}\n\n\t// There is a subscription on `_R_.>` that is created on the system account\n\t// and that will not go away, so discount it.\n\tcheckNumSubs(beforeCreate + 1)\n}\n\nfunc TestJetStreamJWTDeletedAccountIsReEnabled(t *testing.T) {\n\top, _ := nkeys.CreateOperator()\n\topPk, _ := op.PublicKey()\n\tsk, _ := nkeys.CreateOperator()\n\tskPk, _ := sk.PublicKey()\n\topClaim := jwt.NewOperatorClaims(opPk)\n\topClaim.SigningKeys.Add(skPk)\n\topJwt, err := opClaim.Encode(op)\n\trequire_NoError(t, err)\n\tcreateAccountAndUser := func(pubKey, jwt1, creds1 *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 7 * 1024 * 1024, DiskStorage: 7 * 1024 * 1024, Streams: 10}\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(sk)\n\t\trequire_NoError(t, err)\n\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\n\t\tujwt1, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\t}\n\tgenerateRequest := func(accs []string, kp nkeys.KeyPair) []byte {\n\t\tt.Helper()\n\t\topk, _ := kp.PublicKey()\n\t\tc := jwt.NewGenericClaims(opk)\n\t\tc.Data[\"accounts\"] = accs\n\t\tcJwt, err := c.Encode(kp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected no error %v\", err)\n\t\t}\n\t\treturn []byte(cJwt)\n\t}\n\n\t// admin user\n\tvar syspub, sysjwt, sysCreds string\n\tcreateAccountAndUser(&syspub, &sysjwt, &sysCreds)\n\n\tdirSrv := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: \"%s\"}\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tallow_delete: true\n\t\t\tdir: '%s'\n\t\t\ttimeout: \"500ms\"\n\t\t}\n\t`, opJwt, dirSrv, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// update system account jwt\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysjwt, 1)\n\n\t// create account\n\tvar apub, ajwt1, aCreds1 string\n\tkp, _ := nkeys.CreateAccount()\n\tapub, _ = kp.PublicKey()\n\tclaim := jwt.NewAccountClaims(apub)\n\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{\n\t\tMemoryStorage: 7 * 1024 * 1024,\n\t\tDiskStorage: 7 * 1024 * 1024,\n\t\tStreams: 10,\n\t}\n\tajwt1, err = claim.Encode(sk)\n\trequire_NoError(t, err)\n\n\t// user\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub\n\n\tujwt1, err := uclaim.Encode(kp)\n\trequire_NoError(t, err)\n\taCreds1 = genCredsFile(t, ujwt1, seed)\n\n\t// push user account\n\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1)\n\n\tncA, jsA := jsClientConnect(t, s, nats.UserCredentials(aCreds1))\n\tdefer ncA.Close()\n\n\tjsA.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\tjsA.Publish(\"foo\", []byte(\"Hello World\"))\n\tjsA.Publish(\"foo\", []byte(\"Hello Again\"))\n\n\t// JS should be working\n\tai, err := jsA.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, ai.Limits.MaxMemory == 7*1024*1024)\n\trequire_True(t, ai.Limits.MaxStore == 7*1024*1024)\n\trequire_True(t, ai.Tier.Streams == 1)\n\n\t// connect with a different connection and delete the account.\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds))\n\tdefer nc.Close()\n\n\t// delete account\n\tresp, err := nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second)\n\trequire_NoError(t, err)\n\trequire_True(t, strings.Contains(string(resp.Data), `\"message\":\"deleted 1 accounts\"`))\n\n\t// account was disabled and now disconnected, this should get a connection is closed error.\n\t_, err = jsA.AccountInfo()\n\tif err == nil || !errors.Is(err, nats.ErrConnectionClosed) {\n\t\tt.Errorf(\"Expected connection closed error, got: %v\", err)\n\t}\n\tncA.Close()\n\n\t// re-enable, same claims would be detected\n\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1)\n\n\t// expected to get authorization timeout at this time\n\t_, err = nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds1))\n\tif !errors.Is(err, nats.ErrAuthorization) {\n\t\tt.Errorf(\"Expected authorization issue on connect, got: %v\", err)\n\t}\n\n\t// edit the account and push again with updated claims to same account\n\tclaim = jwt.NewAccountClaims(apub)\n\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{\n\t\tMemoryStorage: -1,\n\t\tDiskStorage: 10 * 1024 * 1024,\n\t\tStreams: 10,\n\t}\n\tajwt1, err = claim.Encode(sk)\n\trequire_NoError(t, err)\n\tupdateJwt(t, s.ClientURL(), sysCreds, ajwt1, 1)\n\n\t// reconnect with the updated account\n\tncA, jsA = jsClientConnect(t, s, nats.UserCredentials(aCreds1))\n\tdefer ncA.Close()\n\tai, err = jsA.AccountInfo()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\trequire_True(t, ai.Limits.MaxMemory == -1)\n\trequire_True(t, ai.Limits.MaxStore == 10*1024*1024)\n\trequire_True(t, ai.Tier.Streams == 1)\n\n\t// should be possible to get stream info again\n\tsi, err := jsA.StreamInfo(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif si.State.Msgs != 2 {\n\t\tt.Fatal(\"Unexpected number of messages from recovered stream\")\n\t}\n\tmsg, err := jsA.GetMsg(\"foo\", 1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(msg.Data) != \"Hello World\" {\n\t\tt.Error(\"Unexpected message\")\n\t}\n\tncA.Close()\n}\n\n// Make sure 100MB HA means 100MB of R3, not 33.3MB.\nfunc TestJetStreamJWTHAStorageLimitsAndAccounting(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tnewUser(t, sysKp)\n\n\tmaxFileStorage := int64(100 * 1024 * 1024)\n\tmaxMemStorage := int64(2 * 1024 * 1024)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t// Test max bytes first.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t// Now test actual usage.\n\t// We should be able to send just over 200 of these.\n\tmsg := [500 * 1024]byte{}\n\tfor i := 0; i < 250; i++ {\n\t\tif _, err := js.Publish(\"foo\", msg[:]); err != nil {\n\t\t\trequire_Error(t, err, NewJSAccountResourcesExceededError())\n\t\t\trequire_True(t, i > 200)\n\t\t\tbreak\n\t\t}\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\t// Make sure we are no more then 1 msg below our max in terms of size.\n\tdelta := maxFileStorage - int64(si.State.Bytes)\n\trequire_True(t, int(delta) < len(msg))\n\n\t// Now memory as well.\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t// Test max bytes first.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3, MaxBytes: maxMemStorage, Storage: nats.MemoryStorage, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3, Storage: nats.MemoryStorage, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t// This is much smaller, so should only be able to send 4.\n\tfor i := 0; i < 5; i++ {\n\t\tif _, err := js.Publish(\"foo\", msg[:]); err != nil {\n\t\t\trequire_Error(t, err, NewJSAccountResourcesExceededError())\n\t\t\trequire_Equal(t, i, 4)\n\t\t\tbreak\n\t\t}\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\t// Make sure we are no more then 1 msg below our max in terms of size.\n\tdelta = maxMemStorage - int64(si.State.Bytes)\n\trequire_True(t, int(delta) < len(msg))\n}\n\nfunc TestJetStreamJWTHAStorageLimitsOnScaleAndUpdate(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tnewUser(t, sysKp)\n\n\tmaxFileStorage := int64(5 * 1024 * 1024)\n\tmaxMemStorage := int64(1 * 1024 * 1024)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage}\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{DiskStorage: maxFileStorage, MemoryStorage: maxMemStorage}\n\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t// Test max bytes first.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3, MaxBytes: maxFileStorage, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\t// Now delete\n\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t// Now do 5 1MB streams.\n\tfor i := 1; i <= 5; i++ {\n\t\tsname := fmt.Sprintf(\"TEST%d\", i)\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3, MaxBytes: 1 * 1024 * 1024})\n\t\trequire_NoError(t, err)\n\t}\n\t// Should fail.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST6\", Replicas: 3, MaxBytes: 1 * 1024 * 1024})\n\trequire_Error(t, err, errors.New(\"insufficient storage resources\"))\n\n\t// Update Test1 and Test2 to smaller reservations.\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"TEST1\", Replicas: 3, MaxBytes: 512 * 1024})\n\trequire_NoError(t, err)\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"TEST2\", Replicas: 3, MaxBytes: 512 * 1024})\n\trequire_NoError(t, err)\n\t// Now make sure TEST6 succeeds.\n\tcheckFor(t, 1*time.Second, 500*time.Millisecond, func() error {\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST6\", Replicas: 3, MaxBytes: 1 * 1024 * 1024})\n\t\t// Since the stream leader answers the stream update, and the meta leader determines resources,\n\t\t// we could hit a race condition here. Simply retry if hit.\n\t\tif err != nil && strings.Contains(err.Error(), \"insufficient storage resources\") {\n\t\t\treturn err\n\t\t}\n\t\trequire_NoError(t, err)\n\t\treturn nil\n\t})\n\t// Now delete the R3 version.\n\trequire_NoError(t, js.DeleteStream(\"TEST6\"))\n\t// Now do R1 version and then we will scale up.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST6\", Replicas: 1, MaxBytes: 1 * 1024 * 1024})\n\trequire_NoError(t, err)\n\t// Now make sure scale up works.\n\tcheckFor(t, 1*time.Second, 500*time.Millisecond, func() error {\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"TEST6\", Replicas: 3, MaxBytes: 1 * 1024 * 1024})\n\t\t// Since the stream leader answers the stream add, and the meta leader determines stream not found,\n\t\t// we could hit a race condition here. Simply retry if hit.\n\t\tif err != nil && strings.Contains(err.Error(), \"stream not found\") {\n\t\t\treturn err\n\t\t}\n\t\trequire_NoError(t, err)\n\t\treturn nil\n\t})\n\t// Add in a few more streams to check reserved reporting in account info.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST7\", Replicas: 1, MaxBytes: 2 * 1024 * 1024})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST8\", Replicas: 1, MaxBytes: 256 * 1024, Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST9\", Replicas: 3, MaxBytes: 22 * 1024, Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\t// Now make sure we report reserved correctly.\n\t// Do this direct to server since client does not support it yet.\n\tvar info JSApiAccountInfoResponse\n\tresp, err := nc.Request(\"$JS.API.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(resp.Data, &info))\n\tstats := info.JetStreamAccountStats\n\tr1, r3 := stats.Tiers[\"R1\"], stats.Tiers[\"R3\"]\n\n\trequire_Equal(t, r1.ReservedMemory, 256*1024) // TEST8\n\trequire_Equal(t, r1.ReservedStore, 2*1024*1024) // TEST7\n\trequire_Equal(t, r3.ReservedMemory, 22*1024) // TEST9\n\trequire_Equal(t, r3.ReservedStore, 5*1024*1024) // TEST1-TEST6\n}\n\nfunc TestJetStreamJWTClusteredTiersR3StreamWithR1ConsumersAndAccounting(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tnewUser(t, sysKp)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, Consumer: 10, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, Consumer: 1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now make sure we can add in 10 R1 consumers.\n\tfor i := 1; i <= 10; i++ {\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tName: fmt.Sprintf(\"C-%d\", i),\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tReplicas: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tinfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\n\t// Make sure we account for these properly.\n\tr1 := info.Tiers[\"R1\"]\n\tr3 := info.Tiers[\"R3\"]\n\n\trequire_Equal(t, r1.Streams, 0)\n\trequire_Equal(t, r1.Consumers, 10)\n\trequire_Equal(t, r3.Streams, 1)\n\trequire_Equal(t, r3.Consumers, 0)\n}\n\nfunc TestJetStreamJWTClusterAccountNRG(t *testing.T) {\n\t_, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\n\taExpKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 10, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, aExpKp)\n\n\t_, aExpPub2 := createKey(t)\n\taccClaim2 := jwt.NewAccountClaims(aExpPub2)\n\taccClaim2.Name = \"another_acc\"\n\taccJwt2 := encodeClaim(t, accClaim2, aExpPub2)\n\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt, aExpPub2, accJwt2)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tStorage: FileStorage,\n\t})\n\n\t// We'll try flipping the state a few times and then do some sanity\n\t// checks to check that it took effect.\n\tthirdAcc := jwt.ClusterTraffic(fmt.Sprintf(\"account:%s\", aExpPub2))\n\t// TODO: Not currently testing thirdAcc because we haven't enabled this\n\t// functionality yet. If/when we do enable, this test is ready just by\n\t// uncommenting the third state below.\n\tfor _, state := range []jwt.ClusterTraffic{\"system\", \"owner\" /*, thirdAcc */} {\n\t\taccClaim.ClusterTraffic = state\n\t\taccJwt = encodeClaim(t, accClaim, aExpPub)\n\n\t\tfor _, s := range c.servers {\n\t\t\t// Update the account claim for our \"third account\".\n\t\t\tacc, err := s.lookupAccount(aExpPub2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, s.updateAccountWithClaimJWT(acc, accJwt2))\n\n\t\t\t// Then update the account claim for the asset account.\n\t\t\tacc, err = s.lookupAccount(aExpPub)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, s.updateAccountWithClaimJWT(acc, accJwt))\n\n\t\t\t// Check that everything looks like it should.\n\t\t\trequire_True(t, acc != nil)\n\t\t\trequire_True(t, acc.js != nil)\n\t\t\tswitch state {\n\t\t\tcase \"system\":\n\t\t\t\trequire_Equal(t, acc.nrgAccount, _EMPTY_)\n\t\t\tcase \"owner\":\n\t\t\t\trequire_Equal(t, acc.nrgAccount, aExpPub)\n\t\t\tcase thirdAcc:\n\t\t\t\trequire_Equal(t, acc.nrgAccount, aExpPub2)\n\t\t\t}\n\n\t\t\t// Now get a list of all of the Raft nodes that should\n\t\t\t// have been updated by now.\n\t\t\ts.rnMu.Lock()\n\t\t\traftNodes := make([]*raft, 0, len(s.raftNodes))\n\t\t\tfor _, n := range s.raftNodes {\n\t\t\t\trg := n.(*raft)\n\t\t\t\tif rg.accName != acc.Name {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\traftNodes = append(raftNodes, rg)\n\t\t\t}\n\t\t\ts.rnMu.Unlock()\n\n\t\t\t// Get the Raftz state also.\n\t\t\trz := s.Raftz(&RaftzOptions{AccountFilter: aExpPub})\n\t\t\trequire_NotNil(t, rz)\n\t\t\trza := (*rz)[aExpPub]\n\t\t\trequire_NotNil(t, rza)\n\n\t\t\t// Check whether each of the Raft nodes reports being\n\t\t\t// in-account or not.\n\t\t\tfor _, rg := range raftNodes {\n\t\t\t\trg.Lock()\n\t\t\t\trgAcc := rg.acc\n\t\t\t\trg.Unlock()\n\t\t\t\tswitch state {\n\t\t\t\tcase \"system\":\n\t\t\t\t\trequire_Equal(t, rgAcc.Name, syspub)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].SystemAcc, true)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].TrafficAcc, syspub)\n\t\t\t\tcase \"owner\":\n\t\t\t\t\trequire_Equal(t, rgAcc.Name, aExpPub)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].SystemAcc, false)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].TrafficAcc, aExpPub)\n\t\t\t\tcase thirdAcc:\n\t\t\t\t\trequire_Equal(t, rgAcc.Name, aExpPub2)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].SystemAcc, false)\n\t\t\t\t\trequire_Equal(t, rza[rg.group].TrafficAcc, aExpPub2)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamJWTClusterAccountNRGPersistsAfterRestart(t *testing.T) {\n\t_, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\n\taExpKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.ClusterTraffic = jwt.ClusterTrafficOwner\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 10, Streams: 1}\n\taccClaim.Limits.JetStreamTieredLimits[\"R3\"] = jwt.JetStreamLimits{DiskStorage: 1100, Consumer: 1, Streams: 1}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, aExpKp)\n\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\n\tc := createJetStreamClusterWithTemplate(t, tmlp, \"cluster\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(aExpPub)\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tStorage: FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t// The account had cluster traffic set to \"owner\" already. Restarting servers should remember this setting.\n\tfor _, s := range c.servers {\n\t\tacc, err := s.lookupAccount(aExpPub)\n\t\trequire_NoError(t, err)\n\n\t\t// Check that everything looks like it should.\n\t\trequire_True(t, acc != nil)\n\t\trequire_True(t, acc.js != nil)\n\t\trequire_Equal(t, acc.nrgAccount, aExpPub)\n\n\t\t// Now get a list of all the Raft nodes that should have the correct cluster traffic set.\n\t\ts.rnMu.Lock()\n\t\traftNodes := make([]*raft, 0, len(s.raftNodes))\n\t\tfor _, n := range s.raftNodes {\n\t\t\trg := n.(*raft)\n\t\t\tif rg.accName != acc.Name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\traftNodes = append(raftNodes, rg)\n\t\t}\n\t\ts.rnMu.Unlock()\n\n\t\t// Get the Raftz state also.\n\t\trz := s.Raftz(&RaftzOptions{AccountFilter: aExpPub})\n\t\trequire_NotNil(t, rz)\n\t\trza := (*rz)[aExpPub]\n\t\trequire_NotNil(t, rza)\n\n\t\tfor _, rg := range raftNodes {\n\t\t\trg.Lock()\n\t\t\trgAcc := rg.acc\n\t\t\trg.Unlock()\n\t\t\trequire_Equal(t, rgAcc.Name, aExpPub)\n\t\t\trequire_Equal(t, rza[rg.group].SystemAcc, false)\n\t\t\trequire_Equal(t, rza[rg.group].TrafficAcc, aExpPub)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamJWTUpdateWithPreExistingStream(t *testing.T) {\n\tupdateJwt := func(url string, creds string, pubKey string, jwt string) {\n\t\tt.Helper()\n\t\tc := natsConnect(t, url, nats.UserCredentials(creds))\n\t\tdefer c.Close()\n\t\tif msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t} else {\n\t\t\tcontent := make(map[string]any)\n\t\t\tif err := json.Unmarshal(msg.Data, &content); err != nil {\n\t\t\t\tt.Fatalf(\"%v\", err)\n\t\t\t} else if _, ok := content[\"data\"]; !ok {\n\t\t\t\tt.Fatalf(\"did not get an ok response got: %v\", content)\n\t\t\t}\n\t\t}\n\t}\n\tcreateUserCreds := func(akp nkeys.KeyPair) string {\n\t\tuKp1, _ := nkeys.CreateUser()\n\t\tuSeed1, _ := uKp1.Seed()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject, _ = uKp1.PublicKey()\n\t\tuserJwt1, err := uclaim.Encode(akp)\n\t\trequire_NoError(t, err)\n\t\treturn genCredsFile(t, userJwt1, uSeed1)\n\t}\n\t// Create system account.\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysUKp, _ := nkeys.CreateUser()\n\tsysUSeed, _ := sysUKp.Seed()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject, _ = sysUKp.PublicKey()\n\tsysUserJwt, err := uclaim.Encode(sysKp)\n\trequire_NoError(t, err)\n\tsysKp.Seed()\n\tsysCreds := genCredsFile(t, sysUserJwt, sysUSeed)\n\t// Create exporting account.\n\takpE, _ := nkeys.CreateAccount()\n\taPubE, _ := akpE.PublicKey()\n\tclaimE := jwt.NewAccountClaims(aPubE)\n\taJwtE, err := claimE.Encode(oKp)\n\trequire_NoError(t, err)\n\t// Create importing account.\n\takpI, _ := nkeys.CreateAccount()\n\taPubI, _ := akpI.PublicKey()\n\tclaimI := jwt.NewAccountClaims(aPubI)\n\tclaimI.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 1024 * 1024}\n\tclaimI.Imports.Add(&jwt.Import{\n\t\tName: \"import\",\n\t\tSubject: \"foo\",\n\t\tAccount: aPubE,\n\t\tType: jwt.Stream,\n\t})\n\taJwtI, err := claimI.Encode(oKp)\n\trequire_NoError(t, err)\n\t// Create users.\n\tuserCredsE := createUserCreds(akpE)\n\tuserCredsI := createUserCreds(akpI)\n\t// Start server and update JWTs.\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: \"%s\"}\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, dir, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(s.ClientURL(), sysCreds, aPubI, aJwtI)\n\tupdateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE)\n\n\t// Create stream on importing account before we restart.\n\tnci, js := jsClientConnect(t, s, nats.UserCredentials(userCredsI))\n\tdefer nci.Close()\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Restart server.\n\tnci.Close()\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Reconnect and confirm stream is empty.\n\tnci, js = jsClientConnect(t, s, nats.UserCredentials(userCredsI))\n\tdefer nci.Close()\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\n\t// If an import/export gets added when the stream already existed on startup.\n\t// We should still be able to route those messages.\n\tclaimE.Exports.Add(&jwt.Export{\n\t\tName: \"export\",\n\t\tSubject: \"foo\",\n\t\tType: jwt.Stream,\n\t})\n\taJwtE, err = claimE.Encode(oKp)\n\trequire_NoError(t, err)\n\tupdateJwt(s.ClientURL(), sysCreds, aPubE, aJwtE)\n\n\t// Connect to exporting account and publish a message that should be exported/imported.\n\tnce := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCredsE))\n\tdefer nce.Close()\n\terr = nce.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\t// Confirm the message was captured by the stream on the importing account.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\tif si, err = js.StreamInfo(\"TEST\"); err != nil {\n\t\t\treturn err\n\t\t} else if si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 message in stream, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamAccountResolverNoFetchIfNotMember(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"SYS\"\n\tsysJwt := encodeClaim(t, sysClaim, spub)\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\n\ttempl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 2GB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n`\n\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/\" {\n\t\t\tw.Write([]byte(\"ok\"))\n\t\t} else if strings.HasSuffix(r.URL.Path, spub) {\n\t\t\tw.Write([]byte(sysJwt))\n\t\t} else {\n\t\t\t// Simulate some time being spent, but doesn't respond.\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\tc := createJetStreamClusterWithTemplateAndModHook(t, templ, \"R3S\", 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\treturn conf + fmt.Sprintf(`\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: URL(\"%s\")`, ojwt, spub, ts.URL)\n\t\t})\n\tdefer c.shutdown()\n\n\ts := c.leader()\n\tjs := s.getJetStream()\n\tci := &ClientInfo{Cluster: \"R3S\", Account: aPub}\n\tcfg := &StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}}\n\tsa := &streamAssignment{Client: ci, Config: cfg}\n\tstart := time.Now()\n\t// Simulate some meta operations where this server is not a member.\n\t// The server should not fetch the account from the resolver.\n\tfor range 5 {\n\t\tjs.processStreamAssignment(sa)\n\t}\n\trequire_LessThan(t, time.Since(start), 100*time.Millisecond)\n}\n" }, { "path": "server/jetstream_leafnode_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tjwt \"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc TestJetStreamLeafNodeUniqueServerNameCrossJSDomain(t *testing.T) {\n\tname := \"NOT-UNIQUE\"\n\ttest := func(t *testing.T, s *Server, sIdExpected string, srvs ...*Server) {\n\t\tids := map[string]string{}\n\t\tfor _, srv := range srvs {\n\t\t\tcheckLeafNodeConnectedCount(t, srv, 2)\n\t\t\tids[srv.ID()] = srv.opts.JetStreamDomain\n\t\t}\n\t\t// ensure that an update for every server was received\n\t\tsysNc := natsConnect(t, fmt.Sprintf(\"nats://admin:s3cr3t!@127.0.0.1:%d\", s.opts.Port))\n\t\tdefer sysNc.Close()\n\t\tsub, err := sysNc.SubscribeSync(fmt.Sprintf(serverStatsSubj, \"*\"))\n\t\trequire_NoError(t, err)\n\t\tfor {\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\ttk := strings.Split(m.Subject, \".\")\n\t\t\tif domain, ok := ids[tk[2]]; ok {\n\t\t\t\tdelete(ids, tk[2])\n\t\t\t\trequire_Contains(t, string(m.Data), fmt.Sprintf(`\"domain\":\"%s\"`, domain))\n\t\t\t}\n\t\t\tif len(ids) == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tcnt := 0\n\t\ts.nodeToInfo.Range(func(key, value any) bool {\n\t\t\tcnt++\n\t\t\trequire_Equal(t, value.(nodeInfo).name, name)\n\t\t\trequire_Equal(t, value.(nodeInfo).id, sIdExpected)\n\t\t\treturn true\n\t\t})\n\t\trequire_Equal(t, cnt, 1)\n\t}\n\ttmplA := `\n\t\tlisten: -1\n\t\tserver_name: %s\n\t\tjetstream {\n\t\t\tmax_mem_store: 256MB,\n\t\t\tmax_file_store: 2GB,\n\t\t\tstore_dir: '%s',\n\t\t\tdomain: hub\n\t\t}\n\t\taccounts {\n\t\t\tJSY { users = [ { user: \"y\", pass: \"p\" } ]; jetstream: true }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleaf {\n\t\t\tport: -1\n\t\t}\n `\n\ttmplL := `\n\t\tlisten: -1\n\t\tserver_name: %s\n\t\tjetstream {\n\t\t\tmax_mem_store: 256MB,\n\t\t\tmax_file_store: 2GB,\n\t\t\tstore_dir: '%s',\n\t\t\tdomain: %s\n\t\t}\n\t\taccounts {\n\t\t\tJSY { users = [ { user: \"y\", pass: \"p\" } ]; jetstream: true }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{ urls: [ %s ], account: \"JSY\" }\n\t\t\t\t{ urls: [ %s ], account: \"$SYS\" }\n\t\t\t]\n\t\t}\n `\n\tt.Run(\"same-domain\", func(t *testing.T) {\n\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir())))\n\t\tsA, oA := RunServerWithConfig(confA)\n\t\tdefer sA.Shutdown()\n\t\t// using same domain as sA\n\t\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), \"hub\",\n\t\t\tfmt.Sprintf(\"nats://y:p@127.0.0.1:%d\", oA.LeafNode.Port),\n\t\t\tfmt.Sprintf(\"nats://admin:s3cr3t!@127.0.0.1:%d\", oA.LeafNode.Port))))\n\t\tsL, _ := RunServerWithConfig(confL)\n\t\tdefer sL.Shutdown()\n\t\t// as server name uniqueness is violates, sL.ID() is the expected value\n\t\ttest(t, sA, sL.ID(), sA, sL)\n\t})\n\tt.Run(\"different-domain\", func(t *testing.T) {\n\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, t.TempDir())))\n\t\tsA, oA := RunServerWithConfig(confA)\n\t\tdefer sA.Shutdown()\n\t\t// using different domain as sA\n\t\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, t.TempDir(), \"spoke\",\n\t\t\tfmt.Sprintf(\"nats://y:p@127.0.0.1:%d\", oA.LeafNode.Port),\n\t\t\tfmt.Sprintf(\"nats://admin:s3cr3t!@127.0.0.1:%d\", oA.LeafNode.Port))))\n\t\tsL, _ := RunServerWithConfig(confL)\n\t\tdefer sL.Shutdown()\n\t\tcheckLeafNodeConnectedCount(t, sL, 2)\n\t\tcheckLeafNodeConnectedCount(t, sA, 2)\n\t\t// ensure sA contains only sA.ID\n\t\ttest(t, sA, sA.ID(), sA, sL)\n\t})\n}\n\nfunc TestJetStreamLeafNodeJwtPermsAndJSDomains(t *testing.T) {\n\tcreateAcc := func(js bool) (string, string, nkeys.KeyPair) {\n\t\tkp, _ := nkeys.CreateAccount()\n\t\taPub, _ := kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(aPub)\n\t\tif js {\n\t\t\tclaim.Limits.JetStreamLimits = jwt.JetStreamLimits{\n\t\t\t\tMemoryStorage: 1024 * 1024,\n\t\t\t\tDiskStorage: 1024 * 1024,\n\t\t\t\tStreams: 1, Consumer: 2}\n\t\t}\n\t\taJwt, err := claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\treturn aPub, aJwt, kp\n\t}\n\tsysPub, sysJwt, sysKp := createAcc(false)\n\taccPub, accJwt, accKp := createAcc(true)\n\tnoExpiration := time.Now().Add(time.Hour)\n\t// create user for acc to be used in leaf node.\n\tlnCreds := createUserWithLimit(t, accKp, noExpiration, func(j *jwt.UserPermissionLimits) {\n\t\tj.Sub.Deny.Add(\"subdeny\")\n\t\tj.Pub.Deny.Add(\"pubdeny\")\n\t})\n\tunlimitedCreds := createUserWithLimit(t, accKp, noExpiration, nil)\n\n\tsysCreds := createUserWithLimit(t, sysKp, noExpiration, nil)\n\n\ttmplA := `\noperator: %s\nsystem_account: %s\nresolver: MEMORY\nresolver_preload: {\n %s: %s\n %s: %s\n}\nlisten: 127.0.0.1:-1\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n}\njetstream :{\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 100Mb\n max_file: 100Mb\n}\n`\n\n\ttmplL := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account = SYS\njetstream: {\n domain: ln1\n store_dir: '%s'\n max_mem: 50Mb\n max_file: 50Mb\n}\nleafnodes:{\n remotes:[{ url:nats://127.0.0.1:%d, account: A, credentials: '%s'},\n\t\t\t { url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}]\n}\n`\n\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, ojwt, sysPub,\n\t\tsysPub, sysJwt, accPub, accJwt,\n\t\tt.TempDir())))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\n\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL, t.TempDir(),\n\t\tsA.opts.LeafNode.Port, lnCreds, sA.opts.LeafNode.Port, sysCreds)))\n\tsL, _ := RunServerWithConfig(confL)\n\tdefer sL.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, sA, 2)\n\tcheckLeafNodeConnectedCount(t, sL, 2)\n\n\tncA := natsConnect(t, sA.ClientURL(), nats.UserCredentials(unlimitedCreds))\n\tdefer ncA.Close()\n\n\tncL := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", sL.opts.Port))\n\tdefer ncL.Close()\n\n\ttest := func(subject string, cSub, cPub *nats.Conn, remoteServerForSub *Server, accName string, pass bool) {\n\t\tt.Helper()\n\t\tsub, err := cSub.SubscribeSync(subject)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, cSub.Flush())\n\t\t// ensure the subscription made it across, or if not sent due to sub deny, make sure it could have made it.\n\t\tif remoteServerForSub == nil {\n\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t} else {\n\t\t\tcheckSubInterest(t, remoteServerForSub, accName, subject, time.Second)\n\t\t}\n\t\trequire_NoError(t, cPub.Publish(subject, []byte(\"hello world\")))\n\t\trequire_NoError(t, cPub.Flush())\n\t\tm, err := sub.NextMsg(500 * time.Millisecond)\n\t\tif pass {\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, m.Subject == subject)\n\t\t\trequire_Equal(t, string(m.Data), \"hello world\")\n\t\t} else {\n\t\t\trequire_True(t, err == nats.ErrTimeout)\n\t\t}\n\t}\n\n\tt.Run(\"sub-on-ln-pass\", func(t *testing.T) {\n\t\ttest(\"sub\", ncL, ncA, sA, accPub, true)\n\t})\n\tt.Run(\"sub-on-ln-fail\", func(t *testing.T) {\n\t\ttest(\"subdeny\", ncL, ncA, nil, \"\", false)\n\t})\n\tt.Run(\"pub-on-ln-pass\", func(t *testing.T) {\n\t\ttest(\"pub\", ncA, ncL, sL, \"A\", true)\n\t})\n\tt.Run(\"pub-on-ln-fail\", func(t *testing.T) {\n\t\ttest(\"pubdeny\", ncA, ncL, nil, \"A\", false)\n\t})\n}\n\nfunc TestJetStreamLeafNodeClusterExtensionWithSystemAccount(t *testing.T) {\n\t/*\n\t\tTopologies tested here\n\t\tsame == true\n\t\tA <-> B\n\t\t^ |\\\n\t\t| \\\n\t\t| proxy\n\t\t| \\\n\t\tLA <-> LB\n\n\t\tsame == false\n\t\tA <-> B\n\t\t^ ^\n\t\t| |\n\t\t| proxy\n\t\t| |\n\t\tLA <-> LB\n\n\t\tThe proxy is turned on later, such that the system account connection can be started later, in a controlled way\n\t\tThis explicitly tests the system state before and after this happens.\n\t*/\n\n\ttmplA := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\njetstream :{\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 100Mb\n max_file: 100Mb\n}\nserver_name: A\ncluster: {\n\tname: clust1\n\tlisten: 127.0.0.1:20104\n\troutes=[nats-route://127.0.0.1:20105]\n\tno_advertise: true\n}\n`\n\n\ttmplB := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\njetstream: {\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 100Mb\n max_file: 100Mb\n}\nserver_name: B\ncluster: {\n\tname: clust1\n\tlisten: 127.0.0.1:20105\n\troutes=[nats-route://127.0.0.1:20104]\n\tno_advertise: true\n}\n`\n\n\ttmplLA := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account = SYS\njetstream: {\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 50Mb\n max_file: 50Mb\n\t%s\n}\nserver_name: LA\ncluster: {\n\tname: clustL\n\tlisten: 127.0.0.1:20106\n\troutes=[nats-route://127.0.0.1:20107]\n\tno_advertise: true\n}\nleafnodes:{\n\tno_advertise: true\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t {url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n}\n`\n\n\ttmplLB := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account = SYS\njetstream: {\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 50Mb\n max_file: 50Mb\n\t%s\n}\nserver_name: LB\ncluster: {\n\tname: clustL\n\tlisten: 127.0.0.1:20107\n\troutes=[nats-route://127.0.0.1:20106]\n\tno_advertise: true\n}\nleafnodes:{\n\tno_advertise: true\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t {url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n}\n`\n\n\tfor _, testCase := range []struct {\n\t\t// which topology to pick\n\t\tsame bool\n\t\t// If leaf server should be operational and form a Js cluster prior to joining.\n\t\t// In this setup this would be an error as you give the wrong hint.\n\t\t// But this should work itself out regardless\n\t\tleafFunctionPreJoin bool\n\t}{\n\t\t{true, true},\n\t\t{true, false},\n\t\t{false, true},\n\t\t{false, false}} {\n\t\tt.Run(fmt.Sprintf(\"%t-%t\", testCase.same, testCase.leafFunctionPreJoin), func(t *testing.T) {\n\t\t\tsd1 := t.TempDir()\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, sd1)))\n\t\t\tsA, _ := RunServerWithConfig(confA)\n\t\t\tdefer sA.Shutdown()\n\n\t\t\tsd2 := t.TempDir()\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(tmplB, sd2)))\n\t\t\tsB, _ := RunServerWithConfig(confB)\n\t\t\tdefer sB.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sA, sB)\n\n\t\t\tc := cluster{t: t, servers: []*Server{sA, sB}}\n\t\t\tc.waitOnLeader()\n\n\t\t\t// starting this will allow the second remote in tmplL to successfully connect.\n\t\t\tport := sB.opts.LeafNode.Port\n\t\t\tif testCase.same {\n\t\t\t\tport = sA.opts.LeafNode.Port\n\t\t\t}\n\t\t\tp := &proxyAcceptDetectFailureLate{acceptPort: port}\n\t\t\tdefer p.close()\n\t\t\tlPort := p.runEx(t, true)\n\n\t\t\thint := \"\"\n\t\t\tif testCase.leafFunctionPreJoin {\n\t\t\t\thint = fmt.Sprintf(\"extension_hint: %s\", strings.ToUpper(jsNoExtend))\n\t\t\t}\n\n\t\t\tsd3 := t.TempDir()\n\t\t\t// deliberately pick server sA and proxy\n\t\t\tconfLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, sd3, hint, sA.opts.LeafNode.Port, lPort)))\n\t\t\tsLA, _ := RunServerWithConfig(confLA)\n\t\t\tdefer sLA.Shutdown()\n\n\t\t\tsd4 := t.TempDir()\n\t\t\t// deliberately pick server sA and proxy\n\t\t\tconfLB := createConfFile(t, []byte(fmt.Sprintf(tmplLB, sd4, hint, sA.opts.LeafNode.Port, lPort)))\n\t\t\tsLB, _ := RunServerWithConfig(confLB)\n\t\t\tdefer sLB.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sLA, sLB)\n\n\t\t\tstrmCfg := func(name, placementCluster string) *nats.StreamConfig {\n\t\t\t\tif placementCluster == \"\" {\n\t\t\t\t\treturn &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}}\n\t\t\t\t}\n\t\t\t\treturn &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name},\n\t\t\t\t\tPlacement: &nats.Placement{Cluster: placementCluster}}\n\t\t\t}\n\t\t\t// Only after the system account is fully connected can streams be placed anywhere.\n\t\t\ttestJSFunctions := func(pass bool) {\n\t\t\t\tncA := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", sA.opts.Port))\n\t\t\t\tdefer ncA.Close()\n\t\t\t\tjsA, err := ncA.JetStream()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsA.AddStream(strmCfg(fmt.Sprintf(\"fooA1-%t\", pass), \"\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsA.AddStream(strmCfg(fmt.Sprintf(\"fooA2-%t\", pass), \"clust1\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsA.AddStream(strmCfg(fmt.Sprintf(\"fooA3-%t\", pass), \"clustL\"))\n\t\t\t\tif pass {\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Error(t, err)\n\t\t\t\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\")\n\t\t\t\t}\n\t\t\t\tncL := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", sLA.opts.Port))\n\t\t\t\tdefer ncL.Close()\n\t\t\t\tjsL, err := ncL.JetStream()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsL.AddStream(strmCfg(fmt.Sprintf(\"fooL1-%t\", pass), \"\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsL.AddStream(strmCfg(fmt.Sprintf(\"fooL2-%t\", pass), \"clustL\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsL.AddStream(strmCfg(fmt.Sprintf(\"fooL3-%t\", pass), \"clust1\"))\n\t\t\t\tif pass {\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Error(t, err)\n\t\t\t\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tclusterLnCnt := func(expected int) error {\n\t\t\t\tcnt := 0\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\tcnt += s.NumLeafNodes()\n\t\t\t\t}\n\t\t\t\tif cnt == expected {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"not enought leaf node connections, got %d needed %d\", cnt, expected)\n\t\t\t}\n\n\t\t\t// Even though there are two remotes defined in tmplL, only one will be able to connect.\n\t\t\tcheckFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(2) })\n\t\t\tcheckLeafNodeConnectedCount(t, sLA, 1)\n\t\t\tcheckLeafNodeConnectedCount(t, sLB, 1)\n\t\t\tc.waitOnPeerCount(2)\n\n\t\t\tif testCase.leafFunctionPreJoin {\n\t\t\t\tcl := cluster{t: t, servers: []*Server{sLA, sLB}}\n\t\t\t\tcl.waitOnLeader()\n\t\t\t\tcl.waitOnPeerCount(2)\n\t\t\t\ttestJSFunctions(false)\n\t\t\t} else {\n\t\t\t\t// In cases where the leaf nodes have to wait for the system account to connect,\n\t\t\t\t// JetStream should not be operational during that time\n\t\t\t\tncA := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", sLA.opts.Port))\n\t\t\t\tdefer ncA.Close()\n\t\t\t\tjsA, err := ncA.JetStream()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t_, err = jsA.AddStream(strmCfg(\"fail-false\", \"\"))\n\t\t\t\trequire_Error(t, err)\n\t\t\t}\n\t\t\t// Starting the proxy will connect the system accounts.\n\t\t\t// After they are connected the clusters are merged.\n\t\t\t// Once this happened, all streams in test can be placed anywhere in the cluster.\n\t\t\t// Before that only the cluster the client is connected to can be used for placement\n\t\t\tp.start()\n\n\t\t\t// Even though there are two remotes defined in tmplL, only one will be able to connect.\n\t\t\tcheckFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(4) })\n\t\t\tcheckLeafNodeConnectedCount(t, sLA, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sLB, 2)\n\n\t\t\t// The leader will reside in the main cluster only\n\t\t\tc.waitOnPeerCount(4)\n\t\t\ttestJSFunctions(true)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamLeafNodeClusterMixedModeExtensionWithSystemAccount(t *testing.T) {\n\t/* Topology used in this test:\n\tCLUSTER(A <-> B <-> C (NO JS))\n\t \t ^\n\t |\n\t LA\n\t*/\n\n\t// once every server is up, we expect these peers to be part of the JetStream meta cluster\n\texpectedJetStreamPeers := map[string]struct{}{\n\t\t\"A\": {},\n\t\t\"B\": {},\n\t\t\"LA\": {},\n\t}\n\n\ttmplA := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\njetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb }\nserver_name: A\ncluster: {\n\tname: clust1\n\tlisten: 127.0.0.1:20114\n\troutes=[nats-route://127.0.0.1:20115,nats-route://127.0.0.1:20116]\n\tno_advertise: true\n}\n`\n\n\ttmplB := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\njetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb }\nserver_name: B\ncluster: {\n\tname: clust1\n\tlisten: 127.0.0.1:20115\n\troutes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20116]\n\tno_advertise: true\n}\n`\n\n\ttmplC := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\njetstream: {\n\tenabled: false\n\t%s\n}\nserver_name: C\ncluster: {\n\tname: clust1\n\tlisten: 127.0.0.1:20116\n\troutes=[nats-route://127.0.0.1:20114,nats-route://127.0.0.1:20115]\n\tno_advertise: true\n}\n`\n\n\ttmplLA := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account = SYS\n# the extension hint is to simplify this test. without it present we would need a cluster of size 2\njetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb, extension_hint: will_extend }\nserver_name: LA\nleafnodes:{\n\tno_advertise: true\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t {url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n}\n# add the cluster here so we can test placement\ncluster: { name: clustL }\n`\n\tfor _, withDomain := range []bool{true, false} {\n\t\tt.Run(fmt.Sprintf(\"with-domain:%t\", withDomain), func(t *testing.T) {\n\t\t\tvar jsDisabledDomainString string\n\t\t\tvar jsEnabledDomainString string\n\t\t\tif withDomain {\n\t\t\t\tjsEnabledDomainString = `domain: \"domain\", `\n\t\t\t\tjsDisabledDomainString = `domain: \"domain\"`\n\t\t\t} else {\n\t\t\t\t// in case no domain name is set, fall back to the extension hint.\n\t\t\t\t// since JS is disabled, the value of this does not clash with other uses.\n\t\t\t\tjsDisabledDomainString = \"extension_hint: will_extend\"\n\t\t\t}\n\n\t\t\tsd1 := t.TempDir()\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, jsEnabledDomainString, sd1)))\n\t\t\tsA, _ := RunServerWithConfig(confA)\n\t\t\tdefer sA.Shutdown()\n\n\t\t\tsd2 := t.TempDir()\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(tmplB, jsEnabledDomainString, sd2)))\n\t\t\tsB, _ := RunServerWithConfig(confB)\n\t\t\tdefer sB.Shutdown()\n\n\t\t\tconfC := createConfFile(t, []byte(fmt.Sprintf(tmplC, jsDisabledDomainString)))\n\t\t\tsC, _ := RunServerWithConfig(confC)\n\t\t\tdefer sC.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sA, sB, sC)\n\t\t\tc := cluster{t: t, servers: []*Server{sA, sB, sC}}\n\t\t\tc.waitOnPeerCount(2)\n\n\t\t\tsd3 := t.TempDir()\n\t\t\t// deliberately pick server sC (no JS) to connect to\n\t\t\tconfLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, jsEnabledDomainString, sd3, sC.opts.LeafNode.Port, sC.opts.LeafNode.Port)))\n\t\t\tsLA, _ := RunServerWithConfig(confLA)\n\t\t\tdefer sLA.Shutdown()\n\n\t\t\tcheckLeafNodeConnectedCount(t, sC, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sLA, 2)\n\t\t\tc.waitOnPeerCount(3)\n\t\t\tpeers := c.leader().JetStreamClusterPeers()\n\t\t\tfor _, peer := range peers {\n\t\t\t\tif _, ok := expectedJetStreamPeers[peer]; !ok {\n\t\t\t\t\tt.Fatalf(\"Found unexpected peer %q\", peer)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// helper to create stream config with uniqe name and subject\n\t\t\tcnt := 0\n\t\t\tstrmCfg := func(placementCluster string) *nats.StreamConfig {\n\t\t\t\tname := fmt.Sprintf(\"s-%d\", cnt)\n\t\t\t\tcnt++\n\t\t\t\tif placementCluster == \"\" {\n\t\t\t\t\treturn &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}}\n\t\t\t\t}\n\t\t\t\treturn &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name},\n\t\t\t\t\tPlacement: &nats.Placement{Cluster: placementCluster}}\n\t\t\t}\n\n\t\t\ttest := func(port int, expectedDefPlacement string) {\n\t\t\t\tncA := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", port))\n\t\t\t\tdefer ncA.Close()\n\t\t\t\tjsA, err := ncA.JetStream()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tsi, err := jsA.AddStream(strmCfg(\"\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Contains(t, si.Cluster.Name, expectedDefPlacement)\n\t\t\t\tsi, err = jsA.AddStream(strmCfg(\"clust1\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Contains(t, si.Cluster.Name, \"clust1\")\n\t\t\t\tsi, err = jsA.AddStream(strmCfg(\"clustL\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Contains(t, si.Cluster.Name, \"clustL\")\n\t\t\t}\n\n\t\t\ttest(sA.opts.Port, \"clust1\")\n\t\t\ttest(sB.opts.Port, \"clust1\")\n\t\t\ttest(sC.opts.Port, \"clust1\")\n\t\t\ttest(sLA.opts.Port, \"clustL\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamLeafNodeCredsDenies(t *testing.T) {\n\ttmplL := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account = SYS\njetstream: {\n domain: \"cluster\"\n store_dir: '%s'\n max_mem: 50Mb\n max_file: 50Mb\n}\nleafnodes:{\n remotes:[{url:nats://a1:a1@127.0.0.1:20125, account: A, credentials: '%s' },\n\t\t {url:nats://s1:s1@127.0.0.1:20125, account: SYS, credentials: '%s', deny_imports: foo, deny_exports: bar}]\n}\n`\n\takp, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tcreds := createUserWithLimit(t, akp, time.Time{}, func(pl *jwt.UserPermissionLimits) {\n\t\tpl.Pub.Deny.Add(jsAllAPI)\n\t\tpl.Sub.Deny.Add(jsAllAPI)\n\t})\n\n\tsd := t.TempDir()\n\n\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL, sd, creds, creds)))\n\topts := LoadConfig(confL)\n\tsL, err := NewServer(opts)\n\trequire_NoError(t, err)\n\n\tl := captureNoticeLogger{}\n\tsL.SetLogger(&l, false, false)\n\n\tgo sL.Start()\n\tdefer sL.Shutdown()\n\n\t// wait till the notices got printed\nUNTIL_READY:\n\tfor {\n\t\t<-time.After(50 * time.Millisecond)\n\t\tl.Lock()\n\t\tfor _, n := range l.notices {\n\t\t\tif strings.Contains(n, \"Server is ready\") {\n\t\t\t\tl.Unlock()\n\t\t\t\tbreak UNTIL_READY\n\t\t\t}\n\t\t}\n\t\tl.Unlock()\n\t}\n\n\tl.Lock()\n\tcnt := 0\n\tfor _, n := range l.notices {\n\t\tif strings.Contains(n, \"LeafNode Remote for Account A uses credentials file\") ||\n\t\t\tstrings.Contains(n, \"LeafNode Remote for System Account uses\") ||\n\t\t\tstrings.Contains(n, \"Remote for System Account uses restricted export permissions\") ||\n\t\t\tstrings.Contains(n, \"Remote for System Account uses restricted import permissions\") {\n\t\t\tcnt++\n\t\t}\n\t}\n\tl.Unlock()\n\trequire_True(t, cnt == 4)\n}\n\nfunc TestJetStreamLeafNodeDefaultDomainCfg(t *testing.T) {\n\ttmplHub := `\nlisten: 127.0.0.1:%d\naccounts :{\n A:{ jetstream: %s, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\njetstream : %s\nserver_name: HUB\nleafnodes: {\n\tlisten: 127.0.0.1:%d\n}\n%s\n`\n\n\ttmplL := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\njetstream: { domain: \"%s\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: LEAF\nleafnodes: {\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},%s]\n}\n%s\n`\n\n\ttest := func(domain string, sysShared bool) {\n\t\tconfHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, \"disabled\", \"disabled\", -1, \"\")))\n\t\tsHub, _ := RunServerWithConfig(confHub)\n\t\tdefer sHub.Shutdown()\n\n\t\tnoDomainFix := \"\"\n\t\tif domain == _EMPTY_ {\n\t\t\tnoDomainFix = `default_js_domain:{A:\"\"}`\n\t\t}\n\n\t\tsys := \"\"\n\t\tif sysShared {\n\t\t\tsys = fmt.Sprintf(`{url:nats://s1:s1@127.0.0.1:%d, account: SYS}`, sHub.opts.LeafNode.Port)\n\t\t}\n\n\t\tsdLeaf := t.TempDir()\n\t\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, sys, noDomainFix)))\n\t\tsLeaf, _ := RunServerWithConfig(confL)\n\t\tdefer sLeaf.Shutdown()\n\n\t\tlnCnt := 1\n\t\tif sysShared {\n\t\t\tlnCnt++\n\t\t}\n\n\t\tcheckLeafNodeConnectedCount(t, sHub, lnCnt)\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, lnCnt)\n\n\t\tncA := natsConnect(t, fmt.Sprintf(\"nats://a1:a1@127.0.0.1:%d\", sHub.opts.Port))\n\t\tdefer ncA.Close()\n\t\tjsA, err := ncA.JetStream()\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsA.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, Subjects: []string{\"foo\"}})\n\t\trequire_True(t, err == nats.ErrNoResponders)\n\n\t\t// Add in default domain and restart server\n\t\trequire_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub,\n\t\t\tsHub.opts.Port,\n\t\t\t\"disabled\",\n\t\t\t\"disabled\",\n\t\t\tsHub.opts.LeafNode.Port,\n\t\t\tfmt.Sprintf(`default_js_domain: {A:\"%s\"}`, domain))), 0664))\n\n\t\tsHub.Shutdown()\n\t\tsHub.WaitForShutdown()\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, 0)\n\t\tsHubUpd1, _ := RunServerWithConfig(confHub)\n\t\tdefer sHubUpd1.Shutdown()\n\n\t\tcheckLeafNodeConnectedCount(t, sHubUpd1, lnCnt)\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, lnCnt)\n\n\t\t_, err = jsA.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, Subjects: []string{\"foo\"}})\n\t\trequire_NoError(t, err)\n\n\t\t// Enable jetstream in hub.\n\t\tsdHub := t.TempDir()\n\t\tjsEnabled := fmt.Sprintf(`{ domain: \"%s\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }`, domain, sdHub)\n\t\trequire_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub,\n\t\t\tsHubUpd1.opts.Port,\n\t\t\t\"disabled\",\n\t\t\tjsEnabled,\n\t\t\tsHubUpd1.opts.LeafNode.Port,\n\t\t\tfmt.Sprintf(`default_js_domain: {A:\"%s\"}`, domain))), 0664))\n\n\t\tsHubUpd1.Shutdown()\n\t\tsHubUpd1.WaitForShutdown()\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, 0)\n\t\tsHubUpd2, _ := RunServerWithConfig(confHub)\n\t\tdefer sHubUpd2.Shutdown()\n\n\t\tcheckLeafNodeConnectedCount(t, sHubUpd2, lnCnt)\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, lnCnt)\n\n\t\t_, err = jsA.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1, Subjects: []string{\"bar\"}})\n\t\trequire_NoError(t, err)\n\n\t\t// Enable jetstream in account A of hub\n\t\t// This is a mis config, as you can't have it both ways, local jetstream but default to another one\n\t\trequire_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub,\n\t\t\tsHubUpd2.opts.Port,\n\t\t\t\"enabled\",\n\t\t\tjsEnabled,\n\t\t\tsHubUpd2.opts.LeafNode.Port,\n\t\t\tfmt.Sprintf(`default_js_domain: {A:\"%s\"}`, domain))), 0664))\n\n\t\tif domain != _EMPTY_ {\n\t\t\t// in case no domain name exists there are no additional guard rails, hence no error\n\t\t\t// It is the users responsibility to get this edge case right\n\t\t\tsHubUpd2.Shutdown()\n\t\t\tsHubUpd2.WaitForShutdown()\n\t\t\tcheckLeafNodeConnectedCount(t, sLeaf, 0)\n\t\t\tsHubUpd3, err := NewServer(LoadConfig(confHub))\n\t\t\tsHubUpd3.Shutdown()\n\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Contains(t, err.Error(), `default_js_domain contains account name \"A\" with enabled JetStream`)\n\t\t}\n\t}\n\n\tt.Run(\"with-domain-sys\", func(t *testing.T) {\n\t\ttest(\"domain\", true)\n\t})\n\tt.Run(\"with-domain-nosys\", func(t *testing.T) {\n\t\ttest(\"domain\", false)\n\t})\n\tt.Run(\"no-domain\", func(t *testing.T) {\n\t\ttest(\"\", true)\n\t})\n\tt.Run(\"no-domain\", func(t *testing.T) {\n\t\ttest(\"\", false)\n\t})\n}\n\nfunc TestJetStreamLeafNodeDefaultDomainJwtExplicit(t *testing.T) {\n\ttmplHub := `\nlisten: 127.0.0.1:%d\noperator: %s\nsystem_account: %s\nresolver: MEM\nresolver_preload: {\n\t%s:%s\n\t%s:%s\n}\njetstream : disabled\nserver_name: HUB\nleafnodes: {\n\tlisten: 127.0.0.1:%d\n}\n%s\n`\n\n\ttmplL := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\njetstream: { domain: \"%s\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: LEAF\nleafnodes: {\n remotes:[{url:nats://127.0.0.1:%d, account: A, credentials: '%s'},\n\t\t {url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}]\n}\n%s\n`\n\n\ttest := func(domain string) {\n\t\tnoDomainFix := \"\"\n\t\tif domain == _EMPTY_ {\n\t\t\tnoDomainFix = `default_js_domain:{A:\"\"}`\n\t\t}\n\n\t\tsysKp, syspub := createKey(t)\n\t\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\t\tsysCreds := newUser(t, sysKp)\n\n\t\taKp, aPub := createKey(t)\n\t\taClaim := jwt.NewAccountClaims(aPub)\n\t\taJwt := encodeClaim(t, aClaim, aPub)\n\t\taCreds := newUser(t, aKp)\n\n\t\tconfHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, ojwt, syspub, syspub, sysJwt, aPub, aJwt, -1, \"\")))\n\t\tsHub, _ := RunServerWithConfig(confHub)\n\t\tdefer sHub.Shutdown()\n\n\t\tsdLeaf := t.TempDir()\n\t\tconfL := createConfFile(t, []byte(fmt.Sprintf(tmplL,\n\t\t\tdomain,\n\t\t\tsdLeaf,\n\t\t\tsHub.opts.LeafNode.Port,\n\t\t\taCreds,\n\t\t\tsHub.opts.LeafNode.Port,\n\t\t\tsysCreds,\n\t\t\tnoDomainFix)))\n\t\tsLeaf, _ := RunServerWithConfig(confL)\n\t\tdefer sLeaf.Shutdown()\n\n\t\tcheckLeafNodeConnectedCount(t, sHub, 2)\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, 2)\n\n\t\tncA := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", sHub.opts.Port), createUserCreds(t, nil, aKp))\n\t\tdefer ncA.Close()\n\t\tjsA, err := ncA.JetStream()\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsA.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, Subjects: []string{\"foo\"}})\n\t\trequire_True(t, err == nats.ErrNoResponders)\n\n\t\t// Add in default domain and restart server\n\t\trequire_NoError(t, os.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub,\n\t\t\tsHub.opts.Port, ojwt, syspub, syspub, sysJwt, aPub, aJwt, sHub.opts.LeafNode.Port,\n\t\t\tfmt.Sprintf(`default_js_domain: {%s:\"%s\"}`, aPub, domain))), 0664))\n\n\t\tsHub.Shutdown()\n\t\tsHub.WaitForShutdown()\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, 0)\n\t\tsHubUpd1, _ := RunServerWithConfig(confHub)\n\t\tdefer sHubUpd1.Shutdown()\n\n\t\tcheckLeafNodeConnectedCount(t, sHubUpd1, 2)\n\t\tcheckLeafNodeConnectedCount(t, sLeaf, 2)\n\n\t\t_, err = jsA.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1, Subjects: []string{\"bar\"}})\n\t\trequire_NoError(t, err)\n\t}\n\tt.Run(\"with-domain\", func(t *testing.T) {\n\t\ttest(\"domain\")\n\t})\n\tt.Run(\"no-domain\", func(t *testing.T) {\n\t\ttest(\"\")\n\t})\n}\n\nfunc TestJetStreamLeafNodeDefaultDomainClusterBothEnds(t *testing.T) {\n\t// test to ensure that default domain functions when both ends of the leaf node connection are clusters\n\ttmplHub1 := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enabled, users:[ {user:a1,password:a1}]},\n\tB:{ jetstream: enabled, users:[ {user:b1,password:b1}]}\n}\njetstream : { domain: \"DHUB\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: HUB1\ncluster: {\n\tname: HUB\n\tlisten: 127.0.0.1:20134\n\troutes=[nats-route://127.0.0.1:20135]\n}\nleafnodes: {\n\tlisten:127.0.0.1:-1\n}\n`\n\n\ttmplHub2 := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enabled, users:[ {user:a1,password:a1}]},\n\tB:{ jetstream: enabled, users:[ {user:b1,password:b1}]}\n}\njetstream : { domain: \"DHUB\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: HUB2\ncluster: {\n\tname: HUB\n\tlisten: 127.0.0.1:20135\n\troutes=[nats-route://127.0.0.1:20134]\n}\nleafnodes: {\n\tlisten:127.0.0.1:-1\n}\n`\n\n\ttmplL1 := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enabled, users:[ {user:a1,password:a1}]},\n\tB:{ jetstream: disabled, users:[ {user:b1,password:b1}]}\n}\njetstream: { domain: \"DLEAF\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: LEAF1\ncluster: {\n\tname: LEAF\n\tlisten: 127.0.0.1:20136\n\troutes=[nats-route://127.0.0.1:20137]\n}\nleafnodes: {\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}]\n}\ndefault_js_domain: {B:\"DHUB\"}\n`\n\n\ttmplL2 := `\nlisten: 127.0.0.1:-1\naccounts :{\n A:{ jetstream: enabled, users:[ {user:a1,password:a1}]},\n\tB:{ jetstream: disabled, users:[ {user:b1,password:b1}]}\n}\njetstream: { domain: \"DLEAF\", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }\nserver_name: LEAF2\ncluster: {\n\tname: LEAF\n\tlisten: 127.0.0.1:20137\n\troutes=[nats-route://127.0.0.1:20136]\n}\nleafnodes: {\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}]\n}\ndefault_js_domain: {B:\"DHUB\"}\n`\n\n\tsd1 := t.TempDir()\n\tconfHub1 := createConfFile(t, []byte(fmt.Sprintf(tmplHub1, sd1)))\n\tsHub1, _ := RunServerWithConfig(confHub1)\n\tdefer sHub1.Shutdown()\n\n\tsd2 := t.TempDir()\n\tconfHub2 := createConfFile(t, []byte(fmt.Sprintf(tmplHub2, sd2)))\n\tsHub2, _ := RunServerWithConfig(confHub2)\n\tdefer sHub2.Shutdown()\n\n\tcheckClusterFormed(t, sHub1, sHub2)\n\tc1 := cluster{t: t, servers: []*Server{sHub1, sHub2}}\n\tc1.waitOnPeerCount(2)\n\n\tsd3 := t.TempDir()\n\tconfLeaf1 := createConfFile(t, []byte(fmt.Sprintf(tmplL1, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port)))\n\tsLeaf1, _ := RunServerWithConfig(confLeaf1)\n\tdefer sLeaf1.Shutdown()\n\n\tsd4 := t.TempDir()\n\tconfLeaf2 := createConfFile(t, []byte(fmt.Sprintf(tmplL2, sd4, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port)))\n\tsLeaf2, _ := RunServerWithConfig(confLeaf2)\n\tdefer sLeaf2.Shutdown()\n\n\tcheckClusterFormed(t, sLeaf1, sLeaf2)\n\tc2 := cluster{t: t, servers: []*Server{sLeaf1, sLeaf2}}\n\tc2.waitOnPeerCount(2)\n\n\tcheckLeafNodeConnectedCount(t, sHub1, 4)\n\tcheckLeafNodeConnectedCount(t, sLeaf1, 2)\n\tcheckLeafNodeConnectedCount(t, sLeaf2, 2)\n\n\tncB := natsConnect(t, fmt.Sprintf(\"nats://b1:b1@127.0.0.1:%d\", sLeaf1.getOpts().Port))\n\tdefer ncB.Close()\n\tjsB1, err := ncB.JetStream()\n\trequire_NoError(t, err)\n\tsi, err := jsB1.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"HUB\")\n\n\tjsB2, err := ncB.JetStream(nats.Domain(\"DHUB\"))\n\trequire_NoError(t, err)\n\tsi, err = jsB2.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1, Subjects: []string{\"bar\"}})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"HUB\")\n}\n\nfunc TestJetStreamLeafNodeSvcImportExportCycle(t *testing.T) {\n\taccounts := `\n\taccounts {\n\t\tSYS: {\n\t\t\tusers: [{user: admin, password: admin}]\n\t\t}\n\t\tLEAF_USER: {\n\t\t\tusers: [{user: leaf_user, password: leaf_user}]\n\t\t\timports: [\n\t\t\t\t{service: {account: LEAF_INGRESS, subject: \"foo\"}}\n\t\t\t\t{service: {account: LEAF_INGRESS, subject: \"_INBOX.>\"}}\n\t\t\t\t{service: {account: LEAF_INGRESS, subject: \"$JS.leaf.API.>\"}, to: \"JS.leaf_ingress@leaf.API.>\" }\n\t\t\t]\n\t\t\tjetstream: enabled\n\t\t}\n\t\tLEAF_INGRESS: {\n\t\t\tusers: [{user: leaf_ingress, password: leaf_ingress}]\n\t\t\texports: [\n\t\t\t\t{service: \"foo\", accounts: [LEAF_USER]}\n\t\t\t\t{service: \"_INBOX.>\", accounts: [LEAF_USER]}\n\t\t\t\t{service: \"$JS.leaf.API.>\", response_type: \"stream\", accounts: [LEAF_USER]}\n\t\t\t]\n\t\t\timports: [\n\t\t\t]\n\t\t\tjetstream: enabled\n\t\t}\n\t}\n\tsystem_account: SYS\n\t`\n\n\thconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t%s\n\tlisten: \"127.0.0.1:-1\"\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n\t`, accounts)))\n\tdefer os.Remove(hconf)\n\ts, o := RunServerWithConfig(hconf)\n\tdefer s.Shutdown()\n\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t%s\n\tserver_name: leaf-server\n\tjetstream {\n\t\tstore_dir: '%s'\n\t\tdomain=leaf\n\t}\n\n\tlisten: \"127.0.0.1:-1\"\n\tleafnodes {\n\t\tremotes = [\n\t\t\t{\n\t\t\t\turls: [\"nats-leaf://leaf_ingress:leaf_ingress@127.0.0.1:%v\"]\n\t\t\t\taccount: \"LEAF_INGRESS\"\n\t\t\t}\n\t\t]\n\t}\n\t`, accounts, t.TempDir(), o.LeafNode.Port)))\n\tdefer os.Remove(lconf)\n\tsl, so := RunServerWithConfig(lconf)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, sl)\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://leaf_user:leaf_user@127.0.0.1:%v\", so.Port))\n\tdefer nc.Close()\n\n\tjs, _ := nc.JetStream(nats.APIPrefix(\"JS.leaf_ingress@leaf.API.\"))\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"msg\"))\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamLeafNodeJSClusterMigrateRecovery(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: hub, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"hub\", _EMPTY_, 3, 12232, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: leaf, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"leaf\", 3, 23913)\n\tdefer lnc.shutdown()\n\n\tlnc.waitOnClusterReady()\n\tfor _, s := range lnc.servers {\n\t\ts.setJetStreamMigrateOnRemoteLeaf()\n\t}\n\n\tnc, _ := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tljs, err := nc.JetStream(nats.Domain(\"leaf\"))\n\trequire_NoError(t, err)\n\n\t// Create an asset in the leafnode cluster.\n\tsi, err := ljs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"leaf\")\n\trequire_NotEqual(t, si.Cluster.Leader, noLeader)\n\trequire_Equal(t, len(si.Cluster.Replicas), 2)\n\n\t// Count how many remotes each server in the leafnode cluster is\n\t// supposed to have and then take them down.\n\tremotes := map[*Server]int{}\n\tfor _, s := range lnc.servers {\n\t\tremotes[s] += len(s.leafRemoteCfgs)\n\t\ts.closeAndDisableLeafnodes()\n\t\tcheckLeafNodeConnectedCount(t, s, 0)\n\t}\n\n\t// The Raft nodes in the leafnode cluster now need some time to\n\t// notice that they're no longer receiving AEs from a leader, as\n\t// they should have been forced into observer mode. Check that\n\t// this is the case.\n\ttime.Sleep(maxElectionTimeout)\n\tfor _, s := range lnc.servers {\n\t\ts.rnMu.RLock()\n\t\tfor name, n := range s.raftNodes {\n\t\t\t// We don't expect the metagroup to have turned into an\n\t\t\t// observer but all other assets should have done.\n\t\t\tif name == defaultMetaGroupName {\n\t\t\t\trequire_False(t, n.IsObserver())\n\t\t\t} else {\n\t\t\t\trequire_True(t, n.IsObserver())\n\t\t\t}\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t}\n\n\t// Bring the leafnode connections back up.\n\tfor _, s := range lnc.servers {\n\t\ts.reEnableLeafnodes()\n\t\tcheckLeafNodeConnectedCount(t, s, remotes[s])\n\t}\n\n\t// Wait for nodes to notice they are no longer in observer mode\n\t// and to leave observer mode.\n\ttime.Sleep(maxElectionTimeout)\n\tfor _, s := range lnc.servers {\n\t\ts.rnMu.RLock()\n\t\tfor _, n := range s.raftNodes {\n\t\t\trequire_False(t, n.IsObserver())\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t}\n\n\t// Previously nodes would have left observer mode but then would\n\t// have failed to elect a stream leader as they were stuck on a\n\t// long election timer. Now this should work reliably.\n\tlnc.waitOnStreamLeader(globalAccountName, \"TEST\")\n}\n\nfunc TestJetStreamLeafNodeJSClusterMigrateRecoveryWithDelay(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: hub, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"hub\", _EMPTY_, 3, 12232, true)\n\tdefer c.shutdown()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafNode, \"store_dir:\", \"domain: leaf, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"leaf\", 3, 23913)\n\tdefer lnc.shutdown()\n\n\tlnc.waitOnClusterReady()\n\tdelay := 5 * time.Second\n\tfor _, s := range lnc.servers {\n\t\ts.setJetStreamMigrateOnRemoteLeafWithDelay(delay)\n\t}\n\n\tnc, _ := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tljs, err := nc.JetStream(nats.Domain(\"leaf\"))\n\trequire_NoError(t, err)\n\n\t// Create an asset in the leafnode cluster.\n\tsi, err := ljs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"leaf\")\n\trequire_NotEqual(t, si.Cluster.Leader, noLeader)\n\trequire_Equal(t, len(si.Cluster.Replicas), 2)\n\n\t// Count how many remotes each server in the leafnode cluster is\n\t// supposed to have and then take them down.\n\tremotes := map[*Server]int{}\n\tfor _, s := range lnc.servers {\n\t\tremotes[s] += len(s.leafRemoteCfgs)\n\t\ts.closeAndDisableLeafnodes()\n\t\tcheckLeafNodeConnectedCount(t, s, 0)\n\t}\n\n\t// The Raft nodes in the leafnode cluster now need some time to\n\t// notice that they're no longer receiving AEs from a leader, as\n\t// they should have been forced into observer mode. Check that\n\t// this is the case.\n\t// We expect the nodes to become observers after the delay time.\n\tnow := time.Now()\n\ttimeout := maxElectionTimeout + delay\n\tsuccess := false\n\tfor time.Since(now) <= timeout {\n\t\tallObservers := true\n\t\tfor _, s := range lnc.servers {\n\t\t\ts.rnMu.RLock()\n\t\t\tfor name, n := range s.raftNodes {\n\t\t\t\tif name == defaultMetaGroupName {\n\t\t\t\t\trequire_False(t, n.IsObserver())\n\t\t\t\t} else if n.IsObserver() {\n\t\t\t\t\t// Make sure the migration delay is respected.\n\t\t\t\t\trequire_True(t, time.Since(now) > time.Duration(float64(delay)*0.7))\n\t\t\t\t} else {\n\t\t\t\t\tallObservers = false\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.rnMu.RUnlock()\n\t\t}\n\t\tif allObservers {\n\t\t\tsuccess = true\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\trequire_True(t, success)\n\n\t// Bring the leafnode connections back up.\n\tfor _, s := range lnc.servers {\n\t\ts.reEnableLeafnodes()\n\t\tcheckLeafNodeConnectedCount(t, s, remotes[s])\n\t}\n\n\t// Wait for nodes to notice they are no longer in observer mode\n\t// and to leave observer mode.\n\ttime.Sleep(maxElectionTimeout)\n\tfor _, s := range lnc.servers {\n\t\ts.rnMu.RLock()\n\t\tfor _, n := range s.raftNodes {\n\t\t\trequire_False(t, n.IsObserver())\n\t\t}\n\t\ts.rnMu.RUnlock()\n\t}\n\n\t// Make sure all delay timers in remotes are disabled\n\tfor _, s := range lnc.servers {\n\t\tfor _, r := range s.leafRemoteCfgs {\n\t\t\trequire_True(t, r.jsMigrateTimer == nil)\n\t\t}\n\t}\n\n\t// Previously nodes would have left observer mode but then would\n\t// have failed to elect a stream leader as they were stuck on a\n\t// long election timer. Now this should work reliably.\n\tlnc.waitOnStreamLeader(globalAccountName, \"TEST\")\n}\n\n// This will test that when a mirror or source construct is setup across a leafnode/domain\n// that it will recover quickly once the LN is re-established regardless\n// of backoff state of the internal consumer create.\nfunc TestJetStreamLeafNodeAndMirrorResyncAfterConnectionDown(t *testing.T) {\n\ttmplA := `\n\t\tlisten: -1\n\t\tserver_name: tcm\n\t\tjetstream {\n\t\t\tstore_dir: '%s',\n\t\t\tdomain: TCM\n\t\t}\n\t\taccounts {\n\t\t\tJS { users = [ { user: \"y\", pass: \"p\" } ]; jetstream: true }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleaf { port: -1 }\n `\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, t.TempDir())))\n\tsA, oA := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\n\t// Create a proxy - we will use this to simulate a network down event.\n\trtt, bw := 10*time.Microsecond, 10*1024*1024*1024\n\tproxy := newNetProxy(rtt, bw, bw, fmt.Sprintf(\"nats://y:p@127.0.0.1:%d\", oA.LeafNode.Port))\n\tdefer proxy.stop()\n\n\ttmplB := `\n\t\tlisten: -1\n\t\tserver_name: xmm\n\t\tjetstream {\n\t\t\tstore_dir: '%s',\n\t\t\tdomain: XMM\n\t\t}\n\t\taccounts {\n\t\t\tJS { users = [ { user: \"y\", pass: \"p\" } ]; jetstream: true }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleaf { remotes [ { url: %s, account: \"JS\" } ], reconnect: \"0.25s\" }\n `\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(tmplB, t.TempDir(), proxy.leafURL())))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sA.Shutdown()\n\n\t// Make sure we are connected ok.\n\tcheckLeafNodeConnectedCount(t, sA, 1)\n\tcheckLeafNodeConnectedCount(t, sB, 1)\n\n\t// We will have 3 streams that we will test for proper syncing after\n\t// the network is restored.\n\t//\n\t// 1. Mirror A --> B\n\t// 2. Mirror A <-- B\n\t// 3. Source A <-> B\n\n\t// Connect to sA.\n\tncA, jsA := jsClientConnect(t, sA, nats.UserInfo(\"y\", \"p\"))\n\tdefer ncA.Close()\n\n\t// Connect to sB.\n\tncB, jsB := jsClientConnect(t, sB, nats.UserInfo(\"y\", \"p\"))\n\tdefer ncB.Close()\n\n\t// Add in TEST-A\n\t_, err := jsA.AddStream(&nats.StreamConfig{Name: \"TEST-A\", Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t// Add in TEST-B\n\t_, err = jsB.AddStream(&nats.StreamConfig{Name: \"TEST-B\", Subjects: []string{\"bar\"}})\n\trequire_NoError(t, err)\n\n\t// Now setup mirrors.\n\t_, err = jsB.AddStream(&nats.StreamConfig{\n\t\tName: \"M-A\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST-A\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.TCM.API\"},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = jsA.AddStream(&nats.StreamConfig{\n\t\tName: \"M-B\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST-B\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.XMM.API\"},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Now add in the streams that will source from one another bi-directionally.\n\t_, err = jsA.AddStream(&nats.StreamConfig{\n\t\tName: \"SRC-A\",\n\t\tSubjects: []string{\"A.*\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"SRC-B\",\n\t\t\tFilterSubject: \"B.*\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.XMM.API\"},\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = jsB.AddStream(&nats.StreamConfig{\n\t\tName: \"SRC-B\",\n\t\tSubjects: []string{\"B.*\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"SRC-A\",\n\t\t\tFilterSubject: \"A.*\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.TCM.API\"},\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Now load them up with 500 messages.\n\tinitMsgs := 500\n\tfor i := 0; i < initMsgs; i++ {\n\t\t// Individual Streams\n\t\tjsA.PublishAsync(\"foo\", []byte(\"PAYLOAD\"))\n\t\tjsB.PublishAsync(\"bar\", []byte(\"PAYLOAD\"))\n\t\t// Bi-directional Sources\n\t\tjsA.PublishAsync(\"A.foo\", []byte(\"PAYLOAD\"))\n\t\tjsB.PublishAsync(\"B.bar\", []byte(\"PAYLOAD\"))\n\t}\n\tselect {\n\tcase <-jsA.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\tselect {\n\tcase <-jsB.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Utility to check the number of stream msgs.\n\tcheckStreamMsgs := func(js nats.JetStreamContext, sname string, expected int, perr error) error {\n\t\tt.Helper()\n\t\tif perr != nil {\n\t\t\treturn perr\n\t\t}\n\t\tsi, err := js.StreamInfo(sname)\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != uint64(expected) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs for %s, got state: %+v\", expected, sname, si.State)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Wait til we see all messages.\n\tcheckFor(t, 2*time.Second, 250*time.Millisecond, func() error {\n\t\terr := checkStreamMsgs(jsA, \"TEST-A\", initMsgs, nil)\n\t\terr = checkStreamMsgs(jsB, \"M-A\", initMsgs, err)\n\t\terr = checkStreamMsgs(jsB, \"TEST-B\", initMsgs, err)\n\t\terr = checkStreamMsgs(jsA, \"M-B\", initMsgs, err)\n\t\terr = checkStreamMsgs(jsA, \"SRC-A\", initMsgs*2, err)\n\t\terr = checkStreamMsgs(jsB, \"SRC-B\", initMsgs*2, err)\n\t\treturn err\n\t})\n\n\t// Take down proxy. This will stop any propagation of messages between TEST and M streams.\n\tproxy.stop()\n\n\t// Now add an additional 500 messages to originals on both sides.\n\tfor i := 0; i < initMsgs; i++ {\n\t\t// Individual Streams\n\t\tjsA.PublishAsync(\"foo\", []byte(\"PAYLOAD\"))\n\t\tjsB.PublishAsync(\"bar\", []byte(\"PAYLOAD\"))\n\t\t// Bi-directional Sources\n\t\tjsA.PublishAsync(\"A.foo\", []byte(\"PAYLOAD\"))\n\t\tjsB.PublishAsync(\"B.bar\", []byte(\"PAYLOAD\"))\n\t}\n\tselect {\n\tcase <-jsA.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\tselect {\n\tcase <-jsB.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcancelAndDelayConsumer := func(s *Server, stream string) {\n\t\t// Now make sure internal consumer is at max backoff.\n\t\tacc, err := s.lookupAccount(\"JS\")\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(stream)\n\t\trequire_NoError(t, err)\n\n\t\t// Reset sourceInfo to have lots of failures and last attempt 2 minutes ago.\n\t\t// Lock should be held on parent stream.\n\t\tresetSourceInfo := func(si *sourceInfo) {\n\t\t\t// Do not reset sip here to make sure that the internal logic clears.\n\t\t\tsi.fails = 100\n\t\t\tsi.lreq = time.Now().Add(-2 * time.Minute)\n\t\t}\n\n\t\t// Force the consumer to be canceled and we simulate 100 failed attempts\n\t\t// such that the next time we will try will be a long way out.\n\t\tmset.mu.Lock()\n\t\tif mset.mirror != nil {\n\t\t\tresetSourceInfo(mset.mirror)\n\t\t\tmset.cancelSourceInfo(mset.mirror)\n\t\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t\t} else if len(mset.sources) > 0 {\n\t\t\tfor iname, si := range mset.sources {\n\t\t\t\tresetSourceInfo(si)\n\t\t\t\tmset.cancelSourceInfo(si)\n\t\t\t\tmset.setupSourceConsumer(iname, si.sseq+1, time.Time{})\n\t\t\t}\n\t\t}\n\t\tmset.mu.Unlock()\n\t}\n\n\t// Mirrors\n\tcancelAndDelayConsumer(sA, \"M-B\")\n\tcancelAndDelayConsumer(sB, \"M-A\")\n\t// Now bi-directional sourcing\n\tcancelAndDelayConsumer(sA, \"SRC-A\")\n\tcancelAndDelayConsumer(sB, \"SRC-B\")\n\n\t// Now restart the network proxy.\n\tproxy.start()\n\n\t// Make sure we are connected ok.\n\tcheckLeafNodeConnectedCount(t, sA, 1)\n\tcheckLeafNodeConnectedCount(t, sB, 1)\n\n\t// These should be good before re-sync.\n\trequire_NoError(t, checkStreamMsgs(jsA, \"TEST-A\", initMsgs*2, nil))\n\trequire_NoError(t, checkStreamMsgs(jsB, \"TEST-B\", initMsgs*2, nil))\n\n\tstart := time.Now()\n\t// Wait til we see all messages.\n\tcheckFor(t, 2*time.Minute, 50*time.Millisecond, func() error {\n\t\terr := checkStreamMsgs(jsA, \"M-B\", initMsgs*2, err)\n\t\terr = checkStreamMsgs(jsB, \"M-A\", initMsgs*2, err)\n\t\terr = checkStreamMsgs(jsA, \"SRC-A\", initMsgs*4, err)\n\t\terr = checkStreamMsgs(jsB, \"SRC-B\", initMsgs*4, err)\n\t\treturn err\n\t})\n\tif elapsed := time.Since(start); elapsed > 3*time.Second {\n\t\tt.Fatalf(\"Expected to resync all streams <3s but got %v\", elapsed)\n\t}\n}\n\n// This test will test a 3 node setup where we have a hub node, a gateway node, and a satellite node.\n// This is specifically testing re-sync when there is not a direct Domain with JS match for the first\n// hop connect LN that is signaling.\n//\n//\t\t HUB <---- GW(+JS/DOMAIN) -----> SAT1\n//\t\t ^\n//\t\t |\n//\t +------- GW(-JS/NO DOMAIN) --> SAT2\n//\n// The Gateway node will solicit the satellites but will act as a LN hub.\nfunc TestJetStreamLeafNodeAndMirrorResyncAfterLeafEstablished(t *testing.T) {\n\taccs := `\n\t\taccounts {\n\t\t\tJS { users = [ { user: \"u\", pass: \"p\" } ]; jetstream: true }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t`\n\thubT := `\n\t\tlisten: -1\n\t\tserver_name: hub\n\t\tjetstream { store_dir: '%s', domain: HUB }\n\t\t%s\n\t\tleaf { port: -1 }\n `\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(hubT, t.TempDir(), accs)))\n\tsHub, oHub := RunServerWithConfig(confA)\n\tdefer sHub.Shutdown()\n\n\t// We run the SAT node second to extract out info for solicitation from targeted GW.\n\tsat1T := `\n\t\tlisten: -1\n\t\tserver_name: sat1\n\t\tjetstream { store_dir: '%s', domain: SAT1 }\n\t\t%s\n\t\tleaf { port: -1 }\n `\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(sat1T, t.TempDir(), accs)))\n\tsSat1, oSat1 := RunServerWithConfig(confB)\n\tdefer sSat1.Shutdown()\n\n\tsat2T := `\n\t\tlisten: -1\n\t\tserver_name: sat2\n\t\tjetstream { store_dir: '%s', domain: SAT2 }\n\t\t%s\n\t\tleaf { port: -1 }\n `\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(sat2T, t.TempDir(), accs)))\n\tsSat2, oSat2 := RunServerWithConfig(confC)\n\tdefer sSat2.Shutdown()\n\n\thubLeafPort := fmt.Sprintf(\"nats://u:p@127.0.0.1:%d\", oHub.LeafNode.Port)\n\tsat1LeafPort := fmt.Sprintf(\"nats://u:p@127.0.0.1:%d\", oSat1.LeafNode.Port)\n\tsat2LeafPort := fmt.Sprintf(\"nats://u:p@127.0.0.1:%d\", oSat2.LeafNode.Port)\n\n\tgw1T := `\n\t\tlisten: -1\n\t\tserver_name: gw1\n\t\tjetstream { store_dir: '%s', domain: GW }\n\t\t%s\n\t\tleaf { remotes [ { url: %s, account: \"JS\" }, { url: %s, account: \"JS\", hub: true } ], reconnect: \"0.25s\" }\n `\n\tconfD := createConfFile(t, []byte(fmt.Sprintf(gw1T, t.TempDir(), accs, hubLeafPort, sat1LeafPort)))\n\tsGW1, _ := RunServerWithConfig(confD)\n\tdefer sGW1.Shutdown()\n\n\tgw2T := `\n\t\tlisten: -1\n\t\tserver_name: gw2\n\t\taccounts {\n\t\t\tJS { users = [ { user: \"u\", pass: \"p\" } ] }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleaf { remotes [ { url: %s, account: \"JS\" }, { url: %s, account: \"JS\", hub: true } ], reconnect: \"0.25s\" }\n `\n\tconfE := createConfFile(t, []byte(fmt.Sprintf(gw2T, hubLeafPort, sat2LeafPort)))\n\tsGW2, _ := RunServerWithConfig(confE)\n\tdefer sGW2.Shutdown()\n\n\t// Make sure we are connected ok.\n\tcheckLeafNodeConnectedCount(t, sHub, 2)\n\tcheckLeafNodeConnectedCount(t, sSat1, 1)\n\tcheckLeafNodeConnectedCount(t, sSat2, 1)\n\tcheckLeafNodeConnectedCount(t, sGW1, 2)\n\tcheckLeafNodeConnectedCount(t, sGW2, 2)\n\n\t// Let's place a muxed stream on the hub and have it source from a stream on the Satellite.\n\t// Connect to Hub.\n\tncHub, jsHub := jsClientConnect(t, sHub, nats.UserInfo(\"u\", \"p\"))\n\tdefer ncHub.Close()\n\n\t_, err := jsHub.AddStream(&nats.StreamConfig{Name: \"HUB\", Subjects: []string{\"H.>\"}})\n\trequire_NoError(t, err)\n\n\t// Connect to Sat1.\n\tncSat1, jsSat1 := jsClientConnect(t, sSat1, nats.UserInfo(\"u\", \"p\"))\n\tdefer ncSat1.Close()\n\n\t_, err = jsSat1.AddStream(&nats.StreamConfig{\n\t\tName: \"SAT-1\",\n\t\tSubjects: []string{\"S1.*\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"HUB\",\n\t\t\tFilterSubject: \"H.SAT-1.>\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.HUB.API\"},\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Connect to Sat2.\n\tncSat2, jsSat2 := jsClientConnect(t, sSat2, nats.UserInfo(\"u\", \"p\"))\n\tdefer ncSat2.Close()\n\n\t_, err = jsSat2.AddStream(&nats.StreamConfig{\n\t\tName: \"SAT-2\",\n\t\tSubjects: []string{\"S2.*\"},\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"HUB\",\n\t\t\tFilterSubject: \"H.SAT-2.>\",\n\t\t\tExternal: &nats.ExternalStream{APIPrefix: \"$JS.HUB.API\"},\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Put in 10 msgs each in for each satellite.\n\tfor i := 0; i < 10; i++ {\n\t\tjsHub.Publish(\"H.SAT-1.foo\", []byte(\"CMD\"))\n\t\tjsHub.Publish(\"H.SAT-2.foo\", []byte(\"CMD\"))\n\t}\n\t// Make sure both are sync'd.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsSat1.StreamInfo(\"SAT-1\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 10 {\n\t\t\treturn errors.New(\"SAT-1 Not sync'd yet\")\n\t\t}\n\t\tsi, err = jsSat2.StreamInfo(\"SAT-2\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 10 {\n\t\t\treturn errors.New(\"SAT-2 Not sync'd yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\ttestReconnect := func(t *testing.T, delay time.Duration, expected uint64) {\n\t\t// Now disconnect Sat1 and Sat2. In 2.12 we can do this with active: false, but since this will be\n\t\t// pulled into 2.11.9 just shutdown both gateways.\n\t\tsGW1.Shutdown()\n\t\tcheckLeafNodeConnectedCount(t, sSat1, 0)\n\t\tcheckLeafNodeConnectedCount(t, sHub, 1)\n\n\t\tsGW2.Shutdown()\n\t\tcheckLeafNodeConnectedCount(t, sSat2, 0)\n\t\tcheckLeafNodeConnectedCount(t, sHub, 0)\n\n\t\t// Send 10 more messages for each while GW1 and GW2 are down.\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tjsHub.Publish(\"H.SAT-1.foo\", []byte(\"CMD\"))\n\t\t\tjsHub.Publish(\"H.SAT-2.foo\", []byte(\"CMD\"))\n\t\t}\n\n\t\t// Keep GWs down for delay.\n\t\ttime.Sleep(delay)\n\n\t\tsGW1, _ = RunServerWithConfig(confD)\n\t\t// Make sure we are connected ok.\n\t\tcheckLeafNodeConnectedCount(t, sHub, 1)\n\t\tcheckLeafNodeConnectedCount(t, sSat1, 1)\n\t\tcheckLeafNodeConnectedCount(t, sGW1, 2)\n\n\t\tsGW2, _ = RunServerWithConfig(confE)\n\t\t// Make sure we are connected ok.\n\t\tcheckLeafNodeConnectedCount(t, sHub, 2)\n\t\tcheckLeafNodeConnectedCount(t, sSat2, 1)\n\t\tcheckLeafNodeConnectedCount(t, sGW2, 2)\n\n\t\t// Make sure sync'd in less than a second or two.\n\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := jsSat1.StreamInfo(\"SAT-1\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif si.State.Msgs != expected {\n\t\t\t\treturn fmt.Errorf(\"SAT-1 not sync'd, expected %d got %d\", expected, si.State.Msgs)\n\t\t\t}\n\t\t\tsi, err = jsSat2.StreamInfo(\"SAT-2\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif si.State.Msgs != expected {\n\t\t\t\treturn fmt.Errorf(\"SAT-2 not sync'd, expected %d got %d\", expected, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// We will test two scenarios with amount of time the GWs (link) is down.\n\t// 1. Just a second, we will not have detected the consumer is offline as of yet.\n\t// 2. Just over sourceHealthCheckInterval, meaning we detect it is down and schedule for another try.\n\tt.Run(fmt.Sprintf(\"reconnect-%v\", time.Second), func(t *testing.T) {\n\t\ttestReconnect(t, time.Second, 20)\n\t})\n\tt.Run(fmt.Sprintf(\"reconnect-%v\", sourceHealthCheckInterval+time.Second), func(t *testing.T) {\n\t\ttestReconnect(t, sourceHealthCheckInterval+time.Second, 30)\n\t})\n\tdefer sGW1.Shutdown()\n\tdefer sGW2.Shutdown()\n}\n" }, { "path": "server/jetstream_meta_benchmark_test.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc BenchmarkJetStreamCreate(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tresourcePrefix = \"S\"\n\t\tconcurrency = 12\n\t)\n\n\t// Types of resource that this benchmark creates\n\ttype ResourceType string\n\tconst (\n\t\tStream ResourceType = \"Stream\"\n\t\tKVBucket ResourceType = \"KVBucket\"\n\t\tObjectStore ResourceType = \"ObjStore\"\n\t)\n\n\tresourceTypeCases := []ResourceType{\n\t\tStream,\n\t\tKVBucket,\n\t\tObjectStore,\n\t}\n\n\tbenchmarksCases := []struct {\n\t\tclusterSize int\n\t\treplicas int\n\t\tstorage nats.StorageType\n\t}{\n\t\t{1, 1, nats.MemoryStorage},\n\t\t{3, 3, nats.MemoryStorage},\n\t\t{3, 3, nats.FileStorage},\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\t\tbName := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,storage=%s,C=%d\",\n\t\t\tbc.clusterSize,\n\t\t\tbc.replicas,\n\t\t\tbc.storage.String(),\n\t\t\tconcurrency,\n\t\t)\n\n\t\tb.Run(\n\t\t\tbName,\n\t\t\tfunc(b *testing.B) {\n\t\t\t\tfor _, rt := range resourceTypeCases {\n\t\t\t\t\t//for _, bc := range benchmarksCases {\n\t\t\t\t\trName := fmt.Sprintf(\"resource=%s\", rt)\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\trName,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\n\t\t\t\t\t\t\t\t\t\"Creating %d %s resources in cluster with %d nodes, R=%d, %s storage\",\n\t\t\t\t\t\t\t\t\tb.N,\n\t\t\t\t\t\t\t\t\tstring(rt),\n\t\t\t\t\t\t\t\t\tbc.clusterSize,\n\t\t\t\t\t\t\t\t\tbc.replicas,\n\t\t\t\t\t\t\t\t\tbc.storage,\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Setup server or cluster\n\t\t\t\t\t\t\t_, leaderServer, shutdown, nc, _ := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t// All clients connect to cluster (meta) leader for lower variability\n\t\t\t\t\t\t\tconnectURL := leaderServer.ClientURL()\n\n\t\t\t\t\t\t\t// Wait for all clients and main routine to be ready\n\t\t\t\t\t\t\twgReady := sync.WaitGroup{}\n\t\t\t\t\t\t\twgReady.Add(concurrency + 1)\n\t\t\t\t\t\t\t// Wait for all routines to complete\n\t\t\t\t\t\t\twgComplete := sync.WaitGroup{}\n\t\t\t\t\t\t\twgComplete.Add(concurrency)\n\n\t\t\t\t\t\t\t// Number of operations (divided amongst clients)\n\t\t\t\t\t\t\topsLeft := atomic.Int64{}\n\t\t\t\t\t\t\topsLeft.Store(int64(b.N))\n\t\t\t\t\t\t\ttotalErrors := atomic.Int64{}\n\n\t\t\t\t\t\t\t// Pre-create connections and JS contexts\n\t\t\t\t\t\t\tfor i := 1; i <= concurrency; i++ {\n\t\t\t\t\t\t\t\tnc, js := jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\t\t\t\tgo func(clientId int, nc *nats.Conn, js nats.JetStreamContext) {\n\t\t\t\t\t\t\t\t\tdefer wgComplete.Done()\n\n\t\t\t\t\t\t\t\t\t// Config struct (reused and modified in place for each call)\n\t\t\t\t\t\t\t\t\tstreamConfig := nats.StreamConfig{\n\t\t\t\t\t\t\t\t\t\tName: \"?\",\n\t\t\t\t\t\t\t\t\t\tStorage: bc.storage,\n\t\t\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tkvConfig := nats.KeyValueConfig{\n\t\t\t\t\t\t\t\t\t\tBucket: \"?\",\n\t\t\t\t\t\t\t\t\t\tStorage: bc.storage,\n\t\t\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tobjConfig := nats.ObjectStoreConfig{\n\t\t\t\t\t\t\t\t\t\tBucket: \"?\",\n\t\t\t\t\t\t\t\t\t\tStorage: bc.storage,\n\t\t\t\t\t\t\t\t\t\tReplicas: bc.replicas,\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t// Block until everyone is ready\n\t\t\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t\t\terrCount := int64(0)\n\t\t\t\t\t\t\t\t\tdefer func() {\n\t\t\t\t\t\t\t\t\t\t// Roll up error count on completion\n\t\t\t\t\t\t\t\t\t\ttotalErrors.Add(errCount)\n\t\t\t\t\t\t\t\t\t}()\n\n\t\t\t\t\t\t\t\t\t// Track per-client opCount (just for logging/debugging)\n\t\t\t\t\t\t\t\t\topCount := 0\n\t\t\t\t\t\t\t\t\tfor opsLeft.Add(-1) >= 0 {\n\t\t\t\t\t\t\t\t\t\tvar err error\n\t\t\t\t\t\t\t\t\t\t// Create unique resource name\n\t\t\t\t\t\t\t\t\t\tresourceName := fmt.Sprintf(\"%s_%d_%d\", resourcePrefix, clientId, opCount)\n\t\t\t\t\t\t\t\t\t\tswitch rt {\n\t\t\t\t\t\t\t\t\t\tcase Stream:\n\t\t\t\t\t\t\t\t\t\t\tstreamConfig.Name = resourceName\n\t\t\t\t\t\t\t\t\t\t\t_, err = js.AddStream(&streamConfig)\n\t\t\t\t\t\t\t\t\t\tcase KVBucket:\n\t\t\t\t\t\t\t\t\t\t\tkvConfig.Bucket = resourceName\n\t\t\t\t\t\t\t\t\t\t\t_, err = js.CreateKeyValue(&kvConfig)\n\t\t\t\t\t\t\t\t\t\tcase ObjectStore:\n\t\t\t\t\t\t\t\t\t\t\tobjConfig.Bucket = resourceName\n\t\t\t\t\t\t\t\t\t\t\t_, err = js.CreateObjectStore(&objConfig)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\topCount += 1\n\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Error creating %s (%s): %s\", rt, resourceName, err)\n\t\t\t\t\t\t\t\t\t\t\terrCount += 1\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\tb.Logf(\"Client %d completed %d operations\", clientId, opCount)\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t}(i, nc, js)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Wait for all clients to be ready\n\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t// Start benchmark clock\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\twgComplete.Wait()\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n\nfunc BenchmarkJetStreamCreateConsumers(b *testing.B) {\n\n\tconst (\n\t\tverbose = false\n\t\tstreamName = \"S\"\n\t\tconsumerPrefix = \"C\"\n\t\tconcurrency = 12\n\t)\n\n\tbenchmarksCases := []struct {\n\t\tclusterSize int\n\t\tconsumerReplicas int\n\t\tconsumerStorage nats.StorageType\n\t}{\n\t\t{1, 1, nats.MemoryStorage},\n\t\t{3, 3, nats.MemoryStorage},\n\t\t{3, 3, nats.FileStorage},\n\t}\n\n\ttype ConsumerType string\n\tconst (\n\t\tEphemeral ConsumerType = \"Ephemeral\"\n\t\tDurable ConsumerType = \"Durable\"\n\t)\n\n\tconsumerTypeCases := []ConsumerType{\n\t\tEphemeral,\n\t\tDurable,\n\t}\n\n\tfor _, bc := range benchmarksCases {\n\n\t\tbName := fmt.Sprintf(\n\t\t\t\"N=%d,R=%d,storage=%s,C=%d\",\n\t\t\tbc.clusterSize,\n\t\t\tbc.consumerReplicas,\n\t\t\tbc.consumerStorage.String(),\n\t\t\tconcurrency,\n\t\t)\n\n\t\tb.Run(\n\t\t\tbName,\n\t\t\tfunc(b *testing.B) {\n\n\t\t\t\tfor _, ct := range consumerTypeCases {\n\n\t\t\t\t\tcName := fmt.Sprintf(\"Consumer=%s\", ct)\n\n\t\t\t\t\tb.Run(\n\t\t\t\t\t\tcName,\n\t\t\t\t\t\tfunc(b *testing.B) {\n\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\tb.Logf(\n\t\t\t\t\t\t\t\t\t\"Creating %d consumers in cluster with %d nodes, R=%d, %s storage\",\n\t\t\t\t\t\t\t\t\tb.N,\n\t\t\t\t\t\t\t\t\tbc.clusterSize,\n\t\t\t\t\t\t\t\t\tbc.consumerReplicas,\n\t\t\t\t\t\t\t\t\tbc.consumerStorage,\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Setup server or cluster\n\t\t\t\t\t\t\t_, leaderServer, shutdown, nc, js := startJSClusterAndConnect(b, bc.clusterSize)\n\t\t\t\t\t\t\tdefer shutdown()\n\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t// All clients connect to cluster (meta) leader for lower variability\n\t\t\t\t\t\t\tconnectURL := leaderServer.ClientURL()\n\n\t\t\t\t\t\t\t// Create stream\n\t\t\t\t\t\t\tstreamConfig := nats.StreamConfig{\n\t\t\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\t\t\tStorage: nats.FileStorage,\n\t\t\t\t\t\t\t\tReplicas: bc.clusterSize,\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t_, err := js.AddStream(&streamConfig)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tb.Fatalf(\"Failed to create stream: %s\", err)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Wait for all clients and main routine to be ready\n\t\t\t\t\t\t\twgReady := sync.WaitGroup{}\n\t\t\t\t\t\t\twgReady.Add(concurrency + 1)\n\t\t\t\t\t\t\t// Wait for all routines to complete\n\t\t\t\t\t\t\twgComplete := sync.WaitGroup{}\n\t\t\t\t\t\t\twgComplete.Add(concurrency)\n\n\t\t\t\t\t\t\t// Number of operations (divided amongst clients)\n\t\t\t\t\t\t\topsLeft := atomic.Int64{}\n\t\t\t\t\t\t\topsLeft.Store(int64(b.N))\n\t\t\t\t\t\t\t// Total number of errors\n\t\t\t\t\t\t\ttotalErrors := atomic.Int64{}\n\n\t\t\t\t\t\t\t// Pre-create connections and JS contexts\n\t\t\t\t\t\t\tfor i := 1; i <= concurrency; i++ {\n\t\t\t\t\t\t\t\tnc, js := jsClientConnectURL(b, connectURL)\n\t\t\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\t\t\tgo func(clientId int, nc *nats.Conn, js nats.JetStreamContext) {\n\t\t\t\t\t\t\t\t\tdefer wgComplete.Done()\n\n\t\t\t\t\t\t\t\t\t// Config struct (reused and modified in place for each call)\n\t\t\t\t\t\t\t\t\tcfg := nats.ConsumerConfig{\n\t\t\t\t\t\t\t\t\t\tDurable: \"\",\n\t\t\t\t\t\t\t\t\t\tName: \"\",\n\t\t\t\t\t\t\t\t\t\tReplicas: bc.consumerReplicas,\n\t\t\t\t\t\t\t\t\t\tMemoryStorage: bc.consumerStorage == nats.MemoryStorage,\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\t// Block until everyone is ready\n\t\t\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t\t\terrCount := int64(0)\n\t\t\t\t\t\t\t\t\topCount := 0\n\t\t\t\t\t\t\t\t\tfor opsLeft.Add(-1) >= 0 {\n\t\t\t\t\t\t\t\t\t\tvar err error\n\t\t\t\t\t\t\t\t\t\t// Set unique consumer name\n\t\t\t\t\t\t\t\t\t\tcfg.Name = fmt.Sprintf(\"%s_%d_%d\", consumerPrefix, clientId, opCount)\n\t\t\t\t\t\t\t\t\t\tif ct == Durable {\n\t\t\t\t\t\t\t\t\t\t\tcfg.Durable = cfg.Name\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t_, err = js.AddConsumer(streamName, &cfg)\n\t\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\t\tb.Logf(\"Failed to add consumer: %s\", err)\n\t\t\t\t\t\t\t\t\t\t\terrCount += 1\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\topCount += 1\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\t\t\t\tb.Logf(\"Client %d completed %d operations\", clientId, opCount)\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\ttotalErrors.Add(errCount)\n\n\t\t\t\t\t\t\t\t}(i, nc, js)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t// Wait for all clients to be ready\n\t\t\t\t\t\t\twgReady.Done()\n\t\t\t\t\t\t\twgReady.Wait()\n\n\t\t\t\t\t\t\t// Start benchmark clock\n\t\t\t\t\t\t\tb.ResetTimer()\n\n\t\t\t\t\t\t\twgComplete.Wait()\n\t\t\t\t\t\t\tb.StopTimer()\n\n\t\t\t\t\t\t\tb.ReportMetric(float64(100*(totalErrors.Load()))/float64(b.N), \"%error\")\n\t\t\t\t\t\t},\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t}\n}\n" }, { "path": "server/jetstream_sourcing_scaling_test.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nvar serverConfig1 = `\n\tserver_name: server1\n\tlisten: 127.0.0.1:4222\n\thttp: 8222\n\n\tprof_port = 18222\n\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n \t\tname: my_cluster\n \t\tlisten: 127.0.0.1:4248\n \t\troutes: [nats://127.0.0.1:4249,nats://127.0.0.1:4250]\n\t}\n\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n`\n\nvar serverConfig2 = `\n\tserver_name: server2\n\tlisten: 127.0.0.1:5222\n\thttp: 8223\n\n\tprof_port = 18223\n\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n \t\tname: my_cluster\n \t\tlisten: 127.0.0.1:4249\n \t\troutes: [nats://127.0.0.1:4248,nats://127.0.0.1:4250]\n\t}\n\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n`\n\nvar serverConfig3 = `\n\tserver_name: server3\n\tlisten: 127.0.0.1:6222\n\thttp: 8224\n\n\tprof_port = 18224\n\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n \t\tname: my_cluster\n \t\tlisten: 127.0.0.1:4250\n \t\troutes: [nats://127.0.0.1:4248,nats://127.0.0.1:4249]\n\t}\n\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n`\n\nvar connectURL = \"nats://127.0.0.1:4222,nats://127.0.0.1:4223,nats://127.0.0.1:4224\"\n\nfunc createMyLocalCluster(t *testing.T) *cluster {\n\tc := &cluster{servers: make([]*Server, 0, 3), opts: make([]*Options, 0, 3), name: \"C3\"}\n\n\tstoreDir1 := t.TempDir()\n\ts1, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig1, storeDir1))))\n\tc.servers = append(c.servers, s1)\n\tc.opts = append(c.opts, o)\n\n\tstoreDir2 := t.TempDir()\n\ts2, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig2, storeDir2))))\n\tc.servers = append(c.servers, s2)\n\tc.opts = append(c.opts, o)\n\n\tstoreDir3 := t.TempDir()\n\ts3, o := RunServerWithConfig(createConfFile(t, []byte(fmt.Sprintf(serverConfig3, storeDir3))))\n\tc.servers = append(c.servers, s3)\n\tc.opts = append(c.opts, o)\n\n\tc.t = t\n\n\t// Wait til we are formed and have a leader.\n\tc.checkClusterFormed()\n\tc.waitOnClusterReady()\n\n\treturn c\n}\n\n// This test is being skipped by CI as it takes too long to run and is meant to test the scalability of sourcing\n// rather than being a unit test.\nfunc TestStreamSourcingScalingSourcingManyBenchmark(t *testing.T) {\n\tt.Skip()\n\n\tvar numSourced = 1000\n\tvar numMsgPerSource = 10_000\n\tvar batchSize = 200\n\tvar retries int\n\n\tvar err error\n\n\tc := createMyLocalCluster(t)\n\tdefer c.shutdown()\n\n\tnc, err := nats.Connect(connectURL)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tjs, err := nc.JetStream(nats.MaxWait(10 * time.Second))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error getting JetStream context: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// create n streams to source from\n\tfor i := 0; i < numSourced; i++ {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: fmt.Sprintf(\"sourced-%d\", i),\n\t\t\tSubjects: []string{strconv.Itoa(i)},\n\t\t\tRetention: nats.LimitsPolicy,\n\t\t\tStorage: nats.FileStorage,\n\t\t\tDiscard: nats.DiscardOld,\n\t\t\tReplicas: 1,\n\t\t\tAllowDirect: true,\n\t\t\tMirrorDirect: false,\n\t\t\tDiscardNewPerSubject: false,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tfmt.Printf(\"Streams created\\n\")\n\n\t// publish n messages for each sourced stream\n\tfor j := 0; j < numMsgPerSource; j++ {\n\t\tstart := time.Now()\n\t\tvar pafs = make([]nats.PubAckFuture, numSourced)\n\t\tfor i := 0; i < numSourced; i++ {\n\t\t\tvar err error\n\n\t\t\tfor {\n\t\t\t\tpafs[i], err = js.PublishAsync(strconv.Itoa(i), []byte(strconv.Itoa(j)))\n\t\t\t\tif err != nil {\n\t\t\t\t\tfmt.Printf(\"Error async publishing: %v, retrying\\n\", err)\n\t\t\t\t\tretries++\n\t\t\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\t\t} else {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif i != 0 && i%batchSize == 0 {\n\t\t\t\t<-js.PublishAsyncComplete()\n\t\t\t}\n\n\t\t}\n\n\t\t<-js.PublishAsyncComplete()\n\n\t\tfor i := 0; i < numSourced; i++ {\n\t\t\tselect {\n\t\t\tcase <-pafs[i].Ok():\n\t\t\tcase psae := <-pafs[i].Err():\n\t\t\t\tfmt.Printf(\"Error on PubAckFuture: %v, retrying sync...\\n\", psae)\n\t\t\t\tretries++\n\t\t\t\t_, err = js.Publish(strconv.Itoa(i), []byte(strconv.Itoa(j)))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\n\t\tend := time.Now()\n\t\tif j%1000 == 0 {\n\t\t\tfmt.Printf(\"Published round %d, avg pub latency %v\\n\", j, end.Sub(start)/time.Duration(numSourced))\n\t\t}\n\t}\n\n\tfmt.Printf(\"Messages published\\n\")\n\n\t// create the StreamSources\n\tstreamSources := make([]*nats.StreamSource, numSourced)\n\n\tfor i := 0; i < numSourced; i++ {\n\t\tstreamSources[i] = &nats.StreamSource{Name: fmt.Sprintf(\"sourced-%d\", i), FilterSubject: strconv.Itoa(i)}\n\t}\n\n\t// create a stream that sources from them\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"sourcing\",\n\t\tSubjects: []string{\"foo\"},\n\t\tSources: streamSources,\n\t\tRetention: nats.LimitsPolicy,\n\t\tStorage: nats.FileStorage,\n\t\tDiscard: nats.DiscardOld,\n\t\tReplicas: 3,\n\t\tAllowDirect: true,\n\t\tMirrorDirect: false,\n\t\tDiscardNewPerSubject: false,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"sourcing\")\n\tsl := c.streamLeader(globalAccountName, \"sourcing\")\n\tfmt.Printf(\"Sourcing stream created leader is *** %s ***\\n\", sl.Name())\n\n\texpectedSeq := make([]int, numSourced)\n\n\tstart := time.Now()\n\n\tvar lastMsgs uint64\n\tmset, err := sl.GlobalAccount().lookupStream(\"sourcing\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 5*time.Minute, 1000*time.Millisecond, func() error {\n\t\tmset.mu.RLock()\n\t\tvar state StreamState\n\t\tmset.store.FastState(&state)\n\t\tmset.mu.RUnlock()\n\n\t\tif state.Msgs == uint64(numMsgPerSource*numSourced) {\n\t\t\tfmt.Printf(\"👍 Test passed: expected %d messages, got %d and took %v\\n\", uint64(numMsgPerSource*numSourced), state.Msgs, time.Since(start))\n\t\t\treturn nil\n\t\t} else if state.Msgs < uint64(numMsgPerSource*numSourced) {\n\t\t\tfmt.Printf(\"Current Rate %d per second - Received %d\\n\", state.Msgs-lastMsgs, state.Msgs)\n\t\t\tlastMsgs = state.Msgs\n\t\t\treturn fmt.Errorf(\"Expected %d messages, got %d\", uint64(numMsgPerSource*numSourced), state.Msgs)\n\t\t} else {\n\t\t\tfmt.Printf(\"Too many messages! expected %d (retries=%d), got %d\\n\", uint64(numMsgPerSource*numSourced), retries, state.Msgs)\n\t\t\treturn fmt.Errorf(\"Too many messages: expected %d (retries=%d), got %d\", uint64(numMsgPerSource*numSourced), retries, state.Msgs)\n\t\t}\n\t})\n\n\t// Check that all the messages sourced in the stream are correct\n\t// Note: expects to see exactly increasing matching sequence numbers, so could theoretically fail if some messages\n\t// get recorded 'out of order' (according to the payload value) because asynchronous JS publication is used to\n\t// publish the messages.\n\t// However, that should not happen if the publish 'batch size' is not more than the number of streams being sourced.\n\n\t// create a consumer on sourcing\n\t_, err = js.AddConsumer(\"sourcing\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tsyncSub, err := js.SubscribeSync(\"\", nats.BindStream(\"sourcing\"))\n\trequire_NoError(t, err)\n\n\tstart = time.Now()\n\n\tprint(\"Checking the messages\\n\")\n\tfor i := 0; i < numSourced*numMsgPerSource; i++ {\n\t\tmsg, err := syncSub.NextMsg(30 * time.Second)\n\t\trequire_NoError(t, err)\n\t\tsId, err := strconv.Atoi(msg.Subject)\n\t\trequire_NoError(t, err)\n\t\tseq, err := strconv.Atoi(string(msg.Data))\n\t\trequire_NoError(t, err)\n\t\tif expectedSeq[sId] == seq {\n\t\t\texpectedSeq[sId]++\n\t\t} else {\n\t\t\tt.Fatalf(\"Expected seq number %d got %d for source %d\\n\", expectedSeq[sId], seq, sId)\n\t\t}\n\t\tmsg.Ack()\n\t\tif i%100_000 == 0 {\n\t\t\tnow := time.Now()\n\t\t\tfmt.Printf(\"[%v] Checked %d messages: %f msgs/sec \\n\", now, i, 100_000/now.Sub(start).Seconds())\n\t\t\tstart = now\n\t\t}\n\t}\n\tprint(\"👍 Done. \\n\")\n}\n" }, { "path": "server/jetstream_super_cluster_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests && !skip_js_cluster_tests && !skip_js_cluster_tests_2 && !skip_js_super_cluster_tests\n\npackage server\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc TestJetStreamSuperClusterMetaStepDown(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\tsc.waitOnLeader()\n\n\t// Client based API\n\ts := sc.randomServer()\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tstepdown := func(preferred, cn string, tags []string) *JSApiLeaderStepDownResponse {\n\t\tjreq, err := json.Marshal(&JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tCluster: cn,\n\t\t\t\tTags: tags,\n\t\t\t\tPreferred: preferred,\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(JSApiLeaderStepDown, jreq, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar sdr JSApiLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\t\treturn &sdr\n\t}\n\n\t// Make sure we get correct errors for clusters we don't know about.\n\tt.Run(\"UnknownCluster\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, \"ThisClusterDoesntExist\", nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers we don't know about.\n\tt.Run(\"UnknownPreferredServer\", func(t *testing.T) {\n\t\tsdr := stepdown(\"ThisServerDoesntExist\", _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags that don't match any servers.\n\tt.Run(\"UnknownTag\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{\"thistag:doesntexist\"})\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers that are already leader.\n\tt.Run(\"PreferredServerAlreadyLeader\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\tsdr := stepdown(ml.Name(), _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags and bad or unavailable cluster placement.\n\tt.Run(\"PlacementByPreferredServer\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\tvar preferredServer string\n\tclusters:\n\t\tfor _, c := range sc.clusters {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tif s == ml {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tpreferredServer = s.Name()\n\t\t\t\tbreak clusters\n\t\t\t}\n\t\t}\n\n\t\tsdr := stepdown(preferredServer, _EMPTY_, nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tsc.waitOnLeader()\n\t\tml = sc.leader()\n\t\trequire_Equal(t, ml.Name(), preferredServer)\n\t})\n\n\t// Influence the placement by using the cluster name.\n\tt.Run(\"PlacementByCluster\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\tcn := ml.ClusterName()\n\t\tvar pcn string\n\t\tfor _, c := range sc.clusters {\n\t\t\tif c.name != cn {\n\t\t\t\tpcn = c.name\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, pcn, nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tsc.waitOnLeader()\n\t\tml = sc.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_Equal(t, ml.ClusterName(), pcn)\n\t})\n\n\t// Influence the placement by using tag names.\n\tt.Run(\"PlacementByTag\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"cloud:aws\": {},\n\t\t\t\"cloud:gcp\": {},\n\t\t\t\"cloud:az\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range ml.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tsc.waitOnLeader()\n\t\tml = sc.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_True(t, ml.getOpts().Tags.Contains(chosenTag))\n\t})\n\n\t// Influence the placement by using tag names, we need to match all of them.\n\tt.Run(\"PlacementByMultipleTags\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"cloud:aws\": {},\n\t\t\t\"cloud:gcp\": {},\n\t\t\t\"cloud:az\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range ml.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag, \"node:1\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tsc.waitOnLeader()\n\t\tml = sc.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_True(t, ml.getOpts().Tags.Contains(chosenTag))\n\t\trequire_True(t, ml.getOpts().Tags.Contains(\"node:1\"))\n\t})\n\n\t// Influence the placement by using the cluster name and a tag.\n\tt.Run(\"PlacementByClusterAndTag\", func(t *testing.T) {\n\t\tml := sc.leader()\n\t\trequire_NotNil(t, ml)\n\n\t\tcn := ml.ClusterName()\n\t\tvar pcn string\n\t\tfor _, c := range sc.clusters {\n\t\t\tif c.name != cn {\n\t\t\t\tpcn = c.name\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, pcn, []string{\"node:1\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tsc.waitOnLeader()\n\t\tml = sc.leader()\n\t\trequire_NotNil(t, ml)\n\t\trequire_Equal(t, ml.ClusterName(), pcn)\n\t\trequire_True(t, ml.getOpts().Tags.Contains(\"node:1\"))\n\t})\n}\n\nfunc TestJetStreamSuperClusterStreamStepDown(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tsc.waitOnLeader()\n\ts := sc.randomServer()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t\tPlacement: &nats.Placement{\n\t\t\tCluster: \"C1\",\n\t\t\tTags: []string{\"cloud:aws\"},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tstepdown := func(preferred, cn string, tags []string) *JSApiStreamLeaderStepDownResponse {\n\t\tjreq, err := json.Marshal(&JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tCluster: cn,\n\t\t\t\tTags: tags,\n\t\t\t\tPreferred: preferred,\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"foo\"), jreq, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar sdr JSApiStreamLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\t\treturn &sdr\n\t}\n\n\t// Make sure we get correct errors for clusters we don't know about.\n\tt.Run(\"UnknownCluster\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, \"ThisClusterDoesntExist\", nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers we don't know about.\n\tt.Run(\"UnknownPreferredServer\", func(t *testing.T) {\n\t\tsdr := stepdown(\"ThisServerDoesntExist\", _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags that don't match any servers.\n\tt.Run(\"UnknownTag\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{\"thistag:doesntexist\"})\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for clusters that we aren't assigned to.\n\t// The asset is in C1, not C2, so placement should fail.\n\tt.Run(\"NonParticipantCluster\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, \"C2\", nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers that are already leader.\n\tt.Run(\"PreferredServerAlreadyLeader\", func(t *testing.T) {\n\t\tsl := sc.clusterForName(\"C1\").streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\tsdr := stepdown(sl.Name(), _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags and bad or unavailable cluster placement.\n\tt.Run(\"PlacementByPreferredServer\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tsl := c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\tvar preferredServer string\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpreferredServer = s.Name()\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(preferredServer, _EMPTY_, nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\t\tsl = c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_Equal(t, sl.Name(), preferredServer)\n\t})\n\n\t// Influence the placement by using the cluster name. For streams this doesn't really\n\t// make sense, since the stream can only exist in a single cluster (the one that it\n\t// had its placement in), so this effectively works the same as specifying a stepdown\n\t// without a cluster name specified. Let's just make sure it does the right thing in\n\t// any case.\n\tt.Run(\"PlacementByCluster\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tsl := c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\tsdr := stepdown(_EMPTY_, \"C1\", nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\t\tsl = c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\t\trequire_Equal(t, sl.ClusterName(), \"C1\")\n\t})\n\n\t// Influence the placement by using tag names.\n\tt.Run(\"PlacementByTag\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tsl := c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range sl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\t\tsl = c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\t\trequire_True(t, sl.getOpts().Tags.Contains(chosenTag))\n\t})\n\n\t// Influence the placement by using tag names, we need to match all of them.\n\tt.Run(\"PlacementByMultipleTags\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tsl := c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range sl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag, \"cloud:aws\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\t\tsl = c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\t\trequire_True(t, sl.getOpts().Tags.Contains(chosenTag))\n\t\trequire_True(t, sl.getOpts().Tags.Contains(\"cloud:aws\"))\n\t})\n\n\t// Influence the placement by using the cluster name and a tag. Like with\n\t// PlacementByCluster above, the cluster portion of this request is not really\n\t// doing anything, but it's useful just to ensure the API behaves properly.\n\tt.Run(\"PlacementByClusterAndTag\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tsl := c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range sl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, \"C1\", []string{chosenTag, \"cloud:aws\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\t\tsl = c.streamLeader(globalAccountName, \"foo\")\n\t\trequire_NotNil(t, sl)\n\t\trequire_True(t, sl.getOpts().Tags.Contains(chosenTag))\n\t\trequire_True(t, sl.getOpts().Tags.Contains(\"cloud:aws\"))\n\t\trequire_Equal(t, sl.ClusterName(), \"C1\")\n\t})\n}\n\nfunc TestJetStreamSuperClusterConsumerStepDown(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tsc.waitOnLeader()\n\ts := sc.randomServer()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t\tPlacement: &nats.Placement{\n\t\t\tCluster: \"C1\",\n\t\t\tTags: []string{\"cloud:aws\"},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tName: \"consumer\",\n\t})\n\trequire_NoError(t, err)\n\n\tstepdown := func(preferred, cn string, tags []string) *JSApiStreamLeaderStepDownResponse {\n\t\tjreq, err := json.Marshal(&JSApiLeaderStepdownRequest{\n\t\t\tPlacement: &Placement{\n\t\t\t\tCluster: cn,\n\t\t\t\tTags: tags,\n\t\t\t\tPreferred: preferred,\n\t\t\t},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"foo\", \"consumer\"), jreq, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar sdr JSApiStreamLeaderStepDownResponse\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &sdr))\n\t\treturn &sdr\n\t}\n\n\t// Make sure we get correct errors for clusters we don't know about.\n\tt.Run(\"UnknownCluster\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, \"ThisClusterDoesntExist\", nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers we don't know about.\n\tt.Run(\"UnknownPreferredServer\", func(t *testing.T) {\n\t\tsdr := stepdown(\"ThisServerDoesntExist\", _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags that don't match any servers.\n\tt.Run(\"UnknownTag\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{\"thistag:doesntexist\"})\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for clusters that we aren't assigned to.\n\t// The asset is in C1, not C2, so placement should fail.\n\tt.Run(\"NonParticipantCluster\", func(t *testing.T) {\n\t\tsdr := stepdown(_EMPTY_, \"C2\", nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for servers that are already leader.\n\tt.Run(\"PreferredServerAlreadyLeader\", func(t *testing.T) {\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\tsdr := stepdown(cl.Name(), _EMPTY_, nil)\n\t\trequire_NotNil(t, sdr.Error)\n\t\trequire_Equal(t, sdr.Error.Code, 400)\n\t\trequire_Equal(t, ErrorIdentifier(sdr.Error.ErrCode), JSClusterNoPeersErrF)\n\t})\n\n\t// Make sure we get correct errors for tags and bad or unavailable cluster placement.\n\tt.Run(\"PlacementByPreferredServer\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\tvar preferredServer string\n\t\tfor _, s := range c.servers {\n\t\t\tif s == cl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpreferredServer = s.Name()\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(preferredServer, _EMPTY_, nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif cl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\"); cl == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer leader is nil\")\n\t\t\t}\n\t\t\tif cl.Name() != preferredServer {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q is not preferred %q\", cl.Name(), preferredServer)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n\n\t// Influence the placement by using the cluster name. For consumers this doesn't really\n\t// make sense, since the consumers can only exist in a single cluster (the one that it\n\t// had its placement in), so this effectively works the same as specifying a stepdown\n\t// without a cluster name specified. Let's just make sure it does the right thing in\n\t// any case.\n\tt.Run(\"PlacementByCluster\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\tsdr := stepdown(_EMPTY_, \"C1\", nil)\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif cl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\"); cl == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer leader is nil\")\n\t\t\t}\n\t\t\tif cl.ClusterName() != \"C1\" {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q is not in cluster C1\", cl.Name())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n\n\t// Influence the placement by using tag names.\n\tt.Run(\"PlacementByTag\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range cl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif cl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\"); cl == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer leader is nil\")\n\t\t\t}\n\t\t\tif !cl.getOpts().Tags.Contains(chosenTag) {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q doesn't contain tag %q\", cl.Name(), chosenTag)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n\n\t// Influence the placement by using tag names, we need to match all of them.\n\tt.Run(\"PlacementByMultipleTags\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range cl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, _EMPTY_, []string{chosenTag, \"cloud:aws\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif cl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\"); cl == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer leader is nil\")\n\t\t\t}\n\t\t\tif !cl.getOpts().Tags.Contains(chosenTag) {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q doesn't contain tag %q\", cl.Name(), chosenTag)\n\t\t\t}\n\t\t\tif !cl.getOpts().Tags.Contains(\"cloud:aws\") {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q isn't in cloud:aws\", cl.Name())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n\n\t// Influence the placement by using the cluster name and a tag. Like with\n\t// PlacementByCluster above, the cluster portion of this request is not really\n\t// doing anything, but it's useful just to ensure the API behaves properly.\n\tt.Run(\"PlacementByClusterAndTag\", func(t *testing.T) {\n\t\tc := sc.clusterForName(\"C1\")\n\t\tcl := sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\trequire_NotNil(t, cl)\n\n\t\t// Work out what the tags are of the current leader, so we can pick\n\t\t// different ones.\n\t\tpossibleTags := map[string]struct{}{\n\t\t\t\"node:1\": {},\n\t\t\t\"node:2\": {},\n\t\t\t\"node:3\": {},\n\t\t}\n\t\t// Remove the current leader's tags from the list.\n\t\tfor _, tag := range cl.getOpts().Tags {\n\t\t\tdelete(possibleTags, tag)\n\t\t}\n\t\t// Now pick the first tag as our new chosen tag.\n\t\tvar chosenTag string\n\t\tfor tag := range possibleTags {\n\t\t\tchosenTag = tag\n\t\t\tbreak\n\t\t}\n\n\t\tsdr := stepdown(_EMPTY_, \"C1\", []string{chosenTag, \"cloud:aws\"})\n\t\trequire_Equal(t, sdr.Error, nil)\n\n\t\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"consumer\")\n\t\tcl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\")\n\n\t\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\t\tif cl = sc.clusterForName(\"C1\").consumerLeader(globalAccountName, \"foo\", \"consumer\"); cl == nil {\n\t\t\t\treturn fmt.Errorf(\"consumer leader is nil\")\n\t\t\t}\n\t\t\tif cl.ClusterName() != \"C1\" {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q is not in cluster C1\", cl.Name())\n\t\t\t}\n\t\t\tif !cl.getOpts().Tags.Contains(chosenTag) {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q doesn't contain tag %q\", cl.Name(), chosenTag)\n\t\t\t}\n\t\t\tif !cl.getOpts().Tags.Contains(\"cloud:aws\") {\n\t\t\t\treturn fmt.Errorf(\"consumer leader %q isn't in cloud:aws\", cl.Name())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t})\n}\n\nfunc TestJetStreamSuperClusterUniquePlacementTag(t *testing.T) {\n\ttmlp := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', unique_tag: az}\n\t\tleaf {listen: 127.0.0.1:-1}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\ts := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 5, 2,\n\t\tfunc(serverName, clustername, storeDir, conf string) string {\n\t\t\tazTag := map[string]string{\n\t\t\t\t\"C1-S1\": \"az:same\",\n\t\t\t\t\"C1-S2\": \"az:same\",\n\t\t\t\t\"C1-S3\": \"az:same\",\n\t\t\t\t\"C1-S4\": \"az:same\",\n\t\t\t\t\"C1-S5\": \"az:same\",\n\t\t\t\t\"C2-S1\": \"az:1\",\n\t\t\t\t\"C2-S2\": \"az:2\",\n\t\t\t\t\"C2-S3\": \"az:1\",\n\t\t\t\t\"C2-S4\": \"az:2\",\n\t\t\t\t\"C2-S5\": \"az:1\",\n\t\t\t}\n\t\t\treturn conf + fmt.Sprintf(\"\\nserver_tags: [cloud:%s-tag, %s]\\n\", clustername, azTag[serverName])\n\t\t}, nil)\n\tdefer s.shutdown()\n\n\tinDifferentAz := func(ci *nats.ClusterInfo) (bool, error) {\n\t\tt.Helper()\n\t\tif len(ci.Replicas) == 0 {\n\t\t\treturn true, nil\n\t\t}\n\t\t// if R2 (has replica, this setup does not support R3), test if the server in a cluster picked the same az,\n\t\t// as determined by modulo2 of server number which aligns with az\n\t\tdummy := 0\n\t\tsrvnum1 := 0\n\t\tsrvnum2 := 0\n\t\tif n, _ := fmt.Sscanf(ci.Leader, \"C%d-S%d\", &dummy, &srvnum1); n != 2 {\n\t\t\treturn false, fmt.Errorf(\"couldn't parse leader\")\n\t\t}\n\t\tif n, _ := fmt.Sscanf(ci.Replicas[0].Name, \"C%d-S%d\", &dummy, &srvnum2); n != 2 {\n\t\t\treturn false, fmt.Errorf(\"couldn't parse replica\")\n\t\t}\n\t\treturn srvnum1%2 != srvnum2%2, nil\n\t}\n\n\tnc := natsConnect(t, s.randomServer().ClientURL())\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\tfor i, test := range []struct {\n\t\tplacement *nats.Placement\n\t\treplicas int\n\t\tfail bool\n\t\tcluster string\n\t}{\n\t\t// these pass because replica count is 1\n\t\t{&nats.Placement{Tags: []string{\"az:same\"}}, 1, false, \"C1\"},\n\t\t{&nats.Placement{Tags: []string{\"cloud:C1-tag\", \"az:same\"}}, 1, false, \"C1\"},\n\t\t{&nats.Placement{Tags: []string{\"cloud:C1-tag\"}}, 1, false, \"C1\"},\n\t\t// pass because az is set, which disables the filter\n\t\t{&nats.Placement{Tags: []string{\"az:same\"}}, 2, false, \"C1\"},\n\t\t{&nats.Placement{Tags: []string{\"cloud:C1-tag\", \"az:same\"}}, 2, false, \"C1\"},\n\t\t// fails because this cluster only has the same az\n\t\t{&nats.Placement{Tags: []string{\"cloud:C1-tag\"}}, 2, true, \"\"},\n\t\t// fails because no 3 unique tags exist\n\t\t{&nats.Placement{Tags: []string{\"cloud:C2-tag\"}}, 3, true, \"\"},\n\t\t{nil, 3, true, \"\"},\n\t\t// pass because replica count is low enough\n\t\t{nil, 2, false, \"C2\"},\n\t\t{&nats.Placement{Tags: []string{\"cloud:C2-tag\"}}, 2, false, \"C2\"},\n\t\t// pass because az is provided\n\t\t{&nats.Placement{Tags: []string{\"az:1\"}}, 3, false, \"C2\"},\n\t\t{&nats.Placement{Tags: []string{\"az:2\"}}, 2, false, \"C2\"},\n\t} {\n\t\tname := fmt.Sprintf(\"test-%d\", i)\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tsi, err := js.AddStream(&nats.StreamConfig{Name: name, Replicas: test.replicas, Placement: test.placement})\n\t\t\tif test.fail {\n\t\t\t\trequire_Error(t, err)\n\t\t\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"server tag not unique\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\t\t\tif test.cluster != _EMPTY_ {\n\t\t\t\trequire_Equal(t, si.Cluster.Name, test.cluster)\n\t\t\t}\n\t\t\t// skip placement test if tags call for a particular az\n\t\t\tif test.placement != nil && len(test.placement.Tags) > 0 {\n\t\t\t\tfor _, tag := range test.placement.Tags {\n\t\t\t\t\tif strings.HasPrefix(tag, \"az:\") {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tdiff, err := inDifferentAz(si.Cluster)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, diff)\n\t\t})\n\t}\n\n\tt.Run(\"scale-up-test\", func(t *testing.T) {\n\t\t// create enough streams so we hit it eventually\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tcfg := &nats.StreamConfig{Name: fmt.Sprintf(\"scale-up-%d\", i), Replicas: 1,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:C2-tag\"}}}\n\t\t\tsi, err := js.AddStream(cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.Cluster.Name, \"C2\")\n\t\t\ts.waitOnStreamLeader(globalAccountName, cfg.Name)\n\t\t\tcfg.Replicas = 2\n\t\t\tsi, err = js.UpdateStream(cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.Cluster.Name, \"C2\")\n\t\t\tcheckFor(t, 10, 250*time.Millisecond, func() error {\n\t\t\t\tif si, err := js.StreamInfo(cfg.Name); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if diff, err := inDifferentAz(si.Cluster); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t} else if !diff {\n\t\t\t\t\treturn fmt.Errorf(\"not in different AZ\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t})\n}\n\nfunc TestJetStreamSuperClusterBasics(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Check request origin placement.\n\tif si.Cluster.Name != s.ClusterName() {\n\t\tt.Fatalf(\"Expected stream to be placed in %q, but got %q\", s.ClusterName(), si.Cluster.Name)\n\t}\n\n\t// Check consumers.\n\tsub, err := js.SubscribeSync(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, toSend)\n\tci, err := sub.ConsumerInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ci.Delivered.Consumer != uint64(toSend) || ci.NumAckPending != toSend {\n\t\tt.Fatalf(\"ConsumerInfo is not correct: %+v\", ci)\n\t}\n\n\t// Now check we can place a stream.\n\tpcn := \"C3\"\n\tscResp, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif scResp.Cluster.Name != pcn {\n\t\tt.Fatalf(\"Expected the stream to be placed in %q, got %q\", pcn, scResp.Cluster.Name)\n\t}\n}\n\n// Test that consumer interest across gateways and superclusters is properly identitifed in a remote cluster.\nfunc TestJetStreamSuperClusterCrossClusterConsumerInterest(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Since we need all of the peers accounted for to add the stream wait for all to be present.\n\tsc.waitOnPeerCount(9)\n\n\t// Client based API - Connect to Cluster C1. Stream and consumer will live in C2.\n\ts := sc.clusterForName(\"C1\").randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tpcn := \"C2\"\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 3, Placement: &nats.Placement{Cluster: pcn}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Pull based first.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send a message.\n\tif _, err = js.Publish(\"foo\", []byte(\"CCI\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tfetchMsgs(t, sub, 1, 5*time.Second)\n\n\t// Now check push based delivery.\n\tsub, err = js.SubscribeSync(\"foo\", nats.Durable(\"rip\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, 1)\n\n\t// Send another message.\n\tif _, err = js.Publish(\"foo\", []byte(\"CCI\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\tcheckSubsPending(t, sub, 2)\n}\n\nfunc TestJetStreamSuperClusterPeerReassign(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tpcn := \"C2\"\n\n\t// Create a stream in C2 that sources TEST\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send in 10 messages.\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\t// Now grab info for this stream.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si == nil || si.Config.Name != \"TEST\" {\n\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\t// Check request origin placement.\n\tif si.Cluster.Name != pcn {\n\t\tt.Fatalf(\"Expected stream to be placed in %q, but got %q\", s.ClusterName(), si.Cluster.Name)\n\t}\n\n\t// Now remove a peer that is assigned to the stream.\n\trc := sc.clusterForName(pcn)\n\trs := rc.randomNonStreamLeader(\"$G\", \"TEST\")\n\trc.removeJetStream(rs)\n\n\t// Check the stream info is eventually correct.\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not fetch stream info: %v\", err)\n\t\t}\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\tfor _, peer := range si.Cluster.Replicas {\n\t\t\tif !peer.Current {\n\t\t\t\treturn fmt.Errorf(\"Expected replica to be current: %+v\", peer)\n\t\t\t}\n\t\t\tif !strings.HasPrefix(peer.Name, pcn) {\n\t\t\t\tt.Fatalf(\"Stream peer reassigned to wrong cluster: %q\", peer.Name)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterInterestOnlyMode(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\taccounts {\n\t\t\tone {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers [{user: one, password: password}]\n\t\t\t}\n\t\t\ttwo {\n\t\t\t\t%s\n\t\t\t\tusers [{user: two, password: password}]\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\tname: %s\n\t\t\troutes = [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t\tgateway {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\tgateways = [{name: %s, urls: [\"nats://127.0.0.1:%d\"]}]\n\t\t}\n\t`\n\tstoreDir1 := t.TempDir()\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\t\"S1\", storeDir1, \"\", 23222, \"A\", 23222, \"A\", 11222, \"B\", 11223)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tstoreDir2 := t.TempDir()\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\t\"S2\", storeDir2, \"\", 23223, \"B\", 23223, \"B\", 11223, \"A\", 11222)))\n\ts2, o2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\n\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://two:password@127.0.0.1:%d\", o1.Port))\n\tdefer nc1.Close()\n\tnc1.Publish(\"foo\", []byte(\"some message\"))\n\tnc1.Flush()\n\n\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://two:password@127.0.0.1:%d\", o2.Port))\n\tdefer nc2.Close()\n\tnc2.Publish(\"bar\", []byte(\"some message\"))\n\tnc2.Flush()\n\n\tcheckMode := func(accName string, expectedMode GatewayInterestMode) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tservers := []*Server{s1, s2}\n\t\t\tfor _, s := range servers {\n\t\t\t\tvar gws []*client\n\t\t\t\ts.getInboundGatewayConnections(&gws)\n\t\t\t\tfor _, gw := range gws {\n\t\t\t\t\tvar mode GatewayInterestMode\n\t\t\t\t\tgw.mu.Lock()\n\t\t\t\t\tie := gw.gw.insim[accName]\n\t\t\t\t\tif ie != nil {\n\t\t\t\t\t\tmode = ie.mode\n\t\t\t\t\t}\n\t\t\t\t\tgw.mu.Unlock()\n\t\t\t\t\tif ie == nil {\n\t\t\t\t\t\treturn fmt.Errorf(\"Account %q not in map\", accName)\n\t\t\t\t\t}\n\t\t\t\t\tif mode != expectedMode {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected account %q mode to be %v, got: %v\", accName, expectedMode, mode)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckMode(\"one\", InterestOnly)\n\tcheckMode(\"two\", Optimistic)\n\n\t// Now change account \"two\" to enable JS\n\tchangeCurrentConfigContentWithNewContent(t, conf1, []byte(fmt.Sprintf(template,\n\t\t\"S1\", storeDir1, \"jetstream: enabled\", 23222, \"A\", 23222, \"A\", 11222, \"B\", 11223)))\n\tchangeCurrentConfigContentWithNewContent(t, conf2, []byte(fmt.Sprintf(template,\n\t\t\"S2\", storeDir2, \"jetstream: enabled\", 23223, \"B\", 23223, \"B\", 11223, \"A\", 11222)))\n\n\tif err := s1.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on s1 reload: %v\", err)\n\t}\n\tif err := s2.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on s2 reload: %v\", err)\n\t}\n\n\tcheckMode(\"one\", InterestOnly)\n\tcheckMode(\"two\", InterestOnly)\n}\n\nfunc TestJetStreamSuperClusterConnectionCount(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 2)\n\tdefer sc.shutdown()\n\n\tsysNc := natsConnect(t, sc.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer sysNc.Close()\n\t_, err := sysNc.Request(fmt.Sprintf(accDirectReqSubj, \"ONE\", \"CONNS\"), nil, 100*time.Millisecond)\n\t// this is a timeout as the server only responds when it has connections....\n\t// not convinced this should be that way, but also not the issue to investigate.\n\trequire_True(t, err == nats.ErrTimeout)\n\n\tfor i := 1; i <= 2; i++ {\n\t\tfunc() {\n\t\t\tnc := natsConnect(t, sc.clusterForName(fmt.Sprintf(\"C%d\", i)).randomServer().ClientURL())\n\t\t\tdefer nc.Close()\n\t\t\tjs, err := nc.JetStream()\n\t\t\trequire_NoError(t, err)\n\t\t\tname := fmt.Sprintf(\"foo%d\", 1)\n\t\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: name,\n\t\t\t\tSubjects: []string{name},\n\t\t\t\tReplicas: 3})\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\t}\n\tfunc() {\n\t\tnc := natsConnect(t, sc.clusterForName(\"C1\").randomServer().ClientURL())\n\t\tdefer nc.Close()\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"src\",\n\t\t\tSources: []*nats.StreamSource{{Name: \"foo1\"}, {Name: \"foo2\"}},\n\t\t\tReplicas: 3})\n\t\trequire_NoError(t, err)\n\t}()\n\tfunc() {\n\t\tnc := natsConnect(t, sc.clusterForName(\"C2\").randomServer().ClientURL())\n\t\tdefer nc.Close()\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"mir\",\n\t\t\tMirror: &nats.StreamSource{Name: \"foo2\"},\n\t\t\tReplicas: 3})\n\t\trequire_NoError(t, err)\n\t}()\n\n\t// There should be no active NATS CLIENT connections, but we still need\n\t// to wait a little bit...\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t_, err := sysNc.Request(fmt.Sprintf(accDirectReqSubj, \"ONE\", \"CONNS\"), nil, 100*time.Millisecond)\n\t\tif err != nats.ErrTimeout {\n\t\t\treturn fmt.Errorf(\"Expected timeout, got %v\", err)\n\t\t}\n\t\treturn nil\n\t})\n\tsysNc.Close()\n\n\ts := sc.randomServer()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tacc, err := s.lookupAccount(\"ONE\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not look up account: %v\", err)\n\t\t}\n\t\tif n := acc.NumConnections(); n != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no connections, got %d\", n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterConsumersBrokenGateways(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 1, 2)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.clusterForName(\"C1\").randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// This will be in C1.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create a stream in C2 that sources TEST\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tPlacement: &nats.Placement{Cluster: \"C2\"},\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\"}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Wait for direct consumer to get registered and detect interest across GW.\n\ttime.Sleep(time.Second)\n\n\t// Send 100 msgs over 100ms in separate Go routine.\n\tmsg, toSend, done := []byte(\"Hello\"), 100, make(chan bool)\n\tgo func() {\n\t\t// Send in 10 messages.\n\t\tfor i := 0; i < toSend; i++ {\n\t\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\t\tt.Errorf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t\ttime.Sleep(500 * time.Microsecond)\n\t\t}\n\t\tdone <- true\n\t}()\n\n\tbreakGW := func() {\n\t\ts.gateway.Lock()\n\t\tgw := s.gateway.out[\"C2\"]\n\t\ts.gateway.Unlock()\n\t\tif gw != nil {\n\t\t\tgw.closeConnection(ClientClosed)\n\t\t}\n\t}\n\n\t// Wait til about half way through.\n\ttime.Sleep(20 * time.Millisecond)\n\t// Now break GW connection.\n\tbreakGW()\n\n\t// Wait for GW to reform.\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\twaitForOutboundGateways(t, s, 1, 2*time.Second)\n\t\t}\n\t}\n\n\tselect {\n\tcase <-done:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not complete sending first batch of messages\")\n\t}\n\n\t// Make sure we can deal with data loss at the end.\n\tcheckFor(t, 20*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 100 {\n\t\t\treturn fmt.Errorf(\"Expected to have %d messages, got %d\", 100, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now send 100 more. Will aos break here in the middle.\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"TEST\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t\tif i == 50 {\n\t\t\tbreakGW()\n\t\t}\n\t}\n\n\t// Wait for GW to reform.\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\twaitForOutboundGateways(t, s, 1, 2*time.Second)\n\t\t}\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 200 {\n\t\tt.Fatalf(\"Expected to have %d messages, got %d\", 200, si.State.Msgs)\n\t}\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 200 {\n\t\t\treturn fmt.Errorf(\"Expected to have %d messages, got %d\", 200, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterLeafNodesWithSharedSystemAccountAndSameDomain(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tlnc := sc.createLeafNodes(\"LNC\", 2)\n\tdefer lnc.shutdown()\n\n\t// We want to make sure there is only one leader and its always in the supercluster.\n\tsc.waitOnLeader()\n\n\tif ml := lnc.leader(); ml != nil {\n\t\tt.Fatalf(\"Detected a meta-leader in the leafnode cluster: %s\", ml)\n\t}\n\n\t// leafnodes should have been added into the overall peer count.\n\tsc.waitOnPeerCount(8)\n\n\t// Check here that we auto detect sharing system account as well and auto place the correct\n\t// deny imports and exports.\n\tls := lnc.randomServer()\n\tif ls == nil {\n\t\tt.Fatalf(\"Expected a leafnode server, got none\")\n\t}\n\tgacc := ls.globalAccount().GetName()\n\n\tls.mu.Lock()\n\tvar hasDE, hasDI bool\n\tfor _, ln := range ls.leafs {\n\t\tln.mu.Lock()\n\t\tif ln.leaf.remote.RemoteLeafOpts.LocalAccount == gacc {\n\t\t\tre := ln.perms.pub.deny.Match(jsAllAPI)\n\t\t\thasDE = len(re.psubs)+len(re.qsubs) > 0\n\t\t\trs := ln.perms.sub.deny.Match(jsAllAPI)\n\t\t\thasDI = len(rs.psubs)+len(rs.qsubs) > 0\n\t\t}\n\t\tln.mu.Unlock()\n\t}\n\tls.mu.Unlock()\n\n\tif !hasDE {\n\t\tt.Fatalf(\"No deny export on global account\")\n\t}\n\tif !hasDI {\n\t\tt.Fatalf(\"No deny import on global account\")\n\t}\n\n\t// Make a stream by connecting to the leafnode cluster. Make sure placement is correct.\n\t// Client based API\n\tnc, js := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != \"LNC\" {\n\t\tt.Fatalf(\"Expected default placement to be %q, got %q\", \"LNC\", si.Cluster.Name)\n\t}\n\n\t// Now make sure placement also works if we want to place in a cluster in the supercluster.\n\tpcn := \"C2\"\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 2,\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != pcn {\n\t\tt.Fatalf(\"Expected default placement to be %q, got %q\", pcn, si.Cluster.Name)\n\t}\n}\n\nfunc TestJetStreamSuperClusterLeafNodesWithSharedSystemAccountAndDifferentDomain(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tlnc := sc.createLeafNodesWithDomain(\"LNC\", 2, \"LEAFDOMAIN\")\n\tdefer lnc.shutdown()\n\n\t// We want to make sure there is only one leader and its always in the supercluster.\n\tsc.waitOnLeader()\n\tlnc.waitOnLeader()\n\n\t// even though system account is shared, because domains differ,\n\tsc.waitOnPeerCount(6)\n\tlnc.waitOnPeerCount(2)\n\n\t// Check here that we auto detect sharing system account as well and auto place the correct\n\t// deny imports and exports.\n\tls := lnc.randomServer()\n\tif ls == nil {\n\t\tt.Fatalf(\"Expected a leafnode server, got none\")\n\t}\n\tgacc := ls.globalAccount().GetName()\n\n\tls.mu.Lock()\n\tvar hasDE, hasDI bool\n\tfor _, ln := range ls.leafs {\n\t\tln.mu.Lock()\n\t\tif ln.leaf.remote.RemoteLeafOpts.LocalAccount == gacc {\n\t\t\tre := ln.perms.pub.deny.Match(jsAllAPI)\n\t\t\thasDE = len(re.psubs)+len(re.qsubs) > 0\n\t\t\trs := ln.perms.sub.deny.Match(jsAllAPI)\n\t\t\thasDI = len(rs.psubs)+len(rs.qsubs) > 0\n\t\t}\n\t\tln.mu.Unlock()\n\t}\n\tls.mu.Unlock()\n\n\tif !hasDE {\n\t\tt.Fatalf(\"No deny export on global account\")\n\t}\n\tif !hasDI {\n\t\tt.Fatalf(\"No deny import on global account\")\n\t}\n\n\t// Make a stream by connecting to the leafnode cluster. Make sure placement is correct.\n\t// Client based API\n\tnc, js := jsClientConnect(t, lnc.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 2,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != \"LNC\" {\n\t\tt.Fatalf(\"Expected default placement to be %q, got %q\", \"LNC\", si.Cluster.Name)\n\t}\n\n\t// Now make sure placement does not works for cluster in different domain\n\tpcn := \"C2\"\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 2,\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t})\n\tif err == nil || !strings.Contains(err.Error(), \"no suitable peers for placement\") {\n\t\tt.Fatalf(\"Expected no suitable peers for placement, got: %v\", err)\n\t}\n}\n\nfunc TestJetStreamSuperClusterSingleLeafNodeWithSharedSystemAccount(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tln := sc.createSingleLeafNode(true)\n\tdefer ln.Shutdown()\n\n\t// We want to make sure there is only one leader and its always in the supercluster.\n\tsc.waitOnLeader()\n\n\t// leafnodes should have been added into the overall peer count.\n\tsc.waitOnPeerCount(7)\n\n\t// Now make sure we can place a stream in the leaf node.\n\t// First connect to the leafnode server itself.\n\tnc, js := jsClientConnect(t, ln)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST1\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != \"LNS\" {\n\t\tt.Fatalf(\"Expected to be placed in leafnode with %q as cluster name, got %q\", \"LNS\", si.Cluster.Name)\n\t}\n\t// Now check we can place on here as well but connect to the hub.\n\tnc, js = jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"bar\"},\n\t\tPlacement: &nats.Placement{Cluster: \"LNS\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.Cluster.Name != \"LNS\" {\n\t\tt.Fatalf(\"Expected to be placed in leafnode with %q as cluster name, got %q\", \"LNS\", si.Cluster.Name)\n\t}\n}\n\n// Issue reported with superclusters and leafnodes where first few get next requests for pull subscribers\n// have the wrong subject.\nfunc TestJetStreamSuperClusterGetNextRewrite(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 2, 2)\n\tdefer sc.shutdown()\n\n\t// Will connect the leafnode to cluster C1. We will then connect the \"client\" to cluster C2 to cross gateways.\n\tln := sc.clusterForName(\"C1\").createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(\"C\", \"nojs\")\n\tdefer ln.Shutdown()\n\n\tc2 := sc.clusterForName(\"C2\")\n\tnc, js := jsClientConnectEx(t, c2.randomServer(), []nats.JSOpt{nats.Domain(\"C\")}, nats.UserInfo(\"nojs\", \"p\"))\n\tdefer nc.Close()\n\n\t// Create a stream and add messages.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"ok\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Pull messages and make sure subject rewrite works.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfor _, m := range fetchMsgs(t, sub, 5, time.Second) {\n\t\tif m.Subject != \"foo\" {\n\t\t\tt.Fatalf(\"Expected %q as subject but got %q\", \"foo\", m.Subject)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamSuperClusterEphemeralCleanup(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Create a stream in cluster 0\n\ts := sc.clusters[0].randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsourceInCluster int\n\t\tstreamName string\n\t\tsourceName string\n\t}{\n\t\t{\"local\", 0, \"TEST1\", \"S1\"},\n\t\t{\"remote\", 1, \"TEST2\", \"S2\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: test.streamName, Replicas: 3}); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding %q stream: %v\", test.streamName, err)\n\t\t\t}\n\t\t\tif _, err := js.Publish(test.streamName, []byte(\"hello\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\n\t\t\t// Now create a source for that stream, either in same or remote cluster.\n\t\t\ts2 := sc.clusters[test.sourceInCluster].randomServer()\n\t\t\tnc2, js2 := jsClientConnect(t, s2)\n\t\t\tdefer nc2.Close()\n\n\t\t\tif _, err := js2.AddStream(&nats.StreamConfig{\n\t\t\t\tName: test.sourceName,\n\t\t\t\tStorage: nats.FileStorage,\n\t\t\t\tSources: []*nats.StreamSource{{Name: test.streamName}},\n\t\t\t\tReplicas: 1,\n\t\t\t}); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding source stream: %v\", err)\n\t\t\t}\n\n\t\t\t// Check that TEST(n) has 1 consumer and that S(n) is created and has 1 message.\n\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tsi, err := js2.StreamInfo(test.sourceName)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Get the consumer because we will want to artificially reduce\n\t\t\t// the delete threshold.\n\t\t\tleader := sc.clusters[0].streamLeader(\"$G\", test.streamName)\n\t\t\tmset, err := leader.GlobalAccount().lookupStream(test.streamName)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected to find a stream for %q, got %v\", test.streamName, err)\n\t\t\t}\n\t\t\tcons := mset.getConsumers()[0]\n\t\t\tcons.mu.Lock()\n\t\t\tcons.dthresh = 1250 * time.Millisecond\n\t\t\tactive := cons.active\n\t\t\tdtimerSet := cons.dtmr != nil\n\t\t\tdeliver := cons.cfg.DeliverSubject\n\t\t\tcons.mu.Unlock()\n\n\t\t\tif !active || dtimerSet {\n\t\t\t\tt.Fatalf(\"Invalid values for active=%v dtimerSet=%v\", active, dtimerSet)\n\t\t\t}\n\t\t\t// To add to the mix, let's create a local interest on the delivery subject\n\t\t\t// and stop it. This is to ensure that this does not stop timers that should\n\t\t\t// still be running and monitor the GW interest.\n\t\t\tsub := natsSubSync(t, nc, deliver)\n\t\t\tnatsFlush(t, nc)\n\t\t\tnatsUnsub(t, sub)\n\t\t\tnatsFlush(t, nc)\n\n\t\t\t// Now remove the \"S(n)\" stream...\n\t\t\tif err := js2.DeleteStream(test.sourceName); err != nil {\n\t\t\t\tt.Fatalf(\"Error deleting stream: %v\", err)\n\t\t\t}\n\n\t\t\t// Now check that the stream S(n) is really removed and that\n\t\t\t// the consumer is gone for stream TEST(n).\n\t\t\tcheckFor(t, 5*time.Second, 25*time.Millisecond, func() error {\n\t\t\t\t// First, make sure that stream S(n) has disappeared.\n\t\t\t\tif _, err := js2.StreamInfo(test.sourceName); err == nil {\n\t\t\t\t\treturn fmt.Errorf(\"Stream %q should no longer exist\", test.sourceName)\n\t\t\t\t}\n\t\t\t\tif ndc := mset.numDirectConsumers(); ndc != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %q stream to have 0 consumers, got %v\", test.streamName, ndc)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterGetNextSubRace(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 2, 2)\n\tdefer sc.shutdown()\n\n\t// Will connect the leafnode to cluster C1. We will then connect the \"client\" to cluster C2 to cross gateways.\n\tln := sc.clusterForName(\"C1\").createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(\"C\", \"nojs\")\n\tdefer ln.Shutdown()\n\n\t// Shutdown 1 of the server from C1, (the one LN is not connected to)\n\tfor _, s := range sc.clusterForName(\"C1\").servers {\n\t\ts.mu.Lock()\n\t\tif len(s.leafs) == 0 {\n\t\t\ts.mu.Unlock()\n\t\t\ts.Shutdown()\n\t\t\tbreak\n\t\t}\n\t\ts.mu.Unlock()\n\t}\n\n\t// Wait on meta leader in case shutdown of server above caused an election.\n\tsc.waitOnLeader()\n\n\tvar c2Srv *Server\n\t// Take the server from C2 that has no inbound from C1.\n\tc2 := sc.clusterForName(\"C2\")\n\tfor _, s := range c2.servers {\n\t\tvar gwsa [2]*client\n\t\tgws := gwsa[:0]\n\t\ts.getInboundGatewayConnections(&gws)\n\t\tif len(gws) == 0 {\n\t\t\tc2Srv = s\n\t\t\tbreak\n\t\t}\n\t}\n\tif c2Srv == nil {\n\t\tt.Fatalf(\"Both servers in C2 had an inbound GW connection!\")\n\t}\n\n\tnc, js := jsClientConnectEx(t, c2Srv, []nats.JSOpt{nats.Domain(\"C\")}, nats.UserInfo(\"nojs\", \"p\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"ok\")\n\t}\n\n\t// Wait for all messages to appear in the consumer\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\tci, err := js.ConsumerInfo(\"foo\", \"dur\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif n := ci.NumPending; n != 100 {\n\t\t\treturn fmt.Errorf(\"Expected 100 msgs, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\treq := &JSApiConsumerGetNextRequest{Batch: 1, Expires: 5 * time.Second}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\t// Create this by hand here to make sure we create the subscription\n\t// on the reply subject for every single request\n\tnextSubj := fmt.Sprintf(JSApiRequestNextT, \"foo\", \"dur\")\n\tnextSubj = \"$JS.C.API\" + strings.TrimPrefix(nextSubj, \"$JS.API\")\n\tfor i := 0; i < 100; i++ {\n\t\tinbox := nats.NewInbox()\n\t\tsub := natsSubSync(t, nc, inbox)\n\t\tnatsPubReq(t, nc, nextSubj, inbox, jreq)\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tif len(msg.Header) != 0 && string(msg.Data) != \"ok\" {\n\t\t\tt.Fatalf(\"Unexpected message: header=%+v data=%s\", msg.Header, msg.Data)\n\t\t}\n\t\tsub.Unsubscribe()\n\t}\n}\n\nfunc TestJetStreamSuperClusterPullConsumerAndHeaders(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tc1 := sc.clusterForName(\"C1\")\n\tc2 := sc.clusterForName(\"C2\")\n\n\tnc, js := jsClientConnect(t, c1.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"ORIGIN\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ttoSend := 50\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"ORIGIN\", []byte(\"ok\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tnc2, js2 := jsClientConnect(t, c2.randomServer())\n\tdefer nc2.Close()\n\n\t_, err := js2.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{Name: \"ORIGIN\"}},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Wait for them to be in the sourced stream.\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tif si, _ := js2.StreamInfo(\"S\"); si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs for %q, got %d\", toSend, \"S\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create a pull consumer for the sourced stream.\n\t_, err = js2.AddConsumer(\"S\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now we will connect and request the next message from each server in C1 cluster and check that headers remain in place.\n\tfor _, s := range c1.servers {\n\t\tnc, err := nats.Connect(s.ClientURL())\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tm, err := nc.Request(\"$JS.API.CONSUMER.MSG.NEXT.S.dlc\", nil, 2*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif len(m.Header) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 header element, got %+v\", m.Header)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamSuperClusterStatszActiveServers(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 2, 2)\n\tdefer sc.shutdown()\n\n\tcheckActive := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t\ts := sc.randomServer()\n\t\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Failed to create system client: %v\", err)\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tresp, err := nc.Request(serverStatsPingReqSubj, nil, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar ssm ServerStatsMsg\n\t\t\tif err := json.Unmarshal(resp.Data, &ssm); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif ssm.Stats.ActiveServers != expected {\n\t\t\t\treturn fmt.Errorf(\"Wanted %d, got %d\", expected, ssm.Stats.ActiveServers)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckActive(4)\n\tc := sc.randomCluster()\n\tss := c.randomServer()\n\tss.Shutdown()\n\tcheckActive(3)\n\tc.restartServer(ss)\n\tcheckActive(4)\n}\n\nfunc TestJetStreamSuperClusterSourceAndMirrorConsumersLeaderChange(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tc1 := sc.clusterForName(\"C1\")\n\tc2 := sc.clusterForName(\"C2\")\n\n\tnc, js := jsClientConnect(t, c1.randomServer())\n\tdefer nc.Close()\n\n\tvar sources []*nats.StreamSource\n\tnumStreams := 10\n\n\tfor i := 1; i <= numStreams; i++ {\n\t\tname := fmt.Sprintf(\"O%d\", i)\n\t\tsources = append(sources, &nats.StreamSource{Name: name})\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Place our new stream that will source all the others in different cluster.\n\tnc, js = jsClientConnect(t, c2.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tReplicas: 2,\n\t\tSources: sources,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Force leader change twice.\n\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"S\"), nil, time.Second)\n\tc2.waitOnStreamLeader(\"$G\", \"S\")\n\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"S\"), nil, time.Second)\n\tc2.waitOnStreamLeader(\"$G\", \"S\")\n\n\t// Now make sure we only have a single direct consumer on our origin streams.\n\t// Pick one at random.\n\tname := fmt.Sprintf(\"O%d\", rand.Intn(numStreams-1)+1)\n\tc1.waitOnStreamLeader(\"$G\", name)\n\ts := c1.streamLeader(\"$G\", name)\n\ta, err := s.lookupAccount(\"$G\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmset, err := a.lookupStream(name)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tif ndc := mset.numDirectConsumers(); ndc != 1 {\n\t\t\treturn fmt.Errorf(\"Stream %q wanted 1 direct consumer, got %d\", name, ndc)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create a mirror of selected from above. Will test same scenario.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: name},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Force leader change twice.\n\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"M\"), nil, time.Second)\n\tc2.waitOnStreamLeader(\"$G\", \"M\")\n\tnc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"M\"), nil, time.Second)\n\tc2.waitOnStreamLeader(\"$G\", \"M\")\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tif ndc := mset.numDirectConsumers(); ndc != 2 {\n\t\t\treturn fmt.Errorf(\"Stream %q wanted 2 direct consumers, got %d\", name, ndc)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterPushConsumerInterest(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tqueue string\n\t}{\n\t\t{\"non queue\", _EMPTY_},\n\t\t{\"queue\", \"queue\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttestInterest := func(s *Server) {\n\t\t\t\tt.Helper()\n\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tReplicas: 3,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tvar sub *nats.Subscription\n\t\t\t\tif test.queue != _EMPTY_ {\n\t\t\t\t\tsub, err = js.QueueSubscribeSync(\"foo\", test.queue)\n\t\t\t\t} else {\n\t\t\t\t\tsub, err = js.SubscribeSync(\"foo\", nats.Durable(\"dur\"))\n\t\t\t\t}\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tjs.Publish(\"foo\", []byte(\"msg1\"))\n\t\t\t\t// Since the GW watcher is checking every 1sec, make sure we are\n\t\t\t\t// giving it enough time for the delivery to start.\n\t\t\t\t_, err = sub.NextMsg(2 * time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Create the durable push consumer from cluster \"0\"\n\t\t\ttestInterest(sc.clusters[0].servers[0])\n\n\t\t\t// Now \"move\" to a server in cluster \"1\"\n\t\t\ttestInterest(sc.clusters[1].servers[0])\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterOverflowPlacement(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterMaxBytesTempl, 3, 3)\n\tdefer sc.shutdown()\n\n\tpcn := \"C2\"\n\ts := sc.clusterForName(pcn).randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// With this setup, we opted in for requiring MaxBytes, so this should error.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tReplicas: 3,\n\t})\n\trequire_Error(t, err, NewJSStreamMaxBytesRequiredError())\n\n\t// R=2 on purpose to leave one server empty.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tReplicas: 2,\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now try to add another that will overflow the current cluster's reservation.\n\t// Since we asked explicitly for the same cluster this should fail.\n\t// Note this will not be testing the peer picker since the update has probably not made it to the meta leader.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"bar\",\n\t\tReplicas: 3,\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t})\n\trequire_Contains(t, err.Error(), \"nats: no suitable peers for placement\")\n\t// Now test actual overflow placement. So try again with no placement designation.\n\t// This will test the peer picker's logic since they are updated at this point and the meta leader\n\t// knows it can not place it in C2.\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"bar\",\n\t\tReplicas: 3,\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure we did not get place into C2.\n\tfalt := si.Cluster.Name\n\tif falt == pcn {\n\t\tt.Fatalf(\"Expected to be placed in another cluster besides %q, but got %q\", pcn, falt)\n\t}\n\n\t// One more time that should spill over again to our last cluster.\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"baz\",\n\t\tReplicas: 3,\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure we did not get place into C2.\n\tif salt := si.Cluster.Name; salt == pcn || salt == falt {\n\t\tt.Fatalf(\"Expected to be placed in last cluster besides %q or %q, but got %q\", pcn, falt, salt)\n\t}\n\n\t// Now place a stream of R1 into C2 which should have space.\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"dlc\",\n\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t})\n\trequire_NoError(t, err)\n\n\tif si.Cluster.Name != pcn {\n\t\tt.Fatalf(\"Expected to be placed in our origin cluster %q, but got %q\", pcn, si.Cluster.Name)\n\t}\n}\n\nfunc TestJetStreamSuperClusterConcurrentOverflow(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterMaxBytesTempl, 3, 3)\n\tdefer sc.shutdown()\n\n\tpcn := \"C2\"\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\tvar swg sync.WaitGroup\n\n\tstart := func(name string) {\n\t\tdefer wg.Done()\n\n\t\ts := sc.clusterForName(pcn).randomServer()\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tswg.Done()\n\t\t<-startCh\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tReplicas: 3,\n\t\t\tMaxBytes: 2 * 1024 * 1024 * 1024,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\twg.Add(2)\n\tswg.Add(2)\n\tgo start(\"foo\")\n\tgo start(\"bar\")\n\tswg.Wait()\n\t// Now start both at same time.\n\tclose(startCh)\n\twg.Wait()\n}\n\nfunc TestJetStreamSuperClusterStreamTagPlacement(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tplaceOK := func(connectCluster string, tags []string, expectedCluster string) {\n\t\tt.Helper()\n\t\tnc, js := jsClientConnect(t, sc.clusterForName(connectCluster).randomServer())\n\t\tdefer nc.Close()\n\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tPlacement: &nats.Placement{Tags: tags},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tif si.Cluster.Name != expectedCluster {\n\t\t\tt.Fatalf(\"Failed to place properly in %q, got %q\", expectedCluster, si.Cluster.Name)\n\t\t}\n\t\tjs.DeleteStream(\"TEST\")\n\t}\n\n\tplaceOK(\"C2\", []string{\"cloud:aws\"}, \"C1\")\n\tplaceOK(\"C2\", []string{\"country:jp\"}, \"C3\")\n\tplaceOK(\"C1\", []string{\"cloud:gcp\", \"country:uk\"}, \"C2\")\n\n\t// Case shoud not matter.\n\tplaceOK(\"C1\", []string{\"cloud:GCP\", \"country:UK\"}, \"C2\")\n\tplaceOK(\"C2\", []string{\"Cloud:AwS\", \"Country:uS\"}, \"C1\")\n\n\tplaceErr := func(connectCluster string, tags []string) {\n\t\tt.Helper()\n\t\tnc, js := jsClientConnect(t, sc.clusterForName(connectCluster).randomServer())\n\t\tdefer nc.Close()\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tPlacement: &nats.Placement{Tags: tags},\n\t\t})\n\t\trequire_Contains(t, err.Error(), \"no suitable peers for placement\", \"tags not matched\")\n\t\trequire_Contains(t, err.Error(), tags...)\n\t}\n\n\tplaceErr(\"C1\", []string{\"cloud:GCP\", \"country:US\"})\n\tplaceErr(\"C1\", []string{\"country:DN\"})\n\tplaceErr(\"C1\", []string{\"cloud:DO\"})\n}\n\nfunc TestJetStreamSuperClusterRemovedPeersAndStreamsListAndDelete(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\tpcn := \"C2\"\n\tsc.waitOnLeader()\n\tml := sc.leader()\n\tif ml.ClusterName() == pcn {\n\t\tpcn = \"C1\"\n\t}\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"GONE\",\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"GONE\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: ml.ClusterName()},\n\t})\n\trequire_NoError(t, err)\n\n\t// Put messages in..\n\tnum := 100\n\tfor i := 0; i < num; i++ {\n\t\tjs.PublishAsync(\"GONE\", []byte(\"SLS\"))\n\t\tjs.PublishAsync(\"TEST\", []byte(\"SLS\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tc := sc.clusterForName(pcn)\n\tc.shutdown()\n\n\t// Grab Stream List..\n\tstart := time.Now()\n\tresp, err := nc.Request(JSApiStreamList, nil, 2*time.Second)\n\trequire_NoError(t, err)\n\tif delta := time.Since(start); delta > 100*time.Millisecond {\n\t\tt.Fatalf(\"Stream list call took too long to return: %v\", delta)\n\t}\n\tvar list JSApiStreamListResponse\n\terr = json.Unmarshal(resp.Data, &list)\n\trequire_NoError(t, err)\n\n\tif len(list.Missing) != 1 || list.Missing[0] != \"GONE\" {\n\t\tt.Fatalf(\"Wrong Missing: %+v\", list)\n\t}\n\n\t// Check behavior of stream info as well. We want it to return the stream is offline and not just timeout.\n\t_, err = js.StreamInfo(\"GONE\")\n\t// FIXME(dlc) - Go client not putting nats: prefix on for stream but does for consumer.\n\trequire_Error(t, err, NewJSStreamOfflineError(), errors.New(\"nats: stream is offline\"))\n\n\t// Same for Consumer\n\tstart = time.Now()\n\tresp, err = nc.Request(\"$JS.API.CONSUMER.LIST.GONE\", nil, 2*time.Second)\n\trequire_NoError(t, err)\n\tif delta := time.Since(start); delta > 100*time.Millisecond {\n\t\tt.Fatalf(\"Consumer list call took too long to return: %v\", delta)\n\t}\n\tvar clist JSApiConsumerListResponse\n\terr = json.Unmarshal(resp.Data, &clist)\n\trequire_NoError(t, err)\n\n\tif len(clist.Missing) != 1 || clist.Missing[0] != \"dlc\" {\n\t\tt.Fatalf(\"Wrong Missing: %+v\", clist)\n\t}\n\n\t_, err = js.ConsumerInfo(\"GONE\", \"dlc\")\n\trequire_Error(t, err, NewJSConsumerOfflineError(), errors.New(\"nats: consumer is offline\"))\n\n\t// Make sure delete works.\n\terr = js.DeleteConsumer(\"GONE\", \"dlc\")\n\trequire_NoError(t, err)\n\n\terr = js.DeleteStream(\"GONE\")\n\trequire_NoError(t, err)\n\n\t// Test it is really gone.\n\t_, err = js.StreamInfo(\"GONE\")\n\trequire_Error(t, err, nats.ErrStreamNotFound)\n}\n\nfunc TestJetStreamSuperClusterConsumerDeliverNewBug(t *testing.T) {\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tsc := createJetStreamSuperCluster(t, 3, 3)\n\t\t\tdefer sc.shutdown()\n\n\t\t\tpcn := \"C2\"\n\t\t\tsc.waitOnLeader()\n\t\t\tml := sc.leader()\n\t\t\tif ml.ClusterName() == pcn {\n\t\t\t\tpcn = \"C1\"\n\t\t\t}\n\n\t\t\t// Client based API\n\t\t\tnc, js := jsClientConnect(t, ml)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"T\",\n\t\t\t\tReplicas: 3,\n\t\t\t\tPlacement: &nats.Placement{Cluster: pcn},\n\t\t\t\tStorage: storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Put messages in..\n\t\t\tnum := 100\n\t\t\tfor i := 0; i < num; i++ {\n\t\t\t\tjs.PublishAsync(\"T\", []byte(\"OK\"))\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\tci, err := js.AddConsumer(\"T\", &nats.ConsumerConfig{\n\t\t\t\tDurable: \"d\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\tDeliverPolicy: nats.DeliverNewPolicy,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif ci.Delivered.Consumer != 0 || ci.Delivered.Stream != 100 {\n\t\t\t\tt.Fatalf(\"Incorrect consumer delivered info: %+v\", ci.Delivered)\n\t\t\t}\n\n\t\t\tc := sc.clusterForName(pcn)\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tsd := s.JetStreamConfig().StoreDir\n\t\t\t\ts.Shutdown()\n\t\t\t\tremoveDir(t, sd)\n\t\t\t\ts = c.restartServer(s)\n\t\t\t\tc.waitOnServerHealthz(s)\n\t\t\t\tc.waitOnConsumerLeader(\"$G\", \"T\", \"d\")\n\t\t\t}\n\n\t\t\tc.waitOnConsumerLeader(\"$G\", \"T\", \"d\")\n\n\t\t\t// Each server, after being caught up, needs to fully agree on stream/consumer sequences.\n\t\t\t// For both the in-memory consumer state, as the stored state.\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(\"T\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\to := mset.lookupConsumer(\"d\")\n\t\t\t\trequire_NotNil(t, o)\n\t\t\t\to.mu.RLock()\n\t\t\t\tdefer o.mu.RUnlock()\n\n\t\t\t\tif o.dseq-1 != 0 || o.sseq-1 != 100 {\n\t\t\t\t\tt.Fatalf(\"Incorrect consumer delivered info: dseq=%d, sseq=%d\", o.dseq-1, o.sseq-1)\n\t\t\t\t}\n\t\t\t\tstate, err := o.store.BorrowState()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif state.Delivered.Consumer != 0 || state.Delivered.Stream != 100 {\n\t\t\t\t\tt.Fatalf(\"Incorrect consumer state: consumer_seq=%d, stream_seq=%d\", state.Delivered.Consumer, state.Delivered.Stream)\n\t\t\t\t}\n\t\t\t\tnp, err := o.checkNumPending()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif np != 0 {\n\t\t\t\t\tt.Fatalf(\"Did not expect NumPending, got %d\", np)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// This will test our ability to move streams and consumers between clusters.\nfunc TestJetStreamSuperClusterMovingStreamsAndConsumers(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\treplicas int\n\t}{\n\t\t{\"R1\", 1},\n\t\t{\"R3\", 3},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\treplicas := test.replicas\n\n\t\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"MOVE\",\n\t\t\t\tReplicas: replicas,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\"}},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer js.DeleteStream(\"MOVE\")\n\n\t\t\tif si.Cluster.Name != \"C1\" {\n\t\t\t\tt.Fatalf(\"Failed to place properly in %q, got %q\", \"C1\", si.Cluster.Name)\n\t\t\t}\n\n\t\t\tfor i := 0; i < 1000; i++ {\n\t\t\t\t_, err := js.PublishAsync(\"MOVE\", []byte(\"Moving on up\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Durable Push Consumer, so same R.\n\t\t\tdpushSub, err := js.SubscribeSync(\"MOVE\", nats.Durable(\"dlc\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dpushSub.Unsubscribe()\n\n\t\t\t// Ephemeral Push Consumer, R1.\n\t\t\tepushSub, err := js.SubscribeSync(\"MOVE\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer epushSub.Unsubscribe()\n\n\t\t\t// Durable Pull Consumer, so same R.\n\t\t\tdpullSub, err := js.PullSubscribe(\"MOVE\", \"dlc-pull\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dpullSub.Unsubscribe()\n\n\t\t\t// TODO(dlc) - Server supports ephemeral pulls but Go client does not yet.\n\n\t\t\tsi, err = js.StreamInfo(\"MOVE\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif si.State.Consumers != 3 {\n\t\t\t\tt.Fatalf(\"Expected 3 attached consumers, got %d\", si.State.Consumers)\n\t\t\t}\n\n\t\t\tinitialState := si.State\n\n\t\t\tcheckSubsPending(t, dpushSub, int(initialState.Msgs))\n\t\t\tcheckSubsPending(t, epushSub, int(initialState.Msgs))\n\n\t\t\t// Ack 100\n\t\t\ttoAck := 100\n\t\t\tfor i := 0; i < toAck; i++ {\n\t\t\t\tm, err := dpushSub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tm.AckSync()\n\t\t\t\t// Ephemeral\n\t\t\t\tm, err = epushSub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tm.AckSync()\n\t\t\t}\n\n\t\t\t// Do same with pull subscriber.\n\t\t\tfor _, m := range fetchMsgs(t, dpullSub, toAck, 5*time.Second) {\n\t\t\t\tm.AckSync()\n\t\t\t}\n\n\t\t\t// First make sure we disallow move and replica changes in same update.\n\t\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\tName: \"MOVE\",\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t\t\t\tReplicas: replicas + 1,\n\t\t\t})\n\t\t\trequire_Error(t, err, NewJSStreamMoveAndScaleError())\n\n\t\t\t// Now move to new cluster.\n\t\t\tsi, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\tName: \"MOVE\",\n\t\t\t\tReplicas: replicas,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\tif si.Cluster.Name != \"C1\" {\n\t\t\t\t\treturn fmt.Errorf(\"Expected cluster of %q but got %q\", \"C1\", si.Cluster.Name)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Make sure we can not move an inflight stream and consumers, should error.\n\t\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\tName: \"MOVE\",\n\t\t\t\tReplicas: replicas,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\"}},\n\t\t\t})\n\t\t\trequire_Contains(t, err.Error(), \"stream move already in progress\")\n\n\t\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(\"MOVE\", nats.MaxWait(500*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\t// We should see 2X peers.\n\t\t\t\tnumPeers := len(si.Cluster.Replicas)\n\t\t\t\tif si.Cluster.Leader != _EMPTY_ {\n\t\t\t\t\tnumPeers++\n\t\t\t\t}\n\t\t\t\tif numPeers != 2*replicas {\n\t\t\t\t\t// The move can happen very quick now, so we might already be done.\n\t\t\t\t\tif si.Cluster.Name == \"C2\" {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\treturn fmt.Errorf(\"Expected to see %d replicas, got %d\", 2*replicas, numPeers)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Expect a new leader to emerge and replicas to drop as a leader is elected.\n\t\t\t// We have to check fast or it might complete and we will not see intermediate steps.\n\t\t\tsc.waitOnStreamLeader(\"$G\", \"MOVE\")\n\t\t\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(\"MOVE\", nats.MaxWait(500*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif len(si.Cluster.Replicas) >= 2*replicas {\n\t\t\t\t\treturn fmt.Errorf(\"Expected <%d replicas, got %d\", 2*replicas, len(si.Cluster.Replicas))\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Should see the cluster designation and leader switch to C2.\n\t\t\t// We should also shrink back down to original replica count.\n\t\t\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(\"MOVE\", nats.MaxWait(500*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif si.Cluster.Name != \"C2\" {\n\t\t\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t\t\t}\n\t\t\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t\t\t} else if !strings.HasPrefix(si.Cluster.Leader, \"C2-\") {\n\t\t\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t\t\t}\n\t\t\t\t// Now we want to see that we shrink back to original.\n\t\t\t\tif len(si.Cluster.Replicas) != replicas-1 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", replicas-1, len(si.Cluster.Replicas))\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Check moved state is same as initial state.\n\t\t\tsi, err = js.StreamInfo(\"MOVE\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif !reflect.DeepEqual(si.State, initialState) {\n\t\t\t\tt.Fatalf(\"States do not match after migration:\\n%+v\\nvs\\n%+v\", si.State, initialState)\n\t\t\t}\n\n\t\t\t// Make sure we can still send messages.\n\t\t\taddN := toAck\n\t\t\tfor i := 0; i < addN; i++ {\n\t\t\t\t_, err := js.Publish(\"MOVE\", []byte(\"Done Moved\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsi, err = js.StreamInfo(\"MOVE\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\texpectedPushMsgs := initialState.Msgs + uint64(addN)\n\t\t\texpectedPullMsgs := uint64(addN)\n\n\t\t\tif si.State.Msgs != expectedPushMsgs {\n\t\t\t\tt.Fatalf(\"Expected to be able to send new messages\")\n\t\t\t}\n\n\t\t\t// Now check consumers, make sure the state is correct and that they transferred state and reflect the new messages.\n\t\t\t// We Ack'd 100 and sent another 100, so should be same.\n\t\t\tcheckConsumer := func(sub *nats.Subscription, isPull bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\tci, err := sub.ConsumerInfo()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tvar expectedMsgs uint64\n\t\t\t\t\tif isPull {\n\t\t\t\t\t\texpectedMsgs = expectedPullMsgs\n\t\t\t\t\t} else {\n\t\t\t\t\t\texpectedMsgs = expectedPushMsgs\n\t\t\t\t\t}\n\n\t\t\t\t\tif ci.Delivered.Consumer != expectedMsgs || ci.Delivered.Stream != expectedMsgs {\n\t\t\t\t\t\treturn fmt.Errorf(\"Delivered for %q is not correct: %+v\", ci.Name, ci.Delivered)\n\t\t\t\t\t}\n\t\t\t\t\tif ci.AckFloor.Consumer != uint64(toAck) || ci.AckFloor.Stream != uint64(toAck) {\n\t\t\t\t\t\treturn fmt.Errorf(\"AckFloor for %q is not correct: %+v\", ci.Name, ci.AckFloor)\n\t\t\t\t\t}\n\t\t\t\t\tif isPull && ci.NumAckPending != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"NumAckPending for %q is not correct: %v\", ci.Name, ci.NumAckPending)\n\t\t\t\t\t} else if !isPull && ci.NumAckPending != int(initialState.Msgs) {\n\t\t\t\t\t\treturn fmt.Errorf(\"NumAckPending for %q is not correct: %v\", ci.Name, ci.NumAckPending)\n\t\t\t\t\t}\n\t\t\t\t\t// Make sure the replicas etc are back to what is expected.\n\t\t\t\t\tsi, err := js.StreamInfo(\"MOVE\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tnumExpected := si.Config.Replicas\n\t\t\t\t\tif ci.Config.Durable == _EMPTY_ {\n\t\t\t\t\t\tnumExpected = 1\n\t\t\t\t\t}\n\t\t\t\t\tnumPeers := len(ci.Cluster.Replicas)\n\t\t\t\t\tif ci.Cluster.Leader != _EMPTY_ {\n\t\t\t\t\t\tnumPeers++\n\t\t\t\t\t}\n\t\t\t\t\tif numPeers != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d peers, got %d\", numExpected, numPeers)\n\t\t\t\t\t}\n\t\t\t\t\t// If we are push check sub pending.\n\t\t\t\t\tif !isPull {\n\t\t\t\t\t\tcheckSubsPending(t, sub, int(expectedPushMsgs)-toAck)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tcheckPushConsumer := func(sub *nats.Subscription) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckConsumer(sub, false)\n\t\t\t}\n\t\t\tcheckPullConsumer := func(sub *nats.Subscription) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckConsumer(sub, true)\n\t\t\t}\n\n\t\t\tcheckPushConsumer(dpushSub)\n\t\t\tcheckPushConsumer(epushSub)\n\t\t\tcheckPullConsumer(dpullSub)\n\n\t\t\t// Cleanup\n\t\t\terr = js.DeleteStream(\"MOVE\")\n\t\t\trequire_NoError(t, err)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterMovingStreamsWithMirror(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tReplicas: 1,\n\t\tMirror: &nats.StreamSource{Name: \"SOURCE\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t})\n\trequire_NoError(t, err)\n\n\tdone := make(chan struct{})\n\texited := make(chan struct{})\n\terrors := make(chan error, 1)\n\n\tnumNoResp := uint64(0)\n\n\t// We will run a separate routine and send at 100hz\n\tgo func() {\n\t\tnc, js := jsClientConnect(t, sc.randomServer())\n\t\tdefer nc.Close()\n\n\t\tdefer close(exited)\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase <-time.After(10 * time.Millisecond):\n\t\t\t\t_, err := js.Publish(\"foo\", []byte(\"100HZ\"))\n\t\t\t\tif err == nil {\n\t\t\t\t} else if err == nats.ErrNoStreamResponse {\n\t\t\t\t\tatomic.AddUint64(&numNoResp, 1)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\terrors <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Let it get going.\n\ttime.Sleep(1500 * time.Millisecond)\n\n\t// Now move the source to a new cluster.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 30*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"SOURCE\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster.Name != \"C2\" {\n\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t}\n\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t} else if !strings.HasPrefix(si.Cluster.Leader, \"C2-\") {\n\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t}\n\t\t// Now we want to see that we shrink back to original.\n\t\tif len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", 2, len(si.Cluster.Replicas))\n\t\t}\n\t\t// Let's get to 50+ msgs.\n\t\tif si.State.Msgs < 50 {\n\t\t\treturn fmt.Errorf(\"Only see %d msgs\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tclose(done)\n\t<-exited\n\n\tif nnr := atomic.LoadUint64(&numNoResp); nnr > 0 {\n\t\tif nnr > 5 {\n\t\t\tt.Fatalf(\"Expected no or very few failed message publishes, got %d\", nnr)\n\t\t} else {\n\t\t\tt.Logf(\"Got a few failed publishes: %d\", nnr)\n\t\t}\n\t}\n\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"SOURCE\")\n\t\trequire_NoError(t, err)\n\t\tmi, err := js.StreamInfo(\"MIRROR\")\n\t\trequire_NoError(t, err)\n\n\t\tif !reflect.DeepEqual(si.State, mi.State) {\n\t\t\treturn fmt.Errorf(\"Expected mirror to be the same, got %+v vs %+v\", mi.State, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestJetStreamSuperClusterMovingStreamAndMoveBack(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\treplicas int\n\t}{\n\t\t{\"R1\", 1},\n\t\t{\"R3\", 3},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tjs.DeleteStream(\"TEST\")\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tReplicas: test.replicas,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\"}},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\ttoSend := 10_000\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\t_, err := js.Publish(\"TEST\", []byte(\"HELLO WORLD\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tReplicas: test.replicas,\n\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckMove := func(cluster string) {\n\t\t\t\tt.Helper()\n\t\t\t\tsc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t\t\t\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif si.Cluster.Name != cluster {\n\t\t\t\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t\t\t\t}\n\t\t\t\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\t\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t\t\t\t} else if !strings.HasPrefix(si.Cluster.Leader, cluster) {\n\t\t\t\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t\t\t\t}\n\t\t\t\t\t// Now we want to see that we shrink back to original.\n\t\t\t\t\tif len(si.Cluster.Replicas) != test.replicas-1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", test.replicas-1, len(si.Cluster.Replicas))\n\t\t\t\t\t}\n\t\t\t\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\t\t\t\treturn fmt.Errorf(\"Only see %d msgs\", si.State.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tcheckMove(\"C2\")\n\n\t\t\t// The move could be completed when looking at the stream info, but the meta leader could still\n\t\t\t// deny move updates for a short time while state is cleaned up.\n\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tReplicas: test.replicas,\n\t\t\t\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\"}},\n\t\t\t\t})\n\t\t\t\treturn err\n\t\t\t})\n\n\t\t\tcheckMove(\"C1\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterImportConsumerStreamSubjectRemap(t *testing.T) {\n\ttemplate := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, domain: HUB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts: {\n\t\tJS: {\n\t\t\tjetstream: enabled\n\t\t\tusers: [ {user: js, password: pwd} ]\n\t\t\texports [\n\t\t\t\t# This is streaming to a delivery subject for a push based consumer.\n\t\t\t\t{ stream: \"deliver.ORDERS.*\" }\n\t\t\t\t# This is to ack received messages. This is a service to support sync ack.\n\t\t\t\t{ service: \"$JS.ACK.ORDERS.*.>\" }\n\t\t\t\t# To support ordered consumers, flow control.\n\t\t\t\t{ service: \"$JS.FC.>\" }\n\t\t\t]\n\t\t},\n\t\tIM: {\n\t\t\tusers: [ {user: im, password: pwd} ]\n\t\t\timports [\n\t\t\t\t{ stream: { account: JS, subject: \"deliver.ORDERS.route\" }}\n\t\t\t\t{ stream: { account: JS, subject: \"deliver.ORDERS.gateway\" }}\n\t\t\t\t{ stream: { account: JS, subject: \"deliver.ORDERS.leaf1\" }}\n\t\t\t\t{ stream: { account: JS, subject: \"deliver.ORDERS.leaf2\" }}\n\t\t\t\t{ service: {account: JS, subject: \"$JS.FC.>\" }}\n\t\t\t]\n\t\t},\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] },\n\t}\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}`\n\n\ttest := func(t *testing.T, queue bool) {\n\t\tc := createJetStreamSuperClusterWithTemplate(t, template, 3, 2)\n\t\tdefer c.shutdown()\n\n\t\ts := c.randomServer()\n\t\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"js\", \"pwd\"))\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"ORDERS\",\n\t\t\tSubjects: []string{\"foo\"}, // The JS subject.\n\t\t\tReplicas: 3,\n\t\t\tPlacement: &nats.Placement{Cluster: \"C1\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\n\t\tfor dur, deliver := range map[string]string{\n\t\t\t\"dur-route\": \"deliver.ORDERS.route\",\n\t\t\t\"dur-gateway\": \"deliver.ORDERS.gateway\",\n\t\t\t\"dur-leaf-1\": \"deliver.ORDERS.leaf1\",\n\t\t\t\"dur-leaf-2\": \"deliver.ORDERS.leaf2\",\n\t\t} {\n\t\t\tcfg := &nats.ConsumerConfig{\n\t\t\t\tDurable: dur,\n\t\t\t\tDeliverSubject: deliver,\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t}\n\t\t\tif queue {\n\t\t\t\tcfg.DeliverGroup = \"queue\"\n\t\t\t}\n\t\t\t_, err = js.AddConsumer(\"ORDERS\", cfg)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\ttestCase := func(t *testing.T, s *Server, dSubj string) {\n\t\t\tnc2, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"im\", \"pwd\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer nc2.Close()\n\n\t\t\tvar sub *nats.Subscription\n\t\t\tif queue {\n\t\t\t\tsub, err = nc2.QueueSubscribeSync(dSubj, \"queue\")\n\t\t\t} else {\n\t\t\t\tsub, err = nc2.SubscribeSync(dSubj)\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif m.Subject != \"foo\" {\n\t\t\t\tt.Fatalf(\"Subject not mapped correctly across account boundary, expected %q got %q\", \"foo\", m.Subject)\n\t\t\t}\n\t\t\trequire_False(t, strings.Contains(m.Reply, \"@\"))\n\t\t}\n\n\t\tt.Run(\"route\", func(t *testing.T) {\n\t\t\t// pick random non consumer leader so we receive via route\n\t\t\ts := c.clusterForName(\"C1\").randomNonConsumerLeader(\"JS\", \"ORDERS\", \"dur-route\")\n\t\t\ttestCase(t, s, \"deliver.ORDERS.route\")\n\t\t})\n\t\tt.Run(\"gateway\", func(t *testing.T) {\n\t\t\t// pick server with inbound gateway from consumer leader, so we receive from gateway and have no route in between\n\t\t\tscl := c.clusterForName(\"C1\").consumerLeader(\"JS\", \"ORDERS\", \"dur-gateway\")\n\t\t\tvar sfound *Server\n\t\t\tfor _, s := range c.clusterForName(\"C2\").servers {\n\t\t\t\ts.mu.Lock()\n\t\t\t\tfor _, c := range s.gateway.in {\n\t\t\t\t\tif c.GetName() == scl.info.ID {\n\t\t\t\t\t\tsfound = s\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.mu.Unlock()\n\t\t\t\tif sfound != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\ttestCase(t, sfound, \"deliver.ORDERS.gateway\")\n\t\t})\n\t\tt.Run(\"leaf-post-export\", func(t *testing.T) {\n\t\t\t// create leaf node server connected post export/import\n\t\t\tscl := c.clusterForName(\"C1\").consumerLeader(\"JS\", \"ORDERS\", \"dur-leaf-1\")\n\t\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tport: -1\n\t\t\tleafnodes {\n\t\t\t\tremotes [ { url: \"nats://im:pwd@127.0.0.1:%d\" } ]\n\t\t\t}\n\t\t\tauthorization: {\n\t\t\t\tuser: im,\n\t\t\t\tpassword: pwd\n\t\t\t}\n\t\t`, scl.getOpts().LeafNode.Port)))\n\t\t\ts, _ := RunServerWithConfig(cf)\n\t\t\tdefer s.Shutdown()\n\t\t\tcheckLeafNodeConnected(t, scl)\n\t\t\ttestCase(t, s, \"deliver.ORDERS.leaf1\")\n\t\t})\n\t\tt.Run(\"leaf-pre-export\", func(t *testing.T) {\n\t\t\t// create leaf node server connected pre export, perform export/import on leaf node server\n\t\t\tscl := c.clusterForName(\"C1\").consumerLeader(\"JS\", \"ORDERS\", \"dur-leaf-2\")\n\t\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tport: -1\n\t\t\tleafnodes {\n\t\t\t\tremotes [ { url: \"nats://js:pwd@127.0.0.1:%d\", account: JS2 } ]\n\t\t\t}\n\t\t\taccounts: {\n\t\t\t\tJS2: {\n\t\t\t\t\tusers: [ {user: js, password: pwd} ]\n\t\t\t\t\texports [\n\t\t\t\t\t\t# This is streaming to a delivery subject for a push based consumer.\n\t\t\t\t\t\t{ stream: \"deliver.ORDERS.leaf2\" }\n\t\t\t\t\t\t# This is to ack received messages. This is a service to support sync ack.\n\t\t\t\t\t\t{ service: \"$JS.ACK.ORDERS.*.>\" }\n\t\t\t\t\t\t# To support ordered consumers, flow control.\n\t\t\t\t\t\t{ service: \"$JS.FC.>\" }\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\tIM2: {\n\t\t\t\t\tusers: [ {user: im, password: pwd} ]\n\t\t\t\t\timports [\n\t\t\t\t\t\t{ stream: { account: JS2, subject: \"deliver.ORDERS.leaf2\" }}\n\t\t\t\t\t\t{ service: {account: JS2, subject: \"$JS.FC.>\" }}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t}\n\t\t`, scl.getOpts().LeafNode.Port)))\n\t\t\ts, _ := RunServerWithConfig(cf)\n\t\t\tdefer s.Shutdown()\n\t\t\tcheckLeafNodeConnected(t, scl)\n\t\t\ttestCase(t, s, \"deliver.ORDERS.leaf2\")\n\t\t})\n\t}\n\n\tt.Run(\"noQueue\", func(t *testing.T) {\n\t\ttest(t, false)\n\t})\n\tt.Run(\"queue\", func(t *testing.T) {\n\t\ttest(t, true)\n\t})\n}\n\nfunc TestJetStreamSuperClusterMaxHaAssets(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s', limits: {max_ha_assets: 2}}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`, 3, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\treturn conf\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// speed up statsz reporting\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\ts.mu.Lock()\n\t\t\ts.sys.statsz = 10 * time.Millisecond\n\t\t\ts.sys.cstatsz = s.sys.statsz\n\t\t\ts.sys.stmr.Reset(s.sys.statsz)\n\t\t\ts.mu.Unlock()\n\t\t}\n\t}\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tncSys := natsConnect(t, sc.randomServer().ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer ncSys.Close()\n\tstatszSub, err := ncSys.SubscribeSync(fmt.Sprintf(serverStatsSubj, \"*\"))\n\trequire_NoError(t, err)\n\trequire_NoError(t, ncSys.Flush())\n\n\twaitStatsz := func(peers, haassets int) {\n\t\tt.Helper()\n\t\tfor peersWithExactHaAssets := 0; peersWithExactHaAssets < peers; {\n\t\t\tm, err := statszSub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar statsz ServerStatsMsg\n\t\t\terr = json.Unmarshal(m.Data, &statsz)\n\t\t\trequire_NoError(t, err)\n\t\t\tif statsz.Stats.JetStream == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif haassets == statsz.Stats.JetStream.Stats.HAAssets {\n\t\t\t\tpeersWithExactHaAssets++\n\t\t\t}\n\t\t}\n\t}\n\twaitStatsz(6, 1) // counts _meta_\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S0\", Replicas: 1, Placement: &nats.Placement{Cluster: \"C1\"}})\n\trequire_NoError(t, err)\n\twaitStatsz(6, 1)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S1\", Replicas: 3, Placement: &nats.Placement{Cluster: \"C1\"}})\n\trequire_NoError(t, err)\n\twaitStatsz(3, 2)\n\twaitStatsz(3, 1)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S2\", Replicas: 3, Placement: &nats.Placement{Cluster: \"C1\"}})\n\trequire_NoError(t, err)\n\twaitStatsz(3, 3)\n\twaitStatsz(3, 1)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"S3\", Replicas: 3, Placement: &nats.Placement{Cluster: \"C1\"}})\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"nats: no suitable peers for placement\")\n\trequire_Contains(t, err.Error(), \"miscellaneous issue\")\n\trequire_NoError(t, js.DeleteStream(\"S1\"))\n\twaitStatsz(3, 2)\n\twaitStatsz(3, 1)\n\t_, err = js.AddConsumer(\"S2\", &nats.ConsumerConfig{Durable: \"DUR1\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\twaitStatsz(3, 3)\n\twaitStatsz(3, 1)\n\t_, err = js.AddConsumer(\"S2\", &nats.ConsumerConfig{Durable: \"DUR2\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: insufficient resources\")\n\t_, err = js.AddConsumer(\"S2\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\twaitStatsz(3, 3)\n\twaitStatsz(3, 1)\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"S2\", Replicas: 3, Description: \"foobar\"})\n\trequire_NoError(t, err)\n\twaitStatsz(3, 3)\n\twaitStatsz(3, 1)\n\tsi, err := js.AddStream(&nats.StreamConfig{Name: \"S4\", Replicas: 3})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"C2\")\n\twaitStatsz(3, 3)\n\twaitStatsz(3, 2)\n\tsi, err = js.AddStream(&nats.StreamConfig{Name: \"S5\", Replicas: 3})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.Cluster.Name, \"C2\")\n\twaitStatsz(6, 3)\n\t_, err = js.AddConsumer(\"S4\", &nats.ConsumerConfig{Durable: \"DUR2\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: insufficient resources\")\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"S2\", Replicas: 3, Placement: &nats.Placement{Cluster: \"C2\"}})\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"nats: no suitable peers for placement\")\n\trequire_Contains(t, err.Error(), \"miscellaneous issue\")\n}\n\nfunc TestJetStreamSuperClusterStreamAlternates(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\t// C1\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\", \"country:us\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// C2\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR-1\",\n\t\tReplicas: 1,\n\t\tMirror: &nats.StreamSource{Name: \"SOURCE\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\", \"country:uk\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// C3\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR-2\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: \"SOURCE\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:az\", \"country:jp\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// No client support yet, so do by hand.\n\tgetStreamInfo := func(nc *nats.Conn, expected string) {\n\t\tt.Helper()\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"SOURCE\"), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar si StreamInfo\n\t\terr = json.Unmarshal(resp.Data, &si)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(si.Alternates) == 3)\n\t\trequire_True(t, si.Alternates[0].Cluster == expected)\n\t\tseen := make(map[string]struct{})\n\t\tfor _, alt := range si.Alternates {\n\t\t\tseen[alt.Cluster] = struct{}{}\n\t\t}\n\t\trequire_True(t, len(seen) == 3)\n\t}\n\n\t// Connect to different clusters to check ordering.\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C1\").randomServer())\n\tdefer nc.Close()\n\tgetStreamInfo(nc, \"C1\")\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C2\").randomServer())\n\tdefer nc.Close()\n\tgetStreamInfo(nc, \"C2\")\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C3\").randomServer())\n\tdefer nc.Close()\n\tgetStreamInfo(nc, \"C3\")\n}\n\n// We had a scenario where a consumer would not recover properly on restart due to\n// the cluster state not being set properly when checking source subjects.\nfunc TestJetStreamSuperClusterStateOnRestartPreventsConsumerRecovery(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\t// C1\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\", \"country:us\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// C2\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"DS\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 3,\n\t\tSources: []*nats.StreamSource{{Name: \"SOURCE\"}},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\", \"country:uk\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Bind to DS and match filter subject of SOURCE.\n\t_, err = js.AddConsumer(\"DS\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"foo\",\n\t\tDeliverSubject: \"d\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Send a few messages.\n\tfor i := 0; i < 100; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"HELLO\"))\n\t\trequire_NoError(t, err)\n\t}\n\tsub := natsSubSync(t, nc, \"d\")\n\tnatsNexMsg(t, sub, 5*time.Second)\n\n\tc := sc.clusterForName(\"C2\")\n\tcl := c.consumerLeader(\"$G\", \"DS\", \"dlc\")\n\n\t// Pull source out from underneath the downstream stream.\n\terr = js.DeleteStream(\"SOURCE\")\n\trequire_NoError(t, err)\n\n\tcl.Shutdown()\n\tcl = c.restartServer(cl)\n\tc.waitOnServerHealthz(cl)\n\n\t// Now make sure the consumer is still on this server and has restarted properly.\n\tmset, err := cl.GlobalAccount().lookupStream(\"DS\")\n\trequire_NoError(t, err)\n\tif o := mset.lookupConsumer(\"dlc\"); o == nil {\n\t\tt.Fatalf(\"Consumer was not properly restarted\")\n\t}\n}\n\n// We allow mirrors to opt-in to direct get in a distributed queue group.\nfunc TestJetStreamSuperClusterStreamDirectGetMirrorQueueGroup(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\t// C1\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"SOURCE\",\n\t\tSubjects: []string{\"kv.>\"},\n\t\tMaxMsgsPer: 1,\n\t\tPlacement: &Placement{Tags: []string{\"cloud:aws\", \"country:us\"}},\n\t\tAllowDirect: true,\n\t\tReplicas: 3,\n\t\tStorage: MemoryStorage,\n\t}\n\taddStream(t, nc, cfg)\n\n\tnum := 100\n\tfor i := 0; i < num; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"kv.%d\", i), []byte(\"VAL\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// C2\n\tcfg = &StreamConfig{\n\t\tName: \"M1\",\n\t\tMirror: &StreamSource{Name: \"SOURCE\"},\n\t\tPlacement: &Placement{Tags: []string{\"cloud:gcp\", \"country:uk\"}},\n\t\tMirrorDirect: true,\n\t\tStorage: MemoryStorage,\n\t}\n\taddStream(t, nc, cfg)\n\n\t// C3 (clustered)\n\tcfg = &StreamConfig{\n\t\tName: \"M2\",\n\t\tMirror: &StreamSource{Name: \"SOURCE\"},\n\t\tReplicas: 3,\n\t\tPlacement: &Placement{Tags: []string{\"country:jp\"}},\n\t\tMirrorDirect: true,\n\t\tStorage: MemoryStorage,\n\t}\n\taddStream(t, nc, cfg)\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M2\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != uint64(num) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %d\", num, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Since last one was an R3, check and wait for the direct subscription.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsl := sc.clusterForName(\"C3\").streamLeader(\"$G\", \"M2\")\n\t\tif mset, err := sl.GlobalAccount().lookupStream(\"M2\"); err == nil {\n\t\t\tmset.mu.RLock()\n\t\t\tok := mset.mirrorDirectSub != nil\n\t\t\tmset.mu.RUnlock()\n\t\t\tif ok {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"No dsub yet\")\n\t})\n\n\t// Always do a direct get to the source, but check that we are getting answers from the mirrors when connected to their cluster.\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"SOURCE\")\n\treq := []byte(`{\"last_by_subj\":\"kv.22\"}`)\n\tgetMsg := func(c *nats.Conn) *nats.Msg {\n\t\tm, err := c.Request(getSubj, req, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, string(m.Data) == \"VAL\")\n\t\trequire_True(t, m.Header.Get(JSSequence) == \"23\")\n\t\trequire_True(t, m.Header.Get(JSSubject) == \"kv.22\")\n\t\treturn m\n\t}\n\n\t// C1 -> SOURCE\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C1\").randomServer())\n\tdefer nc.Close()\n\n\tm := getMsg(nc)\n\trequire_True(t, m.Header.Get(JSStream) == \"SOURCE\")\n\n\t// C2 -> M1\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C2\").randomServer())\n\tdefer nc.Close()\n\n\tm = getMsg(nc)\n\trequire_True(t, m.Header.Get(JSStream) == \"M1\")\n\n\t// C3 -> M2\n\tnc, _ = jsClientConnect(t, sc.clusterForName(\"C3\").randomServer())\n\tdefer nc.Close()\n\n\tm = getMsg(nc)\n\trequire_True(t, m.Header.Get(JSStream) == \"M2\")\n}\n\nfunc TestJetStreamSuperClusterTagInducedMoveCancel(t *testing.T) {\n\tserver := map[string]struct{}{}\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tserver[serverName] = struct{}{}\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [%s]\", conf, clusterName)\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\tc := sc.randomCluster()\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"C1\"}},\n\t\tReplicas: 3,\n\t}\n\tsiCreate, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\trequire_Equal(t, siCreate.Cluster.Name, \"C1\")\n\n\ttoSend := uint64(1_000)\n\tfor i := uint64(0); i < toSend; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer ncsys.Close()\n\n\t// cause a move by altering placement tags\n\tcfg.Placement.Tags = []string{\"C2\"}\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Retry in case stream is still being created and without a leader we won't receive a response\n\tvar cancelResp JSApiStreamUpdateResponse\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\trmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamCancelMoveT, \"$G\", \"TEST\"), nil, 1*time.Second)\n\t\tif errors.Is(err, nats.ErrTimeout) {\n\t\t\treturn err\n\t\t}\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &cancelResp))\n\t\treturn nil\n\t})\n\tif cancelResp.Error != nil && ErrorIdentifier(cancelResp.Error.ErrCode) == JSStreamMoveNotInProgress {\n\t\tt.Skip(\"This can happen with delays, when Move completed before Cancel\", cancelResp.Error)\n\t\treturn\n\t}\n\trequire_True(t, cancelResp.Error == nil)\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.Config.Placement != nil {\n\t\t\treturn fmt.Errorf(\"expected placement to be cleared got: %+v\", si.Config.Placement)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterMoveCancel(t *testing.T) {\n\tusageTickOld := usageTick\n\tusageTick = 250 * time.Millisecond\n\tdefer func() {\n\t\tusageTick = usageTickOld\n\t}()\n\n\tserver := map[string]struct{}{}\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tserver[serverName] = struct{}{}\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [%s]\", conf, serverName)\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\tc := sc.randomCluster()\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\tsiCreate, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tstreamPeerSrv := []string{siCreate.Cluster.Leader, siCreate.Cluster.Replicas[0].Name, siCreate.Cluster.Replicas[1].Name}\n\t// determine empty server\n\tfor _, s := range streamPeerSrv {\n\t\tdelete(server, s)\n\t}\n\t// pick left over server in same cluster as other server\n\temptySrv := _EMPTY_\n\tfor s := range server {\n\t\t// server name is prefixed with cluster name\n\t\tif strings.HasPrefix(s, c.name) {\n\t\t\temptySrv = s\n\t\t\tbreak\n\t\t}\n\t}\n\n\texpectedPeers := map[string]struct{}{\n\t\tgetHash(streamPeerSrv[0]): {},\n\t\tgetHash(streamPeerSrv[1]): {},\n\t\tgetHash(streamPeerSrv[2]): {},\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DUR\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{InactiveThreshold: time.Hour, AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tephName := ci.Name\n\n\ttoSend := uint64(1_000)\n\tfor i := uint64(0); i < toSend; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tserverEmpty := func(fromSrv string) error {\n\t\tif jszAfter, err := c.serverByName(fromSrv).Jsz(nil); err != nil {\n\t\t\treturn fmt.Errorf(\"could not fetch JS info for server: %v\", err)\n\t\t} else if jszAfter.Streams != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d streams\", jszAfter.Streams)\n\t\t} else if jszAfter.Consumers != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d consumers\", jszAfter.Consumers)\n\t\t} else if jszAfter.Bytes != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d storage\", jszAfter.Store)\n\t\t}\n\t\treturn nil\n\t}\n\n\tcheckSrvInvariant := func(s *Server, expectedPeers map[string]struct{}) error {\n\t\tjs, cc := s.getJetStreamCluster()\n\t\tjs.mu.Lock()\n\t\tdefer js.mu.Unlock()\n\t\tif sa, ok := cc.streams[\"$G\"][\"TEST\"]; !ok {\n\t\t\treturn fmt.Errorf(\"stream not found\")\n\t\t} else if len(sa.Group.Peers) != len(expectedPeers) {\n\t\t\treturn fmt.Errorf(\"stream peer group size not %d, but %d\", len(expectedPeers), len(sa.Group.Peers))\n\t\t} else if da, ok := sa.consumers[\"DUR\"]; !ok {\n\t\t\treturn fmt.Errorf(\"durable not found\")\n\t\t} else if len(da.Group.Peers) != len(expectedPeers) {\n\t\t\treturn fmt.Errorf(\"durable peer group size not %d, but %d\", len(expectedPeers), len(da.Group.Peers))\n\t\t} else if ea, ok := sa.consumers[ephName]; !ok {\n\t\t\treturn fmt.Errorf(\"ephemeral not found\")\n\t\t} else if len(ea.Group.Peers) != 1 {\n\t\t\treturn fmt.Errorf(\"ephemeral peer group size not 1, but %d\", len(ea.Group.Peers))\n\t\t} else if _, ok := expectedPeers[ea.Group.Peers[0]]; !ok {\n\t\t\treturn fmt.Errorf(\"ephemeral peer not an expected peer\")\n\t\t} else {\n\t\t\tfor _, p := range sa.Group.Peers {\n\t\t\t\tif _, ok := expectedPeers[p]; !ok {\n\t\t\t\t\treturn fmt.Errorf(\"peer not expected\")\n\t\t\t\t}\n\t\t\t\tfound := false\n\t\t\t\tfor _, dp := range da.Group.Peers {\n\t\t\t\t\tif p == dp {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tt.Logf(\"durable peer group does not match stream peer group\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\tncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer ncsys.Close()\n\n\ttime.Sleep(2 * usageTick)\n\taiBefore, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\n\tfor _, moveFromSrv := range streamPeerSrv {\n\t\tmoveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{Server: moveFromSrv, Tags: []string{emptySrv}})\n\t\trequire_NoError(t, err)\n\t\trmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, \"$G\", \"TEST\"), moveReq, 5*time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar moveResp JSApiStreamUpdateResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &moveResp))\n\t\trequire_True(t, moveResp.Error == nil)\n\n\t\trmsg, err = ncsys.Request(fmt.Sprintf(JSApiServerStreamCancelMoveT, \"$G\", \"TEST\"), nil, 5*time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar cancelResp JSApiStreamUpdateResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &cancelResp))\n\t\tif cancelResp.Error != nil && ErrorIdentifier(cancelResp.Error.ErrCode) == JSStreamMoveNotInProgress {\n\t\t\tt.Skip(\"This can happen with delays, when Move completed before Cancel\", cancelResp.Error)\n\t\t\treturn\n\t\t}\n\t\trequire_True(t, cancelResp.Error == nil)\n\n\t\tfor _, sExpected := range streamPeerSrv {\n\t\t\ts := c.serverByName(sExpected)\n\t\t\trequire_True(t, s.JetStreamIsStreamAssigned(\"$G\", \"TEST\"))\n\t\t\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error { return checkSrvInvariant(s, expectedPeers) })\n\t\t}\n\t\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error { return serverEmpty(emptySrv) })\n\t\tcheckFor(t, 3*usageTick, 100*time.Millisecond, func() error {\n\t\t\tif aiAfter, err := js.AccountInfo(); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if aiAfter.Store != aiBefore.Store {\n\t\t\t\treturn fmt.Errorf(\"store before %d and after %d don't match\", aiBefore.Store, aiAfter.Store)\n\t\t\t} else {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterDoubleStreamMove(t *testing.T) {\n\tserver := map[string]struct{}{}\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tserver[serverName] = struct{}{}\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [%s]\", conf, serverName)\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\tc := sc.randomCluster()\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\tsiCreate, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tsrvMoveList := []string{siCreate.Cluster.Leader, siCreate.Cluster.Replicas[0].Name, siCreate.Cluster.Replicas[1].Name}\n\t// determine empty server\n\tfor _, s := range srvMoveList {\n\t\tdelete(server, s)\n\t}\n\t// pick left over server in same cluster as other server\n\tfor s := range server {\n\t\t// server name is prefixed with cluster name\n\t\tif strings.HasPrefix(s, c.name) {\n\t\t\tsrvMoveList = append(srvMoveList, s)\n\t\t\tbreak\n\t\t}\n\t}\n\n\tservers := []*Server{\n\t\tc.serverByName(srvMoveList[0]),\n\t\tc.serverByName(srvMoveList[1]),\n\t\tc.serverByName(srvMoveList[2]),\n\t\tc.serverByName(srvMoveList[3]), // starts out empty\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DUR\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{InactiveThreshold: time.Hour, AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tephName := ci.Name\n\n\ttoSend := uint64(100)\n\tfor i := uint64(0); i < toSend; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer ncsys.Close()\n\n\tmove := func(fromSrv string, toTags ...string) {\n\t\tsEmpty := c.serverByName(fromSrv)\n\t\tjszBefore, err := sEmpty.Jsz(nil)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, jszBefore.Streams == 1)\n\n\t\tmoveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{\n\t\t\tServer: fromSrv, Tags: toTags})\n\t\trequire_NoError(t, err)\n\t\trmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, \"$G\", \"TEST\"), moveReq, 100*time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar moveResp JSApiStreamUpdateResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &moveResp))\n\t\trequire_True(t, moveResp.Error == nil)\n\t}\n\n\tserverEmpty := func(fromSrv string) error {\n\t\tif jszAfter, err := c.serverByName(fromSrv).Jsz(nil); err != nil {\n\t\t\treturn fmt.Errorf(\"could not fetch JS info for server: %v\", err)\n\t\t} else if jszAfter.Streams != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d streams\", jszAfter.Streams)\n\t\t} else if jszAfter.Consumers != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d consumers\", jszAfter.Consumers)\n\t\t} else if jszAfter.Store != 0 {\n\t\t\treturn fmt.Errorf(\"empty server still has %d storage\", jszAfter.Store)\n\t\t}\n\t\treturn nil\n\t}\n\n\tmoveComplete := func(toSrv string, expectedSet ...string) error {\n\t\teSet := map[string]int{}\n\t\tfoundInExpected := false\n\t\tfor i, sExpected := range expectedSet {\n\t\t\teSet[sExpected] = i\n\t\t\ts := c.serverByName(sExpected)\n\t\t\tif !s.JetStreamIsStreamAssigned(\"$G\", \"TEST\") {\n\t\t\t\treturn fmt.Errorf(\"expected stream to be assigned to %s\", sExpected)\n\t\t\t}\n\t\t\t// test list order invariant\n\t\t\tjs, cc := s.getJetStreamCluster()\n\t\t\tsExpHash := getHash(sExpected)\n\t\t\tjs.mu.Lock()\n\t\t\tif sa, ok := cc.streams[\"$G\"][\"TEST\"]; !ok {\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\treturn fmt.Errorf(\"stream not found in cluster\")\n\t\t\t} else if len(sa.Group.Peers) != 3 {\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\treturn fmt.Errorf(\"peers not reset\")\n\t\t\t} else if sa.Group.Peers[i] != sExpHash {\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\treturn fmt.Errorf(\"stream: expected peer %s on index %d, got %s/%s\",\n\t\t\t\t\tsa.Group.Peers[i], i, sExpHash, sExpected)\n\t\t\t} else if ca, ok := sa.consumers[\"DUR\"]; !ok {\n\t\t\t\tjs.mu.Unlock()\n\t\t\t\treturn fmt.Errorf(\"durable not found in stream\")\n\t\t\t} else {\n\t\t\t\tif !slices.Contains(ca.Group.Peers, sExpHash) {\n\t\t\t\t\tjs.mu.Unlock()\n\t\t\t\t\treturn fmt.Errorf(\"consumer expected peer %s/%s bud didn't find in %+v\",\n\t\t\t\t\t\tsExpHash, sExpected, ca.Group.Peers)\n\t\t\t\t}\n\t\t\t\tif ephA, ok := sa.consumers[ephName]; ok {\n\t\t\t\t\tif len(ephA.Group.Peers) != 1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"ephemeral peers not reset\")\n\t\t\t\t\t}\n\t\t\t\t\tfoundInExpected = foundInExpected || (ephA.Group.Peers[0] == cc.meta.ID())\n\t\t\t\t}\n\t\t\t}\n\t\t\tjs.mu.Unlock()\n\t\t}\n\t\tif len(expectedSet) > 0 && !foundInExpected {\n\t\t\treturn fmt.Errorf(\"ephemeral peer not expected\")\n\t\t}\n\t\tfor _, s := range servers {\n\t\t\tif jszAfter, err := c.serverByName(toSrv).Jsz(nil); err != nil {\n\t\t\t\treturn fmt.Errorf(\"could not fetch JS info for server: %v\", err)\n\t\t\t} else if jszAfter.Messages != toSend {\n\t\t\t\treturn fmt.Errorf(\"messages not yet copied, got %d, expected %d\", jszAfter.Messages, toSend)\n\t\t\t}\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\tif si, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second)); err != nil {\n\t\t\t\treturn fmt.Errorf(\"could not fetch stream info: %v\", err)\n\t\t\t} else if len(si.Cluster.Replicas)+1 != si.Config.Replicas {\n\t\t\t\treturn fmt.Errorf(\"not yet downsized replica should be empty has: %d != %d (%s)\",\n\t\t\t\t\tlen(si.Cluster.Replicas)+1, si.Config.Replicas, si.Cluster.Leader)\n\t\t\t} else if si.Cluster.Leader == _EMPTY_ {\n\t\t\t\treturn fmt.Errorf(\"leader not found\")\n\t\t\t} else if len(expectedSet) > 0 {\n\t\t\t\tif _, ok := eSet[si.Cluster.Leader]; !ok {\n\t\t\t\t\treturn fmt.Errorf(\"leader %s not in expected set %+v\", si.Cluster.Leader, eSet)\n\t\t\t\t} else if _, ok := eSet[si.Cluster.Replicas[0].Name]; !ok {\n\t\t\t\t\treturn fmt.Errorf(\"leader %s not in expected set %+v\", si.Cluster.Replicas[0].Name, eSet)\n\t\t\t\t} else if _, ok := eSet[si.Cluster.Replicas[1].Name]; !ok {\n\t\t\t\t\treturn fmt.Errorf(\"leader %s not in expected set %+v\", si.Cluster.Replicas[1].Name, eSet)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Close()\n\t\t}\n\t\treturn nil\n\t}\n\n\tmoveAndCheck := func(from, to string, expectedSet ...string) {\n\t\tt.Helper()\n\t\tmove(from, to)\n\t\tcheckFor(t, 40*time.Second, 100*time.Millisecond, func() error { return moveComplete(to, expectedSet...) })\n\t\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error { return serverEmpty(from) })\n\t}\n\n\tcheckFor(t, 20*time.Second, 1000*time.Millisecond, func() error { return serverEmpty(srvMoveList[3]) })\n\t// first iteration establishes order of server 0-2 (the internal order in the server could be 1,0,2)\n\tmoveAndCheck(srvMoveList[0], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[1], srvMoveList[0])\n\tmoveAndCheck(srvMoveList[2], srvMoveList[1])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[1], srvMoveList[2])\n\t// second iteration iterates in order\n\tmoveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[1], srvMoveList[2], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[1], srvMoveList[0], srvMoveList[2], srvMoveList[3], srvMoveList[0])\n\tmoveAndCheck(srvMoveList[2], srvMoveList[1], srvMoveList[3], srvMoveList[0], srvMoveList[1])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[1], srvMoveList[2])\n\t// iterate in the opposite direction and establish order 2-0\n\tmoveAndCheck(srvMoveList[2], srvMoveList[3], srvMoveList[0], srvMoveList[1], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[1], srvMoveList[2], srvMoveList[0], srvMoveList[3], srvMoveList[2])\n\tmoveAndCheck(srvMoveList[0], srvMoveList[1], srvMoveList[3], srvMoveList[2], srvMoveList[1])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0])\n\t// move server in the middle of list\n\tmoveAndCheck(srvMoveList[1], srvMoveList[3], srvMoveList[2], srvMoveList[0], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[0], srvMoveList[1], srvMoveList[2], srvMoveList[3], srvMoveList[1])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0])\n\t// repeatedly use end\n\tmoveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[2], srvMoveList[1], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0])\n\tmoveAndCheck(srvMoveList[0], srvMoveList[3], srvMoveList[2], srvMoveList[1], srvMoveList[3])\n\tmoveAndCheck(srvMoveList[3], srvMoveList[0], srvMoveList[2], srvMoveList[1], srvMoveList[0])\n}\n\nfunc TestJetStreamSuperClusterPeerEvacuationAndStreamReassignment(t *testing.T) {\n\ts := createJetStreamSuperClusterWithTemplateAndModHook(t, jsClusterTempl, 4, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\treturn fmt.Sprintf(\"%s\\nserver_tags: [cluster:%s, server:%s]\", conf, clusterName, serverName)\n\t\t}, nil)\n\tdefer s.shutdown()\n\n\tc := s.clusterForName(\"C1\")\n\n\t// Client based API\n\tsrv := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, srv)\n\tdefer nc.Close()\n\n\ttest := func(t *testing.T, r int, moveTags []string, targetCluster string, testMigrateTo bool, listFrom bool) {\n\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tReplicas: r,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer js.DeleteStream(\"TEST\")\n\t\tstartSet := map[string]struct{}{\n\t\t\tsi.Cluster.Leader: {},\n\t\t}\n\t\tfor _, p := range si.Cluster.Replicas {\n\t\t\tstartSet[p.Name] = struct{}{}\n\t\t}\n\n\t\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"DUR\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.SubscribeSync(\"foo\")\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\ttoMoveFrom := si.Cluster.Leader\n\t\tif !listFrom {\n\t\t\ttoMoveFrom = _EMPTY_\n\t\t}\n\t\tsLdr := c.serverByName(si.Cluster.Leader)\n\t\tjszBefore, err := sLdr.Jsz(nil)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, jszBefore.Streams == 1)\n\t\trequire_True(t, jszBefore.Consumers >= 1)\n\t\trequire_True(t, jszBefore.Store != 0)\n\n\t\tmigrateToServer := _EMPTY_\n\t\tif testMigrateTo {\n\t\t\t// find an empty server\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tname := s.Name()\n\t\t\t\tfound := si.Cluster.Leader == name\n\t\t\t\tif !found {\n\t\t\t\t\tfor _, r := range si.Cluster.Replicas {\n\t\t\t\t\t\tif r.Name == name {\n\t\t\t\t\t\t\tfound = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tmigrateToServer = name\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tjszAfter, err := c.serverByName(migrateToServer).Jsz(nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, jszAfter.Streams == 0)\n\n\t\t\tmoveTags = append(moveTags, fmt.Sprintf(\"server:%s\", migrateToServer))\n\t\t}\n\n\t\tncsys, err := nats.Connect(srv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\trequire_NoError(t, err)\n\t\tdefer ncsys.Close()\n\n\t\tmoveReq, err := json.Marshal(&JSApiMetaServerStreamMoveRequest{Server: toMoveFrom, Tags: moveTags})\n\t\trequire_NoError(t, err)\n\t\trmsg, err := ncsys.Request(fmt.Sprintf(JSApiServerStreamMoveT, \"$G\", \"TEST\"), moveReq, 100*time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar moveResp JSApiStreamUpdateResponse\n\t\trequire_NoError(t, json.Unmarshal(rmsg.Data, &moveResp))\n\t\trequire_True(t, moveResp.Error == nil)\n\n\t\t// test move to particular server\n\t\tif testMigrateTo {\n\t\t\ttoSrv := c.serverByName(migrateToServer)\n\t\t\tcheckFor(t, 20*time.Second, 1000*time.Millisecond, func() error {\n\t\t\t\tjszAfter, err := toSrv.Jsz(nil)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"could not fetch JS info for server: %v\", err)\n\t\t\t\t}\n\t\t\t\tif jszAfter.Streams != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"server expected to have one stream, has %d\", jszAfter.Streams)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\t// Now wait until the stream is now current.\n\t\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"could not fetch stream info: %v\", err)\n\t\t\t}\n\t\t\tif si.Cluster.Leader == toMoveFrom {\n\t\t\t\treturn fmt.Errorf(\"peer not removed yet: %+v\", toMoveFrom)\n\t\t\t}\n\t\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\t\treturn fmt.Errorf(\"no leader yet\")\n\t\t\t}\n\t\t\tif len(si.Cluster.Replicas) != r-1 {\n\t\t\t\treturn fmt.Errorf(\"not yet downsized replica should be %d has: %d\", r-1, len(si.Cluster.Replicas))\n\t\t\t}\n\t\t\tif si.Config.Replicas != r {\n\t\t\t\treturn fmt.Errorf(\"bad replica count %d\", si.Config.Replicas)\n\t\t\t}\n\t\t\tif si.Cluster.Name != targetCluster {\n\t\t\t\treturn fmt.Errorf(\"stream expected in %s but found in %s\", si.Cluster.Name, targetCluster)\n\t\t\t}\n\t\t\tsNew := s.serverByName(si.Cluster.Leader)\n\t\t\tif jszNew, err := sNew.Jsz(nil); err != nil {\n\t\t\t\treturn err\n\t\t\t} else if jszNew.Streams != 1 {\n\t\t\t\treturn fmt.Errorf(\"new leader has %d streams, not one\", jszNew.Streams)\n\t\t\t} else if jszNew.Store != jszBefore.Store {\n\t\t\t\treturn fmt.Errorf(\"new leader has %d storage, should have %d\", jszNew.Store, jszBefore.Store)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// test draining\n\t\tcheckFor(t, 20*time.Second, time.Second, func() error {\n\t\t\tif !listFrom {\n\t\t\t\t// when needed determine which server move moved away from\n\t\t\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"could not fetch stream info: %v\", err)\n\t\t\t\t}\n\t\t\t\tfor n := range startSet {\n\t\t\t\t\tif n != si.Cluster.Leader {\n\t\t\t\t\t\tvar found bool\n\t\t\t\t\t\tfor _, p := range si.Cluster.Replicas {\n\t\t\t\t\t\t\tif p.Name == n {\n\t\t\t\t\t\t\t\tfound = true\n\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif !found {\n\t\t\t\t\t\t\ttoMoveFrom = n\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif toMoveFrom == _EMPTY_ {\n\t\t\t\treturn fmt.Errorf(\"server to move away from not found\")\n\t\t\t}\n\t\t\tsEmpty := c.serverByName(toMoveFrom)\n\t\t\tjszAfter, err := sEmpty.Jsz(nil)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"could not fetch JS info for server: %v\", err)\n\t\t\t}\n\t\t\tif jszAfter.Streams != 0 {\n\t\t\t\treturn fmt.Errorf(\"empty server still has %d streams\", jszAfter.Streams)\n\t\t\t}\n\t\t\tif jszAfter.Consumers != 0 {\n\t\t\t\treturn fmt.Errorf(\"empty server still has %d consumers\", jszAfter.Consumers)\n\t\t\t}\n\t\t\tif jszAfter.Store != 0 {\n\t\t\t\treturn fmt.Errorf(\"empty server still has %d storage\", jszAfter.Store)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// consume messages from ephemeral consumer\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tfor i := 1; i <= 3; i++ {\n\t\tt.Run(fmt.Sprintf(\"r%d\", i), func(t *testing.T) {\n\t\t\ttest(t, i, nil, \"C1\", false, true)\n\t\t})\n\t\tt.Run(fmt.Sprintf(\"r%d-explicit\", i), func(t *testing.T) {\n\t\t\ttest(t, i, nil, \"C1\", true, true)\n\t\t})\n\t\tt.Run(fmt.Sprintf(\"r%d-nosrc\", i), func(t *testing.T) {\n\t\t\ttest(t, i, nil, \"C1\", false, false)\n\t\t})\n\t}\n\n\tt.Run(\"r3-cluster-move\", func(t *testing.T) {\n\t\ttest(t, 3, []string{\"cluster:C2\"}, \"C2\", false, false)\n\t})\n\tt.Run(\"r3-cluster-move-nosrc\", func(t *testing.T) {\n\t\ttest(t, 3, []string{\"cluster:C2\"}, \"C2\", false, true)\n\t})\n}\n\nfunc TestJetStreamSuperClusterMirrorInheritsAllowDirect(t *testing.T) {\n\tsc := createJetStreamTaggedSuperCluster(t)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"key.*\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\", \"country:us\"}},\n\t\tMaxMsgsPerSubject: 1,\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"KV\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\", \"country:uk\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Do direct grab for now.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"M\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar si StreamInfo\n\terr = json.Unmarshal(resp.Data, &si)\n\trequire_NoError(t, err)\n\n\tif !si.Config.MirrorDirect {\n\t\tt.Fatalf(\"Expected MirrorDirect to be inherited as true\")\n\t}\n}\n\nfunc TestJetStreamSuperClusterSystemLimitsPlacement(t *testing.T) {\n\tconst largeSystemLimit = 1024\n\tconst smallSystemLimit = 512\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {\n\t\t\tmax_mem_store: _MAXMEM_\n\t\t\tmax_file_store: _MAXFILE_\n\t\t\tstore_dir: '%s',\n\t\t}\n\t\tserver_tags: [\n\t\t\t_TAG_\n\t\t]\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tstoreCnf := func(serverName, clusterName, storeDir, conf string) string {\n\t\tswitch {\n\t\tcase strings.HasPrefix(serverName, \"C1\"):\n\t\t\tconf = strings.Replace(conf, \"_MAXMEM_\", fmt.Sprint(largeSystemLimit), 1)\n\t\t\tconf = strings.Replace(conf, \"_MAXFILE_\", fmt.Sprint(largeSystemLimit), 1)\n\t\t\treturn strings.Replace(conf, \"_TAG_\", serverName, 1)\n\t\tcase strings.HasPrefix(serverName, \"C2\"):\n\t\t\tconf = strings.Replace(conf, \"_MAXMEM_\", fmt.Sprint(smallSystemLimit), 1)\n\t\t\tconf = strings.Replace(conf, \"_MAXFILE_\", fmt.Sprint(smallSystemLimit), 1)\n\t\t\treturn strings.Replace(conf, \"_TAG_\", serverName, 1)\n\t\tdefault:\n\t\t\treturn conf\n\t\t}\n\t}\n\n\tsCluster := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, storeCnf, nil)\n\tdefer sCluster.shutdown()\n\n\trequestLeaderStepDown := func(clientURL string) error {\n\t\tnc, err := nats.Connect(clientURL, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tncResp, err := nc.Request(JSApiLeaderStepDown, nil, time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar resp JSApiLeaderStepDownResponse\n\t\tif err := json.Unmarshal(ncResp.Data, &resp); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif resp.Error != nil {\n\t\t\treturn resp.Error\n\t\t}\n\t\tif !resp.Success {\n\t\t\treturn fmt.Errorf(\"leader step down request not successful\")\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// Force large cluster to be leader\n\tvar largeLeader *Server\n\terr := checkForErr(15*time.Second, 500*time.Millisecond, func() error {\n\t\t// Range over cluster A, which is the large cluster.\n\t\tservers := sCluster.clusters[0].servers\n\t\tfor _, s := range servers {\n\t\t\tif s.JetStreamIsLeader() {\n\t\t\t\tlargeLeader = s\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\tif err := requestLeaderStepDown(servers[0].ClientURL()); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to request leader step down: %s\", err)\n\t\t}\n\t\treturn fmt.Errorf(\"leader is not in large cluster\")\n\t})\n\tif err != nil {\n\t\tt.Skipf(\"failed to get desired layout: %s\", err)\n\t}\n\n\tgetStreams := func(jsm nats.JetStreamManager) []string {\n\t\tvar streams []string\n\t\tfor s := range jsm.StreamNames() {\n\t\t\tstreams = append(streams, s)\n\t\t}\n\t\treturn streams\n\t}\n\tnc, js := jsClientConnect(t, largeLeader)\n\tdefer nc.Close()\n\n\tcases := []struct {\n\t\tname string\n\t\tstorage nats.StorageType\n\t\tcreateMaxBytes int64\n\t\tserverTag string\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"file create large stream on small cluster b0\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S1\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on small cluster b0\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S1\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on small cluster b1\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S2\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on small cluster b1\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S2\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on small cluster b2\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S3\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on small cluster b2\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C2-S3\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on large cluster a0\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S1\",\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on large cluster a0\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S1\",\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on large cluster a1\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S2\",\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on large cluster a1\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S2\",\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on large cluster a2\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S3\",\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on large cluster a2\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: smallSystemLimit + 1,\n\t\t\tserverTag: \"C1-S3\",\n\t\t},\n\t}\n\tfor i := 0; i < len(cases) && !t.Failed(); i++ {\n\t\tc := cases[i]\n\t\tt.Run(c.name, func(st *testing.T) {\n\t\t\tvar clusterName string\n\t\t\tif strings.HasPrefix(c.serverTag, \"a\") {\n\t\t\t\tclusterName = \"cluster-a\"\n\t\t\t} else if strings.HasPrefix(c.serverTag, \"b\") {\n\t\t\t\tclusterName = \"cluster-b\"\n\t\t\t}\n\n\t\t\tif s := getStreams(js); len(s) != 0 {\n\t\t\t\tst.Fatalf(\"unexpected stream count, got=%d, want=0\", len(s))\n\t\t\t}\n\n\t\t\tstreamName := fmt.Sprintf(\"TEST-%s\", c.serverTag)\n\t\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: streamName,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: c.storage,\n\t\t\t\tMaxBytes: c.createMaxBytes,\n\t\t\t\tPlacement: &nats.Placement{\n\t\t\t\t\tCluster: clusterName,\n\t\t\t\t\tTags: []string{c.serverTag},\n\t\t\t\t},\n\t\t\t})\n\t\t\tif c.wantErr && err == nil {\n\t\t\t\tif s := getStreams(js); len(s) != 1 {\n\t\t\t\t\tst.Logf(\"unexpected stream count, got=%d, want=1, streams=%v\", len(s), s)\n\t\t\t\t}\n\n\t\t\t\tcfg := si.Config\n\t\t\t\tst.Fatalf(\"unexpected success, maxBytes=%d, cluster=%s, tags=%v\",\n\t\t\t\t\tcfg.MaxBytes, cfg.Placement.Cluster, cfg.Placement.Tags)\n\t\t\t} else if !c.wantErr && err != nil {\n\t\t\t\tif s := getStreams(js); len(s) != 0 {\n\t\t\t\t\tst.Logf(\"unexpected stream count, got=%d, want=0, streams=%v\", len(s), s)\n\t\t\t\t}\n\n\t\t\t\trequire_NoError(st, err)\n\t\t\t}\n\n\t\t\tif err == nil {\n\t\t\t\tif s := getStreams(js); len(s) != 1 {\n\t\t\t\t\tst.Fatalf(\"unexpected stream count, got=%d, want=1\", len(s))\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Delete regardless.\n\t\t\tjs.DeleteStream(streamName)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSuperClusterMixedModeSwitchToInterestOnlyStaticConfig(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf: { listen: 127.0.0.1:-1 }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\taccounts {\n\t\t\tONE {\n\t\t\t\tusers = [ { user: \"one\", pass: \"pwd\" } ]\n\t\t\t\tjetstream: enabled\n\t\t\t}\n\t\t\tTWO { users = [ { user: \"two\", pass: \"pwd\" } ] }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t`\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 5, 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tsname := serverName[strings.Index(serverName, \"-\")+1:]\n\t\t\tswitch sname {\n\t\t\tcase \"S4\", \"S5\":\n\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream: { \", \"#jetstream: { \")\n\t\t\tdefault:\n\t\t\t\tconf = strings.ReplaceAll(conf, \"leaf: { \", \"#leaf: { \")\n\t\t\t}\n\t\t\treturn conf\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Connect our client to a non JS server\n\tc := sc.randomCluster()\n\tvar s *Server\n\tfor _, as := range c.servers {\n\t\tif !as.JetStreamEnabled() {\n\t\t\ts = as\n\t\t\tbreak\n\t\t}\n\t}\n\tif s == nil {\n\t\tt.Fatal(\"Did not find a non JS server!\")\n\t}\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"one\", \"pwd\"))\n\tdefer nc.Close()\n\n\t// Just create a stream and then make sure that all gateways have switched\n\t// to interest-only mode.\n\tsi, err := js.AddStream(&nats.StreamConfig{Name: \"interest\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(\"ONE\", \"interest\")\n\n\tcheck := func(accName string) {\n\t\tt.Helper()\n\t\tfor _, c := range sc.clusters {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\t// Check only JS servers outbound GW connections\n\t\t\t\tif !s.JetStreamEnabled() {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\topts := s.getOpts()\n\t\t\t\tfor _, gw := range opts.Gateway.Gateways {\n\t\t\t\t\tif gw.Name == opts.Gateway.Name {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tcheckGWInterestOnlyMode(t, s, gw.Name, accName)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// Starting v2.9.0, all accounts should be switched to interest-only mode\n\tcheck(\"ONE\")\n\tcheck(\"TWO\")\n\n\tvar gwsa [16]*client\n\tgws := gwsa[:0]\n\n\ts = sc.serverByName(si.Cluster.Leader)\n\t// Get the GW outbound connections\n\ts.getOutboundGatewayConnections(&gws)\n\tfor _, gwc := range gws {\n\t\tgwc.mu.Lock()\n\t\tgwc.nc.Close()\n\t\tgwc.mu.Unlock()\n\t}\n\twaitForOutboundGateways(t, s, 2, 5*time.Second)\n\tcheck(\"ONE\")\n\tcheck(\"TWO\")\n}\n\nfunc TestJetStreamSuperClusterMixedModeSwitchToInterestOnlyOperatorConfig(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\n\tskp, _ := nkeys.CreateAccount()\n\tspub, _ := skp.PublicKey()\n\tnac := jwt.NewAccountClaims(spub)\n\tsjwt, err := nac.Encode(kp)\n\trequire_NoError(t, err)\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac = jwt.NewAccountClaims(apub)\n\t// Set some limits to enable JS.\n\tnac.Limits.JetStreamLimits.DiskStorage = 1024 * 1024\n\tnac.Limits.JetStreamLimits.Streams = 10\n\tajwt, err := nac.Encode(kp)\n\trequire_NoError(t, err)\n\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif strings.HasSuffix(r.URL.Path, spub) {\n\t\t\tw.Write([]byte(sjwt))\n\t\t} else {\n\t\t\tw.Write([]byte(ajwt))\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\toperator := fmt.Sprintf(`\n\t\toperator: %s\n\t\tresolver: URL(\"%s/ngs/v1/accounts/jwt/\")\n\t`, ojwt, ts.URL)\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf: { listen: 127.0.0.1:-1 }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t` + operator\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 5, 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tconf = strings.ReplaceAll(conf, \"system_account: \\\"$SYS\\\"\", fmt.Sprintf(\"system_account: \\\"%s\\\"\", spub))\n\t\t\tsname := serverName[strings.Index(serverName, \"-\")+1:]\n\t\t\tswitch sname {\n\t\t\tcase \"S4\", \"S5\":\n\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream: { \", \"#jetstream: { \")\n\t\t\tdefault:\n\t\t\t\tconf = strings.ReplaceAll(conf, \"leaf: { \", \"#leaf: { \")\n\t\t\t}\n\t\t\treturn conf\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Connect our client to a non JS server\n\tc := sc.randomCluster()\n\tvar s *Server\n\tfor _, as := range c.servers {\n\t\tif !as.JetStreamEnabled() {\n\t\t\ts = as\n\t\t\tbreak\n\t\t}\n\t}\n\tif s == nil {\n\t\tt.Fatal(\"Did not find a non JS server!\")\n\t}\n\tnc, js := jsClientConnect(t, s, createUserCreds(t, nil, akp))\n\tdefer nc.Close()\n\n\t// Prevent 'nats: JetStream not enabled for account' when creating the first stream.\n\tc.waitOnAccount(apub)\n\n\t// Just create a stream and then make sure that all gateways have switched\n\t// to interest-only mode.\n\tsi, err := js.AddStream(&nats.StreamConfig{Name: \"interest\", Replicas: 3})\n\trequire_NoError(t, err)\n\n\tsc.waitOnStreamLeader(apub, \"interest\")\n\n\tcheck := func(s *Server) {\n\t\topts := s.getOpts()\n\t\tfor _, gw := range opts.Gateway.Gateways {\n\t\t\tif gw.Name == opts.Gateway.Name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcheckGWInterestOnlyModeOrNotPresent(t, s, gw.Name, apub, true)\n\t\t}\n\t}\n\ts = sc.serverByName(si.Cluster.Leader)\n\tcheck(s)\n\n\t// Let's cause a leadership change and verify that it still works.\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"interest\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tsc.waitOnStreamLeader(apub, \"interest\")\n\n\tsi, err = js.StreamInfo(\"interest\")\n\trequire_NoError(t, err)\n\ts = sc.serverByName(si.Cluster.Leader)\n\tcheck(s)\n\n\tvar gwsa [16]*client\n\tgws := gwsa[:0]\n\t// Get the GW outbound connections\n\ts.getOutboundGatewayConnections(&gws)\n\tfor _, gwc := range gws {\n\t\tgwc.mu.Lock()\n\t\tgwc.nc.Close()\n\t\tgwc.mu.Unlock()\n\t}\n\twaitForOutboundGateways(t, s, 2, 5*time.Second)\n\tcheck(s)\n}\n\ntype captureGWRewriteLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *captureGWRewriteLogger) Tracef(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"$JS.SNAPSHOT.ACK.TEST\") && strings.Contains(msg, gwReplyPrefix) {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestJetStreamSuperClusterGWReplyRewrite(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.serverByName(\"C1-S1\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\tnc2, _ := jsClientConnect(t, sc.serverByName(\"C2-S2\"))\n\tdefer nc2.Close()\n\n\ts := sc.clusters[0].streamLeader(globalAccountName, \"TEST\")\n\tvar gws []*client\n\ts.getOutboundGatewayConnections(&gws)\n\tfor _, gw := range gws {\n\t\tgw.mu.Lock()\n\t\tgw.trace = true\n\t\tgw.mu.Unlock()\n\t}\n\tl := &captureGWRewriteLogger{ch: make(chan string, 1)}\n\ts.SetLogger(l, false, true)\n\n\t// Send a request through the gateway\n\tsreq := &JSApiStreamSnapshotRequest{\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tChunkSize: 512,\n\t}\n\tnatsSub(t, nc2, sreq.DeliverSubject, func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tnatsFlush(t, nc2)\n\treq, _ := json.Marshal(sreq)\n\trmsg, err := nc2.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiStreamSnapshotResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\t// Now we just want to make sure that the reply has the gateway prefix\n\tselect {\n\tcase <-l.ch:\n\tcase <-time.After(10 * time.Second):\n\t}\n}\n\nfunc TestJetStreamSuperClusterGWOfflineSatus(t *testing.T) {\n\torgEventsHBInterval := eventsHBInterval\n\teventsHBInterval = 500 * time.Millisecond //time.Second\n\tdefer func() { eventsHBInterval = orgEventsHBInterval }()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tgateway {\n\t\t\tname: \"local\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts {\n\t\t\tSYS {\n\t\t\t\tusers [{user: sys, password: pwd}]\n\t\t\t}\n\t\t\tONE {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers [{user: one, password: pwd}]\n\t\t\t}\n\t\t}\n\t\tsystem_account=SYS\n\t`\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"local\", 3)\n\tdefer c.shutdown()\n\n\tvar gwURLs string\n\tfor i, s := range c.servers {\n\t\tif i > 0 {\n\t\t\tgwURLs += \",\"\n\t\t}\n\t\tgwURLs += `\"nats://` + s.GatewayAddr().String() + `\"`\n\t}\n\n\ttmpl2 := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tgateway {\n\t\t\tname: \"remote\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\t__remote__\n\t\t}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts {\n\t\t\tSYS {\n\t\t\t\tusers [{user: sys, password: pwd}]\n\t\t\t}\n\t\t\tONE {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers [{user: one, password: pwd}]\n\t\t\t}\n\t\t}\n\t\tsystem_account=SYS\n\t`\n\tc2 := createJetStreamClusterAndModHook(t, tmpl2, \"remote\", \"R\", 2, 16022, false,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tconf = strings.Replace(conf, \"__remote__\", fmt.Sprintf(\"gateways [ { name: 'local', urls: [%s] } ]\", gwURLs), 1)\n\t\t\treturn conf\n\t\t})\n\tdefer c2.shutdown()\n\n\tfor _, s := range c.servers {\n\t\twaitForOutboundGateways(t, s, 1, 2*time.Second)\n\t}\n\tfor _, s := range c2.servers {\n\t\twaitForOutboundGateways(t, s, 1, 2*time.Second)\n\t}\n\tc.waitOnPeerCount(5)\n\n\t// Simulate going offline without sending shutdown protocol\n\tfor _, s := range c2.servers {\n\t\tc := s.getOutboundGatewayConnection(\"local\")\n\t\tc.setNoReconnect()\n\t\tc.mu.Lock()\n\t\tc.nc.Close()\n\t\tc.mu.Unlock()\n\t}\n\tc2.shutdown()\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tvar ok int\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, err := s.Jsz(nil)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, r := range jsz.Meta.Replicas {\n\t\t\t\tif r.Name == \"RS-1\" && r.Offline {\n\t\t\t\t\tok++\n\t\t\t\t} else if r.Name == \"RS-2\" && r.Offline {\n\t\t\t\t\tok++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif ok != 2 {\n\t\t\treturn fmt.Errorf(\"RS-1 or RS-2 still marked as online\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSuperClusterMovingR1Stream(t *testing.T) {\n\t// Make C2 have some latency.\n\tgwm := gwProxyMap{\n\t\t\"C2\": &gwProxy{\n\t\t\trtt: 10 * time.Millisecond,\n\t\t\tup: 1 * 1024 * 1024 * 1024, // 1gbit\n\t\t\tdown: 1 * 1024 * 1024 * 1024, // 1gbit\n\t\t},\n\t}\n\tsc := createJetStreamTaggedSuperClusterWithGWProxy(t, gwm)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.clusterForName(\"C1\").randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend := 10_000\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.PublishAsync(\"TEST\", []byte(\"HELLO WORLD\"))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Have it move to GCP.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:gcp\"}},\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster.Name != \"C2\" {\n\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t}\n\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t} else if !strings.HasPrefix(si.Cluster.Leader, \"C2\") {\n\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t}\n\t\t// Now we want to see that we shrink back to original.\n\t\tif len(si.Cluster.Replicas) != 0 {\n\t\t\treturn fmt.Errorf(\"Expected 0 replicas, got %d\", len(si.Cluster.Replicas))\n\t\t}\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Only see %d msgs\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/4396\nfunc TestJetStreamSuperClusterR1StreamPeerRemove(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 1, 3)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.serverByName(\"C1-S1\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Call peer remove on the only peer the leader.\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamRemovePeerT, \"TEST\"), []byte(`{\"peer\":\"`+si.Cluster.Leader+`\"}`), time.Second)\n\trequire_NoError(t, err)\n\tvar rpr JSApiStreamRemovePeerResponse\n\trequire_NoError(t, json.Unmarshal(resp.Data, &rpr))\n\trequire_False(t, rpr.Success)\n\trequire_True(t, rpr.Error.ErrCode == 10075)\n\n\t// Stream should still be in place and useable.\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamSuperClusterConsumerPauseAdvisories(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tpauseReq := func(consumer string, deadline time.Time) time.Time {\n\t\tj, err := json.Marshal(JSApiConsumerPauseRequest{\n\t\t\tPauseUntil: deadline,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiConsumerPauseT, \"TEST\", consumer), j, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar res JSApiConsumerPauseResponse\n\t\terr = json.Unmarshal(msg.Data, &res)\n\t\trequire_NoError(t, err)\n\t\treturn res.PauseUntil\n\t}\n\n\tcheckAdvisory := func(msg *nats.Msg, shouldBePaused bool, deadline time.Time) {\n\t\tt.Helper()\n\t\tvar advisory JSConsumerPauseAdvisory\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &advisory))\n\t\trequire_Equal(t, advisory.Stream, \"TEST\")\n\t\trequire_Equal(t, advisory.Consumer, \"my_consumer\")\n\t\trequire_Equal(t, advisory.Paused, shouldBePaused)\n\t\trequire_True(t, advisory.PauseUntil.Equal(deadline))\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tch := make(chan *nats.Msg, 10)\n\t_, err = nc.ChanSubscribe(JSAdvisoryConsumerPausePre+\".TEST.my_consumer\", ch)\n\trequire_NoError(t, err)\n\n\tdeadline := time.Now().Add(time.Second)\n\tjsTestPause_CreateOrUpdateConsumer(t, nc, ActionCreate, \"TEST\", ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tPauseUntil: &deadline,\n\t\tReplicas: 3,\n\t})\n\n\t// First advisory should tell us that the consumer was paused\n\t// on creation.\n\tmsg := require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// The second one for the unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Now we'll pause the consumer for a second using the API.\n\tdeadline = time.Now().Add(time.Second)\n\trequire_True(t, pauseReq(\"my_consumer\", deadline).Equal(deadline))\n\n\t// Third advisory should tell us about the pause via the API.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Finally that should unpause.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, false, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n\n\t// Now we're going to set the deadline into the future so we can\n\t// see what happens when we kick leaders or restart.\n\tdeadline = time.Now().Add(time.Hour)\n\trequire_True(t, pauseReq(\"my_consumer\", deadline).Equal(deadline))\n\n\t// Setting the deadline should have generated an advisory.\n\tmsg = require_ChanRead(t, ch, time.Second*2)\n\tcheckAdvisory(msg, true, deadline)\n\trequire_Len(t, len(ch), 0) // Should only receive one advisory.\n}\n\nfunc TestJetStreamSuperClusterConsumerAckSubjectWithStreamImportProtocolError(t *testing.T) {\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {domain: hub, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\taccounts {\n\t\t\tSYS {}\n\t\t\tone {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers [{user: one, password: password}]\n\t\t\t\texports [{stream: >}]\n\t\t\t}\n\t\t\ttwo {\n\t\t\t\tusers [{user: two, password: password}]\n\t\t\t\timports [{stream: {subject: >, account: one}}]\n\t\t\t}\n\t\t}\n\t\tsystem_account: SYS\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\tname: %s\n\t\t\troutes = [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t\tgateway {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\tgateways = [{name: %s, urls: [\"nats://127.0.0.1:%d\"]}]\n\t\t}\n\t\tleaf {listen: 127.0.0.1:-1}\n\t`\n\tstoreDir1 := t.TempDir()\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\t\"S1\", storeDir1, 23222, \"A\", 23222, \"A\", 11222, \"B\", 11223)))\n\ts1, _ := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tstoreDir2 := t.TempDir()\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\t\"S2\", storeDir2, 23223, \"B\", 23223, \"B\", 11223, \"A\", 11222)))\n\ts2, o2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\twaitForInboundGateways(t, s1, 1, 2*time.Second)\n\twaitForInboundGateways(t, s2, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\n\tmeta := s2.getJetStream().getMetaGroup()\n\trequire_NoError(t, meta.CampaignImmediately())\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif !s1.JetStreamIsLeader() && !s2.JetStreamIsLeader() {\n\t\t\treturn fmt.Errorf(\"neither server is leader\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tleafCfg := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes [ { url: \"nats://one:password@127.0.0.1:%d\" } ]\n\t\t}\n\t`, o2.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(leafCfg)\n\tdefer leaf.Shutdown()\n\n\tncl := clientConnectToServer(t, leaf)\n\tdefer ncl.Close()\n\tjs, err := ncl.JetStream(nats.Domain(\"hub\"))\n\trequire_NoError(t, err)\n\n\tcheckLeafNodeConnected(t, leaf)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"TEST\", Placement: &nats.Placement{Cluster: \"A\"}}, nats.MaxWait(200*time.Millisecond))\n\trequire_NoError(t, err)\n\n\tfor range 2 {\n\t\t_, err = js.Publish(\"TEST\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"TEST\", \"DURABLE\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tmsgs, err := sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmsg := msgs[0]\n\trequire_Equal(t, msg.Subject, \"TEST\")\n\trequire_NoError(t, msg.AckSync())\n\n\t// Since the JetStream ACK subject can encode the publish subject into the reply subject,\n\t// if we subscribe on a stream import this would previously result in a protocol error.\n\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://two:password@127.0.0.1:%d\", o2.Port))\n\tdefer nc2.Close()\n\tsub2, err := nc2.Subscribe(\">\", func(msg *nats.Msg) {})\n\trequire_NoError(t, err)\n\tdefer sub2.Drain()\n\trequire_NoError(t, nc2.Flush())\n\n\t// Ensure we can still receive messages. Would previously timeout due to the protocol error.\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\tmsg = msgs[0]\n\trequire_Equal(t, msg.Subject, \"TEST\")\n\trequire_NoError(t, msg.AckSync())\n}\n" }, { "path": "server/jetstream_test.go", "content": "// Copyright 2019-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\tcrand \"crypto/rand\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"math/big\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/server/sysmem\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nats.go/jetstream\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc TestJetStreamBasicNilConfig(t *testing.T) {\n\ts := RunRandClientPortServer(t)\n\tdefer s.Shutdown()\n\n\tif err := s.EnableJetStream(nil); err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\tif s.SystemAccount() == nil {\n\t\tt.Fatalf(\"Expected system account to be created automatically\")\n\t}\n\t// Grab our config since it was dynamically generated.\n\tconfig := s.JetStreamConfig()\n\tif config == nil {\n\t\tt.Fatalf(\"Expected non-nil config\")\n\t}\n\t// Check dynamic max memory.\n\thwMem := sysmem.Memory()\n\tif hwMem != 0 {\n\t\t// Check if memory being limited via GOMEMLIMIT if being set.\n\t\tif gml := debug.SetMemoryLimit(-1); gml != math.MaxInt64 {\n\t\t\thwMem = gml\n\t\t}\n\t\t// Make sure its about 75%\n\t\test := hwMem / 4 * 3\n\t\tif config.MaxMemory != est {\n\t\t\tt.Fatalf(\"Expected memory to be 80 percent of system memory, got %v vs %v\", config.MaxMemory, est)\n\t\t}\n\t}\n\t// Make sure it was created.\n\tstat, err := os.Stat(config.StoreDir)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected the store directory to be present, %v\", err)\n\t}\n\tif stat == nil || !stat.IsDir() {\n\t\tt.Fatalf(\"Expected a directory\")\n\t}\n}\n\nfunc RunBasicJetStreamServer(t testing.TB) *Server {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.JetStream = true\n\topts.StoreDir = t.TempDir()\n\treturn RunServer(&opts)\n}\n\nfunc RunJetStreamServerOnPort(port int, sd string) *Server {\n\topts := DefaultTestOptions\n\topts.Port = port\n\topts.JetStream = true\n\topts.StoreDir = filepath.Dir(sd)\n\treturn RunServer(&opts)\n}\n\nfunc clientConnectToServer(t *testing.T, s *Server) *nats.Conn {\n\tt.Helper()\n\tnc, err := nats.Connect(s.ClientURL(),\n\t\tnats.Name(\"JS-TEST\"),\n\t\tnats.ReconnectWait(5*time.Millisecond),\n\t\tnats.MaxReconnects(-1))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc clientConnectWithOldRequest(t *testing.T, s *Server) *nats.Conn {\n\tnc, err := nats.Connect(s.ClientURL(), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc TestJetStreamEnableAndDisableAccount(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Global in simple setup should be enabled already.\n\tif !s.GlobalAccount().JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected to have jetstream enabled on global account\")\n\t}\n\tif na := s.JetStreamNumAccounts(); na != 1 {\n\t\tt.Fatalf(\"Expected 1 account, got %d\", na)\n\t}\n\n\tif err := s.GlobalAccount().DisableJetStream(); err != nil {\n\t\tt.Fatalf(\"Did not expect error on disabling account: %v\", err)\n\t}\n\tif na := s.JetStreamNumAccounts(); na != 0 {\n\t\tt.Fatalf(\"Expected no accounts, got %d\", na)\n\t}\n\t// Make sure we unreserved resources.\n\tif rm, rd, err := s.JetStreamReservedResources(); err != nil {\n\t\tt.Fatalf(\"Unexpected error requesting jetstream reserved resources: %v\", err)\n\t} else if rm != 0 || rd != 0 {\n\t\tt.Fatalf(\"Expected reserved memory and store to be 0, got %v and %v\", friendlyBytes(rm), friendlyBytes(rd))\n\t}\n\n\tacc, _ := s.LookupOrRegisterAccount(\"$FOO\")\n\tif err := acc.EnableJetStream(nil, nil); err != nil {\n\t\tt.Fatalf(\"Did not expect error on enabling account: %v\", err)\n\t}\n\tif na := s.JetStreamNumAccounts(); na != 1 {\n\t\tt.Fatalf(\"Expected 1 account, got %d\", na)\n\t}\n\tif err := acc.DisableJetStream(); err != nil {\n\t\tt.Fatalf(\"Did not expect error on disabling account: %v\", err)\n\t}\n\tif na := s.JetStreamNumAccounts(); na != 0 {\n\t\tt.Fatalf(\"Expected no accounts, got %d\", na)\n\t}\n\t// We should get error if disabling something not enabled.\n\tacc, _ = s.LookupOrRegisterAccount(\"$BAR\")\n\tif err := acc.DisableJetStream(); err == nil {\n\t\tt.Fatalf(\"Expected error on disabling account that was not enabled\")\n\t}\n\t// Should get an error for trying to enable a non-registered account.\n\tacc = NewAccount(\"$BAZ\")\n\tif err := acc.EnableJetStream(nil, nil); err == nil {\n\t\tt.Fatalf(\"Expected error on enabling account that was not registered\")\n\t}\n}\n\nfunc TestJetStreamAddStream(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjs.Publish(\"foo\", []byte(\"Hello World!\"))\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 1 {\n\t\t\t\tt.Fatalf(\"Expected 1 message, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tif state.Bytes == 0 {\n\t\t\t\tt.Fatalf(\"Expected non-zero bytes\")\n\t\t\t}\n\n\t\t\tjs.Publish(\"foo\", []byte(\"Hello World Again!\"))\n\t\t\tstate = mset.state()\n\t\t\tif state.Msgs != 2 {\n\t\t\t\tt.Fatalf(\"Expected 2 messages, got %d\", state.Msgs)\n\t\t\t}\n\n\t\t\tif err := mset.delete(); err != nil {\n\t\t\t\tt.Fatalf(\"Got an error deleting the stream: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAddStreamDiscardNew(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tMaxMsgs: 10,\n\t\t\t\tMaxBytes: 4096,\n\t\t\t\tDiscard: DiscardNew,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tMaxMsgs: 10,\n\t\t\t\tMaxBytes: 4096,\n\t\t\t\tDiscard: DiscardNew,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsubj := \"foo\"\n\t\t\ttoSend := 10\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, subj, fmt.Sprintf(\"MSG: %d\", i+1))\n\t\t\t}\n\t\t\t// We expect this one to fail due to discard policy.\n\t\t\tresp, _ := nc.Request(subj, []byte(\"discard me\"), 100*time.Millisecond)\n\t\t\tif resp == nil {\n\t\t\t\tt.Fatalf(\"No response, possible timeout?\")\n\t\t\t}\n\t\t\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != \"maximum messages exceeded\" || pa.Stream != \"foo\" {\n\t\t\t\tt.Fatalf(\"Expected to get an error about maximum messages, got %q\", resp.Data)\n\t\t\t}\n\n\t\t\t// Now do bytes.\n\t\t\tmset.purge(nil)\n\n\t\t\tbig := make([]byte, 8192)\n\t\t\tresp, _ = nc.Request(subj, big, 100*time.Millisecond)\n\t\t\tif resp == nil {\n\t\t\t\tt.Fatalf(\"No response, possible timeout?\")\n\t\t\t}\n\t\t\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != \"maximum bytes exceeded\" || pa.Stream != \"foo\" {\n\t\t\t\tt.Fatalf(\"Expected to get an error about maximum bytes, got %q\", resp.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAutoTuneFSConfig(t *testing.T) {\n\ts := RunRandClientPortServer(t)\n\tdefer s.Shutdown()\n\n\tjsconfig := &JetStreamConfig{MaxMemory: -1, MaxStore: 128 * 1024 * 1024, StoreDir: t.TempDir()}\n\tif err := s.EnableJetStream(jsconfig); err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\n\tmaxMsgSize := int32(512)\n\tstreamConfig := func(name string, maxMsgs, maxBytes int64) *StreamConfig {\n\t\tt.Helper()\n\t\tcfg := &StreamConfig{Name: name, MaxMsgSize: maxMsgSize, Storage: FileStorage}\n\t\tif maxMsgs > 0 {\n\t\t\tcfg.MaxMsgs = maxMsgs\n\t\t}\n\t\tif maxBytes > 0 {\n\t\t\tcfg.MaxBytes = maxBytes\n\t\t}\n\t\treturn cfg\n\t}\n\n\tacc := s.GlobalAccount()\n\n\ttestBlkSize := func(name string, maxMsgs, maxBytes int64, expectedBlkSize uint64) {\n\t\tt.Helper()\n\t\tmset, err := acc.addStream(streamConfig(name, maxMsgs, maxBytes))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t}\n\t\tdefer mset.delete()\n\t\tfsCfg, err := mset.fileStoreConfig()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error retrieving file store: %v\", err)\n\t\t}\n\t\tif fsCfg.BlockSize != expectedBlkSize {\n\t\t\tt.Fatalf(\"Expected auto tuned block size to be %d, got %d\", expectedBlkSize, fsCfg.BlockSize)\n\t\t}\n\t}\n\n\t// Create a dummy stream, to ensure removing stream/account directories don't race.\n\t_, err := acc.addStream(streamConfig(\"dummy\", 1, 0))\n\trequire_NoError(t, err)\n\n\ttestBlkSize(\"foo\", 1, 0, FileStoreMinBlkSize)\n\ttestBlkSize(\"foo\", 1, 512, FileStoreMinBlkSize)\n\ttestBlkSize(\"foo\", 1, 1024*1024, defaultMediumBlockSize)\n\ttestBlkSize(\"foo\", 1, 8*1024*1024, defaultMediumBlockSize)\n\ttestBlkSize(\"foo_bar_baz\", -1, 32*1024*1024, FileStoreMaxBlkSize)\n}\n\nfunc TestJetStreamPubAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsname := \"PUBACK\"\n\tacc := s.GlobalAccount()\n\tmconfig := &StreamConfig{Name: sname, Subjects: []string{\"foo\"}, Storage: MemoryStorage}\n\tmset, err := acc.addStream(mconfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcheckRespDetails := func(resp *nats.Msg, err error, seq uint64) {\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error from send stream msg: %v\", err)\n\t\t}\n\t\tif resp == nil {\n\t\t\tt.Fatalf(\"No response from send stream msg\")\n\t\t}\n\t\tpa := getPubAckResponse(resp.Data)\n\t\tif pa == nil || pa.Error != nil {\n\t\t\tt.Fatalf(\"Expected a valid JetStreamPubAck, got %q\", resp.Data)\n\t\t}\n\t\tif pa.Stream != sname {\n\t\t\tt.Fatalf(\"Expected %q for stream name, got %q\", sname, pa.Stream)\n\t\t}\n\t\tif pa.Sequence != seq {\n\t\t\tt.Fatalf(\"Expected %d for sequence, got %d\", seq, pa.Sequence)\n\t\t}\n\t}\n\n\t// Send messages and make sure pubAck details are correct.\n\tfor i := uint64(1); i <= 1000; i++ {\n\t\tresp, err := nc.Request(\"foo\", []byte(\"HELLO\"), 100*time.Millisecond)\n\t\tcheckRespDetails(resp, err, i)\n\t}\n}\n\nfunc TestJetStreamNextReqFromMsg(t *testing.T) {\n\tbef := time.Now()\n\texpires, _, _, _, _, _, _, err := nextReqFromMsg([]byte(`{\"expires\":5000000000}`)) // nanoseconds\n\trequire_NoError(t, err)\n\tnow := time.Now()\n\tif expires.Before(bef.Add(5*time.Second)) || expires.After(now.Add(5*time.Second)) {\n\t\tt.Fatal(\"Expires out of expected range\")\n\t}\n}\n\nfunc TestJetStreamNoPanicOnRaceBetweenShutdownAndConsumerDelete(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_STREAM\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_STREAM\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tvar cons []*consumer\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\t\tDurable: fmt.Sprintf(\"d%d\", i),\n\t\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer o.delete()\n\t\t\t\tcons = append(cons, o)\n\t\t\t}\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tfor _, c := range cons {\n\t\t\t\t\tc.delete()\n\t\t\t\t}\n\t\t\t}()\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\ts.Shutdown()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAddStreamMaxMsgSize(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tMaxMsgSize: 22,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tMaxMsgSize: 22,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tif _, err := nc.Request(\"foo\", []byte(\"Hello World!\"), time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\ttooBig := []byte(\"1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ\")\n\t\t\tresp, err := nc.Request(\"foo\", tooBig, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error.Description != \"message size exceeds maximum allowed\" {\n\t\t\t\tt.Fatalf(\"Expected to get an error for maximum message size, got %q\", pa.Error)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAddStreamCanonicalNames(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\texpectErr := func(_ *stream, err error) {\n\t\tt.Helper()\n\t\tif !IsNatsErr(err, JSStreamInvalidConfigF) {\n\t\t\tt.Fatalf(\"Expected error but got none\")\n\t\t}\n\t}\n\n\texpectErr(acc.addStream(&StreamConfig{Name: \"foo.bar\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"foo.bar.\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"foo.*\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"foo.>\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"*\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \">\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"*>\"}))\n}\n\nfunc TestJetStreamAddStreamBadSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\texpectAPIErr := func(cfg StreamConfig) {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\t\tresp, _ := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\tvar scResp JSApiStreamCreateResponse\n\t\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\trequire_Error(t, scResp.ToError(), NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid subject\")))\n\t}\n\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\"foo.bar.\"}})\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\"..\"}})\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\".*\"}})\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\".>\"}})\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\" x\"}})\n\texpectAPIErr(StreamConfig{Name: \"MyStream\", Storage: MemoryStorage, Subjects: []string{\"y \"}})\n}\n\nfunc TestJetStreamMaxConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"MAXC\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"in.maxc.>\"},\n\t\tMaxConsumers: 2,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsi, err := js.StreamInfo(\"MAXC\")\n\trequire_NoError(t, err)\n\tif si.Config.MaxConsumers != 2 {\n\t\tt.Fatalf(\"Expected max of 2, got %d\", si.Config.MaxConsumers)\n\t}\n\t// Make sure we get the right error.\n\t// This should succeed.\n\tif _, err := js.PullSubscribe(\"in.maxc.foo\", \"maxc_foo\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Create for the same consumer must be idempotent, and not trigger limit.\n\tif _, err := js.PullSubscribe(\"in.maxc.foo\", \"maxc_foo\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create (with explicit create API) for the same consumer must be idempotent, and not trigger limit.\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"MAXC\",\n\t\tConfig: ConsumerConfig{Durable: \"maxc_baz\"},\n\t\tAction: ActionCreate,\n\t}\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"MAXC\", \"maxc_baz\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", resp.Error)\n\t}\n\n\tmsg, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"MAXC\", \"maxc_baz\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp2 JSApiConsumerInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &resp2))\n\tif resp2.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", resp2.Error)\n\t}\n\n\t// Exceeds limit.\n\tif _, err := js.SubscribeSync(\"in.maxc.bar\"); err == nil {\n\t\tt.Fatalf(\"Expected error but got none\")\n\t}\n}\n\nfunc TestJetStreamAddStreamOverlappingSubjects(t *testing.T) {\n\tmconfig := &StreamConfig{\n\t\tName: \"ok\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\", \"foo.bar.baz.>\"},\n\t}\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\tmset, err := acc.addStream(mconfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\texpectErr := func(_ *stream, err error) {\n\t\tt.Helper()\n\t\tif err == nil || !strings.Contains(err.Error(), \"subjects overlap\") {\n\t\t\tt.Fatalf(\"Expected error but got none\")\n\t\t}\n\t}\n\n\t// Test that any overlapping subjects will fail.\n\texpectErr(acc.addStream(&StreamConfig{Name: \"foo\"}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"a\", Subjects: []string{\"baz\", \"bar\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"b\", Subjects: []string{\">\"}, NoAck: true}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"c\", Subjects: []string{\"baz.33\"}}))\n\n\t// Using NoAck on the following because technically they overlap with the $JS.> namespace...\n\texpectErr(acc.addStream(&StreamConfig{Name: \"d\", Subjects: []string{\"*.33\"}, NoAck: true}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"e\", Subjects: []string{\"*.>\"}, NoAck: true}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"f\", Subjects: []string{\"foo.bar\", \"*.bar.>\"}, NoAck: true}))\n}\n\nfunc TestJetStreamAddStreamOverlapWithJSAPISubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\texpectNoErr := func(_ *stream, err error) {\n\t\tt.Helper()\n\t\tswitch {\n\t\tcase err == nil:\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\texpectErr := func(_ *stream, err error) {\n\t\tt.Helper()\n\t\tswitch {\n\t\tcase err == nil:\n\t\t\tt.Errorf(\"Expected error but got none\")\n\t\tcase !strings.Contains(err.Error(), \"subjects that overlap with jetstream api\"):\n\t\tcase !strings.Contains(err.Error(), \"subjects that overlap with system api\"):\n\t\tcase !strings.Contains(err.Error(), \"capturing all subjects requires no-ack to be true\"):\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Test that any overlapping subjects with our JSAPI should fail.\n\texpectErr(acc.addStream(&StreamConfig{Name: \"a\", Subjects: []string{\"$JS.API.foo\", \"$JS.API.bar\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"b\", Subjects: []string{\"$JS.API.>\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"c\", Subjects: []string{\"$JS.API.*\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"d\", Subjects: []string{\"$JS.>\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"e\", Subjects: []string{\"$SYS.>\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"f\", Subjects: []string{\"*.>\"}}))\n\texpectErr(acc.addStream(&StreamConfig{Name: \"g\", Subjects: []string{\">\"}}))\n\n\t// Events and advisories should be OK.\n\texpectNoErr(acc.addStream(&StreamConfig{Name: \"h\", Subjects: []string{\"$JS.EVENT.>\"}}))\n\texpectNoErr(acc.addStream(&StreamConfig{Name: \"i\", Subjects: []string{\"$SYS.ACCOUNT.>\"}}))\n\n\t// So should a full wild-card with NoAck, but need to clean up overlapping streams first.\n\tfor _, name := range []string{\"h\", \"i\"} {\n\t\tmset, err := acc.lookupStream(name)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, mset.delete())\n\t}\n\texpectNoErr(acc.addStream(&StreamConfig{Name: \"j\", Subjects: []string{\">\"}, NoAck: true}))\n}\n\nfunc TestJetStreamAddStreamSameConfigOK(t *testing.T) {\n\tmconfig := &StreamConfig{\n\t\tName: \"ok\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\", \"foo.bar.baz.>\"},\n\t\tStorage: MemoryStorage,\n\t}\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\tmset, err := acc.addStream(mconfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// Adding again with same config should be idempotent.\n\tif _, err = acc.addStream(mconfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n}\n\nfunc sendStreamMsg(t *testing.T, nc *nats.Conn, subject, msg string) *PubAck {\n\tt.Helper()\n\tresp, err := nc.Request(subject, []byte(msg), 500*time.Millisecond)\n\tif resp == nil {\n\t\tt.Fatalf(\"No response for %q (error: %v)\", msg, err)\n\t}\n\tpa := getPubAckResponse(resp.Data)\n\tif pa == nil || pa.Error != nil {\n\t\tt.Fatalf(\"Expected a valid JetStreamPubAck, got %q\", resp.Data)\n\t}\n\treturn pa.PubAck\n}\n\nfunc TestJetStreamBasicAckPublish(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"foo\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"foo\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfor i := 0; i < 50; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 50 {\n\t\t\t\tt.Fatalf(\"Expected 50 messages, got %d\", state.Msgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStateTimestamps(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"foo\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"foo\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tstart := time.Now()\n\t\t\tdelay := 250 * time.Millisecond\n\t\t\tsendStreamMsg(t, nc, \"foo.bar\", \"Hello World!\")\n\t\t\ttime.Sleep(delay)\n\t\t\tsendStreamMsg(t, nc, \"foo.bar\", \"Hello World Again!\")\n\n\t\t\tstate := mset.state()\n\t\t\tif state.FirstTime.Before(start) {\n\t\t\t\tt.Fatalf(\"Unexpected first message timestamp: %v\", state.FirstTime)\n\t\t\t}\n\t\t\tif state.LastTime.Before(start.Add(delay)) {\n\t\t\t\tt.Fatalf(\"Unexpected last message timestamp: %v\", state.LastTime)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamNoAckStream(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"foo\", Storage: MemoryStorage, NoAck: true}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"foo\", Storage: FileStorage, NoAck: true}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// We can use NoAck to suppress acks even when reply subjects are present.\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tif _, err := nc.Request(\"foo\", []byte(\"Hello World!\"), 25*time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected a timeout error and no response with acks suppressed\")\n\t\t\t}\n\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 1 {\n\t\t\t\tt.Fatalf(\"Expected 1 message, got %d\", state.Msgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamBasicDeliverSubject(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MSET\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MSET\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 100\n\t\t\tsendSubj := \"foo.bar\"\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, strconv.Itoa(i))\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\t// Now create an consumer. Use different connection.\n\t\t\tnc2 := clientConnectToServer(t, s)\n\t\t\tdefer nc2.Close()\n\n\t\t\tsub, _ := nc2.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc2.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Check for our messages.\n\t\t\tcheckMsgs := func(seqOff int) {\n\t\t\t\tt.Helper()\n\n\t\t\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\t// Now let's check the messages\n\t\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\t\tm, _ := sub.NextMsg(time.Second)\n\t\t\t\t\t// JetStream will have the subject match the stream subject, not delivery subject.\n\t\t\t\t\tif m.Subject != sendSubj {\n\t\t\t\t\t\tt.Fatalf(\"Expected original subject of %q, but got %q\", sendSubj, m.Subject)\n\t\t\t\t\t}\n\t\t\t\t\t// Now check that reply subject exists and has a sequence as the last token.\n\t\t\t\t\tif seq := o.seqFromReply(m.Reply); seq != uint64(i+seqOff) {\n\t\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", i+seqOff, seq)\n\t\t\t\t\t}\n\t\t\t\t\t// Ack the message here.\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheckMsgs(1)\n\n\t\t\t// Now send more and make sure delivery picks back up.\n\t\t\tfor i := toSend + 1; i <= toSend*2; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, strconv.Itoa(i))\n\t\t\t}\n\t\t\tstate = mset.state()\n\t\t\tif state.Msgs != uint64(toSend*2) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*2, state.Msgs)\n\t\t\t}\n\n\t\t\tcheckMsgs(101)\n\n\t\t\tcheckSubEmpty := func() {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected sub to have no pending\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckSubEmpty()\n\t\t\to.delete()\n\n\t\t\t// Now check for deliver last, deliver new and deliver by seq.\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverPolicy: DeliverLast})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Did not get expected message, got %v\", err)\n\t\t\t}\n\t\t\t// All Consumers start with sequence #1.\n\t\t\tif seq := o.seqFromReply(m.Reply); seq != 1 {\n\t\t\t\tt.Fatalf(\"Expected sequence to be 1, but got %d\", seq)\n\t\t\t}\n\t\t\t// Check that is the last msg we sent though.\n\t\t\tif mseq, _ := strconv.Atoi(string(m.Data)); mseq != 200 {\n\t\t\t\tt.Fatalf(\"Expected message sequence to be 200, but got %d\", mseq)\n\t\t\t}\n\n\t\t\tcheckSubEmpty()\n\t\t\to.delete()\n\n\t\t\t// Make sure we only got one message.\n\t\t\tif m, err := sub.NextMsg(5 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Expected no msg, got %+v\", m)\n\t\t\t}\n\n\t\t\tcheckSubEmpty()\n\t\t\to.delete()\n\n\t\t\t// Now try by sequence number.\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverPolicy: DeliverByStartSequence, OptStartSeq: 101})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tcheckMsgs(1)\n\n\t\t\t// Now do push based queue-subscribers\n\t\t\tsub, _ = nc2.QueueSubscribeSync(\"_qg_\", \"dev\")\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc2.Flush()\n\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject, DeliverGroup: \"dev\"})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Since we sent another batch need check to be looking for 2x.\n\t\t\ttoSend *= 2\n\t\t\tcheckMsgs(1)\n\t\t})\n\t}\n}\n\nfunc workerModeConfig(name string) *ConsumerConfig {\n\treturn &ConsumerConfig{Durable: name, AckPolicy: AckExplicit}\n}\n\nfunc TestJetStreamBasicWorkQueue(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Create basic work queue mode consumer.\n\t\t\toname := \"WQ\"\n\t\t\to, err := mset.addConsumer(workerModeConfig(oname))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tif o.nextSeq() != 1 {\n\t\t\t\tt.Fatalf(\"Expected to be starting at sequence 1\")\n\t\t\t}\n\n\t\t\tnc := clientConnectWithOldRequest(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tsendSubj := \"bar\"\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tgetNext := func(seqno int) {\n\t\t\t\tt.Helper()\n\t\t\t\tnextMsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error for seq %d: %v\", seqno, err)\n\t\t\t\t}\n\t\t\t\tif nextMsg.Subject != \"bar\" {\n\t\t\t\t\tt.Fatalf(\"Expected subject of %q, got %q\", \"bar\", nextMsg.Subject)\n\t\t\t\t}\n\t\t\t\tif seq := o.seqFromReply(nextMsg.Reply); seq != uint64(seqno) {\n\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", seqno, seq)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Make sure we can get the messages already there.\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tgetNext(i)\n\t\t\t}\n\n\t\t\t// Now we want to make sure we can get a message that is published to the message\n\t\t\t// set as we are waiting for it.\n\t\t\tnextDelay := 50 * time.Millisecond\n\n\t\t\tgo func() {\n\t\t\t\ttime.Sleep(nextDelay)\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}()\n\n\t\t\tstart := time.Now()\n\t\t\tgetNext(toSend + 1)\n\t\t\tif time.Since(start) < nextDelay {\n\t\t\t\tt.Fatalf(\"Received message too quickly\")\n\t\t\t}\n\n\t\t\t// Now do same thing but combine waiting for new ones with sending.\n\t\t\tgo func() {\n\t\t\t\ttime.Sleep(nextDelay)\n\t\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\t\tnc.Request(sendSubj, []byte(\"Hello World!\"), 50*time.Millisecond)\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tfor i := toSend + 2; i < toSend*2+2; i++ {\n\t\t\t\tgetNext(i)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueMaxWaiting(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Make sure these cases fail\n\t\t\tcfg := &ConsumerConfig{Durable: \"foo\", AckPolicy: AckExplicit, MaxWaiting: 10, DeliverSubject: \"_INBOX.22\"}\n\t\t\tif _, err := mset.addConsumer(cfg); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error with MaxWaiting set on non-pull based consumer\")\n\t\t\t}\n\n\t\t\t// Create basic work queue mode consumer.\n\t\t\twcfg := workerModeConfig(\"MAXWQ\")\n\t\t\to, err := mset.addConsumer(wcfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Make sure we set default correctly.\n\t\t\tif cfg := o.config(); cfg.MaxWaiting != JSWaitQueueDefaultMax {\n\t\t\t\tt.Fatalf(\"Expected default max waiting to have been set to %d, got %d\", JSWaitQueueDefaultMax, cfg.MaxWaiting)\n\t\t\t}\n\n\t\t\texpectWaiting := func(expected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\t\tif oi := o.info(); oi.NumWaiting != expected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d waiting, got %d\", expected, oi.NumWaiting)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tnc := clientConnectWithOldRequest(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Like muxed new INBOX.\n\t\t\tsub, _ := nc.SubscribeSync(\"req.*\")\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\tcheckSubPending := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tgetSubj := o.requestNextMsgSubject()\n\t\t\t// Queue up JSWaitQueueDefaultMax requests.\n\t\t\tfor i := 0; i < JSWaitQueueDefaultMax; i++ {\n\t\t\t\tnc.PublishRequest(getSubj, fmt.Sprintf(\"req.%d\", i), nil)\n\t\t\t}\n\t\t\texpectWaiting(JSWaitQueueDefaultMax)\n\n\t\t\t// We are at the max, so we should get a 409 saying that we have\n\t\t\t// exceeded the number of pull requests.\n\t\t\tm, err := nc.Request(getSubj, nil, 100*time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\t// Make sure this is the 409\n\t\t\tif v := m.Header.Get(\"Status\"); v != \"409\" {\n\t\t\t\tt.Fatalf(\"Expected a 409 status code, got %q\", v)\n\t\t\t}\n\t\t\t// The sub for the other requests should not have received anything\n\t\t\tcheckSubPending(0)\n\t\t\t// Now send some messages that should make some of the requests complete\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\t\t\tsendStreamMsg(t, nc, \"bar\", \"Hello World!\")\n\t\t\texpectWaiting(JSWaitQueueDefaultMax - 2)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueWrapWaiting(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tmaxWaiting := 8\n\t\t\twcfg := workerModeConfig(\"WRAP\")\n\t\t\twcfg.MaxWaiting = maxWaiting\n\n\t\t\to, err := mset.addConsumer(wcfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tgetSubj := o.requestNextMsgSubject()\n\n\t\t\texpectWaiting := func(expected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\t\tif oi := o.info(); oi.NumWaiting != expected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d waiting, got %d\", expected, oi.NumWaiting)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsub, _ := nc.SubscribeSync(\"req.*\")\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// Fill up waiting.\n\t\t\tfor i := 0; i < maxWaiting; i++ {\n\t\t\t\tnc.PublishRequest(getSubj, fmt.Sprintf(\"req.%d\", i), nil)\n\t\t\t}\n\t\t\texpectWaiting(maxWaiting)\n\n\t\t\t// Now use 1/2 of the waiting.\n\t\t\tfor i := 0; i < maxWaiting/2; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\t\t\t}\n\t\t\texpectWaiting(maxWaiting / 2)\n\n\t\t\t// Now add in two (2) more pull requests.\n\t\t\tfor i := maxWaiting; i < maxWaiting+2; i++ {\n\t\t\t\tnc.PublishRequest(getSubj, fmt.Sprintf(\"req.%d\", i), nil)\n\t\t\t}\n\t\t\texpectWaiting(maxWaiting/2 + 2)\n\n\t\t\t// Now use second 1/2 of the waiting and the 2 extra.\n\t\t\tfor i := 0; i < maxWaiting/2+2; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"bar\", \"Hello World!\")\n\t\t\t}\n\t\t\texpectWaiting(0)\n\n\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != maxWaiting+2 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected sub to have %d pending, got %d\", maxWaiting+2, nmsgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueRequest(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WRAP\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 25\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"bar\", \"Hello World!\")\n\t\t\t}\n\n\t\t\treply := \"_.consumer._\"\n\t\t\tsub, _ := nc.SubscribeSync(reply)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tgetSubj := o.requestNextMsgSubject()\n\n\t\t\tcheckSubPending := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Create a formal request object.\n\t\t\treq := &JSApiConsumerGetNextRequest{Batch: toSend}\n\t\t\tjreq, _ := json.Marshal(req)\n\t\t\tnc.PublishRequest(getSubj, reply, jreq)\n\n\t\t\tcheckSubPending(toSend)\n\n\t\t\t// Now check that we can ask for NoWait\n\t\t\treq.Batch = 1\n\t\t\treq.NoWait = true\n\t\t\tjreq, _ = json.Marshal(req)\n\n\t\t\tresp, err := nc.Request(getSubj, jreq, 100*time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\tif status := resp.Header.Get(\"Status\"); !strings.HasPrefix(status, \"404\") {\n\t\t\t\tt.Fatalf(\"Expected status code of 404\")\n\t\t\t}\n\t\t\t// Load up more messages.\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\t\t\t}\n\t\t\t// Now we will ask for a batch larger then what is queued up.\n\t\t\treq.Batch = toSend + 10\n\t\t\treq.NoWait = true\n\t\t\tjreq, _ = json.Marshal(req)\n\n\t\t\tnc.PublishRequest(getSubj, reply, jreq)\n\t\t\t// We should now have 2 * toSend + the 404 message.\n\t\t\tcheckSubPending(2*toSend + 1)\n\t\t\tfor i := 0; i < 2*toSend+1; i++ {\n\t\t\t\tsub.NextMsg(time.Millisecond)\n\t\t\t}\n\t\t\tcheckSubPending(0)\n\t\t\tmset.purge(nil)\n\n\t\t\t// Now do expiration\n\t\t\treq.Batch = 1\n\t\t\treq.NoWait = false\n\t\t\treq.Expires = 100 * time.Millisecond\n\t\t\tjreq, _ = json.Marshal(req)\n\n\t\t\tnc.PublishRequest(getSubj, reply, jreq)\n\t\t\t// Let it expire\n\t\t\ttime.Sleep(200 * time.Millisecond)\n\n\t\t\t// Send a few more messages. These should not be delivered to the sub.\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\t\t\tsendStreamMsg(t, nc, \"bar\", \"Hello World!\")\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\t// Expect the request timed out message.\n\t\t\tcheckSubPending(1)\n\t\t\tif resp, _ = sub.NextMsg(time.Millisecond); resp == nil {\n\t\t\t\tt.Fatalf(\"Expected an expired status message\")\n\t\t\t}\n\t\t\tif status := resp.Header.Get(\"Status\"); !strings.HasPrefix(status, \"408\") {\n\t\t\t\tt.Fatalf(\"Expected status code of 408\")\n\t\t\t}\n\n\t\t\t// Send a new request, we should not get the 408 because our previous request\n\t\t\t// should have expired.\n\t\t\tnc.PublishRequest(getSubj, reply, jreq)\n\t\t\tcheckSubPending(1)\n\t\t\tsub.NextMsg(time.Second)\n\t\t\tcheckSubPending(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSubjectFiltering(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MSET\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MSET\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 50\n\t\t\tsubjA := \"foo.A\"\n\t\t\tsubjB := \"foo.B\"\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, subjA, \"Hello World!\")\n\t\t\t\tsendStreamMsg(t, nc, subjB, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend*2) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*2, state.Msgs)\n\t\t\t}\n\n\t\t\tdelivery := nats.NewInbox()\n\t\t\tsub, _ := nc.SubscribeSync(delivery)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: subjB})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Now let's check the messages\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// JetStream will have the subject match the stream subject, not delivery subject.\n\t\t\t\t// We want these to only be subjB.\n\t\t\t\tif m.Subject != subjB {\n\t\t\t\t\tt.Fatalf(\"Expected original subject of %q, but got %q\", subjB, m.Subject)\n\t\t\t\t}\n\t\t\t\t// Now check that reply subject exists and has a sequence as the last token.\n\t\t\t\tif seq := o.seqFromReply(m.Reply); seq != uint64(i) {\n\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", i, seq)\n\t\t\t\t}\n\t\t\t\t// Ack the message here.\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected sub to have no pending\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueSubjectFiltering(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 50\n\t\t\tsubjA := \"foo.A\"\n\t\t\tsubjB := \"foo.B\"\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, subjA, \"Hello World!\")\n\t\t\t\tsendStreamMsg(t, nc, subjB, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend*2) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*2, state.Msgs)\n\t\t\t}\n\n\t\t\toname := \"WQ\"\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: oname, FilterSubject: subjA, AckPolicy: AckExplicit})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tif o.nextSeq() != 1 {\n\t\t\t\tt.Fatalf(\"Expected to be starting at sequence 1\")\n\t\t\t}\n\n\t\t\tgetNext := func(seqno int) {\n\t\t\t\tt.Helper()\n\t\t\t\tnextMsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif nextMsg.Subject != subjA {\n\t\t\t\t\tt.Fatalf(\"Expected subject of %q, got %q\", subjA, nextMsg.Subject)\n\t\t\t\t}\n\t\t\t\tif seq := o.seqFromReply(nextMsg.Reply); seq != uint64(seqno) {\n\t\t\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", seqno, seq)\n\t\t\t\t}\n\t\t\t\tnextMsg.Respond(nil)\n\t\t\t}\n\n\t\t\t// Make sure we can get the messages already there.\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tgetNext(i)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWildcardSubjectFiltering(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"ORDERS\", Storage: MemoryStorage, Subjects: []string{\"orders.*.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"ORDERS\", Storage: FileStorage, Subjects: []string{\"orders.*.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\ttoSend := 100\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"orders.%d.%s\", i, \"NEW\")\n\t\t\t\tsendStreamMsg(t, nc, subj, \"new order\")\n\t\t\t}\n\t\t\t// Randomly move 25 to shipped.\n\t\t\ttoShip := 25\n\t\t\tshipped := make(map[int]bool)\n\t\t\tfor i := 0; i < toShip; {\n\t\t\t\torderId := rand.Intn(toSend-1) + 1\n\t\t\t\tif shipped[orderId] {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsubj := fmt.Sprintf(\"orders.%d.%s\", orderId, \"SHIPPED\")\n\t\t\t\tsendStreamMsg(t, nc, subj, \"shipped order\")\n\t\t\t\tshipped[orderId] = true\n\t\t\t\ti++\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend+toShip) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend+toShip, state.Msgs)\n\t\t\t}\n\n\t\t\tdelivery := nats.NewInbox()\n\t\t\tsub, _ := nc.SubscribeSync(delivery)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// Get all shipped.\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: \"orders.*.SHIPPED\"})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toShip {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toShip)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tfor nmsgs, _, _ := sub.Pending(); nmsgs > 0; nmsgs, _, _ = sub.Pending() {\n\t\t\t\tsub.NextMsg(time.Second)\n\t\t\t}\n\t\t\tif nmsgs, _, _ := sub.Pending(); nmsgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d\", nmsgs)\n\t\t\t}\n\n\t\t\t// Get all new\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: \"orders.*.NEW\"})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tfor nmsgs, _, _ := sub.Pending(); nmsgs > 0; nmsgs, _, _ = sub.Pending() {\n\t\t\t\tsub.NextMsg(time.Second)\n\t\t\t}\n\t\t\tif nmsgs, _, _ := sub.Pending(); nmsgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d\", nmsgs)\n\t\t\t}\n\n\t\t\t// Now grab a single orderId that has shipped, so we should have two messages.\n\t\t\tvar orderId int\n\t\t\tfor orderId = range shipped {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tsubj := fmt.Sprintf(\"orders.%d.*\", orderId)\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: subj})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, 2)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueAckAndNext(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Create basic work queue mode consumer.\n\t\t\toname := \"WQ\"\n\t\t\to, err := mset.addConsumer(workerModeConfig(oname))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tif o.nextSeq() != 1 {\n\t\t\t\tt.Fatalf(\"Expected to be starting at sequence 1\")\n\t\t\t}\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tsendSubj := \"bar\"\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// Kick things off.\n\t\t\t// For normal work queue semantics, you send requests to the subject with stream and consumer name.\n\t\t\t// We will do this to start it off then use ack+next to get other messages.\n\t\t\tnc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, nil)\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error waiting for messages: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tif !bytes.Equal(m.Data, []byte(\"Hello World!\")) {\n\t\t\t\t\tt.Fatalf(\"Got an invalid message from the stream: %q\", m.Data)\n\t\t\t\t}\n\n\t\t\t\tnc.PublishRequest(m.Reply, sub.Subject, AckNext)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueRequestBatch(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: MemoryStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_MSG_SET\", Storage: FileStorage, Subjects: []string{\"foo\", \"bar\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Create basic work queue mode consumer.\n\t\t\toname := \"WQ\"\n\t\t\to, err := mset.addConsumer(workerModeConfig(oname))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tif o.nextSeq() != 1 {\n\t\t\t\tt.Fatalf(\"Expected to be starting at sequence 1\")\n\t\t\t}\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tsendSubj := \"bar\"\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// For normal work queue semantics, you send requests to the subject with stream and consumer name.\n\t\t\t// We will do this to start it off then use ack+next to get other messages.\n\t\t\t// Kick things off with batch size of 50.\n\t\t\tbatchSize := 50\n\t\t\tnc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte(strconv.Itoa(batchSize)))\n\n\t\t\t// We should receive batchSize with no acks or additional requests.\n\t\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != batchSize {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, batchSize)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now queue up the request without messages and add them after.\n\t\t\tsub, _ = nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tmset.purge(nil)\n\n\t\t\tnc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte(strconv.Itoa(batchSize)))\n\t\t\tnc.Flush() // Make sure its registered.\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}\n\n\t\t\t// We should receive batchSize with no acks or additional requests.\n\t\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != batchSize {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, batchSize)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueRetentionStream(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\", mconfig: &StreamConfig{\n\t\t\tName: \"MWQ\",\n\t\t\tStorage: MemoryStorage,\n\t\t\tSubjects: []string{\"MY_WORK_QUEUE.>\"},\n\t\t\tRetention: WorkQueuePolicy},\n\t\t},\n\t\t{name: \"FileStore\", mconfig: &StreamConfig{\n\t\t\tName: \"MWQ\",\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: []string{\"MY_WORK_QUEUE.>\"},\n\t\t\tRetention: WorkQueuePolicy},\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// This type of stream has restrictions which we will test here.\n\t\t\t// DeliverAll is only start mode allowed.\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{DeliverPolicy: DeliverLast}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error with anything but DeliverAll\")\n\t\t\t}\n\n\t\t\t// We will create a non-partitioned consumer. This should succeed.\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\t// Now if we create another this should fail, only can have one non-partitioned.\n\t\t\tif _, err := mset.addConsumer(&ConsumerConfig{}); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on attempt for second consumer for a workqueue\")\n\t\t\t}\n\t\t\to.delete()\n\n\t\t\tif numo := mset.numConsumers(); numo != 0 {\n\t\t\t\tt.Fatalf(\"Expected to have zero consumers, got %d\", numo)\n\t\t\t}\n\n\t\t\t// Now add in an consumer that has a partition.\n\t\t\tpindex := 1\n\t\t\tpConfig := func(pname string) *ConsumerConfig {\n\t\t\t\tdname := fmt.Sprintf(\"PPBO-%d\", pindex)\n\t\t\t\tpindex += 1\n\t\t\t\treturn &ConsumerConfig{Durable: dname, FilterSubject: pname, AckPolicy: AckExplicit}\n\t\t\t}\n\t\t\to, err = mset.addConsumer(pConfig(\"MY_WORK_QUEUE.A\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\t// Now creating another with separate partition should work.\n\t\t\to2, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.B\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o2.delete()\n\n\t\t\t// Anything that would overlap should fail though.\n\t\t\tif _, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.A\")); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on attempt for partitioned consumer for a workqueue\")\n\t\t\t}\n\t\t\tif _, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.B\")); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on attempt for partitioned consumer for a workqueue\")\n\t\t\t}\n\n\t\t\to3, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.C\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\to.delete()\n\t\t\to2.delete()\n\t\t\to3.delete()\n\n\t\t\t// Test with wildcards, first from wider to narrower\n\t\t\to, err = mset.addConsumer(pConfig(\"MY_WORK_QUEUE.>\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tif _, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.*.BAR\")); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on attempt for partitioned consumer for a workqueue\")\n\t\t\t}\n\t\t\to.delete()\n\n\t\t\t// Now from narrower to wider\n\t\t\to, err = mset.addConsumer(pConfig(\"MY_WORK_QUEUE.*.BAR\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tif _, err := mset.addConsumer(pConfig(\"MY_WORK_QUEUE.>\")); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error on attempt for partitioned consumer for a workqueue\")\n\t\t\t}\n\t\t\to.delete()\n\n\t\t\t// Push based will be allowed now, including ephemerals.\n\t\t\t// They can not overlap etc meaning same rules as above apply.\n\t\t\to4, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"DURABLE\",\n\t\t\t\tDeliverSubject: \"SOME.SUBJ\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected Error: %v\", err)\n\t\t\t}\n\t\t\tdefer o4.delete()\n\n\t\t\t// Now try to create an ephemeral\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\t// This should fail at first due to conflict above.\n\t\t\tephCfg := &ConsumerConfig{DeliverSubject: sub.Subject, AckPolicy: AckExplicit}\n\t\t\tif _, err := mset.addConsumer(ephCfg); err == nil {\n\t\t\t\tt.Fatalf(\"Expected an error \")\n\t\t\t}\n\t\t\t// Delete of o4 should clear.\n\t\t\to4.delete()\n\t\t\to5, err := mset.addConsumer(ephCfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected Error: %v\", err)\n\t\t\t}\n\t\t\tdefer o5.delete()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAckAllRedelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_S22\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_S22\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDeliverSubject: sub.Subject,\n\t\t\t\tAckWait: 50 * time.Millisecond,\n\t\t\t\tAckPolicy: AckAll,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// Wait for messages.\n\t\t\t// We will do 5 redeliveries.\n\t\t\tfor i := 1; i <= 5; i++ {\n\t\t\t\tcheckFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend*i {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend*i)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\t// Stop redeliveries.\n\t\t\to.delete()\n\n\t\t\t// Now make sure that they are all redelivered in order for each redelivered batch.\n\t\t\tfor l := 1; l <= 5; l++ {\n\t\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\t\tm, _ := sub.NextMsg(time.Second)\n\t\t\t\t\tif seq := o.streamSeqFromReply(m.Reply); seq != uint64(i) {\n\t\t\t\t\t\tt.Fatalf(\"Expected stream sequence of %d, got %d\", i, seq)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAckReplyStreamPending(t *testing.T) {\n\tmsc := StreamConfig{\n\t\tName: \"MY_WQ\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: MemoryStorage,\n\t\tMaxAge: 1 * time.Second,\n\t\tRetention: WorkQueuePolicy,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.1\", \"Hello World!\")\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\texpectPending := func(ep int) {\n\t\t\t\tt.Helper()\n\t\t\t\t// Now check consumer info.\n\t\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t\tif info, pep := o.info(), ep+1; int(info.NumPending) != pep {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected consumer info pending of %d, got %d\", pep, info.NumPending)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t_, _, _, _, pending := replyInfo(m.Reply)\n\t\t\t\tif pending != uint64(ep) {\n\t\t\t\t\tt.Fatalf(\"Expected ack reply pending of %d, got %d - reply: %q\", ep, pending, m.Reply)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\texpectPending(toSend - 1)\n\t\t\t// Send some more while we are connected.\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.1\", \"Hello World!\")\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\texpectPending(toSend*2 - 2)\n\t\t\t// Purge and send a new one.\n\t\t\tmset.purge(nil)\n\t\t\tnc.Flush()\n\n\t\t\tsendStreamMsg(t, nc, \"foo.1\", \"Hello World!\")\n\t\t\texpectPending(0)\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\")\n\t\t\t}\n\t\t\texpectPending(toSend - 1) // 201\n\t\t\t// Test that delete will not register for consumed messages.\n\t\t\tmset.removeMsg(mset.state().FirstSeq)\n\t\t\texpectPending(toSend - 2) // 202\n\t\t\t// Now remove one that has not been delivered.\n\t\t\tmset.removeMsg(250)\n\t\t\texpectPending(toSend - 4) // 203\n\n\t\t\t// Test Expiration.\n\t\t\tmset.purge(nil)\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.1\", \"Hello World!\")\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\t// Wait for expiration to kick in.\n\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\tif state := mset.state(); state.Msgs != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Stream still has messages\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tsendStreamMsg(t, nc, \"foo.33\", \"Hello World!\")\n\t\t\texpectPending(0)\n\n\t\t\t// Now do filtered consumers.\n\t\t\to.delete()\n\t\t\to, err = mset.addConsumer(&ConsumerConfig{Durable: \"PBO-FILTERED\", AckPolicy: AckExplicit, FilterSubject: \"foo.22\"})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.33\", \"Hello World!\")\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\tif info := o.info(); info.NumPending != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d\", info.NumPending)\n\t\t\t}\n\t\t\t// Now send one message that will match us.\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\")\n\t\t\texpectPending(0)\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\") // 504\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\") // 505\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\") // 506\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\") // 507\n\t\t\texpectPending(3)\n\t\t\tmset.removeMsg(506)\n\t\t\texpectPending(1)\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\")\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t\texpectPending(100)\n\t\t\tmset.purge(nil)\n\t\t\tsendStreamMsg(t, nc, \"foo.22\", \"Hello World!\")\n\t\t\texpectPending(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAckReplyStreamPendingWithAcks(t *testing.T) {\n\tmsc := StreamConfig{\n\t\tName: \"MY_STREAM\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 500\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"Hello Foo!\")\n\t\t\t\tsendStreamMsg(t, nc, \"bar\", \"Hello Bar!\")\n\t\t\t\tsendStreamMsg(t, nc, \"baz\", \"Hello Baz!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend*3) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend*3, state.Msgs)\n\t\t\t}\n\t\t\tdsubj := \"_d_\"\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"D-1\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tFilterSubject: \"foo\",\n\t\t\t\tDeliverSubject: dsubj,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tif info := o.info(); int(info.NumPending) != toSend {\n\t\t\t\tt.Fatalf(\"Expected consumer info pending of %d, got %d\", toSend, info.NumPending)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(dsubj)\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tcheckFor(t, 500*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Should be zero.\n\t\t\tif info := o.info(); int(info.NumPending) != 0 {\n\t\t\t\tt.Fatalf(\"Expected consumer info pending of %d, got %d\", 0, info.NumPending)\n\t\t\t} else if info.NumAckPending != toSend {\n\t\t\t\tt.Fatalf(\"Expected %d to be pending acks, got %d\", toSend, info.NumAckPending)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueAckWaitRedelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage, Retention: WorkQueuePolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage, Retention: WorkQueuePolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tackWait := 100 * time.Millisecond\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit, AckWait: ackWait})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\treqNextMsgSubj := o.requestNextMsgSubject()\n\n\t\t\t// Consume all the messages. But do not ack.\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tnc.PublishRequest(reqNextMsgSubj, sub.Subject, nil)\n\t\t\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error waiting for messages: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 {\n\t\t\t\tt.Fatalf(\"Did not consume all messages, still have %d\", nmsgs)\n\t\t\t}\n\n\t\t\t// All messages should still be there.\n\t\t\tstate = mset.state()\n\t\t\tif int(state.Msgs) != toSend {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\t// Now consume and ack.\n\t\t\tfor i := 1; i <= toSend; i++ {\n\t\t\t\tnc.PublishRequest(reqNextMsgSubj, sub.Subject, nil)\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error waiting for message[%d]: %v\", i, err)\n\t\t\t\t}\n\t\t\t\tsseq, dseq, dcount, _, _ := replyInfo(m.Reply)\n\t\t\t\tif sseq != uint64(i) {\n\t\t\t\t\tt.Fatalf(\"Expected set sequence of %d , got %d\", i, sseq)\n\t\t\t\t}\n\t\t\t\t// Delivery sequences should always increase.\n\t\t\t\tif dseq != uint64(toSend+i) {\n\t\t\t\t\tt.Fatalf(\"Expected delivery sequence of %d , got %d\", toSend+i, dseq)\n\t\t\t\t}\n\t\t\t\tif dcount == 1 {\n\t\t\t\t\tt.Fatalf(\"Expected these to be marked as redelivered\")\n\t\t\t\t}\n\t\t\t\t// Ack the message here.\n\t\t\t\tm.AckSync()\n\t\t\t}\n\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 0 {\n\t\t\t\tt.Fatalf(\"Did not consume all messages, still have %d\", nmsgs)\n\t\t\t}\n\n\t\t\t// Flush acks\n\t\t\tnc.Flush()\n\n\t\t\t// Now check the mset as well, since we have a WorkQueue retention policy this should be empty.\n\t\t\tif state := mset.state(); state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected no messages, got %d\", state.Msgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueNakRedelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage, Retention: WorkQueuePolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage, Retention: WorkQueuePolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 10\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tgetMsg := func(sseq, dseq int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\trsseq, rdseq, _, _, _ := replyInfo(m.Reply)\n\t\t\t\tif rdseq != uint64(dseq) {\n\t\t\t\t\tt.Fatalf(\"Expected delivered sequence of %d , got %d\", dseq, rdseq)\n\t\t\t\t}\n\t\t\t\tif rsseq != uint64(sseq) {\n\t\t\t\t\tt.Fatalf(\"Expected store sequence of %d , got %d\", sseq, rsseq)\n\t\t\t\t}\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\tfor i := 1; i <= 5; i++ {\n\t\t\t\tm := getMsg(i, i)\n\t\t\t\t// Ack the message here.\n\t\t\t\tm.Respond(nil)\n\t\t\t}\n\n\t\t\t// Grab #6\n\t\t\tm := getMsg(6, 6)\n\t\t\t// NAK this one and make sure its processed.\n\t\t\tm.Respond(AckNak)\n\t\t\tnc.Flush()\n\n\t\t\t// When we request again should be store sequence 6 again.\n\t\t\tgetMsg(6, 7)\n\t\t\t// Then we should get 7, 8, etc.\n\t\t\tgetMsg(7, 8)\n\t\t\tgetMsg(8, 9)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueWorkingIndicator(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage, Retention: WorkQueuePolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage, Retention: WorkQueuePolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 2\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tackWait := 100 * time.Millisecond\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit, AckWait: ackWait})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tgetMsg := func(sseq, dseq int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\trsseq, rdseq, _, _, _ := replyInfo(m.Reply)\n\t\t\t\tif rdseq != uint64(dseq) {\n\t\t\t\t\tt.Fatalf(\"Expected delivered sequence of %d , got %d\", dseq, rdseq)\n\t\t\t\t}\n\t\t\t\tif rsseq != uint64(sseq) {\n\t\t\t\t\tt.Fatalf(\"Expected store sequence of %d , got %d\", sseq, rsseq)\n\t\t\t\t}\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\tgetMsg(1, 1)\n\t\t\t// Now wait past ackWait\n\t\t\ttime.Sleep(ackWait * 2)\n\n\t\t\t// We should get 1 back.\n\t\t\tm := getMsg(1, 2)\n\n\t\t\t// Now let's take longer than ackWait to process but signal we are working on the message.\n\t\t\ttimeout := time.Now().Add(3 * ackWait)\n\t\t\tfor time.Now().Before(timeout) {\n\t\t\t\tm.Respond(AckProgress)\n\t\t\t\tnc.Flush()\n\t\t\t\ttime.Sleep(ackWait / 5)\n\t\t\t}\n\t\t\t// We should get 2 here, not 1 since we have indicated we are working on it.\n\t\t\tm2 := getMsg(2, 3)\n\t\t\ttime.Sleep(ackWait / 2)\n\t\t\tm2.Respond(AckProgress)\n\n\t\t\t// Now should get 1 back then 2.\n\t\t\tm = getMsg(1, 4)\n\t\t\tm.Respond(nil)\n\t\t\tgetMsg(2, 5)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamWorkQueueTerminateDelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"MY_WQ\", Storage: MemoryStorage, Retention: WorkQueuePolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"MY_WQ\", Storage: FileStorage, Retention: WorkQueuePolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 22\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, c.mconfig.Name, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\tackWait := 25 * time.Millisecond\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"PBO\", AckPolicy: AckExplicit, AckWait: ackWait})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer o.delete()\n\n\t\t\tgetMsg := func(sseq, dseq int) *nats.Msg {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\trsseq, rdseq, _, _, _ := replyInfo(m.Reply)\n\t\t\t\tif rdseq != uint64(dseq) {\n\t\t\t\t\tt.Fatalf(\"Expected delivered sequence of %d , got %d\", dseq, rdseq)\n\t\t\t\t}\n\t\t\t\tif rsseq != uint64(sseq) {\n\t\t\t\t\tt.Fatalf(\"Expected store sequence of %d , got %d\", sseq, rsseq)\n\t\t\t\t}\n\t\t\t\treturn m\n\t\t\t}\n\n\t\t\t// Make sure we get the correct advisory\n\t\t\tsub, _ := nc.SubscribeSync(JSAdvisoryConsumerMsgTerminatedPre + \".>\")\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tgetMsg(1, 1)\n\t\t\t// Now wait past ackWait\n\t\t\ttime.Sleep(ackWait * 2)\n\n\t\t\t// We should get 1 back.\n\t\t\tm := getMsg(1, 2)\n\t\t\t// Now terminate\n\t\t\tm.Respond([]byte(fmt.Sprintf(\"%s with reason\", string(AckTerm))))\n\t\t\ttime.Sleep(ackWait * 2)\n\n\t\t\t// We should get 2 here, not 1 since we have indicated we wanted to terminate.\n\t\t\tgetMsg(2, 3)\n\n\t\t\t// Check advisory was delivered.\n\t\t\tam, err := sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar adv JSConsumerDeliveryTerminatedAdvisory\n\t\t\tjson.Unmarshal(am.Data, &adv)\n\t\t\tif adv.Stream != \"MY_WQ\" {\n\t\t\t\tt.Fatalf(\"Expected stream of %s, got %s\", \"MY_WQ\", adv.Stream)\n\t\t\t}\n\t\t\tif adv.Consumer != \"PBO\" {\n\t\t\t\tt.Fatalf(\"Expected consumer of %s, got %s\", \"PBO\", adv.Consumer)\n\t\t\t}\n\t\t\tif adv.StreamSeq != 1 {\n\t\t\t\tt.Fatalf(\"Expected stream sequence of %d, got %d\", 1, adv.StreamSeq)\n\t\t\t}\n\t\t\tif adv.ConsumerSeq != 2 {\n\t\t\t\tt.Fatalf(\"Expected consumer sequence of %d, got %d\", 2, adv.ConsumerSeq)\n\t\t\t}\n\t\t\tif adv.Deliveries != 2 {\n\t\t\t\tt.Fatalf(\"Expected delivery count of %d, got %d\", 2, adv.Deliveries)\n\t\t\t}\n\t\t\tif adv.Reason != \"with reason\" {\n\t\t\t\tt.Fatalf(\"Advisory did not have a reason\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamAckNext(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"ACKNXT\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"worker\", AckPolicy: AckExplicit})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tfor i := 0; i < 12; i++ {\n\t\tsendStreamMsg(t, nc, mname, fmt.Sprintf(\"msg %d\", i))\n\t}\n\n\tq := make(chan *nats.Msg, 10)\n\tsub, err := nc.ChanSubscribe(nats.NewInbox(), q)\n\tif err != nil {\n\t\tt.Fatalf(\"SubscribeSync failed: %s\", err)\n\t}\n\n\tnc.PublishRequest(o.requestNextMsgSubject(), sub.Subject, []byte(\"1\"))\n\n\t// normal next should imply 1\n\tmsg := <-q\n\terr = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: AckNext})\n\tif err != nil {\n\t\tt.Fatalf(\"RespondMsg failed: %s\", err)\n\t}\n\n\t// read 1 message and check ack was done etc\n\tmsg = <-q\n\tif len(q) != 0 {\n\t\tt.Fatalf(\"Expected empty q got %d\", len(q))\n\t}\n\tif o.info().AckFloor.Stream != 1 {\n\t\tt.Fatalf(\"First message was not acknowledged\")\n\t}\n\tif !bytes.Equal(msg.Data, []byte(\"msg 1\")) {\n\t\tt.Fatalf(\"wrong message received, expected: msg 1 got %q\", msg.Data)\n\t}\n\n\t// now ack and request 5 more using a naked number\n\terr = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: append(AckNext, []byte(\" 5\")...)})\n\tif err != nil {\n\t\tt.Fatalf(\"RespondMsg failed: %s\", err)\n\t}\n\n\tgetMsgs := func(start, count int) {\n\t\tt.Helper()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)\n\t\tdefer cancel()\n\n\t\tfor i := start; i < count+1; i++ {\n\t\t\tselect {\n\t\t\tcase msg := <-q:\n\t\t\t\texpect := fmt.Sprintf(\"msg %d\", i+1)\n\t\t\t\tif !bytes.Equal(msg.Data, []byte(expect)) {\n\t\t\t\t\tt.Fatalf(\"wrong message received, expected: %s got %#v\", expect, msg)\n\t\t\t\t}\n\t\t\tcase <-ctx.Done():\n\t\t\t\tt.Fatalf(\"did not receive all messages\")\n\t\t\t}\n\t\t}\n\n\t}\n\n\tgetMsgs(1, 5)\n\n\t// now ack and request 5 more using the full request\n\terr = msg.RespondMsg(&nats.Msg{Reply: sub.Subject, Subject: msg.Reply, Data: append(AckNext, []byte(`{\"batch\": 5}`)...)})\n\tif err != nil {\n\t\tt.Fatalf(\"RespondMsg failed: %s\", err)\n\t}\n\n\tgetMsgs(6, 10)\n\n\tif o.info().AckFloor.Stream != 2 {\n\t\tt.Fatalf(\"second message was not acknowledged\")\n\t}\n}\n\nfunc TestJetStreamPublishDeDupe(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"DeDupe\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage, MaxAge: time.Hour, Subjects: []string{\"foo.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// Check Duplicates setting.\n\tduplicates := mset.config().Duplicates\n\tif duplicates != StreamDefaultDuplicatesWindow {\n\t\tt.Fatalf(\"Expected a default of %v, got %v\", StreamDefaultDuplicatesWindow, duplicates)\n\t}\n\n\tcfg := mset.config()\n\t// Make sure can't be negative.\n\tcfg.Duplicates = -25 * time.Millisecond\n\tif err := mset.update(&cfg); err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\t// Make sure can't be longer than age if its set.\n\tcfg.Duplicates = 2 * time.Hour\n\tif err := mset.update(&cfg); err == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tsendMsg := func(seq uint64, id, msg string) *PubAck {\n\t\tt.Helper()\n\t\tm := nats.NewMsg(fmt.Sprintf(\"foo.%d\", seq))\n\t\tm.Header.Add(JSMsgId, id)\n\t\tm.Data = []byte(msg)\n\t\tresp, _ := nc.RequestMsg(m, 100*time.Millisecond)\n\t\tif resp == nil {\n\t\t\tt.Fatalf(\"No response for %q, possible timeout?\", msg)\n\t\t}\n\t\tpa := getPubAckResponse(resp.Data)\n\t\tif pa == nil || pa.Error != nil {\n\t\t\tt.Fatalf(\"Expected a JetStreamPubAck, got %q\", resp.Data)\n\t\t}\n\t\tif pa.Sequence != seq {\n\t\t\tt.Fatalf(\"Did not get correct sequence in PubAck, expected %d, got %d\", seq, pa.Sequence)\n\t\t}\n\t\treturn pa.PubAck\n\t}\n\n\texpect := func(n uint64) {\n\t\tt.Helper()\n\t\tstate := mset.state()\n\t\tif state.Msgs != n {\n\t\t\tt.Fatalf(\"Expected %d messages, got %d\", n, state.Msgs)\n\t\t}\n\t}\n\n\tsendMsg(1, \"AA\", \"Hello DeDupe!\")\n\tsendMsg(2, \"BB\", \"Hello DeDupe!\")\n\tsendMsg(3, \"CC\", \"Hello DeDupe!\")\n\tsendMsg(4, \"ZZ\", \"Hello DeDupe!\")\n\texpect(4)\n\n\tsendMsg(1, \"AA\", \"Hello DeDupe!\")\n\tsendMsg(2, \"BB\", \"Hello DeDupe!\")\n\tsendMsg(4, \"ZZ\", \"Hello DeDupe!\")\n\texpect(4)\n\n\tcfg = mset.config()\n\tcfg.Duplicates = 100 * time.Millisecond\n\tif err := mset.update(&cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnmids := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\tif nids := mset.numMsgIds(); nids != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d message ids, got %d\", expected, nids)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tnmids(4)\n\ttime.Sleep(cfg.Duplicates * 2)\n\n\tsendMsg(5, \"AAA\", \"Hello DeDupe!\")\n\tsendMsg(6, \"BBB\", \"Hello DeDupe!\")\n\tsendMsg(7, \"CCC\", \"Hello DeDupe!\")\n\tsendMsg(8, \"DDD\", \"Hello DeDupe!\")\n\tsendMsg(9, \"ZZZ\", \"Hello DeDupe!\")\n\tnmids(5)\n\t// Eventually will drop to zero.\n\tnmids(0)\n\n\t// Now test server restart\n\tcfg.Duplicates = 30 * time.Minute\n\tif err := mset.update(&cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmset.purge(nil)\n\n\t// Send 5 new messages.\n\tsendMsg(10, \"AAAA\", \"Hello DeDupe!\")\n\tsendMsg(11, \"BBBB\", \"Hello DeDupe!\")\n\tsendMsg(12, \"CCCC\", \"Hello DeDupe!\")\n\tsendMsg(13, \"DDDD\", \"Hello DeDupe!\")\n\tsendMsg(14, \"EEEE\", \"Hello DeDupe!\")\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc = clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tmset, _ = s.GlobalAccount().lookupStream(mname)\n\tif nms := mset.state().Msgs; nms != 5 {\n\t\tt.Fatalf(\"Expected 5 restored messages, got %d\", nms)\n\t}\n\tnmids(5)\n\n\t// Send same and make sure duplicate detection still works.\n\t// Send 5 duplicate messages.\n\tsendMsg(10, \"AAAA\", \"Hello DeDupe!\")\n\tsendMsg(11, \"BBBB\", \"Hello DeDupe!\")\n\tsendMsg(12, \"CCCC\", \"Hello DeDupe!\")\n\tsendMsg(13, \"DDDD\", \"Hello DeDupe!\")\n\tsendMsg(14, \"EEEE\", \"Hello DeDupe!\")\n\n\tif nms := mset.state().Msgs; nms != 5 {\n\t\tt.Fatalf(\"Expected 5 restored messages, got %d\", nms)\n\t}\n\tnmids(5)\n\n\t// Check we set duplicate properly.\n\tpa := sendMsg(10, \"AAAA\", \"Hello DeDupe!\")\n\tif !pa.Duplicate {\n\t\tt.Fatalf(\"Expected duplicate to be set\")\n\t}\n\n\t// Purge should NOT wipe the msgIds. They should still persist.\n\tmset.purge(nil)\n\tnmids(5)\n}\n\nfunc getPubAckResponse(msg []byte) *JSPubAckResponse {\n\tvar par JSPubAckResponse\n\tif err := json.Unmarshal(msg, &par); err != nil {\n\t\treturn nil\n\t}\n\treturn &par\n}\n\nfunc TestJetStreamPublishExpect(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"EXPECT\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage, MaxAge: time.Hour, Subjects: []string{\"foo.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Test that we get no error when expected stream is correct.\n\tm := nats.NewMsg(\"foo.bar\")\n\tm.Data = []byte(\"HELLO\")\n\tm.Header.Set(JSExpectedStream, mname)\n\tresp, err := nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error != nil {\n\t\tt.Fatalf(\"Expected a valid JetStreamPubAck, got %q\", resp.Data)\n\t}\n\n\t// Now test that we get an error back when expecting a different stream.\n\tm.Header.Set(JSExpectedStream, \"ORDERS\")\n\tresp, err = nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got %q\", resp.Data)\n\t}\n\n\t// Now test that we get an error back when expecting a different sequence number.\n\tm.Header.Set(JSExpectedStream, mname)\n\tm.Header.Set(JSExpectedLastSeq, \"10\")\n\tresp, err = nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got %q\", resp.Data)\n\t}\n\n\t// Or if we expect that there are no messages by setting \"0\" as the expected last seq\n\tm.Header.Set(JSExpectedLastSeq, \"0\")\n\tresp, err = nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got %q\", resp.Data)\n\t}\n\n\t// Now send a message with a message ID and make sure we can match that.\n\tm = nats.NewMsg(\"foo.bar\")\n\tm.Data = []byte(\"HELLO\")\n\tm.Header.Set(JSMsgId, \"AAA\")\n\tif _, err = nc.RequestMsg(m, 100*time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now try again with new message ID but require last one to be 'BBB'\n\tm.Header.Set(JSMsgId, \"ZZZ\")\n\tm.Header.Set(JSExpectedLastMsgId, \"BBB\")\n\tresp, err = nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got %q\", resp.Data)\n\t}\n\n\t// Restart the server and make sure we remember/rebuild last seq and last msgId.\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc = clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Our last sequence was 2 and last msgId was \"AAA\"\n\tm = nats.NewMsg(\"foo.baz\")\n\tm.Data = []byte(\"HELLO AGAIN\")\n\tm.Header.Set(JSExpectedLastSeq, \"2\")\n\tm.Header.Set(JSExpectedLastMsgId, \"AAA\")\n\tm.Header.Set(JSMsgId, \"BBB\")\n\tresp, err = nc.RequestMsg(m, 100*time.Millisecond)\n\trequire_NoError(t, err)\n\tif pa := getPubAckResponse(resp.Data); pa == nil || pa.Error != nil {\n\t\tt.Fatalf(\"Expected a valid JetStreamPubAck, got %q\", resp.Data)\n\t}\n}\n\nfunc TestJetStreamRedeliveryAfterServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsendSubj := \"MYQ\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sendSubj, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Now load up some messages.\n\ttoSend := 25\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t}\n\tstate := mset.state()\n\tif state.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t}\n\n\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\tdefer sub.Unsubscribe()\n\tnc.Flush()\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"TO\",\n\t\tDeliverSubject: sub.Subject,\n\t\tAckPolicy: AckExplicit,\n\t\tAckWait: 100 * time.Millisecond,\n\t})\n\trequire_NoError(t, err)\n\tdefer o.delete()\n\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Capture port since it was dynamic.\n\tu, _ := url.Parse(s.ClientURL())\n\tport, _ := strconv.Atoi(u.Port())\n\tsd := s.JetStreamConfig().StoreDir\n\n\t// Stop current\n\ts.Shutdown()\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\t// Don't wait for reconnect from old client.\n\tnc = clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tsub, _ = nc.SubscribeSync(sub.Subject)\n\tdefer sub.Unsubscribe()\n\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamUsageNoReservation(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", replicas)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"FILE\", Storage: nats.FileStorage, Replicas: replicas})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"MEM\", Storage: nats.MemoryStorage, Replicas: replicas})\n\t\trequire_NoError(t, err)\n\n\t\tacc := s.globalAccount()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif streams := acc.numStreams(); streams != 2 {\n\t\t\t\treturn fmt.Errorf(\"not at 2 streams yet, got %d\", streams)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tstats := acc.JetStreamUsage()\n\t\trequire_Equal(t, stats.ReservedStore, 0)\n\t\trequire_Equal(t, stats.ReservedMemory, 0)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamUsageReservationNegativeMaxBytes(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", replicas)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tfileCfg := &nats.StreamConfig{Name: \"FILE\", Storage: nats.FileStorage, Replicas: replicas, MaxBytes: 1024}\n\t\tmemCfg := &nats.StreamConfig{Name: \"MEM\", Storage: nats.MemoryStorage, Replicas: replicas, MaxBytes: 1024}\n\n\t\t_, err := js.AddStream(fileCfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(memCfg)\n\t\trequire_NoError(t, err)\n\n\t\tacc := s.globalAccount()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif streams := acc.numStreams(); streams != 2 {\n\t\t\t\treturn fmt.Errorf(\"not at 2 streams yet, got %d\", streams)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tvalidateReserved := func(total uint64) {\n\t\t\tt.Helper()\n\t\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tstats := acc.JetStreamUsage()\n\t\t\t\tif stats.ReservedMemory != total {\n\t\t\t\t\treturn fmt.Errorf(\"acc stats: expected %d, got %d\", total, stats.ReservedMemory)\n\t\t\t\t}\n\t\t\t\tif stats.ReservedStore != total {\n\t\t\t\t\treturn fmt.Errorf(\"acc stats: expected %d, got %d\", total, stats.ReservedStore)\n\t\t\t\t}\n\t\t\t\tjstats := s.getJetStream().usageStats()\n\t\t\t\tif jstats.ReservedMemory != total {\n\t\t\t\t\treturn fmt.Errorf(\"server stats: expected %d, got %d\", total, jstats.ReservedMemory)\n\t\t\t\t}\n\t\t\t\tif jstats.ReservedStore != total {\n\t\t\t\t\treturn fmt.Errorf(\"server stats: expected %d, got %d\", total, jstats.ReservedStore)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\tvalidateReserved(1024)\n\n\t\t// Resetting back to zero.\n\t\tfileCfg.MaxBytes, memCfg.MaxBytes = 0, 0\n\t\t_, err = js.UpdateStream(fileCfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.UpdateStream(memCfg)\n\t\trequire_NoError(t, err)\n\t\tvalidateReserved(0)\n\n\t\t// Update to reset back again.\n\t\tfileCfg.MaxBytes, memCfg.MaxBytes = 512, 512\n\t\t_, err = js.UpdateStream(fileCfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.UpdateStream(memCfg)\n\t\trequire_NoError(t, err)\n\t\tvalidateReserved(512)\n\n\t\t// Resetting back to -1 should mean zero as well, and should do proper accounting.\n\t\tfileCfg.MaxBytes, memCfg.MaxBytes = -1, -1\n\t\t_, err = js.UpdateStream(fileCfg)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.UpdateStream(memCfg)\n\t\trequire_NoError(t, err)\n\t\tvalidateReserved(0)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamSnapshots(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MY-STREAM\"\n\tsubjects := []string{\"foo\", \"bar\", \"baz\"}\n\tcfg := StreamConfig{\n\t\tName: mname,\n\t\tStorage: FileStorage,\n\t\tSubjects: subjects,\n\t\tMaxMsgs: 1000,\n\t}\n\n\tacc := s.GlobalAccount()\n\tmset, err := acc.addStream(&cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Make sure we send some as floor.\n\ttoSend := rand.Intn(200) + 22\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := fmt.Sprintf(\"Hello World %d\", i)\n\t\tsubj := subjects[rand.Intn(len(subjects))]\n\t\tsendStreamMsg(t, nc, subj, msg)\n\t}\n\n\t// Create up to 10 consumers.\n\tnumConsumers := rand.Intn(10) + 1\n\tvar obs []obsi\n\tfor i := 1; i <= numConsumers; i++ {\n\t\tcname := fmt.Sprintf(\"WQ-%d\", i)\n\t\to, err := mset.addConsumer(workerModeConfig(cname))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Now grab some messages.\n\t\ttoReceive := rand.Intn(toSend/2) + 1\n\t\tfor r := 0; r < toReceive; r++ {\n\t\t\tresp, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif resp != nil {\n\t\t\t\tresp.Respond(nil)\n\t\t\t}\n\t\t}\n\t\tobs = append(obs, obsi{o.config(), toReceive})\n\t}\n\tnc.Flush()\n\n\t// Snapshot state of the stream and consumers.\n\tinfo := info{mset.config(), mset.state(), obs}\n\n\tsr, err := mset.snapshot(5*time.Second, false, true)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting snapshot: %v\", err)\n\t}\n\tzr := sr.Reader\n\tsnapshot, err := io.ReadAll(zr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading snapshot\")\n\t}\n\t// Try to restore from snapshot with current stream present, should error.\n\tr := bytes.NewReader(snapshot)\n\tif _, err := acc.RestoreStream(&info.cfg, r); err == nil {\n\t\tt.Fatalf(\"Expected an error trying to restore existing stream\")\n\t} else if !strings.Contains(err.Error(), \"name already in use\") {\n\t\tt.Fatalf(\"Incorrect error received: %v\", err)\n\t}\n\t// Now delete so we can restore.\n\tpusage := acc.JetStreamUsage()\n\tmset.delete()\n\tr.Reset(snapshot)\n\n\tmset, err = acc.RestoreStream(&info.cfg, r)\n\trequire_NoError(t, err)\n\t// Now compare to make sure they are equal.\n\tif nusage := acc.JetStreamUsage(); !reflect.DeepEqual(nusage, pusage) {\n\t\tt.Fatalf(\"Usage does not match after restore: %+v vs %+v\", nusage, pusage)\n\t}\n\tif state := mset.state(); !reflect.DeepEqual(state, info.state) {\n\t\tt.Fatalf(\"State does not match: %+v vs %+v\", state, info.state)\n\t}\n\tif cfg := mset.config(); !reflect.DeepEqual(cfg, info.cfg) {\n\t\tt.Fatalf(\"Configs do not match: %+v vs %+v\", cfg, info.cfg)\n\t}\n\t// Consumers.\n\tif mset.numConsumers() != len(info.obs) {\n\t\tt.Fatalf(\"Number of consumers do not match: %d vs %d\", mset.numConsumers(), len(info.obs))\n\t}\n\tfor _, oi := range info.obs {\n\t\tif o := mset.lookupConsumer(oi.cfg.Durable); o != nil {\n\t\t\tif uint64(oi.ack+1) != o.nextSeq() {\n\t\t\t\tt.Fatalf(\"[%v] Consumer next seq is not correct: %d vs %d\", o.String(), oi.ack+1, o.nextSeq())\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatalf(\"Expected to get an consumer\")\n\t\t}\n\t}\n\n\t// Now try restoring to a different\n\ts2 := RunBasicJetStreamServer(t)\n\tdefer s2.Shutdown()\n\n\tacc = s2.GlobalAccount()\n\tr.Reset(snapshot)\n\tmset, err = acc.RestoreStream(&info.cfg, r)\n\trequire_NoError(t, err)\n\n\to := mset.lookupConsumer(\"WQ-1\")\n\tif o == nil {\n\t\tt.Fatalf(\"Could not lookup consumer\")\n\t}\n\n\tnc2 := clientConnectToServer(t, s2)\n\tdefer nc2.Close()\n\n\t// Make sure we can read messages.\n\tif _, err := nc2.Request(o.requestNextMsgSubject(), nil, 5*time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error getting next message: %v\", err)\n\t}\n}\n\nfunc TestJetStreamSnapshotsAPI(t *testing.T) {\n\tlopts := DefaultTestOptions\n\tlopts.ServerName = \"LS\"\n\tlopts.Port = -1\n\tlopts.LeafNode.Host = lopts.Host\n\tlopts.LeafNode.Port = -1\n\n\tls := RunServer(&lopts)\n\tdefer ls.Shutdown()\n\n\topts := DefaultTestOptions\n\topts.ServerName = \"S\"\n\topts.Port = -1\n\ttdir := t.TempDir()\n\topts.JetStream = true\n\topts.JetStreamDomain = \"domain\"\n\topts.StoreDir = tdir\n\tmaxStore := int64(1024 * 1024 * 1024)\n\topts.maxStoreSet = true\n\topts.JetStreamMaxStore = maxStore\n\trurl, _ := url.Parse(fmt.Sprintf(\"nats-leaf://%s:%d\", lopts.LeafNode.Host, lopts.LeafNode.Port))\n\topts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\tmname := \"MY-STREAM\"\n\tsubjects := []string{\"foo\", \"bar\", \"baz\"}\n\tcfg := StreamConfig{\n\t\tName: mname,\n\t\tStorage: FileStorage,\n\t\tSubjects: subjects,\n\t\tMaxMsgs: 1000,\n\t}\n\n\tacc := s.GlobalAccount()\n\tmset, err := acc.addStreamWithStore(&cfg, &FileStoreConfig{BlockSize: 128})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := rand.Intn(100) + 1\n\tfor i := 1; i <= toSend; i++ {\n\t\tmsg := fmt.Sprintf(\"Hello World %d\", i)\n\t\tsubj := subjects[rand.Intn(len(subjects))]\n\t\tsendStreamMsg(t, nc, subj, msg)\n\t}\n\n\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\trequire_NoError(t, err)\n\t// Now grab some messages.\n\ttoReceive := rand.Intn(toSend) + 1\n\tfor r := 0; r < toReceive; r++ {\n\t\tresp, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif resp != nil {\n\t\t\tresp.Respond(nil)\n\t\t}\n\t}\n\n\t// Make sure we get proper error for non-existent request, streams,etc,\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"foo\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tvar resp JSApiStreamSnapshotResponse\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error == nil || resp.Error.Code != 400 || resp.Error.Description != \"bad request\" {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\tsreq := &JSApiStreamSnapshotRequest{}\n\treq, _ := json.Marshal(sreq)\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"foo\"), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error == nil || resp.Error.Code != 404 || resp.Error.Description != \"stream not found\" {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error == nil || resp.Error.Code != 400 || resp.Error.Description != \"deliver subject not valid\" {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\t// Set delivery subject, do not subscribe yet. Want this to be an ok pattern.\n\tsreq.DeliverSubject = nats.NewInbox()\n\t// Just for test, usually left alone.\n\tsreq.ChunkSize = 1024\n\treq, _ = json.Marshal(sreq)\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\t// Check that we have the config and the state.\n\tif resp.Config == nil {\n\t\tt.Fatalf(\"Expected a stream config in the response, got %+v\\n\", resp)\n\t}\n\tif resp.State == nil {\n\t\tt.Fatalf(\"Expected a stream state in the response, got %+v\\n\", resp)\n\t}\n\n\t// Grab state for comparison.\n\tstate := *resp.State\n\tconfig := *resp.Config\n\n\t// Setup to process snapshot chunks.\n\tvar snapshot []byte\n\tdone := make(chan bool)\n\n\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t// EOF\n\t\tif len(m.Data) == 0 {\n\t\t\tdone <- true\n\t\t\treturn\n\t\t}\n\t\t// Could be writing to a file here too.\n\t\tsnapshot = append(snapshot, m.Data...)\n\t\t// Flow ack\n\t\tm.Respond(nil)\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\n\t// Now make sure this snapshot is legit.\n\tvar rresp JSApiStreamRestoreResponse\n\trreq := &JSApiStreamRestoreRequest{\n\t\tConfig: config,\n\t\tState: state,\n\t}\n\treq, _ = json.Marshal(rreq)\n\n\t// Make sure we get an error since stream still exists.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif !IsNatsErr(rresp.Error, JSStreamNameExistRestoreFailedErr) {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", rresp.Error)\n\t}\n\n\t// Delete this stream.\n\tmset.delete()\n\n\t// Sending no request message will error now.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Make sure to clear.\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error == nil || rresp.Error.Code != 400 || rresp.Error.Description != \"bad request\" {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", rresp.Error)\n\t}\n\n\t// This should work.\n\trmsg, err = nc.Request(fmt.Sprintf(JSApiStreamRestoreT, mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Make sure to clear.\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\n\t// Can be any size message.\n\tvar chunk [512]byte\n\tfor r := bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tnc.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t}\n\tnc.Request(rresp.DeliverSubject, nil, time.Second)\n\n\tmset, err = acc.lookupStream(mname)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t}\n\tif !reflect.DeepEqual(mset.state(), state) {\n\t\tt.Fatalf(\"Did not match states, %+v vs %+v\", mset.state(), state)\n\t}\n\n\t// Now ask that the stream be checked first.\n\tsreq.ChunkSize = 0\n\tsreq.CheckMsgs = true\n\tsnapshot = snapshot[:0]\n\n\treq, _ = json.Marshal(sreq)\n\tif _, err = nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, mname), req, 5*time.Second); err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\n\t// Now connect through a cluster server and make sure we can get things to work this way as well.\n\t// This client, connecting to a leaf without shared system account and domain needs to provide the domain explicitly.\n\tnc2 := clientConnectToServer(t, ls)\n\tdefer nc2.Close()\n\t// Wait a bit for interest to propagate.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tsnapshot = snapshot[:0]\n\n\treq, _ = json.Marshal(sreq)\n\trmsg, err = nc2.Request(fmt.Sprintf(strings.ReplaceAll(JSApiStreamSnapshotT, JSApiPrefix, \"$JS.domain.API\"), mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\tresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\n\t// Now do a restore through the new client connection.\n\t// Delete this stream first.\n\tmset, err = acc.lookupStream(mname)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t}\n\tstate = mset.state()\n\tmset.delete()\n\n\treq, _ = json.Marshal(rreq)\n\trmsg, err = nc2.Request(fmt.Sprintf(strings.ReplaceAll(JSApiStreamRestoreT, JSApiPrefix, \"$JS.domain.API\"), mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Make sure to clear.\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\n\t// Make sure when we send something without a reply subject the subscription is shutoff.\n\tr := bytes.NewReader(snapshot)\n\tn, _ := r.Read(chunk[:])\n\tnc2.Publish(rresp.DeliverSubject, chunk[:n])\n\tnc2.Flush()\n\tn, _ = r.Read(chunk[:])\n\tif _, err := nc2.Request(rresp.DeliverSubject, chunk[:n], 100*time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Expected restore subscription to be closed\")\n\t}\n\n\treq, _ = json.Marshal(rreq)\n\trmsg, err = nc2.Request(fmt.Sprintf(strings.ReplaceAll(JSApiStreamRestoreT, JSApiPrefix, \"$JS.domain.API\"), mname), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Make sure to clear.\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\n\tfor r := bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\t// Make sure other side responds to reply subjects for ack flow. Optional.\n\t\tif _, err := nc2.Request(rresp.DeliverSubject, chunk[:n], time.Second); err != nil {\n\t\t\tt.Fatalf(\"Restore not honoring reply subjects for ack flow\")\n\t\t}\n\t}\n\n\t// For EOF this will send back stream info or an error.\n\tsi, err := nc2.Request(rresp.DeliverSubject, nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error restoring stream: %v\", err)\n\t}\n\tvar scResp JSApiStreamCreateResponse\n\tif err := json.Unmarshal(si.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error from EOF on restore: %+v\", scResp.Error)\n\t}\n\n\tif !reflect.DeepEqual(scResp.StreamInfo.State, state) {\n\t\tt.Fatalf(\"Did not match states, %+v vs %+v\", scResp.StreamInfo.State, state)\n\t}\n\n\t// Now make sure that if we try to change the name/identity of the stream we get an error.\n\tmset, err = acc.lookupStream(mname)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t}\n\tmset.state()\n\tmset.delete()\n\n\trreq.Config.Name = \"NEW_STREAM\"\n\treq, _ = json.Marshal(rreq)\n\n\trmsg, err = nc2.Request(fmt.Sprintf(strings.ReplaceAll(JSApiStreamRestoreT, JSApiPrefix, \"$JS.domain.API\"), rreq.Config.Name), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\t// Make sure to clear.\n\trresp.Error = nil\n\tjson.Unmarshal(rmsg.Data, &rresp)\n\t// We should not get an error here.\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Got an unexpected error response: %+v\", rresp.Error)\n\t}\n\tfor r := bytes.NewReader(snapshot); ; {\n\t\tn, err := r.Read(chunk[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tnc2.Request(rresp.DeliverSubject, chunk[:n], time.Second)\n\t}\n\n\tsi, err = nc2.Request(rresp.DeliverSubject, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tscResp.Error = nil\n\tif err := json.Unmarshal(si.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.Error == nil {\n\t\tt.Fatalf(\"Expected an error but got none\")\n\t}\n\texpect := \"names do not match\"\n\tif !strings.Contains(scResp.Error.Description, expect) {\n\t\tt.Fatalf(\"Expected description of %q, got %q\", expect, scResp.Error.Description)\n\t}\n}\n\nfunc TestJetStreamPubAckPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", Storage: nats.MemoryStorage}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 1_000_000\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamPubPerfWithFullStream(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttoSend, msg := 1_000_000, []byte(\"OK\")\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\", MaxMsgs: int64(toSend)})\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", msg)\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n\n\t// Now do same amount but knowing we are at our limit.\n\tstart = time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", msg)\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt = time.Since(start)\n\tfmt.Printf(\"\\ntime is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamSnapshotsAPIPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tcfg := StreamConfig{\n\t\tName: \"snap-perf\",\n\t\tStorage: FileStorage,\n\t}\n\n\tacc := s.GlobalAccount()\n\tif _, err := acc.addStream(&cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tmsg := make([]byte, 128*1024)\n\t// If you don't give gzip some data will spend too much time compressing everything to zero.\n\tcrand.Read(msg)\n\n\tfor i := 0; i < 10000; i++ {\n\t\tnc.Publish(\"snap-perf\", msg)\n\t}\n\tnc.Flush()\n\n\tsreq := &JSApiStreamSnapshotRequest{DeliverSubject: nats.NewInbox()}\n\treq, _ := json.Marshal(sreq)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"snap-perf\"), req, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on snapshot request: %v\", err)\n\t}\n\n\tvar resp JSApiStreamSnapshotResponse\n\tjson.Unmarshal(rmsg.Data, &resp)\n\tif resp.Error != nil {\n\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t}\n\n\tdone := make(chan bool)\n\ttotal := 0\n\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t// EOF\n\t\tif len(m.Data) == 0 {\n\t\t\tm.Sub.Unsubscribe()\n\t\t\tdone <- true\n\t\t\treturn\n\t\t}\n\t\t// We don't do anything with the snapshot, just take\n\t\t// note of the size.\n\t\ttotal += len(m.Data)\n\t\t// Flow ack\n\t\tm.Respond(nil)\n\t})\n\tdefer sub.Unsubscribe()\n\n\tstart := time.Now()\n\t// Wait to receive the snapshot.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(30 * time.Second):\n\t\tt.Fatalf(\"Did not receive our snapshot in time\")\n\t}\n\ttd := time.Since(start)\n\tfmt.Printf(\"Received %d bytes in %v\\n\", total, td)\n\tfmt.Printf(\"Rate %.0f MB/s\\n\", float64(total)/td.Seconds()/(1024*1024))\n}\n\nfunc TestJetStreamActiveDelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"ADS\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"ADS\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now load up some messages.\n\t\t\ttoSend := 100\n\t\t\tsendSubj := \"foo.22\"\n\t\t\tfor i := 0; i < toSend; i++ {\n\t\t\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(toSend) {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, state.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"to\", DeliverSubject: \"d\"})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\t// We have no active interest above. So consumer will be considered inactive. Let's subscribe and make sure\n\t\t\t// we get the messages instantly. This will test that we hook interest activation correctly.\n\t\t\tsub, _ := nc.SubscribeSync(\"d\")\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\tcheckFor(t, 100*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamEphemeralConsumers(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"EP\", Storage: MemoryStorage, Subjects: []string{\"foo.*\"}}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"EP\", Storage: FileStorage, Subjects: []string{\"foo.*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif !o.isActive() {\n\t\t\t\tt.Fatalf(\"Expected the consumer to be considered active\")\n\t\t\t}\n\t\t\tif numo := mset.numConsumers(); numo != 1 {\n\t\t\t\tt.Fatalf(\"Expected number of consumers to be 1, got %d\", numo)\n\t\t\t}\n\t\t\t// Set our delete threshold to something low for testing purposes.\n\t\t\to.setInActiveDeleteThreshold(100 * time.Millisecond)\n\n\t\t\t// Make sure works now.\n\t\t\tnc.Request(\"foo.22\", nil, 100*time.Millisecond)\n\t\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, 1)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now close the subscription, this should trip active state on the ephemeral consumer.\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Expected the ephemeral consumer to be considered inactive\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\t// The reason for this still being 1 is that we give some time in case of a reconnect scenario.\n\t\t\t// We detect right away on the interest change but we wait for interest to be re-established.\n\t\t\t// This is in case server goes away but app is fine, we do not want to recycle those consumers.\n\t\t\tif numo := mset.numConsumers(); numo != 1 {\n\t\t\t\tt.Fatalf(\"Expected number of consumers to be 1, got %d\", numo)\n\t\t\t}\n\n\t\t\t// We should delete this one after the delete threshold.\n\t\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tif numo := mset.numConsumers(); numo != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected number of consumers to be 0, got %d\", numo)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMetadata(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Retention: WorkQueuePolicy, Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Retention: WorkQueuePolicy, Storage: FileStorage}},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tnc.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t\tnc.Flush()\n\t\t\t\ttime.Sleep(time.Millisecond)\n\t\t\t}\n\n\t\t\tif state := mset.state(); state.Msgs != 10 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 10, state.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := uint64(1); i <= 10; i++ {\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tsseq, dseq, dcount, ts, _ := replyInfo(m.Reply)\n\n\t\t\t\tmreq := &JSApiMsgGetRequest{Seq: sseq}\n\t\t\t\treq, err := json.Marshal(mreq)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Load the original message from the stream to verify ReplyInfo ts against stored message\n\t\t\t\tsmsgj, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, c.mconfig.Name), req, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Could not retrieve stream message: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tvar resp JSApiMsgGetResponse\n\t\t\t\terr = json.Unmarshal(smsgj.Data, &resp)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t\t\t\t}\n\t\t\t\tif resp.Message == nil || resp.Error != nil {\n\t\t\t\t\tt.Fatalf(\"Did not receive correct response\")\n\t\t\t\t}\n\t\t\t\tsmsg := resp.Message\n\t\t\t\tif ts != smsg.Time.UnixNano() {\n\t\t\t\t\tt.Fatalf(\"Wrong timestamp in ReplyInfo for msg %d, expected %v got %v\", i, ts, smsg.Time.UnixNano())\n\t\t\t\t}\n\t\t\t\tif sseq != i {\n\t\t\t\t\tt.Fatalf(\"Expected set sequence of %d, got %d\", i, sseq)\n\t\t\t\t}\n\t\t\t\tif dseq != i {\n\t\t\t\t\tt.Fatalf(\"Expected delivery sequence of %d, got %d\", i, dseq)\n\t\t\t\t}\n\t\t\t\tif dcount != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count to be 1, got %d\", dcount)\n\t\t\t\t}\n\t\t\t\tm.Respond(AckAck)\n\t\t\t}\n\n\t\t\t// Now make sure we get right response when message is missing.\n\t\t\tmreq := &JSApiMsgGetRequest{Seq: 1}\n\t\t\treq, err := json.Marshal(mreq)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Load the original message from the stream to verify ReplyInfo ts against stored message\n\t\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, c.mconfig.Name), req, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Could not retrieve stream message: %v\", err)\n\t\t\t}\n\t\t\tvar resp JSApiMsgGetResponse\n\t\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t\t\t}\n\t\t\tif resp.Error == nil || resp.Error.Code != 404 || resp.Error.Description != \"no message found\" {\n\t\t\t\tt.Fatalf(\"Did not get correct error response: %+v\", resp.Error)\n\t\t\t}\n\t\t})\n\t}\n}\nfunc TestJetStreamRedeliverCount(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tif _, err = js.Publish(\"DC\", []byte(\"OK!\")); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tcheckFor(t, time.Second, time.Millisecond*250, func() error {\n\t\t\t\tif state := mset.state(); state.Msgs != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d messages, got %d\", 1, state.Msgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := uint64(1); i <= 10; i++ {\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tsseq, dseq, dcount, _, _ := replyInfo(m.Reply)\n\n\t\t\t\t// Make sure we keep getting stream sequence #1\n\t\t\t\tif sseq != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected set sequence of 1, got %d\", sseq)\n\t\t\t\t}\n\t\t\t\tif dseq != i {\n\t\t\t\t\tt.Fatalf(\"Expected delivery sequence of %d, got %d\", i, dseq)\n\t\t\t\t}\n\t\t\t\t// Now make sure dcount is same as dseq (or i).\n\t\t\t\tif dcount != i {\n\t\t\t\t\tt.Fatalf(\"Expected delivery count to be %d, got %d\", i, dcount)\n\t\t\t\t}\n\n\t\t\t\t// Make sure it keeps getting sent back.\n\t\t\t\tm.Respond(AckNak)\n\t\t\t\tnc.Flush()\n\t\t\t}\n\t\t})\n\t}\n}\n\n// We want to make sure that for pull based consumers that if we ack\n// late with no interest the redelivery attempt is removed and we do\n// not get the message back.\nfunc TestJetStreamRedeliverAndLateAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"LA\", Storage: MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: \"DDD\", AckPolicy: AckExplicit, AckWait: 100 * time.Millisecond})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Queue up message\n\tsendStreamMsg(t, nc, \"LA\", \"Hello World!\")\n\n\tnextSubj := o.requestNextMsgSubject()\n\tmsg, err := nc.Request(nextSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// Wait for past ackwait time\n\ttime.Sleep(150 * time.Millisecond)\n\t// Now ack!\n\tmsg.AckSync()\n\t// We should not get this back.\n\tif _, err := nc.Request(nextSubj, nil, 10*time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Message should not have been sent back\")\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/1502\nfunc TestJetStreamPendingNextTimer(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"NT\", Storage: MemoryStorage, Subjects: []string{\"ORDERS.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"DDD\",\n\t\tAckPolicy: AckExplicit,\n\t\tFilterSubject: \"ORDERS.test\",\n\t\tAckWait: 100 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tsendAndReceive := func() {\n\t\tnc := clientConnectToServer(t, s)\n\t\tdefer nc.Close()\n\n\t\t// Queue up message\n\t\tsendStreamMsg(t, nc, \"ORDERS.test\", \"Hello World! #1\")\n\t\tsendStreamMsg(t, nc, \"ORDERS.test\", \"Hello World! #2\")\n\n\t\tnextSubj := o.requestNextMsgSubject()\n\t\tfor i := 0; i < 2; i++ {\n\t\t\tif _, err := nc.Request(nextSubj, nil, time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\t\tnc.Close()\n\t\ttime.Sleep(200 * time.Millisecond)\n\t}\n\n\tsendAndReceive()\n\tsendAndReceive()\n\tsendAndReceive()\n}\n\nfunc TestJetStreamCanNotNakAckd(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 10 msgs\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != 10 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 10, state.Msgs)\n\t\t\t}\n\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tfor i := uint64(1); i <= 10; i++ {\n\t\t\t\tm, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Ack evens.\n\t\t\t\tif i%2 == 0 {\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\t// Fake these for now.\n\t\t\tackReplyT := \"$JS.A.DC.WQ.1.%d.%d\"\n\t\t\tcheckBadNak := func(seq int) {\n\t\t\t\tt.Helper()\n\t\t\t\tif err := nc.Publish(fmt.Sprintf(ackReplyT, seq, seq), AckNak); err != nil {\n\t\t\t\t\tt.Fatalf(\"Error sending nak: %v\", err)\n\t\t\t\t}\n\t\t\t\tnc.Flush()\n\t\t\t\tif _, err := nc.Request(o.requestNextMsgSubject(), nil, 10*time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect new delivery on nak of %d\", seq)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// If the nak took action it will deliver another message, incrementing the next delivery seq.\n\t\t\t// We ack evens above, so these should fail\n\t\t\tfor i := 2; i <= 10; i += 2 {\n\t\t\t\tcheckBadNak(i)\n\t\t\t}\n\n\t\t\t// Now check we can not nak something we do not have.\n\t\t\tcheckBadNak(22)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStreamPurge(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 100 msgs\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != 100 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 100, state.Msgs)\n\t\t\t}\n\t\t\tmset.purge(nil)\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 0, state.Msgs)\n\t\t\t}\n\t\t\t// Make sure first timestamp are reset.\n\t\t\tif !state.FirstTime.IsZero() {\n\t\t\t\tt.Fatalf(\"Expected the state's first time to be zero after purge\")\n\t\t\t}\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tnow := time.Now()\n\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\n\t\t\tstate = mset.state()\n\t\t\tif state.Msgs != 1 {\n\t\t\t\tt.Fatalf(\"Expected %d message, got %d\", 1, state.Msgs)\n\t\t\t}\n\t\t\tif state.FirstTime.Before(now) {\n\t\t\t\tt.Fatalf(\"First time is incorrect after adding messages back in\")\n\t\t\t}\n\t\t\tif state.FirstTime != state.LastTime {\n\t\t\t\tt.Fatalf(\"Expected first and last times to be the same for only message\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStreamPurgeWithConsumer(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 100 msgs\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != 100 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 100, state.Msgs)\n\t\t\t}\n\t\t\t// Now create an consumer and make sure it functions properly.\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\t\t\tnextSubj := o.requestNextMsgSubject()\n\t\t\tfor i := 0; i < 50; i++ {\n\t\t\t\tmsg, err := nc.Request(nextSubj, nil, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Ack.\n\t\t\t\tmsg.Respond(nil)\n\t\t\t}\n\t\t\t// Now grab next 25 without ack.\n\t\t\tfor i := 0; i < 25; i++ {\n\t\t\t\tif _, err := nc.Request(nextSubj, nil, time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tstate := o.info()\n\t\t\tif state.AckFloor.Consumer != 50 {\n\t\t\t\tt.Fatalf(\"Expected ack floor of 50, got %d\", state.AckFloor.Consumer)\n\t\t\t}\n\t\t\tif state.NumAckPending != 25 {\n\t\t\t\tt.Fatalf(\"Expected len(pending) to be 25, got %d\", state.NumAckPending)\n\t\t\t}\n\t\t\t// Now do purge.\n\t\t\tmset.purge(nil)\n\t\t\tif state := mset.state(); state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 0, state.Msgs)\n\t\t\t}\n\t\t\t// Now re-acquire state and check that we did the right thing.\n\t\t\t// Pending should be cleared, and stream sequences should have been set\n\t\t\t// to the total messages before purge + 1.\n\t\t\tstate = o.info()\n\t\t\tif state.NumAckPending != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d\", state.NumAckPending)\n\t\t\t}\n\t\t\tif state.Delivered.Stream != 100 {\n\t\t\t\tt.Fatalf(\"Expected to have setseq now at next seq of 100, got %d\", state.Delivered.Stream)\n\t\t\t}\n\t\t\t// Check AckFloors which should have also been adjusted.\n\t\t\tif state.AckFloor.Stream != 100 {\n\t\t\t\tt.Fatalf(\"Expected ackfloor for setseq to be 100, got %d\", state.AckFloor.Stream)\n\t\t\t}\n\t\t\tif state.AckFloor.Consumer != 75 {\n\t\t\t\tt.Fatalf(\"Expected ackfloor for obsseq to be 75, got %d\", state.AckFloor.Consumer)\n\t\t\t}\n\t\t\t// Also make sure we can get new messages correctly.\n\t\t\tjs.Publish(\"DC\", []byte(\"OK-22\"))\n\t\t\tif msg, err := nc.Request(nextSubj, nil, time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else if string(msg.Data) != \"OK-22\" {\n\t\t\t\tt.Fatalf(\"Received wrong message, wanted 'OK-22', got %q\", msg.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStreamPurgeWithConsumerAndRedelivery(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 100 msgs\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != 100 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 100, state.Msgs)\n\t\t\t}\n\t\t\t// Now create an consumer and make sure it functions properly.\n\t\t\t// This will test redelivery state and purge of the stream.\n\t\t\twcfg := &ConsumerConfig{\n\t\t\t\tDurable: \"WQ\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 20 * time.Millisecond,\n\t\t\t}\n\t\t\to, err := mset.addConsumer(wcfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\t\t\tnextSubj := o.requestNextMsgSubject()\n\t\t\tfor i := 0; i < 50; i++ {\n\t\t\t\t// Do not ack these.\n\t\t\t\tif _, err := nc.Request(nextSubj, nil, time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now wait to make sure we are in a redelivered state.\n\t\t\ttime.Sleep(wcfg.AckWait * 2)\n\t\t\t// Now do purge.\n\t\t\tmset.purge(nil)\n\t\t\tif state := mset.state(); state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 0, state.Msgs)\n\t\t\t}\n\t\t\t// Now get the state and check that we did the right thing.\n\t\t\t// Pending should be cleared, and stream sequences should have been set\n\t\t\t// to the total messages before purge + 1.\n\t\t\tstate := o.info()\n\t\t\tif state.NumAckPending != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d\", state.NumAckPending)\n\t\t\t}\n\t\t\tif state.Delivered.Stream != 100 {\n\t\t\t\tt.Fatalf(\"Expected to have setseq now at next seq of 100, got %d\", state.Delivered.Stream)\n\t\t\t}\n\t\t\t// Check AckFloors which should have also been adjusted.\n\t\t\tif state.AckFloor.Stream != 100 {\n\t\t\t\tt.Fatalf(\"Expected ackfloor for setseq to be 100, got %d\", state.AckFloor.Stream)\n\t\t\t}\n\t\t\tif state.AckFloor.Consumer != 50 {\n\t\t\t\tt.Fatalf(\"Expected ackfloor for obsseq to be 75, got %d\", state.AckFloor.Consumer)\n\t\t\t}\n\t\t\t// Also make sure we can get new messages correctly.\n\t\t\tjs.Publish(\"DC\", []byte(\"OK-22\"))\n\t\t\tif msg, err := nc.Request(nextSubj, nil, time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else if string(msg.Data) != \"OK-22\" {\n\t\t\t\tt.Fatalf(\"Received wrong message, wanted 'OK-22', got %q\", msg.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamInterestRetentionStream(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Storage: MemoryStorage, Retention: InterestPolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Storage: FileStorage, Retention: InterestPolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 100 msgs\n\t\t\ttotalMsgs := 100\n\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\n\t\t\tcheckNumMsgs := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\tif state := mset.state(); state.Msgs != uint64(numExpected) {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d messages, got %d\", numExpected, state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Since we had no interest this should be 0.\n\t\t\tcheckNumMsgs(0)\n\n\t\t\tsyncSub := func() *nats.Subscription {\n\t\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\t\tnc.Flush()\n\t\t\t\treturn sub\n\t\t\t}\n\n\t\t\t// Now create three consumers.\n\t\t\t// 1. AckExplicit\n\t\t\t// 2. AckAll\n\t\t\t// 3. AckNone\n\n\t\t\tsub1 := syncSub()\n\t\t\tmset.addConsumer(&ConsumerConfig{DeliverSubject: sub1.Subject, AckPolicy: AckExplicit})\n\n\t\t\tsub2 := syncSub()\n\t\t\tmset.addConsumer(&ConsumerConfig{DeliverSubject: sub2.Subject, AckPolicy: AckAll})\n\n\t\t\tsub3 := syncSub()\n\t\t\tmset.addConsumer(&ConsumerConfig{DeliverSubject: sub3.Subject, AckPolicy: AckNone})\n\n\t\t\tfor i := 0; i < totalMsgs; i++ {\n\t\t\t\tjs.Publish(\"DC\", []byte(\"OK!\"))\n\t\t\t}\n\n\t\t\tcheckNumMsgs(totalMsgs)\n\n\t\t\t// Wait for all messsages to be pending for each sub.\n\t\t\tfor i, sub := range []*nats.Subscription{sub1, sub2, sub3} {\n\t\t\t\tcheckFor(t, 5*time.Second, 25*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); nmsgs != totalMsgs {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d for sub %d\", nmsgs, totalMsgs, i+1)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tgetAndAck := func(sub *nats.Subscription) {\n\t\t\t\tt.Helper()\n\t\t\t\tif m, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t} else {\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t\tnc.Flush()\n\t\t\t}\n\n\t\t\t// Ack evens for the explicit ack sub.\n\t\t\tvar odds []*nats.Msg\n\t\t\tfor i := 1; i <= totalMsgs; i++ {\n\t\t\t\tif m, err := sub1.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t} else if i%2 == 0 {\n\t\t\t\t\tm.Respond(nil) // Ack evens.\n\t\t\t\t} else {\n\t\t\t\t\todds = append(odds, m)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\tcheckNumMsgs(totalMsgs)\n\n\t\t\t// Now ack first for AckAll sub2\n\t\t\tgetAndAck(sub2)\n\t\t\t// We should be at the same number since we acked 1, explicit acked 2\n\t\t\tcheckNumMsgs(totalMsgs)\n\t\t\t// Now ack second for AckAll sub2\n\t\t\tgetAndAck(sub2)\n\t\t\t// We should now have 1 removed.\n\t\t\tcheckNumMsgs(totalMsgs - 1)\n\t\t\t// Now ack third for AckAll sub2\n\t\t\tgetAndAck(sub2)\n\t\t\t// We should still only have 1 removed.\n\t\t\tcheckNumMsgs(totalMsgs - 1)\n\n\t\t\t// Now ack odds from explicit.\n\t\t\tfor _, m := range odds {\n\t\t\t\tm.Respond(nil) // Ack\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\t// we should have 1, 2, 3 acks now.\n\t\t\tcheckNumMsgs(totalMsgs - 3)\n\n\t\t\tnm, _, _ := sub2.Pending()\n\t\t\t// Now ack last ackAll message. This should clear all of them.\n\t\t\tfor i := 1; i <= nm; i++ {\n\t\t\t\tif m, err := sub2.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t} else if i == nm {\n\t\t\t\t\tm.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\t// Should be zero now.\n\t\t\tcheckNumMsgs(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamInterestRetentionStreamWithFilteredConsumers(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"DC\", Subjects: []string{\"*\"}, Storage: MemoryStorage, Retention: InterestPolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"DC\", Subjects: []string{\"*\"}, Storage: FileStorage, Retention: InterestPolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfsub, err := js.SubscribeSync(\"foo\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer fsub.Unsubscribe()\n\n\t\t\tbsub, err := js.SubscribeSync(\"bar\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer bsub.Unsubscribe()\n\n\t\t\tmsg := []byte(\"FILTERED\")\n\t\t\tsendMsg := func(subj string) {\n\t\t\t\tt.Helper()\n\t\t\t\tif _, err = js.Publish(subj, msg); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tgetAndAck := func(sub *nats.Subscription) {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error getting msg: %v\", err)\n\t\t\t\t}\n\t\t\t\tm.AckSync()\n\t\t\t}\n\n\t\t\tcheckState := func(expected uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tsi, err := js.StreamInfo(\"DC\")\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != expected {\n\t\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", expected, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsendMsg(\"foo\")\n\t\t\tcheckState(1)\n\t\t\tgetAndAck(fsub)\n\t\t\tcheckState(0)\n\t\t\tsendMsg(\"bar\")\n\t\t\tsendMsg(\"foo\")\n\t\t\tcheckState(2)\n\t\t\tgetAndAck(bsub)\n\t\t\tcheckState(1)\n\t\t\tgetAndAck(fsub)\n\t\t\tcheckState(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamInterestRetentionWithWildcardsAndFilteredConsumers(t *testing.T) {\n\tmsc := StreamConfig{\n\t\tName: \"DCWC\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: MemoryStorage,\n\t\tRetention: InterestPolicy,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Send 10 msgs\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo.bar\", \"Hello World!\")\n\t\t\t}\n\t\t\tif state := mset.state(); state.Msgs != 0 {\n\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", 0, state.Msgs)\n\t\t\t}\n\n\t\t\tcfg := &ConsumerConfig{Durable: \"ddd\", FilterSubject: \"foo.bar\", AckPolicy: AckExplicit}\n\t\t\to, err := mset.addConsumer(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tsendStreamMsg(t, nc, \"foo.bar\", \"Hello World!\")\n\t\t\tif state := mset.state(); state.Msgs != 1 {\n\t\t\t\tt.Fatalf(\"Expected %d message, got %d\", 1, state.Msgs)\n\t\t\t} else if state.FirstSeq != 11 {\n\t\t\t\tt.Fatalf(\"Expected %d for first seq, got %d\", 11, state.FirstSeq)\n\t\t\t}\n\t\t\t// Now send to foo.baz, which has no interest, so we should not hold onto this message.\n\t\t\tsendStreamMsg(t, nc, \"foo.baz\", \"Hello World!\")\n\t\t\tif state := mset.state(); state.Msgs != 1 {\n\t\t\t\tt.Fatalf(\"Expected %d message, got %d\", 1, state.Msgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamInterestRetentionStreamWithDurableRestart(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &StreamConfig{Name: \"IK\", Storage: MemoryStorage, Retention: InterestPolicy}},\n\t\t{\"FileStore\", &StreamConfig{Name: \"IK\", Storage: FileStorage, Retention: InterestPolicy}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tcheckNumMsgs := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\t\t\t\tif state := mset.state(); state.Msgs != uint64(numExpected) {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %d messages, got %d\", numExpected, state.Msgs)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tnc.Flush()\n\n\t\t\tcfg := &ConsumerConfig{Durable: \"ivan\", DeliverPolicy: DeliverNew, DeliverSubject: sub.Subject, AckPolicy: AckNone}\n\n\t\t\to, _ := mset.addConsumer(cfg)\n\n\t\t\tsendStreamMsg(t, nc, \"IK\", \"M1\")\n\t\t\tsendStreamMsg(t, nc, \"IK\", \"M2\")\n\n\t\t\tcheckSubPending := func(numExpected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tcheckSubPending(2)\n\t\t\tcheckNumMsgs(0)\n\n\t\t\t// Now stop the subscription.\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Still active consumer\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tsendStreamMsg(t, nc, \"IK\", \"M3\")\n\t\t\tsendStreamMsg(t, nc, \"IK\", \"M4\")\n\n\t\t\tcheckNumMsgs(2)\n\n\t\t\t// Now restart the durable.\n\t\t\tsub, _ = nc.SubscribeSync(nats.NewInbox())\n\t\t\tnc.Flush()\n\t\t\tcfg.DeliverSubject = sub.Subject\n\t\t\tif o, err = mset.addConsumer(cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Error re-establishing the durable consumer: %v\", err)\n\t\t\t}\n\t\t\tcheckSubPending(2)\n\n\t\t\tfor _, expected := range []string{\"M3\", \"M4\"} {\n\t\t\t\tif m, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t} else if string(m.Data) != expected {\n\t\t\t\t\tt.Fatalf(\"Expected %q, got %q\", expected, m.Data)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Should all be gone now.\n\t\t\tcheckNumMsgs(0)\n\n\t\t\t// Now restart again and make sure we do not get any messages.\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\t\tif o.isActive() {\n\t\t\t\t\treturn fmt.Errorf(\"Still active consumer\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\to.delete()\n\n\t\t\tsub, _ = nc.SubscribeSync(nats.NewInbox())\n\t\t\tnc.Flush()\n\n\t\t\tcfg.DeliverSubject = sub.Subject\n\t\t\tcfg.AckPolicy = AckExplicit // Set ack\n\t\t\tif o, err = mset.addConsumer(cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Error re-establishing the durable consumer: %v\", err)\n\t\t\t}\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\tcheckSubPending(0)\n\t\t\tcheckNumMsgs(0)\n\n\t\t\t// Now queue up some messages.\n\t\t\tfor i := 1; i <= 10; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"IK\", fmt.Sprintf(\"M%d\", i))\n\t\t\t}\n\t\t\tcheckNumMsgs(10)\n\t\t\tcheckSubPending(10)\n\n\t\t\t// Create second consumer\n\t\t\tsub2, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tnc.Flush()\n\t\t\tcfg.DeliverSubject = sub2.Subject\n\t\t\tcfg.Durable = \"derek\"\n\t\t\to2, err := mset.addConsumer(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating second durable consumer: %v\", err)\n\t\t\t}\n\n\t\t\t// Now queue up some messages.\n\t\t\tfor i := 11; i <= 20; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"IK\", fmt.Sprintf(\"M%d\", i))\n\t\t\t}\n\t\t\tcheckNumMsgs(20)\n\t\t\tcheckSubPending(20)\n\n\t\t\t// Now make sure deleting the consumers will remove messages from\n\t\t\t// the stream since we are interest retention based.\n\t\t\to.delete()\n\t\t\tcheckNumMsgs(10)\n\n\t\t\to2.delete()\n\t\t\tcheckNumMsgs(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSystemLimits(t *testing.T) {\n\ts := RunRandClientPortServer(t)\n\tdefer s.Shutdown()\n\n\tif _, _, err := s.JetStreamReservedResources(); err == nil {\n\t\tt.Fatalf(\"Expected error requesting jetstream reserved resources when not enabled\")\n\t}\n\t// Create some accounts.\n\tfacc, _ := s.LookupOrRegisterAccount(\"FOO\")\n\tbacc, _ := s.LookupOrRegisterAccount(\"BAR\")\n\tzacc, _ := s.LookupOrRegisterAccount(\"BAZ\")\n\n\tjsconfig := &JetStreamConfig{MaxMemory: 1024, MaxStore: 8192, StoreDir: t.TempDir()}\n\tif err := s.EnableJetStream(jsconfig); err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\n\tif rm, rd, err := s.JetStreamReservedResources(); err != nil {\n\t\tt.Fatalf(\"Unexpected error requesting jetstream reserved resources: %v\", err)\n\t} else if rm != 0 || rd != 0 {\n\t\tt.Fatalf(\"Expected reserved memory and store to be 0, got %d and %d\", rm, rd)\n\t}\n\n\tlimits := func(mem int64, store int64) map[string]JetStreamAccountLimits {\n\t\treturn map[string]JetStreamAccountLimits{\n\t\t\t_EMPTY_: {\n\t\t\t\tMaxMemory: mem,\n\t\t\t\tMaxStore: store,\n\t\t\t\tMaxStreams: -1,\n\t\t\t\tMaxConsumers: -1,\n\t\t\t},\n\t\t}\n\t}\n\n\tif err := facc.EnableJetStream(limits(24, 192), nil); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Use up rest of our resources in memory\n\tif err := bacc.EnableJetStream(limits(1000, 0), nil); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now ask for more memory. Should error.\n\tif err := zacc.EnableJetStream(limits(1000, 0), nil); err == nil {\n\t\tt.Fatalf(\"Expected an error when exhausting memory resource limits\")\n\t}\n\t// Disk too.\n\tif err := zacc.EnableJetStream(limits(0, 10000), nil); err == nil {\n\t\tt.Fatalf(\"Expected an error when exhausting memory resource limits\")\n\t}\n\tfacc.DisableJetStream()\n\tbacc.DisableJetStream()\n\tzacc.DisableJetStream()\n\n\t// Make sure we unreserved resources.\n\tif rm, rd, err := s.JetStreamReservedResources(); err != nil {\n\t\tt.Fatalf(\"Unexpected error requesting jetstream reserved resources: %v\", err)\n\t} else if rm != 0 || rd != 0 {\n\t\tt.Fatalf(\"Expected reserved memory and store to be 0, got %v and %v\", friendlyBytes(rm), friendlyBytes(rd))\n\t}\n\n\tif err := facc.EnableJetStream(limits(24, 192), nil); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Test Adjust\n\tlim := limits(jsconfig.MaxMemory, jsconfig.MaxStore)\n\tl := lim[_EMPTY_]\n\tl.MaxStreams = 10\n\tl.MaxConsumers = 10\n\tlim[_EMPTY_] = l\n\tif err := facc.UpdateJetStreamLimits(lim); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tvar msets []*stream\n\t// Now test max streams and max consumers. Note max consumers is per stream.\n\tfor i := 0; i < 10; i++ {\n\t\tmname := fmt.Sprintf(\"foo.%d\", i)\n\t\tmset, err := facc.addStream(&StreamConfig{Name: strconv.Itoa(i), Storage: MemoryStorage, Subjects: []string{mname}})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t}\n\t\tmsets = append(msets, mset)\n\t}\n\n\t// Remove them all\n\tfor _, mset := range msets {\n\t\tmset.delete()\n\t}\n\n\t// Now try to add one with bytes limit that would exceed the account limit.\n\tif _, err := facc.addStream(&StreamConfig{Name: \"22\", Storage: MemoryStorage, MaxBytes: jsconfig.MaxStore * 2}); err == nil {\n\t\tt.Fatalf(\"Expected error adding stream over limit\")\n\t}\n\n\t// Replicas can't be > 1\n\tif _, err := facc.addStream(&StreamConfig{Name: \"22\", Storage: MemoryStorage, Replicas: 10}); err == nil {\n\t\tt.Fatalf(\"Expected error adding stream over limit\")\n\t}\n\n\t// Test consumers limit against account limit when the stream does not set a limit\n\tmset, err := facc.addStream(&StreamConfig{Name: \"22\", Storage: MemoryStorage, Subjects: []string{\"foo.22\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\toname := fmt.Sprintf(\"O:%d\", i)\n\t\t_, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// This one should fail.\n\tif _, err := mset.addConsumer(&ConsumerConfig{Durable: \"O:22\", AckPolicy: AckExplicit}); err == nil {\n\t\tt.Fatalf(\"Expected error adding consumer over the limit\")\n\t}\n\n\t// Test consumer limit against stream limit\n\tmset.delete()\n\tmset, err = facc.addStream(&StreamConfig{Name: \"22\", Storage: MemoryStorage, Subjects: []string{\"foo.22\"}, MaxConsumers: 5})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tfor i := 0; i < 5; i++ {\n\t\toname := fmt.Sprintf(\"O:%d\", i)\n\t\t_, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// This one should fail.\n\tif _, err := mset.addConsumer(&ConsumerConfig{Durable: \"O:22\", AckPolicy: AckExplicit}); err == nil {\n\t\tt.Fatalf(\"Expected error adding consumer over the limit\")\n\t}\n\n\t// Test the account having smaller limits than the stream\n\tmset.delete()\n\n\tmset, err = facc.addStream(&StreamConfig{Name: \"22\", Storage: MemoryStorage, Subjects: []string{\"foo.22\"}, MaxConsumers: 10})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tl.MaxConsumers = 5\n\tlim[_EMPTY_] = l\n\tif err := facc.UpdateJetStreamLimits(lim); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tfor i := 0; i < 5; i++ {\n\t\toname := fmt.Sprintf(\"O:%d\", i)\n\t\t_, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// This one should fail.\n\tif _, err := mset.addConsumer(&ConsumerConfig{Durable: \"O:22\", AckPolicy: AckExplicit}); err == nil {\n\t\tt.Fatalf(\"Expected error adding consumer over the limit\")\n\t}\n}\n\nfunc TestJetStreamSystemLimitsPlacement(t *testing.T) {\n\tconst smallSystemLimit = 128\n\tconst mediumSystemLimit = smallSystemLimit * 2\n\tconst largeSystemLimit = smallSystemLimit * 3\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {\n\t\t\tmax_mem_store: _MAXMEM_\n\t\t\tmax_file_store: _MAXFILE_\n\t\t\tstore_dir: '%s'\n\t\t}\n\n\t\tserver_tags: [\n\t\t\t_TAG_\n\t\t]\n\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tstoreCnf := func(serverName, clusterName, storeDir, conf string) string {\n\t\tswitch serverName {\n\t\tcase \"S-1\":\n\t\t\tconf = strings.Replace(conf, \"_MAXMEM_\", fmt.Sprint(smallSystemLimit), 1)\n\t\t\tconf = strings.Replace(conf, \"_MAXFILE_\", fmt.Sprint(smallSystemLimit), 1)\n\t\t\treturn strings.Replace(conf, \"_TAG_\", \"small\", 1)\n\t\tcase \"S-2\":\n\t\t\tconf = strings.Replace(conf, \"_MAXMEM_\", fmt.Sprint(mediumSystemLimit), 1)\n\t\t\tconf = strings.Replace(conf, \"_MAXFILE_\", fmt.Sprint(mediumSystemLimit), 1)\n\t\t\treturn strings.Replace(conf, \"_TAG_\", \"medium\", 1)\n\t\tcase \"S-3\":\n\t\t\tconf = strings.Replace(conf, \"_MAXMEM_\", fmt.Sprint(largeSystemLimit), 1)\n\t\t\tconf = strings.Replace(conf, \"_MAXFILE_\", fmt.Sprint(largeSystemLimit), 1)\n\t\t\treturn strings.Replace(conf, \"_TAG_\", \"large\", 1)\n\t\tdefault:\n\t\t\treturn conf\n\t\t}\n\t}\n\n\tcluster := createJetStreamClusterWithTemplateAndModHook(t, tmpl, \"cluster-a\", 3, storeCnf)\n\tdefer cluster.shutdown()\n\n\trequestLeaderStepDown := func(clientURL string) error {\n\t\tnc, err := nats.Connect(clientURL, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tncResp, err := nc.Request(JSApiLeaderStepDown, nil, 3*time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar resp JSApiLeaderStepDownResponse\n\t\tif err := json.Unmarshal(ncResp.Data, &resp); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif resp.Error != nil {\n\t\t\treturn resp.Error\n\t\t}\n\t\tif !resp.Success {\n\t\t\treturn fmt.Errorf(\"leader step down request not successful\")\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tlargeSrv := cluster.servers[2]\n\t// Force large server to be leader\n\terr := checkForErr(15*time.Second, 500*time.Millisecond, func() error {\n\t\tif largeSrv.JetStreamIsLeader() {\n\t\t\treturn nil\n\t\t}\n\n\t\tif err := requestLeaderStepDown(largeSrv.ClientURL()); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn fmt.Errorf(\"large server is not leader\")\n\t})\n\tif err != nil {\n\t\tt.Skipf(\"failed to get desired layout: %s\", err)\n\t}\n\n\tnc, js := jsClientConnect(t, largeSrv)\n\tdefer nc.Close()\n\n\tcases := []struct {\n\t\tname string\n\t\tstorage nats.StorageType\n\t\tcreateMaxBytes int64\n\t\tserverTag string\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"file create large stream on small server\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"small\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on small server\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"small\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on medium server\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"medium\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on medium server\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"medium\",\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file create large stream on large server\",\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"large\",\n\t\t},\n\t\t{\n\t\t\tname: \"memory create large stream on large server\",\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: largeSystemLimit,\n\t\t\tserverTag: \"large\",\n\t\t},\n\t}\n\n\tfor i := 0; i < len(cases) && !t.Failed(); i++ {\n\t\tc := cases[i]\n\t\tt.Run(c.name, func(st *testing.T) {\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: c.storage,\n\t\t\t\tMaxBytes: c.createMaxBytes,\n\t\t\t\tPlacement: &nats.Placement{\n\t\t\t\t\tCluster: \"cluster-a\",\n\t\t\t\t\tTags: []string{c.serverTag},\n\t\t\t\t},\n\t\t\t})\n\t\t\tif c.wantErr && err == nil {\n\t\t\t\tst.Fatalf(\"unexpected stream create success, maxBytes=%d, tag=%s\",\n\t\t\t\t\tc.createMaxBytes, c.serverTag)\n\t\t\t} else if !c.wantErr && err != nil {\n\t\t\t\tst.Fatalf(\"unexpected error: %s\", err)\n\t\t\t}\n\n\t\t\tif err == nil {\n\t\t\t\terr = js.DeleteStream(\"TEST\")\n\t\t\t\trequire_NoError(st, err)\n\t\t\t}\n\t\t})\n\t}\n\n\t// These next two tests should fail because although the stream fits in the\n\t// large and medium server, it doesn't fit on the small server.\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.FileStorage,\n\t\tMaxBytes: smallSystemLimit + 1,\n\t\tReplicas: 3,\n\t})\n\tif err == nil {\n\t\tt.Fatalf(\"unexpected file stream create success, maxBytes=%d, replicas=%d\",\n\t\t\tsi.Config.MaxBytes, si.Config.Replicas)\n\t}\n\n\tsi, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tMaxBytes: smallSystemLimit + 1,\n\t\tReplicas: 3,\n\t})\n\tif err == nil {\n\t\tt.Fatalf(\"unexpected memory stream create success, maxBytes=%d, replicas=%d\",\n\t\t\tsi.Config.MaxBytes, si.Config.Replicas)\n\t}\n\n\tt.Run(\"meta info placement in request - empty request\", func(t *testing.T) {\n\t\tnc, err = nats.Connect(largeSrv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\n\t\tvar resp JSApiLeaderStepDownResponse\n\t\tncResp, err := nc.Request(JSApiLeaderStepDown, []byte(\"{}\"), 3*time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(ncResp.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, resp.Error == nil)\n\t\trequire_True(t, resp.Success)\n\n\t})\n\n\tt.Run(\"meta info placement in request - uninitialized fields\", func(t *testing.T) {\n\t\tnc, err = nats.Connect(largeSrv.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\n\t\tcluster.waitOnClusterReadyWithNumPeers(3)\n\t\tvar resp JSApiLeaderStepDownResponse\n\t\treq, err := json.Marshal(JSApiLeaderStepdownRequest{Placement: nil})\n\t\trequire_NoError(t, err)\n\t\tncResp, err := nc.Request(JSApiLeaderStepDown, req, 10*time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(ncResp.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, resp.Error == nil)\n\t})\n\n}\n\nfunc TestJetStreamStreamLimitUpdate(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\terr := s.GlobalAccount().UpdateJetStreamLimits(map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 128,\n\t\t\tMaxStore: 128,\n\t\t\tMaxStreams: 1,\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, storage := range []nats.StorageType{nats.MemoryStorage, nats.FileStorage} {\n\t\tcfg := &nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: storage,\n\t\t\tMaxBytes: 32,\n\t\t}\n\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Create with same config is idempotent, and must not exceed max streams as it already exists.\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\tcfg.MaxBytes = 16\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\n\t\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t}\n}\n\nfunc TestJetStreamStreamStorageTrackingAndLimits(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tgacc := s.GlobalAccount()\n\n\tal := map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 8192,\n\t\t\tMaxStore: -1,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t}\n\n\tif err := gacc.UpdateJetStreamLimits(al); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tmset, err := gacc.addStream(&StreamConfig{Name: \"LIMITS\", Storage: MemoryStorage, Retention: WorkQueuePolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 100\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"LIMITS\", \"Hello World!\")\n\t}\n\n\tstate := mset.state()\n\tusage := gacc.JetStreamUsage()\n\n\t// Make sure these are working correctly.\n\tif state.Bytes != usage.Memory {\n\t\tt.Fatalf(\"Expected to have stream bytes match memory usage, %d vs %d\", state.Bytes, usage.Memory)\n\t}\n\tif usage.Streams != 1 {\n\t\tt.Fatalf(\"Expected to have 1 stream, got %d\", usage.Streams)\n\t}\n\n\t// Do second stream.\n\tmset2, err := gacc.addStream(&StreamConfig{Name: \"NUM22\", Storage: MemoryStorage, Retention: WorkQueuePolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset2.delete()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"NUM22\", \"Hello World!\")\n\t}\n\n\tstats2 := mset2.state()\n\tusage = gacc.JetStreamUsage()\n\n\tif usage.Memory != (state.Bytes + stats2.Bytes) {\n\t\tt.Fatalf(\"Expected to track both streams, account is %v, stream1 is %v, stream2 is %v\", usage.Memory, state.Bytes, stats2.Bytes)\n\t}\n\n\t// Make sure delete works.\n\tmset2.delete()\n\tstats2 = mset2.state()\n\tusage = gacc.JetStreamUsage()\n\n\tif usage.Memory != (state.Bytes + stats2.Bytes) {\n\t\tt.Fatalf(\"Expected to track both streams, account is %v, stream1 is %v, stream2 is %v\", usage.Memory, state.Bytes, stats2.Bytes)\n\t}\n\n\t// Now drain the first one by consuming the messages.\n\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tmsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tmsg.Respond(nil)\n\t}\n\tnc.Flush()\n\n\tstate = mset.state()\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tusage = gacc.JetStreamUsage()\n\t\tif usage.Memory != 0 {\n\t\t\treturn fmt.Errorf(\"Expected usage memory to be 0, got %d\", usage.Memory)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now send twice the number of messages. Should receive an error at some point, and we will check usage against limits.\n\tvar errSeen string\n\tfor i := 0; i < toSend*2; i++ {\n\t\tresp, _ := nc.Request(\"LIMITS\", []byte(\"The quick brown fox jumped over the...\"), 50*time.Millisecond)\n\t\tif string(resp.Data) != OK {\n\t\t\terrSeen = string(resp.Data)\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif errSeen == \"\" {\n\t\tt.Fatalf(\"Expected to see an error when exceeding the account limits\")\n\t}\n\n\tstate = mset.state()\n\tvar lim JetStreamAccountLimits\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tusage = gacc.JetStreamUsage()\n\t\tlim = al[_EMPTY_]\n\t\tif usage.Memory > uint64(lim.MaxMemory) {\n\t\t\treturn fmt.Errorf(\"Expected memory to not exceed limit of %d, got %d\", lim.MaxMemory, usage.Memory)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// make sure that unlimited accounts work\n\tlim.MaxMemory = -1\n\n\tif err := gacc.UpdateJetStreamLimits(al); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"LIMITS\", \"Hello World!\")\n\t}\n}\n\nfunc TestJetStreamStreamFileTrackingAndLimits(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tgacc := s.GlobalAccount()\n\n\tal := map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxMemory: 8192,\n\t\t\tMaxStore: 9600,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t}\n\n\tif err := gacc.UpdateJetStreamLimits(al); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tmconfig := &StreamConfig{Name: \"LIMITS\", Storage: FileStorage, Retention: WorkQueuePolicy}\n\tmset, err := gacc.addStream(mconfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 100\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"LIMITS\", \"Hello World!\")\n\t}\n\n\tstate := mset.state()\n\tusage := gacc.JetStreamUsage()\n\n\t// Make sure these are working correctly.\n\tif usage.Store != state.Bytes {\n\t\tt.Fatalf(\"Expected to have stream bytes match the store usage, %d vs %d\", usage.Store, state.Bytes)\n\t}\n\tif usage.Streams != 1 {\n\t\tt.Fatalf(\"Expected to have 1 stream, got %d\", usage.Streams)\n\t}\n\n\t// Do second stream.\n\tmconfig2 := &StreamConfig{Name: \"NUM22\", Storage: FileStorage, Retention: WorkQueuePolicy}\n\tmset2, err := gacc.addStream(mconfig2)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset2.delete()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"NUM22\", \"Hello World!\")\n\t}\n\n\tstats2 := mset2.state()\n\tusage = gacc.JetStreamUsage()\n\n\tif usage.Store != (state.Bytes + stats2.Bytes) {\n\t\tt.Fatalf(\"Expected to track both streams, usage is %v, stream1 is %v, stream2 is %v\", usage.Store, state.Bytes, stats2.Bytes)\n\t}\n\n\t// Make sure delete works.\n\tmset2.delete()\n\tstats2 = mset2.state()\n\tusage = gacc.JetStreamUsage()\n\n\tif usage.Store != (state.Bytes + stats2.Bytes) {\n\t\tt.Fatalf(\"Expected to track both streams, account is %v, stream1 is %v, stream2 is %v\", usage.Store, state.Bytes, stats2.Bytes)\n\t}\n\n\t// Now drain the first one by consuming the messages.\n\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tmsg, err := nc.Request(o.requestNextMsgSubject(), nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tmsg.Respond(nil)\n\t}\n\tnc.Flush()\n\n\tstate = mset.state()\n\tusage = gacc.JetStreamUsage()\n\n\tif usage.Memory != 0 {\n\t\tt.Fatalf(\"Expected usage memeory to be 0, got %d\", usage.Memory)\n\t}\n\n\t// Now send twice the number of messages. Should receive an error at some point, and we will check usage against limits.\n\tvar errSeen string\n\tfor i := 0; i < toSend*2; i++ {\n\t\tresp, _ := nc.Request(\"LIMITS\", []byte(\"The quick brown fox jumped over the...\"), 50*time.Millisecond)\n\t\tif string(resp.Data) != OK {\n\t\t\terrSeen = string(resp.Data)\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif errSeen == \"\" {\n\t\tt.Fatalf(\"Expected to see an error when exceeding the account limits\")\n\t}\n\n\tstate = mset.state()\n\tusage = gacc.JetStreamUsage()\n\n\tlim := al[_EMPTY_]\n\tif usage.Memory > uint64(lim.MaxMemory) {\n\t\tt.Fatalf(\"Expected memory to not exceed limit of %d, got %d\", lim.MaxMemory, usage.Memory)\n\t}\n}\n\nfunc TestJetStreamTieredLimits(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tgacc := s.GlobalAccount()\n\n\ttFail := map[string]JetStreamAccountLimits{\n\t\t\"nottheer\": {\n\t\t\tMaxMemory: 8192,\n\t\t\tMaxStore: 9600,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t}\n\n\tif err := gacc.UpdateJetStreamLimits(tFail); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n\n\tmconfig := &StreamConfig{Name: \"LIMITS\", Storage: FileStorage, Retention: WorkQueuePolicy}\n\tmset, err := gacc.addStream(mconfig)\n\tdefer mset.delete()\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"no JetStream default or applicable tiered limit present\")\n\n\ttPass := map[string]JetStreamAccountLimits{\n\t\t\"R1\": {\n\t\t\tMaxMemory: 8192,\n\t\t\tMaxStore: 9600,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxConsumers: -1,\n\t\t},\n\t}\n\n\tif err := gacc.UpdateJetStreamLimits(tPass); err != nil {\n\t\tt.Fatalf(\"Unexpected error updating jetstream account limits: %v\", err)\n\t}\n}\n\ntype obsi struct {\n\tcfg ConsumerConfig\n\tack int\n}\n\ntype info struct {\n\tcfg StreamConfig\n\tstate StreamState\n\tobs []obsi\n}\n\nfunc TestJetStreamSimpleFileRecovery(t *testing.T) {\n\tbase := runtime.NumGoroutine()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tostate := make(map[string]info)\n\n\tnid := nuid.New()\n\trandomSubject := func() string {\n\t\tnid.RandomizePrefix()\n\t\treturn fmt.Sprintf(\"SUBJ.%s\", nid.Next())\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tnumStreams := 10\n\tfor i := 1; i <= numStreams; i++ {\n\t\tmsetName := fmt.Sprintf(\"MMS-%d\", i)\n\t\tsubjects := []string{randomSubject(), randomSubject(), randomSubject()}\n\t\tmsetConfig := StreamConfig{\n\t\t\tName: msetName,\n\t\t\tStorage: FileStorage,\n\t\t\tSubjects: subjects,\n\t\t\tMaxMsgs: 100,\n\t\t}\n\t\tmset, err := acc.addStream(&msetConfig)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error adding stream %q: %v\", msetName, err)\n\t\t}\n\n\t\ttoSend := rand.Intn(100) + 1\n\t\tfor n := 1; n <= toSend; n++ {\n\t\t\tmsg := fmt.Sprintf(\"Hello %d\", n*i)\n\t\t\tsubj := subjects[rand.Intn(len(subjects))]\n\t\t\tsendStreamMsg(t, nc, subj, msg)\n\t\t}\n\t\t// Create up to 5 consumers.\n\t\tnumObs := rand.Intn(5) + 1\n\t\tvar obs []obsi\n\t\tfor n := 1; n <= numObs; n++ {\n\t\t\toname := fmt.Sprintf(\"WQ-%d-%d\", i, n)\n\t\t\to, err := mset.addConsumer(workerModeConfig(oname))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Now grab some messages.\n\t\t\ttoReceive := rand.Intn(toSend) + 1\n\t\t\trsubj := o.requestNextMsgSubject()\n\t\t\tfor r := 0; r < toReceive; r++ {\n\t\t\t\tresp, err := nc.Request(rsubj, nil, time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif resp != nil {\n\t\t\t\t\tresp.Respond(nil)\n\t\t\t\t}\n\t\t\t}\n\t\t\tobs = append(obs, obsi{o.config(), toReceive})\n\t\t}\n\t\tostate[msetName] = info{mset.config(), mset.state(), obs}\n\t}\n\tpusage := acc.JetStreamUsage()\n\tnc.Flush()\n\n\t// Shutdown and restart and make sure things come back.\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tdelta := (runtime.NumGoroutine() - base)\n\t\tif delta > 3 {\n\t\t\treturn fmt.Errorf(\"%d Go routines still exist post Shutdown()\", delta)\n\t\t}\n\t\treturn nil\n\t})\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tacc = s.GlobalAccount()\n\n\tnusage := acc.JetStreamUsage()\n\tif !reflect.DeepEqual(nusage, pusage) {\n\t\tt.Fatalf(\"Usage does not match after restore: %+v vs %+v\", nusage, pusage)\n\t}\n\n\tfor mname, info := range ostate {\n\t\tmset, err := acc.lookupStream(mname)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to find a stream for %q\", mname)\n\t\t}\n\t\tif state := mset.state(); !reflect.DeepEqual(state, info.state) {\n\t\t\tt.Fatalf(\"State does not match: %+v vs %+v\", state, info.state)\n\t\t}\n\t\tif cfg := mset.config(); !reflect.DeepEqual(cfg, info.cfg) {\n\t\t\tt.Fatalf(\"Configs do not match: %+v vs %+v\", cfg, info.cfg)\n\t\t}\n\t\t// Consumers.\n\t\tif mset.numConsumers() != len(info.obs) {\n\t\t\tt.Fatalf(\"Number of consumers do not match: %d vs %d\", mset.numConsumers(), len(info.obs))\n\t\t}\n\t\tfor _, oi := range info.obs {\n\t\t\tif o := mset.lookupConsumer(oi.cfg.Durable); o != nil {\n\t\t\t\tif uint64(oi.ack+1) != o.nextSeq() {\n\t\t\t\t\tt.Fatalf(\"Consumer next seq is not correct: %d vs %d\", oi.ack+1, o.nextSeq())\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Expected to get an consumer\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamPushConsumerFlowControl(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := nc.SubscribeSync(nats.NewInbox())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tDeliverSubject: sub.Subject,\n\t\t\tFlowControl: true,\n\t\t\tHeartbeat: 5 * time.Second,\n\t\t},\n\t}\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"dlc\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", ccResp.Error)\n\t}\n\n\t// Grab the low level consumer so we can manually set the fc max.\n\tif mset, err := s.GlobalAccount().lookupStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Error looking up stream: %v\", err)\n\t} else if obs := mset.lookupConsumer(\"dlc\"); obs == nil {\n\t\tt.Fatalf(\"Error looking up stream: %v\", err)\n\t} else {\n\t\tobs.mu.Lock()\n\t\tobs.setMaxPendingBytes(16 * 1024)\n\t\tobs.mu.Unlock()\n\t}\n\n\tmsgSize := 1024\n\tmsg := make([]byte, msgSize)\n\tcrand.Read(msg)\n\n\tsendBatch := func(n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(\"TEST\", msg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckSubPending := func(numExpected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsendBatch(100)\n\tcheckSubPending(2) // First four data and flowcontrol from slow start pause.\n\n\tvar n int\n\tfor m, err := sub.NextMsg(time.Second); err == nil; m, err = sub.NextMsg(time.Second) {\n\t\tif m.Subject == \"TEST\" {\n\t\t\tn++\n\t\t} else {\n\t\t\t// This should be a FC control message.\n\t\t\tif m.Header.Get(\"Status\") != \"100\" {\n\t\t\t\tt.Fatalf(\"Expected a 100 status code, got %q\", m.Header.Get(\"Status\"))\n\t\t\t}\n\t\t\tif m.Header.Get(\"Description\") != \"FlowControl Request\" {\n\t\t\t\tt.Fatalf(\"Wrong description, got %q\", m.Header.Get(\"Description\"))\n\t\t\t}\n\t\t\tm.Respond(nil)\n\t\t}\n\t}\n\n\tif n != 100 {\n\t\tt.Fatalf(\"Expected to receive all 100 messages but got %d\", n)\n\t}\n}\n\nfunc TestJetStreamFlowControlRequiresHeartbeats(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tFlowControl: true,\n\t}); err == nil || IsNatsErr(err, JSConsumerWithFlowControlNeedsHeartbeats) {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamPushConsumerIdleHeartbeats(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := nc.SubscribeSync(nats.NewInbox())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Test errors first\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: sub.Subject,\n\t\t\tHeartbeat: time.Millisecond,\n\t\t},\n\t}\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.Error == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\t// Set acceptable heartbeat.\n\tobsReq.Config.Heartbeat = 100 * time.Millisecond\n\treq, err = json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\tccResp.Error = nil\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs < 9 {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, 9)\n\t\t}\n\t\treturn nil\n\t})\n\tm, _ := sub.NextMsg(0)\n\tif m.Header.Get(\"Status\") != \"100\" {\n\t\tt.Fatalf(\"Expected a 100 status code, got %q\", m.Header.Get(\"Status\"))\n\t}\n\tif m.Header.Get(\"Description\") != \"Idle Heartbeat\" {\n\t\tt.Fatalf(\"Wrong description, got %q\", m.Header.Get(\"Description\"))\n\t}\n}\n\nfunc TestJetStreamPushConsumerIdleHeartbeatsWithFilterSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\", \"bar\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\thbC := make(chan *nats.Msg, 8)\n\tsub, err := nc.ChanSubscribe(nats.NewInbox(), hbC)\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: sub.Subject,\n\t\t\tFilterSubject: \"bar\",\n\t\t\tHeartbeat: 100 * time.Millisecond,\n\t\t},\n\t}\n\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tst := time.NewTicker(10 * time.Millisecond)\n\tdefer st.Stop()\n\n\tdone := time.NewTimer(time.Second)\n\tdefer done.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-st.C:\n\t\t\tjs.Publish(\"foo\", []byte(\"HELLO FOO\"))\n\t\tcase <-done.C:\n\t\t\tt.Fatalf(\"Expected to have seen idle heartbeats for consumer\")\n\t\tcase <-hbC:\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestJetStreamPushConsumerIdleHeartbeatsWithNoInterest(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tdsubj := \"d.22\"\n\thbC := make(chan *nats.Msg, 8)\n\tsub, err := nc.ChanSubscribe(\"d.>\", hbC)\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\tobsReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: dsubj,\n\t\t\tHeartbeat: 100 * time.Millisecond,\n\t\t},\n\t}\n\n\treq, err := json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", ccResp.Error)\n\t}\n\n\tdone := time.NewTimer(400 * time.Millisecond)\n\tdefer done.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-done.C:\n\t\t\treturn\n\t\tcase m := <-hbC:\n\t\t\tif m.Header.Get(\"Status\") == \"100\" {\n\t\t\t\tt.Fatalf(\"Did not expect to see a heartbeat with no formal interest\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamInfoAPIWithHeaders(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tm := nats.NewMsg(JSApiAccountInfo)\n\tm.Header.Add(\"Accept-Encoding\", \"json\")\n\tm.Header.Add(\"Authorization\", \"s3cr3t\")\n\tm.Data = []byte(\"HELLO-JS!\")\n\n\tresp, err := nc.RequestMsg(m, time.Second)\n\trequire_NoError(t, err)\n\n\tvar info JSApiAccountInfoResponse\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Error != nil {\n\t\tt.Fatalf(\"Received an error: %+v\", info.Error)\n\t}\n}\n\nfunc TestJetStreamRequestAPI(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// This will get the current information about usage and limits for this account.\n\tresp, err := nc.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tvar info JSApiAccountInfoResponse\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// JS API level should be set.\n\trequire_Equal(t, info.API.Level, JSApiLevel)\n\n\t// Now create a stream.\n\tmsetCfg := StreamConfig{\n\t\tName: \"MSET22\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tMaxMsgs: 100,\n\t}\n\treq, err := json.Marshal(msetCfg)\n\trequire_NoError(t, err)\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamCreateT, msetCfg.Name), req, time.Second)\n\tvar scResp JSApiStreamCreateResponse\n\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.StreamInfo == nil || scResp.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", scResp.Error)\n\t}\n\tif time.Since(scResp.Created) > time.Second {\n\t\tt.Fatalf(\"Created time seems wrong: %v\\n\", scResp.Created)\n\t}\n\n\t// Check that the name in config has to match the name in the subject\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamCreateT, \"BOB\"), req, time.Second)\n\tscResp.Error, scResp.StreamInfo = nil, nil\n\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckNatsError(t, scResp.Error, JSStreamMismatchErr)\n\n\t// Check that update works.\n\tmsetCfg.Subjects = []string{\"foo\", \"bar\", \"baz\"}\n\tmsetCfg.MaxBytes = 2222222\n\treq, err = json.Marshal(msetCfg)\n\trequire_NoError(t, err)\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamUpdateT, msetCfg.Name), req, time.Second)\n\tscResp.Error, scResp.StreamInfo = nil, nil\n\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.StreamInfo == nil || scResp.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response: %+v\", scResp.Error)\n\t}\n\n\t// Check that updating a non existing stream fails\n\tcfg := StreamConfig{\n\t\tName: \"UNKNOWN_STREAM\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\treq, err = json.Marshal(cfg)\n\trequire_NoError(t, err)\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), req, time.Second)\n\tscResp.Error, scResp.StreamInfo = nil, nil\n\tif err := json.Unmarshal(resp.Data, &scResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif scResp.StreamInfo != nil || scResp.Error == nil || scResp.Error.Code != 404 {\n\t\tt.Fatalf(\"Unexpected error: %+v\", scResp.Error)\n\t}\n\n\t// Now lookup info again and see that we can see the new stream.\n\tresp, err = nc.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tif err = json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Streams != 1 {\n\t\tt.Fatalf(\"Expected to see 1 Stream, got %d\", info.Streams)\n\t}\n\n\t// Make sure list names works.\n\tresp, err = nc.Request(JSApiStreams, nil, time.Second)\n\trequire_NoError(t, err)\n\tvar namesResponse JSApiStreamNamesResponse\n\tif err = json.Unmarshal(resp.Data, &namesResponse); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif len(namesResponse.Streams) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(namesResponse.Streams))\n\t}\n\tif namesResponse.Total != 1 {\n\t\tt.Fatalf(\"Expected total to be 1 but got %d\", namesResponse.Total)\n\t}\n\tif namesResponse.Offset != 0 {\n\t\tt.Fatalf(\"Expected offset to be 0 but got %d\", namesResponse.Offset)\n\t}\n\tif namesResponse.Limit != JSApiNamesLimit {\n\t\tt.Fatalf(\"Expected limit to be %d but got %d\", JSApiNamesLimit, namesResponse.Limit)\n\t}\n\tif namesResponse.Streams[0] != msetCfg.Name {\n\t\tt.Fatalf(\"Expected to get %q, but got %q\", msetCfg.Name, namesResponse.Streams[0])\n\t}\n\n\t// Now do detailed version.\n\tresp, err = nc.Request(JSApiStreamList, nil, time.Second)\n\trequire_NoError(t, err)\n\tvar listResponse JSApiStreamListResponse\n\tif err = json.Unmarshal(resp.Data, &listResponse); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif len(listResponse.Streams) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(listResponse.Streams))\n\t}\n\tif listResponse.Total != 1 {\n\t\tt.Fatalf(\"Expected total to be 1 but got %d\", listResponse.Total)\n\t}\n\tif listResponse.Offset != 0 {\n\t\tt.Fatalf(\"Expected offset to be 0 but got %d\", listResponse.Offset)\n\t}\n\tif listResponse.Limit != JSApiListLimit {\n\t\tt.Fatalf(\"Expected limit to be %d but got %d\", JSApiListLimit, listResponse.Limit)\n\t}\n\tif listResponse.Streams[0].Config.Name != msetCfg.Name {\n\t\tt.Fatalf(\"Expected to get %q, but got %q\", msetCfg.Name, listResponse.Streams[0].Config.Name)\n\t}\n\n\t// Now send some messages, then we can poll for info on this stream.\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.Request(\"foo\", []byte(\"WELCOME JETSTREAM\"), time.Second)\n\t}\n\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, msetCfg.Name), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar msi StreamInfo\n\tif err = json.Unmarshal(resp.Data, &msi); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif msi.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected to get %d msgs, got %d\", toSend, msi.State.Msgs)\n\t}\n\tif time.Since(msi.Created) > time.Second {\n\t\tt.Fatalf(\"Created time seems wrong: %v\\n\", msi.Created)\n\t}\n\n\t// Looking up one that is not there should yield an error.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"BOB\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar bResp JSApiStreamInfoResponse\n\tif err = json.Unmarshal(resp.Data, &bResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckNatsError(t, bResp.Error, JSStreamNotFoundErr)\n\n\t// Now create a consumer.\n\tdelivery := nats.NewInbox()\n\tobsReq := CreateConsumerRequest{\n\t\tStream: msetCfg.Name,\n\t\tConfig: ConsumerConfig{DeliverSubject: delivery},\n\t}\n\treq, err = json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, msetCfg.Name), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Ephemerals are now not rejected when there is no interest.\n\tif ccResp.ConsumerInfo == nil || ccResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", ccResp)\n\t}\n\tif time.Since(ccResp.Created) > time.Second {\n\t\tt.Fatalf(\"Created time seems wrong: %v\\n\", ccResp.Created)\n\t}\n\n\t// Now create subscription and make sure we get proper response.\n\tsub, _ := nc.SubscribeSync(delivery)\n\tnc.Flush()\n\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != toSend {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, toSend)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check that we get an error if the stream name in the subject does not match the config.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"BOB\"), req, time.Second)\n\trequire_NoError(t, err)\n\tccResp.Error, ccResp.ConsumerInfo = nil, nil\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Since we do not have interest this should have failed.\n\tcheckNatsError(t, ccResp.Error, JSStreamMismatchErr)\n\n\t// Get the list of all of the consumers for our stream.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumersT, msetCfg.Name), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar clResponse JSApiConsumerNamesResponse\n\tif err = json.Unmarshal(resp.Data, &clResponse); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif len(clResponse.Consumers) != 1 {\n\t\tt.Fatalf(\"Expected only 1 consumer but got %d\", len(clResponse.Consumers))\n\t}\n\t// Now let's get info about our consumer.\n\tcName := clResponse.Consumers[0]\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerInfoT, msetCfg.Name, cName), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar oinfo ConsumerInfo\n\tif err = json.Unmarshal(resp.Data, &oinfo); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Do some sanity checking.\n\t// Must match consumer.go\n\tconst randConsumerNameLen = 8\n\tif len(oinfo.Name) != randConsumerNameLen {\n\t\tt.Fatalf(\"Expected ephemeral name, got %q\", oinfo.Name)\n\t}\n\tif len(oinfo.Config.Durable) != 0 {\n\t\tt.Fatalf(\"Expected no durable name, but got %q\", oinfo.Config.Durable)\n\t}\n\tif oinfo.Config.DeliverSubject != delivery {\n\t\tt.Fatalf(\"Expected to have delivery subject of %q, got %q\", delivery, oinfo.Config.DeliverSubject)\n\t}\n\tif oinfo.Delivered.Consumer != 10 {\n\t\tt.Fatalf(\"Expected consumer delivered sequence of 10, got %d\", oinfo.Delivered.Consumer)\n\t}\n\tif oinfo.AckFloor.Consumer != 10 {\n\t\tt.Fatalf(\"Expected ack floor to be 10, got %d\", oinfo.AckFloor.Consumer)\n\t}\n\n\t// Now delete the consumer.\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiConsumerDeleteT, msetCfg.Name, cName), nil, time.Second)\n\tvar cdResp JSApiConsumerDeleteResponse\n\tif err = json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !cdResp.Success || cdResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", ccResp)\n\t}\n\n\t// Make sure we can't create a durable using the ephemeral API endpoint.\n\tobsReq = CreateConsumerRequest{\n\t\tStream: msetCfg.Name,\n\t\tConfig: ConsumerConfig{Durable: \"myd\", DeliverSubject: delivery},\n\t}\n\treq, err = json.Marshal(obsReq)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, msetCfg.Name), req, time.Second)\n\trequire_NoError(t, err)\n\tccResp.Error, ccResp.ConsumerInfo = nil, nil\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckNatsError(t, ccResp.Error, JSConsumerEphemeralWithDurableNameErr)\n\n\t// Now make sure we can create a durable on the subject with the proper name.\n\tresp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, msetCfg.Name, obsReq.Config.Durable), req, time.Second)\n\tccResp.Error, ccResp.ConsumerInfo = nil, nil\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif ccResp.ConsumerInfo == nil || ccResp.Error != nil {\n\t\tt.Fatalf(\"Did not receive correct response\")\n\t}\n\n\t// Make sure empty durable in cfg does not work\n\tobsReq2 := CreateConsumerRequest{\n\t\tStream: msetCfg.Name,\n\t\tConfig: ConsumerConfig{DeliverSubject: delivery},\n\t}\n\treq2, err := json.Marshal(obsReq2)\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiDurableCreateT, msetCfg.Name, obsReq.Config.Durable), req2, time.Second)\n\trequire_NoError(t, err)\n\tccResp.Error, ccResp.ConsumerInfo = nil, nil\n\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckNatsError(t, ccResp.Error, JSConsumerDurableNameNotSetErr)\n\n\t// Now delete a msg.\n\tdreq := JSApiMsgDeleteRequest{Seq: 2}\n\tdreqj, err := json.Marshal(dreq)\n\trequire_NoError(t, err)\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiMsgDeleteT, msetCfg.Name), dreqj, time.Second)\n\tvar delMsgResp JSApiMsgDeleteResponse\n\tif err = json.Unmarshal(resp.Data, &delMsgResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !delMsgResp.Success || delMsgResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", delMsgResp.Error)\n\t}\n\n\t// Now purge the stream.\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, msetCfg.Name), nil, time.Second)\n\tvar pResp JSApiStreamPurgeResponse\n\tif err = json.Unmarshal(resp.Data, &pResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !pResp.Success || pResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", pResp)\n\t}\n\tif pResp.Purged != 9 {\n\t\tt.Fatalf(\"Expected 9 purged, got %d\", pResp.Purged)\n\t}\n\n\t// Now delete the stream.\n\tresp, _ = nc.Request(fmt.Sprintf(JSApiStreamDeleteT, msetCfg.Name), nil, time.Second)\n\tvar dResp JSApiStreamDeleteResponse\n\tif err = json.Unmarshal(resp.Data, &dResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif !dResp.Success || dResp.Error != nil {\n\t\tt.Fatalf(\"Got a bad response %+v\", dResp.Error)\n\t}\n\n\t// Now grab stats again.\n\t// This will get the current information about usage and limits for this account.\n\tresp, err = nc.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Streams != 0 {\n\t\tt.Fatalf(\"Expected no remaining streams, got %d\", info.Streams)\n\t}\n\n\t// Test that we can send nil or an empty legal json for requests that take no args.\n\t// We know this stream does not exist, this just checking request processing.\n\tcheckEmptyReqArg := func(arg string) {\n\t\tt.Helper()\n\t\tvar req []byte\n\t\tif len(arg) > 0 {\n\t\t\treq = []byte(arg)\n\t\t}\n\t\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamDeleteT, \"foo_bar_baz\"), req, time.Second)\n\t\tvar dResp JSApiStreamDeleteResponse\n\t\tif err = json.Unmarshal(resp.Data, &dResp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif dResp.Error == nil || dResp.Error.Code != 404 {\n\t\t\tt.Fatalf(\"Got a bad response, expected a 404 response %+v\", dResp.Error)\n\t\t}\n\t}\n\n\tcheckEmptyReqArg(\"\")\n\tcheckEmptyReqArg(\"{}\")\n\tcheckEmptyReqArg(\" {} \")\n\tcheckEmptyReqArg(\" { } \")\n}\n\nfunc TestJetStreamFilteredStreamNames(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Create some streams.\n\tvar snid int\n\tcreateStream := func(subjects []string) {\n\t\tt.Helper()\n\t\tsnid++\n\t\tname := fmt.Sprintf(\"S-%d\", snid)\n\t\tsc := &StreamConfig{Name: name, Subjects: subjects}\n\t\tif _, err := s.GlobalAccount().addStream(sc); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t}\n\t}\n\n\tcreateStream([]string{\"foo\"}) // S1\n\tcreateStream([]string{\"bar\"}) // S2\n\tcreateStream([]string{\"baz\"}) // S3\n\tcreateStream([]string{\"foo.*\", \"bar.*\"}) // S4\n\tcreateStream([]string{\"foo-1.22\", \"bar-1.33\"}) // S5\n\n\texpectStreams := func(filter string, streams []string) {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(&JSApiStreamNamesRequest{Subject: filter})\n\t\tr, _ := nc.Request(JSApiStreams, req, time.Second)\n\t\tvar resp JSApiStreamNamesResponse\n\t\tif err := json.Unmarshal(r.Data, &resp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif len(resp.Streams) != len(streams) {\n\t\t\tt.Fatalf(\"Expected %d results, got %d\", len(streams), len(resp.Streams))\n\t\t}\n\t}\n\n\texpectStreams(\"foo\", []string{\"S1\"})\n\texpectStreams(\"bar\", []string{\"S2\"})\n\texpectStreams(\"baz\", []string{\"S3\"})\n\texpectStreams(\"*\", []string{\"S1\", \"S2\", \"S3\"})\n\texpectStreams(\">\", []string{\"S1\", \"S2\", \"S3\", \"S4\", \"S5\"})\n\texpectStreams(\"*.*\", []string{\"S4\", \"S5\"})\n\texpectStreams(\"*.22\", []string{\"S4\", \"S5\"})\n}\n\nfunc TestJetStreamUpdateStream(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Test basic updates. We allow changing the subjects, limits, and no_ack along with replicas(TBD w/ cluster)\n\t\t\tcfg := *c.mconfig\n\n\t\t\t// Can't change name.\n\t\t\tcfg.Name = \"bar\"\n\t\t\tif err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), \"name must match\") {\n\t\t\t\tt.Fatalf(\"Expected error trying to update name\")\n\t\t\t}\n\t\t\t// Can't change storage types.\n\t\t\tcfg = *c.mconfig\n\t\t\tif cfg.Storage == FileStorage {\n\t\t\t\tcfg.Storage = MemoryStorage\n\t\t\t} else {\n\t\t\t\tcfg.Storage = FileStorage\n\t\t\t}\n\t\t\tif err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), \"can not change\") {\n\t\t\t\tt.Fatalf(\"Expected error trying to change Storage\")\n\t\t\t}\n\t\t\t// Can't change replicas > 1 for now.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.Replicas = 10\n\t\t\tif err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), \"maximum replicas\") {\n\t\t\t\tt.Fatalf(\"Expected error trying to change Replicas\")\n\t\t\t}\n\t\t\t// Can't change limits policy.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.Retention = WorkQueuePolicy\n\t\t\tif err := mset.update(&cfg); err == nil || !strings.Contains(err.Error(), \"can not change\") {\n\t\t\t\tt.Fatalf(\"Expected error trying to change Retention\")\n\t\t\t}\n\n\t\t\t// Now test changing limits.\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tpending := uint64(100)\n\t\t\tfor i := uint64(0); i < pending; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"0123456789\")\n\t\t\t}\n\t\t\tpendingBytes := mset.state().Bytes\n\n\t\t\tcheckPending := func(msgs, bts uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tstate := mset.state()\n\t\t\t\tif state.Msgs != msgs {\n\t\t\t\t\tt.Fatalf(\"Expected %d messages, got %d\", msgs, state.Msgs)\n\t\t\t\t}\n\t\t\t\tif state.Bytes != bts {\n\t\t\t\t\tt.Fatalf(\"Expected %d bytes, got %d\", bts, state.Bytes)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckPending(pending, pendingBytes)\n\n\t\t\t// Update msgs to higher.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.MaxMsgs = int64(pending * 2)\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\tif mset.config().MaxMsgs != cfg.MaxMsgs {\n\t\t\t\tt.Fatalf(\"Expected the change to take effect, %d vs %d\", mset.config().MaxMsgs, cfg.MaxMsgs)\n\t\t\t}\n\t\t\tcheckPending(pending, pendingBytes)\n\n\t\t\t// Update msgs to lower.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.MaxMsgs = int64(pending / 2)\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\tif mset.config().MaxMsgs != cfg.MaxMsgs {\n\t\t\t\tt.Fatalf(\"Expected the change to take effect, %d vs %d\", mset.config().MaxMsgs, cfg.MaxMsgs)\n\t\t\t}\n\t\t\tcheckPending(pending/2, pendingBytes/2)\n\t\t\t// Now do bytes.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.MaxBytes = int64(pendingBytes / 4)\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\tif mset.config().MaxBytes != cfg.MaxBytes {\n\t\t\t\tt.Fatalf(\"Expected the change to take effect, %d vs %d\", mset.config().MaxBytes, cfg.MaxBytes)\n\t\t\t}\n\t\t\tcheckPending(pending/4, pendingBytes/4)\n\n\t\t\t// Now do age.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.MaxAge = time.Second\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\t// Just wait a bit for expiration.\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\tif mset.config().MaxAge != cfg.MaxAge {\n\t\t\t\tt.Fatalf(\"Expected the change to take effect, %d vs %d\", mset.config().MaxAge, cfg.MaxAge)\n\t\t\t}\n\t\t\tcheckPending(0, 0)\n\n\t\t\t// Now put back to original.\n\t\t\tcfg = *c.mconfig\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\tfor i := uint64(0); i < pending; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"0123456789\")\n\t\t\t}\n\n\t\t\t// subject changes.\n\t\t\t// Add in a subject first.\n\t\t\tcfg = *c.mconfig\n\t\t\tcfg.Subjects = []string{\"foo\", \"bar\"}\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\t// Make sure we can still send to foo.\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"0123456789\")\n\t\t\t// And we can now send to bar.\n\t\t\tsendStreamMsg(t, nc, \"bar\", \"0123456789\")\n\t\t\t// Now delete both and change to baz only.\n\t\t\tcfg.Subjects = []string{\"baz\"}\n\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t}\n\t\t\t// Make sure we do not get response acks for \"foo\" or \"bar\".\n\t\t\tif resp, err := nc.Request(\"foo\", nil, 25*time.Millisecond); err == nil || resp != nil {\n\t\t\t\tt.Fatalf(\"Expected no response from jetstream for deleted subject: %q\", \"foo\")\n\t\t\t}\n\t\t\tif resp, err := nc.Request(\"bar\", nil, 25*time.Millisecond); err == nil || resp != nil {\n\t\t\t\tt.Fatalf(\"Expected no response from jetstream for deleted subject: %q\", \"bar\")\n\t\t\t}\n\t\t\t// Make sure we can send to \"baz\"\n\t\t\tsendStreamMsg(t, nc, \"baz\", \"0123456789\")\n\t\t\tif nmsgs := mset.state().Msgs; nmsgs != pending+3 {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", pending+3, nmsgs)\n\t\t\t}\n\n\t\t\t// FileStore restarts for config save.\n\t\t\tcfg = *c.mconfig\n\t\t\tif cfg.Storage == FileStorage {\n\t\t\t\tcfg.Subjects = []string{\"foo\", \"bar\"}\n\t\t\t\tcfg.MaxMsgs = 2222\n\t\t\t\tcfg.MaxBytes = 3333333\n\t\t\t\tcfg.MaxAge = 22 * time.Hour\n\t\t\t\tif err := mset.update(&cfg); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error %v\", err)\n\t\t\t\t}\n\t\t\t\t// Pull since certain defaults etc are set in processing.\n\t\t\t\tcfg = mset.config()\n\n\t\t\t\t// Restart the\n\t\t\t\t// Capture port since it was dynamic.\n\t\t\t\tu, _ := url.Parse(s.ClientURL())\n\t\t\t\tport, _ := strconv.Atoi(u.Port())\n\n\t\t\t\t// Stop current\n\t\t\t\tsd := s.JetStreamConfig().StoreDir\n\t\t\t\ts.Shutdown()\n\t\t\t\t// Restart.\n\t\t\t\ts = RunJetStreamServerOnPort(port, sd)\n\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\tmset, err = s.GlobalAccount().lookupStream(cfg.Name)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected to find a stream for %q\", cfg.Name)\n\t\t\t\t}\n\t\t\t\trestored_cfg := mset.config()\n\t\t\t\tif !reflect.DeepEqual(cfg, restored_cfg) {\n\t\t\t\t\tt.Fatalf(\"restored configuration does not match: \\n%+v\\n vs \\n%+v\", restored_cfg, cfg)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamDeleteMsg(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tpubTen := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t\tjs.Publish(\"foo\", []byte(\"Hello World!\"))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpubTen()\n\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 10 {\n\t\t\t\tt.Fatalf(\"Expected 10 messages, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tbytesPerMsg := state.Bytes / 10\n\t\t\tif bytesPerMsg == 0 {\n\t\t\t\tt.Fatalf(\"Expected non-zero bytes for msg size\")\n\t\t\t}\n\n\t\t\tdeleteAndCheck := func(seq, expectedFirstSeq uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tbeforeState := mset.state()\n\t\t\t\tif removed, _ := mset.deleteMsg(seq); !removed {\n\t\t\t\t\tt.Fatalf(\"Expected the delete of sequence %d to succeed\", seq)\n\t\t\t\t}\n\t\t\t\texpectedState := beforeState\n\t\t\t\texpectedState.Msgs--\n\t\t\t\texpectedState.Bytes -= bytesPerMsg\n\t\t\t\texpectedState.FirstSeq = expectedFirstSeq\n\n\t\t\t\tsm, err := mset.getMsg(expectedFirstSeq)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error fetching message for seq: %d - %v\", expectedFirstSeq, err)\n\t\t\t\t}\n\t\t\t\texpectedState.FirstTime = sm.Time\n\t\t\t\texpectedState.Deleted = nil\n\t\t\t\texpectedState.NumDeleted = 0\n\n\t\t\t\tafterState := mset.state()\n\t\t\t\tafterState.Deleted = nil\n\t\t\t\tafterState.NumDeleted = 0\n\n\t\t\t\t// Ignore first time in this test.\n\t\t\t\tif !reflect.DeepEqual(afterState, expectedState) {\n\t\t\t\t\tt.Fatalf(\"Stats not what we expected. Expected %+v, got %+v\\n\", expectedState, afterState)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Delete one from the middle\n\t\t\tdeleteAndCheck(5, 1)\n\t\t\t// Now make sure sequences are updated properly.\n\t\t\t// Delete first msg.\n\t\t\tdeleteAndCheck(1, 2)\n\t\t\t// Now last\n\t\t\tdeleteAndCheck(10, 2)\n\t\t\t// Now gaps.\n\t\t\tdeleteAndCheck(3, 2)\n\t\t\tdeleteAndCheck(2, 4)\n\n\t\t\tmset.purge(nil)\n\t\t\t// Put ten more one.\n\t\t\tpubTen()\n\t\t\tdeleteAndCheck(11, 12)\n\t\t\tdeleteAndCheck(15, 12)\n\t\t\tdeleteAndCheck(16, 12)\n\t\t\tdeleteAndCheck(20, 12)\n\n\t\t\t// Only file storage beyond here.\n\t\t\tif c.mconfig.Storage == MemoryStorage {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Capture port since it was dynamic.\n\t\t\tu, _ := url.Parse(s.ClientURL())\n\t\t\tport, _ := strconv.Atoi(u.Port())\n\t\t\tsd := s.JetStreamConfig().StoreDir\n\n\t\t\t// Shutdown the\n\t\t\ts.Shutdown()\n\n\t\t\ts = RunJetStreamServerOnPort(port, sd)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err = s.GlobalAccount().lookupStream(\"foo\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected to get the stream back\")\n\t\t\t}\n\n\t\t\texpected := StreamState{Msgs: 6, Bytes: 6 * bytesPerMsg, FirstSeq: 12, LastSeq: 20, NumSubjects: 1}\n\t\t\tstate = mset.state()\n\t\t\tstate.FirstTime, state.LastTime, state.Deleted, state.NumDeleted = time.Time{}, time.Time{}, nil, 0\n\n\t\t\tif !reflect.DeepEqual(expected, state) {\n\t\t\t\tt.Fatalf(\"State not what we expected. Expected %+v, got %+v\\n\", expected, state)\n\t\t\t}\n\n\t\t\t// Now create an consumer and make sure we get the right sequence.\n\t\t\tnc = clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tdelivery := nats.NewInbox()\n\t\t\tsub, _ := nc.SubscribeSync(delivery)\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: delivery, FilterSubject: \"foo\"})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\texpectedStoreSeq := []uint64{12, 13, 14, 17, 18, 19}\n\n\t\t\tfor i := 0; i < 6; i++ {\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif o.streamSeqFromReply(m.Reply) != expectedStoreSeq[i] {\n\t\t\t\t\tt.Fatalf(\"Expected store seq of %d, got %d\", expectedStoreSeq[i], o.streamSeqFromReply(m.Reply))\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/jetstream/issues/396\nfunc TestJetStreamLimitLockBug(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxMsgs: 10,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxMsgs: 10,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tsendStreamMsg(t, nc, \"foo\", \"ok\")\n\t\t\t}\n\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 10 {\n\t\t\t\tt.Fatalf(\"Expected 10 messages, got %d\", state.Msgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamNextMsgNoInterest(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tcfg := &StreamConfig{Name: \"foo\", Storage: FileStorage}\n\t\t\tmset, err := s.GlobalAccount().addStream(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\n\t\t\tnc := clientConnectWithOldRequest(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Now create an consumer and make sure it functions properly.\n\t\t\to, err := mset.addConsumer(workerModeConfig(\"WQ\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tnextSubj := o.requestNextMsgSubject()\n\n\t\t\t// Queue up a worker but use a short time out.\n\t\t\tif _, err := nc.Request(nextSubj, nil, time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected a timeout error and no response with acks suppressed\")\n\t\t\t}\n\t\t\t// Now send a message, the worker from above will still be known but we want to make\n\t\t\t// sure the system detects that so we will do a request for next msg right behind it.\n\t\t\tnc.Publish(\"foo\", []byte(\"OK\"))\n\t\t\tif msg, err := nc.Request(nextSubj, nil, 5*time.Millisecond); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else {\n\t\t\t\tmsg.Respond(nil) // Ack\n\t\t\t}\n\t\t\t// Now queue up 10 workers.\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tif _, err := nc.Request(nextSubj, nil, time.Microsecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected a timeout error and no response with acks suppressed\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now publish ten messages.\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tnc.Publish(\"foo\", []byte(\"OK\"))\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tif msg, err := nc.Request(nextSubj, nil, 10*time.Millisecond); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error for %d: %v\", i, err)\n\t\t\t\t} else {\n\t\t\t\t\tmsg.Respond(nil) // Ack\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tostate := o.info()\n\t\t\t\tif ostate.AckFloor.Stream != 11 || ostate.NumAckPending > 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Inconsistent ack state: %+v\", ostate)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMsgHeaders(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tmconfig: &StreamConfig{\n\t\t\t\tName: \"foo\",\n\t\t\t\tRetention: LimitsPolicy,\n\t\t\t\tMaxAge: time.Hour,\n\t\t\t\tStorage: FileStorage,\n\t\t\t\tReplicas: 1,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\tnc := clientConnectToServer(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Add(\"Accept-Encoding\", \"json\")\n\t\t\tm.Header.Add(\"Authorization\", \"s3cr3t\")\n\t\t\tm.Data = []byte(\"Hello JetStream Headers - #1!\")\n\n\t\t\tnc.PublishMsg(m)\n\t\t\tnc.Flush()\n\n\t\t\tcheckFor(t, time.Second*2, time.Millisecond*250, func() error {\n\t\t\t\tstate := mset.state()\n\t\t\t\tif state.Msgs != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected 1 message, got %d\", state.Msgs)\n\t\t\t\t}\n\t\t\t\tif state.Bytes == 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected non-zero bytes\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now access raw from stream.\n\t\t\tsm, err := mset.getMsg(1)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error getting stored message: %v\", err)\n\t\t\t}\n\t\t\t// Calculate the []byte version of the headers.\n\t\t\tvar b bytes.Buffer\n\t\t\tb.WriteString(\"NATS/1.0\\r\\n\")\n\t\t\thttp.Header(m.Header).Write(&b)\n\t\t\tb.WriteString(\"\\r\\n\")\n\t\t\thdr := b.Bytes()\n\n\t\t\tif !bytes.Equal(sm.Header, hdr) {\n\t\t\t\tt.Fatalf(\"Message headers do not match, %q vs %q\", hdr, sm.Header)\n\t\t\t}\n\t\t\tif !bytes.Equal(sm.Data, m.Data) {\n\t\t\t\tt.Fatalf(\"Message data do not match, %q vs %q\", m.Data, sm.Data)\n\t\t\t}\n\n\t\t\t// Now do consumer based.\n\t\t\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tnc.Flush()\n\n\t\t\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t\t\t}\n\t\t\tdefer o.delete()\n\n\t\t\tcm, err := sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t\t\t}\n\t\t\t// Check the message.\n\t\t\t// Check out original headers.\n\t\t\tif cm.Header.Get(\"Accept-Encoding\") != \"json\" ||\n\t\t\t\tcm.Header.Get(\"Authorization\") != \"s3cr3t\" {\n\t\t\t\tt.Fatalf(\"Original headers not present\")\n\t\t\t}\n\t\t\tif !bytes.Equal(m.Data, cm.Data) {\n\t\t\t\tt.Fatalf(\"Message payloads are not the same: %q vs %q\", cm.Data, m.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// This will be testing our ability to conditionally rewrite subjects for last mile\n// when working with JetStream. Consumers receive messages that have their subjects\n// rewritten to match the original subject. NATS routing is all subject based except\n// for the last mile to the client.\nfunc TestJetStreamSingleInstanceRemoteAccess(t *testing.T) {\n\tca := createClusterWithName(t, \"A\", 1)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 1, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Connect our leafnode server to cluster B.\n\topts := cb.opts[rand.Intn(len(cb.opts))]\n\ts, _ := runSolicitLeafServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\tif err := s.EnableJetStream(&JetStreamConfig{StoreDir: t.TempDir()}); err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"foo\", Storage: MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\t}\n\n\t// Now create a push based consumer. Connected to the non-jetstream server via a random server on cluster A.\n\tsl := ca.servers[rand.Intn(len(ca.servers))]\n\tnc2 := clientConnectToServer(t, sl)\n\tdefer nc2.Close()\n\n\tsub, _ := nc2.SubscribeSync(nats.NewInbox())\n\tdefer sub.Unsubscribe()\n\n\t// Need to wait for interest to propagate across GW.\n\tnc2.Flush()\n\ttime.Sleep(25 * time.Millisecond)\n\n\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: sub.Subject})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tcheckSubPending := func(numExpected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 200*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numExpected {\n\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, numExpected)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckSubPending(toSend)\n\n\tcheckMsg := func(m *nats.Msg, err error, i int) {\n\t\tt.Helper()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Got an error checking message: %v\", err)\n\t\t}\n\t\tif m.Subject != \"foo\" {\n\t\t\tt.Fatalf(\"Expected original subject of %q, but got %q\", \"foo\", m.Subject)\n\t\t}\n\t\t// Now check that reply subject exists and has a sequence as the last token.\n\t\tif seq := o.seqFromReply(m.Reply); seq != uint64(i) {\n\t\t\tt.Fatalf(\"Expected sequence of %d , got %d\", i, seq)\n\t\t}\n\t}\n\n\t// Now check the subject to make sure its the original one.\n\tfor i := 1; i <= toSend; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\tcheckMsg(m, err, i)\n\t}\n\n\t// Now do a pull based consumer.\n\to, err = mset.addConsumer(workerModeConfig(\"p\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tnextMsg := o.requestNextMsgSubject()\n\tfor i := 1; i <= toSend; i++ {\n\t\tm, err := nc.Request(nextMsg, nil, time.Second)\n\t\tcheckMsg(m, err, i)\n\t}\n}\n\nfunc clientConnectToServerWithUP(t *testing.T, opts *Options, user, pass string) *nats.Conn {\n\tcurl := fmt.Sprintf(\"nats://%s:%s@%s:%d\", user, pass, opts.Host, opts.Port)\n\tnc, err := nats.Connect(curl, nats.Name(\"JS-UP-TEST\"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc TestJetStreamCanNotEnableOnSystemAccount(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsa := s.SystemAccount()\n\tif err := sa.EnableJetStream(nil, nil); err == nil {\n\t\tt.Fatalf(\"Expected an error trying to enable on the system account\")\n\t}\n}\n\nfunc TestJetStreamMultipleAccountsBasics(t *testing.T) {\n\ttdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: ua, password: pwd} ]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tjetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k}\n\t\t\t\tusers: [ {user: ub, password: pwd} ]\n\t\t\t},\n\t\t\tC: {\n\t\t\t\tusers: [ {user: uc, password: pwd} ]\n\t\t\t},\n\t\t}\n\t`, tdir)))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\n\tnca := clientConnectToServerWithUP(t, opts, \"ua\", \"pwd\")\n\tdefer nca.Close()\n\n\tncb := clientConnectToServerWithUP(t, opts, \"ub\", \"pwd\")\n\tdefer ncb.Close()\n\n\tresp, err := ncb.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tvar info JSApiAccountInfoResponse\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tlimits := info.Limits\n\tif limits.MaxStreams != 10 {\n\t\tt.Fatalf(\"Expected 10 for MaxStreams, got %d\", limits.MaxStreams)\n\t}\n\tif limits.MaxConsumers != 1000 {\n\t\tt.Fatalf(\"Expected MaxConsumers of %d, got %d\", 1000, limits.MaxConsumers)\n\t}\n\tgb := int64(1024 * 1024 * 1024)\n\tif limits.MaxMemory != gb {\n\t\tt.Fatalf(\"Expected MaxMemory to be 1GB, got %d\", limits.MaxMemory)\n\t}\n\tif limits.MaxStore != 1024*gb {\n\t\tt.Fatalf(\"Expected MaxStore to be 1TB, got %d\", limits.MaxStore)\n\t}\n\n\tncc := clientConnectToServerWithUP(t, opts, \"uc\", \"pwd\")\n\tdefer ncc.Close()\n\n\texpectNotEnabled := func(resp *nats.Msg, err error) {\n\t\tt.Helper()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error requesting enabled status: %v\", err)\n\t\t}\n\t\tif resp == nil {\n\t\t\tt.Fatalf(\"No response, possible timeout?\")\n\t\t}\n\t\tvar iResp JSApiAccountInfoResponse\n\t\tif err := json.Unmarshal(resp.Data, &iResp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif iResp.Error == nil {\n\t\t\tt.Fatalf(\"Expected an error on not enabled account\")\n\t\t}\n\t}\n\n\t// Check C is not enabled. We expect a negative response, not a timeout.\n\texpectNotEnabled(ncc.Request(JSApiAccountInfo, nil, 250*time.Millisecond))\n\n\t// Now do simple reload and check that we do the right thing. Testing enable and disable and also change in limits\n\tnewConf := []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: disabled\n\t\t\t\tusers: [ {user: ua, password: pwd} ]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tjetstream: {max_mem: 32GB, max_store: 512GB, max_streams: 100, max_consumers: 4k}\n\t\t\t\tusers: [ {user: ub, password: pwd} ]\n\t\t\t},\n\t\t\tC: {\n\t\t\t\tjetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k}\n\t\t\t\tusers: [ {user: uc, password: pwd} ]\n\t\t\t},\n\t\t}\n\t`, tdir))\n\tif err := os.WriteFile(conf, newConf, 0600); err != nil {\n\t\tt.Fatalf(\"Error rewriting server's config file: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on server reload: %v\", err)\n\t}\n\texpectNotEnabled(nca.Request(JSApiAccountInfo, nil, 250*time.Millisecond))\n\n\tresp, _ = ncb.Request(JSApiAccountInfo, nil, 250*time.Millisecond)\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Error != nil {\n\t\tt.Fatalf(\"Expected JetStream to be enabled, got %+v\", info.Error)\n\t}\n\n\tresp, _ = ncc.Request(JSApiAccountInfo, nil, 250*time.Millisecond)\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Error != nil {\n\t\tt.Fatalf(\"Expected JetStream to be enabled, got %+v\", info.Error)\n\t}\n\n\t// Now check that limits have been updated.\n\t// Account B\n\tresp, err = ncb.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tlimits = info.Limits\n\tif limits.MaxStreams != 100 {\n\t\tt.Fatalf(\"Expected 100 for MaxStreams, got %d\", limits.MaxStreams)\n\t}\n\tif limits.MaxConsumers != 4000 {\n\t\tt.Fatalf(\"Expected MaxConsumers of %d, got %d\", 4000, limits.MaxConsumers)\n\t}\n\tif limits.MaxMemory != 32*gb {\n\t\tt.Fatalf(\"Expected MaxMemory to be 32GB, got %d\", limits.MaxMemory)\n\t}\n\tif limits.MaxStore != 512*gb {\n\t\tt.Fatalf(\"Expected MaxStore to be 512GB, got %d\", limits.MaxStore)\n\t}\n\n\t// Account C\n\tresp, err = ncc.Request(JSApiAccountInfo, nil, time.Second)\n\trequire_NoError(t, err)\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tlimits = info.Limits\n\tif limits.MaxStreams != 10 {\n\t\tt.Fatalf(\"Expected 10 for MaxStreams, got %d\", limits.MaxStreams)\n\t}\n\tif limits.MaxConsumers != 1000 {\n\t\tt.Fatalf(\"Expected MaxConsumers of %d, got %d\", 1000, limits.MaxConsumers)\n\t}\n\tif limits.MaxMemory != gb {\n\t\tt.Fatalf(\"Expected MaxMemory to be 1GB, got %d\", limits.MaxMemory)\n\t}\n\tif limits.MaxStore != 1024*gb {\n\t\tt.Fatalf(\"Expected MaxStore to be 1TB, got %d\", limits.MaxStore)\n\t}\n}\n\nfunc TestJetStreamServerResourcesConfig(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 2GB, max_file_store: 1TB, store_dir: %q}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\n\tgb := int64(1024 * 1024 * 1024)\n\tjsc := s.JetStreamConfig()\n\tif jsc.MaxMemory != 2*gb {\n\t\tt.Fatalf(\"Expected MaxMemory to be %d, got %d\", 2*gb, jsc.MaxMemory)\n\t}\n\tif jsc.MaxStore != 1024*gb {\n\t\tt.Fatalf(\"Expected MaxStore to be %d, got %d\", 1024*gb, jsc.MaxStore)\n\t}\n}\n\n// From 2.2.2 to 2.2.3 we fixed a bug that would not consistently place a jetstream directory\n// under the store directory configured. However there were some cases where the directory was\n// created that way and therefore 2.2.3 would start and not recognize the existing accounts,\n// streams and consumers.\nfunc TestJetStreamStoreDirectoryFix(t *testing.T) {\n\tsd := filepath.Join(os.TempDir(), \"sd_test\")\n\tdefer removeDir(t, sd)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(\"listen: 127.0.0.1:-1\\njetstream: {store_dir: %q}\\n\", sd)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.Publish(\"TEST\", []byte(\"TSS\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\t// Push based.\n\tsub, err := js.SubscribeSync(\"TEST\", nats.Durable(\"dlc\"))\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Now shutdown the server.\n\tnc.Close()\n\ts.Shutdown()\n\n\t// Now move stuff up from the jetstream directory etc.\n\tjssd := filepath.Join(sd, JetStreamStoreDir)\n\tfis, _ := os.ReadDir(jssd)\n\t// This will be accounts, move them up one directory.\n\tfor _, fi := range fis {\n\t\tos.Rename(filepath.Join(jssd, fi.Name()), filepath.Join(sd, fi.Name()))\n\t}\n\tremoveDir(t, jssd)\n\n\t// Restart our server. Make sure our assets got moved.\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tvar names []string\n\tfor name := range js.StreamNames() {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 stream but got %d\", len(names))\n\t}\n\tnames = names[:0]\n\tfor name := range js.ConsumerNames(\"TEST\") {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 1 {\n\t\tt.Fatalf(\"Expected only 1 consumer but got %d\", len(names))\n\t}\n}\n\nfunc TestJetStreamPushConsumersPullError(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.Publish(\"TEST\", []byte(\"TSS\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\t// Push based.\n\tsub, err := js.SubscribeSync(\"TEST\")\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\tci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\n\t// Now do a pull. Make sure we get an error.\n\tm, err := nc.Request(fmt.Sprintf(JSApiRequestNextT, \"TEST\", ci.Name), nil, time.Second)\n\trequire_NoError(t, err)\n\tif m.Header.Get(\"Status\") != \"409\" {\n\t\tt.Fatalf(\"Expected a 409 status code, got %q\", m.Header.Get(\"Status\"))\n\t}\n}\n\nfunc TestJetStreamChangeConsumerType(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"test.*\"}})\n\trequire_NoError(t, err)\n\n\t// create pull consumer\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"pull\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// cannot update pull -> push\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"pull\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverSubject: \"foo\",\n\t})\n\trequire_Contains(t, err.Error(), \"can not update pull consumer to push based\")\n\n\t// create push consumer\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"push\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverSubject: \"foo\",\n\t})\n\trequire_NoError(t, err)\n\n\t// cannot change push -> pull\n\t_, err = js.UpdateConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"push\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_Contains(t, err.Error(), \"can not update push consumer to pull based\")\n}\n\n////////////////////////////////////////\n// Benchmark placeholders\n// TODO(dlc) - move\n////////////////////////////////////////\n\nfunc TestJetStreamPubPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tmsetConfig := StreamConfig{\n\t\tName: \"sr22\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\n\tif _, err := acc.addStream(&msetConfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 5_000_000\n\tnumProducers := 5\n\n\tpayload := []byte(\"Hello World\")\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\tfor n := 0; n < numProducers; n++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t<-startCh\n\t\t\tfor i := 0; i < int(toSend)/numProducers; i++ {\n\t\t\t\tnc.Publish(\"foo\", payload)\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t}()\n\t}\n\n\t// Wait for Go routines.\n\ttime.Sleep(20 * time.Millisecond)\n\tstart := time.Now()\n\tclose(startCh)\n\twg.Wait()\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\tstart = time.Now()\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\tfmt.Printf(\"Took %v to restart!\\n\", time.Since(start))\n}\n\nfunc TestJetStreamPubWithAsyncResponsePerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tmsetConfig := StreamConfig{\n\t\tName: \"sr33\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\n\tif _, err := acc.addStream(&msetConfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 1_000_000\n\tpayload := []byte(\"Hello World\")\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.PublishRequest(\"foo\", \"bar\", payload)\n\t}\n\tnc.Flush()\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamPubWithSyncPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\trequire_NoError(t, err)\n\n\ttoSend := 1_000_000\n\tpayload := []byte(\"Hello World\")\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.Publish(\"foo\", payload)\n\t}\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamPubSubPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.GlobalAccount()\n\n\tmsetConfig := StreamConfig{\n\t\tName: \"MSET22\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\n\tmset, err := acc.addStream(&msetConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tvar toSend = 1_000_000\n\tvar received int\n\tdone := make(chan bool)\n\n\tdelivery := \"d\"\n\n\tnc.Subscribe(delivery, func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= toSend {\n\t\t\tdone <- true\n\t\t}\n\t})\n\tnc.Flush()\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: delivery,\n\t\tAckPolicy: AckNone,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t}\n\n\tpayload := []byte(\"Hello World\")\n\n\tstart := time.Now()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.Publish(\"foo\", payload)\n\t}\n\n\t<-done\n\ttt := time.Since(start)\n\tfmt.Printf(\"time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\nfunc TestJetStreamAckExplicitMsgRemoval(t *testing.T) {\n\tconst burstSize = 20\n\n\tfor _, storageType := range []StorageType{\n\t\tFileStorage,\n\t\tMemoryStorage,\n\t} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\t// Start a server\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Create a stream\n\t\t\tcfg := &StreamConfig{\n\t\t\t\tName: \"MY_STREAM\",\n\t\t\t\tStorage: storageType,\n\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\tRetention: InterestPolicy,\n\t\t\t}\n\t\t\tmset, err := s.GlobalAccount().addStream(cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer mset.delete()\n\n\t\t\t// Create 2 connections\n\t\t\tnc1 := clientConnectToServer(t, s)\n\t\t\tdefer nc1.Close()\n\t\t\tnc2 := clientConnectToServer(t, s)\n\t\t\tdefer nc2.Close()\n\n\t\t\t// Create 2 subscription inboxes (for durable consumers DeliverSubject forwarding)\n\t\t\tsub1, err := nc1.SubscribeSync(nats.NewInbox())\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub1.Unsubscribe()\n\t\t\terr = nc1.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsub2, err := nc2.SubscribeSync(nats.NewInbox())\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub2.Unsubscribe()\n\t\t\terr = nc2.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Create 2 durable consumers (that forward messages to the inboxes above)\n\t\t\tdc1, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dur1\",\n\t\t\t\tDeliverSubject: sub1.Subject,\n\t\t\t\tFilterSubject: \"foo.bar\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dc1.delete()\n\n\t\t\tdc2, err := mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dur2\",\n\t\t\t\tDeliverSubject: sub2.Subject,\n\t\t\t\tFilterSubject: \"foo.bar\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 100 * time.Millisecond,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dc2.delete()\n\n\t\t\t// Publish N messages (with subject matching the stream created above)\n\t\t\tfor i := 1; i <= burstSize; i++ {\n\t\t\t\tpubAck := sendStreamMsg(t, nc1, \"foo.bar\", fmt.Sprintf(\"msg_%d\", i))\n\t\t\t\trequire_Equal(t, int(pubAck.Sequence), i)\n\t\t\t}\n\n\t\t\t// Check that the stream contains 2 messages\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != uint64(burstSize) {\n\t\t\t\tt.Fatalf(\"Expected %d messages in stream, found %d\", burstSize, state.Msgs)\n\t\t\t}\n\n\t\t\t// Receive and ack both messages from the 2 subscriptions\n\t\t\tfor subIndex, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tsequences := make(map[uint64]bool, burstSize)\n\n\t\t\t\tfor i := 1; i <= burstSize; i++ {\n\t\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\terr = m.Respond(nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tmetadata, err := m.Metadata()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tsequences[metadata.Sequence.Stream] = true\n\t\t\t\t}\n\n\t\t\t\t// Verify all known sequence numbers are delivered (as opposed to duplicates)\n\t\t\t\tfor i := uint64(1); i <= burstSize; i++ {\n\t\t\t\t\t_, ok := sequences[i]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tt.Fatalf(\"Subscriber %d/2 did not deliver message sequence %d\", subIndex+1, i)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Flush both connections which may have pending ACKs\n\t\t\terr = nc1.Flush()\n\t\t\trequire_NoError(t, err)\n\t\t\terr = nc2.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Verify all messages are eventually dropped from the stream (since all known consumers ack'd them)\n\t\t\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tif state = mset.state(); state.Msgs != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"expected empty stream, but found %d messages\", state.Msgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Shut down one of the subscriptions\n\t\t\terr = sub2.Unsubscribe()\n\t\t\trequire_NoError(t, err)\n\t\t\terr = nc2.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Publish N additional messages\n\t\t\tfor i := burstSize + 1; i <= burstSize+burstSize; i++ {\n\t\t\t\tpubAck := sendStreamMsg(t, nc1, \"foo.bar\", fmt.Sprintf(\"msg_%d\", i))\n\t\t\t\trequire_Equal(t, i, int(pubAck.Sequence))\n\t\t\t}\n\n\t\t\t// Check that the stream contains N messages\n\t\t\tstate = mset.state()\n\t\t\tif state.Msgs != uint64(burstSize) {\n\t\t\t\tt.Fatalf(\"Expected %d messages in stream, found %d\", burstSize, state.Msgs)\n\t\t\t}\n\n\t\t\t// Receive and ack the new messages using the active subscription/consumer\n\t\t\tsequences := make(map[uint64]bool, burstSize)\n\t\t\tfor i := 1; i <= burstSize; i++ {\n\t\t\t\tm, err := sub1.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\terr = m.Respond(nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tmetadata, err := m.Metadata()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tsequences[metadata.Sequence.Stream] = true\n\t\t\t}\n\t\t\t// Verify all known sequence numbers are delivered (as opposed to duplicates)\n\t\t\tfor i := uint64(burstSize) + 1; i <= burstSize+burstSize; i++ {\n\t\t\t\t_, ok := sequences[i]\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"Subscriber 1/2 did not deliver message sequence %d\", i)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Create a different subscription inbox\n\t\t\tnewSub2, err := nc2.SubscribeSync(nats.NewInbox())\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer newSub2.Unsubscribe()\n\t\t\terr = nc2.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Update the second durable consumer with the new inbox subscription\n\t\t\tdc2, err = mset.addConsumer(&ConsumerConfig{\n\t\t\t\tDurable: \"dur2\",\n\t\t\t\tDeliverSubject: newSub2.Subject,\n\t\t\t\tFilterSubject: \"foo.bar\",\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tAckWait: 100 * time.Millisecond,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer dc2.delete()\n\n\t\t\t// Receive and ack the new messages using the recreated/updated subscription/consumer\n\t\t\tsequences = make(map[uint64]bool, burstSize)\n\t\t\tfor i := 1; i <= burstSize; i++ {\n\t\t\t\tm, err := newSub2.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\terr = m.Respond(nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tmetadata, err := m.Metadata()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tsequences[metadata.Sequence.Stream] = true\n\t\t\t}\n\t\t\t// Verify all known sequence numbers are delivered (as opposed to duplicates)\n\t\t\tfor i := uint64(burstSize) + 1; i <= burstSize+burstSize; i++ {\n\t\t\t\t_, ok := sequences[i]\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"Subscriber 2/2 did not deliver message sequence %d\", i)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Flush both connections which may have pending ACKs\n\t\t\terr = nc1.Flush()\n\t\t\trequire_NoError(t, err)\n\t\t\terr = nc2.Flush()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Verify all messages are eventually dropped from the stream, since all known consumers ack'd them\n\t\t\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tif state = mset.state(); state.Msgs != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"expected empty stream, but found %d messages\", state.Msgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStoredMsgsDontDisappearAfterCacheExpiration(t *testing.T) {\n\tsc := &StreamConfig{\n\t\tName: \"MY_STREAM\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo.>\"},\n\t\tRetention: InterestPolicy,\n\t}\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStreamWithStore(sc, &FileStoreConfig{BlockSize: 128, CacheExpire: 15 * time.Millisecond})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc1 := clientConnectWithOldRequest(t, s)\n\tdefer nc1.Close()\n\n\t// Create a durable consumers\n\tsub, _ := nc1.SubscribeSync(nats.NewInbox())\n\tdefer sub.Unsubscribe()\n\tnc1.Flush()\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tDeliverSubject: sub.Subject,\n\t\tFilterSubject: \"foo.bar\",\n\t\tDeliverPolicy: DeliverNew,\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding consumer: %v\", err)\n\t}\n\tdefer o.delete()\n\n\tnc2 := clientConnectWithOldRequest(t, s)\n\tdefer nc2.Close()\n\n\tsendStreamMsg(t, nc2, \"foo.bar\", \"msg1\")\n\n\tmsg, err := sub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Did not get message: %v\", err)\n\t}\n\tif string(msg.Data) != \"msg1\" {\n\t\tt.Fatalf(\"Unexpected message: %q\", msg.Data)\n\t}\n\n\tnc1.Close()\n\n\t// Get the message from the stream\n\tgetMsgSeq := func(seq uint64) {\n\t\tt.Helper()\n\t\tmreq := &JSApiMsgGetRequest{Seq: seq}\n\t\treq, err := json.Marshal(mreq)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tsmsgj, err := nc2.Request(fmt.Sprintf(JSApiMsgGetT, sc.Name), req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not retrieve stream message: %v\", err)\n\t\t}\n\t\tif strings.Contains(string(smsgj.Data), \"code\") {\n\t\t\tt.Fatalf(\"Error: %q\", smsgj.Data)\n\t\t}\n\t}\n\n\tgetMsgSeq(1)\n\n\ttime.Sleep(time.Second)\n\n\tsendStreamMsg(t, nc2, \"foo.bar\", \"msg2\")\n\tsendStreamMsg(t, nc2, \"foo.bar\", \"msg3\")\n\n\tgetMsgSeq(1)\n\tgetMsgSeq(2)\n\tgetMsgSeq(3)\n}\n\nfunc TestJetStreamDeliveryAfterServerRestart(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.JetStream = true\n\topts.StoreDir = t.TempDir()\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{\n\t\tName: \"MY_STREAM\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"foo.>\"},\n\t\tRetention: InterestPolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tinbox := nats.NewInbox()\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tDeliverSubject: inbox,\n\t\tDeliverPolicy: DeliverNew,\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tsub, err := nc.SubscribeSync(inbox)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc.Flush()\n\n\t// Send 1 message\n\tsendStreamMsg(t, nc, \"foo.bar\", \"msg1\")\n\n\t// Make sure we receive it and ack it.\n\tmsg, err := sub.NextMsg(250 * time.Millisecond)\n\tif err != nil {\n\t\tt.Fatalf(\"Did not get message: %v\", err)\n\t}\n\t// Ack it!\n\tmsg.Respond(nil)\n\tnc.Flush()\n\n\t// Shutdown client and server\n\tnc.Close()\n\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\ts.Shutdown()\n\n\topts.Port = -1\n\topts.StoreDir = dir\n\ts = RunServer(&opts)\n\tdefer s.Shutdown()\n\n\t// Lookup stream.\n\tmset, err = s.GlobalAccount().lookupStream(\"MY_STREAM\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up stream: %v\", err)\n\t}\n\n\t// Update consumer's deliver subject with new inbox\n\tinbox = nats.NewInbox()\n\to, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tDeliverSubject: inbox,\n\t\tDeliverPolicy: DeliverNew,\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\tnc = clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Send 2nd message\n\tsendStreamMsg(t, nc, \"foo.bar\", \"msg2\")\n\n\t// Start sub on new inbox\n\tsub, err = nc.SubscribeSync(inbox)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc.Flush()\n\n\t// Should receive message 2.\n\tif _, err := sub.NextMsg(500 * time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Did not get message: %v\", err)\n\t}\n}\n\n// This is for the basics of importing the ability to send to a stream and consume\n// from a consumer that is pull based on push based on a well known delivery subject.\nfunc TestJetStreamAccountImportBasics(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tno_auth_user: rip\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: dlc, password: foo} ]\n\t\t\t\texports [\n\t\t\t\t\t# This is for sending into a stream from other accounts.\n\t\t\t\t\t{ service: \"ORDERS.*\" }\n\t\t\t\t\t# This is for accessing a pull based consumer.\n\t\t\t\t\t{ service: \"$JS.API.CONSUMER.MSG.NEXT.*.*\" }\n\t\t\t\t\t# This is streaming to a delivery subject for a push based consumer.\n\t\t\t\t\t{ stream: \"deliver.ORDERS\" }\n\t\t\t\t\t# This is to ack received messages. This is a service to ack acks..\n\t\t\t\t\t{ service: \"$JS.ACK.ORDERS.*.>\" }\n\t\t\t\t]\n\t\t\t},\n\t\t\tIU: {\n\t\t\t\tusers: [ {user: rip, password: bar} ]\n\t\t\t\timports [\n\t\t\t\t\t{ service: { subject: \"ORDERS.*\", account: JS }, to: \"my.orders.$1\" }\n\t\t\t\t\t{ service: { subject: \"$JS.API.CONSUMER.MSG.NEXT.ORDERS.d\", account: JS }, to: \"nxt.msg\" }\n\t\t\t\t\t{ stream: { subject: \"deliver.ORDERS\", account: JS }, to: \"d\" }\n\t\t\t\t\t{ service: { subject: \"$JS.ACK.ORDERS.*.>\", account: JS } }\n\t\t\t\t]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tacc, err := s.LookupAccount(\"JS\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up account: %v\", err)\n\t}\n\n\tmset, err := acc.addStream(&StreamConfig{Name: \"ORDERS\", Subjects: []string{\"ORDERS.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// This should be the rip user, the one that imports some JS.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Simple publish to a stream.\n\tpubAck := sendStreamMsg(t, nc, \"my.orders.foo\", \"ORDERS-1\")\n\tif pubAck.Stream != \"ORDERS\" || pubAck.Sequence != 1 {\n\t\tt.Fatalf(\"Bad pubAck received: %+v\", pubAck)\n\t}\n\tif msgs := mset.state().Msgs; msgs != 1 {\n\t\tt.Fatalf(\"Expected 1 message, got %d\", msgs)\n\t}\n\n\ttotal := 2\n\tfor i := 2; i <= total; i++ {\n\t\tsendStreamMsg(t, nc, \"my.orders.bar\", fmt.Sprintf(\"ORDERS-%d\", i))\n\t}\n\tif msgs := mset.state().Msgs; msgs != uint64(total) {\n\t\tt.Fatalf(\"Expected %d messages, got %d\", total, msgs)\n\t}\n\n\t// Now test access to a pull based consumer, e.g. workqueue.\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\t// We mapped the next message request, \"$JS.API.CONSUMER.MSG.NEXT.ORDERS.d\" -> \"nxt.msg\"\n\tm, err := nc.Request(\"nxt.msg\", nil, time.Second)\n\trequire_NoError(t, err)\n\tif string(m.Data) != \"ORDERS-1\" {\n\t\tt.Fatalf(\"Expected to receive %q, got %q\", \"ORDERS-1\", m.Data)\n\t}\n\n\t// Now test access to a push based consumer\n\to, err = mset.addConsumer(&ConsumerConfig{\n\t\tDurable: \"p\",\n\t\tDeliverSubject: \"deliver.ORDERS\",\n\t\tAckPolicy: AckExplicit,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\t// We remapped from above, deliver.ORDERS -> d\n\tsub, _ := nc.SubscribeSync(\"d\")\n\tdefer sub.Unsubscribe()\n\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != total {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, total)\n\t\t}\n\t\treturn nil\n\t})\n\n\tm, _ = sub.NextMsg(time.Second)\n\t// Make sure we remapped subject correctly across the account boundary.\n\tif m.Subject != \"ORDERS.foo\" {\n\t\tt.Fatalf(\"Expected subject of %q, got %q\", \"ORDERS.foo\", m.Subject)\n\t}\n\t// Now make sure we can ack messages correctly.\n\tm.Respond(AckAck)\n\tnc.Flush()\n\n\tif info := o.info(); info.AckFloor.Consumer != 1 {\n\t\tt.Fatalf(\"Did not receive the ack properly\")\n\t}\n\n\t// Grab second one now.\n\tm, _ = sub.NextMsg(time.Second)\n\t// Make sure we remapped subject correctly across the account boundary.\n\tif m.Subject != \"ORDERS.bar\" {\n\t\tt.Fatalf(\"Expected subject of %q, got %q\", \"ORDERS.bar\", m.Subject)\n\t}\n\t// Now make sure we can ack messages and get back an ack as well.\n\tresp, err := nc.Request(m.Reply, nil, 100*time.Millisecond)\n\tif resp == nil {\n\t\tt.Fatalf(\"No response (error: %v)\", err)\n\t}\n\tif info := o.info(); info.AckFloor.Consumer != 2 {\n\t\tt.Fatalf(\"Did not receive the ack properly\")\n\t}\n}\n\n// This tests whether we are able to aggregate all JetStream advisory events\n// from all accounts into a single account. Config for this test uses\n// service imports and exports as that allows for gathering all events\n// without having to know the account name and without separate entries\n// for each account in aggregate account config.\n// This test fails as it is not receiving the api audit event ($JS.EVENT.ADVISORY.API).\nfunc TestJetStreamAccountImportJSAdvisoriesAsService(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten=127.0.0.1:-1\n\t\tno_auth_user: pp\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts {\n\t\t\tJS {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: pp, password: foo} ]\n\t\t\t\timports [\n\t\t\t\t\t{ service: { account: AGG, subject: '$JS.EVENT.ADVISORY.ACC.JS.>' }, to: '$JS.EVENT.ADVISORY.>' }\n\t\t\t\t]\n\t\t\t}\n\t\t\tAGG {\n\t\t\t\tusers: [ {user: agg, password: foo} ]\n\t\t\t\texports: [\n\t\t\t\t\t{ service: '$JS.EVENT.ADVISORY.ACC.*.>', response: Singleton, account_token_position: 5 }\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// This should be the pp user, one which manages JetStream assets\n\tncJS, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error during connect: %v\", err)\n\t}\n\tdefer ncJS.Close()\n\n\t// This is the agg user, which should aggregate all JS advisory events.\n\tncAgg, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"agg\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error during connect: %v\", err)\n\t}\n\tdefer ncAgg.Close()\n\n\tjs, err := ncJS.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// user from JS account should receive events on $JS.EVENT.ADVISORY.> subject\n\tsubJS, err := ncJS.SubscribeSync(\"$JS.EVENT.ADVISORY.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer subJS.Unsubscribe()\n\trequire_NoError(t, ncJS.Flush())\n\n\t// user from AGG account should receive events on mapped $JS.EVENT.ADVISORY.ACC.JS.> subject (with account name)\n\tsubAgg, err := ncAgg.SubscribeSync(\"$JS.EVENT.ADVISORY.ACC.JS.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer subAgg.Unsubscribe()\n\trequire_NoError(t, ncAgg.Flush())\n\n\t// add stream using JS account\n\t// this should trigger 2 events:\n\t// - an action event on $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS\n\t// - an api audit event on $JS.EVENT.ADVISORY.API\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"ORDERS\", Subjects: []string{\"ORDERS.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tgotEvents := map[string]int{}\n\tfor i := 0; i < 2; i++ {\n\t\tmsg, err := subJS.NextMsg(time.Second * 2)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tgotEvents[msg.Subject]++\n\t}\n\tif c := gotEvents[\"$JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS\"]; c != 1 {\n\t\tt.Fatalf(\"Should have received one advisory from $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS but got %d\", c)\n\t}\n\tif c := gotEvents[\"$JS.EVENT.ADVISORY.API\"]; c != 1 {\n\t\tt.Fatalf(\"Should have received one advisory from $JS.EVENT.ADVISORY.API but got %d\", c)\n\t}\n\n\t// same set of events should be received by AGG account\n\t// on subjects containing account name (ACC.JS)\n\tgotEvents = map[string]int{}\n\tfor i := 0; i < 2; i++ {\n\t\tmsg, err := subAgg.NextMsg(time.Second * 2)\n\t\trequire_NoError(t, err)\n\t\tvar adv JSAPIAudit\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &adv))\n\t\t// Make sure we have full fidelity info via implicit share.\n\t\tif adv.Client != nil {\n\t\t\trequire_True(t, adv.Client.Host != _EMPTY_)\n\t\t\trequire_True(t, adv.Client.User != _EMPTY_)\n\t\t\trequire_True(t, adv.Client.Lang != _EMPTY_)\n\t\t}\n\t\tgotEvents[msg.Subject]++\n\t}\n\tif c := gotEvents[\"$JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS\"]; c != 1 {\n\t\tt.Fatalf(\"Should have received one advisory from $JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS but got %d\", c)\n\t}\n\tif c := gotEvents[\"$JS.EVENT.ADVISORY.ACC.JS.API\"]; c != 1 {\n\t\tt.Fatalf(\"Should have received one advisory from $JS.EVENT.ADVISORY.ACC.JS.API but got %d\", c)\n\t}\n}\n\n// This tests whether we are able to aggregate all JetStream advisory events\n// from all accounts into a single account. Config for this test uses\n// stream imports and exports as that allows for gathering all events\n// as long as there is a separate stream import entry for each account\n// in aggregate account config.\nfunc TestJetStreamAccountImportJSAdvisoriesAsStream(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten=127.0.0.1:-1\n\t\tno_auth_user: pp\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts {\n\t\t\tJS {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: pp, password: foo} ]\n\t\t\t\texports [\n\t\t\t\t\t{ stream: '$JS.EVENT.ADVISORY.>' }\n\t\t\t\t]\n\t\t\t}\n\t\t\tAGG {\n\t\t\t\tusers: [ {user: agg, password: foo} ]\n\t\t\t\timports: [\n\t\t\t\t\t{ stream: { account: JS, subject: '$JS.EVENT.ADVISORY.>' }, to: '$JS.EVENT.ADVISORY.ACC.JS.>' }\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// This should be the pp user, one which manages JetStream assets\n\tncJS, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error during connect: %v\", err)\n\t}\n\tdefer ncJS.Close()\n\n\t// This is the agg user, which should aggregate all JS advisory events.\n\tncAgg, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"agg\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error during connect: %v\", err)\n\t}\n\tdefer ncAgg.Close()\n\n\tjs, err := ncJS.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// user from JS account should receive events on $JS.EVENT.ADVISORY.> subject\n\tsubJS, err := ncJS.SubscribeSync(\"$JS.EVENT.ADVISORY.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer subJS.Unsubscribe()\n\trequire_NoError(t, ncJS.Flush())\n\n\t// user from AGG account should receive events on mapped $JS.EVENT.ADVISORY.ACC.JS.> subject (with account name)\n\tsubAgg, err := ncAgg.SubscribeSync(\"$JS.EVENT.ADVISORY.ACC.JS.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer subAgg.Unsubscribe()\n\trequire_NoError(t, ncAgg.Flush())\n\n\t// add stream using JS account\n\t// this should trigger 2 events:\n\t// - an action event on $JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS\n\t// - an api audit event on $JS.EVENT.ADVISORY.API\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"ORDERS\", Subjects: []string{\"ORDERS.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\tvar gotAPIAdvisory, gotCreateAdvisory bool\n\tfor i := 0; i < 2; i++ {\n\t\tmsg, err := subJS.NextMsg(time.Second * 2)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on JS account: %v\", err)\n\t\t}\n\t\tswitch msg.Subject {\n\t\tcase \"$JS.EVENT.ADVISORY.STREAM.CREATED.ORDERS\":\n\t\t\tgotCreateAdvisory = true\n\t\tcase \"$JS.EVENT.ADVISORY.API\":\n\t\t\tgotAPIAdvisory = true\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected subject: %q\", msg.Subject)\n\t\t}\n\t}\n\tif !gotAPIAdvisory || !gotCreateAdvisory {\n\t\tt.Fatalf(\"Expected to have received both advisories on JS account (API advisory %v, create advisory %v)\", gotAPIAdvisory, gotCreateAdvisory)\n\t}\n\n\t// same set of events should be received by AGG account\n\t// on subjects containing account name (ACC.JS)\n\tgotAPIAdvisory, gotCreateAdvisory = false, false\n\tfor i := 0; i < 2; i++ {\n\t\tmsg, err := subAgg.NextMsg(time.Second * 2)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on AGG account: %v\", err)\n\t\t}\n\t\tswitch msg.Subject {\n\t\tcase \"$JS.EVENT.ADVISORY.ACC.JS.STREAM.CREATED.ORDERS\":\n\t\t\tgotCreateAdvisory = true\n\t\tcase \"$JS.EVENT.ADVISORY.ACC.JS.API\":\n\t\t\tgotAPIAdvisory = true\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected subject: %q\", msg.Subject)\n\t\t}\n\t}\n\tif !gotAPIAdvisory || !gotCreateAdvisory {\n\t\tt.Fatalf(\"Expected to have received both advisories on AGG account (API advisory %v, create advisory %v)\", gotAPIAdvisory, gotCreateAdvisory)\n\t}\n}\n\n// This is for importing all of JetStream into another account for admin purposes.\nfunc TestJetStreamAccountImportAll(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tno_auth_user: rip\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: dlc, password: foo} ]\n\t\t\t\texports [ { service: \"$JS.API.>\" } ]\n\t\t\t},\n\t\t\tIU: {\n\t\t\t\tusers: [ {user: rip, password: bar} ]\n\t\t\t\timports [ { service: { subject: \"$JS.API.>\", account: JS }, to: \"jsapi.>\"} ]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tacc, err := s.LookupAccount(\"JS\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up account: %v\", err)\n\t}\n\n\tmset, err := acc.addStream(&StreamConfig{Name: \"ORDERS\", Subjects: []string{\"ORDERS.*\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// This should be the rip user, the one that imports all of JS.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tmapSubj := func(subject string) string {\n\t\treturn strings.Replace(subject, \"$JS.API.\", \"jsapi.\", 1)\n\t}\n\n\t// This will get the current information about usage and limits for this account.\n\tresp, err := nc.Request(mapSubj(JSApiAccountInfo), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar info JSApiAccountInfoResponse\n\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif info.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", info.Error)\n\t}\n\t// Lookup streams.\n\tresp, err = nc.Request(mapSubj(JSApiStreams), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar namesResponse JSApiStreamNamesResponse\n\tif err = json.Unmarshal(resp.Data, &namesResponse); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif namesResponse.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", namesResponse.Error)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/1736\nfunc TestJetStreamServerReload(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q }\n\t\taccounts: {\n\t\t\tA: { users: [ {user: ua, password: pwd} ] },\n\t\t\tB: {\n\t\t\t\tjetstream: {max_mem: 1GB, max_store: 1TB, max_streams: 10, max_consumers: 1k}\n\t\t\t\tusers: [ {user: ub, password: pwd} ]\n\t\t\t},\n\t\t\tSYS: { users: [ {user: uc, password: pwd} ] },\n\t\t}\n\t\tno_auth_user: ub\n\t\tsystem_account: SYS\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcheckJSAccount := func() {\n\t\tt.Helper()\n\t\tresp, err := nc.Request(JSApiAccountInfo, nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar info JSApiAccountInfoResponse\n\t\tif err := json.Unmarshal(resp.Data, &info); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tcheckJSAccount()\n\n\tacc, err := s.LookupAccount(\"B\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up account: %v\", err)\n\t}\n\tmset, err := acc.addStream(&StreamConfig{Name: \"22\"})\n\trequire_NoError(t, err)\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, \"22\", fmt.Sprintf(\"MSG: %d\", i+1))\n\t}\n\tif msgs := mset.state().Msgs; msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d messages, got %d\", toSend, msgs)\n\t}\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on server reload: %v\", err)\n\t}\n\n\t// Wait to get reconnected.\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tif !nc.IsConnected() {\n\t\t\treturn fmt.Errorf(\"Not connected\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckJSAccount()\n\tsendStreamMsg(t, nc, \"22\", \"MSG: 22\")\n}\n\nfunc TestJetStreamConfigReloadWithGlobalAccount(t *testing.T) {\n\ttdir := t.TempDir()\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{user: anonymous}\n\t\t\t\t{user: user1, password: %s}\n\t\t\t]\n\t\t}\n\t\tno_auth_user: anonymous\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"pwd\", tdir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcheckJSAccount := func() {\n\t\tt.Helper()\n\t\tif _, err := js.AccountInfo(); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tcheckJSAccount()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(fmt.Sprintf(\"MSG: %d\", i+1))); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tsi, err := js.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs after restart, got %d\", toSend, si.State.Msgs)\n\t}\n\n\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(template, \"pwd2\", tdir)), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during config reload: %v\", err)\n\t}\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Try to add a new stream to the global account\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"bar\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckJSAccount()\n}\n\n// Test that we properly enforce per subject msg limits.\nfunc TestJetStreamMaxMsgsPerSubject(t *testing.T) {\n\tconst maxPer = 5\n\tmsc := StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\"},\n\t\tStorage: MemoryStorage,\n\t\tMaxMsgsPer: maxPer,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t\t}\n\t\t\tdefer mset.delete()\n\n\t\t\t// Client for API requests.\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tpubAndCheck := func(subj string, num int, expectedNumMsgs uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor i := 0; i < num; i++ {\n\t\t\t\t\tif _, err = js.Publish(subj, []byte(\"TSLA\")); err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != expectedNumMsgs {\n\t\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", expectedNumMsgs, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpubAndCheck(\"foo\", 1, 1)\n\t\t\tpubAndCheck(\"foo\", 4, 5)\n\t\t\t// Now make sure our per subject limits kick in..\n\t\t\tpubAndCheck(\"foo\", 2, 5)\n\t\t\tpubAndCheck(\"baz.22\", 5, 10)\n\t\t\tpubAndCheck(\"baz.33\", 5, 15)\n\t\t\t// We are maxed so totals should be same no matter what we add here.\n\t\t\tpubAndCheck(\"baz.22\", 5, 15)\n\t\t\tpubAndCheck(\"baz.33\", 5, 15)\n\n\t\t\t// Now purge and make sure all is still good.\n\t\t\tmset.purge(nil)\n\t\t\tpubAndCheck(\"foo\", 1, 1)\n\t\t\tpubAndCheck(\"foo\", 4, 5)\n\t\t\tpubAndCheck(\"baz.22\", 5, 10)\n\t\t\tpubAndCheck(\"baz.33\", 5, 15)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamGetLastMsgBySubject(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 2,\n\t\t\t\tMaxMsgsPer: 20,\n\t\t\t}\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\tfor i := 0; i < 1000; i++ {\n\t\t\t\tmsg := []byte(fmt.Sprintf(\"VAL-%d\", i+1))\n\t\t\t\tjs.PublishAsync(\"kv.foo\", msg)\n\t\t\t\tjs.PublishAsync(\"kv.bar\", msg)\n\t\t\t\tjs.PublishAsync(\"kv.baz\", msg)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Check that if both set that errors.\n\t\t\tb, _ := json.Marshal(JSApiMsgGetRequest{LastFor: \"kv.foo\", Seq: 950})\n\t\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, \"KV\"), b, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar resp JSApiMsgGetResponse\n\t\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t\t\t}\n\t\t\tif resp.Error == nil {\n\t\t\t\tt.Fatalf(\"Expected an error when both are set, got %+v\", resp.Error)\n\t\t\t}\n\n\t\t\t// Need to do stream GetMsg by hand for now until Go client support lands.\n\t\t\tgetLast := func(subject string) *StoredMsg {\n\t\t\t\tt.Helper()\n\t\t\t\treq := &JSApiMsgGetRequest{LastFor: subject}\n\t\t\t\tb, _ := json.Marshal(req)\n\t\t\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, \"KV\"), b, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tvar resp JSApiMsgGetResponse\n\t\t\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t\t\t\t}\n\t\t\t\tif resp.Message == nil || resp.Error != nil {\n\t\t\t\t\tt.Fatalf(\"Did not receive correct response: %+v\", resp.Error)\n\t\t\t\t}\n\t\t\t\treturn resp.Message\n\t\t\t}\n\t\t\t// Do basic checks.\n\t\t\tbasicCheck := func(subject string, expectedSeq uint64) {\n\t\t\t\tsm := getLast(subject)\n\t\t\t\tif sm == nil {\n\t\t\t\t\tt.Fatalf(\"Expected a message but got none\")\n\t\t\t\t} else if string(sm.Data) != \"VAL-1000\" {\n\t\t\t\t\tt.Fatalf(\"Wrong message payload, wanted %q but got %q\", \"VAL-1000\", sm.Data)\n\t\t\t\t} else if sm.Sequence != expectedSeq {\n\t\t\t\t\tt.Fatalf(\"Wrong message sequence, wanted %d but got %d\", expectedSeq, sm.Sequence)\n\t\t\t\t} else if !subjectIsSubsetMatch(sm.Subject, subject) {\n\t\t\t\t\tt.Fatalf(\"Wrong subject, wanted %q but got %q\", subject, sm.Subject)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tbasicCheck(\"kv.foo\", 2998)\n\t\t\tbasicCheck(\"kv.bar\", 2999)\n\t\t\tbasicCheck(\"kv.baz\", 3000)\n\t\t\tbasicCheck(\"kv.*\", 3000)\n\t\t\tbasicCheck(\">\", 3000)\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/2329\nfunc TestJetStreamGetLastMsgBySubjectAfterUpdate(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsc := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 2,\n\t}\n\tif _, err := js.AddStream(sc); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Now Update and add in other subjects.\n\tsc.Subjects = append(sc.Subjects, \"bar\", \"baz\")\n\tif _, err := js.UpdateStream(sc); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tjs.Publish(\"foo\", []byte(\"OK1\")) // 1\n\tjs.Publish(\"bar\", []byte(\"OK1\")) // 2\n\tjs.Publish(\"foo\", []byte(\"OK2\")) // 3\n\tjs.Publish(\"bar\", []byte(\"OK2\")) // 4\n\n\t// Need to do stream GetMsg by hand for now until Go client support lands.\n\tgetLast := func(subject string) *StoredMsg {\n\t\tt.Helper()\n\t\treq := &JSApiMsgGetRequest{LastFor: subject}\n\t\tb, _ := json.Marshal(req)\n\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, \"TEST\"), b, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar resp JSApiMsgGetResponse\n\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not parse stream message: %v\", err)\n\t\t}\n\t\tif resp.Message == nil || resp.Error != nil {\n\t\t\tt.Fatalf(\"Did not receive correct response: %+v\", resp.Error)\n\t\t}\n\t\treturn resp.Message\n\t}\n\t// Do basic checks.\n\tbasicCheck := func(subject string, expectedSeq uint64) {\n\t\tsm := getLast(subject)\n\t\tif sm == nil {\n\t\t\tt.Fatalf(\"Expected a message but got none\")\n\t\t} else if sm.Sequence != expectedSeq {\n\t\t\tt.Fatalf(\"Wrong message sequence, wanted %d but got %d\", expectedSeq, sm.Sequence)\n\t\t} else if !subjectIsSubsetMatch(sm.Subject, subject) {\n\t\t\tt.Fatalf(\"Wrong subject, wanted %q but got %q\", subject, sm.Subject)\n\t\t}\n\t}\n\n\tbasicCheck(\"foo\", 3)\n\tbasicCheck(\"bar\", 4)\n}\n\nfunc TestJetStreamLastSequenceBySubject(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 3,\n\t\t\t\tMaxMsgsPer: 1,\n\t\t\t}\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v, respmsg: %q\", err, string(m.Data))\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\tjs.PublishAsync(\"kv.foo\", []byte(\"1\"))\n\t\t\tjs.PublishAsync(\"kv.bar\", []byte(\"2\"))\n\t\t\tjs.PublishAsync(\"kv.baz\", []byte(\"3\"))\n\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Now make sure we get an error if the last sequence is not correct per subject.\n\t\t\tpubAndCheck := func(subj, seq string, ok bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tm := nats.NewMsg(subj)\n\t\t\t\tm.Data = []byte(\"HELLO\")\n\t\t\t\tm.Header.Set(JSExpectedLastSubjSeq, seq)\n\t\t\t\t_, err := js.PublishMsg(m)\n\t\t\t\tif ok && err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !ok && err == nil {\n\t\t\t\t\tt.Fatalf(\"Expected to get an error and got none\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpubAndCheck(\"kv.foo\", \"1\", true) // So last is now 4.\n\t\t\tpubAndCheck(\"kv.foo\", \"1\", false) // This should fail.\n\t\t\tpubAndCheck(\"kv.bar\", \"2\", true)\n\t\t\tpubAndCheck(\"kv.bar\", \"5\", true)\n\t\t\tpubAndCheck(\"kv.xxx\", \"5\", false)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamLastSequenceBySubjectWithSubject(t *testing.T) {\n\ttest := func(replicas int, st StorageType) {\n\t\tt.Run(fmt.Sprintf(\"R%d/%s\", replicas, st), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: replicas,\n\t\t\t\tMaxMsgsPer: 1,\n\t\t\t}\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v, respmsg: %q\", err, string(m.Data))\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\tjs.PublishAsync(\"kv.1.foo\", []byte(\"1:1\")) // Last is 1 for kv.1.foo; 1 for kv.1.*;\n\t\t\tjs.PublishAsync(\"kv.1.bar\", []byte(\"1:2\")) // Last is 2 for kv.1.bar; 2 for kv.1.*;\n\t\t\tjs.PublishAsync(\"kv.2.foo\", []byte(\"2:1\")) // Last is 3 for kv.2.foo; 3 for kv.2.*;\n\t\t\tjs.PublishAsync(\"kv.3.bar\", []byte(\"3:1\")) // Last is 4 for kv.3.bar; 4 for kv.3.*;\n\t\t\tjs.PublishAsync(\"kv.1.baz\", []byte(\"1:3\")) // Last is 5 for kv.1.baz; 5 for kv.1.*;\n\t\t\tjs.PublishAsync(\"kv.1.bar\", []byte(\"1:4\")) // Last is 6 for kv.1.baz; 6 for kv.1.*;\n\t\t\tjs.PublishAsync(\"kv.2.baz\", []byte(\"2:2\")) // Last is 7 for kv.2.baz; 7 for kv.2.*;\n\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Now make sure we get an error if the last sequence is not correct per subject.\n\t\t\tpubAndCheck := func(subj, filterSubject, seq string, ok bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tm := nats.NewMsg(subj)\n\t\t\t\tm.Data = []byte(\"HELLO\")\n\n\t\t\t\t// Expect last to be seq.\n\t\t\t\tm.Header.Set(JSExpectedLastSubjSeq, seq)\n\n\t\t\t\t// Constrain the sequence restriction to a specific subject\n\t\t\t\t// e.g. \"kv.1.*\" for kv.1.foo, kv.1.bar, kv.1.baz; kv.2.* for kv.2.foo, kv.2.baz; kv.3.* for kv.3.bar\n\t\t\t\tm.Header.Set(JSExpectedLastSubjSeqSubj, filterSubject)\n\t\t\t\t_, err := js.PublishMsg(m)\n\t\t\t\tif ok && err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !ok && err == nil {\n\t\t\t\t\tt.Fatalf(\"Expected to get an error and got none\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpubAndCheck(\"kv.1.foo\", \"kv.1.*\", \"0\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"0\", false)\n\t\t\tpubAndCheck(\"kv.1.xxx\", \"kv.1.*\", \"0\", false)\n\t\t\tpubAndCheck(\"kv.1.foo\", \"kv.1.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.1.xxx\", \"kv.1.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.2.foo\", \"kv.2.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.2.bar\", \"kv.2.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.2.xxx\", \"kv.2.*\", \"1\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"2\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"3\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"4\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"5\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"6\", true) // Last is 8 for kv.1.bar; 8 for kv.1.*;\n\t\t\tpubAndCheck(\"kv.1.baz\", \"kv.1.*\", \"2\", false)\n\t\t\tpubAndCheck(\"kv.1.bar\", \"kv.1.*\", \"7\", false)\n\t\t\tpubAndCheck(\"kv.1.xxx\", \"kv.1.*\", \"8\", true) // Last is 9 for kv.1.xxx; 9 for kv.1.*;\n\t\t\tpubAndCheck(\"kv.2.foo\", \"kv.2.*\", \"2\", false)\n\t\t\tpubAndCheck(\"kv.2.foo\", \"kv.2.*\", \"7\", true) // Last is 10 for kv.2.foo; 10 for kv.2.*;\n\t\t\tpubAndCheck(\"kv.xxx\", \"kv.*\", \"0\", true) // Last is 0 for kv.xxx; 0 for kv.*;\n\t\t\tpubAndCheck(\"kv.xxx\", \"kv.*.*\", \"0\", false) // Last is 11 for kv.xxx; 11 for kv.*.*;\n\t\t\tpubAndCheck(\"kv.3.xxx\", \"kv.3.*\", \"4\", true) // Last is 12 for kv.3.xxx; 12 for kv.3.*;\n\t\t\tpubAndCheck(\"kv.3.xyz\", \"kv.3.*\", \"12\", true) // Last is 13 for kv.3.xyz; 13 for kv.3.*;\n\n\t\t\t// When using the last-subj-seq-subj header, but the sequence header is missing.\n\t\t\tm = nats.NewMsg(\"kv.invalid\")\n\t\t\tm.Data = []byte(\"HELLO\")\n\t\t\tm.Header.Set(JSExpectedLastSubjSeqSubj, \"kv.invalid\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_Error(t, err, NewJSStreamExpectedLastSeqPerSubjectInvalidError())\n\t\t})\n\t}\n\n\tfor _, replicas := range []int{1, 3} {\n\t\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\t\ttest(replicas, st)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamFilteredConsumersWithWiderFilter(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\", \"N.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Add in some messages.\n\tjs.Publish(\"foo\", []byte(\"OK\"))\n\tjs.Publish(\"bar\", []byte(\"OK\"))\n\tjs.Publish(\"baz\", []byte(\"OK\"))\n\tfor i := 0; i < 12; i++ {\n\t\tjs.Publish(fmt.Sprintf(\"N.%d\", i+1), []byte(\"OK\"))\n\t}\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 15 {\n\t\t\treturn fmt.Errorf(\"Expected 15 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckWider := func(subj string, numExpected int) {\n\t\tsub, err := js.SubscribeSync(subj)\n\t\trequire_NoError(t, err)\n\n\t\tdefer sub.Unsubscribe()\n\t\tcheckSubsPending(t, sub, numExpected)\n\t}\n\n\tcheckWider(\"*\", 3)\n\tcheckWider(\"N.*\", 12)\n\tcheckWider(\"*.*\", 12)\n\tcheckWider(\"N.>\", 12)\n\tcheckWider(\">\", 15)\n}\n\nfunc TestJetStreamMirrorAndSourcesFilteredConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Create Mirror now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\tdsubj := nats.NewInbox()\n\tnc.SubscribeSync(dsubj)\n\tnc.Flush()\n\n\tcreateConsumer := func(sn, fs string) {\n\t\tt.Helper()\n\t\t_, err = js.AddConsumer(sn, &nats.ConsumerConfig{DeliverSubject: dsubj, FilterSubject: fs})\n\t\trequire_NoError(t, err)\n\n\t}\n\n\tcreateConsumer(\"M\", \"foo\")\n\tcreateConsumer(\"M\", \"bar\")\n\tcreateConsumer(\"M\", \"baz.foo\")\n\n\t// Now do some sources.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"O1\", Subjects: []string{\"foo.*\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"O2\", Subjects: []string{\"bar.*\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create Mirror now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{Name: \"O1\"}, {Name: \"O2\"}},\n\t})\n\trequire_NoError(t, err)\n\n\tcreateConsumer(\"S\", \"foo.1\")\n\tcreateConsumer(\"S\", \"bar.1\")\n\n\t// Chaining\n\t// Create Mirror now.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M2\",\n\t\tMirror: &nats.StreamSource{Name: \"M\"},\n\t})\n\trequire_NoError(t, err)\n\n\tcreateConsumer(\"M2\", \"foo\")\n\tcreateConsumer(\"M2\", \"bar\")\n\tcreateConsumer(\"M2\", \"baz.foo\")\n}\n\nfunc TestJetStreamMirrorBasics(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcreateStream := func(cfg *nats.StreamConfig) (*nats.StreamInfo, error) {\n\t\treturn js.AddStream(cfg)\n\t}\n\n\tcreateStreamOk := func(cfg *nats.StreamConfig) {\n\t\tt.Helper()\n\t\tif _, err := createStream(cfg); err != nil {\n\t\t\tt.Fatalf(\"Expected no error, got %+v\", err)\n\t\t}\n\t}\n\n\t// Test we get right config errors etc.\n\tcfg := &nats.StreamConfig{\n\t\tName: \"M1\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tMirror: &nats.StreamSource{Name: \"S1\"},\n\t}\n\t_, err := createStream(cfg)\n\tif err == nil || !strings.Contains(err.Error(), \"stream mirrors can not\") {\n\t\tt.Fatalf(\"Expected error, got %+v\", err)\n\t}\n\n\t// Clear subjects.\n\tcfg.Subjects = nil\n\n\t// Mirrored\n\tscfg := &nats.StreamConfig{\n\t\tName: \"S1\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t}\n\n\t// Create mirrored stream\n\tcreateStreamOk(scfg)\n\n\t// Now create our mirror stream.\n\tcreateStreamOk(cfg)\n\n\t// For now wait for the consumer state to register.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Send 100 messages.\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Faster timeout since we loop below checking for condition.\n\tjs2, err := nc.JetStream(nats.MaxWait(500 * time.Millisecond))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"M1\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 100 {\n\t\t\treturn fmt.Errorf(\"Expected 100 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Purge the mirrored stream.\n\tif err := js.PurgeStream(\"S1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\t// Send 50 more msgs now.\n\tfor i := 0; i < 50; i++ {\n\t\tif _, err := js.Publish(\"bar\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcfg = &nats.StreamConfig{\n\t\tName: \"M2\",\n\t\tStorage: nats.FileStorage,\n\t\tMirror: &nats.StreamSource{Name: \"S1\"},\n\t}\n\n\t// Now create our second mirror stream.\n\tcreateStreamOk(cfg)\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"M2\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 101 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 101, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Send 100 more msgs now. Should be 150 total, 101 first.\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := js.Publish(\"baz\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcfg = &nats.StreamConfig{\n\t\tName: \"M3\",\n\t\tMirror: &nats.StreamSource{Name: \"S1\", OptStartSeq: 150},\n\t}\n\n\tcreateStreamOk(cfg)\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"M3\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 101 {\n\t\t\treturn fmt.Errorf(\"Expected 101 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 150 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 150, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure setting time works ok.\n\tstart := time.Now().UTC().Add(-2 * time.Hour)\n\tcfg = &nats.StreamConfig{\n\t\tName: \"M4\",\n\t\tMirror: &nats.StreamSource{Name: \"S1\", OptStartTime: &start},\n\t}\n\tcreateStreamOk(cfg)\n\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"M4\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 150 {\n\t\t\treturn fmt.Errorf(\"Expected 150 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 101 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 101, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Test subject filtering and transformation\n\tcreateStreamServerStreamConfig := func(cfg *StreamConfig, errToCheck uint16) {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\n\t\trm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar resp JSApiStreamCreateResponse\n\t\tif err := json.Unmarshal(rm.Data, &resp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\tif errToCheck == 0 {\n\t\t\tif resp.Error != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t\t\t}\n\t\t} else {\n\t\t\tif resp.Error.ErrCode != errToCheck {\n\t\t\t\tt.Fatalf(\"Expected error %+v, got: %+v\", errToCheck, resp.Error)\n\t\t\t}\n\t\t}\n\t}\n\n\t// check for errors\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"MBAD\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", FilterSubject: \"foo\", SubjectTransforms: []SubjectTransformConfig{{Source: \"foo\", Destination: \"foo3\"}}},\n\t}, ApiErrors[JSMirrorMultipleFiltersNotAllowed].ErrCode)\n\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"MBAD\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", SubjectTransforms: []SubjectTransformConfig{{Source: \".*.\", Destination: \"foo3\"}}},\n\t}, ApiErrors[JSMirrorInvalidSubjectFilter].ErrCode)\n\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"MBAD\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", SubjectTransforms: []SubjectTransformConfig{{Source: \"*\", Destination: \"{{wildcard(2)}}\"}}},\n\t}, ApiErrors[JSMirrorInvalidTransformDestination].ErrCode)\n\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"MBAD\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", SubjectTransforms: []SubjectTransformConfig{{Source: \"foo\", Destination: \"\"}, {Source: \"foo\", Destination: \"bar\"}}},\n\t}, ApiErrors[JSMirrorOverlappingSubjectFilters].ErrCode)\n\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"M5\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", SubjectTransforms: []SubjectTransformConfig{{Source: \"foo\", Destination: \"foo2\"}}},\n\t}, 0)\n\n\tcreateStreamServerStreamConfig(&StreamConfig{\n\t\tName: \"M6\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"S1\", SubjectTransforms: []SubjectTransformConfig{{Source: \"bar\", Destination: \"bar2\"}, {Source: \"baz\", Destination: \"baz2\"}}},\n\t}, 0)\n\n\t// Send 100 messages on foo (there should already be 50 messages on bar and 100 on baz in the stream)\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tvar f = func(streamName string, subject string, subjectNumMsgs uint64, streamNumMsg uint64, firstSeq uint64, lastSeq uint64) func() error {\n\t\treturn func() error {\n\t\t\tsi, err := js2.StreamInfo(streamName, &nats.StreamInfoRequest{SubjectsFilter: subject})\n\t\t\trequire_NoError(t, err)\n\t\t\tif ss, ok := si.State.Subjects[subject]; !ok {\n\t\t\t\treturn fmt.Errorf(\"expected messages with the transformed subject %s\", subject)\n\t\t\t} else {\n\t\t\t\tif ss != subjectNumMsgs {\n\t\t\t\t\treturn fmt.Errorf(\"expected %d messages on the transformed subject %s but got %d\", subjectNumMsgs, subject, ss)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif si.State.Msgs != streamNumMsg {\n\t\t\t\treturn fmt.Errorf(\"expected %d stream messages, got state: %+v\", streamNumMsg, si.State)\n\t\t\t}\n\t\t\tif si.State.FirstSeq != firstSeq || si.State.LastSeq != lastSeq {\n\t\t\t\treturn fmt.Errorf(\"expected first sequence=%d and last sequence=%d, but got state: %+v\", firstSeq, lastSeq, si.State)\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, f(\"M5\", \"foo2\", 100, 100, 251, 350))\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, f(\"M6\", \"bar2\", 50, 150, 101, 250))\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, f(\"M6\", \"baz2\", 100, 150, 101, 250))\n\n}\n\nfunc TestJetStreamMirrorStripExpectedHeaders(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create source and mirror streams.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"S\"},\n\t})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"foo\")\n\tpubAck, err := js.PublishMsg(m)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t// Mirror should get message.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif si, err := js.StreamInfo(\"M\"); err != nil {\n\t\t\treturn err\n\t\t} else if si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 mirrored msg, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tm.Header.Set(\"Nats-Expected-Stream\", \"S\")\n\tpubAck, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t// Mirror should strip expected headers and store the message.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif si, err := js.StreamInfo(\"M\"); err != nil {\n\t\t\treturn err\n\t\t} else if si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 mirrored msgs, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamMirrorUpdatePreventsSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"ORIGINAL\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"MIRROR\", Mirror: &nats.StreamSource{Name: \"ORIGINAL\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"MIRROR\", Mirror: &nats.StreamSource{Name: \"ORIGINAL\"}, Subjects: []string{\"x\"}})\n\tif err == nil || err.Error() != \"nats: stream mirrors can not contain subjects\" {\n\t\tt.Fatalf(\"Expected to not be able to put subjects on a stream, got: %+v\", err)\n\t}\n}\n\nfunc TestJetStreamSourceBasics(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcreateStream := func(cfg *StreamConfig) {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\n\t\trm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar resp JSApiStreamCreateResponse\n\t\tif err := json.Unmarshal(rm.Data, &resp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif resp.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t\t}\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"test\", Sources: []*nats.StreamSource{{Name: \"\"}}}); err.Error() == \"source stream name is invalid\" {\n\t\tt.Fatal(\"Expected a source stream name is invalid error\")\n\t}\n\n\tfor _, sname := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: sname}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\tsendBatch := func(subject string, n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(subject, []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\t// Populate each one.\n\tsendBatch(\"foo\", 10)\n\tsendBatch(\"bar\", 15)\n\tsendBatch(\"baz\", 25)\n\n\tcfg := &StreamConfig{\n\t\tName: \"MS\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"foo\", SubjectTransforms: []SubjectTransformConfig{{Source: \">\", Destination: \"foo2.>\"}}},\n\t\t\t{Name: \"bar\"},\n\t\t\t{Name: \"baz\"},\n\t\t},\n\t}\n\n\tcreateStream(cfg)\n\n\t// Faster timeout since we loop below checking for condition.\n\tjs2, err := nc.JetStream(nats.MaxWait(250 * time.Millisecond))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MS\")\n\t\trequire_NoError(t, err)\n\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tss, err := js.SubscribeSync(\"foo2.foo\", nats.BindStream(\"MS\"))\n\trequire_NoError(t, err)\n\t// we must have at least one message on the transformed subject name (ie no timeout)\n\t_, err = ss.NextMsg(time.Millisecond)\n\trequire_NoError(t, err)\n\tss.Drain()\n\n\t// Test Source Updates\n\tncfg := &nats.StreamConfig{\n\t\tName: \"MS\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t// Keep foo, bar, remove baz, add dlc\n\t\t\t{Name: \"foo\"},\n\t\t\t{Name: \"bar\"},\n\t\t\t{Name: \"dlc\"},\n\t\t},\n\t}\n\tif _, err := js.UpdateStream(ncfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Test optional start times, filtered subjects etc.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"dlc\", \"rip\", \"jnm\"}}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsendBatch(\"dlc\", 20)\n\tsendBatch(\"rip\", 20)\n\tsendBatch(\"dlc\", 10)\n\tsendBatch(\"jnm\", 10)\n\n\tcfg = &StreamConfig{\n\t\tName: \"FMS\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"TEST\", OptStartSeq: 26},\n\t\t},\n\t}\n\tcreateStream(cfg)\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"FMS\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 35 {\n\t\t\treturn fmt.Errorf(\"Expected 35 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\t// Double check first starting.\n\tm, err := js.GetMsg(\"FMS\", 1)\n\trequire_NoError(t, err)\n\tif shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ {\n\t\tt.Fatalf(\"Expected a header, got none\")\n\t} else if _, _, sseq := streamAndSeq(shdr); sseq != 26 {\n\t\tt.Fatalf(\"Expected header sequence of 26, got %d\", sseq)\n\t}\n\n\t// Test Filters\n\tcfg = &StreamConfig{\n\t\tName: \"FMS2\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"TEST\", OptStartSeq: 11, SubjectTransforms: []SubjectTransformConfig{{Source: \"dlc\", Destination: \"dlc2\"}}},\n\t\t},\n\t}\n\tcreateStream(cfg)\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"FMS2\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 20 {\n\t\t\treturn fmt.Errorf(\"Expected 20 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Double check first starting.\n\tif m, err = js.GetMsg(\"FMS2\", 1); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ {\n\t\tt.Fatalf(\"Expected a header, got none\")\n\t} else if _, _, sseq := streamAndSeq(shdr); sseq != 11 {\n\t\tt.Fatalf(\"Expected header sequence of 11, got %d\", sseq)\n\t}\n\tif m.Subject != \"dlc2\" {\n\t\tt.Fatalf(\"Expected transformed subject dlc2, but got %s instead\", m.Subject)\n\t}\n\n\t// Test Filters\n\tcfg = &StreamConfig{\n\t\tName: \"FMS3\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{\n\t\t\t{Name: \"TEST\", SubjectTransforms: []SubjectTransformConfig{{Source: \"dlc\", Destination: \"dlc2\"}, {Source: \"rip\", Destination: \"\"}}},\n\t\t},\n\t}\n\tcreateStream(cfg)\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"FMS3\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Double check first message\n\tif m, err = js.GetMsg(\"FMS3\", 1); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ {\n\t\tt.Fatalf(\"Expected a header, got none\")\n\t} else if m.Subject != \"dlc2\" {\n\t\tt.Fatalf(\"Expected subject 'dlc2' and got %s\", m.Subject)\n\t}\n\n\t// Double check first message with the other subject\n\tif m, err = js.GetMsg(\"FMS3\", 21); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif shdr := m.Header.Get(JSStreamSource); shdr == _EMPTY_ {\n\t\tt.Fatalf(\"Expected a header, got none\")\n\t} else if m.Subject != \"rip\" {\n\t\tt.Fatalf(\"Expected subject 'rip' and got %s\", m.Subject)\n\t}\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"FMS3\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Subjects[\"jnm\"] != 0 {\n\t\t\treturn fmt.Errorf(\"Unexpected messages from the source found\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// pre 2.10 backwards compatibility\n\ttransformConfig := nats.SubjectTransformConfig{Source: \"B.*\", Destination: \"A.{{Wildcard(1)}}\"}\n\taConfig := nats.StreamConfig{Name: \"A\", Subjects: []string{\"B.*\"}, SubjectTransform: &transformConfig}\n\tif _, err := js.AddStream(&aConfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendBatch(\"B.A\", 1)\n\tsendBatch(\"B.B\", 1)\n\tbConfig := nats.StreamConfig{Name: \"B\", Subjects: []string{\"A.*\"}}\n\n\tif _, err := js.AddStream(&bConfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// fake a message that would have been sourced with pre 2.10\n\tmsg := nats.NewMsg(\"A.A\")\n\t// pre 2.10 header format just stream name and sequence number\n\tmsg.Header.Set(JSStreamSource, \"A 1\")\n\tmsg.Data = []byte(\"OK\")\n\n\tif _, err := js.PublishMsg(msg); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tbConfig.Sources = []*nats.StreamSource{{Name: \"A\"}}\n\tif _, err := js.UpdateStream(&bConfig); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"B\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamSourceWorkingQueueWithLimit(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"test\", Subjects: []string{\"test\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"wq\", MaxMsgs: 100, Discard: nats.DiscardNew, Retention: nats.WorkQueuePolicy,\n\t\tSources: []*nats.StreamSource{{Name: \"test\"}}})\n\trequire_NoError(t, err)\n\n\tsendBatch := func(subject string, n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\t_, err = js.Publish(subject, []byte(\"OK\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\t// Populate each one.\n\tsendBatch(\"test\", 300)\n\n\tcheckFor(t, 3*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 100 {\n\t\t\treturn fmt.Errorf(\"Expected 100 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t_, err = js.AddConsumer(\"wq\", &nats.ConsumerConfig{Durable: \"wqc\", FilterSubject: \"test\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tss, err := js.PullSubscribe(\"test\", \"wqc\", nats.Bind(\"wq\", \"wqc\"))\n\trequire_NoError(t, err)\n\t// we must have at least one message on the transformed subject name (ie no timeout)\n\tf := func(done chan bool) {\n\t\tfor i := 0; i < 300; i++ {\n\t\t\tm, err := ss.Fetch(1, nats.MaxWait(3*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\ttime.Sleep(11 * time.Millisecond)\n\t\t\terr = m[0].Ack()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tdone <- true\n\t}\n\n\tvar doneChan = make(chan bool)\n\tgo f(doneChan)\n\n\tcheckFor(t, 6*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"wq\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs > 0 && si.State.Msgs <= 100 {\n\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got: %d\", si.State.Msgs)\n\t\t} else if si.State.Msgs > 100 {\n\t\t\tt.Fatalf(\"Got more than our 100 message limit: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tselect {\n\tcase <-doneChan:\n\t\tss.Drain()\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n}\n\nfunc TestJetStreamStreamSourceFromKV(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API reuqests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create a kv store\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"test\"})\n\trequire_NoError(t, err)\n\n\t// Create a stream with a source from the kv store\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"test\", Retention: nats.InterestPolicy, Sources: []*nats.StreamSource{{Name: \"KV_\" + kv.Bucket()}}})\n\trequire_NoError(t, err)\n\n\t// Create a interested consumer\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{Durable: \"durable\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tss, err := js.PullSubscribe(\"\", \"\", nats.Bind(\"test\", \"durable\"))\n\trequire_NoError(t, err)\n\n\trev1, err := kv.Create(\"key\", []byte(\"value1\"))\n\trequire_NoError(t, err)\n\n\tm, err := ss.Fetch(1, nats.MaxWait(2*time.Second))\n\trequire_NoError(t, err)\n\trequire_NoError(t, m[0].Ack())\n\tif string(m[0].Data) != \"value1\" {\n\t\tt.Fatalf(\"Expected value1, got %s\", m[0].Data)\n\t}\n\n\trev2, err := kv.Update(\"key\", []byte(\"value2\"), rev1)\n\trequire_NoError(t, err)\n\n\t_, err = kv.Update(\"key\", []byte(\"value3\"), rev2)\n\trequire_NoError(t, err)\n\n\tm, err = ss.Fetch(1, nats.MaxWait(500*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_NoError(t, m[0].Ack())\n\tif string(m[0].Data) != \"value2\" {\n\t\tt.Fatalf(\"Expected value2, got %s\", m[0].Data)\n\t}\n\n\tm, err = ss.Fetch(1, nats.MaxWait(500*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_NoError(t, m[0].Ack())\n\tif string(m[0].Data) != \"value3\" {\n\t\tt.Fatalf(\"Expected value3, got %s\", m[0].Data)\n\t}\n}\n\nfunc TestJetStreamInputTransform(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcreateStream := func(cfg *StreamConfig) {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\n\t\trm, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar resp JSApiStreamCreateResponse\n\t\tif err := json.Unmarshal(rm.Data, &resp); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif resp.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", resp.Error)\n\t\t}\n\t}\n\n\tcreateStream(&StreamConfig{Name: \"T1\", Subjects: []string{\"foo\"}, SubjectTransform: &SubjectTransformConfig{Source: \">\", Destination: \"transformed.>\"}, Storage: MemoryStorage})\n\n\t// publish a message\n\tif _, err := js.Publish(\"foo\", []byte(\"OK\")); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\n\tm, err := js.GetMsg(\"T1\", 1)\n\trequire_NoError(t, err)\n\n\tif m.Subject != \"transformed.foo\" {\n\t\tt.Fatalf(\"Expected message subject transformed.foo, got %s\", m.Subject)\n\t}\n}\n\nfunc TestJetStreamOperatorAccounts(t *testing.T) {\n\ts, _ := RunServerWithConfig(\"./configs/js-op.conf\")\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.UserCredentials(\"./configs/one.creds\"))\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\"}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 100\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\t// Close our user for account one.\n\tnc.Close()\n\n\t// Restart the server.\n\ts.Shutdown()\n\ts, _ = RunServerWithConfig(\"./configs/js-op.conf\")\n\tdefer s.Shutdown()\n\n\tjsz, err := s.Jsz(nil)\n\trequire_NoError(t, err)\n\n\tif jsz.Streams != 1 {\n\t\tt.Fatalf(\"Expected jsz to report our stream on restart\")\n\t}\n\tif jsz.Messages != uint64(toSend) {\n\t\tt.Fatalf(\"Expected jsz to report our %d messages on restart, got %d\", toSend, jsz.Messages)\n\t}\n}\n\nfunc TestJetStreamServerDomainBadConfig(t *testing.T) {\n\tshouldFail := func(domain string) {\n\t\tt.Helper()\n\t\topts := DefaultTestOptions\n\t\topts.JetStreamDomain = domain\n\t\tif err := validateOptions(&opts); err == nil || !strings.Contains(err.Error(), \"invalid domain name\") {\n\t\t\tt.Fatalf(\"Expected bad domain error, got %v\", err)\n\t\t}\n\t}\n\n\tshouldFail(\"HU..B\")\n\tshouldFail(\"HU B\")\n\tshouldFail(\" \")\n\tshouldFail(\"\\t\")\n\tshouldFail(\"CORE.\")\n\tshouldFail(\".CORE\")\n\tshouldFail(\"C.*.O. RE\")\n\tshouldFail(\"C.ORE\")\n}\n\nfunc TestJetStreamServerDomainConfig(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {domain: \"HUB\", store_dir: %q}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\n\tconfig := s.JetStreamConfig()\n\tif config.Domain != \"HUB\" {\n\t\tt.Fatalf(\"Expected %q as domain name, got %q\", \"HUB\", config.Domain)\n\t}\n}\n\nfunc TestJetStreamServerDomainConfigButDisabled(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {domain: \"HUB\", enabled: false}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tif s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream NOT to be enabled\")\n\t}\n\n\topts := s.getOpts()\n\tif opts.JetStreamDomain != \"HUB\" {\n\t\tt.Fatalf(\"Expected %q as opts domain name, got %q\", \"HUB\", opts.JetStreamDomain)\n\t}\n}\n\nfunc TestJetStreamDomainInPubAck(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {domain: \"HUB\", store_dir: %q}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Check by hand for now til it makes its way into Go client.\n\tam, err := nc.Request(\"foo\", nil, time.Second)\n\trequire_NoError(t, err)\n\tvar pa PubAck\n\tjson.Unmarshal(am.Data, &pa)\n\tif pa.Domain != \"HUB\" {\n\t\tt.Fatalf(\"Expected PubAck to have domain of %q, got %q\", \"HUB\", pa.Domain)\n\t}\n}\n\n// Issue #2213\nfunc TestJetStreamDirectConsumersBeingReported(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"TEST\",\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\tif _, err = js.Publish(\"foo\", nil); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"Could not get stream info: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 msg, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Direct consumers should not be reported\n\tif si.State.Consumers != 0 {\n\t\tt.Fatalf(\"Did not expect any consumers, got %d\", si.State.Consumers)\n\t}\n\n\t// Now check for consumer in consumer names list.\n\tvar names []string\n\tfor name := range js.ConsumerNames(\"TEST\") {\n\t\tnames = append(names, name)\n\t}\n\tif len(names) != 0 {\n\t\tt.Fatalf(\"Expected no consumers but got %+v\", names)\n\t}\n\n\t// Now check detailed list.\n\tvar cis []*nats.ConsumerInfo\n\tfor ci := range js.ConsumersInfo(\"TEST\") {\n\t\tcis = append(cis, ci)\n\t}\n\tif len(cis) != 0 {\n\t\tt.Fatalf(\"Expected no consumers but got %+v\", cis)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/2290\nfunc TestJetStreamTemplatedErrorsBug(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.PullSubscribe(\"foo\", \"\")\n\tif err != nil && strings.Contains(err.Error(), \"{err}\") {\n\t\tt.Fatalf(\"Error is not filled in: %v\", err)\n\t}\n}\n\nfunc TestJetStreamServerEncryption(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcstr string\n\t\tcipher StoreCipher\n\t}{\n\t\t{\"Default\", _EMPTY_, ChaCha},\n\t\t{\"ChaCha\", \", cipher: chacha\", ChaCha},\n\t\t{\"AES\", \", cipher: aes\", AES},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\t\tserver_name: S22\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tjetstream: {key: $JS_KEY, store_dir: '%s' %s}\n\t\t\t`\n\t\t\tstoreDir := t.TempDir()\n\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, c.cstr)))\n\n\t\t\tos.Setenv(\"JS_KEY\", \"s3cr3t!!\")\n\t\t\tdefer os.Unsetenv(\"JS_KEY\")\n\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tconfig := s.JetStreamConfig()\n\t\t\tif config == nil {\n\t\t\t\tt.Fatalf(\"Expected config but got none\")\n\t\t\t}\n\t\t\tdefer removeDir(t, config.StoreDir)\n\n\t\t\t// Client based API\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\t\t}\n\t\t\tif _, err := js.AddStream(cfg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tmsg := []byte(\"ENCRYPTED PAYLOAD!!\")\n\t\t\tsendMsg := func(subj string) {\n\t\t\t\tt.Helper()\n\t\t\t\tif _, err := js.Publish(subj, msg); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Send 10 msgs\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsendMsg(\"foo\")\n\t\t\t}\n\n\t\t\t// Now create a consumer.\n\t\t\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tfor i, m := range fetchMsgs(t, sub, 10, 5*time.Second) {\n\t\t\t\tif i < 5 {\n\t\t\t\t\tm.AckSync()\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Grab our state to compare after restart.\n\t\t\tsi, _ := js.StreamInfo(\"TEST\")\n\t\t\tci, _ := js.ConsumerInfo(\"TEST\", \"dlc\")\n\n\t\t\t// Quick check to make sure everything not just plaintext still.\n\t\t\tsdir := filepath.Join(config.StoreDir, \"$G\", \"streams\", \"TEST\")\n\t\t\t// Make sure we can not find any plaintext strings in the target file.\n\t\t\tcheckFor := func(fn string, strs ...string) {\n\t\t\t\tt.Helper()\n\t\t\t\tdata, err := os.ReadFile(fn)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tfor _, str := range strs {\n\t\t\t\t\tif bytes.Contains(data, []byte(str)) {\n\t\t\t\t\t\tt.Fatalf(\"Found %q in body of file contents\", str)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckKeyFile := func(fn string) {\n\t\t\t\tt.Helper()\n\t\t\t\tif _, err := os.Stat(fn); err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected a key file at %q\", fn)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check stream meta.\n\t\t\tcheckEncrypted := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckKeyFile(filepath.Join(sdir, JetStreamMetaFileKey))\n\t\t\t\tcheckFor(filepath.Join(sdir, JetStreamMetaFile), \"TEST\", \"foo\", \"bar\", \"baz\", \"max_msgs\", \"max_bytes\")\n\t\t\t\t// Check a message block.\n\t\t\t\tcheckKeyFile(filepath.Join(sdir, \"msgs\", \"1.key\"))\n\t\t\t\tcheckFor(filepath.Join(sdir, \"msgs\", \"1.blk\"), \"ENCRYPTED PAYLOAD!!\", \"foo\", \"bar\", \"baz\")\n\n\t\t\t\t// Check consumer meta and state.\n\t\t\t\tcheckKeyFile(filepath.Join(sdir, \"obs\", \"dlc\", JetStreamMetaFileKey))\n\t\t\t\tcheckFor(filepath.Join(sdir, \"obs\", \"dlc\", JetStreamMetaFile), \"TEST\", \"dlc\", \"foo\", \"bar\", \"baz\", \"max_msgs\", \"ack_policy\")\n\t\t\t\t// Load and see if we can parse the consumer state.\n\t\t\t\tstate, err := os.ReadFile(filepath.Join(sdir, \"obs\", \"dlc\", \"o.dat\"))\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tif _, err := decodeConsumerState(state); err == nil {\n\t\t\t\t\tt.Fatalf(\"Expected decoding consumer state to fail\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Stop current\n\t\t\ts.Shutdown()\n\n\t\t\tcheckEncrypted()\n\n\t\t\t// Restart.\n\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Connect again.\n\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tsi2, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif !reflect.DeepEqual(si, si2) {\n\t\t\t\tt.Fatalf(\"Stream infos did not match\\n%+v\\nvs\\n%+v\", si, si2)\n\t\t\t}\n\n\t\t\tci2, _ := js.ConsumerInfo(\"TEST\", \"dlc\")\n\t\t\t// Consumer create times can be slightly off after restore from disk.\n\t\t\tnow := time.Now()\n\t\t\tci.Created, ci2.Created = now, now\n\t\t\tci.Delivered.Last, ci2.Delivered.Last = nil, nil\n\t\t\tci.AckFloor.Last, ci2.AckFloor.Last = nil, nil\n\t\t\t// Also clusters will be different.\n\t\t\tci.Cluster, ci2.Cluster = nil, nil\n\t\t\tif !reflect.DeepEqual(ci, ci2) {\n\t\t\t\tt.Fatalf(\"Consumer infos did not match\\n%+v\\nvs\\n%+v\", ci, ci2)\n\t\t\t}\n\n\t\t\t// Send 10 more msgs\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsendMsg(\"foo\")\n\t\t\t}\n\t\t\tif si, err = js.StreamInfo(\"TEST\"); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != 20 {\n\t\t\t\tt.Fatalf(\"Expected 20 msgs total, got %d\", si.State.Msgs)\n\t\t\t}\n\n\t\t\t// Now test snapshots etc.\n\t\t\tacc := s.GlobalAccount()\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tscfg := mset.config()\n\t\t\tsr, err := mset.snapshot(5*time.Second, false, true)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting snapshot: %v\", err)\n\t\t\t}\n\t\t\tsnapshot, err := io.ReadAll(sr.Reader)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading snapshot\")\n\t\t\t}\n\n\t\t\t// Run new server w/o encryption. Make sure we can restore properly (meaning encryption was stripped etc).\n\t\t\tns := RunBasicJetStreamServer(t)\n\t\t\tdefer ns.Shutdown()\n\n\t\t\tnacc := ns.GlobalAccount()\n\t\t\tr := bytes.NewReader(snapshot)\n\t\t\tmset, err = nacc.RestoreStream(&scfg, r)\n\t\t\trequire_NoError(t, err)\n\t\t\tss := mset.store.State()\n\t\t\tif ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {\n\t\t\t\tt.Fatalf(\"Stream states do not match: %+v vs %+v\", ss, si.State)\n\t\t\t}\n\n\t\t\t// Now restore to our encrypted server as well.\n\t\t\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tacc = s.GlobalAccount()\n\t\t\tr.Reset(snapshot)\n\t\t\tmset, err = acc.RestoreStream(&scfg, r)\n\t\t\trequire_NoError(t, err)\n\t\t\tss = mset.store.State()\n\t\t\tif ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {\n\t\t\t\tt.Fatalf(\"Stream states do not match: %+v vs %+v\", ss, si.State)\n\t\t\t}\n\n\t\t\t// Check that all is encrypted like above since we know we need to convert since snapshots always plaintext.\n\t\t\tcheckEncrypted()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamServerEncryptionServerRestarts(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcstr string\n\t\tcipher StoreCipher\n\t}{\n\t\t{\"Default\", _EMPTY_, ChaCha},\n\t\t{\"ChaCha\", \", cipher: chacha\", ChaCha},\n\t\t{\"AES\", \", cipher: aes\", AES},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\t\tserver_name: S22\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tjetstream: {key: $JS_KEY, store_dir: '%s' %s}\n\t\t\t`\n\t\t\tstoreDir := t.TempDir()\n\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, c.cstr)))\n\n\t\t\tos.Setenv(\"JS_KEY\", \"s3cr3t!!\")\n\t\t\tdefer os.Unsetenv(\"JS_KEY\")\n\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tconfig := s.JetStreamConfig()\n\t\t\tif config == nil {\n\t\t\t\tt.Fatalf(\"Expected config but got none\")\n\t\t\t}\n\t\t\tdefer removeDir(t, config.StoreDir)\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Add stream.\n\t\t\tcfg := &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: nats.FileStorage,\n\t\t\t}\n\t\t\t_, err := js.AddStream(cfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Restart to invalidate any in-memory state that adding the stream created.\n\t\t\ts.Shutdown()\n\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc.Close()\n\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tmsg := []byte(\"ENCRYPTED PAYLOAD!!\")\n\t\t\t_, err = js.Publish(\"foo\", msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Restart to invalidate any in-memory state that the publish initialized.\n\t\t\ts.Shutdown()\n\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc.Close()\n\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t// Should still be able to get the data.\n\t\t\tsub, err := js.SubscribeSync(\"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Drain()\n\n\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tmeta, err := m.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, meta.Sequence.Stream, 1)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamDeliverLastPerSubject(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Client for API requests.\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tMaxMsgsPer: 5,\n\t\t\t}\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\t// Interleave them on purpose.\n\t\t\tfor i := 1; i <= 11; i++ {\n\t\t\t\tmsg := []byte(fmt.Sprintf(\"%d\", i))\n\t\t\t\tjs.PublishAsync(\"kv.b1.foo\", msg)\n\t\t\t\tjs.PublishAsync(\"kv.b2.foo\", msg)\n\n\t\t\t\tjs.PublishAsync(\"kv.b1.bar\", msg)\n\t\t\t\tjs.PublishAsync(\"kv.b2.bar\", msg)\n\n\t\t\t\tjs.PublishAsync(\"kv.b1.baz\", msg)\n\t\t\t\tjs.PublishAsync(\"kv.b2.baz\", msg)\n\t\t\t}\n\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(2 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Do quick check that config needs FilteredSubjects otherwise bad config.\n\t\t\tbadReq := CreateConsumerRequest{\n\t\t\t\tStream: \"KV\",\n\t\t\t\tConfig: ConsumerConfig{\n\t\t\t\t\tDeliverSubject: \"b\",\n\t\t\t\t\tDeliverPolicy: DeliverLastPerSubject,\n\t\t\t\t},\n\t\t\t}\n\t\t\treq, err = json.Marshal(badReq)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"KV\"), req, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar ccResp JSApiConsumerCreateResponse\n\t\t\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif ccResp.Error == nil || !strings.Contains(ccResp.Error.Description, \"filter subject is not set\") {\n\t\t\t\tt.Fatalf(\"Expected an error, got none\")\n\t\t\t}\n\n\t\t\t// Now let's consume these via last per subject.\n\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\tStream: \"KV\",\n\t\t\t\tConfig: ConsumerConfig{\n\t\t\t\t\tDeliverSubject: \"d\",\n\t\t\t\t\tDeliverPolicy: DeliverLastPerSubject,\n\t\t\t\t\tFilterSubject: \"kv.b1.*\",\n\t\t\t\t},\n\t\t\t}\n\t\t\treq, err = json.Marshal(obsReq)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tresp, err = nc.Request(fmt.Sprintf(JSApiConsumerCreateT, \"KV\"), req, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tccResp.Error = nil\n\t\t\tif err = json.Unmarshal(resp.Data, &ccResp); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tsub, _ := nc.SubscribeSync(\"d\")\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// Helper to check messages are correct.\n\t\t\tcheckNext := func(subject string, sseq uint64, v string) {\n\t\t\t\tt.Helper()\n\t\t\t\tm, err := sub.NextMsg(time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t\t\t\t}\n\t\t\t\tif m.Subject != subject {\n\t\t\t\t\tt.Fatalf(\"Expected subject %q but got %q\", subject, m.Subject)\n\t\t\t\t}\n\t\t\t\tmeta, err := m.Metadata()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"didn't get metadata: %s\", err)\n\t\t\t\t}\n\t\t\t\tif meta.Sequence.Stream != sseq {\n\t\t\t\t\tt.Fatalf(\"Expected stream seq %d but got %d\", sseq, meta.Sequence.Stream)\n\t\t\t\t}\n\t\t\t\tif string(m.Data) != v {\n\t\t\t\t\tt.Fatalf(\"Expected data of %q but got %q\", v, m.Data)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheckSubsPending(t, sub, 3)\n\n\t\t\t// Now make sure they are what we expect.\n\t\t\tcheckNext(\"kv.b1.foo\", 61, \"11\")\n\t\t\tcheckNext(\"kv.b1.bar\", 63, \"11\")\n\t\t\tcheckNext(\"kv.b1.baz\", 65, \"11\")\n\n\t\t\tmsg := []byte(fmt.Sprintf(\"%d\", 22))\n\t\t\tjs.Publish(\"kv.b1.bar\", msg)\n\t\t\tjs.Publish(\"kv.b2.foo\", msg) // Not filtered through..\n\n\t\t\tcheckSubsPending(t, sub, 1)\n\t\t\tcheckNext(\"kv.b1.bar\", 67, \"22\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamDeliverLastPerSubjectNumPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"KV.>\"},\n\t\tMaxMsgsPerSubject: 5,\n\t\tReplicas: 1,\n\t}); err != nil {\n\t\tt.Fatalf(\"Error adding stream: %v\", err)\n\t}\n\n\tfor i := 0; i < 5; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"msg%d\", i))\n\t\tjs.Publish(\"KV.foo\", msg)\n\t\tjs.Publish(\"KV.bar\", msg)\n\t\tjs.Publish(\"KV.baz\", msg)\n\t\tjs.Publish(\"KV.bat\", msg)\n\t}\n\n\t// Delete some messages\n\tjs.DeleteMsg(\"KV\", 2)\n\tjs.DeleteMsg(\"KV\", 5)\n\n\tci, err := js.AddConsumer(\"KV\", &nats.ConsumerConfig{\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\tFilterSubject: \"KV.>\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error adding consumer: %v\", err)\n\t}\n\tif ci.NumPending != 4 {\n\t\tt.Fatalf(\"Expected 4 pending msgs, got %v\", ci.NumPending)\n\t}\n}\n\n// Issue #2392\nfunc TestJetStreamPurgeEffectsConsumerDelivery(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tjs.Publish(\"foo.a\", []byte(\"show once\"))\n\n\tsub, err := js.SubscribeSync(\"foo.*\", nats.AckWait(250*time.Millisecond), nats.DeliverAll(), nats.AckExplicit())\n\trequire_NoError(t, err)\n\n\tdefer sub.Unsubscribe()\n\n\tcheckSubsPending(t, sub, 1)\n\n\t// Do not ack.\n\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\n\t// Now purge stream.\n\tif err := js.PurgeStream(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\n\tjs.Publish(\"foo.b\", []byte(\"show twice?\"))\n\t// Do not ack again, should show back up.\n\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\t// Make sure we get it back.\n\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n}\n\n// Issue #2403\nfunc TestJetStreamExpireCausesDeadlock(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tMaxMsgs: 10,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"foo.bar\")\n\trequire_NoError(t, err)\n\n\tdefer sub.Unsubscribe()\n\n\t// Publish from two connections to get the write lock request wedged in between\n\t// having the RLock and wanting it again deeper in the stack.\n\tnc2, js2 := jsClientConnect(t, s)\n\tdefer nc2.Close()\n\n\tfor i := 0; i < 1000; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"HELLO\"))\n\t\tjs2.PublishAsync(\"foo.bar\", []byte(\"HELLO\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// If we deadlocked then we will not be able to get stream info.\n\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\n// Issue #2420\nfunc TestJetStreamDefaultMaxMsgsPer(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tMaxMsgs: 10,\n\t})\n\trequire_NoError(t, err)\n\n\tif si.Config.MaxMsgsPerSubject != -1 {\n\t\tt.Fatalf(\"Expected default of -1, got %d\", si.Config.MaxMsgsPerSubject)\n\t}\n}\n\n// Got a report of streams that expire all messages while the server is down report errors when clients reconnect\n// and try to send new messages.\nfunc TestJetStreamExpireAllWhileServerDown(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tMaxAge: 250 * time.Millisecond,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ttoSend := 10_000\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"TEST\", []byte(\"OK\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\ttime.Sleep(300 * time.Millisecond)\n\n\t// Restart after expire.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif si, err := js.StreamInfo(\"TEST\"); err != nil || si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Unexpected stream info state: %+v\", si)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tif si, err := js.StreamInfo(\"TEST\"); err != nil || si.State.Msgs != 10 {\n\t\tt.Fatalf(\"Unexpected stream info state: %+v\", si)\n\t}\n}\n\nfunc TestJetStreamLongStreamNamesAndPubAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: strings.Repeat(\"ZABC\", 256/4)[:255],\n\t\tSubjects: []string{\"foo\"},\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tjs.Publish(\"foo\", []byte(\"HELLO\"))\n}\n\nfunc TestJetStreamPerSubjectPending(t *testing.T) {\n\tfor _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"KV_X\",\n\t\t\t\tSubjects: []string{\"$KV.X.>\"},\n\t\t\t\tMaxMsgsPerSubject: 5,\n\t\t\t\tStorage: st,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"add stream failed: %s\", err)\n\t\t\t}\n\n\t\t\t// the message we will care for\n\t\t\t_, err = js.Publish(\"$KV.X.x.y.z\", []byte(\"hello world\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"publish failed: %s\", err)\n\t\t\t}\n\n\t\t\t// make sure there's some unrelated message after\n\t\t\t_, err = js.Publish(\"$KV.X.1\", []byte(\"hello world\"))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"publish failed: %s\", err)\n\t\t\t}\n\n\t\t\t// we expect the wildcard filter subject to match only the one message and so pending will be 0\n\t\t\tsub, err := js.SubscribeSync(\"$KV.X.x.>\", nats.DeliverLastPerSubject())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"subscribe failed: %s\", err)\n\t\t\t}\n\n\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"next failed: %s\", err)\n\t\t\t}\n\n\t\t\tmeta, err := msg.Metadata()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"meta failed: %s\", err)\n\t\t\t}\n\n\t\t\t// with DeliverLastPerSubject set this is never 0, but without setting that its 0 correctly\n\t\t\tif meta.NumPending != 0 {\n\t\t\t\tt.Fatalf(\"expected numpending 0 got %d\", meta.NumPending)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamPublishExpectNoMsg(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"KV.>\"},\n\t\tMaxMsgsPerSubject: 5,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"add stream failed: %s\", err)\n\t}\n\n\tif _, err = js.Publish(\"KV.22\", []byte(\"hello world\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// This should succeed.\n\tm := nats.NewMsg(\"KV.33\")\n\tm.Header.Set(JSExpectedLastSubjSeq, \"0\")\n\tif _, err := js.PublishMsg(m); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// This should fail.\n\tm = nats.NewMsg(\"KV.22\")\n\tm.Header.Set(JSExpectedLastSubjSeq, \"0\")\n\tif _, err := js.PublishMsg(m); err == nil {\n\t\tt.Fatalf(\"Expected error: %v\", err)\n\t}\n\n\tif err := js.PurgeStream(\"KV\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\n\t// This should succeed now.\n\tif _, err := js.PublishMsg(m); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestJetStreamNegativeDupeWindow(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// we incorrectly set MaxAge to -1 which then as a side effect sets dupe window to -1 which should fail\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: nil,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tMaxConsumers: 1,\n\t\tMaxMsgs: -1,\n\t\tMaxBytes: -1,\n\t\tDiscard: nats.DiscardNew,\n\t\tMaxAge: 0,\n\t\tDuplicates: -1,\n\t\tMaxMsgsPerSubject: -1,\n\t\tMaxMsgSize: -1,\n\t\tStorage: nats.FileStorage,\n\t\tReplicas: 1,\n\t\tNoAck: false,\n\t})\n\tif err == nil || err.Error() != \"nats: duplicates window can not be negative\" {\n\t\tt.Fatalf(\"Expected dupe window error got: %v\", err)\n\t}\n}\n\n// Issue #2551\nfunc TestJetStreamMirroredConsumerFailAfterRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S1\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"create failed: %s\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M1\",\n\t\tStorage: nats.FileStorage,\n\t\tMirror: &nats.StreamSource{Name: \"S1\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"create failed: %s\", err)\n\t}\n\n\t_, err = js.AddConsumer(\"M1\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tFilterSubject: \">\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"consumer create failed: %s\", err)\n\t}\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err = js.StreamInfo(\"M1\")\n\tif err != nil {\n\t\tt.Fatalf(\"%s did not exist after start: %s\", \"M1\", err)\n\t}\n\n\t_, err = js.ConsumerInfo(\"M1\", \"C1\")\n\tif err != nil {\n\t\tt.Fatalf(\"C1 did not exist after start: %s\", err)\n\t}\n}\n\nfunc TestJetStreamDisabledLimitsEnforcementJWT(t *testing.T) {\n\tupdateJwt := func(url string, akp nkeys.KeyPair, pubKey string, jwt string) {\n\t\tt.Helper()\n\t\tc := natsConnect(t, url, createUserCreds(t, nil, akp))\n\t\tdefer c.Close()\n\t\tif msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t} else {\n\t\t\tcontent := make(map[string]any)\n\t\t\tif err := json.Unmarshal(msg.Data, &content); err != nil {\n\t\t\t\tt.Fatalf(\"%v\", err)\n\t\t\t} else if _, ok := content[\"data\"]; !ok {\n\t\t\t\tt.Fatalf(\"did not get an ok response got: %v\", content)\n\t\t\t}\n\t\t}\n\t}\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\t// limits to apply and check\n\tlimits1 := jwt.JetStreamLimits{MemoryStorage: 1024, DiskStorage: 0, Streams: 1, Consumer: 2}\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\tclaim.Limits.JetStreamLimits = limits1\n\taJwt1, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tdir := t.TempDir()\n\tstoreDir1 := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\tjetstream: {store_dir: '%s'}\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, storeDir1, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(s.ClientURL(), sysKp, aPub, aJwt1)\n\tc := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp), nats.ReconnectWait(200*time.Millisecond))\n\tdefer c.Close()\n\t// keep using the same connection\n\tjs, err := c.JetStream()\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"disk\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"disk\"},\n\t})\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamDisabledLimitsEnforcement(t *testing.T) {\n\tstoreDir1 := t.TempDir()\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\taccounts {\n\t\t\tone {\n\t\t\t\tjetstream: {\n\t\t\t\t\tmem: 1024\n\t\t\t\t\tdisk: 0\n\t\t\t\t\tstreams: 1\n\t\t\t\t\tconsumers: 2\n\t\t\t\t}\n\t\t\t\tusers [{user: one, password: password}]\n\t\t\t}\n\t\t}\n\t\tno_auth_user: one\n\t`, storeDir1)))\n\ts, _ := RunServerWithConfig(conf1)\n\tdefer s.Shutdown()\n\n\tc := natsConnect(t, s.ClientURL())\n\tdefer c.Close()\n\t// keep using the same connection\n\tjs, err := c.JetStream()\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"disk\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"disk\"},\n\t})\n\trequire_Error(t, err)\n}\n\n// Issue #2607\nfunc TestJetStreamPurgeAndFilteredConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S\", Subjects: []string{\"FOO.*\"}})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(\"FOO.adam\", []byte(\"M\"))\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(\"FOO.eve\", []byte(\"F\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tci, err := js.AddConsumer(\"S\", &nats.ConsumerConfig{\n\t\tDurable: \"adam\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"FOO.adam\",\n\t})\n\trequire_NoError(t, err)\n\tif ci.NumPending != 10 {\n\t\tt.Fatalf(\"Expected NumPending to be 10, got %d\", ci.NumPending)\n\t}\n\n\tci, err = js.AddConsumer(\"S\", &nats.ConsumerConfig{\n\t\tDurable: \"eve\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubject: \"FOO.eve\",\n\t})\n\trequire_NoError(t, err)\n\tif ci.NumPending != 10 {\n\t\tt.Fatalf(\"Expected NumPending to be 10, got %d\", ci.NumPending)\n\t}\n\n\t// Also check unfiltered with interleaving messages.\n\t_, err = js.AddConsumer(\"S\", &nats.ConsumerConfig{\n\t\tDurable: \"all\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Now purge only adam.\n\tjr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: \"FOO.adam\"})\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"S\"), jr, time.Second)\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"S\")\n\trequire_NoError(t, err)\n\tif si.State.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 messages after purge, got %d\", si.State.Msgs)\n\t}\n\n\tci, err = js.ConsumerInfo(\"S\", \"eve\")\n\trequire_NoError(t, err)\n\tif ci.NumPending != 10 {\n\t\tt.Fatalf(\"Expected NumPending to be 10, got %d\", ci.NumPending)\n\t}\n\n\tci, err = js.ConsumerInfo(\"S\", \"adam\")\n\trequire_NoError(t, err)\n\tif ci.NumPending != 0 {\n\t\tt.Fatalf(\"Expected NumPending to be 0, got %d\", ci.NumPending)\n\t}\n\tif ci.AckFloor.Stream != 20 {\n\t\tt.Fatalf(\"Expected AckFloor for stream to be 20, got %d\", ci.AckFloor.Stream)\n\t}\n\n\tci, err = js.ConsumerInfo(\"S\", \"all\")\n\trequire_NoError(t, err)\n\tif ci.NumPending != 10 {\n\t\tt.Fatalf(\"Expected NumPending to be 10, got %d\", ci.NumPending)\n\t}\n}\n\n// Issue #2662\nfunc TestJetStreamLargeExpiresAndServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmaxAge := 2 * time.Second\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxAge: maxAge,\n\t})\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\t_, err = js.Publish(\"foo\", []byte(\"ok\"))\n\trequire_NoError(t, err)\n\n\t// Wait total of maxAge - 1s.\n\ttime.Sleep(maxAge - time.Since(start) - time.Second)\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tif waited := time.Since(start); waited > maxAge+time.Second {\n\t\tt.Fatalf(\"Waited to long %v vs %v for messages to expire\", waited, maxAge)\n\t}\n}\n\n// Bug that was reported showing memstore not handling max per subject of 1.\nfunc TestJetStreamMessagePerSubjectKeepBug(t *testing.T) {\n\ttest := func(t *testing.T, keep int64, store nats.StorageType) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tMaxMsgsPerSubject: keep,\n\t\t\tStorage: store,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, err = js.Publish(\"TEST\", []byte(fmt.Sprintf(\"test %d\", i)))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tnfo, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tif nfo.State.Msgs != uint64(keep) {\n\t\t\tt.Fatalf(\"Expected %d message got %d\", keep, nfo.State.Msgs)\n\t\t}\n\t}\n\n\tt.Run(\"FileStore\", func(t *testing.T) {\n\t\tt.Run(\"Keep 10\", func(t *testing.T) { test(t, 10, nats.FileStorage) })\n\t\tt.Run(\"Keep 1\", func(t *testing.T) { test(t, 1, nats.FileStorage) })\n\t})\n\n\tt.Run(\"MemStore\", func(t *testing.T) {\n\t\tt.Run(\"Keep 10\", func(t *testing.T) { test(t, 10, nats.MemoryStorage) })\n\t\tt.Run(\"Keep 1\", func(t *testing.T) { test(t, 1, nats.MemoryStorage) })\n\t})\n}\n\nfunc TestJetStreamInvalidDeliverSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{DeliverSubject: \" x\"})\n\trequire_Error(t, err, NewJSConsumerInvalidDeliverSubjectError())\n}\n\nfunc TestJetStreamMemoryCorruption(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\terrCh := make(chan error, 10)\n\tnc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\tselect {\n\t\tcase errCh <- e:\n\t\tdefault:\n\t\t}\n\t})\n\n\t// The storage has to be MemoryStorage to show the issue\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"bucket\", Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\tw1, err := kv.WatchAll()\n\trequire_NoError(t, err)\n\n\tw2, err := kv.WatchAll(nats.MetaOnly())\n\trequire_NoError(t, err)\n\n\tkv.Put(\"key1\", []byte(\"aaa\"))\n\tkv.Put(\"key1\", []byte(\"aab\"))\n\tkv.Put(\"key2\", []byte(\"zza\"))\n\tkv.Put(\"key2\", []byte(\"zzb\"))\n\tkv.Delete(\"key1\")\n\tkv.Delete(\"key2\")\n\tkv.Put(\"key1\", []byte(\"aac\"))\n\tkv.Put(\"key2\", []byte(\"zzc\"))\n\tkv.Delete(\"key1\")\n\tkv.Delete(\"key2\")\n\tkv.Purge(\"key1\")\n\tkv.Purge(\"key2\")\n\n\tcheckUpdates := func(updates <-chan nats.KeyValueEntry) {\n\t\tt.Helper()\n\t\tcount := 0\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-updates:\n\t\t\t\tcount++\n\t\t\t\tif count == 13 {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatal(\"Did not receive all updates\")\n\t\t\t}\n\t\t}\n\t}\n\tcheckUpdates(w1.Updates())\n\tcheckUpdates(w2.Updates())\n\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatal(e)\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\nfunc TestJetStreamRecoverBadStreamSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\tf := filepath.Join(sd, \"$G\", \"streams\", \"TEST\")\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: f}, StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \" baz \"}, // baz has spaces\n\t\tStorage: FileStorage,\n\t})\n\trequire_NoError(t, err)\n\tfs.Stop()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif len(si.Config.Subjects) != 3 {\n\t\tt.Fatalf(\"Expected to recover all subjects\")\n\t}\n}\n\nfunc TestJetStreamRecoverBadMirrorConfigWithSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tsd := s.JetStreamConfig().StoreDir\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\ts.Shutdown()\n\n\tf := filepath.Join(sd, \"$G\", \"streams\", \"M\")\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: f}, StreamConfig{\n\t\tName: \"M\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"}, // Mirrors should not have spaces.\n\t\tMirror: &StreamSource{Name: \"S\"},\n\t\tStorage: FileStorage,\n\t})\n\trequire_NoError(t, err)\n\tfs.Stop()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"M\")\n\trequire_NoError(t, err)\n\n\tif len(si.Config.Subjects) != 0 {\n\t\tt.Fatalf(\"Expected to have NO subjects on mirror\")\n\t}\n}\n\nfunc TestJetStreamCrossAccountsDeliverSubjectInterest(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: a, password: pwd} ]\n\t\t\t\texports [\n\t\t\t\t\t{ stream: \"_d_\" } # For the delivery subject for the consumer\n\t\t\t\t]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tusers: [ {user: b, password: pwd} ]\n\t\t\t\timports [\n\t\t\t\t\t{ stream: { account: A, subject: \"_d_\"}, to: \"foo\" }\n\t\t\t\t]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tmsg, toSend := []byte(\"OK\"), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err := js.PublishAsync(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Now create the consumer as well here manually that we will want to reference from Account B.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", DeliverSubject: \"_d_\"})\n\trequire_NoError(t, err)\n\n\t// Wait to see if the stream import signals to deliver messages with no real subscriber interest.\n\ttime.Sleep(200 * time.Millisecond)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Make sure we have not delivered any messages based on the import signal alone.\n\tif ci.NumPending != uint64(toSend) || ci.Delivered.Consumer != 0 {\n\t\tt.Fatalf(\"Bad consumer info, looks like we started delivering: %+v\", ci)\n\t}\n\n\t// Now create interest in the delivery subject through the import on account B.\n\tnc, _ = jsClientConnect(t, s, nats.UserInfo(\"b\", \"pwd\"))\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\trequire_NoError(t, err)\n\tcheckSubsPending(t, sub, toSend)\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Make sure our consumer info reflects we delivered the messages.\n\tif ci.NumPending != 0 || ci.Delivered.Consumer != uint64(toSend) {\n\t\tt.Fatalf(\"Bad consumer info, looks like we did not deliver: %+v\", ci)\n\t}\n}\n\nfunc TestJetStreamEphemeralPullConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"EC\", Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\tci, err := js.AddConsumer(\"EC\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"EC\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(ci.Name)\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\terr = o.setInActiveDeleteThreshold(50 * time.Millisecond)\n\trequire_NoError(t, err)\n\n\ttime.Sleep(100 * time.Millisecond)\n\t// Should no longer be around.\n\tif o := mset.lookupConsumer(ci.Name); o != nil {\n\t\tt.Fatalf(\"Expected consumer to be closed and removed\")\n\t}\n\n\t// Make sure timer keeps firing etc. and does not delete until interest is gone.\n\tci, err = js.AddConsumer(\"EC\", &nats.ConsumerConfig{AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\tif o = mset.lookupConsumer(ci.Name); o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", ci.Name)\n\t}\n\terr = o.setInActiveDeleteThreshold(50 * time.Millisecond)\n\trequire_NoError(t, err)\n\n\t// Need interest otherwise the requests will be recycled based on no real interest.\n\tsub, err := nc.SubscribeSync(\"xx\")\n\trequire_NoError(t, err)\n\n\treq := &JSApiConsumerGetNextRequest{Batch: 10, Expires: 250 * time.Millisecond}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"EC\", ci.Name)\n\terr = nc.PublishRequest(rsubj, \"xx\", jreq)\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\ttime.Sleep(100 * time.Millisecond)\n\t// Should still be alive here.\n\tif o := mset.lookupConsumer(ci.Name); o == nil {\n\t\tt.Fatalf(\"Expected consumer to still be active\")\n\t}\n\t// Remove interest.\n\tsub.Unsubscribe()\n\t// Make sure this EPC goes away now.\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tif o := mset.lookupConsumer(ci.Name); o != nil {\n\t\t\treturn fmt.Errorf(\"Consumer still present\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamEphemeralPullConsumersInactiveThresholdAndNoWait(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"ECIT\", Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\tci, err := js.AddConsumer(\"ECIT\", &nats.ConsumerConfig{\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tInactiveThreshold: 100 * time.Millisecond,\n\t})\n\trequire_NoError(t, err)\n\n\t// Send 10 no_wait requests every 25ms and consumer should still be present.\n\treq := &JSApiConsumerGetNextRequest{Batch: 10, NoWait: true}\n\tjreq, err := json.Marshal(req)\n\trequire_NoError(t, err)\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"ECIT\", ci.Name)\n\tfor i := 0; i < 10; i++ {\n\t\terr = nc.PublishRequest(rsubj, \"xx\", jreq)\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\ttime.Sleep(25 * time.Millisecond)\n\t}\n\n\t_, err = js.ConsumerInfo(\"ECIT\", ci.Name)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamNakRedeliveryWithNoWait(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"NAK\"))\n\trequire_NoError(t, err)\n\n\tccReq := &CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDurable: \"dlc\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tMaxDeliver: 3,\n\t\t\tAckWait: time.Minute,\n\t\t\tBackOff: []time.Duration{5 * time.Second, 10 * time.Second},\n\t\t},\n\t}\n\t// Do by hand for now until Go client catches up.\n\treq, err := json.Marshal(ccReq)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"dlc\"), req, time.Second)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", ccResp.Error)\n\t}\n\n\trsubj := fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"dlc\")\n\tm, err := nc.Request(rsubj, nil, time.Second)\n\trequire_NoError(t, err)\n\n\t// NAK this message.\n\tdelay, err := json.Marshal(&ConsumerNakOptions{Delay: 500 * time.Millisecond})\n\trequire_NoError(t, err)\n\tdnak := []byte(fmt.Sprintf(\"%s %s\", AckNak, delay))\n\tm.Respond(dnak)\n\n\t// This message should come back to us after 500ms. If we do a one-shot request, with NoWait and Expires\n\t// this will do the right thing and we get the message.\n\t// What we want to test here is a true NoWait request with Expires==0 and eventually seeing the message be redelivered.\n\texpires := time.Now().Add(time.Second)\n\tfor time.Now().Before(expires) {\n\t\tm, err = nc.Request(rsubj, []byte(`{\"batch\":1, \"no_wait\": true}`), time.Second)\n\t\trequire_NoError(t, err)\n\t\tif len(m.Data) > 0 {\n\t\t\t// We got our message, so we are good.\n\t\t\treturn\n\t\t}\n\t\t// So we do not spin.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tt.Fatalf(\"Did not get the message in time\")\n}\n\n// Test that we properly enforce per subject msg limits when DiscardNew is set.\n// DiscardNew should only apply to stream limits, subject based limits should always be DiscardOld.\nfunc TestJetStreamMaxMsgsPerSubjectWithDiscardNew(t *testing.T) {\n\tmsc := StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\", \"x\"},\n\t\tDiscard: DiscardNew,\n\t\tStorage: MemoryStorage,\n\t\tMaxMsgsPer: 4,\n\t\tMaxMsgs: 10,\n\t\tMaxBytes: 500,\n\t}\n\tfsc := msc\n\tfsc.Storage = FileStorage\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &msc},\n\t\t{\"FileStore\", &fsc},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tmset, err := s.GlobalAccount().addStream(c.mconfig)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer mset.delete()\n\n\t\t\t// Client for API requests.\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tpubAndCheck := func(subj string, num int, expectedNumMsgs uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor i := 0; i < num; i++ {\n\t\t\t\t\t_, err = js.Publish(subj, []byte(\"TSLA\"))\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif si.State.Msgs != expectedNumMsgs {\n\t\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", expectedNumMsgs, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpubExpectErr := func(subj string, sz int) {\n\t\t\t\tt.Helper()\n\t\t\t\t_, err = js.Publish(subj, bytes.Repeat([]byte(\"X\"), sz))\n\t\t\t\trequire_Error(t, err, errors.New(\"nats: maximum bytes exceeded\"), errors.New(\"nats: maximum messages exceeded\"))\n\t\t\t}\n\n\t\t\tpubAndCheck(\"foo\", 1, 1)\n\t\t\t// We should treat this as DiscardOld and only have 4 msgs after.\n\t\t\tpubAndCheck(\"foo\", 4, 4)\n\t\t\t// Same thing here, shoud only have 4 foo and 4 bar for total of 8.\n\t\t\tpubAndCheck(\"bar\", 8, 8)\n\t\t\t// We have 8 here, so only 2 left. If we add in a new subject when we have room it will be accepted.\n\t\t\tpubAndCheck(\"baz\", 2, 10)\n\t\t\t// Now we are full, but makeup is foo-4 bar-4 baz-2.\n\t\t\t// We can add to foo and bar since they are at their max and adding new ones there stays the same in terms of total of 10.\n\t\t\tpubAndCheck(\"foo\", 1, 10)\n\t\t\tpubAndCheck(\"bar\", 1, 10)\n\t\t\t// Try to send a large message under an established subject that will exceed the 500 maximum.\n\t\t\t// Even though we have a bar subject and its at its maximum, the message to be dropped is not big enough, so this should err.\n\t\t\tpubExpectErr(\"bar\", 300)\n\t\t\t// Also even though we have room bytes wise, if we introduce a new subject this should fail too on msg limit exceeded.\n\t\t\tpubExpectErr(\"x\", 2)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStreamInfoSubjectsDetails(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tgetInfo := func(t *testing.T, filter string) *StreamInfo {\n\t\tt.Helper()\n\t\t// Need to grab StreamInfo by hand for now.\n\t\treq, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter})\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar si StreamInfo\n\t\terr = json.Unmarshal(resp.Data, &si)\n\t\trequire_NoError(t, err)\n\t\tif si.State.NumSubjects != 3 {\n\t\t\tt.Fatalf(\"Expected NumSubjects to be 3, but got %d\", si.State.NumSubjects)\n\t\t}\n\t\treturn &si\n\t}\n\n\ttestSubjects := func(t *testing.T, st nats.StorageType) {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"*\"},\n\t\t\tStorage: st,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer js.DeleteStream(\"TEST\")\n\n\t\tcounts, msg := []int{22, 33, 44}, []byte(\"ok\")\n\t\t// Now place msgs, foo-22, bar-33 and baz-44.\n\t\tfor i, subj := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\t\tfor n := 0; n < counts[i]; n++ {\n\t\t\t\t_, err = js.Publish(subj, msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\n\t\t// Test all subjects first.\n\t\texpected := map[string]uint64{\"foo\": 22, \"bar\": 33, \"baz\": 44}\n\t\tif si := getInfo(t, nats.AllKeys); !reflect.DeepEqual(si.State.Subjects, expected) {\n\t\t\tt.Fatalf(\"Expected subjects of %+v, but got %+v\", expected, si.State.Subjects)\n\t\t}\n\t\tif si := getInfo(t, \"*\"); !reflect.DeepEqual(si.State.Subjects, expected) {\n\t\t\tt.Fatalf(\"Expected subjects of %+v, but got %+v\", expected, si.State.Subjects)\n\t\t}\n\t\t// Filtered to 1.\n\t\texpected = map[string]uint64{\"foo\": 22}\n\t\tif si := getInfo(t, \"foo\"); !reflect.DeepEqual(si.State.Subjects, expected) {\n\t\t\tt.Fatalf(\"Expected subjects of %+v, but got %+v\", expected, si.State.Subjects)\n\t\t}\n\t}\n\n\tt.Run(\"MemoryStore\", func(t *testing.T) { testSubjects(t, nats.MemoryStorage) })\n\tt.Run(\"FileStore\", func(t *testing.T) { testSubjects(t, nats.FileStorage) })\n}\n\nfunc TestJetStreamStreamInfoSubjectsDetailsWithDeleteAndPurge(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tgetInfo := func(t *testing.T, filter string) *StreamInfo {\n\t\tt.Helper()\n\t\t// Need to grab StreamInfo by hand for now.\n\t\treq, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter})\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar si StreamInfo\n\t\terr = json.Unmarshal(resp.Data, &si)\n\t\trequire_NoError(t, err)\n\t\treturn &si\n\t}\n\n\tcheckResults := func(t *testing.T, expected map[string]uint64) {\n\t\tt.Helper()\n\t\tsi := getInfo(t, nats.AllKeys)\n\t\tif !reflect.DeepEqual(si.State.Subjects, expected) {\n\t\t\tt.Fatalf(\"Expected subjects of %+v, but got %+v\", expected, si.State.Subjects)\n\t\t}\n\t\tif si.State.NumSubjects != len(expected) {\n\t\t\tt.Fatalf(\"Expected NumSubjects to be %d, but got %d\", len(expected), si.State.NumSubjects)\n\t\t}\n\t}\n\n\ttestSubjects := func(t *testing.T, st nats.StorageType) {\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"*\"},\n\t\t\tStorage: st,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer js.DeleteStream(\"TEST\")\n\n\t\tmsg := []byte(\"ok\")\n\t\tjs.Publish(\"foo\", msg) // 1\n\t\tjs.Publish(\"foo\", msg) // 2\n\t\tjs.Publish(\"bar\", msg) // 3\n\t\tjs.Publish(\"baz\", msg) // 4\n\t\tjs.Publish(\"baz\", msg) // 5\n\t\tjs.Publish(\"bar\", msg) // 6\n\t\tjs.Publish(\"bar\", msg) // 7\n\n\t\tcheckResults(t, map[string]uint64{\"foo\": 2, \"bar\": 3, \"baz\": 2})\n\n\t\t// Now delete some messages.\n\t\tjs.DeleteMsg(\"TEST\", 6)\n\n\t\tcheckResults(t, map[string]uint64{\"foo\": 2, \"bar\": 2, \"baz\": 2})\n\n\t\t// Delete and add right back, so no-op\n\t\tjs.DeleteMsg(\"TEST\", 5) // baz\n\t\tjs.Publish(\"baz\", msg) // 8\n\n\t\tcheckResults(t, map[string]uint64{\"foo\": 2, \"bar\": 2, \"baz\": 2})\n\n\t\t// Now do a purge only of bar.\n\t\tjr, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: \"bar\"})\n\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"TEST\"), jr, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tcheckResults(t, map[string]uint64{\"foo\": 2, \"baz\": 2})\n\n\t\t// Now purge everything\n\t\terr = js.PurgeStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\tsi := getInfo(t, nats.AllKeys)\n\t\tif len(si.State.Subjects) != 0 {\n\t\t\tt.Fatalf(\"Expected no subjects, but got %+v\", si.State.Subjects)\n\t\t}\n\t\tif si.State.NumSubjects != 0 {\n\t\t\tt.Fatalf(\"Expected NumSubjects to be 0, but got %d\", si.State.NumSubjects)\n\t\t}\n\t}\n\n\tt.Run(\"MemoryStore\", func(t *testing.T) { testSubjects(t, nats.MemoryStorage) })\n\tt.Run(\"FileStore\", func(t *testing.T) { testSubjects(t, nats.FileStorage) })\n}\n\nfunc TestJetStreamStreamInfoSubjectsDetailsAfterRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tgetInfo := func(t *testing.T, filter string) *StreamInfo {\n\t\tt.Helper()\n\t\t// Need to grab StreamInfo by hand for now.\n\t\treq, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: filter})\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar si StreamInfo\n\t\terr = json.Unmarshal(resp.Data, &si)\n\t\trequire_NoError(t, err)\n\t\treturn &si\n\t}\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t})\n\trequire_NoError(t, err)\n\tdefer js.DeleteStream(\"TEST\")\n\n\tmsg := []byte(\"ok\")\n\tjs.Publish(\"foo\", msg) // 1\n\tjs.Publish(\"foo\", msg) // 2\n\tjs.Publish(\"bar\", msg) // 3\n\tjs.Publish(\"baz\", msg) // 4\n\tjs.Publish(\"baz\", msg) // 5\n\n\tsi := getInfo(t, nats.AllKeys)\n\tif si.State.NumSubjects != 3 {\n\t\tt.Fatalf(\"Expected 3 subjects, but got %d\", si.State.NumSubjects)\n\t}\n\n\t// Stop current\n\tnc.Close()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, _ = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi = getInfo(t, nats.AllKeys)\n\tif si.State.NumSubjects != 3 {\n\t\tt.Fatalf(\"Expected 3 subjects, but got %d\", si.State.NumSubjects)\n\t}\n}\n\n// Issue #2836\nfunc TestJetStreamInterestRetentionBug(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"c1\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\ttest := func(token string, fseq, msgs uint64) {\n\t\tt.Helper()\n\t\tsubj := fmt.Sprintf(\"foo.%s\", token)\n\t\t_, err = js.Publish(subj, nil)\n\t\trequire_NoError(t, err)\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.FirstSeq != fseq {\n\t\t\tt.Fatalf(\"Expected first to be %d, got %d\", fseq, si.State.FirstSeq)\n\t\t}\n\t\tif si.State.Msgs != msgs {\n\t\t\tt.Fatalf(\"Expected msgs to be %d, got %d\", msgs, si.State.Msgs)\n\t\t}\n\t}\n\n\ttest(\"bar\", 1, 1)\n\n\t// Create second filtered consumer.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"c2\", FilterSubject: \"foo.foo\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\ttest(\"bar\", 1, 2)\n}\n\n// Under load testing for K3S and the KINE interface we saw some stalls.\n// These were caused by a dynamic that would not send a second FC item with one\n// pending, but when we sent the next message and got blocked, if that msg would\n// exceed the outstanding FC we would become stalled.\nfunc TestJetStreamFlowControlStall(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"FC\"})\n\trequire_NoError(t, err)\n\n\tmsg := []byte(strings.Repeat(\"X\", 32768))\n\t_, err = js.Publish(\"FC\", msg)\n\trequire_NoError(t, err)\n\n\tmsg = []byte(strings.Repeat(\"X\", 8*32768))\n\t_, err = js.Publish(\"FC\", msg)\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"FC\", msg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"FC\", nats.OrderedConsumer())\n\trequire_NoError(t, err)\n\n\tcheckSubsPending(t, sub, 3)\n}\n\nfunc TestJetStreamStorageReservedBytes(t *testing.T) {\n\tconst systemLimit = 1024\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.JetStream = true\n\topts.JetStreamMaxMemory = systemLimit\n\topts.JetStreamMaxStore = systemLimit\n\topts.StoreDir = t.TempDir()\n\topts.HTTPPort = -1\n\ts := RunServer(&opts)\n\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tgetJetStreamVarz := func(hc *http.Client, addr string) (JetStreamVarz, error) {\n\t\tresp, err := hc.Get(addr)\n\t\tif err != nil {\n\t\t\treturn JetStreamVarz{}, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\n\t\tvar v Varz\n\t\tif err := json.NewDecoder(resp.Body).Decode(&v); err != nil {\n\t\t\treturn JetStreamVarz{}, err\n\t\t}\n\n\t\treturn v.JetStream, nil\n\t}\n\tgetReserved := func(hc *http.Client, addr string, st nats.StorageType) (uint64, error) {\n\t\tjsv, err := getJetStreamVarz(hc, addr)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tif st == nats.MemoryStorage {\n\t\t\treturn jsv.Stats.ReservedMemory, nil\n\t\t}\n\t\treturn jsv.Stats.ReservedStore, nil\n\t}\n\n\tvarzAddr := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\thc := &http.Client{Timeout: 5 * time.Second}\n\n\tjsv, err := getJetStreamVarz(hc, varzAddr)\n\trequire_NoError(t, err)\n\n\tif got, want := systemLimit, int(jsv.Config.MaxMemory); got != want {\n\t\tt.Fatalf(\"Unexpected max memory: got=%d, want=%d\", got, want)\n\t}\n\tif got, want := systemLimit, int(jsv.Config.MaxStore); got != want {\n\t\tt.Fatalf(\"Unexpected max store: got=%d, want=%d\", got, want)\n\t}\n\n\tcases := []struct {\n\t\tname string\n\t\taccountLimit int64\n\t\tstorage nats.StorageType\n\t\tcreateMaxBytes int64\n\t\tupdateMaxBytes int64\n\t\twantUpdateError bool\n\t}{\n\t\t{\n\t\t\tname: \"file reserve 66% of system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: int64(math.Round(float64(systemLimit) * .666)),\n\t\t\tupdateMaxBytes: int64(math.Round(float64(systemLimit)*.666)) + 1,\n\t\t},\n\t\t{\n\t\t\tname: \"memory reserve 66% of system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: int64(math.Round(float64(systemLimit) * .666)),\n\t\t\tupdateMaxBytes: int64(math.Round(float64(systemLimit)*.666)) + 1,\n\t\t},\n\t\t{\n\t\t\tname: \"file update past system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: systemLimit,\n\t\t\tupdateMaxBytes: systemLimit + 1,\n\t\t\twantUpdateError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory update past system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: systemLimit,\n\t\t\tupdateMaxBytes: systemLimit + 1,\n\t\t\twantUpdateError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file update to system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: systemLimit - 1,\n\t\t\tupdateMaxBytes: systemLimit,\n\t\t},\n\t\t{\n\t\t\tname: \"memory update to system limit\",\n\t\t\taccountLimit: -1,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: systemLimit - 1,\n\t\t\tupdateMaxBytes: systemLimit,\n\t\t},\n\t\t{\n\t\t\tname: \"file reserve 66% of account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: int64(math.Round(float64(systemLimit/2) * .666)),\n\t\t\tupdateMaxBytes: int64(math.Round(float64(systemLimit/2)*.666)) + 1,\n\t\t},\n\t\t{\n\t\t\tname: \"memory reserve 66% of account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: int64(math.Round(float64(systemLimit/2) * .666)),\n\t\t\tupdateMaxBytes: int64(math.Round(float64(systemLimit/2)*.666)) + 1,\n\t\t},\n\t\t{\n\t\t\tname: \"file update past account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: (systemLimit / 2),\n\t\t\tupdateMaxBytes: (systemLimit / 2) + 1,\n\t\t\twantUpdateError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"memory update past account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: (systemLimit / 2),\n\t\t\tupdateMaxBytes: (systemLimit / 2) + 1,\n\t\t\twantUpdateError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"file update to account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.FileStorage,\n\t\t\tcreateMaxBytes: (systemLimit / 2) - 1,\n\t\t\tupdateMaxBytes: (systemLimit / 2),\n\t\t},\n\t\t{\n\t\t\tname: \"memory update to account limit\",\n\t\t\taccountLimit: systemLimit / 2,\n\t\t\tstorage: nats.MemoryStorage,\n\t\t\tcreateMaxBytes: (systemLimit / 2) - 1,\n\t\t\tupdateMaxBytes: (systemLimit / 2),\n\t\t},\n\t}\n\tfor i := 0; i < len(cases) && !t.Failed(); i++ {\n\t\tc := cases[i]\n\t\tt.Run(c.name, func(st *testing.T) {\n\t\t\t// Setup limits\n\t\t\terr = s.GlobalAccount().UpdateJetStreamLimits(map[string]JetStreamAccountLimits{\n\t\t\t\t_EMPTY_: {\n\t\t\t\t\tMaxMemory: c.accountLimit,\n\t\t\t\t\tMaxStore: c.accountLimit,\n\t\t\t\t},\n\t\t\t})\n\t\t\trequire_NoError(st, err)\n\n\t\t\t// Create initial stream\n\t\t\tcfg := &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: c.storage,\n\t\t\t\tMaxBytes: c.createMaxBytes,\n\t\t\t}\n\t\t\t_, err = js.AddStream(cfg)\n\t\t\trequire_NoError(st, err)\n\n\t\t\t// Update stream MaxBytes\n\t\t\tcfg.MaxBytes = c.updateMaxBytes\n\t\t\tinfo, err := js.UpdateStream(cfg)\n\t\t\tif c.wantUpdateError && err == nil {\n\t\t\t\tgot := info.Config.MaxBytes\n\t\t\t\tst.Fatalf(\"Unexpected update success, newMaxBytes=%d; systemLimit=%d; accountLimit=%d\",\n\t\t\t\t\tgot, systemLimit, c.accountLimit)\n\t\t\t} else if !c.wantUpdateError && err != nil {\n\t\t\t\tst.Fatalf(\"Unexpected update error: %s\", err)\n\t\t\t}\n\n\t\t\tif !c.wantUpdateError && err == nil {\n\t\t\t\t// If update was successful, then ensure reserved shows new\n\t\t\t\t// amount\n\t\t\t\treserved, err := getReserved(hc, varzAddr, c.storage)\n\t\t\t\trequire_NoError(st, err)\n\t\t\t\tif got, want := reserved, uint64(c.updateMaxBytes); got != want {\n\t\t\t\t\tst.Fatalf(\"Unexpected reserved: %d, want %d\", got, want)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Delete stream\n\t\t\terr = js.DeleteStream(\"TEST\")\n\t\t\trequire_NoError(st, err)\n\n\t\t\t// Ensure reserved shows 0 because we've deleted the stream\n\t\t\treserved, err := getReserved(hc, varzAddr, c.storage)\n\t\t\trequire_NoError(st, err)\n\t\t\tif reserved != 0 {\n\t\t\t\tst.Fatalf(\"Unexpected reserved: %d, want 0\", reserved)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamRestoreBadStream(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tvar rreq JSApiStreamRestoreRequest\n\tbuf, err := os.ReadFile(\"../test/configs/jetstream/restore_bad_stream/backup.json\")\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(buf, &rreq)\n\trequire_NoError(t, err)\n\n\tdata, err := os.Open(\"../test/configs/jetstream/restore_bad_stream/stream.tar.s2\")\n\trequire_NoError(t, err)\n\tdefer data.Close()\n\n\tvar rresp JSApiStreamRestoreResponse\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, rreq.Config.Name), buf, 5*time.Second)\n\trequire_NoError(t, err)\n\tjson.Unmarshal(msg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Error on restore: %+v\", rresp.Error)\n\t}\n\n\tvar chunk [1024]byte\n\tfor {\n\t\tn, err := data.Read(chunk[:])\n\t\tif err == io.EOF {\n\t\t\tbreak\n\t\t}\n\t\trequire_NoError(t, err)\n\n\t\tmsg, err = nc.Request(rresp.DeliverSubject, chunk[:n], 5*time.Second)\n\t\trequire_NoError(t, err)\n\t\tjson.Unmarshal(msg.Data, &rresp)\n\t\tif rresp.Error != nil {\n\t\t\tt.Fatalf(\"Error on restore: %+v\", rresp.Error)\n\t\t}\n\t}\n\tmsg, err = nc.Request(rresp.DeliverSubject, nil, 5*time.Second)\n\trequire_NoError(t, err)\n\tjson.Unmarshal(msg.Data, &rresp)\n\tif rresp.Error == nil || !strings.Contains(rresp.Error.Description, \"unexpected\") {\n\t\tt.Fatalf(\"Expected error about unexpected content, got: %+v\", rresp.Error)\n\t}\n\n\tdir := filepath.Join(s.JetStreamConfig().StoreDir, globalAccountName)\n\tf1 := filepath.Join(dir, \"fail1.txt\")\n\tf2 := filepath.Join(dir, \"fail2.txt\")\n\tfor _, f := range []string{f1, f2} {\n\t\tif _, err := os.Stat(f); err == nil {\n\t\t\tt.Fatalf(\"Found file %s\", f)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamRemoveExternalSource(t *testing.T) {\n\tho := DefaultTestOptions\n\tho.Port = 4000 //-1\n\tho.LeafNode.Host = \"127.0.0.1\"\n\tho.LeafNode.Port = -1\n\ths := RunServer(&ho)\n\tdefer hs.Shutdown()\n\n\tlu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ho.LeafNode.Port))\n\trequire_NoError(t, err)\n\n\tlo1 := DefaultTestOptions\n\tlo1.Port = 4111 //-1\n\tlo1.ServerName = \"a-leaf\"\n\tlo1.JetStream = true\n\tlo1.StoreDir = t.TempDir()\n\tlo1.JetStreamDomain = \"a-leaf\"\n\tlo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}}\n\tl1 := RunServer(&lo1)\n\tdefer l1.Shutdown()\n\n\tlo2 := DefaultTestOptions\n\tlo2.Port = 2111 //-1\n\tlo2.ServerName = \"b-leaf\"\n\tlo2.JetStream = true\n\tlo2.StoreDir = t.TempDir()\n\tlo2.JetStreamDomain = \"b-leaf\"\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}}\n\tl2 := RunServer(&lo2)\n\tdefer l2.Shutdown()\n\n\tcheckLeafNodeConnected(t, l1)\n\tcheckLeafNodeConnected(t, l2)\n\n\tcheckStreamMsgs := func(js nats.JetStreamContext, stream string, expected uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %v messages, got %v\", expected, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsendToStreamTest := func(js nats.JetStreamContext) {\n\t\tt.Helper()\n\t\tfor i := 0; i < 10; i++ {\n\t\t\t_, err = js.Publish(\"test\", []byte(\"hello\"))\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tnca, jsa := jsClientConnect(t, l1)\n\tdefer nca.Close()\n\t_, err = jsa.AddStream(&nats.StreamConfig{Name: \"queue\", Subjects: []string{\"queue\"}})\n\trequire_NoError(t, err)\n\n\t_, err = jsa.AddStream(&nats.StreamConfig{Name: \"testdel\", Subjects: []string{\"testdel\"}})\n\trequire_NoError(t, err)\n\n\tncb, jsb := jsClientConnect(t, l2)\n\tdefer ncb.Close()\n\t_, err = jsb.AddStream(&nats.StreamConfig{Name: \"test\", Subjects: []string{\"test\"}})\n\trequire_NoError(t, err)\n\tsendToStreamTest(jsb)\n\tcheckStreamMsgs(jsb, \"test\", 10)\n\n\t_, err = jsb.AddStream(&nats.StreamConfig{Name: \"testdelsrc1\", Subjects: []string{\"testdelsrc1\"}})\n\trequire_NoError(t, err)\n\t_, err = jsb.AddStream(&nats.StreamConfig{Name: \"testdelsrc2\", Subjects: []string{\"testdelsrc2\"}})\n\trequire_NoError(t, err)\n\n\t// Add test as source to queue\n\tsi, err := jsa.UpdateStream(&nats.StreamConfig{\n\t\tName: \"queue\",\n\t\tSubjects: []string{\"queue\"},\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"test\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.b-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, len(si.Config.Sources) == 1)\n\tcheckStreamMsgs(jsa, \"queue\", 10)\n\n\t// add more entries to \"test\"\n\tsendToStreamTest(jsb)\n\n\t// verify entries are both in \"test\" and \"queue\"\n\tcheckStreamMsgs(jsb, \"test\", 20)\n\tcheckStreamMsgs(jsa, \"queue\", 20)\n\n\t// Remove source\n\tsi, err = jsa.UpdateStream(&nats.StreamConfig{\n\t\tName: \"queue\",\n\t\tSubjects: []string{\"queue\"},\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, len(si.Config.Sources) == 0)\n\n\t// add more entries to \"test\"\n\tsendToStreamTest(jsb)\n\t// verify entries are in \"test\"\n\tcheckStreamMsgs(jsb, \"test\", 30)\n\n\t// But they should not be in \"queue\". We will wait a bit before checking\n\t// to make sure that we are letting enough time for the sourcing to\n\t// incorrectly happen if there is a bug.\n\ttime.Sleep(250 * time.Millisecond)\n\tcheckStreamMsgs(jsa, \"queue\", 20)\n\n\t// Test that we delete correctly. First add source to a \"testdel\"\n\tsi, err = jsa.UpdateStream(&nats.StreamConfig{\n\t\tName: \"testdel\",\n\t\tSubjects: []string{\"testdel\"},\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"testdelsrc1\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.b-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, len(si.Config.Sources) == 1)\n\t// Now add the second one...\n\tsi, err = jsa.UpdateStream(&nats.StreamConfig{\n\t\tName: \"testdel\",\n\t\tSubjects: []string{\"testdel\"},\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"testdelsrc1\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.b-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"testdelsrc2\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.b-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, len(si.Config.Sources) == 2)\n\t// Now check that the stream testdel has still 2 source consumers...\n\tacc, err := l1.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"testdel\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tn := len(mset.sources)\n\tmset.mu.RUnlock()\n\tif n != 2 {\n\t\tt.Fatalf(\"Expected 2 sources, got %v\", n)\n\t}\n\n\t// Restart leaf \"a\"\n\tnca.Close()\n\tl1.Shutdown()\n\tl1 = RunServer(&lo1)\n\tdefer l1.Shutdown()\n\n\t// add more entries to \"test\"\n\tsendToStreamTest(jsb)\n\tcheckStreamMsgs(jsb, \"test\", 40)\n\n\tnca, jsa = jsClientConnect(t, l1)\n\tdefer nca.Close()\n\ttime.Sleep(250 * time.Millisecond)\n\tcheckStreamMsgs(jsa, \"queue\", 20)\n}\n\nfunc TestJetStreamAddStreamWithFilestoreFailure(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Cause failure to create stream with filestore.\n\t// In one of ipQueue changes, this could cause a panic, so verify that we get\n\t// a failure to create, but no panic.\n\tif _, err := s.globalAccount().addStreamWithStore(\n\t\t&StreamConfig{Name: \"TEST\"},\n\t\t&FileStoreConfig{BlockSize: 2 * maxBlockSize}); err == nil {\n\t\tt.Fatal(\"Expected failure, did not get one\")\n\t}\n}\n\ntype checkFastState struct {\n\tcount int64\n\tStreamStore\n}\n\nfunc (s *checkFastState) FastState(state *StreamState) {\n\t// Keep track only when called from checkPending()\n\tif bytes.Contains(debug.Stack(), []byte(\"checkPending(\")) {\n\t\tatomic.AddInt64(&s.count, 1)\n\t}\n\ts.StreamStore.FastState(state)\n}\n\nfunc TestJetStreamBackOffCheckPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// Plug or store to see how many times we invoke FastState, which is done in checkPending\n\tmset.mu.Lock()\n\tst := &checkFastState{StreamStore: mset.store}\n\tmset.store = st\n\tmset.mu.Unlock()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tsendStreamMsg(t, nc, \"foo\", \"Hello World!\")\n\n\tsub, _ := nc.SubscribeSync(nats.NewInbox())\n\tdefer sub.Unsubscribe()\n\tnc.Flush()\n\n\to, err := mset.addConsumer(&ConsumerConfig{\n\t\tDeliverSubject: sub.Subject,\n\t\tAckPolicy: AckExplicit,\n\t\tMaxDeliver: 1000,\n\t\tBackOff: []time.Duration{50 * time.Millisecond, 250 * time.Millisecond, time.Second},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\t// Check the first delivery and the following 2 redeliveries\n\tstart := time.Now()\n\tnatsNexMsg(t, sub, time.Second)\n\tif dur := time.Since(start); dur >= 50*time.Millisecond {\n\t\tt.Fatalf(\"Expected first delivery to be fast, took: %v\", dur)\n\t}\n\tstart = time.Now()\n\tnatsNexMsg(t, sub, time.Second)\n\tif dur := time.Since(start); dur < 25*time.Millisecond || dur > 75*time.Millisecond {\n\t\tt.Fatalf(\"Expected first redelivery to be ~50ms, took: %v\", dur)\n\t}\n\tstart = time.Now()\n\tnatsNexMsg(t, sub, time.Second)\n\tif dur := time.Since(start); dur < 200*time.Millisecond || dur > 300*time.Millisecond {\n\t\tt.Fatalf(\"Expected first redelivery to be ~250ms, took: %v\", dur)\n\t}\n\t// There was a bug that would cause checkPending to be invoked based on the\n\t// ackWait (which in this case would be the first value of BackOff, which\n\t// is 50ms). So we would call checkPending() too many times.\n\ttime.Sleep(500 * time.Millisecond)\n\t// Check now, it should have been invoked 2/3 times.\n\tif n := atomic.LoadInt64(&st.count); n < 2 || n > 3 {\n\t\tt.Fatalf(\"Expected checkPending to be invoked 2/3 times, was %v\", n)\n\t}\n}\n\nfunc TestJetStreamCrossAccounts(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n jetstream {\n\t\t\tstore_dir = %q\n\t\t}\n accounts: {\n A: {\n users: [ {user: a, password: a} ]\n jetstream: enabled\n exports: [\n {service: '$JS.API.>' }\n {service: '$KV.>'}\n {stream: 'accI.>'}\n ]\n },\n I: {\n users: [ {user: i, password: i} ]\n imports: [\n {service: {account: A, subject: '$JS.API.>'}, to: 'fromA.>' }\n {service: {account: A, subject: '$KV.>'}, to: 'fromA.$KV.>' }\n {stream: {subject: 'accI.>', account: A}}\n ]\n }\n\t\t}`, t.TempDir())))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\twatchNext := func(w nats.KeyWatcher) nats.KeyValueEntry {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase e := <-w.Updates():\n\t\t\treturn e\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Fail to get the next update\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tnc1, js1 := jsClientConnect(t, s, nats.UserInfo(\"a\", \"a\"))\n\tdefer nc1.Close()\n\n\tkv1, err := js1.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"Map\", History: 10})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating kv store: %v\", err)\n\t}\n\n\tw1, err := kv1.Watch(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating watcher: %v\", err)\n\t}\n\tif e := watchNext(w1); e != nil {\n\t\tt.Fatalf(\"Expected nil entry, got %+v\", e)\n\t}\n\n\tnc2, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"i\", \"i\"), nats.CustomInboxPrefix(\"accI\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\tjs2, err := nc2.JetStream(nats.APIPrefix(\"fromA\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting jetstream context: %v\", err)\n\t}\n\n\tkv2, err := js2.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"Map\", History: 10})\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating kv store: %v\", err)\n\t}\n\n\tw2, err := kv2.Watch(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating watcher: %v\", err)\n\t}\n\tif e := watchNext(w2); e != nil {\n\t\tt.Fatalf(\"Expected nil entry, got %+v\", e)\n\t}\n\n\t// Do a Put from kv2\n\trev, err := kv2.Put(\"map\", []byte(\"value\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on put: %v\", err)\n\t}\n\n\t// Get from kv1\n\te, err := kv1.Get(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on get: %v\", err)\n\t}\n\tif e.Key() != \"map\" || string(e.Value()) != \"value\" {\n\t\tt.Fatalf(\"Unexpected entry: +%v\", e)\n\t}\n\n\t// Get from kv2\n\te, err = kv2.Get(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on get: %v\", err)\n\t}\n\tif e.Key() != \"map\" || string(e.Value()) != \"value\" {\n\t\tt.Fatalf(\"Unexpected entry: +%v\", e)\n\t}\n\n\t// Watcher 1\n\tif e := watchNext(w1); e == nil || e.Key() != \"map\" || string(e.Value()) != \"value\" {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Watcher 2\n\tif e := watchNext(w2); e == nil || e.Key() != \"map\" || string(e.Value()) != \"value\" {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Try an update form kv2\n\tif _, err := kv2.Update(\"map\", []byte(\"updated\"), rev); err != nil {\n\t\tt.Fatalf(\"Failed to update: %v\", err)\n\t}\n\n\t// Get from kv1\n\te, err = kv1.Get(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on get: %v\", err)\n\t}\n\tif e.Key() != \"map\" || string(e.Value()) != \"updated\" {\n\t\tt.Fatalf(\"Unexpected entry: +%v\", e)\n\t}\n\n\t// Get from kv2\n\te, err = kv2.Get(\"map\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on get: %v\", err)\n\t}\n\tif e.Key() != \"map\" || string(e.Value()) != \"updated\" {\n\t\tt.Fatalf(\"Unexpected entry: +%v\", e)\n\t}\n\n\t// Watcher 1\n\tif e := watchNext(w1); e == nil || e.Key() != \"map\" || string(e.Value()) != \"updated\" {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Watcher 2\n\tif e := watchNext(w2); e == nil || e.Key() != \"map\" || string(e.Value()) != \"updated\" {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Purge from kv2\n\tif err := kv2.Purge(\"map\"); err != nil {\n\t\tt.Fatalf(\"Error on purge: %v\", err)\n\t}\n\n\t// Check purge ok from w1\n\tif e := watchNext(w1); e == nil || e.Operation() != nats.KeyValuePurge {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Check purge ok from w2\n\tif e := watchNext(w2); e == nil || e.Operation() != nats.KeyValuePurge {\n\t\tt.Fatalf(\"Unexpected entry: %+v\", e)\n\t}\n\n\t// Delete purge records from kv2\n\tif err := kv2.PurgeDeletes(nats.DeleteMarkersOlderThan(-1)); err != nil {\n\t\tt.Fatalf(\"Error on purge deletes: %v\", err)\n\t}\n\n\t// Check all gone from js1\n\tif si, err := js1.StreamInfo(\"KV_Map\"); err != nil || si == nil || si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Error getting stream info: err=%v si=%+v\", err, si)\n\t}\n\n\t// Delete key from kv2\n\tif err := kv2.Delete(\"map\"); err != nil {\n\t\tt.Fatalf(\"Error on delete: %v\", err)\n\t}\n\n\t// Check key gone from kv1\n\tif e, err := kv1.Get(\"map\"); err != nats.ErrKeyNotFound || e != nil {\n\t\tt.Fatalf(\"Expected key not found, got err=%v e=%+v\", err, e)\n\t}\n}\n\nfunc TestJetStreamInvalidRestoreRequests(t *testing.T) {\n\ttest := func(t *testing.T, s *Server, replica int) {\n\t\tnc := natsConnect(t, s.ClientURL())\n\t\tdefer nc.Close()\n\t\t// test invalid stream config in restore request\n\t\trequire_fail := func(cfg StreamConfig, errDesc string) {\n\t\t\tt.Helper()\n\t\t\trreq := &JSApiStreamRestoreRequest{\n\t\t\t\tConfig: cfg,\n\t\t\t}\n\t\t\treq, err := json.Marshal(rreq)\n\t\t\trequire_NoError(t, err)\n\t\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, \"fail\"), req, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar rresp JSApiStreamRestoreResponse\n\t\t\tjson.Unmarshal(rmsg.Data, &rresp)\n\t\t\trequire_True(t, rresp.Error != nil)\n\t\t\trequire_Equal(t, rresp.Error.Description, errDesc)\n\t\t}\n\t\trequire_fail(StreamConfig{Name: \"fail\", MaxBytes: 1024, Storage: FileStorage, Replicas: 6},\n\t\t\t\"maximum replicas is 5\")\n\t\trequire_fail(StreamConfig{Name: \"fail\", MaxBytes: 2 * 1012 * 1024, Storage: FileStorage, Replicas: replica},\n\t\t\t\"insufficient storage resources available\")\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"stream\", MaxBytes: 1024, Storage: nats.FileStorage, Replicas: 1})\n\t\trequire_NoError(t, err)\n\t\trequire_fail(StreamConfig{Name: \"fail\", MaxBytes: 1024, Storage: FileStorage},\n\t\t\t\"maximum number of streams reached\")\n\t}\n\n\tcommonAccSection := `\n\t\tno_auth_user: u\n\t\taccounts {\n\t\t\tONE {\n\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\tjetstream: {\n\t\t\t\t\tmax_store: 1Mb\n\t\t\t\t\tmax_streams: 1\n\t\t\t\t}\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}`\n\n\tt.Run(\"clustered\", func(t *testing.T) {\n\t\tc := createJetStreamClusterWithTemplate(t, `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}`+commonAccSection, \"clust\", 3)\n\t\tdefer c.shutdown()\n\t\ts := c.randomServer()\n\t\ttest(t, s, 3)\n\t})\n\tt.Run(\"single\", func(t *testing.T) {\n\t\tstoreDir := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}\n\t\t\t%s`, storeDir, commonAccSection)))\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\t\ttest(t, s, 1)\n\t})\n}\n\nfunc TestJetStreamLimits(t *testing.T) {\n\ttest := func(t *testing.T, s *Server) {\n\t\tnc := natsConnect(t, s.ClientURL())\n\t\tdefer nc.Close()\n\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\n\t\tsi, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, si.Config.Duplicates == time.Minute)\n\n\t\tsi, err = js.AddStream(&nats.StreamConfig{Name: \"bar\", Duplicates: 1500 * time.Millisecond})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, si.Config.Duplicates == 1500*time.Millisecond)\n\n\t\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"bar\", Duplicates: 2 * time.Minute})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: duplicates window can not be larger then server limit of 1m0s\")\n\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: \"baz\", Duplicates: 2 * time.Minute})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: duplicates window can not be larger then server limit of 1m0s\")\n\n\t\tci, err := js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ci.Config.MaxAckPending == 1000)\n\t\trequire_True(t, ci.Config.MaxRequestBatch == 250)\n\n\t\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur2\", AckPolicy: nats.AckExplicitPolicy, MaxRequestBatch: 500})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: consumer max request batch exceeds server limit of 250\")\n\n\t\tci, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur2\", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 500})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, ci.Config.MaxAckPending == 500)\n\t\trequire_True(t, ci.Config.MaxRequestBatch == 250)\n\n\t\t_, err = js.UpdateConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur2\", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 2000})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: consumer max ack pending exceeds system limit of 1000\")\n\n\t\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{Durable: \"dur3\", AckPolicy: nats.AckExplicitPolicy, MaxAckPending: 2000})\n\t\trequire_Error(t, err)\n\t\trequire_Equal(t, err.Error(), \"nats: consumer max ack pending exceeds system limit of 1000\")\n\t}\n\n\tt.Run(\"clustered\", func(t *testing.T) {\n\t\ttmpl := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t\tlimits: {duplicate_window: \"1m\", max_request_batch: 250}\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t\t\tno_auth_user: u\n\t\t\taccounts {\n\t\t\t\tONE {\n\t\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\t\tjetstream: enabled\n\t\t\t\t}\n\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t}`\n\t\tlimitsTest := func(t *testing.T, tmpl string) {\n\t\t\tc := createJetStreamClusterWithTemplate(t, tmpl, \"clust\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts := c.randomServer()\n\t\t\ttest(t, s)\n\t\t}\n\t\t// test with max_ack_pending being defined in operator or account\n\t\tt.Run(\"operator\", func(t *testing.T) {\n\t\t\tlimitsTest(t, strings.Replace(tmpl, \"duplicate_window\", \"max_ack_pending: 1000, duplicate_window\", 1))\n\t\t})\n\t\tt.Run(\"account\", func(t *testing.T) {\n\t\t\tlimitsTest(t, strings.Replace(tmpl, \"jetstream: enabled\", \"jetstream: {max_ack_pending: 1000}\", 1))\n\t\t})\n\t})\n\n\tt.Run(\"single\", func(t *testing.T) {\n\t\ttmpl := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 2MB,\n\t\t\t\tmax_file_store: 8MB,\n\t\t\t\tstore_dir: '%s',\n\t\t\t\tlimits: {duplicate_window: \"1m\", max_request_batch: 250}\n\t\t\t}\n\t\t\tno_auth_user: u\n\t\t\taccounts {\n\t\t\t\tONE {\n\t\t\t\t\tusers = [ { user: \"u\", pass: \"s3cr3t!\" } ]\n\t\t\t\t\tjetstream: enabled\n\t\t\t\t}\n\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t}`\n\t\tlimitsTest := func(t *testing.T, tmpl string) {\n\t\t\tstoreDir := t.TempDir()\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir)))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\trequire_True(t, opts.JetStreamLimits.Duplicates == time.Minute)\n\t\t\ttest(t, s)\n\t\t}\n\t\t// test with max_ack_pending being defined in operator or account\n\t\tt.Run(\"operator\", func(t *testing.T) {\n\t\t\tlimitsTest(t, strings.Replace(tmpl, \"duplicate_window\", \"max_ack_pending: 1000, duplicate_window\", 1))\n\t\t})\n\t\tt.Run(\"account\", func(t *testing.T) {\n\t\t\tlimitsTest(t, strings.Replace(tmpl, \"jetstream: enabled\", \"jetstream: {max_ack_pending: 1000}\", 1))\n\t\t})\n\t})\n}\n\nfunc TestJetStreamImportReload(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}\n\t\taccounts: {\n\t\t\taccount_a: {\n\t\t\t\tusers: [{user: user_a, password: pwd}]\n\t\t\t\texports: [{stream: news.>}]\n\t\t\t}\n\t\t\taccount_b: {\n\t\t\t\tusers: [{user: user_b, password: pwd}]\n\t\t\t\tjetstream: enabled\n\t\t\t\timports: [{stream: {subject: news.>, account: account_a}}]\n\t\t\t}\n\t\t}`, storeDir)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tncA := natsConnect(t, s.ClientURL(), nats.UserInfo(\"user_a\", \"pwd\"))\n\tdefer ncA.Close()\n\n\tncB := natsConnect(t, s.ClientURL(), nats.UserInfo(\"user_b\", \"pwd\"))\n\tdefer ncB.Close()\n\n\tjsB, err := ncB.JetStream()\n\trequire_NoError(t, err)\n\n\t_, err = jsB.AddStream(&nats.StreamConfig{Name: \"news\", Subjects: []string{\"news.>\"}})\n\trequire_NoError(t, err)\n\n\trequire_NoError(t, ncA.Publish(\"news.article\", nil))\n\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := jsB.StreamInfo(\"news\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != 1\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Remove exports/imports\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'}\n\t\taccounts: {\n\t\t\taccount_a: {\n\t\t\t\tusers: [{user: user_a, password: pwd}]\n\t\t\t}\n\t\t\taccount_b: {\n\t\t\t\tusers: [{user: user_b, password: pwd}]\n\t\t\t\tjetstream: enabled\n\t\t\t}\n\t\t}`, storeDir))\n\n\trequire_NoError(t, ncA.Publish(\"news.article\", nil))\n\trequire_NoError(t, ncA.Flush())\n\n\tsi, err := jsB.StreamInfo(\"news\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n}\n\nfunc TestJetStreamRecoverSealedAfterServerRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"foo\", Sealed: true})\n\trequire_NoError(t, err)\n\n\tnc.Close()\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"foo\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 100)\n}\n\nfunc TestJetStreamImportConsumerStreamSubjectRemapSingle(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 4GB, max_file_store: 1TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tJS: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {user: js, password: pwd} ]\n\t\t\t\texports [\n\t\t\t\t\t# This is streaming to a delivery subject for a push based consumer.\n\t\t\t\t\t{ stream: \"deliver.*\" }\n\t\t\t\t\t{ stream: \"foo.*\" }\n\t\t\t\t\t# This is to ack received messages. This is a service to support sync ack.\n\t\t\t\t\t{ service: \"$JS.ACK.ORDERS.*.>\" }\n\t\t\t\t\t# To support ordered consumers, flow control.\n\t\t\t\t\t{ service: \"$JS.FC.>\" }\n\t\t\t\t]\n\t\t\t},\n\t\t\tIM: {\n\t\t\t\tusers: [ {user: im, password: pwd} ]\n\t\t\t\timports [\n\t\t\t\t\t{ stream: { account: JS, subject: \"deliver.ORDERS\" }, to: \"d.*\" }\n\t\t\t\t\t{ stream: { account: JS, subject: \"foo.*\" }, to: \"bar.*\" }\n\t\t\t\t\t{ service: { account: JS, subject: \"$JS.FC.>\" }}\n\t\t\t\t]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ttest := func(t *testing.T, queue bool) {\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"js\", \"pwd\"))\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"ORDERS\",\n\t\t\tSubjects: []string{\"foo\"}, // The JS subject.\n\t\t\tStorage: nats.MemoryStorage},\n\t\t)\n\t\trequire_NoError(t, err)\n\n\t\t_, err = js.Publish(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\n\t\tqueueName := \"\"\n\t\tif queue {\n\t\t\tqueueName = \"queue\"\n\t\t}\n\n\t\t_, err = js.AddConsumer(\"ORDERS\", &nats.ConsumerConfig{\n\t\t\tDeliverSubject: \"deliver.ORDERS\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tDeliverGroup: queueName,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tnc2, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"im\", \"pwd\"))\n\t\trequire_NoError(t, err)\n\t\tdefer nc2.Close()\n\n\t\tvar sub *nats.Subscription\n\t\tif queue {\n\t\t\tsub, err = nc2.QueueSubscribeSync(\"d.ORDERS\", queueName)\n\t\t\trequire_NoError(t, err)\n\t\t} else {\n\t\t\tsub, err = nc2.SubscribeSync(\"d.ORDERS\")\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tif m.Subject != \"foo\" {\n\t\t\tt.Fatalf(\"Subject not mapped correctly across account boundary, expected %q got %q\", \"foo\", m.Subject)\n\t\t}\n\n\t\t// Now do one that would kick in a transform.\n\t\t_, err = js.AddConsumer(\"ORDERS\", &nats.ConsumerConfig{\n\t\t\tDeliverSubject: \"foo.ORDERS\",\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\tDeliverGroup: queueName,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tif queue {\n\t\t\tsub, err = nc2.QueueSubscribeSync(\"bar.ORDERS\", queueName)\n\t\t\trequire_NoError(t, err)\n\t\t} else {\n\t\t\tsub, err = nc2.SubscribeSync(\"bar.ORDERS\")\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tm, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tif m.Subject != \"foo\" {\n\t\t\tt.Fatalf(\"Subject not mapped correctly across account boundary, expected %q got %q\", \"foo\", m.Subject)\n\t\t}\n\t}\n\n\tt.Run(\"noqueue\", func(t *testing.T) {\n\t\ttest(t, false)\n\t})\n\tt.Run(\"queue\", func(t *testing.T) {\n\t\ttest(t, true)\n\t})\n}\n\nfunc TestJetStreamWorkQueueSourceRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsent := 10\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"FOO\",\n\t\tReplicas: 1,\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < sent; i++ {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 1,\n\t\t// TODO test will pass when retention commented out\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tSources: []*nats.StreamSource{{Name: \"FOO\"}}})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dur\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dur\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\n\ttime.Sleep(1 * time.Second)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\trequire_NoError(t, err)\n\trequire_True(t, ci.NumPending == uint64(sent))\n\n\tmsgs, err := sub.Fetch(sent)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == sent)\n\n\tfor i := 0; i < sent; i++ {\n\t\terr = msgs[i].AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dur\")\n\trequire_NoError(t, err)\n\trequire_True(t, ci.NumPending == 0)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 0)\n\n\t// Restart server\n\tnc.Close()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ttime.Sleep(200 * time.Millisecond)\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(nil)\n\t\tif hs.Status == \"ok\" && hs.Error == _EMPTY_ {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"healthz %s %s\", hs.Error, hs.Status)\n\t})\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Expected 0 messages on restart, got %d\", si.State.Msgs)\n\t}\n\n\tctest, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\trequire_NoError(t, err)\n\n\t//TODO (mh) I have experienced in other tests that NumPending has a value of 1 post restart.\n\t// seems to go awary in single server setup. It's also unrelated to work queue\n\t// but that error seems benign.\n\tif ctest.NumPending != 0 {\n\t\tt.Fatalf(\"Expected pending of 0 but got %d\", ctest.NumPending)\n\t}\n\n\tsub, err = js.PullSubscribe(\"foo\", \"dur\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\t_, err = sub.Fetch(1, nats.MaxWait(time.Second))\n\tif err != nats.ErrTimeout {\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestJetStreamWorkQueueSourceNamingRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"C1\", Subjects: []string{\"foo.*\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"C2\", Subjects: []string{\"bar.*\"}})\n\trequire_NoError(t, err)\n\n\tsendCount := 10\n\tfor i := 0; i < sendCount; i++ {\n\t\t_, err = js.Publish(fmt.Sprintf(\"foo.%d\", i), nil)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.Publish(fmt.Sprintf(\"bar.%d\", i), nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// TODO Test will always pass if pending is 0\n\tpending := 1\n\t// For some yet unknown reason this failure seems to require 2 streams to source from.\n\t// This might possibly be timing, as the test sometimes passes\n\tstreams := 2\n\ttotalPending := uint64(streams * pending)\n\ttotalMsgs := streams * sendCount\n\ttotalNonPending := streams * (sendCount - pending)\n\n\t// TODO Test will always pass if this is named A (go returns directory names sorted)\n\t// A: this stream is recovered BEFORE C1/C2, tbh, I'd expect this to be the case to fail, but it isn't\n\t// D: this stream is recovered AFTER C1/C2, which is the case that fails (perhaps it is timing)\n\tsrcName := \"D\"\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: srcName,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tSources: []*nats.StreamSource{{Name: \"C1\"}, {Name: \"C2\"}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Add a consumer and consume all but totalPending messages\n\t_, err = js.AddConsumer(srcName, &nats.ConsumerConfig{Durable: \"dur\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"\", \"dur\", nats.BindStream(srcName))\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 5*time.Second, time.Millisecond*200, func() error {\n\t\tif ci, err := js.ConsumerInfo(srcName, \"dur\"); err != nil {\n\t\t\treturn err\n\t\t} else if ci.NumPending != uint64(totalMsgs) {\n\t\t\treturn fmt.Errorf(\"not enough messages: %d\", ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// consume all but messages we want pending\n\tmsgs, err := sub.Fetch(totalNonPending)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == totalNonPending)\n\n\tfor _, m := range msgs {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\n\tci, err := js.ConsumerInfo(srcName, \"dur\")\n\trequire_NoError(t, err)\n\trequire_True(t, ci.NumPending == totalPending)\n\n\tsi, err := js.StreamInfo(srcName)\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == totalPending)\n\n\t// Restart server\n\tnc.Close()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\ttime.Sleep(200 * time.Millisecond)\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\ths := s.healthz(nil)\n\t\tif hs.Status == \"ok\" && hs.Error == _EMPTY_ {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"healthz %s %s\", hs.Error, hs.Status)\n\t})\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err = js.StreamInfo(srcName)\n\trequire_NoError(t, err)\n\n\tif si.State.Msgs != totalPending {\n\t\tt.Fatalf(\"Expected 0 messages on restart, got %d\", si.State.Msgs)\n\t}\n}\n\nfunc TestJetStreamDisabledHealthz(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"Expected JetStream to be enabled\")\n\t}\n\n\ts.DisableJetStream()\n\n\ths := s.healthz(&HealthzOptions{JSEnabledOnly: true})\n\tif hs.Status == \"unavailable\" && hs.Error == NewJSNotEnabledError().Error() {\n\t\treturn\n\t}\n\n\tt.Fatalf(\"Expected healthz to return error if JetStream is disabled, got status: %s\", hs.Status)\n}\n\nfunc TestJetStreamStreamRepublishCycle(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"RPC\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo.>\", \"bar.*\", \"baz\"},\n\t}\n\n\texpectFail := func() {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(cfg)\n\t\trequire_NoError(t, err)\n\t\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar resp JSApiStreamCreateResponse\n\t\terr = json.Unmarshal(rmsg.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\tif resp.Type != JSApiStreamCreateResponseType {\n\t\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiStreamCreateResponseType)\n\t\t}\n\t\tif resp.Error == nil {\n\t\t\tt.Fatalf(\"Expected error but got none\")\n\t\t}\n\t\tif !strings.Contains(resp.Error.Description, \"republish destination forms a cycle\") {\n\t\t\tt.Fatalf(\"Expected cycle error, got %q\", resp.Error.Description)\n\t\t}\n\t}\n\n\tcfg.RePublish = &RePublish{\n\t\tSource: \"foo.>\",\n\t\tDestination: \"foo.>\",\n\t}\n\texpectFail()\n\n\tcfg.RePublish = &RePublish{\n\t\tSource: \"bar.bar\",\n\t\tDestination: \"foo.bar\",\n\t}\n\texpectFail()\n\n\tcfg.RePublish = &RePublish{\n\t\tSource: \"baz\",\n\t\tDestination: \"bar.bar\",\n\t}\n\texpectFail()\n}\n\nfunc TestJetStreamStreamRepublishOneTokenMatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"Stream1\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"one\", \"four\"},\n\t\tRePublish: &RePublish{\n\t\t\tSource: \"one\",\n\t\t\tDestination: \"uno\",\n\t\t\tHeadersOnly: false,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"uno\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"one\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tif !(len(m.Data) > 0) {\n\t\tt.Fatalf(\"Expected msg data\")\n\t}\n}\n\nfunc TestJetStreamStreamRepublishMultiTokenMatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"Stream1\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"one.>\", \"four.>\"},\n\t\tRePublish: &RePublish{\n\t\t\tSource: \"one.two.>\",\n\t\t\tDestination: \"uno.dos.>\",\n\t\t\tHeadersOnly: false,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"uno.dos.>\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"one.two.three\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tif !(len(m.Data) > 0) {\n\t\tt.Fatalf(\"Expected msg data\")\n\t}\n}\n\nfunc TestJetStreamStreamRepublishAnySubjectMatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"Stream1\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"one.>\", \"four.>\"},\n\t\tRePublish: &RePublish{\n\t\t\tDestination: \"uno.dos.>\",\n\t\t\tHeadersOnly: false,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"uno.dos.>\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"one.two.three\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\tm, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\n\tif !(len(m.Data) > 0) {\n\t\tt.Fatalf(\"Expected msg data\")\n\t}\n}\n\nfunc TestJetStreamStreamRepublishMultiTokenNoMatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"Stream1\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"one.>\", \"four.>\"},\n\t\tRePublish: &RePublish{\n\t\t\tSource: \"one.two.>\",\n\t\t\tDestination: \"uno.dos.>\",\n\t\t\tHeadersOnly: true,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"uno.dos.>\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"four.five.six\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, 0)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamStreamRepublishOneTokenNoMatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"Stream1\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"one\", \"four\"},\n\t\tRePublish: &RePublish{\n\t\t\tSource: \"one\",\n\t\t\tDestination: \"uno\",\n\t\t\tHeadersOnly: true,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"uno\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"four\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, 0)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamStreamRepublishHeadersOnly(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"RPC\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tRePublish: &RePublish{\n\t\t\tDestination: \"RP.>\",\n\t\t\tHeadersOnly: true,\n\t\t},\n\t}\n\taddStream(t, nc, cfg)\n\n\tsub, err := nc.SubscribeSync(\"RP.>\")\n\trequire_NoError(t, err)\n\n\tmsg, toSend := bytes.Repeat([]byte(\"Z\"), 512), 100\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckSubsPending(t, sub, toSend)\n\tlast := \"0\"\n\tfor i := 0; i < toSend; i++ {\n\t\tm, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tif len(m.Data) > 0 {\n\t\t\tt.Fatalf(\"Expected no msg just headers, but got %d bytes\", len(m.Data))\n\t\t}\n\t\tif sz := m.Header.Get(JSMsgSize); sz != \"512\" {\n\t\t\tt.Fatalf(\"Expected msg size hdr, got %q\", sz)\n\t\t}\n\t\tif lastHeader := m.Header.Get(JSLastSequence); lastHeader != last {\n\t\t\tt.Fatalf(\"Expected last message header %q, got %q\", last, lastHeader)\n\t\t}\n\t\tlast = m.Header.Get(JSSequence)\n\t}\n}\n\nfunc TestJetStreamMsgGetNoAdvisory(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"foo\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsub, err := nc.SubscribeSync(\"$JS.EVENT.ADVISORY.>\")\n\trequire_NoError(t, err)\n\n\t_, err = js.GetMsg(\"foo\", 1)\n\trequire_NoError(t, err)\n\n\tcheckSubsPending(t, sub, 0)\n}\n\nfunc TestJetStreamDirectMsgGet(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"DSMG\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tMaxMsgsPer: 1,\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\n\tsendStreamMsg(t, nc, \"foo\", \"foo\")\n\tsendStreamMsg(t, nc, \"bar\", \"bar\")\n\tsendStreamMsg(t, nc, \"baz\", \"baz\")\n\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"DSMG\")\n\tgetMsg := func(req *JSApiMsgGetRequest) *nats.Msg {\n\t\tvar b []byte\n\t\tvar err error\n\t\tif req != nil {\n\t\t\tb, err = json.Marshal(req)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tm, err := nc.Request(getSubj, b, time.Second)\n\t\trequire_NoError(t, err)\n\t\treturn m\n\t}\n\n\tm := getMsg(&JSApiMsgGetRequest{LastFor: \"foo\"})\n\trequire_True(t, string(m.Data) == \"foo\")\n\trequire_True(t, m.Header.Get(JSStream) == \"DSMG\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"1\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"foo\")\n\trequire_True(t, m.Subject != \"foo\")\n\trequire_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_)\n\n\tm = getMsg(&JSApiMsgGetRequest{LastFor: \"bar\"})\n\trequire_True(t, string(m.Data) == \"bar\")\n\trequire_True(t, m.Header.Get(JSStream) == \"DSMG\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"2\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"bar\")\n\trequire_True(t, m.Subject != \"bar\")\n\trequire_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_)\n\n\tm = getMsg(&JSApiMsgGetRequest{LastFor: \"baz\"})\n\trequire_True(t, string(m.Data) == \"baz\")\n\trequire_True(t, m.Header.Get(JSStream) == \"DSMG\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"3\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"baz\")\n\trequire_True(t, m.Subject != \"baz\")\n\trequire_True(t, m.Header.Get(JSTimeStamp) != _EMPTY_)\n\n\t// Test error conditions\n\n\t// Nil request\n\tm = getMsg(nil)\n\trequire_True(t, len(m.Data) == 0)\n\trequire_True(t, m.Header.Get(\"Status\") == \"408\")\n\trequire_True(t, m.Header.Get(\"Description\") == \"Empty Request\")\n\n\t// Empty request\n\tm = getMsg(&JSApiMsgGetRequest{})\n\trequire_True(t, len(m.Data) == 0)\n\trequire_True(t, m.Header.Get(\"Status\") == \"408\")\n\trequire_True(t, m.Header.Get(\"Description\") == \"Empty Request\")\n\n\t// Both set\n\tm = getMsg(&JSApiMsgGetRequest{Seq: 1, LastFor: \"foo\"})\n\trequire_True(t, len(m.Data) == 0)\n\trequire_True(t, m.Header.Get(\"Status\") == \"408\")\n\trequire_True(t, m.Header.Get(\"Description\") == \"Bad Request\")\n\n\t// Not found\n\tm = getMsg(&JSApiMsgGetRequest{LastFor: \"foobar\"})\n\trequire_True(t, len(m.Data) == 0)\n\trequire_True(t, m.Header.Get(\"Status\") == \"404\")\n\trequire_True(t, m.Header.Get(\"Description\") == \"Message Not Found\")\n\n\tm = getMsg(&JSApiMsgGetRequest{Seq: 22})\n\trequire_True(t, len(m.Data) == 0)\n\trequire_True(t, m.Header.Get(\"Status\") == \"404\")\n\trequire_True(t, m.Header.Get(\"Description\") == \"Message Not Found\")\n}\n\n// This allows support for a get next given a sequence as a starting.\n// This allows these to be chained together if needed for sparse streams.\nfunc TestJetStreamDirectMsgGetNext(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"DSMG\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\n\tsendStreamMsg(t, nc, \"foo\", \"foo\")\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"bar\", \"bar\")\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"baz\", \"baz\")\n\t}\n\tsendStreamMsg(t, nc, \"foo\", \"foo\")\n\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"DSMG\")\n\tgetMsg := func(seq uint64, subj string) *nats.Msg {\n\t\treq := []byte(fmt.Sprintf(`{\"seq\": %d, \"next_by_subj\": %q}`, seq, subj))\n\t\tm, err := nc.Request(getSubj, req, time.Second)\n\t\trequire_NoError(t, err)\n\t\treturn m\n\t}\n\n\tm := getMsg(0, \"foo\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"1\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"foo\")\n\n\tm = getMsg(1, \"foo\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"1\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"foo\")\n\n\tm = getMsg(2, \"foo\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"22\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"foo\")\n\n\tm = getMsg(2, \"bar\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"2\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"bar\")\n\n\tm = getMsg(5, \"baz\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"12\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"baz\")\n\n\tm = getMsg(14, \"baz\")\n\trequire_True(t, m.Header.Get(JSSequence) == \"14\")\n\trequire_True(t, m.Header.Get(JSSubject) == \"baz\")\n}\n\n///////////////////////////////////////////////////////////////////////////\n// Simple JetStream Benchmarks\n///////////////////////////////////////////////////////////////////////////\n\nfunc Benchmark__JetStreamPubWithAck(b *testing.B) {\n\ts := RunBasicJetStreamServer(b)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"foo\"})\n\tif err != nil {\n\t\tb.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnc.Request(\"foo\", []byte(\"Hello World!\"), 50*time.Millisecond)\n\t}\n\tb.StopTimer()\n\n\tstate := mset.state()\n\tif int(state.Msgs) != b.N {\n\t\tb.Fatalf(\"Expected %d messages, got %d\", b.N, state.Msgs)\n\t}\n}\n\nfunc Benchmark____JetStreamPubNoAck(b *testing.B) {\n\ts := RunBasicJetStreamServer(b)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"foo\"})\n\tif err != nil {\n\t\tb.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := nc.Publish(\"foo\", []byte(\"Hello World!\")); err != nil {\n\t\t\tb.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\tnc.Flush()\n\tb.StopTimer()\n\n\tstate := mset.state()\n\tif int(state.Msgs) != b.N {\n\t\tb.Fatalf(\"Expected %d messages, got %d\", b.N, state.Msgs)\n\t}\n}\n\nfunc Benchmark_JetStreamPubAsyncAck(b *testing.B) {\n\ts := RunBasicJetStreamServer(b)\n\tdefer s.Shutdown()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: \"foo\"})\n\tif err != nil {\n\t\tb.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.NoReconnect())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Put ack stream on its own connection.\n\tanc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer anc.Close()\n\n\tacks := nats.NewInbox()\n\tsub, _ := anc.Subscribe(acks, func(m *nats.Msg) {\n\t\t// Just eat them for this test.\n\t})\n\t// set max pending to unlimited.\n\tsub.SetPendingLimits(-1, -1)\n\tdefer sub.Unsubscribe()\n\n\tanc.Flush()\n\truntime.GC()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := nc.PublishRequest(\"foo\", acks, []byte(\"Hello World!\")); err != nil {\n\t\t\tb.Fatalf(\"[%d] Unexpected error: %v\", i, err)\n\t\t}\n\t}\n\tnc.Flush()\n\tb.StopTimer()\n\n\tstate := mset.state()\n\tif int(state.Msgs) != b.N {\n\t\tb.Fatalf(\"Expected %d messages, got %d\", b.N, state.Msgs)\n\t}\n}\n\nfunc Benchmark____JetStreamSubNoAck(b *testing.B) {\n\tif b.N < 10000 {\n\t\treturn\n\t}\n\n\ts := RunBasicJetStreamServer(b)\n\tdefer s.Shutdown()\n\n\tmname := \"foo\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname})\n\tif err != nil {\n\t\tb.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.NoReconnect())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Queue up messages.\n\tfor i := 0; i < b.N; i++ {\n\t\tnc.Publish(mname, []byte(\"Hello World!\"))\n\t}\n\tnc.Flush()\n\n\tstate := mset.state()\n\tif state.Msgs != uint64(b.N) {\n\t\tb.Fatalf(\"Expected %d messages, got %d\", b.N, state.Msgs)\n\t}\n\n\ttotal := int32(b.N)\n\treceived := int32(0)\n\tdone := make(chan bool)\n\n\tdeliverTo := \"DM\"\n\toname := \"O\"\n\n\tnc.Subscribe(deliverTo, func(m *nats.Msg) {\n\t\t// We only are done when we receive all, we could check for gaps too.\n\t\tif atomic.AddInt32(&received, 1) >= total {\n\t\t\tdone <- true\n\t\t}\n\t})\n\tnc.Flush()\n\n\tb.ResetTimer()\n\to, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: deliverTo, Durable: oname, AckPolicy: AckNone})\n\tif err != nil {\n\t\tb.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\t<-done\n\tb.StopTimer()\n}\n\nfunc benchJetStreamWorkersAndBatch(b *testing.B, numWorkers, batchSize int) {\n\t// Avoid running at too low of numbers since that chews up memory and GC.\n\tif b.N < numWorkers*batchSize {\n\t\treturn\n\t}\n\n\ts := RunBasicJetStreamServer(b)\n\tdefer s.Shutdown()\n\n\tmname := \"MSET22\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname})\n\tif err != nil {\n\t\tb.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.NoReconnect())\n\tif err != nil {\n\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Queue up messages.\n\tfor i := 0; i < b.N; i++ {\n\t\tnc.Publish(mname, []byte(\"Hello World!\"))\n\t}\n\tnc.Flush()\n\n\tstate := mset.state()\n\tif state.Msgs != uint64(b.N) {\n\t\tb.Fatalf(\"Expected %d messages, got %d\", b.N, state.Msgs)\n\t}\n\n\t// Create basic work queue mode consumer.\n\toname := \"WQ\"\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit})\n\tif err != nil {\n\t\tb.Fatalf(\"Expected no error with registered interest, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\ttotal := int32(b.N)\n\treceived := int32(0)\n\tstart := make(chan bool)\n\tdone := make(chan bool)\n\n\tbatchSizeMsg := []byte(strconv.Itoa(batchSize))\n\treqNextMsgSubj := o.requestNextMsgSubject()\n\n\tfor i := 0; i < numWorkers; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL(), nats.NoReconnect())\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Failed to create client: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tdeliverTo := nats.NewInbox()\n\t\tnc.Subscribe(deliverTo, func(m *nats.Msg) {\n\t\t\tif atomic.AddInt32(&received, 1) >= total {\n\t\t\t\tdone <- true\n\t\t\t}\n\t\t\t// Ack + Next request.\n\t\t\tnc.PublishRequest(m.Reply, deliverTo, AckNext)\n\t\t})\n\t\tnc.Flush()\n\t\tgo func() {\n\t\t\t<-start\n\t\t\tnc.PublishRequest(reqNextMsgSubj, deliverTo, batchSizeMsg)\n\t\t}()\n\t}\n\n\tb.ResetTimer()\n\tclose(start)\n\t<-done\n\tb.StopTimer()\n}\n\nfunc Benchmark___JetStream1x1Worker(b *testing.B) {\n\tbenchJetStreamWorkersAndBatch(b, 1, 1)\n}\n\nfunc Benchmark__JetStream1x1kWorker(b *testing.B) {\n\tbenchJetStreamWorkersAndBatch(b, 1, 1024)\n}\n\nfunc Benchmark_JetStream10x1kWorker(b *testing.B) {\n\tbenchJetStreamWorkersAndBatch(b, 10, 1024)\n}\n\nfunc Benchmark_JetStream4x512Worker(b *testing.B) {\n\tbenchJetStreamWorkersAndBatch(b, 4, 512)\n}\n\nfunc TestJetStreamKVMemoryStorePerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"TEST\", History: 1, Storage: nats.MemoryStorage})\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\tfor i := 0; i < 100_000; i++ {\n\t\t_, err := kv.PutString(fmt.Sprintf(\"foo.%d\", i), \"HELLO\")\n\t\trequire_NoError(t, err)\n\t}\n\tfmt.Printf(\"Took %v for first run\\n\", time.Since(start))\n\n\tstart = time.Now()\n\tfor i := 0; i < 100_000; i++ {\n\t\t_, err := kv.PutString(fmt.Sprintf(\"foo.%d\", i), \"HELLO WORLD\")\n\t\trequire_NoError(t, err)\n\t}\n\tfmt.Printf(\"Took %v for second run\\n\", time.Since(start))\n\n\tstart = time.Now()\n\tfor i := 0; i < 100_000; i++ {\n\t\t_, err := kv.Get(fmt.Sprintf(\"foo.%d\", i))\n\t\trequire_NoError(t, err)\n\t}\n\tfmt.Printf(\"Took %v for get\\n\", time.Since(start))\n}\n\nfunc TestJetStreamKVMemoryStoreDirectGetPerf(t *testing.T) {\n\t// Comment out to run, holding place for now.\n\tt.SkipNow()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tMaxMsgsPer: 1,\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\n\tstart := time.Now()\n\tfor i := 0; i < 100_000; i++ {\n\t\t_, err := js.Publish(fmt.Sprintf(\"foo.%d\", i), []byte(\"HELLO\"))\n\t\trequire_NoError(t, err)\n\t}\n\tfmt.Printf(\"Took %v for put\\n\", time.Since(start))\n\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"TEST\")\n\n\tconst tmpl = \"{\\\"last_by_subj\\\":%q}\"\n\n\tstart = time.Now()\n\tfor i := 0; i < 100_000; i++ {\n\t\treq := []byte(fmt.Sprintf(tmpl, fmt.Sprintf(\"foo.%d\", i)))\n\t\t_, err := nc.Request(getSubj, req, time.Second)\n\t\trequire_NoError(t, err)\n\t}\n\tfmt.Printf(\"Took %v for get\\n\", time.Since(start))\n}\n\nfunc TestJetStreamMultiplePullPerf(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjs.AddStream(&nats.StreamConfig{Name: \"mp22\", Storage: nats.FileStorage})\n\tdefer js.DeleteStream(\"mp22\")\n\n\tn, msg := 1_000_000, []byte(\"OK\")\n\tfor i := 0; i < n; i++ {\n\t\tjs.PublishAsync(\"mp22\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsi, err := js.StreamInfo(\"mp22\")\n\trequire_NoError(t, err)\n\n\tfmt.Printf(\"msgs: %d, total_bytes: %v\\n\", si.State.Msgs, friendlyBytes(int64(si.State.Bytes)))\n\n\t// 10 pull subscribers each asking for 100 msgs.\n\t_, err = js.AddConsumer(\"mp22\", &nats.ConsumerConfig{\n\t\tDurable: \"d\",\n\t\tMaxAckPending: 8_000,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\tnp, bs := 10, 100\n\n\tcount := 0\n\n\tfor i := 0; i < np; i++ {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tsub, err := js.PullSubscribe(\"mp22\", \"d\")\n\t\trequire_NoError(t, err)\n\n\t\twg.Add(1)\n\t\tgo func(sub *nats.Subscription) {\n\t\t\tdefer wg.Done()\n\t\t\t<-startCh\n\t\t\tfor i := 0; i < n/(np*bs); i++ {\n\t\t\t\tmsgs, err := sub.Fetch(bs)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Got error on pull: %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif len(msgs) != bs {\n\t\t\t\t\tt.Logf(\"Expected %d msgs, got %d\", bs, len(msgs))\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tcount += len(msgs)\n\t\t\t\tfor _, m := range msgs {\n\t\t\t\t\tm.Ack()\n\t\t\t\t}\n\t\t\t}\n\t\t}(sub)\n\t}\n\n\tstart := time.Now()\n\tclose(startCh)\n\twg.Wait()\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs [%d]\\n\", tt, n, count)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n}\n\nfunc TestJetStreamMirrorUpdatesNotSupported(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"SOURCE\"})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"SOURCE\"},\n\t}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Promoting a mirror is supported though.\n\tcfg.Mirror = nil\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamMirrorFirstSeqNotSupported(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t_, err := s.gacc.addStream(&StreamConfig{Name: \"SOURCE\"})\n\trequire_NoError(t, err)\n\n\tcfg := &StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &StreamSource{Name: \"SOURCE\"},\n\t\tFirstSeq: 123,\n\t}\n\t_, err = s.gacc.addStream(cfg)\n\trequire_Error(t, err, NewJSMirrorWithFirstSeqError())\n}\n\nfunc TestJetStreamDirectGetBySubject(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\n\t\tONLYME = {\n\t\t\tpublish = { allow = \"$JS.API.DIRECT.GET.KV.vid.22.>\"}\n\t\t}\n\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [\n\t\t\t\t\t{ user: admin, password: s3cr3t },\n\t\t\t\t\t{ user: user, password: pwd, permissions: $ONLYME},\n\t\t\t\t]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"admin\", \"s3cr3t\"))\n\tdefer nc.Close()\n\n\t// Do by hand for now.\n\tcfg := &StreamConfig{\n\t\tName: \"KV\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"vid.*.>\"},\n\t\tMaxMsgsPer: 1,\n\t\tAllowDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\n\t// Add in mirror as well.\n\tcfg = &StreamConfig{\n\t\tName: \"M\",\n\t\tStorage: MemoryStorage,\n\t\tMirror: &StreamSource{Name: \"KV\"},\n\t\tMirrorDirect: true,\n\t}\n\taddStream(t, nc, cfg)\n\n\tv22 := \"vid.22.speed\"\n\tv33 := \"vid.33.speed\"\n\t_, err := js.Publish(v22, []byte(\"100\"))\n\trequire_NoError(t, err)\n\t_, err = js.Publish(v33, []byte(\"55\"))\n\trequire_NoError(t, err)\n\n\t// User the restricted user.\n\tnc, _ = jsClientConnect(t, s, nats.UserInfo(\"user\", \"pwd\"))\n\tdefer nc.Close()\n\n\terrCh := make(chan error, 10)\n\tnc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\tselect {\n\t\tcase errCh <- e:\n\t\tdefault:\n\t\t}\n\t})\n\n\tgetSubj := fmt.Sprintf(JSDirectGetLastBySubjectT, \"KV\", v22)\n\tm, err := nc.Request(getSubj, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_True(t, string(m.Data) == \"100\")\n\n\t// Now attempt to access vid 33 data..\n\tgetSubj = fmt.Sprintf(JSDirectGetLastBySubjectT, \"KV\", v33)\n\t_, err = nc.Request(getSubj, nil, 200*time.Millisecond)\n\trequire_Error(t, err) // timeout here.\n\n\tselect {\n\tcase e := <-errCh:\n\t\tif !strings.HasPrefix(e.Error(), \"nats: permissions violation\") {\n\t\t\tt.Fatalf(\"Expected a permissions violation but got %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Expected to get a permissions error, got none\")\n\t}\n\n\t// Now make sure mirrors are doing right thing with new way as well.\n\tvar sawMirror bool\n\tgetSubj = fmt.Sprintf(JSDirectGetLastBySubjectT, \"KV\", v22)\n\tfor i := 0; i < 100; i++ {\n\t\tm, err := nc.Request(getSubj, nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tif shdr := m.Header.Get(JSStream); shdr == \"M\" {\n\t\t\tsawMirror = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !sawMirror {\n\t\tt.Fatalf(\"Expected to see the mirror respond at least once\")\n\t}\n}\n\nfunc TestJetStreamProperErrorDueToOverlapSubjects(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ttest := func(t *testing.T, s *Server) {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Now do this by end since we want to check the error returned.\n\t\tsc := &nats.StreamConfig{\n\t\t\tName: \"TEST2\",\n\t\t\tSubjects: []string{\"foo.>\"},\n\t\t}\n\t\treq, _ := json.Marshal(sc)\n\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, sc.Name), req, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tvar scResp JSApiStreamCreateResponse\n\t\terr = json.Unmarshal(msg.Data, &scResp)\n\t\trequire_NoError(t, err)\n\n\t\tif scResp.Error == nil || !IsNatsErr(scResp.Error, JSStreamSubjectOverlapErr) {\n\t\t\tt.Fatalf(\"Did not receive correct error: %+v\", scResp)\n\t\t}\n\t}\n\n\tt.Run(\"standalone\", func(t *testing.T) { test(t, s) })\n\tt.Run(\"clustered\", func(t *testing.T) { test(t, c.randomServer()) })\n}\n\nfunc TestJetStreamServerCipherConvert(t *testing.T) {\n\ttmpl := `\n\t\tserver_name: S22\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {key: s3cr3t, store_dir: '%s', cipher: %s}\n\t`\n\tstoreDir := t.TempDir()\n\n\t// Create a stream and a consumer under one cipher, and restart the server with a new cipher.\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, \"AES\")))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfor i := 0; i < 1000; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"TOP SECRET DOCUMENT #%d\", i+1))\n\t\t_, err := js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Make sure consumers convert as well.\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\tfor _, m := range fetchMsgs(t, sub, 100, 5*time.Second) {\n\t\tm.AckSync()\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Stop current\n\ts.Shutdown()\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, \"ChaCha\")))\n\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi2, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif !reflect.DeepEqual(si, si2) {\n\t\tt.Fatalf(\"Stream infos did not match\\n%+v\\nvs\\n%+v\", si, si2)\n\t}\n\n\tci2, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// Consumer create times can be slightly off after restore from disk.\n\tnow := time.Now()\n\tci.Created, ci2.Created = now, now\n\tci.Delivered.Last, ci2.Delivered.Last = nil, nil\n\tci.AckFloor.Last, ci2.AckFloor.Last = nil, nil\n\t// Also clusters will be different.\n\tci.Cluster, ci2.Cluster = nil, nil\n\n\tif !reflect.DeepEqual(ci, ci2) {\n\t\tt.Fatalf(\"Consumer infos did not match\\n%+v\\nvs\\n%+v\", ci, ci2)\n\t}\n}\n\nfunc TestJetStreamAllowDirectAfterUpdate(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t})\n\trequire_NoError(t, err)\n\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\n\tsi, err := js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.AllowDirect)\n\n\t_, err = js.GetLastMsg(\"TEST\", \"foo\", nats.DirectGet(), nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\n\t// Make sure turning off works too.\n\tsi, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tAllowDirect: false,\n\t})\n\trequire_NoError(t, err)\n\trequire_False(t, si.Config.AllowDirect)\n\n\t_, err = js.GetLastMsg(\"TEST\", \"foo\", nats.DirectGet(), nats.MaxWait(100*time.Millisecond))\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamSubjectBasedFilteredConsumers(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [ {\n\t\t\t\t\tuser: u,\n\t\t\t\t\tpassword: p\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\tpublish {\n\t\t\t\t\t\t\tallow: [\n\t\t\t\t\t\t\t\t'ID.>',\n\t\t\t\t\t\t\t\t'$JS.API.INFO',\n\t\t\t\t\t\t\t\t'$JS.API.STREAM.>',\n\t\t\t\t\t\t\t\t'$JS.API.CONSUMER.INFO.>',\n\t\t\t\t\t\t\t\t'$JS.API.CONSUMER.CREATE.TEST.VIN-xxx.ID.foo.>', # Only allow ID.foo.\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\tdeny: [ '$JS.API.CONSUMER.CREATE.*', '$JS.API.CONSUMER.DURABLE.CREATE.*.*']\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} ]\n\t\t\t},\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"u\", \"p\"), nats.ErrorHandler(noOpErrHandler))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ID.*.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(fmt.Sprintf(\"ID.foo.%d\", i*3), nil)\n\t\tjs.Publish(fmt.Sprintf(\"ID.bar.%d\", i*3+1), nil)\n\t\tjs.Publish(fmt.Sprintf(\"ID.baz.%d\", i*3+2), nil)\n\t}\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 300)\n\n\t// Trying to create a consumer with non filtered API should fail.\n\tjs, err = nc.JetStream(nats.MaxWait(200 * time.Millisecond))\n\trequire_NoError(t, err)\n\n\t_, err = js.SubscribeSync(\"ID.foo.*\")\n\trequire_Error(t, err, nats.ErrTimeout, context.DeadlineExceeded)\n\n\t_, err = js.SubscribeSync(\"ID.foo.*\", nats.Durable(\"dlc\"))\n\trequire_Error(t, err, nats.ErrTimeout, context.DeadlineExceeded)\n\n\t// Direct filtered should work.\n\t// Need to do by hand for now.\n\tecSubj := fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"VIN-xxx\", \"ID.foo.*\")\n\n\tcrReq := CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverPolicy: DeliverLast,\n\t\t\tFilterSubject: \"ID.foo.*\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t},\n\t}\n\treq, err := json.Marshal(crReq)\n\trequire_NoError(t, err)\n\n\tresp, err := nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\tvar ccResp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tif ccResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", ccResp.Error)\n\t}\n\tcfg := ccResp.Config\n\tci := ccResp.ConsumerInfo\n\t// Make sure we recognized as an ephemeral (since no durable was set) and that we have an InactiveThreshold.\n\t// Make sure we captured preferred ephemeral name.\n\tif ci.Name != \"VIN-xxx\" {\n\t\tt.Fatalf(\"Did not get correct name, expected %q got %q\", \"xxx\", ci.Name)\n\t}\n\tif cfg.InactiveThreshold == 0 {\n\t\tt.Fatalf(\"Expected default inactive threshold to be set, got %v\", cfg.InactiveThreshold)\n\t}\n\n\t// Make sure we can not use different consumer name since locked above.\n\tecSubj = fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"VIN-zzz\", \"ID.foo.*\")\n\t_, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Now check that we error when we mismatch filtersubject.\n\tcrReq = CreateConsumerRequest{\n\t\tStream: \"TEST\",\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverPolicy: DeliverLast,\n\t\t\tFilterSubject: \"ID.bar.*\",\n\t\t\tAckPolicy: AckExplicit,\n\t\t},\n\t}\n\treq, err = json.Marshal(crReq)\n\trequire_NoError(t, err)\n\n\tecSubj = fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"VIN-xxx\", \"ID.foo.*\")\n\tresp, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_NoError(t, err)\n\terr = json.Unmarshal(resp.Data, &ccResp)\n\trequire_NoError(t, err)\n\tcheckNatsError(t, ccResp.Error, JSConsumerCreateFilterSubjectMismatchErr)\n\n\t// Now make sure if we change subject to match that we can not create a filtered consumer on ID.bar.>\n\tecSubj = fmt.Sprintf(JSApiConsumerCreateExT, \"TEST\", \"VIN-xxx\", \"ID.bar.*\")\n\t_, err = nc.Request(ecSubj, req, 500*time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n}\n\nfunc TestJetStreamStreamSubjectsOverlap(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\", \"foo.A\"},\n\t})\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(err.Error(), \"overlaps\"))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\", \"foo.A\"},\n\t})\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(err.Error(), \"overlaps\"))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.bar.*\", \"foo.*.bar\"},\n\t})\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(err.Error(), \"overlaps\"))\n}\n\nfunc TestJetStreamStreamTransformOverlap(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\",\n\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t{\n\t\t\t\t\tSource: \"foo.*.bar\",\n\t\t\t\t\tDestination: \"baz\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tSource: \"foo.bar.*\",\n\t\t\t\t\tDestination: \"baz\",\n\t\t\t\t},\n\t\t\t},\n\t\t}})\n\trequire_Error(t, err)\n\trequire_True(t, strings.Contains(err.Error(), \"overlap\"))\n\n}\n\nfunc TestJetStreamSuppressAllowDirect(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"key.*\"},\n\t\tMaxMsgsPerSubject: 1,\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, si.Config.AllowDirect)\n\n\tsi, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"key.*\"},\n\t\tMaxMsgsPerSubject: 1,\n\t\tAllowDirect: false,\n\t})\n\trequire_NoError(t, err)\n\trequire_False(t, si.Config.AllowDirect)\n\n\tsendStreamMsg(t, nc, \"key.22\", \"msg\")\n\n\t_, err = js.GetLastMsg(\"TEST\", \"foo\", nats.DirectGet(), nats.MaxWait(100*time.Millisecond))\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamAccountPurge(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\taccKp, accpub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accpub)\n\taccClaim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 * 5\n\taccClaim.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024 * 5\n\taccJwt := encodeClaim(t, accClaim, accpub)\n\taccCreds := newUser(t, accKp)\n\n\tstoreDir := t.TempDir()\n\n\tcfg := createConfFile(t, []byte(fmt.Sprintf(`\n host: 127.0.0.1\n port:-1\n server_name: S1\n operator: %s\n system_account: %s\n resolver: {\n type: full\n dir: '%s/jwt'\n }\n jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s/js'}\n`, ojwt, syspub, storeDir, storeDir)))\n\tdefer os.Remove(cfg)\n\n\ts, o := RunServerWithConfig(cfg)\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)\n\tdefer s.Shutdown()\n\n\tinspectDirs := func(t *testing.T, accTotal int) error {\n\t\tt.Helper()\n\t\tif accTotal == 0 {\n\t\t\tfiles, err := os.ReadDir(filepath.Join(o.StoreDir, \"jetstream\", accpub))\n\t\t\trequire_True(t, len(files) == accTotal || err != nil)\n\t\t} else {\n\t\t\tfiles, err := os.ReadDir(filepath.Join(o.StoreDir, \"jetstream\", accpub, \"streams\"))\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(files) == accTotal)\n\t\t}\n\t\treturn nil\n\t}\n\n\tcreateTestData := func() {\n\t\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\t\tdefer nc.Close()\n\t\tjs, err := nc.JetStream()\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST1\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t_, err = js.AddConsumer(\"TEST1\",\n\t\t\t&nats.ConsumerConfig{Durable: \"DUR1\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy})\n\t\trequire_NoError(t, err)\n\t}\n\n\tpurge := func(t *testing.T) {\n\t\tt.Helper()\n\t\tvar resp JSApiAccountPurgeResponse\n\t\tncsys := natsConnect(t, s.ClientURL(), nats.UserCredentials(sysCreds))\n\t\tdefer ncsys.Close()\n\t\tm, err := ncsys.Request(fmt.Sprintf(JSApiAccountPurgeT, accpub), nil, 5*time.Second)\n\t\trequire_NoError(t, err)\n\t\terr = json.Unmarshal(m.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, resp.Initiated)\n\t}\n\n\tcreateTestData()\n\tinspectDirs(t, 1)\n\tpurge(t)\n\tinspectDirs(t, 0)\n\tcreateTestData()\n\tinspectDirs(t, 1)\n\n\ts.Shutdown()\n\trequire_NoError(t, os.Remove(storeDir+\"/jwt/\"+accpub+\".jwt\"))\n\n\ts, o = RunServerWithConfig(o.ConfigFile)\n\tdefer s.Shutdown()\n\tinspectDirs(t, 1)\n\tpurge(t)\n\tinspectDirs(t, 0)\n}\n\n// For issue https://github.com/nats-io/nats-server/issues/3612\n// Do auto cleanup.\nfunc TestJetStreamDanglingMessageAutoCleanup(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.MaxAckPending(10))\n\trequire_NoError(t, err)\n\n\t// Send 100 msgs\n\tn := 100\n\tfor i := 0; i < n; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\t}\n\n\t// Grab and ack 10 messages.\n\tfor _, m := range fetchMsgs(t, sub, 10, time.Second) {\n\t\tm.AckSync()\n\t}\n\n\tci, err := sub.ConsumerInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, ci.AckFloor.Stream == 10)\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\t// We will hand move the ackfloor to simulate dangling message condition.\n\tcstore := filepath.Join(sd, \"$G\", \"streams\", \"TEST\", \"obs\", \"dlc\", \"o.dat\")\n\n\tbuf, err := os.ReadFile(cstore)\n\trequire_NoError(t, err)\n\n\tstate, err := decodeConsumerState(buf)\n\trequire_NoError(t, err)\n\n\t// Update from 10 for delivered and ack to 90.\n\tstate.Delivered.Stream, state.Delivered.Consumer = 90, 90\n\tstate.AckFloor.Stream, state.AckFloor.Consumer = 90, 90\n\n\terr = os.WriteFile(cstore, encodeConsumerState(state), defaultFilePerms)\n\trequire_NoError(t, err)\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.State.Msgs != 10 {\n\t\tt.Fatalf(\"Expected auto-cleanup to have worked but got %d msgs vs 10\", si.State.Msgs)\n\t}\n}\n\n// Issue https://github.com/nats-io/nats-server/issues/3645\nfunc TestJetStreamMsgIDHeaderCollision(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ORDERS.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"ORDERS.test\")\n\tm.Header.Add(JSMsgId, \"1\")\n\tm.Data = []byte(\"ok\")\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tm.Header = make(nats.Header)\n\tm.Header.Add(\"Orig-Nats-Msg-Id\", \"1\")\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tm.Header = make(nats.Header)\n\tm.Header.Add(\"Original-Nats-Msg-Id\", \"1\")\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tm.Header = make(nats.Header)\n\tm.Header.Add(\"Original-Nats-Msg-Id\", \"1\")\n\tm.Header.Add(\"Really-Original-Nats-Msg-Id\", \"1\")\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tm.Header = make(nats.Header)\n\tm.Header.Add(\"X\", \"Nats-Msg-Id:1\")\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\trequire_True(t, si.State.Msgs == 5)\n}\n\n// https://github.com/nats-io/nats-server/issues/3657\nfunc TestJetStreamServerCrashOnPullConsumerDeleteWithInactiveThresholdAfterAck(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo\", \"msg\")\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.InactiveThreshold(10*time.Second))\n\trequire_NoError(t, err)\n\n\tmsgs := fetchMsgs(t, sub, 1, time.Second)\n\trequire_True(t, len(msgs) == 1)\n\tmsgs[0].Ack()\n\terr = js.DeleteConsumer(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// If server crashes this will fail.\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n}\n\n// createConsumer is a temporary method until nats.go client supports multiple subjects.\n// it is used where lowe level call on mset is not enough, as we want to test error validation.\nfunc createConsumer(t *testing.T, nc *nats.Conn, stream string, config ConsumerConfig) JSApiConsumerCreateResponse {\n\treq, err := json.Marshal(&CreateConsumerRequest{Stream: stream, Config: config})\n\trequire_NoError(t, err)\n\n\tresp, err := nc.Request(fmt.Sprintf(\"$JS.API.CONSUMER.DURABLE.CREATE.%s.%s\", stream, config.Durable), req, time.Second*10)\n\trequire_NoError(t, err)\n\n\tvar apiResp JSApiConsumerCreateResponse\n\trequire_NoError(t, json.Unmarshal(resp.Data, &apiResp))\n\n\treturn apiResp\n}\n\nfunc TestJetStreamBothFiltersSet(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\t_, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events.>\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\tresp := createConsumer(t, nc, \"deliver\", ConsumerConfig{\n\t\tFilterSubjects: []string{\"events.one\", \"events.two\"},\n\t\tFilterSubject: \"events.three\",\n\t\tDurable: \"name\",\n\t})\n\trequire_True(t, resp.Error.ErrCode == 10136)\n}\n\nfunc TestJetStreamMultipleSubjectsPushBasic(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{\n\t\tSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tDurable: \"name\",\n\t\tDeliverSubject: \"push\",\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := nc.SubscribeSync(\"push\")\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"other\", \"10\")\n\tsendStreamMsg(t, nc, \"events\", \"0\")\n\tsendStreamMsg(t, nc, \"data\", \"1\")\n\tsendStreamMsg(t, nc, \"events\", \"2\")\n\tsendStreamMsg(t, nc, \"events\", \"3\")\n\tsendStreamMsg(t, nc, \"other\", \"10\")\n\tsendStreamMsg(t, nc, \"data\", \"4\")\n\tsendStreamMsg(t, nc, \"data\", \"5\")\n\n\tfor i := 0; i < 6; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second * 1)\n\t\trequire_NoError(t, err)\n\t\tif fmt.Sprintf(\"%v\", i) != string(msg.Data) {\n\t\t\tt.Fatalf(\"bad sequence. Expected %v, got %v\", i, string(msg.Data))\n\t\t}\n\t}\n\tinfo, err := js.ConsumerInfo(\"deliver\", \"name\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.AckFloor.Consumer == 6)\n\trequire_True(t, info.AckFloor.Stream == 8)\n}\nfunc TestJetStreamMultipleSubjectsBasic(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tacc := s.GlobalAccount()\n\n\tmset, err := acc.addStream(&StreamConfig{\n\t\tSubjects: []string{\"events\", \"data\", \"other\"},\n\t\tName: \"deliver\",\n\t})\n\trequire_NoError(t, err)\n\n\tmset.addConsumer(&ConsumerConfig{\n\t\tFilterSubjects: []string{\"events\", \"data\"},\n\t\tDurable: \"name\",\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"other\", \"10\")\n\tsendStreamMsg(t, nc, \"events\", \"0\")\n\tsendStreamMsg(t, nc, \"data\", \"1\")\n\tsendStreamMsg(t, nc, \"events\", \"2\")\n\tsendStreamMsg(t, nc, \"events\", \"3\")\n\tsendStreamMsg(t, nc, \"other\", \"10\")\n\tsendStreamMsg(t, nc, \"data\", \"4\")\n\tsendStreamMsg(t, nc, \"data\", \"5\")\n\n\tconsumer, err := js.PullSubscribe(\"\", \"name\", nats.Bind(\"deliver\", \"name\"))\n\trequire_NoError(t, err)\n\n\tmsg, err := consumer.Fetch(6)\n\trequire_NoError(t, err)\n\n\tfor i, msg := range msg {\n\t\tif fmt.Sprintf(\"%v\", i) != string(msg.Data) {\n\t\t\tt.Fatalf(\"bad sequence. Expected %v, got %v\", i, string(msg.Data))\n\t\t}\n\t}\n\t_, err = js.ConsumerInfo(\"deliver\", \"name\")\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamKVDelete(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"deletion\",\n\t\tHistory: 10,\n\t})\n\trequire_NoError(t, err)\n\tkv.Put(\"a\", nil)\n\tkv.Put(\"a.a\", nil)\n\tkv.Put(\"a.b\", nil)\n\tkv.Put(\"a.b.c\", nil)\n\n\tkeys, err := kv.Keys()\n\trequire_NoError(t, err)\n\trequire_True(t, len(keys) == 4)\n\n\tinfo, err := js.AddConsumer(\"KV_deletion\", &nats.ConsumerConfig{\n\t\tName: \"keys\",\n\t\tFilterSubject: \"$KV.deletion.a.*\",\n\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\tDeliverSubject: \"keys\",\n\t\tMaxDeliver: 1,\n\t\tAckPolicy: nats.AckNonePolicy,\n\t\tMemoryStorage: true,\n\t\tFlowControl: true,\n\t\tHeartbeat: time.Second * 5,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, info.NumPending == 2)\n\n\tsub, err := js.SubscribeSync(\"$KV.deletion.a.*\", nats.Bind(\"KV_deletion\", \"keys\"))\n\trequire_NoError(t, err)\n\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\t_, err = sub.NextMsg(time.Second * 1)\n\trequire_NoError(t, err)\n\tmsg, err := sub.NextMsg(time.Second * 1)\n\trequire_True(t, msg == nil)\n\trequire_Error(t, err)\n\n\trequire_NoError(t, kv.Delete(\"a.a\"))\n\trequire_NoError(t, kv.Delete(\"a.b\"))\n\n\twatcher, err := kv.WatchAll()\n\trequire_NoError(t, err)\n\n\tupdates := watcher.Updates()\n\n\tkeys = []string{}\n\tfor v := range updates {\n\t\tif v == nil {\n\t\t\tbreak\n\t\t}\n\t\tif v.Operation() == nats.KeyValueDelete {\n\t\t\tkeys = append(keys, v.Key())\n\t\t}\n\t}\n\trequire_True(t, len(keys) == 2)\n}\n\nfunc TestJetStreamDeliverLastPerSubjectWithKV(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tMaxMsgsPerSubject: 5,\n\t\tSubjects: []string{\"kv.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"kv.a\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a\", \"a\")\n\tsendStreamMsg(t, nc, \"kv.a.b\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a.b\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a.b\", \"a.b\")\n\tsendStreamMsg(t, nc, \"kv.a.b.c\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a.b.c\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a.b.c\", \"bad\")\n\tsendStreamMsg(t, nc, \"kv.a.b.c\", \"a.b.c\")\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"CONSUMER\",\n\t\tFilterSubject: \"kv.>\",\n\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\tDeliverSubject: \"deliver\",\n\t\tMaxDeliver: 1,\n\t\tAckPolicy: nats.AckNonePolicy,\n\t\tMemoryStorage: true,\n\t\tFlowControl: true,\n\t\tHeartbeat: time.Second * 5,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"kv.>\", nats.Bind(\"TEST\", \"CONSUMER\"))\n\trequire_NoError(t, err)\n\n\tfor i := 1; i <= 3; i++ {\n\t\t_, err := sub.NextMsg(time.Second * 1)\n\t\trequire_NoError(t, err)\n\t}\n\n\tmsg, err := sub.NextMsg(time.Second * 1)\n\tif err == nil || msg != nil {\n\t\tt.Fatalf(\"should not get any more messages\")\n\t}\n}\n\nfunc TestJetStreamStreamUpdateSubjectsOverlapOthers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"TEST\", \"foo.a\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"TEST2\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// we expect an error updating stream TEST2 with subject that overlaps that used by TEST\n\t// foo.a fails too, but foo.* also double-check for sophisticated overlap match\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"TEST2\", \"foo.*\"},\n\t})\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"overlap\")\n}\n\nfunc TestJetStreamMetaDataFailOnKernelFault(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\n\tsd := s.JetStreamConfig().StoreDir\n\tsdir := filepath.Join(sd, \"$G\", \"streams\", \"TEST\")\n\ts.Shutdown()\n\n\t// Emulate if the kernel did not flush out to disk the meta information.\n\t// so we will zero out both meta.inf and meta.sum.\n\terr = os.WriteFile(filepath.Join(sdir, JetStreamMetaFile), nil, defaultFilePerms)\n\trequire_NoError(t, err)\n\n\terr = os.WriteFile(filepath.Join(sdir, JetStreamMetaFileSum), nil, defaultFilePerms)\n\trequire_NoError(t, err)\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// The stream will have not been recovered. So err is normal.\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_Error(t, err)\n\n\t// Make sure we are signaled here from healthz\n\ths := s.healthz(nil)\n\tconst expected = \"JetStream stream '$G > TEST' could not be recovered\"\n\tif hs.Status != \"unavailable\" || hs.Error == _EMPTY_ {\n\t\tt.Fatalf(\"Expected healthz to return an error\")\n\t} else if hs.Error != expected {\n\t\tt.Fatalf(\"Expected healthz error %q got %q\", expected, hs.Error)\n\t}\n\n\t// If we add it back, this should recover the msgs.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure we recovered.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 10)\n\n\t// Now if we restart the server, meta should be correct,\n\t// and the stream should be restored.\n\ts.Shutdown()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Make sure we recovered the stream correctly after re-adding.\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 10)\n}\n\n// https://github.com/nats-io/nats-server/issues/3734\nfunc TestJetStreamMsgBlkFailOnKernelFault(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxBytes: 10 * 1024 * 1024, // 10MB\n\t})\n\trequire_NoError(t, err)\n\n\tmsgSize := 1024 * 1024 // 1MB\n\tmsg := make([]byte, msgSize)\n\tcrand.Read(msg)\n\n\tfor i := 0; i < 20; i++ {\n\t\t_, err := js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Bytes < uint64(si.Config.MaxBytes))\n\n\t// Now emulate a kernel fault that fails to write the last blk properly.\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tmset.mu.RLock()\n\tfs := mset.store.(*fileStore)\n\tfs.mu.RLock()\n\trequire_True(t, len(fs.blks) > 2)\n\t// Here we do not grab the last one, which we handle correctly. We grab an interior one near the end.\n\tlmbf := fs.blks[len(fs.blks)-2].mfn\n\tfs.mu.RUnlock()\n\tmset.mu.RUnlock()\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\t// Remove block.\n\trequire_NoError(t, os.Remove(lmbf))\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err = js.GetMsg(\"TEST\", 17)\n\trequire_Error(t, err, nats.ErrMsgNotFound)\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.NumDeleted == 3)\n\n\t// Test detailed version as well.\n\tsi, err = js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{DeletedDetails: true})\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.NumDeleted == 3)\n\tif !reflect.DeepEqual(si.State.Deleted, []uint64{16, 17, 18}) {\n\t\tt.Fatalf(\"Expected deleted of %+v, got %+v\", []uint64{16, 17, 18}, si.State.Deleted)\n\t}\n\n\tfor i := 0; i < 20; i++ {\n\t\t_, err := js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\tif si.State.Bytes > uint64(si.Config.MaxBytes) {\n\t\tt.Fatalf(\"MaxBytes not enforced with empty interior msg blk, max %v, bytes %v\",\n\t\t\tfriendlyBytes(si.Config.MaxBytes), friendlyBytes(int64(si.State.Bytes)))\n\t}\n}\n\nfunc TestJetStreamPurgeExAndAccounting(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcfg *nats.StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tcfg: &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: nats.MemoryStorage,\n\t\t\t\tSubjects: []string{\"*\"},\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tcfg: &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: nats.FileStorage,\n\t\t\t\tSubjects: []string{\"*\"},\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\t// Client for API requests.\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(c.cfg)\n\t\trequire_NoError(t, err)\n\n\t\tmsg := []byte(\"accounting\")\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, err = js.Publish(\"foo\", msg)\n\t\t\trequire_NoError(t, err)\n\t\t\t_, err = js.Publish(\"bar\", msg)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\n\t\tinfo, err := js.AccountInfo()\n\t\trequire_NoError(t, err)\n\n\t\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"foo\"})\n\t\trequire_NoError(t, err)\n\n\t\tninfo, err := js.AccountInfo()\n\t\trequire_NoError(t, err)\n\n\t\t// Make sure we did the proper accounting.\n\t\tif c.cfg.Storage == nats.MemoryStorage {\n\t\t\tif ninfo.Memory != info.Memory/2 {\n\t\t\t\tt.Fatalf(\"Accounting information incorrect for Memory: %d vs %d\",\n\t\t\t\t\tninfo.Memory, info.Memory/2)\n\t\t\t}\n\t\t} else {\n\t\t\tif ninfo.Store != info.Store/2 {\n\t\t\t\tt.Fatalf(\"Accounting information incorrect for FileStore: %d vs %d\",\n\t\t\t\t\tninfo.Store, info.Store/2)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestJetStreamRollup(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tconst STREAM = \"S\"\n\tconst SUBJ = \"S.*\"\n\n\tjs.AddStream(&nats.StreamConfig{\n\t\tName: STREAM,\n\t\tSubjects: []string{SUBJ},\n\t\tAllowRollup: true,\n\t})\n\n\tfor i := 1; i <= 10; i++ {\n\t\tsendStreamMsg(t, nc, \"S.A\", fmt.Sprintf(\"%v\", i))\n\t\tsendStreamMsg(t, nc, \"S.B\", fmt.Sprintf(\"%v\", i))\n\t}\n\n\tsinfo, err := js.StreamInfo(STREAM)\n\trequire_NoError(t, err)\n\trequire_True(t, sinfo.State.Msgs == 20)\n\n\tcinfo, err := js.AddConsumer(STREAM, &nats.ConsumerConfig{\n\t\tDurable: \"DUR-A\",\n\t\tFilterSubject: \"S.A\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, cinfo.NumPending == 10)\n\n\tm := nats.NewMsg(\"S.A\")\n\tm.Header.Set(JSMsgRollup, JSMsgRollupSubject)\n\n\t_, err = js.PublishMsg(m)\n\trequire_NoError(t, err)\n\n\tcinfo, err = js.ConsumerInfo(\"S\", \"DUR-A\")\n\trequire_NoError(t, err)\n\trequire_True(t, cinfo.NumPending == 1)\n\n\tsinfo, err = js.StreamInfo(STREAM)\n\trequire_NoError(t, err)\n\trequire_True(t, sinfo.State.Msgs == 11)\n\n\tcinfo, err = js.AddConsumer(STREAM, &nats.ConsumerConfig{\n\t\tDurable: \"DUR-B\",\n\t\tFilterSubject: \"S.B\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\trequire_True(t, cinfo.NumPending == 10)\n}\n\nfunc TestJetStreamPartialPurgeWithAckPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tnmsgs := 100\n\tfor i := 0; i < nmsgs; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.AckWait(time.Second))\n\trequire_NoError(t, err)\n\n\t// Queue up all for ack pending.\n\t_, err = sub.Fetch(nmsgs)\n\trequire_NoError(t, err)\n\n\tkeep := nmsgs / 2\n\trequire_NoError(t, js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Keep: uint64(keep)}))\n\n\t// Should be able to be redelivered now.\n\ttime.Sleep(2 * time.Second)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\t// Make sure we calculated correctly.\n\trequire_True(t, ci.AckFloor.Consumer == uint64(keep))\n\trequire_True(t, ci.AckFloor.Stream == uint64(keep))\n\trequire_True(t, ci.NumAckPending == keep)\n\trequire_True(t, ci.NumPending == 0)\n\n\tfor i := 0; i < nmsgs; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\t// Make sure we calculated correctly.\n\t// Top 3 will be same.\n\trequire_True(t, ci.AckFloor.Consumer == uint64(keep))\n\trequire_True(t, ci.AckFloor.Stream == uint64(keep))\n\trequire_True(t, ci.NumAckPending == keep)\n\trequire_True(t, ci.NumPending == uint64(nmsgs))\n\trequire_True(t, ci.NumRedelivered == 0)\n\n\tmsgs, err := sub.Fetch(keep)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == keep)\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\t// Make sure we calculated correctly.\n\trequire_True(t, ci.Delivered.Consumer == uint64(nmsgs+keep))\n\trequire_True(t, ci.Delivered.Stream == uint64(nmsgs))\n\trequire_True(t, ci.AckFloor.Consumer == uint64(keep))\n\trequire_True(t, ci.AckFloor.Stream == uint64(keep))\n\trequire_True(t, ci.NumAckPending == keep)\n\trequire_True(t, ci.NumPending == uint64(nmsgs))\n\trequire_True(t, ci.NumRedelivered == keep)\n\n\t// Ack all.\n\tfor _, m := range msgs {\n\t\tm.Ack()\n\t}\n\tnc.Flush()\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\t// Same for Delivered\n\trequire_True(t, ci.Delivered.Consumer == uint64(nmsgs+keep))\n\trequire_True(t, ci.Delivered.Stream == uint64(nmsgs))\n\trequire_True(t, ci.AckFloor.Consumer == uint64(nmsgs+keep))\n\trequire_True(t, ci.AckFloor.Stream == uint64(nmsgs))\n\trequire_True(t, ci.NumAckPending == 0)\n\trequire_True(t, ci.NumPending == uint64(nmsgs))\n\trequire_True(t, ci.NumRedelivered == 0)\n\n\tmsgs, err = sub.Fetch(nmsgs)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == nmsgs)\n\n\t// Ack all again\n\tfor _, m := range msgs {\n\t\tm.Ack()\n\t}\n\tnc.Flush()\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\t// Make sure we calculated correctly.\n\trequire_True(t, ci.Delivered.Consumer == uint64(nmsgs*2+keep))\n\trequire_True(t, ci.Delivered.Stream == uint64(nmsgs*2))\n\trequire_True(t, ci.AckFloor.Consumer == uint64(nmsgs*2+keep))\n\trequire_True(t, ci.AckFloor.Stream == uint64(nmsgs*2))\n\trequire_True(t, ci.NumAckPending == 0)\n\trequire_True(t, ci.NumPending == 0)\n\trequire_True(t, ci.NumRedelivered == 0)\n}\n\nfunc TestJetStreamPurgeWithRedeliveredPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tnmsgs := 100\n\tfor i := 0; i < nmsgs; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"OK\")\n\t}\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\", nats.AckWait(time.Second))\n\trequire_NoError(t, err)\n\n\t// Queue up all for ack pending.\n\tmsgs, err := sub.Fetch(nmsgs)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == nmsgs)\n\n\t// Should be able to be redelivered now.\n\ttime.Sleep(2 * time.Second)\n\n\t// Queue up all for ack pending again.\n\tmsgs, err = sub.Fetch(nmsgs)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == nmsgs)\n\n\trequire_NoError(t, js.PurgeStream(\"TEST\"))\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\n\trequire_True(t, ci.Delivered.Consumer == uint64(2*nmsgs))\n\trequire_True(t, ci.Delivered.Stream == uint64(nmsgs))\n\trequire_True(t, ci.AckFloor.Consumer == uint64(2*nmsgs))\n\trequire_True(t, ci.AckFloor.Stream == uint64(nmsgs))\n\trequire_True(t, ci.NumAckPending == 0)\n\trequire_True(t, ci.NumPending == 0)\n\trequire_True(t, ci.NumRedelivered == 0)\n}\n\nfunc TestJetStreamStreamUpdateWithExternalSource(t *testing.T) {\n\tho := DefaultTestOptions\n\tho.Port = -1\n\tho.LeafNode.Host = \"127.0.0.1\"\n\tho.LeafNode.Port = -1\n\tho.JetStream = true\n\tho.JetStreamDomain = \"hub\"\n\tho.StoreDir = t.TempDir()\n\ths := RunServer(&ho)\n\tdefer hs.Shutdown()\n\n\tlu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ho.LeafNode.Port))\n\trequire_NoError(t, err)\n\n\tlo1 := DefaultTestOptions\n\tlo1.Port = -1\n\tlo1.ServerName = \"a-leaf\"\n\tlo1.JetStream = true\n\tlo1.StoreDir = t.TempDir()\n\tlo1.JetStreamDomain = \"a-leaf\"\n\tlo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lu}}}\n\tl1 := RunServer(&lo1)\n\tdefer l1.Shutdown()\n\n\tcheckLeafNodeConnected(t, l1)\n\n\t// Test sources with `External` provided\n\tncl, jsl := jsClientConnect(t, l1)\n\tdefer ncl.Close()\n\n\t// Hub stream.\n\t_, err = jsl.AddStream(&nats.StreamConfig{Name: \"stream\", Subjects: []string{\"leaf\"}})\n\trequire_NoError(t, err)\n\n\tnch, jsh := jsClientConnect(t, hs)\n\tdefer nch.Close()\n\n\t// Leaf stream.\n\t// Both streams uses the same name, as we're testing if overlap does not check against itself\n\t// if `External` stream has the same name.\n\t_, err = jsh.AddStream(&nats.StreamConfig{\n\t\tName: \"stream\",\n\t\tSubjects: []string{\"hub\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Add `Sources`.\n\t// This should not validate subjects overlap against itself.\n\t_, err = jsh.UpdateStream(&nats.StreamConfig{\n\t\tName: \"stream\",\n\t\tSubjects: []string{\"hub\"},\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"stream\",\n\t\t\t\tFilterSubject: \"leaf\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.a-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Specifying not existing FilterSubject should also be fine, as we do not validate `External` stream.\n\t_, err = jsh.UpdateStream(&nats.StreamConfig{\n\t\tName: \"stream\",\n\t\tSubjects: []string{\"hub\"},\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"stream\",\n\t\t\t\tFilterSubject: \"foo\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.a-leaf.API\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamKVHistoryRegression(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\tjs.DeleteKeyValue(\"TEST\")\n\n\t\t\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\t\t\tBucket: \"TEST\",\n\t\t\t\tHistory: 4,\n\t\t\t\tStorage: storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tr1, err := kv.Create(\"foo\", []byte(\"a\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = kv.Update(\"foo\", []byte(\"ab\"), r1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = kv.Delete(\"foo\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = kv.Create(\"foo\", []byte(\"abc\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = kv.Delete(\"foo\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\thistory, err := kv.History(\"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(history) == 4)\n\n\t\t\t_, err = kv.Update(\"foo\", []byte(\"abcd\"), history[len(history)-1].Revision())\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = kv.Purge(\"foo\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = kv.Create(\"foo\", []byte(\"abcde\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = kv.Purge(\"foo\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\thistory, err = kv.History(\"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(history) == 1)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSnapshotRestoreStallAndHealthz(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS\",\n\t\tSubjects: []string{\"orders.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"orders.created\", \"new order\")\n\t}\n\n\ths := s.healthz(nil)\n\tif hs.Status != \"ok\" || hs.Error != _EMPTY_ {\n\t\tt.Fatalf(\"Expected health to be ok, got %+v\", hs)\n\t}\n\n\t// Simulate the staging directory for restores. This is normally cleaned up\n\t// but since its at the root of the storage directory make sure healthz is not affected.\n\tsnapDir := filepath.Join(s.getJetStream().config.StoreDir, snapStagingDir)\n\trequire_NoError(t, os.MkdirAll(snapDir, defaultDirPerms))\n\n\t// Make sure healthz ok.\n\ths = s.healthz(nil)\n\tif hs.Status != \"ok\" || hs.Error != _EMPTY_ {\n\t\tt.Fatalf(\"Expected health to be ok, got %+v\", hs)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/pull/4163\nfunc TestJetStreamMaxBytesIgnored(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxBytes: 10 * 1024 * 1024,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := bytes.Repeat([]byte(\"A\"), 1024*1024)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"x\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == 9)\n\n\t// Stop current\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\t// We will truncate blk file.\n\tmdir := filepath.Join(sd, \"$G\", \"streams\", \"TEST\", \"msgs\")\n\t// Truncate blk\n\terr = os.WriteFile(filepath.Join(mdir, \"1.blk\"), nil, defaultFilePerms)\n\trequire_NoError(t, err)\n\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"x\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Bytes <= 10*1024*1024)\n}\n\nfunc TestJetStreamLastSequenceBySubjectConcurrent(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc0, js0 := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc0.Close()\n\n\t\t\tnc1, js1 := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc1.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 3,\n\t\t\t}\n\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tm, err := nc0.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tsi, err := js0.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v, respmsg: %q\", err, string(m.Data))\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\tpub := func(js nats.JetStreamContext, subj, data, seq string) {\n\t\t\t\tt.Helper()\n\t\t\t\tm := nats.NewMsg(subj)\n\t\t\t\tm.Data = []byte(data)\n\t\t\t\tm.Header.Set(JSExpectedLastSubjSeq, seq)\n\t\t\t\tjs.PublishMsg(m)\n\t\t\t}\n\n\t\t\tready := make(chan struct{})\n\t\t\twg := &sync.WaitGroup{}\n\t\t\twg.Add(2)\n\n\t\t\tgo func() {\n\t\t\t\t<-ready\n\t\t\t\tpub(js0, \"kv.foo\", \"0-0\", \"0\")\n\t\t\t\tpub(js0, \"kv.foo\", \"0-1\", \"1\")\n\t\t\t\tpub(js0, \"kv.foo\", \"0-2\", \"2\")\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\tgo func() {\n\t\t\t\t<-ready\n\t\t\t\tpub(js1, \"kv.foo\", \"1-0\", \"0\")\n\t\t\t\tpub(js1, \"kv.foo\", \"1-1\", \"1\")\n\t\t\t\tpub(js1, \"kv.foo\", \"1-2\", \"2\")\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tclose(ready)\n\t\t\twg.Wait()\n\n\t\t\t// Read the messages.\n\t\t\tsub, err := js0.PullSubscribe(_EMPTY_, _EMPTY_, nats.BindStream(\"KV\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tmsgs, err := sub.Fetch(10)\n\t\t\trequire_NoError(t, err)\n\t\t\tif len(msgs) != 3 {\n\t\t\t\tt.Errorf(\"Expected 3 messages, got %d\", len(msgs))\n\t\t\t}\n\t\t\tfor i, m := range msgs {\n\t\t\t\tif m.Header.Get(JSExpectedLastSubjSeq) != fmt.Sprint(i) {\n\t\t\t\t\tt.Errorf(\"Expected %d for last sequence, got %q\", i, m.Header.Get(JSExpectedLastSubjSeq))\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamServerReencryption(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tvar i int\n\tfor _, algo := range []struct {\n\t\tfrom string\n\t\tto string\n\t}{\n\t\t{\"aes\", \"aes\"},\n\t\t{\"aes\", \"chacha\"},\n\t\t{\"chacha\", \"chacha\"},\n\t\t{\"chacha\", \"aes\"},\n\t} {\n\t\tfor _, compression := range []StoreCompression{NoCompression, S2Compression} {\n\t\t\tt.Run(fmt.Sprintf(\"%s_to_%s/%s\", algo.from, algo.to, compression), func(t *testing.T) {\n\t\t\t\ti++\n\t\t\t\tstreamName := fmt.Sprintf(\"TEST_%d\", i)\n\t\t\t\tsubjectName := fmt.Sprintf(\"foo_%d\", i)\n\t\t\t\texpected := 30\n\n\t\t\t\tcheckStream := func(js nats.JetStreamContext) {\n\t\t\t\t\tsi, err := js.StreamInfo(streamName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif si.State.Msgs != uint64(expected) {\n\t\t\t\t\t\tt.Fatalf(\"Should be %d messages but got %d messages\", expected, si.State.Msgs)\n\t\t\t\t\t}\n\n\t\t\t\t\tsub, err := js.PullSubscribe(subjectName, \"\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tc := 0\n\t\t\t\t\tfor _, m := range fetchMsgs(t, sub, expected, 5*time.Second) {\n\t\t\t\t\t\tm.AckSync()\n\t\t\t\t\t\tc++\n\t\t\t\t\t}\n\t\t\t\t\tif c != expected {\n\t\t\t\t\t\tt.Fatalf(\"Should have read back %d messages but got %d messages\", expected, c)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// First off, we start up using the original encryption key and algorithm.\n\t\t\t\t// We'll create a stream and populate it with some messages.\n\t\t\t\tt.Run(\"setup\", func(t *testing.T) {\n\t\t\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\t\tserver_name: S22\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tjetstream: {\n\t\t\t\t\t\tkey: %q,\n\t\t\t\t\t\tcipher: %s,\n\t\t\t\t\t\tstore_dir: %q\n\t\t\t\t\t}\n\t\t\t\t`, \"firstencryptionkey\", algo.from, storeDir)))\n\n\t\t\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\tcfg := &StreamConfig{\n\t\t\t\t\t\tName: streamName,\n\t\t\t\t\t\tSubjects: []string{subjectName},\n\t\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t\t\tCompression: compression,\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := jsStreamCreate(t, nc, cfg); err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tpayload := strings.Repeat(\"A\", 512*1024)\n\t\t\t\t\tfor i := 0; i < expected; i++ {\n\t\t\t\t\t\tif _, err := js.Publish(subjectName, []byte(payload)); err != nil {\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tcheckStream(js)\n\t\t\t\t})\n\n\t\t\t\t// Next up, we will restart the server, this time with both the new key\n\t\t\t\t// and algorithm and also the old key. At startup, the server will detect\n\t\t\t\t// the change in encryption key and/or algorithm and re-encrypt the stream.\n\t\t\t\tt.Run(\"reencrypt\", func(t *testing.T) {\n\t\t\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\t\tserver_name: S22\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tjetstream: {\n\t\t\t\t\t\tkey: %q,\n\t\t\t\t\t\tcipher: %s,\n\t\t\t\t\t\tprev_key: %q,\n\t\t\t\t\t\tstore_dir: %q\n\t\t\t\t\t}\n\t\t\t\t`, \"secondencryptionkey\", algo.to, \"firstencryptionkey\", storeDir)))\n\n\t\t\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\tcheckStream(js)\n\t\t\t\t})\n\n\t\t\t\t// Finally, we'll restart the server using only the new key and algorithm.\n\t\t\t\t// At this point everything should have been re-encrypted, so we should still\n\t\t\t\t// be able to access the stream.\n\t\t\t\tt.Run(\"restart\", func(t *testing.T) {\n\t\t\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\t\tserver_name: S22\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tjetstream: {\n\t\t\t\t\t\tkey: %q,\n\t\t\t\t\t\tcipher: %s,\n\t\t\t\t\t\tstore_dir: %q\n\t\t\t\t\t}\n\t\t\t\t`, \"secondencryptionkey\", algo.to, storeDir)))\n\n\t\t\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\tcheckStream(js)\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJetStreamLimitsToInterestPolicy(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.leader())\n\tdefer nc.Close()\n\n\t// This is the index of the consumer that we'll create as R1\n\t// instead of R3, just to prove that it blocks the stream\n\t// update from happening properly.\n\tsingleReplica := 3\n\n\tstreamCfg := nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tStorage: nats.MemoryStorage,\n\t\tReplicas: 3,\n\t}\n\n\tstream, err := js.AddStream(&streamCfg)\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\treplicas := streamCfg.Replicas\n\t\tif i == singleReplica {\n\t\t\t// Make one of the consumers R1 so that we can check\n\t\t\t// that the switch to interest-based retention is also\n\t\t\t// turning it into an R3 consumer.\n\t\t\treplicas = 1\n\t\t}\n\t\tcname := fmt.Sprintf(\"test_%d\", i)\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tName: cname,\n\t\t\tDurable: cname,\n\t\t\tAckPolicy: nats.AckAllPolicy,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 20; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte{1, 2, 3, 4, 5})\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Pull 10 or more messages from the stream. We will never pull\n\t// less than 10, which guarantees that the lowest ack floor of\n\t// all consumers should be 10.\n\tfor i := 0; i < 10; i++ {\n\t\tcname := fmt.Sprintf(\"test_%d\", i)\n\t\tcount := 10 + i // At least 10 messages\n\n\t\tsub, err := js.PullSubscribe(\"foo\", cname)\n\t\trequire_NoError(t, err)\n\n\t\tmsgs, err := sub.Fetch(count)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(msgs), count)\n\t\trequire_NoError(t, msgs[len(msgs)-1].AckSync())\n\n\t\t// At this point the ack floor should match the count of\n\t\t// messages we received.\n\t\tinfo, err := js.ConsumerInfo(\"TEST\", cname)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, info.AckFloor.Consumer, uint64(count))\n\t}\n\n\t// Try updating to interest-based. This should fail because\n\t// we have a consumer that is R1 on an R3 stream.\n\tstreamCfg = stream.Config\n\tstreamCfg.Retention = nats.InterestPolicy\n\t_, err = js.UpdateStream(&streamCfg)\n\trequire_Error(t, err)\n\n\t// Now we'll make the R1 consumer an R3.\n\tcname := fmt.Sprintf(\"test_%d\", singleReplica)\n\tcinfo, err := js.ConsumerInfo(\"TEST\", cname)\n\trequire_NoError(t, err)\n\n\tcinfo.Config.Replicas = streamCfg.Replicas\n\t_, _ = js.UpdateConsumer(\"TEST\", &cinfo.Config)\n\t// TODO(nat): The jsConsumerCreateRequest update doesn't always\n\t// respond when there are no errors updating a consumer, so this\n\t// nearly always returns a timeout, despite actually doing what\n\t// it should. We'll make sure the replicas were updated by doing\n\t// another consumer info just to be sure.\n\t// require_NoError(t, err)\n\tc.waitOnAllCurrent()\n\tc.waitOnConsumerLeader(globalAccountName, \"TEST\", cname)\n\tcinfo, err = js.ConsumerInfo(\"TEST\", cname)\n\trequire_NoError(t, err)\n\trequire_Equal(t, cinfo.Config.Replicas, streamCfg.Replicas)\n\trequire_Equal(t, len(cinfo.Cluster.Replicas), streamCfg.Replicas-1)\n\n\t// This time it should succeed.\n\t_, err = js.UpdateStream(&streamCfg)\n\trequire_NoError(t, err)\n\n\t// We need to wait for all nodes to have applied the new stream\n\t// configuration.\n\tc.waitOnAllCurrent()\n\n\t// Now we should only have 10 messages left in the stream, as\n\t// each consumer has acked at least the first 10 messages.\n\tinfo, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, info.State.FirstSeq, 11)\n\trequire_Equal(t, info.State.Msgs, 10)\n}\n\nfunc TestJetStreamLimitsToInterestPolicyWhileAcking(t *testing.T) {\n\tfor _, st := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.leader())\n\t\t\tdefer nc.Close()\n\t\t\tstreamCfg := nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tRetention: nats.LimitsPolicy,\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 3,\n\t\t\t}\n\n\t\t\tstream, err := js.AddStream(&streamCfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\tctx, cancel := context.WithCancel(context.Background())\n\t\t\tpayload := []byte(strings.Repeat(\"A\", 128))\n\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tfor range time.NewTicker(10 * time.Millisecond).C {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t\tjs.Publish(\"foo\", payload)\n\t\t\t\t}\n\t\t\t}()\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\tcname := fmt.Sprintf(\"test_%d\", i)\n\t\t\t\tsub, err := js.PullSubscribe(\"foo\", cname)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tfor range time.NewTicker(10 * time.Millisecond).C {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\t\tif err != nil && errors.Is(err, nats.ErrTimeout) {\n\t\t\t\t\t\t\tt.Logf(\"ERROR: %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfor _, msg := range msgs {\n\t\t\t\t\t\t\tmsg.Ack()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\t\t\t// Leave running for a few secs then do the change on a different connection.\n\t\t\ttime.Sleep(5 * time.Second)\n\t\t\tnc2, js2 := jsClientConnect(t, c.leader())\n\t\t\tdefer nc2.Close()\n\n\t\t\t// Try updating to interest-based and changing replicas too.\n\t\t\tstreamCfg = stream.Config\n\t\t\tstreamCfg.Retention = nats.InterestPolicy\n\t\t\t_, err = js2.UpdateStream(&streamCfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// We need to wait for all nodes to have applied the new stream\n\t\t\t// configuration.\n\t\t\tc.waitOnAllCurrent()\n\n\t\t\tvar retention nats.RetentionPolicy\n\t\t\tcheckFor(t, 15*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\tinfo, err := js2.StreamInfo(\"TEST\", nats.MaxWait(500*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tretention = info.Config.Retention\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\trequire_Equal(t, retention, nats.InterestPolicy)\n\n\t\t\t// Cancel and wait for goroutines underneath.\n\t\t\tcancel()\n\t\t\twg.Wait()\n\t\t})\n\t}\n}\n\nfunc TestJetStreamUsageSyncDeadlock(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsendStreamMsg(t, nc, \"foo\", \"hello\")\n\n\t// Now purposely mess up the usage that will force a sync.\n\t// Without the fix this will deadlock.\n\tjsa := s.getJetStream().lookupAccount(s.GlobalAccount())\n\tjsa.usageMu.Lock()\n\tst, ok := jsa.usage[_EMPTY_]\n\trequire_True(t, ok)\n\tst.local.store = -1000\n\tjsa.usageMu.Unlock()\n\n\tsendStreamMsg(t, nc, \"foo\", \"hello\")\n}\n\n// https://github.com/nats-io/nats.go/issues/1382\n// https://github.com/nats-io/nats-server/issues/4445\nfunc TestJetStreamChangeMaxMessagesPerSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"one.>\"},\n\t\tMaxMsgsPerSubject: 5,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"one.data\", \"data\")\n\t}\n\n\texpectMsgs := func(num int32) error {\n\t\tt.Helper()\n\n\t\tvar msgs atomic.Int32\n\t\tsub, err := js.Subscribe(\"one.>\", func(msg *nats.Msg) {\n\t\t\tmsgs.Add(1)\n\t\t\tmsg.Ack()\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tif nm := msgs.Load(); nm != num {\n\t\t\t\treturn fmt.Errorf(\"expected to get %v messages, got %v instead\", num, nm)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn nil\n\t}\n\n\trequire_NoError(t, expectMsgs(5))\n\n\tjs.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"one.>\"},\n\t\tMaxMsgsPerSubject: 3,\n\t})\n\n\tinfo, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, info.Config.MaxMsgsPerSubject == 3)\n\trequire_True(t, info.State.Msgs == 3)\n\n\trequire_NoError(t, expectMsgs(3))\n\n\tfor i := 0; i < 10; i++ {\n\t\tsendStreamMsg(t, nc, \"one.data\", \"data\")\n\t}\n\n\trequire_NoError(t, expectMsgs(3))\n}\n\nfunc TestJetStreamSyncInterval(t *testing.T) {\n\tsd := t.TempDir()\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {\n\t\t\tstore_dir: %q\n\t\t\t%s\n\t\t}`\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsync string\n\t\texpected time.Duration\n\t\talways bool\n\t}{\n\t\t{\"Default\", _EMPTY_, defaultSyncInterval, false},\n\t\t{\"10s\", \"sync_interval: 10s\", time.Duration(10 * time.Second), false},\n\t\t{\"Always\", \"sync_interval: always\", defaultSyncInterval, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, sd, test.sync)))\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\topts := s.getOpts()\n\t\t\trequire_True(t, opts.SyncInterval == test.expected)\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"test.>\"},\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tfs := mset.store.(*fileStore)\n\t\t\tfs.mu.RLock()\n\t\t\tfsSync := fs.fcfg.SyncInterval\n\t\t\tsyncAlways := fs.fcfg.SyncAlways\n\t\t\tfs.mu.RUnlock()\n\t\t\trequire_True(t, fsSync == test.expected)\n\t\t\trequire_True(t, syncAlways == test.always)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamFilteredSubjectUsesNewConsumerCreateSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\textEndpoint := make(chan *nats.Msg, 1)\n\tnormalEndpoint := make(chan *nats.Msg, 1)\n\n\t_, err := nc.ChanSubscribe(JSApiConsumerCreateEx, extEndpoint)\n\trequire_NoError(t, err)\n\n\t_, err = nc.ChanSubscribe(JSApiConsumerCreate, normalEndpoint)\n\trequire_NoError(t, err)\n\n\ttestStreamSource := func(name string, shouldBeExtended bool, ss StreamSource) {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\treq := StreamConfig{\n\t\t\t\tName: name,\n\t\t\t\tStorage: MemoryStorage,\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%s\", name)},\n\t\t\t\tSources: []*StreamSource{&ss},\n\t\t\t}\n\t\t\treqJson, err := json.Marshal(req)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = nc.Request(fmt.Sprintf(JSApiStreamCreateT, name), reqJson, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tselect {\n\t\t\tcase <-time.After(time.Second * 5):\n\t\t\t\tt.Fatalf(\"Timed out waiting for receive consumer create\")\n\t\t\tcase <-extEndpoint:\n\t\t\t\tif !shouldBeExtended {\n\t\t\t\t\tt.Fatalf(\"Expected normal consumer create, got extended\")\n\t\t\t\t}\n\t\t\tcase <-normalEndpoint:\n\t\t\t\tif shouldBeExtended {\n\t\t\t\t\tt.Fatalf(\"Expected extended consumer create, got normal\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\ttestStreamSource(\"OneFilterSubject\", true, StreamSource{\n\t\tName: \"source\",\n\t\tFilterSubject: \"bar.>\",\n\t})\n\n\ttestStreamSource(\"OneTransform\", true, StreamSource{\n\t\tName: \"source\",\n\t\tSubjectTransforms: []SubjectTransformConfig{\n\t\t\t{\n\t\t\t\tSource: \"bar.one.>\",\n\t\t\t\tDestination: \"bar.two.>\",\n\t\t\t},\n\t\t},\n\t})\n\n\ttestStreamSource(\"TwoTransforms\", false, StreamSource{\n\t\tName: \"source\",\n\t\tSubjectTransforms: []SubjectTransformConfig{\n\t\t\t{\n\t\t\t\tSource: \"bar.one.>\",\n\t\t\t\tDestination: \"bar.two.>\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: \"baz.one.>\",\n\t\t\t\tDestination: \"baz.two.>\",\n\t\t\t},\n\t\t},\n\t})\n}\n\n// Make sure when we downgrade history to a smaller number that the account info\n// is properly updated and all keys are still accessible.\n// There was a bug calculating next first that was not taking into account the dbit slots.\nfunc TestJetStreamKVReductionInHistory(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tstartHistory := 4\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"TEST\", History: uint8(startHistory)})\n\trequire_NoError(t, err)\n\n\tnumKeys, msg := 1000, bytes.Repeat([]byte(\"ABC\"), 330) // ~1000bytes\n\tfor {\n\t\tkey := fmt.Sprintf(\"%X\", rand.Intn(numKeys)+1)\n\t\t_, err = kv.Put(key, msg)\n\t\trequire_NoError(t, err)\n\t\tstatus, err := kv.Status()\n\t\trequire_NoError(t, err)\n\t\tif status.Values() >= uint64(startHistory*numKeys) {\n\t\t\tbreak\n\t\t}\n\t}\n\tinfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\n\tcheckAllKeys := func() {\n\t\tt.Helper()\n\t\t// Make sure we can retrieve all of the keys.\n\t\tkeys, err := kv.Keys()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, len(keys), numKeys)\n\t\tfor _, key := range keys {\n\t\t\t_, err := kv.Get(key)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\t// Quick sanity check.\n\tcheckAllKeys()\n\n\tsi, err := js.StreamInfo(\"KV_TEST\")\n\trequire_NoError(t, err)\n\t// Adjust down to history of 1.\n\tcfg := si.Config\n\tcfg.MaxMsgsPerSubject = 1\n\t_, err = js.UpdateStream(&cfg)\n\trequire_NoError(t, err)\n\t// Make sure the accounting was updated.\n\tninfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, info.Store > ninfo.Store)\n\n\t// Make sure all keys still accessible.\n\tcheckAllKeys()\n}\n\nfunc TestJetStreamDirectGetBatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Add in messages\n\tfor i := 0; i < 333; i++ {\n\t\tjs.PublishAsync(\"foo.foo\", []byte(\"HELLO\"))\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"WORLD\"))\n\t\tjs.PublishAsync(\"foo.baz\", []byte(\"AGAIN\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// DirectGet is required for batch. Make sure we error correctly if not enabled.\n\tmreq := &JSApiMsgGetRequest{Seq: 1, Batch: 10}\n\treq, _ := json.Marshal(mreq)\n\trr, err := nc.Request(\"$JS.API.STREAM.MSG.GET.TEST\", req, time.Second)\n\trequire_NoError(t, err)\n\tvar resp JSApiMsgGetResponse\n\tjson.Unmarshal(rr.Data, &resp)\n\trequire_True(t, resp.Error != nil)\n\trequire_Equal(t, resp.Error.Code, NewJSBadRequestError().Code)\n\n\t// Update stream to support direct.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// Direct subjects.\n\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\t// We will get multiple responses so can't do normal request.\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req)\n\t\trequire_NoError(t, err)\n\t\treturn sub\n\t}\n\n\t// Batch sizes greater than 1 will have a nil message as the end marker.\n\tcheckResponses := func(sub *nats.Subscription, numPendingStart int, expected ...string) {\n\t\tt.Helper()\n\t\tdefer sub.Unsubscribe()\n\t\tcheckSubsPending(t, sub, len(expected))\n\t\tnp := numPendingStart\n\t\tlast := \"0\"\n\t\tfor i := 0; i < len(expected); i++ {\n\t\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\t// If expected is _EMPTY_ that signals we expect a EOB marker.\n\t\t\tif subj := expected[i]; subj != _EMPTY_ {\n\t\t\t\t// Make sure subject is correct.\n\t\t\t\trequire_Equal(t, expected[i], msg.Header.Get(JSSubject))\n\t\t\t\t// Should have Data field non-zero\n\t\t\t\trequire_True(t, len(msg.Data) > 0)\n\t\t\t\t// Check we have NumPending and it's correct.\n\t\t\t\tif np > 0 {\n\t\t\t\t\tnp--\n\t\t\t\t}\n\t\t\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t\t\trequire_Equal(t, last, msg.Header.Get(JSLastSequence))\n\t\t\t\tlast = msg.Header.Get(JSSequence)\n\t\t\t} else {\n\t\t\t\t// Check for properly formatted EOB marker.\n\t\t\t\t// Should have no body.\n\t\t\t\trequire_Equal(t, len(msg.Data), 0)\n\t\t\t\t// We mark status as 204 - No Content\n\t\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"204\")\n\t\t\t\t// Check description is EOB\n\t\t\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"EOB\")\n\t\t\t\t// Check we have NumPending and it's correct.\n\t\t\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t\t\trequire_Equal(t, last, msg.Header.Get(JSLastSequence))\n\t\t\t}\n\t\t}\n\t}\n\n\t// Run some simple tests.\n\tsub := sendRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 2})\n\tcheckResponses(sub, 999, \"foo.foo\", \"foo.bar\", _EMPTY_)\n\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 3})\n\tcheckResponses(sub, 999, \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\t// Test NextFor works\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 3, NextFor: \"foo.*\"})\n\tcheckResponses(sub, 999, \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 3, NextFor: \"foo.baz\"})\n\tcheckResponses(sub, 333, \"foo.baz\", \"foo.baz\", \"foo.baz\", _EMPTY_)\n\n\t// Test stopping early by starting at 997 with only 3 messages.\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 997, Batch: 10, NextFor: \"foo.*\"})\n\tcheckResponses(sub, 3, \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Add in messages\n\tjs.PublishAsync(\"foo.foo\", []byte(\"HELLO\"))\n\tjs.PublishAsync(\"foo.bar\", []byte(\"WORLD\"))\n\tjs.PublishAsync(\"foo.baz\", []byte(\"AGAIN\"))\n\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Verify that a batch get request starting at a deleted sequence number skips over the deleted messages.\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 10})\n\tcheckResponses(sub, 3, \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\t// verify that a non-batch direct get to a deleted message returns not found.\n\t_, err = js.GetMsg(\"TEST\", 1, nats.DirectGet())\n\trequire_Error(t, err, fmt.Errorf(\"nats: message not found\"))\n\n}\n\nfunc TestJetStreamDirectGetBatchMaxBytes(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t\tCompression: nats.S2Compression,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 512*1024)\n\t// Add in messages\n\tfor i := 0; i < 333; i++ {\n\t\tjs.PublishAsync(\"foo.foo\", msg)\n\t\tjs.PublishAsync(\"foo.bar\", msg)\n\t\tjs.PublishAsync(\"foo.baz\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsendRequestAndCheck := func(mreq *JSApiMsgGetRequest, numExpected int) {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\t// We will get multiple responses so can't do normal request.\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\t\terr = nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req)\n\t\trequire_NoError(t, err)\n\t\t// Make sure we get correct number of responses.\n\t\tcheckSubsPending(t, sub, numExpected)\n\t}\n\n\t// Total msg size being sent back to us.\n\tmsgSize := len(msg) + len(\"foo.foo\")\n\t// We should get 1 msg and 1 EOB\n\tsendRequestAndCheck(&JSApiMsgGetRequest{Seq: 1, Batch: 3, MaxBytes: msgSize}, 2)\n\n\t// Test NextFor tracks as well.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{Seq: 1, NextFor: \"foo.bar\", Batch: 3, MaxBytes: 2 * msgSize}, 3)\n\n\t// Now test no MaxBytes to inherit server max_num_pending.\n\texpected := (int(s.getOpts().MaxPending) / msgSize) + 1\n\tsendRequestAndCheck(&JSApiMsgGetRequest{Seq: 1, Batch: 200}, expected+1)\n}\n\nfunc TestJetStreamMsgGetAsOfTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\tsendRequestAndCheck := func(mreq *JSApiMsgGetRequest, seq uint64, eerr error) {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\trep, err := nc.Request(fmt.Sprintf(JSApiMsgGetT, \"TEST\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar mrep JSApiMsgGetResponse\n\t\terr = json.Unmarshal(rep.Data, &mrep)\n\t\trequire_NoError(t, err)\n\t\tif eerr != nil {\n\t\t\trequire_Error(t, mrep.ToError(), eerr)\n\t\t\treturn\n\t\t}\n\t\trequire_NoError(t, mrep.ToError())\n\t\trequire_Equal(t, seq, mrep.Message.Sequence)\n\t}\n\tt0 := time.Now()\n\n\t// Check for conflicting options.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, LastFor: \"foo.1\"}, 0, NewJSBadRequestError())\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, Seq: 1}, 0, NewJSBadRequestError())\n\n\t// Nothing exists yet in the stream.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 0, NewJSNoMessageFoundError())\n\n\t_, err = js.Publish(\"foo.1\", nil)\n\trequire_NoError(t, err)\n\n\t// Try again with t0 and now it will find the first message.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 1, nil)\n\n\tt1 := time.Now()\n\t_, err = js.Publish(\"foo.2\", nil)\n\trequire_NoError(t, err)\n\n\t// At t0, first message will still be returned first.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 1, nil)\n\t// Unless we combine with NextFor...\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, NextFor: \"foo.2\"}, 2, nil)\n\n\t// At t1 (after first message), the second message will be returned.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t1}, 2, nil)\n\n\tt2 := time.Now()\n\t// t2 is later than the last message so nothing will be found.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t2}, 0, NewJSNoMessageFoundError())\n}\n\nfunc TestJetStreamMsgDirectGetAsOfTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\tsendRequestAndCheck := func(mreq *JSApiMsgGetRequest, seq uint64, eerr string) {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\trep, err := nc.Request(fmt.Sprintf(JSDirectMsgGetT, \"TEST\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tif eerr != \"\" {\n\t\t\trequire_Equal(t, rep.Header.Get(\"Description\"), eerr)\n\t\t\treturn\n\t\t}\n\n\t\tmseq, err := strconv.ParseUint(rep.Header.Get(\"Nats-Sequence\"), 10, 64)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, seq, mseq)\n\t}\n\tt0 := time.Now()\n\n\t// Check for conflicting options.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, LastFor: \"foo.1\"}, 0, \"Bad Request\")\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, Seq: 1}, 0, \"Bad Request\")\n\n\t// Nothing exists yet in the stream.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 0, \"Message Not Found\")\n\n\t_, err = js.Publish(\"foo.1\", nil)\n\trequire_NoError(t, err)\n\n\t// Try again with t0 and now it will find the first message.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 1, \"\")\n\n\tt1 := time.Now()\n\t_, err = js.Publish(\"foo.2\", nil)\n\trequire_NoError(t, err)\n\n\t// At t0, first message will still be returned first.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0}, 1, \"\")\n\t// Unless we combine with NextFor..\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t0, NextFor: \"foo.2\"}, 2, \"\")\n\n\t// At t1 (after first message), the second message will be returned.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t1}, 2, \"\")\n\n\tt2 := time.Now()\n\t// t2 is later than the last message so nothing will be found.\n\tsendRequestAndCheck(&JSApiMsgGetRequest{StartTime: &t2}, 0, \"Message Not Found\")\n}\n\nfunc TestJetStreamSubjectFilteredPurgeClearsPendingAcks(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 5; i++ {\n\t\tjs.Publish(\"foo\", []byte(\"OK\"))\n\t\tjs.Publish(\"bar\", []byte(\"OK\"))\n\t}\n\n\t// Note that there are no subject filters here, this is deliberate\n\t// as previously the purge with filter code was checking for them.\n\t// We want to prove that unfiltered consumers also get purged.\n\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"my_consumer\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: 10,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 10)\n\trequire_Equal(t, ci.NumAckPending, 0)\n\n\tsub, err := js.PullSubscribe(\">\", \"\", nats.Bind(\"TEST\", \"my_consumer\"))\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 10)\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"my_consumer\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 0)\n\trequire_Equal(t, ci.NumAckPending, 10)\n\n\trequire_NoError(t, js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{\n\t\tSubject: \"foo\",\n\t}))\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"my_consumer\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 0)\n\trequire_Equal(t, ci.NumAckPending, 5)\n\n\tfor i := 0; i < 5; i++ {\n\t\tjs.Publish(\"foo\", []byte(\"OK\"))\n\t}\n\tmsgs, err = sub.Fetch(5)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 5)\n\n\tci, err = js.ConsumerInfo(\"TEST\", \"my_consumer\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 0)\n\trequire_Equal(t, ci.NumAckPending, 10)\n}\n\n// Helper function for TestJetStreamConsumerPause*, TestJetStreamClusterConsumerPause*, TestJetStreamSuperClusterConsumerPause*\nfunc jsTestPause_CreateOrUpdateConsumer(t *testing.T, nc *nats.Conn, action ConsumerAction, stream string, cc ConsumerConfig) *JSApiConsumerCreateResponse {\n\tt.Helper()\n\tj, err := json.Marshal(CreateConsumerRequest{\n\t\tStream: stream,\n\t\tConfig: cc,\n\t\tAction: action,\n\t})\n\trequire_NoError(t, err)\n\tsubj := fmt.Sprintf(\"$JS.API.CONSUMER.CREATE.%s.%s\", stream, cc.Name)\n\tm, err := nc.Request(subj, j, time.Second*3)\n\trequire_NoError(t, err)\n\tvar res JSApiConsumerCreateResponse\n\trequire_NoError(t, json.Unmarshal(m.Data, &res))\n\trequire_True(t, res.Config != nil)\n\treturn &res\n}\n\n// Helper function for TestJetStreamConsumerPause*, TestJetStreamClusterConsumerPause*, TestJetStreamSuperClusterConsumerPause*\nfunc jsTestPause_PauseConsumer(t *testing.T, nc *nats.Conn, stream, consumer string, deadline time.Time) time.Time {\n\tt.Helper()\n\tj, err := json.Marshal(JSApiConsumerPauseRequest{\n\t\tPauseUntil: deadline,\n\t})\n\trequire_NoError(t, err)\n\tsubj := fmt.Sprintf(\"$JS.API.CONSUMER.PAUSE.%s.%s\", stream, consumer)\n\tmsg, err := nc.Request(subj, j, time.Second)\n\trequire_NoError(t, err)\n\tvar res JSApiConsumerPauseResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &res))\n\treturn res.PauseUntil\n}\n\nfunc TestJetStreamDirectGetMulti(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcfg *nats.StreamConfig\n\t}{\n\t\t{name: \"MemoryStore\",\n\t\t\tcfg: &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\tAllowDirect: true,\n\t\t\t\tStorage: nats.MemoryStorage,\n\t\t\t}},\n\t\t{name: \"FileStore\",\n\t\t\tcfg: &nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\t\tAllowDirect: true,\n\t\t\t}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(c.cfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Add in messages\n\t\t\tfor i := 0; i < 33; i++ {\n\t\t\t\tjs.PublishAsync(\"foo.foo\", []byte(fmt.Sprintf(\"HELLO-%d\", i)))\n\t\t\t\tjs.PublishAsync(\"foo.bar\", []byte(fmt.Sprintf(\"WORLD-%d\", i)))\n\t\t\t\tjs.PublishAsync(\"foo.baz\", []byte(fmt.Sprintf(\"AGAIN-%d\", i)))\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Direct subjects.\n\t\t\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\t\t\tt.Helper()\n\t\t\t\treq, _ := json.Marshal(mreq)\n\t\t\t\t// We will get multiple responses so can't do normal request.\n\t\t\t\treply := nats.NewInbox()\n\t\t\t\tsub, err := nc.SubscribeSync(reply)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\terr = nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\treturn sub\n\t\t\t}\n\n\t\t\t// Subject / Sequence pair\n\t\t\ttype p struct {\n\t\t\t\tsubj string\n\t\t\t\tseq int\n\t\t\t}\n\t\t\tvar eob p\n\n\t\t\t// Multi-Get will have a nil message as the end marker regardless.\n\t\t\tcheckResponses := func(sub *nats.Subscription, numPendingStart int, expected ...p) {\n\t\t\t\tt.Helper()\n\t\t\t\tlast := \"0\"\n\t\t\t\tdefer sub.Unsubscribe()\n\t\t\t\tcheckSubsPending(t, sub, len(expected))\n\t\t\t\tnp := numPendingStart\n\t\t\t\tfor i := 0; i < len(expected); i++ {\n\t\t\t\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t// If expected is _EMPTY_ that signals we expect a EOB marker.\n\t\t\t\t\tif subj := expected[i].subj; subj != _EMPTY_ {\n\t\t\t\t\t\t// Make sure subject is correct.\n\t\t\t\t\t\trequire_Equal(t, subj, msg.Header.Get(JSSubject))\n\t\t\t\t\t\t// Make sure sequence is correct.\n\t\t\t\t\t\trequire_Equal(t, strconv.Itoa(expected[i].seq), msg.Header.Get(JSSequence))\n\t\t\t\t\t\t// Should have Data field non-zero\n\t\t\t\t\t\trequire_True(t, len(msg.Data) > 0)\n\t\t\t\t\t\t// Check we have NumPending and it's correct.\n\t\t\t\t\t\tif np > 0 {\n\t\t\t\t\t\t\tnp--\n\t\t\t\t\t\t}\n\t\t\t\t\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t\t\t\t\trequire_Equal(t, last, msg.Header.Get(JSLastSequence))\n\t\t\t\t\t\tlast = msg.Header.Get(JSSequence)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Check for properly formatted EOB marker.\n\t\t\t\t\t\t// Should have no body.\n\t\t\t\t\t\trequire_Equal(t, len(msg.Data), 0)\n\t\t\t\t\t\t// We mark status as 204 - No Content\n\t\t\t\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"204\")\n\t\t\t\t\t\t// Check description is EOB\n\t\t\t\t\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"EOB\")\n\t\t\t\t\t\t// Check we have NumPending and it's correct.\n\t\t\t\t\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t\t\t\t\trequire_Equal(t, last, msg.Header.Get(JSLastSequence))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsub := sendRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"foo.*\"}})\n\t\t\tcheckResponses(sub, 3, p{\"foo.foo\", 97}, p{\"foo.bar\", 98}, p{\"foo.baz\", 99}, eob)\n\t\t\t// Check with UpToSeq\n\t\t\tsub = sendRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"foo.*\"}, UpToSeq: 3})\n\t\t\tcheckResponses(sub, 3, p{\"foo.foo\", 1}, p{\"foo.bar\", 2}, p{\"foo.baz\", 3}, eob)\n\t\t\t// check last header sequence number is correct\n\t\t\tsub = sendRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"foo.foo\", \"foo.baz\"}})\n\t\t\tcheckResponses(sub, 2, p{\"foo.foo\", 97}, p{\"foo.baz\", 99}, eob)\n\t\t\t// Test No Results.\n\t\t\tsub = sendRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"bar.*\"}})\n\t\t\tcheckSubsPending(t, sub, 1)\n\t\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\t// Check for properly formatted No Results.\n\t\t\t// Should have no body.\n\t\t\trequire_Equal(t, len(msg.Data), 0)\n\t\t\t// We mark status as 204 - No Content\n\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\t\t\t// Check description is No Results\n\t\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"No Results\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamDirectGetMultiUpToTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\tjs.Publish(\"foo.foo\", []byte(\"1\"))\n\tjs.Publish(\"foo.bar\", []byte(\"1\"))\n\tjs.Publish(\"foo.baz\", []byte(\"1\"))\n\tstart := time.Now()\n\ttime.Sleep(time.Second)\n\tjs.Publish(\"foo.foo\", []byte(\"2\"))\n\tjs.Publish(\"foo.bar\", []byte(\"2\"))\n\tjs.Publish(\"foo.baz\", []byte(\"2\"))\n\tmid := time.Now()\n\ttime.Sleep(time.Second)\n\tjs.Publish(\"foo.foo\", []byte(\"3\"))\n\tjs.Publish(\"foo.bar\", []byte(\"3\"))\n\tjs.Publish(\"foo.baz\", []byte(\"3\"))\n\tend := time.Now()\n\n\t// Direct subjects.\n\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\t// We will get multiple responses so can't do normal request.\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req)\n\t\trequire_NoError(t, err)\n\t\treturn sub\n\t}\n\n\tcheckResponses := func(sub *nats.Subscription, val string, expected ...string) {\n\t\tt.Helper()\n\t\tdefer sub.Unsubscribe()\n\t\tcheckSubsPending(t, sub, len(expected))\n\t\tfor i := 0; i < len(expected); i++ {\n\t\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\t// If expected is _EMPTY_ that signals we expect a EOB marker.\n\t\t\tif subj := expected[i]; subj != _EMPTY_ {\n\t\t\t\t// Make sure subject is correct.\n\t\t\t\trequire_Equal(t, subj, msg.Header.Get(JSSubject))\n\t\t\t\t// Should have Data field non-zero\n\t\t\t\trequire_True(t, len(msg.Data) > 0)\n\t\t\t\t// Make sure the value matches.\n\t\t\t\trequire_Equal(t, string(msg.Data), val)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Make sure you can't set both.\n\tsub := sendRequest(&JSApiMsgGetRequest{Seq: 1, MultiLastFor: []string{\"foo.*\"}, UpToSeq: 3, UpToTime: &start})\n\tcheckSubsPending(t, sub, 1)\n\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\trequire_NoError(t, err)\n\t// Check for properly formatted No Results.\n\t// Should have no body.\n\trequire_Equal(t, len(msg.Data), 0)\n\t// We mark status as 204 - No Content\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"408\")\n\t// Check description is No Results\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Bad Request\")\n\n\t// Valid responses.\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, MultiLastFor: []string{\"foo.*\"}, UpToTime: &start})\n\tcheckResponses(sub, \"1\", \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, MultiLastFor: []string{\"foo.*\"}, UpToTime: &mid})\n\tcheckResponses(sub, \"2\", \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n\n\tsub = sendRequest(&JSApiMsgGetRequest{Seq: 1, MultiLastFor: []string{\"foo.*\"}, UpToTime: &end})\n\tcheckResponses(sub, \"3\", \"foo.foo\", \"foo.bar\", \"foo.baz\", _EMPTY_)\n}\n\nfunc TestJetStreamDirectGetMultiMaxAllowed(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// from stream.go - const maxAllowedResponses = 1024, so max sure > 1024\n\t// Add in messages\n\tfor i := 1; i <= 1025; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"foo.%d\", i), []byte(\"OK\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\treq, _ := json.Marshal(&JSApiMsgGetRequest{Seq: 1, MultiLastFor: []string{\"foo.*\"}})\n\tmsg, err := nc.Request(\"$JS.API.DIRECT.GET.TEST\", req, time.Second)\n\trequire_NoError(t, err)\n\n\t// Check for properly formatted Too Many Results error.\n\t// Should have no body.\n\trequire_Equal(t, len(msg.Data), 0)\n\t// We mark status as 413 - Too Many Results\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"413\")\n\t// Check description is No Results\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Too Many Results\")\n}\n\nfunc TestJetStreamDirectGetMultiPaging(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// We will queue up 500 messages, each 512k big and request them for a multi-get.\n\t// This will not hit the max allowed limit of 1024, but will bump up against max bytes and only return partial results.\n\t// We want to make sure we can pick up where we left off.\n\n\t// Add in messages\n\tdata, sent := bytes.Repeat([]byte(\"Z\"), 512*1024), 500\n\tfor i := 1; i <= sent; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"foo.%d\", i), data)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\t// Wait for all replicas to be correct.\n\ttime.Sleep(time.Second)\n\n\t// Direct subjects.\n\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\tt.Helper()\n\t\treq, _ := json.Marshal(mreq)\n\t\t// We will get multiple responses so can't do normal request.\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\terr = nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req)\n\t\trequire_NoError(t, err)\n\t\treturn sub\n\t}\n\n\t// Setup variables that control procesPartial\n\tstart, seq, np, b, bsz := 1, 1, sent, 0, 128\n\n\tprocessPartial := func(expected int) {\n\t\tt.Helper()\n\t\tsub := sendRequest(&JSApiMsgGetRequest{Seq: uint64(start), Batch: b, MultiLastFor: []string{\"foo.*\"}})\n\t\tcheckSubsPending(t, sub, expected)\n\t\t// Check partial.\n\t\t// We should receive seqs seq-(seq+bsz-1)\n\t\tfor ; seq < start+(expected-1); seq++ {\n\t\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\t// Make sure sequence is correct.\n\t\t\trequire_Equal(t, strconv.Itoa(int(seq)), msg.Header.Get(JSSequence))\n\t\t\t// Check we have NumPending and it's correct.\n\t\t\tif np > 0 {\n\t\t\t\tnp--\n\t\t\t}\n\t\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t}\n\t\t// Now check EOB\n\t\tmsg, err := sub.NextMsg(10 * time.Millisecond)\n\t\trequire_NoError(t, err)\n\t\t// We mark status as 204 - No Content\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"204\")\n\t\t// Check description is EOB\n\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"EOB\")\n\t\t// Check we have NumPending and it's correct.\n\t\trequire_Equal(t, strconv.Itoa(np), msg.Header.Get(JSNumPending))\n\t\t// Check we have LastSequence and it's correct.\n\t\trequire_Equal(t, strconv.Itoa(seq-1), msg.Header.Get(JSLastSequence))\n\t\t// Check we have UpToSequence and it's correct.\n\t\trequire_Equal(t, strconv.Itoa(sent), msg.Header.Get(JSUpToSequence))\n\t\t// Update start\n\t\tstart = seq\n\t}\n\n\tprocessPartial(bsz + 1) // 128 + EOB\n\tprocessPartial(bsz + 1) // 128 + EOB\n\tprocessPartial(bsz + 1) // 128 + EOB\n\t// Last one will be a partial block.\n\tprocessPartial(116 + 1)\n\n\t// Now reset and test that batch is honored as well.\n\tstart, seq, np, b = 1, 1, sent, 100\n\tfor i := 0; i < 5; i++ {\n\t\tprocessPartial(b + 1) // 100 + EOB\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/4878\nfunc TestJetStreamInterestStreamConsumerFilterEdit(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"INTEREST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"interest.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"INTEREST\", &nats.ConsumerConfig{\n\t\tDurable: \"C0\",\n\t\tFilterSubject: \"interest.>\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = js.Publish(fmt.Sprintf(\"interest.%d\", i), []byte(strconv.Itoa(i)))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// we check we got 10 messages\n\tnfo, err := js.StreamInfo(\"INTEREST\")\n\trequire_NoError(t, err)\n\tif nfo.State.Msgs != 10 {\n\t\tt.Fatalf(\"expected 10 messages got %d\", nfo.State.Msgs)\n\t}\n\n\t// now we lower the consumer interest from all subjects to 1,\n\t// then check the stream state and check if interest behavior still works\n\t_, err = js.UpdateConsumer(\"INTEREST\", &nats.ConsumerConfig{\n\t\tDurable: \"C0\",\n\t\tFilterSubject: \"interest.1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// we should now have only one message left\n\tnfo, err = js.StreamInfo(\"INTEREST\")\n\trequire_NoError(t, err)\n\tif nfo.State.Msgs != 1 {\n\t\tt.Fatalf(\"expected 1 message got %d\", nfo.State.Msgs)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/5383\nfunc TestJetStreamInterestStreamWithFilterSubjectsConsumer(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"INTEREST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"interest.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"INTEREST\", &nats.ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubjects: []string{\"interest.a\", \"interest.b\"},\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor _, sub := range []string{\"interest.a\", \"interest.b\", \"interest.c\"} {\n\t\t_, err = js.Publish(sub, nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// we check we got 2 messages, interest.c doesn't have interest\n\tnfo, err := js.StreamInfo(\"INTEREST\")\n\trequire_NoError(t, err)\n\tif nfo.State.Msgs != 2 {\n\t\tt.Fatalf(\"expected 2 messages got %d\", nfo.State.Msgs)\n\t}\n}\n\nfunc TestJetStreamAckAllWithLargeFirstSequenceAndNoAckFloor(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Set first sequence to something very big here. This shows the issue with AckAll the\n\t// first time it is called and existing ack floor is 0.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 10_000_000_000})\n\trequire_NoError(t, err)\n\n\t// Now add in 100 msgs\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(\"foo.bar\", []byte(\"hello\"))\n\t}\n\n\tss, err := js.PullSubscribe(\"foo.*\", \"C1\", nats.AckAll())\n\trequire_NoError(t, err)\n\tmsgs, err := ss.Fetch(10, nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\tstart := time.Now()\n\tmsgs[9].AckSync()\n\tif elapsed := time.Since(start); elapsed > 250*time.Millisecond {\n\t\tt.Fatalf(\"AckSync took too long %v\", elapsed)\n\t}\n\n\t// Make sure next fetch works right away with low timeout.\n\tmsgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t_, err = js.StreamInfo(\"TEST\", nats.MaxWait(250*time.Millisecond))\n\trequire_NoError(t, err)\n\n\t// Now make sure that if we ack in the middle, meaning we still have ack pending,\n\t// that we do the right thing as well.\n\tss, err = js.PullSubscribe(\"foo.*\", \"C2\", nats.AckAll())\n\trequire_NoError(t, err)\n\tmsgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\tstart = time.Now()\n\tmsgs[5].AckSync()\n\tif elapsed := time.Since(start); elapsed > 250*time.Millisecond {\n\t\tt.Fatalf(\"AckSync took too long %v\", elapsed)\n\t}\n\n\t// Make sure next fetch works right away with low timeout.\n\tmsgs, err = ss.Fetch(10, nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\t_, err = js.StreamInfo(\"TEST\", nats.MaxWait(250*time.Millisecond))\n\trequire_NoError(t, err)\n}\n\nfunc TestJetStreamAckAllWithLargeFirstSequenceAndNoAckFloorWithInterestPolicy(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Set first sequence to something very big here. This shows the issue with AckAll the\n\t// first time it is called and existing ack floor is 0.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 10_000_000_000})\n\trequire_NoError(t, err)\n\n\tss, err := js.PullSubscribe(\"foo.*\", \"C1\", nats.AckAll())\n\trequire_NoError(t, err)\n\n\t// Now add in 100 msgs\n\tfor i := 0; i < 100; i++ {\n\t\tjs.Publish(\"foo.bar\", []byte(\"hello\"))\n\t}\n\n\tmsgs, err := ss.Fetch(10, nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\tstart := time.Now()\n\tmsgs[5].AckSync()\n\tif elapsed := time.Since(start); elapsed > 250*time.Millisecond {\n\t\tt.Fatalf(\"AckSync took too long %v\", elapsed)\n\t}\n\n\t// We are testing for run away loops acking messages in the stream that are not there.\n\t_, err = js.StreamInfo(\"TEST\", nats.MaxWait(100*time.Millisecond))\n\trequire_NoError(t, err)\n}\n\n// Allow streams with $JS or $SYS prefixes for audit purposes but require no pub ack be set.\nfunc TestJetStreamAuditStreams(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjsOverlap := errors.New(\"subjects that overlap with jetstream api require no-ack to be true\")\n\tsysOverlap := errors.New(\"subjects that overlap with system api require no-ack to be true\")\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"$JS.>\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"$JS.API.>\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"$JSC.>\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(jsOverlap))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"$SYS.>\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(sysOverlap))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\">\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"capturing all subjects requires no-ack to be true\")))\n\n\t// These should all be ok if no pub ack.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST1\",\n\t\tSubjects: []string{\"$JS.>\"},\n\t\tNoAck: true,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST2\",\n\t\tSubjects: []string{\"$JSC.>\"},\n\t\tNoAck: true,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST3\",\n\t\tSubjects: []string{\"$SYS.>\"},\n\t\tNoAck: true,\n\t})\n\trequire_NoError(t, err)\n\n\t// Since prior behavior did allow $JS.EVENT to be captured without no-ack, these might break\n\t// on a server upgrade so make sure they still work ok without --no-ack.\n\n\t// To avoid overlap error.\n\terr = js.DeleteStream(\"TEST1\")\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST4\",\n\t\tSubjects: []string{\"$JS.EVENT.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Also allow $SYS.ACCOUNT to be captured without no-ack, these also might break\n\t// on a server upgrade so make sure they still work ok without --no-ack.\n\n\t// To avoid overlap error.\n\terr = js.DeleteStream(\"TEST3\")\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST5\",\n\t\tSubjects: []string{\"$SYS.ACCOUNT.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// We will test handling of \">\" on a cluster here.\n\t// Specific test for capturing everything which will require both no-ack and replicas of 1.\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\">\"},\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"capturing all subjects requires no-ack to be true\")))\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\">\"},\n\t\tReplicas: 3,\n\t\tNoAck: true,\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"capturing all subjects requires replicas of 1\")))\n\n\t// Ths should work ok.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\">\"},\n\t\tReplicas: 1,\n\t\tNoAck: true,\n\t})\n\trequire_NoError(t, err)\n}\n\n// https://github.com/nats-io/nats-server/issues/5570\nfunc TestJetStreamBadSubjectMappingStream(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"test\"})\n\trequire_NoError(t, err)\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"mapping\",\n\t\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tSource: \"events.*\",\n\t\t\t\t\t\tDestination: \"events.{{wildcard(1)}}{{split(3,1)}}\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: source transform: invalid mapping destination: too many arguments passed to the function in {{wildcard(1)}}{{split(3,1)}}\")))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"mapping\",\n\t\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tSource: \"events.>.*\",\n\t\t\t\t\t\tDestination: \"events.{{split(1,1)}}\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: source transform source: invalid subject events.>.*\")))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"mapping\",\n\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t{\n\t\t\t\t\tSource: \"events.*\",\n\t\t\t\t\tDestination: \"events.{{split(3,1)}}\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: mirror transform: invalid mapping destination: wildcard index out of range in {{split(3,1)}}: [3]\")))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"mapping\",\n\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t{\n\t\t\t\t\tSource: \"events.>.*\",\n\t\t\t\t\tDestination: \"events.{{split(1,1)}}\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: mirror transform source: invalid subject events.>.*\")))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjectTransform: &nats.SubjectTransformConfig{\n\t\t\tSource: \"events.*\",\n\t\t\tDestination: \"events.{{split(3,1)}}\",\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: stream transform: invalid mapping destination: wildcard index out of range in {{split(3,1)}}: [3]\")))\n\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjectTransform: &nats.SubjectTransformConfig{\n\t\t\tSource: \"events.>.*\",\n\t\t\tDestination: \"events.{{split(1,1)}}\",\n\t\t},\n\t})\n\trequire_Error(t, err, NewJSStreamUpdateError(errors.New(\"nats: stream transform source: invalid subject events.>.*\")))\n}\n\nfunc TestJetStreamInterestStreamWithDuplicateMessages(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"INTEREST\",\n\t\tSubjects: []string{\"interest\"},\n\t\tReplicas: 1,\n\t\tRetention: nats.InterestPolicy,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Publishing the first time should give a sequence, even when there's no interest.\n\tpa, err := js.Publish(\"interest\", nil, nats.MsgId(\"dedupe\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pa.Sequence, 1)\n\trequire_Equal(t, pa.Duplicate, false)\n\n\t// Publishing a duplicate with no interest should return the same sequence as above.\n\tpa, err = js.Publish(\"interest\", nil, nats.MsgId(\"dedupe\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pa.Sequence, 1)\n\trequire_Equal(t, pa.Duplicate, true)\n}\n\nfunc TestJetStreamStreamCreatePedanticMode(t *testing.T) {\n\tcfgFmt := []byte(fmt.Sprintf(`\n jetstream: {\n enabled: true\n max_file_store: 100MB\n store_dir: %s\n limits: {duplicate_window: \"1m\", max_request_batch: 250}\n }\n accounts: {\n myacc: {\n jetstream: enabled\n users: [ { user: user, password: pass } ]\n }\n }\n no_auth_user: user\n\t`, t.TempDir()))\n\n\tconf := createConfFile(t, cfgFmt)\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\tcfg StreamConfigRequest\n\t\tshouldErr bool\n\t\tupdate bool\n\t}{\n\t\t{\n\t\t\tname: \"too_high_duplicate\",\n\t\t\tcfg: StreamConfigRequest{\n\t\t\t\tStreamConfig: StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tMaxAge: time.Minute,\n\t\t\t\t\tDuplicates: time.Hour,\n\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t},\n\t\t\t\tPedantic: true,\n\t\t\t},\n\t\t\tshouldErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"duplicate_over_limits\",\n\t\t\tcfg: StreamConfigRequest{\n\t\t\t\tStreamConfig: StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tMaxAge: time.Hour * 60,\n\t\t\t\t\tDuplicates: time.Hour,\n\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t},\n\t\t\t\tPedantic: false,\n\t\t\t},\n\t\t\tshouldErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"duplicate_window_within_limits\",\n\t\t\tcfg: StreamConfigRequest{\n\t\t\t\tStreamConfig: StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tMaxAge: time.Hour * 60,\n\t\t\t\t\tDuplicates: time.Second * 30,\n\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t},\n\t\t\t\tPedantic: false,\n\t\t\t},\n\t\t\tshouldErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"update_too_high_duplicate\",\n\t\t\tcfg: StreamConfigRequest{\n\t\t\t\tStreamConfig: StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tMaxAge: time.Minute,\n\t\t\t\t\tDuplicates: time.Hour,\n\t\t\t\t\tStorage: FileStorage,\n\t\t\t\t},\n\t\t\t\tPedantic: true,\n\t\t\t},\n\t\t\tupdate: true,\n\t\t\tshouldErr: true,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif !test.update {\n\t\t\t\t_, err := addStreamPedanticWithError(t, nc, &test.cfg)\n\t\t\t\trequire_True(t, (err != nil) == test.shouldErr)\n\t\t\t} else {\n\t\t\t\t_, err := updateStreamPedanticWithError(t, nc, &test.cfg)\n\t\t\t\trequire_True(t, (err != nil) == test.shouldErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStrictMode(t *testing.T) {\n\tcfgFmt := []byte(fmt.Sprintf(`\n\t\tjetstream: {\n\t\t\tstrict: true\n\t\t\tenabled: true\n\t\t\tmax_file_store: 100MB\n\t\t\tstore_dir: %s\n\t\t\tlimits: {duplicate_window: \"1m\", max_request_batch: 250}\n\t\t}\n\t`, t.TempDir()))\n\tconf := createConfFile(t, cfgFmt)\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to NATS: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\tsubject string\n\t\tpayload []byte\n\t\texpectedErr string\n\t}{\n\t\t{\n\t\t\tname: \"Stream Create\",\n\t\t\tsubject: \"$JS.API.STREAM.CREATE.TEST_STREAM\",\n\t\t\tpayload: []byte(`{\"name\":\"TEST_STREAM\",\"subjects\":[\"test.>\"],\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t\t{\n\t\t\tname: \"Stream Update\",\n\t\t\tsubject: \"$JS.API.STREAM.UPDATE.TEST_STREAM\",\n\t\t\tpayload: []byte(`{\"name\":\"TEST_STREAM\",\"subjects\":[\"test.>\"],\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t\t{\n\t\t\tname: \"Stream Delete\",\n\t\t\tsubject: \"$JS.API.STREAM.DELETE.TEST_STREAM\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"expected an empty request payload\",\n\t\t},\n\t\t{\n\t\t\tname: \"Stream Info\",\n\t\t\tsubject: \"$JS.API.STREAM.INFO.TEST_STREAM\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t\t{\n\t\t\tname: \"Consumer Create\",\n\t\t\tsubject: \"$JS.API.CONSUMER.CREATE.TEST_STREAM.TEST_CONSUMER\",\n\t\t\tpayload: []byte(`{\"durable_name\":\"TEST_CONSUMER\",\"ack_policy\":\"explicit\",\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t\t{\n\t\t\tname: \"Consumer Delete\",\n\t\t\tsubject: \"$JS.API.CONSUMER.DELETE.TEST_STREAM.TEST_CONSUMER\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"expected an empty request payload\",\n\t\t},\n\t\t{\n\t\t\tname: \"Consumer Info\",\n\t\t\tsubject: \"$JS.API.CONSUMER.INFO.TEST_STREAM.TEST_CONSUMER\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"expected an empty request payload\",\n\t\t},\n\t\t{\n\t\t\tname: \"Stream List\",\n\t\t\tsubject: \"$JS.API.STREAM.LIST\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t\t{\n\t\t\tname: \"Consumer List\",\n\t\t\tsubject: \"$JS.API.CONSUMER.LIST.TEST_STREAM\",\n\t\t\tpayload: []byte(`{\"extra_field\":\"unexpected\"}`),\n\t\t\texpectedErr: \"invalid JSON\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresp, err := nc.Request(tt.subject, tt.payload, time.Second*10)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Request failed: %v\", err)\n\t\t\t}\n\n\t\t\tvar apiResp ApiResponse = ApiResponse{}\n\n\t\t\tif err := json.Unmarshal(resp.Data, &apiResp); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling response: %v\", err)\n\t\t\t}\n\n\t\t\trequire_NotNil(t, apiResp.Error.Description)\n\t\t\trequire_Contains(t, apiResp.Error.Description, tt.expectedErr)\n\t\t})\n\t}\n}\n\nfunc addConsumerWithError(t *testing.T, nc *nats.Conn, cfg *CreateConsumerRequest) (*ConsumerInfo, *ApiError) {\n\tt.Helper()\n\treq, err := json.Marshal(cfg)\n\trequire_NoError(t, err)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable), req, 5*time.Second)\n\n\trequire_NoError(t, err)\n\tvar resp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(rmsg.Data, &resp)\n\trequire_NoError(t, err)\n\tif resp.Type != JSApiConsumerCreateResponseType {\n\t\tt.Fatalf(\"Invalid response type %s expected %s\", resp.Type, JSApiConsumerCreateResponseType)\n\t}\n\treturn resp.ConsumerInfo, resp.Error\n}\n\nfunc TestJetStreamSourceRemovalAndReAdd(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// The source stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"SRC\",\n\t\tSubjects: []string{\"foo.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// The stream that sources.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"SRC\",\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Now add in 10 msgs.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(fmt.Sprintf(\"foo.%d\", i), []byte(\"test\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Make sure we have 10 msgs in TEST.\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 10 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Do not have all msgs yet, %d of 10\", si.State.Msgs)\n\t})\n\n\t// Now update the TEST stream to no longer source.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Now add in 10 more msgs.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(fmt.Sprintf(\"foo.%d\", i+10), []byte(\"test\"))\n\t\trequire_NoError(t, err)\n\t}\n\t// Make sure we are still stuck at 10 for TEST.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 10)\n\n\t// Now re-add the source to our stream.\n\t_, err = js.UpdateStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSources: []*nats.StreamSource{{\n\t\t\tName: \"SRC\",\n\t\t}},\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure we have 20 msgs now.\n\t// Make sure we have 10 msgs in TEST.\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 20 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Do not have all msgs yet, %d of 20\", si.State.Msgs)\n\t})\n\n\t// Check that we get what we want in the stream.\n\tsub, err := js.PullSubscribe(\"foo.*\", \"d\")\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(20)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 20)\n\n\tfor i, m := range msgs {\n\t\trequire_Equal(t, m.Subject, fmt.Sprintf(\"foo.%d\", i))\n\t}\n}\n\nfunc TestJetStreamRateLimitHighStreamIngest(t *testing.T) {\n\tcfgFmt := []byte(fmt.Sprintf(`\n jetstream: {\n enabled: true\n store_dir: %s\n max_buffered_size: 1kb\n max_buffered_msgs: 1\n }\n `, t.TempDir()))\n\n\tconf := createConfFile(t, cfgFmt)\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\trequire_Equal(t, opts.StreamMaxBufferedSize, 1024)\n\trequire_Equal(t, opts.StreamMaxBufferedMsgs, 1)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Create a reply inbox that we can await API requests on.\n\t// This is instead of using nc.Request().\n\tinbox := nc.NewRespInbox()\n\tresp := make(chan *nats.Msg, 1000)\n\t_, err = nc.ChanSubscribe(inbox, resp)\n\trequire_NoError(t, err)\n\n\t// Publish a large number of messages using Core NATS withou\n\t// waiting for the responses from the API.\n\tmsg := &nats.Msg{\n\t\tSubject: \"test\",\n\t\tReply: inbox,\n\t}\n\tfor i := 0; i < 1000; i++ {\n\t\trequire_NoError(t, nc.PublishMsg(msg))\n\t}\n\n\t// Now sort through the API responses. We're looking for one\n\t// that tells us that we were rate-limited. If we don't find\n\t// one then we fail the test.\n\tvar rateLimited bool\n\tfor i, msg := 0, <-resp; i < 1000; i, msg = i+1, <-resp {\n\t\tif msg.Header.Get(\"Status\") == \"429\" {\n\t\t\trateLimited = true\n\t\t\tbreak\n\t\t}\n\t}\n\trequire_True(t, rateLimited)\n}\n\nfunc TestJetStreamRateLimitHighStreamIngestDefaults(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\tstream, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, stream.msgs.mlen, streamDefaultMaxQueueMsgs)\n\trequire_Equal(t, stream.msgs.msz, streamDefaultMaxQueueBytes)\n}\n\nfunc TestJetStreamStreamConfigClone(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"name\",\n\t\tPlacement: &Placement{Cluster: \"placement\", Tags: []string{\"tag\"}},\n\t\tMirror: &StreamSource{Name: \"mirror\"},\n\t\tSources: []*StreamSource{&StreamSource{Name: \"source\"}},\n\t\tSubjectTransform: &SubjectTransformConfig{Source: \"source\", Destination: \"dest\"},\n\t\tRePublish: &RePublish{Source: \"source\", Destination: \"dest\", HeadersOnly: false},\n\t\tMetadata: make(map[string]string),\n\t}\n\n\t// Copy should be complete.\n\tclone := cfg.clone()\n\trequire_True(t, reflect.DeepEqual(cfg, clone))\n\n\t// Changing fields should not update the original.\n\tclone.Placement.Cluster = \"diff\"\n\trequire_False(t, reflect.DeepEqual(cfg.Placement, clone.Placement))\n\n\tclone.Mirror.Name = \"diff\"\n\trequire_False(t, reflect.DeepEqual(cfg.Mirror, clone.Mirror))\n\n\tclone.Sources[0].Name = \"diff\"\n\trequire_False(t, reflect.DeepEqual(cfg.Sources, clone.Sources))\n\n\tclone.SubjectTransform.Source = \"diff\"\n\trequire_False(t, reflect.DeepEqual(cfg.SubjectTransform, clone.SubjectTransform))\n\n\tclone.RePublish.Source = \"diff\"\n\trequire_False(t, reflect.DeepEqual(cfg.RePublish, clone.RePublish))\n\n\tclone.Metadata[\"key\"] = \"value\"\n\trequire_False(t, reflect.DeepEqual(cfg.Metadata, clone.Metadata))\n}\n\nfunc TestIsJSONObjectOrArray(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tdata []byte\n\t\tvalid bool\n\t}{\n\t\t{\"empty\", []byte{}, false},\n\t\t{\"empty_object\", []byte(\"{}\"), true},\n\t\t{\"empty object, not trimmed\", []byte(\"\\t\\n\\r{ }\"), true},\n\t\t{\"empty_array\", []byte(\"[]\"), true},\n\t\t{\"empty array, not trimmed\", []byte(\"\\t\\n\\r[ ]\"), true},\n\t\t// This is a valid JSON, but it's not a JSON object or array.\n\t\t{\"empty_string\", []byte(\"\\\"\\\"\"), false},\n\t\t{\"whitespace_only\", []byte(\" \"), false},\n\t\t{\"object_with_whitespace\", []byte(\"{ }\"), true},\n\t\t{\"array_with_whitespace\", []byte(\"[ ]\"), true},\n\t\t{\"string_with_whitespace\", []byte(\" \\\"text\\\"\"), false},\n\t\t{\"number\", []byte(\"123\"), false},\n\t\t{\"boolean_true\", []byte(\"true\"), false},\n\t\t{\"boolean_false\", []byte(\"false\"), false},\n\t\t{\"null_value\", []byte(\"null\"), false}, {\"invalid JSON\", []byte(\"invalid\"), false},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\trequire_Equal(t, isJSONObjectOrArray(test.data), test.valid)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSourcingClipStartSeq(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORIGIN\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"SOURCING\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"ORIGIN\",\n\t\t\t\tOptStartSeq: 20,\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Wait for sourcing consumer to be created.\n\ttime.Sleep(time.Second)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"ORIGIN\")\n\trequire_NoError(t, err)\n\trequire_True(t, mset != nil)\n\trequire_Len(t, len(mset.consumers), 1)\n\tfor _, o := range mset.consumers {\n\t\t// Should have been clipped back to below 20 as only\n\t\t// 10 messages in the origin stream.\n\t\trequire_Equal(t, o.sseq, 11)\n\t}\n}\n\nfunc TestJetStreamMirroringClipStartSeq(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORIGIN\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRRORING\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"ORIGIN\",\n\t\t\tOptStartSeq: 20,\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Wait for mirroring consumer to be created.\n\ttime.Sleep(time.Second)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"ORIGIN\")\n\trequire_NoError(t, err)\n\trequire_True(t, mset != nil)\n\trequire_Len(t, len(mset.consumers), 1)\n\tfor _, o := range mset.consumers {\n\t\t// Should have been clipped back to below 20 as only\n\t\t// 10 messages in the origin stream.\n\t\trequire_Equal(t, o.sseq, 11)\n\t}\n}\n\nfunc TestJetStreamDelayedAPIResponses(t *testing.T) {\n\ttdir := t.TempDir()\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\t%sjetstream: {max_mem_store: 64GB, max_file_store: 10TB, store_dir: %q}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"\", tdir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, JSAuditAdvisory)\n\n\tacc := s.GlobalAccount()\n\n\t// Send B, A, D, C and expected to receive A, B, C, D\n\ts.sendDelayedAPIErrResponse(nil, acc, \"B\", _EMPTY_, \"request2\", \"response2\", nil, 500*time.Millisecond)\n\ttime.Sleep(50 * time.Millisecond)\n\ts.sendDelayedAPIErrResponse(nil, acc, \"A\", _EMPTY_, \"request1\", \"response1\", nil, 200*time.Millisecond)\n\ttime.Sleep(50 * time.Millisecond)\n\ts.sendDelayedAPIErrResponse(nil, acc, \"D\", _EMPTY_, \"request4\", \"response4\", nil, 800*time.Millisecond)\n\ttime.Sleep(50 * time.Millisecond)\n\ts.sendDelayedAPIErrResponse(nil, acc, \"C\", _EMPTY_, \"request3\", \"response3\", nil, 650*time.Millisecond)\n\n\tcheck := func(req, resp string) {\n\t\tt.Helper()\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tvar audit JSAPIAudit\n\t\terr := json.Unmarshal(msg.Data, &audit)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, audit.Request, req)\n\t\trequire_Equal(t, audit.Response, resp)\n\t}\n\tcheck(\"request1\", \"response1\")\n\tcheck(\"request2\", \"response2\")\n\tcheck(\"request3\", \"response3\")\n\tcheck(\"request4\", \"response4\")\n\n\t// Verify that if a raft group is canceled, the delayed API response is canceled too.\n\tnode := &raft{quit: make(chan struct{})}\n\tg := &raftGroup{node: node}\n\t// Send delayed API response with this raft group\n\ts.sendDelayedAPIErrResponse(nil, acc, \"E\", _EMPTY_, \"request5\", \"response5\", g, 250*time.Millisecond)\n\ttime.Sleep(50 * time.Millisecond)\n\t// Send that one without a group\n\ts.sendDelayedAPIErrResponse(nil, acc, \"F\", _EMPTY_, \"request6\", \"response6\", nil, 400*time.Millisecond)\n\t// Close the \"request5\"'s channel.\n\tclose(node.quit)\n\t// So we should receive request6, not 5.\n\tcheck(\"request6\", \"response6\")\n\n\t// Check config reload.\n\ts.sendDelayedAPIErrResponse(nil, acc, \"G\", _EMPTY_, \"request7\", \"response7\", nil, 400*time.Millisecond)\n\ttime.Sleep(50 * time.Millisecond)\n\t// Send a bunch more\n\tfor i := 0; i < 10; i++ {\n\t\ts.sendDelayedAPIErrResponse(nil, acc, \"H\", _EMPTY_, \"request8\", \"response8\", nil, 500*time.Millisecond)\n\t}\n\t// Config reload to disable JS.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"#\", tdir))\n\tcheck(\"request7\", \"response7\")\n\t// We should not receive more.\n\tif msg, err := sub.NextMsg(600 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Did not expect to receive: %s\", msg.Data)\n\t}\n\t// Check that the queue is empty.\n\ts.mu.RLock()\n\tq := s.delayedAPIResponses\n\ts.mu.RUnlock()\n\trequire_Equal(t, q.len(), 0)\n\n\t// Restore JS and check delayed response can be received.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"\", tdir))\n\t// Wait until js is re-enabled\n\tcheckFor(t, 10*time.Second, 50*time.Millisecond, func() error {\n\t\tif s.getJetStream() == nil {\n\t\t\treturn ErrJetStreamNotEnabled\n\t\t}\n\t\treturn nil\n\t})\n\ts.sendDelayedAPIErrResponse(nil, acc, \"I\", _EMPTY_, \"request9\", \"response9\", nil, 100*time.Millisecond)\n\tcheck(\"request9\", \"response9\")\n}\n\nfunc TestJetStreamMemoryPurgeClearsSubjectsState(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.MemoryStorage,\n\t})\n\trequire_NoError(t, err)\n\n\tpa, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pa.Sequence, 1)\n\n\t// When requesting stream info, we expect one subject foo with one entry.\n\tsi, err := js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{SubjectsFilter: \">\"})\n\trequire_NoError(t, err)\n\trequire_Len(t, len(si.State.Subjects), 1)\n\trequire_Equal(t, si.State.Subjects[\"foo\"], 1)\n\n\t// After purging, moving the sequence up, the subjects state should be cleared.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100})\n\trequire_NoError(t, err)\n\n\tsi, err = js.StreamInfo(\"TEST\", &nats.StreamInfoRequest{SubjectsFilter: \">\"})\n\trequire_NoError(t, err)\n\trequire_Len(t, len(si.State.Subjects), 0)\n}\n\nfunc TestJetStreamWouldExceedLimits(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tjs := s.getJetStream()\n\trequire_NotNil(t, js)\n\n\t// Storing exactly up to the limit should work.\n\trequire_False(t, js.wouldExceedLimits(MemoryStorage, int(js.config.MaxMemory)))\n\trequire_False(t, js.wouldExceedLimits(FileStorage, int(js.config.MaxStore)))\n\n\t// Storing one more than the max should exceed limits.\n\trequire_True(t, js.wouldExceedLimits(MemoryStorage, int(js.config.MaxMemory)+1))\n\trequire_True(t, js.wouldExceedLimits(FileStorage, int(js.config.MaxStore)+1))\n}\n\nfunc TestJetStreamMessageTTL(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: nats.Header{},\n\t\t\t}\n\n\t\t\tfor i := 1; i <= 10; i++ {\n\t\t\t\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\t\t\t\t_, err := js.PublishMsg(msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 10)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 1)\n\t\t\trequire_Equal(t, si.State.LastSeq, 10)\n\n\t\t\ttime.Sleep(time.Second * 2)\n\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 11)\n\t\t\trequire_Equal(t, si.State.LastSeq, 10)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t})\n\n\tmsg := &nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: nats.Header{},\n\t}\n\n\tfor i := 1; i <= 10; i++ {\n\t\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\t\t_, err := js.PublishMsg(msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 10)\n\trequire_Equal(t, si.State.FirstSeq, 1)\n\trequire_Equal(t, si.State.LastSeq, 10)\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 10)\n\trequire_Equal(t, si.State.FirstSeq, 1)\n\trequire_Equal(t, si.State.LastSeq, 10)\n\n\ttime.Sleep(time.Second * 2)\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 11)\n\trequire_Equal(t, si.State.LastSeq, 10)\n}\n\nfunc TestJetStreamMessageTTLRecovered(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t})\n\n\tmsg := &nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: nats.Header{},\n\t}\n\n\tfor i := 1; i <= 10; i++ {\n\t\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\t\t_, err := js.PublishMsg(msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 10)\n\trequire_Equal(t, si.State.FirstSeq, 1)\n\trequire_Equal(t, si.State.LastSeq, 10)\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\tfn := filepath.Join(sd, globalAccountName, streamsDir, \"TEST\", msgDir, ttlStreamStateFile)\n\trequire_NoError(t, os.RemoveAll(fn))\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 10)\n\trequire_Equal(t, si.State.FirstSeq, 1)\n\trequire_Equal(t, si.State.LastSeq, 10)\n\n\ttime.Sleep(time.Second * 2)\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 11)\n\trequire_Equal(t, si.State.LastSeq, 10)\n}\n\nfunc TestJetStreamMessageTTLInvalid(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: nats.Header{},\n\t\t\t}\n\n\t\t\tmsg.Header.Set(JSMessageTTL, \"500ms\")\n\t\t\t_, err := js.PublishMsg(msg)\n\t\t\trequire_Error(t, err)\n\n\t\t\tmsg.Header.Set(JSMessageTTL, \"something\")\n\t\t\t_, err = js.PublishMsg(msg)\n\t\t\trequire_Error(t, err)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLNotUpdatable(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t})\n\n\t_, err := jsStreamUpdate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: false,\n\t})\n\trequire_Error(t, err)\n}\n\nfunc TestJetStreamMessageTTLNeverExpire(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tMaxAge: time.Second,\n\t\t\t})\n\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: nats.Header{},\n\t\t\t}\n\n\t\t\t// The first message we publish is set to \"never expire\", therefore it\n\t\t\t// won't age out with the MaxAge policy.\n\t\t\tmsg.Header.Set(JSMessageTTL, \"never\")\n\t\t\t_, err := js.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Following messages will be published as normal and will age out.\n\t\t\tmsg.Header.Del(JSMessageTTL)\n\t\t\tfor i := 1; i <= 10; i++ {\n\t\t\t\t_, err := js.PublishMsg(msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 11)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 1)\n\t\t\trequire_Equal(t, si.State.LastSeq, 11)\n\n\t\t\ttime.Sleep(time.Second * 2)\n\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 1)\n\t\t\trequire_Equal(t, si.State.LastSeq, 11)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLDisabled(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t})\n\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: nats.Header{},\n\t\t\t}\n\n\t\t\tmsg.Header.Set(JSMessageTTL, \"1s\")\n\t\t\t_, err := js.PublishMsg(msg)\n\t\t\trequire_Error(t, err)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLWhenSourcing(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"Origin\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TTLEnabled\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSources: []*StreamSource{\n\t\t\t\t\t{Name: \"Origin\"},\n\t\t\t\t},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tttlDisabledConfig := &StreamConfig{\n\t\t\t\tName: \"TTLDisabled\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSources: []*StreamSource{\n\t\t\t\t\t{Name: \"Origin\"},\n\t\t\t\t},\n\t\t\t\tAllowMsgTTL: false,\n\t\t\t}\n\t\t\t_, err = jsStreamCreate(t, nc, ttlDisabledConfig)\n\t\t\trequire_NoError(t, err)\n\n\t\t\thdr := nats.Header{}\n\t\t\thdr.Add(JSMessageTTL, \"1s\")\n\n\t\t\t_, err = js.PublishMsg(&nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: hdr,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor _, stream := range []string{\"TTLEnabled\", \"TTLDisabled\"} {\n\t\t\t\tt.Run(stream, func(t *testing.T) {\n\t\t\t\t\tsc, err := js.PullSubscribe(\"test\", \"consumer\", nats.BindStream(stream))\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tmsgs, err := sc.Fetch(1)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, len(msgs), 1)\n\t\t\t\t\trequire_Equal(t, msgs[0].Header.Get(JSMessageTTL), \"1s\")\n\n\t\t\t\t\ttime.Sleep(time.Second)\n\n\t\t\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif stream != \"TTLDisabled\" {\n\t\t\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\n\t\t\t\t\tttlDisabledConfig.AllowMsgTTL = true\n\t\t\t\t\t_, err = jsStreamUpdate(t, nc, ttlDisabledConfig)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tsi, err = js.StreamInfo(stream)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLWhenMirroring(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"Origin\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TTLEnabled\",\n\t\t\t\tStorage: storage,\n\t\t\t\tMirror: &StreamSource{\n\t\t\t\t\tName: \"Origin\",\n\t\t\t\t},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tttlDisabledConfig := &StreamConfig{\n\t\t\t\tName: \"TTLDisabled\",\n\t\t\t\tStorage: storage,\n\t\t\t\tMirror: &StreamSource{\n\t\t\t\t\tName: \"Origin\",\n\t\t\t\t},\n\t\t\t\tAllowMsgTTL: false,\n\t\t\t}\n\t\t\t_, err = jsStreamCreate(t, nc, ttlDisabledConfig)\n\t\t\trequire_NoError(t, err)\n\n\t\t\thdr := nats.Header{}\n\t\t\thdr.Add(JSMessageTTL, \"1s\")\n\n\t\t\t_, err = js.PublishMsg(&nats.Msg{\n\t\t\t\tSubject: \"test\",\n\t\t\t\tHeader: hdr,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor _, stream := range []string{\"TTLEnabled\", \"TTLDisabled\"} {\n\t\t\t\tt.Run(stream, func(t *testing.T) {\n\t\t\t\t\tsc, err := js.PullSubscribe(\"test\", \"consumer\", nats.BindStream(stream))\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tmsgs, err := sc.Fetch(1)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, len(msgs), 1)\n\t\t\t\t\trequire_Equal(t, msgs[0].Header.Get(JSMessageTTL), \"1s\")\n\n\t\t\t\t\ttime.Sleep(time.Second)\n\n\t\t\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tif stream != \"TTLDisabled\" {\n\t\t\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\n\t\t\t\t\tttlDisabledConfig.AllowMsgTTL = true\n\t\t\t\t\t_, err = jsStreamUpdate(t, nc, ttlDisabledConfig)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tsi, err = js.StreamInfo(stream)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Equal(t, si.State.Msgs, 0)\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSubjectDeleteMarkers(t *testing.T) {\n\tfor _, storage := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tjsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storage,\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tMaxAge: time.Second,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t})\n\n\t\t\tsub, err := js.SubscribeSync(\"test\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, err = js.Publish(\"test\", nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tmsg, err := sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, msg.AckSync())\n\t\t\t}\n\n\t\t\tmsg, err := sub.NextMsg(time.Second * 10)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Header.Get(JSMarkerReason), \"MaxAge\")\n\t\t\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSubjectDeleteMarkersAfterRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tMaxAge: time.Second,\n\t\tAllowMsgTTL: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"test_consumer\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub, err := js.PullSubscribe(\"test\", _EMPTY_, nats.Bind(\"TEST\", \"test_consumer\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\tmsgs, err := sub.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\t}\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_Equal(t, msgs[0].Header.Get(JSMarkerReason), \"MaxAge\")\n\trequire_Equal(t, msgs[0].Header.Get(JSMessageTTL), \"1s\")\n}\n\nfunc TestJetStreamSubjectDeleteMarkersTTLRollupWithMaxAge(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tMaxAge: time.Second,\n\t\tAllowMsgTTL: true,\n\t\tAllowRollup: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\n\tsub, err := js.SubscribeSync(\"test\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\trh := nats.Header{}\n\trh.Set(JSMessageTTL, \"2s\") // MaxAge will get here first.\n\trh.Set(JSMsgRollup, JSMsgRollupSubject)\n\t_, err = js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: rh,\n\t})\n\trequire_NoError(t, err)\n\n\t// Expect to only have the rollup message here.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, 4)\n\n\tmsg, err := sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(JSMsgRollup), JSMsgRollupSubject)\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"2s\")\n\tmeta, err := msg.Metadata()\n\trequire_NoError(t, err)\n\trequire_NoError(t, msg.AckSync())\n\n\tmsg, err = sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, time.Second, time.Since(meta.Timestamp))\n\trequire_Equal(t, msg.Header.Get(JSMarkerReason), \"MaxAge\")\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n}\n\nfunc TestJetStreamSubjectDeleteMarkersTTLRollupWithoutMaxAge(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tAllowMsgTTL: true,\n\t\tAllowRollup: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.SubscribeSync(\"test\")\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 3; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\trh := nats.Header{}\n\trh.Set(JSMessageTTL, \"1s\")\n\trh.Set(JSMsgRollup, JSMsgRollupSubject)\n\t_, err = js.PublishMsg(&nats.Msg{\n\t\tSubject: \"test\",\n\t\tHeader: rh,\n\t})\n\trequire_NoError(t, err)\n\n\t// Expect to only have the rollup message here.\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, 4)\n\n\tmsg, err := sub.NextMsg(time.Second * 10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(JSMsgRollup), JSMsgRollupSubject)\n\trequire_Equal(t, msg.Header.Get(JSMessageTTL), \"1s\")\n\trequire_NoError(t, msg.AckSync())\n\n\t// Wait for the rollup message to hit the TTL.\n\ttime.Sleep(2500 * time.Millisecond)\n\n\t// Now it should be gone, and it will have been replaced with a\n\t// subject delete marker (which is also gone by now).\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 6)\n}\n\nfunc TestJetStreamSubjectDeleteMarkersWithMirror(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"Origin\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tMaxAge: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"Mirror\",\n\t\tStorage: FileStorage,\n\t\tAllowMsgTTL: true,\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\tMirror: &StreamSource{\n\t\t\tName: \"Origin\",\n\t\t},\n\t})\n\trequire_Error(t, err)\n}\n\n// https://github.com/nats-io/nats-server/issues/6538\nfunc TestJetStreamInterestMaxDeliveryReached(t *testing.T) {\n\tmaxWait := 250 * time.Millisecond\n\tfor _, useNak := range []bool{true, false} {\n\t\tfor _, test := range []struct {\n\t\t\ttitle string\n\t\t\taction func(s *Server, sub *nats.Subscription)\n\t\t}{\n\t\t\t{\n\t\t\t\ttitle: \"fetch\",\n\t\t\t\taction: func(s *Server, sub *nats.Subscription) {\n\t\t\t\t\ttime.Sleep(time.Second)\n\n\t\t\t\t\t// max deliver 1 so this will fail\n\t\t\t\t\t_, err := sub.Fetch(1, nats.MaxWait(maxWait))\n\t\t\t\t\trequire_Error(t, err)\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\ttitle: \"expire pending\",\n\t\t\t\taction: func(s *Server, sub *nats.Subscription) {\n\t\t\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\to := mset.lookupConsumer(\"consumer\")\n\t\t\t\t\trequire_NotNil(t, o)\n\n\t\t\t\t\to.mu.Lock()\n\t\t\t\t\to.forceExpirePending()\n\t\t\t\t\to.mu.Unlock()\n\t\t\t\t},\n\t\t\t},\n\t\t} {\n\t\t\ttitle := fmt.Sprintf(\"nak/%s\", test.title)\n\t\t\tif !useNak {\n\t\t\t\ttitle = fmt.Sprintf(\"no-%s\", title)\n\t\t\t}\n\t\t\tt.Run(title, func(t *testing.T) {\n\t\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tStorage: nats.FileStorage,\n\t\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\t\tReplicas: 1,\n\t\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tsub, err := js.PullSubscribe(\"test\", \"consumer\", nats.AckWait(time.Second), nats.MaxDeliver(1))\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t_, err = nc.Request(\"test\", []byte(\"hello\"), maxWait)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tnfo, err := js.StreamInfo(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, nfo.State.Msgs, uint64(1))\n\n\t\t\t\tmsg, err := sub.Fetch(1, nats.MaxWait(maxWait))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, 1, len(msg))\n\t\t\t\tif useNak {\n\t\t\t\t\trequire_NoError(t, msg[0].Nak())\n\t\t\t\t}\n\n\t\t\t\tcnfo, err := js.ConsumerInfo(\"TEST\", \"consumer\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, cnfo.NumAckPending, 1)\n\n\t\t\t\ttest.action(s, sub)\n\n\t\t\t\t// max deliver 1 so this will fail\n\t\t\t\t_, err = sub.Fetch(1, nats.MaxWait(maxWait))\n\t\t\t\trequire_Error(t, err)\n\n\t\t\t\tcnfo, err = js.ConsumerInfo(\"TEST\", \"consumer\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, cnfo.NumAckPending, 0)\n\n\t\t\t\tnfo, err = js.StreamInfo(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, nfo.State.Msgs, uint64(1))\n\n\t\t\t\tsub2, err := js.PullSubscribe(\"test\", \"consumer2\")\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tmsg, err = sub2.Fetch(1)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, 1, len(msg))\n\t\t\t\trequire_NoError(t, msg[0].AckSync())\n\n\t\t\t\tnfo, err = js.StreamInfo(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, nfo.State.Msgs, uint64(1))\n\t\t\t})\n\t\t}\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/7817\nfunc TestJetStreamWQMaxDeliveryReached(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor range 3 {\n\t\t_, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\",\n\t\tnats.BindStream(\"TEST\"),\n\t\tnats.MaxDeliver(2),\n\t\tnats.AckExplicit(),\n\t\tnats.AckWait(200*time.Millisecond),\n\t)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\tmsgs, err := sub.Fetch(3, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 3)\n\n\tmsgs, err = sub.Fetch(3, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 3)\n\n\t_, err = sub.Fetch(1, nats.MaxWait(500*time.Millisecond))\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\tmsgs, err = sub.Fetch(1, nats.MaxWait(500*time.Millisecond))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 1)\n\trequire_NoError(t, msgs[0].AckSync())\n\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"CONSUMER\")\n\trequire_NotNil(t, cl)\n\tmset, err := cl.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"CONSUMER\")\n\trequire_NotNil(t, o)\n\to.mu.RLock()\n\tpending, rdc, adflr, asflr := len(o.pending), len(o.rdc), o.adflr, o.asflr\n\to.mu.RUnlock()\n\trequire_Equal(t, pending, 0)\n\trequire_Equal(t, rdc, 3)\n\trequire_Equal(t, adflr, 7)\n\trequire_Equal(t, asflr, 4)\n\n\tmset.checkInterestState()\n\ttime.Sleep(200 * time.Millisecond)\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"TEST\")\n\t})\n\n\tsm, err := mset.store.LoadMsg(1, nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, sm.subj, \"foo\")\n}\n\n// https://github.com/nats-io/nats-server/issues/6874\nfunc TestJetStreamMaxDeliveryRedeliveredReporting(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t\tRetention: nats.LimitsPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\tmaxWait := 250 * time.Millisecond\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"CONSUMER\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: maxWait,\n\t\tMaxDeliver: 1,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"CONSUMER\", nats.BindStream(\"TEST\"))\n\trequire_NoError(t, err)\n\n\tnfo, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, nfo.State.Msgs, uint64(1))\n\n\tmsgs, err := sub.Fetch(1, nats.MaxWait(maxWait))\n\trequire_NoError(t, err)\n\trequire_Len(t, 1, len(msgs))\n\n\tcnfo, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, cnfo.NumAckPending, 1)\n\trequire_Equal(t, cnfo.NumRedelivered, 0)\n\n\ttime.Sleep(2 * maxWait)\n\n\t// Max deliver 1 so this will fail.\n\t_, err = sub.Fetch(1, nats.MaxWait(maxWait))\n\trequire_Error(t, err)\n\n\t// Redelivered should remain 0, as it doesn't get redelivered with MaxDeliver 1.\n\tcnfo, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, cnfo.NumAckPending, 0)\n\trequire_Equal(t, cnfo.NumRedelivered, 0)\n\n\t// With a higher MaxDeliver we should report it.\n\tcfg.MaxDeliver = 2\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tcnfo, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, cnfo.NumAckPending, 0)\n\trequire_Equal(t, cnfo.NumRedelivered, 1)\n\n\t// Unset should also report.\n\tcfg.MaxDeliver = -1\n\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tcnfo, err = js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, cnfo.NumAckPending, 0)\n\trequire_Equal(t, cnfo.NumRedelivered, 1)\n}\n\nfunc TestJetStreamRecoversStreamFirstSeqWhenNotEmpty(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tName: \"CONSUMER\",\n\t\tFilterSubject: \"test\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1000)\n\trequire_Equal(t, si.State.FirstSeq, 1)\n\trequire_Equal(t, si.State.LastSeq, 1000)\n\n\tps, err := js.PullSubscribe(\"test\", \"\", nats.Bind(\"TEST\", \"CONSUMER\"))\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 500; i++ {\n\t\tmsgs, err := ps.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(msgs), 1)\n\t\trequire_NoError(t, msgs[0].AckSync())\n\t}\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.AckFloor.Stream, 500)\n\n\ts.shutdownJetStream()\n\n\tpath := filepath.Join(s.opts.StoreDir, \"jetstream\", globalAccountName, \"streams\", \"TEST\", msgDir)\n\tdir, err := os.ReadDir(path)\n\trequire_NoError(t, err)\n\n\t// Mangle the last message in the block so that it fails the checksum comparison\n\t// with index.db, which causes us to throw the prior state error and rebuild.\n\t// Once this is done we should still be able to figure out what the previous first\n\t// and last sequence were, even without messages.\n\tfor _, f := range dir {\n\t\tif !strings.HasSuffix(f.Name(), \".blk\") {\n\t\t\tcontinue\n\t\t}\n\t\tst, err := f.Info()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, os.Truncate(filepath.Join(path, f.Name()), st.Size()-1))\n\t}\n\n\trequire_NoError(t, s.restartJetStream())\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 500)\n\trequire_Equal(t, si.State.FirstSeq, 501)\n\trequire_Equal(t, si.State.LastSeq, 1000)\n}\n\nfunc TestJetStreamRecoversStreamFirstSeqWhenEmpty(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.FileStorage,\n\t\tSubjects: []string{\"test\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_, err = js.Publish(\"test\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\trequire_NoError(t, js.PurgeStream(\"TEST\"))\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 1001)\n\trequire_Equal(t, si.State.LastSeq, 1000)\n\n\tci, err := js.ConsumerInfo(\"TEST\", \"CONSUMER\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.AckFloor.Stream, 1000)\n\n\ts.shutdownJetStream()\n\n\tpath := filepath.Join(s.opts.StoreDir, \"jetstream\", globalAccountName, \"streams\", \"TEST\", msgDir)\n\tdir, err := os.ReadDir(path)\n\trequire_NoError(t, err)\n\n\t// Mangle the last message in the block so that it fails the checksum comparison\n\t// with index.db, which causes us to throw the prior state error and rebuild.\n\t// Once this is done we should still be able to figure out what the previous first\n\t// and last sequence were, even without messages.\n\tfor _, f := range dir {\n\t\tif !strings.HasSuffix(f.Name(), \".blk\") {\n\t\t\tcontinue\n\t\t}\n\t\tst, err := f.Info()\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, os.Truncate(filepath.Join(path, f.Name()), st.Size()-1))\n\t}\n\n\ts.restartJetStream()\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 0)\n\trequire_Equal(t, si.State.FirstSeq, 1001)\n\trequire_Equal(t, si.State.LastSeq, 1000)\n}\n\nfunc TestJetStreamUpgradeStreamVersioning(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\n\t// Create stream config.\n\tcfg, apiErr := s.checkStreamCfg(&StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}}, acc, false)\n\trequire_True(t, apiErr == nil)\n\n\t// Create stream.\n\tmset, err := acc.addStream(&cfg)\n\trequire_NoError(t, err)\n\trequire_True(t, mset.cfg.Metadata == nil)\n\n\tfor _, create := range []bool{true, false} {\n\t\ttitle := \"create\"\n\t\tif !create {\n\t\t\ttitle = \"update\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\tif create {\n\t\t\t\t// Create on 2.11+ should be idempotent with previous create on 2.10-.\n\t\t\t\tmcfg := &StreamConfig{}\n\t\t\t\tsetStaticStreamMetadata(mcfg)\n\t\t\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: \"TEST\",\n\t\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\t\tMetadata: setDynamicStreamMetadata(mcfg).Metadata,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdeleteDynamicMetadata(si.Config.Metadata)\n\t\t\t\trequire_Len(t, len(si.Config.Metadata), 0)\n\t\t\t} else {\n\t\t\t\t// Update populates the versioning metadata.\n\t\t\t\tsi, err := js.UpdateStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(si.Config.Metadata), 3)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamUpgradeConsumerVersioning(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(globalAccountName)\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Create consumer config.\n\tcfg := &ConsumerConfig{Durable: \"CONSUMER\", Name: \"CONSUMER\"}\n\tselectedLimits, _, _, apiErr := acc.selectLimits(cfg.replicas(&mset.cfg))\n\tif apiErr != nil {\n\t\trequire_NoError(t, apiErr)\n\t}\n\tsrvLim := &s.getOpts().JetStreamLimits\n\tapiErr = setConsumerConfigDefaults(cfg, &mset.cfg, srvLim, selectedLimits, false)\n\tif apiErr != nil {\n\t\trequire_NoError(t, apiErr)\n\t}\n\n\t// Create consumer.\n\t_, err = mset.addConsumer(cfg)\n\trequire_NoError(t, err)\n\n\tfor _, create := range []bool{true, false} {\n\t\ttitle := \"create\"\n\t\tif !create {\n\t\t\ttitle = \"update\"\n\t\t}\n\t\tt.Run(title, func(t *testing.T) {\n\t\t\tcreateConsumerRequest := func(obsReq CreateConsumerRequest) (*JSApiConsumerInfoResponse, error) {\n\t\t\t\treq, err := json.Marshal(obsReq)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tmsg, err := nc.Request(fmt.Sprintf(JSApiDurableCreateT, \"TEST\", \"CONSUMER\"), req, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tvar resp JSApiConsumerInfoResponse\n\t\t\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\t\t\tif resp.Error != nil {\n\t\t\t\t\treturn nil, resp.Error\n\t\t\t\t}\n\t\t\t\treturn &resp, nil\n\t\t\t}\n\n\t\t\tif create {\n\t\t\t\t// Create on 2.11+ should be idempotent with previous create on 2.10-.\n\t\t\t\tncfg := &ConsumerConfig{Durable: \"CONSUMER\"}\n\t\t\t\tsetStaticConsumerMetadata(ncfg)\n\t\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\t\tStream: \"TEST\",\n\t\t\t\t\tConfig: *setDynamicConsumerMetadata(ncfg),\n\t\t\t\t\tAction: ActionCreate,\n\t\t\t\t}\n\t\t\t\tresp, err := createConsumerRequest(obsReq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdeleteDynamicMetadata(resp.Config.Metadata)\n\t\t\t\trequire_Len(t, len(resp.Config.Metadata), 0)\n\t\t\t} else {\n\t\t\t\t// Update populates the versioning metadata.\n\t\t\t\tobsReq := CreateConsumerRequest{\n\t\t\t\t\tStream: \"TEST\",\n\t\t\t\t\tConfig: ConsumerConfig{Durable: \"CONSUMER\"},\n\t\t\t\t\tAction: ActionUpdate,\n\t\t\t\t}\n\t\t\t\tresp, err := createConsumerRequest(obsReq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Len(t, len(resp.Config.Metadata), 3)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMirrorCrossAccountWithFilteredSubjectAndSubjectTransform(t *testing.T) {\n\tconf := createConfFile(t, fmt.Appendf(nil, `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tjetstream: {store_dir: %q}\n\t\taccounts: {\n\t\t\tPUBLIC_ACCOUNT: {\n\t\t\t\tjetstream: enabled\n\t\t\t\texports: [\n\t\t\t\t\t{ service: \"$JS.API.CONSUMER.CREATE.S.*.public.a\", response_type: stream, accounts: [\"INTERNAL_ACCOUNT\"] },\n\t\t\t\t\t{ service: \"$JS.API.CONSUMER.CREATE.S.*.public.b\", response_type: stream, accounts: [\"INTERNAL_ACCOUNT\"] },\n\t\t\t\t\t{ service: \"$JS.API.CONSUMER.CREATE.S.*.public.c\", response_type: stream, accounts: [\"INTERNAL_ACCOUNT\"] },\n\t\t\t\t\t{ service: \"$JS.FC.>\" },\n\t\t\t\t\t{ stream: \"shared.public.>\", accounts: [\"INTERNAL_ACCOUNT\"] }\n\t\t\t\t]\n\t\t\t\tusers: [ {user: \"public\", password: \"pwd\"} ]\n\t\t\t},\n\n\t\t\tINTERNAL_ACCOUNT: {\n\t\t\t\tjetstream: enabled\n\t\t\t\timports: [\n\t\t\t\t\t{ service: { account: \"PUBLIC_ACCOUNT\", subject: \"$JS.API.CONSUMER.CREATE.S.*.public.a\" }, to: \"JS.PUBLIC_ACCOUNT.CONSUMER.CREATE.S.*.public.a\" },\n\t\t\t\t\t{ service: { account: \"PUBLIC_ACCOUNT\", subject: \"$JS.API.CONSUMER.CREATE.S.*.public.b\" }, to: \"JS.PUBLIC_ACCOUNT.CONSUMER.CREATE.S.*.public.b\" },\n\t\t\t\t\t{ service: { account: \"PUBLIC_ACCOUNT\", subject: \"$JS.API.CONSUMER.CREATE.S.*.public.c\" }, to: \"JS.PUBLIC_ACCOUNT.CONSUMER.CREATE.S.*.public.c\" },\n\t\t\t\t\t{ service: { account: \"PUBLIC_ACCOUNT\", subject: \"$JS.FC.>\" }, to: \"$JS.FC.>\" },\n\t\t\t\t\t{ stream: { account: \"PUBLIC_ACCOUNT\", subject: \"shared.public.>\" }, to: \"shared.public.>\" }\n\t\t\t\t]\n\t\t\t\tusers: [ {user: \"internal\", password: \"pwd\"} ]\n\t\t\t}\n\t\t}\n\t`, t.TempDir()))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tpc, pjs := jsClientConnect(t, s, nats.UserInfo(\"public\", \"pwd\"))\n\tdefer pc.Close()\n\n\t_, err := pjs.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSubjects: []string{\"public.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tic, ijs := jsClientConnect(t, s, nats.UserInfo(\"internal\", \"pwd\"))\n\tdefer ic.Close()\n\n\taddStream := func(name, fs string, transform *nats.SubjectTransformConfig) {\n\t\tt.Helper()\n\t\tsc := &nats.StreamConfig{\n\t\t\tName: name,\n\t\t\tMirror: &nats.StreamSource{\n\t\t\t\tName: \"S\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"JS.PUBLIC_ACCOUNT\",\n\t\t\t\t\tDeliverPrefix: \"shared.public\",\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tif fs != _EMPTY_ {\n\t\t\tsc.Mirror.FilterSubject = fs\n\t\t} else {\n\t\t\tsc.Mirror.SubjectTransforms = append(sc.Mirror.SubjectTransforms, *transform)\n\t\t}\n\t\t_, err = ijs.AddStream(sc)\n\t\trequire_NoError(t, err)\n\t}\n\n\tcheckMirror := func(name string, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tsi, err := ijs.StreamInfo(name)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := int(si.State.Msgs); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %v mirrored message, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Just a subject filter\n\taddStream(\"M1\", \"public.a\", nil)\n\tnatsPub(t, pc, \"public.a\", []byte(\"hello\"))\n\tcheckMirror(\"M1\", 1)\n\n\t// Now try with equivalent subject transform (no destination).\n\taddStream(\"M2\", _EMPTY_, &nats.SubjectTransformConfig{Source: \"public.b\"})\n\tnatsPub(t, pc, \"public.b\", []byte(\"hello\"))\n\tcheckMirror(\"M2\", 1)\n\n\t// And now with a transform destination.\n\taddStream(\"M3\", _EMPTY_, &nats.SubjectTransformConfig{Source: \"public.c\", Destination: \"public.d\"})\n\tnatsPub(t, pc, \"public.c\", []byte(\"hello\"))\n\tcheckMirror(\"M3\", 1)\n\n\tcheckMsg := func(stream, subj string, seq uint64) {\n\t\tt.Helper()\n\t\tmsg, err := ijs.GetMsg(stream, seq)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, msg.Subject, subj)\n\t}\n\tcheckMsg(\"M1\", \"public.a\", 1)\n\tcheckMsg(\"M2\", \"public.b\", 2)\n\tcheckMsg(\"M3\", \"public.d\", 3)\n\n\tic.Close()\n\tpc.Close()\n\ts.Shutdown()\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tic, ijs = jsClientConnect(t, s, nats.UserInfo(\"internal\", \"pwd\"))\n\tdefer ic.Close()\n\n\tcheckMirror(\"M1\", 1)\n\tcheckMirror(\"M2\", 1)\n\tcheckMirror(\"M3\", 1)\n\tcheckMsg(\"M1\", \"public.a\", 1)\n\tcheckMsg(\"M2\", \"public.b\", 2)\n\tcheckMsg(\"M3\", \"public.d\", 3)\n\n\tpc, _ = jsClientConnect(t, s, nats.UserInfo(\"public\", \"pwd\"))\n\tdefer pc.Close()\n\tnatsPub(t, pc, \"public.a\", []byte(\"hello\"))\n\tnatsPub(t, pc, \"public.b\", []byte(\"hello\"))\n\tnatsPub(t, pc, \"public.c\", []byte(\"hello\"))\n\tnatsFlush(t, pc)\n\n\tcheckMirror(\"M1\", 2)\n\tcheckMirror(\"M2\", 2)\n\tcheckMirror(\"M3\", 2)\n\tcheckMsg(\"M1\", \"public.a\", 1)\n\tcheckMsg(\"M2\", \"public.b\", 2)\n\tcheckMsg(\"M3\", \"public.d\", 3)\n\tcheckMsg(\"M1\", \"public.a\", 4)\n\tcheckMsg(\"M2\", \"public.b\", 5)\n\tcheckMsg(\"M3\", \"public.d\", 6)\n}\n\nfunc TestJetStreamFileStoreFirstSeqAfterRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create a stream with a first sequence.\n\tfseq := uint64(10_000)\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.FileStorage,\n\t\tRetention: nats.LimitsPolicy,\n\t\tFirstSeq: fseq,\n\t})\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.FirstSeq, fseq)\n\trequire_Equal(t, si.State.LastSeq, fseq-1)\n\n\t// Publish one message to have some data in the stream.\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, fseq)\n\n\t// Confirm initial stream state.\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, fseq)\n\trequire_Equal(t, si.State.LastSeq, fseq)\n\n\t// Restart the server.\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\tnc.Close()\n\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Stream should come back up with the same state prior to restart.\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 1)\n\trequire_Equal(t, si.State.FirstSeq, fseq)\n\trequire_Equal(t, si.State.LastSeq, fseq)\n}\n\nfunc TestJetStreamCreateStreamWithSubjectDeleteMarkersOptions(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tcfg := StreamConfig{\n\t\tName: \"PEDANTIC\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"pedantic\"},\n\t\tSubjectDeleteMarkerTTL: -time.Millisecond,\n\t}\n\n\t_, err := addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"subject delete marker TTL must not be negative\"))\n\n\tcfg.SubjectDeleteMarkerTTL = time.Millisecond\n\t_, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"subject delete marker TTL must be at least 1 second\"))\n\n\tcfg.SubjectDeleteMarkerTTL = time.Second\n\t_, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"subject delete marker cannot be set if message TTLs are disabled\"))\n\n\tcfg.AllowMsgTTL = true\n\t_, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"subject delete marker cannot be set if roll-ups are disabled\"))\n\n\tcfg.AllowRollup = true\n\tcfg.DenyPurge = true\n\t_, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"roll-ups require the purge permission\"))\n\n\tcfg.DenyPurge = false\n\t_, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\n\t// Automatically setup pre-requisites for SDM.\n\tcfg = StreamConfig{\n\t\tName: \"AUTO\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"auto\"},\n\t\tSubjectDeleteMarkerTTL: time.Second,\n\t}\n\n\tsi, err := addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, false})\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\trequire_Equal(t, si.Config.SubjectDeleteMarkerTTL, time.Second)\n\trequire_True(t, si.Config.AllowMsgTTL)\n\trequire_True(t, si.Config.AllowRollup)\n\trequire_False(t, si.Config.DenyPurge)\n\n\t// Allow updating to use SDM and TTL.\n\tcfg = StreamConfig{\n\t\tName: \"UPDATE\",\n\t\tStorage: FileStorage,\n\t\tSubjects: []string{\"update\"},\n\t}\n\n\tsi, err = addStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, false})\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\trequire_False(t, si.Config.AllowMsgTTL)\n\trequire_False(t, si.Config.AllowRollup)\n\trequire_False(t, si.Config.DenyPurge)\n\n\tcfg.SubjectDeleteMarkerTTL = time.Second\n\tsi, err = updateStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, false})\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\trequire_Equal(t, si.Config.SubjectDeleteMarkerTTL, time.Second)\n\trequire_True(t, si.Config.AllowMsgTTL)\n\trequire_True(t, si.Config.AllowRollup)\n\trequire_False(t, si.Config.DenyPurge)\n\n\t// Should not be allowed to disable msg TTL.\n\tcfg = si.Config\n\tcfg.SubjectDeleteMarkerTTL = 0\n\tcfg.AllowMsgTTL = false\n\t_, err = updateStreamPedanticWithError(t, nc, &StreamConfigRequest{cfg, true})\n\trequire_Error(t, err, errors.New(\"message TTL status can not be disabled\"))\n}\n\nfunc TestJetStreamTHWExpireTasksRace(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tStorage: storageType,\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Send a bunch of message that need to be expired.\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(JSMessageTTL, \"1s\")\n\t\t\tfor i := 0; i < 10_000; i++ {\n\t\t\t\t_, err = js.PublishMsg(m)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Manually lock so that expirations can't be done without us unlocking.\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\tfs.mu.Lock()\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\tms.mu.Lock()\n\t\t\t}\n\n\t\t\t// Wait for all message TTLs to have expired.\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\n\t\t\t// Spawn a number of goroutines that will all want to lock and unlock the store lock.\n\t\t\tn := 50\n\t\t\tvar ready sync.WaitGroup\n\t\t\tvar wg sync.WaitGroup\n\t\t\tready.Add(n)\n\t\t\twg.Add(n)\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tready.Done()\n\t\t\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\t\t\tfs.expireMsgs()\n\t\t\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\t\t\tms.expireMsgs()\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\n\t\t\t// Wait for all goroutines to be ready.\n\t\t\tready.Wait()\n\n\t\t\t// Manually unlock so that goroutines can run expirations in parallel.\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\tfs.mu.Unlock()\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\tms.mu.Unlock()\n\t\t\t}\n\n\t\t\t// Wait for all goroutines to finish.\n\t\t\twg.Wait()\n\n\t\t\t// Run once more to clean up for removed messages.\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\tfs.expireMsgs()\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\tms.expireMsgs()\n\t\t\t}\n\n\t\t\t// Count of entries in the THW should be exactly 0, and not underflow.\n\t\t\tvar hwCount uint64\n\t\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\t\tfs.mu.Lock()\n\t\t\t\thwCount = fs.ttls.Count()\n\t\t\t\tfs.mu.Unlock()\n\t\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\t\tms.mu.Lock()\n\t\t\t\thwCount = ms.ttls.Count()\n\t\t\t\tms.mu.Unlock()\n\t\t\t}\n\t\t\trequire_Equal(t, hwCount, 0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamRejectLargePublishes(t *testing.T) {\n\ttdir := t.TempDir()\n\n\t// The test relies on the MaxPayload being larger than the\n\t// rlBadThresh, otherwise you can't publish a message large\n\t// enough.\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tmax_payload: %d\n\t\tjetstream: {store_dir: %q}\n\t`, rlBadThresh+2048, tdir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"test\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"test\", make([]byte, rlBadThresh-1024))\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"test\", make([]byte, rlBadThresh+1024))\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), ErrMsgTooLarge.Error())\n}\n\nfunc TestJetStreamDirectGetSubjectDeleteMarker(t *testing.T) {\n\tfor _, storageType := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t\tAllowDirect: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tm := nats.NewMsg(\"test\")\n\t\t\tm.Header.Set(JSMessageTTL, \"1s\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfirst, err := js.GetLastMsg(\"TEST\", \"test\", nats.DirectGet())\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, first.Header.Get(JSSequence), \"1\")\n\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\n\t\t\tsecond, err := js.GetLastMsg(\"TEST\", \"test\", nats.DirectGet())\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, second.Header.Get(JSSequence), \"2\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamPurgeExSeqSimple(t *testing.T) {\n\tfor _, storageType := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"test\"},\n\t\t\t\tStorage: storageType,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdata := make([]byte, 1024)\n\t\t\tfor i := 0; i < 10_000; i++ {\n\t\t\t\t_, err = js.Publish(\"test\", data)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 10_000)\n\n\t\t\trequire_NoError(t, js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 9_000}))\n\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 1_001)\n\t\t\trequire_Equal(t, si.State.NumDeleted, 0)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 9_000)\n\t\t\trequire_Equal(t, si.State.LastSeq, 10_000)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamPurgeExSeqInInteriorDeleteGap(t *testing.T) {\n\tfor _, storageType := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"test.*\"},\n\t\t\t\tStorage: storageType,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdata := make([]byte, 1024)\n\t\t\t_, err = js.Publish(\"test.start\", data)\n\t\t\trequire_NoError(t, err)\n\t\t\tfor i := 0; i < 10_000; i++ {\n\t\t\t\t_, err = js.Publish(\"test.mid\", data)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t_, err = js.Publish(\"test.end\", data)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 10_002)\n\n\t\t\trequire_NoError(t, js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Subject: \"test.mid\"}))\n\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 2)\n\t\t\trequire_Equal(t, si.State.NumDeleted, 10_000)\n\n\t\t\trequire_NoError(t, js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 9_000}))\n\n\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, si.State.Msgs, 1)\n\t\t\trequire_Equal(t, si.State.NumDeleted, 0)\n\t\t\trequire_Equal(t, si.State.FirstSeq, 10_002)\n\t\t\trequire_Equal(t, si.State.LastSeq, 10_002)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamDirectGetUpToTime(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tAllowDirect: true,\n\t\tStorage: nats.FileStorage,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := range 10 {\n\t\tsendStreamMsg(t, nc, \"foo\", fmt.Sprintf(\"message %d\", i+1))\n\t}\n\n\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(mreq)\n\t\trequire_NoError(t, err)\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req))\n\t\treturn sub\n\t}\n\n\tcheckResponses := func(t *testing.T, upToTime time.Time, expected ...string) {\n\t\tt.Helper()\n\t\tsub := sendRequest(&JSApiMsgGetRequest{MultiLastFor: []string{\"foo\"}, UpToTime: &upToTime})\n\t\tdefer sub.Unsubscribe()\n\t\tfor _, expect := range expected {\n\t\t\tmsg, err := sub.NextMsg(25 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Header.Get(JSSubject), \"foo\")\n\t\t\trequire_Equal(t, bytesToString(msg.Data), expect)\n\t\t}\n\t\t// By this time we're either at the end of our expected and looking\n\t\t// for an EOB marker (204) or we're not finding anything (404).\n\t\tmsg, err := sub.NextMsg(25 * time.Millisecond)\n\t\trequire_NoError(t, err)\n\t\tif len(expected) == 0 {\n\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\t\t} else {\n\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"204\")\n\t\t}\n\t}\n\n\tt.Run(\"DistantPast\", func(t *testing.T) {\n\t\tcheckResponses(t, time.Time{})\n\t})\n\n\tt.Run(\"DistantFuture\", func(t *testing.T) {\n\t\tcheckResponses(t, time.Unix(0, math.MaxInt64), \"message 10\")\n\t})\n\n\tt.Run(\"BeforeFirstSeq\", func(t *testing.T) {\n\t\tfirst, err := js.GetMsg(\"TEST\", 1)\n\t\trequire_NoError(t, err)\n\t\tcheckResponses(t, first.Time)\n\t})\n\n\tt.Run(\"BeforeFifthSeq\", func(t *testing.T) {\n\t\tfifth, err := js.GetMsg(\"TEST\", 5)\n\t\trequire_NoError(t, err)\n\t\tcheckResponses(t, fifth.Time, \"message 4\")\n\t})\n}\n\nfunc TestJetStreamDirectGetStartTimeSingleMsg(t *testing.T) {\n\tfor _, storage := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tAllowDirect: true,\n\t\t\t\tStorage: storage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsendStreamMsg(t, nc, \"foo\", \"message\")\n\n\t\t\tsendRequest := func(mreq *JSApiMsgGetRequest) *nats.Subscription {\n\t\t\t\tt.Helper()\n\t\t\t\treq, err := json.Marshal(mreq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\treply := nats.NewInbox()\n\t\t\t\tsub, err := nc.SubscribeSync(reply)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_NoError(t, nc.PublishRequest(\"$JS.API.DIRECT.GET.TEST\", reply, req))\n\t\t\t\treturn sub\n\t\t\t}\n\n\t\t\tfirst, err := js.GetMsg(\"TEST\", 1)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfuture := first.Time.Add(10 * time.Second)\n\t\t\tsub := sendRequest(&JSApiMsgGetRequest{StartTime: &future, NextFor: \"foo\", Batch: 1})\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\tmsg, err := sub.NextMsg(25 * time.Millisecond)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"404\")\n\t\t})\n\t}\n}\n\nfunc TestJetStreamStreamRetentionUpdatesConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, tc := range []struct {\n\t\tfrom RetentionPolicy\n\t\tto RetentionPolicy\n\t}{\n\t\t{LimitsPolicy, InterestPolicy},\n\t\t{InterestPolicy, LimitsPolicy},\n\t} {\n\t\tfrom, to, name := tc.from, tc.to, fmt.Sprintf(\"%sTo%s\", tc.from, tc.to)\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tsc, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: name,\n\t\t\t\tSubjects: []string{name},\n\t\t\t\tRetention: from,\n\t\t\t\tStorage: FileStorage,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.AddConsumer(name, &nats.ConsumerConfig{\n\t\t\t\tName: \"test_consumer\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmset, err := s.globalAccount().lookupStream(name)\n\t\t\trequire_NoError(t, err)\n\n\t\t\to := mset.lookupConsumer(\"test_consumer\")\n\t\t\trequire_NotNil(t, err)\n\t\t\trequire_Equal(t, o.retention, from)\n\n\t\t\tsc.Retention = to\n\t\t\t_, err = jsStreamUpdate(t, nc, sc)\n\t\t\trequire_NoError(t, err)\n\n\t\t\trequire_Equal(t, o.retention, to)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMaxMsgsPerSubjectAndDeliverLastPerSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tconst subjects = 300\n\tconst msgs = subjects * 10\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"foo.>\", \"bar.>\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tStorage: nats.FileStorage,\n\t\tMaxMsgsPerSubject: 5,\n\t})\n\trequire_NoError(t, err)\n\n\t// First, publish some messages that match the consumer filter. These\n\t// are the messages that we expect to consume a subset of. The random\n\t// sequences give us interior gaps after MaxMsgsPerSubject has been\n\t// enforced which is needed for this test to work.\n\tfor range msgs {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(subjects))\n\t\t_, err = js.Publish(subj, nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Then publish some messages on a different subject. These won't be\n\t// matched by the consumer filter.\n\tfor range msgs / 10 {\n\t\tsubj := fmt.Sprintf(\"bar.%d\", rand.Intn(subjects/10))\n\t\t_, err = js.Publish(subj, nil)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Add a deliver last per consumer that matches the first batch of\n\t// published messages only. We expect at this point that the skiplist\n\t// of the consumer will be for foo.> messages only, but the resume\n\t// sequence will be the stream last sequence at the time, i.e. the\n\t// last bar.> message.\n\t_, err = js.AddConsumer(\"test\", &nats.ConsumerConfig{\n\t\tName: \"test_consumer\",\n\t\tFilterSubjects: []string{\"foo.>\"},\n\t\tDeliverPolicy: nats.DeliverLastPerSubjectPolicy,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Take a copy of the skiplist and the resume value so that we can\n\t// make sure we receive all the messages we expect.\n\tmset, err := s.globalAccount().lookupStream(\"test\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"test_consumer\")\n\to.mu.RLock()\n\tpending := make(map[uint64]struct{}, len(o.lss.seqs))\n\tfor _, seq := range o.lss.seqs {\n\t\tpending[seq] = struct{}{}\n\t}\n\tresume := o.lss.resume\n\to.mu.RUnlock()\n\n\t// Now fetch the messages from the consumer.\n\tps, err := js.PullSubscribe(_EMPTY_, _EMPTY_, nats.Bind(\"test\", \"test_consumer\"))\n\trequire_NoError(t, err)\n\tfor range subjects {\n\t\tmsgs, err := ps.Fetch(1)\n\t\trequire_NoError(t, err)\n\t\tfor _, msg := range msgs {\n\t\t\tmeta, err := msg.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\t// We must be expecting this sequence and not have seen it already.\n\t\t\t// Once we've seen it, take it out of the map.\n\t\t\t_, ok := pending[meta.Sequence.Stream]\n\t\t\trequire_True(t, ok)\n\t\t\tdelete(pending, meta.Sequence.Stream)\n\t\t\t// Then ack.\n\t\t\trequire_NoError(t, msg.AckSync())\n\t\t}\n\t}\n\n\t// We should have received every message that was in the skiplist.\n\trequire_Len(t, len(pending), 0)\n\n\t// When we've run out of last sequences per subject, the consumer\n\t// should now continue from the resume seq, i.e. the last bar.> message.\n\to.mu.RLock()\n\tsseq := o.sseq\n\to.mu.RUnlock()\n\trequire_Equal(t, sseq, resume+1)\n}\n\nfunc TestJetStreamAllowMsgCounter(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tvar servers []*Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tservers = []*Server{s}\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\tservers = c.servers\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: false,\n\t\t\tReplicas: replicas,\n\t\t}\n\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// A normal publish will succeed, as it's not a counter.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Key\", \"Value\")\n\t\tpa, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pa.Sequence, 1)\n\n\t\t// Stream with disabled counters doesn't allow to publish counters.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageIncrDisabledError())\n\n\t\t// Enabling counters is not allowed.\n\t\tcfg.AllowMsgCounter = true\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change message counter setting\")))\n\n\t\t// Recreate stream with counters enabled.\n\t\trequire_NoError(t, js.DeleteStream(\"TEST\"))\n\t\t_, err = jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\t// Don't allow if missing counter increment.\n\t\tm = nats.NewMsg(\"foo\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageIncrMissingError())\n\n\t\tm.Header.Set(\"Key\", \"Value\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageIncrMissingError())\n\n\t\t// Don't allow if increment contains payload.\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tm.Data = []byte(\"data\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageIncrPayloadError())\n\n\t\t// Don't allow if counter increment is invalid.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"bogus\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageIncrInvalidError())\n\n\t\tvalidateTotal := func(seq uint64, total *big.Int) {\n\t\t\tt.Helper()\n\t\t\trsm, err := js.GetLastMsg(\"TEST\", \"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, rsm.Sequence, seq)\n\n\t\t\tvar val CounterValue\n\t\t\trequire_NoError(t, json.Unmarshal(rsm.Data, &val))\n\t\t\tvar res big.Int\n\t\t\tres.SetString(val.Value, 10)\n\t\t\trequire_Equal(t, res.Int64(), total.Int64())\n\t\t}\n\n\t\tincrement := func(incr string, seq uint64, total *big.Int) {\n\t\t\tt.Helper()\n\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Incr\", incr)\n\n\t\t\tmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar pubAck PubAck\n\t\t\trequire_NoError(t, json.Unmarshal(msg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, seq)\n\t\t\trequire_Equal(t, pubAck.Value, total.String())\n\t\t\tvalidateTotal(seq, total)\n\t\t}\n\n\t\t// Perform and check increments.\n\t\tincrement(\"1\", 1, big.NewInt(1))\n\t\tincrement(\"2\", 2, big.NewInt(3))\n\n\t\t// Can also decrement/reset the counter.\n\t\tincrement(\"-3\", 3, big.NewInt(0))\n\t\tincrement(\"1\", 4, big.NewInt(1))\n\n\t\t// Reset back to zero.\n\t\tincrement(\"-1\", 5, big.NewInt(0))\n\n\t\t// Check de-duplication.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tm.Header.Set(\"Nats-Msg-Id\", \"dedupe\")\n\t\tmsg, err := nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar pubAck1 PubAck\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &pubAck1))\n\t\trequire_False(t, pubAck1.Duplicate)\n\t\trequire_Equal(t, pubAck1.Sequence, 6)\n\t\trequire_Equal(t, pubAck1.Value, \"1\")\n\t\tvalidateTotal(6, big.NewInt(1))\n\n\t\t// Re-send should not up counter, but also not return current value.\n\t\t// When clustered this state can't be guaranteed.\n\t\tmsg, err = nc.RequestMsg(m, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar pubAck2 PubAck\n\t\trequire_NoError(t, json.Unmarshal(msg.Data, &pubAck2))\n\t\trequire_True(t, pubAck2.Duplicate)\n\t\trequire_Equal(t, pubAck2.Sequence, 6)\n\t\trequire_Equal(t, pubAck2.Value, _EMPTY_)\n\t\tvalidateTotal(6, big.NewInt(1))\n\n\t\t// Check rejected headers.\n\t\tfor _, header := range []string{JSMsgRollup, JSExpectedLastSeq, JSExpectedLastSubjSeq, JSExpectedLastSubjSeqSubj, JSExpectedLastMsgId} {\n\t\t\tm = nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\t\tm.Header.Set(header, \"1\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_Error(t, err, NewJSMessageIncrInvalidError())\n\t\t}\n\n\t\t// Manually break a counter in storage.\n\t\tfor _, cs := range servers {\n\t\t\tmset, err := cs.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tseq, _, err := mset.Store().StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, seq, 7)\n\t\t}\n\n\t\t// Should now error, because counter is broken.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSMessageCounterBrokenError())\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAllowMsgCounterMaxPayloadAndSize(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\ts.optsMu.Lock()\n\t\t\ts.opts.MaxPayload = 1024\n\t\t\ts.optsMu.Unlock()\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\tfor _, cs := range c.servers {\n\t\t\t\tcs.optsMu.Lock()\n\t\t\t\tcs.opts.MaxPayload = 1024\n\t\t\t\tcs.optsMu.Unlock()\n\t\t\t}\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\tcfg := &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err := jsStreamCreate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\n\t\tvalidateTotal := func(seq uint64, total *big.Int) {\n\t\t\tt.Helper()\n\t\t\trsm, err := js.GetLastMsg(\"TEST\", \"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, rsm.Sequence, seq)\n\n\t\t\tvar val CounterValue\n\t\t\trequire_NoError(t, json.Unmarshal(rsm.Data, &val))\n\t\t\tvar res big.Int\n\t\t\tres.SetString(val.Value, 10)\n\t\t\trequire_Equal(t, res.Int64(), total.Int64())\n\t\t}\n\n\t\tincrement := func(incr string, seq uint64, total *big.Int) {\n\t\t\tt.Helper()\n\t\t\tm := nats.NewMsg(\"foo\")\n\t\t\tm.Header.Set(\"Nats-Incr\", incr)\n\n\t\t\tmsg, err := nc.RequestMsg(m, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar pubAck PubAck\n\t\t\trequire_NoError(t, json.Unmarshal(msg.Data, &pubAck))\n\t\t\trequire_Equal(t, pubAck.Sequence, seq)\n\t\t\trequire_Equal(t, pubAck.Value, total.String())\n\t\t\tvalidateTotal(seq, total)\n\t\t}\n\n\t\t// Check payload exceeded, positive bound.\n\t\tincrement(\"1\", 1, big.NewInt(1))\n\t\ttooLargeIncrement := strings.Repeat(\"1\", 500)\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", tooLargeIncrement)\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSStreamMessageExceedsMaximumError())\n\n\t\t// Check payload exceeded, negative bound.\n\t\tincrement(\"-2\", 2, big.NewInt(-1))\n\t\tm.Header.Set(\"Nats-Incr\", fmt.Sprintf(\"-%s\", tooLargeIncrement))\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSStreamMessageExceedsMaximumError())\n\n\t\t// Reset back to zero.\n\t\tincrement(\"1\", 3, big.NewInt(0))\n\n\t\t// Exactly equals the size of the message for the first message.\n\t\tcfg.MaxMsgSize = 37\n\t\t_, err = jsStreamUpdate(t, nc, cfg)\n\t\trequire_NoError(t, err)\n\t\tincrement(\"1\", 4, big.NewInt(1))\n\n\t\t// Next increment bumps over MaxMsgSize limit defined above.\n\t\tm.Header.Set(\"Nats-Incr\", \"10\")\n\t\t_, err = js.PublishMsg(m)\n\t\trequire_Error(t, err, NewJSStreamMessageExceedsMaximumError())\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAllowMsgCounterIncompatibleSettings(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// DiscardNew not allowed.\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowMsgCounter: true,\n\t\tDiscard: DiscardNew,\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream cannot use discard new\")))\n\n\t// AllowMsgTTL not allowed.\n\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: FileStorage,\n\t\tAllowMsgCounter: true,\n\t\tAllowMsgTTL: true,\n\t})\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream cannot use message TTLs\")))\n\n\t// Only limits retention is allowed.\n\tfor _, retention := range []RetentionPolicy{InterestPolicy, WorkQueuePolicy} {\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tRetention: retention,\n\t\t})\n\t\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream can only use limits retention\")))\n\t}\n}\n\nfunc TestJetStreamAllowMsgCounterMirror(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Mirror with counters enabled is rejected.\n\t\tmirrorCfg := &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &StreamSource{Name: \"O\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err = jsStreamCreate(t, nc, mirrorCfg)\n\t\trequire_Error(t, err, NewJSMirrorWithCountersError())\n\n\t\t// Mirror with verbatim copying of counters.\n\t\tmirrorCfg.AllowMsgCounter = false\n\t\t_, err = jsStreamCreate(t, nc, mirrorCfg)\n\t\trequire_NoError(t, err)\n\n\t\t// A normal publish will succeed, as it's not a counter.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tpubAck, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Mirror should get message verbatim.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\trsm, err := js.GetMsg(\"M\", 1)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif incr := rsm.Header.Get(\"Nats-Incr\"); incr != \"1\" {\n\t\t\t\treturn fmt.Errorf(\"incorrect increment: %q\", incr)\n\t\t\t}\n\t\t\tvar count CounterValue\n\t\t\tif err := json.Unmarshal(rsm.Data, &count); err != nil {\n\t\t\t\treturn fmt.Errorf(\"JSON error: %w\", err)\n\t\t\t} else if count.Value != \"1\" {\n\t\t\t\treturn fmt.Errorf(\"unexpected value: %s\", rsm.Data)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAllowMsgCounterSourceAggregates(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O1\",\n\t\t\tSubjects: []string{\"foo.1\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O2\",\n\t\t\tSubjects: []string{\"foo.2\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Source will only work if counters are enabled on both streams.\n\t\tsourceCfg := &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tSources: []*StreamSource{\n\t\t\t\t{\n\t\t\t\t\tName: \"O1\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"O2\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err = jsStreamCreate(t, nc, sourceCfg)\n\t\trequire_NoError(t, err)\n\n\t\tm := nats.NewMsg(\"foo.1\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tpubAck, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\tm = nats.NewMsg(\"foo.2\")\n\t\tm.Header.Set(\"Nats-Incr\", \"2\")\n\t\tpubAck, err = js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Source should aggregate.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\trsm, err := js.GetMsg(\"M\", 2)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar count CounterValue\n\t\t\tif err := json.Unmarshal(rsm.Data, &count); err != nil {\n\t\t\t\treturn fmt.Errorf(\"JSON error: %w\", err)\n\t\t\t} else if count.Value != \"3\" {\n\t\t\t\treturn fmt.Errorf(\"unexpected value: %s\", rsm.Data)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAllowMsgCounterSourceVerbatim(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O1\",\n\t\t\tSubjects: []string{\"foo.1\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O2\",\n\t\t\tSubjects: []string{\"foo.2\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Source will only work if counters are enabled on both streams.\n\t\tsourceCfg := &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tSources: []*StreamSource{\n\t\t\t\t{\n\t\t\t\t\tName: \"O1\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"O2\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: false,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\t_, err = jsStreamCreate(t, nc, sourceCfg)\n\t\trequire_NoError(t, err)\n\n\t\tm := nats.NewMsg(\"foo.1\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tpubAck, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Source should store verbatim.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\trsm, err := js.GetMsg(\"M\", 1)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar count CounterValue\n\t\t\tif err := json.Unmarshal(rsm.Data, &count); err != nil {\n\t\t\t\treturn fmt.Errorf(\"JSON error: %w\", err)\n\t\t\t} else if count.Value != \"1\" {\n\t\t\t\treturn fmt.Errorf(\"unexpected value: %s\", rsm.Data)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\tm = nats.NewMsg(\"foo.2\")\n\t\tm.Header.Set(\"Nats-Incr\", \"2\")\n\t\tpubAck, err = js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Source should store verbatim.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\trsm, err := js.GetMsg(\"M\", 2)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar count CounterValue\n\t\t\tif err := json.Unmarshal(rsm.Data, &count); err != nil {\n\t\t\t\treturn fmt.Errorf(\"JSON error: %w\", err)\n\t\t\t} else if count.Value != \"2\" {\n\t\t\t\treturn fmt.Errorf(\"unexpected value: %s\", rsm.Data)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamAllowMsgCounterSourceStartingAboveZero(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O1\",\n\t\t\tSubjects: []string{\"foo.1\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t\tMaxMsgsPer: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O2\",\n\t\t\tSubjects: []string{\"foo.2\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t\tMaxMsgsPer: 1,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := range uint64(5) {\n\t\t\tm := nats.NewMsg(\"foo.1\")\n\t\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\t\tpubAck, err := js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, pubAck.Sequence, i+1)\n\n\t\t\tm = nats.NewMsg(\"foo.2\")\n\t\t\tm.Header.Set(\"Nats-Incr\", \"2\")\n\t\t\tpubAck, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, pubAck.Sequence, i+1)\n\t\t}\n\n\t\t// Source will only work if counters are enabled on both streams.\n\t\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Now make an addition locally on this stream too.\n\t\tm := nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tpubAck, err := js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Source will only work if counters are enabled on both streams.\n\t\t_, err = jsStreamUpdate(t, nc, &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tSources: []*StreamSource{\n\t\t\t\t{\n\t\t\t\t\tName: \"O1\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"O2\",\n\t\t\t\t\tSubjectTransforms: []SubjectTransformConfig{{Source: \"foo.*\", Destination: \"foo\"}},\n\t\t\t\t},\n\t\t\t},\n\t\t\tStorage: FileStorage,\n\t\t\tAllowMsgCounter: true,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\t// Source should aggregate.\n\t\tvar first, second, third, fourth *nats.RawStreamMsg\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfirst, err = js.GetMsg(\"M\", 1)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsecond, err = js.GetMsg(\"M\", 2)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tthird, err = js.GetMsg(\"M\", 3)\n\t\t\treturn err\n\t\t})\n\n\t\t// Now make an addition locally on this stream too.\n\t\tm = nats.NewMsg(\"foo\")\n\t\tm.Header.Set(\"Nats-Incr\", \"1\")\n\t\tpubAck, err = js.PublishMsg(m)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 4)\n\n\t\t// Fetch it back out of the stream.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tfourth, err = js.GetMsg(\"M\", 4)\n\t\t\treturn err\n\t\t})\n\n\t\t// There are no local additions to this counter, but the total\n\t\t// comprises changes from the sources.\n\t\tvar count CounterValue\n\t\trequire_NoError(t, json.Unmarshal(fourth.Data, &count))\n\t\trequire_Equal(t, count.Value, \"17\")\n\n\t\t// The most recent message should contain information about both\n\t\t// sources, so let's check.\n\t\tvar sources CounterSources\n\t\trequire_NoError(t, json.Unmarshal([]byte(fourth.Header.Get(JSMessageCounterSources)), &sources))\n\t\trequire_NotNil(t, sources[\"O1\"])\n\t\trequire_NotNil(t, sources[\"O2\"])\n\t\trequire_Equal(t, sources[\"O1\"][\"foo.1\"], \"5\")\n\t\trequire_Equal(t, sources[\"O2\"][\"foo.2\"], \"10\")\n\n\t\t// Since this is the first time we've seen a message from these\n\t\t// sources, the Nats-Incr header should have been updated to reflect\n\t\t// the correct delta.\n\t\tfor _, rsm := range []*nats.RawStreamMsg{first, second, third, fourth} {\n\t\t\trequire_Equal(t, rsm.Subject, \"foo\") // Subject transform'd\n\t\t\tswitch rsm {\n\t\t\tcase first:\n\t\t\t\trequire_Equal(t, rsm.Header.Get(JSMessageIncr), \"1\")\n\t\t\tcase second, third: // We can't know which order they got sourced in\n\t\t\t\tincr := rsm.Header.Get(JSMessageIncr)\n\t\t\t\trequire_True(t, incr == \"5\" || incr == \"10\")\n\t\t\tcase fourth:\n\t\t\t\trequire_Equal(t, rsm.Header.Get(JSMessageIncr), \"1\")\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamGetNoHeaders(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.FileStorage,\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := &nats.Msg{\n\t\tSubject: \"foo\",\n\t\tHeader: nats.Header{\n\t\t\t\"test\": []string{\"something\"},\n\t\t},\n\t\tData: []byte{1, 2, 3, 4, 5},\n\t}\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\tt.Run(\"MsgGet\", func(t *testing.T) {\n\t\tmsgGet := func(noHeaders bool) (payload []byte, hdrs nats.Header) {\n\t\t\tgetSubj := fmt.Sprintf(JSApiMsgGetT, \"TEST\")\n\t\t\treq := fmt.Appendf(nil, `{\"seq\":1,\"no_hdr\":%v}`, noHeaders)\n\t\t\tresp, err := nc.Request(getSubj, req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\tvar get JSApiMsgGetResponse\n\t\t\trequire_NoError(t, json.Unmarshal(resp.Data, &get))\n\t\t\tpayload = get.Message.Data\n\t\t\tif len(get.Message.Header) > 0 {\n\t\t\t\thdrs, err = nats.DecodeHeadersMsg(get.Message.Header)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tpayload, headers := msgGet(false)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), \"something\")\n\n\t\tpayload, headers = msgGet(true)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), _EMPTY_)\n\t})\n\n\tt.Run(\"DirectGet\", func(t *testing.T) {\n\t\tdirectGet := func(noHeaders bool) (payload []byte, hdrs nats.Header) {\n\t\t\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"TEST\")\n\t\t\treq := fmt.Appendf(nil, `{\"seq\":1,\"no_hdr\":%v}`, noHeaders)\n\t\t\tresp, err := nc.Request(getSubj, req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\treturn resp.Data, resp.Header\n\t\t}\n\n\t\tpayload, headers := directGet(false)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), \"something\")\n\n\t\tpayload, headers = directGet(true)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Stream\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Sequence\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Time-Stamp\"), _EMPTY_)\n\t})\n\n\tt.Run(\"DirectGetLastFor\", func(t *testing.T) {\n\t\tdirectGet := func(noHeaders bool) (payload []byte, hdrs nats.Header) {\n\t\t\tgetSubj := fmt.Sprintf(JSDirectGetLastBySubjectT, \"TEST\", \"foo\")\n\t\t\treq := fmt.Appendf(nil, `{\"no_hdr\":%v}`, noHeaders)\n\t\t\tresp, err := nc.Request(getSubj, req, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\treturn resp.Data, resp.Header\n\t\t}\n\n\t\tpayload, headers := directGet(false)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), \"something\")\n\n\t\tpayload, headers = directGet(true)\n\t\trequire_True(t, bytes.Equal(payload, msg.Data))\n\t\trequire_Equal(t, headers.Get(\"test\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Stream\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Sequence\"), _EMPTY_)\n\t\trequire_Equal(t, headers.Get(\"Nats-Time-Stamp\"), _EMPTY_)\n\t})\n}\n\nfunc TestJetStreamKVNoSubjectDeleteMarkerOnPurgeMarker(t *testing.T) {\n\tfor _, storage := range []jetstream.StorageType{jetstream.FileStorage, jetstream.MemoryStorage} {\n\t\tt.Run(storage.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnectNewAPI(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tctx := context.Background()\n\t\t\tkv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{\n\t\t\t\tBucket: \"bucket\",\n\t\t\t\tHistory: 1,\n\t\t\t\tStorage: storage,\n\t\t\t\tTTL: 2 * time.Second,\n\t\t\t\tLimitMarkerTTL: time.Minute,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tstream, err := js.Stream(ctx, \"KV_bucket\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Purge such that the bucket TTL expires this message.\n\t\t\trequire_NoError(t, kv.Purge(ctx, \"key\"))\n\t\t\trsm, err := stream.GetMsg(ctx, 1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, rsm.Header.Get(\"KV-Operation\"), \"PURGE\")\n\n\t\t\t// The bucket TTL should have removed the message by now.\n\t\t\ttime.Sleep(2500 * time.Millisecond)\n\n\t\t\t// Confirm the purge marker is gone.\n\t\t\t_, err = stream.GetMsg(ctx, 1)\n\t\t\trequire_Error(t, err, jetstream.ErrMsgNotFound)\n\t\t\trequire_Equal(t, rsm.Header.Get(\"KV-Operation\"), \"PURGE\")\n\n\t\t\t// Confirm we don't get a redundant subject delete marker.\n\t\t\t_, err = stream.GetMsg(ctx, 2)\n\t\t\trequire_Error(t, err, jetstream.ErrMsgNotFound)\n\n\t\t\t// Purge with a TTL so it expires this message.\n\t\t\trequire_NoError(t, kv.Purge(ctx, \"key\", jetstream.PurgeTTL(time.Second)))\n\t\t\trsm, err = stream.GetMsg(ctx, 2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, rsm.Header.Get(\"KV-Operation\"), \"PURGE\")\n\n\t\t\t// The purge TTL should have removed the message by now.\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\n\t\t\t// Confirm the purge marker is gone.\n\t\t\t_, err = stream.GetMsg(ctx, 2)\n\t\t\trequire_Error(t, err, jetstream.ErrMsgNotFound)\n\n\t\t\t// Confirm we don't get a redundant subject delete marker.\n\t\t\t_, err = stream.GetMsg(ctx, 3)\n\t\t\trequire_Error(t, err, jetstream.ErrMsgNotFound)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamInvalidConfigValues(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tacc := s.globalAccount()\n\n\t_, err := s.checkStreamCfg(&StreamConfig{Name: \"TEST\", Retention: -1}, acc, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"invalid retention\")))\n\n\t_, err = s.checkStreamCfg(&StreamConfig{Name: \"TEST\", Discard: -1}, acc, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"invalid discard policy\")))\n\n\t_, err = s.checkStreamCfg(&StreamConfig{Name: \"TEST\", Storage: -1}, acc, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"invalid storage type\")))\n\n\t_, err = s.checkStreamCfg(&StreamConfig{Name: \"TEST\", Compression: 255}, acc, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"invalid compression\")))\n\n\t_, err = s.checkStreamCfg(&StreamConfig{Name: \"TEST\", MaxAge: -time.Second}, acc, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(errors.New(\"max age can not be negative\")))\n\n\tscfg := StreamConfig{Name: \"TEST\"}\n\tstreamTests := []struct {\n\t\tname string\n\t\tsetValue func(value int)\n\t\tgetValue func() int\n\t}{\n\t\t{\n\t\t\tname: \"max_msgs\",\n\t\t\tsetValue: func(value int) { scfg.MaxMsgs = int64(value) },\n\t\t\tgetValue: func() int { return int(scfg.MaxMsgs) },\n\t\t},\n\t\t{\n\t\t\tname: \"max_msgs_per_subject\",\n\t\t\tsetValue: func(value int) { scfg.MaxMsgsPer = int64(value) },\n\t\t\tgetValue: func() int { return int(scfg.MaxMsgsPer) },\n\t\t},\n\t\t{\n\t\t\tname: \"max_bytes\",\n\t\t\tsetValue: func(value int) { scfg.MaxBytes = int64(value) },\n\t\t\tgetValue: func() int { return int(scfg.MaxBytes) },\n\t\t},\n\t\t{\n\t\t\tname: \"max_msg_size\",\n\t\t\tsetValue: func(value int) { scfg.MaxMsgSize = int32(value) },\n\t\t\tgetValue: func() int { return int(scfg.MaxMsgSize) },\n\t\t},\n\t\t{\n\t\t\tname: \"max_consumers\",\n\t\t\tsetValue: func(value int) { scfg.MaxConsumers = value },\n\t\t\tgetValue: func() int { return scfg.MaxConsumers },\n\t\t},\n\t}\n\tfor _, streamTest := range streamTests {\n\t\t// Pedantic errors if less than -1.\n\t\tstreamTest.setValue(-10)\n\t\t_, err = s.checkStreamCfg(&scfg, acc, true)\n\t\tif err == nil {\n\t\t\tt.Fatal(\"Expected error for pedantic mode\")\n\t\t}\n\t\trequire_Error(t, err, NewJSPedanticError(fmt.Errorf(\"%s must be set to -1\", streamTest.name)))\n\n\t\t// Pedantic defaults if zero-value.\n\t\tstreamTest.setValue(0)\n\t\tscfg, err = s.checkStreamCfg(&scfg, acc, true)\n\t\tif err != nil {\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_Equal(t, streamTest.getValue(), -1)\n\n\t\t// Non-pedantic defaults.\n\t\tstreamTest.setValue(-10)\n\t\tscfg, err = s.checkStreamCfg(&scfg, acc, false)\n\t\tif err != nil {\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_Equal(t, streamTest.getValue(), -1)\n\t}\n\n\tccfg := &ConsumerConfig{AckPolicy: -1}\n\terr = checkConsumerCfg(ccfg, &JSLimitOpts{}, &scfg, nil, &JetStreamAccountLimits{}, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSConsumerAckPolicyInvalidError())\n\n\tccfg = &ConsumerConfig{ReplayPolicy: -1}\n\terr = checkConsumerCfg(ccfg, &JSLimitOpts{}, &scfg, nil, &JetStreamAccountLimits{}, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSConsumerReplayPolicyInvalidError())\n\n\tccfg = &ConsumerConfig{AckWait: -time.Second}\n\terr = checkConsumerCfg(ccfg, &JSLimitOpts{}, &scfg, nil, &JetStreamAccountLimits{}, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSConsumerAckWaitNegativeError())\n\n\tccfg = &ConsumerConfig{BackOff: []time.Duration{-time.Second}}\n\terr = checkConsumerCfg(ccfg, &JSLimitOpts{}, &scfg, nil, &JetStreamAccountLimits{}, false)\n\trequire_True(t, err != nil)\n\trequire_Error(t, err, NewJSConsumerBackOffNegativeError())\n\n\tccfg = &ConsumerConfig{AckPolicy: AckExplicit}\n\tconsumerTests := []struct {\n\t\tname string\n\t\tsetValue func(value int)\n\t\tgetValue func() int\n\t\tdefaultValue int\n\t}{\n\t\t{\n\t\t\tname: \"max_deliver\",\n\t\t\tsetValue: func(value int) { ccfg.MaxDeliver = value },\n\t\t\tgetValue: func() int { return ccfg.MaxDeliver },\n\t\t\tdefaultValue: -1,\n\t\t},\n\t\t{\n\t\t\tname: \"max_waiting\",\n\t\t\tsetValue: func(value int) { ccfg.MaxWaiting = value },\n\t\t\tgetValue: func() int { return ccfg.MaxWaiting },\n\t\t\tdefaultValue: JSWaitQueueDefaultMax,\n\t\t},\n\t\t{\n\t\t\tname: \"max_batch\",\n\t\t\tsetValue: func(value int) { ccfg.MaxRequestBatch = value },\n\t\t\tgetValue: func() int { return ccfg.MaxRequestBatch },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"max_expires\",\n\t\t\tsetValue: func(value int) { ccfg.MaxRequestExpires = time.Duration(value) },\n\t\t\tgetValue: func() int { return int(ccfg.MaxRequestExpires) },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"max_bytes\",\n\t\t\tsetValue: func(value int) { ccfg.MaxRequestMaxBytes = value },\n\t\t\tgetValue: func() int { return ccfg.MaxRequestMaxBytes },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"idle_heartbeat\",\n\t\t\tsetValue: func(value int) { ccfg.Heartbeat = time.Duration(value) },\n\t\t\tgetValue: func() int { return int(ccfg.Heartbeat) },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"inactive_threshold\",\n\t\t\tsetValue: func(value int) { ccfg.InactiveThreshold = time.Duration(value) },\n\t\t\tgetValue: func() int { return int(ccfg.InactiveThreshold) },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"priority_timeout\",\n\t\t\tsetValue: func(value int) { ccfg.PinnedTTL = time.Duration(value) },\n\t\t\tgetValue: func() int { return int(ccfg.PinnedTTL) },\n\t\t\tdefaultValue: 0,\n\t\t},\n\t}\n\tfor _, consumerTest := range consumerTests {\n\t\t// Pedantic errors if less than -1.\n\t\tconsumerTest.setValue(-10)\n\t\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, true)\n\t\tif err == nil {\n\t\t\tt.Fatal(\"Expected error for pedantic mode\")\n\t\t}\n\t\tif consumerTest.defaultValue == -1 {\n\t\t\trequire_Error(t, err, NewJSPedanticError(fmt.Errorf(\"%s must be set to -1\", consumerTest.name)))\n\t\t} else {\n\t\t\trequire_Error(t, err, NewJSPedanticError(fmt.Errorf(\"%s must not be negative\", consumerTest.name)))\n\t\t}\n\n\t\t// Pedantic defaults if zero-value.\n\t\tconsumerTest.setValue(0)\n\t\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, true)\n\t\tif err != nil {\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_Equal(t, consumerTest.getValue(), consumerTest.defaultValue)\n\n\t\t// Non-pedantic defaults.\n\t\tconsumerTest.setValue(-10)\n\t\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, false)\n\t\tif err != nil {\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\trequire_Equal(t, consumerTest.getValue(), consumerTest.defaultValue)\n\t}\n\n\t// Pedantic errors if less than -1.\n\tccfg.MaxAckPending = -10\n\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, true)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error for pedantic mode\")\n\t}\n\trequire_Error(t, err, NewJSPedanticError(errors.New(\"max_ack_pending must be set to -1\")))\n\n\t// Pedantic defaults if zero-value.\n\tccfg.MaxAckPending = 0\n\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, true)\n\tif err != nil {\n\t\trequire_NoError(t, err)\n\t}\n\trequire_Equal(t, ccfg.MaxAckPending, JsDefaultMaxAckPending)\n\n\t// Non-pedantic defaults.\n\tccfg.MaxAckPending = -10\n\terr = setConsumerConfigDefaults(ccfg, &scfg, &JSLimitOpts{}, &JetStreamAccountLimits{}, false)\n\tif err != nil {\n\t\trequire_NoError(t, err)\n\t}\n\trequire_Equal(t, ccfg.MaxAckPending, -1)\n}\n\nfunc TestJetStreamPromoteMirrorDeletingOrigin(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tmirrorCfg := &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &StreamSource{Name: \"O\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\tmirrorCfg, err = jsStreamCreate(t, nc, mirrorCfg)\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Mirror should get message.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t_, err := js.GetMsg(\"M\", 1)\n\t\t\treturn err\n\t\t})\n\n\t\t// Trying to promote the mirror with a subject conflict should fail\n\t\t// because the origin stream is listening on that subject still.\n\t\tmirrorCfg.Mirror = nil\n\t\tmirrorCfg.Subjects = []string{\"foo\"}\n\t\t_, err = jsStreamUpdate(t, nc, mirrorCfg)\n\t\trequire_Error(t, err)\n\t\tapiErr, ok := err.(*ApiError)\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, apiErr.ErrCode, uint16(JSStreamSubjectOverlapErr))\n\n\t\t// But if we delete the stream, the subject conflict goes away...\n\t\terr = js.DeleteStream(\"O\")\n\t\trequire_NoError(t, err)\n\n\t\t// ... so now it should work.\n\t\tmirrorCfg, err = jsStreamUpdate(t, nc, mirrorCfg)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(mirrorCfg.Subjects), 1)\n\t\trequire_Equal(t, mirrorCfg.Mirror, nil)\n\n\t\t// Make sure the mirror state has gone away.\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"M\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.Mirror != nil {\n\t\t\t\treturn fmt.Errorf(\"expecting no mirror status, got %+v\", si.Mirror)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Now we should be able to publish into the newly promoted mirror.\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t\t// ... and confirm that it was received in the mirror stream.\n\t\t_, err = js.GetMsg(\"M\", 2)\n\t\trequire_NoError(t, err)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamPromoteMirrorUpdatingOrigin(t *testing.T) {\n\ttest := func(t *testing.T, replicas int) {\n\t\tvar s *Server\n\t\tif replicas == 1 {\n\t\t\ts = RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\t\t} else {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\t\t\ts = c.randomServer()\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\toriginCfg, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\tName: \"O\",\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tmirrorCfg := &StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &StreamSource{Name: \"O\"},\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\tmirrorCfg, err = jsStreamCreate(t, nc, mirrorCfg)\n\t\trequire_NoError(t, err)\n\n\t\tpubAck, err := js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t\t// Mirror should get message.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\t_, err := js.GetMsg(\"M\", 1)\n\t\t\treturn err\n\t\t})\n\n\t\t// Trying to promote the mirror with a subject conflict should fail\n\t\t// because the origin stream is listening on that subject still.\n\t\tmirrorCfg.Mirror = nil\n\t\tmirrorCfg.Subjects = []string{\"foo\"}\n\t\t_, err = jsStreamUpdate(t, nc, mirrorCfg)\n\t\trequire_Error(t, err)\n\t\tapiErr, ok := err.(*ApiError)\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, apiErr.ErrCode, uint16(JSStreamSubjectOverlapErr))\n\n\t\t// But if we change the subjects on the origin stream, the subject\n\t\t// conflict goes away...\n\t\toriginCfg.Subjects = []string{\"bar\"}\n\t\t_, err = jsStreamUpdate(t, nc, originCfg)\n\t\trequire_NoError(t, err)\n\n\t\t// ... so now it should work.\n\t\tmirrorCfg, err = jsStreamUpdate(t, nc, mirrorCfg)\n\t\trequire_NoError(t, err)\n\t\trequire_Len(t, len(mirrorCfg.Subjects), 1)\n\t\trequire_Equal(t, mirrorCfg.Mirror, nil)\n\n\t\t// Make sure the mirror state has gone away.\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"M\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.Mirror != nil {\n\t\t\t\treturn fmt.Errorf(\"expecting no mirror status, got %+v\", si.Mirror)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Publishing into the original stream should no longer mirror over\n\t\t// onto the mirror stream...\n\t\tpubAck, err = js.Publish(\"bar\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 2)\n\n\t\t// ... therefore a new publish to the mirror stream should also have\n\t\t// sequence 2 and not 3.\n\t\tpubAck, err = js.Publish(\"foo\", nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, pubAck.Sequence, 2)\n\t}\n\n\tt.Run(\"R1\", func(t *testing.T) { test(t, 1) })\n\tt.Run(\"R3\", func(t *testing.T) { test(t, 3) })\n}\n\nfunc TestJetStreamScheduledMirrorOrSource(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tMirror: &StreamSource{Name: \"M\"},\n\t\tAllowMsgSchedules: true,\n\t})\n\trequire_Error(t, err, NewJSMirrorWithMsgSchedulesError())\n\n\t_, err = jsStreamCreate(t, nc, &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: FileStorage,\n\t\tSources: []*StreamSource{{Name: \"S\"}},\n\t\tAllowMsgSchedules: true,\n\t})\n\trequire_Error(t, err, NewJSSourceWithMsgSchedulesError())\n}\n\nfunc TestJetStreamOfflineStreamAndConsumerAfterDowngrade(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tport := s.getOpts().Port\n\tsd := s.JetStreamConfig().StoreDir\n\n\t_, err := s.globalAccount().addStream(&StreamConfig{\n\t\tName: \"DowngradeStreamTest\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 1,\n\t\tMetadata: map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)},\n\t})\n\trequire_NoError(t, err)\n\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tofflineReason := fmt.Sprintf(\"unsupported - required API level: %d, current API level: %d\", math.MaxInt, JSApiLevel)\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"DowngradeStreamTest\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar si JSApiStreamInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &si))\n\trequire_NotNil(t, si.Error)\n\trequire_Error(t, si.Error, NewJSStreamOfflineReasonError(errors.New(offlineReason)))\n\n\tvar sn JSApiStreamNamesResponse\n\tmsg, err = nc.Request(JSApiStreams, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &sn))\n\trequire_Len(t, len(sn.Streams), 1)\n\trequire_Equal(t, sn.Streams[0], \"DowngradeStreamTest\")\n\n\tvar sl JSApiStreamListResponse\n\tmsg, err = nc.Request(JSApiStreamList, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &sl))\n\trequire_Len(t, len(sl.Streams), 0)\n\trequire_Len(t, len(sl.Missing), 1)\n\trequire_Equal(t, sl.Missing[0], \"DowngradeStreamTest\")\n\trequire_Len(t, len(sl.Offline), 1)\n\trequire_Equal(t, sl.Offline[\"DowngradeStreamTest\"], offlineReason)\n\n\tmset, err := s.globalAccount().lookupStream(\"DowngradeStreamTest\")\n\trequire_NoError(t, err)\n\trequire_True(t, mset.closed.Load())\n\trequire_Equal(t, mset.offlineReason, offlineReason)\n\trequire_NoError(t, mset.delete())\n\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\t_, err = s.globalAccount().addStream(&StreamConfig{\n\t\tName: \"DowngradeConsumerTest\",\n\t\tStorage: FileStorage,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\tmset, err = s.globalAccount().lookupStream(\"DowngradeConsumerTest\")\n\trequire_NoError(t, err)\n\t_, err = mset.addConsumer(&ConsumerConfig{\n\t\tName: \"DowngradeConsumerTest\",\n\t\tMetadata: map[string]string{\"_nats.req.level\": strconv.Itoa(math.MaxInt)},\n\t})\n\trequire_NoError(t, err)\n\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\n\tmset, err = s.globalAccount().lookupStream(\"DowngradeConsumerTest\")\n\trequire_NoError(t, err)\n\trequire_True(t, mset.closed.Load())\n\trequire_Equal(t, mset.offlineReason, \"stopped - unsupported consumer \\\"DowngradeConsumerTest\\\"\")\n\n\tobs := mset.getPublicConsumers()\n\trequire_Len(t, len(obs), 1)\n\trequire_True(t, obs[0].isClosed())\n\trequire_Equal(t, obs[0].offlineReason, offlineReason)\n\n\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumerInfoT, \"DowngradeConsumerTest\", \"DowngradeConsumerTest\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tvar ci JSApiConsumerInfoResponse\n\trequire_NoError(t, json.Unmarshal(msg.Data, &ci))\n\trequire_NotNil(t, ci.Error)\n\trequire_Error(t, ci.Error, NewJSConsumerOfflineReasonError(errors.New(offlineReason)))\n\n\tvar cn JSApiConsumerNamesResponse\n\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumersT, \"DowngradeConsumerTest\"), nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &cn))\n\trequire_Len(t, len(cn.Consumers), 1)\n\trequire_Equal(t, cn.Consumers[0], \"DowngradeConsumerTest\")\n\n\tvar cl JSApiConsumerListResponse\n\tmsg, err = nc.Request(fmt.Sprintf(JSApiConsumerListT, \"DowngradeConsumerTest\"), nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, json.Unmarshal(msg.Data, &cl))\n\trequire_Len(t, len(cl.Consumers), 0)\n\trequire_Len(t, len(cl.Missing), 1)\n\trequire_Equal(t, cl.Missing[0], \"DowngradeConsumerTest\")\n\trequire_Len(t, len(cl.Offline), 1)\n\trequire_Equal(t, cl.Offline[\"DowngradeConsumerTest\"], offlineReason)\n}\n\nfunc TestJetStreamPersistModeAsync(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t\tPersistMode: AsyncPersistMode,\n\t}\n\t_, err := jsStreamCreate(t, nc, cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is only supported on file storage\")))\n\n\tcfg.Storage = FileStorage\n\tsi, err := jsStreamCreate(t, nc, cfg)\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.PersistMode, AsyncPersistMode)\n\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfs := mset.store.(*fileStore)\n\tfs.mu.RLock()\n\tasyncFlush := fs.fcfg.AsyncFlush\n\tfs.mu.RUnlock()\n\trequire_True(t, asyncFlush)\n\n\tcfg.PersistMode = DefaultPersistMode\n\t_, err = jsStreamUpdate(t, nc, cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change persist mode\")))\n}\n\nfunc TestJetStreamRemoveTTLOnRemoveMsg(t *testing.T) {\n\tfor _, storageType := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil, nats.MsgTTL(time.Hour))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvalidateTTLCount := func(count uint64) {\n\t\t\t\tstore := mset.Store()\n\t\t\t\tswitch storageType {\n\t\t\t\tcase nats.FileStorage:\n\t\t\t\t\tfs := store.(*fileStore)\n\t\t\t\t\tfs.mu.RLock()\n\t\t\t\t\tdefer fs.mu.RUnlock()\n\t\t\t\t\trequire_Equal(t, fs.ttls.Count(), count)\n\t\t\t\tcase nats.MemoryStorage:\n\t\t\t\t\tms := store.(*memStore)\n\t\t\t\t\tms.mu.RLock()\n\t\t\t\t\tdefer ms.mu.RUnlock()\n\t\t\t\t\trequire_Equal(t, ms.ttls.Count(), count)\n\t\t\t\t}\n\t\t\t}\n\t\t\tvalidateTTLCount(1)\n\n\t\t\trequire_NoError(t, js.DeleteMsg(\"TEST\", 1))\n\t\t\tvalidateTTLCount(0)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamMessageTTLNotExpiring(t *testing.T) {\n\tfor _, storageType := range []nats.StorageType{nats.FileStorage, nats.MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tAllowMsgTTL: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Triggers the expiry timer once, and needs to be reset to trigger earlier.\n\t\t\t_, err = js.Publish(\"foo\", nil, nats.MsgTTL(time.Hour))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Storing messages with a TTL would continuously reset the timer.\n\t\t\tvar wg sync.WaitGroup\n\t\t\twg.Add(1)\n\t\t\tdefer wg.Wait()\n\t\t\tctx, cancel := context.WithCancel(context.Background())\n\t\t\tdefer cancel()\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tcase <-time.After(100 * time.Millisecond):\n\t\t\t\t\t\tttl := time.Hour.Nanoseconds()\n\t\t\t\t\t\tstore := mset.Store()\n\t\t\t\t\t\tstore.StoreMsg(\"foo\", nil, nil, ttl)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\t// The message should be expired timely.\n\t\t\tpubAck, err := js.Publish(\"foo\", nil, nats.MsgTTL(time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t_, err = js.GetMsg(\"TEST\", pubAck.Sequence)\n\t\t\t\tif err == nil {\n\t\t\t\t\treturn fmt.Errorf(\"message not removed yet\")\n\t\t\t\t}\n\t\t\t\tif !errors.Is(err, nats.ErrMsgNotFound) {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamScheduledMessageNotTriggering(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tAllowMsgSchedules: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdelay := func(d time.Duration) string {\n\t\t\t\treturn fmt.Sprintf(\"@at %s\", time.Now().Add(d).Format(time.RFC3339Nano))\n\t\t\t}\n\n\t\t\t// Triggers the schedule timer once, and needs to be reset to trigger earlier.\n\t\t\tm := nats.NewMsg(\"foo.schedule.first\")\n\t\t\tm.Header.Set(\"Nats-Schedule\", delay(time.Hour))\n\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.msg\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Storing messages with a schedule would continuously reset the timer.\n\t\t\tvar wg sync.WaitGroup\n\t\t\twg.Add(1)\n\t\t\tdefer wg.Wait()\n\t\t\tctx, cancel := context.WithCancel(context.Background())\n\t\t\tdefer cancel()\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tvar i int\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tcase <-time.After(100 * time.Millisecond):\n\t\t\t\t\t\ti++\n\t\t\t\t\t\tms := nats.NewMsg(fmt.Sprintf(\"foo.schedule.%d\", i))\n\t\t\t\t\t\tms.Header.Set(\"Nats-Schedule\", delay(time.Hour))\n\t\t\t\t\t\tms.Header.Set(\"Nats-Schedule-Target\", \"foo.msg\")\n\t\t\t\t\t\tjs.PublishMsg(ms)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\t// The message should be scheduled timely.\n\t\t\tm = nats.NewMsg(\"foo.schedule.validate\")\n\t\t\tm.Header.Set(\"Nats-Schedule\", delay(time.Second))\n\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.msg\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t\tpubAck, err := js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t_, err = js.GetMsg(\"TEST\", pubAck.Sequence)\n\t\t\t\tif err == nil {\n\t\t\t\t\treturn fmt.Errorf(\"message not removed yet\")\n\t\t\t\t}\n\t\t\t\tif !errors.Is(err, nats.ErrMsgNotFound) {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestJetStreamScheduledMessageNotDeactivated(t *testing.T) {\n\tfor _, storageType := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(storageType.String(), func(t *testing.T) {\n\t\t\ts := RunBasicJetStreamServer(t)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := jsStreamCreate(t, nc, &StreamConfig{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tSubjects: []string{\"foo.>\"},\n\t\t\t\tStorage: storageType,\n\t\t\t\tAllowMsgSchedules: true,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdelay := func(d time.Duration) string {\n\t\t\t\treturn fmt.Sprintf(\"@at %s\", time.Now().Add(d).Format(time.RFC3339Nano))\n\t\t\t}\n\n\t\t\t// Message should be scheduled.\n\t\t\tm := nats.NewMsg(\"foo.schedule\")\n\t\t\tm.Header.Set(\"Nats-Schedule\", delay(time.Second))\n\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.msg1\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t_, err = js.GetLastMsg(\"TEST\", \"foo.msg1\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// A message with a schedule is published.\n\t\t\tm = nats.NewMsg(\"foo.schedule\")\n\t\t\tm.Header.Set(\"Nats-Schedule\", delay(time.Second))\n\t\t\tm.Header.Set(\"Nats-Schedule-Target\", \"foo.msg2\")\n\t\t\t_, err = js.PublishMsg(m)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// But, a publish that is not a schedule should deactivate it.\n\t\t\t_, err = js.Publish(\"foo.schedule\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait for some time, and confirm the schedule wasn't triggered.\n\t\t\ttime.Sleep(1500 * time.Millisecond)\n\t\t\t_, err = js.GetLastMsg(\"TEST\", \"foo.msg2\")\n\t\t\trequire_Error(t, err, nats.ErrMsgNotFound)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamScheduledMessageParse(t *testing.T) {\n\t// @at \n\tt.Run(\"@at\", func(t *testing.T) {\n\t\tts := time.Now().UTC()\n\t\tsts, repeat, ok := parseMsgSchedule(fmt.Sprintf(\"@at %s\", ts.Format(time.RFC3339Nano)), _EMPTY_, 0)\n\t\trequire_True(t, ok)\n\t\trequire_False(t, repeat)\n\t\trequire_Equal(t, ts, sts)\n\t})\n\n\t// @every \n\tt.Run(\"@every\", func(t *testing.T) {\n\t\tnow := time.Now().UTC().Round(time.Second)\n\t\tsts, repeat, ok := parseMsgSchedule(\"@every 5s\", _EMPTY_, now.UnixNano())\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_Equal(t, now.Add(5*time.Second), sts)\n\n\t\t// A schedule on an interval should not spam loads of times if it hasn't run in a long while.\n\t\tnow = time.Now().UTC().Round(time.Second)\n\t\tsts, repeat, ok = parseMsgSchedule(\"@every 5s\", _EMPTY_, 0)\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_True(t, !sts.Before(now.Add(5*time.Second)))\n\n\t\t// A schedule can only run at least once every second.\n\t\t_, _, ok = parseMsgSchedule(\"@every 999ms\", _EMPTY_, 0)\n\t\trequire_False(t, ok)\n\t})\n\n\t// pattern\n\tt.Run(\"cron\", func(t *testing.T) {\n\t\tnow := time.Now().UTC().Round(time.Second)\n\t\tsts, repeat, ok := parseMsgSchedule(\"* * * * * *\", _EMPTY_, now.UnixNano())\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_Equal(t, now.Add(time.Second), sts)\n\n\t\t// A schedule based on a cron should run the earliest \"next\" second.\n\t\tnow = time.Now().UTC().Truncate(time.Second).Add(time.Second - time.Nanosecond)\n\t\tsts, repeat, ok = parseMsgSchedule(\"* * * * * *\", _EMPTY_, now.UnixNano())\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_Equal(t, now.Truncate(time.Second).Add(time.Second), sts)\n\n\t\t// A schedule based on cron should not spam loads of times if it hasn't run in a long while.\n\t\tnow = time.Now().UTC().Round(time.Second)\n\t\tsts, repeat, ok = parseMsgSchedule(\"* * * * * *\", _EMPTY_, 0)\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_True(t, !sts.Before(now.Add(time.Second)))\n\n\t\t// Predefined cron patterns, e.g. @hourly.\n\t\tfor _, p := range []struct {\n\t\t\tpattern string\n\t\t\tdelay func(time.Time) time.Time\n\t\t}{\n\t\t\t{pattern: \"@yearly\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(1, 0, 0)\n\t\t\t}},\n\t\t\t{pattern: \"@annually\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(1, 0, 0)\n\t\t\t}},\n\t\t\t{pattern: \"@monthly\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(0, 1, 0)\n\t\t\t}},\n\t\t\t{pattern: \"@weekly\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(0, 0, 7)\n\t\t\t}},\n\t\t\t{pattern: \"@daily\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(0, 0, 1)\n\t\t\t}},\n\t\t\t{pattern: \"@midnight\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.AddDate(0, 0, 1)\n\t\t\t}},\n\t\t\t{pattern: \"@hourly\", delay: func(t time.Time) time.Time {\n\t\t\t\treturn t.Add(time.Hour)\n\t\t\t}},\n\t\t} {\n\t\t\tt.Run(p.pattern, func(t *testing.T) {\n\t\t\t\t// Skip ahead \"current time\", so we can test the added delay is correct.\n\t\t\t\tnow = time.Date(time.Now().Year()+2, 1, 1, 0, 0, 0, 0, time.UTC)\n\t\t\t\t// Also ensure we're on a Sunday of this month, to test \"@weekly\".\n\t\t\t\tif p.pattern == \"@weekly\" {\n\t\t\t\t\tfor now.Weekday() != 0 {\n\t\t\t\t\t\tnow = now.AddDate(0, 0, 1)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tsts, repeat, ok = parseMsgSchedule(p.pattern, _EMPTY_, now.UnixNano())\n\t\t\t\trequire_True(t, ok)\n\t\t\t\trequire_True(t, repeat)\n\t\t\t\trequire_Equal(t, p.delay(now), sts)\n\t\t\t})\n\t\t}\n\t})\n\n\t// pattern with time zone\n\tt.Run(\"cron_tz\", func(t *testing.T) {\n\t\ttz := \"Europe/Amsterdam\"\n\t\tloc, err := time.LoadLocation(tz)\n\t\trequire_NoError(t, err)\n\n\t\tnow := time.Now().UTC().Round(time.Second)\n\t\tsts, repeat, ok := parseMsgSchedule(\"* * * * * *\", tz, now.UnixNano())\n\t\trequire_True(t, ok)\n\t\trequire_True(t, repeat)\n\t\trequire_Equal(t, now.In(loc).Add(time.Second).String(), sts.String())\n\t})\n}\n\nfunc TestJetStreamDirectGetBatchParallelWriteDeadlock(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: nats.FileStorage,\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfor range 2 {\n\t\trequire_NoError(t, mset.processJetStreamMsg(\"foo\", _EMPTY_, nil, nil, 0, 0, nil, false, true))\n\t}\n\n\t// We'll lock the message blocks such that we can't read, but NumPending should still function.\n\tfs := mset.store.(*fileStore)\n\tfs.lockAllMsgBlocks()\n\ttotal, validThrough, err := fs.NumPending(1, _EMPTY_, false)\n\trequire_NoError(t, err)\n\trequire_Equal(t, total, 2)\n\trequire_Equal(t, validThrough, 2)\n\n\t// We'll now run things in the following order:\n\t// - do a read through Direct Batch Get, which is blocked by the message blocks being locked\n\t// - do a write in parallel, which is blocked by the read to complete\n\t// - unlock the message blocks while both read and write goroutines are active\n\t// If there's no deadlock the read and write will complete, and 3 messages will end up in the stream.\n\tvar wg, read sync.WaitGroup\n\tread.Add(1)\n\twg.Add(1)\n\tgo func() {\n\t\tread.Done()\n\t\tmset.getDirectRequest(&JSApiMsgGetRequest{Seq: 1, Batch: 2}, _EMPTY_)\n\t}()\n\tgo func() {\n\t\t// Make sure we enter getDirectRequest first.\n\t\tread.Wait()\n\t\t<-time.After(100 * time.Millisecond)\n\t\twg.Done()\n\t\tmset.processJetStreamMsg(\"foo\", _EMPTY_, nil, nil, 0, 0, nil, false, true)\n\t}()\n\tgo func() {\n\t\t// Run some time after we've entered processJetStreamMsg above.\n\t\twg.Wait()\n\t\t<-time.After(100 * time.Millisecond)\n\t\tfs.unlockAllMsgBlocks()\n\t}()\n\tread.Wait()\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif msgs := mset.state().Msgs; msgs != 3 {\n\t\t\treturn fmt.Errorf(\"expected 3 msgs, got %d\", msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamReloadMetaCompact(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {\n\t\t\tmax_mem_store: 2MB\n\t\t\tmax_file_store: 8MB\n\t\t\tstore_dir: '%s'\n\t\t}\n\t`, storeDir)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\trequire_Equal(t, s.getOpts().JetStreamMetaCompact, 0)\n\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {\n\t\t\tmax_mem_store: 2MB\n\t\t\tmax_file_store: 8MB\n\t\t\tstore_dir: '%s'\n\t\t\tmeta_compact: 100\n\t\t}\n\t`, storeDir))\n\n\trequire_Equal(t, s.getOpts().JetStreamMetaCompact, 100)\n\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {\n\t\t\tmax_mem_store: 2MB\n\t\t\tmax_file_store: 8MB\n\t\t\tstore_dir: '%s'\n\t\t\tmeta_compact: 0\n\t\t}\n\t`, storeDir))\n\n\trequire_Equal(t, s.getOpts().JetStreamMetaCompact, 0)\n}\n\n// https://github.com/nats-io/nats-server/issues/7511\nfunc TestJetStreamImplicitRePublishAfterSubjectTransform(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"a.>\", \"c.>\"},\n\t\tSubjectTransform: &nats.SubjectTransformConfig{Source: \"a.>\", Destination: \"b.>\"},\n\t\tRePublish: &nats.RePublish{Destination: \">\"}, // Implicitly RePublish 'b.>'.\n\t}\n\t// Forms a cycle since the RePublish captures both 'a.>' and 'c.>'\n\t_, err := js.AddStream(cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for republish destination forms a cycle\")))\n\n\t// Doesn't form a cycle as 'a.>' is mapped to 'b.>'. A RePublish for '>' can be translated to 'b.>'.\n\tcfg.Subjects = []string{\"a.>\"}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := nc.SubscribeSync(\"b.>\")\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\n\t// The published message should be transformed and RePublished.\n\t_, err = js.Publish(\"a.hello\", nil)\n\trequire_NoError(t, err)\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Subject, \"b.hello\")\n\n\t// Forms a cycle since the implicit RePublish on 'b.>' is lost.\n\t// The RePublish would now mean publishing to 'c.>' which is a cycle.\n\tcfg.Subjects = []string{\"c.>\"}\n\t_, err = js.UpdateStream(cfg)\n\trequire_Error(t, err, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for republish destination forms a cycle\")))\n}\n\nfunc TestJetStreamStreamMirrorWithoutDuplicateWindow(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tDuplicates: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tpubAck, err := js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\trequire_False(t, pubAck.Duplicate)\n\n\tpubAck, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\trequire_True(t, pubAck.Duplicate)\n\n\ttime.Sleep(1200 * time.Millisecond)\n\tpubAck, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\trequire_False(t, pubAck.Duplicate)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\tDuplicates: 0,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tmset, err := s.globalAccount().lookupStream(\"M\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tstate := mset.state()\n\t\tif state.Msgs != 2 || state.FirstSeq != 1 || state.LastSeq != 2 {\n\t\t\treturn fmt.Errorf(\"incorrect state: %v\", state)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamStreamSourceWithoutDuplicateWindow(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tDuplicates: time.Second,\n\t})\n\trequire_NoError(t, err)\n\n\tpubAck, err := js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\trequire_False(t, pubAck.Duplicate)\n\n\tpubAck, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\trequire_True(t, pubAck.Duplicate)\n\n\ttime.Sleep(1200 * time.Millisecond)\n\tpubAck, err = js.Publish(\"foo\", nil, nats.MsgId(\"msgId\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 2)\n\trequire_False(t, pubAck.Duplicate)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"S\",\n\t\tSources: []*nats.StreamSource{{Name: \"TEST\"}},\n\t\tDuplicates: 0,\n\t})\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tmset, err := s.globalAccount().lookupStream(\"S\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tstate := mset.state()\n\t\tif state.Msgs != 2 || state.FirstSeq != 1 || state.LastSeq != 2 {\n\t\t\treturn fmt.Errorf(\"incorrect state: %v\", state)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamServerEncryptionRecoveryWithoutStreamStateFile(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tcstr string\n\t\tcipher StoreCipher\n\t}{\n\t\t{\"Default\", _EMPTY_, ChaCha},\n\t\t{\"ChaCha\", \", cipher: chacha\", ChaCha},\n\t\t{\"AES\", \", cipher: aes\", AES},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\t\tserver_name: S22\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tjetstream: {key: $JS_KEY, store_dir: '%s' %s}\n\t\t\t`\n\t\t\tstoreDir := t.TempDir()\n\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, c.cstr)))\n\n\t\t\tos.Setenv(\"JS_KEY\", \"s3cr3t!!\")\n\t\t\tdefer os.Unsetenv(\"JS_KEY\")\n\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tconfig := s.JetStreamConfig()\n\t\t\tif config == nil {\n\t\t\t\tt.Fatalf(\"Expected config but got none\")\n\t\t\t}\n\t\t\tdefer removeDir(t, config.StoreDir)\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t_, err = js.Publish(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tbefore := si.State\n\t\t\trequire_Equal(t, before.Msgs, 1)\n\t\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\t\trequire_Equal(t, before.LastSeq, 1)\n\n\t\t\tfor i := range 2 {\n\t\t\t\ts.Shutdown()\n\t\t\t\ts.WaitForShutdown()\n\t\t\t\t// Previously, the server would rely on this file to be present. If it wasn't, it would\n\t\t\t\t// not initialize the keys and erroneously regenerate the meta.key upon the next graceful\n\t\t\t\t// shutdown. A subsequent restart would not allow this stream to be loaded.\n\t\t\t\tif i == 0 {\n\t\t\t\t\tstateFile := filepath.Join(storeDir, JetStreamStoreDir, globalAccountName, streamsDir, \"TEST\", msgDir, streamStreamStateFile)\n\t\t\t\t\trequire_NoError(t, os.Remove(stateFile))\n\t\t\t\t}\n\n\t\t\t\ts, _ = RunServerWithConfig(conf)\n\t\t\t\tdefer s.Shutdown()\n\n\t\t\t\t// Reconnect.\n\t\t\t\tnc.Close()\n\t\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\t// Previously, the next iteration would fail by not finding the stream.\n\t\t\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif state := si.State; !reflect.DeepEqual(state, before) {\n\t\t\t\t\tt.Fatalf(\"Expected state\\n of %+v, \\ngot %+v without index.db state\", before, state)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamFileStoreErrorOpeningBlockAfterTruncate(t *testing.T) {\n\tstoreDir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream: {store_dir: %q}\n\t`, storeDir)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tpubAck, err := js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n\n\t// Shut down the server and manually truncate the message blocks to be entirely empty, simulating data loss.\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfs := mset.store.(*fileStore)\n\tblk := filepath.Join(fs.fcfg.StoreDir, msgDir, \"1.blk\")\n\tindex := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\tnc.Close()\n\ts.Shutdown()\n\n\t// Truncate the block such that it isn't fully empty, but doesn't contain any messages.\n\trequire_NoError(t, os.Truncate(blk, 1))\n\trequire_NoError(t, os.Remove(index))\n\n\t// Restart the server and reconnect.\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Publish another message. Due to the simulated data loss, the stream sequence should continue\n\t// counting after truncating the corrupted data.\n\tpubAck, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\trequire_Equal(t, pubAck.Sequence, 1)\n}\n\nfunc TestJetStreamSourceConfigValidation(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// Not testing with js.AddStream as passing it a nil source causes it to panic.\n\tmsg := nats.Msg{Subject: \"$JS.API.STREAM.CREATE.crash\", Data: []byte(`{\"name\":\"crash\",\"retention\":\"limits\",\"max_consumers\":-1,\"max_msgs_per_subject\":-1,\"max_msgs\":-1,\"max_bytes\":-1,\"max_age\":0,\"max_msg_size\":-1,\"storage\":\"file\",\"discard\":\"old\",\"num_replicas\":1,\"duplicate_window\":120000000000,\"sources\":[null],\"sealed\":false,\"deny_delete\":false,\"deny_purge\":false,\"allow_rollup_hdrs\":false,\"allow_direct\":true,\"mirror_direct\":false,\"consumer_limits\":{}}`)}\n\tresponse, err := nc.Request(msg.Subject, msg.Data, time.Second)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, string(response.Data), `{\"type\":\"io.nats.jetstream.api.v1.stream_create_response\",\"error\":{\"code\":400,\"err_code\":10141,\"description\":\"sourced stream name is invalid\"}}`)\n}\n\n// https://github.com/nats-io/nats-server/issues/6747\nfunc TestJetStreamCleanupNoInterestAboveThreshold(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"a\", \"b\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"A\", FilterSubject: \"a\"})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"B\", FilterSubject: \"b\"})\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tfor _, subj := range []string{\"a\", \"b\"} {\n\t\tfor range 100_000 {\n\t\t\t_, _, err = mset.store.StoreMsg(subj, nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, si.State.Msgs, 200_000)\n\n\t// Deleting the consumer will trigger deletion of the messages that now have no interest.\n\t// Since the total number of messages in the stream is above the threshold, this might take some time.\n\trequire_NoError(t, js.DeleteConsumer(\"TEST\", \"A\"))\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 100_000 {\n\t\t\treturn fmt.Errorf(\"expected 100k messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJetStreamStoreFilterIsAll(t *testing.T) {\n\ttest := func(t *testing.T, storage nats.StorageType) {\n\t\ts := RunBasicJetStreamServer(t)\n\t\tdefer s.Shutdown()\n\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"c\", \"b\", \"a\"},\n\t\t\tStorage: storage,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif fs, ok := mset.store.(*fileStore); ok {\n\t\t\trequire_True(t, fs.filterIsAll([]string{\"b\", \"a\", \"c\"}))\n\t\t} else if ms, ok := mset.store.(*memStore); ok {\n\t\t\trequire_True(t, ms.filterIsAll([]string{\"b\", \"a\", \"c\"}))\n\t\t} else {\n\t\t\tt.Fatal(\"unknown store type\")\n\t\t}\n\t}\n\tt.Run(\"Memory\", func(t *testing.T) { test(t, nats.MemoryStorage) })\n\tt.Run(\"File\", func(t *testing.T) { test(t, nats.FileStorage) })\n}\n\n// Test that flow control works correctly when a stream sources from two\n// different external accounts. Each account's $JS.FC.> service import\n// causes FC replies to fan out to all accounts, but only the correct\n// account's consumer handles the reply. This verifies the fan-out\n// doesn't prevent FC from completing.\nfunc TestJetStreamFlowControlCrossAccountFanOut(t *testing.T) {\n\tconf := createConfFile(t, fmt.Appendf(nil, `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tjetstream: {store_dir: %q}\n\t\taccounts: {\n\t\t\tACCT_A: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: usrA, password: pwd}]\n\t\t\t\texports: [\n\t\t\t\t\t{service: \"$JS.API.CONSUMER.>\"}\n\t\t\t\t\t{stream: \"RI.DELIVER.A.>\"}\n\t\t\t\t\t{service: \"$JS.FC.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tACCT_B: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: usrB, password: pwd}]\n\t\t\t\texports: [\n\t\t\t\t\t{service: \"$JS.API.CONSUMER.>\"}\n\t\t\t\t\t{stream: \"RI.DELIVER.B.>\"}\n\t\t\t\t\t{service: \"$JS.FC.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tSOURCER: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: src, password: pwd}]\n\t\t\t\timports: [\n\t\t\t\t\t{service: {account: ACCT_A, subject: \"$JS.API.CONSUMER.>\"}, to: \"RI.JS.A.API.CONSUMER.>\"}\n\t\t\t\t\t{stream: {account: ACCT_A, subject: \"RI.DELIVER.A.>\"}}\n\t\t\t\t\t{service: {account: ACCT_A, subject: \"$JS.FC.>\"}}\n\t\t\t\t\t{service: {account: ACCT_B, subject: \"$JS.API.CONSUMER.>\"}, to: \"RI.JS.B.API.CONSUMER.>\"}\n\t\t\t\t\t{stream: {account: ACCT_B, subject: \"RI.DELIVER.B.>\"}}\n\t\t\t\t\t{service: {account: ACCT_B, subject: \"$JS.FC.>\"}}\n\t\t\t\t]\n\t\t\t}\n\t\t\t$SYS {users: [{user: admin, password: pwd}]}\n\t\t}\n\t`, t.TempDir()))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create source streams in each account.\n\tncA, jsA := jsClientConnect(t, s, nats.UserInfo(\"usrA\", \"pwd\"))\n\tdefer ncA.Close()\n\t_, err := jsA.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"A\"}})\n\trequire_NoError(t, err)\n\n\tncB, jsB := jsClientConnect(t, s, nats.UserInfo(\"usrB\", \"pwd\"))\n\tdefer ncB.Close()\n\t_, err = jsB.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"B\"}})\n\trequire_NoError(t, err)\n\n\t// Publish 10 x 256KB messages to each source. Total 2.56MB exceeds the\n\t// initial FC threshold (maxpb/2 = 1MB), triggering at least one FC\n\t// round-trip per source.\n\tpayload := []byte(strings.Repeat(\"X\", 256*1024))\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = jsA.Publish(\"A\", payload)\n\t\trequire_NoError(t, err)\n\t\t_, err = jsB.Publish(\"B\", payload)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Create sourcing stream that pulls from both accounts.\n\tncS, jsS := jsClientConnect(t, s, nats.UserInfo(\"src\", \"pwd\"))\n\tdefer ncS.Close()\n\t_, err = jsS.AddStream(&nats.StreamConfig{\n\t\tName: \"DEST\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"RI.JS.A.API\",\n\t\t\t\t\tDeliverPrefix: \"RI.DELIVER.A\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"TEST\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"RI.JS.B.API\",\n\t\t\t\t\tDeliverPrefix: \"RI.DELIVER.B\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Wait for all 20 messages to arrive.\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := jsS.StreamInfo(\"DEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 20 {\n\t\t\treturn fmt.Errorf(\"expected 20 msgs, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Verify both sources converged with zero lag.\n\tsi, err := jsS.StreamInfo(\"DEST\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(si.Sources), 2)\n\tfor _, src := range si.Sources {\n\t\trequire_Equal(t, src.Lag, 0)\n\t}\n\n\t// Verify FC was exercised by checking that the internal consumers on\n\t// each source account ramped up their maxpb beyond the initial value.\n\tfor _, accName := range []string{\"ACCT_A\", \"ACCT_B\"} {\n\t\tacc, err := s.lookupAccount(accName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tconsumers := mset.getConsumers()\n\t\trequire_Equal(t, len(consumers), 1)\n\t\to := consumers[0]\n\t\to.mu.RLock()\n\t\tmaxpb := o.maxpb\n\t\tpblimit := o.pblimit\n\t\to.mu.RUnlock()\n\t\tif maxpb <= pblimit/16 {\n\t\t\tt.Fatalf(\"account %s: expected FC ramp-up (maxpb=%d > initial=%d)\", accName, maxpb, pblimit/16)\n\t\t}\n\t}\n}\n\nfunc TestJetStreamStreamCheckSourcesWithExternal(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"ORIGIN\"})\n\trequire_NoError(t, err)\n\n\t// Specifying both fields errors.\n\tsrc := &nats.StreamSource{\n\t\tName: \"ORIGIN\",\n\t\tFilterSubject: \"filter\",\n\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t{Source: \"filter\"},\n\t\t},\n\t}\n\tcfg := &nats.StreamConfig{Name: \"SOURCE\", Sources: []*nats.StreamSource{src}}\n\tfor _, external := range []*nats.ExternalStream{nil, {}} {\n\t\tsrc.External = external\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_Error(t, err, NewJSSourceMultipleFiltersNotAllowedError())\n\t}\n\n\t// Invalid source subject errors.\n\tsrc = &nats.StreamSource{\n\t\tName: \"ORIGIN\",\n\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t{Source: \".invalid\"},\n\t\t},\n\t}\n\tcfg = &nats.StreamConfig{Name: \"SOURCE\", Sources: []*nats.StreamSource{src}}\n\tfor _, external := range []*nats.ExternalStream{nil, {}} {\n\t\tsrc.External = external\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_Error(t, err, NewJSSourceInvalidSubjectFilterError(fmt.Errorf(\"%w %s\", ErrBadSubject, \".invalid\")))\n\t}\n\n\t// Invalid destination subject errors.\n\tsrc = &nats.StreamSource{\n\t\tName: \"ORIGIN\",\n\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t{Source: \"src\", Destination: \".invalid\"},\n\t\t},\n\t}\n\tcfg = &nats.StreamConfig{Name: \"SOURCE\", Sources: []*nats.StreamSource{src}}\n\tfor _, external := range []*nats.ExternalStream{nil, {}} {\n\t\tsrc.External = external\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_Error(t, err, NewJSSourceInvalidTransformDestinationError(ErrInvalidMappingDestinationSubject))\n\t}\n\n\t// Overlap errors.\n\tsrc = &nats.StreamSource{\n\t\tName: \"ORIGIN\",\n\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t{Source: \"src\"},\n\t\t\t{Source: \"src\"},\n\t\t},\n\t}\n\tcfg = &nats.StreamConfig{Name: \"SOURCE\", Sources: []*nats.StreamSource{src}}\n\tfor _, external := range []*nats.ExternalStream{nil, {}} {\n\t\tsrc.External = external\n\t\t_, err = js.AddStream(cfg)\n\t\trequire_Error(t, err, NewJSSourceOverlappingSubjectFiltersError())\n\t}\n}\n\nfunc TestJetStreamMirrorProcessMsgsNilQuitChannel(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Verify the mirror works.\n\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"MIRROR\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tmset, err := s.globalAccount().lookupStream(\"MIRROR\")\n\trequire_NoError(t, err)\n\n\t// Cancel the current mirror consumer and wait for its processMirrorMsgs\n\t// goroutine to fully exit.\n\tmset.mu.Lock()\n\tmset.cancelMirrorConsumer()\n\tmirror := mset.mirror\n\tmset.mu.Unlock()\n\tmirror.wg.Wait()\n\n\t// Now reproduce the race condition: start processMirrorMsgs while\n\t// mirror.qch is nil. This simulates what happens when cancelSourceInfo\n\t// runs between the goroutine being launched and acquiring mset.mu.\n\t//\n\t// We hold mset.mu while starting the goroutine so that it blocks on\n\t// mset.mu.Lock() inside processMirrorMsgs. When we release the lock,\n\t// it captures siqch = mirror.qch which is nil (left nil by cancelSourceInfo).\n\tmset.mu.Lock()\n\tmirror = mset.mirror\n\tmirror.msgs = newIPQueue[*inMsg](s, \"stream mirror\")\n\tmirror.wg.Add(1)\n\tready := sync.WaitGroup{}\n\tready.Add(1)\n\tif !s.startGoRoutine(func() { mset.processMirrorMsgs(mirror, &ready) }) {\n\t\tmirror.wg.Done()\n\t\tready.Done()\n\t\tmset.mu.Unlock()\n\t\tt.Fatal(\"Failed to start goroutine\")\n\t}\n\tmset.mu.Unlock()\n\tready.Wait()\n\n\t// Verify the goroutine exits promptly. With the bug, mirror.wg.Wait()\n\t// blocks forever because <-siqch on a nil channel never fires.\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tmirror.wg.Wait()\n\t\tclose(done)\n\t}()\n\n\tselect {\n\tcase <-done:\n\t\t// Expected goroutine to exit properly after cancel.\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Mirror goroutine did not exit after cancel: stuck on nil quit channel\")\n\t}\n}\n\nfunc TestJetStreamMirrorSetupStartGoRoutineFailMissingWgDone(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"MIRROR\",\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Verify the mirror works.\n\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"MIRROR\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 msg, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tmset, err := s.globalAccount().lookupStream(\"MIRROR\")\n\trequire_NoError(t, err)\n\n\t// Cancel the current mirror consumer and wait for its goroutine to exit.\n\tmset.mu.Lock()\n\tmset.cancelMirrorConsumer()\n\tmirror := mset.mirror\n\tmset.mu.Unlock()\n\tmirror.wg.Wait()\n\n\t// Simulate shutting down and not able to schedule more goroutines.\n\ts.grMu.Lock()\n\ts.grRunning = false\n\ts.grMu.Unlock()\n\n\t// Set up the mirror consumer which should not allow a new goroutine to be scheduled.\n\tmset.mu.Lock()\n\tmirror.lreq = time.Time{}\n\terr = mset.setupMirrorConsumer()\n\tmset.mu.Unlock()\n\trequire_NoError(t, err)\n\n\t// Give the above setup enough time to start the mirror goroutine.\n\ttime.Sleep(500 * time.Millisecond)\n\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tmirror.wg.Wait()\n\t\tclose(done)\n\t}()\n\n\tselect {\n\tcase <-done:\n\t\t// Expected goroutine to exit properly.\n\tcase <-time.After(2 * time.Second):\n\t\t// wg is stuck because Done() was never called.\n\t\t// Clean up the orphaned wg to prevent leaking the waiter goroutine.\n\t\tmirror.wg.Done()\n\t\tt.Fatal(\"mirror.wg.Wait() blocked indefinitely: missing wg.Done() in startGoRoutine failure path\")\n\t}\n}\n\nfunc TestJetStreamSourcingIntoDiscardNewPerSubject(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"A\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tDuplicates: 100 * time.Millisecond,\n\t})\n\trequire_NoError(t, err)\n\n\tsConfig := StreamConfig{\n\t\tName: \"B\",\n\t\tStorage: MemoryStorage,\n\t\tSources: []*StreamSource{{Name: \"A\"}},\n\t\tMaxMsgsPer: 1,\n\t\tDiscard: DiscardNew,\n\t\tDiscardNewPer: true,\n\t\tDuplicates: 10 * time.Second,\n\t}\n\n\treq, err := json.Marshal(&sConfig)\n\trequire_NoError(t, err)\n\n\tr, err := nc.Request(\"$JS.API.STREAM.CREATE.B\", req, 5*time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiConsumerCreateResponse\n\terr = json.Unmarshal(r.Data, &resp)\n\trequire_NoError(t, err)\n\trequire_Equal(t, resp.Error, nil)\n\n\t_, err = js.Publish(\"foo.1\", ([]byte)(\"1\"))\n\trequire_NoError(t, err)\n\n\t// this will not be sourced as discard new per subject\n\t_, err = js.Publish(\"foo.1\", ([]byte)(\"2\"))\n\trequire_NoError(t, err)\n\n\tvar si *nats.StreamInfo\n\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check the message\n\tmsgp, err := js.GetMsg(\"B\", uint64(1))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.1\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n\n\t// now purge the stream so sourcing can continue\n\trequire_NoError(t, js.PurgeStream(\"B\"))\n\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 1 {\n\t\t\treturn fmt.Errorf(\"expected 1 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// check the message\n\tmsgp, err = js.GetMsg(\"B\", uint64(2))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.1\")\n\trequire_Equal(t, string(msgp.Data), \"2\")\n\n\tmsg := nats.NewMsg(\"foo.2\")\n\tmsg.Data = []byte(\"1\")\n\tmsg.Header.Set(\"Nats-Msg-Id\", \"1\")\n\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// Must be able to move on and source the next message after the duplicate\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// check the message\n\tmsgp, err = js.GetMsg(\"B\", uint64(3))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.2\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n\n\ttime.Sleep(200 * time.Millisecond)\n\n\tmsg = nats.NewMsg(\"foo.3\")\n\tmsg.Data = []byte(\"1\")\n\tmsg.Header.Set(\"Nats-Msg-Id\", \"1\")\n\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// Duplicate message id, should get skipped\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 2 {\n\t\t\treturn fmt.Errorf(\"expected 2 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Must be able to move on\n\t_, err = js.Publish(\"foo.4\", ([]byte)(\"1\"))\n\trequire_NoError(t, err)\n\n\t// Must be able to move on and source the next message after the duplicate\n\tcheckFor(t, 4*time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = js.StreamInfo(\"B\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 3 {\n\t\t\treturn fmt.Errorf(\"expected 3 messages, got %d\", si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\tmsgp, err = js.GetMsg(\"B\", uint64(4))\n\trequire_NoError(t, err)\n\trequire_Equal(t, msgp.Subject, \"foo.4\")\n\trequire_Equal(t, string(msgp.Data), \"1\")\n\n}\n" }, { "path": "server/jetstream_tpm_test.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\n// The tests in this file are not complete, but are a start to test the TPM\n// with default settings. For a complete test we would need to use a TPM\n// configured with an owner password and a SRK password. This is not\n// typically the default or setup in CI/CD environments.\n\nvar jsTPMConfigPassword = `\n\tlisten: 127.0.0.1:-1\n\tjetstream: {\n\t\tstore_dir: %q\n\t\ttpm {\n\t\t\tkeys_file: %q\n\t\t\tencryption_password: \"s3cr3t!\"\n\t\t}\n\t}`\n\nvar jsTPMConfigBadPassword = `\n listen: 127.0.0.1:-1\n\tjetstream: {\n\t\tstore_dir: %q\n\t\ttpm {\n\t\t\tkeys_file: %q\n\t\t\tencryption_password: \"garbage\"\n\t\t}\n\t}`\n\nvar jsTPMConfigPasswordPcr = `\n\tlisten: 127.0.0.1:-1\n\tjetstream: {\n\t\tstore_dir: %q\n\t\ttpm {\n\t\t\tkeys_file: %q\n\t\t\tencryption_password: \"s3cr3t!\"\n\t\t\tpcr: %d\n\t\t}\n\t}`\n\nvar jsTPMConfigAllFields = `\n\tlisten: 127.0.0.1:-1\n\tjetstream: {\n\t\tstore_dir: %q\n\t\ttpm {\n\t\t\tkeys_file: %q\n\t\t\tencryption_password: \"s3cr3t!\"\n\t\t\tpcr: %d\n\t\t\tsrk_password: %q\n\t\t\tcipher: \"aes\"\n\t\t}\n\t}`\n\nvar jsTPMConfigInvalidBothOptionsSet = `\n\tlisten: 127.0.0.1:-1\n\tjetstream: {\n\t\tstore_dir: %q\n\t\tencryption_key: \"foo\"\n\t\ttpm {\n\t\t\tkeys_file: \"bar.json\"\n\t\t\tencryption_password: \"s3cr3t!\"\n\t\t}\n\t}`\n\nfunc getTPMFileName(t *testing.T) string {\n\treturn t.TempDir() + \"/\" + t.Name() + \"/jskeys.json\"\n}\n\nfunc checkSendMessage(t *testing.T, s *Server) {\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"tpm_test\",\n\t\tSubjects: []string{\"tpm_test\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"tpm_test\", []byte(\"hello\"))\n\trequire_NoError(t, err)\n}\n\nfunc checkReceiveMessage(t *testing.T, s *Server) {\n\t// reconnect and get the message\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// get the message\n\tsub, err := js.PullSubscribe(\"tpm_test\", \"cls\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tm := fetchMsgs(t, sub, 1, 5*time.Second)\n\tif m == nil {\n\t\tt.Fatalf(\"Did not receive the message\")\n\t}\n\tif len(m) != 1 {\n\t\tt.Fatalf(\"Expected 1 message, got %d\", len(m))\n\t}\n}\n\nfunc TestJetStreamTPMBasic(t *testing.T) {\n\tfileName := getTPMFileName(t)\n\tcf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPassword, t.TempDir(), fileName)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\t// Note, actual encryption is tested elsewhere. This just tests that the key is generated.\n\tkey := s.opts.JetStreamKey\n\tif !strings.HasPrefix(key, \"SU\") {\n\t\tt.Fatalf(\"expected a TPM key to be generated. Key = %q\", key)\n\t}\n\n\tif _, err := os.Stat(fileName); err != nil {\n\t\tt.Fatalf(\"keys file was not created\")\n\t}\n\n\tcheckSendMessage(t, s)\n\n\t// restart the server.\n\ts.Shutdown()\n\ts, _ = RunServerWithConfig(cf)\n\tif s.Running() == false {\n\t\tt.Fatalf(\"Server should be running\")\n\t}\n\tdefer s.Shutdown()\n\n\t// double check we're encrypted with the same key.\n\tif s.opts.JetStreamKey != key {\n\t\tt.Fatalf(\"expected same key\")\n\t}\n\tcheckReceiveMessage(t, s)\n}\n\n// Warning: Running this test too frequently will considerably slow down\n// these tests and eventually lock out the TPM.\n//\n// Depending on the environment this varies from 10 failures per day with\n// a 24 hour lockout (Dell) to 32 failures per day with a 10\n// minute healing time (decrementing one failure every 10 minutes).\nfunc TestJetStreamTPMKeyBadPassword(t *testing.T) {\n\tfileName := getTPMFileName(t)\n\tcf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPassword,\n\t\tt.TempDir(), fileName)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tcheckSendMessage(t, s)\n\ts.Shutdown()\n\n\t// restart the server.\n\tdefer func() {\n\t\tr := recover()\n\t\tif r == nil {\n\t\t\tt.Fatalf(\"Expected server panic, unexpected start.\")\n\t\t}\n\t}()\n\tcf2 := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigBadPassword,\n\t\tt.TempDir(), fileName)))\n\ts, _ = RunServerWithConfig(cf2)\n\ttime.Sleep(3 * time.Second)\n\tif s.Running() {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Server should NOT be running\")\n\t}\n}\n\nfunc TestJetStreamTPMKeyWithPCR(t *testing.T) {\n\tcf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigPasswordPcr,\n\t\tt.TempDir(), getTPMFileName(t), 18)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tcheckSendMessage(t, s)\n\ts.Shutdown()\n\n\t// restart the server\n\ts, _ = RunServerWithConfig(cf)\n\tif !s.Running() {\n\t\tt.Fatalf(\"Server should be running\")\n\t}\n}\n\nfunc TestJetStreamTPMAll(t *testing.T) {\n\tfileName := getTPMFileName(t)\n\n\t// TODO: When the CI/CD environment supports updating the TPM,\n\t// expand this test with the SRK password.\n\tcf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigAllFields,\n\t\tgetTPMFileName(t), fileName, 19, \"\")))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tcheckSendMessage(t, s)\n\ts.Shutdown()\n\n\t// restart the server.\n\ts, _ = RunServerWithConfig(cf)\n\tif s.Running() == false {\n\t\tt.Fatalf(\"Server should be running\")\n\t}\n\tdefer s.Shutdown()\n\tcheckReceiveMessage(t, s)\n}\n\nfunc TestJetStreamInvalidConfig(t *testing.T) {\n\tcf := createConfFile(t, []byte(fmt.Sprintf(jsTPMConfigInvalidBothOptionsSet,\n\t\tgetTPMFileName(t))))\n\n\tdefer func() {\n\t\tr := recover()\n\t\tif r == nil {\n\t\t\tt.Fatalf(\"Expected server panic, unexpected start.\")\n\t\t}\n\t}()\n\ts, _ := RunServerWithConfig(cf)\n\tif s.Running() == true {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Server should NOT be running\")\n\t}\n}\n" }, { "path": "server/jetstream_versioning.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport \"strconv\"\n\nconst (\n\t// JSApiLevel is the maximum supported JetStream API level for this server.\n\tJSApiLevel int = 4\n\n\tJSRequiredLevelMetadataKey = \"_nats.req.level\"\n\tJSServerVersionMetadataKey = \"_nats.ver\"\n\tJSServerLevelMetadataKey = \"_nats.level\"\n)\n\n// getRequiredApiLevel returns the required API level for the JetStream asset.\nfunc getRequiredApiLevel(metadata map[string]string) string {\n\tif l, ok := metadata[JSRequiredLevelMetadataKey]; ok && l != _EMPTY_ {\n\t\treturn l\n\t}\n\treturn _EMPTY_\n}\n\n// supportsRequiredApiLevel returns whether the required API level for the JetStream asset is supported.\nfunc supportsRequiredApiLevel(metadata map[string]string) bool {\n\tif l := getRequiredApiLevel(metadata); l != _EMPTY_ {\n\t\tli, err := strconv.Atoi(l)\n\t\treturn err == nil && li <= JSApiLevel\n\t}\n\treturn true\n}\n\n// setStaticStreamMetadata sets JetStream stream metadata, like the server version and API level.\n// Any dynamic metadata is removed, it must not be stored and only be added for responses.\nfunc setStaticStreamMetadata(cfg *StreamConfig) {\n\tif cfg.Metadata == nil {\n\t\tcfg.Metadata = make(map[string]string)\n\t} else {\n\t\tdeleteDynamicMetadata(cfg.Metadata)\n\t}\n\n\tvar requiredApiLevel int\n\trequires := func(level int) {\n\t\tif level > requiredApiLevel {\n\t\t\trequiredApiLevel = level\n\t\t}\n\t}\n\n\t// TTLs were added in v2.11 and require API level 1.\n\tif cfg.AllowMsgTTL || cfg.SubjectDeleteMarkerTTL > 0 {\n\t\trequires(1)\n\t}\n\n\t// Counter CRDTs were added in v2.12 and require API level 2.\n\tif cfg.AllowMsgCounter {\n\t\trequires(2)\n\t}\n\n\t// Atomic batch publishing was added in v2.12 and require API level 2.\n\tif cfg.AllowAtomicPublish {\n\t\trequires(2)\n\t}\n\n\t// Message scheduling was added in v2.12 and require API level 2.\n\tif cfg.AllowMsgSchedules {\n\t\trequires(2)\n\t}\n\n\t// Async persist mode was added in v2.12 and requires API level 2.\n\tif cfg.PersistMode == AsyncPersistMode {\n\t\trequires(2)\n\t}\n\n\t// Fast batch publishing was added in v2.14 and requires API level 4.\n\tif cfg.AllowBatchPublish {\n\t\trequires(4)\n\t}\n\n\tcfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)\n}\n\n// setDynamicStreamMetadata adds dynamic fields into the (copied) metadata.\nfunc setDynamicStreamMetadata(cfg *StreamConfig) *StreamConfig {\n\tvar newCfg StreamConfig\n\tif cfg != nil {\n\t\tnewCfg = *cfg\n\t}\n\tnewCfg.Metadata = make(map[string]string)\n\tif cfg != nil {\n\t\tfor key, value := range cfg.Metadata {\n\t\t\tnewCfg.Metadata[key] = value\n\t\t}\n\t}\n\tnewCfg.Metadata[JSServerVersionMetadataKey] = VERSION\n\tnewCfg.Metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel)\n\treturn &newCfg\n}\n\n// copyConsumerMetadata copies versioning fields from metadata of prevCfg into cfg.\n// Removes versioning fields if no previous metadata, updates if set, and removes fields if it doesn't exist in prevCfg.\n// Any dynamic metadata is removed, it must not be stored and only be added for responses.\n//\n// Note: useful when doing equality checks on cfg and prevCfg, but ignoring any versioning metadata differences.\nfunc copyStreamMetadata(cfg *StreamConfig, prevCfg *StreamConfig) {\n\tif cfg.Metadata != nil {\n\t\tdeleteDynamicMetadata(cfg.Metadata)\n\t}\n\tsetOrDeleteInStreamMetadata(cfg, prevCfg, JSRequiredLevelMetadataKey)\n}\n\n// setOrDeleteInConsumerMetadata sets field with key/value in metadata of cfg if set, deletes otherwise.\nfunc setOrDeleteInStreamMetadata(cfg *StreamConfig, prevCfg *StreamConfig, key string) {\n\tif prevCfg != nil && prevCfg.Metadata != nil {\n\t\tif value, ok := prevCfg.Metadata[key]; ok {\n\t\t\tif cfg.Metadata == nil {\n\t\t\t\tcfg.Metadata = make(map[string]string)\n\t\t\t}\n\t\t\tcfg.Metadata[key] = value\n\t\t\treturn\n\t\t}\n\t}\n\tdelete(cfg.Metadata, key)\n\tif len(cfg.Metadata) == 0 {\n\t\tcfg.Metadata = nil\n\t}\n}\n\n// setStaticConsumerMetadata sets JetStream consumer metadata, like the server version and API level.\n// Any dynamic metadata is removed, it must not be stored and only be added for responses.\nfunc setStaticConsumerMetadata(cfg *ConsumerConfig) {\n\tif cfg.Metadata == nil {\n\t\tcfg.Metadata = make(map[string]string)\n\t} else {\n\t\tdeleteDynamicMetadata(cfg.Metadata)\n\t}\n\n\tvar requiredApiLevel int\n\trequires := func(level int) {\n\t\tif level > requiredApiLevel {\n\t\t\trequiredApiLevel = level\n\t\t}\n\t}\n\n\t// Added in 2.11, absent | zero is the feature is not used.\n\t// one could be stricter and say even if its set but the time\n\t// has already passed it is also not needed to restore the consumer\n\tif cfg.PauseUntil != nil && !cfg.PauseUntil.IsZero() {\n\t\trequires(1)\n\t}\n\n\tif cfg.PriorityPolicy != PriorityNone || cfg.PinnedTTL != 0 || len(cfg.PriorityGroups) > 0 {\n\t\trequires(1)\n\t}\n\n\tcfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)\n}\n\n// setDynamicConsumerMetadata adds dynamic fields into the (copied) metadata.\nfunc setDynamicConsumerMetadata(cfg *ConsumerConfig) *ConsumerConfig {\n\tvar newCfg ConsumerConfig\n\tif cfg != nil {\n\t\tnewCfg = *cfg\n\t}\n\tnewCfg.Metadata = make(map[string]string)\n\tif cfg != nil {\n\t\tfor key, value := range cfg.Metadata {\n\t\t\tnewCfg.Metadata[key] = value\n\t\t}\n\t}\n\tnewCfg.Metadata[JSServerVersionMetadataKey] = VERSION\n\tnewCfg.Metadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel)\n\treturn &newCfg\n}\n\n// setDynamicConsumerInfoMetadata adds dynamic fields into the (copied) metadata.\nfunc setDynamicConsumerInfoMetadata(info *ConsumerInfo) *ConsumerInfo {\n\tif info == nil {\n\t\treturn nil\n\t}\n\n\tnewInfo := *info\n\tcfg := setDynamicConsumerMetadata(info.Config)\n\tnewInfo.Config = cfg\n\treturn &newInfo\n}\n\n// copyConsumerMetadata copies versioning fields from metadata of prevCfg into cfg.\n// Removes versioning fields if no previous metadata, updates if set, and removes fields if it doesn't exist in prevCfg.\n// Any dynamic metadata is removed, it must not be stored and only be added for responses.\n//\n// Note: useful when doing equality checks on cfg and prevCfg, but ignoring any versioning metadata differences.\nfunc copyConsumerMetadata(cfg *ConsumerConfig, prevCfg *ConsumerConfig) {\n\tif cfg.Metadata != nil {\n\t\tdeleteDynamicMetadata(cfg.Metadata)\n\t}\n\tsetOrDeleteInConsumerMetadata(cfg, prevCfg, JSRequiredLevelMetadataKey)\n}\n\n// setOrDeleteInConsumerMetadata sets field with key/value in metadata of cfg if set, deletes otherwise.\nfunc setOrDeleteInConsumerMetadata(cfg *ConsumerConfig, prevCfg *ConsumerConfig, key string) {\n\tif prevCfg != nil && prevCfg.Metadata != nil {\n\t\tif value, ok := prevCfg.Metadata[key]; ok {\n\t\t\tif cfg.Metadata == nil {\n\t\t\t\tcfg.Metadata = make(map[string]string)\n\t\t\t}\n\t\t\tcfg.Metadata[key] = value\n\t\t\treturn\n\t\t}\n\t}\n\tdelete(cfg.Metadata, key)\n\tif len(cfg.Metadata) == 0 {\n\t\tcfg.Metadata = nil\n\t}\n}\n\n// deleteDynamicMetadata deletes dynamic fields from the metadata.\nfunc deleteDynamicMetadata(metadata map[string]string) {\n\tdelete(metadata, JSServerVersionMetadataKey)\n\tdelete(metadata, JSServerLevelMetadataKey)\n}\n\n// errorOnRequiredApiLevel returns whether a request should be rejected based on the JSRequiredApiLevel header.\nfunc errorOnRequiredApiLevel(hdr []byte) bool {\n\treqApiLevel := sliceHeader(JSRequiredApiLevel, hdr)\n\tif len(reqApiLevel) == 0 {\n\t\treturn false\n\t}\n\tminLevel, err := strconv.Atoi(string(reqApiLevel))\n\treturn err != nil || JSApiLevel < minLevel\n}\n" }, { "path": "server/jetstream_versioning_test.go", "content": "// Copyright 2024-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_js_tests\n\npackage server\n\nimport (\n\t\"archive/tar\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestGetAndSupportsRequiredApiLevel(t *testing.T) {\n\trequire_Equal(t, getRequiredApiLevel(nil), _EMPTY_)\n\trequire_Equal(t, getRequiredApiLevel(map[string]string{}), _EMPTY_)\n\trequire_Equal(t, getRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: \"1\"}), \"1\")\n\trequire_Equal(t, getRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: \"text\"}), \"text\")\n\n\trequire_True(t, supportsRequiredApiLevel(nil))\n\trequire_True(t, supportsRequiredApiLevel(map[string]string{}))\n\trequire_True(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: \"1\"}))\n\trequire_True(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: strconv.Itoa(JSApiLevel)}))\n\trequire_False(t, supportsRequiredApiLevel(map[string]string{JSRequiredLevelMetadataKey: \"text\"}))\n}\n\nfunc metadataAtLevel(featureLevel string) map[string]string {\n\treturn map[string]string{\n\t\tJSRequiredLevelMetadataKey: featureLevel,\n\t}\n}\n\nfunc metadataPrevious() map[string]string {\n\treturn map[string]string{\n\t\tJSRequiredLevelMetadataKey: \"previous-level\",\n\t}\n}\n\nfunc TestJetStreamSetStaticStreamMetadata(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tdesc string\n\t\tcfg *StreamConfig\n\t\texpectedMetadata map[string]string\n\t}{\n\t\t{\n\t\t\tdesc: \"empty\",\n\t\t\tcfg: &StreamConfig{},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"overwrite-user-provided\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataPrevious()},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"empty-prev-metadata/delete-user-provided\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataPrevious()},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AllowMsgTTL\",\n\t\t\tcfg: &StreamConfig{AllowMsgTTL: true},\n\t\t\texpectedMetadata: metadataAtLevel(\"1\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"SubjectDeleteMarkerTTL\",\n\t\t\tcfg: &StreamConfig{SubjectDeleteMarkerTTL: time.Second},\n\t\t\texpectedMetadata: metadataAtLevel(\"1\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AllowMsgCounter\",\n\t\t\tcfg: &StreamConfig{AllowMsgCounter: true},\n\t\t\texpectedMetadata: metadataAtLevel(\"2\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AllowAtomicPublish\",\n\t\t\tcfg: &StreamConfig{AllowAtomicPublish: true},\n\t\t\texpectedMetadata: metadataAtLevel(\"2\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AllowMsgSchedules\",\n\t\t\tcfg: &StreamConfig{AllowMsgSchedules: true},\n\t\t\texpectedMetadata: metadataAtLevel(\"2\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AsyncPersistMode\",\n\t\t\tcfg: &StreamConfig{PersistMode: AsyncPersistMode},\n\t\t\texpectedMetadata: metadataAtLevel(\"2\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"AllowBatchPublish\",\n\t\t\tcfg: &StreamConfig{AllowBatchPublish: true},\n\t\t\texpectedMetadata: metadataAtLevel(\"4\"),\n\t\t},\n\t} {\n\t\tt.Run(test.desc, func(t *testing.T) {\n\t\t\tsetStaticStreamMetadata(test.cfg)\n\t\t\tlevel := test.cfg.Metadata[JSRequiredLevelMetadataKey]\n\t\t\trequire_Equal(t, level, test.expectedMetadata[JSRequiredLevelMetadataKey])\n\n\t\t\t// Ensure we up the server API level if we introduced a feature that requires it.\n\t\t\tl, err := strconv.Atoi(level)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, l <= JSApiLevel)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSetStaticStreamMetadataRemoveDynamicFields(t *testing.T) {\n\tdynamicMetadata := func() map[string]string {\n\t\treturn map[string]string{\n\t\t\tJSServerVersionMetadataKey: \"dynamic-version\",\n\t\t\tJSServerLevelMetadataKey: \"dynamic-version\",\n\t\t}\n\t}\n\n\tcfg := StreamConfig{Metadata: dynamicMetadata()}\n\tsetStaticStreamMetadata(&cfg)\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel(\"0\")))\n}\n\nfunc TestJetStreamSetDynamicStreamMetadata(t *testing.T) {\n\tcfg := StreamConfig{Metadata: metadataAtLevel(\"0\")}\n\tnewCfg := setDynamicStreamMetadata(&cfg)\n\n\t// Only new metadata must contain dynamic fields.\n\tmetadata := metadataAtLevel(\"0\")\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadata))\n\tmetadata[JSServerVersionMetadataKey] = VERSION\n\tmetadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel)\n\trequire_True(t, reflect.DeepEqual(newCfg.Metadata, metadata))\n}\n\nfunc TestJetStreamCopyStreamMetadata(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tdesc string\n\t\tcfg *StreamConfig\n\t\tprev *StreamConfig\n\t}{\n\t\t{\n\t\t\tdesc: \"no-previous-ignore\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: nil,\n\t\t},\n\t\t{\n\t\t\tdesc: \"nil-previous-metadata-ignore\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &StreamConfig{Metadata: nil},\n\t\t},\n\t\t{\n\t\t\tdesc: \"nil-current-metadata-ignore\",\n\t\t\tcfg: &StreamConfig{Metadata: nil},\n\t\t\tprev: &StreamConfig{Metadata: metadataPrevious()},\n\t\t},\n\t\t{\n\t\t\tdesc: \"copy-previous\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &StreamConfig{Metadata: metadataPrevious()},\n\t\t},\n\t\t{\n\t\t\tdesc: \"delete-missing-fields\",\n\t\t\tcfg: &StreamConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &StreamConfig{Metadata: make(map[string]string)},\n\t\t},\n\t} {\n\t\tt.Run(test.desc, func(t *testing.T) {\n\t\t\tcopyStreamMetadata(test.cfg, test.prev)\n\n\t\t\tvar expectedMetadata map[string]string\n\t\t\tif test.prev != nil {\n\t\t\t\texpectedMetadata = test.prev.Metadata\n\t\t\t}\n\n\t\t\tvalue, ok := expectedMetadata[JSRequiredLevelMetadataKey]\n\t\t\tif ok {\n\t\t\t\trequire_Equal(t, test.cfg.Metadata[JSRequiredLevelMetadataKey], value)\n\t\t\t} else {\n\t\t\t\t// Key shouldn't exist.\n\t\t\t\t_, ok = test.cfg.Metadata[JSRequiredLevelMetadataKey]\n\t\t\t\trequire_False(t, ok)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamCopyStreamMetadataRemoveDynamicFields(t *testing.T) {\n\tdynamicMetadata := func() map[string]string {\n\t\treturn map[string]string{\n\t\t\tJSServerVersionMetadataKey: \"dynamic-version\",\n\t\t\tJSServerLevelMetadataKey: \"dynamic-version\",\n\t\t}\n\t}\n\n\tcfg := StreamConfig{Metadata: dynamicMetadata()}\n\tcopyStreamMetadata(&cfg, nil)\n\trequire_Equal(t, len(cfg.Metadata), 0)\n\n\tcfg = StreamConfig{Metadata: dynamicMetadata()}\n\tprevCfg := StreamConfig{Metadata: metadataAtLevel(\"0\")}\n\tcopyStreamMetadata(&cfg, &prevCfg)\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel(\"0\")))\n}\n\nfunc TestJetStreamSetStaticConsumerMetadata(t *testing.T) {\n\tpauseUntil := time.Unix(0, 0)\n\tpauseUntilZero := time.Time{}\n\tfor _, test := range []struct {\n\t\tdesc string\n\t\tcfg *ConsumerConfig\n\t\texpectedMetadata map[string]string\n\t}{\n\t\t{\n\t\t\tdesc: \"empty\",\n\t\t\tcfg: &ConsumerConfig{},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"overwrite-user-provided\",\n\t\t\tcfg: &ConsumerConfig{Metadata: metadataPrevious()},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"PauseUntil/zero\",\n\t\t\tcfg: &ConsumerConfig{PauseUntil: &pauseUntilZero},\n\t\t\texpectedMetadata: metadataAtLevel(\"0\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"PauseUntil\",\n\t\t\tcfg: &ConsumerConfig{PauseUntil: &pauseUntil},\n\t\t\texpectedMetadata: metadataAtLevel(\"1\"),\n\t\t},\n\t\t{\n\t\t\tdesc: \"Pinned\",\n\t\t\tcfg: &ConsumerConfig{PriorityPolicy: PriorityPinnedClient, PriorityGroups: []string{\"a\"}},\n\t\t\texpectedMetadata: metadataAtLevel(\"1\"),\n\t\t},\n\t} {\n\t\tt.Run(test.desc, func(t *testing.T) {\n\t\t\tsetStaticConsumerMetadata(test.cfg)\n\t\t\tlevel := test.cfg.Metadata[JSRequiredLevelMetadataKey]\n\t\t\trequire_Equal(t, level, test.expectedMetadata[JSRequiredLevelMetadataKey])\n\n\t\t\t// Ensure we up the server API level if we introduced a feature that requires it.\n\t\t\tl, err := strconv.Atoi(level)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, l <= JSApiLevel)\n\t\t})\n\t}\n}\n\nfunc TestJetStreamSetStaticConsumerMetadataRemoveDynamicFields(t *testing.T) {\n\tdynamicMetadata := func() map[string]string {\n\t\treturn map[string]string{\n\t\t\tJSServerVersionMetadataKey: \"dynamic-version\",\n\t\t\tJSServerLevelMetadataKey: \"dynamic-version\",\n\t\t}\n\t}\n\n\tcfg := ConsumerConfig{Metadata: dynamicMetadata()}\n\tsetStaticConsumerMetadata(&cfg)\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel(\"0\")))\n}\n\nfunc TestJetStreamSetDynamicConsumerMetadata(t *testing.T) {\n\tcfg := ConsumerConfig{Metadata: metadataAtLevel(\"0\")}\n\tnewCfg := setDynamicConsumerMetadata(&cfg)\n\n\t// Only new metadata must contain dynamic fields.\n\tmetadata := metadataAtLevel(\"0\")\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadata))\n\tmetadata[JSServerVersionMetadataKey] = VERSION\n\tmetadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel)\n\trequire_True(t, reflect.DeepEqual(newCfg.Metadata, metadata))\n}\n\nfunc TestJetStreamSetDynamicConsumerInfoMetadata(t *testing.T) {\n\tci := ConsumerInfo{Config: &ConsumerConfig{Metadata: metadataAtLevel(\"0\")}}\n\tnewCi := setDynamicConsumerInfoMetadata(&ci)\n\n\t// Configs should not equal, as that would mean we've overwritten the original ConsumerInfo.\n\trequire_False(t, reflect.DeepEqual(ci, newCi))\n\n\t// Only new metadata must contain dynamic fields.\n\tmetadata := metadataAtLevel(\"0\")\n\trequire_True(t, reflect.DeepEqual(ci.Config.Metadata, metadata))\n\tmetadata[JSServerVersionMetadataKey] = VERSION\n\tmetadata[JSServerLevelMetadataKey] = strconv.Itoa(JSApiLevel)\n\trequire_True(t, reflect.DeepEqual(newCi.Config.Metadata, metadata))\n}\n\nfunc TestJetStreamCopyConsumerMetadata(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tdesc string\n\t\tcfg *ConsumerConfig\n\t\tprev *ConsumerConfig\n\t}{\n\t\t{\n\t\t\tdesc: \"no-previous-ignore\",\n\t\t\tcfg: &ConsumerConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: nil,\n\t\t},\n\t\t{\n\t\t\tdesc: \"nil-previous-metadata-ignore\",\n\t\t\tcfg: &ConsumerConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &ConsumerConfig{Metadata: nil},\n\t\t},\n\t\t{\n\t\t\tdesc: \"nil-current-metadata-ignore\",\n\t\t\tcfg: &ConsumerConfig{Metadata: nil},\n\t\t\tprev: &ConsumerConfig{Metadata: metadataPrevious()},\n\t\t},\n\t\t{\n\t\t\tdesc: \"copy-previous\",\n\t\t\tcfg: &ConsumerConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &ConsumerConfig{Metadata: metadataPrevious()},\n\t\t},\n\t\t{\n\t\t\tdesc: \"delete-missing-fields\",\n\t\t\tcfg: &ConsumerConfig{Metadata: metadataAtLevel(\"-1\")},\n\t\t\tprev: &ConsumerConfig{Metadata: make(map[string]string)},\n\t\t},\n\t} {\n\t\tt.Run(test.desc, func(t *testing.T) {\n\t\t\tcopyConsumerMetadata(test.cfg, test.prev)\n\n\t\t\tvar expectedMetadata map[string]string\n\t\t\tif test.prev != nil {\n\t\t\t\texpectedMetadata = test.prev.Metadata\n\t\t\t}\n\n\t\t\tvalue, ok := expectedMetadata[JSRequiredLevelMetadataKey]\n\t\t\tif ok {\n\t\t\t\trequire_Equal(t, test.cfg.Metadata[JSRequiredLevelMetadataKey], value)\n\t\t\t} else {\n\t\t\t\t// Key shouldn't exist.\n\t\t\t\t_, ok = test.cfg.Metadata[JSRequiredLevelMetadataKey]\n\t\t\t\trequire_False(t, ok)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamCopyConsumerMetadataRemoveDynamicFields(t *testing.T) {\n\tdynamicMetadata := func() map[string]string {\n\t\treturn map[string]string{\n\t\t\tJSServerVersionMetadataKey: \"dynamic-version\",\n\t\t\tJSServerLevelMetadataKey: \"dynamic-version\",\n\t\t}\n\t}\n\n\tcfg := ConsumerConfig{Metadata: dynamicMetadata()}\n\tcopyConsumerMetadata(&cfg, nil)\n\trequire_Equal(t, len(cfg.Metadata), 0)\n\n\tcfg = ConsumerConfig{Metadata: dynamicMetadata()}\n\tprevCfg := ConsumerConfig{Metadata: metadataAtLevel(\"0\")}\n\tcopyConsumerMetadata(&cfg, &prevCfg)\n\trequire_True(t, reflect.DeepEqual(cfg.Metadata, metadataAtLevel(\"0\")))\n}\n\ntype server struct {\n\treplicas int\n\tjs nats.JetStreamContext\n\tnc *nats.Conn\n}\n\nconst (\n\tstreamName = \"STREAM\"\n\tconsumerName = \"CONSUMER\"\n)\n\nfunc TestJetStreamMetadataMutations(t *testing.T) {\n\tsingle := RunBasicJetStreamServer(t)\n\tdefer single.Shutdown()\n\tnc, js := jsClientConnect(t, single)\n\tdefer nc.Close()\n\n\tcluster := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer cluster.shutdown()\n\tcnc, cjs := jsClientConnect(t, cluster.randomServer())\n\tdefer cnc.Close()\n\n\t// Test for both single server and clustered mode.\n\tfor _, s := range []server{\n\t\t{1, js, nc},\n\t\t{3, cjs, cnc},\n\t} {\n\t\tt.Run(fmt.Sprintf(\"R%d\", s.replicas), func(t *testing.T) {\n\t\t\tstreamMetadataChecks(t, s)\n\t\t\tconsumerMetadataChecks(t, s)\n\t\t})\n\t}\n}\n\nfunc validateMetadata(metadata map[string]string, expectedFeatureLevel string) bool {\n\treturn metadata[JSRequiredLevelMetadataKey] == expectedFeatureLevel ||\n\t\tmetadata[JSServerVersionMetadataKey] == VERSION ||\n\t\tmetadata[JSServerLevelMetadataKey] == strconv.Itoa(JSApiLevel)\n}\n\nfunc streamMetadataChecks(t *testing.T, s server) {\n\t// Add stream.\n\tsc := nats.StreamConfig{Name: streamName, Replicas: s.replicas}\n\tsi, err := s.js.AddStream(&sc)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(si.Config.Metadata, \"0\"))\n\n\t// (Double) add stream, has different code path for clustered streams.\n\tsc = nats.StreamConfig{Name: streamName, Replicas: s.replicas}\n\tsi, err = s.js.AddStream(&sc)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(si.Config.Metadata, \"0\"))\n\n\t// Stream info.\n\tsi, err = s.js.StreamInfo(streamName)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(si.Config.Metadata, \"0\"))\n\n\t// Update stream.\n\t// Metadata set on creation should be preserved, even if not included in update.\n\tsi, err = s.js.UpdateStream(&sc)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(si.Config.Metadata, \"0\"))\n}\n\nfunc consumerMetadataChecks(t *testing.T, s server) {\n\t// Add consumer.\n\tcc := nats.ConsumerConfig{Name: consumerName, Replicas: s.replicas}\n\tci, err := s.js.AddConsumer(streamName, &cc)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t// Consumer info.\n\tci, err = s.js.ConsumerInfo(streamName, consumerName)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t// Update consumer.\n\t// Metadata set on creation should be preserved, even if not included in update.\n\tci, err = s.js.UpdateConsumer(streamName, &cc)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t// Use pause advisories to know when pause/resume is applied.\n\tpauseCh := make(chan *nats.Msg, 10)\n\t_, err = s.nc.ChanSubscribe(JSAdvisoryConsumerPausePre+\".STREAM.CONSUMER\", pauseCh)\n\trequire_NoError(t, err)\n\n\t// Pause consumer, should up required API level.\n\tjsTestPause_PauseConsumer(t, s.nc, streamName, consumerName, time.Now().Add(time.Second*3))\n\trequire_ChanRead(t, pauseCh, time.Second*2)\n\trequire_Len(t, len(pauseCh), 0)\n\n\tci, err = s.js.ConsumerInfo(streamName, consumerName)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(ci.Config.Metadata, \"1\"))\n\n\t// Unpause consumer, should lower required API level.\n\tsubj := fmt.Sprintf(\"$JS.API.CONSUMER.PAUSE.%s.%s\", streamName, consumerName)\n\t_, err = s.nc.Request(subj, nil, time.Second)\n\trequire_NoError(t, err)\n\trequire_ChanRead(t, pauseCh, time.Second*2)\n\trequire_Len(t, len(pauseCh), 0)\n\n\tci, err = s.js.ConsumerInfo(streamName, consumerName)\n\trequire_NoError(t, err)\n\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t// Test scaling up/down.\n\tif s.replicas == 3 {\n\t\t// Scale down.\n\t\tcc.Replicas = 1\n\t\tci, err = s.js.UpdateConsumer(streamName, &cc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t\tci, err = s.js.ConsumerInfo(streamName, consumerName)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t\t// Scale up.\n\t\tcc.Replicas = 3\n\t\tci, err = s.js.UpdateConsumer(streamName, &cc)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\n\t\tci, err = s.js.ConsumerInfo(streamName, consumerName)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, validateMetadata(ci.Config.Metadata, \"0\"))\n\t}\n}\n\nfunc TestJetStreamMetadataStreamRestoreAndRestart(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\trestoreEmptyStream(t, nc, 1)\n\n\texpectedMetadata := map[string]string{\n\t\tJSServerVersionMetadataKey: VERSION,\n\t\tJSServerLevelMetadataKey: strconv.Itoa(JSApiLevel),\n\t}\n\n\t// Stream restore should result in empty metadata to be preserved, only adding dynamic metadata.\n\tsi, err := js.StreamInfo(streamName)\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata))\n\n\t// Restart server.\n\tport := s.opts.Port\n\tsd := s.StoreDir()\n\tnc.Close()\n\ts.Shutdown()\n\ts = RunJetStreamServerOnPort(port, sd)\n\tdefer s.Shutdown()\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// After restart (or upgrade) metadata data should remain empty, only adding dynamic metadata.\n\tsi, err = js.StreamInfo(streamName)\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata))\n}\n\nfunc TestJetStreamMetadataStreamRestoreAndRestartCluster(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\trestoreEmptyStream(t, nc, 3)\n\n\texpectedMetadata := map[string]string{\n\t\tJSServerVersionMetadataKey: VERSION,\n\t\tJSServerLevelMetadataKey: strconv.Itoa(JSApiLevel),\n\t}\n\n\t// Stream restore should result in empty metadata to be preserved, only adding dynamic metadata.\n\tsi, err := js.StreamInfo(streamName)\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata))\n\n\t// Restart cluster.\n\tc.stopAll()\n\tc.restartAllSamePorts()\n\tdefer c.shutdown()\n\tc.waitOnAllCurrent()\n\tc.waitOnStreamLeader(\"$G\", streamName)\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// After restart (or upgrade) metadata data should remain empty, only adding dynamic metadata.\n\tsi, err = js.StreamInfo(streamName)\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(si.Config.Metadata, expectedMetadata))\n}\n\nfunc restoreEmptyStream(t *testing.T, nc *nats.Conn, replicas int) {\n\trreq := JSApiStreamRestoreRequest{\n\t\tConfig: StreamConfig{\n\t\t\tName: \"STREAM\",\n\t\t\tRetention: LimitsPolicy,\n\t\t\tStorage: FileStorage,\n\t\t\tReplicas: replicas,\n\t\t},\n\t}\n\tbuf, err := json.Marshal(rreq)\n\trequire_NoError(t, err)\n\n\tvar rresp JSApiStreamRestoreResponse\n\tmsg, err := nc.Request(fmt.Sprintf(JSApiStreamRestoreT, rreq.Config.Name), buf, 5*time.Second)\n\trequire_NoError(t, err)\n\tjson.Unmarshal(msg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Error on restore: %+v\", rresp.Error)\n\t}\n\n\t// Construct empty stream.tar.s2 (only containing meta.inf).\n\tfsi := FileStreamInfo{StreamConfig: rreq.Config}\n\tfsij, err := json.Marshal(fsi)\n\trequire_NoError(t, err)\n\n\thdr := &tar.Header{\n\t\tName: JetStreamMetaFile,\n\t\tMode: 0600,\n\t\tUname: \"nats\",\n\t\tGname: \"nats\",\n\t\tSize: int64(len(fsij)),\n\t\tFormat: tar.FormatPAX,\n\t}\n\tvar buffer bytes.Buffer\n\tenc := s2.NewWriter(&buffer)\n\ttw := tar.NewWriter(enc)\n\terr = tw.WriteHeader(hdr)\n\trequire_NoError(t, err)\n\t_, err = tw.Write(fsij)\n\trequire_NoError(t, err)\n\terr = tw.Close()\n\trequire_NoError(t, err)\n\terr = enc.Close()\n\trequire_NoError(t, err)\n\n\tdata := buffer.Bytes()\n\tmsg, err = nc.Request(rresp.DeliverSubject, data, 5*time.Second)\n\trequire_NoError(t, err)\n\tjson.Unmarshal(msg.Data, &rresp)\n\tif rresp.Error != nil {\n\t\tt.Fatalf(\"Error on restore: %+v\", rresp.Error)\n\t}\n\n\tmsg, err = nc.Request(rresp.DeliverSubject, nil, 5*time.Second)\n\trequire_NoError(t, err)\n\n\texpectedMetadata := map[string]string{\n\t\tJSServerVersionMetadataKey: VERSION,\n\t\tJSServerLevelMetadataKey: strconv.Itoa(JSApiLevel),\n\t}\n\n\tvar cresp JSApiStreamCreateResponse\n\terr = json.Unmarshal(msg.Data, &cresp)\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(cresp.Config.Metadata, expectedMetadata))\n}\n\nfunc TestJetStreamApiErrorOnRequiredApiLevel(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tvar subs []*subscription\n\ts.getJetStream().apiSubs.All(&subs)\n\trequire_True(t, len(subs) > 0)\n\tfor _, sub := range subs {\n\t\tapiSubject := string(sub.subject)\n\t\tt.Run(apiSubject, func(t *testing.T) {\n\t\t\treq := nats.NewMsg(apiSubject)\n\t\t\treq.Header.Set(\"Nats-Required-Api-Level\", strconv.Itoa(math.MaxInt))\n\t\t\tmsg, err := nc.RequestMsg(req, time.Second)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar resp ApiResponse\n\t\t\trequire_NoError(t, json.Unmarshal(msg.Data, &resp))\n\t\t\trequire_True(t, resp.Error != nil)\n\t\t\t// Peer remove or stepdown is not supported if not clustered.\n\t\t\tif strings.Contains(apiSubject, \".STEPDOWN.\") || strings.Contains(apiSubject, \".PEER.\") {\n\t\t\t\trequire_Error(t, resp.Error, NewJSClusterRequiredError())\n\t\t\t} else {\n\t\t\t\trequire_Error(t, resp.Error, NewJSRequiredApiLevelError())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJetStreamApiErrorOnRequiredApiLevelDirectGet(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\treq := nats.NewMsg(fmt.Sprintf(JSDirectMsgGetT, \"TEST\"))\n\treq.Header.Set(\"Nats-Required-Api-Level\", strconv.Itoa(math.MaxInt))\n\tmsg, err := nc.RequestMsg(req, time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"412\")\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Required Api Level\")\n\n\treq = nats.NewMsg(fmt.Sprintf(JSDirectGetLastBySubjectT, \"TEST\", \"foo\"))\n\treq.Header.Set(\"Nats-Required-Api-Level\", strconv.Itoa(math.MaxInt))\n\tmsg, err = nc.RequestMsg(req, time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"412\")\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Required Api Level\")\n}\n\nfunc TestJetStreamApiErrorOnRequiredApiLevelPullConsumerNextMsg(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tAllowDirect: true,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"CONSUMER\"})\n\trequire_NoError(t, err)\n\n\treq := nats.NewMsg(fmt.Sprintf(JSApiRequestNextT, \"TEST\", \"CONSUMER\"))\n\treq.Header.Set(\"Nats-Required-Api-Level\", strconv.Itoa(math.MaxInt))\n\tmsg, err := nc.RequestMsg(req, time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, msg.Header.Get(\"Status\"), \"412\")\n\trequire_Equal(t, msg.Header.Get(\"Description\"), \"Required Api Level\")\n}\n" }, { "path": "server/jwt.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nkeys\"\n)\n\n// All JWTs once encoded start with this\nconst jwtPrefix = \"eyJ\"\n\n// ReadOperatorJWT will read a jwt file for an operator claim. This can be a decorated file.\nfunc ReadOperatorJWT(jwtfile string) (*jwt.OperatorClaims, error) {\n\t_, claim, err := readOperatorJWT(jwtfile)\n\treturn claim, err\n}\n\nfunc readOperatorJWT(jwtfile string) (string, *jwt.OperatorClaims, error) {\n\tcontents, err := os.ReadFile(jwtfile)\n\tif err != nil {\n\t\t// Check to see if the JWT has been inlined.\n\t\tif !strings.HasPrefix(jwtfile, jwtPrefix) {\n\t\t\treturn \"\", nil, err\n\t\t}\n\t\t// We may have an inline jwt here.\n\t\tcontents = []byte(jwtfile)\n\t}\n\tdefer wipeSlice(contents)\n\n\ttheJWT, err := jwt.ParseDecoratedJWT(contents)\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\topc, err := jwt.DecodeOperatorClaims(theJWT)\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\treturn theJWT, opc, nil\n}\n\n// Just wipe slice with 'x', for clearing contents of nkey seed file.\nfunc wipeSlice(buf []byte) {\n\tfor i := range buf {\n\t\tbuf[i] = 'x'\n\t}\n}\n\n// validateTrustedOperators will check that we do not have conflicts with\n// assigned trusted keys and trusted operators. If operators are defined we\n// will expand the trusted keys in options.\nfunc validateTrustedOperators(o *Options) error {\n\tif len(o.TrustedOperators) == 0 {\n\t\t// if we have no operator, default sentinel shouldn't be set\n\t\tif o.DefaultSentinel != _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"default sentinel requires operators and accounts\")\n\t\t}\n\t\treturn nil\n\t}\n\tif o.DefaultSentinel != _EMPTY_ {\n\t\tjuc, err := jwt.DecodeUserClaims(o.DefaultSentinel)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"default sentinel JWT not valid\")\n\t\t}\n\n\t\tif !juc.BearerToken && juc.IssuerAccount != \"\" && juc.HasEmptyPermissions() {\n\t\t\t// we cannot resolve the account yet - but this looks like a scoped user\n\t\t\t// it will be rejected at runtime if not valid\n\t\t} else if !juc.BearerToken {\n\t\t\treturn fmt.Errorf(\"default sentinel must be a bearer token\")\n\t\t}\n\t}\n\tif o.AccountResolver == nil {\n\t\treturn fmt.Errorf(\"operators require an account resolver to be configured\")\n\t}\n\tif len(o.Accounts) > 0 {\n\t\treturn fmt.Errorf(\"operators do not allow Accounts to be configured directly\")\n\t}\n\tif len(o.Users) > 0 || len(o.Nkeys) > 0 {\n\t\treturn fmt.Errorf(\"operators do not allow users to be configured directly\")\n\t}\n\tif len(o.TrustedOperators) > 0 && len(o.TrustedKeys) > 0 {\n\t\treturn fmt.Errorf(\"conflicting options for 'TrustedKeys' and 'TrustedOperators'\")\n\t}\n\tif o.SystemAccount != _EMPTY_ {\n\t\tfoundSys := false\n\t\tfoundNonEmpty := false\n\t\tfor _, op := range o.TrustedOperators {\n\t\t\tif op.SystemAccount != _EMPTY_ {\n\t\t\t\tfoundNonEmpty = true\n\t\t\t}\n\t\t\tif op.SystemAccount == o.SystemAccount {\n\t\t\t\tfoundSys = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif foundNonEmpty && !foundSys {\n\t\t\treturn fmt.Errorf(\"system_account in config and operator JWT must be identical\")\n\t\t}\n\t} else if o.TrustedOperators[0].SystemAccount == _EMPTY_ {\n\t\t// In case the system account is neither defined in config nor in the first operator.\n\t\t// If it would be needed due to the nats account resolver, raise an error.\n\t\tswitch o.AccountResolver.(type) {\n\t\tcase *DirAccResolver, *CacheDirAccResolver:\n\t\t\treturn fmt.Errorf(\"using nats based account resolver - the system account needs to be specified in configuration or the operator jwt\")\n\t\t}\n\t}\n\n\tsrvMajor, srvMinor, srvUpdate, _ := versionComponents(VERSION)\n\tfor _, opc := range o.TrustedOperators {\n\t\tif major, minor, update, err := jwt.ParseServerVersion(opc.AssertServerVersion); err != nil {\n\t\t\treturn fmt.Errorf(\"operator %s expects version %s got error instead: %s\",\n\t\t\t\topc.Subject, opc.AssertServerVersion, err)\n\t\t} else if major > srvMajor {\n\t\t\treturn fmt.Errorf(\"operator %s expected major version %d > server major version %d\",\n\t\t\t\topc.Subject, major, srvMajor)\n\t\t} else if srvMajor > major {\n\t\t} else if minor > srvMinor {\n\t\t\treturn fmt.Errorf(\"operator %s expected minor version %d > server minor version %d\",\n\t\t\t\topc.Subject, minor, srvMinor)\n\t\t} else if srvMinor > minor {\n\t\t} else if update > srvUpdate {\n\t\t\treturn fmt.Errorf(\"operator %s expected update version %d > server update version %d\",\n\t\t\t\topc.Subject, update, srvUpdate)\n\t\t}\n\t}\n\t// If we have operators, fill in the trusted keys.\n\t// FIXME(dlc) - We had TrustedKeys before TrustedOperators. The jwt.OperatorClaims\n\t// has a DidSign(). Use that longer term. For now we can expand in place.\n\tfor _, opc := range o.TrustedOperators {\n\t\tif o.TrustedKeys == nil {\n\t\t\to.TrustedKeys = make([]string, 0, 4)\n\t\t}\n\t\tif !opc.StrictSigningKeyUsage {\n\t\t\to.TrustedKeys = append(o.TrustedKeys, opc.Subject)\n\t\t}\n\t\to.TrustedKeys = append(o.TrustedKeys, opc.SigningKeys...)\n\t}\n\tfor _, key := range o.TrustedKeys {\n\t\tif !nkeys.IsValidPublicOperatorKey(key) {\n\t\t\treturn fmt.Errorf(\"trusted Keys %q are required to be a valid public operator nkey\", key)\n\t\t}\n\t}\n\tif len(o.resolverPinnedAccounts) > 0 {\n\t\tfor key := range o.resolverPinnedAccounts {\n\t\t\tif !nkeys.IsValidPublicAccountKey(key) {\n\t\t\t\treturn fmt.Errorf(\"pinned account key %q is not a valid public account nkey\", key)\n\t\t\t}\n\t\t}\n\t\t// ensure the system account (belonging to the operator can always connect)\n\t\tif o.SystemAccount != _EMPTY_ {\n\t\t\to.resolverPinnedAccounts[o.SystemAccount] = struct{}{}\n\t\t}\n\t}\n\n\t// If we have an auth callout defined make sure we are not in operator mode.\n\tif o.AuthCallout != nil {\n\t\treturn errors.New(\"operators do not allow authorization callouts to be configured directly\")\n\t}\n\n\treturn nil\n}\n\nfunc validateSrc(claims *jwt.UserClaims, host string) bool {\n\tif claims == nil {\n\t\treturn false\n\t} else if len(claims.Src) == 0 {\n\t\treturn true\n\t} else if host == \"\" {\n\t\treturn false\n\t}\n\tip := net.ParseIP(host)\n\tif ip == nil {\n\t\treturn false\n\t}\n\tfor _, cidr := range claims.Src {\n\t\tif _, net, err := net.ParseCIDR(cidr); err != nil {\n\t\t\treturn false // should not happen as this jwt is invalid\n\t\t} else if net.Contains(ip) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {\n\tif claims == nil {\n\t\treturn false, time.Duration(0)\n\t} else if len(claims.Times) == 0 {\n\t\treturn true, time.Duration(0)\n\t}\n\tnow := time.Now()\n\tloc := time.Local\n\tif claims.Locale != \"\" {\n\t\tvar err error\n\t\tif loc, err = time.LoadLocation(claims.Locale); err != nil {\n\t\t\treturn false, time.Duration(0) // parsing not expected to fail at this point\n\t\t}\n\t\tnow = now.In(loc)\n\t}\n\tfor _, timeRange := range claims.Times {\n\t\ty, m, d := now.Date()\n\t\tm = m - 1\n\t\td = d - 1\n\t\tstart, err := time.ParseInLocation(\"15:04:05\", timeRange.Start, loc)\n\t\tif err != nil {\n\t\t\treturn false, time.Duration(0) // parsing not expected to fail at this point\n\t\t}\n\t\tend, err := time.ParseInLocation(\"15:04:05\", timeRange.End, loc)\n\t\tif err != nil {\n\t\t\treturn false, time.Duration(0) // parsing not expected to fail at this point\n\t\t}\n\t\tif start.After(end) {\n\t\t\tstart = start.AddDate(y, int(m), d)\n\t\t\td++ // the intent is to be the next day\n\t\t} else {\n\t\t\tstart = start.AddDate(y, int(m), d)\n\t\t}\n\t\tif start.Before(now) {\n\t\t\tend = end.AddDate(y, int(m), d)\n\t\t\tif end.After(now) {\n\t\t\t\treturn true, end.Sub(now)\n\t\t\t}\n\t\t}\n\t}\n\treturn false, time.Duration(0)\n}\n" }, { "path": "server/jwt_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nvar (\n\t// This matches ./configs/nkeys_jwts/test.seed\n\toSeed = []byte(\"SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU\")\n\t// This matches ./configs/nkeys/op.jwt\n\tojwt = \"eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw\"\n\toKp nkeys.KeyPair\n)\n\nfunc init() {\n\tvar err error\n\toKp, err = nkeys.FromSeed(oSeed)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Parsing oSeed failed with: %v\", err))\n\t}\n}\n\nfunc chanRecv(t *testing.T, recvChan <-chan struct{}, limit time.Duration) {\n\tt.Helper()\n\tselect {\n\tcase <-recvChan:\n\tcase <-time.After(limit):\n\t\tt.Fatal(\"Should have received from channel\")\n\t}\n}\n\nfunc opTrustBasicSetup() *Server {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\topts := defaultServerOptions\n\topts.TrustedKeys = []string{pub}\n\ts, c, _, _ := rawSetup(opts)\n\tc.close()\n\treturn s\n}\n\nfunc buildMemAccResolver(s *Server) {\n\tmr := &MemAccResolver{}\n\ts.SetAccountResolver(mr)\n}\n\nfunc addAccountToMemResolver(s *Server, pub, jwtclaim string) {\n\ts.AccountResolver().Store(pub, jwtclaim)\n}\n\nfunc createClient(t *testing.T, s *Server, akp nkeys.KeyPair) (*testAsyncClient, *bufio.Reader, string) {\n\treturn createClientWithIssuer(t, s, akp, \"\")\n}\n\nfunc createClientWithIssuer(t *testing.T, s *Server, akp nkeys.KeyPair, optIssuerAccount string) (*testAsyncClient, *bufio.Reader, string) {\n\tt.Helper()\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tif optIssuerAccount != \"\" {\n\t\tnuc.IssuerAccount = optIssuerAccount\n\t}\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tc, cr, l := newClientForServer(s)\n\n\t// Sign Nonce\n\tvar info nonceInfo\n\tjson.Unmarshal([]byte(l[5:]), &info)\n\tsigraw, _ := nkp.Sign([]byte(info.Nonce))\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"sig\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\", ujwt, sig)\n\treturn c, cr, cs\n}\n\nfunc setupJWTTestWithClaims(t *testing.T, nac *jwt.AccountClaims, nuc *jwt.UserClaims, expected string) (*Server, nkeys.KeyPair, *testAsyncClient, *bufio.Reader) {\n\tt.Helper()\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tif nac == nil {\n\t\tnac = jwt.NewAccountClaims(apub)\n\t} else {\n\t\tnac.Subject = apub\n\t}\n\tajwt, err := nac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tif nuc == nil {\n\t\tnuc = jwt.NewUserClaims(pub)\n\t} else {\n\t\tnuc.Subject = pub\n\t}\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\ts := opTrustBasicSetup()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, l := newClientForServer(s)\n\n\t// Sign Nonce\n\tvar info nonceInfo\n\tjson.Unmarshal([]byte(l[5:]), &info)\n\tsigraw, _ := nkp.Sign([]byte(info.Nonce))\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK/-ERR to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt, sig)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tc.parse([]byte(cs))\n\t\twg.Done()\n\t}()\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, expected) {\n\t\tt.Fatalf(\"Expected %q, got %q\", expected, l)\n\t}\n\twg.Wait()\n\n\treturn s, akp, c, cr\n}\n\nfunc setupJWTTestWitAccountClaims(t *testing.T, nac *jwt.AccountClaims, expected string) (*Server, nkeys.KeyPair, *testAsyncClient, *bufio.Reader) {\n\tt.Helper()\n\treturn setupJWTTestWithClaims(t, nac, nil, expected)\n}\n\n// This is used in test to create account claims and pass it\n// to setupJWTTestWitAccountClaims.\nfunc newJWTTestAccountClaims() *jwt.AccountClaims {\n\t// We call NewAccountClaims() because it sets some defaults.\n\t// However, this call needs a subject, but the real subject will\n\t// be set in setupJWTTestWitAccountClaims(). Use some temporary one\n\t// here.\n\treturn jwt.NewAccountClaims(\"temp\")\n}\n\nfunc setupJWTTestWithUserClaims(t *testing.T, nuc *jwt.UserClaims, expected string) (*Server, *testAsyncClient, *bufio.Reader) {\n\tt.Helper()\n\ts, _, c, cr := setupJWTTestWithClaims(t, nil, nuc, expected)\n\treturn s, c, cr\n}\n\n// This is used in test to create user claims and pass it\n// to setupJWTTestWithUserClaims.\nfunc newJWTTestUserClaims() *jwt.UserClaims {\n\t// As of now, tests could simply do &jwt.UserClaims{}, but in\n\t// case some defaults are later added, we call NewUserClaims().\n\t// However, this call needs a subject, but the real subject will\n\t// be set in setupJWTTestWithUserClaims(). Use some temporary one\n\t// here.\n\treturn jwt.NewUserClaims(\"temp\")\n}\n\nfunc TestJWTUser(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\n\t// Check to make sure we would have an authTimer\n\tif !s.info.AuthRequired {\n\t\tt.Fatalf(\"Expect the server to require auth\")\n\t}\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Don't send jwt field, should fail.\n\tc.parseAsync(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create an account that will be expired.\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tc, cr, cs := createClient(t, s, akp)\n\tdefer c.close()\n\n\t// PING needed to flush the +OK/-ERR to us.\n\t// This should fail too since no account resolver is defined.\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Ok now let's walk through and make sure all is good.\n\t// We will set the account resolver by hand to a memory resolver.\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, cs = createClient(t, s, akp)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG, got %q\", l)\n\t}\n}\n\nfunc TestJWTUserBadTrusted(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\n\t// Check to make sure we would have an authTimer\n\tif !s.info.AuthRequired {\n\t\tt.Fatalf(\"Expect the server to require auth\")\n\t}\n\t// Now place bad trusted key\n\ts.mu.Lock()\n\ts.trustedKeys = []string{\"bad\"}\n\ts.mu.Unlock()\n\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create an account that will be expired.\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, cs := createClient(t, s, akp)\n\tdefer c.close()\n\tc.parseAsync(cs)\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n}\n\n// Test that if a user tries to connect with an expired user JWT we do the right thing.\nfunc TestJWTUserExpired(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.IssuedAt = time.Now().Add(-10 * time.Second).Unix()\n\tnuc.Expires = time.Now().Add(-2 * time.Second).Unix()\n\ts, c, _ := setupJWTTestWithUserClaims(t, nuc, \"-ERR \")\n\tc.close()\n\ts.Shutdown()\n}\n\nfunc TestJWTUserExpiresAfterConnect(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.IssuedAt = time.Now().Unix()\n\tnuc.Expires = time.Now().Add(time.Second).Unix()\n\ts, c, cr := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\tl, err := cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Received %v\", err)\n\t}\n\tif !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG\")\n\t}\n\n\t// Now we should expire after 1 second or so.\n\ttime.Sleep(1250 * time.Millisecond)\n\n\tl, err = cr.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Received %v\", err)\n\t}\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"Expired\") {\n\t\tt.Fatalf(\"Expected 'Expired' to be in the error\")\n\t}\n}\n\nfunc TestJWTUserPermissionClaims(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.Permissions.Pub.Allow.Add(\"foo\")\n\tnuc.Permissions.Pub.Allow.Add(\"bar\")\n\tnuc.Permissions.Pub.Deny.Add(\"baz\")\n\tnuc.Permissions.Sub.Allow.Add(\"foo\")\n\tnuc.Permissions.Sub.Allow.Add(\"bar\")\n\tnuc.Permissions.Sub.Deny.Add(\"baz\")\n\n\ts, c, _ := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\t// Now check client to make sure permissions transferred.\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.perms == nil {\n\t\tt.Fatalf(\"Expected client permissions to be set\")\n\t}\n\n\tif lpa := c.perms.pub.allow.Count(); lpa != 2 {\n\t\tt.Fatalf(\"Expected 2 publish allow subjects, got %d\", lpa)\n\t}\n\tif lpd := c.perms.pub.deny.Count(); lpd != 1 {\n\t\tt.Fatalf(\"Expected 1 publish deny subjects, got %d\", lpd)\n\t}\n\tif lsa := c.perms.sub.allow.Count(); lsa != 2 {\n\t\tt.Fatalf(\"Expected 2 subscribe allow subjects, got %d\", lsa)\n\t}\n\tif lsd := c.perms.sub.deny.Count(); lsd != 1 {\n\t\tt.Fatalf(\"Expected 1 subscribe deny subjects, got %d\", lsd)\n\t}\n}\n\nfunc TestJWTUserResponsePermissionClaims(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.Permissions.Resp = &jwt.ResponsePermission{\n\t\tMaxMsgs: 22,\n\t\tExpires: 100 * time.Millisecond,\n\t}\n\ts, c, _ := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\t// Now check client to make sure permissions transferred.\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.perms == nil {\n\t\tt.Fatalf(\"Expected client permissions to be set\")\n\t}\n\tif c.perms.pub.allow == nil {\n\t\tt.Fatalf(\"Expected client perms for pub allow to be non-nil\")\n\t}\n\tif lpa := c.perms.pub.allow.Count(); lpa != 0 {\n\t\tt.Fatalf(\"Expected 0 publish allow subjects, got %d\", lpa)\n\t}\n\tif c.perms.resp == nil {\n\t\tt.Fatalf(\"Expected client perms for response permissions to be non-nil\")\n\t}\n\tif c.perms.resp.MaxMsgs != nuc.Permissions.Resp.MaxMsgs {\n\t\tt.Fatalf(\"Expected client perms for response permissions MaxMsgs to be same as jwt: %d vs %d\",\n\t\t\tc.perms.resp.MaxMsgs, nuc.Permissions.Resp.MaxMsgs)\n\t}\n\tif c.perms.resp.Expires != nuc.Permissions.Resp.Expires {\n\t\tt.Fatalf(\"Expected client perms for response permissions Expires to be same as jwt: %v vs %v\",\n\t\t\tc.perms.resp.Expires, nuc.Permissions.Resp.Expires)\n\t}\n}\n\nfunc TestJWTUserResponsePermissionClaimsDefaultValues(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.Permissions.Resp = &jwt.ResponsePermission{}\n\ts, c, _ := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\t// Now check client to make sure permissions transferred\n\t// and defaults are set.\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.perms == nil {\n\t\tt.Fatalf(\"Expected client permissions to be set\")\n\t}\n\tif c.perms.pub.allow == nil {\n\t\tt.Fatalf(\"Expected client perms for pub allow to be non-nil\")\n\t}\n\tif lpa := c.perms.pub.allow.Count(); lpa != 0 {\n\t\tt.Fatalf(\"Expected 0 publish allow subjects, got %d\", lpa)\n\t}\n\tif c.perms.resp == nil {\n\t\tt.Fatalf(\"Expected client perms for response permissions to be non-nil\")\n\t}\n\tif c.perms.resp.MaxMsgs != DEFAULT_ALLOW_RESPONSE_MAX_MSGS {\n\t\tt.Fatalf(\"Expected client perms for response permissions MaxMsgs to be default %v, got %v\",\n\t\t\tDEFAULT_ALLOW_RESPONSE_MAX_MSGS, c.perms.resp.MaxMsgs)\n\t}\n\tif c.perms.resp.Expires != DEFAULT_ALLOW_RESPONSE_EXPIRATION {\n\t\tt.Fatalf(\"Expected client perms for response permissions Expires to be default %v, got %v\",\n\t\t\tDEFAULT_ALLOW_RESPONSE_EXPIRATION, c.perms.resp.Expires)\n\t}\n}\n\nfunc TestJWTUserResponsePermissionClaimsNegativeValues(t *testing.T) {\n\tnuc := newJWTTestUserClaims()\n\tnuc.Permissions.Resp = &jwt.ResponsePermission{\n\t\tMaxMsgs: -1,\n\t\tExpires: -1 * time.Second,\n\t}\n\ts, c, _ := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\t// Now check client to make sure permissions transferred\n\t// and negative values are transferred.\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.perms == nil {\n\t\tt.Fatalf(\"Expected client permissions to be set\")\n\t}\n\tif c.perms.pub.allow == nil {\n\t\tt.Fatalf(\"Expected client perms for pub allow to be non-nil\")\n\t}\n\tif lpa := c.perms.pub.allow.Count(); lpa != 0 {\n\t\tt.Fatalf(\"Expected 0 publish allow subjects, got %d\", lpa)\n\t}\n\tif c.perms.resp == nil {\n\t\tt.Fatalf(\"Expected client perms for response permissions to be non-nil\")\n\t}\n\tif c.perms.resp.MaxMsgs != -1 {\n\t\tt.Fatalf(\"Expected client perms for response permissions MaxMsgs to be %v, got %v\",\n\t\t\t-1, c.perms.resp.MaxMsgs)\n\t}\n\tif c.perms.resp.Expires != -1*time.Second {\n\t\tt.Fatalf(\"Expected client perms for response permissions Expires to be %v, got %v\",\n\t\t\t-1*time.Second, c.perms.resp.Expires)\n\t}\n}\n\nfunc TestJWTAccountExpired(t *testing.T) {\n\tnac := newJWTTestAccountClaims()\n\tnac.IssuedAt = time.Now().Add(-10 * time.Second).Unix()\n\tnac.Expires = time.Now().Add(-2 * time.Second).Unix()\n\ts, _, c, _ := setupJWTTestWitAccountClaims(t, nac, \"-ERR \")\n\tdefer s.Shutdown()\n\tdefer c.close()\n}\n\nfunc TestJWTAccountExpiresAfterConnect(t *testing.T) {\n\tnac := newJWTTestAccountClaims()\n\tnow := time.Now()\n\tnac.IssuedAt = now.Add(-10 * time.Second).Unix()\n\tnac.Expires = now.Round(time.Second).Add(time.Second).Unix()\n\ts, akp, c, cr := setupJWTTestWitAccountClaims(t, nac, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tapub, _ := akp.PublicKey()\n\tacc, err := s.LookupAccount(apub)\n\tif acc == nil || err != nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\tif l, _ := cr.ReadString('\\n'); !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected PONG, got %q\", l)\n\t}\n\n\t// Wait for the account to be expired.\n\tcheckFor(t, 3*time.Second, 100*time.Millisecond, func() error {\n\t\tif acc.IsExpired() {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Account not expired yet\")\n\t})\n\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error, got %q\", l)\n\t}\n\tif !strings.Contains(l, \"Expired\") {\n\t\tt.Fatalf(\"Expected 'Expired' to be in the error\")\n\t}\n\n\t// Now make sure that accounts that have expired return an error.\n\tc, cr, cs := createClient(t, s, akp)\n\tdefer c.close()\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n}\n\nfunc TestJWTAccountRenew(t *testing.T) {\n\tnac := newJWTTestAccountClaims()\n\t// Create an account that has expired.\n\tnac.IssuedAt = time.Now().Add(-10 * time.Second).Unix()\n\tnac.Expires = time.Now().Add(-2 * time.Second).Unix()\n\t// Expect an error\n\ts, akp, c, _ := setupJWTTestWitAccountClaims(t, nac, \"-ERR \")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\tapub, _ := akp.PublicKey()\n\n\t// Now update with new expiration\n\tnac.IssuedAt = time.Now().Unix()\n\tnac.Expires = time.Now().Add(5 * time.Second).Unix()\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t// Update the account\n\taddAccountToMemResolver(s, apub, ajwt)\n\tacc, _ := s.LookupAccount(apub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\ts.UpdateAccountClaims(acc, nac)\n\n\t// Now make sure we can connect.\n\tc, cr, cs := createClient(t, s, akp)\n\tdefer c.close()\n\tc.parseAsync(cs)\n\tif l, _ := cr.ReadString('\\n'); !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG, got: %q\", l)\n\t}\n}\n\nfunc TestJWTAccountRenewFromResolver(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tnac.IssuedAt = time.Now().Add(-10 * time.Second).Unix()\n\tnac.Expires = time.Now().Add(time.Second).Unix()\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, apub, ajwt)\n\t// Force it to be loaded by the server and start the expiration timer.\n\tacc, _ := s.LookupAccount(apub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Could not retrieve account for %q\", apub)\n\t}\n\n\t// Create a new user\n\tc, cr, cs := createClient(t, s, akp)\n\tdefer c.close()\n\t// Wait for expiration.\n\ttime.Sleep(1250 * time.Millisecond)\n\n\tc.parseAsync(cs)\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Now update with new expiration\n\tnac.IssuedAt = time.Now().Unix()\n\tnac.Expires = time.Now().Add(5 * time.Second).Unix()\n\tajwt, err = nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t// Update the account\n\taddAccountToMemResolver(s, apub, ajwt)\n\t// Make sure the too quick update suppression does not bite us.\n\tacc.mu.Lock()\n\tacc.updated = time.Now().UTC().Add(-1 * time.Hour)\n\tacc.mu.Unlock()\n\n\t// Do not update the account directly. The resolver should\n\t// happen automatically.\n\n\t// Now make sure we can connect.\n\tc, cr, cs = createClient(t, s, akp)\n\tdefer c.close()\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG, got: %q\", l)\n\t}\n}\n\nfunc TestJWTAccountBasicImportExport(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\n\t// Now create Exports.\n\tstreamExport := &jwt.Export{Subject: \"foo\", Type: jwt.Stream}\n\tstreamExport2 := &jwt.Export{Subject: \"private\", Type: jwt.Stream, TokenReq: true}\n\tserviceExport := &jwt.Export{Subject: \"req.echo\", Type: jwt.Service, TokenReq: true}\n\tserviceExport2 := &jwt.Export{Subject: \"req.add\", Type: jwt.Service, TokenReq: true}\n\n\tfooAC.Exports.Add(streamExport, streamExport2, serviceExport, serviceExport2)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tacc, _ := s.LookupAccount(fooPub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\t// Check to make sure exports transferred over.\n\tif les := len(acc.exports.streams); les != 2 {\n\t\tt.Fatalf(\"Expected exports streams len of 2, got %d\", les)\n\t}\n\tif les := len(acc.exports.services); les != 2 {\n\t\tt.Fatalf(\"Expected exports services len of 2, got %d\", les)\n\t}\n\t_, ok := acc.exports.streams[\"foo\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to map a stream export\")\n\t}\n\tse, ok := acc.exports.services[\"req.echo\"]\n\tif !ok || se == nil {\n\t\tt.Fatalf(\"Expected to map a service export\")\n\t}\n\tif !se.tokenReq {\n\t\tt.Fatalf(\"Expected the service export to require tokens\")\n\t}\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\n\tstreamImport := &jwt.Import{Account: fooPub, Subject: \"foo\", To: \"import.foo\", Type: jwt.Stream}\n\tserviceImport := &jwt.Import{Account: fooPub, Subject: \"req.echo\", Type: jwt.Service}\n\tbarAC.Imports.Add(streamImport, serviceImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\tacc, _ = s.LookupAccount(barPub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\tif les := len(acc.imports.streams); les != 1 {\n\t\tt.Fatalf(\"Expected imports streams len of 1, got %d\", les)\n\t}\n\t// Our service import should have failed without a token.\n\tif les := len(acc.imports.services); les != 0 {\n\t\tt.Fatalf(\"Expected imports services len of 0, got %d\", les)\n\t}\n\n\t// Now add in a bad activation token.\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tserviceImport = &jwt.Import{Account: fooPub, Subject: \"req.echo\", Token: \"not a token\", Type: jwt.Service}\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err = barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\ts.UpdateAccountClaims(acc, barAC)\n\n\t// Our service import should have failed with a bad token.\n\tif les := len(acc.imports.services); les != 0 {\n\t\tt.Fatalf(\"Expected imports services len of 0, got %d\", les)\n\t}\n\n\t// Now make a correct one.\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tserviceImport = &jwt.Import{Account: fooPub, Subject: \"req.echo\", Type: jwt.Service}\n\n\tactivation := jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"req.echo\"\n\tactivation.ImportType = jwt.Service\n\tactJWT, err := activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\tserviceImport.Token = actJWT\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err = barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tvr := jwt.ValidationResults{}\n\tbarAC.Validate(&vr)\n\tif vr.IsBlocking(true) {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", vr)\n\t}\n\n\taddAccountToMemResolver(s, barPub, barJWT)\n\ts.UpdateAccountClaims(acc, barAC)\n\t// Our service import should have succeeded.\n\tif les := len(acc.imports.services); les != 1 {\n\t\tt.Fatalf(\"Expected imports services len of 1, got %d\", les)\n\t}\n\n\t// Now streams\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tstreamImport = &jwt.Import{Account: fooPub, Subject: \"private\", To: \"import.private\", Type: jwt.Stream}\n\n\tbarAC.Imports.Add(streamImport)\n\tbarJWT, err = barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\ts.UpdateAccountClaims(acc, barAC)\n\t// Our stream import should have not succeeded.\n\tif les := len(acc.imports.streams); les != 0 {\n\t\tt.Fatalf(\"Expected imports services len of 0, got %d\", les)\n\t}\n\n\t// Now add in activation.\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tstreamImport = &jwt.Import{Account: fooPub, Subject: \"private\", To: \"import.private\", Type: jwt.Stream}\n\n\tactivation = jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"private\"\n\tactivation.ImportType = jwt.Stream\n\tactJWT, err = activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\tstreamImport.Token = actJWT\n\tbarAC.Imports.Add(streamImport)\n\tbarJWT, err = barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\ts.UpdateAccountClaims(acc, barAC)\n\t// Our stream import should have not succeeded.\n\tif les := len(acc.imports.streams); les != 1 {\n\t\tt.Fatalf(\"Expected imports services len of 1, got %d\", les)\n\t}\n}\n\nfunc TestJWTAccountExportWithResponseType(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\n\t// Now create Exports.\n\tserviceStreamExport := &jwt.Export{Subject: \"test.stream\", Type: jwt.Service, ResponseType: jwt.ResponseTypeStream, TokenReq: false}\n\tserviceChunkExport := &jwt.Export{Subject: \"test.chunk\", Type: jwt.Service, ResponseType: jwt.ResponseTypeChunked, TokenReq: false}\n\tserviceSingletonExport := &jwt.Export{Subject: \"test.single\", Type: jwt.Service, ResponseType: jwt.ResponseTypeSingleton, TokenReq: true}\n\tserviceDefExport := &jwt.Export{Subject: \"test.def\", Type: jwt.Service, TokenReq: true}\n\tserviceOldExport := &jwt.Export{Subject: \"test.old\", Type: jwt.Service, TokenReq: false}\n\n\tfooAC.Exports.Add(serviceStreamExport, serviceSingletonExport, serviceChunkExport, serviceDefExport, serviceOldExport)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tif fooAcc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\tservices := fooAcc.exports.services\n\n\tif len(services) != 5 {\n\t\tt.Fatalf(\"Expected 4 services\")\n\t}\n\n\tse, ok := services[\"test.stream\"]\n\tif !ok || se == nil {\n\t\tt.Fatalf(\"Expected to map a service export\")\n\t}\n\tif se.tokenReq {\n\t\tt.Fatalf(\"Expected the service export to not require tokens\")\n\t}\n\tif se.respType != Streamed {\n\t\tt.Fatalf(\"Expected the service export to respond with a stream\")\n\t}\n\n\tse, ok = services[\"test.chunk\"]\n\tif !ok || se == nil {\n\t\tt.Fatalf(\"Expected to map a service export\")\n\t}\n\tif se.tokenReq {\n\t\tt.Fatalf(\"Expected the service export to not require tokens\")\n\t}\n\tif se.respType != Chunked {\n\t\tt.Fatalf(\"Expected the service export to respond with a stream\")\n\t}\n\n\tse, ok = services[\"test.def\"]\n\tif !ok || se == nil {\n\t\tt.Fatalf(\"Expected to map a service export\")\n\t}\n\tif !se.tokenReq {\n\t\tt.Fatalf(\"Expected the service export to not require tokens\")\n\t}\n\tif se.respType != Singleton {\n\t\tt.Fatalf(\"Expected the service export to respond with a stream\")\n\t}\n\n\tse, ok = services[\"test.single\"]\n\tif !ok || se == nil {\n\t\tt.Fatalf(\"Expected to map a service export\")\n\t}\n\tif !se.tokenReq {\n\t\tt.Fatalf(\"Expected the service export to not require tokens\")\n\t}\n\tif se.respType != Singleton {\n\t\tt.Fatalf(\"Expected the service export to respond with a stream\")\n\t}\n\n\tse, ok = services[\"test.old\"]\n\tif !ok || se == nil || len(se.approved) > 0 {\n\t\tt.Fatalf(\"Service with a singleton response and no tokens should not be nil and have no approvals\")\n\t}\n}\n\nfunc expectPong(t *testing.T, cr *bufio.Reader) {\n\tt.Helper()\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG, got %q\", l)\n\t}\n}\n\nfunc expectMsg(t *testing.T, cr *bufio.Reader, sub, payload string) {\n\tt.Helper()\n\tl, _ := cr.ReadString('\\n')\n\texpected := \"MSG \" + sub\n\tif !strings.HasPrefix(l, expected) {\n\t\tt.Fatalf(\"Expected %q, got %q\", expected, l)\n\t}\n\tl, _ = cr.ReadString('\\n')\n\tif l != payload+\"\\r\\n\" {\n\t\tt.Fatalf(\"Expected %q, got %q\", payload, l)\n\t}\n\texpectPong(t, cr)\n}\n\nfunc TestJWTAccountImportExportUpdates(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tstreamExport := &jwt.Export{Subject: \"foo\", Type: jwt.Stream}\n\n\tfooAC.Exports.Add(streamExport)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tstreamImport := &jwt.Import{Account: fooPub, Subject: \"foo\", To: \"import\", Type: jwt.Stream}\n\n\tbarAC.Imports.Add(streamImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, barKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"SUB import.foo 1\\r\\nPING\\r\\n\")\n\texpectPong(t, cr)\n\n\tcheckShadow := func(expected int) {\n\t\tt.Helper()\n\t\tc.mu.Lock()\n\t\tdefer c.mu.Unlock()\n\t\tsub := c.subs[\"1\"]\n\t\tif ls := len(sub.shadow); ls != expected {\n\t\t\tt.Fatalf(\"Expected shadows to be %d, got %d\", expected, ls)\n\t\t}\n\t}\n\n\t// We created a SUB on foo which should create a shadow subscription.\n\tcheckShadow(1)\n\n\t// Now update bar and remove the import which should make the shadow go away.\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tbarJWT, _ = barAC.Encode(okp)\n\taddAccountToMemResolver(s, barPub, barJWT)\n\tacc, _ := s.LookupAccount(barPub)\n\ts.UpdateAccountClaims(acc, barAC)\n\n\tcheckShadow(0)\n\n\t// Now add it back and make sure the shadow comes back.\n\tstreamImport = &jwt.Import{Account: string(fooPub), Subject: \"foo\", To: \"import\", Type: jwt.Stream}\n\tbarAC.Imports.Add(streamImport)\n\tbarJWT, _ = barAC.Encode(okp)\n\taddAccountToMemResolver(s, barPub, barJWT)\n\ts.UpdateAccountClaims(acc, barAC)\n\n\tcheckShadow(1)\n\n\t// Now change export and make sure it goes away as well. So no exports anymore.\n\tfooAC = jwt.NewAccountClaims(fooPub)\n\tfooJWT, _ = fooAC.Encode(okp)\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\tacc, _ = s.LookupAccount(fooPub)\n\ts.UpdateAccountClaims(acc, fooAC)\n\tcheckShadow(0)\n\n\t// Now add it in but with permission required.\n\tstreamExport = &jwt.Export{Subject: \"foo\", Type: jwt.Stream, TokenReq: true}\n\tfooAC.Exports.Add(streamExport)\n\tfooJWT, _ = fooAC.Encode(okp)\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\ts.UpdateAccountClaims(acc, fooAC)\n\n\tcheckShadow(0)\n\n\t// Now put it back as normal.\n\tfooAC = jwt.NewAccountClaims(fooPub)\n\tstreamExport = &jwt.Export{Subject: \"foo\", Type: jwt.Stream}\n\tfooAC.Exports.Add(streamExport)\n\tfooJWT, _ = fooAC.Encode(okp)\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\ts.UpdateAccountClaims(acc, fooAC)\n\n\tcheckShadow(1)\n}\n\nfunc TestJWTAccountImportActivationExpires(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tstreamExport := &jwt.Export{Subject: \"foo\", Type: jwt.Stream, TokenReq: true}\n\tfooAC.Exports.Add(streamExport)\n\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\tacc, _ := s.LookupAccount(fooPub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tstreamImport := &jwt.Import{Account: fooPub, Subject: \"foo\", To: \"import.\", Type: jwt.Stream}\n\n\tactivation := jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"foo\"\n\tactivation.ImportType = jwt.Stream\n\tnow := time.Now()\n\tactivation.IssuedAt = now.Add(-10 * time.Second).Unix()\n\t// These are second resolution. So round up before adding a second.\n\tactivation.Expires = now.Round(time.Second).Add(time.Second).Unix()\n\tactJWT, err := activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\tstreamImport.Token = actJWT\n\tbarAC.Imports.Add(streamImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\tif acc, _ := s.LookupAccount(barPub); acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, barKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"SUB import.foo 1\\r\\nPING\\r\\n\")\n\texpectPong(t, cr)\n\n\tcheckShadow := func(t *testing.T, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 3*time.Second, 15*time.Millisecond, func() error {\n\t\t\tc.mu.Lock()\n\t\t\tdefer c.mu.Unlock()\n\t\t\tsub := c.subs[\"1\"]\n\t\t\tif ls := len(sub.shadow); ls != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected shadows to be %d, got %d\", expected, ls)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// We created a SUB on foo which should create a shadow subscription.\n\tcheckShadow(t, 1)\n\n\ttime.Sleep(1250 * time.Millisecond)\n\n\t// Should have expired and been removed.\n\tcheckShadow(t, 0)\n}\n\nfunc TestJWTAccountLimitsSubs(t *testing.T) {\n\tfooAC := newJWTTestAccountClaims()\n\tfooAC.Limits.Subs = 10\n\ts, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\tfooPub, _ := fooKP.PublicKey()\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, fooKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\t// Check to make sure we have the limit set.\n\t// Account first\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tfooAcc.mu.RLock()\n\tif fooAcc.msubs != 10 {\n\t\tfooAcc.mu.RUnlock()\n\t\tt.Fatalf(\"Expected account to have msubs of 10, got %d\", fooAcc.msubs)\n\t}\n\tfooAcc.mu.RUnlock()\n\t// Now test that the client has limits too.\n\tc.mu.Lock()\n\tif c.msubs != 10 {\n\t\tc.mu.Unlock()\n\t\tt.Fatalf(\"Expected client msubs to be 10, got %d\", c.msubs)\n\t}\n\tc.mu.Unlock()\n\n\t// Now make sure its enforced.\n\t/// These should all work ok.\n\tfor i := 0; i < 10; i++ {\n\t\tc.parseAsync(fmt.Sprintf(\"SUB foo %d\\r\\nPING\\r\\n\", i))\n\t\texpectPong(t, cr)\n\t}\n\n\t// This one should fail.\n\tc.parseAsync(\"SUB foo 22\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR\") {\n\t\tt.Fatalf(\"Expected an ERR, got: %v\", l)\n\t}\n\tif !strings.Contains(l, \"maximum subscriptions exceeded\") {\n\t\tt.Fatalf(\"Expected an ERR for max subscriptions exceeded, got: %v\", l)\n\t}\n\n\t// Now update the claims and expect if max is lower to be disconnected.\n\tfooAC.Limits.Subs = 5\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\ts.UpdateAccountClaims(fooAcc, fooAC)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR\") {\n\t\tt.Fatalf(\"Expected an ERR, got: %v\", l)\n\t}\n\tif !strings.Contains(l, \"maximum subscriptions exceeded\") {\n\t\tt.Fatalf(\"Expected an ERR for max subscriptions exceeded, got: %v\", l)\n\t}\n}\n\nfunc TestJWTAccountLimitsSubsButServerOverrides(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\t// override with server setting of 2.\n\topts := s.getOpts()\n\topts.MaxSubs = 2\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tfooAC.Limits.Subs = 10\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tfooAcc.mu.RLock()\n\tif fooAcc.msubs != 10 {\n\t\tfooAcc.mu.RUnlock()\n\t\tt.Fatalf(\"Expected account to have msubs of 10, got %d\", fooAcc.msubs)\n\t}\n\tfooAcc.mu.RUnlock()\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, fooKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"SUB foo 1\\r\\nSUB bar 2\\r\\nSUB baz 3\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"maximum subscriptions exceeded\") {\n\t\tt.Fatalf(\"Expected an ERR for max subscriptions exceeded, got: %v\", l)\n\t}\n\t// Read last PONG so does not hold up test.\n\tcr.ReadString('\\n')\n}\n\nfunc TestJWTAccountLimitsMaxPayload(t *testing.T) {\n\tfooAC := newJWTTestAccountClaims()\n\tfooAC.Limits.Payload = 8\n\ts, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tfooPub, _ := fooKP.PublicKey()\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, fooKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\t// Check to make sure we have the limit set.\n\t// Account first\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tfooAcc.mu.RLock()\n\tif fooAcc.mpay != 8 {\n\t\tfooAcc.mu.RUnlock()\n\t\tt.Fatalf(\"Expected account to have mpay of 8, got %d\", fooAcc.mpay)\n\t}\n\tfooAcc.mu.RUnlock()\n\t// Now test that the client has limits too.\n\tc.mu.Lock()\n\tif c.mpay != 8 {\n\t\tc.mu.Unlock()\n\t\tt.Fatalf(\"Expected client to have mpay of 10, got %d\", c.mpay)\n\t}\n\tc.mu.Unlock()\n\n\tc.parseAsync(\"PUB foo 4\\r\\nXXXX\\r\\nPING\\r\\n\")\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"PUB foo 10\\r\\nXXXXXXXXXX\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"Maximum Payload\") {\n\t\tt.Fatalf(\"Expected an ERR for max payload violation, got: %v\", l)\n\t}\n}\n\nfunc TestJWTAccountLimitsMaxPayloadButServerOverrides(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\t// override with server setting of 4.\n\topts := s.getOpts()\n\topts.MaxPayload = 4\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tfooAC.Limits.Payload = 8\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\t// Create a client.\n\tc, cr, cs := createClient(t, s, fooKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"PUB foo 6\\r\\nXXXXXX\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"Maximum Payload\") {\n\t\tt.Fatalf(\"Expected an ERR for max payload violation, got: %v\", l)\n\t}\n}\n\nfunc TestJWTAccountLimitsMaxConns(t *testing.T) {\n\tfooAC := newJWTTestAccountClaims()\n\tfooAC.Limits.Conn = 8\n\ts, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tnewClient := func(expPre string) *testAsyncClient {\n\t\tt.Helper()\n\t\t// Create a client.\n\t\tc, cr, cs := createClient(t, s, fooKP)\n\t\tc.parseAsync(cs)\n\t\tl, _ := cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, expPre) {\n\t\t\tt.Fatalf(\"Expected a response starting with %q, got %q\", expPre, l)\n\t\t}\n\t\treturn c\n\t}\n\n\t// A connection is created in setupJWTTestWitAccountClaims(), so limit\n\t// to 7 here (8 total).\n\tfor i := 0; i < 7; i++ {\n\t\tc := newClient(\"PONG\")\n\t\tdefer c.close()\n\t}\n\t// Now this one should fail.\n\tc = newClient(\"-ERR \")\n\tc.close()\n}\n\n// This will test that we can switch from a public export to a private\n// one and back with export claims to make sure the claim update mechanism\n// is working properly.\nfunc TestJWTAccountServiceImportAuthSwitch(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tserviceExport := &jwt.Export{Subject: \"ngs.usage.*\", Type: jwt.Service}\n\tfooAC.Exports.Add(serviceExport)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tserviceImport := &jwt.Import{Account: fooPub, Subject: \"ngs.usage\", To: \"ngs.usage.DEREK\", Type: jwt.Service}\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\t// Create a client that will send the request\n\tca, cra, csa := createClient(t, s, barKP)\n\tdefer ca.close()\n\tca.parseAsync(csa)\n\texpectPong(t, cra)\n\n\t// Create the client that will respond to the requests.\n\tcb, crb, csb := createClient(t, s, fooKP)\n\tdefer cb.close()\n\tcb.parseAsync(csb)\n\texpectPong(t, crb)\n\n\t// Create Subscriber.\n\tcb.parseAsync(\"SUB ngs.usage.* 1\\r\\nPING\\r\\n\")\n\texpectPong(t, crb)\n\n\t// Send Request\n\tca.parseAsync(\"PUB ngs.usage 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should receive the request mapped into our account. PING needed to flush.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, crb, \"ngs.usage.DEREK\", \"hi\")\n\n\t// Now update to make the export private.\n\tfooACPrivate := jwt.NewAccountClaims(fooPub)\n\tserviceExport = &jwt.Export{Subject: \"ngs.usage.*\", Type: jwt.Service, TokenReq: true}\n\tfooACPrivate.Exports.Add(serviceExport)\n\tfooJWTPrivate, err := fooACPrivate.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWTPrivate)\n\tacc, _ := s.LookupAccount(fooPub)\n\ts.UpdateAccountClaims(acc, fooACPrivate)\n\n\t// Send Another Request\n\tca.parseAsync(\"PUB ngs.usage 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should not receive the request this time.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectPong(t, crb)\n\n\t// Now put it back again to public and make sure it works again.\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\ts.UpdateAccountClaims(acc, fooAC)\n\n\t// Send Request\n\tca.parseAsync(\"PUB ngs.usage 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should receive the request mapped into our account. PING needed to flush.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, crb, \"ngs.usage.DEREK\", \"hi\")\n}\n\nfunc TestJWTAccountServiceImportExpires(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tserviceExport := &jwt.Export{Subject: \"foo\", Type: jwt.Service}\n\n\tfooAC.Exports.Add(serviceExport)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tserviceImport := &jwt.Import{Account: fooPub, Subject: \"foo\", Type: jwt.Service}\n\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\t// Create a client that will send the request\n\tca, cra, csa := createClient(t, s, barKP)\n\tdefer ca.close()\n\tca.parseAsync(csa)\n\texpectPong(t, cra)\n\n\t// Create the client that will respond to the requests.\n\tcb, crb, csb := createClient(t, s, fooKP)\n\tdefer cb.close()\n\tcb.parseAsync(csb)\n\texpectPong(t, crb)\n\n\t// Create Subscriber.\n\tcb.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectPong(t, crb)\n\n\t// Send Request\n\tca.parseAsync(\"PUB foo 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should receive the request. PING needed to flush.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, crb, \"foo\", \"hi\")\n\n\t// Now update the exported service to require auth.\n\tfooAC = jwt.NewAccountClaims(fooPub)\n\tserviceExport = &jwt.Export{Subject: \"foo\", Type: jwt.Service, TokenReq: true}\n\n\tfooAC.Exports.Add(serviceExport)\n\tfooJWT, err = fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\tacc, _ := s.LookupAccount(fooPub)\n\ts.UpdateAccountClaims(acc, fooAC)\n\n\t// Send Another Request\n\tca.parseAsync(\"PUB foo 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should not receive the request this time.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectPong(t, crb)\n\n\t// Now get an activation token such that it will work, but will expire.\n\tbarAC = jwt.NewAccountClaims(barPub)\n\tserviceImport = &jwt.Import{Account: fooPub, Subject: \"foo\", Type: jwt.Service}\n\n\tnow := time.Now()\n\tactivation := jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"foo\"\n\tactivation.ImportType = jwt.Service\n\tactivation.IssuedAt = now.Add(-10 * time.Second).Unix()\n\tactivation.Expires = now.Add(time.Second).Round(time.Second).Unix()\n\tactJWT, err := activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\tserviceImport.Token = actJWT\n\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err = barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\tacc, _ = s.LookupAccount(barPub)\n\ts.UpdateAccountClaims(acc, barAC)\n\n\t// Now it should work again.\n\t// Send Another Request\n\tca.parseAsync(\"PUB foo 3\\r\\nhi2\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should receive the request. PING needed to flush.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, crb, \"foo\", \"hi2\")\n\n\t// Now wait for it to expire, then retry.\n\twaitTime := time.Duration(activation.Expires-time.Now().Unix()) * time.Second\n\ttime.Sleep(waitTime + 250*time.Millisecond)\n\n\t// Send Another Request\n\tca.parseAsync(\"PUB foo 3\\r\\nhi3\\r\\nPING\\r\\n\")\n\texpectPong(t, cra)\n\n\t// We should NOT receive the request. PING needed to flush.\n\tcb.parseAsync(\"PING\\r\\n\")\n\texpectPong(t, crb)\n}\n\nfunc TestJWTAccountURLResolver(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuseTLS bool\n\t}{\n\t\t{\"plain\", false},\n\t\t{\"tls\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tkp, _ := nkeys.FromSeed(oSeed)\n\t\t\takp, _ := nkeys.CreateAccount()\n\t\t\tapub, _ := akp.PublicKey()\n\t\t\tnac := jwt.NewAccountClaims(apub)\n\t\t\tajwt, err := nac.Encode(kp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t\t\t}\n\n\t\t\thf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\t\tw.Write([]byte(ajwt))\n\t\t\t})\n\t\t\tvar ts *httptest.Server\n\t\t\tif test.useTLS {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\t\t\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\t\t\t}\n\t\t\t\ttlsConfig, err := GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t\tts = httptest.NewUnstartedServer(hf)\n\t\t\t\tts.TLS = tlsConfig\n\t\t\t\tts.StartTLS()\n\t\t\t} else {\n\t\t\t\tts = httptest.NewServer(hf)\n\t\t\t}\n\t\t\tdefer ts.Close()\n\n\t\t\tconfTemplate := `\n\t\t\t\toperator: %s\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tresolver: URL(\"%s/ngs/v1/accounts/jwt/\")\n\t\t\t\tresolver_tls {\n\t\t\t\t\tcert_file: \"../test/configs/certs/client-cert.pem\"\n\t\t\t\t\tkey_file: \"../test/configs/certs/client-key.pem\"\n\t\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\t}\n\t\t\t`\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL)))\n\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tpub, _ := kp.PublicKey()\n\t\t\topts.TrustedKeys = []string{pub}\n\t\t\tdefer s.Shutdown()\n\n\t\t\tacc, _ := s.LookupAccount(apub)\n\t\t\tif acc == nil {\n\t\t\t\tt.Fatalf(\"Expected to receive an account\")\n\t\t\t}\n\t\t\tif acc.Name != apub {\n\t\t\t\tt.Fatalf(\"Account name did not match claim key\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJWTAccountURLResolverTimeout(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tbasePath := \"/ngs/v1/accounts/jwt/\"\n\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == basePath {\n\t\t\tw.Write([]byte(\"ok\"))\n\t\t\treturn\n\t\t}\n\t\t// Purposely be slow on account lookup.\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tdefer ts.Close()\n\n\tconfTemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tresolver: URL(\"%s%s\")\n `\n\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ts.URL, basePath)))\n\n\ts, opts := RunServerWithConfig(conf)\n\tpub, _ := kp.PublicKey()\n\topts.TrustedKeys = []string{pub}\n\tdefer s.Shutdown()\n\n\t// Lower default timeout to speed-up test\n\ts.AccountResolver().(*URLAccResolver).c.Timeout = 50 * time.Millisecond\n\n\tacc, _ := s.LookupAccount(apub)\n\tif acc != nil {\n\t\tt.Fatalf(\"Expected to not receive an account due to timeout\")\n\t}\n}\n\nfunc TestJWTAccountURLResolverNoFetchOnReload(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tdefer ts.Close()\n\n\tconfTemplate := `\n\t\toperator: %s\n\t\tlisten: 127.0.0.1:-1\n\t\tresolver: URL(\"%s/ngs/v1/accounts/jwt/\")\n `\n\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tacc, _ := s.LookupAccount(apub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to receive an account\")\n\t}\n\n\t// Reload would produce a DATA race during the DeepEqual check for the account resolver,\n\t// so close the current one and we will create a new one that keeps track of fetch calls.\n\tts.Close()\n\n\tfetch := int32(0)\n\tts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tatomic.AddInt32(&fetch, 1)\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tdefer ts.Close()\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, ojwt, ts.URL)))\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\tif atomic.LoadInt32(&fetch) != 0 {\n\t\tt.Fatalf(\"Fetch invoked during reload\")\n\t}\n\n\t// Now stop the resolver and make sure that on startup, we report URL resolver failure\n\ts.Shutdown()\n\ts = nil\n\tts.Close()\n\n\topts := LoadConfig(conf)\n\tif s, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), \"could not fetch\") {\n\t\tif s != nil {\n\t\t\ts.Shutdown()\n\t\t}\n\t\tt.Fatalf(\"Expected error regarding account resolver, got %v\", err)\n\t}\n}\n\nfunc TestJWTAccountURLResolverFetchFailureInServer1(t *testing.T) {\n\tconst subj = \"test\"\n\tconst crossAccSubj = \"test\"\n\t// Create Exporting Account\n\texpkp, _ := nkeys.CreateAccount()\n\texppub, _ := expkp.PublicKey()\n\texpac := jwt.NewAccountClaims(exppub)\n\texpac.Exports.Add(&jwt.Export{\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\texpjwt, err := expac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create importing Account\n\timpkp, _ := nkeys.CreateAccount()\n\timppub, _ := impkp.PublicKey()\n\timpac := jwt.NewAccountClaims(imppub)\n\timpac.Imports.Add(&jwt.Import{\n\t\tAccount: exppub,\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\timpjwt, err := impac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Simulate an account server that drops the first request to exppub\n\tchanImpA := make(chan struct{}, 10)\n\tdefer close(chanImpA)\n\tchanExpS := make(chan struct{}, 10)\n\tdefer close(chanExpS)\n\tchanExpF := make(chan struct{}, 1)\n\tdefer close(chanExpF)\n\tfailureCnt := int32(0)\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/A/\" {\n\t\t\t// Server startup\n\t\t\tw.Write(nil)\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+imppub {\n\t\t\tw.Write([]byte(impjwt))\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+exppub {\n\t\t\tif atomic.AddInt32(&failureCnt, 1) <= 1 {\n\t\t\t\t// skip the write to simulate the failure\n\t\t\t\tchanExpF <- struct{}{}\n\t\t\t} else {\n\t\t\t\tw.Write([]byte(expjwt))\n\t\t\t\tchanExpS <- struct{}{}\n\t\t\t}\n\t\t} else {\n\t\t\tt.Fatal(\"not expected\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\t// Create server\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/A/\")\n `, ojwt, ts.URL)))\n\tsA := RunServer(LoadConfig(confA))\n\tdefer sA.Shutdown()\n\t// server observed one fetch on startup\n\tchanRecv(t, chanImpA, 10*time.Second)\n\t// Create first client\n\tncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp))\n\tdefer ncA.Close()\n\t// create a test subscription\n\tsubA, err := ncA.SubscribeSync(subj)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error during subscribe: %v\", err)\n\t}\n\tdefer subA.Unsubscribe()\n\t// Connect of client triggered a fetch of both accounts\n\t// the fetch for the imported account will fail\n\tchanRecv(t, chanImpA, 10*time.Second)\n\tchanRecv(t, chanExpF, 10*time.Second)\n\t// create second client for user exporting\n\tncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp))\n\tdefer ncB.Close()\n\tchanRecv(t, chanExpS, 10*time.Second)\n\t// Connect of client triggered another fetch, this time passing\n\tcheckSubInterest(t, sA, imppub, subj, 10*time.Second)\n\tcheckSubInterest(t, sA, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue\n}\n\nfunc TestJWTAccountURLResolverFetchFailurePushReorder(t *testing.T) {\n\tconst subj = \"test\"\n\tconst crossAccSubj = \"test\"\n\t// Create System Account\n\tsyskp, _ := nkeys.CreateAccount()\n\tsyspub, _ := syskp.PublicKey()\n\tsysAc := jwt.NewAccountClaims(syspub)\n\tsysjwt, err := sysAc.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create Exporting Account\n\texpkp, _ := nkeys.CreateAccount()\n\texppub, _ := expkp.PublicKey()\n\texpac := jwt.NewAccountClaims(exppub)\n\texpjwt1, err := expac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\texpac.Exports.Add(&jwt.Export{\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\texpjwt2, err := expac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create importing Account\n\timpkp, _ := nkeys.CreateAccount()\n\timppub, _ := impkp.PublicKey()\n\timpac := jwt.NewAccountClaims(imppub)\n\timpac.Imports.Add(&jwt.Import{\n\t\tAccount: exppub,\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\timpjwt, err := impac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Simulate an account server that does not serve the updated jwt for exppub\n\tchanImpA := make(chan struct{}, 10)\n\tdefer close(chanImpA)\n\tchanExpS := make(chan struct{}, 10)\n\tdefer close(chanExpS)\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/A/\" {\n\t\t\t// Server startup\n\t\t\tw.Write(nil)\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+imppub {\n\t\t\tw.Write([]byte(impjwt))\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+exppub {\n\t\t\t// respond with jwt that does not have the export\n\t\t\t// this simulates an ordering issue\n\t\t\tw.Write([]byte(expjwt1))\n\t\t\tchanExpS <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+syspub {\n\t\t\tw.Write([]byte(sysjwt))\n\t\t} else {\n\t\t\tt.Fatal(\"not expected\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/A/\")\n\t\tsystem_account: %s\n `, ojwt, ts.URL, syspub)))\n\tsA := RunServer(LoadConfig(confA))\n\tdefer sA.Shutdown()\n\t// server observed one fetch on startup\n\tchanRecv(t, chanImpA, 10*time.Second)\n\t// Create first client\n\tncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp))\n\tdefer ncA.Close()\n\t// create a test subscription\n\tsubA, err := ncA.SubscribeSync(subj)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error during subscribe: %v\", err)\n\t}\n\tdefer subA.Unsubscribe()\n\t// Connect of client triggered a fetch of both accounts\n\t// the fetch for the imported account will fail\n\tchanRecv(t, chanImpA, 10*time.Second)\n\tchanRecv(t, chanExpS, 10*time.Second)\n\t// create second client for user exporting\n\tncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp))\n\tdefer ncB.Close()\n\t// update expjwt2, this will correct the import issue\n\tsysc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, syskp))\n\tdefer sysc.Close()\n\tnatsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjNew, exppub), []byte(expjwt2))\n\tsysc.Flush()\n\t// updating expjwt should cause this to pass\n\tcheckSubInterest(t, sA, imppub, subj, 10*time.Second)\n\tcheckSubInterest(t, sA, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue\n}\n\ntype captureDebugLogger struct {\n\tDummyLogger\n\tdbgCh chan string\n}\n\nfunc (l *captureDebugLogger) Debugf(format string, v ...any) {\n\tselect {\n\tcase l.dbgCh <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\nfunc TestJWTAccountURLResolverPermanentFetchFailure(t *testing.T) {\n\tconst crossAccSubj = \"test\"\n\texpkp, _ := nkeys.CreateAccount()\n\texppub, _ := expkp.PublicKey()\n\timpkp, _ := nkeys.CreateAccount()\n\timppub, _ := impkp.PublicKey()\n\t// Create System Account\n\tsyskp, _ := nkeys.CreateAccount()\n\tsyspub, _ := syskp.PublicKey()\n\tsysAc := jwt.NewAccountClaims(syspub)\n\tsysjwt, err := sysAc.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create 2 Accounts. Each importing from the other, but NO matching export\n\texpac := jwt.NewAccountClaims(exppub)\n\texpac.Imports.Add(&jwt.Import{\n\t\tAccount: imppub,\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\texpjwt, err := expac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create importing Account\n\timpac := jwt.NewAccountClaims(imppub)\n\timpac.Imports.Add(&jwt.Import{\n\t\tAccount: exppub,\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\timpjwt, err := impac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Simulate an account server that does not serve the updated jwt for exppub\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/A/\" {\n\t\t\t// Server startup\n\t\t\tw.Write(nil)\n\t\t} else if r.URL.Path == \"/A/\"+imppub {\n\t\t\tw.Write([]byte(impjwt))\n\t\t} else if r.URL.Path == \"/A/\"+exppub {\n\t\t\tw.Write([]byte(expjwt))\n\t\t} else if r.URL.Path == \"/A/\"+syspub {\n\t\t\tw.Write([]byte(sysjwt))\n\t\t} else {\n\t\t\tt.Fatal(\"not expected\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/A/\")\n\t\tsystem_account: %s\n `, ojwt, ts.URL, syspub)))\n\to := LoadConfig(confA)\n\tsA := RunServer(o)\n\tdefer sA.Shutdown()\n\tl := &captureDebugLogger{dbgCh: make(chan string, 100)} // has enough space to not block\n\tsA.SetLogger(l, true, false)\n\t// Create clients\n\tncA := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, impkp))\n\tdefer ncA.Close()\n\tncB := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, expkp))\n\tdefer ncB.Close()\n\tsysc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, syskp))\n\tdefer sysc.Close()\n\t// push accounts\n\tnatsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjNew, imppub), []byte(impjwt))\n\tnatsPub(t, sysc, fmt.Sprintf(accUpdateEventSubjOld, exppub), []byte(expjwt))\n\tsysc.Flush()\n\timportErrCnt := 0\n\ttmr := time.NewTimer(500 * time.Millisecond)\n\tdefer tmr.Stop()\n\tfor {\n\t\tselect {\n\t\tcase line := <-l.dbgCh:\n\t\t\tif strings.HasPrefix(line, \"Error adding stream import to account\") {\n\t\t\t\timportErrCnt++\n\t\t\t}\n\t\tcase <-tmr.C:\n\t\t\t// connecting and updating, each cause 3 traces (2 + 1 on iteration) + 1 xtra fetch\n\t\t\tif importErrCnt != 7 {\n\t\t\t\tt.Fatalf(\"Expected 7 debug traces, got %d\", importErrCnt)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestJWTAccountURLResolverFetchFailureInCluster(t *testing.T) {\n\tassertChanLen := func(x int, chans ...chan struct{}) {\n\t\tt.Helper()\n\t\tfor _, c := range chans {\n\t\t\tif len(c) != x {\n\t\t\t\tt.Fatalf(\"length of channel is not %d\", x)\n\t\t\t}\n\t\t}\n\t}\n\tconst subj = \">\"\n\tconst crossAccSubj = \"test\"\n\t// Create Exporting Account\n\texpkp, _ := nkeys.CreateAccount()\n\texppub, _ := expkp.PublicKey()\n\texpac := jwt.NewAccountClaims(exppub)\n\texpac.Exports.Add(&jwt.Export{\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\texpjwt, err := expac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create importing Account\n\timpkp, _ := nkeys.CreateAccount()\n\timppub, _ := impkp.PublicKey()\n\timpac := jwt.NewAccountClaims(imppub)\n\timpac.Imports.Add(&jwt.Import{\n\t\tAccount: exppub,\n\t\tSubject: crossAccSubj,\n\t\tType: jwt.Stream,\n\t})\n\timpac.Exports.Add(&jwt.Export{\n\t\tSubject: \"srvc\",\n\t\tType: jwt.Service,\n\t})\n\timpjwt, err := impac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create User\n\tnkp, _ := nkeys.CreateUser()\n\tuSeed, _ := nkp.Seed()\n\tupub, _ := nkp.PublicKey()\n\tnuc := newJWTTestUserClaims()\n\tnuc.Subject = upub\n\tuJwt, err := nuc.Encode(impkp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tcreds := genCredsFile(t, uJwt, uSeed)\n\t// Simulate an account server that drops the first request to /B/acc\n\tchanImpA := make(chan struct{}, 4)\n\tdefer close(chanImpA)\n\tchanImpB := make(chan struct{}, 4)\n\tdefer close(chanImpB)\n\tchanExpA := make(chan struct{}, 4)\n\tdefer close(chanExpA)\n\tchanExpB := make(chan struct{}, 4)\n\tdefer close(chanExpB)\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/A/\" {\n\t\t\t// Server A startup\n\t\t\tw.Write(nil)\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/B/\" {\n\t\t\t// Server B startup\n\t\t\tw.Write(nil)\n\t\t\tchanImpB <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+imppub {\n\t\t\t// First Client connecting to Server A\n\t\t\tw.Write([]byte(impjwt))\n\t\t\tchanImpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/B/\"+imppub {\n\t\t\t// Second Client connecting to Server B\n\t\t\tw.Write([]byte(impjwt))\n\t\t\tchanImpB <- struct{}{}\n\t\t} else if r.URL.Path == \"/A/\"+exppub {\n\t\t\t// First Client connecting to Server A\n\t\t\tw.Write([]byte(expjwt))\n\t\t\tchanExpA <- struct{}{}\n\t\t} else if r.URL.Path == \"/B/\"+exppub {\n\t\t\t// Second Client connecting to Server B\n\t\t\tw.Write([]byte(expjwt))\n\t\t\tchanExpB <- struct{}{}\n\t\t} else {\n\t\t\tt.Fatal(\"not expected\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\t// Create seed server A\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/A/\")\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tno_advertise: true\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n `, ojwt, ts.URL)))\n\tsA := RunServer(LoadConfig(confA))\n\tdefer sA.Shutdown()\n\t// Create Server B (using no_advertise to prevent failover)\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/B/\")\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tno_advertise: true\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `, ojwt, ts.URL, sA.opts.Cluster.Port)))\n\tsB := RunServer(LoadConfig(confB))\n\tdefer sB.Shutdown()\n\t// startup cluster\n\tcheckClusterFormed(t, sA, sB)\n\t// Both server observed one fetch on startup\n\tchanRecv(t, chanImpA, 10*time.Second)\n\tchanRecv(t, chanImpB, 10*time.Second)\n\tassertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB)\n\t// Create first client, directly connects to A\n\turlA := fmt.Sprintf(\"nats://%s:%d\", sA.opts.Host, sA.opts.Port)\n\tncA, err := nats.Connect(urlA, nats.UserCredentials(creds),\n\t\tnats.DisconnectErrHandler(func(_ *nats.Conn, err error) {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t\t}\n\t\t}),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v %s\", err, urlA)\n\t}\n\tdefer ncA.Close()\n\t// create a test subscription\n\tsubA, err := ncA.SubscribeSync(subj)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error during subscribe: %v\", err)\n\t}\n\tdefer subA.Unsubscribe()\n\t// Connect of client triggered a fetch by Server A\n\tchanRecv(t, chanImpA, 10*time.Second)\n\tchanRecv(t, chanExpA, 10*time.Second)\n\tassertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB)\n\t//time.Sleep(10 * time.Second)\n\t// create second client, directly connect to B\n\turlB := fmt.Sprintf(\"nats://%s:%d\", sB.opts.Host, sB.opts.Port)\n\tncB, err := nats.Connect(urlB, nats.UserCredentials(creds), nats.NoReconnect())\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v %s\", err, urlB)\n\t}\n\tdefer ncB.Close()\n\t// Connect of client triggered a fetch by Server B\n\tchanRecv(t, chanImpB, 10*time.Second)\n\tchanRecv(t, chanExpB, 10*time.Second)\n\tassertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB)\n\tcheckClusterFormed(t, sA, sB)\n\t// the route subscription was lost due to the failed fetch\n\t// Now we test if some recover mechanism is in play\n\tcheckSubInterest(t, sB, imppub, subj, 10*time.Second) // Will fail as a result of this issue\n\tcheckSubInterest(t, sB, exppub, crossAccSubj, 10*time.Second) // Will fail as a result of this issue\n\tif err := ncB.Publish(subj, []byte(\"msg\")); err != nil {\n\t\tt.Fatalf(\"Expected to publish %v\", err)\n\t}\n\t// expect the message from B to flow to A\n\tif m, err := subA.NextMsg(10 * time.Second); err != nil {\n\t\tt.Fatalf(\"Expected to receive a message %v\", err)\n\t} else if string(m.Data) != \"msg\" {\n\t\tt.Fatalf(\"Expected to receive 'msg', got: %s\", string(m.Data))\n\t}\n\tassertChanLen(0, chanImpA, chanImpB, chanExpA, chanExpB)\n}\n\nfunc TestJWTAccountURLResolverReturnDifferentOperator(t *testing.T) {\n\t// Create a valid chain of op/acc/usr using a different operator\n\t// This is so we can test if the server rejects this chain.\n\t// Create Operator\n\top, _ := nkeys.CreateOperator()\n\t// Create Account, this account is the one returned by the resolver\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(op)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create User\n\tnkp, _ := nkeys.CreateUser()\n\tuSeed, _ := nkp.Seed()\n\tupub, _ := nkp.PublicKey()\n\tnuc := newJWTTestUserClaims()\n\tnuc.Subject = upub\n\tuJwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tcreds := genCredsFile(t, uJwt, uSeed)\n\t// Simulate an account server that was hijacked/mis configured\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tdefer ts.Close()\n\t// Create Server\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: URL(\"%s/A/\")\n `, ojwt, ts.URL)))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\t// Create first client, directly connects to A\n\turlA := fmt.Sprintf(\"nats://%s:%d\", sA.opts.Host, sA.opts.Port)\n\tif _, err := nats.Connect(urlA, nats.UserCredentials(creds),\n\t\tnats.DisconnectErrHandler(func(_ *nats.Conn, err error) {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t\t}\n\t\t}),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t}),\n\t); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\t// Test if the server has the account in memory. (shouldn't)\n\tif v, ok := sA.accounts.Load(apub); ok {\n\t\tt.Fatalf(\"Expected account to NOT be in memory: %v\", v.(*Account))\n\t}\n}\n\nfunc TestJWTUserSigningKey(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\n\t// Check to make sure we would have an authTimer\n\tif !s.info.AuthRequired {\n\t\tt.Fatalf(\"Expect the server to require auth\")\n\t}\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\t// Don't send jwt field, should fail.\n\tc.parseAsync(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create an account\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\n\t// Create a signing key for the account\n\taskp, _ := nkeys.CreateAccount()\n\taspub, _ := askp.PublicKey()\n\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t// Create a client with the account signing key\n\tc, cr, cs := createClientWithIssuer(t, s, askp, apub)\n\tdefer c.close()\n\n\t// PING needed to flush the +OK/-ERR to us.\n\t// This should fail too since no account resolver is defined.\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Ok now let's walk through and make sure all is good.\n\t// We will set the account resolver by hand to a memory resolver.\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\t// Create a client with a signing key\n\tc, cr, cs = createClientWithIssuer(t, s, askp, apub)\n\tdefer c.close()\n\t// should fail because the signing key is not known\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error: %v\", l)\n\t}\n\n\t// add a signing key\n\tnac.SigningKeys.Add(aspub)\n\t// update the memory resolver\n\tacc, _ := s.LookupAccount(apub)\n\ts.UpdateAccountClaims(acc, nac)\n\n\t// Create a client with a signing key\n\tc, cr, cs = createClientWithIssuer(t, s, askp, apub)\n\tdefer c.close()\n\n\t// expect this to work\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"PONG\") {\n\t\tt.Fatalf(\"Expected a PONG, got %q\", l)\n\t}\n\n\tisClosed := func() bool {\n\t\tc.mu.Lock()\n\t\tdefer c.mu.Unlock()\n\t\treturn c.isClosed()\n\t}\n\n\tif isClosed() {\n\t\tt.Fatal(\"expected client to be alive\")\n\t}\n\t// remove the signing key should bounce client\n\tnac.SigningKeys = nil\n\tacc, _ = s.LookupAccount(apub)\n\ts.UpdateAccountClaims(acc, nac)\n\n\tif !isClosed() {\n\t\tt.Fatal(\"expected client to be gone\")\n\t}\n}\n\nfunc TestJWTAccountImportSignerRemoved(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Exporter keys\n\tsrvKP, _ := nkeys.CreateAccount()\n\tsrvPK, _ := srvKP.PublicKey()\n\tsrvSignerKP, _ := nkeys.CreateAccount()\n\tsrvSignerPK, _ := srvSignerKP.PublicKey()\n\n\t// Importer keys\n\tclientKP, _ := nkeys.CreateAccount()\n\tclientPK, _ := clientKP.PublicKey()\n\n\tcreateSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) {\n\t\tac := jwt.NewAccountClaims(srvPK)\n\t\tac.SigningKeys.Add(signingKeys...)\n\t\tac.Exports.Add(&jwt.Export{Subject: \"foo\", Type: jwt.Service, TokenReq: true})\n\t\tac.Exports.Add(&jwt.Export{Subject: \"bar\", Type: jwt.Stream, TokenReq: true})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating exporter JWT: %v\", err)\n\t\t}\n\t\treturn token, ac\n\t}\n\n\tcreateImportToken := func(sub string, kind jwt.ExportType) string {\n\t\tactC := jwt.NewActivationClaims(clientPK)\n\t\tactC.IssuerAccount = srvPK\n\t\tactC.ImportType = kind\n\t\tactC.ImportSubject = jwt.Subject(sub)\n\t\ttoken, err := actC.Encode(srvSignerKP)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\treturn token\n\t}\n\n\tcreateClientJwt := func() string {\n\t\tac := jwt.NewAccountClaims(clientPK)\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"foo\", Type: jwt.Service, Token: createImportToken(\"foo\", jwt.Service)})\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"bar\", Type: jwt.Stream, Token: createImportToken(\"bar\", jwt.Stream)})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating importer JWT: %v\", err)\n\t\t}\n\t\treturn token\n\t}\n\n\tsrvJWT, _ := createSrvJwt(srvSignerPK)\n\taddAccountToMemResolver(s, srvPK, srvJWT)\n\n\tclientJWT := createClientJwt()\n\taddAccountToMemResolver(s, clientPK, clientJWT)\n\n\t// Create a client that will send the request\n\tclient, clientReader, clientCS := createClient(t, s, clientKP)\n\tdefer client.close()\n\tclient.parseAsync(clientCS)\n\texpectPong(t, clientReader)\n\n\tcheckShadow := func(expected int) {\n\t\tt.Helper()\n\t\tclient.mu.Lock()\n\t\tdefer client.mu.Unlock()\n\t\tsub := client.subs[\"1\"]\n\t\tcount := 0\n\t\tif sub != nil {\n\t\t\tcount = len(sub.shadow)\n\t\t}\n\t\tif count != expected {\n\t\t\tt.Fatalf(\"Expected shadows to be %d, got %d\", expected, count)\n\t\t}\n\t}\n\n\tcheckShadow(0)\n\t// Create the client that will respond to the requests.\n\tsrv, srvReader, srvCS := createClient(t, s, srvKP)\n\tdefer srv.close()\n\tsrv.parseAsync(srvCS)\n\texpectPong(t, srvReader)\n\n\t// Create Subscriber.\n\tsrv.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectPong(t, srvReader)\n\n\t// Send Request\n\tclient.parseAsync(\"PUB foo 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, clientReader)\n\n\t// We should receive the request. PING needed to flush.\n\tsrv.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, srvReader, \"foo\", \"hi\")\n\n\tclient.parseAsync(\"SUB bar 1\\r\\nPING\\r\\n\")\n\texpectPong(t, clientReader)\n\tcheckShadow(1)\n\n\tsrv.parseAsync(\"PUB bar 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, srvReader)\n\n\t// We should receive from stream. PING needed to flush.\n\tclient.parseAsync(\"PING\\r\\n\")\n\texpectMsg(t, clientReader, \"bar\", \"hi\")\n\n\t// Now update the exported service no signer\n\tsrvJWT, srvAC := createSrvJwt()\n\taddAccountToMemResolver(s, srvPK, srvJWT)\n\tacc, _ := s.LookupAccount(srvPK)\n\ts.UpdateAccountClaims(acc, srvAC)\n\n\t// Send Another Request\n\tclient.parseAsync(\"PUB foo 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, clientReader)\n\n\t// We should not receive the request this time.\n\tsrv.parseAsync(\"PING\\r\\n\")\n\texpectPong(t, srvReader)\n\n\t// Publish on the stream\n\tsrv.parseAsync(\"PUB bar 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectPong(t, srvReader)\n\n\t// We should not receive from the stream this time\n\tclient.parseAsync(\"PING\\r\\n\")\n\texpectPong(t, clientReader)\n\tcheckShadow(0)\n}\n\nfunc TestJWTAccountImportSignerDeadlock(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Exporter keys\n\tsrvKP, _ := nkeys.CreateAccount()\n\tsrvPK, _ := srvKP.PublicKey()\n\tsrvSignerKP, _ := nkeys.CreateAccount()\n\tsrvSignerPK, _ := srvSignerKP.PublicKey()\n\n\t// Importer keys\n\tclientKP, _ := nkeys.CreateAccount()\n\tclientPK, _ := clientKP.PublicKey()\n\n\tcreateSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) {\n\t\tac := jwt.NewAccountClaims(srvPK)\n\t\tac.SigningKeys.Add(signingKeys...)\n\t\tac.Exports.Add(&jwt.Export{Subject: \"foo\", Type: jwt.Service, TokenReq: true})\n\t\tac.Exports.Add(&jwt.Export{Subject: \"bar\", Type: jwt.Stream, TokenReq: true})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating exporter JWT: %v\", err)\n\t\t}\n\t\treturn token, ac\n\t}\n\n\tcreateImportToken := func(sub string, kind jwt.ExportType) string {\n\t\tactC := jwt.NewActivationClaims(clientPK)\n\t\tactC.IssuerAccount = srvPK\n\t\tactC.ImportType = kind\n\t\tactC.ImportSubject = jwt.Subject(sub)\n\t\ttoken, err := actC.Encode(srvSignerKP)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\treturn token\n\t}\n\n\tcreateClientJwt := func() string {\n\t\tac := jwt.NewAccountClaims(clientPK)\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"foo\", Type: jwt.Service, Token: createImportToken(\"foo\", jwt.Service)})\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"bar\", Type: jwt.Stream, Token: createImportToken(\"bar\", jwt.Stream)})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating importer JWT: %v\", err)\n\t\t}\n\t\treturn token\n\t}\n\n\tsrvJWT, _ := createSrvJwt(srvSignerPK)\n\taddAccountToMemResolver(s, srvPK, srvJWT)\n\n\tclientJWT := createClientJwt()\n\taddAccountToMemResolver(s, clientPK, clientJWT)\n\n\tacc, _ := s.LookupAccount(srvPK)\n\t// Have a go routine that constantly gets/releases the acc's write lock.\n\t// There was a bug that could cause AddServiceImportWithClaim to deadlock.\n\tch := make(chan bool, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tacc.mu.Lock()\n\t\t\t\tacc.mu.Unlock()\n\t\t\t\ttime.Sleep(time.Millisecond)\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Create a client that will send the request\n\tclient, clientReader, clientCS := createClient(t, s, clientKP)\n\tdefer client.close()\n\tclient.parseAsync(clientCS)\n\texpectPong(t, clientReader)\n\n\tclose(ch)\n\twg.Wait()\n}\n\nfunc TestJWTAccountImportWrongIssuerAccount(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 2)}\n\ts.SetLogger(l, false, false)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Exporter keys\n\tsrvKP, _ := nkeys.CreateAccount()\n\tsrvPK, _ := srvKP.PublicKey()\n\tsrvSignerKP, _ := nkeys.CreateAccount()\n\tsrvSignerPK, _ := srvSignerKP.PublicKey()\n\n\t// Importer keys\n\tclientKP, _ := nkeys.CreateAccount()\n\tclientPK, _ := clientKP.PublicKey()\n\n\tcreateSrvJwt := func(signingKeys ...string) (string, *jwt.AccountClaims) {\n\t\tac := jwt.NewAccountClaims(srvPK)\n\t\tac.SigningKeys.Add(signingKeys...)\n\t\tac.Exports.Add(&jwt.Export{Subject: \"foo\", Type: jwt.Service, TokenReq: true})\n\t\tac.Exports.Add(&jwt.Export{Subject: \"bar\", Type: jwt.Stream, TokenReq: true})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating exporter JWT: %v\", err)\n\t\t}\n\t\treturn token, ac\n\t}\n\n\tcreateImportToken := func(sub string, kind jwt.ExportType) string {\n\t\tactC := jwt.NewActivationClaims(clientPK)\n\t\t// Reference ourselves, which is wrong.\n\t\tactC.IssuerAccount = clientPK\n\t\tactC.ImportType = kind\n\t\tactC.ImportSubject = jwt.Subject(sub)\n\t\ttoken, err := actC.Encode(srvSignerKP)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\treturn token\n\t}\n\n\tcreateClientJwt := func() string {\n\t\tac := jwt.NewAccountClaims(clientPK)\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"foo\", Type: jwt.Service, Token: createImportToken(\"foo\", jwt.Service)})\n\t\tac.Imports.Add(&jwt.Import{Account: srvPK, Subject: \"bar\", Type: jwt.Stream, Token: createImportToken(\"bar\", jwt.Stream)})\n\t\ttoken, err := ac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating importer JWT: %v\", err)\n\t\t}\n\t\treturn token\n\t}\n\n\tsrvJWT, _ := createSrvJwt(srvSignerPK)\n\taddAccountToMemResolver(s, srvPK, srvJWT)\n\n\tclientJWT := createClientJwt()\n\taddAccountToMemResolver(s, clientPK, clientJWT)\n\n\t// Create a client that will send the request\n\tclient, clientReader, clientCS := createClient(t, s, clientKP)\n\tdefer client.close()\n\tclient.parseAsync(clientCS)\n\tif l, _, err := clientReader.ReadLine(); err != nil {\n\t\tt.Fatalf(\"Expected no Error, got: %v\", err)\n\t} else if !strings.Contains(string(l), \"-ERR 'Authorization Violation'\") {\n\t\tt.Fatalf(\"Expected Error, got: %v\", l)\n\t}\n}\n\nfunc TestJWTUserRevokedOnAccountUpdate(t *testing.T) {\n\tnac := newJWTTestAccountClaims()\n\ts, akp, c, cr := setupJWTTestWitAccountClaims(t, nac, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\texpectPong(t, cr)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\tapub, _ := akp.PublicKey()\n\n\tc.mu.Lock()\n\tpub := c.user.Nkey\n\tc.mu.Unlock()\n\n\t// Now revoke the user.\n\tnac.Revoke(pub)\n\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t// Update the account on the server.\n\taddAccountToMemResolver(s, apub, ajwt)\n\tacc, err := s.LookupAccount(apub)\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up the account: %v\", err)\n\t}\n\n\t// This is simulating a system update for the account claims.\n\tgo s.updateAccountWithClaimJWT(acc, ajwt)\n\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"Revoked\") {\n\t\tt.Fatalf(\"Expected 'Revoked' to be in the error\")\n\t}\n}\n\nfunc TestJWTUserRevoked(t *testing.T) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create a new user that we will make sure has been revoked.\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\t// Revoke the user right away.\n\tnac.Revoke(pub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t// Sign for the user.\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, l := newClientForServer(s)\n\tdefer c.close()\n\n\t// Sign Nonce\n\tvar info nonceInfo\n\tjson.Unmarshal([]byte(l[5:]), &info)\n\tsigraw, _ := nkp.Sign([]byte(info.Nonce))\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK/-ERR to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"sig\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\", jwt, sig)\n\n\tc.parseAsync(cs)\n\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\tif !strings.Contains(l, \"Authorization\") {\n\t\tt.Fatalf(\"Expected 'Revoked' to be in the error\")\n\t}\n}\n\n// Test that an account update that revokes an import authorization cancels the import.\nfunc TestJWTImportTokenRevokedAfter(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\n\t// Now create Exports.\n\texport := &jwt.Export{Subject: \"foo.private\", Type: jwt.Stream, TokenReq: true}\n\n\tfooAC.Exports.Add(export)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tsimport := &jwt.Import{Account: fooPub, Subject: \"foo.private\", Type: jwt.Stream}\n\n\tactivation := jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"foo.private\"\n\tactivation.ImportType = jwt.Stream\n\tactJWT, err := activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\n\tsimport.Token = actJWT\n\tbarAC.Imports.Add(simport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\t// Now revoke the export.\n\tdecoded, _ := jwt.DecodeActivationClaims(actJWT)\n\texport.Revoke(decoded.Subject)\n\n\tfooJWT, err = fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tif fooAcc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\t// Now lookup bar account and make sure it was revoked.\n\tacc, _ := s.LookupAccount(barPub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\tif les := len(acc.imports.streams); les != 0 {\n\t\tt.Fatalf(\"Expected imports streams len of 0, got %d\", les)\n\t}\n}\n\n// Test that an account update that revokes an import authorization cancels the import.\nfunc TestJWTImportTokenRevokedBefore(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\n\t// Now create Exports.\n\texport := &jwt.Export{Subject: \"foo.private\", Type: jwt.Stream, TokenReq: true}\n\n\tfooAC.Exports.Add(export)\n\n\t// Import account\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\tsimport := &jwt.Import{Account: fooPub, Subject: \"foo.private\", Type: jwt.Stream}\n\n\tactivation := jwt.NewActivationClaims(barPub)\n\tactivation.ImportSubject = \"foo.private\"\n\tactivation.ImportType = jwt.Stream\n\tactJWT, err := activation.Encode(fooKP)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating activation token: %v\", err)\n\t}\n\n\tsimport.Token = actJWT\n\tbarAC.Imports.Add(simport)\n\n\t// Now revoke the export.\n\tdecoded, _ := jwt.DecodeActivationClaims(actJWT)\n\texport.Revoke(decoded.Subject)\n\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tif fooAcc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\n\t// Now lookup bar account and make sure it was revoked.\n\tacc, _ := s.LookupAccount(barPub)\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to retrieve the account\")\n\t}\n\tif les := len(acc.imports.streams); les != 0 {\n\t\tt.Fatalf(\"Expected imports streams len of 0, got %d\", les)\n\t}\n}\n\nfunc TestJWTCircularAccountServiceImport(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\n\tbarKP, _ := nkeys.CreateAccount()\n\tbarPub, _ := barKP.PublicKey()\n\tbarAC := jwt.NewAccountClaims(barPub)\n\n\t// Create service export/import for account foo\n\tserviceExport := &jwt.Export{Subject: \"foo\", Type: jwt.Service, TokenReq: true}\n\tserviceImport := &jwt.Import{Account: barPub, Subject: \"bar\", Type: jwt.Service}\n\n\tfooAC.Exports.Add(serviceExport)\n\tfooAC.Imports.Add(serviceImport)\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\t// Create service export/import for account bar\n\tserviceExport = &jwt.Export{Subject: \"bar\", Type: jwt.Service, TokenReq: true}\n\tserviceImport = &jwt.Import{Account: fooPub, Subject: \"foo\", Type: jwt.Service}\n\n\tbarAC.Exports.Add(serviceExport)\n\tbarAC.Imports.Add(serviceImport)\n\tbarJWT, err := barAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, barPub, barJWT)\n\n\tc, cr, cs := createClient(t, s, fooKP)\n\tdefer c.close()\n\n\tc.parseAsync(cs)\n\texpectPong(t, cr)\n\n\tc.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectPong(t, cr)\n}\n\n// This test ensures that connected clients are properly evicted\n// (no deadlock) if the max conns of an account has been lowered\n// and the account is being updated (following expiration during\n// a lookup).\nfunc TestJWTAccountLimitsMaxConnsAfterExpired(t *testing.T) {\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create accounts and imports/exports.\n\tfooKP, _ := nkeys.CreateAccount()\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAC := jwt.NewAccountClaims(fooPub)\n\tfooAC.Limits.Conn = 10\n\tfooJWT, err := fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\tnewClient := func(expPre string) *testAsyncClient {\n\t\tt.Helper()\n\t\t// Create a client.\n\t\tc, cr, cs := createClient(t, s, fooKP)\n\t\tc.parseAsync(cs)\n\t\tl, _ := cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, expPre) {\n\t\t\tt.Fatalf(\"Expected a response starting with %q, got %q\", expPre, l)\n\t\t}\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tif _, _, err := cr.ReadLine(); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t\treturn c\n\t}\n\n\tfor i := 0; i < 4; i++ {\n\t\tc := newClient(\"PONG\")\n\t\tdefer c.close()\n\t}\n\n\t// We will simulate that the account has expired. When\n\t// a new client will connect, the server will do a lookup\n\t// and find the account expired, which then will cause\n\t// a fetch and a rebuild of the account. Since max conns\n\t// is now lower, some clients should have been removed.\n\tacc, _ := s.LookupAccount(fooPub)\n\tacc.mu.Lock()\n\tacc.expired.Store(true)\n\tacc.updated = time.Now().UTC().Add(-2 * time.Second) // work around updating to quickly\n\tacc.mu.Unlock()\n\n\t// Now update with new expiration and max connections lowered to 2\n\tfooAC.Limits.Conn = 2\n\tfooJWT, err = fooAC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\taddAccountToMemResolver(s, fooPub, fooJWT)\n\n\t// Cause the lookup that will detect that account was expired\n\t// and rebuild it, and kick clients out.\n\tc := newClient(\"-ERR \")\n\tdefer c.close()\n\n\tacc, _ = s.LookupAccount(fooPub)\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tacc.mu.RLock()\n\t\tnumClients := len(acc.clients)\n\t\tacc.mu.RUnlock()\n\t\tif numClients != 2 {\n\t\t\treturn fmt.Errorf(\"Should have 2 clients, got %v\", numClients)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestJWTBearerToken(t *testing.T) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tnuc := newJWTTestUserClaims()\n\tnuc.Subject = pub\n\t// Set bearer token.\n\tnuc.BearerToken = true\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Skip nonce signature...\n\n\t// PING needed to flush the +OK/-ERR to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tc.parse([]byte(cs))\n\t\twg.Done()\n\t}()\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected +OK, got %s\", l)\n\t}\n\twg.Wait()\n}\n\nfunc TestJWTBearerWithIssuerSameAsAccountToken(t *testing.T) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tnuc := newJWTTestUserClaims()\n\t// we are setting the issuer account here to trigger verification\n\t// of the issuer - the account has no signing keys, but the issuer\n\t// account is set to the public key of the account which should be OK.\n\tnuc.IssuerAccount = apub\n\tnuc.Subject = pub\n\t// Set bearer token.\n\tnuc.BearerToken = true\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Skip nonce signature...\n\n\t// PING needed to flush the +OK/-ERR to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tc.parse([]byte(cs))\n\t\twg.Done()\n\t}()\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected +OK, got %s\", l)\n\t}\n\twg.Wait()\n}\n\nfunc TestJWTBearerWithBadIssuerToken(t *testing.T) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tnuc := newJWTTestUserClaims()\n\tbakp, _ := nkeys.CreateAccount()\n\tbapub, _ := bakp.PublicKey()\n\tnuc.IssuerAccount = bapub\n\tnuc.Subject = pub\n\t// Set bearer token.\n\tnuc.BearerToken = true\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\ts := opTrustBasicSetup()\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, _ := newClientForServer(s)\n\tdefer c.close()\n\n\t// Skip nonce signature...\n\n\t// PING needed to flush the +OK/-ERR to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tc.parse([]byte(cs))\n\t\twg.Done()\n\t}()\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR\") {\n\t\tt.Fatalf(\"Expected -ERR, got %s\", l)\n\t}\n\twg.Wait()\n}\n\nfunc TestJWTExpiredUserCredentialsRenewal(t *testing.T) {\n\tcreateTmpFile := func(t *testing.T, content []byte) string {\n\t\tt.Helper()\n\t\tconf := createTempFile(t, _EMPTY_)\n\t\tfName := conf.Name()\n\t\tconf.Close()\n\t\tif err := os.WriteFile(fName, content, 0666); err != nil {\n\t\t\tt.Fatalf(\"Error writing conf file: %v\", err)\n\t\t}\n\t\treturn fName\n\t}\n\twaitTime := func(ch chan bool, timeout time.Duration) error {\n\t\tselect {\n\t\tcase <-ch:\n\t\t\treturn nil\n\t\tcase <-time.After(timeout):\n\t\t}\n\t\treturn errors.New(\"timeout\")\n\t}\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, err := nkeys.CreateAccount()\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account\")\n\t}\n\taPub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(aPub)\n\taJwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tkp, _ := nkeys.FromSeed(oSeed)\n\toPub, _ := kp.PublicKey()\n\topts := defaultServerOptions\n\topts.TrustedKeys = []string{oPub}\n\ts := RunServer(&opts)\n\tif s == nil {\n\t\tt.Fatal(\"Server did not start\")\n\t}\n\tdefer s.Shutdown()\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, aPub, aJwt)\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tuSeed, _ := nkp.Seed()\n\tnuc := newJWTTestUserClaims()\n\tnuc.Subject = pub\n\tnuc.Expires = time.Now().Add(time.Second).Unix()\n\tuJwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\tcreds, err := jwt.FormatUserConfig(uJwt, uSeed)\n\tif err != nil {\n\t\tt.Fatalf(\"Error encoding credentials: %v\", err)\n\t}\n\tchainedFile := createTmpFile(t, creds)\n\n\trch := make(chan bool)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", s.opts.Host, s.opts.Port)\n\tnc, err := nats.Connect(url,\n\t\tnats.UserCredentials(chainedFile),\n\t\tnats.ReconnectWait(25*time.Millisecond),\n\t\tnats.ReconnectJitter(0, 0),\n\t\tnats.MaxReconnects(2),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t\tnats.ReconnectHandler(func(nc *nats.Conn) {\n\t\t\trch <- true\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v %s\", err, url)\n\t}\n\tdefer nc.Close()\n\n\t// Place new credentials underneath.\n\tnuc.Expires = time.Now().Add(30 * time.Second).Unix()\n\tuJwt, err = nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error encoding user jwt: %v\", err)\n\t}\n\tcreds, err = jwt.FormatUserConfig(uJwt, uSeed)\n\tif err != nil {\n\t\tt.Fatalf(\"Error encoding credentials: %v\", err)\n\t}\n\tif err := os.WriteFile(chainedFile, creds, 0666); err != nil {\n\t\tt.Fatalf(\"Error writing conf file: %v\", err)\n\t}\n\n\t// Make sure we get disconnected and reconnected first.\n\tif err := waitTime(rch, 2*time.Second); err != nil {\n\t\tt.Fatal(\"Should have reconnected.\")\n\t}\n\n\t// We should not have been closed.\n\tif nc.IsClosed() {\n\t\tt.Fatal(\"Got disconnected when we should have reconnected.\")\n\t}\n\n\t// Check that we clear the lastErr that can cause the disconnect.\n\t// Our reconnect CB will happen before the clear. So check after a bit.\n\ttime.Sleep(50 * time.Millisecond)\n\tif nc.LastError() != nil {\n\t\tt.Fatalf(\"Expected lastErr to be cleared, got %q\", nc.LastError())\n\t}\n}\n\nfunc updateJwt(t *testing.T, url string, creds string, jwt string, respCnt int) int {\n\tt.Helper()\n\trequire_NextMsg := func(sub *nats.Subscription) bool {\n\t\tt.Helper()\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tcontent := make(map[string]any)\n\t\tjson.Unmarshal(msg.Data, &content)\n\t\tif _, ok := content[\"data\"]; ok {\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\tc := natsConnect(t, url, nats.UserCredentials(creds),\n\t\tnats.DisconnectErrHandler(func(_ *nats.Conn, err error) {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t\t}\n\t\t}),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tt.Fatal(\"error not expected in this test\", err)\n\t\t}),\n\t)\n\tdefer c.Close()\n\tresp := c.NewRespInbox()\n\tsub := natsSubSync(t, c, resp)\n\terr := sub.AutoUnsubscribe(respCnt)\n\trequire_NoError(t, err)\n\trequire_NoError(t, c.PublishRequest(accClaimsReqSubj, resp, []byte(jwt)))\n\tpassCnt := 0\n\tfor i := 0; i < respCnt; i++ {\n\t\tif require_NextMsg(sub) {\n\t\t\tpassCnt++\n\t\t}\n\t}\n\treturn passCnt\n}\n\nfunc require_JWTAbsent(t *testing.T, dir string, pub string) {\n\tt.Helper()\n\t_, err := os.Stat(filepath.Join(dir, pub+\".jwt\"))\n\trequire_Error(t, err)\n\trequire_True(t, os.IsNotExist(err))\n}\n\nfunc require_JWTPresent(t *testing.T, dir string, pub string) {\n\tt.Helper()\n\t_, err := os.Stat(filepath.Join(dir, pub+\".jwt\"))\n\trequire_NoError(t, err)\n}\n\nfunc require_JWTEqual(t *testing.T, dir string, pub string, jwt string) {\n\tt.Helper()\n\tcontent, err := os.ReadFile(filepath.Join(dir, pub+\".jwt\"))\n\trequire_NoError(t, err)\n\trequire_Equal(t, string(content), jwt)\n}\n\nfunc createTempFile(t testing.TB, prefix string) *os.File {\n\tt.Helper()\n\ttempDir := t.TempDir()\n\tf, err := os.CreateTemp(tempDir, prefix)\n\trequire_NoError(t, err)\n\treturn f\n}\n\nfunc removeDir(t testing.TB, dir string) {\n\tt.Helper()\n\tif err := os.RemoveAll(dir); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc removeFile(t testing.TB, p string) {\n\tt.Helper()\n\tif err := os.Remove(p); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc writeJWT(t *testing.T, dir string, pub string, jwt string) {\n\tt.Helper()\n\terr := os.WriteFile(filepath.Join(dir, pub+\".jwt\"), []byte(jwt), 0644)\n\trequire_NoError(t, err)\n}\n\nfunc TestJWTAccountNATSResolverFetch(t *testing.T) {\n\torigEventsHBInterval := eventsHBInterval\n\teventsHBInterval = 50 * time.Millisecond // speed up eventing\n\tdefer func() { eventsHBInterval = origEventsHBInterval }()\n\trequire_NoLocalOrRemoteConnections := func(account string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tfor _, srv := range srvs {\n\t\t\tif acc, ok := srv.accounts.Load(account); ok {\n\t\t\t\tcheckAccClientsCount(t, acc.(*Account), 0)\n\t\t\t}\n\t\t}\n\t}\n\t// After each connection check, require_XConnection and connect assures that\n\t// listed server have no connections for the account used\n\trequire_1Connection := func(url, creds, acc string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tfunc() {\n\t\t\tt.Helper()\n\t\t\tc := natsConnect(t, url, nats.UserCredentials(creds))\n\t\t\tdefer c.Close()\n\t\t\tif _, err := nats.Connect(url, nats.UserCredentials(creds)); err == nil {\n\t\t\t\tt.Fatal(\"Second connection was supposed to fail due to limits\")\n\t\t\t} else if !strings.Contains(err.Error(), ErrTooManyAccountConnections.Error()) {\n\t\t\t\tt.Fatal(\"Second connection was supposed to fail with too many conns\")\n\t\t\t}\n\t\t}()\n\t\trequire_NoLocalOrRemoteConnections(acc, srvs...)\n\t}\n\trequire_2Connection := func(url, creds, acc string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tfunc() {\n\t\t\tt.Helper()\n\t\t\tc1 := natsConnect(t, url, nats.UserCredentials(creds))\n\t\t\tdefer c1.Close()\n\t\t\tc2 := natsConnect(t, url, nats.UserCredentials(creds))\n\t\t\tdefer c2.Close()\n\t\t\tif _, err := nats.Connect(url, nats.UserCredentials(creds)); err == nil {\n\t\t\t\tt.Fatal(\"Third connection was supposed to fail due to limits\")\n\t\t\t} else if !strings.Contains(err.Error(), ErrTooManyAccountConnections.Error()) {\n\t\t\t\tt.Fatal(\"Third connection was supposed to fail with too many conns\")\n\t\t\t}\n\t\t}()\n\t\trequire_NoLocalOrRemoteConnections(acc, srvs...)\n\t}\n\tconnect := func(url string, credsfile string, acc string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tnc := natsConnect(t, url, nats.UserCredentials(credsfile), nats.Timeout(5*time.Second))\n\t\tnc.Close()\n\t\trequire_NoLocalOrRemoteConnections(acc, srvs...)\n\t}\n\tcreateAccountAndUser := func(limit bool, done chan struct{}, pubKey, jwt1, jwt2, creds *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tif limit {\n\t\t\tclaim.Limits.Conn = 1\n\t\t}\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\t// need to assure that create time differs (resolution is sec)\n\t\ttime.Sleep(time.Millisecond * 1100)\n\t\t// create updated claim allowing more connections\n\t\tif limit {\n\t\t\tclaim.Limits.Conn = 2\n\t\t}\n\t\t*jwt2, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tujwt, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds = genCredsFile(t, ujwt, seed)\n\t\tdone <- struct{}{}\n\t}\n\t// Create Accounts and corresponding user creds. Do so concurrently to speed up the test\n\tdoneChan := make(chan struct{}, 5)\n\tdefer close(doneChan)\n\tvar syspub, sysjwt, dummy1, sysCreds string\n\tgo createAccountAndUser(false, doneChan, &syspub, &sysjwt, &dummy1, &sysCreds)\n\tvar apub, ajwt1, ajwt2, aCreds string\n\tgo createAccountAndUser(true, doneChan, &apub, &ajwt1, &ajwt2, &aCreds)\n\tvar bpub, bjwt1, bjwt2, bCreds string\n\tgo createAccountAndUser(true, doneChan, &bpub, &bjwt1, &bjwt2, &bCreds)\n\tvar cpub, cjwt1, cjwt2, cCreds string\n\tgo createAccountAndUser(true, doneChan, &cpub, &cjwt1, &cjwt2, &cCreds)\n\tvar dpub, djwt1, dummy2, dCreds string // extra user used later in the test in order to test limits\n\tgo createAccountAndUser(true, doneChan, &dpub, &djwt1, &dummy2, &dCreds)\n\tfor i := 0; i < cap(doneChan); i++ {\n\t\t<-doneChan\n\t}\n\t// Create one directory for each server\n\tdirA := t.TempDir()\n\tdirB := t.TempDir()\n\tdirC := t.TempDir()\n\t// simulate a restart of the server by storing files in them\n\t// Server A/B will completely sync, so after startup each server\n\t// will contain the union off all stored/configured jwt\n\t// Server C will send out lookup requests for jwt it does not store itself\n\twriteJWT(t, dirA, apub, ajwt1)\n\twriteJWT(t, dirB, bpub, bjwt1)\n\twriteJWT(t, dirC, cpub, cjwt1)\n\t// Create seed server A (using no_advertise to prevent fail over)\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-A\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t\tlimit: 4\n\t\t}\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t}\n `, ojwt, syspub, dirA, cpub, cjwt1)))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\t// during startup resolver_preload causes the directory to contain data\n\trequire_JWTPresent(t, dirA, cpub)\n\t// Create Server B (using no_advertise to prevent fail over)\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-B\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t\tlimit: 4\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `, ojwt, syspub, dirB, sA.opts.Cluster.Port)))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\t// Create Server C (using no_advertise to prevent fail over)\n\tfmtC := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-C\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: cache\n\t\t\tdir: '%s'\n\t\t\tttl: \"%dms\"\n\t\t\tlimit: 4\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `\n\tconfClongTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, 10000, sA.opts.Cluster.Port)))\n\tconfCshortTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, 1000, sA.opts.Cluster.Port)))\n\tsC, _ := RunServerWithConfig(confClongTTL) // use long ttl to assure it is not kicking\n\tdefer sC.Shutdown()\n\t// startup cluster\n\tcheckClusterFormed(t, sA, sB, sC)\n\ttime.Sleep(500 * time.Millisecond) // wait for the protocol to converge\n\t// Check all accounts\n\trequire_JWTPresent(t, dirA, apub) // was already present on startup\n\trequire_JWTPresent(t, dirB, apub) // was copied from server A\n\trequire_JWTAbsent(t, dirC, apub)\n\trequire_JWTPresent(t, dirA, bpub) // was copied from server B\n\trequire_JWTPresent(t, dirB, bpub) // was already present on startup\n\trequire_JWTAbsent(t, dirC, bpub)\n\trequire_JWTPresent(t, dirA, cpub) // was present in preload\n\trequire_JWTPresent(t, dirB, cpub) // was copied from server A\n\trequire_JWTPresent(t, dirC, cpub) // was already present on startup\n\t// This is to test that connecting to it still works\n\trequire_JWTAbsent(t, dirA, syspub)\n\trequire_JWTAbsent(t, dirB, syspub)\n\trequire_JWTAbsent(t, dirC, syspub)\n\t// system account client can connect to every server\n\tconnect(sA.ClientURL(), sysCreds, \"\")\n\tconnect(sB.ClientURL(), sysCreds, \"\")\n\tconnect(sC.ClientURL(), sysCreds, \"\")\n\tcheckClusterFormed(t, sA, sB, sC)\n\t// upload system account and require a response from each server\n\tpassCnt := updateJwt(t, sA.ClientURL(), sysCreds, sysjwt, 3)\n\trequire_True(t, passCnt == 3)\n\trequire_JWTPresent(t, dirA, syspub) // was just received\n\trequire_JWTPresent(t, dirB, syspub) // was just received\n\trequire_JWTPresent(t, dirC, syspub) // was just received\n\t// Only files missing are in C, which is only caching\n\tconnect(sC.ClientURL(), aCreds, apub, sA, sB, sC)\n\tconnect(sC.ClientURL(), bCreds, bpub, sA, sB, sC)\n\trequire_JWTPresent(t, dirC, apub) // was looked up form A or B\n\trequire_JWTPresent(t, dirC, bpub) // was looked up from A or B\n\t// Check limits and update jwt B connecting to server A\n\tfor port, v := range map[string]struct{ pub, jwt, creds string }{\n\t\tsB.ClientURL(): {bpub, bjwt2, bCreds},\n\t\tsC.ClientURL(): {cpub, cjwt2, cCreds},\n\t} {\n\t\trequire_1Connection(sA.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\trequire_1Connection(sB.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\trequire_1Connection(sC.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\tpassCnt := updateJwt(t, port, sysCreds, v.jwt, 3)\n\t\trequire_True(t, passCnt == 3)\n\t\trequire_2Connection(sA.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\trequire_2Connection(sB.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\trequire_2Connection(sC.ClientURL(), v.creds, v.pub, sA, sB, sC)\n\t\trequire_JWTEqual(t, dirA, v.pub, v.jwt)\n\t\trequire_JWTEqual(t, dirB, v.pub, v.jwt)\n\t\trequire_JWTEqual(t, dirC, v.pub, v.jwt)\n\t}\n\t// Simulates A having missed an update\n\t// shutting B down as it has it will directly connect to A and connect right away\n\tsB.Shutdown()\n\twriteJWT(t, dirB, apub, ajwt2) // this will be copied to server A\n\tsB, _ = RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\tcheckClusterFormed(t, sA, sB, sC)\n\ttime.Sleep(500 * time.Millisecond) // wait for the protocol to converge\n\t// Restart server C. this is a workaround to force C to do a lookup in the absence of account cleanup\n\tsC.Shutdown()\n\tsC, _ = RunServerWithConfig(confClongTTL) //TODO remove this once we clean up accounts\n\tdefer sC.Shutdown()\n\trequire_JWTEqual(t, dirA, apub, ajwt2) // was copied from server B\n\trequire_JWTEqual(t, dirB, apub, ajwt2) // was restarted with this\n\trequire_JWTEqual(t, dirC, apub, ajwt1) // still contains old cached value\n\trequire_2Connection(sA.ClientURL(), aCreds, apub, sA, sB, sC)\n\trequire_2Connection(sB.ClientURL(), aCreds, apub, sA, sB, sC)\n\trequire_1Connection(sC.ClientURL(), aCreds, apub, sA, sB, sC)\n\t// Restart server C. this is a workaround to force C to do a lookup in the absence of account cleanup\n\tsC.Shutdown()\n\tsC, _ = RunServerWithConfig(confCshortTTL) //TODO remove this once we clean up accounts\n\tdefer sC.Shutdown()\n\trequire_JWTEqual(t, dirC, apub, ajwt1) // still contains old cached value\n\tcheckClusterFormed(t, sA, sB, sC)\n\t// Force next connect to do a lookup exceeds ttl\n\tfname := filepath.Join(dirC, apub+\".jwt\")\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t_, err := os.Stat(fname)\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"File not removed in time\")\n\t})\n\tconnect(sC.ClientURL(), aCreds, apub, sA, sB, sC) // When lookup happens\n\trequire_JWTEqual(t, dirC, apub, ajwt2) // was looked up form A or B\n\trequire_2Connection(sC.ClientURL(), aCreds, apub, sA, sB, sC)\n\t// Test exceeding limit. For the exclusive directory resolver, limit is a stop gap measure.\n\t// It is not expected to be hit. When hit the administrator is supposed to take action.\n\tpassCnt = updateJwt(t, sA.ClientURL(), sysCreds, djwt1, 3)\n\trequire_True(t, passCnt == 1) // Only Server C updated\n\tfor _, srv := range []*Server{sA, sB, sC} {\n\t\tif a, ok := srv.accounts.Load(syspub); ok {\n\t\t\tacc := a.(*Account)\n\t\t\tcheckFor(t, time.Second, 20*time.Millisecond, func() error {\n\t\t\t\tacc.mu.Lock()\n\t\t\t\tdefer acc.mu.Unlock()\n\t\t\t\tif acc.ctmr != nil {\n\t\t\t\t\treturn fmt.Errorf(\"Timer still exists\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestJWTAccountNATSResolverCrossClusterFetch(t *testing.T) {\n\tconnect := func(url string, credsfile string) {\n\t\tt.Helper()\n\t\tnc := natsConnect(t, url, nats.UserCredentials(credsfile))\n\t\tnc.Close()\n\t}\n\tcreateAccountAndUser := func(done chan struct{}, pubKey, jwt1, jwt2, creds *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\t// need to assure that create time differs (resolution is sec)\n\t\ttime.Sleep(time.Millisecond * 1100)\n\t\t// create updated claim\n\t\tclaim.Tags.Add(\"tag\")\n\t\t*jwt2, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tujwt, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds = genCredsFile(t, ujwt, seed)\n\t\tdone <- struct{}{}\n\t}\n\t// Create Accounts and corresponding user creds. Do so concurrently to speed up the test\n\tdoneChan := make(chan struct{}, 3)\n\tdefer close(doneChan)\n\tvar syspub, sysjwt, dummy1, sysCreds string\n\tgo createAccountAndUser(doneChan, &syspub, &sysjwt, &dummy1, &sysCreds)\n\tvar apub, ajwt1, ajwt2, aCreds string\n\tgo createAccountAndUser(doneChan, &apub, &ajwt1, &ajwt2, &aCreds)\n\tvar bpub, bjwt1, bjwt2, bCreds string\n\tgo createAccountAndUser(doneChan, &bpub, &bjwt1, &bjwt2, &bCreds)\n\tfor i := 0; i < cap(doneChan); i++ {\n\t\t<-doneChan\n\t}\n\t// Create one directory for each server\n\tdirAA := t.TempDir()\n\tdirAB := t.TempDir()\n\tdirBA := t.TempDir()\n\tdirBB := t.TempDir()\n\t// simulate a restart of the server by storing files in them\n\t// Server AA & AB will completely sync\n\t// Server BA & BB will completely sync\n\t// Be aware that no syncing will occur between cluster\n\twriteJWT(t, dirAA, apub, ajwt1)\n\twriteJWT(t, dirBA, bpub, bjwt1)\n\t// Create seed server A (using no_advertise to prevent fail over)\n\tconfAA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-A-A\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tgateway: {\n\t\t\tname: \"clust-A\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: clust-A\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t}\n `, ojwt, syspub, dirAA)))\n\tsAA, _ := RunServerWithConfig(confAA)\n\tdefer sAA.Shutdown()\n\t// Create Server B (using no_advertise to prevent fail over)\n\tconfAB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-A-B\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tgateway: {\n\t\t\tname: \"clust-A\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tcluster {\n\t\t\tname: clust-A\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `, ojwt, syspub, dirAB, sAA.opts.Cluster.Port)))\n\tsAB, _ := RunServerWithConfig(confAB)\n\tdefer sAB.Shutdown()\n\t// Create Server C (using no_advertise to prevent fail over)\n\tconfBA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-B-A\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tgateway: {\n\t\t\tname: \"clust-B\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tgateways: [\n\t\t\t\t{name: \"clust-A\", url: \"nats://127.0.0.1:%d\"},\n\t\t\t]\n\t\t}\n\t\tcluster {\n\t\t\tname: clust-B\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t}\n `, ojwt, syspub, dirBA, sAA.opts.Gateway.Port)))\n\tsBA, _ := RunServerWithConfig(confBA)\n\tdefer sBA.Shutdown()\n\t// Create Server BA (using no_advertise to prevent fail over)\n\tconfBB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-B-B\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tcluster {\n\t\t\tname: clust-B\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n\t\tgateway: {\n\t\t\tname: \"clust-B\"\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tgateways: [\n\t\t\t\t{name: \"clust-A\", url: \"nats://127.0.0.1:%d\"},\n\t\t\t]\n\t\t}\n `, ojwt, syspub, dirBB, sBA.opts.Cluster.Port, sAA.opts.Cluster.Port)))\n\tsBB, _ := RunServerWithConfig(confBB)\n\tdefer sBB.Shutdown()\n\t// Assert topology\n\tcheckClusterFormed(t, sAA, sAB)\n\tcheckClusterFormed(t, sBA, sBB)\n\twaitForOutboundGateways(t, sAA, 1, 5*time.Second)\n\twaitForOutboundGateways(t, sAB, 1, 5*time.Second)\n\twaitForOutboundGateways(t, sBA, 1, 5*time.Second)\n\twaitForOutboundGateways(t, sBB, 1, 5*time.Second)\n\ttime.Sleep(500 * time.Millisecond) // wait for the protocol to converge\n\tupdateJwt(t, sAA.ClientURL(), sysCreds, sysjwt, 4) // update system account jwt on all server\n\trequire_JWTEqual(t, dirAA, syspub, sysjwt) // assure this update made it to every server\n\trequire_JWTEqual(t, dirAB, syspub, sysjwt) // assure this update made it to every server\n\trequire_JWTEqual(t, dirBA, syspub, sysjwt) // assure this update made it to every server\n\trequire_JWTEqual(t, dirBB, syspub, sysjwt) // assure this update made it to every server\n\trequire_JWTAbsent(t, dirAA, bpub) // assure that jwt are not synced across cluster\n\trequire_JWTAbsent(t, dirAB, bpub) // assure that jwt are not synced across cluster\n\trequire_JWTAbsent(t, dirBA, apub) // assure that jwt are not synced across cluster\n\trequire_JWTAbsent(t, dirBB, apub) // assure that jwt are not synced across cluster\n\tconnect(sAA.ClientURL(), aCreds) // connect to cluster where jwt was initially stored\n\tconnect(sAB.ClientURL(), aCreds) // connect to cluster where jwt was initially stored\n\tconnect(sBA.ClientURL(), bCreds) // connect to cluster where jwt was initially stored\n\tconnect(sBB.ClientURL(), bCreds) // connect to cluster where jwt was initially stored\n\ttime.Sleep(500 * time.Millisecond) // wait for the protocol to (NOT) converge\n\trequire_JWTAbsent(t, dirAA, bpub) // assure that jwt are still not synced across cluster\n\trequire_JWTAbsent(t, dirAB, bpub) // assure that jwt are still not synced across cluster\n\trequire_JWTAbsent(t, dirBA, apub) // assure that jwt are still not synced across cluster\n\trequire_JWTAbsent(t, dirBB, apub) // assure that jwt are still not synced across cluster\n\t// We have verified that account B does not exist in cluster A, neither does account A in cluster B\n\t// Despite that clients from account B can connect to server A, same for account A in cluster B\n\tconnect(sAA.ClientURL(), bCreds) // connect to cluster where jwt was not initially stored\n\tconnect(sAB.ClientURL(), bCreds) // connect to cluster where jwt was not initially stored\n\tconnect(sBA.ClientURL(), aCreds) // connect to cluster where jwt was not initially stored\n\tconnect(sBB.ClientURL(), aCreds) // connect to cluster where jwt was not initially stored\n\trequire_JWTEqual(t, dirAA, bpub, bjwt1) // assure that now jwt used in connect is stored\n\trequire_JWTEqual(t, dirAB, bpub, bjwt1) // assure that now jwt used in connect is stored\n\trequire_JWTEqual(t, dirBA, apub, ajwt1) // assure that now jwt used in connect is stored\n\trequire_JWTEqual(t, dirBB, apub, ajwt1) // assure that now jwt used in connect is stored\n\tupdateJwt(t, sAA.ClientURL(), sysCreds, bjwt2, 4) // update bjwt, expect updates from everywhere\n\tupdateJwt(t, sBA.ClientURL(), sysCreds, ajwt2, 4) // update ajwt, expect updates from everywhere\n\trequire_JWTEqual(t, dirAA, bpub, bjwt2) // assure that jwt got updated accordingly\n\trequire_JWTEqual(t, dirAB, bpub, bjwt2) // assure that jwt got updated accordingly\n\trequire_JWTEqual(t, dirBA, apub, ajwt2) // assure that jwt got updated accordingly\n\trequire_JWTEqual(t, dirBB, apub, ajwt2) // assure that jwt got updated accordingly\n}\n\nfunc newTimeRange(start time.Time, dur time.Duration) jwt.TimeRange {\n\treturn jwt.TimeRange{Start: start.Format(\"15:04:05\"), End: start.Add(dur).Format(\"15:04:05\")}\n}\n\nfunc createUserWithLimit(t *testing.T, accKp nkeys.KeyPair, expiration time.Time, limits func(*jwt.UserPermissionLimits)) string {\n\tt.Helper()\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub\n\tif limits != nil {\n\t\tlimits(&uclaim.UserPermissionLimits)\n\t}\n\tif !expiration.IsZero() {\n\t\tuclaim.Expires = expiration.Unix()\n\t}\n\tvr := jwt.ValidationResults{}\n\tuclaim.Validate(&vr)\n\trequire_Len(t, len(vr.Errors()), 0)\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn genCredsFile(t, ujwt, seed)\n}\n\nfunc TestJWTUserLimits(t *testing.T) {\n\t// helper for time\n\tinAnHour := time.Now().Add(time.Hour)\n\tinTwoHours := time.Now().Add(2 * time.Hour)\n\tdoNotExpire := time.Now().AddDate(1, 0, 0)\n\t// create account\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n `, ojwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\tfor _, v := range []struct {\n\t\tpass bool\n\t\tf func(*jwt.UserPermissionLimits)\n\t}{\n\t\t{true, nil},\n\t\t{false, func(j *jwt.UserPermissionLimits) { j.Src.Set(\"8.8.8.8/8\") }},\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Src.Set(\"8.8.8.8/0\") }},\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Src.Set(\"127.0.0.1/8\") }},\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Src.Set(\"8.8.8.8/8,127.0.0.1/8\") }},\n\t\t{false, func(j *jwt.UserPermissionLimits) { j.Src.Set(\"8.8.8.8/8,9.9.9.9/8\") }},\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), time.Hour)) }},\n\t\t{false, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = append(j.Times, newTimeRange(time.Now().Add(time.Hour), time.Hour))\n\t\t}},\n\t\t{true, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = append(j.Times, newTimeRange(inAnHour, time.Hour), newTimeRange(time.Now(), time.Hour))\n\t\t}}, // last one is within range\n\t\t{false, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = append(j.Times, newTimeRange(inAnHour, time.Hour), newTimeRange(inTwoHours, time.Hour))\n\t\t}}, // out of range\n\t\t{false, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = append(j.Times, newTimeRange(inAnHour, 3*time.Hour), newTimeRange(inTwoHours, 2*time.Hour))\n\t\t}}, // overlapping [a[]b] out of range*/\n\t\t{false, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = append(j.Times, newTimeRange(inAnHour, 3*time.Hour), newTimeRange(inTwoHours, time.Hour))\n\t\t}}, // overlapping [a[b]] out of range\n\t\t// next day tests where end < begin\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), 25*time.Hour)) }},\n\t\t{true, func(j *jwt.UserPermissionLimits) { j.Times = append(j.Times, newTimeRange(time.Now(), -time.Hour)) }},\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tcreds := createUserWithLimit(t, kp, doNotExpire, v.f)\n\t\t\tif c, err := nats.Connect(sA.ClientURL(), nats.UserCredentials(creds)); err == nil {\n\t\t\t\tc.Close()\n\t\t\t\tif !v.pass {\n\t\t\t\t\tt.Fatalf(\"Expected failure got none\")\n\t\t\t\t}\n\t\t\t} else if v.pass {\n\t\t\t\tt.Fatalf(\"Expected success got %v\", err)\n\t\t\t} else if !strings.Contains(err.Error(), \"Authorization Violation\") {\n\t\t\t\tt.Fatalf(\"Expected error other than %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJWTTimeExpiration(t *testing.T) {\n\tvalidFor := 1500 * time.Millisecond\n\tvalidRange := 500 * time.Millisecond\n\tdoNotExpire := time.Now().AddDate(1, 0, 0)\n\t// create account\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n `, ojwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\tfor _, l := range []string{\"\", \"Europe/Berlin\", \"America/New_York\"} {\n\t\tt.Run(\"simple expiration \"+l, func(t *testing.T) {\n\t\t\tstart := time.Now()\n\t\t\tcreds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) {\n\t\t\t\tif l == _EMPTY_ {\n\t\t\t\t\tj.Times = []jwt.TimeRange{newTimeRange(start, validFor)}\n\t\t\t\t} else {\n\t\t\t\t\tloc, err := time.LoadLocation(l)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tj.Times = []jwt.TimeRange{newTimeRange(start.In(loc), validFor)}\n\t\t\t\t\tj.Locale = l\n\t\t\t\t}\n\t\t\t})\n\t\t\tdisconnectChan := make(chan struct{})\n\t\t\tdefer close(disconnectChan)\n\t\t\terrChan := make(chan struct{})\n\t\t\tdefer close(errChan)\n\t\t\tc := natsConnect(t, sA.ClientURL(),\n\t\t\t\tnats.UserCredentials(creds),\n\t\t\t\tnats.DisconnectErrHandler(func(conn *nats.Conn, err error) {\n\t\t\t\t\tif err != io.EOF {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tdisconnectChan <- struct{}{}\n\t\t\t\t}),\n\t\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\t\tif err != nats.ErrAuthExpired {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tnow := time.Now()\n\t\t\t\t\tstop := start.Add(validFor)\n\t\t\t\t\t// assure event happens within a second of stop\n\t\t\t\t\tif stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) {\n\t\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t\t}\n\t\t\t\t}))\n\t\t\tdefer c.Close()\n\t\t\tchanRecv(t, errChan, 10*time.Second)\n\t\t\tchanRecv(t, disconnectChan, 10*time.Second)\n\t\t\trequire_True(t, c.IsReconnecting())\n\t\t\trequire_False(t, c.IsConnected())\n\t\t})\n\t}\n\tt.Run(\"double expiration\", func(t *testing.T) {\n\t\tstart1 := time.Now()\n\t\tstart2 := start1.Add(2 * validFor)\n\t\tcreds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) {\n\t\t\tj.Times = []jwt.TimeRange{newTimeRange(start1, validFor), newTimeRange(start2, validFor)}\n\t\t})\n\t\terrChan := make(chan struct{})\n\t\tdefer close(errChan)\n\t\treConnectChan := make(chan struct{})\n\t\tdefer close(reConnectChan)\n\t\tc := natsConnect(t, sA.ClientURL(),\n\t\t\tnats.UserCredentials(creds),\n\t\t\tnats.ReconnectHandler(func(conn *nats.Conn) {\n\t\t\t\treConnectChan <- struct{}{}\n\t\t\t}),\n\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\tif err != nats.ErrAuthExpired {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tnow := time.Now()\n\t\t\t\tstop := start1.Add(validFor)\n\t\t\t\t// assure event happens within a second of stop\n\t\t\t\tif stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) {\n\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tstop = start2.Add(validFor)\n\t\t\t\t// assure event happens within a second of stop\n\t\t\t\tif stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) {\n\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t}\n\t\t\t}))\n\t\tdefer c.Close()\n\t\tchanRecv(t, errChan, 10*time.Second)\n\t\tchanRecv(t, reConnectChan, 10*time.Second)\n\t\trequire_False(t, c.IsReconnecting())\n\t\trequire_True(t, c.IsConnected())\n\t\tchanRecv(t, errChan, 10*time.Second)\n\t})\n\tt.Run(\"lower jwt expiration overwrites time\", func(t *testing.T) {\n\t\tstart := time.Now()\n\t\tcreds := createUserWithLimit(t, kp, start.Add(validFor), func(j *jwt.UserPermissionLimits) { j.Times = []jwt.TimeRange{newTimeRange(start, 2*validFor)} })\n\t\tdisconnectChan := make(chan struct{})\n\t\tdefer close(disconnectChan)\n\t\terrChan := make(chan struct{})\n\t\tdefer close(errChan)\n\t\tc := natsConnect(t, sA.ClientURL(),\n\t\t\tnats.UserCredentials(creds),\n\t\t\tnats.DisconnectErrHandler(func(conn *nats.Conn, err error) {\n\t\t\t\tif err != io.EOF {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdisconnectChan <- struct{}{}\n\t\t\t}),\n\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\tif err != nats.ErrAuthExpired {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tnow := time.Now()\n\t\t\t\tstop := start.Add(validFor)\n\t\t\t\t// assure event happens within a second of stop\n\t\t\t\tif stop.Add(-validRange).Before(stop) && now.Before(stop.Add(validRange)) {\n\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t}\n\t\t\t}))\n\t\tdefer c.Close()\n\t\tchanRecv(t, errChan, 10*time.Second)\n\t\tchanRecv(t, disconnectChan, 10*time.Second)\n\t\trequire_True(t, c.IsReconnecting())\n\t\trequire_False(t, c.IsConnected())\n\t})\n}\n\nfunc NewJwtAccountClaim(name string) (nkeys.KeyPair, string, *jwt.AccountClaims) {\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tclaim := jwt.NewAccountClaims(sysPub)\n\tclaim.Name = name\n\treturn sysKp, sysPub, claim\n}\n\nfunc TestJWTSysImportForDifferentAccount(t *testing.T) {\n\t_, sysPub, sysClaim := NewJwtAccountClaim(\"SYS\")\n\tsysClaim.Exports.Add(&jwt.Export{\n\t\tType: jwt.Service,\n\t\tSubject: \"$SYS.REQ.ACCOUNT.*.INFO\",\n\t})\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// create account\n\taKp, aPub, claim := NewJwtAccountClaim(\"A\")\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\tSubject: \"$SYS.REQ.ACCOUNT.*.INFO\",\n\t\tLocalSubject: \"COMMON.ADVISORY.SYS.REQ.ACCOUNT.*.INFO\",\n\t\tAccount: sysPub,\n\t})\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\n\tnc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp))\n\tdefer nc.Close()\n\t// user for account a requests for a different account, the system account\n\tm, err := nc.Request(fmt.Sprintf(\"COMMON.ADVISORY.SYS.REQ.ACCOUNT.%s.INFO\", sysPub), nil, time.Second)\n\trequire_NoError(t, err)\n\tresp := &ServerAPIResponse{}\n\trequire_NoError(t, json.Unmarshal(m.Data, resp))\n\trequire_True(t, resp.Error == nil)\n}\n\nfunc TestJWTSysImportFromNothing(t *testing.T) {\n\t_, sysPub, sysClaim := NewJwtAccountClaim(\"SYS\")\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// create account\n\taKp, aPub, claim := NewJwtAccountClaim(\"A\")\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\t// fails as it's not for own account, but system account\n\t\tSubject: jwt.Subject(fmt.Sprintf(\"$SYS.REQ.ACCOUNT.%s.CONNZ\", sysPub)),\n\t\tLocalSubject: \"fail1\",\n\t\tAccount: sysPub,\n\t})\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\t// fails as it's not for own account but all accounts\n\t\tSubject: \"$SYS.REQ.ACCOUNT.*.CONNZ\",\n\t\tLocalSubject: \"fail2.*\",\n\t\tAccount: sysPub,\n\t})\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\tSubject: jwt.Subject(fmt.Sprintf(\"$SYS.REQ.ACCOUNT.%s.CONNZ\", aPub)),\n\t\tLocalSubject: \"pass\",\n\t\tAccount: sysPub,\n\t})\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\n\tnc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp))\n\tdefer nc.Close()\n\t// user for account a requests for a different account, the system account\n\t_, err = nc.Request(\"pass\", nil, time.Second)\n\trequire_NoError(t, err)\n\t// default import\n\t_, err = nc.Request(\"$SYS.REQ.ACCOUNT.PING.CONNZ\", nil, time.Second)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(\"fail1\", nil, time.Second)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"no responders\")\n\t// fails even for own account, as the import itself is bad\n\t_, err = nc.Request(\"fail2.\"+aPub, nil, time.Second)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"no responders\")\n}\n\nfunc TestJWTSysImportOverwritePublic(t *testing.T) {\n\t_, sysPub, sysClaim := NewJwtAccountClaim(\"SYS\")\n\t// this changes the export permissions to allow for requests for every account\n\tsysClaim.Exports.Add(&jwt.Export{\n\t\tType: jwt.Service,\n\t\tSubject: \"$SYS.REQ.ACCOUNT.*.>\",\n\t})\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// create account\n\taKp, aPub, claim := NewJwtAccountClaim(\"A\")\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\tSubject: jwt.Subject(fmt.Sprintf(\"$SYS.REQ.ACCOUNT.%s.CONNZ\", sysPub)),\n\t\tLocalSubject: \"pass1\",\n\t\tAccount: sysPub,\n\t})\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\tSubject: jwt.Subject(fmt.Sprintf(\"$SYS.REQ.ACCOUNT.%s.CONNZ\", aPub)),\n\t\tLocalSubject: \"pass2\",\n\t\tAccount: sysPub,\n\t})\n\tclaim.Imports.Add(&jwt.Import{\n\t\tType: jwt.Service,\n\t\tSubject: \"$SYS.REQ.ACCOUNT.*.CONNZ\",\n\t\tLocalSubject: \"pass3.*\",\n\t\tAccount: sysPub,\n\t})\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\n\tnc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp))\n\tdefer nc.Close()\n\t// user for account a requests for a different account, the system account\n\t_, err = nc.Request(\"pass1\", nil, time.Second)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(\"pass2\", nil, time.Second)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(\"pass3.\"+sysPub, nil, time.Second)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(\"pass3.\"+aPub, nil, time.Second)\n\trequire_NoError(t, err)\n\t_, err = nc.Request(\"pass3.PING\", nil, time.Second)\n\trequire_NoError(t, err)\n}\n\nfunc TestJWTSysImportOverwriteToken(t *testing.T) {\n\t_, sysPub, sysClaim := NewJwtAccountClaim(\"SYS\")\n\t// this changes the export permissions in a way that the internal imports can't satisfy\n\tsysClaim.Exports.Add(&jwt.Export{\n\t\tType: jwt.Service,\n\t\tSubject: \"$SYS.REQ.>\",\n\t\tTokenReq: true,\n\t})\n\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\t// create account\n\taKp, aPub, claim := NewJwtAccountClaim(\"A\")\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, sysPub, sysPub, sysJwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\n\tnc := natsConnect(t, sA.ClientURL(), createUserCreds(t, nil, aKp))\n\tdefer nc.Close()\n\t// make sure the internal import still got added\n\t_, err = nc.Request(\"$SYS.REQ.ACCOUNT.PING.CONNZ\", nil, time.Second)\n\trequire_NoError(t, err)\n}\n\nfunc TestJWTLimits(t *testing.T) {\n\tdoNotExpire := time.Now().AddDate(1, 0, 0)\n\t// create account\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n `, ojwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\terrChan := make(chan struct{})\n\tdefer close(errChan)\n\tt.Run(\"subs\", func(t *testing.T) {\n\t\tcreds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { j.Subs = 1 })\n\t\tc := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds),\n\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\tif e := conn.LastError(); e != nil && strings.Contains(e.Error(), \"maximum subscriptions exceeded\") {\n\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t}\n\t\t\t}),\n\t\t)\n\t\tdefer c.Close()\n\t\tif _, err := c.Subscribe(\"foo\", func(msg *nats.Msg) {}); err != nil {\n\t\t\tt.Fatalf(\"couldn't subscribe: %v\", err)\n\t\t}\n\t\tif _, err = c.Subscribe(\"bar\", func(msg *nats.Msg) {}); err != nil {\n\t\t\tt.Fatalf(\"expected error got: %v\", err)\n\t\t}\n\t\tchanRecv(t, errChan, time.Second)\n\t})\n\tt.Run(\"payload\", func(t *testing.T) {\n\t\tcreds := createUserWithLimit(t, kp, doNotExpire, func(j *jwt.UserPermissionLimits) { j.Payload = 5 })\n\t\tc := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds))\n\t\tdefer c.Close()\n\t\tif err := c.Flush(); err != nil {\n\t\t\tt.Fatalf(\"flush failed %v\", err)\n\t\t}\n\t\tif err := c.Publish(\"foo\", []byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"couldn't publish: %v\", err)\n\t\t}\n\t\tif err := c.Publish(\"foo\", []byte(\"worldX\")); err != nats.ErrMaxPayload {\n\t\t\tt.Fatalf(\"couldn't publish: %v\", err)\n\t\t}\n\t})\n}\n\nfunc TestJWTTemplates(t *testing.T) {\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tukp, _ := nkeys.CreateUser()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"myname\"\n\tuclaim.Subject = upub\n\tuclaim.SetScoped(true)\n\tuclaim.IssuerAccount = aPub\n\tuclaim.Tags.Add(\"foo:foo1\")\n\tuclaim.Tags.Add(\"foo:foo2\")\n\tuclaim.Tags.Add(\"bar:bar1\")\n\tuclaim.Tags.Add(\"bar:bar2\")\n\tuclaim.Tags.Add(\"bar:bar3\")\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Pub.Allow.Add(\"{{tag(foo)}}.none.{{tag(bar)}}\")\n\tlim.Pub.Deny.Add(\"{{tag(foo)}}.{{account-tag(acc)}}\")\n\tlim.Sub.Allow.Add(\"{{tag(NOT_THERE)}}\") // expect to not emit this\n\tlim.Sub.Deny.Add(\"foo.{{name()}}.{{subject()}}.{{account-name()}}.{{account-subject()}}.bar\")\n\tacc := &Account{nameTag: \"accname\", tags: []string{\"acc:acc1\", \"acc:acc2\"}}\n\n\tresLim, err := processUserPermissionsTemplate(lim, uclaim, acc)\n\trequire_NoError(t, err)\n\n\ttest := func(expectedSubjects []string, res jwt.StringList) {\n\t\tt.Helper()\n\t\trequire_True(t, len(res) == len(expectedSubjects))\n\t\tfor _, expetedSubj := range expectedSubjects {\n\t\t\trequire_True(t, res.Contains(expetedSubj))\n\t\t}\n\t}\n\n\ttest(resLim.Pub.Allow, []string{\"foo1.none.bar1\", \"foo1.none.bar2\", \"foo1.none.bar3\",\n\t\t\"foo2.none.bar1\", \"foo2.none.bar2\", \"foo2.none.bar3\"})\n\n\ttest(resLim.Pub.Deny, []string{\"foo1.acc1\", \"foo1.acc2\", \"foo2.acc1\", \"foo2.acc2\"})\n\n\trequire_True(t, len(resLim.Sub.Allow) == 0)\n\trequire_True(t, len(resLim.Sub.Deny) == 2)\n\trequire_Contains(t, resLim.Sub.Deny[0], fmt.Sprintf(\"foo.myname.%s.accname.%s.bar\", upub, aPub))\n\t// added in to compensate for sub allow not resolving\n\trequire_Contains(t, resLim.Sub.Deny[1], \">\")\n\n\tlim.Pub.Deny.Add(\"{{tag(NOT_THERE)}}\")\n\t_, err = processUserPermissionsTemplate(lim, uclaim, acc)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"generated invalid subject\")\n}\n\nfunc TestJWTInLineTemplates(t *testing.T) {\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tukp, _ := nkeys.CreateUser()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"myname\"\n\tuclaim.Subject = upub\n\tuclaim.SetScoped(true)\n\tuclaim.IssuerAccount = aPub\n\tuclaim.Tags.Add(\"bucket:a\")\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Pub.Allow.Add(\"$JS.API.STREAM.INFO.KV_{{tag(bucket)}}\")\n\tacc := &Account{nameTag: \"accname\", tags: []string{\"acc:acc1\", \"acc:acc2\"}}\n\n\tresLim, err := processUserPermissionsTemplate(lim, uclaim, acc)\n\trequire_NoError(t, err)\n\n\ttest := func(expectedSubjects []string, res jwt.StringList) {\n\t\tt.Helper()\n\t\trequire_True(t, len(res) == len(expectedSubjects))\n\t\tfor _, expetedSubj := range expectedSubjects {\n\t\t\trequire_True(t, res.Contains(expetedSubj))\n\t\t}\n\t}\n\n\ttest(resLim.Pub.Allow, []string{\"$JS.API.STREAM.INFO.KV_a\"})\n}\n\nfunc TestJWTTemplateGoodTagAfterBadTag(t *testing.T) {\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tukp, _ := nkeys.CreateUser()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"myname\"\n\tuclaim.Subject = upub\n\tuclaim.SetScoped(true)\n\tuclaim.IssuerAccount = aPub\n\tuclaim.Tags.Add(\"foo:foo1\")\n\n\tlim := jwt.UserPermissionLimits{}\n\tlim.Pub.Deny.Add(\"{{tag(NOT_THERE)}}.{{tag(foo)}}\")\n\tacc := &Account{nameTag: \"accname\", tags: []string{\"acc:acc1\", \"acc:acc2\"}}\n\n\t_, err := processUserPermissionsTemplate(lim, uclaim, acc)\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \"generated invalid subject\")\n}\n\nfunc TestJWTLimitsTemplate(t *testing.T) {\n\tkp, _ := nkeys.CreateAccount()\n\taPub, _ := kp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\taSignScopedKp, aSignScopedPub := createKey(t)\n\tsigner := jwt.NewUserScope()\n\tsigner.Key = aSignScopedPub\n\tsigner.Template.Pub.Deny.Add(\"denied\")\n\tsigner.Template.Pub.Allow.Add(\"foo.{{name()}}\")\n\tsigner.Template.Sub.Allow.Add(\"foo.{{name()}}\")\n\tclaim.SigningKeys.AddScopedSigner(signer)\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n `, ojwt, aPub, aJwt)))\n\tsA, _ := RunServerWithConfig(conf)\n\tdefer sA.Shutdown()\n\terrChan := make(chan struct{})\n\tdefer close(errChan)\n\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Name = \"myname\"\n\tuclaim.Subject = upub\n\tuclaim.SetScoped(true)\n\tuclaim.IssuerAccount = aPub\n\n\tujwt, err := uclaim.Encode(aSignScopedKp)\n\trequire_NoError(t, err)\n\tcreds := genCredsFile(t, ujwt, seed)\n\n\tt.Run(\"pass\", func(t *testing.T) {\n\t\tc := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds))\n\t\tdefer c.Close()\n\t\tsub, err := c.SubscribeSync(\"foo.myname\")\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, c.Flush())\n\t\trequire_NoError(t, c.Publish(\"foo.myname\", nil))\n\t\t_, err = sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t})\n\tt.Run(\"fail\", func(t *testing.T) {\n\t\tc := natsConnect(t, sA.ClientURL(), nats.UserCredentials(creds),\n\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\t\tif strings.Contains(err.Error(), `Permissions Violation for Publish to \"foo.othername\"`) {\n\t\t\t\t\terrChan <- struct{}{}\n\t\t\t\t}\n\t\t\t}))\n\t\tdefer c.Close()\n\t\trequire_NoError(t, c.Publish(\"foo.othername\", nil))\n\t\tselect {\n\t\tcase <-errChan:\n\t\tcase <-time.After(time.Second * 2):\n\t\t\trequire_True(t, false)\n\t\t}\n\t})\n}\n\nfunc TestJWTNoOperatorMode(t *testing.T) {\n\tfor _, login := range []bool{true, false} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\topts := DefaultOptions()\n\t\t\tif login {\n\t\t\t\topts.Users = append(opts.Users, &User{Username: \"u\", Password: \"pwd\"})\n\t\t\t}\n\t\t\tsA := RunServer(opts)\n\t\t\tdefer sA.Shutdown()\n\t\t\tkp, _ := nkeys.CreateAccount()\n\t\t\tcreds := createUserWithLimit(t, kp, time.Now().Add(time.Hour), nil)\n\t\t\turl := sA.ClientURL()\n\t\t\tif login {\n\t\t\t\turl = fmt.Sprintf(\"nats://u:pwd@%s:%d\", sA.opts.Host, sA.opts.Port)\n\t\t\t}\n\t\t\tc := natsConnect(t, url, nats.UserCredentials(creds))\n\t\t\tdefer c.Close()\n\t\t\tsA.mu.Lock()\n\t\t\tdefer sA.mu.Unlock()\n\t\t\tif len(sA.clients) != 1 {\n\t\t\t\tt.Fatalf(\"Expected exactly one client\")\n\t\t\t}\n\t\t\tfor _, v := range sA.clients {\n\t\t\t\tif v.opts.JWT != \"\" {\n\t\t\t\t\tt.Fatalf(\"Expected no jwt %v\", v.opts.JWT)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJWTUserRevocation(t *testing.T) {\n\ttest := func(all bool) {\n\t\tcreateAccountAndUser := func(done chan struct{}, pubKey, jwt1, jwt2, creds1, creds2 *string) {\n\t\t\tt.Helper()\n\t\t\tkp, _ := nkeys.CreateAccount()\n\t\t\t*pubKey, _ = kp.PublicKey()\n\t\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\t\tvar err error\n\t\t\t*jwt1, err = claim.Encode(oKp)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tukp, _ := nkeys.CreateUser()\n\t\t\tseed, _ := ukp.Seed()\n\t\t\tupub, _ := ukp.PublicKey()\n\t\t\tuclaim := newJWTTestUserClaims()\n\t\t\tuclaim.Subject = upub\n\n\t\t\tujwt1, err := uclaim.Encode(kp)\n\t\t\trequire_NoError(t, err)\n\t\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\n\t\t\t// create updated claim need to assure that issue time differs\n\t\t\tif all {\n\t\t\t\tclaim.Revoke(jwt.All) // revokes all jwt from now on\n\t\t\t} else {\n\t\t\t\tclaim.Revoke(upub) // revokes this jwt from now on\n\t\t\t}\n\t\t\ttime.Sleep(time.Millisecond * 1100)\n\t\t\t*jwt2, err = claim.Encode(oKp)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tujwt2, err := uclaim.Encode(kp)\n\t\t\trequire_NoError(t, err)\n\t\t\t*creds2 = genCredsFile(t, ujwt2, seed)\n\n\t\t\tdone <- struct{}{}\n\t\t}\n\t\t// Create Accounts and corresponding revoked and non revoked user creds. Do so concurrently to speed up the test\n\t\tdoneChan := make(chan struct{}, 2)\n\t\tdefer close(doneChan)\n\t\tvar syspub, sysjwt, dummy1, sysCreds, dummyCreds string\n\t\tgo createAccountAndUser(doneChan, &syspub, &sysjwt, &dummy1, &sysCreds, &dummyCreds)\n\t\tvar apub, ajwt1, ajwt2, aCreds1, aCreds2 string\n\t\tgo createAccountAndUser(doneChan, &apub, &ajwt1, &ajwt2, &aCreds1, &aCreds2)\n\t\tfor i := 0; i < cap(doneChan); i++ {\n\t\t\t<-doneChan\n\t\t}\n\t\tdirSrv := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, syspub, dirSrv)))\n\t\tsrv, _ := RunServerWithConfig(conf)\n\t\tdefer srv.Shutdown()\n\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt\n\t\tupdateJwt(t, srv.ClientURL(), sysCreds, ajwt1, 1) // set account jwt without revocation\n\t\tncSys := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds), nats.Name(\"conn name\"))\n\t\tdefer ncSys.Close()\n\t\tncChan := make(chan *nats.Msg, 10)\n\t\tdefer close(ncChan)\n\t\tsub, _ := ncSys.ChanSubscribe(fmt.Sprintf(disconnectEventSubj, apub), ncChan) // observe disconnect message\n\t\tdefer sub.Unsubscribe()\n\t\t// use credentials that will be revoked ans assure that the connection will be disconnected\n\t\tnc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds1),\n\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\t\tif err != nil && strings.Contains(err.Error(), \"authentication revoked\") {\n\t\t\t\t\tdoneChan <- struct{}{}\n\t\t\t\t}\n\t\t\t}),\n\t\t)\n\t\tdefer nc.Close()\n\t\t// update account jwt to contain revocation\n\t\tif updateJwt(t, srv.ClientURL(), sysCreds, ajwt2, 1) != 1 {\n\t\t\tt.Fatalf(\"Expected jwt update to pass\")\n\t\t}\n\t\t// assure that nc got disconnected due to the revocation\n\t\tselect {\n\t\tcase <-doneChan:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"Expected connection to have failed\")\n\t\t}\n\t\tm := <-ncChan\n\t\trequire_Len(t, strings.Count(string(m.Data), apub), 3)\n\t\trequire_True(t, strings.Contains(string(m.Data), `\"jwt\":\"eyJ0`))\n\t\t// try again with old credentials. Expected to fail\n\t\tif nc1, err := nats.Connect(srv.ClientURL(), nats.UserCredentials(aCreds1)); err == nil {\n\t\t\tnc1.Close()\n\t\t\tt.Fatalf(\"Expected revoked credentials to fail\")\n\t\t}\n\t\t// Assure new creds pass\n\t\tnc2 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds2))\n\t\tdefer nc2.Close()\n\t}\n\tt.Run(\"specific-key\", func(t *testing.T) {\n\t\ttest(false)\n\t})\n\tt.Run(\"all-key\", func(t *testing.T) {\n\t\ttest(true)\n\t})\n}\n\nfunc TestJWTActivationRevocation(t *testing.T) {\n\ttest := func(all bool) {\n\t\tsysKp, syspub := createKey(t)\n\t\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\t\tsysCreds := newUser(t, sysKp)\n\n\t\taExpKp, aExpPub := createKey(t)\n\t\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\t\taExpClaim.Name = \"Export\"\n\t\taExpClaim.Exports.Add(&jwt.Export{\n\t\t\tSubject: \"foo\",\n\t\t\tType: jwt.Stream,\n\t\t\tTokenReq: true,\n\t\t})\n\t\taExp1Jwt := encodeClaim(t, aExpClaim, aExpPub)\n\t\taExpCreds := newUser(t, aExpKp)\n\n\t\taImpKp, aImpPub := createKey(t)\n\n\t\tac := &jwt.ActivationClaims{}\n\t\tac.Subject = aImpPub\n\t\tac.ImportSubject = \"foo\"\n\t\tac.ImportType = jwt.Stream\n\t\ttoken, err := ac.Encode(aExpKp)\n\t\trequire_NoError(t, err)\n\n\t\trevPubKey := aImpPub\n\t\tif all {\n\t\t\trevPubKey = jwt.All\n\t\t}\n\n\t\taExpClaim.Exports[0].RevokeAt(revPubKey, time.Now())\n\t\taExp2Jwt := encodeClaim(t, aExpClaim, aExpPub)\n\n\t\taExpClaim.Exports[0].ClearRevocation(revPubKey)\n\t\taExp3Jwt := encodeClaim(t, aExpClaim, aExpPub)\n\n\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\taImpClaim.Name = \"Import\"\n\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\tSubject: \"foo\",\n\t\t\tType: jwt.Stream,\n\t\t\tAccount: aExpPub,\n\t\t\tToken: token,\n\t\t})\n\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\taImpCreds := newUser(t, aImpKp)\n\n\t\tdirSrv := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, syspub, dirSrv)))\n\n\t\tt.Run(\"token-expired-on-connect\", func(t *testing.T) {\n\t\t\tsrv, _ := RunServerWithConfig(conf)\n\t\t\tdefer srv.Shutdown()\n\t\t\tdefer removeDir(t, dirSrv) // clean jwt directory\n\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aExp2Jwt, 1) // set account jwt without revocation\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1)\n\n\t\t\tncExp1 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds))\n\t\t\tdefer ncExp1.Close()\n\n\t\t\tncImp := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds))\n\t\t\tdefer ncImp.Close()\n\n\t\t\tsub, err := ncImp.SubscribeSync(\"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, ncImp.Flush())\n\t\t\trequire_NoError(t, ncExp1.Publish(\"foo\", []byte(\"1\")))\n\t\t\t_, err = sub.NextMsg(time.Second)\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Equal(t, err.Error(), \"nats: timeout\")\n\t\t})\n\n\t\tt.Run(\"token-expired-on-update\", func(t *testing.T) {\n\t\t\tsrv, _ := RunServerWithConfig(conf)\n\t\t\tdefer srv.Shutdown()\n\t\t\tdefer removeDir(t, dirSrv) // clean jwt directory\n\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aExp1Jwt, 1) // set account jwt without revocation\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1)\n\n\t\t\tncExp1 := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds))\n\t\t\tdefer ncExp1.Close()\n\n\t\t\tncImp := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds))\n\t\t\tdefer ncImp.Close()\n\n\t\t\tsub, err := ncImp.SubscribeSync(\"foo\")\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_NoError(t, ncImp.Flush())\n\t\t\trequire_NoError(t, ncExp1.Publish(\"foo\", []byte(\"1\")))\n\t\t\tm1, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, string(m1.Data), \"1\")\n\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aExp2Jwt, 1) // set account jwt with revocation\n\n\t\t\trequire_NoError(t, ncExp1.Publish(\"foo\", []byte(\"2\")))\n\t\t\t_, err = sub.NextMsg(time.Second)\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Equal(t, err.Error(), \"nats: timeout\")\n\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aExp3Jwt, 1) // set account with revocation cleared\n\n\t\t\trequire_NoError(t, ncExp1.Publish(\"foo\", []byte(\"3\")))\n\t\t\tm2, err := sub.NextMsg(time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, string(m2.Data), \"3\")\n\t\t})\n\t}\n\tt.Run(\"specific-key\", func(t *testing.T) {\n\t\ttest(false)\n\t})\n\tt.Run(\"all-key\", func(t *testing.T) {\n\t\ttest(true)\n\t})\n}\n\nfunc TestJWTAccountFetchTimeout(t *testing.T) {\n\tcreateAccountAndUser := func(pubKey, jwt1, creds1 *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tujwt1, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\t}\n\tfor _, cfg := range []string{\n\t\t`type: full`,\n\t\t`type: cache`,\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tvar syspub, sysjwt, sysCreds string\n\t\t\tcreateAccountAndUser(&syspub, &sysjwt, &sysCreds)\n\t\t\tvar apub, ajwt1, aCreds1 string\n\t\t\tcreateAccountAndUser(&apub, &ajwt1, &aCreds1)\n\t\t\tdirSrv := t.TempDir()\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\t%s\n\t\t\ttimeout: \"100ms\"\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, syspub, cfg, dirSrv)))\n\t\t\tsrv, _ := RunServerWithConfig(conf)\n\t\t\tdefer srv.Shutdown()\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt\n\t\t\tstart := time.Now()\n\t\t\tnc, err := nats.Connect(srv.ClientURL(), nats.UserCredentials(aCreds1))\n\t\t\tif err == nil {\n\t\t\t\tt.Fatal(\"expected an error, got none\")\n\t\t\t} else if !strings.Contains(err.Error(), \"Authorization Violation\") {\n\t\t\t\tt.Fatalf(\"expected an authorization violation, got: %v\", err)\n\t\t\t}\n\t\t\tif time.Since(start) > 300*time.Millisecond {\n\t\t\t\tt.Fatal(\"expected timeout earlier\")\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\nfunc TestJWTAccountOps(t *testing.T) {\n\top, _ := nkeys.CreateOperator()\n\topPk, _ := op.PublicKey()\n\tsk, _ := nkeys.CreateOperator()\n\tskPk, _ := sk.PublicKey()\n\topClaim := jwt.NewOperatorClaims(opPk)\n\topClaim.SigningKeys.Add(skPk)\n\topJwt, err := opClaim.Encode(op)\n\trequire_NoError(t, err)\n\tcreateAccountAndUser := func(pubKey, jwt1, creds1 *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(sk)\n\t\trequire_NoError(t, err)\n\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\n\t\tujwt1, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds1 = genCredsFile(t, ujwt1, seed)\n\t}\n\tgenerateRequest := func(accs []string, kp nkeys.KeyPair) []byte {\n\t\tt.Helper()\n\t\topk, _ := kp.PublicKey()\n\t\tc := jwt.NewGenericClaims(opk)\n\t\tc.Data[\"accounts\"] = accs\n\t\tcJwt, err := c.Encode(kp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected no error %v\", err)\n\t\t}\n\t\treturn []byte(cJwt)\n\t}\n\tfor _, cfg := range []string{\n\t\t`type: full\n \t\tallow_delete: true`,\n\t\t`type: cache`,\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tvar syspub, sysjwt, sysCreds string\n\t\t\tcreateAccountAndUser(&syspub, &sysjwt, &sysCreds)\n\t\t\tvar apub, ajwt1, aCreds1 string\n\t\t\tcreateAccountAndUser(&apub, &ajwt1, &aCreds1)\n\t\t\tdirSrv := t.TempDir()\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\t%s\n\t\t\tdir: '%s'\n\t\t}\n `, opJwt, syspub, cfg, dirSrv)))\n\t\t\tdisconnectErrChan := make(chan struct{}, 1)\n\t\t\tdefer close(disconnectErrChan)\n\t\t\tsrv, _ := RunServerWithConfig(conf)\n\t\t\tdefer srv.Shutdown()\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysjwt, 1) // update system account jwt\n\t\t\t// push jwt (for full resolver)\n\t\t\tupdateJwt(t, srv.ClientURL(), sysCreds, ajwt1, 1) // set jwt\n\t\t\tnc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds))\n\t\t\tdefer nc.Close()\n\t\t\t// simulate nas resolver in case of a lookup request (cache)\n\t\t\tnc.Subscribe(fmt.Sprintf(accLookupReqSubj, apub), func(msg *nats.Msg) {\n\t\t\t\tmsg.Respond([]byte(ajwt1))\n\t\t\t})\n\t\t\t// connect so there is a reason to cache the request and so disconnect can be observed\n\t\t\tncA := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds1), nats.NoReconnect(),\n\t\t\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\t\t\tif err != nil && strings.Contains(err.Error(), \"account authentication expired\") {\n\t\t\t\t\t\tdisconnectErrChan <- struct{}{}\n\t\t\t\t\t}\n\t\t\t\t}))\n\t\t\tdefer ncA.Close()\n\t\t\tresp, err := nc.Request(accListReqSubj, nil, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, strings.Contains(string(resp.Data), apub))\n\t\t\trequire_True(t, strings.Contains(string(resp.Data), syspub))\n\t\t\t// delete nothing\n\t\t\tresp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{}, op), time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, strings.Contains(string(resp.Data), `\"message\":\"deleted 0 accounts\"`))\n\t\t\t// issue delete, twice to also delete a non existing account\n\t\t\t// also switch which key used to sign the request\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tresp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{apub}, sk), time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, strings.Contains(string(resp.Data), `\"message\":\"deleted 1 accounts\"`))\n\t\t\t\tresp, err = nc.Request(accListReqSubj, nil, time.Second)\n\t\t\t\trequire_False(t, strings.Contains(string(resp.Data), apub))\n\t\t\t\trequire_True(t, strings.Contains(string(resp.Data), syspub))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif i > 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tselect {\n\t\t\t\tcase <-disconnectErrChan:\n\t\t\t\tcase <-time.After(time.Second):\n\t\t\t\t\tt.Fatal(\"Callback not executed\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc createKey(t *testing.T) (nkeys.KeyPair, string) {\n\tt.Helper()\n\tkp, _ := nkeys.CreateAccount()\n\tsyspub, _ := kp.PublicKey()\n\treturn kp, syspub\n}\n\nfunc encodeClaim(t *testing.T, claim *jwt.AccountClaims, _ string) string {\n\tt.Helper()\n\ttheJWT, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\treturn theJWT\n}\n\n// returns user creds\nfunc newUserEx(t *testing.T, accKp nkeys.KeyPair, scoped bool, issuerAccount string) string {\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub\n\tuclaim.SetScoped(scoped)\n\tif issuerAccount != _EMPTY_ {\n\t\tuclaim.IssuerAccount = issuerAccount\n\t}\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn genCredsFile(t, ujwt, seed)\n}\n\n// returns user creds\nfunc newUser(t *testing.T, accKp nkeys.KeyPair) string {\n\treturn newUserEx(t, accKp, false, \"\")\n}\n\nfunc TestJWTHeader(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\ttest := func(share bool) {\n\t\taExpKp, aExpPub := createKey(t)\n\t\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\t\taExpClaim.Exports.Add(&jwt.Export{\n\t\t\tName: \"test\",\n\t\t\tSubject: \"srvc\",\n\t\t\tType: jwt.Service,\n\t\t\tTokenReq: false,\n\t\t\tLatency: &jwt.ServiceLatency{\n\t\t\t\tSampling: jwt.Headers,\n\t\t\t\tResults: \"res\",\n\t\t\t},\n\t\t})\n\t\taExpJwt := encodeClaim(t, aExpClaim, aExpPub)\n\t\taExpCreds := newUser(t, aExpKp)\n\n\t\taImpKp, aImpPub := createKey(t)\n\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\tName: \"test\",\n\t\t\tSubject: \"srvc\",\n\t\t\tAccount: aExpPub,\n\t\t\tType: jwt.Service,\n\t\t\tShare: share,\n\t\t})\n\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\taImpCreds := newUser(t, aImpKp)\n\n\t\tdirSrv := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, syspub, dirSrv)))\n\t\tsrv, _ := RunServerWithConfig(conf)\n\t\tdefer srv.Shutdown()\n\t\tupdateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt\n\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aExpJwt, 1)\n\t\tupdateJwt(t, srv.ClientURL(), sysCreds, aImpJwt, 1)\n\n\t\texpNc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aExpCreds))\n\t\tdefer expNc.Close()\n\t\tresChan := make(chan *nats.Msg, 1)\n\t\texpNc.ChanSubscribe(\"res\", resChan)\n\t\tsub, err := expNc.Subscribe(\"srvc\", func(msg *nats.Msg) {\n\t\t\tmsg.Respond(nil)\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer sub.Unsubscribe()\n\n\t\timpNc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aImpCreds))\n\t\tdefer impNc.Close()\n\t\t// send request w/o header\n\t\t_, err = impNc.Request(\"srvc\", []byte(\"msg1\"), time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, len(resChan) == 0)\n\n\t\t_, err = impNc.RequestMsg(&nats.Msg{\n\t\t\tSubject: \"srvc\", Data: []byte(\"msg2\"), Header: nats.Header{\n\t\t\t\t\"X-B3-Sampled\": []string{\"1\"},\n\t\t\t\t\"Share\": []string{\"Me\"}}}, time.Second)\n\t\trequire_NoError(t, err)\n\t\tselect {\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"should have received a response\")\n\t\tcase m := <-resChan:\n\t\t\tobj := map[string]any{}\n\t\t\terr = json.Unmarshal(m.Data, &obj)\n\t\t\trequire_NoError(t, err)\n\t\t\t// test if shared is honored\n\t\t\treqInfo := obj[\"requestor\"].(map[string]any)\n\t\t\t// fields always set\n\t\t\trequire_True(t, reqInfo[\"acc\"] != nil)\n\t\t\trequire_True(t, reqInfo[\"rtt\"] != nil)\n\n\t\t\t// fields only set when shared\n\t\t\t_, ok1 := reqInfo[\"lang\"]\n\t\t\t_, ok2 := reqInfo[\"ver\"]\n\t\t\t_, ok3 := reqInfo[\"host\"]\n\t\t\t_, ok4 := reqInfo[\"start\"]\n\t\t\tif !share {\n\t\t\t\tok1 = !ok1\n\t\t\t\tok2 = !ok2\n\t\t\t\tok3 = !ok3\n\t\t\t\tok4 = !ok4\n\t\t\t}\n\t\t\trequire_True(t, ok1)\n\t\t\trequire_True(t, ok2)\n\t\t\trequire_True(t, ok3)\n\t\t\trequire_True(t, ok4)\n\n\t\t}\n\t\trequire_True(t, len(resChan) == 0)\n\t}\n\ttest(true)\n\ttest(false)\n}\n\nfunc TestJWTAccountImportsWithWildcardSupport(t *testing.T) {\n\ttest := func(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds string, jsEnabled bool, exSubExpect, exPub, imReq, imSubExpect string) {\n\t\tt.Helper()\n\n\t\tvar jsSetting string\n\t\tif jsEnabled {\n\t\t\tjsSetting = \"jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb}\"\n\t\t}\n\n\t\t_, aSysPub := createKey(t)\n\t\taSysClaim := jwt.NewAccountClaims(aSysPub)\n\t\taSysJwt := encodeClaim(t, aSysClaim, aSysPub)\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t}\n\t\tsystem_account: %s\n\t\t%s\n\t\t`, ojwt, aExpPub, aExpJwt, aImpPub, aImpJwt, aSysPub, aSysJwt, aSysPub, jsSetting)))\n\n\t\ts, opts := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\n\t\tncExp := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.UserCredentials(aExpCreds))\n\t\tdefer ncExp.Close()\n\n\t\tncImp := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.UserCredentials(aImpCreds))\n\t\tdefer ncImp.Close()\n\n\t\t// Create subscriber for the service endpoint in foo.\n\t\t_, err := ncExp.Subscribe(exSubExpect, func(m *nats.Msg) {\n\t\t\tm.Respond([]byte(\"yes!\"))\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tncExp.Flush()\n\n\t\t// Now test service import.\n\t\tif resp, err := ncImp.Request(imReq, []byte(\"yes?\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected a response to request %s got: %v\", imReq, err)\n\t\t} else if string(resp.Data) != \"yes!\" {\n\t\t\tt.Fatalf(\"Expected a response of %q, got %q\", \"yes!\", resp.Data)\n\t\t}\n\t\tsubBar, err := ncImp.SubscribeSync(imSubExpect)\n\t\trequire_NoError(t, err)\n\t\tncImp.Flush()\n\n\t\tncExp.Publish(exPub, []byte(\"event!\"))\n\n\t\tif m, err := subBar.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected a stream message got %v\", err)\n\t\t} else if string(m.Data) != \"event!\" {\n\t\t\tt.Fatalf(\"Expected a response of %q, got %q\", \"event!\", m.Data)\n\t\t}\n\t}\n\tcreateExporter := func() (string, string, string) {\n\t\tt.Helper()\n\t\taExpKp, aExpPub := createKey(t)\n\t\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\t\taExpClaim.Name = \"Export\"\n\t\taExpClaim.Exports.Add(&jwt.Export{\n\t\t\tSubject: \"$request.*.$in.*.>\",\n\t\t\tType: jwt.Service,\n\t\t}, &jwt.Export{\n\t\t\tSubject: \"$events.*.$in.*.>\",\n\t\t\tType: jwt.Stream,\n\t\t})\n\t\taExpJwt := encodeClaim(t, aExpClaim, aExpPub)\n\t\taExpCreds := newUser(t, aExpKp)\n\t\treturn aExpPub, aExpJwt, aExpCreds\n\t}\n\tt.Run(\"To\", func(t *testing.T) {\n\t\taExpPub, aExpJwt, aExpCreds := createExporter()\n\t\taImpKp, aImpPub := createKey(t)\n\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\taImpClaim.Name = \"Import\"\n\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\tSubject: \"my.request.*.*.>\",\n\t\t\tType: jwt.Service,\n\t\t\tTo: \"$request.*.$in.*.>\", // services have local and remote switched between Subject and To\n\t\t\tAccount: aExpPub,\n\t\t}, &jwt.Import{\n\t\t\tSubject: \"$events.*.$in.*.>\",\n\t\t\tType: jwt.Stream,\n\t\t\tTo: \"prefix\",\n\t\t\tAccount: aExpPub,\n\t\t})\n\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\taImpCreds := newUser(t, aImpKp)\n\t\ttest(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, false,\n\t\t\t\"$request.1.$in.2.bar\", \"$events.1.$in.2.bar\",\n\t\t\t\"my.request.1.2.bar\", \"prefix.$events.1.$in.2.bar\")\n\t})\n\tt.Run(\"LocalSubject-No-Reorder\", func(t *testing.T) {\n\t\taExpPub, aExpJwt, aExpCreds := createExporter()\n\t\taImpKp, aImpPub := createKey(t)\n\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\taImpClaim.Name = \"Import\"\n\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\tSubject: \"$request.*.$in.*.>\",\n\t\t\tType: jwt.Service,\n\t\t\tLocalSubject: \"my.request.*.*.>\",\n\t\t\tAccount: aExpPub,\n\t\t}, &jwt.Import{\n\t\t\tSubject: \"$events.*.$in.*.>\",\n\t\t\tType: jwt.Stream,\n\t\t\tLocalSubject: \"my.events.*.*.>\",\n\t\t\tAccount: aExpPub,\n\t\t})\n\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\taImpCreds := newUser(t, aImpKp)\n\t\ttest(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, false,\n\t\t\t\"$request.1.$in.2.bar\", \"$events.1.$in.2.bar\",\n\t\t\t\"my.request.1.2.bar\", \"my.events.1.2.bar\")\n\t})\n\tt.Run(\"LocalSubject-Reorder\", func(t *testing.T) {\n\t\tfor _, jsEnabled := range []bool{false, true} {\n\t\t\tt.Run(fmt.Sprintf(\"%t\", jsEnabled), func(t *testing.T) {\n\t\t\t\taExpPub, aExpJwt, aExpCreds := createExporter()\n\t\t\t\taImpKp, aImpPub := createKey(t)\n\t\t\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\t\t\taImpClaim.Name = \"Import\"\n\t\t\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\t\t\tSubject: \"$request.*.$in.*.>\",\n\t\t\t\t\tType: jwt.Service,\n\t\t\t\t\tLocalSubject: \"my.request.$2.$1.>\",\n\t\t\t\t\tAccount: aExpPub,\n\t\t\t\t}, &jwt.Import{\n\t\t\t\t\tSubject: \"$events.*.$in.*.>\",\n\t\t\t\t\tType: jwt.Stream,\n\t\t\t\t\tLocalSubject: \"my.events.$2.$1.>\",\n\t\t\t\t\tAccount: aExpPub,\n\t\t\t\t})\n\t\t\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\t\t\taImpCreds := newUser(t, aImpKp)\n\t\t\t\ttest(aExpPub, aExpJwt, aExpCreds, aImpPub, aImpJwt, aImpCreds, jsEnabled,\n\t\t\t\t\t\"$request.2.$in.1.bar\", \"$events.1.$in.2.bar\",\n\t\t\t\t\t\"my.request.1.2.bar\", \"my.events.2.1.bar\")\n\t\t\t})\n\t\t}\n\t})\n}\n\nfunc TestJWTAccountTokenImportMisuse(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taExpKp, aExpPub := createKey(t)\n\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\taExpClaim.Name = \"Export\"\n\taExpClaim.Exports.Add(&jwt.Export{\n\t\tSubject: \"$events.*.$in.*.>\",\n\t\tType: jwt.Stream,\n\t\tTokenReq: true,\n\t}, &jwt.Export{\n\t\tSubject: \"foo\",\n\t\tType: jwt.Stream,\n\t\tTokenReq: true,\n\t})\n\taExpJwt := encodeClaim(t, aExpClaim, aExpPub)\n\n\tcreateImportingAccountClaim := func(aImpKp nkeys.KeyPair, aExpPub string, ac *jwt.ActivationClaims) (string, string) {\n\t\tt.Helper()\n\t\ttoken, err := ac.Encode(aExpKp)\n\t\trequire_NoError(t, err)\n\n\t\taImpPub, err := aImpKp.PublicKey()\n\t\trequire_NoError(t, err)\n\t\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\t\taImpClaim.Name = \"Import\"\n\t\taImpClaim.Imports.Add(&jwt.Import{\n\t\t\tSubject: \"$events.*.$in.*.>\",\n\t\t\tType: jwt.Stream,\n\t\t\tAccount: aExpPub,\n\t\t\tToken: token,\n\t\t})\n\t\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\t\taImpCreds := newUser(t, aImpKp)\n\t\treturn aImpJwt, aImpCreds\n\t}\n\n\ttestConnect := func(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds string) {\n\t\tt.Helper()\n\t\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tif r.URL.Path == \"/A/\" {\n\t\t\t\t// Server startup\n\t\t\t\tw.Write(nil)\n\t\t\t} else if r.URL.Path == \"/A/\"+aExpPub {\n\t\t\t\tw.Write([]byte(aExpJwt))\n\t\t\t} else if r.URL.Path == \"/A/\"+aImpPub {\n\t\t\t\tw.Write([]byte(aImpJwt))\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"not expected\")\n\t\t\t}\n\t\t}))\n\t\tdefer ts.Close()\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\toperator: %s\n\t\t\tresolver: URL(\"%s/A/\")\n\t\t`, ojwt, ts.URL)))\n\n\t\ts, opts := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\n\t\tncImp, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.UserCredentials(aImpCreds))\n\t\trequire_Error(t, err) // misuse needs to result in an error\n\t\tdefer ncImp.Close()\n\t}\n\n\ttestNatsResolver := func(aImpJwt string) {\n\t\tt.Helper()\n\t\tdirSrv := t.TempDir()\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\toperator: %s\n\t\t\tsystem_account: %s\n\t\t\tresolver: {\n\t\t\t\ttype: full\n\t\t\t\tdir: '%s'\n\t\t\t}\n\t\t`, ojwt, syspub, dirSrv)))\n\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\n\t\trequire_True(t, updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1) == 1)\n\t\trequire_True(t, updateJwt(t, s.ClientURL(), sysCreds, aExpJwt, 1) == 1)\n\t\trequire_True(t, updateJwt(t, s.ClientURL(), sysCreds, aImpJwt, 1) == 0) // assure this did not succeed\n\t}\n\n\tt.Run(\"wrong-account\", func(t *testing.T) {\n\t\taImpKp, aImpPub := createKey(t)\n\t\tac := &jwt.ActivationClaims{}\n\t\t_, ac.Subject = createKey(t) // on purpose issue this token for another account\n\t\tac.ImportSubject = \"$events.*.$in.*.>\"\n\t\tac.ImportType = jwt.Stream\n\n\t\taImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac)\n\t\ttestConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds)\n\t\ttestNatsResolver(aImpJwt)\n\t})\n\n\tt.Run(\"different-subject\", func(t *testing.T) {\n\t\taImpKp, aImpPub := createKey(t)\n\t\tac := &jwt.ActivationClaims{}\n\t\tac.Subject = aImpPub\n\t\tac.ImportSubject = \"foo\" // on purpose use a subject from another export\n\t\tac.ImportType = jwt.Stream\n\n\t\taImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac)\n\t\ttestConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds)\n\t\ttestNatsResolver(aImpJwt)\n\t})\n\n\tt.Run(\"non-existing-subject\", func(t *testing.T) {\n\t\taImpKp, aImpPub := createKey(t)\n\t\tac := &jwt.ActivationClaims{}\n\t\tac.Subject = aImpPub\n\t\tac.ImportSubject = \"does-not-exist-or-from-different-export\" // on purpose use a non exported subject\n\t\tac.ImportType = jwt.Stream\n\n\t\taImpJwt, aImpCreds := createImportingAccountClaim(aImpKp, aExpPub, ac)\n\t\ttestConnect(aExpPub, aExpJwt, aImpPub, aImpJwt, aImpCreds)\n\t\ttestNatsResolver(aImpJwt)\n\t})\n}\n\nfunc TestJWTResponseThreshold(t *testing.T) {\n\trespThresh := 20 * time.Millisecond\n\taExpKp, aExpPub := createKey(t)\n\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\taExpClaim.Name = \"Export\"\n\taExpClaim.Exports.Add(&jwt.Export{\n\t\tSubject: \"srvc\",\n\t\tType: jwt.Service,\n\t\tResponseThreshold: respThresh,\n\t})\n\taExpJwt := encodeClaim(t, aExpClaim, aExpPub)\n\taExpCreds := newUser(t, aExpKp)\n\n\taImpKp, aImpPub := createKey(t)\n\taImpClaim := jwt.NewAccountClaims(aImpPub)\n\taImpClaim.Name = \"Import\"\n\taImpClaim.Imports.Add(&jwt.Import{\n\t\tSubject: \"srvc\",\n\t\tType: jwt.Service,\n\t\tAccount: aExpPub,\n\t})\n\taImpJwt := encodeClaim(t, aImpClaim, aImpPub)\n\taImpCreds := newUser(t, aImpKp)\n\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t}\n\t\t`, ojwt, aExpPub, aExpJwt, aImpPub, aImpJwt)))\n\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tncExp := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.UserCredentials(aExpCreds))\n\tdefer ncExp.Close()\n\n\tncImp := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.UserCredentials(aImpCreds))\n\tdefer ncImp.Close()\n\n\tdelayChan := make(chan time.Duration, 1)\n\n\t// Create subscriber for the service endpoint in foo.\n\t_, err := ncExp.Subscribe(\"srvc\", func(m *nats.Msg) {\n\t\ttime.Sleep(<-delayChan)\n\t\tm.Respond([]byte(\"yes!\"))\n\t})\n\trequire_NoError(t, err)\n\tncExp.Flush()\n\n\tt.Run(\"No-Timeout\", func(t *testing.T) {\n\t\tdelayChan <- respThresh / 2\n\t\tif resp, err := ncImp.Request(\"srvc\", []byte(\"yes?\"), 4*respThresh); err != nil {\n\t\t\tt.Fatalf(\"Expected a response to request srvc got: %v\", err)\n\t\t} else if string(resp.Data) != \"yes!\" {\n\t\t\tt.Fatalf(\"Expected a response of %q, got %q\", \"yes!\", resp.Data)\n\t\t}\n\t})\n\tt.Run(\"Timeout\", func(t *testing.T) {\n\t\tdelayChan <- 2 * respThresh\n\t\tif _, err := ncImp.Request(\"srvc\", []byte(\"yes?\"), 4*respThresh); err == nil || err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Expected a timeout\")\n\t\t}\n\t})\n}\n\nfunc TestJWTJetStreamTiers(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, accPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, MemoryStorage: 0, Consumer: 2, Streams: 2}\n\taccJwt1 := encodeClaim(t, accClaim, accPub)\n\taccCreds := newUser(t, accKp)\n\n\tstart := time.Now()\n\n\tstoreDir := t.TempDir()\n\n\tdirSrv := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t`, storeDir, ojwt, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t// Test tiers up to stream limits\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-1\", Replicas: 1, Subjects: []string{\"testR1-1\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-2\", Replicas: 1, Subjects: []string{\"testR1-2\"}})\n\trequire_NoError(t, err)\n\n\t// Test exceeding tiered stream limit\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-3\", Replicas: 1, Subjects: []string{\"testR1-3\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum number of streams reached\")\n\n\t// Test tiers up to consumer limits\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur1\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur3\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\t// test exceeding tiered consumer limits\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur4\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum consumers limit reached\")\n\t_, err = js.AddConsumer(\"testR1-1\", &nats.ConsumerConfig{Durable: \"dur5\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum consumers limit reached\")\n\n\t// test tiered storage limit\n\tmsg := [512]byte{}\n\t_, err = js.Publish(\"testR1-1\", msg[:])\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"testR1-2\", msg[:])\n\trequire_NoError(t, err)\n\n\tainfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 1100)\n\n\t// test exceeding tiered storage limit\n\t_, err = js.Publish(\"testR1-1\", []byte(\"1\"))\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\n\t// Check that storage has not increased after the rejected publish.\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 1100)\n\n\ttime.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1650, MemoryStorage: 0, Consumer: 1, Streams: 3}\n\taccJwt2 := encodeClaim(t, accClaim, accPub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1)\n\n\t// test same sequence as before, add stream, fail add stream, add consumer, fail add consumer, publish, fail publish\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-3\", Replicas: 1, Subjects: []string{\"testR1-3\"}})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"testR1-4\", Replicas: 1, Subjects: []string{\"testR1-4\"}})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum number of streams reached\")\n\t_, err = js.AddConsumer(\"testR1-3\", &nats.ConsumerConfig{Durable: \"dur6\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"testR1-3\", &nats.ConsumerConfig{Durable: \"dur7\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: maximum consumers limit reached\")\n\n\t// At this point it will be exactly at the DiskStorage limit so it should not fail.\n\t_, err = js.Publish(\"testR1-3\", msg[:])\n\trequire_NoError(t, err)\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 1650)\n\n\t_, err = js.Publish(\"testR1-3\", []byte(\"1\"))\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\n\t// Should remain at the same usage.\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 1650)\n}\n\nfunc TestJWTJetStreamMaxAckPending(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, accPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit,\n\t\tConsumer: jwt.NoLimit, Streams: jwt.NoLimit, MaxAckPending: int64(1000),\n\t}\n\taccJwt1 := encodeClaim(t, accClaim, accPub)\n\taccCreds := newUser(t, accKp)\n\n\tstart := time.Now()\n\n\tstoreDir := t.TempDir()\n\n\tdirSrv := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t`, storeDir, ojwt, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tDurable: \"dur1\", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: consumer max ack pending exceeds system limit of 1000\")\n\n\tci, err := js.AddConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tDurable: \"dur2\", AckPolicy: nats.AckAllPolicy, MaxAckPending: 500})\n\trequire_NoError(t, err)\n\trequire_True(t, ci.Config.MaxAckPending == 500)\n\n\t_, err = js.UpdateConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tDurable: \"dur2\", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: consumer max ack pending exceeds system limit of 1000\")\n\n\ttime.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: jwt.NoLimit, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit,\n\t\tStreams: jwt.NoLimit, MaxAckPending: int64(2000)}\n\taccJwt2 := encodeClaim(t, accClaim, accPub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1)\n\n\tci, err = js.UpdateConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tDurable: \"dur2\", AckPolicy: nats.AckAllPolicy, MaxAckPending: 2000})\n\trequire_NoError(t, err)\n\trequire_True(t, ci.Config.MaxAckPending == 2000)\n}\n\nfunc TestJWTJetStreamMaxStore(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, accPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1024, MemoryStorage: jwt.NoLimit,\n\t\tConsumer: jwt.NoLimit, Streams: jwt.NoLimit,\n\t\tDiskMaxStreamBytes: 1024, MaxBytesRequired: false,\n\t}\n\taccJwt1 := encodeClaim(t, accClaim, accPub)\n\taccCreds := newUser(t, accKp)\n\n\tstart := time.Now()\n\n\tstoreDir := t.TempDir()\n\n\tdirSrv := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t`, storeDir, ojwt, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream()\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, MaxBytes: 2048})\n\trequire_Error(t, err, NewJSStorageResourcesExceededError())\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"foo\", Replicas: 1, MaxBytes: 1024})\n\trequire_NoError(t, err)\n\n\tmsg := [900]byte{}\n\t_, err = js.Publish(\"foo\", msg[:])\n\trequire_NoError(t, err)\n\t_, err = js.Publish(\"foo\", msg[:]) // exceeds storage limit\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\n\ttime.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 3072, MemoryStorage: jwt.NoLimit, Consumer: jwt.NoLimit, Streams: jwt.NoLimit,\n\t\tDiskMaxStreamBytes: 2048, MaxBytesRequired: true}\n\taccJwt2 := encodeClaim(t, accClaim, accPub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt2, 1)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1, MaxBytes: 3000})\n\trequire_Error(t, err, NewJSStorageResourcesExceededError())\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1, MaxBytes: 2048})\n\trequire_NoError(t, err)\n\n\tainfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 933)\n\n\t// This should be exactly at the limit of the account.\n\t_, err = js.Publish(\"foo\", []byte(strings.Repeat(\"A\", 991)))\n\trequire_NoError(t, err)\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 1024)\n\t_, err = js.Publish(\"bar\", []byte(strings.Repeat(\"A\", 2015)))\n\trequire_NoError(t, err)\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 3072)\n\n\t// Exceed storage limit.\n\t_, err = js.Publish(\"bar\", []byte(\"1\"))\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: resource limits exceeded for account\")\n\n\t// Confirm no changes after rejected publish.\n\tainfo, err = js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_Equal(t, ainfo.Tiers[\"R1\"].Store, 3072)\n\n\t// test disabling max bytes required\n\t_, err = js.UpdateStream(&nats.StreamConfig{Name: \"bar\", Replicas: 1})\n\trequire_Error(t, err)\n\trequire_Equal(t, err.Error(), \"nats: account requires a stream config to have max bytes set\")\n}\n\nfunc TestJWTQueuePermissions(t *testing.T) {\n\taExpKp, aExpPub := createKey(t)\n\taExpClaim := jwt.NewAccountClaims(aExpPub)\n\taExpJwt := encodeClaim(t, aExpClaim, aExpPub)\n\tnewUser := func(t *testing.T, permType string) string {\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tswitch permType {\n\t\tcase \"allow\":\n\t\t\tuclaim.Permissions.Sub.Allow.Add(\"foo.> *.dev\")\n\t\tcase \"deny\":\n\t\t\tuclaim.Permissions.Sub.Deny.Add(\"foo.> *.dev\")\n\t\t}\n\t\tujwt, err := uclaim.Encode(aExpKp)\n\t\trequire_NoError(t, err)\n\t\treturn genCredsFile(t, ujwt, seed)\n\t}\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver = MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t}`, ojwt, aExpPub, aExpJwt)))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\topts.NoLog, opts.NoSigs = true, true\n\terrChan := make(chan error, 1)\n\tdefer close(errChan)\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tpermType string\n\t\tqueue string\n\t\terrExpected bool\n\t}{\n\t\t{\"allow\", \"queue.dev\", false},\n\t\t{\"allow\", \"\", true},\n\t\t{\"allow\", \"bad\", true},\n\t\t{\"deny\", \"\", false},\n\t\t{\"deny\", \"queue.dev\", true},\n\t} {\n\t\tt.Run(test.permType+test.queue, func(t *testing.T) {\n\t\t\tusrCreds := newUser(t, test.permType)\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port),\n\t\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\t\terrChan <- err\n\t\t\t\t}),\n\t\t\t\tnats.UserCredentials(usrCreds))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"No error expected: %v\", err)\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tif test.queue == \"\" {\n\t\t\t\tif _, err := nc.Subscribe(\"foo.bar\", func(msg *nats.Msg) {}); err != nil {\n\t\t\t\t\tt.Fatalf(\"no error expected: %v\", err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif _, err := nc.QueueSubscribe(\"foo.bar\", test.queue, func(msg *nats.Msg) {}); err != nil {\n\t\t\t\t\tt.Fatalf(\"no error expected: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t\tselect {\n\t\t\tcase err := <-errChan:\n\t\t\t\tif !test.errExpected {\n\t\t\t\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(err.Error(), `Permissions Violation for Subscription to \"foo.bar\"`) {\n\t\t\t\t\tt.Fatalf(\"error %v\", err)\n\t\t\t\t}\n\t\t\tcase <-time.After(150 * time.Millisecond):\n\t\t\t\tif test.errExpected {\n\t\t\t\t\tt.Fatal(\"Expected an error\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\n\t}\n}\n\nfunc TestJWTScopedSigningKeys(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\t_, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.Name = \"acc\"\n\n\taSignNonScopedKp, aSignNonScopedPub := createKey(t)\n\taccClaim.SigningKeys.Add(aSignNonScopedPub)\n\n\taSignScopedKp, aSignScopedPub := createKey(t)\n\tsigner := jwt.NewUserScope()\n\tsigner.Key = aSignScopedPub\n\tsigner.Template.Pub.Deny.Add(\"denied\")\n\tsigner.Template.Payload = 5\n\tsigner.Template.AllowedConnectionTypes.Add(jwt.ConnectionTypeStandard)\n\taccClaim.SigningKeys.AddScopedSigner(signer)\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\n\taNonScopedCreds := newUserEx(t, aSignNonScopedKp, false, aExpPub)\n\taBadScopedCreds := newUserEx(t, aSignScopedKp, false, aExpPub)\n\taScopedCreds := newUserEx(t, aSignScopedKp, true, aExpPub)\n\n\tdirSrv := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_tls: true\n\t\t}\n `, ojwt, syspub, dirSrv)))\n\ts, opts := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\twsUrl := fmt.Sprintf(\"ws://%s:%d\", opts.Websocket.Host, opts.Websocket.Port)\n\terrChan := make(chan error, 1)\n\tdefer close(errChan)\n\tawaitError := func(expected bool) {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase err := <-errChan:\n\t\t\tif !expected {\n\t\t\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t\t\t}\n\t\tcase <-time.After(150 * time.Millisecond):\n\t\t\tif expected {\n\t\t\t\tt.Fatal(\"Expected an error\")\n\t\t\t}\n\t\t}\n\t}\n\terrHdlr := nats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\terrChan <- err\n\t})\n\tif updateJwt(t, url, sysCreds, sysJwt, 1) != 1 {\n\t\tt.Error(\"Expected update to pass\")\n\t} else if updateJwt(t, url, sysCreds, accJwt, 1) != 1 {\n\t\tt.Error(\"Expected update to pass\")\n\t}\n\tt.Run(\"bad-scoped-signing-key\", func(t *testing.T) {\n\t\t_, err := nats.Connect(url, nats.UserCredentials(aBadScopedCreds))\n\t\trequire_Error(t, err)\n\t})\n\tt.Run(\"regular-signing-key\", func(t *testing.T) {\n\t\tnc := natsConnect(t, url, nats.UserCredentials(aNonScopedCreds), errHdlr)\n\t\tdefer nc.Close()\n\t\tnc.Flush()\n\t\terr := nc.Publish(\"denied\", nil)\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\tawaitError(false)\n\t})\n\tt.Run(\"scoped-signing-key-client-side\", func(t *testing.T) {\n\t\tnc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr)\n\t\tdefer nc.Close()\n\t\tnc.Flush()\n\t\terr := nc.Publish(\"too-long\", []byte(\"way.too.long.for.payload.limit\"))\n\t\trequire_Error(t, err)\n\t\trequire_True(t, strings.Contains(err.Error(), ErrMaxPayload.Error()))\n\t})\n\tt.Run(\"scoped-signing-key-server-side\", func(t *testing.T) {\n\t\tnc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr)\n\t\tdefer nc.Close()\n\t\tnc.Flush()\n\t\terr := nc.Publish(\"denied\", nil)\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\tawaitError(true)\n\t})\n\tt.Run(\"scoped-signing-key-allowed-conn-types\", func(t *testing.T) {\n\t\t_, err := nats.Connect(wsUrl, nats.UserCredentials(aScopedCreds))\n\t\trequire_Error(t, err)\n\t})\n\tt.Run(\"scoped-signing-key-reload\", func(t *testing.T) {\n\t\treconChan := make(chan struct{}, 1)\n\t\tdefer close(reconChan)\n\t\tmsgChan := make(chan *nats.Msg, 2)\n\t\tdefer close(msgChan)\n\t\tnc := natsConnect(t, url, nats.UserCredentials(aScopedCreds), errHdlr,\n\t\t\tnats.DisconnectErrHandler(func(conn *nats.Conn, err error) {\n\t\t\t\tif err != nil {\n\t\t\t\t\terrChan <- err\n\t\t\t\t}\n\t\t\t}),\n\t\t\tnats.ReconnectHandler(func(conn *nats.Conn) {\n\t\t\t\treconChan <- struct{}{}\n\t\t\t}),\n\t\t\tnats.ReconnectWait(100*time.Millisecond),\n\t\t)\n\t\tdefer nc.Close()\n\t\t_, err := nc.ChanSubscribe(\"denied\", msgChan)\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\terr = nc.Publish(\"denied\", nil)\n\t\trequire_NoError(t, err)\n\t\tawaitError(true)\n\t\trequire_Len(t, len(msgChan), 0)\n\t\t// Alter scoped permissions and update\n\t\tsigner.Template.Payload = -1\n\t\tsigner.Template.Pub.Deny.Remove(\"denied\")\n\t\tsigner.Template.AllowedConnectionTypes.Add(jwt.ConnectionTypeWebsocket)\n\t\taccClaim.SigningKeys.AddScopedSigner(signer)\n\t\taccUpdatedJwt := encodeClaim(t, accClaim, aExpPub)\n\t\tif updateJwt(t, url, sysCreds, accUpdatedJwt, 1) != 1 {\n\t\t\tt.Error(\"Expected update to pass\")\n\t\t}\n\t\t// disconnect triggered by update\n\t\tawaitError(true)\n\t\t<-reconChan\n\t\tnc.Flush()\n\t\terr = nc.Publish(\"denied\", []byte(\"way.too.long.for.old.payload.limit\"))\n\t\trequire_NoError(t, err)\n\t\tawaitError(false)\n\t\tmsg := <-msgChan\n\t\trequire_Equal(t, string(msg.Data), \"way.too.long.for.old.payload.limit\")\n\t\trequire_Len(t, len(msgChan), 0)\n\t\tnc.Close()\n\t\tnc, err = nats.Connect(wsUrl, nats.UserCredentials(aScopedCreds))\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\tdefer nc.Close()\n\t})\n\trequire_Len(t, len(errChan), 0)\n}\n\nfunc TestJWTStrictSigningKeys(t *testing.T) {\n\tnewAccount := func(opKp nkeys.KeyPair) (nkeys.KeyPair, nkeys.KeyPair, string, *jwt.AccountClaims, string) {\n\t\taccId, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\taccIdPub, err := accId.PublicKey()\n\t\trequire_NoError(t, err)\n\n\t\taccSig, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\taccSigPub, err := accSig.PublicKey()\n\t\trequire_NoError(t, err)\n\n\t\taClaim := jwt.NewAccountClaims(accIdPub)\n\t\taClaim.SigningKeys.Add(accSigPub)\n\t\ttheJwt, err := aClaim.Encode(opKp)\n\t\trequire_NoError(t, err)\n\t\treturn accId, accSig, accIdPub, aClaim, theJwt\n\t}\n\n\topId, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\topIdPub, err := opId.PublicKey()\n\trequire_NoError(t, err)\n\n\topSig, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\topSigPub, err := opSig.PublicKey()\n\trequire_NoError(t, err)\n\n\taBadBadKp, aBadGoodKp, aBadPub, _, aBadJwt := newAccount(opId)\n\taGoodBadKp, aGoodGoodKp, aGoodPub, _, aGoodJwt := newAccount(opSig)\n\t_, aSysKp, aSysPub, _, aSysJwt := newAccount(opSig)\n\n\toClaim := jwt.NewOperatorClaims(opIdPub)\n\toClaim.StrictSigningKeyUsage = true\n\toClaim.SigningKeys.Add(opSigPub)\n\toClaim.SystemAccount = aSysPub\n\toJwt, err := oClaim.Encode(opId)\n\trequire_NoError(t, err)\n\n\tuBadBadCreds := newUserEx(t, aBadBadKp, false, aBadPub)\n\tuBadGoodCreds := newUserEx(t, aBadGoodKp, false, aBadPub)\n\tuGoodBadCreds := newUserEx(t, aGoodBadKp, false, aGoodPub)\n\tuGoodGoodCreds := newUserEx(t, aGoodGoodKp, false, aGoodPub)\n\tuSysCreds := newUserEx(t, aSysKp, false, aSysPub)\n\n\tconnectTest := func(url string) {\n\t\tfor _, test := range []struct {\n\t\t\tcreds string\n\t\t\tfail bool\n\t\t}{\n\t\t\t{uBadBadCreds, true},\n\t\t\t{uBadGoodCreds, true},\n\t\t\t{uGoodBadCreds, true},\n\t\t\t{uGoodGoodCreds, false},\n\t\t} {\n\t\t\tnc, err := nats.Connect(url, nats.UserCredentials(test.creds))\n\t\t\tnc.Close()\n\t\t\tif test.fail {\n\t\t\t\trequire_Error(t, err)\n\t\t\t} else {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tt.Run(\"resolver\", func(t *testing.T) {\n\t\tdirSrv := t.TempDir()\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t}\n\t\t`, oJwt, dirSrv, aSysPub, aSysJwt)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\t\turl := s.ClientURL()\n\t\tif updateJwt(t, url, uSysCreds, aBadJwt, 1) != 0 {\n\t\t\tt.Fatal(\"Expected negative response\")\n\t\t}\n\t\tif updateJwt(t, url, uSysCreds, aGoodJwt, 1) != 1 {\n\t\t\tt.Fatal(\"Expected positive response\")\n\t\t}\n\t\tconnectTest(url)\n\t})\n\n\tt.Run(\"mem-resolver\", func(t *testing.T) {\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver: MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t}\n\t\t`, oJwt, aSysPub, aSysJwt, aBadPub, aBadJwt, aGoodPub, aGoodJwt)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\t\tconnectTest(s.ClientURL())\n\t})\n}\n\nfunc TestJWTAccountProtectedImport(t *testing.T) {\n\tsrvFmt := `\n\t\tport: -1\n\t\toperator = %s\n\t\tresolver: MEMORY\n\t\tresolver_preload = {\n\t\t\t%s : \"%s\"\n\t\t\t%s : \"%s\"\n\t\t} `\n\tsetupAccounts := func(pass bool) (nkeys.KeyPair, string, string, string, nkeys.KeyPair, string, string, string, string) {\n\t\t// Create accounts and imports/exports.\n\t\texportKP, _ := nkeys.CreateAccount()\n\t\texportPub, _ := exportKP.PublicKey()\n\t\texportAC := jwt.NewAccountClaims(exportPub)\n\t\texportAC.Exports.Add(&jwt.Export{Subject: \"service.*\", Type: jwt.Service, AccountTokenPosition: 2})\n\t\texportAC.Exports.Add(&jwt.Export{Subject: \"stream.*\", Type: jwt.Stream, AccountTokenPosition: 2})\n\t\texportJWT, err := exportAC.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\t// create alternative exporter jwt without account token pos set\n\t\texportAC.Exports = jwt.Exports{}\n\t\texportAC.Exports.Add(&jwt.Export{Subject: \"service.*\", Type: jwt.Service})\n\t\texportAC.Exports.Add(&jwt.Export{Subject: \"stream.*\", Type: jwt.Stream})\n\t\texportJWTNoPos, err := exportAC.Encode(oKp)\n\t\trequire_NoError(t, err)\n\n\t\timportKP, _ := nkeys.CreateAccount()\n\t\timportPub, _ := importKP.PublicKey()\n\t\timportAc := jwt.NewAccountClaims(importPub)\n\t\tsrvcSub, strmSub := \"service.foo\", \"stream.foo\"\n\t\tif pass {\n\t\t\tsrvcSub = fmt.Sprintf(\"service.%s\", importPub)\n\t\t\tstrmSub = fmt.Sprintf(\"stream.%s\", importPub)\n\t\t}\n\t\timportAc.Imports.Add(&jwt.Import{Account: exportPub, Subject: jwt.Subject(srvcSub), Type: jwt.Service})\n\t\timportAc.Imports.Add(&jwt.Import{Account: exportPub, Subject: jwt.Subject(strmSub), Type: jwt.Stream})\n\t\timportJWT, err := importAc.Encode(oKp)\n\t\trequire_NoError(t, err)\n\n\t\treturn exportKP, exportPub, exportJWT, exportJWTNoPos, importKP, importPub, importJWT, srvcSub, strmSub\n\t}\n\tt.Run(\"pass\", func(t *testing.T) {\n\t\texportKp, exportPub, exportJWT, _, importKp, importPub, importJWT, srvcSub, strmSub := setupAccounts(true)\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(srvFmt, ojwt, exportPub, exportJWT, importPub, importJWT)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\t\tncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp))\n\t\tdefer ncExp.Close()\n\t\tncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp))\n\t\tdefer ncImp.Close()\n\t\tt.Run(\"service\", func(t *testing.T) {\n\t\t\tsub, err := ncExp.Subscribe(\"service.*\", func(msg *nats.Msg) {\n\t\t\t\tmsg.Respond([]byte(\"world\"))\n\t\t\t})\n\t\t\tdefer sub.Unsubscribe()\n\t\t\trequire_NoError(t, err)\n\t\t\tncExp.Flush()\n\t\t\tmsg, err := ncImp.Request(srvcSub, []byte(\"hello\"), time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, string(msg.Data), \"world\")\n\t\t})\n\t\tt.Run(\"stream\", func(t *testing.T) {\n\t\t\tmsgChan := make(chan *nats.Msg, 4)\n\t\t\tdefer close(msgChan)\n\t\t\tsub, err := ncImp.ChanSubscribe(strmSub, msgChan)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\trequire_NoError(t, err)\n\t\t\tncImp.Flush()\n\t\t\terr = ncExp.Publish(\"stream.foo\", []byte(\"hello\"))\n\t\t\trequire_NoError(t, err)\n\t\t\terr = ncExp.Publish(strmSub, []byte(\"hello\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tmsg := <-msgChan\n\t\t\trequire_Equal(t, string(msg.Data), \"hello\")\n\t\t\trequire_True(t, len(msgChan) == 0)\n\t\t})\n\t})\n\tt.Run(\"fail\", func(t *testing.T) {\n\t\texportKp, exportPub, exportJWT, _, importKp, importPub, importJWT, srvcSub, strmSub := setupAccounts(false)\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(srvFmt, ojwt, exportPub, exportJWT, importPub, importJWT)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\t\tncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp))\n\t\tdefer ncExp.Close()\n\t\tncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp))\n\t\tdefer ncImp.Close()\n\t\tt.Run(\"service\", func(t *testing.T) {\n\t\t\tsub, err := ncExp.Subscribe(\"service.*\", func(msg *nats.Msg) {\n\t\t\t\tmsg.Respond([]byte(\"world\"))\n\t\t\t})\n\t\t\tdefer sub.Unsubscribe()\n\t\t\trequire_NoError(t, err)\n\t\t\tncExp.Flush()\n\t\t\t_, err = ncImp.Request(srvcSub, []byte(\"hello\"), time.Second)\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Contains(t, err.Error(), \"no responders available for request\")\n\t\t})\n\t\tt.Run(\"stream\", func(t *testing.T) {\n\t\t\tmsgChan := make(chan *nats.Msg, 4)\n\t\t\tdefer close(msgChan)\n\t\t\t_, err := ncImp.ChanSubscribe(strmSub, msgChan)\n\t\t\trequire_NoError(t, err)\n\t\t\tncImp.Flush()\n\t\t\terr = ncExp.Publish(\"stream.foo\", []byte(\"hello\"))\n\t\t\trequire_NoError(t, err)\n\t\t\terr = ncExp.Publish(strmSub, []byte(\"hello\"))\n\t\t\trequire_NoError(t, err)\n\t\t\tselect {\n\t\t\tcase <-msgChan:\n\t\t\t\tt.Fatal(\"did not expect a message\")\n\t\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\t}\n\t\t\trequire_True(t, len(msgChan) == 0)\n\t\t})\n\t})\n\tt.Run(\"reload-off-2-on\", func(t *testing.T) {\n\t\texportKp, exportPub, exportJWTOn, exportJWTOff, importKp, _, importJWT, srvcSub, strmSub := setupAccounts(false)\n\t\tdirSrv := t.TempDir()\n\t\t// set up system account. Relying bootstrapping system account to not create JWT\n\t\tsysAcc, err := nkeys.CreateAccount()\n\t\trequire_NoError(t, err)\n\t\tsysPub, err := sysAcc.PublicKey()\n\t\trequire_NoError(t, err)\n\t\tsysUsrCreds := newUserEx(t, sysAcc, false, sysPub)\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\toperator = %s\n\t\tsystem_account = %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}`, ojwt, sysPub, dirSrv)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\tdefer s.Shutdown()\n\t\tupdateJwt(t, s.ClientURL(), sysUsrCreds, importJWT, 1)\n\t\tupdateJwt(t, s.ClientURL(), sysUsrCreds, exportJWTOff, 1)\n\t\tncExp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, exportKp))\n\t\tdefer ncExp.Close()\n\t\tncImp := natsConnect(t, s.ClientURL(), createUserCreds(t, s, importKp))\n\t\tdefer ncImp.Close()\n\t\tmsgChan := make(chan *nats.Msg, 4)\n\t\tdefer close(msgChan)\n\t\t// ensure service passes\n\t\tsubSrvc, err := ncExp.Subscribe(\"service.*\", func(msg *nats.Msg) {\n\t\t\tmsg.Respond([]byte(\"world\"))\n\t\t})\n\t\tdefer subSrvc.Unsubscribe()\n\t\trequire_NoError(t, err)\n\t\tncExp.Flush()\n\t\trespMst, err := ncImp.Request(srvcSub, []byte(\"hello\"), time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, string(respMst.Data), \"world\")\n\t\t// ensure stream passes\n\t\tsubStrm, err := ncImp.ChanSubscribe(strmSub, msgChan)\n\t\tdefer subStrm.Unsubscribe()\n\t\trequire_NoError(t, err)\n\t\tncImp.Flush()\n\t\terr = ncExp.Publish(strmSub, []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t\tmsg := <-msgChan\n\t\trequire_Equal(t, string(msg.Data), \"hello\")\n\t\trequire_True(t, len(msgChan) == 0)\n\n\t\tupdateJwt(t, s.ClientURL(), sysUsrCreds, exportJWTOn, 1)\n\n\t\t// ensure service fails\n\t\t_, err = ncImp.Request(srvcSub, []byte(\"hello\"), time.Second)\n\t\trequire_Error(t, err, nats.ErrNoResponders)\n\t\ts.AccountResolver().Store(exportPub, exportJWTOn)\n\t\t// ensure stream fails\n\t\terr = ncExp.Publish(strmSub, []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t\tselect {\n\t\tcase <-msgChan:\n\t\t\tt.Fatal(\"did not expect a message\")\n\t\tcase <-time.After(250 * time.Millisecond):\n\t\t}\n\t\trequire_True(t, len(msgChan) == 0)\n\t})\n}\n\n// Headers are ignored in claims update, but passing them should not cause error.\nfunc TestJWTClaimsUpdateWithHeaders(t *testing.T) {\n\tskp, spub := createKey(t)\n\tnewUser(t, skp)\n\n\tsclaim := jwt.NewAccountClaims(spub)\n\tencodeClaim(t, sclaim, spub)\n\n\takp, apub := createKey(t)\n\tnewUser(t, akp)\n\tclaim := jwt.NewAccountClaims(apub)\n\tjwtClaim := encodeClaim(t, claim, apub)\n\n\tdirSrv := t.TempDir()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tallow_delete: true\n\t\t}\n `, ojwt, spub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttype zapi struct {\n\t\tServer *ServerInfo\n\t\tData *Connz\n\t\tError *ApiError\n\t}\n\n\tsc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, skp))\n\tdefer sc.Close()\n\t// Pass claims update with headers.\n\tmsg := &nats.Msg{\n\t\tSubject: \"$SYS.REQ.CLAIMS.UPDATE\",\n\t\tData: []byte(jwtClaim),\n\t\tHeader: map[string][]string{\"key\": {\"value\"}},\n\t}\n\tresp, err := sc.RequestMsg(msg, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar cz zapi\n\tif err := json.Unmarshal(resp.Data, &cz); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cz.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t}\n\n\t// Pass claims delete with headers.\n\topk, err := oKp.PublicKey()\n\trequire_NoError(t, err)\n\tc := jwt.NewGenericClaims(opk)\n\tc.Data[\"accounts\"] = []string{apub}\n\tdjwt, err := c.Encode(oKp)\n\trequire_NoError(t, err)\n\tmsg.Subject = \"$SYS.REQ.CLAIMS.DELETE\"\n\tmsg.Data = []byte(djwt)\n\tresp, err = sc.RequestMsg(msg, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcz = zapi{}\n\tif err := json.Unmarshal(resp.Data, &cz); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cz.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t}\n}\n\nfunc TestJWTMappings(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\t// create two jwt, one with and one without mapping\n\taKp, aPub := createKey(t)\n\taClaim := jwt.NewAccountClaims(aPub)\n\taJwtNoM := encodeClaim(t, aClaim, aPub)\n\taClaim.AddMapping(\"foo1\", jwt.WeightedMapping{Subject: \"bar1\"})\n\taJwtMap1 := encodeClaim(t, aClaim, aPub)\n\n\taClaim.Mappings = map[jwt.Subject][]jwt.WeightedMapping{}\n\taClaim.AddMapping(\"foo2\", jwt.WeightedMapping{Subject: \"bar2\"})\n\taJwtMap2 := encodeClaim(t, aClaim, aPub)\n\n\tdirSrv := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, syspub, dirSrv)))\n\tsrv, _ := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\tupdateJwt(t, srv.ClientURL(), sysCreds, sysJwt, 1) // update system account jwt\n\n\ttest := func(pub, sub string, fail bool) {\n\t\tt.Helper()\n\t\tnc := natsConnect(t, srv.ClientURL(), createUserCreds(t, srv, aKp))\n\t\tdefer nc.Close()\n\t\ts, err := nc.SubscribeSync(sub)\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\terr = nc.Publish(pub, nil)\n\t\trequire_NoError(t, err)\n\t\t_, err = s.NextMsg(500 * time.Millisecond)\n\t\tswitch {\n\t\tcase fail && err == nil:\n\t\t\tt.Fatal(\"expected error, got none\")\n\t\tcase !fail && err != nil:\n\t\t\tt.Fatalf(\"expected no error, got %v\", err)\n\t\t}\n\t}\n\n\t// turn mappings on\n\trequire_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtMap1, 1))\n\ttest(\"foo1\", \"bar1\", false)\n\t// alter mappings\n\trequire_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtMap2, 1))\n\ttest(\"foo1\", \"bar1\", true)\n\ttest(\"foo2\", \"bar2\", false)\n\t// turn mappings off\n\trequire_Len(t, 1, updateJwt(t, srv.ClientURL(), sysCreds, aJwtNoM, 1))\n\ttest(\"foo2\", \"bar2\", true)\n}\n\nfunc TestJWTOperatorPinnedAccounts(t *testing.T) {\n\tkps, pubs, jwts := [4]nkeys.KeyPair{}, [4]string{}, [4]string{}\n\tfor i := 0; i < 4; i++ {\n\t\tkps[i], pubs[i] = createKey(t)\n\t\tjwts[i] = encodeClaim(t, jwt.NewAccountClaims(pubs[i]), pubs[i])\n\t}\n\t// create system account user credentials, index 0 is handled as system account\n\tnewUser(t, kps[0])\n\n\tcfgCommon := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s:%s\n\t\t\t%s:%s\n\t\t\t%s:%s\n\t\t\t%s:%s\n\t\t}`, ojwt, pubs[0], pubs[0], jwts[0], pubs[1], jwts[1], pubs[2], jwts[2], pubs[3], jwts[3])\n\tcfgFmt := cfgCommon + `\n\t\tresolver_pinned_accounts: [%s, %s]\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, pubs[1], pubs[2])))\n\tsrv, _ := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\tconnectPass := func(keys ...nkeys.KeyPair) {\n\t\tfor _, kp := range keys {\n\t\t\tnc, err := nats.Connect(srv.ClientURL(), createUserCreds(t, srv, kp))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer nc.Close()\n\t\t}\n\t}\n\tvar pinnedFail uint64\n\tconnectFail := func(key nkeys.KeyPair) {\n\t\t_, err := nats.Connect(srv.ClientURL(), createUserCreds(t, srv, key))\n\t\trequire_Error(t, err)\n\t\trequire_Contains(t, err.Error(), \"Authorization Violation\")\n\t\tv, err := srv.Varz(&VarzOptions{})\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, pinnedFail+1 == v.PinnedAccountFail)\n\t\tpinnedFail = v.PinnedAccountFail\n\t}\n\n\tconnectPass(kps[0], kps[1], kps[2]) // make sure user from accounts listed and system account (index 0) work\n\tconnectFail(kps[3]) // make sure the other user does not work\n\t// reload and test again\n\treloadUpdateConfig(t, srv, conf, fmt.Sprintf(cfgFmt, pubs[2], pubs[3]))\n\tconnectPass(kps[0], kps[2], kps[3]) // make sure user from accounts listed and system account (index 0) work\n\tconnectFail(kps[1]) // make sure the other user does not work\n\t// completely disable and test again\n\treloadUpdateConfig(t, srv, conf, cfgCommon)\n\tconnectPass(kps[0], kps[1], kps[2], kps[3]) // make sure every account and system account (index 0) can connect\n\t// re-enable and test again\n\treloadUpdateConfig(t, srv, conf, fmt.Sprintf(cfgFmt, pubs[2], pubs[3]))\n\tconnectPass(kps[0], kps[2], kps[3]) // make sure user from accounts listed and system account (index 0) work\n\tconnectFail(kps[1]) // make sure the other user does not work\n}\n\nfunc TestJWTNoSystemAccountButNatsResolver(t *testing.T) {\n\tdirSrv := t.TempDir()\n\tfor _, resType := range []string{\"full\", \"cache\"} {\n\t\tt.Run(resType, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\toperator: %s\n\t\t\tresolver: {\n\t\t\t\ttype: %s\n\t\t\t\tdir: '%s'\n\t\t\t}`, ojwt, resType, dirSrv)))\n\t\t\topts := LoadConfig(conf)\n\t\t\ts, err := NewServer(opts)\n\t\t\t// Since the server cannot be stopped, since it did not start,\n\t\t\t// let's manually close the account resolver to avoid leaking go routines.\n\t\t\topts.AccountResolver.Close()\n\t\t\ts.Shutdown()\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Contains(t, err.Error(), \"the system account needs to be specified in configuration or the operator jwt\")\n\t\t})\n\t}\n}\n\nfunc TestJWTAccountConnzAccessAfterClaimUpdate(t *testing.T) {\n\tskp, spub := createKey(t)\n\tnewUser(t, skp)\n\n\tsclaim := jwt.NewAccountClaims(spub)\n\tsclaim.AddMapping(\"foo.bar\", jwt.WeightedMapping{Subject: \"foo.baz\"})\n\tsjwt := encodeClaim(t, sclaim, spub)\n\n\t// create two jwt, one with and one without mapping\n\takp, apub := createKey(t)\n\tnewUser(t, akp)\n\tclaim := jwt.NewAccountClaims(apub)\n\tjwt1 := encodeClaim(t, claim, apub)\n\tclaim.AddMapping(\"foo.bar\", jwt.WeightedMapping{Subject: \"foo.baz\"})\n\tjwt2 := encodeClaim(t, claim, apub)\n\n\tdirSrv := t.TempDir()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n `, ojwt, spub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttype zapi struct {\n\t\tServer *ServerInfo\n\t\tData *Connz\n\t\tError *ApiError\n\t}\n\n\tupdateJWT := func(jwt string) {\n\t\tt.Helper()\n\t\tsc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, skp))\n\t\tdefer sc.Close()\n\t\tresp, err := sc.Request(\"$SYS.REQ.CLAIMS.UPDATE\", []byte(jwt), time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar cz zapi\n\t\tif err := json.Unmarshal(resp.Data, &cz); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cz.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t\t}\n\t}\n\n\tupdateJWT(jwt1)\n\n\tnc := natsConnect(t, s.ClientURL(), createUserCreds(t, s, akp))\n\tdefer nc.Close()\n\n\tdoRequest := func() {\n\t\tt.Helper()\n\t\tresp, err := nc.Request(\"$SYS.REQ.SERVER.PING.CONNZ\", nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tvar cz zapi\n\t\tif err := json.Unmarshal(resp.Data, &cz); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cz.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t\t}\n\t}\n\n\tdoRequest()\n\tupdateJWT(jwt2)\n\t// If we accidentally wipe the system import this will fail with no responders.\n\tdoRequest()\n\t// Now test updating system account.\n\tupdateJWT(sjwt)\n\t// If export was wiped this would fail with timeout.\n\tdoRequest()\n}\n\nfunc TestJWTAccountWeightedMappingInSuperCluster(t *testing.T) {\n\tskp, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"SYS\"\n\tsysCreds := newUser(t, skp)\n\n\takp, apub := createKey(t)\n\taUsr := createUserCreds(t, nil, akp)\n\tclaim := jwt.NewAccountClaims(apub)\n\taJwtMap := encodeClaim(t, claim, apub)\n\n\t// We are using the createJetStreamSuperClusterWithTemplateAndModHook()\n\t// helper, but this test is not about JetStream...\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n `\n\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 3,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tdirSrv := t.TempDir()\n\t\t\treturn fmt.Sprintf(`%s\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: {\n\t\t\t\t\ttype: full\n\t\t\t\t\tdir: '%s'\n\t\t\t\t}\n\t\t\t`, conf, ojwt, spub, dirSrv)\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Update from C2\n\trequire_Len(t, 1, updateJwt(t, sc.clusterForName(\"C2\").randomServer().ClientURL(), sysCreds, aJwtMap, 1))\n\n\t// We will connect our services in the C3 cluster.\n\tnc1 := natsConnect(t, sc.clusterForName(\"C3\").randomServer().ClientURL(), aUsr)\n\tdefer nc1.Close()\n\tnc2 := natsConnect(t, sc.clusterForName(\"C3\").randomServer().ClientURL(), aUsr)\n\tdefer nc2.Close()\n\n\tnatsSub(t, nc1, \"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"foo\"))\n\t})\n\tnatsSub(t, nc1, \"bar.v1\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"v1\"))\n\t})\n\tnatsSub(t, nc2, \"bar.v2\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"v2\"))\n\t})\n\tnatsFlush(t, nc1)\n\tnatsFlush(t, nc2)\n\n\t// Now we will update the account to add weighted subject mapping\n\tclaim.Mappings = map[jwt.Subject][]jwt.WeightedMapping{}\n\t// Start with foo->bar.v2 at 40%, the server will auto-add foo->foo at 60%.\n\twm := []jwt.WeightedMapping{{Subject: \"bar.v2\", Weight: 40}}\n\tclaim.AddMapping(\"foo\", wm...)\n\taJwtMap = encodeClaim(t, claim, apub)\n\n\t// We will update from C2\n\trequire_Len(t, 1, updateJwt(t, sc.clusterForName(\"C2\").randomServer().ClientURL(), sysCreds, aJwtMap, 1))\n\n\ttime.Sleep(time.Second)\n\n\t// And we will publish from C1\n\tnc := natsConnect(t, sc.clusterForName(\"C1\").randomServer().ClientURL(), aUsr)\n\tdefer nc.Close()\n\n\tvar foo, v1, v2 int\n\tpubAndCount := func() {\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tmsg, err := nc.Request(\"foo\", []byte(\"req\"), 500*time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tswitch string(msg.Data) {\n\t\t\tcase \"foo\":\n\t\t\t\tfoo++\n\t\t\tcase \"v1\":\n\t\t\t\tv1++\n\t\t\tcase \"v2\":\n\t\t\t\tv2++\n\t\t\t}\n\t\t}\n\t}\n\tpubAndCount()\n\tif foo < 550 || foo > 650 {\n\t\tt.Fatalf(\"Expected foo to receive 60%%, got %v/1000\", foo)\n\t}\n\tif v1 != 0 {\n\t\tt.Fatalf(\"Expected v1 to receive no message, got %v/1000\", v1)\n\t}\n\tif v2 < 350 || v2 > 450 {\n\t\tt.Fatalf(\"Expected v2 to receive 40%%, got %v/1000\", v2)\n\t}\n\n\t// Now send a new update with foo-> bar.v2(40) and bar.v1(60).\n\t// The auto-add of \"foo\" should no longer be used by the server.\n\twm = []jwt.WeightedMapping{\n\t\t{Subject: \"bar.v2\", Weight: 40},\n\t\t{Subject: \"bar.v1\", Weight: 60},\n\t}\n\tclaim.AddMapping(\"foo\", wm...)\n\taJwtMap = encodeClaim(t, claim, apub)\n\n\t// We will update from C2\n\trequire_Len(t, 1, updateJwt(t, sc.clusterForName(\"C2\").randomServer().ClientURL(), sysCreds, aJwtMap, 1))\n\n\ttime.Sleep(time.Second)\n\n\tfoo, v1, v2 = 0, 0, 0\n\tpubAndCount()\n\tif foo != 0 {\n\t\tt.Fatalf(\"Expected foo to receive no message, got %v/1000\", foo)\n\t}\n\tif v1 < 550 || v1 > 650 {\n\t\tt.Fatalf(\"Expected v1 to receive 60%%, got %v/1000\", v1)\n\t}\n\tif v2 < 350 || v2 > 450 {\n\t\tt.Fatalf(\"Expected v2 to receive 40%%, got %v/1000\", v2)\n\t}\n}\n\nfunc TestJWTServerOperatorModeNoAuthRequired(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\takp, apub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(apub)\n\taccClaim.Name = \"TEST\"\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tnuc := jwt.NewUserClaims(upub)\n\tujwt, err := nuc.Encode(akp)\n\trequire_NoError(t, err)\n\tcreds := genCredsFile(t, ujwt, seed)\n\n\tdirSrv := t.TempDir()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-A\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tinterval: \"200ms\"\n\t\t\tlimit: 4\n\t\t}\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n `, ojwt, spub, dirSrv, spub, sysJwt, apub, accJwt)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(creds))\n\tdefer nc.Close()\n\n\trequire_True(t, nc.AuthRequired())\n}\n\nfunc TestJWTServerOperatorModeUserInfoExpiration(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"$SYS\"\n\tsysJwt, err := sysClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\takp, apub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(apub)\n\taccClaim.Name = \"TEST\"\n\taccJwt, err := accClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\toperator: %s\n\t\t\t\tsystem_account: %s\n\t\t\t\tresolver: MEM\n\t\t\t\tresolver_preload: {\n\t\t\t\t\t%s: %s\n\t\t\t\t\t%s: %s\n\t\t\t\t}\n\t\t\t`, ojwt, spub, apub, accJwt, spub, sysJwt)))\n\tdefer removeFile(t, conf)\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\texpires := time.Now().Add(time.Minute)\n\tcreds := createUserWithLimit(t, akp, expires, func(j *jwt.UserPermissionLimits) {})\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserCredentials(creds))\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(\"$SYS.REQ.USER.INFO\", nil, time.Second)\n\trequire_NoError(t, err)\n\tnow := time.Now()\n\n\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\terr = json.Unmarshal(resp.Data, &response)\n\trequire_NoError(t, err)\n\n\tuserInfo := response.Data.(*UserInfo)\n\trequire_True(t, userInfo.Expires != 0)\n\n\t// We need to round the expiration time to the second because the server\n\t// will truncate the expiration time to the second.\n\texpiresDurRounded := expires.Sub(now).Truncate(time.Second)\n\n\t// Checking range to avoid flaky tests where the expiration time is\n\t// off by a couple of seconds.\n\trequire_True(t, expiresDurRounded >= userInfo.Expires-2*time.Second && expiresDurRounded <= userInfo.Expires+2*time.Second)\n}\n\nfunc TestJWTAccountNATSResolverWrongCreds(t *testing.T) {\n\trequire_NoLocalOrRemoteConnections := func(account string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tfor _, srv := range srvs {\n\t\t\tif acc, ok := srv.accounts.Load(account); ok {\n\t\t\t\tcheckAccClientsCount(t, acc.(*Account), 0)\n\t\t\t}\n\t\t}\n\t}\n\tconnect := func(url string, credsfile string, acc string, srvs ...*Server) {\n\t\tt.Helper()\n\t\tnc := natsConnect(t, url, nats.UserCredentials(credsfile), nats.Timeout(5*time.Second))\n\t\tnc.Close()\n\t\trequire_NoLocalOrRemoteConnections(acc, srvs...)\n\t}\n\tcreateAccountAndUser := func(pubKey, jwt1, jwt2, creds *string) {\n\t\tt.Helper()\n\t\tkp, _ := nkeys.CreateAccount()\n\t\t*pubKey, _ = kp.PublicKey()\n\t\tclaim := jwt.NewAccountClaims(*pubKey)\n\t\tvar err error\n\t\t*jwt1, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\t*jwt2, err = claim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tujwt, err := uclaim.Encode(kp)\n\t\trequire_NoError(t, err)\n\t\t*creds = genCredsFile(t, ujwt, seed)\n\t}\n\t// Create Accounts and corresponding user creds.\n\tvar syspub, sysjwt, dummy1, sysCreds string\n\tcreateAccountAndUser(&syspub, &sysjwt, &dummy1, &sysCreds)\n\n\tvar apub, ajwt1, ajwt2, aCreds string\n\tcreateAccountAndUser(&apub, &ajwt1, &ajwt2, &aCreds)\n\n\tvar bpub, bjwt1, bjwt2, bCreds string\n\tcreateAccountAndUser(&bpub, &bjwt1, &bjwt2, &bCreds)\n\n\t// The one that is going to be missing.\n\tvar cpub, cjwt1, cjwt2, cCreds string\n\tcreateAccountAndUser(&cpub, &cjwt1, &cjwt2, &cCreds)\n\t// Create one directory for each server\n\tdirA := t.TempDir()\n\tdirB := t.TempDir()\n\tdirC := t.TempDir()\n\n\t// Store accounts on servers A and B, then let C sync on its own.\n\twriteJWT(t, dirA, apub, ajwt1)\n\twriteJWT(t, dirB, bpub, bjwt1)\n\n\t/////////////////////////////////////////\n\t// //\n\t// Server A: has creds from client A //\n\t// //\n\t/////////////////////////////////////////\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-A\n\t\toperator: %s\n\t\tsystem_account: %s\n debug: true\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tallow_delete: true\n\t\t\ttimeout: \"1.5s\"\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t}\n `, ojwt, syspub, dirA, apub, ajwt1)))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\trequire_JWTPresent(t, dirA, apub)\n\n\t/////////////////////////////////////////\n\t// //\n\t// Server B: has creds from client B //\n\t// //\n\t/////////////////////////////////////////\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-B\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tallow_delete: true\n\t\t\ttimeout: \"1.5s\"\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `, ojwt, syspub, dirB, sA.opts.Cluster.Port)))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\n\t/////////////////////////////////////////\n\t// //\n\t// Server C: has no creds //\n\t// //\n\t/////////////////////////////////////////\n\tfmtC := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: srv-C\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t\tallow_delete: true\n\t\t\ttimeout: \"1.5s\"\n\t\t\tinterval: \"200ms\"\n\t\t}\n\t\tcluster {\n\t\t\tname: clust\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tno_advertise: true\n\t\t\troutes [\n\t\t\t\tnats-route://127.0.0.1:%d\n\t\t\t]\n\t\t}\n `\n\tconfClongTTL := createConfFile(t, []byte(fmt.Sprintf(fmtC, ojwt, syspub, dirC, sA.opts.Cluster.Port)))\n\tsC, _ := RunServerWithConfig(confClongTTL) // use long ttl to assure it is not kicking\n\tdefer sC.Shutdown()\n\n\t// startup cluster\n\tcheckClusterFormed(t, sA, sB, sC)\n\ttime.Sleep(1 * time.Second) // wait for the protocol to converge\n\t// // Check all accounts\n\trequire_JWTPresent(t, dirA, apub) // was already present on startup\n\trequire_JWTPresent(t, dirB, apub) // was copied from server A\n\trequire_JWTPresent(t, dirA, bpub) // was copied from server B\n\trequire_JWTPresent(t, dirB, bpub) // was already present on startup\n\n\t// There should be no state about the missing account.\n\trequire_JWTAbsent(t, dirA, cpub)\n\trequire_JWTAbsent(t, dirB, cpub)\n\trequire_JWTAbsent(t, dirC, cpub)\n\n\t// system account client can connect to every server\n\tconnect(sA.ClientURL(), sysCreds, \"\")\n\tconnect(sB.ClientURL(), sysCreds, \"\")\n\tconnect(sC.ClientURL(), sysCreds, \"\")\n\n\t// A and B clients can connect to any server.\n\tconnect(sA.ClientURL(), aCreds, \"\")\n\tconnect(sB.ClientURL(), aCreds, \"\")\n\tconnect(sC.ClientURL(), aCreds, \"\")\n\tconnect(sA.ClientURL(), bCreds, \"\")\n\tconnect(sB.ClientURL(), bCreds, \"\")\n\tconnect(sC.ClientURL(), bCreds, \"\")\n\n\t// Check that trying to connect with bad credentials should not hang until the fetch timeout\n\t// and instead return a faster response when an account is not found.\n\t_, err := nats.Connect(sC.ClientURL(), nats.UserCredentials(cCreds), nats.Timeout(500*time.Second))\n\tif err != nil && !errors.Is(err, nats.ErrAuthorization) {\n\t\tt.Fatalf(\"Expected auth error: %v\", err)\n\t}\n}\n\n// Issue 5480: https://github.com/nats-io/nats-server/issues/5480\nfunc TestJWTImportsOnServerRestartAndClientsReconnect(t *testing.T) {\n\ttype namedCreds struct {\n\t\tname string\n\t\tcreds nats.Option\n\t}\n\tpreload := make(map[string]string)\n\tusers := make(map[string]*namedCreds)\n\n\t// sys account\n\t_, sysAcc, sysAccClaim := NewJwtAccountClaim(\"sys\")\n\tsysAccJWT, err := sysAccClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\tpreload[sysAcc] = sysAccJWT\n\n\t// main account, other accounts will import from this.\n\tmainAccKP, mainAcc, mainAccClaim := NewJwtAccountClaim(\"main\")\n\tmainAccClaim.Exports.Add(&jwt.Export{\n\t\tType: jwt.Stream,\n\t\tSubject: \"city.>\",\n\t})\n\n\t// main account user\n\tmainUserClaim := jwt.NewUserClaims(\"publisher\")\n\tmainUserClaim.Permissions = jwt.Permissions{\n\t\tPub: jwt.Permission{\n\t\t\tAllow: []string{\"city.>\"},\n\t\t},\n\t}\n\tmainCreds := createUserCredsEx(t, mainUserClaim, mainAccKP)\n\n\t// The main account will be importing from all other accounts.\n\tmaxAccounts := 100\n\tfor i := 0; i < maxAccounts; i++ {\n\t\tname := fmt.Sprintf(\"secondary-%d\", i)\n\t\taccKP, acc, accClaim := NewJwtAccountClaim(name)\n\n\t\taccClaim.Exports.Add(&jwt.Export{\n\t\t\tType: jwt.Stream,\n\t\t\tSubject: \"internal.*\",\n\t\t})\n\t\taccClaim.Imports.Add(&jwt.Import{\n\t\t\tType: jwt.Stream,\n\t\t\tSubject: jwt.Subject(fmt.Sprintf(\"city.%d-1.*\", i)),\n\t\t\tAccount: mainAcc,\n\t\t})\n\n\t\t// main account imports from the secondary accounts\n\t\tmainAccClaim.Imports.Add(&jwt.Import{\n\t\t\tType: jwt.Stream,\n\t\t\tSubject: jwt.Subject(fmt.Sprintf(\"internal.%d\", i)),\n\t\t\tAccount: acc,\n\t\t})\n\n\t\taccJWT, err := accClaim.Encode(oKp)\n\t\trequire_NoError(t, err)\n\t\tpreload[acc] = accJWT\n\n\t\tuserClaim := jwt.NewUserClaims(\"subscriber\")\n\t\tuserClaim.Permissions = jwt.Permissions{\n\t\t\tSub: jwt.Permission{\n\t\t\t\tAllow: []string{\"city.>\", \"internal.*\"},\n\t\t\t},\n\t\t\tPub: jwt.Permission{\n\t\t\t\tAllow: []string{\"internal.*\"},\n\t\t\t},\n\t\t}\n\t\tuserCreds := createUserCredsEx(t, userClaim, accKP)\n\t\tusers[acc] = &namedCreds{name, userCreds}\n\t}\n\tmainAccJWT, err := mainAccClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\tpreload[mainAcc] = mainAccJWT\n\n\t// Start the server with the preload.\n\tresolverPreload, err := json.Marshal(preload)\n\trequire_NoError(t, err)\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n http: 127.0.0.1:8222\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n\t`, ojwt, sysAcc, string(resolverPreload))))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Have a connection ready for each one of the accounts.\n\ttype namedSub struct {\n\t\tname string\n\t\tsub *nats.Subscription\n\t}\n\tsubs := make(map[string]*namedSub)\n\tfor acc, user := range users {\n\t\tnc := natsConnect(t, s.ClientURL(), user.creds,\n\t\t\t// Make the clients attempt to reconnect too fast,\n\t\t\t// changing this to be above ~200ms mitigates the issue.\n\t\t\tnats.ReconnectWait(15*time.Millisecond),\n\t\t\tnats.Name(user.name),\n\t\t\tnats.MaxReconnects(-1),\n\t\t)\n\t\tdefer nc.Close()\n\n\t\tsub, err := nc.SubscribeSync(\"city.>\")\n\t\trequire_NoError(t, err)\n\t\tsubs[acc] = &namedSub{user.name, sub}\n\t}\n\n\tnc := natsConnect(t, s.ClientURL(), mainCreds, nats.ReconnectWait(15*time.Millisecond), nats.MaxReconnects(-1))\n\tdefer nc.Close()\n\n\tsend := func(t *testing.T) {\n\t\tt.Helper()\n\t\tfor i := 0; i < maxAccounts; i++ {\n\t\t\tnc.Publish(fmt.Sprintf(\"city.%d-1.A4BDB048-69DC-4F10-916C-2B998249DC11\", i), []byte(fmt.Sprintf(\"test:%d\", i)))\n\t\t}\n\t\tnc.Flush()\n\t}\n\n\tctx, done := context.WithCancel(context.Background())\n\tdefer done()\n\tgo func() {\n\t\tfor range time.NewTicker(200 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tsend(t)\n\t\t}\n\t}()\n\n\treceive := func(t *testing.T) {\n\t\tt.Helper()\n\t\treceived := 0\n\t\tfor _, nsub := range subs {\n\t\t\t// Drain first any pending messages.\n\t\t\tpendingMsgs, _, _ := nsub.sub.Pending()\n\t\t\tfor i, _ := 0, 0; i < pendingMsgs; i++ {\n\t\t\t\tnsub.sub.NextMsg(500 * time.Millisecond)\n\t\t\t}\n\n\t\t\t_, err = nsub.sub.NextMsg(500 * time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"WRN: Failed to receive message on account %q: %v\", nsub.name, err)\n\t\t\t} else {\n\t\t\t\treceived++\n\t\t\t}\n\t\t}\n\t\tif received < (maxAccounts / 2) {\n\t\t\tt.Fatalf(\"Too many missed messages after restart. Received %d\", received)\n\t\t}\n\t}\n\treceive(t)\n\ttime.Sleep(1 * time.Second)\n\n\trestart := func(t *testing.T) *Server {\n\t\tt.Helper()\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t\ts, _ = RunServerWithConfig(conf)\n\n\t\thctx, hcancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\tdefer hcancel()\n\t\tfor range time.NewTicker(2 * time.Second).C {\n\t\t\tselect {\n\t\t\tcase <-hctx.Done():\n\t\t\t\tt.Logf(\"WRN: Timed out waiting for healthz from %s\", s)\n\t\t\tdefault:\n\t\t\t}\n\n\t\t\tstatus := s.healthz(nil)\n\t\t\tif status.StatusCode == 200 {\n\t\t\t\treturn s\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Takes a few restarts for issue to show up.\n\tfor i := 0; i < 5; i++ {\n\t\ts := restart(t)\n\t\tdefer s.Shutdown()\n\t\ttime.Sleep(2 * time.Second)\n\t\treceive(t)\n\t}\n}\n\nfunc TestDefaultSentinelUser(t *testing.T) {\n\tvar err error\n\tpreload := make(map[string]string)\n\n\t_, sysPub, sysAC := NewJwtAccountClaim(\"SYS\")\n\tpreload[sysPub], err = sysAC.Encode(oKp)\n\trequire_NoError(t, err)\n\n\taKP, aPub, aAC := NewJwtAccountClaim(\"A\")\n\taScopedKP, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taScopedPK, err := aScopedKP.PublicKey()\n\trequire_NoError(t, err)\n\n\tsentinelScope := jwt.NewUserScope()\n\tsentinelScope.Key = aScopedPK\n\tsentinelScope.Role = \"sentinel\"\n\tsentinelScope.Description = \"Sentinel Role\"\n\tsentinelScope.Template = jwt.UserPermissionLimits{\n\t\tBearerToken: true,\n\t\tPermissions: jwt.Permissions{\n\t\t\tPub: jwt.Permission{Deny: []string{\">\"}},\n\t\t\tSub: jwt.Permission{Deny: []string{\">\"}},\n\t\t},\n\t}\n\taAC.SigningKeys.AddScopedSigner(sentinelScope)\n\n\tpreload[aPub], err = aAC.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tpreloadConfig, err := json.MarshalIndent(preload, \"\", \" \")\n\trequire_NoError(t, err)\n\n\t// test that the user will be rejected\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n`, ojwt, sysPub, preloadConfig)))\n\n\tns, _ := RunServerWithConfig(conf)\n\tdefer ns.Shutdown()\n\t_, err = nats.Connect(ns.ClientURL(), nats.MaxReconnects(0))\n\trequire_Error(t, err)\n\trequire_True(t, errors.Is(err, nats.ErrAuthorization))\n\tns.Shutdown()\n\n\t// test that user can connect\n\tuKP, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tuPub, err := uKP.PublicKey()\n\trequire_NoError(t, err)\n\tuc := jwt.NewUserClaims(uPub)\n\tuc.BearerToken = false\n\tuc.Name = \"sentinel\"\n\tsentinelToken, err := uc.Encode(aKP)\n\trequire_NoError(t, err)\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n\t\t\tdefault_sentinel: %s\n`, ojwt, sysPub, preloadConfig, sentinelToken)))\n\n\t// test non-bearer sentinel is rejected\n\topts, err := ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\t_, err = NewServer(opts)\n\trequire_Error(t, err, fmt.Errorf(\"default sentinel must be a bearer token\"))\n\n\t// correct and start server\n\tuc.BearerToken = true\n\tsentinelToken, err = uc.Encode(aKP)\n\trequire_NoError(t, err)\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n\t\t\tdefault_sentinel: %s\n`, ojwt, sysPub, preloadConfig, sentinelToken)))\n\n\tns, _ = RunServerWithConfig(conf)\n\tnc, err := nats.Connect(ns.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tr, err := nc.Request(\"$SYS.REQ.USER.INFO\", nil, time.Second*5)\n\trequire_NoError(t, err)\n\ttype SR struct {\n\t\tData UserInfo `json:\"data\"`\n\t}\n\tvar ui SR\n\trequire_NoError(t, json.Unmarshal(r.Data, &ui))\n\trequire_Equal(t, ui.Data.UserID, uPub)\n\tns.Shutdown()\n\n\t// now lets make a sentinel that is a scoped user with bearer token\n\tuc = jwt.NewUserClaims(uPub)\n\tuc.IssuerAccount = aPub\n\tuc.UserPermissionLimits = jwt.UserPermissionLimits{}\n\n\tsentinelToken, err = uc.Encode(aScopedKP)\n\trequire_NoError(t, err)\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n\t\t\tdefault_sentinel: %s\n`, ojwt, sysPub, preloadConfig, sentinelToken)))\n\tns, _ = RunServerWithConfig(conf)\n\tdefer ns.Shutdown()\n\tnc, err = nats.Connect(ns.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n}\n\nfunc TestJWTUpdateAccountClaimsStreamAndServiceImportDeadlock(t *testing.T) {\n\tfor _, exportType := range []jwt.ExportType{jwt.Stream, jwt.Service} {\n\t\tt.Run(exportType.String(), func(t *testing.T) {\n\t\t\ts := opTrustBasicSetup()\n\t\t\tdefer s.Shutdown()\n\t\t\tbuildMemAccResolver(s)\n\n\t\t\t// Get operator.\n\t\t\tokp, err := nkeys.FromSeed(oSeed)\n\t\t\trequire_NoError(t, err)\n\n\t\t\ttype Acc struct {\n\t\t\t\tpub string\n\t\t\t\tac *jwt.AccountClaims\n\t\t\t\ta *Account\n\t\t\t\tc *client\n\t\t\t}\n\n\t\t\t// Create accounts.\n\t\t\tvar accs []*Acc\n\t\t\tnumAccounts := 10\n\t\t\tfor i := 0; i < numAccounts; i++ {\n\t\t\t\taKp, err := nkeys.CreateAccount()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\taPub, err := aKp.PublicKey()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\taAC := jwt.NewAccountClaims(aPub)\n\t\t\t\taJWT, err := aAC.Encode(okp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\taddAccountToMemResolver(s, aPub, aJWT)\n\n\t\t\t\taAcc, err := s.LookupAccount(aPub)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\taAcc.mu.Lock()\n\t\t\t\tc := aAcc.internalClient()\n\t\t\t\taAcc.mu.Unlock()\n\t\t\t\taAcc.addClient(c)\n\n\t\t\t\taccs = append(accs, &Acc{aPub, aAC, aAcc, c})\n\t\t\t}\n\n\t\t\taddImportExport := func(i int, acc *Acc) {\n\t\t\t\tlocalSubject := fmt.Sprintf(\"%s.%d\", acc.pub, i)\n\t\t\t\tacc.ac.Exports.Add(&jwt.Export{Subject: jwt.Subject(localSubject), Type: exportType})\n\t\t\t\tfor _, oAcc := range accs {\n\t\t\t\t\tif acc.pub == oAcc.pub {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\texternalSubject := fmt.Sprintf(\"%s.%d\", oAcc.pub, i)\n\t\t\t\t\tacc.ac.Imports.Add(&jwt.Import{Account: oAcc.pub, Subject: jwt.Subject(externalSubject), Type: exportType})\n\t\t\t\t}\n\t\t\t}\n\t\t\ttest := func(i int) {\n\t\t\t\tvar start sync.WaitGroup\n\t\t\t\tvar release sync.WaitGroup\n\t\t\t\tvar finish sync.WaitGroup\n\t\t\t\tstart.Add(numAccounts)\n\t\t\t\trelease.Add(1)\n\t\t\t\tfinish.Add(numAccounts)\n\n\t\t\t\t// Add imports/exports to both accounts and update in parallel, should not deadlock.\n\t\t\t\tfor _, acc := range accs {\n\t\t\t\t\tacc := acc\n\t\t\t\t\tgo func() {\n\t\t\t\t\t\tdefer finish.Done()\n\t\t\t\t\t\taddImportExport(i, acc)\n\t\t\t\t\t\tjwt, err := acc.ac.Encode(okp)\n\t\t\t\t\t\taddAccountToMemResolver(s, acc.pub, jwt)\n\t\t\t\t\t\tstart.Done()\n\t\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\t\trelease.Wait()\n\t\t\t\t\t\ts.UpdateAccountClaims(acc.a, acc.ac)\n\t\t\t\t\t}()\n\t\t\t\t}\n\n\t\t\t\tstart.Wait()\n\n\t\t\t\t// Lock all clients, once we release below we'll get all claim updates\n\t\t\t\t// in the same place after initial checks.\n\t\t\t\tfor _, acc := range accs {\n\t\t\t\t\tacc.c.mu.Lock()\n\t\t\t\t}\n\t\t\t\trelease.Done()\n\n\t\t\t\t// Wait some time for them to reach that point and be blocked on the client lock.\n\t\t\t\ttime.Sleep(time.Second)\n\t\t\t\tfor _, acc := range accs {\n\t\t\t\t\tacc.c.mu.Unlock()\n\t\t\t\t}\n\n\t\t\t\t// Eventually all goroutines should finish.\n\t\t\t\tfinish.Wait()\n\t\t\t}\n\n\t\t\t// Repeat test multiple times, increasing the amount of imports/exports along the way.\n\t\t\tfor i := 0; i < 30; i++ {\n\t\t\t\ttest(i)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJWTJetStreamClientsExcludedForMaxConnsUpdate(t *testing.T) {\n\tsysKp, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\tsysCreds := newUser(t, sysKp)\n\n\taccKp, accPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accPub)\n\taccClaim.Name = \"acc\"\n\taccClaim.Limits.JetStreamTieredLimits[\"R1\"] = jwt.JetStreamLimits{\n\t\tDiskStorage: 1100, MemoryStorage: 0, Consumer: 2, Streams: 2}\n\taccClaim.Limits.Conn = 5\n\taccJwt1 := encodeClaim(t, accClaim, accPub)\n\taccCreds := newUser(t, accKp)\n\n\tstoreDir := t.TempDir()\n\n\tdirSrv := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s1\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t`, storeDir, ojwt, syspub, dirSrv)))\n\n\ts, _ := RunServerWithConfig(cf)\n\tdefer s.Shutdown()\n\n\tupdateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\n\tnc, js := jsClientConnectURL(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 1, Subjects: []string{\"foo\"}})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\n\taccClaim.Limits.Conn = 1\n\taccJwt1 = encodeClaim(t, accClaim, accPub)\n\tupdateJwt(t, s.ClientURL(), sysCreds, accJwt1, 1)\n\n\t// Manually reconnect.\n\tnc.Close()\n\tnc, js = jsClientConnectURL(t, s.ClientURL(), nats.UserCredentials(accCreds))\n\tdefer nc.Close()\n\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n}\n\nfunc TestJWTClusterUserInfoContainsPermissions(t *testing.T) {\n\ttmpl := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: %s\n\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\t\tcluster {\n\t\t\t\tname: %s\n\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\troutes = [%s]\n\t\t\t}\n\t`\n\topFrag := `\n\t\t\toperator: %s\n\t\t\tsystem_account: %s\n\t\t\tresolver: { type: MEM }\n\t\t\tresolver_preload = {\n\t\t\t\t%s : %s\n\t\t\t\t%s : %s\n\t\t\t}\n\t\t`\n\n\t_, syspub := createKey(t)\n\tsysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub)\n\n\taccKp, aExpPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(aExpPub)\n\taccClaim.DefaultPermissions.Sub = jwt.Permission{\n\t\tDeny: []string{\"foo\"},\n\t}\n\taccJwt := encodeClaim(t, accClaim, aExpPub)\n\taccCreds := newUser(t, accKp)\n\n\ttemplate := tmpl + fmt.Sprintf(opFrag, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt)\n\tc := createJetStreamClusterWithTemplate(t, template, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Since it's a bit of a race whether the local server responds via the\n\t// service import before a remote server does, we need to keep trying.\n\t// In 1000 attempts it is quite easy to reproduce the problem.\n\ttest := func() {\n\t\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserCredentials(accCreds))\n\t\tdefer nc.Close()\n\n\t\tresp, err := nc.Request(userDirectInfoSubj, nil, time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tresponse := ServerAPIResponse{Data: &UserInfo{}}\n\t\trequire_NoError(t, json.Unmarshal(resp.Data, &response))\n\n\t\tuserInfo := response.Data.(*UserInfo)\n\t\trequire_NotNil(t, userInfo.Permissions)\n\t}\n\tfor range 1000 {\n\t\ttest()\n\t}\n}\n\nfunc TestJWTAccountLimitsOverflowInt32(t *testing.T) {\n\t// Without clamping, int32 truncation of values > MaxInt32 causes:\n\t// - mleafs/mconns wrapping to negative, triggering panics in\n\t// updateRemoteServer (slice bounds out of range) and rejecting\n\t// all connections.\n\tfooAC := newJWTTestAccountClaims()\n\tfooAC.Limits.Conn = math.MaxInt32 + 1\n\tfooAC.Limits.LeafNodeConn = math.MaxInt32 + 1\n\tfooAC.Limits.Subs = math.MaxInt32 + 1\n\tfooAC.Limits.Payload = math.MaxInt32 + 1\n\n\ts, fooKP, c, _ := setupJWTTestWitAccountClaims(t, fooAC, \"+OK\")\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tfooPub, _ := fooKP.PublicKey()\n\tfooAcc, _ := s.LookupAccount(fooPub)\n\tfooAcc.mu.RLock()\n\tmconns := fooAcc.mconns\n\tmleafs := fooAcc.mleafs\n\tmsubs := fooAcc.msubs\n\tmpay := fooAcc.mpay\n\tfooAcc.mu.RUnlock()\n\n\t// All account limits should be clamped to math.MaxInt32.\n\tif mconns != math.MaxInt32 {\n\t\tt.Fatalf(\"Expected account mconns to be MaxInt32 (%d), got %d\", math.MaxInt32, mconns)\n\t}\n\tif mleafs != math.MaxInt32 {\n\t\tt.Fatalf(\"Expected account mleafs to be MaxInt32 (%d), got %d\", math.MaxInt32, mleafs)\n\t}\n\tif msubs != math.MaxInt32 {\n\t\tt.Fatalf(\"Expected account msubs to be MaxInt32 (%d), got %d\", math.MaxInt32, msubs)\n\t}\n\tif mpay != math.MaxInt32 {\n\t\tt.Fatalf(\"Expected account mpay to be MaxInt32 (%d), got %d\", math.MaxInt32, mpay)\n\t}\n\n\t// Simulate a remote server update — without clamping this panics with:\n\t// panic: runtime error: slice bounds out of range [2147483648:0]\n\tclients := fooAcc.updateRemoteServer(&AccountNumConns{\n\t\tServer: ServerInfo{\n\t\t\tID: \"fake-server-1\",\n\t\t\tName: \"fake-nats-1\",\n\t\t},\n\t\tAccountStat: AccountStat{\n\t\t\tAccount: fooPub,\n\t\t\tConns: 1,\n\t\t\tLeafNodes: 1,\n\t\t},\n\t})\n\tif len(clients) != 0 {\n\t\tt.Fatalf(\"Expected no clients to disconnect, got %d\", len(clients))\n\t}\n}\n\nfunc TestJWTUserLimitsOverflowInt32SubPub(t *testing.T) {\n\tt.Run(\"Subs\", func(t *testing.T) {\n\t\tnuc := newJWTTestUserClaims()\n\t\t// Without clamping, int32(math.MaxInt32+1) wraps to MinInt32,\n\t\t// making subsAtLimit() always true and blocking all subscriptions.\n\t\tnuc.Limits.Subs = math.MaxInt32 + 1\n\t\ts, c, cr := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\t\tdefer s.Shutdown()\n\t\tdefer c.close()\n\n\t\texpectPong(t, cr)\n\n\t\t// With clamping, subscriptions should succeed.\n\t\t// Before, this would have been `-ERR 'maximum subscriptions exceeded`\n\t\tc.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\t\tl, _ := cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, \"+OK\") {\n\t\t\tt.Fatalf(\"Expected +OK, got %q\", l)\n\t\t}\n\t})\n\n\tt.Run(\"Payload\", func(t *testing.T) {\n\t\tnuc := newJWTTestUserClaims()\n\t\t// Without clamping, int32(math.MaxInt32+1) wraps to MinInt32,\n\t\t// making int64(size) > int64(negative) always true and rejecting\n\t\t// all publishes and closing the connection.\n\t\tnuc.Limits.Payload = math.MaxInt32 + 1\n\t\ts, c, cr := setupJWTTestWithUserClaims(t, nuc, \"+OK\")\n\t\tdefer s.Shutdown()\n\t\tdefer c.close()\n\n\t\texpectPong(t, cr)\n\n\t\t// With clamping, publish should succeed.\n\t\t// Before, this would have caused `-ERR 'Maximum Payload Violation'`\n\t\t// then disconnect the client.\n\t\tc.parseAsync(\"PUB baz 5\\r\\nhello\\r\\nPING\\r\\n\")\n\t\tl, _ := cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, \"+OK\") {\n\t\t\tt.Fatalf(\"Expected +OK, got %q\", l)\n\t\t}\n\t})\n\n\tt.Run(\"ScopedSigningKey\", func(t *testing.T) {\n\t\t// Without clamping in the scoped signing key path (client.go\n\t\t// userScope.Template.Limits), int32 truncation of values >\n\t\t// MaxInt32 would wrap to negative and reject all subs/publishes.\n\t\takp, _ := nkeys.CreateAccount()\n\t\tapub, _ := akp.PublicKey()\n\t\tnac := jwt.NewAccountClaims(apub)\n\n\t\t// Create a scoped signing key with overflow limits.\n\t\tskp, _ := nkeys.CreateAccount()\n\t\tspub, _ := skp.PublicKey()\n\t\tscope := jwt.NewUserScope()\n\t\tscope.Key = spub\n\t\tscope.Template.Limits.Subs = math.MaxInt32 + 1\n\t\tscope.Template.Limits.Payload = math.MaxInt32 + 1\n\t\tnac.SigningKeys.AddScopedSigner(scope)\n\n\t\tajwt, err := nac.Encode(oKp)\n\t\trequire_NoError(t, err)\n\n\t\t// Create user signed by the scoped signing key.\n\t\t// SetScoped(true) clears UserPermissionLimits so the\n\t\t// scope template is used instead of user claims.\n\t\tukp, _ := nkeys.CreateUser()\n\t\tupub, _ := ukp.PublicKey()\n\t\tnuc := jwt.NewUserClaims(upub)\n\t\tnuc.IssuerAccount = apub\n\t\tnuc.SetScoped(true)\n\t\tujwt, err := nuc.Encode(skp)\n\t\trequire_NoError(t, err)\n\n\t\ts := opTrustBasicSetup()\n\t\tdefer s.Shutdown()\n\t\tbuildMemAccResolver(s)\n\t\taddAccountToMemResolver(s, apub, ajwt)\n\n\t\tc, cr, l := newClientForServer(s)\n\t\tdefer c.close()\n\n\t\tvar info nonceInfo\n\t\tjson.Unmarshal([]byte(l[5:]), &info)\n\t\tsigraw, _ := ukp.Sign([]byte(info.Nonce))\n\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t\tcs := fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", ujwt, sig)\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tc.parse([]byte(cs))\n\t\t\twg.Done()\n\t\t}()\n\t\tl, _ = cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, \"+OK\") {\n\t\t\tt.Fatalf(\"Expected +OK on CONNECT, got %q\", l)\n\t\t}\n\t\twg.Wait()\n\t\texpectPong(t, cr)\n\n\t\t// SUB must succeed — without clamping, msubs overflows to\n\t\t// negative and subsAtLimit() always returns true.\n\t\tc.parseAsync(\"SUB foo 1\\r\\nPING\\r\\n\")\n\t\tl, _ = cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, \"+OK\") {\n\t\t\tt.Fatalf(\"Expected +OK on SUB, got %q\", l)\n\t\t}\n\t\texpectPong(t, cr)\n\n\t\t// PUB must succeed — without clamping, mpay overflows to\n\t\t// negative and the payload check always triggers.\n\t\tc.parseAsync(\"PUB baz 5\\r\\nhello\\r\\nPING\\r\\n\")\n\t\tl, _ = cr.ReadString('\\n')\n\t\tif !strings.HasPrefix(l, \"+OK\") {\n\t\t\tt.Fatalf(\"Expected +OK on PUB, got %q\", l)\n\t\t}\n\t})\n}\n" }, { "path": "server/leafnode.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nconst (\n\t// Warning when user configures leafnode TLS insecure\n\tleafnodeTLSInsecureWarning = \"TLS certificate chain and hostname of solicited leafnodes will not be verified. DO NOT USE IN PRODUCTION!\"\n\n\t// When a loop is detected, delay the reconnect of solicited connection.\n\tleafNodeReconnectDelayAfterLoopDetected = 30 * time.Second\n\n\t// When a server receives a message causing a permission violation, the\n\t// connection is closed and it won't attempt to reconnect for that long.\n\tleafNodeReconnectAfterPermViolation = 30 * time.Second\n\n\t// When we have the same cluster name as the hub.\n\tleafNodeReconnectDelayAfterClusterNameSame = 30 * time.Second\n\n\t// Prefix for loop detection subject\n\tleafNodeLoopDetectionSubjectPrefix = \"$LDS.\"\n\n\t// Path added to URL to indicate to WS server that the connection is a\n\t// LEAF connection as opposed to a CLIENT.\n\tleafNodeWSPath = \"/leafnode\"\n\n\t// When a soliciting leafnode is rejected because it does not meet the\n\t// configured minimum version, delay the next reconnect attempt by this long.\n\tleafNodeMinVersionReconnectDelay = 5 * time.Second\n)\n\ntype leaf struct {\n\t// We have any auth stuff here for solicited connections.\n\tremote *leafNodeCfg\n\t// isSpoke tells us what role we are playing.\n\t// Used when we receive a connection but otherside tells us they are a hub.\n\tisSpoke bool\n\t// remoteCluster is when we are a hub but the spoke leafnode is part of a cluster.\n\tremoteCluster string\n\t// remoteServer holds onto the remote server's name or ID.\n\tremoteServer string\n\t// domain name of remote server\n\tremoteDomain string\n\t// account name of remote server\n\tremoteAccName string\n\t// Whether or not we want to propagate east-west interest from other LNs.\n\tisolated bool\n\t// Used to suppress sub and unsub interest. Same as routes but our audience\n\t// here is tied to this leaf node. This will hold all subscriptions except this\n\t// leaf nodes. This represents all the interest we want to send to the other side.\n\tsmap map[string]int32\n\t// This map will contain all the subscriptions that have been added to the smap\n\t// during initLeafNodeSmapAndSendSubs. It is short lived and is there to avoid\n\t// race between processing of a sub where sub is added to account sublist but\n\t// updateSmap has not be called on that \"thread\", while in the LN readloop,\n\t// when processing CONNECT, initLeafNodeSmapAndSendSubs is invoked and add\n\t// this subscription to smap. When processing of the sub then calls updateSmap,\n\t// we would add it a second time in the smap causing later unsub to suppress the LS-.\n\ttsub map[*subscription]struct{}\n\ttsubt *time.Timer\n\t// Selected compression mode, which may be different from the server configured mode.\n\tcompression string\n\t// This is for GW map replies.\n\tgwSub *subscription\n}\n\n// Used for remote (solicited) leafnodes.\ntype leafNodeCfg struct {\n\tsync.RWMutex\n\t*RemoteLeafOpts\n\turls []*url.URL\n\tcurURL *url.URL\n\ttlsName string\n\tusername string\n\tpassword string\n\tperms *Permissions\n\tconnDelay time.Duration // Delay before a connect, could be used while detecting loop condition, etc..\n\tjsMigrateTimer *time.Timer\n}\n\n// Check to see if this is a solicited leafnode. We do special processing for solicited.\nfunc (c *client) isSolicitedLeafNode() bool {\n\treturn c.kind == LEAF && c.leaf.remote != nil\n}\n\n// Returns true if this is a solicited leafnode and is not configured to be treated as a hub or a receiving\n// connection leafnode where the otherside has declared itself to be the hub.\nfunc (c *client) isSpokeLeafNode() bool {\n\treturn c.kind == LEAF && c.leaf.isSpoke\n}\n\nfunc (c *client) isHubLeafNode() bool {\n\treturn c.kind == LEAF && !c.leaf.isSpoke\n}\n\nfunc (c *client) isIsolatedLeafNode() bool {\n\t// TODO(nat): In future we may want to pass in and consider an isolation\n\t// group name here, which the hub and/or leaf could provide, so that we\n\t// can isolate away certain LNs but not others on an opt-in basis. For\n\t// now we will just isolate all LN interest until then.\n\treturn c.kind == LEAF && c.leaf.isolated\n}\n\n// This will spin up go routines to solicit the remote leaf node connections.\nfunc (s *Server) solicitLeafNodeRemotes(remotes []*RemoteLeafOpts) {\n\tsysAccName := _EMPTY_\n\tsAcc := s.SystemAccount()\n\tif sAcc != nil {\n\t\tsysAccName = sAcc.Name\n\t}\n\taddRemote := func(r *RemoteLeafOpts, isSysAccRemote bool) *leafNodeCfg {\n\t\ts.mu.Lock()\n\t\tremote := newLeafNodeCfg(r)\n\t\tcreds := remote.Credentials\n\t\taccName := remote.LocalAccount\n\t\ts.leafRemoteCfgs = append(s.leafRemoteCfgs, remote)\n\t\t// Print notice if\n\t\tif isSysAccRemote {\n\t\t\tif len(remote.DenyExports) > 0 {\n\t\t\t\ts.Noticef(\"Remote for System Account uses restricted export permissions\")\n\t\t\t}\n\t\t\tif len(remote.DenyImports) > 0 {\n\t\t\t\ts.Noticef(\"Remote for System Account uses restricted import permissions\")\n\t\t\t}\n\t\t}\n\t\ts.mu.Unlock()\n\t\tif creds != _EMPTY_ {\n\t\t\tcontents, err := os.ReadFile(creds)\n\t\t\tdefer wipeSlice(contents)\n\t\t\tif err != nil {\n\t\t\t\ts.Errorf(\"Error reading LeafNode Remote Credentials file %q: %v\", creds, err)\n\t\t\t} else if items := credsRe.FindAllSubmatch(contents, -1); len(items) < 2 {\n\t\t\t\ts.Errorf(\"LeafNode Remote Credentials file %q malformed\", creds)\n\t\t\t} else if _, err := nkeys.FromSeed(items[1][1]); err != nil {\n\t\t\t\ts.Errorf(\"LeafNode Remote Credentials file %q has malformed seed\", creds)\n\t\t\t} else if uc, err := jwt.DecodeUserClaims(string(items[0][1])); err != nil {\n\t\t\t\ts.Errorf(\"LeafNode Remote Credentials file %q has malformed user jwt\", creds)\n\t\t\t} else if isSysAccRemote {\n\t\t\t\tif !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil {\n\t\t\t\t\ts.Noticef(\"LeafNode Remote for System Account uses credentials file %q with restricted permissions\", creds)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif !uc.Permissions.Pub.Empty() || !uc.Permissions.Sub.Empty() || uc.Permissions.Resp != nil {\n\t\t\t\t\ts.Noticef(\"LeafNode Remote for Account %s uses credentials file %q with restricted permissions\", accName, creds)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn remote\n\t}\n\tfor _, r := range remotes {\n\t\t// We need to call this, even if the leaf is disabled. This is so that\n\t\t// the number of internal configuration matches the options' remote leaf\n\t\t// configuration required for configuration reload.\n\t\tremote := addRemote(r, r.LocalAccount == sysAccName)\n\t\tif !r.Disabled {\n\t\t\ts.startGoRoutine(func() { s.connectToRemoteLeafNode(remote, true) })\n\t\t}\n\t}\n}\n\nfunc (s *Server) remoteLeafNodeStillValid(remote *leafNodeCfg) bool {\n\tif remote.Disabled {\n\t\treturn false\n\t}\n\tfor _, ri := range s.getOpts().LeafNode.Remotes {\n\t\t// FIXME(dlc) - What about auth changes?\n\t\tif reflect.DeepEqual(ri.URLs, remote.URLs) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Ensure that leafnode is properly configured.\nfunc validateLeafNode(o *Options) error {\n\tif err := validateLeafNodeAuthOptions(o); err != nil {\n\t\treturn err\n\t}\n\n\t// Users can bind to any local account, if its empty we will assume the $G account.\n\tfor _, r := range o.LeafNode.Remotes {\n\t\tif r.LocalAccount == _EMPTY_ {\n\t\t\tr.LocalAccount = globalAccountName\n\t\t}\n\t}\n\n\t// In local config mode, check that leafnode configuration refers to accounts that exist.\n\tif len(o.TrustedOperators) == 0 {\n\t\taccNames := map[string]struct{}{}\n\t\tfor _, a := range o.Accounts {\n\t\t\taccNames[a.Name] = struct{}{}\n\t\t}\n\t\t// global account is always created\n\t\taccNames[DEFAULT_GLOBAL_ACCOUNT] = struct{}{}\n\t\t// in the context of leaf nodes, empty account means global account\n\t\taccNames[_EMPTY_] = struct{}{}\n\t\t// system account either exists or, if not disabled, will be created\n\t\tif o.SystemAccount == _EMPTY_ && !o.NoSystemAccount {\n\t\t\taccNames[DEFAULT_SYSTEM_ACCOUNT] = struct{}{}\n\t\t}\n\t\tcheckAccountExists := func(accName string, cfgType string) error {\n\t\t\tif _, ok := accNames[accName]; !ok {\n\t\t\t\treturn fmt.Errorf(\"cannot find local account %q specified in leafnode %s\", accName, cfgType)\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tif err := checkAccountExists(o.LeafNode.Account, \"authorization\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, lu := range o.LeafNode.Users {\n\t\t\tif lu.Account == nil { // means global account\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := checkAccountExists(lu.Account.Name, \"authorization\"); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tfor _, r := range o.LeafNode.Remotes {\n\t\t\tif err := checkAccountExists(r.LocalAccount, \"remote\"); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif len(o.LeafNode.Users) != 0 {\n\t\t\treturn fmt.Errorf(\"operator mode does not allow specifying users in leafnode config\")\n\t\t}\n\t\tfor _, r := range o.LeafNode.Remotes {\n\t\t\tif !nkeys.IsValidPublicAccountKey(r.LocalAccount) {\n\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\"operator mode requires account nkeys in remotes. \" +\n\t\t\t\t\t\t\"Please add an `account` key to each remote in your `leafnodes` section, to assign it to an account. \" +\n\t\t\t\t\t\t\"Each account value should be a 56 character public key, starting with the letter 'A'\")\n\t\t\t}\n\t\t}\n\t\tif o.LeafNode.Port != 0 && o.LeafNode.Account != \"\" && !nkeys.IsValidPublicAccountKey(o.LeafNode.Account) {\n\t\t\treturn fmt.Errorf(\"operator mode and non account nkeys are incompatible\")\n\t\t}\n\t}\n\n\t// Validate compression settings\n\tif o.LeafNode.Compression.Mode != _EMPTY_ {\n\t\tif err := validateAndNormalizeCompressionOption(&o.LeafNode.Compression, CompressionS2Auto); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// If a remote has a websocket scheme, all need to have it.\n\tfor _, rcfg := range o.LeafNode.Remotes {\n\t\t// Validate proxy configuration\n\t\tif _, err := validateLeafNodeProxyOptions(rcfg); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif len(rcfg.URLs) >= 2 {\n\t\t\tfirstIsWS, ok := isWSURL(rcfg.URLs[0]), true\n\t\t\tfor i := 1; i < len(rcfg.URLs); i++ {\n\t\t\t\tu := rcfg.URLs[i]\n\t\t\t\tif isWS := isWSURL(u); isWS && !firstIsWS || !isWS && firstIsWS {\n\t\t\t\t\tok = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\treturn fmt.Errorf(\"remote leaf node configuration cannot have a mix of websocket and non-websocket urls: %q\", redactURLList(rcfg.URLs))\n\t\t\t}\n\t\t}\n\t\t// Validate compression settings\n\t\tif rcfg.Compression.Mode != _EMPTY_ {\n\t\t\tif err := validateAndNormalizeCompressionOption(&rcfg.Compression, CompressionS2Auto); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tif o.LeafNode.Port == 0 {\n\t\treturn nil\n\t}\n\n\t// If MinVersion is defined, check that it is valid.\n\tif mv := o.LeafNode.MinVersion; mv != _EMPTY_ {\n\t\tif err := checkLeafMinVersionConfig(mv); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// The checks below will be done only when detecting that we are configured\n\t// with gateways. So if an option validation needs to be done regardless,\n\t// it MUST be done before this point!\n\n\tif o.Gateway.Name == _EMPTY_ && o.Gateway.Port == 0 {\n\t\treturn nil\n\t}\n\t// If we are here we have both leaf nodes and gateways defined, make sure there\n\t// is a system account defined.\n\tif o.SystemAccount == _EMPTY_ {\n\t\treturn fmt.Errorf(\"leaf nodes and gateways (both being defined) require a system account to also be configured\")\n\t}\n\tif err := validatePinnedCerts(o.LeafNode.TLSPinnedCerts); err != nil {\n\t\treturn fmt.Errorf(\"leafnode: %v\", err)\n\t}\n\treturn nil\n}\n\nfunc checkLeafMinVersionConfig(mv string) error {\n\tif ok, err := versionAtLeastCheckError(mv, 2, 8, 0); !ok || err != nil {\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid leafnode's minimum version: %v\", err)\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"the minimum version should be at least 2.8.0\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Used to validate user names in LeafNode configuration.\n// - rejects mix of single and multiple users.\n// - rejects duplicate user names.\nfunc validateLeafNodeAuthOptions(o *Options) error {\n\tif len(o.LeafNode.Users) == 0 {\n\t\treturn nil\n\t}\n\tif o.LeafNode.Username != _EMPTY_ {\n\t\treturn fmt.Errorf(\"can not have a single user/pass and a users array\")\n\t}\n\tif o.LeafNode.Nkey != _EMPTY_ {\n\t\treturn fmt.Errorf(\"can not have a single nkey and a users array\")\n\t}\n\tusers := map[string]struct{}{}\n\tfor _, u := range o.LeafNode.Users {\n\t\tif _, exists := users[u.Username]; exists {\n\t\t\treturn fmt.Errorf(\"duplicate user %q detected in leafnode authorization\", u.Username)\n\t\t}\n\t\tusers[u.Username] = struct{}{}\n\t}\n\treturn nil\n}\n\nfunc validateLeafNodeProxyOptions(remote *RemoteLeafOpts) ([]string, error) {\n\tvar warnings []string\n\n\tif remote.Proxy.URL == _EMPTY_ {\n\t\treturn warnings, nil\n\t}\n\n\tproxyURL, err := url.Parse(remote.Proxy.URL)\n\tif err != nil {\n\t\treturn warnings, fmt.Errorf(\"invalid proxy URL: %v\", err)\n\t}\n\n\tif proxyURL.Scheme != \"http\" && proxyURL.Scheme != \"https\" {\n\t\treturn warnings, fmt.Errorf(\"proxy URL scheme must be http or https, got: %s\", proxyURL.Scheme)\n\t}\n\n\tif proxyURL.Host == _EMPTY_ {\n\t\treturn warnings, fmt.Errorf(\"proxy URL must specify a host\")\n\t}\n\n\tif remote.Proxy.Timeout < 0 {\n\t\treturn warnings, fmt.Errorf(\"proxy timeout must be >= 0\")\n\t}\n\n\tif (remote.Proxy.Username == _EMPTY_) != (remote.Proxy.Password == _EMPTY_) {\n\t\treturn warnings, fmt.Errorf(\"proxy username and password must both be specified or both be empty\")\n\t}\n\n\tif len(remote.URLs) > 0 {\n\t\thasWebSocketURL := false\n\t\thasNonWebSocketURL := false\n\n\t\tfor _, remoteURL := range remote.URLs {\n\t\t\tif remoteURL.Scheme == wsSchemePrefix || remoteURL.Scheme == wsSchemePrefixTLS {\n\t\t\t\thasWebSocketURL = true\n\t\t\t\tif (remoteURL.Scheme == wsSchemePrefixTLS) &&\n\t\t\t\t\tremote.TLSConfig == nil && !remote.TLS {\n\t\t\t\t\treturn warnings, fmt.Errorf(\"proxy is configured but remote URL %s requires TLS and no TLS configuration is provided. When using proxy with TLS endpoints, ensure TLS is properly configured for the leafnode remote\", remoteURL.String())\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\thasNonWebSocketURL = true\n\t\t\t}\n\t\t}\n\n\t\tif !hasWebSocketURL {\n\t\t\twarnings = append(warnings, \"proxy configuration will be ignored: proxy settings only apply to WebSocket connections (ws:// or wss://), but all configured URLs use TCP connections (nats://)\")\n\t\t} else if hasNonWebSocketURL {\n\t\t\twarnings = append(warnings, \"proxy configuration will only be used for WebSocket URLs: proxy settings do not apply to TCP connections (nats://)\")\n\t\t}\n\t}\n\n\treturn warnings, nil\n}\n\n// Update remote LeafNode TLS configurations after a config reload.\nfunc (s *Server) updateRemoteLeafNodesTLSConfig(opts *Options) {\n\tmax := len(opts.LeafNode.Remotes)\n\tif max == 0 {\n\t\treturn\n\t}\n\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\n\t// Changes in the list of remote leaf nodes is not supported.\n\t// However, make sure that we don't go over the arrays.\n\tif len(s.leafRemoteCfgs) < max {\n\t\tmax = len(s.leafRemoteCfgs)\n\t}\n\tfor i := 0; i < max; i++ {\n\t\tro := opts.LeafNode.Remotes[i]\n\t\tcfg := s.leafRemoteCfgs[i]\n\t\tif ro.TLSConfig != nil {\n\t\t\tcfg.Lock()\n\t\t\tcfg.TLSConfig = ro.TLSConfig.Clone()\n\t\t\tcfg.TLSHandshakeFirst = ro.TLSHandshakeFirst\n\t\t\tcfg.Unlock()\n\t\t}\n\t}\n}\n\nfunc (s *Server) reConnectToRemoteLeafNode(remote *leafNodeCfg) {\n\tdelay := s.getOpts().LeafNode.ReconnectInterval\n\tselect {\n\tcase <-time.After(delay):\n\tcase <-s.quitCh:\n\t\ts.grWG.Done()\n\t\treturn\n\t}\n\ts.connectToRemoteLeafNode(remote, false)\n}\n\n// Creates a leafNodeCfg object that wraps the RemoteLeafOpts.\nfunc newLeafNodeCfg(remote *RemoteLeafOpts) *leafNodeCfg {\n\tcfg := &leafNodeCfg{\n\t\tRemoteLeafOpts: remote,\n\t\turls: make([]*url.URL, 0, len(remote.URLs)),\n\t}\n\tif len(remote.DenyExports) > 0 || len(remote.DenyImports) > 0 {\n\t\tperms := &Permissions{}\n\t\tif len(remote.DenyExports) > 0 {\n\t\t\tperms.Publish = &SubjectPermission{Deny: remote.DenyExports}\n\t\t}\n\t\tif len(remote.DenyImports) > 0 {\n\t\t\tperms.Subscribe = &SubjectPermission{Deny: remote.DenyImports}\n\t\t}\n\t\tcfg.perms = perms\n\t}\n\t// Start with the one that is configured. We will add to this\n\t// array when receiving async leafnode INFOs.\n\tcfg.urls = append(cfg.urls, cfg.URLs...)\n\t// If allowed to randomize, do it on our copy of URLs\n\tif !remote.NoRandomize {\n\t\trand.Shuffle(len(cfg.urls), func(i, j int) {\n\t\t\tcfg.urls[i], cfg.urls[j] = cfg.urls[j], cfg.urls[i]\n\t\t})\n\t}\n\t// If we are TLS make sure we save off a proper servername if possible.\n\t// Do same for user/password since we may need them to connect to\n\t// a bare URL that we get from INFO protocol.\n\tfor _, u := range cfg.urls {\n\t\tcfg.saveTLSHostname(u)\n\t\tcfg.saveUserPassword(u)\n\t\t// If the url(s) have the \"wss://\" scheme, and we don't have a TLS\n\t\t// config, mark that we should be using TLS anyway.\n\t\tif !cfg.TLS && isWSSURL(u) {\n\t\t\tcfg.TLS = true\n\t\t}\n\t}\n\treturn cfg\n}\n\n// Will pick an URL from the list of available URLs.\nfunc (cfg *leafNodeCfg) pickNextURL() *url.URL {\n\tcfg.Lock()\n\tdefer cfg.Unlock()\n\t// If the current URL is the first in the list and we have more than\n\t// one URL, then move that one to end of the list.\n\tif cfg.curURL != nil && len(cfg.urls) > 1 && urlsAreEqual(cfg.curURL, cfg.urls[0]) {\n\t\tfirst := cfg.urls[0]\n\t\tcopy(cfg.urls, cfg.urls[1:])\n\t\tcfg.urls[len(cfg.urls)-1] = first\n\t}\n\tcfg.curURL = cfg.urls[0]\n\treturn cfg.curURL\n}\n\n// Returns the current URL\nfunc (cfg *leafNodeCfg) getCurrentURL() *url.URL {\n\tcfg.RLock()\n\tdefer cfg.RUnlock()\n\treturn cfg.curURL\n}\n\n// Returns how long the server should wait before attempting\n// to solicit a remote leafnode connection.\nfunc (cfg *leafNodeCfg) getConnectDelay() time.Duration {\n\tcfg.RLock()\n\tdelay := cfg.connDelay\n\tcfg.RUnlock()\n\treturn delay\n}\n\n// Sets the connect delay.\nfunc (cfg *leafNodeCfg) setConnectDelay(delay time.Duration) {\n\tcfg.Lock()\n\tcfg.connDelay = delay\n\tcfg.Unlock()\n}\n\n// Ensure that non-exported options (used in tests) have\n// been properly set.\nfunc (s *Server) setLeafNodeNonExportedOptions() {\n\topts := s.getOpts()\n\ts.leafNodeOpts.dialTimeout = opts.LeafNode.dialTimeout\n\tif s.leafNodeOpts.dialTimeout == 0 {\n\t\t// Use same timeouts as routes for now.\n\t\ts.leafNodeOpts.dialTimeout = DEFAULT_ROUTE_DIAL\n\t}\n\ts.leafNodeOpts.resolver = opts.LeafNode.resolver\n\tif s.leafNodeOpts.resolver == nil {\n\t\ts.leafNodeOpts.resolver = net.DefaultResolver\n\t}\n}\n\nconst sharedSysAccDelay = 250 * time.Millisecond\n\n// establishHTTPProxyTunnel establishes an HTTP CONNECT tunnel through a proxy server\nfunc establishHTTPProxyTunnel(proxyURL, targetHost string, timeout time.Duration, username, password string) (net.Conn, error) {\n\tproxyAddr, err := url.Parse(proxyURL)\n\tif err != nil {\n\t\t// This should not happen since proxy URL is validated during configuration parsing\n\t\treturn nil, fmt.Errorf(\"unexpected proxy URL parse error (URL was pre-validated): %v\", err)\n\t}\n\n\t// Connect to the proxy server\n\tconn, err := natsDialTimeout(\"tcp\", proxyAddr.Host, timeout)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to connect to proxy: %v\", err)\n\t}\n\n\t// Set deadline for the entire proxy handshake\n\tif err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {\n\t\tconn.Close()\n\t\treturn nil, fmt.Errorf(\"failed to set deadline: %v\", err)\n\t}\n\n\treq := &http.Request{\n\t\tMethod: http.MethodConnect,\n\t\tURL: &url.URL{Opaque: targetHost}, // Opaque is required for CONNECT\n\t\tHost: targetHost,\n\t\tHeader: make(http.Header),\n\t}\n\n\t// Add proxy authentication if provided\n\tif username != \"\" && password != \"\" {\n\t\treq.Header.Set(\"Proxy-Authorization\", \"Basic \"+base64.StdEncoding.EncodeToString([]byte(username+\":\"+password)))\n\t}\n\n\tif err := req.Write(conn); err != nil {\n\t\tconn.Close()\n\t\treturn nil, fmt.Errorf(\"failed to write CONNECT request: %v\", err)\n\t}\n\n\tresp, err := http.ReadResponse(bufio.NewReader(conn), req)\n\tif err != nil {\n\t\tconn.Close()\n\t\treturn nil, fmt.Errorf(\"failed to read proxy response: %v\", err)\n\t}\n\n\tif resp.StatusCode != http.StatusOK {\n\t\tresp.Body.Close()\n\t\tconn.Close()\n\t\treturn nil, fmt.Errorf(\"proxy CONNECT failed: %s\", resp.Status)\n\t}\n\n\t// Close the response body\n\tresp.Body.Close()\n\n\t// Clear the deadline now that we've finished the proxy handshake\n\tif err := conn.SetDeadline(time.Time{}); err != nil {\n\t\tconn.Close()\n\t\treturn nil, fmt.Errorf(\"failed to clear deadline: %v\", err)\n\t}\n\n\treturn conn, nil\n}\n\nfunc (s *Server) connectToRemoteLeafNode(remote *leafNodeCfg, firstConnect bool) {\n\tdefer s.grWG.Done()\n\n\tif remote == nil || len(remote.URLs) == 0 {\n\t\ts.Debugf(\"Empty remote leafnode definition, nothing to connect\")\n\t\treturn\n\t}\n\n\topts := s.getOpts()\n\treconnectDelay := opts.LeafNode.ReconnectInterval\n\ts.mu.RLock()\n\tdialTimeout := s.leafNodeOpts.dialTimeout\n\tresolver := s.leafNodeOpts.resolver\n\tvar isSysAcc bool\n\tif s.eventsEnabled() {\n\t\tisSysAcc = remote.LocalAccount == s.sys.account.Name\n\t}\n\tjetstreamMigrateDelay := remote.JetStreamClusterMigrateDelay\n\ts.mu.RUnlock()\n\n\t// If we are sharing a system account and we are not standalone delay to gather some info prior.\n\tif firstConnect && isSysAcc && !s.standAloneMode() {\n\t\ts.Debugf(\"Will delay first leafnode connect to shared system account due to clustering\")\n\t\tremote.setConnectDelay(sharedSysAccDelay)\n\t}\n\n\tif connDelay := remote.getConnectDelay(); connDelay > 0 {\n\t\tselect {\n\t\tcase <-time.After(connDelay):\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t\tremote.setConnectDelay(0)\n\t}\n\n\tvar conn net.Conn\n\n\tconst connErrFmt = \"Error trying to connect as leafnode to remote server %q (attempt %v): %v\"\n\n\t// Capture proxy configuration once before the loop with proper locking\n\tremote.RLock()\n\tproxyURL := remote.Proxy.URL\n\tproxyUsername := remote.Proxy.Username\n\tproxyPassword := remote.Proxy.Password\n\tproxyTimeout := remote.Proxy.Timeout\n\tremote.RUnlock()\n\n\t// Set default proxy timeout if not specified\n\tif proxyTimeout == 0 {\n\t\tproxyTimeout = dialTimeout\n\t}\n\n\tattempts := 0\n\n\tfor s.isRunning() && s.remoteLeafNodeStillValid(remote) {\n\t\trURL := remote.pickNextURL()\n\t\turl, err := s.getRandomIP(resolver, rURL.Host, nil)\n\t\tif err == nil {\n\t\t\tvar ipStr string\n\t\t\tif url != rURL.Host {\n\t\t\t\tipStr = fmt.Sprintf(\" (%s)\", url)\n\t\t\t}\n\t\t\t// Some test may want to disable remotes from connecting\n\t\t\tif s.isLeafConnectDisabled() {\n\t\t\t\ts.Debugf(\"Will not attempt to connect to remote server on %q%s, leafnodes currently disabled\", rURL.Host, ipStr)\n\t\t\t\terr = ErrLeafNodeDisabled\n\t\t\t} else {\n\t\t\t\ts.Debugf(\"Trying to connect as leafnode to remote server on %q%s\", rURL.Host, ipStr)\n\n\t\t\t\t// Check if proxy is configured\n\t\t\t\tif proxyURL != _EMPTY_ {\n\t\t\t\t\ttargetHost := rURL.Host\n\t\t\t\t\t// If URL doesn't include port, add the default port for the scheme\n\t\t\t\t\tif rURL.Port() == _EMPTY_ {\n\t\t\t\t\t\tdefaultPort := \"80\"\n\t\t\t\t\t\tif rURL.Scheme == wsSchemePrefixTLS {\n\t\t\t\t\t\t\tdefaultPort = \"443\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttargetHost = net.JoinHostPort(rURL.Hostname(), defaultPort)\n\t\t\t\t\t}\n\n\t\t\t\t\tconn, err = establishHTTPProxyTunnel(proxyURL, targetHost, proxyTimeout, proxyUsername, proxyPassword)\n\t\t\t\t} else {\n\t\t\t\t\t// Direct connection\n\t\t\t\t\tconn, err = natsDialTimeout(\"tcp\", url, dialTimeout)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\tjitter := time.Duration(rand.Int63n(int64(reconnectDelay)))\n\t\t\tdelay := reconnectDelay + jitter\n\t\t\tattempts++\n\t\t\tif s.shouldReportConnectErr(firstConnect, attempts) {\n\t\t\t\ts.Errorf(connErrFmt, rURL.Host, attempts, err)\n\t\t\t} else {\n\t\t\t\ts.Debugf(connErrFmt, rURL.Host, attempts, err)\n\t\t\t}\n\t\t\tremote.Lock()\n\t\t\t// if we are using a delay to start migrating assets, kick off a migrate timer.\n\t\t\tif remote.jsMigrateTimer == nil && jetstreamMigrateDelay > 0 {\n\t\t\t\tremote.jsMigrateTimer = time.AfterFunc(jetstreamMigrateDelay, func() {\n\t\t\t\t\ts.checkJetStreamMigrate(remote)\n\t\t\t\t})\n\t\t\t}\n\t\t\tremote.Unlock()\n\t\t\tselect {\n\t\t\tcase <-s.quitCh:\n\t\t\t\tremote.cancelMigrateTimer()\n\t\t\t\treturn\n\t\t\tcase <-time.After(delay):\n\t\t\t\t// Check if we should migrate any JetStream assets immediately while this remote is down.\n\t\t\t\t// This will be used if JetStreamClusterMigrateDelay was not set\n\t\t\t\tif jetstreamMigrateDelay == 0 {\n\t\t\t\t\ts.checkJetStreamMigrate(remote)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tremote.cancelMigrateTimer()\n\t\tif !s.remoteLeafNodeStillValid(remote) {\n\t\t\tconn.Close()\n\t\t\treturn\n\t\t}\n\n\t\t// We have a connection here to a remote server.\n\t\t// Go ahead and create our leaf node and return.\n\t\ts.createLeafNode(conn, rURL, remote, nil)\n\n\t\t// Clear any observer states if we had them.\n\t\ts.clearObserverState(remote)\n\n\t\treturn\n\t}\n}\n\nfunc (cfg *leafNodeCfg) cancelMigrateTimer() {\n\tcfg.Lock()\n\tstopAndClearTimer(&cfg.jsMigrateTimer)\n\tcfg.Unlock()\n}\n\n// This will clear any observer state such that stream or consumer assets on this server can become leaders again.\nfunc (s *Server) clearObserverState(remote *leafNodeCfg) {\n\ts.mu.RLock()\n\taccName := remote.LocalAccount\n\ts.mu.RUnlock()\n\n\tacc, err := s.LookupAccount(accName)\n\tif err != nil {\n\t\ts.Warnf(\"Error looking up account [%s] checking for JetStream clear observer state on a leafnode\", accName)\n\t\treturn\n\t}\n\n\tacc.jscmMu.Lock()\n\tdefer acc.jscmMu.Unlock()\n\n\t// Walk all streams looking for any clustered stream, skip otherwise.\n\tfor _, mset := range acc.streams() {\n\t\tnode := mset.raftNode()\n\t\tif node == nil {\n\t\t\t// Not R>1\n\t\t\tcontinue\n\t\t}\n\t\t// Check consumers\n\t\tfor _, o := range mset.getConsumers() {\n\t\t\tif n := o.raftNode(); n != nil {\n\t\t\t\t// Ensure we can become a leader again.\n\t\t\t\tn.SetObserver(false)\n\t\t\t}\n\t\t}\n\t\t// Ensure we can not become a leader again.\n\t\tnode.SetObserver(false)\n\t}\n}\n\n// Check to see if we should migrate any assets from this account.\nfunc (s *Server) checkJetStreamMigrate(remote *leafNodeCfg) {\n\ts.mu.RLock()\n\taccName, shouldMigrate := remote.LocalAccount, remote.JetStreamClusterMigrate\n\ts.mu.RUnlock()\n\n\tif !shouldMigrate {\n\t\treturn\n\t}\n\n\tacc, err := s.LookupAccount(accName)\n\tif err != nil {\n\t\ts.Warnf(\"Error looking up account [%s] checking for JetStream migration on a leafnode\", accName)\n\t\treturn\n\t}\n\n\tacc.jscmMu.Lock()\n\tdefer acc.jscmMu.Unlock()\n\n\t// Walk all streams looking for any clustered stream, skip otherwise.\n\t// If we are the leader force stepdown.\n\tfor _, mset := range acc.streams() {\n\t\tnode := mset.raftNode()\n\t\tif node == nil {\n\t\t\t// Not R>1\n\t\t\tcontinue\n\t\t}\n\t\t// Collect any consumers\n\t\tfor _, o := range mset.getConsumers() {\n\t\t\tif n := o.raftNode(); n != nil {\n\t\t\t\tn.StepDown()\n\t\t\t\t// Ensure we can not become a leader while in this state.\n\t\t\t\tn.SetObserver(true)\n\t\t\t}\n\t\t}\n\t\t// Stepdown if this stream was leader.\n\t\tnode.StepDown()\n\t\t// Ensure we can not become a leader while in this state.\n\t\tnode.SetObserver(true)\n\t}\n}\n\n// Helper for checking.\nfunc (s *Server) isLeafConnectDisabled() bool {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.leafDisableConnect\n}\n\n// Save off the tlsName for when we use TLS and mix hostnames and IPs. IPs usually\n// come from the server we connect to.\n//\n// We used to save the name only if there was a TLSConfig or scheme equal to \"tls\".\n// However, this was causing failures for users that did not set the scheme (and\n// their remote connections did not have a tls{} block).\n// We now save the host name regardless in case the remote returns an INFO indicating\n// that TLS is required.\nfunc (cfg *leafNodeCfg) saveTLSHostname(u *url.URL) {\n\tif cfg.tlsName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil {\n\t\tcfg.tlsName = u.Hostname()\n\t}\n}\n\n// Save off the username/password for when we connect using a bare URL\n// that we get from the INFO protocol.\nfunc (cfg *leafNodeCfg) saveUserPassword(u *url.URL) {\n\tif cfg.username == _EMPTY_ && u.User != nil {\n\t\tcfg.username = u.User.Username()\n\t\tcfg.password, _ = u.User.Password()\n\t}\n}\n\n// This starts the leafnode accept loop in a go routine, unless it\n// is detected that the server has already been shutdown.\nfunc (s *Server) startLeafNodeAcceptLoop() {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tport := opts.LeafNode.Port\n\tif port == -1 {\n\t\tport = 0\n\t}\n\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\ts.mu.Lock()\n\thp := net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(port))\n\tl, e := natsListen(\"tcp\", hp)\n\ts.leafNodeListenerErr = e\n\tif e != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Error listening on leafnode port: %d - %v\", opts.LeafNode.Port, e)\n\t\treturn\n\t}\n\n\ts.Noticef(\"Listening for leafnode connections on %s\",\n\t\tnet.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port)))\n\n\ttlsRequired := opts.LeafNode.TLSConfig != nil\n\ttlsVerify := tlsRequired && opts.LeafNode.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert\n\t// Do not set compression in this Info object, it would possibly cause\n\t// issues when sending asynchronous INFO to the remote.\n\tinfo := Info{\n\t\tID: s.info.ID,\n\t\tName: s.info.Name,\n\t\tVersion: s.info.Version,\n\t\tGitCommit: gitCommit,\n\t\tGoVersion: runtime.Version(),\n\t\tAuthRequired: true,\n\t\tTLSRequired: tlsRequired,\n\t\tTLSVerify: tlsVerify,\n\t\tMaxPayload: s.info.MaxPayload, // TODO(dlc) - Allow override?\n\t\tHeaders: s.supportsHeaders(),\n\t\tJetStream: opts.JetStream,\n\t\tDomain: opts.JetStreamDomain,\n\t\tProto: s.getServerProto(),\n\t\tInfoOnConnect: true,\n\t\tJSApiLevel: JSApiLevel,\n\t}\n\t// If we have selected a random port...\n\tif port == 0 {\n\t\t// Write resolved port back to options.\n\t\topts.LeafNode.Port = l.Addr().(*net.TCPAddr).Port\n\t}\n\n\ts.leafNodeInfo = info\n\t// Possibly override Host/Port and set IP based on Cluster.Advertise\n\tif err := s.setLeafNodeInfoHostPortAndIP(); err != nil {\n\t\ts.Fatalf(\"Error setting leafnode INFO with LeafNode.Advertise value of %s, err=%v\", opts.LeafNode.Advertise, err)\n\t\tl.Close()\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\ts.leafURLsMap[s.leafNodeInfo.IP]++\n\ts.generateLeafNodeInfoJSON()\n\n\t// Setup state that can enable shutdown\n\ts.leafNodeListener = l\n\n\t// As of now, a server that does not have remotes configured would\n\t// never solicit a connection, so we should not have to warn if\n\t// InsecureSkipVerify is set in main LeafNodes config (since\n\t// this TLS setting matters only when soliciting a connection).\n\t// Still, warn if insecure is set in any of LeafNode block.\n\t// We need to check remotes, even if tls is not required on accept.\n\twarn := tlsRequired && opts.LeafNode.TLSConfig.InsecureSkipVerify\n\tif !warn {\n\t\tfor _, r := range opts.LeafNode.Remotes {\n\t\t\tif r.TLSConfig != nil && r.TLSConfig.InsecureSkipVerify {\n\t\t\t\twarn = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif warn {\n\t\ts.Warnf(leafnodeTLSInsecureWarning)\n\t}\n\tgo s.acceptConnections(l, \"Leafnode\", func(conn net.Conn) { s.createLeafNode(conn, nil, nil, nil) }, nil)\n\ts.mu.Unlock()\n}\n\n// RegEx to match a creds file with user JWT and Seed.\nvar credsRe = regexp.MustCompile(`\\s*(?:(?:[-]{3,}.*[-]{3,}\\r?\\n)([\\w\\-.=]+)(?:\\r?\\n[-]{3,}.*[-]{3,}(\\r?\\n|\\z)))`)\n\n// clusterName is provided as argument to avoid lock ordering issues with the locked client c\n// Lock should be held entering here.\nfunc (c *client) sendLeafConnect(clusterName string, headers bool) error {\n\t// We support basic user/pass and operator based user JWT with signatures.\n\tcinfo := leafConnectInfo{\n\t\tVersion: VERSION,\n\t\tID: c.srv.info.ID,\n\t\tDomain: c.srv.info.Domain,\n\t\tName: c.srv.info.Name,\n\t\tHub: c.leaf.remote.Hub,\n\t\tCluster: clusterName,\n\t\tHeaders: headers,\n\t\tJetStream: c.acc.jetStreamConfigured(),\n\t\tDenyPub: c.leaf.remote.DenyImports,\n\t\tCompression: c.leaf.compression,\n\t\tRemoteAccount: c.acc.GetName(),\n\t\tProto: c.srv.getServerProto(),\n\t\tIsolate: c.leaf.remote.RequestIsolation,\n\t}\n\n\t// If a signature callback is specified, this takes precedence over anything else.\n\tif cb := c.leaf.remote.SignatureCB; cb != nil {\n\t\tnonce := c.nonce\n\t\tc.mu.Unlock()\n\t\tjwt, sigraw, err := cb(nonce)\n\t\tc.mu.Lock()\n\t\tif err == nil && c.isClosed() {\n\t\t\terr = ErrConnectionClosed\n\t\t}\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Error signing the nonce: %v\", err)\n\t\t\treturn err\n\t\t}\n\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\t\tcinfo.JWT, cinfo.Sig = jwt, sig\n\n\t} else if creds := c.leaf.remote.Credentials; creds != _EMPTY_ {\n\t\t// Check for credentials first, that will take precedence..\n\t\tc.Debugf(\"Authenticating with credentials file %q\", c.leaf.remote.Credentials)\n\t\tcontents, err := os.ReadFile(creds)\n\t\tif err != nil {\n\t\t\tc.Errorf(\"%v\", err)\n\t\t\treturn err\n\t\t}\n\t\tdefer wipeSlice(contents)\n\t\titems := credsRe.FindAllSubmatch(contents, -1)\n\t\tif len(items) < 2 {\n\t\t\tc.Errorf(\"Credentials file malformed\")\n\t\t\treturn err\n\t\t}\n\t\t// First result should be the user JWT.\n\t\t// We copy here so that the file containing the seed will be wiped appropriately.\n\t\traw := items[0][1]\n\t\ttmp := make([]byte, len(raw))\n\t\tcopy(tmp, raw)\n\t\t// Seed is second item.\n\t\tkp, err := nkeys.FromSeed(items[1][1])\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Credentials file has malformed seed\")\n\t\t\treturn err\n\t\t}\n\t\t// Wipe our key on exit.\n\t\tdefer kp.Wipe()\n\n\t\tsigraw, _ := kp.Sign(c.nonce)\n\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\t\tcinfo.JWT = bytesToString(tmp)\n\t\tcinfo.Sig = sig\n\t} else if nkey := c.leaf.remote.Nkey; nkey != _EMPTY_ {\n\t\tkp, err := nkeys.FromSeed([]byte(nkey))\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Remote nkey has malformed seed\")\n\t\t\treturn err\n\t\t}\n\t\t// Wipe our key on exit.\n\t\tdefer kp.Wipe()\n\t\tsigraw, _ := kp.Sign(c.nonce)\n\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\t\tpkey, _ := kp.PublicKey()\n\t\tcinfo.Nkey = pkey\n\t\tcinfo.Sig = sig\n\t}\n\t// In addition, and this is to allow auth callout, set user/password or\n\t// token if applicable.\n\tif userInfo := c.leaf.remote.curURL.User; userInfo != nil {\n\t\tcinfo.User = userInfo.Username()\n\t\tvar ok bool\n\t\tcinfo.Pass, ok = userInfo.Password()\n\t\t// For backward compatibility, if only username is provided, set both\n\t\t// Token and User, not just Token.\n\t\tif !ok {\n\t\t\tcinfo.Token = cinfo.User\n\t\t}\n\t} else if c.leaf.remote.username != _EMPTY_ {\n\t\tcinfo.User = c.leaf.remote.username\n\t\tcinfo.Pass = c.leaf.remote.password\n\t\t// For backward compatibility, if only username is provided, set both\n\t\t// Token and User, not just Token.\n\t\tif cinfo.Pass == _EMPTY_ {\n\t\t\tcinfo.Token = cinfo.User\n\t\t}\n\t}\n\tb, err := json.Marshal(cinfo)\n\tif err != nil {\n\t\tc.Errorf(\"Error marshaling CONNECT to remote leafnode: %v\\n\", err)\n\t\treturn err\n\t}\n\t// Although this call is made before the writeLoop is created,\n\t// we don't really need to send in place. The protocol will be\n\t// sent out by the writeLoop.\n\tc.enqueueProto([]byte(fmt.Sprintf(ConProto, b)))\n\treturn nil\n}\n\n// Makes a deep copy of the LeafNode Info structure.\n// The server lock is held on entry.\nfunc (s *Server) copyLeafNodeInfo() *Info {\n\tclone := s.leafNodeInfo\n\t// Copy the array of urls.\n\tif len(s.leafNodeInfo.LeafNodeURLs) > 0 {\n\t\tclone.LeafNodeURLs = append([]string(nil), s.leafNodeInfo.LeafNodeURLs...)\n\t}\n\treturn &clone\n}\n\n// Adds a LeafNode URL that we get when a route connects to the Info structure.\n// Regenerates the JSON byte array so that it can be sent to LeafNode connections.\n// Returns a boolean indicating if the URL was added or not.\n// Server lock is held on entry\nfunc (s *Server) addLeafNodeURL(urlStr string) bool {\n\tif s.leafURLsMap.addUrl(urlStr) {\n\t\ts.generateLeafNodeInfoJSON()\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Removes a LeafNode URL of the route that is disconnecting from the Info structure.\n// Regenerates the JSON byte array so that it can be sent to LeafNode connections.\n// Returns a boolean indicating if the URL was removed or not.\n// Server lock is held on entry.\nfunc (s *Server) removeLeafNodeURL(urlStr string) bool {\n\t// Don't need to do this if we are removing the route connection because\n\t// we are shuting down...\n\tif s.isShuttingDown() {\n\t\treturn false\n\t}\n\tif s.leafURLsMap.removeUrl(urlStr) {\n\t\ts.generateLeafNodeInfoJSON()\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Server lock is held on entry\nfunc (s *Server) generateLeafNodeInfoJSON() {\n\ts.leafNodeInfo.Cluster = s.cachedClusterName()\n\ts.leafNodeInfo.LeafNodeURLs = s.leafURLsMap.getAsStringSlice()\n\ts.leafNodeInfo.WSConnectURLs = s.websocket.connectURLsMap.getAsStringSlice()\n\ts.leafNodeInfoJSON = generateInfoJSON(&s.leafNodeInfo)\n}\n\n// Sends an async INFO protocol so that the connected servers can update\n// their list of LeafNode urls.\nfunc (s *Server) sendAsyncLeafNodeInfo() {\n\tfor _, c := range s.leafs {\n\t\tc.mu.Lock()\n\t\tc.enqueueProto(s.leafNodeInfoJSON)\n\t\tc.mu.Unlock()\n\t}\n}\n\n// Called when an inbound leafnode connection is accepted or we create one for a solicited leafnode.\nfunc (s *Server) createLeafNode(conn net.Conn, rURL *url.URL, remote *leafNodeCfg, ws *websocket) *client {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tmaxPay := int32(opts.MaxPayload)\n\tmaxSubs := int32(opts.MaxSubs)\n\t// For system, maxSubs of 0 means unlimited, so re-adjust here.\n\tif maxSubs == 0 {\n\t\tmaxSubs = -1\n\t}\n\tnow := time.Now().UTC()\n\n\tc := &client{srv: s, nc: conn, kind: LEAF, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now}\n\t// Do not update the smap here, we need to do it in initLeafNodeSmapAndSendSubs\n\tc.leaf = &leaf{}\n\n\t// If the leafnode subject interest should be isolated, flag it here.\n\ts.optsMu.RLock()\n\tif c.leaf.isolated = s.opts.LeafNode.IsolateLeafnodeInterest; !c.leaf.isolated && remote != nil {\n\t\tc.leaf.isolated = remote.LocalIsolation\n\t}\n\ts.optsMu.RUnlock()\n\n\t// For accepted LN connections, ws will be != nil if it was accepted\n\t// through the Websocket port.\n\tc.ws = ws\n\n\t// For remote, check if the scheme starts with \"ws\", if so, we will initiate\n\t// a remote Leaf Node connection as a websocket connection.\n\tif remote != nil && rURL != nil && isWSURL(rURL) {\n\t\tremote.RLock()\n\t\tc.ws = &websocket{compress: remote.Websocket.Compression, maskwrite: !remote.Websocket.NoMasking}\n\t\tremote.RUnlock()\n\t}\n\n\t// Determines if we are soliciting the connection or not.\n\tvar solicited bool\n\tvar acc *Account\n\tvar remoteSuffix string\n\tif remote != nil {\n\t\t// For now, if lookup fails, we will constantly try\n\t\t// to recreate this LN connection.\n\t\tlacc := remote.LocalAccount\n\t\tvar err error\n\t\tacc, err = s.LookupAccount(lacc)\n\t\tif err != nil {\n\t\t\t// An account not existing is something that can happen with nats/http account resolver and the account\n\t\t\t// has not yet been pushed, or the request failed for other reasons.\n\t\t\t// remote needs to be set or retry won't happen\n\t\t\tc.leaf.remote = remote\n\t\t\tc.closeConnection(MissingAccount)\n\t\t\ts.Errorf(\"Unable to lookup account %s for solicited leafnode connection: %v\", lacc, err)\n\t\t\treturn nil\n\t\t}\n\t\tremoteSuffix = fmt.Sprintf(\" for account: %s\", acc.traceLabel())\n\t}\n\n\tc.mu.Lock()\n\tc.initClient()\n\tc.Noticef(\"Leafnode connection created%s %s\", remoteSuffix, c.opts.Name)\n\n\tvar (\n\t\ttlsFirst bool\n\t\ttlsFirstFallback time.Duration\n\t\tinfoTimeout time.Duration\n\t)\n\tif remote != nil {\n\t\tsolicited = true\n\t\tremote.Lock()\n\t\tc.leaf.remote = remote\n\t\tc.setPermissions(remote.perms)\n\t\tif !c.leaf.remote.Hub {\n\t\t\tc.leaf.isSpoke = true\n\t\t}\n\t\ttlsFirst = remote.TLSHandshakeFirst\n\t\tinfoTimeout = remote.FirstInfoTimeout\n\t\tremote.Unlock()\n\t\tc.acc = acc\n\t} else {\n\t\tc.flags.set(expectConnect)\n\t\tif ws != nil {\n\t\t\tc.Debugf(\"Leafnode compression=%v\", c.ws.compress)\n\t\t}\n\t\ttlsFirst = opts.LeafNode.TLSHandshakeFirst\n\t\tif f := opts.LeafNode.TLSHandshakeFirstFallback; f > 0 {\n\t\t\ttlsFirstFallback = f\n\t\t}\n\t}\n\tc.mu.Unlock()\n\n\tvar nonce [nonceLen]byte\n\tvar info *Info\n\n\t// Grab this before the client lock below.\n\tif !solicited {\n\t\t// Grab server variables\n\t\ts.mu.Lock()\n\t\tinfo = s.copyLeafNodeInfo()\n\t\t// For tests that want to simulate old servers, do not set the compression\n\t\t// on the INFO protocol if configured with CompressionNotSupported.\n\t\tif cm := opts.LeafNode.Compression.Mode; cm != CompressionNotSupported {\n\t\t\tinfo.Compression = cm\n\t\t}\n\t\t// We always send a nonce for LEAF connections. Do not change that without\n\t\t// taking into account presence of proxy trusted keys.\n\t\ts.generateNonce(nonce[:])\n\t\ts.mu.Unlock()\n\t}\n\n\t// Grab lock\n\tc.mu.Lock()\n\n\tvar preBuf []byte\n\tif solicited {\n\t\t// For websocket connection, we need to send an HTTP request,\n\t\t// and get the response before starting the readLoop to get\n\t\t// the INFO, etc..\n\t\tif c.isWebsocket() {\n\t\t\tvar err error\n\t\t\tvar closeReason ClosedState\n\n\t\t\tpreBuf, closeReason, err = c.leafNodeSolicitWSConnection(opts, rURL, remote)\n\t\t\tif err != nil {\n\t\t\t\tc.Errorf(\"Error soliciting websocket connection: %v\", err)\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tif closeReason != 0 {\n\t\t\t\t\tc.closeConnection(closeReason)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t} else {\n\t\t\t// If configured to do TLS handshake first\n\t\t\tif tlsFirst {\n\t\t\t\tif _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil {\n\t\t\t\t\tc.mu.Unlock()\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We need to wait for the info, but not for too long.\n\t\t\tc.nc.SetReadDeadline(time.Now().Add(infoTimeout))\n\t\t}\n\n\t\t// We will process the INFO from the readloop and finish by\n\t\t// sending the CONNECT and finish registration later.\n\t} else {\n\t\t// Send our info to the other side.\n\t\t// Remember the nonce we sent here for signatures, etc.\n\t\tc.nonce = make([]byte, nonceLen)\n\t\tcopy(c.nonce, nonce[:])\n\t\tinfo.Nonce = bytesToString(c.nonce)\n\t\tinfo.CID = c.cid\n\t\tproto := generateInfoJSON(info)\n\n\t\tvar pre []byte\n\t\t// We need first to check for \"TLS First\" fallback delay.\n\t\tif tlsFirstFallback > 0 {\n\t\t\t// We wait and see if we are getting any data. Since we did not send\n\t\t\t// the INFO protocol yet, only clients that use TLS first should be\n\t\t\t// sending data (the TLS handshake). We don't really check the content:\n\t\t\t// if it is a rogue agent and not an actual client performing the\n\t\t\t// TLS handshake, the error will be detected when performing the\n\t\t\t// handshake on our side.\n\t\t\tpre = make([]byte, 4)\n\t\t\tc.nc.SetReadDeadline(time.Now().Add(tlsFirstFallback))\n\t\t\tn, _ := io.ReadFull(c.nc, pre[:])\n\t\t\tc.nc.SetReadDeadline(time.Time{})\n\t\t\t// If we get any data (regardless of possible timeout), we will proceed\n\t\t\t// with the TLS handshake.\n\t\t\tif n > 0 {\n\t\t\t\tpre = pre[:n]\n\t\t\t} else {\n\t\t\t\t// We did not get anything so we will send the INFO protocol.\n\t\t\t\tpre = nil\n\t\t\t\t// Set the boolean to false for the rest of the function.\n\t\t\t\ttlsFirst = false\n\t\t\t}\n\t\t}\n\n\t\tif !tlsFirst {\n\t\t\t// We have to send from this go routine because we may\n\t\t\t// have to block for TLS handshake before we start our\n\t\t\t// writeLoop go routine. The other side needs to receive\n\t\t\t// this before it can initiate the TLS handshake..\n\t\t\tc.sendProtoNow(proto)\n\n\t\t\t// The above call could have marked the connection as closed (due to TCP error).\n\t\t\tif c.isClosed() {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.closeConnection(WriteError)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\t// Check to see if we need to spin up TLS.\n\t\tif !c.isWebsocket() && info.TLSRequired {\n\t\t\t// If we have a prebuffer create a multi-reader.\n\t\t\tif len(pre) > 0 {\n\t\t\t\tc.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)}\n\t\t\t}\n\t\t\t// Perform server-side TLS handshake.\n\t\t\tif err := c.doTLSServerHandshake(tlsHandshakeLeaf, opts.LeafNode.TLSConfig, opts.LeafNode.TLSTimeout, opts.LeafNode.TLSPinnedCerts); err != nil {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\t// If the user wants the TLS handshake to occur first, now that it is\n\t\t// done, send the INFO protocol.\n\t\tif tlsFirst {\n\t\t\tc.flags.set(didTLSFirst)\n\t\t\tc.sendProtoNow(proto)\n\t\t\tif c.isClosed() {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.closeConnection(WriteError)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\t// Leaf nodes will always require a CONNECT to let us know\n\t\t// when we are properly bound to an account.\n\t\t//\n\t\t// If compression is configured, we can't set the authTimer here because\n\t\t// it would cause the parser to fail any incoming protocol that is not a\n\t\t// CONNECT (and we need to exchange INFO protocols for compression\n\t\t// negotiation). So instead, use the ping timer until we are done with\n\t\t// negotiation and can set the auth timer.\n\t\ttimeout := secondsToDuration(opts.LeafNode.AuthTimeout)\n\t\tif needsCompression(opts.LeafNode.Compression.Mode) {\n\t\t\tc.ping.tmr = time.AfterFunc(timeout, func() {\n\t\t\t\tc.authTimeout()\n\t\t\t})\n\t\t} else {\n\t\t\tc.setAuthTimer(timeout)\n\t\t}\n\t}\n\n\t// Keep track in case server is shutdown before we can successfully register.\n\tif !s.addToTempClients(c.cid, c) {\n\t\tc.mu.Unlock()\n\t\tc.setNoReconnect()\n\t\tc.closeConnection(ServerShutdown)\n\t\treturn nil\n\t}\n\n\t// Spin up the read loop.\n\ts.startGoRoutine(func() { c.readLoop(preBuf) })\n\n\t// We will spin the write loop for solicited connections only\n\t// when processing the INFO and after switching to TLS if needed.\n\tif !solicited {\n\t\ts.startGoRoutine(func() { c.writeLoop() })\n\t}\n\n\tc.mu.Unlock()\n\n\treturn c\n}\n\n// Will perform the client-side TLS handshake if needed. Assumes that this\n// is called by the solicit side (remote will be non nil). Returns `true`\n// if TLS is required, `false` otherwise.\n// Lock held on entry.\nfunc (c *client) leafClientHandshakeIfNeeded(remote *leafNodeCfg, opts *Options) (bool, error) {\n\t// Check if TLS is required and gather TLS config variables.\n\ttlsRequired, tlsConfig, tlsName, tlsTimeout := c.leafNodeGetTLSConfigForSolicit(remote)\n\tif !tlsRequired {\n\t\treturn false, nil\n\t}\n\n\t// If TLS required, peform handshake.\n\t// Get the URL that was used to connect to the remote server.\n\trURL := remote.getCurrentURL()\n\n\t// Perform the client-side TLS handshake.\n\tif resetTLSName, err := c.doTLSClientHandshake(tlsHandshakeLeaf, rURL, tlsConfig, tlsName, tlsTimeout, opts.LeafNode.TLSPinnedCerts); err != nil {\n\t\t// Check if we need to reset the remote's TLS name.\n\t\tif resetTLSName {\n\t\t\tremote.Lock()\n\t\t\tremote.tlsName = _EMPTY_\n\t\t\tremote.Unlock()\n\t\t}\n\t\treturn false, err\n\t}\n\treturn true, nil\n}\n\nfunc (c *client) processLeafnodeInfo(info *Info) {\n\tc.mu.Lock()\n\tif c.leaf == nil || c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\ts := c.srv\n\topts := s.getOpts()\n\tremote := c.leaf.remote\n\tdidSolicit := remote != nil\n\tfirstINFO := !c.flags.isSet(infoReceived)\n\n\t// In case of websocket, the TLS handshake has been already done.\n\t// So check only for non websocket connections and for configurations\n\t// where the TLS Handshake was not done first.\n\tif didSolicit && !c.flags.isSet(handshakeComplete) && !c.isWebsocket() && !remote.TLSHandshakeFirst {\n\t\t// If the server requires TLS, we need to set this in the remote\n\t\t// otherwise if there is no TLS configuration block for the remote,\n\t\t// the solicit side will not attempt to perform the TLS handshake.\n\t\tif firstINFO && info.TLSRequired {\n\t\t\t// Check for TLS/proxy configuration mismatch\n\t\t\tif remote.Proxy.URL != _EMPTY_ && !remote.TLS && remote.TLSConfig == nil {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tc.Errorf(\"TLS configuration mismatch: Hub requires TLS but leafnode remote is not configured for TLS. When using a proxy, ensure TLS leafnode configuration matches the Hub requirements.\")\n\t\t\t\tc.closeConnection(TLSHandshakeError)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tremote.TLS = true\n\t\t}\n\t\tif _, err := c.leafClientHandshakeIfNeeded(remote, opts); err != nil {\n\t\t\tc.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Check for compression, unless already done.\n\tif firstINFO && !c.flags.isSet(compressionNegotiated) {\n\t\t// Prevent from getting back here.\n\t\tc.flags.set(compressionNegotiated)\n\n\t\tvar co *CompressionOpts\n\t\tif !didSolicit {\n\t\t\tco = &opts.LeafNode.Compression\n\t\t} else {\n\t\t\tco = &remote.Compression\n\t\t}\n\t\tif needsCompression(co.Mode) {\n\t\t\t// Release client lock since following function will need server lock.\n\t\t\tc.mu.Unlock()\n\t\t\tcompress, err := s.negotiateLeafCompression(c, didSolicit, info.Compression, co)\n\t\t\tif err != nil {\n\t\t\t\tc.sendErrAndErr(err.Error())\n\t\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif compress {\n\t\t\t\t// Done for now, will get back another INFO protocol...\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No compression because one side does not want/can't, so proceed.\n\t\t\tc.mu.Lock()\n\t\t\t// Check that the connection did not close if the lock was released.\n\t\t\tif c.isClosed() {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t} else {\n\t\t\t// Coming from an old server, the Compression field would be the empty\n\t\t\t// string. For servers that are configured with CompressionNotSupported,\n\t\t\t// this makes them behave as old servers.\n\t\t\tif info.Compression == _EMPTY_ || co.Mode == CompressionNotSupported {\n\t\t\t\tc.leaf.compression = CompressionNotSupported\n\t\t\t} else {\n\t\t\t\tc.leaf.compression = CompressionOff\n\t\t\t}\n\t\t}\n\t\t// Accepting side does not normally process an INFO protocol during\n\t\t// initial connection handshake. So we keep it consistent by returning\n\t\t// if we are not soliciting.\n\t\tif !didSolicit {\n\t\t\t// If we had created the ping timer instead of the auth timer, we will\n\t\t\t// clear the ping timer and set the auth timer now that the compression\n\t\t\t// negotiation is done.\n\t\t\tif info.Compression != _EMPTY_ && c.ping.tmr != nil {\n\t\t\t\tclearTimer(&c.ping.tmr)\n\t\t\t\tc.setAuthTimer(secondsToDuration(opts.LeafNode.AuthTimeout))\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t\t// Fall through and process the INFO protocol as usual.\n\t}\n\n\t// Note: For now, only the initial INFO has a nonce. We\n\t// will probably do auto key rotation at some point.\n\tif firstINFO {\n\t\t// Mark that the INFO protocol has been received.\n\t\tc.flags.set(infoReceived)\n\t\t// Prevent connecting to non leafnode port. Need to do this only for\n\t\t// the first INFO, not for async INFO updates...\n\t\t//\n\t\t// Content of INFO sent by the server when accepting a tcp connection.\n\t\t// -------------------------------------------------------------------\n\t\t// Listen Port Of | CID | ClientConnectURLs | LeafNodeURLs | Gateway |\n\t\t// -------------------------------------------------------------------\n\t\t// CLIENT | X* | X** | | |\n\t\t// ROUTE | | X** | X*** | |\n\t\t// GATEWAY | | | | X |\n\t\t// LEAFNODE | X | | X | |\n\t\t// -------------------------------------------------------------------\n\t\t// * Not on older servers.\n\t\t// ** Not if \"no advertise\" is enabled.\n\t\t// *** Not if leafnode's \"no advertise\" is enabled.\n\t\t//\n\t\t// As seen from above, a solicited LeafNode connection should receive\n\t\t// from the remote server an INFO with CID and LeafNodeURLs. Anything\n\t\t// else should be considered an attempt to connect to a wrong port.\n\t\tif didSolicit && (info.CID == 0 || info.LeafNodeURLs == nil) {\n\t\t\tc.mu.Unlock()\n\t\t\tc.Errorf(ErrConnectedToWrongPort.Error())\n\t\t\tc.closeConnection(WrongPort)\n\t\t\treturn\n\t\t}\n\t\t// Reject a cluster that contains spaces.\n\t\tif info.Cluster != _EMPTY_ && strings.Contains(info.Cluster, \" \") {\n\t\t\tc.mu.Unlock()\n\t\t\tc.sendErrAndErr(ErrClusterNameHasSpaces.Error())\n\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\treturn\n\t\t}\n\t\t// Capture a nonce here.\n\t\tc.nonce = []byte(info.Nonce)\n\t\tif info.TLSRequired && didSolicit {\n\t\t\tremote.TLS = true\n\t\t}\n\t\tsupportsHeaders := c.srv.supportsHeaders()\n\t\tc.headers = supportsHeaders && info.Headers\n\n\t\t// Remember the remote server.\n\t\t// Pre 2.2.0 servers are not sending their server name.\n\t\t// In that case, use info.ID, which, for those servers, matches\n\t\t// the content of the field `Name` in the leafnode CONNECT protocol.\n\t\tif info.Name == _EMPTY_ {\n\t\t\tc.leaf.remoteServer = info.ID\n\t\t} else {\n\t\t\tc.leaf.remoteServer = info.Name\n\t\t}\n\t\tc.leaf.remoteDomain = info.Domain\n\t\tc.leaf.remoteCluster = info.Cluster\n\t\t// We send the protocol version in the INFO protocol.\n\t\t// Keep track of it, so we know if this connection supports message\n\t\t// tracing for instance.\n\t\tc.opts.Protocol = info.Proto\n\t}\n\n\t// For both initial INFO and async INFO protocols, Possibly\n\t// update our list of remote leafnode URLs we can connect to.\n\tif didSolicit && (len(info.LeafNodeURLs) > 0 || len(info.WSConnectURLs) > 0) {\n\t\t// Consider the incoming array as the most up-to-date\n\t\t// representation of the remote cluster's list of URLs.\n\t\tc.updateLeafNodeURLs(info)\n\t}\n\n\t// Check to see if we have permissions updates here.\n\tif info.Import != nil || info.Export != nil {\n\t\tperms := &Permissions{\n\t\t\tPublish: info.Export,\n\t\t\tSubscribe: info.Import,\n\t\t}\n\t\t// Check if we have local deny clauses that we need to merge.\n\t\tif remote := c.leaf.remote; remote != nil {\n\t\t\tif len(remote.DenyExports) > 0 {\n\t\t\t\tif perms.Publish == nil {\n\t\t\t\t\tperms.Publish = &SubjectPermission{}\n\t\t\t\t}\n\t\t\t\tperms.Publish.Deny = append(perms.Publish.Deny, remote.DenyExports...)\n\t\t\t}\n\t\t\tif len(remote.DenyImports) > 0 {\n\t\t\t\tif perms.Subscribe == nil {\n\t\t\t\t\tperms.Subscribe = &SubjectPermission{}\n\t\t\t\t}\n\t\t\t\tperms.Subscribe.Deny = append(perms.Subscribe.Deny, remote.DenyImports...)\n\t\t\t}\n\t\t}\n\t\tc.setPermissions(perms)\n\t}\n\n\tvar resumeConnect bool\n\n\t// If this is a remote connection and this is the first INFO protocol,\n\t// then we need to finish the connect process by sending CONNECT, etc..\n\tif firstINFO && didSolicit {\n\t\t// Clear deadline that was set in createLeafNode while waiting for the INFO.\n\t\tc.nc.SetDeadline(time.Time{})\n\t\tresumeConnect = true\n\t} else if !firstINFO && didSolicit {\n\t\tc.leaf.remoteAccName = info.RemoteAccount\n\t}\n\n\t// Check if we have the remote account information and if so make sure it's stored.\n\tif info.RemoteAccount != _EMPTY_ {\n\t\ts.leafRemoteAccounts.Store(c.acc.Name, info.RemoteAccount)\n\t}\n\tc.mu.Unlock()\n\n\tfinishConnect := info.ConnectInfo\n\tif resumeConnect && s != nil {\n\t\ts.leafNodeResumeConnectProcess(c)\n\t\tif !info.InfoOnConnect {\n\t\t\tfinishConnect = true\n\t\t}\n\t}\n\tif finishConnect {\n\t\ts.leafNodeFinishConnectProcess(c)\n\t}\n\n\t// Check to see if we need to kick any internal source or mirror consumers.\n\t// This will be a no-op if JetStream not enabled for this server or if the bound account\n\t// does not have jetstream.\n\ts.checkInternalSyncConsumers(c.acc)\n}\n\nfunc (s *Server) negotiateLeafCompression(c *client, didSolicit bool, infoCompression string, co *CompressionOpts) (bool, error) {\n\t// Negotiate the appropriate compression mode (or no compression)\n\tcm, err := selectCompressionMode(co.Mode, infoCompression)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tc.mu.Lock()\n\t// For \"auto\" mode, set the initial compression mode based on RTT\n\tif cm == CompressionS2Auto {\n\t\tif c.rttStart.IsZero() {\n\t\t\tc.rtt = computeRTT(c.start)\n\t\t}\n\t\tcm = selectS2AutoModeBasedOnRTT(c.rtt, co.RTTThresholds)\n\t}\n\t// Keep track of the negotiated compression mode.\n\tc.leaf.compression = cm\n\tcid := c.cid\n\tvar nonce string\n\tif !didSolicit {\n\t\tnonce = bytesToString(c.nonce)\n\t}\n\tc.mu.Unlock()\n\n\tif !needsCompression(cm) {\n\t\treturn false, nil\n\t}\n\n\t// If we end-up doing compression...\n\n\t// Generate an INFO with the chosen compression mode.\n\ts.mu.Lock()\n\tinfo := s.copyLeafNodeInfo()\n\tinfo.Compression, info.CID, info.Nonce = compressionModeForInfoProtocol(co, cm), cid, nonce\n\tinfoProto := generateInfoJSON(info)\n\ts.mu.Unlock()\n\n\t// If we solicited, then send this INFO protocol BEFORE switching\n\t// to compression writer. However, if we did not, we send it after.\n\tc.mu.Lock()\n\tif didSolicit {\n\t\tc.enqueueProto(infoProto)\n\t\t// Make sure it is completely flushed (the pending bytes goes to\n\t\t// 0) before proceeding.\n\t\tfor c.out.pb > 0 && !c.isClosed() {\n\t\t\tc.flushOutbound()\n\t\t}\n\t}\n\t// This is to notify the readLoop that it should switch to a\n\t// (de)compression reader.\n\tc.in.flags.set(switchToCompression)\n\t// Create the compress writer before queueing the INFO protocol for\n\t// a route that did not solicit. It will make sure that that proto\n\t// is sent with compression on.\n\tc.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...)\n\tif !didSolicit {\n\t\tc.enqueueProto(infoProto)\n\t}\n\tc.mu.Unlock()\n\treturn true, nil\n}\n\n// When getting a leaf node INFO protocol, use the provided\n// array of urls to update the list of possible endpoints.\nfunc (c *client) updateLeafNodeURLs(info *Info) {\n\tcfg := c.leaf.remote\n\tcfg.Lock()\n\tdefer cfg.Unlock()\n\n\t// We have ensured that if a remote has a WS scheme, then all are.\n\t// So check if first is WS, then add WS URLs, otherwise, add non WS ones.\n\tif len(cfg.URLs) > 0 && isWSURL(cfg.URLs[0]) {\n\t\t// It does not really matter if we use \"ws://\" or \"wss://\" here since\n\t\t// we will have already marked that the remote should use TLS anyway.\n\t\t// But use proper scheme for log statements, etc...\n\t\tproto := wsSchemePrefix\n\t\tif cfg.TLS {\n\t\t\tproto = wsSchemePrefixTLS\n\t\t}\n\t\tc.doUpdateLNURLs(cfg, proto, info.WSConnectURLs)\n\t\treturn\n\t}\n\tc.doUpdateLNURLs(cfg, \"nats-leaf\", info.LeafNodeURLs)\n}\n\nfunc (c *client) doUpdateLNURLs(cfg *leafNodeCfg, scheme string, URLs []string) {\n\tcfg.urls = make([]*url.URL, 0, 1+len(URLs))\n\t// Add the ones we receive in the protocol\n\tfor _, surl := range URLs {\n\t\turl, err := url.Parse(fmt.Sprintf(\"%s://%s\", scheme, surl))\n\t\tif err != nil {\n\t\t\t// As per below, the URLs we receive should not have contained URL info, so this should be safe to log.\n\t\t\tc.Errorf(\"Error parsing url %q: %v\", surl, err)\n\t\t\tcontinue\n\t\t}\n\t\t// Do not add if it's the same as what we already have configured.\n\t\tvar dup bool\n\t\tfor _, u := range cfg.URLs {\n\t\t\t// URLs that we receive never have user info, but the\n\t\t\t// ones that were configured may have. Simply compare\n\t\t\t// host and port to decide if they are equal or not.\n\t\t\tif url.Host == u.Host && url.Port() == u.Port() {\n\t\t\t\tdup = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !dup {\n\t\t\tcfg.urls = append(cfg.urls, url)\n\t\t\tcfg.saveTLSHostname(url)\n\t\t}\n\t}\n\t// Add the configured one\n\tcfg.urls = append(cfg.urls, cfg.URLs...)\n}\n\n// Similar to setInfoHostPortAndGenerateJSON, but for leafNodeInfo.\nfunc (s *Server) setLeafNodeInfoHostPortAndIP() error {\n\topts := s.getOpts()\n\tif opts.LeafNode.Advertise != _EMPTY_ {\n\t\tadvHost, advPort, err := parseHostPort(opts.LeafNode.Advertise, opts.LeafNode.Port)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ts.leafNodeInfo.Host = advHost\n\t\ts.leafNodeInfo.Port = advPort\n\t} else {\n\t\ts.leafNodeInfo.Host = opts.LeafNode.Host\n\t\ts.leafNodeInfo.Port = opts.LeafNode.Port\n\t\t// If the host is \"0.0.0.0\" or \"::\" we need to resolve to a public IP.\n\t\t// This will return at most 1 IP.\n\t\thostIsIPAny, ips, err := s.getNonLocalIPsIfHostIsIPAny(s.leafNodeInfo.Host, false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif hostIsIPAny {\n\t\t\tif len(ips) == 0 {\n\t\t\t\ts.Errorf(\"Could not find any non-local IP for leafnode's listen specification %q\",\n\t\t\t\t\ts.leafNodeInfo.Host)\n\t\t\t} else {\n\t\t\t\t// Take the first from the list...\n\t\t\t\ts.leafNodeInfo.Host = ips[0]\n\t\t\t}\n\t\t}\n\t}\n\t// Use just host:port for the IP\n\ts.leafNodeInfo.IP = net.JoinHostPort(s.leafNodeInfo.Host, strconv.Itoa(s.leafNodeInfo.Port))\n\tif opts.LeafNode.Advertise != _EMPTY_ {\n\t\ts.Noticef(\"Advertise address for leafnode is set to %s\", s.leafNodeInfo.IP)\n\t}\n\treturn nil\n}\n\n// Add the connection to the map of leaf nodes.\n// If `checkForDup` is true (invoked when a leafnode is accepted), then we check\n// if a connection already exists for the same server name and account.\n// That can happen when the remote is attempting to reconnect while the accepting\n// side did not detect the connection as broken yet.\n// But it can also happen when there is a misconfiguration and the remote is\n// creating two (or more) connections that bind to the same account on the accept\n// side.\n// When a duplicate is found, the new connection is accepted and the old is closed\n// (this solves the stale connection situation). An error is returned to help the\n// remote detect the misconfiguration when the duplicate is the result of that\n// misconfiguration.\nfunc (s *Server) addLeafNodeConnection(c *client, srvName, clusterName string, checkForDup bool) {\n\tvar accName string\n\tc.mu.Lock()\n\tcid := c.cid\n\tacc := c.acc\n\tif acc != nil {\n\t\taccName = acc.Name\n\t}\n\tmyRemoteDomain := c.leaf.remoteDomain\n\tmySrvName := c.leaf.remoteServer\n\tremoteAccName := c.leaf.remoteAccName\n\tmyClustName := c.leaf.remoteCluster\n\tsolicited := c.leaf.remote != nil\n\tc.mu.Unlock()\n\n\tvar old *client\n\ts.mu.Lock()\n\t// We check for empty because in some test we may send empty CONNECT{}\n\tif checkForDup && srvName != _EMPTY_ {\n\t\tfor _, ol := range s.leafs {\n\t\t\tol.mu.Lock()\n\t\t\t// We care here only about non solicited Leafnode. This function\n\t\t\t// is more about replacing stale connections than detecting loops.\n\t\t\t// We have code for the loop detection elsewhere, which also delays\n\t\t\t// attempt to reconnect.\n\t\t\tif !ol.isSolicitedLeafNode() && ol.leaf.remoteServer == srvName &&\n\t\t\t\tol.leaf.remoteCluster == clusterName && ol.acc.Name == accName &&\n\t\t\t\tremoteAccName != _EMPTY_ && ol.leaf.remoteAccName == remoteAccName {\n\t\t\t\told = ol\n\t\t\t}\n\t\t\tol.mu.Unlock()\n\t\t\tif old != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\t// Store new connection in the map\n\ts.leafs[cid] = c\n\ts.mu.Unlock()\n\ts.removeFromTempClients(cid)\n\n\t// If applicable, evict the old one.\n\tif old != nil {\n\t\told.sendErrAndErr(DuplicateRemoteLeafnodeConnection.String())\n\t\told.closeConnection(DuplicateRemoteLeafnodeConnection)\n\t\tc.Warnf(\"Replacing connection from same server\")\n\t}\n\n\tsrvDecorated := func() string {\n\t\tif myClustName == _EMPTY_ {\n\t\t\treturn mySrvName\n\t\t}\n\t\treturn fmt.Sprintf(\"%s/%s\", mySrvName, myClustName)\n\t}\n\n\topts := s.getOpts()\n\tsysAcc := s.SystemAccount()\n\tjs := s.getJetStream()\n\tvar meta *raft\n\tif js != nil {\n\t\tif mg := js.getMetaGroup(); mg != nil {\n\t\t\tmeta = mg.(*raft)\n\t\t}\n\t}\n\tblockMappingOutgoing := false\n\t// Deny (non domain) JetStream API traffic unless system account is shared\n\t// and domain names are identical and extending is not disabled\n\n\t// Check if backwards compatibility has been enabled and needs to be acted on\n\tforceSysAccDeny := false\n\tif len(opts.JsAccDefaultDomain) > 0 {\n\t\tif acc == sysAcc {\n\t\t\tfor _, d := range opts.JsAccDefaultDomain {\n\t\t\t\tif d == _EMPTY_ {\n\t\t\t\t\t// Extending JetStream via leaf node is mutually exclusive with a domain mapping to the empty/default domain.\n\t\t\t\t\t// As soon as one mapping to \"\" is found, disable the ability to extend JS via a leaf node.\n\t\t\t\t\tc.Noticef(\"Not extending remote JetStream domain %q due to presence of empty default domain\", myRemoteDomain)\n\t\t\t\t\tforceSysAccDeny = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t} else if domain, ok := opts.JsAccDefaultDomain[accName]; ok && domain == _EMPTY_ {\n\t\t\t// for backwards compatibility with old setups that do not have a domain name set\n\t\t\tc.Debugf(\"Skipping deny %q for account %q due to default domain\", jsAllAPI, accName)\n\t\t\treturn\n\t\t}\n\t}\n\n\t// If the server has JS disabled, it may still be part of a JetStream that could be extended.\n\t// This is either signaled by js being disabled and a domain set,\n\t// or in cases where no domain name exists, an extension hint is set.\n\t// However, this is only relevant in mixed setups.\n\t//\n\t// If the system account connects but default domains are present, JetStream can't be extended.\n\tif opts.JetStreamDomain != myRemoteDomain || (!opts.JetStream && (opts.JetStreamDomain == _EMPTY_ && opts.JetStreamExtHint != jsWillExtend)) ||\n\t\tsysAcc == nil || acc == nil || forceSysAccDeny {\n\t\t// If domain names mismatch always deny. This applies to system accounts as well as non system accounts.\n\t\t// Not having a system account, account or JetStream disabled is considered a mismatch as well.\n\t\tif acc != nil && acc == sysAcc {\n\t\t\tc.Noticef(\"System account connected from %s\", srvDecorated())\n\t\t\tc.Noticef(\"JetStream not extended, domains differ\")\n\t\t\tc.mergeDenyPermissionsLocked(both, denyAllJs)\n\t\t\t// When a remote with a system account is present in a server, unless otherwise disabled, the server will be\n\t\t\t// started in observer mode. Now that it is clear that this not used, turn the observer mode off.\n\t\t\tif solicited && meta != nil && meta.IsObserver() {\n\t\t\t\tmeta.setObserver(false, extNotExtended)\n\t\t\t\tc.Debugf(\"Turning JetStream metadata controller Observer Mode off\")\n\t\t\t\t// Take note that the domain was not extended to avoid this state from startup.\n\t\t\t\twritePeerState(js.config.StoreDir, meta.currentPeerState())\n\t\t\t\t// Meta controller can't be leader yet.\n\t\t\t\t// Yet it is possible that due to observer mode every server already stopped campaigning.\n\t\t\t\t// Therefore this server needs to be kicked into campaigning gear explicitly.\n\t\t\t\tmeta.Campaign()\n\t\t\t}\n\t\t} else {\n\t\t\tc.Noticef(\"JetStream using domains: local %q, remote %q\", opts.JetStreamDomain, myRemoteDomain)\n\t\t\tc.mergeDenyPermissionsLocked(both, denyAllClientJs)\n\t\t}\n\t\tblockMappingOutgoing = true\n\t} else if acc == sysAcc {\n\t\t// system account and same domain\n\t\ts.sys.client.Noticef(\"Extending JetStream domain %q as System Account connected from server %s\",\n\t\t\tmyRemoteDomain, srvDecorated())\n\t\t// In an extension use case, pin leadership to server remotes connect to.\n\t\t// Therefore, server with a remote that are not already in observer mode, need to be put into it.\n\t\tif solicited && meta != nil && !meta.IsObserver() {\n\t\t\tmeta.setObserver(true, extExtended)\n\t\t\tc.Debugf(\"Turning JetStream metadata controller Observer Mode on - System Account Connected\")\n\t\t\t// Take note that the domain was not extended to avoid this state next startup.\n\t\t\twritePeerState(js.config.StoreDir, meta.currentPeerState())\n\t\t\t// If this server is the leader already, step down so a new leader can be elected (that is not an observer)\n\t\t\tmeta.StepDown()\n\t\t}\n\t} else {\n\t\t// This deny is needed in all cases (system account shared or not)\n\t\t// If the system account is shared, jsAllAPI traffic will go through the system account.\n\t\t// So in order to prevent duplicate delivery (from system and actual account) suppress it on the account.\n\t\t// If the system account is NOT shared, jsAllAPI traffic has no business\n\t\tc.Debugf(\"Adding deny %+v for account %q\", denyAllClientJs, accName)\n\t\tc.mergeDenyPermissionsLocked(both, denyAllClientJs)\n\t}\n\t// If we have a specified JetStream domain we will want to add a mapping to\n\t// allow access cross domain for each non-system account.\n\tif opts.JetStreamDomain != _EMPTY_ && opts.JetStream && acc != nil && acc != sysAcc {\n\t\tfor src, dest := range generateJSMappingTable(opts.JetStreamDomain) {\n\t\t\tif err := acc.AddMapping(src, dest); err != nil {\n\t\t\t\tc.Debugf(\"Error adding JetStream domain mapping: %s\", err.Error())\n\t\t\t} else {\n\t\t\t\tc.Debugf(\"Adding JetStream Domain Mapping %q -> %s to account %q\", src, dest, accName)\n\t\t\t}\n\t\t}\n\t\tif blockMappingOutgoing {\n\t\t\tsrc := fmt.Sprintf(jsDomainAPI, opts.JetStreamDomain)\n\t\t\t// make sure that messages intended for this domain, do not leave the cluster via this leaf node connection\n\t\t\t// This is a guard against a miss-config with two identical domain names and will only cover some forms\n\t\t\t// of this issue, not all of them.\n\t\t\t// This guards against a hub and a spoke having the same domain name.\n\t\t\t// But not two spokes having the same one and the request coming from the hub.\n\t\t\tc.mergeDenyPermissionsLocked(pub, []string{src})\n\t\t\tc.Debugf(\"Adding deny %q for outgoing messages to account %q\", src, accName)\n\t\t}\n\t}\n}\n\nfunc (s *Server) removeLeafNodeConnection(c *client) {\n\tc.mu.Lock()\n\tcid := c.cid\n\tif c.leaf != nil {\n\t\tif c.leaf.tsubt != nil {\n\t\t\tc.leaf.tsubt.Stop()\n\t\t\tc.leaf.tsubt = nil\n\t\t}\n\t\tif c.leaf.gwSub != nil {\n\t\t\ts.gwLeafSubs.Remove(c.leaf.gwSub)\n\t\t\t// We need to set this to nil for GC to release the connection\n\t\t\tc.leaf.gwSub = nil\n\t\t}\n\t}\n\tproxyKey := c.proxyKey\n\tc.mu.Unlock()\n\ts.mu.Lock()\n\tdelete(s.leafs, cid)\n\tif proxyKey != _EMPTY_ {\n\t\ts.removeProxiedConn(proxyKey, cid)\n\t}\n\ts.mu.Unlock()\n\ts.removeFromTempClients(cid)\n}\n\n// Connect information for solicited leafnodes.\ntype leafConnectInfo struct {\n\tVersion string `json:\"version,omitempty\"`\n\tNkey string `json:\"nkey,omitempty\"`\n\tJWT string `json:\"jwt,omitempty\"`\n\tSig string `json:\"sig,omitempty\"`\n\tUser string `json:\"user,omitempty\"`\n\tPass string `json:\"pass,omitempty\"`\n\tToken string `json:\"auth_token,omitempty\"`\n\tID string `json:\"server_id,omitempty\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tName string `json:\"name,omitempty\"`\n\tHub bool `json:\"is_hub,omitempty\"`\n\tCluster string `json:\"cluster,omitempty\"`\n\tHeaders bool `json:\"headers,omitempty\"`\n\tJetStream bool `json:\"jetstream,omitempty\"`\n\tDenyPub []string `json:\"deny_pub,omitempty\"`\n\tIsolate bool `json:\"isolate,omitempty\"`\n\n\t// There was an existing field called:\n\t// >> Comp bool `json:\"compression,omitempty\"`\n\t// that has never been used. With support for compression, we now need\n\t// a field that is a string. So we use a different json tag:\n\tCompression string `json:\"compress_mode,omitempty\"`\n\n\t// Just used to detect wrong connection attempts.\n\tGateway string `json:\"gateway,omitempty\"`\n\n\t// Tells the accept side which account the remote is binding to.\n\tRemoteAccount string `json:\"remote_account,omitempty\"`\n\n\t// The accept side of a LEAF connection, unlike ROUTER and GATEWAY, receives\n\t// only the CONNECT protocol, and no INFO. So we need to send the protocol\n\t// version as part of the CONNECT. It will indicate if a connection supports\n\t// some features, such as message tracing.\n\t// We use `protocol` as the JSON tag, so this is automatically unmarshal'ed\n\t// in the low level process CONNECT.\n\tProto int `json:\"protocol,omitempty\"`\n}\n\n// processLeafNodeConnect will process the inbound connect args.\n// Once we are here we are bound to an account, so can send any interest that\n// we would have to the other side.\nfunc (c *client) processLeafNodeConnect(s *Server, arg []byte, lang string) error {\n\t// Way to detect clients that incorrectly connect to the route listen\n\t// port. Client provided \"lang\" in the CONNECT protocol while LEAFNODEs don't.\n\tif lang != _EMPTY_ {\n\t\tc.sendErrAndErr(ErrClientConnectedToLeafNodePort.Error())\n\t\tc.closeConnection(WrongPort)\n\t\treturn ErrClientConnectedToLeafNodePort\n\t}\n\n\t// Unmarshal as a leaf node connect protocol\n\tproto := &leafConnectInfo{}\n\tif err := json.Unmarshal(arg, proto); err != nil {\n\t\treturn err\n\t}\n\n\t// Reject a cluster that contains spaces.\n\tif proto.Cluster != _EMPTY_ && strings.Contains(proto.Cluster, \" \") {\n\t\tc.sendErrAndErr(ErrClusterNameHasSpaces.Error())\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn ErrClusterNameHasSpaces\n\t}\n\n\t// Check for cluster name collisions.\n\tif cn := s.cachedClusterName(); cn != _EMPTY_ && proto.Cluster != _EMPTY_ && proto.Cluster == cn {\n\t\tc.sendErrAndErr(ErrLeafNodeHasSameClusterName.Error())\n\t\tc.closeConnection(ClusterNamesIdentical)\n\t\treturn ErrLeafNodeHasSameClusterName\n\t}\n\n\t// Reject if this has Gateway which means that it would be from a gateway\n\t// connection that incorrectly connects to the leafnode port.\n\tif proto.Gateway != _EMPTY_ {\n\t\terrTxt := fmt.Sprintf(\"Rejecting connection from gateway %q on the leafnode port\", proto.Gateway)\n\t\tc.Errorf(errTxt)\n\t\tc.sendErr(errTxt)\n\t\tc.closeConnection(WrongGateway)\n\t\treturn ErrWrongGateway\n\t}\n\n\tif mv := s.getOpts().LeafNode.MinVersion; mv != _EMPTY_ {\n\t\tmajor, minor, update, _ := versionComponents(mv)\n\t\tif !versionAtLeast(proto.Version, major, minor, update) {\n\t\t\t// Send back an INFO so recent remote servers process the rejection\n\t\t\t// cleanly, then close immediately. The soliciting side applies the\n\t\t\t// reconnect delay when it processes the error.\n\t\t\ts.sendPermsAndAccountInfo(c)\n\t\t\tc.sendErrAndErr(fmt.Sprintf(\"%s %q\", ErrLeafNodeMinVersionRejected, mv))\n\t\t\tc.closeConnection(MinimumVersionRequired)\n\t\t\treturn ErrMinimumVersionRequired\n\t\t}\n\t}\n\n\t// Check if this server supports headers.\n\tsupportHeaders := c.srv.supportsHeaders()\n\n\tc.mu.Lock()\n\t// Leaf Nodes do not do echo or verbose or pedantic.\n\tc.opts.Verbose = false\n\tc.opts.Echo = false\n\tc.opts.Pedantic = false\n\t// This inbound connection will be marked as supporting headers if this server\n\t// support headers and the remote has sent in the CONNECT protocol that it does\n\t// support headers too.\n\tc.headers = supportHeaders && proto.Headers\n\t// If the compression level is still not set, set it based on what has been\n\t// given to us in the CONNECT protocol.\n\tif c.leaf.compression == _EMPTY_ {\n\t\t// But if proto.Compression is _EMPTY_, set it to CompressionNotSupported\n\t\tif proto.Compression == _EMPTY_ {\n\t\t\tc.leaf.compression = CompressionNotSupported\n\t\t} else {\n\t\t\tc.leaf.compression = proto.Compression\n\t\t}\n\t}\n\n\t// Remember the remote server.\n\tc.leaf.remoteServer = proto.Name\n\t// Remember the remote account name\n\tc.leaf.remoteAccName = proto.RemoteAccount\n\t// Remember if the leafnode requested isolation.\n\tc.leaf.isolated = c.leaf.isolated || proto.Isolate\n\n\t// If the other side has declared itself a hub, so we will take on the spoke role.\n\tif proto.Hub {\n\t\tc.leaf.isSpoke = true\n\t}\n\n\t// The soliciting side is part of a cluster.\n\tif proto.Cluster != _EMPTY_ {\n\t\tc.leaf.remoteCluster = proto.Cluster\n\t}\n\n\tc.leaf.remoteDomain = proto.Domain\n\n\t// When a leaf solicits a connection to a hub, the perms that it will use on the soliciting leafnode's\n\t// behalf are correct for them, but inside the hub need to be reversed since data is flowing in the opposite direction.\n\tif !c.isSolicitedLeafNode() && c.perms != nil {\n\t\tsp, pp := c.perms.sub, c.perms.pub\n\t\tc.perms.sub, c.perms.pub = pp, sp\n\t\tif c.opts.Import != nil {\n\t\t\tc.darray = c.opts.Import.Deny\n\t\t} else {\n\t\t\tc.darray = nil\n\t\t}\n\t}\n\n\t// Set the Ping timer\n\tc.setFirstPingTimer()\n\n\t// If we received pub deny permissions from the other end, merge with existing ones.\n\tc.mergeDenyPermissions(pub, proto.DenyPub)\n\n\tacc := c.acc\n\tc.mu.Unlock()\n\n\t// Register the cluster, even if empty, as long as we are acting as a hub.\n\tif !proto.Hub {\n\t\tacc.registerLeafNodeCluster(proto.Cluster)\n\t}\n\n\t// Add in the leafnode here since we passed through auth at this point.\n\ts.addLeafNodeConnection(c, proto.Name, proto.Cluster, true)\n\n\t// If we have permissions bound to this leafnode we need to send then back to the\n\t// origin server for local enforcement.\n\ts.sendPermsAndAccountInfo(c)\n\n\t// Create and initialize the smap since we know our bound account now.\n\t// This will send all registered subs too.\n\ts.initLeafNodeSmapAndSendSubs(c)\n\n\t// Announce the account connect event for a leaf node.\n\t// This will be a no-op as needed.\n\ts.sendLeafNodeConnect(c.acc)\n\n\t// Check to see if we need to kick any internal source or mirror consumers.\n\t// This will be a no-op if JetStream not enabled for this server or if the bound account\n\t// does not have jetstream.\n\ts.checkInternalSyncConsumers(acc)\n\n\treturn nil\n}\n\n// checkInternalSyncConsumers\nfunc (s *Server) checkInternalSyncConsumers(acc *Account) {\n\t// Grab our js\n\tjs := s.getJetStream()\n\n\t// Only applicable if we have JS and the leafnode has JS as well.\n\t// We check for remote JS outside.\n\tif !js.isEnabled() || acc == nil {\n\t\treturn\n\t}\n\n\t// We will check all streams in our local account. They must be a leader and\n\t// be sourcing or mirroring. We will check the external config on the stream itself\n\t// if this is cross domain, or if the remote domain is empty, meaning we might be\n\t// extending the system across this leafnode connection and hence we would be extending\n\t// our own domain.\n\tjsa := js.lookupAccount(acc)\n\tif jsa == nil {\n\t\treturn\n\t}\n\n\tvar streams []*stream\n\tjsa.mu.RLock()\n\tfor _, mset := range jsa.streams {\n\t\tmset.cfgMu.RLock()\n\t\t// We need to have a mirror or source defined.\n\t\t// We do not want to force another lock here to look for leader status,\n\t\t// so collect and after we release jsa will make sure.\n\t\tif mset.cfg.Mirror != nil || len(mset.cfg.Sources) > 0 {\n\t\t\tstreams = append(streams, mset)\n\t\t}\n\t\tmset.cfgMu.RUnlock()\n\t}\n\tjsa.mu.RUnlock()\n\n\t// Now loop through all candidates and check if we are the leader and have NOT\n\t// created the sync up consumer.\n\tfor _, mset := range streams {\n\t\tmset.retryDisconnectedSyncConsumers()\n\t}\n}\n\n// Returns the remote cluster name. This is set only once so does not require a lock.\nfunc (c *client) remoteCluster() string {\n\tif c.leaf == nil {\n\t\treturn _EMPTY_\n\t}\n\treturn c.leaf.remoteCluster\n}\n\n// Sends back an info block to the soliciting leafnode to let it know about\n// its permission settings for local enforcement.\nfunc (s *Server) sendPermsAndAccountInfo(c *client) {\n\t// Copy\n\ts.mu.Lock()\n\tinfo := s.copyLeafNodeInfo()\n\ts.mu.Unlock()\n\tc.mu.Lock()\n\tinfo.CID = c.cid\n\tinfo.Import = c.opts.Import\n\tinfo.Export = c.opts.Export\n\tinfo.RemoteAccount = c.acc.Name\n\t// s.SystemAccount() uses an atomic operation and does not get the server lock, so this is safe.\n\tinfo.IsSystemAccount = c.acc == s.SystemAccount()\n\tinfo.ConnectInfo = true\n\tc.enqueueProto(generateInfoJSON(info))\n\tc.mu.Unlock()\n}\n\n// Snapshot the current subscriptions from the sublist into our smap which\n// we will keep updated from now on.\n// Also send the registered subscriptions.\nfunc (s *Server) initLeafNodeSmapAndSendSubs(c *client) {\n\tacc := c.acc\n\tif acc == nil {\n\t\tc.Debugf(\"Leafnode does not have an account bound\")\n\t\treturn\n\t}\n\t// Collect all account subs here.\n\t_subs := [1024]*subscription{}\n\tsubs := _subs[:0]\n\tims := []string{}\n\n\t// Hold the client lock otherwise there can be a race and miss some subs.\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tacc.mu.RLock()\n\taccName := acc.Name\n\taccNTag := acc.nameTag\n\n\t// To make printing look better when no friendly name present.\n\tif accNTag != _EMPTY_ {\n\t\taccNTag = \"/\" + accNTag\n\t}\n\n\t// If we are solicited we only send interest for local clients.\n\tif c.isSpokeLeafNode() {\n\t\tacc.sl.localSubs(&subs, true)\n\t} else {\n\t\tacc.sl.All(&subs)\n\t}\n\n\t// Check if we have an existing service import reply.\n\tsiReply := copyBytes(acc.siReply)\n\n\t// Since leaf nodes only send on interest, if the bound\n\t// account has import services we need to send those over.\n\tfor isubj := range acc.imports.services {\n\t\tif c.isSpokeLeafNode() && !c.canSubscribe(isubj) {\n\t\t\tc.Debugf(\"Not permitted to import service %q on behalf of %s%s\", isubj, accName, accNTag)\n\t\t\tcontinue\n\t\t}\n\t\tims = append(ims, isubj)\n\t}\n\t// Likewise for mappings.\n\tfor _, m := range acc.mappings {\n\t\tif c.isSpokeLeafNode() && !c.canSubscribe(m.src) {\n\t\t\tc.Debugf(\"Not permitted to import mapping %q on behalf of %s%s\", m.src, accName, accNTag)\n\t\t\tcontinue\n\t\t}\n\t\tims = append(ims, m.src)\n\t}\n\n\t// Create a unique subject that will be used for loop detection.\n\tlds := acc.lds\n\tacc.mu.RUnlock()\n\n\t// Check if we have to create the LDS.\n\tif lds == _EMPTY_ {\n\t\tlds = leafNodeLoopDetectionSubjectPrefix + nuid.Next()\n\t\tacc.mu.Lock()\n\t\tacc.lds = lds\n\t\tacc.mu.Unlock()\n\t}\n\n\t// Now check for gateway interest. Leafnodes will put this into\n\t// the proper mode to propagate, but they are not held in the account.\n\tgwsa := [16]*client{}\n\tgws := gwsa[:0]\n\ts.getOutboundGatewayConnections(&gws)\n\tfor _, cgw := range gws {\n\t\tcgw.mu.Lock()\n\t\tgw := cgw.gw\n\t\tcgw.mu.Unlock()\n\t\tif gw != nil {\n\t\t\tif ei, _ := gw.outsim.Load(accName); ei != nil {\n\t\t\t\tif e := ei.(*outsie); e != nil && e.sl != nil {\n\t\t\t\t\te.sl.All(&subs)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tapplyGlobalRouting := s.gateway.enabled\n\tif c.isSpokeLeafNode() {\n\t\t// Add a fake subscription for this solicited leafnode connection\n\t\t// so that we can send back directly for mapped GW replies.\n\t\t// We need to keep track of this subscription so it can be removed\n\t\t// when the connection is closed so that the GC can release it.\n\t\tc.leaf.gwSub = &subscription{client: c, subject: []byte(gwReplyPrefix + \">\")}\n\t\tc.srv.gwLeafSubs.Insert(c.leaf.gwSub)\n\t}\n\n\t// Now walk the results and add them to our smap\n\trc := c.leaf.remoteCluster\n\tc.leaf.smap = make(map[string]int32)\n\tfor _, sub := range subs {\n\t\t// Check perms regardless of role.\n\t\tif c.perms != nil && !c.canSubscribe(string(sub.subject)) {\n\t\t\tc.Debugf(\"Not permitted to subscribe to %q on behalf of %s%s\", sub.subject, accName, accNTag)\n\t\t\tcontinue\n\t\t}\n\t\t// Don't advertise interest from leafnodes to other isolated leafnodes.\n\t\tif sub.client.kind == LEAF && c.isIsolatedLeafNode() {\n\t\t\tcontinue\n\t\t}\n\t\t// We ignore ourselves here.\n\t\t// Also don't add the subscription if it has a origin cluster and the\n\t\t// cluster name matches the one of the client we are sending to.\n\t\tif c != sub.client && (sub.origin == nil || (bytesToString(sub.origin) != rc)) {\n\t\t\tcount := int32(1)\n\t\t\tif len(sub.queue) > 0 && sub.qw > 0 {\n\t\t\t\tcount = sub.qw\n\t\t\t}\n\t\t\tc.leaf.smap[keyFromSub(sub)] += count\n\t\t\tif c.leaf.tsub == nil {\n\t\t\t\tc.leaf.tsub = make(map[*subscription]struct{})\n\t\t\t}\n\t\t\tc.leaf.tsub[sub] = struct{}{}\n\t\t}\n\t}\n\t// FIXME(dlc) - We need to update appropriately on an account claims update.\n\tfor _, isubj := range ims {\n\t\tc.leaf.smap[isubj]++\n\t}\n\t// If we have gateways enabled we need to make sure the other side sends us responses\n\t// that have been augmented from the original subscription.\n\t// TODO(dlc) - Should we lock this down more?\n\tif applyGlobalRouting {\n\t\tc.leaf.smap[oldGWReplyPrefix+\"*.>\"]++\n\t\tc.leaf.smap[gwReplyPrefix+\">\"]++\n\t}\n\t// Detect loops by subscribing to a specific subject and checking\n\t// if this sub is coming back to us.\n\tc.leaf.smap[lds]++\n\n\t// Check if we need to add an existing siReply to our map.\n\t// This will be a prefix so add on the wildcard.\n\tif siReply != nil {\n\t\twcsub := append(siReply, '>')\n\t\tc.leaf.smap[string(wcsub)]++\n\t}\n\t// Queue all protocols. There is no max pending limit for LN connection,\n\t// so we don't need chunking. The writes will happen from the writeLoop.\n\tvar b bytes.Buffer\n\tfor key, n := range c.leaf.smap {\n\t\tc.writeLeafSub(&b, key, n)\n\t}\n\tif b.Len() > 0 {\n\t\tc.enqueueProto(b.Bytes())\n\t}\n\tif c.leaf.tsub != nil {\n\t\t// Clear the tsub map after 5 seconds.\n\t\tc.leaf.tsubt = time.AfterFunc(5*time.Second, func() {\n\t\t\tc.mu.Lock()\n\t\t\tif c.leaf != nil {\n\t\t\t\tc.leaf.tsub = nil\n\t\t\t\tc.leaf.tsubt = nil\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t})\n\t}\n}\n\n// updateInterestForAccountOnGateway called from gateway code when processing RS+ and RS-.\nfunc (s *Server) updateInterestForAccountOnGateway(accName string, sub *subscription, delta int32) {\n\t// Since we're in the gateway's readLoop, and we would otherwise block, don't allow fetching.\n\tacc, err := s.lookupOrFetchAccount(accName, false)\n\tif acc == nil || err != nil {\n\t\ts.Debugf(\"No or bad account for %q, failed to update interest from gateway\", accName)\n\t\treturn\n\t}\n\tacc.updateLeafNodes(sub, delta)\n}\n\n// updateLeafNodesEx will make sure to update the account smap for the subscription.\n// Will also forward to all leaf nodes as needed.\n// If `hubOnly` is true, then will update only leaf nodes that connect to this server\n// (that is, for which this server acts as a hub to them).\nfunc (acc *Account) updateLeafNodesEx(sub *subscription, delta int32, hubOnly bool) {\n\tif acc == nil || sub == nil {\n\t\treturn\n\t}\n\n\t// We will do checks for no leafnodes and same cluster here inline and under the\n\t// general account read lock.\n\t// If we feel we need to update the leafnodes we will do that out of line to avoid\n\t// blocking routes or GWs.\n\n\tacc.mu.RLock()\n\t// First check if we even have leafnodes here.\n\tif acc.nleafs == 0 {\n\t\tacc.mu.RUnlock()\n\t\treturn\n\t}\n\n\t// Is this a loop detection subject.\n\tisLDS := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix))\n\n\t// Capture the cluster even if its empty.\n\tvar cluster string\n\tif sub.origin != nil {\n\t\tcluster = bytesToString(sub.origin)\n\t}\n\n\t// If we have an isolated cluster we can return early, as long as it is not a loop detection subject.\n\t// Empty clusters will return false for the check.\n\tif !isLDS && acc.isLeafNodeClusterIsolated(cluster) {\n\t\tacc.mu.RUnlock()\n\t\treturn\n\t}\n\n\t// We can release the general account lock.\n\tacc.mu.RUnlock()\n\n\t// We can hold the list lock here to avoid having to copy a large slice.\n\tacc.lmu.RLock()\n\tdefer acc.lmu.RUnlock()\n\n\t// Do this once.\n\tsubject := string(sub.subject)\n\n\t// Walk the connected leafnodes.\n\tfor _, ln := range acc.lleafs {\n\t\tif ln == sub.client {\n\t\t\tcontinue\n\t\t}\n\t\tln.mu.Lock()\n\t\t// Don't advertise interest from leafnodes to other isolated leafnodes.\n\t\tif sub.client.kind == LEAF && ln.isIsolatedLeafNode() {\n\t\t\tln.mu.Unlock()\n\t\t\tcontinue\n\t\t}\n\t\t// If `hubOnly` is true, it means that we want to update only leafnodes\n\t\t// that connect to this server (so isHubLeafNode() would return `true`).\n\t\tif hubOnly && !ln.isHubLeafNode() {\n\t\t\tln.mu.Unlock()\n\t\t\tcontinue\n\t\t}\n\t\t// Check to make sure this sub does not have an origin cluster that matches the leafnode.\n\t\t// If skipped, make sure that we still let go the \"$LDS.\" subscription that allows\n\t\t// the detection of loops as long as different cluster.\n\t\tclusterDifferent := cluster != ln.remoteCluster()\n\t\tif (isLDS && clusterDifferent) || ((cluster == _EMPTY_ || clusterDifferent) && (delta <= 0 || ln.canSubscribe(subject))) {\n\t\t\tln.updateSmap(sub, delta, isLDS)\n\t\t}\n\t\tln.mu.Unlock()\n\t}\n}\n\n// updateLeafNodes will make sure to update the account smap for the subscription.\n// Will also forward to all leaf nodes as needed.\nfunc (acc *Account) updateLeafNodes(sub *subscription, delta int32) {\n\tacc.updateLeafNodesEx(sub, delta, false)\n}\n\n// This will make an update to our internal smap and determine if we should send out\n// an interest update to the remote side.\n// Lock should be held.\nfunc (c *client) updateSmap(sub *subscription, delta int32, isLDS bool) {\n\tif c.leaf.smap == nil {\n\t\treturn\n\t}\n\n\t// If we are solicited make sure this is a local client or a non-solicited leaf node\n\tskind := sub.client.kind\n\tupdateClient := skind == CLIENT || skind == SYSTEM || skind == JETSTREAM || skind == ACCOUNT\n\tif !isLDS && c.isSpokeLeafNode() && !(updateClient || (skind == LEAF && !sub.client.isSpokeLeafNode())) {\n\t\treturn\n\t}\n\n\t// For additions, check if that sub has just been processed during initLeafNodeSmapAndSendSubs\n\tif delta > 0 && c.leaf.tsub != nil {\n\t\tif _, present := c.leaf.tsub[sub]; present {\n\t\t\tdelete(c.leaf.tsub, sub)\n\t\t\tif len(c.leaf.tsub) == 0 {\n\t\t\t\tc.leaf.tsub = nil\n\t\t\t\tc.leaf.tsubt.Stop()\n\t\t\t\tc.leaf.tsubt = nil\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\tkey := keyFromSub(sub)\n\tn, ok := c.leaf.smap[key]\n\tif delta < 0 && !ok {\n\t\treturn\n\t}\n\n\t// We will update if its a queue, if count is zero (or negative), or we were 0 and are N > 0.\n\tupdate := sub.queue != nil || (n <= 0 && n+delta > 0) || (n > 0 && n+delta <= 0)\n\tn += delta\n\tif n > 0 {\n\t\tc.leaf.smap[key] = n\n\t} else {\n\t\tdelete(c.leaf.smap, key)\n\t}\n\tif update {\n\t\tc.sendLeafNodeSubUpdate(key, n)\n\t}\n}\n\n// Used to force add subjects to the subject map.\nfunc (c *client) forceAddToSmap(subj string) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.leaf.smap == nil {\n\t\treturn\n\t}\n\tn := c.leaf.smap[subj]\n\tif n != 0 {\n\t\treturn\n\t}\n\t// Place into the map since it was not there.\n\tc.leaf.smap[subj] = 1\n\tc.sendLeafNodeSubUpdate(subj, 1)\n}\n\n// Used to force remove a subject from the subject map.\nfunc (c *client) forceRemoveFromSmap(subj string) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\n\tif c.leaf.smap == nil {\n\t\treturn\n\t}\n\tn := c.leaf.smap[subj]\n\tif n == 0 {\n\t\treturn\n\t}\n\tn--\n\tif n == 0 {\n\t\t// Remove is now zero\n\t\tdelete(c.leaf.smap, subj)\n\t\tc.sendLeafNodeSubUpdate(subj, 0)\n\t} else {\n\t\tc.leaf.smap[subj] = n\n\t}\n}\n\n// Send the subscription interest change to the other side.\n// Lock should be held.\nfunc (c *client) sendLeafNodeSubUpdate(key string, n int32) {\n\t// If we are a spoke, we need to check if we are allowed to send this subscription over to the hub.\n\tif c.isSpokeLeafNode() {\n\t\tcheckPerms := true\n\t\tif len(key) > 0 && (key[0] == '$' || key[0] == '_') {\n\t\t\tif strings.HasPrefix(key, leafNodeLoopDetectionSubjectPrefix) ||\n\t\t\t\tstrings.HasPrefix(key, oldGWReplyPrefix) ||\n\t\t\t\tstrings.HasPrefix(key, gwReplyPrefix) {\n\t\t\t\tcheckPerms = false\n\t\t\t}\n\t\t}\n\t\tif checkPerms {\n\t\t\tvar subject string\n\t\t\tif sep := strings.IndexByte(key, ' '); sep != -1 {\n\t\t\t\tsubject = key[:sep]\n\t\t\t} else {\n\t\t\t\tsubject = key\n\t\t\t}\n\t\t\tif !c.canSubscribe(subject) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\t// If we are here we can send over to the other side.\n\t_b := [64]byte{}\n\tb := bytes.NewBuffer(_b[:0])\n\tc.writeLeafSub(b, key, n)\n\tc.enqueueProto(b.Bytes())\n}\n\n// Helper function to build the key.\nfunc keyFromSub(sub *subscription) string {\n\tvar sb strings.Builder\n\tsb.Grow(len(sub.subject) + len(sub.queue) + 1)\n\tsb.Write(sub.subject)\n\tif sub.queue != nil {\n\t\t// Just make the key subject spc group, e.g. 'foo bar'\n\t\tsb.WriteByte(' ')\n\t\tsb.Write(sub.queue)\n\t}\n\treturn sb.String()\n}\n\nconst (\n\tkeyRoutedSub = \"R\"\n\tkeyRoutedSubByte = 'R'\n\tkeyRoutedLeafSub = \"L\"\n\tkeyRoutedLeafSubByte = 'L'\n)\n\n// Helper function to build the key that prevents collisions between normal\n// routed subscriptions and routed subscriptions on behalf of a leafnode.\n// Keys will look like this:\n// \"R foo\" -> plain routed sub on \"foo\"\n// \"R foo bar\" -> queue routed sub on \"foo\", queue \"bar\"\n// \"L foo bar\" -> plain routed leaf sub on \"foo\", leaf \"bar\"\n// \"L foo bar baz\" -> queue routed sub on \"foo\", queue \"bar\", leaf \"baz\"\nfunc keyFromSubWithOrigin(sub *subscription) string {\n\tvar sb strings.Builder\n\tsb.Grow(2 + len(sub.origin) + 1 + len(sub.subject) + 1 + len(sub.queue))\n\tleaf := len(sub.origin) > 0\n\tif leaf {\n\t\tsb.WriteByte(keyRoutedLeafSubByte)\n\t} else {\n\t\tsb.WriteByte(keyRoutedSubByte)\n\t}\n\tsb.WriteByte(' ')\n\tsb.Write(sub.subject)\n\tif sub.queue != nil {\n\t\tsb.WriteByte(' ')\n\t\tsb.Write(sub.queue)\n\t}\n\tif leaf {\n\t\tsb.WriteByte(' ')\n\t\tsb.Write(sub.origin)\n\t}\n\treturn sb.String()\n}\n\n// Lock should be held.\nfunc (c *client) writeLeafSub(w *bytes.Buffer, key string, n int32) {\n\tif key == _EMPTY_ {\n\t\treturn\n\t}\n\tif n > 0 {\n\t\tw.WriteString(\"LS+ \" + key)\n\t\t// Check for queue semantics, if found write n.\n\t\tif strings.Contains(key, \" \") {\n\t\t\tw.WriteString(\" \")\n\t\t\tvar b [12]byte\n\t\t\tvar i = len(b)\n\t\t\tfor l := n; l > 0; l /= 10 {\n\t\t\t\ti--\n\t\t\t\tb[i] = digits[l%10]\n\t\t\t}\n\t\t\tw.Write(b[i:])\n\t\t\tif c.trace {\n\t\t\t\targ := fmt.Sprintf(\"%s %d\", key, n)\n\t\t\t\tc.traceOutOp(\"LS+\", []byte(arg))\n\t\t\t}\n\t\t} else if c.trace {\n\t\t\tc.traceOutOp(\"LS+\", []byte(key))\n\t\t}\n\t} else {\n\t\tw.WriteString(\"LS- \" + key)\n\t\tif c.trace {\n\t\t\tc.traceOutOp(\"LS-\", []byte(key))\n\t\t}\n\t}\n\tw.WriteString(CR_LF)\n}\n\n// processLeafSub will process an inbound sub request for the remote leaf node.\nfunc (c *client) processLeafSub(argo []byte) (err error) {\n\t// Indicate activity.\n\tc.in.subs++\n\n\tsrv := c.srv\n\tif srv == nil {\n\t\treturn nil\n\t}\n\n\t// Copy so we do not reference a potentially large buffer\n\targ := make([]byte, len(argo))\n\tcopy(arg, argo)\n\n\targs := splitArg(arg)\n\tsub := &subscription{client: c}\n\n\tdelta := int32(1)\n\tswitch len(args) {\n\tcase 1:\n\t\tsub.queue = nil\n\tcase 3:\n\t\tsub.queue = args[1]\n\t\tsub.qw = int32(parseSize(args[2]))\n\t\t// TODO: (ik) We should have a non empty queue name and a queue\n\t\t// weight >= 1. For 2.11, we may want to return an error if that\n\t\t// is not the case, but for now just overwrite `delta` if queue\n\t\t// weight is greater than 1 (it is possible after a reconnect/\n\t\t// server restart to receive a queue weight > 1 for a new sub).\n\t\tif sub.qw > 1 {\n\t\t\tdelta = sub.qw\n\t\t}\n\tdefault:\n\t\treturn fmt.Errorf(\"processLeafSub Parse Error: '%s'\", arg)\n\t}\n\tsub.subject = args[0]\n\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\n\tacc := c.acc\n\t// Guard against LS+ arriving before CONNECT has been processed, which\n\t// can happen when compression is enabled.\n\tif acc == nil {\n\t\tc.mu.Unlock()\n\t\tc.sendErr(\"Authorization Violation\")\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn nil\n\t}\n\t// Check if we have a loop.\n\tldsPrefix := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix))\n\n\tif ldsPrefix && bytesToString(sub.subject) == acc.getLDSubject() {\n\t\tc.mu.Unlock()\n\t\tc.handleLeafNodeLoop(true)\n\t\treturn nil\n\t}\n\n\t// Check permissions if applicable. (but exclude the $LDS, $GR and _GR_)\n\tcheckPerms := true\n\tif sub.subject[0] == '$' || sub.subject[0] == '_' {\n\t\tif ldsPrefix ||\n\t\t\tbytes.HasPrefix(sub.subject, []byte(oldGWReplyPrefix)) ||\n\t\t\tbytes.HasPrefix(sub.subject, []byte(gwReplyPrefix)) {\n\t\t\tcheckPerms = false\n\t\t}\n\t}\n\n\t// If we are a hub check that we can publish to this subject.\n\tif checkPerms {\n\t\tsubj := string(sub.subject)\n\t\tif subjectIsLiteral(subj) && !c.pubAllowedFullCheck(subj, true, true) {\n\t\t\tc.mu.Unlock()\n\t\t\tc.leafSubPermViolation(sub.subject)\n\t\t\tc.Debugf(fmt.Sprintf(\"Permissions Violation for Subscription to %q\", sub.subject))\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t// Check if we have a maximum on the number of subscriptions.\n\tif c.subsAtLimit() {\n\t\tc.mu.Unlock()\n\t\tc.maxSubsExceeded()\n\t\treturn nil\n\t}\n\n\t// If we have an origin cluster associated mark that in the sub.\n\tif rc := c.remoteCluster(); rc != _EMPTY_ {\n\t\tsub.origin = []byte(rc)\n\t}\n\n\t// Like Routes, we store local subs by account and subject and optionally queue name.\n\t// If we have a queue it will have a trailing weight which we do not want.\n\tif sub.queue != nil {\n\t\tsub.sid = arg[:len(arg)-len(args[2])-1]\n\t} else {\n\t\tsub.sid = arg\n\t}\n\tkey := bytesToString(sub.sid)\n\tosub := c.subs[key]\n\tif osub == nil {\n\t\tc.subs[key] = sub\n\t\t// Now place into the account sl.\n\t\tif err := acc.sl.Insert(sub); err != nil {\n\t\t\tdelete(c.subs, key)\n\t\t\tc.mu.Unlock()\n\t\t\tc.Errorf(\"Could not insert subscription: %v\", err)\n\t\t\tc.sendErr(\"Invalid Subscription\")\n\t\t\treturn nil\n\t\t}\n\t} else if sub.queue != nil {\n\t\t// For a queue we need to update the weight.\n\t\tdelta = sub.qw - atomic.LoadInt32(&osub.qw)\n\t\tatomic.StoreInt32(&osub.qw, sub.qw)\n\t\tacc.sl.UpdateRemoteQSub(osub)\n\t}\n\tspoke := c.isSpokeLeafNode()\n\tc.mu.Unlock()\n\n\t// Only add in shadow subs if a new sub or qsub.\n\tif osub == nil {\n\t\tif err := c.addShadowSubscriptions(acc, sub); err != nil {\n\t\t\tc.Errorf(err.Error())\n\t\t}\n\t}\n\n\t// If we are not solicited, treat leaf node subscriptions similar to a\n\t// client subscription, meaning we forward them to routes, gateways and\n\t// other leaf nodes as needed.\n\tif !spoke {\n\t\t// If we are routing add to the route map for the associated account.\n\t\tsrv.updateRouteSubscriptionMap(acc, sub, delta)\n\t\tif srv.gateway.enabled {\n\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, delta)\n\t\t}\n\t}\n\t// Now check on leafnode updates for other leaf nodes. We understand solicited\n\t// and non-solicited state in this call so we will do the right thing.\n\tacc.updateLeafNodes(sub, delta)\n\n\treturn nil\n}\n\n// If the leafnode is a solicited, set the connect delay based on default\n// or private option (for tests). Sends the error to the other side, log and\n// close the connection.\nfunc (c *client) handleLeafNodeLoop(sendErr bool) {\n\taccName, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterLoopDetected)\n\terrTxt := fmt.Sprintf(\"Loop detected for leafnode account=%q. Delaying attempt to reconnect for %v\", accName, delay)\n\tif sendErr {\n\t\tc.sendErr(errTxt)\n\t}\n\n\tc.Errorf(errTxt)\n\t// If we are here with \"sendErr\" false, it means that this is the server\n\t// that received the error. The other side will have closed the connection,\n\t// but does not hurt to close here too.\n\tc.closeConnection(ProtocolViolation)\n}\n\n// processLeafUnsub will process an inbound unsub request for the remote leaf node.\nfunc (c *client) processLeafUnsub(arg []byte) error {\n\t// Indicate any activity, so pub and sub or unsubs.\n\tc.in.subs++\n\n\tsrv := c.srv\n\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\n\tacc := c.acc\n\t// Guard against LS- arriving before CONNECT has been processed.\n\tif acc == nil {\n\t\tc.mu.Unlock()\n\t\tc.sendErr(\"Authorization Violation\")\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn nil\n\t}\n\n\tspoke := c.isSpokeLeafNode()\n\t// We store local subs by account and subject and optionally queue name.\n\t// LS- will have the arg exactly as the key.\n\tsub, ok := c.subs[string(arg)]\n\tif !ok {\n\t\t// If not found, don't try to update routes/gws/leaf nodes.\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\tdelta := int32(1)\n\tif len(sub.queue) > 0 {\n\t\tdelta = sub.qw\n\t}\n\tc.mu.Unlock()\n\n\tc.unsubscribe(acc, sub, true, true)\n\tif !spoke {\n\t\t// If we are routing subtract from the route map for the associated account.\n\t\tsrv.updateRouteSubscriptionMap(acc, sub, -delta)\n\t\t// Gateways\n\t\tif srv.gateway.enabled {\n\t\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, -delta)\n\t\t}\n\t}\n\t// Now check on leafnode updates for other leaf nodes.\n\tacc.updateLeafNodes(sub, -delta)\n\treturn nil\n}\n\nfunc (c *client) processLeafHeaderMsgArgs(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\targs := c.argsa[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 0, 1, 2:\n\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Parse Error: '%s'\", args)\n\tcase 3:\n\t\tc.pa.reply = nil\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[1]\n\t\tc.pa.hdr = parseSize(args[1])\n\t\tc.pa.szb = args[2]\n\t\tc.pa.size = parseSize(args[2])\n\tcase 4:\n\t\tc.pa.reply = args[1]\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[2]\n\t\tc.pa.hdr = parseSize(args[2])\n\t\tc.pa.szb = args[3]\n\t\tc.pa.size = parseSize(args[3])\n\tdefault:\n\t\t// args[1] is our reply indicator. Should be + or | normally.\n\t\tif len(args[1]) != 1 {\n\t\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'\", args[1])\n\t\t}\n\t\tswitch args[1][0] {\n\t\tcase '+':\n\t\t\tc.pa.reply = args[2]\n\t\tcase '|':\n\t\t\tc.pa.reply = nil\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Bad or Missing Reply Indicator: '%s'\", args[1])\n\t\t}\n\t\t// Grab header size.\n\t\tc.pa.hdb = args[len(args)-2]\n\t\tc.pa.hdr = parseSize(c.pa.hdb)\n\n\t\t// Grab size.\n\t\tc.pa.szb = args[len(args)-1]\n\t\tc.pa.size = parseSize(c.pa.szb)\n\n\t\t// Grab queue names.\n\t\tif c.pa.reply != nil {\n\t\t\tc.pa.queues = args[3 : len(args)-2]\n\t\t} else {\n\t\t\tc.pa.queues = args[2 : len(args)-2]\n\t\t}\n\t}\n\tif c.pa.hdr < 0 {\n\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Bad or Missing Header Size: '%s'\", arg)\n\t}\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Bad or Missing Size: '%s'\", args)\n\t}\n\tif c.pa.hdr > c.pa.size {\n\t\treturn fmt.Errorf(\"processLeafHeaderMsgArgs Header Size larger then TotalSize: '%s'\", arg)\n\t}\n\n\t// Common ones processed after check for arg length\n\tc.pa.subject = args[0]\n\n\treturn nil\n}\n\nfunc (c *client) processLeafMsgArgs(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\targs := c.argsa[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 0, 1:\n\t\treturn fmt.Errorf(\"processLeafMsgArgs Parse Error: '%s'\", args)\n\tcase 2:\n\t\tc.pa.reply = nil\n\t\tc.pa.queues = nil\n\t\tc.pa.szb = args[1]\n\t\tc.pa.size = parseSize(args[1])\n\tcase 3:\n\t\tc.pa.reply = args[1]\n\t\tc.pa.queues = nil\n\t\tc.pa.szb = args[2]\n\t\tc.pa.size = parseSize(args[2])\n\tdefault:\n\t\t// args[1] is our reply indicator. Should be + or | normally.\n\t\tif len(args[1]) != 1 {\n\t\t\treturn fmt.Errorf(\"processLeafMsgArgs Bad or Missing Reply Indicator: '%s'\", args[1])\n\t\t}\n\t\tswitch args[1][0] {\n\t\tcase '+':\n\t\t\tc.pa.reply = args[2]\n\t\tcase '|':\n\t\t\tc.pa.reply = nil\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"processLeafMsgArgs Bad or Missing Reply Indicator: '%s'\", args[1])\n\t\t}\n\t\t// Grab size.\n\t\tc.pa.szb = args[len(args)-1]\n\t\tc.pa.size = parseSize(c.pa.szb)\n\n\t\t// Grab queue names.\n\t\tif c.pa.reply != nil {\n\t\t\tc.pa.queues = args[3 : len(args)-1]\n\t\t} else {\n\t\t\tc.pa.queues = args[2 : len(args)-1]\n\t\t}\n\t}\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processLeafMsgArgs Bad or Missing Size: '%s'\", args)\n\t}\n\n\t// Common ones processed after check for arg length\n\tc.pa.subject = args[0]\n\n\treturn nil\n}\n\n// processInboundLeafMsg is called to process an inbound msg from a leaf node.\nfunc (c *client) processInboundLeafMsg(msg []byte) {\n\t// Update statistics\n\t// The msg includes the CR_LF, so pull back out for accounting.\n\tc.in.msgs++\n\tc.in.bytes += int32(len(msg) - LEN_CR_LF)\n\n\tsrv, acc, subject := c.srv, c.acc, string(c.pa.subject)\n\n\t// Mostly under testing scenarios.\n\tif srv == nil || acc == nil {\n\t\treturn\n\t}\n\n\t// Match the subscriptions. We will use our own L1 map if\n\t// it's still valid, avoiding contention on the shared sublist.\n\tvar r *SublistResult\n\tvar ok bool\n\n\tgenid := atomic.LoadUint64(&c.acc.sl.genid)\n\tif genid == c.in.genid && c.in.results != nil {\n\t\tr, ok = c.in.results[subject]\n\t} else {\n\t\t// Reset our L1 completely.\n\t\tc.in.results = make(map[string]*SublistResult)\n\t\tc.in.genid = genid\n\t}\n\n\t// Go back to the sublist data structure.\n\tif !ok {\n\t\tr = c.acc.sl.Match(subject)\n\t\t// Prune the results cache. Keeps us from unbounded growth. Random delete.\n\t\tif len(c.in.results) >= maxResultCacheSize {\n\t\t\tn := 0\n\t\t\tfor subj := range c.in.results {\n\t\t\t\tdelete(c.in.results, subj)\n\t\t\t\tif n++; n > pruneSize {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Then add the new cache entry.\n\t\tc.in.results[subject] = r\n\t}\n\n\t// Collect queue names if needed.\n\tvar qnames [][]byte\n\n\t// Check for no interest, short circuit if so.\n\t// This is the fanout scale.\n\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\tflag := pmrNoFlag\n\t\t// If we have queue subs in this cluster, then if we run in gateway\n\t\t// mode and the remote gateways have queue subs, then we need to\n\t\t// collect the queue groups this message was sent to so that we\n\t\t// exclude them when sending to gateways.\n\t\tif len(r.qsubs) > 0 && c.srv.gateway.enabled &&\n\t\t\tatomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 {\n\t\t\tflag |= pmrCollectQueueNames\n\t\t}\n\t\t// If this is a mapped subject that means the mapped interest\n\t\t// is what got us here, but this might not have a queue designation\n\t\t// If that is the case, make sure we ignore to process local queue subscribers.\n\t\tif len(c.pa.mapped) > 0 && len(c.pa.queues) == 0 {\n\t\t\tflag |= pmrIgnoreEmptyQueueFilter\n\t\t}\n\t\t_, qnames = c.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, flag)\n\t}\n\n\t// Now deal with gateways\n\tif c.srv.gateway.enabled {\n\t\tc.sendMsgToGateways(acc, msg, c.pa.subject, c.pa.reply, qnames, true)\n\t}\n}\n\n// Handles a subscription permission violation.\n// See leafPermViolation() for details.\nfunc (c *client) leafSubPermViolation(subj []byte) {\n\tc.leafPermViolation(false, subj)\n}\n\n// Common function to process publish or subscribe leafnode permission violation.\n// Sends the permission violation error to the remote, logs it and closes the connection.\n// If this is from a server soliciting, the reconnection will be delayed.\nfunc (c *client) leafPermViolation(pub bool, subj []byte) {\n\tif c.isSpokeLeafNode() {\n\t\t// For spokes these are no-ops since the hub server told us our permissions.\n\t\t// We just need to not send these over to the other side since we will get cutoff.\n\t\treturn\n\t}\n\t// FIXME(dlc) ?\n\tc.setLeafConnectDelayIfSoliciting(leafNodeReconnectAfterPermViolation)\n\tvar action string\n\tif pub {\n\t\tc.sendErr(fmt.Sprintf(\"Permissions Violation for Publish to %q\", subj))\n\t\taction = \"Publish\"\n\t} else {\n\t\tc.sendErr(fmt.Sprintf(\"Permissions Violation for Subscription to %q\", subj))\n\t\taction = \"Subscription\"\n\t}\n\tc.Errorf(\"%s Violation on %q - Check other side configuration\", action, subj)\n\t// TODO: add a new close reason that is more appropriate?\n\tc.closeConnection(ProtocolViolation)\n}\n\n// Invoked from generic processErr() for LEAF connections.\nfunc (c *client) leafProcessErr(errStr string) {\n\t// Check if we got a cluster name collision.\n\tif strings.Contains(errStr, ErrLeafNodeHasSameClusterName.Error()) {\n\t\t_, delay := c.setLeafConnectDelayIfSoliciting(leafNodeReconnectDelayAfterClusterNameSame)\n\t\tc.Errorf(\"Leafnode connection dropped with same cluster name error. Delaying attempt to reconnect for %v\", delay)\n\t\treturn\n\t}\n\tif strings.Contains(errStr, ErrLeafNodeMinVersionRejected.Error()) {\n\t\t_, delay := c.setLeafConnectDelayIfSoliciting(leafNodeMinVersionReconnectDelay)\n\t\tc.Errorf(\"Leafnode connection dropped due to minimum version requirement. Delaying attempt to reconnect for %v\", delay)\n\t\treturn\n\t}\n\n\t// We will look for Loop detected error coming from the other side.\n\t// If we solicit, set the connect delay.\n\tif !strings.Contains(errStr, \"Loop detected\") {\n\t\treturn\n\t}\n\tc.handleLeafNodeLoop(false)\n}\n\n// If this leaf connection solicits, sets the connect delay to the given value,\n// or the one from the server option's LeafNode.connDelay if one is set (for tests).\n// Returns the connection's account name and delay.\nfunc (c *client) setLeafConnectDelayIfSoliciting(delay time.Duration) (string, time.Duration) {\n\tc.mu.Lock()\n\tif c.isSolicitedLeafNode() {\n\t\tif s := c.srv; s != nil {\n\t\t\tif srvdelay := s.getOpts().LeafNode.connDelay; srvdelay != 0 {\n\t\t\t\tdelay = srvdelay\n\t\t\t}\n\t\t}\n\t\tc.leaf.remote.setConnectDelay(delay)\n\t}\n\taccName := c.acc.Name\n\tc.mu.Unlock()\n\treturn accName, delay\n}\n\n// For the given remote Leafnode configuration, this function returns\n// if TLS is required, and if so, will return a clone of the TLS Config\n// (since some fields will be changed during handshake), the TLS server\n// name that is remembered, and the TLS timeout.\nfunc (c *client) leafNodeGetTLSConfigForSolicit(remote *leafNodeCfg) (bool, *tls.Config, string, float64) {\n\tvar (\n\t\ttlsConfig *tls.Config\n\t\ttlsName string\n\t\ttlsTimeout float64\n\t)\n\n\tremote.RLock()\n\tdefer remote.RUnlock()\n\n\ttlsRequired := remote.TLS || remote.TLSConfig != nil\n\tif tlsRequired {\n\t\tif remote.TLSConfig != nil {\n\t\t\ttlsConfig = remote.TLSConfig.Clone()\n\t\t} else {\n\t\t\ttlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}\n\t\t}\n\t\ttlsName = remote.tlsName\n\t\ttlsTimeout = remote.TLSTimeout\n\t\tif tlsTimeout == 0 {\n\t\t\ttlsTimeout = float64(TLS_TIMEOUT / time.Second)\n\t\t}\n\t}\n\n\treturn tlsRequired, tlsConfig, tlsName, tlsTimeout\n}\n\n// Initiates the LeafNode Websocket connection by:\n// - doing the TLS handshake if needed\n// - sending the HTTP request\n// - waiting for the HTTP response\n//\n// Since some bufio reader is used to consume the HTTP response, this function\n// returns the slice of buffered bytes (if any) so that the readLoop that will\n// be started after that consume those first before reading from the socket.\n// The boolean\n//\n// Lock held on entry.\nfunc (c *client) leafNodeSolicitWSConnection(opts *Options, rURL *url.URL, remote *leafNodeCfg) ([]byte, ClosedState, error) {\n\tremote.RLock()\n\tcompress := remote.Websocket.Compression\n\t// By default the server will mask outbound frames, but it can be disabled with this option.\n\tnoMasking := remote.Websocket.NoMasking\n\tinfoTimeout := remote.FirstInfoTimeout\n\tremote.RUnlock()\n\t// Will do the client-side TLS handshake if needed.\n\ttlsRequired, err := c.leafClientHandshakeIfNeeded(remote, opts)\n\tif err != nil {\n\t\t// 0 will indicate that the connection was already closed\n\t\treturn nil, 0, err\n\t}\n\n\t// For http request, we need the passed URL to contain either http or https scheme.\n\tscheme := \"http\"\n\tif tlsRequired {\n\t\tscheme = \"https\"\n\t}\n\t// We will use the `/leafnode` path to tell the accepting WS server that it should\n\t// create a LEAF connection, not a CLIENT.\n\t// In case we use the user's URL path in the future, make sure we append the user's\n\t// path to our `/leafnode` path.\n\tlpath := leafNodeWSPath\n\tif curPath := rURL.EscapedPath(); curPath != _EMPTY_ {\n\t\tif curPath[0] == '/' {\n\t\t\tcurPath = curPath[1:]\n\t\t}\n\t\tlpath = path.Join(curPath, lpath)\n\t} else {\n\t\tlpath = lpath[1:]\n\t}\n\tustr := fmt.Sprintf(\"%s://%s/%s\", scheme, rURL.Host, lpath)\n\tu, _ := url.Parse(ustr)\n\treq := &http.Request{\n\t\tMethod: \"GET\",\n\t\tURL: u,\n\t\tProto: \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader: make(http.Header),\n\t\tHost: u.Host,\n\t}\n\twsKey, err := wsMakeChallengeKey()\n\tif err != nil {\n\t\treturn nil, WriteError, err\n\t}\n\n\treq.Header[\"Upgrade\"] = []string{\"websocket\"}\n\treq.Header[\"Connection\"] = []string{\"Upgrade\"}\n\treq.Header[\"Sec-WebSocket-Key\"] = []string{wsKey}\n\treq.Header[\"Sec-WebSocket-Version\"] = []string{\"13\"}\n\tif compress {\n\t\treq.Header.Add(\"Sec-WebSocket-Extensions\", wsPMCReqHeaderValue)\n\t}\n\tif noMasking {\n\t\treq.Header.Add(wsNoMaskingHeader, wsNoMaskingValue)\n\t}\n\tc.nc.SetDeadline(time.Now().Add(infoTimeout))\n\tif err := req.Write(c.nc); err != nil {\n\t\treturn nil, WriteError, err\n\t}\n\n\tvar resp *http.Response\n\n\tbr := bufio.NewReaderSize(c.nc, MAX_CONTROL_LINE_SIZE)\n\tresp, err = http.ReadResponse(br, req)\n\tif err == nil &&\n\t\t(resp.StatusCode != 101 ||\n\t\t\t!strings.EqualFold(resp.Header.Get(\"Upgrade\"), \"websocket\") ||\n\t\t\t!strings.EqualFold(resp.Header.Get(\"Connection\"), \"upgrade\") ||\n\t\t\tresp.Header.Get(\"Sec-Websocket-Accept\") != wsAcceptKey(wsKey)) {\n\n\t\terr = fmt.Errorf(\"invalid websocket connection\")\n\t}\n\t// Check compression extension...\n\tif err == nil && c.ws.compress {\n\t\t// Check that not only permessage-deflate extension is present, but that\n\t\t// we also have server and client no context take over.\n\t\tsrvCompress, noCtxTakeover := wsPMCExtensionSupport(resp.Header, false)\n\n\t\t// If server does not support compression, then simply disable it in our side.\n\t\tif !srvCompress {\n\t\t\tc.ws.compress = false\n\t\t} else if !noCtxTakeover {\n\t\t\terr = fmt.Errorf(\"compression negotiation error\")\n\t\t}\n\t}\n\t// Same for no masking...\n\tif err == nil && noMasking {\n\t\t// Check if server accepts no masking\n\t\tif resp.Header.Get(wsNoMaskingHeader) != wsNoMaskingValue {\n\t\t\t// Nope, need to mask our writes as any client would do.\n\t\t\tc.ws.maskwrite = true\n\t\t}\n\t}\n\tif resp != nil {\n\t\tresp.Body.Close()\n\t}\n\tif err != nil {\n\t\treturn nil, ReadError, err\n\t}\n\tc.Debugf(\"Leafnode compression=%v masking=%v\", c.ws.compress, c.ws.maskwrite)\n\n\tvar preBuf []byte\n\t// We have to slurp whatever is in the bufio reader and pass that to the readloop.\n\tif n := br.Buffered(); n != 0 {\n\t\tpreBuf, _ = br.Peek(n)\n\t}\n\treturn preBuf, 0, nil\n}\n\nconst connectProcessTimeout = 2 * time.Second\n\n// This is invoked for remote LEAF remote connections after processing the INFO\n// protocol.\nfunc (s *Server) leafNodeResumeConnectProcess(c *client) {\n\tclusterName := s.ClusterName()\n\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tif err := c.sendLeafConnect(clusterName, c.headers); err != nil {\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(WriteError)\n\t\treturn\n\t}\n\n\t// Spin up the write loop.\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\t// timeout leafNodeFinishConnectProcess\n\tc.ping.tmr = time.AfterFunc(connectProcessTimeout, func() {\n\t\tc.mu.Lock()\n\t\t// check if leafNodeFinishConnectProcess was called and prevent later leafNodeFinishConnectProcess\n\t\tif !c.flags.setIfNotSet(connectProcessFinished) {\n\t\t\tc.mu.Unlock()\n\t\t\treturn\n\t\t}\n\t\tclearTimer(&c.ping.tmr)\n\t\tclosed := c.isClosed()\n\t\tc.mu.Unlock()\n\t\tif !closed {\n\t\t\tc.sendErrAndDebug(\"Stale Leaf Node Connection - Closing\")\n\t\t\tc.closeConnection(StaleConnection)\n\t\t}\n\t})\n\tc.mu.Unlock()\n\tc.Debugf(\"Remote leafnode connect msg sent\")\n}\n\n// This is invoked for remote LEAF connections after processing the INFO\n// protocol and leafNodeResumeConnectProcess.\n// This will send LS+ the CONNECT protocol and register the leaf node.\nfunc (s *Server) leafNodeFinishConnectProcess(c *client) {\n\tc.mu.Lock()\n\tif !c.flags.setIfNotSet(connectProcessFinished) {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\ts.removeLeafNodeConnection(c)\n\t\treturn\n\t}\n\tremote := c.leaf.remote\n\t// Check if we will need to send the system connect event.\n\tremote.RLock()\n\tsendSysConnectEvent := remote.Hub\n\tremote.RUnlock()\n\n\t// Capture account before releasing lock\n\tacc := c.acc\n\t// cancel connectProcessTimeout\n\tclearTimer(&c.ping.tmr)\n\tc.mu.Unlock()\n\n\t// Make sure we register with the account here.\n\tif err := c.registerWithAccount(acc); err != nil {\n\t\tif err == ErrTooManyAccountConnections {\n\t\t\tc.maxAccountConnExceeded()\n\t\t\treturn\n\t\t} else if err == ErrLeafNodeLoop {\n\t\t\tc.handleLeafNodeLoop(true)\n\t\t\treturn\n\t\t}\n\t\tc.Errorf(\"Registering leaf with account %s resulted in error: %v\", acc.Name, err)\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn\n\t}\n\ts.addLeafNodeConnection(c, _EMPTY_, _EMPTY_, false)\n\ts.initLeafNodeSmapAndSendSubs(c)\n\tif sendSysConnectEvent {\n\t\ts.sendLeafNodeConnect(acc)\n\t}\n\n\t// The above functions are not atomically under the client\n\t// lock doing those operations. It is possible - since we\n\t// have started the read/write loops - that the connection\n\t// is closed before or in between. This would leave the\n\t// closed LN connection possible registered with the account\n\t// and/or the server's leafs map. So check if connection\n\t// is closed, and if so, manually cleanup.\n\tc.mu.Lock()\n\tclosed := c.isClosed()\n\tif !closed {\n\t\tc.setFirstPingTimer()\n\t}\n\tc.mu.Unlock()\n\tif closed {\n\t\ts.removeLeafNodeConnection(c)\n\t\tif prev := acc.removeClient(c); prev == 1 {\n\t\t\ts.decActiveAccounts()\n\t\t}\n\t}\n}\n" }, { "path": "server/leafnode_proxy_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\n// Basic HTTP proxy for testing\ntype testHTTPProxy struct {\n\tlistener net.Listener\n\tport int\n\tusername string\n\tpassword string\n\tstarted bool\n\tcloseDelay time.Duration // Delay before closing connections for robustness\n\tconnections []net.Conn // Track connections for cleanup\n}\n\nfunc createTestHTTPProxy(username, password string) *testHTTPProxy {\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tport := l.Addr().(*net.TCPAddr).Port\n\n\tproxy := &testHTTPProxy{\n\t\tlistener: l,\n\t\tport: port,\n\t\tusername: username,\n\t\tpassword: password,\n\t\tcloseDelay: 100 * time.Millisecond, // Default delay for test robustness\n\t\tconnections: make([]net.Conn, 0),\n\t}\n\n\treturn proxy\n}\n\nfunc (p *testHTTPProxy) setCloseDelay(delay time.Duration) {\n\tp.closeDelay = delay\n}\n\nfunc (p *testHTTPProxy) start() {\n\tif p.started {\n\t\treturn\n\t}\n\tp.started = true\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := p.listener.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tp.connections = append(p.connections, conn)\n\t\t\tgo p.handleConnection(conn)\n\t\t}\n\t}()\n}\n\nfunc (p *testHTTPProxy) handleConnection(conn net.Conn) {\n\tdefer func() {\n\t\tif p.closeDelay > 0 {\n\t\t\ttime.Sleep(p.closeDelay)\n\t\t}\n\t\tconn.Close()\n\t}()\n\n\t// Set read timeout to prevent hanging on malformed requests\n\tconn.SetReadDeadline(time.Now().Add(10 * time.Second))\n\n\t// Read the CONNECT request\n\tbuffer := make([]byte, 4096)\n\tn, err := conn.Read(buffer)\n\tif err != nil {\n\t\treturn\n\t}\n\n\trequest := string(buffer[:n])\n\tlines := strings.Split(request, \"\\r\\n\")\n\n\tif len(lines) == 0 || !strings.HasPrefix(lines[0], \"CONNECT \") {\n\t\tconn.Write([]byte(\"HTTP/1.1 400 Bad Request\\r\\n\\r\\n\"))\n\t\treturn\n\t}\n\n\t// Check authentication if required\n\tif p.username != _EMPTY_ || p.password != _EMPTY_ {\n\t\tauthFound := false\n\t\tfor _, line := range lines {\n\t\t\tif strings.HasPrefix(line, \"Proxy-Authorization: Basic \") {\n\t\t\t\tauthFound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !authFound {\n\t\t\tconn.Write([]byte(\"HTTP/1.1 407 Proxy Authentication Required\\r\\n\\r\\n\"))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Extract target host from CONNECT line\n\tparts := strings.Fields(lines[0])\n\tif len(parts) < 3 {\n\t\tconn.Write([]byte(\"HTTP/1.1 400 Bad Request\\r\\n\\r\\n\"))\n\t\treturn\n\t}\n\n\ttargetHost := parts[1]\n\n\t// Connect to target with timeout\n\ttarget, err := net.DialTimeout(\"tcp\", targetHost, 5*time.Second)\n\tif err != nil {\n\t\tconn.Write([]byte(\"HTTP/1.1 502 Bad Gateway\\r\\n\\r\\n\"))\n\t\treturn\n\t}\n\tdefer target.Close()\n\n\t// Send success response\n\tconn.Write([]byte(\"HTTP/1.1 200 Connection established\\r\\n\\r\\n\"))\n\n\t// Clear read deadline for ongoing connection\n\tconn.SetReadDeadline(time.Time{})\n\n\t// Relay data between client and target with proper error handling\n\tdone := make(chan bool, 2)\n\n\t// Client to target\n\tgo func() {\n\t\tdefer func() {\n\t\t\tdone <- true\n\t\t\ttarget.Close()\n\t\t}()\n\t\tbuffer := make([]byte, 32*1024)\n\t\tfor {\n\t\t\tconn.SetReadDeadline(time.Now().Add(30 * time.Second))\n\t\t\tn, err := conn.Read(buffer)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttarget.SetWriteDeadline(time.Now().Add(10 * time.Second))\n\t\t\t_, err = target.Write(buffer[:n])\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Target to client\n\tgo func() {\n\t\tdefer func() {\n\t\t\tdone <- true\n\t\t\tconn.Close()\n\t\t}()\n\t\tbuffer := make([]byte, 32*1024)\n\t\tfor {\n\t\t\ttarget.SetReadDeadline(time.Now().Add(30 * time.Second))\n\t\t\tn, err := target.Read(buffer)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tconn.SetWriteDeadline(time.Now().Add(10 * time.Second))\n\t\t\t_, err = conn.Write(buffer[:n])\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Wait for either direction to finish\n\t<-done\n}\n\nfunc (p *testHTTPProxy) stop() {\n\tif p.listener != nil {\n\t\tp.listener.Close()\n\t}\n\t// Close all tracked connections with delay for robustness\n\tfor _, conn := range p.connections {\n\t\tgo func(c net.Conn) {\n\t\t\tif p.closeDelay > 0 {\n\t\t\t\ttime.Sleep(p.closeDelay)\n\t\t\t}\n\t\t\tc.Close()\n\t\t}(conn)\n\t}\n}\n\nfunc (p *testHTTPProxy) url() string {\n\treturn fmt.Sprintf(\"http://127.0.0.1:%d\", p.port)\n}\n\nfunc TestLeafNodeHttpProxyConfigParsing(t *testing.T) {\n\t// Test valid proxy configuration\n\tconf := `\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\tproxy {\n\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\tusername: \"user\"\n\t\t\t\t\t\tpassword: \"pass\"\n\t\t\t\t\t\ttimeout: \"10s\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\n\tconfigFile := createConfFile(t, []byte(conf))\n\n\topts, err := ProcessConfigFile(configFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing config: %v\", err)\n\t}\n\n\tif len(opts.LeafNode.Remotes) != 1 {\n\t\tt.Fatalf(\"Expected 1 remote, got %d\", len(opts.LeafNode.Remotes))\n\t}\n\n\tremote := opts.LeafNode.Remotes[0]\n\tif remote.Proxy.URL != \"http://proxy.example.com:8080\" {\n\t\tt.Errorf(\"Expected proxy URL 'http://proxy.example.com:8080', got '%s'\", remote.Proxy.URL)\n\t}\n\tif remote.Proxy.Username != \"user\" {\n\t\tt.Errorf(\"Expected proxy username 'user', got '%s'\", remote.Proxy.Username)\n\t}\n\tif remote.Proxy.Password != \"pass\" {\n\t\tt.Errorf(\"Expected proxy password 'pass', got '%s'\", remote.Proxy.Password)\n\t}\n\tif remote.Proxy.Timeout != 10*time.Second {\n\t\tt.Errorf(\"Expected proxy timeout 10s, got %v\", remote.Proxy.Timeout)\n\t}\n}\n\nfunc TestLeafNodeHttpProxyConfigWarnings(t *testing.T) {\n\ttestCases := []struct {\n\t\tname string\n\t\tconfig string\n\t\texpectWarning bool\n\t\twarningMatch string\n\t}{\n\t\t{\n\t\t\tname: \"proxy with only TCP URLs\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpectWarning: true,\n\t\t\twarningMatch: \"proxy configuration will be ignored\",\n\t\t},\n\t\t{\n\t\t\tname: \"proxy with mixed TCP and WebSocket URLs\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turls: [\"nats://127.0.0.1:7422\", \"ws://127.0.0.1:8080\"]\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpectWarning: true,\n\t\t\twarningMatch: \"proxy configuration will only be used for WebSocket URLs\",\n\t\t},\n\t\t{\n\t\t\tname: \"proxy with only WebSocket URLs\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes = [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"ws://127.0.0.1:7422\"\n\t\t\t\t\t\t\tproxy {\n\t\t\t\t\t\t\t\turl: \"http://proxy.example.com:8080\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpectWarning: false,\n\t\t},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tconfigFile := createConfFile(t, []byte(tc.config))\n\n\t\t\topts, err := ProcessConfigFile(configFile)\n\n\t\t\tif tc.expectWarning {\n\t\t\t\t// With ProcessConfigFile, warnings don't cause errors\n\t\t\t\t// The configuration should be valid but might log warnings\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected valid configuration with warnings, but got error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif opts == nil {\n\t\t\t\t\tt.Fatal(\"Expected valid options but got nil\")\n\t\t\t\t}\n\t\t\t\t// Note: With ProcessConfigFile, warnings are filtered out and not returned as errors\n\t\t\t\t// The test verifies that the configuration is valid despite having warning conditions\n\t\t\t} else {\n\t\t\t\t// No warnings expected - should parse successfully\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected no error but got: %v\", err)\n\t\t\t\t}\n\t\t\t\tif opts == nil {\n\t\t\t\t\tt.Fatal(\"Expected valid options but got nil\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeHttpProxyConnection(t *testing.T) {\n\t// Create a hub server with WebSocket support using config file\n\thubConfig := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\n\thub, hubOpts := RunServerWithConfig(hubConfig)\n\tdefer hub.Shutdown()\n\n\t// Create HTTP proxy\n\tproxy := createTestHTTPProxy(_EMPTY_, _EMPTY_)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create spoke server with proxy configuration via config file\n\tconfigContent := fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\t\tproxy {\n\t\t\t\t\t\turl: \"%s\"\n\t\t\t\t\t\ttimeout: 5s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hubOpts.Websocket.Port, proxy.url())\n\n\tconfigFile := createConfFile(t, []byte(configContent))\n\n\tspoke, _ := RunServerWithConfig(configFile)\n\tdefer spoke.Shutdown()\n\n\t// Verify leafnode connections are established\n\tcheckLeafNodeConnected(t, spoke)\n\tcheckLeafNodeConnected(t, hub)\n}\n\nfunc TestLeafNodeHttpProxyConnectionToTCP(t *testing.T) {\n\t// Create a hub server using config file\n\thubConfig := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\n\thub, hubOpts := RunServerWithConfig(hubConfig)\n\tdefer hub.Shutdown()\n\n\t// Create HTTP proxy\n\tproxy := createTestHTTPProxy(_EMPTY_, _EMPTY_)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create spoke server with proxy configuration via config file\n\tconfigContent := fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t\tproxy {\n\t\t\t\t\t\turl: \"%s\"\n\t\t\t\t\t\ttimeout: 5s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hubOpts.LeafNode.Port, proxy.url())\n\n\tconfigFile := createConfFile(t, []byte(configContent))\n\n\tspoke, _ := RunServerWithConfig(configFile)\n\tdefer spoke.Shutdown()\n\n\t// Verify leafnode connections are established\n\tcheckLeafNodeConnected(t, spoke)\n\tcheckLeafNodeConnected(t, hub)\n}\n\nfunc TestLeafNodeHttpProxyWithAuthentication(t *testing.T) {\n\t// Create a hub server with WebSocket support using config file\n\thubConfig := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\n\thub, hubOpts := RunServerWithConfig(hubConfig)\n\tdefer hub.Shutdown()\n\n\t// Create HTTP proxy with authentication\n\tproxy := createTestHTTPProxy(\"testuser\", \"testpass\")\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create spoke server with proxy configuration including auth via config file\n\tconfigContent := fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\t\tproxy {\n\t\t\t\t\t\turl: \"%s\"\n\t\t\t\t\t\tusername: \"testuser\"\n\t\t\t\t\t\tpassword: \"testpass\"\n\t\t\t\t\t\ttimeout: 5s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hubOpts.Websocket.Port, proxy.url())\n\n\tconfigFile := createConfFile(t, []byte(configContent))\n\n\tspoke, _ := RunServerWithConfig(configFile)\n\tdefer spoke.Shutdown()\n\n\t// Verify leafnode connections are established\n\tcheckLeafNodeConnected(t, spoke)\n\tcheckLeafNodeConnected(t, hub)\n}\n\nfunc TestLeafNodeHttpProxyTLSMismatchDetection(t *testing.T) {\n\t// This test simulates the TLS mismatch scenario described in the feedback:\n\t// - Leafnode configured with proxy but no TLS\n\t// - Hub requires TLS\n\t// - Connection should fail with appropriate error message\n\n\t// Create hub server with TLS required using config file\n\thubConfig := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls: {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t}\n\t\t}\n\t`))\n\n\thub, hubOpts := RunServerWithConfig(hubConfig)\n\tdefer hub.Shutdown()\n\n\t// Create HTTP proxy\n\tproxy := createTestHTTPProxy(_EMPTY_, _EMPTY_)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create spoke server with proxy but no TLS configuration (intentional mismatch)\n\tspokeConfigContent := fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\t\tproxy {\n\t\t\t\t\t\turl: \"%s\"\n\t\t\t\t\t\ttimeout: 5s\n\t\t\t\t\t}\n\t\t\t\t\t# Intentionally no TLS configuration to create mismatch\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, hubOpts.LeafNode.Port, proxy.url())\n\n\tspokeConfig := createConfFile(t, []byte(spokeConfigContent))\n\tspoke, _ := RunServerWithConfig(spokeConfig)\n\tdefer spoke.Shutdown()\n\n\t// Wait and verify that connection was NOT established due to TLS mismatch\n\t// First attempt happens during RunServerWithConfig(), then retries every 50ms\n\ttime.Sleep(250 * time.Millisecond)\n\n\tif spoke.NumLeafNodes() != 0 {\n\t\tt.Errorf(\"Expected 0 leafnode connections due to TLS mismatch, got %d\", spoke.NumLeafNodes())\n\t}\n}\n\nfunc TestLeafNodeHttpProxyTunnelBasic(t *testing.T) {\n\t// Create HTTP proxy with longer delay for robustness\n\tproxy := createTestHTTPProxy(_EMPTY_, _EMPTY_)\n\tproxy.setCloseDelay(200 * time.Millisecond)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create a simple TCP server to connect to through proxy\n\tlistener, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test server: %v\", err)\n\t}\n\tdefer listener.Close()\n\n\ttargetPort := listener.Addr().(*net.TCPAddr).Port\n\ttargetHost := fmt.Sprintf(\"127.0.0.1:%d\", targetPort)\n\n\terrCh := make(chan error, 1)\n\n\t// Accept one connection with proper timeout handling\n\tgo func() {\n\t\tconn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"unable to accept: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tdefer conn.Close()\n\n\t\t// Set read deadline to prevent hanging forever\n\t\tconn.SetReadDeadline(time.Now().Add(5 * time.Second))\n\n\t\t// Read the incoming data first\n\t\tbuffer := make([]byte, 1024)\n\t\tn, err := conn.Read(buffer)\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"server failed to read: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\treceivedMsg := string(buffer[:n])\n\t\tif receivedMsg != \"Hello\" {\n\t\t\terrCh <- fmt.Errorf(\"server expected 'Hello', got '%s'\", receivedMsg)\n\t\t\treturn\n\t\t}\n\n\t\t// Send response\n\t\tconn.SetWriteDeadline(time.Now().Add(5 * time.Second))\n\t\t_, err = conn.Write([]byte(\"Hello from target server\"))\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"server failed to write: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// Wait a bit to ensure the client has time to read before closing\n\t\ttime.Sleep(50 * time.Millisecond)\n\t\terrCh <- nil\n\t}()\n\n\t// Test establishing proxy tunnel with timeout\n\tconn, err := establishHTTPProxyTunnel(proxy.url(), targetHost, 10*time.Second, _EMPTY_, _EMPTY_)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to establish proxy tunnel: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\t// Test that we can communicate through the tunnel with deadlines\n\tconn.SetWriteDeadline(time.Now().Add(5 * time.Second))\n\t_, err = conn.Write([]byte(\"Hello\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to proxy tunnel: %v\", err)\n\t}\n\n\tconn.SetReadDeadline(time.Now().Add(5 * time.Second))\n\tbuffer := make([]byte, 1024)\n\tn, err := conn.Read(buffer)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read from proxy tunnel: %v\", err)\n\t}\n\n\tresponse := string(buffer[:n])\n\tif response != \"Hello from target server\" {\n\t\tt.Errorf(\"Unexpected response: '%s', expected 'Hello from target server'\", response)\n\t}\n\n\tselect {\n\tcase err := <-errCh:\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"%v\", err)\n\t\t}\n\tcase <-time.After(3 * time.Second):\n\t\tt.Fatal(\"Server goroutine didn't complete in time, but test data was exchanged successfully\")\n\t}\n}\n\nfunc TestLeafNodeHttpProxyTunnelWithAuth(t *testing.T) {\n\t// Create HTTP proxy with authentication and delay for robustness\n\tproxy := createTestHTTPProxy(\"testuser\", \"testpass\")\n\tproxy.setCloseDelay(200 * time.Millisecond)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Create a simple TCP server to connect to through proxy\n\tlistener, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test server: %v\", err)\n\t}\n\tdefer listener.Close()\n\n\ttargetPort := listener.Addr().(*net.TCPAddr).Port\n\ttargetHost := fmt.Sprintf(\"127.0.0.1:%d\", targetPort)\n\n\terrCh := make(chan error, 1)\n\n\t// Accept one connection with proper timeout handling\n\tgo func() {\n\t\tconn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"unable to accept: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tdefer conn.Close()\n\n\t\t// Set read deadline to prevent hanging\n\t\tconn.SetReadDeadline(time.Now().Add(5 * time.Second))\n\n\t\t// Read the incoming data first\n\t\tbuffer := make([]byte, 1024)\n\t\t_, err = conn.Read(buffer)\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"unable to read: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// Send response\n\t\tconn.SetWriteDeadline(time.Now().Add(5 * time.Second))\n\t\tif _, err := conn.Write([]byte(\"Hello from authenticated server\")); err != nil {\n\t\t\terrCh <- fmt.Errorf(\"unable to write: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\terrCh <- nil\n\t}()\n\n\t// Test establishing proxy tunnel with authentication and timeout\n\tconn, err := establishHTTPProxyTunnel(proxy.url(), targetHost, 10*time.Second, \"testuser\", \"testpass\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to establish proxy tunnel with auth: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\t// Test that we can communicate through the tunnel with deadlines\n\tconn.SetWriteDeadline(time.Now().Add(5 * time.Second))\n\t_, err = conn.Write([]byte(\"Hello\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to proxy tunnel: %v\", err)\n\t}\n\n\tconn.SetReadDeadline(time.Now().Add(5 * time.Second))\n\tbuffer := make([]byte, 1024)\n\tn, err := conn.Read(buffer)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read from proxy tunnel: %v\", err)\n\t}\n\n\tresponse := string(buffer[:n])\n\tif response != \"Hello from authenticated server\" {\n\t\tt.Errorf(\"Unexpected response: %s\", response)\n\t}\n\n\tselect {\n\tcase err := <-errCh:\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"%v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Server goroutine didn't complete in time, but test data was exchanged successfully\")\n\t}\n}\n\nfunc TestLeafNodeHttpProxyTunnelFailsWithoutAuth(t *testing.T) {\n\t// Create HTTP proxy with authentication required and delay for robustness\n\tproxy := createTestHTTPProxy(\"testuser\", \"testpass\")\n\tproxy.setCloseDelay(200 * time.Millisecond)\n\tproxy.start()\n\tdefer proxy.stop()\n\n\t// Try to establish tunnel without providing credentials (should fail quickly)\n\t_, err := establishHTTPProxyTunnel(proxy.url(), \"127.0.0.1:80\", 10*time.Second, _EMPTY_, _EMPTY_)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error when connecting without authentication\")\n\t}\n\n\tif !strings.Contains(err.Error(), \"proxy CONNECT failed\") {\n\t\tt.Errorf(\"Expected proxy authentication error, got: %v\", err)\n\t}\n\n\t// Verify the error contains the expected HTTP response code\n\tif !strings.Contains(err.Error(), \"407\") {\n\t\tt.Errorf(\"Expected HTTP 407 error in response, got: %v\", err)\n\t}\n}\n\n// TestLeafNodeProxyValidationProgrammatic tests proxy validation when configuring server programmatically\nfunc TestLeafNodeHttpProxyValidationProgrammatic(t *testing.T) {\n\ttests := []struct {\n\t\t// name is the name of the test.\n\t\tname string\n\n\t\t// setupOptions creates the Options configuration for the test.\n\t\tsetupOptions func() *Options\n\n\t\t// err is the expected error. nil means no error expected.\n\t\terr error\n\t}{\n\t\t{\n\t\t\tname: \"invalid proxy scheme\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"ftp://proxy.example.com:21\"\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"proxy URL scheme must be http or https\"),\n\t\t},\n\t\t{\n\t\t\tname: \"empty proxy URL - no validation performed\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = _EMPTY_\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: nil, // No error expected for empty URL\n\t\t},\n\t\t{\n\t\t\tname: \"invalid proxy URL parse failure\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"ht!tp://invalid-url-with-bad-characters\"\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"invalid proxy URL\"),\n\t\t},\n\t\t{\n\t\t\tname: \"missing proxy host\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://\"\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"proxy URL must specify a host\"),\n\t\t},\n\t\t{\n\t\t\tname: \"username without password\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://proxy.example.com:8080\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Username = \"user\"\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"proxy username and password must both be specified\"),\n\t\t},\n\t\t{\n\t\t\tname: \"password without username\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://proxy.example.com:8080\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Password = \"pass\"\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"proxy username and password must both be specified\"),\n\t\t},\n\t\t{\n\t\t\tname: \"negative timeout value\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://proxy.example.com:8080\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Timeout = -5 * time.Second\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: errors.New(\"proxy timeout must be >= 0\"),\n\t\t},\n\t\t{\n\t\t\tname: \"zero timeout value - valid\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://proxy.example.com:8080\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Timeout = 0\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: nil, // No error expected for zero timeout\n\t\t},\n\t\t{\n\t\t\tname: \"valid proxy configuration with authentication\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"http://proxy.example.com:8080\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Username = \"user\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Password = \"pass\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Timeout = 10 * time.Second\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: nil, // No error expected\n\t\t},\n\t\t{\n\t\t\tname: \"valid proxy configuration without authentication\",\n\t\t\tsetupOptions: func() *Options {\n\t\t\t\topts := &Options{}\n\t\t\t\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t\t{\n\t\t\t\t\t\tURLs: []*url.URL{{Scheme: wsSchemePrefix, Host: \"127.0.0.1:7422\"}},\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.URL = \"https://proxy.example.com:3128\"\n\t\t\t\topts.LeafNode.Remotes[0].Proxy.Timeout = 30 * time.Second\n\t\t\t\treturn opts\n\t\t\t},\n\t\t\terr: nil, // No error expected\n\t\t},\n\t}\n\n\tcheckErr := func(t *testing.T, err, expectedErr error) {\n\t\tt.Helper()\n\t\tswitch {\n\t\tcase err == nil && expectedErr == nil:\n\t\t\t// OK\n\t\tcase err != nil && expectedErr == nil:\n\t\t\tt.Errorf(\"Unexpected error after validating options: %s\", err)\n\t\tcase err == nil && expectedErr != nil:\n\t\t\tt.Errorf(\"Expected %q error after validating invalid options but got nothing\", expectedErr)\n\t\tcase err != nil && expectedErr != nil:\n\t\t\tif !strings.Contains(err.Error(), expectedErr.Error()) {\n\t\t\t\tt.Errorf(\"Expected error containing %q, got: %q\", expectedErr.Error(), err.Error())\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := test.setupOptions()\n\t\t\terr := validateLeafNode(opts)\n\t\t\tcheckErr(t, err, test.err)\n\t\t})\n\t}\n}\n" }, { "path": "server/leafnode_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nkeys\"\n\n\t\"github.com/klauspost/compress/s2\"\n\tjwt \"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n\t\"github.com/nats-io/nats-server/v2/internal/testhelper\"\n)\n\ntype captureLeafNodeRandomIPLogger struct {\n\tDummyLogger\n\tch chan struct{}\n\tips [3]int\n}\n\nfunc (c *captureLeafNodeRandomIPLogger) Debugf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"hostname_to_resolve\") {\n\t\tippos := strings.Index(msg, \"127.0.0.\")\n\t\tif ippos != -1 {\n\t\t\tn := int(msg[ippos+8] - '1')\n\t\t\tc.ips[n]++\n\t\t\tfor _, v := range c.ips {\n\t\t\t\tif v < 2 {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\t// All IPs got at least some hit, we are done.\n\t\t\tc.ch <- struct{}{}\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeRandomIP(t *testing.T) {\n\tu, err := url.Parse(\"nats://hostname_to_resolve:1234\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing: %v\", err)\n\t}\n\n\tresolver := &myDummyDNSResolver{ips: []string{\"127.0.0.1\", \"127.0.0.2\", \"127.0.0.3\"}}\n\n\to := DefaultOptions()\n\to.Host = \"127.0.0.1\"\n\to.Port = -1\n\to.LeafNode.Port = 0\n\to.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\to.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\to.LeafNode.resolver = resolver\n\to.LeafNode.dialTimeout = 15 * time.Millisecond\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tl := &captureLeafNodeRandomIPLogger{ch: make(chan struct{})}\n\ts.SetLogger(l, true, true)\n\n\tselect {\n\tcase <-l.ch:\n\tcase <-time.After(3 * time.Second):\n\t\tt.Fatalf(\"Does not seem to have used random IPs\")\n\t}\n}\n\nfunc TestLeafNodeRandomRemotes(t *testing.T) {\n\t// 16! possible permutations.\n\torderedURLs := make([]*url.URL, 0, 16)\n\tfor i := 0; i < cap(orderedURLs); i++ {\n\t\torderedURLs = append(orderedURLs, &url.URL{\n\t\t\tScheme: \"nats-leaf\",\n\t\t\tHost: fmt.Sprintf(\"host%d:7422\", i),\n\t\t})\n\t}\n\n\to := DefaultOptions()\n\to.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{NoRandomize: true},\n\t\t{NoRandomize: false},\n\t}\n\to.LeafNode.Remotes[0].URLs = make([]*url.URL, cap(orderedURLs))\n\tcopy(o.LeafNode.Remotes[0].URLs, orderedURLs)\n\to.LeafNode.Remotes[1].URLs = make([]*url.URL, cap(orderedURLs))\n\tcopy(o.LeafNode.Remotes[1].URLs, orderedURLs)\n\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\tr1 := s.leafRemoteCfgs[0]\n\tr2 := s.leafRemoteCfgs[1]\n\ts.mu.Unlock()\n\n\tr1.RLock()\n\tgotOrdered := r1.urls\n\tr1.RUnlock()\n\tif got, want := len(gotOrdered), len(orderedURLs); got != want {\n\t\tt.Fatalf(\"Unexpected rem0 len URLs, got %d, want %d\", got, want)\n\t}\n\n\t// These should be IN order.\n\tfor i := range orderedURLs {\n\t\tif got, want := gotOrdered[i].String(), orderedURLs[i].String(); got != want {\n\t\t\tt.Fatalf(\"Unexpected ordered url, got %s, want %s\", got, want)\n\t\t}\n\t}\n\n\tr2.RLock()\n\tgotRandom := r2.urls\n\tr2.RUnlock()\n\tif got, want := len(gotRandom), len(orderedURLs); got != want {\n\t\tt.Fatalf(\"Unexpected rem1 len URLs, got %d, want %d\", got, want)\n\t}\n\n\t// These should be OUT of order.\n\tvar random bool\n\tfor i := range orderedURLs {\n\t\tif gotRandom[i].String() != orderedURLs[i].String() {\n\t\t\trandom = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !random {\n\t\tt.Fatal(\"Expected urls to be random\")\n\t}\n}\n\ntype testLoopbackResolver struct{}\n\nfunc (r *testLoopbackResolver) LookupHost(ctx context.Context, host string) ([]string, error) {\n\treturn []string{\"127.0.0.1\"}, nil\n}\n\nfunc TestLeafNodeTLSWithCerts(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tu, err := url.Parse(fmt.Sprintf(\"nats://localhost:%d\", o1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/client.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/client-key.pem\"\n\t\t\t\t\t\ttimeout: 2\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String())))\n\to2, err := ProcessConfigFile(conf2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\to2.NoLog, o2.NoSigs = true, true\n\to2.LeafNode.resolver = &testLoopbackResolver{}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckFor(t, 3*time.Second, 10*time.Millisecond, func() error {\n\t\tif nln := s1.NumLeafNodes(); nln != 1 {\n\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeTLSRemoteWithNoCerts(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tu, err := url.Parse(fmt.Sprintf(\"nats://localhost:%d\", o1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String())))\n\to2, err := ProcessConfigFile(conf2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\n\tif len(o2.LeafNode.Remotes) == 0 {\n\t\tt.Fatal(\"Expected at least a single leaf remote\")\n\t}\n\n\tvar (\n\t\tgot float64 = o2.LeafNode.Remotes[0].TLSTimeout\n\t\texpected float64 = 5\n\t)\n\tif got != expected {\n\t\tt.Fatalf(\"Expected %v, got: %v\", expected, got)\n\t}\n\to2.NoLog, o2.NoSigs = true, true\n\to2.LeafNode.resolver = &testLoopbackResolver{}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckFor(t, 3*time.Second, 10*time.Millisecond, func() error {\n\t\tif nln := s1.NumLeafNodes(); nln != 1 {\n\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Here we only process options without starting the server\n\t// and without a root CA for the remote.\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\ttimeout: 10\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String())))\n\to3, err := ProcessConfigFile(conf3)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\n\tif len(o3.LeafNode.Remotes) == 0 {\n\t\tt.Fatal(\"Expected at least a single leaf remote\")\n\t}\n\tgot = o3.LeafNode.Remotes[0].TLSTimeout\n\texpected = 10\n\tif got != expected {\n\t\tt.Fatalf(\"Expected %v, got: %v\", expected, got)\n\t}\n\n\t// Here we only process options without starting the server\n\t// and check the default for leafnode remotes.\n\tconf4 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String())))\n\to4, err := ProcessConfigFile(conf4)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\n\tif len(o4.LeafNode.Remotes) == 0 {\n\t\tt.Fatal(\"Expected at least a single leaf remote\")\n\t}\n\tgot = o4.LeafNode.Remotes[0].TLSTimeout\n\texpected = float64(DEFAULT_LEAF_TLS_TIMEOUT) / float64(time.Second)\n\tif int(got) != int(expected) {\n\t\tt.Fatalf(\"Expected %v, got: %v\", expected, got)\n\t}\n}\n\ntype captureErrorLogger struct {\n\tDummyLogger\n\terrCh chan string\n}\n\nfunc (l *captureErrorLogger) Errorf(format string, v ...any) {\n\tselect {\n\tcase l.errCh <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\nfunc TestLeafNodeAccountNotFound(t *testing.T) {\n\tob := DefaultOptions()\n\tob.LeafNode.Host = \"127.0.0.1\"\n\tob.LeafNode.Port = -1\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.LeafNode.Port))\n\n\toa := DefaultOptions()\n\toa.Cluster.Name = \"xyz\"\n\toa.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\toa.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tLocalAccount: \"foo\",\n\t\t\tURLs: []*url.URL{u},\n\t\t},\n\t}\n\t// Expected to fail\n\tif _, err := NewServer(oa); err == nil || !strings.Contains(err.Error(), \"local account\") {\n\t\tt.Fatalf(\"Expected server to fail with error about no local account, got %v\", err)\n\t}\n\toa.Accounts = []*Account{NewAccount(\"foo\")}\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 1)}\n\tsa.SetLogger(l, false, false)\n\n\tcheckLeafNodeConnected(t, sa)\n\n\t// Now simulate account is removed with config reload, or it expires.\n\tsa.accounts.Delete(\"foo\")\n\n\t// Restart B (with same Port)\n\tsb.Shutdown()\n\tsb = RunServer(ob)\n\tdefer sb.Shutdown()\n\n\t// Wait for report of error\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif !strings.Contains(e, \"Unable to lookup account\") {\n\t\t\tt.Fatalf(\"Expected error about no local account, got %s\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get the error\")\n\t}\n\n\t// TODO below test is bogus. Instead add the account, do a reload, and make sure the connection works.\n\n\t// For now, sa would try to recreate the connection for ever.\n\t// Check that lid is increasing...\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif lid := atomic.LoadUint64(&sa.gcid); lid < 3 {\n\t\t\treturn fmt.Errorf(\"Seems like connection was not retried, lid currently only at %d\", lid)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// This test ensures that we can connect using proper user/password\n// to a LN URL that was discovered through the INFO protocol.\n// We also check that the password doesn't leak to debug/trace logs.\nfunc TestLeafNodeBasicAuthFailover(t *testing.T) {\n\t// Something a little longer than \"pwd\" to prevent false positives amongst many log lines;\n\t// don't make it complex enough to be subject to %-escaping, we want a simple needle search.\n\tfatalPassword := \"pwdfatal\"\n\n\tcontent := `\n\tlisten: \"127.0.0.1:-1\"\n\tcluster {\n\t\tname: \"abc\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\t%s\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tauthorization {\n\t\t\tuser: foo\n\t\t\tpassword: %s\n\t\t\ttimeout: 1\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(content, \"\", fatalPassword)))\n\n\tsb1, ob1 := RunServerWithConfig(conf)\n\tdefer sb1.Shutdown()\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(content, fmt.Sprintf(\"routes: [nats://127.0.0.1:%d]\", ob1.Cluster.Port), fatalPassword)))\n\n\tsb2, _ := RunServerWithConfig(conf)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tcontent = `\n\tport: -1\n\taccounts {\n\t\tfoo {}\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tremotes [\n\t\t\t{\n\t\t\t\taccount: \"foo\"\n\t\t\t\turl: \"nats://foo:%s@127.0.0.1:%d\"\n\t\t\t}\n\t\t]\n\t}\n\t`\n\tconf = createConfFile(t, []byte(fmt.Sprintf(content, fatalPassword, ob1.LeafNode.Port)))\n\n\tsa, _ := RunServerWithConfig(conf)\n\tdefer sa.Shutdown()\n\n\tl := testhelper.NewDummyLogger(100)\n\tsa.SetLogger(l, true, true) // we want debug & trace logs, to check for passwords in them\n\n\tcheckLeafNodeConnected(t, sa)\n\n\t// Shutdown sb1, sa should reconnect to sb2\n\tsb1.Shutdown()\n\n\t// Wait a bit to make sure there was a disconnect and attempt to reconnect.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Should be able to reconnect\n\tcheckLeafNodeConnected(t, sa)\n\n\t// Look at all our logs for the password; at time of writing it doesn't appear\n\t// but we want to safe-guard against it.\n\tl.CheckForProhibited(t, \"fatal password\", fatalPassword)\n}\n\nfunc TestLeafNodeRTT(t *testing.T) {\n\tob := DefaultOptions()\n\tob.PingInterval = 15 * time.Millisecond\n\tob.LeafNode.Host = \"127.0.0.1\"\n\tob.LeafNode.Port = -1\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tlnBURL, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.LeafNode.Port))\n\toa := DefaultOptions()\n\toa.Cluster.Name = \"xyz\"\n\toa.PingInterval = 15 * time.Millisecond\n\toa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lnBURL}}}\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tcheckLeafNodeConnected(t, sa)\n\n\tcheckRTT := func(t *testing.T, s *Server) time.Duration {\n\t\tt.Helper()\n\t\tvar ln *client\n\t\ts.mu.Lock()\n\t\tfor _, l := range s.leafs {\n\t\t\tln = l\n\t\t\tbreak\n\t\t}\n\t\ts.mu.Unlock()\n\t\tvar rtt time.Duration\n\t\tcheckFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error {\n\t\t\tln.mu.Lock()\n\t\t\trtt = ln.rtt\n\t\t\tln.mu.Unlock()\n\t\t\tif rtt == 0 {\n\t\t\t\treturn fmt.Errorf(\"RTT not tracked\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn rtt\n\t}\n\n\tprevA := checkRTT(t, sa)\n\tprevB := checkRTT(t, sb)\n\n\t// Wait to see if RTT is updated\n\tcheckUpdated := func(t *testing.T, s *Server, prev time.Duration) {\n\t\tattempts := 0\n\t\ttimeout := time.Now().Add(2 * firstPingInterval)\n\t\tfor time.Now().Before(timeout) {\n\t\t\tif rtt := checkRTT(t, s); rtt != prev {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tattempts++\n\t\t\tif attempts == 5 {\n\t\t\t\ts.mu.Lock()\n\t\t\t\tfor _, ln := range s.leafs {\n\t\t\t\t\tln.mu.Lock()\n\t\t\t\t\tln.rtt = 0\n\t\t\t\t\tln.mu.Unlock()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ts.mu.Unlock()\n\t\t\t}\n\t\t\ttime.Sleep(15 * time.Millisecond)\n\t\t}\n\t\tt.Fatalf(\"RTT probably not updated\")\n\t}\n\tcheckUpdated(t, sa, prevA)\n\tcheckUpdated(t, sb, prevB)\n\n\tsa.Shutdown()\n\tsb.Shutdown()\n\n\t// Now check that initial RTT is computed prior to first PingInterval\n\t// Get new options to avoid possible race changing the ping interval.\n\tob = DefaultOptions()\n\tob.Cluster.Name = \"xyz\"\n\tob.PingInterval = time.Minute\n\tob.LeafNode.Host = \"127.0.0.1\"\n\tob.LeafNode.Port = -1\n\tsb = RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tlnBURL, _ = url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.LeafNode.Port))\n\toa = DefaultOptions()\n\toa.PingInterval = time.Minute\n\toa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{lnBURL}}}\n\tsa = RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tcheckLeafNodeConnected(t, sa)\n\n\tcheckRTT(t, sa)\n\tcheckRTT(t, sb)\n}\n\nfunc TestLeafNodeValidateAuthOptions(t *testing.T) {\n\topts := DefaultOptions()\n\topts.LeafNode.Username = \"user1\"\n\topts.LeafNode.Password = \"pwd\"\n\topts.LeafNode.Users = []*User{{Username: \"user\", Password: \"pwd\"}}\n\tif _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(),\n\t\t\"can not have a single user/pass and a users array\") {\n\t\tt.Fatalf(\"Expected error about mixing single/multi users, got %v\", err)\n\t}\n\n\t// Check duplicate user names\n\topts.LeafNode.Username = _EMPTY_\n\topts.LeafNode.Password = _EMPTY_\n\topts.LeafNode.Users = append(opts.LeafNode.Users, &User{Username: \"user\", Password: \"pwd\"})\n\tif _, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), \"duplicate user\") {\n\t\tt.Fatalf(\"Expected error about duplicate user, got %v\", err)\n\t}\n}\n\nfunc TestLeafNodeBasicAuthSingleton(t *testing.T) {\n\topts := DefaultOptions()\n\topts.LeafNode.Port = -1\n\topts.LeafNode.Account = \"unknown\"\n\tif s, err := NewServer(opts); err == nil || !strings.Contains(err.Error(), \"cannot find\") {\n\t\tif s != nil {\n\t\t\ts.Shutdown()\n\t\t}\n\t\tt.Fatalf(\"Expected error about account not found, got %v\", err)\n\t}\n\n\ttemplate := `\n\t\tport: -1\n\t\taccounts: {\n\t\t\tACC1: { users = [{user: \"user1\", password: \"user1\"}] }\n\t\t\tACC2: { users = [{user: \"user2\", password: \"user2\"}] }\n\t\t}\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t %s\n account: \"ACC1\"\n }\n\t\t}\n\t`\n\tfor iter, test := range []struct {\n\t\tname string\n\t\tuserSpec string\n\t\tlnURLCreds string\n\t\tshouldFail bool\n\t}{\n\t\t{\"user creds required and no user so fails\", \"\", \"\", true},\n\t\t{\"user creds required and pick user2 associated to ACC2\", \"\", \"user2:user2@\", false},\n\t\t{\"user creds required and unknown user should fail\", \"\", \"unknown:user@\", true},\n\t\t{\"user creds required so binds to ACC1\", \"user: \\\"ln\\\"\\npass: \\\"pwd\\\"\", \"ln:pwd@\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(template, test.userSpec)))\n\t\t\ts1, o1 := RunServerWithConfig(conf)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\t// Create a sub on \"foo\" for account ACC1 (user user1), which is the one\n\t\t\t// bound to the accepted LN connection.\n\t\t\tncACC1 := natsConnect(t, fmt.Sprintf(\"nats://user1:user1@%s:%d\", o1.Host, o1.Port))\n\t\t\tdefer ncACC1.Close()\n\t\t\tsub1 := natsSubSync(t, ncACC1, \"foo\")\n\t\t\tnatsFlush(t, ncACC1)\n\n\t\t\t// Create a sub on \"foo\" for account ACC2 (user user2). This one should\n\t\t\t// not receive any message.\n\t\t\tncACC2 := natsConnect(t, fmt.Sprintf(\"nats://user2:user2@%s:%d\", o1.Host, o1.Port))\n\t\t\tdefer ncACC2.Close()\n\t\t\tsub2 := natsSubSync(t, ncACC2, \"foo\")\n\t\t\tnatsFlush(t, ncACC2)\n\n\t\t\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tleafnodes: {\n\t\t\t\t\tremotes = [ { url: \"nats-leaf://%s%s:%d\" } ]\n\t\t\t\t}\n\t\t\t`, test.lnURLCreds, o1.LeafNode.Host, o1.LeafNode.Port)))\n\t\t\ts2, _ := RunServerWithConfig(conf)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tif test.shouldFail {\n\t\t\t\t// Wait a bit and ensure that there is no leaf node connection\n\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\tif n := s1.NumLeafNodes(); n != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected no leafnode connection, got %v\", n)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tcheckLeafNodeConnected(t, s2)\n\n\t\t\tnc := natsConnect(t, s2.ClientURL())\n\t\t\tdefer nc.Close()\n\t\t\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\t\t\t// If url contains known user, even when there is no credentials\n\t\t\t// required, the connection will be bound to the user's account.\n\t\t\tif iter == 1 {\n\t\t\t\t// Should not receive on \"ACC1\", but should on \"ACC2\"\n\t\t\t\tif _, err := sub1.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected timeout error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tnatsNexMsg(t, sub2, time.Second)\n\t\t\t} else {\n\t\t\t\t// Should receive on \"ACC1\"...\n\t\t\t\tnatsNexMsg(t, sub1, time.Second)\n\t\t\t\t// but not received on \"ACC2\" since leafnode bound to account \"ACC1\".\n\t\t\t\tif _, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected timeout error, got %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeBasicAuthMultiple(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts: {\n\t\t\tS1ACC1: { users = [{user: \"user1\", password: \"user1\"}] }\n\t\t\tS1ACC2: { users = [{user: \"user2\", password: \"user2\"}] }\n\t\t}\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t users = [\n\t\t\t\t {user: \"ln1\", password: \"ln1\", account: \"S1ACC1\"}\n\t\t\t\t {user: \"ln2\", password: \"ln2\", account: \"S1ACC2\"}\n\t\t\t\t {user: \"ln3\", password: \"ln3\"}\n\t\t\t ]\n }\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf)\n\tdefer s1.Shutdown()\n\n\t// Make sure that we reject a LN connection if user does not match\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tremotes = [{url: \"nats-leaf://wron:user@%s:%d\"}]\n\t\t}\n\t`, o1.LeafNode.Host, o1.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(conf)\n\tdefer s2.Shutdown()\n\t// Give a chance for s2 to attempt to connect and make sure that s1\n\t// did not register a LN connection.\n\ttime.Sleep(100 * time.Millisecond)\n\tif n := s1.NumLeafNodes(); n != 0 {\n\t\tt.Fatalf(\"Expected no leafnode connection, got %v\", n)\n\t}\n\ts2.Shutdown()\n\n\tncACC1 := natsConnect(t, fmt.Sprintf(\"nats://user1:user1@%s:%d\", o1.Host, o1.Port))\n\tdefer ncACC1.Close()\n\tsub1 := natsSubSync(t, ncACC1, \"foo\")\n\tnatsFlush(t, ncACC1)\n\n\tncACC2 := natsConnect(t, fmt.Sprintf(\"nats://user2:user2@%s:%d\", o1.Host, o1.Port))\n\tdefer ncACC2.Close()\n\tsub2 := natsSubSync(t, ncACC2, \"foo\")\n\tnatsFlush(t, ncACC2)\n\n\t// We will start s2 with 2 LN connections that should bind local account S2ACC1\n\t// to account S1ACC1 and S2ACC2 to account S1ACC2 on s1.\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tS2ACC1 { users = [{user: \"user1\", password: \"user1\"}] }\n\t\t\tS2ACC2 { users = [{user: \"user2\", password: \"user2\"}] }\n\t\t}\n\t\tleafnodes: {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats-leaf://ln1:ln1@%s:%d\"\n\t\t\t\t\taccount: \"S2ACC1\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: \"nats-leaf://ln2:ln2@%s:%d\"\n\t\t\t\t\taccount: \"S2ACC2\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.LeafNode.Host, o1.LeafNode.Port, o1.LeafNode.Host, o1.LeafNode.Port)))\n\ts2, o2 := RunServerWithConfig(conf)\n\tdefer s2.Shutdown()\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif nln := s2.NumLeafNodes(); nln != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 connected leafnodes for server %q, got %d\", s2.ID(), nln)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Create a user connection on s2 that binds to S2ACC1 (use user1).\n\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://user1:user1@%s:%d\", o2.Host, o2.Port))\n\tdefer nc1.Close()\n\n\t// Create an user connection on s2 that binds to S2ACC2 (use user2).\n\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://user2:user2@%s:%d\", o2.Host, o2.Port))\n\tdefer nc2.Close()\n\n\t// Now if a message is published from nc1, sub1 should receive it since\n\t// their account are bound together.\n\tnatsPub(t, nc1, \"foo\", []byte(\"hello\"))\n\tnatsNexMsg(t, sub1, time.Second)\n\t// But sub2 should not receive it since different account.\n\tif _, err := sub2.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout error, got %v\", err)\n\t}\n\n\t// Now use nc2 (S2ACC2) to publish\n\tnatsPub(t, nc2, \"foo\", []byte(\"hello\"))\n\t// Expect sub2 to receive and sub1 not to.\n\tnatsNexMsg(t, sub2, time.Second)\n\tif _, err := sub1.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected timeout error, got %v\", err)\n\t}\n\n\t// Now check that we don't panic if no account is specified for\n\t// a given user.\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tremotes = [\n\t\t\t\t{ url: \"nats-leaf://ln3:ln3@%s:%d\" }\n\t\t\t]\n\t\t}\n\t`, o1.LeafNode.Host, o1.LeafNode.Port)))\n\ts3, _ := RunServerWithConfig(conf)\n\tdefer s3.Shutdown()\n}\n\ntype loopDetectedLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *loopDetectedLogger) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"Loop\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeLoop(t *testing.T) {\n\ttest := func(t *testing.T, cluster bool) {\n\t\t// This test requires that we set the port to known value because\n\t\t// we want A point to B and B to A.\n\t\toa := DefaultOptions()\n\t\toa.ServerName = \"A\"\n\t\tif !cluster {\n\t\t\toa.Cluster.Port = 0\n\t\t\toa.Cluster.Name = _EMPTY_\n\t\t}\n\t\toa.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\t\toa.LeafNode.Port = 1234\n\t\tub, _ := url.Parse(\"nats://127.0.0.1:5678\")\n\t\toa.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ub}}}\n\t\toa.LeafNode.connDelay = 50 * time.Millisecond\n\t\tsa := RunServer(oa)\n\t\tdefer sa.Shutdown()\n\n\t\tla := &loopDetectedLogger{ch: make(chan string, 1)}\n\t\tsa.SetLogger(la, false, false)\n\n\t\tob := DefaultOptions()\n\t\tob.ServerName = \"B\"\n\t\tif !cluster {\n\t\t\tob.Cluster.Port = 0\n\t\t\tob.Cluster.Name = _EMPTY_\n\t\t} else {\n\t\t\tob.Cluster.Name = \"xyz\"\n\t\t}\n\t\tob.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\t\tob.LeafNode.Port = 5678\n\t\tua, _ := url.Parse(\"nats://127.0.0.1:1234\")\n\t\tob.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}}\n\t\tob.LeafNode.connDelay = 50 * time.Millisecond\n\t\tsb := RunServer(ob)\n\t\tdefer sb.Shutdown()\n\n\t\tlb := &loopDetectedLogger{ch: make(chan string, 1)}\n\t\tsb.SetLogger(lb, false, false)\n\n\t\tselect {\n\t\tcase <-la.ch:\n\t\t\t// OK!\n\t\tcase <-lb.ch:\n\t\t\t// OK!\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not get any error regarding loop\")\n\t\t}\n\n\t\tsb.Shutdown()\n\t\tob.Port = -1\n\t\tob.Cluster.Port = -1\n\t\tob.LeafNode.Remotes = nil\n\t\tsb = RunServer(ob)\n\t\tdefer sb.Shutdown()\n\n\t\tcheckLeafNodeConnected(t, sa)\n\t}\n\tt.Run(\"standalone\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"cluster\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestLeafNodeLoopFromDAG(t *testing.T) {\n\t// We want B & C to point to A, A itself does not point to any other server.\n\t// We need to cancel clustering since now this will suppress on its own.\n\toa := DefaultOptions()\n\toa.ServerName = \"A\"\n\toa.LeafNode.connDelay = 50 * time.Millisecond\n\toa.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\toa.LeafNode.Port = -1\n\toa.Cluster = ClusterOpts{}\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tua, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", oa.LeafNode.Port))\n\n\t// B will point to A\n\tob := DefaultOptions()\n\tob.ServerName = \"B\"\n\tob.LeafNode.connDelay = 50 * time.Millisecond\n\tob.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\tob.LeafNode.Port = -1\n\tob.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}}\n\tob.Cluster = ClusterOpts{}\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tub, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.LeafNode.Port))\n\n\tcheckLeafNodeConnected(t, sa)\n\tcheckLeafNodeConnected(t, sb)\n\n\t// C will point to A and B\n\toc := DefaultOptions()\n\toc.ServerName = \"C\"\n\toc.LeafNode.connDelay = 50 * time.Millisecond\n\toc.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\toc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ua}}, {URLs: []*url.URL{ub}}}\n\toc.LeafNode.connDelay = 100 * time.Millisecond // Allow logger to be attached before connecting.\n\toc.Cluster = ClusterOpts{}\n\tsc := RunServer(oc)\n\n\tlc := &loopDetectedLogger{ch: make(chan string, 1)}\n\tsc.SetLogger(lc, false, false)\n\n\t// We should get an error.\n\tselect {\n\tcase <-lc.ch:\n\t\t// OK\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get any error regarding loop\")\n\t}\n\n\t// C should not be connected to anything.\n\tcheckLeafNodeConnectedCount(t, sc, 0)\n\t// A and B are connected to each other.\n\tcheckLeafNodeConnectedCount(t, sa, 1)\n\tcheckLeafNodeConnectedCount(t, sb, 1)\n\n\t// Shutdown C and restart without the loop.\n\tsc.Shutdown()\n\toc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{ub}}}\n\n\tsc = RunServer(oc)\n\tdefer sc.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, sa, 1)\n\tcheckLeafNodeConnectedCount(t, sb, 2)\n\tcheckLeafNodeConnectedCount(t, sc, 1)\n}\n\nfunc TestLeafNodeCloseTLSConnection(t *testing.T) {\n\topts := DefaultOptions()\n\topts.DisableShortFirstPing = true\n\topts.LeafNode.Host = \"127.0.0.1\"\n\topts.LeafNode.Port = -1\n\topts.LeafNode.TLSTimeout = 100\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server.pem\",\n\t\tKeyFile: \"./configs/certs/key.pem\",\n\t\tInsecure: true,\n\t}\n\ttlsConf, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\topts.LeafNode.TLSConfig = tlsConf\n\topts.NoLog = true\n\topts.NoSigs = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tendpoint := net.JoinHostPort(opts.LeafNode.Host, fmt.Sprintf(\"%d\", opts.LeafNode.Port))\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on dial: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\tbr := bufio.NewReaderSize(conn, 100)\n\tif _, err := br.ReadString('\\n'); err != nil {\n\t\tt.Fatalf(\"Unexpected error reading INFO: %v\", err)\n\t}\n\n\ttlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})\n\tdefer tlsConn.Close()\n\tif err := tlsConn.Handshake(); err != nil {\n\t\tt.Fatalf(\"Unexpected error during handshake: %v\", err)\n\t}\n\tconnectOp := []byte(\"CONNECT {\\\"name\\\":\\\"leaf\\\",\\\"verbose\\\":false,\\\"pedantic\\\":false}\\r\\n\")\n\tif _, err := tlsConn.Write(connectOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tif _, err := tlsConn.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing PING: %v\", err)\n\t}\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Get leaf connection\n\tvar leaf *client\n\ts.mu.Lock()\n\tfor _, l := range s.leafs {\n\t\tleaf = l\n\t\tbreak\n\t}\n\ts.mu.Unlock()\n\t// Fill the buffer. We want to timeout on write so that nc.Close()\n\t// would block due to a write that cannot complete.\n\tbuf := make([]byte, 64*1024)\n\tdone := false\n\tfor !done {\n\t\tleaf.nc.SetWriteDeadline(time.Now().Add(time.Second))\n\t\tif _, err := leaf.nc.Write(buf); err != nil {\n\t\t\tdone = true\n\t\t}\n\t\tleaf.nc.SetWriteDeadline(time.Time{})\n\t}\n\tch := make(chan bool)\n\tgo func() {\n\t\tselect {\n\t\tcase <-ch:\n\t\t\treturn\n\t\tcase <-time.After(3 * time.Second):\n\t\t\tfmt.Println(\"!!!! closeConnection is blocked, test will hang !!!\")\n\t\t\treturn\n\t\t}\n\t}()\n\t// Close the route\n\tleaf.closeConnection(SlowConsumerWriteDeadline)\n\tch <- true\n}\n\nfunc TestLeafNodeTLSSaveName(t *testing.T) {\n\topts := DefaultOptions()\n\topts.LeafNode.Host = \"127.0.0.1\"\n\topts.LeafNode.Port = -1\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-noip.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key-noip.pem\",\n\t\tInsecure: true,\n\t}\n\ttlsConf, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\topts.LeafNode.TLSConfig = tlsConf\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tlo := DefaultOptions()\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://localhost:%d\", opts.LeafNode.Port))\n\tlo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\tlo.LeafNode.ReconnectInterval = 15 * time.Millisecond\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\t// We know connection will fail, but it should not fail because of error such as:\n\t// \"cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs\"\n\t// This would mean that we are not saving the hostname to use during the TLS handshake.\n\n\tle := &captureErrorLogger{errCh: make(chan string, 100)}\n\tln.SetLogger(le, false, false)\n\n\ttm := time.NewTimer(time.Second)\n\tvar done bool\n\tfor !done {\n\t\tselect {\n\t\tcase err := <-le.errCh:\n\t\t\tif strings.Contains(err, \"doesn't contain any IP SANs\") {\n\t\t\t\tt.Fatalf(\"Got this error: %q\", err)\n\t\t\t}\n\t\tcase <-tm.C:\n\t\t\tdone = true\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeRemoteWrongPort(t *testing.T) {\n\tfor _, test1 := range []struct {\n\t\tname string\n\t\tclusterAdvertise bool\n\t\tleafnodeAdvertise bool\n\t}{\n\t\t{\"advertise_on\", false, false},\n\t\t{\"cluster_no_advertise\", true, false},\n\t\t{\"leafnode_no_advertise\", false, true},\n\t} {\n\t\tt.Run(test1.name, func(t *testing.T) {\n\t\t\toa := DefaultOptions()\n\t\t\t// Make sure we have all ports (client, route, gateway) and we will try\n\t\t\t// to create a leafnode to connection to each and make sure we get the error.\n\t\t\toa.Cluster.NoAdvertise = test1.clusterAdvertise\n\t\t\toa.Cluster.Name = \"A\"\n\t\t\toa.Cluster.Host = \"127.0.0.1\"\n\t\t\toa.Cluster.Port = -1\n\t\t\toa.Gateway.Host = \"127.0.0.1\"\n\t\t\toa.Gateway.Port = -1\n\t\t\toa.Gateway.Name = \"A\"\n\t\t\toa.LeafNode.Host = \"127.0.0.1\"\n\t\t\toa.LeafNode.Port = -1\n\t\t\toa.LeafNode.NoAdvertise = test1.leafnodeAdvertise\n\t\t\toa.Accounts = []*Account{NewAccount(\"sys\")}\n\t\t\toa.SystemAccount = \"sys\"\n\t\t\tsa := RunServer(oa)\n\t\t\tdefer sa.Shutdown()\n\n\t\t\tob := DefaultOptions()\n\t\t\tob.Cluster.NoAdvertise = test1.clusterAdvertise\n\t\t\tob.Cluster.Name = \"A\"\n\t\t\tob.Cluster.Host = \"127.0.0.1\"\n\t\t\tob.Cluster.Port = -1\n\t\t\tob.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa.Cluster.Host, oa.Cluster.Port))\n\t\t\tob.Gateway.Host = \"127.0.0.1\"\n\t\t\tob.Gateway.Port = -1\n\t\t\tob.Gateway.Name = \"A\"\n\t\t\tob.LeafNode.Host = \"127.0.0.1\"\n\t\t\tob.LeafNode.Port = -1\n\t\t\tob.LeafNode.NoAdvertise = test1.leafnodeAdvertise\n\t\t\tob.Accounts = []*Account{NewAccount(\"sys\")}\n\t\t\tob.SystemAccount = \"sys\"\n\t\t\tsb := RunServer(ob)\n\t\t\tdefer sb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sa, sb)\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tport int\n\t\t\t}{\n\t\t\t\t{\"client\", oa.Port},\n\t\t\t\t{\"cluster\", oa.Cluster.Port},\n\t\t\t\t{\"gateway\", oa.Gateway.Port},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\toc := DefaultOptions()\n\t\t\t\t\t// Server with the wrong config against non leafnode port.\n\t\t\t\t\tleafURL, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", test.port))\n\t\t\t\t\toc.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{leafURL}}}\n\t\t\t\t\toc.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\t\t\t\t\tsc := RunServer(oc)\n\t\t\t\t\tdefer sc.Shutdown()\n\t\t\t\t\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\t\t\t\t\tsc.SetLogger(l, true, true)\n\n\t\t\t\t\tselect {\n\t\t\t\t\tcase e := <-l.errCh:\n\t\t\t\t\t\tif strings.Contains(e, ErrConnectedToWrongPort.Error()) {\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\tcase <-time.After(2 * time.Second):\n\t\t\t\t\t\tt.Fatalf(\"Did not get any error about connecting to wrong port for %q - %q\",\n\t\t\t\t\t\t\ttest1.name, test.name)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeRemoteIsHub(t *testing.T) {\n\toa := testDefaultOptionsForGateway(\"A\")\n\toa.Accounts = []*Account{NewAccount(\"sys\")}\n\toa.SystemAccount = \"sys\"\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tlno := DefaultOptions()\n\tlno.LeafNode.Host = \"127.0.0.1\"\n\tlno.LeafNode.Port = -1\n\tln := RunServer(lno)\n\tdefer ln.Shutdown()\n\n\tob1 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\tob1.Accounts = []*Account{NewAccount(\"sys\")}\n\tob1.SystemAccount = \"sys\"\n\tob1.Cluster.Host = \"127.0.0.1\"\n\tob1.Cluster.Port = -1\n\tob1.LeafNode.Host = \"127.0.0.1\"\n\tob1.LeafNode.Port = -1\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", lno.LeafNode.Port))\n\tob1.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tHub: true,\n\t\t},\n\t}\n\tsb1 := RunServer(ob1)\n\tdefer sb1.Shutdown()\n\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\n\tcheckLeafNodeConnected(t, sb1)\n\n\t// For now, due to issue 977, let's restart the leafnode so that the\n\t// leafnode connect is propagated in the super-cluster.\n\tln.Shutdown()\n\tln = RunServer(lno)\n\tdefer ln.Shutdown()\n\tcheckLeafNodeConnected(t, sb1)\n\n\t// Connect another server in cluster B\n\tob2 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa)\n\tob2.Accounts = []*Account{NewAccount(\"sys\")}\n\tob2.SystemAccount = \"sys\"\n\tob2.Cluster.Host = \"127.0.0.1\"\n\tob2.Cluster.Port = -1\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tsb2 := RunServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\n\texpectedSubs := ln.NumSubscriptions() + 2\n\n\t// Create sub on \"foo\" connected to sa\n\tncA := natsConnect(t, sa.ClientURL())\n\tdefer ncA.Close()\n\tsubFoo := natsSubSync(t, ncA, \"foo\")\n\n\t// Create sub on \"bar\" connected to sb2\n\tncB2 := natsConnect(t, sb2.ClientURL())\n\tdefer ncB2.Close()\n\tsubBar := natsSubSync(t, ncB2, \"bar\")\n\n\t// Make sure subscriptions have propagated to the leafnode.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif subs := ln.NumSubscriptions(); subs < expectedSubs {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d\", subs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Create pub connection on leafnode\n\tncLN := natsConnect(t, ln.ClientURL())\n\tdefer ncLN.Close()\n\n\t// Publish on foo and make sure it is received.\n\tnatsPub(t, ncLN, \"foo\", []byte(\"msg\"))\n\tnatsNexMsg(t, subFoo, time.Second)\n\n\t// Publish on foo and make sure it is received.\n\tnatsPub(t, ncLN, \"bar\", []byte(\"msg\"))\n\tnatsNexMsg(t, subBar, time.Second)\n}\n\nfunc TestLeafNodePermissions(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tln1 := RunServer(lo1)\n\tdefer ln1.Shutdown()\n\n\terrLog := &captureErrorLogger{errCh: make(chan string, 1)}\n\tln1.SetLogger(errLog, false, false)\n\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", lo1.LeafNode.Host, lo1.LeafNode.Port))\n\tlo2 := DefaultOptions()\n\tlo2.Cluster.Name = \"xyz\"\n\tlo2.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tlo2.LeafNode.connDelay = 100 * time.Millisecond\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tDenyExports: []string{\"export.*\", \"export\"},\n\t\t\tDenyImports: []string{\"import.*\", \"import\"},\n\t\t},\n\t}\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\tcheckLeafNodeConnected(t, ln1)\n\n\t// Create clients on ln1 and ln2\n\tnc1, err := nats.Connect(ln1.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc1.Close()\n\tnc2, err := nats.Connect(ln2.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckSubs := func(acc *Account, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := acc.TotalSubs(); n != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d subs, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Create a sub on \">\" on LN1\n\tsubAll := natsSubSync(t, nc1, \">\")\n\t// this should be registered in LN2 (there is 1 sub for LN1 $LDS subject) + SYS IMPORTS\n\tcheckSubs(ln2.globalAccount(), 12)\n\n\t// Check deny export clause from messages published from LN2\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsubject string\n\t\treceived bool\n\t}{\n\t\t{\"do not send on export.bat\", \"export.bat\", false},\n\t\t{\"do not send on export\", \"export\", false},\n\t\t{\"send on foo\", \"foo\", true},\n\t\t{\"send on export.this.one\", \"export.this.one\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tnc2.Publish(test.subject, []byte(\"msg\"))\n\t\t\tif test.received {\n\t\t\t\tnatsNexMsg(t, subAll, time.Second)\n\t\t\t} else {\n\t\t\t\tif _, err := subAll.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\tt.Fatalf(\"Should not have received message on %q\", test.subject)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tsubAll.Unsubscribe()\n\t// Goes down by 1.\n\tcheckSubs(ln2.globalAccount(), 11)\n\n\t// We used to make sure we would not do subscriptions however that\n\t// was incorrect. We need to check publishes, not the subscriptions.\n\t// For instance if we can publish across a leafnode to foo, and the\n\t// other side has a subsxcription for '*' the message should cross\n\t// the leafnode. The old way would not allow this.\n\n\t// Now check deny import clause.\n\t// As of now, we don't suppress forwarding of subscriptions on LN2 that\n\t// match the deny import clause to be forwarded to LN1. However, messages\n\t// should still not be able to come back to LN2.\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsubSubject string\n\t\tpubSubject string\n\t\tok bool\n\t}{\n\t\t{\"reject import on import.*\", \"import.*\", \"import.bad\", false},\n\t\t{\"reject import on import\", \"import\", \"import\", false},\n\t\t{\"accepts import on foo\", \"foo\", \"foo\", true},\n\t\t{\"accepts import on import.this.one.ok\", \"import.*.>\", \"import.this.one.ok\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsub := natsSubSync(t, nc2, test.subSubject)\n\t\t\tcheckSubs(ln2.globalAccount(), 12)\n\n\t\t\tif !test.ok {\n\t\t\t\tnc1.Publish(test.pubSubject, []byte(\"msg\"))\n\t\t\t\tif _, err := sub.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\tt.Fatalf(\"Did not expect to get the message\")\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tcheckSubs(ln1.globalAccount(), 11)\n\t\t\t\tnc1.Publish(test.pubSubject, []byte(\"msg\"))\n\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t\t}\n\t\t\tsub.Unsubscribe()\n\t\t\tcheckSubs(ln1.globalAccount(), 10)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodePermissionsConcurrentAccess(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tln1 := RunServer(lo1)\n\tdefer ln1.Shutdown()\n\n\tnc1 := natsConnect(t, ln1.ClientURL())\n\tdefer nc1.Close()\n\n\tnatsSub(t, nc1, \"_INBOX.>\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc1)\n\n\tch := make(chan struct{}, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\n\tpublish := func(nc *nats.Conn) {\n\t\tdefer wg.Done()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tnc.Publish(nats.NewInbox(), []byte(\"hello\"))\n\t\t\t}\n\t\t}\n\t}\n\n\tgo publish(nc1)\n\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", lo1.LeafNode.Host, lo1.LeafNode.Port))\n\tlo2 := DefaultOptions()\n\tlo2.Cluster.Name = \"xyz\"\n\tlo2.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tlo2.LeafNode.connDelay = 500 * time.Millisecond\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tDenyExports: []string{\"foo\"},\n\t\t\tDenyImports: []string{\"bar\"},\n\t\t},\n\t}\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\tnc2 := natsConnect(t, ln2.ClientURL())\n\tdefer nc2.Close()\n\n\tnatsSub(t, nc2, \"_INBOX.>\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc2)\n\n\tgo publish(nc2)\n\n\tcheckLeafNodeConnected(t, ln1)\n\tcheckLeafNodeConnected(t, ln2)\n\n\ttime.Sleep(50 * time.Millisecond)\n\tclose(ch)\n\twg.Wait()\n}\n\nfunc TestLeafNodePubAllowedPruning(t *testing.T) {\n\tc := &client{}\n\tc.setPermissions(&Permissions{Publish: &SubjectPermission{Allow: []string{\"foo\"}}})\n\n\tgr := 100\n\twg := sync.WaitGroup{}\n\twg.Add(gr)\n\tfor i := 0; i < gr; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor j := 0; j < 100; j++ {\n\t\t\t\tc.pubAllowed(nats.NewInbox())\n\t\t\t}\n\t\t}()\n\t}\n\n\twg.Wait()\n\t// The cache prune function does try for a bit to make sure the cache\n\t// is below the maxPermCacheSize value, but depending on the machine\n\t// this runs on, it may be that it is still a bit over. If so, run\n\t// pubAllowed one more time and we must get below.\n\tif n := int(atomic.LoadInt32(&c.perms.pcsz)); n > maxPermCacheSize {\n\t\tc.pubAllowed(nats.NewInbox())\n\t}\n\tif n := int(atomic.LoadInt32(&c.perms.pcsz)); n > maxPermCacheSize {\n\t\tt.Fatalf(\"Expected size to be less than %v, got %v\", maxPermCacheSize, n)\n\t}\n\tif n := atomic.LoadInt32(&c.perms.prun); n != 0 {\n\t\tt.Fatalf(\"c.perms.prun should be 0, was %v\", n)\n\t}\n}\n\nfunc TestLeafNodeExportPermissionsNotForSpecialSubs(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.Accounts = []*Account{NewAccount(\"SYS\")}\n\tlo1.SystemAccount = \"SYS\"\n\tlo1.Cluster.Name = \"A\"\n\tlo1.Gateway.Name = \"A\"\n\tlo1.Gateway.Port = -1\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tln1 := RunServer(lo1)\n\tdefer ln1.Shutdown()\n\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", lo1.LeafNode.Host, lo1.LeafNode.Port))\n\tlo2 := DefaultOptions()\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tDenyExports: []string{\">\"},\n\t\t},\n\t}\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\tcheckLeafNodeConnected(t, ln1)\n\n\t// The deny is totally restrictive, but make sure that we still accept the $LDS, $GR and _GR_ go from LN1.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t// We should have registered the 3 subs from the accepting leafnode.\n\t\tif n := ln2.globalAccount().TotalSubs(); n != 9 {\n\t\t\treturn fmt.Errorf(\"Expected %d subs, got %v\", 9, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Make sure that if the node that detects the loop (and sends the error and\n// close the connection) is the accept side, the remote node (the one that solicits)\n// properly use the reconnect delay.\nfunc TestLeafNodeLoopDetectedOnAcceptSide(t *testing.T) {\n\tbo := DefaultOptions()\n\tbo.LeafNode.Host = \"127.0.0.1\"\n\tbo.LeafNode.Port = -1\n\tb := RunServer(bo)\n\tdefer b.Shutdown()\n\n\tl := &loopDetectedLogger{ch: make(chan string, 1)}\n\tb.SetLogger(l, false, false)\n\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", bo.LeafNode.Port))\n\n\tao := testDefaultOptionsForGateway(\"A\")\n\tao.Accounts = []*Account{NewAccount(\"SYS\")}\n\tao.SystemAccount = \"SYS\"\n\tao.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tao.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tHub: true,\n\t\t},\n\t}\n\ta := RunServer(ao)\n\tdefer a.Shutdown()\n\n\tco := testGatewayOptionsFromToWithServers(t, \"C\", \"A\", a)\n\tco.Accounts = []*Account{NewAccount(\"SYS\")}\n\tco.SystemAccount = \"SYS\"\n\tco.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tco.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t\tHub: true,\n\t\t},\n\t}\n\tc := RunServer(co)\n\tdefer c.Shutdown()\n\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase <-l.ch:\n\t\t\t// OK\n\t\tcase <-time.After(200 * time.Millisecond):\n\t\t\t// We are likely to detect from each A and C servers,\n\t\t\t// but consider a failure if we did not receive any.\n\t\t\tif i == 0 {\n\t\t\t\tt.Fatalf(\"Should have detected loop\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// The reconnect attempt is set to 5ms, but the default loop delay\n\t// is 30 seconds, so we should not get any new error for that long.\n\t// Check if we are getting more errors..\n\tselect {\n\tcase e := <-l.ch:\n\t\tt.Fatalf(\"Should not have gotten another error, got %q\", e)\n\tcase <-time.After(50 * time.Millisecond):\n\t\t// OK!\n\t}\n}\n\nfunc TestLeafNodeHubWithGateways(t *testing.T) {\n\tao := DefaultOptions()\n\tao.ServerName = \"A\"\n\tao.LeafNode.Host = \"127.0.0.1\"\n\tao.LeafNode.Port = -1\n\ta := RunServer(ao)\n\tdefer a.Shutdown()\n\n\tua, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ao.LeafNode.Port))\n\n\tbo := testDefaultOptionsForGateway(\"B\")\n\tbo.ServerName = \"B\"\n\tbo.Accounts = []*Account{NewAccount(\"SYS\")}\n\tbo.SystemAccount = \"SYS\"\n\tbo.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tbo.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{ua},\n\t\t\tHub: true,\n\t\t},\n\t}\n\tb := RunServer(bo)\n\tdefer b.Shutdown()\n\n\tdo := DefaultOptions()\n\tdo.ServerName = \"D\"\n\tdo.LeafNode.Host = \"127.0.0.1\"\n\tdo.LeafNode.Port = -1\n\td := RunServer(do)\n\tdefer d.Shutdown()\n\n\tud, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", do.LeafNode.Port))\n\n\tco := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", b)\n\tco.ServerName = \"C\"\n\tco.Accounts = []*Account{NewAccount(\"SYS\")}\n\tco.SystemAccount = \"SYS\"\n\tco.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tco.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{ud},\n\t\t\tHub: true,\n\t\t},\n\t}\n\tc := RunServer(co)\n\tdefer c.Shutdown()\n\n\twaitForInboundGateways(t, b, 1, 2*time.Second)\n\twaitForInboundGateways(t, c, 1, 2*time.Second)\n\tcheckLeafNodeConnected(t, a)\n\tcheckLeafNodeConnected(t, d)\n\n\t// Create a responder on D\n\tncD := natsConnect(t, d.ClientURL())\n\tdefer ncD.Close()\n\n\tncD.Subscribe(\"service\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"reply\"))\n\t})\n\tncD.Flush()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tacc := a.globalAccount()\n\t\tif r := acc.sl.Match(\"service\"); r != nil && len(r.psubs) == 1 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"subscription still not registered\")\n\t})\n\n\t// Create requestor on A and send the request, expect a reply.\n\tncA := natsConnect(t, a.ClientURL())\n\tdefer ncA.Close()\n\tif msg, err := ncA.Request(\"service\", []byte(\"request\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Failed to get reply: %v\", err)\n\t} else if string(msg.Data) != \"reply\" {\n\t\tt.Fatalf(\"Unexpected reply: %q\", msg.Data)\n\t}\n}\n\nfunc TestLeafNodeTmpClients(t *testing.T) {\n\tao := DefaultOptions()\n\tao.LeafNode.Host = \"127.0.0.1\"\n\tao.LeafNode.Port = -1\n\ta := RunServer(ao)\n\tdefer a.Shutdown()\n\n\tc, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", ao.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer c.Close()\n\t// Read info\n\tbr := bufio.NewReader(c)\n\tbr.ReadLine()\n\n\tcheckTmp := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\ta.grMu.Lock()\n\t\t\tl := len(a.grTmpClients)\n\t\t\ta.grMu.Unlock()\n\t\t\tif l != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected tmp map to have %v entries, got %v\", expected, l)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckTmp(1)\n\n\t// Close client and wait check that it is removed.\n\tc.Close()\n\tcheckTmp(0)\n\n\t// Check with normal leafnode connection that once connected,\n\t// the tmp map is also emptied.\n\tbo := DefaultOptions()\n\tbo.Cluster.Name = \"xyz\"\n\tbo.LeafNode.ReconnectInterval = 5 * time.Millisecond\n\tu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", ao.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating url: %v\", err)\n\t}\n\tbo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\tb := RunServer(bo)\n\tdefer b.Shutdown()\n\n\tcheckLeafNodeConnected(t, b)\n\tcheckTmp(0)\n}\n\nfunc TestLeafNodeTLSVerifyAndMap(t *testing.T) {\n\taccName := \"MyAccount\"\n\tacc := NewAccount(accName)\n\tcertUserName := \"CN=example.com,OU=NATS.io\"\n\tusers := []*User{{Username: certUserName, Account: acc}}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tleafUsers bool\n\t\tprovideCert bool\n\t}{\n\t\t{\"no users override, provides cert\", false, true},\n\t\t{\"no users override, does not provide cert\", false, false},\n\t\t{\"users override, provides cert\", true, true},\n\t\t{\"users override, does not provide cert\", true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.Accounts = []*Account{acc}\n\t\t\to.LeafNode.Host = \"127.0.0.1\"\n\t\t\to.LeafNode.Port = -1\n\t\t\tif test.leafUsers {\n\t\t\t\to.LeafNode.Users = users\n\t\t\t} else {\n\t\t\t\to.Users = users\n\t\t\t}\n\t\t\ttc := &TLSConfigOpts{\n\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/server.pem\",\n\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/server-key.pem\",\n\t\t\t\tCaFile: \"../test/configs/certs/tlsauth/ca.pem\",\n\t\t\t\tVerify: true,\n\t\t\t}\n\t\t\ttlsc, err := GenTLSConfig(tc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating tls config: %v\", err)\n\t\t\t}\n\t\t\to.LeafNode.TLSConfig = tlsc\n\t\t\to.LeafNode.TLSMap = true\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tslo := DefaultOptions()\n\t\t\tslo.Cluster.Name = \"xyz\"\n\n\t\t\tsltlsc := &tls.Config{}\n\t\t\tif test.provideCert {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/client.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/client-key.pem\",\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\tsltlsc, err = GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tsltlsc.InsecureSkipVerify = true\n\t\t\tu, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", o.LeafNode.Host, o.LeafNode.Port))\n\t\t\tslo.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t\t\t{\n\t\t\t\t\tTLSConfig: sltlsc,\n\t\t\t\t\tURLs: []*url.URL{u},\n\t\t\t\t},\n\t\t\t}\n\t\t\tsl := RunServer(slo)\n\t\t\tdefer sl.Shutdown()\n\n\t\t\tif !test.provideCert {\n\t\t\t\t// Wait a bit and make sure we are not connecting\n\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\tcheckLeafNodeConnectedCount(t, s, 0)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tcheckLeafNodeConnected(t, s)\n\n\t\t\tvar uname string\n\t\t\tvar accname string\n\t\t\ts.mu.Lock()\n\t\t\tfor _, c := range s.leafs {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tuname = c.opts.Username\n\t\t\t\tif c.acc != nil {\n\t\t\t\t\taccname = c.acc.GetName()\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\t\t\t}\n\t\t\ts.mu.Unlock()\n\t\t\tif uname != certUserName {\n\t\t\t\tt.Fatalf(\"Expected username %q, got %q\", certUserName, uname)\n\t\t\t}\n\t\t\tif accname != accName {\n\t\t\t\tt.Fatalf(\"Expected account %q, got %v\", accName, accname)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype chanLogger struct {\n\tDummyLogger\n\ttriggerChan chan string\n}\n\nfunc (l *chanLogger) Warnf(format string, v ...any) {\n\tl.triggerChan <- fmt.Sprintf(format, v...)\n}\n\nfunc (l *chanLogger) Errorf(format string, v ...any) {\n\tl.triggerChan <- fmt.Sprintf(format, v...)\n}\n\nconst (\n\ttestLeafNodeTLSVerifyAndMapSrvA = `\nlisten: 127.0.0.1:-1\nleaf {\n\tlisten: \"127.0.0.1:-1\"\n\ttls {\n\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\ttimeout: 2\n\t\tverify_and_map: true\n\t}\n\tauthorization {\n\t\tusers [{\n\t\t\tuser: \"%s\"\n\t\t}]\n\t}\n}\n`\n\ttestLeafNodeTLSVerifyAndMapSrvB = `\nlisten: -1\nleaf {\n\tremotes [\n\t\t{\n\t\t\turl: \"tls://user-provided-in-url@localhost:%d\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t}\n\t\t}\n\t]\n}`\n)\n\nfunc TestLeafNodeTLSVerifyAndMapCfgPass(t *testing.T) {\n\tl := &chanLogger{triggerChan: make(chan string, 100)}\n\tdefer close(l.triggerChan)\n\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvA, \"localhost\")))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\tsrvA.SetLogger(l, true, true)\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvB, optsA.LeafNode.Port)))\n\tob := LoadConfig(confB)\n\tob.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tsrvB := RunServer(ob)\n\tdefer srvB.Shutdown()\n\n\t// Now make sure that the leaf node connection is up and the correct account was picked\n\tcheckFor(t, 10*time.Second, 10*time.Millisecond, func() error {\n\t\tfor _, srv := range []*Server{srvA, srvB} {\n\t\t\tif nln := srv.NumLeafNodes(); nln != 1 {\n\t\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t\t}\n\t\t\tif leafz, err := srv.Leafz(nil); err != nil {\n\t\t\t\tif len(leafz.Leafs) != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"Number of leaf nodes returned by LEAFZ is not one: %d\", len(leafz.Leafs))\n\t\t\t\t} else if leafz.Leafs[0].Account != DEFAULT_GLOBAL_ACCOUNT {\n\t\t\t\t\treturn fmt.Errorf(\"Account used is not $G: %s\", leafz.Leafs[0].Account)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\t// Make sure that the user name in the url was ignored and a warning printed\n\tfor {\n\t\tselect {\n\t\tcase w := <-l.triggerChan:\n\t\t\tif w == `User \"user-provided-in-url\" found in connect proto, but user required from cert` {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Did not get expected warning\")\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeTLSVerifyAndMapCfgFail(t *testing.T) {\n\tl := &chanLogger{triggerChan: make(chan string, 100)}\n\tdefer close(l.triggerChan)\n\n\t// use certificate with SAN localhost, but configure the server to not accept it\n\t// instead provide a name matching the user (to be matched by failed\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvA, \"user-provided-in-url\")))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\tsrvA.SetLogger(l, true, true)\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(testLeafNodeTLSVerifyAndMapSrvB, optsA.LeafNode.Port)))\n\tob := LoadConfig(confB)\n\tob.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tsrvB := RunServer(ob)\n\tdefer srvB.Shutdown()\n\n\t// Now make sure that the leaf node connection is down\n\tcheckFor(t, 10*time.Second, 10*time.Millisecond, func() error {\n\t\tfor _, srv := range []*Server{srvA, srvB} {\n\t\t\tif nln := srv.NumLeafNodes(); nln != 0 {\n\t\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\t// Make sure that the connection was closed for the right reason\n\tfor {\n\t\tselect {\n\t\tcase w := <-l.triggerChan:\n\t\t\tif strings.Contains(w, ErrAuthentication.Error()) {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Did not get expected warning\")\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeOriginClusterInfo(t *testing.T) {\n\thopts := DefaultOptions()\n\thopts.ServerName = \"hub\"\n\thopts.LeafNode.Port = -1\n\n\thub := RunServer(hopts)\n\tdefer hub.Shutdown()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [ { url: \"nats://127.0.0.1:%d\" } ]\n\t\t}\n\t`, hopts.LeafNode.Port)))\n\n\topts, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoLog, opts.NoSigs = true, true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Check the info on the leadnode client in the hub.\n\tgrabLeaf := func() *client {\n\t\tvar l *client\n\t\thub.mu.Lock()\n\t\tfor _, l = range hub.leafs {\n\t\t\tbreak\n\t\t}\n\t\thub.mu.Unlock()\n\t\treturn l\n\t}\n\n\tl := grabLeaf()\n\tif rc := l.remoteCluster(); rc != \"\" {\n\t\tt.Fatalf(\"Expected an empty remote cluster, got %q\", rc)\n\t}\n\n\ts.Shutdown()\n\n\t// Now make our leafnode part of a cluster.\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [ { url: \"nats://127.0.0.1:%d\" } ]\n\t\t}\n\t\tcluster {\n\t\t\tname: \"xyz\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`, hopts.LeafNode.Port)))\n\n\topts, err = ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoLog, opts.NoSigs = true, true\n\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\tl = grabLeaf()\n\tif rc := l.remoteCluster(); rc != \"xyz\" {\n\t\tt.Fatalf(\"Expected a remote cluster name of \\\"xyz\\\", got %q\", rc)\n\t}\n\tpcid := l.cid\n\n\t// Now make sure that if we update our cluster name, simulating the settling\n\t// of dynamic cluster names between competing servers.\n\ts.setClusterName(\"xyz\")\n\t// Make sure we disconnect and reconnect.\n\tcheckLeafNodeConnectedCount(t, s, 0)\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, hub)\n\n\tl = grabLeaf()\n\tif rc := l.remoteCluster(); rc != \"xyz\" {\n\t\tt.Fatalf(\"Expected a remote cluster name of \\\"xyz\\\", got %q\", rc)\n\t}\n\t// Make sure we reconnected and have a new CID.\n\tif l.cid == pcid {\n\t\tt.Fatalf(\"Expected a different id, got the same\")\n\t}\n}\n\ntype proxyAcceptDetectFailureLate struct {\n\tsync.Mutex\n\twg sync.WaitGroup\n\tacceptPort int\n\tl net.Listener\n\tsrvs []net.Conn\n\tleaf net.Conn\n\tstartChan chan struct{}\n}\n\nfunc (p *proxyAcceptDetectFailureLate) run(t *testing.T) int {\n\treturn p.runEx(t, false)\n}\n\nfunc (p *proxyAcceptDetectFailureLate) runEx(t *testing.T, needStart bool) int {\n\tl, err := natsListen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on listen: %v\", err)\n\t}\n\tp.Lock()\n\tvar startChan chan struct{}\n\tif needStart {\n\t\tstartChan = make(chan struct{})\n\t\tp.startChan = startChan\n\t}\n\tp.l = l\n\tp.Unlock()\n\tport := l.Addr().(*net.TCPAddr).Port\n\tp.wg.Add(1)\n\tgo func() {\n\t\tdefer p.wg.Done()\n\t\tdefer l.Close()\n\t\tdefer func() {\n\t\t\tp.Lock()\n\t\t\tfor _, c := range p.srvs {\n\t\t\t\tc.Close()\n\t\t\t}\n\t\t\tp.Unlock()\n\t\t}()\n\t\tif startChan != nil {\n\t\t\t<-startChan\n\t\t}\n\t\tfor {\n\t\t\tc, err := l.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tsrv, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", p.acceptPort))\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tp.Lock()\n\t\t\tp.leaf = c\n\t\t\tp.srvs = append(p.srvs, srv)\n\t\t\tp.Unlock()\n\n\t\t\ttransfer := func(c1, c2 net.Conn) {\n\t\t\t\tvar buf [1024]byte\n\t\t\t\tfor {\n\t\t\t\t\tn, err := c1.Read(buf[:])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif _, err := c2.Write(buf[:n]); err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tgo transfer(srv, c)\n\t\t\tgo transfer(c, srv)\n\t\t}\n\t}()\n\treturn port\n}\n\nfunc (p *proxyAcceptDetectFailureLate) start() {\n\tp.Lock()\n\tif p.startChan != nil {\n\t\tclose(p.startChan)\n\t\tp.startChan = nil\n\t}\n\tp.Unlock()\n}\n\nfunc (p *proxyAcceptDetectFailureLate) close() {\n\tp.Lock()\n\tif p.startChan != nil {\n\t\tclose(p.startChan)\n\t\tp.startChan = nil\n\t}\n\tp.l.Close()\n\tp.Unlock()\n\n\tp.wg.Wait()\n}\n\ntype oldConnReplacedLogger struct {\n\tDummyLogger\n\terrCh chan string\n\twarnCh chan string\n}\n\nfunc (l *oldConnReplacedLogger) Errorf(format string, v ...any) {\n\tselect {\n\tcase l.errCh <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\nfunc (l *oldConnReplacedLogger) Warnf(format string, v ...any) {\n\tselect {\n\tcase l.warnCh <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\n// This test will simulate that the accept side does not detect the connection\n// has been closed early enough. The soliciting side will attempt to reconnect\n// and we should not be getting the \"loop detected\" error.\nfunc TestLeafNodeLoopDetectedDueToReconnect(t *testing.T) {\n\to := DefaultOptions()\n\to.LeafNode.Host = \"127.0.0.1\"\n\to.LeafNode.Port = -1\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tl := &oldConnReplacedLogger{errCh: make(chan string, 10), warnCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tp := &proxyAcceptDetectFailureLate{acceptPort: o.LeafNode.Port}\n\tdefer p.close()\n\tport := p.run(t)\n\n\taurl, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\tol := DefaultOptions()\n\tol.Cluster.Name = \"cde\"\n\tol.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tol.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{aurl}}}\n\tsl := RunServer(ol)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, sl)\n\n\t// Cause disconnect client side...\n\tp.Lock()\n\tp.leaf.Close()\n\tp.Unlock()\n\n\t// Make sure we did not get the loop detected error\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif strings.Contains(e, \"Loop detected\") {\n\t\t\tt.Fatalf(\"Loop detected: %v\", e)\n\t\t}\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// We are ok\n\t}\n\n\t// Now make sure we got the warning\n\tselect {\n\tcase w := <-l.warnCh:\n\t\tif !strings.Contains(w, \"Replacing connection from same server\") {\n\t\t\tt.Fatalf(\"Unexpected warning: %v\", w)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get expected warning\")\n\t}\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, sl)\n}\n\nfunc TestLeafNodeTwoRemotesBindToSameHubAccount(t *testing.T) {\n\topts := DefaultOptions()\n\topts.LeafNode.Host = \"127.0.0.1\"\n\topts.LeafNode.Port = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\taccount string\n\t\tfail bool\n\t}{\n\t\t{\"different local accounts\", \"b\", false},\n\t\t{\"same local accounts\", \"a\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tcluster { name: ln22, listen: 127.0.0.1:-1 }\n\t\t\taccounts {\n\t\t\t\ta { users [ {user: a, password: a} ]}\n\t\t\t\tb { users [ {user: b, password: b} ]}\n\t\t\t}\n\t\t\tleafnodes {\n\t\t\t\tremotes = [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: nats-leaf://127.0.0.1:%[1]d\n\t\t\t\t\t\taccount: a\n\t\t\t\t\t}\n\t\t\t\t\t{\n\t\t\t\t\t\turl: nats-leaf://127.0.0.1:%[1]d\n\t\t\t\t\t\taccount: %s\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t\t`\n\t\t\tlconf := createConfFile(t, []byte(fmt.Sprintf(conf, opts.LeafNode.Port, test.account)))\n\n\t\t\tlopts, err := ProcessConfigFile(lconf)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error loading config file: %v\", err)\n\t\t\t}\n\t\t\tlopts.NoLog = false\n\t\t\tln, err := NewServer(lopts)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t\t\t}\n\t\t\tdefer ln.Shutdown()\n\t\t\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\t\t\tln.SetLogger(l, false, false)\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tln.Start()\n\t\t\t}()\n\n\t\t\tselect {\n\t\t\tcase err := <-l.errCh:\n\t\t\t\tif test.fail && !strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) {\n\t\t\t\t\tt.Fatalf(\"Did not get expected duplicate connection error: %v\", err)\n\t\t\t\t} else if !test.fail && strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) {\n\t\t\t\t\tt.Fatalf(\"Incorrectly detected a duplicate connection: %v\", err)\n\t\t\t\t}\n\t\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\t\tif test.fail {\n\t\t\t\t\tt.Fatal(\"Did not get expected error\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tln.Shutdown()\n\t\t\twg.Wait()\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeNoDuplicateWithinCluster(t *testing.T) {\n\t// This set the cluster name to \"abc\"\n\toSrv1 := DefaultOptions()\n\toSrv1.ServerName = \"srv1\"\n\toSrv1.LeafNode.Host = \"127.0.0.1\"\n\toSrv1.LeafNode.Port = -1\n\tsrv1 := RunServer(oSrv1)\n\tdefer srv1.Shutdown()\n\n\tu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\n\toLeaf1 := DefaultOptions()\n\toLeaf1.ServerName = \"leaf1\"\n\toLeaf1.Cluster.Name = \"xyz\"\n\toLeaf1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\tleaf1 := RunServer(oLeaf1)\n\tdefer leaf1.Shutdown()\n\n\tleaf1ClusterURL := fmt.Sprintf(\"nats://127.0.0.1:%d\", oLeaf1.Cluster.Port)\n\n\toLeaf2 := DefaultOptions()\n\toLeaf2.ServerName = \"leaf2\"\n\toLeaf2.Cluster.Name = \"xyz\"\n\toLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\toLeaf2.Routes = RoutesFromStr(leaf1ClusterURL)\n\tleaf2 := RunServer(oLeaf2)\n\tdefer leaf2.Shutdown()\n\n\tcheckClusterFormed(t, leaf1, leaf2)\n\n\tcheckLeafNodeConnectedCount(t, srv1, 2)\n\tcheckLeafNodeConnected(t, leaf1)\n\tcheckLeafNodeConnected(t, leaf2)\n\n\tncSrv1 := natsConnect(t, srv1.ClientURL())\n\tdefer ncSrv1.Close()\n\tnatsQueueSub(t, ncSrv1, \"foo\", \"queue\", func(m *nats.Msg) {\n\t\tm.Data = []byte(\"from srv1\")\n\t\tm.RespondMsg(m)\n\t})\n\n\tncLeaf1 := natsConnect(t, leaf1.ClientURL())\n\tdefer ncLeaf1.Close()\n\tnatsQueueSub(t, ncLeaf1, \"foo\", \"queue\", func(m *nats.Msg) {\n\t\tm.Data = []byte(\"from leaf1\")\n\t\tm.RespondMsg(m)\n\t})\n\n\tncLeaf2 := natsConnect(t, leaf2.ClientURL())\n\tdefer ncLeaf2.Close()\n\n\t// Check that \"foo\" interest is available everywhere.\n\tfor _, s := range []*Server{srv1, leaf1, leaf2} {\n\t\tgacc := s.GlobalAccount()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := gacc.Interest(\"foo\"); n != 2 {\n\t\t\t\treturn fmt.Errorf(\"Expected interest for %q to be 2, got %v\", \"foo\", n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Send requests (from leaf2). For this test to make sure that\n\t// there is no duplicate, we want to make sure that we check for\n\t// multiple replies and that the reply subject subscription has\n\t// been propagated everywhere.\n\tsub := natsSubSync(t, ncLeaf2, \"reply_subj\")\n\tnatsFlush(t, ncLeaf2)\n\n\t// Here we have a single sub on \"reply_subj\" so using checkSubInterest is ok.\n\tcheckSubInterest(t, srv1, globalAccountName, \"reply_subj\", time.Second)\n\tcheckSubInterest(t, leaf1, globalAccountName, \"reply_subj\", time.Second)\n\tcheckSubInterest(t, leaf2, globalAccountName, \"reply_subj\", time.Second)\n\n\tfor i := 0; i < 100; i++ {\n\t\t// Now send the request\n\t\treqID := fmt.Sprintf(\"req.%d\", i)\n\t\tmsg := nats.NewMsg(\"foo\")\n\t\tmsg.Data = []byte(\"req\")\n\t\tmsg.Header.Set(\"ReqId\", reqID)\n\t\tmsg.Reply = sub.Subject\n\t\tif err := ncLeaf2.PublishMsg(msg); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t\t// Check that we get the reply\n\t\treplyMsg := natsNexMsg(t, sub, time.Second)\n\t\t// But make sure no duplicate. We do so by checking that the reply's\n\t\t// header ReqId matches our current reqID.\n\t\tif respReqID := replyMsg.Header.Get(\"ReqId\"); respReqID != reqID {\n\t\t\tt.Fatalf(\"Current request is %q, got duplicate with %q\", reqID, respReqID)\n\t\t}\n\t\t// We also should have preferred the queue sub that is in the leaf cluster.\n\t\tif string(replyMsg.Data) != \"from leaf1\" {\n\t\t\tt.Fatalf(\"Expected reply from leaf1, got %q\", replyMsg.Data)\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeLMsgSplit(t *testing.T) {\n\t// This set the cluster name to \"abc\"\n\toSrv1 := DefaultOptions()\n\toSrv1.LeafNode.Host = \"127.0.0.1\"\n\toSrv1.LeafNode.Port = -1\n\tsrv1 := RunServer(oSrv1)\n\tdefer srv1.Shutdown()\n\n\toSrv2 := DefaultOptions()\n\toSrv2.LeafNode.Host = \"127.0.0.1\"\n\toSrv2.LeafNode.Port = -1\n\toSrv2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv1.Cluster.Port))\n\tsrv2 := RunServer(oSrv2)\n\tdefer srv2.Shutdown()\n\n\tcheckClusterFormed(t, srv1, srv2)\n\n\tu1, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\tu2, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv2.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\n\toLeaf1 := DefaultOptions()\n\toLeaf1.Cluster.Name = \"xyz\"\n\toLeaf1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u1, u2}}}\n\tleaf1 := RunServer(oLeaf1)\n\tdefer leaf1.Shutdown()\n\n\toLeaf2 := DefaultOptions()\n\toLeaf2.Cluster.Name = \"xyz\"\n\toLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u1, u2}}}\n\toLeaf2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", oLeaf1.Cluster.Port))\n\tleaf2 := RunServer(oLeaf2)\n\tdefer leaf2.Shutdown()\n\n\tcheckClusterFormed(t, leaf1, leaf2)\n\n\tcheckLeafNodeConnected(t, leaf1)\n\tcheckLeafNodeConnected(t, leaf2)\n\n\tncSrv2 := natsConnect(t, srv2.ClientURL())\n\tdefer ncSrv2.Close()\n\tnatsQueueSub(t, ncSrv2, \"foo\", \"queue\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"from srv2\"))\n\t})\n\n\t// Check that \"foo\" interest is available everywhere.\n\tcheckSubInterest(t, srv1, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, srv2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, leaf1, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, leaf2, globalAccountName, \"foo\", time.Second)\n\n\t// Not required, but have a request payload that is more than 100 bytes\n\treqPayload := make([]byte, 150)\n\tfor i := 0; i < len(reqPayload); i++ {\n\t\treqPayload[i] = byte((i % 26)) + 'A'\n\t}\n\n\t// Send repeated requests (from scratch) from leaf-2:\n\tsendReq := func() {\n\t\tt.Helper()\n\n\t\tncLeaf2 := natsConnect(t, leaf2.ClientURL())\n\t\tdefer ncLeaf2.Close()\n\n\t\tif _, err := ncLeaf2.Request(\"foo\", reqPayload, time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not receive reply: %v\", err)\n\t\t}\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tsendReq()\n\t}\n}\n\ntype parseRouteLSUnsubLogger struct {\n\tDummyLogger\n\tgotTrace chan struct{}\n\tgotErr chan error\n}\n\nfunc (l *parseRouteLSUnsubLogger) Errorf(format string, v ...any) {\n\terr := fmt.Errorf(format, v...)\n\tselect {\n\tcase l.gotErr <- err:\n\tdefault:\n\t}\n}\n\nfunc (l *parseRouteLSUnsubLogger) Tracef(format string, v ...any) {\n\ttrace := fmt.Sprintf(format, v...)\n\tif strings.Contains(trace, \"LS- xyz $G foo bar\") {\n\t\tl.gotTrace <- struct{}{}\n\t}\n}\n\nfunc TestLeafNodeRouteParseLSUnsub(t *testing.T) {\n\t// This set the cluster name to \"abc\"\n\toSrv1 := DefaultOptions()\n\toSrv1.LeafNode.Host = \"127.0.0.1\"\n\toSrv1.LeafNode.Port = -1\n\tsrv1 := RunServer(oSrv1)\n\tdefer srv1.Shutdown()\n\n\tl := &parseRouteLSUnsubLogger{gotTrace: make(chan struct{}, 1), gotErr: make(chan error, 1)}\n\tsrv1.SetLogger(l, true, true)\n\n\toSrv2 := DefaultOptions()\n\toSrv2.LeafNode.Host = \"127.0.0.1\"\n\toSrv2.LeafNode.Port = -1\n\toSrv2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv1.Cluster.Port))\n\tsrv2 := RunServer(oSrv2)\n\tdefer srv2.Shutdown()\n\n\tcheckClusterFormed(t, srv1, srv2)\n\n\tu2, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", oSrv2.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\n\toLeaf2 := DefaultOptions()\n\toLeaf2.Cluster.Name = \"xyz\"\n\toLeaf2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u2}}}\n\tleaf2 := RunServer(oLeaf2)\n\tdefer leaf2.Shutdown()\n\n\tcheckLeafNodeConnected(t, srv2)\n\tcheckLeafNodeConnected(t, leaf2)\n\n\tncLeaf2 := natsConnect(t, leaf2.ClientURL())\n\tdefer ncLeaf2.Close()\n\n\tsub := natsQueueSubSync(t, ncLeaf2, \"foo\", \"bar\")\n\t// The issue was with the unsubscribe of this queue subscription\n\tnatsUnsub(t, sub)\n\n\t// We should get the trace\n\tselect {\n\tcase <-l.gotTrace:\n\t\t// OK!\n\tcase <-time.After(100 * time.Millisecond):\n\t\tt.Fatalf(\"Did not get LS- trace\")\n\t}\n\t// And no error...\n\tselect {\n\tcase e := <-l.gotErr:\n\t\tt.Fatalf(\"There was an error on server 1: %q\", e.Error())\n\tcase <-time.After(100 * time.Millisecond):\n\t\t// OK!\n\t}\n}\n\nfunc TestLeafNodeOperatorBadCfg(t *testing.T) {\n\tsysAcc, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tsysAccPk, err := sysAcc.PublicKey()\n\trequire_NoError(t, err)\n\ttmpDir := t.TempDir()\n\n\tconfigTmpl := `\n\t\tport: -1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: {\n\t\t\ttype: cache\n\t\t\tdir: '%s'\n\t\t}\n\t\tleafnodes: {\n\t\t\t%s\n\t\t}\n\t`\n\n\tcases := []struct {\n\t\tname string\n\t\terrorText string\n\t\tcfg string\n\t}{\n\t\t{\n\t\t\tname: \"Operator with Leafnode\",\n\t\t\terrorText: \"operator mode does not allow specifying users in leafnode config\",\n\t\t\tcfg: `\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tusers = [{user: \"u\", password: \"p\"}]\n\t\t\t}`,\n\t\t},\n\t\t{\n\t\t\tname: \"Operator with NKey\",\n\t\t\terrorText: \"operator mode and non account nkeys are incompatible\",\n\t\t\tcfg: `\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\taccount: notankey\n\t\t\t}`,\n\t\t},\n\t\t{\n\t\t\tname: \"Operator remote account NKeys\",\n\t\t\terrorText: \"operator mode requires account nkeys in remotes. \" +\n\t\t\t\t\"Please add an `account` key to each remote in your `leafnodes` section, to assign it to an account. \" +\n\t\t\t\t\"Each account value should be a 56 character public key, starting with the letter 'A'\",\n\t\t\tcfg: `remotes: [{url: u}]`,\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(configTmpl, ojwt, sysAccPk, tmpDir, c.cfg)))\n\t\t\topts := LoadConfig(conf)\n\t\t\ts, err := NewServer(opts)\n\t\t\tif err == nil {\n\t\t\t\ts.Shutdown()\n\t\t\t\tt.Fatal(\"Expected an error\")\n\t\t\t}\n\t\t\t// Since the server cannot be stopped, since it did not start,\n\t\t\t// let's manually close the account resolver to avoid leaking go routines.\n\t\t\topts.AccountResolver.Close()\n\t\t\tif err.Error() != c.errorText {\n\t\t\t\tt.Fatalf(\"Expected error %q but got %q\", c.errorText, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeTLSConfigReload(t *testing.T) {\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\t%s\n\t\t\t\ttimeout: 2\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tlg := &captureErrorLogger{errCh: make(chan string, 10)}\n\tsrvA.SetLogger(lg, false, false)\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, optsA.LeafNode.Port)))\n\n\toptsB, err := ProcessConfigFile(confB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\toptsB.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\toptsB.NoLog, optsB.NoSigs = true, true\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for the error\n\tselect {\n\tcase err := <-lg.errCh:\n\t\t// Since Go 1.18, we had to regenerate certs to not have to use GODEBUG=\"x509sha1=1\"\n\t\t// But on macOS, with our test CA certs, no SCTs included, it will fail\n\t\t// for the reason \"x509: “localhost” certificate is not standards compliant\"\n\t\t// instead of \"unknown authority\".\n\t\tif !strings.Contains(err, \"unknown\") && !strings.Contains(err, \"compliant\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get TLS error\")\n\t}\n\n\t// Add the CA to srvA\n\treloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file: \"../test/configs/certs/ca.pem\"`))\n\n\t// Now make sure that srvB can create a LN connection.\n\tcheckFor(t, 3*time.Second, 10*time.Millisecond, func() error {\n\t\tif nln := srvB.NumLeafNodes(); nln != 1 {\n\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeTLSConfigReloadForRemote(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`))\n\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tlg := &captureErrorLogger{errCh: make(chan string, 10)}\n\tsrvA.SetLogger(lg, false, false)\n\n\ttemplate := `\n\t\tlisten: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\t\t\t%s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.LeafNode.Port, \"\")))\n\n\tsrvB, _ := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for the error\n\tselect {\n\tcase err := <-lg.errCh:\n\t\tif !strings.Contains(err, \"bad certificate\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get TLS error\")\n\t}\n\n\t// Add the CA to srvB\n\treloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.LeafNode.Port, `ca_file: \"../test/configs/certs/ca.pem\"`))\n\n\t// Now make sure that srvB can create a LN connection.\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif nln := srvB.NumLeafNodes(); nln != 1 {\n\t\t\treturn fmt.Errorf(\"Number of leaf nodes is %d\", nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc testDefaultLeafNodeWSOptions() *Options {\n\to := DefaultOptions()\n\to.Websocket.Host = \"127.0.0.1\"\n\to.Websocket.Port = -1\n\to.Websocket.NoTLS = true\n\to.LeafNode.Host = \"127.0.0.1\"\n\to.LeafNode.Port = -1\n\treturn o\n}\n\nfunc testDefaultRemoteLeafNodeWSOptions(t *testing.T, o *Options, tls bool) *Options {\n\t// Use some path in the URL.. we don't use that, but internally\n\t// the server will prefix the path with /leafnode so that the\n\t// WS webserver knows that it needs to create a LEAF connection.\n\tu, _ := url.Parse(fmt.Sprintf(\"ws://127.0.0.1:%d/some/path\", o.Websocket.Port))\n\tlo := DefaultOptions()\n\tlo.Cluster.Name = \"LN\"\n\tremote := &RemoteLeafOpts{URLs: []*url.URL{u}}\n\tif tls {\n\t\ttc := &TLSConfigOpts{\n\t\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\t}\n\t\ttlsConf, err := GenTLSConfig(tc)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t\t}\n\t\t// GenTLSConfig sets the CA in ClientCAs, but since here we act\n\t\t// as a client, set RootCAs...\n\t\ttlsConf.RootCAs = tlsConf.ClientCAs\n\t\tremote.TLSConfig = tlsConf\n\t}\n\tlo.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\treturn lo\n}\n\nfunc TestLeafNodeWSMixURLs(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\turls []string\n\t}{\n\t\t{\"mix 1\", []string{\"nats://127.0.0.1:1234\", \"ws://127.0.0.1:5678\", \"wss://127.0.0.1:9012\"}},\n\t\t{\"mix 2\", []string{\"ws://127.0.0.1:1234\", \"nats://127.0.0.1:5678\", \"wss://127.0.0.1:9012\"}},\n\t\t{\"mix 3\", []string{\"wss://127.0.0.1:1234\", \"ws://127.0.0.1:5678\", \"nats://127.0.0.1:9012\"}},\n\t\t{\"mix 4\", []string{\"ws://127.0.0.1:1234\", \"nats://127.0.0.1:9012\"}},\n\t\t{\"mix 5\", []string{\"nats://127.0.0.1:1234\", \"ws://127.0.0.1:9012\"}},\n\t\t{\"mix 6\", []string{\"wss://127.0.0.1:1234\", \"nats://127.0.0.1:9012\"}},\n\t\t{\"mix 7\", []string{\"nats://127.0.0.1:1234\", \"wss://127.0.0.1:9012\"}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\tremote := &RemoteLeafOpts{}\n\t\t\turls := make([]*url.URL, 0, 3)\n\t\t\tfor _, ustr := range test.urls {\n\t\t\t\tu, err := url.Parse(ustr)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t\t\t\t}\n\t\t\t\turls = append(urls, u)\n\t\t\t}\n\t\t\tremote.URLs = urls\n\t\t\to.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\t\t\ts, err := NewServer(o)\n\t\t\tif err == nil || !strings.Contains(err.Error(), \"mix\") {\n\t\t\t\tif s != nil {\n\t\t\t\t\ts.Shutdown()\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype testConnTrackSize struct {\n\tsync.Mutex\n\tnet.Conn\n\tsz int\n}\n\nfunc (c *testConnTrackSize) Write(p []byte) (int, error) {\n\tc.Lock()\n\tdefer c.Unlock()\n\tn, err := c.Conn.Write(p)\n\tc.sz += n\n\treturn n, err\n}\n\nfunc TestLeafNodeWSBasic(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmasking bool\n\t\ttls bool\n\t\tacceptCompression bool\n\t\tremoteCompression bool\n\t}{\n\t\t{\"masking plain no compression\", true, false, false, false},\n\t\t{\"masking plain compression\", true, false, true, true},\n\t\t{\"masking plain compression disagree\", true, false, false, true},\n\t\t{\"masking plain compression disagree 2\", true, false, true, false},\n\t\t{\"masking tls no compression\", true, true, false, false},\n\t\t{\"masking tls compression\", true, true, true, true},\n\t\t{\"masking tls compression disagree\", true, true, false, true},\n\t\t{\"masking tls compression disagree 2\", true, true, true, false},\n\t\t{\"no masking plain no compression\", false, false, false, false},\n\t\t{\"no masking plain compression\", false, false, true, true},\n\t\t{\"no masking plain compression disagree\", false, false, false, true},\n\t\t{\"no masking plain compression disagree 2\", false, false, true, false},\n\t\t{\"no masking tls no compression\", false, true, false, false},\n\t\t{\"no masking tls compression\", false, true, true, true},\n\t\t{\"no masking tls compression disagree\", false, true, false, true},\n\t\t{\"no masking tls compression disagree 2\", false, true, true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testDefaultLeafNodeWSOptions()\n\t\t\to.Websocket.NoTLS = !test.tls\n\t\t\tif test.tls {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\t\t\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\t\t\t}\n\t\t\t\ttlsConf, err := GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t\t\t\t}\n\t\t\t\to.Websocket.TLSConfig = tlsConf\n\t\t\t}\n\t\t\to.Websocket.Compression = test.acceptCompression\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tlo := testDefaultRemoteLeafNodeWSOptions(t, o, test.tls)\n\t\t\tlo.LeafNode.Remotes[0].Websocket.Compression = test.remoteCompression\n\t\t\tlo.LeafNode.Remotes[0].Websocket.NoMasking = !test.masking\n\t\t\tln := RunServer(lo)\n\t\t\tdefer ln.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, s)\n\t\t\tcheckLeafNodeConnected(t, ln)\n\n\t\t\tvar trackSizeConn *testConnTrackSize\n\t\t\tif !test.tls {\n\t\t\t\tvar cln *client\n\t\t\t\tln.mu.Lock()\n\t\t\t\tfor _, l := range ln.leafs {\n\t\t\t\t\tcln = l\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tln.mu.Unlock()\n\t\t\t\tcln.mu.Lock()\n\t\t\t\ttrackSizeConn = &testConnTrackSize{Conn: cln.nc}\n\t\t\t\tcln.nc = trackSizeConn\n\t\t\t\tcln.mu.Unlock()\n\t\t\t}\n\n\t\t\tnc1 := natsConnect(t, s.ClientURL())\n\t\t\tdefer nc1.Close()\n\t\t\tsub1 := natsSubSync(t, nc1, \"foo\")\n\t\t\tnatsFlush(t, nc1)\n\n\t\t\tcheckSubInterest(t, ln, globalAccountName, \"foo\", time.Second)\n\n\t\t\tnc2 := natsConnect(t, ln.ClientURL())\n\t\t\tdefer nc2.Close()\n\t\t\tmsg1Payload := make([]byte, 2048)\n\t\t\tfor i := 0; i < len(msg1Payload); i++ {\n\t\t\t\tmsg1Payload[i] = 'A'\n\t\t\t}\n\t\t\tnatsPub(t, nc2, \"foo\", msg1Payload)\n\n\t\t\tmsg := natsNexMsg(t, sub1, time.Second)\n\t\t\tif !bytes.Equal(msg.Data, msg1Payload) {\n\t\t\t\tt.Fatalf(\"Invalid message: %q\", msg.Data)\n\t\t\t}\n\n\t\t\tsub2 := natsSubSync(t, nc2, \"bar\")\n\t\t\tnatsFlush(t, nc2)\n\n\t\t\tcheckSubInterest(t, s, globalAccountName, \"bar\", time.Second)\n\n\t\t\tmsg2Payload := make([]byte, 2048)\n\t\t\tfor i := 0; i < len(msg2Payload); i++ {\n\t\t\t\tmsg2Payload[i] = 'B'\n\t\t\t}\n\t\t\tnatsPub(t, nc1, \"bar\", msg2Payload)\n\n\t\t\tmsg = natsNexMsg(t, sub2, time.Second)\n\t\t\tif !bytes.Equal(msg.Data, msg2Payload) {\n\t\t\t\tt.Fatalf(\"Invalid message: %q\", msg.Data)\n\t\t\t}\n\n\t\t\tif !test.tls {\n\t\t\t\ttrackSizeConn.Lock()\n\t\t\t\tsize := trackSizeConn.sz\n\t\t\t\ttrackSizeConn.Unlock()\n\n\t\t\t\tif test.acceptCompression && test.remoteCompression {\n\t\t\t\t\tif size >= 1024 {\n\t\t\t\t\t\tt.Fatalf(\"Seems that there was no compression: size=%v\", size)\n\t\t\t\t\t}\n\t\t\t\t} else if size < 2048 {\n\t\t\t\t\tt.Fatalf(\"Seems compression was on while it should not: size=%v\", size)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeWSRemoteCompressAndMaskingOptions(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcompress bool\n\t\tcompStr string\n\t\tnoMasking bool\n\t\tnoMaskStr string\n\t}{\n\t\t{\"compression masking\", true, \"true\", false, \"false\"},\n\t\t{\"compression no masking\", true, \"true\", true, \"true\"},\n\t\t{\"no compression masking\", false, \"false\", false, \"false\"},\n\t\t{\"no compression no masking\", false, \"false\", true, \"true\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{url: \"ws://127.0.0.1:1234\", ws_compression: %s, ws_no_masking: %s}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`, test.compStr, test.noMaskStr)))\n\t\t\to, err := ProcessConfigFile(conf)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error loading conf: %v\", err)\n\t\t\t}\n\t\t\tif nr := len(o.LeafNode.Remotes); nr != 1 {\n\t\t\t\tt.Fatalf(\"Expected 1 remote, got %v\", nr)\n\t\t\t}\n\t\t\tr := o.LeafNode.Remotes[0]\n\t\t\tif cur := r.Websocket.Compression; cur != test.compress {\n\t\t\t\tt.Fatalf(\"Expected compress to be %v, got %v\", test.compress, cur)\n\t\t\t}\n\t\t\tif cur := r.Websocket.NoMasking; cur != test.noMasking {\n\t\t\t\tt.Fatalf(\"Expected ws_masking to be %v, got %v\", test.noMasking, cur)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeWSNoMaskingRejected(t *testing.T) {\n\twsTestRejectNoMasking = true\n\tdefer func() { wsTestRejectNoMasking = false }()\n\n\to := testDefaultLeafNodeWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlo := testDefaultRemoteLeafNodeWSOptions(t, o, false)\n\tlo.LeafNode.Remotes[0].Websocket.NoMasking = true\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln)\n\n\tvar cln *client\n\tln.mu.Lock()\n\tfor _, l := range ln.leafs {\n\t\tcln = l\n\t\tbreak\n\t}\n\tln.mu.Unlock()\n\n\tcln.mu.Lock()\n\tmaskWrite := cln.ws.maskwrite\n\tcln.mu.Unlock()\n\n\tif !maskWrite {\n\t\tt.Fatal(\"Leafnode remote connection should mask writes, it does not\")\n\t}\n}\n\nfunc TestLeafNodeWSSubPath(t *testing.T) {\n\to := testDefaultLeafNodeWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlo := testDefaultRemoteLeafNodeWSOptions(t, o, false)\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\t// Confirm that it can connect using the subpath.\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln)\n\n\t// Add another leafnode that tries to connect to the subpath\n\t// but intercept the attempt for the test.\n\to2 := testDefaultLeafNodeWSOptions()\n\tlo2 := testDefaultRemoteLeafNodeWSOptions(t, o2, false)\n\tattempts := make(chan string, 2)\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tattempts <- r.URL.String()\n\t}))\n\tdefer ts.Close()\n\tu, _ := url.Parse(fmt.Sprintf(\"%v/some/path\", ts.URL))\n\tu.Scheme = \"ws\"\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{u},\n\t\t},\n\t}\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\texpected := \"/some/path/leafnode\"\n\tselect {\n\tcase got := <-attempts:\n\t\tif got != expected {\n\t\t\tt.Fatalf(\"Expected: %v, got: %v\", expected, got)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timed out waiting for leaf ws connect attempt\")\n\t}\n}\n\nfunc TestLeafNodeWSFailedConnection(t *testing.T) {\n\to := testDefaultLeafNodeWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlo := testDefaultRemoteLeafNodeWSOptions(t, o, true)\n\tlo.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tel := &captureErrorLogger{errCh: make(chan string, 100)}\n\tln.SetLogger(el, false, false)\n\n\tselect {\n\tcase err := <-el.errCh:\n\t\tif !strings.Contains(err, \"handshake error\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"No error reported!\")\n\t}\n\tln.Shutdown()\n\ts.Shutdown()\n\n\tlst, err := natsListen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error starting listener: %v\", err)\n\t}\n\tdefer lst.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tc, err := lst.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttime.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)\n\t\t\tif rand.Intn(2) == 1 {\n\t\t\t\tc.Write([]byte(\"something\\r\\n\"))\n\t\t\t}\n\t\t\tc.Close()\n\t\t}\n\t}()\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tport := lst.Addr().(*net.TCPAddr).Port\n\tu, _ := url.Parse(fmt.Sprintf(\"ws://127.0.0.1:%d\", port))\n\tlo = DefaultOptions()\n\tlo.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\tlo.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\tln, _ = NewServer(lo)\n\tel = &captureErrorLogger{errCh: make(chan string, 100)}\n\tln.SetLogger(el, false, false)\n\n\tgo func() {\n\t\tln.Start()\n\t\twg.Done()\n\t}()\n\n\ttimeout := time.NewTimer(time.Second)\n\tfor i := 0; i < 10; i++ {\n\t\tselect {\n\t\tcase err := <-el.errCh:\n\t\t\tif !strings.Contains(err, \"Error soliciting\") {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\tcase <-timeout.C:\n\t\t\tt.Fatal(\"No error reported!\")\n\t\t}\n\t}\n\tln.Shutdown()\n\tlst.Close()\n\twg.Wait()\n}\n\nfunc TestLeafNodeWSAuth(t *testing.T) {\n\ttemplate := `\n\t\tport: -1\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{user: \"user\", pass: \"puser\", connection_types: [\"%s\"]}\n\t\t\t\t{user: \"leaf\", pass: \"pleaf\", connection_types: [\"%s\"%s]}\n\t\t\t]\n\t\t}\n\t\twebsocket {\n\t\t\tport: -1\n\t\t\tno_tls: true\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`\n\ts, o, conf := runReloadServerWithContent(t,\n\t\t[]byte(fmt.Sprintf(template, jwt.ConnectionTypeStandard, jwt.ConnectionTypeLeafnode, \"\")))\n\tdefer s.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tlo := testDefaultRemoteLeafNodeWSOptions(t, o, false)\n\tu, _ := url.Parse(fmt.Sprintf(\"ws://leaf:pleaf@127.0.0.1:%d\", o.Websocket.Port))\n\tremote := &RemoteLeafOpts{URLs: []*url.URL{u}}\n\tlo.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\tlo.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tvar lasterr string\n\ttm := time.NewTimer(2 * time.Second)\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase lasterr = <-l.errCh:\n\t\t\tif strings.Contains(lasterr, \"authentication\") {\n\t\t\t\tdone = true\n\t\t\t}\n\t\tcase <-tm.C:\n\t\t\tt.Fatalf(\"Expected auth error, got %v\", lasterr)\n\t\t}\n\t}\n\n\tws := fmt.Sprintf(`, \"%s\"`, jwt.ConnectionTypeLeafnodeWS)\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template,\n\t\tjwt.ConnectionTypeStandard, jwt.ConnectionTypeLeafnode, ws))\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln)\n\n\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://user:puser@127.0.0.1:%d\", o.Port))\n\tdefer nc1.Close()\n\n\tsub := natsSubSync(t, nc1, \"foo\")\n\tnatsFlush(t, nc1)\n\n\tcheckSubInterest(t, ln, globalAccountName, \"foo\", time.Second)\n\n\tnc2 := natsConnect(t, ln.ClientURL())\n\tdefer nc2.Close()\n\n\tnatsPub(t, nc2, \"foo\", []byte(\"msg1\"))\n\tmsg := natsNexMsg(t, sub, time.Second)\n\n\tif md := string(msg.Data); md != \"msg1\" {\n\t\tt.Fatalf(\"Invalid message: %q\", md)\n\t}\n}\n\nfunc TestLeafNodeWSGossip(t *testing.T) {\n\to1 := testDefaultLeafNodeWSOptions()\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\t// Now connect from a server that knows only about s1\n\tlo := testDefaultRemoteLeafNodeWSOptions(t, o1, false)\n\tlo.LeafNode.ReconnectInterval = 15 * time.Millisecond\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnected(t, s1)\n\tcheckLeafNodeConnected(t, ln)\n\n\t// Now add a routed server to s1\n\to2 := testDefaultLeafNodeWSOptions()\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\t// Wait for cluster to form\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now shutdown s1 and check that ln is able to reconnect to s2.\n\ts1.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\tcheckLeafNodeConnected(t, ln)\n\n\t// Make sure that the reconnection was as a WS connection, not simply to\n\t// the regular LN port.\n\tvar s2lc *client\n\ts2.mu.Lock()\n\tfor _, l := range s2.leafs {\n\t\ts2lc = l\n\t\tbreak\n\t}\n\ts2.mu.Unlock()\n\n\ts2lc.mu.Lock()\n\tisWS := s2lc.isWebsocket()\n\ts2lc.mu.Unlock()\n\n\tif !isWS {\n\t\tt.Fatal(\"Leafnode connection is not websocket!\")\n\t}\n}\n\n// This test was showing an issue if one set maxBufSize to very small value,\n// such as maxBufSize = 10. With such small value, we would get a corruption\n// in that LMSG would arrive with missing bytes. We are now always making\n// a copy when dealing with messages that are bigger than maxBufSize.\nfunc TestLeafNodeWSNoBufferCorruption(t *testing.T) {\n\to := testDefaultLeafNodeWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlo1 := testDefaultRemoteLeafNodeWSOptions(t, o, false)\n\tlo1.LeafNode.ReconnectInterval = 15 * time.Millisecond\n\tln1 := RunServer(lo1)\n\tdefer ln1.Shutdown()\n\n\tlo2 := DefaultOptions()\n\tlo2.Cluster.Name = \"LN\"\n\tlo2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.Cluster.Port))\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\tcheckClusterFormed(t, ln1, ln2)\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln1)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tsub := natsSubSync(t, nc, \"foo\")\n\n\tnc1 := natsConnect(t, ln1.ClientURL())\n\tdefer nc1.Close()\n\n\tnc2 := natsConnect(t, ln2.ClientURL())\n\tdefer nc2.Close()\n\tsub2 := natsSubSync(t, nc2, \"foo\")\n\n\tcheckSubInterest(t, s, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, ln2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, ln1, globalAccountName, \"foo\", time.Second)\n\n\tpayload := make([]byte, 100*1024)\n\tfor i := 0; i < len(payload); i++ {\n\t\tpayload[i] = 'A'\n\t}\n\tnatsPub(t, nc1, \"foo\", payload)\n\n\tcheckMsgRcv := func(sub *nats.Subscription) {\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tif !bytes.Equal(msg.Data, payload) {\n\t\t\tt.Fatalf(\"Invalid message content: %q\", msg.Data)\n\t\t}\n\t}\n\tcheckMsgRcv(sub2)\n\tcheckMsgRcv(sub)\n}\n\nfunc TestLeafNodeWSRemoteNoTLSBlockWithWSSProto(t *testing.T) {\n\to := testDefaultLeafNodeWSOptions()\n\to.Websocket.NoTLS = false\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t}\n\ttlsConf, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t}\n\to.Websocket.TLSConfig = tlsConf\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\t// The test will make sure that if the protocol is \"wss://\", a TLS handshake must\n\t// be initiated, regardless of the presence of a TLS config block in config file\n\t// or here directly.\n\t// A bug was causing the absence of TLS config block to initiate a non TLS connection\n\t// even if \"wss://\" proto was specified, which would lead to \"invalid websocket connection\"\n\t// errors in the log.\n\t// With the fix, the connection will fail because the remote will fail to verify\n\t// the root CA, but at least, we will make sure that this is not an \"invalid websocket connection\"\n\n\tu, _ := url.Parse(fmt.Sprintf(\"wss://127.0.0.1:%d/some/path\", o.Websocket.Port))\n\tlo := DefaultOptions()\n\tlo.Cluster.Name = \"LN\"\n\tremote := &RemoteLeafOpts{URLs: []*url.URL{u}}\n\tlo.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\tlo.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\tln := RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\tln.SetLogger(l, false, false)\n\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif strings.Contains(e, \"invalid websocket connection\") {\n\t\t\tt.Fatalf(\"The remote did not try to create a TLS connection: %v\", e)\n\t\t}\n\t\t// OK!\n\t\treturn\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Connection should fail\")\n\t}\n}\n\nfunc TestLeafNodeWSNoAuthUser(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tport: -1\n\taccounts {\n\t\tA { users [ {user: a, password: a} ]}\n\t\tB { users [ {user: b, password: b} ]}\n\t}\n\twebsocket {\n\t\tport: -1\n\t\tno_tls: true\n\t\tno_auth_user: a\n\t}\n\tleafnodes {\n\t\tport: -1\n\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc1 := natsConnect(t, fmt.Sprintf(\"nats://a:a@127.0.0.1:%d\", o.Port))\n\tdefer nc1.Close()\n\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\tport: -1\n\taccounts {\n\t\tA { users [ {user: a, password: a} ]}\n\t\tB { users [ {user: b, password: b} ]}\n\t}\n\tleafnodes {\n\t\tremotes [\n\t\t\t{\n\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\taccount: A\n\t\t\t}\n\t\t]\n\t}\n\t`, o.Websocket.Port)))\n\n\tln, lo := RunServerWithConfig(lconf)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln)\n\n\tnc2 := natsConnect(t, fmt.Sprintf(\"nats://a:a@127.0.0.1:%d\", lo.Port))\n\tdefer nc2.Close()\n\n\tsub := natsSubSync(t, nc2, \"foo\")\n\tnatsFlush(t, nc2)\n\n\tcheckSubInterest(t, s, \"A\", \"foo\", time.Second)\n\n\tnatsPub(t, nc1, \"foo\", []byte(\"msg1\"))\n\tmsg := natsNexMsg(t, sub, time.Second)\n\n\tif md := string(msg.Data); md != \"msg1\" {\n\t\tt.Fatalf(\"Invalid message: %q\", md)\n\t}\n}\n\nfunc TestLeafNodeStreamImport(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.LeafNode.Port = -1\n\taccA := NewAccount(\"A\")\n\to1.Accounts = []*Account{accA}\n\to1.Users = []*User{{Username: \"a\", Password: \"a\", Account: accA}}\n\to1.LeafNode.Account = \"A\"\n\to1.NoAuthUser = \"a\"\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.LeafNode.Port = -1\n\to2.Cluster.Name = \"xyz\"\n\n\taccB := NewAccount(\"B\")\n\tif err := accB.AddStreamExport(\">\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding stream export: %v\", err)\n\t}\n\n\taccC := NewAccount(\"C\")\n\tif err := accC.AddStreamImport(accB, \">\", \"\"); err != nil {\n\t\tt.Fatalf(\"Error adding stream import: %v\", err)\n\t}\n\n\to2.Accounts = []*Account{accB, accC}\n\to2.Users = []*User{{Username: \"b\", Password: \"b\", Account: accB}, {Username: \"c\", Password: \"c\", Account: accC}}\n\to2.NoAuthUser = \"b\"\n\tu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\to2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}, LocalAccount: \"C\"}}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tnc1 := natsConnect(t, s1.ClientURL())\n\tdefer nc1.Close()\n\n\tsub := natsSubSync(t, nc1, \"a\")\n\n\tcheckSubInterest(t, s2, \"C\", \"a\", time.Second)\n\n\tnc2 := natsConnect(t, s2.ClientURL())\n\tdefer nc2.Close()\n\n\tnatsPub(t, nc2, \"a\", []byte(\"hello?\"))\n\n\tnatsNexMsg(t, sub, time.Second)\n}\n\nfunc TestLeafNodeRouteSubWithOrigin(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tlo1.Cluster.Name = \"local\"\n\tlo1.Cluster.Host = \"127.0.0.1\"\n\tlo1.Cluster.Port = -1\n\tl1 := RunServer(lo1)\n\tdefer l1.Shutdown()\n\n\tlo2 := DefaultOptions()\n\tlo2.LeafNode.Host = \"127.0.0.1\"\n\tlo2.LeafNode.Port = -1\n\tlo2.Cluster.Name = \"local\"\n\tlo2.Cluster.Host = \"127.0.0.1\"\n\tlo2.Cluster.Port = -1\n\tlo2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.Cluster.Port))\n\tl2 := RunServer(lo2)\n\tdefer l2.Shutdown()\n\n\tcheckClusterFormed(t, l1, l2)\n\n\tu1, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.LeafNode.Port))\n\turls := []*url.URL{u1}\n\n\tro1 := DefaultOptions()\n\tro1.Cluster.Name = \"remote\"\n\tro1.Cluster.Host = \"127.0.0.1\"\n\tro1.Cluster.Port = -1\n\tro1.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}}\n\tr1 := RunServer(ro1)\n\tdefer r1.Shutdown()\n\n\tcheckLeafNodeConnected(t, r1)\n\n\tnc := natsConnect(t, r1.ClientURL(), nats.NoReconnect())\n\tdefer nc.Close()\n\tnatsSubSync(t, nc, \"foo\")\n\tnatsQueueSubSync(t, nc, \"bar\", \"baz\")\n\tcheckSubInterest(t, l2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, l2, globalAccountName, \"bar\", time.Second)\n\n\t// Now shutdown the leafnode and check that any subscription for $G on l2 are gone.\n\tr1.Shutdown()\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tacc := l2.GlobalAccount()\n\t\tif n := acc.TotalSubs(); n != 5 {\n\t\t\treturn fmt.Errorf(\"Account %q should have 5 subs, got %v\", acc.GetName(), n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeLoopDetectionWithMultipleClusters(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tlo1.Cluster.Name = \"local\"\n\tlo1.Cluster.Host = \"127.0.0.1\"\n\tlo1.Cluster.Port = -1\n\tl1 := RunServer(lo1)\n\tdefer l1.Shutdown()\n\n\tlo2 := DefaultOptions()\n\tlo2.LeafNode.Host = \"127.0.0.1\"\n\tlo2.LeafNode.Port = -1\n\tlo2.Cluster.Name = \"local\"\n\tlo2.Cluster.Host = \"127.0.0.1\"\n\tlo2.Cluster.Port = -1\n\tlo2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.Cluster.Port))\n\tl2 := RunServer(lo2)\n\tdefer l2.Shutdown()\n\n\tcheckClusterFormed(t, l1, l2)\n\n\tro1 := DefaultOptions()\n\tro1.Cluster.Name = \"remote\"\n\tro1.Cluster.Host = \"127.0.0.1\"\n\tro1.Cluster.Port = -1\n\tro1.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{\n\t\t{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", lo1.LeafNode.Port)},\n\t\t{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", lo2.LeafNode.Port)},\n\t}}}\n\tr1 := RunServer(ro1)\n\tdefer r1.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 100)}\n\tr1.SetLogger(l, false, false)\n\n\tro2 := DefaultOptions()\n\tro2.Cluster.Name = \"remote\"\n\tro2.Cluster.Host = \"127.0.0.1\"\n\tro2.Cluster.Port = -1\n\tro2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ro1.Cluster.Port))\n\tro2.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tro2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{\n\t\t{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", lo1.LeafNode.Port)},\n\t\t{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", lo2.LeafNode.Port)},\n\t}}}\n\tr2 := RunServer(ro2)\n\tdefer r2.Shutdown()\n\n\tcheckClusterFormed(t, r1, r2)\n\tcheckLeafNodeConnected(t, r1)\n\tcheckLeafNodeConnected(t, r2)\n\n\tl1.Shutdown()\n\n\t// Now wait for r1 and r2 to reconnect, they should not have a problem of loop detection.\n\tcheckLeafNodeConnected(t, r1)\n\tcheckLeafNodeConnected(t, r2)\n\n\t// Wait and make sure we don't have a loop error\n\ttimeout := time.NewTimer(500 * time.Millisecond)\n\tfor {\n\t\tselect {\n\t\tcase err := <-l.errCh:\n\t\t\tif strings.Contains(err, \"Loop detected\") {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\tcase <-timeout.C:\n\t\t\t// OK, we are done.\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeUnsubOnRouteDisconnect(t *testing.T) {\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.Host = \"127.0.0.1\"\n\tlo1.LeafNode.Port = -1\n\tlo1.Cluster.Name = \"local\"\n\tlo1.Cluster.Host = \"127.0.0.1\"\n\tlo1.Cluster.Port = -1\n\tl1 := RunServer(lo1)\n\tdefer l1.Shutdown()\n\n\tlo2 := DefaultOptions()\n\tlo2.LeafNode.Host = \"127.0.0.1\"\n\tlo2.LeafNode.Port = -1\n\tlo2.Cluster.Name = \"local\"\n\tlo2.Cluster.Host = \"127.0.0.1\"\n\tlo2.Cluster.Port = -1\n\tlo2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.Cluster.Port))\n\tl2 := RunServer(lo2)\n\tdefer l2.Shutdown()\n\n\tcheckClusterFormed(t, l1, l2)\n\n\tu1, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo1.LeafNode.Port))\n\tu2, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo2.LeafNode.Port))\n\turls := []*url.URL{u1, u2}\n\n\tro1 := DefaultOptions()\n\t// DefaultOptions sets a cluster name, so make sure they are different.\n\t// Also, we don't have r1 and r2 clustered in this test, so set port to 0.\n\tro1.Cluster.Name = _EMPTY_\n\tro1.Cluster.Port = 0\n\tro1.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tro1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}}\n\tr1 := RunServer(ro1)\n\tdefer r1.Shutdown()\n\n\tro2 := DefaultOptions()\n\tro1.Cluster.Name = _EMPTY_\n\tro2.Cluster.Port = 0\n\tro2.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\t// Have this one point only to l2\n\tro2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u2}}}\n\tr2 := RunServer(ro2)\n\tdefer r2.Shutdown()\n\n\tcheckLeafNodeConnected(t, r1)\n\tcheckLeafNodeConnected(t, r2)\n\n\t// Create a subscription on r1.\n\tnc := natsConnect(t, r1.ClientURL())\n\tdefer nc.Close()\n\tsub := natsSubSync(t, nc, \"foo\")\n\tnatsFlush(t, nc)\n\n\tcheckSubInterest(t, l2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, r2, globalAccountName, \"foo\", time.Second)\n\n\tnc2 := natsConnect(t, r2.ClientURL())\n\tdefer nc2.Close()\n\tnatsPub(t, nc, \"foo\", []byte(\"msg1\"))\n\n\t// Check message received\n\tnatsNexMsg(t, sub, time.Second)\n\n\t// Now shutdown l1, l2 should update subscription interest to r2.\n\t// When r1 reconnects to l2, subscription should be updated too.\n\tl1.Shutdown()\n\n\t// Wait a bit (so that the check of interest is not OK just because\n\t// the route would not have been yet detected as broken), and check\n\t// interest still present on r2, l2.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckSubInterest(t, l2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, r2, globalAccountName, \"foo\", time.Second)\n\n\t// Check again that message received ok\n\tnatsPub(t, nc, \"foo\", []byte(\"msg2\"))\n\tnatsNexMsg(t, sub, time.Second)\n\n\t// Now close client. Interest should disappear on r2. Due to a bug,\n\t// it was not.\n\tnc.Close()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tacc := r2.GlobalAccount()\n\t\tif n := acc.Interest(\"foo\"); n != 0 {\n\t\t\treturn fmt.Errorf(\"Still interest on subject: %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeNoPingBeforeConnect(t *testing.T) {\n\to := DefaultOptions()\n\to.LeafNode.Port = -1\n\to.LeafNode.AuthTimeout = 0.5\n\t// For this test we need to disable compression, because we do use\n\t// the ping timer instead of the auth timer before the negotiation\n\t// is complete.\n\to.LeafNode.Compression.Mode = CompressionOff\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.LeafNode.Port)\n\tc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\tdefer c.Close()\n\n\t// Read the info\n\tbr := bufio.NewReader(c)\n\tc.SetReadDeadline(time.Now().Add(time.Second))\n\tl, _, err := br.ReadLine()\n\tif err != nil {\n\t\tt.Fatalf(\"Error on read: %v\", err)\n\t}\n\tif !strings.HasPrefix(string(l), \"INFO\") {\n\t\tt.Fatalf(\"Wrong proto: %q\", l)\n\t}\n\n\tvar leaf *client\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\ts.grMu.Lock()\n\t\tfor _, l := range s.grTmpClients {\n\t\t\tleaf = l\n\t\t\tbreak\n\t\t}\n\t\ts.grMu.Unlock()\n\t\tif leaf == nil {\n\t\t\treturn fmt.Errorf(\"No leaf connection found\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure that ping timer is not set\n\tleaf.mu.Lock()\n\tptmrSet := leaf.ping.tmr != nil\n\tleaf.mu.Unlock()\n\n\tif ptmrSet {\n\t\tt.Fatal(\"Ping timer was set before CONNECT was processed\")\n\t}\n\n\t// Send CONNECT\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error writing connect: %v\", err)\n\t}\n\n\t// Check that we correctly set the timer now\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tleaf.mu.Lock()\n\t\tptmrSet := leaf.ping.tmr != nil\n\t\tleaf.mu.Unlock()\n\t\tif !ptmrSet {\n\t\t\treturn fmt.Errorf(\"Timer still not set\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Reduce the first ping..\n\tleaf.mu.Lock()\n\tleaf.ping.tmr.Reset(15 * time.Millisecond)\n\tleaf.mu.Unlock()\n\n\t// Now consume that PING (we may get LS+, etc..)\n\tfor {\n\t\tc.SetReadDeadline(time.Now().Add(time.Second))\n\t\tl, _, err = br.ReadLine()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on read: %v\", err)\n\t\t}\n\t\tif strings.HasPrefix(string(l), \"PING\") {\n\t\t\tcheckLeafNodeConnected(t, s)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeNoMsgLoop(t *testing.T) {\n\thubConf := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tFOO {\n\t\t\t\tusers [\n\t\t\t\t\t{username: leaf, password: pass}\n\t\t\t\t\t{username: user, password: pass}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tname: \"hub\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tauthorization {\n\t\t\t\taccount: FOO\n\t\t\t}\n\t\t}\n\t`\n\tconfigS1 := createConfFile(t, []byte(fmt.Sprintf(hubConf, \"\")))\n\ts1, o1 := RunServerWithConfig(configS1)\n\tdefer s1.Shutdown()\n\n\tconfigS2S3 := createConfFile(t, []byte(fmt.Sprintf(hubConf, fmt.Sprintf(`routes: [\"nats://127.0.0.1:%d\"]`, o1.Cluster.Port))))\n\ts2, o2 := RunServerWithConfig(configS2S3)\n\tdefer s2.Shutdown()\n\n\ts3, _ := RunServerWithConfig(configS2S3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tcontentLN := `\n\t\tlisten: \"127.0.0.1:%d\"\n\t\taccounts {\n\t\t\tFOO {\n\t\t\t\tusers [\n\t\t\t\t\t{username: leaf, password: pass}\n\t\t\t\t\t{username: user, password: pass}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://leaf:pass@127.0.0.1:%d\"\n\t\t\t\t\taccount: FOO\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tlnconf := createConfFile(t, []byte(fmt.Sprintf(contentLN, -1, o1.LeafNode.Port)))\n\tsl1, slo1 := RunServerWithConfig(lnconf)\n\tdefer sl1.Shutdown()\n\n\tsl2, slo2 := RunServerWithConfig(lnconf)\n\tdefer sl2.Shutdown()\n\n\tcheckLeafNodeConnected(t, sl1)\n\tcheckLeafNodeConnected(t, sl2)\n\n\t// Create users on each leafnode\n\tnc1, err := nats.Connect(fmt.Sprintf(\"nats://user:pass@127.0.0.1:%d\", slo1.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\trch := make(chan struct{}, 1)\n\tnc2, err := nats.Connect(\n\t\tfmt.Sprintf(\"nats://user:pass@127.0.0.1:%d\", slo2.Port),\n\t\tnats.ReconnectWait(50*time.Millisecond),\n\t\tnats.ReconnectHandler(func(_ *nats.Conn) {\n\t\t\trch <- struct{}{}\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Create queue subs on sl2\n\tnc2.QueueSubscribe(\"foo\", \"bar\", func(_ *nats.Msg) {})\n\tnc2.QueueSubscribe(\"foo\", \"bar\", func(_ *nats.Msg) {})\n\tnc2.Flush()\n\n\t// Wait for interest to propagate to sl1\n\tcheckSubInterest(t, sl1, \"FOO\", \"foo\", 250*time.Millisecond)\n\n\t// Create sub on sl1\n\tch := make(chan *nats.Msg, 10)\n\tnc1.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tselect {\n\t\tcase ch <- m:\n\t\tdefault:\n\t\t}\n\t})\n\tnc1.Flush()\n\n\tcheckSubInterest(t, sl2, \"FOO\", \"foo\", 250*time.Millisecond)\n\n\t// Produce from sl1\n\tnc1.Publish(\"foo\", []byte(\"msg1\"))\n\n\t// Check message is received by plain sub\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not receive message\")\n\t}\n\n\t// Restart leaf node, this time make sure we connect to 2nd server.\n\tsl2.Shutdown()\n\n\t// Use config file but this time reuse the client port and set the 2nd server for\n\t// the remote leaf node port.\n\tlnconf = createConfFile(t, []byte(fmt.Sprintf(contentLN, slo2.Port, o2.LeafNode.Port)))\n\tsl2, _ = RunServerWithConfig(lnconf)\n\tdefer sl2.Shutdown()\n\n\tcheckLeafNodeConnected(t, sl2)\n\n\t// Wait for client to reconnect\n\tselect {\n\tcase <-rch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Did not reconnect\")\n\t}\n\n\t// Produce a new messages\n\tfor i := 0; i < 10; i++ {\n\t\tnc1.Publish(\"foo\", []byte(fmt.Sprintf(\"msg%d\", 2+i)))\n\n\t\t// Check sub receives 1 message\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"Did not receive message\")\n\t\t}\n\t\t// Check that there is no more...\n\t\tselect {\n\t\tcase m := <-ch:\n\t\t\tt.Fatalf(\"Loop: received second message %s\", m.Data)\n\t\tcase <-time.After(50 * time.Millisecond):\n\t\t\t// OK\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeInterestPropagationDaisychain(t *testing.T) {\n\taTmpl := `\n\t\tport: %d\n\t\tleafnodes {\n\t\t port: %d\n\t\t}\n\t\t`\n\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(aTmpl, -1, -1)))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\n\taPort := sA.opts.Port\n\taLeafPort := sA.opts.LeafNode.Port\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes = [{\n\t\t\t\turl:\"nats://127.0.0.1:%d\"\n\t\t\t}]\n\t\t}`, aLeafPort)))\n\tsB, _ := RunServerWithConfig(confB)\n\tdefer sB.Shutdown()\n\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes = [{url:\"nats://127.0.0.1:%d\"}]\n\t\t}`, sB.opts.LeafNode.Port)))\n\tsC, _ := RunServerWithConfig(confC)\n\tdefer sC.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, sC, 1)\n\tcheckLeafNodeConnectedCount(t, sB, 2)\n\tcheckLeafNodeConnectedCount(t, sA, 1)\n\n\tncC := natsConnect(t, sC.ClientURL())\n\tdefer ncC.Close()\n\t_, err := ncC.SubscribeSync(\"foo\")\n\trequire_NoError(t, err)\n\trequire_NoError(t, ncC.Flush())\n\n\tcheckSubInterest(t, sC, \"$G\", \"foo\", time.Second)\n\tcheckSubInterest(t, sB, \"$G\", \"foo\", time.Second)\n\tcheckSubInterest(t, sA, \"$G\", \"foo\", time.Second)\n\n\tncA := natsConnect(t, sA.ClientURL())\n\tdefer ncA.Close()\n\n\tsA.Shutdown()\n\tsA.WaitForShutdown()\n\n\tconfAA := createConfFile(t, []byte(fmt.Sprintf(aTmpl, aPort, aLeafPort)))\n\tsAA, _ := RunServerWithConfig(confAA)\n\tdefer sAA.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, sAA, 1)\n\tcheckLeafNodeConnectedCount(t, sB, 2)\n\tcheckLeafNodeConnectedCount(t, sC, 1)\n\n\tcheckSubInterest(t, sC, \"$G\", \"foo\", time.Second)\n\tcheckSubInterest(t, sB, \"$G\", \"foo\", time.Second)\n\tcheckSubInterest(t, sAA, \"$G\", \"foo\", time.Second) // failure issue 2448\n}\n\nfunc TestLeafNodeQueueGroupDistribution(t *testing.T) {\n\thc := createClusterWithName(t, \"HUB\", 3)\n\tdefer hc.shutdown()\n\n\t// Now have a cluster of leafnodes with each one connecting to corresponding HUB(n) node.\n\tc1 := `\n\tserver_name: LEAF1\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1 }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, hc.opts[0].LeafNode.Port)))\n\tln1, lopts1 := RunServerWithConfig(lconf1)\n\tdefer ln1.Shutdown()\n\n\tc2 := `\n\tserver_name: LEAF2\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, hc.opts[1].LeafNode.Port)))\n\tln2, _ := RunServerWithConfig(lconf2)\n\tdefer ln2.Shutdown()\n\n\tc3 := `\n\tserver_name: LEAF3\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, hc.opts[2].LeafNode.Port)))\n\tln3, _ := RunServerWithConfig(lconf3)\n\tdefer ln3.Shutdown()\n\n\t// Check leaf cluster is formed and all connected to the HUB.\n\tlnServers := []*Server{ln1, ln2, ln3}\n\tcheckClusterFormed(t, lnServers...)\n\tfor _, s := range lnServers {\n\t\tcheckLeafNodeConnected(t, s)\n\t}\n\t// Check each node in the hub has 1 connection from the leaf cluster.\n\tfor i := 0; i < 3; i++ {\n\t\tcheckLeafNodeConnectedCount(t, hc.servers[i], 1)\n\t}\n\n\t// Create a client and qsub on LEAF1 and LEAF2.\n\tnc1 := natsConnect(t, ln1.ClientURL())\n\tdefer nc1.Close()\n\tvar qsub1Count atomic.Int32\n\tnatsQueueSub(t, nc1, \"foo\", \"queue1\", func(_ *nats.Msg) {\n\t\tqsub1Count.Add(1)\n\t})\n\tnatsFlush(t, nc1)\n\n\tnc2 := natsConnect(t, ln2.ClientURL())\n\tdefer nc2.Close()\n\tvar qsub2Count atomic.Int32\n\tnatsQueueSub(t, nc2, \"foo\", \"queue1\", func(_ *nats.Msg) {\n\t\tqsub2Count.Add(1)\n\t})\n\tnatsFlush(t, nc2)\n\n\t// Make sure that the propagation interest is done before sending.\n\tfor _, s := range hc.servers {\n\t\tgacc := s.GlobalAccount()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := gacc.Interest(\"foo\"); n != 2 {\n\t\t\t\treturn fmt.Errorf(\"Expected interest for %q to be 2, got %v\", \"foo\", n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsendAndCheck := func(idx int) {\n\t\tt.Helper()\n\t\tnchub := natsConnect(t, hc.servers[idx].ClientURL())\n\t\tdefer nchub.Close()\n\t\ttotal := 1000\n\t\tfor i := 0; i < total; i++ {\n\t\t\tnatsPub(t, nchub, \"foo\", []byte(\"from hub\"))\n\t\t}\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif trecv := int(qsub1Count.Load() + qsub2Count.Load()); trecv != total {\n\t\t\t\treturn fmt.Errorf(\"Expected %v messages, got %v\", total, trecv)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Now that we have made sure that all messages were received,\n\t\t// check that qsub1 and qsub2 are getting at least some.\n\t\tif n := int(qsub1Count.Load()); n <= total/10 {\n\t\t\tt.Fatalf(\"Expected qsub1 to get some messages, but got %v\", n)\n\t\t}\n\t\tif n := int(qsub2Count.Load()); n <= total/10 {\n\t\t\tt.Fatalf(\"Expected qsub2 to get some messages, but got %v\", n)\n\t\t}\n\t\t// Reset the counters.\n\t\tqsub1Count.Store(0)\n\t\tqsub2Count.Store(0)\n\t}\n\t// Send from HUB1\n\tsendAndCheck(0)\n\t// Send from HUB2\n\tsendAndCheck(1)\n\t// Send from HUB3\n\tsendAndCheck(2)\n}\n\nfunc TestLeafNodeQueueGroupDistributionVariant(t *testing.T) {\n\thc := createClusterWithName(t, \"HUB\", 3)\n\tdefer hc.shutdown()\n\n\t// Now have a cluster of leafnodes with LEAF1 and LEAF2 connecting to HUB1.\n\tc1 := `\n\tserver_name: LEAF1\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1 }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, hc.opts[0].LeafNode.Port)))\n\tln1, lopts1 := RunServerWithConfig(lconf1)\n\tdefer ln1.Shutdown()\n\n\tc2 := `\n\tserver_name: LEAF2\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, hc.opts[0].LeafNode.Port)))\n\tln2, _ := RunServerWithConfig(lconf2)\n\tdefer ln2.Shutdown()\n\n\t// And LEAF3 to HUB3\n\tc3 := `\n\tserver_name: LEAF3\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, hc.opts[2].LeafNode.Port)))\n\tln3, _ := RunServerWithConfig(lconf3)\n\tdefer ln3.Shutdown()\n\n\t// Check leaf cluster is formed and all connected to the HUB.\n\tlnServers := []*Server{ln1, ln2, ln3}\n\tcheckClusterFormed(t, lnServers...)\n\tfor _, s := range lnServers {\n\t\tcheckLeafNodeConnected(t, s)\n\t}\n\t// Check that HUB1 has 2 leaf connections, HUB2 has 0 and HUB3 has 1.\n\tcheckLeafNodeConnectedCount(t, hc.servers[0], 2)\n\tcheckLeafNodeConnectedCount(t, hc.servers[1], 0)\n\tcheckLeafNodeConnectedCount(t, hc.servers[2], 1)\n\n\t// Create a client and qsub on LEAF1 and LEAF2.\n\tnc1 := natsConnect(t, ln1.ClientURL())\n\tdefer nc1.Close()\n\tvar qsub1Count atomic.Int32\n\tnatsQueueSub(t, nc1, \"foo\", \"queue1\", func(_ *nats.Msg) {\n\t\tqsub1Count.Add(1)\n\t})\n\tnatsFlush(t, nc1)\n\n\tnc2 := natsConnect(t, ln2.ClientURL())\n\tdefer nc2.Close()\n\tvar qsub2Count atomic.Int32\n\tnatsQueueSub(t, nc2, \"foo\", \"queue1\", func(_ *nats.Msg) {\n\t\tqsub2Count.Add(1)\n\t})\n\tnatsFlush(t, nc2)\n\n\t// Make sure that the propagation interest is done before sending.\n\tfor i, s := range hc.servers {\n\t\tgacc := s.GlobalAccount()\n\t\tvar ei int\n\t\tswitch i {\n\t\tcase 0:\n\t\t\tei = 2\n\t\tdefault:\n\t\t\tei = 1\n\t\t}\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif n := gacc.Interest(\"foo\"); n != ei {\n\t\t\t\treturn fmt.Errorf(\"Expected interest for %q to be %d, got %v\", \"foo\", ei, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsendAndCheck := func(idx int) {\n\t\tt.Helper()\n\t\tnchub := natsConnect(t, hc.servers[idx].ClientURL())\n\t\tdefer nchub.Close()\n\t\ttotal := 1000\n\t\tfor i := 0; i < total; i++ {\n\t\t\tnatsPub(t, nchub, \"foo\", []byte(\"from hub\"))\n\t\t}\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif trecv := int(qsub1Count.Load() + qsub2Count.Load()); trecv != total {\n\t\t\t\treturn fmt.Errorf(\"Expected %v messages, got %v\", total, trecv)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Now that we have made sure that all messages were received,\n\t\t// check that qsub1 and qsub2 are getting at least some.\n\t\tif n := int(qsub1Count.Load()); n <= total/10 {\n\t\t\tt.Fatalf(\"Expected qsub1 to get some messages, but got %v (qsub2=%v)\", n, qsub2Count.Load())\n\t\t}\n\t\tif n := int(qsub2Count.Load()); n <= total/10 {\n\t\t\tt.Fatalf(\"Expected qsub2 to get some messages, but got %v (qsub1=%v)\", n, qsub1Count.Load())\n\t\t}\n\t\t// Reset the counters.\n\t\tqsub1Count.Store(0)\n\t\tqsub2Count.Store(0)\n\t}\n\t// Send from HUB1\n\tsendAndCheck(0)\n\t// Send from HUB2\n\tsendAndCheck(1)\n\t// Send from HUB3\n\tsendAndCheck(2)\n}\n\nfunc TestLeafNodeQueueGroupDistributionWithDaisyChainAndGateway(t *testing.T) {\n\tSetGatewaysSolicitDelay(0)\n\tdefer ResetGatewaysSolicitDelay()\n\n\t// We create a sort of a ladder of servers with connections that look like this:\n\t//\n\t// D1 <--- route ---> D2\n\t// | |\n\t// GW GW\n\t// | |\n\t// C1 <--- route ---> C2\n\t// | |\n\t// Leaf Leaf\n\t// | |\n\t// B1 <--- route ---> B2\n\t// | |\n\t// Leaf Leaf\n\t// | |\n\t// A1 <--- route ---> A2\n\t//\n\t// We will then place queue subscriptions (different sub-tests) on A1, A2\n\t// B1, B2, D1 and D2.\n\n\taccs := `\n\t\taccounts {\n\t\t\tSYS: {users: [{user:sys, password: pwd}]}\n\t\t\tUSER: {users: [{user:user, password: pwd}]}\n\t\t}\n\t\tsystem_account: SYS\n\t`\n\tdConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"D\"\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t\tgateway {\n\t\t\tname: \"D\"\n\t\t\tport: -1\n\t\t}\n\t`\n\td1Conf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, \"GW1\", _EMPTY_)))\n\td1, d1Opts := RunServerWithConfig(d1Conf)\n\tdefer d1.Shutdown()\n\n\td2Conf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, \"GW2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", d1Opts.Cluster.Port))))\n\td2, d2Opts := RunServerWithConfig(d2Conf)\n\tdefer d2.Shutdown()\n\n\tcheckClusterFormed(t, d1, d2)\n\n\tleafCConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: C\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t\tgateway {\n\t\t\tname: C\n\t\t\tport: -1\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: D\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tc1Conf := createConfFile(t, []byte(fmt.Sprintf(leafCConf, accs, \"C1\", _EMPTY_, d1Opts.Gateway.Port)))\n\tc1, c1Opts := RunServerWithConfig(c1Conf)\n\tdefer c1.Shutdown()\n\n\twaitForOutboundGateways(t, c1, 1, time.Second)\n\twaitForInboundGateways(t, d1, 1, time.Second)\n\n\tc2Conf := createConfFile(t, []byte(fmt.Sprintf(leafCConf, accs, \"C2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", c1Opts.Cluster.Port), d2Opts.Gateway.Port)))\n\tc2, c2Opts := RunServerWithConfig(c2Conf)\n\tdefer c2.Shutdown()\n\n\twaitForOutboundGateways(t, c2, 1, time.Second)\n\twaitForInboundGateways(t, d2, 1, time.Second)\n\n\tcheckClusterFormed(t, c1, c2)\n\n\tleafABConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://user:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: USER\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tb1Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, \"B1\", \"B\", _EMPTY_, c1Opts.LeafNode.Port)))\n\tb1, b1Opts := RunServerWithConfig(b1Conf)\n\tdefer b1.Shutdown()\n\n\tcheckLeafNodeConnected(t, b1)\n\tcheckLeafNodeConnected(t, c1)\n\n\tb2Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, \"B2\", \"B\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", b1Opts.Cluster.Port), c2Opts.LeafNode.Port)))\n\tb2, b2Opts := RunServerWithConfig(b2Conf)\n\tdefer b2.Shutdown()\n\n\tcheckLeafNodeConnected(t, b2)\n\tcheckLeafNodeConnected(t, c2)\n\tcheckClusterFormed(t, b1, b2)\n\n\ta1Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, \"A1\", \"A\", _EMPTY_, b1Opts.LeafNode.Port)))\n\ta1, a1Opts := RunServerWithConfig(a1Conf)\n\tdefer a1.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, b1, 2)\n\tcheckLeafNodeConnected(t, a1)\n\n\ta2Conf := createConfFile(t, []byte(fmt.Sprintf(leafABConf, accs, \"A2\", \"A\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", a1Opts.Cluster.Port), b2Opts.LeafNode.Port)))\n\ta2, _ := RunServerWithConfig(a2Conf)\n\tdefer a2.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, b2, 2)\n\tcheckLeafNodeConnected(t, a2)\n\tcheckClusterFormed(t, a1, a2)\n\n\t// Create our client connections to all servers where we may need to have\n\t// queue subscriptions.\n\tncD1 := natsConnect(t, d1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncD1.Close()\n\tncD2 := natsConnect(t, d2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncD2.Close()\n\tncB1 := natsConnect(t, b1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncB1.Close()\n\tncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncB2.Close()\n\tncA1 := natsConnect(t, a1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncA1.Close()\n\tncA2 := natsConnect(t, a2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncA2.Close()\n\n\t// Helper to check that the interest is propagated to all servers\n\tcheckInterest := func(t *testing.T, subj string) {\n\t\tt.Helper()\n\t\tfor _, s := range []*Server{a1, a2, b1, b2, c1, c2, d1, d2} {\n\t\t\tacc, err := s.LookupAccount(\"USER\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tif acc.Interest(subj) != 0 {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Still no interest on %q in server %q\", subj, s)\n\t\t\t})\n\t\t}\n\t}\n\n\t// Helper to send messages on given subject. We are always sending\n\t// from cluster B in this test, but we pick randomly between B1 and B2.\n\ttotal := 1000\n\tsend := func(t *testing.T, subj string) {\n\t\tfor i := 0; i < total; i++ {\n\t\t\tvar nc *nats.Conn\n\t\t\tif fastrand.Uint32n(2) == 0 {\n\t\t\t\tnc = ncB1\n\t\t\t} else {\n\t\t\t\tnc = ncB2\n\t\t\t}\n\t\t\tnatsPub(t, nc, subj, []byte(fmt.Sprintf(\"msg_%d\", i+1)))\n\t\t}\n\t}\n\n\tconst queue = \"queue\"\n\n\tfor i, test := range []struct {\n\t\tname string\n\t\ta1 bool\n\t\ta2 bool\n\t\tb1 bool\n\t\tb2 bool\n\t\td1 bool\n\t\td2 bool\n\t}{\n\t\t// Cases with QSubs in A, B and D\n\t\t{\"A1 __ B1 __ D1 __\", true, false, true, false, true, false},\n\t\t{\"A1 __ B1 __ __ D2\", true, false, true, false, false, true},\n\t\t{\"A1 __ B1 __ D1 D2\", true, false, true, false, true, true},\n\n\t\t{\"A1 __ __ B2 D1 __\", true, false, false, true, true, false},\n\t\t{\"A1 __ __ B2 __ D2\", true, false, false, true, false, true},\n\t\t{\"A1 __ __ B2 D1 D2\", true, false, false, true, true, true},\n\n\t\t{\"A1 __ B1 B2 D1 __\", true, false, true, true, true, false},\n\t\t{\"A1 __ B1 B2 __ D2\", true, false, true, true, false, true},\n\t\t{\"A1 __ B1 B2 D1 D2\", true, false, true, true, true, true},\n\n\t\t{\"__ A2 B1 __ D1 __\", false, true, true, false, true, false},\n\t\t{\"__ A2 B1 __ __ D2\", false, true, true, false, false, true},\n\t\t{\"__ A2 B1 __ D1 D2\", false, true, true, false, true, true},\n\n\t\t{\"__ A2 __ B2 D1 __\", false, true, false, true, true, false},\n\t\t{\"__ A2 __ B2 __ D2\", false, true, false, true, false, true},\n\t\t{\"__ A2 __ B2 D1 D2\", false, true, false, true, true, true},\n\n\t\t{\"__ A2 B1 B2 D1 __\", false, true, true, true, true, false},\n\t\t{\"__ A2 B1 B2 __ D2\", false, true, true, true, false, true},\n\t\t{\"__ A2 B1 B2 D1 D2\", false, true, true, true, true, true},\n\n\t\t{\"A1 A2 B1 __ D1 __\", true, true, true, false, true, false},\n\t\t{\"A1 A2 B1 __ __ D2\", true, true, true, false, false, true},\n\t\t{\"A1 A2 B1 __ D1 D2\", true, true, true, false, true, true},\n\n\t\t{\"A1 A2 __ B2 D1 __\", true, true, false, true, true, false},\n\t\t{\"A1 A2 __ B2 __ D2\", true, true, false, true, false, true},\n\t\t{\"A1 A2 __ B2 D1 D2\", true, true, false, true, true, true},\n\n\t\t{\"A1 A2 B1 B2 D1 __\", true, true, true, true, true, false},\n\t\t{\"A1 A2 B1 B2 __ D2\", true, true, true, true, false, true},\n\t\t{\"A1 A2 B1 B2 D1 D2\", true, true, true, true, true, true},\n\n\t\t// Now without any QSub in B cluster (so just A and D)\n\t\t{\"A1 __ __ __ D1 __\", true, false, false, false, true, false},\n\t\t{\"A1 __ __ __ __ D2\", true, false, false, false, false, true},\n\t\t{\"A1 __ __ __ D1 D2\", true, false, false, false, true, true},\n\n\t\t{\"__ A2 __ __ D1 __\", false, true, false, false, true, false},\n\t\t{\"__ A2 __ __ __ D2\", false, true, false, false, false, true},\n\t\t{\"__ A2 __ __ D1 D2\", false, true, false, false, true, true},\n\n\t\t{\"A1 A2 __ __ D1 __\", true, true, false, false, true, false},\n\t\t{\"A1 A2 __ __ __ D2\", true, true, false, false, false, true},\n\t\t{\"A1 A2 __ __ D1 D2\", true, true, false, false, true, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i+1)\n\t\t\tvar aCount, bCount, dCount atomic.Int32\n\t\t\tif test.a1 {\n\t\t\t\tqsA1 := natsQueueSub(t, ncA1, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\taCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsA1.Unsubscribe()\n\t\t\t}\n\t\t\tif test.a2 {\n\t\t\t\tqsA2 := natsQueueSub(t, ncA2, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\taCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsA2.Unsubscribe()\n\t\t\t}\n\t\t\tif test.b1 {\n\t\t\t\tqsB1 := natsQueueSub(t, ncB1, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\tbCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsB1.Unsubscribe()\n\t\t\t}\n\t\t\tif test.b2 {\n\t\t\t\tqsB2 := natsQueueSub(t, ncB2, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\tbCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsB2.Unsubscribe()\n\t\t\t}\n\t\t\tif test.d1 {\n\t\t\t\tqsD1 := natsQueueSub(t, ncD1, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\tdCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsD1.Unsubscribe()\n\t\t\t}\n\t\t\tif test.d2 {\n\t\t\t\tqsD2 := natsQueueSub(t, ncD2, subj, queue, func(_ *nats.Msg) {\n\t\t\t\t\tdCount.Add(1)\n\t\t\t\t})\n\t\t\t\tdefer qsD2.Unsubscribe()\n\t\t\t}\n\t\t\tcheckInterest(t, subj)\n\n\t\t\t// Now send messages\n\t\t\tsend(t, subj)\n\n\t\t\t// Check that appropriate queue subs receive all messages.\n\t\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tn := aCount.Load() + bCount.Load() + dCount.Load()\n\t\t\t\tif n == int32(total) {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Got only %v/%v messages (a=%v b=%v d=%v)\", n, total, aCount.Load(), bCount.Load(), dCount.Load())\n\t\t\t})\n\t\t\t// When there is (are) qsub(s) on b, then only B should\n\t\t\t// get the messages. Otherwise, it should be between A and D\n\t\t\tif test.b1 || test.b2 {\n\t\t\t\trequire_Equal(t, aCount.Load(), 0)\n\t\t\t\trequire_Equal(t, dCount.Load(), 0)\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, bCount.Load(), 0)\n\t\t\t\t// We should have receive some on A and D\n\t\t\t\trequire_True(t, aCount.Load() > 0)\n\t\t\t\trequire_True(t, dCount.Load() > 0)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeAndGatewaysSingleMsgPerQueueGroup(t *testing.T) {\n\tSetGatewaysSolicitDelay(0)\n\tdefer ResetGatewaysSolicitDelay()\n\n\taccs := `\n\t\taccounts {\n\t\t\tSYS: {users: [{user:sys, password: pwd}]}\n\t\t\tUSER: {users: [{user:user, password: pwd}]}\n\t\t}\n\t\tsystem_account: SYS\n\t`\n\tgwUSConfTmpl := `\n\t\t%s\n\t\tserver_name: GW_US\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tgateway {\n\t\t\tname: US\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tgwUSConf := createConfFile(t, []byte(fmt.Sprintf(gwUSConfTmpl, accs)))\n\tgwUS, gwUSOpts := RunServerWithConfig(gwUSConf)\n\tdefer gwUS.Shutdown()\n\n\tgwEUConfTmpl := `\n\t\t%s\n\t\tserver_name: GW_EU\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tgateway {\n\t\t\tname: EU\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: US\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tgwEUConf := createConfFile(t, []byte(fmt.Sprintf(gwEUConfTmpl, accs, gwUSOpts.Gateway.Port)))\n\tgwEU, gwEUOpts := RunServerWithConfig(gwEUConf)\n\tdefer gwEU.Shutdown()\n\n\twaitForOutboundGateways(t, gwUS, 1, time.Second)\n\twaitForOutboundGateways(t, gwEU, 1, time.Second)\n\twaitForInboundGateways(t, gwUS, 1, time.Second)\n\twaitForInboundGateways(t, gwEU, 1, time.Second)\n\n\tleafConfTmpl := `\n\t\t%s\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://user:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: USER\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tleafUSConf := createConfFile(t, []byte(fmt.Sprintf(leafConfTmpl, accs, \"LEAF_US\", gwUSOpts.LeafNode.Port)))\n\tleafUS, _ := RunServerWithConfig(leafUSConf)\n\tdefer leafUS.Shutdown()\n\tcheckLeafNodeConnected(t, leafUS)\n\n\tleafEUConf := createConfFile(t, []byte(fmt.Sprintf(leafConfTmpl, accs, \"LEAF_EU\", gwEUOpts.LeafNode.Port)))\n\tleafEU, _ := RunServerWithConfig(leafEUConf)\n\tdefer leafEU.Shutdown()\n\tcheckLeafNodeConnected(t, leafEU)\n\n\t// Order is important! (see rest of test to understand why)\n\tvar usLeafQ, usLeafPS, usGWQ, usGWPS, euGWQ, euGWPS, euLeafQ, euLeafPS, euLeafQBaz atomic.Int32\n\tcounters := []*atomic.Int32{&usLeafQ, &usLeafPS, &usGWQ, &usGWPS, &euGWQ, &euGWPS, &euLeafQ, &euLeafPS, &euLeafQBaz}\n\tcounterNames := []string{\"usLeafQ\", \"usLeafPS\", \"usGWQ\", \"usGWPS\", \"euGWQ\", \"euGWPS\", \"euLeafQ\", \"euLeafPS\", \"euLeafQBaz\"}\n\tif len(counters) != len(counterNames) {\n\t\tpanic(\"Fix test!\")\n\t}\n\tresetCounters := func() {\n\t\tfor _, a := range counters {\n\t\t\ta.Store(0)\n\t\t}\n\t}\n\n\t// This test will always produce from the US leaf.\n\tncProd := natsConnect(t, leafUS.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncProd.Close()\n\n\ttotal := int32(1)\n\tcheck := func(t *testing.T, expected []int32) {\n\t\ttime.Sleep(50 * time.Millisecond)\n\t\tresetCounters()\n\t\tfor i := 0; i < int(total); i++ {\n\t\t\tnatsPub(t, ncProd, \"foo.1\", []byte(\"hello\"))\n\t\t}\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tfor i := 0; i < len(expected); i++ {\n\t\t\t\tif n := counters[i].Load(); n != expected[i] {\n\t\t\t\t\treturn fmt.Errorf(\"Expected counter %q to be %v, got %v\", counterNames[i], expected[i], n)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// \"usLeafQ\", \"usLeafPS\", \"usGWQ\", \"usGWPS\", \"euGWQ\", \"euGWPS\", \"euLeafQ\", \"euLeafPS\", \"euLeafQBaz\"\n\tfor _, test := range []struct {\n\t\tsubs []int\n\t\texpected []int32\n\t}{\n\t\t// We will always have the qsub on leaf EU, and have some permutations\n\t\t// of queue and plain subs on other server(s) and check we get the\n\t\t// expected distribution.\n\n\t\t// Simple test firs, qsubs on leaf US and leaf EU, all messages stay in leaf US.\n\t\t{\n\t\t\t[]int{1, 0, 0, 0, 0, 0, 1, 0, 0},\n\t\t\t[]int32{total, 0, 0, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t// Move the queue sub from leaf US to GW US.\n\t\t{\n\t\t\t[]int{0, 0, 1, 0, 0, 0, 1, 0, 0},\n\t\t\t[]int32{0, 0, total, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t// Now move it to GW EU.\n\t\t{\n\t\t\t[]int{0, 0, 0, 0, 1, 0, 1, 0, 0},\n\t\t\t[]int32{0, 0, 0, 0, total, 0, 0, 0, 0},\n\t\t},\n\n\t\t// More combinations...\n\t\t{\n\t\t\t[]int{1, 1, 0, 0, 0, 0, 1, 0, 0},\n\t\t\t[]int32{total, total, 0, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{0, 1, 1, 0, 0, 0, 1, 0, 0},\n\t\t\t[]int32{0, total, total, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{0, 1, 1, 1, 0, 0, 1, 0, 0},\n\t\t\t[]int32{0, total, total, total, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{0, 1, 0, 1, 1, 0, 1, 0, 0},\n\t\t\t[]int32{0, total, 0, total, total, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{0, 1, 0, 1, 1, 1, 1, 0, 0},\n\t\t\t[]int32{0, total, 0, total, total, total, 0, 0, 0},\n\t\t},\n\t\t// If we have the qsub in leaf US, does not matter if we have\n\t\t// qsubs in GW US and EU, only leaf US should receive the messages,\n\t\t// but plain sub in GW servers should get them too.\n\t\t{\n\t\t\t[]int{1, 1, 1, 0, 0, 0, 1, 0, 0},\n\t\t\t[]int32{total, total, 0, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 0, 0, 1, 0, 0},\n\t\t\t[]int32{total, total, 0, total, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 1, 0, 1, 0, 0},\n\t\t\t[]int32{total, total, 0, total, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 1, 1, 1, 0, 0},\n\t\t\t[]int32{total, total, 0, total, 0, total, 0, 0, 0},\n\t\t},\n\t\t// Now back to a qsub on leaf US and leaf EU, but introduce plain sub\n\t\t// interest in leaf EU\n\t\t{\n\t\t\t[]int{1, 0, 0, 0, 0, 0, 1, 1, 0},\n\t\t\t[]int32{total, 0, 0, 0, 0, 0, 0, total, 0},\n\t\t},\n\t\t// And add a different queue group in leaf EU and it should get the messages too.\n\t\t{\n\t\t\t[]int{1, 0, 0, 0, 0, 0, 1, 1, 1},\n\t\t\t[]int32{total, 0, 0, 0, 0, 0, 0, total, total},\n\t\t},\n\t\t// Keep plain and baz queue sub interests in leaf EU and add more combinations.\n\t\t{\n\t\t\t[]int{1, 1, 0, 0, 0, 0, 1, 1, 1},\n\t\t\t[]int32{total, total, 0, 0, 0, 0, 0, total, total},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 0, 0, 0, 1, 1, 1},\n\t\t\t[]int32{total, total, 0, 0, 0, 0, 0, total, total},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 0, 0, 1, 1, 1},\n\t\t\t[]int32{total, total, 0, total, 0, 0, 0, total, total},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 1, 0, 1, 1, 1},\n\t\t\t[]int32{total, total, 0, total, 0, 0, 0, total, total},\n\t\t},\n\t\t{\n\t\t\t[]int{1, 1, 1, 1, 1, 1, 1, 1, 1},\n\t\t\t[]int32{total, total, 0, total, 0, total, 0, total, total},\n\t\t},\n\t} {\n\t\tt.Run(_EMPTY_, func(t *testing.T) {\n\t\t\tif len(test.subs) != len(counters) || len(test.expected) != len(counters) {\n\t\t\t\tpanic(\"Fix test\")\n\t\t\t}\n\n\t\t\tncUS := natsConnect(t, leafUS.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\t\t\tdefer ncUS.Close()\n\n\t\t\tncGWUS := natsConnect(t, gwUS.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\t\t\tdefer ncGWUS.Close()\n\n\t\t\tncGWEU := natsConnect(t, gwEU.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\t\t\tdefer ncGWEU.Close()\n\n\t\t\tncEU := natsConnect(t, leafEU.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\t\t\tdefer ncEU.Close()\n\n\t\t\tif test.subs[0] > 0 {\n\t\t\t\tnatsQueueSub(t, ncUS, \"foo.*\", \"bar\", func(_ *nats.Msg) {\n\t\t\t\t\tusLeafQ.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tif test.subs[1] > 0 {\n\t\t\t\tnatsSub(t, ncUS, \"foo.>\", func(_ *nats.Msg) {\n\t\t\t\t\tusLeafPS.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tnatsFlush(t, ncUS)\n\t\t\tif test.subs[2] > 0 {\n\t\t\t\tnatsQueueSub(t, ncGWUS, \"foo.*\", \"bar\", func(_ *nats.Msg) {\n\t\t\t\t\tusGWQ.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tif test.subs[3] > 0 {\n\t\t\t\tnatsSub(t, ncGWUS, \"foo.>\", func(_ *nats.Msg) {\n\t\t\t\t\tusGWPS.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tnatsFlush(t, ncGWUS)\n\t\t\tif test.subs[4] > 0 {\n\t\t\t\tnatsQueueSub(t, ncGWEU, \"foo.*\", \"bar\", func(_ *nats.Msg) {\n\t\t\t\t\teuGWQ.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tif test.subs[5] > 0 {\n\t\t\t\tnatsSub(t, ncGWEU, \"foo.>\", func(_ *nats.Msg) {\n\t\t\t\t\teuGWPS.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tnatsFlush(t, ncGWEU)\n\t\t\tif test.subs[6] > 0 {\n\t\t\t\tnatsQueueSub(t, ncEU, \"foo.*\", \"bar\", func(_ *nats.Msg) {\n\t\t\t\t\teuLeafQ.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tif test.subs[7] > 0 {\n\t\t\t\tnatsSub(t, ncEU, \"foo.>\", func(_ *nats.Msg) {\n\t\t\t\t\teuLeafPS.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tif test.subs[8] > 0 {\n\t\t\t\t// Create on different group, called \"baz\"\n\t\t\t\tnatsQueueSub(t, ncEU, \"foo.*\", \"baz\", func(_ *nats.Msg) {\n\t\t\t\t\teuLeafQBaz.Add(1)\n\t\t\t\t})\n\t\t\t}\n\t\t\tnatsFlush(t, ncEU)\n\n\t\t\t// Check that we have what we expect.\n\t\t\tcheck(t, test.expected)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeQueueGroupWeightCorrectOnConnectionCloseInSuperCluster(t *testing.T) {\n\tSetGatewaysSolicitDelay(0)\n\tdefer ResetGatewaysSolicitDelay()\n\n\t//\n\t// D\n\t// |\n\t// Leaf\n\t// |\n\t// v\n\t// C\n\t// ^ ^\n\t// / \\\n\t// GW GW\n\t// / \\\n\t// v \\\n\t// B1 <--- route ---> B2 <----*----------*\n\t// ^ <---* | |\n\t// | | Leaf Leaf\n\t// Leaf *-- Leaf ---* | |\n\t// | | | |\n\t// A1 <--- route ---> A2 OTHER1 OTHER2\n\t//\n\n\taccs := `\n\t\taccounts {\n\t\t\tSYS: {users: [{user:sys, password: pwd}]}\n\t\t\tUSER: {users: [{user:user, password: pwd}]}\n\t\t}\n\t\tsystem_account: SYS\n\t`\n\tbConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_advertise: true\n\t\t\t%s\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tsb1Conf := createConfFile(t, []byte(fmt.Sprintf(bConf, accs, \"B1\", _EMPTY_)))\n\tsb1, sb1o := RunServerWithConfig(sb1Conf)\n\tdefer sb1.Shutdown()\n\n\tsb2Conf := createConfFile(t, []byte(fmt.Sprintf(bConf, accs, \"B2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", sb1o.Cluster.Port))))\n\tsb2, sb2o := RunServerWithConfig(sb2Conf)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tcConf := `\n\t\t%s\n\t\tserver_name: C\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: \"C\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tgateway {\n\t\t\tname: \"C\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: B\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tscConf := createConfFile(t, []byte(fmt.Sprintf(cConf, accs, sb1o.Gateway.Port)))\n\tsc, sco := RunServerWithConfig(scConf)\n\tdefer sc.Shutdown()\n\n\twaitForOutboundGateways(t, sc, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sc, 2, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 1, 2*time.Second)\n\n\tdConf := `\n\t\t%s\n\t\tserver_name: D\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: \"D\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://user:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: USER\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tsdConf := createConfFile(t, []byte(fmt.Sprintf(dConf, accs, sco.LeafNode.Port)))\n\tsd, _ := RunServerWithConfig(sdConf)\n\tdefer sd.Shutdown()\n\n\tcheckLeafNodeConnected(t, sc)\n\tcheckLeafNodeConnected(t, sd)\n\n\taConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: A\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_advertise: true\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\treconnect: \"10ms\"\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://user:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: USER\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\ta1Conf := createConfFile(t, []byte(fmt.Sprintf(aConf, accs, \"A1\", _EMPTY_, sb1o.LeafNode.Port)))\n\tsa1, sa1o := RunServerWithConfig(a1Conf)\n\tdefer sa1.Shutdown()\n\n\tcheckLeafNodeConnected(t, sa1)\n\tcheckLeafNodeConnected(t, sb1)\n\n\ta2Conf := createConfFile(t, []byte(fmt.Sprintf(aConf, accs, \"A2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", sa1o.Cluster.Port), sb1o.LeafNode.Port)))\n\tsa2, _ := RunServerWithConfig(a2Conf)\n\tdefer sa2.Shutdown()\n\n\tcheckClusterFormed(t, sa1, sa2)\n\tcheckLeafNodeConnected(t, sa2)\n\tcheckLeafNodeConnectedCount(t, sb1, 2)\n\n\totherLeafsConf := `\n\t\t%s\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://user:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: USER\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\to1Conf := createConfFile(t, []byte(fmt.Sprintf(otherLeafsConf, accs, \"OTHERLEAF1\", sb2o.LeafNode.Port)))\n\tso1, _ := RunServerWithConfig(o1Conf)\n\tdefer so1.Shutdown()\n\tcheckLeafNodeConnected(t, so1)\n\tcheckLeafNodeConnectedCount(t, sb2, 1)\n\n\to2Conf := createConfFile(t, []byte(fmt.Sprintf(otherLeafsConf, accs, \"OTHERLEAF2\", sb2o.LeafNode.Port)))\n\tso2, _ := RunServerWithConfig(o2Conf)\n\tdefer so2.Shutdown()\n\tcheckLeafNodeConnected(t, so2)\n\tcheckLeafNodeConnectedCount(t, sb2, 2)\n\n\t// Helper to check that the interest is propagated to all servers\n\tcheckInterest := func(t *testing.T, expected []int, expectedGW int32) {\n\t\tt.Helper()\n\t\tsubj := \"foo\"\n\t\tfor i, s := range []*Server{sa1, sa2, so1, so2, sb1, sb2, sc, sd} {\n\t\t\tif s == sc || !s.isRunning() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tacc, err := s.LookupAccount(\"USER\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tn := acc.Interest(subj)\n\t\t\t\tif n == expected[i] {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Expected interest count for server %q to be %v, got %v\", s, expected[i], n)\n\t\t\t})\n\t\t}\n\t\t// For server C, need to check in gateway's account.\n\t\tcheckForRegisteredQSubInterest(t, sc, \"B\", \"USER\", \"foo\", expected[6], time.Second)\n\n\t\t// For server B1 and B2, check that we have the proper counts in the map.\n\t\tfor _, s := range []*Server{sb1, sb2} {\n\t\t\tif !s.isRunning() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t\ts.gateway.pasi.Lock()\n\t\t\t\taccMap := s.gateway.pasi.m\n\t\t\t\tst := accMap[\"USER\"]\n\t\t\t\tvar n int32\n\t\t\t\tentry, ok := st[\"foo bar\"]\n\t\t\t\tif ok {\n\t\t\t\t\tn = entry.n\n\t\t\t\t}\n\t\t\t\ts.gateway.pasi.Unlock()\n\t\t\t\tif n == expectedGW {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Expected GW interest count for server %q to be %v, got %v\", s, expectedGW, n)\n\t\t\t})\n\t\t}\n\t}\n\n\tncA1 := natsConnect(t, sa1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncA1.Close()\n\tfor i := 0; i < 3; i++ {\n\t\tnatsQueueSubSync(t, ncA1, \"foo\", \"bar\")\n\t}\n\tnatsFlush(t, ncA1)\n\t// With 3 queue subs on A1, we should have for servers (in order checked in checkInterest)\n\t// for A1: 3 locals, for all others, 1 for the remote sub from A1.\n\t// B1 and B2 GW map will be 3 (1 for each sub)\n\tcheckInterest(t, []int{3, 1, 1, 1, 1, 1, 1, 1}, 3)\n\n\tncA2 := natsConnect(t, sa2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncA2.Close()\n\tncA2qsub1 := natsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tncA2qsub2 := natsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tnatsFlush(t, ncA2)\n\t// A1 will have 1 more for remote sub, same for A2 (2 locals + 1 remote).\n\t// B1 will have 2 interest (1 per leaf connection)\n\t// B1 and B2 GW map goes to 5.\n\tcheckInterest(t, []int{4, 3, 1, 1, 2, 1, 1, 1}, 5)\n\n\tncOther1 := natsConnect(t, so1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncOther1.Close()\n\tnatsQueueSubSync(t, ncOther1, \"foo\", \"bar\")\n\tnatsQueueSubSync(t, ncOther1, \"foo\", \"bar\")\n\tnatsFlush(t, ncOther1)\n\t// A1, A2 will have one more because of routed interest\n\t// O1 will have 3 (2 locals + 1 for remote interest)\n\t// O2 has still 1 for remote interest\n\t// B1 has 1 more because of new leaf interest and B2 because of routed interest.\n\t// B1 and B2 GW map goes to 7.\n\tcheckInterest(t, []int{5, 4, 3, 1, 3, 2, 1, 1}, 7)\n\n\tncOther2 := natsConnect(t, so2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncOther2.Close()\n\tnatsQueueSubSync(t, ncOther2, \"foo\", \"bar\")\n\tnatsFlush(t, ncOther2)\n\t// O2 1 more for local interest\n\t// B2 1 more for the new leaf interest\n\t// B1 and B2 GW map goes to 8.\n\tcheckInterest(t, []int{5, 4, 3, 2, 3, 3, 1, 1}, 8)\n\n\t// Stop the server so1.\n\tso1.Shutdown()\n\tso1.WaitForShutdown()\n\tcheckLeafNodeConnectedCount(t, sb2, 1)\n\t// Now check interest still valid, but wait a little bit to make sure that\n\t// even with the bug where we would send an RS- through the gateway, there\n\t// would be enough time for it to propagate before we check for interest.\n\ttime.Sleep(250 * time.Millisecond)\n\t// O1 is stopped, so expect 0\n\t// B2 has 1 less because leaf connection went away.\n\t// B1 and B2 GW map goes down to 6.\n\tcheckInterest(t, []int{5, 4, 0, 2, 3, 2, 1, 1}, 6)\n\n\t// Store server sa1.\n\tsa1.Shutdown()\n\tsa1.WaitForShutdown()\n\tcheckLeafNodeConnectedCount(t, sb1, 1)\n\ttime.Sleep(250 * time.Millisecond)\n\t// A1 and O1 are gone, so 0\n\t// A2 has 1 less due to loss of routed interest\n\t// B1 has 1 less because 1 leaf connection went away.\n\t// B1 and B2 GW map goes down to 3.\n\tcheckInterest(t, []int{0, 3, 0, 2, 2, 2, 1, 1}, 3)\n\n\t// Now remove the queue subs from A2\n\tncA2qsub1.Unsubscribe()\n\tnatsFlush(t, ncA2)\n\t// A2 has 1 less\n\tcheckInterest(t, []int{0, 2, 0, 2, 2, 2, 1, 1}, 2)\n\n\tncA2qsub2.Unsubscribe()\n\tnatsFlush(t, ncA2)\n\t// A2 has 1 (no more locals but still interest for O2).\n\t// O2 has 1 (no more for remote interest, only local).\n\t// B1, B2 has 1 less since no interest from any of its leaf connections.\n\tcheckInterest(t, []int{0, 1, 0, 1, 1, 1, 1, 1}, 1)\n\n\t// Removing (closing connection) of the sub on O2 will remove\n\t// interest globally.\n\tncOther2.Close()\n\tcheckInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0)\n\n\t// Resubscribe now, and again, interest should be propagated.\n\tnatsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tnatsFlush(t, ncA2)\n\tcheckInterest(t, []int{0, 1, 0, 1, 1, 1, 1, 1}, 1)\n\n\tnatsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tnatsFlush(t, ncA2)\n\tcheckInterest(t, []int{0, 2, 0, 1, 1, 1, 1, 1}, 2)\n\n\t// Close the client connection that has the 2 queue subs.\n\tncA2.Close()\n\tcheckInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0)\n\n\t// Now we will test when a route is lost on a server that has gateway enabled\n\t// that we update counts properly.\n\tncB2 := natsConnect(t, sb2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncB2.Close()\n\tnatsQueueSubSync(t, ncB2, \"foo\", \"bar\")\n\tnatsQueueSubSync(t, ncB2, \"foo\", \"bar\")\n\tnatsQueueSubSync(t, ncB2, \"foo\", \"bar\")\n\tnatsFlush(t, ncB2)\n\tcheckInterest(t, []int{0, 1, 0, 1, 1, 3, 1, 1}, 3)\n\n\tncB1 := natsConnect(t, sb1.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncB1.Close()\n\tnatsQueueSubSync(t, ncB1, \"foo\", \"bar\")\n\tnatsQueueSubSync(t, ncB1, \"foo\", \"bar\")\n\tcheckInterest(t, []int{0, 1, 0, 1, 3, 4, 1, 1}, 5)\n\n\t// Now shutdown B2\n\tsb2.Shutdown()\n\tsa1.WaitForShutdown()\n\ttime.Sleep(250 * time.Millisecond)\n\tcheckInterest(t, []int{0, 1, 0, 0, 2, 0, 1, 1}, 2)\n\n\tncB1.Close()\n\tcheckInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0)\n\n\t// Now we will create 2 qsubs to sa2 that is still running.\n\tncA2 = natsConnect(t, sa2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncA2.Close()\n\tqsub1 := natsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tqsub2 := natsQueueSubSync(t, ncA2, \"foo\", \"bar\")\n\tnatsFlush(t, ncA2)\n\t// sa1 is down, so 0, sa2 has 2 queue subs, so1 is down, so 0, so2 is up, so\n\t// 1 for remote interest, same for sb1. Server sb2 is down, so 0, then 1 for\n\t// remote interest for sc and sd. The expected tally for the gateway to \"C\"\n\t// should be 2.\n\tcheckInterest(t, []int{0, 2, 0, 1, 1, 0, 1, 1}, 2)\n\n\t// Close the leaf connection between sb1 and sa2\n\tsa2.mu.Lock()\n\tfor _, l := range sa2.leafs {\n\t\tl.mu.Lock()\n\t\tl.nc.Close()\n\t\tl.mu.Unlock()\n\t}\n\tsa2.mu.Unlock()\n\ttime.Sleep(50 * time.Millisecond)\n\tcheckLeafNodeConnected(t, sa2)\n\t// Should be the same counts\n\tcheckInterest(t, []int{0, 2, 0, 1, 1, 0, 1, 1}, 2)\n\n\t// Unsubscribe one of the queue sub.\n\tqsub2.Unsubscribe()\n\tnatsFlush(t, ncA2)\n\ttime.Sleep(50 * time.Millisecond)\n\t// One less for the local interest on sa2 and 1 less for the expected GW count.\n\tcheckInterest(t, []int{0, 1, 0, 1, 1, 0, 1, 1}, 1)\n\n\t// Verify that interest works by publishing from server \"C\".\n\tncC := natsConnect(t, sc.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncC.Close()\n\tnatsPub(t, ncC, \"foo\", []byte(\"hello\"))\n\n\t// Message should be received by qsub1.\n\tnatsNexMsg(t, qsub1, time.Second)\n\n\t// Now stop sa2 and instead start sb2.\n\tncA2.Close()\n\tsa2.Shutdown()\n\tsa2.WaitForShutdown()\n\n\t// Wait for counts to go down to 0\n\tcheckInterest(t, []int{0, 0, 0, 0, 0, 0, 0, 0}, 0)\n\n\t// Restart sb2 to form route to sb1.\n\tsb2, _ = RunServerWithConfig(sb2Conf)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\n\t// Create 2 queue subs on sb2.\n\tncB2 = natsConnect(t, sb2.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncB2.Close()\n\n\tqsub1 = natsQueueSubSync(t, ncB2, \"foo\", \"bar\")\n\tqsub2 = natsQueueSubSync(t, ncB2, \"foo\", \"bar\")\n\tnatsFlush(t, ncB2)\n\t// sa1 and sa2 are down, so 0, so1 is down, so 0, so2 is up, so 1 for remote interest,\n\t// same for sb1. Server sb2 has 2 local queue subs, so 2. Servers sc and sd have 1 for\n\t// remote interest. The expected tally for the gateway to \"C\" should be 2.\n\tcheckInterest(t, []int{0, 0, 0, 1, 1, 2, 1, 1}, 2)\n\n\t// Now close the route(s) between sb1 and sb2.\n\tsb2.mu.Lock()\n\tsb2.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tr.nc.Close()\n\t\tr.mu.Unlock()\n\t})\n\tsb2.mu.Unlock()\n\ttime.Sleep(50 * time.Millisecond)\n\tcheckClusterFormed(t, sb1, sb2)\n\t// Should be the same counts\n\tcheckInterest(t, []int{0, 0, 0, 1, 1, 2, 1, 1}, 2)\n\n\t// Unsubscribe one of the queue sub.\n\tqsub2.Unsubscribe()\n\tnatsFlush(t, ncB2)\n\ttime.Sleep(50 * time.Millisecond)\n\t// One less for the local interest on sb2 and 1 less for the expected GW count.\n\tcheckInterest(t, []int{0, 0, 0, 1, 1, 1, 1, 1}, 1)\n\n\t// Verify that interest works by publishing from server \"C\".\n\tnatsPub(t, ncC, \"foo\", []byte(\"hello\"))\n\n\t// Message should be received by qsub1.\n\tnatsNexMsg(t, qsub1, time.Second)\n}\n\nfunc TestLeafNodeQueueInterestAndWeightCorrectAfterServerRestartOrConnectionClose(t *testing.T) {\n\n\t// Note that this is not what a normal configuration should be. Users should\n\t// configure each leafnode to have the URLs of both B1 and B2 so that when\n\t// a server fails, the leaf can reconnect to the other running server. But\n\t// we force it to be this way to demonstrate what the issue was and see that\n\t// it is now fixed.\n\t//\n\t// B1 <--- route ---> B2\n\t// | |\n\t// Leaf Leaf\n\t// | |\n\t// A1 <--- route ---> A2\n\t//\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpinnedAccount string\n\t}{\n\t\t{\"without pinned account\", _EMPTY_},\n\t\t{\"with pinned account\", \"accounts: [\\\"A\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tleafBConf := `\n\t\t\t\taccounts { A { users: [{user:a, password: pwd}] } }\n\t\t\t\tserver_name: %s\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: HUB\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\tno_advertise: true\n\t\t\t\t}\n\t\t\t`\n\t\t\tb1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B1\", _EMPTY_, test.pinnedAccount)))\n\t\t\tb1, b1Opts := RunServerWithConfig(b1Conf)\n\t\t\tdefer b1.Shutdown()\n\n\t\t\tb2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B2\",\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", b1Opts.Cluster.Port), test.pinnedAccount)))\n\t\t\tb2, b2Opts := RunServerWithConfig(b2Conf)\n\t\t\tdefer b2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, b1, b2)\n\n\t\t\tleafAConf := `\n\t\t\t\taccounts { A { users: [{user:a, password: pwd}] } }\n\t\t\t\tserver_name: %s\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: LEAF\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\taccount: A\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t\tno_advertise: true\n\t\t\t\t}\n\t\t\t`\n\t\t\ta1Conf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, \"A1\", _EMPTY_, test.pinnedAccount, b1Opts.LeafNode.Port)))\n\t\t\ta1, a1Opts := RunServerWithConfig(a1Conf)\n\t\t\tdefer a1.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, b1)\n\t\t\tcheckLeafNodeConnected(t, a1)\n\n\t\t\ta2Conf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, \"A2\",\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", a1Opts.Cluster.Port), test.pinnedAccount, b2Opts.LeafNode.Port)))\n\t\t\ta2, _ := RunServerWithConfig(a2Conf)\n\t\t\tdefer a2.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, b2)\n\t\t\tcheckLeafNodeConnected(t, a2)\n\t\t\tcheckClusterFormed(t, a1, a2)\n\n\t\t\t// Create a client on A2 and 3 queue subs.\n\t\t\tncA2 := natsConnect(t, a2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncA2.Close()\n\n\t\t\tvar qsubs []*nats.Subscription\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tqsubs = append(qsubs, natsQueueSub(t, ncA2, \"foo\", \"queue\", func(_ *nats.Msg) {}))\n\t\t\t}\n\t\t\tnatsFlush(t, ncA2)\n\n\t\t\tsubj := \"foo\"\n\t\t\tcheckInterest := func(expected bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor _, s := range []*Server{a1, a2, b1, b2} {\n\t\t\t\t\tacc, err := s.LookupAccount(\"A\")\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\t\t\t\ti := acc.Interest(subj)\n\t\t\t\t\t\tif expected && i == 0 {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"Still no interest on %q in server %q\", subj, s)\n\t\t\t\t\t\t} else if !expected && i > 0 {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"Still interest on %q in server %q\", subj, s)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckInterest(true)\n\n\t\t\t// Check that Leafz from A1 (which connects to B1) has the expected sub\n\t\t\t// interest on \"foo\".\n\t\t\tcheckLeafA1 := func(expected bool) {\n\t\t\t\tt.Helper()\n\t\t\t\t// We will wait a bit before checking Leafz since with the bug, it would\n\t\t\t\t// take a bit of time after the action to reproduce the issue for the\n\t\t\t\t// LS+ to be sent to the wrong cluster, or the interest to not be removed.\n\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\t// Now check Leafz\n\t\t\t\tleafsz, err := a1.Leafz(&LeafzOptions{Subscriptions: true})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, leafsz.NumLeafs, 1)\n\t\t\t\trequire_True(t, leafsz.Leafs[0] != nil)\n\t\t\t\tlz := leafsz.Leafs[0]\n\t\t\t\trequire_Equal(t, lz.Name, \"B1\")\n\t\t\t\trequire_Equal(t, lz.NumSubs, uint32(len(lz.Subs)))\n\t\t\t\tvar ok bool\n\t\t\t\tfor _, sub := range lz.Subs {\n\t\t\t\t\tif sub == \"foo\" {\n\t\t\t\t\t\tif expected {\n\t\t\t\t\t\t\tok = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\tt.Fatalf(\"Did not expect to have the %q subscription\", sub)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif expected && !ok {\n\t\t\t\t\tt.Fatalf(\"Expected to have the %q subscription\", \"foo\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckLeafA1(false)\n\n\t\t\t// Now restart server \"B1\". We need to create a conf file with the ports\n\t\t\t// that it used.\n\t\t\trestartBConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\taccounts { A { users: [{user:a, password: pwd}] } }\n\t\t\t\tserver_name: B1\n\t\t\t\tlisten: \"127.0.0.1:%d\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: HUB\n\t\t\t\t\tlisten: \"127.0.0.1:%d\"\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tlisten: \"127.0.0.1:%d\"\n\t\t\t\t\tno_advertise: true\n\t\t\t\t}\n\t\t\t`, b1Opts.Port, b1Opts.Cluster.Port, test.pinnedAccount, b1Opts.LeafNode.Port)))\n\t\t\tb1.Shutdown()\n\t\t\tb1, _ = RunServerWithConfig(restartBConf)\n\t\t\tdefer b1.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, b1)\n\t\t\tcheckLeafNodeConnected(t, a1)\n\n\t\t\t// Stop one of the queue sub.\n\t\t\tqsubs[0].Unsubscribe()\n\t\t\tnatsFlush(t, ncA2)\n\n\t\t\t// Check that \"foo\" does not show up in the subscription list\n\t\t\t// for the leaf from A1 to B1.\n\t\t\tcheckLeafA1(false)\n\n\t\t\t// Now stop the other 2 and check again.\n\t\t\tqsubs[1].Unsubscribe()\n\t\t\tqsubs[2].Unsubscribe()\n\t\t\tnatsFlush(t, ncA2)\n\t\t\tcheckInterest(false)\n\n\t\t\tcheckLeafA1(false)\n\n\t\t\t// Now recreate 3 queue subs.\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tnatsQueueSub(t, ncA2, \"foo\", \"queue\", func(_ *nats.Msg) {})\n\t\t\t}\n\t\t\t// Check interest is present in all servers\n\t\t\tcheckInterest(true)\n\t\t\t// But A1's leaf to B1 should still not have a sub interest for \"foo\".\n\t\t\tcheckLeafA1(false)\n\n\t\t\t// Now stop the client connection instead of removing queue sub\n\t\t\t// one at a time. This will ensure that we properly handle an LS-\n\t\t\t// on B2 with an interest with a queue weight more than 1 still\n\t\t\t// present at the time of processing.\n\t\t\tncA2.Close()\n\t\t\tcheckInterest(false)\n\n\t\t\tcheckLeafA1(false)\n\n\t\t\t// We will now test that if the queue subs are created on B2,\n\t\t\t// we have proper interest on A1, but when we close the connection,\n\t\t\t// the interest disappears.\n\t\t\tncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncB2.Close()\n\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tnatsQueueSub(t, ncB2, \"foo\", \"queue\", func(_ *nats.Msg) {})\n\t\t\t}\n\t\t\tcheckInterest(true)\n\t\t\tcheckLeafA1(true)\n\t\t\t// Close the connection, so all queue subs should be removed at once.\n\t\t\tncB2.Close()\n\t\t\tcheckInterest(false)\n\t\t\tcheckLeafA1(false)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeQueueWeightCorrectOnRestart(t *testing.T) {\n\tleafBConf := `\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: HUB\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_advertise: true\n\t\t}\n\t`\n\tb1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B1\", _EMPTY_)))\n\tb1, b1Opts := RunServerWithConfig(b1Conf)\n\tdefer b1.Shutdown()\n\n\tb2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", b1Opts.Cluster.Port))))\n\tb2, b2Opts := RunServerWithConfig(b2Conf)\n\tdefer b2.Shutdown()\n\n\tcheckClusterFormed(t, b1, b2)\n\n\tleafAConf := `\n\t\tserver_name: LEAF\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes: [{url: \"nats://127.0.0.1:%d\"}]\n\t\t\treconnect: \"50ms\"\n\t\t}\n\t`\n\taConf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, b2Opts.LeafNode.Port)))\n\ta, _ := RunServerWithConfig(aConf)\n\tdefer a.Shutdown()\n\n\tcheckLeafNodeConnected(t, b2)\n\tcheckLeafNodeConnected(t, a)\n\n\tnc := natsConnect(t, a.ClientURL())\n\tdefer nc.Close()\n\n\tfor i := 0; i < 2; i++ {\n\t\tnatsQueueSubSync(t, nc, \"foo\", \"queue\")\n\t}\n\tnatsFlush(t, nc)\n\n\tcheckQueueWeight := func() {\n\t\tfor _, s := range []*Server{b1, b2} {\n\t\t\tgacc := s.GlobalAccount()\n\t\t\tgacc.mu.RLock()\n\t\t\tsl := gacc.sl\n\t\t\tgacc.mu.RUnlock()\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t// For remote queue interest, Match() will expand to queue weight.\n\t\t\t\t// So we should have 1 group and 2 queue subs present.\n\t\t\t\tres := sl.Match(\"foo\")\n\t\t\t\tfor _, qsubs := range res.qsubs {\n\t\t\t\t\tfor _, sub := range qsubs {\n\t\t\t\t\t\tif string(sub.subject) == \"foo\" && string(sub.queue) == \"queue\" && atomic.LoadInt32(&sub.qw) == 2 {\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Server %q does not have expected queue interest with expected weight\", s)\n\t\t\t})\n\t\t}\n\t}\n\tcheckQueueWeight()\n\n\t// Now restart server \"B2\". We need to create a conf file with the ports\n\t// that it used.\n\trestartBConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: B2\n\t\tlisten: \"127.0.0.1:%d\"\n\t\tcluster {\n\t\t\tname: HUB\n\t\t\tlisten: \"127.0.0.1:%d\"\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:%d\"\n\t\t\tno_advertise: true\n\t\t}\n\t`, b2Opts.Port, b2Opts.Cluster.Port, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", b1Opts.Cluster.Port), b2Opts.LeafNode.Port)))\n\tb2.Shutdown()\n\tb2, _ = RunServerWithConfig(restartBConf)\n\tdefer b2.Shutdown()\n\n\tcheckLeafNodeConnected(t, b2)\n\tcheckLeafNodeConnected(t, a)\n\tcheckQueueWeight()\n}\n\nfunc TestLeafNodeRoutedSubKeyDifferentBetweenLeafSubAndRoutedSub(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpinnedAccount string\n\t\tlnocu bool\n\t}{\n\t\t{\"without pinned account\", _EMPTY_, true},\n\t\t{\"with pinned account\", \"accounts: [\\\"XYZ\\\"]\", true},\n\t\t{\"old server without pinned account\", _EMPTY_, false},\n\t\t{\"old server with pinned account\", \"accounts: [\\\"XYZ\\\"]\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tleafBConf := `\n\t\t\t\taccounts: {XYZ {users:[{user:a, password:pwd}]}}\n\t\t\t\tserver_name: %s\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: HUB\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\tno_advertise: true\n\t\t\t\t}\n\t\t\t`\n\t\t\tb1Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B1\", _EMPTY_, test.pinnedAccount)))\n\t\t\tb1, b1Opts := RunServerWithConfig(b1Conf)\n\t\t\tdefer b1.Shutdown()\n\n\t\t\tb2Conf := createConfFile(t, []byte(fmt.Sprintf(leafBConf, \"B2\",\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", b1Opts.Cluster.Port), test.pinnedAccount)))\n\t\t\tb2, b2Opts := RunServerWithConfig(b2Conf)\n\t\t\tdefer b2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, b1, b2)\n\n\t\t\t// To make route connections behave like if the server was connected\n\t\t\t// to an older server, change the routes' lnocu field.\n\t\t\tif !test.lnocu {\n\t\t\t\tfor _, s := range []*Server{b1, b2} {\n\t\t\t\t\ts.mu.RLock()\n\t\t\t\t\ts.forEachRoute(func(r *client) {\n\t\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\t\tr.route.lnocu = false\n\t\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\t})\n\t\t\t\t\ts.mu.RUnlock()\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// This leaf will have a cluster name that matches an account name.\n\t\t\t// The idea is to make sure that hub servers are not using incorrect\n\t\t\t// keys to differentiate a routed queue interest on subject \"A\" with\n\t\t\t// queue name \"foo\" for account \"A\" in their cluster: \"RS+ A A foo\"\n\t\t\t// with a leafnode plain subscription, which since there is an origin\n\t\t\t// would be: \"LS+ A A foo\", that is, origin is \"A\", account is \"A\"\n\t\t\t// and subject is \"foo\".\n\t\t\tleafAConf := `\n\t\t\t\taccounts: {XYZ {users:[{user:a, password:pwd}]}}\n\t\t\t\tserver_name: LEAF\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: XYZ\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\taccount: \"XYZ\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`\n\t\t\taConf := createConfFile(t, []byte(fmt.Sprintf(leafAConf, b2Opts.LeafNode.Port)))\n\t\t\ta, _ := RunServerWithConfig(aConf)\n\t\t\tdefer a.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, b2)\n\t\t\tcheckLeafNodeConnected(t, a)\n\n\t\t\tncB2 := natsConnect(t, b2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncB2.Close()\n\t\t\t// Create a plain sub on \"foo\"\n\t\t\tnatsSubSync(t, ncB2, \"foo\")\n\t\t\t// And a queue sub on \"XYZ\" with queue name \"foo\"\n\t\t\tnatsQueueSubSync(t, ncB2, \"XYZ\", \"foo\")\n\t\t\tnatsFlush(t, ncB2)\n\n\t\t\tncA := natsConnect(t, a.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\t\t\t// From the leafnode, create a plain sub on \"foo\"\n\t\t\tnatsSubSync(t, ncA, \"foo\")\n\t\t\t// And a queue sub on \"XYZ\" with queue name \"foo\"\n\t\t\tnatsQueueSubSync(t, ncA, \"XYZ\", \"foo\")\n\t\t\tnatsFlush(t, ncA)\n\n\t\t\t// Check the acc.rm on B2\n\t\t\tb2Acc, err := b2.LookupAccount(\"XYZ\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\trsubKey := keyFromSubWithOrigin(&subscription{subject: []byte(\"foo\")})\n\t\t\trqsubKey := keyFromSubWithOrigin(&subscription{subject: []byte(\"XYZ\"), queue: []byte(\"foo\")})\n\t\t\trlsubKey := keyFromSubWithOrigin(&subscription{origin: []byte(\"XYZ\"), subject: []byte(\"foo\")})\n\t\t\trlqsubKey := keyFromSubWithOrigin(&subscription{origin: []byte(\"XYZ\"), subject: []byte(\"XYZ\"), queue: []byte(\"foo\")})\n\t\t\t// Ensure all keys are different\n\t\t\trequire_True(t, rsubKey != rqsubKey && rqsubKey != rlsubKey && rlsubKey != rlqsubKey)\n\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tb2Acc.mu.RLock()\n\t\t\t\tdefer b2Acc.mu.RUnlock()\n\t\t\t\tfor _, key := range []string{rsubKey, rqsubKey, rlsubKey, rlqsubKey} {\n\t\t\t\t\tv, ok := b2Acc.rm[key]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn fmt.Errorf(\"Did not find key %q for sub: %+v\", key, sub)\n\t\t\t\t\t}\n\t\t\t\t\tif v != 1 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Key %q v=%v for sub: %+v\", key, v, sub)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Now check that on B1, we have 2 distinct subs for the route.\n\t\t\tb1Acc, err := b1.LookupAccount(\"XYZ\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar route *client\n\n\t\t\tif test.pinnedAccount == _EMPTY_ {\n\t\t\t\tb1Acc.mu.RLock()\n\t\t\t\trIdx := b1Acc.routePoolIdx\n\t\t\t\tb1Acc.mu.RUnlock()\n\t\t\t\tb1.mu.RLock()\n\t\t\t\tb1.forEachRouteIdx(rIdx, func(r *client) bool {\n\t\t\t\t\troute = r\n\t\t\t\t\treturn false\n\t\t\t\t})\n\t\t\t\tb1.mu.RUnlock()\n\t\t\t} else {\n\t\t\t\tb1.mu.RLock()\n\t\t\t\tremotes := b1.accRoutes[\"XYZ\"]\n\t\t\t\tfor _, r := range remotes {\n\t\t\t\t\troute = r\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tb1.mu.RUnlock()\n\t\t\t}\n\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\t// Check that route.subs has 4 entries for the subs we\n\t\t\t\t// created in this test.\n\t\t\t\tvar entries []string\n\t\t\t\troute.mu.Lock()\n\t\t\t\tfor key := range route.subs {\n\t\t\t\t\tif strings.Contains(key, \"foo\") {\n\t\t\t\t\t\tentries = append(entries, key)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\troute.mu.Unlock()\n\t\t\t\t// With new servers, we expect 4 entries, but with older servers,\n\t\t\t\t// we have collisions and have only 2.\n\t\t\t\tvar expected int\n\t\t\t\tif test.lnocu {\n\t\t\t\t\texpected = 4\n\t\t\t\t} else {\n\t\t\t\t\texpected = 2\n\t\t\t\t}\n\t\t\t\tif len(entries) != expected {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d entries with %q, got this: %q\", expected, \"foo\", entries)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\t// Close the connections and expect all gone.\n\t\t\tncB2.Close()\n\t\t\tncA.Close()\n\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tb2Acc.mu.RLock()\n\t\t\t\tdefer b2Acc.mu.RUnlock()\n\t\t\t\tfor _, key := range []string{rsubKey, rqsubKey, rlsubKey, rlqsubKey} {\n\t\t\t\t\tif _, ok := b2Acc.rm[key]; ok {\n\t\t\t\t\t\treturn fmt.Errorf(\"Key %q still present\", key)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\t\tvar entries []string\n\t\t\t\troute.mu.Lock()\n\t\t\t\tfor key := range route.subs {\n\t\t\t\t\tif strings.Contains(key, \"foo\") {\n\t\t\t\t\t\tentries = append(entries, key)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\troute.mu.Unlock()\n\t\t\t\tif len(entries) != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Still routed subscriptions on %q: %q\", \"foo\", entries)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeQueueGroupWithLateLNJoin(t *testing.T) {\n\t/*\n\n\t\tTopology: A cluster of leafnodes LN2 and LN3, connect\n\t\tto a cluster C1, C2.\n\n\t\tsub(foo) sub(foo)\n\t\t \\ /\n\t\t C1 <-> C2\n\t\t ^ ^\n\t\t | |\n\t\t LN2 <-> LN3\n\t\t / \\\n\t\tsub(foo) sub(foo)\n\n\t\tOnce the above is set, start LN1 that connects to C1.\n\n\t\t sub(foo) sub(foo)\n\t\t \\ /\n\t\tLN1 -> C1 <-> C2\n\t\t ^ ^\n\t\t | |\n\t\t LN2 <-> LN3\n\t\t / \\\n\t\t sub(foo) sub(foo)\n\n\t\tRemove subs to LN3, C2 and C1.\n\n\t\tLN1 -> C1 <-> C2\n\t\t ^ ^\n\t\t | |\n\t\t LN2 <-> LN3\n\t\t /\n\t\t sub(foo)\n\n\t\tPublish from LN1 and verify message is received by sub on LN2.\n\n\t\tpub(foo)\n\t\t\\\n\t\tLN1 -> C1 <-> C2\n\t\t ^ ^\n\t\t | |\n\t\t LN2 <-> LN3\n\t\t /\n\t\t sub(foo)\n\t*/\n\tco1 := DefaultOptions()\n\tco1.LeafNode.Host = \"127.0.0.1\"\n\tco1.LeafNode.Port = -1\n\tco1.Cluster.Name = \"ngs\"\n\tco1.Cluster.Host = \"127.0.0.1\"\n\tco1.Cluster.Port = -1\n\tc1 := RunServer(co1)\n\tdefer c1.Shutdown()\n\n\tco2 := DefaultOptions()\n\tco2.LeafNode.Host = \"127.0.0.1\"\n\tco2.LeafNode.Port = -1\n\tco2.Cluster.Name = \"ngs\"\n\tco2.Cluster.Host = \"127.0.0.1\"\n\tco2.Cluster.Port = -1\n\tco2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", co1.Cluster.Port))\n\tc2 := RunServer(co2)\n\tdefer c2.Shutdown()\n\n\tcheckClusterFormed(t, c1, c2)\n\n\tlo2 := DefaultOptions()\n\tlo2.Cluster.Name = \"local\"\n\tlo2.Cluster.Host = \"127.0.0.1\"\n\tlo2.Cluster.Port = -1\n\tlo2.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tlo2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", co1.LeafNode.Port)}}}}\n\tln2 := RunServer(lo2)\n\tdefer ln2.Shutdown()\n\n\tlo3 := DefaultOptions()\n\tlo3.Cluster.Name = \"local\"\n\tlo3.Cluster.Host = \"127.0.0.1\"\n\tlo3.Cluster.Port = -1\n\tlo3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", lo2.Cluster.Port))\n\tlo3.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tlo3.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", co2.LeafNode.Port)}}}}\n\tln3 := RunServer(lo3)\n\tdefer ln3.Shutdown()\n\n\tcheckClusterFormed(t, ln2, ln3)\n\tcheckLeafNodeConnected(t, ln2)\n\tcheckLeafNodeConnected(t, ln3)\n\n\tcln2 := natsConnect(t, ln2.ClientURL())\n\tdefer cln2.Close()\n\tsln2 := natsQueueSubSync(t, cln2, \"foo\", \"qgroup\")\n\tnatsFlush(t, cln2)\n\n\tcln3 := natsConnect(t, ln3.ClientURL())\n\tdefer cln3.Close()\n\tsln3 := natsQueueSubSync(t, cln3, \"foo\", \"qgroup\")\n\tnatsFlush(t, cln3)\n\n\tcc1 := natsConnect(t, c1.ClientURL())\n\tdefer cc1.Close()\n\tsc1 := natsQueueSubSync(t, cc1, \"foo\", \"qgroup\")\n\tnatsFlush(t, cc1)\n\n\tcc2 := natsConnect(t, c2.ClientURL())\n\tdefer cc2.Close()\n\tsc2 := natsQueueSubSync(t, cc2, \"foo\", \"qgroup\")\n\tnatsFlush(t, cc2)\n\n\tcheckSubInterest(t, c1, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, c2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, ln2, globalAccountName, \"foo\", time.Second)\n\tcheckSubInterest(t, ln3, globalAccountName, \"foo\", time.Second)\n\n\tlo1 := DefaultOptions()\n\tlo1.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tlo1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{{Scheme: \"nats\", Host: fmt.Sprintf(\"127.0.0.1:%d\", co1.LeafNode.Port)}}}}\n\tln1 := RunServer(lo1)\n\tdefer ln1.Shutdown()\n\n\tcheckLeafNodeConnected(t, ln1)\n\tcheckSubInterest(t, ln1, globalAccountName, \"foo\", time.Second)\n\n\tsln3.Unsubscribe()\n\tnatsFlush(t, cln3)\n\tsc2.Unsubscribe()\n\tnatsFlush(t, cc2)\n\tsc1.Unsubscribe()\n\tnatsFlush(t, cc1)\n\n\tcln1 := natsConnect(t, ln1.ClientURL())\n\tdefer cln1.Close()\n\n\tnatsPub(t, cln1, \"foo\", []byte(\"hello\"))\n\tnatsNexMsg(t, sln2, time.Second)\n}\n\nfunc TestLeafNodeJetStreamDomainMapCrossTalk(t *testing.T) {\n\taccs := `\naccounts :{\n A:{ jetstream: enable, users:[ {user:a1,password:a1}]},\n SYS:{ users:[ {user:s1,password:s1}]},\n}\nsystem_account: SYS\n`\n\n\tsd1 := t.TempDir()\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\nlisten: 127.0.0.1:-1\n%s\njetstream: { domain: da, store_dir: '%s', max_mem: 50Mb, max_file: 50Mb }\nleafnodes: {\n\tlisten: 127.0.0.1:-1\n\tno_advertise: true\n\tauthorization: {\n\t\ttimeout: 0.5\n\t}\n}\n`, accs, sd1)))\n\tsA, _ := RunServerWithConfig(confA)\n\tdefer sA.Shutdown()\n\n\tsd2 := t.TempDir()\n\tconfL := createConfFile(t, []byte(fmt.Sprintf(`\nlisten: 127.0.0.1:-1\n%s\njetstream: { domain: dl, store_dir: '%s', max_mem: 50Mb, max_file: 50Mb }\nleafnodes:{\n\tno_advertise: true\n remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},\n\t\t {url:nats://s1:s1@127.0.0.1:%d, account: SYS}]\n}\n`, accs, sd2, sA.opts.LeafNode.Port, sA.opts.LeafNode.Port)))\n\tsL, _ := RunServerWithConfig(confL)\n\tdefer sL.Shutdown()\n\n\tncA := natsConnect(t, sA.ClientURL(), nats.UserInfo(\"a1\", \"a1\"))\n\tdefer ncA.Close()\n\tncL := natsConnect(t, sL.ClientURL(), nats.UserInfo(\"a1\", \"a1\"))\n\tdefer ncL.Close()\n\n\ttest := func(jsA, jsL nats.JetStreamContext) {\n\t\tkvA, err := jsA.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"bucket\"})\n\t\trequire_NoError(t, err)\n\t\tkvL, err := jsL.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"bucket\"})\n\t\trequire_NoError(t, err)\n\n\t\t_, err = kvA.Put(\"A\", nil)\n\t\trequire_NoError(t, err)\n\t\t_, err = kvL.Put(\"L\", nil)\n\t\trequire_NoError(t, err)\n\n\t\t// check for unwanted cross talk\n\t\t_, err = kvA.Get(\"A\")\n\t\trequire_NoError(t, err)\n\t\t_, err = kvA.Get(\"l\")\n\t\trequire_Error(t, err)\n\t\trequire_True(t, err == nats.ErrKeyNotFound)\n\n\t\t_, err = kvL.Get(\"A\")\n\t\trequire_Error(t, err)\n\t\trequire_True(t, err == nats.ErrKeyNotFound)\n\t\t_, err = kvL.Get(\"L\")\n\t\trequire_NoError(t, err)\n\n\t\terr = jsA.DeleteKeyValue(\"bucket\")\n\t\trequire_NoError(t, err)\n\t\terr = jsL.DeleteKeyValue(\"bucket\")\n\t\trequire_NoError(t, err)\n\t}\n\n\tjsA, err := ncA.JetStream()\n\trequire_NoError(t, err)\n\tjsL, err := ncL.JetStream()\n\trequire_NoError(t, err)\n\ttest(jsA, jsL)\n\n\tjsAL, err := ncA.JetStream(nats.Domain(\"dl\"))\n\trequire_NoError(t, err)\n\tjsLA, err := ncL.JetStream(nats.Domain(\"da\"))\n\trequire_NoError(t, err)\n\ttest(jsAL, jsLA)\n\n\tjsAA, err := ncA.JetStream(nats.Domain(\"da\"))\n\trequire_NoError(t, err)\n\tjsLL, err := ncL.JetStream(nats.Domain(\"dl\"))\n\trequire_NoError(t, err)\n\ttest(jsAA, jsLL)\n}\n\ntype checkLeafMinVersionLogger struct {\n\tDummyLogger\n\terrCh chan time.Time\n\tconnCh chan time.Time\n}\n\nfunc (l *checkLeafMinVersionLogger) Errorf(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"minimum version\") {\n\t\tselect {\n\t\tcase l.errCh <- time.Now():\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc (l *checkLeafMinVersionLogger) Noticef(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"Leafnode connection created\") {\n\t\tselect {\n\t\tcase l.connCh <- time.Now():\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeMinVersion(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tmin_version: 2.8.0\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\trconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{url: \"nats://127.0.0.1:%d\" }\n\t\t\t]\n\t\t}\n\t`, o.LeafNode.Port)))\n\tln, _ := RunServerWithConfig(rconf)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\tcheckLeafNodeConnected(t, ln)\n\n\tln.Shutdown()\n\ts.Shutdown()\n\n\t// Now makes sure we validate options, not just config file.\n\tfor _, test := range []struct {\n\t\tname string\n\t\tversion string\n\t\terr string\n\t}{\n\t\t{\"invalid version\", \"abc\", \"semver\"},\n\t\t{\"version too low\", \"2.7.9\", \"the minimum version should be at least 2.8.0\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to.Port = -1\n\t\t\to.LeafNode.Port = -1\n\t\t\to.LeafNode.MinVersion = test.version\n\t\t\tif s, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tif s != nil {\n\t\t\t\t\ts.Shutdown()\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"Expected error to contain %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n\n\t// Ok, so now to verify that a server rejects a leafnode connection\n\t// we will set the min_version above our current VERSION. So first\n\t// decompose the version:\n\tmajor, minor, _, err := versionComponents(VERSION)\n\tif err != nil {\n\t\tt.Fatalf(\"The current server version %q is not valid: %v\", VERSION, err)\n\t}\n\t// Let's make our minimum server an minor version above\n\tmv := fmt.Sprintf(\"%d.%d.0\", major, minor+1)\n\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tmin_version: \"%s\"\n\t\t}\n\t`, mv)))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tl := &checkLeafMinVersionLogger{errCh: make(chan time.Time, 1), connCh: make(chan time.Time, 1)}\n\ts.SetLogger(l, false, false)\n\n\trconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{url: \"nats://127.0.0.1:%d\" }\n\t\t\t]\n\t\t}\n\t`, o.LeafNode.Port)))\n\tlo := LoadConfig(rconf)\n\tlo.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tln = RunServer(lo)\n\tdefer ln.Shutdown()\n\n\tselect {\n\tcase <-l.connCh:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Remote did not try to connect\")\n\t}\n\n\tvar rejectAt time.Time\n\tselect {\n\tcase rejectAt = <-l.errCh:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get the minimum version required error\")\n\t}\n\n\t// Since we have a very small reconnect interval, the next attempt should be\n\t// delayed by the dedicated minimum version reconnect delay on the soliciting\n\t// side, not by the normal reconnect interval.\n\tselect {\n\tcase secondAttemptAt := <-l.connCh:\n\t\tif elapsed := secondAttemptAt.Sub(rejectAt); elapsed < leafNodeMinVersionReconnectDelay {\n\t\t\tt.Fatalf(\"Expected reconnect attempt after at least %v, got %v\", leafNodeMinVersionReconnectDelay, elapsed)\n\t\t}\n\tcase <-time.After(7 * time.Second):\n\t\tt.Fatal(\"Did not get the reconnect attempt\")\n\t}\n}\n\nfunc TestLeafNodeStreamAndShadowSubs(t *testing.T) {\n\thubConf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization: {\n\t\t\t user: leaf\n\t\t\t password: leaf\n\t\t\t account: B\n\t\t\t}\n\t\t}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t users = [{user: usrA, password: usrA}]\n\t\t\t exports: [{stream: foo.*.>}]\n\t\t\t}\n\t\t\tB: {\n\t\t\t imports: [{stream: {account: A, subject: foo.*.>}}]\n\t\t\t}\n\t\t}\n\t`))\n\thub, hubo := RunServerWithConfig(hubConf)\n\tdefer hub.Shutdown()\n\n\tleafConfContet := fmt.Sprintf(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats-leaf://leaf:leaf@127.0.0.1:%d\"\n\t\t\t\t\taccount: B\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\taccounts: {\n\t\t\tB: {\n\t\t\t\texports: [{stream: foo.*.>}]\n\t\t\t}\n\t\t\tC: {\n\t\t\t\tusers: [{user: usrC, password: usrC}]\n\t\t\t\timports: [{stream: {account: B, subject: foo.bar.>}}]\n\t\t\t}\n\t\t}\n\t`, hubo.LeafNode.Port)\n\tleafConf := createConfFile(t, []byte(leafConfContet))\n\tleafo := LoadConfig(leafConf)\n\tleafo.LeafNode.ReconnectInterval = 50 * time.Millisecond\n\tleaf := RunServer(leafo)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\tsubPubAndCheck := func() {\n\t\tt.Helper()\n\n\t\tncl, err := nats.Connect(leaf.ClientURL(), nats.UserInfo(\"usrC\", \"usrC\"))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t\t}\n\t\tdefer ncl.Close()\n\n\t\t// This will send an LS+ to the \"hub\" server.\n\t\tsub, err := ncl.SubscribeSync(\"foo.*.baz\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t\t}\n\t\tncl.Flush()\n\n\t\tncm, err := nats.Connect(hub.ClientURL(), nats.UserInfo(\"usrA\", \"usrA\"))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t\t}\n\t\tdefer ncm.Close()\n\n\t\t// Try a few times in case subject interest has not propagated yet\n\t\tfor i := 0; i < 5; i++ {\n\t\t\tncm.Publish(\"foo.bar.baz\", []byte(\"msg\"))\n\t\t\tif _, err := sub.NextMsg(time.Second); err == nil {\n\t\t\t\t// OK, done!\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tt.Fatal(\"Message was not received\")\n\t}\n\tsubPubAndCheck()\n\n\t// Now cause a restart of the accepting side so that the leaf connection\n\t// is recreated.\n\thub.Shutdown()\n\thub = RunServer(hubo)\n\tdefer hub.Shutdown()\n\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\tsubPubAndCheck()\n\n\t// Issue a config reload even though we make no modification. There was\n\t// a defect that caused the interest propagation to break.\n\t// Set the ReconnectInterval to the default value so that reload does not complain.\n\tleaf.getOpts().LeafNode.ReconnectInterval = DEFAULT_LEAF_NODE_RECONNECT\n\treloadUpdateConfig(t, leaf, leafConf, leafConfContet)\n\n\t// Check again\n\tsubPubAndCheck()\n}\n\nfunc TestLeafNodeAuthConfigReload(t *testing.T) {\n\ttemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts { test: {} }\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:7422\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t}\n\t\t\tauthorization {\n\t\t\t\t# These are only fields allowed atm.\n\t\t\t\tusers = [ { user: test, password: \"s3cret1\", account: \"test\" } ]\n\t\t\t}\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(template))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tlg := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(lg, false, false)\n\n\t// Reload here should work ok.\n\treloadUpdateConfig(t, s, conf, template)\n}\n\nfunc TestLeafNodeSignatureCB(t *testing.T) {\n\tcontent := `\n\t\tport: -1\n\t\tserver_name: OP\n\t\toperator = \"../test/configs/nkeys/op.jwt\"\n\t\tresolver = MEMORY\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t_, akp := createAccount(s)\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\tlopts := &DefaultTestOptions\n\tu, err := url.Parse(fmt.Sprintf(\"nats://%s:%d\", opts.LeafNode.Host, opts.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\tremote := &RemoteLeafOpts{URLs: []*url.URL{u}}\n\tremote.SignatureCB = func(nonce []byte) (string, []byte, error) {\n\t\treturn \"\", nil, fmt.Errorf(\"on purpose\")\n\t}\n\tlopts.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\tlopts.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\tsl := RunServer(lopts)\n\tdefer sl.Shutdown()\n\n\tslog := &captureErrorLogger{errCh: make(chan string, 10)}\n\tsl.SetLogger(slog, false, false)\n\n\t// Now check that the leafnode got the error that the callback returned.\n\tselect {\n\tcase err := <-slog.errCh:\n\t\tif !strings.Contains(err, \"on purpose\") {\n\t\t\tt.Fatalf(\"Expected error from cb, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get expected error\")\n\t}\n\n\tsl.Shutdown()\n\t// Now check what happens if the connection is closed while in the callback.\n\tblockCh := make(chan struct{})\n\tremote.SignatureCB = func(nonce []byte) (string, []byte, error) {\n\t\t<-blockCh\n\t\tsig, err := kp.Sign(nonce)\n\t\treturn ujwt, sig, err\n\t}\n\tsl = RunServer(lopts)\n\tdefer sl.Shutdown()\n\n\t// Recreate the logger so that we are sure not to have possible previous errors\n\tslog = &captureErrorLogger{errCh: make(chan string, 10)}\n\tsl.SetLogger(slog, false, false)\n\n\t// Get the leaf connection from the temp clients map and close it.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tvar c *client\n\t\tsl.grMu.Lock()\n\t\tfor _, cli := range sl.grTmpClients {\n\t\t\tc = cli\n\t\t}\n\t\tsl.grMu.Unlock()\n\t\tif c == nil {\n\t\t\treturn fmt.Errorf(\"Client still not found in temp map\")\n\t\t}\n\t\tc.closeConnection(ClientClosed)\n\t\treturn nil\n\t})\n\n\t// Release the callback, and check we get the appropriate error.\n\tclose(blockCh)\n\tselect {\n\tcase err := <-slog.errCh:\n\t\tif !strings.Contains(err, ErrConnectionClosed.Error()) {\n\t\t\tt.Fatalf(\"Expected error that connection was closed, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not get expected error\")\n\t}\n\n\tsl.Shutdown()\n\t// Change to a good CB and now it should work\n\tremote.SignatureCB = func(nonce []byte) (string, []byte, error) {\n\t\tsig, err := kp.Sign(nonce)\n\t\treturn ujwt, sig, err\n\t}\n\tsl = RunServer(lopts)\n\tdefer sl.Shutdown()\n\tcheckLeafNodeConnected(t, sl)\n}\n\ntype testLeafTraceLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *testLeafTraceLogger) Tracef(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\t// We will sub to 'baz' and to 'bar', so filter on 'ba' prefix.\n\tif strings.Contains(msg, \"[LS+ ba\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\n// Make sure permissioned denied subs do not make it to the leafnode even if existing.\nfunc TestLeafNodePermsSuppressSubs(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tauthorization {\n\t\t PERMS = {\n\t\t publish = \"foo\"\n\t\t subscribe = [\"_INBOX.>\"]\n\t\t }\n\t\t users = [\n\t\t {user: \"user\", password: \"pass\"}\n\t\t {user: \"ln\", password: \"pass\" , permissions: $PERMS }\n\t\t ]\n\t\t}\n\t\tno_auth_user: user\n\n\t\tleafnodes {\n\t\t listen: 127.0.0.1:7422\n\t\t}\n\t`))\n\n\tlconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t remotes = [ { url: \"nats://ln:pass@127.0.0.1\" } ]\n\t\t}\n\t\ttrace = true\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Connect client to the hub.\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// This should not be seen on leafnode side since we only allow pub to \"foo\"\n\t_, err = nc.SubscribeSync(\"baz\")\n\trequire_NoError(t, err)\n\n\tln, _ := RunServerWithConfig(lconf)\n\tdefer ln.Shutdown()\n\n\t// Setup logger to capture trace events.\n\tl := &testLeafTraceLogger{ch: make(chan string, 10)}\n\tln.SetLogger(l, true, true)\n\n\tcheckLeafNodeConnected(t, ln)\n\n\t// Need to have ot reconnect to trigger since logger attaches too late.\n\tln.mu.Lock()\n\tfor _, c := range ln.leafs {\n\t\tc.mu.Lock()\n\t\tc.nc.Close()\n\t\tc.mu.Unlock()\n\t}\n\tln.mu.Unlock()\n\tcheckLeafNodeConnectedCount(t, ln, 0)\n\tcheckLeafNodeConnectedCount(t, ln, 1)\n\n\tselect {\n\tcase msg := <-l.ch:\n\t\tt.Fatalf(\"Unexpected LS+ seen on leafnode: %s\", msg)\n\tcase <-time.After(50 * time.Millisecond):\n\t\t// OK\n\t}\n\n\t// Now double check that new subs also do not propagate.\n\t// This behavior was working already.\n\t_, err = nc.SubscribeSync(\"bar\")\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase msg := <-l.ch:\n\t\tt.Fatalf(\"Unexpected LS+ seen on leafnode: %s\", msg)\n\tcase <-time.After(50 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\nfunc TestLeafNodeDuplicateMsg(t *testing.T) {\n\t// This involves 2 clusters with leafnodes to each other with a different\n\t// account, and those accounts import/export a subject that caused\n\t// duplicate messages. This test requires static ports since we need to\n\t// have A->B and B->A.\n\ta1Conf := createConfFile(t, []byte(`\n\tcluster : {\n\t\tname : A\n\t\tport : -1\n\t}\n\tleafnodes : {\n\t\tport : 14333\n\t\tremotes : [{\n\t\t\taccount : A\n\t\t\turls : [nats://leafa:pwd@127.0.0.1:24333]\n\t\t}]\n\t}\n\tport : -1\n\tserver_name : A_1\n\n\taccounts:{\n\t\tA:{\n\t\t\tusers:[\n\t\t\t\t{user: leafa, password: pwd},\n\t\t\t\t{user: usera, password: usera, permissions: {\n\t\t\t\t\tpublish:{ allow:[\"iot.b.topic\"] }\n\t\t\t\t\tsubscribe:{ allow:[\"iot.a.topic\"] }\n\t\t\t\t}}\n\t\t\t]\n\t\t\timports:[\n\t\t\t\t{stream:{account:\"B\", subject:\"iot.a.topic\"}}\n\t\t\t]\n\t\t},\n\t\tB:{\n\t\t\tusers:[\n\t\t\t\t{user: leafb, password: pwd},\n\t\t\t]\n\t\t\texports:[\n\t\t\t\t{stream: \"iot.a.topic\", accounts: [\"A\"]}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\ta1, oa1 := RunServerWithConfig(a1Conf)\n\tdefer a1.Shutdown()\n\n\ta2Conf := createConfFile(t, []byte(fmt.Sprintf(`\n\tcluster : {\n\t\tname : A\n\t\tport : -1\n\t\troutes : [nats://127.0.0.1:%d]\n\t}\n\tleafnodes : {\n\t\tport : 14334\n\t\tremotes : [{\n\t\t\taccount : A\n\t\t\turls : [nats://leafa:pwd@127.0.0.1:24334]\n\t\t}]\n\t}\n\tport : -1\n\tserver_name : A_2\n\n\taccounts:{\n\t\tA:{\n\t\t\tusers:[\n\t\t\t\t{user: leafa, password: pwd},\n\t\t\t\t{user: usera, password: usera, permissions: {\n\t\t\t\t\tpublish:{ allow:[\"iot.b.topic\"] }\n\t\t\t\t\tsubscribe:{ allow:[\"iot.a.topic\"] }\n\t\t\t\t}}\n\t\t\t]\n\t\t\timports:[\n\t\t\t\t{stream:{account:\"B\", subject:\"iot.a.topic\"}}\n\t\t\t]\n\t\t},\n\t\tB:{\n\t\t\tusers:[\n\t\t\t\t{user: leafb, password: pwd},\n\t\t\t]\n\t\t\texports:[\n\t\t\t\t{stream: \"iot.a.topic\", accounts: [\"A\"]}\n\t\t\t]\n\t\t}\n\t}`, oa1.Cluster.Port)))\n\ta2, _ := RunServerWithConfig(a2Conf)\n\tdefer a2.Shutdown()\n\n\tcheckClusterFormed(t, a1, a2)\n\n\tb1Conf := createConfFile(t, []byte(`\n\tcluster : {\n\t\tname : B\n\t\tport : -1\n\t}\n\tleafnodes : {\n\t\tport : 24333\n\t\tremotes : [{\n\t\t\taccount : B\n\t\t\turls : [nats://leafb:pwd@127.0.0.1:14333]\n\t\t}]\n\t}\n\tport : -1\n\tserver_name : B_1\n\n\taccounts:{\n\t\tA:{\n\t\t\tusers:[\n\t\t\t\t{user: leafa, password: pwd},\n\t\t\t]\n\t\t\texports:[\n\t\t\t\t{stream: \"iot.b.topic\", accounts: [\"B\"]}\n\t\t\t]\n\t\t},\n\t\tB:{\n\t\t\tusers:[\n\t\t\t\t{user: leafb, password: pwd},\n\t\t\t\t{user: userb, password: userb, permissions: {\n\t\t\t\t\tpublish:{ allow:[\"iot.a.topic\"] },\n\t\t\t\t\tsubscribe:{ allow:[\"iot.b.topic\"] }\n\t\t\t\t}}\n\t\t\t]\n\t\t\timports:[\n\t\t\t\t{stream:{account:\"A\", subject:\"iot.b.topic\"}}\n\t\t\t]\n\t\t}\n\t}`))\n\tb1, ob1 := RunServerWithConfig(b1Conf)\n\tdefer b1.Shutdown()\n\n\tb2Conf := createConfFile(t, []byte(fmt.Sprintf(`\n\tcluster : {\n\t\tname : B\n\t\tport : -1\n\t\troutes : [nats://127.0.0.1:%d]\n\t}\n\tleafnodes : {\n\t\tport : 24334\n\t\tremotes : [{\n\t\t\taccount : B\n\t\t\turls : [nats://leafb:pwd@127.0.0.1:14334]\n\t\t}]\n\t}\n\tport : -1\n\tserver_name : B_2\n\n\taccounts:{\n\t\tA:{\n\t\t\tusers:[\n\t\t\t\t{user: leafa, password: pwd},\n\t\t\t]\n\t\t\texports:[\n\t\t\t\t{stream: \"iot.b.topic\", accounts: [\"B\"]}\n\t\t\t]\n\t\t},\n\t\tB:{\n\t\t\tusers:[\n\t\t\t\t{user: leafb, password: pwd},\n\t\t\t\t{user: userb, password: userb, permissions: {\n\t\t\t\t\tpublish:{ allow:[\"iot.a.topic\"] },\n\t\t\t\t\tsubscribe:{ allow:[\"iot.b.topic\"] }\n\t\t\t\t}}\n\t\t\t]\n\t\t\timports:[\n\t\t\t\t{stream:{account:\"A\", subject:\"iot.b.topic\"}}\n\t\t\t]\n\t\t}\n\t}`, ob1.Cluster.Port)))\n\tb2, _ := RunServerWithConfig(b2Conf)\n\tdefer b2.Shutdown()\n\n\tcheckClusterFormed(t, b1, b2)\n\n\tcheckLeafNodeConnectedCount(t, a1, 2)\n\tcheckLeafNodeConnectedCount(t, a2, 2)\n\tcheckLeafNodeConnectedCount(t, b1, 2)\n\tcheckLeafNodeConnectedCount(t, b2, 2)\n\n\tcheck := func(t *testing.T, subSrv *Server, pubSrv *Server) {\n\n\t\tsc := natsConnect(t, subSrv.ClientURL(), nats.UserInfo(\"userb\", \"userb\"))\n\t\tdefer sc.Close()\n\n\t\tsubject := \"iot.b.topic\"\n\t\tsub := natsSubSync(t, sc, subject)\n\n\t\t// Wait for this to be available in A cluster\n\t\tcheckSubInterest(t, a1, \"A\", subject, time.Second)\n\t\tcheckSubInterest(t, a2, \"A\", subject, time.Second)\n\n\t\tpb := natsConnect(t, pubSrv.ClientURL(), nats.UserInfo(\"usera\", \"usera\"))\n\t\tdefer pb.Close()\n\n\t\tnatsPub(t, pb, subject, []byte(\"msg\"))\n\t\tnatsNexMsg(t, sub, time.Second)\n\t\t// Should be only 1\n\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\t\tt.Fatalf(\"Received duplicate on %q: %s\", msg.Subject, msg.Data)\n\t\t}\n\t}\n\tt.Run(\"sub_b1_pub_a1\", func(t *testing.T) { check(t, b1, a1) })\n\tt.Run(\"sub_b1_pub_a2\", func(t *testing.T) { check(t, b1, a2) })\n\tt.Run(\"sub_b2_pub_a1\", func(t *testing.T) { check(t, b2, a1) })\n\tt.Run(\"sub_b2_pub_a2\", func(t *testing.T) { check(t, b2, a2) })\n}\n\nfunc TestLeafNodeTLSHandshakeFirstVerifyNoInfoSent(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tport : -1\n\t\tleafnodes : {\n\t\t\tport : -1\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t\thandshake_first: true\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(confHub)\n\tdefer s1.Shutdown()\n\n\tc, err := net.DialTimeout(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", o1.LeafNode.Port), 2*time.Second)\n\trequire_NoError(t, err)\n\tdefer c.Close()\n\n\tbuf := make([]byte, 1024)\n\t// We will wait for up to 500ms to see if the server is sending (incorrectly)\n\t// the INFO.\n\tc.SetReadDeadline(time.Now().Add(500 * time.Millisecond))\n\tn, err := c.Read(buf)\n\tc.SetReadDeadline(time.Time{})\n\t// If we did not get an error, this is an issue...\n\tif err == nil {\n\t\tt.Fatalf(\"Should not have received anything, got n=%v buf=%s\", n, buf[:n])\n\t}\n\t// We expect a timeout error\n\tif ne, ok := err.(net.Error); !ok || !ne.Timeout() {\n\t\tt.Fatalf(\"Expected a timeout error, got %v\", err)\n\t}\n}\n\nfunc TestLeafNodeTLSHandshakeFirstFallbackDelayConfigValues(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\t\ttimeout: \t1\n\t\t\t\tfirst: \t\t%s\n\t\t\t}\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tval string\n\t\tfirst bool\n\t\tdelay time.Duration\n\t}{\n\t\t{\"first as boolean true\", \"true\", true, 0},\n\t\t{\"first as boolean false\", \"false\", false, 0},\n\t\t{\"first as string true\", \"\\\"true\\\"\", true, 0},\n\t\t{\"first as string false\", \"\\\"false\\\"\", false, 0},\n\t\t{\"first as string on\", \"on\", true, 0},\n\t\t{\"first as string off\", \"off\", false, 0},\n\t\t{\"first as string auto\", \"auto\", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY},\n\t\t{\"first as string auto_fallback\", \"auto_fallback\", true, DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY},\n\t\t{\"first as fallback duration\", \"300ms\", true, 300 * time.Millisecond},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.val)))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tif test.first {\n\t\t\t\tif !o.LeafNode.TLSHandshakeFirst {\n\t\t\t\t\tt.Fatal(\"Expected tls first to be true, was not\")\n\t\t\t\t}\n\t\t\t\tif test.delay != o.LeafNode.TLSHandshakeFirstFallback {\n\t\t\t\t\tt.Fatalf(\"Expected fallback delay to be %v, got %v\", test.delay, o.LeafNode.TLSHandshakeFirstFallback)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif o.LeafNode.TLSHandshakeFirst {\n\t\t\t\t\tt.Fatal(\"Expected tls first to be false, was not\")\n\t\t\t\t}\n\t\t\t\tif o.LeafNode.TLSHandshakeFirstFallback != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected fallback delay to be 0, got %v\", o.LeafNode.TLSHandshakeFirstFallback)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeTLSHandshakeFirst(t *testing.T) {\n\ttmpl1 := `\n\t\tport : -1\n\t\tleafnodes : {\n\t\t\tport : -1\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t\thandshake_first: %s\n\t\t\t}\n\t\t}\n\t`\n\tconfHub := createConfFile(t, []byte(fmt.Sprintf(tmpl1, \"true\")))\n\ts1, o1 := RunServerWithConfig(confHub)\n\tdefer s1.Shutdown()\n\n\ttmpl2 := `\n\t\tport: -1\n\t\tleafnodes : {\n\t\t\tport : -1\n\t\t\tremotes : [\n\t\t\t\t{\n\t\t\t\t\turls : [tls://127.0.0.1:%d]\n\t\t\t\t\ttls {\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/client-cert.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/client-key.pem\"\n\t\t\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\t\t\ttimeout: 2\n\t\t\t\t\t\tfirst: %s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tconfSpoke := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, \"true\")))\n\ts2, _ := RunServerWithConfig(confSpoke)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\ts2.Shutdown()\n\n\t// Now check that there will be a failure if the remote does not ask for\n\t// handshake first since the hub is configured that way.\n\t// Set a logger on s1 to capture errors\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts1.SetLogger(l, false, false)\n\n\tconfSpoke = createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, \"false\")))\n\ts2, _ = RunServerWithConfig(confSpoke)\n\tdefer s2.Shutdown()\n\n\tselect {\n\tcase err := <-l.errCh:\n\t\tif !strings.Contains(err, \"handshake error\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Did not get TLS handshake failure\")\n\t}\n\n\t// Check configuration reload for this remote\n\treloadUpdateConfig(t, s2, confSpoke, fmt.Sprintf(tmpl2, o1.LeafNode.Port, \"true\"))\n\tcheckLeafNodeConnected(t, s2)\n\ts2.Shutdown()\n\n\t// Drain the logger error channel\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase <-l.errCh:\n\t\tdefault:\n\t\t\tdone = true\n\t\t}\n\t}\n\n\t// Now change the config on the hub\n\treloadUpdateConfig(t, s1, confHub, fmt.Sprintf(tmpl1, \"false\"))\n\t// Restart s2\n\ts2, _ = RunServerWithConfig(confSpoke)\n\tdefer s2.Shutdown()\n\n\tselect {\n\tcase err := <-l.errCh:\n\t\tif !strings.Contains(err, \"handshake error\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Did not get TLS handshake failure\")\n\t}\n\n\t// Reload again with \"true\"\n\treloadUpdateConfig(t, s1, confHub, fmt.Sprintf(tmpl1, \"true\"))\n\tcheckLeafNodeConnected(t, s2)\n\n\t// Shutdown the remote\n\ts2.Shutdown()\n\n\t// Reload the hub with handshake_first with a duration\n\treloadUpdateConfig(t, s1, confHub, fmt.Sprintf(tmpl1, `\"250ms\"`))\n\n\t// Now start s2 with a remote that does not use handshake_first (set to false)\n\tconfSpoke = createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, \"false\")))\n\ts2, _ = RunServerWithConfig(confSpoke)\n\tdefer s2.Shutdown()\n\tcheckLeafNodeConnected(t, s2)\n}\n\nfunc TestLeafNodeTLSHandshakeEvenForRemoteWithNoTLSBlock(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tport : -1\n\t\tleafnodes : {\n\t\t\tport : -1\n\t\t\ttls {\n\t\t\t\tcert_file: \"../test/configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(confHub)\n\tdefer s1.Shutdown()\n\n\ttmpl2 := `\n\t\tport: -1\n\t\tleafnodes : {\n\t\t\tport : -1\n\t\t\tremotes : [\n\t\t\t\t{\n\t\t\t\t\turls : [tls://127.0.0.1:%d]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tconfSpoke := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(confSpoke)\n\tdefer s2.Shutdown()\n\n\tl := &captureDebugLogger{dbgCh: make(chan string, 100)}\n\ts2.SetLogger(l, true, false)\n\n\ttm := time.NewTimer(2 * time.Second)\n\tdefer tm.Stop()\n\tfor {\n\t\tselect {\n\t\tcase l := <-l.dbgCh:\n\t\t\tif strings.Contains(l, \"Starting TLS\") {\n\t\t\t\t// OK!\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-tm.C:\n\t\t\tt.Fatalf(\"Did not perform a TLS handshake\")\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeCompressionOptions(t *testing.T) {\n\torg := testDefaultLeafNodeCompression\n\ttestDefaultLeafNodeCompression = _EMPTY_\n\tdefer func() { testDefaultLeafNodeCompression = org }()\n\n\ttmpl := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: %s\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmode string\n\t\trttVals []int\n\t\texpected string\n\t\trtts []time.Duration\n\t}{\n\t\t{\"boolean enabled\", \"true\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string enabled\", \"enabled\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string EnaBled\", \"EnaBled\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string on\", \"on\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string ON\", \"ON\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string fast\", \"fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string Fast\", \"Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_fast\", \"s2_fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_Fast\", \"s2_Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"boolean disabled\", \"false\", nil, CompressionOff, nil},\n\t\t{\"string disabled\", \"disabled\", nil, CompressionOff, nil},\n\t\t{\"string DisableD\", \"DisableD\", nil, CompressionOff, nil},\n\t\t{\"string off\", \"off\", nil, CompressionOff, nil},\n\t\t{\"string OFF\", \"OFF\", nil, CompressionOff, nil},\n\t\t{\"better\", \"better\", nil, CompressionS2Better, nil},\n\t\t{\"Better\", \"Better\", nil, CompressionS2Better, nil},\n\t\t{\"s2_better\", \"s2_better\", nil, CompressionS2Better, nil},\n\t\t{\"S2_BETTER\", \"S2_BETTER\", nil, CompressionS2Better, nil},\n\t\t{\"best\", \"best\", nil, CompressionS2Best, nil},\n\t\t{\"BEST\", \"BEST\", nil, CompressionS2Best, nil},\n\t\t{\"s2_best\", \"s2_best\", nil, CompressionS2Best, nil},\n\t\t{\"S2_BEST\", \"S2_BEST\", nil, CompressionS2Best, nil},\n\t\t{\"auto no rtts\", \"auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"s2_auto no rtts\", \"s2_auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"auto\", \"{mode: auto, rtt_thresholds: [%s]}\", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}},\n\t\t{\"Auto\", \"{Mode: Auto, thresholds: [%s]}\", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}},\n\t\t{\"s2_auto\", \"{mode: s2_auto, thresholds: [%s]}\", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}},\n\t\t{\"s2_AUTO\", \"{mode: s2_AUTO, thresholds: [%s]}\", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}},\n\t\t{\"s2_auto:-10,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,15\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}},\n\t\t{\"s2_auto:0,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,0,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,0\", \"{mode: s2_auto, rtt_thresholds: [%s]}\", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar val string\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\tvar rtts string\n\t\t\t\tfor i, v := range test.rttVals {\n\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\trtts += \", \"\n\t\t\t\t\t}\n\t\t\t\t\trtts += fmt.Sprintf(\"%dms\", v)\n\t\t\t\t}\n\t\t\t\tval = fmt.Sprintf(test.mode, rtts)\n\t\t\t} else {\n\t\t\t\tval = test.mode\n\t\t\t}\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val)))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tif cm := o.LeafNode.Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, o.LeafNode.Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, o.LeafNode.Compression.RTTThresholds)\n\t\t\t}\n\t\t\tport := s.getOpts().Port\n\t\t\tleafNodePort := s.getOpts().LeafNode.Port\n\t\t\ts.Shutdown()\n\n\t\t\to.Port = port\n\t\t\to.LeafNode.Port = leafNodePort\n\t\t\to.LeafNode.Compression.Mode = test.mode\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\to.LeafNode.Compression.Mode = CompressionS2Auto\n\t\t\t\to.LeafNode.Compression.RTTThresholds = o.LeafNode.Compression.RTTThresholds[:0]\n\t\t\t\tfor _, v := range test.rttVals {\n\t\t\t\t\to.LeafNode.Compression.RTTThresholds = append(o.LeafNode.Compression.RTTThresholds, time.Duration(v)*time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts = RunServer(o)\n\t\t\tdefer s.Shutdown()\n\t\t\tif cm := o.LeafNode.Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, o.LeafNode.Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, o.LeafNode.Compression.RTTThresholds)\n\t\t\t}\n\t\t})\n\t}\n\n\t// Same, but with remotes\n\ttmpl = `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://127.0.0.1:1234\"\n\t\t\t\t\tcompression: %s\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmode string\n\t\trttVals []int\n\t\texpected string\n\t\trtts []time.Duration\n\t}{\n\t\t{\"boolean enabled\", \"true\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string enabled\", \"enabled\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string EnaBled\", \"EnaBled\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string on\", \"on\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string ON\", \"ON\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"string fast\", \"fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string Fast\", \"Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_fast\", \"s2_fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_Fast\", \"s2_Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"boolean disabled\", \"false\", nil, CompressionOff, nil},\n\t\t{\"string disabled\", \"disabled\", nil, CompressionOff, nil},\n\t\t{\"string DisableD\", \"DisableD\", nil, CompressionOff, nil},\n\t\t{\"string off\", \"off\", nil, CompressionOff, nil},\n\t\t{\"string OFF\", \"OFF\", nil, CompressionOff, nil},\n\t\t{\"better\", \"better\", nil, CompressionS2Better, nil},\n\t\t{\"Better\", \"Better\", nil, CompressionS2Better, nil},\n\t\t{\"s2_better\", \"s2_better\", nil, CompressionS2Better, nil},\n\t\t{\"S2_BETTER\", \"S2_BETTER\", nil, CompressionS2Better, nil},\n\t\t{\"best\", \"best\", nil, CompressionS2Best, nil},\n\t\t{\"BEST\", \"BEST\", nil, CompressionS2Best, nil},\n\t\t{\"s2_best\", \"s2_best\", nil, CompressionS2Best, nil},\n\t\t{\"S2_BEST\", \"S2_BEST\", nil, CompressionS2Best, nil},\n\t\t{\"auto no rtts\", \"auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"s2_auto no rtts\", \"s2_auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"auto\", \"{mode: auto, rtt_thresholds: [%s]}\", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}},\n\t\t{\"Auto\", \"{Mode: Auto, thresholds: [%s]}\", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}},\n\t\t{\"s2_auto\", \"{mode: s2_auto, thresholds: [%s]}\", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}},\n\t\t{\"s2_AUTO\", \"{mode: s2_AUTO, thresholds: [%s]}\", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}},\n\t\t{\"s2_auto:-10,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,15\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}},\n\t\t{\"s2_auto:0,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,0,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,0\", \"{mode: s2_auto, rtt_thresholds: [%s]}\", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}},\n\t} {\n\t\tt.Run(\"remote leaf \"+test.name, func(t *testing.T) {\n\t\t\tvar val string\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\tvar rtts string\n\t\t\t\tfor i, v := range test.rttVals {\n\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\trtts += \", \"\n\t\t\t\t\t}\n\t\t\t\t\trtts += fmt.Sprintf(\"%dms\", v)\n\t\t\t\t}\n\t\t\t\tval = fmt.Sprintf(test.mode, rtts)\n\t\t\t} else {\n\t\t\t\tval = test.mode\n\t\t\t}\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val)))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tr := o.LeafNode.Remotes[0]\n\n\t\t\tif cm := r.Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, r.Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, r.Compression.RTTThresholds)\n\t\t\t}\n\t\t\tport := s.getOpts().Port\n\t\t\tleafNodePort := s.getOpts().LeafNode.Port\n\t\t\ts.Shutdown()\n\n\t\t\to.Port = port\n\t\t\to.LeafNode.Port = leafNodePort\n\t\t\to.LeafNode.Remotes[0].Compression.Mode = test.mode\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\to.LeafNode.Remotes[0].Compression.Mode = CompressionS2Auto\n\t\t\t\to.LeafNode.Remotes[0].Compression.RTTThresholds = o.LeafNode.Remotes[0].Compression.RTTThresholds[:0]\n\t\t\t\tfor _, v := range test.rttVals {\n\t\t\t\t\to.LeafNode.Remotes[0].Compression.RTTThresholds = append(o.LeafNode.Remotes[0].Compression.RTTThresholds, time.Duration(v)*time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts = RunServer(o)\n\t\t\tdefer s.Shutdown()\n\t\t\tif cm := o.LeafNode.Remotes[0].Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, o.LeafNode.Remotes[0].Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, o.LeafNode.Remotes[0].Compression.RTTThresholds)\n\t\t\t}\n\t\t})\n\t}\n\n\t// Test that with no compression specified, we default to \"s2_auto\"\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tif o.LeafNode.Compression.Mode != CompressionS2Auto {\n\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", CompressionAccept, o.LeafNode.Compression.Mode)\n\t}\n\tif !reflect.DeepEqual(defaultCompressionS2AutoRTTThresholds, o.LeafNode.Compression.RTTThresholds) {\n\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", defaultCompressionS2AutoRTTThresholds, o.LeafNode.Compression.RTTThresholds)\n\t}\n\t// Same for remotes\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [ { url: \"nats://127.0.0.1:1234\" } ]\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tif cm := o.LeafNode.Remotes[0].Compression.Mode; cm != CompressionS2Auto {\n\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", CompressionAccept, cm)\n\t}\n\tif !reflect.DeepEqual(defaultCompressionS2AutoRTTThresholds, o.LeafNode.Remotes[0].Compression.RTTThresholds) {\n\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", defaultCompressionS2AutoRTTThresholds, o.LeafNode.Remotes[0].Compression.RTTThresholds)\n\t}\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmode string\n\t\trtts []time.Duration\n\t\terr string\n\t}{\n\t\t{\"unsupported mode\", \"gzip\", nil, \"unsupported\"},\n\t\t{\"not ascending order\", \"s2_auto\", []time.Duration{\n\t\t\t5 * time.Millisecond,\n\t\t\t10 * time.Millisecond,\n\t\t\t2 * time.Millisecond,\n\t\t}, \"ascending\"},\n\t\t{\"too many thresholds\", \"s2_auto\", []time.Duration{\n\t\t\t5 * time.Millisecond,\n\t\t\t10 * time.Millisecond,\n\t\t\t20 * time.Millisecond,\n\t\t\t40 * time.Millisecond,\n\t\t\t60 * time.Millisecond,\n\t\t}, \"more than 4\"},\n\t\t{\"all 0\", \"s2_auto\", []time.Duration{0, 0, 0, 0}, \"at least one\"},\n\t\t{\"single 0\", \"s2_auto\", []time.Duration{0}, \"at least one\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.LeafNode.Port = -1\n\t\t\to.LeafNode.Compression = CompressionOpts{test.mode, test.rtts}\n\t\t\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Same with remotes\n\t\t\to.LeafNode.Compression = CompressionOpts{}\n\t\t\to.LeafNode.Remotes = []*RemoteLeafOpts{{Compression: CompressionOpts{test.mode, test.rtts}}}\n\t\t\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeCompression(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"Hub\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\tC { users: [{user: c, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: s2_fast\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tport := o1.LeafNode.Port\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"Spoke\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\tC { users: [{user: c, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\", compression: s2_better }\n\t\t\t\t{ url: \"nats://b:pwd@127.0.0.1:%d\", account: \"B\", compression: s2_best }\n\t\t\t\t{ url: \"nats://c:pwd@127.0.0.1:%d\", account: \"C\", compression: off }\n\t\t\t]\n\t\t}\n\t`, port, port, port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, s1, 3)\n\tcheckLeafNodeConnectedCount(t, s2, 3)\n\n\ts1.mu.RLock()\n\tfor _, l := range s1.leafs {\n\t\tl.mu.Lock()\n\t\tl.nc = &testConnSentBytes{Conn: l.nc}\n\t\tl.mu.Unlock()\n\t}\n\ts1.mu.RUnlock()\n\n\tvar payloads [][]byte\n\ttotalPayloadSize := 0\n\tcount := 26\n\tfor i := 0; i < count; i++ {\n\t\tn := rand.Intn(2048) + 1\n\t\tp := make([]byte, n)\n\t\tfor j := 0; j < n; j++ {\n\t\t\tp[j] = byte(i) + 'A'\n\t\t}\n\t\ttotalPayloadSize += len(p)\n\t\tpayloads = append(payloads, p)\n\t}\n\n\tcheck := func(acc, user, subj string) {\n\t\tt.Helper()\n\t\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc2.Close()\n\t\tsub := natsSubSync(t, nc2, subj)\n\t\tnatsFlush(t, nc2)\n\t\tcheckSubInterest(t, s1, acc, subj, time.Second)\n\n\t\tnc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc1.Close()\n\n\t\tfor i := 0; i < count; i++ {\n\t\t\tnatsPub(t, nc1, subj, payloads[i])\n\t\t}\n\t\tfor i := 0; i < count; i++ {\n\t\t\tm := natsNexMsg(t, sub, time.Second)\n\t\t\tif !bytes.Equal(m.Data, payloads[i]) {\n\t\t\t\tt.Fatalf(\"Expected payload %q - got %q\", payloads[i], m.Data)\n\t\t\t}\n\t\t}\n\n\t\t// Also check that the leafnode stats shows that compression likely occurred\n\t\tvar out int\n\t\ts1.mu.RLock()\n\t\tfor _, l := range s1.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tif l.acc.Name == acc && l.nc != nil {\n\t\t\t\tnc := l.nc.(*testConnSentBytes)\n\t\t\t\tnc.Lock()\n\t\t\t\tout = nc.sent\n\t\t\t\tnc.sent = 0\n\t\t\t\tnc.Unlock()\n\t\t\t}\n\t\t\tl.mu.Unlock()\n\t\t}\n\t\ts1.mu.RUnlock()\n\t\t// Except for account \"C\", where compression should be off,\n\t\t// \"out\" should at least be smaller than totalPayloadSize, use 20%.\n\t\tif acc == \"C\" {\n\t\t\tif int(out) < totalPayloadSize {\n\t\t\t\tt.Fatalf(\"Expected s1's sent bytes to be at least payload size (%v), got %v\", totalPayloadSize, out)\n\t\t\t}\n\t\t} else {\n\t\t\tlimit := totalPayloadSize * 80 / 100\n\t\t\tif int(out) > limit {\n\t\t\t\tt.Fatalf(\"Expected s1's sent bytes to be less than %v, got %v (total payload was %v)\", limit, out, totalPayloadSize)\n\t\t\t}\n\t\t}\n\t}\n\tcheck(\"A\", \"a\", \"foo\")\n\tcheck(\"B\", \"b\", \"bar\")\n\tcheck(\"C\", \"c\", \"baz\")\n\n\t// Check compression settings. S1 should always be s2_fast, except for account \"C\"\n\t// since \"C\" wanted compression \"off\"\n\tl, err := s1.Leafz(nil)\n\trequire_NoError(t, err)\n\tfor _, r := range l.Leafs {\n\t\tswitch r.Account {\n\t\tcase \"C\":\n\t\t\tif r.Compression != CompressionOff {\n\t\t\t\tt.Fatalf(\"Expected compression of remote for C account to be %q, got %q\", CompressionOff, r.Compression)\n\t\t\t}\n\t\tdefault:\n\t\t\tif r.Compression != CompressionS2Fast {\n\t\t\t\tt.Fatalf(\"Expected compression of remote for %s account to be %q, got %q\", r.Account, CompressionS2Fast, r.Compression)\n\t\t\t}\n\t\t}\n\t}\n\n\tl, err = s2.Leafz(nil)\n\trequire_NoError(t, err)\n\tfor _, r := range l.Leafs {\n\t\tswitch r.Account {\n\t\tcase \"A\":\n\t\t\tif r.Compression != CompressionS2Better {\n\t\t\t\tt.Fatalf(\"Expected compression for A account to be %q, got %q\", CompressionS2Better, r.Compression)\n\t\t\t}\n\t\tcase \"B\":\n\t\t\tif r.Compression != CompressionS2Best {\n\t\t\t\tt.Fatalf(\"Expected compression for B account to be %q, got %q\", CompressionS2Best, r.Compression)\n\t\t\t}\n\t\tcase \"C\":\n\t\t\tif r.Compression != CompressionOff {\n\t\t\t\tt.Fatalf(\"Expected compression for C account to be %q, got %q\", CompressionOff, r.Compression)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc BenchmarkLeafNodeCompression(b *testing.B) {\n\tconf1 := createConfFile(b, []byte(`\n\t\tport: -1\n\t\tserver_name: \"Hub\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\tC { users: [{user: c, password: pwd}] }\n\t\t\tD { users: [{user: d, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tport := o1.LeafNode.Port\n\tconf2 := createConfFile(b, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"Spoke\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\tC { users: [{user: c, password: pwd}] }\n\t\t\tD { users: [{user: d, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\", compression: s2_better }\n\t\t\t\t{ url: \"nats://b:pwd@127.0.0.1:%d\", account: \"B\", compression: s2_best }\n\t\t\t\t{ url: \"nats://c:pwd@127.0.0.1:%d\", account: \"C\", compression: s2_fast }\n\t\t\t\t{ url: \"nats://d:pwd@127.0.0.1:%d\", account: \"D\", compression: off }\n\t\t\t]\n\t\t}\n\t`, port, port, port, port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnectedCount(b, s1, 4)\n\tcheckLeafNodeConnectedCount(b, s2, 4)\n\n\tl, err := s2.Leafz(nil)\n\trequire_NoError(b, err)\n\tfor _, r := range l.Leafs {\n\t\tswitch {\n\t\tcase r.Account == \"A\" && r.Compression == CompressionS2Better:\n\t\tcase r.Account == \"B\" && r.Compression == CompressionS2Best:\n\t\tcase r.Account == \"C\" && r.Compression == CompressionS2Fast:\n\t\tcase r.Account == \"D\" && r.Compression == CompressionOff:\n\t\tdefault:\n\t\t\tb.Fatalf(\"Account %q had incorrect compression mode %q on leaf connection\", r.Account, r.Compression)\n\t\t}\n\t}\n\n\tmsg := make([]byte, 1024)\n\tfor _, p := range []struct {\n\t\talgo string\n\t\tuser string\n\t}{\n\t\t{\"Better\", \"a\"},\n\t\t{\"Best\", \"b\"},\n\t\t{\"Fast\", \"c\"},\n\t\t{\"Off\", \"d\"},\n\t} {\n\t\tnc1 := natsConnect(b, s1.ClientURL(), nats.UserInfo(p.user, \"pwd\"))\n\t\tnc2 := natsConnect(b, s2.ClientURL(), nats.UserInfo(p.user, \"pwd\"))\n\n\t\tsub, err := nc1.SubscribeSync(\"foo\")\n\t\trequire_NoError(b, err)\n\n\t\ttime.Sleep(time.Second)\n\n\t\tb.Run(p.algo, func(b *testing.B) {\n\t\t\tstart := time.Now()\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\terr = nc2.Publish(\"foo\", msg)\n\t\t\t\trequire_NoError(b, err)\n\n\t\t\t\t_, err = sub.NextMsg(time.Second)\n\t\t\t\trequire_NoError(b, err)\n\t\t\t}\n\n\t\t\tb.ReportMetric(float64(len(msg)*b.N)/1024/1024, \"MB\")\n\t\t\tb.ReportMetric(float64(len(msg)*b.N)/1024/1024/float64(time.Since(start).Seconds()), \"MB/sec\")\n\t\t})\n\n\t\tnc1.Close()\n\t\tnc2.Close()\n\t}\n}\n\nfunc TestLeafNodeCompressionMatrixModes(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\ts1 string\n\t\ts2 string\n\t\ts1Expected string\n\t\ts2Expected string\n\t}{\n\t\t{\"off off\", \"off\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"off accept\", \"off\", \"accept\", CompressionOff, CompressionOff},\n\t\t{\"off on\", \"off\", \"on\", CompressionOff, CompressionOff},\n\t\t{\"off better\", \"off\", \"better\", CompressionOff, CompressionOff},\n\t\t{\"off best\", \"off\", \"best\", CompressionOff, CompressionOff},\n\n\t\t{\"accept off\", \"accept\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"accept accept\", \"accept\", \"accept\", CompressionOff, CompressionOff},\n\t\t// Note: \"on\", means s2_auto, which will mean uncompressed since RTT is low.\n\t\t{\"accept on\", \"accept\", \"on\", CompressionS2Fast, CompressionS2Uncompressed},\n\t\t{\"accept better\", \"accept\", \"better\", CompressionS2Better, CompressionS2Better},\n\t\t{\"accept best\", \"accept\", \"best\", CompressionS2Best, CompressionS2Best},\n\n\t\t{\"on off\", \"on\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"on accept\", \"on\", \"accept\", CompressionS2Uncompressed, CompressionS2Fast},\n\t\t{\"on on\", \"on\", \"on\", CompressionS2Uncompressed, CompressionS2Uncompressed},\n\t\t{\"on better\", \"on\", \"better\", CompressionS2Uncompressed, CompressionS2Better},\n\t\t{\"on best\", \"on\", \"best\", CompressionS2Uncompressed, CompressionS2Best},\n\n\t\t{\"better off\", \"better\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"better accept\", \"better\", \"accept\", CompressionS2Better, CompressionS2Better},\n\t\t{\"better on\", \"better\", \"on\", CompressionS2Better, CompressionS2Uncompressed},\n\t\t{\"better better\", \"better\", \"better\", CompressionS2Better, CompressionS2Better},\n\t\t{\"better best\", \"better\", \"best\", CompressionS2Better, CompressionS2Best},\n\n\t\t{\"best off\", \"best\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"best accept\", \"best\", \"accept\", CompressionS2Best, CompressionS2Best},\n\t\t{\"best on\", \"best\", \"on\", CompressionS2Best, CompressionS2Uncompressed},\n\t\t{\"best better\", \"best\", \"better\", CompressionS2Best, CompressionS2Better},\n\t\t{\"best best\", \"best\", \"best\", CompressionS2Best, CompressionS2Best},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: %s\n\t\t\t\t}\n\t\t\t`, test.s1)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{url: \"nats://127.0.0.1:%d\", compression: %s}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`, o1.LeafNode.Port, test.s2)))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, s2)\n\n\t\t\tnc1 := natsConnect(t, s1.ClientURL())\n\t\t\tdefer nc1.Close()\n\n\t\t\tnc2 := natsConnect(t, s2.ClientURL())\n\t\t\tdefer nc2.Close()\n\n\t\t\tpayload := make([]byte, 128)\n\t\t\tcheck := func(ncp, ncs *nats.Conn, subj string, s *Server) {\n\t\t\t\tt.Helper()\n\t\t\t\tsub := natsSubSync(t, ncs, subj)\n\t\t\t\tcheckSubInterest(t, s, globalAccountName, subj, time.Second)\n\t\t\t\tnatsPub(t, ncp, subj, payload)\n\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\n\t\t\t\tfor _, srv := range []*Server{s1, s2} {\n\t\t\t\t\tlz, err := srv.Leafz(nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tvar expected string\n\t\t\t\t\tif srv == s1 {\n\t\t\t\t\t\texpected = test.s1Expected\n\t\t\t\t\t} else {\n\t\t\t\t\t\texpected = test.s2Expected\n\t\t\t\t\t}\n\t\t\t\t\tif cm := lz.Leafs[0].Compression; cm != expected {\n\t\t\t\t\t\tt.Fatalf(\"Server %s - expected compression %q, got %q\", srv, expected, cm)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck(nc1, nc2, \"foo\", s1)\n\t\t\tcheck(nc2, nc1, \"bar\", s2)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeCompressionWithOlderServer(t *testing.T) {\n\ttmpl1 := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: \"%s\"\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, CompressionS2Fast)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\ttmpl2 := `\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{url: \"nats://127.0.0.1:%d\", compression: \"%s\"}\n\t\t\t]\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, CompressionNotSupported)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\tgetLeafCompMode := func(s *Server) string {\n\t\tvar cm string\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\tfor _, l := range s1.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tcm = l.leaf.compression\n\t\t\tl.mu.Unlock()\n\t\t\treturn cm\n\t\t}\n\t\treturn _EMPTY_\n\t}\n\tfor _, s := range []*Server{s1, s2} {\n\t\tif cm := getLeafCompMode(s); cm != CompressionNotSupported {\n\t\t\tt.Fatalf(\"Expected compression not supported, got %q\", cm)\n\t\t}\n\t}\n\n\ts2.Shutdown()\n\ts1.Shutdown()\n\n\tconf1 = createConfFile(t, []byte(fmt.Sprintf(tmpl1, CompressionNotSupported)))\n\ts1, o1 = RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 = createConfFile(t, []byte(fmt.Sprintf(tmpl2, o1.LeafNode.Port, CompressionS2Fast)))\n\ts2, _ = RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\tfor _, s := range []*Server{s1, s2} {\n\t\tif cm := getLeafCompMode(s); cm != CompressionNotSupported {\n\t\t\tt.Fatalf(\"Expected compression not supported, got %q\", cm)\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeCompressionAuto(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\ts1Ping string\n\t\ts1Compression string\n\t\ts2Ping string\n\t\ts2Compression string\n\t\tcheckS1 bool\n\t}{\n\t\t{\"remote side\", \"10s\", CompressionS2Fast, \"100ms\", \"{mode: s2_auto, rtt_thresholds: [10ms, 20ms, 30ms]}\", false},\n\t\t{\"accept side\", \"100ms\", \"{mode: s2_auto, rtt_thresholds: [10ms, 20ms, 30ms]}\", \"10s\", CompressionS2Fast, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tping_interval: \"%s\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tcompression: %s\n\t\t\t\t}\n\t\t\t`, test.s1Ping, test.s1Compression)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\t// Start with 0ms RTT\n\t\t\tnp := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.LeafNode.Port), true)\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tping_interval: \"%s\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{url: %s, compression %s}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`, test.s2Ping, np.routeURL(), test.s2Compression)))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\t\t\tdefer np.stop()\n\n\t\t\tcheckLeafNodeConnected(t, s2)\n\n\t\t\tcheckComp := func(expected string) {\n\t\t\t\tt.Helper()\n\t\t\t\tvar s *Server\n\t\t\t\tif test.checkS1 {\n\t\t\t\t\ts = s1\n\t\t\t\t} else {\n\t\t\t\t\ts = s2\n\t\t\t\t}\n\t\t\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\ts.mu.RLock()\n\t\t\t\t\tdefer s.mu.RUnlock()\n\t\t\t\t\tfor _, l := range s.leafs {\n\t\t\t\t\t\tl.mu.Lock()\n\t\t\t\t\t\tcm := l.leaf.compression\n\t\t\t\t\t\tl.mu.Unlock()\n\t\t\t\t\t\tif cm != expected {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"Leaf %v compression mode expected to be %q, got %q\", l, expected, cm)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tcheckComp(CompressionS2Uncompressed)\n\n\t\t\t// Change the proxy RTT and we should get compression \"fast\"\n\t\t\tnp.updateRTT(15 * time.Millisecond)\n\t\t\tcheckComp(CompressionS2Fast)\n\n\t\t\t// Now 25ms, and get \"better\"\n\t\t\tnp.updateRTT(25 * time.Millisecond)\n\t\t\tcheckComp(CompressionS2Better)\n\n\t\t\t// Above 35 and we should get \"best\"\n\t\t\tnp.updateRTT(35 * time.Millisecond)\n\t\t\tcheckComp(CompressionS2Best)\n\n\t\t\t// Down to 1ms and again should get \"uncompressed\"\n\t\t\tnp.updateRTT(1 * time.Millisecond)\n\t\t\tcheckComp(CompressionS2Uncompressed)\n\t\t})\n\t}\n\n\t// Make sure that if compression is off on one side, the update of RTT does\n\t// not trigger a compression change.\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: off\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\t// Start with 0ms RTT\n\tnp := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.LeafNode.Port), true)\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tping_interval: \"50ms\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{url: %s, compression s2_auto}\n\t\t\t]\n\t\t}\n\t`, np.routeURL())))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\tdefer np.stop()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\t// Even with a bug of updating compression level while it should have been\n\t// off, the check done below would almost always pass because after\n\t// reconnecting, there could be a chance to get at first compression set\n\t// to \"off\". So we will double check that the leaf node CID did not change\n\t// at the end of the test.\n\tgetCID := func() uint64 {\n\t\ts2.mu.RLock()\n\t\tdefer s2.mu.RUnlock()\n\t\tfor _, l := range s2.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tcid := l.cid\n\t\t\tl.mu.Unlock()\n\t\t\treturn cid\n\t\t}\n\t\treturn 0\n\t}\n\toldCID := getCID()\n\n\tcheckCompOff := func() {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\ts2.mu.RLock()\n\t\t\tdefer s2.mu.RUnlock()\n\t\t\tif len(s2.leafs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"Leaf not currently connected\")\n\t\t\t}\n\t\t\tfor _, l := range s2.leafs {\n\t\t\t\tl.mu.Lock()\n\t\t\t\tcm := l.leaf.compression\n\t\t\t\tl.mu.Unlock()\n\t\t\t\tif cm != CompressionOff {\n\t\t\t\t\treturn fmt.Errorf(\"Leaf %v compression mode expected to be %q, got %q\", l, CompressionOff, cm)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckCompOff()\n\n\t// Now change RTT and again, make sure that it is still off\n\tnp.updateRTT(20 * time.Millisecond)\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckCompOff()\n\tif cid := getCID(); cid != oldCID {\n\t\tt.Fatalf(\"Leafnode has reconnected, cid was %v, now %v\", oldCID, cid)\n\t}\n}\n\nfunc TestLeafNodeCompressionWithWSCompression(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\twebsocket {\n\t\t\tport: -1\n\t\t\tno_tls: true\n\t\t\tcompression: true\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: s2_fast\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\t\tws_compression: true\n\t\t\t\t\tcompression: s2_fast\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.Websocket.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\tnc1 := natsConnect(t, s1.ClientURL())\n\tdefer nc1.Close()\n\n\tsub := natsSubSync(t, nc1, \"foo\")\n\tcheckSubInterest(t, s2, globalAccountName, \"foo\", time.Second)\n\n\tnc2 := natsConnect(t, s2.ClientURL())\n\tdefer nc2.Close()\n\n\tpayload := make([]byte, 1024)\n\tfor i := 0; i < len(payload); i++ {\n\t\tpayload[i] = 'A'\n\t}\n\tnatsPub(t, nc2, \"foo\", payload)\n\tmsg := natsNexMsg(t, sub, time.Second)\n\trequire_True(t, len(msg.Data) == 1024)\n\tfor i := 0; i < len(msg.Data); i++ {\n\t\tif msg.Data[i] != 'A' {\n\t\t\tt.Fatalf(\"Invalid msg: %s\", msg.Data)\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeCompressionWithWSGetNeedsData(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\twebsocket {\n\t\t\tport: -1\n\t\t\tno_tls: true\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: s2_fast\n\t\t}\n\t`))\n\tsrv1, o1 := RunServerWithConfig(conf1)\n\tdefer srv1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"ws://127.0.0.1:%d\"\n\t\t\t\t\tws_no_masking: true\n\t\t\t\t\tcompression: s2_fast\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.Websocket.Port)))\n\tsrv2, _ := RunServerWithConfig(conf2)\n\tdefer srv2.Shutdown()\n\n\tcheckLeafNodeConnected(t, srv2)\n\n\tnc1 := natsConnect(t, srv1.ClientURL())\n\tdefer nc1.Close()\n\n\tsub := natsSubSync(t, nc1, \"foo\")\n\tcheckSubInterest(t, srv2, globalAccountName, \"foo\", time.Second)\n\n\t// We want to have the payload more than 126 bytes so that the websocket\n\t// code need to read 2 bytes for the length. See below.\n\tpayload := \"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\tsentBytes := []byte(\"LMSG foo 156\\r\\n\" + payload + \"\\r\\n\")\n\th, _ := wsCreateFrameHeader(false, false, wsBinaryMessage, len(sentBytes))\n\tcombined := &bytes.Buffer{}\n\tcombined.Write(h)\n\tcombined.Write(sentBytes)\n\ttoSend := combined.Bytes()\n\n\t// We will make a compressed block that cuts the websocket header that\n\t// makes the reader want to read bytes directly from the connection.\n\t// We want to make sure that we are not going to get compressed data\n\t// without going through the (de)compress library. So for that, compress\n\t// the first 3 bytes.\n\tb := &bytes.Buffer{}\n\tw := s2.NewWriter(b)\n\tw.Write(toSend[:3])\n\tw.Close()\n\n\tvar nc net.Conn\n\tsrv2.mu.RLock()\n\tfor _, l := range srv2.leafs {\n\t\tl.mu.Lock()\n\t\tnc = l.nc\n\t\tl.mu.Unlock()\n\t}\n\tsrv2.mu.RUnlock()\n\n\tnc.Write(b.Bytes())\n\n\t// Pause to make sure other side just gets a partial of the whole WS frame.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tb.Reset()\n\tw.Reset(b)\n\tw.Write(toSend[3:])\n\tw.Close()\n\n\tnc.Write(b.Bytes())\n\n\tmsg := natsNexMsg(t, sub, time.Second)\n\trequire_True(t, len(msg.Data) == 156)\n\trequire_Equal(t, string(msg.Data), payload)\n}\n\nfunc TestLeafNodeCompressionAuthTimeout(t *testing.T) {\n\thconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"hub\"\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\ttimeout: 0.75\n\t\t\t}\n\t\t}\n\t`))\n\tsh, oh := RunServerWithConfig(hconf)\n\tdefer sh.Shutdown()\n\n\tsconfTmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"spoke\"\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://127.0.0.1:%d\" }\n\t\t\t]\n\t\t}\n\t`\n\ts1conf := createConfFile(t, []byte(fmt.Sprintf(sconfTmpl, \"SP1\", _EMPTY_, oh.LeafNode.Port)))\n\ts1, o1 := RunServerWithConfig(s1conf)\n\tdefer s1.Shutdown()\n\n\ts2conf := createConfFile(t, []byte(fmt.Sprintf(sconfTmpl, \"SP2\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), oh.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(s2conf)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckLeafNodeConnected(t, s1)\n\tcheckLeafNodeConnected(t, s2)\n\n\tgetCID := func(s *Server) uint64 {\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\tvar cid uint64\n\t\tfor _, l := range s.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tcid = l.cid\n\t\t\tl.mu.Unlock()\n\t\t}\n\t\treturn cid\n\t}\n\tleaf1 := getCID(s1)\n\tleaf2 := getCID(s2)\n\n\t// Wait for more than auth timeout\n\ttime.Sleep(time.Second)\n\n\tcheckLeafNodeConnected(t, s1)\n\tcheckLeafNodeConnected(t, s2)\n\tif l1 := getCID(s1); l1 != leaf1 {\n\t\tt.Fatalf(\"Leaf connection first connection had CID %v, now %v\", leaf1, l1)\n\t}\n\tif l2 := getCID(s2); l2 != leaf2 {\n\t\tt.Fatalf(\"Leaf connection first connection had CID %v, now %v\", leaf2, l2)\n\t}\n}\n\nfunc TestLeafNodeWithWeightedDQRequestsToSuperClusterWithSeparateAccounts(t *testing.T) {\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, ONE and TWO.\n\tvar lnTmpl = `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t{{leaf}}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n\t\t`\n\n\tvar leafFrag = `\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tremotes [\n\t\t\t\t{ urls: [ %s ] }\n\t\t\t\t{ urls: [ %s ] }\n\t\t\t]\n\t\t}`\n\n\t// We want to have two leaf node connections that join to the same local account on the leafnode servers,\n\t// but connect to different accounts in different clusters.\n\tc1 := sc.clusters[0] // Will connect to account ONE\n\tc2 := sc.clusters[1] // Will connect to account TWO\n\n\tgenLeafTmpl := func(tmpl string) string {\n\t\tt.Helper()\n\n\t\tvar ln1, ln2 []string\n\t\tfor _, s := range c1.servers {\n\t\t\tif s.ClusterName() != c1.name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tln1 = append(ln1, fmt.Sprintf(\"nats://one:p@%s:%d\", ln.Host, ln.Port))\n\t\t}\n\n\t\tfor _, s := range c2.servers {\n\t\t\tif s.ClusterName() != c2.name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tln2 = append(ln2, fmt.Sprintf(\"nats://two:p@%s:%d\", ln.Host, ln.Port))\n\t\t}\n\t\treturn strings.Replace(tmpl, \"{{leaf}}\", fmt.Sprintf(leafFrag, strings.Join(ln1, \", \"), strings.Join(ln2, \", \")), 1)\n\t}\n\n\ttmpl := strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"SA\"), 1)\n\ttmpl = genLeafTmpl(tmpl)\n\n\tln := createJetStreamCluster(t, tmpl, \"SA\", \"SA-\", 3, 22280, false)\n\tln.waitOnClusterReady()\n\tdefer ln.shutdown()\n\n\tfor _, s := range ln.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 2)\n\t}\n\n\t// Now connect DQ subscribers to each cluster and they separate accounts, and make sure we get the right behavior, balanced between\n\t// them when requests originate from the leaf cluster.\n\n\t// Create 5 clients for each cluster / account\n\tvar c1c, c2c []*nats.Conn\n\tfor i := 0; i < 5; i++ {\n\t\tnc1, _ := jsClientConnect(t, c1.randomServer(), nats.UserInfo(\"one\", \"p\"))\n\t\tdefer nc1.Close()\n\t\tc1c = append(c1c, nc1)\n\t\tnc2, _ := jsClientConnect(t, c2.randomServer(), nats.UserInfo(\"two\", \"p\"))\n\t\tdefer nc2.Close()\n\t\tc2c = append(c2c, nc2)\n\t}\n\n\tcreateSubs := func(num int, conns []*nats.Conn) (subs []*nats.Subscription) {\n\t\tfor i := 0; i < num; i++ {\n\t\t\tnc := conns[rand.Intn(len(conns))]\n\t\t\tsub, err := nc.QueueSubscribeSync(\"REQUEST\", \"MC\")\n\t\t\trequire_NoError(t, err)\n\t\t\tsubs = append(subs, sub)\n\t\t\tnc.Flush()\n\t\t}\n\t\t// Let subs propagate.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\treturn subs\n\t}\n\tcloseSubs := func(subs []*nats.Subscription) {\n\t\tfor _, sub := range subs {\n\t\t\tsub.Unsubscribe()\n\t\t}\n\t}\n\n\t// Simple test first.\n\tsubs1 := createSubs(1, c1c)\n\tdefer closeSubs(subs1)\n\tsubs2 := createSubs(1, c2c)\n\tdefer closeSubs(subs2)\n\n\tsendRequests := func(num int) {\n\t\tt.Helper()\n\t\t// Now connect to the leaf cluster and send some requests.\n\t\tnc, _ := jsClientConnect(t, ln.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfor i := 0; i < num; i++ {\n\t\t\trequire_NoError(t, nc.Publish(\"REQUEST\", []byte(\"HELP\")))\n\t\t}\n\t\tnc.Flush()\n\t}\n\n\tpending := func(subs []*nats.Subscription) (total int) {\n\t\tt.Helper()\n\t\tfor _, sub := range subs {\n\t\t\tn, _, err := sub.Pending()\n\t\t\trequire_NoError(t, err)\n\t\t\ttotal += n\n\t\t}\n\t\treturn total\n\t}\n\n\tnum := 1000\n\tcheckAllReceived := func() error {\n\t\ttotal := pending(subs1) + pending(subs2)\n\t\tif total == num {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all received: %d vs %d\", total, num)\n\t}\n\n\tcheckBalanced := func(total, pc1, pc2 int) {\n\t\tt.Helper()\n\t\ttf := float64(total)\n\t\te1 := tf * (float64(pc1) / 100.00)\n\t\te2 := tf * (float64(pc2) / 100.00)\n\t\tdelta := tf / 10\n\t\tp1 := float64(pending(subs1))\n\t\tif p1 < e1-delta || p1 > e1+delta {\n\t\t\tt.Fatalf(\"Value out of range for subs1, expected %v got %v\", e1, p1)\n\t\t}\n\t\tp2 := float64(pending(subs2))\n\t\tif p2 < e2-delta || p2 > e2+delta {\n\t\t\tt.Fatalf(\"Value out of range for subs2, expected %v got %v\", e2, p2)\n\t\t}\n\t}\n\n\t// Now connect to the leaf cluster and send some requests.\n\n\t// Simple 50/50\n\tsendRequests(num)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllReceived)\n\tcheckBalanced(num, 50, 50)\n\n\tcloseSubs(subs1)\n\tcloseSubs(subs2)\n\n\t// Now test unbalanced. 10/90\n\tsubs1 = createSubs(1, c1c)\n\tdefer closeSubs(subs1)\n\tsubs2 = createSubs(9, c2c)\n\tdefer closeSubs(subs2)\n\n\tsendRequests(num)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllReceived)\n\tcheckBalanced(num, 10, 90)\n\n\t// Now test draining the subs as we are sending from an initial balanced situation simulating a draining of a cluster.\n\n\tcloseSubs(subs1)\n\tcloseSubs(subs2)\n\tsubs1, subs2 = nil, nil\n\n\t// These subs slightly different.\n\tvar r1, r2 atomic.Uint64\n\tfor i := 0; i < 20; i++ {\n\t\tnc := c1c[rand.Intn(len(c1c))]\n\t\tsub, err := nc.QueueSubscribe(\"REQUEST\", \"MC\", func(m *nats.Msg) { r1.Add(1) })\n\t\trequire_NoError(t, err)\n\t\tsubs1 = append(subs1, sub)\n\t\tnc.Flush()\n\n\t\tnc = c2c[rand.Intn(len(c2c))]\n\t\tsub, err = nc.QueueSubscribe(\"REQUEST\", \"MC\", func(m *nats.Msg) { r2.Add(1) })\n\t\trequire_NoError(t, err)\n\t\tsubs2 = append(subs2, sub)\n\t\tnc.Flush()\n\t}\n\tdefer closeSubs(subs1)\n\tdefer closeSubs(subs2)\n\n\tnc, _ := jsClientConnect(t, ln.randomServer())\n\tdefer nc.Close()\n\n\tfor i, dindex := 0, 1; i < num; i++ {\n\t\trequire_NoError(t, nc.Publish(\"REQUEST\", []byte(\"HELP\")))\n\t\t// Check if we have more to simulate draining.\n\t\t// Will drain within first ~100 requests using 20% rand test below.\n\t\t// Will leave 1 behind.\n\t\tif dindex < len(subs1)-1 && rand.Intn(6) > 4 {\n\t\t\tsub := subs1[dindex]\n\t\t\tdindex++\n\t\t\tsub.Drain()\n\t\t}\n\t}\n\tnc.Flush()\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\ttotal := int(r1.Load() + r2.Load())\n\t\tif total == num {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all received: %d vs %d\", total, num)\n\t})\n\trequire_True(t, r2.Load() > r1.Load())\n}\n\nfunc TestLeafNodeWithWeightedDQRequestsToSuperClusterWithStreamImportAccounts(t *testing.T) {\n\tvar tmpl = `\n\tlisten: 127.0.0.1:-1\n\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf { listen: 127.0.0.1:-1 }\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\tEFG {\n\t\t\tusers = [ { user: \"efg\", pass: \"p\" } ]\n\t\t\tjetstream: enabled\n\t\t\timports [\n\t\t\t\t{ stream: { account: STL, subject: \"REQUEST\"} }\n\t\t\t\t{ stream: { account: KSC, subject: \"REQUEST\"} }\n\t\t\t]\n\t\t\texports [ { stream: \"RESPONSE\" } ]\n\t\t}\n\t\tSTL {\n\t\t\tusers = [ { user: \"stl\", pass: \"p\" } ]\n\t\t\texports [ { stream: \"REQUEST\" } ]\n\t\t\timports [ { stream: { account: EFG, subject: \"RESPONSE\"} } ]\n\t\t}\n\t\tKSC {\n\t\t\tusers = [ { user: \"ksc\", pass: \"p\" } ]\n\t\t\texports [ { stream: \"REQUEST\" } ]\n\t\t\timports [ { stream: { account: EFG, subject: \"RESPONSE\"} } ]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}`\n\n\tsc := createJetStreamSuperClusterWithTemplate(t, tmpl, 5, 2)\n\tdefer sc.shutdown()\n\n\t// Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, STL and KSC.\n\tvar lnTmpl = `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t{{leaf}}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n\t\t`\n\n\tvar leafFrag = `\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tremotes [\n\t\t\t\t{ urls: [ %s ] }\n\t\t\t\t{ urls: [ %s ] }\n\t\t\t\t{ urls: [ %s ] ; deny_export: [REQUEST, RESPONSE], deny_import: RESPONSE }\n\t\t\t]\n\t\t}`\n\n\t// We want to have two leaf node connections that join to the same local account on the leafnode servers,\n\t// but connect to different accounts in different clusters.\n\tc1 := sc.clusters[0] // Will connect to account KSC\n\tc2 := sc.clusters[1] // Will connect to account STL\n\n\tgenLeafTmpl := func(tmpl string) string {\n\t\tt.Helper()\n\n\t\tvar ln1, ln2, ln3 []string\n\t\tfor _, s := range c1.servers {\n\t\t\tif s.ClusterName() != c1.name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tln1 = append(ln1, fmt.Sprintf(\"nats://ksc:p@%s:%d\", ln.Host, ln.Port))\n\t\t}\n\n\t\tfor _, s := range c2.servers {\n\t\t\tif s.ClusterName() != c2.name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tln := s.getOpts().LeafNode\n\t\t\tln2 = append(ln2, fmt.Sprintf(\"nats://stl:p@%s:%d\", ln.Host, ln.Port))\n\t\t\tln3 = append(ln3, fmt.Sprintf(\"nats://efg:p@%s:%d\", ln.Host, ln.Port))\n\t\t}\n\t\treturn strings.Replace(tmpl, \"{{leaf}}\", fmt.Sprintf(leafFrag, strings.Join(ln1, \", \"), strings.Join(ln2, \", \"), strings.Join(ln3, \", \")), 1)\n\t}\n\n\ttmpl = strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"SA\"), 1)\n\ttmpl = genLeafTmpl(tmpl)\n\n\tln := createJetStreamCluster(t, tmpl, \"SA\", \"SA-\", 3, 22280, false)\n\tln.waitOnClusterReady()\n\tdefer ln.shutdown()\n\n\tfor _, s := range ln.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 3)\n\t}\n\n\t// Now connect DQ subscribers to each cluster but to the global account.\n\n\t// Create 5 clients for each cluster / account\n\tvar c1c, c2c []*nats.Conn\n\tfor i := 0; i < 5; i++ {\n\t\tnc1, _ := jsClientConnect(t, c1.randomServer(), nats.UserInfo(\"efg\", \"p\"))\n\t\tdefer nc1.Close()\n\t\tc1c = append(c1c, nc1)\n\t\tnc2, _ := jsClientConnect(t, c2.randomServer(), nats.UserInfo(\"efg\", \"p\"))\n\t\tdefer nc2.Close()\n\t\tc2c = append(c2c, nc2)\n\t}\n\n\tcreateSubs := func(num int, conns []*nats.Conn) (subs []*nats.Subscription) {\n\t\tfor i := 0; i < num; i++ {\n\t\t\tnc := conns[rand.Intn(len(conns))]\n\t\t\tsub, err := nc.QueueSubscribeSync(\"REQUEST\", \"MC\")\n\t\t\trequire_NoError(t, err)\n\t\t\tsubs = append(subs, sub)\n\t\t\tnc.Flush()\n\t\t}\n\t\t// Let subs propagate.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\treturn subs\n\t}\n\tcloseSubs := func(subs []*nats.Subscription) {\n\t\tfor _, sub := range subs {\n\t\t\tsub.Unsubscribe()\n\t\t}\n\t}\n\n\t// Simple test first.\n\tsubs1 := createSubs(1, c1c)\n\tdefer closeSubs(subs1)\n\tsubs2 := createSubs(1, c2c)\n\tdefer closeSubs(subs2)\n\n\tsendRequests := func(num int) {\n\t\tt.Helper()\n\t\t// Now connect to the leaf cluster and send some requests.\n\t\tnc, _ := jsClientConnect(t, ln.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfor i := 0; i < num; i++ {\n\t\t\trequire_NoError(t, nc.Publish(\"REQUEST\", []byte(\"HELP\")))\n\t\t}\n\t\tnc.Flush()\n\t}\n\n\tpending := func(subs []*nats.Subscription) (total int) {\n\t\tt.Helper()\n\t\tfor _, sub := range subs {\n\t\t\tn, _, err := sub.Pending()\n\t\t\trequire_NoError(t, err)\n\t\t\ttotal += n\n\t\t}\n\t\treturn total\n\t}\n\n\tnum := 1000\n\tcheckAllReceived := func() error {\n\t\ttotal := pending(subs1) + pending(subs2)\n\t\tif total == num {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all received: %d vs %d\", total, num)\n\t}\n\n\tcheckBalanced := func(total, pc1, pc2 int) {\n\t\tt.Helper()\n\t\ttf := float64(total)\n\t\te1 := tf * (float64(pc1) / 100.00)\n\t\te2 := tf * (float64(pc2) / 100.00)\n\t\tdelta := tf / 10\n\t\tp1 := float64(pending(subs1))\n\t\tif p1 < e1-delta || p1 > e1+delta {\n\t\t\tt.Fatalf(\"Value out of range for subs1, expected %v got %v\", e1, p1)\n\t\t}\n\t\tp2 := float64(pending(subs2))\n\t\tif p2 < e2-delta || p2 > e2+delta {\n\t\t\tt.Fatalf(\"Value out of range for subs2, expected %v got %v\", e2, p2)\n\t\t}\n\t}\n\n\t// Now connect to the leaf cluster and send some requests.\n\n\t// Simple 50/50\n\tsendRequests(num)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllReceived)\n\tcheckBalanced(num, 50, 50)\n\n\tcloseSubs(subs1)\n\tcloseSubs(subs2)\n\n\t// Now test unbalanced. 10/90\n\tsubs1 = createSubs(1, c1c)\n\tdefer closeSubs(subs1)\n\tsubs2 = createSubs(9, c2c)\n\tdefer closeSubs(subs2)\n\n\tsendRequests(num)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllReceived)\n\tcheckBalanced(num, 10, 90)\n\n\tcloseSubs(subs1)\n\tcloseSubs(subs2)\n\n\t// Now test unbalanced. 80/20\n\tsubs1 = createSubs(80, c1c)\n\tdefer closeSubs(subs1)\n\tsubs2 = createSubs(20, c2c)\n\tdefer closeSubs(subs2)\n\n\tsendRequests(num)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllReceived)\n\tcheckBalanced(num, 80, 20)\n\n\t// Now test draining the subs as we are sending from an initial balanced situation simulating a draining of a cluster.\n\n\tcloseSubs(subs1)\n\tcloseSubs(subs2)\n\tsubs1, subs2 = nil, nil\n\n\t// These subs slightly different.\n\tvar r1, r2 atomic.Uint64\n\tfor i := 0; i < 20; i++ {\n\t\tnc := c1c[rand.Intn(len(c1c))]\n\t\tsub, err := nc.QueueSubscribe(\"REQUEST\", \"MC\", func(m *nats.Msg) { r1.Add(1) })\n\t\trequire_NoError(t, err)\n\t\tsubs1 = append(subs1, sub)\n\t\tnc.Flush()\n\n\t\tnc = c2c[rand.Intn(len(c2c))]\n\t\tsub, err = nc.QueueSubscribe(\"REQUEST\", \"MC\", func(m *nats.Msg) { r2.Add(1) })\n\t\trequire_NoError(t, err)\n\t\tsubs2 = append(subs2, sub)\n\t\tnc.Flush()\n\t}\n\t// Let's them propagate\n\ttime.Sleep(100 * time.Millisecond)\n\tdefer closeSubs(subs1)\n\tdefer closeSubs(subs2)\n\n\tnc, _ := jsClientConnect(t, ln.randomServer())\n\tdefer nc.Close()\n\n\tfor i, dindex := 0, 1; i < num; i++ {\n\t\trequire_NoError(t, nc.Publish(\"REQUEST\", []byte(\"HELP\")))\n\t\t// Check if we have more to simulate draining.\n\t\t// Will drain within first ~100 requests using 20% rand test below.\n\t\t// Will leave 1 behind.\n\t\tif dindex < len(subs1)-1 && rand.Intn(6) > 4 {\n\t\t\tsub := subs1[dindex]\n\t\t\tdindex++\n\t\t\tsub.Drain()\n\t\t}\n\t}\n\tnc.Flush()\n\n\t// Drain can lose some messages since it only checks pending messages known to the client,\n\t// and is not aware of other messages that are inflight on other servers.\n\tnumMin := num * 99 / 100\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\ttotal := int(r1.Load() + r2.Load())\n\t\tif total >= numMin {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all received: %d vs %d (minimum %d)\", total, num, numMin)\n\t})\n\trequire_True(t, r2.Load() > r1.Load())\n\n\t// Now check opposite flow for responses.\n\n\t// Create 10 subscribers.\n\tvar rsubs []*nats.Subscription\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc, _ := jsClientConnect(t, ln.randomServer())\n\t\tdefer nc.Close()\n\t\tsub, err := nc.QueueSubscribeSync(\"RESPONSE\", \"SA\")\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\trsubs = append(rsubs, sub)\n\t}\n\t// Let's them propagate\n\ttime.Sleep(100 * time.Millisecond)\n\n\tnc, _ = jsClientConnect(t, ln.randomServer())\n\tdefer nc.Close()\n\t_, err := nc.SubscribeSync(\"RESPONSE\")\n\trequire_NoError(t, err)\n\tnc.Flush()\n\n\t// Now connect and send responses from EFG in cloud.\n\tnc, _ = jsClientConnect(t, sc.randomServer(), nats.UserInfo(\"efg\", \"p\"))\n\tdefer nc.Close()\n\n\tfor i := 0; i < 100; i++ {\n\t\trequire_NoError(t, nc.Publish(\"RESPONSE\", []byte(\"OK\")))\n\t}\n\tnc.Flush()\n\n\tcheckAllRespReceived := func() error {\n\t\tp := pending(rsubs)\n\t\tif p == 100 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all responses received: %d vs %d\", p, 100)\n\t}\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived)\n}\n\nfunc TestLeafNodeWithWeightedDQResponsesWithStreamImportAccountsWithUnsub(t *testing.T) {\n\tvar tmpl = `\n\tlisten: 127.0.0.1:-1\n\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tleaf { listen: 127.0.0.1:-1 }\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\tEFG {\n\t\t\tusers = [ { user: \"efg\", pass: \"p\" } ]\n\t\t\tjetstream: enabled\n\t\t\texports [ { stream: \"RESPONSE\" } ]\n\t\t}\n\t\tSTL {\n\t\t\tusers = [ { user: \"stl\", pass: \"p\" } ]\n\t\t\timports [ { stream: { account: EFG, subject: \"RESPONSE\"} } ]\n\t\t}\n\t\tKSC {\n\t\t\tusers = [ { user: \"ksc\", pass: \"p\" } ]\n\t\t\timports [ { stream: { account: EFG, subject: \"RESPONSE\"} } ]\n\t\t}\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}`\n\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"US-CENTRAL\", 3)\n\tdefer c.shutdown()\n\n\t// Now create a leafnode cluster that has 2 LNs, one to each cluster but on separate accounts, STL and KSC.\n\tvar lnTmpl = `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t{{leaf}}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }}\n\t\t`\n\n\tvar leafFrag = `\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tremotes [ { urls: [ %s ] } ]\n\t\t}`\n\n\tgenLeafTmpl := func(tmpl string) string {\n\t\tt.Helper()\n\n\t\tvar ln []string\n\t\tfor _, s := range c.servers {\n\t\t\tlno := s.getOpts().LeafNode\n\t\t\tln = append(ln, fmt.Sprintf(\"nats://ksc:p@%s:%d\", lno.Host, lno.Port))\n\t\t}\n\t\treturn strings.Replace(tmpl, \"{{leaf}}\", fmt.Sprintf(leafFrag, strings.Join(ln, \", \")), 1)\n\t}\n\n\ttmpl = strings.Replace(lnTmpl, \"store_dir:\", fmt.Sprintf(`domain: \"%s\", store_dir:`, \"SA\"), 1)\n\ttmpl = genLeafTmpl(tmpl)\n\n\tln := createJetStreamCluster(t, tmpl, \"SA\", \"SA-\", 3, 22280, false)\n\tln.waitOnClusterReady()\n\tdefer ln.shutdown()\n\n\tfor _, s := range ln.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 1)\n\t}\n\n\t// Create 10 subscribers.\n\tvar rsubs []*nats.Subscription\n\n\tcloseSubs := func(subs []*nats.Subscription) {\n\t\tfor _, sub := range subs {\n\t\t\tsub.Unsubscribe()\n\t\t}\n\t}\n\n\tcheckAllRespReceived := func() error {\n\t\tt.Helper()\n\t\tvar total int\n\t\tfor _, sub := range rsubs {\n\t\t\tn, _, err := sub.Pending()\n\t\t\trequire_NoError(t, err)\n\t\t\ttotal += n\n\t\t}\n\t\tif total == 100 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not all responses received: %d vs %d\", total, 100)\n\t}\n\n\ts := ln.randomServer()\n\tfor i := 0; i < 4; i++ {\n\t\tnc, _ := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tsub, err := nc.QueueSubscribeSync(\"RESPONSE\", \"SA\")\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\trsubs = append(rsubs, sub)\n\t}\n\n\t// Now connect and send responses from EFG in cloud.\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"efg\", \"p\"))\n\tdefer nc.Close()\n\tfor i := 0; i < 100; i++ {\n\t\trequire_NoError(t, nc.Publish(\"RESPONSE\", []byte(\"OK\")))\n\t}\n\tnc.Flush()\n\n\t// Make sure all received.\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived)\n\n\tcheckAccountInterest := func(s *Server, accName string) *SublistResult {\n\t\tt.Helper()\n\t\tacc, err := s.LookupAccount(accName)\n\t\trequire_NoError(t, err)\n\t\tacc.mu.RLock()\n\t\tr := acc.sl.Match(\"RESPONSE\")\n\t\tacc.mu.RUnlock()\n\t\treturn r\n\t}\n\n\tcheckInterest := func() error {\n\t\tt.Helper()\n\t\tfor _, s := range c.servers {\n\t\t\tif r := checkAccountInterest(s, \"KSC\"); len(r.psubs)+len(r.qsubs) > 0 {\n\t\t\t\treturn fmt.Errorf(\"Subs still present for %q: %+v\", \"KSC\", r)\n\t\t\t}\n\t\t\tif r := checkAccountInterest(s, \"EFG\"); len(r.psubs)+len(r.qsubs) > 0 {\n\t\t\t\treturn fmt.Errorf(\"Subs still present for %q: %+v\", \"EFG\", r)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Now unsub them and create new ones on a different server.\n\tcloseSubs(rsubs)\n\trsubs = rsubs[:0]\n\n\t// Also restart the server that we had all the rsubs on.\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\ts = ln.restartServer(s)\n\tln.waitOnClusterReady()\n\tln.waitOnServerCurrent(s)\n\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkInterest)\n\n\tfor i := 0; i < 4; i++ {\n\t\tnc, _ := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tsub, err := nc.QueueSubscribeSync(\"RESPONSE\", \"SA\")\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t\trsubs = append(rsubs, sub)\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\trequire_NoError(t, nc.Publish(\"RESPONSE\", []byte(\"OK\")))\n\t}\n\tnc.Flush()\n\n\t// Make sure all received.\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkAllRespReceived)\n\n\tcloseSubs(rsubs)\n\tcheckFor(t, time.Second, 200*time.Millisecond, checkInterest)\n}\n\nfunc TestLeafNodeTwoRemotesToSameHubAccount(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"hub\"\n\t\taccounts {\n\t\t\tHA { users: [{user: ha, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"spoke\"\n\t\taccounts {\n\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\tB { users: [{user: B, password: pwd}] }\n\t\t\tC { users: [{user: C, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://ha:pwd@127.0.0.1:%d\"\n\t\t\t\t\tlocal: \"A\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://ha:pwd@127.0.0.1:%d\"\n\t\t\t\t\tlocal: \"C\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.LeafNode.Port, o1.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts2.SetLogger(l, false, false)\n\n\tcheckLeafNodeConnectedCount(t, s2, 2)\n\n\t// Make sure we don't get duplicate leafnode connection errors\n\tdeadline := time.NewTimer(1500 * time.Millisecond)\n\tfor done := false; !done; {\n\t\tselect {\n\t\tcase err := <-l.errCh:\n\t\t\tif strings.Contains(err, DuplicateRemoteLeafnodeConnection.String()) {\n\t\t\t\tt.Fatalf(\"Got error: %v\", err)\n\t\t\t}\n\t\tcase <-deadline.C:\n\t\t\tdone = true\n\t\t}\n\t}\n\n\tnca := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\tdefer nca.Close()\n\tsuba := natsSubSync(t, nca, \"A\")\n\tncb := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"B\", \"pwd\"))\n\tdefer ncb.Close()\n\tsubb := natsSubSync(t, ncb, \"B\")\n\tncc := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"C\", \"pwd\"))\n\tdefer ncc.Close()\n\tsubc := natsSubSync(t, ncc, \"C\")\n\tsubs := map[string]*nats.Subscription{\"A\": suba, \"B\": subb, \"C\": subc}\n\n\tfor _, subj := range []string{\"A\", \"C\"} {\n\t\tcheckSubInterest(t, s1, \"HA\", subj, time.Second)\n\t}\n\n\tnc := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"ha\", \"pwd\"))\n\tdefer nc.Close()\n\n\tfor _, subj := range []string{\"A\", \"B\", \"C\"} {\n\t\tnatsPub(t, nc, subj, []byte(\"hello\"))\n\t}\n\n\tfor _, subj := range []string{\"A\", \"B\", \"C\"} {\n\t\tvar expected bool\n\t\tif subj != \"B\" {\n\t\t\texpected = true\n\t\t}\n\t\tsub := subs[subj]\n\t\tif expected {\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t} else {\n\t\t\tif _, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected timeout error, got %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeTwoRemotesToSameHubAccountWithClusters(t *testing.T) {\n\thubTmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tHA { users: [{user: HA, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"hub\"\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`\n\tconfH1 := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, \"H1\", _EMPTY_)))\n\tsh1, oh1 := RunServerWithConfig(confH1)\n\tdefer sh1.Shutdown()\n\n\tconfH2 := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, \"H2\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", oh1.Cluster.Port))))\n\tsh2, oh2 := RunServerWithConfig(confH2)\n\tdefer sh2.Shutdown()\n\n\tcheckClusterFormed(t, sh1, sh2)\n\n\tspokeTmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\tB { users: [{user: B, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"spoke\"\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://HA:pwd@127.0.0.1:%d\"\n\t\t\t\t\tlocal: \"A\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://HA:pwd@127.0.0.1:%d\"\n\t\t\t\t\tlocal: \"B\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsp2Leafport int\n\t}{\n\t\t{\"connect to different hub servers\", oh2.LeafNode.Port},\n\t\t{\"connect to same hub server\", oh1.LeafNode.Port},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconfSP1 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP1\", _EMPTY_, oh1.LeafNode.Port, oh1.LeafNode.Port)))\n\t\t\tsp1, osp1 := RunServerWithConfig(confSP1)\n\t\t\tdefer sp1.Shutdown()\n\n\t\t\tconfSP2 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP2\",\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", osp1.Cluster.Port), test.sp2Leafport, test.sp2Leafport)))\n\t\t\tsp2, _ := RunServerWithConfig(confSP2)\n\t\t\tdefer sp2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sp1, sp2)\n\t\t\tcheckLeafNodeConnectedCount(t, sp1, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sp2, 2)\n\n\t\t\tvar conns []*nats.Conn\n\t\t\tcreateConn := func(s *Server, user string) {\n\t\t\t\tt.Helper()\n\t\t\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\t\t\tconns = append(conns, nc)\n\t\t\t}\n\t\t\tcreateConn(sh1, \"HA\")\n\t\t\tcreateConn(sh2, \"HA\")\n\t\t\tcreateConn(sp1, \"A\")\n\t\t\tcreateConn(sp2, \"A\")\n\t\t\tcreateConn(sp1, \"B\")\n\t\t\tcreateConn(sp2, \"B\")\n\t\t\tfor _, nc := range conns {\n\t\t\t\tdefer nc.Close()\n\t\t\t}\n\n\t\t\tcheck := func(subConn *nats.Conn, subj string, checkA, checkB bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tsub := natsSubSync(t, subConn, subj)\n\t\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t\tcheckSubInterest(t, sh1, \"HA\", subj, time.Second)\n\t\t\t\tcheckSubInterest(t, sh2, \"HA\", subj, time.Second)\n\t\t\t\tif checkA {\n\t\t\t\t\tcheckSubInterest(t, sp1, \"A\", subj, time.Second)\n\t\t\t\t\tcheckSubInterest(t, sp2, \"A\", subj, time.Second)\n\t\t\t\t}\n\t\t\t\tif checkB {\n\t\t\t\t\tcheckSubInterest(t, sp1, \"B\", subj, time.Second)\n\t\t\t\t\tcheckSubInterest(t, sp2, \"B\", subj, time.Second)\n\t\t\t\t}\n\n\t\t\t\tfor i, ncp := range conns {\n\t\t\t\t\t// Don't publish from account \"A\" connections if we are\n\t\t\t\t\t// dealing with account \"B\", and vice-versa.\n\t\t\t\t\tif !checkA && i >= 2 && i <= 3 {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif !checkB && i >= 4 {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tnatsPub(t, ncp, subj, []byte(\"hello\"))\n\t\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t\t\t\t// Make sure we don't get a duplicate\n\t\t\t\t\tif msg, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected message or error: msg=%v - err=%v\", msg, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck(conns[0], \"HA.1\", true, true)\n\t\t\tcheck(conns[1], \"HA.2\", true, true)\n\t\t\tcheck(conns[2], \"SPA.1\", true, false)\n\t\t\tcheck(conns[3], \"SPA.2\", true, false)\n\t\t\tcheck(conns[4], \"SPB.1\", false, true)\n\t\t\tcheck(conns[5], \"SPB.2\", false, true)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeSameLocalAccountToMultipleHubs(t *testing.T) {\n\thub1Conf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: hub1\n\t\taccounts {\n\t\t\thub1 { users: [{user: hub1, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\tsh1, oh1 := RunServerWithConfig(hub1Conf)\n\tdefer sh1.Shutdown()\n\n\thub2Conf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: hub2\n\t\taccounts {\n\t\t\thub2 { users: [{user: hub2, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\tsh2, oh2 := RunServerWithConfig(hub2Conf)\n\tdefer sh2.Shutdown()\n\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: leaf\n\t\taccounts {\n\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\tB { users: [{user: B, password: pwd}] }\n\t\t\tC { users: [{user: C, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: nats://hub1:pwd@127.0.0.1:%[1]d\n\t\t\t\t\tlocal: \"A\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: nats://hub1:pwd@127.0.0.1:%[1]d\n\t\t\t\t\tlocal: \"C\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: nats://hub2:pwd@127.0.0.1:%[2]d\n\t\t\t\t\tlocal: \"A\"\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: nats://hub2:pwd@127.0.0.1:%[2]d\n\t\t\t\t\tlocal: \"B\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, oh1.LeafNode.Port, oh2.LeafNode.Port)))\n\ts, _ := RunServerWithConfig(lconf)\n\tdefer s.Shutdown()\n\n\t// The leafnode to hub1 should have 2 connections (A and C)\n\t// while the one to hub2 should have 2 connections (A and B)\n\tcheckLeafNodeConnectedCount(t, sh1, 2)\n\tcheckLeafNodeConnectedCount(t, sh2, 2)\n\tcheckLeafNodeConnectedCount(t, s, 4)\n\n\tnca := natsConnect(t, s.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\tdefer nca.Close()\n\tsuba := natsSubSync(t, nca, \"A\")\n\tncb := natsConnect(t, s.ClientURL(), nats.UserInfo(\"B\", \"pwd\"))\n\tdefer ncb.Close()\n\tsubb := natsSubSync(t, ncb, \"B\")\n\tncc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"C\", \"pwd\"))\n\tdefer ncc.Close()\n\tsubc := natsSubSync(t, ncc, \"C\")\n\n\tcheckSubInterest(t, sh1, \"hub1\", \"A\", time.Second)\n\tcheckSubNoInterest(t, sh1, \"hub1\", \"B\", time.Second)\n\tcheckSubInterest(t, sh1, \"hub1\", \"C\", time.Second)\n\n\tcheckSubInterest(t, sh2, \"hub2\", \"A\", time.Second)\n\tcheckSubInterest(t, sh2, \"hub2\", \"B\", time.Second)\n\tcheckSubNoInterest(t, sh2, \"hub2\", \"C\", time.Second)\n\n\tnch1 := natsConnect(t, sh1.ClientURL(), nats.UserInfo(\"hub1\", \"pwd\"))\n\tdefer nch1.Close()\n\tnch2 := natsConnect(t, sh2.ClientURL(), nats.UserInfo(\"hub2\", \"pwd\"))\n\tdefer nch2.Close()\n\n\tcheckNoMsg := func(sub *nats.Subscription) {\n\t\tt.Helper()\n\t\tif msg, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Unexpected message: %s\", msg.Data)\n\t\t}\n\t}\n\n\tcheckSub := func(sub *nats.Subscription, subj, payload string) {\n\t\tt.Helper()\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\trequire_Equal(t, subj, msg.Subject)\n\t\trequire_Equal(t, payload, string(msg.Data))\n\t\t// Make sure we don't get duplicates\n\t\tcheckNoMsg(sub)\n\t}\n\n\tnatsPub(t, nch1, \"A\", []byte(\"msgA1\"))\n\tcheckSub(suba, \"A\", \"msgA1\")\n\tnatsPub(t, nch1, \"B\", []byte(\"msgB1\"))\n\tcheckNoMsg(subb)\n\tnatsPub(t, nch1, \"C\", []byte(\"msgC1\"))\n\tcheckSub(subc, \"C\", \"msgC1\")\n\n\tnatsPub(t, nch2, \"A\", []byte(\"msgA2\"))\n\tcheckSub(suba, \"A\", \"msgA2\")\n\tnatsPub(t, nch2, \"B\", []byte(\"msgB2\"))\n\tcheckSub(subb, \"B\", \"msgB2\")\n\tnatsPub(t, nch2, \"C\", []byte(\"msgC2\"))\n\tcheckNoMsg(subc)\n}\n\nfunc TestLeafNodeSlowConsumer(t *testing.T) {\n\tao := DefaultOptions()\n\tao.LeafNode.Host = \"127.0.0.1\"\n\tao.LeafNode.Port = -1\n\tao.WriteDeadline = 1 * time.Millisecond\n\ta := RunServer(ao)\n\tdefer a.Shutdown()\n\n\tc, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", ao.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\ttime.Sleep(5 * time.Millisecond)\n\ta.mu.Lock()\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\ta.grMu.Lock()\n\t\tdefer a.grMu.Unlock()\n\t\tfor _, cli := range a.grTmpClients {\n\t\t\tcli.out.wdl = time.Nanosecond\n\t\t\treturn nil\n\t\t}\n\t\treturn nil\n\t})\n\ta.mu.Unlock()\n\n\t// Only leafnode slow consumers that made it past connect are tracked\n\t// in the slow consumers counter.\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error writing connect: %v\", err)\n\t}\n\t// Read info\n\tbr := bufio.NewReader(c)\n\tbr.ReadLine()\n\tfor i := 0; i < 10; i++ {\n\t\tif _, err := c.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error writing PING: %v\", err)\n\t\t}\n\t}\n\tdefer c.Close()\n\ttimeout := time.Now().Add(time.Second)\n\tvar (\n\t\tgot uint64\n\t\texpected uint64 = 1\n\t)\n\tfor time.Now().Before(timeout) {\n\t\tgot = a.NumSlowConsumersLeafs()\n\t\tif got == expected {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(1 * time.Millisecond)\n\t}\n\tt.Fatalf(\"Timed out waiting for slow consumer leafnodes, got: %v, expected: %v\", got, expected)\n}\n\n// https://github.com/nats-io/nats-server/issues/4367\nfunc TestLeafNodeDQMultiAccountExportImport(t *testing.T) {\n\tbConf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: cluster-b-0\n\t\taccounts {\n\t\t\t$SYS: { users: [ { user: admin, password: pwd } ] },\n\t\t\tAGG: {\n\t\t\t\texports: [ { service: \"PING.>\" } ]\n\t\t\t\tusers: [ { user: agg, password: agg } ]\n\t\t\t}\n\t\t}\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t`))\n\n\tsb, ob := RunServerWithConfig(bConf)\n\tdefer sb.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { store_dir: '%s' }\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\t\taccounts {\n\t\t\t$SYS: { users: [ { user: admin, password: pwd } ] },\n\t\t\tA: {\n\t\t\t\tmappings: { \"A.>\" : \">\" }\n\t\t\t\texports: [ { service: A.> } ]\n\t\t\t\tusers: [ { user: a, password: a } ]\n\t\t\t},\n\t\t\tAGG: {\n\t\t\t\timports: [ { service: { subject: A.>, account: A } } ]\n\t\t\t\tusers: [ { user: agg, password: agg } ]\n\t\t\t},\n\t\t}\n\t\tleaf {\n\t\t\tremotes: [ {\n\t\t\t\turls: [ nats-leaf://agg:agg@127.0.0.1:{LEAF_PORT} ]\n\t\t\t\taccount: AGG\n\t\t\t} ]\n\t\t}\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", ob.LeafNode.Port), 1)\n\tc := createJetStreamCluster(t, tmpl, \"cluster-a\", \"cluster-a-\", 3, 22110, false)\n\tdefer c.shutdown()\n\n\t// Make sure all servers are connected via leafnode to the hub, the b server.\n\tfor _, s := range c.servers {\n\t\tcheckLeafNodeConnectedCount(t, s, 1)\n\t}\n\n\t// Connect to a server in the cluster and create a DQ listener.\n\tnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"a\", \"a\"))\n\tdefer nc.Close()\n\n\tvar got atomic.Int32\n\n\tnatsQueueSub(t, nc, \"PING\", \"Q\", func(m *nats.Msg) {\n\t\tgot.Add(1)\n\t\tm.Respond([]byte(\"REPLY\"))\n\t})\n\n\t// Now connect to B and send the request.\n\tncb, _ := jsClientConnect(t, sb, nats.UserInfo(\"agg\", \"agg\"))\n\tdefer ncb.Close()\n\n\t_, err := ncb.Request(\"A.PING\", []byte(\"REQUEST\"), time.Second)\n\trequire_NoError(t, err)\n\trequire_Equal(t, got.Load(), 1)\n}\n\n// https://github.com/nats-io/nats-server/issues/4934\nfunc TestLeafNodeServerReloadSubjectMappings(t *testing.T) {\n\tstmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-server\n\t\tmappings = { \"source1\": \"target\" }\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t`\n\tconf := createConfFile(t, []byte(stmpl))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-leaf\n\t\tleaf {\n\t\t\tremotes: [ {\n\t\t\t\turls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ]\n\t\t\t} ]\n\t\t}\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", o.LeafNode.Port), 1)\n\tlConf := createConfFile(t, []byte(tmpl))\n\tl, _ := RunServerWithConfig(lConf)\n\tdefer l.Shutdown()\n\n\tcheckLeafNodeConnected(t, l)\n\n\t// Create our subscriber.\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tsub := natsSubSync(t, nc, \"target\")\n\tnatsFlush(t, nc)\n\n\t// Create our publisher.\n\tncl := natsConnect(t, l.ClientURL())\n\tdefer ncl.Close()\n\t// Publish our message.\n\tncl.Publish(\"source1\", []byte(\"OK\"))\n\n\t// Make sure we receive it.\n\tcheckSubsPending(t, sub, 1)\n\n\t// Now change mapping.\n\treloadUpdateConfig(t, s, conf, strings.Replace(stmpl, \"source1\", \"source2\", 1))\n\t// Also make sure we do not have subscription interest for source1 on leaf anymore.\n\tcheckSubInterest(t, l, globalAccountName, \"source2\", 2*time.Second)\n\n\t// Publish our new message.\n\tncl.Publish(\"source2\", []byte(\"OK\"))\n\n\t// Make sure we receive it.\n\tcheckSubsPending(t, sub, 2)\n\n\t// Also make sure we do not have subscription interest for source1 on leaf anymore.\n\tcheckSubNoInterest(t, l, globalAccountName, \"source1\", 2*time.Second)\n}\n\n// https://github.com/nats-io/nats-server/issues/5099\nfunc TestLeafNodeServerReloadSubjectMappingsWithSameSubject(t *testing.T) {\n\tstmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-server\n\t\tmappings = { \"source\": \"target1\" }\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t`\n\tconf := createConfFile(t, []byte(stmpl))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-leaf\n\t\tleaf {\n\t\t\tremotes: [ { urls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ] } ]\n\t\t}\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", o.LeafNode.Port), 1)\n\tlConf := createConfFile(t, []byte(tmpl))\n\tl, _ := RunServerWithConfig(lConf)\n\tdefer l.Shutdown()\n\n\tcheckLeafNodeConnected(t, l)\n\n\t// Create our subscriber.\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tsub1 := natsSubSync(t, nc, \"target1\")\n\tsub2 := natsSubSync(t, nc, \"target2\")\n\tnatsFlush(t, nc)\n\n\t// Create our publisher.\n\tncl := natsConnect(t, l.ClientURL())\n\tdefer ncl.Close()\n\t// Publish our message.\n\tncl.Publish(\"source\", []byte(\"OK\"))\n\n\t// Make sure we receive it.\n\tcheckSubsPending(t, sub1, 1)\n\t// Make sure the other does not.\n\tcheckSubsPending(t, sub2, 0)\n\n\t// Now change mapping, but only the \"to\" subject, keeping same \"from\"\n\treloadUpdateConfig(t, s, conf, strings.Replace(stmpl, \"target1\", \"target2\", 1))\n\tcheckLeafNodeConnected(t, l)\n\n\t// Publish our new message.\n\tncl.Publish(\"source\", []byte(\"OK\"))\n\n\t// Make sure we receive it.\n\tcheckSubsPending(t, sub2, 1)\n\t// Make sure the other does not.\n\tcheckSubsPending(t, sub1, 1)\n}\n\nfunc TestLeafNodeNkeyAuth(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-server\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tauthorization: { nkey: UCSTG5CRF5GEJERAFKUUYRODGABTBVWY2NPE4GGKRQVQOH74PIAKTVKO }\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-leaf\n\t\tleaf {\n\t\t\tremotes: [ {\n\t\t\t\turl: nats-leaf://127.0.0.1:{LEAF_PORT}\n\t\t\t\tseed: SUACJN3OSKWWPQXME4JUNFJ3PARXPO657GGNWNU7PK7G3AUQQYHLW26XH4\n\t\t\t} ]\n\t\t}\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", o.LeafNode.Port), 1)\n\tlConf := createConfFile(t, []byte(tmpl))\n\tl, _ := RunServerWithConfig(lConf)\n\tdefer l.Shutdown()\n\n\tcheckLeafNodeConnected(t, l)\n}\n\nfunc TestLeafNodeAccountNkeysAuth(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-server\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\taccounts {\n A { users [ {nkey: UCSTG5CRF5GEJERAFKUUYRODGABTBVWY2NPE4GGKRQVQOH74PIAKTVKO } ] }\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-leaf\n\t\tleaf {\n\t\t\tremotes: [ {\n\t\t\t\turl: nats-leaf://127.0.0.1:{LEAF_PORT}\n\t\t\t\tseed: SUACJN3OSKWWPQXME4JUNFJ3PARXPO657GGNWNU7PK7G3AUQQYHLW26XH4\n\t\t\t} ]\n\t\t}\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", o.LeafNode.Port), 1)\n\tlConf := createConfFile(t, []byte(tmpl))\n\tl, _ := RunServerWithConfig(lConf)\n\tdefer l.Shutdown()\n\n\tcheckLeafNodeConnected(t, l)\n}\n\n// https://github.com/nats-io/nats-server/issues/5117\nfunc TestLeafNodeLoopDetectionOnActualLoop(t *testing.T) {\n\t// Setup: B --[leaf]--> A C --[leaf]--> A C --[leaf] --> B\n\taccConf := `\n\t\taccounts: {\n\t\t\tAPP: {\n\t\t\t\tusers: [ { user:u, password: u,\n\t\t\t\t\tpermissions: { publish = \"u.>\", subscribe = \"u.>\" }} ]\n\t\t\t}\n\t\t\t$SYS: { users = [ {user: \"s\", password: \"s\"} ] }\n\t\t}`\n\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: a1\n\t\tport: -1\n\t\tcluster: { name: A }\n\t\tleafnodes {\n\t\t\tport: 17422\n\t\t}\n\t\t%s`, accConf)))\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: b1\n\t\tport: -1\n\t\tcluster: { name: B }\n\t\tleafnodes {\n\t\t\tport: 17432\n\t\t\tremotes [\n\t\t\t\t{ urls: [\"nats-leaf://u:u@localhost:17422\"], account: \"APP\" }\n\t\t\t]\n\t\t\treconnect: \"2s\"\n\t\t}\n\t\t%s`, accConf)))\n\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: c1\n\t\tport: -1\n\t\tcluster: { name: C }\n\t\tleafnodes {\n\t\t\tport: 17442\n\t\t\tremotes [\n\t\t\t\t{ urls: [\"nats-leaf://u:u@localhost:17422\"], account: \"APP\" }\n\t\t\t\t# This one creates the loop\n\t\t\t\t{ urls: [\"nats-leaf://u:u@localhost:17432\"], account: \"APP\" }\n\t\t\t]\n\t\t\treconnect: \"0.5s\"\n\t\t}\n\t\t%s`, accConf)))\n\n\t// Start order will be B -> C -> A\n\t// We will force C to connect to A first before B using different reconnect intervals.\n\t// If B connects first we detect loops fine. If C connects first we do not.\n\n\tsrvB, _ := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\tlb := &loopDetectedLogger{ch: make(chan string, 1)}\n\tsrvB.SetLogger(lb, false, false)\n\n\tsrvC, _ := RunServerWithConfig(confC)\n\tdefer srvC.Shutdown()\n\tlc := &loopDetectedLogger{ch: make(chan string, 1)}\n\tsrvC.SetLogger(lc, false, false)\n\n\t// C should connect to B\n\tcheckLeafNodeConnectedCount(t, srvC, 1)\n\n\tsrvA, _ := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\tla := &loopDetectedLogger{ch: make(chan string, 1)}\n\tsrvA.SetLogger(la, false, false)\n\n\tselect {\n\tcase <-la.ch:\n\tcase <-lb.ch:\n\tcase <-lc.ch:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not get any error regarding loop\")\n\t}\n}\n\nfunc TestLeafNodeConnectionSucceedsEvenWithDelayedFirstINFO(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\twebsocket bool\n\t}{\n\t\t{\"regular\", false},\n\t\t{\"websocket\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tob := DefaultOptions()\n\t\t\tob.ServerName = \"HUB\"\n\t\t\tob.LeafNode.Host = \"127.0.0.1\"\n\t\t\tob.LeafNode.Port = -1\n\t\t\tob.LeafNode.AuthTimeout = 10\n\t\t\tif test.websocket {\n\t\t\t\tob.Websocket.Host = \"127.0.0.1\"\n\t\t\t\tob.Websocket.Port = -1\n\t\t\t\tob.Websocket.HandshakeTimeout = 10 * time.Second\n\t\t\t\tob.Websocket.AuthTimeout = 10\n\t\t\t\tob.Websocket.NoTLS = true\n\t\t\t}\n\t\t\tsb := RunServer(ob)\n\t\t\tdefer sb.Shutdown()\n\n\t\t\tvar port int\n\t\t\tvar scheme string\n\t\t\tif test.websocket {\n\t\t\t\tport = ob.Websocket.Port\n\t\t\t\tscheme = wsSchemePrefix\n\t\t\t} else {\n\t\t\t\tport = ob.LeafNode.Port\n\t\t\t\tscheme = \"nats\"\n\t\t\t}\n\n\t\t\turlStr := fmt.Sprintf(\"%s://127.0.0.1:%d\", scheme, port)\n\t\t\tproxy := createNetProxy(1100*time.Millisecond, 1024*1024*1024, 1024*1024*1024, urlStr, true)\n\t\t\tdefer proxy.stop()\n\t\t\tproxyURL := proxy.clientURL()\n\t\t\t_, proxyPort, err := net.SplitHostPort(proxyURL[len(scheme)+3:])\n\t\t\trequire_NoError(t, err)\n\n\t\t\tlnBURL, err := url.Parse(fmt.Sprintf(\"%s://127.0.0.1:%s\", scheme, proxyPort))\n\t\t\trequire_NoError(t, err)\n\n\t\t\toa := DefaultOptions()\n\t\t\toa.ServerName = \"SPOKE\"\n\t\t\toa.Cluster.Name = \"xyz\"\n\t\t\tremote := &RemoteLeafOpts{\n\t\t\t\tURLs: []*url.URL{lnBURL},\n\t\t\t\tFirstInfoTimeout: 3 * time.Second,\n\t\t\t}\n\t\t\toa.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\t\t\tsa := RunServer(oa)\n\t\t\tdefer sa.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, sa)\n\t\t})\n\t}\n}\n\ntype captureLeafConnClosed struct {\n\tDummyLogger\n\tch chan struct{}\n}\n\nfunc (l *captureLeafConnClosed) Noticef(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"Leafnode connection closed: Read Error\") {\n\t\tselect {\n\t\tcase l.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeDetectsStaleConnectionIfNoInfo(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\twebsocket bool\n\t}{\n\t\t{\"regular\", false},\n\t\t{\"websocket\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer l.Close()\n\n\t\t\tch := make(chan struct{})\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tc, err := l.Accept()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdefer c.Close()\n\t\t\t\t<-ch\n\t\t\t}()\n\n\t\t\tvar scheme string\n\t\t\tif test.websocket {\n\t\t\t\tscheme = wsSchemePrefix\n\t\t\t} else {\n\t\t\t\tscheme = \"nats\"\n\t\t\t}\n\t\t\turlStr := fmt.Sprintf(\"%s://%s\", scheme, l.Addr())\n\t\t\tlnBURL, err := url.Parse(urlStr)\n\t\t\trequire_NoError(t, err)\n\n\t\t\toa := DefaultOptions()\n\t\t\toa.ServerName = \"SPOKE\"\n\t\t\toa.Cluster.Name = \"xyz\"\n\t\t\tremote := &RemoteLeafOpts{\n\t\t\t\tURLs: []*url.URL{lnBURL},\n\t\t\t\tFirstInfoTimeout: 250 * time.Millisecond,\n\t\t\t}\n\t\t\toa.LeafNode.Remotes = []*RemoteLeafOpts{remote}\n\t\t\toa.DisableShortFirstPing = false\n\t\t\toa.NoLog = false\n\t\t\tsa, err := NewServer(oa)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sa.Shutdown()\n\n\t\t\tlog := &captureLeafConnClosed{ch: make(chan struct{}, 1)}\n\t\t\tsa.SetLogger(log, false, false)\n\t\t\tsa.Start()\n\n\t\t\tselect {\n\t\t\tcase <-log.ch:\n\t\t\t\t// OK\n\t\t\tcase <-time.After(750 * time.Millisecond):\n\t\t\t\tt.Fatalf(\"Connection was not closed\")\n\t\t\t}\n\n\t\t\tsa.Shutdown()\n\t\t\tclose(ch)\n\t\t\twg.Wait()\n\t\t\tsa.WaitForShutdown()\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/5473\nfunc TestLeafNodeDupeDeliveryQueueSubAndPlainSub(t *testing.T) {\n\tclusterCommonConf := `\n\t\taccounts: {\n\t\t\ttenant: {\n\t\t\t\tusers: [ { user:t, password: t } ]\n\t\t\t\texports: [{stream: system-a.events.>}]\n\t\t\t}\n\t\t\tsystem-a: {\n\t\t\t\tusers: [ { user:sa, password: sa } ]\n\t\t\t\timports: [\n\t\t\t\t\t{stream: {subject: system-a.events.>, account: tenant}, prefix: tenant}\n\t\t\t\t]\n\t\t\t}\n\t\t\t$SYS: { users = [ {user: \"s\", password: \"s\"} ] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes: [{\n\t\t\t\turls: [ \"nats-leaf://sa:sa@127.0.0.1:17422\" ]\n\t\t\t\taccount: system-a\n\t\t\t}]\n\t\t}`\n\n\tconfCluster0 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: a-0\n\t\tport: -1\n\t\tcluster: {\n\t\t\tname: cluster-a\n\t\t\tlisten: 127.0.0.1:16122\n\t\t\troutes = [ nats://127.0.0.1:16123 ]\n\t\t\tpool_size: -1\n\t\t}\n\t\t%s`, clusterCommonConf)))\n\n\tconfCluster1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: a-1\n\t\tport: -1\n\t\tcluster: {\n\t\t\tname: cluster-a\n\t\t\tlisten: 127.0.0.1:16123\n\t\t\troutes = [ nats://127.0.0.1:16122 ]\n\t\t\tpool_size: -1\n\t\t}\n\t\t%s`, clusterCommonConf)))\n\n\tserverB := createConfFile(t, []byte(`\n\t\tserver_name: b\n\t\tport: -1\n\t\tleafnodes: { port: 17422 }\n\t\taccounts: {\n\t\t\tsystem-a: {\n\t\t\t\tusers: [ { user: sa, password: sa } ]\n\t\t\t\texports: [{stream: *.system-a.>}]\n\t\t\t}\n\t\t\tsystem-b: {\n\t\t\t\tusers: [ { user: sb, password: sb } ]\n\t\t\t\timports: [ {stream: {subject: *.system-a.>, account: system-a }}]\n\t\t\t}\n\t\t\t$SYS: { users = [ {user: \"s\", password: \"s\"} ] }\n\t\t}`))\n\n\t// Start server B\n\tsrvB, _ := RunServerWithConfig(serverB)\n\tdefer srvB.Shutdown()\n\n\t// Start the cluster servers.\n\tsrvA0, _ := RunServerWithConfig(confCluster0)\n\tdefer srvA0.Shutdown()\n\t// Make sure this is connected first before starting the second server in cluster A.\n\tcheckLeafNodeConnectedCount(t, srvB, 1)\n\t// Start second A server.\n\tsrvA1, _ := RunServerWithConfig(confCluster1)\n\tdefer srvA1.Shutdown()\n\t// Make sure they are routed together.\n\tcheckNumRoutes(t, srvA0, 1)\n\tcheckNumRoutes(t, srvA1, 1)\n\t// Make sure each cluster server is connected to server B.\n\tcheckLeafNodeConnectedCount(t, srvB, 2)\n\n\t// Create plain subscriber on server B attached to system-b account.\n\tncB := natsConnect(t, srvB.ClientURL(), nats.UserInfo(\"sb\", \"sb\"))\n\tdefer ncB.Close()\n\tsub := natsSubSync(t, ncB, \"*.system-a.events.>\")\n\tsubq := natsQueueSubSync(t, ncB, \"*.system-a.events.objectnotfound\", \"SBQ\")\n\tnatsFlush(t, ncB)\n\n\t// Create a subscription on SA1 (we will send from SA0). We want to make sure that\n\t// when subscription on B is removed, this does not affect the subject interest\n\t// in SA0 on behalf of SA1.\n\tncSAA1 := natsConnect(t, srvA1.ClientURL(), nats.UserInfo(\"sa\", \"sa\"))\n\tdefer ncSAA1.Close()\n\tsub2 := natsSubSync(t, ncSAA1, \"*.system-a.events.>\")\n\tsubq2 := natsQueueSubSync(t, ncSAA1, \"*.system-a.events.objectnotfound\", \"SBQ\")\n\tnatsFlush(t, ncSAA1)\n\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Connect to cluster A on SA0.\n\tncA := natsConnect(t, srvA0.ClientURL(), nats.UserInfo(\"t\", \"t\"))\n\tdefer ncA.Close()\n\n\tnatsPub(t, ncA, \"system-a.events.objectnotfound\", []byte(\"EventA\"))\n\tnatsFlush(t, ncA)\n\n\tnatsNexMsg(t, sub, time.Second)\n\tnatsNexMsg(t, sub2, time.Second)\n\tif _, err := subq.NextMsg(250 * time.Millisecond); err != nil {\n\t\tnatsNexMsg(t, subq2, time.Second)\n\t}\n\n\t// Unsubscribe the subscriptions from server B.\n\tnatsUnsub(t, sub)\n\tnatsUnsub(t, subq)\n\tnatsFlush(t, ncB)\n\n\t// Wait for subject propagation.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Publish again, subscriptions on SA1 should receive it.\n\tnatsPub(t, ncA, \"system-a.events.objectnotfound\", []byte(\"EventA\"))\n\tnatsFlush(t, ncA)\n\n\tnatsNexMsg(t, sub2, time.Second)\n\tnatsNexMsg(t, subq2, time.Second)\n}\n\nfunc TestLeafNodeServerKickClient(t *testing.T) {\n\tstmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-server\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t`\n\tconf := createConfFile(t, []byte(stmpl))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: test-leaf\n\t\tleaf { remotes: [ { urls: [ nats-leaf://127.0.0.1:{LEAF_PORT} ] } ] }\n\t`\n\ttmpl = strings.Replace(tmpl, \"{LEAF_PORT}\", fmt.Sprintf(\"%d\", o.LeafNode.Port), 1)\n\tlConf := createConfFile(t, []byte(tmpl))\n\tl, _ := RunServerWithConfig(lConf)\n\tdefer l.Shutdown()\n\n\tcheckLeafNodeConnected(t, l)\n\n\t// We want to make sure we can kick the leafnode connections as well as client connections.\n\tconns, err := s.Connz(&ConnzOptions{Account: globalAccountName})\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(conns.Conns), 1)\n\tlid := conns.Conns[0].Cid\n\n\tdisconnectTime := time.Now()\n\terr = s.DisconnectClientByID(lid)\n\trequire_NoError(t, err)\n\n\t// Wait until we are reconnected.\n\tcheckLeafNodeConnected(t, s)\n\n\t// Look back up again and make sure start time indicates a restart, meaning kick worked.\n\tconns, err = s.Connz(&ConnzOptions{Account: globalAccountName})\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(conns.Conns), 1)\n\tln := conns.Conns[0]\n\trequire_True(t, lid != ln.Cid)\n\trequire_True(t, ln.Start.After(disconnectTime))\n}\n\nfunc TestLeafNodeBannerNoClusterNameIfNoCluster(t *testing.T) {\n\tu, err := url.Parse(\"nats://127.0.0.1:1234\")\n\trequire_NoError(t, err)\n\n\topts := DefaultOptions()\n\topts.ServerName = \"LEAF_SERVER\"\n\topts.Cluster.Name = _EMPTY_\n\topts.Cluster.Port = 0\n\topts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\topts.NoLog = false\n\n\ts, err := NewServer(opts)\n\trequire_NoError(t, err)\n\tdefer func() {\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t}()\n\tl := &captureNoticeLogger{}\n\ts.SetLogger(l, false, false)\n\ts.Start()\n\n\tif !s.ReadyForConnections(time.Second) {\n\t\tt.Fatal(\"Server not ready!\")\n\t}\n\n\tl.Lock()\n\tfor _, n := range l.notices {\n\t\tif strings.Contains(n, \"Cluster: \") {\n\t\t\tl.Unlock()\n\t\t\tt.Fatalf(\"Cluster name should not be displayed, got %q\", n)\n\t\t}\n\t}\n\tl.Unlock()\n}\n\nfunc TestLeafNodeCredFormatting(t *testing.T) {\n\t//create the operator/sys/account tree\n\toKP, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\toPK, err := oKP.PublicKey()\n\trequire_NoError(t, err)\n\n\toc := jwt.NewOperatorClaims(oPK)\n\toc.Name = \"O\"\n\toJWT, err := oc.Encode(oKP)\n\trequire_NoError(t, err)\n\n\tsysKP, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\tsysPK, err := sysKP.PublicKey()\n\trequire_NoError(t, err)\n\n\tsys := jwt.NewAccountClaims(sysPK)\n\tsys.Name = \"SYS\"\n\tsysJWT, err := sys.Encode(oKP)\n\trequire_NoError(t, err)\n\n\taKP, err := nkeys.CreateAccount()\n\trequire_NoError(t, err)\n\taPK, err := aKP.PublicKey()\n\trequire_NoError(t, err)\n\n\tac := jwt.NewAccountClaims(aPK)\n\tac.Name = \"A\"\n\taJWT, err := ac.Encode(oKP)\n\trequire_NoError(t, err)\n\n\tuKP, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tuSeed, err := uKP.Seed()\n\trequire_NoError(t, err)\n\tuPK, err := uKP.PublicKey()\n\trequire_NoError(t, err)\n\n\t// build the config\n\tstmpl := fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEM\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t\t%s: %s\n\t\t}\n\t\tleaf { listen: 127.0.0.1:-1 }\n\t`, oJWT, sysPK, sysPK, sysJWT, aPK, aJWT)\n\tconf := createConfFile(t, []byte(stmpl))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// create the leaf node\n\t// generate the user credentials\n\tuc := jwt.NewUserClaims(uPK)\n\tuc.Name = \"U\"\n\tuc.Limits.Data = -1\n\tuc.Limits.Payload = -1\n\tuc.Permissions.Pub.Allow.Add(\">\")\n\tuc.Permissions.Sub.Allow.Add(\">\")\n\tuJWT, err := uc.Encode(aKP)\n\trequire_NoError(t, err)\n\n\trunLeaf := func(t *testing.T, creds []byte) {\n\t\tfile, err := os.CreateTemp(\"\", \"tmp-*.creds\")\n\t\trequire_NoError(t, err)\n\t\t_, err = file.Write(creds)\n\t\trequire_NoError(t, err)\n\t\trequire_NoError(t, file.Close())\n\n\t\ttemplate := fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tleafnodes {\n\t\t\t\tremotes: [\n\t\t\t\t\t{\n\t\t\t\t\t\turls: [ nats-leaf://127.0.0.1:%d ]\n\t\t\t\t\t\tcredentials: \"%s\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}`, o.LeafNode.Port, file.Name())\n\t\tconf := createConfFile(t, []byte(template))\n\t\tleaf, _ := RunServerWithConfig(conf)\n\t\tdefer leaf.Shutdown()\n\t\tdefer os.Remove(file.Name())\n\t\tcheckLeafNodeConnected(t, leaf)\n\t}\n\n\tcreds, err := jwt.FormatUserConfig(uJWT, uSeed)\n\trequire_NoError(t, err)\n\n\trunLeaf(t, creds)\n\trunLeaf(t, bytes.ReplaceAll(creds, []byte{'\\n'}, []byte{'\\r', '\\n'}))\n}\n\nfunc TestLeafNodePermissionWithLiteralSubjectAndQueueInterest(t *testing.T) {\n\thconf := createConfFile(t, []byte(`\n\t\tserver_name: \"HUB\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [\n\t\t\t\t\t{ user: \"user\", password: \"pwd\",\n\t\t\t\t\t\tpermissions: {\n\t\t\t\t\t\t\tsubscribe: { allow: [\"_INBOX.>\", \"my.subject\"] }\n\t\t\t\t\t\t\tpublish: {allow: [\">\"]}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(hconf)\n\tdefer hub.Shutdown()\n\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tserver_name: \"LEAF\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes: [\n\t\t\t\t{url: \"nats://user:pwd@127.0.0.1:%d\", account: A}\n\t\t\t]\n\t\t}\n\t\taccounts {\n\t\t\tA { users: [{user: user, password: pwd}] }\n\t\t}\n\t`, ohub.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(lconf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\tncLeaf := natsConnect(t, leaf.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncLeaf.Close()\n\tnatsQueueSub(t, ncLeaf, \"my.subject\", \"queue\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"OK\"))\n\t})\n\tnatsFlush(t, ncLeaf)\n\n\tncHub := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"user\", \"pwd\"))\n\tdefer ncHub.Close()\n\n\tvar resp *nats.Msg\n\tvar err error\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t// Make sure we don't fail the test on the first \"no responders\", might\n\t\t// take time for the sub to propagate.\n\t\tresp, err = ncHub.Request(\"my.subject\", []byte(\"hello\"), time.Second)\n\t\treturn err\n\t})\n\trequire_Equal(t, \"OK\", string(resp.Data))\n}\n\nfunc TestLeafNodePermissionWithGateways(t *testing.T) {\n\tusConf := createConfFile(t, []byte(`\n\t\tserver_name: \"US\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tgateway {\n\t\t\tname: \"US\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\taccounts {\n\t\t\tsys { users: [{user: sys, password: sys}] }\n\t\t\tleaf { users: [{user: leaf, password: leaf}] }\n\t\t}\n\t\tsystem_account: sys\n\t`))\n\tus, ous := RunServerWithConfig(usConf)\n\tdefer us.Shutdown()\n\n\teuConf := createConfFile(t, fmt.Appendf(nil, `\n\t\tserver_name: \"EU\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tgateway {\n\t\t\tname: \"EU\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tgateways: [\n\t\t\t\t{\n\t\t\t\t\tname: \"US\"\n\t\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\taccounts {\n\t\t\tsys { users: [{user: sys, password: sys}] }\n\t\t\tleaf {\n\t\t\t\tusers: [\n\t\t\t\t\t{\n\t\t\t\t\t\tuser: leaf\n\t\t\t\t\t\tpassword: leaf\n\t\t\t\t\t\tpermissions: {\n\t\t\t\t\t\t\tpublish: \"bar\"\n\t\t\t\t\t\t\tsubscribe: \"foo\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tsystem_account: sys\n\t\t`, ous.Gateway.Port))\n\teu, oeu := RunServerWithConfig(euConf)\n\tdefer eu.Shutdown()\n\n\twaitForOutboundGateways(t, us, 1, 2*time.Second)\n\twaitForOutboundGateways(t, eu, 1, 2*time.Second)\n\twaitForInboundGateways(t, us, 1, 2*time.Second)\n\twaitForInboundGateways(t, eu, 1, 2*time.Second)\n\n\tleafConf := createConfFile(t, fmt.Appendf(nil, `\n\t\tserver_name: \"LEAF\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes: [\n\t\t\t\t{ url: \"nats://leaf:leaf@127.0.0.1:%d\" }\n\t\t\t]\n\t\t}\n\t`, oeu.LeafNode.Port))\n\tleaf, _ := RunServerWithConfig(leafConf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnected(t, leaf)\n\n\t// Run the service from EU leafnode.\n\tncEU := natsConnect(t, leaf.ClientURL())\n\tdefer ncEU.Close()\n\tnatsSub(t, ncEU, \"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"response\"))\n\t})\n\tnatsFlush(t, ncEU)\n\n\t// Wait for subject interest to propagate.\n\tcheckGWInterestOnlyModeInterestOn(t, us, \"EU\", \"leaf\", \"foo\")\n\n\t// Connect to the US server\n\tncUS := natsConnect(t, us.ClientURL(), nats.UserInfo(\"leaf\", \"leaf\"))\n\tdefer ncUS.Close()\n\n\t// Create a subscription on \"bar\" and send request on \"foo\"\n\tsub := natsSubSync(t, ncUS, \"bar\")\n\t// Wait for subject to propagate so we know that EU server\n\t// would know about the \"bar\" subscription.\n\tcheckGWInterestOnlyModeInterestOn(t, eu, \"US\", \"leaf\", \"bar\")\n\t// Send the request and make sure we receive the reply.\n\tnatsPubReq(t, ncUS, \"foo\", \"bar\", []byte(\"request\"))\n\treply := natsNexMsg(t, sub, time.Second)\n\tif string(reply.Data) != \"response\" {\n\t\tt.Fatalf(\"Invalid response: %q\", reply.Data)\n\t}\n\t// Make sure that we don't blindly accept any reply because there\n\t// is the routing protocol. So create a sub on \"baz\" that the leaf\n\t// would not be allowed to reply to. We should not get the reply\n\t// to our request.\n\tsub2 := natsSubSync(t, ncUS, \"baz\")\n\tcheckGWInterestOnlyModeInterestOn(t, eu, \"US\", \"leaf\", \"baz\")\n\t// Send the request. We should not get the message since the\n\t// leaf server does not have permission to publish on \"baz\".\n\tnatsPubReq(t, ncUS, \"foo\", \"baz\", []byte(\"request2\"))\n\tif msg, err := sub2.NextMsg(250 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Should not have received the reply, got %q\", msg.Data)\n\t}\n}\n\nfunc TestLeafNodesDisableRemote(t *testing.T) {\n\thubConf := createConfFile(t, []byte(`\n\t\tserver_name: \"HUB\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\tleaf1 { users: [{user: leaf1, password: pwd}] }\n\t\t\tleaf2 { users: [{user: leaf2, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(hubConf)\n\tdefer hub.Shutdown()\n\n\tport := ohub.LeafNode.Port\n\tleafTmpl := `\n\t\tserver_name: \"LEAF\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\treconnect_interval: \"50ms\"\n\t\t\tremotes: [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://leaf1:pwd@127.0.0.1:%d\"\n\t\t\t\t\tdisabled: %v\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://leaf2:pwd@127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\t// Start with \"disabled: true\" to make sure that we don't solicit it\n\t// when starting the server.\n\tleafConf := createConfFile(t, fmt.Appendf(nil, leafTmpl, port, true, port))\n\tleaf, _ := RunServerWithConfig(leafConf)\n\tdefer leaf.Shutdown()\n\n\t// Wait for more than the reconnect interval to make sure that we don't\n\t// reconnect the connection that should have been disabled.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Verify that we have only 1 leaf\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\treconnectCh := make(chan struct{}, 2)\n\tncl1 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"leaf1\", \"pwd\"),\n\t\tnats.ReconnectWait(50*time.Millisecond),\n\t\tnats.ReconnectJitter(time.Millisecond, time.Millisecond),\n\t\tnats.ReconnectHandler(func(_ *nats.Conn) {\n\t\t\treconnectCh <- struct{}{}\n\t\t}))\n\tdefer ncl1.Close()\n\tsub1 := natsSubSync(t, ncl1, \"foo\")\n\tnatsFlush(t, ncl1)\n\n\tncl2 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"leaf2\", \"pwd\"),\n\t\tnats.ReconnectWait(50*time.Millisecond),\n\t\tnats.ReconnectJitter(time.Millisecond, time.Millisecond),\n\t\tnats.ReconnectHandler(func(_ *nats.Conn) {\n\t\t\treconnectCh <- struct{}{}\n\t\t}))\n\tdefer ncl2.Close()\n\tsub2 := natsSubSync(t, ncl2, \"foo\")\n\tnatsFlush(t, ncl2)\n\n\tcheckSubInterest(t, leaf, globalAccountName, \"foo\", time.Second)\n\n\tnc := natsConnect(t, leaf.ClientURL())\n\tdefer nc.Close()\n\n\tnatsPub(t, nc, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc)\n\n\t// We should not receive on leaf1\n\t_, err := sub1.NextMsg(100 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// But should receive on leaf2.\n\tmsg := natsNexMsg(t, sub2, time.Second)\n\trequire_Equal(t, string(msg.Data), \"hello\")\n\n\t// Enable leaf1, which means set \"disabled\" to false.\n\treloadUpdateConfig(t, leaf, leafConf, fmt.Sprintf(leafTmpl, port, false, port))\n\n\t// Check that we have 2 leaf node connections now.\n\tcheckLeafNodeConnectedCount(t, hub, 2)\n\tcheckLeafNodeConnectedCount(t, leaf, 2)\n\n\t// Verify connectivity.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello2\"))\n\n\tmsg = natsNexMsg(t, sub1, time.Second)\n\trequire_Equal(t, string(msg.Data), \"hello2\")\n\n\tmsg = natsNexMsg(t, sub2, time.Second)\n\trequire_Equal(t, string(msg.Data), \"hello2\")\n\n\t// Disable again.\n\treloadUpdateConfig(t, leaf, leafConf, fmt.Sprintf(leafTmpl, port, true, port))\n\n\t// Wait for more than the reconnect interval to make sure that we don't\n\t// reconnect the connection that should have been disabled.\n\ttime.Sleep(100 * time.Millisecond)\n\t// Verify that we have only 1 leaf\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\t// Now send a message again.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello3\"))\n\tnatsFlush(t, nc)\n\t// We should not receive on leaf1\n\t_, err = sub1.NextMsg(100 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// We should still receive for leaf2\n\tmsg = natsNexMsg(t, sub2, time.Second)\n\trequire_Equal(t, string(msg.Data), \"hello3\")\n\n\t// Enable again.\n\treloadUpdateConfig(t, leaf, leafConf, fmt.Sprintf(leafTmpl, port, false, port))\n\n\tcheckLeafNodeConnectedCount(t, hub, 2)\n\tcheckLeafNodeConnectedCount(t, leaf, 2)\n\n\t// Now shutdown hub.\n\thub.Shutdown()\n\n\t// Wait to be disconnected\n\tcheckLeafNodeConnectedCount(t, leaf, 0)\n\n\t// Wait at least some reconnect interval.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Now disable leaf1 one more time to make sure we stop soliciting when we were\n\t// not currently connected.\n\treloadUpdateConfig(t, leaf, leafConf, fmt.Sprintf(leafTmpl, port, true, port))\n\n\t// Restart hub\n\thub = RunServer(ohub)\n\tdefer hub.Shutdown()\n\n\t// Verify that we have only 1 leaf\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n\n\t// Wait for clients to reconnect\n\tfor range 2 {\n\t\tselect {\n\t\tcase <-reconnectCh:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Client failed to reconnect\")\n\t\t}\n\t}\n\n\t// Wait for subject propagation\n\tcheckSubInterest(t, leaf, globalAccountName, \"foo\", time.Second)\n\n\t// Now send a message again.\n\tnatsPub(t, nc, \"foo\", []byte(\"hello4\"))\n\tnatsFlush(t, nc)\n\t// We should not receive on leaf1\n\t_, err = sub1.NextMsg(100 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// We should still receive for leaf2\n\tmsg = natsNexMsg(t, sub2, time.Second)\n\trequire_Equal(t, string(msg.Data), \"hello4\")\n}\n\nfunc TestLeafNodeIsolatedLeafSubjectPropagationGlobal(t *testing.T) {\n\tfor tname, isolated := range map[string]bool{\n\t\t\"Isolated\": true,\n\t\t\"Normal\": false,\n\t} {\n\t\tt.Run(tname, func(t *testing.T) {\n\t\t\thubTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tHA { users: [{user: HA, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tisolate_leafnode_interest: %v\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfH := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, \"H1\", isolated)))\n\t\t\tsh, oh := RunServerWithConfig(confH)\n\t\t\tdefer sh.Shutdown()\n\n\t\t\tspokeTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://HA:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\tlocal: \"A\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`\n\n\t\t\tconfSP1 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP1\", oh.LeafNode.Port)))\n\t\t\tsp1, _ := RunServerWithConfig(confSP1)\n\t\t\tdefer sp1.Shutdown()\n\n\t\t\tconfSP2 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP2\", oh.LeafNode.Port)))\n\t\t\tsp2, _ := RunServerWithConfig(confSP2)\n\t\t\tdefer sp2.Shutdown()\n\n\t\t\tcheckLeafNodeConnectedCount(t, sh, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sp1, 1)\n\t\t\tcheckLeafNodeConnectedCount(t, sp2, 1)\n\n\t\t\t// We expect that the hub side will have answered the request from the spoke for\n\t\t\t// isolation if needed, but the spokes themselves will not themselves isolate\n\t\t\t// subscription interest in the other direction unless also configured to do so.\n\t\t\tfor _, c := range sh.leafs {\n\t\t\t\trequire_Equal(t, c.leaf.isolated, isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp1.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp2.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\n\t\t\tnch := natsConnect(t, sh.ClientURL(), nats.UserInfo(\"HA\", \"pwd\"))\n\t\t\tnc1 := natsConnect(t, sp1.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\n\t\t\t// Create a north-south subscription on the hub that should be visible to both spokes.\n\t\t\tnssub, err := nch.SubscribeSync(\"northsouth\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"northsouth\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\t\t\tcheckSubInterest(t, sp2, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\n\t\t\t// The spoke subscriptions should be visible to the hub in all cases, but only\n\t\t\t// visible to other spokes if they are not isolated.\n\t\t\tewsub, err := nc1.SubscribeSync(\"eastwest\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"eastwest\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"eastwest\", time.Second) // Visible to the spoke with the sub.\n\t\t\tif isolated {\n\t\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Not visible to the other spoke.\n\t\t\t} else {\n\t\t\t\tcheckSubInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Visible to the other spoke.\n\t\t\t}\n\t\t\trequire_NoError(t, ewsub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second)\n\n\t\t\t// ... but a subscription from the hub should be visible to both.\n\t\t\trequire_NoError(t, nssub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"northsouth\", time.Second)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeIsolatedLeafSubjectPropagationRequestIsolation(t *testing.T) {\n\tfor tname, isolated := range map[string]bool{\n\t\t\"Isolated\": true,\n\t\t\"Normal\": false,\n\t} {\n\t\tt.Run(tname, func(t *testing.T) {\n\t\t\thubTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tHA { users: [{user: HA, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfH := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, \"H1\")))\n\t\t\tsh, oh := RunServerWithConfig(confH)\n\t\t\tdefer sh.Shutdown()\n\n\t\t\tspokeTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://HA:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\tlocal: \"A\"\n\t\t\t\t\t\t\trequest_isolation: %v\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`\n\n\t\t\tconfSP1 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP1\", oh.LeafNode.Port, isolated)))\n\t\t\tsp1, _ := RunServerWithConfig(confSP1)\n\t\t\tdefer sp1.Shutdown()\n\n\t\t\tconfSP2 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP2\", oh.LeafNode.Port, isolated)))\n\t\t\tsp2, _ := RunServerWithConfig(confSP2)\n\t\t\tdefer sp2.Shutdown()\n\n\t\t\tcheckLeafNodeConnectedCount(t, sh, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sp1, 1)\n\t\t\tcheckLeafNodeConnectedCount(t, sp2, 1)\n\n\t\t\t// We expect that the hub side will have answered the request from the spoke for\n\t\t\t// isolation if needed, but the spokes themselves will not themselves isolate\n\t\t\t// subscription interest in the other direction unless also configured to do so.\n\t\t\tfor _, c := range sh.leafs {\n\t\t\t\trequire_Equal(t, c.leaf.isolated, isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp1.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp2.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\n\t\t\tnch := natsConnect(t, sh.ClientURL(), nats.UserInfo(\"HA\", \"pwd\"))\n\t\t\tnc1 := natsConnect(t, sp1.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\n\t\t\t// Create a north-south subscription on the hub that should be visible to both spokes.\n\t\t\tnssub, err := nch.SubscribeSync(\"northsouth\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"northsouth\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\t\t\tcheckSubInterest(t, sp2, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\n\t\t\t// The spoke subscriptions should be visible to the hub in all cases, but only\n\t\t\t// visible to other spokes if they are not isolated.\n\t\t\tewsub, err := nc1.SubscribeSync(\"eastwest\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"eastwest\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"eastwest\", time.Second) // Visible to the spoke with the sub.\n\t\t\tif isolated {\n\t\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Not visible to the other spoke.\n\t\t\t} else {\n\t\t\t\tcheckSubInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Visible to the other spoke.\n\t\t\t}\n\t\t\trequire_NoError(t, ewsub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second)\n\n\t\t\t// ... but a subscription from the hub should be visible to both.\n\t\t\trequire_NoError(t, nssub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"northsouth\", time.Second)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeIsolatedLeafSubjectPropagationLocalIsolation(t *testing.T) {\n\tfor tname, isolated := range map[string]bool{\n\t\t\"Isolated\": true,\n\t\t\"Normal\": false,\n\t} {\n\t\tt.Run(tname, func(t *testing.T) {\n\t\t\tspokeTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: A, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t}\n\t\t\t`\n\n\t\t\tconfSP1 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP1\")))\n\t\t\tsp1, osp1 := RunServerWithConfig(confSP1)\n\t\t\tdefer sp1.Shutdown()\n\n\t\t\tconfSP2 := createConfFile(t, []byte(fmt.Sprintf(spokeTmpl, \"SP2\")))\n\t\t\tsp2, osp2 := RunServerWithConfig(confSP2)\n\t\t\tdefer sp2.Shutdown()\n\n\t\t\thubTmpl := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"%s\"\n\t\t\t\taccounts {\n\t\t\t\t\tHA { users: [{user: HA, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://A:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\tlocal: \"HA\"\n\t\t\t\t\t\t\tisolate: %v\n\t\t\t\t\t\t\thub: true\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://A:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\tlocal: \"HA\"\n\t\t\t\t\t\t\tisolate: %v\n\t\t\t\t\t\t\thub: true\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfH := createConfFile(t, []byte(fmt.Sprintf(hubTmpl, \"H1\", osp1.LeafNode.Port, isolated, osp2.LeafNode.Port, isolated)))\n\t\t\tsh, _ := RunServerWithConfig(confH)\n\t\t\tdefer sh.Shutdown()\n\n\t\t\tcheckLeafNodeConnectedCount(t, sh, 2)\n\t\t\tcheckLeafNodeConnectedCount(t, sp1, 1)\n\t\t\tcheckLeafNodeConnectedCount(t, sp2, 1)\n\n\t\t\t// We expect that the hub side will have answered the request from the spoke for\n\t\t\t// isolation if needed, but the spokes themselves will not themselves isolate\n\t\t\t// subscription interest in the other direction unless also configured to do so.\n\t\t\tfor _, c := range sh.leafs {\n\t\t\t\trequire_Equal(t, c.leaf.isolated, isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp1.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\t\t\tfor _, c := range sp2.leafs {\n\t\t\t\trequire_False(t, c.leaf.isolated)\n\t\t\t}\n\n\t\t\tnch := natsConnect(t, sh.ClientURL(), nats.UserInfo(\"HA\", \"pwd\"))\n\t\t\tnc1 := natsConnect(t, sp1.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\n\t\t\t// Create a north-south subscription on the hub that should be visible to both spokes.\n\t\t\tnssub, err := nch.SubscribeSync(\"northsouth\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"northsouth\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\t\t\tcheckSubInterest(t, sp2, \"A\", \"northsouth\", time.Second) // Visible to both spokes.\n\n\t\t\t// The spoke subscriptions should be visible to the hub in all cases, but only\n\t\t\t// visible to other spokes if they are not isolated.\n\t\t\tewsub, err := nc1.SubscribeSync(\"eastwest\")\n\t\t\trequire_NoError(t, err)\n\t\t\tcheckSubInterest(t, sh, \"HA\", \"eastwest\", time.Second) // Visible to the hub.\n\t\t\tcheckSubInterest(t, sp1, \"A\", \"eastwest\", time.Second) // Visible to the spoke with the sub.\n\t\t\tif isolated {\n\t\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Not visible to the other spoke.\n\t\t\t} else {\n\t\t\t\tcheckSubInterest(t, sp2, \"A\", \"eastwest\", time.Second) // Visible to the other spoke.\n\t\t\t}\n\t\t\trequire_NoError(t, ewsub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"eastwest\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"eastwest\", time.Second)\n\n\t\t\t// ... but a subscription from the hub should be visible to both.\n\t\t\trequire_NoError(t, nssub.Unsubscribe())\n\t\t\tcheckSubNoInterest(t, sh, \"HA\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp1, \"A\", \"northsouth\", time.Second)\n\t\t\tcheckSubNoInterest(t, sp2, \"A\", \"northsouth\", time.Second)\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeDaisyChainWithAccountImportExport(t *testing.T) {\n\thubConf := createConfFile(t, []byte(`\n\t\tserver_name: hub\n\t\tlisten: \"127.0.0.1:-1\"\n\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\taccounts {\n\t\t\tSYS: {\n\t\t\t\tusers: [{ user: s, password: s}],\n\t\t\t},\n\t\t\tODC: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [\n\t\t\t\t\t{\n\t\t\t\t\t\tuser: u, password: u,\n\t\t\t\t\t\tpermissions: {\n\t\t\t\t\t\t\tpublish: {deny: [\"local.>\",\"hub2leaf.>\"]}\n\t\t\t\t\t\t\tsubscribe: {deny: [\"local.>\",\"leaf2leaf.>\"]}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(hubConf)\n\tdefer hub.Shutdown()\n\n\tstoreDir := t.TempDir()\n\tleafJSConf := createConfFile(t, fmt.Appendf(nil, `\n\t\tserver_name: leaf-js\n\t\tlisten: \"127.0.0.1:-1\"\n\n\t\tjetstream {\n\t\t\tstore_dir=\"%s/leaf-js\"\n\t\t\tdomain=leaf-js\n\t\t}\n\t\taccounts {\n\t\t\tODC: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{ user: u, password: u}]\n\t\t\t},\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turls: [\"leaf://u:u@127.0.0.1:%d\"] # connects to hub\n\t\t\t\t\taccount: ODC\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, storeDir, ohub.LeafNode.Port))\n\tleafJS, _ := RunServerWithConfig(leafJSConf)\n\tdefer leafJS.Shutdown()\n\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leafJS)\n\n\totherConf := createConfFile(t, []byte(`\n\t\tserver_name: other\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\tother, oother := RunServerWithConfig(otherConf)\n\tdefer other.Shutdown()\n\n\ttmpl := `\n\t\tserver_name: %s\n\t\tlisten: \"127.0.0.1:-1\"\n\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turls: [\"leaf://u:u@127.0.0.1:%d\"]\n\t\t\t\t\taccount: ODC_DEV\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turls: [\"leaf://127.0.0.1:%d\"]\n\t\t\t\t\taccount: ODC_DEV\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tcluster {\n\t\t\tname: \"hubsh\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t%s\n\t\t}\n\t\taccounts: {\n\t\t\tODC_DEV: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: o, password: o}\n\t\t\t\t]\n\t\t\t\timports: [\n\t\t\t\t\t{service: {account: \"SH1\", subject: \"$JS.leaf-sh.API.>\"}}\n\t\t\t\t\t{stream: {account: \"SH1\", subject: \"sync.leaf-sh.jspush.>\"}}\n\t\t\t\t]\n\t\t\t\texports: [\n\t\t\t\t\t{stream: \">\"}\n\t\t\t\t\t{service: \">\", response_type: \"Singleton\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tSH1: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: s, password: s}\n\t\t\t\t]\n\t\t\t\texports: [\n\t\t\t\t\t{service: \"$JS.leaf-sh.API.>\", response_type: \"Stream\"}\n\t\t\t\t\t{stream: \"sync.leaf-sh.jspush.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`\n\thubSh1Conf := createConfFile(t, fmt.Appendf(nil, tmpl, \"hubsh1\", ohub.LeafNode.Port, oother.LeafNode.Port, _EMPTY_))\n\thubSh1, ohubSh1 := RunServerWithConfig(hubSh1Conf)\n\tdefer hubSh1.Shutdown()\n\n\thubSh2Conf := createConfFile(t, fmt.Appendf(nil, tmpl, \"hubsh2\", ohub.LeafNode.Port, oother.LeafNode.Port,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", ohubSh1.Cluster.Port)))\n\thubSh2, ohubSh2 := RunServerWithConfig(hubSh2Conf)\n\tdefer hubSh2.Shutdown()\n\n\tcheckClusterFormed(t, hubSh1, hubSh2)\n\n\tcheckLeafNodeConnectedCount(t, hub, 3)\n\tcheckLeafNodeConnectedCount(t, hubSh1, 2)\n\tcheckLeafNodeConnectedCount(t, hubSh2, 2)\n\n\tleafShConf := createConfFile(t, fmt.Appendf(nil, `\n\t\tserver_name: leafsh\n\t\tlisten: \"127.0.0.1:-1\"\n\n\t\tjetstream {\n\t\t\tstore_dir=\"%s/leafsh\"\n\t\t\tdomain=leaf-sh\n\t\t}\n\t\taccounts {\n\t\t\tSH: {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: u, password: u}]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turls: [\"leaf://s:s@127.0.0.1:%d\"]\n\t\t\t\t\taccount: SH\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, storeDir, ohubSh2.LeafNode.Port))\n\tleafSh, _ := RunServerWithConfig(leafShConf)\n\tdefer leafSh.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, hubSh2, 3)\n\tcheckLeafNodeConnected(t, leafSh)\n\n\tncLeafSh, jsLeafSh := jsClientConnect(t, leafSh, nats.UserInfo(\"u\", \"u\"))\n\tdefer ncLeafSh.Close()\n\n\tsc := &nats.StreamConfig{\n\t\tName: \"leaf-sh\",\n\t\tSubjects: []string{\"leaf2leaf.>\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tStorage: nats.FileStorage,\n\t\tAllowRollup: true,\n\t\tAllowDirect: true,\n\t}\n\t_, err := jsLeafSh.AddStream(sc)\n\trequire_NoError(t, err)\n\n\tncLeafJS, jsLeafJS := jsClientConnect(t, leafJS, nats.UserInfo(\"u\", \"u\"))\n\tdefer ncLeafJS.Close()\n\n\tsc = &nats.StreamConfig{\n\t\tName: \"leaf-js\",\n\t\tRetention: nats.LimitsPolicy,\n\t\tStorage: nats.FileStorage,\n\t\tAllowRollup: true,\n\t\tAllowDirect: true,\n\t\tSources: []*nats.StreamSource{\n\t\t\t{\n\t\t\t\tName: \"leaf-sh\",\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.leaf-sh.API\",\n\t\t\t\t\tDeliverPrefix: \"sync.leaf-sh.jspush\"},\n\t\t\t},\n\t\t},\n\t}\n\t_, err = jsLeafJS.AddStream(sc)\n\trequire_NoError(t, err)\n\n\tfor range 10 {\n\t\t_, err = jsLeafSh.Publish(\"leaf2leaf.v1.test\", []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\tcheck := func(js nats.JetStreamContext, stream string) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := si.State.Msgs; n != 10 {\n\t\t\t\treturn fmt.Errorf(\"Expected 10 messages, got %v\", n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheck(jsLeafSh, \"leaf-sh\")\n\tcheck(jsLeafJS, \"leaf-js\")\n\n\tacc := other.GlobalAccount()\n\tacc.mu.RLock()\n\tsr := acc.sl.ReverseMatch(\"sync.leaf-sh.jspush.>\")\n\tacc.mu.RUnlock()\n\trequire_Len(t, len(sr.psubs), 0)\n}\n\nfunc TestLeafNodeConfigureWriteDeadline(t *testing.T) {\n\to1, o2 := DefaultOptions(), DefaultOptions()\n\n\to1.LeafNode.WriteDeadline = 5 * time.Second\n\to1.LeafNode.Host = \"127.0.0.1\"\n\to1.LeafNode.Port = -1\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\ts1URL, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.LeafNode.Port))\n\to2.Cluster.Name = \"somethingelse\"\n\to2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{s1URL}}}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\ts1.mu.RLock()\n\tdefer s1.mu.RUnlock()\n\n\tfor _, r := range s1.leafs {\n\t\trequire_Equal(t, r.out.wdl, 5*time.Second)\n\t}\n}\n\nfunc TestLeafNodeConfigureWriteTimeoutPolicy(t *testing.T) {\n\tfor name, policy := range map[string]WriteTimeoutPolicy{\n\t\t\"Default\": WriteTimeoutPolicyDefault,\n\t\t\"Retry\": WriteTimeoutPolicyRetry,\n\t\t\"Close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\to1 := testDefaultOptionsForGateway(\"B\")\n\t\t\to1.Gateway.WriteTimeout = policy\n\t\t\ts1 := runGatewayServer(o1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\to2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s1)\n\t\t\ts2 := runGatewayServer(o2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, s2, 1, time.Second)\n\t\t\twaitForInboundGateways(t, s1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, s1, 1, time.Second)\n\n\t\t\ts1.mu.RLock()\n\t\t\tdefer s1.mu.RUnlock()\n\n\t\t\tfor _, r := range s1.leafs {\n\t\t\t\tif policy == WriteTimeoutPolicyDefault {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, WriteTimeoutPolicyRetry)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, policy)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/7441\nfunc TestLeafNodesBasicTokenAuth(t *testing.T) {\n\thubConf := createConfFile(t, []byte(`\n\t\tserver_name: \"HUB\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tauthorization {\n\t\t\ttoken: secret\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(hubConf)\n\tdefer hub.Shutdown()\n\n\tport := ohub.LeafNode.Port\n\tleafTmpl := `\n\t\tserver_name: \"LEAF\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tleafnodes {\n\t\t\tremotes: [\n\t\t\t\t{ url: \"nats://secret@localhost:%d\" }\n\t\t\t]\n\t\t}\n\t`\n\tleafConf := createConfFile(t, fmt.Appendf(nil, leafTmpl, port))\n\tleaf, _ := RunServerWithConfig(leafConf)\n\tdefer leaf.Shutdown()\n\n\t// Verify that we have only 1 leaf\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnected(t, leaf)\n}\n\nfunc TestLeafNodeNoAccPanicOnLeafSubBeforeConnect(t *testing.T) {\n\to := DefaultOptions()\n\to.LeafNode.Port = -1\n\t// Default compression is s2_auto, which causes the auth timer to use\n\t// c.ping.tmr instead of c.atmr. This makes awaitingAuth() return false,\n\t// allowing LS+ through the parser before CONNECT sets c.acc.\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.LeafNode.Port)\n\tc, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer c.Close()\n\n\t// Read the INFO.\n\tbr := bufio.NewReader(c)\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tl, _, err := br.ReadLine()\n\trequire_NoError(t, err)\n\tif !strings.HasPrefix(string(l), \"INFO\") {\n\t\tt.Fatalf(\"Expected INFO, got %q\", l)\n\t}\n\n\t// Send LS+ without CONNECT first. This should not panic the server.\n\t_, err = c.Write([]byte(\"LS+ test\\r\\n\"))\n\trequire_NoError(t, err)\n\n\t// The server should close the connection.\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tbuf := make([]byte, 256)\n\tfor {\n\t\t_, err = c.Read(buf)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Make sure the server is still running.\n\ts.mu.Lock()\n\tshutdown := s.isShuttingDown()\n\ts.mu.Unlock()\n\tif shutdown {\n\t\tt.Fatal(\"Server should not have shutdown\")\n\t}\n}\n\nfunc TestLeafNodeNoAccPanicOnLeafUnsubBeforeConnect(t *testing.T) {\n\to := DefaultOptions()\n\to.LeafNode.Port = -1\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.LeafNode.Port)\n\tc, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer c.Close()\n\n\t// Read the INFO.\n\tbr := bufio.NewReader(c)\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tl, _, err := br.ReadLine()\n\trequire_NoError(t, err)\n\tif !strings.HasPrefix(string(l), \"INFO\") {\n\t\tt.Fatalf(\"Expected INFO, got %q\", l)\n\t}\n\n\t// Send LS- without CONNECT first. This should not panic the server.\n\t_, err = c.Write([]byte(\"LS- test\\r\\n\"))\n\trequire_NoError(t, err)\n\n\t// The server should close the connection.\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tbuf := make([]byte, 256)\n\tfor {\n\t\t_, err = c.Read(buf)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Make sure the server is still running.\n\ts.mu.Lock()\n\tshutdown := s.isShuttingDown()\n\ts.mu.Unlock()\n\tif shutdown {\n\t\tt.Fatal(\"Server should not have shutdown\")\n\t}\n}\n\nfunc TestLeafNodeLeafSubBeforeConnectCompressionEffect(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcompression string\n\t\t// With compression off, the auth timer (c.atmr) is set, so\n\t\t// awaitingAuth() returns true and the parser blocks LS+ with\n\t\t// an auth violation. With compression on (default s2_auto),\n\t\t// c.ping.tmr is used instead, awaitingAuth() returns false,\n\t\t// and LS+ reaches processLeafSub where our nil acc guard\n\t\t// catches it. Both paths send the same \"Authorization Violation\"\n\t\t// error for consistency.\n\t}{\n\t\t{\"compression off\", CompressionOff},\n\t\t{\"compression s2_auto\", CompressionS2Auto},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.LeafNode.Port = -1\n\t\t\to.LeafNode.Compression.Mode = test.compression\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.LeafNode.Port)\n\t\t\tc, err := net.Dial(\"tcp\", addr)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer c.Close()\n\n\t\t\tbr := bufio.NewReader(c)\n\t\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\tl, _, err := br.ReadLine()\n\t\t\trequire_NoError(t, err)\n\t\t\tif !strings.HasPrefix(string(l), \"INFO\") {\n\t\t\t\tt.Fatalf(\"Expected INFO, got %q\", l)\n\t\t\t}\n\n\t\t\t// Send LS+ without CONNECT.\n\t\t\t_, err = c.Write([]byte(\"LS+ test\\r\\n\"))\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Read the error response before the connection is closed.\n\t\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\tl, _, err = br.ReadLine()\n\t\t\trequire_NoError(t, err)\n\t\t\terrMsg := string(l)\n\n\t\t\tif !strings.Contains(errMsg, \"Authorization Violation\") {\n\t\t\t\tt.Fatalf(\"Expected auth violation error, got %q\", errMsg)\n\t\t\t}\n\n\t\t\t// Make sure the server is still running.\n\t\t\ts.mu.Lock()\n\t\t\tshutdown := s.isShuttingDown()\n\t\t\ts.mu.Unlock()\n\t\t\tif shutdown {\n\t\t\t\tt.Fatal(\"Server should not have shutdown\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeNoAccPanicOnLeafSubBeforeConnectOperatorMode(t *testing.T) {\n\t// Setup operator JWT-based server with leafnode port.\n\t// This confirms that even with full operator/JWT auth configured,\n\t// a raw TCP connection can bypass auth and trigger the panic.\n\tsysAcc, _ := nkeys.CreateAccount()\n\tsysAccPub, _ := sysAcc.PublicKey()\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\topub, _ := okp.PublicKey()\n\n\t// Create operator claim with system account.\n\toc := jwt.NewOperatorClaims(opub)\n\toc.SystemAccount = sysAccPub\n\toperatorJwt, err := oc.Encode(okp)\n\trequire_NoError(t, err)\n\n\t// Create the system account JWT.\n\tsysAccClaim := jwt.NewAccountClaims(sysAccPub)\n\tsysAccJwt, err := sysAccClaim.Encode(okp)\n\trequire_NoError(t, err)\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: OP\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: MEMORY\n\t\tresolver_preload: {\n\t\t\t%s: %s\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`, operatorJwt, sysAccPub, sysAccPub, sysAccJwt)))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", opts.LeafNode.Port)\n\tc, err := net.Dial(\"tcp\", addr)\n\trequire_NoError(t, err)\n\tdefer c.Close()\n\n\t// Read the INFO.\n\tbr := bufio.NewReader(c)\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tl, _, err := br.ReadLine()\n\trequire_NoError(t, err)\n\tif !strings.HasPrefix(string(l), \"INFO\") {\n\t\tt.Fatalf(\"Expected INFO, got %q\", l)\n\t}\n\n\t// Send LS+ without CONNECT first, bypassing JWT auth entirely.\n\t// Without the fix this would panic on nil c.acc dereference.\n\t_, err = c.Write([]byte(\"LS+ test\\r\\n\"))\n\trequire_NoError(t, err)\n\n\t// The server should close the connection.\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tbuf := make([]byte, 256)\n\tfor {\n\t\t_, err = c.Read(buf)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Make sure the server is still running.\n\ts.mu.Lock()\n\tshutdown := s.isShuttingDown()\n\ts.mu.Unlock()\n\tif shutdown {\n\t\tt.Fatal(\"Server should not have shutdown\")\n\t}\n}\n" }, { "path": "server/log.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\tsrvlog \"github.com/nats-io/nats-server/v2/logger\"\n)\n\n// Logger interface of the NATS Server\ntype Logger interface {\n\n\t// Log a notice statement\n\tNoticef(format string, v ...any)\n\n\t// Log a warning statement\n\tWarnf(format string, v ...any)\n\n\t// Log a fatal error\n\tFatalf(format string, v ...any)\n\n\t// Log an error\n\tErrorf(format string, v ...any)\n\n\t// Log a debug statement\n\tDebugf(format string, v ...any)\n\n\t// Log a trace statement\n\tTracef(format string, v ...any)\n}\n\n// ConfigureLogger configures and sets the logger for the server.\nfunc (s *Server) ConfigureLogger() {\n\tvar (\n\t\tlog Logger\n\n\t\t// Snapshot server options.\n\t\topts = s.getOpts()\n\t)\n\n\tif opts.NoLog {\n\t\treturn\n\t}\n\n\tsyslog := opts.Syslog\n\tif isWindowsService() && opts.LogFile == \"\" {\n\t\t// Enable syslog if no log file is specified and we're running as a\n\t\t// Windows service so that logs are written to the Windows event log.\n\t\tsyslog = true\n\t}\n\n\tif opts.LogFile != \"\" {\n\t\tlog = srvlog.NewFileLogger(opts.LogFile, opts.Logtime, opts.Debug, opts.Trace, true, srvlog.LogUTC(opts.LogtimeUTC))\n\t\tif opts.LogSizeLimit > 0 {\n\t\t\tif l, ok := log.(*srvlog.Logger); ok {\n\t\t\t\tl.SetSizeLimit(opts.LogSizeLimit)\n\t\t\t}\n\t\t}\n\t\tif opts.LogMaxFiles > 0 {\n\t\t\tif l, ok := log.(*srvlog.Logger); ok {\n\t\t\t\tal := int(opts.LogMaxFiles)\n\t\t\t\tif int64(al) != opts.LogMaxFiles {\n\t\t\t\t\t// set to default (no max) on overflow\n\t\t\t\t\tal = 0\n\t\t\t\t}\n\t\t\t\tl.SetMaxNumFiles(al)\n\t\t\t}\n\t\t}\n\t} else if opts.RemoteSyslog != \"\" {\n\t\tlog = srvlog.NewRemoteSysLogger(opts.RemoteSyslog, opts.Debug, opts.Trace)\n\t} else if syslog {\n\t\tlog = srvlog.NewSysLogger(opts.Debug, opts.Trace)\n\t} else {\n\t\tcolors := true\n\t\t// Check to see if stderr is being redirected and if so turn off color\n\t\t// Also turn off colors if we're running on Windows where os.Stderr.Stat() returns an invalid handle-error\n\t\tstat, err := os.Stderr.Stat()\n\t\tif err != nil || (stat.Mode()&os.ModeCharDevice) == 0 {\n\t\t\tcolors = false\n\t\t}\n\t\tlog = srvlog.NewStdLogger(opts.Logtime, opts.Debug, opts.Trace, colors, true, srvlog.LogUTC(opts.LogtimeUTC))\n\t}\n\n\ts.SetLoggerV2(log, opts.Debug, opts.Trace, opts.TraceVerbose)\n}\n\n// Returns our current logger.\nfunc (s *Server) Logger() Logger {\n\ts.logging.Lock()\n\tdefer s.logging.Unlock()\n\treturn s.logging.logger\n}\n\n// SetLogger sets the logger of the server\nfunc (s *Server) SetLogger(logger Logger, debugFlag, traceFlag bool) {\n\ts.SetLoggerV2(logger, debugFlag, traceFlag, false)\n}\n\n// SetLogger sets the logger of the server\nfunc (s *Server) SetLoggerV2(logger Logger, debugFlag, traceFlag, sysTrace bool) {\n\tif debugFlag {\n\t\tatomic.StoreInt32(&s.logging.debug, 1)\n\t} else {\n\t\tatomic.StoreInt32(&s.logging.debug, 0)\n\t}\n\tif traceFlag {\n\t\tatomic.StoreInt32(&s.logging.trace, 1)\n\t} else {\n\t\tatomic.StoreInt32(&s.logging.trace, 0)\n\t}\n\tif sysTrace {\n\t\tatomic.StoreInt32(&s.logging.traceSysAcc, 1)\n\t} else {\n\t\tatomic.StoreInt32(&s.logging.traceSysAcc, 0)\n\t}\n\ts.logging.Lock()\n\tif s.logging.logger != nil {\n\t\t// Check to see if the logger implements io.Closer. This could be a\n\t\t// logger from another process embedding the NATS server or a dummy\n\t\t// test logger that may not implement that interface.\n\t\tif l, ok := s.logging.logger.(io.Closer); ok {\n\t\t\tif err := l.Close(); err != nil {\n\t\t\t\ts.Errorf(\"Error closing logger: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\ts.logging.logger = logger\n\ts.logging.Unlock()\n}\n\n// ReOpenLogFile if the logger is a file based logger, close and re-open the file.\n// This allows for file rotation by 'mv'ing the file then signaling\n// the process to trigger this function.\nfunc (s *Server) ReOpenLogFile() {\n\t// Check to make sure this is a file logger.\n\ts.logging.RLock()\n\tll := s.logging.logger\n\ts.logging.RUnlock()\n\n\tif ll == nil {\n\t\ts.Noticef(\"File log re-open ignored, no logger\")\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tif opts.LogFile == \"\" {\n\t\ts.Noticef(\"File log re-open ignored, not a file logger\")\n\t} else {\n\t\tfileLog := srvlog.NewFileLogger(\n\t\t\topts.LogFile, opts.Logtime,\n\t\t\topts.Debug, opts.Trace, true,\n\t\t\tsrvlog.LogUTC(opts.LogtimeUTC),\n\t\t)\n\t\ts.SetLogger(fileLog, opts.Debug, opts.Trace)\n\t\tif opts.LogSizeLimit > 0 {\n\t\t\tfileLog.SetSizeLimit(opts.LogSizeLimit)\n\t\t}\n\t\ts.Noticef(\"File log re-opened\")\n\t}\n}\n\n// Noticef logs a notice statement\nfunc (s *Server) Noticef(format string, v ...any) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Noticef(format, v...)\n\t}, format, v...)\n}\n\n// Errorf logs an error\nfunc (s *Server) Errorf(format string, v ...any) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Errorf(format, v...)\n\t}, format, v...)\n}\n\n// Error logs an error with a scope\nfunc (s *Server) Errors(scope any, e error) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Errorf(format, v...)\n\t}, \"%s - %s\", scope, UnpackIfErrorCtx(e))\n}\n\n// Error logs an error with a context\nfunc (s *Server) Errorc(ctx string, e error) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Errorf(format, v...)\n\t}, \"%s: %s\", ctx, UnpackIfErrorCtx(e))\n}\n\n// Error logs an error with a scope and context\nfunc (s *Server) Errorsc(scope any, ctx string, e error) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Errorf(format, v...)\n\t}, \"%s - %s: %s\", scope, ctx, UnpackIfErrorCtx(e))\n}\n\n// Warnf logs a warning error\nfunc (s *Server) Warnf(format string, v ...any) {\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Warnf(format, v...)\n\t}, format, v...)\n}\n\nfunc (s *Server) rateLimitFormatWarnf(format string, v ...any) {\n\tif _, loaded := s.rateLimitLogging.LoadOrStore(format, time.Now()); loaded {\n\t\treturn\n\t}\n\tstatement := fmt.Sprintf(format, v...)\n\ts.Warnf(\"%s\", statement)\n}\n\nfunc (s *Server) RateLimitErrorf(format string, v ...any) {\n\tstatement := fmt.Sprintf(format, v...)\n\tif _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\ts.Errorf(\"%s\", statement)\n}\n\nfunc (s *Server) RateLimitWarnf(format string, v ...any) {\n\tstatement := fmt.Sprintf(format, v...)\n\tif _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\ts.Warnf(\"%s\", statement)\n}\n\nfunc (s *Server) RateLimitDebugf(format string, v ...any) {\n\tstatement := fmt.Sprintf(format, v...)\n\tif _, loaded := s.rateLimitLogging.LoadOrStore(statement, time.Now()); loaded {\n\t\treturn\n\t}\n\ts.Debugf(\"%s\", statement)\n}\n\n// Fatalf logs a fatal error\nfunc (s *Server) Fatalf(format string, v ...any) {\n\tif s.isShuttingDown() {\n\t\ts.Errorf(format, v)\n\t\treturn\n\t}\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Fatalf(format, v...)\n\t}, format, v...)\n}\n\n// Debugf logs a debug statement\nfunc (s *Server) Debugf(format string, v ...any) {\n\tif atomic.LoadInt32(&s.logging.debug) == 0 {\n\t\treturn\n\t}\n\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Debugf(format, v...)\n\t}, format, v...)\n}\n\n// Tracef logs a trace statement\nfunc (s *Server) Tracef(format string, v ...any) {\n\tif atomic.LoadInt32(&s.logging.trace) == 0 {\n\t\treturn\n\t}\n\n\ts.executeLogCall(func(logger Logger, format string, v ...any) {\n\t\tlogger.Tracef(format, v...)\n\t}, format, v...)\n}\n\nfunc (s *Server) executeLogCall(f func(logger Logger, format string, v ...any), format string, args ...any) {\n\ts.logging.RLock()\n\tdefer s.logging.RUnlock()\n\tif s.logging.logger == nil {\n\t\treturn\n\t}\n\n\tf(s.logging.logger, format, args...)\n}\n" }, { "path": "server/log_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/testhelper\"\n\t\"github.com/nats-io/nats-server/v2/logger\"\n)\n\nfunc TestSetLogger(t *testing.T) {\n\tserver := &Server{}\n\tdefer server.SetLogger(nil, false, false)\n\tdl := &DummyLogger{}\n\tserver.SetLogger(dl, true, true)\n\n\t// We assert that the logger has change to the DummyLogger\n\t_ = server.logging.logger.(*DummyLogger)\n\n\tif server.logging.debug != 1 {\n\t\tt.Fatalf(\"Expected debug 1, received value %d\\n\", server.logging.debug)\n\t}\n\n\tif server.logging.trace != 1 {\n\t\tt.Fatalf(\"Expected trace 1, received value %d\\n\", server.logging.trace)\n\t}\n\n\t// Check traces\n\texpectedStr := \"This is a Notice\"\n\tserver.Noticef(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\texpectedStr = \"This is an Error\"\n\tserver.Errorf(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\texpectedStr = \"This is a Fatal\"\n\tserver.Fatalf(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\texpectedStr = \"This is a Debug\"\n\tserver.Debugf(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\texpectedStr = \"This is a Trace\"\n\tserver.Tracef(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\texpectedStr = \"This is a Warning\"\n\tserver.Tracef(expectedStr)\n\tdl.CheckContent(t, expectedStr)\n\n\t// Make sure that we can reset to fal\n\tserver.SetLogger(dl, false, false)\n\tif server.logging.debug != 0 {\n\t\tt.Fatalf(\"Expected debug 0, got %v\", server.logging.debug)\n\t}\n\tif server.logging.trace != 0 {\n\t\tt.Fatalf(\"Expected trace 0, got %v\", server.logging.trace)\n\t}\n\t// Now, Debug and Trace should not produce anything\n\tdl.Msg = \"\"\n\tserver.Debugf(\"This Debug should not be traced\")\n\tdl.CheckContent(t, \"\")\n\tserver.Tracef(\"This Trace should not be traced\")\n\tdl.CheckContent(t, \"\")\n}\n\ntype DummyLogger = testhelper.DummyLogger\n\nfunc TestReOpenLogFile(t *testing.T) {\n\t// We can't rename the file log when still opened on Windows, so skip\n\tif runtime.GOOS == \"windows\" {\n\t\tt.SkipNow()\n\t}\n\ts := &Server{opts: &Options{}}\n\tdefer s.SetLogger(nil, false, false)\n\n\t// First check with no logger\n\ts.SetLogger(nil, false, false)\n\ts.ReOpenLogFile()\n\n\t// Then when LogFile is not provided.\n\tdl := &DummyLogger{}\n\ts.SetLogger(dl, false, false)\n\ts.ReOpenLogFile()\n\tdl.CheckContent(t, \"File log re-open ignored, not a file logger\")\n\n\t// Set a File log\n\ts.opts.LogFile = filepath.Join(t.TempDir(), \"test.log\")\n\tfileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC))\n\ts.SetLogger(fileLog, false, false)\n\t// Add some log\n\texpectedStr := \"This is a Notice\"\n\ts.Noticef(expectedStr)\n\t// Check content of log\n\tbuf, err := os.ReadFile(s.opts.LogFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tif !strings.Contains(string(buf), expectedStr) {\n\t\tt.Fatalf(\"Expected log to contain: %q, got %q\", expectedStr, string(buf))\n\t}\n\t// Close the file and rename it\n\tif err := os.Rename(s.opts.LogFile, s.opts.LogFile+\".bak\"); err != nil {\n\t\tt.Fatalf(\"Unable to rename log file: %v\", err)\n\t}\n\t// Now re-open LogFile\n\ts.ReOpenLogFile()\n\t// Content should indicate that we have re-opened the log\n\tbuf, err = os.ReadFile(s.opts.LogFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tif strings.HasSuffix(string(buf), \"File log-reopened\") {\n\t\tt.Fatalf(\"File should indicate that file log was re-opened, got: %v\", string(buf))\n\t}\n\t// Make sure we can append to the log\n\ts.Noticef(\"New message\")\n\tbuf, err = os.ReadFile(s.opts.LogFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tif strings.HasSuffix(string(buf), \"New message\") {\n\t\tt.Fatalf(\"New message was not appended after file was re-opened, got: %v\", string(buf))\n\t}\n}\n\nfunc TestFileLoggerSizeLimitAndReopen(t *testing.T) {\n\tfile := createTempFile(t, \"log_\")\n\tfile.Close()\n\n\ts := &Server{opts: &Options{}}\n\tdefer s.SetLogger(nil, false, false)\n\n\t// Set a File log\n\ts.opts.LogFile = file.Name()\n\ts.opts.Logtime = true\n\ts.opts.LogSizeLimit = 1000\n\ts.ConfigureLogger()\n\n\t// Add a trace\n\ts.Noticef(\"this is a notice\")\n\n\t// Do a re-open...\n\ts.ReOpenLogFile()\n\n\t// Content should indicate that we have re-opened the log\n\tbuf, err := os.ReadFile(s.opts.LogFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tif strings.HasSuffix(string(buf), \"File log-reopened\") {\n\t\tt.Fatalf(\"File should indicate that file log was re-opened, got: %v\", string(buf))\n\t}\n\n\t// Now make sure that the limit is still honored.\n\ttxt := make([]byte, 800)\n\tfor i := 0; i < len(txt); i++ {\n\t\ttxt[i] = 'A'\n\t}\n\ts.Noticef(string(txt))\n\tfor i := 0; i < len(txt); i++ {\n\t\ttxt[i] = 'B'\n\t}\n\ts.Noticef(string(txt))\n\n\tbuf, err = os.ReadFile(s.opts.LogFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tsbuf := string(buf)\n\tif strings.Contains(sbuf, \"AAAAA\") || strings.Contains(sbuf, \"BBBBB\") {\n\t\tt.Fatalf(\"Looks like file was not rotated: %s\", sbuf)\n\t}\n\tif !strings.Contains(sbuf, \"Rotated log, backup saved\") {\n\t\tt.Fatalf(\"File should have been rotated, was not: %s\", sbuf)\n\t}\n}\n\nfunc TestNoPasswordsFromConnectTrace(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoLog = false\n\topts.Trace = true\n\topts.Username = \"derek\"\n\topts.Password = \"s3cr3t\"\n\topts.PingInterval = 2 * time.Minute\n\tsetBaselineOptions(opts)\n\ts := &Server{opts: opts}\n\tdl := testhelper.NewDummyLogger(100)\n\ts.SetLogger(dl, false, true)\n\n\t_ = s.logging.logger.(*DummyLogger)\n\tif s.logging.trace != 1 {\n\t\tt.Fatalf(\"Expected trace 1, received value %d\\n\", s.logging.trace)\n\t}\n\tdefer s.SetLogger(nil, false, false)\n\n\tc, _, _ := newClientForServer(s)\n\tdefer c.close()\n\n\tconnectOp := []byte(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"s3cr3t\\\"}\\r\\n\")\n\terr := c.parse(connectOp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received error: %v\\n\", err)\n\t}\n\n\tdl.CheckForProhibited(t, \"password found\", \"s3cr3t\")\n}\n\nfunc TestRemovePassFromTrace(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\t\"user and pass\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass extra space\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\": \\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\": \\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass is empty\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass is empty whitespace\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\" \\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass whitespace\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\": \\\"s3cr3t\\\" }\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\": \\\"[REDACTED]\\\" }\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"only pass\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"s3cr3t\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json\",\n\t\t\t\"CONNECT {pass:s3cr3t , password = s3cr3t}\",\n\t\t\t\"CONNECT {pass:[REDACTED], password = [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json no whitespace after key\",\n\t\t\t\"CONNECT {pass:s3cr3t , password= s3cr3t}\",\n\t\t\t\"CONNECT {pass:[REDACTED], password= [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"both pass and wrong password key\",\n\t\t\t`CONNECT {\"pass\":\"s3cr3t4\", \"password\": \"s3cr3t4\"}`,\n\t\t\t`CONNECT {\"pass\":\"[REDACTED]\", \"password\": \"[REDACTED]\"}`,\n\t\t},\n\t\t{\n\t\t\t\"invalid json\",\n\t\t\t\"CONNECT {user = hello, password = s3cr3t}\",\n\t\t\t\"CONNECT {user = hello, password = [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"pass\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json with only pass key\",\n\t\t\t\"CONNECT {pass:s3cr3t\\r\\n\",\n\t\t\t\"CONNECT {pass:[REDACTED]\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid password key also filtered\",\n\t\t\t\"CONNECT {\\\"password\\\":\\\"s3cr3t\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"password\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"single long password with whitespace\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"secret password which is very long\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"single long pass key is filtered\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"duplicate keys all filtered\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\",\\\"pass\\\":\\\"BBBBBBBBBBBBBBBBBBBB\\\",\\\"password\\\":\\\"CCCCCCCCCCCCCCCC\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"pass\\\":\\\"[REDACTED]\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"password\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json with multiple keys all filtered\",\n\t\t\t\"CONNECT {pass = \\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\",pass= \\\"BBBBBBBBBBBBBBBBBBBB\\\",password =\\\"CCCCCCCCCCCCCCCC\\\"}\\r\\n\",\n\t\t\t\"CONNECT {pass = \\\"[REDACTED]\\\",pass= \\\"[REDACTED]\\\",password =\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect protocol\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"pass\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass are filterered\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect using password key with user and password being the same\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"...\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"...\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect with user password and name all the same\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect extra white space at the beginning\",\n\t\t\t\"CONNECT \t {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\t\"CONNECT \t {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toutput := removeSecretsFromTrace([]byte(test.input))\n\t\t\tif !bytes.Equal(output, []byte(test.expected)) {\n\t\t\t\tt.Errorf(\"\\nExpected %q\\n got: %q\", test.expected, string(output))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRemoveAuthTokenFromTrace(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\t\"user and auth_token\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass extra space\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\": \\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\": \\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass is empty\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\"\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass is empty whitespace\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\" \\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and pass whitespace\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\": \\\"s3cr3t\\\" }\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"auth_token\\\": \\\"[REDACTED]\\\" }\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"only pass\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"s3cr3t\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json\",\n\t\t\t\"CONNECT {auth_token:s3cr3t , password = s3cr3t}\",\n\t\t\t\"CONNECT {auth_token:[REDACTED], password = [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json no whitespace after key\",\n\t\t\t\"CONNECT {auth_token:s3cr3t , password= s3cr3t}\",\n\t\t\t\"CONNECT {auth_token:[REDACTED], password= [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"both pass and wrong password key\",\n\t\t\t`CONNECT {\"auth_token\":\"s3cr3t4\", \"password\": \"s3cr3t4\"}`,\n\t\t\t`CONNECT {\"auth_token\":\"[REDACTED]\", \"password\": \"[REDACTED]\"}`,\n\t\t},\n\t\t{\n\t\t\t\"invalid json\",\n\t\t\t\"CONNECT {user = hello, auth_token = s3cr3t}\",\n\t\t\t\"CONNECT {user = hello, auth_token = [REDACTED]}\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"auth_token\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json with only pass key\",\n\t\t\t\"CONNECT {auth_token:s3cr3t\\r\\n\",\n\t\t\t\"CONNECT {auth_token:[REDACTED]\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid password key also filtered\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"s3cr3t\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"single long password with whitespace\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"secret password which is very long\\\",}\\r\\n\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"[REDACTED]\\\",}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"single long pass key is filtered\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"duplicate keys all filtered\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\",\\\"auth_token\\\":\\\"BBB\\\",\\\"pass\\\":\\\"BBBBBBBBBBBBBBBBBBBB\\\",\\\"password\\\":\\\"CCCCCCCCCCCCCCCC\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"pass\\\":\\\"[REDACTED]\\\",\\\"password\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"invalid json with multiple keys all filtered\",\n\t\t\t\"CONNECT {auth_token = \\\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\\\",pass= \\\"BBBBBBBBBBBBBBBBBBBB\\\",password =\\\"CCCCCCCCCCCCCCCC\\\"}\\r\\n\",\n\t\t\t\"CONNECT {auth_token = \\\"[REDACTED]\\\",pass= \\\"[REDACTED]\\\",password =\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect protocol\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"auth_token\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"foo\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"APM7JU94z77YzP6WTBEiuw\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"user and token are filterered\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect using token key with user and token being the same\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"...\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"...\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect with user, token and name all the same\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t\t\"CONNECT {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"s3cr3t\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\t\"complete connect extra white space at the beginning\",\n\t\t\t\"CONNECT \t {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"s3cr3t\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\t\"CONNECT \t {\\\"echo\\\":true,\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"s3cr3t\\\",\\\"auth_token\\\":\\\"[REDACTED]\\\",\\\"tls_required\\\":false,\\\"name\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toutput := removeSecretsFromTrace([]byte(test.input))\n\t\t\tif !bytes.Equal(output, []byte(test.expected)) {\n\t\t\t\tt.Errorf(\"\\nExpected %q\\n got: %q\", test.expected, string(output))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRemoveAdditionalAuthFieldsFromTrace(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"sig\",\n\t\t\tinput: \"CONNECT {\\\"sig\\\":\\\"signature\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"sig\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"nkey\",\n\t\t\tinput: \"CONNECT {\\\"nkey\\\":\\\"nkey\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"nkey\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"proxy sig\",\n\t\t\tinput: \"CONNECT {\\\"proxy_sig\\\":\\\"proxy-signature\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"proxy_sig\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"proxy sig followed by sig\",\n\t\t\tinput: \"CONNECT {\\\"proxy_sig\\\":\\\"proxy-signature\\\",\\\"sig\\\":\\\"sig\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"proxy_sig\\\":\\\"[REDACTED]\\\",\\\"sig\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"sig followed by proxy sig\",\n\t\t\tinput: \"CONNECT {\\\"sig\\\":\\\"sig\\\",\\\"proxy_sig\\\":\\\"proxy-signature\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"sig\\\":\\\"[REDACTED]\\\",\\\"proxy_sig\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"sig marker inside another string still redacts real sig\",\n\t\t\tinput: \"CONNECT {\\\"name\\\":\\\"foo sig:bar\\\",\\\"sig\\\":\\\"REAL\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"name\\\":\\\"foo sig:[REDACTED]\\\",\\\"sig\\\":\\\"[REDACTED]\\\",\\\"user\\\":\\\"foo\\\"}\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"malformed sig without object delimiters\",\n\t\t\tinput: \"CONNECT sig=abc\\r\\n\",\n\t\t\texpected: \"CONNECT sig=[REDACTED]\\r\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"sig after malformed quoted field is redacted\",\n\t\t\tinput: \"CONNECT {\\\"name\\\":\\\"unterminated,\\\"sig\\\":\\\"abc\\\"}\\r\\n\",\n\t\t\texpected: \"CONNECT {\\\"name\\\":\\\"unterminated,\\\"sig\\\":\\\"[REDACTED]\\\"}\\r\\n\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toutput := removeSecretsFromTrace([]byte(test.input))\n\t\t\tif !bytes.Equal(output, []byte(test.expected)) {\n\t\t\t\tt.Errorf(\"\\nExpected %q\\n got: %q\", test.expected, string(output))\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/memstore.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\tcrand \"crypto/rand\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"math\"\n\t\"slices\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/ats\"\n\t\"github.com/nats-io/nats-server/v2/server/avl\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nats-server/v2/server/stree\"\n\t\"github.com/nats-io/nats-server/v2/server/thw\"\n)\n\n// TODO(dlc) - This is a fairly simplistic approach but should do for now.\ntype memStore struct {\n\tmu sync.RWMutex\n\tcfg StreamConfig\n\tstate StreamState\n\tmsgs map[uint64]*StoreMsg\n\tfss *stree.SubjectTree[SimpleState]\n\tdmap avl.SequenceSet\n\tmaxp int64\n\tscb StorageUpdateHandler\n\trmcb StorageRemoveMsgHandler\n\tpmsgcb ProcessJetStreamMsgHandler\n\tageChk *time.Timer // Timer to expire messages.\n\tageChkRun bool // Whether message expiration is currently running.\n\tageChkTime int64 // When the message expiration is scheduled to run.\n\tconsumers int\n\treceivedAny bool\n\tttls *thw.HashWheel\n\tscheduling *MsgScheduling\n\tsdm *SDMMeta\n}\n\nfunc newMemStore(cfg *StreamConfig) (*memStore, error) {\n\tif cfg == nil {\n\t\treturn nil, fmt.Errorf(\"config required\")\n\t}\n\tif cfg.Storage != MemoryStorage {\n\t\treturn nil, fmt.Errorf(\"memStore requires memory storage type in config\")\n\t}\n\tms := &memStore{\n\t\tmsgs: make(map[uint64]*StoreMsg),\n\t\tfss: stree.NewSubjectTree[SimpleState](),\n\t\tmaxp: cfg.MaxMsgsPer,\n\t\tcfg: *cfg,\n\t}\n\t// Only create a THW if we're going to allow TTLs.\n\tif cfg.AllowMsgTTL {\n\t\tms.ttls = thw.NewHashWheel()\n\t}\n\tif cfg.AllowMsgSchedules {\n\t\tms.scheduling = newMsgScheduling(ms.runMsgScheduling)\n\t}\n\tif cfg.FirstSeq > 0 {\n\t\tif _, err := ms.purge(cfg.FirstSeq); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Register with access time service.\n\tats.Register()\n\n\treturn ms, nil\n}\n\nfunc (ms *memStore) UpdateConfig(cfg *StreamConfig) error {\n\tif cfg == nil {\n\t\treturn fmt.Errorf(\"config required\")\n\t}\n\tif cfg.Storage != MemoryStorage {\n\t\treturn fmt.Errorf(\"memStore requires memory storage type in config\")\n\t}\n\n\tms.mu.Lock()\n\tms.cfg = *cfg\n\t// Create or delete the THW if needed.\n\tif cfg.AllowMsgTTL && ms.ttls == nil {\n\t\tms.recoverTTLState()\n\t} else if !cfg.AllowMsgTTL && ms.ttls != nil {\n\t\tms.ttls = nil\n\t}\n\tif cfg.AllowMsgSchedules && ms.scheduling == nil {\n\t\tms.recoverMsgSchedulingState()\n\t} else if !cfg.AllowMsgSchedules && ms.scheduling != nil {\n\t\tms.scheduling = nil\n\t}\n\t// Limits checks and enforcement.\n\tms.enforceMsgLimit()\n\tms.enforceBytesLimit()\n\t// Do age timers.\n\tif ms.ageChk == nil && ms.cfg.MaxAge != 0 {\n\t\tms.startAgeChk()\n\t}\n\tif ms.ageChk != nil && ms.cfg.MaxAge == 0 {\n\t\tms.ageChk.Stop()\n\t\tms.ageChk = nil\n\t\tms.ageChkTime = 0\n\t}\n\t// Make sure to update MaxMsgsPer\n\tif cfg.MaxMsgsPer < -1 {\n\t\tcfg.MaxMsgsPer = -1\n\t}\n\tmaxp := ms.maxp\n\tms.maxp = cfg.MaxMsgsPer\n\t// If the value is smaller, or was unset before, we need to enforce that.\n\tif ms.maxp > 0 && (maxp == 0 || ms.maxp < maxp) {\n\t\tlm := uint64(ms.maxp)\n\t\tms.fss.IterFast(func(subj []byte, ss *SimpleState) bool {\n\t\t\tif ss.Msgs > lm {\n\t\t\t\tms.enforcePerSubjectLimit(bytesToString(subj), ss)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n\tms.mu.Unlock()\n\n\tif cfg.MaxAge != 0 || cfg.AllowMsgTTL {\n\t\tms.expireMsgs()\n\t}\n\tif cfg.AllowMsgSchedules {\n\t\tms.runMsgScheduling()\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (ms *memStore) recoverTTLState() {\n\tms.ttls = thw.NewHashWheel()\n\tif ms.state.Msgs == 0 {\n\t\treturn\n\t}\n\n\tvar (\n\t\tseq uint64\n\t\tsmv StoreMsg\n\t\tsm *StoreMsg\n\t)\n\tdefer ms.resetAgeChk(0)\n\tfor sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, 0, &smv); sm != nil; sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, seq+1, &smv) {\n\t\tif len(sm.hdr) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif ttl, _ := getMessageTTL(sm.hdr); ttl > 0 {\n\t\t\texpires := time.Duration(sm.ts) + (time.Second * time.Duration(ttl))\n\t\t\tms.ttls.Add(seq, int64(expires))\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (ms *memStore) recoverMsgSchedulingState() {\n\tms.scheduling = newMsgScheduling(ms.runMsgScheduling)\n\tif ms.state.Msgs == 0 {\n\t\treturn\n\t}\n\n\tvar (\n\t\tseq uint64\n\t\tsmv StoreMsg\n\t\tsm *StoreMsg\n\t)\n\tdefer ms.scheduling.resetTimer()\n\tfor sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, 0, &smv); sm != nil; sm, seq, _ = ms.loadNextMsgLocked(fwcs, true, seq+1, &smv) {\n\t\tif len(sm.hdr) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif schedule, ok := nextMessageSchedule(sm.hdr, sm.ts); ok && !schedule.IsZero() {\n\t\t\tms.scheduling.init(seq, sm.subj, schedule.UnixNano())\n\t\t}\n\t}\n}\n\n// Stores a raw message with expected sequence number and timestamp.\n// Lock should be held.\nfunc (ms *memStore) storeRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) error {\n\tif ms.msgs == nil {\n\t\treturn ErrStoreClosed\n\t}\n\n\t// Tracking by subject.\n\tvar ss *SimpleState\n\tvar asl bool\n\tif len(subj) > 0 {\n\t\tvar ok bool\n\t\tif ss, ok = ms.fss.Find(stringToBytes(subj)); ok {\n\t\t\tasl = ms.maxp > 0 && ss.Msgs >= uint64(ms.maxp)\n\t\t}\n\t}\n\n\t// Check if we are discarding new messages when we reach the limit.\n\t// If we are clustered, we do the enforcement above and should not disqualify\n\t// the message here since it could cause replicas to drift.\n\tif discardNewCheck && ms.cfg.Discard == DiscardNew {\n\t\tif asl && ms.cfg.DiscardNewPer {\n\t\t\treturn ErrMaxMsgsPerSubject\n\t\t}\n\t\tif ms.cfg.MaxMsgs > 0 && ms.state.Msgs >= uint64(ms.cfg.MaxMsgs) {\n\t\t\t// If we are tracking max messages per subject and are at the limit we will replace, so this is ok.\n\t\t\tif !asl {\n\t\t\t\treturn ErrMaxMsgs\n\t\t\t}\n\t\t}\n\t\tif ms.cfg.MaxBytes > 0 && ms.state.Bytes+memStoreMsgSize(subj, hdr, msg) > uint64(ms.cfg.MaxBytes) {\n\t\t\tif !asl {\n\t\t\t\treturn ErrMaxBytes\n\t\t\t}\n\t\t\t// If we are here we are at a subject maximum, need to determine if dropping last message gives us enough room.\n\t\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\t\tms.recalculateForSubj(subj, ss)\n\t\t\t}\n\t\t\tsm, ok := ms.msgs[ss.First]\n\t\t\tif !ok || memStoreMsgSize(sm.subj, sm.hdr, sm.msg) < memStoreMsgSize(subj, hdr, msg) {\n\t\t\t\treturn ErrMaxBytes\n\t\t\t}\n\t\t}\n\t}\n\n\tif seq != ms.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn ErrSequenceMismatch\n\t\t}\n\t\tseq = ms.state.LastSeq + 1\n\t}\n\n\t// Adjust first if needed.\n\tnow := time.Unix(0, ts).UTC()\n\tif ms.state.Msgs == 0 {\n\t\tms.state.FirstSeq = seq\n\t\tms.state.FirstTime = now\n\t}\n\n\t// Make copies\n\t// TODO(dlc) - Maybe be smarter here.\n\tif len(msg) > 0 {\n\t\tmsg = copyBytes(msg)\n\t}\n\tif len(hdr) > 0 {\n\t\thdr = copyBytes(hdr)\n\t}\n\n\t// FIXME(dlc) - Could pool at this level?\n\tsm := &StoreMsg{subj, nil, nil, make([]byte, 0, len(hdr)+len(msg)), seq, ts}\n\tsm.buf = append(sm.buf, hdr...)\n\tsm.buf = append(sm.buf, msg...)\n\tif len(hdr) > 0 {\n\t\tsm.hdr = sm.buf[:len(hdr)]\n\t}\n\tsm.msg = sm.buf[len(hdr):]\n\tms.msgs[seq] = sm\n\tms.state.Msgs++\n\tms.state.Bytes += memStoreMsgSize(subj, hdr, msg)\n\tms.state.LastSeq = seq\n\tms.state.LastTime = now\n\n\t// Track per subject.\n\tif len(subj) > 0 {\n\t\tif ss != nil {\n\t\t\tss.Msgs++\n\t\t\tss.Last = seq\n\t\t\tss.lastNeedsUpdate = false\n\t\t\t// Check per subject limits.\n\t\t\tif ms.maxp > 0 && ss.Msgs > uint64(ms.maxp) {\n\t\t\t\tms.enforcePerSubjectLimit(subj, ss)\n\t\t\t}\n\t\t} else {\n\t\t\tms.fss.Insert([]byte(subj), SimpleState{Msgs: 1, First: seq, Last: seq})\n\t\t}\n\t}\n\n\t// Limits checks and enforcement.\n\tms.enforceMsgLimit()\n\tms.enforceBytesLimit()\n\n\t// Per-message TTL.\n\tif ms.ttls != nil && ttl > 0 {\n\t\texpires := time.Duration(ts) + (time.Second * time.Duration(ttl))\n\t\tms.ttls.Add(seq, int64(expires))\n\t}\n\n\t// Check if we have and need the age expiration timer running.\n\tswitch {\n\tcase ms.ttls != nil && ttl > 0:\n\t\tms.resetAgeChk(0)\n\tcase ms.ageChk == nil && (ms.cfg.MaxAge > 0 || ms.ttls != nil):\n\t\tms.startAgeChk()\n\t}\n\n\t// Message scheduling.\n\tif ms.scheduling != nil {\n\t\tif schedule, ok := nextMessageSchedule(hdr, ts); ok && !schedule.IsZero() {\n\t\t\tms.scheduling.add(seq, subj, schedule.UnixNano())\n\t\t} else {\n\t\t\tms.scheduling.removeSubject(subj)\n\t\t}\n\n\t\t// Check for a repeating schedule and update such that it triggers again.\n\t\tif scheduleNext := bytesToString(sliceHeader(JSScheduleNext, hdr)); scheduleNext != _EMPTY_ && scheduleNext != JSScheduleNextPurge {\n\t\t\tscheduler := getMessageScheduler(hdr)\n\t\t\tif next, err := time.Parse(time.RFC3339Nano, scheduleNext); err == nil && scheduler != _EMPTY_ {\n\t\t\t\tms.scheduling.update(scheduler, next.UnixNano())\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// StoreRawMsg stores a raw message with expected sequence number and timestamp.\nfunc (ms *memStore) StoreRawMsg(subj string, hdr, msg []byte, seq uint64, ts, ttl int64, discardNewCheck bool) error {\n\tms.mu.Lock()\n\terr := ms.storeRawMsg(subj, hdr, msg, seq, ts, ttl, discardNewCheck)\n\tcb := ms.scb\n\t// Check if first message timestamp requires expiry\n\t// sooner than initial replica expiry timer set to MaxAge when initializing.\n\tif !ms.receivedAny && ms.cfg.MaxAge != 0 && ts > 0 {\n\t\tms.receivedAny = true\n\t\t// Calculate duration when the next expireMsgs should be called.\n\t\tms.resetAgeChk(int64(time.Millisecond) * 50)\n\t}\n\tms.mu.Unlock()\n\n\tif err == nil && cb != nil {\n\t\tcb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj)\n\t}\n\n\treturn err\n}\n\n// Store stores a message.\nfunc (ms *memStore) StoreMsg(subj string, hdr, msg []byte, ttl int64) (uint64, int64, error) {\n\tms.mu.Lock()\n\tseq, ts := ms.state.LastSeq+1, time.Now().UnixNano()\n\t// This is called for a R1 with no expected sequence number, so perform DiscardNew checks on the store-level.\n\terr := ms.storeRawMsg(subj, hdr, msg, seq, ts, ttl, true)\n\tcb := ms.scb\n\tms.mu.Unlock()\n\n\tif err != nil {\n\t\tseq, ts = 0, 0\n\t} else if cb != nil {\n\t\tcb(1, int64(memStoreMsgSize(subj, hdr, msg)), seq, subj)\n\t}\n\n\treturn seq, ts, err\n}\n\n// SkipMsg will use the next sequence number but not store anything.\nfunc (ms *memStore) SkipMsg(seq uint64) (uint64, error) {\n\t// Grab time.\n\tnow := time.Unix(0, ats.AccessTime()).UTC()\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\t// Check sequence matches our last sequence.\n\tif seq != ms.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn 0, ErrSequenceMismatch\n\t\t}\n\t\tseq = ms.state.LastSeq + 1\n\t}\n\n\tms.state.LastSeq = seq\n\tms.state.LastTime = now\n\tif ms.state.Msgs == 0 {\n\t\tms.state.FirstSeq = seq + 1\n\t\tms.state.FirstTime = time.Time{}\n\t} else {\n\t\tms.dmap.Insert(seq)\n\t}\n\treturn seq, nil\n}\n\n// Skip multiple msgs.\nfunc (ms *memStore) SkipMsgs(seq uint64, num uint64) error {\n\t// Grab time.\n\tnow := time.Unix(0, ats.AccessTime()).UTC()\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\t// Check sequence matches our last sequence.\n\tif seq != ms.state.LastSeq+1 {\n\t\tif seq > 0 {\n\t\t\treturn ErrSequenceMismatch\n\t\t}\n\t\tseq = ms.state.LastSeq + 1\n\t}\n\tlseq := seq + num - 1\n\n\tms.state.LastSeq = lseq\n\tms.state.LastTime = now\n\tif ms.state.Msgs == 0 {\n\t\tms.state.FirstSeq, ms.state.FirstTime = lseq+1, time.Time{}\n\t} else {\n\t\tfor ; seq <= lseq; seq++ {\n\t\t\tms.dmap.Insert(seq)\n\t\t}\n\t}\n\treturn nil\n}\n\n// FlushAllPending flushes all data that was still pending to be written.\nfunc (ms *memStore) FlushAllPending() error {\n\t// Noop, in-memory store doesn't use async applying.\n\treturn nil\n}\n\n// RegisterStorageUpdates registers a callback for updates to storage changes.\n// It will present number of messages and bytes as a signed integer and an\n// optional sequence number of the message if a single.\nfunc (ms *memStore) RegisterStorageUpdates(cb StorageUpdateHandler) {\n\tms.mu.Lock()\n\tms.scb = cb\n\tms.mu.Unlock()\n}\n\n// RegisterStorageRemoveMsg registers a callback to remove messages.\n// Replicated streams should propose removals, R1 can remove inline.\nfunc (ms *memStore) RegisterStorageRemoveMsg(cb StorageRemoveMsgHandler) {\n\tms.mu.Lock()\n\tms.rmcb = cb\n\tms.mu.Unlock()\n}\n\n// RegisterProcessJetStreamMsg registers a callback to process new JetStream messages.\nfunc (ms *memStore) RegisterProcessJetStreamMsg(cb ProcessJetStreamMsgHandler) {\n\tms.mu.Lock()\n\tms.pmsgcb = cb\n\tms.mu.Unlock()\n}\n\n// GetSeqFromTime looks for the first sequence number that has the message\n// with >= timestamp.\nfunc (ms *memStore) GetSeqFromTime(t time.Time) uint64 {\n\tts := t.UnixNano()\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\tif len(ms.msgs) == 0 {\n\t\treturn ms.state.LastSeq + 1\n\t}\n\tif ts <= ms.msgs[ms.state.FirstSeq].ts {\n\t\treturn ms.state.FirstSeq\n\t}\n\t// LastSeq is not guaranteed to be present since last does not go backwards.\n\tvar lmsg *StoreMsg\n\tfor lseq := ms.state.LastSeq; lseq > ms.state.FirstSeq; lseq-- {\n\t\tif lmsg = ms.msgs[lseq]; lmsg != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tif lmsg == nil {\n\t\treturn ms.state.LastSeq + 1\n\t}\n\n\tlast := lmsg.ts\n\tif ts == last {\n\t\treturn lmsg.seq\n\t}\n\tif ts > last {\n\t\treturn ms.state.LastSeq + 1\n\t}\n\n\tvar (\n\t\tcts int64\n\t\tcseq uint64\n\t\toff uint64\n\t)\n\n\t// Using a binary search, but need to be aware of interior deletes.\n\tfseq := ms.state.FirstSeq\n\tlseq := ms.state.LastSeq\n\tseq := lseq + 1\nloop:\n\tfor fseq <= lseq {\n\t\tmid := fseq + (lseq-fseq)/2\n\t\toff = 0\n\t\t// Potentially skip over gaps. We keep the original middle but keep track of a\n\t\t// potential delete range with an offset.\n\t\tfor {\n\t\t\tmsg := ms.msgs[mid+off]\n\t\t\tif msg == nil {\n\t\t\t\toff++\n\t\t\t\tif mid+off <= lseq {\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\t// Continue search to the left. Purposely ignore the skipped deletes here.\n\t\t\t\t\tlseq = mid - 1\n\t\t\t\t\tcontinue loop\n\t\t\t\t}\n\t\t\t}\n\t\t\tcts = msg.ts\n\t\t\tcseq = msg.seq\n\t\t\tbreak\n\t\t}\n\t\tif cts >= ts {\n\t\t\tseq = cseq\n\t\t\tif mid == fseq {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// Continue search to the left.\n\t\t\tlseq = mid - 1\n\t\t} else {\n\t\t\t// Continue search to the right (potentially skipping over interior deletes).\n\t\t\tfseq = mid + off + 1\n\t\t}\n\t}\n\treturn seq\n}\n\n// FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence.\nfunc (ms *memStore) FilteredState(sseq uint64, subj string) (SimpleState, error) {\n\t// This needs to be a write lock, as filteredStateLocked can\n\t// mutate the per-subject state.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\treturn ms.filteredStateLocked(sseq, subj, false), nil\n}\n\nfunc (ms *memStore) filteredStateLocked(sseq uint64, filter string, lastPerSubject bool) SimpleState {\n\tif sseq < ms.state.FirstSeq {\n\t\tsseq = ms.state.FirstSeq\n\t}\n\n\t// If past the end no results.\n\tif sseq > ms.state.LastSeq {\n\t\treturn SimpleState{}\n\t}\n\n\tif filter == _EMPTY_ {\n\t\tfilter = fwcs\n\t}\n\tisAll := filter == fwcs\n\n\t// First check if we can optimize this part.\n\t// This means we want all and the starting sequence was before this block.\n\tif isAll && sseq <= ms.state.FirstSeq {\n\t\ttotal := ms.state.Msgs\n\t\tif lastPerSubject {\n\t\t\ttotal = uint64(ms.fss.Size())\n\t\t}\n\t\treturn SimpleState{\n\t\t\tMsgs: total,\n\t\t\tFirst: ms.state.FirstSeq,\n\t\t\tLast: ms.state.LastSeq,\n\t\t}\n\t}\n\n\t_tsa, _fsa := [32]string{}, [32]string{}\n\ttsa, fsa := _tsa[:0], _fsa[:0]\n\twc := subjectHasWildcard(filter)\n\tif wc {\n\t\tfsa = tokenizeSubjectIntoSlice(fsa[:0], filter)\n\t}\n\t// 1. See if we match any subs from fss.\n\t// 2. If we match and the sseq is past ss.Last then we can use meta only.\n\t// 3. If we match we need to do a partial, break and clear any totals and do a full scan like num pending.\n\n\tisMatch := func(subj string) bool {\n\t\tif isAll {\n\t\t\treturn true\n\t\t}\n\t\tif !wc {\n\t\t\treturn subj == filter\n\t\t}\n\t\ttsa = tokenizeSubjectIntoSlice(tsa[:0], subj)\n\t\treturn isSubsetMatchTokenized(tsa, fsa)\n\t}\n\n\tvar ss SimpleState\n\tupdate := func(fss *SimpleState) {\n\t\tmsgs, first, last := fss.Msgs, fss.First, fss.Last\n\t\tif lastPerSubject {\n\t\t\tmsgs, first = 1, last\n\t\t}\n\t\tss.Msgs += msgs\n\t\tif ss.First == 0 || first < ss.First {\n\t\t\tss.First = first\n\t\t}\n\t\tif last > ss.Last {\n\t\t\tss.Last = last\n\t\t}\n\t}\n\n\tvar havePartial bool\n\tvar totalSkipped uint64\n\t// We will track start and end sequences as we go.\n\tms.fss.Match(stringToBytes(filter), func(subj []byte, fss *SimpleState) {\n\t\tif fss.firstNeedsUpdate || fss.lastNeedsUpdate {\n\t\t\tms.recalculateForSubj(bytesToString(subj), fss)\n\t\t}\n\t\tif sseq <= fss.First {\n\t\t\tupdate(fss)\n\t\t} else if sseq <= fss.Last {\n\t\t\t// We matched but it is a partial.\n\t\t\thavePartial = true\n\t\t\t// Don't break here, we will update to keep tracking last.\n\t\t\tupdate(fss)\n\t\t} else {\n\t\t\ttotalSkipped += fss.Msgs\n\t\t}\n\t})\n\n\t// If we did not encounter any partials we can return here.\n\tif !havePartial {\n\t\treturn ss\n\t}\n\n\t// If we are here we need to scan the msgs.\n\t// Capture first and last sequences for scan and then clear what we had.\n\tfirst, last := ss.First, ss.Last\n\t// To track if we decide to exclude we need to calculate first.\n\tvar needScanFirst bool\n\tif first < sseq {\n\t\tfirst = sseq\n\t\tneedScanFirst = true\n\t}\n\n\t// Now we want to check if it is better to scan inclusive and recalculate that way\n\t// or leave and scan exclusive and adjust our totals.\n\t// ss.Last is always correct here.\n\ttoScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last\n\tvar seen map[string]bool\n\tif lastPerSubject {\n\t\tseen = make(map[string]bool)\n\t}\n\tif toScan < toExclude {\n\t\tss.Msgs, ss.First = 0, 0\n\n\t\tupdate := func(sm *StoreMsg) {\n\t\t\tss.Msgs++\n\t\t\tif ss.First == 0 {\n\t\t\t\tss.First = sm.seq\n\t\t\t}\n\t\t\tif seen != nil {\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\t// Check if easier to just scan msgs vs the sequence range.\n\t\t// This can happen with lots of interior deletes.\n\t\tif last-first > uint64(len(ms.msgs)) {\n\t\t\tfor _, sm := range ms.msgs {\n\t\t\t\tif sm.seq >= first && sm.seq <= last && !seen[sm.subj] && isMatch(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := first; seq <= last; seq++ {\n\t\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// We will adjust from the totals above by scanning what we need to exclude.\n\t\tss.First = first\n\t\tss.Msgs += totalSkipped\n\t\tvar adjust uint64\n\t\tvar tss *SimpleState\n\n\t\tupdate := func(sm *StoreMsg) {\n\t\t\tif lastPerSubject {\n\t\t\t\ttss, _ = ms.fss.Find(stringToBytes(sm.subj))\n\t\t\t}\n\t\t\t// If we are last per subject, make sure to only adjust if all messages are before our first.\n\t\t\tif tss == nil || tss.Last < first {\n\t\t\t\tadjust++\n\t\t\t}\n\t\t\tif seen != nil {\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\t// Check if easier to just scan msgs vs the sequence range.\n\t\tif first-ms.state.FirstSeq > uint64(len(ms.msgs)) {\n\t\t\tfor _, sm := range ms.msgs {\n\t\t\t\tif sm.seq < first && !seen[sm.subj] && isMatch(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := ms.state.FirstSeq; seq < first; seq++ {\n\t\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Now do range at end.\n\t\tfor seq := last + 1; seq < ms.state.LastSeq; seq++ {\n\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && isMatch(sm.subj) {\n\t\t\t\tadjust++\n\t\t\t\tif seen != nil {\n\t\t\t\t\tseen[sm.subj] = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tss.Msgs -= adjust\n\t\tif needScanFirst {\n\t\t\t// Check if easier to just scan msgs vs the sequence range.\n\t\t\t// Since we will need to scan all of the msgs vs below where we break on the first match,\n\t\t\t// we will only do so if a few orders of magnitude lower.\n\t\t\tif last-first > 100*uint64(len(ms.msgs)) {\n\t\t\t\tlow := ms.state.LastSeq\n\t\t\t\tfor _, sm := range ms.msgs {\n\t\t\t\t\tif sm.seq >= first && sm.seq < last && isMatch(sm.subj) {\n\t\t\t\t\t\tif sm.seq < low {\n\t\t\t\t\t\t\tlow = sm.seq\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif low < ms.state.LastSeq {\n\t\t\t\t\tss.First = low\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor seq := first; seq < last; seq++ {\n\t\t\t\t\tif sm, ok := ms.msgs[seq]; ok && isMatch(sm.subj) {\n\t\t\t\t\t\tss.First = seq\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn ss\n}\n\n// SubjectsState returns a map of SimpleState for all matching subjects.\nfunc (ms *memStore) SubjectsState(subject string) map[string]SimpleState {\n\t// This needs to be a write lock, as we can mutate the per-subject state.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tif ms.fss.Size() == 0 {\n\t\treturn nil\n\t}\n\n\tif subject == _EMPTY_ {\n\t\tsubject = fwcs\n\t}\n\n\tfss := make(map[string]SimpleState)\n\tms.fss.Match(stringToBytes(subject), func(subj []byte, ss *SimpleState) {\n\t\tsubjs := string(subj)\n\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\tms.recalculateForSubj(subjs, ss)\n\t\t}\n\t\toss := fss[subjs]\n\t\tif oss.First == 0 { // New\n\t\t\tfss[subjs] = *ss\n\t\t} else {\n\t\t\t// Merge here.\n\t\t\toss.Last, oss.Msgs = ss.Last, oss.Msgs+ss.Msgs\n\t\t\tfss[subjs] = oss\n\t\t}\n\t})\n\treturn fss\n}\n\n// AllLastSeqs will return a sorted list of last sequences for all subjects.\nfunc (ms *memStore) AllLastSeqs() ([]uint64, error) {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\treturn ms.allLastSeqsLocked()\n}\n\n// allLastSeqsLocked will return a sorted list of last sequences for all\n// subjects, but won't take the lock to do it, to avoid the issue of compounding\n// read locks causing a deadlock with a write lock.\nfunc (ms *memStore) allLastSeqsLocked() ([]uint64, error) {\n\tif len(ms.msgs) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tseqs := make([]uint64, 0, ms.fss.Size())\n\tms.fss.IterFast(func(subj []byte, ss *SimpleState) bool {\n\t\t// Check if we need to recalculate. We only care about the last sequence.\n\t\tif ss.lastNeedsUpdate {\n\t\t\tms.recalculateForSubj(bytesToString(subj), ss)\n\t\t}\n\t\tseqs = append(seqs, ss.Last)\n\t\treturn true\n\t})\n\n\tslices.Sort(seqs)\n\treturn seqs, nil\n}\n\n// Helper to determine if the filter(s) represent all the subjects.\n// Most clients send in subjects even if they match the stream's ingest subjects.\n// Lock should be held.\nfunc (ms *memStore) filterIsAll(filters []string) bool {\n\tif len(filters) != len(ms.cfg.Subjects) {\n\t\treturn false\n\t}\n\t// Sort so we can compare.\n\tslices.Sort(filters)\n\tslices.Sort(ms.cfg.Subjects)\n\tfor i, subj := range filters {\n\t\tif !subjectIsSubsetMatch(ms.cfg.Subjects[i], subj) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// MultiLastSeqs will return a sorted list of sequences that match all subjects presented in filters.\n// We will not exceed the maxSeq, which if 0 becomes the store's last sequence.\nfunc (ms *memStore) MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) {\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tif len(ms.msgs) == 0 {\n\t\treturn nil, nil\n\t}\n\n\t// See if we can short circuit if we think they are asking for all last sequences and have no maxSeq or maxAllowed set.\n\tif maxSeq == 0 && maxAllowed <= 0 && ms.filterIsAll(filters) {\n\t\treturn ms.allLastSeqsLocked()\n\t}\n\n\t// Implied last sequence.\n\tif maxSeq == 0 {\n\t\tmaxSeq = ms.state.LastSeq\n\t}\n\n\tseqs := make([]uint64, 0, 64)\n\tseen := make(map[uint64]struct{})\n\n\taddIfNotDupe := func(seq uint64) {\n\t\tif _, ok := seen[seq]; !ok {\n\t\t\tseqs = append(seqs, seq)\n\t\t\tseen[seq] = struct{}{}\n\t\t}\n\t}\n\n\tfor _, filter := range filters {\n\t\tms.fss.Match(stringToBytes(filter), func(subj []byte, ss *SimpleState) {\n\t\t\tif ss.lastNeedsUpdate {\n\t\t\t\tms.recalculateForSubj(bytesToString(subj), ss)\n\t\t\t}\n\t\t\tif ss.Last <= maxSeq {\n\t\t\t\taddIfNotDupe(ss.Last)\n\t\t\t} else if ss.Msgs > 1 {\n\t\t\t\t// The last is greater than maxSeq.\n\t\t\t\ts := bytesToString(subj)\n\t\t\t\tfor seq := maxSeq; seq > 0; seq-- {\n\t\t\t\t\tif sm, ok := ms.msgs[seq]; ok && sm.subj == s {\n\t\t\t\t\t\taddIfNotDupe(seq)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t\t// If maxAllowed was sepcified check that we will not exceed that.\n\t\tif maxAllowed > 0 && len(seqs) > maxAllowed {\n\t\t\treturn nil, ErrTooManyResults\n\t\t}\n\t}\n\tslices.Sort(seqs)\n\treturn seqs, nil\n}\n\n// SubjectsTotals return message totals per subject.\nfunc (ms *memStore) SubjectsTotals(filterSubject string) map[string]uint64 {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\treturn ms.subjectsTotalsLocked(filterSubject)\n}\n\n// Lock should be held.\nfunc (ms *memStore) subjectsTotalsLocked(filterSubject string) map[string]uint64 {\n\tif ms.fss.Size() == 0 {\n\t\treturn nil\n\t}\n\n\t_tsa, _fsa := [32]string{}, [32]string{}\n\ttsa, fsa := _tsa[:0], _fsa[:0]\n\tfsa = tokenizeSubjectIntoSlice(fsa[:0], filterSubject)\n\tisAll := filterSubject == _EMPTY_ || filterSubject == fwcs\n\n\tfst := make(map[string]uint64)\n\tms.fss.Match(stringToBytes(filterSubject), func(subj []byte, ss *SimpleState) {\n\t\tsubjs := string(subj)\n\t\tif isAll {\n\t\t\tfst[subjs] = ss.Msgs\n\t\t} else {\n\t\t\tif tsa = tokenizeSubjectIntoSlice(tsa[:0], subjs); isSubsetMatchTokenized(tsa, fsa) {\n\t\t\t\tfst[subjs] = ss.Msgs\n\t\t\t}\n\t\t}\n\t})\n\treturn fst\n}\n\n// NumPending will return the number of pending messages matching the filter subject starting at sequence.\nfunc (ms *memStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error) {\n\t// This needs to be a write lock, as filteredStateLocked can mutate the per-subject state.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tss := ms.filteredStateLocked(sseq, filter, lastPerSubject)\n\treturn ss.Msgs, ms.state.LastSeq, nil\n}\n\n// NumPending will return the number of pending messages matching any subject in the sublist starting at sequence.\nfunc (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error) {\n\tif sl == nil {\n\t\treturn ms.NumPending(sseq, fwcs, lastPerSubject)\n\t}\n\n\t// This needs to be a write lock, as we can mutate the per-subject state.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tvar ss SimpleState\n\tif sseq < ms.state.FirstSeq {\n\t\tsseq = ms.state.FirstSeq\n\t}\n\t// If past the end no results.\n\tif sseq > ms.state.LastSeq {\n\t\treturn 0, ms.state.LastSeq, nil\n\t}\n\n\tupdate := func(fss *SimpleState) {\n\t\tmsgs, first, last := fss.Msgs, fss.First, fss.Last\n\t\tif lastPerSubject {\n\t\t\tmsgs, first = 1, last\n\t\t}\n\t\tss.Msgs += msgs\n\t\tif ss.First == 0 || first < ss.First {\n\t\t\tss.First = first\n\t\t}\n\t\tif last > ss.Last {\n\t\t\tss.Last = last\n\t\t}\n\t}\n\n\tvar havePartial bool\n\tvar totalSkipped uint64\n\t// We will track start and end sequences as we go.\n\tstree.IntersectGSL[SimpleState](ms.fss, sl, func(subj []byte, fss *SimpleState) {\n\t\tif fss.firstNeedsUpdate || fss.lastNeedsUpdate {\n\t\t\tms.recalculateForSubj(bytesToString(subj), fss)\n\t\t}\n\t\tif sseq <= fss.First {\n\t\t\tupdate(fss)\n\t\t} else if sseq <= fss.Last {\n\t\t\t// We matched but it is a partial.\n\t\t\thavePartial = true\n\t\t\t// Don't break here, we will update to keep tracking last.\n\t\t\tupdate(fss)\n\t\t} else {\n\t\t\ttotalSkipped += fss.Msgs\n\t\t}\n\t})\n\n\t// If we did not encounter any partials we can return here.\n\tif !havePartial {\n\t\treturn ss.Msgs, ms.state.LastSeq, nil\n\t}\n\n\t// If we are here we need to scan the msgs.\n\t// Capture first and last sequences for scan and then clear what we had.\n\tfirst, last := ss.First, ss.Last\n\t// To track if we decide to exclude we need to calculate first.\n\tif first < sseq {\n\t\tfirst = sseq\n\t}\n\n\t// Now we want to check if it is better to scan inclusive and recalculate that way\n\t// or leave and scan exclusive and adjust our totals.\n\t// ss.Last is always correct here.\n\ttoScan, toExclude := last-first, first-ms.state.FirstSeq+ms.state.LastSeq-ss.Last\n\tvar seen map[string]bool\n\tif lastPerSubject {\n\t\tseen = make(map[string]bool)\n\t}\n\tif toScan < toExclude {\n\t\tss.Msgs, ss.First = 0, 0\n\n\t\tupdate := func(sm *StoreMsg) {\n\t\t\tss.Msgs++\n\t\t\tif ss.First == 0 {\n\t\t\t\tss.First = sm.seq\n\t\t\t}\n\t\t\tif seen != nil {\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\t// Check if easier to just scan msgs vs the sequence range.\n\t\t// This can happen with lots of interior deletes.\n\t\tif last-first > uint64(len(ms.msgs)) {\n\t\t\tfor _, sm := range ms.msgs {\n\t\t\t\tif sm.seq >= first && sm.seq <= last && !seen[sm.subj] && sl.HasInterest(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := first; seq <= last; seq++ {\n\t\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// We will adjust from the totals above by scanning what we need to exclude.\n\t\tss.First = first\n\t\tss.Msgs += totalSkipped\n\t\tvar adjust uint64\n\t\tvar tss *SimpleState\n\n\t\tupdate := func(sm *StoreMsg) {\n\t\t\tif lastPerSubject {\n\t\t\t\ttss, _ = ms.fss.Find(stringToBytes(sm.subj))\n\t\t\t}\n\t\t\t// If we are last per subject, make sure to only adjust if all messages are before our first.\n\t\t\tif tss == nil || tss.Last < first {\n\t\t\t\tadjust++\n\t\t\t}\n\t\t\tif seen != nil {\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\t// Check if easier to just scan msgs vs the sequence range.\n\t\tif first-ms.state.FirstSeq > uint64(len(ms.msgs)) {\n\t\t\tfor _, sm := range ms.msgs {\n\t\t\t\tif sm.seq < first && !seen[sm.subj] && sl.HasInterest(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := ms.state.FirstSeq; seq < first; seq++ {\n\t\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) {\n\t\t\t\t\tupdate(sm)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Now do range at end.\n\t\tfor seq := last + 1; seq < ms.state.LastSeq; seq++ {\n\t\t\tif sm, ok := ms.msgs[seq]; ok && !seen[sm.subj] && sl.HasInterest(sm.subj) {\n\t\t\t\tadjust++\n\t\t\t\tif seen != nil {\n\t\t\t\t\tseen[sm.subj] = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tss.Msgs -= adjust\n\t}\n\n\treturn ss.Msgs, ms.state.LastSeq, nil\n}\n\n// Will check the msg limit for this tracked subject.\n// Lock should be held.\nfunc (ms *memStore) enforcePerSubjectLimit(subj string, ss *SimpleState) {\n\tif ms.maxp <= 0 {\n\t\treturn\n\t}\n\tfor nmsgs := ss.Msgs; nmsgs > uint64(ms.maxp); nmsgs = ss.Msgs {\n\t\tif ss.firstNeedsUpdate || ss.lastNeedsUpdate {\n\t\t\tms.recalculateForSubj(subj, ss)\n\t\t}\n\t\tif !ms.removeMsg(ss.First, false) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// Will check the msg limit and drop firstSeq msg if needed.\n// Lock should be held.\nfunc (ms *memStore) enforceMsgLimit() {\n\tif ms.cfg.Discard != DiscardOld {\n\t\treturn\n\t}\n\tif ms.cfg.MaxMsgs <= 0 || ms.state.Msgs <= uint64(ms.cfg.MaxMsgs) {\n\t\treturn\n\t}\n\tfor nmsgs := ms.state.Msgs; nmsgs > uint64(ms.cfg.MaxMsgs); nmsgs = ms.state.Msgs {\n\t\tms.deleteFirstMsgOrPanic()\n\t}\n}\n\n// Will check the bytes limit and drop msgs if needed.\n// Lock should be held.\nfunc (ms *memStore) enforceBytesLimit() {\n\tif ms.cfg.Discard != DiscardOld {\n\t\treturn\n\t}\n\tif ms.cfg.MaxBytes <= 0 || ms.state.Bytes <= uint64(ms.cfg.MaxBytes) {\n\t\treturn\n\t}\n\tfor bs := ms.state.Bytes; bs > uint64(ms.cfg.MaxBytes); bs = ms.state.Bytes {\n\t\tms.deleteFirstMsgOrPanic()\n\t}\n}\n\n// Will start the age check timer.\n// Lock should be held.\nfunc (ms *memStore) startAgeChk() {\n\tif ms.ageChk != nil {\n\t\treturn\n\t}\n\tif ms.cfg.MaxAge != 0 || ms.ttls != nil {\n\t\tms.ageChk = time.AfterFunc(ms.cfg.MaxAge, ms.expireMsgs)\n\t}\n}\n\n// Lock should be held.\nfunc (ms *memStore) resetAgeChk(delta int64) {\n\t// If we're already expiring messages, it will make sure to reset.\n\t// Don't trigger again, as that could result in many expire goroutines.\n\tif ms.ageChkRun {\n\t\treturn\n\t}\n\n\tvar next int64 = math.MaxInt64\n\tif ms.ttls != nil {\n\t\tnext = ms.ttls.GetNextExpiration(next)\n\t}\n\n\t// If there's no MaxAge and there's nothing waiting to be expired then\n\t// don't bother continuing. The next storeRawMsg() will wake us up if\n\t// needs be.\n\tif ms.cfg.MaxAge <= 0 && next == math.MaxInt64 {\n\t\tclearTimer(&ms.ageChk)\n\t\treturn\n\t}\n\n\t// Check to see if we should be firing sooner than MaxAge for an expiring TTL.\n\tfireIn := ms.cfg.MaxAge\n\n\t// If delta for next-to-expire message is unset, but we still have messages to remove.\n\t// Assume messages are removed through proposals, and we need to speed up subsequent age check.\n\tif delta == 0 && ms.state.Msgs > 0 {\n\t\tif until := 2 * time.Second; until < fireIn {\n\t\t\tfireIn = until\n\t\t}\n\t}\n\n\tif next < math.MaxInt64 {\n\t\t// Looks like there's a next expiration, use it either if there's no\n\t\t// MaxAge set or if it looks to be sooner than MaxAge is.\n\t\tif until := time.Until(time.Unix(0, next)); fireIn == 0 || until < fireIn {\n\t\t\tfireIn = until\n\t\t}\n\t}\n\n\t// If not then look at the delta provided (usually gap to next age expiry).\n\tif delta > 0 {\n\t\tif fireIn == 0 || time.Duration(delta) < fireIn {\n\t\t\tfireIn = time.Duration(delta)\n\t\t}\n\t}\n\n\t// Make sure we aren't firing too often either way, otherwise we can\n\t// negatively impact stream ingest performance.\n\tif fireIn < 250*time.Millisecond {\n\t\tfireIn = 250 * time.Millisecond\n\t}\n\n\t// If we want to kick the timer to run later than what was assigned before, don't reset it.\n\t// Otherwise, we could get in a situation where the timer is continuously reset, and it never runs.\n\texpires := ats.AccessTime() + fireIn.Nanoseconds()\n\tif ms.ageChkTime > 0 && expires > ms.ageChkTime {\n\t\treturn\n\t}\n\n\tms.ageChkTime = expires\n\tif ms.ageChk != nil {\n\t\tms.ageChk.Reset(fireIn)\n\t} else {\n\t\tms.ageChk = time.AfterFunc(fireIn, ms.expireMsgs)\n\t}\n}\n\n// Lock should be held.\nfunc (ms *memStore) cancelAgeChk() {\n\tif ms.ageChk != nil {\n\t\tms.ageChk.Stop()\n\t\tms.ageChk = nil\n\t\tms.ageChkTime = 0\n\t}\n}\n\n// Will expire msgs that are too old.\nfunc (ms *memStore) expireMsgs() {\n\tvar smv StoreMsg\n\tvar sm *StoreMsg\n\tms.mu.Lock()\n\tmaxAge := int64(ms.cfg.MaxAge)\n\tminAge := time.Now().UnixNano() - maxAge\n\trmcb := ms.rmcb\n\tpmsgcb := ms.pmsgcb\n\tsdmTTL := int64(ms.cfg.SubjectDeleteMarkerTTL.Seconds())\n\tsdmEnabled := sdmTTL > 0\n\n\t// If SDM is enabled, but handlers aren't set up yet. Try again later.\n\tif sdmEnabled && (rmcb == nil || pmsgcb == nil) {\n\t\tms.resetAgeChk(0)\n\t\tms.mu.Unlock()\n\t\treturn\n\t}\n\tms.ageChkRun = true\n\tms.mu.Unlock()\n\n\tif maxAge > 0 {\n\t\tvar seq uint64\n\t\tfor sm, seq, _ = ms.LoadNextMsg(fwcs, true, 0, &smv); sm != nil && sm.ts <= minAge; sm, seq, _ = ms.LoadNextMsg(fwcs, true, seq+1, &smv) {\n\t\t\tif len(sm.hdr) > 0 {\n\t\t\t\tif ttl, err := getMessageTTL(sm.hdr); err == nil && ttl < 0 {\n\t\t\t\t\t// The message has a negative TTL, therefore it must \"never expire\".\n\t\t\t\t\tminAge = time.Now().UnixNano() - maxAge\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif sdmEnabled {\n\t\t\t\tif last, ok := ms.shouldProcessSdm(seq, sm.subj); ok {\n\t\t\t\t\tsdm := last && !isSubjectDeleteMarker(sm.hdr)\n\t\t\t\t\tms.handleRemovalOrSdm(seq, sm.subj, sdm, sdmTTL)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tms.mu.Lock()\n\t\t\t\tms.removeMsg(seq, false)\n\t\t\t\tms.mu.Unlock()\n\t\t\t}\n\t\t\t// Recalculate in case we are expiring a bunch.\n\t\t\tminAge = time.Now().UnixNano() - maxAge\n\t\t}\n\t}\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\t// TODO: Not great that we're holding the lock here, but the timed hash wheel isn't thread-safe.\n\tnextTTL := int64(math.MaxInt64)\n\tvar rmSeqs []thw.HashWheelEntry\n\tif ms.ttls != nil {\n\t\tms.ttls.ExpireTasks(func(seq uint64, ts int64) bool {\n\t\t\trmSeqs = append(rmSeqs, thw.HashWheelEntry{Seq: seq, Expires: ts})\n\t\t\t// We might need to remove messages out of band, those can fail, and we can be shutdown halfway\n\t\t\t// through so don't remove from THW just yet.\n\t\t\treturn false\n\t\t})\n\t\tif maxAge > 0 {\n\t\t\t// Only check if we're expiring something in the next MaxAge interval, saves us a bit\n\t\t\t// of work if MaxAge will beat us to the next expiry anyway.\n\t\t\tnextTTL = ms.ttls.GetNextExpiration(time.Now().Add(time.Duration(maxAge)).UnixNano())\n\t\t} else {\n\t\t\tnextTTL = ms.ttls.GetNextExpiration(math.MaxInt64)\n\t\t}\n\t}\n\n\t// Remove messages collected by THW.\n\tif !sdmEnabled {\n\t\tfor _, rm := range rmSeqs {\n\t\t\tms.removeMsg(rm.Seq, false)\n\t\t}\n\t} else {\n\t\t// THW is unordered, so must sort by sequence and must not be holding the lock.\n\t\tms.mu.Unlock()\n\t\tslices.SortFunc(rmSeqs, func(a, b thw.HashWheelEntry) int {\n\t\t\tif a.Seq == b.Seq {\n\t\t\t\treturn 0\n\t\t\t} else if a.Seq < b.Seq {\n\t\t\t\treturn -1\n\t\t\t} else {\n\t\t\t\treturn 1\n\t\t\t}\n\t\t})\n\t\tfor _, rm := range rmSeqs {\n\t\t\t// Need to grab subject for the specified sequence if for SDM, and check\n\t\t\t// if the message hasn't been removed in the meantime.\n\t\t\t// We need to grab the message and check if we should process SDM while holding the lock,\n\t\t\t// otherwise we can race if a deletion of this message is in progress.\n\t\t\tms.mu.Lock()\n\t\t\tsm, _ = ms.loadMsgLocked(rm.Seq, &smv, false)\n\t\t\tif sm == nil {\n\t\t\t\tms.ttls.Remove(rm.Seq, rm.Expires)\n\t\t\t\tms.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlast, ok := ms.shouldProcessSdmLocked(rm.Seq, sm.subj)\n\t\t\tms.mu.Unlock()\n\t\t\tif ok {\n\t\t\t\tsdm := last && !isSubjectDeleteMarker(sm.hdr)\n\t\t\t\tms.handleRemovalOrSdm(rm.Seq, sm.subj, sdm, sdmTTL)\n\t\t\t}\n\t\t}\n\t\tms.mu.Lock()\n\t}\n\n\t// Only cancel if no message left, not on potential lookup error that would result in sm == nil.\n\tms.ageChkRun, ms.ageChkTime = false, 0\n\tif ms.state.Msgs == 0 && nextTTL == math.MaxInt64 {\n\t\tms.cancelAgeChk()\n\t} else {\n\t\tif sm == nil {\n\t\t\tms.resetAgeChk(0)\n\t\t} else {\n\t\t\tms.resetAgeChk(sm.ts - minAge)\n\t\t}\n\t}\n}\n\nfunc (ms *memStore) shouldProcessSdm(seq uint64, subj string) (bool, bool) {\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\treturn ms.shouldProcessSdmLocked(seq, subj)\n}\n\n// Lock should be held.\nfunc (ms *memStore) shouldProcessSdmLocked(seq uint64, subj string) (bool, bool) {\n\tif ms.sdm == nil {\n\t\tms.sdm = newSDMMeta()\n\t}\n\n\tif p, ok := ms.sdm.pending[seq]; ok {\n\t\t// Don't allow more proposals for the same sequence if we already did recently.\n\t\tif time.Since(time.Unix(0, p.ts)) < 2*time.Second {\n\t\t\treturn p.last, false\n\t\t}\n\t\t// If we're about to use the cached value, and we knew it was last before,\n\t\t// quickly check that we don't have more remaining messages for the subject now.\n\t\t// Which means we are not the last anymore and must reset to not remove later data.\n\t\tif p.last {\n\t\t\tmsgs := ms.subjectsTotalsLocked(subj)[subj]\n\t\t\tnumPending := ms.sdm.totals[subj]\n\t\t\tif remaining := msgs - numPending; remaining > 0 {\n\t\t\t\tp.last = false\n\t\t\t}\n\t\t}\n\t\tms.sdm.pending[seq] = SDMBySeq{p.last, time.Now().UnixNano()}\n\t\treturn p.last, true\n\t}\n\n\tmsgs := ms.subjectsTotalsLocked(subj)[subj]\n\tif msgs == 0 {\n\t\treturn false, true\n\t}\n\tnumPending := ms.sdm.totals[subj]\n\tremaining := msgs - numPending\n\treturn ms.sdm.trackPending(seq, subj, remaining == 1), true\n}\n\nfunc (ms *memStore) handleRemovalOrSdm(seq uint64, subj string, sdm bool, sdmTTL int64) {\n\tif sdm {\n\t\tvar _hdr [128]byte\n\t\thdr := fmt.Appendf(\n\t\t\t_hdr[:0],\n\t\t\t\"NATS/1.0\\r\\n%s: %s\\r\\n%s: %s\\r\\n%s: %s\\r\\n\\r\\n\",\n\t\t\tJSMarkerReason, JSMarkerReasonMaxAge,\n\t\t\tJSMessageTTL, time.Duration(sdmTTL)*time.Second,\n\t\t\tJSMsgRollup, JSMsgRollupSubject,\n\t\t)\n\t\tmsg := &inMsg{\n\t\t\tsubj: subj,\n\t\t\thdr: hdr,\n\t\t}\n\t\tms.pmsgcb(msg)\n\t} else {\n\t\tms.rmcb(seq)\n\t}\n}\n\n// Will run through scheduled messages.\nfunc (ms *memStore) runMsgScheduling() {\n\t// TODO: Not great that we're holding the lock here, but the timed hash wheel and message scheduling isn't thread-safe.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\t// If scheduling is enabled, but handler isn't set up yet. Try again later.\n\tif ms.scheduling == nil {\n\t\treturn\n\t}\n\tif ms.pmsgcb == nil {\n\t\tms.scheduling.resetTimer()\n\t\treturn\n\t}\n\tms.scheduling.running = true\n\n\tscheduledMsgs := ms.scheduling.getScheduledMessages(\n\t\tfunc(seq uint64, smv *StoreMsg) *StoreMsg {\n\t\t\tsm, _ := ms.loadMsgLocked(seq, smv, false)\n\t\t\treturn sm\n\t\t},\n\t\tfunc(subj string, smv *StoreMsg) *StoreMsg {\n\t\t\tsm, _ := ms.loadLastLocked(subj, smv)\n\t\t\treturn sm\n\t\t},\n\t)\n\tif len(scheduledMsgs) > 0 {\n\t\tms.mu.Unlock()\n\t\tfor _, msg := range scheduledMsgs {\n\t\t\tms.pmsgcb(msg)\n\t\t}\n\t\tms.mu.Lock()\n\t}\n\n\tms.scheduling.running, ms.scheduling.deadline = false, 0\n\tms.scheduling.resetTimer()\n}\n\n// PurgeEx will remove messages based on subject filters, sequence and number of messages to keep.\n// Will return the number of purged messages.\nfunc (ms *memStore) PurgeEx(subject string, sequence, keep uint64) (purged uint64, err error) {\n\tif subject == _EMPTY_ || subject == fwcs {\n\t\tif keep == 0 && sequence == 0 {\n\t\t\treturn ms.purge(0)\n\t\t}\n\t\tif sequence > 1 {\n\t\t\treturn ms.compact(sequence)\n\t\t} else if keep > 0 {\n\t\t\tms.mu.RLock()\n\t\t\tmsgs, lseq := ms.state.Msgs, ms.state.LastSeq\n\t\t\tms.mu.RUnlock()\n\t\t\tif keep >= msgs {\n\t\t\t\treturn 0, nil\n\t\t\t}\n\t\t\treturn ms.compact(lseq - keep + 1)\n\t\t}\n\t\treturn 0, nil\n\n\t}\n\teq := compareFn(subject)\n\tif ss, _ := ms.FilteredState(1, subject); ss.Msgs > 0 {\n\t\tif keep > 0 {\n\t\t\tif keep >= ss.Msgs {\n\t\t\t\treturn 0, nil\n\t\t\t}\n\t\t\tss.Msgs -= keep\n\t\t}\n\t\tlast := ss.Last\n\t\tif sequence > 1 {\n\t\t\tlast = sequence - 1\n\t\t}\n\t\tms.mu.Lock()\n\t\tfor seq := ss.First; seq <= last; seq++ {\n\t\t\tif sm, ok := ms.msgs[seq]; ok && eq(sm.subj, subject) {\n\t\t\t\tif ok := ms.removeMsg(sm.seq, false); ok {\n\t\t\t\t\tpurged++\n\t\t\t\t\tif purged >= ss.Msgs {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tms.mu.Unlock()\n\t}\n\treturn purged, nil\n}\n\n// Purge will remove all messages from this store.\n// Will return the number of purged messages.\nfunc (ms *memStore) Purge() (uint64, error) {\n\treturn ms.purge(0)\n}\n\nfunc (ms *memStore) purge(fseq uint64) (uint64, error) {\n\tms.mu.Lock()\n\tpurged := uint64(len(ms.msgs))\n\tcb := ms.scb\n\tbytes := int64(ms.state.Bytes)\n\tif fseq == 0 {\n\t\tfseq = ms.state.LastSeq + 1\n\t} else if fseq < ms.state.LastSeq {\n\t\tms.mu.Unlock()\n\t\treturn 0, fmt.Errorf(\"partial purges not supported on memory store\")\n\t}\n\tms.state.FirstSeq = fseq\n\tms.state.LastSeq = fseq - 1\n\tms.state.FirstTime = time.Time{}\n\tms.state.Bytes = 0\n\tms.state.Msgs = 0\n\tif ms.msgs != nil {\n\t\tms.msgs = make(map[uint64]*StoreMsg)\n\t}\n\tms.fss = stree.NewSubjectTree[SimpleState]()\n\tms.dmap.Empty()\n\tms.sdm.empty()\n\tms.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -bytes, 0, _EMPTY_)\n\t}\n\n\treturn purged, nil\n}\n\n// Compact will remove all messages from this store up to\n// but not including the seq parameter.\n// Will return the number of purged messages.\nfunc (ms *memStore) Compact(seq uint64) (uint64, error) {\n\treturn ms.compact(seq)\n}\n\nfunc (ms *memStore) compact(seq uint64) (uint64, error) {\n\tif seq == 0 {\n\t\treturn ms.Purge()\n\t}\n\n\tvar purged, bytes uint64\n\n\tms.mu.Lock()\n\t// Short-circuit if the store was already compacted past this point.\n\tif ms.state.FirstSeq > seq {\n\t\tms.mu.Unlock()\n\t\treturn purged, nil\n\t}\n\n\tcb := ms.scb\n\tif seq <= ms.state.LastSeq {\n\t\tfseq := ms.state.FirstSeq\n\t\t// Determine new first sequence.\n\t\tfor ; seq <= ms.state.LastSeq; seq++ {\n\t\t\tif sm, ok := ms.msgs[seq]; ok {\n\t\t\t\tms.state.FirstSeq = seq\n\t\t\t\tms.state.FirstTime = time.Unix(0, sm.ts).UTC()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tfor seq := seq - 1; seq >= fseq; seq-- {\n\t\t\tif sm := ms.msgs[seq]; sm != nil {\n\t\t\t\tbytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t\t\tpurged++\n\t\t\t\tms.removeSeqPerSubject(sm.subj, seq)\n\t\t\t\t// Must delete message after updating per-subject info, to be consistent with file store.\n\t\t\t\tdelete(ms.msgs, seq)\n\t\t\t} else if !ms.dmap.IsEmpty() {\n\t\t\t\tms.dmap.Delete(seq)\n\t\t\t}\n\t\t}\n\t\tif purged > ms.state.Msgs {\n\t\t\tpurged = ms.state.Msgs\n\t\t}\n\t\tms.state.Msgs -= purged\n\t\tif bytes > ms.state.Bytes {\n\t\t\tbytes = ms.state.Bytes\n\t\t}\n\t\tms.state.Bytes -= bytes\n\t} else {\n\t\t// We are compacting past the end of our range. Do purge and set sequences correctly\n\t\t// such that the next message placed will have seq.\n\t\tpurged = uint64(len(ms.msgs))\n\t\tbytes = ms.state.Bytes\n\t\tms.state.Bytes = 0\n\t\tms.state.Msgs = 0\n\t\tms.state.FirstSeq = seq\n\t\tms.state.FirstTime = time.Time{}\n\t\tms.state.LastSeq = seq - 1\n\t\t// Reset msgs, fss and dmap.\n\t\tms.msgs = make(map[uint64]*StoreMsg)\n\t\tms.fss = stree.NewSubjectTree[SimpleState]()\n\t\tms.dmap.Empty()\n\t\tms.sdm.empty()\n\t}\n\tms.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn purged, nil\n}\n\n// Will completely reset our store.\nfunc (ms *memStore) reset() error {\n\tms.mu.Lock()\n\tvar purged, bytes uint64\n\tcb := ms.scb\n\tif cb != nil {\n\t\tfor _, sm := range ms.msgs {\n\t\t\tpurged++\n\t\t\tbytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t}\n\t}\n\n\t// Reset\n\tms.state.FirstSeq = 0\n\tms.state.FirstTime = time.Time{}\n\tms.state.LastSeq = 0\n\tms.state.LastTime = time.Now().UTC()\n\t// Update msgs and bytes.\n\tms.state.Msgs = 0\n\tms.state.Bytes = 0\n\t// Reset msgs, fss and dmap.\n\tms.msgs = make(map[uint64]*StoreMsg)\n\tms.fss = stree.NewSubjectTree[SimpleState]()\n\tms.dmap.Empty()\n\tms.sdm.empty()\n\n\tms.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn nil\n}\n\n// Truncate will truncate a stream store up to seq. Sequence needs to be valid.\nfunc (ms *memStore) Truncate(seq uint64) error {\n\t// Check for request to reset.\n\tif seq == 0 {\n\t\treturn ms.reset()\n\t}\n\n\tvar purged, bytes uint64\n\n\tms.mu.Lock()\n\tlsm, ok := ms.msgs[seq]\n\tlastTime := ms.state.LastTime\n\tif ok && lsm != nil {\n\t\tlastTime = time.Unix(0, lsm.ts).UTC()\n\t}\n\n\tfor i := ms.state.LastSeq; i > seq; i-- {\n\t\tif sm := ms.msgs[i]; sm != nil {\n\t\t\tpurged++\n\t\t\tbytes += memStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\t\t\tms.removeSeqPerSubject(sm.subj, i)\n\t\t\t// Must delete message after updating per-subject info, to be consistent with file store.\n\t\t\tdelete(ms.msgs, i)\n\t\t} else if !ms.dmap.IsEmpty() {\n\t\t\tms.dmap.Delete(i)\n\t\t}\n\t}\n\t// Reset last.\n\tms.state.LastSeq = seq\n\tms.state.LastTime = lastTime\n\t// Update msgs and bytes.\n\tif purged > ms.state.Msgs {\n\t\tpurged = ms.state.Msgs\n\t}\n\tms.state.Msgs -= purged\n\tif bytes > ms.state.Bytes {\n\t\tbytes = ms.state.Bytes\n\t}\n\tms.state.Bytes -= bytes\n\n\tcb := ms.scb\n\tms.mu.Unlock()\n\n\tif cb != nil {\n\t\tcb(-int64(purged), -int64(bytes), 0, _EMPTY_)\n\t}\n\n\treturn nil\n}\n\nfunc (ms *memStore) deleteFirstMsgOrPanic() {\n\tif !ms.deleteFirstMsg() {\n\t\tpanic(\"jetstream memstore has inconsistent state, can't find first seq msg\")\n\t}\n}\n\nfunc (ms *memStore) deleteFirstMsg() bool {\n\treturn ms.removeMsg(ms.state.FirstSeq, false)\n}\n\n// SubjectForSeq will return what the subject is for this sequence if found.\nfunc (ms *memStore) SubjectForSeq(seq uint64) (string, error) {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\tif seq < ms.state.FirstSeq {\n\t\treturn _EMPTY_, ErrStoreMsgNotFound\n\t}\n\tif sm, ok := ms.msgs[seq]; ok {\n\t\t// Copy the subject, as it's used elsewhere, and we've released the lock in the meantime.\n\t\treturn copyString(sm.subj), nil\n\t}\n\treturn _EMPTY_, ErrStoreMsgNotFound\n}\n\n// LoadMsg will lookup the message by sequence number and return it if found.\nfunc (ms *memStore) LoadMsg(seq uint64, smp *StoreMsg) (*StoreMsg, error) {\n\treturn ms.loadMsgLocked(seq, smp, true)\n}\n\n// loadMsgLocked will lookup the message by sequence number and return it if found.\nfunc (ms *memStore) loadMsgLocked(seq uint64, smp *StoreMsg, needMSLock bool) (*StoreMsg, error) {\n\tif needMSLock {\n\t\tms.mu.RLock()\n\t}\n\tsm, ok := ms.msgs[seq]\n\tlast := ms.state.LastSeq\n\tif needMSLock {\n\t\tms.mu.RUnlock()\n\t}\n\n\tif !ok || sm == nil {\n\t\tvar err = ErrStoreEOF\n\t\tif seq <= last {\n\t\t\terr = ErrStoreMsgNotFound\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tif smp == nil {\n\t\tsmp = new(StoreMsg)\n\t}\n\tsm.copy(smp)\n\treturn smp, nil\n}\n\n// LoadLastMsg will return the last message we have that matches a given subject.\n// The subject can be a wildcard.\nfunc (ms *memStore) LoadLastMsg(subject string, smp *StoreMsg) (*StoreMsg, error) {\n\t// This needs to be a write lock, as filteredStateLocked can\n\t// mutate the per-subject state.\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\treturn ms.loadLastLocked(subject, smp)\n}\n\n// Lock should be held.\nfunc (ms *memStore) loadLastLocked(subject string, smp *StoreMsg) (*StoreMsg, error) {\n\tvar sm *StoreMsg\n\tvar ok bool\n\n\tif subject == _EMPTY_ || subject == fwcs {\n\t\tsm, ok = ms.msgs[ms.state.LastSeq]\n\t} else if subjectIsLiteral(subject) {\n\t\tvar ss *SimpleState\n\t\tif ss, ok = ms.fss.Find(stringToBytes(subject)); ok && ss.Msgs > 0 {\n\t\t\t// Check if we need to recalculate. We only care about the last sequence.\n\t\t\tif ss.lastNeedsUpdate {\n\t\t\t\tms.recalculateForSubj(subject, ss)\n\t\t\t}\n\t\t\tsm, ok = ms.msgs[ss.Last]\n\t\t}\n\t} else if ss := ms.filteredStateLocked(1, subject, true); ss.Msgs > 0 {\n\t\tsm, ok = ms.msgs[ss.Last]\n\t}\n\tif !ok || sm == nil {\n\t\treturn nil, ErrStoreMsgNotFound\n\t}\n\n\tif smp == nil {\n\t\tsmp = new(StoreMsg)\n\t}\n\tsm.copy(smp)\n\treturn smp, nil\n}\n\n// LoadNextMsgMulti will find the next message matching any entry in the sublist.\nfunc (ms *memStore) LoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) {\n\t// TODO(dlc) - for now simple linear walk to get started.\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\n\tif start < ms.state.FirstSeq {\n\t\tstart = ms.state.FirstSeq\n\t}\n\n\t// If past the end no results.\n\tif start > ms.state.LastSeq || ms.state.Msgs == 0 {\n\t\treturn nil, ms.state.LastSeq, ErrStoreEOF\n\t}\n\n\t// Initial setup.\n\tfseq, lseq := start, ms.state.LastSeq\n\n\tfor nseq := fseq; nseq <= lseq; nseq++ {\n\t\tsm, ok := ms.msgs[nseq]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tif sl.HasInterest(sm.subj) {\n\t\t\tif smp == nil {\n\t\t\t\tsmp = new(StoreMsg)\n\t\t\t}\n\t\t\tsm.copy(smp)\n\t\t\treturn smp, nseq, nil\n\t\t}\n\t}\n\treturn nil, ms.state.LastSeq, ErrStoreEOF\n}\n\n// LoadNextMsg will find the next message matching the filter subject starting at the start sequence.\n// The filter subject can be a wildcard.\nfunc (ms *memStore) LoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) {\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\treturn ms.loadNextMsgLocked(filter, wc, start, smp)\n}\n\n// Find sequence bounds matching a wildcard filter from ms.fss.\n// Returns (first, last, true) if there is at least one matching\n// subject at or after start (start <= first <= last).\n// Returns (0, 0, false) if the subject does not exist or has no\n// messages at or after start.\n// Lock should be held.\nfunc (ms *memStore) nextWildcardMatchLocked(filter string, start uint64) (uint64, uint64, bool) {\n\tfound := false\n\tfirst, last := ms.state.LastSeq, uint64(0)\n\tms.fss.MatchUntil(stringToBytes(filter), func(subj []byte, ss *SimpleState) bool {\n\t\tms.recalculateForSubj(string(subj), ss)\n\n\t\t// Skip matches that are below our starting sequence\n\t\tif start > ss.Last {\n\t\t\treturn true\n\t\t}\n\n\t\t// A match was found, adjust the bounds accordingly\n\t\tfound = true\n\t\tif ss.First < first {\n\t\t\tfirst = ss.First\n\t\t}\n\t\tif ss.Last > last {\n\t\t\tlast = ss.Last\n\t\t}\n\n\t\t// If first > start, there may be more matches between\n\t\t// start and first, in which case we keep searching.\n\t\t// If not, we have a match between start and last, we\n\t\t// can break out of the search.\n\t\t// This could be further optimized: if first and start\n\t\t// are \"close\", we could just extend the linear search,\n\t\t// especially if we know that the remaining ms.fss to\n\t\t// explore is large.\n\t\treturn first > start\n\t})\n\tif !found {\n\t\treturn 0, 0, false\n\t}\n\treturn max(first, start), last, found\n}\n\n// Find sequence bounds matching a literal filter from ms.fss.\n// Returns (first, last, true) if there is a matching literal\n// subject at or after start (start <= first <= last).\n// Returns (0, 0, false) if the subject does not exist or has no\n// messages at or after start.\n// Lock should be held.\nfunc (ms *memStore) nextLiteralMatchLocked(filter string, start uint64) (uint64, uint64, bool) {\n\tss, ok := ms.fss.Find(stringToBytes(filter))\n\tif !ok {\n\t\treturn 0, 0, false\n\t}\n\tms.recalculateForSubj(filter, ss)\n\tif start > ss.Last {\n\t\treturn 0, 0, false\n\t}\n\treturn max(start, ss.First), ss.Last, true\n}\n\n// Returns true if LoadNextMsg should perform a linear scan,\n// false if it should use the subject tree to try to reduce\n// the search space.\n// Lock should be held.\nfunc (ms *memStore) shouldLinearScan(filter string, wc bool, start uint64) bool {\n\t// Skip scan of ms.fss if number of messages in the block are less than\n\t// 1/2 the number of subjects in ms.fss. Or we have a wc and lots of fss entries.\n\tconst linearScanMaxFSS = 256\n\tisAll := filter == fwcs\n\treturn isAll || 2*int(ms.state.LastSeq-start) < ms.fss.Size() || (wc && ms.fss.Size() > linearScanMaxFSS)\n}\n\n// Lock should be held.\nfunc (ms *memStore) loadNextMsgLocked(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) {\n\tif start < ms.state.FirstSeq {\n\t\tstart = ms.state.FirstSeq\n\t}\n\n\t// If past the end no results.\n\tif start > ms.state.LastSeq || ms.state.Msgs == 0 {\n\t\treturn nil, ms.state.LastSeq, ErrStoreEOF\n\t}\n\n\tif filter == _EMPTY_ {\n\t\tfilter = fwcs\n\t}\n\tisAll := filter == fwcs\n\n\t// Initial setup.\n\tfseq, lseq := start, ms.state.LastSeq\n\n\tif !ms.shouldLinearScan(filter, wc, start) {\n\t\tvar found bool\n\t\tif wc {\n\t\t\tfseq, lseq, found = ms.nextWildcardMatchLocked(filter, start)\n\t\t} else {\n\t\t\tfseq, lseq, found = ms.nextLiteralMatchLocked(filter, start)\n\t\t}\n\t\tif !found {\n\t\t\treturn nil, ms.state.LastSeq, ErrStoreEOF\n\t\t}\n\t}\n\n\teq := subjectsEqual\n\tif wc {\n\t\teq = matchLiteral\n\t}\n\n\tfor nseq := fseq; nseq <= lseq; nseq++ {\n\t\tif sm, ok := ms.msgs[nseq]; ok && (isAll || eq(sm.subj, filter)) {\n\t\t\tif smp == nil {\n\t\t\t\tsmp = new(StoreMsg)\n\t\t\t}\n\t\t\tsm.copy(smp)\n\t\t\treturn smp, nseq, nil\n\t\t}\n\t}\n\treturn nil, ms.state.LastSeq, ErrStoreEOF\n}\n\n// Will load the next non-deleted msg starting at the start sequence and walking backwards.\nfunc (ms *memStore) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error) {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\n\tif ms.msgs == nil {\n\t\treturn nil, ErrStoreClosed\n\t}\n\tif ms.state.Msgs == 0 || start < ms.state.FirstSeq {\n\t\treturn nil, ErrStoreEOF\n\t}\n\tif start > ms.state.LastSeq {\n\t\tstart = ms.state.LastSeq\n\t}\n\n\tfor seq := start; seq >= ms.state.FirstSeq; seq-- {\n\t\tif sm, ok := ms.msgs[seq]; ok {\n\t\t\tif smp == nil {\n\t\t\t\tsmp = new(StoreMsg)\n\t\t\t}\n\t\t\tsm.copy(smp)\n\t\t\treturn smp, nil\n\t\t}\n\t}\n\treturn nil, ErrStoreEOF\n}\n\n// LoadPrevMsgMulti will find the previous message matching any entry in the sublist.\nfunc (ms *memStore) LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) {\n\t// TODO(dlc) - for now simple linear walk to get started.\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\n\tif start > ms.state.LastSeq {\n\t\tstart = ms.state.LastSeq\n\t}\n\n\t// If past the start no results.\n\tif start < ms.state.FirstSeq || ms.state.Msgs == 0 {\n\t\treturn nil, ms.state.FirstSeq, ErrStoreEOF\n\t}\n\n\t// Initial setup.\n\tfseq, lseq := start, ms.state.FirstSeq\n\n\tfor nseq := fseq; nseq >= lseq; nseq-- {\n\t\tsm, ok := ms.msgs[nseq]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tif sl.HasInterest(sm.subj) {\n\t\t\tif smp == nil {\n\t\t\t\tsmp = new(StoreMsg)\n\t\t\t}\n\t\t\tsm.copy(smp)\n\t\t\treturn smp, nseq, nil\n\t\t}\n\t}\n\treturn nil, ms.state.LastSeq, ErrStoreEOF\n}\n\n// RemoveMsg will remove the message from this store.\n// Will return the number of bytes removed.\nfunc (ms *memStore) RemoveMsg(seq uint64) (bool, error) {\n\tms.mu.Lock()\n\tremoved := ms.removeMsg(seq, false)\n\tms.mu.Unlock()\n\treturn removed, nil\n}\n\n// EraseMsg will remove the message and rewrite its contents.\nfunc (ms *memStore) EraseMsg(seq uint64) (bool, error) {\n\tms.mu.Lock()\n\tremoved := ms.removeMsg(seq, true)\n\tms.mu.Unlock()\n\treturn removed, nil\n}\n\n// Performs logic to update first sequence number.\n// Lock should be held.\nfunc (ms *memStore) updateFirstSeq(seq uint64) {\n\tif seq != ms.state.FirstSeq {\n\t\t// Interior delete.\n\t\treturn\n\t}\n\tvar nsm *StoreMsg\n\tvar ok bool\n\tfor nseq := ms.state.FirstSeq + 1; nseq <= ms.state.LastSeq; nseq++ {\n\t\tif nsm, ok = ms.msgs[nseq]; ok {\n\t\t\tbreak\n\t\t}\n\t}\n\toldFirst := ms.state.FirstSeq\n\tif nsm != nil {\n\t\tms.state.FirstSeq = nsm.seq\n\t\tms.state.FirstTime = time.Unix(0, nsm.ts).UTC()\n\t} else {\n\t\t// Like purge.\n\t\tms.state.FirstSeq = ms.state.LastSeq + 1\n\t\tms.state.FirstTime = time.Time{}\n\t}\n\n\tif oldFirst == ms.state.FirstSeq-1 {\n\t\tms.dmap.Delete(oldFirst)\n\t} else {\n\t\tfor seq := oldFirst; seq < ms.state.FirstSeq; seq++ {\n\t\t\tms.dmap.Delete(seq)\n\t\t}\n\t}\n}\n\n// Remove a seq from the fss and select new first.\n// Lock should be held.\nfunc (ms *memStore) removeSeqPerSubject(subj string, seq uint64) {\n\tss, ok := ms.fss.Find(stringToBytes(subj))\n\tif !ok {\n\t\treturn\n\t}\n\tms.sdm.removeSeqAndSubject(seq, subj)\n\tif ss.Msgs == 1 {\n\t\tms.fss.Delete(stringToBytes(subj))\n\t\treturn\n\t}\n\tss.Msgs--\n\n\t// Only one left\n\tif ss.Msgs == 1 {\n\t\tif !ss.lastNeedsUpdate && seq != ss.Last {\n\t\t\tss.First = ss.Last\n\t\t\tss.firstNeedsUpdate = false\n\t\t\treturn\n\t\t}\n\t\tif !ss.firstNeedsUpdate && seq != ss.First {\n\t\t\tss.Last = ss.First\n\t\t\tss.lastNeedsUpdate = false\n\t\t\treturn\n\t\t}\n\t}\n\n\t// We can lazily calculate the first/last sequence when needed.\n\tss.firstNeedsUpdate = seq == ss.First || ss.firstNeedsUpdate\n\tss.lastNeedsUpdate = seq == ss.Last || ss.lastNeedsUpdate\n}\n\n// Will recalculate the first and/or last sequence for this subject.\n// Lock should be held.\nfunc (ms *memStore) recalculateForSubj(subj string, ss *SimpleState) {\n\tif ss.firstNeedsUpdate {\n\t\ttseq := ss.First + 1\n\t\tif tseq < ms.state.FirstSeq {\n\t\t\ttseq = ms.state.FirstSeq\n\t\t}\n\t\tfor ; tseq <= ss.Last; tseq++ {\n\t\t\tif sm := ms.msgs[tseq]; sm != nil && sm.subj == subj {\n\t\t\t\tss.First = tseq\n\t\t\t\tss.firstNeedsUpdate = false\n\t\t\t\tif ss.Msgs == 1 {\n\t\t\t\t\tss.Last = tseq\n\t\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif ss.lastNeedsUpdate {\n\t\ttseq := ss.Last - 1\n\t\tif tseq > ms.state.LastSeq {\n\t\t\ttseq = ms.state.LastSeq\n\t\t}\n\t\tfor ; tseq >= ss.First; tseq-- {\n\t\t\tif sm := ms.msgs[tseq]; sm != nil && sm.subj == subj {\n\t\t\t\tss.Last = tseq\n\t\t\t\tss.lastNeedsUpdate = false\n\t\t\t\tif ss.Msgs == 1 {\n\t\t\t\t\tss.First = tseq\n\t\t\t\t\tss.firstNeedsUpdate = false\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Removes the message referenced by seq.\n// Lock should be held.\nfunc (ms *memStore) removeMsg(seq uint64, secure bool) bool {\n\tvar ss uint64\n\tsm, ok := ms.msgs[seq]\n\tif !ok {\n\t\treturn false\n\t}\n\n\tss = memStoreMsgSize(sm.subj, sm.hdr, sm.msg)\n\n\tif ms.state.Msgs > 0 {\n\t\tms.state.Msgs--\n\t\tif ss > ms.state.Bytes {\n\t\t\tss = ms.state.Bytes\n\t\t}\n\t\tms.state.Bytes -= ss\n\t}\n\tms.dmap.Insert(seq)\n\tms.updateFirstSeq(seq)\n\n\t// Remove any per subject tracking.\n\tms.removeSeqPerSubject(sm.subj, seq)\n\tif ms.ttls != nil {\n\t\tif ttl, err := getMessageTTL(sm.hdr); err == nil {\n\t\t\texpires := time.Duration(sm.ts) + (time.Second * time.Duration(ttl))\n\t\t\tms.ttls.Remove(seq, int64(expires))\n\t\t}\n\t}\n\n\tif secure {\n\t\tif len(sm.hdr) > 0 {\n\t\t\tsm.hdr = make([]byte, len(sm.hdr))\n\t\t\tcrand.Read(sm.hdr)\n\t\t}\n\t\tif len(sm.msg) > 0 {\n\t\t\tsm.msg = make([]byte, len(sm.msg))\n\t\t\tcrand.Read(sm.msg)\n\t\t}\n\t\tsm.seq, sm.ts = 0, 0\n\t}\n\n\t// Must delete message after updating per-subject info, to be consistent with file store.\n\tdelete(ms.msgs, seq)\n\n\tif ms.scb != nil {\n\t\t// We do not want to hold any locks here.\n\t\tms.mu.Unlock()\n\t\tif ms.scb != nil {\n\t\t\tdelta := int64(ss)\n\t\t\tms.scb(-1, -delta, seq, sm.subj)\n\t\t}\n\t\tms.mu.Lock()\n\t}\n\n\treturn ok\n}\n\n// Type returns the type of the underlying store.\nfunc (ms *memStore) Type() StorageType {\n\treturn MemoryStorage\n}\n\n// FastState will fill in state with only the following.\n// Msgs, Bytes, First and Last Sequence and Time and NumDeleted.\nfunc (ms *memStore) FastState(state *StreamState) {\n\tms.mu.RLock()\n\tstate.Msgs = ms.state.Msgs\n\tstate.Bytes = ms.state.Bytes\n\tstate.FirstSeq = ms.state.FirstSeq\n\tstate.FirstTime = ms.state.FirstTime\n\tstate.LastSeq = ms.state.LastSeq\n\tstate.LastTime = ms.state.LastTime\n\tif state.LastSeq > state.FirstSeq {\n\t\tstate.NumDeleted = int((state.LastSeq - state.FirstSeq + 1) - state.Msgs)\n\t\tif state.NumDeleted < 0 {\n\t\t\tstate.NumDeleted = 0\n\t\t}\n\t}\n\tstate.Consumers = ms.consumers\n\tstate.NumSubjects = ms.fss.Size()\n\tms.mu.RUnlock()\n}\n\nfunc (ms *memStore) State() StreamState {\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tstate := ms.state\n\tstate.Consumers = ms.consumers\n\tstate.NumSubjects = ms.fss.Size()\n\tstate.Deleted = nil\n\n\t// Calculate interior delete details.\n\tif numDeleted := int((state.LastSeq - state.FirstSeq + 1) - state.Msgs); numDeleted > 0 {\n\t\tstate.Deleted = make([]uint64, 0, numDeleted)\n\t\tfseq, lseq := state.FirstSeq, state.LastSeq\n\t\tms.dmap.Range(func(seq uint64) bool {\n\t\t\tif seq < fseq || seq > lseq {\n\t\t\t\tms.dmap.Delete(seq)\n\t\t\t} else {\n\t\t\t\tstate.Deleted = append(state.Deleted, seq)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n\tif len(state.Deleted) > 0 {\n\t\tstate.NumDeleted = len(state.Deleted)\n\t}\n\n\treturn state\n}\n\nfunc (ms *memStore) Utilization() (total, reported uint64, err error) {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\treturn ms.state.Bytes, ms.state.Bytes, nil\n}\n\nfunc memStoreMsgSizeRaw(slen, hlen, mlen int) uint64 {\n\treturn uint64(slen + hlen + mlen + 16) // 8*2 for seq + age\n}\n\nfunc memStoreMsgSize(subj string, hdr, msg []byte) uint64 {\n\treturn memStoreMsgSizeRaw(len(subj), len(hdr), len(msg))\n}\n\n// ResetState resets any state that's temporary. For example when changing leaders.\nfunc (ms *memStore) ResetState() {\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\tif ms.scheduling != nil {\n\t\tms.scheduling.clearInflight()\n\t}\n}\n\n// Delete is same as Stop for memory store.\nfunc (ms *memStore) Delete(_ bool) error {\n\treturn ms.Stop()\n}\n\nfunc (ms *memStore) Stop() error {\n\tms.mu.Lock()\n\tif ms.msgs == nil {\n\t\tms.mu.Unlock()\n\t\treturn nil\n\t}\n\tif ms.ageChk != nil {\n\t\tms.ageChk.Stop()\n\t\tms.ageChk = nil\n\t\tms.ageChkTime = 0\n\t}\n\tms.msgs = nil\n\tms.mu.Unlock()\n\n\t// These can't come back, so stop is same as Delete.\n\tms.Purge()\n\n\t// Unregister from the access time service.\n\tats.Unregister()\n\n\treturn nil\n}\n\nfunc (ms *memStore) isClosed() bool {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\treturn ms.msgs == nil\n}\n\ntype consumerMemStore struct {\n\tmu sync.Mutex\n\tms StreamStore\n\tcfg ConsumerConfig\n\tstate ConsumerState\n\tclosed bool\n}\n\nfunc (ms *memStore) ConsumerStore(name string, _ time.Time, cfg *ConsumerConfig) (ConsumerStore, error) {\n\tif ms == nil {\n\t\treturn nil, fmt.Errorf(\"memstore is nil\")\n\t}\n\tif ms.isClosed() {\n\t\treturn nil, ErrStoreClosed\n\t}\n\tif cfg == nil || name == _EMPTY_ {\n\t\treturn nil, fmt.Errorf(\"bad consumer config\")\n\t}\n\to := &consumerMemStore{ms: ms, cfg: *cfg}\n\tms.AddConsumer(o)\n\treturn o, nil\n}\n\nfunc (ms *memStore) AddConsumer(o ConsumerStore) error {\n\tms.mu.Lock()\n\tms.consumers++\n\tms.mu.Unlock()\n\treturn nil\n}\n\nfunc (ms *memStore) RemoveConsumer(o ConsumerStore) error {\n\tms.mu.Lock()\n\tif ms.consumers > 0 {\n\t\tms.consumers--\n\t}\n\tms.mu.Unlock()\n\treturn nil\n}\n\nfunc (ms *memStore) Snapshot(_ time.Duration, _, _ bool) (*SnapshotResult, error) {\n\treturn nil, fmt.Errorf(\"no impl\")\n}\n\n// Binary encoded state snapshot, >= v2.10 server.\nfunc (ms *memStore) EncodedStreamState(failed uint64) ([]byte, error) {\n\tms.mu.RLock()\n\tdefer ms.mu.RUnlock()\n\n\t// Quick calculate num deleted.\n\tnumDeleted := int((ms.state.LastSeq - ms.state.FirstSeq + 1) - ms.state.Msgs)\n\tif numDeleted < 0 {\n\t\tnumDeleted = 0\n\t}\n\n\t// Encoded is Msgs, Bytes, FirstSeq, LastSeq, Failed, NumDeleted and optional DeletedBlocks\n\tvar buf [1024]byte\n\tbuf[0], buf[1] = streamStateMagic, streamStateVersion\n\tn := hdrLen\n\tn += binary.PutUvarint(buf[n:], ms.state.Msgs)\n\tn += binary.PutUvarint(buf[n:], ms.state.Bytes)\n\tn += binary.PutUvarint(buf[n:], ms.state.FirstSeq)\n\tn += binary.PutUvarint(buf[n:], ms.state.LastSeq)\n\tn += binary.PutUvarint(buf[n:], failed)\n\tn += binary.PutUvarint(buf[n:], uint64(numDeleted))\n\n\tb := buf[0:n]\n\n\tif numDeleted > 0 {\n\t\tbuf := ms.dmap.Encode(nil)\n\t\tb = append(b, buf...)\n\t}\n\n\treturn b, nil\n}\n\n// SyncDeleted will make sure this stream has same deleted state as dbs.\nfunc (ms *memStore) SyncDeleted(dbs DeleteBlocks) error {\n\tif len(dbs) == 0 {\n\t\treturn nil\n\t}\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\t// For now we share one dmap, so if we have one entry here check if states are the same.\n\t// Note this will work for any DeleteBlock type, but we expect this to be a dmap too.\n\tif len(dbs) == 1 {\n\t\tmin, max, num := ms.dmap.State()\n\t\tif pmin, pmax, pnum := dbs[0].State(); pmin == min && pmax == max && pnum == num {\n\t\t\treturn nil\n\t\t}\n\t}\n\tlseq := ms.state.LastSeq\n\tfor _, db := range dbs {\n\t\t// Skip if beyond our current state.\n\t\tif first, _, _ := db.State(); first > lseq {\n\t\t\tcontinue\n\t\t}\n\t\tdb.Range(func(seq uint64) bool {\n\t\t\tms.removeMsg(seq, false)\n\t\t\treturn true\n\t\t})\n\t}\n\treturn nil\n}\n\nfunc (o *consumerMemStore) Update(state *ConsumerState) error {\n\t// Sanity checks.\n\tif state.AckFloor.Consumer > state.Delivered.Consumer {\n\t\treturn fmt.Errorf(\"bad ack floor for consumer\")\n\t}\n\tif state.AckFloor.Stream > state.Delivered.Stream {\n\t\treturn fmt.Errorf(\"bad ack floor for stream\")\n\t}\n\n\t// Copy to our state.\n\tvar pending map[uint64]*Pending\n\tvar redelivered map[uint64]uint64\n\tif len(state.Pending) > 0 {\n\t\tpending = make(map[uint64]*Pending, len(state.Pending))\n\t\tfor seq, p := range state.Pending {\n\t\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t\t\tif seq <= state.AckFloor.Stream || seq > state.Delivered.Stream {\n\t\t\t\treturn fmt.Errorf(\"bad pending entry, sequence [%d] out of range\", seq)\n\t\t\t}\n\t\t}\n\t}\n\tif len(state.Redelivered) > 0 {\n\t\tredelivered = make(map[uint64]uint64, len(state.Redelivered))\n\t\tfor seq, dc := range state.Redelivered {\n\t\t\tredelivered[seq] = dc\n\t\t}\n\t}\n\n\t// Replace our state.\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// Check to see if this is an outdated update.\n\tif state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream {\n\t\treturn ErrStoreOldUpdate\n\t}\n\n\to.state.Delivered = state.Delivered\n\to.state.AckFloor = state.AckFloor\n\to.state.Pending = pending\n\to.state.Redelivered = redelivered\n\n\treturn nil\n}\n\nfunc (o *consumerMemStore) ForceUpdate(state *ConsumerState) error {\n\t// Sanity checks.\n\tif state.AckFloor.Consumer > state.Delivered.Consumer {\n\t\treturn fmt.Errorf(\"bad ack floor for consumer\")\n\t}\n\tif state.AckFloor.Stream > state.Delivered.Stream {\n\t\treturn fmt.Errorf(\"bad ack floor for stream\")\n\t}\n\n\t// Copy to our state.\n\tvar pending map[uint64]*Pending\n\tvar redelivered map[uint64]uint64\n\tif len(state.Pending) > 0 {\n\t\tpending = make(map[uint64]*Pending, len(state.Pending))\n\t\tfor seq, p := range state.Pending {\n\t\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t\t\tif seq <= state.AckFloor.Stream || seq > state.Delivered.Stream {\n\t\t\t\treturn fmt.Errorf(\"bad pending entry, sequence [%d] out of range\", seq)\n\t\t\t}\n\t\t}\n\t}\n\tif len(state.Redelivered) > 0 {\n\t\tredelivered = make(map[uint64]uint64, len(state.Redelivered))\n\t\tfor seq, dc := range state.Redelivered {\n\t\t\tredelivered[seq] = dc\n\t\t}\n\t}\n\n\t// Replace our state.\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\to.state.Delivered = state.Delivered\n\to.state.AckFloor = state.AckFloor\n\to.state.Pending = pending\n\to.state.Redelivered = redelivered\n\n\treturn nil\n}\n\n// SetStarting sets our starting stream sequence.\nfunc (o *consumerMemStore) SetStarting(sseq uint64) error {\n\to.mu.Lock()\n\to.state.Delivered.Stream = sseq\n\to.state.AckFloor.Stream = sseq\n\to.mu.Unlock()\n\treturn nil\n}\n\n// UpdateStarting updates our starting stream sequence.\nfunc (o *consumerMemStore) UpdateStarting(sseq uint64) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif sseq > o.state.Delivered.Stream {\n\t\to.state.Delivered.Stream = sseq\n\t\t// For AckNone just update delivered and ackfloor at the same time.\n\t\tif o.cfg.AckPolicy == AckNone {\n\t\t\to.state.AckFloor.Stream = sseq\n\t\t}\n\t}\n}\n\n// Reset all values in the store, and reset the starting sequence.\nfunc (o *consumerMemStore) Reset(sseq uint64) error {\n\to.mu.Lock()\n\to.state = ConsumerState{}\n\to.mu.Unlock()\n\treturn o.SetStarting(sseq)\n}\n\n// HasState returns if this store has a recorded state.\nfunc (o *consumerMemStore) HasState() bool {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\t// We have a running state.\n\treturn o.state.Delivered.Consumer != 0 || o.state.Delivered.Stream != 0\n}\n\nfunc (o *consumerMemStore) UpdateDelivered(dseq, sseq, dc uint64, ts int64) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif dc != 1 && o.cfg.AckPolicy == AckNone {\n\t\treturn ErrNoAckPolicy\n\t}\n\n\t// On restarts the old leader may get a replay from the raft logs that are old.\n\tif dseq <= o.state.AckFloor.Consumer {\n\t\treturn nil\n\t}\n\n\t// See if we expect an ack for this.\n\tif o.cfg.AckPolicy != AckNone {\n\t\t// Need to create pending records here.\n\t\tif o.state.Pending == nil {\n\t\t\to.state.Pending = make(map[uint64]*Pending)\n\t\t}\n\t\tvar p *Pending\n\t\t// Check for an update to a message already delivered.\n\t\tif sseq <= o.state.Delivered.Stream {\n\t\t\tif p = o.state.Pending[sseq]; p != nil {\n\t\t\t\t// Do not update p.Sequence, that should be the original delivery sequence.\n\t\t\t\tp.Timestamp = ts\n\t\t\t}\n\t\t} else {\n\t\t\t// Add to pending.\n\t\t\to.state.Pending[sseq] = &Pending{dseq, ts}\n\t\t}\n\t\t// Update delivered as needed.\n\t\tif dseq > o.state.Delivered.Consumer {\n\t\t\to.state.Delivered.Consumer = dseq\n\t\t}\n\t\tif sseq > o.state.Delivered.Stream {\n\t\t\to.state.Delivered.Stream = sseq\n\t\t}\n\n\t\tif dc > 1 {\n\t\t\tif maxdc := uint64(o.cfg.MaxDeliver); maxdc > 0 && dc > maxdc {\n\t\t\t\t// Make sure to remove from pending.\n\t\t\t\tdelete(o.state.Pending, sseq)\n\t\t\t}\n\t\t\tif o.state.Redelivered == nil {\n\t\t\t\to.state.Redelivered = make(map[uint64]uint64)\n\t\t\t}\n\t\t\t// Only update if greater than what we already have.\n\t\t\tif o.state.Redelivered[sseq] < dc-1 {\n\t\t\t\to.state.Redelivered[sseq] = dc - 1\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// For AckNone just update delivered and ackfloor at the same time.\n\t\tif dseq > o.state.Delivered.Consumer {\n\t\t\to.state.Delivered.Consumer = dseq\n\t\t\to.state.AckFloor.Consumer = dseq\n\t\t}\n\t\tif sseq > o.state.Delivered.Stream {\n\t\t\to.state.Delivered.Stream = sseq\n\t\t\to.state.AckFloor.Stream = sseq\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (o *consumerMemStore) UpdateAcks(dseq, sseq uint64) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.cfg.AckPolicy == AckNone {\n\t\treturn ErrNoAckPolicy\n\t}\n\n\t// On restarts the old leader may get a replay from the raft logs that are old.\n\tif dseq <= o.state.AckFloor.Consumer {\n\t\treturn nil\n\t}\n\n\tif len(o.state.Pending) == 0 || o.state.Pending[sseq] == nil {\n\t\tdelete(o.state.Redelivered, sseq)\n\t\treturn ErrStoreMsgNotFound\n\t}\n\n\t// Check for AckAll here.\n\tif o.cfg.AckPolicy == AckAll {\n\t\tsgap := sseq - o.state.AckFloor.Stream\n\t\to.state.AckFloor.Consumer = dseq\n\t\to.state.AckFloor.Stream = sseq\n\t\tif sgap > uint64(len(o.state.Pending)) {\n\t\t\tfor seq := range o.state.Pending {\n\t\t\t\tif seq <= sseq {\n\t\t\t\t\tdelete(o.state.Pending, seq)\n\t\t\t\t\tdelete(o.state.Redelivered, seq)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor seq := sseq; seq > sseq-sgap && len(o.state.Pending) > 0; seq-- {\n\t\t\t\tdelete(o.state.Pending, seq)\n\t\t\t\tdelete(o.state.Redelivered, seq)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\t// AckExplicit\n\n\t// First delete from our pending state.\n\tif p, ok := o.state.Pending[sseq]; ok {\n\t\tdelete(o.state.Pending, sseq)\n\t\tif dseq > p.Sequence && p.Sequence > 0 {\n\t\t\tdseq = p.Sequence // Use the original.\n\t\t}\n\t}\n\n\tif len(o.state.Pending) == 0 {\n\t\to.state.AckFloor.Consumer = o.state.Delivered.Consumer\n\t\to.state.AckFloor.Stream = o.state.Delivered.Stream\n\t} else if dseq == o.state.AckFloor.Consumer+1 {\n\t\to.state.AckFloor.Consumer = dseq\n\t\to.state.AckFloor.Stream = sseq\n\n\t\tif o.state.Delivered.Consumer > dseq {\n\t\t\tfor ss := sseq + 1; ss <= o.state.Delivered.Stream; ss++ {\n\t\t\t\tif p, ok := o.state.Pending[ss]; ok {\n\t\t\t\t\tif p.Sequence > 0 {\n\t\t\t\t\t\to.state.AckFloor.Consumer = p.Sequence - 1\n\t\t\t\t\t\to.state.AckFloor.Stream = ss - 1\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// We do these regardless.\n\tdelete(o.state.Redelivered, sseq)\n\n\treturn nil\n}\n\nfunc (o *consumerMemStore) UpdateConfig(cfg *ConsumerConfig) error {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\t// This is mostly unchecked here. We are assuming the upper layers have done sanity checking.\n\to.cfg = *cfg\n\treturn nil\n}\n\nfunc (o *consumerMemStore) Stop() error {\n\to.mu.Lock()\n\to.closed = true\n\tms := o.ms\n\to.mu.Unlock()\n\tms.RemoveConsumer(o)\n\treturn nil\n}\n\nfunc (o *consumerMemStore) Delete() error {\n\treturn o.Stop()\n}\n\nfunc (o *consumerMemStore) StreamDelete() error {\n\treturn o.Stop()\n}\n\nfunc (o *consumerMemStore) State() (*ConsumerState, error) {\n\treturn o.stateWithCopy(true)\n}\n\n// This will not copy pending or redelivered, so should only be done under the\n// consumer owner's lock.\nfunc (o *consumerMemStore) BorrowState() (*ConsumerState, error) {\n\treturn o.stateWithCopy(false)\n}\n\nfunc (o *consumerMemStore) stateWithCopy(doCopy bool) (*ConsumerState, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.closed {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\tstate := &ConsumerState{}\n\n\tstate.Delivered = o.state.Delivered\n\tstate.AckFloor = o.state.AckFloor\n\tif len(o.state.Pending) > 0 {\n\t\tif doCopy {\n\t\t\tstate.Pending = o.copyPending()\n\t\t} else {\n\t\t\tstate.Pending = o.state.Pending\n\t\t}\n\t}\n\tif len(o.state.Redelivered) > 0 {\n\t\tif doCopy {\n\t\t\tstate.Redelivered = o.copyRedelivered()\n\t\t} else {\n\t\t\tstate.Redelivered = o.state.Redelivered\n\t\t}\n\t}\n\treturn state, nil\n}\n\n// EncodedState for this consumer store.\nfunc (o *consumerMemStore) EncodedState() ([]byte, error) {\n\to.mu.Lock()\n\tdefer o.mu.Unlock()\n\n\tif o.closed {\n\t\treturn nil, ErrStoreClosed\n\t}\n\n\treturn encodeConsumerState(&o.state), nil\n}\n\nfunc (o *consumerMemStore) copyPending() map[uint64]*Pending {\n\tpending := make(map[uint64]*Pending, len(o.state.Pending))\n\tfor seq, p := range o.state.Pending {\n\t\tpending[seq] = &Pending{p.Sequence, p.Timestamp}\n\t}\n\treturn pending\n}\n\nfunc (o *consumerMemStore) copyRedelivered() map[uint64]uint64 {\n\tredelivered := make(map[uint64]uint64, len(o.state.Redelivered))\n\tfor seq, dc := range o.state.Redelivered {\n\t\tredelivered[seq] = dc\n\t}\n\treturn redelivered\n}\n\n// Type returns the type of the underlying store.\nfunc (o *consumerMemStore) Type() StorageType { return MemoryStorage }\n" }, { "path": "server/memstore_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_store_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"reflect\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc TestMemStoreBasics(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tnow := time.Now().UnixNano()\n\tif seq, ts, err := ms.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t} else if seq != 1 {\n\t\tt.Fatalf(\"Expected sequence to be 1, got %d\", seq)\n\t} else if ts < now || ts > now+int64(time.Millisecond) {\n\t\tt.Fatalf(\"Expected timestamp to be current, got %v\", ts-now)\n\t}\n\n\tstate := ms.State()\n\tif state.Msgs != 1 {\n\t\tt.Fatalf(\"Expected 1 msg, got %d\", state.Msgs)\n\t}\n\texpectedSize := memStoreMsgSize(subj, nil, msg)\n\tif state.Bytes != expectedSize {\n\t\tt.Fatalf(\"Expected %d bytes, got %d\", expectedSize, state.Bytes)\n\t}\n\tsm, err := ms.LoadMsg(1, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t}\n\tif sm.subj != subj {\n\t\tt.Fatalf(\"Subjects don't match, original %q vs %q\", subj, sm.subj)\n\t}\n\tif !bytes.Equal(sm.msg, msg) {\n\t\tt.Fatalf(\"Msgs don't match, original %q vs %q\", msg, sm.msg)\n\t}\n}\n\nfunc TestMemStoreMsgLimit(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxMsgs: 10})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfor i := 0; i < 10; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstate := ms.State()\n\tif state.Msgs != 10 {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", 10, state.Msgs)\n\t}\n\tif _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t}\n\tstate = ms.State()\n\tif state.Msgs != 10 {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", 10, state.Msgs)\n\t}\n\tif state.LastSeq != 11 {\n\t\tt.Fatalf(\"Expected the last sequence to be 11 now, but got %d\", state.LastSeq)\n\t}\n\tif state.FirstSeq != 2 {\n\t\tt.Fatalf(\"Expected the first sequence to be 2 now, but got %d\", state.FirstSeq)\n\t}\n\t// Make sure we can not lookup seq 1.\n\tif _, err := ms.LoadMsg(1, nil); err == nil {\n\t\tt.Fatalf(\"Expected error looking up seq 1 but got none\")\n\t}\n}\n\nfunc TestMemStoreBytesLimit(t *testing.T) {\n\tsubj, msg := \"foo\", make([]byte, 512)\n\tstoredMsgSize := memStoreMsgSize(subj, nil, msg)\n\n\ttoStore := uint64(1024)\n\tmaxBytes := storedMsgSize * toStore\n\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfor i := uint64(0); i < toStore; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstate := ms.State()\n\tif state.Msgs != toStore {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\tif state.Bytes != storedMsgSize*toStore {\n\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t}\n\n\t// Now send 10 more and check that bytes limit enforced.\n\tfor i := 0; i < 10; i++ {\n\t\tif _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t}\n\t}\n\tstate = ms.State()\n\tif state.Msgs != toStore {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\tif state.Bytes != storedMsgSize*toStore {\n\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t}\n\tif state.FirstSeq != 11 {\n\t\tt.Fatalf(\"Expected first sequence to be 11, got %d\", state.FirstSeq)\n\t}\n\tif state.LastSeq != toStore+10 {\n\t\tt.Fatalf(\"Expected last sequence to be %d, got %d\", toStore+10, state.LastSeq)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/4771\nfunc TestMemStoreBytesLimitWithDiscardNew(t *testing.T) {\n\tsubj, msg := \"tiny\", make([]byte, 7)\n\tstoredMsgSize := memStoreMsgSize(subj, nil, msg)\n\n\ttoStore := uint64(3)\n\tmaxBytes := 100\n\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes), Discard: DiscardNew})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Now send 10 messages and check that bytes limit enforced.\n\tfor i := 0; i < 10; i++ {\n\t\t_, _, err := ms.StoreMsg(subj, nil, msg, 0)\n\t\tif i < int(toStore) {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t\t}\n\t\t} else if !errors.Is(err, ErrMaxBytes) {\n\t\t\tt.Fatalf(\"Storing msg should result in: %v\", ErrMaxBytes)\n\t\t}\n\t}\n\tstate := ms.State()\n\tif state.Msgs != toStore {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\tif state.Bytes != storedMsgSize*toStore {\n\t\tt.Fatalf(\"Expected bytes to be %d, got %d\", storedMsgSize*toStore, state.Bytes)\n\t}\n}\n\nfunc TestMemStoreAgeLimit(t *testing.T) {\n\tmaxAge := 10 * time.Millisecond\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxAge: maxAge})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Store some messages. Does not really matter how many.\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\ttoStore := 100\n\tfor i := 0; i < toStore; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstate := ms.State()\n\tif state.Msgs != uint64(toStore) {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\tcheckExpired := func(t *testing.T) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, maxAge, func() error {\n\t\t\tstate = ms.State()\n\t\t\tif state.Msgs != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no msgs, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tif state.Bytes != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no bytes, got %d\", state.Bytes)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Let them expire\n\tcheckExpired(t)\n\t// Now add some more and make sure that timer will fire again.\n\tfor i := 0; i < toStore; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstate = ms.State()\n\tif state.Msgs != uint64(toStore) {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\tcheckExpired(t)\n}\n\nfunc TestMemStoreTimeStamps(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tlast := time.Now().UnixNano()\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfor i := 0; i < 10; i++ {\n\t\ttime.Sleep(5 * time.Microsecond)\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tvar smv StoreMsg\n\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\tsm, err := ms.LoadMsg(seq, &smv)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t\t}\n\t\t// These should be different\n\t\tif sm.ts <= last {\n\t\t\tt.Fatalf(\"Expected different timestamps, got %v\", sm.ts)\n\t\t}\n\t\tlast = sm.ts\n\t}\n}\n\nfunc TestMemStorePurge(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfor i := 0; i < 10; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tif state := ms.State(); state.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs, got %d\", state.Msgs)\n\t}\n\tms.Purge()\n\tif state := ms.State(); state.Msgs != 0 {\n\t\tt.Fatalf(\"Expected no msgs, got %d\", state.Msgs)\n\t}\n}\n\nfunc TestMemStoreCompact(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfor i := 0; i < 10; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tif state := ms.State(); state.Msgs != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs, got %d\", state.Msgs)\n\t}\n\tn, err := ms.Compact(6)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n != 5 {\n\t\tt.Fatalf(\"Expected to have purged 5 msgs, got %d\", n)\n\t}\n\tstate := ms.State()\n\tif state.Msgs != 5 {\n\t\tt.Fatalf(\"Expected 5 msgs, got %d\", state.Msgs)\n\t}\n\tif state.FirstSeq != 6 {\n\t\tt.Fatalf(\"Expected first seq of 6, got %d\", state.FirstSeq)\n\t}\n\t// Now test that compact will also reset first if seq > last\n\tn, err = ms.Compact(100)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n != 5 {\n\t\tt.Fatalf(\"Expected to have purged 5 msgs, got %d\", n)\n\t}\n\tif state = ms.State(); state.FirstSeq != 100 {\n\t\tt.Fatalf(\"Expected first seq of 100, got %d\", state.FirstSeq)\n\t}\n}\n\nfunc TestMemStoreEraseMsg(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tms.StoreMsg(subj, nil, msg, 0)\n\tsm, err := ms.LoadMsg(1, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t}\n\tif !bytes.Equal(msg, sm.msg) {\n\t\tt.Fatalf(\"Expected same msg, got %q vs %q\", sm.msg, msg)\n\t}\n\tif removed, _ := ms.EraseMsg(1); !removed {\n\t\tt.Fatalf(\"Expected erase msg to return success\")\n\t}\n}\n\nfunc TestMemStoreMsgHeaders(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, hdr, msg := \"foo\", []byte(\"name:derek\"), []byte(\"Hello World\")\n\tif sz := int(memStoreMsgSize(subj, hdr, msg)); sz != (len(subj) + len(hdr) + len(msg) + 16) {\n\t\tt.Fatalf(\"Wrong size for stored msg with header\")\n\t}\n\tms.StoreMsg(subj, hdr, msg, 0)\n\tsm, err := ms.LoadMsg(1, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error looking up msg: %v\", err)\n\t}\n\tif !bytes.Equal(msg, sm.msg) {\n\t\tt.Fatalf(\"Expected same msg, got %q vs %q\", sm.msg, msg)\n\t}\n\tif !bytes.Equal(hdr, sm.hdr) {\n\t\tt.Fatalf(\"Expected same hdr, got %q vs %q\", sm.hdr, hdr)\n\t}\n\tif removed, _ := ms.EraseMsg(1); !removed {\n\t\tt.Fatalf(\"Expected erase msg to return success\")\n\t}\n}\n\nfunc TestMemStoreStreamStateDeleted(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, toStore := \"foo\", uint64(10)\n\tfor i := uint64(1); i <= toStore; i++ {\n\t\tmsg := []byte(fmt.Sprintf(\"[%08d] Hello World!\", i))\n\t\tif _, _, err := ms.StoreMsg(subj, nil, msg, 0); err != nil {\n\t\t\tt.Fatalf(\"Error storing msg: %v\", err)\n\t\t}\n\t}\n\tstate := ms.State()\n\tif len(state.Deleted) != 0 {\n\t\tt.Fatalf(\"Expected deleted to be empty\")\n\t}\n\t// Now remove some interior messages.\n\tvar expected []uint64\n\tfor seq := uint64(2); seq < toStore; seq += 2 {\n\t\tms.RemoveMsg(seq)\n\t\texpected = append(expected, seq)\n\t}\n\tstate = ms.State()\n\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t}\n\t// Now fill the gap by deleting 1 and 3\n\tms.RemoveMsg(1)\n\tms.RemoveMsg(3)\n\texpected = expected[2:]\n\tstate = ms.State()\n\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t}\n\tif state.FirstSeq != 5 {\n\t\tt.Fatalf(\"Expected first seq to be 5, got %d\", state.FirstSeq)\n\t}\n\tms.Purge()\n\tif state = ms.State(); len(state.Deleted) != 0 {\n\t\tt.Fatalf(\"Expected no deleted after purge, got %+v\\n\", state.Deleted)\n\t}\n}\n\nfunc TestMemStoreStreamTruncate(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\ttseq := uint64(50)\n\n\tsubj, toStore := \"foo\", uint64(100)\n\tfor i := uint64(1); i < tseq; i++ {\n\t\t_, _, err := ms.StoreMsg(subj, nil, []byte(\"ok\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\tsubj = \"bar\"\n\tfor i := tseq; i <= toStore; i++ {\n\t\t_, _, err := ms.StoreMsg(subj, nil, []byte(\"ok\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tif state := ms.State(); state.Msgs != toStore {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", toStore, state.Msgs)\n\t}\n\n\tif err := ms.Truncate(tseq); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif state := ms.State(); state.Msgs != tseq {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", tseq, state.Msgs)\n\t}\n\n\t// Now make sure we report properly if we have some deleted interior messages.\n\tms.RemoveMsg(10)\n\tms.RemoveMsg(20)\n\tms.RemoveMsg(30)\n\tms.RemoveMsg(40)\n\n\ttseq = uint64(25)\n\tif err := ms.Truncate(tseq); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tstate := ms.State()\n\tif state.Msgs != tseq-2 {\n\t\tt.Fatalf(\"Expected %d msgs, got %d\", tseq-2, state.Msgs)\n\t}\n\tif state.NumSubjects != 1 {\n\t\tt.Fatalf(\"Expected only 1 subject, got %d\", state.NumSubjects)\n\t}\n\texpected := []uint64{10, 20}\n\tif !reflect.DeepEqual(state.Deleted, expected) {\n\t\tt.Fatalf(\"Expected deleted to be %+v, got %+v\\n\", expected, state.Deleted)\n\t}\n}\n\nfunc TestMemStorePurgeExWithSubject(t *testing.T) {\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfor i := 0; i < 100; i++ {\n\t\t_, _, err = ms.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// This should purge all.\n\tms.PurgeEx(\"foo\", 1, 0)\n\trequire_True(t, ms.State().Msgs == 0)\n}\n\nfunc TestMemStoreUpdateMaxMsgsPerSubject(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tMaxMsgsPer: 10,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Make sure this is honored on an update.\n\tcfg.MaxMsgsPer = 50\n\terr = ms.UpdateConfig(cfg)\n\trequire_NoError(t, err)\n\n\tnumStored := 22\n\tfor i := 0; i < numStored; i++ {\n\t\t_, _, err = ms.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tss := ms.SubjectsState(\"foo\")[\"foo\"]\n\tif ss.Msgs != uint64(numStored) {\n\t\tt.Fatalf(\"Expected to have %d stored, got %d\", numStored, ss.Msgs)\n\t}\n\n\t// Now make sure we trunk if setting to lower value.\n\tcfg.MaxMsgsPer = 10\n\terr = ms.UpdateConfig(cfg)\n\trequire_NoError(t, err)\n\n\tss = ms.SubjectsState(\"foo\")[\"foo\"]\n\tif ss.Msgs != 10 {\n\t\tt.Fatalf(\"Expected to have %d stored, got %d\", 10, ss.Msgs)\n\t}\n}\n\nfunc TestMemStoreStreamTruncateReset(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\tfor i := 0; i < 1000; i++ {\n\t\t_, _, err := ms.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Reset everything\n\trequire_NoError(t, ms.Truncate(0))\n\n\tstate := ms.State()\n\trequire_True(t, state.Msgs == 0)\n\trequire_True(t, state.Bytes == 0)\n\trequire_True(t, state.FirstSeq == 0)\n\trequire_True(t, state.LastSeq == 0)\n\trequire_True(t, state.NumSubjects == 0)\n\trequire_True(t, state.NumDeleted == 0)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_, _, err := ms.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tstate = ms.State()\n\trequire_True(t, state.Msgs == 1000)\n\trequire_True(t, state.Bytes == 30000)\n\trequire_True(t, state.FirstSeq == 1)\n\trequire_True(t, state.LastSeq == 1000)\n\trequire_True(t, state.NumSubjects == 1)\n\trequire_True(t, state.NumDeleted == 0)\n}\n\nfunc TestMemStoreStreamCompactMultiBlockSubjectInfo(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"foo.*\"},\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfor i := 0; i < 1000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t_, _, err := ms.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Compact such that we know we throw blocks away from the beginning.\n\tdeleted, err := ms.Compact(501)\n\trequire_NoError(t, err)\n\trequire_True(t, deleted == 500)\n\n\t// Make sure we adjusted for subjects etc.\n\tstate := ms.State()\n\trequire_True(t, state.NumSubjects == 500)\n}\n\nfunc TestMemStoreSubjectsTotals(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"*.*\"},\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfmap := make(map[int]int)\n\tbmap := make(map[int]int)\n\n\tvar m map[int]int\n\tvar ft string\n\n\tfor i := 0; i < 10_000; i++ {\n\t\t// Flip coin for prefix\n\t\tif rand.Intn(2) == 0 {\n\t\t\tft, m = \"foo\", fmap\n\t\t} else {\n\t\t\tft, m = \"bar\", bmap\n\t\t}\n\t\tdt := rand.Intn(100)\n\t\tsubj := fmt.Sprintf(\"%s.%d\", ft, dt)\n\t\tm[dt]++\n\n\t\t_, _, err := ms.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now test SubjectsTotal\n\tfor dt, total := range fmap {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", dt)\n\t\tm := ms.SubjectsTotals(subj)\n\t\tif m[subj] != uint64(total) {\n\t\t\tt.Fatalf(\"Expected %q to have %d total, got %d\", subj, total, m[subj])\n\t\t}\n\t}\n\n\t// Check fmap.\n\tif st := ms.SubjectsTotals(\"foo.*\"); len(st) != len(fmap) {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", len(fmap), \"foo.*\", len(st))\n\t} else {\n\t\texpected := 0\n\t\tfor _, n := range fmap {\n\t\t\texpected += n\n\t\t}\n\t\treceived := uint64(0)\n\t\tfor _, n := range st {\n\t\t\treceived += n\n\t\t}\n\t\tif received != uint64(expected) {\n\t\t\tt.Fatalf(\"Expected %d total but got %d\", expected, received)\n\t\t}\n\t}\n\n\t// Check bmap.\n\tif st := ms.SubjectsTotals(\"bar.*\"); len(st) != len(bmap) {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", len(bmap), \"bar.*\", len(st))\n\t} else {\n\t\texpected := 0\n\t\tfor _, n := range bmap {\n\t\t\texpected += n\n\t\t}\n\t\treceived := uint64(0)\n\t\tfor _, n := range st {\n\t\t\treceived += n\n\t\t}\n\t\tif received != uint64(expected) {\n\t\t\tt.Fatalf(\"Expected %d total but got %d\", expected, received)\n\t\t}\n\t}\n\n\t// All with pwc match.\n\tif st, expected := ms.SubjectsTotals(\"*.*\"), len(bmap)+len(fmap); len(st) != expected {\n\t\tt.Fatalf(\"Expected %d subjects for %q, got %d\", expected, \"*.*\", len(st))\n\t}\n}\n\nfunc TestMemStoreNumPending(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: MemoryStorage,\n\t\tSubjects: []string{\"*.*.*.*\"},\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\ttokens := []string{\"foo\", \"bar\", \"baz\"}\n\tgenSubj := func() string {\n\t\treturn fmt.Sprintf(\"%s.%s.%s.%s\",\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t)\n\t}\n\n\tfor i := 0; i < 50_000; i++ {\n\t\tsubj := genSubj()\n\t\t_, _, err := ms.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tstate := ms.State()\n\n\t// Scan one by one for sanity check against other calculations.\n\tsanityCheck := func(sseq uint64, filter string) SimpleState {\n\t\tt.Helper()\n\t\tvar ss SimpleState\n\t\tvar smv StoreMsg\n\t\t// For here we know 0 is invalid, set to 1.\n\t\tif sseq == 0 {\n\t\t\tsseq = 1\n\t\t}\n\t\tfor seq := sseq; seq <= state.LastSeq; seq++ {\n\t\t\tsm, err := ms.LoadMsg(seq, &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Encountered error %v loading sequence: %d\", err, seq)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif subjectIsSubsetMatch(sm.subj, filter) {\n\t\t\t\tss.Msgs++\n\t\t\t\tss.Last = seq\n\t\t\t\tif ss.First == 0 || seq < ss.First {\n\t\t\t\t\tss.First = seq\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn ss\n\t}\n\n\tcheck := func(sseq uint64, filter string) {\n\t\tt.Helper()\n\t\tnp, lvs, err := ms.NumPending(sseq, filter, false)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"NumPending error: %v\", err)\n\t\t}\n\t\tss, err := ms.FilteredState(sseq, filter)\n\t\trequire_NoError(t, err)\n\t\tsss := sanityCheck(sseq, filter)\n\t\tif lvs != state.LastSeq {\n\t\t\tt.Fatalf(\"Expected NumPending to return valid through last of %d but got %d\", state.LastSeq, lvs)\n\t\t}\n\t\tif ss.Msgs != np {\n\t\t\tt.Fatalf(\"NumPending of %d did not match ss.Msgs of %d\", np, ss.Msgs)\n\t\t}\n\t\tif ss != sss {\n\t\t\tt.Fatalf(\"Failed sanity check, expected %+v got %+v\", sss, ss)\n\t\t}\n\t}\n\n\tsanityCheckLastOnly := func(sseq uint64, filter string) SimpleState {\n\t\tt.Helper()\n\t\tvar ss SimpleState\n\t\tvar smv StoreMsg\n\t\t// For here we know 0 is invalid, set to 1.\n\t\tif sseq == 0 {\n\t\t\tsseq = 1\n\t\t}\n\t\tseen := make(map[string]bool)\n\t\tfor seq := state.LastSeq; seq >= sseq; seq-- {\n\t\t\tsm, err := ms.LoadMsg(seq, &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Encountered error %v loading sequence: %d\", err, seq)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !seen[sm.subj] && subjectIsSubsetMatch(sm.subj, filter) {\n\t\t\t\tss.Msgs++\n\t\t\t\tif ss.Last == 0 {\n\t\t\t\t\tss.Last = seq\n\t\t\t\t}\n\t\t\t\tif ss.First == 0 || seq < ss.First {\n\t\t\t\t\tss.First = seq\n\t\t\t\t}\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\treturn ss\n\t}\n\n\tcheckLastOnly := func(sseq uint64, filter string) {\n\t\tt.Helper()\n\t\tnp, lvs, err := ms.NumPending(sseq, filter, true)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"NumPending error: %v\", err)\n\t\t}\n\t\tss := sanityCheckLastOnly(sseq, filter)\n\t\tif lvs != state.LastSeq {\n\t\t\tt.Fatalf(\"Expected NumPending to return valid through last of %d but got %d\", state.LastSeq, lvs)\n\t\t}\n\t\tif ss.Msgs != np {\n\t\t\tt.Fatalf(\"NumPending of %d did not match ss.Msgs of %d\", np, ss.Msgs)\n\t\t}\n\t}\n\n\tstartSeqs := []uint64{0, 1, 2, 200, 444, 555, 2222, 8888, 12_345, 28_222, 33_456, 44_400, 49_999}\n\tcheckSubs := []string{\"foo.>\", \"*.bar.>\", \"foo.bar.*.baz\", \"*.bar.>\", \"*.foo.bar.*\", \"foo.foo.bar.baz\"}\n\n\tfor _, filter := range checkSubs {\n\t\tfor _, start := range startSeqs {\n\t\t\tcheck(start, filter)\n\t\t\tcheckLastOnly(start, filter)\n\t\t}\n\t}\n}\n\nfunc TestMemStoreInitialFirstSeq(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tStorage: MemoryStorage,\n\t\tFirstSeq: 1000,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tseq, _, err := ms.StoreMsg(\"A\", nil, []byte(\"OK\"), 0)\n\trequire_NoError(t, err)\n\tif seq != 1000 {\n\t\tt.Fatalf(\"Message should have been sequence 1000 but was %d\", seq)\n\t}\n\n\tseq, _, err = ms.StoreMsg(\"B\", nil, []byte(\"OK\"), 0)\n\trequire_NoError(t, err)\n\tif seq != 1001 {\n\t\tt.Fatalf(\"Message should have been sequence 1001 but was %d\", seq)\n\t}\n\n\tvar state StreamState\n\tms.FastState(&state)\n\tswitch {\n\tcase state.Msgs != 2:\n\t\tt.Fatalf(\"Expected 2 messages, got %d\", state.Msgs)\n\tcase state.FirstSeq != 1000:\n\t\tt.Fatalf(\"Expected first seq 1000, got %d\", state.FirstSeq)\n\tcase state.LastSeq != 1001:\n\t\tt.Fatalf(\"Expected last seq 1001, got %d\", state.LastSeq)\n\t}\n}\n\nfunc TestMemStoreDeleteBlocks(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Put in 10_000 msgs.\n\ttotal := 10_000\n\tfor i := 0; i < total; i++ {\n\t\t_, _, err := ms.StoreMsg(\"A\", nil, []byte(\"OK\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now pick 5k random sequences.\n\tdelete := 5000\n\tdeleteMap := make(map[int]struct{}, delete)\n\tfor len(deleteMap) < delete {\n\t\tdeleteMap[rand.Intn(total)+1] = struct{}{}\n\t}\n\t// Now remove?\n\tfor seq := range deleteMap {\n\t\tms.RemoveMsg(uint64(seq))\n\t}\n\n\tvar state StreamState\n\tms.FastState(&state)\n\n\t// For now we just track via one dmap.\n\tms.mu.RLock()\n\tdmap := ms.dmap.Clone()\n\tms.mu.RUnlock()\n\n\trequire_True(t, dmap.Size() == state.NumDeleted)\n}\n\n// https://github.com/nats-io/nats-server/issues/4850\nfunc TestMemStoreGetSeqFromTimeWithLastDeleted(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Put in 1000 msgs.\n\ttotal := 1000\n\tvar st time.Time\n\tfor i := 1; i <= total; i++ {\n\t\t_, _, err := ms.StoreMsg(\"A\", nil, []byte(\"OK\"), 0)\n\t\trequire_NoError(t, err)\n\t\tif i == total/2 {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\tst = time.Now()\n\t\t}\n\t}\n\t// Delete last 100\n\tfor seq := total - 100; seq <= total; seq++ {\n\t\tms.RemoveMsg(uint64(seq))\n\t}\n\n\t// Make sure this does not panic with last sequence no longer accessible.\n\tseq := ms.GetSeqFromTime(st)\n\t// Make sure we get the right value.\n\trequire_Equal(t, seq, 501)\n}\n\nfunc TestMemStoreSkipMsgs(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// Test on empty FS first.\n\t// Make sure wrong starting sequence fails.\n\terr = ms.SkipMsgs(10, 100)\n\trequire_Error(t, err, ErrSequenceMismatch)\n\n\terr = ms.SkipMsgs(1, 100)\n\trequire_NoError(t, err)\n\n\tstate := ms.State()\n\trequire_Equal(t, state.FirstSeq, 101)\n\trequire_Equal(t, state.LastSeq, 100)\n\n\t// Now add alot.\n\terr = ms.SkipMsgs(101, 100_000)\n\trequire_NoError(t, err)\n\tstate = ms.State()\n\trequire_Equal(t, state.FirstSeq, 100_101)\n\trequire_Equal(t, state.LastSeq, 100_100)\n\n\t// Now add in a message, and then skip to check dmap.\n\tms, err = newMemStore(cfg)\n\trequire_NoError(t, err)\n\tms.StoreMsg(\"foo\", nil, nil, 0)\n\n\terr = ms.SkipMsgs(2, 10)\n\trequire_NoError(t, err)\n\tstate = ms.State()\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 11)\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.NumDeleted, 10)\n\trequire_Equal(t, len(state.Deleted), 10)\n\n\t// Check Fast State too.\n\tstate.Deleted = nil\n\tms.FastState(&state)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 11)\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.NumDeleted, 10)\n}\n\nfunc TestMemStoreMultiLastSeqs(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.*\", \"bar.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 0; i < 33; i++ {\n\t\tms.StoreMsg(\"foo.foo\", nil, msg, 0)\n\t\tms.StoreMsg(\"foo.bar\", nil, msg, 0)\n\t\tms.StoreMsg(\"foo.baz\", nil, msg, 0)\n\t}\n\tfor i := 0; i < 33; i++ {\n\t\tms.StoreMsg(\"bar.foo\", nil, msg, 0)\n\t\tms.StoreMsg(\"bar.bar\", nil, msg, 0)\n\t\tms.StoreMsg(\"bar.baz\", nil, msg, 0)\n\t}\n\n\tcheckResults := func(seqs, expected []uint64) {\n\t\tt.Helper()\n\t\tif len(seqs) != len(expected) {\n\t\t\tt.Fatalf(\"Expected %+v got %+v\", expected, seqs)\n\t\t}\n\t\tfor i := range seqs {\n\t\t\tif seqs[i] != expected[i] {\n\t\t\t\tt.Fatalf(\"Expected %+v got %+v\", expected, seqs)\n\t\t\t}\n\t\t}\n\t}\n\n\t// UpTo sequence 3. Tests block split.\n\tseqs, err := ms.MultiLastSeqs([]string{\"foo.*\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\t// Up to last sequence of the stream.\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo.*\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n\t// Check for bar.* at the end.\n\tseqs, err = ms.MultiLastSeqs([]string{\"bar.*\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\t// This should find nothing.\n\tseqs, err = ms.MultiLastSeqs([]string{\"bar.*\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, nil)\n\n\t// Do multiple subjects explicitly.\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo.foo\", \"foo.bar\", \"foo.baz\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo.foo\", \"foo.bar\", \"foo.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n\tseqs, err = ms.MultiLastSeqs([]string{\"bar.foo\", \"bar.bar\", \"bar.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\tseqs, err = ms.MultiLastSeqs([]string{\"bar.foo\", \"bar.bar\", \"bar.baz\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, nil)\n\n\t// Check single works\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo.foo\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1})\n\n\t// Now test that we properly de-duplicate between filters.\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo.*\", \"foo.bar\"}, 3, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{1, 2, 3})\n\tseqs, err = ms.MultiLastSeqs([]string{\"bar.>\", \"bar.bar\", \"bar.baz\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{196, 197, 198})\n\n\t// All\n\tseqs, err = ms.MultiLastSeqs([]string{\">\"}, 0, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99, 196, 197, 198})\n\tseqs, err = ms.MultiLastSeqs([]string{\">\"}, 99, -1)\n\trequire_NoError(t, err)\n\tcheckResults(seqs, []uint64{97, 98, 99})\n}\n\nfunc TestMemStoreMultiLastSeqsMaxAllowed(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 100; i++ {\n\t\tms.StoreMsg(fmt.Sprintf(\"foo.%d\", i), nil, msg, 0)\n\t}\n\t// Test that if we specify maxAllowed that we get the correct error.\n\tseqs, err := ms.MultiLastSeqs([]string{\"foo.*\"}, 0, 10)\n\trequire_True(t, seqs == nil)\n\trequire_Error(t, err, ErrTooManyResults)\n}\n\n// Bug would cause PurgeEx to fail if it encountered a deleted msg at sequence to delete up to.\nfunc TestMemStorePurgeExWithDeletedMsgs(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\tms.StoreMsg(\"foo\", nil, msg, 0)\n\t}\n\tms.RemoveMsg(2)\n\tms.RemoveMsg(9) // This was the bug\n\n\tn, err := ms.PurgeEx(_EMPTY_, 9, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, n, 7)\n\n\tvar state StreamState\n\tms.FastState(&state)\n\trequire_Equal(t, state.FirstSeq, 10)\n\trequire_Equal(t, state.LastSeq, 10)\n\trequire_Equal(t, state.Msgs, 1)\n}\n\n// When all messages are deleted we should have a state of first = last + 1.\nfunc TestMemStoreDeleteAllFirstSequenceCheck(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tfor i := 1; i <= 10; i++ {\n\t\tms.StoreMsg(\"foo\", nil, msg, 0)\n\t}\n\tfor seq := uint64(1); seq <= 10; seq++ {\n\t\tms.RemoveMsg(seq)\n\t}\n\tvar state StreamState\n\tms.FastState(&state)\n\trequire_Equal(t, state.FirstSeq, 11)\n\trequire_Equal(t, state.LastSeq, 10)\n\trequire_Equal(t, state.Msgs, 0)\n}\n\nfunc TestMemStoreNumPendingMulti(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"ev.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\ttotalMsgs := 100_000\n\ttotalSubjects := 10_000\n\tnumFiltered := 5000\n\tstartSeq := uint64(5_000 + rand.Intn(90_000))\n\n\tsubjects := make([]string, 0, totalSubjects)\n\tfor i := 0; i < totalSubjects; i++ {\n\t\tsubjects = append(subjects, fmt.Sprintf(\"ev.%s\", nuid.Next()))\n\t}\n\n\t// Put in 100k msgs with random subjects.\n\tmsg := bytes.Repeat([]byte(\"ZZZ\"), 333)\n\tfor i := 0; i < totalMsgs; i++ {\n\t\t_, _, err = ms.StoreMsg(subjects[rand.Intn(totalSubjects)], nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now we want to do a calculate NumPendingMulti.\n\tfilters := gsl.NewSublist[struct{}]()\n\tfor filters.Count() < uint32(numFiltered) {\n\t\tfilter := subjects[rand.Intn(totalSubjects)]\n\t\tif !filters.HasInterest(filter) {\n\t\t\tfilters.Insert(filter, struct{}{})\n\t\t}\n\t}\n\n\t// Use new function.\n\ttotal, _, err := ms.NumPendingMulti(startSeq, filters, false)\n\trequire_NoError(t, err)\n\n\t// Check our results.\n\tvar checkTotal uint64\n\tvar smv StoreMsg\n\tfor seq := startSeq; seq <= uint64(totalMsgs); seq++ {\n\t\tsm, err := ms.LoadMsg(seq, &smv)\n\t\trequire_NoError(t, err)\n\t\tif filters.HasInterest(sm.subj) {\n\t\t\tcheckTotal++\n\t\t}\n\t}\n\trequire_Equal(t, total, checkTotal)\n}\n\nfunc TestMemStoreNumPendingBug(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\t// 12 msgs total\n\tfor _, subj := range []string{\"foo.foo\", \"foo.bar\", \"foo.baz\", \"foo.zzz\"} {\n\t\tms.StoreMsg(\"foo.aaa\", nil, nil, 0)\n\t\tms.StoreMsg(subj, nil, nil, 0)\n\t\tms.StoreMsg(subj, nil, nil, 0)\n\t}\n\ttotal, _, err := ms.NumPending(4, \"foo.*\", false)\n\trequire_NoError(t, err)\n\n\tvar checkTotal uint64\n\tvar smv StoreMsg\n\tfor seq := 4; seq <= 12; seq++ {\n\t\tsm, err := ms.LoadMsg(uint64(seq), &smv)\n\t\trequire_NoError(t, err)\n\t\tif subjectIsSubsetMatch(sm.subj, \"foo.*\") {\n\t\t\tcheckTotal++\n\t\t}\n\t}\n\trequire_Equal(t, total, checkTotal)\n}\n\nfunc TestMemStorePurgeLeaksDmap(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, _, err = ms.StoreMsg(\"foo\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := uint64(2); i <= 9; i++ {\n\t\t_, err = ms.RemoveMsg(i)\n\t\trequire_NoError(t, err)\n\t}\n\tms.mu.Lock()\n\tdmaps := ms.dmap.Size()\n\tms.mu.Unlock()\n\trequire_Equal(t, dmaps, 8)\n\n\tpurged, err := ms.Purge()\n\trequire_NoError(t, err)\n\trequire_Equal(t, purged, 2)\n\n\tms.mu.Lock()\n\tdmaps = ms.dmap.Size()\n\tms.mu.Unlock()\n\trequire_Equal(t, dmaps, 0)\n}\n\nfunc TestMemStoreMessageTTL(t *testing.T) {\n\tfs, err := newMemStore(\n\t\t&StreamConfig{Name: \"zzz\", Subjects: []string{\"test\"}, Storage: MemoryStorage, AllowMsgTTL: true},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tttl := int64(1) // 1 second\n\n\tfor i := 1; i <= 10; i++ {\n\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, ttl)\n\t\trequire_NoError(t, err)\n\t}\n\n\tvar ss StreamState\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 10)\n\n\ttime.Sleep(time.Second * 2)\n\n\tfs.FastState(&ss)\n\trequire_Equal(t, ss.FirstSeq, 11)\n\trequire_Equal(t, ss.LastSeq, 10)\n\trequire_Equal(t, ss.Msgs, 0)\n}\n\nfunc TestMemStoreSubjectDeleteMarkers(t *testing.T) {\n\tfs, err := newMemStore(\n\t\t&StreamConfig{\n\t\t\tName: \"zzz\", Subjects: []string{\"test\"}, Storage: MemoryStorage,\n\t\t\tMaxAge: time.Second, AllowMsgTTL: true,\n\t\t\tSubjectDeleteMarkerTTL: time.Second,\n\t\t},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Capture subject delete marker proposals.\n\tch := make(chan *inMsg, 1)\n\tfs.rmcb = func(seq uint64) {\n\t\t_, err := fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t}\n\tfs.pmsgcb = func(im *inMsg) {\n\t\tch <- im\n\t}\n\n\t// Store three messages that will expire because of MaxAge.\n\tfor i := 0; i < 3; i++ {\n\t\t_, _, err = fs.StoreMsg(\"test\", nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Wait for MaxAge to pass.\n\ttime.Sleep(time.Second + time.Millisecond*500)\n\n\t// We should have placed a subject delete marker.\n\tim := require_ChanRead(t, ch, time.Second*5)\n\trequire_Equal(t, bytesToString(getHeader(JSMarkerReason, im.hdr)), JSMarkerReasonMaxAge)\n\trequire_Equal(t, bytesToString(getHeader(JSMessageTTL, im.hdr)), \"1s\")\n}\n\nfunc TestMemStoreAllLastSeqs(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"*.*\"},\n\t\tMaxMsgsPer: 50,\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubjs := []string{\"foo.foo\", \"foo.bar\", \"foo.baz\", \"bar.foo\", \"bar.bar\", \"bar.baz\"}\n\tmsg := []byte(\"abc\")\n\n\tfor i := 0; i < 100_000; i++ {\n\t\tsubj := subjs[rand.Intn(len(subjs))]\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\n\texpected := make([]uint64, 0, len(subjs))\n\tvar smv StoreMsg\n\tfor _, subj := range subjs {\n\t\tsm, err := ms.LoadLastMsg(subj, &smv)\n\t\trequire_NoError(t, err)\n\t\texpected = append(expected, sm.seq)\n\t}\n\tslices.Sort(expected)\n\n\tseqs, err := ms.AllLastSeqs()\n\trequire_NoError(t, err)\n\trequire_True(t, reflect.DeepEqual(seqs, expected))\n}\n\nfunc TestMemStoreUpdateConfigTTLState(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\">\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\trequire_Equal(t, ms.ttls, nil)\n\n\tcfg.AllowMsgTTL = true\n\trequire_NoError(t, ms.UpdateConfig(cfg))\n\trequire_NotEqual(t, ms.ttls, nil)\n\n\tcfg.AllowMsgTTL = false\n\trequire_NoError(t, ms.UpdateConfig(cfg))\n\trequire_Equal(t, ms.ttls, nil)\n}\n\nfunc TestMemStoreSubjectForSeq(t *testing.T) {\n\tcfg := StreamConfig{\n\t\tName: \"foo\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(&cfg)\n\trequire_NoError(t, err)\n\n\tseq, _, err := ms.StoreMsg(\"foo.bar\", nil, nil, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, seq, 1)\n\n\t_, err = ms.SubjectForSeq(0)\n\trequire_Error(t, err, ErrStoreMsgNotFound)\n\n\tsubj, err := ms.SubjectForSeq(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, subj, \"foo.bar\")\n\n\t_, err = ms.SubjectForSeq(2)\n\trequire_Error(t, err, ErrStoreMsgNotFound)\n}\n\nfunc TestMemStoreMessageSchedule(t *testing.T) {\n\tfs, err := newMemStore(\n\t\t&StreamConfig{\n\t\t\tName: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: MemoryStorage,\n\t\t\tAllowMsgSchedules: true,\n\t\t},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Capture message schedule proposals.\n\tch := make(chan *inMsg, 1)\n\tfs.pmsgcb = func(im *inMsg) {\n\t\tch <- im\n\t}\n\n\t// Store a single message schedule.\n\tschedule := time.Now().Add(time.Second).Format(time.RFC3339Nano)\n\thdr := genHeader(nil, JSSchedulePattern, fmt.Sprintf(\"@at %s\", schedule))\n\thdr = genHeader(hdr, JSScheduleTarget, \"foo.target\")\n\t_, _, err = fs.StoreMsg(\"foo.schedule\", hdr, nil, 0)\n\trequire_NoError(t, err)\n\n\t// We should have published a scheduled message.\n\tim := require_ChanRead(t, ch, time.Second*5)\n\trequire_Equal(t, im.subj, \"foo.target\")\n\trequire_Equal(t, bytesToString(getHeader(JSScheduler, im.hdr)), \"foo.schedule\")\n\trequire_Equal(t, bytesToString(getHeader(JSScheduleNext, im.hdr)), JSScheduleNextPurge)\n}\n\nfunc TestMemStoreNextWildcardMatch(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"msg\")\n\tstoreN := func(subj string, n int) {\n\t\tt.Helper()\n\t\tfor range n {\n\t\t\t_, _, err := ms.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\tstoreN(\"foo.bar.a\", 1)\n\tstoreN(\"foo.baz.bar\", 10)\n\tstoreN(\"foo.bar.b\", 1)\n\tstoreN(\"foo.baz.bar\", 10)\n\tstoreN(\"foo.baz.bar.no.match\", 10)\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tfirst, last, found := ms.nextWildcardMatchLocked(\"foo.bar.*\", 0)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 1)\n\trequire_Equal(t, last, 12)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.bar.*\", 1)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 1)\n\trequire_Equal(t, last, 12)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.bar.*\", 2)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 12)\n\trequire_Equal(t, last, 12)\n\n\t_, _, found = ms.nextWildcardMatchLocked(\"foo.bar.*\", first+1)\n\trequire_False(t, found)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.baz.*\", 1)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 2)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.baz.*\", 11)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 11)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.baz.*\", 12)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 12)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.baz.*\", 22)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 22)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.baz.*\", 23)\n\trequire_False(t, found)\n\trequire_Equal(t, first, 0)\n\trequire_Equal(t, last, 0)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.nope.*\", 1)\n\trequire_False(t, found)\n\trequire_Equal(t, first, 0)\n\trequire_Equal(t, last, 0)\n\n\tfirst, last, found = ms.nextWildcardMatchLocked(\"foo.>\", 1)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 1)\n\trequire_Equal(t, last, 32)\n}\n\nfunc TestMemStoreNextLiteralMatch(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"msg\")\n\tstoreN := func(subj string, n int) {\n\t\tt.Helper()\n\t\tfor range n {\n\t\t\t_, _, err := ms.StoreMsg(subj, nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n\tstoreN(\"foo.bar.a\", 1) // seq 1\n\tstoreN(\"foo.baz.bar\", 10) // seqs 2-11\n\tstoreN(\"foo.bar.b\", 1) // seq 12\n\tstoreN(\"foo.baz.bar\", 10) // seqs 13-22\n\tstoreN(\"foo.baz.bar.no.match\", 10) // seqs 23-32\n\n\tms.mu.Lock()\n\tdefer ms.mu.Unlock()\n\n\tfirst, last, found := ms.nextLiteralMatchLocked(\"foo.bar.a\", 0)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 1)\n\trequire_Equal(t, last, 1)\n\n\t_, _, found = ms.nextLiteralMatchLocked(\"foo.bar.a\", 2)\n\trequire_False(t, found)\n\n\tfirst, last, found = ms.nextLiteralMatchLocked(\"foo.baz.bar\", 1)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 2)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextLiteralMatchLocked(\"foo.baz.bar\", 11)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 11)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextLiteralMatchLocked(\"foo.baz.bar\", 22)\n\trequire_True(t, found)\n\trequire_Equal(t, first, 22)\n\trequire_Equal(t, last, 22)\n\n\tfirst, last, found = ms.nextLiteralMatchLocked(\"foo.baz.bar\", 23)\n\trequire_False(t, found)\n\trequire_Equal(t, first, 0)\n\trequire_Equal(t, last, 0)\n\n\tfirst, last, found = ms.nextLiteralMatchLocked(\"foo.nope\", 1)\n\trequire_False(t, found)\n\trequire_Equal(t, first, 0)\n\trequire_Equal(t, last, 0)\n\n}\n\nfunc TestMemStoreMultiLastSeqsDoesNotUseStaleLastValue(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo\", \"bar\", \"baz\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tsubjects := []string{\"foo\", \"bar\", \"foo\", \" baz\", \"foo\", \"foo\", \"bar\", \"foo\", \"bar\", \"baz\"}\n\tfor _, s := range subjects {\n\t\t_, _, err := ms.StoreMsg(s, nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Initially we expect last seq for `foo` to be 8\n\tseqs, err := ms.MultiLastSeqs([]string{\"foo\", \"bar\"}, 0, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(seqs), 2)\n\trequire_Equal(t, seqs[0], uint64(8))\n\n\t// Remove latest foo message\n\tremoved, err := ms.RemoveMsg(8)\n\trequire_NoError(t, err)\n\trequire_True(t, removed)\n\n\t// If bug is present: MultiLastSeqs returns last sequence\n\t// 8 for subject `foo`.\n\t// After removal, expect last sequence 6 for subject `foo`.\n\tseqs, err = ms.MultiLastSeqs([]string{\"foo\", \"bar\"}, 0, 0)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(seqs), 2)\n\trequire_Equal(t, seqs[0], uint64(6))\n}\n\n///////////////////////////////////////////////////////////////////////////\n// Benchmarks\n///////////////////////////////////////////////////////////////////////////\n\nfunc Benchmark_MemStoreNumPendingWithLargeInteriorDeletesScan(b *testing.B) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.*.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(b, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tms.StoreMsg(\"foo.bar.baz\", nil, msg, 0)\n\tfor i := 1; i <= 1_000_000; i++ {\n\t\tms.SkipMsg(0)\n\t}\n\tms.StoreMsg(\"foo.bar.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttotal, _, err := ms.NumPending(600_000, \"foo.*.baz\", false)\n\t\trequire_NoError(b, err)\n\t\tif total != 1 {\n\t\t\tb.Fatalf(\"Expected total of 2 got %d\", total)\n\t\t}\n\t}\n}\n\nfunc Benchmark_MemStoreNumPendingWithLargeInteriorDeletesExclude(b *testing.B) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"foo.*.*\"},\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(b, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"abc\")\n\tms.StoreMsg(\"foo.bar.baz\", nil, msg, 0)\n\tfor i := 1; i <= 1_000_000; i++ {\n\t\tms.SkipMsg(0)\n\t}\n\tms.StoreMsg(\"foo.bar.baz\", nil, msg, 0)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttotal, _, err := ms.NumPending(400_000, \"foo.*.baz\", false)\n\t\trequire_NoError(b, err)\n\t\tif total != 1 {\n\t\t\tb.Fatalf(\"Expected total of 2 got %d\", total)\n\t\t}\n\t}\n}\n\nfunc Benchmark_MemStoreSubjectStateConsistencyOptimizationPerf(b *testing.B) {\n\tcfg := &StreamConfig{Name: \"TEST\", Subjects: []string{\"foo.*\"}, Storage: MemoryStorage, MaxMsgsPer: 1}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(b, err)\n\tdefer ms.Stop()\n\n\t// Do R rounds of storing N messages.\n\t// MaxMsgsPer=1, so every unique subject that's placed only exists in the stream once.\n\t// If R=2, N=3 that means we'd place foo.0, foo.1, foo.2 in the first round, and the second\n\t// round we'd place foo.2, foo.1, foo.0, etc. This is intentional so that without any\n\t// optimizations we'd need to scan either 1 in the optimal case or N in the worst case.\n\t// Which is way more expensive than always knowing what the sequences are and it being O(1).\n\tr := max(2, b.N)\n\tn := 40_000\n\tb.ResetTimer()\n\tfor i := 0; i < r; i++ {\n\t\tfor j := 0; j < n; j++ {\n\t\t\td := j\n\t\t\tif i%2 == 0 {\n\t\t\t\td = n - j - 1\n\t\t\t}\n\t\t\tsubject := fmt.Sprintf(\"foo.%d\", d)\n\t\t\t_, _, err = ms.StoreMsg(subject, nil, nil, 0)\n\t\t\trequire_NoError(b, err)\n\t\t}\n\t}\n}\n\n// This benchmark populates a memstore and then measures\n// the time it takes to load the entire store using\n// LoadNextMsg repeatedly until the store returns ErrStoreEOF.\nfunc Benchmark_MemStoreLoadNextMsgFiltered(b *testing.B) {\n\tcases := []struct {\n\t\tname string\n\t\tmsgs int\n\t\tmatchingMsgEvery int\n\t\tfilter string\n\t\twc bool\n\t\tmatchingSubject func(i int) string\n\t\texpectLinear bool\n\t}{\n\t\t{\n\t\t\tname: \"wildcard_linear_scan\",\n\t\t\tmsgs: 10_000_000,\n\t\t\tmatchingMsgEvery: 10_000,\n\t\t\tfilter: \"foo.baz.*\",\n\t\t\twc: true,\n\t\t\tmatchingSubject: func(i int) string {\n\t\t\t\treturn fmt.Sprintf(\"foo.baz.%d\", i)\n\t\t\t},\n\t\t\texpectLinear: true,\n\t\t},\n\t\t{\n\t\t\tname: \"wildcard_bounded_scan\",\n\t\t\tmsgs: 10_000_000,\n\t\t\tmatchingMsgEvery: 100_000,\n\t\t\tfilter: \"foo.baz.*\",\n\t\t\twc: true,\n\t\t\tmatchingSubject: func(i int) string {\n\t\t\t\treturn fmt.Sprintf(\"foo.baz.%d\", i)\n\t\t\t},\n\t\t\texpectLinear: false,\n\t\t},\n\t\t{\n\t\t\tname: \"literal_bounded_scan\",\n\t\t\tmsgs: 10_000_000,\n\t\t\tmatchingMsgEvery: 100_000,\n\t\t\tfilter: \"foo.baz\",\n\t\t\twc: false,\n\t\t\tmatchingSubject: func(i int) string {\n\t\t\t\treturn \"foo.baz\"\n\t\t\t},\n\t\t\texpectLinear: false,\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tb.Run(tc.name, func(b *testing.B) {\n\t\t\tcfg := &StreamConfig{Name: \"TEST\", Subjects: []string{\"foo.>\"}, Storage: MemoryStorage}\n\t\t\tms, err := newMemStore(cfg)\n\t\t\trequire_NoError(b, err)\n\t\t\tdefer ms.Stop()\n\n\t\t\tmsg := []byte(\"ok\")\n\t\t\tfor i := range tc.msgs {\n\t\t\t\tsubject := \"foo.bar\"\n\t\t\t\tif i%tc.matchingMsgEvery == 0 {\n\t\t\t\t\tsubject = tc.matchingSubject(i)\n\t\t\t\t}\n\t\t\t\t_, _, err = ms.StoreMsg(subject, nil, msg, 0)\n\t\t\t\trequire_NoError(b, err)\n\t\t\t}\n\n\t\t\tms.mu.Lock()\n\t\t\trequire_Equal(b, ms.shouldLinearScan(tc.filter, tc.wc, 1), tc.expectLinear)\n\t\t\tms.mu.Unlock()\n\n\t\t\tvar smv StoreMsg\n\t\t\texpectedMatches := uint64(tc.msgs / tc.matchingMsgEvery)\n\n\t\t\tb.ResetTimer()\n\t\t\tfor b.Loop() {\n\t\t\t\tvar start uint64\n\t\t\t\tvar count uint64\n\t\t\t\tfor {\n\t\t\t\t\t_, start, err = ms.LoadNextMsg(tc.filter, tc.wc, start, &smv)\n\t\t\t\t\tstart++\n\t\t\t\t\tif err == ErrStoreEOF {\n\t\t\t\t\t\trequire_Equal(b, count, expectedMatches)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\trequire_NoError(b, err)\n\t\t\t\t\tcount++\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/monitor.go", "content": "// Copyright 2013-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\t\"crypto/sha256\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"expvar\"\n\t\"fmt\"\n\t\"maps\"\n\t\"math\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"runtime/pprof\"\n\t\"slices\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/server/pse\"\n)\n\n// Connz represents detailed information on current client connections.\ntype Connz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tNumConns int `json:\"num_connections\"`\n\tTotal int `json:\"total\"`\n\tOffset int `json:\"offset\"`\n\tLimit int `json:\"limit\"`\n\tConns []*ConnInfo `json:\"connections\"`\n}\n\n// ConnzOptions are the options passed to Connz()\ntype ConnzOptions struct {\n\t// Sort indicates how the results will be sorted. Check SortOpt for possible values.\n\t// Only the sort by connection ID (ByCid) is ascending, all others are descending.\n\tSort SortOpt `json:\"sort\"`\n\n\t// Username indicates if user names should be included in the results.\n\tUsername bool `json:\"auth\"`\n\n\t// Subscriptions indicates if subscriptions should be included in the results.\n\tSubscriptions bool `json:\"subscriptions\"`\n\n\t// SubscriptionsDetail indicates if subscription details should be included in the results\n\tSubscriptionsDetail bool `json:\"subscriptions_detail\"`\n\n\t// Offset is used for pagination. Connz() only returns connections starting at this\n\t// offset from the global results.\n\tOffset int `json:\"offset\"`\n\n\t// Limit is the maximum number of connections that should be returned by Connz().\n\tLimit int `json:\"limit\"`\n\n\t// Filter for this explicit client connection.\n\tCID uint64 `json:\"cid\"`\n\n\t// Filter for this explicit client connection based on the MQTT client ID\n\tMQTTClient string `json:\"mqtt_client\"`\n\n\t// Filter by connection state.\n\tState ConnState `json:\"state\"`\n\n\t// The below options only apply if auth is true.\n\n\t// Filter by username.\n\tUser string `json:\"user\"`\n\n\t// Filter by account.\n\tAccount string `json:\"acc\"`\n\n\t// Filter by subject interest\n\tFilterSubject string `json:\"filter_subject\"`\n}\n\n// ConnState is for filtering states of connections. We will only have two, open and closed.\ntype ConnState int\n\nconst (\n\t// ConnOpen filters on open clients.\n\tConnOpen = ConnState(iota)\n\t// ConnClosed filters on closed clients.\n\tConnClosed\n\t// ConnAll returns all clients.\n\tConnAll\n)\n\n// ConnInfo has detailed information on a per connection basis.\ntype ConnInfo struct {\n\tCid uint64 `json:\"cid\"`\n\tKind string `json:\"kind,omitempty\"`\n\tType string `json:\"type,omitempty\"`\n\tIP string `json:\"ip\"`\n\tPort int `json:\"port\"`\n\tStart time.Time `json:\"start\"`\n\tLastActivity time.Time `json:\"last_activity\"`\n\tStop *time.Time `json:\"stop,omitempty\"`\n\tReason string `json:\"reason,omitempty\"`\n\tRTT string `json:\"rtt,omitempty\"`\n\tUptime string `json:\"uptime\"`\n\tIdle string `json:\"idle\"`\n\tPending int `json:\"pending_bytes\"`\n\tInMsgs int64 `json:\"in_msgs\"`\n\tOutMsgs int64 `json:\"out_msgs\"`\n\tInBytes int64 `json:\"in_bytes\"`\n\tOutBytes int64 `json:\"out_bytes\"`\n\tStalls int64 `json:\"stalls,omitempty\"`\n\tNumSubs uint32 `json:\"subscriptions\"`\n\tName string `json:\"name,omitempty\"`\n\tLang string `json:\"lang,omitempty\"`\n\tVersion string `json:\"version,omitempty\"`\n\tTLSVersion string `json:\"tls_version,omitempty\"`\n\tTLSCipher string `json:\"tls_cipher_suite,omitempty\"`\n\tTLSPeerCerts []*TLSPeerCert `json:\"tls_peer_certs,omitempty\"`\n\tTLSFirst bool `json:\"tls_first,omitempty\"`\n\tAuthorizedUser string `json:\"authorized_user,omitempty\"`\n\tAccount string `json:\"account,omitempty\"`\n\tSubs []string `json:\"subscriptions_list,omitempty\"`\n\tSubsDetail []SubDetail `json:\"subscriptions_list_detail,omitempty\"`\n\tJWT string `json:\"jwt,omitempty\"`\n\tIssuerKey string `json:\"issuer_key,omitempty\"`\n\tNameTag string `json:\"name_tag,omitempty\"`\n\tTags jwt.TagList `json:\"tags,omitempty\"`\n\tMQTTClient string `json:\"mqtt_client,omitempty\"` // This is the MQTT client id\n\tProxy *ProxyInfo `json:\"proxy,omitempty\"`\n\n\t// Internal\n\trtt int64 // For fast sorting\n}\n\n// ProxyInfo represents the information about this proxied connection.\ntype ProxyInfo struct {\n\tKey string `json:\"key\"`\n}\n\n// TLSPeerCert contains basic information about a TLS peer certificate\ntype TLSPeerCert struct {\n\tSubject string `json:\"subject,omitempty\"`\n\tSubjectPKISha256 string `json:\"spki_sha256,omitempty\"`\n\tCertSha256 string `json:\"cert_sha256,omitempty\"`\n}\n\n// DefaultConnListSize is the default size of the connection list.\nconst DefaultConnListSize = 1024\n\n// DefaultSubListSize is the default size of the subscriptions list.\nconst DefaultSubListSize = 1024\n\nconst defaultStackBufSize = 10000\n\nfunc newSubsDetailList(client *client) []SubDetail {\n\tsubsDetail := make([]SubDetail, 0, len(client.subs))\n\tfor _, sub := range client.subs {\n\t\tsubsDetail = append(subsDetail, newClientSubDetail(sub))\n\t}\n\treturn subsDetail\n}\n\nfunc newSubsList(client *client) []string {\n\tsubs := make([]string, 0, len(client.subs))\n\tfor _, sub := range client.subs {\n\t\tsubs = append(subs, string(sub.subject))\n\t}\n\treturn subs\n}\n\n// Connz returns a Connz struct containing information about connections.\nfunc (s *Server) Connz(opts *ConnzOptions) (*Connz, error) {\n\tvar (\n\t\tsortOpt = ByCid\n\t\tauth bool\n\t\tsubs bool\n\t\tsubsDet bool\n\t\toffset int\n\t\tlimit = DefaultConnListSize\n\t\tcid = uint64(0)\n\t\tstate = ConnOpen\n\t\tuser string\n\t\tacc string\n\t\ta *Account\n\t\tfilter string\n\t\tmqttCID string\n\t)\n\n\tif opts != nil {\n\t\t// If no sort option given or sort is by uptime, then sort by cid\n\t\tif opts.Sort != _EMPTY_ {\n\t\t\tsortOpt = opts.Sort\n\t\t\tif !sortOpt.IsValid() {\n\t\t\t\treturn nil, fmt.Errorf(\"invalid sorting option: %s\", sortOpt)\n\t\t\t}\n\t\t}\n\n\t\t// Auth specifics.\n\t\tauth = opts.Username\n\t\tuser = opts.User\n\t\tacc = opts.Account\n\t\tmqttCID = opts.MQTTClient\n\n\t\tsubs = opts.Subscriptions\n\t\tsubsDet = opts.SubscriptionsDetail\n\t\toffset = opts.Offset\n\t\tif offset < 0 {\n\t\t\toffset = 0\n\t\t}\n\t\tlimit = opts.Limit\n\t\tif limit <= 0 {\n\t\t\tlimit = DefaultConnListSize\n\t\t}\n\t\t// state\n\t\tstate = opts.State\n\n\t\t// ByStop only makes sense on closed connections\n\t\tif sortOpt == ByStop && state != ConnClosed {\n\t\t\treturn nil, fmt.Errorf(\"sort by stop only valid on closed connections\")\n\t\t}\n\t\t// ByReason is the same.\n\t\tif sortOpt == ByReason && state != ConnClosed {\n\t\t\treturn nil, fmt.Errorf(\"sort by reason only valid on closed connections\")\n\t\t}\n\t\t// If searching by CID\n\t\tif opts.CID > 0 {\n\t\t\tcid = opts.CID\n\t\t\tlimit = 1\n\t\t}\n\t\t// If filtering by subject.\n\t\tif opts.FilterSubject != _EMPTY_ && opts.FilterSubject != fwcs {\n\t\t\tif acc == _EMPTY_ {\n\t\t\t\treturn nil, fmt.Errorf(\"filter by subject only valid with account filtering\")\n\t\t\t}\n\t\t\tfilter = opts.FilterSubject\n\t\t}\n\t}\n\n\tc := &Connz{\n\t\tOffset: offset,\n\t\tLimit: limit,\n\t\tNow: time.Now().UTC(),\n\t}\n\n\t// Open clients\n\tvar openClients []*client\n\t// Hold for closed clients if requested.\n\tvar closedClients []*closedClient\n\n\tvar clist map[uint64]*client\n\n\tif acc != _EMPTY_ {\n\t\tvar err error\n\t\ta, err = s.lookupAccount(acc)\n\t\tif err != nil {\n\t\t\treturn c, nil\n\t\t}\n\t\ta.mu.RLock()\n\t\tclist = make(map[uint64]*client, a.numLocalConnections())\n\t\tfor c := range a.clients {\n\t\t\tif c.kind == CLIENT || c.kind == LEAF {\n\t\t\t\tclist[c.cid] = c\n\t\t\t}\n\t\t}\n\t\ta.mu.RUnlock()\n\t}\n\n\t// Walk the open client list with server lock held.\n\ts.mu.RLock()\n\t// Default to all client unless filled in above.\n\tif clist == nil {\n\t\tclist = make(map[uint64]*client, len(s.clients)+len(s.leafs))\n\t\tmaps.Copy(clist, s.clients)\n\t\tmaps.Copy(clist, s.leafs)\n\t}\n\n\t// copy the server id for monitoring\n\tc.ID = s.info.ID\n\n\t// Number of total clients. The resulting ConnInfo array\n\t// may be smaller if pagination is used.\n\tswitch state {\n\tcase ConnOpen:\n\t\tc.Total = len(clist)\n\tcase ConnClosed:\n\t\tclosedClients = s.closed.closedClients()\n\t\tc.Total = len(closedClients)\n\tcase ConnAll:\n\t\tc.Total = len(clist)\n\t\tclosedClients = s.closed.closedClients()\n\t\tc.Total += len(closedClients)\n\t}\n\n\t// We may need to filter these connections.\n\tif acc != _EMPTY_ && len(closedClients) > 0 {\n\t\tvar ccc []*closedClient\n\t\tfor _, cc := range closedClients {\n\t\t\tif cc.acc != acc {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tccc = append(ccc, cc)\n\t\t}\n\t\tc.Total -= (len(closedClients) - len(ccc))\n\t\tclosedClients = ccc\n\t}\n\n\ttotalClients := c.Total\n\tif cid > 0 { // Meaning we only want 1.\n\t\ttotalClients = 1\n\t}\n\tif state == ConnOpen || state == ConnAll {\n\t\topenClients = make([]*client, 0, totalClients)\n\t}\n\n\t// Data structures for results.\n\tvar conns []ConnInfo // Limits allocs for actual ConnInfos.\n\tvar pconns ConnInfos\n\n\tswitch state {\n\tcase ConnOpen:\n\t\tconns = make([]ConnInfo, totalClients)\n\t\tpconns = make(ConnInfos, totalClients)\n\tcase ConnClosed:\n\t\tpconns = make(ConnInfos, totalClients)\n\tcase ConnAll:\n\t\tconns = make([]ConnInfo, cap(openClients))\n\t\tpconns = make(ConnInfos, totalClients)\n\t}\n\n\t// Search by individual CID.\n\tif cid > 0 {\n\t\t// Let's first check if user also selects on ConnOpen or ConnAll\n\t\t// and look for opened connections.\n\t\tif state == ConnOpen || state == ConnAll {\n\t\t\tif client := s.clients[cid]; client != nil {\n\t\t\t\topenClients = append(openClients, client)\n\t\t\t\tclosedClients = nil\n\t\t\t}\n\t\t}\n\t\t// If we did not find, and the user selected for ConnClosed or ConnAll,\n\t\t// look for closed connections.\n\t\tif len(openClients) == 0 && (state == ConnClosed || state == ConnAll) {\n\t\t\tcopyClosed := closedClients\n\t\t\tclosedClients = nil\n\t\t\tfor _, cc := range copyClosed {\n\t\t\t\tif cc.Cid == cid {\n\t\t\t\t\tclosedClients = []*closedClient{cc}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// Gather all open clients.\n\t\tif state == ConnOpen || state == ConnAll {\n\t\t\tfor _, client := range clist {\n\t\t\t\t// If we have an account specified we need to filter.\n\t\t\t\tif acc != _EMPTY_ && (client.acc == nil || client.acc.Name != acc) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Do user filtering second\n\t\t\t\tif user != _EMPTY_ && client.getRawAuthUserLock() != user {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Do mqtt client ID filtering next\n\t\t\t\tif mqttCID != _EMPTY_ && client.getMQTTClientID() != mqttCID {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\topenClients = append(openClients, client)\n\t\t\t}\n\t\t}\n\t}\n\ts.mu.RUnlock()\n\n\t// Filter by subject now if needed. We do this outside of server lock.\n\tif filter != _EMPTY_ {\n\t\tvar oc []*client\n\t\tfor _, c := range openClients {\n\t\t\tc.mu.Lock()\n\t\t\tfor _, sub := range c.subs {\n\t\t\t\tif SubjectsCollide(filter, string(sub.subject)) {\n\t\t\t\t\toc = append(oc, c)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\topenClients = oc\n\t\t}\n\t}\n\n\t// Just return with empty array if nothing here.\n\tif len(openClients) == 0 && len(closedClients) == 0 {\n\t\tc.Conns = ConnInfos{}\n\t\treturn c, nil\n\t}\n\n\t// Now whip through and generate ConnInfo entries\n\t// Open Clients\n\ti := 0\n\tfor _, client := range openClients {\n\t\tclient.mu.Lock()\n\t\tci := &conns[i]\n\t\tci.fill(client, client.nc, c.Now, auth)\n\t\t// Fill in subscription data if requested.\n\t\tif len(client.subs) > 0 {\n\t\t\tif subsDet {\n\t\t\t\tci.SubsDetail = newSubsDetailList(client)\n\t\t\t} else if subs {\n\t\t\t\tci.Subs = newSubsList(client)\n\t\t\t}\n\t\t}\n\t\t// Fill in user if auth requested.\n\t\tif auth {\n\t\t\tci.AuthorizedUser = client.getRawAuthUser()\n\t\t\tif name := client.acc.GetName(); name != globalAccountName {\n\t\t\t\tci.Account = name\n\t\t\t}\n\t\t\tci.JWT = client.opts.JWT\n\t\t\tci.IssuerKey = issuerForClient(client)\n\t\t\tci.Tags = client.tags\n\t\t\tci.NameTag = client.acc.getNameTag()\n\t\t}\n\t\tclient.mu.Unlock()\n\t\tpconns[i] = ci\n\t\ti++\n\t}\n\t// Closed Clients\n\tvar needCopy bool\n\tif subs || auth {\n\t\tneedCopy = true\n\t}\n\tfor _, cc := range closedClients {\n\t\t// If we have an account specified we need to filter.\n\t\tif acc != _EMPTY_ && cc.acc != acc {\n\t\t\tcontinue\n\t\t}\n\t\t// Do user filtering second\n\t\tif user != _EMPTY_ && cc.user != user {\n\t\t\tcontinue\n\t\t}\n\t\t// Do mqtt client ID filtering next\n\t\tif mqttCID != _EMPTY_ && cc.MQTTClient != mqttCID {\n\t\t\tcontinue\n\t\t}\n\t\t// Copy if needed for any changes to the ConnInfo\n\t\tif needCopy {\n\t\t\tcx := *cc\n\t\t\tcc = &cx\n\t\t}\n\t\t// Fill in subscription data if requested.\n\t\tif len(cc.subs) > 0 {\n\t\t\tif subsDet {\n\t\t\t\tcc.SubsDetail = cc.subs\n\t\t\t} else if subs {\n\t\t\t\tcc.Subs = make([]string, 0, len(cc.subs))\n\t\t\t\tfor _, sub := range cc.subs {\n\t\t\t\t\tcc.Subs = append(cc.Subs, sub.Subject)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Fill in user if auth requested.\n\t\tif auth {\n\t\t\tcc.AuthorizedUser = cc.user\n\t\t\tif cc.acc != _EMPTY_ && (cc.acc != globalAccountName) {\n\t\t\t\tcc.Account = cc.acc\n\t\t\t\tif acc, err := s.LookupAccount(cc.acc); err == nil {\n\t\t\t\t\tcc.NameTag = acc.getNameTag()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tpconns[i] = &cc.ConnInfo\n\t\ti++\n\t}\n\n\t// This will trip if we have filtered out client connections.\n\tif len(pconns) != i {\n\t\tpconns = pconns[:i]\n\t\ttotalClients = i\n\t}\n\n\tswitch sortOpt {\n\tcase ByCid, ByStart:\n\t\tsort.Sort(SortByCid{pconns})\n\tcase BySubs:\n\t\tsort.Sort(sort.Reverse(SortBySubs{pconns}))\n\tcase ByPending:\n\t\tsort.Sort(sort.Reverse(SortByPending{pconns}))\n\tcase ByOutMsgs:\n\t\tsort.Sort(sort.Reverse(SortByOutMsgs{pconns}))\n\tcase ByInMsgs:\n\t\tsort.Sort(sort.Reverse(SortByInMsgs{pconns}))\n\tcase ByOutBytes:\n\t\tsort.Sort(sort.Reverse(SortByOutBytes{pconns}))\n\tcase ByInBytes:\n\t\tsort.Sort(sort.Reverse(SortByInBytes{pconns}))\n\tcase ByLast:\n\t\tsort.Sort(sort.Reverse(SortByLast{pconns}))\n\tcase ByIdle:\n\t\tsort.Sort(sort.Reverse(SortByIdle{pconns, c.Now}))\n\tcase ByUptime:\n\t\tsort.Sort(SortByUptime{pconns, time.Now()})\n\tcase ByStop:\n\t\tsort.Sort(sort.Reverse(SortByStop{pconns}))\n\tcase ByReason:\n\t\tsort.Sort(SortByReason{pconns})\n\tcase ByRTT:\n\t\tsort.Sort(sort.Reverse(SortByRTT{pconns}))\n\t}\n\n\tminoff := c.Offset\n\tmaxoff := c.Offset + c.Limit\n\n\tmaxIndex := totalClients\n\n\t// Make sure these are sane.\n\tif minoff > maxIndex {\n\t\tminoff = maxIndex\n\t}\n\tif maxoff > maxIndex {\n\t\tmaxoff = maxIndex\n\t}\n\n\t// Now pare down to the requested size.\n\t// TODO(dlc) - for very large number of connections we\n\t// could save the whole list in a hash, send hash on first\n\t// request and allow users to use has for subsequent pages.\n\t// Low TTL, say < 1sec.\n\tc.Conns = pconns[minoff:maxoff]\n\tc.NumConns = len(c.Conns)\n\n\treturn c, nil\n}\n\n// Fills in the ConnInfo from the client.\n// client should be locked.\nfunc (ci *ConnInfo) fill(client *client, nc net.Conn, now time.Time, auth bool) {\n\t// For fast sort if required.\n\trtt := client.getRTT()\n\tci.rtt = int64(rtt)\n\n\tci.Cid = client.cid\n\tci.MQTTClient = client.getMQTTClientID()\n\tci.Kind = client.kindString()\n\tci.Type = client.clientTypeString()\n\tci.Start = client.start\n\tci.LastActivity = client.last\n\tci.Uptime = myUptime(now.Sub(client.start))\n\tci.Idle = myUptime(now.Sub(client.last))\n\tci.RTT = rtt.String()\n\tci.OutMsgs = client.outMsgs\n\tci.OutBytes = client.outBytes\n\tci.NumSubs = uint32(len(client.subs))\n\tci.Pending = int(client.out.pb)\n\tci.Name = client.opts.Name\n\tci.Lang = client.opts.Lang\n\tci.Version = client.opts.Version\n\t// inMsgs and inBytes are updated outside of the client's lock, so\n\t// we need to use atomic here.\n\tci.InMsgs = atomic.LoadInt64(&client.inMsgs)\n\tci.InBytes = atomic.LoadInt64(&client.inBytes)\n\tci.Stalls = atomic.LoadInt64(&client.stalls)\n\tci.Proxy = createProxyInfo(client)\n\n\t// If the connection is gone, too bad, we won't set TLSVersion and TLSCipher.\n\t// Exclude clients that are still doing handshake so we don't block in\n\t// ConnectionState().\n\tif client.flags.isSet(handshakeComplete) && nc != nil {\n\t\tif conn, ok := nc.(*tls.Conn); ok {\n\t\t\tcs := conn.ConnectionState()\n\t\t\tci.TLSVersion = tlsVersion(cs.Version)\n\t\t\tci.TLSCipher = tls.CipherSuiteName(cs.CipherSuite)\n\t\t\tif auth && len(cs.PeerCertificates) > 0 {\n\t\t\t\tci.TLSPeerCerts = makePeerCerts(cs.PeerCertificates)\n\t\t\t}\n\t\t\tci.TLSFirst = client.flags.isSet(didTLSFirst)\n\t\t}\n\t}\n\n\tif client.port != 0 {\n\t\tci.Port = int(client.port)\n\t\tci.IP = client.host\n\t}\n}\n\n// If this client came from a trusted proxy, this will return a ProxyInfo\n// to be used in ConnInfo or LeafInfo.\n//\n// Client lock must be held on entry.\nfunc createProxyInfo(c *client) *ProxyInfo {\n\tif c.proxyKey == _EMPTY_ {\n\t\treturn nil\n\t}\n\treturn &ProxyInfo{Key: c.proxyKey}\n}\n\nfunc makePeerCerts(pc []*x509.Certificate) []*TLSPeerCert {\n\tres := make([]*TLSPeerCert, len(pc))\n\tfor i, c := range pc {\n\t\ttmp := sha256.Sum256(c.RawSubjectPublicKeyInfo)\n\t\tssha := hex.EncodeToString(tmp[:])\n\t\ttmp = sha256.Sum256(c.Raw)\n\t\tcsha := hex.EncodeToString(tmp[:])\n\t\tres[i] = &TLSPeerCert{Subject: c.Subject.String(), SubjectPKISha256: ssha, CertSha256: csha}\n\t}\n\treturn res\n}\n\n// Assume lock is held\nfunc (c *client) getRTT() time.Duration {\n\tif c.rtt == 0 {\n\t\t// If a real client, go ahead and send ping now to get a value\n\t\t// for RTT. For tests and telnet, or if client is closing, etc skip.\n\t\tif c.opts.Lang != _EMPTY_ {\n\t\t\tc.sendRTTPingLocked()\n\t\t}\n\t\treturn 0\n\t}\n\tvar rtt time.Duration\n\tif c.rtt > time.Microsecond && c.rtt < time.Millisecond {\n\t\trtt = c.rtt.Truncate(time.Microsecond)\n\t} else {\n\t\trtt = c.rtt.Truncate(time.Nanosecond)\n\t}\n\treturn rtt\n}\n\nfunc decodeBool(w http.ResponseWriter, r *http.Request, param string) (bool, error) {\n\tstr := r.URL.Query().Get(param)\n\tif str == _EMPTY_ {\n\t\treturn false, nil\n\t}\n\tval, err := strconv.ParseBool(str)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(fmt.Sprintf(\"Error decoding boolean for '%s': %v\", param, err)))\n\t\treturn false, err\n\t}\n\treturn val, nil\n}\n\nfunc decodeUint64(w http.ResponseWriter, r *http.Request, param string) (uint64, error) {\n\tstr := r.URL.Query().Get(param)\n\tif str == _EMPTY_ {\n\t\treturn 0, nil\n\t}\n\tval, err := strconv.ParseUint(str, 10, 64)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(fmt.Sprintf(\"Error decoding uint64 for '%s': %v\", param, err)))\n\t\treturn 0, err\n\t}\n\treturn val, nil\n}\n\nfunc decodeInt(w http.ResponseWriter, r *http.Request, param string) (int, error) {\n\tstr := r.URL.Query().Get(param)\n\tif str == _EMPTY_ {\n\t\treturn 0, nil\n\t}\n\tval, err := strconv.Atoi(str)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(fmt.Sprintf(\"Error decoding int for '%s': %v\", param, err)))\n\t\treturn 0, err\n\t}\n\treturn val, nil\n}\n\nfunc decodeState(w http.ResponseWriter, r *http.Request) (ConnState, error) {\n\tstr := r.URL.Query().Get(\"state\")\n\tif str == _EMPTY_ {\n\t\treturn ConnOpen, nil\n\t}\n\tswitch strings.ToLower(str) {\n\tcase \"open\":\n\t\treturn ConnOpen, nil\n\tcase \"closed\":\n\t\treturn ConnClosed, nil\n\tcase \"any\", \"all\":\n\t\treturn ConnAll, nil\n\t}\n\t// We do not understand intended state here.\n\tw.WriteHeader(http.StatusBadRequest)\n\terr := fmt.Errorf(\"Error decoding state for %s\", str)\n\tw.Write([]byte(err.Error()))\n\treturn 0, err\n}\n\nfunc decodeSubs(w http.ResponseWriter, r *http.Request) (subs bool, subsDet bool, err error) {\n\tsubsDet = strings.ToLower(r.URL.Query().Get(\"subs\")) == \"detail\"\n\tif !subsDet {\n\t\tsubs, err = decodeBool(w, r, \"subs\")\n\t}\n\treturn\n}\n\n// HandleConnz process HTTP requests for connection information.\nfunc (s *Server) HandleConnz(w http.ResponseWriter, r *http.Request) {\n\tsortOpt := SortOpt(r.URL.Query().Get(\"sort\"))\n\tauth, err := decodeBool(w, r, \"auth\")\n\tif err != nil {\n\t\treturn\n\t}\n\tsubs, subsDet, err := decodeSubs(w, r)\n\tif err != nil {\n\t\treturn\n\t}\n\toffset, err := decodeInt(w, r, \"offset\")\n\tif err != nil {\n\t\treturn\n\t}\n\tlimit, err := decodeInt(w, r, \"limit\")\n\tif err != nil {\n\t\treturn\n\t}\n\tcid, err := decodeUint64(w, r, \"cid\")\n\tif err != nil {\n\t\treturn\n\t}\n\tstate, err := decodeState(w, r)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tuser := r.URL.Query().Get(\"user\")\n\tacc := r.URL.Query().Get(\"acc\")\n\tmqttCID := r.URL.Query().Get(\"mqtt_client\")\n\n\tconnzOpts := &ConnzOptions{\n\t\tSort: sortOpt,\n\t\tUsername: auth,\n\t\tSubscriptions: subs,\n\t\tSubscriptionsDetail: subsDet,\n\t\tOffset: offset,\n\t\tLimit: limit,\n\t\tCID: cid,\n\t\tMQTTClient: mqttCID,\n\t\tState: state,\n\t\tUser: user,\n\t\tAccount: acc,\n\t}\n\n\ts.mu.Lock()\n\ts.httpReqStats[ConnzPath]++\n\ts.mu.Unlock()\n\n\tc, err := s.Connz(connzOpts)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\tb, err := json.MarshalIndent(c, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /connz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// Routez represents detailed information on current client connections.\ntype Routez struct {\n\tID string `json:\"server_id\"`\n\tName string `json:\"server_name\"`\n\tNow time.Time `json:\"now\"`\n\tImport *SubjectPermission `json:\"import,omitempty\"`\n\tExport *SubjectPermission `json:\"export,omitempty\"`\n\tNumRoutes int `json:\"num_routes\"`\n\tRoutes []*RouteInfo `json:\"routes\"`\n}\n\n// RoutezOptions are options passed to Routez\ntype RoutezOptions struct {\n\t// Subscriptions indicates that Routez will return a route's subscriptions\n\tSubscriptions bool `json:\"subscriptions\"`\n\t// SubscriptionsDetail indicates if subscription details should be included in the results\n\tSubscriptionsDetail bool `json:\"subscriptions_detail\"`\n}\n\n// RouteInfo has detailed information on a per connection basis.\ntype RouteInfo struct {\n\tRid uint64 `json:\"rid\"`\n\tRemoteID string `json:\"remote_id\"`\n\tRemoteName string `json:\"remote_name\"`\n\tDidSolicit bool `json:\"did_solicit\"`\n\tIsConfigured bool `json:\"is_configured\"`\n\tIP string `json:\"ip\"`\n\tPort int `json:\"port\"`\n\tStart time.Time `json:\"start\"`\n\tLastActivity time.Time `json:\"last_activity\"`\n\tRTT string `json:\"rtt,omitempty\"`\n\tUptime string `json:\"uptime\"`\n\tIdle string `json:\"idle\"`\n\tImport *SubjectPermission `json:\"import,omitempty\"`\n\tExport *SubjectPermission `json:\"export,omitempty\"`\n\tPending int `json:\"pending_size\"`\n\tInMsgs int64 `json:\"in_msgs\"`\n\tOutMsgs int64 `json:\"out_msgs\"`\n\tInBytes int64 `json:\"in_bytes\"`\n\tOutBytes int64 `json:\"out_bytes\"`\n\tNumSubs uint32 `json:\"subscriptions\"`\n\tSubs []string `json:\"subscriptions_list,omitempty\"`\n\tSubsDetail []SubDetail `json:\"subscriptions_list_detail,omitempty\"`\n\tAccount string `json:\"account,omitempty\"`\n\tCompression string `json:\"compression,omitempty\"`\n}\n\n// Routez returns a Routez struct containing information about routes.\nfunc (s *Server) Routez(routezOpts *RoutezOptions) (*Routez, error) {\n\trs := &Routez{\n\t\tNow: time.Now().UTC(),\n\t\tRoutes: []*RouteInfo{},\n\t}\n\n\tif routezOpts == nil {\n\t\troutezOpts = &RoutezOptions{}\n\t}\n\n\ts.mu.Lock()\n\trs.NumRoutes = s.numRoutes()\n\n\t// copy the server id for monitoring\n\trs.ID = s.info.ID\n\n\t// Check for defined permissions for all connected routes.\n\tif perms := s.getOpts().Cluster.Permissions; perms != nil {\n\t\trs.Import = perms.Import\n\t\trs.Export = perms.Export\n\t}\n\trs.Name = s.info.Name\n\n\taddRoute := func(r *client) {\n\t\tr.mu.Lock()\n\t\tri := &RouteInfo{\n\t\t\tRid: r.cid,\n\t\t\tRemoteID: r.route.remoteID,\n\t\t\tRemoteName: r.route.remoteName,\n\t\t\tDidSolicit: r.route.didSolicit,\n\t\t\tIsConfigured: r.route.routeType == Explicit,\n\t\t\tInMsgs: atomic.LoadInt64(&r.inMsgs),\n\t\t\tOutMsgs: r.outMsgs,\n\t\t\tInBytes: atomic.LoadInt64(&r.inBytes),\n\t\t\tOutBytes: r.outBytes,\n\t\t\tNumSubs: uint32(len(r.subs)),\n\t\t\tImport: r.opts.Import,\n\t\t\tPending: int(r.out.pb),\n\t\t\tExport: r.opts.Export,\n\t\t\tRTT: r.getRTT().String(),\n\t\t\tStart: r.start,\n\t\t\tLastActivity: r.last,\n\t\t\tUptime: myUptime(rs.Now.Sub(r.start)),\n\t\t\tIdle: myUptime(rs.Now.Sub(r.last)),\n\t\t\tAccount: string(r.route.accName),\n\t\t\tCompression: r.route.compression,\n\t\t}\n\n\t\tif len(r.subs) > 0 {\n\t\t\tif routezOpts.SubscriptionsDetail {\n\t\t\t\tri.SubsDetail = newSubsDetailList(r)\n\t\t\t} else if routezOpts.Subscriptions {\n\t\t\t\tri.Subs = newSubsList(r)\n\t\t\t}\n\t\t}\n\n\t\tswitch conn := r.nc.(type) {\n\t\tcase *net.TCPConn, *tls.Conn:\n\t\t\taddr := conn.RemoteAddr().(*net.TCPAddr)\n\t\t\tri.Port = addr.Port\n\t\t\tri.IP = addr.IP.String()\n\t\t}\n\t\tr.mu.Unlock()\n\t\trs.Routes = append(rs.Routes, ri)\n\t}\n\n\t// Walk the list\n\ts.forEachRoute(func(r *client) {\n\t\taddRoute(r)\n\t})\n\ts.mu.Unlock()\n\treturn rs, nil\n}\n\n// HandleRoutez process HTTP requests for route information.\nfunc (s *Server) HandleRoutez(w http.ResponseWriter, r *http.Request) {\n\tsubs, subsDetail, err := decodeSubs(w, r)\n\tif err != nil {\n\t\treturn\n\t}\n\n\topts := RoutezOptions{Subscriptions: subs, SubscriptionsDetail: subsDetail}\n\n\ts.mu.Lock()\n\ts.httpReqStats[RoutezPath]++\n\ts.mu.Unlock()\n\n\t// As of now, no error is ever returned.\n\trs, _ := s.Routez(&opts)\n\tb, err := json.MarshalIndent(rs, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /routez request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// Subsz represents detail information on current connections.\ntype Subsz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\t*SublistStats\n\tTotal int `json:\"total\"`\n\tOffset int `json:\"offset\"`\n\tLimit int `json:\"limit\"`\n\tSubs []SubDetail `json:\"subscriptions_list,omitempty\"`\n}\n\n// SubszOptions are the options passed to Subsz.\n// As of now, there are no options defined.\ntype SubszOptions struct {\n\t// Offset is used for pagination. Subsz() only returns connections starting at this\n\t// offset from the global results.\n\tOffset int `json:\"offset\"`\n\n\t// Limit is the maximum number of subscriptions that should be returned by Subsz().\n\tLimit int `json:\"limit\"`\n\n\t// Subscriptions indicates if subscription details should be included in the results.\n\tSubscriptions bool `json:\"subscriptions\"`\n\n\t// Filter based on this account name.\n\tAccount string `json:\"account,omitempty\"`\n\n\t// Test the list against this subject. Needs to be literal since it signifies a publish subject.\n\t// We will only return subscriptions that would match if a message was sent to this subject.\n\tTest string `json:\"test,omitempty\"`\n}\n\n// SubDetail is for verbose information for subscriptions.\ntype SubDetail struct {\n\tAccount string `json:\"account,omitempty\"`\n\tAccountTag string `json:\"account_tag,omitempty\"`\n\tSubject string `json:\"subject\"`\n\tQueue string `json:\"qgroup,omitempty\"`\n\tSid string `json:\"sid\"`\n\tMsgs int64 `json:\"msgs\"`\n\tMax int64 `json:\"max,omitempty\"`\n\tCid uint64 `json:\"cid\"`\n}\n\n// Subscription client should be locked and guaranteed to be present.\nfunc newSubDetail(sub *subscription) SubDetail {\n\tsd := newClientSubDetail(sub)\n\tsd.Account = sub.client.acc.GetName()\n\tsd.AccountTag = sub.client.acc.getNameTag()\n\treturn sd\n}\n\n// For subs details under clients.\nfunc newClientSubDetail(sub *subscription) SubDetail {\n\treturn SubDetail{\n\t\tSubject: string(sub.subject),\n\t\tQueue: string(sub.queue),\n\t\tSid: string(sub.sid),\n\t\tMsgs: sub.nm,\n\t\tMax: sub.max,\n\t\tCid: sub.client.cid,\n\t}\n}\n\n// Subsz returns a Subsz struct containing subjects statistics\nfunc (s *Server) Subsz(opts *SubszOptions) (*Subsz, error) {\n\tvar (\n\t\tsubdetail bool\n\t\ttest bool\n\t\toffset int\n\t\ttestSub string\n\t\tfilterAcc string\n\t\tlimit = DefaultSubListSize\n\t)\n\n\tif opts != nil {\n\t\tsubdetail = opts.Subscriptions\n\t\toffset = opts.Offset\n\t\tif offset < 0 {\n\t\t\toffset = 0\n\t\t}\n\t\tlimit = opts.Limit\n\t\tif limit <= 0 {\n\t\t\tlimit = DefaultSubListSize\n\t\t}\n\t\tif opts.Test != _EMPTY_ {\n\t\t\ttestSub = opts.Test\n\t\t\ttest = true\n\t\t\tif !IsValidLiteralSubject(testSub) {\n\t\t\t\treturn nil, fmt.Errorf(\"invalid test subject, must be valid publish subject: %s\", testSub)\n\t\t\t}\n\t\t}\n\t\tif opts.Account != _EMPTY_ {\n\t\t\tfilterAcc = opts.Account\n\t\t}\n\t}\n\n\tslStats := &SublistStats{}\n\n\t// FIXME(dlc) - Make account aware.\n\tsz := &Subsz{\n\t\tID: s.info.ID,\n\t\tNow: time.Now().UTC(),\n\t\tSublistStats: slStats,\n\t\tTotal: 0,\n\t\tOffset: offset,\n\t\tLimit: limit,\n\t\tSubs: nil,\n\t}\n\n\tif subdetail {\n\t\tvar raw [4096]*subscription\n\t\tsubs := raw[:0]\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\tif filterAcc != _EMPTY_ && acc.GetName() != filterAcc {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tslStats.add(acc.sl.Stats())\n\t\t\tacc.sl.localSubs(&subs, false)\n\t\t\treturn true\n\t\t})\n\n\t\tdetails := make([]SubDetail, 0, len(subs))\n\t\ti := 0\n\t\t// TODO(dlc) - may be inefficient and could just do normal match when total subs is large and filtering.\n\t\tfor _, sub := range subs {\n\t\t\t// Check for filter\n\t\t\tif test && !matchLiteral(testSub, string(sub.subject)) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif sub.client == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsub.client.mu.Lock()\n\t\t\tdetails = append(details, newSubDetail(sub))\n\t\t\tsub.client.mu.Unlock()\n\t\t\ti++\n\t\t}\n\t\tminoff := sz.Offset\n\t\tmaxoff := sz.Offset + sz.Limit\n\n\t\tmaxIndex := i\n\n\t\t// Make sure these are sane.\n\t\tif minoff > maxIndex {\n\t\t\tminoff = maxIndex\n\t\t}\n\t\tif maxoff > maxIndex {\n\t\t\tmaxoff = maxIndex\n\t\t}\n\t\tsz.Subs = details[minoff:maxoff]\n\t\tsz.Total = len(details)\n\t} else {\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\tif filterAcc != _EMPTY_ && acc.GetName() != filterAcc {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tslStats.add(acc.sl.Stats())\n\t\t\treturn true\n\t\t})\n\t}\n\n\treturn sz, nil\n}\n\n// HandleSubsz processes HTTP requests for subjects stats.\nfunc (s *Server) HandleSubsz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[SubszPath]++\n\ts.mu.Unlock()\n\n\tsubs, err := decodeBool(w, r, \"subs\")\n\tif err != nil {\n\t\treturn\n\t}\n\toffset, err := decodeInt(w, r, \"offset\")\n\tif err != nil {\n\t\treturn\n\t}\n\tlimit, err := decodeInt(w, r, \"limit\")\n\tif err != nil {\n\t\treturn\n\t}\n\ttestSub := r.URL.Query().Get(\"test\")\n\t// Filtered account.\n\tfilterAcc := r.URL.Query().Get(\"acc\")\n\n\tsubszOpts := &SubszOptions{\n\t\tSubscriptions: subs,\n\t\tOffset: offset,\n\t\tLimit: limit,\n\t\tAccount: filterAcc,\n\t\tTest: testSub,\n\t}\n\n\tst, err := s.Subsz(subszOpts)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\n\tvar b []byte\n\tb, err = json.MarshalIndent(st, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /subscriptionsz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// HandleStacksz processes HTTP requests for getting stacks\nfunc (s *Server) HandleStacksz(w http.ResponseWriter, r *http.Request) {\n\t// Do not get any lock here that would prevent getting the stacks\n\t// if we were to have a deadlock somewhere.\n\tvar defaultBuf [defaultStackBufSize]byte\n\tsize := defaultStackBufSize\n\tbuf := defaultBuf[:size]\n\tn := 0\n\tfor {\n\t\tn = runtime.Stack(buf, true)\n\t\tif n < size {\n\t\t\tbreak\n\t\t}\n\t\tsize *= 2\n\t\tbuf = make([]byte, size)\n\t}\n\t// Handle response\n\tResponseHandler(w, r, buf[:n])\n}\n\ntype IpqueueszStatusIPQ struct {\n\tPending int `json:\"pending\"`\n\tInProgress int `json:\"in_progress,omitempty\"`\n}\n\ntype IpqueueszStatus map[string]IpqueueszStatusIPQ\n\nfunc (s *Server) Ipqueuesz(opts *IpqueueszOptions) *IpqueueszStatus {\n\tall, qfilter := opts.All, opts.Filter\n\tqueues := IpqueueszStatus{}\n\ts.ipQueues.Range(func(k, v any) bool {\n\t\tvar pending, inProgress int\n\t\tname := k.(string)\n\t\tqueue, ok := v.(interface {\n\t\t\tlen() int\n\t\t\tinProgress() int64\n\t\t})\n\t\tif ok {\n\t\t\tpending = queue.len()\n\t\t\tinProgress = int(queue.inProgress())\n\t\t}\n\t\tif !all && (pending == 0 && inProgress == 0) {\n\t\t\treturn true\n\t\t} else if qfilter != _EMPTY_ && !strings.Contains(name, qfilter) {\n\t\t\treturn true\n\t\t}\n\t\tqueues[name] = IpqueueszStatusIPQ{Pending: pending, InProgress: inProgress}\n\t\treturn true\n\t})\n\treturn &queues\n}\n\nfunc (s *Server) HandleIPQueuesz(w http.ResponseWriter, r *http.Request) {\n\tall, err := decodeBool(w, r, \"all\")\n\tif err != nil {\n\t\treturn\n\t}\n\tqfilter := r.URL.Query().Get(\"queues\")\n\n\tqueues := s.Ipqueuesz(&IpqueueszOptions{\n\t\tAll: all,\n\t\tFilter: qfilter,\n\t})\n\n\tb, _ := json.MarshalIndent(queues, \"\", \" \")\n\tResponseHandler(w, r, b)\n}\n\n// Varz will output server information on the monitoring port at /varz.\ntype Varz struct {\n\tID string `json:\"server_id\"` // ID is the unique server ID generated at start\n\tName string `json:\"server_name\"` // Name is the configured server name, equals ID when not set\n\tVersion string `json:\"version\"` // Version is the version of the running server\n\tProto int `json:\"proto\"` // Proto is the protocol version this server supports\n\tGitCommit string `json:\"git_commit,omitempty\"` // GitCommit is the git repository commit hash that the build corresponds with\n\tGoVersion string `json:\"go\"` // GoVersion is the version of Go used to build this binary\n\tHost string `json:\"host\"` // Host is the hostname the server runs on\n\tPort int `json:\"port\"` // Port is the port the server listens on for client connections\n\tAuthRequired bool `json:\"auth_required,omitempty\"` // AuthRequired indicates if users are required to authenticate to join the server\n\tTLSRequired bool `json:\"tls_required,omitempty\"` // TLSRequired indicates if connections must use TLS when connecting to this server\n\tTLSVerify bool `json:\"tls_verify,omitempty\"` // TLSVerify indicates if full TLS verification will be performed\n\tTLSOCSPPeerVerify bool `json:\"tls_ocsp_peer_verify,omitempty\"` // TLSOCSPPeerVerify indicates if the OCSP protocol will be used to verify peers\n\tIP string `json:\"ip,omitempty\"` // IP is the IP address the server listens on if set\n\tClientConnectURLs []string `json:\"connect_urls,omitempty\"` // ClientConnectURLs is the list of URLs NATS clients can use to connect to this server\n\tWSConnectURLs []string `json:\"ws_connect_urls,omitempty\"` // WSConnectURLs is the list of URLs websocket clients can use to connect to this server\n\tMaxConn int `json:\"max_connections\"` // MaxConn is the maximum amount of connections the server can accept\n\tMaxSubs int `json:\"max_subscriptions,omitempty\"` // MaxSubs is the maximum amount of subscriptions the server can manage\n\tPingInterval time.Duration `json:\"ping_interval\"` // PingInterval is the interval the server will send PING messages during periods of inactivity on a connection\n\tMaxPingsOut int `json:\"ping_max\"` // MaxPingsOut is the number of unanswered PINGs after which the connection will be considered stale\n\tHTTPHost string `json:\"http_host\"` // HTTPHost is the HTTP host monitoring connections are accepted on\n\tHTTPPort int `json:\"http_port\"` // HTTPPort is the port monitoring connections are accepted on\n\tHTTPBasePath string `json:\"http_base_path\"` // HTTPBasePath is the path prefix for access to monitor endpoints\n\tHTTPSPort int `json:\"https_port\"` // HTTPSPort is the HTTPS host monitoring connections are accepted on`\n\tAuthTimeout float64 `json:\"auth_timeout\"` // AuthTimeout is the amount of seconds connections have to complete authentication\n\tMaxControlLine int32 `json:\"max_control_line\"` // MaxControlLine is the amount of bytes a signal control message may be\n\tMaxPayload int `json:\"max_payload\"` // MaxPayload is the maximum amount of bytes a message may have as payload\n\tMaxPending int64 `json:\"max_pending\"` // MaxPending is the maximum amount of unprocessed bytes a connection may have\n\tCluster ClusterOptsVarz `json:\"cluster,omitempty\"` // Cluster is the Cluster state\n\tGateway GatewayOptsVarz `json:\"gateway,omitempty\"` // Gateway is the Super Cluster state\n\tLeafNode LeafNodeOptsVarz `json:\"leaf,omitempty\"` // LeafNode is the Leafnode state\n\tMQTT MQTTOptsVarz `json:\"mqtt,omitempty\"` // MQTT is the MQTT state\n\tWebsocket WebsocketOptsVarz `json:\"websocket,omitempty\"` // Websocket is the Websocket client state\n\tJetStream JetStreamVarz `json:\"jetstream,omitempty\"` // JetStream is the JetStream state\n\tTLSTimeout float64 `json:\"tls_timeout\"` // TLSTimeout is how long TLS operations have to complete\n\tWriteDeadline time.Duration `json:\"write_deadline\"` // WriteDeadline is the maximum time writes to sockets have to complete\n\tWriteTimeout string `json:\"write_timeout,omitempty\"` // WriteTimeout is the closure policy for write deadline errors\n\tStart time.Time `json:\"start\"` // Start is time when the server was started\n\tNow time.Time `json:\"now\"` // Now is the current time of the server\n\tUptime string `json:\"uptime\"` // Uptime is how long the server has been running\n\tMem int64 `json:\"mem\"` // Mem is the resident memory allocation\n\tCores int `json:\"cores\"` // Cores is the number of cores the process has access to\n\tMaxProcs int `json:\"gomaxprocs\"` // MaxProcs is the configured GOMAXPROCS value\n\tMemLimit int64 `json:\"gomemlimit,omitempty\"` // MemLimit is the configured GOMEMLIMIT value\n\tCPU float64 `json:\"cpu\"` // CPU is the current total CPU usage\n\tConnections int `json:\"connections\"` // Connections is the current connected connections\n\tTotalConnections uint64 `json:\"total_connections\"` // TotalConnections is the total connections the server have ever handled\n\tRoutes int `json:\"routes\"` // Routes is the number of connected route servers\n\tRemotes int `json:\"remotes\"` // Remotes is the configured route remote endpoints\n\tLeafs int `json:\"leafnodes\"` // Leafs is the number connected leafnode clients\n\tInMsgs int64 `json:\"in_msgs\"` // InMsgs is the number of messages this server received\n\tOutMsgs int64 `json:\"out_msgs\"` // OutMsgs is the number of message this server sent\n\tInBytes int64 `json:\"in_bytes\"` // InBytes is the number of bytes this server received\n\tOutBytes int64 `json:\"out_bytes\"` // OutMsgs is the number of bytes this server sent\n\tSlowConsumers int64 `json:\"slow_consumers\"` // SlowConsumers is the total count of clients that were disconnected since start due to being slow consumers\n\tStaleConnections int64 `json:\"stale_connections\"` // StaleConnections is the total count of stale connections that were detected\n\tStalledClients int64 `json:\"stalled_clients\"` // StalledClients is the total number of times that clients have been stalled.\n\tSubscriptions uint32 `json:\"subscriptions\"` // Subscriptions is the count of active subscriptions\n\tHTTPReqStats map[string]uint64 `json:\"http_req_stats\"` // HTTPReqStats is the number of requests each HTTP endpoint received\n\tConfigLoadTime time.Time `json:\"config_load_time\"` // ConfigLoadTime is the time the configuration was loaded or reloaded\n\tConfigDigest string `json:\"config_digest\"` // ConfigDigest is a calculated hash of the current configuration\n\tTags jwt.TagList `json:\"tags,omitempty\"` // Tags are the tags assigned to the server in configuration\n\tMetadata map[string]string `json:\"metadata,omitempty\"` // Metadata is the metadata assigned to the server in configuration\n\tTrustedOperatorsJwt []string `json:\"trusted_operators_jwt,omitempty\"` // TrustedOperatorsJwt is the JWTs for all trusted operators\n\tTrustedOperatorsClaim []*jwt.OperatorClaims `json:\"trusted_operators_claim,omitempty\"` // TrustedOperatorsClaim is the decoded claims for each trusted operator\n\tSystemAccount string `json:\"system_account,omitempty\"` // SystemAccount is the name of the System account\n\tPinnedAccountFail uint64 `json:\"pinned_account_fails,omitempty\"` // PinnedAccountFail is how often user logon fails due to the issuer account not being pinned.\n\tOCSPResponseCache *OCSPResponseCacheVarz `json:\"ocsp_peer_cache,omitempty\"` // OCSPResponseCache is the state of the OCSP cache\n\tSlowConsumersStats *SlowConsumersStats `json:\"slow_consumer_stats\"` // SlowConsumersStats are statistics about all detected Slow Consumer\n\tStaleConnectionStats *StaleConnectionStats `json:\"stale_connection_stats,omitempty\"` // StaleConnectionStats are statistics about all detected Stale Connections\n\tProxies *ProxiesOptsVarz `json:\"proxies,omitempty\"` // Proxies hold information about network proxy devices\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificate of this server\n}\n\n// JetStreamVarz contains basic runtime information about jetstream\ntype JetStreamVarz struct {\n\tConfig *JetStreamConfig `json:\"config,omitempty\"` // Config is the active JetStream configuration\n\tStats *JetStreamStats `json:\"stats,omitempty\"` // Stats is the statistics for the JetStream server\n\tMeta *MetaClusterInfo `json:\"meta,omitempty\"` // Meta is information about the JetStream metalayer\n\tLimits *JSLimitOpts `json:\"limits,omitempty\"` // Limits are the configured JetStream limits\n}\n\n// ClusterOptsVarz contains monitoring cluster information\ntype ClusterOptsVarz struct {\n\tName string `json:\"name,omitempty\"` // Name is the configured cluster name\n\tHost string `json:\"addr,omitempty\"` // Host is the host the cluster listens on for connections\n\tPort int `json:\"cluster_port,omitempty\"` // Port is the port the cluster listens on for connections\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"` // AuthTimeout is the time cluster connections have to complete authentication\n\tURLs []string `json:\"urls,omitempty\"` // URLs is the list of cluster URLs\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tTLSRequired bool `json:\"tls_required,omitempty\"` // TLSRequired indicates if TLS is required for connections\n\tTLSVerify bool `json:\"tls_verify,omitempty\"` // TLSVerify indicates if full verification of TLS connections is performed\n\tPoolSize int `json:\"pool_size,omitempty\"` // PoolSize is the configured route connection pool size\n\tWriteDeadline time.Duration `json:\"write_deadline,omitempty\"` // WriteDeadline is the maximum time writes to sockets have to complete\n\tWriteTimeout string `json:\"write_timeout,omitempty\"` // WriteTimeout is the closure policy for write deadline errors\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificate\n}\n\n// GatewayOptsVarz contains monitoring gateway information\ntype GatewayOptsVarz struct {\n\tName string `json:\"name,omitempty\"` // Name is the configured cluster name\n\tHost string `json:\"host,omitempty\"` // Host is the host the gateway listens on for connections\n\tPort int `json:\"port,omitempty\"` // Port is the post gateway connections listens on\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"` // AuthTimeout is the time cluster connections have to complete authentication\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tTLSRequired bool `json:\"tls_required,omitempty\"` // TLSRequired indicates if TLS is required for connections\n\tTLSVerify bool `json:\"tls_verify,omitempty\"` // TLSVerify indicates if full verification of TLS connections is performed\n\tAdvertise string `json:\"advertise,omitempty\"` // Advertise is the URL advertised to remote gateway clients\n\tConnectRetries int `json:\"connect_retries,omitempty\"` // ConnectRetries is how many connection attempts the route will make\n\tGateways []RemoteGatewayOptsVarz `json:\"gateways,omitempty\"` // Gateways is state of configured gateway remotes\n\tRejectUnknown bool `json:\"reject_unknown,omitempty\"` // RejectUnknown indicates if unknown cluster connections will be rejected\n\tWriteDeadline time.Duration `json:\"write_deadline,omitempty\"` // WriteDeadline is the maximum time writes to sockets have to complete\n\tWriteTimeout string `json:\"write_timeout,omitempty\"` // WriteTimeout is the closure policy for write deadline errors\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificaet\n}\n\n// RemoteGatewayOptsVarz contains monitoring remote gateway information\ntype RemoteGatewayOptsVarz struct {\n\tName string `json:\"name\"` // Name is the name of the remote gateway\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tURLs []string `json:\"urls,omitempty\"` // URLs is the list of Gateway URLs\n}\n\n// LeafNodeOptsVarz contains monitoring leaf node information\ntype LeafNodeOptsVarz struct {\n\tHost string `json:\"host,omitempty\"` // Host is the host the server listens on\n\tPort int `json:\"port,omitempty\"` // Port is the port the server listens on\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"` // AuthTimeout is the time Leafnode connections have to complete authentication\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tTLSRequired bool `json:\"tls_required,omitempty\"` // TLSRequired indicates if TLS is required for connections\n\tTLSVerify bool `json:\"tls_verify,omitempty\"` // TLSVerify indicates if full verification of TLS connections is performed\n\tRemotes []RemoteLeafOptsVarz `json:\"remotes,omitempty\"` // Remotes is state of configured Leafnode remotes\n\tTLSOCSPPeerVerify bool `json:\"tls_ocsp_peer_verify,omitempty\"` // TLSOCSPPeerVerify indicates if OCSP verification will be performed\n\tWriteDeadline time.Duration `json:\"write_deadline,omitempty\"` // WriteDeadline is the maximum time writes to sockets have to complete\n\tWriteTimeout string `json:\"write_timeout,omitempty\"` // WriteTimeout is the closure policy for write deadline errors\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificate\n}\n\n// DenyRules Contains lists of subjects not allowed to be imported/exported\ntype DenyRules struct {\n\tExports []string `json:\"exports,omitempty\"` // Exports are denied exports\n\tImports []string `json:\"imports,omitempty\"` // Imports are denied imports\n}\n\n// RemoteLeafOptsVarz contains monitoring remote leaf node information\ntype RemoteLeafOptsVarz struct {\n\tLocalAccount string `json:\"local_account,omitempty\"` // LocalAccount is the local account this leaf is logged into\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tURLs []string `json:\"urls,omitempty\"` // URLs is the list of URLs for the remote Leafnode connection\n\tDeny *DenyRules `json:\"deny,omitempty\"` // Deny is the configured import and exports that the Leafnode may not access\n\tTLSOCSPPeerVerify bool `json:\"tls_ocsp_peer_verify,omitempty\"` // TLSOCSPPeerVerify indicates if OCSP verification will be done\n}\n\n// MQTTOptsVarz contains monitoring MQTT information\ntype MQTTOptsVarz struct {\n\tHost string `json:\"host,omitempty\"` // Host is the host the server listens on\n\tPort int `json:\"port,omitempty\"` // Port is the port the server listens on\n\tNoAuthUser string `json:\"no_auth_user,omitempty\"` // NoAuthUser is the user that will be used for unauthenticated connections\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"` // AuthTimeout is how long authentication has to complete\n\tTLSMap bool `json:\"tls_map,omitempty\"` // TLSMap indicates if TLS Mapping is enabled\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"` // TLSTimeout is how long TLS operations have to complete\n\tTLSPinnedCerts []string `json:\"tls_pinned_certs,omitempty\"` // TLSPinnedCerts is the list of certificates pinned to this connection\n\tJsDomain string `json:\"js_domain,omitempty\"` // JsDomain is the JetStream domain used for MQTT state\n\tAckWait time.Duration `json:\"ack_wait,omitempty\"` // AckWait is how long the internal JetStream state store will allow acks to complete\n\tMaxAckPending uint16 `json:\"max_ack_pending,omitempty\"` // MaxAckPending is how many outstanding acks the internal JetStream state store will allow\n\tTLSOCSPPeerVerify bool `json:\"tls_ocsp_peer_verify,omitempty\"` // TLSOCSPPeerVerify indicates if OCSP verification will be done\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificate\n}\n\n// WebsocketOptsVarz contains monitoring websocket information\ntype WebsocketOptsVarz struct {\n\tHost string `json:\"host,omitempty\"` // Host is the host the server listens on\n\tPort int `json:\"port,omitempty\"` // Port is the port the server listens on\n\tAdvertise string `json:\"advertise,omitempty\"` // Advertise is the connection URL the server advertises\n\tNoAuthUser string `json:\"no_auth_user,omitempty\"` // NoAuthUser is the user that will be used for unauthenticated connections\n\tJWTCookie string `json:\"jwt_cookie,omitempty\"` // JWTCookie is the name of a cookie the server will read for the connection JWT\n\tHandshakeTimeout time.Duration `json:\"handshake_timeout,omitempty\"` // HandshakeTimeout is how long the connection has to complete the websocket setup\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"` // AuthTimeout is how long authentication has to complete\n\tNoTLS bool `json:\"no_tls,omitempty\"` // NoTLS indicates if TLS is disabled\n\tTLSMap bool `json:\"tls_map,omitempty\"` // TLSMap indicates if TLS Mapping is enabled\n\tTLSPinnedCerts []string `json:\"tls_pinned_certs,omitempty\"` // TLSPinnedCerts is the list of certificates pinned to this connection\n\tSameOrigin bool `json:\"same_origin,omitempty\"` // SameOrigin indicates if same origin connections are allowed\n\tAllowedOrigins []string `json:\"allowed_origins,omitempty\"` // AllowedOrigins list of configured trusted origins\n\tCompression bool `json:\"compression,omitempty\"` // Compression indicates if compression is supported\n\tTLSOCSPPeerVerify bool `json:\"tls_ocsp_peer_verify,omitempty\"` // TLSOCSPPeerVerify indicates if OCSP verification will be done\n\tTLSCertNotAfter time.Time `json:\"tls_cert_not_after,omitzero\"` // TLSCertNotAfter is the expiration date of the TLS certificate\n}\n\n// OCSPResponseCacheVarz contains OCSP response cache information\ntype OCSPResponseCacheVarz struct {\n\tType string `json:\"cache_type,omitempty\"` // Type is the kind of cache being used\n\tHits int64 `json:\"cache_hits,omitempty\"` // Hits is how many times the cache was able to answer a request\n\tMisses int64 `json:\"cache_misses,omitempty\"` // Misses is how many times the cache failed to answer a request\n\tResponses int64 `json:\"cached_responses,omitempty\"` // Responses is how many responses are currently stored in the cache\n\tRevokes int64 `json:\"cached_revoked_responses,omitempty\"` // Revokes is how many of the stored cache entries are revokes\n\tGoods int64 `json:\"cached_good_responses,omitempty\"` // Goods is how many of the stored cache entries are good responses\n\tUnknowns int64 `json:\"cached_unknown_responses,omitempty\"` // Unknowns is how many of the stored cache entries are unknown responses\n}\n\n// ProxiesOptsVarz contains proxies information\ntype ProxiesOptsVarz struct {\n\tTrusted []*ProxyOptsVarz `json:\"trusted,omitempty\"` // Trusted holds a list of trusted proxies\n}\n\n// ProxyOptsVarz contains proxy information\ntype ProxyOptsVarz struct {\n\tKey string `json:\"key\"` // Key is the public key of the proxy\n}\n\n// VarzOptions are the options passed to Varz().\n// Currently, there are no options defined.\ntype VarzOptions struct{}\n\n// SlowConsumersStats contains information about the slow consumers from different type of connections.\ntype SlowConsumersStats struct {\n\tClients uint64 `json:\"clients\"` // Clients is how many Clients were slow consumers\n\tRoutes uint64 `json:\"routes\"` // Routes is how many Routes were slow consumers\n\tGateways uint64 `json:\"gateways\"` // Gateways is how many Gateways were slow consumers\n\tLeafs uint64 `json:\"leafs\"` // Leafs is how many Leafnodes were slow consumers\n}\n\n// StaleConnectionStats contains information about the stale connections from different type of connections.\ntype StaleConnectionStats struct {\n\tClients uint64 `json:\"clients\"` // Clients is how many Client connections became stale connections\n\tRoutes uint64 `json:\"routes\"` // Routes is how many Route connections became stale connections\n\tGateways uint64 `json:\"gateways\"` // Gateways is how many Gateway connections became stale connections\n\tLeafs uint64 `json:\"leafs\"` // Leafs is how many Leafnode connections became stale connections\n}\n\nfunc myUptime(d time.Duration) string {\n\t// Just use total seconds for uptime, and display days / years\n\ttsecs := d / time.Second\n\ttmins := tsecs / 60\n\tthrs := tmins / 60\n\ttdays := thrs / 24\n\ttyrs := tdays / 365\n\n\tif tyrs > 0 {\n\t\treturn fmt.Sprintf(\"%dy%dd%dh%dm%ds\", tyrs, tdays%365, thrs%24, tmins%60, tsecs%60)\n\t}\n\tif tdays > 0 {\n\t\treturn fmt.Sprintf(\"%dd%dh%dm%ds\", tdays, thrs%24, tmins%60, tsecs%60)\n\t}\n\tif thrs > 0 {\n\t\treturn fmt.Sprintf(\"%dh%dm%ds\", thrs, tmins%60, tsecs%60)\n\t}\n\tif tmins > 0 {\n\t\treturn fmt.Sprintf(\"%dm%ds\", tmins, tsecs%60)\n\t}\n\treturn fmt.Sprintf(\"%ds\", tsecs)\n}\n\nfunc tlsCertNotAfter(config *tls.Config) time.Time {\n\tif config == nil || len(config.Certificates) == 0 {\n\t\treturn time.Time{}\n\t}\n\tcert := config.Certificates[0]\n\tleaf := cert.Leaf\n\tif leaf == nil {\n\t\tvar err error\n\t\tleaf, err = x509.ParseCertificate(cert.Certificate[0])\n\t\tif err != nil {\n\t\t\treturn time.Time{}\n\t\t}\n\t}\n\treturn leaf.NotAfter\n}\n\n// HandleRoot will show basic info and links to others handlers.\nfunc (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {\n\t// This feels dumb to me, but is required: https://code.google.com/p/go/issues/detail?id=4799\n\tif r.URL.Path != s.httpBasePath {\n\t\thttp.NotFound(w, r)\n\t\treturn\n\t}\n\ts.mu.Lock()\n\ts.httpReqStats[RootPath]++\n\ts.mu.Unlock()\n\n\t// Calculate source url. If git set go directly to that tag, otherwise just main.\n\tvar srcUrl string\n\tif gitCommit == _EMPTY_ {\n\t\tsrcUrl = \"https://github.com/nats-io/nats-server\"\n\t} else if serverVersion != _EMPTY_ {\n\t\tsrcUrl = fmt.Sprintf(\"https://github.com/nats-io/nats-server/tree/%s\", serverVersion)\n\t} else {\n\t\tsrcUrl = fmt.Sprintf(\"https://github.com/nats-io/nats-server/tree/%s\", gitCommit)\n\t}\n\n\tfmt.Fprintf(w, `\n\t\n\t\n\t\n\t\n\t\n \n\n\tv%s\n\n\t\n\t
\n\tGeneral %s\n\tJetStream %s\n\tConnections %s\n\tAccounts %s\n\tAccount Stats %s\n\tSubscriptions %s\n\tRoutes %s\n\tLeafNodes %s\n\tGateways %s\n\tRaft Groups %s\n\tHealth Probe %s\n\tExpvar %s\n Help\n \n`,\n\t\tsrcUrl,\n\t\tVERSION,\n\t\ts.basePath(VarzPath), VarzPath,\n\t\ts.basePath(JszPath), JszPath,\n\t\ts.basePath(ConnzPath), ConnzPath,\n\t\ts.basePath(AccountzPath), AccountzPath,\n\t\ts.basePath(AccountStatzPath), AccountStatzPath,\n\t\ts.basePath(SubszPath), SubszPath,\n\t\ts.basePath(RoutezPath), RoutezPath,\n\t\ts.basePath(LeafzPath), LeafzPath,\n\t\ts.basePath(GatewayzPath), GatewayzPath,\n\t\ts.basePath(RaftzPath), RaftzPath,\n\t\ts.basePath(HealthzPath), HealthzPath,\n\t\ts.basePath(ExpvarzPath), ExpvarzPath,\n\t)\n}\n\nfunc (s *Server) updateJszVarz(js *jetStream, v *JetStreamVarz, doConfig bool) {\n\tif doConfig {\n\t\tjs.mu.RLock()\n\t\t// We want to snapshot the config since it will then be available outside\n\t\t// of the js lock. So make a copy first, then point to this copy.\n\t\tcfg := js.config\n\t\tv.Config = &cfg\n\t\tjs.mu.RUnlock()\n\t}\n\tv.Stats = js.usageStats()\n\tv.Limits = &s.getOpts().JetStreamLimits\n\tif mg := js.getMetaGroup(); mg != nil {\n\t\tif ci := s.raftNodeToClusterInfo(mg); ci != nil {\n\t\t\tv.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Size: mg.ClusterSize()}\n\t\t\tif ci.Leader == s.info.Name {\n\t\t\t\tv.Meta.Replicas = ci.Replicas\n\t\t\t}\n\t\t\tif ipq := s.jsAPIRoutedReqs; ipq != nil {\n\t\t\t\tv.Meta.PendingRequests = ipq.len()\n\t\t\t}\n\t\t\tif ipq := s.jsAPIRoutedInfoReqs; ipq != nil {\n\t\t\t\tv.Meta.PendingInfos = ipq.len()\n\t\t\t}\n\t\t\tv.Meta.Pending = v.Meta.PendingRequests + v.Meta.PendingInfos\n\t\t\tv.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg)\n\t\t}\n\t}\n}\n\n// Varz returns a Varz struct containing the server information.\nfunc (s *Server) Varz(varzOpts *VarzOptions) (*Varz, error) {\n\tvar rss, vss int64\n\tvar pcpu float64\n\n\t// We want to do that outside of the lock.\n\tpse.ProcUsage(&pcpu, &rss, &vss)\n\n\ts.mu.RLock()\n\t// We need to create a new instance of Varz (with no reference\n\t// whatsoever to anything stored in the server) since the user\n\t// has access to the returned value.\n\tv := s.createVarz(pcpu, rss)\n\ts.mu.RUnlock()\n\n\tif js := s.getJetStream(); js != nil {\n\t\ts.updateJszVarz(js, &v.JetStream, true)\n\t}\n\n\treturn v, nil\n}\n\n// Returns a Varz instance.\n// Server lock is held on entry.\nfunc (s *Server) createVarz(pcpu float64, rss int64) *Varz {\n\tinfo := s.info\n\topts := s.getOpts()\n\tc := &opts.Cluster\n\tgw := &opts.Gateway\n\tln := &opts.LeafNode\n\tmqtt := &opts.MQTT\n\tws := &opts.Websocket\n\tclustTlsReq := c.TLSConfig != nil\n\tgatewayTlsReq := gw.TLSConfig != nil\n\tleafTlsReq := ln.TLSConfig != nil\n\tleafTlsVerify := leafTlsReq && ln.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert\n\tleafTlsOCSPPeerVerify := s.ocspPeerVerify && leafTlsReq && ln.tlsConfigOpts.OCSPPeerConfig != nil && ln.tlsConfigOpts.OCSPPeerConfig.Verify\n\tmqttTlsOCSPPeerVerify := s.ocspPeerVerify && mqtt.TLSConfig != nil && mqtt.tlsConfigOpts.OCSPPeerConfig != nil && mqtt.tlsConfigOpts.OCSPPeerConfig.Verify\n\twsTlsOCSPPeerVerify := s.ocspPeerVerify && ws.TLSConfig != nil && ws.tlsConfigOpts.OCSPPeerConfig != nil && ws.tlsConfigOpts.OCSPPeerConfig.Verify\n\tvarz := &Varz{\n\t\tID: info.ID,\n\t\tVersion: info.Version,\n\t\tProto: info.Proto,\n\t\tGitCommit: info.GitCommit,\n\t\tGoVersion: info.GoVersion,\n\t\tName: info.Name,\n\t\tHost: info.Host,\n\t\tPort: info.Port,\n\t\tIP: info.IP,\n\t\tHTTPHost: opts.HTTPHost,\n\t\tHTTPPort: opts.HTTPPort,\n\t\tHTTPBasePath: opts.HTTPBasePath,\n\t\tHTTPSPort: opts.HTTPSPort,\n\t\tCluster: ClusterOptsVarz{\n\t\t\tName: info.Cluster,\n\t\t\tHost: c.Host,\n\t\t\tPort: c.Port,\n\t\t\tAuthTimeout: c.AuthTimeout,\n\t\t\tTLSTimeout: c.TLSTimeout,\n\t\t\tTLSRequired: clustTlsReq,\n\t\t\tTLSVerify: clustTlsReq,\n\t\t\tPoolSize: opts.Cluster.PoolSize,\n\t\t\tWriteDeadline: opts.Cluster.WriteDeadline,\n\t\t\tWriteTimeout: opts.Cluster.WriteTimeout.String(),\n\t\t},\n\t\tGateway: GatewayOptsVarz{\n\t\t\tName: gw.Name,\n\t\t\tHost: gw.Host,\n\t\t\tPort: gw.Port,\n\t\t\tAuthTimeout: gw.AuthTimeout,\n\t\t\tTLSTimeout: gw.TLSTimeout,\n\t\t\tTLSRequired: gatewayTlsReq,\n\t\t\tTLSVerify: gatewayTlsReq,\n\t\t\tAdvertise: gw.Advertise,\n\t\t\tConnectRetries: gw.ConnectRetries,\n\t\t\tGateways: []RemoteGatewayOptsVarz{},\n\t\t\tRejectUnknown: gw.RejectUnknown,\n\t\t\tWriteDeadline: opts.Cluster.WriteDeadline,\n\t\t\tWriteTimeout: opts.Cluster.WriteTimeout.String(),\n\t\t},\n\t\tLeafNode: LeafNodeOptsVarz{\n\t\t\tHost: ln.Host,\n\t\t\tPort: ln.Port,\n\t\t\tAuthTimeout: ln.AuthTimeout,\n\t\t\tTLSTimeout: ln.TLSTimeout,\n\t\t\tTLSRequired: leafTlsReq,\n\t\t\tTLSVerify: leafTlsVerify,\n\t\t\tTLSOCSPPeerVerify: leafTlsOCSPPeerVerify,\n\t\t\tRemotes: []RemoteLeafOptsVarz{},\n\t\t\tWriteDeadline: opts.Cluster.WriteDeadline,\n\t\t\tWriteTimeout: opts.Cluster.WriteTimeout.String(),\n\t\t},\n\t\tMQTT: MQTTOptsVarz{\n\t\t\tHost: mqtt.Host,\n\t\t\tPort: mqtt.Port,\n\t\t\tNoAuthUser: mqtt.NoAuthUser,\n\t\t\tAuthTimeout: mqtt.AuthTimeout,\n\t\t\tTLSMap: mqtt.TLSMap,\n\t\t\tTLSTimeout: mqtt.TLSTimeout,\n\t\t\tJsDomain: mqtt.JsDomain,\n\t\t\tAckWait: mqtt.AckWait,\n\t\t\tMaxAckPending: mqtt.MaxAckPending,\n\t\t\tTLSOCSPPeerVerify: mqttTlsOCSPPeerVerify,\n\t\t},\n\t\tWebsocket: WebsocketOptsVarz{\n\t\t\tHost: ws.Host,\n\t\t\tPort: ws.Port,\n\t\t\tAdvertise: ws.Advertise,\n\t\t\tNoAuthUser: ws.NoAuthUser,\n\t\t\tJWTCookie: ws.JWTCookie,\n\t\t\tAuthTimeout: ws.AuthTimeout,\n\t\t\tNoTLS: ws.NoTLS,\n\t\t\tTLSMap: ws.TLSMap,\n\t\t\tSameOrigin: ws.SameOrigin,\n\t\t\tAllowedOrigins: copyStrings(ws.AllowedOrigins),\n\t\t\tCompression: ws.Compression,\n\t\t\tHandshakeTimeout: ws.HandshakeTimeout,\n\t\t\tTLSOCSPPeerVerify: wsTlsOCSPPeerVerify,\n\t\t},\n\t\tStart: s.start.UTC(),\n\t\tMaxSubs: opts.MaxSubs,\n\t\tCores: runtime.NumCPU(),\n\t\tMaxProcs: runtime.GOMAXPROCS(0),\n\t\tTrustedOperatorsJwt: opts.operatorJWT,\n\t\tTrustedOperatorsClaim: opts.TrustedOperators,\n\t}\n\tif mm := debug.SetMemoryLimit(-1); mm < math.MaxInt64 {\n\t\tvarz.MemLimit = mm\n\t}\n\t// If this is a leaf without cluster, reset the cluster name (that is otherwise\n\t// set to the server name).\n\tif s.leafNoCluster {\n\t\tvarz.Cluster.Name = _EMPTY_\n\t}\n\tif len(opts.Routes) > 0 {\n\t\tvarz.Cluster.URLs = urlsToStrings(opts.Routes)\n\t}\n\tif l := len(gw.Gateways); l > 0 {\n\t\trgwa := make([]RemoteGatewayOptsVarz, l)\n\t\tfor i, r := range gw.Gateways {\n\t\t\trgwa[i] = RemoteGatewayOptsVarz{\n\t\t\t\tName: r.Name,\n\t\t\t\tTLSTimeout: r.TLSTimeout,\n\t\t\t}\n\t\t}\n\t\tvarz.Gateway.Gateways = rgwa\n\t}\n\tif l := len(ln.Remotes); l > 0 {\n\t\trlna := make([]RemoteLeafOptsVarz, l)\n\t\tfor i, r := range ln.Remotes {\n\t\t\tvar deny *DenyRules\n\t\t\tif len(r.DenyImports) > 0 || len(r.DenyExports) > 0 {\n\t\t\t\tdeny = &DenyRules{\n\t\t\t\t\tImports: r.DenyImports,\n\t\t\t\t\tExports: r.DenyExports,\n\t\t\t\t}\n\t\t\t}\n\t\t\tremoteTlsOCSPPeerVerify := s.ocspPeerVerify && r.tlsConfigOpts != nil && r.tlsConfigOpts.OCSPPeerConfig != nil && r.tlsConfigOpts.OCSPPeerConfig.Verify\n\n\t\t\trlna[i] = RemoteLeafOptsVarz{\n\t\t\t\tLocalAccount: r.LocalAccount,\n\t\t\t\tURLs: urlsToStrings(r.URLs),\n\t\t\t\tTLSTimeout: r.TLSTimeout,\n\t\t\t\tDeny: deny,\n\t\t\t\tTLSOCSPPeerVerify: remoteTlsOCSPPeerVerify,\n\t\t\t}\n\t\t}\n\t\tvarz.LeafNode.Remotes = rlna\n\t}\n\n\t// Finish setting it up with fields that can be updated during\n\t// configuration reload and runtime.\n\ts.updateVarzConfigReloadableFields(varz)\n\ts.updateVarzRuntimeFields(varz, true, pcpu, rss)\n\treturn varz\n}\n\nfunc urlsToStrings(urls []*url.URL) []string {\n\tsURLs := make([]string, len(urls))\n\tfor i, u := range urls {\n\t\tsURLs[i] = u.Host\n\t}\n\treturn sURLs\n}\n\n// Invoked during configuration reload once options have possibly be changed\n// and config load time has been set. If s.varz has not been initialized yet\n// (because no pooling of /varz has been made), this function does nothing.\n// Server lock is held on entry.\nfunc (s *Server) updateVarzConfigReloadableFields(v *Varz) {\n\tif v == nil {\n\t\treturn\n\t}\n\topts := s.getOpts()\n\tinfo := &s.info\n\tv.AuthRequired = info.AuthRequired\n\tv.TLSRequired = info.TLSRequired\n\tv.TLSVerify = info.TLSVerify\n\tv.MaxConn = opts.MaxConn\n\tv.PingInterval = opts.PingInterval\n\tv.MaxPingsOut = opts.MaxPingsOut\n\tv.AuthTimeout = opts.AuthTimeout\n\tv.MaxControlLine = opts.MaxControlLine\n\tv.MaxPayload = int(opts.MaxPayload)\n\tv.MaxPending = opts.MaxPending\n\tv.TLSTimeout = opts.TLSTimeout\n\tv.WriteDeadline = opts.WriteDeadline\n\tv.WriteTimeout = opts.WriteTimeout.String()\n\tv.ConfigLoadTime = s.configTime.UTC()\n\tv.ConfigDigest = opts.configDigest\n\tv.Tags = opts.Tags\n\tv.Metadata = opts.Metadata\n\t// Update route URLs if applicable\n\tif s.varzUpdateRouteURLs {\n\t\tv.Cluster.URLs = urlsToStrings(opts.Routes)\n\t\ts.varzUpdateRouteURLs = false\n\t}\n\tif s.sys != nil && s.sys.account != nil {\n\t\tv.SystemAccount = s.sys.account.GetName()\n\t}\n\tv.MQTT.TLSPinnedCerts = getPinnedCertsAsSlice(opts.MQTT.TLSPinnedCerts)\n\tv.Websocket.TLSPinnedCerts = getPinnedCertsAsSlice(opts.Websocket.TLSPinnedCerts)\n\n\tv.TLSOCSPPeerVerify = s.ocspPeerVerify && v.TLSRequired && s.opts.tlsConfigOpts != nil && s.opts.tlsConfigOpts.OCSPPeerConfig != nil && s.opts.tlsConfigOpts.OCSPPeerConfig.Verify\n\n\tv.TLSCertNotAfter = tlsCertNotAfter(opts.TLSConfig)\n\tv.Cluster.TLSCertNotAfter = tlsCertNotAfter(opts.Cluster.TLSConfig)\n\tv.Gateway.TLSCertNotAfter = tlsCertNotAfter(opts.Gateway.TLSConfig)\n\tv.LeafNode.TLSCertNotAfter = tlsCertNotAfter(opts.LeafNode.TLSConfig)\n\tv.MQTT.TLSCertNotAfter = tlsCertNotAfter(opts.MQTT.TLSConfig)\n\tv.Websocket.TLSCertNotAfter = tlsCertNotAfter(opts.Websocket.TLSConfig)\n\n\tif opts.Proxies != nil {\n\t\tif v.Proxies == nil {\n\t\t\tv.Proxies = &ProxiesOptsVarz{}\n\t\t}\n\t\ttrusted := make([]*ProxyOptsVarz, 0, len(opts.Proxies.Trusted))\n\t\tfor _, t := range opts.Proxies.Trusted {\n\t\t\ttrusted = append(trusted, &ProxyOptsVarz{Key: t.Key})\n\t\t}\n\t\tv.Proxies.Trusted = trusted\n\t} else {\n\t\tv.Proxies = nil\n\t}\n}\n\nfunc getPinnedCertsAsSlice(certs PinnedCertSet) []string {\n\tif len(certs) == 0 {\n\t\treturn nil\n\t}\n\tres := make([]string, 0, len(certs))\n\tfor cn := range certs {\n\t\tres = append(res, cn)\n\t}\n\treturn res\n}\n\n// Updates the runtime Varz fields, that is, fields that change during\n// runtime and that should be updated any time Varz() or polling of /varz\n// is done.\n// Server lock is held on entry.\nfunc (s *Server) updateVarzRuntimeFields(v *Varz, forceUpdate bool, pcpu float64, rss int64) {\n\tv.Now = time.Now().UTC()\n\tv.Uptime = myUptime(time.Since(s.start))\n\tv.Mem = rss\n\tv.CPU = pcpu\n\tif l := len(s.info.ClientConnectURLs); l > 0 {\n\t\tv.ClientConnectURLs = append([]string(nil), s.info.ClientConnectURLs...)\n\t}\n\tif l := len(s.info.WSConnectURLs); l > 0 {\n\t\tv.WSConnectURLs = append([]string(nil), s.info.WSConnectURLs...)\n\t}\n\tv.Connections = len(s.clients)\n\tv.TotalConnections = s.totalClients\n\tv.Routes = s.numRoutes()\n\tv.Remotes = s.numRemotes()\n\tv.Leafs = len(s.leafs)\n\tv.InMsgs = atomic.LoadInt64(&s.inMsgs)\n\tv.InBytes = atomic.LoadInt64(&s.inBytes)\n\tv.OutMsgs = atomic.LoadInt64(&s.outMsgs)\n\tv.OutBytes = atomic.LoadInt64(&s.outBytes)\n\tv.SlowConsumers = atomic.LoadInt64(&s.slowConsumers)\n\tv.StalledClients = atomic.LoadInt64(&s.stalls)\n\tv.SlowConsumersStats = &SlowConsumersStats{\n\t\tClients: s.NumSlowConsumersClients(),\n\t\tRoutes: s.NumSlowConsumersRoutes(),\n\t\tGateways: s.NumSlowConsumersGateways(),\n\t\tLeafs: s.NumSlowConsumersLeafs(),\n\t}\n\tv.StaleConnections = atomic.LoadInt64(&s.staleConnections)\n\tv.StaleConnectionStats = &StaleConnectionStats{\n\t\tClients: s.NumStaleConnectionsClients(),\n\t\tRoutes: s.NumStaleConnectionsRoutes(),\n\t\tGateways: s.NumStaleConnectionsGateways(),\n\t\tLeafs: s.NumStaleConnectionsLeafs(),\n\t}\n\tv.PinnedAccountFail = atomic.LoadUint64(&s.pinnedAccFail)\n\n\t// Make sure to reset in case we are re-using.\n\tv.Subscriptions = 0\n\ts.accounts.Range(func(k, val any) bool {\n\t\tacc := val.(*Account)\n\t\tv.Subscriptions += acc.sl.Count()\n\t\treturn true\n\t})\n\n\tv.HTTPReqStats = make(map[string]uint64, len(s.httpReqStats))\n\tfor key, val := range s.httpReqStats {\n\t\tv.HTTPReqStats[key] = val\n\t}\n\n\t// Update Gateway remote urls if applicable\n\tgw := s.gateway\n\tgw.RLock()\n\tif gw.enabled {\n\t\tfor i := 0; i < len(v.Gateway.Gateways); i++ {\n\t\t\tg := &v.Gateway.Gateways[i]\n\t\t\trgw := gw.remotes[g.Name]\n\t\t\tif rgw != nil {\n\t\t\t\trgw.RLock()\n\t\t\t\t// forceUpdate is needed if user calls Varz() programmatically,\n\t\t\t\t// since we need to create a new instance every time and the\n\t\t\t\t// gateway's varzUpdateURLs may have been set to false after\n\t\t\t\t// a web /varz inspection.\n\t\t\t\tif forceUpdate || rgw.varzUpdateURLs {\n\t\t\t\t\t// Make reuse of backend array\n\t\t\t\t\tg.URLs = g.URLs[:0]\n\t\t\t\t\t// rgw.urls is a map[string]*url.URL where the key is\n\t\t\t\t\t// already in the right format (host:port, without any\n\t\t\t\t\t// user info present).\n\t\t\t\t\tfor u := range rgw.urls {\n\t\t\t\t\t\tg.URLs = append(g.URLs, u)\n\t\t\t\t\t}\n\t\t\t\t\trgw.varzUpdateURLs = false\n\t\t\t\t}\n\t\t\t\trgw.RUnlock()\n\t\t\t} else if g.Name == gw.name && len(gw.ownCfgURLs) > 0 {\n\t\t\t\t// This is a remote that correspond to this very same server.\n\t\t\t\t// We report the URLs that were configured (if any).\n\t\t\t\t// Since we don't support changes to the gateway configuration\n\t\t\t\t// at this time, we could do this only if g.URLs has not been already\n\t\t\t\t// set, but let's do it regardless in case we add support for\n\t\t\t\t// gateway config reload.\n\t\t\t\tg.URLs = g.URLs[:0]\n\t\t\t\tg.URLs = append(g.URLs, gw.ownCfgURLs...)\n\t\t\t}\n\t\t}\n\t}\n\tgw.RUnlock()\n\n\tif s.ocsprc != nil && s.ocsprc.Type() != \"none\" {\n\t\tstats := s.ocsprc.Stats()\n\t\tif stats != nil {\n\t\t\tv.OCSPResponseCache = &OCSPResponseCacheVarz{\n\t\t\t\ts.ocsprc.Type(),\n\t\t\t\tstats.Hits,\n\t\t\t\tstats.Misses,\n\t\t\t\tstats.Responses,\n\t\t\t\tstats.Revokes,\n\t\t\t\tstats.Goods,\n\t\t\t\tstats.Unknowns,\n\t\t\t}\n\t\t}\n\t}\n}\n\n// HandleVarz will process HTTP requests for server information.\nfunc (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {\n\tvar rss, vss int64\n\tvar pcpu float64\n\n\t// We want to do that outside of the lock.\n\tpse.ProcUsage(&pcpu, &rss, &vss)\n\n\t// In response to http requests, we want to minimize mem copies\n\t// so we use an object stored in the server. Creating/collecting\n\t// server metrics is done under server lock, but we don't want\n\t// to marshal under that lock. Still, we need to prevent concurrent\n\t// http requests to /varz to update s.varz while marshal is\n\t// happening, so we need a new lock that serialize those http\n\t// requests and include marshaling.\n\ts.varzMu.Lock()\n\n\t// Use server lock to create/update the server's varz object.\n\ts.mu.Lock()\n\tvar created bool\n\ts.httpReqStats[VarzPath]++\n\tif s.varz == nil {\n\t\ts.varz = s.createVarz(pcpu, rss)\n\t\tcreated = true\n\t} else {\n\t\ts.updateVarzRuntimeFields(s.varz, false, pcpu, rss)\n\t}\n\ts.mu.Unlock()\n\t// Since locking is jetStream -> Server, need to update jetstream\n\t// varz outside of server lock.\n\n\tif js := s.getJetStream(); js != nil {\n\t\tvar v JetStreamVarz\n\t\t// Work on stack variable\n\t\ts.updateJszVarz(js, &v, created)\n\t\t// Now update server's varz\n\t\ts.mu.RLock()\n\t\tsv := &s.varz.JetStream\n\t\tif created {\n\t\t\tsv.Config = v.Config\n\t\t}\n\t\tsv.Stats = v.Stats\n\t\tsv.Meta = v.Meta\n\t\tsv.Limits = v.Limits\n\t\ts.mu.RUnlock()\n\t}\n\n\t// Do the marshaling outside of server lock, but under varzMu lock.\n\tb, err := json.MarshalIndent(s.varz, \"\", \" \")\n\ts.varzMu.Unlock()\n\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /varz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// GatewayzOptions are the options passed to Gatewayz()\ntype GatewayzOptions struct {\n\t// Name will output only remote gateways with this name\n\tName string `json:\"name\"`\n\n\t// Accounts indicates if accounts with its interest should be included in the results.\n\tAccounts bool `json:\"accounts\"`\n\n\t// AccountName will limit the list of accounts to that account name (makes Accounts implicit)\n\tAccountName string `json:\"account_name\"`\n\n\t// AccountSubscriptions indicates if subscriptions should be included in the results.\n\t// Note: This is used only if `Accounts` or `AccountName` are specified.\n\tAccountSubscriptions bool `json:\"subscriptions\"`\n\n\t// AccountSubscriptionsDetail indicates if subscription details should be included in the results.\n\t// Note: This is used only if `Accounts` or `AccountName` are specified.\n\tAccountSubscriptionsDetail bool `json:\"subscriptions_detail\"`\n}\n\n// Gatewayz represents detailed information on Gateways\ntype Gatewayz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tName string `json:\"name,omitempty\"`\n\tHost string `json:\"host,omitempty\"`\n\tPort int `json:\"port,omitempty\"`\n\tOutboundGateways map[string]*RemoteGatewayz `json:\"outbound_gateways\"`\n\tInboundGateways map[string][]*RemoteGatewayz `json:\"inbound_gateways\"`\n}\n\n// RemoteGatewayz represents information about an outbound connection to a gateway\ntype RemoteGatewayz struct {\n\tIsConfigured bool `json:\"configured\"`\n\tConnection *ConnInfo `json:\"connection,omitempty\"`\n\tAccounts []*AccountGatewayz `json:\"accounts,omitempty\"`\n}\n\n// AccountGatewayz represents interest mode for this account\ntype AccountGatewayz struct {\n\tName string `json:\"name\"`\n\tInterestMode string `json:\"interest_mode\"`\n\tNoInterestCount int `json:\"no_interest_count,omitempty\"`\n\tInterestOnlyThreshold int `json:\"interest_only_threshold,omitempty\"`\n\tTotalSubscriptions int `json:\"num_subs,omitempty\"`\n\tNumQueueSubscriptions int `json:\"num_queue_subs,omitempty\"`\n\tSubs []string `json:\"subscriptions_list,omitempty\"`\n\tSubsDetail []SubDetail `json:\"subscriptions_list_detail,omitempty\"`\n}\n\n// Gatewayz returns a Gatewayz struct containing information about gateways.\nfunc (s *Server) Gatewayz(opts *GatewayzOptions) (*Gatewayz, error) {\n\tsrvID := s.ID()\n\tnow := time.Now().UTC()\n\tgw := s.gateway\n\tgw.RLock()\n\tif !gw.enabled || gw.info == nil {\n\t\tgw.RUnlock()\n\t\tgwz := &Gatewayz{\n\t\t\tID: srvID,\n\t\t\tNow: now,\n\t\t\tOutboundGateways: map[string]*RemoteGatewayz{},\n\t\t\tInboundGateways: map[string][]*RemoteGatewayz{},\n\t\t}\n\t\treturn gwz, nil\n\t}\n\t// Here gateways are enabled, so fill up more.\n\tgwz := &Gatewayz{\n\t\tID: srvID,\n\t\tNow: now,\n\t\tName: gw.name,\n\t\tHost: gw.info.Host,\n\t\tPort: gw.info.Port,\n\t}\n\tgw.RUnlock()\n\n\tgwz.OutboundGateways = s.createOutboundsRemoteGatewayz(opts, now)\n\tgwz.InboundGateways = s.createInboundsRemoteGatewayz(opts, now)\n\n\treturn gwz, nil\n}\n\n// Based on give options struct, returns if there is a filtered\n// Gateway Name and if we should do report Accounts.\n// Note that if Accounts is false but AccountName is not empty,\n// then Accounts is implicitly set to true.\nfunc getMonitorGWOptions(opts *GatewayzOptions) (string, bool) {\n\tvar name string\n\tvar accs bool\n\tif opts != nil {\n\t\tif opts.Name != _EMPTY_ {\n\t\t\tname = opts.Name\n\t\t}\n\t\taccs = opts.Accounts\n\t\tif !accs && opts.AccountName != _EMPTY_ {\n\t\t\taccs = true\n\t\t}\n\t}\n\treturn name, accs\n}\n\n// Returns a map of gateways outbound connections.\n// Based on options, will include a single or all gateways,\n// with no/single/or all accounts interest information.\nfunc (s *Server) createOutboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string]*RemoteGatewayz {\n\ttargetGWName, doAccs := getMonitorGWOptions(opts)\n\n\tif targetGWName != _EMPTY_ {\n\t\tc := s.getOutboundGatewayConnection(targetGWName)\n\t\tif c == nil {\n\t\t\treturn nil\n\t\t}\n\t\toutbounds := make(map[string]*RemoteGatewayz, 1)\n\t\t_, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs)\n\t\toutbounds[targetGWName] = rgw\n\t\treturn outbounds\n\t}\n\n\tvar connsa [16]*client\n\tvar conns = connsa[:0]\n\n\ts.getOutboundGatewayConnections(&conns)\n\n\toutbounds := make(map[string]*RemoteGatewayz, len(conns))\n\tfor _, c := range conns {\n\t\tname, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs)\n\t\tif rgw != nil {\n\t\t\toutbounds[name] = rgw\n\t\t}\n\t}\n\treturn outbounds\n}\n\n// Returns a RemoteGatewayz for a given outbound gw connection\nfunc createOutboundRemoteGatewayz(c *client, opts *GatewayzOptions, now time.Time, doAccs bool) (string, *RemoteGatewayz) {\n\tvar name string\n\tvar rgw *RemoteGatewayz\n\n\tc.mu.Lock()\n\tif c.gw != nil {\n\t\trgw = &RemoteGatewayz{}\n\t\tif doAccs {\n\t\t\trgw.Accounts = createOutboundAccountsGatewayz(opts, c.gw)\n\t\t}\n\t\tif c.gw.cfg != nil {\n\t\t\trgw.IsConfigured = !c.gw.cfg.isImplicit()\n\t\t}\n\t\trgw.Connection = &ConnInfo{}\n\t\trgw.Connection.fill(c, c.nc, now, false)\n\t\tname = c.gw.name\n\t}\n\tc.mu.Unlock()\n\n\treturn name, rgw\n}\n\n// Returns the list of accounts for this outbound gateway connection.\n// Based on the options, it will be a single or all accounts for\n// this outbound.\nfunc createOutboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz {\n\tif gw.outsim == nil {\n\t\treturn nil\n\t}\n\n\tvar accName string\n\tif opts != nil {\n\t\taccName = opts.AccountName\n\t}\n\tif accName != _EMPTY_ {\n\t\tei, ok := gw.outsim.Load(accName)\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\ta := createAccountOutboundGatewayz(opts, accName, ei)\n\t\treturn []*AccountGatewayz{a}\n\t}\n\n\taccs := make([]*AccountGatewayz, 0, 4)\n\tgw.outsim.Range(func(k, v any) bool {\n\t\tname := k.(string)\n\t\ta := createAccountOutboundGatewayz(opts, name, v)\n\t\taccs = append(accs, a)\n\t\treturn true\n\t})\n\treturn accs\n}\n\n// Returns an AccountGatewayz for this gateway outbound connection\nfunc createAccountOutboundGatewayz(opts *GatewayzOptions, name string, ei any) *AccountGatewayz {\n\ta := &AccountGatewayz{\n\t\tName: name,\n\t\tInterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch,\n\t}\n\tif ei != nil {\n\t\te := ei.(*outsie)\n\t\te.RLock()\n\t\ta.InterestMode = e.mode.String()\n\t\ta.NoInterestCount = len(e.ni)\n\t\ta.NumQueueSubscriptions = e.qsubs\n\t\ta.TotalSubscriptions = int(e.sl.Count())\n\t\tif opts.AccountSubscriptions || opts.AccountSubscriptionsDetail {\n\t\t\tvar subsa [4096]*subscription\n\t\t\tsubs := subsa[:0]\n\t\t\te.sl.All(&subs)\n\t\t\tif opts.AccountSubscriptions {\n\t\t\t\ta.Subs = make([]string, 0, len(subs))\n\t\t\t} else {\n\t\t\t\ta.SubsDetail = make([]SubDetail, 0, len(subs))\n\t\t\t}\n\t\t\tfor _, sub := range subs {\n\t\t\t\tif opts.AccountSubscriptions {\n\t\t\t\t\ta.Subs = append(a.Subs, string(sub.subject))\n\t\t\t\t} else {\n\t\t\t\t\ta.SubsDetail = append(a.SubsDetail, newClientSubDetail(sub))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\te.RUnlock()\n\t} else {\n\t\ta.InterestMode = Optimistic.String()\n\t}\n\treturn a\n}\n\n// Returns a map of gateways inbound connections.\n// Each entry is an array of RemoteGatewayz since a given server\n// may have more than one inbound from the same remote gateway.\n// Based on options, will include a single or all gateways,\n// with no/single/or all accounts interest information.\nfunc (s *Server) createInboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string][]*RemoteGatewayz {\n\ttargetGWName, doAccs := getMonitorGWOptions(opts)\n\n\tvar connsa [16]*client\n\tvar conns = connsa[:0]\n\ts.getInboundGatewayConnections(&conns)\n\n\tm := make(map[string][]*RemoteGatewayz)\n\tfor _, c := range conns {\n\t\tc.mu.Lock()\n\t\tif c.gw != nil && (targetGWName == _EMPTY_ || targetGWName == c.gw.name) {\n\t\t\tigws := m[c.gw.name]\n\t\t\tif igws == nil {\n\t\t\t\tigws = make([]*RemoteGatewayz, 0, 2)\n\t\t\t}\n\t\t\trgw := &RemoteGatewayz{}\n\t\t\tif doAccs {\n\t\t\t\trgw.Accounts = createInboundAccountsGatewayz(opts, c.gw)\n\t\t\t}\n\t\t\trgw.Connection = &ConnInfo{}\n\t\t\trgw.Connection.fill(c, c.nc, now, false)\n\t\t\tigws = append(igws, rgw)\n\t\t\tm[c.gw.name] = igws\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n\treturn m\n}\n\n// Returns the list of accounts for this inbound gateway connection.\n// Based on the options, it will be a single or all accounts for\n// this inbound.\nfunc createInboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz {\n\tif gw.insim == nil {\n\t\treturn nil\n\t}\n\n\tvar accName string\n\tif opts != nil {\n\t\taccName = opts.AccountName\n\t}\n\tif accName != _EMPTY_ {\n\t\te, ok := gw.insim[accName]\n\t\tif !ok {\n\t\t\treturn nil\n\t\t}\n\t\ta := createInboundAccountGatewayz(accName, e)\n\t\treturn []*AccountGatewayz{a}\n\t}\n\n\taccs := make([]*AccountGatewayz, 0, 4)\n\tfor name, e := range gw.insim {\n\t\ta := createInboundAccountGatewayz(name, e)\n\t\taccs = append(accs, a)\n\t}\n\treturn accs\n}\n\n// Returns an AccountGatewayz for this gateway inbound connection\nfunc createInboundAccountGatewayz(name string, e *insie) *AccountGatewayz {\n\ta := &AccountGatewayz{\n\t\tName: name,\n\t\tInterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch,\n\t}\n\tif e != nil {\n\t\ta.InterestMode = e.mode.String()\n\t\ta.NoInterestCount = len(e.ni)\n\t} else {\n\t\ta.InterestMode = Optimistic.String()\n\t}\n\treturn a\n}\n\n// HandleGatewayz process HTTP requests for route information.\nfunc (s *Server) HandleGatewayz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[GatewayzPath]++\n\ts.mu.Unlock()\n\n\tsubs, subsDet, err := decodeSubs(w, r)\n\tif err != nil {\n\t\treturn\n\t}\n\taccs, err := decodeBool(w, r, \"accs\")\n\tif err != nil {\n\t\treturn\n\t}\n\tgwName := r.URL.Query().Get(\"gw_name\")\n\taccName := r.URL.Query().Get(\"acc_name\")\n\tif accName != _EMPTY_ {\n\t\taccs = true\n\t}\n\n\topts := &GatewayzOptions{\n\t\tName: gwName,\n\t\tAccounts: accs,\n\t\tAccountName: accName,\n\t\tAccountSubscriptions: subs,\n\t\tAccountSubscriptionsDetail: subsDet,\n\t}\n\tgw, err := s.Gatewayz(opts)\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\tb, err := json.MarshalIndent(gw, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /gatewayz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// Leafz represents detailed information on Leafnodes.\ntype Leafz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tNumLeafs int `json:\"leafnodes\"`\n\tLeafs []*LeafInfo `json:\"leafs\"`\n}\n\n// LeafzOptions are options passed to Leafz\ntype LeafzOptions struct {\n\t// Subscriptions indicates that Leafz will return a leafnode's subscriptions\n\tSubscriptions bool `json:\"subscriptions\"`\n\tAccount string `json:\"account\"`\n}\n\n// LeafInfo has detailed information on each remote leafnode connection.\ntype LeafInfo struct {\n\tID uint64 `json:\"id\"`\n\tName string `json:\"name\"`\n\tIsSpoke bool `json:\"is_spoke\"`\n\tIsIsolated bool `json:\"is_isolated,omitempty\"`\n\tAccount string `json:\"account\"`\n\tIP string `json:\"ip\"`\n\tPort int `json:\"port\"`\n\tRTT string `json:\"rtt,omitempty\"`\n\tInMsgs int64 `json:\"in_msgs\"`\n\tOutMsgs int64 `json:\"out_msgs\"`\n\tInBytes int64 `json:\"in_bytes\"`\n\tOutBytes int64 `json:\"out_bytes\"`\n\tNumSubs uint32 `json:\"subscriptions\"`\n\tSubs []string `json:\"subscriptions_list,omitempty\"`\n\tCompression string `json:\"compression,omitempty\"`\n\tProxy *ProxyInfo `json:\"proxy,omitempty\"`\n}\n\n// Leafz returns a Leafz structure containing information about leafnodes.\nfunc (s *Server) Leafz(opts *LeafzOptions) (*Leafz, error) {\n\t// Grab leafnodes\n\tvar lconns []*client\n\ts.mu.Lock()\n\tif len(s.leafs) > 0 {\n\t\tlconns = make([]*client, 0, len(s.leafs))\n\t\tfor _, ln := range s.leafs {\n\t\t\tif opts != nil && opts.Account != _EMPTY_ {\n\t\t\t\tln.mu.Lock()\n\t\t\t\tok := ln.acc.Name == opts.Account\n\t\t\t\tln.mu.Unlock()\n\t\t\t\tif !ok {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tlconns = append(lconns, ln)\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\tleafnodes := make([]*LeafInfo, 0, len(lconns))\n\n\tif len(lconns) > 0 {\n\t\tfor _, ln := range lconns {\n\t\t\tln.mu.Lock()\n\t\t\tlni := &LeafInfo{\n\t\t\t\tID: ln.cid,\n\t\t\t\tName: ln.leaf.remoteServer,\n\t\t\t\tIsSpoke: ln.isSpokeLeafNode(),\n\t\t\t\tIsIsolated: ln.leaf.isolated,\n\t\t\t\tAccount: ln.acc.Name,\n\t\t\t\tIP: ln.host,\n\t\t\t\tPort: int(ln.port),\n\t\t\t\tRTT: ln.getRTT().String(),\n\t\t\t\tInMsgs: atomic.LoadInt64(&ln.inMsgs),\n\t\t\t\tOutMsgs: ln.outMsgs,\n\t\t\t\tInBytes: atomic.LoadInt64(&ln.inBytes),\n\t\t\t\tOutBytes: ln.outBytes,\n\t\t\t\tNumSubs: uint32(len(ln.subs)),\n\t\t\t\tCompression: ln.leaf.compression,\n\t\t\t\tProxy: createProxyInfo(ln),\n\t\t\t}\n\t\t\tif opts != nil && opts.Subscriptions {\n\t\t\t\tlni.Subs = make([]string, 0, len(ln.subs))\n\t\t\t\tfor _, sub := range ln.subs {\n\t\t\t\t\tlni.Subs = append(lni.Subs, string(sub.subject))\n\t\t\t\t}\n\t\t\t}\n\t\t\tln.mu.Unlock()\n\t\t\tleafnodes = append(leafnodes, lni)\n\t\t}\n\t}\n\n\treturn &Leafz{\n\t\tID: s.ID(),\n\t\tNow: time.Now().UTC(),\n\t\tNumLeafs: len(leafnodes),\n\t\tLeafs: leafnodes,\n\t}, nil\n}\n\n// HandleLeafz process HTTP requests for leafnode information.\nfunc (s *Server) HandleLeafz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[LeafzPath]++\n\ts.mu.Unlock()\n\n\tsubs, err := decodeBool(w, r, \"subs\")\n\tif err != nil {\n\t\treturn\n\t}\n\tl, err := s.Leafz(&LeafzOptions{subs, r.URL.Query().Get(\"acc\")})\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\tb, err := json.MarshalIndent(l, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /leafz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// Leafz represents detailed information on Leafnodes.\ntype AccountStatz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tAccounts []*AccountStat `json:\"account_statz\"`\n}\n\n// AccountStatzOptions are options passed to account stats requests.\ntype AccountStatzOptions struct {\n\tAccounts []string `json:\"accounts\"`\n\tIncludeUnused bool `json:\"include_unused\"`\n}\n\n// Leafz returns a AccountStatz structure containing summary information about accounts.\nfunc (s *Server) AccountStatz(opts *AccountStatzOptions) (*AccountStatz, error) {\n\tstz := &AccountStatz{\n\t\tID: s.ID(),\n\t\tNow: time.Now().UTC(),\n\t\tAccounts: []*AccountStat{},\n\t}\n\tif opts == nil || len(opts.Accounts) == 0 {\n\t\ts.accounts.Range(func(key, a any) bool {\n\t\t\tacc := a.(*Account)\n\t\t\tacc.mu.RLock()\n\t\t\tif (opts != nil && opts.IncludeUnused) || acc.numLocalConnections() != 0 {\n\t\t\t\tstz.Accounts = append(stz.Accounts, acc.statz())\n\t\t\t}\n\t\t\tacc.mu.RUnlock()\n\t\t\treturn true\n\t\t})\n\t} else {\n\t\tfor _, a := range opts.Accounts {\n\t\t\tif acc, ok := s.accounts.Load(a); ok {\n\t\t\t\tacc := acc.(*Account)\n\t\t\t\tacc.mu.RLock()\n\t\t\t\tif opts.IncludeUnused || acc.numLocalConnections() != 0 {\n\t\t\t\t\tstz.Accounts = append(stz.Accounts, acc.statz())\n\t\t\t\t}\n\t\t\t\tacc.mu.RUnlock()\n\t\t\t}\n\t\t}\n\t}\n\treturn stz, nil\n}\n\n// HandleAccountStatz process HTTP requests for statz information of all accounts.\nfunc (s *Server) HandleAccountStatz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[AccountStatzPath]++\n\ts.mu.Unlock()\n\n\tunused, err := decodeBool(w, r, \"unused\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\tl, err := s.AccountStatz(&AccountStatzOptions{IncludeUnused: unused})\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\tb, err := json.MarshalIndent(l, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to %s request: %v\", AccountStatzPath, err)\n\t\treturn\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\n// ResponseHandler handles responses for monitoring routes.\nfunc ResponseHandler(w http.ResponseWriter, r *http.Request, data []byte) {\n\thandleResponse(http.StatusOK, w, r, data)\n}\n\n// handleResponse handles responses for monitoring routes with a specific HTTP status code.\nfunc handleResponse(code int, w http.ResponseWriter, r *http.Request, data []byte) {\n\t// Get callback from request\n\tcallback := r.URL.Query().Get(\"callback\")\n\tif callback != _EMPTY_ {\n\t\t// Response for JSONP\n\t\tw.Header().Set(\"Content-Type\", \"application/javascript\")\n\t\tw.WriteHeader(code)\n\t\tfmt.Fprintf(w, \"%s(%s)\", callback, data)\n\t} else {\n\t\t// Otherwise JSON\n\t\tw.Header().Set(\"Content-Type\", \"application/json\")\n\t\tw.Header().Set(\"Access-Control-Allow-Origin\", \"*\")\n\t\tw.WriteHeader(code)\n\t\tw.Write(data)\n\t}\n}\n\nfunc (reason ClosedState) String() string {\n\tswitch reason {\n\tcase ClientClosed:\n\t\treturn \"Client Closed\"\n\tcase AuthenticationTimeout:\n\t\treturn \"Authentication Timeout\"\n\tcase AuthenticationViolation:\n\t\treturn \"Authentication Failure\"\n\tcase TLSHandshakeError:\n\t\treturn \"TLS Handshake Failure\"\n\tcase SlowConsumerPendingBytes:\n\t\treturn \"Slow Consumer (Pending Bytes)\"\n\tcase SlowConsumerWriteDeadline:\n\t\treturn \"Slow Consumer (Write Deadline)\"\n\tcase WriteError:\n\t\treturn \"Write Error\"\n\tcase ReadError:\n\t\treturn \"Read Error\"\n\tcase ParseError:\n\t\treturn \"Parse Error\"\n\tcase StaleConnection:\n\t\treturn \"Stale Connection\"\n\tcase ProtocolViolation:\n\t\treturn \"Protocol Violation\"\n\tcase BadClientProtocolVersion:\n\t\treturn \"Bad Client Protocol Version\"\n\tcase WrongPort:\n\t\treturn \"Incorrect Port\"\n\tcase MaxConnectionsExceeded:\n\t\treturn \"Maximum Connections Exceeded\"\n\tcase MaxAccountConnectionsExceeded:\n\t\treturn \"Maximum Account Connections Exceeded\"\n\tcase MaxPayloadExceeded:\n\t\treturn \"Maximum Message Payload Exceeded\"\n\tcase MaxControlLineExceeded:\n\t\treturn \"Maximum Control Line Exceeded\"\n\tcase MaxSubscriptionsExceeded:\n\t\treturn \"Maximum Subscriptions Exceeded\"\n\tcase DuplicateRoute:\n\t\treturn \"Duplicate Route\"\n\tcase RouteRemoved:\n\t\treturn \"Route Removed\"\n\tcase ServerShutdown:\n\t\treturn \"Server Shutdown\"\n\tcase AuthenticationExpired:\n\t\treturn \"Authentication Expired\"\n\tcase WrongGateway:\n\t\treturn \"Wrong Gateway\"\n\tcase MissingAccount:\n\t\treturn \"Missing Account\"\n\tcase Revocation:\n\t\treturn \"Credentials Revoked\"\n\tcase InternalClient:\n\t\treturn \"Internal Client\"\n\tcase MsgHeaderViolation:\n\t\treturn \"Message Header Violation\"\n\tcase NoRespondersRequiresHeaders:\n\t\treturn \"No Responders Requires Headers\"\n\tcase ClusterNameConflict:\n\t\treturn \"Cluster Name Conflict\"\n\tcase DuplicateRemoteLeafnodeConnection:\n\t\treturn \"Duplicate Remote LeafNode Connection\"\n\tcase DuplicateClientID:\n\t\treturn \"Duplicate Client ID\"\n\tcase DuplicateServerName:\n\t\treturn \"Duplicate Server Name\"\n\tcase MinimumVersionRequired:\n\t\treturn \"Minimum Version Required\"\n\tcase ClusterNamesIdentical:\n\t\treturn \"Cluster Names Identical\"\n\tcase Kicked:\n\t\treturn \"Kicked\"\n\tcase ProxyNotTrusted:\n\t\treturn \"Proxy Not Trusted\"\n\tcase ProxyRequired:\n\t\treturn \"Proxy Required\"\n\t}\n\n\treturn \"Unknown State\"\n}\n\n// AccountzOptions are options passed to Accountz\ntype AccountzOptions struct {\n\t// Account indicates that Accountz will return details for the account\n\tAccount string `json:\"account\"`\n}\n\nfunc newExtServiceLatency(l *serviceLatency) *jwt.ServiceLatency {\n\tif l == nil {\n\t\treturn nil\n\t}\n\treturn &jwt.ServiceLatency{\n\t\tSampling: jwt.SamplingRate(l.sampling),\n\t\tResults: jwt.Subject(l.subject),\n\t}\n}\n\ntype ExtImport struct {\n\tjwt.Import\n\tInvalid bool `json:\"invalid\"`\n\tShare bool `json:\"share\"`\n\tTracking bool `json:\"tracking\"`\n\tTrackingHdr http.Header `json:\"tracking_header,omitempty\"`\n\tLatency *jwt.ServiceLatency `json:\"latency,omitempty\"`\n\tM1 *ServiceLatency `json:\"m1,omitempty\"`\n}\n\ntype ExtExport struct {\n\tjwt.Export\n\tApprovedAccounts []string `json:\"approved_accounts,omitempty\"`\n\tRevokedAct map[string]time.Time `json:\"revoked_activations,omitempty\"`\n}\n\ntype ExtVrIssues struct {\n\tDescription string `json:\"description\"`\n\tBlocking bool `json:\"blocking\"`\n\tTime bool `json:\"time_check\"`\n}\n\ntype ExtMap map[string][]*MapDest\n\ntype AccountInfo struct {\n\tAccountName string `json:\"account_name\"`\n\tLastUpdate time.Time `json:\"update_time,omitempty\"`\n\tIsSystem bool `json:\"is_system,omitempty\"`\n\tExpired bool `json:\"expired\"`\n\tComplete bool `json:\"complete\"`\n\tJetStream bool `json:\"jetstream_enabled\"`\n\tLeafCnt int `json:\"leafnode_connections\"`\n\tClientCnt int `json:\"client_connections\"`\n\tSubCnt uint32 `json:\"subscriptions\"`\n\tMappings ExtMap `json:\"mappings,omitempty\"`\n\tExports []ExtExport `json:\"exports,omitempty\"`\n\tImports []ExtImport `json:\"imports,omitempty\"`\n\tJwt string `json:\"jwt,omitempty\"`\n\tIssuerKey string `json:\"issuer_key,omitempty\"`\n\tNameTag string `json:\"name_tag,omitempty\"`\n\tTags jwt.TagList `json:\"tags,omitempty\"`\n\tClaim *jwt.AccountClaims `json:\"decoded_jwt,omitempty\"`\n\tVr []ExtVrIssues `json:\"validation_result_jwt,omitempty\"`\n\tRevokedUser map[string]time.Time `json:\"revoked_user,omitempty\"`\n\tSublist *SublistStats `json:\"sublist_stats,omitempty\"`\n\tResponses map[string]ExtImport `json:\"responses,omitempty\"`\n}\n\ntype Accountz struct {\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tSystemAccount string `json:\"system_account,omitempty\"`\n\tAccounts []string `json:\"accounts,omitempty\"`\n\tAccount *AccountInfo `json:\"account_detail,omitempty\"`\n}\n\n// HandleAccountz process HTTP requests for account information.\nfunc (s *Server) HandleAccountz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[AccountzPath]++\n\ts.mu.Unlock()\n\tif l, err := s.Accountz(&AccountzOptions{r.URL.Query().Get(\"acc\")}); err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t} else if b, err := json.MarshalIndent(l, \"\", \" \"); err != nil {\n\t\ts.Errorf(\"Error marshaling response to %s request: %v\", AccountzPath, err)\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t} else {\n\t\tResponseHandler(w, r, b) // Handle response\n\t}\n}\n\nfunc (s *Server) Accountz(optz *AccountzOptions) (*Accountz, error) {\n\ta := &Accountz{\n\t\tID: s.ID(),\n\t\tNow: time.Now().UTC(),\n\t}\n\tif sacc := s.SystemAccount(); sacc != nil {\n\t\ta.SystemAccount = sacc.GetName()\n\t}\n\tif optz == nil || optz.Account == _EMPTY_ {\n\t\ta.Accounts = []string{}\n\t\ts.accounts.Range(func(key, value any) bool {\n\t\t\ta.Accounts = append(a.Accounts, key.(string))\n\t\t\treturn true\n\t\t})\n\t\treturn a, nil\n\t}\n\taInfo, err := s.accountInfo(optz.Account)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.Account = aInfo\n\treturn a, nil\n}\n\nfunc newExtImport(v *serviceImport) ExtImport {\n\timp := ExtImport{\n\t\tInvalid: true,\n\t\tImport: jwt.Import{Type: jwt.Service},\n\t}\n\tif v != nil {\n\t\timp.Share = v.share\n\t\timp.Tracking = v.tracking\n\t\timp.Invalid = v.invalid\n\t\timp.Import = jwt.Import{\n\t\t\tSubject: jwt.Subject(v.to),\n\t\t\tAccount: v.acc.Name,\n\t\t\tType: jwt.Service,\n\t\t\t// Deprecated so we duplicate. Use LocalSubject.\n\t\t\tTo: jwt.Subject(v.from),\n\t\t\tLocalSubject: jwt.RenamingSubject(v.from),\n\t\t}\n\t\timp.TrackingHdr = v.trackingHdr\n\t\timp.Latency = newExtServiceLatency(v.latency)\n\t\tif v.m1 != nil {\n\t\t\tm1 := *v.m1\n\t\t\timp.M1 = &m1\n\t\t}\n\t}\n\treturn imp\n}\n\nfunc (s *Server) accountInfo(accName string) (*AccountInfo, error) {\n\tvar a *Account\n\tif v, ok := s.accounts.Load(accName); !ok {\n\t\treturn nil, fmt.Errorf(\"Account %s does not exist\", accName)\n\t} else {\n\t\ta = v.(*Account)\n\t}\n\tisSys := a == s.SystemAccount()\n\ta.mu.RLock()\n\tdefer a.mu.RUnlock()\n\tvar vrIssues []ExtVrIssues\n\tclaim, _ := jwt.DecodeAccountClaims(a.claimJWT) // ignore error\n\tif claim != nil {\n\t\tvr := jwt.ValidationResults{}\n\t\tclaim.Validate(&vr)\n\t\tvrIssues = make([]ExtVrIssues, len(vr.Issues))\n\t\tfor i, v := range vr.Issues {\n\t\t\tvrIssues[i] = ExtVrIssues{v.Description, v.Blocking, v.TimeCheck}\n\t\t}\n\t}\n\tcollectRevocations := func(revocations map[string]int64) map[string]time.Time {\n\t\tl := len(revocations)\n\t\tif l == 0 {\n\t\t\treturn nil\n\t\t}\n\t\trev := make(map[string]time.Time, l)\n\t\tfor k, v := range revocations {\n\t\t\trev[k] = time.Unix(v, 0)\n\t\t}\n\t\treturn rev\n\t}\n\texports := []ExtExport{}\n\tfor k, v := range a.exports.services {\n\t\te := ExtExport{\n\t\t\tExport: jwt.Export{\n\t\t\t\tSubject: jwt.Subject(k),\n\t\t\t\tType: jwt.Service,\n\t\t\t},\n\t\t\tApprovedAccounts: []string{},\n\t\t}\n\t\tif v != nil {\n\t\t\te.Latency = newExtServiceLatency(v.latency)\n\t\t\te.TokenReq = v.tokenReq\n\t\t\te.ResponseType = jwt.ResponseType(v.respType.String())\n\t\t\tfor name := range v.approved {\n\t\t\t\te.ApprovedAccounts = append(e.ApprovedAccounts, name)\n\t\t\t}\n\t\t\te.RevokedAct = collectRevocations(v.actsRevoked)\n\t\t}\n\t\texports = append(exports, e)\n\t}\n\tfor k, v := range a.exports.streams {\n\t\te := ExtExport{\n\t\t\tExport: jwt.Export{\n\t\t\t\tSubject: jwt.Subject(k),\n\t\t\t\tType: jwt.Stream,\n\t\t\t},\n\t\t\tApprovedAccounts: []string{},\n\t\t}\n\t\tif v != nil {\n\t\t\te.TokenReq = v.tokenReq\n\t\t\tfor name := range v.approved {\n\t\t\t\te.ApprovedAccounts = append(e.ApprovedAccounts, name)\n\t\t\t}\n\t\t\te.RevokedAct = collectRevocations(v.actsRevoked)\n\t\t}\n\t\texports = append(exports, e)\n\t}\n\timports := []ExtImport{}\n\tfor _, v := range a.imports.streams {\n\t\timp := ExtImport{\n\t\t\tInvalid: true,\n\t\t\tImport: jwt.Import{Type: jwt.Stream},\n\t\t}\n\t\tif v != nil {\n\t\t\timp.Invalid = v.invalid\n\t\t\timp.Import = jwt.Import{\n\t\t\t\tSubject: jwt.Subject(v.from),\n\t\t\t\tAccount: v.acc.Name,\n\t\t\t\tType: jwt.Stream,\n\t\t\t\tLocalSubject: jwt.RenamingSubject(v.to),\n\t\t\t}\n\t\t}\n\t\timports = append(imports, imp)\n\t}\n\tfor _, sis := range a.imports.services {\n\t\tfor _, v := range sis {\n\t\t\timports = append(imports, newExtImport(v))\n\t\t}\n\t}\n\tresponses := map[string]ExtImport{}\n\tfor k, v := range a.exports.responses {\n\t\tresponses[k] = newExtImport(v)\n\t}\n\tmappings := ExtMap{}\n\tfor _, m := range a.mappings {\n\t\tvar dests []*MapDest\n\t\tvar src string\n\t\tif m == nil {\n\t\t\tsrc = \"nil\"\n\t\t\tif _, ok := mappings[src]; ok { // only set if not present (keep orig in case nil is used)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdests = append(dests, &MapDest{})\n\t\t} else {\n\t\t\tsrc = m.src\n\t\t\tfor _, d := range m.dests {\n\t\t\t\tdests = append(dests, &MapDest{d.tr.dest, d.weight, _EMPTY_})\n\t\t\t}\n\t\t\tfor c, cd := range m.cdests {\n\t\t\t\tfor _, d := range cd {\n\t\t\t\t\tdests = append(dests, &MapDest{d.tr.dest, d.weight, c})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tmappings[src] = dests\n\t}\n\treturn &AccountInfo{\n\t\tAccountName: accName,\n\t\tLastUpdate: a.updated.UTC(),\n\t\tIsSystem: isSys,\n\t\tExpired: a.expired.Load(),\n\t\tComplete: !a.incomplete,\n\t\tJetStream: a.js != nil,\n\t\tLeafCnt: a.numLocalLeafNodes(),\n\t\tClientCnt: a.numLocalConnections(),\n\t\tSubCnt: a.sl.Count(),\n\t\tMappings: mappings,\n\t\tExports: exports,\n\t\tImports: imports,\n\t\tJwt: a.claimJWT,\n\t\tIssuerKey: a.Issuer,\n\t\tNameTag: a.getNameTagLocked(),\n\t\tTags: a.tags,\n\t\tClaim: claim,\n\t\tVr: vrIssues,\n\t\tRevokedUser: collectRevocations(a.usersRevoked),\n\t\tSublist: a.sl.Stats(),\n\t\tResponses: responses,\n\t}, nil\n}\n\n// JSzOptions are options passed to Jsz\ntype JSzOptions struct {\n\tAccount string `json:\"account,omitempty\"`\n\tAccounts bool `json:\"accounts,omitempty\"`\n\tStreams bool `json:\"streams,omitempty\"`\n\tConsumer bool `json:\"consumer,omitempty\"`\n\tDirectConsumer bool `json:\"direct_consumer,omitempty\"`\n\tConfig bool `json:\"config,omitempty\"`\n\tLeaderOnly bool `json:\"leader_only,omitempty\"`\n\tOffset int `json:\"offset,omitempty\"`\n\tLimit int `json:\"limit,omitempty\"`\n\tRaftGroups bool `json:\"raft,omitempty\"`\n\tStreamLeaderOnly bool `json:\"stream_leader_only,omitempty\"`\n}\n\n// HealthzOptions are options passed to Healthz\ntype HealthzOptions struct {\n\t// Deprecated: Use JSEnabledOnly instead\n\tJSEnabled bool `json:\"js-enabled,omitempty\"`\n\tJSEnabledOnly bool `json:\"js-enabled-only,omitempty\"`\n\tJSServerOnly bool `json:\"js-server-only,omitempty\"`\n\tJSMetaOnly bool `json:\"js-meta-only,omitempty\"`\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream,omitempty\"`\n\tConsumer string `json:\"consumer,omitempty\"`\n\tDetails bool `json:\"details,omitempty\"`\n}\n\n// ProfilezOptions are options passed to Profilez\ntype ProfilezOptions struct {\n\tName string `json:\"name\"`\n\tDebug int `json:\"debug\"`\n\tDuration time.Duration `json:\"duration,omitempty\"`\n}\n\n// IpqueueszOptions are options passed to Ipqueuesz\ntype IpqueueszOptions struct {\n\tAll bool `json:\"all\"`\n\tFilter string `json:\"filter\"`\n}\n\n// RaftzOptions are options passed to Raftz\ntype RaftzOptions struct {\n\tAccountFilter string `json:\"account\"`\n\tGroupFilter string `json:\"group\"`\n}\n\n// StreamDetail shows information about the stream state and its consumers.\ntype StreamDetail struct {\n\tName string `json:\"name\"`\n\tCreated time.Time `json:\"created\"`\n\tCluster *ClusterInfo `json:\"cluster,omitempty\"`\n\tConfig *StreamConfig `json:\"config,omitempty\"`\n\tState StreamState `json:\"state,omitempty\"`\n\tConsumer []*ConsumerInfo `json:\"consumer_detail,omitempty\"`\n\tDirectConsumer []*ConsumerInfo `json:\"direct_consumer_detail,omitempty\"`\n\tMirror *StreamSourceInfo `json:\"mirror,omitempty\"`\n\tSources []*StreamSourceInfo `json:\"sources,omitempty\"`\n\tRaftGroup string `json:\"stream_raft_group,omitempty\"`\n\tConsumerRaftGroups []*RaftGroupDetail `json:\"consumer_raft_groups,omitempty\"`\n}\n\n// RaftGroupDetail shows information details about the Raft group.\ntype RaftGroupDetail struct {\n\tName string `json:\"name\"`\n\tRaftGroup string `json:\"raft_group,omitempty\"`\n}\n\ntype AccountDetail struct {\n\tName string `json:\"name\"`\n\tId string `json:\"id\"`\n\tJetStreamStats\n\tStreams []StreamDetail `json:\"stream_detail,omitempty\"`\n}\n\n// MetaSnapshotStats shows information about meta snapshots.\ntype MetaSnapshotStats struct {\n\tPendingEntries uint64 `json:\"pending_entries\"` // PendingEntries is the count of pending entries in the meta layer\n\tPendingSize uint64 `json:\"pending_size\"` // PendingSize is the size in bytes of pending entries in the meta layer\n\tLastTime time.Time `json:\"last_time,omitempty\"` // LastTime is when the last meta snapshot was taken\n\tLastDuration time.Duration `json:\"last_duration,omitempty\"` // LastDuration is how long the last meta snapshot took\n}\n\n// metaClusterSnapshotStats returns snapshot statistics for the meta group.\nfunc (s *Server) metaClusterSnapshotStats(js *jetStream, mg RaftNode) *MetaSnapshotStats {\n\tentries, bytes := mg.Size()\n\tsnap := &MetaSnapshotStats{\n\t\tPendingEntries: entries,\n\t\tPendingSize: bytes,\n\t}\n\n\tjs.mu.RLock()\n\tcluster := js.cluster\n\tjs.mu.RUnlock()\n\n\tif cluster != nil {\n\t\ttimeNanos := atomic.LoadInt64(&cluster.lastMetaSnapTime)\n\t\tdurationNanos := atomic.LoadInt64(&cluster.lastMetaSnapDuration)\n\t\tif timeNanos > 0 {\n\t\t\tsnap.LastTime = time.Unix(0, timeNanos).UTC()\n\t\t}\n\t\tif durationNanos > 0 {\n\t\t\tsnap.LastDuration = time.Duration(durationNanos)\n\t\t}\n\t}\n\n\treturn snap\n}\n\n// MetaClusterInfo shows information about the meta group.\ntype MetaClusterInfo struct {\n\tName string `json:\"name,omitempty\"` // Name is the name of the cluster\n\tLeader string `json:\"leader,omitempty\"` // Leader is the server name of the cluster leader\n\tPeer string `json:\"peer,omitempty\"` // Peer is unique ID of the leader\n\tReplicas []*PeerInfo `json:\"replicas,omitempty\"` // Replicas is a list of known peers\n\tSize int `json:\"cluster_size\"` // Size is the known size of the cluster\n\tPending int `json:\"pending\"` // Pending is how many RAFT messages are not yet processed\n\tPendingRequests int `json:\"pending_requests\"` // PendingRequests is how many CRUD operations are queued for processing\n\tPendingInfos int `json:\"pending_infos\"` // PendingInfos is how many info operations are queued for processing\n\tSnapshot *MetaSnapshotStats `json:\"snapshot\"` // Snapshot contains meta snapshot statistics\n}\n\n// JSInfo has detailed information on JetStream.\ntype JSInfo struct {\n\tJetStreamStats\n\tID string `json:\"server_id\"`\n\tNow time.Time `json:\"now\"`\n\tDisabled bool `json:\"disabled,omitempty\"`\n\tConfig JetStreamConfig `json:\"config,omitempty\"`\n\tLimits *JSLimitOpts `json:\"limits,omitempty\"`\n\tStreams int `json:\"streams\"`\n\tStreamsLeader int `json:\"streams_leader,omitempty\"`\n\tConsumers int `json:\"consumers\"`\n\tConsumersLeader int `json:\"consumers_leader,omitempty\"`\n\tMessages uint64 `json:\"messages\"`\n\tBytes uint64 `json:\"bytes\"`\n\tMeta *MetaClusterInfo `json:\"meta_cluster,omitempty\"`\n\tAccountDetails []*AccountDetail `json:\"account_details,omitempty\"`\n\tTotal int `json:\"total\"`\n}\n\nfunc (s *Server) accountDetail(jsa *jsAccount, optStreams, optConsumers, optDirectConsumers, optCfg, optRaft, optStreamLeader bool) *AccountDetail {\n\tjsa.mu.RLock()\n\tacc := jsa.account\n\tname := acc.GetName()\n\tid := name\n\tif acc.nameTag != _EMPTY_ {\n\t\tname = acc.nameTag\n\t}\n\tjsa.usageMu.RLock()\n\ttotalMem, totalStore := jsa.storageTotals()\n\tdetail := AccountDetail{\n\t\tName: name,\n\t\tId: id,\n\t\tJetStreamStats: JetStreamStats{\n\t\t\tMemory: totalMem,\n\t\t\tStore: totalStore,\n\t\t\tAPI: JetStreamAPIStats{\n\t\t\t\tTotal: jsa.apiTotal,\n\t\t\t\tErrors: jsa.apiErrors,\n\t\t\t},\n\t\t},\n\t\tStreams: make([]StreamDetail, 0, len(jsa.streams)),\n\t}\n\tif reserved, ok := jsa.limits[_EMPTY_]; ok {\n\t\tdetail.JetStreamStats.ReservedMemory = uint64(reserved.MaxMemory)\n\t\tdetail.JetStreamStats.ReservedStore = uint64(reserved.MaxStore)\n\t}\n\tjsa.usageMu.RUnlock()\n\n\tvar streams []*stream\n\tif optStreams {\n\t\tfor _, stream := range jsa.streams {\n\t\t\tstreams = append(streams, stream)\n\t\t}\n\t}\n\tjsa.mu.RUnlock()\n\n\tif js := s.getJetStream(); js != nil && optStreams {\n\t\tfor _, stream := range streams {\n\t\t\trgroup := stream.raftGroup()\n\t\t\tci := js.clusterInfo(rgroup)\n\t\t\tvar cfg *StreamConfig\n\t\t\tif optCfg {\n\t\t\t\tc := stream.config()\n\t\t\t\tcfg = &c\n\t\t\t}\n\t\t\t// Skip if we are only looking for stream leaders.\n\t\t\tif optStreamLeader && ci != nil && ci.Leader != s.Name() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsdet := StreamDetail{\n\t\t\t\tName: stream.name(),\n\t\t\t\tCreated: stream.createdTime(),\n\t\t\t\tState: stream.state(),\n\t\t\t\tCluster: ci,\n\t\t\t\tConfig: cfg,\n\t\t\t\tMirror: stream.mirrorInfo(),\n\t\t\t\tSources: stream.sourcesInfo(),\n\t\t\t}\n\t\t\tif optRaft && rgroup != nil {\n\t\t\t\tsdet.RaftGroup = rgroup.Name\n\t\t\t\tsdet.ConsumerRaftGroups = make([]*RaftGroupDetail, 0)\n\t\t\t}\n\t\t\tif optConsumers {\n\t\t\t\tfor _, consumer := range stream.getPublicConsumers() {\n\t\t\t\t\tcInfo := consumer.info()\n\t\t\t\t\tif cInfo == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif !optCfg {\n\t\t\t\t\t\tcInfo.Config = nil\n\t\t\t\t\t}\n\t\t\t\t\tsdet.Consumer = append(sdet.Consumer, cInfo)\n\t\t\t\t\tif optRaft {\n\t\t\t\t\t\tcrgroup := consumer.raftGroup()\n\t\t\t\t\t\tif crgroup != nil {\n\t\t\t\t\t\t\tsdet.ConsumerRaftGroups = append(sdet.ConsumerRaftGroups,\n\t\t\t\t\t\t\t\t&RaftGroupDetail{cInfo.Name, crgroup.Name},\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif optDirectConsumers {\n\t\t\t\t\tfor _, consumer := range stream.getDirectConsumers() {\n\t\t\t\t\t\tcInfo := consumer.info()\n\t\t\t\t\t\tif cInfo == nil {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif !optCfg {\n\t\t\t\t\t\t\tcInfo.Config = nil\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsdet.DirectConsumer = append(sdet.Consumer, cInfo)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tdetail.Streams = append(detail.Streams, sdet)\n\t\t}\n\t}\n\treturn &detail\n}\n\nfunc (s *Server) JszAccount(opts *JSzOptions) (*AccountDetail, error) {\n\tjs := s.getJetStream()\n\tif js == nil {\n\t\treturn nil, fmt.Errorf(\"jetstream not enabled\")\n\t}\n\tacc := opts.Account\n\taccount, ok := s.accounts.Load(acc)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"account %q not found\", acc)\n\t}\n\tjs.mu.RLock()\n\tjsa, ok := js.accounts[account.(*Account).Name]\n\tjs.mu.RUnlock()\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"account %q not jetstream enabled\", acc)\n\t}\n\treturn s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.DirectConsumer, opts.Config, opts.RaftGroups, opts.StreamLeaderOnly), nil\n}\n\n// helper to get cluster info from node via dummy group\nfunc (s *Server) raftNodeToClusterInfo(node RaftNode) *ClusterInfo {\n\tif node == nil {\n\t\treturn nil\n\t}\n\tpeers := node.Peers()\n\tpeerList := make([]string, len(peers))\n\tfor i, p := range peers {\n\t\tpeerList[i] = p.ID\n\t}\n\tgroup := &raftGroup{\n\t\tName: _EMPTY_,\n\t\tPeers: peerList,\n\t\tnode: node,\n\t}\n\treturn s.getJetStream().clusterInfo(group)\n}\n\n// Jsz returns a Jsz structure containing information about JetStream.\nfunc (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) {\n\t// set option defaults\n\tif opts == nil {\n\t\topts = &JSzOptions{}\n\t}\n\tif opts.Offset < 0 {\n\t\topts.Offset = 0\n\t}\n\tif opts.Limit == 0 {\n\t\topts.Limit = 1024\n\t}\n\tif opts.Consumer {\n\t\topts.Streams = true\n\t}\n\tif opts.Streams && opts.Account == _EMPTY_ {\n\t\topts.Accounts = true\n\t}\n\n\tjsi := &JSInfo{\n\t\tID: s.ID(),\n\t\tNow: time.Now().UTC(),\n\t}\n\n\tjs := s.getJetStream()\n\tif js == nil || !js.isEnabled() {\n\t\tif opts.LeaderOnly {\n\t\t\treturn nil, fmt.Errorf(\"%w: not leader\", errSkipZreq)\n\t\t}\n\n\t\tjsi.Disabled = true\n\t\treturn jsi, nil\n\t}\n\n\tjsi.Limits = &s.getOpts().JetStreamLimits\n\n\tjs.mu.RLock()\n\tisLeader := js.cluster == nil || js.cluster.isLeader()\n\tjs.mu.RUnlock()\n\n\tif opts.LeaderOnly && !isLeader {\n\t\treturn nil, fmt.Errorf(\"%w: not leader\", errSkipZreq)\n\t}\n\n\tvar accounts []*jsAccount\n\n\tjs.mu.RLock()\n\tjsi.Config = js.config\n\tfor _, info := range js.accounts {\n\t\taccounts = append(accounts, info)\n\t}\n\tjs.mu.RUnlock()\n\n\tjsi.Total = len(accounts)\n\n\tif mg := js.getMetaGroup(); mg != nil {\n\t\tif ci := s.raftNodeToClusterInfo(mg); ci != nil {\n\t\t\tjsi.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Size: mg.ClusterSize()}\n\t\t\tif isLeader {\n\t\t\t\tjsi.Meta.Replicas = ci.Replicas\n\t\t\t}\n\t\t\tif ipq := s.jsAPIRoutedReqs; ipq != nil {\n\t\t\t\tjsi.Meta.PendingRequests = ipq.len()\n\t\t\t}\n\t\t\tif ipq := s.jsAPIRoutedInfoReqs; ipq != nil {\n\t\t\t\tjsi.Meta.PendingInfos = ipq.len()\n\t\t\t}\n\t\t\tjsi.Meta.Pending = jsi.Meta.PendingRequests + jsi.Meta.PendingInfos\n\t\t\tjsi.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg)\n\t\t}\n\t}\n\n\tjsi.JetStreamStats = *js.usageStats()\n\n\t// If a specific account is requested, track the index.\n\tfilterIdx := -1\n\n\t// Calculate the stats of all accounts and streams regardless of the filtering.\n\tfor i, jsa := range accounts {\n\t\tif jsa.acc().GetName() == opts.Account {\n\t\t\tfilterIdx = i\n\t\t}\n\n\t\tjsa.mu.RLock()\n\t\tstreams := make([]*stream, 0, len(jsa.streams))\n\t\tfor _, stream := range jsa.streams {\n\t\t\tstreams = append(streams, stream)\n\t\t}\n\t\tjsa.mu.RUnlock()\n\n\t\tjsi.Streams += len(streams)\n\t\tfor _, stream := range streams {\n\t\t\tstreamState := stream.state()\n\t\t\tjsi.Messages += streamState.Msgs\n\t\t\tjsi.Bytes += streamState.Bytes\n\t\t\tjsi.Consumers += streamState.Consumers\n\t\t\tif opts.RaftGroups {\n\t\t\t\tif node := stream.raftNode(); node == nil || node.Leader() {\n\t\t\t\t\tjsi.StreamsLeader++\n\t\t\t\t}\n\t\t\t\tfor _, consumer := range stream.getPublicConsumers() {\n\t\t\t\t\tif node := consumer.raftNode(); node == nil || node.Leader() {\n\t\t\t\t\t\tjsi.ConsumersLeader++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Targeted account takes precedence.\n\tif filterIdx >= 0 {\n\t\taccounts = accounts[filterIdx : filterIdx+1]\n\t} else if opts.Accounts {\n\n\t\tif opts.Limit > 0 {\n\t\t\t// Sort by name for a consistent read (barring any concurrent changes)\n\t\t\tslices.SortFunc(accounts, func(i, j *jsAccount) int { return cmp.Compare(i.acc().Name, j.acc().Name) })\n\n\t\t\t// Offset larger than the number of accounts.\n\t\t\toffset := min(opts.Offset, len(accounts))\n\t\t\taccounts = accounts[offset:]\n\n\t\t\tlimit := min(opts.Limit, len(accounts))\n\t\t\taccounts = accounts[:limit]\n\t\t}\n\t} else {\n\t\taccounts = nil\n\t}\n\n\tif len(accounts) > 0 {\n\t\tjsi.AccountDetails = make([]*AccountDetail, 0, len(accounts))\n\n\t\tfor _, jsa := range accounts {\n\t\t\tdetail := s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.DirectConsumer, opts.Config, opts.RaftGroups, opts.StreamLeaderOnly)\n\t\t\tjsi.AccountDetails = append(jsi.AccountDetails, detail)\n\t\t}\n\t}\n\n\treturn jsi, nil\n}\n\n// HandleJsz process HTTP requests for jetstream information.\nfunc (s *Server) HandleJsz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[JszPath]++\n\ts.mu.Unlock()\n\taccounts, err := decodeBool(w, r, \"accounts\")\n\tif err != nil {\n\t\treturn\n\t}\n\tstreams, err := decodeBool(w, r, \"streams\")\n\tif err != nil {\n\t\treturn\n\t}\n\tconsumers, err := decodeBool(w, r, \"consumers\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdirectConsumers, err := decodeBool(w, r, \"direct-consumers\")\n\tif err != nil {\n\t\treturn\n\t}\n\tconfig, err := decodeBool(w, r, \"config\")\n\tif err != nil {\n\t\treturn\n\t}\n\toffset, err := decodeInt(w, r, \"offset\")\n\tif err != nil {\n\t\treturn\n\t}\n\tlimit, err := decodeInt(w, r, \"limit\")\n\tif err != nil {\n\t\treturn\n\t}\n\tleader, err := decodeBool(w, r, \"leader-only\")\n\tif err != nil {\n\t\treturn\n\t}\n\trgroups, err := decodeBool(w, r, \"raft\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\tsleader, err := decodeBool(w, r, \"stream-leader-only\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\tl, err := s.Jsz(&JSzOptions{\n\t\tAccount: r.URL.Query().Get(\"acc\"),\n\t\tAccounts: accounts,\n\t\tStreams: streams,\n\t\tConsumer: consumers,\n\t\tDirectConsumer: directConsumers,\n\t\tConfig: config,\n\t\tLeaderOnly: leader,\n\t\tOffset: offset,\n\t\tLimit: limit,\n\t\tRaftGroups: rgroups,\n\t\tStreamLeaderOnly: sleader,\n\t})\n\tif err != nil {\n\t\tw.WriteHeader(http.StatusBadRequest)\n\t\tw.Write([]byte(err.Error()))\n\t\treturn\n\t}\n\tb, err := json.MarshalIndent(l, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /jsz request: %v\", err)\n\t}\n\n\t// Handle response\n\tResponseHandler(w, r, b)\n}\n\ntype HealthStatus struct {\n\tStatus string `json:\"status\"`\n\tStatusCode int `json:\"status_code,omitempty\"`\n\tError string `json:\"error,omitempty\"`\n\tErrors []HealthzError `json:\"errors,omitempty\"`\n}\n\ntype HealthzError struct {\n\tType HealthZErrorType `json:\"type\"`\n\tAccount string `json:\"account,omitempty\"`\n\tStream string `json:\"stream,omitempty\"`\n\tConsumer string `json:\"consumer,omitempty\"`\n\tError string `json:\"error,omitempty\"`\n}\n\ntype HealthZErrorType int\n\nconst (\n\tHealthzErrorConn HealthZErrorType = iota\n\tHealthzErrorBadRequest\n\tHealthzErrorJetStream\n\tHealthzErrorAccount\n\tHealthzErrorStream\n\tHealthzErrorConsumer\n)\n\nfunc (t HealthZErrorType) String() string {\n\tswitch t {\n\tcase HealthzErrorConn:\n\t\treturn \"CONNECTION\"\n\tcase HealthzErrorBadRequest:\n\t\treturn \"BAD_REQUEST\"\n\tcase HealthzErrorJetStream:\n\t\treturn \"JETSTREAM\"\n\tcase HealthzErrorAccount:\n\t\treturn \"ACCOUNT\"\n\tcase HealthzErrorStream:\n\t\treturn \"STREAM\"\n\tcase HealthzErrorConsumer:\n\t\treturn \"CONSUMER\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\nfunc (t HealthZErrorType) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(t.String())\n}\n\nfunc (t *HealthZErrorType) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase `\"CONNECTION\"`:\n\t\t*t = HealthzErrorConn\n\tcase `\"BAD_REQUEST\"`:\n\t\t*t = HealthzErrorBadRequest\n\tcase `\"JETSTREAM\"`:\n\t\t*t = HealthzErrorJetStream\n\tcase `\"ACCOUNT\"`:\n\t\t*t = HealthzErrorAccount\n\tcase `\"STREAM\"`:\n\t\t*t = HealthzErrorStream\n\tcase `\"CONSUMER\"`:\n\t\t*t = HealthzErrorConsumer\n\tdefault:\n\t\treturn fmt.Errorf(\"unknown healthz error type %q\", data)\n\t}\n\treturn nil\n}\n\n// https://datatracker.ietf.org/doc/html/draft-inadarei-api-health-check\nfunc (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) {\n\ts.mu.Lock()\n\ts.httpReqStats[HealthzPath]++\n\ts.mu.Unlock()\n\n\tjsEnabled, err := decodeBool(w, r, \"js-enabled\")\n\tif err != nil {\n\t\treturn\n\t}\n\tif jsEnabled {\n\t\ts.Warnf(\"Healthcheck: js-enabled deprecated, use js-enabled-only instead\")\n\t}\n\tjsEnabledOnly, err := decodeBool(w, r, \"js-enabled-only\")\n\tif err != nil {\n\t\treturn\n\t}\n\tjsServerOnly, err := decodeBool(w, r, \"js-server-only\")\n\tif err != nil {\n\t\treturn\n\t}\n\tjsMetaOnly, err := decodeBool(w, r, \"js-meta-only\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\tincludeDetails, err := decodeBool(w, r, \"details\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\ths := s.healthz(&HealthzOptions{\n\t\tJSEnabled: jsEnabled,\n\t\tJSEnabledOnly: jsEnabledOnly,\n\t\tJSServerOnly: jsServerOnly,\n\t\tJSMetaOnly: jsMetaOnly,\n\t\tAccount: r.URL.Query().Get(\"account\"),\n\t\tStream: r.URL.Query().Get(\"stream\"),\n\t\tConsumer: r.URL.Query().Get(\"consumer\"),\n\t\tDetails: includeDetails,\n\t})\n\n\tcode := hs.StatusCode\n\tif hs.Error != _EMPTY_ {\n\t\ts.Warnf(\"Healthcheck failed: %q\", hs.Error)\n\t} else if len(hs.Errors) != 0 {\n\t\ts.Warnf(\"Healthcheck failed: %d errors\", len(hs.Errors))\n\t}\n\t// Remove StatusCode from JSON representation when responding via HTTP\n\t// since this is already in the response.\n\ths.StatusCode = 0\n\tb, err := json.Marshal(hs)\n\tif err != nil {\n\t\ts.Errorf(\"Error marshaling response to /healthz request: %v\", err)\n\t}\n\n\thandleResponse(code, w, r, b)\n}\n\n// Generate health status.\nfunc (s *Server) healthz(opts *HealthzOptions) *HealthStatus {\n\tvar health = &HealthStatus{Status: \"ok\"}\n\n\t// set option defaults\n\tif opts == nil {\n\t\topts = &HealthzOptions{}\n\t}\n\tdetails := opts.Details\n\tdefer func() {\n\t\t// for response with details enabled, set status to either \"error\" or \"ok\"\n\t\tif details {\n\t\t\tif len(health.Errors) != 0 {\n\t\t\t\thealth.Status = \"error\"\n\t\t\t} else {\n\t\t\t\thealth.Status = \"ok\"\n\t\t\t}\n\t\t}\n\t\t// if no specific status code was set, set it based on the presence of errors\n\t\tif health.StatusCode == 0 {\n\t\t\tif health.Error != _EMPTY_ || len(health.Errors) != 0 {\n\t\t\t\thealth.StatusCode = http.StatusServiceUnavailable\n\t\t\t} else {\n\t\t\t\thealth.StatusCode = http.StatusOK\n\t\t\t}\n\t\t}\n\t}()\n\n\tif opts.Account == _EMPTY_ && opts.Stream != _EMPTY_ {\n\t\thealth.StatusCode = http.StatusBadRequest\n\t\tif !details {\n\t\t\thealth.Status = \"error\"\n\t\t\thealth.Error = fmt.Sprintf(\"%q must not be empty when checking stream health\", \"account\")\n\t\t} else {\n\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\tType: HealthzErrorBadRequest,\n\t\t\t\tError: fmt.Sprintf(\"%q must not be empty when checking stream health\", \"account\"),\n\t\t\t})\n\t\t}\n\t\treturn health\n\t}\n\n\tif opts.Stream == _EMPTY_ && opts.Consumer != _EMPTY_ {\n\t\thealth.StatusCode = http.StatusBadRequest\n\t\tif !details {\n\t\t\thealth.Status = \"error\"\n\t\t\thealth.Error = fmt.Sprintf(\"%q must not be empty when checking consumer health\", \"stream\")\n\t\t} else {\n\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\tType: HealthzErrorBadRequest,\n\t\t\t\tError: fmt.Sprintf(\"%q must not be empty when checking consumer health\", \"stream\"),\n\t\t\t})\n\t\t}\n\t\treturn health\n\t}\n\n\tif err := s.readyForConnections(time.Millisecond); err != nil {\n\t\thealth.StatusCode = http.StatusInternalServerError\n\t\thealth.Status = \"error\"\n\t\tif !details {\n\t\t\thealth.Error = err.Error()\n\t\t} else {\n\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\tType: HealthzErrorConn,\n\t\t\t\tError: err.Error(),\n\t\t\t})\n\t\t}\n\t\treturn health\n\t}\n\n\t// If JSServerOnly is true, then do not check further accounts, streams and consumers.\n\tif opts.JSServerOnly {\n\t\treturn health\n\t}\n\n\tsopts := s.getOpts()\n\n\t// If JS is not enabled in the config, we stop.\n\tif !sopts.JetStream {\n\t\treturn health\n\t}\n\n\t// Access the Jetstream state to perform additional checks.\n\tjs := s.getJetStream()\n\tconst na = \"unavailable\"\n\tif !js.isEnabled() {\n\t\thealth.StatusCode = http.StatusServiceUnavailable\n\t\thealth.Status = na\n\t\tif !details {\n\t\t\thealth.Error = NewJSNotEnabledError().Error()\n\t\t} else {\n\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\tType: HealthzErrorJetStream,\n\t\t\t\tError: NewJSNotEnabledError().Error(),\n\t\t\t})\n\t\t}\n\t\treturn health\n\t}\n\t// Only check if JS is enabled, skip meta and asset check.\n\tif opts.JSEnabledOnly || opts.JSEnabled {\n\t\treturn health\n\t}\n\n\t// Clustered JetStream\n\tjs.mu.RLock()\n\tcc := js.cluster\n\tjs.mu.RUnlock()\n\n\t// Currently single server we make sure the streams were recovered.\n\tif cc == nil {\n\t\tsdir := js.config.StoreDir\n\t\t// Whip through account folders and pull each stream name.\n\t\tfis, _ := os.ReadDir(sdir)\n\t\tvar accFound, streamFound, consumerFound bool\n\t\tfor _, fi := range fis {\n\t\t\tif fi.Name() == snapStagingDir {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif opts.Account != _EMPTY_ {\n\t\t\t\tif fi.Name() != opts.Account {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\taccFound = true\n\t\t\t}\n\t\t\tacc, err := s.LookupAccount(fi.Name())\n\t\t\tif err != nil {\n\t\t\t\tif !details {\n\t\t\t\t\thealth.Status = na\n\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream account '%s' could not be resolved\", fi.Name())\n\t\t\t\t\treturn health\n\t\t\t\t}\n\t\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\t\tAccount: fi.Name(),\n\t\t\t\t\tError: fmt.Sprintf(\"JetStream account '%s' could not be resolved\", fi.Name()),\n\t\t\t\t})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsfis, _ := os.ReadDir(filepath.Join(sdir, fi.Name(), \"streams\"))\n\t\t\tfor _, sfi := range sfis {\n\t\t\t\tif opts.Stream != _EMPTY_ {\n\t\t\t\t\tif sfi.Name() != opts.Stream {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tstreamFound = true\n\t\t\t\t}\n\t\t\t\tstream := sfi.Name()\n\t\t\t\ts, err := acc.lookupStream(stream)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif !details {\n\t\t\t\t\t\thealth.Status = na\n\t\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream stream '%s > %s' could not be recovered\", acc, stream)\n\t\t\t\t\t\treturn health\n\t\t\t\t\t}\n\t\t\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\tAccount: acc.Name,\n\t\t\t\t\t\tStream: stream,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream stream '%s > %s' could not be recovered\", acc, stream),\n\t\t\t\t\t})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif streamWerr := s.getWriteErr(); streamWerr != nil {\n\t\t\t\t\tif !details {\n\t\t\t\t\t\thealth.Status = na\n\t\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream stream '%s > %s' write error: %v\", acc, stream, streamWerr)\n\t\t\t\t\t\treturn health\n\t\t\t\t\t}\n\t\t\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\tAccount: acc.Name,\n\t\t\t\t\t\tStream: stream,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream stream '%s > %s' write error: %v\", acc, stream, streamWerr),\n\t\t\t\t\t})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif streamFound {\n\t\t\t\t\t// if consumer option is passed, verify that the consumer exists on stream\n\t\t\t\t\tif opts.Consumer != _EMPTY_ {\n\t\t\t\t\t\tfor _, cons := range s.consumers {\n\t\t\t\t\t\t\tif cons.name == opts.Consumer {\n\t\t\t\t\t\t\t\tconsumerFound = true\n\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif accFound {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif opts.Account != _EMPTY_ && !accFound {\n\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\tif !details {\n\t\t\t\thealth.Status = na\n\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream account %q not found\", opts.Account)\n\t\t\t} else {\n\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream account %q not found\", opts.Account),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn health\n\t\t}\n\t\tif opts.Stream != _EMPTY_ && !streamFound {\n\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\tif !details {\n\t\t\t\thealth.Status = na\n\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream stream %q not found on account %q\", opts.Stream, opts.Account)\n\t\t\t} else {\n\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\tStream: opts.Stream,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream stream %q not found on account %q\", opts.Stream, opts.Account),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn health\n\t\t}\n\t\tif opts.Consumer != _EMPTY_ && !consumerFound {\n\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\tif !details {\n\t\t\t\thealth.Status = na\n\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream consumer %q not found for stream %q on account %q\", opts.Consumer, opts.Stream, opts.Account)\n\t\t\t} else {\n\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorConsumer,\n\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\tStream: opts.Stream,\n\t\t\t\t\t\tConsumer: opts.Consumer,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream consumer %q not found for stream %q on account %q\", opts.Consumer, opts.Stream, opts.Account),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn health\n\t}\n\n\t// If we are here we want to check for any assets assigned to us.\n\tvar meta RaftNode\n\tjs.mu.RLock()\n\tmeta = cc.meta\n\tjs.mu.RUnlock()\n\n\t// Check meta layer health.\n\tvar metaNoLeader, metaClosed, metaUnhealthy bool\n\tvar metaWerr error\n\tif meta != nil {\n\t\tmetaNoLeader = meta.GroupLeader() == _EMPTY_\n\t\tmetaClosed = meta.State() == Closed\n\t\tmetaUnhealthy = !meta.Healthy()\n\t\tmetaWerr = meta.GetWriteErr()\n\t}\n\tmetaRecovering := js.isMetaRecovering()\n\tif meta == nil || metaNoLeader || metaClosed || metaUnhealthy || metaWerr != nil || metaRecovering {\n\t\tvar desc string\n\t\tif metaWerr != nil {\n\t\t\tdesc = fmt.Sprintf(\"JetStream meta layer write error: %v\", metaWerr)\n\t\t} else if metaClosed {\n\t\t\tdesc = \"JetStream meta layer is not running\"\n\t\t} else if meta != nil && metaRecovering {\n\t\t\tdesc = \"JetStream is still recovering meta layer\"\n\t\t} else if meta == nil || metaNoLeader {\n\t\t\tdesc = \"JetStream has not established contact with a meta leader\"\n\t\t} else {\n\t\t\tdesc = \"JetStream is not current with the meta leader\"\n\t\t}\n\t\tif !details {\n\t\t\thealth.Status = na\n\t\t\thealth.Error = desc\n\t\t} else {\n\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t{\n\t\t\t\t\tType: HealthzErrorJetStream,\n\t\t\t\t\tError: desc,\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\t\treturn health\n\t}\n\n\t// Skips doing full healthz and only checks the meta leader.\n\tif opts.JSMetaOnly {\n\t\treturn health\n\t}\n\n\t// Range across all accounts, the streams assigned to them, and the consumers.\n\t// If they are assigned to this server check their status.\n\tourID := meta.ID()\n\n\t// Copy the meta layer so we do not need to hold the js read lock for an extended period of time.\n\tvar streams map[string]map[string]*streamAssignment\n\tjs.mu.RLock()\n\tif opts.Account == _EMPTY_ {\n\t\t// Collect all relevant streams and consumers.\n\t\tstreams = make(map[string]map[string]*streamAssignment, len(cc.streams))\n\t\tfor acc, asa := range cc.streams {\n\t\t\tnasa := make(map[string]*streamAssignment)\n\t\t\tfor stream, sa := range asa {\n\t\t\t\t// If we are a member and we are not being restored, select for check.\n\t\t\t\tif sa.Group.isMember(ourID) && sa.Restore == nil {\n\t\t\t\t\tcsa := sa.copyGroup()\n\t\t\t\t\tcsa.consumers = make(map[string]*consumerAssignment)\n\t\t\t\t\tfor consumer, ca := range sa.consumers {\n\t\t\t\t\t\tif ca.Group.isMember(ourID) {\n\t\t\t\t\t\t\t// Use original here. Not a copy.\n\t\t\t\t\t\t\tcsa.consumers[consumer] = ca\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tnasa[stream] = csa\n\t\t\t\t}\n\t\t\t}\n\t\t\tstreams[acc] = nasa\n\t\t}\n\t} else {\n\t\tstreams = make(map[string]map[string]*streamAssignment, 1)\n\t\tasa, ok := cc.streams[opts.Account]\n\t\tif !ok {\n\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\tif !details {\n\t\t\t\thealth.Status = na\n\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream account %q not found\", opts.Account)\n\t\t\t} else {\n\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t{\n\t\t\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream account %q not found\", opts.Account),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t}\n\t\t\tjs.mu.RUnlock()\n\t\t\treturn health\n\t\t}\n\t\tnasa := make(map[string]*streamAssignment)\n\t\tif opts.Stream != _EMPTY_ {\n\t\t\tsa, ok := asa[opts.Stream]\n\t\t\tif !ok || !sa.Group.isMember(ourID) {\n\t\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\t\tif !details {\n\t\t\t\t\thealth.Status = na\n\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream stream %q not found on account %q\", opts.Stream, opts.Account)\n\t\t\t\t} else {\n\t\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\t\tStream: opts.Stream,\n\t\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream stream %q not found on account %q\", opts.Stream, opts.Account),\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\treturn health\n\t\t\t}\n\t\t\tcsa := sa.copyGroup()\n\t\t\tcsa.consumers = make(map[string]*consumerAssignment)\n\t\t\tvar consumerFound bool\n\t\t\tfor consumer, ca := range sa.consumers {\n\t\t\t\tif opts.Consumer != _EMPTY_ {\n\t\t\t\t\tif consumer != opts.Consumer || !ca.Group.isMember(ourID) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tconsumerFound = true\n\t\t\t\t}\n\t\t\t\t// If we are a member and we are not being restored, select for check.\n\t\t\t\tif sa.Group.isMember(ourID) && sa.Restore == nil {\n\t\t\t\t\tcsa.consumers[consumer] = ca\n\t\t\t\t}\n\t\t\t\tif consumerFound {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif opts.Consumer != _EMPTY_ && !consumerFound {\n\t\t\t\thealth.StatusCode = http.StatusNotFound\n\t\t\t\tif !details {\n\t\t\t\t\thealth.Status = na\n\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream consumer %q not found for stream %q on account %q\", opts.Consumer, opts.Stream, opts.Account)\n\t\t\t\t} else {\n\t\t\t\t\thealth.Errors = []HealthzError{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tType: HealthzErrorConsumer,\n\t\t\t\t\t\t\tAccount: opts.Account,\n\t\t\t\t\t\t\tStream: opts.Stream,\n\t\t\t\t\t\t\tConsumer: opts.Consumer,\n\t\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream consumer %q not found for stream %q on account %q\", opts.Consumer, opts.Stream, opts.Account),\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t\treturn health\n\t\t\t}\n\t\t\tnasa[opts.Stream] = csa\n\t\t} else {\n\t\t\tfor stream, sa := range asa {\n\t\t\t\t// If we are a member and we are not being restored, select for check.\n\t\t\t\tif sa.Group.isMember(ourID) && sa.Restore == nil {\n\t\t\t\t\tcsa := sa.copyGroup()\n\t\t\t\t\tcsa.consumers = make(map[string]*consumerAssignment)\n\t\t\t\t\tfor consumer, ca := range sa.consumers {\n\t\t\t\t\t\tif ca.Group.isMember(ourID) {\n\t\t\t\t\t\t\tcsa.consumers[consumer] = ca\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tnasa[stream] = csa\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tstreams[opts.Account] = nasa\n\t}\n\tjs.mu.RUnlock()\n\n\t// Use our copy to traverse so we do not need to hold the js lock.\n\tfor accName, asa := range streams {\n\t\tacc, err := s.LookupAccount(accName)\n\t\tif err != nil && len(asa) > 0 {\n\t\t\tif !details {\n\t\t\t\thealth.Status = na\n\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream can not lookup account %q: %v\", accName, err)\n\t\t\t\treturn health\n\t\t\t}\n\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\tType: HealthzErrorAccount,\n\t\t\t\tAccount: accName,\n\t\t\t\tError: fmt.Sprintf(\"JetStream can not lookup account %q: %v\", accName, err),\n\t\t\t})\n\t\t\tcontinue\n\t\t}\n\n\t\tfor stream, sa := range asa {\n\t\t\tif sa != nil && sa.unsupported != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Make sure we can look up\n\t\t\tif err := js.isStreamHealthy(acc, sa); err != nil {\n\t\t\t\tif !details {\n\t\t\t\t\thealth.Status = na\n\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream stream '%s > %s' is not current: %s\", accName, stream, err)\n\t\t\t\t\treturn health\n\t\t\t\t}\n\t\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\t\tType: HealthzErrorStream,\n\t\t\t\t\tAccount: accName,\n\t\t\t\t\tStream: stream,\n\t\t\t\t\tError: fmt.Sprintf(\"JetStream stream '%s > %s' is not current: %s\", accName, stream, err),\n\t\t\t\t})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmset, _ := acc.lookupStream(stream)\n\t\t\t// Now check consumers.\n\t\t\tfor consumer, ca := range sa.consumers {\n\t\t\t\tif err := js.isConsumerHealthy(mset, consumer, ca); err != nil {\n\t\t\t\t\tif !details {\n\t\t\t\t\t\thealth.Status = na\n\t\t\t\t\t\thealth.Error = fmt.Sprintf(\"JetStream consumer '%s > %s > %s' is not current: %s\", acc, stream, consumer, err)\n\t\t\t\t\t\treturn health\n\t\t\t\t\t}\n\t\t\t\t\thealth.Errors = append(health.Errors, HealthzError{\n\t\t\t\t\t\tType: HealthzErrorConsumer,\n\t\t\t\t\t\tAccount: accName,\n\t\t\t\t\t\tStream: stream,\n\t\t\t\t\t\tConsumer: consumer,\n\t\t\t\t\t\tError: fmt.Sprintf(\"JetStream consumer '%s > %s > %s' is not current: %s\", acc, stream, consumer, err),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// Success.\n\treturn health\n}\n\n// Healthz returns the health status of the server.\nfunc (s *Server) Healthz(opts *HealthzOptions) *HealthStatus {\n\treturn s.healthz(opts)\n}\n\ntype ExpvarzStatus struct {\n\tMemstats json.RawMessage `json:\"memstats\"`\n\tCmdline json.RawMessage `json:\"cmdline\"`\n}\n\nfunc (s *Server) expvarz(_ *ExpvarzEventOptions) *ExpvarzStatus {\n\tvar stat ExpvarzStatus\n\n\tconst memStatsKey = \"memstats\"\n\tconst cmdLineKey = \"cmdline\"\n\n\texpvar.Do(func(v expvar.KeyValue) {\n\t\tswitch v.Key {\n\t\tcase memStatsKey:\n\t\t\tstat.Memstats = json.RawMessage(v.Value.String())\n\n\t\tcase cmdLineKey:\n\t\t\tstat.Cmdline = json.RawMessage(v.Value.String())\n\t\t}\n\t})\n\n\treturn &stat\n}\n\ntype ProfilezStatus struct {\n\tProfile []byte `json:\"profile\"`\n\tError string `json:\"error\"`\n}\n\nfunc (s *Server) profilez(opts *ProfilezOptions) *ProfilezStatus {\n\tvar buffer bytes.Buffer\n\tswitch opts.Name {\n\tcase _EMPTY_:\n\t\treturn &ProfilezStatus{\n\t\t\tError: \"Profile name not specified\",\n\t\t}\n\tcase \"cpu\":\n\t\tif opts.Duration <= 0 || opts.Duration > 15*time.Second {\n\t\t\treturn &ProfilezStatus{\n\t\t\t\tError: fmt.Sprintf(\"Duration %s should be between 0s and 15s\", opts.Duration),\n\t\t\t}\n\t\t}\n\t\tif err := pprof.StartCPUProfile(&buffer); err != nil {\n\t\t\treturn &ProfilezStatus{\n\t\t\t\tError: fmt.Sprintf(\"Failed to start CPU profile: %s\", err),\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(opts.Duration)\n\t\tpprof.StopCPUProfile()\n\tdefault:\n\t\tprofile := pprof.Lookup(opts.Name)\n\t\tif profile == nil {\n\t\t\treturn &ProfilezStatus{\n\t\t\t\tError: fmt.Sprintf(\"Profile %q not found\", opts.Name),\n\t\t\t}\n\t\t}\n\t\tif err := profile.WriteTo(&buffer, opts.Debug); err != nil {\n\t\t\treturn &ProfilezStatus{\n\t\t\t\tError: fmt.Sprintf(\"Profile %q error: %s\", opts.Name, err),\n\t\t\t}\n\t\t}\n\t}\n\treturn &ProfilezStatus{\n\t\tProfile: buffer.Bytes(),\n\t}\n}\n\ntype RaftzGroup struct {\n\tID string `json:\"id\"`\n\tState string `json:\"state\"`\n\tSize int `json:\"size\"`\n\tQuorumNeeded int `json:\"quorum_needed\"`\n\tObserver bool `json:\"observer,omitempty\"`\n\tPaused bool `json:\"paused,omitempty\"`\n\tCommitted uint64 `json:\"committed\"`\n\tApplied uint64 `json:\"applied\"`\n\tCatchingUp bool `json:\"catching_up,omitempty\"`\n\tLeader string `json:\"leader,omitempty\"`\n\tLeaderSince *time.Time `json:\"leader_since,omitempty\"`\n\tEverHadLeader bool `json:\"ever_had_leader\"`\n\tTerm uint64 `json:\"term\"`\n\tVote string `json:\"voted_for,omitempty\"`\n\tPTerm uint64 `json:\"pterm\"`\n\tPIndex uint64 `json:\"pindex\"`\n\tSystemAcc bool `json:\"system_account\"`\n\tTrafficAcc string `json:\"traffic_account\"`\n\tIPQPropLen int `json:\"ipq_proposal_len\"`\n\tIPQEntryLen int `json:\"ipq_entry_len\"`\n\tIPQRespLen int `json:\"ipq_resp_len\"`\n\tIPQApplyLen int `json:\"ipq_apply_len\"`\n\tWAL StreamState `json:\"wal\"`\n\tWALError error `json:\"wal_error,omitempty\"`\n\tPeers map[string]RaftzGroupPeer `json:\"peers\"`\n}\n\ntype RaftzGroupPeer struct {\n\tName string `json:\"name\"`\n\tKnown bool `json:\"known\"`\n\tLastReplicatedIndex uint64 `json:\"last_replicated_index,omitempty\"`\n\tLastSeen string `json:\"last_seen,omitempty\"`\n}\n\ntype RaftzStatus map[string]map[string]RaftzGroup\n\nfunc (s *Server) HandleRaftz(w http.ResponseWriter, r *http.Request) {\n\tif s.raftNodes == nil {\n\t\tw.WriteHeader(404)\n\t\tw.Write([]byte(\"No Raft nodes registered\"))\n\t\treturn\n\t}\n\n\tgroups := s.Raftz(&RaftzOptions{\n\t\tAccountFilter: r.URL.Query().Get(\"acc\"),\n\t\tGroupFilter: r.URL.Query().Get(\"group\"),\n\t})\n\n\tif groups == nil {\n\t\tw.WriteHeader(404)\n\t\tw.Write([]byte(\"No Raft nodes returned, check supplied filters\"))\n\t\treturn\n\t}\n\n\tb, _ := json.MarshalIndent(groups, \"\", \" \")\n\tResponseHandler(w, r, b)\n}\n\nfunc (s *Server) Raftz(opts *RaftzOptions) *RaftzStatus {\n\tafilter, gfilter := opts.AccountFilter, opts.GroupFilter\n\n\tif afilter == _EMPTY_ {\n\t\tif sys := s.SystemAccount(); sys != nil {\n\t\t\tafilter = sys.Name\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tgroups := map[string]RaftNode{}\n\tinfos := RaftzStatus{} // account -> group ID\n\n\ts.rnMu.RLock()\n\tif gfilter != _EMPTY_ {\n\t\tif rg, ok := s.raftNodes[gfilter]; ok && rg != nil {\n\t\t\tif n, ok := rg.(*raft); ok {\n\t\t\t\tif n.accName == afilter {\n\t\t\t\t\tgroups[gfilter] = rg\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfor name, rg := range s.raftNodes {\n\t\t\tif rg == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif n, ok := rg.(*raft); ok {\n\t\t\t\tif n.accName != afilter {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tgroups[name] = rg\n\t\t\t}\n\t\t}\n\t}\n\ts.rnMu.RUnlock()\n\n\tfor name, rg := range groups {\n\t\tn, ok := rg.(*raft)\n\t\tif n == nil || !ok {\n\t\t\tcontinue\n\t\t}\n\t\tif _, ok := infos[n.accName]; !ok {\n\t\t\tinfos[n.accName] = map[string]RaftzGroup{}\n\t\t}\n\t\t// Only take the lock once, using the public RaftNode functions would\n\t\t// cause us to take and release the locks over and over again.\n\t\tn.RLock()\n\t\tinfo := RaftzGroup{\n\t\t\tID: n.id,\n\t\t\tState: RaftState(n.state.Load()).String(),\n\t\t\tSize: n.csz,\n\t\t\tQuorumNeeded: n.qn,\n\t\t\tObserver: n.observer,\n\t\t\tPaused: n.paused,\n\t\t\tCommitted: n.commit,\n\t\t\tApplied: n.applied,\n\t\t\tCatchingUp: n.catchup != nil,\n\t\t\tLeader: n.leader,\n\t\t\tLeaderSince: n.leaderSince.Load(),\n\t\t\tEverHadLeader: n.pleader.Load(),\n\t\t\tTerm: n.term,\n\t\t\tVote: n.vote,\n\t\t\tPTerm: n.pterm,\n\t\t\tPIndex: n.pindex,\n\t\t\tSystemAcc: n.IsSystemAccount(),\n\t\t\tTrafficAcc: n.acc.GetName(),\n\t\t\tIPQPropLen: n.prop.len(),\n\t\t\tIPQEntryLen: n.entry.len(),\n\t\t\tIPQRespLen: n.resp.len(),\n\t\t\tIPQApplyLen: n.apply.len(),\n\t\t\tWALError: n.werr,\n\t\t\tPeers: map[string]RaftzGroupPeer{},\n\t\t}\n\t\tn.wal.FastState(&info.WAL)\n\t\tfor id, p := range n.peers {\n\t\t\tif id == n.id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpeer := RaftzGroupPeer{\n\t\t\t\tName: s.serverNameForNode(id),\n\t\t\t\tKnown: p.kp,\n\t\t\t\tLastReplicatedIndex: p.li,\n\t\t\t}\n\t\t\tif !p.ts.IsZero() {\n\t\t\t\tpeer.LastSeen = time.Since(p.ts).String()\n\t\t\t}\n\t\t\tinfo.Peers[id] = peer\n\t\t}\n\t\tn.RUnlock()\n\t\tinfos[n.accName][name] = info\n\t}\n\n\treturn &infos\n}\n" }, { "path": "server/monitor_sort_opts.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"time\"\n)\n\n// ConnInfos represents a connection info list. We use pointers since it will be sorted.\ntype ConnInfos []*ConnInfo\n\n// For sorting\n// Len returns length for sorting.\nfunc (cl ConnInfos) Len() int { return len(cl) }\n\n// Swap will sawap the elements.\nfunc (cl ConnInfos) Swap(i, j int) { cl[i], cl[j] = cl[j], cl[i] }\n\n// SortOpt is a helper type to sort clients\ntype SortOpt string\n\n// Possible sort options\nconst (\n\tByCid SortOpt = \"cid\" // By connection ID\n\tByStart SortOpt = \"start\" // By connection start time, same as CID\n\tBySubs SortOpt = \"subs\" // By number of subscriptions\n\tByPending SortOpt = \"pending\" // By amount of data in bytes waiting to be sent to client\n\tByOutMsgs SortOpt = \"msgs_to\" // By number of messages sent\n\tByInMsgs SortOpt = \"msgs_from\" // By number of messages received\n\tByOutBytes SortOpt = \"bytes_to\" // By amount of bytes sent\n\tByInBytes SortOpt = \"bytes_from\" // By amount of bytes received\n\tByLast SortOpt = \"last\" // By the last activity\n\tByIdle SortOpt = \"idle\" // By the amount of inactivity\n\tByUptime SortOpt = \"uptime\" // By the amount of time connections exist\n\tByStop SortOpt = \"stop\" // By the stop time for a closed connection\n\tByReason SortOpt = \"reason\" // By the reason for a closed connection\n\tByRTT SortOpt = \"rtt\" // By the round trip time\n)\n\n// Individual sort options provide the Less for sort.Interface. Len and Swap are on cList.\n// CID\ntype SortByCid struct{ ConnInfos }\n\nfunc (l SortByCid) Less(i, j int) bool { return l.ConnInfos[i].Cid < l.ConnInfos[j].Cid }\n\n// Number of Subscriptions\ntype SortBySubs struct{ ConnInfos }\n\nfunc (l SortBySubs) Less(i, j int) bool { return l.ConnInfos[i].NumSubs < l.ConnInfos[j].NumSubs }\n\n// Pending Bytes\ntype SortByPending struct{ ConnInfos }\n\nfunc (l SortByPending) Less(i, j int) bool { return l.ConnInfos[i].Pending < l.ConnInfos[j].Pending }\n\n// Outbound Msgs\ntype SortByOutMsgs struct{ ConnInfos }\n\nfunc (l SortByOutMsgs) Less(i, j int) bool { return l.ConnInfos[i].OutMsgs < l.ConnInfos[j].OutMsgs }\n\n// Inbound Msgs\ntype SortByInMsgs struct{ ConnInfos }\n\nfunc (l SortByInMsgs) Less(i, j int) bool { return l.ConnInfos[i].InMsgs < l.ConnInfos[j].InMsgs }\n\n// Outbound Bytes\ntype SortByOutBytes struct{ ConnInfos }\n\nfunc (l SortByOutBytes) Less(i, j int) bool { return l.ConnInfos[i].OutBytes < l.ConnInfos[j].OutBytes }\n\n// Inbound Bytes\ntype SortByInBytes struct{ ConnInfos }\n\nfunc (l SortByInBytes) Less(i, j int) bool { return l.ConnInfos[i].InBytes < l.ConnInfos[j].InBytes }\n\n// Last Activity\ntype SortByLast struct{ ConnInfos }\n\nfunc (l SortByLast) Less(i, j int) bool {\n\treturn l.ConnInfos[i].LastActivity.UnixNano() < l.ConnInfos[j].LastActivity.UnixNano()\n}\n\n// Idle time\ntype SortByIdle struct {\n\tConnInfos\n\tnow time.Time\n}\n\nfunc (l SortByIdle) Less(i, j int) bool {\n\treturn l.now.Sub(l.ConnInfos[i].LastActivity) < l.now.Sub(l.ConnInfos[j].LastActivity)\n}\n\n// Uptime\ntype SortByUptime struct {\n\tConnInfos\n\tnow time.Time\n}\n\nfunc (l SortByUptime) Less(i, j int) bool {\n\tci := l.ConnInfos[i]\n\tcj := l.ConnInfos[j]\n\tvar upi, upj time.Duration\n\tif ci.Stop == nil || ci.Stop.IsZero() {\n\t\tupi = l.now.Sub(ci.Start)\n\t} else {\n\t\tupi = ci.Stop.Sub(ci.Start)\n\t}\n\tif cj.Stop == nil || cj.Stop.IsZero() {\n\t\tupj = l.now.Sub(cj.Start)\n\t} else {\n\t\tupj = cj.Stop.Sub(cj.Start)\n\t}\n\treturn upi < upj\n}\n\n// Stop\ntype SortByStop struct{ ConnInfos }\n\nfunc (l SortByStop) Less(i, j int) bool {\n\tciStop := l.ConnInfos[i].Stop\n\tcjStop := l.ConnInfos[j].Stop\n\treturn ciStop.Before(*cjStop)\n}\n\n// Reason\ntype SortByReason struct{ ConnInfos }\n\nfunc (l SortByReason) Less(i, j int) bool {\n\treturn l.ConnInfos[i].Reason < l.ConnInfos[j].Reason\n}\n\n// RTT - Default is descending\ntype SortByRTT struct{ ConnInfos }\n\nfunc (l SortByRTT) Less(i, j int) bool { return l.ConnInfos[i].rtt < l.ConnInfos[j].rtt }\n\n// IsValid determines if a sort option is valid\nfunc (s SortOpt) IsValid() bool {\n\tswitch s {\n\tcase _EMPTY_, ByCid, ByStart, BySubs, ByPending, ByOutMsgs, ByInMsgs, ByOutBytes, ByInBytes, ByLast, ByIdle, ByUptime, ByStop, ByReason, ByRTT:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n" }, { "path": "server/monitor_test.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nconst CLIENT_PORT = -1\nconst MONITOR_PORT = -1\nconst CLUSTER_PORT = -1\n\nfunc DefaultMonitorOptions() *Options {\n\treturn &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: CLIENT_PORT,\n\t\tHTTPHost: \"127.0.0.1\",\n\t\tHTTPPort: MONITOR_PORT,\n\t\tHTTPBasePath: \"/\",\n\t\tServerName: \"monitor_server\",\n\t\tNoLog: true,\n\t\tNoSigs: true,\n\t\tTags: []string{\"tag\"},\n\t\tMetadata: map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"},\n\t}\n}\n\nfunc runMonitorJSServer(t *testing.T, clientPort int, monitorPort int, clusterPort int, routePort int) (*Server, *Options) {\n\tresetPreviousHTTPConnections()\n\ttmpDir := t.TempDir()\n\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:%d\n\t\thttp: 127.0.0.1:%d\n\t\tsystem_account: SYS\n\t\taccounts {\n\t\t\tSYS {\n\t\t\t\tusers [{user: sys, password: pwd}]\n\t\t\t}\n\t\t\tACC {\n\t\t\t\tusers [{user: usr, password: pwd}]\n\t\t\t\t// In clustered mode, these reservations will not impact any one server.\n\t\t\t\tjetstream: {max_store: 4Mb, max_memory: 5Mb}\n\t\t\t}\n\t\t\tBCC_TO_HAVE_ONE_EXTRA {\n\t\t\t\tusers [{user: usr2, password: pwd}]\n\t\t\t\tjetstream: enabled\n\t\t\t}\n\t\t}\n\t\tjetstream: {\n\t\t\tmax_mem_store: 10Mb\n\t\t\tmax_file_store: 10Mb\n\t\t\tstore_dir: '%s'\n\t\t\tunique_tag: az\n\t\t\tlimits: {\n\t\t\t\tmax_ha_assets: 1000\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tname: cluster_name\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes: [nats-route://127.0.0.1:%d]\n\t\t}\n\t\tserver_name: server_%d\n\t\tserver_tags: [ \"az:%d\", \"tag\" ] `, clientPort, monitorPort, tmpDir, clusterPort, routePort, clientPort, clientPort)))\n\n\treturn RunServerWithConfig(cf)\n}\n\nfunc runMonitorServer() *Server {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\treturn RunServer(opts)\n}\n\nfunc runMonitorServerWithAccounts() *Server {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\taA := NewAccount(\"A\")\n\taB := NewAccount(\"B\")\n\topts.Accounts = append(opts.Accounts, aA, aB)\n\topts.Users = append(opts.Users,\n\t\t&User{Username: \"a\", Password: \"a\", Account: aA},\n\t\t&User{Username: \"b\", Password: \"b\", Account: aB})\n\treturn RunServer(opts)\n}\n\nfunc runMonitorServerNoHTTPPort() *Server {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.HTTPPort = 0\n\treturn RunServer(opts)\n}\n\nfunc resetPreviousHTTPConnections() {\n\thttp.DefaultTransport.(*http.Transport).CloseIdleConnections()\n}\n\nfunc TestMyUptime(t *testing.T) {\n\t// Make sure we print this stuff right.\n\tvar d time.Duration\n\tvar s string\n\n\td = 22 * time.Second\n\ts = myUptime(d)\n\tif s != \"22s\" {\n\t\tt.Fatalf(\"Expected `22s`, go ``%s`\", s)\n\t}\n\td = 4*time.Minute + d\n\ts = myUptime(d)\n\tif s != \"4m22s\" {\n\t\tt.Fatalf(\"Expected `4m22s`, go ``%s`\", s)\n\t}\n\td = 4*time.Hour + d\n\ts = myUptime(d)\n\tif s != \"4h4m22s\" {\n\t\tt.Fatalf(\"Expected `4h4m22s`, go ``%s`\", s)\n\t}\n\td = 32*24*time.Hour + d\n\ts = myUptime(d)\n\tif s != \"32d4h4m22s\" {\n\t\tt.Fatalf(\"Expected `32d4h4m22s`, go ``%s`\", s)\n\t}\n\td = 22*365*24*time.Hour + d\n\ts = myUptime(d)\n\tif s != \"22y32d4h4m22s\" {\n\t\tt.Fatalf(\"Expected `22y32d4h4m22s`, go ``%s`\", s)\n\t}\n}\n\n// Make sure that we do not run the http server for monitoring unless asked.\nfunc TestMonitorNoPort(t *testing.T) {\n\ts := runMonitorServerNoHTTPPort()\n\tdefer s.Shutdown()\n\n\t// this test might be meaningless now that we're testing with random ports?\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", 11245)\n\tif resp, err := http.Get(url + \"varz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n\tif resp, err := http.Get(url + \"healthz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n\tif resp, err := http.Get(url + \"connz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n}\n\nvar (\n\tappJSONContent = \"application/json\"\n\tappJSContent = \"application/javascript\"\n\ttextPlain = \"text/plain; charset=utf-8\"\n\ttextHTML = \"text/html; charset=utf-8\"\n)\n\nfunc readBodyEx(t *testing.T, url string, status int, content string) []byte {\n\tt.Helper()\n\tresp, err := http.Get(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\tif resp.StatusCode != status {\n\t\tt.Fatalf(\"Expected a %d response, got %d\\n%s\", status, resp.StatusCode, string(body))\n\t}\n\tct := resp.Header.Get(\"Content-Type\")\n\tif ct != content {\n\t\tt.Fatalf(\"Expected %q content-type, got %q\\n\", content, ct)\n\t}\n\t// Check the CORS header for \"application/json\" requests only.\n\tif ct == appJSONContent {\n\t\tacao := resp.Header.Get(\"Access-Control-Allow-Origin\")\n\t\tif acao != \"*\" {\n\t\t\tt.Fatalf(\"Expected with %q Content-Type an Access-Control-Allow-Origin header with value %q, got %q\\n\", appJSONContent, \"*\", acao)\n\t\t}\n\t}\n\treturn body\n}\n\nfunc TestMonitorHTTPBasePath(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.HTTPBasePath = \"/nats\"\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/nats\", s.MonitorAddr().Port)\n\treadBodyEx(t, url, http.StatusOK, textHTML)\n}\n\nfunc readBody(t *testing.T, url string) []byte {\n\tt.Helper()\n\treturn readBodyEx(t, url, http.StatusOK, appJSONContent)\n}\n\nfunc pollVarz(t *testing.T, s *Server, mode int, url string, opts *VarzOptions) *Varz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tv := &Varz{}\n\t\tbody := readBody(t, url)\n\t\tif err := json.Unmarshal(body, v); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn v\n\t}\n\tv, err := s.Varz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Varz: %v\", err)\n\t}\n\treturn v\n}\n\n// https://github.com/nats-io/nats-server/issues/2170\n// Just the ever increasing subs part.\nfunc TestMonitorVarzSubscriptionsResetProperly(t *testing.T) {\n\t// Run with JS to create a bunch of subs to start.\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.JetStream = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// This bug seems to only happen via the http endpoint, not direct calls.\n\t// Every time you call it doubles.\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tosubs := pollVarz(t, s, 0, url, nil).Subscriptions\n\t// Make sure we get same number back.\n\tif v := pollVarz(t, s, 0, url, nil); v.Subscriptions != osubs {\n\t\tt.Fatalf(\"Expected subs to stay the same, %d vs %d\", osubs, v.Subscriptions)\n\t}\n}\n\nfunc TestMonitorHandleVarz(t *testing.T) {\n\ts, _ := runMonitorJSServer(t, -1, -1, 0, 0)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s, mode, url+\"varz\", nil)\n\n\t\t// Do some sanity checks on values\n\t\tif time.Since(v.Start) > 10*time.Second {\n\t\t\tt.Fatal(\"Expected start time to be within 10 seconds.\")\n\t\t}\n\t}\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tnc := createClientConnWithUserSubscribeAndPublish(t, s, \"sys\", \"pwd\")\n\tdefer nc.Close()\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s, mode, url+\"varz\", nil)\n\n\t\tif v.Connections != 1 {\n\t\t\tt.Fatalf(\"Expected Connections of 1, got %v\\n\", v.Connections)\n\t\t}\n\t\tif v.TotalConnections < 1 {\n\t\t\tt.Fatalf(\"Expected Total Connections of at least 1, got %v\\n\", v.TotalConnections)\n\t\t}\n\t\tif v.InMsgs != 1 {\n\t\t\tt.Fatalf(\"Expected InMsgs of 1, got %v\\n\", v.InMsgs)\n\t\t}\n\t\tif v.OutMsgs != 1 {\n\t\t\tt.Fatalf(\"Expected OutMsgs of 1, got %v\\n\", v.OutMsgs)\n\t\t}\n\t\tif v.InBytes != 5 {\n\t\t\tt.Fatalf(\"Expected InBytes of 5, got %v\\n\", v.InBytes)\n\t\t}\n\t\tif v.OutBytes != 5 {\n\t\t\tt.Fatalf(\"Expected OutBytes of 5, got %v\\n\", v.OutBytes)\n\t\t}\n\t\tif v.Subscriptions <= 10 {\n\t\t\tt.Fatalf(\"Expected Subscriptions of at least 10, got %v\\n\", v.Subscriptions)\n\t\t}\n\t\tif v.Name != \"server_-1\" {\n\t\t\tt.Fatalf(\"Expected ServerName to be 'monitor_server' got %q\", v.Name)\n\t\t}\n\t\tif !v.Tags.Contains(\"tag\") {\n\t\t\tt.Fatalf(\"Expected tags to be 'tag' got %v\", v.Tags)\n\t\t}\n\t\tif v.JetStream.Config == nil {\n\t\t\tt.Fatalf(\"JS Config not set\")\n\t\t}\n\t\tsd := filepath.Join(s.opts.StoreDir, \"jetstream\")\n\t\tif v.JetStream.Config.StoreDir != sd {\n\t\t\tt.Fatalf(\"JS Config is invalid expected %q got %q\", sd, v.JetStream.Config.StoreDir)\n\t\t}\n\t\tif v.JetStream.Stats == nil {\n\t\t\tt.Fatalf(\"JS Stats not set\")\n\t\t}\n\t\tif v.JetStream.Stats.Accounts != 2 {\n\t\t\tt.Fatalf(\"Invalid stats expected 2 accounts got %d\", v.JetStream.Stats.Accounts)\n\t\t}\n\t\tif v.JetStream.Limits == nil {\n\t\t\tt.Fatalf(\"JS limits not set\")\n\t\t}\n\t\tif v.JetStream.Limits.MaxHAAssets != 1000 {\n\t\t\tt.Fatalf(\"Expected 1000 max_ha_assets got %q\", v.JetStream.Limits.MaxHAAssets)\n\t\t}\n\t}\n\n\t// Test JSONP\n\treadBodyEx(t, url+\"varz?callback=callback\", http.StatusOK, appJSContent)\n}\n\nfunc pollConnz(t *testing.T, s *Server, mode int, url string, opts *ConnzOptions) *Connz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tbody := readBody(t, url)\n\t\tc := &Connz{}\n\t\tif err := json.Unmarshal(body, &c); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn c\n\t}\n\tc, err := s.Connz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Connz(): %v\", err)\n\t}\n\treturn c\n}\n\nfunc TestMonitorConnz(t *testing.T) {\n\ts := runMonitorServerWithAccounts()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\ttestConnz := func(mode int) {\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\n\t\t// Test contents..\n\t\trequire_Equal(t, c.NumConns, 0)\n\t\trequire_Equal(t, c.Total, 0)\n\t\trequire_Equal(t, len(c.Conns), 0)\n\n\t\t// Test with connections.\n\t\tnc := createClientConnWithUserSubscribeAndPublish(t, s, \"a\", \"a\")\n\t\tdefer nc.Close()\n\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\tc = pollConnz(t, s, mode, url+\"connz?auth=1\", &ConnzOptions{Username: true})\n\n\t\trequire_Equal(t, c.NumConns, 1)\n\t\trequire_Equal(t, c.Total, 1)\n\t\trequire_Equal(t, len(c.Conns), 1)\n\t\trequire_Equal(t, c.Limit, DefaultConnListSize)\n\t\trequire_Equal(t, c.Offset, 0)\n\n\t\t// Test inside details of each connection\n\t\tci := c.Conns[0]\n\n\t\trequire_NotEqual(t, ci.Cid, 0)\n\t\trequire_Equal(t, ci.IP, \"127.0.0.1\")\n\t\trequire_NotEqual(t, ci.Port, 0)\n\t\trequire_Equal(t, ci.NumSubs, 0)\n\t\trequire_Equal(t, len(ci.Subs), 0)\n\t\trequire_Equal(t, len(ci.SubsDetail), 0)\n\t\trequire_Equal(t, ci.InMsgs, 1)\n\t\trequire_Equal(t, ci.OutMsgs, 1)\n\t\trequire_Equal(t, ci.InBytes, 5)\n\t\trequire_Equal(t, ci.OutBytes, 5)\n\t\trequire_False(t, ci.Start.IsZero())\n\t\trequire_NotEqual(t, ci.Uptime, \"\")\n\t\trequire_False(t, ci.LastActivity.IsZero())\n\t\trequire_False(t, ci.LastActivity.UnixNano() < ci.Start.UnixNano())\n\t\trequire_NotEqual(t, ci.Idle, \"\")\n\t\t// This is a change, we now expect them to be set for connections when the\n\t\t// client sends a connect.\n\t\trequire_NotEqual(t, ci.RTT, \"\")\n\n\t\trequire_Equal(t, ci.Account, \"A\")\n\t\trequire_Equal(t, ci.NameTag, \"A\")\n\t}\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\ttestConnz(mode)\n\t\tcheckClientsCount(t, s, 0)\n\t}\n\n\t// Test JSONP\n\treadBodyEx(t, url+\"connz?callback=callback\", http.StatusOK, appJSContent)\n}\n\nfunc TestMonitorConnzBadParams(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?\", s.MonitorAddr().Port)\n\treadBodyEx(t, url+\"auth=xxx\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, url+\"subs=xxx\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, url+\"offset=xxx\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, url+\"limit=xxx\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, url+\"state=xxx\", http.StatusBadRequest, textPlain)\n}\n\nfunc TestMonitorConnzWithSubs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"hello.foo\", func(m *nats.Msg) {})\n\tensureServerActivityRecorded(t, nc)\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?subs=1\", &ConnzOptions{Subscriptions: true})\n\t\t// Test inside details of each connection\n\t\tci := c.Conns[0]\n\t\tif len(ci.Subs) != 1 || ci.Subs[0] != \"hello.foo\" {\n\t\t\tt.Fatalf(\"Expected subs of 1, got %v\\n\", ci.Subs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzWithSubsDetail(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"hello.foo\", func(m *nats.Msg) {})\n\tensureServerActivityRecorded(t, nc)\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?subs=detail\", &ConnzOptions{SubscriptionsDetail: true})\n\t\t// Test inside details of each connection\n\t\tci := c.Conns[0]\n\t\tif len(ci.SubsDetail) != 1 || ci.SubsDetail[0].Subject != \"hello.foo\" {\n\t\t\tt.Fatalf(\"Expected subsdetail of 1, got %v\\n\", ci.Subs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorClosedConnzWithSubsDetail(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\n\tnc.Subscribe(\"hello.foo\", func(m *nats.Msg) {})\n\tensureServerActivityRecorded(t, nc)\n\tnc.Close()\n\n\ts.mu.Lock()\n\tfor len(s.clients) != 0 {\n\t\ts.mu.Unlock()\n\t\t<-time.After(100 * time.Millisecond)\n\t\ts.mu.Lock()\n\t}\n\ts.mu.Unlock()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?state=closed&subs=detail\", &ConnzOptions{State: ConnClosed,\n\t\t\tSubscriptionsDetail: true})\n\t\t// Test inside details of each connection\n\t\tci := c.Conns[0]\n\t\tif len(ci.SubsDetail) != 1 || ci.SubsDetail[0].Subject != \"hello.foo\" {\n\t\t\tt.Fatalf(\"Expected subsdetail of 1, got %v\\n\", ci.Subs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzWithCID(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\t// The one we will request\n\tcid := 5\n\ttotal := 10\n\n\t// Create 10\n\tfor i := 1; i <= total; i++ {\n\t\tnc := createClientConnSubscribeAndPublish(t, s)\n\t\tdefer nc.Close()\n\t\tif i == cid {\n\t\t\tnc.Subscribe(\"hello.foo\", func(m *nats.Msg) {})\n\t\t\tnc.Subscribe(\"hello.bar\", func(m *nats.Msg) {})\n\t\t\tensureServerActivityRecorded(t, nc)\n\t\t}\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?cid=%d\", s.MonitorAddr().Port, cid)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url, &ConnzOptions{CID: uint64(cid)})\n\t\t// Test inside details of each connection\n\t\tif len(c.Conns) != 1 {\n\t\t\tt.Fatalf(\"Expected only one connection, but got %d\\n\", len(c.Conns))\n\t\t}\n\t\tif c.NumConns != 1 {\n\t\t\tt.Fatalf(\"Expected NumConns to be 1, but got %d\\n\", c.NumConns)\n\t\t}\n\t\tci := c.Conns[0]\n\t\tif ci.Cid != uint64(cid) {\n\t\t\tt.Fatalf(\"Expected to receive connection %v, but received %v\\n\", cid, ci.Cid)\n\t\t}\n\t\tif ci.NumSubs != 2 {\n\t\t\tt.Fatalf(\"Expected to receive connection with %d subs, but received %d\\n\", 2, ci.NumSubs)\n\t\t}\n\t\t// Now test a miss\n\t\tbadUrl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?cid=%d\", s.MonitorAddr().Port, 100)\n\t\tc = pollConnz(t, s, mode, badUrl, &ConnzOptions{CID: uint64(100)})\n\t\tif len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected no connections, got %d\\n\", len(c.Conns))\n\t\t}\n\t\tif c.NumConns != 0 {\n\t\t\tt.Fatalf(\"Expected NumConns of 0, got %d\\n\", c.NumConns)\n\t\t}\n\t}\n}\n\n// Helper to map to connection name\nfunc createConnMap(cz *Connz) map[string]*ConnInfo {\n\tcm := make(map[string]*ConnInfo)\n\tfor _, c := range cz.Conns {\n\t\tcm[c.Name] = c\n\t}\n\treturn cm\n}\n\nfunc getFooAndBar(cm map[string]*ConnInfo) (*ConnInfo, *ConnInfo) {\n\treturn cm[\"foo\"], cm[\"bar\"]\n}\n\nfunc ensureServerActivityRecorded(t *testing.T, nc *nats.Conn) {\n\tnc.Flush()\n\terr := nc.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"Error flushing: %v\\n\", err)\n\t}\n}\n\nfunc TestMonitorConnzRTT(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\ttestRTT := func(mode int) {\n\t\t// Test with connections.\n\t\tnc := createClientConnSubscribeAndPublish(t, s)\n\t\tdefer nc.Close()\n\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\n\t\tif c.NumConns != 1 {\n\t\t\tt.Fatalf(\"Expected 1 connection, got %d\\n\", c.NumConns)\n\t\t}\n\n\t\t// Send a server side PING to record RTT\n\t\ts.mu.Lock()\n\t\tci := c.Conns[0]\n\t\tsc := s.clients[ci.Cid]\n\t\tif sc == nil {\n\t\t\tt.Fatalf(\"Error looking up client %v\\n\", ci.Cid)\n\t\t}\n\t\ts.mu.Unlock()\n\t\tsc.mu.Lock()\n\t\tsc.sendPing()\n\t\tsc.mu.Unlock()\n\n\t\t// Wait for client to respond with PONG\n\t\ttime.Sleep(20 * time.Millisecond)\n\n\t\t// Repoll for updated information.\n\t\tc = pollConnz(t, s, mode, url+\"connz\", nil)\n\t\tci = c.Conns[0]\n\n\t\trtt, err := time.ParseDuration(ci.RTT)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not parse RTT properly, %v (ci.RTT=%v)\", err, ci.RTT)\n\t\t}\n\t\tif rtt <= 0 {\n\t\t\tt.Fatal(\"Expected RTT to be valid and non-zero\\n\")\n\t\t}\n\t\tif (runtime.GOOS == \"windows\" && rtt > 20*time.Millisecond) ||\n\t\t\trtt > 20*time.Millisecond || rtt < 100*time.Nanosecond {\n\t\t\tt.Fatalf(\"Invalid RTT of %s\\n\", ci.RTT)\n\t\t}\n\t}\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\ttestRTT(mode)\n\t\tcheckClientsCount(t, s, 0)\n\t}\n}\n\nfunc TestMonitorConnzLastActivity(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\turl += \"connz?subs=1\"\n\topts := &ConnzOptions{Subscriptions: true}\n\n\tvar sleepTime time.Duration\n\tif runtime.GOOS == \"windows\" {\n\t\tsleepTime = 10 * time.Millisecond\n\t}\n\n\ttestActivity := func(mode int) {\n\t\tncFoo := createClientConnWithName(t, \"foo\", s)\n\t\tdefer ncFoo.Close()\n\n\t\tncBar := createClientConnWithName(t, \"bar\", s)\n\t\tdefer ncBar.Close()\n\n\t\t// Test inside details of each connection\n\t\tciFoo, ciBar := getFooAndBar(createConnMap(pollConnz(t, s, mode, url, opts)))\n\n\t\t// Test that LastActivity is non-zero\n\t\tif ciFoo.LastActivity.IsZero() {\n\t\t\tt.Fatalf(\"Expected LastActivity for connection '%s'to be valid\\n\", ciFoo.Name)\n\t\t}\n\t\tif ciBar.LastActivity.IsZero() {\n\t\t\tt.Fatalf(\"Expected LastActivity for connection '%s'to be valid\\n\", ciBar.Name)\n\t\t}\n\t\t// Foo should be older than Bar\n\t\tif ciFoo.LastActivity.After(ciBar.LastActivity) {\n\t\t\tt.Fatal(\"Expected connection 'foo' to be older than 'bar'\\n\")\n\t\t}\n\n\t\tfooLA := ciFoo.LastActivity\n\t\tbarLA := ciBar.LastActivity\n\n\t\tensureServerActivityRecorded(t, ncFoo)\n\t\tensureServerActivityRecorded(t, ncBar)\n\n\t\ttime.Sleep(sleepTime)\n\n\t\t// Sub should trigger update.\n\t\tsub, _ := ncFoo.Subscribe(\"hello.world\", func(m *nats.Msg) {})\n\t\tensureServerActivityRecorded(t, ncFoo)\n\n\t\tciFoo, _ = getFooAndBar(createConnMap(pollConnz(t, s, mode, url, opts)))\n\t\tnextLA := ciFoo.LastActivity\n\t\tif fooLA.Equal(nextLA) {\n\t\t\tt.Fatalf(\"Subscribe should have triggered update to LastActivity %+v\\n\", ciFoo)\n\t\t}\n\t\tfooLA = nextLA\n\n\t\ttime.Sleep(sleepTime)\n\n\t\t// Publish and Message Delivery should trigger as well. So both connections\n\t\t// should have updates.\n\t\tncBar.Publish(\"hello.world\", []byte(\"Hello\"))\n\n\t\tensureServerActivityRecorded(t, ncFoo)\n\t\tensureServerActivityRecorded(t, ncBar)\n\n\t\tciFoo, ciBar = getFooAndBar(createConnMap(pollConnz(t, s, mode, url, opts)))\n\t\tnextLA = ciBar.LastActivity\n\t\tif barLA.Equal(nextLA) {\n\t\t\tt.Fatalf(\"Publish should have triggered update to LastActivity\\n\")\n\t\t}\n\n\t\t// Message delivery on ncFoo should have triggered as well.\n\t\tnextLA = ciFoo.LastActivity\n\t\tif fooLA.Equal(nextLA) {\n\t\t\tt.Fatalf(\"Message delivery should have triggered update to LastActivity\\n\")\n\t\t}\n\t\tfooLA = nextLA\n\n\t\ttime.Sleep(sleepTime)\n\n\t\t// Unsub should trigger as well\n\t\tsub.Unsubscribe()\n\t\tensureServerActivityRecorded(t, ncFoo)\n\n\t\tciFoo, _ = getFooAndBar(createConnMap(pollConnz(t, s, mode, url, opts)))\n\t\tnextLA = ciFoo.LastActivity\n\t\tif fooLA.Equal(nextLA) {\n\t\t\tt.Fatalf(\"Message delivery should have triggered update to LastActivity\\n\")\n\t\t}\n\t}\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\ttestActivity(mode)\n\t}\n}\n\nfunc TestMonitorConnzWithOffsetAndLimit(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?offset=1&limit=1\", &ConnzOptions{Offset: 1, Limit: 1})\n\t\tif c.Conns == nil || len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected 0 connections in array, got %p\\n\", c.Conns)\n\t\t}\n\n\t\t// Test that when given negative values, 0 or default is used\n\t\tc = pollConnz(t, s, mode, url+\"connz?offset=-1&limit=-1\", &ConnzOptions{Offset: -11, Limit: -11})\n\t\tif c.Conns == nil || len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected 0 connections in array, got %p\\n\", c.Conns)\n\t\t}\n\t\tif c.Offset != 0 {\n\t\t\tt.Fatalf(\"Expected offset to be 0, and limit to be %v, got %v and %v\",\n\t\t\t\tDefaultConnListSize, c.Offset, c.Limit)\n\t\t}\n\t}\n\n\tcl1 := createClientConnSubscribeAndPublish(t, s)\n\tdefer cl1.Close()\n\n\tcl2 := createClientConnSubscribeAndPublish(t, s)\n\tdefer cl2.Close()\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?offset=1&limit=1\", &ConnzOptions{Offset: 1, Limit: 1})\n\t\tif c.Limit != 1 {\n\t\t\tt.Fatalf(\"Expected limit of 1, got %v\\n\", c.Limit)\n\t\t}\n\n\t\tif c.Offset != 1 {\n\t\t\tt.Fatalf(\"Expected offset of 1, got %v\\n\", c.Offset)\n\t\t}\n\n\t\tif len(c.Conns) != 1 {\n\t\t\tt.Fatalf(\"Expected conns of 1, got %v\\n\", len(c.Conns))\n\t\t}\n\n\t\tif c.NumConns != 1 {\n\t\t\tt.Fatalf(\"Expected NumConns to be 1, got %v\\n\", c.NumConns)\n\t\t}\n\n\t\tif c.Total != 2 {\n\t\t\tt.Fatalf(\"Expected Total to be at least 2, got %v\", c.Total)\n\t\t}\n\n\t\tc = pollConnz(t, s, mode, url+\"connz?offset=2&limit=1\", &ConnzOptions{Offset: 2, Limit: 1})\n\t\tif c.Limit != 1 {\n\t\t\tt.Fatalf(\"Expected limit of 1, got %v\\n\", c.Limit)\n\t\t}\n\n\t\tif c.Offset != 2 {\n\t\t\tt.Fatalf(\"Expected offset of 2, got %v\\n\", c.Offset)\n\t\t}\n\n\t\tif len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected conns of 0, got %v\\n\", len(c.Conns))\n\t\t}\n\n\t\tif c.NumConns != 0 {\n\t\t\tt.Fatalf(\"Expected NumConns to be 0, got %v\\n\", c.NumConns)\n\t\t}\n\n\t\tif c.Total != 2 {\n\t\t\tt.Fatalf(\"Expected Total to be 2, got %v\", c.Total)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzDefaultSorted(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tclients := make([]*nats.Conn, 4)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\t\tif c.Conns[0].Cid > c.Conns[1].Cid ||\n\t\t\tc.Conns[1].Cid > c.Conns[2].Cid ||\n\t\t\tc.Conns[2].Cid > c.Conns[3].Cid {\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by cid, got %v < %v\\n\", c.Conns[0].Cid, c.Conns[3].Cid)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByCid(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tclients := make([]*nats.Conn, 4)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=cid\", &ConnzOptions{Sort: ByCid})\n\t\tif c.Conns[0].Cid > c.Conns[1].Cid ||\n\t\t\tc.Conns[1].Cid > c.Conns[2].Cid ||\n\t\t\tc.Conns[2].Cid > c.Conns[3].Cid {\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by cid, got [%v, %v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].Cid, c.Conns[1].Cid, c.Conns[2].Cid, c.Conns[3].Cid)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByStart(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tclients := make([]*nats.Conn, 4)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=start\", &ConnzOptions{Sort: ByStart})\n\t\tif c.Conns[0].Start.After(c.Conns[1].Start) ||\n\t\t\tc.Conns[1].Start.After(c.Conns[2].Start) ||\n\t\t\tc.Conns[2].Start.After(c.Conns[3].Start) {\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by startime, got [%v, %v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].Start, c.Conns[1].Start, c.Conns[2].Start, c.Conns[3].Start)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByBytesAndMsgs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\t// Create a connection and make it send more messages than others\n\tfirstClient := createClientConnSubscribeAndPublish(t, s)\n\tfor i := 0; i < 100; i++ {\n\t\tfirstClient.Publish(\"foo\", []byte(\"Hello World\"))\n\t}\n\tdefer firstClient.Close()\n\tfirstClient.Flush()\n\n\tclients := make([]*nats.Conn, 3)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=bytes_to\", &ConnzOptions{Sort: ByOutBytes})\n\t\tif c.Conns[0].OutBytes < c.Conns[1].OutBytes ||\n\t\t\tc.Conns[0].OutBytes < c.Conns[2].OutBytes ||\n\t\t\tc.Conns[0].OutBytes < c.Conns[3].OutBytes {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by bytes to, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].OutBytes, c.Conns[1].OutBytes, c.Conns[2].OutBytes, c.Conns[3].OutBytes)\n\t\t}\n\n\t\tc = pollConnz(t, s, mode, url+\"connz?sort=msgs_to\", &ConnzOptions{Sort: ByOutMsgs})\n\t\tif c.Conns[0].OutMsgs < c.Conns[1].OutMsgs ||\n\t\t\tc.Conns[0].OutMsgs < c.Conns[2].OutMsgs ||\n\t\t\tc.Conns[0].OutMsgs < c.Conns[3].OutMsgs {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by msgs from, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].OutMsgs, c.Conns[1].OutMsgs, c.Conns[2].OutMsgs, c.Conns[3].OutMsgs)\n\t\t}\n\n\t\tc = pollConnz(t, s, mode, url+\"connz?sort=bytes_from\", &ConnzOptions{Sort: ByInBytes})\n\t\tif c.Conns[0].InBytes < c.Conns[1].InBytes ||\n\t\t\tc.Conns[0].InBytes < c.Conns[2].InBytes ||\n\t\t\tc.Conns[0].InBytes < c.Conns[3].InBytes {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by bytes from, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].InBytes, c.Conns[1].InBytes, c.Conns[2].InBytes, c.Conns[3].InBytes)\n\t\t}\n\n\t\tc = pollConnz(t, s, mode, url+\"connz?sort=msgs_from\", &ConnzOptions{Sort: ByInMsgs})\n\t\tif c.Conns[0].InMsgs < c.Conns[1].InMsgs ||\n\t\t\tc.Conns[0].InMsgs < c.Conns[2].InMsgs ||\n\t\t\tc.Conns[0].InMsgs < c.Conns[3].InMsgs {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by msgs from, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].InMsgs, c.Conns[1].InMsgs, c.Conns[2].InMsgs, c.Conns[3].InMsgs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByPending(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfirstClient := createClientConnSubscribeAndPublish(t, s)\n\tfirstClient.Subscribe(\"hello.world\", func(m *nats.Msg) {})\n\tclients := make([]*nats.Conn, 3)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\tdefer firstClient.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=pending\", &ConnzOptions{Sort: ByPending})\n\t\tif c.Conns[0].Pending < c.Conns[1].Pending ||\n\t\t\tc.Conns[0].Pending < c.Conns[2].Pending ||\n\t\t\tc.Conns[0].Pending < c.Conns[3].Pending {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by number of pending, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].Pending, c.Conns[1].Pending, c.Conns[2].Pending, c.Conns[3].Pending)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedBySubs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfirstClient := createClientConnSubscribeAndPublish(t, s)\n\tfirstClient.Subscribe(\"hello.world\", func(m *nats.Msg) {})\n\tdefer firstClient.Close()\n\n\tclients := make([]*nats.Conn, 3)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=subs\", &ConnzOptions{Sort: BySubs})\n\t\tif c.Conns[0].NumSubs < c.Conns[1].NumSubs ||\n\t\t\tc.Conns[0].NumSubs < c.Conns[2].NumSubs ||\n\t\t\tc.Conns[0].NumSubs < c.Conns[3].NumSubs {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by number of subs, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].NumSubs, c.Conns[1].NumSubs, c.Conns[2].NumSubs, c.Conns[3].NumSubs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByLast(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfirstClient := createClientConnSubscribeAndPublish(t, s)\n\tdefer firstClient.Close()\n\tfirstClient.Subscribe(\"hello.world\", func(m *nats.Msg) {})\n\tfirstClient.Flush()\n\n\tclients := make([]*nats.Conn, 3)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t\tclients[i].Flush()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=last\", &ConnzOptions{Sort: ByLast})\n\t\tif c.Conns[0].LastActivity.UnixNano() < c.Conns[1].LastActivity.UnixNano() ||\n\t\t\tc.Conns[1].LastActivity.UnixNano() < c.Conns[2].LastActivity.UnixNano() ||\n\t\t\tc.Conns[2].LastActivity.UnixNano() < c.Conns[3].LastActivity.UnixNano() {\n\t\t\tt.Fatalf(\"Expected conns sorted in descending order by lastActivity, got %v < one of [%v, %v, %v]\\n\",\n\t\t\t\tc.Conns[0].LastActivity, c.Conns[1].LastActivity, c.Conns[2].LastActivity, c.Conns[3].LastActivity)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByUptime(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfor i := 0; i < 4; i++ {\n\t\tclient := createClientConnSubscribeAndPublish(t, s)\n\t\tdefer client.Close()\n\t\t// Since we check times (now-start) does not have to be big.\n\t\ttime.Sleep(50 * time.Millisecond)\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?sort=uptime\", &ConnzOptions{Sort: ByUptime})\n\t\tnow := time.Now()\n\t\tups := make([]int, 4)\n\t\tfor i := 0; i < 4; i++ {\n\t\t\tups[i] = int(now.Sub(c.Conns[i].Start))\n\t\t}\n\t\tif !sort.IntsAreSorted(ups) {\n\t\t\td := make([]time.Duration, 4)\n\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\td[i] = time.Duration(ups[i])\n\t\t\t}\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by uptime (now-Start), got %+v\\n\", d)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByUptimeClosedConn(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfor i := time.Duration(1); i <= 4; i++ {\n\t\tc := createClientConnSubscribeAndPublish(t, s)\n\n\t\t// Grab client and asjust start time such that\n\t\tclient := s.getClient(uint64(i))\n\t\tif client == nil {\n\t\t\tt.Fatalf(\"Could nopt retrieve client for %d\\n\", i)\n\t\t}\n\t\tclient.mu.Lock()\n\t\tclient.start = client.start.Add(-10 * (4 - i) * time.Second)\n\t\tclient.mu.Unlock()\n\n\t\tc.Close()\n\t}\n\n\tcheckClosedConns(t, s, 4, time.Second)\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?state=closed&sort=uptime\", &ConnzOptions{State: ConnClosed, Sort: ByUptime})\n\t\tups := make([]int, 4)\n\t\tfor i := 0; i < 4; i++ {\n\t\t\tups[i] = int(c.Conns[i].Stop.Sub(c.Conns[i].Start))\n\t\t}\n\t\tif !sort.IntsAreSorted(ups) {\n\t\t\td := make([]time.Duration, 4)\n\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\td[i] = time.Duration(ups[i])\n\t\t\t}\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by uptime, got %+v\\n\", d)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByStopOnOpen(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// 4 clients\n\tfor i := 0; i < 4; i++ {\n\t\tc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not create client: %v\\n\", err)\n\t\t}\n\t\tdefer c.Close()\n\t}\n\n\tc, err := s.Connz(&ConnzOptions{Sort: ByStop})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected err to be non-nil, got %+v\\n\", c)\n\t}\n}\n\nfunc TestMonitorConnzSortedByStopTimeClosedConn(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// 4 clients\n\tfor i := 0; i < 4; i++ {\n\t\tc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not create client: %v\\n\", err)\n\t\t}\n\t\tc.Close()\n\t}\n\tcheckClosedConns(t, s, 4, time.Second)\n\n\t// Now adjust the Stop times for these with some random values.\n\ts.mu.Lock()\n\tnow := time.Now().UTC()\n\tccs := s.closed.closedClients()\n\tfor _, cc := range ccs {\n\t\tnewStop := now.Add(time.Duration(rand.Int()%120) * -time.Minute)\n\t\tcc.Stop = &newStop\n\t}\n\ts.mu.Unlock()\n\n\turl = fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?state=closed&sort=stop\", &ConnzOptions{State: ConnClosed, Sort: ByStop})\n\t\tups := make([]int, 4)\n\t\tnowU := time.Now().UnixNano()\n\t\tfor i := 0; i < 4; i++ {\n\t\t\tups[i] = int(nowU - c.Conns[i].Stop.UnixNano())\n\t\t}\n\t\tif !sort.IntsAreSorted(ups) {\n\t\t\td := make([]time.Duration, 4)\n\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\td[i] = time.Duration(ups[i])\n\t\t\t}\n\t\t\tt.Fatalf(\"Expected conns sorted in ascending order by stop time, got %+v\\n\", d)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByReason(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// 20 clients\n\tfor i := 0; i < 20; i++ {\n\t\tc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not create client: %v\\n\", err)\n\t\t}\n\t\tc.Close()\n\t}\n\tcheckClosedConns(t, s, 20, time.Second)\n\n\t// Now adjust the Reasons for these with some random values.\n\ts.mu.Lock()\n\tccs := s.closed.closedClients()\n\tmax := int(ServerShutdown)\n\tfor _, cc := range ccs {\n\t\tcc.Reason = ClosedState(rand.Int() % max).String()\n\t}\n\ts.mu.Unlock()\n\n\turl = fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz?state=closed&sort=reason\", &ConnzOptions{State: ConnClosed, Sort: ByReason})\n\t\trs := make([]string, 20)\n\t\tfor i := 0; i < 20; i++ {\n\t\t\trs[i] = c.Conns[i].Reason\n\t\t}\n\t\tif !sort.StringsAreSorted(rs) {\n\t\t\tt.Fatalf(\"Expected conns sorted in order by stop reason, got %#v\\n\", rs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzSortedByReasonOnOpen(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// 4 clients\n\tfor i := 0; i < 4; i++ {\n\t\tc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not create client: %v\\n\", err)\n\t\t}\n\t\tdefer c.Close()\n\t}\n\n\tc, err := s.Connz(&ConnzOptions{Sort: ByReason})\n\tif err == nil {\n\t\tt.Fatalf(\"Expected err to be non-nil, got %+v\\n\", c)\n\t}\n}\n\nfunc TestMonitorConnzSortedByIdle(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://%s/connz?sort=idle\", s.MonitorAddr())\n\tnow := time.Now()\n\n\tclients := []struct {\n\t\tstart time.Time // Client start time.\n\t\tlast time.Time // Client last activity time.\n\t}{\n\t\t{start: now.Add(-10 * time.Second), last: now.Add(-5 * time.Second)},\n\t\t{start: now.Add(-20 * time.Second), last: now.Add(-10 * time.Second)},\n\t\t{start: now.Add(-3 * time.Second), last: now.Add(-2 * time.Second)},\n\t\t{start: now.Add(-30 * time.Second), last: now.Add(-20 * time.Second)},\n\t}\n\n\ttestIdle := func(mode int) {\n\t\t// Connect the specified number of clients.\n\t\tfor _, c := range clients {\n\t\t\tclientConn := createClientConnSubscribeAndPublish(t, s)\n\t\t\tdefer clientConn.Close()\n\n\t\t\tcid, err := clientConn.GetClientID()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error getting the client CID: %v\", err)\n\t\t\t}\n\n\t\t\tclient := s.getClient(cid)\n\t\t\tif client == nil {\n\t\t\t\tt.Fatalf(\"error looking up client %d\", cid)\n\t\t\t}\n\n\t\t\t// Change the client's start and last activity times.\n\t\t\tclient.mu.Lock()\n\t\t\tclient.start = c.start\n\t\t\tclient.last = c.last\n\t\t\tclient.mu.Unlock()\n\t\t}\n\n\t\tconnz := pollConnz(t, s, mode, url, &ConnzOptions{Sort: ByIdle})\n\n\t\twantConns := len(clients)\n\t\tgotConns := len(connz.Conns)\n\n\t\tif gotConns != wantConns {\n\t\t\tt.Fatalf(\"want %d connections, got %d\", wantConns, gotConns)\n\t\t}\n\n\t\tidleDurations := getConnsIdleDurations(t, connz.Conns)\n\n\t\tif !sortedDurationsDesc(idleDurations) {\n\t\t\tt.Errorf(\"want durations sorted in descending order, got %v\", idleDurations)\n\t\t}\n\t}\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\ttestIdle(mode)\n\t}\n}\n\n// getConnsIdleDurations returns a slice of parsed idle durations from a connection info slice.\nfunc getConnsIdleDurations(t *testing.T, conns []*ConnInfo) []time.Duration {\n\tt.Helper()\n\n\tdurations := make([]time.Duration, 0, len(conns))\n\n\tfor _, conn := range conns {\n\t\tidle, err := time.ParseDuration(conn.Idle)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error parsing duration %q: %v\", conn.Idle, err)\n\t\t}\n\t\tdurations = append(durations, idle)\n\t}\n\n\treturn durations\n}\n\n// sortedDurationsDesc checks if a time.Duration slice is sorted in descending order.\nfunc sortedDurationsDesc(durations []time.Duration) bool {\n\treturn sort.SliceIsSorted(durations, func(i, j int) bool {\n\t\t// Must be longer than the next duration.\n\t\treturn durations[i] > durations[j]\n\t})\n}\n\nfunc TestMonitorConnzSortByIdleTime(t *testing.T) {\n\tnow := time.Now().UTC()\n\n\tcases := map[string]ConnInfos{\n\t\t\"zero values\": {{}, {}, {}, {}},\n\t\t\"equal last activity times\": {\n\t\t\t{Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-time.Minute)},\n\t\t\t{Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-time.Minute)},\n\t\t\t{Start: now.Add(-10 * time.Second), LastActivity: now.Add(-time.Minute)},\n\t\t\t{Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)},\n\t\t},\n\t\t\"last activity in the future\": {\n\t\t\t{Start: now.Add(-50 * time.Minute), LastActivity: now.Add(10 * time.Minute)}, // +10m\n\t\t\t{Start: now.Add(-30 * time.Minute), LastActivity: now.Add(5 * time.Minute)}, // +5m\n\t\t\t{Start: now.Add(-24 * time.Hour), LastActivity: now.Add(2 * time.Second)}, // +2s\n\t\t\t{Start: now.Add(-10 * time.Second), LastActivity: now.Add(15 * time.Minute)}, // +15m\n\t\t\t{Start: now.Add(-2 * time.Hour), LastActivity: now.Add(time.Minute)}, // +1m\n\t\t},\n\t\t\"unsorted\": {\n\t\t\t{Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-10 * time.Minute)}, // 10m ago\n\t\t\t{Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-5 * time.Minute)}, // 5m ago\n\t\t\t{Start: now.Add(-24 * time.Hour), LastActivity: now.Add(-2 * time.Second)}, // 2s ago\n\t\t\t{Start: now.Add(-10 * time.Second), LastActivity: now.Add(-15 * time.Minute)}, // 15m ago\n\t\t\t{Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)}, // 1m ago\n\t\t},\n\t\t\"unsorted with zero value start time\": {\n\t\t\t{LastActivity: now.Add(-10 * time.Minute)}, // 10m ago\n\t\t\t{LastActivity: now.Add(-5 * time.Minute)}, // 5m ago\n\t\t\t{LastActivity: now.Add(-2 * time.Second)}, // 2s ago\n\t\t\t{LastActivity: now.Add(-15 * time.Minute)}, // 15m ago\n\t\t\t{LastActivity: now.Add(-time.Minute)}, // 1m ago\n\t\t},\n\t\t\"sorted\": {\n\t\t\t{Start: now.Add(-24 * time.Hour), LastActivity: now.Add(-2 * time.Second)}, // 2s ago\n\t\t\t{Start: now.Add(-2 * time.Hour), LastActivity: now.Add(-time.Minute)}, // 1m ago\n\t\t\t{Start: now.Add(-30 * time.Minute), LastActivity: now.Add(-5 * time.Minute)}, // 5m ago\n\t\t\t{Start: now.Add(-50 * time.Minute), LastActivity: now.Add(-10 * time.Minute)}, // 10m ago\n\t\t\t{Start: now.Add(-10 * time.Second), LastActivity: now.Add(-15 * time.Minute)}, // 15m ago\n\t\t},\n\t\t\"sorted with zero value start time\": {\n\t\t\t{LastActivity: now.Add(-2 * time.Second)}, // 2s ago\n\t\t\t{LastActivity: now.Add(-time.Minute)}, // 1m ago\n\t\t\t{LastActivity: now.Add(-5 * time.Minute)}, // 5m ago\n\t\t\t{LastActivity: now.Add(-10 * time.Minute)}, // 10m ago\n\t\t\t{LastActivity: now.Add(-15 * time.Minute)}, // 15m ago\n\t\t},\n\t}\n\n\tfor name, conns := range cases {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tsort.Sort(SortByIdle{conns, now})\n\n\t\t\tidleDurations := getIdleDurations(conns, now)\n\n\t\t\tif !sortedDurationsAsc(idleDurations) {\n\t\t\t\tt.Errorf(\"want durations sorted in ascending order, got %v\", idleDurations)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// getIdleDurations returns a slice of idle durations from a connection info list up until now time.\nfunc getIdleDurations(conns ConnInfos, now time.Time) []time.Duration {\n\tdurations := make([]time.Duration, 0, len(conns))\n\n\tfor _, conn := range conns {\n\t\tdurations = append(durations, now.Sub(conn.LastActivity))\n\t}\n\n\treturn durations\n}\n\n// sortedDurationsAsc checks if a time.Duration slice is sorted in ascending order.\nfunc sortedDurationsAsc(durations []time.Duration) bool {\n\treturn sort.SliceIsSorted(durations, func(i, j int) bool {\n\t\treturn durations[i] < durations[j]\n\t})\n}\n\nfunc TestMonitorConnzSortBadRequest(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfirstClient := createClientConnSubscribeAndPublish(t, s)\n\tfirstClient.Subscribe(\"hello.world\", func(m *nats.Msg) {})\n\tclients := make([]*nats.Conn, 3)\n\tfor i := range clients {\n\t\tclients[i] = createClientConnSubscribeAndPublish(t, s)\n\t\tdefer clients[i].Close()\n\t}\n\tdefer firstClient.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\treadBodyEx(t, url+\"connz?sort=foo\", http.StatusBadRequest, textPlain)\n\n\tif _, err := s.Connz(&ConnzOptions{Sort: \"foo\"}); err == nil {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n}\n\nfunc pollRoutez(t *testing.T, s *Server, mode int, url string, opts *RoutezOptions) *Routez {\n\tt.Helper()\n\tif mode == 0 {\n\t\trz := &Routez{}\n\t\tbody := readBody(t, url)\n\t\tif err := json.Unmarshal(body, rz); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn rz\n\t}\n\trz, err := s.Routez(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Routez: %v\", err)\n\t}\n\treturn rz\n}\n\nfunc TestMonitorConnzWithRoutes(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.Cluster.Name = \"A\"\n\topts.Cluster.Host = \"127.0.0.1\"\n\topts.Cluster.Port = CLUSTER_PORT\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\topts = &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"A\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: -1,\n\t\t},\n\t\tNoLog: true,\n\t\tNoSigs: true,\n\t\tNoSystemAccount: true,\n\t}\n\trouteURL, _ := url.Parse(fmt.Sprintf(\"nats-route://127.0.0.1:%d\", s.ClusterAddr().Port))\n\topts.Routes = []*url.URL{routeURL}\n\n\tstart := time.Now()\n\tsc := RunServer(opts)\n\tdefer sc.Shutdown()\n\n\tcheckClusterFormed(t, s, sc)\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\t\t// Test contents..\n\t\t// Make sure routes don't show up under connz, but do under routez\n\t\tif c.NumConns != 0 {\n\t\t\tt.Fatalf(\"Expected 0 connections, got %d\", c.NumConns)\n\t\t}\n\t\tif c.Conns == nil || len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected 0 connections in array, got %p\", c.Conns)\n\t\t}\n\t}\n\n\tnc := createClientConnSubscribeAndPublish(t, sc)\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"hello.bar\", func(m *nats.Msg) {})\n\tnc.Flush()\n\tcheckExpectedSubs(t, 1, s, sc)\n\n\t// Now check routez\n\turls := []string{\"routez\", \"routez?subs=1\", \"routez?subs=detail\"}\n\tfor subs, urlSuffix := range urls {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\trz := pollRoutez(t, s, mode, url+urlSuffix, &RoutezOptions{Subscriptions: subs == 1, SubscriptionsDetail: subs == 2})\n\n\t\t\tif rz.NumRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\t\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, rz.NumRoutes)\n\t\t\t}\n\n\t\t\tif len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE {\n\t\t\t\tt.Fatalf(\"Expected route array of %d, got %v\", DEFAULT_ROUTE_POOL_SIZE, len(rz.Routes))\n\t\t\t}\n\n\t\t\troute := rz.Routes[0]\n\n\t\t\tif route.DidSolicit {\n\t\t\t\tt.Fatalf(\"Expected unsolicited route, got %v\", route.DidSolicit)\n\t\t\t}\n\n\t\t\tif route.Start.IsZero() {\n\t\t\t\tt.Fatalf(\"Expected Start to be set, got %+v\", route)\n\t\t\t} else if route.Start.Before(start) {\n\t\t\t\tt.Fatalf(\"Unexpected start time: route was started around %v, got %v\", start, route.Start)\n\t\t\t}\n\t\t\tif route.LastActivity.IsZero() {\n\t\t\t\tt.Fatalf(\"Expected LastActivity to be set, got %+v\", route)\n\t\t\t}\n\t\t\tif route.Uptime == _EMPTY_ {\n\t\t\t\tt.Fatalf(\"Expected Uptime to be set, it was not\")\n\t\t\t}\n\t\t\tif route.Idle == _EMPTY_ {\n\t\t\t\tt.Fatalf(\"Expected Idle to be set, it was not\")\n\t\t\t}\n\n\t\t\t// Don't ask for subs, so there should not be any\n\t\t\tif subs == 0 {\n\t\t\t\tif len(route.Subs) != 0 {\n\t\t\t\t\tt.Fatalf(\"There should not be subs, got %v\", len(route.Subs))\n\t\t\t\t}\n\t\t\t} else if subs == 1 {\n\t\t\t\tif len(route.Subs) != 1 && len(route.SubsDetail) != 0 {\n\t\t\t\t\tt.Fatalf(\"There should be 1 sub, got %v\", len(route.Subs))\n\t\t\t\t}\n\t\t\t} else if subs == 2 {\n\t\t\t\tif len(route.SubsDetail) != 1 && len(route.Subs) != 0 {\n\t\t\t\t\tt.Fatalf(\"There should be 1 sub, got %v\", len(route.SubsDetail))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Test JSONP\n\treadBodyEx(t, url+\"routez?callback=callback\", http.StatusOK, appJSContent)\n}\n\nfunc TestMonitorRoutezWithBadParams(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/routez?\", s.MonitorAddr().Port)\n\treadBodyEx(t, url+\"subs=xxx\", http.StatusBadRequest, textPlain)\n}\n\nfunc pollSubsz(t *testing.T, s *Server, mode int, url string, opts *SubszOptions) *Subsz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tbody := readBody(t, url)\n\t\tsz := &Subsz{}\n\t\tif err := json.Unmarshal(body, sz); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn sz\n\t}\n\tsz, err := s.Subsz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Subsz: %v\", err)\n\t}\n\treturn sz\n}\n\nfunc TestSubsz(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz\", nil)\n\n\t\trequire_Equal(t, sl.NumSubs, 0)\n\t\trequire_Equal(t, sl.NumInserts, 1)\n\t\trequire_Equal(t, sl.NumMatches, 1)\n\n\t\tfor _, s := range sl.Subs {\n\t\t\tswitch s.Account {\n\t\t\tcase DEFAULT_GLOBAL_ACCOUNT:\n\t\t\t\trequire_Equal(t, s.AccountTag, DEFAULT_GLOBAL_ACCOUNT)\n\t\t\tcase DEFAULT_SYSTEM_ACCOUNT:\n\t\t\t\trequire_Equal(t, s.AccountTag, DEFAULT_SYSTEM_ACCOUNT)\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unknown account: %q\", s.Account)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Test JSONP\n\treadBodyEx(t, url+\"subsz?callback=callback\", http.StatusOK, appJSContent)\n}\n\nfunc TestSubszOperatorMode(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, sysKp, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\tsysPub, _ := sysKp.PublicKey()\n\taccPub, _ := accKp.PublicKey()\n\n\t_, aCreds := createUser(t, accKp)\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz\", nil)\n\n\t\tfor _, s := range sl.Subs {\n\t\t\tswitch s.Account {\n\t\t\tcase DEFAULT_GLOBAL_ACCOUNT:\n\t\t\t\trequire_Equal(t, s.AccountTag, DEFAULT_GLOBAL_ACCOUNT)\n\t\t\tcase sysPub:\n\t\t\t\trequire_Equal(t, s.AccountTag, sysName)\n\t\t\tcase accPub:\n\t\t\t\trequire_Equal(t, s.AccountTag, accName)\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unknown account: %q\", s.Account)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorSubszDetails(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"foo.*\", func(m *nats.Msg) {})\n\tnc.Subscribe(\"foo.bar\", func(m *nats.Msg) {})\n\tnc.Subscribe(\"foo.foo\", func(m *nats.Msg) {})\n\n\tnc.Publish(\"foo.bar\", []byte(\"Hello\"))\n\tnc.Publish(\"foo.baz\", []byte(\"Hello\"))\n\tnc.Publish(\"foo.foo\", []byte(\"Hello\"))\n\n\tnc.Flush()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz?subs=1\", &SubszOptions{Subscriptions: true})\n\t\tif sl.NumSubs != 3 {\n\t\t\tt.Fatalf(\"Expected NumSubs of 3, got %d\\n\", sl.NumSubs)\n\t\t}\n\t\tif sl.Total != 3 {\n\t\t\tt.Fatalf(\"Expected Total of 3, got %d\\n\", sl.Total)\n\t\t}\n\t\tif len(sl.Subs) != 3 {\n\t\t\tt.Fatalf(\"Expected subscription details for 3 subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t}\n}\n\nfunc TestMonitorSubszWithOffsetAndLimit(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tfor i := 0; i < 200; i++ {\n\t\tnc.Subscribe(fmt.Sprintf(\"foo.%d\", i), func(m *nats.Msg) {})\n\t}\n\tnc.Flush()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz?subs=1&offset=10&limit=100\", &SubszOptions{Subscriptions: true, Offset: 10, Limit: 100})\n\t\tif sl.NumSubs != 200 {\n\t\t\tt.Fatalf(\"Expected NumSubs of 200, got %d\\n\", sl.NumSubs)\n\t\t}\n\t\tif sl.Total != 200 {\n\t\t\tt.Fatalf(\"Expected Total of 200, got %d\\n\", sl.Total)\n\t\t}\n\t\tif sl.Offset != 10 {\n\t\t\tt.Fatalf(\"Expected Offset of 10, got %d\\n\", sl.Offset)\n\t\t}\n\t\tif sl.Limit != 100 {\n\t\t\tt.Fatalf(\"Expected Total of 100, got %d\\n\", sl.Limit)\n\t\t}\n\t\tif len(sl.Subs) != 100 {\n\t\t\tt.Fatalf(\"Expected subscription details for 100 subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t}\n}\n\nfunc TestMonitorSubszTestPubSubject(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"foo.*\", func(m *nats.Msg) {})\n\tnc.Subscribe(\"foo.bar\", func(m *nats.Msg) {})\n\tnc.Subscribe(\"foo.foo\", func(m *nats.Msg) {})\n\tnc.Flush()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz?subs=1&test=foo.foo\", &SubszOptions{Subscriptions: true, Test: \"foo.foo\"})\n\t\tif sl.Total != 2 {\n\t\t\tt.Fatalf(\"Expected Total of 2 match, got %d\\n\", sl.Total)\n\t\t}\n\t\tif len(sl.Subs) != 2 {\n\t\t\tt.Fatalf(\"Expected subscription details for 2 matching subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t\tsl = pollSubsz(t, s, mode, url+\"subsz?subs=1&test=foo\", &SubszOptions{Subscriptions: true, Test: \"foo\"})\n\t\tif len(sl.Subs) != 0 {\n\t\t\tt.Fatalf(\"Expected no matching subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t}\n\t// Make sure we get an error with invalid test subject.\n\ttestUrl := url + \"subsz?subs=1&\"\n\treadBodyEx(t, testUrl+\"test=*\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, testUrl+\"test=foo.*\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, testUrl+\"test=foo.>\", http.StatusBadRequest, textPlain)\n\treadBodyEx(t, testUrl+\"test=foo..bar\", http.StatusBadRequest, textPlain)\n}\n\nfunc TestMonitorSubszMultiAccount(t *testing.T) {\n\ts := runMonitorServerWithAccounts()\n\tdefer s.Shutdown()\n\n\tncA := createClientConnWithUserSubscribeAndPublish(t, s, \"a\", \"a\")\n\tdefer ncA.Close()\n\n\tncA.Subscribe(\"foo.*\", func(m *nats.Msg) {})\n\tncA.Subscribe(\"foo.bar\", func(m *nats.Msg) {})\n\tncA.Subscribe(\"foo.foo\", func(m *nats.Msg) {})\n\n\tncA.Publish(\"foo.bar\", []byte(\"Hello\"))\n\tncA.Publish(\"foo.baz\", []byte(\"Hello\"))\n\tncA.Publish(\"foo.foo\", []byte(\"Hello\"))\n\n\tncA.Flush()\n\n\tncB := createClientConnWithUserSubscribeAndPublish(t, s, \"b\", \"b\")\n\tdefer ncB.Close()\n\n\tncB.Subscribe(\"foo.*\", func(m *nats.Msg) {})\n\tncB.Subscribe(\"foo.bar\", func(m *nats.Msg) {})\n\tncB.Subscribe(\"foo.foo\", func(m *nats.Msg) {})\n\n\tncB.Publish(\"foo.bar\", []byte(\"Hello\"))\n\tncB.Publish(\"foo.baz\", []byte(\"Hello\"))\n\tncB.Publish(\"foo.foo\", []byte(\"Hello\"))\n\n\tncB.Flush()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz?subs=1\", &SubszOptions{Subscriptions: true})\n\t\tif sl.NumSubs != 6 {\n\t\t\tt.Fatalf(\"Expected NumSubs of 6, got %d\\n\", sl.NumSubs)\n\t\t}\n\t\tif sl.Total != 6 {\n\t\t\tt.Fatalf(\"Expected Total of 6, got %d\\n\", sl.Total)\n\t\t}\n\t\tif len(sl.Subs) != 6 {\n\t\t\tt.Fatalf(\"Expected subscription details for 6 subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t\tfor _, sd := range sl.Subs {\n\t\t\tif sd.Account != \"A\" && sd.Account != \"B\" {\n\t\t\t\tt.Fatalf(\"Expected account information to be present and be 'A' or 'B', got %q\", sd.Account)\n\t\t\t}\n\t\t}\n\n\t\t// Now make sure we can filter on account.\n\t\tsl = pollSubsz(t, s, mode, url+\"subsz?subs=1&acc=A\", &SubszOptions{Account: \"A\", Subscriptions: true})\n\t\tif sl.NumSubs != 3 {\n\t\t\tt.Fatalf(\"Expected NumSubs of 3, got %d\\n\", sl.NumSubs)\n\t\t}\n\t\tif sl.Total != 3 {\n\t\t\tt.Fatalf(\"Expected Total of 6, got %d\\n\", sl.Total)\n\t\t}\n\t\tif len(sl.Subs) != 3 {\n\t\t\tt.Fatalf(\"Expected subscription details for 6 subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t\tfor _, sd := range sl.Subs {\n\t\t\tif sd.Account != \"A\" {\n\t\t\t\tt.Fatalf(\"Expected account information to be present and be 'A', got %q\", sd.Account)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorSubszMultiAccountWithOffsetAndLimit(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tncA := createClientConnWithUserSubscribeAndPublish(t, s, \"a\", \"a\")\n\tdefer ncA.Close()\n\n\tfor i := 0; i < 200; i++ {\n\t\tncA.Subscribe(fmt.Sprintf(\"foo.%d\", i), func(m *nats.Msg) {})\n\t}\n\tncA.Flush()\n\n\tncB := createClientConnWithUserSubscribeAndPublish(t, s, \"b\", \"b\")\n\tdefer ncB.Close()\n\n\tfor i := 0; i < 200; i++ {\n\t\tncB.Subscribe(fmt.Sprintf(\"foo.%d\", i), func(m *nats.Msg) {})\n\t}\n\tncB.Flush()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tsl := pollSubsz(t, s, mode, url+\"subsz?subs=1&offset=10&limit=100\", &SubszOptions{Subscriptions: true, Offset: 10, Limit: 100})\n\t\tif sl.NumSubs != 400 {\n\t\t\tt.Fatalf(\"Expected NumSubs of 200, got %d\\n\", sl.NumSubs)\n\t\t}\n\t\tif sl.Total != 400 {\n\t\t\tt.Fatalf(\"Expected Total of 400, got %d\\n\", sl.Total)\n\t\t}\n\t\tif sl.Offset != 10 {\n\t\t\tt.Fatalf(\"Expected Offset of 10, got %d\\n\", sl.Offset)\n\t\t}\n\t\tif sl.Limit != 100 {\n\t\t\tt.Fatalf(\"Expected Total of 100, got %d\\n\", sl.Limit)\n\t\t}\n\t\tif len(sl.Subs) != 100 {\n\t\t\tt.Fatalf(\"Expected subscription details for 100 subs, got %d\\n\", len(sl.Subs))\n\t\t}\n\t}\n}\n\n// Tests handle root\nfunc TestMonitorHandleRoot(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnc := createClientConnSubscribeAndPublish(t, s)\n\tdefer nc.Close()\n\n\tresp, err := http.Get(fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != http.StatusOK {\n\t\tt.Fatalf(\"Expected a %d response, got %d\\n\", http.StatusOK, resp.StatusCode)\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error reading body: Got %v\\n\", err)\n\t}\n\tfor _, b := range body {\n\t\tif b > unicode.MaxASCII {\n\t\t\tt.Fatalf(\"Expected body to contain only ASCII characters, but got %v\\n\", b)\n\t\t}\n\t}\n\n\tct := resp.Header.Get(\"Content-Type\")\n\tif !strings.Contains(ct, \"text/html\") {\n\t\tt.Fatalf(\"Expected text/html response, got %s\\n\", ct)\n\t}\n}\n\nfunc TestMonitorConnzWithNamedClient(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tclientName := \"test-client\"\n\tnc := createClientConnWithName(t, clientName, s)\n\tdefer nc.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\t// Confirm server is exposing client name in monitoring endpoint.\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\t\tgot := len(c.Conns)\n\t\texpected := 1\n\t\tif got != expected {\n\t\t\tt.Fatalf(\"Expected %d connection in array, got %d\\n\", expected, got)\n\t\t}\n\n\t\tconn := c.Conns[0]\n\t\tif conn.Name != clientName {\n\t\t\tt.Fatalf(\"Expected client to have name %q. got %q\", clientName, conn.Name)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzWithStateForClosedConns(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tnumEach := 10\n\t// Create 10 closed, and 10 to leave open.\n\tfor i := 0; i < numEach; i++ {\n\t\tnc := createClientConnSubscribeAndPublish(t, s)\n\t\tnc.Subscribe(\"hello.closed.conns\", func(m *nats.Msg) {})\n\t\tnc.Close()\n\t\tnc = createClientConnSubscribeAndPublish(t, s)\n\t\tnc.Subscribe(\"hello.open.conns\", func(m *nats.Msg) {})\n\t\tdefer nc.Close()\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\t// Look at all open\n\t\t\tc := pollConnz(t, s, mode, url+\"connz?state=open\", &ConnzOptions{State: ConnOpen})\n\t\t\tif lc := len(c.Conns); lc != numEach {\n\t\t\t\treturn fmt.Errorf(\"Expected %d connections in array, got %d\", numEach, lc)\n\t\t\t}\n\t\t\t// Look at all closed\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?state=closed\", &ConnzOptions{State: ConnClosed})\n\t\t\tif lc := len(c.Conns); lc != numEach {\n\t\t\t\treturn fmt.Errorf(\"Expected %d connections in array, got %d\", numEach, lc)\n\t\t\t}\n\t\t\t// Look at all\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?state=ALL\", &ConnzOptions{State: ConnAll})\n\t\t\tif lc := len(c.Conns); lc != numEach*2 {\n\t\t\t\treturn fmt.Errorf(\"Expected %d connections in array, got %d\", 2*numEach, lc)\n\t\t\t}\n\t\t\t// Look at CID #1, which is in closed.\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=1&state=open\", &ConnzOptions{CID: 1, State: ConnOpen})\n\t\t\tif lc := len(c.Conns); lc != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no connections in open array, got %d\", lc)\n\t\t\t}\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=1&state=closed\", &ConnzOptions{CID: 1, State: ConnClosed})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in closed array, got %d\", lc)\n\t\t\t}\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=1&state=ALL\", &ConnzOptions{CID: 1, State: ConnAll})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in closed array, got %d\", lc)\n\t\t\t}\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=1&state=closed&subs=true\",\n\t\t\t\t&ConnzOptions{CID: 1, State: ConnClosed, Subscriptions: true})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in closed array, got %d\", lc)\n\t\t\t}\n\t\t\tci := c.Conns[0]\n\t\t\tif ci.NumSubs != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected NumSubs to be 1, got %d\", ci.NumSubs)\n\t\t\t}\n\t\t\tif len(ci.Subs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected len(ci.Subs) to be 1 also, got %d\", len(ci.Subs))\n\t\t\t}\n\t\t\t// Now ask for same thing without subs and make sure they are not returned.\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=1&state=closed&subs=false\",\n\t\t\t\t&ConnzOptions{CID: 1, State: ConnClosed, Subscriptions: false})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in closed array, got %d\", lc)\n\t\t\t}\n\t\t\tci = c.Conns[0]\n\t\t\tif ci.NumSubs != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected NumSubs to be 1, got %d\", ci.NumSubs)\n\t\t\t}\n\t\t\tif len(ci.Subs) != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected len(ci.Subs) to be 0 since subs=false, got %d\", len(ci.Subs))\n\t\t\t}\n\n\t\t\t// CID #2 is in open\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=2&state=open\", &ConnzOptions{CID: 2, State: ConnOpen})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in open array, got %d\", lc)\n\t\t\t}\n\t\t\t// It should also work if we ask for \"state=all\"\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=2&state=all\", &ConnzOptions{CID: 2, State: ConnAll})\n\t\t\tif lc := len(c.Conns); lc != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected a connection in open array, got %d\", lc)\n\t\t\t}\n\t\t\t// But not for \"state=closed\"\n\t\t\tc = pollConnz(t, s, mode, url+\"connz?cid=2&state=closed\", &ConnzOptions{CID: 2, State: ConnClosed})\n\t\t\tif lc := len(c.Conns); lc != 0 {\n\t\t\t\treturn fmt.Errorf(\"Expected no connections in closed array, got %d\", lc)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n}\n\n// Make sure options for ConnInfo like subs=1, authuser, etc do not cause a race.\nfunc TestMonitorConnzClosedConnsRace(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\t// Create 100 closed connections.\n\tfor i := 0; i < 100; i++ {\n\t\tnc := createClientConnSubscribeAndPublish(t, s)\n\t\tnc.Close()\n\t}\n\n\turlWithoutSubs := fmt.Sprintf(\"http://127.0.0.1:%d/connz?state=closed\", s.MonitorAddr().Port)\n\turlWithSubs := urlWithoutSubs + \"&subs=true\"\n\n\tcheckClosedConns(t, s, 100, 2*time.Second)\n\n\twg := &sync.WaitGroup{}\n\n\tfn := func(url string) {\n\t\tdeadline := time.Now().Add(1 * time.Second)\n\t\tfor time.Now().Before(deadline) {\n\t\t\tc := pollConnz(t, s, 0, url, nil)\n\t\t\tif len(c.Conns) != 100 {\n\t\t\t\tt.Errorf(\"Incorrect Results: %+v\\n\", c)\n\t\t\t}\n\t\t}\n\t\twg.Done()\n\t}\n\n\twg.Add(2)\n\tgo fn(urlWithSubs)\n\tgo fn(urlWithoutSubs)\n\twg.Wait()\n}\n\n// Make sure a bad client that is disconnected right away has proper values.\nfunc TestMonitorConnzClosedConnsBadClient(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\n\trc, err := net.Dial(\"tcp\", net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\trc.Close()\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\n\tc := pollConnz(t, s, 1, \"\", &ConnzOptions{State: ConnClosed})\n\tif len(c.Conns) != 1 {\n\t\tt.Errorf(\"Incorrect Results: %+v\\n\", c)\n\t}\n\tci := c.Conns[0]\n\n\tuptime := ci.Stop.Sub(ci.Start)\n\tidle, err := time.ParseDuration(ci.Idle)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse Idle: %v\\n\", err)\n\t}\n\tif idle > uptime {\n\t\tt.Fatalf(\"Idle can't be larger then uptime, %v vs %v\\n\", idle, uptime)\n\t}\n\tif ci.LastActivity.IsZero() {\n\t\tt.Fatalf(\"LastActivity should not be Zero\\n\")\n\t}\n}\n\n// Make sure a bad client that tries to connect plain to TLS has proper values.\nfunc TestMonitorConnzClosedConnsBadTLSClient(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\ttc := &TLSConfigOpts{}\n\ttc.CertFile = \"configs/certs/server.pem\"\n\ttc.KeyFile = \"configs/certs/key.pem\"\n\n\tvar err error\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.TLSTimeout = 1.5 // 1.5 seconds\n\topts.TLSConfig, err = GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating TSL config: %v\", err)\n\t}\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\topts = s.getOpts()\n\n\trc, err := net.Dial(\"tcp\", net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\trc.Write([]byte(\"CONNECT {}\\r\\n\"))\n\trc.Close()\n\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\n\tc := pollConnz(t, s, 1, \"\", &ConnzOptions{State: ConnClosed})\n\tif len(c.Conns) != 1 {\n\t\tt.Errorf(\"Incorrect Results: %+v\\n\", c)\n\t}\n\tci := c.Conns[0]\n\n\tuptime := ci.Stop.Sub(ci.Start)\n\tidle, err := time.ParseDuration(ci.Idle)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse Idle: %v\\n\", err)\n\t}\n\tif idle > uptime {\n\t\tt.Fatalf(\"Idle can't be larger then uptime, %v vs %v\\n\", idle, uptime)\n\t}\n\tif ci.LastActivity.IsZero() {\n\t\tt.Fatalf(\"LastActivity should not be Zero\\n\")\n\t}\n}\n\n// Create a connection to test ConnInfo\nfunc createClientConnWithUserSubscribeAndPublish(t *testing.T, s *Server, user, pwd string) *nats.Conn {\n\tnatsURL := \"\"\n\tif user == \"\" {\n\t\tnatsURL = fmt.Sprintf(\"nats://127.0.0.1:%d\", s.Addr().(*net.TCPAddr).Port)\n\t} else {\n\t\tnatsURL = fmt.Sprintf(\"nats://%s:%s@127.0.0.1:%d\", user, pwd, s.Addr().(*net.TCPAddr).Port)\n\t}\n\tclient := nats.GetDefaultOptions()\n\tclient.Servers = []string{natsURL}\n\tnc, err := client.Connect()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v to: %s\\n\", err, natsURL)\n\t}\n\n\tch := make(chan bool)\n\tinbox := nats.NewInbox()\n\tsub, err := nc.Subscribe(inbox, func(m *nats.Msg) { ch <- true })\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing to `%s`: %v\\n\", inbox, err)\n\t}\n\tnc.Publish(inbox, []byte(\"Hello\"))\n\t// Wait for message\n\t<-ch\n\tsub.Unsubscribe()\n\tclose(ch)\n\tnc.Flush()\n\treturn nc\n}\n\nfunc createClientConnSubscribeAndPublish(t *testing.T, s *Server) *nats.Conn {\n\treturn createClientConnWithUserSubscribeAndPublish(t, s, \"\", \"\")\n}\n\nfunc createClientConnWithName(t *testing.T, name string, s *Server) *nats.Conn {\n\tnatsURI := fmt.Sprintf(\"nats://127.0.0.1:%d\", s.Addr().(*net.TCPAddr).Port)\n\n\tclient := nats.GetDefaultOptions()\n\tclient.Servers = []string{natsURI}\n\tclient.Name = name\n\tnc, err := client.Connect()\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\treturn nc\n}\n\nfunc TestMonitorStacksz(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tbody := readBody(t, url+\"stacksz\")\n\t// Check content\n\tstr := string(body)\n\tif !strings.Contains(str, \"HandleStacksz\") {\n\t\tt.Fatalf(\"Result does not seem to contain server's stacks:\\n%v\", str)\n\t}\n}\n\nfunc TestMonitorConcurrentMonitoring(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\t// Get some endpoints. Make sure we have at least varz,\n\t// and the more the merrier.\n\tendpoints := []string{\"varz\", \"varz\", \"varz\", \"connz\", \"connz\", \"subsz\", \"subsz\", \"routez\", \"routez\"}\n\twg := &sync.WaitGroup{}\n\twg.Add(len(endpoints))\n\tech := make(chan string, len(endpoints))\n\n\tfor _, e := range endpoints {\n\t\tgo func(endpoint string) {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; i < 50; i++ {\n\t\t\t\tresp, err := http.Get(url + endpoint)\n\t\t\t\tif err != nil {\n\t\t\t\t\tech <- fmt.Sprintf(\"Expected no error: Got %v\\n\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdefer resp.Body.Close()\n\t\t\t\tif resp.StatusCode != http.StatusOK {\n\t\t\t\t\tech <- fmt.Sprintf(\"Expected a %v response, got %d\\n\", http.StatusOK, resp.StatusCode)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tct := resp.Header.Get(\"Content-Type\")\n\t\t\t\tif ct != \"application/json\" {\n\t\t\t\t\tech <- fmt.Sprintf(\"Expected application/json content-type, got %s\\n\", ct)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif _, err := io.ReadAll(resp.Body); err != nil {\n\t\t\t\t\tech <- fmt.Sprintf(\"Got an error reading the body: %v\\n\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tresp.Body.Close()\n\t\t\t}\n\t\t}(e)\n\t}\n\twg.Wait()\n\t// Check for any errors\n\tselect {\n\tcase err := <-ech:\n\t\tt.Fatal(err)\n\tdefault:\n\t}\n}\n\nfunc TestMonitorHandler(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\thandler := s.HTTPHandler()\n\tif handler == nil {\n\t\tt.Fatal(\"HTTP Handler should be set\")\n\t}\n\ts.Shutdown()\n\thandler = s.HTTPHandler()\n\tif handler != nil {\n\t\tt.Fatal(\"HTTP Handler should be nil\")\n\t}\n}\n\nfunc TestMonitorRoutezRace(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\tsrvAOpts := DefaultMonitorOptions()\n\tsrvAOpts.NoSystemAccount = true\n\tsrvAOpts.Cluster.Name = \"B\"\n\tsrvAOpts.Cluster.Port = -1\n\tsrvA := RunServer(srvAOpts)\n\tdefer srvA.Shutdown()\n\n\tsrvBOpts := nextServerOpts(srvAOpts)\n\tsrvBOpts.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\n\tdoneCh := make(chan struct{})\n\tgo func() {\n\t\tdefer func() {\n\t\t\tdoneCh <- struct{}{}\n\t\t}()\n\t\tfor i := 0; i < 10; i++ {\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\t// Reset ports\n\t\t\tsrvBOpts.Port = -1\n\t\t\tsrvBOpts.Cluster.Port = -1\n\t\t\tsrvB := RunServer(srvBOpts)\n\t\t\ttime.Sleep(20 * time.Millisecond)\n\t\t\tsrvB.Shutdown()\n\t\t}\n\t}()\n\tdone := false\n\tfor !done {\n\t\tif _, err := srvA.Routez(nil); err != nil {\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t}\n\t\tselect {\n\t\tcase <-doneCh:\n\t\t\tdone = true\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzTLSInHandshake(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\ttc := &TLSConfigOpts{}\n\ttc.CertFile = \"configs/certs/server.pem\"\n\ttc.KeyFile = \"configs/certs/key.pem\"\n\n\tvar err error\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.TLSTimeout = 1.5 // 1.5 seconds\n\topts.TLSConfig, err = GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating TSL config: %v\", err)\n\t}\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Create bare TCP connection to delay client TLS handshake\n\tc, err := net.Dial(\"tcp\", net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\tdefer c.Close()\n\n\t// Wait for the connection to be registered\n\tcheckClientsCount(t, s, 1)\n\n\tstart := time.Now()\n\tendpoint := fmt.Sprintf(\"http://%s:%d/connz\", opts.HTTPHost, s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tconnz := pollConnz(t, s, mode, endpoint, nil)\n\t\tduration := time.Since(start)\n\t\tif duration >= 1500*time.Millisecond {\n\t\t\tt.Fatalf(\"Looks like connz blocked on handshake, took %v\", duration)\n\t\t}\n\t\tif len(connz.Conns) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 conn, got %v\", len(connz.Conns))\n\t\t}\n\t\tconn := connz.Conns[0]\n\t\t// TLS fields should be not set\n\t\tif conn.TLSVersion != \"\" || conn.TLSCipher != \"\" {\n\t\t\tt.Fatalf(\"Expected TLS fields to not be set, got version:%v cipher:%v\", conn.TLSVersion, conn.TLSCipher)\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzTLSCfg(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\ttc := &TLSConfigOpts{}\n\ttc.CertFile = \"configs/certs/server.pem\"\n\ttc.KeyFile = \"configs/certs/key.pem\"\n\n\tvar err error\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.TLSTimeout = 1.5 // 1.5 seconds\n\topts.TLSConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\topts.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert\n\topts.Gateway.TLSConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\topts.Gateway.TLSTimeout = 1.5\n\topts.LeafNode.TLSConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\topts.LeafNode.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert\n\topts.LeafNode.TLSTimeout = 1.5\n\topts.Cluster.TLSConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\topts.Cluster.TLSTimeout = 1.5\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheck := func(verify, required bool, timeout float64) {\n\t\tt.Helper()\n\t\tif !verify {\n\t\t\tt.Fatalf(\"Expected tls_verify to be true\")\n\t\t}\n\t\tif !required {\n\t\t\tt.Fatalf(\"Expected tls_required to be true\")\n\t\t}\n\t\tif timeout != 1.5 {\n\t\t\tt.Fatalf(\"Expected tls_timeout to be 1.5\")\n\t\t}\n\t}\n\n\tstart := time.Now()\n\tendpoint := fmt.Sprintf(\"http://%s:%d/varz\", opts.HTTPHost, s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tvarz := pollVarz(t, s, mode, endpoint, nil)\n\t\tduration := time.Since(start)\n\t\tif duration >= 1500*time.Millisecond {\n\t\t\tt.Fatalf(\"Looks like varz blocked on handshake, took %v\", duration)\n\t\t}\n\t\tcheck(varz.TLSVerify, varz.TLSRequired, varz.TLSTimeout)\n\t\tcheck(varz.Cluster.TLSVerify, varz.Cluster.TLSRequired, varz.Cluster.TLSTimeout)\n\t\tcheck(varz.Gateway.TLSVerify, varz.Gateway.TLSRequired, varz.Gateway.TLSTimeout)\n\t\tcheck(varz.LeafNode.TLSVerify, varz.LeafNode.TLSRequired, varz.LeafNode.TLSTimeout)\n\t}\n}\n\nfunc TestMonitorConnzTLSPeerCerts(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\ttc := &TLSConfigOpts{}\n\ttc.CertFile = \"../test/configs/certs/server-cert.pem\"\n\ttc.KeyFile = \"../test/configs/certs/server-key.pem\"\n\ttc.CaFile = \"../test/configs/certs/ca.pem\"\n\ttc.Verify = true\n\ttc.Timeout = 2.0\n\n\tvar err error\n\topts := DefaultMonitorOptions()\n\topts.TLSConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(),\n\t\tnats.ClientCert(\"../test/configs/certs/client-cert.pem\", \"../test/configs/certs/client-key.pem\"),\n\t\tnats.RootCAs(\"../test/configs/certs/ca.pem\"))\n\tdefer nc.Close()\n\n\tendpoint := fmt.Sprintf(\"http://%s:%d/connz\", opts.HTTPHost, s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\t// Without \"auth\" option, we should not get the details\n\t\tconnz := pollConnz(t, s, mode, endpoint, nil)\n\t\trequire_True(t, len(connz.Conns) == 1)\n\t\tc := connz.Conns[0]\n\t\tif c.TLSPeerCerts != nil {\n\t\t\tt.Fatalf(\"Did not expect TLSPeerCerts when auth is not specified: %+v\", c.TLSPeerCerts)\n\t\t}\n\t\t// Now specify \"auth\" option\n\t\tconnz = pollConnz(t, s, mode, endpoint+\"?auth=1\", &ConnzOptions{Username: true})\n\t\trequire_True(t, len(connz.Conns) == 1)\n\t\tc = connz.Conns[0]\n\t\tif c.TLSPeerCerts == nil {\n\t\t\tt.Fatal(\"Expected TLSPeerCerts to be set, was not\")\n\t\t} else if len(c.TLSPeerCerts) != 1 {\n\t\t\tt.Fatalf(\"Unexpected peer certificates: %+v\", c.TLSPeerCerts)\n\t\t} else {\n\t\t\tfor _, d := range c.TLSPeerCerts {\n\t\t\t\tif d.Subject != \"CN=localhost,OU=nats.io,O=Synadia,ST=California,C=US\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected subject: %s\", d.Subject)\n\t\t\t\t}\n\t\t\t\tif len(d.SubjectPKISha256) != 64 {\n\t\t\t\t\tt.Fatalf(\"Unexpected spki_sha256: %s\", d.SubjectPKISha256)\n\t\t\t\t}\n\t\t\t\tif len(d.CertSha256) != 64 {\n\t\t\t\t\tt.Fatalf(\"Unexpected cert_sha256: %s\", d.CertSha256)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorServerIDs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tmurl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s, mode, murl+\"varz\", nil)\n\t\tif v.ID == _EMPTY_ {\n\t\t\tt.Fatal(\"Varz ID is empty\")\n\t\t}\n\t\tc := pollConnz(t, s, mode, murl+\"connz\", nil)\n\t\tif c.ID == _EMPTY_ {\n\t\t\tt.Fatal(\"Connz ID is empty\")\n\t\t}\n\t\tr := pollRoutez(t, s, mode, murl+\"routez\", nil)\n\t\tif r.ID == _EMPTY_ {\n\t\t\tt.Fatal(\"Routez ID is empty\")\n\t\t}\n\t\tif v.ID != c.ID || v.ID != r.ID {\n\t\t\tt.Fatalf(\"Varz ID [%s] is not equal to Connz ID [%s] or Routez ID [%s]\", v.ID, c.ID, r.ID)\n\t\t}\n\t}\n}\n\nfunc TestMonitorHttpStatsNoUpdatedWhenUsingServerFuncs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfor i := 0; i < 10; i++ {\n\t\ts.Varz(nil)\n\t\ts.Connz(nil)\n\t\ts.Routez(nil)\n\t\ts.Subsz(nil)\n\t}\n\n\tv, _ := s.Varz(nil)\n\tendpoints := []string{VarzPath, ConnzPath, RoutezPath, SubszPath}\n\tfor _, e := range endpoints {\n\t\tstats := v.HTTPReqStats[e]\n\t\tif stats != 0 {\n\t\t\tt.Fatalf(\"Expected HTTPReqStats for %q to be 0, got %v\", e, stats)\n\t\t}\n\t}\n}\n\nfunc TestMonitorClusterEmptyWhenNotDefined(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tbody := readBody(t, fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port))\n\tvar v map[string]any\n\tif err := json.Unmarshal(body, &v); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\t// Cluster can empty, or be defined but that needs to be empty.\n\tc, ok := v[\"cluster\"]\n\tif !ok {\n\t\treturn\n\t}\n\tif len(c.(map[string]any)) != 0 {\n\t\tt.Fatalf(\"Expected an empty cluster definition, instead got %+v\\n\", c)\n\t}\n}\n\nfunc TestMonitorRoutezPermissions(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.Cluster.Name = \"A\"\n\topts.Cluster.Host = \"127.0.0.1\"\n\topts.Cluster.Port = -1\n\topts.Cluster.Permissions = &RoutePermissions{\n\t\tImport: &SubjectPermission{\n\t\t\tAllow: []string{\"foo\"},\n\t\t},\n\t\tExport: &SubjectPermission{\n\t\t\tAllow: []string{\"*\"},\n\t\t\tDeny: []string{\"foo\", \"nats\"},\n\t\t},\n\t}\n\n\ts1 := RunServer(opts)\n\tdefer s1.Shutdown()\n\n\topts = DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.ServerName = \"monitor_server_2\"\n\topts.Cluster.Host = \"127.0.0.1\"\n\topts.Cluster.Name = \"A\"\n\topts.Cluster.Port = -1\n\trouteURL, _ := url.Parse(fmt.Sprintf(\"nats-route://127.0.0.1:%d\", s1.ClusterAddr().Port))\n\topts.Routes = []*url.URL{routeURL}\n\topts.HTTPPort = -1\n\n\ts2 := RunServer(opts)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\turls := []string{\n\t\tfmt.Sprintf(\"http://127.0.0.1:%d/routez\", s1.MonitorAddr().Port),\n\t\tfmt.Sprintf(\"http://127.0.0.1:%d/routez\", s2.MonitorAddr().Port),\n\t}\n\tservers := []*Server{s1, s2}\n\n\tfor i, url := range urls {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\trz := pollRoutez(t, servers[i], mode, url, nil)\n\t\t\t// For server 1, we expect to see imports and exports\n\t\t\tif i == 0 {\n\t\t\t\tif rz.Import == nil || rz.Import.Allow == nil ||\n\t\t\t\t\tlen(rz.Import.Allow) != 1 || rz.Import.Allow[0] != \"foo\" ||\n\t\t\t\t\trz.Import.Deny != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected Import %v\", rz.Import)\n\t\t\t\t}\n\t\t\t\tif rz.Export == nil || rz.Export.Allow == nil || rz.Export.Deny == nil ||\n\t\t\t\t\tlen(rz.Export.Allow) != 1 || rz.Export.Allow[0] != \"*\" ||\n\t\t\t\t\tlen(rz.Export.Deny) != 2 || rz.Export.Deny[0] != \"foo\" || rz.Export.Deny[1] != \"nats\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected Export %v\", rz.Export)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// We expect to see NO imports and exports for server B by default.\n\t\t\t\tif rz.Import != nil {\n\t\t\t\t\tt.Fatal(\"Routez body should NOT contain \\\"import\\\" information.\")\n\t\t\t\t}\n\t\t\t\tif rz.Export != nil {\n\t\t\t\t\tt.Fatal(\"Routez body should NOT contain \\\"export\\\" information.\")\n\t\t\t\t}\n\t\t\t\t// We do expect to see them show up for the information we have on Server A though.\n\t\t\t\tif len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE {\n\t\t\t\t\tt.Fatalf(\"Expected route array of %d, got %v\\n\", DEFAULT_ROUTE_POOL_SIZE, len(rz.Routes))\n\t\t\t\t}\n\t\t\t\troute := rz.Routes[0]\n\t\t\t\tif route.Import == nil || route.Import.Allow == nil ||\n\t\t\t\t\tlen(route.Import.Allow) != 1 || route.Import.Allow[0] != \"foo\" ||\n\t\t\t\t\troute.Import.Deny != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected Import %v\", route.Import)\n\t\t\t\t}\n\t\t\t\tif route.Export == nil || route.Export.Allow == nil || route.Export.Deny == nil ||\n\t\t\t\t\tlen(route.Export.Allow) != 1 || route.Export.Allow[0] != \"*\" ||\n\t\t\t\t\tlen(route.Export.Deny) != 2 || route.Export.Deny[0] != \"foo\" || route.Export.Deny[1] != \"nats\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected Export %v\", route.Export)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Benchmark our Connz generation. Don't use HTTP here, just measure server endpoint.\nfunc Benchmark_Connz(b *testing.B) {\n\truntime.MemProfileRate = 0\n\n\ts := runMonitorServerNoHTTPPort()\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// Create 250 connections with 100 subs each.\n\tfor i := 0; i < 250; i++ {\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Error on connection[%d] to %s: %v\", i, url, err)\n\t\t}\n\t\tfor x := 0; x < 100; x++ {\n\t\t\tsubj := fmt.Sprintf(\"foo.%d\", x)\n\t\t\tnc.Subscribe(subj, func(m *nats.Msg) {})\n\t\t}\n\t\tnc.Flush()\n\t\tdefer nc.Close()\n\t}\n\n\tb.ResetTimer()\n\truntime.MemProfileRate = 1\n\n\tcopts := &ConnzOptions{Subscriptions: false}\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := s.Connz(copts)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Error on Connz(): %v\", err)\n\t\t}\n\t}\n}\n\nfunc Benchmark_Varz(b *testing.B) {\n\truntime.MemProfileRate = 0\n\n\ts := runMonitorServerNoHTTPPort()\n\tdefer s.Shutdown()\n\n\tb.ResetTimer()\n\truntime.MemProfileRate = 1\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := s.Varz(nil)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Error on Connz(): %v\", err)\n\t\t}\n\t}\n}\n\nfunc Benchmark_VarzHttp(b *testing.B) {\n\truntime.MemProfileRate = 0\n\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tmurl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\n\tb.ResetTimer()\n\truntime.MemProfileRate = 1\n\n\tfor i := 0; i < b.N; i++ {\n\t\tv := &Varz{}\n\t\tresp, err := http.Get(murl)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tbody, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t\t}\n\t\tif err := json.Unmarshal(body, v); err != nil {\n\t\t\tb.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\tresp.Body.Close()\n\t}\n}\n\nfunc TestMonitorVarzRaces(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tmurl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tdone := make(chan struct{})\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tv := pollVarz(t, s, i, murl, nil)\n\t\t\t\t// Check the field that we are setting in main thread\n\t\t\t\t// to ensure that we have a copy and there is no\n\t\t\t\t// race with fields set in s.info and s.opts\n\t\t\t\tif v.ID == \"abc\" || v.MaxConn == -1 {\n\t\t\t\t\t// We will not get there. Need to have something\n\t\t\t\t\t// otherwise staticcheck will report empty branch\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tselect {\n\t\t\t\tcase <-done:\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < 1000; i++ {\n\t\t// Simulate a change in server's info and options\n\t\t// by changing something.\n\t\ts.mu.Lock()\n\t\ts.info.ID = fmt.Sprintf(\"serverid_%d\", i)\n\t\ts.opts.MaxConn = 100 + i\n\t\ts.mu.Unlock()\n\t\ttime.Sleep(time.Nanosecond)\n\t}\n\tclose(done)\n\twg.Wait()\n\n\t// Now check that there is no race doing parallel polling\n\twg.Add(3)\n\tdone = make(chan struct{})\n\tpoll := func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\t\tpollVarz(t, s, mode, murl, nil)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}\n\tfor i := 0; i < 3; i++ {\n\t\tgo poll()\n\t}\n\ttime.Sleep(500 * time.Millisecond)\n\tclose(done)\n\twg.Wait()\n}\n\nfunc testMonitorStructPresent(t *testing.T, tag string) {\n\tt.Helper()\n\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tbody := readBody(t, varzURL)\n\tif !bytes.Contains(body, []byte(`\"`+tag+`\": {}`)) {\n\t\tt.Fatalf(\"%s should be present and empty, got %s\", tag, body)\n\t}\n}\n\nfunc TestMonitorCluster(t *testing.T) {\n\ttestMonitorStructPresent(t, \"cluster\")\n\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.Cluster.Name = \"A\"\n\topts.Cluster.Port = -1\n\topts.Cluster.AuthTimeout = 1\n\topts.Routes = RoutesFromStr(\"nats://127.0.0.1:1234\")\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\texpected := ClusterOptsVarz{\n\t\t\"A\",\n\t\topts.Cluster.Host,\n\t\topts.Cluster.Port,\n\t\topts.Cluster.AuthTimeout,\n\t\t[]string{\"127.0.0.1:1234\"},\n\t\topts.Cluster.TLSTimeout,\n\t\topts.Cluster.TLSConfig != nil,\n\t\topts.Cluster.TLSConfig != nil,\n\t\tDEFAULT_ROUTE_POOL_SIZE,\n\t\t0,\n\t\t_EMPTY_,\n\t\ttime.Time{},\n\t}\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tcheck := func(t *testing.T, v *Varz) {\n\t\t\tt.Helper()\n\t\t\tif !reflect.DeepEqual(v.Cluster, expected) {\n\t\t\t\tt.Fatalf(\"mode=%v - expected %+v, got %+v\", mode, expected, v.Cluster)\n\t\t\t}\n\t\t}\n\t\tv := pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\n\t\t// Having this here to make sure that if fields are added in ClusterOptsVarz,\n\t\t// we make sure to update this test (compiler will report an error if we don't)\n\t\t_ = ClusterOptsVarz{\"\", \"\", 0, 0, nil, 2, false, false, 0, 0, _EMPTY_, time.Time{}}\n\n\t\t// Alter the fields to make sure that we have a proper deep copy\n\t\t// of what may be stored in the server. Anything we change here\n\t\t// should not affect the next returned value.\n\t\tv.Cluster.Name = \"wrong\"\n\t\tv.Cluster.Host = \"wrong\"\n\t\tv.Cluster.Port = 0\n\t\tv.Cluster.AuthTimeout = 0\n\t\tv.Cluster.URLs = []string{\"wrong\"}\n\t\tv = pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\t}\n}\n\nfunc TestMonitorClusterURLs(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\to2 := DefaultOptions()\n\to2.Cluster.Host = \"127.0.0.1\"\n\to2.Cluster.Name = \"A\"\n\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\ts2ClusterHostPort := fmt.Sprintf(\"127.0.0.1:%d\", s2.ClusterAddr().Port)\n\n\ttemplate := `\n\t\tport: -1\n\t\thttp: -1\n\t\tcluster: {\n\t\t\tname: \"A\"\n\t\t\tport: -1\n\t\t\troutes [\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t\t]\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"nats://\"+s2ClusterHostPort, \"\")))\n\ts1, _ := RunServerWithConfig(conf)\n\tdefer s1.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Check /varz cluster{} to see the URLs from s1 to s2\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s1.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s1, mode, varzURL, nil)\n\t\tif n := len(v.Cluster.URLs); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 1 URL, got %v\", mode, n)\n\t\t}\n\t\tif v.Cluster.URLs[0] != s2ClusterHostPort {\n\t\t\tt.Fatalf(\"mode=%v - Expected url %q, got %q\", mode, s2ClusterHostPort, v.Cluster.URLs[0])\n\t\t}\n\t}\n\n\totherClusterHostPort := \"127.0.0.1:1234\"\n\t// Now update the config and add a route\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, \"nats://\"+s2ClusterHostPort, \"nats://\"+otherClusterHostPort)))\n\n\tif err := s1.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\t// Verify cluster still ok\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now verify that s1 reports in /varz the new URL\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, s1, mode, varzURL, nil)\n\t\t\tif n := len(v.Cluster.URLs); n != 2 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected 2 URL, got %v\", mode, n)\n\t\t\t}\n\t\t\tgotS2 := false\n\t\t\tgotOther := false\n\t\t\tfor _, u := range v.Cluster.URLs {\n\t\t\t\tif u == s2ClusterHostPort {\n\t\t\t\t\tgotS2 = true\n\t\t\t\t} else if u == otherClusterHostPort {\n\t\t\t\t\tgotOther = true\n\t\t\t\t} else {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Incorrect url: %q\", mode, u)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !gotS2 {\n\t\t\t\tt.Fatalf(\"mode=%v - Did not get cluster URL for s2\", mode)\n\t\t\t}\n\t\t\tif !gotOther {\n\t\t\t\tt.Fatalf(\"mode=%v - Did not get the new cluster URL\", mode)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Remove all routes from config\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, \"\", \"\")))\n\n\tif err := s1.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\t// Now verify that s1 reports no ULRs in /varz\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, s1, mode, varzURL, nil)\n\t\t\tif n := len(v.Cluster.URLs); n != 0 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected 0 URL, got %v\", mode, n)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestMonitorGateway(t *testing.T) {\n\ttestMonitorStructPresent(t, \"gateway\")\n\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.Gateway.Name = \"A\"\n\topts.Gateway.Port = -1\n\topts.Gateway.AuthTimeout = 1\n\topts.Gateway.TLSTimeout = 1\n\topts.Gateway.Advertise = \"127.0.0.1\"\n\topts.Gateway.ConnectRetries = 1\n\topts.Gateway.RejectUnknown = false\n\tu1, _ := url.Parse(\"nats://ivan:pwd@localhost:1234\")\n\tu2, _ := url.Parse(\"nats://localhost:1235\")\n\topts.Gateway.Gateways = []*RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tTLSTimeout: 1,\n\t\t\tURLs: []*url.URL{\n\t\t\t\tu1,\n\t\t\t\tu2,\n\t\t\t},\n\t\t},\n\t}\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\texpected := GatewayOptsVarz{\n\t\t\"A\",\n\t\topts.Gateway.Host,\n\t\topts.Gateway.Port,\n\t\topts.Gateway.AuthTimeout,\n\t\topts.Gateway.TLSTimeout,\n\t\topts.Gateway.TLSConfig != nil,\n\t\topts.Gateway.TLSConfig != nil,\n\t\topts.Gateway.Advertise,\n\t\topts.Gateway.ConnectRetries,\n\t\t[]RemoteGatewayOptsVarz{{\"B\", 1, nil}},\n\t\topts.Gateway.RejectUnknown,\n\t\t0,\n\t\t_EMPTY_,\n\t\ttime.Time{},\n\t}\n\t// Since URLs array is not guaranteed to be always the same order,\n\t// we don't add it in the expected GatewayOptsVarz, instead we\n\t// maintain here.\n\texpectedURLs := []string{\"localhost:1234\", \"localhost:1235\"}\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tcheck := func(t *testing.T, v *Varz) {\n\t\t\tt.Helper()\n\t\t\tvar urls []string\n\t\t\tif len(v.Gateway.Gateways) == 1 {\n\t\t\t\turls = v.Gateway.Gateways[0].URLs\n\t\t\t\tv.Gateway.Gateways[0].URLs = nil\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(v.Gateway, expected) {\n\t\t\t\tt.Fatalf(\"mode=%v - expected %+v, got %+v\", mode, expected, v.Gateway)\n\t\t\t}\n\t\t\t// Now compare urls\n\t\t\tfor _, u := range expectedURLs {\n\t\t\t\tok := false\n\t\t\t\tfor _, u2 := range urls {\n\t\t\t\t\tif u == u2 {\n\t\t\t\t\t\tok = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"mode=%v - expected urls to be %v, got %v\", mode, expected.Gateways[0].URLs, urls)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tv := pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\n\t\t// Having this here to make sure that if fields are added in GatewayOptsVarz,\n\t\t// we make sure to update this test (compiler will report an error if we don't)\n\t\t_ = GatewayOptsVarz{\"\", \"\", 0, 0, 0, false, false, \"\", 0, []RemoteGatewayOptsVarz{{\"\", 0, nil}}, false, 0, \"default\", time.Time{}}\n\n\t\t// Alter the fields to make sure that we have a proper deep copy\n\t\t// of what may be stored in the server. Anything we change here\n\t\t// should not affect the next returned value.\n\t\tv.Gateway.Name = \"wrong\"\n\t\tv.Gateway.Host = \"wrong\"\n\t\tv.Gateway.Port = 0\n\t\tv.Gateway.AuthTimeout = 1234.5\n\t\tv.Gateway.TLSTimeout = 1234.5\n\t\tv.Gateway.Advertise = \"wrong\"\n\t\tv.Gateway.ConnectRetries = 1234\n\t\tv.Gateway.Gateways[0].Name = \"wrong\"\n\t\tv.Gateway.Gateways[0].TLSTimeout = 1234.5\n\t\tv.Gateway.Gateways[0].URLs = []string{\"wrong\"}\n\t\tv.Gateway.RejectUnknown = true\n\t\tv = pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\t}\n}\n\nfunc TestMonitorGatewayURLsUpdated(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\t// Start a1 that has a single URL to sb1.\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\toa.HTTPHost = \"127.0.0.1\"\n\toa.HTTPPort = MONITOR_PORT\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", sa.MonitorAddr().Port)\n\t// Check the /varz gateway's URLs\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, sa, mode, varzURL, nil)\n\t\tif n := len(v.Gateway.Gateways); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 1 remote gateway, got %v\", mode, n)\n\t\t}\n\t\tgw := v.Gateway.Gateways[0]\n\t\tif n := len(gw.URLs); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 1 url, got %v\", mode, n)\n\t\t}\n\t\texpected := oa.Gateway.Gateways[0].URLs[0].Host\n\t\tif u := gw.URLs[0]; u != expected {\n\t\t\tt.Fatalf(\"mode=%v - Expected URL %q, got %q\", mode, expected, u)\n\t\t}\n\t}\n\n\t// Now start sb2 that clusters with sb1. sa should add to its list of URLs\n\t// sb2 gateway's connect URL.\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\t// Wait for sb1 and sb2 to connect\n\tcheckClusterFormed(t, sb1, sb2)\n\t// sb2 should be made aware of gateway A and connect to sa\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\t// Now check that URLs in /varz get updated\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, sa, mode, varzURL, nil)\n\t\t\tif n := len(v.Gateway.Gateways); n != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 1 remote gateway, got %v\", mode, n)\n\t\t\t}\n\t\t\tgw := v.Gateway.Gateways[0]\n\t\t\tif n := len(gw.URLs); n != 2 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 2 urls, got %v\", mode, n)\n\t\t\t}\n\n\t\t\tgotSB1 := false\n\t\t\tgotSB2 := false\n\t\t\tfor _, u := range gw.URLs {\n\t\t\t\tif u == fmt.Sprintf(\"127.0.0.1:%d\", sb1.GatewayAddr().Port) {\n\t\t\t\t\tgotSB1 = true\n\t\t\t\t} else if u == fmt.Sprintf(\"127.0.0.1:%d\", sb2.GatewayAddr().Port) {\n\t\t\t\t\tgotSB2 = true\n\t\t\t\t} else {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Incorrect URL to gateway B: %v\", mode, u)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !gotSB1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Did not get URL to sb1\", mode)\n\t\t\t}\n\t\t\tif !gotSB2 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Did not get URL to sb2\", mode)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now stop sb2 and make sure that its removal is reflected in varz.\n\tsb2.Shutdown()\n\t// Wait for it to disappear from sa.\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\t// Now check that URLs in /varz get updated.\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, sa, mode, varzURL, nil)\n\t\t\tif n := len(v.Gateway.Gateways); n != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 1 remote gateway, got %v\", mode, n)\n\t\t\t}\n\t\t\tgw := v.Gateway.Gateways[0]\n\t\t\tif n := len(gw.URLs); n != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 1 url, got %v\", mode, n)\n\t\t\t}\n\t\t\tu := gw.URLs[0]\n\t\t\tif u != fmt.Sprintf(\"127.0.0.1:%d\", sb1.GatewayAddr().Port) {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Did not get URL to sb1\", mode)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestMonitorGatewayReportItsOwnURLs(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\t// In this test, we show that if a server has its own gateway information\n\t// as a remote (which is the case when remote gateway definitions is copied\n\t// on all clusters), we display the defined URLs.\n\toa := testGatewayOptionsFromToWithURLs(t, \"A\", \"A\", []string{\"nats://127.0.0.1:1234\", \"nats://127.0.0.1:1235\"})\n\toa.HTTPHost = \"127.0.0.1\"\n\toa.HTTPPort = MONITOR_PORT\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", sa.MonitorAddr().Port)\n\t// Check the /varz gateway's URLs\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, sa, mode, varzURL, nil)\n\t\tif n := len(v.Gateway.Gateways); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 1 remote gateway, got %v\", mode, n)\n\t\t}\n\t\tgw := v.Gateway.Gateways[0]\n\t\tif n := len(gw.URLs); n != 2 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 2 urls, got %v\", mode, gw.URLs)\n\t\t}\n\t\texpected := []string{\"127.0.0.1:1234\", \"127.0.0.1:1235\"}\n\t\tif !reflect.DeepEqual(gw.URLs, expected) {\n\t\t\tt.Fatalf(\"mode=%v - Expected URLs %q, got %q\", mode, expected, gw.URLs)\n\t\t}\n\t}\n}\n\nfunc TestMonitorLeafNode(t *testing.T) {\n\ttestMonitorStructPresent(t, \"leaf\")\n\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.NoSystemAccount = true\n\topts.LeafNode.Port = -1\n\topts.LeafNode.AuthTimeout = 1\n\topts.LeafNode.TLSTimeout = 1\n\topts.Accounts = []*Account{NewAccount(\"acc\")}\n\tu, _ := url.Parse(\"nats://ivan:pwd@localhost:1234\")\n\topts.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tLocalAccount: \"acc\",\n\t\t\tURLs: []*url.URL{u},\n\t\t\tTLSTimeout: 1,\n\t\t},\n\t}\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\texpected := LeafNodeOptsVarz{\n\t\topts.LeafNode.Host,\n\t\topts.LeafNode.Port,\n\t\topts.LeafNode.AuthTimeout,\n\t\topts.LeafNode.TLSTimeout,\n\t\topts.LeafNode.TLSConfig != nil,\n\t\topts.LeafNode.TLSConfig != nil,\n\t\t[]RemoteLeafOptsVarz{\n\t\t\t{\n\t\t\t\t\"acc\", 1, []string{\"localhost:1234\"}, nil, false,\n\t\t\t},\n\t\t},\n\t\tfalse,\n\t\t0,\n\t\t_EMPTY_,\n\t\ttime.Time{},\n\t}\n\n\tvarzURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\n\tfor mode := 0; mode < 2; mode++ {\n\t\tcheck := func(t *testing.T, v *Varz) {\n\t\t\tt.Helper()\n\t\t\t// Issue 5913. When we have solicited leafnodes but no clustering\n\t\t\t// and no clustername, we may need a stable clustername so we use\n\t\t\t// the server name as cluster name. However, we should not expose\n\t\t\t// it in /varz.\n\t\t\tif v.Cluster.Name != _EMPTY_ {\n\t\t\t\tt.Fatalf(\"mode=%v - unexpected cluster name: %s\", mode, v.Cluster.Name)\n\t\t\t}\n\t\t\t// Check rest is as expected.\n\t\t\tif !reflect.DeepEqual(v.LeafNode, expected) {\n\t\t\t\tt.Fatalf(\"mode=%v - expected %+v, got %+v\", mode, expected, v.LeafNode)\n\t\t\t}\n\t\t}\n\t\tv := pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\n\t\t// Having this here to make sure that if fields are added in ClusterOptsVarz,\n\t\t// we make sure to update this test (compiler will report an error if we don't)\n\t\t_ = LeafNodeOptsVarz{\"\", 0, 0, 0, false, false, []RemoteLeafOptsVarz{{\"\", 0, nil, nil, false}}, false, 0, _EMPTY_, time.Time{}}\n\n\t\t// Alter the fields to make sure that we have a proper deep copy\n\t\t// of what may be stored in the server. Anything we change here\n\t\t// should not affect the next returned value.\n\t\tv.LeafNode.Host = \"wrong\"\n\t\tv.LeafNode.Port = 0\n\t\tv.LeafNode.AuthTimeout = 1234.5\n\t\tv.LeafNode.TLSTimeout = 1234.5\n\t\tv.LeafNode.Remotes[0].LocalAccount = \"wrong\"\n\t\tv.LeafNode.Remotes[0].URLs = append(v.LeafNode.Remotes[0].URLs, \"wrong\")\n\t\tv.LeafNode.Remotes[0].TLSTimeout = 1234.5\n\t\tv = pollVarz(t, s, mode, varzURL, nil)\n\t\tcheck(t, v)\n\t}\n}\n\nfunc pollGatewayz(t *testing.T, s *Server, mode int, url string, opts *GatewayzOptions) *Gatewayz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tg := &Gatewayz{}\n\t\tbody := readBody(t, url)\n\t\tif err := json.Unmarshal(body, g); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn g\n\t}\n\tg, err := s.Gatewayz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Gatewayz: %v\", err)\n\t}\n\treturn g\n}\n\nfunc TestMonitorGatewayz(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\t// First check that without gateway configured\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/gatewayz\", s.MonitorAddr().Port)\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\tg := pollGatewayz(t, s, pollMode, url, nil)\n\t\t// Expect Name and port to be empty\n\t\tif g.Name != _EMPTY_ || g.Port != 0 {\n\t\t\tt.Fatalf(\"Expected no gateway, got %+v\", g)\n\t\t}\n\t}\n\ts.Shutdown()\n\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\t// Start a1 that has a single URL to sb1.\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb1)\n\toa.HTTPHost = \"127.0.0.1\"\n\toa.HTTPPort = MONITOR_PORT\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb1, 1, 2*time.Second)\n\n\tgatewayzURL := fmt.Sprintf(\"http://127.0.0.1:%d/gatewayz\", sa.MonitorAddr().Port)\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\tg := pollGatewayz(t, sa, pollMode, gatewayzURL, nil)\n\t\tif g.Host != oa.Gateway.Host {\n\t\t\tt.Fatalf(\"mode=%v - Expected host to be %q, got %q\", pollMode, oa.Gateway.Host, g.Host)\n\t\t}\n\t\tif g.Port != oa.Gateway.Port {\n\t\t\tt.Fatalf(\"mode=%v - Expected port to be %v, got %v\", pollMode, oa.Gateway.Port, g.Port)\n\t\t}\n\t\tif n := len(g.OutboundGateways); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected outbound to 1 gateway, got %v\", pollMode, n)\n\t\t}\n\t\tif n := len(g.InboundGateways); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected inbound from 1 gateway, got %v\", pollMode, n)\n\t\t}\n\t\tog := g.OutboundGateways[\"B\"]\n\t\tif og == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected to find outbound connection to B, got none\", pollMode)\n\t\t}\n\t\tif !og.IsConfigured {\n\t\t\tt.Fatalf(\"mode=%v - Expected gw connection to be configured, was not\", pollMode)\n\t\t}\n\t\tif og.Connection == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected outbound connection to B to be set, wat not\", pollMode)\n\t\t}\n\t\tif og.Connection.Name != sb1.ID() {\n\t\t\tt.Fatalf(\"mode=%v - Expected outbound connection to B to have name %q, got %q\", pollMode, sb1.ID(), og.Connection.Name)\n\t\t}\n\t\tif n := len(og.Accounts); n != 0 {\n\t\t\tt.Fatalf(\"mode=%v - Expected no account, got %v\", pollMode, n)\n\t\t}\n\t\tig := g.InboundGateways[\"B\"]\n\t\tif ig == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected to find inbound connection from B, got none\", pollMode)\n\t\t}\n\t\tif n := len(ig); n != 1 {\n\t\t\tt.Fatalf(\"mode=%v - Expected 1 inbound connection, got %v\", pollMode, n)\n\t\t}\n\t\tigc := ig[0]\n\t\tif igc.Connection == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected inbound connection to B to be set, wat not\", pollMode)\n\t\t}\n\t\tif igc.Connection.Name != sb1.ID() {\n\t\t\tt.Fatalf(\"mode=%v - Expected inbound connection to B to have name %q, got %q\", pollMode, sb1.ID(), igc.Connection.Name)\n\t\t}\n\t}\n\n\t// Now start sb2 that clusters with sb1. sa should add to its list of URLs\n\t// sb2 gateway's connect URL.\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", sb1.ClusterAddr().Port))\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\t// Wait for sb1 and sb2 to connect\n\tcheckClusterFormed(t, sb1, sb2)\n\t// sb2 should be made aware of gateway A and connect to sa\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\t// Now check that URLs in /varz get updated\n\tcheckGatewayB := func(t *testing.T, url string, opts *GatewayzOptions) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\t\tg := pollGatewayz(t, sa, pollMode, url, opts)\n\t\t\t\tif n := len(g.OutboundGateways); n != 1 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Expected outbound to 1 gateway, got %v\", pollMode, n)\n\t\t\t\t}\n\t\t\t\t// The InboundGateways is a map with key the gateway names,\n\t\t\t\t// then value is array of connections. So should be 1 here.\n\t\t\t\tif n := len(g.InboundGateways); n != 1 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Expected inbound from 1 gateway, got %v\", pollMode, n)\n\t\t\t\t}\n\t\t\t\tig := g.InboundGateways[\"B\"]\n\t\t\t\tif ig == nil {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Expected to find inbound connection from B, got none\", pollMode)\n\t\t\t\t}\n\t\t\t\tif n := len(ig); n != 2 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Expected 2 inbound connections from gateway B, got %v\", pollMode, n)\n\t\t\t\t}\n\t\t\t\tgotSB1 := false\n\t\t\t\tgotSB2 := false\n\t\t\t\tfor _, rg := range ig {\n\t\t\t\t\tif rg.Connection != nil {\n\t\t\t\t\t\tif rg.Connection.Name == sb1.ID() {\n\t\t\t\t\t\t\tgotSB1 = true\n\t\t\t\t\t\t} else if rg.Connection.Name == sb2.ID() {\n\t\t\t\t\t\t\tgotSB2 = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !gotSB1 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Missing inbound connection from sb1\", pollMode)\n\t\t\t\t}\n\t\t\t\tif !gotSB2 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Missing inbound connection from sb2\", pollMode)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckGatewayB(t, gatewayzURL, nil)\n\n\t// Start a new cluser C that connects to B. A should see it as\n\t// a non-configured gateway.\n\toc := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", sb1)\n\tsc := runGatewayServer(oc)\n\tdefer sc.Shutdown()\n\n\t// All servers should have 2 outbound connections (one for each other cluster)\n\twaitForOutboundGateways(t, sa, 2, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 2, 2*time.Second)\n\twaitForOutboundGateways(t, sb2, 2, 2*time.Second)\n\twaitForOutboundGateways(t, sc, 2, 2*time.Second)\n\n\t// Server sa should have 3 inbounds now\n\twaitForInboundGateways(t, sa, 3, 2*time.Second)\n\n\t// Check gatewayz again to see that we have C now.\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\tg := pollGatewayz(t, sa, pollMode, gatewayzURL, nil)\n\t\t\tif n := len(g.OutboundGateways); n != 2 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected outbound to 2 gateways, got %v\", pollMode, n)\n\t\t\t}\n\t\t\t// The InboundGateways is a map with key the gateway names,\n\t\t\t// then value is array of connections. So should be 2 here.\n\t\t\tif n := len(g.InboundGateways); n != 2 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected inbound from 2 gateways, got %v\", pollMode, n)\n\t\t\t}\n\t\t\tog := g.OutboundGateways[\"C\"]\n\t\t\tif og == nil {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected to find outbound connection to C, got none\", pollMode)\n\t\t\t}\n\t\t\tif og.IsConfigured {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected IsConfigured for gateway C to be false, was true\", pollMode)\n\t\t\t}\n\t\t\tif og.Connection == nil {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected connection to C, got none\", pollMode)\n\t\t\t}\n\t\t\tif og.Connection.Name != sc.ID() {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected outbound connection to C to have name %q, got %q\", pollMode, sc.ID(), og.Connection.Name)\n\t\t\t}\n\t\t\tig := g.InboundGateways[\"C\"]\n\t\t\tif ig == nil {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected to find inbound connection from C, got none\", pollMode)\n\t\t\t}\n\t\t\tif n := len(ig); n != 1 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected 1 inbound connections from gateway C, got %v\", pollMode, n)\n\t\t\t}\n\t\t\tigc := ig[0]\n\t\t\tif igc.Connection == nil {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected connection to C, got none\", pollMode)\n\t\t\t}\n\t\t\tif igc.Connection.Name != sc.ID() {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected outbound connection to C to have name %q, got %q\", pollMode, sc.ID(), og.Connection.Name)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Select only 1 gateway by passing the name to option/url\n\topts := &GatewayzOptions{Name: \"B\"}\n\tcheckGatewayB(t, gatewayzURL+\"?gw_name=B\", opts)\n\n\t// Stop gateway C and check that we have only B, with and without filter.\n\tsc.Shutdown()\n\tcheckGatewayB(t, gatewayzURL+\"?gw_name=B\", opts)\n\tcheckGatewayB(t, gatewayzURL, nil)\n}\n\nfunc TestMonitorGatewayzAccounts(t *testing.T) {\n\tGatewayDoNotForceInterestOnlyMode(true)\n\tdefer GatewayDoNotForceInterestOnlyMode(false)\n\n\tresetPreviousHTTPConnections()\n\n\t// Create bunch of Accounts\n\ttotalAccounts := 15\n\taccounts := \"\"\n\tfor i := 0; i < totalAccounts; i++ {\n\t\tacc := fmt.Sprintf(\"\tacc_%d: { users=[{user:user_%d, password: pwd}] }\\n\", i, i)\n\t\taccounts += acc\n\t}\n\n\tbConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\taccounts {\n\t\t\t%s\n\t\t}\n\t\tport: -1\n\t\thttp: -1\n\t\tgateway: {\n\t\t\tname: \"B\"\n\t\t\tport: -1\n\t\t}\n\t\tno_sys_acc = true\n\t`, accounts)))\n\n\tsb, ob := RunServerWithConfig(bConf)\n\tdefer sb.Shutdown()\n\tsb.SetLogger(&DummyLogger{}, true, true)\n\n\t// Start a1 that has a single URL to sb1.\n\taConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\taccounts {\n\t\t\t%s\n\t\t}\n\t\tport: -1\n\t\thttp: -1\n\t\tgateway: {\n\t\t\tname: \"A\"\n\t\t\tport: -1\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"B\"\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tno_sys_acc = true\n\t`, accounts, sb.GatewayAddr().Port)))\n\n\tsa, oa := RunServerWithConfig(aConf)\n\tdefer sa.Shutdown()\n\tsa.SetLogger(&DummyLogger{}, true, true)\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\t// Create clients for each account on A and publish a message\n\t// so that list of accounts appear in gatewayz\n\tproduceMsgsFromA := func(t *testing.T) {\n\t\tt.Helper()\n\t\tfor i := 0; i < totalAccounts; i++ {\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://user_%d:pwd@%s:%d\", i, oa.Host, oa.Port))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t\t}\n\t\t\tnc.Publish(\"foo\", []byte(\"hello\"))\n\t\t\tnc.Flush()\n\t\t\tnc.Close()\n\t\t}\n\t}\n\tproduceMsgsFromA(t)\n\n\t// Wait for A- for all accounts\n\tgwc := sa.getOutboundGatewayConnection(\"B\")\n\tfor i := 0; i < totalAccounts; i++ {\n\t\tcheckForAccountNoInterest(t, gwc, fmt.Sprintf(\"acc_%d\", i), true, 2*time.Second)\n\t}\n\n\t// Check accounts...\n\tgatewayzURL := fmt.Sprintf(\"http://127.0.0.1:%d/gatewayz\", sa.MonitorAddr().Port)\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t// First, without asking for it, they should not be present.\n\t\tg := pollGatewayz(t, sa, pollMode, gatewayzURL, nil)\n\t\tog := g.OutboundGateways[\"B\"]\n\t\tif og == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected outbound gateway to B, got none\", pollMode)\n\t\t}\n\t\tif n := len(og.Accounts); n != 0 {\n\t\t\tt.Fatalf(\"mode=%v - Expected accounts list to not be present by default, got %v\", pollMode, n)\n\t\t}\n\t\t// Now ask for the accounts\n\t\tg = pollGatewayz(t, sa, pollMode, gatewayzURL+\"?accs=1\", &GatewayzOptions{Accounts: true})\n\t\tog = g.OutboundGateways[\"B\"]\n\t\tif og == nil {\n\t\t\tt.Fatalf(\"mode=%v - Expected outbound gateway to B, got none\", pollMode)\n\t\t}\n\t\tif n := len(og.Accounts); n != totalAccounts {\n\t\t\tt.Fatalf(\"mode=%v - Expected to get all %d accounts, got %v\", pollMode, totalAccounts, n)\n\t\t}\n\t\t// Now account details\n\t\tfor _, acc := range og.Accounts {\n\t\t\tif acc.InterestMode != Optimistic.String() {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected optimistic mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t}\n\t\t\t// Since there is no interest at all on B, the publish\n\t\t\t// will have resulted in total account no interest, so\n\t\t\t// the number of no interest (subject wise) should be 0\n\t\t\tif acc.NoInterestCount != 0 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected 0 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t}\n\t\t\tif acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected total subs to be 0, got %v - and num queue subs to be 0, got %v\",\n\t\t\t\t\tpollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check inbound on B\n\tgwURLServerB := fmt.Sprintf(\"http://127.0.0.1:%d/gatewayz\", sb.MonitorAddr().Port)\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\t// First, without asking for it, they should not be present.\n\t\t\tg := pollGatewayz(t, sb, pollMode, gwURLServerB, nil)\n\t\t\tigs := g.InboundGateways[\"A\"]\n\t\t\tif igs == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound gateway to A, got none\", pollMode)\n\t\t\t}\n\t\t\tif len(igs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected single inbound, got %v\", pollMode, len(igs))\n\t\t\t}\n\t\t\tig := igs[0]\n\t\t\tif n := len(ig.Accounts); n != 0 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected no account, got %v\", pollMode, n)\n\t\t\t}\n\t\t\t// Check that list of accounts\n\t\t\tg = pollGatewayz(t, sb, pollMode, gwURLServerB+\"?accs=1\", &GatewayzOptions{Accounts: true})\n\t\t\tigs = g.InboundGateways[\"A\"]\n\t\t\tif igs == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound gateway to A, got none\", pollMode)\n\t\t\t}\n\t\t\tif len(igs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected single inbound, got %v\", pollMode, len(igs))\n\t\t\t}\n\t\t\tig = igs[0]\n\t\t\tif ig.Connection == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound connection from A to be set, wat not\", pollMode)\n\t\t\t}\n\t\t\tif ig.Connection.Name != sa.ID() {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected inbound connection from A to have name %q, got %q\", pollMode, sa.ID(), ig.Connection.Name)\n\t\t\t}\n\t\t\tif n := len(ig.Accounts); n != totalAccounts {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected to get all %d accounts, got %v\", pollMode, totalAccounts, n)\n\t\t\t}\n\t\t\t// Now account details\n\t\t\tfor _, acc := range ig.Accounts {\n\t\t\t\tif acc.InterestMode != Optimistic.String() {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected optimistic mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t\t}\n\t\t\t\t// Since there is no interest at all on B, the publish\n\t\t\t\t// will have resulted in total account no interest, so\n\t\t\t\t// the number of no interest (subject wise) should be 0\n\t\t\t\tif acc.NoInterestCount != 0 {\n\t\t\t\t\tt.Fatalf(\"mode=%v - Expected 0 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t\t}\n\t\t\t\t// For inbound gateway, NumQueueSubscriptions and TotalSubscriptions\n\t\t\t\t// are not relevant.\n\t\t\t\tif acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v\",\n\t\t\t\t\t\tpollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create subscriptions on B to prevent A- and check on subject no interest\n\tfor i := 0; i < totalAccounts; i++ {\n\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://user_%d:pwd@%s:%d\", i, ob.Host, ob.Port))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\t// Create a queue sub so it shows up in gatewayz\n\t\tnc.QueueSubscribeSync(\"bar\", \"queue\")\n\t\t// Create plain subscriptions on baz.0, baz.1 and baz.2.\n\t\t// Create to for each subject. Since gateways will send\n\t\t// only once per subject, the number of subs should be 3, not 6.\n\t\tfor j := 0; j < 3; j++ {\n\t\t\tsubj := fmt.Sprintf(\"baz.%d\", j)\n\t\t\tnc.SubscribeSync(subj)\n\t\t\tnc.SubscribeSync(subj)\n\t\t}\n\t\tnc.Flush()\n\t}\n\n\tfor i := 0; i < totalAccounts; i++ {\n\t\taccName := fmt.Sprintf(\"acc_%d\", i)\n\t\tcheckForRegisteredQSubInterest(t, sa, \"B\", accName, \"bar\", 1, 2*time.Second)\n\t}\n\n\t// Resend msgs from A on foo, on all accounts. There will be no interest on this subject.\n\tproduceMsgsFromA(t)\n\n\tfor i := 0; i < totalAccounts; i++ {\n\t\taccName := fmt.Sprintf(\"acc_%d\", i)\n\t\tcheckForSubjectNoInterest(t, gwc, accName, \"foo\", true, 2*time.Second)\n\t\t// Verify that we still have the queue interest registered\n\t\tcheckForRegisteredQSubInterest(t, sa, \"B\", accName, \"bar\", 1, 2*time.Second)\n\t}\n\n\t// Check accounts...\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\tg := pollGatewayz(t, sa, pollMode, gatewayzURL+\"?accs=1\", &GatewayzOptions{Accounts: true})\n\t\t\tog := g.OutboundGateways[\"B\"]\n\t\t\tif og == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected outbound gateway to B, got none\", pollMode)\n\t\t\t}\n\t\t\tif n := len(og.Accounts); n != totalAccounts {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected to get all %d accounts, got %v\", pollMode, totalAccounts, n)\n\t\t\t}\n\t\t\t// Now account details\n\t\t\tfor _, acc := range og.Accounts {\n\t\t\t\tif acc.InterestMode != Optimistic.String() {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected optimistic mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t\t}\n\t\t\t\tif acc.NoInterestCount != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 1 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t\t}\n\t\t\t\tif acc.NumQueueSubscriptions != 1 || acc.TotalSubscriptions != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected total subs to be 1, got %v - and num queue subs to be 1, got %v\",\n\t\t\t\t\t\tpollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check inbound on server B\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\t// Ask for accounts list\n\t\t\tg := pollGatewayz(t, sb, pollMode, gwURLServerB+\"?accs=1\", &GatewayzOptions{Accounts: true})\n\t\t\tigs := g.InboundGateways[\"A\"]\n\t\t\tif igs == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound gateway to A, got none\", pollMode)\n\t\t\t}\n\t\t\tif len(igs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected single inbound, got %v\", pollMode, len(igs))\n\t\t\t}\n\t\t\tig := igs[0]\n\t\t\tif ig.Connection == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound connection from A to be set, wat not\", pollMode)\n\t\t\t}\n\t\t\tif ig.Connection.Name != sa.ID() {\n\t\t\t\tt.Fatalf(\"mode=%v - Expected inbound connection from A to have name %q, got %q\", pollMode, sa.ID(), ig.Connection.Name)\n\t\t\t}\n\t\t\tif n := len(ig.Accounts); n != totalAccounts {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected to get all %d accounts, got %v\", pollMode, totalAccounts, n)\n\t\t\t}\n\t\t\t// Now account details\n\t\t\tfor _, acc := range ig.Accounts {\n\t\t\t\tif acc.InterestMode != Optimistic.String() {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected optimistic mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t\t}\n\t\t\t\tif acc.NoInterestCount != 1 {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 1 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t\t}\n\t\t\t\t// For inbound gateway, NumQueueSubscriptions and TotalSubscriptions\n\t\t\t\t// are not relevant.\n\t\t\t\tif acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v\",\n\t\t\t\t\t\tpollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make one of the account to switch to interest only\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://user_1:pwd@%s:%d\", oa.Host, oa.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tfor i := 0; i < 1100; i++ {\n\t\tnc.Publish(fmt.Sprintf(\"foo.%d\", i), []byte(\"hello\"))\n\t}\n\tnc.Flush()\n\tnc.Close()\n\n\t// Check that we can select single account\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\tg := pollGatewayz(t, sa, pollMode, gatewayzURL+\"?gw_name=B&acc_name=acc_1\", &GatewayzOptions{Name: \"B\", AccountName: \"acc_1\"})\n\t\t\tog := g.OutboundGateways[\"B\"]\n\t\t\tif og == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected outbound gateway to B, got none\", pollMode)\n\t\t\t}\n\t\t\tif n := len(og.Accounts); n != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected to get 1 account, got %v\", pollMode, n)\n\t\t\t}\n\t\t\t// Now account details\n\t\t\tacc := og.Accounts[0]\n\t\t\tif acc.InterestMode != InterestOnly.String() {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected interest-only mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t}\n\t\t\t// Since we switched, this should be set to 0\n\t\t\tif acc.NoInterestCount != 0 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 0 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t}\n\t\t\t// We have created 3 subs on that account on B, and 1 queue sub.\n\t\t\t// So total should be 4 and 1 for queue sub.\n\t\t\tif acc.NumQueueSubscriptions != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected num queue subs to be 1, got %v\",\n\t\t\t\t\tpollMode, acc.NumQueueSubscriptions)\n\t\t\t}\n\t\t\tif acc.TotalSubscriptions != 4 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected total subs to be 4, got %v\",\n\t\t\t\t\tpollMode, acc.TotalSubscriptions)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check inbound on B now...\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\tg := pollGatewayz(t, sb, pollMode, gwURLServerB+\"?gw_name=A&acc_name=acc_1\", &GatewayzOptions{Name: \"A\", AccountName: \"acc_1\"})\n\t\t\tigs := g.InboundGateways[\"A\"]\n\t\t\tif igs == nil {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected inbound gateway from A, got none\", pollMode)\n\t\t\t}\n\t\t\tif len(igs) != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected single inbound, got %v\", pollMode, len(igs))\n\t\t\t}\n\t\t\tig := igs[0]\n\t\t\tif n := len(ig.Accounts); n != 1 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected to get 1 account, got %v\", pollMode, n)\n\t\t\t}\n\t\t\t// Now account details\n\t\t\tacc := ig.Accounts[0]\n\t\t\tif acc.InterestMode != InterestOnly.String() {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected interest-only mode, got %q\", pollMode, acc.InterestMode)\n\t\t\t}\n\t\t\tif acc.InterestMode != InterestOnly.String() {\n\t\t\t\treturn fmt.Errorf(\"Should be in %q mode, got %q\", InterestOnly.String(), acc.InterestMode)\n\t\t\t}\n\t\t\t// Since we switched, this should be set to 0\n\t\t\tif acc.NoInterestCount != 0 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - Expected 0 no-interest, got %v\", pollMode, acc.NoInterestCount)\n\t\t\t}\n\t\t\t// Again, for inbound, these should be always 0.\n\t\t\tif acc.NumQueueSubscriptions != 0 || acc.TotalSubscriptions != 0 {\n\t\t\t\treturn fmt.Errorf(\"mode=%v - For inbound connection, expected num queue subs and total subs to be 0, got %v and %v\",\n\t\t\t\t\tpollMode, acc.TotalSubscriptions, acc.NumQueueSubscriptions)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestMonitorGatewayzWithSubs(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\taA := NewAccount(\"A\")\n\taB := NewAccount(\"B\")\n\tob.Accounts = append(ob.Accounts, aA, aB)\n\tob.Users = append(ob.Users,\n\t\t&User{Username: \"a\", Password: \"a\", Account: aA},\n\t\t&User{Username: \"b\", Password: \"b\", Account: aB})\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb)\n\toa.HTTPHost = \"127.0.0.1\"\n\toa.HTTPPort = MONITOR_PORT\n\taA = NewAccount(\"A\")\n\taB = NewAccount(\"B\")\n\toa.Accounts = append(oa.Accounts, aA, aB)\n\toa.Users = append(oa.Users,\n\t\t&User{Username: \"a\", Password: \"a\", Account: aA},\n\t\t&User{Username: \"b\", Password: \"b\", Account: aB})\n\tsa := runGatewayServer(oa)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 1, 2*time.Second)\n\n\twaitForOutboundGateways(t, sb, 1, 2*time.Second)\n\twaitForInboundGateways(t, sb, 1, 2*time.Second)\n\n\tncA := natsConnect(t, sb.ClientURL(), nats.UserInfo(\"a\", \"a\"))\n\tdefer ncA.Close()\n\tnatsSubSync(t, ncA, \"foo\")\n\tnatsFlush(t, ncA)\n\n\tncB := natsConnect(t, sb.ClientURL(), nats.UserInfo(\"b\", \"b\"))\n\tdefer ncB.Close()\n\tnatsSubSync(t, ncB, \"foo\")\n\tnatsQueueSubSync(t, ncB, \"bar\", \"baz\")\n\tnatsFlush(t, ncB)\n\n\tcheckGWInterestOnlyModeInterestOn(t, sa, \"B\", \"A\", \"foo\")\n\tcheckGWInterestOnlyModeInterestOn(t, sa, \"B\", \"B\", \"foo\")\n\tcheckForRegisteredQSubInterest(t, sa, \"B\", \"B\", \"bar\", 1, time.Second)\n\n\tfor _, test := range []struct {\n\t\turl string\n\t\tallAccs bool\n\t\topts *GatewayzOptions\n\t}{\n\t\t{\"accs=1&subs=1\", true, &GatewayzOptions{Accounts: true, AccountSubscriptions: true}},\n\t\t{\"accs=1&subs=detail\", true, &GatewayzOptions{Accounts: true, AccountSubscriptionsDetail: true}},\n\t\t{\"acc_name=B&subs=1\", false, &GatewayzOptions{AccountName: \"B\", AccountSubscriptions: true}},\n\t\t{\"acc_name=B&subs=detail\", false, &GatewayzOptions{AccountName: \"B\", AccountSubscriptionsDetail: true}},\n\t} {\n\t\tt.Run(test.url, func(t *testing.T) {\n\t\t\tgatewayzURL := fmt.Sprintf(\"http://127.0.0.1:%d/gatewayz?%s\", sa.MonitorAddr().Port, test.url)\n\t\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\t\tgw := pollGatewayz(t, sa, pollMode, gatewayzURL, test.opts)\n\t\t\t\trequire_Equal(t, len(gw.OutboundGateways), 1)\n\t\t\t\togw, ok := gw.OutboundGateways[\"B\"]\n\t\t\t\trequire_True(t, ok)\n\t\t\t\trequire_NotNil(t, ogw)\n\t\t\t\tvar expected int\n\t\t\t\tif test.allAccs {\n\t\t\t\t\texpected = 3 // A + B + $G\n\t\t\t\t} else {\n\t\t\t\t\texpected = 1 // B\n\t\t\t\t}\n\t\t\t\trequire_Len(t, len(ogw.Accounts), expected)\n\t\t\t\taccs := map[string]*AccountGatewayz{}\n\t\t\t\tfor _, a := range ogw.Accounts {\n\t\t\t\t\t// Do not include the global account there.\n\t\t\t\t\tif a.Name == globalAccountName {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\taccs[a.Name] = a\n\t\t\t\t}\n\t\t\t\t// Update the expected number of accounts if we asked for all accounts.\n\t\t\t\tif test.allAccs {\n\t\t\t\t\texpected--\n\t\t\t\t}\n\t\t\t\t// The account B should always be present.\n\t\t\t\t_, ok = accs[\"B\"]\n\t\t\t\trequire_True(t, ok)\n\t\t\t\tif expected == 2 {\n\t\t\t\t\t_, ok = accs[\"A\"]\n\t\t\t\t\trequire_True(t, ok)\n\t\t\t\t}\n\t\t\t\t// Now that we know we have the proper account(s), check the content.\n\t\t\t\tfor n, a := range accs {\n\t\t\t\t\trequire_NotNil(t, a)\n\t\t\t\t\trequire_Equal(t, a.Name, n)\n\t\t\t\t\ttotalSubs := 1\n\t\t\t\t\tvar numQueueSubs int\n\t\t\t\t\tif n == \"B\" {\n\t\t\t\t\t\ttotalSubs++\n\t\t\t\t\t\tnumQueueSubs = 1\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, a.TotalSubscriptions, totalSubs)\n\t\t\t\t\trequire_Equal(t, a.NumQueueSubscriptions, numQueueSubs)\n\n\t\t\t\t\tm := map[string]*SubDetail{}\n\t\t\t\t\tif test.opts.AccountSubscriptions {\n\t\t\t\t\t\trequire_Len(t, len(a.Subs), totalSubs)\n\t\t\t\t\t\trequire_Len(t, len(a.SubsDetail), 0)\n\t\t\t\t\t\tfor _, sub := range a.Subs {\n\t\t\t\t\t\t\tm[sub] = nil\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Len(t, len(a.Subs), 0)\n\t\t\t\t\t\trequire_Len(t, len(a.SubsDetail), totalSubs)\n\t\t\t\t\t\tfor _, sub := range a.SubsDetail {\n\t\t\t\t\t\t\tm[sub.Subject] = &sub\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tsd, ok := m[\"foo\"]\n\t\t\t\t\trequire_True(t, ok)\n\t\t\t\t\tif test.opts.AccountSubscriptionsDetail {\n\t\t\t\t\t\trequire_NotNil(t, sd)\n\t\t\t\t\t\trequire_Equal(t, sd.Queue, _EMPTY_)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_True(t, sd == nil)\n\t\t\t\t\t}\n\t\t\t\t\tsd, ok = m[\"bar\"]\n\t\t\t\t\tif numQueueSubs == 1 {\n\t\t\t\t\t\trequire_True(t, ok)\n\t\t\t\t\t\tif test.opts.AccountSubscriptionsDetail {\n\t\t\t\t\t\t\trequire_NotNil(t, sd)\n\t\t\t\t\t\t\trequire_Equal(t, sd.Queue, \"baz\")\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\trequire_True(t, sd == nil)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_False(t, ok)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMonitorRoutezRTT(t *testing.T) {\n\t// Do not change default PingInterval and expect RTT to still be reported\n\n\tob := DefaultOptions()\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := DefaultOptions()\n\toa.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ob.Cluster.Host, ob.Cluster.Port))\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tcheckClusterFormed(t, sa, sb)\n\n\tcheckRouteInfo := func(t *testing.T, s *Server) {\n\t\tt.Helper()\n\t\troutezURL := fmt.Sprintf(\"http://127.0.0.1:%d/routez\", s.MonitorAddr().Port)\n\t\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\t\tcheckFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error {\n\t\t\t\trz := pollRoutez(t, s, pollMode, routezURL, nil)\n\t\t\t\t// Pool size + 1 for system account\n\t\t\t\tif len(rz.Routes) != DEFAULT_ROUTE_POOL_SIZE+1 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d route, got %v\", DEFAULT_ROUTE_POOL_SIZE+1, len(rz.Routes))\n\t\t\t\t}\n\t\t\t\tfor _, ri := range rz.Routes {\n\t\t\t\t\tif ri.RTT == _EMPTY_ {\n\t\t\t\t\t\treturn fmt.Errorf(\"Route's RTT not reported\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t}\n\tcheckRouteInfo(t, sa)\n\tcheckRouteInfo(t, sb)\n}\n\nfunc pollLeafz(t *testing.T, s *Server, mode int, url string, opts *LeafzOptions) *Leafz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tl := &Leafz{}\n\t\tbody := readBody(t, url)\n\t\tif err := json.Unmarshal(body, l); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn l\n\t}\n\tl, err := s.Leafz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Leafz: %v\", err)\n\t}\n\treturn l\n}\n\nfunc TestMonitorOpJWT(t *testing.T) {\n\tcontent := `\n\tlisten: \"127.0.0.1:-1\"\n\thttp: \"127.0.0.1:-1\"\n\toperator = \"../test/configs/nkeys/op.jwt\"\n\tresolver = MEMORY\n\t`\n\tconf := createConfFile(t, []byte(content))\n\tsa, _ := RunServerWithConfig(conf)\n\tdefer sa.Shutdown()\n\n\ttheJWT, err := os.ReadFile(\"../test/configs/nkeys/op.jwt\")\n\trequire_NoError(t, err)\n\ttheJWT = []byte(strings.Split(string(theJWT), \"\\n\")[1])\n\tclaim, err := jwt.DecodeOperatorClaims(string(theJWT))\n\trequire_NoError(t, err)\n\n\tpollURL := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", sa.MonitorAddr().Port)\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollVarz(t, sa, pollMode, pollURL, nil)\n\n\t\tif len(l.TrustedOperatorsJwt) != 1 {\n\t\t\tt.Fatalf(\"Expected one operator jwt\")\n\t\t}\n\t\tif len(l.TrustedOperatorsClaim) != 1 {\n\t\t\tt.Fatalf(\"Expected one operator claim\")\n\t\t}\n\t\tif l.TrustedOperatorsJwt[0] != string(theJWT) {\n\t\t\tt.Fatalf(\"Expected operator to be identical to configuration\")\n\t\t}\n\t\tif !reflect.DeepEqual(l.TrustedOperatorsClaim[0], claim) {\n\t\t\tt.Fatal(\"claims need to be equal\")\n\t\t}\n\t}\n}\n\nfunc TestMonitorLeafz(t *testing.T) {\n\tcontent := `\n\tserver_name: \"hub\"\n\tlisten: \"127.0.0.1:-1\"\n\thttp: \"127.0.0.1:-1\"\n\toperator = \"../test/configs/nkeys/op.jwt\"\n\tresolver = MEMORY\n\tping_interval = 1\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\tsb, ob := RunServerWithConfig(conf)\n\tdefer sb.Shutdown()\n\n\tcreateAcc := func(t *testing.T) (*Account, string) {\n\t\tt.Helper()\n\t\tacc, akp := createAccount(sb)\n\t\tkp, _ := nkeys.CreateUser()\n\t\tpub, _ := kp.PublicKey()\n\t\tnuc := jwt.NewUserClaims(pub)\n\t\tujwt, err := nuc.Encode(akp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t\t}\n\t\tseed, _ := kp.Seed()\n\t\tcreds := genCredsFile(t, ujwt, seed)\n\t\treturn acc, creds\n\t}\n\tacc1, mycreds1 := createAcc(t)\n\tacc2, mycreds2 := createAcc(t)\n\tleafName := \"my-leaf-node\"\n\n\tcontent = `\n\t\tport: -1\n\t\thttp: \"127.0.0.1:-1\"\n\t\tping_interval = 1\n\t\tserver_name: %s\n\t\taccounts {\n\t\t\t%s {\n\t\t\t\tusers [\n\t\t\t\t\t{user: user1, password: pwd}\n\t\t\t\t]\n\t\t\t}\n\t\t\t%s {\n\t\t\t\tusers [\n\t\t\t\t\t{user: user2, password: pwd}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tisolate_leafnode_interest: true\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\taccount: \"%s\"\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\taccount: \"%s\"\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content,\n\t\tleafName,\n\t\tacc1.Name, acc2.Name,\n\t\tacc1.Name, ob.LeafNode.Port, mycreds1,\n\t\tacc2.Name, ob.LeafNode.Port, mycreds2)\n\tconf = createConfFile(t, []byte(config))\n\tsa, oa := RunServerWithConfig(conf)\n\tdefer sa.Shutdown()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif n := sa.NumLeafNodes(); n != 2 {\n\t\t\treturn fmt.Errorf(\"Expected 2 leaf connections, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Wait for initial RTT to be computed\n\ttime.Sleep(firstPingInterval + 500*time.Millisecond)\n\n\tch := make(chan bool, 1)\n\tnc1B := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.Port), nats.UserCredentials(mycreds1))\n\tdefer nc1B.Close()\n\tnatsSub(t, nc1B, \"foo\", func(_ *nats.Msg) { ch <- true })\n\tnatsSub(t, nc1B, \"bar\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc1B)\n\n\tnc2B := natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", ob.Port), nats.UserCredentials(mycreds2))\n\tdefer nc2B.Close()\n\tnatsSub(t, nc2B, \"bar\", func(_ *nats.Msg) { ch <- true })\n\tnatsSub(t, nc2B, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc2B)\n\n\tnc1A := natsConnect(t, fmt.Sprintf(\"nats://user1:pwd@127.0.0.1:%d\", oa.Port))\n\tdefer nc1A.Close()\n\tnatsPub(t, nc1A, \"foo\", []byte(\"hello\"))\n\tnatsFlush(t, nc1A)\n\n\twaitCh(t, ch, \"Did not get the message\")\n\n\tnc2A := natsConnect(t, fmt.Sprintf(\"nats://user2:pwd@127.0.0.1:%d\", oa.Port))\n\tdefer nc2A.Close()\n\tnatsPub(t, nc2A, \"bar\", []byte(\"hello\"))\n\tnatsPub(t, nc2A, \"bar\", []byte(\"hello\"))\n\tnatsFlush(t, nc2A)\n\n\twaitCh(t, ch, \"Did not get the message\")\n\twaitCh(t, ch, \"Did not get the message\")\n\n\t// Let's poll server A\n\tpollURL := fmt.Sprintf(\"http://127.0.0.1:%d/leafz?subs=1\", sa.MonitorAddr().Port)\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Subscriptions: true})\n\t\tif l.ID != sa.ID() {\n\t\t\tt.Fatalf(\"Expected ID to be %q, got %q\", sa.ID(), l.ID)\n\t\t}\n\t\tif l.Now.IsZero() {\n\t\t\tt.Fatalf(\"Expected Now to be set, was not\")\n\t\t}\n\t\tif l.NumLeafs != 2 {\n\t\t\tt.Fatalf(\"Expected NumLeafs to be 2, got %v\", l.NumLeafs)\n\t\t}\n\t\tif len(l.Leafs) != 2 {\n\t\t\tt.Fatalf(\"Expected array to be len 2, got %v\", len(l.Leafs))\n\t\t}\n\t\tfor _, ln := range l.Leafs {\n\t\t\tif ln.Account == acc1.Name {\n\t\t\t\tif ln.OutMsgs != 1 || ln.OutBytes == 0 || ln.InMsgs != 0 || ln.InBytes != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected 1 OutMsgs/Bytes and 0 InMsgs/Bytes, got %+v\", ln)\n\t\t\t\t}\n\t\t\t} else if ln.Account == acc2.Name {\n\t\t\t\tif ln.OutMsgs != 2 || ln.OutBytes == 0 || ln.InMsgs != 0 || ln.InBytes != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected 2 OutMsgs/Bytes and 0 InMsgs/Bytes, got %+v\", ln)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Expected account to be %q or %q, got %q\", acc1.Name, acc2.Name, ln.Account)\n\t\t\t}\n\t\t\tif ln.Name != \"hub\" {\n\t\t\t\tt.Fatalf(\"Expected name to be %q, got %q\", \"hub\", ln.Name)\n\t\t\t}\n\t\t\tif !ln.IsSpoke {\n\t\t\t\tt.Fatal(\"Expected leafnode connection to be spoke\")\n\t\t\t}\n\t\t\tif !ln.IsIsolated {\n\t\t\t\tt.Fatal(\"Expected leafnode connection to be isolated\")\n\t\t\t}\n\t\t\tif ln.RTT == \"\" {\n\t\t\t\tt.Fatalf(\"RTT not tracked?\")\n\t\t\t}\n\t\t\tif ln.NumSubs != 3 {\n\t\t\t\tt.Fatalf(\"Expected 3 subs, got %v\", ln.NumSubs)\n\t\t\t}\n\t\t\tif len(ln.Subs) != 3 {\n\t\t\t\tt.Fatalf(\"Expected subs to be returned, got %v\", len(ln.Subs))\n\t\t\t}\n\t\t\tvar foundFoo bool\n\t\t\tvar foundBar bool\n\t\t\tfor _, sub := range ln.Subs {\n\t\t\t\tif sub == \"foo\" {\n\t\t\t\t\tfoundFoo = true\n\t\t\t\t} else if sub == \"bar\" {\n\t\t\t\t\tfoundBar = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !foundFoo {\n\t\t\t\tt.Fatal(\"Did not find subject foo\")\n\t\t\t}\n\t\t\tif !foundBar {\n\t\t\t\tt.Fatal(\"Did not find subject bar\")\n\t\t\t}\n\t\t}\n\t}\n\t// Make sure that if we don't ask for subs, we don't get them\n\tpollURL = fmt.Sprintf(\"http://127.0.0.1:%d/leafz\", sa.MonitorAddr().Port)\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollLeafz(t, sa, pollMode, pollURL, nil)\n\t\tfor _, ln := range l.Leafs {\n\t\t\tif ln.NumSubs != 3 {\n\t\t\t\tt.Fatalf(\"Number of subs should be 3, got %v\", ln.NumSubs)\n\t\t\t}\n\t\t\tif len(ln.Subs) != 0 {\n\t\t\t\tt.Fatalf(\"Subs should not have been returned, got %v\", ln.Subs)\n\t\t\t}\n\t\t}\n\t}\n\t// Make sure that we can request per account - existing account\n\tpollURL = fmt.Sprintf(\"http://127.0.0.1:%d/leafz?acc=%s\", sa.MonitorAddr().Port, acc1.Name)\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Account: acc1.Name})\n\t\tfor _, ln := range l.Leafs {\n\t\t\tif ln.Account != acc1.Name {\n\t\t\t\tt.Fatalf(\"Expected leaf node to be from account %s, got: %v\", acc1.Name, ln)\n\t\t\t}\n\t\t}\n\t\tif len(l.Leafs) != 1 {\n\t\t\tt.Fatalf(\"Expected only two leaf node for this account, got: %v\", len(l.Leafs))\n\t\t}\n\t}\n\t// Make sure that we can request per account - non existing account\n\tpollURL = fmt.Sprintf(\"http://127.0.0.1:%d/leafz?acc=%s\", sa.MonitorAddr().Port, \"DOESNOTEXIST\")\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollLeafz(t, sa, pollMode, pollURL, &LeafzOptions{Account: \"DOESNOTEXIST\"})\n\t\tif len(l.Leafs) != 0 {\n\t\t\tt.Fatalf(\"Expected no leaf node for this account, got: %v\", len(l.Leafs))\n\t\t}\n\t}\n\t// Now polling server B.\n\tpollURL = fmt.Sprintf(\"http://127.0.0.1:%d/leafz?subs=1\", sb.MonitorAddr().Port)\n\tfor pollMode := 1; pollMode < 2; pollMode++ {\n\t\tl := pollLeafz(t, sb, pollMode, pollURL, &LeafzOptions{Subscriptions: true})\n\t\tif l.ID != sb.ID() {\n\t\t\tt.Fatalf(\"Expected ID to be %q, got %q\", sb.ID(), l.ID)\n\t\t}\n\t\tif l.Now.IsZero() {\n\t\t\tt.Fatalf(\"Expected Now to be set, was not\")\n\t\t}\n\t\tif l.NumLeafs != 2 {\n\t\t\tt.Fatalf(\"Expected NumLeafs to be 1, got %v\", l.NumLeafs)\n\t\t}\n\t\tif len(l.Leafs) != 2 {\n\t\t\tt.Fatalf(\"Expected array to be len 2, got %v\", len(l.Leafs))\n\t\t}\n\t\tfor _, ln := range l.Leafs {\n\t\t\tif ln.Account == acc1.Name {\n\t\t\t\tif ln.OutMsgs != 0 || ln.OutBytes != 0 || ln.InMsgs != 1 || ln.InBytes == 0 {\n\t\t\t\t\tt.Fatalf(\"Expected 1 InMsgs/Bytes and 0 OutMsgs/Bytes, got %+v\", ln)\n\t\t\t\t}\n\t\t\t} else if ln.Account == acc2.Name {\n\t\t\t\tif ln.OutMsgs != 0 || ln.OutBytes != 0 || ln.InMsgs != 2 || ln.InBytes == 0 {\n\t\t\t\t\tt.Fatalf(\"Expected 2 InMsgs/Bytes and 0 OutMsgs/Bytes, got %+v\", ln)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Expected account to be %q or %q, got %q\", acc1.Name, acc2.Name, ln.Account)\n\t\t\t}\n\t\t\tif ln.Name != leafName {\n\t\t\t\tt.Fatalf(\"Expected name to be %q, got %q\", leafName, ln.Name)\n\t\t\t}\n\t\t\tif ln.IsSpoke {\n\t\t\t\tt.Fatal(\"Expected leafnode connection to be hub\")\n\t\t\t}\n\t\t\tif ln.RTT == \"\" {\n\t\t\t\tt.Fatalf(\"RTT not tracked?\")\n\t\t\t}\n\t\t\t// LDS should be only one.\n\t\t\tif ln.NumSubs != 5 || len(ln.Subs) != 5 {\n\t\t\t\tt.Fatalf(\"Expected 5 subs, got %v (%v)\", ln.NumSubs, ln.Subs)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc pollAccountz(t *testing.T, s *Server, mode int, url string, opts *AccountzOptions) *Accountz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tbody := readBody(t, url)\n\t\ta := &Accountz{}\n\t\tif err := json.Unmarshal(body, a); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn a\n\t}\n\ta, err := s.Accountz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on Accountz(): %v\", err)\n\t}\n\treturn a\n}\n\nfunc pollAccountStatz(t *testing.T, s *Server, mode int, url string, opts *AccountStatzOptions) *AccountStatz {\n\tt.Helper()\n\tif mode == 0 {\n\t\tbody := readBody(t, url)\n\t\tas := &AccountStatz{}\n\t\tif err := json.Unmarshal(body, as); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn as\n\t}\n\tas, err := s.AccountStatz(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on AccountStatz(): %v\", err)\n\t}\n\treturn as\n}\n\nfunc TestMonitorAccountz(t *testing.T) {\n\ts := RunServer(DefaultMonitorOptions())\n\tdefer s.Shutdown()\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s\", s.MonitorAddr().Port, AccountzPath), nil)\n\n\t\trequire_Equal(t, a.ID, s.ID())\n\t\trequire_Equal(t, len(a.Accounts), 2)\n\t\tfor _, acc := range a.Accounts {\n\t\t\tswitch acc {\n\t\t\tcase DEFAULT_SYSTEM_ACCOUNT, DEFAULT_GLOBAL_ACCOUNT:\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unexpected account: %s\", acc)\n\t\t\t}\n\t\t}\n\t\trequire_Equal(t, a.SystemAccount, DEFAULT_SYSTEM_ACCOUNT)\n\t}\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?acc=$SYS\", s.MonitorAddr().Port, AccountzPath), &AccountzOptions{Account: DEFAULT_SYSTEM_ACCOUNT})\n\n\t\trequire_NotNil(t, a.Account)\n\t\trequire_Equal(t, a.Account.AccountName, DEFAULT_SYSTEM_ACCOUNT)\n\t\trequire_Equal(t, a.Account.NameTag, DEFAULT_SYSTEM_ACCOUNT)\n\t\trequire_True(t, a.Account.IsSystem)\n\t\trequire_Equal(t, a.SystemAccount, DEFAULT_SYSTEM_ACCOUNT)\n\t}\n}\n\nfunc TestMonitorAccountStatz(t *testing.T) {\n\ts := RunServer(DefaultMonitorOptions())\n\tdefer s.Shutdown()\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountStatz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?unused=1\", s.MonitorAddr().Port, AccountStatzPath), &AccountStatzOptions{IncludeUnused: true})\n\n\t\trequire_Equal(t, a.ID, s.ID())\n\t\trequire_Equal(t, len(a.Accounts), 2)\n\n\t\t// Check accounts.\n\t\tfor _, acc := range a.Accounts {\n\t\t\tswitch acc.Account {\n\t\t\tcase DEFAULT_GLOBAL_ACCOUNT:\n\t\t\t\trequire_Equal(t, acc.Name, DEFAULT_GLOBAL_ACCOUNT)\n\t\t\t\trequire_Equal(t, acc.Conns, 0)\n\t\t\tcase DEFAULT_SYSTEM_ACCOUNT:\n\t\t\t\trequire_Equal(t, acc.Name, DEFAULT_SYSTEM_ACCOUNT)\n\t\t\t\trequire_Equal(t, acc.Conns, 0)\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unexpected account: %+v\", acc)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc runMonitorServerWithOperator(t *testing.T, sysName, accName string) ([]*Server, nkeys.KeyPair, nkeys.KeyPair) {\n\tt.Helper()\n\n\tresetPreviousHTTPConnections()\n\n\tsysKp, sysPub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(sysPub)\n\tsysClaim.Name = sysName\n\tsysJwt := encodeClaim(t, sysClaim, sysPub)\n\n\taccKp, accPub := createKey(t)\n\taccClaim := jwt.NewAccountClaims(accPub)\n\taccClaim.Name = accName\n\taccClaim.Limits.JetStreamLimits.DiskStorage = -1\n\taccClaim.Limits.JetStreamLimits.MemoryStorage = -1\n\n\taccJwt := encodeClaim(t, accClaim, accPub)\n\n\tvar servers []*Server\n\n\t// Main cluster\n\tfor i, test := range []struct {\n\t\tport int\n\t\tmport int\n\t\tcport int\n\t\troute1 int\n\t\tgport int\n\t\tgateway1 int\n\t\tlport int\n\t}{\n\t\t{7500, 7501, 7502, 5502, 8500, 8503, 7433},\n\t\t{5500, 5501, 5502, 7502, 8501, 8503, 7434},\n\t\t{6050, 6051, 6052, 7502, 8502, 8503, 7435},\n\t} {\n\t\tdir := t.TempDir()\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\thttp: 127.0.0.1:%d\n\t\t\tjetstream: {\n\t\t\t\tmax_mem_store: 10Mb\n\t\t\t\tmax_file_store: 10Mb\n\t\t\t\tstore_dir: '%s'\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tname: c1\n\t\t\t\tlisten: %d\n\t\t\t\troutes: [\n\t\t\t\t\tnats-route://127.0.0.1:%d,\n\t\t\t\t]\n\t\t\t}\n\t\t\tgateway {\n\t\t\t\tname: c1\n\t\t\t\tport: %d\n\t\t\t\tgateways: [\n\t\t\t\t\t{name: c2, urls: [nats://127.0.0.1:%d]},\n\t\t\t\t]\n\t\t\t}\n\t\t\tleafnodes {\n\t\t\t\tlisten: %d\n\t\t\t}\n\t\t\tserver_name: %s\n\t\t\toperator: %s\n\t\t\tresolver: MEMORY\n\t\t\tsystem_account: %s\n\t\t\tresolver_preload {\n\t\t\t\t%s : %s\n\t\t\t\t%s : %s\n\t\t\t}\n\t\t`, test.port, test.mport, dir, test.cport, test.route1, test.gport, test.gateway1, test.lport, fmt.Sprintf(\"n%d\", i), ojwt, sysPub, accPub, accJwt, sysPub, sysJwt)))\n\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tservers = append(servers, s)\n\t}\n\n\t// Gateway\n\tfor i, test := range []struct {\n\t\tport int\n\t\tmport int\n\t\tcport int\n\t\tgport int\n\t\tgateway1 int\n\t}{\n\t\t{7503, 7504, 6053, 8503, 8500},\n\t} {\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\thttp: 127.0.0.1:%d\n\t\t\tcluster {\n\t\t\t\tname: c2\n\t\t\t\tlisten: %d\n\t\t\t}\n\t\t\tgateway {\n\t\t\t\tname: c2\n\t\t\t\tport: %d\n\t\t\t\tgateways: [\n\t\t\t\t\t{name: c1, urls: [nats://127.0.0.1:%d]},\n\t\t\t\t]\n\t\t\t}\n\t\t\tserver_name: %s\n\t\t\toperator: %s\n\t\t\tresolver: MEMORY\n\t\t\tsystem_account: %s\n\t\t\tresolver_preload {\n\t\t\t\t%s : %s\n\t\t\t\t%s : %s\n\t\t\t}\n\t\t`, test.port, test.mport, test.cport, test.gport, test.gateway1, fmt.Sprintf(\"n%d\", i+3), ojwt, sysPub, accPub, accJwt, sysPub, sysJwt)))\n\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tservers = append(servers, s)\n\t}\n\n\t_, credsFile := createUser(t, accKp)\n\n\t// Leafnode\n\tfor _, test := range []struct {\n\t\tport int\n\t\tmport int\n\t\tlport int\n\t}{\n\t\t{7505, 7506, 7433},\n\t} {\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\thttp: 127.0.0.1:%d\n\t\t\tleafnodes: {\n\t\t\t\tremotes: [\n\t\t\t\t\t{url: \"nats://127.0.0.1:%d\", credentials: \"%s\", account: \"APP\"},\n\t\t\t\t]\n\t\t\t}\n\t\t\taccounts {\n\t\t\t\tSYS: {\n\t\t\t\t\tusers: [\n\t\t\t\t\t\t{user: \"sys\", password: \"sys\"},\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t\tAPP: {\n\t\t\t\t\tusers: [\n\t\t\t\t\t\t{user: \"app\", password: \"app\"},\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t\tno_auth_user: \"app\"\n\t\t`, test.port, test.mport, test.lport, credsFile)))\n\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tservers = append(servers, s)\n\t}\n\n\tcheckForJSClusterUp(t, servers[:3]...)\n\twaitForOutboundGateways(t, servers[0], 1, 2*time.Second)\n\twaitForOutboundGateways(t, servers[3], 1, 2*time.Second)\n\n\treturn servers, sysKp, accKp\n}\n\nfunc createUser(t *testing.T, accKp nkeys.KeyPair) (string, string) {\n\tt.Helper()\n\n\tukp, _ := nkeys.CreateUser()\n\tseed, _ := ukp.Seed()\n\tupub, _ := ukp.PublicKey()\n\tuclaim := newJWTTestUserClaims()\n\tuclaim.Subject = upub\n\tujwt, err := uclaim.Encode(accKp)\n\trequire_NoError(t, err)\n\treturn upub, genCredsFile(t, ujwt, seed)\n}\n\nfunc TestMonitorAccountzOperatorMode(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, sysKp, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\tsysPub, _ := sysKp.PublicKey()\n\taccPub, _ := accKp.PublicKey()\n\n\t_, aCreds := createUser(t, accKp)\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s\", s.MonitorAddr().Port, AccountzPath), nil)\n\n\t\trequire_Equal(t, a.ID, s.ID())\n\t\trequire_Equal(t, len(a.Accounts), 3)\n\t\tfor _, acc := range a.Accounts {\n\t\t\tswitch acc {\n\t\t\tcase sysPub, accPub, DEFAULT_GLOBAL_ACCOUNT:\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unexpected account: %s\", acc)\n\t\t\t}\n\t\t}\n\t\trequire_Equal(t, a.SystemAccount, sysPub)\n\t}\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?acc=%s\", s.MonitorAddr().Port, AccountzPath, sysPub), &AccountzOptions{Account: sysPub})\n\n\t\trequire_NotNil(t, a.Account)\n\t\trequire_Equal(t, a.Account.AccountName, sysPub)\n\t\trequire_Equal(t, a.Account.NameTag, sysName)\n\t\trequire_True(t, a.Account.IsSystem)\n\t\trequire_Equal(t, a.SystemAccount, sysPub)\n\t}\n\n}\n\nfunc TestMonitorAccountStatzOperatorMode(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, sysKp, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\tsysPub, _ := sysKp.PublicKey()\n\taccPub, _ := accKp.PublicKey()\n\n\t_, aCreds := createUser(t, accKp)\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollAccountStatz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?unused=1\", s.MonitorAddr().Port, AccountStatzPath), &AccountStatzOptions{IncludeUnused: true})\n\n\t\trequire_Equal(t, a.ID, s.ID())\n\t\trequire_Equal(t, len(a.Accounts), 3)\n\n\t\t// Check accounts.\n\t\tfor _, acc := range a.Accounts {\n\t\t\tswitch acc.Account {\n\t\t\tcase accPub:\n\t\t\t\trequire_Equal(t, acc.Name, accName)\n\t\t\t\trequire_True(t, acc.NumSubs > 0)\n\t\t\t\trequire_Equal(t, acc.Conns, 1)\n\t\t\tcase sysPub:\n\t\t\t\trequire_Equal(t, acc.Name, sysName)\n\t\t\t\trequire_Equal(t, acc.Conns, 0)\n\t\t\tcase DEFAULT_GLOBAL_ACCOUNT:\n\t\t\t\trequire_Equal(t, acc.Name, DEFAULT_GLOBAL_ACCOUNT)\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unexpected account: %+v\", acc)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorAccountStatzDataStatsOperatorMode(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, _, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\t// First three servers are the cluster.\n\tn0 := srvs[0]\n\tn1 := srvs[1]\n\n\t// Gateway server.\n\tn3 := srvs[3]\n\n\t// Leafnode server.\n\tn4 := srvs[4]\n\n\taccPub, _ := accKp.PublicKey()\n\n\t_, aCreds := createUser(t, accKp)\n\n\tn0c, err := nats.Connect(n0.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer n0c.Close()\n\n\tn1c, err := nats.Connect(n1.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer n1c.Close()\n\n\tn3c, err := nats.Connect(n3.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer n3c.Close()\n\n\t// No auth user for leafnode.\n\tn4c, err := nats.Connect(n4.ClientURL())\n\trequire_NoError(t, err)\n\tdefer n4c.Close()\n\n\t// Subscription over a route.\n\t_, err = n1c.Subscribe(\"foo\", func(m *nats.Msg) {})\n\trequire_NoError(t, err)\n\n\t// Subscription over a gateway.\n\t_, err = n3c.Subscribe(\"bar\", func(m *nats.Msg) {})\n\trequire_NoError(t, err)\n\n\t// Subscription over a leafnode.\n\t_, err = n4c.Subscribe(\"baz\", func(m *nats.Msg) {})\n\trequire_NoError(t, err)\n\n\t// Subscription propagation.\n\ttime.Sleep(10 * time.Millisecond)\n\n\terr = n0c.Publish(\"foo\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\n\terr = n0c.Publish(\"bar\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\n\terr = n0c.Publish(\"baz\", []byte(\"Hello\"))\n\trequire_NoError(t, err)\n\n\t// Publish propagation.\n\ttime.Sleep(10 * time.Millisecond)\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\tfor _, s := range srvs {\n\t\t\ta := pollAccountStatz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?unused=1\", s.MonitorAddr().Port, AccountStatzPath), &AccountStatzOptions{IncludeUnused: true})\n\n\t\t\tfor _, acc := range a.Accounts {\n\t\t\t\tif acc.Account != accPub {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tswitch s.Name() {\n\t\t\t\tcase \"n0\":\n\t\t\t\t\t// Should have received three messages due to the three publishes.\n\t\t\t\t\t// Should have sent one over a route to n1 for foo.\n\t\t\t\t\t// Should have sent one over a gateway to n3 for bar.\n\t\t\t\t\t// Should have sent one over a leaf node to n4 for baz.\n\t\t\t\t\trequire_Equal(t, acc.Sent.Msgs, 3)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Routes.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Gateways.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Leafs.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Received.Msgs, 3)\n\t\t\t\tcase \"n1\":\n\t\t\t\t\t// Should have sent 1 message to a client.\n\t\t\t\t\t// Should have received 1 message from n0.\n\t\t\t\t\trequire_Equal(t, acc.Sent.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Routes.Msgs, 0)\n\t\t\t\t\trequire_Equal(t, acc.Received.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Received.Routes.Msgs, 1)\n\t\t\t\tcase \"n2\":\n\t\t\t\t\t// Should have not received anything.\n\t\t\t\t\trequire_Equal(t, acc.Sent.Msgs, 0)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Bytes, 0)\n\t\t\t\t// Gateway, connected to n0\n\t\t\t\tcase \"n3\":\n\t\t\t\t\t// Should have received 1 message from n0.\n\t\t\t\t\t// Should have sent 1 message to a client.\n\t\t\t\t\trequire_Equal(t, acc.Sent.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Sent.Gateways.Msgs, 0)\n\t\t\t\t\trequire_Equal(t, acc.Received.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Received.Gateways.Msgs, 1)\n\t\t\t\t// Leafnode, connected to n0\n\t\t\t\tcase \"n4\":\n\t\t\t\t\t// Should have received 1 message from n0.\n\t\t\t\t\t// Should have sent 1 message to a client.\n\t\t\t\t\trequire_Equal(t, acc.Sent.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Received.Msgs, 1)\n\t\t\t\t\trequire_Equal(t, acc.Received.Leafs.Msgs, 1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorAccountzAccountIssuerUpdate(t *testing.T) {\n\t// create an operator set of keys\n\tokp, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\topk, err := okp.PublicKey()\n\trequire_NoError(t, err)\n\n\t// create the system account\n\t_, sysPK := createKey(t)\n\tsysAc := jwt.NewAccountClaims(sysPK)\n\tsysAc.Name = \"SYS\"\n\tsysJwt, err := sysAc.Encode(okp)\n\trequire_NoError(t, err)\n\n\t// create the operator with the system\n\toc := jwt.NewOperatorClaims(opk)\n\toc.Name = \"O\"\n\t// add a signing keys\n\tosk1, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\topk1, err := osk1.PublicKey()\n\trequire_NoError(t, err)\n\t// add a second signing key\n\tosk2, err := nkeys.CreateOperator()\n\trequire_NoError(t, err)\n\topk2, err := osk2.PublicKey()\n\trequire_NoError(t, err)\n\toc.SigningKeys.Add(opk1, opk2)\n\t// set the system account\n\toc.SystemAccount = sysPK\n\t// generate\n\toJWT, err := oc.Encode(okp)\n\trequire_NoError(t, err)\n\n\t// create an account\n\takp, apk := createKey(t)\n\tac := jwt.NewAccountClaims(apk)\n\tac.Name = \"A\"\n\t// sign with the signing key\n\taJWT, err := ac.Encode(osk1)\n\trequire_NoError(t, err)\n\n\t// build the mem-resolver\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\thttp: 127.0.0.1:-1\n\t\toperator = %s\n\t\tresolver = MEMORY\n\t\tsystem_account: %s\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, oJWT, sysPK, sysPK, sysJwt, apk, aJWT)))\n\n\t// start the server\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// create an user for account A, or we don't see\n\t// the account in accountsz\n\tcreateUser := func() (string, string) {\n\t\tukp, _ := nkeys.CreateUser()\n\t\tseed, _ := ukp.Seed()\n\t\tupub, _ := ukp.PublicKey()\n\t\tuclaim := newJWTTestUserClaims()\n\t\tuclaim.Subject = upub\n\t\tujwt, err := uclaim.Encode(akp)\n\t\trequire_NoError(t, err)\n\t\treturn upub, genCredsFile(t, ujwt, seed)\n\t}\n\n\t_, aCreds := createUser()\n\t// connect the user\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// lookup the account\n\tdata := readBody(t, fmt.Sprintf(\"http://127.0.0.1:%d%s?acc=%s\", s.MonitorAddr().Port, AccountzPath, apk))\n\tvar ci Accountz\n\trequire_NoError(t, json.Unmarshal(data, &ci))\n\trequire_Equal(t, ci.Account.IssuerKey, opk1)\n\n\t// now update the account\n\taJWT, err = ac.Encode(osk2)\n\trequire_NoError(t, err)\n\n\tupdatedConf := []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\thttp: 127.0.0.1:-1\n\t\toperator = %s\n\t\tresolver = MEMORY\n\t\tsystem_account: %s\n\t\tresolver_preload = {\n\t\t\t%s : %s\n\t\t\t%s : %s\n\t\t}\n\t`, oJWT, sysPK, sysPK, sysJwt, apk, aJWT))\n\t// update the configuration file\n\trequire_NoError(t, os.WriteFile(conf, updatedConf, 0666))\n\t// reload\n\trequire_NoError(t, s.Reload())\n\n\tdata = readBody(t, fmt.Sprintf(\"http://127.0.0.1:%d%s?acc=%s\", s.MonitorAddr().Port, AccountzPath, apk))\n\trequire_NoError(t, json.Unmarshal(data, &ci))\n\trequire_Equal(t, ci.Account.IssuerKey, opk2)\n}\n\nfunc TestMonitorAuthorizedUsers(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(seed)\n\tusrNKey, _ := kp.PublicKey()\n\topts := DefaultMonitorOptions()\n\topts.Nkeys = []*NkeyUser{{Nkey: string(usrNKey)}}\n\topts.Users = []*User{{Username: \"user\", Password: \"pwd\"}}\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckAuthUser := func(expected string) {\n\t\tt.Helper()\n\t\tresetPreviousHTTPConnections()\n\t\turl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?auth=true\", s.MonitorAddr().Port)\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tconnz := pollConnz(t, s, mode, url, &ConnzOptions{Username: true})\n\t\t\tif l := len(connz.Conns); l != 1 {\n\t\t\t\tt.Fatalf(\"Expected 1, got %v\", l)\n\t\t\t}\n\t\t\tconn := connz.Conns[0]\n\t\t\tau := conn.AuthorizedUser\n\t\t\tif au == _EMPTY_ {\n\t\t\t\tt.Fatal(\"AuthorizedUser is empty!\")\n\t\t\t}\n\t\t\tif au != expected {\n\t\t\t\tt.Fatalf(\"Expected %q, got %q\", expected, au)\n\t\t\t}\n\t\t}\n\t}\n\n\tc := natsConnect(t, fmt.Sprintf(\"nats://user:pwd@127.0.0.1:%d\", opts.Port))\n\tdefer c.Close()\n\tcheckAuthUser(\"user\")\n\tc.Close()\n\n\tc = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port),\n\t\tnats.Nkey(usrNKey, func(nonce []byte) ([]byte, error) {\n\t\t\treturn kp.Sign(nonce)\n\t\t}))\n\tdefer c.Close()\n\t// we should get the user's NKey\n\tcheckAuthUser(usrNKey)\n\tc.Close()\n\n\ts.Shutdown()\n\topts = DefaultMonitorOptions()\n\topts.Authorization = \"sometoken\"\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port),\n\t\tnats.Token(\"sometoken\"))\n\tdefer c.Close()\n\t// We should get the token specified by the user\n\tcheckAuthUser(\"[REDACTED]\")\n\tc.Close()\n\ts.Shutdown()\n\n\topts = DefaultMonitorOptions()\n\t// User an operator seed\n\tkp, _ = nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\topts.TrustedKeys = []string{pub}\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tupub, _ := nkp.PublicKey()\n\tnuc := jwt.NewUserClaims(upub)\n\tjwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\tbuildMemAccResolver(s)\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc = natsConnect(t, fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port),\n\t\tnats.UserJWT(\n\t\t\tfunc() (string, error) { return jwt, nil },\n\t\t\tfunc(nonce []byte) ([]byte, error) { return nkp.Sign(nonce) }))\n\tdefer c.Close()\n\t// we should get the user's pubkey\n\tcheckAuthUser(upub)\n}\n\n// Helper function to check that a JS cluster is formed\nfunc checkForJSClusterUp(t *testing.T, servers ...*Server) {\n\tt.Helper()\n\t// We will use the other JetStream helpers here.\n\tc := &cluster{t: t, servers: servers}\n\tc.checkClusterFormed()\n\tc.waitOnClusterReady()\n}\n\nfunc pollJsz(t *testing.T, s *Server, mode int, url string, opts *JSzOptions) *JSInfo {\n\tt.Helper()\n\n\tif mode == 0 {\n\t\tbody := readBody(t, url)\n\t\tinfo := &JSInfo{}\n\t\terr := json.Unmarshal(body, info)\n\t\trequire_NoError(t, err)\n\t\treturn info\n\t}\n\n\tinfo, err := s.Jsz(opts)\n\trequire_NoError(t, err)\n\treturn info\n}\n\nfunc TestMonitorJszNonJszServer(t *testing.T) {\n\tsrv := RunServer(DefaultOptions())\n\tdefer srv.Shutdown()\n\n\tif !srv.ReadyForConnections(5 * time.Second) {\n\t\tt.Fatalf(\"server did not become ready\")\n\t}\n\n\tjsi, err := srv.Jsz(&JSzOptions{})\n\tif err != nil {\n\t\tt.Fatalf(\"jsi failed: %v\", err)\n\t}\n\tif jsi.ID != srv.ID() {\n\t\tt.Fatalf(\"did not receive valid info\")\n\t}\n\n\tjsi, err = srv.Jsz(&JSzOptions{LeaderOnly: true})\n\tif !errors.Is(err, errSkipZreq) {\n\t\tt.Fatalf(\"expected a skip z req error: %v\", err)\n\t}\n\tif jsi != nil {\n\t\tt.Fatalf(\"expected no jsi: %v\", jsi)\n\t}\n}\n\nfunc TestMonitorJsz(t *testing.T) {\n\treadJsInfo := func(url string) *JSInfo {\n\t\tt.Helper()\n\t\tbody := readBody(t, url)\n\t\tinfo := &JSInfo{}\n\t\terr := json.Unmarshal(body, info)\n\t\trequire_NoError(t, err)\n\t\treturn info\n\t}\n\tsrvs := []*Server{}\n\tfor _, test := range []struct {\n\t\tport int\n\t\tmport int\n\t\tcport int\n\t\trouted int\n\t}{\n\t\t{7500, 7501, 7502, 5502},\n\t\t{5500, 5501, 5502, 7502},\n\t} {\n\t\ts, _ := runMonitorJSServer(t, test.port, test.mport, test.cport, test.routed)\n\t\tdefer s.Shutdown()\n\t\tsrvs = append(srvs, s)\n\t}\n\tcheckClusterFormed(t, srvs...)\n\tcheckForJSClusterUp(t, srvs...)\n\n\tnc := natsConnect(t, \"nats://usr:pwd@127.0.0.1:7500\")\n\tdefer nc.Close()\n\tjs, err := nc.JetStream(nats.MaxWait(5 * time.Second))\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"my-stream-replicated\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tReplicas: 2,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"my-stream-non-replicated\",\n\t\tSubjects: []string{\"baz\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"my-stream-mirror\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"my-stream-replicated\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"my-stream-replicated\", &nats.ConsumerConfig{\n\t\tDurable: \"my-consumer-replicated\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"my-stream-non-replicated\", &nats.ConsumerConfig{\n\t\tDurable: \"my-consumer-non-replicated\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\t_, err = js.AddConsumer(\"my-stream-mirror\", &nats.ConsumerConfig{\n\t\tDurable: \"my-consumer-mirror\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\tnc.Flush()\n\t_, err = js.Publish(\"foo\", nil)\n\trequire_NoError(t, err)\n\t// Wait for mirror replication\n\ttime.Sleep(200 * time.Millisecond)\n\n\tmonUrl1 := fmt.Sprintf(\"http://127.0.0.1:%d/jsz\", 7501)\n\tmonUrl2 := fmt.Sprintf(\"http://127.0.0.1:%d/jsz\", 5501)\n\n\tt.Run(\"default\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url)\n\t\t\tif len(info.AccountDetails) != 0 {\n\t\t\t\tt.Fatalf(\"expected no account to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif info.Streams == 0 {\n\t\t\t\tt.Fatalf(\"expected stream count to be 3 but got %d\", info.Streams)\n\t\t\t}\n\t\t\tif info.Consumers == 0 {\n\t\t\t\tt.Fatalf(\"expected consumer count to be 3 but got %d\", info.Consumers)\n\t\t\t}\n\t\t\tif info.Messages != 2 {\n\t\t\t\tt.Fatalf(\"expected two message but got %d\", info.Messages)\n\t\t\t}\n\t\t\tif info.Limits.MaxHAAssets != 1000 {\n\t\t\t\tt.Fatalf(\"expected max_ha_assets limit to be 1000 got %v\", info.Limits)\n\t\t\t}\n\t\t\tif info.Total != 2 {\n\t\t\t\tt.Fatalf(\"expected total to be 2 but got %d\", info.Total)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"accounts\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?accounts=true\")\n\n\t\t\trequire_Equal(t, len(info.AccountDetails), 2)\n\n\t\t\tfor _, acc := range info.AccountDetails {\n\t\t\t\tswitch acc.Id {\n\t\t\t\tcase \"ACC\":\n\t\t\t\t\trequire_Equal(t, acc.Name, \"ACC\")\n\t\t\t\tcase \"BCC_TO_HAVE_ONE_EXTRA\":\n\t\t\t\t\trequire_Equal(t, acc.Name, \"BCC_TO_HAVE_ONE_EXTRA\")\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected account: %s\", acc.Name)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"accounts reserved metrics\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?accounts=true&acc=ACC\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected single account\")\n\t\t\t}\n\t\t\tacc := info.AccountDetails[0]\n\t\t\tgot := int(acc.ReservedMemory)\n\t\t\texpected := 5242880\n\t\t\tif got != expected {\n\t\t\t\tt.Errorf(\"Expected: %v, got: %v\", expected, got)\n\t\t\t}\n\t\t\tgot = int(acc.ReservedStore)\n\t\t\texpected = 4194304\n\t\t\tif got != expected {\n\t\t\t\tt.Errorf(\"Expected: %v, got: %v\", expected, got)\n\t\t\t}\n\n\t\t\tinfo = readJsInfo(url + \"?accounts=true&acc=BCC_TO_HAVE_ONE_EXTRA\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected single account\")\n\t\t\t}\n\t\t\tacc = info.AccountDetails[0]\n\t\t\tgot = int(acc.ReservedMemory)\n\t\t\texpected = -1\n\t\t\tif got != expected {\n\t\t\t\tt.Errorf(\"Expected: %v, got: %v\", expected, got)\n\t\t\t}\n\t\t\tgot = int(acc.ReservedStore)\n\t\t\texpected = -1\n\t\t\tif got != expected {\n\t\t\t\tt.Errorf(\"Expected: %v, got: %v\", expected, got)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"offset-too-big\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?accounts=true&offset=10\")\n\t\t\tif len(info.AccountDetails) != 0 {\n\t\t\t\tt.Fatalf(\"expected no accounts to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"limit\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?accounts=true&limit=1\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif info := readJsInfo(url + \"?accounts=true&offset=1&limit=1\"); len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"offset-stable\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo1 := readJsInfo(url + \"?accounts=true&offset=0&limit=1\")\n\t\t\tif len(info1.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info1)\n\t\t\t}\n\t\t\tinfo2 := readJsInfo(url + \"?accounts=true&offset=0&limit=1\")\n\t\t\tif len(info2.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info2)\n\t\t\t}\n\t\t\tif info1.AccountDetails[0].Name != info2.AccountDetails[0].Name {\n\t\t\t\tt.Fatalf(\"absent changes, same offset should result in same account but got: %v %v\",\n\t\t\t\t\tinfo1.AccountDetails[0].Name, info2.AccountDetails[0].Name)\n\t\t\t}\n\n\t\t\tinfo1 = readJsInfo(url + \"?accounts=true&offset=1&limit=1\")\n\t\t\tif len(info1.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info1)\n\t\t\t}\n\t\t\tinfo2 = readJsInfo(url + \"?accounts=true&offset=1&limit=1\")\n\t\t\tif len(info2.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected one account to be returned by %s but got %v\", url, info2)\n\t\t\t}\n\t\t\tif info1.AccountDetails[0].Name != info2.AccountDetails[0].Name {\n\t\t\t\tt.Fatalf(\"absent changes, same offset should result in same account but gut: %v %v\",\n\t\t\t\t\tinfo1.AccountDetails[0].Name, info2.AccountDetails[0].Name)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"filter-account\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif info.AccountDetails[0].Name != \"ACC\" {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(info.AccountDetails[0].Streams) != 0 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"streams\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC&streams=true\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(info.AccountDetails[0].Streams) == 0 {\n\t\t\t\tt.Fatalf(\"expected streams to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(info.AccountDetails[0].Streams[0].Consumer) != 0 {\n\t\t\t\tt.Fatalf(\"expected no consumers to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"stream-leader-only\", func(t *testing.T) {\n\t\t// First server\n\t\tinfo := readJsInfo(monUrl1 + \"?streams=true&stream-leader-only=1\")\n\t\tfor _, a := range info.AccountDetails {\n\t\t\tfor _, s := range a.Streams {\n\t\t\t\tif s.Cluster.Leader != srvs[0].serverName() {\n\t\t\t\t\tt.Fatalf(\"expected stream leader to be %s but got %s\", srvs[0].serverName(), s.Cluster.Leader)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tinfo = readJsInfo(monUrl2 + \"?streams=true&stream-leader-only=1\")\n\t\tfor _, a := range info.AccountDetails {\n\t\t\tfor _, s := range a.Streams {\n\t\t\t\tif s.Cluster.Leader != srvs[1].serverName() {\n\t\t\t\t\tt.Fatalf(\"expected stream leader to be %s but got %s\", srvs[0].serverName(), s.Cluster.Leader)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"consumers\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC&consumers=true\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(info.AccountDetails[0].Streams[0].Consumer) == 0 {\n\t\t\t\tt.Fatalf(\"expected consumers to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif info.AccountDetails[0].Streams[0].Config != nil {\n\t\t\t\tt.Fatal(\"Config expected to not be present\")\n\t\t\t}\n\t\t\tif info.AccountDetails[0].Streams[0].Consumer[0].Config != nil {\n\t\t\t\tt.Fatal(\"Config expected to not be present\")\n\t\t\t}\n\t\t\tif len(info.AccountDetails[0].Streams[0].ConsumerRaftGroups) != 0 {\n\t\t\t\tt.Fatalf(\"expected consumer raft groups to not be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"direct-consumers\", func(t *testing.T) {\n\t\t// It could take time for the sourcing to set up.\n\t\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\t\tinfo := readJsInfo(url + \"?acc=ACC&consumers=true&direct-consumers=true\")\n\t\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t\t}\n\t\t\t\tif slices.ContainsFunc(info.AccountDetails[0].Streams, func(stream StreamDetail) bool {\n\t\t\t\t\treturn len(stream.DirectConsumer) > 0\n\t\t\t\t}) {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"expected direct consumer info to be present on one of the servers\")\n\t\t})\n\t})\n\tt.Run(\"config\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC&consumers=true&config=true\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif info.AccountDetails[0].Streams[0].Config == nil {\n\t\t\t\tt.Fatal(\"Config expected to be present\")\n\t\t\t}\n\t\t\tif info.AccountDetails[0].Streams[0].Consumer[0].Config == nil {\n\t\t\t\tt.Fatal(\"Config expected to be present\")\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"replication\", func(t *testing.T) {\n\t\t// The replication lag may only be present on the leader\n\t\treplicationFound := false\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC&streams=true\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tstreamFound := false\n\t\t\tfor _, stream := range info.AccountDetails[0].Streams {\n\t\t\t\tif stream.Name == \"my-stream-mirror\" {\n\t\t\t\t\tstreamFound = true\n\t\t\t\t\tif stream.Mirror != nil {\n\t\t\t\t\t\treplicationFound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !streamFound {\n\t\t\t\tt.Fatalf(\"Did not locate my-stream-mirror stream in results\")\n\t\t\t}\n\t\t}\n\t\tif !replicationFound {\n\t\t\tt.Fatal(\"ReplicationLag expected to be present for my-stream-mirror stream\")\n\t\t}\n\t})\n\tt.Run(\"cluster-info\", func(t *testing.T) {\n\t\tfound := 0\n\t\tfor i, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"\")\n\t\t\tif info.Meta.Peer != getHash(info.Meta.Leader) {\n\t\t\t\tt.Fatalf(\"Invalid Peer: %+v\", info.Meta)\n\t\t\t}\n\t\t\tif info.Meta.Replicas != nil {\n\t\t\t\tfound++\n\t\t\t\tfor _, r := range info.Meta.Replicas {\n\t\t\t\t\tif r.Peer == _EMPTY_ {\n\t\t\t\t\t\tt.Fatalf(\"Replicas' Peer is empty: %+v\", r)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif info.Meta.Leader != srvs[i].Name() {\n\t\t\t\t\tt.Fatalf(\"received cluster info from non leader: leader %s, server: %s\", info.Meta.Leader, srvs[i].Name())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif found == 0 {\n\t\t\tt.Fatalf(\"did not receive cluster info from any node\")\n\t\t}\n\t\tif found > 1 {\n\t\t\tt.Fatalf(\"received cluster info from multiple nodes\")\n\t\t}\n\t})\n\tt.Run(\"meta-snapshot-stats\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url)\n\t\t\trequire_True(t, info.Meta != nil)\n\t\t\trequire_True(t, info.Meta.Snapshot != nil)\n\n\t\t\tsnapshot := info.Meta.Snapshot\n\n\t\t\t// In case no snapshots have happened there would be some pending entries.\n\t\t\tif snapshot.LastTime.IsZero() {\n\t\t\t\trequire_True(t, snapshot.PendingEntries >= 1)\n\t\t\t\trequire_True(t, snapshot.PendingSize >= 1)\n\t\t\t}\n\n\t\t\t// Force meta snapshots on both servers to test snapshot timing.\n\t\t\tfor _, srv := range srvs {\n\t\t\t\tif js := srv.getJetStream(); js != nil {\n\t\t\t\t\tif mg := js.getMetaGroup(); mg != nil {\n\t\t\t\t\t\tif snap, _, _, err := js.metaSnapshot(); err == nil {\n\t\t\t\t\t\t\tmg.InstallSnapshot(snap, false)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Wait for snapshot timing to be recorded\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\t// Get latest stats again.\n\t\t\tinfo = readJsInfo(url)\n\t\t\trequire_True(t, info.Meta != nil)\n\t\t\trequire_True(t, info.Meta.Snapshot != nil)\n\n\t\t\tsnapshot = info.Meta.Snapshot\n\n\t\t\trequire_True(t, !snapshot.LastTime.IsZero())\n\t\t\t// Assert that snapshot time is in UTC\n\t\t\trequire_Equal(t, snapshot.LastTime.Location(), time.UTC)\n\n\t\t\t// Assert that duration is non-negative and reasonable\n\t\t\trequire_True(t, snapshot.LastDuration >= 0)\n\t\t\trequire_True(t, snapshot.LastDuration < 30*time.Second)\n\n\t\t\t// Assert that snapshot time is recent.\n\t\t\trequire_True(t, time.Since(snapshot.LastTime) < 5*time.Minute)\n\t\t}\n\t})\n\tt.Run(\"account-non-existing\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=DOES_NOT_EXIST\")\n\t\t\tif len(info.AccountDetails) != 0 {\n\t\t\t\tt.Fatalf(\"expected no account to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"account-non-existing-with-stream-details\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=DOES_NOT_EXIST&streams=true\")\n\t\t\tif len(info.AccountDetails) != 0 {\n\t\t\t\tt.Fatalf(\"expected no account to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"unique-tag-exists\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url)\n\t\t\tif len(info.Config.UniqueTag) == 0 {\n\t\t\t\tt.Fatalf(\"expected unique_tag to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t}\n\t})\n\tt.Run(\"raftgroups\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url + \"?acc=ACC&consumers=true&raft=true\")\n\t\t\tif len(info.AccountDetails) != 1 {\n\t\t\t\tt.Fatalf(\"expected account ACC to be returned by %s but got %v\", url, info)\n\t\t\t}\n\n\t\t\t// We will have two streams and order is not guaranteed. So grab the one we want.\n\t\t\tvar si StreamDetail\n\t\t\tif info.AccountDetails[0].Streams[0].Name == \"my-stream-replicated\" {\n\t\t\t\tsi = info.AccountDetails[0].Streams[0]\n\t\t\t} else {\n\t\t\t\tsi = info.AccountDetails[0].Streams[1]\n\t\t\t}\n\n\t\t\tif len(si.Consumer) == 0 {\n\t\t\t\tt.Fatalf(\"expected consumers to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(si.ConsumerRaftGroups) == 0 {\n\t\t\t\tt.Fatalf(\"expected consumer raft groups to be returned by %s but got %v\", url, info)\n\t\t\t}\n\t\t\tif len(si.RaftGroup) == 0 {\n\t\t\t\tt.Fatal(\"expected stream raft group info to be included\")\n\t\t\t}\n\t\t\tcrgroup := si.ConsumerRaftGroups[0]\n\t\t\tif crgroup.Name != \"my-consumer-replicated\" && crgroup.Name != \"my-consumer-mirror\" {\n\t\t\t\tt.Fatalf(\"expected consumer name to be included in raft group info, got: %v\", crgroup.Name)\n\t\t\t}\n\t\t\tif len(crgroup.RaftGroup) == 0 {\n\t\t\t\tt.Fatal(\"expected consumer raft group info to be included\")\n\t\t\t}\n\t\t\trequire_True(t, si.Cluster.SystemAcc)\n\t\t\trequire_Equal(t, si.Cluster.TrafficAcc, \"SYS\")\n\t\t}\n\t})\n\tt.Run(\"js-api-level\", func(t *testing.T) {\n\t\tfor _, url := range []string{monUrl1, monUrl2} {\n\t\t\tinfo := readJsInfo(url)\n\t\t\trequire_Equal(t, info.API.Level, JSApiLevel)\n\t\t}\n\t})\n}\n\nfunc TestMonitorJszOperatorMode(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, _, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\taccPub, _ := accKp.PublicKey()\n\n\t_, aCreds := createUser(t, accKp)\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// Create a stream so the APP account shows up in Jsz.\n\tjs, err := nc.JetStream(nats.MaxWait(5 * time.Second))\n\trequire_NoError(t, err)\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"my-stream\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tMaxBytes: 1024,\n\t})\n\trequire_NoError(t, err)\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\ta := pollJsz(t, s, pollMode, fmt.Sprintf(\"http://127.0.0.1:%d%s?accounts=1\", s.MonitorAddr().Port, JszPath), &JSzOptions{Accounts: true})\n\n\t\trequire_Equal(t, a.ID, s.ID())\n\t\trequire_Equal(t, len(a.AccountDetails), 1)\n\n\t\t// Check accounts.\n\t\td := a.AccountDetails[0]\n\t\trequire_Equal(t, d.Id, accPub)\n\t\trequire_Equal(t, d.Name, accName)\n\t}\n}\n\nfunc TestMonitorReloadTLSConfig(t *testing.T) {\n\ttemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttps: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: '%s'\n\t\t\tkey_file: '%s'\n\t\t\tca_file: '../test/configs/certs/ca.pem'\n\n\t\t\t# Set this to make sure that it does not impact secure monitoring\n\t\t\t# (which it did, see issue: https://github.com/nats-io/nats-server/issues/2980)\n\t\t\tverify_and_map: true\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\t\"../test/configs/certs/server-noip.pem\",\n\t\t\"../test/configs/certs/server-key-noip.pem\")))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", s.MonitorAddr().Port)\n\tc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t}\n\tdefer c.Close()\n\n\ttc := &TLSConfigOpts{CaFile: \"../test/configs/certs/ca.pem\"}\n\ttlsConfig, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t}\n\ttlsConfig.ServerName = \"127.0.0.1\"\n\ttlsConfig.RootCAs = tlsConfig.ClientCAs\n\ttlsConfig.ClientCAs = nil\n\tc = tls.Client(c, tlsConfig.Clone())\n\tif err := c.(*tls.Conn).Handshake(); err == nil || !strings.Contains(err.Error(), \"SAN\") {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.Close()\n\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template,\n\t\t\"../test/configs/certs/server-cert.pem\",\n\t\t\"../test/configs/certs/server-key.pem\"))\n\n\tc, err = net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t}\n\tdefer c.Close()\n\n\tc = tls.Client(c, tlsConfig.Clone())\n\tif err := c.(*tls.Conn).Handshake(); err != nil {\n\t\tt.Fatalf(\"Error on TLS handshake: %v\", err)\n\t}\n\n\t// Need to read something to see if there is a problem with the certificate or not.\n\tvar buf [64]byte\n\tc.SetReadDeadline(time.Now().Add(250 * time.Millisecond))\n\t_, err = c.Read(buf[:])\n\tif ne, ok := err.(net.Error); !ok || !ne.Timeout() {\n\t\tt.Fatalf(\"Error: %v\", err)\n\t}\n}\n\nfunc TestMonitorMQTT(t *testing.T) {\n\to := DefaultOptions()\n\to.HTTPHost = \"127.0.0.1\"\n\to.HTTPPort = -1\n\to.ServerName = \"mqtt_server\"\n\to.Users = []*User{{Username: \"someuser\"}}\n\tpinnedCerts := make(PinnedCertSet)\n\tpinnedCerts[\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\"] = struct{}{}\n\to.MQTT = MQTTOpts{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tNoAuthUser: \"someuser\",\n\t\tJsDomain: \"js\",\n\t\tAuthTimeout: 2.0,\n\t\tTLSMap: true,\n\t\tTLSTimeout: 3.0,\n\t\tTLSPinnedCerts: pinnedCerts,\n\t\tAckWait: 4 * time.Second,\n\t\tMaxAckPending: 256,\n\t}\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\texpected := &MQTTOptsVarz{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: o.MQTT.Port,\n\t\tNoAuthUser: \"someuser\",\n\t\tJsDomain: \"js\",\n\t\tAuthTimeout: 2.0,\n\t\tTLSMap: true,\n\t\tTLSTimeout: 3.0,\n\t\tTLSPinnedCerts: []string{\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\"},\n\t\tAckWait: 4 * time.Second,\n\t\tMaxAckPending: 256,\n\t}\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s, mode, url, nil)\n\t\tvm := &v.MQTT\n\t\tif !reflect.DeepEqual(vm, expected) {\n\t\t\tt.Fatalf(\"Expected\\n%+v\\nGot:\\n%+v\", expected, vm)\n\t\t}\n\t}\n}\n\nfunc TestMonitorWebsocket(t *testing.T) {\n\to := DefaultOptions()\n\to.HTTPHost = \"127.0.0.1\"\n\to.HTTPPort = -1\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\to.TrustedKeys = []string{pub}\n\to.Users = []*User{{Username: \"someuser\"}}\n\tpinnedCerts := make(PinnedCertSet)\n\tpinnedCerts[\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\"] = struct{}{}\n\to.Websocket = WebsocketOpts{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tAdvertise: \"somehost:8080\",\n\t\tNoAuthUser: \"someuser\",\n\t\tJWTCookie: \"somecookiename\",\n\t\tAuthTimeout: 2.0,\n\t\tNoTLS: true,\n\t\tTLSMap: true,\n\t\tTLSPinnedCerts: pinnedCerts,\n\t\tSameOrigin: true,\n\t\tAllowedOrigins: []string{\"https://origin1\", \"https://origin2\"},\n\t\tCompression: true,\n\t\tHandshakeTimeout: 4 * time.Second,\n\t}\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\texpected := &WebsocketOptsVarz{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: o.Websocket.Port,\n\t\tAdvertise: \"somehost:8080\",\n\t\tNoAuthUser: \"someuser\",\n\t\tJWTCookie: \"somecookiename\",\n\t\tAuthTimeout: 2.0,\n\t\tNoTLS: true,\n\t\tTLSMap: true,\n\t\tTLSPinnedCerts: []string{\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\"},\n\t\tSameOrigin: true,\n\t\tAllowedOrigins: []string{\"https://origin1\", \"https://origin2\"},\n\t\tCompression: true,\n\t\tHandshakeTimeout: 4 * time.Second,\n\t}\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tv := pollVarz(t, s, mode, url, nil)\n\t\tvw := &v.Websocket\n\t\tif !reflect.DeepEqual(vw, expected) {\n\t\t\tt.Fatalf(\"Expected\\n%+v\\nGot:\\n%+v\", expected, vw)\n\t\t}\n\t}\n}\n\nfunc TestMonitorServerIDZRequest(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: TEST22\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`))\n\tdefer removeFile(t, conf)\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsubject := fmt.Sprintf(serverPingReqSubj, \"IDZ\")\n\tresp, err := nc.Request(subject, nil, time.Second)\n\trequire_NoError(t, err)\n\n\tvar sid ServerID\n\terr = json.Unmarshal(resp.Data, &sid)\n\trequire_NoError(t, err)\n\n\trequire_True(t, sid.Name == \"TEST22\")\n\trequire_True(t, strings.HasPrefix(sid.ID, \"N\"))\n}\n\nfunc TestMonitorProfilez(t *testing.T) {\n\ts := RunServer(DefaultOptions())\n\tdefer s.Shutdown()\n\n\t// Then start profiling.\n\ts.StartProfiler()\n\n\t// Now check that all of the profiles that we expect are\n\t// returning instead of erroring.\n\tfor _, try := range []*ProfilezOptions{\n\t\t{Name: \"allocs\", Debug: 0},\n\t\t{Name: \"allocs\", Debug: 1},\n\t\t{Name: \"block\", Debug: 0},\n\t\t{Name: \"goroutine\", Debug: 0},\n\t\t{Name: \"goroutine\", Debug: 1},\n\t\t{Name: \"goroutine\", Debug: 2},\n\t\t{Name: \"heap\", Debug: 0},\n\t\t{Name: \"heap\", Debug: 1},\n\t\t{Name: \"mutex\", Debug: 0},\n\t\t{Name: \"threadcreate\", Debug: 0},\n\t} {\n\t\tif ps := s.profilez(try); ps.Error != _EMPTY_ {\n\t\t\tt.Fatalf(\"Unexpected error on %v: %s\", try, ps.Error)\n\t\t}\n\t}\n}\n\nfunc TestMonitorRoutezPoolSize(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\thttp: -1\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: 5\n\t\t}\n\t\tno_sys_acc: true\n\t`))\n\tdefer removeFile(t, conf1)\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf23 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\thttp: -1\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\tpool_size: 5\n\t\t}\n\t\tno_sys_acc: true\n\t`, o1.Cluster.Port)))\n\tdefer removeFile(t, conf23)\n\n\ts2, _ := RunServerWithConfig(conf23)\n\tdefer s2.Shutdown()\n\ts3, _ := RunServerWithConfig(conf23)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tfor i, s := range []*Server{s1, s2, s3} {\n\t\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, s, mode, url, nil)\n\t\t\tif v.Cluster.PoolSize != 5 {\n\t\t\t\tt.Fatalf(\"Expected Cluster.PoolSize==5, got %v\", v.Cluster.PoolSize)\n\t\t\t}\n\t\t\tif v.Remotes != 2 {\n\t\t\t\tt.Fatalf(\"Expected Remotes==2, got %v\", v.Remotes)\n\t\t\t}\n\t\t\tif v.Routes != 10 {\n\t\t\t\tt.Fatalf(\"Expected NumRoutes==10, got %v\", v.Routes)\n\t\t\t}\n\t\t}\n\n\t\turl = fmt.Sprintf(\"http://127.0.0.1:%d/routez\", s.MonitorAddr().Port)\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollRoutez(t, s, mode, url, nil)\n\t\t\tif v.NumRoutes != 10 {\n\t\t\t\tt.Fatalf(\"Expected NumRoutes==10, got %v\", v.NumRoutes)\n\t\t\t}\n\t\t\tif n := len(v.Routes); n != 10 {\n\t\t\t\tt.Fatalf(\"Expected len(Routes)==10, got %v\", n)\n\t\t\t}\n\t\t\tremotes := make(map[string]int)\n\t\t\tfor _, r := range v.Routes {\n\t\t\t\tremotes[r.RemoteID]++\n\t\t\t}\n\t\t\tif n := len(remotes); n != 2 {\n\t\t\t\tt.Fatalf(\"Expected routes for 2 different servers, got %v\", n)\n\t\t\t}\n\t\t\tswitch i {\n\t\t\tcase 0:\n\t\t\t\tif n := remotes[s2.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S1 to S2, got %v\", n)\n\t\t\t\t}\n\t\t\t\tif n := remotes[s3.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S1 to S3, got %v\", n)\n\t\t\t\t}\n\t\t\tcase 1:\n\t\t\t\tif n := remotes[s1.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S2 to S1, got %v\", n)\n\t\t\t\t}\n\t\t\t\tif n := remotes[s3.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S2 to S3, got %v\", n)\n\t\t\t\t}\n\t\t\tcase 2:\n\t\t\t\tif n := remotes[s1.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S3 to S1, got %v\", n)\n\t\t\t\t}\n\t\t\t\tif n := remotes[s2.ID()]; n != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected 5 routes from S3 to S2, got %v\", n)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorRoutezPerAccount(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\thttp: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`))\n\tdefer removeFile(t, conf1)\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf23 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\thttp: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`, o1.Cluster.Port)))\n\tdefer removeFile(t, conf23)\n\n\ts2, _ := RunServerWithConfig(conf23)\n\tdefer s2.Shutdown()\n\ts3, _ := RunServerWithConfig(conf23)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tfor _, s := range []*Server{s1, s2, s3} {\n\t\t// Default pool size + account \"A\" + system account (added by default)\n\t\tenr := 2 * (DEFAULT_ROUTE_POOL_SIZE + 1 + 1)\n\t\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollVarz(t, s, mode, url, nil)\n\t\t\tif v.Remotes != 2 {\n\t\t\t\tt.Fatalf(\"Expected Remotes==2, got %v\", v.Remotes)\n\t\t\t}\n\t\t\tif v.Routes != enr {\n\t\t\t\tt.Fatalf(\"Expected NumRoutes==%d, got %v\", enr, v.Routes)\n\t\t\t}\n\t\t}\n\n\t\turl = fmt.Sprintf(\"http://127.0.0.1:%d/routez\", s.MonitorAddr().Port)\n\t\tfor mode := 0; mode < 2; mode++ {\n\t\t\tv := pollRoutez(t, s, mode, url, nil)\n\t\t\tif v.NumRoutes != enr {\n\t\t\t\tt.Fatalf(\"Expected NumRoutes==%d, got %v\", enr, v.NumRoutes)\n\t\t\t}\n\t\t\tif n := len(v.Routes); n != enr {\n\t\t\t\tt.Fatalf(\"Expected len(Routes)==%d, got %v\", enr, n)\n\t\t\t}\n\t\t\tremotes := make(map[string]int)\n\t\t\tfor _, r := range v.Routes {\n\t\t\t\tvar acc int\n\t\t\t\tif r.Account == \"A\" {\n\t\t\t\t\tacc = 1\n\t\t\t\t}\n\t\t\t\tremotes[r.RemoteID] += acc\n\t\t\t}\n\t\t\tif n := len(remotes); n != 2 {\n\t\t\t\tt.Fatalf(\"Expected routes for 2 different servers, got %v\", n)\n\t\t\t}\n\t\t\tfor remoteID, v := range remotes {\n\t\t\t\tif v != 1 {\n\t\t\t\t\tt.Fatalf(\"Expected one and only one connection for account A for remote %q, got %v\", remoteID, v)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzOperatorAccountNames(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, _, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\taccPub, _ := accKp.PublicKey()\n\n\t// Now create 2 users.\n\t_, creds := createUser(t, accKp)\n\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(creds))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tfor pollMode := 0; pollMode < 2; pollMode++ {\n\t\turl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?auth=1\", s.MonitorAddr().Port)\n\t\tconnz := pollConnz(t, s, pollMode, url, &ConnzOptions{Username: true})\n\t\trequire_Equal(t, connz.NumConns, 2)\n\t\tidx := slices.IndexFunc(connz.Conns, func(c *ConnInfo) bool {\n\t\t\treturn c.Kind == kindStringMap[CLIENT]\n\t\t})\n\t\tci := connz.Conns[idx]\n\t\trequire_Equal(t, ci.Account, accPub)\n\t\trequire_Equal(t, ci.NameTag, accName)\n\t}\n}\n\nfunc TestMonitorConnzOperatorModeFilterByUser(t *testing.T) {\n\tsysName := \"SYS\"\n\taccName := \"APP\"\n\n\tsrvs, _, accKp := runMonitorServerWithOperator(t, sysName, accName)\n\tfor _, s := range srvs {\n\t\tdefer s.Shutdown()\n\t}\n\ts := srvs[0]\n\n\t// Now create 2 users.\n\taUser, aCreds := createUser(t, accKp)\n\tbUser, bCreds := createUser(t, accKp)\n\n\tvar users []*nats.Conn\n\n\t// Create 2 for A\n\tfor i := 0; i < 2; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(aCreds))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t\tusers = append(users, nc)\n\t}\n\t// Create 5 for B\n\tfor i := 0; i < 5; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(bCreds))\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t\tusers = append(users, nc)\n\t}\n\n\t// Test A\n\tconnz := pollConnz(t, s, 1, _EMPTY_, &ConnzOptions{User: aUser, Username: true})\n\trequire_True(t, connz.NumConns == 2)\n\tfor _, ci := range connz.Conns {\n\t\trequire_True(t, ci.AuthorizedUser == aUser)\n\t}\n\t// Test B\n\tconnz = pollConnz(t, s, 1, _EMPTY_, &ConnzOptions{User: bUser, Username: true})\n\trequire_True(t, connz.NumConns == 5)\n\tfor _, ci := range connz.Conns {\n\t\trequire_True(t, ci.AuthorizedUser == bUser)\n\t}\n\n\t// Make sure URL access is the same.\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\turlFull := url + fmt.Sprintf(\"connz?auth=true&user=%s\", aUser)\n\tconnz = pollConnz(t, s, 0, urlFull, nil)\n\trequire_True(t, connz.NumConns == 2)\n\tfor _, ci := range connz.Conns {\n\t\trequire_True(t, ci.AuthorizedUser == aUser)\n\t}\n\n\t// Now test closed filtering as well.\n\tfor _, nc := range users {\n\t\tnc.Close()\n\t}\n\t// Let them process and be moved to closed ring buffer in server.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tconnz = pollConnz(t, s, 1, _EMPTY_, &ConnzOptions{User: aUser, Username: true, State: ConnClosed})\n\trequire_True(t, connz.NumConns == 2)\n\tfor _, ci := range connz.Conns {\n\t\trequire_True(t, ci.AuthorizedUser == aUser)\n\t}\n}\n\nfunc TestMonitorConnzSortByRTT(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc, err := nats.Connect(s.ClientURL())\n\t\trequire_NoError(t, err)\n\t\tdefer nc.Close()\n\t}\n\n\tconnz := pollConnz(t, s, 1, _EMPTY_, &ConnzOptions{Sort: ByRTT})\n\trequire_True(t, connz.NumConns == 10)\n\n\tvar rtt int64\n\tfor _, ci := range connz.Conns {\n\t\tif rtt == 0 {\n\t\t\trtt = ci.rtt\n\t\t} else {\n\t\t\tif ci.rtt > rtt {\n\t\t\t\tt.Fatalf(\"RTT not in descending order: %v vs %v\",\n\t\t\t\t\ttime.Duration(rtt), time.Duration(ci.rtt))\n\t\t\t}\n\t\t\trtt = ci.rtt\n\t\t}\n\t}\n\n\t// Make sure url works as well.\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/connz?sort=rtt\", s.MonitorAddr().Port)\n\tconnz = pollConnz(t, s, 0, url, nil)\n\trequire_True(t, connz.NumConns == 10)\n\n\trtt = 0\n\tfor _, ci := range connz.Conns {\n\t\tcrttd, err := time.ParseDuration(ci.RTT)\n\t\trequire_NoError(t, err)\n\t\tcrtt := int64(crttd)\n\t\tif rtt == 0 {\n\t\t\trtt = crtt\n\t\t} else {\n\t\t\tif crtt > rtt {\n\t\t\t\tt.Fatalf(\"RTT not in descending order: %v vs %v\",\n\t\t\t\t\ttime.Duration(rtt), time.Duration(crtt))\n\t\t\t}\n\t\t\trtt = ci.rtt\n\t\t}\n\t}\n}\n\nfunc TestMonitorConnzIncludesLeafnodes(t *testing.T) {\n\tcontent := `\n\t\tserver_name: \"hub\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\toperator = \"../test/configs/nkeys/op.jwt\"\n\t\tresolver = MEMORY\n\t\tping_interval = 1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\tsb, ob := RunServerWithConfig(conf)\n\tdefer sb.Shutdown()\n\n\tcreateAcc := func(t *testing.T) (*Account, string) {\n\t\tt.Helper()\n\t\tacc, akp := createAccount(sb)\n\t\tkp, _ := nkeys.CreateUser()\n\t\tpub, _ := kp.PublicKey()\n\t\tnuc := jwt.NewUserClaims(pub)\n\t\tujwt, err := nuc.Encode(akp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t\t}\n\t\tseed, _ := kp.Seed()\n\t\tcreds := genCredsFile(t, ujwt, seed)\n\t\treturn acc, creds\n\t}\n\tacc, mycreds := createAcc(t)\n\tleafName := \"my-leaf-node\"\n\n\tcontent = `\n\t\tport: -1\n\t\thttp: \"127.0.0.1:-1\"\n\t\tping_interval = 1\n\t\tserver_name: %s\n\t\taccounts {\n\t\t\t%s {\n\t\t\t\tusers [\n\t\t\t\t\t{user: user1, password: pwd}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\taccount: \"%s\"\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content,\n\t\tleafName,\n\t\tacc.Name,\n\t\tacc.Name, ob.LeafNode.Port, mycreds)\n\tconf = createConfFile(t, []byte(config))\n\tsa, _ := RunServerWithConfig(conf)\n\tdefer sa.Shutdown()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif n := sa.NumLeafNodes(); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 leaf connection, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor test, options := range map[string]*ConnzOptions{\n\t\t\"WithoutAccount\": {},\n\t\t\"WithAccount\": {Account: acc.Name},\n\t} {\n\t\tt.Run(test, func(t *testing.T) {\n\t\t\tc := pollConnz(t, sb, 1, \"http://127.0.0.1:%d/connz\", options)\n\t\t\trequire_Equal(t, c.NumConns, 1)\n\t\t\trequire_Equal(t, c.Conns[0].Kind, kindStringMap[LEAF])\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/4144\nfunc TestMonitorAccountszMappingOrderReporting(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: 127.0.0.1:-1\n\tserver_name: SR22\n\taccounts {\n\t\tCLOUD {\n\t\t\texports [ { service: \"downlink.>\" } ]\n\t\t}\n\t\tCLOUD2 {\n\t\t\texports [ { service: \"downlink.>\" } ]\n\t\t}\n\t\tAPP {\n\t\t\timports [\n\t\t\t\t{ service: { account: CLOUD, subject: \"downlink.>\"}, to: \"event.>\"}\n\t\t\t\t{ service: { account: CLOUD2, subject: \"downlink.>\"}, to: \"event.>\"}\n\t\t\t]\n\t\t}\n\t}`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\taz, err := s.Accountz(&AccountzOptions{\"APP\"})\n\trequire_NoError(t, err)\n\trequire_NotNil(t, az.Account)\n\trequire_True(t, len(az.Account.Imports) > 0)\n\n\tvar found bool\n\tm := map[string]struct{}{}\n\tfor _, si := range az.Account.Imports {\n\t\tif si.Import.Subject == \"downlink.>\" {\n\t\t\tfound = true\n\t\t\trequire_True(t, si.Import.LocalSubject == \"event.>\")\n\t\t\tm[si.Import.Account] = struct{}{}\n\t\t}\n\t}\n\trequire_True(t, found)\n\tif len(m) != 2 {\n\t\tt.Fatalf(\"Expected imports from CLOUD and CLOUD2, got %v\", m)\n\t}\n}\n\n// createCallbackURL adds a callback query parameter for JSONP requests.\nfunc createCallbackURL(t *testing.T, endpoint string) string {\n\tt.Helper()\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tparams := u.Query()\n\tparams.Set(\"callback\", \"callback\")\n\n\tu.RawQuery = params.Encode()\n\n\treturn u.String()\n}\n\n// stripCallback removes the JSONP callback function from the response.\n// Returns the JSON body without the wrapping callback function.\n// If there's no callback function, the data is returned as is.\nfunc stripCallback(data []byte) []byte {\n\t// Cut the JSONP callback function with the opening parentheses.\n\t_, after, found := bytes.Cut(data, []byte(\"(\"))\n\n\tif found {\n\t\treturn bytes.TrimSuffix(after, []byte(\")\"))\n\t}\n\n\treturn data\n}\n\n// expectHealthStatus makes 1 regular and 1 JSONP request to the URL and checks the\n// HTTP status code, Content-Type header and health status string.\nfunc expectHealthStatus(t *testing.T, url string, statusCode int, wantStatus string) {\n\tt.Helper()\n\n\t// First check for regular requests.\n\tbody := readBodyEx(t, url, statusCode, appJSONContent)\n\tcheckHealthStatus(t, body, wantStatus)\n\n\t// Another check for JSONP requests.\n\tjsonpURL := createCallbackURL(t, url) // Adds a callback query param.\n\tjsonpBody := readBodyEx(t, jsonpURL, statusCode, appJSContent)\n\tcheckHealthStatus(t, stripCallback(jsonpBody), wantStatus)\n}\n\n// checkHealthStatus checks the health status from a JSON response.\nfunc checkHealthStatus(t *testing.T, body []byte, wantStatus string) {\n\tt.Helper()\n\n\th := &HealthStatus{}\n\n\tif err := json.Unmarshal(body, h); err != nil {\n\t\tt.Fatalf(\"error unmarshalling the body: %v\", err)\n\t}\n\n\tif h.Status != wantStatus {\n\t\tt.Errorf(\"want health status %q, got %q\", wantStatus, h.Status)\n\t}\n}\n\n// checkHealthzEndpoint makes requests to the /healthz endpoint and checks the health status.\nfunc checkHealthzEndpoint(t *testing.T, address string, statusCode int, wantStatus string) {\n\tt.Helper()\n\n\tcases := map[string]string{\n\t\t\"healthz\": fmt.Sprintf(\"http://%s/healthz\", address),\n\t\t\"js-enabled-only\": fmt.Sprintf(\"http://%s/healthz?js-enabled-only=true\", address),\n\t\t\"js-server-only\": fmt.Sprintf(\"http://%s/healthz?js-server-only=true\", address),\n\t}\n\n\tfor name, url := range cases {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\texpectHealthStatus(t, url, statusCode, wantStatus)\n\t\t})\n\t}\n}\n\nfunc TestMonitorHealthzStatusOK(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tcheckHealthzEndpoint(t, s.MonitorAddr().String(), http.StatusOK, \"ok\")\n}\n\nfunc TestMonitorHealthzStatusError(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\t// Intentionally causing an error in readyForConnections().\n\t// Note: Private field access, taking advantage of having the tests in the same package.\n\ts.mu.Lock()\n\tsl := s.listener\n\ts.listener = nil\n\ts.mu.Unlock()\n\n\tcheckHealthzEndpoint(t, s.MonitorAddr().String(), http.StatusInternalServerError, \"error\")\n\n\t// Restore for proper shutdown.\n\ts.mu.Lock()\n\ts.listener = sl\n\ts.mu.Unlock()\n}\n\nfunc TestMonitorHealthzStatusUnavailable(t *testing.T) {\n\topts := DefaultMonitorOptions()\n\topts.JetStream = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tif !s.JetStreamEnabled() {\n\t\tt.Fatalf(\"want JetStream to be enabled first\")\n\t}\n\n\terr := s.DisableJetStream()\n\n\tif err != nil {\n\t\tt.Fatalf(\"got an error disabling JetStream: %v\", err)\n\t}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\turl string\n\t\tstatusCode int\n\t\twantStatus string\n\t}{\n\t\t{\n\t\t\t\"healthz\",\n\t\t\tfmt.Sprintf(\"http://%s/healthz?\", s.MonitorAddr().String()),\n\t\t\thttp.StatusServiceUnavailable,\n\t\t\t\"unavailable\",\n\t\t},\n\t\t{\n\t\t\t\"js-enabled-only\",\n\t\t\tfmt.Sprintf(\"http://%s/healthz?js-enabled-only=true\", s.MonitorAddr().String()),\n\t\t\thttp.StatusServiceUnavailable,\n\t\t\t\"unavailable\",\n\t\t},\n\t\t{\n\t\t\t\"js-server-only\",\n\t\t\tfmt.Sprintf(\"http://%s/healthz?js-server-only=true\", s.MonitorAddr().String()),\n\t\t\thttp.StatusOK,\n\t\t\t\"ok\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Run(\"no-details\", func(t *testing.T) {\n\t\t\t\texpectHealthStatus(t, test.url, test.statusCode, test.wantStatus)\n\t\t\t})\n\t\t\tt.Run(\"with-details\", func(t *testing.T) {\n\t\t\t\tdetailsUrl := fmt.Sprintf(\"%s&details=true\", test.url)\n\t\t\t\tif test.wantStatus != \"ok\" {\n\t\t\t\t\ttest.wantStatus = \"error\"\n\t\t\t\t}\n\t\t\t\texpectHealthStatus(t, detailsUrl, test.statusCode, test.wantStatus)\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestServerHealthz(t *testing.T) {\n\tt.Run(\"BasicHealth\", func(t *testing.T) {\n\t\ts := runMonitorServer()\n\t\tdefer s.Shutdown()\n\n\t\t// Test with nil options\n\t\tstatus := s.Healthz(nil)\n\t\tif status == nil {\n\t\t\tt.Fatal(\"Expected non-nil HealthStatus\")\n\t\t}\n\t\tif status.Status != \"ok\" {\n\t\t\tt.Fatalf(\"Expected status 'ok', got %q\", status.Status)\n\t\t}\n\t\tif status.StatusCode != http.StatusOK {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusOK, status.StatusCode)\n\t\t}\n\n\t\t// Test with empty options\n\t\tstatus = s.Healthz(&HealthzOptions{})\n\t\tif status == nil {\n\t\t\tt.Fatal(\"Expected non-nil HealthStatus\")\n\t\t}\n\t\tif status.Status != \"ok\" {\n\t\t\tt.Fatalf(\"Expected status 'ok', got %q\", status.Status)\n\t\t}\n\n\t\t// Test with JSServerOnly option\n\t\tstatus = s.Healthz(&HealthzOptions{JSServerOnly: true})\n\t\tif status == nil {\n\t\t\tt.Fatal(\"Expected non-nil HealthStatus\")\n\t\t}\n\t\tif status.Status != \"ok\" {\n\t\t\tt.Fatalf(\"Expected status 'ok', got %q\", status.Status)\n\t\t}\n\t})\n\n\tt.Run(\"BadRequestMissingAccount\", func(t *testing.T) {\n\t\ts := runMonitorServer()\n\t\tdefer s.Shutdown()\n\n\t\t// Stream without account should return bad request\n\t\tstatus := s.Healthz(&HealthzOptions{Stream: \"TEST\"})\n\t\tif status.StatusCode != http.StatusBadRequest {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusBadRequest, status.StatusCode)\n\t\t}\n\t\tif status.Status != \"error\" {\n\t\t\tt.Fatalf(\"Expected status 'error', got %q\", status.Status)\n\t\t}\n\t})\n\n\tt.Run(\"BadRequestMissingStream\", func(t *testing.T) {\n\t\ts := runMonitorServer()\n\t\tdefer s.Shutdown()\n\n\t\t// Consumer without stream should return bad request\n\t\tstatus := s.Healthz(&HealthzOptions{Account: \"ACC\", Consumer: \"TEST\"})\n\t\tif status.StatusCode != http.StatusBadRequest {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusBadRequest, status.StatusCode)\n\t\t}\n\t\tif status.Status != \"error\" {\n\t\t\tt.Fatalf(\"Expected status 'error', got %q\", status.Status)\n\t\t}\n\t})\n\n\tt.Run(\"DetailsOption\", func(t *testing.T) {\n\t\ts := runMonitorServer()\n\t\tdefer s.Shutdown()\n\n\t\t// Test with Details option - should still return ok for healthy server\n\t\tstatus := s.Healthz(&HealthzOptions{Details: true})\n\t\tif status == nil {\n\t\t\tt.Fatal(\"Expected non-nil HealthStatus\")\n\t\t}\n\t\tif status.Status != \"ok\" {\n\t\t\tt.Fatalf(\"Expected status 'ok', got %q\", status.Status)\n\t\t}\n\n\t\t// Test bad request with details - should populate Errors slice\n\t\tstatus = s.Healthz(&HealthzOptions{Stream: \"TEST\", Details: true})\n\t\tif status.StatusCode != http.StatusBadRequest {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusBadRequest, status.StatusCode)\n\t\t}\n\t\tif len(status.Errors) == 0 {\n\t\t\tt.Fatal(\"Expected Errors slice to be populated with Details=true\")\n\t\t}\n\t\tif status.Errors[0].Type != HealthzErrorBadRequest {\n\t\t\tt.Fatalf(\"Expected error type %v, got %v\", HealthzErrorBadRequest, status.Errors[0].Type)\n\t\t}\n\t})\n\n\tt.Run(\"ServerNotReady\", func(t *testing.T) {\n\t\ts := runMonitorServer()\n\t\tdefer s.Shutdown()\n\n\t\t// Simulate server not ready by removing listener\n\t\ts.mu.Lock()\n\t\tsl := s.listener\n\t\ts.listener = nil\n\t\ts.mu.Unlock()\n\n\t\tstatus := s.Healthz(nil)\n\t\tif status.StatusCode != http.StatusInternalServerError {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusInternalServerError, status.StatusCode)\n\t\t}\n\t\tif status.Status != \"error\" {\n\t\t\tt.Fatalf(\"Expected status 'error', got %q\", status.Status)\n\t\t}\n\n\t\t// Restore for proper shutdown\n\t\ts.mu.Lock()\n\t\ts.listener = sl\n\t\ts.mu.Unlock()\n\t})\n\n\tt.Run(\"JetStreamUnavailable\", func(t *testing.T) {\n\t\topts := DefaultMonitorOptions()\n\t\topts.JetStream = true\n\t\ts := RunServer(opts)\n\t\tdefer s.Shutdown()\n\n\t\tif !s.JetStreamEnabled() {\n\t\t\tt.Fatalf(\"want JetStream to be enabled first\")\n\t\t}\n\n\t\terr := s.DisableJetStream()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"got an error disabling JetStream: %v\", err)\n\t\t}\n\n\t\t// Should report unavailable when JS is disabled\n\t\tstatus := s.Healthz(nil)\n\t\tif status.StatusCode != http.StatusServiceUnavailable {\n\t\t\tt.Fatalf(\"Expected status code %d, got %d\", http.StatusServiceUnavailable, status.StatusCode)\n\t\t}\n\t\tif status.Status != \"unavailable\" {\n\t\t\tt.Fatalf(\"Expected status 'unavailable', got %q\", status.Status)\n\t\t}\n\n\t\t// JSServerOnly should still report ok\n\t\tstatus = s.Healthz(&HealthzOptions{JSServerOnly: true})\n\t\tif status.Status != \"ok\" {\n\t\t\tt.Fatalf(\"Expected status 'ok' with JSServerOnly, got %q\", status.Status)\n\t\t}\n\t})\n}\n\n// When we converted ipq to use generics we still were using sync.Map. Currently you can not convert\n// any or any to a generic parameterized type. So this stopped working and panics.\n// Copyright 2013-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n// Make sure that we do not run the http server for monitoring unless asked.\n// https://github.com/nats-io/nats-server/issues/2170\n// Just the ever increasing subs part.\n// Helper to map to connection name\n// getConnsIdleDurations returns a slice of parsed idle durations from a connection info slice.\n// sortedDurationsDesc checks if a time.Duration slice is sorted in descending order.\n// getIdleDurations returns a slice of idle durations from a connection info list up until now time.\n// sortedDurationsAsc checks if a time.Duration slice is sorted in ascending order.\n// Tests handle root\n// Make sure options for ConnInfo like subs=1, authuser, etc do not cause a race.\n// Make sure a bad client that is disconnected right away has proper values.\n// Make sure a bad client that tries to connect plain to TLS has proper values.\n// Create a connection to test ConnInfo\n// Benchmark our Connz generation. Don't use HTTP here, just measure server endpoint.\n// Helper function to check that a JS cluster is formed\n// https://github.com/nats-io/nats-server/issues/4144\n// createCallbackURL adds a callback query parameter for JSONP requests.\n// stripCallback removes the JSONP callback function from the response.\n// Returns the JSON body without the wrapping callback function.\n// If there's no callback function, the data is returned as is.\n// expectHealthStatus makes 1 regular and 1 JSONP request to the URL and checks the\n// HTTP status code, Content-Type header and health status string.\n// checkHealthStatus checks the health status from a JSON response.\n// checkHealthzEndpoint makes requests to the /healthz endpoint and checks the health status.\n// When we converted ipq to use generics we still were using sync.Map. Currently you can not convert\n// any or any to a generic parameterized type. So this stopped working and panics.\nfunc TestMonitorIpqzWithGenerics(t *testing.T) {\n\topts := DefaultMonitorOptions()\n\topts.JetStream = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://%s/ipqueuesz?all=1\", s.MonitorAddr().String())\n\tbody := readBody(t, url)\n\trequire_True(t, len(body) > 0)\n\n\tqueues := IpqueueszStatus{}\n\trequire_NoError(t, json.Unmarshal(body, &queues))\n\trequire_True(t, len(queues) >= 4)\n\t_, ok := queues[\"SendQ\"]\n\trequire_True(t, ok)\n}\n\nfunc TestMonitorVarzSyncInterval(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.JetStream = true\n\topts.SyncInterval = 22 * time.Second\n\topts.SyncAlways = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\n\tjscfg := pollVarz(t, s, 0, url, nil).JetStream.Config\n\trequire_True(t, jscfg.SyncInterval == opts.SyncInterval)\n\trequire_True(t, jscfg.SyncAlways)\n}\n\nfunc TestMonitorVarzJSApiLevel(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\topts.JetStream = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\n\tvarz := pollVarz(t, s, 0, url, nil)\n\tapiLevel := varz.JetStream.Stats.API.Level\n\trequire_Equal(t, apiLevel, JSApiLevel)\n}\n\nfunc TestMonitorVarzMetadata(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tv, err := s.Varz(nil)\n\trequire_NoError(t, err)\n\n\texpected := map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"}\n\tif !reflect.DeepEqual(expected, v.Metadata) {\n\t\tt.Fatalf(\"expected: %v, got: %v\", expected, v.Metadata)\n\t}\n}\n\nfunc TestMonitorVarzTLSCertEndDate(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultMonitorOptions()\n\ttlsConfig, err := GenTLSConfig(\n\t\t&TLSConfigOpts{\n\t\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t}\n\n\topts.TLSConfig = tlsConfig\n\topts.Cluster.TLSConfig = tlsConfig\n\topts.Gateway.TLSConfig = tlsConfig\n\topts.LeafNode.TLSConfig = tlsConfig\n\topts.MQTT.TLSConfig = tlsConfig\n\topts.Websocket.TLSConfig = tlsConfig\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/varz\", s.MonitorAddr().Port)\n\tv := pollVarz(t, s, 0, url, nil)\n\n\texpected := time.Date(2032, 8, 24, 20, 23, 02, 0, time.UTC)\n\n\tcheck := func(t *testing.T, notAfter time.Time) {\n\t\tt.Helper()\n\t\tif notAfter != expected {\n\t\t\tt.Fatalf(\"Expected expiration date '%v', got '%v'\", expected, notAfter)\n\t\t}\n\t}\n\n\tcheck(t, v.TLSCertNotAfter)\n\tcheck(t, v.Cluster.TLSCertNotAfter)\n\tcheck(t, v.Gateway.TLSCertNotAfter)\n\tcheck(t, v.LeafNode.TLSCertNotAfter)\n\tcheck(t, v.MQTT.TLSCertNotAfter)\n\tcheck(t, v.Websocket.TLSCertNotAfter)\n}\n\nfunc TestMetaClusterInfoSnapshotStats(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Create a stream to generate some meta activity.\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tleader := c.leader()\n\trequire_True(t, leader != nil)\n\n\t// Check Jsz() includes Snapshot.\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tjsi, err := leader.Jsz(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif jsi.Meta == nil {\n\t\t\treturn errors.New(\"expected meta cluster info from Jsz\")\n\t\t}\n\t\tif jsi.Meta.Snapshot == nil {\n\t\t\treturn errors.New(\"expected snapshot stats in Jsz meta cluster info\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check Varz() includes Snapshot.\n\tcheckFor(t, 5*time.Second, 250*time.Millisecond, func() error {\n\t\tv, err := leader.Varz(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif v.JetStream.Meta == nil {\n\t\t\treturn errors.New(\"expected meta cluster info from Varz\")\n\t\t}\n\t\tif v.JetStream.Meta.Snapshot == nil {\n\t\t\treturn errors.New(\"expected snapshot stats in Varz meta cluster info\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check STATSZ event includes Snapshot.\n\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer snc.Close()\n\n\tch := make(chan *nats.Msg, 1)\n\t_, err = snc.ChanSubscribe(fmt.Sprintf(serverStatsSubj, leader.ID()), ch)\n\trequire_NoError(t, err)\n\n\tmsg := require_ChanRead(t, ch, 5*time.Second)\n\tvar m ServerStatsMsg\n\trequire_NoError(t, json.Unmarshal(msg.Data, &m))\n\trequire_True(t, m.Stats.JetStream != nil)\n\trequire_True(t, m.Stats.JetStream.Meta != nil)\n\trequire_True(t, m.Stats.JetStream.Meta.Snapshot != nil)\n}\n" }, { "path": "server/mqtt.go", "content": "// Copyright 2020-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"cmp\"\n\t\"crypto/tls\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// References to \"spec\" here is from https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.pdf\n\nconst (\n\tmqttPacketConnect = byte(0x10)\n\tmqttPacketConnectAck = byte(0x20)\n\tmqttPacketPub = byte(0x30)\n\tmqttPacketPubAck = byte(0x40)\n\tmqttPacketPubRec = byte(0x50)\n\tmqttPacketPubRel = byte(0x60)\n\tmqttPacketPubComp = byte(0x70)\n\tmqttPacketSub = byte(0x80)\n\tmqttPacketSubAck = byte(0x90)\n\tmqttPacketUnsub = byte(0xa0)\n\tmqttPacketUnsubAck = byte(0xb0)\n\tmqttPacketPing = byte(0xc0)\n\tmqttPacketPingResp = byte(0xd0)\n\tmqttPacketDisconnect = byte(0xe0)\n\tmqttPacketMask = byte(0xf0)\n\tmqttPacketFlagMask = byte(0x0f)\n\n\tmqttProtoLevel = byte(0x4)\n\n\t// Connect flags\n\tmqttConnFlagReserved = byte(0x1)\n\tmqttConnFlagCleanSession = byte(0x2)\n\tmqttConnFlagWillFlag = byte(0x04)\n\tmqttConnFlagWillQoS = byte(0x18)\n\tmqttConnFlagWillRetain = byte(0x20)\n\tmqttConnFlagPasswordFlag = byte(0x40)\n\tmqttConnFlagUsernameFlag = byte(0x80)\n\n\t// Publish flags\n\tmqttPubFlagRetain = byte(0x01)\n\tmqttPubFlagQoS = byte(0x06)\n\tmqttPubFlagDup = byte(0x08)\n\tmqttPubQos1 = byte(0x1 << 1)\n\tmqttPubQoS2 = byte(0x2 << 1)\n\n\t// Subscribe flags\n\tmqttSubscribeFlags = byte(0x2)\n\tmqttSubAckFailure = byte(0x80)\n\n\t// Unsubscribe flags\n\tmqttUnsubscribeFlags = byte(0x2)\n\n\t// ConnAck returned codes\n\tmqttConnAckRCConnectionAccepted = byte(0x0)\n\tmqttConnAckRCUnacceptableProtocolVersion = byte(0x1)\n\tmqttConnAckRCIdentifierRejected = byte(0x2)\n\tmqttConnAckRCServerUnavailable = byte(0x3)\n\tmqttConnAckRCBadUserOrPassword = byte(0x4)\n\tmqttConnAckRCNotAuthorized = byte(0x5)\n\tmqttConnAckRCQoS2WillRejected = byte(0x10)\n\n\t// Maximum payload size of a control packet\n\tmqttMaxPayloadSize = 0xFFFFFFF\n\n\t// Topic/Filter characters\n\tmqttTopicLevelSep = '/'\n\tmqttSingleLevelWC = '+'\n\tmqttMultiLevelWC = '#'\n\tmqttReservedPre = '$'\n\n\t// This is appended to the sid of a subscription that is\n\t// created on the upper level subject because of the MQTT\n\t// wildcard '#' semantic.\n\tmqttMultiLevelSidSuffix = \" fwc\"\n\n\t// This is the prefix used for all subjects used by MQTT code.\n\tmqttPrefix = \"$MQTT.\"\n\n\t// This is the prefix for NATS subscriptions subjects associated as delivery\n\t// subject of JS consumer. We want to make them unique so will prevent users\n\t// MQTT subscriptions to start with this.\n\tmqttSubPrefix = mqttPrefix + \"sub.\"\n\n\t// Stream name for MQTT messages on a given account\n\tmqttStreamName = \"$MQTT_msgs\"\n\tmqttStreamSubjectPrefix = mqttPrefix + \"msgs.\"\n\n\t// Stream name for MQTT retained messages on a given account\n\tmqttRetainedMsgsStreamName = \"$MQTT_rmsgs\"\n\tmqttRetainedMsgsStreamSubject = mqttPrefix + \"rmsgs.\"\n\n\t// Stream name for MQTT sessions on a given account\n\tmqttSessStreamName = \"$MQTT_sess\"\n\tmqttSessStreamSubjectPrefix = mqttPrefix + \"sess.\"\n\n\t// Stream name prefix for MQTT sessions on a given account\n\tmqttSessionsStreamNamePrefix = \"$MQTT_sess_\"\n\n\t// Stream name and subject for incoming MQTT QoS2 messages\n\tmqttQoS2IncomingMsgsStreamName = \"$MQTT_qos2in\"\n\tmqttQoS2IncomingMsgsStreamSubjectPrefix = mqttPrefix + \"qos2.in.\"\n\n\t// Stream name and subjects for outgoing MQTT QoS (PUBREL) messages\n\tmqttOutStreamName = \"$MQTT_out\"\n\tmqttOutSubjectPrefix = mqttPrefix + \"out.\"\n\tmqttPubRelSubjectPrefix = mqttPrefix + \"out.pubrel.\"\n\tmqttPubRelDeliverySubjectPrefix = mqttPrefix + \"deliver.pubrel.\"\n\tmqttPubRelConsumerDurablePrefix = \"$MQTT_PUBREL_\"\n\n\t// As per spec, MQTT server may not redeliver QoS 1 and 2 messages to\n\t// clients, except after client reconnects. However, NATS Server will\n\t// redeliver unacknowledged messages after this default interval. This can\n\t// be changed with the server.Options.MQTT.AckWait option.\n\tmqttDefaultAckWait = 30 * time.Second\n\n\t// This is the default for the outstanding number of pending QoS 1\n\t// messages sent to a session with QoS 1 subscriptions.\n\tmqttDefaultMaxAckPending = 1024\n\n\t// A session's list of subscriptions cannot have a cumulative MaxAckPending\n\t// of more than this limit.\n\tmqttMaxAckTotalLimit = 0xFFFF\n\n\t// Prefix of the reply subject for JS API requests.\n\tmqttJSARepliesPrefix = mqttPrefix + \"JSA.\"\n\n\t// Those are tokens that are used for the reply subject of JS API requests.\n\t// For instance \"$MQTT.JSA..SC.\" is the reply subject\n\t// for a request to create a stream (where is the server name hash),\n\t// while \"$MQTT.JSA..SL.\" is for a stream lookup, etc...\n\tmqttJSAIdTokenPos = 3\n\tmqttJSATokenPos = 4\n\tmqttJSAClientIDPos = 5\n\tmqttJSAStreamCreate = \"SC\"\n\tmqttJSAStreamUpdate = \"SU\"\n\tmqttJSAStreamLookup = \"SL\"\n\tmqttJSAStreamDel = \"SD\"\n\tmqttJSAConsumerCreate = \"CC\"\n\tmqttJSAConsumerLookup = \"CL\"\n\tmqttJSAConsumerDel = \"CD\"\n\tmqttJSAMsgStore = \"MS\"\n\tmqttJSAMsgLoad = \"ML\"\n\tmqttJSAMsgDelete = \"MD\"\n\tmqttJSASessPersist = \"SP\"\n\tmqttJSARetainedMsgDel = \"RD\"\n\tmqttJSAStreamNames = \"SN\"\n\n\t// This is how long to keep a client in the flappers map before closing the\n\t// connection. This prevent quick reconnect from those clients that keep\n\t// wanting to connect with a client ID already in use.\n\tmqttSessFlappingJailDur = time.Second\n\n\t// This is how frequently the timer to cleanup the sessions flappers map is firing.\n\tmqttSessFlappingCleanupInterval = 5 * time.Second\n\n\t// Default retry delay if transfer of old session streams to new one fails\n\tmqttDefaultTransferRetry = 5 * time.Second\n\n\t// For Websocket URLs\n\tmqttWSPath = \"/mqtt\"\n\n\tmqttInitialPubHeader = 16 // An overkill, should need 7 bytes max\n\tmqttProcessSubTooLong = 100 * time.Millisecond\n\tmqttDefaultRetainedCacheTTL = 2 * time.Minute\n\tmqttRetainedTransferTimeout = 10 * time.Second\n\tmqttDefaultJSAPITimeout = 5 * time.Second\n\tmqttRetainedFlagDelMarker = '-'\n)\n\nconst (\n\tsparkbNBIRTH = \"NBIRTH\"\n\tsparkbDBIRTH = \"DBIRTH\"\n\tsparkbNDEATH = \"NDEATH\"\n\tsparkbDDEATH = \"DDEATH\"\n)\n\nvar (\n\tsparkbNamespaceTopicPrefix = []byte(\"spBv1.0/\")\n\tsparkbCertificatesTopicPrefix = []byte(\"$sparkplug/certificates/\")\n)\n\nvar (\n\tmqttPingResponse = []byte{mqttPacketPingResp, 0x0}\n\tmqttProtoName = []byte(\"MQTT\")\n\tmqttOldProtoName = []byte(\"MQIsdp\")\n\tmqttSessJailDur = mqttSessFlappingJailDur\n\tmqttFlapCleanItvl = mqttSessFlappingCleanupInterval\n\tmqttRetainedCacheTTL = mqttDefaultRetainedCacheTTL\n)\n\nvar (\n\terrMQTTNotWebsocketPort = errors.New(\"MQTT clients over websocket must connect to the Websocket port, not the MQTT port\")\n\terrMQTTTopicFilterCannotBeEmpty = errors.New(\"topic filter cannot be empty\")\n\terrMQTTMalformedVarInt = errors.New(\"malformed variable int\")\n\terrMQTTSecondConnectPacket = errors.New(\"received a second CONNECT packet\")\n\terrMQTTServerNameMustBeSet = errors.New(\"mqtt requires server name to be explicitly set\")\n\terrMQTTUserMixWithUsersNKeys = errors.New(\"mqtt authentication username not compatible with presence of users/nkeys\")\n\terrMQTTTokenMixWIthUsersNKeys = errors.New(\"mqtt authentication token not compatible with presence of users/nkeys\")\n\terrMQTTAckWaitMustBePositive = errors.New(\"ack wait must be a positive value\")\n\terrMQTTJSAPITimeoutMustBePositive = errors.New(\"JS API timeout must be a positive value\")\n\terrMQTTStandaloneNeedsJetStream = errors.New(\"mqtt requires JetStream to be enabled if running in standalone mode\")\n\terrMQTTConnFlagReserved = errors.New(\"connect flags reserved bit not set to 0\")\n\terrMQTTWillAndRetainFlag = errors.New(\"if Will flag is set to 0, Will Retain flag must be 0 too\")\n\terrMQTTPasswordFlagAndNoUser = errors.New(\"password flag set but username flag is not\")\n\terrMQTTCIDEmptyNeedsCleanFlag = errors.New(\"when client ID is empty, clean session flag must be set to 1\")\n\terrMQTTEmptyWillTopic = errors.New(\"empty Will topic not allowed\")\n\terrMQTTEmptyUsername = errors.New(\"empty user name not allowed\")\n\terrMQTTTopicIsEmpty = errors.New(\"topic cannot be empty\")\n\terrMQTTPacketIdentifierIsZero = errors.New(\"packet identifier cannot be 0\")\n\terrMQTTUnsupportedCharacters = errors.New(\"character ' ' not supported for MQTT topics\")\n\terrMQTTInvalidSession = errors.New(\"invalid MQTT session\")\n\terrMQTTInvalidRetainFlags = errors.New(\"invalid retained message flags\")\n)\n\ntype srvMQTT struct {\n\tlistener net.Listener\n\tlistenerErr error\n\tauthOverride bool\n\tsessmgr mqttSessionManager\n}\n\ntype mqttSessionManager struct {\n\tmu sync.RWMutex\n\tsessions map[string]*mqttAccountSessionManager // key is account name\n}\n\ntype mqttAccountSessionManager struct {\n\tmu sync.RWMutex\n\tsessions map[string]*mqttSession // key is MQTT client ID\n\tsessByHash map[string]*mqttSession // key is MQTT client ID hash\n\tsessLocked map[string]struct{} // key is MQTT client ID and indicate that a session can not be taken by a new client at this time\n\tflappers map[string]int64 // When connection connects with client ID already in use\n\tflapTimer *time.Timer // Timer to perform some cleanup of the flappers map\n\tsl *Sublist // sublist allowing to find retained messages for given subscription\n\tretmsgs map[string]*mqttRetainedMsgRef // retained messages\n\trmsCache *sync.Map // map[subject]mqttRetainedMsg\n\tjsa mqttJSA\n\tdomainTk string // Domain (with trailing \".\"), or possibly empty. This is added to session subject.\n}\n\ntype mqttJSAResponse struct {\n\treply string // will be used to map to the original request in jsa.NewRequestExMulti\n\tvalue any\n}\n\ntype mqttJSA struct {\n\tmu sync.Mutex\n\tid string\n\tc *client\n\tsendq *ipQueue[*mqttJSPubMsg]\n\trplyr string\n\treplies sync.Map // [string]chan *mqttJSAResponse\n\tnuid *nuid.NUID\n\tquitCh chan struct{}\n\tdomain string // Domain or possibly empty. This is added to session subject.\n\tdomainSet bool // covers if domain was set, even to empty\n\ttimeout time.Duration\n}\n\ntype mqttJSPubMsg struct {\n\tsubj string\n\treply string\n\thdr int\n\tmsg []byte\n}\n\ntype mqttRetMsgDel struct {\n\tSubject string `json:\"subject\"`\n\tSeq uint64 `json:\"seq\"`\n}\n\ntype mqttSession struct {\n\t// subsMu is a \"quick\" version of the session lock, sufficient for the QoS0\n\t// callback. It only guarantees that a new subscription is initialized, and\n\t// its retained messages if any have been queued up for delivery. The QoS12\n\t// callback uses the session lock.\n\tmu sync.Mutex\n\tsubsMu sync.RWMutex\n\n\tid string // client ID\n\tidHash string // client ID hash\n\tc *client\n\tjsa *mqttJSA\n\tsubs map[string]byte // Key is MQTT SUBSCRIBE filter, value is the subscription QoS\n\tcons map[string]*ConsumerConfig\n\tpubRelConsumer *ConsumerConfig\n\tpubRelSubscribed bool\n\tpubRelDeliverySubject string\n\tpubRelDeliverySubjectB []byte\n\tpubRelSubject string\n\tseq uint64\n\n\t// pendingPublish maps packet identifiers (PI) to JetStream ACK subjects for\n\t// QoS1 and 2 PUBLISH messages pending delivery to the session's client.\n\tpendingPublish map[uint16]*mqttPending\n\n\t// pendingPubRel maps PIs to JetStream ACK subjects for QoS2 PUBREL\n\t// messages pending delivery to the session's client.\n\tpendingPubRel map[uint16]*mqttPending\n\n\t// cpending maps delivery attempts (that come with a JS ACK subject) to\n\t// existing PIs.\n\tcpending map[string]map[uint64]uint16 // composite key: jsDur, sseq\n\n\t// \"Last used\" publish packet identifier (PI). starting point searching for the next available.\n\tlast_pi uint16\n\n\t// Maximum number of pending acks for this session.\n\tmaxp uint16\n\ttmaxack int\n\tclean bool\n\tdomainTk string\n}\n\ntype mqttPersistedSession struct {\n\tOrigin string `json:\"origin,omitempty\"`\n\tID string `json:\"id,omitempty\"`\n\tClean bool `json:\"clean,omitempty\"`\n\tSubs map[string]byte `json:\"subs,omitempty\"`\n\tCons map[string]*ConsumerConfig `json:\"cons,omitempty\"`\n\tPubRel *ConsumerConfig `json:\"pubrel,omitempty\"`\n}\n\ntype mqttRetainedMsg struct {\n\tOrigin string `json:\"origin,omitempty\"`\n\tSubject string `json:\"subject,omitempty\"`\n\tTopic string `json:\"topic,omitempty\"`\n\tMsg []byte `json:\"msg,omitempty\"`\n\tFlags byte `json:\"flags,omitempty\"`\n\tSource string `json:\"source,omitempty\"`\n\n\texpiresFromCache time.Time\n}\n\ntype mqttRetainedMsgRef struct {\n\tsseq uint64\n\tsub *subscription\n}\n\n// mqttSub contains fields associated with a MQTT subscription, and is added to\n// the main subscription struct for MQTT message delivery subscriptions. The\n// delivery callbacks may get invoked before sub.mqtt is set up, so they should\n// acquire either sess.mu or sess.subsMu before accessing it.\ntype mqttSub struct {\n\t// The sub's QOS and the JS durable name. They can change when\n\t// re-subscribing, and are used in the delivery callbacks. They can be\n\t// quickly accessed using sess.subsMu.RLock, or under the main session lock.\n\tqos byte\n\tjsDur string\n\n\t// Pending serialization of retained messages to be sent when subscription\n\t// is registered. The sub's delivery callbacks must wait until `prm` is\n\t// ready (can block on sess.mu for that, too).\n\tprm [][]byte\n\n\t// If this subscription needs to be checked for being reserved. E.g. '#' or\n\t// '*' or '*/'. It is set up at the time of subscription and is immutable\n\t// after that.\n\treserved bool\n}\n\ntype mqtt struct {\n\tr *mqttReader\n\tcp *mqttConnectProto\n\tpp *mqttPublish\n\tasm *mqttAccountSessionManager // quick reference to account session manager, immutable after processConnect()\n\tsess *mqttSession // quick reference to session, immutable after processConnect()\n\tcid string // client ID\n\n\t// rejectQoS2Pub tells the MQTT client to not accept QoS2 PUBLISH, instead\n\t// error and terminate the connection.\n\trejectQoS2Pub bool\n\n\t// downgradeQOS2Sub tells the MQTT client to downgrade QoS2 SUBSCRIBE\n\t// requests to QoS1.\n\tdowngradeQoS2Sub bool\n}\n\ntype mqttPending struct {\n\tsseq uint64 // stream sequence\n\tjsAckSubject string // the ACK subject to send the ack to\n\tjsDur string // JS durable name\n}\n\ntype mqttConnectProto struct {\n\trd time.Duration\n\twill *mqttWill\n\tflags byte\n}\n\ntype mqttIOReader interface {\n\tio.Reader\n\tSetReadDeadline(time.Time) error\n}\n\ntype mqttReader struct {\n\treader mqttIOReader\n\tbuf []byte\n\tpos int\n\tpstart int\n\tpbuf []byte\n}\n\ntype mqttWriter struct {\n\tbytes.Buffer\n}\n\ntype mqttWill struct {\n\ttopic []byte\n\tsubject []byte\n\tmapped []byte\n\tmessage []byte\n\tqos byte\n\tretain bool\n}\n\ntype mqttFilter struct {\n\tfilter string\n\tqos byte\n\t// Used only for tracing and should not be used after parsing of (un)sub protocols.\n\tttopic []byte\n}\n\ntype mqttPublish struct {\n\ttopic []byte\n\tsubject []byte\n\tmapped []byte\n\tmsg []byte\n\tsz int\n\tpi uint16\n\tflags byte\n}\n\n// When we re-encode incoming MQTT PUBLISH messages for NATS delivery, we add\n// the following headers:\n// - \"Nmqtt-Pub\" (*always) indicates that the message originated from MQTT, and\n// contains the original message QoS.\n// - \"Nmqtt-Subject\" contains the original MQTT subject from mqttParsePub.\n// - \"Nmqtt-Mapped\" contains the mapping during mqttParsePub.\n//\n// When we submit a PUBREL for delivery, we add a \"Nmqtt-PubRel\" header that\n// contains the PI.\nconst (\n\t// NATS header that indicates that the message originated from MQTT and\n\t// stores the published message QOS.\n\tmqttNatsHeader = \"Nmqtt-Pub\"\n\n\t// NATS headers to store retained message metadata (along with the original\n\t// message as binary).\n\tmqttNatsRetainedMessageTopic = \"Nmqtt-RTopic\"\n\tmqttNatsRetainedMessageOrigin = \"Nmqtt-ROrigin\"\n\tmqttNatsRetainedMessageFlags = \"Nmqtt-RFlags\"\n\tmqttNatsRetainedMessageSource = \"Nmqtt-RSource\"\n\n\t// NATS header that indicates that the message is an MQTT PubRel and stores\n\t// the PI.\n\tmqttNatsPubRelHeader = \"Nmqtt-PubRel\"\n\n\t// NATS headers to store the original MQTT subject and the subject mapping.\n\tmqttNatsHeaderSubject = \"Nmqtt-Subject\"\n\tmqttNatsHeaderMapped = \"Nmqtt-Mapped\"\n)\n\ntype mqttParsedPublishNATSHeader struct {\n\tqos byte\n\tsubject []byte\n\tmapped []byte\n}\n\nfunc (s *Server) startMQTT() {\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\tsopts := s.getOpts()\n\to := &sopts.MQTT\n\n\tvar hl net.Listener\n\tvar err error\n\n\tport := o.Port\n\tif port == -1 {\n\t\tport = 0\n\t}\n\thp := net.JoinHostPort(o.Host, strconv.Itoa(port))\n\ts.mu.Lock()\n\ts.mqtt.sessmgr.sessions = make(map[string]*mqttAccountSessionManager)\n\thl, err = natsListen(\"tcp\", hp)\n\ts.mqtt.listenerErr = err\n\tif err != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Unable to listen for MQTT connections: %v\", err)\n\t\treturn\n\t}\n\tif port == 0 {\n\t\to.Port = hl.Addr().(*net.TCPAddr).Port\n\t}\n\ts.mqtt.listener = hl\n\tscheme := \"mqtt\"\n\tif o.TLSConfig != nil {\n\t\tscheme = \"tls\"\n\t}\n\ts.Noticef(\"Listening for MQTT clients on %s://%s:%d\", scheme, o.Host, o.Port)\n\tgo s.acceptConnections(hl, \"MQTT\", func(conn net.Conn) { s.createMQTTClient(conn, nil) }, nil)\n\ts.mu.Unlock()\n}\n\n// This is similar to createClient() but has some modifications specifi to MQTT clients.\n// The comments have been kept to minimum to reduce code size. Check createClient() for\n// more details.\nfunc (s *Server) createMQTTClient(conn net.Conn, ws *websocket) *client {\n\topts := s.getOpts()\n\n\tmaxPay := int32(opts.MaxPayload)\n\tmaxSubs := int32(opts.MaxSubs)\n\tif maxSubs == 0 {\n\t\tmaxSubs = -1\n\t}\n\tnow := time.Now()\n\n\tmqtt := &mqtt{\n\t\trejectQoS2Pub: opts.MQTT.rejectQoS2Pub,\n\t\tdowngradeQoS2Sub: opts.MQTT.downgradeQoS2Sub,\n\t}\n\tc := &client{srv: s, nc: conn, mpay: maxPay, msubs: maxSubs, start: now, last: now, mqtt: mqtt, ws: ws}\n\tc.headers = true\n\tc.mqtt.pp = &mqttPublish{}\n\t// MQTT clients don't send NATS CONNECT protocols. So make it an \"echo\"\n\t// client, but disable verbose and pedantic (by not setting them).\n\tc.opts.Echo = true\n\n\tc.registerWithAccount(s.globalAccount())\n\n\ts.mu.Lock()\n\t// Check auth, override if applicable.\n\tauthRequired := s.info.AuthRequired || s.mqtt.authOverride\n\ts.totalClients++\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\tif authRequired {\n\t\tc.flags.set(expectConnect)\n\t}\n\tc.initClient()\n\tc.Debugf(\"Client connection created\")\n\tc.mu.Unlock()\n\n\ts.mu.Lock()\n\tif !s.isRunning() || s.ldm {\n\t\tif s.isShuttingDown() {\n\t\t\tconn.Close()\n\t\t}\n\t\ts.mu.Unlock()\n\t\treturn c\n\t}\n\n\tif opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) {\n\t\ts.mu.Unlock()\n\t\tc.maxConnExceeded()\n\t\treturn nil\n\t}\n\ts.clients[c.cid] = c\n\n\t// Websocket TLS handshake is already done when getting to this function.\n\ttlsRequired := opts.MQTT.TLSConfig != nil && ws == nil\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\n\t// In case connection has already been closed\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(WriteError)\n\t\treturn nil\n\t}\n\n\tvar pre []byte\n\tif tlsRequired && opts.AllowNonTLS {\n\t\tpre = make([]byte, 4)\n\t\tc.nc.SetReadDeadline(time.Now().Add(secondsToDuration(opts.MQTT.TLSTimeout)))\n\t\tn, _ := io.ReadFull(c.nc, pre[:])\n\t\tc.nc.SetReadDeadline(time.Time{})\n\t\tpre = pre[:n]\n\t\tif n > 0 && pre[0] == 0x16 {\n\t\t\ttlsRequired = true\n\t\t} else {\n\t\t\ttlsRequired = false\n\t\t}\n\t}\n\n\tif tlsRequired {\n\t\tif len(pre) > 0 {\n\t\t\tc.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)}\n\t\t\tpre = nil\n\t\t}\n\n\t\t// Perform server-side TLS handshake.\n\t\tif err := c.doTLSServerHandshake(tlsHandshakeMQTT, opts.MQTT.TLSConfig, opts.MQTT.TLSTimeout, opts.MQTT.TLSPinnedCerts); err != nil {\n\t\t\tc.mu.Unlock()\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif authRequired {\n\t\ttimeout := opts.AuthTimeout\n\t\t// Possibly override with MQTT specific value.\n\t\tif opts.MQTT.AuthTimeout != 0 {\n\t\t\ttimeout = opts.MQTT.AuthTimeout\n\t\t}\n\t\tc.setAuthTimer(secondsToDuration(timeout))\n\t}\n\n\t// No Ping timer for MQTT clients...\n\n\ts.startGoRoutine(func() { c.readLoop(pre) })\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\tif tlsRequired {\n\t\tc.Debugf(\"TLS handshake complete\")\n\t\tcs := c.nc.(*tls.Conn).ConnectionState()\n\t\tc.Debugf(\"TLS version %s, cipher suite %s\", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite))\n\t}\n\n\tc.mu.Unlock()\n\n\treturn c\n}\n\n// Given the mqtt options, we check if any auth configuration\n// has been provided. If so, possibly create users/nkey users and\n// store them in s.mqtt.users/nkeys.\n// Also update a boolean that indicates if auth is required for\n// mqtt clients.\n// Server lock is held on entry.\nfunc (s *Server) mqttConfigAuth(opts *MQTTOpts) {\n\tmqtt := &s.mqtt\n\t// If any of those is specified, we consider that there is an override.\n\tmqtt.authOverride = opts.Username != _EMPTY_ || opts.Token != _EMPTY_ || opts.NoAuthUser != _EMPTY_\n}\n\n// Validate the mqtt related options.\nfunc validateMQTTOptions(o *Options) error {\n\tmo := &o.MQTT\n\t// If no port is defined, we don't care about other options\n\tif mo.Port == 0 {\n\t\treturn nil\n\t}\n\t// We have to force the server name to be explicitly set and be unique when\n\t// in cluster mode.\n\tif o.ServerName == _EMPTY_ && (o.Cluster.Port != 0 || o.Gateway.Port != 0) {\n\t\treturn errMQTTServerNameMustBeSet\n\t}\n\t// If there is a NoAuthUser, we need to have Users defined and\n\t// the user to be present.\n\tif mo.NoAuthUser != _EMPTY_ {\n\t\tif err := validateNoAuthUser(o, mo.NoAuthUser); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Token/Username not possible if there are users/nkeys\n\tif len(o.Users) > 0 || len(o.Nkeys) > 0 {\n\t\tif mo.Username != _EMPTY_ {\n\t\t\treturn errMQTTUserMixWithUsersNKeys\n\t\t}\n\t\tif mo.Token != _EMPTY_ {\n\t\t\treturn errMQTTTokenMixWIthUsersNKeys\n\t\t}\n\t}\n\tif mo.AckWait < 0 {\n\t\treturn errMQTTAckWaitMustBePositive\n\t}\n\tif mo.JSAPITimeout < 0 {\n\t\treturn errMQTTJSAPITimeoutMustBePositive\n\t}\n\t// If strictly standalone and there is no JS enabled, then it won't work...\n\t// For leafnodes, we could either have remote(s) and it would be ok, or no\n\t// remote but accept from a remote side that has \"hub\" property set, which\n\t// then would ok too. So we fail only if we have no leafnode config at all.\n\tif !o.JetStream && o.Cluster.Port == 0 && o.Gateway.Port == 0 &&\n\t\to.LeafNode.Port == 0 && len(o.LeafNode.Remotes) == 0 {\n\t\treturn errMQTTStandaloneNeedsJetStream\n\t}\n\tif err := validatePinnedCerts(mo.TLSPinnedCerts); err != nil {\n\t\treturn fmt.Errorf(\"mqtt: %v\", err)\n\t}\n\tif mo.ConsumerReplicas > 0 && mo.StreamReplicas > 0 && mo.ConsumerReplicas > mo.StreamReplicas {\n\t\treturn fmt.Errorf(\"mqtt: consumer_replicas (%v) cannot be higher than stream_replicas (%v)\",\n\t\t\tmo.ConsumerReplicas, mo.StreamReplicas)\n\t}\n\treturn nil\n}\n\n// Returns true if this connection is from a MQTT client.\n// Lock held on entry.\nfunc (c *client) isMqtt() bool {\n\treturn c.mqtt != nil\n}\n\n// If this is an MQTT client, returns the session client ID,\n// otherwise returns the empty string.\n// Lock held on entry\nfunc (c *client) getMQTTClientID() string {\n\tif !c.isMqtt() {\n\t\treturn _EMPTY_\n\t}\n\treturn c.mqtt.cid\n}\n\n// Parse protocols inside the given buffer.\n// This is invoked from the readLoop.\nfunc (c *client) mqttParse(buf []byte) error {\n\tc.mu.Lock()\n\ts := c.srv\n\ttrace := c.trace\n\tconnected := c.flags.isSet(connectReceived)\n\tmqtt := c.mqtt\n\tr := mqtt.r\n\tvar rd time.Duration\n\tif mqtt.cp != nil {\n\t\trd = mqtt.cp.rd\n\t\tif rd > 0 {\n\t\t\tr.reader.SetReadDeadline(time.Time{})\n\t\t}\n\t}\n\thasMappings := c.in.flags.isSet(hasMappings)\n\tc.mu.Unlock()\n\n\tr.reset(buf)\n\n\tvar err error\n\tvar b byte\n\tvar pl int\n\tvar complete bool\n\n\tfor err == nil && r.hasMore() {\n\n\t\t// Keep track of the starting of the packet, in case we have a partial\n\t\tr.pstart = r.pos\n\n\t\t// Read packet type and flags\n\t\tif b, err = r.readByte(\"packet type\"); err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\t// Packet type\n\t\tpt := b & mqttPacketMask\n\n\t\t// If client was not connected yet, the first packet must be\n\t\t// a mqttPacketConnect otherwise we fail the connection.\n\t\tif !connected && pt != mqttPacketConnect {\n\t\t\t// If the buffer indicates that it may be a websocket handshake\n\t\t\t// but the client is not websocket, it means that the client\n\t\t\t// connected to the MQTT port instead of the Websocket port.\n\t\t\tif bytes.HasPrefix(buf, []byte(\"GET \")) && !c.isWebsocket() {\n\t\t\t\terr = errMQTTNotWebsocketPort\n\t\t\t} else {\n\t\t\t\terr = fmt.Errorf(\"the first packet should be a CONNECT (%v), got %v\", mqttPacketConnect, pt)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tif err = mqttCheckFixedHeaderFlags(pt, b&mqttPacketFlagMask); err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\tpl, complete, err = r.readPacketLen()\n\t\tif err != nil || !complete {\n\t\t\tbreak\n\t\t}\n\t\tif err = mqttCheckRemainingLength(pt, pl); err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\tswitch pt {\n\t\t// Packets that we receive back when we act as the \"sender\": PUBACK,\n\t\t// PUBREC, PUBCOMP.\n\t\tcase mqttPacketPubAck:\n\t\t\tvar pi uint16\n\t\t\tpi, err = mqttParsePIPacket(r)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PUBACK\", errOrTrace(err, fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\terr = c.mqttProcessPubAck(pi)\n\t\t\t}\n\n\t\tcase mqttPacketPubRec:\n\t\t\tvar pi uint16\n\t\t\tpi, err = mqttParsePIPacket(r)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PUBREC\", errOrTrace(err, fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\terr = c.mqttProcessPubRec(pi)\n\t\t\t}\n\n\t\tcase mqttPacketPubComp:\n\t\t\tvar pi uint16\n\t\t\tpi, err = mqttParsePIPacket(r)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PUBCOMP\", errOrTrace(err, fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tc.mqttProcessPubComp(pi)\n\t\t\t}\n\n\t\t// Packets where we act as the \"receiver\": PUBLISH, PUBREL, SUBSCRIBE, UNSUBSCRIBE.\n\t\tcase mqttPacketPub:\n\t\t\tpp := c.mqtt.pp\n\t\t\tpp.flags = b & mqttPacketFlagMask\n\t\t\terr = c.mqttParsePub(r, pl, pp, hasMappings)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PUBLISH\", errOrTrace(err, mqttPubTrace(pp)))\n\t\t\t\tif err == nil {\n\t\t\t\t\tc.mqttTraceMsg(pp.msg)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\terr = s.mqttProcessPub(c, pp, trace)\n\t\t\t}\n\n\t\tcase mqttPacketPubRel:\n\t\t\tvar pi uint16\n\t\t\tpi, err = mqttParsePIPacket(r)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PUBREL\", errOrTrace(err, fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\terr = s.mqttProcessPubRel(c, pi, trace)\n\t\t\t}\n\n\t\tcase mqttPacketSub:\n\t\t\tvar pi uint16 // packet identifier\n\t\t\tvar filters []*mqttFilter\n\t\t\tvar subs []*subscription\n\t\t\tpi, filters, err = c.mqttParseSubs(r, b, pl)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"SUBSCRIBE\", errOrTrace(err, mqttSubscribeTrace(pi, filters)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tsubs, err = c.mqttProcessSubs(filters)\n\t\t\t\tif err == nil && trace {\n\t\t\t\t\tc.traceOutOp(\"SUBACK\", []byte(fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tc.mqttEnqueueSubAck(pi, filters)\n\t\t\t\tc.mqttSendRetainedMsgsToNewSubs(subs)\n\t\t\t}\n\n\t\tcase mqttPacketUnsub:\n\t\t\tvar pi uint16 // packet identifier\n\t\t\tvar filters []*mqttFilter\n\t\t\tpi, filters, err = c.mqttParseUnsubs(r, b, pl)\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"UNSUBSCRIBE\", errOrTrace(err, mqttUnsubscribeTrace(pi, filters)))\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\terr = c.mqttProcessUnsubs(filters)\n\t\t\t\tif err == nil && trace {\n\t\t\t\t\tc.traceOutOp(\"UNSUBACK\", []byte(fmt.Sprintf(\"pi=%v\", pi)))\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tc.mqttEnqueueUnsubAck(pi)\n\t\t\t}\n\n\t\t// Packets that we get both as a receiver and sender: PING, CONNECT, DISCONNECT\n\t\tcase mqttPacketPing:\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"PINGREQ\", nil)\n\t\t\t}\n\t\t\tc.mqttEnqueuePingResp()\n\t\t\tif trace {\n\t\t\t\tc.traceOutOp(\"PINGRESP\", nil)\n\t\t\t}\n\n\t\tcase mqttPacketConnect:\n\t\t\t// It is an error to receive a second connect packet\n\t\t\tif connected {\n\t\t\t\terr = errMQTTSecondConnectPacket\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tvar rc byte\n\t\t\tvar cp *mqttConnectProto\n\t\t\tvar sessp bool\n\t\t\trc, cp, err = c.mqttParseConnect(r, hasMappings)\n\t\t\t// Add the client id to the client's string, regardless of error.\n\t\t\t// We may still get the client_id if the call above fails somewhere\n\t\t\t// after parsing the client ID itself.\n\t\t\tc.ncs.Store(fmt.Sprintf(\"%s - %q\", c, c.mqtt.cid))\n\t\t\tif trace && cp != nil {\n\t\t\t\tc.traceInOp(\"CONNECT\", errOrTrace(err, c.mqttConnectTrace(cp)))\n\t\t\t}\n\t\t\tif rc != 0 {\n\t\t\t\tc.mqttEnqueueConnAck(rc, sessp)\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceOutOp(\"CONNACK\", []byte(fmt.Sprintf(\"sp=%v rc=%v\", sessp, rc)))\n\t\t\t\t}\n\t\t\t} else if err == nil {\n\t\t\t\tif err = s.mqttProcessConnect(c, cp, trace); err != nil {\n\t\t\t\t\terr = fmt.Errorf(\"unable to connect: %v\", err)\n\t\t\t\t} else {\n\t\t\t\t\t// Add this debug statement so users running in Debug mode\n\t\t\t\t\t// will have the client id printed here for the first time.\n\t\t\t\t\tc.Debugf(\"Client connected\")\n\t\t\t\t\tconnected = true\n\t\t\t\t\trd = cp.rd\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase mqttPacketDisconnect:\n\t\t\tif trace {\n\t\t\t\tc.traceInOp(\"DISCONNECT\", nil)\n\t\t\t}\n\t\t\t// Normal disconnect, we need to discard the will.\n\t\t\t// Spec [MQTT-3.1.2-8]\n\t\t\tc.mu.Lock()\n\t\t\tif c.mqtt.cp != nil {\n\t\t\t\tc.mqtt.cp.will = nil\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\ts.mqttHandleClosedClient(c)\n\t\t\tc.closeConnection(ClientClosed)\n\t\t\treturn nil\n\n\t\tdefault:\n\t\t\terr = fmt.Errorf(\"received unknown packet type %d\", pt>>4)\n\t\t}\n\t}\n\tif err == nil && rd > 0 {\n\t\tr.reader.SetReadDeadline(time.Now().Add(rd))\n\t}\n\treturn err\n}\n\nfunc mqttCheckFixedHeaderFlags(packetType, flags byte) error {\n\tvar expected byte\n\tswitch packetType {\n\tcase mqttPacketConnect, mqttPacketPubAck, mqttPacketPubRec, mqttPacketPubComp,\n\t\tmqttPacketPing, mqttPacketDisconnect:\n\t\texpected = 0\n\tcase mqttPacketPubRel, mqttPacketSub, mqttPacketUnsub:\n\t\texpected = 0x2\n\tcase mqttPacketPub:\n\t\treturn nil\n\tdefault:\n\t\treturn nil\n\t}\n\tif flags != expected {\n\t\treturn fmt.Errorf(\"invalid fixed header flags %x for packet type %x\", flags, packetType)\n\t}\n\treturn nil\n}\n\nfunc mqttCheckRemainingLength(packetType byte, pl int) error {\n\tvar expected int\n\tswitch packetType {\n\tcase mqttPacketConnect, mqttPacketPub, mqttPacketSub, mqttPacketUnsub:\n\t\treturn nil\n\tcase mqttPacketPubAck, mqttPacketPubRec, mqttPacketPubRel, mqttPacketPubComp:\n\t\texpected = 2\n\tcase mqttPacketPing, mqttPacketDisconnect:\n\t\texpected = 0\n\tdefault:\n\t\treturn nil\n\t}\n\tif pl != expected {\n\t\treturn fmt.Errorf(\"invalid remaining length %d for packet type %x\", pl, packetType)\n\t}\n\treturn nil\n}\n\nfunc (c *client) mqttTraceMsg(msg []byte) {\n\tmaxTrace := c.srv.getOpts().MaxTracedMsgLen\n\tif maxTrace > 0 && len(msg) > maxTrace {\n\t\tc.Tracef(\"<<- MSG_PAYLOAD: [\\\"%s...\\\"]\", msg[:maxTrace])\n\t} else {\n\t\tc.Tracef(\"<<- MSG_PAYLOAD: [%q]\", msg)\n\t}\n}\n\n// The MQTT client connection has been closed, or the DISCONNECT packet was received.\n// For a \"clean\" session, we will delete the session, otherwise, simply removing\n// the binding. We will also send the \"will\" message if applicable.\n//\n// Runs from the client's readLoop.\n// No lock held on entry.\nfunc (s *Server) mqttHandleClosedClient(c *client) {\n\tc.mu.Lock()\n\tasm := c.mqtt.asm\n\tsess := c.mqtt.sess\n\tc.mu.Unlock()\n\n\t// If asm or sess are nil, it means that we have failed a client\n\t// before it was associated with a session, so nothing more to do.\n\tif asm == nil || sess == nil {\n\t\treturn\n\t}\n\n\t// Add this session to the locked map for the rest of the execution.\n\tif err := asm.lockSession(sess, c); err != nil {\n\t\treturn\n\t}\n\tdefer asm.unlockSession(sess)\n\n\tasm.mu.Lock()\n\t// Clear the client from the session, but session may stay.\n\tsess.mu.Lock()\n\tsess.c = nil\n\tdoClean := sess.clean\n\tsess.mu.Unlock()\n\t// If it was a clean session, then we remove from the account manager,\n\t// and we will call clear() outside of any lock.\n\tif doClean {\n\t\tasm.removeSession(sess, false)\n\t}\n\t// Remove in case it was in the flappers map.\n\tasm.removeSessFromFlappers(sess.id)\n\tasm.mu.Unlock()\n\n\t// This needs to be done outside of any lock.\n\tif doClean {\n\t\tif err := sess.clear(true); err != nil {\n\t\t\tc.Errorf(err.Error())\n\t\t}\n\t}\n\n\t// Now handle the \"will\". This function will be a no-op if there is no \"will\" to send.\n\ts.mqttHandleWill(c)\n}\n\n// Updates the MaxAckPending for all MQTT sessions, updating the\n// JetStream consumers and updating their max ack pending and forcing\n// a expiration of pending messages.\n//\n// Runs from a server configuration reload routine.\n// No lock held on entry.\nfunc (s *Server) mqttUpdateMaxAckPending(newmaxp uint16) {\n\tmsm := &s.mqtt.sessmgr\n\ts.accounts.Range(func(k, _ any) bool {\n\t\taccName := k.(string)\n\t\tmsm.mu.RLock()\n\t\tasm := msm.sessions[accName]\n\t\tmsm.mu.RUnlock()\n\t\tif asm == nil {\n\t\t\t// Move to next account\n\t\t\treturn true\n\t\t}\n\t\tasm.mu.RLock()\n\t\tfor _, sess := range asm.sessions {\n\t\t\tsess.mu.Lock()\n\t\t\tsess.maxp = newmaxp\n\t\t\tsess.mu.Unlock()\n\t\t}\n\t\tasm.mu.RUnlock()\n\t\treturn true\n\t})\n}\n\nfunc (s *Server) mqttGetJSAForAccount(acc string) *mqttJSA {\n\tsm := &s.mqtt.sessmgr\n\n\tsm.mu.RLock()\n\tasm := sm.sessions[acc]\n\tsm.mu.RUnlock()\n\n\tif asm == nil {\n\t\treturn nil\n\t}\n\n\tasm.mu.RLock()\n\tjsa := &asm.jsa\n\tasm.mu.RUnlock()\n\treturn jsa\n}\n\nfunc (s *Server) mqttStoreQoSMsgForAccountOnNewSubject(hdr int, msg []byte, acc, subject string) {\n\tif s == nil || hdr <= 0 {\n\t\treturn\n\t}\n\th := mqttParsePublishNATSHeader(msg[:hdr])\n\tif h == nil || h.qos == 0 {\n\t\treturn\n\t}\n\tjsa := s.mqttGetJSAForAccount(acc)\n\tif jsa == nil {\n\t\treturn\n\t}\n\tjsa.storeMsg(mqttStreamSubjectPrefix+subject, hdr, msg)\n}\n\nfunc mqttParsePublishNATSHeader(headerBytes []byte) *mqttParsedPublishNATSHeader {\n\tif len(headerBytes) == 0 {\n\t\treturn nil\n\t}\n\n\tpubValue := getHeader(mqttNatsHeader, headerBytes)\n\tif len(pubValue) == 0 {\n\t\treturn nil\n\t}\n\treturn &mqttParsedPublishNATSHeader{\n\t\tqos: pubValue[0] - '0',\n\t\tsubject: getHeader(mqttNatsHeaderSubject, headerBytes),\n\t\tmapped: getHeader(mqttNatsHeaderMapped, headerBytes),\n\t}\n}\n\nfunc mqttParsePubRelNATSHeader(headerBytes []byte) uint16 {\n\tif len(headerBytes) == 0 {\n\t\treturn 0\n\t}\n\n\tpubrelValue := getHeader(mqttNatsPubRelHeader, headerBytes)\n\tif len(pubrelValue) == 0 {\n\t\treturn 0\n\t}\n\tpi, _ := strconv.ParseUint(string(pubrelValue), 10, 16)\n\treturn uint16(pi)\n}\n\n// Returns the MQTT sessions manager for a given account.\n// If new, creates the required JetStream streams/consumers\n// for handling of sessions and messages.\nfunc (s *Server) getOrCreateMQTTAccountSessionManager(c *client) (*mqttAccountSessionManager, error) {\n\tsm := &s.mqtt.sessmgr\n\n\tc.mu.Lock()\n\tacc := c.acc\n\tc.mu.Unlock()\n\taccName := acc.GetName()\n\n\tsm.mu.RLock()\n\tasm, ok := sm.sessions[accName]\n\tsm.mu.RUnlock()\n\n\tif ok {\n\t\treturn asm, nil\n\t}\n\n\t// We will pass the quitCh to the account session manager if we happen to create it.\n\ts.mu.Lock()\n\tquitCh := s.quitCh\n\ts.mu.Unlock()\n\n\t// Not found, now take the write lock and check again\n\tsm.mu.Lock()\n\tdefer sm.mu.Unlock()\n\tasm, ok = sm.sessions[accName]\n\tif ok {\n\t\treturn asm, nil\n\t}\n\t// Need to create one here.\n\tasm, err := s.mqttCreateAccountSessionManager(acc, quitCh)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsm.sessions[accName] = asm\n\treturn asm, nil\n}\n\n// Creates JS streams/consumers for handling of sessions and messages for this account.\n//\n// Global session manager lock is held on entry.\nfunc (s *Server) mqttCreateAccountSessionManager(acc *Account, quitCh chan struct{}) (*mqttAccountSessionManager, error) {\n\tvar err error\n\n\taccName := acc.GetName()\n\n\topts := s.getOpts()\n\tc := s.createInternalAccountClient()\n\tc.acc = acc\n\n\tid := s.NodeName()\n\n\tmqttJSAPITimeout := opts.MQTT.JSAPITimeout\n\tif mqttJSAPITimeout == 0 {\n\t\tmqttJSAPITimeout = mqttDefaultJSAPITimeout\n\t}\n\n\treplicas := opts.MQTT.StreamReplicas\n\tif replicas <= 0 {\n\t\treplicas = s.mqttDetermineReplicas()\n\t}\n\tqname := fmt.Sprintf(\"[ACC:%s] MQTT \", accName)\n\tas := &mqttAccountSessionManager{\n\t\tsessions: make(map[string]*mqttSession),\n\t\tsessByHash: make(map[string]*mqttSession),\n\t\tsessLocked: make(map[string]struct{}),\n\t\tflappers: make(map[string]int64),\n\t\tjsa: mqttJSA{\n\t\t\tid: id,\n\t\t\tc: c,\n\t\t\trplyr: mqttJSARepliesPrefix + id + \".\",\n\t\t\tsendq: newIPQueue[*mqttJSPubMsg](s, qname+\"send\"),\n\t\t\tnuid: nuid.New(),\n\t\t\tquitCh: quitCh,\n\t\t\ttimeout: mqttJSAPITimeout,\n\t\t},\n\t\trmsCache: &sync.Map{},\n\t}\n\t// TODO record domain name in as here\n\n\t// The domain to communicate with may be required for JS calls.\n\t// Search from specific (per account setting) to generic (mqtt setting)\n\tif opts.JsAccDefaultDomain != nil {\n\t\tif d, ok := opts.JsAccDefaultDomain[accName]; ok {\n\t\t\tif d != _EMPTY_ {\n\t\t\t\tas.jsa.domain = d\n\t\t\t}\n\t\t\tas.jsa.domainSet = true\n\t\t}\n\t\t// in case domain was set to empty, check if there are more generic domain overwrites\n\t}\n\tif as.jsa.domain == _EMPTY_ {\n\t\tif d := opts.MQTT.JsDomain; d != _EMPTY_ {\n\t\t\tas.jsa.domain = d\n\t\t\tas.jsa.domainSet = true\n\t\t}\n\t}\n\t// We need to include the domain in the subject prefix used to store sessions in the $MQTT_sess stream.\n\tif as.jsa.domainSet {\n\t\tif as.jsa.domain != _EMPTY_ {\n\t\t\tas.domainTk = as.jsa.domain + \".\"\n\t\t}\n\t} else if d := s.getOpts().JetStreamDomain; d != _EMPTY_ {\n\t\tas.domainTk = d + \".\"\n\t}\n\tif as.jsa.domainSet {\n\t\ts.Noticef(\"Creating MQTT streams/consumers with replicas %v for account %q in domain %q\", replicas, accName, as.jsa.domain)\n\t} else {\n\t\ts.Noticef(\"Creating MQTT streams/consumers with replicas %v for account %q\", replicas, accName)\n\t}\n\n\tvar subs []*subscription\n\tvar success bool\n\tcloseCh := make(chan struct{})\n\n\tdefer func() {\n\t\tif success {\n\t\t\treturn\n\t\t}\n\t\tfor _, sub := range subs {\n\t\t\tc.processUnsub(sub.sid)\n\t\t}\n\t\tclose(closeCh)\n\t}()\n\n\t// We create all subscriptions before starting the go routine that will do\n\t// sends otherwise we could get races.\n\t// Note that using two different clients (one for the subs, one for the\n\t// sends) would cause other issues such as registration of recent subs in\n\t// the \"sub\" client would be invisible to the check for GW routed replies\n\t// (shouldMapReplyForGatewaySend) since the client there would be the \"sender\".\n\n\tjsa := &as.jsa\n\tsid := int64(1)\n\t// This is a subscription that will process all JS API replies. We could split to\n\t// individual subscriptions if needed, but since there is a bit of common code,\n\t// that seemed like a good idea to be all in one place.\n\tif err := as.createSubscription(jsa.rplyr+\">\",\n\t\tas.processJSAPIReplies, &sid, &subs); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// We will listen for replies to session persist requests so that we can\n\t// detect the use of a session with the same client ID anywhere in the cluster.\n\t// `$MQTT.JSA.{js-id}.SP.{client-id-hash}.{uuid}`\n\tif err := as.createSubscription(mqttJSARepliesPrefix+\"*.\"+mqttJSASessPersist+\".*.*\",\n\t\tas.processSessionPersist, &sid, &subs); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// We create the subscription on \"$MQTT.sub.\" to limit the subjects\n\t// that a user would allow permissions on.\n\trmsubj := mqttSubPrefix + nuid.Next()\n\tif err := as.createSubscription(rmsubj, as.processRetainedMsg, &sid, &subs); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Create a subscription to be notified of retained messages delete requests.\n\trmdelsubj := mqttJSARepliesPrefix + \"*.\" + mqttJSARetainedMsgDel\n\tif err := as.createSubscription(rmdelsubj, as.processRetainedMsgDel, &sid, &subs); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// No more creation of subscriptions past this point otherwise RACEs may happen.\n\n\t// Start the go routine that will send JS API requests.\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\t\tas.sendJSAPIrequests(s, c, accName, closeCh)\n\t})\n\n\t// Start the go routine that will clean up cached retained messages that expired.\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\t\tas.cleanupRetainedMessageCache(s, closeCh)\n\t})\n\n\tlookupStream := func(stream, txt string) (*StreamInfo, error) {\n\t\tsi, err := jsa.lookupStream(stream)\n\t\tif err != nil {\n\t\t\tif IsNatsErr(err, JSStreamNotFoundErr) {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t\treturn nil, fmt.Errorf(\"lookup %s stream for account %q: %v\", txt, accName, err)\n\t\t}\n\t\tif opts.MQTT.StreamReplicas == 0 {\n\t\t\treturn si, nil\n\t\t}\n\t\tsr := 1\n\t\tif si.Cluster != nil {\n\t\t\tsr += len(si.Cluster.Replicas)\n\t\t}\n\t\tif replicas != sr {\n\t\t\ts.Warnf(\"MQTT %s stream replicas mismatch: current is %v but configuration is %v for '%s > %s'\",\n\t\t\t\ttxt, sr, replicas, accName, stream)\n\t\t}\n\t\treturn si, nil\n\t}\n\n\tif si, err := lookupStream(mqttSessStreamName, \"sessions\"); err != nil {\n\t\treturn nil, err\n\t} else if si == nil {\n\t\t// Create the stream for the sessions.\n\t\tcfg := &StreamConfig{\n\t\t\tName: mqttSessStreamName,\n\t\t\tSubjects: []string{mqttSessStreamSubjectPrefix + as.domainTk + \">\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tMaxMsgsPer: 1,\n\t\t}\n\t\tif _, created, err := jsa.createStream(cfg); err == nil && created {\n\t\t\tas.transferUniqueSessStreamsToMuxed(s)\n\t\t} else if isErrorOtherThan(err, JSStreamNameExistErr) {\n\t\t\treturn nil, fmt.Errorf(\"create sessions stream for account %q: %v\", accName, err)\n\t\t}\n\t}\n\n\tif si, err := lookupStream(mqttStreamName, \"messages\"); err != nil {\n\t\treturn nil, err\n\t} else if si == nil {\n\t\t// Create the stream for the messages.\n\t\tcfg := &StreamConfig{\n\t\t\tName: mqttStreamName,\n\t\t\tSubjects: []string{mqttStreamSubjectPrefix + \">\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: InterestPolicy,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\tif _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) {\n\t\t\treturn nil, fmt.Errorf(\"create messages stream for account %q: %v\", accName, err)\n\t\t}\n\t}\n\n\tif si, err := lookupStream(mqttQoS2IncomingMsgsStreamName, \"QoS2 incoming messages\"); err != nil {\n\t\treturn nil, err\n\t} else if si == nil {\n\t\t// Create the stream for the incoming QoS2 messages that have not been\n\t\t// PUBREL-ed by the sender. Subject is\n\t\t// \"$MQTT.qos2..\", the .PI is to achieve exactly\n\t\t// once for each PI.\n\t\tcfg := &StreamConfig{\n\t\t\tName: mqttQoS2IncomingMsgsStreamName,\n\t\t\tSubjects: []string{mqttQoS2IncomingMsgsStreamSubjectPrefix + \">\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tDiscard: DiscardNew,\n\t\t\tMaxMsgsPer: 1,\n\t\t\tDiscardNewPer: true,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\tif _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) {\n\t\t\treturn nil, fmt.Errorf(\"create QoS2 incoming messages stream for account %q: %v\", accName, err)\n\t\t}\n\t}\n\n\tif si, err := lookupStream(mqttOutStreamName, \"QoS2 outgoing PUBREL\"); err != nil {\n\t\treturn nil, err\n\t} else if si == nil {\n\t\t// Create the stream for the incoming QoS2 messages that have not been\n\t\t// PUBREL-ed by the sender. NATS messages are submitted as\n\t\t// \"$MQTT.pubrel.\"\n\t\tcfg := &StreamConfig{\n\t\t\tName: mqttOutStreamName,\n\t\t\tSubjects: []string{mqttOutSubjectPrefix + \">\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: InterestPolicy,\n\t\t\tReplicas: replicas,\n\t\t}\n\t\tif _, _, err := jsa.createStream(cfg); isErrorOtherThan(err, JSStreamNameExistErr) {\n\t\t\treturn nil, fmt.Errorf(\"create QoS2 outgoing PUBREL stream for account %q: %v\", accName, err)\n\t\t}\n\t}\n\n\t// This is the only case where we need \"si\" after lookup/create\n\tneedToTransfer := true\n\tsi, err := lookupStream(mqttRetainedMsgsStreamName, \"retained messages\")\n\tswitch {\n\tcase err != nil:\n\t\treturn nil, err\n\n\tcase si == nil:\n\t\t// Create the stream for retained messages.\n\t\tcfg := &StreamConfig{\n\t\t\tName: mqttRetainedMsgsStreamName,\n\t\t\tSubjects: []string{mqttRetainedMsgsStreamSubject + \">\"},\n\t\t\tStorage: FileStorage,\n\t\t\tRetention: LimitsPolicy,\n\t\t\tReplicas: replicas,\n\t\t\tMaxMsgsPer: 1,\n\t\t}\n\t\t// We will need \"si\" outside of this block.\n\t\tsi, _, err = jsa.createStream(cfg)\n\t\tif err != nil {\n\t\t\tif isErrorOtherThan(err, JSStreamNameExistErr) {\n\t\t\t\treturn nil, fmt.Errorf(\"create retained messages stream for account %q: %v\", accName, err)\n\t\t\t}\n\t\t\t// Suppose we had a race and the stream was actually created by another\n\t\t\t// node, we really need \"si\" after that, so lookup the stream again here.\n\t\t\tsi, err = lookupStream(mqttRetainedMsgsStreamName, \"retained messages\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tneedToTransfer = false\n\n\tdefault:\n\t\tneedToTransfer = si.Config.MaxMsgsPer != 1\n\t}\n\n\t// Doing this check outside of above if/else due to possible race when\n\t// creating the stream.\n\twantedSubj := mqttRetainedMsgsStreamSubject + \">\"\n\tif len(si.Config.Subjects) != 1 || si.Config.Subjects[0] != wantedSubj {\n\t\t// Update only the Subjects at this stage, not MaxMsgsPer yet.\n\t\tsi.Config.Subjects = []string{wantedSubj}\n\t\tif si, err = jsa.updateStream(&si.Config); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to update stream config: %w\", err)\n\t\t}\n\t}\n\n\ttransferRMS := func() error {\n\t\tif !needToTransfer {\n\t\t\treturn nil\n\t\t}\n\n\t\tas.transferRetainedToPerKeySubjectStream(s)\n\n\t\t// We need another lookup to have up-to-date si.State values in order\n\t\t// to load all retained messages.\n\t\tsi, err = lookupStream(mqttRetainedMsgsStreamName, \"retained messages\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tneedToTransfer = false\n\t\treturn nil\n\t}\n\n\t// Attempt to transfer all \"single subject\" retained messages to new\n\t// subjects. It may fail, will log its own error; ignore it the first time\n\t// and proceed to updating MaxMsgsPer. Then we invoke transferRMS() again,\n\t// which will get another chance to resolve the error; if not we bail there.\n\tif err = transferRMS(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Now, if the stream does not have MaxMsgsPer set to 1, and there are no\n\t// more messages on the single $MQTT.rmsgs subject, update the stream again.\n\tif si.Config.MaxMsgsPer != 1 {\n\t\tsi.Config.MaxMsgsPer = 1\n\t\t// We will need an up-to-date si, so don't use local variable here.\n\t\tif si, err = jsa.updateStream(&si.Config); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to update stream config: %w\", err)\n\t\t}\n\t}\n\n\t// If we failed the first time, there is now at most one lingering message\n\t// in the old subject. Try again (it will be a NO-OP if succeeded the first\n\t// time).\n\tif err = transferRMS(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Opportunistically delete the old (legacy) consumer, from v2.10.10 and\n\t// before. Ignore any errors that might arise.\n\trmLegacyDurName := mqttRetainedMsgsStreamName + \"_\" + jsa.id\n\tjsa.deleteConsumer(mqttRetainedMsgsStreamName, rmLegacyDurName, true)\n\n\t// Create a new, uniquely names consumer for retained messages for this\n\t// server. The prior one will expire eventually.\n\tccfg := &CreateConsumerRequest{\n\t\tStream: mqttRetainedMsgsStreamName,\n\t\tConfig: ConsumerConfig{\n\t\t\tName: mqttRetainedMsgsStreamName + \"_\" + nuid.Next(),\n\t\t\tFilterSubject: mqttRetainedMsgsStreamSubject + \">\",\n\t\t\tDeliverSubject: rmsubj,\n\t\t\tReplayPolicy: ReplayInstant,\n\t\t\tAckPolicy: AckNone,\n\t\t\tInactiveThreshold: 5 * time.Minute,\n\t\t},\n\t}\n\tif _, err := jsa.createEphemeralConsumer(ccfg); err != nil {\n\t\treturn nil, fmt.Errorf(\"create retained messages consumer for account %q: %v\", accName, err)\n\t}\n\n\t// Set this so that on defer we don't cleanup.\n\tsuccess = true\n\n\treturn as, nil\n}\n\nfunc (s *Server) mqttDetermineReplicas() int {\n\t// If not clustered, then replica will be 1.\n\tif !s.JetStreamIsClustered() {\n\t\treturn 1\n\t}\n\topts := s.getOpts()\n\treplicas := 0\n\tfor _, u := range opts.Routes {\n\t\thost := u.Hostname()\n\t\t// If this is an IP just add one.\n\t\tif net.ParseIP(host) != nil {\n\t\t\treplicas++\n\t\t} else {\n\t\t\taddrs, _ := net.LookupHost(host)\n\t\t\treplicas += len(addrs)\n\t\t}\n\t}\n\tif replicas < 1 {\n\t\treplicas = 1\n\t} else if replicas > 3 {\n\t\treplicas = 3\n\t}\n\treturn replicas\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// JS APIs related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (jsa *mqttJSA) newRequest(kind, subject string, hdr int, msg []byte) (any, error) {\n\treturn jsa.newRequestEx(kind, subject, _EMPTY_, hdr, msg)\n}\n\nfunc (jsa *mqttJSA) prefixDomain(subject string) string {\n\tif jsa.domain != _EMPTY_ {\n\t\t// rewrite js api prefix with domain\n\t\tif sub := strings.TrimPrefix(subject, JSApiPrefix+\".\"); sub != subject {\n\t\t\tsubject = fmt.Sprintf(\"$JS.%s.API.%s\", jsa.domain, sub)\n\t\t}\n\t}\n\treturn subject\n}\n\nfunc (jsa *mqttJSA) newRequestEx(kind, subject, cidHash string, hdr int, msg []byte) (any, error) {\n\tresponses, err := jsa.newRequestExMulti(kind, subject, cidHash, []int{hdr}, [][]byte{msg})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(responses) != 1 {\n\t\treturn nil, fmt.Errorf(\"unreachable: invalid number of responses (%d)\", len(responses))\n\t}\n\treturn responses[0].value, nil\n}\n\n// newRequestExMulti sends multiple messages on the same subject and waits for\n// all responses. It returns the same number of responses in the same order as\n// msgs parameter. In case of a timeout it returns an error as well as all\n// responses received as a sparsely populated array, matching msgs, with nils\n// for the values that have not yet been received.\n//\n// Note that each response may represent an error and should be inspected as\n// such by the caller.\nfunc (jsa *mqttJSA) newRequestExMulti(kind, subject, cidHash string, hdrs []int, msgs [][]byte) ([]*mqttJSAResponse, error) {\n\tif len(hdrs) != len(msgs) {\n\t\treturn nil, fmt.Errorf(\"unreachable: invalid number of messages (%d) or header offsets (%d)\", len(msgs), len(hdrs))\n\t}\n\tresponseCh := make(chan *mqttJSAResponse, len(msgs))\n\n\t// Generate and queue all outgoing requests, have all results reported to\n\t// responseCh, and store a map of reply subjects to the original subjects'\n\t// indices.\n\tr2i := map[string]int{}\n\tfor i, msg := range msgs {\n\t\thdr := hdrs[i]\n\t\tvar sb strings.Builder\n\t\t// Either we use nuid.Next() which uses a global lock, or our own nuid object, but\n\t\t// then it needs to be \"write\" protected. This approach will reduce across account\n\t\t// contention since we won't use the global nuid's lock.\n\t\tjsa.mu.Lock()\n\t\tuid := jsa.nuid.Next()\n\t\tsb.WriteString(jsa.rplyr)\n\t\tjsa.mu.Unlock()\n\n\t\tsb.WriteString(kind)\n\t\tsb.WriteByte(btsep)\n\t\tif cidHash != _EMPTY_ {\n\t\t\tsb.WriteString(cidHash)\n\t\t\tsb.WriteByte(btsep)\n\t\t}\n\t\tsb.WriteString(uid)\n\t\treply := sb.String()\n\n\t\t// Add responseCh to the reply channel map. It will be cleaned out on\n\t\t// timeout (see below), or in processJSAPIReplies upon receiving the\n\t\t// response.\n\t\tjsa.replies.Store(reply, responseCh)\n\n\t\tsubject = jsa.prefixDomain(subject)\n\t\tjsa.sendq.push(&mqttJSPubMsg{\n\t\t\tsubj: subject,\n\t\t\treply: reply,\n\t\t\thdr: hdr,\n\t\t\tmsg: msg,\n\t\t})\n\t\tr2i[reply] = i\n\t}\n\n\t// Wait for all responses to come back, or for the timeout to expire. We\n\t// don't want to use time.After() which causes memory growth because the\n\t// timer can't be stopped and will need to expire to then be garbage\n\t// collected.\n\tc := 0\n\tresponses := make([]*mqttJSAResponse, len(msgs))\n\tstart := time.Now()\n\tt := time.NewTimer(jsa.timeout)\n\tdefer t.Stop()\n\tfor {\n\t\tselect {\n\t\tcase r := <-responseCh:\n\t\t\ti := r2i[r.reply]\n\t\t\tresponses[i] = r\n\t\t\tc++\n\t\t\tif c == len(msgs) {\n\t\t\t\treturn responses, nil\n\t\t\t}\n\n\t\tcase <-jsa.quitCh:\n\t\t\treturn nil, ErrServerNotRunning\n\n\t\tcase <-t.C:\n\t\t\tvar reply string\n\t\t\tnow := time.Now()\n\t\t\tfor reply = range r2i { // preserve the last value for Errorf\n\t\t\t\tjsa.replies.Delete(reply)\n\t\t\t}\n\n\t\t\tif len(msgs) == 1 {\n\t\t\t\treturn responses, fmt.Errorf(\"timeout after %v: request type %q on %q (reply=%q)\", now.Sub(start), kind, subject, reply)\n\t\t\t} else {\n\t\t\t\treturn responses, fmt.Errorf(\"timeout after %v: request type %q on %q: got %d out of %d\", now.Sub(start), kind, subject, c, len(msgs))\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (jsa *mqttJSA) sendAck(ackSubject string) {\n\t// Send to the ack subject with no payload.\n\tjsa.sendMsg(ackSubject, nil)\n}\n\nfunc (jsa *mqttJSA) sendMsg(subj string, msg []byte) {\n\tif subj == _EMPTY_ {\n\t\treturn\n\t}\n\t// We pass -1 for the hdr so that the send loop does not need to\n\t// add the \"client info\" header. This is not a JS API request per se.\n\tjsa.sendq.push(&mqttJSPubMsg{subj: subj, msg: msg, hdr: -1})\n}\n\nfunc (jsa *mqttJSA) createEphemeralConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) {\n\tcfgb, err := json.Marshal(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsubj := fmt.Sprintf(JSApiConsumerCreateT, cfg.Stream)\n\tccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tccr := ccri.(*JSApiConsumerCreateResponse)\n\treturn ccr, ccr.ToError()\n}\n\nfunc (jsa *mqttJSA) createDurableConsumer(cfg *CreateConsumerRequest) (*JSApiConsumerCreateResponse, error) {\n\tcfgb, err := json.Marshal(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsubj := fmt.Sprintf(JSApiDurableCreateT, cfg.Stream, cfg.Config.Durable)\n\tccri, err := jsa.newRequest(mqttJSAConsumerCreate, subj, 0, cfgb)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tccr := ccri.(*JSApiConsumerCreateResponse)\n\treturn ccr, ccr.ToError()\n}\n\n// if noWait is specified, does not wait for the JS response, returns nil\nfunc (jsa *mqttJSA) deleteConsumer(streamName, consName string, noWait bool) (*JSApiConsumerDeleteResponse, error) {\n\tsubj := fmt.Sprintf(JSApiConsumerDeleteT, streamName, consName)\n\tif noWait {\n\t\tjsa.sendMsg(subj, nil)\n\t\treturn nil, nil\n\t}\n\n\tcdri, err := jsa.newRequest(mqttJSAConsumerDel, subj, 0, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcdr := cdri.(*JSApiConsumerDeleteResponse)\n\treturn cdr, cdr.ToError()\n}\n\nfunc (jsa *mqttJSA) createStream(cfg *StreamConfig) (*StreamInfo, bool, error) {\n\tcfgb, err := json.Marshal(cfg)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\tscri, err := jsa.newRequest(mqttJSAStreamCreate, fmt.Sprintf(JSApiStreamCreateT, cfg.Name), 0, cfgb)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\tscr := scri.(*JSApiStreamCreateResponse)\n\treturn scr.StreamInfo, scr.DidCreate, scr.ToError()\n}\n\nfunc (jsa *mqttJSA) updateStream(cfg *StreamConfig) (*StreamInfo, error) {\n\tcfgb, err := json.Marshal(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tscri, err := jsa.newRequest(mqttJSAStreamUpdate, fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), 0, cfgb)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tscr := scri.(*JSApiStreamUpdateResponse)\n\treturn scr.StreamInfo, scr.ToError()\n}\n\nfunc (jsa *mqttJSA) lookupStream(name string) (*StreamInfo, error) {\n\tslri, err := jsa.newRequest(mqttJSAStreamLookup, fmt.Sprintf(JSApiStreamInfoT, name), 0, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tslr := slri.(*JSApiStreamInfoResponse)\n\treturn slr.StreamInfo, slr.ToError()\n}\n\nfunc (jsa *mqttJSA) deleteStream(name string) (bool, error) {\n\tsdri, err := jsa.newRequest(mqttJSAStreamDel, fmt.Sprintf(JSApiStreamDeleteT, name), 0, nil)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tsdr := sdri.(*JSApiStreamDeleteResponse)\n\treturn sdr.Success, sdr.ToError()\n}\n\nfunc (jsa *mqttJSA) loadLastMsgFor(streamName string, subject string) (*StoredMsg, error) {\n\tmreq := &JSApiMsgGetRequest{LastFor: subject}\n\treq, err := json.Marshal(mreq)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmr := lmri.(*JSApiMsgGetResponse)\n\treturn lmr.Message, lmr.ToError()\n}\n\nfunc (jsa *mqttJSA) loadLastMsgForMulti(streamName string, subjects []string) ([]*JSApiMsgGetResponse, error) {\n\tmarshaled := make([][]byte, 0, len(subjects))\n\theaderBytes := make([]int, 0, len(subjects))\n\tfor _, subject := range subjects {\n\t\tmreq := &JSApiMsgGetRequest{LastFor: subject}\n\t\tbb, err := json.Marshal(mreq)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmarshaled = append(marshaled, bb)\n\t\theaderBytes = append(headerBytes, 0)\n\t}\n\n\tall, err := jsa.newRequestExMulti(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), _EMPTY_, headerBytes, marshaled)\n\t// all has the same order as subjects, preserve it as we unmarshal\n\tresponses := make([]*JSApiMsgGetResponse, len(all))\n\tfor i, v := range all {\n\t\tif v != nil {\n\t\t\tresponses[i] = v.value.(*JSApiMsgGetResponse)\n\t\t}\n\t}\n\treturn responses, err\n}\n\nfunc (jsa *mqttJSA) loadNextMsgFor(streamName string, subject string) (*StoredMsg, error) {\n\tmreq := &JSApiMsgGetRequest{NextFor: subject}\n\treq, err := json.Marshal(mreq)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmr := lmri.(*JSApiMsgGetResponse)\n\treturn lmr.Message, lmr.ToError()\n}\n\nfunc (jsa *mqttJSA) loadMsg(streamName string, seq uint64) (*StoredMsg, error) {\n\tmreq := &JSApiMsgGetRequest{Seq: seq}\n\treq, err := json.Marshal(mreq)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmri, err := jsa.newRequest(mqttJSAMsgLoad, fmt.Sprintf(JSApiMsgGetT, streamName), 0, req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlmr := lmri.(*JSApiMsgGetResponse)\n\treturn lmr.Message, lmr.ToError()\n}\n\nfunc (jsa *mqttJSA) storeMsgNoWait(subject string, hdrLen int, msg []byte) {\n\tjsa.sendq.push(&mqttJSPubMsg{\n\t\tsubj: subject,\n\t\tmsg: msg,\n\t\thdr: hdrLen,\n\t})\n}\n\nfunc (jsa *mqttJSA) storeMsg(subject string, headers int, msg []byte) (*JSPubAckResponse, error) {\n\tsmri, err := jsa.newRequest(mqttJSAMsgStore, subject, headers, msg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsmr := smri.(*JSPubAckResponse)\n\treturn smr, smr.ToError()\n}\n\nfunc (jsa *mqttJSA) storeSessionMsg(domainTk, cidHash string, hdr int, msg []byte) (*JSPubAckResponse, error) {\n\t// Compute subject where the session is being stored\n\tsubject := mqttSessStreamSubjectPrefix + domainTk + cidHash\n\n\t// Passing cidHash will add it to the JS reply subject, so that we can use\n\t// it in processSessionPersist.\n\tsmri, err := jsa.newRequestEx(mqttJSASessPersist, subject, cidHash, hdr, msg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsmr := smri.(*JSPubAckResponse)\n\treturn smr, smr.ToError()\n}\n\nfunc (jsa *mqttJSA) loadSessionMsg(domainTk, cidHash string) (*StoredMsg, error) {\n\tsubject := mqttSessStreamSubjectPrefix + domainTk + cidHash\n\treturn jsa.loadLastMsgFor(mqttSessStreamName, subject)\n}\n\nfunc (jsa *mqttJSA) deleteMsg(stream string, seq uint64, wait bool) error {\n\tdreq := JSApiMsgDeleteRequest{Seq: seq, NoErase: true}\n\treq, _ := json.Marshal(dreq)\n\tsubj := jsa.prefixDomain(fmt.Sprintf(JSApiMsgDeleteT, stream))\n\tif !wait {\n\t\tjsa.sendq.push(&mqttJSPubMsg{\n\t\t\tsubj: subj,\n\t\t\tmsg: req,\n\t\t})\n\t\treturn nil\n\t}\n\tdmi, err := jsa.newRequest(mqttJSAMsgDelete, subj, 0, req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdm := dmi.(*JSApiMsgDeleteResponse)\n\treturn dm.ToError()\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// Account Sessions Manager related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\n// Returns true if `err` is not nil and does not match the api error with ErrorIdentifier id\nfunc isErrorOtherThan(err error, id ErrorIdentifier) bool {\n\treturn err != nil && !IsNatsErr(err, id)\n}\n\n// Process JS API replies.\n//\n// Can run from various go routines (consumer's loop, system send loop, etc..).\nfunc (as *mqttAccountSessionManager) processJSAPIReplies(_ *subscription, pc *client, _ *Account, subject, _ string, msg []byte) {\n\ttoken := tokenAt(subject, mqttJSATokenPos)\n\tif token == _EMPTY_ {\n\t\treturn\n\t}\n\tjsa := &as.jsa\n\tchi, ok := jsa.replies.Load(subject)\n\tif !ok {\n\t\treturn\n\t}\n\tjsa.replies.Delete(subject)\n\tch := chi.(chan *mqttJSAResponse)\n\tout := func(value any) {\n\t\tch <- &mqttJSAResponse{reply: subject, value: value}\n\t}\n\tswitch token {\n\tcase mqttJSAStreamCreate:\n\t\tvar resp = &JSApiStreamCreateResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAStreamUpdate:\n\t\tvar resp = &JSApiStreamUpdateResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAStreamLookup:\n\t\tvar resp = &JSApiStreamInfoResponse{}\n\t\tif err := json.Unmarshal(msg, &resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAStreamDel:\n\t\tvar resp = &JSApiStreamDeleteResponse{}\n\t\tif err := json.Unmarshal(msg, &resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAConsumerCreate:\n\t\tvar resp = &JSApiConsumerCreateResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAConsumerDel:\n\t\tvar resp = &JSApiConsumerDeleteResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAMsgStore, mqttJSASessPersist:\n\t\tvar resp = &JSPubAckResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAMsgLoad:\n\t\tvar resp = &JSApiMsgGetResponse{}\n\t\tif err := json.Unmarshal(msg, &resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAStreamNames:\n\t\tvar resp = &JSApiStreamNamesResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tcase mqttJSAMsgDelete:\n\t\tvar resp = &JSApiMsgDeleteResponse{}\n\t\tif err := json.Unmarshal(msg, resp); err != nil {\n\t\t\tresp.Error = NewJSInvalidJSONError(err)\n\t\t}\n\t\tout(resp)\n\tdefault:\n\t\tpc.Warnf(\"Unknown reply code %q\", token)\n\t}\n}\n\n// This will both load all retained messages and process updates from the cluster.\n//\n// Run from various go routines (JS consumer, etc..).\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) processRetainedMsg(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\th, m := c.msgParts(rmsg)\n\t// We need to strip the trailing \"\\r\\n\".\n\tif l := len(m); l >= LEN_CR_LF {\n\t\tm = m[:l-LEN_CR_LF]\n\t}\n\trm, err := mqttDecodeRetainedMessage(subject, h, m)\n\tif err != nil {\n\t\treturn\n\t}\n\t// The as.jsa.id is immutable, so no need to have a rlock here.\n\tlocal := rm.Origin == as.jsa.id\n\t// Get the stream sequence for this message.\n\tseq, _, _ := ackReplyInfo(reply)\n\tif len(m) == 0 {\n\t\t// An empty payload means that we need to remove the retained message.\n\t\trmSeq := as.removeRetainedMsg(rm.Subject, 0)\n\t\tif local {\n\t\t\tif rmSeq > 0 {\n\t\t\t\t// This is for backward compatibility reasons.\n\t\t\t\t// Should be removed in a future release.\n\t\t\t\tas.notifyRetainedMsgDeleted(rm.Subject, rmSeq)\n\t\t\t}\n\t\t\t// Delete this very message we just processed, we don't need it anymore.\n\t\t\tas.deleteRetainedMsg(seq)\n\t\t}\n\t} else {\n\t\t// Add this retained message. The `rm.Msg` references some buffer that we\n\t\t// don't own. But addRetainedMsg() will take care of making a copy of\n\t\t// `rm.Msg` it `rm` ends-up being stored in the cache.\n\t\tas.addRetainedMsg(rm.Subject, &mqttRetainedMsgRef{sseq: seq}, rm)\n\t}\n}\n\n// NOTE: This is maintained for backward compatibility reasons. Should be removed in 2.14/2.15?\nfunc (as *mqttAccountSessionManager) processRetainedMsgDel(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tidHash := tokenAt(subject, 3)\n\tif idHash == _EMPTY_ || idHash == as.jsa.id {\n\t\treturn\n\t}\n\t_, msg := c.msgParts(rmsg)\n\tif len(msg) < LEN_CR_LF {\n\t\treturn\n\t}\n\tvar drm mqttRetMsgDel\n\tif err := json.Unmarshal(msg, &drm); err != nil {\n\t\treturn\n\t}\n\tas.removeRetainedMsg(drm.Subject, drm.Seq)\n}\n\n// This will receive all JS API replies for a request to store a session record,\n// including the reply for our own server, which we will ignore.\n// This allows us to detect that some application somewhere else in the cluster\n// is connecting with the same client ID, and therefore we need to close the\n// connection that is currently using this client ID.\n//\n// Can run from various go routines (system send loop, etc..).\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) processSessionPersist(_ *subscription, pc *client, _ *Account, subject, _ string, rmsg []byte) {\n\t// Ignore our own responses here (they are handled elsewhere)\n\tif tokenAt(subject, mqttJSAIdTokenPos) == as.jsa.id {\n\t\treturn\n\t}\n\tcIDHash := tokenAt(subject, mqttJSAClientIDPos)\n\t_, msg := pc.msgParts(rmsg)\n\tif len(msg) < LEN_CR_LF {\n\t\treturn\n\t}\n\tvar par = &JSPubAckResponse{}\n\tif err := json.Unmarshal(msg, par); err != nil {\n\t\treturn\n\t}\n\tif err := par.Error; err != nil {\n\t\treturn\n\t}\n\tas.mu.RLock()\n\t// Note that as.domainTk includes a terminal '.', so strip to compare to PubAck.Domain.\n\tdl := len(as.domainTk)\n\tif dl > 0 {\n\t\tdl--\n\t}\n\tignore := par.Domain != as.domainTk[:dl]\n\tas.mu.RUnlock()\n\tif ignore {\n\t\treturn\n\t}\n\n\tas.mu.Lock()\n\tdefer as.mu.Unlock()\n\tsess, ok := as.sessByHash[cIDHash]\n\tif !ok {\n\t\treturn\n\t}\n\t// If our current session's stream sequence is higher, it means that this\n\t// update is stale, so we don't do anything here.\n\tsess.mu.Lock()\n\tignore = par.Sequence < sess.seq\n\tsess.mu.Unlock()\n\tif ignore {\n\t\treturn\n\t}\n\tas.removeSession(sess, false)\n\tsess.mu.Lock()\n\tif ec := sess.c; ec != nil {\n\t\tas.addSessToFlappers(sess.id)\n\t\tec.Warnf(\"Closing because a remote connection has started with the same client ID: %q\", sess.id)\n\t\t// Disassociate the client from the session so that on client close,\n\t\t// nothing will be done with regards to cleaning up the session,\n\t\t// such as deleting stream, etc..\n\t\tsess.c = nil\n\t\t// Remove in separate go routine.\n\t\tgo ec.closeConnection(DuplicateClientID)\n\t}\n\tsess.mu.Unlock()\n}\n\n// Adds this client ID to the flappers map, and if needed start the timer\n// for map cleanup.\n//\n// Lock held on entry.\nfunc (as *mqttAccountSessionManager) addSessToFlappers(clientID string) {\n\tas.flappers[clientID] = time.Now().UnixNano()\n\tif as.flapTimer == nil {\n\t\tas.flapTimer = time.AfterFunc(mqttFlapCleanItvl, func() {\n\t\t\tas.mu.Lock()\n\t\t\tdefer as.mu.Unlock()\n\t\t\t// In case of shutdown, this will be nil\n\t\t\tif as.flapTimer == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnow := time.Now().UnixNano()\n\t\t\tfor cID, tm := range as.flappers {\n\t\t\t\tif now-tm > int64(mqttSessJailDur) {\n\t\t\t\t\tdelete(as.flappers, cID)\n\t\t\t\t}\n\t\t\t}\n\t\t\tas.flapTimer.Reset(mqttFlapCleanItvl)\n\t\t})\n\t}\n}\n\n// Remove this client ID from the flappers map.\n//\n// Lock held on entry.\nfunc (as *mqttAccountSessionManager) removeSessFromFlappers(clientID string) {\n\tdelete(as.flappers, clientID)\n\t// Do not stop/set timer to nil here. Better leave the timer run at its\n\t// regular interval and detect that there is nothing to do. The timer\n\t// will be stopped on shutdown.\n}\n\n// Helper to create a subscription. It updates the sid and array of subscriptions.\nfunc (as *mqttAccountSessionManager) createSubscription(subject string, cb msgHandler, sid *int64, subs *[]*subscription) error {\n\tsub, err := as.jsa.c.processSub([]byte(subject), nil, []byte(strconv.FormatInt(*sid, 10)), cb, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\t*sid++\n\t*subs = append(*subs, sub)\n\treturn nil\n}\n\n// A timer loop to cleanup up expired cached retained messages for a given MQTT account.\n// The closeCh is used by the caller to be able to interrupt this routine\n// if the rest of the initialization fails, since the quitCh is really\n// only used when the server shutdown.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) cleanupRetainedMessageCache(s *Server, closeCh chan struct{}) {\n\ttt := time.NewTicker(mqttRetainedCacheTTL)\n\tdefer tt.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-tt.C:\n\t\t\t// Set a limit to the number of retained messages to scan since we\n\t\t\t// lock as for it. Since the map enumeration gives random order we\n\t\t\t// should eventually clean up everything.\n\t\t\ti, maxScan := 0, 10*1000\n\t\t\tnow := time.Now()\n\t\t\tas.rmsCache.Range(func(key, value any) bool {\n\t\t\t\trm := value.(*mqttRetainedMsg)\n\t\t\t\tif now.After(rm.expiresFromCache) {\n\t\t\t\t\tas.rmsCache.Delete(key)\n\t\t\t\t}\n\t\t\t\ti++\n\t\t\t\treturn i < maxScan\n\t\t\t})\n\n\t\tcase <-closeCh:\n\t\t\treturn\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Loop to send JS API requests for a given MQTT account.\n// The closeCh is used by the caller to be able to interrupt this routine\n// if the rest of the initialization fails, since the quitCh is really\n// only used when the server shutdown.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) sendJSAPIrequests(s *Server, c *client, accName string, closeCh chan struct{}) {\n\tvar cluster string\n\tif s.JetStreamEnabled() && !as.jsa.domainSet {\n\t\t// Only request the own cluster when it is clear that\n\t\tcluster = s.cachedClusterName()\n\t}\n\tas.mu.RLock()\n\tsendq := as.jsa.sendq\n\tquitCh := as.jsa.quitCh\n\tci := ClientInfo{Account: accName, Cluster: cluster}\n\tacc := c.acc\n\tas.mu.RUnlock()\n\n\t// The account session manager does not have a suhtdown API per-se, instead,\n\t// we will cleanup things when this go routine exits after detecting that the\n\t// server is shutdown or the initialization of the account manager failed.\n\tdefer func() {\n\t\tas.mu.Lock()\n\t\tif as.flapTimer != nil {\n\t\t\tas.flapTimer.Stop()\n\t\t\tas.flapTimer = nil\n\t\t}\n\t\tas.mu.Unlock()\n\t}()\n\n\tb, _ := json.Marshal(ci)\n\thdrStart := bytes.Buffer{}\n\thdrStart.WriteString(hdrLine)\n\thttp.Header{ClientInfoHdr: []string{string(b)}}.Write(&hdrStart)\n\thdrStart.WriteString(CR_LF)\n\thdrStart.WriteString(CR_LF)\n\thdrb := hdrStart.Bytes()\n\n\tfor {\n\t\tselect {\n\t\tcase <-sendq.ch:\n\t\t\tpmis := sendq.pop()\n\t\t\tfor _, r := range pmis {\n\t\t\t\tvar nsize int\n\n\t\t\t\tmsg := r.msg\n\t\t\t\t// If r.hdr is set to -1, it means that there is no need for any header.\n\t\t\t\tif r.hdr != -1 {\n\t\t\t\t\tbb := bytes.Buffer{}\n\t\t\t\t\tif r.hdr > 0 {\n\t\t\t\t\t\t// This means that the header has been set by the caller and is\n\t\t\t\t\t\t// already part of `msg`, so simply set c.pa.hdr to the given value.\n\t\t\t\t\t\tc.pa.hdr = r.hdr\n\t\t\t\t\t\tnsize = len(msg)\n\t\t\t\t\t\tmsg = append(msg, _CRLF_...)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// We need the ClientInfo header, so add it here.\n\t\t\t\t\t\tbb.Write(hdrb)\n\t\t\t\t\t\tc.pa.hdr = bb.Len()\n\t\t\t\t\t\tbb.Write(r.msg)\n\t\t\t\t\t\tnsize = bb.Len()\n\t\t\t\t\t\tbb.WriteString(_CRLF_)\n\t\t\t\t\t\tmsg = bb.Bytes()\n\t\t\t\t\t}\n\t\t\t\t\tc.pa.hdb = []byte(strconv.Itoa(c.pa.hdr))\n\t\t\t\t} else {\n\t\t\t\t\tc.pa.hdr = -1\n\t\t\t\t\tc.pa.hdb = nil\n\t\t\t\t\tnsize = len(msg)\n\t\t\t\t\tmsg = append(msg, _CRLF_...)\n\t\t\t\t}\n\n\t\t\t\tc.pa.subject = []byte(r.subj)\n\t\t\t\tc.pa.reply = []byte(r.reply)\n\t\t\t\tc.pa.size = nsize\n\t\t\t\tc.pa.szb = []byte(strconv.Itoa(nsize))\n\t\t\t\tc.pa.mapped = nil\n\n\t\t\t\tif acc.hasMappings() {\n\t\t\t\t\tif changed := c.selectMappedSubject(); changed {\n\t\t\t\t\t\tc.traceOutOp(\"MAPPINGS\", fmt.Appendf(nil, \"%s -> %s\", c.pa.mapped, c.pa.subject))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tc.processInboundClientMsg(msg)\n\t\t\t\tc.flushClients(0)\n\t\t\t}\n\t\t\tsendq.recycle(&pmis)\n\n\t\tcase <-closeCh:\n\t\t\treturn\n\t\tcase <-quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Add/Replace this message from the retained messages map.\n// If a message for this topic already existed, the existing record is updated\n// with the provided information.\n// Lock not held on entry.\nfunc (as *mqttAccountSessionManager) addRetainedMsg(key string, rf *mqttRetainedMsgRef, rm *mqttRetainedMsg) {\n\tas.mu.Lock()\n\tdefer as.mu.Unlock()\n\tif as.retmsgs == nil {\n\t\tas.retmsgs = make(map[string]*mqttRetainedMsgRef)\n\t\tas.sl = NewSublistWithCache()\n\t} else {\n\t\t// Check if we already had one retained message. If so, update the existing one.\n\t\tif erf, exists := as.retmsgs[key]; exists {\n\t\t\t// Update the stream sequence with the new value.\n\t\t\terf.sseq = rf.sseq\n\t\t\t// Update the in-memory retained message cache but only for messages\n\t\t\t// that are already in the cache, i.e. have been (recently) used.\n\t\t\t// If that is the case, we ask setCachedRetainedMsg() to make a copy\n\t\t\t// of rm.Msg bytes slice.\n\t\t\tas.setCachedRetainedMsg(key, rm, true, true)\n\t\t\treturn\n\t\t}\n\t}\n\trf.sub = &subscription{subject: []byte(key)}\n\tas.retmsgs[key] = rf\n\tas.sl.Insert(rf.sub)\n}\n\n// Remove the retained message stored with the `subject` key from the map/cache.\n// When invoked from the retained message stream's consumer, this function will\n// be called with `seq == 0`, this is because add/remove are serialized in this\n// stream and so the request is to remove the current retained message.\n// But in some conditions, we will invoke this function from some other places\n// with `seq > 0` which means that the retained message will be removed only if\n// its sequence is the same than the provided one.\n// This function returns the sequence associated with the existing retained\n// message that is being removed (used with `seq == 0`) and returns 0 if the\n// retained message was not removed from the map (not found or sequence did not\n// match).\nfunc (as *mqttAccountSessionManager) removeRetainedMsg(subject string, seq uint64) uint64 {\n\tas.mu.Lock()\n\tdefer as.mu.Unlock()\n\trm, ok := as.retmsgs[subject]\n\tif !ok || (seq > 0 && rm.sseq != seq) {\n\t\treturn 0\n\t}\n\tseq = rm.sseq\n\tas.rmsCache.Delete(subject)\n\tdelete(as.retmsgs, subject)\n\tas.sl.Remove(rm.sub)\n\treturn seq\n}\n\n// First check if this session's client ID is already in the \"locked\" map,\n// which if it is the case means that another client is now bound to this\n// session and this should return an error.\n// If not in the \"locked\" map, but the client is not bound with this session,\n// then same error is returned.\n// Finally, if all checks ok, then the session's ID is added to the \"locked\" map.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) lockSession(sess *mqttSession, c *client) error {\n\tas.mu.Lock()\n\tdefer as.mu.Unlock()\n\tvar fail bool\n\tif _, fail = as.sessLocked[sess.id]; !fail {\n\t\tsess.mu.Lock()\n\t\tfail = sess.c != c\n\t\tsess.mu.Unlock()\n\t}\n\tif fail {\n\t\treturn fmt.Errorf(\"another session is in use with client ID %q\", sess.id)\n\t}\n\tas.sessLocked[sess.id] = struct{}{}\n\treturn nil\n}\n\n// Remove the session from the \"locked\" map.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) unlockSession(sess *mqttSession) {\n\tas.mu.Lock()\n\tdelete(as.sessLocked, sess.id)\n\tas.mu.Unlock()\n}\n\n// Simply adds the session to the various sessions maps.\n// The boolean `lock` indicates if this function should acquire the lock\n// prior to adding to the maps.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) addSession(sess *mqttSession, lock bool) {\n\tif lock {\n\t\tas.mu.Lock()\n\t}\n\tas.sessions[sess.id] = sess\n\tas.sessByHash[sess.idHash] = sess\n\tif lock {\n\t\tas.mu.Unlock()\n\t}\n}\n\n// Simply removes the session from the various sessions maps.\n// The boolean `lock` indicates if this function should acquire the lock\n// prior to removing from the maps.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) removeSession(sess *mqttSession, lock bool) {\n\tif lock {\n\t\tas.mu.Lock()\n\t}\n\tdelete(as.sessions, sess.id)\n\tdelete(as.sessByHash, sess.idHash)\n\tif lock {\n\t\tas.mu.Unlock()\n\t}\n}\n\n// Helper to set the sub's mqtt fields and possibly serialize (pre-loaded)\n// retained messages.\n//\n// Session lock held on entry. Acquires the subs lock and holds it for\n// the duration. Non-MQTT messages coming into mqttDeliverMsgCbQoS0 will be\n// waiting.\nfunc (sess *mqttSession) processQOS12Sub(\n\tc *client, // subscribing client.\n\tsubject, sid []byte, isReserved bool, qos byte, jsDurName string, h msgHandler, // subscription parameters.\n) (*subscription, error) {\n\treturn sess.processSub(c, subject, sid, isReserved, qos, jsDurName, h, false, nil, false, nil)\n}\n\nfunc (sess *mqttSession) processSub(\n\tc *client, // subscribing client.\n\tsubject, sid []byte, isReserved bool, qos byte, jsDurName string, h msgHandler, // subscription parameters.\n\tinitShadow bool, // do we need to scan for shadow subscriptions? (not for QOS1+)\n\trms map[string]*mqttRetainedMsg, // preloaded rms (can be empty, or missing items if errors)\n\ttrace bool, // trace serialized retained messages in the log?\n\tas *mqttAccountSessionManager, // needed only for rms serialization.\n) (*subscription, error) {\n\tstart := time.Now()\n\tdefer func() {\n\t\telapsed := time.Since(start)\n\t\tif elapsed > mqttProcessSubTooLong {\n\t\t\tc.Warnf(\"Took too long to process subscription for %q: %v\", subject, elapsed)\n\t\t}\n\t}()\n\n\t// Hold subsMu to prevent QOS0 messages callback from doing anything until\n\t// the (MQTT) sub is initialized.\n\tsess.subsMu.Lock()\n\tdefer sess.subsMu.Unlock()\n\n\tsub, err := c.processSub(subject, nil, sid, h, false)\n\tif err != nil {\n\t\t// c.processSub already called c.Errorf(), so no need here.\n\t\treturn nil, err\n\t}\n\tsubs := []*subscription{sub}\n\tif initShadow {\n\t\tsubs = append(subs, sub.shadow...)\n\t}\n\tfor _, ss := range subs {\n\t\tif ss.mqtt == nil {\n\t\t\t// reserved is set only once and once the subscription has been\n\t\t\t// created it can be considered immutable.\n\t\t\tss.mqtt = &mqttSub{\n\t\t\t\treserved: isReserved,\n\t\t\t}\n\t\t}\n\t\t// QOS and jsDurName can be changed on an existing subscription, so\n\t\t// accessing it later requires a lock.\n\t\tss.mqtt.qos = qos\n\t\tss.mqtt.jsDur = jsDurName\n\t}\n\n\tif len(rms) > 0 {\n\t\t// Only deal with retained messages for the normal subscription,\n\t\t// not the shadow one (which is for a different account and subject).\n\t\tas.serializeRetainedMsgsForSub(rms, sess, c, sub, trace)\n\t}\n\n\treturn sub, nil\n}\n\n// Process subscriptions for the given session/client.\n//\n// When `fromSubProto` is false, it means that this is invoked from the CONNECT\n// protocol, when restoring subscriptions that were saved for this session.\n// In that case, there is no need to update the session record.\n//\n// When `fromSubProto` is true, it means that this call is invoked from the\n// processing of the SUBSCRIBE protocol, which means that the session needs to\n// be updated. It also means that if a subscription on same subject with same\n// QoS already exist, we should not be recreating the subscription/JS durable,\n// since it was already done when processing the CONNECT protocol.\n//\n// Runs from the client's readLoop.\n// Lock not held on entry, but session is in the locked map.\nfunc (as *mqttAccountSessionManager) processSubs(sess *mqttSession, c *client,\n\tfilters []*mqttFilter, fromSubProto, trace bool) ([]*subscription, error) {\n\n\t// Helper to determine if we need to create a separate top-level\n\t// subscription for a wildcard.\n\tfwc := func(subject string) (bool, string, string) {\n\t\tif !mqttNeedSubForLevelUp(subject) {\n\t\t\treturn false, _EMPTY_, _EMPTY_\n\t\t}\n\t\t// Say subject is \"foo.>\", remove the \".>\" so that it becomes \"foo\"\n\t\tfwcsubject := subject[:len(subject)-2]\n\t\t// Change the sid to \"foo fwc\"\n\t\tfwcsid := fwcsubject + mqttMultiLevelSidSuffix\n\n\t\treturn true, fwcsubject, fwcsid\n\t}\n\n\trmSubjects := map[string]uint64{}\n\t// Preload retained messages for all requested subscriptions. Also, since\n\t// it's the first iteration over the filter list, do some cleanup.\n\tfor _, f := range filters {\n\t\tif f.qos > 2 {\n\t\t\tf.qos = 2\n\t\t}\n\t\tif c.mqtt.downgradeQoS2Sub && f.qos == 2 {\n\t\t\tc.Warnf(\"Downgrading subscription QoS2 to QoS1 for %q, as configured\", f.filter)\n\t\t\tf.qos = 1\n\t\t}\n\n\t\t// Do not allow subscribing to our internal subjects.\n\t\t//\n\t\t// TODO: (levb: not sure why since one can subscribe to `#` and it'll\n\t\t// include everything; I guess this would discourage? Otherwise another\n\t\t// candidate for DO NOT DELIVER prefix list).\n\t\tif strings.HasPrefix(f.filter, mqttSubPrefix) {\n\t\t\tf.qos = mqttSubAckFailure\n\t\t\tcontinue\n\t\t}\n\n\t\tif f.qos == 2 {\n\t\t\tif err := sess.ensurePubRelConsumerSubscription(c); err != nil {\n\t\t\t\tc.Errorf(\"failed to initialize PUBREL processing: %v\", err)\n\t\t\t\tf.qos = mqttSubAckFailure\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// Find retained messages.\n\t\tif fromSubProto {\n\t\t\tas.addRetainedSubjectsForSubject(rmSubjects, f.filter)\n\t\t\tif need, subject, _ := fwc(f.filter); need {\n\t\t\t\tas.addRetainedSubjectsForSubject(rmSubjects, subject)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar rms map[string]*mqttRetainedMsg\n\tif len(rmSubjects) > 0 {\n\t\t// Make the best effort to load retained messages.\n\t\trms = as.loadRetainedMessages(rmSubjects, c)\n\t}\n\n\t// Small helper to add the consumer config to the session.\n\taddJSConsToSess := func(sid string, cc *ConsumerConfig) {\n\t\tif cc == nil {\n\t\t\treturn\n\t\t}\n\t\tif sess.cons == nil {\n\t\t\tsess.cons = make(map[string]*ConsumerConfig)\n\t\t}\n\t\tsess.cons[sid] = cc\n\t}\n\n\tvar err error\n\tsubs := make([]*subscription, 0, len(filters))\n\tfor _, f := range filters {\n\t\t// Skip what's already been identified as a failure.\n\t\tif f.qos == mqttSubAckFailure {\n\t\t\tcontinue\n\t\t}\n\t\tsubject := f.filter\n\t\tbsubject := []byte(subject)\n\t\tsid := subject\n\t\tbsid := bsubject\n\t\tisReserved := isMQTTReservedSubscription(subject)\n\n\t\tvar jscons *ConsumerConfig\n\t\tvar jssub *subscription\n\n\t\t// Note that if a subscription already exists on this subject, the\n\t\t// existing sub is returned. Need to update the qos.\n\t\tvar sub *subscription\n\t\tvar err error\n\n\t\tconst processShadowSubs = true\n\n\t\tas.mu.Lock()\n\t\tsess.mu.Lock()\n\t\tsub, err = sess.processSub(c,\n\t\t\tbsubject, bsid, isReserved, f.qos, // main subject\n\t\t\t_EMPTY_, mqttDeliverMsgCbQoS0, // no jsDur for QOS0\n\t\t\tprocessShadowSubs,\n\t\t\trms, trace, as)\n\t\tsess.mu.Unlock()\n\t\tas.mu.Unlock()\n\n\t\tif err != nil {\n\t\t\tf.qos = mqttSubAckFailure\n\t\t\tsess.cleanupFailedSub(c, sub, jscons, jssub)\n\t\t\tcontinue\n\t\t}\n\n\t\t// This will create (if not already exist) a JS consumer for\n\t\t// subscriptions of QoS >= 1. But if a JS consumer already exists and\n\t\t// the subscription for same subject is now a QoS==0, then the JS\n\t\t// consumer will be deleted.\n\t\tjscons, jssub, err = sess.processJSConsumer(c, subject, sid, f.qos, fromSubProto)\n\t\tif err != nil {\n\t\t\tf.qos = mqttSubAckFailure\n\t\t\tsess.cleanupFailedSub(c, sub, jscons, jssub)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Process the wildcard subject if needed.\n\t\tif need, fwcsubject, fwcsid := fwc(subject); need {\n\t\t\tvar fwjscons *ConsumerConfig\n\t\t\tvar fwjssub *subscription\n\t\t\tvar fwcsub *subscription\n\n\t\t\t// See note above about existing subscription.\n\t\t\tas.mu.Lock()\n\t\t\tsess.mu.Lock()\n\t\t\tfwcsub, err = sess.processSub(c,\n\t\t\t\t[]byte(fwcsubject), []byte(fwcsid), isReserved, f.qos, // FWC (top-level wildcard) subject\n\t\t\t\t_EMPTY_, mqttDeliverMsgCbQoS0, // no jsDur for QOS0\n\t\t\t\tprocessShadowSubs,\n\t\t\t\trms, trace, as)\n\t\t\tsess.mu.Unlock()\n\t\t\tas.mu.Unlock()\n\t\t\tif err != nil {\n\t\t\t\t// c.processSub already called c.Errorf(), so no need here.\n\t\t\t\tf.qos = mqttSubAckFailure\n\t\t\t\tsess.cleanupFailedSub(c, sub, jscons, jssub)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tfwjscons, fwjssub, err = sess.processJSConsumer(c, fwcsubject, fwcsid, f.qos, fromSubProto)\n\t\t\tif err != nil {\n\t\t\t\t// c.processSub already called c.Errorf(), so no need here.\n\t\t\t\tf.qos = mqttSubAckFailure\n\t\t\t\tsess.cleanupFailedSub(c, sub, jscons, jssub)\n\t\t\t\tsess.cleanupFailedSub(c, fwcsub, fwjscons, fwjssub)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tsubs = append(subs, fwcsub)\n\t\t\taddJSConsToSess(fwcsid, fwjscons)\n\t\t}\n\n\t\tsubs = append(subs, sub)\n\t\taddJSConsToSess(sid, jscons)\n\t}\n\n\tif fromSubProto {\n\t\terr = sess.update(filters, true)\n\t}\n\n\treturn subs, err\n}\n\n// Retained publish messages matching this subscription are serialized in the\n// subscription's `prm` mqtt writer. This buffer will be queued for outbound\n// after the subscription is processed and SUBACK is sent or possibly when\n// server processes an incoming published message matching the newly\n// registered subscription.\n//\n// Runs from the client's readLoop.\n// Account session manager lock held on entry.\n// Session lock held on entry.\nfunc (as *mqttAccountSessionManager) serializeRetainedMsgsForSub(rms map[string]*mqttRetainedMsg, sess *mqttSession, c *client, sub *subscription, trace bool) {\n\tif len(as.retmsgs) == 0 || len(rms) == 0 {\n\t\treturn\n\t}\n\tresult := as.sl.ReverseMatch(string(sub.subject))\n\tif len(result.psubs) == 0 {\n\t\treturn\n\t}\n\ttoTrace := []mqttPublish{}\n\tfor _, psub := range result.psubs {\n\n\t\trm := rms[string(psub.subject)]\n\t\tif rm == nil {\n\t\t\t// This should not happen since we pre-load messages into rms before\n\t\t\t// calling serialize.\n\t\t\tcontinue\n\t\t}\n\t\tvar pi uint16\n\t\tqos := min(mqttGetQoS(rm.Flags), sub.mqtt.qos)\n\t\tif c.mqtt.rejectQoS2Pub && qos == 2 {\n\t\t\tc.Warnf(\"Rejecting retained message with QoS2 for subscription %q, as configured\", sub.subject)\n\t\t\tcontinue\n\t\t}\n\t\tif qos > 0 {\n\t\t\tpi = sess.trackPublishRetained()\n\n\t\t\t// If we failed to get a PI for this message, send it as a QoS0, the\n\t\t\t// best we can do?\n\t\t\tif pi == 0 {\n\t\t\t\tqos = 0\n\t\t\t}\n\t\t}\n\n\t\t// Need to use the subject for the retained message, not the `sub` subject.\n\t\t// We can find the published retained message in rm.sub.subject.\n\t\t// Set the RETAIN flag: [MQTT-3.3.1-8].\n\t\tflags, headerBytes := mqttMakePublishHeader(pi, qos, false, true, []byte(rm.Topic), len(rm.Msg))\n\t\tc.mu.Lock()\n\t\tsub.mqtt.prm = append(sub.mqtt.prm, headerBytes, rm.Msg)\n\t\tc.mu.Unlock()\n\t\tif trace {\n\t\t\ttoTrace = append(toTrace, mqttPublish{\n\t\t\t\ttopic: []byte(rm.Topic),\n\t\t\t\tflags: flags,\n\t\t\t\tpi: pi,\n\t\t\t\tsz: len(rm.Msg),\n\t\t\t})\n\t\t}\n\t}\n\tfor _, pp := range toTrace {\n\t\tc.traceOutOp(\"PUBLISH\", []byte(mqttPubTrace(&pp)))\n\t}\n}\n\n// Appends the stored message subjects for all retained message records that\n// match the given subscription's `subject` (which could have wildcards).\n//\n// Account session manager NOT lock held on entry.\nfunc (as *mqttAccountSessionManager) addRetainedSubjectsForSubject(list map[string]uint64, topSubject string) {\n\tas.mu.RLock()\n\tif len(as.retmsgs) == 0 {\n\t\tas.mu.RUnlock()\n\t\treturn\n\t}\n\tresult := as.sl.ReverseMatch(topSubject)\n\tas.mu.RUnlock()\n\n\tfor _, sub := range result.psubs {\n\t\tif _, ok := list[string(sub.subject)]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tvar seq uint64\n\t\tas.mu.RLock()\n\t\tif rm, ok := as.retmsgs[string(sub.subject)]; ok {\n\t\t\tseq = rm.sseq\n\t\t}\n\t\tas.mu.RUnlock()\n\t\tif seq > 0 {\n\t\t\tlist[string(sub.subject)] = seq\n\t\t}\n\t}\n}\n\ntype warner interface {\n\tWarnf(format string, v ...any)\n}\n\n// Loads a list of retained messages given a list of stored message subjects.\nfunc (as *mqttAccountSessionManager) loadRetainedMessages(subjects map[string]uint64, w warner) map[string]*mqttRetainedMsg {\n\trms := make(map[string]*mqttRetainedMsg, len(subjects))\n\tss := []string{}\n\tfor s := range subjects {\n\t\tif rm := as.getCachedRetainedMsg(s); rm != nil {\n\t\t\trms[s] = rm\n\t\t} else {\n\t\t\tss = append(ss, mqttRetainedMsgsStreamSubject+s)\n\t\t}\n\t}\n\n\tif len(ss) == 0 {\n\t\treturn rms\n\t}\n\n\t// Although we have the stream sequence for a given subject, we still use\n\t// the load with \"last for subject\" because it will cover the cases where a\n\t// new retained message has arrived since we collected the subject/seq pair.\n\t// If we were doing a load \"by seq\" and the message is not found, we would\n\t// incorrectly remove the retained message from our map.\n\tresults, err := as.jsa.loadLastMsgForMulti(mqttRetainedMsgsStreamName, ss)\n\t// If an error occurred, warn, but then proceed with what we got.\n\tif err != nil {\n\t\tw.Warnf(\"error loading retained messages: %v\", err)\n\t}\n\tfor i, result := range results {\n\t\tif result == nil {\n\t\t\tcontinue // skip requests that timed out\n\t\t}\n\t\tif err := result.ToError(); err != nil {\n\t\t\t// Skip the \"$MQTT.rmsgs.\" prefix...\n\t\t\tsubj := ss[i][len(mqttRetainedMsgsStreamSubject):]\n\t\t\tif IsNatsErr(err, JSNoMessageFoundErr) {\n\t\t\t\t// If there is no message for that subject, delete from our map.\n\t\t\t\t// The good thing here is that we handle the race where a retained\n\t\t\t\t// message may just arrive and be replacing it in the map. The\n\t\t\t\t// removeRetainedMsg() function below will not remove if the sequence\n\t\t\t\t// does not match.\n\t\t\t\tseq := subjects[subj]\n\t\t\t\tas.removeRetainedMsg(subj, seq)\n\t\t\t}\n\t\t\tw.Warnf(\"failed to load retained message for subject %q: %v\", subj, err)\n\t\t\tcontinue\n\t\t}\n\t\trm, err := mqttDecodeRetainedMessage(result.Message.Subject, result.Message.Header, result.Message.Data)\n\t\tif err != nil {\n\t\t\t// Unlikely that we can recover from that, so remove the message.\n\t\t\t// (see comment above if failing to load the message).\n\t\t\tsubj := ss[i][len(mqttRetainedMsgsStreamSubject):]\n\t\t\tseq := subjects[subj]\n\t\t\tas.removeRetainedMsg(subj, seq)\n\t\t\tw.Warnf(\"failed to decode retained message for subject %q: %v\", subj, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Add the loaded retained message to the cache, and to the results map.\n\t\t// We don't need setCachedRetainedMsg() to clone the `rm.Msg` bytes slice\n\t\t// since we own it.\n\t\tas.setCachedRetainedMsg(rm.Subject, rm, false, false)\n\t\trms[rm.Subject] = rm\n\t}\n\treturn rms\n}\n\n// Composes a NATS message for a storeable mqttRetainedMsg.\n// If the body is empty, the flags are encoded in a way that will cause older\n// servers to fail to decode the message in processRetainedMsg callback and\n// will simply ignore it, which is what we want.\nfunc mqttEncodeRetainedMessage(rm *mqttRetainedMsg) (natsMsg []byte, headerLen int) {\n\tdelRM := len(rm.Msg) == 0\n\n\t// No need to encode the subject, we can restore it from topic.\n\tl := len(hdrLine)\n\tl += len(mqttNatsRetainedMessageTopic) + 1 + len(rm.Topic) + 2 // 1 byte for ':', 2 bytes for CRLF\n\tif rm.Origin != _EMPTY_ {\n\t\tl += len(mqttNatsRetainedMessageOrigin) + 1 + len(rm.Origin) + 2 // 1 byte for ':', 2 bytes for CRLF\n\t}\n\tif rm.Source != _EMPTY_ {\n\t\tl += len(mqttNatsRetainedMessageSource) + 1 + len(rm.Source) + 2 // 1 byte for ':', 2 bytes for CRLF\n\t}\n\tl += len(mqttNatsRetainedMessageFlags) + 1 + 2 + 2 // 1 byte for ':', 2 bytes for the flags, 2 bytes for CRLF\n\tl += 2 // 2 bytes for the extra CRLF after the header\n\tif delRM {\n\t\tl++ // Will add the delete marker before the flag\n\t} else {\n\t\tl += len(rm.Msg)\n\t}\n\n\tbuf := bytes.NewBuffer(make([]byte, 0, l))\n\n\tbuf.WriteString(hdrLine)\n\n\tbuf.WriteString(mqttNatsRetainedMessageTopic)\n\tbuf.WriteByte(':')\n\tbuf.WriteString(rm.Topic)\n\tbuf.WriteString(_CRLF_)\n\n\tbuf.WriteString(mqttNatsRetainedMessageFlags)\n\tbuf.WriteByte(':')\n\tif delRM {\n\t\tbuf.WriteByte(mqttRetainedFlagDelMarker)\n\t}\n\tbuf.WriteString(strconv.FormatUint(uint64(rm.Flags), 16))\n\tbuf.WriteString(_CRLF_)\n\n\tif rm.Origin != _EMPTY_ {\n\t\tbuf.WriteString(mqttNatsRetainedMessageOrigin)\n\t\tbuf.WriteByte(':')\n\t\tbuf.WriteString(rm.Origin)\n\t\tbuf.WriteString(_CRLF_)\n\t}\n\tif rm.Source != _EMPTY_ {\n\t\tbuf.WriteString(mqttNatsRetainedMessageSource)\n\t\tbuf.WriteByte(':')\n\t\tbuf.WriteString(rm.Source)\n\t\tbuf.WriteString(_CRLF_)\n\t}\n\n\t// End of header, finalize\n\tbuf.WriteString(_CRLF_)\n\theaderLen = buf.Len()\n\tbuf.Write(rm.Msg)\n\treturn buf.Bytes(), headerLen\n}\n\nfunc mqttSliceHeaders(headers map[string][]byte, hdr []byte) {\n\t// Skip the hdrLine\n\tif !bytes.HasPrefix(hdr, stringToBytes(hdrLine)) {\n\t\treturn\n\t}\n\tcrLFAsBytes := stringToBytes(CR_LF)\n\tfor i := len(hdrLine); i < len(hdr); {\n\t\t// Search for key/val delimiter.\n\t\tdel := bytes.IndexByte(hdr[i:], ':')\n\t\t// Not found or key is length 0, we stop.\n\t\tif del < 0 || del == i {\n\t\t\tbreak\n\t\t}\n\t\tkeyStart := i\n\t\t// Walk back to remove spaces between the key and ':' if applicable.\n\t\tindex := keyStart + del - 1\n\t\tfor index > keyStart && hdr[index] == ' ' {\n\t\t\tindex--\n\t\t}\n\t\tkey := hdr[keyStart : index+1]\n\t\t// If what we had is only spaces, we stop.\n\t\tif len(key) == 0 {\n\t\t\tbreak\n\t\t}\n\t\ti += del + 1\n\t\tvalStart := i\n\t\t// Search for `\\r\\n`.\n\t\tnl := bytes.Index(hdr[valStart:], crLFAsBytes)\n\t\t// If we don't find, we stop.\n\t\tif nl < 0 {\n\t\t\tbreak\n\t\t}\n\t\t// Look if the caller is interested in this key.\n\t\tif _, ok := headers[bytesToString(key)]; ok {\n\t\t\tindex := valStart\n\t\t\t// Remove possible spaces between the ':' and the value.\n\t\t\tfor index < valStart+nl && hdr[index] == ' ' {\n\t\t\t\tindex++\n\t\t\t}\n\t\t\t// Create a slice and limit capacity to the value range.\n\t\t\tval := hdr[index : valStart+nl : valStart+nl]\n\t\t\t// Record in the caller's map the value for this key.\n\t\t\theaders[bytesToString(key)] = val\n\t\t}\n\t\t// Reposition to past the `\\r\\n`.\n\t\ti += nl + 2\n\t}\n}\n\n// Decodes a retained message based on the content of the header `h`.\n// The returned `*mqttRetainedMsg` object will hold a reference to `m`.\n// If the buffer `m` is not owned by the caller, it is the caller\n// responsibility to make a copy of the byte slice.\nfunc mqttDecodeRetainedMessage(subject string, h, m []byte) (*mqttRetainedMsg, error) {\n\theaders := map[string][]byte{\n\t\tmqttNatsRetainedMessageOrigin: nil,\n\t\tmqttNatsRetainedMessageFlags: nil,\n\t\tmqttNatsRetainedMessageSource: nil,\n\t}\n\tvar rm *mqttRetainedMsg\n\t// Retrieve the values for the above headers.\n\tmqttSliceHeaders(headers, h)\n\t// Get the flag header.\n\tfHeader := headers[mqttNatsRetainedMessageFlags]\n\t// If we don't, it could be that this is an old retained message that\n\t// was JSON encoded.\n\tif len(fHeader) > 0 {\n\t\tif len(fHeader) > 1 && fHeader[0] == mqttRetainedFlagDelMarker {\n\t\t\tfHeader = fHeader[1:]\n\t\t}\n\t\tflagsUint, err := strconv.ParseUint(bytesToString(fHeader), 16, 8)\n\t\tif err != nil {\n\t\t\t// Since the error is currently not reported in the server, we\n\t\t\t// will simply replace with this one.\n\t\t\treturn nil, errMQTTInvalidRetainFlags\n\t\t}\n\t\trm = &mqttRetainedMsg{\n\t\t\tFlags: byte(flagsUint),\n\t\t\tOrigin: string(headers[mqttNatsRetainedMessageOrigin]),\n\t\t\tSource: string(headers[mqttNatsRetainedMessageSource]),\n\t\t\tMsg: m,\n\t\t}\n\t} else {\n\t\tif err := json.Unmarshal(m, &rm); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\t// Now check that the values are correct.\n\t//\n\t// For \"Flags\", anything at or above binary (1111) is too big.\n\tif rm.Flags >= mqttPacketFlagMask {\n\t\treturn nil, errMQTTInvalidRetainFlags\n\t}\n\tif qos := mqttGetQoS(rm.Flags); qos > 2 {\n\t\treturn nil, errMQTTInvalidRetainFlags\n\t}\n\t// We store `Topic` in the retained message because we used to store\n\t// all retained messages under the same subject `$MQTT_rmsgs` in\n\t// the retained messages stream. That is no longer the case, and to\n\t// cover setups where the retained message stream is sourced from another\n\t// account and has some subject transforms, simply reconstruct the\n\t// topic/subject based on the `subject` passed to this function.\n\trm.Subject = strings.TrimPrefix(subject, mqttRetainedMsgsStreamSubject)\n\trm.Topic = bytesToString(natsSubjectStrToMQTTTopic(rm.Subject))\n\treturn rm, nil\n}\n\n// Creates the session stream (limit msgs of 1) for this client ID if it does\n// not already exist. If it exists, recover the single record to rebuild the\n// state of the session. If there is a session record but this session is not\n// registered in the runtime of this server, then a request is made to the\n// owner to close the client associated with this session since specification\n// [MQTT-3.1.4-2] specifies that if the ClientId represents a Client already\n// connected to the Server then the Server MUST disconnect the existing client.\n//\n// Runs from the client's readLoop.\n// Lock not held on entry, but session is in the locked map.\nfunc (as *mqttAccountSessionManager) createOrRestoreSession(clientID string, opts *Options) (*mqttSession, bool, error) {\n\tjsa := &as.jsa\n\tformatError := func(errTxt string, err error) (*mqttSession, bool, error) {\n\t\taccName := jsa.c.acc.GetName()\n\t\treturn nil, false, fmt.Errorf(\"%s for account %q, session %q: %v\", errTxt, accName, clientID, err)\n\t}\n\n\thash := getHash(clientID)\n\tsmsg, err := jsa.loadSessionMsg(as.domainTk, hash)\n\tif err != nil {\n\t\tif isErrorOtherThan(err, JSNoMessageFoundErr) {\n\t\t\treturn formatError(\"loading session record\", err)\n\t\t}\n\t\t// Message not found, so reate the session...\n\t\t// Create a session and indicate that this session did not exist.\n\t\tsess := mqttSessionCreate(jsa, clientID, hash, 0, opts)\n\t\tsess.domainTk = as.domainTk\n\t\treturn sess, false, nil\n\t}\n\t// We need to recover the existing record now.\n\tps := &mqttPersistedSession{}\n\tif err := json.Unmarshal(smsg.Data, ps); err != nil {\n\t\treturn formatError(fmt.Sprintf(\"unmarshal of session record at sequence %v\", smsg.Sequence), err)\n\t}\n\n\t// Restore this session (even if we don't own it), the caller will do the right thing.\n\tsess := mqttSessionCreate(jsa, clientID, hash, smsg.Sequence, opts)\n\tsess.domainTk = as.domainTk\n\tsess.clean = ps.Clean\n\tsess.subs = ps.Subs\n\tsess.cons = ps.Cons\n\tsess.pubRelConsumer = ps.PubRel\n\tas.addSession(sess, true)\n\treturn sess, true, nil\n}\n\n// Sends a request to delete a message, but does not wait for the response.\n//\n// No lock held on entry.\nfunc (as *mqttAccountSessionManager) deleteRetainedMsg(seq uint64) {\n\tas.jsa.deleteMsg(mqttRetainedMsgsStreamName, seq, false)\n}\n\n// Sends a message indicating that a retained message on a given subject and stream sequence\n// is being removed.\n// NOTE: This is maintained for backward compatibility reasons. Should be removed in 2.14/2.15?\nfunc (as *mqttAccountSessionManager) notifyRetainedMsgDeleted(subject string, seq uint64) {\n\treq := mqttRetMsgDel{\n\t\tSubject: subject,\n\t\tSeq: seq,\n\t}\n\tb, _ := json.Marshal(&req)\n\tjsa := &as.jsa\n\tjsa.sendq.push(&mqttJSPubMsg{\n\t\tsubj: jsa.rplyr + mqttJSARetainedMsgDel,\n\t\tmsg: b,\n\t})\n}\n\nfunc (as *mqttAccountSessionManager) transferUniqueSessStreamsToMuxed(log *Server) {\n\t// Set retry to true, will be set to false on success.\n\tretry := true\n\tdefer func() {\n\t\tif retry {\n\t\t\tnext := mqttDefaultTransferRetry\n\t\t\tlog.Warnf(\"Failed to transfer all MQTT session streams, will try again in %v\", next)\n\t\t\ttime.AfterFunc(next, func() { as.transferUniqueSessStreamsToMuxed(log) })\n\t\t}\n\t}()\n\n\tjsa := &as.jsa\n\tsni, err := jsa.newRequestEx(mqttJSAStreamNames, JSApiStreams, _EMPTY_, 0, nil)\n\tif err != nil {\n\t\tlog.Errorf(\"Unable to transfer MQTT session streams: %v\", err)\n\t\treturn\n\t}\n\tsnames := sni.(*JSApiStreamNamesResponse)\n\tif snames.Error != nil {\n\t\tlog.Errorf(\"Unable to transfer MQTT session streams: %v\", snames.ToError())\n\t\treturn\n\t}\n\tvar oldMQTTSessStreams []string\n\tfor _, sn := range snames.Streams {\n\t\tif strings.HasPrefix(sn, mqttSessionsStreamNamePrefix) {\n\t\t\toldMQTTSessStreams = append(oldMQTTSessStreams, sn)\n\t\t}\n\t}\n\tns := len(oldMQTTSessStreams)\n\tif ns == 0 {\n\t\t// Nothing to do\n\t\tretry = false\n\t\treturn\n\t}\n\tlog.Noticef(\"Transferring %v MQTT session streams...\", ns)\n\tfor _, sn := range oldMQTTSessStreams {\n\t\tlog.Noticef(\" Transferring stream %q to %q\", sn, mqttSessStreamName)\n\t\tsmsg, err := jsa.loadLastMsgFor(sn, sn)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\" Unable to load session record: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tps := &mqttPersistedSession{}\n\t\tif err := json.Unmarshal(smsg.Data, ps); err != nil {\n\t\t\tlog.Warnf(\" Unable to unmarshal the content of this stream, may not be a legitimate MQTT session stream, skipping\")\n\t\t\tcontinue\n\t\t}\n\t\t// Store record to MQTT session stream\n\t\tif _, err := jsa.storeSessionMsg(as.domainTk, getHash(ps.ID), 0, smsg.Data); err != nil {\n\t\t\tlog.Errorf(\" Unable to transfer the session record: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tjsa.deleteStream(sn)\n\t}\n\tlog.Noticef(\"Transfer of %v MQTT session streams done!\", ns)\n\tretry = false\n}\n\nfunc (as *mqttAccountSessionManager) transferRetainedToPerKeySubjectStream(log *Server) error {\n\tjsa := &as.jsa\n\tvar processed int\n\tvar transferred int\n\n\tstart := time.Now()\n\tdeadline := start.Add(mqttRetainedTransferTimeout)\n\tfor {\n\t\t// Try and look up messages on the original undivided \"$MQTT.rmsgs\" subject.\n\t\t// If nothing is returned here, we assume to have migrated all old messages.\n\t\tsmsg, err := jsa.loadNextMsgFor(mqttRetainedMsgsStreamName, \"$MQTT.rmsgs\")\n\t\tif IsNatsErr(err, JSNoMessageFoundErr) {\n\t\t\t// We've ran out of messages to transfer, done.\n\t\t\tbreak\n\t\t}\n\t\tif err != nil {\n\t\t\tlog.Warnf(\" Unable to transfer a retained message: failed to load from '$MQTT.rmsgs': %s\", err)\n\t\t\treturn err\n\t\t}\n\n\t\t// Unmarshal the message so that we can obtain the subject name. Do not\n\t\t// use mqttDecodeRetainedMessage() here because these messages are from\n\t\t// older versions, and contain the full JSON encoding in payload.\n\t\tvar rmsg mqttRetainedMsg\n\t\tif err = json.Unmarshal(smsg.Data, &rmsg); err == nil {\n\t\t\t// Store the message again, this time with the new per-key subject.\n\t\t\tsubject := mqttRetainedMsgsStreamSubject + rmsg.Subject\n\t\t\tif _, err = jsa.storeMsg(subject, 0, smsg.Data); err != nil {\n\t\t\t\tlog.Errorf(\" Unable to transfer the retained message with sequence %d: %v\", smsg.Sequence, err)\n\t\t\t}\n\t\t\ttransferred++\n\t\t} else {\n\t\t\tlog.Warnf(\" Unable to unmarshal retained message with sequence %d, skipping\", smsg.Sequence)\n\t\t}\n\n\t\t// Delete the original message.\n\t\tif err := jsa.deleteMsg(mqttRetainedMsgsStreamName, smsg.Sequence, true); err != nil {\n\t\t\tlog.Errorf(\" Unable to clean up the retained message with sequence %d: %v\", smsg.Sequence, err)\n\t\t\treturn err\n\t\t}\n\t\tprocessed++\n\n\t\tnow := time.Now()\n\t\tif now.After(deadline) {\n\t\t\terr := fmt.Errorf(\"timed out while transferring retained messages from '$MQTT.rmsgs' after %v, %d processed, %d successfully transferred\", now.Sub(start), processed, transferred)\n\t\t\tlog.Noticef(err.Error())\n\t\t\treturn err\n\t\t}\n\t}\n\tif processed > 0 {\n\t\tlog.Noticef(\"Processed %d messages from '$MQTT.rmsgs', successfully transferred %d in %v\", processed, transferred, time.Since(start))\n\t} else {\n\t\tlog.Debugf(\"No messages found to transfer from '$MQTT.rmsgs'\")\n\t}\n\treturn nil\n}\n\nfunc (as *mqttAccountSessionManager) getCachedRetainedMsg(subject string) *mqttRetainedMsg {\n\tv, ok := as.rmsCache.Load(subject)\n\tif !ok {\n\t\treturn nil\n\t}\n\trm := v.(*mqttRetainedMsg)\n\tif rm.expiresFromCache.Before(time.Now()) {\n\t\tas.rmsCache.Delete(subject)\n\t\treturn nil\n\t}\n\treturn rm\n}\n\n// If cache is enabled, the expiration for the `rm` is bumped by\n// `mqttRetainedCacheTTL` seconds.\n// If `onlyReplace` is true, then the `rm` object is stored in the cache using\n// the `subject` key only if there was already an object stored under that key.\n// If `copyMsgBytes` is true, then the `rm.Msg` bytes are copied (because it\n// references some buffer that is not owned by the caller).\n//\n// Note: currently `onlyReplace` and `cloneMsgBytes` always have the same\n// value (all `true` or all `false`) however we use different booleans to\n// better express the intent.\nfunc (as *mqttAccountSessionManager) setCachedRetainedMsg(subject string, rm *mqttRetainedMsg, onlyReplace, copyMsgBytes bool) {\n\tif rm == nil {\n\t\treturn\n\t}\n\trm.expiresFromCache = time.Now().Add(mqttRetainedCacheTTL)\n\tif onlyReplace {\n\t\tif _, ok := as.rmsCache.Load(subject); !ok {\n\t\t\treturn\n\t\t}\n\t}\n\tif copyMsgBytes {\n\t\trm.Msg = copyBytes(rm.Msg)\n\t}\n\tas.rmsCache.Store(subject, rm)\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// MQTT session related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\n// Returns a new mqttSession object with max ack pending set based on\n// option or use mqttDefaultMaxAckPending if no option set.\nfunc mqttSessionCreate(jsa *mqttJSA, id, idHash string, seq uint64, opts *Options) *mqttSession {\n\tmaxp := opts.MQTT.MaxAckPending\n\tif maxp == 0 {\n\t\tmaxp = mqttDefaultMaxAckPending\n\t}\n\n\treturn &mqttSession{\n\t\tjsa: jsa,\n\t\tid: id,\n\t\tidHash: idHash,\n\t\tseq: seq,\n\t\tmaxp: maxp,\n\t\tpubRelSubject: mqttPubRelSubjectPrefix + idHash,\n\t\tpubRelDeliverySubject: mqttPubRelDeliverySubjectPrefix + idHash,\n\t\tpubRelDeliverySubjectB: []byte(mqttPubRelDeliverySubjectPrefix + idHash),\n\t}\n}\n\n// Persists a session. Note that if the session's current client does not match\n// the given client, nothing is done.\n//\n// Lock not held on entry.\nfunc (sess *mqttSession) save() error {\n\tsess.mu.Lock()\n\tps := mqttPersistedSession{\n\t\tOrigin: sess.jsa.id,\n\t\tID: sess.id,\n\t\tClean: sess.clean,\n\t\tSubs: sess.subs,\n\t\tCons: sess.cons,\n\t\tPubRel: sess.pubRelConsumer,\n\t}\n\tb, _ := json.Marshal(&ps)\n\n\tdomainTk, cidHash := sess.domainTk, sess.idHash\n\tseq := sess.seq\n\tsess.mu.Unlock()\n\n\tvar hdr int\n\tif seq != 0 {\n\t\tbb := bytes.Buffer{}\n\t\tbb.WriteString(hdrLine)\n\t\tbb.WriteString(JSExpectedLastSubjSeq)\n\t\tbb.WriteString(\":\")\n\t\tbb.WriteString(strconv.FormatInt(int64(seq), 10))\n\t\tbb.WriteString(CR_LF)\n\t\tbb.WriteString(CR_LF)\n\t\thdr = bb.Len()\n\t\tbb.Write(b)\n\t\tb = bb.Bytes()\n\t}\n\n\tresp, err := sess.jsa.storeSessionMsg(domainTk, cidHash, hdr, b)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to persist session %q (seq=%v): %v\", ps.ID, seq, err)\n\t}\n\tsess.mu.Lock()\n\tsess.seq = resp.Sequence\n\tsess.mu.Unlock()\n\treturn nil\n}\n\n// Clear the session.\n//\n// Runs from the client's readLoop.\n// Lock not held on entry, but session is in the locked map.\nfunc (sess *mqttSession) clear(noWait bool) error {\n\tvar durs []string\n\tvar pubRelDur string\n\n\tsess.mu.Lock()\n\tid := sess.id\n\tseq := sess.seq\n\tif l := len(sess.cons); l > 0 {\n\t\tdurs = make([]string, 0, l)\n\t}\n\tfor sid, cc := range sess.cons {\n\t\tdelete(sess.cons, sid)\n\t\tdurs = append(durs, cc.Durable)\n\t}\n\tif sess.pubRelConsumer != nil {\n\t\tpubRelDur = sess.pubRelConsumer.Durable\n\t}\n\n\tsess.subs = nil\n\tsess.pendingPublish = nil\n\tsess.pendingPubRel = nil\n\tsess.cpending = nil\n\tsess.pubRelConsumer = nil\n\tsess.seq = 0\n\tsess.tmaxack = 0\n\tsess.mu.Unlock()\n\n\tfor _, dur := range durs {\n\t\tif _, err := sess.jsa.deleteConsumer(mqttStreamName, dur, noWait); isErrorOtherThan(err, JSConsumerNotFoundErr) {\n\t\t\treturn fmt.Errorf(\"unable to delete consumer %q for session %q: %v\", dur, sess.id, err)\n\t\t}\n\t}\n\tif pubRelDur != _EMPTY_ {\n\t\t_, err := sess.jsa.deleteConsumer(mqttOutStreamName, pubRelDur, noWait)\n\t\tif isErrorOtherThan(err, JSConsumerNotFoundErr) {\n\t\t\treturn fmt.Errorf(\"unable to delete consumer %q for session %q: %v\", pubRelDur, sess.id, err)\n\t\t}\n\t}\n\n\tif seq > 0 {\n\t\terr := sess.jsa.deleteMsg(mqttSessStreamName, seq, !noWait)\n\t\t// Ignore the various errors indicating that the message (or sequence)\n\t\t// is already deleted, can happen in a cluster.\n\t\tif isErrorOtherThan(err, JSSequenceNotFoundErrF) {\n\t\t\tif isErrorOtherThan(err, JSStreamMsgDeleteFailedF) || !strings.Contains(err.Error(), ErrStoreMsgNotFound.Error()) {\n\t\t\t\treturn fmt.Errorf(\"unable to delete session %q record at sequence %v: %v\", id, seq, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// This will update the session record for this client in the account's MQTT\n// sessions stream if the session had any change in the subscriptions.\n//\n// Runs from the client's readLoop.\n// Lock not held on entry, but session is in the locked map.\nfunc (sess *mqttSession) update(filters []*mqttFilter, add bool) error {\n\t// Evaluate if we need to persist anything.\n\tvar needUpdate bool\n\tfor _, f := range filters {\n\t\tif add {\n\t\t\tif f.qos == mqttSubAckFailure {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif qos, ok := sess.subs[f.filter]; !ok || qos != f.qos {\n\t\t\t\tif sess.subs == nil {\n\t\t\t\t\tsess.subs = make(map[string]byte)\n\t\t\t\t}\n\t\t\t\tsess.subs[f.filter] = f.qos\n\t\t\t\tneedUpdate = true\n\t\t\t}\n\t\t} else {\n\t\t\tif _, ok := sess.subs[f.filter]; ok {\n\t\t\t\tdelete(sess.subs, f.filter)\n\t\t\t\tneedUpdate = true\n\t\t\t}\n\t\t}\n\t}\n\tvar err error\n\tif needUpdate {\n\t\terr = sess.save()\n\t}\n\treturn err\n}\n\nfunc (sess *mqttSession) bumpPI() uint16 {\n\tvar avail bool\n\tnext := sess.last_pi\n\tfor i := 0; i < 0xFFFF; i++ {\n\t\tnext++\n\t\tif next == 0 {\n\t\t\tnext = 1\n\t\t}\n\n\t\t_, usedInPublish := sess.pendingPublish[next]\n\t\t_, usedInPubRel := sess.pendingPubRel[next]\n\t\tif !usedInPublish && !usedInPubRel {\n\t\t\tsess.last_pi = next\n\t\t\tavail = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !avail {\n\t\treturn 0\n\t}\n\treturn sess.last_pi\n}\n\n// trackPublishRetained is invoked when a retained (QoS) message is published.\n// It need a new PI to be allocated, so we add it to the pendingPublish map,\n// with an empty value. Since cpending (not pending) is used to serialize the PI\n// mappings, we need to add this PI there as well. Make a unique key by using\n// mqttRetainedMsgsStreamName for the durable name, and PI for sseq.\n//\n// Lock held on entry\nfunc (sess *mqttSession) trackPublishRetained() uint16 {\n\t// Make sure we initialize the tracking maps.\n\tif sess.pendingPublish == nil {\n\t\tsess.pendingPublish = make(map[uint16]*mqttPending)\n\t}\n\tif sess.cpending == nil {\n\t\tsess.cpending = make(map[string]map[uint64]uint16)\n\t}\n\n\tpi := sess.bumpPI()\n\tif pi == 0 {\n\t\treturn 0\n\t}\n\tsess.pendingPublish[pi] = &mqttPending{}\n\n\treturn pi\n}\n\n// trackPublish is invoked when a (QoS) PUBLISH message is to be delivered. It\n// detects an untracked (new) message based on its sequence extracted from its\n// delivery-time jsAckSubject, and adds it to the tracking maps. Returns a PI to\n// use for the message (new, or previously used), and whether this is a\n// duplicate delivery attempt.\n//\n// Lock held on entry\nfunc (sess *mqttSession) trackPublish(jsDur, jsAckSubject string) (uint16, bool) {\n\tvar dup bool\n\tvar pi uint16\n\n\tif jsAckSubject == _EMPTY_ || jsDur == _EMPTY_ {\n\t\treturn 0, false\n\t}\n\n\t// Make sure we initialize the tracking maps.\n\tif sess.pendingPublish == nil {\n\t\tsess.pendingPublish = make(map[uint16]*mqttPending)\n\t}\n\tif sess.cpending == nil {\n\t\tsess.cpending = make(map[string]map[uint64]uint16)\n\t}\n\n\t// Get the stream sequence and duplicate flag from the ack reply subject.\n\tsseq, _, dcount := ackReplyInfo(jsAckSubject)\n\tif dcount > 1 {\n\t\tdup = true\n\t}\n\n\tvar ack *mqttPending\n\tsseqToPi, ok := sess.cpending[jsDur]\n\tif !ok {\n\t\tsseqToPi = make(map[uint64]uint16)\n\t\tsess.cpending[jsDur] = sseqToPi\n\t} else {\n\t\tpi = sseqToPi[sseq]\n\t}\n\n\tif pi != 0 {\n\t\t// There is a possible race between a PUBLISH re-delivery calling us,\n\t\t// and a PUBREC received already having submitting a PUBREL into JS . If\n\t\t// so, indicate no need for (re-)delivery by returning a PI of 0.\n\t\t_, usedForPubRel := sess.pendingPubRel[pi]\n\t\tif /*dup && */ usedForPubRel {\n\t\t\treturn 0, false\n\t\t}\n\n\t\t// We should have a pending JS ACK for this PI.\n\t\tack = sess.pendingPublish[pi]\n\t} else {\n\t\t// sess.maxp will always have a value > 0.\n\t\tif len(sess.pendingPublish) >= int(sess.maxp) {\n\t\t\t// Indicate that we did not assign a packet identifier.\n\t\t\t// The caller will not send the message to the subscription\n\t\t\t// and JS will redeliver later, based on consumer's AckWait.\n\t\t\treturn 0, false\n\t\t}\n\n\t\tpi = sess.bumpPI()\n\t\tif pi == 0 {\n\t\t\treturn 0, false\n\t\t}\n\n\t\tsseqToPi[sseq] = pi\n\t}\n\n\tif ack == nil {\n\t\tsess.pendingPublish[pi] = &mqttPending{\n\t\t\tjsDur: jsDur,\n\t\t\tsseq: sseq,\n\t\t\tjsAckSubject: jsAckSubject,\n\t\t}\n\t} else {\n\t\tack.jsAckSubject = jsAckSubject\n\t\tack.sseq = sseq\n\t\tack.jsDur = jsDur\n\t}\n\n\treturn pi, dup\n}\n\n// Stops a PI from being tracked as a PUBLISH. It can still be in use for a\n// pending PUBREL.\n//\n// Lock held on entry\nfunc (sess *mqttSession) untrackPublish(pi uint16) (jsAckSubject string) {\n\tack, ok := sess.pendingPublish[pi]\n\tif !ok {\n\t\treturn _EMPTY_\n\t}\n\n\tdelete(sess.pendingPublish, pi)\n\tif len(sess.pendingPublish) == 0 {\n\t\tsess.last_pi = 0\n\t}\n\n\tif len(sess.cpending) != 0 && ack.jsDur != _EMPTY_ {\n\t\tif sseqToPi := sess.cpending[ack.jsDur]; sseqToPi != nil {\n\t\t\tdelete(sseqToPi, ack.sseq)\n\t\t}\n\t}\n\n\treturn ack.jsAckSubject\n}\n\n// trackAsPubRel is invoked in 2 cases: (a) when we receive a PUBREC and we need\n// to change from tracking the PI as a PUBLISH to a PUBREL; and (b) when we\n// attempt to deliver the PUBREL to record the JS ack subject for it.\n//\n// Lock held on entry\nfunc (sess *mqttSession) trackAsPubRel(pi uint16, jsAckSubject string) {\n\tif sess.pubRelConsumer == nil {\n\t\t// The cosumer MUST be set up already.\n\t\treturn\n\t}\n\tjsDur := sess.pubRelConsumer.Durable\n\n\tif sess.pendingPubRel == nil {\n\t\tsess.pendingPubRel = make(map[uint16]*mqttPending)\n\t}\n\n\tif jsAckSubject == _EMPTY_ {\n\t\tsess.pendingPubRel[pi] = &mqttPending{\n\t\t\tjsDur: jsDur,\n\t\t}\n\t\treturn\n\t}\n\n\tsseq, _, _ := ackReplyInfo(jsAckSubject)\n\n\tif sess.cpending == nil {\n\t\tsess.cpending = make(map[string]map[uint64]uint16)\n\t}\n\tsseqToPi := sess.cpending[jsDur]\n\tif sseqToPi == nil {\n\t\tsseqToPi = make(map[uint64]uint16)\n\t\tsess.cpending[jsDur] = sseqToPi\n\t}\n\tsseqToPi[sseq] = pi\n\tsess.pendingPubRel[pi] = &mqttPending{\n\t\tjsDur: sess.pubRelConsumer.Durable,\n\t\tsseq: sseq,\n\t\tjsAckSubject: jsAckSubject,\n\t}\n}\n\n// Stops a PI from being tracked as a PUBREL.\n//\n// Lock held on entry\nfunc (sess *mqttSession) untrackPubRel(pi uint16) (jsAckSubject string) {\n\tack, ok := sess.pendingPubRel[pi]\n\tif !ok {\n\t\treturn _EMPTY_\n\t}\n\n\tdelete(sess.pendingPubRel, pi)\n\n\tif sess.pubRelConsumer != nil && len(sess.cpending) > 0 {\n\t\tif sseqToPi := sess.cpending[ack.jsDur]; sseqToPi != nil {\n\t\t\tdelete(sseqToPi, ack.sseq)\n\t\t}\n\t}\n\n\treturn ack.jsAckSubject\n}\n\n// Sends a consumer delete request, but does not wait for response.\n//\n// Lock not held on entry.\nfunc (sess *mqttSession) deleteConsumer(cc *ConsumerConfig) {\n\tsess.mu.Lock()\n\tsess.tmaxack -= cc.MaxAckPending\n\tsess.jsa.deleteConsumer(mqttStreamName, cc.Durable, true)\n\tsess.mu.Unlock()\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// CONNECT protocol related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\n// Parse the MQTT connect protocol\nfunc (c *client) mqttParseConnect(r *mqttReader, hasMappings bool) (byte, *mqttConnectProto, error) {\n\t// Protocol name\n\tproto, err := r.readBytes(\"protocol name\", false)\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\n\t// Spec [MQTT-3.1.2-1]\n\tif !bytes.Equal(proto, mqttProtoName) {\n\t\t// Check proto name against v3.1 to report better error\n\t\tif bytes.Equal(proto, mqttOldProtoName) {\n\t\t\treturn 0, nil, fmt.Errorf(\"older protocol %q not supported\", proto)\n\t\t}\n\t\treturn 0, nil, fmt.Errorf(\"expected connect packet with protocol name %q, got %q\", mqttProtoName, proto)\n\t}\n\n\t// Protocol level\n\tlevel, err := r.readByte(\"protocol level\")\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\t// Spec [MQTT-3.1.2-2]\n\tif level != mqttProtoLevel {\n\t\treturn mqttConnAckRCUnacceptableProtocolVersion, nil, fmt.Errorf(\"unacceptable protocol version of %v\", level)\n\t}\n\n\tcp := &mqttConnectProto{}\n\t// Connect flags\n\tcp.flags, err = r.readByte(\"flags\")\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\n\t// Spec [MQTT-3.1.2-3]\n\tif cp.flags&mqttConnFlagReserved != 0 {\n\t\treturn 0, nil, errMQTTConnFlagReserved\n\t}\n\n\tvar hasWill bool\n\twqos := (cp.flags & mqttConnFlagWillQoS) >> 3\n\twretain := cp.flags&mqttConnFlagWillRetain != 0\n\t// Spec [MQTT-3.1.2-11]\n\tif cp.flags&mqttConnFlagWillFlag == 0 {\n\t\t// Spec [MQTT-3.1.2-13]\n\t\tif wqos != 0 {\n\t\t\treturn 0, nil, fmt.Errorf(\"if Will flag is set to 0, Will QoS must be 0 too, got %v\", wqos)\n\t\t}\n\t\t// Spec [MQTT-3.1.2-15]\n\t\tif wretain {\n\t\t\treturn 0, nil, errMQTTWillAndRetainFlag\n\t\t}\n\t} else {\n\t\t// Spec [MQTT-3.1.2-14]\n\t\tif wqos == 3 {\n\t\t\treturn 0, nil, fmt.Errorf(\"if Will flag is set to 1, Will QoS can be 0, 1 or 2, got %v\", wqos)\n\t\t}\n\t\thasWill = true\n\t}\n\n\tif c.mqtt.rejectQoS2Pub && hasWill && wqos == 2 {\n\t\treturn mqttConnAckRCQoS2WillRejected, nil, fmt.Errorf(\"server does not accept QoS2 for Will messages\")\n\t}\n\n\t// Spec [MQTT-3.1.2-19]\n\thasUser := cp.flags&mqttConnFlagUsernameFlag != 0\n\t// Spec [MQTT-3.1.2-21]\n\thasPassword := cp.flags&mqttConnFlagPasswordFlag != 0\n\t// Spec [MQTT-3.1.2-22]\n\tif !hasUser && hasPassword {\n\t\treturn 0, nil, errMQTTPasswordFlagAndNoUser\n\t}\n\n\t// Keep alive\n\tvar ka uint16\n\tka, err = r.readUint16(\"keep alive\")\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\t// Spec [MQTT-3.1.2-24]\n\tif ka > 0 {\n\t\tcp.rd = time.Duration(float64(ka)*1.5) * time.Second\n\t}\n\n\t// Payload starts here and order is mandated by:\n\t// Spec [MQTT-3.1.3-1]: client ID, will topic, will message, username, password\n\n\t// Client ID\n\tc.mqtt.cid, err = r.readString(\"client ID\")\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\t// Spec [MQTT-3.1.3-7]\n\tif c.mqtt.cid == _EMPTY_ {\n\t\tif cp.flags&mqttConnFlagCleanSession == 0 {\n\t\t\treturn mqttConnAckRCIdentifierRejected, nil, errMQTTCIDEmptyNeedsCleanFlag\n\t\t}\n\t\t// Spec [MQTT-3.1.3-6]\n\t\tc.mqtt.cid = nuid.Next()\n\t}\n\t// Spec [MQTT-3.1.3-4] and [MQTT-3.1.3-9]\n\tif err := mqttValidateString(c.mqtt.cid, \"client ID\"); err != nil {\n\t\treturn mqttConnAckRCIdentifierRejected, nil, err\n\t}\n\n\tif hasWill {\n\t\tcp.will = &mqttWill{\n\t\t\tqos: wqos,\n\t\t\tretain: wretain,\n\t\t}\n\t\tvar topic []byte\n\t\t// Need to make a copy since we need to hold to this topic after the\n\t\t// parsing of this protocol.\n\t\ttopic, err = r.readBytes(\"Will topic\", true)\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\tif len(topic) == 0 {\n\t\t\treturn 0, nil, errMQTTEmptyWillTopic\n\t\t}\n\t\tif err := mqttValidateTopic(topic, \"Will topic\"); err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\t// Convert MQTT topic to NATS subject\n\t\tcp.will.subject, err = mqttTopicToNATSPubSubject(topic)\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\t// Check for subject mapping.\n\t\tif hasMappings {\n\t\t\t// For selectMappedSubject to work, we need to have c.pa.subject set.\n\t\t\t// If there is a change, c.pa.mapped will be set after the call.\n\t\t\tc.pa.subject = cp.will.subject\n\t\t\tif changed := c.selectMappedSubject(); changed {\n\t\t\t\t// We need to keep track of the NATS subject/mapped in the `cp` structure.\n\t\t\t\tcp.will.subject = c.pa.subject\n\t\t\t\tcp.will.mapped = c.pa.mapped\n\t\t\t\t// We also now need to map the original MQTT topic to the new topic\n\t\t\t\t// based on the new subject.\n\t\t\t\ttopic = natsSubjectToMQTTTopic(cp.will.subject)\n\t\t\t}\n\t\t\t// Reset those now.\n\t\t\tc.pa.subject, c.pa.mapped = nil, nil\n\t\t}\n\t\tcp.will.topic = topic\n\t\t// Now \"will\" message.\n\t\t// Ask for a copy since we need to hold to this after parsing of this protocol.\n\t\tcp.will.message, err = r.readBytes(\"Will message\", true)\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t}\n\n\tif hasUser {\n\t\tc.opts.Username, err = r.readString(\"user name\")\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\tif c.opts.Username == _EMPTY_ {\n\t\t\treturn mqttConnAckRCBadUserOrPassword, nil, errMQTTEmptyUsername\n\t\t}\n\t\t// Spec [MQTT-3.1.3-11]\n\t\tif err := mqttValidateString(c.opts.Username, \"user name\"); err != nil {\n\t\t\treturn mqttConnAckRCBadUserOrPassword, nil, err\n\t\t}\n\t}\n\n\tif hasPassword {\n\t\tc.opts.Password, err = r.readString(\"password\")\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\tc.opts.Token = c.opts.Password\n\t\tc.opts.JWT = c.opts.Password\n\t}\n\treturn 0, cp, nil\n}\n\nfunc (c *client) mqttConnectTrace(cp *mqttConnectProto) string {\n\ttrace := fmt.Sprintf(\"clientID=%s\", c.mqtt.cid)\n\tif cp.rd > 0 {\n\t\ttrace += fmt.Sprintf(\" keepAlive=%v\", cp.rd)\n\t}\n\tif cp.will != nil {\n\t\ttrace += fmt.Sprintf(\" will=(topic=%s QoS=%v retain=%v)\",\n\t\t\tcp.will.topic, cp.will.qos, cp.will.retain)\n\t}\n\tif cp.flags&mqttConnFlagCleanSession != 0 {\n\t\ttrace += \" clean\"\n\t}\n\tif c.opts.Username != _EMPTY_ {\n\t\ttrace += fmt.Sprintf(\" username=%s\", c.opts.Username)\n\t}\n\tif c.opts.Password != _EMPTY_ {\n\t\ttrace += \" password=****\"\n\t}\n\treturn trace\n}\n\n// Process the CONNECT packet.\n//\n// For the first session on the account, an account session manager will be created,\n// along with the JetStream streams/consumer necessary for the working of MQTT.\n//\n// The session, identified by a client ID, will be registered, or if already existing,\n// will be resumed. If the session exists but is associated with an existing client,\n// the old client is evicted, as per the specifications.\n//\n// Due to specific locking requirements around JS API requests, we cannot hold some\n// locks for the entire duration of processing of some protocols, therefore, we use\n// a map that registers the client ID in a \"locked\" state. If a different client tries\n// to connect and the server detects that the client ID is in that map, it will try\n// a little bit until it is not, or fail the new client, since we can't protect\n// processing of protocols in the original client. This is not expected to happen often.\n//\n// Runs from the client's readLoop.\n// No lock held on entry.\nfunc (s *Server) mqttProcessConnect(c *client, cp *mqttConnectProto, trace bool) error {\n\tsendConnAck := func(rc byte, sessp bool) {\n\t\tc.mqttEnqueueConnAck(rc, sessp)\n\t\tif trace {\n\t\t\tc.traceOutOp(\"CONNACK\", []byte(fmt.Sprintf(\"sp=%v rc=%v\", sessp, rc)))\n\t\t}\n\t}\n\n\tc.mu.Lock()\n\tcid := c.mqtt.cid\n\tc.clearAuthTimer()\n\tc.mu.Unlock()\n\tif !s.isClientAuthorized(c) {\n\t\tif trace {\n\t\t\tc.traceOutOp(\"CONNACK\", []byte(fmt.Sprintf(\"sp=%v rc=%v\", false, mqttConnAckRCNotAuthorized)))\n\t\t}\n\t\tc.authViolation()\n\t\treturn ErrAuthentication\n\t}\n\t// Now that we are authenticated, we have the client bound to the account.\n\t// Get the account's level MQTT sessions manager. If it does not exists yet,\n\t// this will create it along with the streams where sessions and messages\n\t// are stored.\n\tasm, err := s.getOrCreateMQTTAccountSessionManager(c)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Most of the session state is altered only in the readLoop so does not\n\t// need locking. For things that can be access in the readLoop and in\n\t// callbacks, we will use explicit locking.\n\t// To prevent other clients to connect with the same client ID, we will\n\t// add the client ID to a \"locked\" map so that the connect somewhere else\n\t// is put on hold.\n\t// This keep track of how many times this client is detecting that its\n\t// client ID is in the locked map. After a short amount, the server will\n\t// fail this inbound client.\n\tlocked := 0\n\nCHECK:\n\tasm.mu.Lock()\n\t// Check if different applications keep trying to connect with the same\n\t// client ID at the same time.\n\tif tm, ok := asm.flappers[cid]; ok {\n\t\t// If the last time it tried to connect was more than 1 sec ago,\n\t\t// then accept and remove from flappers map.\n\t\tif time.Now().UnixNano()-tm > int64(mqttSessJailDur) {\n\t\t\tasm.removeSessFromFlappers(cid)\n\t\t} else {\n\t\t\t// Will hold this client for a second and then close it. We\n\t\t\t// do this so that if the client has a reconnect feature we\n\t\t\t// don't end-up with very rapid flapping between apps.\n\t\t\t// We need to wait in place and not schedule the connection\n\t\t\t// close because if this is a misbehaved client that does\n\t\t\t// not wait for the CONNACK and sends other protocols, the\n\t\t\t// server would not have a fully setup client and may panic.\n\t\t\tasm.mu.Unlock()\n\t\t\tselect {\n\t\t\tcase <-s.quitCh:\n\t\t\tcase <-time.After(mqttSessJailDur):\n\t\t\t}\n\t\t\tc.closeConnection(DuplicateClientID)\n\t\t\treturn ErrConnectionClosed\n\t\t}\n\t}\n\t// If an existing session is in the process of processing some packet, we can't\n\t// evict the old client just yet. So try again to see if the state clears, but\n\t// if it does not, then we have no choice but to fail the new client instead of\n\t// the old one.\n\tif _, ok := asm.sessLocked[cid]; ok {\n\t\tasm.mu.Unlock()\n\t\tif locked++; locked == 10 {\n\t\t\treturn fmt.Errorf(\"other session with client ID %q is in the process of connecting\", cid)\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tgoto CHECK\n\t}\n\n\t// Register this client ID the \"locked\" map for the duration if this function.\n\tasm.sessLocked[cid] = struct{}{}\n\t// And remove it on exit, regardless of error or not.\n\tdefer func() {\n\t\tasm.mu.Lock()\n\t\tdelete(asm.sessLocked, cid)\n\t\tasm.mu.Unlock()\n\t}()\n\n\t// Is the client requesting a clean session or not.\n\tcleanSess := cp.flags&mqttConnFlagCleanSession != 0\n\t// Session present? Assume false, will be set to true only when applicable.\n\tsessp := false\n\t// Do we have an existing session for this client ID\n\tes, exists := asm.sessions[cid]\n\tasm.mu.Unlock()\n\n\t// The session is not in the map, but may be on disk, so try to recover\n\t// or create the stream if not.\n\tif !exists {\n\t\tes, exists, err = asm.createOrRestoreSession(cid, s.getOpts())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif exists {\n\t\t// Clear the session if client wants a clean session.\n\t\t// Also, Spec [MQTT-3.2.2-1]: don't report session present\n\t\tif cleanSess || es.clean {\n\t\t\t// Spec [MQTT-3.1.2-6]: If CleanSession is set to 1, the Client and\n\t\t\t// Server MUST discard any previous Session and start a new one.\n\t\t\t// This Session lasts as long as the Network Connection. State data\n\t\t\t// associated with this Session MUST NOT be reused in any subsequent\n\t\t\t// Session.\n\t\t\tif err := es.clear(false); err != nil {\n\t\t\t\tasm.removeSession(es, true)\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\t// Report to the client that the session was present\n\t\t\tsessp = true\n\t\t}\n\t\t// Spec [MQTT-3.1.4-2]. If the ClientId represents a Client already\n\t\t// connected to the Server then the Server MUST disconnect the existing\n\t\t// client.\n\t\t// Bind with the new client. This needs to be protected because can be\n\t\t// accessed outside of the readLoop.\n\t\tes.mu.Lock()\n\t\tec := es.c\n\t\tes.c = c\n\t\tes.clean = cleanSess\n\t\t// Clear this flag so we resubscribe to PUBREL subject is needed.\n\t\tes.pubRelSubscribed = false\n\t\tes.mu.Unlock()\n\t\tif ec != nil {\n\t\t\t// Remove \"will\" of existing client before closing\n\t\t\tec.mu.Lock()\n\t\t\tec.mqtt.cp.will = nil\n\t\t\tec.mu.Unlock()\n\t\t\t// Add to the map of the flappers\n\t\t\tasm.mu.Lock()\n\t\t\tasm.addSessToFlappers(cid)\n\t\t\tasm.mu.Unlock()\n\t\t\tc.Warnf(\"Replacing old client %q since both have the same client ID %q\", ec, cid)\n\t\t\t// Close old client in separate go routine\n\t\t\tgo ec.closeConnection(DuplicateClientID)\n\t\t}\n\t} else {\n\t\t// Spec [MQTT-3.2.2-3]: if the Server does not have stored Session state,\n\t\t// it MUST set Session Present to 0 in the CONNACK packet.\n\t\tes.mu.Lock()\n\t\tes.c, es.clean = c, cleanSess\n\t\tes.mu.Unlock()\n\t\t// Now add this new session into the account sessions\n\t\tasm.addSession(es, true)\n\t}\n\t// We would need to save only if it did not exist previously, but we save\n\t// always in case we are running in cluster mode. This will notify other\n\t// running servers that this session is being used.\n\tif err := es.save(); err != nil {\n\t\tasm.removeSession(es, true)\n\t\treturn err\n\t}\n\tc.mu.Lock()\n\tc.flags.set(connectReceived)\n\tc.mqtt.cp = cp\n\tc.mqtt.asm = asm\n\tc.mqtt.sess = es\n\tc.mu.Unlock()\n\n\t// Spec [MQTT-3.2.0-1]: CONNACK must be the first protocol sent to the session.\n\tsendConnAck(mqttConnAckRCConnectionAccepted, sessp)\n\n\t// Process possible saved subscriptions.\n\tif l := len(es.subs); l > 0 {\n\t\tfilters := make([]*mqttFilter, 0, l)\n\t\tfor subject, qos := range es.subs {\n\t\t\tfilters = append(filters, &mqttFilter{filter: subject, qos: qos})\n\t\t}\n\t\tif _, err := asm.processSubs(es, c, filters, false, trace); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *client) mqttEnqueueConnAck(rc byte, sessionPresent bool) {\n\tproto := [4]byte{mqttPacketConnectAck, 2, 0, rc}\n\tc.mu.Lock()\n\t// Spec [MQTT-3.2.2-4]. If return code is different from 0, then\n\t// session present flag must be set to 0.\n\tif rc == 0 {\n\t\tif sessionPresent {\n\t\t\tproto[2] = 1\n\t\t}\n\t}\n\tc.enqueueProto(proto[:])\n\tc.mu.Unlock()\n}\n\nfunc (s *Server) mqttHandleWill(c *client) {\n\tc.mu.Lock()\n\tif c.mqtt.cp == nil {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\twill := c.mqtt.cp.will\n\tif will == nil {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\tpp := c.mqtt.pp\n\tpp.topic = will.topic\n\tpp.subject = will.subject\n\tpp.mapped = will.mapped\n\tpp.msg = will.message\n\tpp.sz = len(will.message)\n\tpp.pi = 0\n\tpp.flags = will.qos << 1\n\tif will.retain {\n\t\tpp.flags |= mqttPubFlagRetain\n\t}\n\tc.mu.Unlock()\n\ts.mqttInitiateMsgDelivery(c, pp)\n\tc.flushClients(0)\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// PUBLISH protocol related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (c *client) mqttParsePub(r *mqttReader, pl int, pp *mqttPublish, hasMappings bool) error {\n\tqos := mqttGetQoS(pp.flags)\n\tif qos > 2 {\n\t\treturn fmt.Errorf(\"QoS=%v is invalid in MQTT\", qos)\n\t}\n\n\tif c.mqtt.rejectQoS2Pub && qos == 2 {\n\t\treturn fmt.Errorf(\"QoS=2 is disabled for PUBLISH messages\")\n\t}\n\n\t// Keep track of where we are when starting to read the variable header\n\tstart := r.pos\n\n\tvar err error\n\tpp.topic, err = r.readBytes(\"topic\", false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(pp.topic) == 0 {\n\t\treturn errMQTTTopicIsEmpty\n\t}\n\tif err := mqttValidateTopic(pp.topic, \"topic\"); err != nil {\n\t\treturn err\n\t}\n\t// Convert the topic to a NATS subject. This call will also check that\n\t// there is no MQTT wildcards (Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1])\n\t// Note that this may not result in a copy if there is no conversion.\n\t// It is good because after the message is processed we won't have a\n\t// reference to the buffer and we save a copy.\n\tpp.subject, err = mqttTopicToNATSPubSubject(pp.topic)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Check for subject mapping.\n\tif hasMappings {\n\t\t// For selectMappedSubject to work, we need to have c.pa.subject set.\n\t\t// If there is a change, c.pa.mapped will be set after the call.\n\t\tc.pa.subject = pp.subject\n\t\tif changed := c.selectMappedSubject(); changed {\n\t\t\t// We need to keep track of the NATS subject/mapped in the `pp` structure.\n\t\t\tpp.subject = c.pa.subject\n\t\t\tpp.mapped = c.pa.mapped\n\t\t\t// We also now need to map the original MQTT topic to the new topic\n\t\t\t// based on the new subject.\n\t\t\tpp.topic = natsSubjectToMQTTTopic(pp.subject)\n\t\t}\n\t\t// Reset those now.\n\t\tc.pa.subject, c.pa.mapped = nil, nil\n\t}\n\n\tif qos > 0 {\n\t\tpp.pi, err = r.readUint16(\"packet identifier\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif pp.pi == 0 {\n\t\t\treturn fmt.Errorf(\"with QoS=%v, packet identifier cannot be 0\", qos)\n\t\t}\n\t} else {\n\t\tpp.pi = 0\n\t}\n\n\t// The message payload will be the total packet length minus\n\t// what we have consumed for the variable header\n\tpp.sz = pl - (r.pos - start)\n\tif pp.sz > 0 {\n\t\tstart = r.pos\n\t\tr.pos += pp.sz\n\t\tpp.msg = r.buf[start:r.pos]\n\t} else {\n\t\tpp.msg = nil\n\t}\n\treturn nil\n}\n\nfunc mqttValidateTopic(topic []byte, field string) error {\n\tif !utf8.Valid(topic) {\n\t\treturn fmt.Errorf(\"invalid utf8 for %s %q\", field, topic)\n\t}\n\tif bytes.IndexByte(topic, 0) >= 0 {\n\t\treturn fmt.Errorf(\"invalid null character in %s %q\", field, topic)\n\t}\n\treturn nil\n}\n\nfunc mqttValidateString(value string, field string) error {\n\tif !utf8.ValidString(value) {\n\t\treturn fmt.Errorf(\"invalid utf8 for %s %q\", field, value)\n\t}\n\tif strings.IndexByte(value, 0) >= 0 {\n\t\treturn fmt.Errorf(\"invalid null character in %s %q\", field, value)\n\t}\n\treturn nil\n}\n\nfunc mqttPubTrace(pp *mqttPublish) string {\n\tdup := pp.flags&mqttPubFlagDup != 0\n\tqos := mqttGetQoS(pp.flags)\n\tretain := mqttIsRetained(pp.flags)\n\tvar piStr string\n\tif pp.pi > 0 {\n\t\tpiStr = fmt.Sprintf(\" pi=%v\", pp.pi)\n\t}\n\treturn fmt.Sprintf(\"%s dup=%v QoS=%v retain=%v size=%v%s\",\n\t\tpp.topic, dup, qos, retain, pp.sz, piStr)\n}\n\n// mqttComputeNatsMsgSize computes the size the NATS message to be delivered\n// based on a MQTT PUBLISH packet.\n// encodePP: whether to encode complete MQTT PUBLISH packet header information\n// - false: initial delivery (QoS 0/1) needs only base header\n// - true: QoS2 storage needs to encode Nmqtt-Subject and Nmqtt-Mapped\nfunc mqttComputeNatsMsgSize(pp *mqttPublish, encodePP bool) int {\n\tsize := len(hdrLine) +\n\t\tlen(mqttNatsHeader) + 2 + 2 + // 2 for ':', and 2 for CRLF\n\t\t2 + // end-of-header CRLF\n\t\tpp.sz\n\tif encodePP {\n\t\tsize += len(mqttNatsHeaderSubject) + 1 + // +1 for ':'\n\t\t\tlen(pp.subject) + 2 // 2 for CRLF\n\n\t\tif len(pp.mapped) > 0 {\n\t\t\tsize += len(mqttNatsHeaderMapped) + 1 + // +1 for ':'\n\t\t\t\tlen(pp.mapped) + 2 // 2 for CRLF\n\t\t}\n\t}\n\treturn size\n}\n\n// Composes a NATS message from a MQTT PUBLISH packet. The message includes an\n// internal header containint the original packet's QoS, and for QoS2 packets\n// the original subject.\n//\n// Example (QoS2, subject: \"foo.bar\"):\n//\n//\tNATS/1.0\\r\\n\n//\tNmqtt-Pub:2foo.bar\\r\\n\n//\t\\r\\n\nfunc mqttNewDeliverableMessage(pp *mqttPublish, encodePP bool) (natsMsg []byte, headerLen int) {\n\tsize := mqttComputeNatsMsgSize(pp, encodePP)\n\n\tbuf := bytes.NewBuffer(make([]byte, 0, size))\n\n\tqos := mqttGetQoS(pp.flags)\n\n\tbuf.WriteString(hdrLine)\n\tbuf.WriteString(mqttNatsHeader)\n\tbuf.WriteByte(':')\n\tbuf.WriteByte(qos + '0')\n\tbuf.WriteString(_CRLF_)\n\n\tif encodePP {\n\t\tbuf.WriteString(mqttNatsHeaderSubject)\n\t\tbuf.WriteByte(':')\n\t\tbuf.Write(pp.subject)\n\t\tbuf.WriteString(_CRLF_)\n\n\t\tif len(pp.mapped) > 0 {\n\t\t\tbuf.WriteString(mqttNatsHeaderMapped)\n\t\t\tbuf.WriteByte(':')\n\t\t\tbuf.Write(pp.mapped)\n\t\t\tbuf.WriteString(_CRLF_)\n\t\t}\n\t}\n\n\t// End of header\n\tbuf.WriteString(_CRLF_)\n\n\theaderLen = buf.Len()\n\n\tbuf.Write(pp.msg)\n\treturn buf.Bytes(), headerLen\n}\n\n// Composes a NATS message for a pending PUBREL packet. The message includes an\n// internal header containing the PI for PUBREL/PUBCOMP.\n//\n// Example (PI:123):\n//\n//\t\tNATS/1.0\\r\\n\n//\t \tNmqtt-PubRel:123\\r\\n\n//\t\t\\r\\n\nfunc mqttNewDeliverablePubRel(pi uint16) (natsMsg []byte, headerLen int) {\n\tsize := len(hdrLine) +\n\t\tlen(mqttNatsPubRelHeader) + 6 + 2 + // 6 for ':65535', and 2 for CRLF\n\t\t2 // end-of-header CRLF\n\tbuf := bytes.NewBuffer(make([]byte, 0, size))\n\tbuf.WriteString(hdrLine)\n\tbuf.WriteString(mqttNatsPubRelHeader)\n\tbuf.WriteByte(':')\n\tbuf.WriteString(strconv.FormatInt(int64(pi), 10))\n\tbuf.WriteString(_CRLF_)\n\tbuf.WriteString(_CRLF_)\n\treturn buf.Bytes(), buf.Len()\n}\n\n// Process the PUBLISH packet.\n//\n// Runs from the client's readLoop.\n// No lock held on entry.\nfunc (s *Server) mqttProcessPub(c *client, pp *mqttPublish, trace bool) error {\n\tqos := mqttGetQoS(pp.flags)\n\n\t// Enforce max_payload using existing client max payload logic (mpay) by\n\t// checking the total NATS message size that would be processed.\n\tif maxPayload := atomic.LoadInt32(&c.mpay); maxPayload != jwt.NoLimit {\n\t\tif total := mqttComputeNatsMsgSize(pp, qos == 2); total > int(maxPayload) {\n\t\t\tc.maxPayloadViolation(total, maxPayload)\n\t\t\treturn ErrMaxPayload\n\t\t}\n\t}\n\n\tswitch qos {\n\tcase 0:\n\t\treturn s.mqttInitiateMsgDelivery(c, pp)\n\n\tcase 1:\n\t\t// [MQTT-4.3.2-2]. Initiate onward delivery of the Application Message,\n\t\t// Send PUBACK.\n\t\t//\n\t\t// The receiver is not required to complete delivery of the Application\n\t\t// Message before sending the PUBACK. When its original sender receives\n\t\t// the PUBACK packet, ownership of the Application Message is\n\t\t// transferred to the receiver.\n\t\terr := s.mqttInitiateMsgDelivery(c, pp)\n\t\tif err == nil {\n\t\t\tc.mqttEnqueuePubResponse(mqttPacketPubAck, pp.pi, trace)\n\t\t}\n\t\treturn err\n\n\tcase 2:\n\t\t// [MQTT-4.3.3-2]. Method A, Store message, send PUBREC.\n\t\t//\n\t\t// The receiver is not required to complete delivery of the Application\n\t\t// Message before sending the PUBREC or PUBCOMP. When its original\n\t\t// sender receives the PUBREC packet, ownership of the Application\n\t\t// Message is transferred to the receiver.\n\t\terr := s.mqttStoreQoS2MsgOnce(c, pp)\n\t\tif err == nil {\n\t\t\tc.mqttEnqueuePubResponse(mqttPacketPubRec, pp.pi, trace)\n\t\t}\n\t\treturn err\n\n\tdefault:\n\t\treturn fmt.Errorf(\"unreachable: invalid QoS in mqttProcessPub: %v\", qos)\n\t}\n}\n\nfunc (s *Server) mqttInitiateMsgDelivery(c *client, pp *mqttPublish) error {\n\tnatsMsg, headerLen := mqttNewDeliverableMessage(pp, false)\n\n\t// Set the client's pubarg for processing.\n\tc.pa.subject = pp.subject\n\tc.pa.mapped = pp.mapped\n\tc.pa.reply = nil\n\tc.pa.hdr = headerLen\n\tc.pa.hdb = []byte(strconv.FormatInt(int64(c.pa.hdr), 10))\n\tc.pa.size = len(natsMsg)\n\tc.pa.szb = []byte(strconv.FormatInt(int64(c.pa.size), 10))\n\tdefer func() {\n\t\tc.pa.subject = nil\n\t\tc.pa.mapped = nil\n\t\tc.pa.reply = nil\n\t\tc.pa.hdr = -1\n\t\tc.pa.hdb = nil\n\t\tc.pa.size = 0\n\t\tc.pa.szb = nil\n\t}()\n\n\t_, permIssue := c.processInboundClientMsg(natsMsg)\n\tif permIssue {\n\t\treturn nil\n\t}\n\n\t// If QoS 0 messages don't need to be stored, other (1 and 2) do. Store them\n\t// JetStream under \"$MQTT.msgs.\"\n\tif qos := mqttGetQoS(pp.flags); qos == 0 {\n\t\treturn nil\n\t}\n\n\t// We need to call flushClients now since this we may have called c.addToPCD\n\t// with destination clients (possibly a route). Without calling flushClients\n\t// the following call may then be stuck waiting for a reply that may never\n\t// come because the destination is not flushed (due to c.out.fsp > 0,\n\t// see addToPCD and writeLoop for details).\n\tc.flushClients(0)\n\n\t_, err := c.mqtt.sess.jsa.storeMsg(mqttStreamSubjectPrefix+string(c.pa.subject), headerLen, natsMsg)\n\n\treturn err\n}\n\nvar mqttMaxMsgErrPattern = fmt.Sprintf(\"%s (%v)\", ErrMaxMsgsPerSubject.Error(), JSStreamStoreFailedF)\n\nfunc (s *Server) mqttStoreQoS2MsgOnce(c *client, pp *mqttPublish) error {\n\t// `true` means encode the MQTT PUBLISH packet in the NATS message header.\n\tnatsMsg, headerLen := mqttNewDeliverableMessage(pp, true)\n\n\t// Do not broadcast the message until it has been deduplicated and released\n\t// by the sender. Instead store this QoS2 message as\n\t// \"$MQTT.qos2..\". If the message is a duplicate, we get back\n\t// a ErrMaxMsgsPerSubject, otherwise it does not change the flow, still need\n\t// to send a PUBREC back to the client. The original subject (translated\n\t// from MQTT topic) is included in the NATS header of the stored message to\n\t// use for latter delivery.\n\t_, err := c.mqtt.sess.jsa.storeMsg(c.mqttQoS2InternalSubject(pp.pi), headerLen, natsMsg)\n\n\t// TODO: would prefer a more robust and performant way of checking the\n\t// error, but it comes back wrapped as an API result.\n\tif err != nil &&\n\t\t(isErrorOtherThan(err, JSStreamStoreFailedF) || err.Error() != mqttMaxMsgErrPattern) {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *client) mqttQoS2InternalSubject(pi uint16) string {\n\treturn mqttQoS2IncomingMsgsStreamSubjectPrefix + c.mqtt.cid + \".\" + strconv.FormatUint(uint64(pi), 10)\n}\n\n// Process a PUBREL packet (QoS2, acting as Receiver).\n//\n// Runs from the client's readLoop.\n// No lock held on entry.\nfunc (s *Server) mqttProcessPubRel(c *client, pi uint16, trace bool) error {\n\t// Once done with the processing, send a PUBCOMP back to the client.\n\tdefer c.mqttEnqueuePubResponse(mqttPacketPubComp, pi, trace)\n\n\t// See if there is a message pending for this pi. All failures are treated\n\t// as \"not found\".\n\tasm := c.mqtt.asm\n\tstored, _ := asm.jsa.loadLastMsgFor(mqttQoS2IncomingMsgsStreamName, c.mqttQoS2InternalSubject(pi))\n\n\tif stored == nil {\n\t\t// No message found, nothing to do.\n\t\treturn nil\n\t}\n\t// Best attempt to delete the message from the QoS2 stream.\n\tasm.jsa.deleteMsg(mqttQoS2IncomingMsgsStreamName, stored.Sequence, true)\n\n\t// only MQTT QoS2 messages should be here, and they must have a subject.\n\th := mqttParsePublishNATSHeader(stored.Header)\n\tif h == nil || h.qos != 2 || len(h.subject) == 0 {\n\t\treturn errors.New(\"invalid message in QoS2 PUBREL stream\")\n\t}\n\n\tpp := &mqttPublish{\n\t\ttopic: natsSubjectToMQTTTopic(h.subject),\n\t\tsubject: h.subject,\n\t\tmapped: h.mapped,\n\t\tmsg: stored.Data,\n\t\tsz: len(stored.Data),\n\t\tpi: pi,\n\t\tflags: h.qos << 1,\n\t}\n\n\treturn s.mqttInitiateMsgDelivery(c, pp)\n}\n\n// Invoked when processing an inbound client message. If the \"retain\" flag is\n// set, the message is stored so it can be later resent to (re)starting\n// subscriptions that match the subject.\n//\n// Invoked from the MQTT publisher's readLoop. No client lock is held on entry.\nfunc (c *client) mqttHandlePubRetain() {\n\tpp := c.mqtt.pp\n\tretainMQTT := mqttIsRetained(pp.flags)\n\tisBirth, _, isCertificate := sparkbParseBirthDeathTopic(pp.topic)\n\tretainSparkbBirth := isBirth && !isCertificate\n\n\t// [tck-id-topics-nbirth-mqtt] NBIRTH messages MUST be published with MQTT\n\t// QoS equal to 0 and retain equal to false.\n\t//\n\t// [tck-id-conformance-mqtt-aware-nbirth-mqtt-retain] A Sparkplug Aware MQTT\n\t// Server MUST make NBIRTH messages available on the topic:\n\t// $sparkplug/certificates/namespace/group_id/NBIRTH/edge_node_id with the\n\t// MQTT retain flag set to true.\n\tif retainMQTT == retainSparkbBirth {\n\t\t// (retainSparkbBirth && retainMQTT) : not valid, so ignore altogether.\n\t\t// (!retainSparkbBirth && !retainMQTT) : nothing to do.\n\t\treturn\n\t}\n\n\tasm := c.mqtt.asm\n\tkey := string(pp.subject)\n\n\t// Always clear the retain flag to deliver a normal published message.\n\tdefer func() {\n\t\tpp.flags &= ^mqttPubFlagRetain\n\t}()\n\n\t// Spec [MQTT-3.3.1-11]. Payload of size 0 removes the retained message, but\n\t// should still be delivered as a normal message.\n\t//\n\t// We used to delete the message here from our map, the stream, and notify\n\t// the network about the delete. We no longer do that. Instead, we store\n\t// the message with an empty body. When servers will get the empty body\n\t// in processRetainedMsg, then will remove the message from their map. This\n\t// effectively serializes all add/remove of retained messages without the\n\t// need for \"network\" notifications about deletes (we still support that\n\t// for backward compatibility but will be pulled in future releases).\n\n\trm := &mqttRetainedMsg{\n\t\tOrigin: asm.jsa.id,\n\t\tMsg: pp.msg, // will copy these bytes later as we process rm.\n\t\tFlags: pp.flags,\n\t\tSource: c.opts.Username,\n\t}\n\n\tif retainSparkbBirth {\n\t\t// [tck-id-conformance-mqtt-aware-store] A Sparkplug Aware MQTT Server\n\t\t// MUST store NBIRTH and DBIRTH messages as they pass through the MQTT\n\t\t// Server.\n\t\t//\n\t\t// [tck-id-conformance-mqtt-aware-nbirth-mqtt-topic]. A Sparkplug Aware\n\t\t// MQTT Server MUST make NBIRTH messages available on a topic of the\n\t\t// form: $sparkplug/certificates/namespace/group_id/NBIRTH/edge_node_id\n\t\t//\n\t\t// [tck-id-conformance-mqtt-aware-dbirth-mqtt-topic] A Sparkplug Aware\n\t\t// MQTT Server MUST make DBIRTH messages available on a topic of the\n\t\t// form:\n\t\t// $sparkplug/certificates/namespace/group_id/DBIRTH/edge_node_id/device_id\n\t\ttopic := append(sparkbCertificatesTopicPrefix, pp.topic...)\n\t\tsubject, _ := mqttTopicToNATSPubSubject(topic)\n\t\trm.Topic = string(topic)\n\t\trm.Subject = string(subject)\n\n\t\t// will use to save the retained message.\n\t\tkey = string(subject)\n\n\t\t// Store the retained message with the RETAIN flag set.\n\t\trm.Flags |= mqttPubFlagRetain\n\n\t\tif pp.sz > 0 {\n\t\t\t// Copy the payload out of pp since we will be sending the message\n\t\t\t// asynchronously.\n\t\t\tmsg := make([]byte, pp.sz)\n\t\t\tcopy(msg, pp.msg[:pp.sz])\n\t\t\tasm.jsa.sendMsg(key, msg)\n\t\t}\n\n\t} else { // isRetained\n\t\t// Spec [MQTT-3.3.1-5]. Store the retained message with its QoS.\n\t\t//\n\t\t// When coming from a publish protocol, `pp` is referencing a stack\n\t\t// variable that itself possibly references the read buffer.\n\t\trm.Topic = string(pp.topic)\n\t}\n\n\t// Set the key to the subject of the message for retained, or the composed\n\t// $sparkplug subject for sparkB.\n\trm.Subject = key\n\trmBytes, hdr := mqttEncodeRetainedMessage(rm) // will copy the payload bytes\n\t_, err := asm.jsa.storeMsg(mqttRetainedMsgsStreamSubject+key, hdr, rmBytes)\n\tif err != nil {\n\t\tc.mu.Lock()\n\t\tacc := c.acc\n\t\tc.mu.Unlock()\n\t\tc.Errorf(\"unable to store retained message for account %q, subject %q: %v\",\n\t\t\tacc.GetName(), key, err)\n\t}\n}\n\n// After a config reload, it is possible that the source of a publish retained\n// message is no longer allowed to publish on the given topic. If that is the\n// case, the retained message is removed from the map and will no longer be\n// sent to (re)starting subscriptions.\n//\n// Server lock MUST NOT be held on entry.\nfunc (s *Server) mqttCheckPubRetainedPerms() {\n\tsm := &s.mqtt.sessmgr\n\tsm.mu.RLock()\n\tdone := len(sm.sessions) == 0\n\tsm.mu.RUnlock()\n\n\tif done {\n\t\treturn\n\t}\n\n\ts.mu.Lock()\n\tusers := make(map[string]*User, len(s.users))\n\tfor un, u := range s.users {\n\t\tusers[un] = u\n\t}\n\ts.mu.Unlock()\n\n\t// First get a list of all of the sessions.\n\tsm.mu.RLock()\n\tasms := make([]*mqttAccountSessionManager, 0, len(sm.sessions))\n\tfor _, asm := range sm.sessions {\n\t\tasms = append(asms, asm)\n\t}\n\tsm.mu.RUnlock()\n\n\ttype retainedMsg struct {\n\t\tsubj string\n\t\trmsg *mqttRetainedMsgRef\n\t}\n\n\t// For each session we will obtain a list of retained messages.\n\tvar _rms [128]retainedMsg\n\trms := _rms[:0]\n\tfor _, asm := range asms {\n\t\t// Get all of the retained messages. Then we will sort them so\n\t\t// that they are in sequence order, which should help the file\n\t\t// store to not have to load out-of-order blocks so often.\n\t\tasm.mu.RLock()\n\t\trms = rms[:0] // reuse slice\n\t\tfor subj, rf := range asm.retmsgs {\n\t\t\trms = append(rms, retainedMsg{\n\t\t\t\tsubj: subj,\n\t\t\t\trmsg: rf,\n\t\t\t})\n\t\t}\n\t\tjsaID := asm.jsa.id\n\t\tasm.mu.RUnlock()\n\t\tslices.SortFunc(rms, func(i, j retainedMsg) int { return cmp.Compare(i.rmsg.sseq, j.rmsg.sseq) })\n\n\t\tperms := map[string]*perm{}\n\t\tfor _, rf := range rms {\n\t\t\tjsm, err := asm.jsa.loadMsg(mqttRetainedMsgsStreamName, rf.rmsg.sseq)\n\t\t\tif err != nil || jsm == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trm, err := mqttDecodeRetainedMessage(jsm.Subject, jsm.Header, jsm.Data)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// We deal only with messages that have a source (the username that produced\n\t\t\t// this message) and were produced on this server.\n\t\t\tif rm.Source == _EMPTY_ || rm.Origin != jsaID {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Lookup source from global users.\n\t\t\tu := users[rm.Source]\n\t\t\tif u != nil {\n\t\t\t\tp, ok := perms[rm.Source]\n\t\t\t\tif !ok {\n\t\t\t\t\tp = generatePubPerms(u.Permissions)\n\t\t\t\t\tperms[rm.Source] = p // possibly nil\n\t\t\t\t}\n\t\t\t\t// If there is permission and no longer allowed to publish in\n\t\t\t\t// the subject, remove the publish retained message from the map.\n\t\t\t\tif p != nil && !pubAllowed(p, rf.subj) {\n\t\t\t\t\tu = nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Not present or permissions have changed such that the source can't\n\t\t\t// publish on that subject anymore: delete this retained message.\n\t\t\tif u == nil {\n\t\t\t\t// Set the payload to empty to notify that we are deleting this\n\t\t\t\t// retained message. We will send this message async.\n\t\t\t\trm.Msg = nil\n\t\t\t\trmBytes, hdrLen := mqttEncodeRetainedMessage(rm)\n\t\t\t\tasm.jsa.storeMsgNoWait(mqttRetainedMsgsStreamSubject+rm.Subject, hdrLen, rmBytes)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Helper to generate only pub permissions from a Permissions object\nfunc generatePubPerms(perms *Permissions) *perm {\n\t// If given permissions is `nil`, then it means that permissions block\n\t// has been removed (so the user is now allowed to publish on everything)\n\t// or was never there in the first place. Returning `nil` will let the\n\t// caller know that there are no permissions to enforce.\n\tif perms == nil {\n\t\treturn nil\n\t}\n\tvar p *perm\n\tif perms.Publish.Allow != nil {\n\t\tp = &perm{}\n\t\tp.allow = NewSublistWithCache()\n\t\tfor _, pubSubject := range perms.Publish.Allow {\n\t\t\tsub := &subscription{subject: []byte(pubSubject)}\n\t\t\tp.allow.Insert(sub)\n\t\t}\n\t}\n\tif len(perms.Publish.Deny) > 0 {\n\t\tif p == nil {\n\t\t\tp = &perm{}\n\t\t}\n\t\tp.deny = NewSublistWithCache()\n\t\tfor _, pubSubject := range perms.Publish.Deny {\n\t\t\tsub := &subscription{subject: []byte(pubSubject)}\n\t\t\tp.deny.Insert(sub)\n\t\t}\n\t}\n\treturn p\n}\n\n// Helper that checks if given `perms` allow to publish on the given `subject`\nfunc pubAllowed(perms *perm, subject string) bool {\n\tallowed := true\n\tif perms.allow != nil {\n\t\tnp, _ := perms.allow.NumInterest(subject)\n\t\tallowed = np != 0\n\t}\n\t// If we have a deny list and are currently allowed, check that as well.\n\tif allowed && perms.deny != nil {\n\t\tnp, _ := perms.deny.NumInterest(subject)\n\t\tallowed = np == 0\n\t}\n\treturn allowed\n}\n\nfunc (c *client) mqttEnqueuePubResponse(packetType byte, pi uint16, trace bool) {\n\tproto := [4]byte{packetType, 0x2, 0, 0}\n\tproto[2] = byte(pi >> 8)\n\tproto[3] = byte(pi)\n\n\t// Bits 3,2,1 and 0 of the fixed header in the PUBREL Control Packet are\n\t// reserved and MUST be set to 0,0,1 and 0 respectively. The Server MUST treat\n\t// any other value as malformed and close the Network Connection [MQTT-3.6.1-1].\n\tif packetType == mqttPacketPubRel {\n\t\tproto[0] |= 0x2\n\t}\n\n\tc.mu.Lock()\n\tc.enqueueProto(proto[:4])\n\tc.mu.Unlock()\n\n\tif trace {\n\t\tname := \"(???)\"\n\t\tswitch packetType {\n\t\tcase mqttPacketPubAck:\n\t\t\tname = \"PUBACK\"\n\t\tcase mqttPacketPubRec:\n\t\t\tname = \"PUBREC\"\n\t\tcase mqttPacketPubRel:\n\t\t\tname = \"PUBREL\"\n\t\tcase mqttPacketPubComp:\n\t\t\tname = \"PUBCOMP\"\n\t\t}\n\t\tc.traceOutOp(name, []byte(fmt.Sprintf(\"pi=%v\", pi)))\n\t}\n}\n\nfunc mqttParsePIPacket(r *mqttReader) (uint16, error) {\n\tpi, err := r.readUint16(\"packet identifier\")\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif pi == 0 {\n\t\treturn 0, errMQTTPacketIdentifierIsZero\n\t}\n\treturn pi, nil\n}\n\n// Process a PUBACK (QoS1) or a PUBREC (QoS2) packet, acting as Sender. Set\n// isPubRec to false to process as a PUBACK.\n//\n// Runs from the client's readLoop. No lock held on entry.\nfunc (c *client) mqttProcessPublishReceived(pi uint16, isPubRec bool) (err error) {\n\tsess := c.mqtt.sess\n\tif sess == nil {\n\t\treturn errMQTTInvalidSession\n\t}\n\n\tvar jsAckSubject string\n\tsess.mu.Lock()\n\t// Must be the same client, and the session must have been setup for QoS2.\n\tif sess.c != c {\n\t\tsess.mu.Unlock()\n\t\treturn errMQTTInvalidSession\n\t}\n\tif isPubRec {\n\t\t// The JS ACK subject for the PUBREL will be filled in at the delivery\n\t\t// attempt.\n\t\tsess.trackAsPubRel(pi, _EMPTY_)\n\t}\n\tjsAckSubject = sess.untrackPublish(pi)\n\tsess.mu.Unlock()\n\n\tif isPubRec {\n\t\tnatsMsg, headerLen := mqttNewDeliverablePubRel(pi)\n\t\t_, err = sess.jsa.storeMsg(sess.pubRelSubject, headerLen, natsMsg)\n\t\tif err != nil {\n\t\t\t// Failure to send out PUBREL will terminate the connection.\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Send the ack to JS to remove the pending message from the consumer.\n\tsess.jsa.sendAck(jsAckSubject)\n\treturn nil\n}\n\nfunc (c *client) mqttProcessPubAck(pi uint16) error {\n\treturn c.mqttProcessPublishReceived(pi, false)\n}\n\nfunc (c *client) mqttProcessPubRec(pi uint16) error {\n\treturn c.mqttProcessPublishReceived(pi, true)\n}\n\n// Runs from the client's readLoop. No lock held on entry.\nfunc (c *client) mqttProcessPubComp(pi uint16) {\n\tsess := c.mqtt.sess\n\tif sess == nil {\n\t\treturn\n\t}\n\n\tvar jsAckSubject string\n\tsess.mu.Lock()\n\tif sess.c != c {\n\t\tsess.mu.Unlock()\n\t\treturn\n\t}\n\tjsAckSubject = sess.untrackPubRel(pi)\n\tsess.mu.Unlock()\n\n\t// Send the ack to JS to remove the pending message from the consumer.\n\tsess.jsa.sendAck(jsAckSubject)\n}\n\n// Return the QoS from the given PUBLISH protocol's flags\nfunc mqttGetQoS(flags byte) byte {\n\treturn flags & mqttPubFlagQoS >> 1\n}\n\nfunc mqttIsRetained(flags byte) bool {\n\treturn flags&mqttPubFlagRetain != 0\n}\n\nfunc sparkbParseBirthDeathTopic(topic []byte) (isBirth, isDeath, isCertificate bool) {\n\tif bytes.HasPrefix(topic, sparkbCertificatesTopicPrefix) {\n\t\tisCertificate = true\n\t\ttopic = topic[len(sparkbCertificatesTopicPrefix):]\n\t}\n\tif !bytes.HasPrefix(topic, sparkbNamespaceTopicPrefix) {\n\t\treturn false, false, false\n\t}\n\ttopic = topic[len(sparkbNamespaceTopicPrefix):]\n\n\tparts := bytes.Split(topic, []byte{'/'})\n\tif len(parts) < 3 || len(parts) > 4 {\n\t\treturn false, false, false\n\t}\n\ttyp := bytesToString(parts[1])\n\tswitch typ {\n\tcase sparkbNBIRTH, sparkbDBIRTH:\n\t\tisBirth = true\n\tcase sparkbNDEATH, sparkbDDEATH:\n\t\tisDeath = true\n\tdefault:\n\t\treturn false, false, false\n\t}\n\treturn isBirth, isDeath, isCertificate\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// SUBSCRIBE related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (c *client) mqttParseSubs(r *mqttReader, b byte, pl int) (uint16, []*mqttFilter, error) {\n\treturn c.mqttParseSubsOrUnsubs(r, b, pl, true)\n}\n\nfunc (c *client) mqttParseSubsOrUnsubs(r *mqttReader, b byte, pl int, sub bool) (uint16, []*mqttFilter, error) {\n\tvar expectedFlag byte\n\tvar action string\n\tif sub {\n\t\texpectedFlag = mqttSubscribeFlags\n\t} else {\n\t\texpectedFlag = mqttUnsubscribeFlags\n\t\taction = \"un\"\n\t}\n\t// Spec [MQTT-3.8.1-1], [MQTT-3.10.1-1]\n\tif rf := b & 0xf; rf != expectedFlag {\n\t\treturn 0, nil, fmt.Errorf(\"wrong %ssubscribe reserved flags: %x\", action, rf)\n\t}\n\tpi, err := mqttParsePIPacket(r)\n\tif err != nil {\n\t\treturn 0, nil, err\n\t}\n\tend := r.pos + (pl - 2)\n\tvar filters []*mqttFilter\n\tfor r.pos < end {\n\t\t// Don't make a copy now because, this will happen during conversion\n\t\t// or when processing the sub.\n\t\ttopic, err := r.readBytes(\"topic filter\", false)\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\tif len(topic) == 0 {\n\t\t\treturn 0, nil, errMQTTTopicFilterCannotBeEmpty\n\t\t}\n\t\t// Spec [MQTT-3.8.3-1], [MQTT-3.10.3-1]\n\t\tif err := mqttValidateTopic(topic, \"topic filter\"); err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\tvar qos byte\n\t\t// We are going to report if we had an error during the conversion,\n\t\t// but we don't fail the parsing. When processing the sub, we will\n\t\t// have an error then, and the processing of subs code will send\n\t\t// the proper mqttSubAckFailure flag for this given subscription.\n\t\tfilter, err := mqttFilterToNATSSubject(topic)\n\t\tif err != nil {\n\t\t\tc.Errorf(\"invalid topic %q: %v\", topic, err)\n\t\t}\n\t\tif sub {\n\t\t\tqos, err = r.readByte(\"QoS\")\n\t\t\tif err != nil {\n\t\t\t\treturn 0, nil, err\n\t\t\t}\n\t\t\t// Spec [MQTT-3-8.3-4].\n\t\t\tif qos > 2 {\n\t\t\t\treturn 0, nil, fmt.Errorf(\"subscribe QoS value must be 0, 1 or 2, got %v\", qos)\n\t\t\t}\n\t\t}\n\t\tf := &mqttFilter{ttopic: topic, filter: string(filter), qos: qos}\n\t\tfilters = append(filters, f)\n\t}\n\t// Spec [MQTT-3.8.3-3], [MQTT-3.10.3-2]\n\tif len(filters) == 0 {\n\t\treturn 0, nil, fmt.Errorf(\"%ssubscribe protocol must contain at least 1 topic filter\", action)\n\t}\n\treturn pi, filters, nil\n}\n\nfunc mqttSubscribeTrace(pi uint16, filters []*mqttFilter) string {\n\tvar sep string\n\tsb := &strings.Builder{}\n\tsb.WriteString(\"[\")\n\tfor i, f := range filters {\n\t\tsb.WriteString(sep)\n\t\tsb.Write(f.ttopic)\n\t\tsb.WriteString(\" (\")\n\t\tsb.WriteString(f.filter)\n\t\tsb.WriteString(\") QoS=\")\n\t\tsb.WriteString(fmt.Sprintf(\"%v\", f.qos))\n\t\tif i == 0 {\n\t\t\tsep = \", \"\n\t\t}\n\t}\n\tsb.WriteString(fmt.Sprintf(\"] pi=%v\", pi))\n\treturn sb.String()\n}\n\n// For a MQTT QoS0 subscription, we create a single NATS subscription on the\n// actual subject, for instance \"foo.bar\".\n//\n// For a MQTT QoS1+ subscription, we create 2 subscriptions, one on \"foo.bar\"\n// (as for QoS0, but sub.mqtt.qos will be 1 or 2), and one on the subject\n// \"$MQTT.sub.\" which is the delivery subject of the JS durable consumer\n// with the filter subject \"$MQTT.msgs.foo.bar\".\n//\n// This callback delivers messages to the client as QoS0 messages, either\n// because: (a) they have been produced as MQTT QoS0 messages (and therefore\n// only this callback can receive them); (b) they are MQTT QoS1+ published\n// messages but this callback is for a subscription that is QoS0; or (c) the\n// published messages come from (other) NATS publishers on the subject.\n//\n// This callback must reject a message if it is known to be a QoS1+ published\n// message and this is the callback for a QoS1+ subscription because in that\n// case, it will be handled by the other callback. This avoid getting duplicate\n// deliveries.\nfunc mqttDeliverMsgCbQoS0(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif pc.kind == JETSTREAM && len(reply) > 0 && strings.HasPrefix(reply, jsAckPre) {\n\t\treturn\n\t}\n\n\t// This is the client associated with the subscription.\n\tcc := sub.client\n\n\t// This is immutable\n\tsess := cc.mqtt.sess\n\n\t// Lock here, otherwise we may be called with sub.mqtt == nil. Ignore\n\t// wildcard subscriptions if this subject starts with '$', per Spec\n\t// [MQTT-4.7.2-1].\n\tsess.subsMu.RLock()\n\tsubQoS := sub.mqtt.qos\n\tignore := mqttMustIgnoreForReservedSub(sub, subject)\n\tsess.subsMu.RUnlock()\n\n\tif ignore {\n\t\treturn\n\t}\n\n\thdr, msg := pc.msgParts(rmsg)\n\tvar topic []byte\n\tif pc.isMqtt() {\n\t\t// This is an MQTT publisher directly connected to this server.\n\n\t\t// Check the subscription's QoS. If the message was published with a\n\t\t// QoS>0 and the sub has the QoS>0 then the message will be delivered by\n\t\t// mqttDeliverMsgCbQoS12.\n\t\tmsgQoS := mqttGetQoS(pc.mqtt.pp.flags)\n\t\tif subQoS > 0 && msgQoS > 0 {\n\t\t\treturn\n\t\t}\n\t\ttopic = pc.mqtt.pp.topic\n\t\t// If the subject is different than the one in pp.subject, then some\n\t\t// mapping/transform occurred and we need to recreate the topic.\n\t\tif subject != bytesToString(pc.mqtt.pp.subject) {\n\t\t\ttopic = natsSubjectStrToMQTTTopic(subject)\n\t\t}\n\n\t} else {\n\t\t// Non MQTT client, could be NATS publisher, or ROUTER, etc..\n\t\th := mqttParsePublishNATSHeader(hdr)\n\n\t\t// Check the subscription's QoS. If the message was published with a\n\t\t// QoS>0 (in the header) and the sub has the QoS>0 then the message will\n\t\t// be delivered by mqttDeliverMsgCbQoS12.\n\t\tif subQoS > 0 && h != nil && h.qos > 0 {\n\t\t\treturn\n\t\t}\n\n\t\t// If size is more than what a MQTT client can handle, we should probably reject,\n\t\t// for now just truncate.\n\t\tif len(msg) > mqttMaxPayloadSize {\n\t\t\tmsg = msg[:mqttMaxPayloadSize]\n\t\t}\n\t\ttopic = natsSubjectStrToMQTTTopic(subject)\n\t}\n\n\t// Message never has a packet identifier nor is marked as duplicate.\n\tpc.mqttEnqueuePublishMsgTo(cc, sub, 0, 0, false, topic, msg)\n}\n\n// This is the callback attached to a JS durable subscription for a MQTT QoS 1+\n// sub. Only JETSTREAM should be sending a message to this subject (the delivery\n// subject associated with the JS durable consumer), but in cluster mode, this\n// can be coming from a route, gw, etc... We make sure that if this is the case,\n// the message contains a NATS/MQTT header that indicates that this is a\n// published QoS1+ message.\nfunc mqttDeliverMsgCbQoS12(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) {\n\t// Message on foo.bar is stored under $MQTT.msgs.foo.bar, so the subject has to be\n\t// at least as long as the stream subject prefix \"$MQTT.msgs.\", and after removing\n\t// the prefix, has to be at least 1 character long.\n\tif len(subject) < len(mqttStreamSubjectPrefix)+1 {\n\t\treturn\n\t}\n\n\thdr, msg := pc.msgParts(rmsg)\n\th := mqttParsePublishNATSHeader(hdr)\n\tif pc.kind != JETSTREAM && (h == nil || h.qos == 0) {\n\t\t// MQTT QoS 0 messages must be ignored, they will be delivered by the\n\t\t// other callback, the direct NATS subscription. All JETSTREAM messages\n\t\t// will have the header.\n\t\treturn\n\t}\n\n\t// This is the client associated with the subscription.\n\tcc := sub.client\n\n\t// This is immutable\n\tsess := cc.mqtt.sess\n\n\t// We lock to check some of the subscription's fields and if we need to keep\n\t// track of pending acks, etc. There is no need to acquire the subsMu RLock\n\t// since sess.Lock is overarching for modifying subscriptions.\n\tsess.mu.Lock()\n\tif sess.c != cc || sub.mqtt == nil {\n\t\tsess.mu.Unlock()\n\t\treturn\n\t}\n\n\t// In this callback we handle only QoS-published messages to QoS\n\t// subscriptions. Ignore if either is 0, will be delivered by the other\n\t// callback, mqttDeliverMsgCbQos1.\n\tvar qos byte\n\tif h != nil {\n\t\tqos = h.qos\n\t}\n\tif qos > sub.mqtt.qos {\n\t\tqos = sub.mqtt.qos\n\t}\n\tif qos == 0 {\n\t\tsess.mu.Unlock()\n\t\treturn\n\t}\n\n\t// Check for reserved subject violation. If so, we will send the ack to\n\t// remove the message, and do nothing else.\n\tstrippedSubj := subject[len(mqttStreamSubjectPrefix):]\n\tif mqttMustIgnoreForReservedSub(sub, strippedSubj) {\n\t\tsess.mu.Unlock()\n\t\tsess.jsa.sendAck(reply)\n\t\treturn\n\t}\n\n\tpi, dup := sess.trackPublish(sub.mqtt.jsDur, reply)\n\tsess.mu.Unlock()\n\n\tif pi == 0 {\n\t\t// We have reached max pending, don't send the message now.\n\t\t// JS will cause a redelivery and if by then the number of pending\n\t\t// messages has fallen below threshold, the message will be resent.\n\t\treturn\n\t}\n\n\toriginalTopic := natsSubjectStrToMQTTTopic(strippedSubj)\n\tpc.mqttEnqueuePublishMsgTo(cc, sub, pi, qos, dup, originalTopic, msg)\n}\n\nfunc mqttDeliverPubRelCb(sub *subscription, pc *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif sub.client.mqtt == nil || sub.client.mqtt.sess == nil || reply == _EMPTY_ {\n\t\treturn\n\t}\n\n\thdr, _ := pc.msgParts(rmsg)\n\tpi := mqttParsePubRelNATSHeader(hdr)\n\tif pi == 0 {\n\t\treturn\n\t}\n\n\t// This is the client associated with the subscription.\n\tcc := sub.client\n\n\t// This is immutable\n\tsess := cc.mqtt.sess\n\n\tsess.mu.Lock()\n\tif sess.c != cc || sess.pubRelConsumer == nil {\n\t\tsess.mu.Unlock()\n\t\treturn\n\t}\n\tsess.trackAsPubRel(pi, reply)\n\ttrace := cc.trace\n\tsess.mu.Unlock()\n\n\tcc.mqttEnqueuePubResponse(mqttPacketPubRel, pi, trace)\n}\n\n// The MQTT Server MUST NOT match Topic Filters starting with a wildcard\n// character (# or +) with Topic Names beginning with a $ character, Spec\n// [MQTT-4.7.2-1]. We will return true if there is a violation.\n//\n// Session or subMu lock must be held on entry to protect access to sub.mqtt.\nfunc mqttMustIgnoreForReservedSub(sub *subscription, subject string) bool {\n\t// If the subject does not start with $ nothing to do here.\n\tif !sub.mqtt.reserved || len(subject) == 0 || subject[0] != mqttReservedPre {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Check if a sub is a reserved wildcard. E.g. '#', '*', or '*/\" prefix.\nfunc isMQTTReservedSubscription(subject string) bool {\n\tif len(subject) == 1 && (subject[0] == fwc || subject[0] == pwc) {\n\t\treturn true\n\t}\n\t// Match \"*.<>\"\n\tif len(subject) > 1 && (subject[0] == pwc && subject[1] == btsep) {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc sparkbReplaceDeathTimestamp(msg []byte) []byte {\n\tconst VARINT = 0\n\tconst TIMESTAMP = 1\n\n\torig := msg\n\tbuf := bytes.NewBuffer(make([]byte, 0, len(msg)+16)) // 16 bytes should be enough if we need to add a timestamp\n\twriteDeathTimestamp := func() {\n\t\t// [tck-id-conformance-mqtt-aware-ndeath-timestamp] A Sparkplug Aware\n\t\t// MQTT Server MAY replace the timestamp of NDEATH messages. If it does,\n\t\t// it MUST set the timestamp to the UTC time at which it attempts to\n\t\t// deliver the NDEATH to subscribed clients\n\t\t//\n\t\t// sparkB spec: 6.4.1. Google Protocol Buffer Schema\n\t\t// optional uint64 timestamp = 1; // Timestamp at message sending time\n\t\t//\n\t\t// SparkplugB timestamps are milliseconds since epoch, represented as\n\t\t// uint64 in go, transmitted as protobuf varint.\n\t\tts := uint64(time.Now().UnixMilli())\n\t\tbuf.Write(protoEncodeVarint(TIMESTAMP<<3 | VARINT))\n\t\tbuf.Write(protoEncodeVarint(ts))\n\t}\n\n\tfor len(msg) > 0 {\n\t\tfieldNumericID, fieldType, size, err := protoScanField(msg)\n\t\tif err != nil {\n\t\t\treturn orig\n\t\t}\n\t\tif fieldType != VARINT || fieldNumericID != TIMESTAMP {\n\t\t\t// Add the field as is\n\t\t\tbuf.Write(msg[:size])\n\t\t\tmsg = msg[size:]\n\t\t\tcontinue\n\t\t}\n\n\t\twriteDeathTimestamp()\n\n\t\t// Add the rest of the message as is, we are done\n\t\tbuf.Write(msg[size:])\n\t\treturn buf.Bytes()\n\t}\n\n\t// Add timestamp if we did not find one.\n\twriteDeathTimestamp()\n\n\treturn buf.Bytes()\n}\n\n// Common function to mqtt delivery callbacks to serialize and send the message\n// to the `cc` client.\nfunc (c *client) mqttEnqueuePublishMsgTo(cc *client, sub *subscription, pi uint16, qos byte, dup bool, topic, msg []byte) {\n\t// [tck-id-conformance-mqtt-aware-nbirth-mqtt-retain] A Sparkplug Aware\n\t// MQTT Server MUST make NBIRTH messages available on the topic:\n\t// $sparkplug/certificates/namespace/group_id/NBIRTH/edge_node_id with\n\t// the MQTT retain flag set to true\n\t//\n\t// [tck-id-conformance-mqtt-aware-dbirth-mqtt-retain] A Sparkplug Aware\n\t// MQTT Server MUST make DBIRTH messages available on the topic:\n\t// $sparkplug/certificates/namespace/group_id/DBIRTH/edge_node_id/device_id\n\t// with the MQTT retain flag set to true\n\t//\n\t// $sparkplug/certificates messages are sent as NATS messages, so we\n\t// need to add the retain flag when sending them to MQTT clients.\n\n\tretain := false\n\tisBirth, isDeath, isCertificate := sparkbParseBirthDeathTopic(topic)\n\tif isBirth && qos == 0 {\n\t\tretain = isCertificate\n\t} else if isDeath && !isCertificate {\n\t\tmsg = sparkbReplaceDeathTimestamp(msg)\n\t}\n\n\tflags, headerBytes := mqttMakePublishHeader(pi, qos, dup, retain, topic, len(msg))\n\n\tcc.mu.Lock()\n\tif sub.mqtt.prm != nil {\n\t\tfor _, data := range sub.mqtt.prm {\n\t\t\tcc.queueOutbound(data)\n\t\t}\n\t\tsub.mqtt.prm = nil\n\t}\n\tcc.queueOutbound(headerBytes)\n\tcc.queueOutbound(msg)\n\tc.addToPCD(cc)\n\ttrace := cc.trace\n\tcc.mu.Unlock()\n\n\tif trace {\n\t\tpp := mqttPublish{\n\t\t\ttopic: topic,\n\t\t\tflags: flags,\n\t\t\tpi: pi,\n\t\t\tsz: len(msg),\n\t\t}\n\t\tcc.traceOutOp(\"PUBLISH\", []byte(mqttPubTrace(&pp)))\n\t}\n}\n\n// Serializes to the given writer the message for the given subject.\nfunc (w *mqttWriter) WritePublishHeader(pi uint16, qos byte, dup, retained bool, topic []byte, msgLen int) byte {\n\t// Compute len (will have to add packet id if message is sent as QoS>=1)\n\tpkLen := 2 + len(topic) + msgLen\n\tvar flags byte\n\n\t// Set flags for dup/retained/qos1\n\tif dup {\n\t\tflags |= mqttPubFlagDup\n\t}\n\tif retained {\n\t\tflags |= mqttPubFlagRetain\n\t}\n\tif qos > 0 {\n\t\tpkLen += 2\n\t\tflags |= qos << 1\n\t}\n\n\tw.WriteByte(mqttPacketPub | flags)\n\tw.WriteVarInt(pkLen)\n\tw.WriteBytes(topic)\n\tif qos > 0 {\n\t\tw.WriteUint16(pi)\n\t}\n\n\treturn flags\n}\n\n// Serializes to the given writer the message for the given subject.\nfunc mqttMakePublishHeader(pi uint16, qos byte, dup, retained bool, topic []byte, msgLen int) (byte, []byte) {\n\theaderBuf := newMQTTWriter(mqttInitialPubHeader + len(topic))\n\tflags := headerBuf.WritePublishHeader(pi, qos, dup, retained, topic, msgLen)\n\treturn flags, headerBuf.Bytes()\n}\n\n// Process the SUBSCRIBE packet.\n//\n// Process the list of subscriptions and update the given filter\n// with the QoS that has been accepted (or failure).\n//\n// Spec [MQTT-3.8.4-3] says that if an exact same subscription is\n// found, it needs to be replaced with the new one (possibly updating\n// the qos) and that the flow of publications must not be interrupted,\n// which I read as the replacement cannot be a \"remove then add\" if there\n// is a chance that in between the 2 actions, published messages\n// would be \"lost\" because there would not be any matching subscription.\n//\n// Run from client's readLoop.\n// No lock held on entry.\nfunc (c *client) mqttProcessSubs(filters []*mqttFilter) ([]*subscription, error) {\n\t// Those things are immutable, but since processing subs is not\n\t// really in the fast path, let's get them under the client lock.\n\tc.mu.Lock()\n\tasm := c.mqtt.asm\n\tsess := c.mqtt.sess\n\ttrace := c.trace\n\tc.mu.Unlock()\n\n\tif err := asm.lockSession(sess, c); err != nil {\n\t\treturn nil, err\n\t}\n\tdefer asm.unlockSession(sess)\n\treturn asm.processSubs(sess, c, filters, true, trace)\n}\n\n// Cleanup that is performed in processSubs if there was an error.\n//\n// Runs from client's readLoop.\n// Lock not held on entry, but session is in the locked map.\nfunc (sess *mqttSession) cleanupFailedSub(c *client, sub *subscription, cc *ConsumerConfig, jssub *subscription) {\n\tif sub != nil {\n\t\tc.processUnsub(sub.sid)\n\t}\n\tif jssub != nil {\n\t\tc.processUnsub(jssub.sid)\n\t}\n\tif cc != nil {\n\t\tsess.deleteConsumer(cc)\n\t}\n}\n\n// Make sure we are set up to deliver PUBREL messages to this QoS2-subscribed\n// session.\nfunc (sess *mqttSession) ensurePubRelConsumerSubscription(c *client) error {\n\n\tsess.mu.Lock()\n\tpubRelSubscribed := sess.pubRelSubscribed\n\tpubRelDeliverySubjectB := sess.pubRelDeliverySubjectB\n\tpubRelDeliverySubject := sess.pubRelDeliverySubject\n\tpubRelConsumer := sess.pubRelConsumer\n\tsess.mu.Unlock()\n\n\t// Subscribe before the consumer is created so we don't loose any messages.\n\tif !pubRelSubscribed {\n\t\t_, err := c.processSub(pubRelDeliverySubjectB, nil, pubRelDeliverySubjectB, mqttDeliverPubRelCb, false)\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Unable to create subscription for JetStream consumer on %q: %v\", pubRelDeliverySubject, err)\n\t\t\treturn err\n\t\t}\n\t\tsess.mu.Lock()\n\t\tsess.pubRelSubscribed = true\n\t\tsess.mu.Unlock()\n\t}\n\n\t// If the JS consumer already exists, we are done.\n\tif pubRelConsumer != nil {\n\t\treturn nil\n\t}\n\n\topts := c.srv.getOpts()\n\tackWait := opts.MQTT.AckWait\n\tif ackWait == 0 {\n\t\tackWait = mqttDefaultAckWait\n\t}\n\tmaxAckPending := int(opts.MQTT.MaxAckPending)\n\tif maxAckPending == 0 {\n\t\tmaxAckPending = mqttDefaultMaxAckPending\n\t}\n\n\tsess.mu.Lock()\n\tpubRelSubject := sess.pubRelSubject\n\ttmaxack := sess.tmaxack\n\tidHash := sess.idHash\n\tid := sess.id\n\tsess.mu.Unlock()\n\n\t// Check that the limit of subs' maxAckPending are not going over the limit\n\tif after := tmaxack + maxAckPending; after > mqttMaxAckTotalLimit {\n\t\treturn fmt.Errorf(\"max_ack_pending for all consumers would be %v which exceeds the limit of %v\",\n\t\t\tafter, mqttMaxAckTotalLimit)\n\t}\n\n\tccr := &CreateConsumerRequest{\n\t\tStream: mqttOutStreamName,\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: pubRelDeliverySubject,\n\t\t\tDurable: mqttPubRelConsumerDurablePrefix + idHash,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tDeliverPolicy: DeliverNew,\n\t\t\tFilterSubject: pubRelSubject,\n\t\t\tAckWait: ackWait,\n\t\t\tMaxAckPending: maxAckPending,\n\t\t\tMemoryStorage: opts.MQTT.ConsumerMemoryStorage,\n\t\t},\n\t}\n\tif opts.MQTT.ConsumerInactiveThreshold > 0 {\n\t\tccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold\n\t}\n\tif _, err := sess.jsa.createDurableConsumer(ccr); err != nil {\n\t\tc.Errorf(\"Unable to add JetStream consumer for PUBREL for client %q: err=%v\", id, err)\n\t\treturn err\n\t}\n\tpubRelConsumer = &ccr.Config\n\ttmaxack += maxAckPending\n\n\tsess.mu.Lock()\n\tsess.pubRelConsumer = pubRelConsumer\n\tsess.tmaxack = tmaxack\n\tsess.mu.Unlock()\n\n\treturn nil\n}\n\n// When invoked with a QoS of 0, looks for an existing JS durable consumer for\n// the given sid and if one is found, delete the JS durable consumer and unsub\n// the NATS subscription on the delivery subject.\n//\n// With a QoS > 0, creates or update the existing JS durable consumer along with\n// its NATS subscription on a delivery subject.\n//\n// Session lock is acquired and released as needed. Session is in the locked\n// map.\nfunc (sess *mqttSession) processJSConsumer(c *client, subject, sid string,\n\tqos byte, fromSubProto bool) (*ConsumerConfig, *subscription, error) {\n\n\tsess.mu.Lock()\n\tcc, exists := sess.cons[sid]\n\ttmaxack := sess.tmaxack\n\tidHash := sess.idHash\n\tsess.mu.Unlock()\n\n\t// Check if we are already a JS consumer for this SID.\n\tif exists {\n\t\t// If current QoS is 0, it means that we need to delete the existing\n\t\t// one (that was QoS > 0)\n\t\tif qos == 0 {\n\t\t\t// The JS durable consumer's delivery subject is on a NUID of\n\t\t\t// the form: mqttSubPrefix + . It is also used as the sid\n\t\t\t// for the NATS subscription, so use that for the lookup.\n\t\t\tc.mu.Lock()\n\t\t\tsub := c.subs[cc.DeliverSubject]\n\t\t\tc.mu.Unlock()\n\n\t\t\tsess.mu.Lock()\n\t\t\tdelete(sess.cons, sid)\n\t\t\tsess.mu.Unlock()\n\n\t\t\tsess.deleteConsumer(cc)\n\t\t\tif sub != nil {\n\t\t\t\tc.processUnsub(sub.sid)\n\t\t\t}\n\t\t\treturn nil, nil, nil\n\t\t}\n\t\t// If this is called when processing SUBSCRIBE protocol, then if\n\t\t// the JS consumer already exists, we are done (it was created\n\t\t// during the processing of CONNECT).\n\t\tif fromSubProto {\n\t\t\treturn nil, nil, nil\n\t\t}\n\t}\n\t// Here it means we don't have a JS consumer and if we are QoS 0,\n\t// we have nothing to do.\n\tif qos == 0 {\n\t\treturn nil, nil, nil\n\t}\n\tvar err error\n\tvar inbox string\n\tif exists {\n\t\tinbox = cc.DeliverSubject\n\t} else {\n\t\tinbox = mqttSubPrefix + nuid.Next()\n\t\topts := c.srv.getOpts()\n\t\tackWait := opts.MQTT.AckWait\n\t\tif ackWait == 0 {\n\t\t\tackWait = mqttDefaultAckWait\n\t\t}\n\t\tmaxAckPending := int(opts.MQTT.MaxAckPending)\n\t\tif maxAckPending == 0 {\n\t\t\tmaxAckPending = mqttDefaultMaxAckPending\n\t\t}\n\n\t\t// Check that the limit of subs' maxAckPending are not going over the limit\n\t\tif after := tmaxack + maxAckPending; after > mqttMaxAckTotalLimit {\n\t\t\treturn nil, nil, fmt.Errorf(\"max_ack_pending for all consumers would be %v which exceeds the limit of %v\",\n\t\t\t\tafter, mqttMaxAckTotalLimit)\n\t\t}\n\n\t\tdurName := idHash + \"_\" + nuid.Next()\n\t\tccr := &CreateConsumerRequest{\n\t\t\tStream: mqttStreamName,\n\t\t\tConfig: ConsumerConfig{\n\t\t\t\tDeliverSubject: inbox,\n\t\t\t\tDurable: durName,\n\t\t\t\tAckPolicy: AckExplicit,\n\t\t\t\tDeliverPolicy: DeliverNew,\n\t\t\t\tFilterSubject: mqttStreamSubjectPrefix + subject,\n\t\t\t\tAckWait: ackWait,\n\t\t\t\tMaxAckPending: maxAckPending,\n\t\t\t\tMemoryStorage: opts.MQTT.ConsumerMemoryStorage,\n\t\t\t},\n\t\t}\n\t\tif opts.MQTT.ConsumerInactiveThreshold > 0 {\n\t\t\tccr.Config.InactiveThreshold = opts.MQTT.ConsumerInactiveThreshold\n\t\t}\n\t\tif _, err := sess.jsa.createDurableConsumer(ccr); err != nil {\n\t\t\tc.Errorf(\"Unable to add JetStream consumer for subscription on %q: err=%v\", subject, err)\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tcc = &ccr.Config\n\t\ttmaxack += maxAckPending\n\t}\n\n\t// This is an internal subscription on subject like \"$MQTT.sub.\" that is setup\n\t// for the JS durable's deliver subject.\n\tsess.mu.Lock()\n\tsess.tmaxack = tmaxack\n\tsub, err := sess.processQOS12Sub(c, []byte(inbox), []byte(inbox),\n\t\tisMQTTReservedSubscription(subject), qos, cc.Durable, mqttDeliverMsgCbQoS12)\n\tsess.mu.Unlock()\n\n\tif err != nil {\n\t\tsess.deleteConsumer(cc)\n\t\tc.Errorf(\"Unable to create subscription for JetStream consumer on %q: %v\", subject, err)\n\t\treturn nil, nil, err\n\t}\n\treturn cc, sub, nil\n}\n\n// Queues the published retained messages for each subscription and signals\n// the writeLoop.\nfunc (c *client) mqttSendRetainedMsgsToNewSubs(subs []*subscription) {\n\tc.mu.Lock()\n\tfor _, sub := range subs {\n\t\tif sub.mqtt != nil && sub.mqtt.prm != nil {\n\t\t\tfor _, data := range sub.mqtt.prm {\n\t\t\t\tc.queueOutbound(data)\n\t\t\t}\n\t\t\tsub.mqtt.prm = nil\n\t\t}\n\t}\n\tc.flushSignal()\n\tc.mu.Unlock()\n}\n\nfunc (c *client) mqttEnqueueSubAck(pi uint16, filters []*mqttFilter) {\n\tw := newMQTTWriter(7 + len(filters))\n\tw.WriteByte(mqttPacketSubAck)\n\t// packet length is 2 (for packet identifier) and 1 byte per filter.\n\tw.WriteVarInt(2 + len(filters))\n\tw.WriteUint16(pi)\n\tfor _, f := range filters {\n\t\tw.WriteByte(f.qos)\n\t}\n\tc.mu.Lock()\n\tc.enqueueProto(w.Bytes())\n\tc.mu.Unlock()\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// UNSUBSCRIBE related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (c *client) mqttParseUnsubs(r *mqttReader, b byte, pl int) (uint16, []*mqttFilter, error) {\n\treturn c.mqttParseSubsOrUnsubs(r, b, pl, false)\n}\n\n// Process the UNSUBSCRIBE packet.\n//\n// Given the list of topics, this is going to unsubscribe the low level NATS subscriptions\n// and delete the JS durable consumers when applicable.\n//\n// Runs from the client's readLoop.\n// No lock held on entry.\nfunc (c *client) mqttProcessUnsubs(filters []*mqttFilter) error {\n\t// Those things are immutable, but since processing unsubs is not\n\t// really in the fast path, let's get them under the client lock.\n\tc.mu.Lock()\n\tasm := c.mqtt.asm\n\tsess := c.mqtt.sess\n\tc.mu.Unlock()\n\n\tif err := asm.lockSession(sess, c); err != nil {\n\t\treturn err\n\t}\n\tdefer asm.unlockSession(sess)\n\n\tremoveJSCons := func(sid string) {\n\t\tcc, ok := sess.cons[sid]\n\t\tif ok {\n\t\t\tdelete(sess.cons, sid)\n\t\t\tsess.deleteConsumer(cc)\n\t\t\t// Need lock here since these are accessed by callbacks\n\t\t\tsess.mu.Lock()\n\t\t\tif seqPis, ok := sess.cpending[cc.Durable]; ok {\n\t\t\t\tdelete(sess.cpending, cc.Durable)\n\t\t\t\tfor _, pi := range seqPis {\n\t\t\t\t\tdelete(sess.pendingPublish, pi)\n\t\t\t\t}\n\t\t\t\tif len(sess.pendingPublish) == 0 {\n\t\t\t\t\tsess.last_pi = 0\n\t\t\t\t}\n\t\t\t}\n\t\t\tsess.mu.Unlock()\n\t\t}\n\t}\n\tfor _, f := range filters {\n\t\tsid := f.filter\n\t\t// Remove JS Consumer if one exists for this sid\n\t\tremoveJSCons(sid)\n\t\tif err := c.processUnsub([]byte(sid)); err != nil {\n\t\t\tc.Errorf(\"error unsubscribing from %q: %v\", sid, err)\n\t\t}\n\t\tif mqttNeedSubForLevelUp(sid) {\n\t\t\tsubject := sid[:len(sid)-2]\n\t\t\tsid = subject + mqttMultiLevelSidSuffix\n\t\t\tremoveJSCons(sid)\n\t\t\tif err := c.processUnsub([]byte(sid)); err != nil {\n\t\t\t\tc.Errorf(\"error unsubscribing from %q: %v\", subject, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn sess.update(filters, false)\n}\n\nfunc (c *client) mqttEnqueueUnsubAck(pi uint16) {\n\tw := newMQTTWriter(4)\n\tw.WriteByte(mqttPacketUnsubAck)\n\tw.WriteVarInt(2)\n\tw.WriteUint16(pi)\n\tc.mu.Lock()\n\tc.enqueueProto(w.Bytes())\n\tc.mu.Unlock()\n}\n\nfunc mqttUnsubscribeTrace(pi uint16, filters []*mqttFilter) string {\n\tvar sep string\n\tsb := strings.Builder{}\n\tsb.WriteString(\"[\")\n\tfor i, f := range filters {\n\t\tsb.WriteString(sep)\n\t\tsb.Write(f.ttopic)\n\t\tsb.WriteString(\" (\")\n\t\tsb.WriteString(f.filter)\n\t\tsb.WriteString(\")\")\n\t\tif i == 0 {\n\t\t\tsep = \", \"\n\t\t}\n\t}\n\tsb.WriteString(fmt.Sprintf(\"] pi=%v\", pi))\n\treturn sb.String()\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// PINGREQ/PINGRESP related functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (c *client) mqttEnqueuePingResp() {\n\tc.mu.Lock()\n\tc.enqueueProto(mqttPingResponse)\n\tc.mu.Unlock()\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// Trace functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc errOrTrace(err error, trace string) []byte {\n\tif err != nil {\n\t\treturn []byte(err.Error())\n\t}\n\treturn []byte(trace)\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// Subject/Topic conversion functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\n// Converts an MQTT Topic Name to a NATS Subject (used by PUBLISH)\n// See mqttToNATSSubjectConversion() for details.\nfunc mqttTopicToNATSPubSubject(mt []byte) ([]byte, error) {\n\treturn mqttToNATSSubjectConversion(mt, false)\n}\n\n// Converts an MQTT Topic Filter to a NATS Subject (used by SUBSCRIBE)\n// See mqttToNATSSubjectConversion() for details.\nfunc mqttFilterToNATSSubject(filter []byte) ([]byte, error) {\n\treturn mqttToNATSSubjectConversion(filter, true)\n}\n\n// Converts an MQTT Topic Name or Filter to a NATS Subject.\n// In MQTT:\n// - a Topic Name does not have wildcard (PUBLISH uses only topic names).\n// - a Topic Filter can include wildcards (SUBSCRIBE uses those).\n// - '+' and '#' are wildcard characters (single and multiple levels respectively)\n// - '/' is the topic level separator.\n//\n// Conversion that occurs:\n// - '/' is replaced with '/.' if it is the first character in mt\n// - '/' is replaced with './' if the last or next character in mt is '/'\n// For instance, foo//bar would become foo./.bar\n// - '/' is replaced with '.' for all other conditions (foo/bar -> foo.bar)\n// - '.' is replaced with '//'.\n// - ' ' cause an error to be returned.\n//\n// If there is no need to convert anything (say \"foo\" remains \"foo\"), then\n// the no memory is allocated and the returned slice is the original `mt`.\nfunc mqttToNATSSubjectConversion(mt []byte, wcOk bool) ([]byte, error) {\n\tvar cp bool\n\tvar j int\n\tres := mt\n\n\tmakeCopy := func(i int) {\n\t\tcp = true\n\t\tres = make([]byte, 0, len(mt)+10)\n\t\tif i > 0 {\n\t\t\tres = append(res, mt[:i]...)\n\t\t}\n\t}\n\n\tend := len(mt) - 1\n\tfor i := 0; i < len(mt); i++ {\n\t\tswitch mt[i] {\n\t\tcase mqttTopicLevelSep:\n\t\t\tif i == 0 || res[j-1] == btsep {\n\t\t\t\tif !cp {\n\t\t\t\t\tmakeCopy(0)\n\t\t\t\t}\n\t\t\t\tres = append(res, mqttTopicLevelSep, btsep)\n\t\t\t\tj++\n\t\t\t} else if i == end || mt[i+1] == mqttTopicLevelSep {\n\t\t\t\tif !cp {\n\t\t\t\t\tmakeCopy(i)\n\t\t\t\t}\n\t\t\t\tres = append(res, btsep, mqttTopicLevelSep)\n\t\t\t\tj++\n\t\t\t} else {\n\t\t\t\tif !cp {\n\t\t\t\t\tmakeCopy(i)\n\t\t\t\t}\n\t\t\t\tres = append(res, btsep)\n\t\t\t}\n\t\tcase ' ':\n\t\t\t// As of now, we cannot support ' ' in the MQTT topic/filter.\n\t\t\treturn nil, errMQTTUnsupportedCharacters\n\t\tcase btsep:\n\t\t\tif !cp {\n\t\t\t\tmakeCopy(i)\n\t\t\t}\n\t\t\tres = append(res, mqttTopicLevelSep, mqttTopicLevelSep)\n\t\t\tj++\n\t\tcase mqttSingleLevelWC, mqttMultiLevelWC:\n\t\t\tif !wcOk {\n\t\t\t\t// Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1]\n\t\t\t\t// The wildcard characters can be used in Topic Filters, but MUST NOT be used within a Topic Name\n\t\t\t\treturn nil, fmt.Errorf(\"wildcards not allowed in publish's topic: %q\", mt)\n\t\t\t}\n\t\t\tif !cp {\n\t\t\t\tmakeCopy(i)\n\t\t\t}\n\t\t\tif mt[i] == mqttSingleLevelWC {\n\t\t\t\tres = append(res, pwc)\n\t\t\t} else {\n\t\t\t\tres = append(res, fwc)\n\t\t\t}\n\t\tdefault:\n\t\t\tif cp {\n\t\t\t\tres = append(res, mt[i])\n\t\t\t}\n\t\t}\n\t\tj++\n\t}\n\tif cp && res[j-1] == btsep {\n\t\tres = append(res, mqttTopicLevelSep)\n\t\tj++\n\t}\n\treturn res[:j], nil\n}\n\n// Converts a NATS subject to MQTT topic. This is for publish\n// messages only, so there is no checking for wildcards.\n// Rules are reversed of mqttToNATSSubjectConversion.\nfunc natsSubjectStrToMQTTTopic(subject string) []byte {\n\treturn natsSubjectToMQTTTopic(stringToBytes(subject))\n}\n\nfunc natsSubjectToMQTTTopic(subject []byte) []byte {\n\ttopic := make([]byte, len(subject))\n\tend := len(subject) - 1\n\tvar j int\n\tfor i := 0; i < len(subject); i++ {\n\t\tswitch subject[i] {\n\t\tcase mqttTopicLevelSep:\n\t\t\tif i < end {\n\t\t\t\tswitch c := subject[i+1]; c {\n\t\t\t\tcase btsep, mqttTopicLevelSep:\n\t\t\t\t\tif c == btsep {\n\t\t\t\t\t\ttopic[j] = mqttTopicLevelSep\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttopic[j] = btsep\n\t\t\t\t\t}\n\t\t\t\t\tj++\n\t\t\t\t\ti++\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\t\tcase btsep:\n\t\t\ttopic[j] = mqttTopicLevelSep\n\t\t\tj++\n\t\tdefault:\n\t\t\ttopic[j] = subject[i]\n\t\t\tj++\n\t\t}\n\t}\n\treturn topic[:j]\n}\n\n// Returns true if the subject has more than 1 token and ends with \".>\"\nfunc mqttNeedSubForLevelUp(subject string) bool {\n\tif len(subject) < 3 {\n\t\treturn false\n\t}\n\tend := len(subject)\n\tif subject[end-2] == '.' && subject[end-1] == fwc {\n\t\treturn true\n\t}\n\treturn false\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// MQTT Reader functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (r *mqttReader) reset(buf []byte) {\n\tif l := len(r.pbuf); l > 0 {\n\t\ttmp := make([]byte, l+len(buf))\n\t\tcopy(tmp, r.pbuf)\n\t\tcopy(tmp[l:], buf)\n\t\tbuf = tmp\n\t\tr.pbuf = nil\n\t}\n\tr.buf = buf\n\tr.pos = 0\n\tr.pstart = 0\n}\n\nfunc (r *mqttReader) hasMore() bool {\n\treturn r.pos != len(r.buf)\n}\n\nfunc (r *mqttReader) readByte(field string) (byte, error) {\n\tif r.pos == len(r.buf) {\n\t\treturn 0, fmt.Errorf(\"error reading %s: %v\", field, io.EOF)\n\t}\n\tb := r.buf[r.pos]\n\tr.pos++\n\treturn b, nil\n}\n\nfunc (r *mqttReader) readPacketLen() (int, bool, error) {\n\treturn r.readPacketLenWithCheck(true)\n}\n\nfunc (r *mqttReader) readPacketLenWithCheck(check bool) (int, bool, error) {\n\tm := 1\n\tv := 0\n\tfor {\n\t\tvar b byte\n\t\tif r.pos != len(r.buf) {\n\t\t\tb = r.buf[r.pos]\n\t\t\tr.pos++\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t\tv += int(b&0x7f) * m\n\t\tif (b & 0x80) == 0 {\n\t\t\tif check && r.pos+v > len(r.buf) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\treturn v, true, nil\n\t\t}\n\t\tm *= 0x80\n\t\tif m > 0x200000 {\n\t\t\treturn 0, false, errMQTTMalformedVarInt\n\t\t}\n\t}\n\tr.pbuf = make([]byte, len(r.buf)-r.pstart)\n\tcopy(r.pbuf, r.buf[r.pstart:])\n\treturn 0, false, nil\n}\n\nfunc (r *mqttReader) readString(field string) (string, error) {\n\tvar s string\n\tbs, err := r.readBytes(field, false)\n\tif err == nil {\n\t\ts = string(bs)\n\t}\n\treturn s, err\n}\n\nfunc (r *mqttReader) readBytes(field string, cp bool) ([]byte, error) {\n\tluint, err := r.readUint16(field)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tl := int(luint)\n\tif l == 0 {\n\t\treturn nil, nil\n\t}\n\tstart := r.pos\n\tif start+l > len(r.buf) {\n\t\treturn nil, fmt.Errorf(\"error reading %s: %v\", field, io.ErrUnexpectedEOF)\n\t}\n\tr.pos += l\n\tb := r.buf[start:r.pos]\n\tif cp {\n\t\tb = copyBytes(b)\n\t}\n\treturn b, nil\n}\n\nfunc (r *mqttReader) readUint16(field string) (uint16, error) {\n\tif len(r.buf)-r.pos < 2 {\n\t\treturn 0, fmt.Errorf(\"error reading %s: %v\", field, io.ErrUnexpectedEOF)\n\t}\n\tstart := r.pos\n\tr.pos += 2\n\treturn binary.BigEndian.Uint16(r.buf[start:r.pos]), nil\n}\n\n//////////////////////////////////////////////////////////////////////////////\n//\n// MQTT Writer functions\n//\n//////////////////////////////////////////////////////////////////////////////\n\nfunc (w *mqttWriter) WriteUint16(i uint16) {\n\tw.WriteByte(byte(i >> 8))\n\tw.WriteByte(byte(i))\n}\n\nfunc (w *mqttWriter) WriteString(s string) {\n\tw.WriteBytes([]byte(s))\n}\n\nfunc (w *mqttWriter) WriteBytes(bs []byte) {\n\tw.WriteUint16(uint16(len(bs)))\n\tw.Write(bs)\n}\n\nfunc (w *mqttWriter) WriteVarInt(value int) {\n\tfor {\n\t\tb := byte(value & 0x7f)\n\t\tvalue >>= 7\n\t\tif value > 0 {\n\t\t\tb |= 0x80\n\t\t}\n\t\tw.WriteByte(b)\n\t\tif value == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc newMQTTWriter(cap int) *mqttWriter {\n\tw := &mqttWriter{}\n\tw.Grow(cap)\n\treturn w\n}\n" }, { "path": "server/mqtt_ex_bench_test.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_mqtt_tests\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n)\n\nconst (\n\tKB = 1024\n)\n\ntype mqttBenchMatrix struct {\n\tQOS []int\n\tMessageSize []int\n\tTopics []int\n\tPublishers []int\n\tSubscribers []int\n}\n\ntype mqttBenchContext struct {\n\tQOS int\n\tMessageSize int\n\tTopics int\n\tPublishers int\n\tSubscribers int\n\n\tHost string\n\tPort int\n}\n\nvar mqttBenchDefaultMatrix = mqttBenchMatrix{\n\tQOS: []int{0, 1, 2},\n\tMessageSize: []int{100, 1 * KB, 10 * KB},\n\tTopics: []int{100},\n\tPublishers: []int{1},\n\tSubscribers: []int{1},\n}\n\ntype MQTTBenchmarkResult struct {\n\tOps int `json:\"ops\"`\n\tNS map[string]time.Duration `json:\"ns\"`\n\tBytes int64 `json:\"bytes\"`\n}\n\nfunc BenchmarkXMQTT(b *testing.B) {\n\tif mqttTestCommandPath == \"\" {\n\t\tb.Skip(`\"mqtt-test\" command is not found in $PATH.`)\n\t}\n\n\tbc := mqttBenchContext{}\n\tb.Run(\"Server\", func(b *testing.B) {\n\t\tb.Cleanup(bc.startServer(b, false))\n\t\tbc.runAll(b)\n\t})\n\n\tb.Run(\"Cluster\", func(b *testing.B) {\n\t\tb.Cleanup(bc.startCluster(b, false))\n\t\tbc.runAll(b)\n\t})\n\n\tb.Run(\"Server-no-RMSCache\", func(b *testing.B) {\n\t\tb.Cleanup(bc.startServer(b, true))\n\n\t\tbc.benchmarkSubRet(b)\n\t})\n\n\tb.Run(\"Cluster-no-RMSCache\", func(b *testing.B) {\n\t\tb.Cleanup(bc.startCluster(b, true))\n\n\t\tbc.benchmarkSubRet(b)\n\t})\n}\n\nfunc (bc mqttBenchContext) runAll(b *testing.B) {\n\tbc.benchmarkPub(b)\n\tbc.benchmarkPubRetained(b)\n\tbc.benchmarkPubSub(b)\n\tbc.benchmarkSubRet(b)\n}\n\n// makes a copy of bc\nfunc (bc mqttBenchContext) benchmarkPub(b *testing.B) {\n\tm := mqttBenchDefaultMatrix.\n\t\tNoSubscribers().\n\t\tNoTopics()\n\n\tb.Run(\"PUB\", func(b *testing.B) {\n\t\tm.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) {\n\t\t\tbc.runAndReport(b, \"pub\",\n\t\t\t\t\"--qos\", strconv.Itoa(bc.QOS),\n\t\t\t\t\"--messages\", strconv.Itoa(b.N),\n\t\t\t\t\"--size\", strconv.Itoa(bc.MessageSize),\n\t\t\t\t\"--publishers\", strconv.Itoa(bc.Publishers),\n\t\t\t)\n\t\t})\n\t})\n}\n\n// makes a copy of bc\nfunc (bc mqttBenchContext) benchmarkPubRetained(b *testing.B) {\n\t// This bench is meaningless for QOS0 since the client considers the message\n\t// sent as soon as it's written out. It is also useless for QOS2 since the\n\t// flow takes a lot longer, and the difference of publishing as retained or\n\t// not is lost in the noise.\n\tm := mqttBenchDefaultMatrix.\n\t\tNoSubscribers().\n\t\tNoTopics().\n\t\tQOS1Only()\n\n\tb.Run(\"PUBRET\", func(b *testing.B) {\n\t\tm.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) {\n\t\t\tbc.runAndReport(b, \"pub\", \"--retain\",\n\t\t\t\t\"--qos\", strconv.Itoa(bc.QOS),\n\t\t\t\t\"--messages\", strconv.Itoa(b.N),\n\t\t\t\t\"--size\", strconv.Itoa(bc.MessageSize),\n\t\t\t\t\"--publishers\", strconv.Itoa(bc.Publishers),\n\t\t\t)\n\t\t})\n\t})\n}\n\n// makes a copy of bc\nfunc (bc mqttBenchContext) benchmarkPubSub(b *testing.B) {\n\t// This test uses a single built-in topic, and a built-in publisher, so no\n\t// reason to run it for topics and publishers.\n\tm := mqttBenchDefaultMatrix.\n\t\tNoTopics().\n\t\tNoPublishers()\n\n\tb.Run(\"PUBSUB\", func(b *testing.B) {\n\t\tm.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) {\n\t\t\tbc.runAndReport(b, \"pubsub\",\n\t\t\t\t\"--qos\", strconv.Itoa(bc.QOS),\n\t\t\t\t\"--messages\", strconv.Itoa(b.N),\n\t\t\t\t\"--size\", strconv.Itoa(bc.MessageSize),\n\t\t\t\t\"--subscribers\", strconv.Itoa(bc.Subscribers),\n\t\t\t)\n\t\t})\n\t})\n}\n\n// makes a copy of bc\nfunc (bc mqttBenchContext) benchmarkSubRet(b *testing.B) {\n\t// This test uses a built-in publisher, and it makes most sense to measure\n\t// the retained message delivery \"overhead\" on a QoS0 subscription; without\n\t// the extra time involved in actually subscribing.\n\tm := mqttBenchDefaultMatrix.\n\t\tNoPublishers().\n\t\tQOS0Only()\n\n\tb.Run(\"SUBRET\", func(b *testing.B) {\n\t\tm.runMatrix(b, bc, func(b *testing.B, bc *mqttBenchContext) {\n\t\t\tbc.runAndReport(b, \"subret\",\n\t\t\t\t\"--qos\", strconv.Itoa(bc.QOS),\n\t\t\t\t\"--topics\", strconv.Itoa(bc.Topics), // number of retained messages\n\t\t\t\t\"--subscribers\", strconv.Itoa(bc.Subscribers),\n\t\t\t\t\"--size\", strconv.Itoa(bc.MessageSize),\n\t\t\t\t\"--repeat\", strconv.Itoa(b.N), // number of subscribe requests\n\t\t\t)\n\t\t})\n\t})\n}\n\nfunc (bc mqttBenchContext) runAndReport(b *testing.B, name string, extraArgs ...string) {\n\tb.Helper()\n\tr := mqttRunExCommandTest(b, name, mqttNewDial(\"\", \"\", bc.Host, bc.Port, \"\"), extraArgs...)\n\tr.report(b)\n}\n\nfunc (bc *mqttBenchContext) startServer(b *testing.B, disableRMSCache bool) func() {\n\tb.Helper()\n\tb.StopTimer()\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(b, o)\n\n\to = s.getOpts()\n\tbc.Host = o.MQTT.Host\n\tbc.Port = o.MQTT.Port\n\tmqttInitTestServer(b, mqttNewDial(\"\", \"\", bc.Host, bc.Port, \"\"))\n\treturn func() {\n\t\ttestMQTTShutdownServer(s)\n\t}\n}\n\nfunc (bc *mqttBenchContext) startCluster(b *testing.B, disableRMSCache bool) func() {\n\tb.Helper()\n\tb.StopTimer()\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tstream_replicas: 3\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\n\tcl := createJetStreamClusterWithTemplate(b, conf, \"MQTT\", 3)\n\to := cl.randomNonLeader().getOpts()\n\tbc.Host = o.MQTT.Host\n\tbc.Port = o.MQTT.Port\n\tmqttInitTestServer(b, mqttNewDial(\"\", \"\", bc.Host, bc.Port, \"\"))\n\treturn func() {\n\t\tcl.shutdown()\n\t}\n}\n\nfunc mqttBenchWrapForMatrixField(\n\tvFieldPtr *int,\n\tarr []int,\n\tf func(b *testing.B, bc *mqttBenchContext),\n\tnamef func(int) string,\n) func(b *testing.B, bc *mqttBenchContext) {\n\tif len(arr) == 0 {\n\t\treturn f\n\t}\n\treturn func(b *testing.B, bc *mqttBenchContext) {\n\t\tfor _, value := range arr {\n\t\t\t*vFieldPtr = value\n\t\t\tb.Run(namef(value), func(b *testing.B) {\n\t\t\t\tf(b, bc)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc (m mqttBenchMatrix) runMatrix(b *testing.B, bc mqttBenchContext, f func(*testing.B, *mqttBenchContext)) {\n\tb.Helper()\n\tf = mqttBenchWrapForMatrixField(&bc.MessageSize, m.MessageSize, f, func(size int) string {\n\t\treturn sizeKB(size)\n\t})\n\tf = mqttBenchWrapForMatrixField(&bc.Topics, m.Topics, f, func(n int) string {\n\t\treturn fmt.Sprintf(\"%dtopics\", n)\n\t})\n\tf = mqttBenchWrapForMatrixField(&bc.Publishers, m.Publishers, f, func(n int) string {\n\t\treturn fmt.Sprintf(\"%dpubc\", n)\n\t})\n\tf = mqttBenchWrapForMatrixField(&bc.Subscribers, m.Subscribers, f, func(n int) string {\n\t\treturn fmt.Sprintf(\"%dsubc\", n)\n\t})\n\tf = mqttBenchWrapForMatrixField(&bc.QOS, m.QOS, f, func(qos int) string {\n\t\treturn fmt.Sprintf(\"QOS%d\", qos)\n\t})\n\tb.ResetTimer()\n\tb.StartTimer()\n\tf(b, &bc)\n}\n\nfunc (m mqttBenchMatrix) NoSubscribers() mqttBenchMatrix {\n\tm.Subscribers = nil\n\treturn m\n}\n\nfunc (m mqttBenchMatrix) NoTopics() mqttBenchMatrix {\n\tm.Topics = nil\n\treturn m\n}\n\nfunc (m mqttBenchMatrix) NoPublishers() mqttBenchMatrix {\n\tm.Publishers = nil\n\treturn m\n}\n\nfunc (m mqttBenchMatrix) QOS0Only() mqttBenchMatrix {\n\tm.QOS = []int{0}\n\treturn m\n}\n\nfunc (m mqttBenchMatrix) QOS1Only() mqttBenchMatrix {\n\tm.QOS = []int{1}\n\treturn m\n}\n\nfunc sizeKB(size int) string {\n\tunit := \"B\"\n\tN := size\n\tif size >= KB {\n\t\tunit = \"KB\"\n\t\tN = (N + KB/2) / KB\n\t}\n\treturn fmt.Sprintf(\"%d%s\", N, unit)\n}\n\nfunc (r MQTTBenchmarkResult) report(b *testing.B) {\n\t// Disable the default ns metric in favor of custom X_ms/op.\n\tb.ReportMetric(0, \"ns/op\")\n\n\t// Disable MB/s since the github benchmarking action misinterprets the sign\n\t// of the result (treats it as less is better).\n\tb.SetBytes(0)\n\t// b.SetBytes(r.Bytes)\n\n\tfor unit, ns := range r.NS {\n\t\tnsOp := float64(ns) / float64(r.Ops)\n\t\tb.ReportMetric(nsOp/1000000, unit+\"_ms/op\")\n\t}\n\t// Diable ReportAllocs() since it confuses the github benchmarking action\n\t// with the noise.\n\t// b.ReportAllocs()\n}\n" }, { "path": "server/mqtt_ex_test_test.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_mqtt_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nuid\"\n)\n\ntype mqttDial string\n\ntype mqttTarget struct {\n\tsingleServers []*Server\n\tclusters []*cluster\n\tconfigs []mqttTestConfig\n\tall []mqttDial\n}\n\ntype mqttTestConfig struct {\n\tname string\n\tpub []mqttDial\n\tsub []mqttDial\n}\n\nfunc TestXMQTTCompliance(t *testing.T) {\n\tif mqttCLICommandPath == _EMPTY_ {\n\t\tt.Skip(`\"mqtt\" command is not found in $PATH nor $MQTT_CLI. See https://hivemq.github.io/mqtt-cli/docs/installation/#debian-package for installation instructions`)\n\t}\n\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\to = s.getOpts()\n\tdefer testMQTTShutdownServer(s)\n\n\tcmd := exec.Command(mqttCLICommandPath, \"test\", \"-V\", \"3\", \"-p\", strconv.Itoa(o.MQTT.Port))\n\n\toutput, err := cmd.CombinedOutput()\n\tt.Log(string(output))\n\tif err != nil {\n\t\tif exitError, ok := err.(*exec.ExitError); ok {\n\t\t\tt.Fatalf(\"mqtt cli exited with error: %v\", exitError)\n\t\t}\n\t}\n}\n\nfunc TestXMQTTRetainedMessages(t *testing.T) {\n\tif mqttTestCommandPath == _EMPTY_ {\n\t\tt.Skip(`\"mqtt-test\" command is not found in $PATH.`)\n\t}\n\n\tfor _, topo := range []struct {\n\t\tname string\n\t\tmakef func(testing.TB) *mqttTarget\n\t}{\n\t\t{\n\t\t\tname: \"single server\",\n\t\t\tmakef: mqttMakeTestServer,\n\t\t},\n\t\t{\n\t\t\tname: \"cluster\",\n\t\t\tmakef: mqttMakeTestCluster(4, \"\"),\n\t\t},\n\t} {\n\t\tt.Run(topo.name, func(t *testing.T) {\n\t\t\ttarget := topo.makef(t)\n\t\t\tt.Cleanup(target.Shutdown)\n\n\t\t\t// initialize the MQTT assets by \"touching\" all nodes in the\n\t\t\t// cluster, but then reload to start with fresh server state.\n\t\t\tfor _, dial := range target.all {\n\t\t\t\tmqttInitTestServer(t, dial)\n\t\t\t}\n\n\t\t\tnumRMS := 100\n\t\t\tstrNumRMS := strconv.Itoa(numRMS)\n\t\t\ttopics := make([]string, len(target.configs))\n\n\t\t\tfor i, tc := range target.configs {\n\t\t\t\t// Publish numRMS retained messages one at a time,\n\t\t\t\t// round-robin across pub nodes. Remember the topic for each\n\t\t\t\t// test config to check the subs after reload.\n\t\t\t\ttopic := \"subret_\" + nuid.Next()\n\t\t\t\ttopics[i] = topic\n\t\t\t\tiNode := 0\n\t\t\t\tfor i := 0; i < numRMS; i++ {\n\t\t\t\t\tpubTopic := fmt.Sprintf(\"%s/%d\", topic, i)\n\t\t\t\t\tdial := tc.pub[iNode%len(tc.pub)]\n\t\t\t\t\tmqttRunExCommandTest(t, \"pub\", dial,\n\t\t\t\t\t\t\"--retain\",\n\t\t\t\t\t\t\"--topic\", pubTopic,\n\t\t\t\t\t\t\"--qos\", \"0\",\n\t\t\t\t\t\t\"--size\", \"128\", // message size 128 bytes\n\t\t\t\t\t)\n\t\t\t\t\tiNode++\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Check all sub nodes for retained messages\n\t\t\tfor i, tc := range target.configs {\n\t\t\t\tfor _, dial := range tc.sub {\n\t\t\t\t\tmqttRunExCommandTest(t, \"sub\", dial,\n\t\t\t\t\t\t\"--retained\", strNumRMS,\n\t\t\t\t\t\t\"--qos\", \"0\",\n\t\t\t\t\t\t\"--topic\", topics[i],\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Reload the target\n\t\t\ttarget.Reload(t)\n\n\t\t\t// Now check again\n\t\t\tfor i, tc := range target.configs {\n\t\t\t\tfor _, dial := range tc.sub {\n\t\t\t\t\tmqttRunExCommandTestRetry(t, 1, \"sub\", dial,\n\t\t\t\t\t\t\"--retained\", strNumRMS,\n\t\t\t\t\t\t\"--qos\", \"0\",\n\t\t\t\t\t\t\"--topic\", topics[i],\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc mqttInitTestServer(tb testing.TB, dial mqttDial) {\n\ttb.Helper()\n\tmqttRunExCommandTestRetry(tb, 5, \"pub\", dial)\n}\n\nfunc mqttNewDialForServer(s *Server, username, password string) mqttDial {\n\to := s.getOpts().MQTT\n\treturn mqttNewDial(username, password, o.Host, o.Port, s.Name())\n}\n\nfunc mqttNewDial(username, password, host string, port int, comment string) mqttDial {\n\td := \"\"\n\tswitch {\n\tcase username != \"\" && password != \"\":\n\t\td = fmt.Sprintf(\"%s:%s@%s:%d\", username, password, host, port)\n\tcase username != \"\":\n\t\td = fmt.Sprintf(\"%s@%s:%d\", username, host, port)\n\tdefault:\n\t\td = fmt.Sprintf(\"%s:%d\", host, port)\n\t}\n\tif comment != \"\" {\n\t\td += \"#\" + comment\n\t}\n\treturn mqttDial(d)\n}\n\nfunc (d mqttDial) Get() (u, p, s, c string) {\n\tif d == \"\" {\n\t\treturn \"\", \"\", \"127.0.0.1:1883\", \"\"\n\t}\n\tin := string(d)\n\tif i := strings.LastIndex(in, \"#\"); i != -1 {\n\t\tc = in[i+1:]\n\t\tin = in[:i]\n\t}\n\tif i := strings.LastIndex(in, \"@\"); i != -1 {\n\t\tup := in[:i]\n\t\tin = in[i+1:]\n\t\tu = up\n\t\tif i := strings.Index(up, \":\"); i != -1 {\n\t\t\tu = up[:i]\n\t\t\tp = up[i+1:]\n\t\t}\n\t}\n\ts = in\n\treturn u, p, s, c\n}\n\nfunc (d mqttDial) Name() string {\n\t_, _, _, c := d.Get()\n\treturn c\n}\n\nfunc (t *mqttTarget) Reload(tb testing.TB) {\n\ttb.Helper()\n\n\tfor _, c := range t.clusters {\n\t\tc.stopAll()\n\t\tc.restartAllSamePorts()\n\t}\n\n\tfor i, s := range t.singleServers {\n\t\to := s.getOpts()\n\t\ts.Shutdown()\n\t\tt.singleServers[i] = testMQTTRunServer(tb, o)\n\t}\n\n\tfor _, dial := range t.all {\n\t\tmqttInitTestServer(tb, dial)\n\t}\n}\n\nfunc (t *mqttTarget) Shutdown() {\n\tfor _, c := range t.clusters {\n\t\tc.shutdown()\n\t}\n\tfor _, s := range t.singleServers {\n\t\ttestMQTTShutdownServer(s)\n\t}\n}\n\nfunc mqttMakeTestServer(tb testing.TB) *mqttTarget {\n\ttb.Helper()\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(tb, o)\n\tall := []mqttDial{mqttNewDialForServer(s, \"\", \"\")}\n\treturn &mqttTarget{\n\t\tsingleServers: []*Server{s},\n\t\tall: all,\n\t\tconfigs: []mqttTestConfig{\n\t\t\t{\n\t\t\t\tname: \"single server\",\n\t\t\t\tpub: all,\n\t\t\t\tsub: all,\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc mqttMakeTestCluster(size int, domain string) func(tb testing.TB) *mqttTarget {\n\treturn func(tb testing.TB) *mqttTarget {\n\t\ttb.Helper()\n\t\tif size < 3 {\n\t\t\ttb.Fatal(\"cluster size must be at least 3\")\n\t\t}\n\n\t\tif domain != \"\" {\n\t\t\tdomain = \"domain: \" + domain + \", \"\n\t\t}\n\t\tclusterConf := `\n\tlisten: 127.0.0.1:-1\n\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, ` + domain + `store_dir: '%s'}\n\n\tleafnodes {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tmqtt {\n\t\tlisten: 127.0.0.1:-1\n\t\tstream_replicas: 3\n\t}\n\n\taccounts {\n\t\tONE { users = [ { user: \"one\", pass: \"p\" } ]; jetstream: enabled }\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n`\n\t\tcl := createJetStreamClusterWithTemplate(tb, clusterConf, \"MQTT\", size)\n\t\tcl.waitOnLeader()\n\n\t\tall := []mqttDial{}\n\t\tfor _, s := range cl.servers {\n\t\t\tall = append(all, mqttNewDialForServer(s, \"one\", \"p\"))\n\t\t}\n\n\t\treturn &mqttTarget{\n\t\t\tclusters: []*cluster{cl},\n\t\t\tall: all,\n\t\t\tconfigs: []mqttTestConfig{\n\t\t\t\t{\n\t\t\t\t\tname: \"publish to one\",\n\t\t\t\t\tpub: []mqttDial{\n\t\t\t\t\t\tmqttNewDialForServer(cl.randomServer(), \"one\", \"p\"),\n\t\t\t\t\t},\n\t\t\t\t\tsub: all,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tname: \"publish to all\",\n\t\t\t\t\tpub: all,\n\t\t\t\t\tsub: all,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n}\n\nvar mqttCLICommandPath = func() string {\n\tp := os.Getenv(\"MQTT_CLI\")\n\tif p == \"\" {\n\t\tp, _ = exec.LookPath(\"mqtt\")\n\t}\n\treturn p\n}()\n\nvar mqttTestCommandPath = func() string {\n\tp, _ := exec.LookPath(\"mqtt-test\")\n\treturn p\n}()\n\nfunc mqttRunExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) *MQTTBenchmarkResult {\n\ttb.Helper()\n\treturn mqttRunExCommandTestRetry(tb, 1, subCommand, dial, extraArgs...)\n}\n\nfunc mqttRunExCommandTestRetry(tb testing.TB, n int, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult) {\n\ttb.Helper()\n\tvar err error\n\tfor i := 0; i < n; i++ {\n\t\tif r, err = mqttTryExCommandTest(tb, subCommand, dial, extraArgs...); err == nil {\n\t\t\treturn r\n\t\t}\n\n\t\tif i < (n - 1) {\n\t\t\ttb.Logf(\"failed to %q %s to %q on attempt %v, will retry.\", subCommand, extraArgs, dial.Name(), i)\n\t\t} else {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc mqttTryExCommandTest(tb testing.TB, subCommand string, dial mqttDial, extraArgs ...string) (r *MQTTBenchmarkResult, err error) {\n\ttb.Helper()\n\tif mqttTestCommandPath == \"\" {\n\t\ttb.Skip(`\"mqtt-test\" command is not found in $PATH.`)\n\t}\n\n\targs := []string{subCommand, // \"-q\",\n\t\t\"-s\", string(dial),\n\t}\n\targs = append(args, extraArgs...)\n\tcmd := exec.Command(mqttTestCommandPath, args...)\n\n\tstdout, err := cmd.StdoutPipe()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error executing %q: %v\", cmd.String(), err)\n\t}\n\tdefer stdout.Close()\n\terrbuf := bytes.Buffer{}\n\tcmd.Stderr = &errbuf\n\tif err = cmd.Start(); err != nil {\n\t\treturn nil, fmt.Errorf(\"error executing %q: %v\", cmd.String(), err)\n\t}\n\tout, err := io.ReadAll(stdout)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error executing %q: failed to read output: %v\", cmd.String(), err)\n\t}\n\tif err = cmd.Wait(); err != nil {\n\t\treturn nil, fmt.Errorf(\"error executing %q: %v\\n\\n%s\\n\\n%s\", cmd.String(), err, string(out), errbuf.String())\n\t}\n\n\tr = &MQTTBenchmarkResult{}\n\tif err := json.Unmarshal(out, r); err != nil {\n\t\ttb.Fatalf(\"error executing %q: failed to decode output: %v\\n\\n%s\\n\\n%s\", cmd.String(), err, string(out), errbuf.String())\n\t}\n\treturn r, nil\n}\n" }, { "path": "server/mqtt_test.go", "content": "// Copyright 2020-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_mqtt_tests\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nvar testMQTTTimeout = 10 * time.Second\n\nvar jsClusterTemplWithLeafAndMQTT = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t{{leaf}}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\tmqtt {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\ntype mqttWrapAsWs struct {\n\tnet.Conn\n\tt testing.TB\n\tbr *bufio.Reader\n\ttmp []byte\n}\n\nfunc (c *mqttWrapAsWs) Write(p []byte) (int, error) {\n\tproto := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, p)\n\treturn c.Conn.Write(proto)\n}\n\nfunc (c *mqttWrapAsWs) Read(p []byte) (int, error) {\n\tfor {\n\t\tif n := len(c.tmp); n > 0 {\n\t\t\tif len(p) < n {\n\t\t\t\tn = len(p)\n\t\t\t}\n\t\t\tcopy(p, c.tmp[:n])\n\t\t\tc.tmp = c.tmp[n:]\n\t\t\treturn n, nil\n\t\t}\n\t\tc.tmp = testWSReadFrame(c.t, c.br)\n\t}\n}\n\nfunc testMQTTReadPacket(t testing.TB, r *mqttReader) (byte, int) {\n\tt.Helper()\n\n\tvar b byte\n\tvar pl int\n\tvar err error\n\n\trd := r.reader\n\n\tfill := func() []byte {\n\t\tt.Helper()\n\t\tvar buf [512]byte\n\n\t\tn, err := rd.Read(buf[:])\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading data: %v\", err)\n\t\t}\n\t\treturn copyBytes(buf[:n])\n\t}\n\n\trd.SetReadDeadline(time.Now().Add(testMQTTTimeout))\n\tfor {\n\t\tr.pstart = r.pos\n\t\tif !r.hasMore() {\n\t\t\tr.reset(fill())\n\t\t\tcontinue\n\t\t}\n\t\tb, err = r.readByte(\"packet type\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading packet: %v\", err)\n\t\t}\n\t\tvar complete bool\n\t\tpl, complete, err = r.readPacketLen()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading packet: %v\", err)\n\t\t}\n\t\tif !complete {\n\t\t\tr.reset(fill())\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\trd.SetReadDeadline(time.Time{})\n\treturn b, pl\n}\n\nfunc testMQTTReadPIPacket(expectedType byte, t testing.TB, r *mqttReader, expectedPI uint16) {\n\tt.Helper()\n\tb, _ := testMQTTReadPacket(t, r)\n\tif pt := b & mqttPacketMask; pt != expectedType {\n\t\tt.Fatalf(\"Expected packet %x, got %x\", expectedType, pt)\n\t}\n\trpi, err := r.readUint16(\"packet identifier\")\n\tif err != nil || rpi != expectedPI {\n\t\tt.Fatalf(\"Expected PI %v got: %v, err=%v\", expectedPI, rpi, err)\n\t}\n}\n\nfunc TestMQTTReader(t *testing.T) {\n\tr := &mqttReader{}\n\tr.reset([]byte{0, 2, 'a', 'b'})\n\tbs, err := r.readBytes(\"\", false)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsbs := string(bs)\n\tif sbs != \"ab\" {\n\t\tt.Fatalf(`expected \"ab\", got %q`, sbs)\n\t}\n\n\tr.reset([]byte{0, 2, 'a', 'b'})\n\tbs, err = r.readBytes(\"\", true)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbs[0], bs[1] = 'c', 'd'\n\tif bytes.Equal(bs, r.buf[2:]) {\n\t\tt.Fatal(\"readBytes should have returned a copy\")\n\t}\n\n\tr.reset([]byte{'a', 'b'})\n\tif b, err := r.readByte(\"\"); err != nil || b != 'a' {\n\t\tt.Fatalf(\"Error reading byte: b=%v err=%v\", b, err)\n\t}\n\tif !r.hasMore() {\n\t\tt.Fatal(\"expected to have more, did not\")\n\t}\n\tif b, err := r.readByte(\"\"); err != nil || b != 'b' {\n\t\tt.Fatalf(\"Error reading byte: b=%v err=%v\", b, err)\n\t}\n\tif r.hasMore() {\n\t\tt.Fatal(\"expected to not have more\")\n\t}\n\tif _, err := r.readByte(\"test\"); err == nil || !strings.Contains(err.Error(), \"error reading test\") {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\n\tr.reset([]byte{0, 2, 'a', 'b'})\n\tif s, err := r.readString(\"\"); err != nil || s != \"ab\" {\n\t\tt.Fatalf(\"Error reading string: s=%q err=%v\", s, err)\n\t}\n\n\tr.reset([]byte{10})\n\tif _, err := r.readUint16(\"uint16\"); err == nil || !strings.Contains(err.Error(), \"error reading uint16\") {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\n\tr.reset([]byte{0x82, 0xff, 0x3})\n\tl, _, err := r.readPacketLenWithCheck(false)\n\tif err != nil {\n\t\tt.Fatal(\"error getting packet len\")\n\t}\n\tif l != 0xff82 {\n\t\tt.Fatalf(\"expected length 0xff82 got 0x%x\", l)\n\t}\n\tr.reset([]byte{0xff, 0xff, 0xff, 0xff, 0xff})\n\tif _, _, err := r.readPacketLenWithCheck(false); err == nil || !strings.Contains(err.Error(), \"malformed\") {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n\n\tr.reset([]byte{0, 2, 'a', 'b', mqttPacketPub, 0x82, 0xff, 0x3})\n\tr.readString(\"\")\n\tfor i := 0; i < 2; i++ {\n\t\tr.pstart = r.pos\n\t\tb, err := r.readByte(\"\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading byte: %v\", err)\n\t\t}\n\t\tif pt := b & mqttPacketMask; pt != mqttPacketPub {\n\t\t\tt.Fatalf(\"Unexpected byte: %v\", b)\n\t\t}\n\t\tpl, complete, err := r.readPacketLen()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif complete {\n\t\t\tt.Fatal(\"Expected to be incomplete\")\n\t\t}\n\t\tif pl != 0 {\n\t\t\tt.Fatalf(\"Expected pl to be 0, got %v\", pl)\n\t\t}\n\t\tif i > 0 {\n\t\t\tbreak\n\t\t}\n\t\tif !bytes.Equal(r.pbuf, []byte{mqttPacketPub, 0x82, 0xff, 0x3}) {\n\t\t\tt.Fatalf(\"Invalid recorded partial: %v\", r.pbuf)\n\t\t}\n\t\tr.reset([]byte{'a', 'b', 'c'})\n\t\tif !bytes.Equal(r.buf, []byte{mqttPacketPub, 0x82, 0xff, 0x3, 'a', 'b', 'c'}) {\n\t\t\tt.Fatalf(\"Invalid buffer: %v\", r.buf)\n\t\t}\n\t\tif r.pbuf != nil {\n\t\t\tt.Fatalf(\"Partial buffer should have been reset, got %v\", r.pbuf)\n\t\t}\n\t\tif r.pos != 0 {\n\t\t\tt.Fatalf(\"Pos should have been reset, got %v\", r.pos)\n\t\t}\n\t\tif r.pstart != 0 {\n\t\t\tt.Fatalf(\"Pstart should have been reset, got %v\", r.pstart)\n\t\t}\n\t}\n\t// On second pass, the pbuf should have been extended with 'abc'\n\tif !bytes.Equal(r.pbuf, []byte{mqttPacketPub, 0x82, 0xff, 0x3, 'a', 'b', 'c'}) {\n\t\tt.Fatalf(\"Invalid recorded partial: %v\", r.pbuf)\n\t}\n}\n\nfunc TestMQTTWriter(t *testing.T) {\n\tw := newMQTTWriter(0)\n\tw.WriteUint16(1234)\n\n\tr := &mqttReader{}\n\tr.reset(w.Bytes())\n\tif v, err := r.readUint16(\"\"); err != nil || v != 1234 {\n\t\tt.Fatalf(\"unexpected value: v=%v err=%v\", v, err)\n\t}\n\n\tw.Reset()\n\tw.WriteString(\"test\")\n\tr.reset(w.Bytes())\n\tif len(r.buf) != 6 {\n\t\tt.Fatalf(\"Expected 2 bytes size before string, got %v\", r.buf)\n\t}\n\n\tw.Reset()\n\tw.WriteBytes([]byte(\"test\"))\n\tr.reset(w.Bytes())\n\tif len(r.buf) != 6 {\n\t\tt.Fatalf(\"Expected 2 bytes size before bytes, got %v\", r.buf)\n\t}\n\n\tints := []int{\n\t\t0, 1, 127, 128, 16383, 16384, 2097151, 2097152, 268435455,\n\t}\n\tlens := []int{\n\t\t1, 1, 1, 2, 2, 3, 3, 4, 4,\n\t}\n\n\ttl := 0\n\tw.Reset()\n\tfor i, v := range ints {\n\t\tw.WriteVarInt(v)\n\t\ttl += lens[i]\n\t\tif tl != w.Len() {\n\t\t\tt.Fatalf(\"expected len %d, got %d\", tl, w.Len())\n\t\t}\n\t}\n\n\tr.reset(w.Bytes())\n\tfor _, v := range ints {\n\t\tx, _, _ := r.readPacketLenWithCheck(false)\n\t\tif v != x {\n\t\t\tt.Fatalf(\"expected %d, got %d\", v, x)\n\t\t}\n\t}\n}\n\nfunc testMQTTDefaultOptions() *Options {\n\to := DefaultOptions()\n\to.ServerName = nuid.Next()\n\to.Cluster.Port = 0\n\to.Gateway.Name = \"\"\n\to.Gateway.Port = 0\n\to.LeafNode.Port = 0\n\to.Websocket.Port = 0\n\to.MQTT.Host = \"127.0.0.1\"\n\to.MQTT.Port = -1\n\to.JetStream = true\n\treturn o\n}\n\nfunc testMQTTRunServer(t testing.TB, o *Options) *Server {\n\tt.Helper()\n\to.NoLog = false\n\tif o.StoreDir == _EMPTY_ {\n\t\to.StoreDir = t.TempDir()\n\t}\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tl := &DummyLogger{}\n\ts.SetLogger(l, true, true)\n\ts.Start()\n\tif err := s.readyForConnections(3 * time.Second); err != nil {\n\t\ttestMQTTShutdownServer(s)\n\t\tt.Fatal(err)\n\t}\n\treturn s\n}\n\nfunc testMQTTShutdownRestartedServer(s **Server) {\n\tsrv := *s\n\ttestMQTTShutdownServer(srv)\n\t*s = nil\n}\n\nfunc testMQTTShutdownServer(s *Server) {\n\tif c := s.JetStreamConfig(); c != nil {\n\t\tdir := strings.TrimSuffix(c.StoreDir, JetStreamStoreDir)\n\t\tdefer os.RemoveAll(dir)\n\t}\n\ts.Shutdown()\n}\n\nfunc testMQTTDefaultTLSOptions(t *testing.T, verify bool) *Options {\n\tt.Helper()\n\to := testMQTTDefaultOptions()\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\tVerify: verify,\n\t}\n\tvar err error\n\to.MQTT.TLSConfig, err = GenTLSConfig(tc)\n\to.MQTT.TLSTimeout = 2.0\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating tls config: %v\", err)\n\t}\n\treturn o\n}\n\nfunc TestMQTTServerNameRequired(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tcluster {\n\t\t\tport: -1\n\t\t}\n\t\tmqtt {\n\t\t\tport: -1\n\t\t}\n\t`))\n\to, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\tif _, err := NewServer(o); err == nil || err.Error() != errMQTTServerNameMustBeSet.Error() {\n\t\tt.Fatalf(\"Expected error about requiring server name to be set, got %v\", err)\n\t}\n\n\tconf = createConfFile(t, []byte(`\n\t\tmqtt {\n\t\t\tport: -1\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\tif _, err := NewServer(o); err == nil || err.Error() != errMQTTStandaloneNeedsJetStream.Error() {\n\t\tt.Fatalf(`Expected errMQTTStandaloneNeedsJetStream (\"next in line\"), got %v`, err)\n\t}\n}\n\nfunc TestMQTTStandaloneRequiresJetStream(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tserver_name: mqtt\n\t\tmqtt {\n\t\t\tport: -1\n\t\t\ttls {\n\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t}\n\t\t}\n\t`))\n\to, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\tif _, err := NewServer(o); err == nil || err.Error() != errMQTTStandaloneNeedsJetStream.Error() {\n\t\tt.Fatalf(\"Expected error about requiring JetStream in standalone mode, got %v\", err)\n\t}\n}\n\nfunc TestMQTTConfig(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tserver_name: mqtt\n\t\tmqtt {\n\t\t\tport: -1\n\t\t\ttls {\n\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t}\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\tif o.MQTT.TLSConfig == nil {\n\t\tt.Fatal(\"expected TLS config to be set\")\n\t}\n}\n\nfunc TestMQTTValidateOptions(t *testing.T) {\n\tnmqtto := DefaultOptions()\n\tmqtto := testMQTTDefaultOptions()\n\tfor _, test := range []struct {\n\t\tname string\n\t\tgetOpts func() *Options\n\t\terr error\n\t}{\n\t\t{\"mqtt disabled\", func() *Options { return nmqtto.Clone() }, nil},\n\t\t{\"mqtt username not allowed if users specified\", func() *Options {\n\t\t\to := mqtto.Clone()\n\t\t\to.Users = []*User{{Username: \"abc\", Password: \"pwd\"}}\n\t\t\to.MQTT.Username = \"b\"\n\t\t\to.MQTT.Password = \"pwd\"\n\t\t\treturn o\n\t\t}, errMQTTUserMixWithUsersNKeys},\n\t\t{\"mqtt token not allowed if users specified\", func() *Options {\n\t\t\to := mqtto.Clone()\n\t\t\to.Nkeys = []*NkeyUser{{Nkey: \"abc\"}}\n\t\t\to.MQTT.Token = \"mytoken\"\n\t\t\treturn o\n\t\t}, errMQTTTokenMixWIthUsersNKeys},\n\t\t{\"ack wait should be >=0\", func() *Options {\n\t\t\to := mqtto.Clone()\n\t\t\to.MQTT.AckWait = -10 * time.Second\n\t\t\treturn o\n\t\t}, errMQTTAckWaitMustBePositive},\n\t\t{\"js api timeout should be >=0\", func() *Options {\n\t\t\to := mqtto.Clone()\n\t\t\to.MQTT.JSAPITimeout = -10 * time.Second\n\t\t\treturn o\n\t\t}, errMQTTJSAPITimeoutMustBePositive},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\terr := validateMQTTOptions(test.getOpts())\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else if test.err != nil && (err == nil || err.Error() != test.err.Error()) {\n\t\t\t\tt.Fatalf(\"Expected error to contain %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTParseOptions(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcontent string\n\t\tcheckOpt func(*MQTTOpts) error\n\t\terr string\n\t}{\n\t\t// Negative tests\n\t\t{\"bad type\", \"mqtt: []\", nil, \"to be a map\"},\n\t\t{\"bad listen\", \"mqtt: { listen: [] }\", nil, \"port or host:port\"},\n\t\t{\"bad port\", `mqtt: { port: \"abc\" }`, nil, \"not int64\"},\n\t\t{\"bad host\", `mqtt: { host: 123 }`, nil, \"not string\"},\n\t\t{\"bad tls\", `mqtt: { tls: 123 }`, nil, \"not map[string]interface {}\"},\n\t\t{\"unknown field\", `mqtt: { this_does_not_exist: 123 }`, nil, \"unknown\"},\n\t\t{\"ack wait\", `mqtt: {ack_wait: abc}`, nil, \"invalid duration\"},\n\t\t{\"max ack pending\", `mqtt: {max_ack_pending: abc}`, nil, \"not int64\"},\n\t\t{\"max ack pending too high\", `mqtt: {max_ack_pending: 12345678}`, nil, \"invalid value\"},\n\t\t{\"js_api_timeout bad duration\", `mqtt: {js_api_timeout: abc}`, nil, \"invalid duration\"},\n\t\t// Positive tests\n\t\t{\"tls gen fails\", `\n\t\t\tmqtt {\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t}\n\t\t\t}`, nil, \"missing 'key_file'\"},\n\t\t{\"listen port only\", `mqtt { listen: 1234 }`, func(o *MQTTOpts) error {\n\t\t\tif o.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected 1234, got %v\", o.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"listen host and port\", `mqtt { listen: \"localhost:1234\" }`, func(o *MQTTOpts) error {\n\t\t\tif o.Host != \"localhost\" || o.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected localhost:1234, got %v:%v\", o.Host, o.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"host\", `mqtt { host: \"localhost\" }`, func(o *MQTTOpts) error {\n\t\t\tif o.Host != \"localhost\" {\n\t\t\t\treturn fmt.Errorf(\"expected localhost, got %v\", o.Host)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"port\", `mqtt { port: 1234 }`, func(o *MQTTOpts) error {\n\t\t\tif o.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected 1234, got %v\", o.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"tls config\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.TLSConfig == nil {\n\t\t\t\t\treturn fmt.Errorf(\"TLSConfig should have been set\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"no auth user\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tno_auth_user: \"noauthuser\"\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.NoAuthUser != \"noauthuser\" {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid NoAuthUser value: %q\", o.NoAuthUser)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"auth block\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tauthorization {\n\t\t\t\t\tuser: \"mqttuser\"\n\t\t\t\t\tpassword: \"pwd\"\n\t\t\t\t\ttoken: \"token\"\n\t\t\t\t\ttimeout: 2.0\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.Username != \"mqttuser\" || o.Password != \"pwd\" || o.Token != \"token\" || o.AuthTimeout != 2.0 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid auth block: %+v\", o)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"auth timeout as int\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 2\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.AuthTimeout != 2.0 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid auth timeout: %v\", o.AuthTimeout)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"ack wait\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tack_wait: \"10s\"\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.AckWait != 10*time.Second {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid ack wait: %v\", o.AckWait)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"max ack pending\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tmax_ack_pending: 123\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.MaxAckPending != 123 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid max ack pending: %v\", o.MaxAckPending)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"reject_qos2_publish\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\treject_qos2_publish: true\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif !o.rejectQoS2Pub {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid: expected rejectQoS2Pub to be set\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"downgrade_qos2_subscribe\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tdowngrade_qos2_subscribe: true\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif !o.downgradeQoS2Sub {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid: expected downgradeQoS2Sub to be set\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"js_api_timeout\",\n\t\t\t`\n\t\t\tmqtt {\n\t\t\t\tjs_api_timeout: \"60s\"\n\t\t\t}\n\t\t\t`, func(o *MQTTOpts) error {\n\t\t\t\tif o.JSAPITimeout != 60*time.Second {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid JS API timeout: %v\", o.JSAPITimeout)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(test.content))\n\t\t\to, err := ProcessConfigFile(conf)\n\t\t\tif test.err != _EMPTY_ {\n\t\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\t\tt.Fatalf(\"For content: %q, expected error about %q, got %v\", test.content, test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else if err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error for content %q: %v\", test.content, err)\n\t\t\t}\n\t\t\tif err := test.checkOpt(&o.MQTT); err != nil {\n\t\t\t\tt.Fatalf(\"Incorrect option for content %q: %v\", test.content, err.Error())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTStart(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc, err := net.Dial(\"tcp\", net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to create tcp connection to mqtt port: %v\", err)\n\t}\n\tnc.Close()\n\n\t// Check failure to start due to port in use\n\to2 := testMQTTDefaultOptions()\n\to2.MQTT.Port = o.MQTT.Port\n\ts2, err := NewServer(o2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tdefer s2.Shutdown()\n\tl := &captureFatalLogger{fatalCh: make(chan string, 1)}\n\ts2.SetLogger(l, false, false)\n\ts2.Start()\n\n\tselect {\n\tcase e := <-l.fatalCh:\n\t\tif !strings.Contains(e, \"Unable to listen for MQTT connections\") {\n\t\t\tt.Fatalf(\"Unexpected error: %q\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Should have gotten a fatal error\")\n\t}\n}\n\nfunc TestMQTTTLS(t *testing.T) {\n\to := testMQTTDefaultTLSOptions(t, false)\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tc, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{tls: true}, o.MQTT.Host, o.MQTT.Port, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tc.Close()\n\tc = nil\n\ttestMQTTShutdownServer(s)\n\n\t// Force client cert verification\n\to = testMQTTDefaultTLSOptions(t, true)\n\ts = testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tc, _, err = testMQTTConnectRetryWithError(t, &mqttConnInfo{tls: true}, o.MQTT.Host, o.MQTT.Port, 0)\n\tif err == nil || c != nil {\n\t\tif c != nil {\n\t\t\tc.Close()\n\t\t}\n\t\tt.Fatal(\"Handshake expected to fail since client did not provide cert\")\n\t}\n\n\t// Add client cert.\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/client-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/client-key.pem\",\n\t}\n\ttlsc, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\ttlsc.InsecureSkipVerify = true\n\tc, _, err = testMQTTConnectRetryWithError(t, &mqttConnInfo{\n\t\ttls: true,\n\t\ttlsc: tlsc,\n\t}, o.MQTT.Host, o.MQTT.Port, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tc.Close()\n\tc = nil\n\ttestMQTTShutdownServer(s)\n\n\t// Lower TLS timeout so low that we should fail\n\to.MQTT.TLSTimeout = 0.001\n\ts = testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc, err := net.Dial(\"tcp\", net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to create tcp connection to mqtt port: %v\", err)\n\t}\n\tdefer nc.Close()\n\ttime.Sleep(100 * time.Millisecond)\n\ttlsConn := tls.Client(nc, tlsc)\n\ttlsConn.SetDeadline(time.Now().Add(time.Second))\n\tif err := tlsConn.Handshake(); err == nil {\n\t\tt.Fatal(\"Expected failure, did not get one\")\n\t}\n}\n\ntype mqttConnInfo struct {\n\tclientID string\n\tcleanSess bool\n\tkeepAlive uint16\n\twill *mqttWill\n\tuser string\n\tpass string\n\tws bool\n\ttls bool\n\ttlsc *tls.Config\n}\n\nfunc testMQTTGetClient(t testing.TB, s *Server, clientID string) *client {\n\tt.Helper()\n\tvar mc *client\n\ts.mu.Lock()\n\tfor _, c := range s.clients {\n\t\tc.mu.Lock()\n\t\tif c.isMqtt() && c.mqtt.cid == clientID {\n\t\t\tmc = c\n\t\t}\n\t\tc.mu.Unlock()\n\t\tif mc != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\ts.mu.Unlock()\n\tif mc == nil {\n\t\tt.Fatalf(\"Did not find client %q\", clientID)\n\t}\n\treturn mc\n}\n\nfunc testMQTTGetAccountSessionManager(t *testing.T, s *Server, cid string) *mqttAccountSessionManager {\n\tt.Helper()\n\tc := testMQTTGetClient(t, s, cid)\n\trequire_NotNil(t, c)\n\tasm := c.mqtt.asm\n\trequire_NotNil(t, asm)\n\treturn asm\n}\n\nfunc testMQTTRead(c net.Conn) ([]byte, error) {\n\tvar buf [512]byte\n\t// Make sure that test does not block\n\tc.SetReadDeadline(time.Now().Add(testMQTTTimeout))\n\tn, err := c.Read(buf[:])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tc.SetReadDeadline(time.Time{})\n\treturn copyBytes(buf[:n]), nil\n}\n\nfunc testMQTTWrite(c net.Conn, buf []byte) (int, error) {\n\tc.SetWriteDeadline(time.Now().Add(testMQTTTimeout))\n\tn, err := c.Write(buf)\n\tc.SetWriteDeadline(time.Time{})\n\treturn n, err\n}\n\nfunc testMQTTConnect(t testing.TB, ci *mqttConnInfo, host string, port int) (net.Conn, *mqttReader) {\n\tt.Helper()\n\treturn testMQTTConnectRetry(t, ci, host, port, 0)\n}\n\nfunc testMQTTConnectRetry(t testing.TB, ci *mqttConnInfo, host string, port int, retryCount int) (net.Conn, *mqttReader) {\n\tt.Helper()\n\tc, r, err := testMQTTConnectRetryWithError(t, ci, host, port, retryCount)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn c, r\n}\n\nfunc testMQTTConnectRetryWithError(t testing.TB, ci *mqttConnInfo, host string, port int, retryCount int) (net.Conn, *mqttReader, error) {\n\tretry := func(c net.Conn) bool {\n\t\tif c != nil {\n\t\t\tc.Close()\n\t\t}\n\t\tif retryCount == 0 {\n\t\t\treturn false\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t\tretryCount--\n\t\treturn true\n\t}\n\n\taddr := net.JoinHostPort(host, fmt.Sprintf(\"%d\", port))\n\tvar c net.Conn\n\tvar err error\nRETRY:\n\tif ci.ws {\n\t\tvar br *bufio.Reader\n\t\tc, br, _, err = testNewWSClientWithError(t, testWSClientOptions{\n\t\t\thost: host,\n\t\t\tport: port,\n\t\t\tnoTLS: !ci.tls,\n\t\t\tpath: mqttWSPath,\n\t\t})\n\t\tif err == nil {\n\t\t\tc = &mqttWrapAsWs{Conn: c, t: t, br: br}\n\t\t}\n\t} else {\n\t\tc, err = net.Dial(\"tcp\", addr)\n\t\tif err == nil && ci.tls {\n\t\t\ttc := ci.tlsc\n\t\t\tif tc == nil {\n\t\t\t\ttc = &tls.Config{InsecureSkipVerify: true}\n\t\t\t}\n\t\t\tc = tls.Client(c, tc)\n\t\t\tc.SetDeadline(time.Now().Add(time.Second))\n\t\t\terr = c.(*tls.Conn).Handshake()\n\t\t\tc.SetDeadline(time.Time{})\n\t\t}\n\t}\n\tif err != nil {\n\t\tif retry(c) {\n\t\t\tgoto RETRY\n\t\t}\n\t\treturn nil, nil, fmt.Errorf(\"Error creating mqtt connection: %v\", err)\n\t}\n\n\tproto := mqttCreateConnectProto(ci)\n\tif _, err := testMQTTWrite(c, proto); err != nil {\n\t\tif retry(c) {\n\t\t\tgoto RETRY\n\t\t}\n\t\treturn nil, nil, fmt.Errorf(\"Error writing connect: %v\", err)\n\t}\n\n\tbuf, err := testMQTTRead(c)\n\tif err != nil {\n\t\tif retry(c) {\n\t\t\tgoto RETRY\n\t\t}\n\t\treturn nil, nil, fmt.Errorf(\"Error reading: %v\", err)\n\t}\n\tmr := &mqttReader{reader: c}\n\tmr.reset(buf)\n\n\treturn c, mr, nil\n}\n\nfunc mqttCreateConnectProto(ci *mqttConnInfo) []byte {\n\tflags := byte(0)\n\tif ci.cleanSess {\n\t\tflags |= mqttConnFlagCleanSession\n\t}\n\tif ci.will != nil {\n\t\tflags |= mqttConnFlagWillFlag | (ci.will.qos << 3)\n\t\tif ci.will.retain {\n\t\t\tflags |= mqttConnFlagWillRetain\n\t\t}\n\t}\n\tif ci.user != _EMPTY_ {\n\t\tflags |= mqttConnFlagUsernameFlag\n\t}\n\tif ci.pass != _EMPTY_ {\n\t\tflags |= mqttConnFlagPasswordFlag\n\t}\n\n\tpkLen := 2 + len(mqttProtoName) +\n\t\t1 + // proto level\n\t\t1 + // flags\n\t\t2 + // keepAlive\n\t\t2 + len(ci.clientID)\n\n\tif ci.will != nil {\n\t\tpkLen += 2 + len(ci.will.topic)\n\t\tpkLen += 2 + len(ci.will.message)\n\t}\n\tif ci.user != _EMPTY_ {\n\t\tpkLen += 2 + len(ci.user)\n\t}\n\tif ci.pass != _EMPTY_ {\n\t\tpkLen += 2 + len(ci.pass)\n\t}\n\n\tw := newMQTTWriter(0)\n\tw.WriteByte(mqttPacketConnect)\n\tw.WriteVarInt(pkLen)\n\tw.WriteString(string(mqttProtoName))\n\tw.WriteByte(0x4)\n\tw.WriteByte(flags)\n\tw.WriteUint16(ci.keepAlive)\n\tw.WriteString(ci.clientID)\n\tif ci.will != nil {\n\t\tw.WriteBytes(ci.will.topic)\n\t\tw.WriteBytes(ci.will.message)\n\t}\n\tif ci.user != _EMPTY_ {\n\t\tw.WriteString(ci.user)\n\t}\n\tif ci.pass != _EMPTY_ {\n\t\tw.WriteBytes([]byte(ci.pass))\n\t}\n\treturn w.Bytes()\n}\n\nfunc testMQTTCheckConnAck(t testing.TB, r *mqttReader, rc byte, sessionPresent bool) {\n\tt.Helper()\n\tb, pl := testMQTTReadPacket(t, r)\n\tpt := b & mqttPacketMask\n\tif pt != mqttPacketConnectAck {\n\t\tt.Fatalf(\"Expected ConnAck (%x), got %x\", mqttPacketConnectAck, pt)\n\t}\n\tif pl != 2 {\n\t\tt.Fatalf(\"ConnAck packet length should be 2, got %v\", pl)\n\t}\n\tcaf, err := r.readByte(\"connack flags\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading packet length: %v\", err)\n\t}\n\tif caf&0xfe != 0 {\n\t\tt.Fatalf(\"ConnAck flag bits 7-1 should all be 0, got %x\", caf>>1)\n\t}\n\tif sp := caf == 1; sp != sessionPresent {\n\t\tt.Fatalf(\"Expected session present flag=%v got %v\", sessionPresent, sp)\n\t}\n\tcarc, err := r.readByte(\"connack return code\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading returned code: %v\", err)\n\t}\n\tif carc != rc {\n\t\tt.Fatalf(\"Expected return code to be %v, got %v\", rc, carc)\n\t}\n}\n\nfunc testMQTTCheckPubAck(t testing.TB, r *mqttReader, packetType byte) {\n\tt.Helper()\n\tb, pl := testMQTTReadPacket(t, r)\n\tpt := b & mqttPacketMask\n\tif pt != packetType {\n\t\tt.Fatalf(\"Expected %x, got %x\", packetType, pt)\n\t}\n\tr.pos += pl\n}\n\nfunc TestMQTTRequiresJSEnabled(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\tacc := NewAccount(\"mqtt\")\n\to.Accounts = []*Account{acc}\n\to.Users = []*User{{Username: \"mqtt\", Account: acc}}\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\taddr := net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port))\n\tc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating mqtt connection: %v\", err)\n\t}\n\tdefer c.Close()\n\n\tproto := mqttCreateConnectProto(&mqttConnInfo{cleanSess: true, user: \"mqtt\"})\n\tif _, err := testMQTTWrite(c, proto); err != nil {\n\t\tt.Fatalf(\"Error writing connect: %v\", err)\n\t}\n\tif _, err := testMQTTRead(c); err == nil {\n\t\tt.Fatal(\"Expected failure, did not get one\")\n\t}\n}\n\nfunc testMQTTEnableJSForAccount(t *testing.T, s *Server, accName string) {\n\tt.Helper()\n\tacc, err := s.LookupAccount(accName)\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\tlimits := map[string]JetStreamAccountLimits{\n\t\t_EMPTY_: {\n\t\t\tMaxConsumers: -1,\n\t\t\tMaxStreams: -1,\n\t\t\tMaxMemory: 1024 * 1024,\n\t\t\tMaxStore: 1024 * 1024,\n\t\t},\n\t}\n\tif err := acc.EnableJetStream(limits, nil); err != nil {\n\t\tt.Fatalf(\"Error enabling JS: %v\", err)\n\t}\n}\n\nfunc TestMQTTTLSVerifyAndMap(t *testing.T) {\n\taccName := \"MyAccount\"\n\tacc := NewAccount(accName)\n\tcertUserName := \"CN=example.com,OU=NATS.io\"\n\tusers := []*User{{Username: certUserName, Account: acc}}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tfiltering bool\n\t\tprovideCert bool\n\t}{\n\t\t{\"no filtering, client provides cert\", false, true},\n\t\t{\"no filtering, client does not provide cert\", false, false},\n\t\t{\"filtering, client provides cert\", true, true},\n\t\t{\"filtering, client does not provide cert\", true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\to.Host = \"localhost\"\n\t\t\to.Accounts = []*Account{acc}\n\t\t\to.Users = users\n\t\t\tif test.filtering {\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt})\n\t\t\t}\n\t\t\ttc := &TLSConfigOpts{\n\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/server.pem\",\n\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/server-key.pem\",\n\t\t\t\tCaFile: \"../test/configs/certs/tlsauth/ca.pem\",\n\t\t\t\tVerify: true,\n\t\t\t}\n\t\t\ttlsc, err := GenTLSConfig(tc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating tls config: %v\", err)\n\t\t\t}\n\t\t\to.MQTT.TLSConfig = tlsc\n\t\t\to.MQTT.TLSTimeout = 2.0\n\t\t\to.MQTT.TLSMap = true\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\ttestMQTTEnableJSForAccount(t, s, accName)\n\n\t\t\ttlscc := &tls.Config{}\n\t\t\tif test.provideCert {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/client.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/client-key.pem\",\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\ttlscc, err = GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\ttlscc.InsecureSkipVerify = true\n\t\t\tif test.provideCert {\n\t\t\t\ttlscc.MinVersion = tls.VersionTLS13\n\t\t\t}\n\t\t\tmc, r, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\ttls: true,\n\t\t\t\ttlsc: tlscc,\n\t\t\t}, o.MQTT.Host, o.MQTT.Port, 0)\n\t\t\tif !test.provideCert {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected error, did not get one\")\n\t\t\t\t} else if !strings.Contains(err.Error(), \"bad certificate\") && !strings.Contains(err.Error(), \"certificate required\") {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading: %v\", err)\n\t\t\t}\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tvar c *client\n\t\t\ts.mu.Lock()\n\t\t\tfor _, sc := range s.clients {\n\t\t\t\tsc.mu.Lock()\n\t\t\t\tif sc.isMqtt() {\n\t\t\t\t\tc = sc\n\t\t\t\t}\n\t\t\t\tsc.mu.Unlock()\n\t\t\t\tif c != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.mu.Unlock()\n\t\t\tif c == nil {\n\t\t\t\tt.Fatal(\"Client not found\")\n\t\t\t}\n\n\t\t\tvar uname string\n\t\t\tvar accname string\n\t\t\tc.mu.Lock()\n\t\t\tuname = c.opts.Username\n\t\t\tif c.acc != nil {\n\t\t\t\taccname = c.acc.GetName()\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\tif uname != certUserName {\n\t\t\t\tt.Fatalf(\"Expected username %q, got %q\", certUserName, uname)\n\t\t\t}\n\t\t\tif accname != accName {\n\t\t\t\tt.Fatalf(\"Expected account %q, got %v\", accName, accname)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTBasicAuth(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\tuser string\n\t\tpass string\n\t\trc byte\n\t}{\n\t\t{\n\t\t\t\"top level auth, no override, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"mqtt\", \"client\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, no override, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, mqtt auth, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.MQTT.Username = \"mqtt\"\n\t\t\t\to.MQTT.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, mqtt auth, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.MQTT.Username = \"mqtt\"\n\t\t\t\to.MQTT.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"mqtt\", \"client\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, mqtt override, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\to.MQTT.Username = \"mqtt\"\n\t\t\t\to.MQTT.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, mqtt override, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\to.MQTT.Username = \"mqtt\"\n\t\t\t\to.MQTT.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"mqtt\", \"client\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tci := &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\tuser: test.user,\n\t\t\t\tpass: test.pass,\n\t\t\t}\n\t\t\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, test.rc, false)\n\t\t})\n\t}\n}\n\nfunc TestMQTTAuthTimeout(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tat float64\n\t\tmat float64\n\t\tok bool\n\t}{\n\t\t{\"use top-level auth timeout\", 0.5, 0.0, true},\n\t\t{\"use mqtt auth timeout\", 0.5, 0.05, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\to.AuthTimeout = test.at\n\t\t\to.MQTT.Username = \"mqtt\"\n\t\t\to.MQTT.Password = \"client\"\n\t\t\to.MQTT.AuthTimeout = test.mat\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tmc, err := net.Dial(\"tcp\", net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port)))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t\t\t}\n\t\t\tdefer mc.Close()\n\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\tci := &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\tuser: \"mqtt\",\n\t\t\t\tpass: \"client\",\n\t\t\t}\n\t\t\tproto := mqttCreateConnectProto(ci)\n\t\t\tif _, err := testMQTTWrite(mc, proto); err != nil {\n\t\t\t\tif test.ok {\n\t\t\t\t\tt.Fatalf(\"Error sending connect: %v\", err)\n\t\t\t\t}\n\t\t\t\t// else it is ok since we got disconnected due to auth timeout\n\t\t\t\treturn\n\t\t\t}\n\t\t\tbuf, err := testMQTTRead(mc)\n\t\t\tif err != nil {\n\t\t\t\tif test.ok {\n\t\t\t\t\tt.Fatalf(\"Error reading: %v\", err)\n\t\t\t\t}\n\t\t\t\t// else it is ok since we got disconnected due to auth timeout\n\t\t\t\treturn\n\t\t\t}\n\t\t\tr := &mqttReader{reader: mc}\n\t\t\tr.reset(buf)\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\ttestMQTTPublish(t, mc, r, 1, false, false, \"foo\", 1, []byte(\"msg\"))\n\t\t})\n\t}\n}\n\nfunc TestMQTTTokenAuth(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\ttoken string\n\t\trc byte\n\t}{\n\t\t{\n\t\t\t\"top level auth, no override, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Authorization = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"badtoken\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, no override, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Authorization = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"goodtoken\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, mqtt auth, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.MQTT.Token = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"badtoken\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, mqtt auth, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.MQTT.Token = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"goodtoken\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, mqtt override, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Authorization = \"clienttoken\"\n\t\t\t\to.MQTT.Token = \"mqtttoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"clienttoken\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, mqtt override, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Authorization = \"clienttoken\"\n\t\t\t\to.MQTT.Token = \"mqtttoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"mqtttoken\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tci := &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\tuser: \"ignore_use_token\",\n\t\t\t\tpass: test.token,\n\t\t\t}\n\t\t\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, test.rc, false)\n\t\t})\n\t}\n}\n\nfunc TestMQTTJWTWithAllowedConnectionTypes(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\t// Create System Account\n\tsyskp, _ := nkeys.CreateAccount()\n\tsyspub, _ := syskp.PublicKey()\n\tsysAc := jwt.NewAccountClaims(syspub)\n\tsysjwt, err := sysAc.Encode(oKp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\t// Create memory resolver and store system account\n\tmr := &MemAccResolver{}\n\tmr.Store(syspub, sysjwt)\n\tif err != nil {\n\t\tt.Fatalf(\"Error saving system account JWT to memory resolver: %v\", err)\n\t}\n\t// Add system account and memory resolver to server options\n\to.SystemAccount = syspub\n\to.AccountResolver = mr\n\tsetupAddTrusted(o)\n\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconnectionTypes []string\n\t\trc byte\n\t}{\n\t\t{\"not allowed\", []string{jwt.ConnectionTypeStandard}, mqttConnAckRCNotAuthorized},\n\t\t{\"allowed\", []string{jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeMqtt)}, mqttConnAckRCConnectionAccepted},\n\t\t{\"allowed with unknown\", []string{jwt.ConnectionTypeMqtt, \"SomeNewType\"}, mqttConnAckRCConnectionAccepted},\n\t\t{\"not allowed with unknown\", []string{\"SomeNewType\"}, mqttConnAckRCNotAuthorized},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\tnuc := newJWTTestUserClaims()\n\t\t\tnuc.AllowedConnectionTypes = test.connectionTypes\n\t\t\tnuc.BearerToken = true\n\n\t\t\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t\t\takp, _ := nkeys.CreateAccount()\n\t\t\tapub, _ := akp.PublicKey()\n\t\t\tnac := jwt.NewAccountClaims(apub)\n\t\t\t// Enable Jetstream on account with lax limitations\n\t\t\tnac.Limits.JetStreamLimits.Consumer = -1\n\t\t\tnac.Limits.JetStreamLimits.Streams = -1\n\t\t\tnac.Limits.JetStreamLimits.MemoryStorage = 1024 * 1024\n\t\t\tnac.Limits.JetStreamLimits.DiskStorage = 1024 * 1024\n\t\t\tajwt, err := nac.Encode(okp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t\t\t}\n\n\t\t\tnkp, _ := nkeys.CreateUser()\n\t\t\tpub, _ := nkp.PublicKey()\n\t\t\tnuc.Subject = pub\n\t\t\tjwt, err := nuc.Encode(akp)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t\t\t}\n\n\t\t\taddAccountToMemResolver(s, apub, ajwt)\n\n\t\t\tci := &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\tuser: \"ignore_use_token\",\n\t\t\t\tpass: jwt,\n\t\t\t}\n\n\t\t\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, test.rc, false)\n\t\t})\n\t}\n}\n\nfunc TestMQTTUsersAuth(t *testing.T) {\n\tusers := []*User{{Username: \"user\", Password: \"pwd\"}}\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\tuser string\n\t\tpass string\n\t\trc byte\n\t}{\n\t\t{\n\t\t\t\"no filtering, wrong user\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Users = users\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"wronguser\", \"pwd\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"no filtering, correct user\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Users = users\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"filtering, user not allowed\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Users = users\n\t\t\t\t// Only allowed for regular clients\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t\t{\n\t\t\t\"filtering, user allowed\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Users = users\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", mqttConnAckRCConnectionAccepted,\n\t\t},\n\t\t{\n\t\t\t\"filtering, wrong password\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testMQTTDefaultOptions()\n\t\t\t\to.Users = users\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeMqtt})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"badpassword\", mqttConnAckRCNotAuthorized,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tci := &mqttConnInfo{\n\t\t\t\tcleanSess: true,\n\t\t\t\tuser: test.user,\n\t\t\t\tpass: test.pass,\n\t\t\t}\n\t\t\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, test.rc, false)\n\t\t})\n\t}\n}\n\nfunc TestMQTTNoAuthUserValidation(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.Users = []*User{{Username: \"user\", Password: \"pwd\"}}\n\t// Should fail because it is not part of o.Users.\n\to.MQTT.NoAuthUser = \"notfound\"\n\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), \"not present as user\") {\n\t\tt.Fatalf(\"Expected error saying not present as user, got %v\", err)\n\t}\n\n\t// Set a valid no auth user for global options, but still should fail because\n\t// of o.MQTT.NoAuthUser\n\to.NoAuthUser = \"user\"\n\to.MQTT.NoAuthUser = \"notfound\"\n\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), \"not present as user\") {\n\t\tt.Fatalf(\"Expected error saying not present as user, got %v\", err)\n\t}\n}\n\nfunc TestMQTTNoAuthUser(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\toverride bool\n\t\tuseAuth bool\n\t\texpectedUser string\n\t\texpectedAcc string\n\t}{\n\t\t{\"no override, no user provided\", false, false, \"noauth\", \"normal\"},\n\t\t{\"no override, user povided\", false, true, \"user\", \"normal\"},\n\t\t{\"override, no user provided\", true, false, \"mqttnoauth\", \"mqtt\"},\n\t\t{\"override, user provided\", true, true, \"mqttuser\", \"mqtt\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\tnormalAcc := NewAccount(\"normal\")\n\t\t\tmqttAcc := NewAccount(\"mqtt\")\n\t\t\to.Accounts = []*Account{normalAcc, mqttAcc}\n\t\t\to.Users = []*User{\n\t\t\t\t{Username: \"noauth\", Password: \"pwd\", Account: normalAcc},\n\t\t\t\t{Username: \"user\", Password: \"pwd\", Account: normalAcc},\n\t\t\t\t{Username: \"mqttnoauth\", Password: \"pwd\", Account: mqttAcc},\n\t\t\t\t{Username: \"mqttuser\", Password: \"pwd\", Account: mqttAcc},\n\t\t\t}\n\t\t\to.NoAuthUser = \"noauth\"\n\t\t\tif test.override {\n\t\t\t\to.MQTT.NoAuthUser = \"mqttnoauth\"\n\t\t\t}\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\ttestMQTTEnableJSForAccount(t, s, \"normal\")\n\t\t\ttestMQTTEnableJSForAccount(t, s, \"mqtt\")\n\n\t\t\tci := &mqttConnInfo{clientID: \"mqtt\", cleanSess: true}\n\t\t\tif test.useAuth {\n\t\t\t\tci.user = test.expectedUser\n\t\t\t\tci.pass = \"pwd\"\n\t\t\t}\n\t\t\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tc := testMQTTGetClient(t, s, \"mqtt\")\n\t\t\tc.mu.Lock()\n\t\t\tuname := c.opts.Username\n\t\t\taname := c.acc.GetName()\n\t\t\tc.mu.Unlock()\n\t\t\tif uname != test.expectedUser {\n\t\t\t\tt.Fatalf(\"Expected selected user to be %q, got %q\", test.expectedUser, uname)\n\t\t\t}\n\t\t\tif aname != test.expectedAcc {\n\t\t\t\tt.Fatalf(\"Expected selected account to be %q, got %q\", test.expectedAcc, aname)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTConnectNotFirstPacket(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tc, err := net.Dial(\"tcp\", net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t}\n\tdefer c.Close()\n\n\ttestMQTTSendPublishPacket(t, c, 0, false, false, \"foo\", 0, []byte(\"hello\"))\n\ttestMQTTExpectDisconnect(t, c)\n\n\tselect {\n\tcase err := <-l.errCh:\n\t\tif !strings.Contains(err, \"should be a CONNECT\") {\n\t\t\tt.Fatalf(\"Expected error about first packet being a CONNECT, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not log any error\")\n\t}\n}\n\nfunc TestMQTTSecondConnect(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tproto := mqttCreateConnectProto(&mqttConnInfo{cleanSess: true})\n\tif _, err := testMQTTWrite(mc, proto); err != nil {\n\t\tt.Fatalf(\"Error writing connect: %v\", err)\n\t}\n\ttestMQTTExpectDisconnect(t, mc)\n}\n\nfunc TestMQTTParseConnect(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto []byte\n\t\terr string\n\t\twantErr bool\n\t}{\n\t\t{\"packet in buffer error\", []byte{0}, io.ErrUnexpectedEOF.Error(), false},\n\t\t{\"bad proto name\", []byte{0, 4, 'B', 'A', 'D'}, \"protocol name\", false},\n\t\t{\"invalid proto name\", []byte{0, 3, 'B', 'A', 'D'}, \"expected connect packet with protocol name\", false},\n\t\t{\"old proto not supported\", []byte{0, 6, 'M', 'Q', 'I', 's', 'd', 'p'}, \"older protocol\", false},\n\t\t{\"error on protocol level\", []byte{0, 4, 'M', 'Q', 'T', 'T'}, \"protocol level\", false},\n\t\t{\"unacceptable protocol version\", []byte{0, 4, 'M', 'Q', 'T', 'T', 10}, \"unacceptable protocol version\", false},\n\t\t{\"error on flags\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel}, \"flags\", false},\n\t\t{\"reserved flag\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1}, errMQTTConnFlagReserved.Error(), false},\n\t\t{\"will qos without will flag\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1 << 3}, \"if Will flag is set to 0, Will QoS must be 0 too\", false},\n\t\t{\"will retain without will flag\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 1 << 5}, errMQTTWillAndRetainFlag.Error(), false},\n\t\t{\"will qos\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 3<<3 | 1<<2}, \"if Will flag is set to 1, Will QoS can be 0, 1 or 2\", false},\n\t\t{\"no user but password\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagPasswordFlag}, errMQTTPasswordFlagAndNoUser.Error(), false},\n\t\t{\"missing keep alive\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0}, \"keep alive\", false},\n\t\t{\"missing client ID\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1}, \"client ID\", false},\n\t\t{\"empty client ID\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1, 0, 0}, errMQTTCIDEmptyNeedsCleanFlag.Error(), false},\n\t\t{\"invalid utf8 client ID\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1, 0, 1, 241}, \"invalid utf8 for client ID\", false},\n\t\t{\"client ID with null byte\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, 0, 0, 1, 0, 3, 'a', 0, 'b'}, \"\", true},\n\t\t{\"missing will topic\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0}, \"Will topic\", false},\n\t\t{\"empty will topic\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 0}, errMQTTEmptyWillTopic.Error(), false},\n\t\t{\"invalid utf8 will topic\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 241}, \"invalid utf8 for Will topic\", false},\n\t\t{\"invalid wildcard will topic\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, '#'}, \"wildcards not allowed\", false},\n\t\t{\"error on will message\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagWillFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 'a', 0, 3}, \"Will message\", false},\n\t\t{\"error on username\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0}, \"user name\", false},\n\t\t{\"empty username\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 0}, errMQTTEmptyUsername.Error(), false},\n\t\t{\"invalid utf8 username\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 241}, \"invalid utf8 for user name\", false},\n\t\t{\"username with null byte\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 3, 'a', 0, 'b'}, \"\", true},\n\t\t{\"error on password\", []byte{0, 4, 'M', 'Q', 'T', 'T', mqttProtoLevel, mqttConnFlagUsernameFlag | mqttConnFlagPasswordFlag | mqttConnFlagCleanSession, 0, 0, 0, 0, 0, 1, 'a'}, \"password\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tr := &mqttReader{}\n\t\t\tr.reset(test.proto)\n\t\t\tmqtt := &mqtt{r: r}\n\t\t\tc := &client{mqtt: mqtt}\n\t\t\t_, _, err := c.mqttParseConnect(r, false)\n\t\t\tif test.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected an error, got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTConnectFailsOnParse(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\taddr := net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port))\n\tc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating mqtt connection: %v\", err)\n\t}\n\n\tpkLen := 2 + len(mqttProtoName) +\n\t\t1 + // proto level\n\t\t1 + // flags\n\t\t2 + // keepAlive\n\t\t2 + len(\"mqtt\")\n\n\tw := newMQTTWriter(0)\n\tw.WriteByte(mqttPacketConnect)\n\tw.WriteVarInt(pkLen)\n\tw.WriteString(string(mqttProtoName))\n\tw.WriteByte(0x7)\n\tw.WriteByte(mqttConnFlagCleanSession)\n\tw.WriteUint16(0)\n\tw.WriteString(\"mqtt\")\n\tc.Write(w.Bytes())\n\n\tbuf, err := testMQTTRead(c)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading: %v\", err)\n\t}\n\tr := &mqttReader{reader: c}\n\tr.reset(buf)\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCUnacceptableProtocolVersion, false)\n}\n\nfunc TestMQTTConnKeepAlive(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true, keepAlive: 1}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, mc, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\n\ttime.Sleep(2 * time.Second)\n\ttestMQTTExpectDisconnect(t, mc)\n}\n\nfunc TestMQTTMalformedFixedHeaderFlagsCauseDisconnect(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpacket func(t *testing.T) (net.Conn, []byte)\n\t}{\n\t\t{\n\t\t\tname: \"connect\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\taddr := net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port))\n\t\t\t\tc, err := net.Dial(\"tcp\", addr)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error creating mqtt connection: %v\", err)\n\t\t\t\t}\n\t\t\t\tproto := mqttCreateConnectProto(&mqttConnInfo{cleanSess: true})\n\t\t\t\tproto[0] |= 0x1\n\t\t\t\treturn c, proto\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pingreq\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPing | 0x1, 0}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"puback\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubAck | 0x1, 2, 0, 1}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubrec\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubRec | 0x1, 2, 0, 1}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubrel\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubRel | 0x1, 2, 0, 1}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubcomp\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubComp | 0x1, 2, 0, 1}\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, packet := test.packet(t)\n\t\t\tdefer c.Close()\n\n\t\t\tif _, err := testMQTTWrite(c, packet); err != nil {\n\t\t\t\tt.Fatalf(\"Error writing malformed %s packet: %v\", test.name, err)\n\t\t\t}\n\t\t\ttestMQTTExpectDisconnect(t, c)\n\t\t})\n\t}\n}\n\nfunc TestMQTTMalformedRemainingLengthCausesDisconnect(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpacket func(t *testing.T) (net.Conn, []byte)\n\t}{\n\t\t{\n\t\t\tname: \"pingreq\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPing, 1, 0}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"puback\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubAck, 3, 0, 1, 0}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubrec\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubRec, 3, 0, 1, 0}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubrel\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubRel | 0x2, 3, 0, 1, 0}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"pubcomp\",\n\t\t\tpacket: func(t *testing.T) (net.Conn, []byte) {\n\t\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\treturn mc, []byte{mqttPacketPubComp, 3, 0, 1, 0}\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, packet := test.packet(t)\n\t\t\tdefer c.Close()\n\n\t\t\tif _, err := testMQTTWrite(c, packet); err != nil {\n\t\t\t\tt.Fatalf(\"Error writing malformed %s packet: %v\", test.name, err)\n\t\t\t}\n\t\t\ttestMQTTExpectDisconnect(t, c)\n\t\t})\n\t}\n}\n\nfunc TestMQTTDontSetPinger(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.PingInterval = 15 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"mqtt\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tc := testMQTTGetClient(t, s, \"mqtt\")\n\tc.mu.Lock()\n\ttimerSet := c.ping.tmr != nil\n\tc.mu.Unlock()\n\tif timerSet {\n\t\tt.Fatalf(\"Ping timer should not be set for MQTT clients\")\n\t}\n\n\t// Wait a bit and expect nothing (and connection should still be valid)\n\ttestMQTTExpectNothing(t, r)\n\ttestMQTTPublish(t, mc, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n}\n\nfunc TestMQTTTopicAndSubjectConversion(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmqttTopic string\n\t\tnatsSubject string\n\t\terr string\n\t}{\n\t\t{\"/\", \"/\", \"/./\", \"\"},\n\t\t{\"//\", \"//\", \"/././\", \"\"},\n\t\t{\"///\", \"///\", \"/./././\", \"\"},\n\t\t{\"////\", \"////\", \"/././././\", \"\"},\n\t\t{\"foo\", \"foo\", \"foo\", \"\"},\n\t\t{\"/foo\", \"/foo\", \"/.foo\", \"\"},\n\t\t{\"//foo\", \"//foo\", \"/./.foo\", \"\"},\n\t\t{\"///foo\", \"///foo\", \"/././.foo\", \"\"},\n\t\t{\"///foo/\", \"///foo/\", \"/././.foo./\", \"\"},\n\t\t{\"///foo//\", \"///foo//\", \"/././.foo././\", \"\"},\n\t\t{\"///foo///\", \"///foo///\", \"/././.foo./././\", \"\"},\n\t\t{\"//.foo.//\", \"//.foo.//\", \"/././/foo//././\", \"\"},\n\t\t{\"foo/bar\", \"foo/bar\", \"foo.bar\", \"\"},\n\t\t{\"/foo/bar\", \"/foo/bar\", \"/.foo.bar\", \"\"},\n\t\t{\"/foo/bar/\", \"/foo/bar/\", \"/.foo.bar./\", \"\"},\n\t\t{\"foo/bar/baz\", \"foo/bar/baz\", \"foo.bar.baz\", \"\"},\n\t\t{\"/foo/bar/baz\", \"/foo/bar/baz\", \"/.foo.bar.baz\", \"\"},\n\t\t{\"/foo/bar/baz/\", \"/foo/bar/baz/\", \"/.foo.bar.baz./\", \"\"},\n\t\t{\"bar\", \"bar/\", \"bar./\", \"\"},\n\t\t{\"bar//\", \"bar//\", \"bar././\", \"\"},\n\t\t{\"bar///\", \"bar///\", \"bar./././\", \"\"},\n\t\t{\"foo//bar\", \"foo//bar\", \"foo./.bar\", \"\"},\n\t\t{\"foo///bar\", \"foo///bar\", \"foo././.bar\", \"\"},\n\t\t{\"foo////bar\", \"foo////bar\", \"foo./././.bar\", \"\"},\n\t\t{\".\", \".\", \"//\", \"\"},\n\t\t{\"..\", \"..\", \"////\", \"\"},\n\t\t{\"...\", \"...\", \"//////\", \"\"},\n\t\t{\"./\", \"./\", \"//./\", \"\"},\n\t\t{\".//.\", \".//.\", \"//././/\", \"\"},\n\t\t{\"././.\", \"././.\", \"//.//.//\", \"\"},\n\t\t{\"././/.\", \"././/.\", \"//.//././/\", \"\"},\n\t\t{\".foo\", \".foo\", \"//foo\", \"\"},\n\t\t{\"foo.\", \"foo.\", \"foo//\", \"\"},\n\t\t{\".foo.\", \".foo.\", \"//foo//\", \"\"},\n\t\t{\"foo../bar/\", \"foo../bar/\", \"foo////.bar./\", \"\"},\n\t\t{\"foo../bar/.\", \"foo../bar/.\", \"foo////.bar.//\", \"\"},\n\t\t{\"/foo/\", \"/foo/\", \"/.foo./\", \"\"},\n\t\t{\"./foo/.\", \"./foo/.\", \"//.foo.//\", \"\"},\n\t\t{\"foo.bar/baz\", \"foo.bar/baz\", \"foo//bar.baz\", \"\"},\n\t\t// These should produce errors\n\t\t{\"foo/+\", \"foo/+\", \"\", \"wildcards not allowed in publish\"},\n\t\t{\"foo/#\", \"foo/#\", \"\", \"wildcards not allowed in publish\"},\n\t\t{\"foo bar\", \"foo bar\", \"\", \"not supported\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tres, err := mqttTopicToNATSPubSubject([]byte(test.mqttTopic))\n\t\t\tif test.err != _EMPTY_ {\n\t\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttoNATS := string(res)\n\t\t\tif toNATS != test.natsSubject {\n\t\t\t\tt.Fatalf(\"Expected subject %q got %q\", test.natsSubject, toNATS)\n\t\t\t}\n\n\t\t\tres = natsSubjectToMQTTTopic(res)\n\t\t\tbackToMQTT := string(res)\n\t\t\tif backToMQTT != test.mqttTopic {\n\t\t\t\tt.Fatalf(\"Expected topic %q got %q (NATS conversion was %q)\", test.mqttTopic, backToMQTT, toNATS)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTFilterConversion(t *testing.T) {\n\t// Similar to TopicConversion test except that wildcards are OK here.\n\t// So testing only those.\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmqttTopic string\n\t\tnatsSubject string\n\t}{\n\t\t{\"single level wildcard\", \"+\", \"*\"},\n\t\t{\"single level wildcard\", \"/+\", \"/.*\"},\n\t\t{\"single level wildcard\", \"+/\", \"*./\"},\n\t\t{\"single level wildcard\", \"/+/\", \"/.*./\"},\n\t\t{\"single level wildcard\", \"foo/+\", \"foo.*\"},\n\t\t{\"single level wildcard\", \"foo/+/\", \"foo.*./\"},\n\t\t{\"single level wildcard\", \"foo/+/bar\", \"foo.*.bar\"},\n\t\t{\"single level wildcard\", \"foo/+/+\", \"foo.*.*\"},\n\t\t{\"single level wildcard\", \"foo/+/+/\", \"foo.*.*./\"},\n\t\t{\"single level wildcard\", \"foo/+/+/bar\", \"foo.*.*.bar\"},\n\t\t{\"single level wildcard\", \"foo//+\", \"foo./.*\"},\n\t\t{\"single level wildcard\", \"foo//+/\", \"foo./.*./\"},\n\t\t{\"single level wildcard\", \"foo//+//\", \"foo./.*././\"},\n\t\t{\"single level wildcard\", \"foo//+//bar\", \"foo./.*./.bar\"},\n\t\t{\"single level wildcard\", \"foo///+///bar\", \"foo././.*././.bar\"},\n\t\t{\"single level wildcard\", \"foo.bar///+///baz\", \"foo//bar././.*././.baz\"},\n\n\t\t{\"multi level wildcard\", \"#\", \">\"},\n\t\t{\"multi level wildcard\", \"/#\", \"/.>\"},\n\t\t{\"multi level wildcard\", \"/foo/#\", \"/.foo.>\"},\n\t\t{\"multi level wildcard\", \"foo/#\", \"foo.>\"},\n\t\t{\"multi level wildcard\", \"foo//#\", \"foo./.>\"},\n\t\t{\"multi level wildcard\", \"foo///#\", \"foo././.>\"},\n\t\t{\"multi level wildcard\", \"foo/bar/#\", \"foo.bar.>\"},\n\t\t{\"multi level wildcard\", \"foo/bar.baz/#\", \"foo.bar//baz.>\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tres, err := mqttFilterToNATSSubject([]byte(test.mqttTopic))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error: %v\", err)\n\t\t\t}\n\t\t\tif string(res) != test.natsSubject {\n\t\t\t\tt.Fatalf(\"Expected subject %q got %q\", test.natsSubject, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTParseSub(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto []byte\n\t\tb byte\n\t\tpl int\n\t\terr string\n\t\twantErr bool\n\t}{\n\t\t{\"reserved flag\", nil, 3, 0, \"wrong subscribe reserved flags\", false},\n\t\t{\"ensure packet loaded\", []byte{1, 2}, mqttSubscribeFlags, 10, io.ErrUnexpectedEOF.Error(), false},\n\t\t{\"error reading packet id\", []byte{1}, mqttSubscribeFlags, 1, \"reading packet identifier\", false},\n\t\t{\"packet id cannot be zero\", []byte{0, 0}, mqttSubscribeFlags, 2, errMQTTPacketIdentifierIsZero.Error(), false},\n\t\t{\"missing filters\", []byte{0, 1}, mqttSubscribeFlags, 2, \"subscribe protocol must contain at least 1 topic filter\", false},\n\t\t{\"error reading topic\", []byte{0, 1, 0, 2, 'a'}, mqttSubscribeFlags, 5, \"topic filter\", false},\n\t\t{\"empty topic\", []byte{0, 1, 0, 0}, mqttSubscribeFlags, 4, errMQTTTopicFilterCannotBeEmpty.Error(), false},\n\t\t{\"invalid utf8 topic\", []byte{0, 1, 0, 1, 241}, mqttSubscribeFlags, 5, \"invalid utf8 for topic filter\", false},\n\t\t{\"topic with null byte\", []byte{0, 1, 0, 3, 'a', 0, 'b', 0}, mqttSubscribeFlags, 8, \"\", true},\n\t\t{\"missing qos\", []byte{0, 1, 0, 1, 'a'}, mqttSubscribeFlags, 5, \"QoS\", false},\n\t\t{\"invalid qos\", []byte{0, 1, 0, 1, 'a', 3}, mqttSubscribeFlags, 6, \"subscribe QoS value must be 0, 1 or 2\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tr := &mqttReader{}\n\t\t\tr.reset(test.proto)\n\t\t\tmqtt := &mqtt{r: r}\n\t\t\tc := &client{mqtt: mqtt}\n\t\t\t_, _, err := c.mqttParseSubsOrUnsubs(r, test.b, test.pl, true)\n\t\t\tif test.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected an error, got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testMQTTSub(t testing.TB, pi uint16, c net.Conn, r *mqttReader, filters []*mqttFilter, expected []byte) {\n\tt.Helper()\n\tw := newMQTTWriter(0)\n\tpkLen := 2 // for pi\n\tfor i := 0; i < len(filters); i++ {\n\t\tf := filters[i]\n\t\tpkLen += 2 + len(f.filter) + 1\n\t}\n\tw.WriteByte(mqttPacketSub | mqttSubscribeFlags)\n\tw.WriteVarInt(pkLen)\n\tw.WriteUint16(pi)\n\tfor i := 0; i < len(filters); i++ {\n\t\tf := filters[i]\n\t\tw.WriteBytes([]byte(f.filter))\n\t\tw.WriteByte(f.qos)\n\t}\n\tif _, err := testMQTTWrite(c, w.Bytes()); err != nil {\n\t\tt.Fatalf(\"Error writing SUBSCRIBE protocol: %v\", err)\n\t}\n\tb, pl := testMQTTReadPacket(t, r)\n\tif pt := b & mqttPacketMask; pt != mqttPacketSubAck {\n\t\tt.Fatalf(\"Expected SUBACK packet %x, got %x\", mqttPacketSubAck, pt)\n\t}\n\trpi, err := r.readUint16(\"packet identifier\")\n\tif err != nil || rpi != pi {\n\t\tt.Fatalf(\"Error with packet identifier expected=%v got: %v err=%v\", pi, rpi, err)\n\t}\n\tfor i, rem := 0, pl-2; rem > 0; rem-- {\n\t\tqos, err := r.readByte(\"filter qos\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif expected != nil && qos != expected[i] {\n\t\t\tt.Fatalf(\"For topic filter %q expected qos of %v, got %v\",\n\t\t\t\tfilters[i].filter, expected[i], qos)\n\t\t}\n\t\ti++\n\t}\n}\n\nfunc TestMQTTSubAck(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tsubs := []*mqttFilter{\n\t\t{filter: \"foo\", qos: 0},\n\t\t{filter: \"bar\", qos: 1},\n\t\t{filter: \"baz\", qos: 2},\n\t\t{filter: \"foo/#/bar\", qos: 0}, // Invalid sub, so we should receive a result of mqttSubAckFailure\n\t}\n\texpected := []byte{\n\t\t0,\n\t\t1,\n\t\t2,\n\t\tmqttSubAckFailure,\n\t}\n\ttestMQTTSub(t, 1, mc, r, subs, expected)\n}\n\nfunc TestMQTTQoS2SubDowngrade(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.downgradeQoS2Sub = true\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tsubs := []*mqttFilter{\n\t\t{filter: \"bar\", qos: 1},\n\t\t{filter: \"baz\", qos: 2},\n\t}\n\texpected := []byte{\n\t\t1,\n\t\t1,\n\t}\n\ttestMQTTSub(t, 1, mc, r, subs, expected)\n}\n\nfunc testMQTTFlush(t testing.TB, c net.Conn, bw *bufio.Writer, r *mqttReader) {\n\tt.Helper()\n\tw := newMQTTWriter(0)\n\tw.WriteByte(mqttPacketPing)\n\tw.WriteByte(0)\n\tif bw != nil {\n\t\tbw.Write(w.Bytes())\n\t\tbw.Flush()\n\t} else {\n\t\tc.Write(w.Bytes())\n\t}\n\tab, l := testMQTTReadPacket(t, r)\n\tif pt := ab & mqttPacketMask; pt != mqttPacketPingResp {\n\t\tt.Fatalf(\"Expected ping response got %x\", pt)\n\t}\n\tif l != 0 {\n\t\tt.Fatalf(\"Expected PINGRESP length to be 0, got %v\", l)\n\t}\n}\n\nfunc testMQTTExpectNothing(t testing.TB, r *mqttReader) {\n\tt.Helper()\n\t// First, check that we don't have buffered data.\n\tif r.hasMore() {\n\t\tt.Fatalf(\"Expected nothing, got %v\", r.buf[r.pos:])\n\t}\n\t// Then, try to read from the reader with some timeout.\n\tvar buf [128]byte\n\tr.reader.SetReadDeadline(time.Now().Add(100 * time.Millisecond))\n\tif n, err := r.reader.Read(buf[:]); err == nil {\n\t\tt.Fatalf(\"Expected nothing, got %v\", buf[:n])\n\t}\n\tr.reader.SetReadDeadline(time.Time{})\n}\n\nfunc testMQTTCheckPubMsg(t testing.TB, c net.Conn, r *mqttReader, topic string, expectedFlags byte, payload []byte) uint16 {\n\tt.Helper()\n\tpi := testMQTTCheckPubMsgNoAck(t, c, r, topic, expectedFlags, payload)\n\tif pi == 0 {\n\t\treturn 0\n\t}\n\tqos := mqttGetQoS(expectedFlags)\n\tswitch qos {\n\tcase 1:\n\t\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\tcase 2:\n\t\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, pi)\n\t}\n\treturn pi\n}\n\nfunc testMQTTCheckPubMsgNoAck(t testing.TB, c net.Conn, r *mqttReader, topic string, expectedFlags byte, payload []byte) uint16 {\n\tt.Helper()\n\tpflags, pi := testMQTTGetPubMsg(t, c, r, topic, payload)\n\tif pflags != expectedFlags {\n\t\tt.Fatalf(\"Expected flags to be %x, got %x\", expectedFlags, pflags)\n\t}\n\treturn pi\n}\n\nfunc testMQTTGetPubMsg(t testing.TB, c net.Conn, r *mqttReader, topic string, payload []byte) (byte, uint16) {\n\tt.Helper()\n\tflags, pi, _, _ := testMQTTGetPubMsgEx(t, c, r, topic, payload)\n\treturn flags, pi\n}\n\nfunc testMQTTGetPubMsgEx(t testing.TB, _ net.Conn, r *mqttReader, topic string, payload []byte) (byte, uint16, string, []byte) {\n\tt.Helper()\n\tb, pl := testMQTTReadPacket(t, r)\n\tif pt := b & mqttPacketMask; pt != mqttPacketPub {\n\t\tt.Fatalf(\"Expected PUBLISH packet %x, got %x\", mqttPacketPub, pt)\n\t}\n\treturn testMQTTGetPubMsgExEx(t, nil, r, b, pl, topic, payload)\n}\n\nfunc testMQTTGetPubMsgExEx(t testing.TB, _ net.Conn, r *mqttReader, b byte, pl int, topic string, payload []byte) (byte, uint16, string, []byte) {\n\tt.Helper()\n\tpflags := b & mqttPacketFlagMask\n\tqos := (pflags & mqttPubFlagQoS) >> 1\n\tstart := r.pos\n\tptopic, err := r.readString(\"topic name\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif topic != _EMPTY_ && ptopic != topic {\n\t\tt.Fatalf(\"Expected topic %q, got %q\", topic, ptopic)\n\t}\n\tvar pi uint16\n\tif qos > 0 {\n\t\tpi, err = r.readUint16(\"packet identifier\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tmsgLen := pl - (r.pos - start)\n\tif r.pos+msgLen > len(r.buf) {\n\t\tt.Fatalf(\"computed message length goes beyond buffer: ml=%v pos=%v lenBuf=%v\",\n\t\t\tmsgLen, r.pos, len(r.buf))\n\t}\n\tppayload := r.buf[r.pos : r.pos+msgLen]\n\tif payload != nil && !bytes.Equal(payload, ppayload) {\n\t\tt.Fatalf(\"Expected payload %q, got %q\", payload, ppayload)\n\t}\n\tr.pos += msgLen\n\treturn pflags, pi, ptopic, ppayload\n}\n\nfunc testMQTTReadPubPacket(t testing.TB, r *mqttReader) (flags byte, pi uint16, topic string, payload []byte) {\n\tt.Helper()\n\tb, pl := testMQTTReadPacket(t, r)\n\tif pt := b & mqttPacketMask; pt != mqttPacketPub {\n\t\tt.Fatalf(\"Expected PUBLISH packet %x, got %x\", mqttPacketPub, pt)\n\t}\n\tflags = b & mqttPacketFlagMask\n\tstart := r.pos\n\ttopic, err := r.readString(\"topic name\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tqos := (flags & mqttPubFlagQoS) >> 1\n\tif qos > 0 {\n\t\tpi, err = r.readUint16(\"packet identifier\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tmsgLen := pl - (r.pos - start)\n\tif r.pos+msgLen > len(r.buf) {\n\t\tt.Fatalf(\"computed message length goes beyond buffer: ml=%v pos=%v lenBuf=%v\",\n\t\t\tmsgLen, r.pos, len(r.buf))\n\t}\n\tpayload = r.buf[r.pos : r.pos+msgLen]\n\tr.pos += msgLen\n\treturn flags, pi, topic, payload\n}\n\nfunc testMQTTSendPIPacket(packetType byte, t testing.TB, c net.Conn, pi uint16) {\n\tt.Helper()\n\tw := newMQTTWriter(0)\n\tw.WriteByte(packetType)\n\tw.WriteVarInt(2)\n\tw.WriteUint16(pi)\n\tif _, err := testMQTTWrite(c, w.Bytes()); err != nil {\n\t\tt.Fatalf(\"Error writing packet type %v: %v\", packetType, err)\n\t}\n}\n\nfunc testMQTTWritePublishPacket(t testing.TB, w *mqttWriter, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) {\n\tt.Helper()\n\tw.WritePublishHeader(pi, qos, dup, retain, []byte(topic), len(payload))\n\tif _, err := w.Write(payload); err != nil {\n\t\tt.Fatalf(\"Error writing PUBLISH proto: %v\", err)\n\t}\n}\n\nfunc testMQTTSendPublishPacket(t testing.TB, c net.Conn, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) {\n\tt.Helper()\n\tc.SetWriteDeadline(time.Now().Add(testMQTTTimeout))\n\t_, header := mqttMakePublishHeader(pi, qos, dup, retain, []byte(topic), len(payload))\n\tif _, err := c.Write(header); err != nil {\n\t\tt.Fatalf(\"Error writing PUBLISH header: %v\", err)\n\t}\n\tif _, err := c.Write(payload); err != nil {\n\t\tt.Fatalf(\"Error writing PUBLISH payload: %v\", err)\n\t}\n\tc.SetWriteDeadline(time.Time{})\n}\n\nfunc testMQTTPublish(t testing.TB, c net.Conn, r *mqttReader, qos byte, dup, retain bool, topic string, pi uint16, payload []byte) {\n\tt.Helper()\n\ttestMQTTSendPublishPacket(t, c, qos, dup, retain, topic, pi, payload)\n\tswitch qos {\n\tcase 1:\n\t\tb, _ := testMQTTReadPacket(t, r)\n\t\tif pt := b & mqttPacketMask; pt != mqttPacketPubAck {\n\t\t\tt.Fatalf(\"Expected PUBACK packet %x, got %x\", mqttPacketPubAck, pt)\n\t\t}\n\t\trpi, err := r.readUint16(\"packet identifier\")\n\t\tif err != nil || rpi != pi {\n\t\t\tt.Fatalf(\"Error with packet identifier expected=%v got: %v err=%v\", pi, rpi, err)\n\t\t}\n\n\tcase 2:\n\t\tb, _ := testMQTTReadPacket(t, r)\n\t\tif pt := b & mqttPacketMask; pt != mqttPacketPubRec {\n\t\t\tt.Fatalf(\"Expected PUBREC packet %x, got %x\", mqttPacketPubRec, pt)\n\t\t}\n\t\trpi, err := r.readUint16(\"packet identifier\")\n\t\tif err != nil || rpi != pi {\n\t\t\tt.Fatalf(\"Error with packet identifier expected=%v got: %v err=%v\", pi, rpi, err)\n\t\t}\n\n\t\ttestMQTTSendPIPacket(mqttPacketPubRel|0x2, t, c, pi)\n\n\t\tb, _ = testMQTTReadPacket(t, r)\n\t\tif pt := b & mqttPacketMask; pt != mqttPacketPubComp {\n\t\t\tt.Fatalf(\"Expected PUBCOMP packet %x, got %x\", mqttPacketPubComp, pt)\n\t\t}\n\t\trpi, err = r.readUint16(\"packet identifier\")\n\t\tif err != nil || rpi != pi {\n\t\t\tt.Fatalf(\"Error with packet identifier expected=%v got: %v err=%v\", pi, rpi, err)\n\t\t}\n\n\t\ttestMQTTFlush(t, c, nil, r)\n\t}\n}\n\nfunc TestMQTTParsePub(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tflags byte\n\t\tproto []byte\n\t\tpl int\n\t\terr string\n\t\twantErr bool\n\t}{\n\t\t{\"qos not supported\", (3 << 1), nil, 0, \"QoS=3 is invalid in MQTT\", false},\n\t\t{\"packet in buffer error\", 0, nil, 10, io.ErrUnexpectedEOF.Error(), false},\n\t\t{\"error on topic\", 0, []byte{0, 3, 'f', 'o'}, 4, \"topic\", false},\n\t\t{\"empty topic\", 0, []byte{0, 0}, 2, errMQTTTopicIsEmpty.Error(), false},\n\t\t{\"wildcards topic\", 0, []byte{0, 1, '#'}, 3, \"wildcards not allowed\", false},\n\t\t{\"invalid utf8 topic\", 0, []byte{0, 1, 241}, 3, \"\", true},\n\t\t{\"topic with null byte\", 0, []byte{0, 3, 'f', 0, 'o'}, 5, \"\", true},\n\t\t{\"error on packet identifier\", mqttPubQos1, []byte{0, 3, 'f', 'o', 'o'}, 5, \"packet identifier\", false},\n\t\t{\"invalid packet identifier\", mqttPubQos1, []byte{0, 3, 'f', 'o', 'o', 0, 0}, 7, errMQTTPacketIdentifierIsZero.Error(), false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tr := &mqttReader{}\n\t\t\tr.reset(test.proto)\n\t\t\tmqtt := &mqtt{r: r}\n\t\t\tc := &client{mqtt: mqtt}\n\t\t\tpp := &mqttPublish{flags: test.flags}\n\t\t\terr := c.mqttParsePub(r, test.pl, pp, false)\n\t\t\tif test.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected an error, got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTParsePIMsg(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto []byte\n\t\terr string\n\t}{\n\t\t{\"packet in buffer error\", nil, io.ErrUnexpectedEOF.Error()},\n\t\t{\"error reading packet identifier\", []byte{0}, \"packet identifier\"},\n\t\t{\"invalid packet identifier\", []byte{0, 0}, errMQTTPacketIdentifierIsZero.Error()},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tr := &mqttReader{}\n\t\t\tr.reset(test.proto)\n\t\t\tif _, err := mqttParsePIPacket(r); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTPublish(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, mcp, mpr, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTPublish(t, mcp, mpr, 1, false, false, \"foo\", 1, []byte(\"msg\"))\n\ttestMQTTPublish(t, mcp, mpr, 2, false, false, \"foo\", 2, []byte(\"msg\"))\n}\n\nfunc TestMQTTQoS2PubReject(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.rejectQoS2Pub = true\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, mcp, mpr, 1, false, false, \"foo\", 1, []byte(\"msg\"))\n\n\ttestMQTTSendPublishPacket(t, mcp, 2, false, false, \"foo\", 2, []byte(\"msg\"))\n\ttestMQTTExpectDisconnect(t, mcp)\n}\n\nfunc TestMQTTSub(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmqttSubTopic string\n\t\tnatsPubSubject string\n\t\tmqttPubTopic string\n\t\tok bool\n\t}{\n\t\t{\"1 level match\", \"foo\", \"foo\", \"foo\", true},\n\t\t{\"1 level no match\", \"foo\", \"bar\", \"bar\", false},\n\t\t{\"2 levels match\", \"foo/bar\", \"foo.bar\", \"foo/bar\", true},\n\t\t{\"2 levels no match\", \"foo/bar\", \"foo.baz\", \"foo/baz\", false},\n\t\t{\"3 levels match\", \"/foo/bar\", \"/.foo.bar\", \"/foo/bar\", true},\n\t\t{\"3 levels no match\", \"/foo/bar\", \"/.foo.baz\", \"/foo/baz\", false},\n\n\t\t{\"single level wc\", \"foo/+\", \"foo.bar.baz\", \"foo/bar/baz\", false},\n\t\t{\"single level wc\", \"foo/+\", \"foo.bar./\", \"foo/bar/\", false},\n\t\t{\"single level wc\", \"foo/+\", \"foo.bar\", \"foo/bar\", true},\n\t\t{\"single level wc\", \"foo/+\", \"foo./\", \"foo/\", true},\n\t\t{\"single level wc\", \"foo/+\", \"foo\", \"foo\", false},\n\t\t{\"single level wc\", \"foo/+\", \"/.foo\", \"/foo\", false},\n\n\t\t{\"multiple level wc\", \"foo/#\", \"foo.bar.baz./\", \"foo/bar/baz/\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"foo.bar.baz\", \"foo/bar/baz\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"foo.bar./\", \"foo/bar/\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"foo.bar\", \"foo/bar\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"foo./\", \"foo/\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"foo\", \"foo\", true},\n\t\t{\"multiple level wc\", \"foo/#\", \"/.foo\", \"/foo\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: test.mqttSubTopic, qos: 0}}, []byte{0})\n\t\t\ttestMQTTFlush(t, mc, nil, r)\n\n\t\t\tnatsPub(t, nc, test.natsPubSubject, []byte(\"msg\"))\n\t\t\tif test.ok {\n\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, test.mqttPubTopic, 0, []byte(\"msg\"))\n\t\t\t} else {\n\t\t\t\ttestMQTTExpectNothing(t, r)\n\t\t\t}\n\n\t\t\ttestMQTTPublish(t, mcp, mpr, 0, false, false, test.mqttPubTopic, 0, []byte(\"msg\"))\n\t\t\tif test.ok {\n\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, test.mqttPubTopic, 0, []byte(\"msg\"))\n\t\t\t} else {\n\t\t\t\ttestMQTTExpectNothing(t, r)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTSubQoS2(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttopic := \"foo/bar/baz\"\n\tmqttTopic0 := \"foo/#\"\n\tmqttTopic1 := \"foo/bar/#\"\n\tmqttTopic2 := topic\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic0, qos: 0}}, []byte{0})\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic1, qos: 1}}, []byte{1})\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic2, qos: 2}}, []byte{2})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\tfor pubQoS, expectedCounts := range map[byte]map[byte]int{\n\t\t0: {0: 3},\n\t\t1: {0: 1, 1: 2},\n\t\t2: {0: 1, 1: 1, 2: 1},\n\t} {\n\t\tt.Run(fmt.Sprintf(\"pubQoS %v\", pubQoS), func(t *testing.T) {\n\t\t\tpubPI := uint16(456)\n\n\t\t\ttestMQTTPublish(t, mcp, mpr, pubQoS, false, false, topic, pubPI, []byte(\"msg\"))\n\n\t\t\tqosCounts := map[byte]int{}\n\t\t\tdelivered := map[uint16]byte{}\n\n\t\t\t// We have 3 subscriptions, each should receive the message, with the\n\t\t\t// QoS that maybe \"trimmed\" to that of the subscription.\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\tflags, pi := testMQTTGetPubMsg(t, mc, r, topic, []byte(\"msg\"))\n\t\t\t\tdelivered[pi] = flags\n\t\t\t\tqosCounts[mqttGetQoS(flags)]++\n\t\t\t}\n\n\t\t\tfor pi, flags := range delivered {\n\t\t\t\tswitch mqttGetQoS(flags) {\n\t\t\t\tcase 1:\n\t\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi)\n\n\t\t\t\tcase 2:\n\t\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubRec, t, mc, pi)\n\t\t\t\t\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, pi)\n\t\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, mc, pi)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(qosCounts, expectedCounts) {\n\t\t\t\tt.Fatalf(\"Expected QoS %#v, got %#v\", expectedCounts, qosCounts)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTSubQoS2Restart(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tpubTopic := \"foo/bar\"\n\n\tpublish := func(msg string) {\n\t\tt.Helper()\n\t\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer mcp.Close()\n\t\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTPublish(t, mcp, mpr, 2, false, false, pubTopic, 1, []byte(msg))\n\t\ttestMQTTFlush(t, mcp, nil, mpr)\n\t}\n\n\tcreateSub := func(present bool) (net.Conn, *mqttReader) {\n\t\tt.Helper()\n\t\tc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, present)\n\t\tif !present {\n\t\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo/bar\", qos: 2}}, []byte{2})\n\t\t\ttestMQTTSub(t, 2, c, r, []*mqttFilter{{filter: \"foo/+\", qos: 2}}, []byte{2})\n\t\t\ttestMQTTFlush(t, c, nil, r)\n\t\t}\n\t\treturn c, r\n\t}\n\n\tc, r := createSub(false)\n\tdefer c.Close()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\tcheckSub := func(count int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tfor c := range js.Consumers(mqttOutStreamName) {\n\t\t\t\tacc := s.GlobalAccount()\n\t\t\t\tres := acc.sl.Match(c.Config.DeliverSubject)\n\t\t\t\tif n := len(res.psubs); n != count {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %v subscription on %q, got %v\",\n\t\t\t\t\t\tcount, c.Config.DeliverSubject, n)\n\t\t\t\t} else {\n\t\t\t\t\t// We got the JS consumer and verified that it has the\n\t\t\t\t\t// expected number of subscription, so we are done.\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\t// No JS consumer was found.\n\t\t\treturn fmt.Errorf(\"Did not find a JS consumer for %q\", mqttOutStreamName)\n\t\t})\n\t}\n\tcheckSub(1)\n\n\tpublish(\"msg1\")\n\n\tconsume := func(msg string) {\n\t\tt.Helper()\n\t\tvar pis []uint16\n\t\tfor range 2 {\n\t\t\tb, pl := testMQTTReadPacket(t, r)\n\t\t\tif pt := b & mqttPacketMask; pt != mqttPacketPub {\n\t\t\t\tt.Fatalf(\"Expected PUBLISH packet %x, got %x\", mqttPacketPub, pt)\n\t\t\t}\n\t\t\tflags, pi, _, _ := testMQTTGetPubMsgExEx(t, c, r, b, pl, pubTopic, []byte(msg))\n\t\t\trequire_Equal(t, flags, byte(mqttPubQoS2))\n\t\t\tpis = append(pis, pi)\n\t\t}\n\t\tfor _, pi := range pis {\n\t\t\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, pi)\n\t\t\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, pi)\n\t\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, pi)\n\t\t}\n\t\ttestMQTTExpectNothing(t, r)\n\t}\n\n\tconsume(\"msg1\")\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\tcheckSub(0)\n\n\tpublish(\"msg2\")\n\n\tc, r = createSub(true)\n\tdefer c.Close()\n\tcheckSub(1)\n\tconsume(\"msg2\")\n\n\tpublish(\"msg3\")\n\tconsume(\"msg3\")\n}\n\nfunc TestMQTTSubQoS1(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tmqttTopic := \"foo/bar\"\n\n\t// Subscribe with QoS 1\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: mqttTopic, qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish from NATS, which means QoS 0\n\tnatsPub(t, nc, \"foo.bar\", []byte(\"NATS\"))\n\t// Will receive as QoS 0\n\ttestMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte(\"NATS\"))\n\ttestMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte(\"NATS\"))\n\n\t// Publish from MQTT with QoS 0\n\ttestMQTTPublish(t, mcp, mpr, 0, false, false, mqttTopic, 0, []byte(\"msg\"))\n\t// Will receive as QoS 0\n\ttestMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, mqttTopic, 0, []byte(\"msg\"))\n\n\t// Publish from MQTT with QoS 1\n\ttestMQTTPublish(t, mcp, mpr, 1, false, false, mqttTopic, 1, []byte(\"msg\"))\n\tpflags1, pi1 := testMQTTGetPubMsg(t, mc, r, mqttTopic, []byte(\"msg\"))\n\tif pflags1 != 0x2 {\n\t\tt.Fatalf(\"Expected flags to be 0x2, got %v\", pflags1)\n\t}\n\tpflags2, pi2 := testMQTTGetPubMsg(t, mc, r, mqttTopic, []byte(\"msg\"))\n\tif pflags2 != 0x2 {\n\t\tt.Fatalf(\"Expected flags to be 0x2, got %v\", pflags2)\n\t}\n\tif pi1 == pi2 {\n\t\tt.Fatalf(\"packet identifier for message 1: %v should be different from message 2\", pi1)\n\t}\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi1)\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, mc, pi2)\n}\n\nfunc getSubQoS(sub *subscription) int {\n\tif sub.mqtt != nil {\n\t\treturn int(sub.mqtt.qos)\n\t}\n\treturn -1\n}\n\nfunc TestMQTTSubDups(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", user: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Test with single SUBSCRIBE protocol but multiple filters\n\tfilters := []*mqttFilter{\n\t\t{filter: \"foo\", qos: 1},\n\t\t{filter: \"foo\", qos: 0},\n\t}\n\ttestMQTTSub(t, 1, mc, r, filters, []byte{1, 0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// And also with separate SUBSCRIBE protocols\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"bar\", qos: 0}}, []byte{0})\n\t// Ask for QoS 1 but server will downgrade to 1\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish and test msg received only once\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"bar\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"bar\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Check that the QoS for subscriptions have been updated to the latest received filter\n\tvar err error\n\tsubc := testMQTTGetClient(t, s, \"sub\")\n\tsubc.mu.Lock()\n\tif subc.opts.Username != \"sub\" {\n\t\terr = fmt.Errorf(\"wrong user name\")\n\t}\n\tif err == nil {\n\t\tif sub := subc.subs[\"foo\"]; sub == nil || getSubQoS(sub) != 0 {\n\t\t\terr = fmt.Errorf(\"subscription foo QoS should be 0, got %v\", getSubQoS(sub))\n\t\t}\n\t}\n\tif err == nil {\n\t\tif sub := subc.subs[\"bar\"]; sub == nil || getSubQoS(sub) != 1 {\n\t\t\terr = fmt.Errorf(\"subscription bar QoS should be 1, got %v\", getSubQoS(sub))\n\t\t}\n\t}\n\tsubc.mu.Unlock()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Now subscribe on \"foo/#\" which means that a PUBLISH on \"foo\" will be received\n\t// by this subscription and also the one on \"foo\".\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish and test msg received twice\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\n\tcheckWCSub := func(expectedQoS int) {\n\t\tt.Helper()\n\n\t\tsubc.mu.Lock()\n\t\tdefer subc.mu.Unlock()\n\n\t\t// When invoked with expectedQoS==1, we have the following subs:\n\t\t// foo (QoS-0), bar (QoS-1), foo.> (QoS-1)\n\t\t// which means (since QoS-1 have a JS consumer + sub for delivery\n\t\t// and foo.> causes a \"foo fwc\") that we should have the following\n\t\t// number of NATS subs: foo (1), bar (2), foo.> (2) and \"foo fwc\" (2),\n\t\t// so total=7.\n\t\t// When invoked with expectedQoS==0, it means that we have replaced\n\t\t// foo/# QoS-1 to QoS-0, so we should have 2 less NATS subs,\n\t\t// so total=5\n\t\texpected := 7\n\t\tif expectedQoS == 0 {\n\t\t\texpected = 5\n\t\t}\n\t\tif lenmap := len(subc.subs); lenmap != expected {\n\t\t\tt.Fatalf(\"Subs map should have %v entries, got %v\", expected, lenmap)\n\t\t}\n\t\tif sub, ok := subc.subs[\"foo.>\"]; !ok {\n\t\t\tt.Fatal(\"Expected sub foo.> to be present but was not\")\n\t\t} else if getSubQoS(sub) != expectedQoS {\n\t\t\tt.Fatalf(\"Expected sub foo.> QoS to be %v, got %v\", expectedQoS, getSubQoS(sub))\n\t\t}\n\t\tif sub, ok := subc.subs[\"foo fwc\"]; !ok {\n\t\t\tt.Fatal(\"Expected sub foo fwc to be present but was not\")\n\t\t} else if getSubQoS(sub) != expectedQoS {\n\t\t\tt.Fatalf(\"Expected sub foo fwc QoS to be %v, got %v\", expectedQoS, getSubQoS(sub))\n\t\t}\n\t\t// Make sure existing sub on \"foo\" qos was not changed.\n\t\tif sub, ok := subc.subs[\"foo\"]; !ok {\n\t\t\tt.Fatal(\"Expected sub foo to be present but was not\")\n\t\t} else if getSubQoS(sub) != 0 {\n\t\t\tt.Fatalf(\"Expected sub foo QoS to be 0, got %v\", getSubQoS(sub))\n\t\t}\n\t}\n\tcheckWCSub(1)\n\n\t// Sub again on same subject with lower QoS\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish and test msg received twice\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\tcheckWCSub(0)\n}\n\nfunc TestMQTTSubWithSpaces(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{user: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo bar\", qos: 0}}, []byte{mqttSubAckFailure})\n}\n\nfunc TestMQTTSubCaseSensitive(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{user: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"Foo/Bar\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"Foo/Bar\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"Foo/Bar\", 0, []byte(\"msg\"))\n\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tnatsPub(t, nc, \"Foo.Bar\", []byte(\"nats\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"Foo/Bar\", 0, []byte(\"nats\"))\n\n\tnatsPub(t, nc, \"foo.bar\", []byte(\"nats\"))\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTPubSubMatrix(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnatsPub bool\n\t\tmqttPub bool\n\t\tmqttPubQoS byte\n\t\tnatsSub bool\n\t\tmqttSubQoS0 bool\n\t\tmqttSubQoS1 bool\n\t}{\n\t\t{\"NATS to MQTT sub QoS-0\", true, false, 0, false, true, false},\n\t\t{\"NATS to MQTT sub QoS-1\", true, false, 0, false, false, true},\n\t\t{\"NATS to MQTT sub QoS-0 and QoS-1\", true, false, 0, false, true, true},\n\n\t\t{\"MQTT QoS-0 to NATS sub\", false, true, 0, true, false, false},\n\t\t{\"MQTT QoS-0 to MQTT sub QoS-0\", false, true, 0, false, true, false},\n\t\t{\"MQTT QoS-0 to MQTT sub QoS-1\", false, true, 0, false, false, true},\n\t\t{\"MQTT QoS-0 to NATS sub and MQTT sub QoS-0\", false, true, 0, true, true, false},\n\t\t{\"MQTT QoS-0 to NATS sub and MQTT sub QoS-1\", false, true, 0, true, false, true},\n\t\t{\"MQTT QoS-0 to all subs\", false, true, 0, true, true, true},\n\n\t\t{\"MQTT QoS-1 to NATS sub\", false, true, 1, true, false, false},\n\t\t{\"MQTT QoS-1 to MQTT sub QoS-0\", false, true, 1, false, true, false},\n\t\t{\"MQTT QoS-1 to MQTT sub QoS-1\", false, true, 1, false, false, true},\n\t\t{\"MQTT QoS-1 to NATS sub and MQTT sub QoS-0\", false, true, 1, true, true, false},\n\t\t{\"MQTT QoS-1 to NATS sub and MQTT sub QoS-1\", false, true, 1, true, false, true},\n\t\t{\"MQTT QoS-1 to all subs\", false, true, 1, true, true, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tnc := natsConnect(t, s.ClientURL())\n\t\t\tdefer nc.Close()\n\n\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tmc1, r1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc1.Close()\n\t\t\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tmc2, r2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc2.Close()\n\t\t\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\t// First setup subscriptions based on test options.\n\t\t\tvar ns *nats.Subscription\n\t\t\tif test.natsSub {\n\t\t\t\tns = natsSubSync(t, nc, \"foo\")\n\t\t\t}\n\t\t\tif test.mqttSubQoS0 {\n\t\t\t\ttestMQTTSub(t, 1, mc1, r1, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\t\t\t\ttestMQTTFlush(t, mc1, nil, r1)\n\t\t\t}\n\t\t\tif test.mqttSubQoS1 {\n\t\t\t\ttestMQTTSub(t, 1, mc2, r2, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\t\t\t\ttestMQTTFlush(t, mc2, nil, r2)\n\t\t\t}\n\n\t\t\t// Just as a barrier\n\t\t\tnatsFlush(t, nc)\n\n\t\t\t// Now publish\n\t\t\tif test.natsPub {\n\t\t\t\tnatsPub(t, nc, \"foo\", []byte(\"msg\"))\n\t\t\t} else {\n\t\t\t\ttestMQTTPublish(t, mc, r, test.mqttPubQoS, false, false, \"foo\", 1, []byte(\"msg\"))\n\t\t\t}\n\n\t\t\t// Check message received\n\t\t\tif test.natsSub {\n\t\t\t\tnatsNexMsg(t, ns, time.Second)\n\t\t\t\t// Make sure no other is received\n\t\t\t\tif msg, err := ns.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\tt.Fatalf(\"Should not have gotten a second message, got %v\", msg)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.mqttSubQoS0 {\n\t\t\t\ttestMQTTCheckPubMsg(t, mc1, r1, \"foo\", 0, []byte(\"msg\"))\n\t\t\t\ttestMQTTExpectNothing(t, r1)\n\t\t\t}\n\t\t\tif test.mqttSubQoS1 {\n\t\t\t\tvar expectedFlag byte\n\t\t\t\tif test.mqttPubQoS > 0 {\n\t\t\t\t\texpectedFlag = test.mqttPubQoS << 1\n\t\t\t\t}\n\t\t\t\ttestMQTTCheckPubMsg(t, mc2, r2, \"foo\", expectedFlag, []byte(\"msg\"))\n\t\t\t\ttestMQTTExpectNothing(t, r2)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTPreventSubWithMQTTSubPrefix(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, r,\n\t\t[]*mqttFilter{{filter: strings.ReplaceAll(mqttSubPrefix, \".\", \"/\") + \"foo/bar\", qos: 1}},\n\t\t[]byte{mqttSubAckFailure})\n}\n\nfunc TestMQTTSubWithNATSStream(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\tmcp, rp := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTFlush(t, mcp, nil, rp)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tsc := &StreamConfig{\n\t\tName: \"test\",\n\t\tStorage: FileStorage,\n\t\tRetention: InterestPolicy,\n\t\tSubjects: []string{\"foo.>\"},\n\t}\n\tmset, err := s.GlobalAccount().addStream(sc)\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to create stream: %v\", err)\n\t}\n\n\tsub := natsSubSync(t, nc, \"bar\")\n\tcc := &ConsumerConfig{\n\t\tDurable: \"dur\",\n\t\tAckPolicy: AckExplicit,\n\t\tDeliverSubject: \"bar\",\n\t}\n\tif _, err := mset.addConsumer(cc); err != nil {\n\t\tt.Fatalf(\"Unable to add consumer: %v\", err)\n\t}\n\n\t// Now send message from NATS\n\tresp, err := nc.Request(\"foo.bar\", []byte(\"nats\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tar := &ApiResponse{}\n\tif err := json.Unmarshal(resp.Data, ar); err != nil || ar.Error != nil {\n\t\tt.Fatalf(\"Unexpected response: err=%v resp=%+v\", err, ar.Error)\n\t}\n\n\t// Check that message is received by both\n\tcheckRecv := func(content string, flags byte) {\n\t\tt.Helper()\n\t\tif msg := natsNexMsg(t, sub, time.Second); string(msg.Data) != content {\n\t\t\tt.Fatalf(\"Expected %q, got %q\", content, msg.Data)\n\t\t}\n\t\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", flags, []byte(content))\n\t}\n\tcheckRecv(\"nats\", 0)\n\n\t// Send from MQTT as a QoS0\n\ttestMQTTPublish(t, mcp, rp, 0, false, false, \"foo/bar\", 0, []byte(\"qos0\"))\n\tcheckRecv(\"qos0\", 0)\n\n\t// Send from MQTT as a QoS1\n\ttestMQTTPublish(t, mcp, rp, 1, false, false, \"foo/bar\", 1, []byte(\"qos1\"))\n\tcheckRecv(\"qos1\", mqttPubQos1)\n}\n\nfunc TestMQTTTrackPendingOverrun(t *testing.T) {\n\tsess := mqttSession{}\n\n\tsess.last_pi = 0xFFFF\n\tpi := sess.trackPublishRetained()\n\tif pi != 1 {\n\t\tt.Fatalf(\"Expected 1, got %v\", pi)\n\t}\n\n\tp := &mqttPending{}\n\tfor i := 1; i <= 0xFFFF; i++ {\n\t\tsess.pendingPublish[uint16(i)] = p\n\t}\n\tpi, _ = sess.trackPublish(\"test\", \"test\")\n\tif pi != 0 {\n\t\tt.Fatalf(\"Expected 0, got %v\", pi)\n\t}\n\n\tdelete(sess.pendingPublish, 1234)\n\tpi = sess.trackPublishRetained()\n\tif pi != 1234 {\n\t\tt.Fatalf(\"Expected 1234, got %v\", pi)\n\t}\n}\n\nfunc TestMQTTSubRestart(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Start an MQTT subscription QoS=1 on \"foo\"\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Now start a NATS subscription on \">\" (anything that would match the JS consumer delivery subject)\n\tnatsSubSync(t, nc, \">\")\n\tnatsFlush(t, nc)\n\n\t// Restart the MQTT client\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\tmc, r = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\t// Restart an MQTT subscription QoS=1 on \"foo\"\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\tpc, pr := testMQTTConnect(t, &mqttConnInfo{clientID: \"pub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer pc.Close()\n\ttestMQTTCheckConnAck(t, pr, mqttConnAckRCConnectionAccepted, false)\n\n\t// Publish a message QoS1\n\ttestMQTTPublish(t, pc, pr, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\t// Make sure we receive it\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n\n\t// Now \"restart\" the subscription but as a Qos0\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish a message QoS\n\ttestMQTTPublish(t, pc, pr, 1, false, false, \"foo\", 1, []byte(\"msg2\"))\n\t// Make sure we receive but as a QoS0\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg2\"))\n}\n\nfunc testMQTTGetClusterTemplaceNoLeaf() string {\n\treturn strings.Replace(jsClusterTemplWithLeafAndMQTT, \"{{leaf}}\", \"\", 1)\n}\n\nfunc TestMQTTSubPropagation(t *testing.T) {\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", 2)\n\tdefer cl.shutdown()\n\n\to := cl.opts[0]\n\ts2 := cl.servers[1]\n\tnc := natsConnect(t, s2.ClientURL())\n\tdefer nc.Close()\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Because in MQTT foo/# means foo.> but also foo, check that this is propagated\n\tcheckSubInterest(t, s2, globalAccountName, \"foo\", time.Second)\n\n\t// Publish on foo.bar, foo./ and foo and we should receive them\n\tnatsPub(t, nc, \"foo.bar\", []byte(\"hello\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", 0, []byte(\"hello\"))\n\n\tnatsPub(t, nc, \"foo./\", []byte(\"from\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo/\", 0, []byte(\"from\"))\n\n\tnatsPub(t, nc, \"foo\", []byte(\"NATS\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"NATS\"))\n}\n\nfunc TestMQTTCluster(t *testing.T) {\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", 2)\n\tdefer cl.shutdown()\n\n\tfor _, topTest := range []struct {\n\t\tname string\n\t\trestart bool\n\t}{\n\t\t{\"first_start\", true},\n\t\t{\"restart\", false},\n\t} {\n\t\tt.Run(topTest.name, func(t *testing.T) {\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tsubQos byte\n\t\t\t}{\n\t\t\t\t{\"qos_0\", 0},\n\t\t\t\t{\"qos_1\", 1},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tclientID := nuid.Next()\n\n\t\t\t\t\to := cl.opts[0]\n\t\t\t\t\tmc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: clientID, cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5)\n\t\t\t\t\tdefer mc.Close()\n\t\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\t\t\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\", qos: test.subQos}}, []byte{test.subQos})\n\t\t\t\t\ttestMQTTFlush(t, mc, nil, r)\n\n\t\t\t\t\tcheck := func(mc net.Conn, r *mqttReader, o *Options, s *Server) {\n\t\t\t\t\t\tt.Helper()\n\n\t\t\t\t\t\tnc := natsConnect(t, s.ClientURL())\n\t\t\t\t\t\tdefer nc.Close()\n\n\t\t\t\t\t\tnatsPub(t, nc, \"foo.bar\", []byte(\"fromNats\"))\n\t\t\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", 0, []byte(\"fromNats\"))\n\n\t\t\t\t\t\tmpc, pr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\t\t\tdefer mpc.Close()\n\t\t\t\t\t\ttestMQTTCheckConnAck(t, pr, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\t\t\t\ttestMQTTPublish(t, mpc, pr, 0, false, false, \"foo/baz\", 0, []byte(\"mqtt_qos0\"))\n\t\t\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, \"foo/baz\", 0, []byte(\"mqtt_qos0\"))\n\n\t\t\t\t\t\ttestMQTTPublish(t, mpc, pr, 1, false, false, \"foo/bat\", 1, []byte(\"mqtt_qos1\"))\n\t\t\t\t\t\texpectedQoS := byte(0)\n\t\t\t\t\t\tif test.subQos == 1 {\n\t\t\t\t\t\t\texpectedQoS = mqttPubQos1\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bat\", expectedQoS, []byte(\"mqtt_qos1\"))\n\t\t\t\t\t\ttestMQTTDisconnect(t, mpc, nil)\n\t\t\t\t\t}\n\t\t\t\t\tcheck(mc, r, cl.opts[0], cl.servers[0])\n\t\t\t\t\tcheck(mc, r, cl.opts[1], cl.servers[1])\n\n\t\t\t\t\t// Start the same subscription from the other server. It should disconnect\n\t\t\t\t\t// the one connected in the first server.\n\t\t\t\t\to = cl.opts[1]\n\n\t\t\t\t\tmc2, r2 := testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\t\tdefer mc2.Close()\n\t\t\t\t\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true)\n\n\t\t\t\t\t// Expect first connection to be closed.\n\t\t\t\t\ttestMQTTExpectDisconnect(t, mc)\n\n\t\t\t\t\t// Now re-run the checks\n\t\t\t\t\tcheck(mc2, r2, cl.opts[0], cl.servers[0])\n\t\t\t\t\tcheck(mc2, r2, cl.opts[1], cl.servers[1])\n\n\t\t\t\t\t// Disconnect our sub and restart with clean session then disconnect again to clear the state.\n\t\t\t\t\ttestMQTTDisconnect(t, mc2, nil)\n\t\t\t\t\tmc2.Close()\n\t\t\t\t\tmc2, r2 = testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\t\t\tdefer mc2.Close()\n\t\t\t\t\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\t\ttestMQTTFlush(t, mc2, nil, r2)\n\t\t\t\t\ttestMQTTDisconnect(t, mc2, nil)\n\n\t\t\t\t\t// Remove the session from the flappers so we can restart the test\n\t\t\t\t\t// without failure and have to wait for 1sec before being able to reconnect.\n\t\t\t\t\ts := cl.servers[0]\n\t\t\t\t\tsm := &s.mqtt.sessmgr\n\t\t\t\t\tsm.mu.Lock()\n\t\t\t\t\tasm := sm.sessions[globalAccountName]\n\t\t\t\t\tsm.mu.Unlock()\n\t\t\t\t\tif asm != nil {\n\t\t\t\t\t\tasm.mu.Lock()\n\t\t\t\t\t\tdelete(asm.flappers, clientID)\n\t\t\t\t\t\tasm.mu.Unlock()\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t\tif !t.Failed() && topTest.restart {\n\t\t\t\tcl.stopAll()\n\t\t\t\tcl.restartAll()\n\n\t\t\t\tstreams := []string{mqttStreamName, mqttRetainedMsgsStreamName, mqttSessStreamName}\n\t\t\t\tfor _, sn := range streams {\n\t\t\t\t\tcl.waitOnStreamLeader(globalAccountName, sn)\n\t\t\t\t}\n\n\t\t\t\tmset, err := cl.randomServer().GlobalAccount().lookupStream(mqttRetainedMsgsStreamName)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected to find a stream for %q\", mqttRetainedMsgsStreamName)\n\t\t\t\t}\n\n\t\t\t\trmConsumerNames := []string{}\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tfor _, o := range mset.consumers {\n\t\t\t\t\trmConsumerNames = append(rmConsumerNames, o.name)\n\t\t\t\t}\n\t\t\t\tmset.mu.RUnlock()\n\n\t\t\t\tfor _, consumerName := range rmConsumerNames {\n\t\t\t\t\tcl.waitOnConsumerLeader(globalAccountName, mqttRetainedMsgsStreamName, consumerName)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testMQTTConnectDisconnect(t *testing.T, o *Options, clientID string, clean bool, found bool) {\n\tt.Helper()\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: clientID, cleanSess: clean}, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, found)\n\ttestMQTTDisconnectEx(t, mc, nil, false)\n\tmc.Close()\n}\n\nfunc TestMQTTClusterConnectDisconnectClean(t *testing.T) {\n\t// The purpose of this test was to illustrate and verify\n\t// https://github.com/nats-io/nats-server/pull/4734. Due to its timing\n\t// sensitivity it has never been 100% reliable, prone to flaking on an\n\t// occasional MQTT Connect failure (API timeout?). Skip for now, to reduce\n\t// the flaky noise.\n\tt.Skip()\n\n\tnServers := 3\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", nServers)\n\tdefer cl.shutdown()\n\n\tclientID := nuid.Next()\n\n\t// test runs a connect/disconnect against a random server in the cluster, as\n\t// specified.\n\tN := 100\n\tfor n := 0; n < N; n++ {\n\t\ttestMQTTConnectDisconnect(t, cl.opts[rand.Intn(nServers)], clientID, true, false)\n\t}\n}\n\nfunc TestMQTTClusterConnectDisconnectPersist(t *testing.T) {\n\t// The purpose of this test was to illustrate and verify\n\t// https://github.com/nats-io/nats-server/pull/4734. Due to its timing\n\t// sensitivity it has never been 100% reliable, prone to flaking on an\n\t// occasional MQTT Connect failure (API timeout?). Skip for now, to reduce\n\t// the flaky noise.\n\tt.Skip()\n\n\tnServers := 3\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", nServers)\n\tdefer cl.shutdown()\n\n\tclientID := nuid.Next()\n\n\t// test runs a connect/disconnect against a random server in the cluster, as\n\t// specified.\n\tN := 20\n\tfor n := 0; n < N; n++ {\n\t\t// First clean sessions on all servers\n\t\tfor i := 0; i < nServers; i++ {\n\t\t\ttestMQTTConnectDisconnect(t, cl.opts[i], clientID, true, false)\n\t\t}\n\n\t\ttestMQTTConnectDisconnect(t, cl.opts[0], clientID, false, false)\n\t\ttestMQTTConnectDisconnect(t, cl.opts[1], clientID, false, true)\n\t\ttestMQTTConnectDisconnect(t, cl.opts[2], clientID, false, true)\n\t\ttestMQTTConnectDisconnect(t, cl.opts[0], clientID, false, true)\n\t}\n}\n\nfunc TestMQTTClusterRetainedMsg(t *testing.T) {\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", 2)\n\tdefer cl.shutdown()\n\n\tsrv1Opts := cl.opts[0]\n\tsrv2Opts := cl.opts[1]\n\n\t// Connect subscription on server 1.\n\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, rc)\n\n\t// Create a publisher from server 2.\n\tmp, rp := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port)\n\tdefer mp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// Send retained message.\n\ttestMQTTPublish(t, mp, rp, 1, false, true, \"foo/bar\", 1, []byte(\"retained\"))\n\t// Check it is received.\n\ttestMQTTCheckPubMsg(t, mc, rc, \"foo/bar\", mqttPubQos1, []byte(\"retained\"))\n\n\t// Start a new subscription on server 1 and make sure we receive the retained message\n\tmc2, rc2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\ttestMQTTCheckPubMsg(t, mc2, rc2, \"foo/bar\", mqttPubQos1|mqttPubFlagRetain, []byte(\"retained\"))\n\ttestMQTTDisconnect(t, mc2, nil)\n\tmc2.Close()\n\n\t// Send an empty retained message which should remove it from storage, but still be delivered.\n\ttestMQTTPublish(t, mp, rp, 1, false, true, \"foo/bar\", 1, []byte(\"\"))\n\ttestMQTTCheckPubMsg(t, mc, rc, \"foo/bar\", mqttPubQos1, []byte(\"\"))\n\n\t// Now shutdown the consumer connection\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\t// Reconnect to server where the retained message was published (server 2)\n\tmc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\t// The retained message should not be delivered.\n\ttestMQTTExpectNothing(t, rc)\n\t// Now disconnect and reconnect back to first server\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\t// Now reconnect to the server 1, which is not where the messages were published, and check\n\t// that we don't receive the message.\n\tmc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\ttestMQTTExpectNothing(t, rc)\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\t// Will now test network deletes\n\n\t// Create a subscription on server 1 and server 2\n\tmc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub_one\", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc, nil, rc)\n\n\tmc2, rc2 = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub_two\", cleanSess: false}, srv2Opts.MQTT.Host, srv2Opts.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc2, nil, rc2)\n\n\t// Publish 1 retained message (producer is connected to server 2)\n\ttestMQTTPublish(t, mp, rp, 1, false, true, \"bar\", 1, []byte(\"msg1\"))\n\n\t// Make sure messages are received by both\n\ttestMQTTCheckPubMsg(t, mc, rc, \"bar\", mqttPubQos1, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, mc2, rc2, \"bar\", mqttPubQos1, []byte(\"msg1\"))\n\n\t// Now send an empty retained message that should delete it. For the one on server 1,\n\t// this will be a network delete.\n\ttestMQTTPublish(t, mp, rp, 1, false, true, \"bar\", 1, []byte(\"\"))\n\ttestMQTTCheckPubMsg(t, mc, rc, \"bar\", mqttPubQos1, []byte(\"\"))\n\ttestMQTTCheckPubMsg(t, mc2, rc2, \"bar\", mqttPubQos1, []byte(\"\"))\n\n\t// Now send a new retained message\n\ttestMQTTPublish(t, mp, rp, 1, false, true, \"bar\", 1, []byte(\"msg2\"))\n\n\t// Again, verify that they all receive it.\n\ttestMQTTCheckPubMsg(t, mc, rc, \"bar\", mqttPubQos1, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsg(t, mc2, rc2, \"bar\", mqttPubQos1, []byte(\"msg2\"))\n\n\t// But now, restart the consumer that was in the server that processed the\n\t// original network delete.\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\tmc, rc = testMQTTConnect(t, &mqttConnInfo{clientID: \"sub_one\", cleanSess: false}, srv1Opts.MQTT.Host, srv1Opts.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\t// Expect the message to be delivered as retained\n\ttestMQTTCheckPubMsg(t, mc, rc, \"bar\", mqttPubQos1|mqttPubFlagRetain, []byte(\"msg2\"))\n}\n\nfunc TestMQTTRetainedMsgMigration(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create the retained messages stream to listen on the old subject first.\n\t// The server will correct this when the migration takes place.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: mqttRetainedMsgsStreamName,\n\t\tSubjects: []string{`$MQTT.rmsgs`},\n\t\tStorage: nats.FileStorage,\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Publish some retained messages on the old \"$MQTT.rmsgs\" subject.\n\tconst N = 100\n\tfor i := 0; i < N; i++ {\n\t\tmsg := fmt.Sprintf(\n\t\t\t`{\"origin\":\"b5IQZNtG\",\"subject\":\"test%d\",\"topic\":\"test%d\",\"msg\":\"YmFy\",\"flags\":1}`, i, i,\n\t\t)\n\t\t_, err := js.Publish(`$MQTT.rmsgs`, []byte(msg))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Check that the old subject looks right.\n\tsi, err := js.StreamInfo(mqttRetainedMsgsStreamName, &nats.StreamInfoRequest{\n\t\tSubjectsFilter: `$MQTT.>`,\n\t})\n\trequire_NoError(t, err)\n\tif si.State.NumSubjects != 1 {\n\t\tt.Fatalf(\"expected 1 subject, got %d\", si.State.NumSubjects)\n\t}\n\tif n := si.State.Subjects[`$MQTT.rmsgs`]; n != N {\n\t\tt.Fatalf(\"expected to find %d messages on the original subject but found %d\", N, n)\n\t}\n\n\t// Create an MQTT client, this will cause a migration to take place.\n\tmc, rc := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\n\tas := testMQTTGetAccountSessionManager(t, s, \"sub\")\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tas.mu.RLock()\n\t\tdefer as.mu.RUnlock()\n\t\tif n := len(as.retmsgs); n != N {\n\t\t\treturn fmt.Errorf(\"Got only %v retained messages\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"+\", qos: 0}}, []byte{0})\n\ttopics := map[string]struct{}{}\n\tfor i := 0; i < N; i++ {\n\t\t_, _, topic, _ := testMQTTGetPubMsgEx(t, mc, rc, _EMPTY_, []byte(\"bar\"))\n\t\ttopics[topic] = struct{}{}\n\t}\n\tif len(topics) != N {\n\t\tt.Fatalf(\"Unexpected topics: %v\", topics)\n\t}\n\n\t// Now look at the stream, there should be N messages on the new\n\t// divided subjects and none on the old undivided subject.\n\tsi, err = js.StreamInfo(mqttRetainedMsgsStreamName, &nats.StreamInfoRequest{\n\t\tSubjectsFilter: `$MQTT.>`,\n\t})\n\trequire_NoError(t, err)\n\tif si.State.NumSubjects != N {\n\t\tt.Fatalf(\"expected %d subjects, got %d\", N, si.State.NumSubjects)\n\t}\n\tif n := si.State.Subjects[`$MQTT.rmsgs`]; n > 0 {\n\t\tt.Fatalf(\"expected to find no messages on the original subject but found %d\", n)\n\t}\n\n\t// Check that the message counts look right. There should be one\n\t// retained message per key.\n\tfor i := 0; i < N; i++ {\n\t\texpected := fmt.Sprintf(`$MQTT.rmsgs.test%d`, i)\n\t\tn, ok := si.State.Subjects[expected]\n\t\tif !ok {\n\t\t\tt.Fatalf(\"expected to find %q but didn't\", expected)\n\t\t}\n\t\tif n != 1 {\n\t\t\tt.Fatalf(\"expected %q to have 1 message but had %d\", expected, n)\n\t\t}\n\t}\n}\n\nfunc TestMQTTRetainedNoMsgBodyCorruption(t *testing.T) {\n\tf := func() {\n\t\to := testMQTTDefaultOptions()\n\t\ts := testMQTTRunServer(t, o)\n\t\tdefer testMQTTShutdownServer(s)\n\n\t\t// Send a retained message.\n\t\tc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"pub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTPublish(t, c, r, 0, false, true, \"foo/bar\", 0, []byte(\"retained 1\"))\n\t\ttestMQTTFlush(t, c, nil, r)\n\n\t\tcheckRetained := func(msg string) {\n\t\t\tt.Helper()\n\t\t\tc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer c.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo/#\", qos: 0}}, []byte{0})\n\t\t\ttestMQTTCheckPubMsg(t, c, r, \"foo/bar\", mqttPubFlagRetain, []byte(msg))\n\t\t}\n\t\t// Subscribe to make it load into the cache.\n\t\tcheckRetained(\"retained 1\")\n\n\t\t// Now send another one.\n\t\ttestMQTTPublish(t, c, r, 0, false, true, \"foo/bar\", 0, []byte(\"retained 2\"))\n\t\ttestMQTTFlush(t, c, nil, r)\n\n\t\t// Check it is updated\n\t\tcheckRetained(\"retained 2\")\n\n\t\t// Now we will simulate an update coming from another server\n\t\t// if we were in cluster mode.\n\t\tnc := natsConnect(t, s.ClientURL())\n\t\tdefer nc.Close()\n\n\t\tmsg := nats.NewMsg(\"$MQTT.rmsgs.foo.bar\")\n\t\tmsg.Header.Set(mqttNatsRetainedMessageOrigin, \"XXXXXXXX\")\n\t\tmsg.Header.Set(mqttNatsRetainedMessageTopic, \"foo/bar\")\n\t\tmsg.Header.Set(mqttNatsRetainedMessageFlags, \"1\")\n\t\tmsg.Data = []byte(\"retained 3\")\n\t\tnc.PublishMsg(msg)\n\t\tnatsFlush(t, nc)\n\n\t\t// Have a continuous flow of updates coming in\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tch := make(chan struct{})\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor {\n\t\t\t\tnc.PublishMsg(msg)\n\t\t\t\tselect {\n\t\t\t\tcase <-ch:\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\n\t\t// Retrieve the account session manager using the \"pub\" client we have.\n\t\tas := testMQTTGetAccountSessionManager(t, s, \"pub\")\n\t\tas.mu.RLock()\n\t\tcache := as.rmsCache\n\t\tas.mu.RUnlock()\n\n\t\t// Wait to make sure at least the first update occurs\n\t\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\t\tv, ok := cache.Load(\"foo.bar\")\n\t\t\tif !ok {\n\t\t\t\treturn errors.New(\"not in the cache\")\n\t\t\t}\n\t\t\trm := v.(*mqttRetainedMsg)\n\t\t\tif !bytes.Equal(rm.Msg, []byte(\"retained 3\")) {\n\t\t\t\treturn fmt.Errorf(\"Retained message not updated, got %q\", rm.Msg)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\t// Repeat starting a subscription to check the retained message and\n\t\t// make sure it is not corrupted. With the bug, the payload will at\n\t\t// the very least contain trailing \"\\r\\n\" and possibly be corrupted\n\t\t// (and the race detector would report a race).\n\t\tfor range 50 {\n\t\t\tcheckRetained(\"retained 3\")\n\t\t}\n\t\tclose(ch)\n\t\twg.Wait()\n\t}\n\tfor range 5 {\n\t\tf()\n\t}\n}\n\nfunc TestMQTTClusterReplicasCount(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tsize int\n\t\treplicas int\n\t}{\n\t\t{1, 1},\n\t\t{2, 2},\n\t\t{3, 3},\n\t\t{5, 3},\n\t} {\n\t\tt.Run(fmt.Sprintf(\"size %v\", test.size), func(t *testing.T) {\n\t\t\tvar s *Server\n\t\t\tvar o *Options\n\t\t\tif test.size == 1 {\n\t\t\t\to = testMQTTDefaultOptions()\n\t\t\t\ts = testMQTTRunServer(t, o)\n\t\t\t\tdefer testMQTTShutdownServer(s)\n\t\t\t} else {\n\t\t\t\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", test.size)\n\t\t\t\tdefer cl.shutdown()\n\t\t\t\to = cl.opts[0]\n\t\t\t\ts = cl.randomServer()\n\t\t\t}\n\n\t\t\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\t\t\ttestMQTTFlush(t, mc, nil, rc)\n\n\t\t\tnc := natsConnect(t, s.ClientURL())\n\t\t\tdefer nc.Close()\n\n\t\t\t// Check the replicas of all MQTT streams\n\t\t\tjs, err := nc.JetStream()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error getting js: %v\", err)\n\t\t\t}\n\t\t\tfor _, sname := range []string{\n\t\t\t\tmqttStreamName,\n\t\t\t\tmqttRetainedMsgsStreamName,\n\t\t\t\tmqttSessStreamName,\n\t\t\t} {\n\t\t\t\tt.Run(sname, func(t *testing.T) {\n\t\t\t\t\tsi, err := js.StreamInfo(sname)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Error getting stream info: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t\tif si.Config.Replicas != test.replicas {\n\t\t\t\t\t\tt.Fatalf(\"Expected %v replicas, got %v\", test.replicas, si.Config.Replicas)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTClusterCanCreateSessionWithOnServerDown(t *testing.T) {\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", 3)\n\tdefer cl.shutdown()\n\to := cl.opts[0]\n\n\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\tmc.Close()\n\n\t// Shutdown one of the server.\n\tsd := cl.servers[1].StoreDir()\n\tdefer os.RemoveAll(strings.TrimSuffix(sd, JetStreamStoreDir))\n\tcl.servers[1].Shutdown()\n\n\t// Make sure there is a meta leader\n\tcl.waitOnPeerCount(2)\n\tcl.waitOnLeader()\n\n\t// Now try to create a new session. Since we use a single stream now for all sessions,\n\t// this should succeed.\n\to = cl.opts[2]\n\t// We may still get failures because of some JS APIs may timeout while things\n\t// settle, so try again for a certain amount of times.\n\tmc, rc = testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n}\n\nfunc TestMQTTClusterPlacement(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\tc := sc.randomCluster()\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafAndMQTT, \"SPOKE\", 3, 22111)\n\tdefer lnc.shutdown()\n\n\tsc.waitOnPeerCount(9)\n\tsc.waitOnLeader()\n\n\tfor i := 0; i < 10; i++ {\n\t\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, lnc.opts[i%3].MQTT.Host, lnc.opts[i%3].MQTT.Port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\t}\n\n\t// Now check that MQTT assets have been created in the LEAF node's side, not the Hub.\n\tnc := natsConnect(t, lnc.servers[0].ClientURL())\n\tdefer nc.Close()\n\tjs, err := nc.JetStream()\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to get JetStream: %v\", err)\n\t}\n\tcount := 0\n\tfor si := range js.StreamsInfo() {\n\t\tif si.Cluster == nil || si.Cluster.Name != \"SPOKE\" {\n\t\t\tt.Fatalf(\"Expected asset %q to be placed on spoke cluster, was placed on %+v\", si.Config.Name, si.Cluster)\n\t\t}\n\t\tfor _, repl := range si.Cluster.Replicas {\n\t\t\tif !strings.HasPrefix(repl.Name, \"SPOKE-\") {\n\t\t\t\tt.Fatalf(\"Replica on the wrong cluster: %+v\", repl)\n\t\t\t}\n\t\t}\n\t\tif si.State.Consumers > 0 {\n\t\t\tfor ci := range js.ConsumersInfo(si.Config.Name) {\n\t\t\t\tif ci.Cluster == nil || ci.Cluster.Name != \"SPOKE\" {\n\t\t\t\t\tt.Fatalf(\"Expected asset %q to be placed on spoke cluster, was placed on %+v\", ci.Name, si.Cluster)\n\t\t\t\t}\n\t\t\t\tfor _, repl := range ci.Cluster.Replicas {\n\t\t\t\t\tif !strings.HasPrefix(repl.Name, \"SPOKE-\") {\n\t\t\t\t\t\tt.Fatalf(\"Replica on the wrong cluster: %+v\", repl)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcount++\n\t}\n\tif count == 0 {\n\t\tt.Fatal(\"No stream found!\")\n\t}\n}\n\nfunc TestMQTTLeafnodeWithoutJSToClusterWithJSNoSharedSysAcc(t *testing.T) {\n\ttest := func(t *testing.T, resolution int) {\n\t\tgetClusterOpts := func(name string, i int) *Options {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\to.ServerName = name\n\t\t\to.Cluster.Name = \"hub\"\n\t\t\t// first two test cases rely on domain not being set in hub\n\t\t\tif resolution > 1 {\n\t\t\t\to.JetStreamDomain = \"DOMAIN\"\n\t\t\t}\n\t\t\to.Cluster.Host = \"127.0.0.1\"\n\t\t\to.Cluster.Port = 2790 + i\n\t\t\to.Routes = RoutesFromStr(\"nats://127.0.0.1:2791,nats://127.0.0.1:2792,nats://127.0.0.1:2793\")\n\t\t\to.LeafNode.Host = \"127.0.0.1\"\n\t\t\to.LeafNode.Port = -1\n\t\t\treturn o\n\t\t}\n\t\to1 := getClusterOpts(\"S1\", 1)\n\t\ts1 := testMQTTRunServer(t, o1)\n\t\tdefer testMQTTShutdownServer(s1)\n\n\t\to2 := getClusterOpts(\"S2\", 2)\n\t\ts2 := testMQTTRunServer(t, o2)\n\t\tdefer testMQTTShutdownServer(s2)\n\n\t\to3 := getClusterOpts(\"S3\", 3)\n\t\ts3 := testMQTTRunServer(t, o3)\n\t\tdefer testMQTTShutdownServer(s3)\n\n\t\tcluster := []*Server{s1, s2, s3}\n\t\tcheckClusterFormed(t, cluster...)\n\t\tcheckFor(t, 10*time.Second, 50*time.Millisecond, func() error {\n\t\t\tfor _, s := range cluster {\n\t\t\t\tif s.JetStreamIsLeader() {\n\t\t\t\t\t// Need to wait for usage updates now to propagate to meta leader.\n\t\t\t\t\tif len(s.JetStreamClusterPeers()) == len(cluster) {\n\t\t\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"no leader yet\")\n\t\t})\n\n\t\t// Now define a leafnode that has mqtt enabled, but no JS. This should still work.\n\t\tlno := testMQTTDefaultOptions()\n\t\t// Make sure jetstream is not explicitly defined here.\n\t\tlno.JetStream = false\n\t\tswitch resolution {\n\t\tcase 0:\n\t\t\t// turn off jetstream in $G by adding another account and set mqtt domain option and set account default\n\t\t\tlno.Accounts = append(lno.Accounts, NewAccount(\"unused-account\"))\n\t\t\tfallthrough\n\t\tcase 1:\n\t\t\tlno.JsAccDefaultDomain = map[string]string{\"$G\": \"\"}\n\t\tcase 2:\n\t\t\tlno.JsAccDefaultDomain = map[string]string{\"$G\": o1.JetStreamDomain}\n\t\tcase 3:\n\t\t\t// turn off jetstream in $G by adding another account and set mqtt domain option\n\t\t\tlno.Accounts = append(lno.Accounts, NewAccount(\"unused-account\"))\n\t\t\tfallthrough\n\t\tcase 4:\n\t\t\t// actual solution\n\t\t\tlno.MQTT.JsDomain = o1.JetStreamDomain\n\t\tcase 5:\n\t\t\t// set per account overwrite and disable js in $G\n\t\t\tlno.Accounts = append(lno.Accounts, NewAccount(\"unused-account\"))\n\t\t\tlno.JsAccDefaultDomain = map[string]string{\n\t\t\t\t\"$G\": o1.JetStreamDomain,\n\t\t\t}\n\t\t}\n\t\t// Whenever an account was added to disable JS in $G, enable it in the server\n\t\tif len(lno.Accounts) > 0 {\n\t\t\tlno.JetStream = true\n\t\t\tlno.JetStreamDomain = \"OTHER\"\n\t\t\tlno.StoreDir = t.TempDir()\n\t\t}\n\n\t\t// Use RoutesFromStr() to make an array of urls\n\t\turls := RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d,nats://127.0.0.1:%d,nats://127.0.0.1:%d\",\n\t\t\to1.LeafNode.Port, o2.LeafNode.Port, o3.LeafNode.Port))\n\t\tlno.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: urls}}\n\n\t\tln := RunServer(lno)\n\t\tdefer testMQTTShutdownServer(ln)\n\n\t\tcheckLeafNodeConnected(t, ln)\n\n\t\t// Now connect to leafnode and subscribe\n\t\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: true}, lno.MQTT.Host, lno.MQTT.Port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\t\ttestMQTTFlush(t, mc, nil, rc)\n\n\t\tconnectAndPublish := func(o *Options) {\n\t\t\tmp, rp := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\t\t\tdefer mp.Close()\n\t\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTPublish(t, mp, rp, 1, false, false, \"foo\", 1, []byte(\"msg\"))\n\t\t}\n\t\t// Connect a publisher from leafnode and publish, verify message is received.\n\t\tconnectAndPublish(lno)\n\t\ttestMQTTCheckPubMsg(t, mc, rc, \"foo\", mqttPubQos1, []byte(\"msg\"))\n\n\t\t// Connect from one server in the cluster check it works from there too.\n\t\tconnectAndPublish(o3)\n\t\ttestMQTTCheckPubMsg(t, mc, rc, \"foo\", mqttPubQos1, []byte(\"msg\"))\n\n\t\t// Connect from a server in the hub and subscribe\n\t\tmc2, rc2 := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub2\", cleanSess: true}, o2.MQTT.Host, o2.MQTT.Port, 5)\n\t\tdefer mc2.Close()\n\t\ttestMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\t\ttestMQTTFlush(t, mc2, nil, rc2)\n\n\t\t// Connect a publisher from leafnode and publish, verify message is received.\n\t\tconnectAndPublish(lno)\n\t\ttestMQTTCheckPubMsg(t, mc2, rc2, \"foo\", mqttPubQos1, []byte(\"msg\"))\n\n\t\t// Connect from one server in the cluster check it works from there too.\n\t\tconnectAndPublish(o1)\n\t\ttestMQTTCheckPubMsg(t, mc2, rc2, \"foo\", mqttPubQos1, []byte(\"msg\"))\n\t}\n\tt.Run(\"backwards-compatibility-default-js-enabled-in-leaf\", func(t *testing.T) {\n\t\ttest(t, 0) // test with JsAccDefaultDomain set to default (pointing at hub) but jetstream enabled in leaf node too\n\t})\n\tt.Run(\"backwards-compatibility-default-js-disabled-in-leaf\", func(t *testing.T) {\n\t\t// test with JsAccDefaultDomain set. Checks if it works with backwards compatibility code for empty domain\n\t\ttest(t, 1)\n\t})\n\tt.Run(\"backwards-compatibility-domain-js-disabled-in-leaf\", func(t *testing.T) {\n\t\t// test with JsAccDefaultDomain set. Checks if it works with backwards compatibility code for domain set\n\t\ttest(t, 2) // test with domain set in mqtt client\n\t})\n\tt.Run(\"mqtt-explicit-js-enabled-in-leaf\", func(t *testing.T) {\n\t\ttest(t, 3) // test with domain set in mqtt client (pointing at hub) but jetstream enabled in leaf node too\n\t})\n\tt.Run(\"mqtt-explicit-js-disabled-in-leaf\", func(t *testing.T) {\n\t\ttest(t, 4) // test with domain set in mqtt client\n\t})\n\tt.Run(\"backwards-compatibility-domain-js-enabled-in-leaf\", func(t *testing.T) {\n\t\ttest(t, 5) // test with JsAccDefaultDomain set to domain (pointing at hub) but jetstream enabled in leaf node too\n\t})\n}\n\nfunc TestMQTTImportExport(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\taccounts {\n\t\t\torg {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: org, password: pwd}]\n\t\t\t\timports = [{stream: {account: \"device\", subject: \"foo\"}, prefix: \"org\"}]\n\t\t\t}\n\t\t\tdevice {\n\t\t\t\tusers: [{user: device, password: pwd}]\n\t\t\t\texports = [{stream: \"foo\"}]\n\t\t\t}\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tno_auth_user: device\n\t`, t.TempDir())))\n\tdefer os.Remove(conf)\n\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tmc1, rc1 := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub1\", user: \"org\", pass: \"pwd\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc1.Close()\n\ttestMQTTCheckConnAck(t, rc1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc1, rc1, []*mqttFilter{{filter: \"org/foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc1, nil, rc1)\n\n\tmc2, rc2 := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub2\", user: \"org\", pass: \"pwd\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, rc2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc2, rc2, []*mqttFilter{{filter: \"org/foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc2, nil, rc2)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tnatsPub(t, nc, \"foo\", []byte(\"msg\"))\n\n\t// Verify message is received on receiver side.\n\ttestMQTTCheckPubMsg(t, mc1, rc1, \"org/foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc2, rc2, \"org/foo\", 0, []byte(\"msg\"))\n}\n\nfunc TestMQTTSessionMovingDomains(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterTemplWithLeafAndMQTT, \"{{leaf}}\", `leafnodes { listen: 127.0.0.1:-1 }`, 1)\n\ttmpl = strings.Replace(tmpl, \"store_dir:\", \"domain: HUB, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 22020, true)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, \"store_dir:\", \"domain: SPOKE, store_dir:\", 1)\n\tlnc := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"SPOKE\", 3, 22111)\n\tdefer lnc.shutdown()\n\tlnc.waitOnPeerCount(3)\n\n\tconnectSubAndDisconnect := func(host string, port int, present bool) {\n\t\tt.Helper()\n\t\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, host, port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, present)\n\t\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\t\ttestMQTTFlush(t, mc, nil, rc)\n\t\ttestMQTTDisconnect(t, mc, nil)\n\t}\n\n\t// Create a session on the HUB. Make sure we don't use \"clean\" session so that\n\t// it is not removed when the client connection closes.\n\tfor i := 0; i < 7; i++ {\n\t\tvar present bool\n\t\tif i > 0 {\n\t\t\tpresent = true\n\t\t}\n\t\tconnectSubAndDisconnect(c.opts[0].MQTT.Host, c.opts[0].MQTT.Port, present)\n\t}\n\n\t// Now move to the SPOKE cluster, this is a brand new session there, so should not be present.\n\tconnectSubAndDisconnect(lnc.opts[1].MQTT.Host, lnc.opts[1].MQTT.Port, false)\n\n\t// Move back to HUB cluster. Make it interesting by connecting to a different\n\t// server in that cluster. This should work, and present flag should be true.\n\tconnectSubAndDisconnect(c.opts[2].MQTT.Host, c.opts[2].MQTT.Port, true)\n}\n\ntype remoteConnSameClientIDLogger struct {\n\tDummyLogger\n\twarn chan string\n}\n\nfunc (l *remoteConnSameClientIDLogger) Warnf(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"remote connection has started with the same client ID\") {\n\t\tl.warn <- msg\n\t}\n}\n\nfunc TestMQTTSessionsDifferentDomains(t *testing.T) {\n\ttmpl := strings.Replace(jsClusterTemplWithLeafAndMQTT, \"{{leaf}}\", `leafnodes { listen: 127.0.0.1:-1 }`, 1)\n\tc := createJetStreamCluster(t, tmpl, \"HUB\", _EMPTY_, 3, 22020, true)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, \"store_dir:\", \"domain: LEAF1, store_dir:\", 1)\n\tlnc1 := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"LEAF1\", 2, 22111)\n\tdefer lnc1.shutdown()\n\tlnc1.waitOnPeerCount(2)\n\n\tl := &remoteConnSameClientIDLogger{warn: make(chan string, 10)}\n\tlnc1.servers[0].SetLogger(l, false, false)\n\n\ttmpl = strings.Replace(jsClusterTemplWithLeafAndMQTT, \"store_dir:\", \"domain: LEAF2, store_dir:\", 1)\n\tlnc2 := c.createLeafNodesWithTemplateAndStartPort(tmpl, \"LEAF2\", 2, 23111)\n\tdefer lnc2.shutdown()\n\tlnc2.waitOnPeerCount(2)\n\n\to := &(lnc1.opts[0].MQTT)\n\tmc1, rc1 := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.Host, o.Port, 5)\n\tdefer mc1.Close()\n\ttestMQTTCheckConnAck(t, rc1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc1, rc1, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc1, nil, rc1)\n\n\to = &(lnc2.opts[0].MQTT)\n\tconnectSubAndDisconnect := func(host string, port int, present bool) {\n\t\tt.Helper()\n\t\tmc, rc := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, host, port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, present)\n\t\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\t\ttestMQTTFlush(t, mc, nil, rc)\n\t\ttestMQTTDisconnect(t, mc, nil)\n\t}\n\n\tfor i := 0; i < 2; i++ {\n\t\tconnectSubAndDisconnect(o.Host, o.Port, i > 0)\n\t}\n\n\tselect {\n\tcase w := <-l.warn:\n\t\tt.Fatalf(\"Got a warning: %v\", w)\n\tcase <-time.After(500 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\nfunc TestMQTTParseUnsub(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto []byte\n\t\tb byte\n\t\tpl int\n\t\terr string\n\t\twantErr bool\n\t}{\n\t\t{\"reserved flag\", nil, 3, 0, \"wrong unsubscribe reserved flags\", false},\n\t\t{\"ensure packet loaded\", []byte{1, 2}, mqttUnsubscribeFlags, 10, io.ErrUnexpectedEOF.Error(), false},\n\t\t{\"error reading packet id\", []byte{1}, mqttUnsubscribeFlags, 1, \"reading packet identifier\", false},\n\t\t{\"packet id cannot be zero\", []byte{0, 0}, mqttUnsubscribeFlags, 2, errMQTTPacketIdentifierIsZero.Error(), false},\n\t\t{\"missing filters\", []byte{0, 1}, mqttUnsubscribeFlags, 2, \"subscribe protocol must contain at least 1 topic filter\", false},\n\t\t{\"error reading topic\", []byte{0, 1, 0, 2, 'a'}, mqttUnsubscribeFlags, 5, \"topic filter\", false},\n\t\t{\"empty topic\", []byte{0, 1, 0, 0}, mqttUnsubscribeFlags, 4, errMQTTTopicFilterCannotBeEmpty.Error(), false},\n\t\t{\"invalid utf8 topic\", []byte{0, 1, 0, 1, 241}, mqttUnsubscribeFlags, 5, \"invalid utf8 for topic filter\", false},\n\t\t{\"topic with null byte\", []byte{0, 1, 0, 3, 'a', 0, 'b'}, mqttUnsubscribeFlags, 7, \"\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tr := &mqttReader{}\n\t\t\tr.reset(test.proto)\n\t\t\tmqtt := &mqtt{r: r}\n\t\t\tc := &client{mqtt: mqtt}\n\t\t\t_, _, err := c.mqttParseSubsOrUnsubs(r, test.b, test.pl, false)\n\t\t\tif test.wantErr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected an error, got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testMQTTUnsub(t *testing.T, pi uint16, c net.Conn, r *mqttReader, filters []*mqttFilter) {\n\tt.Helper()\n\tw := newMQTTWriter(0)\n\tpkLen := 2 // for pi\n\tfor i := 0; i < len(filters); i++ {\n\t\tf := filters[i]\n\t\tpkLen += 2 + len(f.filter)\n\t}\n\tw.WriteByte(mqttPacketUnsub | mqttUnsubscribeFlags)\n\tw.WriteVarInt(pkLen)\n\tw.WriteUint16(pi)\n\tfor i := 0; i < len(filters); i++ {\n\t\tf := filters[i]\n\t\tw.WriteBytes([]byte(f.filter))\n\t}\n\tif _, err := testMQTTWrite(c, w.Bytes()); err != nil {\n\t\tt.Fatalf(\"Error writing UNSUBSCRIBE protocol: %v\", err)\n\t}\n\tb, _ := testMQTTReadPacket(t, r)\n\tif pt := b & mqttPacketMask; pt != mqttPacketUnsubAck {\n\t\tt.Fatalf(\"Expected UNSUBACK packet %x, got %x\", mqttPacketUnsubAck, pt)\n\t}\n\trpi, err := r.readUint16(\"packet identifier\")\n\tif err != nil || rpi != pi {\n\t\tt.Fatalf(\"Error with packet identifier expected=%v got: %v err=%v\", pi, rpi, err)\n\t}\n}\n\nfunc TestMQTTUnsub(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmcp, mpr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcp.Close()\n\ttestMQTTCheckConnAck(t, mpr, mqttConnAckRCConnectionAccepted, false)\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{user: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish and test msg received\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo\", 0, []byte(\"msg\"))\n\n\t// Unsubscribe\n\ttestMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\"}})\n\n\t// Publish and test msg not received\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Use of wildcards subs\n\tfilters := []*mqttFilter{\n\t\t{filter: \"foo/bar\", qos: 0},\n\t\t{filter: \"foo/#\", qos: 0},\n\t}\n\ttestMQTTSub(t, 1, mc, r, filters, []byte{0, 0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\t// Publish and check that message received twice\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", 0, []byte(\"msg\"))\n\n\t// Unsub the wildcard one\n\ttestMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/#\"}})\n\t// Publish and check that message received once\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTCheckPubMsg(t, mc, r, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Unsub last\n\ttestMQTTUnsub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/bar\"}})\n\t// Publish and test msg not received\n\ttestMQTTPublish(t, mcp, r, 0, false, false, \"foo/bar\", 0, []byte(\"msg\"))\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc testMQTTExpectDisconnect(t testing.TB, c net.Conn) {\n\tt.Helper()\n\tbuf, err := testMQTTRead(c)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected connection to be disconnected, got %s\", buf)\n\t}\n\t// Distinguish real disconnection (EOF, connection reset) from timeout\n\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\tt.Fatal(\"Expected a disconnect but got a timeout error\")\n\t}\n}\n\nfunc TestMQTTPublishTopicErrors(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\ttopic string\n\t}{\n\t\t{\"empty\", \"\"},\n\t\t{\"with single level wildcard\", \"foo/+\"},\n\t\t{\"with multiple level wildcard\", \"foo/#\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTPublish(t, mc, r, 0, false, false, test.topic, 0, []byte(\"msg\"))\n\t\t\ttestMQTTExpectDisconnect(t, mc)\n\t\t})\n\t}\n}\n\nfunc testMQTTDisconnect(t testing.TB, c net.Conn, bw *bufio.Writer) {\n\tt.Helper()\n\ttestMQTTDisconnectEx(t, c, bw, true)\n}\n\nfunc testMQTTDisconnectEx(t testing.TB, c net.Conn, bw *bufio.Writer, wait bool) {\n\tt.Helper()\n\tw := newMQTTWriter(0)\n\tw.WriteByte(mqttPacketDisconnect)\n\tw.WriteByte(0)\n\tif bw != nil {\n\t\tbw.Write(w.Bytes())\n\t\tbw.Flush()\n\t} else {\n\t\tc.Write(w.Bytes())\n\t}\n\tif wait {\n\t\ttestMQTTExpectDisconnect(t, c)\n\t}\n}\n\nfunc TestMQTTWill(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"will.topic\")\n\tnatsFlush(t, nc)\n\n\twillMsg := []byte(\"bye\")\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\twillExpected bool\n\t\twillQoS byte\n\t}{\n\t\t{\"will qos 0\", true, 0},\n\t\t{\"will qos 1\", true, 1},\n\t\t{\"will qos 2\", true, 2},\n\t\t{\"proper disconnect no will\", false, 0},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mcs.Close()\n\t\t\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"will/#\", qos: 2}}, []byte{2})\n\t\t\ttestMQTTFlush(t, mcs, nil, rs)\n\n\t\t\tmc, r := testMQTTConnect(t,\n\t\t\t\t&mqttConnInfo{\n\t\t\t\t\tcleanSess: true,\n\t\t\t\t\twill: &mqttWill{\n\t\t\t\t\t\ttopic: []byte(\"will/topic\"),\n\t\t\t\t\t\tmessage: willMsg,\n\t\t\t\t\t\tqos: test.willQoS,\n\t\t\t\t\t},\n\t\t\t\t}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tif test.willExpected {\n\t\t\t\tmc.Close()\n\t\t\t\ttestMQTTCheckPubMsg(t, mcs, rs, \"will/topic\", test.willQoS<<1, willMsg)\n\t\t\t\twm := natsNexMsg(t, sub, time.Second)\n\t\t\t\tif !bytes.Equal(wm.Data, willMsg) {\n\t\t\t\t\tt.Fatalf(\"Expected will message to be %q, got %q\", willMsg, wm.Data)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ttestMQTTDisconnect(t, mc, nil)\n\t\t\t\ttestMQTTExpectNothing(t, rs)\n\t\t\t\tif wm, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\t\t\t\tt.Fatalf(\"Should not have receive a message, got subj=%q data=%q\",\n\t\t\t\t\t\twm.Subject, wm.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTQoS2WillReject(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.rejectQoS2Pub = true\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcs.Close()\n\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"will/#\", qos: 2}}, []byte{2})\n\ttestMQTTFlush(t, mcs, nil, rs)\n\n\tmc, r := testMQTTConnect(t,\n\t\t&mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\twill: &mqttWill{\n\t\t\t\ttopic: []byte(\"will/topic\"),\n\t\t\t\tmessage: []byte(\"bye\"),\n\t\t\t\tqos: 2,\n\t\t\t},\n\t\t}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCQoS2WillRejected, false)\n}\n\nfunc TestMQTTWillRetain(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpubQoS byte\n\t\tsubQoS byte\n\t}{\n\t\t{\"pub QoS0 sub QoS0\", 0, 0},\n\t\t{\"pub QoS0 sub QoS1\", 0, 1},\n\t\t{\"pub QoS1 sub QoS0\", 1, 0},\n\t\t{\"pub QoS1 sub QoS1\", 1, 1},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testMQTTDefaultOptions()\n\t\t\ts := testMQTTRunServer(t, o)\n\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\tmces, res := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mces.Close()\n\t\t\ttestMQTTCheckConnAck(t, res, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTSub(t, 1, mces, res, []*mqttFilter{{filter: \"will/#\", qos: test.subQoS}}, []byte{test.subQoS})\n\n\t\t\twillTopic := []byte(\"will/topic\")\n\t\t\twillMsg := []byte(\"bye\")\n\n\t\t\tmc, r := testMQTTConnect(t,\n\t\t\t\t&mqttConnInfo{\n\t\t\t\t\tcleanSess: true,\n\t\t\t\t\twill: &mqttWill{\n\t\t\t\t\t\ttopic: willTopic,\n\t\t\t\t\t\tmessage: willMsg,\n\t\t\t\t\t\tqos: test.pubQoS,\n\t\t\t\t\t\tretain: true,\n\t\t\t\t\t},\n\t\t\t\t}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\t// Disconnect the client\n\t\t\tmc.Close()\n\n\t\t\t// Wait for the server to process the connection close, which will\n\t\t\t// cause the \"will\" message to be published (and retained).\n\t\t\tcheckClientsCount(t, s, 1)\n\n\t\t\t// Create subscription on will topic and expect will message.\n\t\t\tmcs, rs := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mcs.Close()\n\t\t\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"will/#\", qos: test.subQoS}}, []byte{test.subQoS})\n\t\t\tpflags, _ := testMQTTGetPubMsg(t, mcs, rs, \"will/topic\", willMsg)\n\t\t\tif !mqttIsRetained(pflags) {\n\t\t\t\tt.Fatalf(\"expected retain flag to be set, it was not: %v\", pflags)\n\t\t\t}\n\t\t\t// Expected QoS will be the lesser of the pub/sub QoS.\n\t\t\texpectedQoS := test.pubQoS\n\t\t\tif test.subQoS == 0 {\n\t\t\t\texpectedQoS = 0\n\t\t\t}\n\t\t\tif qos := mqttGetQoS(pflags); qos != expectedQoS {\n\t\t\t\tt.Fatalf(\"expected qos to be %v, got %v\", expectedQoS, qos)\n\t\t\t}\n\n\t\t\t// The existing subscription (prior to sending the will) should receive\n\t\t\t// the will but the retain flag should not be set.\n\t\t\tpflags, _ = testMQTTGetPubMsg(t, mces, res, \"will/topic\", willMsg)\n\t\t\tif mqttIsRetained(pflags) {\n\t\t\t\tt.Fatalf(\"expected retain flag to not be set, it was: %v\", pflags)\n\t\t\t}\n\t\t\t// Expected QoS will be the lesser of the pub/sub QoS.\n\t\t\tif qos := mqttGetQoS(pflags); qos != expectedQoS {\n\t\t\t\tt.Fatalf(\"expected qos to be %v, got %v\", expectedQoS, qos)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTWillRetainPermViolation(t *testing.T) {\n\ttemplate := `\n\t\tport: -1\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tserver_name: mqtt\n\t\tauthorization {\n\t\t\tmqtt_perms = {\n\t\t\t\tpublish = [\"%s\"]\n\t\t\t\tsubscribe = [\"foo\", \"bar\"]\n\t\t\t}\n\t\t\tusers = [\n\t\t\t\t{user: mqtt, password: pass, permissions: $mqtt_perms}\n\t\t\t]\n\t\t}\n\t\tmqtt {\n\t\t\tport: -1\n\t\t}\n\t`\n\ttdir := t.TempDir()\n\tconf := createConfFile(t, fmt.Appendf(nil, template, tdir, \"foo\"))\n\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{\n\t\tcleanSess: true,\n\t\tuser: \"mqtt\",\n\t\tpass: \"pass\",\n\t}\n\n\t// We create first a connection with the Will topic that the publisher\n\t// is allowed to publish to.\n\tci.will = &mqttWill{\n\t\ttopic: []byte(\"foo\"),\n\t\tmessage: []byte(\"bye\"),\n\t\tqos: 1,\n\t\tretain: true,\n\t}\n\tmc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Disconnect, which will cause the Will to be sent with retain flag.\n\tmc.Close()\n\n\t// Wait for the server to process the connection close, which will\n\t// cause the \"will\" message to be published (and retained).\n\tcheckClientsCount(t, s, 0)\n\n\t// Create a subscription on the Will subject and we should receive it.\n\tci.will = nil\n\tmcs, rs := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcs.Close()\n\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\tpflags, _ := testMQTTGetPubMsg(t, mcs, rs, \"foo\", []byte(\"bye\"))\n\tif !mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"expected retain flag to be set, it was not: %v\", pflags)\n\t}\n\tif qos := mqttGetQoS(pflags); qos != 1 {\n\t\tt.Fatalf(\"expected qos to be 1, got %v\", qos)\n\t}\n\ttestMQTTDisconnect(t, mcs, nil)\n\tmcs.Close()\n\n\t// Now create another connection with a Will that client is not allowed to publish to.\n\tci.will = &mqttWill{\n\t\ttopic: []byte(\"bar\"),\n\t\tmessage: []byte(\"bye\"),\n\t\tqos: 1,\n\t\tretain: true,\n\t}\n\tmc, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Disconnect, to cause Will to be produced, but in that case should not be stored\n\t// since user not allowed to publish on \"bar\".\n\tmc.Close()\n\n\t// Wait for the server to process the connection close, which will\n\t// cause the \"will\" message to be published (and retained).\n\tcheckClientsCount(t, s, 0)\n\n\t// Create sub on \"bar\" which user is allowed to subscribe to.\n\tci.will = nil\n\tmcs, rs = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcs.Close()\n\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\t// No Will should be published since it should not have been stored in the first place.\n\ttestMQTTExpectNothing(t, rs)\n\ttestMQTTDisconnect(t, mcs, nil)\n\tmcs.Close()\n\n\t// Now remove permission to publish on \"foo\" and check that a new subscription\n\t// on \"foo\" is now not getting the will message because the original user no\n\t// longer has permission to do so.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template, tdir, \"baz\"))\n\n\tmcs, rs = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcs.Close()\n\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mcs, rs, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTExpectNothing(t, rs)\n\ttestMQTTDisconnect(t, mcs, nil)\n}\n\nfunc TestMQTTPublishRetain(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tbuf := strings.Builder{}\n\tfor i := 0; i < 128; i++ { // 128Kb\n\t\tfor j := 0; j < 64; j++ { // 16 * 64 = 1Kb\n\t\t\tbuf.Write([]byte(\"0123456789abcdef\"))\n\t\t}\n\t}\n\tlarge := buf.String()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tretained bool\n\t\tsentValue string\n\t\texpectedValue string\n\t\tsubGetsIt bool\n\t}{\n\t\t{\"publish large retained\", true, large, large, true},\n\t\t{\"publish retained\", true, \"retained\", \"retained\", true},\n\t\t{\"publish not retained\", false, \"not retained\", \"retained\", true},\n\t\t{\"remove retained\", true, \"\", \"\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc1.Close()\n\t\t\ttestMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTPublish(t, mc1, rs1, 0, false, test.retained, \"foo\", 0, []byte(test.sentValue))\n\t\t\ttestMQTTFlush(t, mc1, nil, rs1)\n\n\t\t\tmc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc2.Close()\n\t\t\ttestMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\ttestMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: \"foo/#\", qos: 1}}, []byte{1})\n\n\t\t\tif test.subGetsIt {\n\t\t\t\tpflags, _ := testMQTTGetPubMsg(t, mc2, rs2, \"foo\", []byte(test.expectedValue))\n\t\t\t\tif !mqttIsRetained(pflags) {\n\t\t\t\t\tt.Fatalf(\"retain flag should have been set, it was not: flags=%v\", pflags)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ttestMQTTExpectNothing(t, rs2)\n\t\t\t}\n\n\t\t\ttestMQTTDisconnect(t, mc1, nil)\n\t\t\ttestMQTTDisconnect(t, mc2, nil)\n\t\t})\n\t}\n}\n\nfunc TestMQTTQoS2RetainedReject(t *testing.T) {\n\t// Start the server with QOS2 enabled, and submit retained messages with QoS\n\t// 1 and 2.\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tmc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, mc1, rs1, 2, false, true, \"foo\", 1, []byte(\"qos2 failed\"))\n\ttestMQTTPublish(t, mc1, rs1, 1, false, true, \"bar\", 2, []byte(\"qos1 retained\"))\n\ttestMQTTFlush(t, mc1, nil, rs1)\n\ttestMQTTDisconnect(t, mc1, nil)\n\tmc1.Close()\n\ts.Shutdown()\n\n\t// Restart the server with QOS2 disabled; we should be using the same\n\t// JetStream store, so the retained message should still be there.\n\to.MQTT.rejectQoS2Pub = true\n\ts = testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: \"bar/#\", qos: 2}}, []byte{2})\n\tpflags, _ := testMQTTGetPubMsg(t, mc2, rs2, \"bar\", []byte(\"qos1 retained\"))\n\tif !mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"retain flag should have been set, it was not: flags=%v\", pflags)\n\t}\n\n\ttestMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: \"foo/#\", qos: 2}}, []byte{2})\n\ttestMQTTExpectNothing(t, rs2)\n\ttestMQTTDisconnect(t, mc2, nil)\n}\n\nfunc TestMQTTRetainFlag(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc1, rs1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc1.Close()\n\ttestMQTTCheckConnAck(t, rs1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, mc1, rs1, 0, false, true, \"foo/0\", 0, []byte(\"flag set\"))\n\ttestMQTTPublish(t, mc1, rs1, 0, false, true, \"foo/1\", 0, []byte(\"flag set\"))\n\ttestMQTTFlush(t, mc1, nil, rs1)\n\n\tmc2, rs2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, rs2, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: \"foo/0\", qos: 0}}, []byte{0})\n\tpflags, _ := testMQTTGetPubMsg(t, mc2, rs2, \"foo/0\", []byte(\"flag set\"))\n\tif !mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"retain flag should have been set, it was not: flags=%v\", pflags)\n\t}\n\n\ttestMQTTSub(t, 1, mc2, rs2, []*mqttFilter{{filter: \"foo/1\", qos: 1}}, []byte{1})\n\tpflags, _ = testMQTTGetPubMsg(t, mc2, rs2, \"foo/1\", []byte(\"flag set\"))\n\tif !mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"retain flag should have been set, it was not: flags=%v\", pflags)\n\t}\n\n\t// For existing subscriptions, RETAIN flag should not be set: [MQTT-3.3.1-9].\n\ttestMQTTPublish(t, mc1, rs1, 0, false, true, \"foo/0\", 0, []byte(\"flag not set\"))\n\ttestMQTTFlush(t, mc1, nil, rs1)\n\n\tpflags, _ = testMQTTGetPubMsg(t, mc2, rs2, \"foo/0\", []byte(\"flag not set\"))\n\tif mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"retain flag should not have been set, it was: flags=%v\", pflags)\n\t}\n\n\ttestMQTTPublish(t, mc1, rs1, 0, false, true, \"foo/1\", 0, []byte(\"flag not set\"))\n\ttestMQTTFlush(t, mc1, nil, rs1)\n\n\tpflags, _ = testMQTTGetPubMsg(t, mc2, rs2, \"foo/1\", []byte(\"flag not set\"))\n\tif mqttIsRetained(pflags) {\n\t\tt.Fatalf(\"retain flag should not have been set, it was: flags=%v\", pflags)\n\t}\n}\n\nfunc TestMQTTPublishRetainPermViolation(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.Users = []*User{\n\t\t{\n\t\t\tUsername: \"mqtt1\",\n\t\t\tPassword: \"pass\",\n\t\t\tPermissions: &Permissions{\n\t\t\t\tPublish: &SubjectPermission{Allow: []string{\"foo\"}},\n\t\t\t\tSubscribe: &SubjectPermission{Allow: []string{\"bar\"}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tUsername: \"mqtt2\",\n\t\t\tPassword: \"pass\",\n\t\t\tPermissions: &Permissions{\n\t\t\t\tPublish: &SubjectPermission{Allow: []string{\"foo\", \"bar\"}},\n\t\t\t\tSubscribe: &SubjectPermission{Allow: []string{\">\"}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tUsername: \"mqtt3\",\n\t\t\tPassword: \"pass\",\n\t\t\tPermissions: &Permissions{\n\t\t\t\tPublish: &SubjectPermission{Allow: []string{\"foo.bar\", \"baz\", \"barbaz\"}},\n\t\t\t\tSubscribe: &SubjectPermission{Allow: []string{\">\"}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tUsername: \"mqtt4\",\n\t\t\tPassword: \"pass\",\n\t\t},\n\t}\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tvar asm *mqttAccountSessionManager\n\n\tpubRetained := func(user, subject string) {\n\t\tt.Helper()\n\t\tmc, rs := testMQTTConnect(t, &mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\tclientID: \"pub\",\n\t\t\tuser: user,\n\t\t\tpass: \"pass\",\n\t\t}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTPublish(t, mc, rs, 0, false, true, subject, 0, []byte(\"retained\"))\n\t\ttestMQTTFlush(t, mc, nil, rs)\n\t\tif asm == nil {\n\t\t\tasm = testMQTTGetAccountSessionManager(t, s, \"pub\")\n\t\t}\n\t\ttestMQTTDisconnect(t, mc, nil)\n\t}\n\tconsumeRetained := func(user, subject string, expected bool) {\n\t\tt.Helper()\n\t\tmc, rs := testMQTTConnect(t, &mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\tuser: user,\n\t\t\tpass: \"pass\",\n\t\t}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, rs, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, mc, rs, []*mqttFilter{{filter: subject, qos: 0}}, []byte{0})\n\t\tif expected {\n\t\t\ttestMQTTCheckPubMsg(t, mc, rs, subject, mqttPubFlagRetain, []byte(\"retained\"))\n\t\t} else {\n\t\t\ttestMQTTExpectNothing(t, rs)\n\t\t}\n\t\ttestMQTTDisconnect(t, mc, nil)\n\t}\n\n\t// With user \"mqtt\", publish a retained message on \"bar\".\n\t// Since this user has no permission, the server should not have stored it.\n\tpubRetained(\"mqtt1\", \"bar\")\n\n\t// Verify that we can't get it with a new subscription.\n\tconsumeRetained(\"mqtt1\", \"bar\", false)\n\n\t// Use the user \"mqtt2\" that has permissions to publish on foo and bar.\n\t// Publish on \"foo\" and check retained message can be received.\n\tpubRetained(\"mqtt2\", \"foo\")\n\tconsumeRetained(\"mqtt2\", \"foo\", true)\n\n\t// For user \"mqtt3\", we will publish on \"foo/bar\" and check retained\n\t// message is properly received.\n\tpubRetained(\"mqtt3\", \"foo/bar\")\n\tconsumeRetained(\"mqtt3\", \"foo/bar\", true)\n\n\t// Simulate a message that would have been produced in a different server\n\t// on subject \"barbaz\". We will use user \"mqtt4\" that has no pub permissions\n\t// since we need to send to low-level \"$MQTT.rmsgs.barbaz\" subject...\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"mqtt4\", \"pass\"))\n\tdefer nc.Close()\n\tmsg := nats.NewMsg(\"$MQTT.rmsgs.barbaz\")\n\tmsg.Header.Set(mqttNatsRetainedMessageOrigin, \"SomeOtherServer\")\n\tmsg.Header.Set(mqttNatsRetainedMessageTopic, \"barbaz\")\n\tmsg.Header.Set(mqttNatsRetainedMessageFlags, \"1\")\n\tmsg.Data = []byte(\"retained\")\n\tnc.PublishMsg(msg)\n\tnatsFlush(t, nc)\n\n\t// Wait a bit to make sure it is processed.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Then check that it can be received\n\tconsumeRetained(\"mqtt3\", \"barbaz\", true)\n\n\t// Same with user \"mqtt4\" that does not have permissions defined, which\n\t// means allowed to pub/sub on everything.\n\tpubRetained(\"mqtt4\", \"bat\")\n\tconsumeRetained(\"mqtt4\", \"bat\", true)\n\n\t// Do a config reload and make sure that the server does not panic\n\t// and we can still get the retained messages.\n\tno := *o\n\t// Remove the \"bar\" publish permission from \"mqtt2\"\n\tno.Users[1].Permissions.Publish = &SubjectPermission{Allow: []string{\"foo\"}}\n\t// And the \"foo.bar\" and \"barbaz\" publish permissions from \"mqtt3\"\n\tno.Users[2].Permissions.Publish = &SubjectPermission{Allow: []string{\"baz\"}}\n\terr := s.ReloadOptions(&no)\n\trequire_NoError(t, err)\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tasm.mu.RLock()\n\t\tdefer asm.mu.RUnlock()\n\t\tif _, ok := asm.retmsgs[\"foo.bar\"]; ok {\n\t\t\treturn errors.New(\"foo.bar subject still in map\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Still message on \"bar\" should not exist\n\tconsumeRetained(\"mqtt1\", \"bar\", false)\n\t// This one should still be able to be received\n\tconsumeRetained(\"mqtt2\", \"foo\", true)\n\t// Retained message on \"foo.bar\" should have been removed.\n\tconsumeRetained(\"mqtt3\", \"foo/bar\", false)\n\t// However, message on \"barbaz\" should have been left alone since it\n\t// was produced on a different server.\n\tconsumeRetained(\"mqtt3\", \"barbaz\", true)\n\t// And finally, this user that had no permission should still be able\n\t// to get the retained message on \"bat\".\n\tconsumeRetained(\"mqtt4\", \"bat\", true)\n}\n\nfunc TestMQTTPermissionsViolation(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.Users = []*User{\n\t\t{\n\t\t\tUsername: \"mqtt\",\n\t\t\tPassword: \"pass\",\n\t\t\tPermissions: &Permissions{\n\t\t\t\tPublish: &SubjectPermission{Allow: []string{\"foo.bar\"}},\n\t\t\t\tSubscribe: &SubjectPermission{Allow: []string{\"foo.*\"}, Deny: []string{\"foo.baz\"}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tUsername: \"mqtt2\",\n\t\t\tPassword: \"pass\",\n\t\t\tPermissions: &Permissions{\n\t\t\t\tPublish: &SubjectPermission{Allow: []string{\"foo\"}},\n\t\t\t\t// We use to require explicit allow permission on \"$MQTT.sub.>\" when any\n\t\t\t\t// subscribe permission were provided. We now implicitly allow it, but\n\t\t\t\t// we want to make sure that, if needs arise, one can block the server\n\t\t\t\t// to subscribe on \"$MQTT.sub\".\n\t\t\t\tSubscribe: &SubjectPermission{Allow: []string{\"foo\"}, Deny: []string{\"$MQTT.sub.>\"}},\n\t\t\t},\n\t\t},\n\t}\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{\n\t\tuser: \"mqtt\",\n\t\tpass: \"pass\",\n\t}\n\tci.clientID = \"sub\"\n\tmc, rc := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/+\", qos: 1}}, []byte{1})\n\t// Should not be allowed to subscribe on specifically on \"foo/baz\"\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/baz\", qos: 1}}, []byte{mqttSubAckFailure})\n\ttestMQTTFlush(t, mc, nil, rc)\n\n\tci.clientID = \"pub\"\n\tci.cleanSess = true\n\tmp, rp := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// These should be received since publisher has the right to publish on foo.bar\n\ttestMQTTPublish(t, mp, rp, 0, false, false, \"foo/bar\", 0, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, mc, rc, \"foo/bar\", 0, []byte(\"msg1\"))\n\ttestMQTTPublish(t, mp, rp, 1, false, false, \"foo/bar\", 1, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsg(t, mc, rc, \"foo/bar\", mqttPubQos1, []byte(\"msg2\"))\n\n\t// But these should not be cause pub has no permission to publish on foo.baz\n\ttestMQTTPublish(t, mp, rp, 0, false, false, \"foo/baz\", 0, []byte(\"msg3\"))\n\ttestMQTTExpectNothing(t, rc)\n\ttestMQTTPublish(t, mp, rp, 1, false, false, \"foo/baz\", 1, []byte(\"msg4\"))\n\ttestMQTTExpectNothing(t, rc)\n\n\t// Disconnect publisher\n\ttestMQTTDisconnect(t, mp, nil)\n\tmp.Close()\n\t// Disconnect subscriber and restart it to make sure that it does not receive msg3/msg4\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\tci.cleanSess = false\n\tci.clientID = \"sub\"\n\tmc, rc = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/+\", qos: 1}}, []byte{1})\n\ttestMQTTExpectNothing(t, rc)\n\tmc.Close()\n\n\tci.cleanSess = true\n\tci.user = \"mqtt2\"\n\tci.clientID = \"sub\"\n\tmc, rc = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, rc, mqttConnAckRCConnectionAccepted, false)\n\t// This should fail because although this user is allowed to subscribe on \"foo\"\n\t// we deny permission to subscribe on \"$MQTT.sub.>\", so the server won't be\n\t// able to create the internal NATS subscription for the JS consumer.\n\ttestMQTTSub(t, 1, mc, rc, []*mqttFilter{{filter: \"foo/baz\", qos: 1}}, []byte{mqttSubAckFailure})\n}\n\nfunc TestMQTTCleanSession(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{\n\t\tclientID: \"me\",\n\t\tcleanSess: false,\n\t}\n\tc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\tc, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\tci.cleanSess = true\n\tc, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTDisconnect(t, c, nil)\n}\n\nfunc TestMQTTDuplicateClientID(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{\n\t\tclientID: \"me\",\n\t\tcleanSess: false,\n\t}\n\tc1, r1 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c1.Close()\n\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\n\tc2, r2 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true)\n\n\t// The old client should be disconnected.\n\ttestMQTTExpectDisconnect(t, c1)\n}\n\nfunc TestMQTTPersistedSession(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, c, r,\n\t\t[]*mqttFilter{\n\t\t\t{filter: \"foo/#\", qos: 1},\n\t\t\t{filter: \"bar\", qos: 1},\n\t\t\t{filter: \"baz\", qos: 0},\n\t\t},\n\t\t[]byte{1, 1, 0})\n\ttestMQTTFlush(t, c, nil, r)\n\n\t// Shutdown server, close connection and restart server. It should\n\t// have restored the session and consumers.\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\ts.Shutdown()\n\tc.Close()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// There is already the defer for shutdown at top of function\n\n\t// Create a publisher that will send qos1 so we verify that messages\n\t// are stored for the persisted sessions.\n\tc, r = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, c, r, 1, false, false, \"foo/bar\", 1, []byte(\"msg0\"))\n\ttestMQTTFlush(t, c, nil, r)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\t// Recreate session\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\t// Since consumers have been recovered, messages should be received\n\t// (MQTT does not need client to recreate consumers for a recovered\n\t// session)\n\n\t// Check that qos1 publish message is received.\n\ttestMQTTCheckPubMsg(t, c, r, \"foo/bar\", mqttPubQos1, []byte(\"msg0\"))\n\n\t// Flush to prevent publishes to be done too soon since we are\n\t// receiving the CONNACK before the subscriptions are restored.\n\ttestMQTTFlush(t, c, nil, r)\n\n\t// Now publish some messages to all subscriptions.\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tnatsPub(t, nc, \"foo.bar\", []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo/bar\", 0, []byte(\"msg1\"))\n\n\tnatsPub(t, nc, \"foo\", []byte(\"msg2\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", 0, []byte(\"msg2\"))\n\n\tnatsPub(t, nc, \"bar\", []byte(\"msg3\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"bar\", 0, []byte(\"msg3\"))\n\n\tnatsPub(t, nc, \"baz\", []byte(\"msg4\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"baz\", 0, []byte(\"msg4\"))\n\n\t// Now unsub \"bar\" and verify that message published on this topic\n\t// is not received.\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"bar\"}})\n\tnatsPub(t, nc, \"bar\", []byte(\"msg5\"))\n\ttestMQTTExpectNothing(t, r)\n\n\tnc.Close()\n\ts.Shutdown()\n\tc.Close()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// There is already the defer for shutdown at top of function\n\n\t// Recreate a client\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\tnc = natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tnatsPub(t, nc, \"foo.bar\", []byte(\"msg6\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo/bar\", 0, []byte(\"msg6\"))\n\n\tnatsPub(t, nc, \"foo\", []byte(\"msg7\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", 0, []byte(\"msg7\"))\n\n\t// Make sure that we did not recover bar.\n\tnatsPub(t, nc, \"bar\", []byte(\"msg8\"))\n\ttestMQTTExpectNothing(t, r)\n\n\tnatsPub(t, nc, \"baz\", []byte(\"msg9\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"baz\", 0, []byte(\"msg9\"))\n\n\t// Have the sub client send a subscription downgrading the qos1 subscription.\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo/#\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, c, nil, r)\n\n\tnc.Close()\n\ts.Shutdown()\n\tc.Close()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// There is already the defer for shutdown at top of function\n\n\t// Recreate the sub client\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\t// Publish as a qos1\n\tc2, r2 := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, c2, r2, 1, false, false, \"foo/bar\", 1, []byte(\"msg10\"))\n\n\t// Verify that it is received as qos0 which is the qos of the subscription.\n\ttestMQTTCheckPubMsg(t, c, r, \"foo/bar\", 0, []byte(\"msg10\"))\n\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\ttestMQTTDisconnect(t, c2, nil)\n\tc2.Close()\n\n\t// Finally, recreate the sub with clean session and ensure that all is gone\n\tcisub.cleanSess = true\n\tfor i := 0; i < 2; i++ {\n\t\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\tnc = natsConnect(t, s.ClientURL())\n\t\tdefer nc.Close()\n\n\t\tnatsPub(t, nc, \"foo.bar\", []byte(\"msg11\"))\n\t\ttestMQTTExpectNothing(t, r)\n\n\t\tnatsPub(t, nc, \"foo\", []byte(\"msg12\"))\n\t\ttestMQTTExpectNothing(t, r)\n\n\t\t// Make sure that we did not recover bar.\n\t\tnatsPub(t, nc, \"bar\", []byte(\"msg13\"))\n\t\ttestMQTTExpectNothing(t, r)\n\n\t\tnatsPub(t, nc, \"baz\", []byte(\"msg14\"))\n\t\ttestMQTTExpectNothing(t, r)\n\n\t\ttestMQTTDisconnect(t, c, nil)\n\t\tc.Close()\n\t\tnc.Close()\n\n\t\ts.Shutdown()\n\t\to.Port = -1\n\t\to.MQTT.Port = -1\n\t\to.StoreDir = dir\n\t\ts = testMQTTRunServer(t, o)\n\t\t// There is already the defer for shutdown at top of function\n\t}\n}\n\nfunc TestMQTTRecoverSessionAndAddNewSub(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub1\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\t// Shutdown server, close connection and restart server. It should\n\t// have restored the session and consumers.\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\ts.Shutdown()\n\tc.Close()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// No need for defer since it is done top of function\n\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\t// Now add sub and make sure it does not crash\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c, nil, r)\n\n\t// Now repeat with a new client but without server restart.\n\tcisub2 := &mqttConnInfo{clientID: \"sub2\", cleanSess: false}\n\tc2, r2 := testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTDisconnect(t, c2, nil)\n\tc2.Close()\n\n\tc2, r2 = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, c2, r2, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c2, nil, r2)\n}\n\nfunc TestMQTTRecoverSessionWithSubAndClientResendSub(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tcisub1 := &mqttConnInfo{clientID: \"sub1\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub1, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Have a client send a SUBSCRIBE protocol for foo, QoS1\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\t// Restart the server now.\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\ts.Shutdown()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// No need for defer since it is done top of function\n\n\t// Now restart the client. Since the client was created with cleanSess==false,\n\t// the server will have recorded the subscriptions for this client.\n\tc, r = testMQTTConnect(t, cisub1, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\t// At this point, the server has recreated the subscription on foo, QoS1.\n\n\t// For applications that restart, it is possible (likely) that they\n\t// will resend their SUBSCRIBE protocols, so do so now:\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c, nil, r)\n\n\tcheckNumSub := func(clientID string) {\n\t\tt.Helper()\n\n\t\t// Find the MQTT client...\n\t\tmc := testMQTTGetClient(t, s, clientID)\n\n\t\t// Check how many NATS subscriptions are registered.\n\t\tvar fooSub int\n\t\tvar otherSub int\n\t\tmc.mu.Lock()\n\t\tfor _, sub := range mc.subs {\n\t\t\tswitch string(sub.subject) {\n\t\t\tcase \"foo\":\n\t\t\t\tfooSub++\n\t\t\tdefault:\n\t\t\t\totherSub++\n\t\t\t}\n\t\t}\n\t\tmc.mu.Unlock()\n\n\t\t// We should have 2 subscriptions, one on \"foo\", and one for the JS durable\n\t\t// consumer's delivery subject.\n\t\tif fooSub != 1 {\n\t\t\tt.Fatalf(\"Expected 1 sub on 'foo', got %v\", fooSub)\n\t\t}\n\t\tif otherSub != 1 {\n\t\t\tt.Fatalf(\"Expected 1 subscription for JS durable, got %v\", otherSub)\n\t\t}\n\t}\n\tcheckNumSub(\"sub1\")\n\n\tc.Close()\n\n\t// Now same but without the server restart in-between.\n\tcisub2 := &mqttConnInfo{clientID: \"sub2\", cleanSess: false}\n\tc, r = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\t// Restart client\n\tc, r = testMQTTConnect(t, cisub2, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c, nil, r)\n\t// Check client subs\n\tcheckNumSub(\"sub2\")\n}\n\nfunc TestMQTTFlappingSession(t *testing.T) {\n\tmqttSessJailDur = 250 * time.Millisecond\n\tmqttFlapCleanItvl = 350 * time.Millisecond\n\tdefer func() {\n\t\tmqttSessJailDur = mqttSessFlappingJailDur\n\t\tmqttFlapCleanItvl = mqttSessFlappingCleanupInterval\n\t}()\n\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{clientID: \"flapper\", cleanSess: false}\n\tc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Let's get a handle on the asm to check things later.\n\tasm := testMQTTGetAccountSessionManager(t, s, \"flapper\")\n\n\t// Start a new connection with the same clientID, which should replace\n\t// the old one and put it in the flappers map.\n\tc2, r2 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, true)\n\n\t// Should be disconnected...\n\ttestMQTTExpectDisconnect(t, c)\n\n\t// Now try to reconnect \"c\" and we should fail. We have to do this manually,\n\t// since we expect it to fail.\n\taddr := net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port))\n\tc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating mqtt connection: %v\", err)\n\t}\n\tdefer c.Close()\n\tproto := mqttCreateConnectProto(ci)\n\tif _, err := testMQTTWrite(c, proto); err != nil {\n\t\tt.Fatalf(\"Error writing protocols: %v\", err)\n\t}\n\t// Misbehave and send a SUB protocol without waiting for the CONNACK\n\tw := newMQTTWriter(0)\n\tpkLen := 2 // for pi\n\t// Topic \"foo\"\n\tpkLen += 2 + 3 + 1\n\tw.WriteByte(mqttPacketSub | mqttSubscribeFlags)\n\tw.WriteVarInt(pkLen)\n\tw.WriteUint16(1)\n\tw.WriteBytes([]byte(\"foo\"))\n\tw.WriteByte(1)\n\tif _, err := testMQTTWrite(c, w.Bytes()); err != nil {\n\t\tt.Fatalf(\"Error writing protocols: %v\", err)\n\t}\n\t// Now read the CONNACK and we should have been disconnected.\n\tif _, err := testMQTTRead(c); err == nil {\n\t\tt.Fatal(\"Expected connection to fail\")\n\t}\n\n\t// This should be in the flappers map, but after 250ms should be cleared.\n\tfor i := 0; i < 2; i++ {\n\t\tasm.mu.RLock()\n\t\t_, present := asm.flappers[\"flapper\"]\n\t\tasm.mu.RUnlock()\n\t\tif i == 0 {\n\t\t\tif !present {\n\t\t\t\tt.Fatal(\"Did not find the client ID in the flappers map\")\n\t\t\t}\n\t\t\t// Wait for more than the cleanup interval\n\t\t\ttime.Sleep(mqttFlapCleanItvl + 100*time.Millisecond)\n\t\t} else if present {\n\t\t\tt.Fatal(\"The client ID should have been cleared from the map\")\n\t\t}\n\t}\n}\n\nfunc TestMQTTLockedSession(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tsubClientID := \"sub\"\n\tci := &mqttConnInfo{clientID: subClientID, cleanSess: false}\n\tc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tsm := &s.mqtt.sessmgr\n\tsm.mu.Lock()\n\tasm := sm.sessions[globalAccountName]\n\tsm.mu.Unlock()\n\tif asm == nil {\n\t\tt.Fatalf(\"account session manager not found\")\n\t}\n\n\t// It is possible, however unlikely, to have received CONNACK while\n\t// mqttProcessConnect is still running, and the session remains locked. Wait\n\t// for it to finish.\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tasm.mu.RLock()\n\t\tdefer asm.mu.RUnlock()\n\t\tif _, stillLocked := asm.sessLocked[subClientID]; stillLocked {\n\t\t\treturn fmt.Errorf(\"session still locked\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Get the session for \"sub\"\n\tcli := testMQTTGetClient(t, s, subClientID)\n\tsess := cli.mqtt.sess\n\n\t// Pretend that the session above is locked.\n\tif err := asm.lockSession(sess, cli); err != nil {\n\t\tt.Fatalf(\"Unable to lock session: %v\", err)\n\t}\n\tdefer asm.unlockSession(sess)\n\n\t// Now try to connect another client that wants to use \"sub\".\n\t// We can't use testMQTTConnect() because it is going to fail.\n\taddr := net.JoinHostPort(o.MQTT.Host, fmt.Sprintf(\"%d\", o.MQTT.Port))\n\tc2, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating mqtt connection: %v\", err)\n\t}\n\tdefer c2.Close()\n\tproto := mqttCreateConnectProto(ci)\n\tif _, err := testMQTTWrite(c2, proto); err != nil {\n\t\tt.Fatalf(\"Error writing connect: %v\", err)\n\t}\n\tif _, err := testMQTTRead(c2); err == nil {\n\t\tt.Fatal(\"Expected connection to fail\")\n\t}\n\n\t// Now try again, but this time release the session while waiting\n\t// to connect and it should succeed.\n\ttime.AfterFunc(250*time.Millisecond, func() { asm.unlockSession(sess) })\n\tc3, r3 := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c3.Close()\n\ttestMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, true)\n}\n\nfunc TestMQTTPersistRetainedMsg(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, c, r, 1, false, true, \"moo/foo\", 1, []byte(\"foo1\"))\n\ttestMQTTPublish(t, c, r, 1, false, true, \"moo/foo\", 1, []byte(\"foo2\"))\n\ttestMQTTPublish(t, c, r, 1, false, true, \"moo/bar\", 1, []byte(\"bar1\"))\n\ttestMQTTPublish(t, c, r, 0, false, true, \"moo/baz\", 1, []byte(\"baz1\"))\n\t// Remove bar\n\ttestMQTTPublish(t, c, r, 1, false, true, \"moo/bar\", 1, nil)\n\ttestMQTTFlush(t, c, nil, r)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\ts.Shutdown()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tt.Run(\"many subs one topic\", func(t *testing.T) {\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"moo/foo\", qos: 1}}, []byte{1})\n\t\ttestMQTTCheckPubMsg(t, c, r, \"moo/foo\", mqttPubFlagRetain|mqttPubQos1, []byte(\"foo2\"))\n\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"moo/baz\", qos: 1}}, []byte{1})\n\t\ttestMQTTCheckPubMsg(t, c, r, \"moo/baz\", mqttPubFlagRetain, []byte(\"baz1\"))\n\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"moo/bar\", qos: 1}}, []byte{1})\n\t\ttestMQTTExpectNothing(t, r)\n\t})\n\n\trunSingleSubscribe := func(filters []*mqttFilter) func(t *testing.T) {\n\t\treturn func(t *testing.T) {\n\t\t\ttestMQTTSub(t, 1, c, r, filters, nil)\n\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tflags, pi, topic, payload := testMQTTReadPubPacket(t, r)\n\t\t\t\tif (flags & mqttPubFlagRetain) == 0 {\n\t\t\t\t\tt.Fatalf(\"Expected flags to have retain set, got %v\", flags)\n\t\t\t\t}\n\t\t\t\tif (flags & mqttPubFlagDup) != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected flags to not have Dup set, got %v\", flags)\n\t\t\t\t}\n\t\t\t\tvar expQOS byte\n\t\t\t\tvar expPayload []byte\n\t\t\t\tswitch topic {\n\t\t\t\tcase \"moo/foo\":\n\t\t\t\t\texpQOS = mqttPubQos1\n\t\t\t\t\texpPayload = []byte(\"foo2\")\n\t\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\t\t\t\tcase \"moo/baz\":\n\t\t\t\t\texpQOS = 0\n\t\t\t\t\texpPayload = []byte(\"baz1\")\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected topic: %v\", topic)\n\t\t\t\t}\n\t\t\t\tif (flags & mqttPubFlagQoS) != expQOS {\n\t\t\t\t\tt.Fatalf(\"Expected flags to have QOS %x set, got %x\", expQOS, flags)\n\t\t\t\t}\n\t\t\t\tif string(payload) != string(expPayload) {\n\t\t\t\t\tt.Fatalf(\"Expected payload to be %q, got %v\", expPayload, string(payload))\n\t\t\t\t}\n\t\t\t}\n\t\t\ttestMQTTExpectNothing(t, r)\n\t\t}\n\t}\n\n\tt.Run(\"one sub many topics\", runSingleSubscribe([]*mqttFilter{\n\t\t{filter: \"moo/foo\", qos: 1},\n\t\t{filter: \"moo/baz\", qos: 1},\n\t\t{filter: \"moo/bar\", qos: 1},\n\t}))\n\n\tt.Run(\"one sub wildcard plus\", runSingleSubscribe([]*mqttFilter{\n\t\t{filter: \"moo/+\", qos: 1},\n\t}))\n\n\tt.Run(\"one sub wildcard hash\", runSingleSubscribe([]*mqttFilter{\n\t\t{filter: \"moo/#\", qos: 1},\n\t}))\n\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n}\n\nfunc TestMQTTRetainedMsgCleanup(t *testing.T) {\n\tmqttRetainedCacheTTL = 250 * time.Millisecond\n\tdefer func() { mqttRetainedCacheTTL = mqttDefaultRetainedCacheTTL }()\n\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{clientID: \"cache\", cleanSess: true}\n\tc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Send a retained message.\n\ttestMQTTPublish(t, c, r, 1, false, true, \"foo\", 1, []byte(\"msg\"))\n\ttestMQTTFlush(t, c, nil, r)\n\n\t// Start a subscription.\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1|mqttPubFlagRetain, []byte(\"msg\"))\n\n\ttime.Sleep(2 * mqttRetainedCacheTTL)\n\n\t// Make sure not in cache anymore\n\tasm := testMQTTGetAccountSessionManager(t, s, \"cache\")\n\tif v, ok := asm.rmsCache.Load(\"foo\"); ok {\n\t\tt.Fatalf(\"Should not be in cache, got %+v\", v)\n\t}\n}\n\nfunc TestMQTTRestoreRetainedMsgs(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{clientID: \"retain\", cleanSess: true}\n\tc, r := testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t// Send several retained messages on different topic\n\ttestMQTTPublish(t, c, r, 1, false, true, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTPublish(t, c, r, 1, false, true, \"bar\", 1, []byte(\"msg2\"))\n\ttestMQTTPublish(t, c, r, 1, false, true, \"baz\", 1, []byte(\"msg3\"))\n\n\t// Remove the two last ones (by sending empty body)\n\ttestMQTTPublish(t, c, r, 1, false, true, \"bar\", 1, []byte(\"\"))\n\ttestMQTTPublish(t, c, r, 1, false, true, \"baz\", 1, []byte(\"\"))\n\ttestMQTTFlush(t, c, nil, r)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\t// Now restart the server. We had a bug where we would wait to restore retained\n\t// messages based on stream last sequence, which was wrong.\n\ts.Shutdown()\n\ts, err := NewServer(o)\n\trequire_NoError(t, err)\n\tl := &captureWarnLogger{warn: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\ts.Start()\n\tdefer testMQTTShutdownServer(s)\n\n\ttime.Sleep(500 * time.Millisecond)\n\tstart := time.Now()\n\tc, r = testMQTTConnect(t, ci, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\tdur := time.Since(start)\n\tif dur > 2*time.Second {\n\t\tvar warnMsg string\n\t\tselect {\n\t\tcase warnMsg = <-l.warn:\n\t\tdefault:\n\t\t}\n\t\tt.Fatalf(\"Likely timing out restoring retained messages (%s)\", warnMsg)\n\t}\n}\n\nfunc TestMQTTConnAckFirstPacket(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.NoLog, o.Debug, o.Trace = true, false, false\n\ts := RunServer(o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tch := make(chan struct{}, 1)\n\tready := make(chan struct{})\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tclose(ready)\n\t\tfor {\n\t\t\tnc.Publish(\"foo\", []byte(\"msg\"))\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\t<-ready\n\tfor i := 0; i < 100; i++ {\n\t\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\t\tw := newMQTTWriter(0)\n\t\tw.WriteByte(mqttPacketDisconnect)\n\t\tw.WriteByte(0)\n\t\tc.Write(w.Bytes())\n\t\t// Wait to be disconnected, we can't use testMQTTDisconnect() because\n\t\t// it would fail because we may still receive some NATS messages.\n\t\tvar b [10]byte\n\t\tfor {\n\t\t\tif _, err := c.Read(b[:]); err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tc.Close()\n\t}\n\tclose(ch)\n\twg.Wait()\n}\n\nfunc TestMQTTRedeliveryAckWait(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 250 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"foo1\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 2, []byte(\"foo2\"))\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\tfor i := 0; i < 2; i++ {\n\t\tflags := mqttPubQos1\n\t\tif i > 0 {\n\t\t\tflags |= mqttPubFlagDup\n\t\t}\n\t\tpi1 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", flags, []byte(\"foo1\"))\n\t\tpi2 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", flags, []byte(\"foo2\"))\n\n\t\tif pi1 != 1 || pi2 != 2 {\n\t\t\tt.Fatalf(\"Unexpected pi values: %v, %v\", pi1, pi2)\n\t\t}\n\t}\n\t// Ack first message\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, 1)\n\t// Redelivery should only be for second message now\n\tfor i := 0; i < 2; i++ {\n\t\tflags := mqttPubQos1 | mqttPubFlagDup\n\t\tpi := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", flags, []byte(\"foo2\"))\n\t\tif pi != 2 {\n\t\t\tt.Fatalf(\"Unexpected pi to be 2, got %v\", pi)\n\t\t}\n\t}\n\n\t// Restart client, should receive second message with pi==2\n\tc.Close()\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\t// Check that message is received with proper pi\n\tpi := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1|mqttPubFlagDup, []byte(\"foo2\"))\n\tif pi != 2 {\n\t\tt.Fatalf(\"Unexpected pi to be 2, got %v\", pi)\n\t}\n\t// Now ack second message\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, 2)\n\t// Flush to make sure it is processed before checking client's maps\n\ttestMQTTFlush(t, c, nil, r)\n\n\t// Look for the sub client\n\tmc := testMQTTGetClient(t, s, \"sub\")\n\tmc.mu.Lock()\n\tsess := mc.mqtt.sess\n\tsess.mu.Lock()\n\tlpi := len(sess.pendingPublish)\n\tvar lsseq int\n\tfor _, sseqToPi := range sess.cpending {\n\t\tlsseq += len(sseqToPi)\n\t}\n\tsess.mu.Unlock()\n\tmc.mu.Unlock()\n\tif lpi != 0 || lsseq != 0 {\n\t\tt.Fatalf(\"Maps should be empty, got %v, %v\", lpi, lsseq)\n\t}\n}\n\n// - [MQTT-3.10.4-3] If a Server deletes a Subscription It MUST complete the\n// delivery of any QoS 1 or QoS 2 messages which it has started to send to the\n// Client.\n//\n// Test flow:\n// - Subscribe to foo, publish 3 QoS2 messages.\n// - After one is PUBCOMP-ed, and one is PUBREC-ed, Unsubscribe.\n// - See that the remaining 2 are fully delivered.\nfunc TestMQTTQoS2InflightMsgsDeliveredAfterUnsubscribe(t *testing.T) {\n\t// This test has proven flaky on Travis, so skip for now. Line 4926, the 3rd\n\t// testMQTTCheckPubMsgNoAck sometimes returns `data1`, instead of `data3`\n\t// that we are expecting. It must be a retry since we did not acknowledge\n\t// `data1` until later.\n\tt.Skip()\n\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 10 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tvar qos2 byte = 2\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: qos2}}, []byte{qos2})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// send 3 messages\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", 441, []byte(\"data1\"))\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", 442, []byte(\"data2\"))\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", 443, []byte(\"data3\"))\n\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\tsubPI1 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data1\"))\n\tsubPI2 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data2\"))\n\t// subPI3 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data3\"))\n\t_ = testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data3\"))\n\n\t// fully receive first message\n\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI1)\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI1)\n\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI1)\n\n\t// Do not PUBCOMP the 2nd message yet.\n\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI2)\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI2)\n\n\t// Unsubscribe\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: qos2}})\n\n\t// We expect that PI2 and PI3 will continue to be delivered, from their\n\t// respective states.\n\tgotPI2PubRel := false\n\n\t// TODO: Currently, we do not get the unacknowledged PUBLISH re-delivered\n\t// after an UNSUBSCRIBE. Ongoing discussion if we should/must.\n\t// gotPI3Publish := false\n\t// gotPI3PubRel := false\n\tfor !gotPI2PubRel /* || !gotPI3Publish || !gotPI3PubRel */ {\n\t\tb, _ /* len */ := testMQTTReadPacket(t, r)\n\t\tswitch b & mqttPacketMask {\n\t\tcase mqttPacketPubRel:\n\t\t\tpi, err := r.readUint16(\"packet identifier\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"got unexpected error: %v\", err)\n\t\t\t}\n\t\t\tswitch pi {\n\t\t\tcase subPI2:\n\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, pi)\n\t\t\t\tgotPI2PubRel = true\n\t\t\t// case subPI3:\n\t\t\t// \ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, pi)\n\t\t\t// \tgotPI3PubRel = true\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Expected PI %v got: %v\", subPI2, pi)\n\t\t\t}\n\n\t\t// case mqttPacketPub:\n\t\t// \t_, pi, _ := testMQTTGetPubMsgExEx(t, c, r, b, len, \"foo\", []byte(\"data3\"))\n\t\t// \tif pi != subPI3 {\n\t\t// \t\tt.Fatalf(\"Expected PI %v got: %v\", subPI3, pi)\n\t\t// \t}\n\t\t// \tgotPI3Publish = true\n\t\t// \ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI3)\n\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected packet type: %v\", b&mqttPacketMask)\n\t\t}\n\t}\n\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTQoS2RejectPublishDuplicates(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tvar qos2 byte = 2\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: qos2}}, []byte{qos2})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// Publish 3 different with same PI before we get any PUBREC back, then\n\t// complete the PUBREL/PUBCOMP flow as needed. Only one message (first\n\t// payload) should be delivered. PUBRECs,\n\tvar pubPI uint16 = 444\n\ttestMQTTSendPublishPacket(t, cp, qos2, false, false, \"foo\", pubPI, []byte(\"data1\"))\n\ttestMQTTSendPublishPacket(t, cp, qos2, true, false, \"foo\", pubPI, []byte(\"data2\"))\n\ttestMQTTSendPublishPacket(t, cp, qos2, false, false, \"foo\", pubPI, []byte(\"data3\"))\n\n\tfor i := 0; i < 3; i++ {\n\t\t// [MQTT-4.3.3-1] The receiver\n\t\t//\n\t\t// - MUST respond with a PUBREC containing the Packet Identifier from\n\t\t// the incoming PUBLISH Packet, having accepted ownership of the\n\t\t// Application Message.\n\t\t//\n\t\t// - Until it has received the corresponding PUBREL packet, the Receiver\n\t\t// MUST acknowledge any subsequent PUBLISH packet with the same Packet\n\t\t// Identifier by sending a PUBREC. It MUST NOT cause duplicate messages\n\t\t// to be delivered to any onward recipients in this case.\n\t\ttestMQTTReadPIPacket(mqttPacketPubRec, t, rp, pubPI)\n\t}\n\tfor i := 0; i < 3; i++ {\n\t\ttestMQTTSendPIPacket(mqttPacketPubRel|0x2, t, cp, pubPI)\n\t}\n\tfor i := 0; i < 3; i++ {\n\t\t// [MQTT-4.3.3-1] MUST respond to a PUBREL packet by sending a PUBCOMP\n\t\t// packet containing the same Packet Identifier as the PUBREL.\n\t\ttestMQTTReadPIPacket(mqttPacketPubComp, t, rp, pubPI)\n\t}\n\n\t// [MQTT-4.3.3-1] After it has sent a PUBCOMP, the receiver MUST treat any\n\t// subsequent PUBLISH packet that contains that Packet Identifier as being a\n\t// new publication.\n\t//\n\t// Publish another message, identical to the first one. Since the server\n\t// already sent us a PUBCOMP, it will deliver this message, for a total of 2\n\t// delivered.\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", pubPI, []byte(\"data5\"))\n\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\t// Verify we got a total of 2 messages.\n\tsubPI1 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data1\"))\n\tsubPI2 := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data5\"))\n\tfor _, pi := range []uint16{subPI1, subPI2} {\n\t\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, pi)\n\t\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, pi)\n\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, pi)\n\t}\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTQoS2RetriesPublish(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 100 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tvar qos2 byte = 2\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: qos2}}, []byte{qos2})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// Publish a message and close the pub connection.\n\tvar pubPI uint16 = 444\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", pubPI, []byte(\"data1\"))\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\t// See that we got the message delivered to the sub, but don't PUBREC it\n\t// yet.\n\tsubPI := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data1\"))\n\n\t// See that the message is redelivered again 2 times, with the DUP on, before we PUBREC it.\n\tfor i := 0; i < 2; i++ {\n\t\texpectedFlags := mqttPubQoS2 | mqttPubFlagDup\n\t\tpi := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", expectedFlags, []byte(\"data1\"))\n\t\tif pi != subPI {\n\t\t\tt.Fatalf(\"Expected pi to be %v, got %v\", subPI, pi)\n\t\t}\n\t}\n\n\t// Finish the exchange and make sure there are no more attempts.\n\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI)\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI)\n\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI)\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTQoS2RetriesPubRel(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 50 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tvar qos2 byte = 2\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: qos2}}, []byte{qos2})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t// Publish a message and close the pub connection.\n\tvar pubPI uint16 = 444\n\ttestMQTTPublish(t, cp, rp, qos2, false, false, \"foo\", pubPI, []byte(\"data1\"))\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\t// See that we got the message delivered to the sub, PUBREC it and expect a\n\t// PUBREL from the server.\n\tsubPI := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQoS2, []byte(\"data1\"))\n\ttestMQTTSendPIPacket(mqttPacketPubRec, t, c, subPI)\n\n\t// See that we get PUBREL redelivered several times, there's no DUP flag to\n\t// check.\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI)\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI)\n\ttestMQTTReadPIPacket(mqttPacketPubRel, t, r, subPI)\n\n\t// Finish the exchange and make sure there are no more attempts.\n\ttestMQTTSendPIPacket(mqttPacketPubComp, t, c, subPI)\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTAckWaitConfigChange(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 250 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tsendMsg := func(topic, payload string) {\n\t\tt.Helper()\n\t\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\t\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer cp.Close()\n\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t\ttestMQTTPublish(t, cp, rp, 1, false, false, topic, 1, []byte(payload))\n\t\ttestMQTTDisconnect(t, cp, nil)\n\t\tcp.Close()\n\t}\n\tsendMsg(\"foo\", \"msg1\")\n\n\tfor i := 0; i < 2; i++ {\n\t\tflags := mqttPubQos1\n\t\tif i > 0 {\n\t\t\tflags |= mqttPubFlagDup\n\t\t}\n\t\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", flags, []byte(\"msg1\"))\n\t}\n\n\t// Restart the server with a different AckWait option value.\n\t// Verify that MQTT sub restart succeeds. It will keep the\n\t// original value.\n\tc.Close()\n\ts.Shutdown()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.MQTT.AckWait = 10 * time.Millisecond\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// There is already the defer for shutdown at top of function\n\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1|mqttPubFlagDup, []byte(\"msg1\"))\n\tstart := time.Now()\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1|mqttPubFlagDup, []byte(\"msg1\"))\n\tif dur := time.Since(start); dur < 200*time.Millisecond {\n\t\tt.Fatalf(\"AckWait seem to have changed for existing subscription: %v\", dur)\n\t}\n\n\t// Create new subscription\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\tsendMsg(\"bar\", \"msg2\")\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"bar\", mqttPubQos1, []byte(\"msg2\"))\n\tstart = time.Now()\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"bar\", mqttPubQos1|mqttPubFlagDup, []byte(\"msg2\"))\n\tif dur := time.Since(start); dur > 50*time.Millisecond {\n\t\tt.Fatalf(\"AckWait new value not used by new sub: %v\", dur)\n\t}\n\tc.Close()\n}\n\nfunc TestMQTTUnsubscribeWithPendingAcks(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = 250 * time.Millisecond\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg\"))\n\ttestMQTTDisconnect(t, cp, nil)\n\tcp.Close()\n\n\tfor i := 0; i < 2; i++ {\n\t\tflags := mqttPubQos1\n\t\tif i > 0 {\n\t\t\tflags |= mqttPubFlagDup\n\t\t}\n\t\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", flags, []byte(\"msg\"))\n\t}\n\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"foo\"}})\n\ttestMQTTFlush(t, c, nil, r)\n\n\tmc := testMQTTGetClient(t, s, \"sub\")\n\tmc.mu.Lock()\n\tsess := mc.mqtt.sess\n\tsess.mu.Lock()\n\tpal := len(sess.pendingPublish)\n\tsess.mu.Unlock()\n\tmc.mu.Unlock()\n\tif pal != 0 {\n\t\tt.Fatalf(\"Expected pending ack map to be empty, got %v\", pal)\n\t}\n}\n\nfunc TestMQTTMaxAckPending(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.MaxAckPending = 1\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownRestartedServer(&s)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg2\"))\n\n\tpi := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n\t// Check that we don't receive the second one due to max ack pending\n\ttestMQTTExpectNothing(t, r)\n\n\t// Now ack first message\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\t// Now we should receive message 2\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg2\"))\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\t// Give a chance to the server to \"close\" the consumer\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tfor ci := range js.ConsumersInfo(\"$MQTT_msgs\") {\n\t\t\tif ci.PushBound {\n\t\t\t\treturn fmt.Errorf(\"Consumer still connected\")\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Send 2 messages while sub is offline\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg3\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg4\"))\n\n\t// Restart consumer\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\t// Should receive only message 3\n\tpi = testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg3\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Ack and get the next\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg4\"))\n\n\t// Make sure this message gets ack'ed\n\tmcli := testMQTTGetClient(t, s, cisub.clientID)\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tmcli.mu.Lock()\n\t\tsess := mcli.mqtt.sess\n\t\tsess.mu.Lock()\n\t\tnp := len(sess.pendingPublish)\n\t\tsess.mu.Unlock()\n\t\tmcli.mu.Unlock()\n\t\tif np != 0 {\n\t\t\treturn fmt.Errorf(\"Still %v pending messages\", np)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check that change to config does not prevent restart of sub.\n\tcp.Close()\n\tc.Close()\n\ts.Shutdown()\n\n\to.Port = -1\n\to.MQTT.Port = -1\n\to.MQTT.MaxAckPending = 2\n\to.StoreDir = dir\n\ts = testMQTTRunServer(t, o)\n\t// There is already the defer for shutdown at top of function\n\n\tcp, rp = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg5\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg6\"))\n\n\t// Restart consumer\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, true)\n\n\t// Should receive only message 5\n\tpi = testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg5\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Ack and get the next\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg6\"))\n}\n\nfunc TestMQTTMaxAckPendingForMultipleSubs(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.AckWait = time.Second\n\to.MQTT.MaxAckPending = 1\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: true}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\tpi := testMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n\n\t// Now send a second message but on topic bar\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"bar\", 1, []byte(\"msg2\"))\n\n\t// JS allows us to limit per consumer, but we apply the limit to the\n\t// session, so although JS will attempt to delivery this message,\n\t// the MQTT code will suppress it.\n\ttestMQTTExpectNothing(t, r)\n\n\t// Ack the first message.\n\ttestMQTTSendPIPacket(mqttPacketPubAck, t, c, pi)\n\n\t// Now we should get the second message\n\ttestMQTTCheckPubMsg(t, c, r, \"bar\", mqttPubQos1|mqttPubFlagDup, []byte(\"msg2\"))\n}\n\nfunc TestMQTTMaxAckPendingOverLimit(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\to.MQTT.MaxAckPending = 20000\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcheckTMax := func(sess *mqttSession, expected int) {\n\t\tt.Helper()\n\t\tsess.mu.Lock()\n\t\ttmax := sess.tmaxack\n\t\tsess.mu.Unlock()\n\t\tif tmax != expected {\n\t\t\tt.Fatalf(\"Expected current tmax to be %v, got %v\", expected, tmax)\n\t\t}\n\t}\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tmc := testMQTTGetClient(t, s, \"sub\")\n\tmc.mu.Lock()\n\tsess := mc.mqtt.sess\n\tmc.mu.Unlock()\n\n\t// After this one, total would be 20000\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\tcheckTMax(sess, 20000)\n\t// This one will count for 2, so total will be 60000\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar/#\", qos: 1}}, []byte{1})\n\tcheckTMax(sess, 60000)\n\n\t// This should fail\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{mqttSubAckFailure})\n\tcheckTMax(sess, 60000)\n\n\t// Remove the one with wildcard\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"bar/#\"}})\n\tcheckTMax(sess, 20000)\n\n\t// Now we could add 2 more without wildcards\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\tcheckTMax(sess, 40000)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"baz\", qos: 1}}, []byte{1})\n\tcheckTMax(sess, 60000)\n\n\t// Again, this one should fail\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bat\", qos: 1}}, []byte{mqttSubAckFailure})\n\tcheckTMax(sess, 60000)\n\n\t// Now remove all and check that we are at 0\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"foo\"}})\n\tcheckTMax(sess, 40000)\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"bar\"}})\n\tcheckTMax(sess, 20000)\n\ttestMQTTUnsub(t, 1, c, r, []*mqttFilter{{filter: \"baz\"}})\n\tcheckTMax(sess, 0)\n}\n\nfunc TestMQTTConfigReload(t *testing.T) {\n\ttemplate := `\n\t\tjetstream: true\n\t\tserver_name: mqtt\n\t\tmqtt {\n\t\t\tport: -1\n\t\t\tack_wait: %s\n\t\t\tmax_ack_pending: %s\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, `\"5s\"`, `10000`)))\n\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tif val := o.MQTT.AckWait; val != 5*time.Second {\n\t\tt.Fatalf(\"Invalid ackwait: %v\", val)\n\t}\n\tif val := o.MQTT.MaxAckPending; val != 10000 {\n\t\tt.Fatalf(\"Invalid ackwait: %v\", val)\n\t}\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `\"250ms\"`, `1`)))\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg2\"))\n\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n\tstart := time.Now()\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1|mqttPubFlagDup, []byte(\"msg1\"))\n\tif dur := time.Since(start); dur > 500*time.Millisecond {\n\t\tt.Fatalf(\"AckWait not applied? dur=%v\", dur)\n\t}\n\tc.Close()\n\tcp.Close()\n\ttestMQTTShutdownServer(s)\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `\"30s\"`, `1`)))\n\ts, o = RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub.cleanSess = true\n\tc, r = testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tcipub = &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tcp, rp = testMQTTConnect(t, cipub, o.MQTT.Host, o.MQTT.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg2\"))\n\n\ttestMQTTCheckPubMsgNoAck(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n\ttestMQTTExpectNothing(t, r)\n\n\t// Increase the max ack pending\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, `\"30s\"`, `10`)))\n\t// Reload now\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\t// Reload will have effect only on new subscriptions.\n\t// Create a new subscription, and we should not be able to get the 2 messages.\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"bar\", 1, []byte(\"msg3\"))\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"bar\", 1, []byte(\"msg4\"))\n\n\ttestMQTTCheckPubMsg(t, c, r, \"bar\", mqttPubQos1, []byte(\"msg3\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"bar\", mqttPubQos1, []byte(\"msg4\"))\n}\n\nfunc TestMQTTStreamInfoReturnsNonEmptySubject(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\t// Check that we can query all MQTT streams. MQTT streams are\n\t// created without subject filter, however, if we return them like this,\n\t// the 'nats' utility will fail to display them due to some xml validation.\n\tfor _, sname := range []string{\n\t\tmqttStreamName,\n\t\tmqttRetainedMsgsStreamName,\n\t} {\n\t\tt.Run(sname, func(t *testing.T) {\n\t\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, sname), nil, time.Second)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tvar bResp JSApiStreamInfoResponse\n\t\t\tif err = json.Unmarshal(resp.Data, &bResp); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif len(bResp.Config.Subjects) == 0 {\n\t\t\t\tt.Fatalf(\"No subject returned, which will cause nats tooling to fail: %+v\", bResp.Config)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTWebsocketToMQTTPort(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tci := &mqttConnInfo{cleanSess: true, ws: true}\n\tif _, _, err := testMQTTConnectRetryWithError(t, ci, o.MQTT.Host, o.MQTT.Port, 0); err == nil {\n\t\tt.Fatal(\"Expected error during connect\")\n\t}\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif !strings.Contains(e, errMQTTNotWebsocketPort.Error()) {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"No error regarding wrong port\")\n\t}\n}\n\nfunc TestMQTTWebsocket(t *testing.T) {\n\ttdir := t.TempDir()\n\ttemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\taccounts {\n\t\t\tMQTT {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers [\n\t\t\t\t\t{user: \"mqtt\", pass: \"pwd\", connection_types: [\"%s\"%s]}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t`\n\ts, o, conf := runReloadServerWithContent(t, []byte(fmt.Sprintf(template, tdir, jwt.ConnectionTypeMqtt, \"\")))\n\tdefer testMQTTShutdownServer(s)\n\n\tcisub := &mqttConnInfo{clientID: \"sub\", user: \"mqtt\", pass: \"pwd\", ws: true}\n\tc, r := testMQTTConnect(t, cisub, o.Websocket.Host, o.Websocket.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCNotAuthorized, false)\n\tc.Close()\n\n\tws := fmt.Sprintf(`, \"%s\"`, jwt.ConnectionTypeMqttWS)\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template, tdir, jwt.ConnectionTypeMqtt, ws))\n\n\tcisub = &mqttConnInfo{clientID: \"sub\", user: \"mqtt\", pass: \"pwd\", ws: true}\n\tc, r = testMQTTConnect(t, cisub, o.Websocket.Host, o.Websocket.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c, nil, r)\n\n\tcipub := &mqttConnInfo{clientID: \"pub\", user: \"mqtt\", pass: \"pwd\", ws: true}\n\tcp, rp := testMQTTConnect(t, cipub, o.Websocket.Host, o.Websocket.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n}\n\ntype chunkWriteConn struct {\n\tnet.Conn\n}\n\nfunc (cwc *chunkWriteConn) Write(p []byte) (int, error) {\n\tmax := len(p)\n\tcs := rand.Intn(max) + 1\n\tif cs < max {\n\t\tif pn, perr := cwc.Conn.Write(p[:cs]); perr != nil {\n\t\t\treturn pn, perr\n\t\t}\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tif pn, perr := cwc.Conn.Write(p[cs:]); perr != nil {\n\t\t\treturn pn, perr\n\t\t}\n\t\treturn len(p), nil\n\t}\n\treturn cwc.Conn.Write(p)\n}\n\nfunc TestMQTTPartial(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tws bool\n\t}{\n\t\t{\"standard\", false},\n\t\t{\"websocket\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tci := &mqttConnInfo{cleanSess: true, ws: test.ws}\n\t\t\thost, port := o.MQTT.Host, o.MQTT.Port\n\t\t\tif test.ws {\n\t\t\t\thost, port = o.Websocket.Host, o.Websocket.Port\n\t\t\t}\n\t\t\tc, r := testMQTTConnect(t, ci, host, port)\n\t\t\tdefer c.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\tc = &chunkWriteConn{Conn: c}\n\n\t\t\tcp, rp := testMQTTConnect(t, ci, host, port)\n\t\t\tdefer cp.Close()\n\t\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\t\t\tcp = &chunkWriteConn{Conn: cp}\n\n\t\t\tsubj := nuid.Next()\n\t\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: subj, qos: 1}}, []byte{1})\n\t\t\ttestMQTTFlush(t, c, nil, r)\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\ttestMQTTPublish(t, cp, rp, 1, false, false, subj, 1, []byte(\"msg\"))\n\t\t\t\ttestMQTTCheckPubMsg(t, c, r, subj, mqttPubQos1, []byte(\"msg\"))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTWebsocketTLS(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: '../test/configs/certs/server-cert.pem'\n\t\t\t\tkey_file: '../test/configs/certs/server-key.pem'\n\t\t\t\tca_file: '../test/configs/certs/ca.pem'\n\t\t\t}\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", ws: true, tls: true}, o.Websocket.Host, o.Websocket.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c, nil, r)\n\n\tcp, rp := testMQTTConnect(t, &mqttConnInfo{clientID: \"pub\", ws: true, tls: true}, o.Websocket.Host, o.Websocket.Port)\n\tdefer cp.Close()\n\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, cp, rp, 1, false, false, \"foo\", 1, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubQos1, []byte(\"msg1\"))\n}\n\nfunc TestMQTTTransferSessionStreamsToMuxed(t *testing.T) {\n\tcl := createJetStreamClusterWithTemplate(t, testMQTTGetClusterTemplaceNoLeaf(), \"MQTT\", 3)\n\tdefer cl.shutdown()\n\n\tnc, js := jsClientConnect(t, cl.randomServer())\n\tdefer nc.Close()\n\n\t// Create 2 streams that start with \"$MQTT_sess_\" to check for transfer to new\n\t// mux'ed unique \"$MQTT_sess\" stream. One of this stream will not contain a\n\t// proper session record, and we will check that the stream does not get deleted.\n\tsessStreamName1 := mqttSessionsStreamNamePrefix + getHash(\"sub\")\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: sessStreamName1,\n\t\tSubjects: []string{sessStreamName1},\n\t\tReplicas: 3,\n\t\tMaxMsgs: 1,\n\t}); err != nil {\n\t\tt.Fatalf(\"Unable to add stream: %v\", err)\n\t}\n\t// Then add the session record\n\tps := mqttPersistedSession{\n\t\tID: \"sub\",\n\t\tSubs: map[string]byte{\"foo\": 1},\n\t\tCons: map[string]*ConsumerConfig{\"foo\": {\n\t\t\tDurable: \"d6INCtp3_cK39H5WHEtOSU7sLy2oQv3\",\n\t\t\tDeliverSubject: \"$MQTT.sub.cK39H5WHEtOSU7sLy2oQrR\",\n\t\t\tDeliverPolicy: DeliverNew,\n\t\t\tAckPolicy: AckExplicit,\n\t\t\tFilterSubject: \"$MQTT.msgs.foo\",\n\t\t\tMaxAckPending: 1024,\n\t\t}},\n\t}\n\tb, _ := json.Marshal(&ps)\n\tif _, err := js.Publish(sessStreamName1, b); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\n\t// Create the stream that has \"$MQTT_sess_\" prefix, but that is not really a MQTT session stream\n\tsessStreamName2 := mqttSessionsStreamNamePrefix + \"ivan\"\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: sessStreamName2,\n\t\tSubjects: []string{sessStreamName2},\n\t\tReplicas: 3,\n\t\tMaxMsgs: 1,\n\t}); err != nil {\n\t\tt.Fatalf(\"Unable to add stream: %v\", err)\n\t}\n\tif _, err := js.Publish(sessStreamName2, []byte(\"some content\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\n\tcl.waitOnStreamLeader(globalAccountName, sessStreamName1)\n\tcl.waitOnStreamLeader(globalAccountName, sessStreamName2)\n\n\t// Now create a real MQTT connection\n\to := cl.opts[0]\n\tsc, sr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub\"}, o.MQTT.Host, o.MQTT.Port, 10)\n\tdefer sc.Close()\n\ttestMQTTCheckConnAck(t, sr, mqttConnAckRCConnectionAccepted, true)\n\n\t// Check that old session stream is gone, but the non session stream is still present.\n\tvar gotIt = false\n\tfor info := range js.StreamsInfo() {\n\t\tif strings.HasPrefix(info.Config.Name, mqttSessionsStreamNamePrefix) {\n\t\t\tif strings.HasSuffix(info.Config.Name, \"_ivan\") {\n\t\t\t\tgotIt = true\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"The stream %q should have been deleted\", info.Config.Name)\n\t\t\t}\n\t\t}\n\t}\n\tif !gotIt {\n\t\tt.Fatalf(\"The stream %q should not have been deleted\", mqttSessionsStreamNamePrefix+\"ivan\")\n\t}\n\n\t// We want to check that the record was properly transferred.\n\trmsg, err := js.GetMsg(mqttSessStreamName, 2)\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to get session message: %v\", err)\n\t}\n\tps2 := &mqttPersistedSession{}\n\tif err := json.Unmarshal(rmsg.Data, ps2); err != nil {\n\t\tt.Fatalf(\"Error unpacking session record: %v\", err)\n\t}\n\tif ps2.ID != \"sub\" {\n\t\tt.Fatalf(\"Unexpected session record, %+v vs %+v\", ps2, ps)\n\t}\n\tif qos, ok := ps2.Subs[\"foo\"]; !ok || qos != 1 {\n\t\tt.Fatalf(\"Unexpected session record, %+v vs %+v\", ps2, ps)\n\t}\n\tif cons, ok := ps2.Cons[\"foo\"]; !ok || !reflect.DeepEqual(cons, ps.Cons[\"foo\"]) {\n\t\tt.Fatalf(\"Unexpected session record, %+v vs %+v\", ps2, ps)\n\t}\n\n\t// Make sure we don't attempt to transfer again by creating a subscription\n\t// on the \"stream names\" API, which is used to get the list of streams to transfer\n\tsub := natsSubSync(t, nc, JSApiStreams)\n\n\t// Make sure to connect an MQTT client from a different node so that this node\n\t// gets a connection for the account for the first time and tries to create\n\t// all MQTT streams, etc..\n\to = cl.opts[1]\n\tsc, sr = testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"sub2\"}, o.MQTT.Host, o.MQTT.Port, 10)\n\tdefer sc.Close()\n\ttestMQTTCheckConnAck(t, sr, mqttConnAckRCConnectionAccepted, false)\n\n\tif _, err := sub.NextMsg(200 * time.Millisecond); err == nil {\n\t\tt.Fatal(\"Looks like attempt to transfer was done again\")\n\t}\n}\n\nfunc TestMQTTConnectAndDisconnectEvent(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\thttp: \"127.0.0.1:-1\"\n\t\tserver_name: \"mqtt\"\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\taccounts {\n\t\t\tMQTT {\n\t\t\t\tjetstream: enabled\n\t\t\t\tusers: [{user: \"mqtt\", password: \"pwd\"}]\n\t\t\t}\n\t\t\tSYS {\n\t\t\t\tusers: [{user: \"sys\", password: \"pwd\"}]\n\t\t\t}\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`, t.TempDir())))\n\tdefer os.Remove(conf)\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"sys\", \"pwd\"))\n\tdefer nc.Close()\n\n\taccConn := natsSubSync(t, nc, fmt.Sprintf(connectEventSubj, \"MQTT\"))\n\taccDisc := natsSubSync(t, nc, fmt.Sprintf(disconnectEventSubj, \"MQTT\"))\n\taccAuth := natsSubSync(t, nc, fmt.Sprintf(authErrorEventSubj, s.ID()))\n\tnatsFlush(t, nc)\n\n\tcheckConnEvent := func(data []byte, expected string) {\n\t\tt.Helper()\n\t\tvar ce ConnectEventMsg\n\t\tjson.Unmarshal(data, &ce)\n\t\tif ce.Client.MQTTClient != expected {\n\t\t\tt.Fatalf(\"Expected client ID %q, got this connect event: %+v\", expected, ce)\n\t\t}\n\t}\n\tcheckDiscEvent := func(data []byte, expected string) {\n\t\tt.Helper()\n\t\tvar de DisconnectEventMsg\n\t\tjson.Unmarshal(data, &de)\n\t\tif de.Client.MQTTClient != expected {\n\t\t\tt.Fatalf(\"Expected client ID %q, got this disconnect event: %+v\", expected, de)\n\t\t}\n\t}\n\n\tc1, r1 := testMQTTConnect(t, &mqttConnInfo{user: \"mqtt\", pass: \"pwd\", clientID: \"conn1\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c1.Close()\n\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\n\tcm := natsNexMsg(t, accConn, time.Second)\n\tcheckConnEvent(cm.Data, \"conn1\")\n\n\tc2, r2 := testMQTTConnect(t, &mqttConnInfo{user: \"mqtt\", pass: \"pwd\", clientID: \"conn2\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\n\tcm = natsNexMsg(t, accConn, time.Second)\n\tcheckConnEvent(cm.Data, \"conn2\")\n\n\tc3, r3 := testMQTTConnect(t, &mqttConnInfo{user: \"mqtt\", pass: \"pwd\", clientID: \"conn3\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c3.Close()\n\ttestMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false)\n\n\tcm = natsNexMsg(t, accConn, time.Second)\n\tcheckConnEvent(cm.Data, \"conn3\")\n\n\ttestMQTTDisconnect(t, c3, nil)\n\tc3.Close()\n\tcm = natsNexMsg(t, accDisc, time.Second)\n\tcheckDiscEvent(cm.Data, \"conn3\")\n\n\t// Now try a bad auth\n\tc4, r4 := testMQTTConnect(t, &mqttConnInfo{clientID: \"conn4\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c4.Close()\n\ttestMQTTCheckConnAck(t, r4, mqttConnAckRCNotAuthorized, false)\n\t// This will generate an auth error, which is a disconnect event\n\tcm = natsNexMsg(t, accAuth, time.Second)\n\tcheckDiscEvent(cm.Data, \"conn4\")\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", s.MonitorAddr().Port)\n\tfor mode := 0; mode < 2; mode++ {\n\t\tc := pollConnz(t, s, mode, url+\"connz\", nil)\n\t\tif c.Conns == nil || len(c.Conns) != 3 {\n\t\t\tt.Fatalf(\"Expected 3 connections in array, got %v\", len(c.Conns))\n\t\t}\n\n\t\t// Check that client ID is present\n\t\tfor _, conn := range c.Conns {\n\t\t\tif conn.Type == clientTypeStringMap[MQTT] && conn.MQTTClient == _EMPTY_ {\n\t\t\t\tt.Fatalf(\"Expected a client ID to be set, got %+v\", conn)\n\t\t\t}\n\t\t}\n\n\t\t// Check that we can select based on client ID:\n\t\tc = pollConnz(t, s, mode, url+\"connz?mqtt_client=conn2\", &ConnzOptions{MQTTClient: \"conn2\"})\n\t\tif c.Conns == nil || len(c.Conns) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 connection in array, got %v\", len(c.Conns))\n\t\t}\n\t\tif c.Conns[0].MQTTClient != \"conn2\" {\n\t\t\tt.Fatalf(\"Unexpected client ID: %+v\", c.Conns[0])\n\t\t}\n\n\t\t// Check that we have the closed ones\n\t\tc = pollConnz(t, s, mode, url+\"connz?state=closed\", &ConnzOptions{State: ConnClosed})\n\t\tif c.Conns == nil || len(c.Conns) != 2 {\n\t\t\tt.Fatalf(\"Expected 2 connections in array, got %v\", len(c.Conns))\n\t\t}\n\t\tfor _, conn := range c.Conns {\n\t\t\tif conn.MQTTClient == _EMPTY_ {\n\t\t\t\tt.Fatalf(\"Expected a client ID, got %+v\", conn)\n\t\t\t}\n\t\t}\n\n\t\t// Check that we can select with client ID for closed state\n\t\tc = pollConnz(t, s, mode, url+\"connz?state=closed&mqtt_client=conn3\", &ConnzOptions{State: ConnClosed, MQTTClient: \"conn3\"})\n\t\tif c.Conns == nil || len(c.Conns) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 connection in array, got %v\", len(c.Conns))\n\t\t}\n\t\tif c.Conns[0].MQTTClient != \"conn3\" {\n\t\t\tt.Fatalf(\"Unexpected client ID: %+v\", c.Conns[0])\n\t\t}\n\t\t// Check that we can select with client ID for closed state (but in this case not found)\n\t\tc = pollConnz(t, s, mode, url+\"connz?state=closed&mqtt_client=conn5\", &ConnzOptions{State: ConnClosed, MQTTClient: \"conn5\"})\n\t\tif len(c.Conns) != 0 {\n\t\t\tt.Fatalf(\"Expected 0 connection in array, got %v\", len(c.Conns))\n\t\t}\n\t}\n\n\treply := nc.NewRespInbox()\n\treplySub := natsSubSync(t, nc, reply)\n\n\t// Test system events now\n\tfor _, test := range []struct {\n\t\topt any\n\t\tcid string\n\t}{\n\t\t{&ConnzOptions{MQTTClient: \"conn1\"}, \"conn1\"},\n\t\t{&ConnzOptions{MQTTClient: \"conn3\", State: ConnClosed}, \"conn3\"},\n\t\t{&ConnzOptions{MQTTClient: \"conn4\", State: ConnClosed}, \"conn4\"},\n\t\t{&ConnzOptions{MQTTClient: \"conn5\"}, _EMPTY_},\n\t\t{json.RawMessage(`{\"mqtt_client\":\"conn1\"}`), \"conn1\"},\n\t\t{json.RawMessage(fmt.Sprintf(`{\"mqtt_client\":\"conn3\", \"state\":%v}`, ConnClosed)), \"conn3\"},\n\t\t{json.RawMessage(fmt.Sprintf(`{\"mqtt_client\":\"conn4\", \"state\":%v}`, ConnClosed)), \"conn4\"},\n\t\t{json.RawMessage(`{\"mqtt_client\":\"conn5\"}`), _EMPTY_},\n\t} {\n\t\tt.Run(\"sys connz\", func(t *testing.T) {\n\t\t\tb, _ := json.Marshal(test.opt)\n\n\t\t\t// set a header to make sure request parsing knows to ignore them\n\t\t\tnc.PublishMsg(&nats.Msg{\n\t\t\t\tSubject: fmt.Sprintf(\"%s.CONNZ\", serverStatsPingReqSubj),\n\t\t\t\tReply: reply,\n\t\t\t\tData: b,\n\t\t\t})\n\n\t\t\tmsg := natsNexMsg(t, replySub, time.Second)\n\t\t\tvar response ServerAPIResponse\n\t\t\tif err := json.Unmarshal(msg.Data, &response); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling response json: %v\", err)\n\t\t\t}\n\t\t\ttmp, _ := json.Marshal(response.Data)\n\t\t\tcz := &Connz{}\n\t\t\tif err := json.Unmarshal(tmp, cz); err != nil {\n\t\t\t\tt.Fatalf(\"Error unmarshalling connz: %v\", err)\n\t\t\t}\n\t\t\tif test.cid == _EMPTY_ {\n\t\t\t\tif len(cz.Conns) != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected no connections, got %v\", len(cz.Conns))\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif len(cz.Conns) != 1 {\n\t\t\t\tt.Fatalf(\"Expected single connection, got %v\", len(cz.Conns))\n\t\t\t}\n\t\t\tconn := cz.Conns[0]\n\t\t\tif conn.MQTTClient != test.cid {\n\t\t\t\tt.Fatalf(\"Expected client ID %q, got %q\", test.cid, conn.MQTTClient)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMQTTClientIDInLogStatements(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tl := &captureDebugLogger{dbgCh: make(chan string, 10)}\n\ts.SetLogger(l, true, false)\n\n\tcisub := &mqttConnInfo{clientID: \"my_client_id\", cleanSess: false}\n\tc, r := testMQTTConnect(t, cisub, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTDisconnect(t, c, nil)\n\tc.Close()\n\n\ttm := time.NewTimer(2 * time.Second)\n\tvar connected bool\n\tvar disconnected bool\n\tfor {\n\t\tselect {\n\t\tcase dl := <-l.dbgCh:\n\t\t\tif strings.Contains(dl, \"my_client_id\") {\n\t\t\t\tif strings.Contains(dl, \"Client connected\") {\n\t\t\t\t\tconnected = true\n\t\t\t\t} else if strings.Contains(dl, \"Client connection closed\") {\n\t\t\t\t\tdisconnected = true\n\t\t\t\t}\n\t\t\t\tif connected && disconnected {\n\t\t\t\t\t// OK!\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-tm.C:\n\t\t\tt.Fatal(\"Did not get the debug statements or client_id in them\")\n\t\t}\n\t}\n}\n\nfunc TestMQTTStreamReplicasOverride(t *testing.T) {\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tstream_replicas: 3\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tcl := createJetStreamClusterWithTemplate(t, conf, \"MQTT\", 3)\n\tdefer cl.shutdown()\n\n\tconnectAndCheck := func(restarted bool) {\n\t\tt.Helper()\n\n\t\to := cl.opts[0]\n\t\tmc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"test\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, restarted)\n\n\t\tnc, js := jsClientConnect(t, cl.servers[2])\n\t\tdefer nc.Close()\n\n\t\tstreams := []string{mqttStreamName, mqttRetainedMsgsStreamName, mqttSessStreamName}\n\t\tfor _, sn := range streams {\n\t\t\tsi, err := js.StreamInfo(sn)\n\t\t\trequire_NoError(t, err)\n\t\t\tif n := len(si.Cluster.Replicas); n != 2 {\n\t\t\t\tt.Fatalf(\"Expected stream %q to have 2 replicas, got %v\", sn, n)\n\t\t\t}\n\t\t}\n\t}\n\tconnectAndCheck(false)\n\n\tcl.stopAll()\n\tfor _, o := range cl.opts {\n\t\to.MQTT.StreamReplicas = 2\n\t}\n\tcl.restartAllSamePorts()\n\tcl.waitOnStreamLeader(globalAccountName, mqttStreamName)\n\tcl.waitOnStreamLeader(globalAccountName, mqttRetainedMsgsStreamName)\n\tcl.waitOnStreamLeader(globalAccountName, mqttSessStreamName)\n\n\tl := &captureWarnLogger{warn: make(chan string, 10)}\n\tcl.servers[0].SetLogger(l, false, false)\n\n\tconnectAndCheck(true)\n\n\tselect {\n\tcase w := <-l.warn:\n\t\tif !strings.Contains(w, \"current is 3 but configuration is 2\") {\n\t\t\tt.Fatalf(\"Unexpected warning: %q\", w)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Should have warned against replicas mismatch\")\n\t}\n}\n\nfunc TestMQTTStreamReplicasConfigReload(t *testing.T) {\n\ttdir := t.TempDir()\n\ttmpl := `\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tserver_name: mqtt\n\t\tmqtt {\n\t\t\tport: -1\n\t\t\tstream_replicas: %v\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, 3)))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\t_, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{clientID: \"mqtt\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 0)\n\tif err == nil {\n\t\tt.Fatal(\"Expected to fail, did not\")\n\t}\n\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif !strings.Contains(e, NewJSStreamReplicasNotSupportedError().Description) {\n\t\t\tt.Fatalf(\"Expected error regarding replicas, got %v\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get the error regarding replicas count\")\n\t}\n\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, 1))\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"mqtt\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n}\n\nfunc TestMQTTStreamReplicasInsufficientResources(t *testing.T) {\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tstream_replicas: 5\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tcl := createJetStreamClusterWithTemplate(t, conf, \"MQTT\", 3)\n\tdefer cl.shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\tfor _, s := range cl.servers {\n\t\ts.SetLogger(l, false, false)\n\t}\n\n\to := cl.opts[1]\n\t_, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{clientID: \"mqtt\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 0)\n\tif err == nil {\n\t\tt.Fatal(\"Expected to fail, did not\")\n\t}\n\n\tselect {\n\tcase e := <-l.errCh:\n\t\tif !strings.Contains(e, fmt.Sprintf(\"%d\", NewJSClusterNoPeersError(errors.New(\"\")).ErrCode)) {\n\t\t\tt.Fatalf(\"Expected error regarding no peers error, got %v\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get the error regarding replicas count\")\n\t}\n}\n\nfunc TestMQTTConsumerReplicasValidate(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsr int\n\t\tcr int\n\t\terr bool\n\t}{\n\t\t{\"stream replicas neg\", -1, 3, false},\n\t\t{\"stream replicas 0\", 0, 3, false},\n\t\t{\"consumer replicas neg\", 0, -1, false},\n\t\t{\"consumer replicas 0\", -1, 0, false},\n\t\t{\"consumer replicas too high\", 1, 2, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to.MQTT.StreamReplicas = test.sr\n\t\t\to.MQTT.ConsumerReplicas = test.cr\n\t\t\terr := validateMQTTOptions(o)\n\t\t\tif test.err {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected error, did not get one\")\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(err.Error(), \"cannot be higher\") {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t// OK\n\t\t\t\treturn\n\t\t\t} else if err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// This was ill-advised since the messages stream is currently interest policy.\n// Interest policy streams require consumers match the replica count.\n// Will leave her for now to make sure we do not override.\nfunc TestMQTTConsumerReplicasOverride(t *testing.T) {\n\tconf := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tstream_replicas: 5\n\t\t\tconsumer_replicas: 1\n\t\t\tconsumer_memory_storage: true\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tcl := createJetStreamClusterWithTemplate(t, conf, \"MQTT\", 5)\n\tdefer cl.shutdown()\n\n\tconnectAndCheck := func(subject string, restarted bool) {\n\t\tt.Helper()\n\n\t\to := cl.opts[0]\n\t\tmc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"test\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port, 5)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, restarted)\n\t\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: subject, qos: 1}}, []byte{1})\n\n\t\tnc, js := jsClientConnect(t, cl.servers[2])\n\t\tdefer nc.Close()\n\n\t\tfor ci := range js.ConsumersInfo(mqttStreamName) {\n\t\t\tif ci.Config.FilterSubject == mqttStreamSubjectPrefix+subject {\n\t\t\t\tif rf := len(ci.Cluster.Replicas) + 1; rf != 5 {\n\t\t\t\t\tt.Fatalf(\"Expected consumer to be R5, got: %d\", rf)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tconnectAndCheck(\"foo\", false)\n\n\tcl.stopAll()\n\tfor _, o := range cl.opts {\n\t\to.MQTT.ConsumerReplicas = 2\n\t\to.MQTT.ConsumerMemoryStorage = false\n\t}\n\tcl.restartAllSamePorts()\n\tcl.waitOnStreamLeader(globalAccountName, mqttStreamName)\n\tcl.waitOnStreamLeader(globalAccountName, mqttRetainedMsgsStreamName)\n\tcl.waitOnStreamLeader(globalAccountName, mqttSessStreamName)\n\n\tconnectAndCheck(\"bar\", true)\n}\n\nfunc TestMQTTConsumerMemStorageReload(t *testing.T) {\n\ttdir := t.TempDir()\n\ttmpl := `\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tserver_name: mqtt\n\t\tmqtt {\n\t\t\tport: -1\n\t\t\tconsumer_memory_storage: %s\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, \"false\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: false}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, \"true\"))\n\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tmset, err := s.GlobalAccount().lookupStream(mqttStreamName)\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up stream: %v\", err)\n\t}\n\tvar cons *consumer\n\tmset.mu.RLock()\n\tfor _, c := range mset.consumers {\n\t\tcons = c\n\t\tbreak\n\t}\n\tmset.mu.RUnlock()\n\tcons.mu.RLock()\n\tst := cons.store.Type()\n\tcons.mu.RUnlock()\n\tif st != MemoryStorage {\n\t\tt.Fatalf(\"Expected storage %v, got %v\", MemoryStorage, st)\n\t}\n}\n\n// Test for auto-cleanup of consumers.\nfunc TestMQTTConsumerInactiveThreshold(t *testing.T) {\n\ttdir := t.TempDir()\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tconsumer_inactive_threshold: %q\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, \"0.2s\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tmc, r := testMQTTConnectRetry(t, &mqttConnInfo{clientID: \"test\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tci := <-js.ConsumersInfo(\"$MQTT_msgs\")\n\n\t// Make sure we clean up this consumer.\n\tmc.Close()\n\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t_, err := js.ConsumerInfo(ci.Stream, ci.Name)\n\t\tif err == nil {\n\t\t\treturn fmt.Errorf(\"Consumer still present\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Check reload.\n\t// We will not redo existing consumers however.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, tdir, \"22s\"))\n\tif opts := s.getOpts(); opts.MQTT.ConsumerInactiveThreshold != 22*time.Second {\n\t\tt.Fatalf(\"Expected reloaded value of %v but got %v\", 22*time.Second, opts.MQTT.ConsumerInactiveThreshold)\n\t}\n}\n\nfunc TestMQTTSubjectMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\n\t\tmappings = {\n\t\t\tfoo0: bar0\n\t\t\tfoo1: bar1\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tfor _, qos := range []byte{0, 1} {\n\t\tt.Run(fmt.Sprintf(\"qos%d\", qos), func(t *testing.T) {\n\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"sub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\n\t\t\tbar := fmt.Sprintf(\"bar%v\", qos)\n\t\t\tfoo := fmt.Sprintf(\"foo%v\", qos)\n\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: bar, qos: qos}}, []byte{qos})\n\t\t\ttestMQTTFlush(t, mc, nil, r)\n\n\t\t\tmcp, rp := testMQTTConnect(t, &mqttConnInfo{clientID: \"pub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mcp.Close()\n\t\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\tnatsPub(t, nc, foo, []byte(\"msg0\"))\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte(\"msg0\"))\n\n\t\t\ttestMQTTPublish(t, mcp, rp, 0, false, false, foo, 0, []byte(\"msg1\"))\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte(\"msg1\"))\n\n\t\t\ttestMQTTPublish(t, mcp, rp, 0, false, true, foo, 0, []byte(\"msg1_retained\"))\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte(\"msg1_retained\"))\n\n\t\t\ttestMQTTPublish(t, mcp, rp, 1, false, false, foo, 1, []byte(\"msg2\"))\n\t\t\t// For the receiving side, the expected QoS is based on the subscription's QoS,\n\t\t\t// not the publisher.\n\t\t\texpected := byte(0)\n\t\t\tif qos == 1 {\n\t\t\t\texpected = mqttPubQos1\n\t\t\t}\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte(\"msg2\"))\n\n\t\t\ttestMQTTPublish(t, mcp, rp, 1, false, true, foo, 1, []byte(\"msg2_retained\"))\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte(\"msg2_retained\"))\n\n\t\t\ttestMQTTDisconnect(t, mcp, nil)\n\t\t\tmcp.Close()\n\n\t\t\t// Try the with the \"will\" with QoS0 first\n\t\t\tmcp, rp = testMQTTConnect(t, &mqttConnInfo{\n\t\t\t\tclientID: \"pub\",\n\t\t\t\tcleanSess: true,\n\t\t\t\twill: &mqttWill{topic: []byte(foo), qos: 0, message: []byte(\"willmsg1\")}},\n\t\t\t\to.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mcp.Close()\n\t\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTFlush(t, mcp, nil, rp)\n\n\t\t\t// Close the connection without proper disconnect for will to be sent\n\t\t\tmcp.Close()\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, 0, []byte(\"willmsg1\"))\n\n\t\t\t// Try the with the \"will\" with QoS1 now\n\t\t\tmcp, rp = testMQTTConnect(t, &mqttConnInfo{\n\t\t\t\tclientID: \"pub\",\n\t\t\t\tcleanSess: true,\n\t\t\t\twill: &mqttWill{topic: []byte(foo), qos: 1, message: []byte(\"willmsg2\")}},\n\t\t\t\to.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mcp.Close()\n\t\t\ttestMQTTCheckConnAck(t, rp, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTFlush(t, mcp, nil, rp)\n\n\t\t\t// Close the connection without proper disconnect for will to be sent\n\t\t\tmcp.Close()\n\t\t\ttestMQTTCheckPubMsgNoAck(t, mc, r, bar, expected, []byte(\"willmsg2\"))\n\n\t\t\tsi, err := js.StreamInfo(\"$MQTT_msgs\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif qos == 0 {\n\t\t\t\trequire_True(t, si.State.Msgs == 0)\n\t\t\t\trequire_True(t, si.State.NumSubjects == 0)\n\t\t\t} else {\n\t\t\t\t// Number of QoS1 messages: 1 regular, 1 retained, 1 from will.\n\t\t\t\trequire_True(t, si.State.Msgs == 3)\n\t\t\t\trequire_True(t, si.State.NumSubjects == 1)\n\t\t\t}\n\t\t\ttestMQTTDisconnect(t, mc, nil)\n\t\t})\n\t}\n}\n\nfunc TestMQTTSubjectMappingWithImportExport(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers = [{user: \"a\", password: \"pwd\"}]\n\t\t\t\tmappings = {\n\t\t\t\t\tbar: foo\n\t\t\t\t}\n\t\t\t\texports = [{service: \"bar\"}]\n\t\t\t\tjetstream: enabled\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers = [{user: \"b\", password: \"pwd\"}]\n\t\t\t\tmappings = {\n\t\t\t\t\tfoo: bar\n\t\t\t\t}\n\t\t\t\timports = [{service: {account: \"A\", subject: \"bar\"}}]\n\t\t\t\tjetstream: enabled\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tc1, r1 := testMQTTConnect(t, &mqttConnInfo{user: \"a\", pass: \"pwd\", clientID: \"a1\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c1.Close()\n\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, c1, nil, r1)\n\n\tc2, r2 := testMQTTConnect(t, &mqttConnInfo{user: \"a\", pass: \"pwd\", clientID: \"a2\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c2, r2, []*mqttFilter{{filter: \"foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c2, nil, r2)\n\n\tc3, r3 := testMQTTConnect(t, &mqttConnInfo{user: \"a\", pass: \"pwd\", clientID: \"a3\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c3.Close()\n\ttestMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c3, r3, []*mqttFilter{{filter: \"bar\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, c3, nil, r3)\n\n\tc4, r4 := testMQTTConnect(t, &mqttConnInfo{user: \"a\", pass: \"pwd\", clientID: \"a4\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c4.Close()\n\ttestMQTTCheckConnAck(t, r4, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c4, r4, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, c4, nil, r4)\n\n\tbc, br := testMQTTConnect(t, &mqttConnInfo{user: \"b\", pass: \"pwd\", clientID: \"b0\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer bc.Close()\n\ttestMQTTCheckConnAck(t, br, mqttConnAckRCConnectionAccepted, false)\n\n\tbc1, br1 := testMQTTConnect(t, &mqttConnInfo{user: \"b\", pass: \"pwd\", clientID: \"b1\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer bc1.Close()\n\ttestMQTTCheckConnAck(t, br1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, bc1, br1, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, bc1, nil, br1)\n\n\tbc2, br2 := testMQTTConnect(t, &mqttConnInfo{user: \"b\", pass: \"pwd\", clientID: \"b2\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer bc2.Close()\n\ttestMQTTCheckConnAck(t, br2, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, bc2, br2, []*mqttFilter{{filter: \"b\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, bc2, nil, br2)\n\n\tbc3, br3 := testMQTTConnect(t, &mqttConnInfo{user: \"b\", pass: \"pwd\", clientID: \"b3\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer bc3.Close()\n\ttestMQTTCheckConnAck(t, br3, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, bc3, br3, []*mqttFilter{{filter: \"bar\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, bc3, nil, br3)\n\n\tbc4, br4 := testMQTTConnect(t, &mqttConnInfo{user: \"b\", pass: \"pwd\", clientID: \"b4\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer bc4.Close()\n\ttestMQTTCheckConnAck(t, br4, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, bc4, br4, []*mqttFilter{{filter: \"bar\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, bc4, nil, br4)\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://b:pwd@%s:%d\", o.Host, o.Port))\n\tdefer nc.Close()\n\n\tnatsPub(t, nc, \"foo\", []byte(\"msg0\"))\n\ttestMQTTCheckPubMsgNoAck(t, c1, r1, \"foo\", 0, []byte(\"msg0\"))\n\ttestMQTTCheckPubMsgNoAck(t, c2, r2, \"foo\", 0, []byte(\"msg0\"))\n\ttestMQTTExpectNothing(t, r3)\n\ttestMQTTExpectNothing(t, r4)\n\ttestMQTTExpectNothing(t, br1)\n\ttestMQTTExpectNothing(t, br2)\n\ttestMQTTCheckPubMsgNoAck(t, bc3, br3, \"bar\", 0, []byte(\"msg0\"))\n\ttestMQTTCheckPubMsgNoAck(t, bc4, br4, \"bar\", 0, []byte(\"msg0\"))\n\n\ttestMQTTPublish(t, bc, br, 0, false, false, \"foo\", 0, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsgNoAck(t, c1, r1, \"foo\", 0, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsgNoAck(t, c2, r2, \"foo\", 0, []byte(\"msg1\"))\n\ttestMQTTExpectNothing(t, r3)\n\ttestMQTTExpectNothing(t, r4)\n\ttestMQTTExpectNothing(t, br1)\n\ttestMQTTExpectNothing(t, br2)\n\ttestMQTTCheckPubMsgNoAck(t, bc3, br3, \"bar\", 0, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsgNoAck(t, bc4, br4, \"bar\", 0, []byte(\"msg1\"))\n\n\ttestMQTTPublish(t, bc, br, 1, false, false, \"foo\", 1, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsgNoAck(t, c1, r1, \"foo\", 0, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsgNoAck(t, c2, r2, \"foo\", mqttPubQos1, []byte(\"msg2\"))\n\ttestMQTTExpectNothing(t, r3)\n\ttestMQTTExpectNothing(t, r4)\n\ttestMQTTExpectNothing(t, br1)\n\ttestMQTTExpectNothing(t, br2)\n\ttestMQTTCheckPubMsgNoAck(t, bc3, br3, \"bar\", 0, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsgNoAck(t, bc4, br4, \"bar\", mqttPubQos1, []byte(\"msg2\"))\n\n\t// Connection nc is for account B\n\tcheck := func(nc *nats.Conn, subj string) {\n\t\tt.Helper()\n\t\treq, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: \">\"})\n\t\trequire_NoError(t, err)\n\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"$MQTT_msgs\"), req, time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar si StreamInfo\n\t\terr = json.Unmarshal(resp.Data, &si)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, si.State.Msgs == 1)\n\t\trequire_True(t, si.State.NumSubjects == 1)\n\t\trequire_True(t, si.State.Subjects[subj] == 1)\n\t}\n\t// Currently, nc is for account B\n\tcheck(nc, \"$MQTT.msgs.bar\")\n\n\tnc.Close()\n\tnc = natsConnect(t, fmt.Sprintf(\"nats://a:pwd@%s:%d\", o.Host, o.Port))\n\tdefer nc.Close()\n\tcheck(nc, \"$MQTT.msgs.foo\")\n}\n\nfunc TestMQTTSubRetainedRace(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\n\tuseCases := []struct {\n\t\tname string\n\t\tf func(t *testing.T, s *Server, o *Options, subTopic, pubTopic string, QOS byte)\n\t}{\n\t\t{\"new top level\", testMQTTNewSubRetainedRace},\n\t\t{\"existing top level\", testMQTTNewSubWithExistingTopLevelRetainedRace},\n\t}\n\tpubTopic := \"/bar\"\n\tsubTopics := []string{\"#\", \"/bar\", \"/+\", \"/#\"}\n\tQOS := []byte{0, 1, 2}\n\tfor _, tc := range useCases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfor _, subTopic := range subTopics {\n\t\t\t\tt.Run(subTopic, func(t *testing.T) {\n\t\t\t\t\tfor _, qos := range QOS {\n\t\t\t\t\t\tt.Run(fmt.Sprintf(\"QOS%d\", qos), func(t *testing.T) {\n\t\t\t\t\t\t\ts := testMQTTRunServer(t, o)\n\t\t\t\t\t\t\tdefer testMQTTShutdownServer(s)\n\n\t\t\t\t\t\t\ttc.f(t, s, o, subTopic, pubTopic, qos)\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testMQTTNewSubRetainedRace(t *testing.T, s *Server, o *Options, subTopic, pubTopic string, QOS byte) {\n\texpectedFlags := (QOS << 1) | mqttPubFlagRetain\n\tpayload := []byte(\"testmsg\")\n\n\tpubID := nuid.Next()\n\tpubc, pubr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: pubID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 3)\n\ttestMQTTCheckConnAck(t, pubr, mqttConnAckRCConnectionAccepted, false)\n\tdefer testMQTTDisconnectEx(t, pubc, nil, true)\n\tdefer pubc.Close()\n\ttestMQTTPublish(t, pubc, pubr, QOS, false, true, pubTopic, 1, payload)\n\n\t// Wait for retained messages stream to be populated.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmset, err := acc.lookupStream(mqttRetainedMsgsStreamName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif mset.state().Msgs != 1 {\n\t\t\treturn errors.New(\"retained message not populated yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tsubID := nuid.Next()\n\tsubc, subr := testMQTTConnect(t, &mqttConnInfo{clientID: subID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(t, subr, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: subTopic, qos: QOS}}, []byte{QOS})\n\n\ttestMQTTCheckPubMsg(t, subc, subr, pubTopic, expectedFlags, payload)\n\tif QOS == 2 {\n\t\ttestMQTTCheckPubAck(t, subr, mqttPacketPubRel)\n\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, subc, 1)\n\t}\n\n\ttestMQTTDisconnectEx(t, subc, nil, true)\n\tsubc.Close()\n}\n\nfunc testMQTTNewSubWithExistingTopLevelRetainedRace(t *testing.T, s *Server, o *Options, subTopic, pubTopic string, QOS byte) {\n\texpectedFlags := (QOS << 1) | mqttPubFlagRetain\n\tpayload := []byte(\"testmsg\")\n\n\tpubID := nuid.Next()\n\tpubc, pubr := testMQTTConnectRetry(t, &mqttConnInfo{clientID: pubID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 3)\n\ttestMQTTCheckConnAck(t, pubr, mqttConnAckRCConnectionAccepted, false)\n\tdefer testMQTTDisconnectEx(t, pubc, nil, true)\n\tdefer pubc.Close()\n\n\tsubID := nuid.Next()\n\tsubc, subr := testMQTTConnect(t, &mqttConnInfo{clientID: subID, cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(t, subr, mqttConnAckRCConnectionAccepted, false)\n\n\t// Subscribe to `#` first.\n\ttestMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: `#`, qos: 0}}, []byte{0})\n\ttestMQTTExpectNothing(t, subr)\n\n\t// Publish the retained message with QOS2, see that the `#` subscriber gets it.\n\ttestMQTTPublish(t, pubc, pubr, 2, false, true, pubTopic, 1, payload)\n\ttestMQTTCheckPubMsg(t, subc, subr, pubTopic, 0, payload)\n\ttestMQTTExpectNothing(t, subr)\n\n\t// Wait for retained messages stream to be populated.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmset, err := acc.lookupStream(mqttRetainedMsgsStreamName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif mset.state().Msgs != 1 {\n\t\t\treturn errors.New(\"retained message not populated yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now subscribe to the topic we want to test. We should get the retained\n\t// message there.\n\ttestMQTTSub(t, 1, subc, subr, []*mqttFilter{{filter: subTopic, qos: QOS}}, []byte{QOS})\n\ttestMQTTCheckPubMsg(t, subc, subr, pubTopic, expectedFlags, payload)\n\tif QOS == 2 {\n\t\ttestMQTTCheckPubAck(t, subr, mqttPacketPubRel)\n\t\ttestMQTTSendPIPacket(mqttPacketPubComp, t, subc, 1)\n\t}\n\n\ttestMQTTDisconnectEx(t, subc, nil, true)\n\tsubc.Close()\n}\n\n// Issue https://github.com/nats-io/nats-server/issues/3924\n// The MQTT Server MUST NOT match Topic Filters starting with a wildcard character (# or +),\n// with Topic Names beginning with a $ character [MQTT-4.7.2-1]\nfunc TestMQTTSubjectWildcardStart(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\tmc1, r1 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc1.Close()\n\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\n\tmc2, r2 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\n\tmc3, r3 := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc3.Close()\n\ttestMQTTCheckConnAck(t, r3, mqttConnAckRCConnectionAccepted, false)\n\n\t// These will fail already with the bug due to messages already being queue up before the subAck.\n\ttestMQTTSub(t, 1, mc1, r1, []*mqttFilter{{filter: \"*\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc1, nil, r1)\n\n\ttestMQTTSub(t, 1, mc2, r2, []*mqttFilter{{filter: \"#\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc2, nil, r2)\n\n\ttestMQTTSub(t, 1, mc3, r3, []*mqttFilter{{filter: \"*/foo\", qos: 1}}, []byte{1})\n\ttestMQTTFlush(t, mc2, nil, r2)\n\n\t// Just as a barrier\n\tnatsFlush(t, nc)\n\n\t// Now publish\n\n\t// NATS Publish\n\tmsg := []byte(\"HELLO WORLD\")\n\tnatsPub(t, nc, \"foo\", msg)\n\n\t// Check messages received\n\ttestMQTTCheckPubMsg(t, mc1, r1, \"foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r1)\n\n\ttestMQTTCheckPubMsg(t, mc2, r2, \"foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r2)\n\n\ttestMQTTExpectNothing(t, r3)\n\n\t// Anything that starts with $ is reserved against wildcard subjects like above.\n\tnatsPub(t, nc, \"$JS.foo\", msg)\n\ttestMQTTExpectNothing(t, r1)\n\ttestMQTTExpectNothing(t, r2)\n\ttestMQTTExpectNothing(t, r3)\n\n\t// Now do MQTT QoS-0\n\ttestMQTTPublish(t, mc, r, 0, false, false, \"foo\", 0, msg)\n\n\ttestMQTTCheckPubMsg(t, mc1, r1, \"foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r1)\n\n\ttestMQTTCheckPubMsg(t, mc2, r2, \"foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r2)\n\n\ttestMQTTExpectNothing(t, r3)\n\n\ttestMQTTPublish(t, mc, r, 0, false, false, \"$JS/foo\", 1, msg)\n\n\ttestMQTTExpectNothing(t, r1)\n\ttestMQTTExpectNothing(t, r2)\n\ttestMQTTExpectNothing(t, r3)\n\n\t// Now do MQTT QoS-1\n\tmsg = []byte(\"HELLO WORLD - RETAINED\")\n\ttestMQTTPublish(t, mc, r, 1, false, false, \"$JS/foo\", 4, msg)\n\n\ttestMQTTExpectNothing(t, r1)\n\ttestMQTTExpectNothing(t, r2)\n\ttestMQTTExpectNothing(t, r3)\n\n\ttestMQTTPublish(t, mc, r, 1, false, false, \"foo\", 2, msg)\n\n\ttestMQTTCheckPubMsg(t, mc1, r1, \"foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r1)\n\n\ttestMQTTCheckPubMsg(t, mc2, r2, \"foo\", 2, msg)\n\ttestMQTTExpectNothing(t, r2)\n\n\ttestMQTTExpectNothing(t, r3)\n\n\ttestMQTTPublish(t, mc, r, 1, false, false, \"foo/foo\", 3, msg)\n\n\ttestMQTTExpectNothing(t, r1)\n\n\ttestMQTTCheckPubMsg(t, mc2, r2, \"foo/foo\", 2, msg)\n\ttestMQTTExpectNothing(t, r2)\n\n\ttestMQTTCheckPubMsg(t, mc3, r3, \"foo/foo\", 2, msg)\n\ttestMQTTExpectNothing(t, r3)\n\n\t// Make sure we did not retain the messages prefixed with $.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(mqttStreamName)\n\trequire_NoError(t, err)\n\n\trequire_True(t, si.State.Msgs == 0)\n}\n\nfunc TestMQTTTopicWithDot(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"mqtt\", \"pwd\"))\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"*.*\")\n\n\tc1, r1 := testMQTTConnect(t, &mqttConnInfo{user: \"mqtt\", pass: \"pwd\", clientID: \"conn1\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c1.Close()\n\ttestMQTTCheckConnAck(t, r1, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: \"spBv1.0/plant1\", qos: 0}}, []byte{0})\n\ttestMQTTSub(t, 1, c1, r1, []*mqttFilter{{filter: \"spBv1.0/plant2\", qos: 1}}, []byte{1})\n\n\tc2, r2 := testMQTTConnect(t, &mqttConnInfo{user: \"mqtt\", pass: \"pwd\", clientID: \"conn2\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c2.Close()\n\ttestMQTTCheckConnAck(t, r2, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTPublish(t, c2, r2, 0, false, false, \"spBv1.0/plant1\", 0, []byte(\"msg1\"))\n\ttestMQTTCheckPubMsg(t, c1, r1, \"spBv1.0/plant1\", 0, []byte(\"msg1\"))\n\tmsg := natsNexMsg(t, sub, time.Second)\n\trequire_Equal(t, msg.Subject, \"spBv1//0.plant1\")\n\n\ttestMQTTPublish(t, c2, r2, 1, false, false, \"spBv1.0/plant2\", 1, []byte(\"msg2\"))\n\ttestMQTTCheckPubMsg(t, c1, r1, \"spBv1.0/plant2\", mqttPubQos1, []byte(\"msg2\"))\n\tmsg = natsNexMsg(t, sub, time.Second)\n\trequire_Equal(t, msg.Subject, \"spBv1//0.plant2\")\n\n\tnatsPub(t, nc, \"spBv1//0.plant1\", []byte(\"msg3\"))\n\ttestMQTTCheckPubMsg(t, c1, r1, \"spBv1.0/plant1\", 0, []byte(\"msg3\"))\n\tmsg = natsNexMsg(t, sub, time.Second)\n\trequire_Equal(t, msg.Subject, \"spBv1//0.plant1\")\n\n\tnatsPub(t, nc, \"spBv1//0.plant2\", []byte(\"msg4\"))\n\ttestMQTTCheckPubMsg(t, c1, r1, \"spBv1.0/plant2\", 0, []byte(\"msg4\"))\n\tmsg = natsNexMsg(t, sub, time.Second)\n\trequire_Equal(t, msg.Subject, \"spBv1//0.plant2\")\n}\n\n// Issue https://github.com/nats-io/nats-server/issues/4291\nfunc TestMQTTJetStreamRepublishAndQoS0Subscribers(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`, t.TempDir())))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Setup stream with republish on it.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRePublish: &nats.RePublish{\n\t\t\tSource: \"foo\",\n\t\t\tDestination: \"mqtt.foo\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Create QoS0 subscriber to catch re-publishes.\n\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"mqtt/foo\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, mc, nil, r)\n\n\tmsg := []byte(\"HELLO WORLD\")\n\t_, err = js.Publish(\"foo\", msg)\n\trequire_NoError(t, err)\n\n\ttestMQTTCheckPubMsg(t, mc, r, \"mqtt/foo\", 0, msg)\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTDecodeRetainedMessage(t *testing.T) {\n\ttdir := t.TempDir()\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: mqtt\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\n\t\tmqtt {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tconsumer_inactive_threshold: %q\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, tdir, \"0.2s\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer testMQTTShutdownServer(s)\n\n\t// Connect and publish a retained message, this will be in the \"newer\" form,\n\t// with the metadata in the header.\n\tmc, r := testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, mc, r, 0, false, true, \"foo/1\", 0, []byte(\"msg1\"))\n\tmc.Close()\n\n\t// Store a \"legacy\", JSON-encoded payload directly into the stream.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\trm := mqttRetainedMsg{\n\t\tOrigin: \"test\",\n\t\tSubject: \"foo.2\",\n\t\tTopic: \"foo/2\",\n\t\tFlags: mqttPubFlagRetain,\n\t\tMsg: []byte(\"msg2\"),\n\t}\n\tjsonData, _ := json.Marshal(rm)\n\t_, err := js.PublishMsg(&nats.Msg{\n\t\tSubject: mqttRetainedMsgsStreamSubject + rm.Subject,\n\t\tData: jsonData,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Error publishing retained message to JS directly: %v\", err)\n\t}\n\n\t// Restart the server to see that it picks up both retained messages on restart.\n\ts.Shutdown()\n\ts = RunServer(o)\n\tdefer testMQTTShutdownServer(s)\n\n\t// Connect again, subscribe, and check that we get both messages.\n\tmc, r = testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/+\", qos: 0}}, []byte{0})\n\tfor i := 0; i < 2; i++ {\n\t\tb, pl := testMQTTReadPacket(t, r)\n\t\tif pt := b & mqttPacketMask; pt != mqttPacketPub {\n\t\t\tt.Fatalf(\"Expected PUBLISH packet %x, got %x\", mqttPacketPub, pt)\n\t\t}\n\t\t_, _, topic, _ := testMQTTGetPubMsgExEx(t, mc, r, mqttPubFlagRetain, pl, \"\", nil)\n\t\tif string(topic) != \"foo/1\" && string(topic) != \"foo/2\" {\n\t\t\tt.Fatalf(\"Expected foo/1 or foo/2, got %q\", topic)\n\t\t}\n\t}\n\ttestMQTTExpectNothing(t, r)\n\tmc.Close()\n\n\t// Clear both retained messages.\n\tmc, r = testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, mc, r, 0, false, true, \"foo/1\", 0, []byte{})\n\ttestMQTTPublish(t, mc, r, 0, false, true, \"foo/2\", 0, []byte{})\n\ttestMQTTFlush(t, mc, nil, r)\n\ttestMQTTDisconnect(t, mc, nil)\n\tmc.Close()\n\n\t// Connect again, subscribe, and check that we get nothing.\n\tmc, r = testMQTTConnectRetry(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port, 5)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: \"foo/+\", qos: 0}}, []byte{0})\n\ttestMQTTExpectNothing(t, r)\n}\n\nfunc TestMQTTSparkbDeathHandling(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\t// protoMetrics is protobuf-encoded test data, with no timestamp.\n\t//\n\t//\tp := &sproto.Payload{\n\t// \t\tMetrics: []*sproto.Payload_Metric{\n\t// \t\t\tnewSparkbMetric_Uint64(sparkbBDSeqMetric, uint64(sparkbBDSeq)),\n\t// \t\t\tnewSparkbMetric(sparkbDeviceMetric1, \"value-at-death-1\"),\n\t// \t\t\tnewSparkbMetric(sparkbDeviceMetric2, \"value-at-death-2\"),\n\t// \t\t},\n\t// \t\tTimestamp: nil,\n\t// \t}\n\tprotoMetrics := []byte{0x12, 0x0b, 0x0a, 0x05, 0x62, 0x64, 0x53, 0x65, 0x71, 0x20, 0x04, 0x58, 0x02, 0x12, 0x24, 0x0a, 0x0e, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x2d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x31, 0x20, 0x0c, 0x7a, 0x10, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2d, 0x61, 0x74, 0x2d, 0x64, 0x65, 0x61, 0x74, 0x68, 0x2d, 0x31, 0x12, 0x24, 0x0a, 0x0e, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x2d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x32, 0x20, 0x0c, 0x7a, 0x10, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2d, 0x61, 0x74, 0x2d, 0x64, 0x65, 0x61, 0x74, 0x68, 0x2d, 0x32}\n\n\tproto0Timestamp := []byte{0x08, 0x00}\n\n\tprotoDeadbeefTimestamp := []byte{0x08, 0xef, 0xfd, 0xb6, 0xf5, 0xfd, 0xde, 0xef, 0xd6, 0xde, 0x01}\n\n\tmcPub, rPub := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcPub.Close()\n\ttestMQTTCheckConnAck(t, rPub, mqttConnAckRCConnectionAccepted, false)\n\n\tmcSub, rSub := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer mcSub.Close()\n\ttestMQTTCheckConnAck(t, rSub, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, mcSub, rSub, []*mqttFilter{{filter: \"spBv1.0/ggg/#\", qos: 0}}, []byte{0})\n\n\tfor _, test := range []*struct {\n\t\tname string\n\t\tpayload []byte\n\t\texpectTimestampOffset int\n\t\texpectUnchanged bool\n\t}{\n\t\t{\"append\", protoMetrics, len(protoMetrics), false},\n\t\t{\"replace at the end\", append(protoMetrics, proto0Timestamp...), len(protoMetrics), false},\n\t\t{\"replace at the start\", append(protoDeadbeefTimestamp, protoMetrics...), 0, false},\n\t\t{\"invalid\", []byte{0xde, 0xad, 0xbe, 0xef}, 0, true}, // invalid payload\n\t\t{\"invalid fixed32\", []byte{0x0d, 0x01}, 0, true},\n\t\t{\"invalid fixed64\", []byte{0x09, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}, 0, true},\n\t} {\n\t\tvar sendPI uint16\n\t\tfor _, topic := range []string{\"NDEATH/nnn\", \"DDEATH/nnn/ddd\"} {\n\t\t\tsendPI++\n\t\t\tt.Run(test.name+\" \"+topic, func(t *testing.T) {\n\t\t\t\ttestMQTTPublish(t, mcPub, rPub, 1, false, false, \"spBv1.0/ggg/\"+topic, sendPI, test.payload)\n\n\t\t\t\tflags, pi, _, received := testMQTTGetPubMsgEx(t, mcSub, rSub, \"\", nil)\n\t\t\t\tif mqttGetQoS(flags) == 1 {\n\t\t\t\t\ttestMQTTSendPIPacket(mqttPacketPubAck, t, mcSub, pi)\n\t\t\t\t}\n\n\t\t\t\tif test.expectUnchanged {\n\t\t\t\t\tif !bytes.Equal(test.payload, received) {\n\t\t\t\t\t\tt.Fatalf(\"Expected payload to be unchanged, got %q\", received)\n\t\t\t\t\t}\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tif test.expectTimestampOffset > len(received) {\n\t\t\t\t\tt.Fatalf(\"Expected timestamp offset %v to be less than %v\", test.expectTimestampOffset, len(received))\n\t\t\t\t}\n\t\t\t\tif test.expectTimestampOffset > 0 {\n\t\t\t\t\tif !bytes.Equal(protoMetrics, received[:test.expectTimestampOffset]) {\n\t\t\t\t\t\tt.Fatalf(\"Expected payload to be unchanged up to %v, got %q\", test.expectTimestampOffset, received)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\ttsBytes := received[test.expectTimestampOffset:]\n\t\t\t\tif tsBytes[0] != 0x08 {\n\t\t\t\t\tt.Fatalf(\"Expected timestamp to start with 0x08, got %x\", tsBytes)\n\t\t\t\t}\n\t\t\t\tv, _, err := protoScanVarint(tsBytes[1:])\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error scanning varint: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tts := time.UnixMilli(int64(v))\n\t\t\t\tif time.Now().Before(ts) {\n\t\t\t\t\tt.Fatalf(\"Expected timestamp to be in the past, got %v\", ts)\n\t\t\t\t}\n\t\t\t\tif time.Since(ts) > time.Second {\n\t\t\t\t\tt.Fatalf(\"Expected timestamp to be within a second, got %v\", time.Since(ts))\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestMQTTSparkbBirthHandling(t *testing.T) {\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\t// Publish an NBIRTH message. Make sure it is received as both the original\n\t// subject, and as a $sparkplug/certificates one.\n\n\tconst NBORN = \"NODE BORN FAKE MESSAGE\"\n\tconst DBORN = \"DEVICE BORN FAKE MESSAGE\"\n\n\ttests := []*struct {\n\t\ttopic string\n\t\tpayload string\n\t\tflags byte\n\t\tmc net.Conn\n\t\tr *mqttReader\n\t}{\n\t\t{\n\t\t\ttopic: \"spBv1.0/ggg/NBIRTH/nnn\",\n\t\t\tpayload: NBORN,\n\t\t\tflags: 0,\n\t\t},\n\t\t{\n\t\t\ttopic: \"spBv1.0/ggg/DBIRTH/nnn/ddd\",\n\t\t\tpayload: DBORN,\n\t\t\tflags: 0,\n\t\t},\n\t\t{\n\t\t\ttopic: \"$sparkplug/certificates/spBv1.0/ggg/NBIRTH/nnn\",\n\t\t\tpayload: NBORN,\n\t\t\tflags: mqttPubFlagRetain,\n\t\t},\n\t\t{\n\t\t\ttopic: \"$sparkplug/certificates/spBv1.0/ggg/DBIRTH/nnn/ddd\",\n\t\t\tpayload: DBORN,\n\t\t\tflags: mqttPubFlagRetain,\n\t\t},\n\t}\n\n\tt.Run(\"$sparkplug messages delivered\", func(t *testing.T) {\n\t\t// Connect the subscribers, unique for each topic we monitor.\n\t\tfor i, test := range tests {\n\t\t\ttest.mc, test.r = testMQTTConnect(t, &mqttConnInfo{cleanSess: true, clientID: fmt.Sprintf(\"sub-%v\", i)}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer test.mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, test.r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t\t// Subscribne at QoS2 to make sure the messages are posted at QoS0 and\n\t\t\t// not truncated to sub QoS.\n\t\t\ttestMQTTSub(t, 1, test.mc, test.r, []*mqttFilter{{filter: test.topic, qos: 2}}, []byte{2})\n\t\t}\n\n\t\t// connect the publisher\n\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true, clientID: \"pub\"}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer mc.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\n\t\t// {Publish} [ND]BIRTH messages. Note that IRL these are protobuf payloads.\n\t\ttestMQTTPublish(t, mc, r, 0, false, false, \"spBv1.0/ggg/NBIRTH/nnn\", 0, []byte(\"NODE BORN FAKE MESSAGE\"))\n\t\ttestMQTTPublish(t, mc, r, 0, false, false, \"spBv1.0/ggg/DBIRTH/nnn/ddd\", 0, []byte(\"DEVICE BORN FAKE MESSAGE\"))\n\n\t\t// Check that the subscribers got the messages.\n\t\tfor _, test := range tests {\n\t\t\ttestMQTTCheckPubMsg(t, test.mc, test.r, test.topic, test.flags, []byte(test.payload))\n\t\t}\n\t})\n\n\tt.Run(\"$sparkplug messages retained\", func(t *testing.T) {\n\t\t// Connect/subscribe again, and make sure that only retained messages are there.\n\t\tfor i, test := range tests {\n\t\t\tmc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true, clientID: fmt.Sprintf(\"sub-%v\", i+100)}, o.MQTT.Host, o.MQTT.Port)\n\t\t\tdefer mc.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTSub(t, 1, mc, r, []*mqttFilter{{filter: test.topic, qos: 2}}, []byte{2})\n\t\t\tif test.flags&mqttPubFlagRetain != 0 {\n\t\t\t\ttestMQTTCheckPubMsg(t, mc, r, test.topic, test.flags, []byte(test.payload))\n\t\t\t} else {\n\t\t\t\ttestMQTTExpectNothing(t, r)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestMQTTMaxPayloadEnforced(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tserver_name: test_mqtt_max_payload\n\t\tport: -1\n\t\tmax_payload: 1024\n\t\tmqtt { listen: \"127.0.0.1:-1\" }\n\t\tjetstream: { domain: \"TEST\", max_mem_store: 8MB, max_file_store: 8MB, store_dir: \"`+t.TempDir()+`\" }\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tmp := o.MQTT.Port\n\thost := o.MQTT.Host\n\tmc, _ := testMQTTConnect(t, &mqttConnInfo{clientID: \"cid\", cleanSess: true}, host, mp)\n\tdefer mc.Close()\n\n\toversized := bytes.Repeat([]byte{'A'}, 1500)\n\ttestMQTTSendPublishPacket(t, mc, 0, false, false, \"foo\", 0, oversized)\n\n\ttestMQTTExpectDisconnect(t, mc)\n}\n\nfunc TestMQTTJSApiMapping(t *testing.T) {\n\ttd := t.TempDir()\n\thubDir := filepath.Join(td, \"hub\")\n\thubConf := createConfFile(t, fmt.Appendf(nil, `\n server_name: hub\n listen: \"127.0.0.1:-1\"\n jetstream {\n domain: \"HUB\"\n store_dir: \"%s\"\n }\n mqtt {\n listen: \"127.0.0.1:-1\"\n }\n leafnodes {\n listen: \"127.0.0.1:-1\"\n isolate: true\n }\n accounts: {\n SYS: { users: [ { user:s, password:x } ] }\n HUB: {\n jetstream: true\n users: [\n {\n user:h\n password:x\n permissions: {\n subscribe: {allow: [\"foo\", \"bar\", \"_INBOX.>\"], deny: [\"baz\"] }\n }\n }\n ]\n }\n }\n system_account: SYS\n `, hubDir))\n\thub, ohub := RunServerWithConfig(hubConf)\n\tdefer hub.Shutdown()\n\n\tleafDir := filepath.Join(td, \"leaf\")\n\tleafTmpl := `\n server_name: leaf\n listen: \"127.0.0.1:-1\"\n jetstream {\n domain: \"LEAF\"\n store_dir: \"%s\"\n }\n mqtt {\n listen: \"127.0.0.1:-1\"\n js_api_timeout: \"500ms\"\n }\n leafnodes {\n remotes [\n { urls: [ nats-leaf://s:x@127.0.0.1:%d ], account: SYS }\n { urls: [ nats-leaf://h:x@127.0.0.1:%d ], account: HUB }\n ]\n }\n accounts: {\n SYS: { users: [ { user:s, password:x } ] }\n HUB: {\n jetstream: false\n users: [ { user:h, password:x } ]\n mappings: {\n \"$JS.API.>\" : \"$JS.HUB.API.>\"\n %s\n }\n }\n LEAF: {\n jetstream: true\n users: [ { user:l, password:x } ]\n }\n }\n system_account: SYS\n `\n\tleafConf := createConfFile(t, fmt.Appendf(nil, leafTmpl, leafDir, ohub.LeafNode.Port, ohub.LeafNode.Port, \"\"))\n\tleaf, oleaf := RunServerWithConfig(leafConf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, leaf, 2)\n\n\ttestRetained := func(user string, port int, msg string) {\n\t\tt.Helper()\n\t\t// Create a producer\n\t\tc, r := testMQTTConnect(t, &mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\tuser: user,\n\t\t\tpass: \"x\",\n\t\t}, \"127.0.0.1\", port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTPublish(t, c, r, 0, false, true, \"foo\", 0, []byte(msg))\n\t\ttestMQTTFlush(t, c, nil, r)\n\t\ttestMQTTDisconnect(t, c, nil)\n\t\tc.Close()\n\n\t\t// Create a consumer\n\t\tc, r = testMQTTConnect(t, &mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\tuser: user,\n\t\t\tpass: \"x\",\n\t\t}, \"127.0.0.1\", port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\t\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubFlagRetain, []byte(msg))\n\t}\n\n\t// Connect to leaf\n\ttestRetained(\"l\", oleaf.MQTT.Port, \"hi leaf\")\n\n\t// Now same test in the hub\n\ttestRetained(\"h\", ohub.MQTT.Port, \"hi hub\")\n\n\t// Now test the hub account in the leaf node. We expect it to fail\n\t// because we don't have the mapping for the sessions to be persisted\n\t// in the hub.\n\t_, _, err := testMQTTConnectRetryWithError(t, &mqttConnInfo{\n\t\tcleanSess: true,\n\t\tuser: \"h\",\n\t\tpass: \"x\",\n\t}, oleaf.MQTT.Host, oleaf.MQTT.Port, 0)\n\tif err == nil {\n\t\tt.Fatal(\"Expected failure to connect, but did connect\")\n\t}\n\n\t// Config reload the leaf server to add the missing mapping.\n\treloadUpdateConfig(t, leaf, leafConf, fmt.Sprintf(leafTmpl,\n\t\tleafDir, ohub.LeafNode.Port, ohub.LeafNode.Port, `\"$MQTT.sess.LEAF.>\" : \"$MQTT.sess.HUB.>\"`))\n\n\tc, r := testMQTTConnect(t, &mqttConnInfo{\n\t\tcleanSess: true,\n\t\tuser: \"h\",\n\t\tpass: \"x\",\n\t}, \"127.0.0.1\", oleaf.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubFlagRetain, []byte(\"hi hub\"))\n}\n\nfunc TestMQTTMappingsQoS0(t *testing.T) {\n\ttd := t.TempDir()\n\tdir := filepath.Join(td, \"js\")\n\tconf := createConfFile(t, fmt.Appendf(nil, `\n\t\tserver_name: server\n\t\tjetstream {\n\t\t\tdomain: \"A\"\n\t\t\tstore_dir: \"%s\"\n\t\t}\n\t\tmqtt {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\taccounts: {\n\t\t\tA: {\n\t\t\t\tjetstream: true\n\t\t\t\tusers: [ { user:a, password:x }]\n\t\t\t\texports: [ { stream: \"foo.>\" }]\n\t\t\t\tmappings: { \"baz.>\" : \"bazz.>\" }\n\t\t\t}\n\t\t\tB: {\n\t\t\t\tjetstream: true\n\t\t\t\tusers: [ { user:b, password:x }]\n\t\t\t\timports: [ { stream: { account: A, subject: \"foo.>\" }, to: \"bar.>\" } ]\n\t\t\t}\n\t\t\tC: {\n\t\t\t\tjetstream: true\n\t\t\t\tusers: [ { user:c, password:x }]\n\t\t\t\timports: [ { stream: { account: A, subject: \"foo.>\" }, to: \"baz.>\" } ]\n\t\t\t}\n\t\t\tD: {\n\t\t\t\tjetstream: true\n\t\t\t\tusers: [ { user:d, password:x }]\n\t\t\t\t# Same imports than for account C\n\t\t\t\timports: [ { stream: { account: A, subject: \"foo.>\" }, to: \"baz.>\" } ]\n\t\t\t}\n\t\t}\n\t`, dir))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// First, we check for mapping inside the same account A.\n\taSubConn, aSubReader := testMQTTConnect(t, &mqttConnInfo{\n\t\tcleanSess: true,\n\t\tclientID: \"sub\",\n\t\tuser: \"a\",\n\t\tpass: \"x\",\n\t}, \"127.0.0.1\", o.MQTT.Port)\n\tdefer aSubConn.Close()\n\ttestMQTTCheckConnAck(t, aSubReader, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTSub(t, 1, aSubConn, aSubReader, []*mqttFilter{{filter: \"#\", qos: 0}}, []byte{0})\n\ttestMQTTFlush(t, aSubConn, nil, aSubReader)\n\n\taPubConn, aPubReader := testMQTTConnect(t, &mqttConnInfo{\n\t\tcleanSess: true,\n\t\tclientID: \"pub\",\n\t\tuser: \"a\",\n\t\tpass: \"x\",\n\t}, \"127.0.0.1\", o.MQTT.Port)\n\tdefer aPubConn.Close()\n\ttestMQTTCheckConnAck(t, aPubReader, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, aPubConn, aPubReader, 0, false, true, \"baz/x\", 0, []byte(\"msg1\"))\n\t// Because of mapping, the subscription should receive the message on \"bazz/x\"\n\ttestMQTTCheckPubMsg(t, aSubConn, aSubReader, \"bazz/x\", 0, []byte(\"msg1\"))\n\n\t// Helper to create a connection and sub for a given user on given topic\n\tcreateSub := func(user, topic string) (net.Conn, *mqttReader) {\n\t\tt.Helper()\n\t\tc, r := testMQTTConnect(t, &mqttConnInfo{\n\t\t\tcleanSess: true,\n\t\t\tclientID: nuid.Next(),\n\t\t\tuser: user,\n\t\t\tpass: \"x\",\n\t\t}, \"127.0.0.1\", o.MQTT.Port)\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: topic, qos: 0}}, []byte{0})\n\t\ttestMQTTFlush(t, c, nil, r)\n\t\treturn c, r\n\t}\n\tbConn1, bReader1 := createSub(\"b\", \"bar/#\")\n\tdefer bConn1.Close()\n\tbConn2, bReader2 := createSub(\"b\", \"bar/+\")\n\tdefer bConn2.Close()\n\tcConn1, cReader1 := createSub(\"c\", \"baz/#\")\n\tdefer cConn1.Close()\n\tcConn2, cReader2 := createSub(\"c\", \"baz/+\")\n\tdefer cConn2.Close()\n\tdConn1, dReader1 := createSub(\"d\", \"baz/#\")\n\tdefer dConn1.Close()\n\tdConn2, dReader2 := createSub(\"d\", \"baz/+\")\n\tdefer dConn2.Close()\n\n\t// Now from \"A\" publisher, publishes on \"foo/x\", and with mapping across\n\t// accounts, we should get the expected results. We send 2 messages in\n\t// a row and verify that we get those, and no more than that.\n\ttestMQTTPublish(t, aPubConn, aPubReader, 0, false, true, \"foo/x\", 0, []byte(\"msg2\"))\n\ttestMQTTPublish(t, aPubConn, aPubReader, 0, false, true, \"foo/x\", 0, []byte(\"msg3\"))\n\n\t// \"B\" consumers should receive on \"bar/x\"\n\tconns := []net.Conn{bConn1, bConn2}\n\treaders := []*mqttReader{bReader1, bReader2}\n\tfor i := range len(conns) {\n\t\ttestMQTTCheckPubMsg(t, conns[i], readers[i], \"bar/x\", 0, []byte(\"msg2\"))\n\t\ttestMQTTCheckPubMsg(t, conns[i], readers[i], \"bar/x\", 0, []byte(\"msg3\"))\n\t\ttestMQTTExpectNothing(t, readers[i])\n\t}\n\t// For \"C\" and \"D\" consumers, it should be \"baz/x\"\n\tconns = []net.Conn{cConn1, cConn2, dConn1, dConn2}\n\treaders = []*mqttReader{cReader1, cReader2, dReader1, dReader2}\n\tfor i := range len(conns) {\n\t\ttestMQTTCheckPubMsg(t, conns[i], readers[i], \"baz/x\", 0, []byte(\"msg2\"))\n\t\ttestMQTTCheckPubMsg(t, conns[i], readers[i], \"baz/x\", 0, []byte(\"msg3\"))\n\t\ttestMQTTExpectNothing(t, readers[i])\n\t}\n}\n\nfunc TestMQTTSliceHeadersAndDecodeRetainedMessage(t *testing.T) {\n\t// First check low level mqttSliceHeaders\n\tfor _, test := range []struct {\n\t\tname string\n\t\thdr string\n\t\texpected []string\n\t}{\n\t\t// Valid cases\n\t\t{\"one key and some random\", hdrLine + \"key1:val1\\r\\nsomeotherkey:someval\\r\\n\", []string{\"val1\", _EMPTY_, _EMPTY_}},\n\t\t{\"two keys\", hdrLine + \"key2:val2\\r\\nthisisnotkey1:someval\\r\\nkey1:val1\\r\\n\", []string{\"val1\", \"val2\", _EMPTY_}},\n\t\t{\"three keys\", hdrLine + \"key2:val2\\r\\nkey3:val3\\r\\nkey1:val1\\r\\n\", []string{\"val1\", \"val2\", \"val3\"}},\n\t\t{\"space before value\", hdrLine + \"somekey:someval\\r\\nkey2: val2withspacebefore\\r\\n\", []string{_EMPTY_, \"val2withspacebefore\", _EMPTY_}},\n\t\t{\"space between key and colon sign\", hdrLine + \"key1:val1\\r\\nkey2 :val2\\r\\nkey3 : val3\\r\\n\", []string{\"val1\", \"val2\", \"val3\"}},\n\t\t// Error cases\n\t\t{\"no hdr line\", \"key1:val1\\r\\n\", []string{_EMPTY_, _EMPTY_, _EMPTY_}},\n\t\t{\"key length 0\", hdrLine + \"key1:val1\\r\\n:val2\\r\\nkey3:val3\\r\\n\", []string{\"val1\", _EMPTY_, _EMPTY_}},\n\t\t{\"key is only spaces\", hdrLine + \"key1:val1\\r\\nkey2:val2\\r\\n :val3\\r\\n\", []string{\"val1\", \"val2\", _EMPTY_}},\n\t\t{\"value no crlf\", hdrLine + \"key1:val1\\r\\nkey2:val2\", []string{\"val1\", _EMPTY_, _EMPTY_}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\theaders := map[string][]byte{\n\t\t\t\t\"key1\": nil,\n\t\t\t\t\"key2\": nil,\n\t\t\t\t\"key3\": nil,\n\t\t\t}\n\t\t\tmqttSliceHeaders(headers, []byte(test.hdr))\n\t\t\tfor i := range len(headers) {\n\t\t\t\tkey := fmt.Sprintf(\"key%d\", i+1)\n\t\t\t\tval := string(headers[key])\n\t\t\t\tif ev := test.expected[i]; ev != val {\n\t\t\t\t\tt.Fatalf(\"For key %q, expected value to be %q, got %q\", key, ev, val)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\t// Now test mqttDecodeRetainedMessage() itself.\n\tt.Run(\"flag with delete marker\", func(t *testing.T) {\n\t\thdr := fmt.Appendf(nil, \"%sNmqtt-RFlags:%c1\\r\\n\\r\\n\", hdrLine, mqttRetainedFlagDelMarker)\n\t\trm, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, nil)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rm.Flags, 1)\n\t})\n\tt.Run(\"flag not a number\", func(t *testing.T) {\n\t\thdr := fmt.Appendf(nil, \"%sNmqtt-RFlags:bad\\r\\n\\r\\n\", hdrLine)\n\t\t_, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, []byte(\"msg\"))\n\t\trequire_Error(t, err, errMQTTInvalidRetainFlags)\n\t})\n\tt.Run(\"flag not a number with delete marker\", func(t *testing.T) {\n\t\thdr := fmt.Appendf(nil, \"%sNmqtt-RFlags:%cad\\r\\n\\r\\n\", hdrLine, mqttRetainedFlagDelMarker)\n\t\t_, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, []byte(\"msg\"))\n\t\trequire_Error(t, err, errMQTTInvalidRetainFlags)\n\t})\n\tt.Run(\"flag too big\", func(t *testing.T) {\n\t\thdr := fmt.Appendf(nil, \"%sNmqtt-RFlags:%c15\\r\\n\\r\\n\", hdrLine, mqttRetainedFlagDelMarker)\n\t\t_, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, []byte(\"msg\"))\n\t\trequire_Error(t, err, errMQTTInvalidRetainFlags)\n\t})\n\tt.Run(\"flag invalid qos\", func(t *testing.T) {\n\t\thdr := fmt.Appendf(nil, \"%sNmqtt-RFlags:%c7\\r\\n\\r\\n\", hdrLine, mqttRetainedFlagDelMarker)\n\t\t_, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, []byte(\"msg\"))\n\t\trequire_Error(t, err, errMQTTInvalidRetainFlags)\n\t})\n\tt.Run(\"decode retained msg with space before header value\", func(t *testing.T) {\n\t\tmsg, hdrLen := mqttEncodeRetainedMessage(&mqttRetainedMsg{\n\t\t\tTopic: \"foo/x\",\n\t\t\tOrigin: \" Origin\", // Add spaces in front on purpose\n\t\t\tSource: \"Source\",\n\t\t\tFlags: 1,\n\t\t\tMsg: []byte(\"msg1\"),\n\t\t})\n\t\thdr := msg[:hdrLen]\n\t\tmsg = msg[hdrLen:]\n\t\trm, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.foo.x\", hdr, msg)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rm.Topic, \"foo/x\")\n\t\trequire_Equal(t, rm.Subject, \"foo.x\")\n\t\trequire_Equal(t, rm.Origin, \"Origin\")\n\t\trequire_Equal(t, rm.Source, \"Source\")\n\t\trequire_Equal(t, rm.Flags, 1)\n\t\trequire_Equal(t, string(rm.Msg), \"msg1\")\n\t})\n\tt.Run(\"decode retained msg with subject transformed\", func(t *testing.T) {\n\t\tmsg, hdrLen := mqttEncodeRetainedMessage(&mqttRetainedMsg{\n\t\t\tTopic: \"foo/x\",\n\t\t\tOrigin: \"Origin\",\n\t\t\tSource: \"Source\",\n\t\t\tFlags: 1,\n\t\t\tMsg: []byte(\"msg2\"),\n\t\t})\n\t\thdr := msg[:hdrLen]\n\t\tmsg = msg[hdrLen:]\n\t\t// Use different subject when calling the function. Make sure the\n\t\t// topic is properly reflecting the subject.\n\t\trm, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, msg)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rm.Topic, \"bar/x\")\n\t\trequire_Equal(t, rm.Subject, \"bar.x\")\n\t\trequire_Equal(t, rm.Origin, \"Origin\")\n\t\trequire_Equal(t, rm.Source, \"Source\")\n\t\trequire_Equal(t, rm.Flags, 1)\n\t\trequire_Equal(t, string(rm.Msg), \"msg2\")\n\t})\n\tt.Run(\"decode deleted retained message\", func(t *testing.T) {\n\t\tmsg, hdrLen := mqttEncodeRetainedMessage(&mqttRetainedMsg{\n\t\t\tTopic: \"foo/x\",\n\t\t\tOrigin: \"Origin\",\n\t\t\tSource: \"Source\",\n\t\t\tFlags: 1,\n\t\t\tMsg: nil,\n\t\t})\n\t\thdr := msg[:hdrLen]\n\t\tmsg = msg[hdrLen:]\n\t\t// Use different subject too\n\t\trm, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", hdr, msg)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rm.Topic, \"bar/x\")\n\t\trequire_Equal(t, rm.Subject, \"bar.x\")\n\t\trequire_Equal(t, rm.Origin, \"Origin\")\n\t\trequire_Equal(t, rm.Source, \"Source\")\n\t\trequire_Equal(t, rm.Flags, 1)\n\t\trequire_Len(t, len(rm.Msg), 0)\n\t})\n\tt.Run(\"decode retained message as JSON with bad flags\", func(t *testing.T) {\n\t\trmo := &mqttRetainedMsg{Flags: 15}\n\t\tmsg, err := json.Marshal(rmo)\n\t\trequire_NoError(t, err)\n\t\t_, err = mqttDecodeRetainedMessage(\"$MQTT.rmsgs.foo.x\", nil, msg)\n\t\trequire_Error(t, err, errMQTTInvalidRetainFlags)\n\t})\n\tt.Run(\"decode retained message as JSON subject transform\", func(t *testing.T) {\n\t\trmo := &mqttRetainedMsg{\n\t\t\tTopic: \"foo/x\",\n\t\t\tOrigin: \"Origin\",\n\t\t\tSource: \"Source\",\n\t\t\tFlags: 1,\n\t\t\tMsg: []byte(\"hello\"),\n\t\t}\n\t\tmsg, err := json.Marshal(rmo)\n\t\trequire_NoError(t, err)\n\t\trm, err := mqttDecodeRetainedMessage(\"$MQTT.rmsgs.bar.x\", nil, msg)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, rm.Topic, \"bar/x\")\n\t\trequire_Equal(t, rm.Subject, \"bar.x\")\n\t\trequire_Equal(t, rm.Origin, \"Origin\")\n\t\trequire_Equal(t, rm.Source, \"Source\")\n\t\trequire_Equal(t, rm.Flags, 1)\n\t\trequire_Equal(t, string(rm.Msg), \"hello\")\n\t})\n}\n\nfunc TestMQTTRetainedMsgRemovedFromMapIfNotInStream(t *testing.T) {\n\tmqttRetainedCacheTTL = 250 * time.Millisecond\n\tdefer func() { mqttRetainedCacheTTL = mqttDefaultRetainedCacheTTL }()\n\n\to := testMQTTDefaultOptions()\n\ts := testMQTTRunServer(t, o)\n\tdefer testMQTTShutdownServer(s)\n\n\tc, r := testMQTTConnect(t, &mqttConnInfo{clientID: \"pub\", cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\tdefer c.Close()\n\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\ttestMQTTPublish(t, c, r, 0, false, true, \"foo\", 0, []byte(\"msg1\"))\n\ttestMQTTFlush(t, c, nil, r)\n\n\tcheckRetained := func(expected string) {\n\t\tt.Helper()\n\t\tc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, o.MQTT.Host, o.MQTT.Port)\n\t\tdefer c.Close()\n\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: \"foo\", qos: 0}}, []byte{0})\n\t\tif expected == _EMPTY_ {\n\t\t\ttestMQTTExpectNothing(t, r)\n\t\t} else {\n\t\t\ttestMQTTCheckPubMsg(t, c, r, \"foo\", mqttPubFlagRetain, []byte(expected))\n\t\t}\n\t}\n\tcheckRetained(\"msg1\")\n\n\ttestMQTTPublish(t, c, r, 0, false, true, \"foo\", 0, []byte(\"msg2\"))\n\ttestMQTTFlush(t, c, nil, r)\n\n\tcheckRetained(\"msg2\")\n\n\t// Now we will get the current sequence for the retained message and\n\t// remove it from the stream. We expect to get a warning that indicates\n\t// that the load failed. Restarting the subscription should not longer\n\t// cause this warning and the retained message should have been removed\n\t// from the map/cache.\n\tl := &captureWarnLogger{warn: make(chan string, 10)}\n\ts.SetLogger(l, false, false)\n\n\tasm := testMQTTGetAccountSessionManager(t, s, \"pub\")\n\t// Make sure it is in the cache\n\trm := asm.getCachedRetainedMsg(\"foo\")\n\trequire_NotNil(t, rm)\n\t// Get the mqttRetainedMsgRef from the map\n\tasm.mu.RLock()\n\trf, ok := asm.retmsgs[\"foo\"]\n\tasm.mu.RUnlock()\n\trequire_True(t, ok)\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\terr := js.DeleteMsg(mqttRetainedMsgsStreamName, rf.sseq)\n\trequire_NoError(t, err)\n\n\t// Wait for more than the cache TTL\n\ttime.Sleep(2 * mqttRetainedCacheTTL)\n\n\tcheckRetained(_EMPTY_)\n\n\t// We should have got a warning.\n\tselect {\n\tcase w := <-l.warn:\n\t\tif !strings.Contains(w, ApiErrors[JSNoMessageFoundErr].Description) {\n\t\t\tt.Fatalf(\"Unexpected warning: %q\", w)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Test timed out\")\n\t}\n\n\t// But restarting it should not cause the server to try to load the retained\n\t// message again. So we should not have a warning.\n\tcheckRetained(_EMPTY_)\n\n\tselect {\n\tcase w := <-l.warn:\n\t\tif strings.Contains(w, ApiErrors[JSNoMessageFoundErr].Description) {\n\t\t\tt.Fatalf(\"Got the warning: %q\", w)\n\t\t}\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// OK\n\t}\n\n\t// Finally, check that the retmsgs map is empty.\n\tasm.mu.RLock()\n\tok = len(asm.retmsgs) == 0\n\tasm.mu.RUnlock()\n\trequire_True(t, ok)\n}\n\nfunc TestMQTTCrossAccountRetain(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\timportTransform string\n\t\tsourceDest string\n\t\tbDest string\n\t\tgetLastMsgSubj string\n\t}{\n\t\t{\"without transform\", \"\", \"\", \"foo/x\", \"$MQTT.rmsgs.foo.x\"},\n\t\t{\"with transform\", `, to: \"foobar.>\"`, \"$MQTT.rmsgs.foobar.>\", \"foobar/x\", \"$MQTT.rmsgs.foobar.x\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttd := t.TempDir()\n\t\t\tdir := filepath.Join(td, \"js\")\n\t\t\tconf := createConfFile(t, fmt.Appendf(nil, `\n\t\t\t\tserver_name: server\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tjetstream {\n\t\t\t\t\tdomain: \"MYDOMAIN\"\n\t\t\t\t\tstore_dir: \"%s\"\n\t\t\t\t}\n\t\t\t\tmqtt {\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t}\n\t\t\t\taccounts: {\n\t\t\t\t\tA: {\n\t\t\t\t\t\tjetstream: true\n\t\t\t\t\t\tusers: [ { user:a, password:x }]\n\t\t\t\t\t\texports: [\n\t\t\t\t\t\t\t{ stream: \"foo.>\" }\n\t\t\t\t\t\t\t{ service: \"$JS.API.>\", response_type: stream }\n\t\t\t\t\t\t\t{ stream: \"a2b.>\" }\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t\tB: {\n\t\t\t\t\t\tjetstream: true\n\t\t\t\t\t\tusers: [ { user:b, password:x }]\n\t\t\t\t\t\timports: [\n\t\t\t\t\t\t\t{ stream: { account: A, subject: \"foo.>\" }%s }\n\t\t\t\t\t\t\t{ service: { account: A, subject: \"$JS.API.>\"}, to: \"A.$JS.API.>\" }\n\t\t\t\t\t\t\t{ stream: { account: A, subject: \"a2b.>\" } }\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`, dir, test.importTransform))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Connect a user on \"B\" to create the MQTT assets.\n\t\t\tc, r := testMQTTConnect(t, &mqttConnInfo{cleanSess: true, user: \"b\", pass: \"x\"}, \"127.0.0.1\", o.MQTT.Port)\n\t\t\tdefer c.Close()\n\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\ttestMQTTDisconnect(t, c, nil)\n\t\t\tc.Close()\n\n\t\t\tpubRetained := func(user, dest, msg string) {\n\t\t\t\tt.Helper()\n\t\t\t\tc, r := testMQTTConnect(t, &mqttConnInfo{\n\t\t\t\t\tcleanSess: true,\n\t\t\t\t\tuser: user,\n\t\t\t\t\tpass: \"x\",\n\t\t\t\t}, \"127.0.0.1\", o.MQTT.Port)\n\t\t\t\tdefer c.Close()\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\ttestMQTTPublish(t, c, r, 0, false, true, dest, 0, []byte(msg))\n\t\t\t\ttestMQTTFlush(t, c, nil, r)\n\t\t\t}\n\n\t\t\t// Publish a retained message from account \"A\".\n\t\t\tretainInAMsg := \"Retain in A\"\n\t\t\tpubRetained(\"a\", \"foo/x\", retainInAMsg)\n\n\t\t\t// Now we are going to do something unusual, which is to update\n\t\t\t// the MQTT retain stream in \"B\" to source from \"A\".\n\t\t\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"b\", \"x\"))\n\t\t\tdefer nc.Close()\n\n\t\t\tsi, err := js.StreamInfo(mqttRetainedMsgsStreamName)\n\t\t\trequire_NoError(t, err)\n\t\t\tsrc := &nats.StreamSource{\n\t\t\t\tName: mqttRetainedMsgsStreamName,\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"A.$JS.API\",\n\t\t\t\t\tDeliverPrefix: \"a2b\",\n\t\t\t\t},\n\t\t\t\tSubjectTransforms: []nats.SubjectTransformConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tSource: mqttRetainedMsgsStreamSubject + \"foo.>\",\n\t\t\t\t\t\tDestination: test.sourceDest,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\tsi.Config.Sources = []*nats.StreamSource{src}\n\t\t\t_, err = js.UpdateStream(&si.Config)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Now wait to make sure that the \"B\" retained messages stream\n\t\t\t// contains the message with body \"Retain in A\"\n\t\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t\tmsg, err := js.GetLastMsg(mqttRetainedMsgsStreamName, test.getLastMsgSubj)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif !bytes.Contains(msg.Data, []byte(retainInAMsg)) {\n\t\t\t\t\treturn fmt.Errorf(\"Message is not from A: %q\", msg.Data)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\n\t\t\tgetRetained := func(user, dest, msg string) {\n\t\t\t\tt.Helper()\n\t\t\t\tc, r := testMQTTConnect(t, &mqttConnInfo{\n\t\t\t\t\tcleanSess: true,\n\t\t\t\t\tuser: user,\n\t\t\t\t\tpass: \"x\",\n\t\t\t\t}, \"127.0.0.1\", o.MQTT.Port)\n\t\t\t\ttestMQTTCheckConnAck(t, r, mqttConnAckRCConnectionAccepted, false)\n\t\t\t\ttestMQTTSub(t, 1, c, r, []*mqttFilter{{filter: dest, qos: 0}}, []byte{0})\n\t\t\t\tif msg == _EMPTY_ {\n\t\t\t\t\ttestMQTTExpectNothing(t, r)\n\t\t\t\t} else {\n\t\t\t\t\ttestMQTTCheckPubMsg(t, c, r, dest, mqttPubFlagRetain, []byte(msg))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// The retained message in account A should of course be \"Retain in A\"\n\t\t\tgetRetained(\"a\", \"foo/x\", retainInAMsg)\n\t\t\t// But because of the sourcing, the retained message in \"B\" should be\n\t\t\t// the retained message from the \"A\" account.\n\t\t\tgetRetained(\"b\", test.bDest, retainInAMsg)\n\n\t\t\t// Now publish a retained message from \"B\" account and make sure that\n\t\t\t// it is correctly replacing \"Retain in A\".\n\t\t\tretainInBMsg := \"Retain in B\"\n\t\t\tpubRetained(\"b\", test.bDest, retainInBMsg)\n\t\t\t// Check that we can receive it.\n\t\t\tgetRetained(\"b\", test.bDest, retainInBMsg)\n\t\t\t// And \"A\" still has the \"Retain in A\" message.\n\t\t\tgetRetained(\"a\", \"foo/x\", retainInAMsg)\n\n\t\t\t// Publish from \"A\" a new message:\n\t\t\tretainInAMsg = \"Retain in A2\"\n\t\t\tpubRetained(\"a\", \"foo/x\", retainInAMsg)\n\t\t\t// Make sure that this retained appears on \"A\" and \"B\".\n\t\t\tgetRetained(\"a\", \"foo/x\", retainInAMsg)\n\t\t\tgetRetained(\"b\", test.bDest, retainInAMsg)\n\n\t\t\t// Now publish an empty body retained message from \"A\". This\n\t\t\t// should remove the retained message from both \"A\" and \"B\".\n\t\t\tpubRetained(\"a\", \"foo/x\", _EMPTY_)\n\n\t\t\t// We will check that the message gets removed from the stream.\n\t\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t_, err := js.GetLastMsg(mqttRetainedMsgsStreamName, test.getLastMsgSubj)\n\t\t\t\tif err == nats.ErrMsgNotFound {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Message still present or unexpected error %v\", err)\n\t\t\t})\n\t\t\t// The helper will use \"expect nothing\" if the given string is empty.\n\t\t\tgetRetained(\"a\", \"foo/x\", _EMPTY_)\n\t\t\tgetRetained(\"b\", test.bDest, _EMPTY_)\n\t\t})\n\t}\n}\n\n//////////////////////////////////////////////////////////////////////////\n//\n// Benchmarks\n//\n//////////////////////////////////////////////////////////////////////////\n\nconst (\n\tmqttPubSubj = \"a\"\n\tmqttBenchBufLen = 32768\n)\n\nfunc mqttBenchPubQoS0(b *testing.B, subject, payload string, numSubs int) {\n\tb.StopTimer()\n\tb.ReportAllocs()\n\to := testMQTTDefaultOptions()\n\ts := RunServer(o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{clientID: \"pub\", cleanSess: true}\n\tc, br := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(b, br, mqttConnAckRCConnectionAccepted, false)\n\tw := newMQTTWriter(0)\n\ttestMQTTWritePublishPacket(b, w, 0, false, false, subject, 0, []byte(payload))\n\tsendOp := w.Bytes()\n\n\tdch := make(chan error, 1)\n\ttotalSize := int64(len(sendOp))\n\tcdch := 0\n\n\tcreateSub := func(i int) {\n\t\tci := &mqttConnInfo{clientID: fmt.Sprintf(\"sub%d\", i), cleanSess: true}\n\t\tcs, brs := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port)\n\t\ttestMQTTCheckConnAck(b, brs, mqttConnAckRCConnectionAccepted, false)\n\n\t\ttestMQTTSub(b, 1, cs, brs, []*mqttFilter{{filter: subject, qos: 0}}, []byte{0})\n\t\ttestMQTTFlush(b, cs, nil, brs)\n\n\t\tw := newMQTTWriter(0)\n\t\tvarHeaderAndPayload := 2 + len(subject) + len(payload)\n\t\tw.WriteVarInt(varHeaderAndPayload)\n\t\tsize := 1 + w.Len() + varHeaderAndPayload\n\t\ttotalSize += int64(size)\n\n\t\tgo func() {\n\t\t\tmqttBenchConsumeMsgQoS0(cs, int64(b.N)*int64(size), dch)\n\t\t\tcs.Close()\n\t\t}()\n\t}\n\tfor i := 0; i < numSubs; i++ {\n\t\tcreateSub(i + 1)\n\t\tcdch++\n\t}\n\n\tbw := bufio.NewWriterSize(c, mqttBenchBufLen)\n\tb.SetBytes(totalSize)\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\ttestMQTTFlush(b, c, bw, br)\n\tfor i := 0; i < cdch; i++ {\n\t\tif e := <-dch; e != nil {\n\t\t\tb.Fatal(e.Error())\n\t\t}\n\t}\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc mqttBenchConsumeMsgQoS0(c net.Conn, total int64, dch chan<- error) {\n\tvar buf [mqttBenchBufLen]byte\n\tvar err error\n\tvar n int\n\tfor size := int64(0); size < total; {\n\t\tn, err = c.Read(buf[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tsize += int64(n)\n\t}\n\tdch <- err\n}\n\nfunc mqttBenchPubQoS1(b *testing.B, subject, payload string, numSubs int) {\n\tb.StopTimer()\n\to := testMQTTDefaultOptions()\n\to.MQTT.MaxAckPending = 0xFFFF\n\ts := RunServer(o)\n\tdefer testMQTTShutdownServer(s)\n\n\tci := &mqttConnInfo{cleanSess: true}\n\tc, br := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port)\n\ttestMQTTCheckConnAck(b, br, mqttConnAckRCConnectionAccepted, false)\n\n\tw := newMQTTWriter(0)\n\ttestMQTTWritePublishPacket(b, w, 1, false, false, subject, 1, []byte(payload))\n\t// For reported bytes we will count the PUBLISH + PUBACK (4 bytes)\n\ttotalSize := int64(w.Len() + 4)\n\tw.Reset()\n\n\tpi := uint16(1)\n\tmaxpi := uint16(60000)\n\tppich := make(chan error, 10)\n\tdch := make(chan error, 1+numSubs)\n\tcdch := 1\n\t// Start go routine to consume PUBACK for published QoS 1 messages.\n\tgo mqttBenchConsumePubAck(c, b.N, dch, ppich)\n\n\tcreateSub := func(i int) {\n\t\tci := &mqttConnInfo{clientID: fmt.Sprintf(\"sub%d\", i), cleanSess: true}\n\t\tcs, brs := testMQTTConnect(b, ci, o.MQTT.Host, o.MQTT.Port)\n\t\ttestMQTTCheckConnAck(b, brs, mqttConnAckRCConnectionAccepted, false)\n\n\t\ttestMQTTSub(b, 1, cs, brs, []*mqttFilter{{filter: subject, qos: 1}}, []byte{1})\n\t\ttestMQTTFlush(b, cs, nil, brs)\n\n\t\tw := newMQTTWriter(0)\n\t\tvarHeaderAndPayload := 2 + len(subject) + 2 + len(payload)\n\t\tw.WriteVarInt(varHeaderAndPayload)\n\t\tsize := 1 + w.Len() + varHeaderAndPayload\n\t\t// Add to the bytes reported the size of message sent to subscriber + PUBACK (4 bytes)\n\t\ttotalSize += int64(size + 4)\n\n\t\tgo func() {\n\t\t\tmqttBenchConsumeMsgQos1(cs, b.N, size, dch)\n\t\t\tcs.Close()\n\t\t}()\n\t}\n\tfor i := 0; i < numSubs; i++ {\n\t\tcreateSub(i + 1)\n\t\tcdch++\n\t}\n\n\tflush := func() {\n\t\tb.Helper()\n\t\tif _, err := c.Write(w.Bytes()); err != nil {\n\t\t\tb.Fatalf(\"Error on write: %v\", err)\n\t\t}\n\t\tw.Reset()\n\t}\n\n\tb.SetBytes(totalSize)\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif pi <= maxpi {\n\t\t\ttestMQTTWritePublishPacket(b, w, 1, false, false, subject, pi, []byte(payload))\n\t\t\tpi++\n\t\t\tif w.Len() >= mqttBenchBufLen {\n\t\t\t\tflush()\n\t\t\t}\n\t\t} else {\n\t\t\tif w.Len() > 0 {\n\t\t\t\tflush()\n\t\t\t}\n\t\t\tif pi > 60000 {\n\t\t\t\tpi = 1\n\t\t\t\tmaxpi = 0\n\t\t\t}\n\t\t\tif e := <-ppich; e != nil {\n\t\t\t\tb.Fatal(e.Error())\n\t\t\t}\n\t\t\tmaxpi += 10000\n\t\t\ti--\n\t\t}\n\t}\n\tif w.Len() > 0 {\n\t\tflush()\n\t}\n\tfor i := 0; i < cdch; i++ {\n\t\tif e := <-dch; e != nil {\n\t\t\tb.Fatal(e.Error())\n\t\t}\n\t}\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc mqttBenchConsumeMsgQos1(c net.Conn, total, size int, dch chan<- error) {\n\tvar buf [mqttBenchBufLen]byte\n\tpubAck := [4]byte{mqttPacketPubAck, 0x2, 0, 0}\n\tvar err error\n\tvar n int\n\tvar pi uint16\n\tvar prev int\n\tfor i := 0; i < total; {\n\t\tn, err = c.Read(buf[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tn += prev\n\t\tfor ; n >= size; n -= size {\n\t\t\ti++\n\t\t\tpi++\n\t\t\tpubAck[2] = byte(pi >> 8)\n\t\t\tpubAck[3] = byte(pi)\n\t\t\tif _, err = c.Write(pubAck[:4]); err != nil {\n\t\t\t\tdch <- err\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif pi == 60000 {\n\t\t\t\tpi = 0\n\t\t\t}\n\t\t}\n\t\tprev = n\n\t}\n\tdch <- err\n}\n\nfunc mqttBenchConsumePubAck(c net.Conn, total int, dch, ppich chan<- error) {\n\tvar buf [mqttBenchBufLen]byte\n\tvar err error\n\tvar n int\n\tvar pi uint16\n\tvar prev int\n\tfor i := 0; i < total; {\n\t\tn, err = c.Read(buf[:])\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tn += prev\n\t\tfor ; n >= 4; n -= 4 {\n\t\t\ti++\n\t\t\tpi++\n\t\t\tif pi%10000 == 0 {\n\t\t\t\tppich <- nil\n\t\t\t}\n\t\t\tif pi == 60001 {\n\t\t\t\tpi = 0\n\t\t\t}\n\t\t}\n\t\tprev = n\n\t}\n\tppich <- err\n\tdch <- err\n}\n\nfunc BenchmarkMQTT_QoS0_Pub_______0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, \"\", 0)\n}\n\nfunc BenchmarkMQTT_QoS0_Pub_______8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 0)\n}\n\nfunc BenchmarkMQTT_QoS0_Pub______32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 0)\n}\n\nfunc BenchmarkMQTT_QoS0_Pub_____128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 0)\n}\n\nfunc BenchmarkMQTT_QoS0_Pub_____256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 0)\n}\n\nfunc BenchmarkMQTT_QoS0_Pub_______1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 0)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1___0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, \"\", 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1___8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1__32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1_128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1_256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub1___1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 1)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2___0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, \"\", 2)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2___8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(8), 2)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2__32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(32), 2)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2_128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(128), 2)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2_256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(256), 2)\n}\n\nfunc BenchmarkMQTT_QoS0_PubSub2___1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS0(b, mqttPubSubj, sizedString(1024), 2)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub_______0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, \"\", 0)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub_______8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 0)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub______32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 0)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub_____128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 0)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub_____256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 0)\n}\n\nfunc BenchmarkMQTT_QoS1_Pub_______1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 0)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1___0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, \"\", 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1___8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1__32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1_128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1_256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub1___1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 1)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2___0b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, \"\", 2)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2___8b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(8), 2)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2__32b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(32), 2)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2_128b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(128), 2)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2_256b_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(256), 2)\n}\n\nfunc BenchmarkMQTT_QoS1_PubSub2___1K_Payload(b *testing.B) {\n\tmqttBenchPubQoS1(b, mqttPubSubj, sizedString(1024), 2)\n}\n" }, { "path": "server/msgtrace.go", "content": "// Copyright 2024-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nconst (\n\tMsgTraceDest = \"Nats-Trace-Dest\"\n\tMsgTraceDestDisabled = \"trace disabled\" // This must be an invalid NATS subject\n\tMsgTraceHop = \"Nats-Trace-Hop\"\n\tMsgTraceOriginAccount = \"Nats-Trace-Origin-Account\"\n\tMsgTraceOnly = \"Nats-Trace-Only\"\n\n\t// External trace header. Note that this header is normally in lower\n\t// case (https://www.w3.org/TR/trace-context/#header-name). Vendors\n\t// MUST expect the header in any case (upper, lower, mixed), and\n\t// SHOULD send the header name in lowercase. We used to change it\n\t// to lower case, but no longer do that in 2.14.\n\ttraceParentHdr = \"traceparent\"\n)\n\nvar (\n\ttraceDestHdrAsBytes = stringToBytes(MsgTraceDest)\n\ttraceDestDisabledAsBytes = stringToBytes(MsgTraceDestDisabled)\n\ttraceParentHdrAsBytes = stringToBytes(traceParentHdr)\n\tcrLFAsBytes = stringToBytes(CR_LF)\n\tdashAsBytes = stringToBytes(\"-\")\n)\n\ntype MsgTraceType string\n\n// Type of message trace events in the MsgTraceEvents list.\n// This is needed to unmarshal the list.\nconst (\n\tMsgTraceIngressType = \"in\"\n\tMsgTraceSubjectMappingType = \"sm\"\n\tMsgTraceStreamExportType = \"se\"\n\tMsgTraceServiceImportType = \"si\"\n\tMsgTraceJetStreamType = \"js\"\n\tMsgTraceEgressType = \"eg\"\n)\n\ntype MsgTraceEvent struct {\n\tServer ServerInfo `json:\"server\"`\n\tRequest MsgTraceRequest `json:\"request\"`\n\tHops int `json:\"hops,omitempty\"`\n\tEvents MsgTraceEvents `json:\"events\"`\n}\n\ntype MsgTraceRequest struct {\n\t// We are not making this an http.Header so that header name case is preserved.\n\tHeader map[string][]string `json:\"header,omitempty\"`\n\tMsgSize int `json:\"msgsize,omitempty\"`\n}\n\ntype MsgTraceEvents []MsgTrace\n\ntype MsgTrace interface {\n\tnew() MsgTrace\n\ttyp() MsgTraceType\n}\n\ntype MsgTraceBase struct {\n\tType MsgTraceType `json:\"type\"`\n\tTimestamp time.Time `json:\"ts\"`\n}\n\ntype MsgTraceIngress struct {\n\tMsgTraceBase\n\tKind int `json:\"kind\"`\n\tCID uint64 `json:\"cid\"`\n\tName string `json:\"name,omitempty\"`\n\tAccount string `json:\"acc\"`\n\tSubject string `json:\"subj\"`\n\tError string `json:\"error,omitempty\"`\n}\n\ntype MsgTraceSubjectMapping struct {\n\tMsgTraceBase\n\tMappedTo string `json:\"to\"`\n}\n\ntype MsgTraceStreamExport struct {\n\tMsgTraceBase\n\tAccount string `json:\"acc\"`\n\tTo string `json:\"to\"`\n}\n\ntype MsgTraceServiceImport struct {\n\tMsgTraceBase\n\tAccount string `json:\"acc\"`\n\tFrom string `json:\"from\"`\n\tTo string `json:\"to\"`\n}\n\ntype MsgTraceJetStream struct {\n\tMsgTraceBase\n\tStream string `json:\"stream\"`\n\tSubject string `json:\"subject,omitempty\"`\n\tNoInterest bool `json:\"nointerest,omitempty\"`\n\tError string `json:\"error,omitempty\"`\n}\n\ntype MsgTraceEgress struct {\n\tMsgTraceBase\n\tKind int `json:\"kind\"`\n\tCID uint64 `json:\"cid\"`\n\tName string `json:\"name,omitempty\"`\n\tHop string `json:\"hop,omitempty\"`\n\tAccount string `json:\"acc,omitempty\"`\n\tSubscription string `json:\"sub,omitempty\"`\n\tQueue string `json:\"queue,omitempty\"`\n\tError string `json:\"error,omitempty\"`\n\n\t// This is for applications that unmarshal the trace events\n\t// and want to link an egress to route/leaf/gateway with\n\t// the MsgTraceEvent from that server.\n\tLink *MsgTraceEvent `json:\"-\"`\n}\n\n// -------------------------------------------------------------\n\nfunc (t MsgTraceBase) typ() MsgTraceType { return t.Type }\nfunc (MsgTraceIngress) new() MsgTrace { return &MsgTraceIngress{} }\nfunc (MsgTraceSubjectMapping) new() MsgTrace { return &MsgTraceSubjectMapping{} }\nfunc (MsgTraceStreamExport) new() MsgTrace { return &MsgTraceStreamExport{} }\nfunc (MsgTraceServiceImport) new() MsgTrace { return &MsgTraceServiceImport{} }\nfunc (MsgTraceJetStream) new() MsgTrace { return &MsgTraceJetStream{} }\nfunc (MsgTraceEgress) new() MsgTrace { return &MsgTraceEgress{} }\n\nvar msgTraceInterfaces = map[MsgTraceType]MsgTrace{\n\tMsgTraceIngressType: MsgTraceIngress{},\n\tMsgTraceSubjectMappingType: MsgTraceSubjectMapping{},\n\tMsgTraceStreamExportType: MsgTraceStreamExport{},\n\tMsgTraceServiceImportType: MsgTraceServiceImport{},\n\tMsgTraceJetStreamType: MsgTraceJetStream{},\n\tMsgTraceEgressType: MsgTraceEgress{},\n}\n\nfunc (t *MsgTraceEvents) UnmarshalJSON(data []byte) error {\n\tvar raw []json.RawMessage\n\terr := json.Unmarshal(data, &raw)\n\tif err != nil {\n\t\treturn err\n\t}\n\t*t = make(MsgTraceEvents, len(raw))\n\tvar tt MsgTraceBase\n\tfor i, r := range raw {\n\t\tif err = json.Unmarshal(r, &tt); err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttr, ok := msgTraceInterfaces[tt.Type]\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"unknown trace type %v\", tt.Type)\n\t\t}\n\t\tte := tr.new()\n\t\tif err := json.Unmarshal(r, te); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t(*t)[i] = te\n\t}\n\treturn nil\n}\n\nfunc getTraceAs[T MsgTrace](e any) *T {\n\tv, ok := e.(*T)\n\tif ok {\n\t\treturn v\n\t}\n\treturn nil\n}\n\nfunc (t *MsgTraceEvent) Ingress() *MsgTraceIngress {\n\tif len(t.Events) < 1 {\n\t\treturn nil\n\t}\n\treturn getTraceAs[MsgTraceIngress](t.Events[0])\n}\n\nfunc (t *MsgTraceEvent) SubjectMapping() *MsgTraceSubjectMapping {\n\tfor _, e := range t.Events {\n\t\tif e.typ() == MsgTraceSubjectMappingType {\n\t\t\treturn getTraceAs[MsgTraceSubjectMapping](e)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (t *MsgTraceEvent) StreamExports() []*MsgTraceStreamExport {\n\tvar se []*MsgTraceStreamExport\n\tfor _, e := range t.Events {\n\t\tif e.typ() == MsgTraceStreamExportType {\n\t\t\tse = append(se, getTraceAs[MsgTraceStreamExport](e))\n\t\t}\n\t}\n\treturn se\n}\n\nfunc (t *MsgTraceEvent) ServiceImports() []*MsgTraceServiceImport {\n\tvar si []*MsgTraceServiceImport\n\tfor _, e := range t.Events {\n\t\tif e.typ() == MsgTraceServiceImportType {\n\t\t\tsi = append(si, getTraceAs[MsgTraceServiceImport](e))\n\t\t}\n\t}\n\treturn si\n}\n\nfunc (t *MsgTraceEvent) JetStream() *MsgTraceJetStream {\n\tfor _, e := range t.Events {\n\t\tif e.typ() == MsgTraceJetStreamType {\n\t\t\treturn getTraceAs[MsgTraceJetStream](e)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (t *MsgTraceEvent) Egresses() []*MsgTraceEgress {\n\tvar eg []*MsgTraceEgress\n\tfor _, e := range t.Events {\n\t\tif e.typ() == MsgTraceEgressType {\n\t\t\teg = append(eg, getTraceAs[MsgTraceEgress](e))\n\t\t}\n\t}\n\treturn eg\n}\n\nconst (\n\terrMsgTraceOnlyNoSupport = \"Not delivered because remote does not support message tracing\"\n\terrMsgTraceNoSupport = \"Message delivered but remote does not support message tracing so no trace event generated from there\"\n\terrMsgTraceNoEcho = \"Not delivered because of no echo\"\n\terrMsgTracePubViolation = \"Not delivered because publish denied for this subject\"\n\terrMsgTraceSubDeny = \"Not delivered because subscription denies this subject\"\n\terrMsgTraceSubClosed = \"Not delivered because subscription is closed\"\n\terrMsgTraceClientClosed = \"Not delivered because client is closed\"\n\terrMsgTraceAutoSubExceeded = \"Not delivered because auto-unsubscribe exceeded\"\n\terrMsgTraceFastProdNoStall = \"Not delivered because fast producer not stalled and consumer is slow\"\n)\n\ntype msgTrace struct {\n\tready int32\n\tsrv *Server\n\tacc *Account\n\t// Origin account name, set only if acc is nil when acc lookup failed.\n\toan string\n\tdest string\n\tevent *MsgTraceEvent\n\tjs *MsgTraceJetStream\n\thop string\n\tnhop string\n\ttonly bool // Will only trace the message, not do delivery.\n\tct compressionType\n}\n\n// This will be false outside of the tests, so when building the server binary,\n// any code where you see `if msgTraceRunInTests` statement will be compiled\n// out, so this will have no performance penalty.\nvar (\n\tmsgTraceRunInTests bool\n\tmsgTraceCheckSupport bool\n)\n\n// Returns the message trace object, if message is being traced,\n// and `true` if we want to only trace, not actually deliver the message.\nfunc (c *client) isMsgTraceEnabled() (*msgTrace, bool) {\n\tt := c.pa.trace\n\tif t == nil {\n\t\treturn nil, false\n\t}\n\treturn t, t.tonly\n}\n\n// For LEAF/ROUTER/GATEWAY, return false if the remote does not support\n// message tracing (important if the tracing requests trace-only).\nfunc (c *client) msgTraceSupport() bool {\n\t// Exclude client connection from the protocol check.\n\treturn c.kind == CLIENT || c.opts.Protocol >= MsgTraceProto\n}\n\nfunc getConnName(c *client) string {\n\tswitch c.kind {\n\tcase ROUTER:\n\t\tif n := c.route.remoteName; n != _EMPTY_ {\n\t\t\treturn n\n\t\t}\n\tcase GATEWAY:\n\t\tif n := c.gw.remoteName; n != _EMPTY_ {\n\t\t\treturn n\n\t\t}\n\tcase LEAF:\n\t\tif n := c.leaf.remoteServer; n != _EMPTY_ {\n\t\t\treturn n\n\t\t}\n\t}\n\treturn c.opts.Name\n}\n\nfunc getCompressionType(cts string) compressionType {\n\tif cts == _EMPTY_ {\n\t\treturn noCompression\n\t}\n\tcts = strings.ToLower(cts)\n\tif strings.Contains(cts, \"snappy\") || strings.Contains(cts, \"s2\") {\n\t\treturn snappyCompression\n\t}\n\tif strings.Contains(cts, \"gzip\") {\n\t\treturn gzipCompression\n\t}\n\treturn unsupportedCompression\n}\n\nfunc (c *client) initMsgTrace() *msgTrace {\n\t// The code in the \"if\" statement is only running in test mode.\n\tif msgTraceRunInTests {\n\t\t// Check the type of client that tries to initialize a trace struct.\n\t\tif !(c.kind == CLIENT || c.kind == ROUTER || c.kind == GATEWAY || c.kind == LEAF) {\n\t\t\tpanic(fmt.Sprintf(\"Unexpected client type %q trying to initialize msgTrace\", c.kindString()))\n\t\t}\n\t\t// In some tests, we want to make a server behave like an old server\n\t\t// and so even if a trace header is received, we want the server to\n\t\t// simply ignore it.\n\t\tif msgTraceCheckSupport {\n\t\t\tif c.srv == nil || c.srv.getServerProto() < MsgTraceProto {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\tif c.pa.hdr <= 0 {\n\t\treturn nil\n\t}\n\thdr := c.msgBuf[:c.pa.hdr]\n\theaders, external := genHeaderMapIfTraceHeadersPresent(hdr)\n\tif len(headers) == 0 {\n\t\treturn nil\n\t}\n\t// Little helper to give us the first value of a given header, or _EMPTY_\n\t// if key is not present.\n\tgetHdrVal := func(key string) string {\n\t\tvv, ok := headers[key]\n\t\tif !ok {\n\t\t\treturn _EMPTY_\n\t\t}\n\t\treturn vv[0]\n\t}\n\tvar (\n\t\tdest string\n\t\ttraceOnly bool\n\t)\n\t// Check for traceOnly only if not external.\n\tif !external {\n\t\tif to := getHdrVal(MsgTraceOnly); to != _EMPTY_ {\n\t\t\ttos := strings.ToLower(to)\n\t\t\tswitch tos {\n\t\t\tcase \"1\", \"true\", \"on\":\n\t\t\t\ttraceOnly = true\n\t\t\t}\n\t\t}\n\t\tdest = getHdrVal(MsgTraceDest)\n\t\t// Check the destination to see if this is a valid public subject.\n\t\tif !IsValidPublishSubject(dest) {\n\t\t\t// We still have to return a msgTrace object (if traceOnly is set)\n\t\t\t// because if we don't, the message will end-up being delivered to\n\t\t\t// applications, which may break them. We report the error in any case.\n\t\t\tc.Errorf(\"Destination %q is not valid, won't be able to trace events\", dest)\n\t\t\tif !traceOnly {\n\t\t\t\t// We can bail, tracing will be disabled for this message.\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\tvar (\n\t\t// Account to use when sending the trace event\n\t\tacc *Account\n\t\t// Ingress' account name\n\t\tian string\n\t\t// Origin account name\n\t\toan string\n\t\t// The hop \"id\", taken from headers only when not from CLIENT\n\t\thop string\n\t)\n\tif c.kind == ROUTER || c.kind == GATEWAY || c.kind == LEAF {\n\t\t// The ingress account name will always be c.pa.account, but `acc` may\n\t\t// be different if we have an origin account header.\n\t\tif c.kind == LEAF {\n\t\t\tian = c.acc.GetName()\n\t\t} else {\n\t\t\tian = string(c.pa.account)\n\t\t}\n\t\t// The remote will have set the origin account header only if the\n\t\t// message changed account (think of service imports).\n\t\toan = getHdrVal(MsgTraceOriginAccount)\n\t\tif oan == _EMPTY_ {\n\t\t\t// For LEAF or ROUTER with pinned-account, we can use the c.acc.\n\t\t\tif c.kind == LEAF || (c.kind == ROUTER && len(c.route.accName) > 0) {\n\t\t\t\tacc = c.acc\n\t\t\t} else {\n\t\t\t\t// We will lookup account with c.pa.account (or ian).\n\t\t\t\toan = ian\n\t\t\t}\n\t\t}\n\t\t// Unless we already got the account, we need to look it up.\n\t\tif acc == nil {\n\t\t\t// We don't want to do account resolving here.\n\t\t\tif acci, ok := c.srv.accounts.Load(oan); ok {\n\t\t\t\tacc = acci.(*Account)\n\t\t\t\t// Since we have looked-up the account, we don't need oan, so\n\t\t\t\t// clear it in case it was set.\n\t\t\t\toan = _EMPTY_\n\t\t\t} else {\n\t\t\t\t// We still have to return a msgTrace object (if traceOnly is set)\n\t\t\t\t// because if we don't, the message will end-up being delivered to\n\t\t\t\t// applications, which may break them. We report the error in any case.\n\t\t\t\tc.Errorf(\"Account %q was not found, won't be able to trace events\", oan)\n\t\t\t\tif !traceOnly {\n\t\t\t\t\t// We can bail, tracing will be disabled for this message.\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Check the hop header\n\t\thop = getHdrVal(MsgTraceHop)\n\t} else {\n\t\tacc = c.acc\n\t\tian = acc.GetName()\n\t}\n\t// If external, we need to have the account's trace destination set,\n\t// otherwise, we are not enabling tracing.\n\tif external {\n\t\tvar sampling int\n\t\tif acc != nil {\n\t\t\tdest, sampling = acc.getTraceDestAndSampling()\n\t\t}\n\t\tif dest == _EMPTY_ {\n\t\t\t// No account destination, no tracing for external trace headers.\n\t\t\treturn nil\n\t\t}\n\t\t// Check sampling, but only from origin server.\n\t\tif c.kind == CLIENT && !sample(sampling) {\n\t\t\t// Need to disable tracing so that if the message is routed, it won't\n\t\t\t// trigger a trace there.\n\t\t\tc.msgBuf = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, c.msgBuf)\n\t\t\treturn nil\n\t\t}\n\t}\n\tc.pa.trace = &msgTrace{\n\t\tsrv: c.srv,\n\t\tacc: acc,\n\t\toan: oan,\n\t\tdest: dest,\n\t\tct: getCompressionType(getHdrVal(acceptEncodingHeader)),\n\t\thop: hop,\n\t\tevent: &MsgTraceEvent{\n\t\t\tRequest: MsgTraceRequest{\n\t\t\t\tHeader: headers,\n\t\t\t\tMsgSize: c.pa.size,\n\t\t\t},\n\t\t\tEvents: append(MsgTraceEvents(nil), &MsgTraceIngress{\n\t\t\t\tMsgTraceBase: MsgTraceBase{\n\t\t\t\t\tType: MsgTraceIngressType,\n\t\t\t\t\tTimestamp: time.Now(),\n\t\t\t\t},\n\t\t\t\tKind: c.kind,\n\t\t\t\tCID: c.cid,\n\t\t\t\tName: getConnName(c),\n\t\t\t\tAccount: ian,\n\t\t\t\tSubject: string(c.pa.subject),\n\t\t\t}),\n\t\t},\n\t\ttonly: traceOnly,\n\t}\n\treturn c.pa.trace\n}\n\nfunc sample(sampling int) bool {\n\t// Option parsing should ensure that sampling is [1..100], but consider\n\t// any value outside of this range to be 100%.\n\tif sampling <= 0 || sampling >= 100 {\n\t\treturn true\n\t}\n\treturn rand.Int31n(100) <= int32(sampling)\n}\n\n// This function will return the header as a map (instead of http.Header because\n// we want to preserve the header names' case) and a boolean that indicates if\n// the headers have been lifted due to the presence of the external trace header\n// only.\n// Note that because of the traceParentHdr, the search is done in a case\n// insensitive way. We used to rewrite it in lower case but no longer do since v2.14.\nfunc genHeaderMapIfTraceHeadersPresent(hdr []byte) (map[string][]string, bool) {\n\n\tvar (\n\t\t_keys = [64][]byte{}\n\t\t_vals = [64][]byte{}\n\t\tm map[string][]string\n\t\ttraceDestHdrFound bool\n\t\ttraceParentHdrFound bool\n\t)\n\t// Skip the hdrLine\n\tif !bytes.HasPrefix(hdr, stringToBytes(hdrLine)) {\n\t\treturn nil, false\n\t}\n\n\tkeys := _keys[:0]\n\tvals := _vals[:0]\n\n\tfor i := len(hdrLine); i < len(hdr); {\n\t\t// Search for key/val delimiter\n\t\tdel := bytes.IndexByte(hdr[i:], ':')\n\t\tif del < 0 {\n\t\t\tbreak\n\t\t}\n\t\tkeyStart := i\n\t\tkey := hdr[keyStart : keyStart+del]\n\t\ti += del + 1\n\t\tfor i < len(hdr) && (hdr[i] == ' ' || hdr[i] == '\\t') {\n\t\t\ti++\n\t\t}\n\t\tvalStart := i\n\t\tnl := bytes.Index(hdr[valStart:], crLFAsBytes)\n\t\tif nl < 0 {\n\t\t\tbreak\n\t\t}\n\t\tvalEnd := valStart + nl\n\t\tfor valEnd > valStart && (hdr[valEnd-1] == ' ' || hdr[valEnd-1] == '\\t') {\n\t\t\tvalEnd--\n\t\t}\n\t\tval := hdr[valStart:valEnd]\n\t\tif len(key) > 0 && len(val) > 0 {\n\t\t\tvals = append(vals, val)\n\n\t\t\t// We search for our special keys only if not already found.\n\n\t\t\t// Check for the external trace header.\n\t\t\t// Search needs to be case insensitive.\n\t\t\tif !traceParentHdrFound && bytes.EqualFold(key, traceParentHdrAsBytes) {\n\t\t\t\t// We will now check if the value has sampling or not.\n\t\t\t\t// TODO(ik): Not sure if this header can have multiple values\n\t\t\t\t// or not, and if so, what would be the rule to check for\n\t\t\t\t// sampling. What is done here is to check them all until we\n\t\t\t\t// found one with sampling.\n\t\t\t\ttk := bytes.Split(val, dashAsBytes)\n\t\t\t\tif len(tk) == 4 && len([]byte(tk[3])) == 2 {\n\t\t\t\t\tif hexVal, err := strconv.ParseInt(bytesToString(tk[3]), 16, 8); err == nil {\n\t\t\t\t\t\tif hexVal&0x1 == 0x1 {\n\t\t\t\t\t\t\ttraceParentHdrFound = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if !traceDestHdrFound && bytes.Equal(key, traceDestHdrAsBytes) {\n\t\t\t\t// This is the Nats-Trace-Dest header, check the value to see\n\t\t\t\t// if it indicates that the trace was disabled.\n\t\t\t\tif bytes.Equal(val, traceDestDisabledAsBytes) {\n\t\t\t\t\treturn nil, false\n\t\t\t\t}\n\t\t\t\ttraceDestHdrFound = true\n\t\t\t}\n\t\t\t// Add to the keys and preserve the key's case\n\t\t\tkeys = append(keys, key)\n\t\t}\n\t\ti += nl + 2\n\t}\n\tif !traceDestHdrFound && !traceParentHdrFound {\n\t\treturn nil, false\n\t}\n\tm = make(map[string][]string, len(keys))\n\tfor i, k := range keys {\n\t\thname := string(k)\n\t\tm[hname] = append(m[hname], string(vals[i]))\n\t}\n\treturn m, !traceDestHdrFound && traceParentHdrFound\n}\n\n// Special case where we create a trace event before parsing the message.\n// This is for cases where the connection will be closed when detecting\n// an error during early message processing (for instance max payload).\nfunc (c *client) initAndSendIngressErrEvent(hdr []byte, dest string, ingressError error) {\n\tif ingressError == nil {\n\t\treturn\n\t}\n\tct := getAcceptEncoding(hdr)\n\tt := &msgTrace{\n\t\tsrv: c.srv,\n\t\tacc: c.acc,\n\t\tdest: dest,\n\t\tct: ct,\n\t\tevent: &MsgTraceEvent{\n\t\t\tRequest: MsgTraceRequest{MsgSize: c.pa.size},\n\t\t\tEvents: append(MsgTraceEvents(nil), &MsgTraceIngress{\n\t\t\t\tMsgTraceBase: MsgTraceBase{\n\t\t\t\t\tType: MsgTraceIngressType,\n\t\t\t\t\tTimestamp: time.Now(),\n\t\t\t\t},\n\t\t\t\tKind: c.kind,\n\t\t\t\tCID: c.cid,\n\t\t\t\tName: getConnName(c),\n\t\t\t\tError: ingressError.Error(),\n\t\t\t}),\n\t\t},\n\t}\n\tt.sendEvent()\n}\n\n// Returns `true` if message tracing is enabled and we are tracing only,\n// that is, we are not going to deliver the inbound message, returns\n// `false` otherwise (no tracing, or tracing and message delivery).\nfunc (t *msgTrace) traceOnly() bool {\n\treturn t != nil && t.tonly\n}\n\nfunc (t *msgTrace) setOriginAccountHeaderIfNeeded(c *client, acc *Account, msg []byte) []byte {\n\tvar oan string\n\t// If t.acc is set, only check that, not t.oan.\n\tif t.acc != nil {\n\t\tif t.acc != acc {\n\t\t\toan = t.acc.GetName()\n\t\t}\n\t} else if t.oan != acc.GetName() {\n\t\toan = t.oan\n\t}\n\tif oan != _EMPTY_ {\n\t\tmsg = c.setHeader(MsgTraceOriginAccount, oan, msg)\n\t}\n\treturn msg\n}\n\nfunc (t *msgTrace) setHopHeader(c *client, msg []byte) []byte {\n\te := t.event\n\te.Hops++\n\tif len(t.hop) > 0 {\n\t\tt.nhop = fmt.Sprintf(\"%s.%d\", t.hop, e.Hops)\n\t} else {\n\t\tt.nhop = fmt.Sprintf(\"%d\", e.Hops)\n\t}\n\treturn c.setHeader(MsgTraceHop, t.nhop, msg)\n}\n\nfunc (t *msgTrace) setIngressError(err string) {\n\tif i := t.event.Ingress(); i != nil {\n\t\ti.Error = err\n\t}\n}\n\nfunc (t *msgTrace) addSubjectMappingEvent(subj []byte) {\n\tif t == nil {\n\t\treturn\n\t}\n\tt.event.Events = append(t.event.Events, &MsgTraceSubjectMapping{\n\t\tMsgTraceBase: MsgTraceBase{\n\t\t\tType: MsgTraceSubjectMappingType,\n\t\t\tTimestamp: time.Now(),\n\t\t},\n\t\tMappedTo: string(subj),\n\t})\n}\n\nfunc (t *msgTrace) addEgressEvent(dc *client, sub *subscription, err string) {\n\tif t == nil {\n\t\treturn\n\t}\n\te := &MsgTraceEgress{\n\t\tMsgTraceBase: MsgTraceBase{\n\t\t\tType: MsgTraceEgressType,\n\t\t\tTimestamp: time.Now(),\n\t\t},\n\t\tKind: dc.kind,\n\t\tCID: dc.cid,\n\t\tName: getConnName(dc),\n\t\tHop: t.nhop,\n\t\tError: err,\n\t}\n\tt.nhop = _EMPTY_\n\t// Specific to CLIENT connections...\n\tif dc.kind == CLIENT {\n\t\t// Set the subscription's subject and possibly queue name.\n\t\te.Subscription = string(sub.subject)\n\t\tif len(sub.queue) > 0 {\n\t\t\te.Queue = string(sub.queue)\n\t\t}\n\t}\n\tif dc.kind == CLIENT || dc.kind == LEAF {\n\t\tif i := t.event.Ingress(); i != nil {\n\t\t\t// If the Ingress' account is different from the destination's\n\t\t\t// account, add the account name into the Egress trace event.\n\t\t\t// This would happen with service imports.\n\t\t\tif dcAccName := dc.acc.GetName(); dcAccName != i.Account {\n\t\t\t\te.Account = dcAccName\n\t\t\t}\n\t\t}\n\t}\n\tt.event.Events = append(t.event.Events, e)\n}\n\nfunc (t *msgTrace) addStreamExportEvent(dc *client, to []byte) {\n\tif t == nil {\n\t\treturn\n\t}\n\tdc.mu.Lock()\n\taccName := dc.acc.GetName()\n\tdc.mu.Unlock()\n\tt.event.Events = append(t.event.Events, &MsgTraceStreamExport{\n\t\tMsgTraceBase: MsgTraceBase{\n\t\t\tType: MsgTraceStreamExportType,\n\t\t\tTimestamp: time.Now(),\n\t\t},\n\t\tAccount: accName,\n\t\tTo: string(to),\n\t})\n}\n\nfunc (t *msgTrace) addServiceImportEvent(accName, from, to string) {\n\tif t == nil {\n\t\treturn\n\t}\n\tt.event.Events = append(t.event.Events, &MsgTraceServiceImport{\n\t\tMsgTraceBase: MsgTraceBase{\n\t\t\tType: MsgTraceServiceImportType,\n\t\t\tTimestamp: time.Now(),\n\t\t},\n\t\tAccount: accName,\n\t\tFrom: from,\n\t\tTo: to,\n\t})\n}\n\nfunc (t *msgTrace) addJetStreamEvent(streamName string) {\n\tif t == nil {\n\t\treturn\n\t}\n\tt.js = &MsgTraceJetStream{\n\t\tMsgTraceBase: MsgTraceBase{\n\t\t\tType: MsgTraceJetStreamType,\n\t\t\tTimestamp: time.Now(),\n\t\t},\n\t\tStream: streamName,\n\t}\n\tt.event.Events = append(t.event.Events, t.js)\n}\n\nfunc (t *msgTrace) updateJetStreamEvent(subject string, noInterest bool) {\n\tif t == nil {\n\t\treturn\n\t}\n\t// JetStream event should have been created in addJetStreamEvent\n\tif t.js == nil {\n\t\treturn\n\t}\n\tt.js.Subject = subject\n\tt.js.NoInterest = noInterest\n\t// Update the timestamp since this is more accurate than when it\n\t// was first added in addJetStreamEvent().\n\tt.js.Timestamp = time.Now()\n}\n\nfunc (t *msgTrace) sendEventFromJetStream(err error) {\n\tif t == nil {\n\t\treturn\n\t}\n\t// JetStream event should have been created in addJetStreamEvent\n\tif t.js == nil {\n\t\treturn\n\t}\n\tif err != nil {\n\t\tt.js.Error = err.Error()\n\t}\n\tt.sendEvent()\n}\n\nfunc (t *msgTrace) sendEvent() {\n\tif t == nil {\n\t\treturn\n\t}\n\tif t.js != nil {\n\t\tready := atomic.AddInt32(&t.ready, 1) == 2\n\t\tif !ready {\n\t\t\treturn\n\t\t}\n\t}\n\tt.srv.sendInternalAccountSysMsg(t.acc, t.dest, &t.event.Server, t.event, t.ct)\n}\n" }, { "path": "server/msgtrace_test.go", "content": "// Copyright 2024-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_msgtrace_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc init() {\n\tmsgTraceRunInTests = true\n}\n\nfunc TestMsgTraceConnName(t *testing.T) {\n\tc := &client{kind: ROUTER, route: &route{remoteName: \"somename\"}}\n\tc.opts.Name = \"someid\"\n\n\t// If route.remoteName is set, it will take precedence.\n\tval := getConnName(c)\n\trequire_Equal[string](t, val, \"somename\")\n\t// When not set, we revert to c.opts.Name\n\tc.route.remoteName = _EMPTY_\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"someid\")\n\n\t// Now same for GW.\n\tc.route = nil\n\tc.gw = &gateway{remoteName: \"somename\"}\n\tc.kind = GATEWAY\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"somename\")\n\t// Revert to c.opts.Name\n\tc.gw.remoteName = _EMPTY_\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"someid\")\n\n\t// For LeafNode now\n\tc.gw = nil\n\tc.leaf = &leaf{remoteServer: \"somename\"}\n\tc.kind = LEAF\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"somename\")\n\t// But if not set...\n\tc.leaf.remoteServer = _EMPTY_\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"someid\")\n\n\tc.leaf = nil\n\tc.kind = CLIENT\n\tval = getConnName(c)\n\trequire_Equal[string](t, val, \"someid\")\n}\n\nfunc TestMsgTraceGenHeaderMap(t *testing.T) {\n\ttraceparentDifferentCase := \"TrAcEpArEnT\"\n\tfor _, test := range []struct {\n\t\tname string\n\t\theader []byte\n\t\texpected map[string][]string\n\t\texternal bool\n\t}{\n\t\t{\"missing header line\", []byte(\"Nats-Trace-Dest: val\\r\\n\"), nil, false},\n\t\t{\"no trace header present\", []byte(hdrLine + \"Header1: val1\\r\\nHeader2: val2\\r\\n\"), nil, false},\n\t\t{\"trace header with some prefix\", []byte(hdrLine + \"Some-Prefix-\" + MsgTraceDest + \": some value\\r\\n\"), nil, false},\n\t\t{\"trace header with some suffix\", []byte(hdrLine + MsgTraceDest + \"-Some-Suffix: some value\\r\\n\"), nil, false},\n\t\t{\"trace header with space before colon\", []byte(hdrLine + MsgTraceDest + \" : some value\\r\\n\"), nil, false},\n\t\t{\"trace header with missing cr_lf for value\", []byte(hdrLine + MsgTraceDest + \": bogus\"), nil, false},\n\t\t{\"trace header with empty value\", []byte(hdrLine + MsgTraceDest + \": \\r\\n\"), nil, false},\n\t\t{\"external trace header with some prefix\", []byte(hdrLine + \"Some-Prefix-\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\n\"), nil, false},\n\t\t{\"external trace header with some suffix\", []byte(hdrLine + traceParentHdr + \"-Some-Suffix: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\n\"), nil, false},\n\t\t{\"external header with space before colon\", []byte(hdrLine + traceParentHdr + \" : 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\n\"), nil, false},\n\t\t{\"external header with missing cr_lf for value\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"), nil, false},\n\t\t{\"external header with empty value\", []byte(hdrLine + traceParentHdr + \": \\r\\n\"), nil, false},\n\t\t{\"trace header first\", []byte(hdrLine + MsgTraceDest + \": some.dest\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\"}}, false},\n\t\t{\"trace header last\", []byte(hdrLine + \"Some-Header: some value\\r\\n\" + MsgTraceDest + \": some.dest\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\"}}, false},\n\t\t{\"trace header multiple values\", []byte(hdrLine + MsgTraceDest + \": some.dest\\r\\nSome-Header: some value\\r\\n\" + MsgTraceDest + \": some.dest.2\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\", \"some.dest.2\"}}, false},\n\t\t{\"trace header and some empty key\", []byte(hdrLine + MsgTraceDest + \": some.dest\\r\\n: bogus\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\"}}, false},\n\t\t{\"trace header and some header missing cr_lf for value\", []byte(hdrLine + MsgTraceDest + \": some.dest\\r\\nSome-Header: bogus\"),\n\t\t\tmap[string][]string{MsgTraceDest: {\"some.dest\"}}, false},\n\t\t{\"trace header and trims value\", []byte(hdrLine + MsgTraceDest + \": some.dest \\r\\n\"),\n\t\t\tmap[string][]string{MsgTraceDest: {\"some.dest\"}}, false},\n\t\t{\"trace header and external after\", []byte(hdrLine + MsgTraceDest + \": some.dest\\r\\n\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, false},\n\t\t{\"trace header and external before\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\n\" + MsgTraceDest + \": some.dest\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, MsgTraceDest: {\"some.dest\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, false},\n\t\t{\"external malformed\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-01\\r\\n\"), nil, false},\n\t\t{\"external first and sampling\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, true},\n\t\t{\"external middle and sampling\", []byte(hdrLine + \"Some-Header: some value1\\r\\n\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\nSome-Header: some value2\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value1\", \"some value2\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, true},\n\t\t{\"external last and sampling\", []byte(hdrLine + \"Some-Header: some value\\r\\n\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, true},\n\t\t{\"external sampling with not just 01\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-27\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-27\"}}, true},\n\t\t{\"external trims value\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01 \\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, traceParentHdr: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, true},\n\t\t{\"external with different case and sampling\", []byte(hdrLine + traceparentDifferentCase + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\\r\\nSome-Header: some value\\r\\n\"),\n\t\t\tmap[string][]string{\"Some-Header\": {\"some value\"}, traceparentDifferentCase: {\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}, true},\n\t\t{\"external first and not sampling\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\\r\\nSome-Header: some value\\r\\n\"), nil, false},\n\t\t{\"external middle and not sampling\", []byte(hdrLine + \"Some-Header: some value1\\r\\n\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\\r\\nSome-Header: some value2\\r\\n\"), nil, false},\n\t\t{\"external last and not sampling\", []byte(hdrLine + \"Some-Header: some value\\r\\n\" + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\\r\\n\"), nil, false},\n\t\t{\"external not sampling with not just 00\", []byte(hdrLine + traceParentHdr + \": 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-22\\r\\nSome-Header: some value\\r\\n\"), nil, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tm, ext := genHeaderMapIfTraceHeadersPresent(test.header)\n\t\t\tif test.external != ext {\n\t\t\t\tt.Fatalf(\"Expected external to be %v, got %v\", test.external, ext)\n\t\t\t}\n\t\t\tif len(test.expected) != len(m) {\n\t\t\t\tt.Fatalf(\"Expected map to be of size %v, got %v\", len(test.expected), len(m))\n\t\t\t}\n\t\t\t// If external, we should find traceParentHdr\n\t\t\tif test.external {\n\t\t\t\theaderName := traceParentHdr\n\t\t\t\tif _, ok := m[headerName]; !ok {\n\t\t\t\t\t// There is a test where we use different case, so check for that one too.\n\t\t\t\t\theaderName = traceparentDifferentCase\n\t\t\t\t\tif _, ok := m[headerName]; !ok {\n\t\t\t\t\t\tt.Fatalf(\"Expected traceparent header to be present, it was not: %+v\", m)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// We no longer rewrite the header, so take that into consideration.\n\t\t\t\tif !bytes.Contains(test.header, []byte(headerName)) {\n\t\t\t\t\tt.Fatalf(\"Header should have been rewritten to have the traceparent in lower case: %s\", test.header)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor k, vv := range m {\n\t\t\t\tevv, ok := test.expected[k]\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"Did not find header %q in resulting map: %+v\", k, m)\n\t\t\t\t}\n\t\t\t\tfor i, v := range vv {\n\t\t\t\t\tif evv[i] != v {\n\t\t\t\t\t\tt.Fatalf(\"Expected value %v of key %q to be %q, got %q\", i, k, evv[i], v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceBasic(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tmappings = {\n\t\t\tfoo: bar\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tcid, err := nc.GetClientID()\n\trequire_NoError(t, err)\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\t// Send trace message to a dummy subject to check that resulting trace's\n\t// SubjectMapping and Egress are nil.\n\tmsg := nats.NewMsg(\"dummy\")\n\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\tmsg.Data = []byte(\"hello!\")\n\terr = nc.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\tvar e MsgTraceEvent\n\tjson.Unmarshal(traceMsg.Data, &e)\n\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t// We don't remove the headers, so we will find the tracing header there.\n\trequire_True(t, e.Request.Header != nil)\n\trequire_Equal[int](t, len(e.Request.Header), 2)\n\t// The message size is 6 + whatever size for the 2 trace headers.\n\t// Let's just make sure that size is > 20...\n\trequire_True(t, e.Request.MsgSize > 20)\n\tingress := e.Ingress()\n\trequire_True(t, ingress != nil)\n\trequire_True(t, ingress.Kind == CLIENT)\n\trequire_True(t, ingress.Timestamp != time.Time{})\n\trequire_Equal[uint64](t, ingress.CID, cid)\n\trequire_Equal[string](t, ingress.Name, _EMPTY_)\n\trequire_Equal[string](t, ingress.Account, globalAccountName)\n\trequire_Equal[string](t, ingress.Subject, \"dummy\")\n\trequire_Equal[string](t, ingress.Error, _EMPTY_)\n\trequire_True(t, e.SubjectMapping() == nil)\n\trequire_True(t, e.StreamExports() == nil)\n\trequire_True(t, e.ServiceImports() == nil)\n\trequire_True(t, e.JetStream() == nil)\n\trequire_True(t, e.Egresses() == nil)\n\n\t// Now setup subscriptions that generate interest on the subject.\n\tnc2 := natsConnect(t, s.ClientURL(), nats.Name(\"sub1And2\"))\n\tdefer nc2.Close()\n\tsub1 := natsSubSync(t, nc2, \"bar\")\n\tsub2 := natsSubSync(t, nc2, \"bar\")\n\tnatsFlush(t, nc2)\n\tnc2CID, _ := nc2.GetClientID()\n\n\tnc3 := natsConnect(t, s.ClientURL())\n\tdefer nc3.Close()\n\tsub3 := natsSubSync(t, nc3, \"*\")\n\tnatsFlush(t, nc3)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg = nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(\"Some-App-Header\", \"some value\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr = nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tif expected {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t\t// We don't remove message trace header, so we should have\n\t\t\t\t\t// 2 headers (the app + trace destination)\n\t\t\t\t\trequire_True(t, len(appMsg.Header) == 2)\n\t\t\t\t\trequire_Equal[string](t, appMsg.Header.Get(\"Some-App-Header\"), \"some value\")\n\t\t\t\t\trequire_Equal[string](t, appMsg.Header.Get(MsgTraceDest), traceSub.Subject)\n\t\t\t\t}\n\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2, sub3} {\n\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t}\n\n\t\t\ttraceMsg = natsNexMsg(t, traceSub, time.Second)\n\t\t\te = MsgTraceEvent{}\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\trequire_True(t, e.Request.Header != nil)\n\t\t\t// We should have the app header and the trace header(s) too.\n\t\t\texpected := 2\n\t\t\tif !test.deliverMsg {\n\t\t\t\t// The \"trace-only\" header is added.\n\t\t\t\texpected++\n\t\t\t}\n\t\t\trequire_Equal[int](t, len(e.Request.Header), expected)\n\t\t\trequire_Equal[string](t, e.Request.Header[\"Some-App-Header\"][0], \"some value\")\n\t\t\t// The message size is 6 + whatever size for the 2 trace headers.\n\t\t\t// Let's just make sure that size is > 20...\n\t\t\trequire_True(t, e.Request.MsgSize > 20)\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\trequire_True(t, ingress.Timestamp != time.Time{})\n\t\t\trequire_Equal[string](t, ingress.Account, globalAccountName)\n\t\t\trequire_Equal[string](t, ingress.Subject, \"foo\")\n\t\t\tsm := e.SubjectMapping()\n\t\t\trequire_True(t, sm != nil)\n\t\t\trequire_True(t, sm.Timestamp != time.Time{})\n\t\t\trequire_Equal[string](t, sm.MappedTo, \"bar\")\n\t\t\tegress := e.Egresses()\n\t\t\trequire_Equal[int](t, len(egress), 3)\n\t\t\tvar sub1And2 int\n\t\t\tfor _, eg := range egress {\n\t\t\t\t// All Egress should be clients\n\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\trequire_True(t, eg.Timestamp != time.Time{})\n\t\t\t\t// For nc2CID, we should have two egress\n\t\t\t\tif eg.CID == nc2CID {\n\t\t\t\t\t// Check name\n\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1And2\")\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"bar\")\n\t\t\t\t\tsub1And2++\n\t\t\t\t} else {\n\t\t\t\t\t// No name set\n\t\t\t\t\trequire_Equal[string](t, eg.Name, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t}\n\t\t\t}\n\t\t\trequire_Equal[int](t, sub1And2, 2)\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceIngressMaxPayloadError(t *testing.T) {\n\to := DefaultOptions()\n\to.MaxPayload = 1024\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsSub(t, nc, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc)\n\n\t// Ensure the subscription is known by the server we're connected to.\n\tcheckSubInterest(t, s, globalAccountName, \"my.trace.subj\", time.Second)\n\tcheckSubInterest(t, s, globalAccountName, \"foo\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tnc2, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", o.Port))\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer nc2.Close()\n\n\t\t\tnc2.Write([]byte(\"CONNECT {\\\"protocol\\\":1,\\\"headers\\\":true,\\\"no_responders\\\":true}\\r\\n\"))\n\n\t\t\tvar traceOnlyHdr string\n\t\t\tif !test.deliverMsg {\n\t\t\t\ttraceOnlyHdr = fmt.Sprintf(\"%s:true\\r\\n\", MsgTraceOnly)\n\t\t\t}\n\t\t\thdr := fmt.Sprintf(\"%s%s:%s\\r\\n%s\\r\\n\", hdrLine, MsgTraceDest, traceSub.Subject, traceOnlyHdr)\n\t\t\thPub := fmt.Sprintf(\"HPUB foo %d 2048\\r\\n%sAAAAAAAAAAAAAAAAAA...\", len(hdr), hdr)\n\t\t\tnc2.Write([]byte(hPub))\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\trequire_True(t, e.Request.Header == nil)\n\t\t\trequire_True(t, e.Ingress() != nil)\n\t\t\trequire_Contains(t, e.Ingress().Error, ErrMaxPayload.Error())\n\t\t\trequire_True(t, e.Egresses() == nil)\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceIngressMaxPayloadErrorDoesNotScanPayloadForTraceDest(t *testing.T) {\n\to := DefaultOptions()\n\to.MaxPayload = 1024\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\tcheckSubInterest(t, s, globalAccountName, \"my.trace.subj\", time.Second)\n\n\tnc2, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", o.Port))\n\trequire_NoError(t, err)\n\tdefer nc2.Close()\n\n\t_, err = nc2.Write([]byte(\"CONNECT {\\\"protocol\\\":1,\\\"headers\\\":true,\\\"no_responders\\\":true}\\r\\n\"))\n\trequire_NoError(t, err)\n\n\t// Payload contains a valid header, but the server should\n\t// not interpret it as such.\n\tpayload := fmt.Sprintf(\"AA\\r\\n%s:%s\\r\\n\", MsgTraceDest, traceSub.Subject)\n\n\thPub := fmt.Sprintf(\"HPUB foo %d 2048\\r\\n%s%s\", len(hdrLine), hdrLine, payload)\n\t_, err = nc2.Write([]byte(hPub))\n\trequire_NoError(t, err)\n\n\t// If bug is present: we receive a trace msg, even though\n\t// no trace header was set.\n\tif traceMsg, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Should not have received trace message: %s\", traceMsg.Data)\n\t}\n}\n\nfunc TestMsgTraceIngressErrors(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [\n\t\t\t\t\t{\n\t\t\t\t\t\tuser: a\n\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\tpermissions {\n\t\t\t\t\t\t\tsubscribe: [\"my.trace.subj\", \"foo\"]\n\t\t\t\t\t\t\tpublish {\n\t\t\t\t\t\t\t\tallow: [\"foo\", \"bar.>\"]\n\t\t\t\t\t\t\t\tdeny: [\"bar.baz\"]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsSub(t, nc, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc)\n\n\t// Ensure the subscription is known by the server we're connected to.\n\tcheckSubInterest(t, s, \"A\", \"my.trace.subj\", time.Second)\n\tcheckSubInterest(t, s, \"A\", \"foo\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tnc2 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer nc2.Close()\n\n\t\t\tsendMsg := func(subj, reply, errTxt string) {\n\t\t\t\tmsg := nats.NewMsg(subj)\n\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t}\n\t\t\t\tmsg.Reply = reply\n\t\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\t\tnc2.PublishMsg(msg)\n\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\trequire_True(t, e.Request.Header != nil)\n\t\t\t\trequire_Contains(t, e.Ingress().Error, errTxt)\n\t\t\t\trequire_True(t, e.Egresses() == nil)\n\t\t\t}\n\n\t\t\t// Send to a subject that causes permission violation\n\t\t\tsendMsg(\"bar.baz\", _EMPTY_, \"Permissions Violation for Publish to\")\n\n\t\t\t// Send to a subject that is reserved for GW replies\n\t\t\tsendMsg(gwReplyPrefix+\"foo\", _EMPTY_, \"Permissions Violation for Publish to\")\n\n\t\t\t// Send with a Reply that is reserved\n\t\t\tsendMsg(\"foo\", replyPrefix+\"bar\", \"Permissions Violation for Publish with Reply of\")\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceEgressErrors(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [\n\t\t\t\t\t{\n\t\t\t\t\t\tuser: a\n\t\t\t\t\t\tpassword: pwd\n\t\t\t\t\t\tpermissions {\n\t\t\t\t\t\t\tsubscribe: {\n\t\t\t\t\t\t\t\tallow: [\"my.trace.subj\", \"foo\", \"bar.>\"]\n\t\t\t\t\t\t\t\tdeny: \"bar.bat\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tpublish {\n\t\t\t\t\t\t\t\tallow: [\"foo\", \"bar.>\"]\n\t\t\t\t\t\t\t\tdeny: [\"bar.baz\"]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\t// Ensure the subscription is known by the server we're connected to.\n\tcheckSubInterest(t, s, \"A\", \"my.trace.subj\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsendMsg := func(pubc *nats.Conn, subj, errTxt string) {\n\t\t\t\tt.Helper()\n\n\t\t\t\tmsg := nats.NewMsg(subj)\n\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t}\n\t\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\t\tpubc.PublishMsg(msg)\n\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\tegress := e.Egresses()\n\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\trequire_Contains(t, egress[0].Error, errTxt)\n\t\t\t}\n\n\t\t\t// Test no-echo.\n\t\t\tnc2 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.NoEcho())\n\t\t\tdefer nc2.Close()\n\t\t\tnatsSubSync(t, nc2, \"foo\")\n\t\t\tsendMsg(nc2, \"foo\", errMsgTraceNoEcho)\n\t\t\tnc2.Close()\n\n\t\t\t// Test deny sub.\n\t\t\tnc2 = natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer nc2.Close()\n\t\t\tnatsSubSync(t, nc2, \"bar.>\")\n\t\t\tsendMsg(nc2, \"bar.bat\", errMsgTraceSubDeny)\n\t\t\tnc2.Close()\n\n\t\t\t// Test sub closed\n\t\t\tnc2 = natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer nc2.Close()\n\t\t\tnatsSubSync(t, nc2, \"bar.>\")\n\t\t\tnatsFlush(t, nc2)\n\t\t\t// Aritifially change the closed status of the subscription\n\t\t\tcid, err := nc2.GetClientID()\n\t\t\trequire_NoError(t, err)\n\t\t\tc := s.GetClient(cid)\n\t\t\tc.mu.Lock()\n\t\t\tfor _, sub := range c.subs {\n\t\t\t\tif string(sub.subject) == \"bar.>\" {\n\t\t\t\t\tsub.close()\n\t\t\t\t}\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t\tsendMsg(nc2, \"bar.bar\", errMsgTraceSubClosed)\n\t\t\tnc2.Close()\n\n\t\t\t// The following applies only when doing delivery.\n\t\t\tif test.deliverMsg {\n\t\t\t\t// Test auto-unsub exceeded\n\t\t\t\tnc2 = natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\t\tdefer nc2.Close()\n\t\t\t\tsub := natsSubSync(t, nc2, \"bar.>\")\n\t\t\t\terr := sub.AutoUnsubscribe(10)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tnatsFlush(t, nc2)\n\n\t\t\t\t// Modify sub.nm to be already over the 10 limit\n\t\t\t\tcid, err := nc2.GetClientID()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tc := s.GetClient(cid)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tfor _, sub := range c.subs {\n\t\t\t\t\tif string(sub.subject) == \"bar.>\" {\n\t\t\t\t\t\tsub.nm = 20\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\n\t\t\t\tsendMsg(nc2, \"bar.bar\", errMsgTraceAutoSubExceeded)\n\t\t\t\tnc2.Close()\n\n\t\t\t\t// Test client closed\n\t\t\t\tnc2 = natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\t\tdefer nc2.Close()\n\t\t\t\tnatsSubSync(t, nc2, \"bar.>\")\n\t\t\t\tcid, err = nc2.GetClientID()\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tc = s.GetClient(cid)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.out.stc = make(chan struct{})\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tmsg := nats.NewMsg(\"bar.bar\")\n\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t}\n\t\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\t\tnc2.PublishMsg(msg)\n\t\t\t\t// This needs to be less than default stall time, which now is 2ms.\n\t\t\t\ttime.Sleep(1 * time.Millisecond)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.flags.set(closeConnection)\n\t\t\t\tc.mu.Unlock()\n\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\tegress := e.Egresses()\n\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\trequire_Contains(t, egress[0].Error, errMsgTraceClientClosed)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.flags.clear(closeConnection)\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tnc2.Close()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithQueueSub(t *testing.T) {\n\to := DefaultOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\tnc2 := natsConnect(t, s.ClientURL(), nats.Name(\"sub1\"))\n\tdefer nc2.Close()\n\tsub1 := natsQueueSubSync(t, nc2, \"foo\", \"bar\")\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, s.ClientURL(), nats.Name(\"sub2\"))\n\tdefer nc3.Close()\n\tsub2 := natsQueueSubSync(t, nc3, \"foo\", \"bar\")\n\tsub3 := natsQueueSubSync(t, nc3, \"*\", \"baz\")\n\tnatsFlush(t, nc3)\n\n\t// Ensure the subscription is known by the server we're connected to.\n\tcheckSubInterest(t, s, globalAccountName, \"my.trace.subj\", time.Second)\n\tcheckSubInterest(t, s, globalAccountName, \"foo\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Data = []byte(\"hello1\")\n\t\t\t} else {\n\t\t\t\tmsg.Data = []byte(\"hello2\")\n\t\t\t}\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.deliverMsg {\n\t\t\t\t// Only one should have got the message...\n\t\t\t\tmsg1, err1 := sub1.NextMsg(100 * time.Millisecond)\n\t\t\t\tmsg2, err2 := sub2.NextMsg(100 * time.Millisecond)\n\t\t\t\tif err1 == nil && err2 == nil {\n\t\t\t\t\tt.Fatalf(\"Only one message should have been received\")\n\t\t\t\t}\n\t\t\t\tvar val string\n\t\t\t\tif msg1 != nil {\n\t\t\t\t\tval = string(msg1.Data)\n\t\t\t\t} else {\n\t\t\t\t\tval = string(msg2.Data)\n\t\t\t\t}\n\t\t\t\trequire_Equal[string](t, val, \"hello2\")\n\t\t\t\t// Queue baz should also have received the message\n\t\t\t\tmsg := natsNexMsg(t, sub3, time.Second)\n\t\t\t\trequire_Equal[string](t, string(msg.Data), \"hello2\")\n\t\t\t}\n\t\t\t// Check that no (more) messages are received.\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2, sub3} {\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected no message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\trequire_Equal[string](t, ingress.Subject, \"foo\")\n\t\t\tegress := e.Egresses()\n\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\tvar qbar, qbaz int\n\t\t\tfor _, eg := range egress {\n\t\t\t\tswitch eg.Queue {\n\t\t\t\tcase \"bar\":\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"foo\")\n\t\t\t\t\tqbar++\n\t\t\t\tcase \"baz\":\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t\tqbaz++\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Wrong queue name: %q\", eg.Queue)\n\t\t\t\t}\n\t\t\t}\n\t\t\trequire_Equal[int](t, qbar, 1)\n\t\t\trequire_Equal[int](t, qbaz, 1)\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithRoutes(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA { users: [{user:A, password: pwd}] }\n\t\t\tB { users: [{user:B, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\taccounts: [\"A\"]\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckDummy := func(user string) {\n\t\tnc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc.Close()\n\n\t\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\t\tnatsFlush(t, nc)\n\n\t\t// Send trace message to a dummy subject to check that resulting trace\n\t\t// is as expected.\n\t\tmsg := nats.NewMsg(\"dummy\")\n\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\tmsg.Data = []byte(\"hello!\")\n\t\terr := nc.PublishMsg(msg)\n\t\trequire_NoError(t, err)\n\n\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\tvar e MsgTraceEvent\n\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\tingress := e.Ingress()\n\t\trequire_True(t, ingress != nil)\n\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t// \"user\" is same than account name in this test.\n\t\trequire_Equal[string](t, ingress.Account, user)\n\t\trequire_Equal[string](t, ingress.Subject, \"dummy\")\n\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\trequire_True(t, e.Egresses() == nil)\n\n\t\t// We should also not get an event from the remote server.\n\t\tif msg, err := traceSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Expected no message, got %s\", msg.Data)\n\t\t}\n\t}\n\tcheckDummy(\"A\")\n\tcheckDummy(\"B\")\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tacc string\n\t}{\n\t\t{\"pinned account\", \"A\"},\n\t\t{\"reg account\", \"B\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tacc := test.acc\n\t\t\t// Now create subscriptions on both s1 and s2\n\t\t\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(acc, \"pwd\"), nats.Name(\"sub2\"))\n\t\t\tdefer nc2.Close()\n\t\t\tsub2 := natsQueueSubSync(t, nc2, \"foo.*\", \"my_queue\")\n\n\t\t\tnc3 := natsConnect(t, s2.ClientURL(), nats.UserInfo(acc, \"pwd\"), nats.Name(\"sub3\"))\n\t\t\tdefer nc3.Close()\n\t\t\tsub3 := natsQueueSubSync(t, nc3, \"*.*\", \"my_queue_2\")\n\n\t\t\tcheckSubInterest(t, s1, acc, \"foo.bar\", time.Second)\n\n\t\t\tnc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(acc, \"pwd\"), nats.Name(\"sub1\"))\n\t\t\tdefer nc1.Close()\n\t\t\tsub1 := natsSubSync(t, nc1, \"*.bar\")\n\n\t\t\tnct := natsConnect(t, s1.ClientURL(), nats.UserInfo(acc, \"pwd\"), nats.Name(\"tracer\"))\n\t\t\tdefer nct.Close()\n\t\t\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"foo.bar\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\t\t\terr := nct.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\t\t\tif expected {\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2, sub3} {\n\t\t\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t\t\t}\n\n\t\t\t\t\tcheck := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, acc)\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t\tif eg.Kind == CLIENT {\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*.bar\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\trequire_True(t, eg.Kind == ROUTER)\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, s2.Name())\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s2.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, acc)\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\t\tvar gotSub2, gotSub3 int\n\t\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\t\tswitch eg.Name {\n\t\t\t\t\t\t\t\tcase \"sub2\":\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"foo.*\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\t\t\t\tgotSub2++\n\t\t\t\t\t\t\t\tcase \"sub3\":\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*.*\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue_2\")\n\t\t\t\t\t\t\t\t\tgotSub3++\n\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress name: %+v\", eg)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\trequire_Equal[int](t, gotSub2, 1)\n\t\t\t\t\t\t\trequire_Equal[int](t, gotSub3, 1)\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// We should get 2 events. Order is not guaranteed.\n\t\t\t\t\tcheck()\n\t\t\t\t\tcheck()\n\t\t\t\t\t// Make sure we are not receiving more traces\n\t\t\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithRouteToOldServer(t *testing.T) {\n\tmsgTraceCheckSupport = true\n\tdefer func() { msgTraceCheckSupport = false }()\n\ttmpl := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\to2 := LoadConfig(conf2)\n\t// Make this server behave like an older server\n\to2.overrideProto = setServerProtoForTest(MsgTraceProto - 1)\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now create subscriptions on both s1 and s2\n\tnc2 := natsConnect(t, s2.ClientURL(), nats.Name(\"sub2\"))\n\tdefer nc2.Close()\n\tsub2 := natsSubSync(t, nc2, \"foo\")\n\n\tcheckSubInterest(t, s1, globalAccountName, \"foo\", time.Second)\n\n\tnc1 := natsConnect(t, s1.ClientURL(), nats.Name(\"sub1\"))\n\tdefer nc1.Close()\n\tsub1 := natsSubSync(t, nc1, \"foo\")\n\n\tnct := natsConnect(t, s1.ClientURL(), nats.Name(\"tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t// Ensure the subscription is known by the server we're connected to.\n\tcheckSubInterest(t, s1, globalAccountName, \"my.trace.subj\", time.Second)\n\tcheckSubInterest(t, s2, globalAccountName, \"my.trace.subj\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr := nct.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tif expected {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t}\n\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Even if a server does not support tracing, as long as the header\n\t\t\t// TraceOnly is not set, the message should be forwarded to the remote.\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t}\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\t\tegress := e.Egresses()\n\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\tfor _, ci := range egress {\n\t\t\t\tswitch ci.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, ci.Name, \"sub1\")\n\t\t\t\tcase ROUTER:\n\t\t\t\t\trequire_Equal[string](t, ci.Name, s2.Name())\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceNoSupport)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceOnlyNoSupport)\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", ci)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We should not get a second trace\n\t\t\tif msg, err := traceSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Did not expect other trace, got %s\", msg.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithLeafNode(t *testing.T) {\n\tfor _, mainTest := range []struct {\n\t\tname string\n\t\tfromHub bool\n\t\tleafUseLocalAcc bool\n\t}{\n\t\t{\"from hub\", true, false},\n\t\t{\"from leaf\", false, false},\n\t\t{\"from hub with local account\", true, true},\n\t\t{\"from leaf with local account\", false, true},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\tconfHub := createConfFile(t, []byte(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"A\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"a\", password: \"pwd\"}]}\n\t\t\t\t\tB { users: [{user: \"b\", password: \"pwd\"}]}\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t}\n\t\t\t`))\n\t\t\thub, ohub := RunServerWithConfig(confHub)\n\t\t\tdefer hub.Shutdown()\n\n\t\t\tvar accs string\n\t\t\tvar lacc string\n\t\t\tif mainTest.leafUseLocalAcc {\n\t\t\t\taccs = `accounts { B { users: [{user: \"b\", password: \"pwd\"}]} }`\n\t\t\t\tlacc = `account: B`\n\t\t\t}\n\t\t\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"B\"\n\t\t\t\t%s\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t\t\t\t%s\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t\t`, accs, ohub.LeafNode.Port, lacc)))\n\t\t\tleaf, _ := RunServerWithConfig(confLeaf)\n\t\t\tdefer leaf.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, hub)\n\t\t\tcheckLeafNodeConnected(t, leaf)\n\n\t\t\tvar s1, s2 *Server\n\t\t\tif mainTest.fromHub {\n\t\t\t\ts1, s2 = hub, leaf\n\t\t\t} else {\n\t\t\t\ts1, s2 = leaf, hub\n\t\t\t}\n\t\t\t// Now create subscriptions on both s1 and s2\n\t\t\topts := []nats.Option{nats.Name(\"sub2\")}\n\t\t\tvar user string\n\t\t\t// If fromHub, then it means that s2 is the leaf.\n\t\t\tif mainTest.fromHub {\n\t\t\t\tif mainTest.leafUseLocalAcc {\n\t\t\t\t\tuser = \"b\"\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// s2 is the hub, always connect with user \"a'\"\n\t\t\t\tuser = \"a\"\n\t\t\t}\n\t\t\tif user != _EMPTY_ {\n\t\t\t\topts = append(opts, nats.UserInfo(user, \"pwd\"))\n\t\t\t}\n\t\t\tnc2 := natsConnect(t, s2.ClientURL(), opts...)\n\t\t\tdefer nc2.Close()\n\t\t\tsub2 := natsSubSync(t, nc2, \"foo\")\n\n\t\t\tif mainTest.fromHub {\n\t\t\t\tcheckSubInterest(t, s1, \"A\", \"foo\", time.Second)\n\t\t\t} else if mainTest.leafUseLocalAcc {\n\t\t\t\tcheckSubInterest(t, s1, \"B\", \"foo\", time.Second)\n\t\t\t} else {\n\t\t\t\tcheckSubInterest(t, s1, globalAccountName, \"foo\", time.Second)\n\t\t\t}\n\n\t\t\tuser = _EMPTY_\n\t\t\topts = []nats.Option{nats.Name(\"sub1\")}\n\t\t\tif mainTest.fromHub {\n\t\t\t\t// s1 is the hub, so we need user \"a\"\n\t\t\t\tuser = \"a\"\n\t\t\t} else if mainTest.leafUseLocalAcc {\n\t\t\t\t// s1 is the leaf, we need user \"b\" if leafUseLocalAcc\n\t\t\t\tuser = \"b\"\n\t\t\t}\n\t\t\tif user != _EMPTY_ {\n\t\t\t\topts = append(opts, nats.UserInfo(user, \"pwd\"))\n\t\t\t}\n\t\t\tnc1 := natsConnect(t, s1.ClientURL(), opts...)\n\t\t\tdefer nc1.Close()\n\t\t\tsub1 := natsSubSync(t, nc1, \"foo\")\n\n\t\t\topts = []nats.Option{nats.Name(\"tracer\")}\n\t\t\tif user != _EMPTY_ {\n\t\t\t\topts = append(opts, nats.UserInfo(user, \"pwd\"))\n\t\t\t}\n\t\t\tnct := natsConnect(t, s1.ClientURL(), opts...)\n\t\t\tdefer nct.Close()\n\t\t\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\t\t\terr := nct.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\t\t\tif expected {\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t\t\t}\n\n\t\t\t\t\tcheck := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t\tswitch eg.Kind {\n\t\t\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\t\tcase LEAF:\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, s2.Name())\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Error, _EMPTY_)\n\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase LEAF:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s2.Name())\n\t\t\t\t\t\t\trequire_True(t, ingress.Kind == LEAF)\n\t\t\t\t\t\t\trequire_Equal(t, ingress.Name, s1.Name())\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tcheck()\n\t\t\t\t\tcheck()\n\t\t\t\t\t// Make sure we are not receiving more traces\n\t\t\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithLeafNodeToOldServer(t *testing.T) {\n\tmsgTraceCheckSupport = true\n\tdefer func() { msgTraceCheckSupport = false }()\n\tfor _, mainTest := range []struct {\n\t\tname string\n\t\tfromHub bool\n\t}{\n\t\t{\"from hub\", true},\n\t\t{\"from leaf\", false},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\tconfHub := createConfFile(t, []byte(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t}\n\t\t\t`))\n\t\t\tohub := LoadConfig(confHub)\n\t\t\tif !mainTest.fromHub {\n\t\t\t\t// Make this server behave like an older server\n\t\t\t\tohub.overrideProto = setServerProtoForTest(MsgTraceProto - 1)\n\t\t\t}\n\t\t\thub := RunServer(ohub)\n\t\t\tdefer hub.Shutdown()\n\n\t\t\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tleafnodes {\n\t\t\t\t\tremotes [{url: \"nats://127.0.0.1:%d\"}]\n\t\t\t\t}\n\t\t\t\t`, ohub.LeafNode.Port)))\n\t\t\toleaf := LoadConfig(confLeaf)\n\t\t\tif mainTest.fromHub {\n\t\t\t\t// Make this server behave like an older server\n\t\t\t\toleaf.overrideProto = setServerProtoForTest(MsgTraceProto - 1)\n\t\t\t}\n\t\t\tleaf := RunServer(oleaf)\n\t\t\tdefer leaf.Shutdown()\n\n\t\t\tcheckLeafNodeConnected(t, hub)\n\t\t\tcheckLeafNodeConnected(t, leaf)\n\n\t\t\tvar s1, s2 *Server\n\t\t\tif mainTest.fromHub {\n\t\t\t\ts1, s2 = hub, leaf\n\t\t\t} else {\n\t\t\t\ts1, s2 = leaf, hub\n\t\t\t}\n\n\t\t\t// Now create subscriptions on both s1 and s2\n\t\t\tnc2 := natsConnect(t, s2.ClientURL(), nats.Name(\"sub2\"))\n\t\t\tdefer nc2.Close()\n\t\t\tsub2 := natsSubSync(t, nc2, \"foo\")\n\n\t\t\tcheckSubInterest(t, s1, globalAccountName, \"foo\", time.Second)\n\n\t\t\tnc1 := natsConnect(t, s1.ClientURL(), nats.Name(\"sub1\"))\n\t\t\tdefer nc1.Close()\n\t\t\tsub1 := natsSubSync(t, nc1, \"foo\")\n\n\t\t\tnct := natsConnect(t, s1.ClientURL(), nats.Name(\"tracer\"))\n\t\t\tdefer nct.Close()\n\t\t\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\t\t\terr := nct.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\t\t\tif expected {\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Even if a server does not support tracing, as long as the header\n\t\t\t\t\t// TraceOnly is not set, the message should be forwarded to the remote.\n\t\t\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t\t\t}\n\n\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\tfor _, ci := range egress {\n\t\t\t\t\t\tswitch ci.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal[string](t, ci.Name, \"sub1\")\n\t\t\t\t\t\tcase LEAF:\n\t\t\t\t\t\t\trequire_Equal[string](t, ci.Name, s2.Name())\n\t\t\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceNoSupport)\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceOnlyNoSupport)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", ci)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// We should not get a second trace\n\t\t\t\t\tif msg, err := traceSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\t\tt.Fatalf(\"Did not expect other trace, got %s\", msg.Data)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithLeafNodeDaisyChain(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"pwd\"}]}\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(confHub)\n\tdefer hub.Shutdown()\n\n\tconfLeaf1 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\taccounts {\n\t\t\tB { users: [{user: \"b\", password: \"pwd\"}]}\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: B}]\n\t\t}\n\t`, ohub.LeafNode.Port)))\n\tleaf1, oleaf1 := RunServerWithConfig(confLeaf1)\n\tdefer leaf1.Shutdown()\n\n\tconfLeaf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"C\"\n\t\taccounts {\n\t\t\tC { users: [{user: \"c\", password: \"pwd\"}]}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [{url: \"nats://b:pwd@127.0.0.1:%d\", account: C}]\n\t\t}\n\t`, oleaf1.LeafNode.Port)))\n\tleaf2, _ := RunServerWithConfig(confLeaf2)\n\tdefer leaf2.Shutdown()\n\n\tcheckLeafNodeConnected(t, hub)\n\tcheckLeafNodeConnectedCount(t, leaf1, 2)\n\tcheckLeafNodeConnected(t, leaf2)\n\n\tnct := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"Tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\tnatsFlush(t, nct)\n\t// Make sure that subject interest travels down to leaf2\n\tcheckSubInterest(t, leaf2, \"C\", traceSub.Subject, time.Second)\n\n\tnc1 := natsConnect(t, leaf1.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"sub1\"))\n\tdefer nc1.Close()\n\n\tnc2 := natsConnect(t, leaf2.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"sub2\"))\n\tdefer nc2.Close()\n\tsub2 := natsQueueSubSync(t, nc2, \"foo.bar\", \"my_queue\")\n\tnatsFlush(t, nc2)\n\n\t// Check the subject interest makes it to leaf1\n\tcheckSubInterest(t, leaf1, \"B\", \"foo.bar\", time.Second)\n\n\t// Now create the sub on leaf1\n\tsub1 := natsSubSync(t, nc1, \"foo.*\")\n\tnatsFlush(t, nc1)\n\n\t// Check that subject interest registered on \"hub\"\n\tcheckSubInterest(t, hub, \"A\", \"foo.bar\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo.bar\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr := nct.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tif expected {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t}\n\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t}\n\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == LEAF)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, leaf1.Name())\n\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\tcase LEAF:\n\t\t\t\t\tswitch e.Server.Name {\n\t\t\t\t\tcase leaf1.Name():\n\t\t\t\t\t\trequire_Equal(t, ingress.Name, hub.Name())\n\t\t\t\t\t\trequire_Equal(t, ingress.Account, \"B\")\n\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\tswitch eg.Kind {\n\t\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"foo.*\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\tcase LEAF:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, leaf2.Name())\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\tcase leaf2.Name():\n\t\t\t\t\t\trequire_Equal(t, ingress.Name, leaf1.Name())\n\t\t\t\t\t\trequire_Equal(t, ingress.Account, \"C\")\n\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"foo.bar\")\n\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithGateways(t *testing.T) {\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.NoSystemAccount = false\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\to1.NoSystemAccount = false\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\to3 := testGatewayOptionsFromToWithServers(t, \"C\", \"B\", s2)\n\to3.NoSystemAccount = false\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 2, time.Second)\n\twaitForInboundGateways(t, s1, 2, time.Second)\n\twaitForInboundGateways(t, s2, 2, time.Second)\n\twaitForOutboundGateways(t, s2, 2, time.Second)\n\twaitForInboundGateways(t, s3, 2, time.Second)\n\twaitForOutboundGateways(t, s3, 2, time.Second)\n\n\tnc2 := natsConnect(t, s2.ClientURL(), nats.Name(\"sub2\"))\n\tdefer nc2.Close()\n\tsub2 := natsQueueSubSync(t, nc2, \"foo.*\", \"my_queue_2\")\n\n\tnc22 := natsConnect(t, s2.ClientURL(), nats.Name(\"sub22\"))\n\tdefer nc22.Close()\n\tsub22 := natsQueueSubSync(t, nc22, \"*.*\", \"my_queue_22\")\n\n\tnc3 := natsConnect(t, s3.ClientURL(), nats.Name(\"sub3\"))\n\tdefer nc3.Close()\n\tsub3 := natsQueueSubSync(t, nc3, \"foo.*\", \"my_queue_3\")\n\n\tnc32 := natsConnect(t, s3.ClientURL(), nats.Name(\"sub32\"))\n\tdefer nc32.Close()\n\tsub32 := natsQueueSubSync(t, nc32, \"*.*\", \"my_queue_32\")\n\n\tnc1 := natsConnect(t, s1.ClientURL(), nats.Name(\"sub1\"))\n\tdefer nc1.Close()\n\tsub1 := natsSubSync(t, nc1, \"*.bar\")\n\n\tnct := natsConnect(t, s1.ClientURL(), nats.Name(\"tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t// Ensure the subscription is known by the server we're connected to.\n\trequire_NoError(t, nct.Flush())\n\ttime.Sleep(100 * time.Millisecond)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo.bar\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr := nct.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tif expected {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal(t, string(appMsg.Data), \"hello!\")\n\t\t\t\t}\n\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2, sub22, sub3, sub32} {\n\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t}\n\n\t\t\tvar previousHop string\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal(t, e.Server.Name, s1.Name())\n\t\t\t\t\trequire_Equal(t, ingress.Account, globalAccountName)\n\t\t\t\t\trequire_Equal(t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal(t, len(egress), 3)\n\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\tswitch eg.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal(t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\trequire_Equal(t, eg.Subscription, \"*.bar\")\n\t\t\t\t\t\t\trequire_Equal(t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\tif eg.Name != s2.Name() && eg.Name != s3.Name() {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expected name to be %q or %q, got %q\", s2.Name(), s3.Name(), eg.Name)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\trequire_Equal(t, eg.Error, _EMPTY_)\n\t\t\t\t\t\t\trequire_Equal(t, eg.Subscription, _EMPTY_)\n\t\t\t\t\t\t\trequire_Equal(t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tcase GATEWAY:\n\t\t\t\t\trequire_True(t, e.Request.Header != nil)\n\t\t\t\t\trequire_Len(t, len(e.Request.Header[MsgTraceHop]), 1)\n\t\t\t\t\thop := e.Request.Header[MsgTraceHop][0]\n\t\t\t\t\trequire_True(t, hop == \"1\" || hop == \"2\")\n\t\t\t\t\tif previousHop == _EMPTY_ {\n\t\t\t\t\t\tpreviousHop = hop\n\t\t\t\t\t} else if hop == previousHop {\n\t\t\t\t\t\tt.Fatalf(\"Expected different hop value, got the same %q\", hop)\n\t\t\t\t\t}\n\t\t\t\t\tvar sub2Name, queue2Name, sub3Name, queue3Name string\n\t\t\t\t\tswitch e.Server.Name {\n\t\t\t\t\tcase s2.Name():\n\t\t\t\t\t\trequire_Equal(t, e.Server.Cluster, \"B\")\n\t\t\t\t\t\tsub2Name, sub3Name = \"sub2\", \"sub22\"\n\t\t\t\t\t\tqueue2Name, queue3Name = \"my_queue_2\", \"my_queue_22\"\n\t\t\t\t\tcase s3.Name():\n\t\t\t\t\t\trequire_Equal(t, e.Server.Cluster, \"C\")\n\t\t\t\t\t\tsub2Name, sub3Name = \"sub3\", \"sub32\"\n\t\t\t\t\t\tqueue2Name, queue3Name = \"my_queue_3\", \"my_queue_32\"\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tt.Fatalf(\"Unexpected server name %q\", e.Server.Name)\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, ingress.Account, globalAccountName)\n\t\t\t\t\trequire_Equal(t, ingress.Subject, \"foo.bar\")\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal(t, len(egress), 2)\n\t\t\t\t\tvar gotSub2, gotSub3 int\n\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\tswitch eg.Name {\n\t\t\t\t\t\tcase sub2Name:\n\t\t\t\t\t\t\trequire_Equal(t, eg.Subscription, \"foo.*\")\n\t\t\t\t\t\t\trequire_Equal(t, eg.Queue, queue2Name)\n\t\t\t\t\t\t\tgotSub2++\n\t\t\t\t\t\tcase sub3Name:\n\t\t\t\t\t\t\trequire_Equal(t, eg.Subscription, \"*.*\")\n\t\t\t\t\t\t\trequire_Equal(t, eg.Queue, queue3Name)\n\t\t\t\t\t\t\tgotSub3++\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress name: %+v\", eg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, gotSub2, 1)\n\t\t\t\t\trequire_Equal(t, gotSub3, 1)\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We should get 3 events\n\t\t\tfor range 3 {\n\t\t\t\tcheck()\n\t\t\t}\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceWithGatewayToOldServer(t *testing.T) {\n\tmsgTraceCheckSupport = true\n\tdefer func() { msgTraceCheckSupport = false }()\n\n\to2 := testDefaultOptionsForGateway(\"B\")\n\to2.NoSystemAccount = false\n\t// Make this server behave like an older server\n\to2.overrideProto = setServerProtoForTest(MsgTraceProto - 1)\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s2)\n\to1.NoSystemAccount = false\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, time.Second)\n\twaitForInboundGateways(t, s2, 1, time.Second)\n\twaitForOutboundGateways(t, s2, 1, time.Second)\n\n\tnc2 := natsConnect(t, s2.ClientURL(), nats.Name(\"sub2\"))\n\tdefer nc2.Close()\n\tsub2 := natsSubSync(t, nc2, \"foo\")\n\n\tnc1 := natsConnect(t, s1.ClientURL(), nats.Name(\"sub1\"))\n\tdefer nc1.Close()\n\tsub1 := natsSubSync(t, nc1, \"foo\")\n\n\tnct := natsConnect(t, s1.ClientURL(), nats.Name(\"tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\n\t// Ensure the subscription is known by the server we're connected to.\n\trequire_NoError(t, nct.Flush())\n\ttime.Sleep(100 * time.Millisecond)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr := nct.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tcheckAppMsg := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tif expected {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t\t\t}\n\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Even if a server does not support tracing, as long as the header\n\t\t\t// TraceOnly is not set, the message should be forwarded to the remote.\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tcheckAppMsg(sub, test.deliverMsg)\n\t\t\t}\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\tswitch ingress.Kind {\n\t\t\tcase CLIENT:\n\t\t\t\trequire_Equal[string](t, e.Server.Name, s1.Name())\n\t\t\t\tegress := e.Egresses()\n\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\tfor _, ci := range egress {\n\t\t\t\t\tswitch ci.Kind {\n\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\trequire_Equal[string](t, ci.Name, \"sub1\")\n\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\trequire_Equal[string](t, ci.Name, s2.Name())\n\t\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceNoSupport)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\trequire_Contains(t, ci.Error, errMsgTraceOnlyNoSupport)\n\t\t\t\t\t\t}\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", ci)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t}\n\t\t\t// We should not get a second trace\n\t\t\tif msg, err := traceSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Did not expect other trace, got %s\", msg.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceServiceImport(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [ { service: \">\", allow_trace: %v} ]\n\t\t\t\tmappings = {\n\t\t\t\t\tbar: bozo\n\t\t\t\t}\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [ { service: {account: \"A\", subject:\">\"} } ]\n\t\t\t\texports: [ { service: \">\", allow_trace: %v} ]\n\t\t\t}\n\t\t\tC {\n\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\texports: [ { service: \">\", allow_trace: %v } ]\n\t\t\t}\n\t\t\tD {\n\t\t\t\tusers: [{user: d, password: pwd}]\n\t\t\t\timports: [\n\t\t\t\t\t{ service: {account: \"B\", subject:\"bar\"}, to: baz }\n\t\t\t\t\t{ service: {account: \"C\", subject:\">\"} }\n\t\t\t\t]\n\t\t\t\tmappings = {\n\t\t\t\t\t\tbat: baz\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, false, false, false)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"d\", \"pwd\"), nats.Name(\"Requestor\"))\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tsub := natsSubSync(t, nc, \"my.service.response.inbox\")\n\n\tnc2 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"ServiceA\"))\n\tdefer nc2.Close()\n\trecv := int32(0)\n\tnatsQueueSub(t, nc2, \"*\", \"my_queue\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&recv, 1)\n\t\tm.Respond(m.Data)\n\t})\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"ServiceC\"))\n\tdefer nc3.Close()\n\tnatsSub(t, nc3, \"baz\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&recv, 1)\n\t\tm.Respond(m.Data)\n\t})\n\tnatsFlush(t, nc3)\n\n\tfor mainIter, mainTest := range []struct {\n\t\tname string\n\t\tallow bool\n\t}{\n\t\t{\"not allowed\", false},\n\t\t{\"allowed\", true},\n\t\t{\"not allowed again\", false},\n\t} {\n\t\tatomic.StoreInt32(&recv, 0)\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"bat\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Data = []byte(\"request1\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tmsg.Data = []byte(\"request2\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Reply = sub.Subject\n\n\t\t\t\t\terr := nc.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"request2\")\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || (err != nats.ErrTimeout && err != nats.ErrNoResponders) {\n\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t\t}\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\t// Just to make sure that message was not delivered to service\n\t\t\t\t\t\t// responders, wait a bit and check the recv value.\n\t\t\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\t\t\tif n := atomic.LoadInt32(&recv); n != 0 {\n\t\t\t\t\t\t\tt.Fatalf(\"Expected no message to be received, but service callback fired %d times\", n)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"D\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bat\")\n\t\t\t\t\tsm := e.SubjectMapping()\n\t\t\t\t\trequire_True(t, sm != nil)\n\t\t\t\t\trequire_Equal[string](t, sm.MappedTo, \"baz\")\n\t\t\t\t\tsimps := e.ServiceImports()\n\t\t\t\t\trequire_True(t, simps != nil)\n\t\t\t\t\tvar expectedServices int\n\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\texpectedServices = 3\n\t\t\t\t\t} else {\n\t\t\t\t\t\texpectedServices = 2\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal[int](t, len(simps), expectedServices)\n\t\t\t\t\tfor _, si := range simps {\n\t\t\t\t\t\trequire_True(t, si.Timestamp != time.Time{})\n\t\t\t\t\t\tswitch si.Account {\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"baz\")\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bar\")\n\t\t\t\t\t\tcase \"A\":\n\t\t\t\t\t\t\tif !mainTest.allow {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Without allow_trace, we should not see service for account A\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"bar\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bozo\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected account name: %s\", si.Account)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\tif !mainTest.allow {\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 0)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\tvar gotA, gotC bool\n\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t// All Egress should be clients\n\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\t// We should have one for ServiceA and one for ServiceC\n\t\t\t\t\t\t\tif eg.Name == \"ServiceA\" {\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, \"A\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\t\t\tgotA = true\n\t\t\t\t\t\t\t} else if eg.Name == \"ServiceC\" {\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, \"C\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\t\tgotC = true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif !gotA {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not get Egress for serviceA: %+v\", egress)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif !gotC {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not get Egress for serviceC: %+v\", egress)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Make sure we properly remove the responses.\n\t\t\t\t\tcheckResp := func(an string) {\n\t\t\t\t\t\tacc, err := s.lookupAccount(an)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\t\t\tif n := acc.NumPendingAllResponses(); n != 0 {\n\t\t\t\t\t\t\t\treturn fmt.Errorf(\"Still %d responses pending for account %q on server %s\", n, acc, s)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t\tfor _, acc := range []string{\"A\", \"B\", \"C\", \"D\"} {\n\t\t\t\t\t\tcheckResp(acc)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t\tswitch mainIter {\n\t\t\tcase 0:\n\t\t\t\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, true, true, true))\n\t\t\tcase 1:\n\t\t\t\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, false, false, false))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceServiceImportWithSuperCluster(t *testing.T) {\n\tfor _, mainTest := range []struct {\n\t\tname string\n\t\tallowStr string\n\t\tallow bool\n\t}{\n\t\t{\"allowed\", \"true\", true},\n\t\t{\"not allowed\", \"false\", false},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tserver_name: %s\n\t\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t\t\tcluster {\n\t\t\t\t\tname: %s\n\t\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\t\troutes = [%s]\n\t\t\t\t}\n\t\t\t\taccounts {\n\t\t\t\t\tA {\n\t\t\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\t\t\texports: [ { service: \">\", allow_trace: ` + mainTest.allowStr + ` } ]\n\t\t\t\t\t\tmappings = {\n\t\t\t\t\t\t\tbar: bozo\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttrace_dest: \"a.trace.subj\"\n\t\t\t\t\t}\n\t\t\t\t\tB {\n\t\t\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\t\t\timports: [ { service: {account: \"A\", subject:\">\"} } ]\n\t\t\t\t\t\texports: [ { service: \">\" , allow_trace: ` + mainTest.allowStr + ` } ]\n\t\t\t\t\t\ttrace_dest: \"b.trace.subj\"\n\t\t\t\t\t}\n\t\t\t\t\tC {\n\t\t\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\t\t\texports: [ { service: \">\" , allow_trace: ` + mainTest.allowStr + ` } ]\n\t\t\t\t\t\ttrace_dest: \"c.trace.subj\"\n\t\t\t\t\t}\n\t\t\t\t\tD {\n\t\t\t\t\t\tusers: [{user: d, password: pwd}]\n\t\t\t\t\t\timports: [\n\t\t\t\t\t\t\t{ service: {account: \"B\", subject:\"bar\"}, to: baz }\n\t\t\t\t\t\t\t{ service: {account: \"C\", subject:\">\"} }\n\t\t\t\t\t\t]\n\t\t\t\t\t\tmappings = {\n\t\t\t\t\t\t\t\tbat: baz\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttrace_dest: \"d.trace.subj\"\n\t\t\t\t\t}\n\t\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t\t}\n\t\t\t`\n\t\t\tsc := createJetStreamSuperClusterWithTemplate(t, tmpl, 3, 2)\n\t\t\tdefer sc.shutdown()\n\n\t\t\tsfornc := sc.clusters[0].servers[0]\n\t\t\tnc := natsConnect(t, sfornc.ClientURL(), nats.UserInfo(\"d\", \"pwd\"), nats.Name(\"Requestor\"))\n\t\t\tdefer nc.Close()\n\n\t\t\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\t\t\tsub := natsSubSync(t, nc, \"my.service.response.inbox\")\n\n\t\t\tsfornc2 := sc.clusters[0].servers[1]\n\t\t\tnc2 := natsConnect(t, sfornc2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"ServiceA\"))\n\t\t\tdefer nc2.Close()\n\t\t\tsubSvcA := natsQueueSubSync(t, nc2, \"*\", \"my_queue\")\n\t\t\tnatsFlush(t, nc2)\n\n\t\t\tsfornc3 := sc.clusters[1].servers[0]\n\t\t\tnc3 := natsConnect(t, sfornc3.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"ServiceC\"))\n\t\t\tdefer nc3.Close()\n\t\t\tsubSvcC := natsSubSync(t, nc3, \"baz\")\n\t\t\tnatsFlush(t, nc3)\n\n\t\t\t// Create a subscription for each account trace destination to make\n\t\t\t// sure that we are not sending it there.\n\t\t\tvar accSubs []*nats.Subscription\n\t\t\tfor _, user := range []string{\"a\", \"b\", \"c\", \"d\"} {\n\t\t\t\tnc := natsConnect(t, sfornc.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\t\t\tdefer nc.Close()\n\t\t\t\taccSubs = append(accSubs, natsSubSync(t, nc, user+\".trace.subj\"))\n\t\t\t}\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"bat\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\t// We add the traceParentHdr header to make sure that it is\n\t\t\t\t\t// deactivated in addition to the Nats-Trace-Dest header too\n\t\t\t\t\t// when needed.\n\t\t\t\t\ttraceParentHdrVal := \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"\n\t\t\t\t\tmsg.Header.Set(traceParentHdr, traceParentHdrVal)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Data = []byte(\"request1\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tmsg.Data = []byte(\"request2\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Reply = sub.Subject\n\n\t\t\t\t\terr := nc.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\tprocessSvc := func(sub *nats.Subscription) {\n\t\t\t\t\t\t\tt.Helper()\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\tif hv := appMsg.Header.Get(traceParentHdr); hv != traceParentHdrVal {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expecting header with %q, but got %q\", traceParentHdrVal, hv)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t// This test causes a message to be routed to the\n\t\t\t\t\t\t\t// service responders. When not allowing, we need\n\t\t\t\t\t\t\t// to make sure that the trace header has been\n\t\t\t\t\t\t\t// disabled. Not receiving the trace event from\n\t\t\t\t\t\t\t// the remote is not enough to verify since the\n\t\t\t\t\t\t\t// trace would not reach the origin server because\n\t\t\t\t\t\t\t// the origin account header will not be present.\n\t\t\t\t\t\t\tvar expected string\n\t\t\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\t\t\texpected = traceSub.Subject\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\texpected = MsgTraceDestDisabled\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif hv := appMsg.Header.Get(MsgTraceDest); hv != expected {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expecting header with %q, but got %q\", expected, hv)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tappMsg.Respond(appMsg.Data)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tprocessSvc(subSvcA)\n\t\t\t\t\t\tprocessSvc(subSvcC)\n\n\t\t\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"request2\")\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || (err != nats.ErrTimeout && err != nats.ErrNoResponders) {\n\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t\t}\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\t// Just to make sure that message was not delivered to service\n\t\t\t\t\t\t// responders, wait a bit and check the recv value.\n\t\t\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\t\t\tfor _, sub := range []*nats.Subscription{subSvcA, subSvcC} {\n\t\t\t\t\t\t\tif msg, err := sub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expected no message to be received, but service subscription got %s\", msg.Data)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tcheck := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"D\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bat\")\n\t\t\t\t\t\t\tsm := e.SubjectMapping()\n\t\t\t\t\t\t\trequire_True(t, sm != nil)\n\t\t\t\t\t\t\trequire_Equal[string](t, sm.MappedTo, \"baz\")\n\t\t\t\t\t\t\tsimps := e.ServiceImports()\n\t\t\t\t\t\t\trequire_True(t, simps != nil)\n\t\t\t\t\t\t\tvar expectedServices int\n\t\t\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\t\t\texpectedServices = 3\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\texpectedServices = 2\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\trequire_Equal[int](t, len(simps), expectedServices)\n\t\t\t\t\t\t\tfor _, si := range simps {\n\t\t\t\t\t\t\t\tswitch si.Account {\n\t\t\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"baz\")\n\t\t\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bar\")\n\t\t\t\t\t\t\t\tcase \"A\":\n\t\t\t\t\t\t\t\t\tif !mainTest.allow {\n\t\t\t\t\t\t\t\t\t\tt.Fatalf(\"Without allow_trace, we should not see service for account A\")\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"bar\")\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bozo\")\n\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected account name: %s\", si.Account)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\tif !mainTest.allow {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 0)\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t\t\tswitch eg.Kind {\n\t\t\t\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, sfornc2.Name())\n\t\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, sfornc3.Name())\n\t\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc2.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bozo\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"ServiceA\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc3.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"C\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"baz\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"ServiceC\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// We should receive 3 events when allowed, a single when not.\n\t\t\t\t\tcheck()\n\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\tcheck()\n\t\t\t\t\t\tcheck()\n\t\t\t\t\t}\n\t\t\t\t\t// Make sure we are not receiving more traces\n\t\t\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t\t\t}\n\t\t\t\t\t// Make sure that we never receive on any of the account\n\t\t\t\t\t// trace destination's sub.\n\t\t\t\t\tfor _, sub := range accSubs {\n\t\t\t\t\t\tif tm, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message on account's trace sub, got %s\", tm.Data)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Make sure we properly remove the responses.\n\t\t\t\t\tcheckResp := func(an string) {\n\t\t\t\t\t\tfor _, s := range []*Server{sfornc, sfornc2, sfornc3} {\n\t\t\t\t\t\t\tacc, err := s.lookupAccount(an)\n\t\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\t\t\t\tif n := acc.NumPendingAllResponses(); n != 0 {\n\t\t\t\t\t\t\t\t\treturn fmt.Errorf(\"Still %d responses pending for account %q on server %s\", n, acc, s)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tfor _, acc := range []string{\"A\", \"B\", \"C\", \"D\"} {\n\t\t\t\t\t\tcheckResp(acc)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceServiceImportWithLeafNodeHub(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S1\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [ { service: \">\", allow_trace: true } ]\n\t\t\t\tmappings = {\n\t\t\t\t\tbar: bozo\n\t\t\t\t}\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [ { service: {account: \"A\", subject:\">\"} } ]\n\t\t\t\texports: [ { service: \">\", allow_trace: true } ]\n\t\t\t}\n\t\t\tC {\n\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\texports: [ { service: \">\", allow_trace: true } ]\n\t\t\t}\n\t\t\tD {\n\t\t\t\tusers: [{user: d, password: pwd}]\n\t\t\t\timports: [\n\t\t\t\t\t{ service: {account: \"B\", subject:\"bar\"}, to: baz }\n\t\t\t\t\t{ service: {account: \"C\", subject:\">\"} }\n\t\t\t\t]\n\t\t\t\tmappings = {\n\t\t\t\t\t\tbat: baz\n\t\t\t\t}\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(confHub)\n\tdefer hub.Shutdown()\n\n\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S2\"\n\t\tleafnodes {\n\t\t\tremotes [{url: \"nats://d:pwd@127.0.0.1:%d\"}]\n\t\t}\n\t`, ohub.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(confLeaf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, hub, 1)\n\tcheckLeafNodeConnectedCount(t, leaf, 1)\n\n\tnc2 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"ServiceA\"))\n\tdefer nc2.Close()\n\trecv := int32(0)\n\tnatsQueueSub(t, nc2, \"*\", \"my_queue\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&recv, 1)\n\t\tm.Respond(m.Data)\n\t})\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"ServiceC\"))\n\tdefer nc3.Close()\n\tnatsSub(t, nc3, \"baz\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&recv, 1)\n\t\tm.Respond(m.Data)\n\t})\n\tnatsFlush(t, nc3)\n\n\tnc := natsConnect(t, leaf.ClientURL(), nats.Name(\"Requestor\"))\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tsub := natsSubSync(t, nc, \"my.service.response.inbox\")\n\n\tcheckSubInterest(t, leaf, globalAccountName, \"bat\", time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"bat\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Data = []byte(\"request1\")\n\t\t\t} else {\n\t\t\t\tmsg.Data = []byte(\"request2\")\n\t\t\t}\n\t\t\tmsg.Reply = sub.Subject\n\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.deliverMsg {\n\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"request2\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Check that no (more) messages are received.\n\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t}\n\t\t\tif !test.deliverMsg {\n\t\t\t\t// Just to make sure that message was not delivered to service\n\t\t\t\t// responders, wait a bit and check the recv value.\n\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\tif n := atomic.LoadInt32(&recv); n != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected no message to be received, but service callback fired %d times\", n)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, \"S2\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, globalAccountName)\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bat\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == LEAF)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, \"S1\")\n\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\tcase LEAF:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"D\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bat\")\n\t\t\t\t\tsm := e.SubjectMapping()\n\t\t\t\t\trequire_True(t, sm != nil)\n\t\t\t\t\trequire_Equal[string](t, sm.MappedTo, \"baz\")\n\t\t\t\t\tsimps := e.ServiceImports()\n\t\t\t\t\trequire_True(t, simps != nil)\n\t\t\t\t\trequire_Equal[int](t, len(simps), 3)\n\t\t\t\t\tfor _, si := range simps {\n\t\t\t\t\t\tswitch si.Account {\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"baz\")\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bar\")\n\t\t\t\t\t\tcase \"A\":\n\t\t\t\t\t\t\trequire_Equal[string](t, si.From, \"bar\")\n\t\t\t\t\t\t\trequire_Equal[string](t, si.To, \"bozo\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected account name: %s\", si.Account)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\tswitch eg.Account {\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"ServiceC\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"baz\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tcase \"A\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"ServiceA\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We should receive 2 events.\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\n\t\t\t// Make sure we properly remove the responses.\n\t\t\tcheckResp := func(an string) {\n\t\t\t\tacc, err := hub.lookupAccount(an)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\tif n := acc.NumPendingAllResponses(); n != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Still %d responses for account %q pending on %s\", n, an, hub)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tfor _, acc := range []string{\"A\", \"B\", \"C\", \"D\"} {\n\t\t\t\tcheckResp(acc)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceServiceImportWithLeafNodeLeaf(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S1\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [ { service: \"bar\", allow_trace: true } ]\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [{ service: {account: \"A\", subject:\"bar\"}, to: baz }]\n\t\t\t}\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(confHub)\n\tdefer hub.Shutdown()\n\n\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S2\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [ { service: \"bar\"} ]\n\t\t\t}\n\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: A\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://b:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: B\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, ohub.LeafNode.Port, ohub.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(confLeaf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, hub, 2)\n\tcheckLeafNodeConnectedCount(t, leaf, 2)\n\n\tnc2 := natsConnect(t, leaf.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"ServiceA\"))\n\tdefer nc2.Close()\n\trecv := int32(0)\n\tnatsQueueSub(t, nc2, \"*\", \"my_queue\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&recv, 1)\n\t\tm.Respond(m.Data)\n\t})\n\tnatsFlush(t, nc2)\n\n\tnc := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"Requestor\"))\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tsub := natsSubSync(t, nc, \"my.service.response.inbox\")\n\n\t// Check that hub has a subscription interest on \"baz\"\n\tcheckSubInterest(t, hub, \"A\", \"baz\", time.Second)\n\t// And check that the leaf has the sub interest on the trace subject\n\tcheckSubInterest(t, leaf, \"B\", traceSub.Subject, time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"baz\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Data = []byte(\"request1\")\n\t\t\t} else {\n\t\t\t\tmsg.Data = []byte(\"request2\")\n\t\t\t}\n\t\t\tmsg.Reply = sub.Subject\n\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.deliverMsg {\n\t\t\t\tappMsg := natsNexMsg(t, sub, time.Second)\n\t\t\t\trequire_Equal[string](t, string(appMsg.Data), \"request2\")\n\t\t\t}\n\t\t\t// Check that no (more) messages are received.\n\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || (err != nats.ErrTimeout && err != nats.ErrNoResponders) {\n\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t}\n\t\t\tif !test.deliverMsg {\n\t\t\t\t// Just to make sure that message was not delivered to service\n\t\t\t\t// responders, wait a bit and check the recv value.\n\t\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t\tif n := atomic.LoadInt32(&recv); n != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected no message to be received, but service callback fired %d times\", n)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, \"S1\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Requestor\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"B\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"baz\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\tsimps := e.ServiceImports()\n\t\t\t\t\trequire_True(t, simps != nil)\n\t\t\t\t\trequire_Equal[int](t, len(simps), 1)\n\t\t\t\t\tsi := simps[0]\n\t\t\t\t\trequire_Equal[string](t, si.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, si.From, \"baz\")\n\t\t\t\t\trequire_Equal[string](t, si.To, \"bar\")\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == LEAF)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, \"S2\")\n\t\t\t\t\trequire_Equal[string](t, eg.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\tcase LEAF:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, \"ServiceA\")\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"*\")\n\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We should receive 2 events.\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\n\t\t\t// Make sure we properly remove the responses.\n\t\t\tcheckResp := func(an string) {\n\t\t\t\tacc, err := leaf.lookupAccount(an)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\tif n := acc.NumPendingAllResponses(); n != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Still %d responses for account %q pending on %s\", n, an, leaf)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tfor _, acc := range []string{\"A\", \"B\"} {\n\t\t\t\tcheckResp(acc)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceStreamExport(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [\n\t\t\t\t\t{ stream: \"info.*.*.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"B.info.$2.$1.>\", allow_trace: %v } ]\n\t\t\t}\n\t\t\tC {\n\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"C.info.$1.$2.>\", allow_trace: %v } ]\n\t\t\t}\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, false, false)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"Tracer\"))\n\tdefer nc.Close()\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\n\tnc2 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"sub1\"))\n\tdefer nc2.Close()\n\tsub1 := natsSubSync(t, nc2, \"B.info.*.*.>\")\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, s.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"sub2\"))\n\tdefer nc3.Close()\n\tsub2 := natsQueueSubSync(t, nc3, \"C.info.>\", \"my_queue\")\n\tnatsFlush(t, nc3)\n\n\tfor mainIter, mainTest := range []struct {\n\t\tname string\n\t\tallow bool\n\t}{\n\t\t{\"not allowed\", false},\n\t\t{\"allowed\", true},\n\t\t{\"not allowed again\", false},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"info.11.22.bar\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Data = []byte(\"hello\")\n\n\t\t\t\t\terr := nc.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\tappMsg := natsNexMsg(t, sub1, time.Second)\n\t\t\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"B.info.22.11.bar\")\n\t\t\t\t\t\tappMsg = natsNexMsg(t, sub2, time.Second)\n\t\t\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"C.info.11.22.bar\")\n\t\t\t\t\t}\n\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || err != nats.ErrTimeout {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\tstexps := e.StreamExports()\n\t\t\t\t\trequire_True(t, stexps != nil)\n\t\t\t\t\trequire_Equal[int](t, len(stexps), 2)\n\t\t\t\t\tfor _, se := range stexps {\n\t\t\t\t\t\trequire_True(t, se.Timestamp != time.Time{})\n\t\t\t\t\t\tswitch se.Account {\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"B.info.22.11.bar\")\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"C.info.11.22.bar\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected stream export: %+v\", se)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\tswitch eg.Account {\n\t\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Equal[int](t, len(egress), 0)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t\tswitch mainIter {\n\t\t\tcase 0:\n\t\t\t\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, true, true))\n\t\t\tcase 1:\n\t\t\t\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, false, false))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceStreamExportWithSuperCluster(t *testing.T) {\n\tfor _, mainTest := range []struct {\n\t\tname string\n\t\tallowStr string\n\t\tallow bool\n\t}{\n\t\t{\"allowed\", \"true\", true},\n\t\t{\"not allowed\", \"false\", false},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tserver_name: %s\n\t\t\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\t\t\tcluster {\n\t\t\t\t\tname: %s\n\t\t\t\t\tlisten: 127.0.0.1:%d\n\t\t\t\t\troutes = [%s]\n\t\t\t\t}\n\t\t\t\taccounts {\n\t\t\t\t\tA {\n\t\t\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\t\t\texports: [\n\t\t\t\t\t\t\t{ stream: \"info.*.*.>\"}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t\tB {\n\t\t\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"B.info.$2.$1.>\", allow_trace: ` + mainTest.allowStr + ` } ]\n\t\t\t\t\t}\n\t\t\t\t\tC {\n\t\t\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"C.info.$1.$2.>\", allow_trace: ` + mainTest.allowStr + ` } ]\n\t\t\t\t\t}\n\t\t\t\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t\t\t\t}\n\t\t\t`\n\t\t\tsc := createJetStreamSuperClusterWithTemplate(t, tmpl, 2, 2)\n\t\t\tdefer sc.shutdown()\n\n\t\t\tsfornc := sc.clusters[0].servers[0]\n\t\t\tnc := natsConnect(t, sfornc.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"Tracer\"))\n\t\t\tdefer nc.Close()\n\t\t\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\n\t\t\tsfornc2 := sc.clusters[0].servers[1]\n\t\t\tnc2 := natsConnect(t, sfornc2.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"sub1\"))\n\t\t\tdefer nc2.Close()\n\t\t\tsub1 := natsSubSync(t, nc2, \"B.info.*.*.>\")\n\t\t\tnatsFlush(t, nc2)\n\t\t\tcheckSubInterest(t, sfornc2, \"A\", traceSub.Subject, time.Second)\n\n\t\t\tsfornc3 := sc.clusters[1].servers[0]\n\t\t\tnc3 := natsConnect(t, sfornc3.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"sub2\"))\n\t\t\tdefer nc3.Close()\n\t\t\tsub2 := natsQueueSubSync(t, nc3, \"C.info.>\", \"my_queue\")\n\t\t\tnatsFlush(t, nc3)\n\n\t\t\tcheckSubInterest(t, sfornc, \"A\", \"info.1.2.3.4\", time.Second)\n\t\t\tfor _, s := range sc.clusters[0].servers {\n\t\t\t\tcheckForRegisteredQSubInterest(t, s, \"C2\", \"A\", \"info.1.2.3\", 1, time.Second)\n\t\t\t}\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(\"info.11.22.bar\")\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\tmsg.Data = []byte(\"hello\")\n\n\t\t\t\t\terr := nc.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\tappMsg := natsNexMsg(t, sub1, time.Second)\n\t\t\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"B.info.22.11.bar\")\n\t\t\t\t\t\tappMsg = natsNexMsg(t, sub2, time.Second)\n\t\t\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"C.info.11.22.bar\")\n\t\t\t\t\t}\n\t\t\t\t\t// Check that no (more) messages are received.\n\t\t\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || err != nats.ErrTimeout {\n\t\t\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tcheck := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\t\t\trequire_True(t, e.StreamExports() == nil)\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\t\t\tswitch eg.Kind {\n\t\t\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, sfornc2.Name())\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, sfornc3.Name())\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc2.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, sfornc.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\t\t\tstexps := e.StreamExports()\n\t\t\t\t\t\t\trequire_True(t, stexps != nil)\n\t\t\t\t\t\t\trequire_Equal[int](t, len(stexps), 1)\n\t\t\t\t\t\t\tse := stexps[0]\n\t\t\t\t\t\t\trequire_Equal[string](t, se.Account, \"B\")\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"B.info.22.11.bar\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 0)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, sfornc3.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, sfornc.Name())\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\t\t\tstexps := e.StreamExports()\n\t\t\t\t\t\t\trequire_True(t, stexps != nil)\n\t\t\t\t\t\t\trequire_Equal[int](t, len(stexps), 1)\n\t\t\t\t\t\t\tse := stexps[0]\n\t\t\t\t\t\t\trequire_Equal[string](t, se.Account, \"C\")\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"C.info.11.22.bar\")\n\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\tif mainTest.allow {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\t\teg := egress[0]\n\t\t\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 0)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// We expect 3 events\n\t\t\t\t\tcheck()\n\t\t\t\t\tcheck()\n\t\t\t\t\tcheck()\n\t\t\t\t\t// Make sure we are not receiving more traces\n\t\t\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceStreamExportWithLeafNode_Hub(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S1\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [\n\t\t\t\t\t{ stream: \"info.*.*.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"B.info.$2.$1.>\", allow_trace: true } ]\n\t\t\t}\n\t\t\tC {\n\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"C.info.$1.$2.>\", allow_trace: true } ]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(confHub)\n\tdefer hub.Shutdown()\n\n\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S2\"\n\t\taccounts {\n\t\t\tLEAF { users: [{user: leaf, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://a:pwd@127.0.0.1:%d\", account: \"LEAF\" }\n\t\t\t]\n\t\t}\n\t`, ohub.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(confLeaf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, hub, 1)\n\tcheckLeafNodeConnectedCount(t, leaf, 1)\n\n\tnc := natsConnect(t, leaf.ClientURL(), nats.UserInfo(\"leaf\", \"pwd\"), nats.Name(\"Tracer\"))\n\tdefer nc.Close()\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\n\tcheckSubInterest(t, hub, \"A\", traceSub.Subject, time.Second)\n\n\tnc2 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"sub1\"))\n\tdefer nc2.Close()\n\tsub1 := natsSubSync(t, nc2, \"B.info.*.*.>\")\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"sub2\"))\n\tdefer nc3.Close()\n\tsub2 := natsQueueSubSync(t, nc3, \"C.info.>\", \"my_queue\")\n\tnatsFlush(t, nc3)\n\n\tacc, err := leaf.LookupAccount(\"LEAF\")\n\trequire_NoError(t, err)\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tacc.mu.RLock()\n\t\tsl := acc.sl\n\t\tacc.mu.RUnlock()\n\t\tr := sl.Match(\"info.1.2.3\")\n\t\tok := len(r.psubs) > 0\n\t\tif ok && (len(r.qsubs) == 0 || len(r.qsubs[0]) == 0) {\n\t\t\tok = false\n\t\t}\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"Subscription interest not yet propagated\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"info.11.22.bar\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello\")\n\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.deliverMsg {\n\t\t\t\tappMsg := natsNexMsg(t, sub1, time.Second)\n\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"B.info.22.11.bar\")\n\t\t\t\tappMsg = natsNexMsg(t, sub2, time.Second)\n\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"C.info.11.22.bar\")\n\t\t\t}\n\t\t\t// Check that no (more) messages are received.\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"LEAF\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\trequire_True(t, e.StreamExports() == nil)\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == LEAF)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\tcase LEAF:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\tstexps := e.StreamExports()\n\t\t\t\t\trequire_True(t, stexps != nil)\n\t\t\t\t\trequire_Equal[int](t, len(stexps), 2)\n\t\t\t\t\tfor _, se := range stexps {\n\t\t\t\t\t\tswitch se.Account {\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"B.info.22.11.bar\")\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"C.info.11.22.bar\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected stream export: %+v\", se)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\tswitch eg.Account {\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We expect 2 events\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceStreamExportWithLeafNode_Leaf(t *testing.T) {\n\tconfHub := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S1\"\n\t\taccounts {\n\t\t\tHUB { users: [{user: hub, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\thub, ohub := RunServerWithConfig(confHub)\n\tdefer hub.Shutdown()\n\n\tconfLeaf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: \"S2\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\texports: [\n\t\t\t\t\t{ stream: \"info.*.*.>\"}\n\t\t\t\t]\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: b, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"B.info.$2.$1.>\", allow_trace: true } ]\n\t\t\t}\n\t\t\tC {\n\t\t\t\tusers: [{user: c, password: pwd}]\n\t\t\t\timports: [ { stream: {account: \"A\", subject:\"info.*.*.>\"}, to: \"C.info.$1.$2.>\", allow_trace: true } ]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://hub:pwd@127.0.0.1:%d\", account: \"A\" }\n\t\t\t]\n\t\t}\n\t`, ohub.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(confLeaf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnectedCount(t, hub, 1)\n\tcheckLeafNodeConnectedCount(t, leaf, 1)\n\n\tnc := natsConnect(t, hub.ClientURL(), nats.UserInfo(\"hub\", \"pwd\"), nats.Name(\"Tracer\"))\n\tdefer nc.Close()\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\n\tcheckSubInterest(t, leaf, \"A\", traceSub.Subject, time.Second)\n\n\tnc2 := natsConnect(t, leaf.ClientURL(), nats.UserInfo(\"b\", \"pwd\"), nats.Name(\"sub1\"))\n\tdefer nc2.Close()\n\tsub1 := natsSubSync(t, nc2, \"B.info.*.*.>\")\n\tnatsFlush(t, nc2)\n\n\tnc3 := natsConnect(t, leaf.ClientURL(), nats.UserInfo(\"c\", \"pwd\"), nats.Name(\"sub2\"))\n\tdefer nc3.Close()\n\tsub2 := natsQueueSubSync(t, nc3, \"C.info.>\", \"my_queue\")\n\tnatsFlush(t, nc3)\n\n\tacc, err := hub.LookupAccount(\"HUB\")\n\trequire_NoError(t, err)\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tacc.mu.RLock()\n\t\tsl := acc.sl\n\t\tacc.mu.RUnlock()\n\t\tr := sl.Match(\"info.1.2.3\")\n\t\tok := len(r.psubs) > 0\n\t\tif ok && (len(r.qsubs) == 0 || len(r.qsubs[0]) == 0) {\n\t\t\tok = false\n\t\t}\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"Subscription interest not yet propagated\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"info.11.22.bar\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = []byte(\"hello\")\n\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.deliverMsg {\n\t\t\t\tappMsg := natsNexMsg(t, sub1, time.Second)\n\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"B.info.22.11.bar\")\n\t\t\t\tappMsg = natsNexMsg(t, sub2, time.Second)\n\t\t\t\trequire_Equal[string](t, appMsg.Subject, \"C.info.11.22.bar\")\n\t\t\t}\n\t\t\t// Check that no (more) messages are received.\n\t\t\tfor _, sub := range []*nats.Subscription{sub1, sub2} {\n\t\t\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); msg != nil || err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Did not expect application message, got msg=%v err=%v\", msg, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck := func() {\n\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"HUB\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\trequire_True(t, e.StreamExports() == nil)\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\teg := egress[0]\n\t\t\t\t\trequire_True(t, eg.Kind == LEAF)\n\t\t\t\t\trequire_Equal[string](t, eg.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, eg.Account, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Subscription, _EMPTY_)\n\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\tcase LEAF:\n\t\t\t\t\trequire_Equal[string](t, e.Server.Name, leaf.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Name, hub.Name())\n\t\t\t\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\t\t\t\trequire_Equal[string](t, ingress.Subject, \"info.11.22.bar\")\n\t\t\t\t\trequire_True(t, e.SubjectMapping() == nil)\n\t\t\t\t\trequire_True(t, e.ServiceImports() == nil)\n\t\t\t\t\tstexps := e.StreamExports()\n\t\t\t\t\trequire_True(t, stexps != nil)\n\t\t\t\t\trequire_Equal[int](t, len(stexps), 2)\n\t\t\t\t\tfor _, se := range stexps {\n\t\t\t\t\t\tswitch se.Account {\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"B.info.22.11.bar\")\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, se.To, \"C.info.11.22.bar\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected stream export: %+v\", se)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\trequire_Equal[int](t, len(egress), 2)\n\t\t\t\t\tfor _, eg := range egress {\n\t\t\t\t\t\trequire_True(t, eg.Kind == CLIENT)\n\t\t\t\t\t\tswitch eg.Account {\n\t\t\t\t\t\tcase \"B\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub1\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t\t\t\t\t\tcase \"C\":\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Name, \"sub2\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info.*.*.>\")\n\t\t\t\t\t\t\trequire_Equal[string](t, eg.Queue, \"my_queue\")\n\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We expect 2 events\n\t\t\tcheck()\n\t\t\tcheck()\n\t\t\t// Make sure we are not receiving more traces\n\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceJetStream(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.JetStream = true\n\topts.JetStreamMaxMemory = 270\n\topts.StoreDir = t.TempDir()\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tStorage: nats.MemoryStorage,\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1,\n\t\tAllowRollup: true,\n\t\tSubjectTransform: &nats.SubjectTransformConfig{\n\t\t\tSource: \"foo\",\n\t\t\tDestination: \"bar\",\n\t\t},\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tnct := natsConnect(t, s.ClientURL(), nats.Name(\"Tracer\"))\n\tdefer nct.Close()\n\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\tnatsFlush(t, nct)\n\n\tmsg := nats.NewMsg(\"foo\")\n\tmsg.Header.Set(JSMsgId, \"MyId\")\n\tmsg.Data = make([]byte, 50)\n\t_, err = js.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\tcheckStream := func(t *testing.T, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := si.State.Msgs; int(n) != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d messages, got %v\", expected, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckStream(t, 1)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdeliverMsg bool\n\t}{\n\t\t{\"just trace\", false},\n\t\t{\"deliver msg\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tif !test.deliverMsg {\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t}\n\t\t\tmsg.Data = make([]byte, 50)\n\t\t\terr = nct.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Wait a bit and check if message should be in the stream or not.\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tif test.deliverMsg {\n\t\t\t\tcheckStream(t, 2)\n\t\t\t} else {\n\t\t\t\tcheckStream(t, 1)\n\t\t\t}\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\tjs := e.JetStream()\n\t\t\trequire_True(t, js != nil)\n\t\t\trequire_True(t, js.Timestamp != time.Time{})\n\t\t\trequire_Equal[string](t, js.Stream, \"TEST\")\n\t\t\trequire_Equal[string](t, js.Subject, \"bar\")\n\t\t\trequire_False(t, js.NoInterest)\n\t\t\trequire_Equal[string](t, js.Error, _EMPTY_)\n\t\t})\n\t}\n\n\tjst, err := nct.JetStream()\n\trequire_NoError(t, err)\n\n\tmset, err := s.globalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Now we will not ask for delivery and use headers that will fail checks\n\t// and make sure that message is not added, that the stream's clfs is not\n\t// increased, and that the JS trace shows the error.\n\tnewMsg := func() *nats.Msg {\n\t\tmsg = nats.NewMsg(\"foo\")\n\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\tmsg.Data = []byte(\"hello\")\n\t\treturn msg\n\t}\n\n\tmsgCount := 2\n\tfor _, test := range []struct {\n\t\tname string\n\t\theaderName string\n\t\theaderVal string\n\t\texpectedErr string\n\t\tspecial int\n\t}{\n\t\t{\"unexpected stream name\", JSExpectedStream, \"WRONG\", \"expected stream does not match\", 0},\n\t\t{\"duplicate id\", JSMsgId, \"MyId\", \"duplicate\", 0},\n\t\t{\"last seq by subject mismatch\", JSExpectedLastSubjSeq, \"10\", \"last sequence by subject mismatch\", 0},\n\t\t{\"last seq mismatch\", JSExpectedLastSeq, \"10\", \"last sequence mismatch\", 0},\n\t\t{\"last msgid mismatch\", JSExpectedLastMsgId, \"MyId3\", \"last msgid mismatch\", 0},\n\t\t{\"invalid rollup command\", JSMsgRollup, \"wrong\", \"rollup value invalid: \\\"wrong\\\"\", 0},\n\t\t{\"rollup not permitted\", JSMsgRollup, JSMsgRollupSubject, \"rollup not permitted\", 1},\n\t\t{\"max msg size\", _EMPTY_, _EMPTY_, ErrMaxPayload.Error(), 2},\n\t\t{\"normal message ok\", _EMPTY_, _EMPTY_, _EMPTY_, 3},\n\t\t{\"insufficient resources\", _EMPTY_, _EMPTY_, NewJSInsufficientResourcesError().Error(), 0},\n\t\t{\"stream sealed\", _EMPTY_, _EMPTY_, NewJSStreamSealedError().Error(), 4},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg = newMsg()\n\t\t\tif test.headerName != _EMPTY_ {\n\t\t\t\tmsg.Header.Set(test.headerName, test.headerVal)\n\t\t\t}\n\t\t\tswitch test.special {\n\t\t\tcase 1:\n\t\t\t\t// Update stream to prevent rollups, and set a max size.\n\t\t\t\tcfg.AllowRollup = false\n\t\t\t\tcfg.MaxMsgSize = 100\n\t\t\t\t_, err = js.UpdateStream(cfg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\tcase 2:\n\t\t\t\tmsg.Data = make([]byte, 200)\n\t\t\tcase 3:\n\t\t\t\tpa, err := jst.Publish(\"foo\", make([]byte, 100))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tmsgCount++\n\t\t\t\tcheckStream(t, msgCount)\n\t\t\t\trequire_Equal[uint64](t, pa.Sequence, 3)\n\t\t\t\treturn\n\t\t\tcase 4:\n\t\t\t\tcfg.Sealed = true\n\t\t\t\t_, err = js.UpdateStream(cfg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\tdefault:\n\t\t\t}\n\t\t\tjst.PublishMsg(msg)\n\n\t\t\t// Message count should not have increased and stay at 2.\n\t\t\tcheckStream(t, msgCount)\n\t\t\t// Check that clfs does not increase\n\t\t\tmset.mu.RLock()\n\t\t\tclfs := mset.getCLFS()\n\t\t\tmset.mu.RUnlock()\n\t\t\tif clfs != 0 {\n\t\t\t\tt.Fatalf(\"Stream's clfs was expected to be 0, is %d\", clfs)\n\t\t\t}\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\tjs := e.JetStream()\n\t\t\trequire_True(t, js != nil)\n\t\t\trequire_Equal[string](t, js.Stream, \"TEST\")\n\t\t\trequire_Equal[string](t, js.Subject, _EMPTY_)\n\t\t\trequire_False(t, js.NoInterest)\n\t\t\tif et := js.Error; !strings.Contains(et, test.expectedErr) {\n\t\t\t\tt.Fatalf(\"Expected JS error to contain %q, got %q\", test.expectedErr, et)\n\t\t\t}\n\t\t})\n\t}\n\n\t// Create a stream with interest retention policy\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"NO_INTEREST\",\n\t\tSubjects: []string{\"baz\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\tmsg = nats.NewMsg(\"baz\")\n\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\tmsg.Data = []byte(\"hello\")\n\terr = nct.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\tvar e MsgTraceEvent\n\tjson.Unmarshal(traceMsg.Data, &e)\n\trequire_Equal[string](t, e.Server.Name, s.Name())\n\tingress := e.Ingress()\n\trequire_True(t, ingress != nil)\n\trequire_True(t, ingress.Kind == CLIENT)\n\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\trequire_Equal[int](t, len(e.Egresses()), 0)\n\tejs := e.JetStream()\n\trequire_True(t, js != nil)\n\trequire_Equal[string](t, ejs.Stream, \"NO_INTEREST\")\n\trequire_Equal[string](t, ejs.Subject, \"baz\")\n\trequire_True(t, ejs.NoInterest)\n\trequire_Equal[string](t, ejs.Error, _EMPTY_)\n\n\t// Create a new stream to just check that when consumer a JS message,\n\t// the trace header has been disabled.\n\tcfg = &nats.StreamConfig{\n\t\tName: \"TEST_TRACE_DISABLED\",\n\t\tSubjects: []string{\"disabled\"},\n\t\tRetention: nats.LimitsPolicy,\n\t}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"disabled\", \"dur\")\n\trequire_NoError(t, err)\n\n\tmsg = nats.NewMsg(\"disabled\")\n\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\tmsg.Data = []byte(\"hello\")\n\terr = nct.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// Check trace msg is still OK\n\ttraceMsg = natsNexMsg(t, traceSub, time.Second)\n\te = MsgTraceEvent{}\n\tjson.Unmarshal(traceMsg.Data, &e)\n\trequire_Equal[string](t, e.Server.Name, s.Name())\n\tingress = e.Ingress()\n\trequire_True(t, ingress != nil)\n\trequire_True(t, ingress.Kind == CLIENT)\n\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\trequire_Equal[int](t, len(e.Egresses()), 0)\n\tejs = e.JetStream()\n\trequire_True(t, js != nil)\n\trequire_Equal[string](t, ejs.Stream, \"TEST_TRACE_DISABLED\")\n\trequire_Equal[string](t, ejs.Subject, \"disabled\")\n\trequire_False(t, ejs.NoInterest)\n\trequire_Equal[string](t, ejs.Error, _EMPTY_)\n\n\t// Now consume the message.\n\tjmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(jmsgs), 1)\n\tjmsg := jmsgs[0]\n\trequire_True(t, len(jmsg.Header) > 0)\n\trequire_Equal(t, string(jmsg.Data), \"hello\")\n\t// When consuming, the message tracing should have been disabled,\n\t// so we should have the MsgTraceDest header set to MsgTraceDestDisabled\n\trequire_Equal(t, jmsg.Header.Get(MsgTraceDest), MsgTraceDestDisabled)\n\n\t// Verify that no trace message was generated.\n\t_, err = traceSub.NextMsg(100 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n}\n\nfunc TestMsgTraceJetStreamWithSuperCluster(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\ttraceDest := \"my.trace.subj\"\n\n\t// Hack to set the trace destination for the global account in order\n\t// to make sure that the traceParentHdr header is disabled when a message\n\t// is stored in JetStream, which will prevent emitting a trace\n\t// when such message is retrieved and traverses a route.\n\t// Without the account destination set, the trace would not be\n\t// triggered, but that does not mean that we would have been\n\t// doing the right thing of disabling the header.\n\tfor _, cl := range sc.clusters {\n\t\tfor _, s := range cl.servers {\n\t\t\tacc, err := s.LookupAccount(globalAccountName)\n\t\t\trequire_NoError(t, err)\n\t\t\tacc.setTraceDest(traceDest)\n\t\t}\n\t}\n\n\tc1 := sc.clusters[0]\n\tc2 := sc.clusters[1]\n\tnc, js := jsClientConnect(t, c1.randomServer())\n\tdefer nc.Close()\n\n\tcheckStream := func(t *testing.T, stream string, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 15*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif n := si.State.Msgs; int(n) != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d messages for stream %q, got %v\", expected, stream, n)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tpayload := make([]byte, 50)\n\n\tfor mainIter, mainTest := range []struct {\n\t\tname string\n\t\tstream string\n\t}{\n\t\t{\"from stream leader\", \"TEST1\"},\n\t\t{\"from non stream leader\", \"TEST2\"},\n\t\t{\"from other cluster\", \"TEST3\"},\n\t} {\n\t\tt.Run(mainTest.name, func(t *testing.T) {\n\t\t\tcfg := &nats.StreamConfig{\n\t\t\t\tName: mainTest.stream,\n\t\t\t\tReplicas: 3,\n\t\t\t\tAllowRollup: true,\n\t\t\t}\n\t\t\t_, err := js.AddStream(cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\tsc.waitOnStreamLeader(globalAccountName, mainTest.stream)\n\n\t\t\t// The streams are created from c1 cluster.\n\t\t\tslSrv := c1.streamLeader(globalAccountName, mainTest.stream)\n\n\t\t\t// Store some messages\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\t_, err = js.Publish(mainTest.stream, payload)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// We will connect the app that sends the trace message to a server\n\t\t\t// that is either the stream leader, a random server in c1, or\n\t\t\t// a server in c2 (to go through a GW).\n\t\t\tvar s *Server\n\t\t\tswitch mainIter {\n\t\t\tcase 0:\n\t\t\t\ts = slSrv\n\t\t\tcase 1:\n\t\t\t\ts = c1.randomNonStreamLeader(globalAccountName, mainTest.stream)\n\t\t\tcase 2:\n\t\t\t\ts = c2.randomServer()\n\t\t\t}\n\n\t\t\tnct := natsConnect(t, s.ClientURL(), nats.Name(\"Tracer\"))\n\t\t\tdefer nct.Close()\n\n\t\t\ttraceSub := natsSubSync(t, nct, traceDest)\n\t\t\tnatsFlush(t, nct)\n\n\t\t\tfor _, test := range []struct {\n\t\t\t\tname string\n\t\t\t\tdeliverMsg bool\n\t\t\t}{\n\t\t\t\t{\"just trace\", false},\n\t\t\t\t{\"deliver msg\", true},\n\t\t\t} {\n\t\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\t\tmsg := nats.NewMsg(mainTest.stream)\n\t\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\t\tif !test.deliverMsg {\n\t\t\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\t\t}\n\t\t\t\t\t// We add the traceParentHdr header to make sure that it\n\t\t\t\t\t// is deactivated in addition to the Nats-Trace-Dest\n\t\t\t\t\t// header too when needed.\n\t\t\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t\t\tmsg.Header.Set(JSMsgId, \"MyId\")\n\t\t\t\t\tmsg.Data = payload\n\t\t\t\t\terr = nct.PublishMsg(msg)\n\t\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\t\tif test.deliverMsg {\n\t\t\t\t\t\tcheckStream(t, mainTest.stream, 6)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tcheckStream(t, mainTest.stream, 5)\n\t\t\t\t\t}\n\n\t\t\t\t\tvar (\n\t\t\t\t\t\tclientOK bool\n\t\t\t\t\t\tgatewayOK bool\n\t\t\t\t\t\trouteOK bool\n\t\t\t\t\t)\n\n\t\t\t\t\tcheck := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\t\tcheckJS := func() {\n\t\t\t\t\t\t\tt.Helper()\n\t\t\t\t\t\t\tjs := e.JetStream()\n\t\t\t\t\t\t\trequire_True(t, js != nil)\n\t\t\t\t\t\t\trequire_Equal[string](t, js.Stream, mainTest.stream)\n\t\t\t\t\t\t\trequire_Equal[string](t, js.Subject, mainTest.stream)\n\t\t\t\t\t\t\trequire_False(t, js.NoInterest)\n\t\t\t\t\t\t\trequire_Equal[string](t, js.Error, _EMPTY_)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\tswitch mainIter {\n\t\t\t\t\t\tcase 0:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\tcase 1:\n\t\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\t\tci := egress[0]\n\t\t\t\t\t\t\t\trequire_True(t, ci.Kind == ROUTER)\n\t\t\t\t\t\t\t\trequire_Equal[string](t, ci.Name, slSrv.Name())\n\t\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, slSrv.Name())\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase 2:\n\t\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\t\tci := egress[0]\n\t\t\t\t\t\t\t\trequire_True(t, ci.Kind == GATEWAY)\n\t\t\t\t\t\t\t\t// It could have gone to any server in the C1 cluster.\n\t\t\t\t\t\t\t\t// If it is not the stream leader, it should be\n\t\t\t\t\t\t\t\t// routed to it.\n\t\t\t\t\t\t\t\tclientOK = true\n\t\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, s.Name())\n\t\t\t\t\t\t\t\t// If the server that emitted this event is the\n\t\t\t\t\t\t\t\t// stream leader, then we should have the stream,\n\t\t\t\t\t\t\t\t// otherwise, it should be routed.\n\t\t\t\t\t\t\t\tif e.Server.Name == slSrv.Name() {\n\t\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\t\t\t// Set this so that we know that we don't expect\n\t\t\t\t\t\t\t\t\t// to have the route receive it.\n\t\t\t\t\t\t\t\t\trouteOK = true\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\tegress := e.Egresses()\n\t\t\t\t\t\t\t\t\trequire_Equal[int](t, len(egress), 1)\n\t\t\t\t\t\t\t\t\tci := egress[0]\n\t\t\t\t\t\t\t\t\trequire_True(t, ci.Kind == ROUTER)\n\t\t\t\t\t\t\t\t\trequire_Equal[string](t, ci.Name, slSrv.Name())\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tgatewayOK = true\n\t\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, slSrv.Name())\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\t\trouteOK = true\n\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tcheck()\n\t\t\t\t\tif mainIter > 0 {\n\t\t\t\t\t\t// There will be at least 2 events\n\t\t\t\t\t\tcheck()\n\t\t\t\t\t\t// For the last test, there may be a 3rd.\n\t\t\t\t\t\tif mainIter == 2 && !(clientOK && gatewayOK && routeOK) {\n\t\t\t\t\t\t\tcheck()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Make sure we are not receiving more traces\n\t\t\t\t\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tjst, err := nct.JetStream()\n\t\t\trequire_NoError(t, err)\n\n\t\t\tnewMsg := func() *nats.Msg {\n\t\t\t\tmsg := nats.NewMsg(mainTest.stream)\n\t\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\tmsg.Header.Set(MsgTraceOnly, \"true\")\n\t\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\t\treturn msg\n\t\t\t}\n\n\t\t\tmsgCount := 6\n\t\t\tfor _, subtest := range []struct {\n\t\t\t\tname string\n\t\t\t\theaderName string\n\t\t\t\theaderVal string\n\t\t\t\texpectedErr string\n\t\t\t\tspecial int\n\t\t\t}{\n\t\t\t\t{\"unexpected stream name\", JSExpectedStream, \"WRONG\", \"expected stream does not match\", 0},\n\t\t\t\t{\"duplicate id\", JSMsgId, \"MyId\", \"duplicate\", 0},\n\t\t\t\t{\"last seq by subject mismatch\", JSExpectedLastSubjSeq, \"3\", \"last sequence by subject mismatch\", 0},\n\t\t\t\t{\"last seq mismatch\", JSExpectedLastSeq, \"10\", \"last sequence mismatch\", 0},\n\t\t\t\t{\"last msgid mismatch\", JSExpectedLastMsgId, \"MyId3\", \"last msgid mismatch\", 0},\n\t\t\t\t{\"invalid rollup command\", JSMsgRollup, \"wrong\", \"rollup value invalid: \\\"wrong\\\"\", 0},\n\t\t\t\t{\"rollup not permitted\", JSMsgRollup, JSMsgRollupSubject, \"rollup not permitted\", 1},\n\t\t\t\t{\"max msg size\", _EMPTY_, _EMPTY_, ErrMaxPayload.Error(), 2},\n\t\t\t\t{\"new message ok\", _EMPTY_, _EMPTY_, _EMPTY_, 3},\n\t\t\t\t{\"stream sealed\", _EMPTY_, _EMPTY_, NewJSStreamSealedError().Error(), 4},\n\t\t\t} {\n\t\t\t\tt.Run(subtest.name, func(t *testing.T) {\n\t\t\t\t\tmsg := newMsg()\n\t\t\t\t\tif subtest.headerName != _EMPTY_ {\n\t\t\t\t\t\tmsg.Header.Set(subtest.headerName, subtest.headerVal)\n\t\t\t\t\t}\n\t\t\t\t\tswitch subtest.special {\n\t\t\t\t\tcase 1:\n\t\t\t\t\t\t// Update stream to prevent rollups, and set a max size.\n\t\t\t\t\t\tcfg.AllowRollup = false\n\t\t\t\t\t\tcfg.MaxMsgSize = 100\n\t\t\t\t\t\t_, err = js.UpdateStream(cfg)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tcase 2:\n\t\t\t\t\t\tmsg.Data = make([]byte, 200)\n\t\t\t\t\tcase 3:\n\t\t\t\t\t\tpa, err := jst.Publish(mainTest.stream, []byte(\"hello\"))\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\tmsgCount++\n\t\t\t\t\t\tcheckStream(t, mainTest.stream, msgCount)\n\t\t\t\t\t\trequire_Equal[uint64](t, pa.Sequence, 7)\n\t\t\t\t\t\treturn\n\t\t\t\t\tcase 4:\n\t\t\t\t\t\tcfg.Sealed = true\n\t\t\t\t\t\t_, err = js.UpdateStream(cfg)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t\tjst.PublishMsg(msg)\n\t\t\t\t\tcheckStream(t, mainTest.stream, msgCount)\n\n\t\t\t\t\tvar (\n\t\t\t\t\t\tclientOK bool\n\t\t\t\t\t\tgatewayOK bool\n\t\t\t\t\t\trouteOK bool\n\t\t\t\t\t)\n\n\t\t\t\t\tcheckJSTrace := func() {\n\t\t\t\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\t\t\t\t\tcheckJS := func() {\n\t\t\t\t\t\t\tt.Helper()\n\t\t\t\t\t\t\tjs := e.JetStream()\n\t\t\t\t\t\t\trequire_True(t, e.JetStream() != nil)\n\t\t\t\t\t\t\trequire_Equal[string](t, js.Stream, mainTest.stream)\n\t\t\t\t\t\t\trequire_Equal[string](t, js.Subject, _EMPTY_)\n\t\t\t\t\t\t\trequire_False(t, js.NoInterest)\n\t\t\t\t\t\t\tif et := js.Error; !strings.Contains(et, subtest.expectedErr) {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expected JS error to contain %q, got %q\", subtest.expectedErr, et)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tingress := e.Ingress()\n\t\t\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\t\t\t// We will focus only on the trace message that\n\t\t\t\t\t\t// includes the JetStream event.\n\t\t\t\t\t\tswitch mainIter {\n\t\t\t\t\t\tcase 0:\n\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t\t\t\t\trequire_True(t, ingress.Kind == CLIENT)\n\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, \"Tracer\")\n\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\tcase 1:\n\t\t\t\t\t\t\tif ingress.Kind == ROUTER {\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, slSrv.Name())\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\trequire_True(t, e.JetStream() != nil)\n\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\tcase 2:\n\t\t\t\t\t\t\tswitch ingress.Kind {\n\t\t\t\t\t\t\tcase CLIENT:\n\t\t\t\t\t\t\t\tclientOK = true\n\t\t\t\t\t\t\tcase GATEWAY:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, ingress.Name, s.Name())\n\t\t\t\t\t\t\t\t// If the server that emitted this event is the\n\t\t\t\t\t\t\t\t// stream leader, then we should have the stream,\n\t\t\t\t\t\t\t\t// otherwise, it should be routed.\n\t\t\t\t\t\t\t\tif e.Server.Name == slSrv.Name() {\n\t\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\t\t\t// We don't expect the route event\n\t\t\t\t\t\t\t\t\trouteOK = true\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tgatewayOK = true\n\t\t\t\t\t\t\tcase ROUTER:\n\t\t\t\t\t\t\t\trequire_Equal[string](t, e.Server.Name, slSrv.Name())\n\t\t\t\t\t\t\t\trequire_Equal[int](t, len(e.Egresses()), 0)\n\t\t\t\t\t\t\t\tcheckJS()\n\t\t\t\t\t\t\t\trouteOK = true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tcheckJSTrace()\n\t\t\t\t\tif mainIter > 0 {\n\t\t\t\t\t\t// There will be at least 2 events\n\t\t\t\t\t\tcheckJSTrace()\n\t\t\t\t\t\t// For the last test, there may be a 3rd.\n\t\t\t\t\t\tif mainIter == 2 && !(clientOK && gatewayOK && routeOK) {\n\t\t\t\t\t\t\tcheckJSTrace()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n\n\t// Now cause a step-down, and verify count is as expected.\n\tfor _, stream := range []string{\"TEST1\", \"TEST2\", \"TEST3\"} {\n\t\t_, err := nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, stream), nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tsc.waitOnStreamLeader(globalAccountName, stream)\n\t\tcheckStream(t, stream, 7)\n\t}\n\n\ts := c1.randomNonStreamLeader(globalAccountName, \"TEST1\")\n\t// Try to get a message that will come from a route and make sure that\n\t// this does not trigger a trace message.\n\tnct := natsConnect(t, s.ClientURL(), nats.Name(\"Tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, traceDest)\n\tnatsFlush(t, nct)\n\n\tjct, err := nct.JetStream()\n\trequire_NoError(t, err)\n\n\tsub, err := jct.SubscribeSync(\"TEST1\")\n\trequire_NoError(t, err)\n\tfor i := range 7 {\n\t\tjmsg, err := sub.NextMsg(time.Second)\n\t\trequire_NoError(t, err)\n\t\tif i < 5 {\n\t\t\trequire_Len(t, len(jmsg.Header), 0)\n\t\t\trequire_True(t, bytes.Equal(jmsg.Data, payload))\n\t\t\tcontinue\n\t\t}\n\t\tif i == 5 {\n\t\t\trequire_True(t, jmsg.Header != nil)\n\t\t\trequire_Equal(t, jmsg.Header.Get(JSMsgId), \"MyId\")\n\t\t\trequire_Equal(t, jmsg.Header.Get(traceParentHdr), \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\trequire_Equal(t, jmsg.Header.Get(MsgTraceDest), MsgTraceDestDisabled)\n\t\t\trequire_True(t, bytes.Equal(jmsg.Data, payload))\n\t\t\tcontinue\n\t\t}\n\t\trequire_Len(t, len(jmsg.Header), 0)\n\t\trequire_Equal(t, string(jmsg.Data), \"hello\")\n\t}\n\n\tmsg, err := traceSub.NextMsg(250 * time.Millisecond)\n\tif err != nats.ErrTimeout {\n\t\tif msg != nil {\n\t\t\tt.Fatalf(\"Expected timeout, got msg headers=%+v data=%s\", msg.Header, msg.Data)\n\t\t}\n\t\tt.Fatalf(\"Expected timeout, got err=%v\", err)\n\t}\n}\n\nfunc TestMsgTraceWithCompression(t *testing.T) {\n\to := DefaultOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\tfor _, test := range []struct {\n\t\tcompressAlgo string\n\t\texpectedHdr string\n\t\tunsupported bool\n\t}{\n\t\t{\"gzip\", \"gzip\", false},\n\t\t{\"snappy\", \"snappy\", false},\n\t\t{\"s2\", \"snappy\", false},\n\t\t{\"bad one\", \"identity\", true},\n\t} {\n\t\tt.Run(test.compressAlgo, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tmsg.Header.Set(acceptEncodingHeader, test.compressAlgo)\n\t\t\tmsg.Data = []byte(\"hello!\")\n\t\t\terr := nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\t\tdata := traceMsg.Data\n\t\t\teh := traceMsg.Header.Get(contentEncodingHeader)\n\t\t\trequire_Equal[string](t, eh, test.expectedHdr)\n\t\t\tif test.unsupported {\n\t\t\t\t// We should be able to unmarshal directly\n\t\t\t} else {\n\t\t\t\tswitch test.expectedHdr {\n\t\t\t\tcase \"gzip\":\n\t\t\t\t\tzr, err := gzip.NewReader(bytes.NewReader(data))\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdata, err = io.ReadAll(zr)\n\t\t\t\t\tif err != nil && err != io.ErrUnexpectedEOF {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t\terr = zr.Close()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\tcase \"snappy\":\n\t\t\t\t\tsr := s2.NewReader(bytes.NewReader(data))\n\t\t\t\t\tdata, err = io.ReadAll(sr)\n\t\t\t\t\tif err != nil && err != io.ErrUnexpectedEOF {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tvar e MsgTraceEvent\n\t\t\terr = json.Unmarshal(data, &e)\n\t\t\trequire_NoError(t, err)\n\t\t\tingress := e.Ingress()\n\t\t\trequire_True(t, ingress != nil)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\trequire_Equal[string](t, ingress.Subject, \"foo\")\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceHops(t *testing.T) {\n\t// Will have a test with following toplogy\n\t//\n\t// =================== ===================\n\t// = C1 cluster = = C2 cluster =\n\t// =================== <--- Gateway ---> ===================\n\t// = C1-S1 <-> C1-S2 = = C2-S1 =\n\t// =================== ===================\n\t// ^ ^ ^\n\t// | Leafnode | | Leafnode\n\t// | | |\n\t// =================== ===================\n\t// = C3 cluster = = C4 cluster =\n\t// =================== ===================\n\t// = C3-S1 <-> C3-S2 = = C4-S1 =\n\t// =================== ===================\n\t// ^\n\t// | Leafnode\n\t// |-------|\n\t// ===================\n\t// = C5 cluster =\n\t// ===================\n\t// = C5-S1 <-> C5-S2 =\n\t// ===================\n\t//\n\t// And a subscription on \"foo\" attached to all servers, and the subscription\n\t// on the trace subject attached to c1-s1 (and where the trace message will\n\t// be sent from).\n\t//\n\tcommonTmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s-%s\"\n\t\taccounts {\n\t\t\tA { users: [{user:\"a\", pass: \"pwd\"}] }\n\t\t\t$SYS { users: [{user:\"admin\", pass: \"s3cr3t!\"}] }\n\t\t}\n\t\tsystem_account: \"$SYS\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"%s\"\n\t\t\t%s\n\t\t}\n\t`\n\tgenCommon := func(cname, sname string, routePort int) string {\n\t\tvar routes string\n\t\tif routePort > 0 {\n\t\t\troutes = fmt.Sprintf(`routes: [\"nats://127.0.0.1:%d\"]`, routePort)\n\t\t}\n\t\treturn fmt.Sprintf(commonTmpl, cname, sname, cname, routes)\n\t}\n\tc1s1conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tgateway {\n\t\t\tport: -1\n\t\t\tname: \"C1\"\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`, genCommon(\"C1\", \"S1\", 0))))\n\tc1s1, oc1s1 := RunServerWithConfig(c1s1conf)\n\tdefer c1s1.Shutdown()\n\n\tc1s2conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tgateway {\n\t\t\tport: -1\n\t\t\tname: \"C1\"\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`, genCommon(\"C1\", \"S2\", oc1s1.Cluster.Port))))\n\tc1s2, oc1s2 := RunServerWithConfig(c1s2conf)\n\tdefer c1s2.Shutdown()\n\n\tcheckClusterFormed(t, c1s1, c1s2)\n\n\tc2s1conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tgateway {\n\t\t\tport: -1\n\t\t\tname: \"C2\"\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"C1\"\n\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`, genCommon(\"C2\", \"S1\", 0), oc1s1.Gateway.Port)))\n\tc2s1, oc2s1 := RunServerWithConfig(c2s1conf)\n\tdefer c2s1.Shutdown()\n\n\tc4s1conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tleafnodes {\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\"}]\n\t\t}\n\t`, genCommon(\"C4\", \"S1\", 0), oc2s1.LeafNode.Port)))\n\tc4s1, _ := RunServerWithConfig(c4s1conf)\n\tdefer c4s1.Shutdown()\n\n\tfor _, s := range []*Server{c1s1, c1s2, c2s1} {\n\t\twaitForOutboundGateways(t, s, 1, time.Second)\n\t}\n\twaitForInboundGateways(t, c2s1, 2, time.Second)\n\n\tc3s1conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\"}]\n\t\t}\n\t`, genCommon(\"C3\", \"S1\", 0), oc1s1.LeafNode.Port)))\n\tc3s1, oc3s1 := RunServerWithConfig(c3s1conf)\n\tdefer c3s1.Shutdown()\n\n\tc3s2conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\"}]\n\t\t}\n\t\tsystem_account: \"$SYS\"\n\t`, genCommon(\"C3\", \"S2\", oc3s1.Cluster.Port), oc1s2.LeafNode.Port)))\n\tc3s2, oc3s2 := RunServerWithConfig(c3s2conf)\n\tdefer c3s2.Shutdown()\n\n\tcheckClusterFormed(t, c3s1, c3s2)\n\tcheckLeafNodeConnected(t, c1s1)\n\tcheckLeafNodeConnected(t, c1s2)\n\tcheckLeafNodeConnected(t, c3s1)\n\tcheckLeafNodeConnected(t, c3s2)\n\n\tc5s1conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tleafnodes {\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\"}]\n\t\t}\n\t`, genCommon(\"C5\", \"S1\", 0), oc3s2.LeafNode.Port)))\n\tc5s1, oc5s1 := RunServerWithConfig(c5s1conf)\n\tdefer c5s1.Shutdown()\n\n\tc5s2conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t%s\n\t\tleafnodes {\n\t\t\tremotes [{url: \"nats://a:pwd@127.0.0.1:%d\", account: \"A\"}]\n\t\t}\n\t`, genCommon(\"C5\", \"S2\", oc5s1.Cluster.Port), oc3s2.LeafNode.Port)))\n\tc5s2, _ := RunServerWithConfig(c5s2conf)\n\tdefer c5s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, c5s1)\n\tcheckLeafNodeConnected(t, c5s2)\n\tcheckLeafNodeConnectedCount(t, c3s2, 3)\n\n\tnct := natsConnect(t, c1s1.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(\"Tracer\"))\n\tdefer nct.Close()\n\ttraceSub := natsSubSync(t, nct, \"my.trace.subj\")\n\tnatsFlush(t, nct)\n\n\tallServers := []*Server{c1s1, c1s2, c2s1, c3s1, c3s2, c4s1, c5s1, c5s2}\n\t// Check that the subscription interest on the trace subject reaches all servers.\n\tfor _, s := range allServers {\n\t\tif s == c2s1 {\n\t\t\t// Gateway needs to be checked differently.\n\t\t\tcheckGWInterestOnlyModeInterestOn(t, c2s1, \"C1\", \"A\", traceSub.Subject)\n\t\t\tcontinue\n\t\t}\n\t\tcheckSubInterest(t, s, \"A\", traceSub.Subject, time.Second)\n\t}\n\n\tvar subs []*nats.Subscription\n\t// Now create a subscription on \"foo\" on all servers (do in reverse order).\n\tfor i := len(allServers) - 1; i >= 0; i-- {\n\t\ts := allServers[i]\n\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(\"a\", \"pwd\"), nats.Name(fmt.Sprintf(\"sub%d\", i+1)))\n\t\tdefer nc.Close()\n\t\tsubs = append(subs, natsSubSync(t, nc, \"foo\"))\n\t\tnatsFlush(t, nc)\n\t}\n\n\t// Check sub interest on \"foo\" on all servers.\n\tfor _, s := range allServers {\n\t\tcheckSubInterest(t, s, \"A\", \"foo\", time.Second)\n\t}\n\n\t// Now send a trace message from c1s1\n\tmsg := nats.NewMsg(\"foo\")\n\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\tmsg.Data = []byte(\"hello!\")\n\terr := nct.PublishMsg(msg)\n\trequire_NoError(t, err)\n\n\t// Check that all subscriptions received the message\n\tfor i, sub := range subs {\n\t\tappMsg, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error getting app message for server %q\", allServers[i])\n\t\t}\n\t\trequire_Equal[string](t, string(appMsg.Data), \"hello!\")\n\t\t// Check that no (more) messages are received.\n\t\tif msg, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Did not expect application message, got %s\", msg.Data)\n\t\t}\n\t}\n\n\tevents := make(map[string]*MsgTraceEvent, 8)\n\t// We expect 8 events\n\tfor i := 0; i < 8; i++ {\n\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\tvar e MsgTraceEvent\n\t\tjson.Unmarshal(traceMsg.Data, &e)\n\n\t\tvar hop string\n\t\tif hopVals := e.Request.Header[MsgTraceHop]; len(hopVals) > 0 {\n\t\t\thop = hopVals[0]\n\t\t}\n\t\tevents[hop] = &e\n\t}\n\t// Make sure we are not receiving more traces\n\tif tm, err := traceSub.NextMsg(250 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Should not have received trace message: %s\", tm.Data)\n\t}\n\n\tcheckIngress := func(e *MsgTraceEvent, kind int, name, hop string) *MsgTraceIngress {\n\t\tt.Helper()\n\t\tingress := e.Ingress()\n\t\trequire_True(t, ingress != nil)\n\t\trequire_True(t, ingress.Kind == kind)\n\t\trequire_Equal[string](t, ingress.Account, \"A\")\n\t\trequire_Equal[string](t, ingress.Subject, \"foo\")\n\t\trequire_Equal[string](t, ingress.Name, name)\n\t\tvar hhop string\n\t\tif hopVals := e.Request.Header[MsgTraceHop]; len(hopVals) > 0 {\n\t\t\thhop = hopVals[0]\n\t\t}\n\t\trequire_Equal[string](t, hhop, hop)\n\t\treturn ingress\n\t}\n\n\tcheckEgressClient := func(eg *MsgTraceEgress, name string) {\n\t\tt.Helper()\n\t\trequire_True(t, eg.Kind == CLIENT)\n\t\trequire_Equal[string](t, eg.Name, name)\n\t\trequire_Equal[string](t, eg.Hop, _EMPTY_)\n\t\trequire_Equal[string](t, eg.Subscription, \"foo\")\n\t\trequire_Equal[string](t, eg.Queue, _EMPTY_)\n\t}\n\n\t// First, we should have an event without a \"hop\" header, that is the\n\t// ingress from the client.\n\te, ok := events[_EMPTY_]\n\trequire_True(t, ok)\n\tcheckIngress(e, CLIENT, \"Tracer\", _EMPTY_)\n\trequire_Equal[int](t, e.Hops, 3)\n\tegress := e.Egresses()\n\trequire_Equal[int](t, len(egress), 4)\n\tvar (\n\t\tleafC3S1Hop string\n\t\tleafC3S2Hop string\n\t\tleafC4S1Hop string\n\t\tleafC5S1Hop string\n\t\tleafC5S2Hop string\n\t\trouteC1S2Hop string\n\t\tgwC2S1Hop string\n\t)\n\tfor _, eg := range egress {\n\t\tswitch eg.Kind {\n\t\tcase CLIENT:\n\t\t\tcheckEgressClient(eg, \"sub1\")\n\t\tcase ROUTER:\n\t\t\trequire_Equal[string](t, eg.Name, c1s2.Name())\n\t\t\trouteC1S2Hop = eg.Hop\n\t\tcase LEAF:\n\t\t\trequire_Equal[string](t, eg.Name, c3s1.Name())\n\t\t\tleafC3S1Hop = eg.Hop\n\t\tcase GATEWAY:\n\t\t\trequire_Equal[string](t, eg.Name, c2s1.Name())\n\t\t\tgwC2S1Hop = eg.Hop\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t}\n\t}\n\t// All \"hop\" ids should be not empty and different from each other\n\trequire_True(t, leafC3S1Hop != _EMPTY_ && routeC1S2Hop != _EMPTY_ && gwC2S1Hop != _EMPTY_)\n\trequire_True(t, leafC3S1Hop != routeC1S2Hop && leafC3S1Hop != gwC2S1Hop && routeC1S2Hop != gwC2S1Hop)\n\n\t// Now check the routed server in C1 (c1s2)\n\te, ok = events[routeC1S2Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, ROUTER, c1s1.Name(), routeC1S2Hop)\n\trequire_Equal[int](t, e.Hops, 1)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 2)\n\tfor _, eg := range egress {\n\t\tswitch eg.Kind {\n\t\tcase CLIENT:\n\t\t\tcheckEgressClient(eg, \"sub2\")\n\t\tcase LEAF:\n\t\t\trequire_Equal[string](t, eg.Name, c3s2.Name())\n\t\t\trequire_Equal[string](t, eg.Hop, routeC1S2Hop+\".1\")\n\t\t\tleafC3S2Hop = eg.Hop\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t}\n\t}\n\trequire_True(t, leafC3S2Hop != _EMPTY_)\n\n\t// Let's check the gateway server\n\te, ok = events[gwC2S1Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, GATEWAY, c1s1.Name(), gwC2S1Hop)\n\trequire_Equal[int](t, e.Hops, 1)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 2)\n\tfor _, eg := range egress {\n\t\tswitch eg.Kind {\n\t\tcase CLIENT:\n\t\t\tcheckEgressClient(eg, \"sub3\")\n\t\tcase LEAF:\n\t\t\trequire_Equal[string](t, eg.Name, c4s1.Name())\n\t\t\trequire_Equal[string](t, eg.Hop, gwC2S1Hop+\".1\")\n\t\t\tleafC4S1Hop = eg.Hop\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t}\n\t}\n\trequire_True(t, leafC4S1Hop != _EMPTY_)\n\n\t// Let's check the C3 cluster, starting at C3-S1\n\te, ok = events[leafC3S1Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, LEAF, c1s1.Name(), leafC3S1Hop)\n\trequire_Equal[int](t, e.Hops, 0)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 1)\n\tcheckEgressClient(egress[0], \"sub4\")\n\n\t// Now C3-S2\n\te, ok = events[leafC3S2Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, LEAF, c1s2.Name(), leafC3S2Hop)\n\trequire_Equal[int](t, e.Hops, 2)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 3)\n\tfor _, eg := range egress {\n\t\tswitch eg.Kind {\n\t\tcase CLIENT:\n\t\t\tcheckEgressClient(eg, \"sub5\")\n\t\tcase LEAF:\n\t\t\trequire_True(t, eg.Name == c5s1.Name() || eg.Name == c5s2.Name())\n\t\t\trequire_True(t, eg.Hop == leafC3S2Hop+\".1\" || eg.Hop == leafC3S2Hop+\".2\")\n\t\t\tif eg.Name == c5s1.Name() {\n\t\t\t\tleafC5S1Hop = eg.Hop\n\t\t\t} else {\n\t\t\t\tleafC5S2Hop = eg.Hop\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected egress: %+v\", eg)\n\t\t}\n\t}\n\t// The leafC5SxHop must be different and not empty\n\trequire_True(t, leafC5S1Hop != _EMPTY_ && leafC5S1Hop != leafC5S2Hop && leafC5S2Hop != _EMPTY_)\n\n\t// Check the C4 cluster\n\te, ok = events[leafC4S1Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, LEAF, c2s1.Name(), leafC4S1Hop)\n\trequire_Equal[int](t, e.Hops, 0)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 1)\n\tcheckEgressClient(egress[0], \"sub6\")\n\n\t// Finally, the C5 cluster, starting with C5-S1\n\te, ok = events[leafC5S1Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, LEAF, c3s2.Name(), leafC5S1Hop)\n\trequire_Equal[int](t, e.Hops, 0)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 1)\n\tcheckEgressClient(egress[0], \"sub7\")\n\n\t// Then C5-S2\n\te, ok = events[leafC5S2Hop]\n\trequire_True(t, ok)\n\tcheckIngress(e, LEAF, c3s2.Name(), leafC5S2Hop)\n\trequire_Equal[int](t, e.Hops, 0)\n\tegress = e.Egresses()\n\trequire_Equal[int](t, len(egress), 1)\n\tcheckEgressClient(egress[0], \"sub8\")\n}\n\nfunc TestMsgTraceTriggeredByExternalHeader(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user:A, password: pwd}]\n\t\t\t\ttrace_dest: \"acc.trace.dest\"\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user:B, password: pwd}]\n\t\t\t\t%s\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", _EMPTY_, fmt.Sprintf(`routes: [\"nats://127.0.0.1:%d\"]`, o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\tdefer nc2.Close()\n\tappSub := natsSubSync(t, nc2, \"foo\")\n\tnatsFlush(t, nc2)\n\n\tcheckSubInterest(t, s1, \"A\", \"foo\", time.Second)\n\n\tnc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"A\", \"pwd\"))\n\tdefer nc1.Close()\n\n\ttraceSub := natsSubSync(t, nc1, \"trace.dest\")\n\taccTraceSub := natsSubSync(t, nc1, \"acc.trace.dest\")\n\tnatsFlush(t, nc1)\n\n\tcheckSubInterest(t, s1, \"A\", traceSub.Subject, time.Second)\n\tcheckSubInterest(t, s1, \"A\", accTraceSub.Subject, time.Second)\n\n\tvar msgCount int\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsetHeaders func(h nats.Header)\n\t\ttraceTriggered bool\n\t\ttraceOnly bool\n\t\texpectedAccSub bool\n\t}{\n\t\t// Tests with external header only (no Nats-Trace-Dest). In this case, the\n\t\t// trace is triggered based on sampling (last token is `-01`). The presence\n\t\t// of Nats-Trace-Only has no effect and message should always be delivered\n\t\t// to the application.\n\t\t{\"only external header sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\ttrue},\n\t\t{\"only external header with different case and sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(\"TraceParent\", \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\ttrue},\n\t\t{\"only external header sampling but not simply 01\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-25\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\ttrue},\n\t\t{\"only external header no sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\")\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse},\n\t\t{\"external header sampling and trace only\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t\th.Set(MsgTraceOnly, \"true\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\ttrue},\n\t\t{\"external header no sampling and trace only\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\")\n\t\t\t\th.Set(MsgTraceOnly, \"true\")\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse},\n\t\t// Tests where Nats-Trace-Dest is present, so ignore external header and\n\t\t// always deliver to the Nats-Trace-Dest, not the account.\n\t\t{\"trace dest and external header sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\tfalse},\n\t\t{\"trace dest and external header no sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\tfalse},\n\t\t{\"trace dest with trace only and external header sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\th.Set(MsgTraceOnly, \"true\")\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\ttrue,\n\t\t\tfalse},\n\t\t{\"trace dest with trace only and external header no sampling\",\n\t\t\tfunc(h nats.Header) {\n\t\t\t\th.Set(MsgTraceDest, traceSub.Subject)\n\t\t\t\th.Set(MsgTraceOnly, \"true\")\n\t\t\t\th.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00\")\n\t\t\t},\n\t\t\ttrue,\n\t\t\ttrue,\n\t\t\tfalse},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\ttest.setHeaders(msg.Header)\n\t\t\tmsgCount++\n\t\t\tmsgPayload := fmt.Sprintf(\"msg%d\", msgCount)\n\t\t\tmsg.Data = []byte(msgPayload)\n\t\t\terr := nc1.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif !test.traceOnly {\n\t\t\t\tappMsg := natsNexMsg(t, appSub, time.Second)\n\t\t\t\trequire_Equal[string](t, string(appMsg.Data), msgPayload)\n\t\t\t}\n\t\t\t// Make sure we don't receive more (or not if trace only)\n\t\t\tif appMsg, err := appSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected no app message, got %q\", appMsg.Data)\n\t\t\t}\n\n\t\t\tcheckTrace := func(sub *nats.Subscription) {\n\t\t\t\t// We should receive 2 traces, 1 per server.\n\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\ttm := natsNexMsg(t, sub, time.Second)\n\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\terr := json.Unmarshal(tm.Data, &e)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif test.traceTriggered {\n\t\t\t\tif test.expectedAccSub {\n\t\t\t\t\tcheckTrace(accTraceSub)\n\t\t\t\t} else {\n\t\t\t\t\tcheckTrace(traceSub)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Make sure no trace is received in the other trace sub\n\t\t\t// or no trace received at all.\n\t\t\tfor _, sub := range []*nats.Subscription{accTraceSub, traceSub} {\n\t\t\t\tif tm, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected no trace for the trace sub on %q, got %q\", sub.Subject, tm.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tnc1.Close()\n\tnc2.Close()\n\n\t// Now replace connections and subs for account \"B\"\n\tnc2 = natsConnect(t, s2.ClientURL(), nats.UserInfo(\"B\", \"pwd\"))\n\tdefer nc2.Close()\n\tappSub = natsSubSync(t, nc2, \"foo\")\n\tnatsFlush(t, nc2)\n\n\tcheckSubInterest(t, s1, \"B\", \"foo\", time.Second)\n\n\tnc1 = natsConnect(t, s1.ClientURL(), nats.UserInfo(\"B\", \"pwd\"))\n\tdefer nc1.Close()\n\n\ttraceSub = natsSubSync(t, nc1, \"trace.dest\")\n\taccTraceSub = natsSubSync(t, nc1, \"acc.trace.dest\")\n\tnatsFlush(t, nc1)\n\n\tcheckSubInterest(t, s1, \"B\", traceSub.Subject, time.Second)\n\tcheckSubInterest(t, s1, \"B\", accTraceSub.Subject, time.Second)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\treload bool\n\t}{\n\t\t{\"external header but no account destination\", true},\n\t\t{\"external header with account destination added through config reload\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\terr := nc1.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Application should receive the message\n\t\t\tappMsg := natsNexMsg(t, appSub, time.Second)\n\t\t\trequire_Equal[string](t, string(appMsg.Data), \"hello\")\n\t\t\t// Only once...\n\t\t\tif appMsg, err := appSub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected no app message, got %q\", appMsg.Data)\n\t\t\t}\n\t\t\tif !test.reload {\n\t\t\t\t// We should receive the traces (1 per server) on the account destination\n\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\ttm := natsNexMsg(t, accTraceSub, time.Second)\n\t\t\t\t\tvar e MsgTraceEvent\n\t\t\t\t\terr := json.Unmarshal(tm.Data, &e)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// No (or no more) trace message should be received.\n\t\t\tfor _, sub := range []*nats.Subscription{accTraceSub, traceSub} {\n\t\t\t\tif tm, err := sub.NextMsg(100 * time.Millisecond); err != nats.ErrTimeout {\n\t\t\t\t\tt.Fatalf(\"Expected no trace for the trace sub on %q, got %q\", sub.Subject, tm.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Do the config reload and we will repeat the test and now\n\t\t\t// we should receive the trace message into the account\n\t\t\t// destination trace.\n\t\t\tif test.reload {\n\t\t\t\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", `trace_dest: \"acc.trace.dest\"`, _EMPTY_))\n\t\t\t\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", `trace_dest: \"acc.trace.dest\"`, fmt.Sprintf(`routes: [\"nats://127.0.0.1:%d\"]`, o1.Cluster.Port)))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceAccountTraceDestJWTUpdate(t *testing.T) {\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysCreds := newUser(t, sysKp)\n\t// create account A\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n `, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\n\tnc := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp))\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"acc.trace.dest\")\n\tnatsFlush(t, nc)\n\n\tfor i, test := range []struct {\n\t\tname string\n\t\ttraceTriggered bool\n\t}{\n\t\t{\"no acc dest\", false},\n\t\t{\"adding trace dest\", true},\n\t\t{\"removing trace dest\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\tmsg.Data = []byte(\"hello\")\n\t\t\terr = nc.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tif test.traceTriggered {\n\t\t\t\ttm := natsNexMsg(t, sub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\terr = json.Unmarshal(tm.Data, &e)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// Simple check\n\t\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\t}\n\t\t\t// No (more) trace message expected.\n\t\t\ttm, err := sub.NextMsg(250 * time.Millisecond)\n\t\t\tif err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected no trace message, got %s\", tm.Data)\n\t\t\t}\n\t\t\tif i < 2 {\n\t\t\t\tif i == 0 {\n\t\t\t\t\tclaim.Trace = &jwt.MsgTrace{Destination: \"acc.trace.dest\"}\n\t\t\t\t} else {\n\t\t\t\t\tclaim.Trace = nil\n\t\t\t\t}\n\t\t\t\taJwt, err = claim.Encode(oKp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceServiceJWTUpdate(t *testing.T) {\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysCreds := newUser(t, sysKp)\n\t// create account A\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\taClaim := jwt.NewAccountClaims(aPub)\n\tserviceExport := &jwt.Export{Subject: \"req\", Type: jwt.Service}\n\taClaim.Exports.Add(serviceExport)\n\taJwt, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\t// create account B\n\tbkp, _ := nkeys.CreateAccount()\n\tbPub, _ := bkp.PublicKey()\n\tbClaim := jwt.NewAccountClaims(bPub)\n\tserviceImport := &jwt.Import{Account: aPub, Subject: \"req\", Type: jwt.Service}\n\tbClaim.Imports.Add(serviceImport)\n\tbJwt, err := bClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n\t`, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, bJwt, 1)\n\n\tncA := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp), nats.Name(\"Service\"))\n\tdefer ncA.Close()\n\n\tnatsSub(t, ncA, \"req\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"resp\"))\n\t})\n\tnatsFlush(t, ncA)\n\n\tncB := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, bkp))\n\tdefer ncB.Close()\n\n\tsub := natsSubSync(t, ncB, \"trace.dest\")\n\tnatsFlush(t, ncB)\n\n\tfor i, test := range []struct {\n\t\tname string\n\t\tallowTrace bool\n\t}{\n\t\t{\"trace not allowed\", false},\n\t\t{\"trace allowed\", true},\n\t\t{\"trace not allowed again\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"req\")\n\t\t\tmsg.Header.Set(MsgTraceDest, sub.Subject)\n\t\t\tmsg.Data = []byte(\"request\")\n\t\t\treply, err := ncB.RequestMsg(msg, time.Second)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal[string](t, string(reply.Data), \"resp\")\n\n\t\t\ttm := natsNexMsg(t, sub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\terr = json.Unmarshal(tm.Data, &e)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\trequire_Equal[string](t, e.Ingress().Account, bPub)\n\t\t\tsis := e.ServiceImports()\n\t\t\trequire_Equal[int](t, len(sis), 1)\n\t\t\tsi := sis[0]\n\t\t\trequire_Equal[string](t, si.Account, aPub)\n\t\t\tegresses := e.Egresses()\n\t\t\tif !test.allowTrace {\n\t\t\t\trequire_Equal[int](t, len(egresses), 0)\n\t\t\t} else {\n\t\t\t\trequire_Equal[int](t, len(egresses), 1)\n\t\t\t\teg := egresses[0]\n\t\t\t\trequire_Equal[string](t, eg.Name, \"Service\")\n\t\t\t\trequire_Equal[string](t, eg.Account, aPub)\n\t\t\t\trequire_Equal[string](t, eg.Subscription, \"req\")\n\t\t\t}\n\t\t\t// No (more) trace message expected.\n\t\t\ttm, err = sub.NextMsg(250 * time.Millisecond)\n\t\t\tif err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected no trace message, got %s\", tm.Data)\n\t\t\t}\n\t\t\tif i < 2 {\n\t\t\t\t// Set AllowTrace to true at the first iteration, then\n\t\t\t\t// false at the second.\n\t\t\t\taClaim.Exports[0].AllowTrace = (i == 0)\n\t\t\t\taJwt, err = aClaim.Encode(oKp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceStreamJWTUpdate(t *testing.T) {\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysCreds := newUser(t, sysKp)\n\t// create account A\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\taClaim := jwt.NewAccountClaims(aPub)\n\tstreamExport := &jwt.Export{Subject: \"info\", Type: jwt.Stream}\n\taClaim.Exports.Add(streamExport)\n\taJwt, err := aClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\t// create account B\n\tbkp, _ := nkeys.CreateAccount()\n\tbPub, _ := bkp.PublicKey()\n\tbClaim := jwt.NewAccountClaims(bPub)\n\tstreamImport := &jwt.Import{Account: aPub, Subject: \"info\", To: \"b\", Type: jwt.Stream}\n\tbClaim.Imports.Add(streamImport)\n\tbJwt, err := bClaim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: -1\n\t\toperator: %s\n\t\tresolver: {\n\t\t\ttype: full\n\t\t\tdir: '%s'\n\t\t}\n\t\tsystem_account: %s\n\t`, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\tupdateJwt(t, s.ClientURL(), sysCreds, bJwt, 1)\n\n\tncA := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp))\n\tdefer ncA.Close()\n\n\ttraceSub := natsSubSync(t, ncA, \"trace.dest\")\n\tnatsFlush(t, ncA)\n\n\tncB := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, bkp), nats.Name(\"BInfo\"))\n\tdefer ncB.Close()\n\n\tappSub := natsSubSync(t, ncB, \"b.info\")\n\tnatsFlush(t, ncB)\n\n\tfor i, test := range []struct {\n\t\tname string\n\t\tallowTrace bool\n\t}{\n\t\t{\"trace not allowed\", false},\n\t\t{\"trace allowed\", true},\n\t\t{\"trace not allowed again\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tmsg := nats.NewMsg(\"info\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tmsg.Data = []byte(\"some info\")\n\t\t\terr = ncA.PublishMsg(msg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\tappMsg := natsNexMsg(t, appSub, time.Second)\n\t\t\trequire_Equal[string](t, string(appMsg.Data), \"some info\")\n\n\t\t\ttm := natsNexMsg(t, traceSub, time.Second)\n\t\t\tvar e MsgTraceEvent\n\t\t\terr = json.Unmarshal(tm.Data, &e)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal[string](t, e.Server.Name, s.Name())\n\t\t\tses := e.StreamExports()\n\t\t\trequire_Equal[int](t, len(ses), 1)\n\t\t\tse := ses[0]\n\t\t\trequire_Equal[string](t, se.Account, bPub)\n\t\t\trequire_Equal[string](t, se.To, \"b.info\")\n\t\t\tegresses := e.Egresses()\n\t\t\tif !test.allowTrace {\n\t\t\t\trequire_Equal[int](t, len(egresses), 0)\n\t\t\t} else {\n\t\t\t\trequire_Equal[int](t, len(egresses), 1)\n\t\t\t\teg := egresses[0]\n\t\t\t\trequire_Equal[string](t, eg.Name, \"BInfo\")\n\t\t\t\trequire_Equal[string](t, eg.Account, bPub)\n\t\t\t\trequire_Equal[string](t, eg.Subscription, \"info\")\n\t\t\t}\n\t\t\t// No (more) trace message expected.\n\t\t\ttm, err = traceSub.NextMsg(250 * time.Millisecond)\n\t\t\tif err != nats.ErrTimeout {\n\t\t\t\tt.Fatalf(\"Expected no trace message, got %s\", tm.Data)\n\t\t\t}\n\t\t\tif i < 2 {\n\t\t\t\t// Set AllowTrace to true at the first iteration, then\n\t\t\t\t// false at the second.\n\t\t\t\tbClaim.Imports[0].AllowTrace = (i == 0)\n\t\t\t\tbJwt, err = bClaim.Encode(oKp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tupdateJwt(t, s.ClientURL(), sysCreds, bJwt, 1)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceParseAccountDestWithSampling(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password: pwd}]\n\t\t\t\t%s\n\t\t\t}\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsamplingStr string\n\t\twant int\n\t}{\n\t\t{\"trace sampling no dest\", `msg_trace: {sampling: 50}`, 0},\n\t\t{\"trace dest only\", `msg_trace: {dest: foo}`, 100},\n\t\t{\"trace dest with number only\", `msg_trace: {dest: foo, sampling: 20}`, 20},\n\t\t{\"trace dest with percentage\", `msg_trace: {dest: foo, sampling: 50%}`, 50},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.samplingStr)))\n\t\t\to := LoadConfig(conf)\n\t\t\t_, sampling := o.Accounts[0].getTraceDestAndSampling()\n\t\t\trequire_Equal[int](t, test.want, sampling)\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceAccountDestWithSampling(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: %s\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: a, password:pwd}]\n\t\t\t\tmsg_trace: {dest: \"acc.dest\"%s}\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troutes := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", _EMPTY_, routes)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc2.Close()\n\tnatsSub(t, nc2, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, nc2)\n\n\tnc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc1.Close()\n\tsub := natsSubSync(t, nc1, \"acc.dest\")\n\tnatsFlush(t, nc1)\n\n\tcheckSubInterest(t, s1, \"A\", \"foo\", time.Second)\n\tcheckSubInterest(t, s2, \"A\", \"acc.dest\", time.Second)\n\n\tfor iter, test := range []struct {\n\t\tname string\n\t\tsamplingStr string\n\t\tsampling int\n\t}{\n\t\t// Sampling is considered 100% if not specified or <=0 or >= 100.\n\t\t// To disable sampling, the account destination should not be specified.\n\t\t{\"no sampling specified\", _EMPTY_, 100},\n\t\t{\"sampling specified\", \", sampling: \\\"25%\\\"\", 25},\n\t\t{\"no sampling again\", _EMPTY_, 100},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif iter > 0 {\n\t\t\t\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", test.samplingStr, _EMPTY_))\n\t\t\t\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", test.samplingStr, routes))\n\t\t\t}\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\ttotal := 400\n\t\t\tfor i := 0; i < total; i++ {\n\t\t\t\terr := nc1.PublishMsg(msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t// Wait a bit to make sure that we received all traces that should\n\t\t\t// have been received.\n\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\tn, _, err := sub.Pending()\n\t\t\trequire_NoError(t, err)\n\t\t\tfromClient := 0\n\t\t\tfromRoute := 0\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tmsg = natsNexMsg(t, sub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\terr = json.Unmarshal(msg.Data, &e)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tingress := e.Ingress()\n\t\t\t\trequire_True(t, ingress != nil)\n\t\t\t\tswitch ingress.Kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\tfromClient++\n\t\t\t\tcase ROUTER:\n\t\t\t\t\tfromRoute++\n\t\t\t\tdefault:\n\t\t\t\t\tt.Fatalf(\"Unexpected ingress: %+v\", ingress)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// There should be as many messages coming from the origin server\n\t\t\t// and the routed server. This checks that if sampling is not 100%\n\t\t\t// then when a message is routed, the header is properly deactivated.\n\t\t\trequire_Equal[int](t, fromClient, fromRoute)\n\t\t\t// Now check that if sampling was 100%, we have the total number\n\t\t\t// of published messages.\n\t\t\tif test.sampling == 100 {\n\t\t\t\trequire_Equal[int](t, fromClient, total)\n\t\t\t} else {\n\t\t\t\t// Otherwise, we should have no more (but let's be conservative)\n\t\t\t\t// than the sampling number.\n\t\t\t\trequire_LessThan[int](t, fromClient, int(float64(test.sampling*total/100)*1.35))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMsgTraceAccDestWithSamplingJWTUpdate(t *testing.T) {\n\t// create system account\n\tsysKp, _ := nkeys.CreateAccount()\n\tsysPub, _ := sysKp.PublicKey()\n\tsysCreds := newUser(t, sysKp)\n\t// create account A\n\takp, _ := nkeys.CreateAccount()\n\taPub, _ := akp.PublicKey()\n\tclaim := jwt.NewAccountClaims(aPub)\n\tclaim.Trace = &jwt.MsgTrace{Destination: \"acc.trace.dest\"}\n\taJwt, err := claim.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tdir := t.TempDir()\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tlisten: -1\n\t\t\toperator: %s\n\t\t\tresolver: {\n\t\t\t\ttype: full\n\t\t\t\tdir: '%s'\n\t\t\t}\n\t\t\tsystem_account: %s\n\t\t`, ojwt, dir, sysPub)))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\n\tnc := natsConnect(t, s.ClientURL(), createUserCreds(t, nil, akp))\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"acc.trace.dest\")\n\tnatsFlush(t, nc)\n\n\tfor iter, test := range []struct {\n\t\tname string\n\t\tsampling int\n\t}{\n\t\t{\"no sampling specified\", 100},\n\t\t{\"sampling\", 25},\n\t\t{\"set back sampling to 0\", 100},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif iter > 0 {\n\t\t\t\tclaim.Trace = &jwt.MsgTrace{Destination: \"acc.trace.dest\", Sampling: test.sampling}\n\t\t\t\taJwt, err = claim.Encode(oKp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tupdateJwt(t, s.ClientURL(), sysCreds, aJwt, 1)\n\t\t\t}\n\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(traceParentHdr, \"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\")\n\t\t\tmsg.Data = []byte(\"hello\")\n\n\t\t\ttotal := 400\n\t\t\tfor i := 0; i < total; i++ {\n\t\t\t\terr := nc.PublishMsg(msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t// Wait a bit to make sure that we received all traces that should\n\t\t\t// have been received.\n\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\tn, _, err := sub.Pending()\n\t\t\trequire_NoError(t, err)\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tmsg = natsNexMsg(t, sub, time.Second)\n\t\t\t\tvar e MsgTraceEvent\n\t\t\t\terr = json.Unmarshal(msg.Data, &e)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\t// Now check that if sampling was 100%, we have the total number\n\t\t\t// of published messages.\n\t\t\tif test.sampling == 100 {\n\t\t\t\trequire_Equal[int](t, n, total)\n\t\t\t} else {\n\t\t\t\t// Otherwise, we should have no more (but let's be conservative)\n\t\t\t\t// than the sampling number.\n\t\t\t\trequire_LessThan[int](t, n, int(float64(test.sampling*total/100)*1.35))\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/nkey.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\tcrand \"crypto/rand\"\n\t\"encoding/base64\"\n)\n\n// Raw length of the nonce challenge\nconst (\n\tnonceRawLen = 11\n\tnonceLen = 15 // base64.RawURLEncoding.EncodedLen(nonceRawLen)\n)\n\n// NonceRequired tells us if we should send a nonce.\nfunc (s *Server) NonceRequired() bool {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\treturn s.nonceRequired()\n}\n\n// nonceRequired tells us if we should send a nonce.\n// Lock should be held on entry.\nfunc (s *Server) nonceRequired() bool {\n\treturn s.getOpts().AlwaysEnableNonce || len(s.nkeys) > 0 || s.trustedKeys != nil || len(s.proxiesKeyPairs) > 0\n}\n\n// Generate a nonce for INFO challenge.\n// Assumes server lock is held\nfunc (s *Server) generateNonce(n []byte) {\n\tvar raw [nonceRawLen]byte\n\tdata := raw[:]\n\tcrand.Read(data)\n\tbase64.RawURLEncoding.Encode(n, data)\n}\n" }, { "path": "server/nkey_test.go", "content": "// Copyright 2018-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tcrand \"crypto/rand\"\n\tmrand \"math/rand\"\n\n\t\"github.com/nats-io/nkeys\"\n)\n\n// Nonce has to be a string since we used different encoding by default than json.Unmarshal.\ntype nonceInfo struct {\n\tId string `json:\"server_id\"`\n\tCID uint64 `json:\"client_id,omitempty\"`\n\tNonce string `json:\"nonce,omitempty\"`\n}\n\n// This is a seed for a user. We can extract public and private keys from this for testing.\nvar seed = []byte(\"SUAKYRHVIOREXV7EUZTBHUHL7NUMHPMAS7QMDU3GTIUWEI5LDNOXD43IZY\")\n\nfunc nkeyBasicSetup() (*Server, *testAsyncClient, *bufio.Reader, string) {\n\tkp, _ := nkeys.FromSeed(seed)\n\tpub, _ := kp.PublicKey()\n\topts := defaultServerOptions\n\topts.Nkeys = []*NkeyUser{{Nkey: string(pub)}}\n\treturn rawSetup(opts)\n}\n\nfunc mixedSetup() (*Server, *testAsyncClient, *bufio.Reader, string) {\n\tkp, _ := nkeys.FromSeed(seed)\n\tpub, _ := kp.PublicKey()\n\topts := defaultServerOptions\n\topts.Nkeys = []*NkeyUser{{Nkey: string(pub)}}\n\topts.Users = []*User{{Username: \"derek\", Password: \"foo\"}}\n\treturn rawSetup(opts)\n}\n\nfunc TestServerInfoNonceAlwaysEnabled(t *testing.T) {\n\topts := defaultServerOptions\n\topts.AlwaysEnableNonce = true\n\ts, c, _, l := rawSetup(opts)\n\tdefer s.WaitForShutdown()\n\tdefer s.Shutdown()\n\tdefer c.close()\n\n\tif !strings.HasPrefix(l, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", l)\n\t}\n\n\tvar info nonceInfo\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with AlwaysEnableNonce set\")\n\t}\n}\n\nfunc TestServerInfoNonce(t *testing.T) {\n\tc, l := setUpClientWithResponse()\n\tdefer c.close()\n\tif !strings.HasPrefix(l, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", l)\n\t}\n\t// Make sure payload is proper json\n\tvar info nonceInfo\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce != \"\" {\n\t\tt.Fatalf(\"Expected an empty nonce with no nkeys defined\")\n\t}\n\n\t// Now setup server with auth and nkeys to trigger nonce generation\n\ts, c, _, l := nkeyBasicSetup()\n\tdefer c.close()\n\n\tif !strings.HasPrefix(l, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", l)\n\t}\n\t// Make sure payload is proper json\n\terr = json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\n\t// Make sure new clients get new nonces\n\toldNonce := info.Nonce\n\n\tc, _, l = newClientForServer(s)\n\tdefer c.close()\n\n\terr = json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce\")\n\t}\n\tif strings.Compare(oldNonce, info.Nonce) == 0 {\n\t\tt.Fatalf(\"Expected subsequent nonces to be different\\n\")\n\t}\n}\n\nfunc TestNkeyClientConnect(t *testing.T) {\n\ts, c, cr, _ := nkeyBasicSetup()\n\tdefer c.close()\n\t// Send CONNECT with no signature or nkey, should fail.\n\tconnectOp := \"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\n\"\n\tc.parseAsync(connectOp)\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\tkp, _ := nkeys.FromSeed(seed)\n\tpubKey, _ := kp.PublicKey()\n\n\t// Send nkey but no signature\n\tc, cr, _ = newClientForServer(s)\n\tdefer c.close()\n\tcs := fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q, \\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\n\", pubKey)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Now improperly sign etc.\n\tc, cr, _ = newClientForServer(s)\n\tdefer c.close()\n\tcs = fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":%q,\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\n\", pubKey, \"bad_sig\")\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"-ERR \") {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Now properly sign the nonce\n\tc, cr, l = newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\tvar info nonceInfo\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err := kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs = fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n}\n\nfunc TestMixedClientConnect(t *testing.T) {\n\ts, c, cr, _ := mixedSetup()\n\tdefer c.close()\n\t// Normal user/pass\n\tc.parseAsync(\"CONNECT {\\\"user\\\":\\\"derek\\\",\\\"pass\\\":\\\"foo\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\")\n\tl, _ := cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n\n\tkp, _ := nkeys.FromSeed(seed)\n\tpubKey, _ := kp.PublicKey()\n\n\tc, cr, l = newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\tvar info nonceInfo\n\terr := json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err := kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n}\n\nfunc TestMixedClientConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n authorization {\n users = [\n {nkey: \"UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV\"}\n {user: alice, password: foo}\n ]\n }`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error processing config file: %v\", err)\n\t}\n\tif len(opts.Nkeys) != 1 {\n\t\tt.Fatalf(\"Expected 1 nkey, got %d\", len(opts.Nkeys))\n\t}\n\tif len(opts.Users) != 1 {\n\t\tt.Fatalf(\"Expected 1 user, got %d\", len(opts.Users))\n\t}\n}\n\nfunc BenchmarkCryptoRandGeneration(b *testing.B) {\n\tdata := make([]byte, 16)\n\tfor i := 0; i < b.N; i++ {\n\t\tcrand.Read(data)\n\t}\n}\n\nfunc BenchmarkMathRandGeneration(b *testing.B) {\n\tdata := make([]byte, 16)\n\tprng := mrand.New(mrand.NewSource(time.Now().UnixNano()))\n\tfor i := 0; i < b.N; i++ {\n\t\tprng.Read(data)\n\t}\n}\n\nfunc BenchmarkNonceGeneration(b *testing.B) {\n\tdata := make([]byte, nonceRawLen)\n\tb64 := make([]byte, nonceLen)\n\tprand := mrand.New(mrand.NewSource(time.Now().UnixNano()))\n\tfor i := 0; i < b.N; i++ {\n\t\tprand.Read(data)\n\t\tbase64.RawURLEncoding.Encode(b64, data)\n\t}\n}\n\nfunc BenchmarkPublicVerify(b *testing.B) {\n\tdata := make([]byte, nonceRawLen)\n\tnonce := make([]byte, nonceLen)\n\tcrand.Read(data)\n\tbase64.RawURLEncoding.Encode(nonce, data)\n\n\tuser, err := nkeys.CreateUser()\n\tif err != nil {\n\t\tb.Fatalf(\"Error creating User Nkey: %v\", err)\n\t}\n\tsig, err := user.Sign(nonce)\n\tif err != nil {\n\t\tb.Fatalf(\"Error sigining nonce: %v\", err)\n\t}\n\tpk, err := user.PublicKey()\n\tif err != nil {\n\t\tb.Fatalf(\"Could not extract public key from user: %v\", err)\n\t}\n\tpub, err := nkeys.FromPublicKey(pk)\n\tif err != nil {\n\t\tb.Fatalf(\"Could not create public key pair from public key string: %v\", err)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tif err := pub.Verify(nonce, sig); err != nil {\n\t\t\tb.Fatalf(\"Error verifying nonce: %v\", err)\n\t\t}\n\t}\n}\n" }, { "path": "server/norace_1_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"crypto/hmac\"\n\tcrand \"crypto/rand\"\n\t\"crypto/sha256\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// IMPORTANT: Tests in this file are not executed when running with the -race flag.\n// The test name should be prefixed with TestNoRace so we can run only\n// those tests: go test -run=TestNoRace ...\n\nfunc TestNoRaceAvoidSlowConsumerBigMessages(t *testing.T) {\n\topts := DefaultOptions() // Use defaults to make sure they avoid pending slow consumer.\n\topts.NoSystemAccount = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc1, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tdata := make([]byte, 1024*1024) // 1MB payload\n\tcrand.Read(data)\n\n\texpected := int32(500)\n\treceived := int32(0)\n\n\tdone := make(chan bool)\n\n\t// Create Subscription.\n\tnc1.Subscribe(\"slow.consumer\", func(m *nats.Msg) {\n\t\t// Just eat it so that we are not measuring\n\t\t// code time, just delivery.\n\t\tatomic.AddInt32(&received, 1)\n\t\tif received >= expected {\n\t\t\tdone <- true\n\t\t}\n\t})\n\n\t// Create Error handler\n\tnc1.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {\n\t\tt.Fatalf(\"Received an error on the subscription's connection: %v\\n\", err)\n\t})\n\n\tnc1.Flush()\n\n\tfor i := 0; i < int(expected); i++ {\n\t\tnc2.Publish(\"slow.consumer\", data)\n\t}\n\tnc2.Flush()\n\n\tselect {\n\tcase <-done:\n\t\treturn\n\tcase <-time.After(10 * time.Second):\n\t\tr := atomic.LoadInt32(&received)\n\t\tif s.NumSlowConsumers() > 0 {\n\t\t\tt.Fatalf(\"Did not receive all large messages due to slow consumer status: %d of %d\", r, expected)\n\t\t}\n\t\tt.Fatalf(\"Failed to receive all large messages: %d of %d\\n\", r, expected)\n\t}\n}\n\nfunc TestNoRaceRoutedQueueAutoUnsubscribe(t *testing.T) {\n\toptsA, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\toptsA.NoSigs, optsA.NoLog = true, true\n\toptsA.NoSystemAccount = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvARouteURL := fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, srvA.ClusterAddr().Port)\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(srvARouteURL)\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for these 2 to connect to each other\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Have a client connection to each server\n\tncA, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncA.Close()\n\n\tncB, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsB.Host, optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncB.Close()\n\n\trbar := int32(0)\n\tbarCb := func(m *nats.Msg) {\n\t\tatomic.AddInt32(&rbar, 1)\n\t}\n\trbaz := int32(0)\n\tbazCb := func(m *nats.Msg) {\n\t\tatomic.AddInt32(&rbaz, 1)\n\t}\n\n\t// Create 125 queue subs with auto-unsubscribe to each server for\n\t// group bar and group baz. So 250 total per queue group.\n\tcons := []*nats.Conn{ncA, ncB}\n\tfor _, c := range cons {\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tqsub, err := c.QueueSubscribe(\"foo\", \"bar\", barCb)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t\t}\n\t\t\tif err := qsub.AutoUnsubscribe(1); err != nil {\n\t\t\t\tt.Fatalf(\"Error on auto-unsubscribe: %v\", err)\n\t\t\t}\n\t\t\tqsub, err = c.QueueSubscribe(\"foo\", \"baz\", bazCb)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t\t}\n\t\t\tif err := qsub.AutoUnsubscribe(1); err != nil {\n\t\t\t\tt.Fatalf(\"Error on auto-unsubscribe: %v\", err)\n\t\t\t}\n\t\t}\n\t\tc.Subscribe(\"TEST.COMPLETE\", func(m *nats.Msg) {})\n\t}\n\n\t// We coelasce now so for each server we will have all local (200) plus\n\t// two from the remote side for each queue group. We also create one more\n\t// and will wait til each server has 204 subscriptions, that will make sure\n\t// that we have everything setup.\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsubsA := srvA.NumSubscriptions()\n\t\tsubsB := srvB.NumSubscriptions()\n\t\tif subsA != 204 || subsB != 204 {\n\t\t\treturn fmt.Errorf(\"Not all subs processed yet: %d and %d\", subsA, subsB)\n\t\t}\n\t\treturn nil\n\t})\n\n\texpected := int32(200)\n\t// Now send messages from each server\n\tfor i := int32(0); i < expected; i++ {\n\t\tc := cons[i%2]\n\t\tc.Publish(\"foo\", []byte(\"Don't Drop Me!\"))\n\t}\n\tfor _, c := range cons {\n\t\tc.Flush()\n\t}\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tnbar := atomic.LoadInt32(&rbar)\n\t\tnbaz := atomic.LoadInt32(&rbaz)\n\t\tif nbar == expected && nbaz == expected {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Did not receive all %d queue messages, received %d for 'bar' and %d for 'baz'\",\n\t\t\texpected, atomic.LoadInt32(&rbar), atomic.LoadInt32(&rbaz))\n\t})\n}\n\nfunc TestNoRaceClosedSlowConsumerWriteDeadline(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoSystemAccount = true\n\topts.WriteDeadline = 10 * time.Millisecond // Make very small to trip.\n\topts.MaxPending = 500 * 1024 * 1024 // Set high so it will not trip here.\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.DialTimeout(\"tcp\", fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port), 3*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer c.Close()\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\nPING\\r\\nSUB foo 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error sending protocols to server: %v\", err)\n\t}\n\t// Reduce socket buffer to increase reliability of data backing up in the server destined\n\t// for our subscribed client.\n\tc.(*net.TCPConn).SetReadBuffer(128)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tsender, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer sender.Close()\n\n\tpayload := make([]byte, 1024*1024)\n\tfor i := 0; i < 100; i++ {\n\t\tif err := sender.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\t// Flush sender connection to ensure that all data has been sent.\n\tif err := sender.Flush(); err != nil {\n\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t}\n\n\t// At this point server should have closed connection c.\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, SlowConsumerWriteDeadline)\n}\n\nfunc TestNoRaceClosedSlowConsumerPendingBytes(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoSystemAccount = true\n\topts.WriteDeadline = 30 * time.Second // Wait for long time so write deadline does not trigger slow consumer.\n\topts.MaxPending = 1 * 1024 * 1024 // Set to low value (1MB) to allow SC to trip.\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.DialTimeout(\"tcp\", fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port), 3*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer c.Close()\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\nPING\\r\\nSUB foo 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error sending protocols to server: %v\", err)\n\t}\n\t// Reduce socket buffer to increase reliability of data backing up in the server destined\n\t// for our subscribed client.\n\tc.(*net.TCPConn).SetReadBuffer(128)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tsender, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer sender.Close()\n\n\tpayload := make([]byte, 1024*1024)\n\tfor i := 0; i < 100; i++ {\n\t\tif err := sender.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\t// Flush sender connection to ensure that all data has been sent.\n\tif err := sender.Flush(); err != nil {\n\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t}\n\n\t// At this point server should have closed connection c.\n\tcheckClosedConns(t, s, 1, 2*time.Second)\n\tconns := s.closedClients()\n\tif lc := len(conns); lc != 1 {\n\t\tt.Fatalf(\"len(conns) expected to be %d, got %d\\n\", 1, lc)\n\t}\n\tcheckReason(t, conns[0].Reason, SlowConsumerPendingBytes)\n}\n\nfunc TestNoRaceSlowConsumerPendingBytes(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoSystemAccount = true\n\topts.WriteDeadline = 30 * time.Second // Wait for long time so write deadline does not trigger slow consumer.\n\topts.MaxPending = 1 * 1024 * 1024 // Set to low value (1MB) to allow SC to trip.\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.DialTimeout(\"tcp\", fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port), 3*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer c.Close()\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\nPING\\r\\nSUB foo 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error sending protocols to server: %v\", err)\n\t}\n\t// Reduce socket buffer to increase reliability of data backing up in the server destined\n\t// for our subscribed client.\n\tc.(*net.TCPConn).SetReadBuffer(128)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tsender, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer sender.Close()\n\n\tpayload := make([]byte, 1024*1024)\n\tfor i := 0; i < 100; i++ {\n\t\tif err := sender.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\t// Flush sender connection to ensure that all data has been sent.\n\tif err := sender.Flush(); err != nil {\n\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t}\n\n\t// At this point server should have closed connection c.\n\n\t// On certain platforms, it may take more than one call before\n\t// getting the error.\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := c.Write([]byte(\"PUB bar 5\\r\\nhello\\r\\n\")); err != nil {\n\t\t\t// ok\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatal(\"Connection should have been closed\")\n}\n\nfunc TestNoRaceGatewayNoMissingReplies(t *testing.T) {\n\t// This test will have following setup:\n\t//\n\t// responder1\t\t requestor\n\t// | |\n\t// v v\n\t// [A1]<-------gw------------[B1]\n\t// | \\ |\n\t// | \\______gw__________ | route\n\t// | _\\| |\n\t// [ ]--------gw----------->[ ]\n\t// [A2]<-------gw------------[B2]\n\t// [ ] [ ]\n\t// ^\n\t// |\n\t// responder2\n\t//\n\t// There is a possible race that when the requestor creates\n\t// a subscription on the reply subject, the subject interest\n\t// being sent from the inbound gateway, and B1 having none,\n\t// the SUB first goes to B2 before being sent to A1 from\n\t// B2's inbound GW. But the request can go from B1 to A1\n\t// right away and the responder1 connecting to A1 may send\n\t// back the reply before the interest on the reply makes it\n\t// to A1 (from B2).\n\t// This test will also verify that if the responder is instead\n\t// connected to A2, the reply is properly received by requestor\n\t// on B1.\n\n\t// For this test we want to be in interestOnly mode, so\n\t// make it happen quickly\n\tgatewayMaxRUnsubBeforeSwitch = 1\n\tdefer func() { gatewayMaxRUnsubBeforeSwitch = defaultGatewayMaxRUnsubBeforeSwitch }()\n\n\t// Start with setting up A2 and B2.\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\toa2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb2)\n\tsa2 := runGatewayServer(oa2)\n\tdefer sa2.Shutdown()\n\n\twaitForOutboundGateways(t, sa2, 1, time.Second)\n\twaitForInboundGateways(t, sa2, 1, time.Second)\n\twaitForOutboundGateways(t, sb2, 1, time.Second)\n\twaitForInboundGateways(t, sb2, 1, time.Second)\n\n\t// Now start A1 which will connect to B2\n\toa1 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", sb2)\n\toa1.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa2.Cluster.Host, oa2.Cluster.Port))\n\tsa1 := runGatewayServer(oa1)\n\tdefer sa1.Shutdown()\n\n\twaitForOutboundGateways(t, sa1, 1, time.Second)\n\twaitForInboundGateways(t, sb2, 2, time.Second)\n\n\tcheckClusterFormed(t, sa1, sa2)\n\n\t// Finally, start B1 that will connect to A1.\n\tob1 := testGatewayOptionsFromToWithServers(t, \"B\", \"A\", sa1)\n\tob1.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ob2.Cluster.Host, ob2.Cluster.Port))\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\t// Check that we have the outbound gateway from B1 to A1\n\tcheckFor(t, 3*time.Second, 15*time.Millisecond, func() error {\n\t\tc := sb1.getOutboundGatewayConnection(\"A\")\n\t\tif c == nil {\n\t\t\treturn fmt.Errorf(\"Outbound connection to A not created yet\")\n\t\t}\n\t\tc.mu.Lock()\n\t\tname := c.opts.Name\n\t\tnc := c.nc\n\t\tc.mu.Unlock()\n\t\tif name != sa1.ID() {\n\t\t\t// Force a disconnect\n\t\t\tnc.Close()\n\t\t\treturn fmt.Errorf(\"Was unable to have B1 connect to A1\")\n\t\t}\n\t\treturn nil\n\t})\n\n\twaitForInboundGateways(t, sa1, 1, time.Second)\n\tcheckClusterFormed(t, sb1, sb2)\n\n\ta1URL := fmt.Sprintf(\"nats://%s:%d\", oa1.Host, oa1.Port)\n\ta2URL := fmt.Sprintf(\"nats://%s:%d\", oa2.Host, oa2.Port)\n\tb1URL := fmt.Sprintf(\"nats://%s:%d\", ob1.Host, ob1.Port)\n\tb2URL := fmt.Sprintf(\"nats://%s:%d\", ob2.Host, ob2.Port)\n\n\tncb1 := natsConnect(t, b1URL)\n\tdefer ncb1.Close()\n\n\tncb2 := natsConnect(t, b2URL)\n\tdefer ncb2.Close()\n\n\tnatsSubSync(t, ncb1, \"just.a.sub\")\n\tnatsSubSync(t, ncb2, \"just.a.sub\")\n\tcheckExpectedSubs(t, 2, sb1, sb2)\n\n\t// For this test, we want A to be checking B's interest in order\n\t// to send messages (which would cause replies to be dropped if\n\t// there is no interest registered on A). So from A servers,\n\t// send to various subjects and cause B's to switch to interestOnly\n\t// mode.\n\tnca1 := natsConnect(t, a1URL)\n\tdefer nca1.Close()\n\tfor i := 0; i < 10; i++ {\n\t\tnatsPub(t, nca1, fmt.Sprintf(\"reject.%d\", i), []byte(\"hello\"))\n\t}\n\tnca2 := natsConnect(t, a2URL)\n\tdefer nca2.Close()\n\tfor i := 0; i < 10; i++ {\n\t\tnatsPub(t, nca2, fmt.Sprintf(\"reject.%d\", i), []byte(\"hello\"))\n\t}\n\n\tcheckSwitchedMode := func(t *testing.T, s *Server) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tvar switchedMode bool\n\t\t\tc := s.getOutboundGatewayConnection(\"B\")\n\t\t\tei, _ := c.gw.outsim.Load(globalAccountName)\n\t\t\tif ei != nil {\n\t\t\t\te := ei.(*outsie)\n\t\t\t\te.RLock()\n\t\t\t\tswitchedMode = e.ni == nil && e.mode == InterestOnly\n\t\t\t\te.RUnlock()\n\t\t\t}\n\t\t\tif !switchedMode {\n\t\t\t\treturn fmt.Errorf(\"Still not switched mode\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckSwitchedMode(t, sa1)\n\tcheckSwitchedMode(t, sa2)\n\n\t// Setup a subscriber on _INBOX.> on each of A's servers.\n\ttotal := 1000\n\texpected := int32(total)\n\trcvOnA := int32(0)\n\tqrcvOnA := int32(0)\n\tnatsSub(t, nca1, \"myreply.>\", func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&rcvOnA, 1)\n\t})\n\tnatsQueueSub(t, nca2, \"myreply.>\", \"bar\", func(_ *nats.Msg) {\n\t\tatomic.AddInt32(&qrcvOnA, 1)\n\t})\n\tcheckExpectedSubs(t, 2, sa1, sa2)\n\n\t// Ok.. so now we will run the actual test where we\n\t// create a responder on A1 and make sure that every\n\t// single request from B1 gets the reply. Will repeat\n\t// test with responder connected to A2.\n\tsendReqs := func(t *testing.T, subConn *nats.Conn) {\n\t\tt.Helper()\n\t\tresponder := natsSub(t, subConn, \"foo\", func(m *nats.Msg) {\n\t\t\tm.Respond([]byte(\"reply\"))\n\t\t})\n\t\tnatsFlush(t, subConn)\n\t\tcheckExpectedSubs(t, 3, sa1, sa2)\n\n\t\t// We are not going to use Request() because this sets\n\t\t// a wildcard subscription on an INBOX and less likely\n\t\t// to produce the race. Instead we will explicitly set\n\t\t// the subscription on the reply subject and create one\n\t\t// per request.\n\t\tfor i := 0; i < total/2; i++ {\n\t\t\treply := fmt.Sprintf(\"myreply.%d\", i)\n\t\t\treplySub := natsQueueSubSync(t, ncb1, reply, \"bar\")\n\t\t\tnatsFlush(t, ncb1)\n\n\t\t\t// Let's make sure we have interest on B2.\n\t\t\tif r := sb2.globalAccount().sl.Match(reply); len(r.qsubs) == 0 {\n\t\t\t\tcheckFor(t, time.Second, time.Millisecond, func() error {\n\t\t\t\t\tif r := sb2.globalAccount().sl.Match(reply); len(r.qsubs) == 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"B still not registered interest on %s\", reply)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tnatsPubReq(t, ncb1, \"foo\", reply, []byte(\"request\"))\n\t\t\tif _, err := replySub.NextMsg(time.Second); err != nil {\n\t\t\t\tt.Fatalf(\"Did not receive reply: %v\", err)\n\t\t\t}\n\t\t\tnatsUnsub(t, replySub)\n\t\t}\n\n\t\tresponder.Unsubscribe()\n\t\tnatsFlush(t, subConn)\n\t\tcheckExpectedSubs(t, 2, sa1, sa2)\n\t}\n\tsendReqs(t, nca1)\n\tsendReqs(t, nca2)\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif n := atomic.LoadInt32(&rcvOnA); n != expected {\n\t\t\treturn fmt.Errorf(\"Subs on A expected to get %v replies, got %v\", expected, n)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// We should not have received a single message on the queue sub\n\t// on cluster A because messages will have been delivered to\n\t// the member on cluster B.\n\tif n := atomic.LoadInt32(&qrcvOnA); n != 0 {\n\t\tt.Fatalf(\"Queue sub on A should not have received message, got %v\", n)\n\t}\n}\n\nfunc TestNoRaceRouteMemUsage(t *testing.T) {\n\toa := DefaultOptions()\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tob := DefaultOptions()\n\tob.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa.Cluster.Host, oa.Cluster.Port))\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tcheckClusterFormed(t, sa, sb)\n\n\tresponder := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\tdefer responder.Close()\n\tfor i := 0; i < 10; i++ {\n\t\tnatsSub(t, responder, \"foo\", func(m *nats.Msg) {\n\t\t\tm.Respond(m.Data)\n\t\t})\n\t}\n\tnatsFlush(t, responder)\n\n\tpayload := make([]byte, 50*1024)\n\n\tbURL := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\n\t// Capture mem usage\n\tmem := runtime.MemStats{}\n\truntime.ReadMemStats(&mem)\n\tinUseBefore := mem.HeapInuse\n\n\tfor i := 0; i < 100; i++ {\n\t\trequestor := natsConnect(t, bURL)\n\t\t// Don't use a defer here otherwise that will make the memory check fail!\n\t\t// We are closing the connection just after these few instructions that\n\t\t// are not calling t.Fatal() anyway.\n\t\tinbox := nats.NewInbox()\n\t\tsub := natsSubSync(t, requestor, inbox)\n\t\tnatsPubReq(t, requestor, \"foo\", inbox, payload)\n\t\tfor j := 0; j < 10; j++ {\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t}\n\t\trequestor.Close()\n\t}\n\n\truntime.GC()\n\tdebug.FreeOSMemory()\n\truntime.ReadMemStats(&mem)\n\tinUseNow := mem.HeapInuse\n\tif inUseNow > 3*inUseBefore {\n\t\tt.Fatalf(\"Heap in-use before was %v, now %v: too high\", inUseBefore, inUseNow)\n\t}\n}\n\nfunc TestNoRaceRouteCache(t *testing.T) {\n\tmaxPerAccountCacheSize = 20\n\tclosedSubsCheckInterval = 250 * time.Millisecond\n\n\tdefer func() {\n\t\tmaxPerAccountCacheSize = defaultMaxPerAccountCacheSize\n\t\tclosedSubsCheckInterval = defaultClosedSubsCheckInterval\n\t}()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuseQueue bool\n\t}{\n\t\t{\"plain_sub\", false},\n\t\t{\"queue_sub\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\n\t\t\toa := DefaultOptions()\n\t\t\toa.NoSystemAccount = true\n\t\t\toa.Cluster.PoolSize = -1\n\t\t\tsa := RunServer(oa)\n\t\t\tdefer sa.Shutdown()\n\n\t\t\tob := DefaultOptions()\n\t\t\tob.NoSystemAccount = true\n\t\t\tob.Cluster.PoolSize = -1\n\t\t\tob.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", oa.Cluster.Host, oa.Cluster.Port))\n\t\t\tsb := RunServer(ob)\n\t\t\tdefer sb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, sa, sb)\n\n\t\t\tresponder := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port))\n\t\t\tdefer responder.Close()\n\t\t\tnatsSub(t, responder, \"foo\", func(m *nats.Msg) {\n\t\t\t\tm.Respond(m.Data)\n\t\t\t})\n\t\t\tnatsFlush(t, responder)\n\n\t\t\tcheckExpectedSubs(t, 1, sa)\n\t\t\tcheckExpectedSubs(t, 1, sb)\n\n\t\t\tbURL := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\t\t\trequestor := natsConnect(t, bURL)\n\t\t\tdefer requestor.Close()\n\n\t\t\tch := make(chan struct{}, 1)\n\t\t\tcb := func(_ *nats.Msg) {\n\t\t\t\tselect {\n\t\t\t\tcase ch <- struct{}{}:\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsendReqs := func(t *testing.T, nc *nats.Conn, count int, unsub bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\t\tinbox := nats.NewInbox()\n\t\t\t\t\tvar sub *nats.Subscription\n\t\t\t\t\tif test.useQueue {\n\t\t\t\t\t\tsub = natsQueueSub(t, nc, inbox, \"queue\", cb)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsub = natsSub(t, nc, inbox, cb)\n\t\t\t\t\t}\n\t\t\t\t\tnatsPubReq(t, nc, \"foo\", inbox, []byte(\"hello\"))\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ch:\n\t\t\t\t\tcase <-time.After(time.Second):\n\t\t\t\t\t\tt.Fatalf(\"Failed to get reply\")\n\t\t\t\t\t}\n\t\t\t\t\tif unsub {\n\t\t\t\t\t\tnatsUnsub(t, sub)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tsendReqs(t, requestor, maxPerAccountCacheSize+1, true)\n\n\t\t\tvar route *client\n\t\t\tsb.mu.Lock()\n\t\t\troute = getFirstRoute(sb)\n\t\t\tsb.mu.Unlock()\n\n\t\t\tcheckExpected := func(t *testing.T, expected int) {\n\t\t\t\tt.Helper()\n\t\t\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\troute.mu.Lock()\n\t\t\t\t\tn := len(route.in.pacache)\n\t\t\t\t\troute.mu.Unlock()\n\t\t\t\t\tif n != expected {\n\t\t\t\t\t\treturn fmt.Errorf(\"Expected %v subs in the cache, got %v\", expected, n)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Expect the cache to be full\n\t\t\tcheckExpected(t, maxPerAccountCacheSize)\n\n\t\t\t// Wait for more than the orphan check\n\t\t\ttime.Sleep(2 * closedSubsCheckInterval)\n\n\t\t\t// Add a new subs up to point where new prunes would occur\n\t\t\tsendReqs(t, requestor, maxPerAccountCacheSize+1, false)\n\n\t\t\t// Now closed subs should have been removed, but we have added\n\t\t\t// new subs, so shouldn't exceed maxPerAccountCacheSize still\n\t\t\tcheckExpected(t, maxPerAccountCacheSize)\n\n\t\t\t// Now try wil implicit unsubscribe (due to connection close)\n\t\t\tsendReqs(t, requestor, maxPerAccountCacheSize+1, false)\n\t\t\trequestor.Close()\n\n\t\t\t// Cache should be empty now due to implicit unsubscribes\n\t\t\tcheckExpected(t, 0)\n\n\t\t\t// Wait for more than the orphan check\n\t\t\ttime.Sleep(2 * closedSubsCheckInterval)\n\n\t\t\t// Now create new connection and send maxPerAccountCacheSize+1\n\t\t\t// and that should cause all subs from previous connection to be\n\t\t\t// removed from cache\n\t\t\trequestor = natsConnect(t, bURL)\n\t\t\tdefer requestor.Close()\n\t\t\tsendReqs(t, requestor, maxPerAccountCacheSize+1, false)\n\n\t\t\t// We still won't have exceeded maxPerAccountCacheSize regardless\n\t\t\tcheckExpected(t, maxPerAccountCacheSize)\n\t\t})\n\t}\n}\n\nfunc TestNoRaceFetchAccountDoesNotRegisterAccountTwice(t *testing.T) {\n\tsa, oa, sb, ob, _ := runTrustedGateways(t)\n\tdefer sa.Shutdown()\n\tdefer sb.Shutdown()\n\n\t// Let's create a user account.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tjwt, _ := nac.Encode(okp)\n\tuserAcc := pub\n\n\t// Replace B's account resolver with one that introduces\n\t// delay during the Fetch()\n\tsac := &slowAccResolver{AccountResolver: sb.AccountResolver()}\n\tsb.SetAccountResolver(sac)\n\n\t// Add the account in sa and sb\n\taddAccountToMemResolver(sa, userAcc, jwt)\n\taddAccountToMemResolver(sb, userAcc, jwt)\n\n\t// Tell the slow account resolver which account to slow down\n\tsac.Lock()\n\tsac.acc = userAcc\n\tsac.Unlock()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d\", oa.Host, oa.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d\", ob.Host, ob.Port)\n\n\tnca, err := nats.Connect(urlA, createUserCreds(t, sa, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to A: %v\", err)\n\t}\n\tdefer nca.Close()\n\n\t// Since there is an optimistic send, this message will go to B\n\t// and on processing this message, B will lookup/fetch this\n\t// account, which can produce race with the fetch of this\n\t// account from A's system account that sent a notification\n\t// about this account, or with the client connect just after\n\t// that.\n\tnca.Publish(\"foo\", []byte(\"hello\"))\n\n\t// Now connect and create a subscription on B\n\tncb, err := nats.Connect(urlB, createUserCreds(t, sb, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting to A: %v\", err)\n\t}\n\tdefer ncb.Close()\n\tsub, err := ncb.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tncb.Flush()\n\n\t// Now send messages from A and B should ultimately start to receive\n\t// them (once the subscription has been correctly registered)\n\tok := false\n\tfor i := 0; i < 10; i++ {\n\t\tnca.Publish(\"foo\", []byte(\"hello\"))\n\t\tif _, err := sub.NextMsg(100 * time.Millisecond); err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tok = true\n\t\tbreak\n\t}\n\tif !ok {\n\t\tt.Fatalf(\"B should be able to receive messages\")\n\t}\n\n\tcheckTmpAccounts := func(t *testing.T, s *Server) {\n\t\tt.Helper()\n\t\tempty := true\n\t\ts.tmpAccounts.Range(func(_, _ any) bool {\n\t\t\tempty = false\n\t\t\treturn false\n\t\t})\n\t\tif !empty {\n\t\t\tt.Fatalf(\"tmpAccounts is not empty\")\n\t\t}\n\t}\n\tcheckTmpAccounts(t, sa)\n\tcheckTmpAccounts(t, sb)\n}\n\nfunc TestNoRaceWriteDeadline(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoSystemAccount = true\n\topts.WriteDeadline = 30 * time.Millisecond\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.DialTimeout(\"tcp\", fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port), 3*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer c.Close()\n\tif _, err := c.Write([]byte(\"CONNECT {}\\r\\nPING\\r\\nSUB foo 1\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Error sending protocols to server: %v\", err)\n\t}\n\t// Reduce socket buffer to increase reliability of getting\n\t// write deadline errors.\n\tc.(*net.TCPConn).SetReadBuffer(4)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tsender, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer sender.Close()\n\n\tpayload := make([]byte, 1000000)\n\ttotal := 1000\n\tfor i := 0; i < total; i++ {\n\t\tif err := sender.Publish(\"foo\", payload); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\t// Flush sender connection to ensure that all data has been sent.\n\tif err := sender.Flush(); err != nil {\n\t\tt.Fatalf(\"Error on flush: %v\", err)\n\t}\n\n\t// At this point server should have closed connection c.\n\n\t// On certain platforms, it may take more than one call before\n\t// getting the error.\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := c.Write([]byte(\"PUB bar 5\\r\\nhello\\r\\n\")); err != nil {\n\t\t\t// ok\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatal(\"Connection should have been closed\")\n}\n\nfunc TestNoRaceLeafNodeClusterNameConflictDeadlock(t *testing.T) {\n\to := DefaultOptions()\n\to.LeafNode.Port = -1\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tu, err := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\n\to1 := DefaultOptions()\n\to1.ServerName = \"A1\"\n\to1.Cluster.Name = \"clusterA\"\n\to1.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\tcheckLeafNodeConnected(t, s1)\n\n\to2 := DefaultOptions()\n\to2.ServerName = \"A2\"\n\to2.Cluster.Name = \"clusterA\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to2.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\tcheckClusterFormed(t, s1, s2)\n\n\to3 := DefaultOptions()\n\to3.ServerName = \"A3\"\n\to3.Cluster.Name = \"\" // intentionally not set\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to3.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\ts3 := RunServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckLeafNodeConnected(t, s3)\n\tcheckClusterFormed(t, s1, s2, s3)\n}\n\n// This test is same than TestAccountAddServiceImportRace but running\n// without the -race flag, it would capture more easily the possible\n// duplicate sid, resulting in less than expected number of subscriptions\n// in the account's internal subscriptions map.\nfunc TestNoRaceAccountAddServiceImportRace(t *testing.T) {\n\tTestAccountAddServiceImportRace(t)\n}\n\n// Similar to the routed version. Make sure we receive all of the\n// messages with auto-unsubscribe enabled.\nfunc TestNoRaceQueueAutoUnsubscribe(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\trbar := int32(0)\n\tbarCb := func(m *nats.Msg) {\n\t\tatomic.AddInt32(&rbar, 1)\n\t}\n\trbaz := int32(0)\n\tbazCb := func(m *nats.Msg) {\n\t\tatomic.AddInt32(&rbaz, 1)\n\t}\n\n\t// Create 1000 subscriptions with auto-unsubscribe of 1.\n\t// Do two groups, one bar and one baz.\n\ttotal := 1000\n\tfor i := 0; i < total; i++ {\n\t\tqsub, err := nc.QueueSubscribe(\"foo\", \"bar\", barCb)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\tif err := qsub.AutoUnsubscribe(1); err != nil {\n\t\t\tt.Fatalf(\"Error on auto-unsubscribe: %v\", err)\n\t\t}\n\t\tqsub, err = nc.QueueSubscribe(\"foo\", \"baz\", bazCb)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\tif err := qsub.AutoUnsubscribe(1); err != nil {\n\t\t\tt.Fatalf(\"Error on auto-unsubscribe: %v\", err)\n\t\t}\n\t}\n\tnc.Flush()\n\n\texpected := int32(total)\n\tfor i := int32(0); i < expected; i++ {\n\t\tnc.Publish(\"foo\", []byte(\"Don't Drop Me!\"))\n\t}\n\tnc.Flush()\n\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tnbar := atomic.LoadInt32(&rbar)\n\t\tnbaz := atomic.LoadInt32(&rbaz)\n\t\tif nbar == expected && nbaz == expected {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Did not receive all %d queue messages, received %d for 'bar' and %d for 'baz'\",\n\t\t\texpected, atomic.LoadInt32(&rbar), atomic.LoadInt32(&rbaz))\n\t})\n}\n\nfunc TestNoRaceAcceptLoopsDoNotLeaveOpenedConn(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\turl func(o *Options) (string, int)\n\t}{\n\t\t{\"client\", func(o *Options) (string, int) { return o.Host, o.Port }},\n\t\t{\"route\", func(o *Options) (string, int) { return o.Cluster.Host, o.Cluster.Port }},\n\t\t{\"gateway\", func(o *Options) (string, int) { return o.Gateway.Host, o.Gateway.Port }},\n\t\t{\"leafnode\", func(o *Options) (string, int) { return o.LeafNode.Host, o.LeafNode.Port }},\n\t\t{\"websocket\", func(o *Options) (string, int) { return o.Websocket.Host, o.Websocket.Port }},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.DisableShortFirstPing = true\n\t\t\to.Accounts = []*Account{NewAccount(\"$SYS\")}\n\t\t\to.SystemAccount = \"$SYS\"\n\t\t\to.Cluster.Name = \"abc\"\n\t\t\to.Cluster.Host = \"127.0.0.1\"\n\t\t\to.Cluster.Port = -1\n\t\t\to.Gateway.Name = \"abc\"\n\t\t\to.Gateway.Host = \"127.0.0.1\"\n\t\t\to.Gateway.Port = -1\n\t\t\to.LeafNode.Host = \"127.0.0.1\"\n\t\t\to.LeafNode.Port = -1\n\t\t\to.Websocket.Host = \"127.0.0.1\"\n\t\t\to.Websocket.Port = -1\n\t\t\to.Websocket.HandshakeTimeout = 1\n\t\t\to.Websocket.NoTLS = true\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\thost, port := test.url(o)\n\t\t\turl := net.JoinHostPort(host, fmt.Sprintf(\"%d\", port))\n\t\t\tvar conns []net.Conn\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(1)\n\t\t\tdone := make(chan struct{}, 1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\t// Have an upper limit\n\t\t\t\tfor i := 0; i < 200; i++ {\n\t\t\t\t\tc, err := net.Dial(\"tcp\", url)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tconns = append(conns, c)\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-done:\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t\ttime.Sleep(15 * time.Millisecond)\n\t\t\ts.Shutdown()\n\t\t\tclose(done)\n\t\t\twg.Wait()\n\t\t\tfor _, c := range conns {\n\t\t\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\t\tbr := bufio.NewReader(c)\n\t\t\t\t// Read INFO for connections that were accepted\n\t\t\t\t_, _, err := br.ReadLine()\n\t\t\t\tif err == nil {\n\t\t\t\t\t// After that, the connection should be closed,\n\t\t\t\t\t// so we should get an error here.\n\t\t\t\t\t_, _, err = br.ReadLine()\n\t\t\t\t}\n\t\t\t\t// We expect an io.EOF or any other error indicating the use of a closed\n\t\t\t\t// connection, but we should not get the timeout error.\n\t\t\t\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\t\t\t\terr = nil\n\t\t\t\t}\n\t\t\t\tif err == nil {\n\t\t\t\t\tvar buf [10]byte\n\t\t\t\t\tc.SetDeadline(time.Now().Add(2 * time.Second))\n\t\t\t\t\tc.Write([]byte(\"C\"))\n\t\t\t\t\t_, err = c.Read(buf[:])\n\t\t\t\t\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\t\t\t\t\terr = nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatalf(\"Connection should have been closed\")\n\t\t\t\t}\n\t\t\t\tc.Close()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNoRaceJetStreamDeleteStreamManyConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MYS\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Storage: FileStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\t// This number needs to be higher than the internal sendq size to trigger what this test is testing.\n\tfor i := 0; i < 2000; i++ {\n\t\t_, err := mset.addConsumer(&ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"D-%d\", i),\n\t\t\tDeliverSubject: fmt.Sprintf(\"deliver.%d\", i),\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating consumer: %v\", err)\n\t\t}\n\t}\n\t// With bug this would not return and would hang.\n\tmset.delete()\n}\n\n// We used to swap accounts on an inbound message when processing service imports.\n// Until JetStream this was kinda ok, but with JetStream we can have pull consumers\n// trying to access the clients account in another Go routine now which causes issues.\n// This is not limited to the case above, its just the one that exposed it.\n// This test is to show that issue and that the fix works, meaning we no longer swap c.acc.\nfunc TestNoRaceJetStreamServiceImportAccountSwapIssue(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tbeforeSubs := s.NumSubscriptions()\n\n\t// How long we want both sides to run.\n\ttimeout := time.Now().Add(3 * time.Second)\n\terrs := make(chan error, 1)\n\n\t// Publishing side, which will signal the consumer that is waiting and which will access c.acc. If publish\n\t// operation runs concurrently we will catch c.acc being $SYS some of the time.\n\tgo func() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tfor time.Now().Before(timeout) {\n\t\t\t// This will signal the delivery of the pull messages.\n\t\t\tjs.Publish(\"foo\", []byte(\"Hello\"))\n\t\t\t// This will swap the account because of JetStream service import.\n\t\t\t// We can get an error here with the bug or not.\n\t\t\tif _, err := js.StreamInfo(\"TEST\"); err != nil {\n\t\t\t\terrs <- err\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\terrs <- nil\n\t}()\n\n\t// Pull messages flow.\n\tvar received int\n\tfor time.Now().Before(timeout.Add(2 * time.Second)) {\n\t\tif msgs, err := sub.Fetch(1, nats.MaxWait(200*time.Millisecond)); err == nil {\n\t\t\tfor _, m := range msgs {\n\t\t\t\treceived++\n\t\t\t\tm.AckSync()\n\t\t\t}\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\t// Wait on publisher Go routine and check for errors.\n\tif err := <-errs; err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Double check all received.\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif int(si.State.Msgs) != received {\n\t\tt.Fatalf(\"Expected to receive %d msgs, only got %d\", si.State.Msgs, received)\n\t}\n\t// Now check for leaked subs from the fetch call above. That is what we first saw from the bug.\n\tif afterSubs := s.NumSubscriptions(); afterSubs != beforeSubs {\n\t\tt.Fatalf(\"Leaked subscriptions: %d before, %d after\", beforeSubs, afterSubs)\n\t}\n}\n\nfunc TestNoRaceJetStreamAPIStreamListPaging(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Create 2X limit\n\tstreamsNum := 2 * JSApiNamesLimit\n\tfor i := 1; i <= streamsNum; i++ {\n\t\tname := fmt.Sprintf(\"STREAM-%06d\", i)\n\t\tcfg := StreamConfig{Name: name, Storage: MemoryStorage}\n\t\t_, err := s.GlobalAccount().addStream(&cfg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t\t}\n\t}\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\treqList := func(offset int) []byte {\n\t\tt.Helper()\n\t\tvar req []byte\n\t\tif offset > 0 {\n\t\t\treq, _ = json.Marshal(&ApiPagedRequest{Offset: offset})\n\t\t}\n\t\tresp, err := nc.Request(JSApiStreams, req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting stream list: %v\", err)\n\t\t}\n\t\treturn resp.Data\n\t}\n\n\tcheckResp := func(resp []byte, expectedLen, expectedOffset int) {\n\t\tt.Helper()\n\t\tvar listResponse JSApiStreamNamesResponse\n\t\tif err := json.Unmarshal(resp, &listResponse); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif len(listResponse.Streams) != expectedLen {\n\t\t\tt.Fatalf(\"Expected only %d streams but got %d\", expectedLen, len(listResponse.Streams))\n\t\t}\n\t\tif listResponse.Total != streamsNum {\n\t\t\tt.Fatalf(\"Expected total to be %d but got %d\", streamsNum, listResponse.Total)\n\t\t}\n\t\tif listResponse.Offset != expectedOffset {\n\t\t\tt.Fatalf(\"Expected offset to be %d but got %d\", expectedOffset, listResponse.Offset)\n\t\t}\n\t\tif expectedLen < 1 {\n\t\t\treturn\n\t\t}\n\t\t// Make sure we get the right stream.\n\t\tsname := fmt.Sprintf(\"STREAM-%06d\", expectedOffset+1)\n\t\tif listResponse.Streams[0] != sname {\n\t\t\tt.Fatalf(\"Expected stream %q to be first, got %q\", sname, listResponse.Streams[0])\n\t\t}\n\t}\n\n\tcheckResp(reqList(0), JSApiNamesLimit, 0)\n\tcheckResp(reqList(JSApiNamesLimit), JSApiNamesLimit, JSApiNamesLimit)\n\tcheckResp(reqList(streamsNum), 0, streamsNum)\n\tcheckResp(reqList(streamsNum-22), 22, streamsNum-22)\n\tcheckResp(reqList(streamsNum+22), 0, streamsNum)\n}\n\nfunc TestNoRaceJetStreamAPIConsumerListPaging(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tsname := \"MYSTREAM\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: sname})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\t// Client for API requests.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\tconsumersNum := JSApiNamesLimit\n\tfor i := 1; i <= consumersNum; i++ {\n\t\tdsubj := fmt.Sprintf(\"d.%d\", i)\n\t\tsub, _ := nc.SubscribeSync(dsubj)\n\t\tdefer sub.Unsubscribe()\n\t\tnc.Flush()\n\n\t\t_, err := mset.addConsumer(&ConsumerConfig{DeliverSubject: dsubj})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\treqListSubject := fmt.Sprintf(JSApiConsumersT, sname)\n\treqList := func(offset int) []byte {\n\t\tt.Helper()\n\t\tvar req []byte\n\t\tif offset > 0 {\n\t\t\treq, _ = json.Marshal(&JSApiConsumersRequest{ApiPagedRequest: ApiPagedRequest{Offset: offset}})\n\t\t}\n\t\tresp, err := nc.Request(reqListSubject, req, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error getting stream list: %v\", err)\n\t\t}\n\t\treturn resp.Data\n\t}\n\n\tcheckResp := func(resp []byte, expectedLen, expectedOffset int) {\n\t\tt.Helper()\n\t\tvar listResponse JSApiConsumerNamesResponse\n\t\tif err := json.Unmarshal(resp, &listResponse); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif len(listResponse.Consumers) != expectedLen {\n\t\t\tt.Fatalf(\"Expected only %d streams but got %d\", expectedLen, len(listResponse.Consumers))\n\t\t}\n\t\tif listResponse.Total != consumersNum {\n\t\t\tt.Fatalf(\"Expected total to be %d but got %d\", consumersNum, listResponse.Total)\n\t\t}\n\t\tif listResponse.Offset != expectedOffset {\n\t\t\tt.Fatalf(\"Expected offset to be %d but got %d\", expectedOffset, listResponse.Offset)\n\t\t}\n\t}\n\n\tcheckResp(reqList(0), JSApiNamesLimit, 0)\n\tcheckResp(reqList(consumersNum-22), 22, consumersNum-22)\n\tcheckResp(reqList(consumersNum+22), 0, consumersNum)\n}\n\nfunc TestNoRaceJetStreamWorkQueueLoadBalance(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tmname := \"MY_MSG_SET\"\n\tmset, err := s.GlobalAccount().addStream(&StreamConfig{Name: mname, Subjects: []string{\"foo\", \"bar\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error adding message set: %v\", err)\n\t}\n\tdefer mset.delete()\n\n\t// Create basic work queue mode consumer.\n\toname := \"WQ\"\n\to, err := mset.addConsumer(&ConsumerConfig{Durable: oname, AckPolicy: AckExplicit})\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error with durable, got %v\", err)\n\t}\n\tdefer o.delete()\n\n\t// To send messages.\n\tnc := clientConnectToServer(t, s)\n\tdefer nc.Close()\n\n\t// For normal work queue semantics, you send requests to the subject with stream and consumer name.\n\treqMsgSubj := o.requestNextMsgSubject()\n\n\tnumWorkers := 25\n\tcounts := make([]int32, numWorkers)\n\tvar received int32\n\n\trwg := &sync.WaitGroup{}\n\trwg.Add(numWorkers)\n\n\twg := &sync.WaitGroup{}\n\twg.Add(numWorkers)\n\tch := make(chan bool)\n\n\ttoSend := 1000\n\n\tfor i := 0; i < numWorkers; i++ {\n\t\tnc := clientConnectToServer(t, s)\n\t\tdefer nc.Close()\n\n\t\tgo func(index int32) {\n\t\t\trwg.Done()\n\t\t\tdefer wg.Done()\n\t\t\t<-ch\n\n\t\t\tfor counter := &counts[index]; ; {\n\t\t\t\tm, err := nc.Request(reqMsgSubj, nil, 100*time.Millisecond)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tm.Respond(nil)\n\t\t\t\tatomic.AddInt32(counter, 1)\n\t\t\t\tif total := atomic.AddInt32(&received, 1); total >= int32(toSend) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}(int32(i))\n\t}\n\n\t// Wait for requestors to be ready\n\trwg.Wait()\n\tclose(ch)\n\n\tsendSubj := \"bar\"\n\tfor i := 0; i < toSend; i++ {\n\t\tsendStreamMsg(t, nc, sendSubj, \"Hello World!\")\n\t}\n\n\t// Wait for test to complete.\n\twg.Wait()\n\n\ttarget := toSend / numWorkers\n\tdelta := target/2 + 5\n\tlow, high := int32(target-delta), int32(target+delta)\n\n\tfor i := 0; i < numWorkers; i++ {\n\t\tif msgs := atomic.LoadInt32(&counts[i]); msgs < low || msgs > high {\n\t\t\tt.Fatalf(\"Messages received for worker [%d] too far off from target of %d, got %d\", i, target, msgs)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterLargeStreamInlineCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"LSS\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsr := c.randomNonStreamLeader(\"$G\", \"TEST\")\n\tsr.Shutdown()\n\n\t// In case sr was meta leader.\n\tc.waitOnLeader()\n\n\tmsg, toSend := []byte(\"Hello JS Clustering\"), 5000\n\n\t// Now fill up stream.\n\tfor i := 0; i < toSend; i++ {\n\t\tif _, err = js.Publish(\"foo\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tsi, err := js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Check active state as well, shows that the owner answered.\n\tif si.State.Msgs != uint64(toSend) {\n\t\tt.Fatalf(\"Expected %d msgs, got bad state: %+v\", toSend, si.State)\n\t}\n\n\t// Kill our current leader to make just 2.\n\tc.streamLeader(\"$G\", \"TEST\").Shutdown()\n\n\t// Now restart the shutdown peer and wait for it to be current.\n\tsr = c.restartServer(sr)\n\tc.waitOnStreamCurrent(sr, \"$G\", \"TEST\")\n\n\t// Ask other servers to stepdown as leader so that sr becomes the leader.\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\t\tif sl := c.streamLeader(\"$G\", \"TEST\"); sl != sr {\n\t\t\tsl.JetStreamStepdownStream(\"$G\", \"TEST\")\n\t\t\treturn fmt.Errorf(\"Server %s is not leader yet\", sr)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsi, err = js.StreamInfo(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Check that we have all of our messsages stored.\n\t// Wait for a bit for upper layers to process.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", toSend, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterStreamCreateAndLostQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(JSAdvisoryStreamQuorumLostPre + \".*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"NO-LQ-START\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tc.waitOnStreamLeader(\"$G\", \"NO-LQ-START\")\n\tcheckSubsPending(t, sub, 0)\n\n\tc.stopAll()\n\t// Start up the one we were connected to first and wait for it to be connected.\n\ts = c.restartServer(s)\n\tnc, err = nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsub, err = nc.SubscribeSync(JSAdvisoryStreamQuorumLostPre + \".*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tnc.Flush()\n\n\tc.restartAll()\n\tc.waitOnStreamLeader(\"$G\", \"NO-LQ-START\")\n\n\tcheckSubsPending(t, sub, 0)\n}\n\nfunc TestNoRaceJetStreamSuperClusterMirrors(t *testing.T) {\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.clusterForName(\"C2\").randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create source stream.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S1\", Subjects: []string{\"foo\", \"bar\"}, Replicas: 3, Placement: &nats.Placement{Cluster: \"C2\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Needed while Go client does not have mirror support.\n\tcreateStream := func(cfg *nats.StreamConfig) {\n\t\tt.Helper()\n\t\tif _, err := js.AddStream(cfg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", err)\n\t\t}\n\t}\n\n\t// Send 100 messages.\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := js.Publish(\"foo\", []byte(\"MIRRORS!\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcreateStream(&nats.StreamConfig{\n\t\tName: \"M1\",\n\t\tMirror: &nats.StreamSource{Name: \"S1\"},\n\t\tPlacement: &nats.Placement{Cluster: \"C1\"},\n\t})\n\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M1\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 100 {\n\t\t\treturn fmt.Errorf(\"Expected 100 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Purge the source stream.\n\tif err := js.PurgeStream(\"S1\"); err != nil {\n\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t}\n\t// Send 50 more msgs now.\n\tfor i := 0; i < 50; i++ {\n\t\tif _, err := js.Publish(\"bar\", []byte(\"OK\")); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\n\tcreateStream(&nats.StreamConfig{\n\t\tName: \"M2\",\n\t\tMirror: &nats.StreamSource{Name: \"S1\"},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: \"C3\"},\n\t})\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M2\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 101 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 101, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsl := sc.clusterForName(\"C3\").streamLeader(\"$G\", \"M2\")\n\tdoneCh := make(chan bool)\n\n\t// Now test that if the mirror get's interrupted that it picks up where it left off etc.\n\tgo func() {\n\t\t// Send 100 more messages.\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tif _, err := js.Publish(\"foo\", []byte(\"MIRRORS!\")); err != nil {\n\t\t\t\tt.Errorf(\"Unexpected publish on %d error: %v\", i, err)\n\t\t\t}\n\t\t\ttime.Sleep(2 * time.Millisecond)\n\t\t}\n\t\tdoneCh <- true\n\t}()\n\n\ttime.Sleep(20 * time.Millisecond)\n\tsl.Shutdown()\n\n\t<-doneCh\n\tsc.clusterForName(\"C3\").waitOnStreamLeader(\"$G\", \"M2\")\n\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M2\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 150 {\n\t\t\treturn fmt.Errorf(\"Expected 150 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 101 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 101, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamSuperClusterMixedModeMirrors(t *testing.T) {\n\t// Unlike the similar sources test, this test is not reliably catching the bug\n\t// that would cause mirrors to not have the expected messages count.\n\t// Still, adding this test in case we have a regression and we are lucky in\n\t// getting the failure while running this.\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf: { listen: 127.0.0.1:-1 }\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 7, 4,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tsname := serverName[strings.Index(serverName, \"-\")+1:]\n\t\t\tswitch sname {\n\t\t\tcase \"S5\", \"S6\", \"S7\":\n\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream: { \", \"#jetstream: { \")\n\t\t\tdefault:\n\t\t\t\tconf = strings.ReplaceAll(conf, \"leaf: { \", \"#leaf: { \")\n\t\t\t}\n\t\t\treturn conf\n\t\t}, nil)\n\tdefer sc.shutdown()\n\n\t// Connect our client to a non JS server\n\tc := sc.randomCluster()\n\tvar s *Server\n\tfor {\n\t\tif as := c.randomServer(); !as.JetStreamEnabled() {\n\t\t\ts = as\n\t\t\tbreak\n\t\t}\n\t}\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tnumStreams := 10\n\ttoSend := 1000\n\terrCh := make(chan error, numStreams)\n\twg := sync.WaitGroup{}\n\twg.Add(numStreams)\n\t// Create 10 origin streams\n\tfor i := 0; i < 10; i++ {\n\t\tgo func(idx int) {\n\t\t\tdefer wg.Done()\n\t\t\tname := fmt.Sprintf(\"S%d\", idx+1)\n\t\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\t\terrCh <- fmt.Errorf(\"unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tc.waitOnStreamLeader(globalAccountName, name)\n\t\t\t// Load them up with a bunch of messages.\n\t\t\tfor n := 0; n < toSend; n++ {\n\t\t\t\tm := nats.NewMsg(name)\n\t\t\t\tm.Header.Set(\"stream\", name)\n\t\t\t\tm.Header.Set(\"idx\", strconv.FormatInt(int64(n+1), 10))\n\t\t\t\tif err := nc.PublishMsg(m); err != nil {\n\t\t\t\t\terrCh <- fmt.Errorf(\"unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\tselect {\n\tcase err := <-errCh:\n\t\tt.Fatal(err)\n\tdefault:\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\t// Now create our mirrors\n\t\twg := sync.WaitGroup{}\n\t\tmirrorsCount := 10\n\t\twg.Add(mirrorsCount)\n\t\terrCh := make(chan error, 1)\n\t\tfor m := 0; m < mirrorsCount; m++ {\n\t\t\tsname := fmt.Sprintf(\"S%d\", rand.Intn(10)+1)\n\t\t\tgo func(sname string, mirrorIdx int) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"M%d\", mirrorIdx),\n\t\t\t\t\tMirror: &nats.StreamSource{Name: sname},\n\t\t\t\t\tReplicas: 3,\n\t\t\t\t}); err != nil {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase errCh <- err:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(sname, m+1)\n\t\t}\n\t\twg.Wait()\n\t\tselect {\n\t\tcase err := <-errCh:\n\t\t\tt.Fatalf(\"Error creating mirrors: %v\", err)\n\t\tdefault:\n\t\t}\n\t\t// Now check the mirrors have all expected messages\n\t\tfor m := 0; m < mirrorsCount; m++ {\n\t\t\tname := fmt.Sprintf(\"M%d\", m+1)\n\t\t\tcheckFor(t, 15*time.Second, 500*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(name)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Could not retrieve stream info\")\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != uint64(toSend) {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", toSend, si.State)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\terr := js.DeleteStream(name)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamSuperClusterSources(t *testing.T) {\n\towt := srcConsumerWaitTime\n\tsrcConsumerWaitTime = 2 * time.Second\n\tdefer func() { srcConsumerWaitTime = owt }()\n\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.clusterForName(\"C1\").randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Create our source streams.\n\tfor _, sname := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 1}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tsendBatch := func(subject string, n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tmsg := fmt.Sprintf(\"MSG-%d\", i+1)\n\t\t\tif _, err := js.Publish(subject, []byte(msg)); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\t// Populate each one.\n\tsendBatch(\"foo\", 10)\n\tsendBatch(\"bar\", 15)\n\tsendBatch(\"baz\", 25)\n\n\t// Needed while Go client does not have mirror support for creating mirror or source streams.\n\tcreateStream := func(cfg *nats.StreamConfig) {\n\t\tt.Helper()\n\t\tif _, err := js.AddStream(cfg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", err)\n\t\t}\n\t}\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"MS\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{Name: \"foo\"},\n\t\t\t{Name: \"bar\"},\n\t\t\t{Name: \"baz\"},\n\t\t},\n\t}\n\n\tcreateStream(cfg)\n\ttime.Sleep(time.Second)\n\n\t// Faster timeout since we loop below checking for condition.\n\tjs2, err := nc.JetStream(nats.MaxWait(50 * time.Millisecond))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MS\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Purge the source streams.\n\tfor _, sname := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\tif err := js.PurgeStream(sname); err != nil {\n\t\t\tt.Fatalf(\"Unexpected purge error: %v\", err)\n\t\t}\n\t}\n\n\tif err := js.DeleteStream(\"MS\"); err != nil {\n\t\tt.Fatalf(\"Unexpected delete error: %v\", err)\n\t}\n\n\t// Send more msgs now.\n\tsendBatch(\"foo\", 10)\n\tsendBatch(\"bar\", 15)\n\tsendBatch(\"baz\", 25)\n\n\tcfg = &nats.StreamConfig{\n\t\tName: \"MS2\",\n\t\tSources: []*nats.StreamSource{\n\t\t\t{Name: \"foo\"},\n\t\t\t{Name: \"bar\"},\n\t\t\t{Name: \"baz\"},\n\t\t},\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: \"C3\"},\n\t}\n\n\tcreateStream(cfg)\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js2.StreamInfo(\"MS2\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 50 {\n\t\t\treturn fmt.Errorf(\"Expected 50 msgs, got state: %+v\", si.State)\n\t\t}\n\t\tif si.State.FirstSeq != 1 {\n\t\t\treturn fmt.Errorf(\"Expected start seq of 1, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n\tsl := sc.clusterForName(\"C3\").streamLeader(\"$G\", \"MS2\")\n\tdoneCh := make(chan bool)\n\n\tif sl == sc.leader() {\n\t\tsnc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\tdefer snc.Close()\n\t\t_, err := snc.Request(JSApiLeaderStepDown, nil, time.Second)\n\t\trequire_NoError(t, err)\n\t\tsc.waitOnLeader()\n\t}\n\n\t// Now test that if the mirror get's interrupted that it picks up where it left off etc.\n\tgo func() {\n\t\t// Send 50 more messages each.\n\t\tfor i := 0; i < 50; i++ {\n\t\t\tmsg := fmt.Sprintf(\"R-MSG-%d\", i+1)\n\t\t\tfor _, sname := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\t\t\tm := nats.NewMsg(sname)\n\t\t\t\tm.Data = []byte(msg)\n\t\t\t\tif _, err := js.PublishMsg(m); err != nil {\n\t\t\t\t\tt.Errorf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\ttime.Sleep(2 * time.Millisecond)\n\t\t}\n\t\tdoneCh <- true\n\t}()\n\n\ttime.Sleep(20 * time.Millisecond)\n\tsl.Shutdown()\n\n\tsc.clusterForName(\"C3\").waitOnStreamLeader(\"$G\", \"MS2\")\n\t<-doneCh\n\n\tcheckFor(t, 20*time.Second, time.Second, func() error {\n\t\tsi, err := js2.StreamInfo(\"MS2\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != 200 {\n\t\t\treturn fmt.Errorf(\"Expected 200 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterSourcesMuxd(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"SMUX\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Send in 10000 messages.\n\tmsg, toSend := make([]byte, 1024), 10000\n\tcrand.Read(msg)\n\n\tvar sources []*nats.StreamSource\n\t// Create 10 origin streams.\n\tfor i := 1; i <= 10; i++ {\n\t\tname := fmt.Sprintf(\"O-%d\", i)\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\t// Make sure we have a leader before publishing, especially since we use\n\t\t// non JS publisher, we would not know if the messages made it to those\n\t\t// streams or not.\n\t\tc.waitOnStreamLeader(globalAccountName, name)\n\t\t// Load them up with a bunch of messages.\n\t\tfor n := 0; n < toSend; n++ {\n\t\t\tif err := nc.Publish(name, msg); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t\tsources = append(sources, &nats.StreamSource{Name: name})\n\t}\n\n\t// Now create our downstream stream that sources from all of them.\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"S\", Replicas: 2, Sources: sources}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not retrieve stream info\")\n\t\t}\n\t\tif si.State.Msgs != uint64(10*toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", toSend*10, si.State)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestNoRaceJetStreamSuperClusterMixedModeSources(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: { domain: ngs, max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\t\tleaf: { listen: 127.0.0.1:-1 }\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tsc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 7, 2,\n\t\tfunc(serverName, clusterName, storeDir, conf string) string {\n\t\t\tsname := serverName[strings.Index(serverName, \"-\")+1:]\n\t\t\tswitch sname {\n\t\t\tcase \"S5\", \"S6\", \"S7\":\n\t\t\t\tconf = strings.ReplaceAll(conf, \"jetstream: { \", \"#jetstream: { \")\n\t\t\tdefault:\n\t\t\t\tconf = strings.ReplaceAll(conf, \"leaf: { \", \"#leaf: { \")\n\t\t\t}\n\t\t\treturn conf\n\t\t}, nil)\n\tdefer sc.shutdown()\n\t// Connect our client to a non JS server\n\tc := sc.randomCluster()\n\tvar s *Server\n\tfor {\n\t\tif as := c.randomServer(); !as.JetStreamEnabled() {\n\t\t\ts = as\n\t\t\tbreak\n\t\t}\n\t}\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tnumStreams := 100\n\ttoSend := 1000\n\tvar sources []*nats.StreamSource\n\terrCh := make(chan error, numStreams)\n\tsrcCh := make(chan *nats.StreamSource, numStreams)\n\twg := sync.WaitGroup{}\n\twg.Add(numStreams)\n\t// Create 100 origin streams.\n\tfor i := 1; i <= numStreams; i++ {\n\t\tgo func(idx int) {\n\t\t\tdefer wg.Done()\n\n\t\t\tname := fmt.Sprintf(\"O-%d\", idx)\n\t\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\t\terrCh <- fmt.Errorf(\"unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tc.waitOnStreamLeader(globalAccountName, name)\n\t\t\t// Load them up with a bunch of messages.\n\t\t\tfor n := 0; n < toSend; n++ {\n\t\t\t\tm := nats.NewMsg(name)\n\t\t\t\tm.Header.Set(\"stream\", name)\n\t\t\t\tm.Header.Set(\"idx\", strconv.FormatInt(int64(n+1), 10))\n\t\t\t\tif err := nc.PublishMsg(m); err != nil {\n\t\t\t\t\terrCh <- fmt.Errorf(\"unexpected publish error: %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tsrcCh <- &nats.StreamSource{Name: name}\n\t\t}(i)\n\t}\n\twg.Wait()\n\tselect {\n\tcase err := <-errCh:\n\t\tt.Fatal(err)\n\tdefault:\n\t}\n\tfor i := 0; i < numStreams; i++ {\n\t\tsources = append(sources, <-srcCh)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\t// Now create our downstream stream that sources from all of them.\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"S\", Replicas: 3, Sources: sources}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\tcheckFor(t, 15*time.Second, 1000*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"S\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Could not retrieve stream info\")\n\t\t\t}\n\t\t\tif si.State.Msgs != uint64(numStreams*toSend) {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got state: %+v\", numStreams*toSend, si.State)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\terr := js.DeleteStream(\"S\")\n\t\trequire_NoError(t, err)\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterExtendedStreamPurgeStall(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\tcerr := func(t *testing.T, err error) {\n\t\tt.Helper()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"unexepected err: %s\", err)\n\t\t}\n\t}\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"kv.>\"},\n\t\tStorage: nats.FileStorage,\n\t})\n\tcerr(t, err)\n\n\t// 100kb messages spread over 1000 different subjects\n\tbody := make([]byte, 100*1024)\n\tfor i := 0; i < 50000; i++ {\n\t\tif _, err := js.PublishAsync(fmt.Sprintf(\"kv.%d\", i%1000), body); err != nil {\n\t\t\tcerr(t, err)\n\t\t}\n\t}\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tif si, err = js.StreamInfo(\"KV\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs == 50000 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"waiting for more\")\n\t})\n\n\tjp, _ := json.Marshal(&JSApiStreamPurgeRequest{Subject: \"kv.20\"})\n\tstart := time.Now()\n\tres, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"KV\"), jp, time.Minute)\n\telapsed := time.Since(start)\n\tcerr(t, err)\n\tpres := JSApiStreamPurgeResponse{}\n\terr = json.Unmarshal(res.Data, &pres)\n\tcerr(t, err)\n\tif !pres.Success {\n\t\tt.Fatalf(\"purge failed: %#v\", pres)\n\t}\n\tif elapsed > time.Second {\n\t\tt.Fatalf(\"Purge took too long %s\", elapsed)\n\t}\n\tv, _ := s.Varz(nil)\n\tif v.Mem > 100*1024*1024 { // 100MB limit but in practice < 100MB -> Was ~7GB when failing.\n\t\tt.Fatalf(\"Used too much memory: %v\", friendlyBytes(v.Mem))\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterMirrorExpirationAndMissingSequences(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"MMS\", 9)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsendBatch := func(n int) {\n\t\tt.Helper()\n\t\t// Send a batch to a given subject.\n\t\tfor i := 0; i < n; i++ {\n\t\t\tif _, err := js.Publish(\"TEST\", []byte(\"OK\")); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckStream := func(stream string, num uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 20*time.Second, 20*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(stream)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != num {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", num, si.State.Msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckMirror := func(num uint64) { t.Helper(); checkStream(\"M\", num) }\n\tcheckTest := func(num uint64) { t.Helper(); checkStream(\"TEST\", num) }\n\n\t// Origin\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tMaxAge: 500 * time.Millisecond,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tts := c.streamLeader(\"$G\", \"TEST\")\n\tml := c.leader()\n\n\t// Create mirror now.\n\tfor ms := ts; ms == ts || ms == ml; {\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"M\",\n\t\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\t\tReplicas: 2,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tms = c.streamLeader(\"$G\", \"M\")\n\t\tif ts == ms || ms == ml {\n\t\t\t// Delete and retry.\n\t\t\tjs.DeleteStream(\"M\")\n\t\t}\n\t}\n\n\tsendBatch(10)\n\tcheckMirror(10)\n\n\t// Now shutdown the server with the mirror.\n\tms := c.streamLeader(\"$G\", \"M\")\n\tms.Shutdown()\n\tc.waitOnLeader()\n\n\t// Send more messages but let them expire.\n\tsendBatch(10)\n\tcheckTest(0)\n\n\tc.restartServer(ms)\n\tc.checkClusterFormed()\n\tc.waitOnStreamLeader(\"$G\", \"M\")\n\n\tsendBatch(10)\n\tcheckMirror(20)\n}\n\nfunc TestNoRaceJetStreamClusterLargeActiveOnReplica(t *testing.T) {\n\t// Uncomment to run.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"LAG\", 3)\n\tdefer c.shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\ttimeout := time.Now().Add(60 * time.Second)\n\tfor time.Now().Before(timeout) {\n\t\tsi, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\t\tReplicas: 3,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tfor _, r := range si.Cluster.Replicas {\n\t\t\tif r.Active > 5*time.Second {\n\t\t\t\tt.Fatalf(\"Bad Active value: %+v\", r)\n\t\t\t}\n\t\t}\n\t\tif err := js.DeleteStream(\"TEST\"); err != nil {\n\t\t\tt.Fatalf(\"Unexpected delete error: %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamSuperClusterRIPStress(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine.\n\tskip(t)\n\n\tsc := createJetStreamSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Client based API\n\ts := sc.clusterForName(\"C2\").randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tscm := make(map[string][]string)\n\n\t// Create 50 streams per cluster.\n\tfor _, cn := range []string{\"C1\", \"C2\", \"C3\"} {\n\t\tvar streams []string\n\t\tfor i := 0; i < 50; i++ {\n\t\t\tsn := fmt.Sprintf(\"%s-S%d\", cn, i+1)\n\t\t\tstreams = append(streams, sn)\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: sn,\n\t\t\t\tReplicas: 3,\n\t\t\t\tPlacement: &nats.Placement{Cluster: cn},\n\t\t\t\tMaxAge: 2 * time.Minute,\n\t\t\t\tMaxMsgs: 50_000,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\t\tscm[cn] = streams\n\t}\n\n\tsourceForCluster := func(cn string) []*nats.StreamSource {\n\t\tvar sns []string\n\t\tswitch cn {\n\t\tcase \"C1\":\n\t\t\tsns = scm[\"C2\"]\n\t\tcase \"C2\":\n\t\t\tsns = scm[\"C3\"]\n\t\tcase \"C3\":\n\t\t\tsns = scm[\"C1\"]\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unknown cluster %q\", cn)\n\t\t}\n\t\tvar ss []*nats.StreamSource\n\t\tfor _, sn := range sns {\n\t\t\tss = append(ss, &nats.StreamSource{Name: sn})\n\t\t}\n\t\treturn ss\n\t}\n\n\t// Mux all 50 streams from one cluster to a single stream across a GW connection to another cluster.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"C1-S-MUX\",\n\t\tReplicas: 2,\n\t\tPlacement: &nats.Placement{Cluster: \"C1\"},\n\t\tSources: sourceForCluster(\"C2\"),\n\t\tMaxAge: time.Minute,\n\t\tMaxMsgs: 20_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"C2-S-MUX\",\n\t\tReplicas: 2,\n\t\tPlacement: &nats.Placement{Cluster: \"C2\"},\n\t\tSources: sourceForCluster(\"C3\"),\n\t\tMaxAge: time.Minute,\n\t\tMaxMsgs: 20_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"C3-S-MUX\",\n\t\tReplicas: 2,\n\t\tPlacement: &nats.Placement{Cluster: \"C3\"},\n\t\tSources: sourceForCluster(\"C1\"),\n\t\tMaxAge: time.Minute,\n\t\tMaxMsgs: 20_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now create mirrors for our mux'd streams.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"C1-MIRROR\",\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: \"C1\"},\n\t\tMirror: &nats.StreamSource{Name: \"C3-S-MUX\"},\n\t\tMaxAge: 5 * time.Minute,\n\t\tMaxMsgs: 10_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"C2-MIRROR\",\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: \"C2\"},\n\t\tMirror: &nats.StreamSource{Name: \"C2-S-MUX\"},\n\t\tMaxAge: 5 * time.Minute,\n\t\tMaxMsgs: 10_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"C3-MIRROR\",\n\t\tReplicas: 3,\n\t\tPlacement: &nats.Placement{Cluster: \"C3\"},\n\t\tMirror: &nats.StreamSource{Name: \"C1-S-MUX\"},\n\t\tMaxAge: 5 * time.Minute,\n\t\tMaxMsgs: 10_000,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tvar jsc []nats.JetStream\n\n\t// Create 64 clients.\n\tfor i := 0; i < 64; i++ {\n\t\ts := sc.randomCluster().randomServer()\n\t\tnc, _ := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tjs, err := nc.JetStream(nats.PublishAsyncMaxPending(8 * 1024))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tjsc = append(jsc, js)\n\t}\n\n\tmsg := make([]byte, 1024)\n\tcrand.Read(msg)\n\n\t// 10 minutes\n\texpires := time.Now().Add(480 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfor _, sns := range scm {\n\t\t\trand.Shuffle(len(sns), func(i, j int) { sns[i], sns[j] = sns[j], sns[i] })\n\t\t\tfor _, sn := range sns {\n\t\t\t\tjs := jsc[rand.Intn(len(jsc))]\n\t\t\t\tif _, err = js.PublishAsync(sn, msg); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(10 * time.Millisecond)\n\t}\n}\n\nfunc TestNoRaceJetStreamSlowFilteredInitialPendingAndFirstMsg(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Create directly here to force multiple blocks, etc.\n\ta, err := s.LookupAccount(\"$G\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmset, err := a.addStreamWithStore(\n\t\t&StreamConfig{\n\t\t\tName: \"S\",\n\t\t\tSubjects: []string{\"foo\", \"bar\", \"baz\", \"foo.bar.baz\", \"foo.*\"},\n\t\t},\n\t\t&FileStoreConfig{\n\t\t\tBlockSize: 4 * 1024 * 1024,\n\t\t\tAsyncFlush: true,\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 100_000 // 500k total though.\n\n\t// Messages will be 'foo' 'bar' 'baz' repeated 100k times.\n\t// Then 'foo.bar.baz' all contigous for 100k.\n\t// Then foo.N for 1-100000\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"HELLO\"))\n\t\tjs.PublishAsync(\"bar\", []byte(\"WORLD\"))\n\t\tjs.PublishAsync(\"baz\", []byte(\"AGAIN\"))\n\t}\n\t// Make contiguous block of same subject.\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo.bar.baz\", []byte(\"ALL-TOGETHER\"))\n\t}\n\t// Now add some more at the end.\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"foo.%d\", i+1), []byte(\"LATER\"))\n\t}\n\n\tcheckFor(t, 10*time.Second, 250*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.State.Msgs != uint64(5*toSend) {\n\t\t\treturn fmt.Errorf(\"Expected %d msgs, got %d\", 5*toSend, si.State.Msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Threshold for taking too long.\n\tconst thresh = 150 * time.Millisecond\n\n\tvar dindex int\n\ttestConsumerCreate := func(subj string, startSeq, expectedNumPending uint64) {\n\t\tt.Helper()\n\t\tdindex++\n\t\tdname := fmt.Sprintf(\"dur-%d\", dindex)\n\t\tcfg := ConsumerConfig{FilterSubject: subj, Durable: dname, AckPolicy: AckExplicit}\n\t\tif startSeq > 1 {\n\t\t\tcfg.OptStartSeq, cfg.DeliverPolicy = startSeq, DeliverByStartSequence\n\t\t}\n\t\tstart := time.Now()\n\t\to, err := mset.addConsumer(&cfg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif delta := time.Since(start); delta > thresh {\n\t\t\tt.Fatalf(\"Creating consumer for %q and start: %d took too long: %v\", subj, startSeq, delta)\n\t\t}\n\t\tif ci := o.info(); ci.NumPending != expectedNumPending {\n\t\t\tt.Fatalf(\"Expected NumPending of %d, got %d\", expectedNumPending, ci.NumPending)\n\t\t}\n\t}\n\n\ttestConsumerCreate(\"foo.100000\", 1, 1)\n\ttestConsumerCreate(\"foo.100000\", 222_000, 1)\n\ttestConsumerCreate(\"foo\", 1, 100_000)\n\ttestConsumerCreate(\"foo\", 4, 100_000-1)\n\ttestConsumerCreate(\"foo.bar.baz\", 1, 100_000)\n\ttestConsumerCreate(\"foo.bar.baz\", 350_001, 50_000)\n\ttestConsumerCreate(\"*\", 1, 300_000)\n\ttestConsumerCreate(\"*\", 4, 300_000-3)\n\ttestConsumerCreate(\">\", 1, 500_000)\n\ttestConsumerCreate(\">\", 50_000, 500_000-50_000+1)\n\ttestConsumerCreate(\"foo.10\", 1, 1)\n\n\t// Also test that we do not take long if the start sequence is later in the stream.\n\tsub, err := js.PullSubscribe(\"foo.100000\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tstart := time.Now()\n\tfetchMsgs(t, sub, 1, time.Second)\n\tif delta := time.Since(start); delta > thresh {\n\t\tt.Fatalf(\"Took too long for pull subscriber to fetch the message: %v\", delta)\n\t}\n\n\t// Now do some deletes and make sure these are handled correctly.\n\t// Delete 3 foo messages.\n\t_, err = mset.removeMsg(1)\n\trequire_NoError(t, err)\n\t_, err = mset.removeMsg(4)\n\trequire_NoError(t, err)\n\t_, err = mset.removeMsg(7)\n\trequire_NoError(t, err)\n\ttestConsumerCreate(\"foo\", 1, 100_000-3)\n\n\t// Make sure wider scoped subjects do the right thing from a pending perspective.\n\to, err := mset.addConsumer(&ConsumerConfig{FilterSubject: \">\", Durable: \"cat\", AckPolicy: AckExplicit})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tci, expected := o.info(), uint64(500_000-3)\n\tif ci.NumPending != expected {\n\t\tt.Fatalf(\"Expected NumPending of %d, got %d\", expected, ci.NumPending)\n\t}\n\t// Send another and make sure its captured by our wide scope consumer.\n\t_, err = js.Publish(\"foo\", []byte(\"HELLO AGAIN\"))\n\trequire_NoError(t, err)\n\t// Due to consumer signaling it might not immediately be reflected.\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tif ci = o.info(); ci.NumPending != expected+1 {\n\t\t\treturn fmt.Errorf(\"Expected the consumer to recognize the wide scoped consumer, wanted pending of %d, got %d\", expected+1, ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Stop current server and test restart..\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\t// Restart.\n\ts = RunJetStreamServerOnPort(-1, sd)\n\tdefer s.Shutdown()\n\n\ta, err = s.LookupAccount(\"$G\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tmset, err = a.lookupStream(\"S\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Make sure we recovered our per subject state on restart.\n\ttestConsumerCreate(\"foo.100000\", 1, 1)\n\ttestConsumerCreate(\"foo\", 1, 100_000-2)\n}\n\nfunc TestNoRaceJetStreamFileStoreBufferReuse(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine.\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tcfg := &StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\", \"bar\", \"baz\"}, Storage: FileStorage}\n\tif _, err := s.GlobalAccount().addStreamWithStore(cfg, nil); err != nil {\n\t\tt.Fatalf(\"Unexpected error adding stream: %v\", err)\n\t}\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 200_000\n\n\tm := nats.NewMsg(\"foo\")\n\tm.Data = make([]byte, 8*1024)\n\tcrand.Read(m.Data)\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tm.Reply = _EMPTY_\n\t\tswitch i % 3 {\n\t\tcase 0:\n\t\t\tm.Subject = \"foo\"\n\t\tcase 1:\n\t\t\tm.Subject = \"bar\"\n\t\tcase 2:\n\t\t\tm.Subject = \"baz\"\n\t\t}\n\t\tm.Header.Set(\"X-ID2\", fmt.Sprintf(\"XXXXX-%d\", i))\n\t\tif _, err := js.PublishMsgAsync(m); err != nil {\n\t\t\tt.Fatalf(\"Err on publish: %v\", err)\n\t\t}\n\t}\n\t<-js.PublishAsyncComplete()\n\tfmt.Printf(\"TOOK %v to publish\\n\", time.Since(start))\n\n\tv, err := s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tfmt.Printf(\"MEM AFTER PUBLISH is %v\\n\", friendlyBytes(v.Mem))\n\n\tsi, _ := js.StreamInfo(\"TEST\")\n\tfmt.Printf(\"si is %+v\\n\", si.State)\n\n\treceived := 0\n\tdone := make(chan bool)\n\n\tcb := func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= toSend {\n\t\t\tdone <- true\n\t\t}\n\t}\n\n\tstart = time.Now()\n\tsub, err := js.Subscribe(\"*\", cb, nats.EnableFlowControl(), nats.IdleHeartbeat(time.Second), nats.AckNone())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\t<-done\n\tfmt.Printf(\"TOOK %v to consume\\n\", time.Since(start))\n\n\tv, err = s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tfmt.Printf(\"MEM AFTER SUBSCRIBE is %v\\n\", friendlyBytes(v.Mem))\n}\n\n// Report of slow restart for a server that has many messages that have expired while it was not running.\nfunc TestNoRaceJetStreamSlowRestartWithManyExpiredMsgs(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.JetStream = true\n\ts := RunServer(&opts)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tttl := 2 * time.Second\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS\",\n\t\tSubjects: []string{\"orders.*\"},\n\t\tMaxAge: ttl,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Attach a consumer who is filtering on a wildcard subject as well.\n\t// This does not affect it like I thought originally but will keep it here.\n\t_, err = js.AddConsumer(\"ORDERS\", &nats.ConsumerConfig{\n\t\tDurable: \"c22\",\n\t\tFilterSubject: \"orders.*\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Now fill up with messages.\n\ttoSend := 100_000\n\tfor i := 1; i <= toSend; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"orders.%d\", i), []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\n\tsdir := strings.TrimSuffix(s.JetStreamConfig().StoreDir, JetStreamStoreDir)\n\ts.Shutdown()\n\n\t// Let them expire while not running.\n\ttime.Sleep(ttl + 500*time.Millisecond)\n\n\tstart := time.Now()\n\topts.Port = -1\n\topts.StoreDir = sdir\n\ts = RunServer(&opts)\n\telapsed := time.Since(start)\n\tdefer s.Shutdown()\n\n\tif elapsed > 2*time.Second {\n\t\tt.Fatalf(\"Took %v for restart which is too long\", elapsed)\n\t}\n\n\t// Check everything is correct.\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"ORDERS\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif si.State.Msgs != 0 {\n\t\tt.Fatalf(\"Expected no msgs after restart, got %d\", si.State.Msgs)\n\t}\n}\n\nfunc TestNoRaceJetStreamStalledMirrorsAfterExpire(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 1,\n\t\tMaxAge: 100 * time.Millisecond,\n\t}\n\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Error creating stream: %v\", err)\n\t}\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 2,\n\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendBatch := func(batch int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < batch; i++ {\n\t\t\tjs.PublishAsync(\"foo.bar\", []byte(\"Hello\"))\n\t\t}\n\t\tselect {\n\t\tcase <-js.PublishAsyncComplete():\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t}\n\t}\n\n\tnumMsgs := 10_000\n\tsendBatch(numMsgs)\n\n\t// Turn off expiration so we can test we did not stall.\n\tcfg.MaxAge = 0\n\tif _, err := js.UpdateStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsendBatch(numMsgs)\n\n\t// Wait for mirror to be caught up.\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"M\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.LastSeq != uint64(2*numMsgs) {\n\t\t\treturn fmt.Errorf(\"Expected %d as last sequence, got state: %+v\", 2*numMsgs, si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// We will use JetStream helpers to create supercluster but this test is about exposing the ability to access\n// account scoped connz with subject interest filtering.\nfunc TestNoRaceJetStreamSuperClusterAccountConnz(t *testing.T) {\n\t// This has 4 different account, 3 general and system.\n\tsc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Create 20 connections on account one and two\n\t// Create JetStream assets for each as well to make sure by default we do not report them.\n\tnum := 20\n\tfor i := 0; i < num; i++ {\n\t\tnc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo(\"one\", \"p\"), nats.Name(\"one\"))\n\t\tdefer nc.Close()\n\n\t\tif i%2 == 0 {\n\t\t\tnc.SubscribeSync(\"foo\")\n\t\t} else {\n\t\t\tnc.SubscribeSync(\"bar\")\n\t\t}\n\n\t\tnc, js := jsClientConnect(t, sc.randomServer(), nats.UserInfo(\"two\", \"p\"), nats.Name(\"two\"))\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(\"baz\")\n\t\tnc.SubscribeSync(\"foo.bar.*\")\n\t\tnc.SubscribeSync(fmt.Sprintf(\"id.%d\", i+1))\n\n\t\tjs.AddStream(&nats.StreamConfig{Name: fmt.Sprintf(\"TEST:%d\", i+1)})\n\t}\n\n\ttype czapi struct {\n\t\tServer *ServerInfo\n\t\tData *Connz\n\t\tError *ApiError\n\t}\n\n\tparseConnz := func(buf []byte) *Connz {\n\t\tt.Helper()\n\t\tvar cz czapi\n\t\tif err := json.Unmarshal(buf, &cz); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cz.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t\t}\n\t\treturn cz.Data\n\t}\n\n\tdoRequest := func(reqSubj, acc, filter string, expected int) {\n\t\tt.Helper()\n\t\tnc, _ := jsClientConnect(t, sc.randomServer(), nats.UserInfo(acc, \"p\"), nats.Name(acc))\n\t\tdefer nc.Close()\n\n\t\tmch := make(chan *nats.Msg, 9)\n\t\tsub, _ := nc.ChanSubscribe(nats.NewInbox(), mch)\n\n\t\tvar req []byte\n\t\tif filter != _EMPTY_ {\n\t\t\treq, _ = json.Marshal(&ConnzOptions{FilterSubject: filter})\n\t\t}\n\n\t\tif err := nc.PublishRequest(reqSubj, sub.Subject, req); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\t// So we can igniore ourtselves.\n\t\tcid, _ := nc.GetClientID()\n\t\tsid := nc.ConnectedServerId()\n\n\t\twt := time.NewTimer(200 * time.Millisecond)\n\t\tvar conns []*ConnInfo\n\tLOOP:\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase m := <-mch:\n\t\t\t\tif len(m.Data) == 0 {\n\t\t\t\t\tt.Fatalf(\"No responders\")\n\t\t\t\t}\n\t\t\t\tcr := parseConnz(m.Data)\n\t\t\t\t// For account scoped, NumConns and Total should be the same (sans limits and offsets).\n\t\t\t\t// It Total should not include other accounts since that would leak information about the system.\n\t\t\t\tif filter == _EMPTY_ && cr.NumConns != cr.Total {\n\t\t\t\t\tt.Fatalf(\"NumConns and Total should be same with account scoped connz, got %+v\", cr)\n\t\t\t\t}\n\t\t\t\tfor _, c := range cr.Conns {\n\t\t\t\t\tif c.Name != acc {\n\t\t\t\t\t\tt.Fatalf(\"Got wrong account: %q vs %q for %+v\", acc, c.Account, c)\n\t\t\t\t\t}\n\t\t\t\t\tif !(c.Cid == cid && cr.ID == sid) {\n\t\t\t\t\t\tconns = append(conns, c)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twt.Reset(200 * time.Millisecond)\n\t\t\tcase <-wt.C:\n\t\t\t\tbreak LOOP\n\t\t\t}\n\t\t}\n\t\tif len(conns) != expected {\n\t\t\tt.Fatalf(\"Expected to see %d conns but got %d\", expected, len(conns))\n\t\t}\n\t}\n\n\tdoSysRequest := func(acc string, expected int) {\n\t\tt.Helper()\n\t\tdoRequest(\"$SYS.REQ.SERVER.PING.CONNZ\", acc, _EMPTY_, expected)\n\t}\n\tdoAccRequest := func(acc string, expected int) {\n\t\tt.Helper()\n\t\tdoRequest(\"$SYS.REQ.ACCOUNT.PING.CONNZ\", acc, _EMPTY_, expected)\n\t}\n\tdoFiltered := func(acc, filter string, expected int) {\n\t\tt.Helper()\n\t\tdoRequest(\"$SYS.REQ.SERVER.PING.CONNZ\", acc, filter, expected)\n\t}\n\n\tdoSysRequest(\"one\", 20)\n\tdoAccRequest(\"one\", 20)\n\n\tdoSysRequest(\"two\", 20)\n\tdoAccRequest(\"two\", 20)\n\n\t// Now check filtering.\n\tdoFiltered(\"one\", _EMPTY_, 20)\n\tdoFiltered(\"one\", \">\", 20)\n\tdoFiltered(\"one\", \"bar\", 10)\n\tdoFiltered(\"two\", \"bar\", 0)\n\tdoFiltered(\"two\", \"id.1\", 1)\n\tdoFiltered(\"two\", \"id.*\", 20)\n\tdoFiltered(\"two\", \"foo.bar.*\", 20)\n\tdoFiltered(\"two\", \"foo.>\", 20)\n}\n\nfunc TestNoRaceCompressedConnz(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tdoRequest := func(compress string) {\n\t\tt.Helper()\n\t\tm := nats.NewMsg(\"$SYS.REQ.ACCOUNT.PING.CONNZ\")\n\t\tm.Header.Add(\"Accept-Encoding\", compress)\n\t\tresp, err := nc.RequestMsg(m, time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tbuf := resp.Data\n\n\t\t// Make sure we have an encoding header.\n\t\tce := resp.Header.Get(\"Content-Encoding\")\n\t\tswitch strings.ToLower(ce) {\n\t\tcase \"gzip\":\n\t\t\tzr, err := gzip.NewReader(bytes.NewReader(buf))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tdefer zr.Close()\n\t\t\tbuf, err = io.ReadAll(zr)\n\t\t\tif err != nil && err != io.ErrUnexpectedEOF {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\tcase \"snappy\", \"s2\":\n\t\t\tsr := s2.NewReader(bytes.NewReader(buf))\n\t\t\tbuf, err = io.ReadAll(sr)\n\t\t\tif err != nil && err != io.ErrUnexpectedEOF {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unknown content-encoding of %q\", ce)\n\t\t}\n\n\t\tvar cz ServerAPIConnzResponse\n\t\tif err := json.Unmarshal(buf, &cz); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif cz.Error != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %+v\", cz.Error)\n\t\t}\n\t}\n\n\tdoRequest(\"gzip\")\n\tdoRequest(\"snappy\")\n\tdoRequest(\"s2\")\n}\n\nfunc TestNoRaceJetStreamClusterExtendedStreamPurge(t *testing.T) {\n\tfor _, st := range []StorageType{FileStorage, MemoryStorage} {\n\t\tt.Run(st.String(), func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcfg := StreamConfig{\n\t\t\t\tName: \"KV\",\n\t\t\t\tSubjects: []string{\"kv.>\"},\n\t\t\t\tStorage: st,\n\t\t\t\tReplicas: 2,\n\t\t\t\tMaxMsgsPer: 100,\n\t\t\t}\n\t\t\treq, err := json.Marshal(cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tc.waitOnStreamLeader(\"$G\", \"KV\")\n\n\t\t\tsi, err := js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si == nil || si.Config.Name != \"KV\" {\n\t\t\t\tt.Fatalf(\"StreamInfo is not correct %+v\", si)\n\t\t\t}\n\n\t\t\tfor i := 0; i < 1000; i++ {\n\t\t\t\tjs.PublishAsync(\"kv.foo\", []byte(\"OK\")) // 1 * i\n\t\t\t\tjs.PublishAsync(\"kv.bar\", []byte(\"OK\")) // 2 * i\n\t\t\t\tjs.PublishAsync(\"kv.baz\", []byte(\"OK\")) // 3 * i\n\t\t\t}\n\t\t\t// First is 2700, last is 3000\n\t\t\tfor i := 0; i < 700; i++ {\n\t\t\t\tjs.PublishAsync(fmt.Sprintf(\"kv.%d\", i+1), []byte(\"OK\"))\n\t\t\t}\n\t\t\t// Now first is 2700, last is 3700\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(10 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\tsi, err = js.StreamInfo(\"KV\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != 1000 {\n\t\t\t\tt.Fatalf(\"Expected %d msgs, got %d\", 1000, si.State.Msgs)\n\t\t\t}\n\n\t\t\tshouldFail := func(preq *JSApiStreamPurgeRequest) {\n\t\t\t\treq, _ := json.Marshal(preq)\n\t\t\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"KV\"), req, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tvar pResp JSApiStreamPurgeResponse\n\t\t\t\tif err = json.Unmarshal(resp.Data, &pResp); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif pResp.Success || pResp.Error == nil {\n\t\t\t\t\tt.Fatalf(\"Expected an error response but got none\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Sequence and Keep should be mutually exclusive.\n\t\t\tshouldFail(&JSApiStreamPurgeRequest{Sequence: 10, Keep: 10})\n\n\t\t\tpurge := func(preq *JSApiStreamPurgeRequest, newTotal uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\treq, _ := json.Marshal(preq)\n\t\t\t\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamPurgeT, \"KV\"), req, time.Second)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tvar pResp JSApiStreamPurgeResponse\n\t\t\t\tif err = json.Unmarshal(resp.Data, &pResp); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !pResp.Success || pResp.Error != nil {\n\t\t\t\t\tt.Fatalf(\"Got a bad response %+v\", pResp)\n\t\t\t\t}\n\t\t\t\tsi, err = js.StreamInfo(\"KV\")\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != newTotal {\n\t\t\t\t\tt.Fatalf(\"Expected total after purge to be %d but got %d\", newTotal, si.State.Msgs)\n\t\t\t\t}\n\t\t\t}\n\t\t\texpectLeft := func(subject string, expected uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tci, err := js.AddConsumer(\"KV\", &nats.ConsumerConfig{Durable: \"dlc\", FilterSubject: subject, AckPolicy: nats.AckExplicitPolicy})\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tdefer js.DeleteConsumer(\"KV\", \"dlc\")\n\t\t\t\tif ci.NumPending != expected {\n\t\t\t\t\tt.Fatalf(\"Expected %d remaining but got %d\", expected, ci.NumPending)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.foo\"}, 900)\n\t\t\texpectLeft(\"kv.foo\", 0)\n\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.bar\", Keep: 1}, 801)\n\t\t\texpectLeft(\"kv.bar\", 1)\n\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.baz\", Sequence: 2851}, 751)\n\t\t\texpectLeft(\"kv.baz\", 50)\n\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.*\"}, 0)\n\n\t\t\t// RESET\n\t\t\tjs.DeleteStream(\"KV\")\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tc.waitOnStreamLeader(\"$G\", \"KV\")\n\n\t\t\tif _, err := js.StreamInfo(\"KV\"); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Put in 100.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.PublishAsync(\"kv.foo\", []byte(\"OK\"))\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.foo\", Keep: 10}, 10)\n\t\t\tpurge(&JSApiStreamPurgeRequest{Subject: \"kv.foo\", Keep: 10}, 10)\n\t\t\texpectLeft(\"kv.foo\", 10)\n\n\t\t\t// RESET AGAIN\n\t\t\tjs.DeleteStream(\"KV\")\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tc.waitOnStreamLeader(\"$G\", \"KV\")\n\n\t\t\tif _, err := js.StreamInfo(\"KV\"); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Put in 100.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.Publish(\"kv.foo\", []byte(\"OK\"))\n\t\t\t}\n\t\t\tpurge(&JSApiStreamPurgeRequest{Keep: 10}, 10)\n\t\t\texpectLeft(\">\", 10)\n\n\t\t\t// RESET AGAIN\n\t\t\tjs.DeleteStream(\"KV\")\n\t\t\t// Do manually for now.\n\t\t\tnc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), req, time.Second)\n\t\t\tif _, err := js.StreamInfo(\"KV\"); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\t// Put in 100.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tjs.Publish(\"kv.foo\", []byte(\"OK\"))\n\t\t\t}\n\t\t\tpurge(&JSApiStreamPurgeRequest{Sequence: 90}, 11) // Up to 90 so we keep that, hence the 11.\n\t\t\texpectLeft(\">\", 11)\n\t\t})\n\t}\n}\n\nfunc TestNoRaceJetStreamFileStoreCompaction(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"KV\",\n\t\tSubjects: []string{\"KV.>\"},\n\t\tMaxMsgsPerSubject: 1,\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttoSend := 10_000\n\tdata := make([]byte, 4*1024)\n\tcrand.Read(data)\n\n\t// First one.\n\tjs.PublishAsync(\"KV.FM\", data)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"KV.%d\", i+1), data)\n\t}\n\t// Do again and overwrite the previous batch.\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"KV.%d\", i+1), data)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Now check by hand the utilization level.\n\tmset, err := s.GlobalAccount().lookupStream(\"KV\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\ttotal, used, _ := mset.Store().Utilization()\n\tif pu := 100.0 * float32(used) / float32(total); pu < 80.0 {\n\t\tt.Fatalf(\"Utilization is less than 80%%, got %.2f\", pu)\n\t}\n}\n\nfunc TestNoRaceJetStreamEncryptionEnabledOnRestartWithExpire(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tconfig := s.JetStreamConfig()\n\tif config == nil {\n\t\tt.Fatalf(\"Expected config but got none\")\n\t}\n\tdefer removeDir(t, config.StoreDir)\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 10_000\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tMaxMsgs: int64(toSend),\n\t}\n\tif _, err := js.AddStream(cfg); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tdata := make([]byte, 4*1024) // 4K payload\n\tcrand.Read(data)\n\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", data)\n\t\tjs.PublishAsync(\"bar\", data)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dlc\", AckPolicy: nats.AckExplicitPolicy})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Restart\n\tnc.Close()\n\ts.Shutdown()\n\n\tncs := fmt.Sprintf(\"\\nlisten: 127.0.0.1:-1\\njetstream: {key: %q, store_dir: %q}\\n\", \"s3cr3t!\", config.StoreDir)\n\tconf = createConfFile(t, []byte(ncs))\n\n\t// Try to drain entropy to see if effects startup time.\n\tdrain := make([]byte, 32*1024*1024) // Pull 32Mb of crypto rand.\n\tcrand.Read(drain)\n\n\tstart := time.Now()\n\ts, _ = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tdd := time.Since(start)\n\tif dd > 5*time.Second {\n\t\tt.Fatalf(\"Restart took longer than expected: %v\", dd)\n\t}\n}\n\n// This test was from Ivan K. and showed a bug in the filestore implementation.\n// This is skipped by default since it takes >40s to run.\nfunc TestNoRaceJetStreamOrderedConsumerMissingMsg(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"benchstream\",\n\t\tSubjects: []string{\"testsubject\"},\n\t\tReplicas: 1,\n\t}); err != nil {\n\t\tt.Fatalf(\"add stream failed: %s\", err)\n\t}\n\n\ttotal := 1_000_000\n\n\tnumSubs := 10\n\tch := make(chan struct{}, numSubs)\n\twg := sync.WaitGroup{}\n\twg.Add(numSubs)\n\terrCh := make(chan error, 1)\n\tfor i := 0; i < numSubs; i++ {\n\t\tnc, js := jsClientConnect(t, s)\n\t\tdefer nc.Close()\n\t\tgo func(nc *nats.Conn, js nats.JetStreamContext) {\n\t\t\tdefer wg.Done()\n\t\t\treceived := 0\n\t\t\t_, err := js.Subscribe(\"testsubject\", func(m *nats.Msg) {\n\t\t\t\tmeta, _ := m.Metadata()\n\t\t\t\tif meta.Sequence.Consumer != meta.Sequence.Stream {\n\t\t\t\t\tnc.Close()\n\t\t\t\t\terrCh <- fmt.Errorf(\"Bad meta: %+v\", meta)\n\t\t\t\t}\n\t\t\t\treceived++\n\t\t\t\tif received == total {\n\t\t\t\t\tch <- struct{}{}\n\t\t\t\t}\n\t\t\t}, nats.OrderedConsumer())\n\t\t\tif err != nil {\n\t\t\t\tselect {\n\t\t\t\tcase errCh <- fmt.Errorf(\"Error creating sub: %v\", err):\n\t\t\t\tdefault:\n\t\t\t\t}\n\n\t\t\t}\n\t\t}(nc, js)\n\t}\n\twg.Wait()\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatal(e)\n\tdefault:\n\t}\n\n\tpayload := make([]byte, 500)\n\tfor i := 1; i <= total; i++ {\n\t\tjs.PublishAsync(\"testsubject\", payload)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not send all messages\")\n\t}\n\n\t// Now wait for consumers to be done:\n\tfor i := 0; i < numSubs; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(10 * time.Second):\n\t\t\tt.Fatal(\"Did not receive all messages for all consumers in time\")\n\t\t}\n\t}\n\n}\n\n// Issue #2488 - Bad accounting, can not reproduce the stalled consumers after last several PRs.\n// Issue did show bug in ack logic for no-ack and interest based retention.\nfunc TestNoRaceJetStreamClusterInterestPolicyAckNone(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdurable string\n\t}{\n\t\t{\"durable\", \"dlc\"},\n\t\t{\"ephemeral\", _EMPTY_},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\t// Client based API\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: \"cluster\",\n\t\t\t\tSubjects: []string{\"cluster.*\"},\n\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t\tDiscard: nats.DiscardOld,\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tvar received uint32\n\t\t\tmh := func(m *nats.Msg) {\n\t\t\t\tatomic.AddUint32(&received, 1)\n\t\t\t}\n\n\t\t\topts := []nats.SubOpt{nats.DeliverNew(), nats.AckNone()}\n\t\t\tif test.durable != _EMPTY_ {\n\t\t\t\topts = append(opts, nats.Durable(test.durable))\n\t\t\t}\n\t\t\t_, err = js.Subscribe(\"cluster.created\", mh, opts...)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\tmsg := []byte(\"ACK ME\")\n\t\t\tconst total = uint32(1_000)\n\t\t\tfor i := 0; i < int(total); i++ {\n\t\t\t\tif _, err := js.Publish(\"cluster.created\", msg); err != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\t//time.Sleep(100 * time.Microsecond)\n\t\t\t}\n\n\t\t\t// Wait for all messages to be received.\n\t\t\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tr := atomic.LoadUint32(&received)\n\t\t\t\tif r == total {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"Received only %d out of %d\", r, total)\n\t\t\t})\n\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tsi, err := js.StreamInfo(\"cluster\")\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error getting stream info: %v\", err)\n\t\t\t\t}\n\t\t\t\tif si.State.Msgs != 0 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected no messages, got %d\", si.State.Msgs)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\n// There was a bug in the filestore compact code that would cause a store\n// with JSExpectedLastSubjSeq to fail with \"wrong last sequence: 0\"\nfunc TestNoRaceJetStreamLastSubjSeqAndFilestoreCompact(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"MQTT_sess\",\n\t\tSubjects: []string{\"MQTT.sess.>\"},\n\t\tStorage: nats.FileStorage,\n\t\tRetention: nats.LimitsPolicy,\n\t\tReplicas: 1,\n\t\tMaxMsgsPerSubject: 1,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfirstPayload := make([]byte, 40)\n\tsecondPayload := make([]byte, 380)\n\tfor iter := 0; iter < 2; iter++ {\n\t\tfor i := 0; i < 4000; i++ {\n\t\t\tsubj := \"MQTT.sess.\" + getHash(fmt.Sprintf(\"client_%d\", i))\n\t\t\tpa, err := js.Publish(subj, firstPayload)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t\t}\n\t\t\tm := nats.NewMsg(subj)\n\t\t\tm.Data = secondPayload\n\t\t\teseq := strconv.FormatInt(int64(pa.Sequence), 10)\n\t\t\tm.Header.Set(JSExpectedLastSubjSeq, eseq)\n\t\t\tif _, err := js.PublishMsg(m); err != nil {\n\t\t\t\tt.Fatalf(\"Error on publish (iter=%v seq=%v): %v\", iter+1, pa.Sequence, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Issue #2548\nfunc TestNoRaceJetStreamClusterMemoryStreamConsumerRaftGrowth(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"memory-leak\",\n\t\tSubjects: []string{\"memory-leak\"},\n\t\tRetention: nats.LimitsPolicy,\n\t\tMaxMsgs: 1000,\n\t\tDiscard: nats.DiscardOld,\n\t\tMaxAge: time.Minute,\n\t\tStorage: nats.MemoryStorage,\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t_, err = js.QueueSubscribe(\"memory-leak\", \"q1\", func(msg *nats.Msg) {\n\t\ttime.Sleep(1 * time.Second)\n\t\tmsg.AckSync()\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Send 10k (Must be > 8192 which is compactNumMin from monitorConsumer.\n\tmsg := []byte(\"NATS is a connective technology that powers modern distributed systems.\")\n\tfor i := 0; i < 10_000; i++ {\n\t\tif _, err := js.Publish(\"memory-leak\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// We will verify here that the underlying raft layer for the leader is not > 8192\n\tcl := c.consumerLeader(\"$G\", \"memory-leak\", \"q1\")\n\tmset, err := cl.GlobalAccount().lookupStream(\"memory-leak\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\to := mset.lookupConsumer(\"q1\")\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", \"q1\")\n\t}\n\tnode := o.raftNode().(*raft)\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tif ms := node.wal.(*memStore); ms.State().Msgs > 8192 {\n\t\t\treturn fmt.Errorf(\"Did not compact the raft memory WAL\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterCorruptWAL(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsub, err := js.PullSubscribe(\"foo\", \"dlc\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnumMsgs := 1000\n\tfor i := 0; i < numMsgs; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"WAL\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tfor i, m := range fetchMsgs(t, sub, 200, 5*time.Second) {\n\t\t// Ack first 50 and every other even on after that..\n\t\tif i < 50 || i%2 == 1 {\n\t\t\tm.AckSync()\n\t\t}\n\t}\n\t// Make sure acks processed.\n\ttime.Sleep(200 * time.Millisecond)\n\tnc.Close()\n\n\t// Check consumer consistency.\n\tcheckConsumerWith := func(delivered, ackFloor uint64, ackPending int) {\n\t\tt.Helper()\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif ci.Delivered.Consumer != ci.Delivered.Stream || ci.Delivered.Consumer != delivered {\n\t\t\t\treturn fmt.Errorf(\"Expected %d for delivered, got %+v\", delivered, ci.Delivered)\n\t\t\t}\n\t\t\tif ci.AckFloor.Consumer != ci.AckFloor.Stream || ci.AckFloor.Consumer != ackFloor {\n\t\t\t\treturn fmt.Errorf(\"Expected %d for ack floor, got %+v\", ackFloor, ci.AckFloor)\n\t\t\t}\n\t\t\tnm := uint64(numMsgs)\n\t\t\tif ci.NumPending != nm-delivered {\n\t\t\t\treturn fmt.Errorf(\"Expected num pending to be %d, got %d\", nm-delivered, ci.NumPending)\n\t\t\t}\n\t\t\tif ci.NumAckPending != ackPending {\n\t\t\t\treturn fmt.Errorf(\"Expected num ack pending to be %d, got %d\", ackPending, ci.NumAckPending)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckConsumer := func() {\n\t\tt.Helper()\n\t\tcheckConsumerWith(200, 50, 75)\n\t}\n\n\tcheckConsumer()\n\n\t// Grab the consumer leader.\n\tcl := c.consumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tmset, err := cl.GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\to := mset.lookupConsumer(\"dlc\")\n\tif o == nil {\n\t\tt.Fatalf(\"Error looking up consumer %q\", \"dlc\")\n\t}\n\t// Grab underlying raft node and the WAL (filestore) and we will attempt to \"corrupt\" it.\n\tnode := o.raftNode().(*raft)\n\t// We are doing a stop here to prevent the internal consumer snapshot from happening on exit\n\tnode.Stop()\n\tfs := node.wal.(*fileStore)\n\tfcfg, cfg := fs.fcfg, fs.cfg.StreamConfig\n\t// Stop all the servers.\n\tc.stopAll()\n\n\t// Manipulate directly with cluster down.\n\tfs, err = newFileStore(fcfg, cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tstate := fs.State()\n\tsm, err := fs.LoadMsg(state.LastSeq, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tae, err := decodeAppendEntry(sm.msg, nil, _EMPTY_)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tdentry := func(dseq, sseq, dc uint64, ts int64) []byte {\n\t\tb := make([]byte, 4*binary.MaxVarintLen64+1)\n\t\tb[0] = byte(updateDeliveredOp)\n\t\tn := 1\n\t\tn += binary.PutUvarint(b[n:], dseq)\n\t\tn += binary.PutUvarint(b[n:], sseq)\n\t\tn += binary.PutUvarint(b[n:], dc)\n\t\tn += binary.PutVarint(b[n:], ts)\n\t\treturn b[:n]\n\t}\n\n\t// Let's put a non-contigous AppendEntry into the system.\n\tae.pindex += 10\n\t// Add in delivered record.\n\tae.entries = []*Entry{{EntryNormal, dentry(1000, 1000, 1, time.Now().UnixNano())}}\n\tencoded, err := ae.encode(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif _, _, err := fs.StoreMsg(_EMPTY_, nil, encoded, 0); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tfs.Stop()\n\n\tc.restartAllSamePorts()\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\n\tcheckConsumer()\n\n\t// Now we will truncate out the WAL out from underneath the leader.\n\t// Grab the consumer leader.\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcl = c.consumerLeader(\"$G\", \"TEST\", \"dlc\")\n\tmset, err = cl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to = mset.lookupConsumer(\"dlc\")\n\trequire_NoError(t, err)\n\n\t// Grab underlying raft node and the WAL (filestore) and truncate it.\n\t// This will simulate the WAL losing state due to truncate and we want to make sure it recovers.\n\n\tfs = o.raftNode().(*raft).wal.(*fileStore)\n\tstate = fs.State()\n\terr = fs.Truncate(state.FirstSeq)\n\trequire_NoError(t, err)\n\tstate = fs.State()\n\n\tsub, err = js.PullSubscribe(\"foo\", \"dlc\")\n\trequire_NoError(t, err)\n\n\t// This will cause us to stepdown and truncate our WAL.\n\tsub.Fetch(100)\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"dlc\")\n\t// We can't trust the results sans that we have a leader back in place and the ackFloor.\n\tci, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\trequire_NoError(t, err)\n\tif ci.AckFloor.Consumer != ci.AckFloor.Stream || ci.AckFloor.Consumer != 50 {\n\t\tt.Fatalf(\"Expected %d for ack floor, got %+v\", 50, ci.AckFloor)\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterInterestRetentionDeadlock(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// This can trigger deadlock with current architecture.\n\t// If stream is !limitsRetention and consumer is DIRECT and ack none we will try to place the msg seq\n\t// onto a chan for the stream to consider removing. All conditions above must hold to trigger.\n\n\t// We will attempt to trigger here with a stream mirror setup which uses and R=1 DIRECT consumer to replicate msgs.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S\", Retention: nats.InterestPolicy, Storage: nats.MemoryStorage})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Create a mirror which will create the consumer profile to trigger.\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"M\", Mirror: &nats.StreamSource{Name: \"S\"}})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\t// Queue up alot of messages.\n\tnumRequests := 20_000\n\tfor i := 0; i < numRequests; i++ {\n\t\tjs.PublishAsync(\"S\", []byte(\"Q\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"S\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif si.State.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected 0 msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterMaxConsumersAndDirect(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// We want to max sure max consumer limits do not affect mirrors or sources etc.\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"S\", Storage: nats.MemoryStorage, MaxConsumers: 1})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tvar mirrors []string\n\tfor i := 0; i < 10; i++ {\n\t\t// Create a mirror.\n\t\tmname := fmt.Sprintf(\"M-%d\", i+1)\n\t\tmirrors = append(mirrors, mname)\n\t\t_, err = js.AddStream(&nats.StreamConfig{Name: mname, Mirror: &nats.StreamSource{Name: \"S\"}})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Queue up messages.\n\tnumRequests := 20\n\tfor i := 0; i < numRequests; i++ {\n\t\tjs.Publish(\"S\", []byte(\"Q\"))\n\t}\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, mname := range mirrors {\n\t\t\tsi, err := js.StreamInfo(mname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif si.State.Msgs != uint64(numRequests) {\n\t\t\t\treturn fmt.Errorf(\"Expected %d msgs for %q, got state: %+v\", numRequests, mname, si.State)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Make sure when we try to hard reset a stream state in a cluster that we also re-create the consumers.\nfunc TestNoRaceJetStreamClusterStreamReset(t *testing.T) {\n\t// Speed up raft\n\tomin, omax, ohb := minElectionTimeout, maxElectionTimeout, hbInterval\n\tminElectionTimeout = 250 * time.Millisecond\n\tmaxElectionTimeout = time.Second\n\thbInterval = 50 * time.Millisecond\n\tdefer func() {\n\t\tminElectionTimeout = omin\n\t\tmaxElectionTimeout = omax\n\t\thbInterval = ohb\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 2,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tnumRequests := 20\n\tfor i := 0; i < numRequests; i++ {\n\t\tjs.Publish(\"foo.created\", []byte(\"REQ\"))\n\t}\n\n\t// Durable.\n\tsub, err := js.SubscribeSync(\"foo.created\", nats.Durable(\"d1\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.Msgs == uint64(numRequests))\n\n\t// Let settle a bit for Go routine checks.\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Grab number go routines.\n\tbase := runtime.NumGoroutine()\n\n\t// Make the consumer busy here by async sending a bunch of messages.\n\tfor i := 0; i < numRequests*10; i++ {\n\t\tjs.PublishAsync(\"foo.created\", []byte(\"REQ\"))\n\t}\n\n\t// Grab a server that is the consumer leader for the durable.\n\tcl := c.consumerLeader(\"$G\", \"TEST\", \"d1\")\n\tmset, err := cl.GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// Do a hard reset here by hand.\n\tmset.resetClusteredState(nil)\n\n\t// Wait til we have the consumer leader re-elected.\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"d1\")\n\n\t// Make sure we can get the consumer info eventually.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\t_, err := js.ConsumerInfo(\"TEST\", \"d1\", nats.MaxWait(250*time.Millisecond))\n\t\treturn err\n\t})\n\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tif after := runtime.NumGoroutine(); base > after {\n\t\t\treturn fmt.Errorf(\"Expected %d go routines, got %d\", base, after)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Simulate a low level write error on our consumer and make sure we can recover etc.\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tif cl = c.consumerLeader(\"$G\", \"TEST\", \"d1\"); cl != nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.New(\"waiting on consumer leader\")\n\t})\n\n\tmset, err = cl.GlobalAccount().lookupStream(\"TEST\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\to := mset.lookupConsumer(\"d1\")\n\tif o == nil {\n\t\tt.Fatalf(\"Did not retrieve consumer\")\n\t}\n\tnode := o.raftNode().(*raft)\n\tif node == nil {\n\t\tt.Fatalf(\"could not retrieve the raft node for consumer\")\n\t}\n\n\tnc.Close()\n\tnode.setWriteErr(io.ErrShortWrite)\n\n\tc.stopAll()\n\tc.restartAll()\n\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\tc.waitOnConsumerLeader(\"$G\", \"TEST\", \"d1\")\n}\n\n// Reports of high cpu on compaction for a KV store.\nfunc TestNoRaceJetStreamKeyValueCompaction(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"COMPACT\",\n\t\tReplicas: 3,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tvalue := strings.Repeat(\"A\", 128*1024)\n\tfor i := 0; i < 5_000; i++ {\n\t\tkey := fmt.Sprintf(\"K-%d\", rand.Intn(256)+1)\n\t\tif _, err := kv.PutString(key, value); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n}\n\n// Trying to recreate an issue rip saw with KV and server restarts complaining about\n// mismatch for a few minutes and growing memory.\nfunc TestNoRaceJetStreamClusterStreamSeqMismatchIssue(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"MM\",\n\t\tReplicas: 3,\n\t\tTTL: 500 * time.Millisecond,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 1; i <= 10; i++ {\n\t\tif _, err := kv.PutString(\"k\", \"1\"); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\t// Close in case we are connected here. Will recreate.\n\tnc.Close()\n\n\t// Shutdown a non-leader.\n\ts := c.randomNonStreamLeader(\"$G\", \"KV_MM\")\n\ts.Shutdown()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err = js.KeyValue(\"MM\")\n\trequire_NoError(t, err)\n\n\t// Now change the state of the stream such that we have to do a compact upon restart\n\t// of the downed server.\n\tfor i := 1; i <= 10; i++ {\n\t\tif _, err := kv.PutString(\"k\", \"2\"); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Raft could save us here so need to run a compact on the leader.\n\tsnapshotLeader := func() {\n\t\tsl := c.streamLeader(\"$G\", \"KV_MM\")\n\t\tif sl == nil {\n\t\t\tt.Fatalf(\"Did not get the leader\")\n\t\t}\n\t\tmset, err := sl.GlobalAccount().lookupStream(\"KV_MM\")\n\t\trequire_NoError(t, err)\n\t\tnode := mset.raftNode()\n\t\tif node == nil {\n\t\t\tt.Fatalf(\"Could not get stream group\")\n\t\t}\n\t\tif err := node.InstallSnapshot(mset.stateSnapshot(), false); err != nil {\n\t\t\tt.Fatalf(\"Error installing snapshot: %v\", err)\n\t\t}\n\t}\n\n\t// Now wait for expiration\n\ttime.Sleep(time.Second)\n\n\tsnapshotLeader()\n\n\ts = c.restartServer(s)\n\tc.waitOnServerCurrent(s)\n\n\t// We want to make sure we do not reset the raft state on a catchup due to no request yield.\n\t// Bug was if we did not actually request any help from snapshot we did not set mset.lseq properly.\n\t// So when we send next batch that would cause raft reset due to cluster reset for our stream.\n\tmset, err := s.GlobalAccount().lookupStream(\"KV_MM\")\n\trequire_NoError(t, err)\n\n\tfor i := 1; i <= 10; i++ {\n\t\tif _, err := kv.PutString(\"k1\", \"X\"); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tc.waitOnStreamCurrent(s, \"$G\", \"KV_MM\")\n\n\t// Make sure we did not reset our stream.\n\tmsetNew, err := s.GlobalAccount().lookupStream(\"KV_MM\")\n\trequire_NoError(t, err)\n\tif msetNew != mset {\n\t\tt.Fatalf(\"Stream was reset\")\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterStreamDropCLFS(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"CLFS\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Will work\n\t_, err = kv.Create(\"k.1\", []byte(\"X\"))\n\trequire_NoError(t, err)\n\t// Drive up CLFS state on leader.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = kv.Create(\"k.1\", []byte(\"X\"))\n\t\trequire_Error(t, err)\n\t}\n\t// Bookend with new key success.\n\t_, err = kv.Create(\"k.2\", []byte(\"Z\"))\n\trequire_NoError(t, err)\n\n\t// Close in case we are connected here. Will recreate.\n\tnc.Close()\n\n\t// Shutdown, which will also clear clfs.\n\ts := c.randomNonStreamLeader(\"$G\", \"KV_CLFS\")\n\ts.Shutdown()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tkv, err = js.KeyValue(\"CLFS\")\n\trequire_NoError(t, err)\n\n\t// Drive up CLFS state on leader.\n\tfor i := 0; i < 10; i++ {\n\t\t_, err = kv.Create(\"k.1\", []byte(\"X\"))\n\t\trequire_Error(t, err)\n\t}\n\n\tsl := c.streamLeader(\"$G\", \"KV_CLFS\")\n\tif sl == nil {\n\t\tt.Fatalf(\"Did not get the leader\")\n\t}\n\tmset, err := sl.GlobalAccount().lookupStream(\"KV_CLFS\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\tif node == nil {\n\t\tt.Fatalf(\"Could not get stream group\")\n\t}\n\tif err := node.InstallSnapshot(mset.stateSnapshot(), false); err != nil {\n\t\tt.Fatalf(\"Error installing snapshot: %v\", err)\n\t}\n\n\t_, err = kv.Create(\"k.3\", []byte(\"ZZZ\"))\n\trequire_NoError(t, err)\n\n\ts = c.restartServer(s)\n\tc.waitOnServerCurrent(s)\n\n\tmset, err = s.GlobalAccount().lookupStream(\"KV_CLFS\")\n\trequire_NoError(t, err)\n\n\t_, err = kv.Create(\"k.4\", []byte(\"YYY\"))\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamCurrent(s, \"$G\", \"KV_CLFS\")\n\n\t// Make sure we did not reset our stream.\n\tmsetNew, err := s.GlobalAccount().lookupStream(\"KV_CLFS\")\n\trequire_NoError(t, err)\n\tif msetNew != mset {\n\t\tt.Fatalf(\"Stream was reset\")\n\t}\n}\n\nfunc TestNoRaceJetStreamMemstoreWithLargeInteriorDeletes(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tMaxMsgsPerSubject: 1,\n\t\tStorage: nats.MemoryStorage,\n\t})\n\trequire_NoError(t, err)\n\n\tacc, err := s.lookupAccount(\"$G\")\n\trequire_NoError(t, err)\n\tmset, err := acc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tmsg := []byte(\"Hello World!\")\n\tif _, err := js.PublishAsync(\"foo\", msg); err != nil {\n\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t}\n\tfor i := 1; i <= 1_000_000; i++ {\n\t\tif _, err := js.PublishAsync(\"bar\", msg); err != nil {\n\t\t\tt.Fatalf(\"Unexpected publish error: %v\", err)\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tnow := time.Now()\n\tss := mset.stateWithDetail(true)\n\t// Before the fix the snapshot for this test would be > 200ms on my setup.\n\tif elapsed := time.Since(now); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Took too long to snapshot: %v\", elapsed)\n\t} else if elapsed > 50*time.Millisecond {\n\t\tt.Logf(\"WRN: Took longer than usual to snapshot: %v\", elapsed)\n\t}\n\n\tif ss.Msgs != 2 || ss.FirstSeq != 1 || ss.LastSeq != 1_000_001 || ss.NumDeleted != 999999 {\n\t\t// To not print out on error.\n\t\tss.Deleted = nil\n\t\tt.Fatalf(\"Bad State: %+v\", ss)\n\t}\n}\n\n// This is related to an issue reported where we were exhausting threads by trying to\n// cleanup too many consumers at the same time.\n// https://github.com/nats-io/nats-server/issues/2742\nfunc TestNoRaceJetStreamConsumerFileStoreConcurrentDiskIO(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\t// Artificially adjust our environment for this test.\n\tgmp := runtime.GOMAXPROCS(32)\n\tdefer runtime.GOMAXPROCS(gmp)\n\n\tmaxT := debug.SetMaxThreads(1050) // 1024 now\n\tdefer debug.SetMaxThreads(maxT)\n\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: \"MT\", Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tstartCh := make(chan bool)\n\tvar wg sync.WaitGroup\n\tvar swg sync.WaitGroup\n\n\tts := time.Now().UnixNano()\n\n\t// Create 1000 consumerStores\n\tn := 1000\n\tswg.Add(n)\n\n\tfor i := 1; i <= n; i++ {\n\t\tname := fmt.Sprintf(\"o%d\", i)\n\t\to, err := fs.ConsumerStore(name, time.Time{}, &ConsumerConfig{AckPolicy: AckExplicit})\n\t\trequire_NoError(t, err)\n\t\twg.Add(1)\n\t\tswg.Done()\n\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t// Will make everyone run concurrently.\n\t\t\t<-startCh\n\t\t\to.UpdateDelivered(22, 22, 1, ts)\n\t\t\tbuf, _ := o.(*consumerFileStore).encodeState()\n\t\t\to.(*consumerFileStore).writeState(buf)\n\t\t\to.Delete()\n\t\t}()\n\t}\n\n\tswg.Wait()\n\tclose(startCh)\n\twg.Wait()\n}\n\nfunc TestNoRaceJetStreamClusterHealthz(t *testing.T) {\n\tc := createJetStreamCluster(t, jsClusterAccountsTempl, \"HZ\", _EMPTY_, 3, 23033, true)\n\tdefer c.shutdown()\n\n\tnc1, js1 := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"one\", \"p\"))\n\tdefer nc1.Close()\n\n\tnc2, js2 := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"two\", \"p\"))\n\tdefer nc2.Close()\n\n\tvar err error\n\tfor _, sname := range []string{\"foo\", \"bar\", \"baz\"} {\n\t\t_, err = js1.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3})\n\t\trequire_NoError(t, err)\n\t\t_, err = js2.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3})\n\t\trequire_NoError(t, err)\n\t}\n\t// R1\n\t_, err = js1.AddStream(&nats.StreamConfig{Name: \"r1\", Replicas: 1})\n\trequire_NoError(t, err)\n\n\t// Now shutdown then send a bunch of data.\n\ts := c.servers[0]\n\ts.Shutdown()\n\n\tfor i := 0; i < 5_000; i++ {\n\t\t_, err = js1.PublishAsync(\"foo\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t\t_, err = js2.PublishAsync(\"bar\", []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js1.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\tselect {\n\tcase <-js2.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\ts = c.restartServer(s)\n\topts := s.getOpts()\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.HTTPPort = 11222\n\terr = s.StartMonitoring()\n\trequire_NoError(t, err)\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/healthz\", opts.HTTPPort)\n\n\tgetHealth := func() (int, *HealthStatus) {\n\t\tresp, err := http.Get(url)\n\t\trequire_NoError(t, err)\n\t\tdefer resp.Body.Close()\n\t\tbody, err := io.ReadAll(resp.Body)\n\t\trequire_NoError(t, err)\n\t\tvar hs HealthStatus\n\t\terr = json.Unmarshal(body, &hs)\n\t\trequire_NoError(t, err)\n\t\treturn resp.StatusCode, &hs\n\t}\n\n\terrors := 0\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tcode, hs := getHealth()\n\t\tif code >= 200 && code < 300 {\n\t\t\treturn nil\n\t\t}\n\t\terrors++\n\t\treturn fmt.Errorf(\"Got %d status with %+v\", code, hs)\n\t})\n\tif errors == 0 {\n\t\tt.Fatalf(\"Expected to have some errors until we became current, got none\")\n\t}\n}\n\n// Test that we can receive larger messages with stream subject details.\n// Also test that we will fail at some point and the user can fall back to\n// an orderedconsumer like we do with watch for KV Keys() call.\nfunc TestNoRaceJetStreamStreamInfoSubjectDetailsLimits(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tjetstream {\n\t\t\tstore_dir = %q\n\t\t}\n\t\taccounts: {\n\t\t default: {\n\t\t\tjetstream: true\n\t\t\tusers: [ {user: me, password: pwd} ]\n\t\t\tlimits { max_payload: 512 }\n\t\t }\n\t\t}\n\t`, t.TempDir())))\n\n\ts, _ := RunServerWithConfig(conf)\n\tif config := s.JetStreamConfig(); config != nil {\n\t\tdefer removeDir(t, config.StoreDir)\n\t}\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s, nats.UserInfo(\"me\", \"pwd\"))\n\tdefer nc.Close()\n\n\t// Make sure to flush so we process the 2nd INFO.\n\tnc.Flush()\n\n\t// Make sure we cannot send larger than 512 bytes.\n\t// But we can receive larger.\n\tsub, err := nc.SubscribeSync(\"foo\")\n\trequire_NoError(t, err)\n\terr = nc.Publish(\"foo\", []byte(strings.Repeat(\"A\", 600)))\n\trequire_Error(t, err, nats.ErrMaxPayload)\n\tsub.Unsubscribe()\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\", \"X.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\tn := JSMaxSubjectDetails\n\tfor i := 0; i < n; i++ {\n\t\t_, err := js.PublishAsync(fmt.Sprintf(\"X.%d\", i), []byte(\"OK\"))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Need to grab StreamInfo by hand for now.\n\treq, err := json.Marshal(&JSApiStreamInfoRequest{SubjectsFilter: \"X.*\"})\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), req, 5*time.Second)\n\trequire_NoError(t, err)\n\tvar si StreamInfo\n\terr = json.Unmarshal(resp.Data, &si)\n\trequire_NoError(t, err)\n\tif len(si.State.Subjects) != n {\n\t\tt.Fatalf(\"Expected to get %d subject details, got %d\", n, len(si.State.Subjects))\n\t}\n\n\t// Now add one more message to check pagination\n\t_, err = js.Publish(\"foo\", []byte(\"TOO MUCH\"))\n\trequire_NoError(t, err)\n\n\treq, err = json.Marshal(&JSApiStreamInfoRequest{ApiPagedRequest: ApiPagedRequest{Offset: n}, SubjectsFilter: nats.AllKeys})\n\trequire_NoError(t, err)\n\tresp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, \"TEST\"), req, 5*time.Second)\n\trequire_NoError(t, err)\n\tvar sir JSApiStreamInfoResponse\n\terr = json.Unmarshal(resp.Data, &sir)\n\trequire_NoError(t, err)\n\tif len(sir.State.Subjects) != 1 {\n\t\tt.Fatalf(\"Expected to get 1 extra subject detail, got %d\", len(sir.State.Subjects))\n\t}\n}\n\nfunc TestNoRaceJetStreamSparseConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmsg := []byte(\"ok\")\n\n\tcases := []struct {\n\t\tname string\n\t\tmconfig *nats.StreamConfig\n\t}{\n\t\t{\"MemoryStore\", &nats.StreamConfig{Name: \"TEST\", Storage: nats.MemoryStorage, MaxMsgsPerSubject: 25_000_000,\n\t\t\tSubjects: []string{\"*\"}}},\n\t\t{\"FileStore\", &nats.StreamConfig{Name: \"TEST\", Storage: nats.FileStorage, MaxMsgsPerSubject: 25_000_000,\n\t\t\tSubjects: []string{\"*\"}}},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\tjs.DeleteStream(\"TEST\")\n\t\t\t_, err := js.AddStream(c.mconfig)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// We will purposely place foo msgs near the beginning, then in middle, then at the end.\n\t\t\tfor n := 0; n < 2; n++ {\n\t\t\t\t_, err = js.PublishAsync(\"foo\", msg, nats.StallWait(800*time.Millisecond))\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tfor i := 0; i < 1_000_000; i++ {\n\t\t\t\t\t_, err = js.PublishAsync(\"bar\", msg, nats.StallWait(800*time.Millisecond))\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t\t_, err = js.PublishAsync(\"foo\", msg, nats.StallWait(800*time.Millisecond))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\n\t\t\t// Now create a consumer on foo.\n\t\t\tci, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{DeliverSubject: \"x.x\", FilterSubject: \"foo\", AckPolicy: nats.AckNonePolicy})\n\t\t\trequire_NoError(t, err)\n\n\t\t\tdone, received := make(chan bool), uint64(0)\n\n\t\t\tcb := func(m *nats.Msg) {\n\t\t\t\treceived++\n\t\t\t\tif received >= ci.NumPending {\n\t\t\t\t\tdone <- true\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsub, err := nc.Subscribe(\"x.x\", cb)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer sub.Unsubscribe()\n\t\t\tstart := time.Now()\n\t\t\tvar elapsed time.Duration\n\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\telapsed = time.Since(start)\n\t\t\tcase <-time.After(10 * time.Second):\n\t\t\t\tt.Fatal(\"Did not receive all messages for all consumers in time\")\n\t\t\t}\n\n\t\t\tif elapsed > 500*time.Millisecond {\n\t\t\t\tt.Fatalf(\"Getting all messages took longer than expected: %v\", elapsed)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNoRaceJetStreamConsumerFilterPerfDegradation(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream(nats.PublishAsyncMaxPending(256))\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"test\",\n\t\tSubjects: []string{\"test.*.subj\"},\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend := 50_000\n\tcount := 0\n\tch := make(chan struct{}, 6)\n\t_, err = js.Subscribe(\"test.*.subj\", func(m *nats.Msg) {\n\t\tm.Ack()\n\t\tif count++; count == toSend {\n\t\t\tch <- struct{}{}\n\t\t}\n\t}, nats.DeliverNew(), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\tmsg := make([]byte, 1024)\n\tsent := int32(0)\n\tsend := func() {\n\t\tdefer func() { ch <- struct{}{} }()\n\t\tfor i := 0; i < toSend/5; i++ {\n\t\t\tmsgID := atomic.AddInt32(&sent, 1)\n\t\t\t_, err := js.Publish(fmt.Sprintf(\"test.%d.subj\", msgID), msg)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tfor i := 0; i < 5; i++ {\n\t\tgo send()\n\t}\n\ttimeout := time.NewTimer(10 * time.Second)\n\tfor i := 0; i < 6; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-timeout.C:\n\t\t\tt.Fatal(\"Took too long\")\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamFileStoreKeyFileCleanup(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tprf := func(context []byte) ([]byte, error) {\n\t\th := hmac.New(sha256.New, []byte(\"dlc22\"))\n\t\tif _, err := h.Write(context); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn h.Sum(nil), nil\n\t}\n\n\tfs, err := newFileStoreWithCreated(\n\t\tFileStoreConfig{StoreDir: storeDir, BlockSize: 1024 * 1024},\n\t\tStreamConfig{Name: \"TEST\", Storage: FileStorage},\n\t\ttime.Now(),\n\t\tprf, nil)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tn, msg := 10_000, []byte(strings.Repeat(\"Z\", 1024))\n\tfor i := 0; i < n; i++ {\n\t\t_, _, err := fs.StoreMsg(fmt.Sprintf(\"X.%d\", i), nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tvar seqs []uint64\n\tfor i := 1; i <= n; i++ {\n\t\tseqs = append(seqs, uint64(i))\n\t}\n\t// Randomly delete msgs, make sure we cleanup as we empty the message blocks.\n\trand.Shuffle(len(seqs), func(i, j int) { seqs[i], seqs[j] = seqs[j], seqs[i] })\n\n\tfor _, seq := range seqs {\n\t\t_, err := fs.RemoveMsg(seq)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// We will have cleanup the main .blk and .idx sans the lmb, but we should not have any *.fss files.\n\tkms, err := filepath.Glob(filepath.Join(storeDir, msgDir, keyScanAll))\n\trequire_NoError(t, err)\n\n\tif len(kms) > 1 {\n\t\tt.Fatalf(\"Expected to find only 1 key file, found %d\", len(kms))\n\t}\n}\n\nfunc TestNoRaceJetStreamMsgIdPerfDuringCatchup(t *testing.T) {\n\t// Uncomment to run. Needs to be on a bigger machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.serverByName(\"S-1\"))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// This will be the one we restart.\n\tsl := c.streamLeader(\"$G\", \"TEST\")\n\t// Now move leader.\n\t_, err = nc.Request(fmt.Sprintf(JSApiStreamLeaderStepDownT, \"TEST\"), nil, time.Second)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(\"$G\", \"TEST\")\n\n\t// Connect to new leader.\n\tnc, _ = jsClientConnect(t, c.streamLeader(\"$G\", \"TEST\"))\n\tdefer nc.Close()\n\n\tjs, err = nc.JetStream(nats.PublishAsyncMaxPending(1024))\n\trequire_NoError(t, err)\n\n\tn, ss, sr := 1_000_000, 250_000, 800_000\n\tm := nats.NewMsg(\"TEST\")\n\tm.Data = []byte(strings.Repeat(\"Z\", 2048))\n\n\t// Target rate 10k msgs/sec\n\tstart := time.Now()\n\n\tfor i := 0; i < n; i++ {\n\t\tm.Header.Set(JSMsgId, strconv.Itoa(i))\n\t\t_, err := js.PublishMsgAsync(m)\n\t\trequire_NoError(t, err)\n\t\t//time.Sleep(42 * time.Microsecond)\n\t\tif i == ss {\n\t\t\tfmt.Printf(\"SD\")\n\t\t\tsl.Shutdown()\n\t\t} else if i == sr {\n\t\t\tnc.Flush()\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(10 * time.Second):\n\t\t\t}\n\t\t\tfmt.Printf(\"RS\")\n\t\t\tsl = c.restartServer(sl)\n\t\t}\n\t\tif i%10_000 == 0 {\n\t\t\tfmt.Print(\"#\")\n\t\t}\n\t}\n\tfmt.Println()\n\n\t// Wait to receive all messages.\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\ttt := time.Since(start)\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tfmt.Printf(\"Took %v to send %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n\n\tc.waitOnStreamCurrent(sl, \"$G\", \"TEST\")\n\tfor _, s := range c.servers {\n\t\tmset, _ := s.GlobalAccount().lookupStream(\"TEST\")\n\t\tif state := mset.store.State(); state.Msgs != uint64(n) {\n\t\t\tt.Fatalf(\"Expected server %v to have correct number of msgs %d but got %d\", s, n, state.Msgs)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamRebuildDeDupeAndMemoryPerf(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"DD\"})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"DD\")\n\tm.Data = []byte(strings.Repeat(\"Z\", 2048))\n\n\tstart := time.Now()\n\n\tn := 1_000_000\n\tfor i := 0; i < n; i++ {\n\t\tm.Header.Set(JSMsgId, strconv.Itoa(i))\n\t\t_, err := js.PublishMsgAsync(m)\n\t\trequire_NoError(t, err)\n\t}\n\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\ttt := time.Since(start)\n\tsi, err := js.StreamInfo(\"DD\")\n\trequire_NoError(t, err)\n\n\tfmt.Printf(\"Took %v to send %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n\n\tv, _ := s.Varz(nil)\n\tfmt.Printf(\"Memory AFTER SEND: %v\\n\", friendlyBytes(v.Mem))\n\n\tmset, err := s.GlobalAccount().lookupStream(\"DD\")\n\trequire_NoError(t, err)\n\n\tmset.mu.Lock()\n\tmset.ddMu.Lock()\n\tstart = time.Now()\n\tmset.rebuildDedupe()\n\tfmt.Printf(\"TOOK %v to rebuild dd\\n\", time.Since(start))\n\tmset.ddMu.Unlock()\n\tmset.mu.Unlock()\n\n\tv, _ = s.Varz(nil)\n\tfmt.Printf(\"Memory: %v\\n\", friendlyBytes(v.Mem))\n\n\t// Now do an ephemeral consumer and whip through every message. Doing same calculations.\n\tstart = time.Now()\n\treceived, done := 0, make(chan bool)\n\tsub, err := js.Subscribe(\"DD\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= n {\n\t\t\tdone <- true\n\t\t}\n\t}, nats.OrderedConsumer())\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase <-done:\n\tcase <-time.After(10 * time.Second):\n\t\tif s.NumSlowConsumers() > 0 {\n\t\t\tt.Fatalf(\"Did not receive all large messages due to slow consumer status: %d of %d\", received, n)\n\t\t}\n\t\tt.Fatalf(\"Failed to receive all large messages: %d of %d\\n\", received, n)\n\t}\n\n\tfmt.Printf(\"TOOK %v to receive all %d msgs\\n\", time.Since(start), n)\n\tsub.Unsubscribe()\n\n\tv, _ = s.Varz(nil)\n\tfmt.Printf(\"Memory: %v\\n\", friendlyBytes(v.Mem))\n}\n\nfunc TestNoRaceJetStreamMemoryUsageOnLimitedStreamWithMirror(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{Name: \"DD\", Subjects: []string{\"ORDERS.*\"}, MaxMsgs: 10_000})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tMirror: &nats.StreamSource{Name: \"DD\"},\n\t\tMaxMsgs: 10_000,\n\t})\n\trequire_NoError(t, err)\n\n\tm := nats.NewMsg(\"ORDERS.0\")\n\tm.Data = []byte(strings.Repeat(\"Z\", 2048))\n\n\tstart := time.Now()\n\n\tn := 1_000_000\n\tfor i := 0; i < n; i++ {\n\t\tm.Subject = fmt.Sprintf(\"ORDERS.%d\", i)\n\t\tm.Header.Set(JSMsgId, strconv.Itoa(i))\n\t\t_, err := js.PublishMsgAsync(m)\n\t\trequire_NoError(t, err)\n\t}\n\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\ttt := time.Since(start)\n\tsi, err := js.StreamInfo(\"DD\")\n\trequire_NoError(t, err)\n\n\tfmt.Printf(\"Took %v to send %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n\n\tv, _ := s.Varz(nil)\n\tfmt.Printf(\"Memory AFTER SEND: %v\\n\", friendlyBytes(v.Mem))\n}\n\nfunc TestNoRaceJetStreamOrderedConsumerLongRTTPerformance(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream(nats.PublishAsyncMaxPending(1000))\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{Name: \"OCP\"})\n\trequire_NoError(t, err)\n\n\tn, msg := 100_000, []byte(strings.Repeat(\"D\", 30_000))\n\n\tfor i := 0; i < n; i++ {\n\t\t_, err := js.PublishAsync(\"OCP\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Approximately 3GB\n\tsi, err := js.StreamInfo(\"OCP\")\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\treceived, done := 0, make(chan bool)\n\tsub, err := js.Subscribe(\"OCP\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= n {\n\t\t\tdone <- true\n\t\t}\n\t}, nats.OrderedConsumer())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Wait to receive all messages.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(30 * time.Second):\n\t\tt.Fatalf(\"Did not receive all of our messages\")\n\t}\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n\n\tsub.Unsubscribe()\n\n\trtt := 10 * time.Millisecond\n\tbw := 10 * 1024 * 1024 * 1024\n\tproxy := newNetProxy(rtt, bw, bw, s.ClientURL())\n\tdefer proxy.stop()\n\n\tnc, err = nats.Connect(proxy.clientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\tjs, err = nc.JetStream()\n\trequire_NoError(t, err)\n\n\tstart, received = time.Now(), 0\n\tsub, err = js.Subscribe(\"OCP\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= n {\n\t\t\tdone <- true\n\t\t}\n\t}, nats.OrderedConsumer())\n\trequire_NoError(t, err)\n\tdefer sub.Unsubscribe()\n\n\t// Wait to receive all messages.\n\tselect {\n\tcase <-done:\n\tcase <-time.After(60 * time.Second):\n\t\tt.Fatalf(\"Did not receive all of our messages\")\n\t}\n\n\ttt = time.Since(start)\n\tfmt.Printf(\"Proxy RTT: %v, UP: %d, DOWN: %d\\n\", rtt, bw, bw)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, n)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(n)/tt.Seconds())\n\tfmt.Printf(\"%.0f mb/s\\n\\n\", float64(si.State.Bytes/(1024*1024))/tt.Seconds())\n}\n\nvar jsClusterStallCatchupTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 32GB, store_dir: '%s'}\n\n\tleaf {\n\t\tlisten: 127.0.0.1:-1\n\t}\n\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\n// Test our global stall gate for outstanding catchup bytes.\nfunc TestNoRaceJetStreamClusterCatchupStallGate(t *testing.T) {\n\tskip(t)\n\n\tc := createJetStreamClusterWithTemplate(t, jsClusterStallCatchupTempl, \"GSG\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// ~100k per message.\n\tmsg := []byte(strings.Repeat(\"A\", 99_960))\n\n\t// Create 200 streams with 100MB.\n\t// Each server has ~2GB\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 20; i++ {\n\t\twg.Add(1)\n\t\tgo func(x int) {\n\t\t\tdefer wg.Done()\n\t\t\tfor n := 1; n <= 10; n++ {\n\t\t\t\tsn := fmt.Sprintf(\"S-%d\", n+x)\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: sn,\n\t\t\t\t\tReplicas: 3,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\t\t_, err := js.Publish(sn, msg)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}(i * 20)\n\t}\n\twg.Wait()\n\n\tinfo, err := js.AccountInfo()\n\trequire_NoError(t, err)\n\trequire_True(t, info.Streams == 200)\n\n\truntime.GC()\n\tdebug.FreeOSMemory()\n\n\t// Now bring a server down and wipe its storage.\n\ts := c.servers[0]\n\tvz, err := s.Varz(nil)\n\trequire_NoError(t, err)\n\tfmt.Printf(\"MEM BEFORE is %v\\n\", friendlyBytes(vz.Mem))\n\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\tremoveDir(t, sd)\n\ts = c.restartServer(s)\n\n\tc.waitOnServerHealthz(s)\n\n\truntime.GC()\n\tdebug.FreeOSMemory()\n\n\tvz, err = s.Varz(nil)\n\trequire_NoError(t, err)\n\tfmt.Printf(\"MEM AFTER is %v\\n\", friendlyBytes(vz.Mem))\n}\n\nfunc TestNoRaceJetStreamClusterCatchupBailMidway(t *testing.T) {\n\tskip(t)\n\n\tc := createJetStreamClusterWithTemplate(t, jsClusterStallCatchupTempl, \"GSG\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\tnc, js := jsClientConnect(t, ml)\n\tdefer nc.Close()\n\n\tmsg := []byte(strings.Repeat(\"A\", 480))\n\n\tfor i := 0; i < maxConcurrentSyncRequests*2; i++ {\n\t\tsn := fmt.Sprintf(\"CUP-%d\", i+1)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: sn,\n\t\t\tReplicas: 3,\n\t\t})\n\t\trequire_NoError(t, err)\n\n\t\tfor i := 0; i < 10_000; i++ {\n\t\t\t_, err := js.PublishAsync(sn, msg)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tselect {\n\t\tcase <-js.PublishAsyncComplete():\n\t\tcase <-time.After(10 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t}\n\t}\n\n\tjsz, _ := ml.Jsz(nil)\n\texpectedMsgs := jsz.Messages\n\n\t// Now select a server and shut it down, removing the storage directory.\n\ts := c.randomNonLeader()\n\tsd := s.JetStreamConfig().StoreDir\n\ts.Shutdown()\n\tremoveDir(t, sd)\n\n\t// Now restart the server.\n\ts = c.restartServer(s)\n\n\t// We want to force the follower to bail before the catchup through the\n\t// upper level catchup logic completes.\n\tcheckFor(t, 5*time.Second, 10*time.Millisecond, func() error {\n\t\tjsz, _ := s.Jsz(nil)\n\t\tif jsz.Messages > expectedMsgs/2 {\n\t\t\ts.Shutdown()\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not enough yet\")\n\t})\n\n\t// Now restart the server.\n\ts = c.restartServer(s)\n\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\tjsz, _ := s.Jsz(nil)\n\t\tif jsz.Messages == expectedMsgs {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Not enough yet\")\n\t})\n}\n\nfunc TestNoRaceJetStreamAccountLimitsAndRestart(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterAccountLimitsTempl, \"A3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tfor i := 0; i < 20_000; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"A\")); err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif i == 5_000 {\n\t\t\tsnl := c.randomNonStreamLeader(\"$JS\", \"TEST\")\n\t\t\tsnl.Shutdown()\n\t\t}\n\t}\n\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$JS\", \"TEST\")\n\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$JS\", \"TEST\")\n\t})\n\tfor _, cs := range c.servers {\n\t\tc.waitOnStreamCurrent(cs, \"$JS\", \"TEST\")\n\t}\n}\n\nfunc TestNoRaceJetStreamAccountLimitsAndRestartForceSnapshot(t *testing.T) {\n\tc := createJetStreamClusterWithTemplate(t, jsClusterAccountLimitsTempl, \"A3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tif _, err := js.AddStream(&nats.StreamConfig{Name: \"TEST\", Replicas: 3}); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tsnl := c.randomNonStreamLeader(\"$JS\", \"TEST\")\n\tfor i := 0; i < 20_000; i++ {\n\t\tif _, err := js.Publish(\"TEST\", []byte(\"A\")); err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif i == 5_000 {\n\t\t\tsnl.Shutdown()\n\t\t}\n\t}\n\n\t// Wait for the remaining servers to converge on the state.\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$JS\", \"TEST\")\n\t})\n\t// Then install snapshots so the shutdown server needs to catchup based on a snapshot.\n\tfor _, s := range c.servers {\n\t\tif s == snl {\n\t\t\tcontinue\n\t\t}\n\t\tacc, err := s.lookupAccount(\"$JS\")\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tn := mset.raftNode().(*raft)\n\t\trequire_NoError(t, n.InstallSnapshot(mset.stateSnapshot(), false))\n\t}\n\n\tc.stopAll()\n\tc.restartAll()\n\tc.waitOnLeader()\n\tc.waitOnStreamLeader(\"$JS\", \"TEST\")\n\n\tcheckFor(t, 2*time.Second, 500*time.Millisecond, func() error {\n\t\treturn checkState(t, c, \"$JS\", \"TEST\")\n\t})\n\tfor _, cs := range c.servers {\n\t\tc.waitOnStreamCurrent(cs, \"$JS\", \"TEST\")\n\t}\n}\n\nfunc TestNoRaceJetStreamPullConsumersAndInteriorDeletes(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"ID\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"foo\",\n\t\tReplicas: 3,\n\t\tMaxMsgs: 50000,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnStreamLeader(globalAccountName, \"foo\")\n\n\t_, err = js.AddConsumer(\"foo\", &nats.ConsumerConfig{\n\t\tDurable: \"foo\",\n\t\tFilterSubject: \"foo\",\n\t\tMaxAckPending: 20000,\n\t\tAckWait: 5 * time.Second,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnConsumerLeader(globalAccountName, \"foo\", \"foo\")\n\n\trcv := int32(0)\n\tprods := 5\n\tcons := 5\n\twg := sync.WaitGroup{}\n\twg.Add(prods + cons)\n\ttoSend := 100000\n\n\tfor i := 0; i < cons; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tsub, err := js.PullSubscribe(\"foo\", \"foo\")\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor {\n\t\t\t\tmsgs, err := sub.Fetch(200, nats.MaxWait(250*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\tif n := int(atomic.LoadInt32(&rcv)); n >= toSend {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor _, m := range msgs {\n\t\t\t\t\tm.Ack()\n\t\t\t\t\tatomic.AddInt32(&rcv, 1)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tfor i := 0; i < prods; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tfor i := 0; i < toSend/prods; i++ {\n\t\t\t\tjs.Publish(\"foo\", []byte(\"hello\"))\n\t\t\t}\n\t\t}()\n\t}\n\n\ttime.Sleep(time.Second)\n\tresp, err := nc.Request(fmt.Sprintf(JSApiConsumerLeaderStepDownT, \"foo\", \"foo\"), nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tvar cdResp JSApiConsumerLeaderStepDownResponse\n\tif err := json.Unmarshal(resp.Data, &cdResp); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif cdResp.Error != nil {\n\t\tt.Fatalf(\"Unexpected error: %+v\", cdResp.Error)\n\t}\n\tch := make(chan struct{})\n\tgo func() {\n\t\twg.Wait()\n\t\tclose(ch)\n\t}()\n\tselect {\n\tcase <-ch:\n\t\t// OK\n\tcase <-time.After(30 * time.Second):\n\t\tt.Fatalf(\"Consumers took too long to consume all messages\")\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterInterestPullConsumerStreamLimitBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tlimit := uint64(1000)\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t\tMaxMsgs: int64(limit),\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{Durable: \"dur\", AckPolicy: nats.AckExplicitPolicy})\n\trequire_NoError(t, err)\n\n\tqch := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\t// Publisher\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; ; i++ {\n\t\t\tselect {\n\t\t\tcase <-qch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\t_, err := js.Publish(\"foo\", []byte(\"BUG!\"))\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// Only sleep every so often.\n\t\t\t\tif i%100 == 0 {\n\t\t\t\t\ttime.Sleep(time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(time.Second)\n\n\t// Pull Consumers\n\twg.Add(100)\n\tfor i := 0; i < 100; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tnc := natsConnect(t, c.randomServer().ClientURL())\n\t\t\tdefer nc.Close()\n\n\t\t\tjs, err := nc.JetStream(nats.MaxWait(time.Second))\n\t\t\trequire_NoError(t, err)\n\n\t\t\tvar sub *nats.Subscription\n\t\t\tfor j := 0; j < 5; j++ {\n\t\t\t\tsub, err = js.PullSubscribe(\"foo\", \"dur\")\n\t\t\t\tif err == nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor {\n\t\t\t\tpt := time.NewTimer(time.Duration(rand.Intn(300)) * time.Millisecond)\n\t\t\t\tselect {\n\t\t\t\tcase <-pt.C:\n\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Logf(\"Got a Fetch error: %v\", err)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif len(msgs) > 0 {\n\t\t\t\t\t\tgo func() {\n\t\t\t\t\t\t\tackDelay := time.Duration(rand.Intn(375)+15) * time.Millisecond\n\t\t\t\t\t\t\tm := msgs[0]\n\t\t\t\t\t\t\ttime.AfterFunc(ackDelay, func() { m.AckSync() })\n\t\t\t\t\t\t}()\n\t\t\t\t\t}\n\t\t\t\tcase <-qch:\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Make sure we have hit the limit for the number of messages we expected.\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs < limit {\n\t\t\treturn fmt.Errorf(\"Not hit limit yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tclose(qch)\n\twg.Wait()\n\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tci, err := js.ConsumerInfo(\"TEST\", \"dur\")\n\t\trequire_NoError(t, err)\n\n\t\tnp := ci.NumPending + uint64(ci.NumAckPending)\n\t\tif np != si.State.Msgs {\n\t\t\treturn fmt.Errorf(\"Expected NumPending to be %d got %d\", si.State.Msgs-uint64(ci.NumAckPending), ci.NumPending)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Test that all peers have the direct access subs that participate in a queue group,\n// but only when they are current and ready. So we will start with R1, add in messages\n// then scale up while also still adding messages.\nfunc TestNoRaceJetStreamClusterDirectAccessAllPeersSubs(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"JSC\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Start as R1\n\tcfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"kv.>\"},\n\t\tMaxMsgsPer: 10,\n\t\tAllowDirect: true,\n\t\tReplicas: 1,\n\t\tStorage: FileStorage,\n\t}\n\taddStream(t, nc, cfg)\n\n\t// Seed with enough messages to start then we will scale up while still adding more messages.\n\tnum, msg := 1000, bytes.Repeat([]byte(\"XYZ\"), 64)\n\tfor i := 0; i < num; i++ {\n\t\tjs.PublishAsync(fmt.Sprintf(\"kv.%d\", i), msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tgetSubj := fmt.Sprintf(JSDirectMsgGetT, \"TEST\")\n\tgetMsg := func(key string) *nats.Msg {\n\t\tt.Helper()\n\t\treq := []byte(fmt.Sprintf(`{\"last_by_subj\":%q}`, key))\n\t\tm, err := nc.Request(getSubj, req, time.Second)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, m.Header.Get(JSSubject) == key)\n\t\treturn m\n\t}\n\n\t// Just make sure we can succeed here.\n\tgetMsg(\"kv.22\")\n\n\t// Now crank up a go routine to continue sending more messages.\n\tqch := make(chan bool)\n\tvar wg sync.WaitGroup\n\n\tfor i := 0; i < 5; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-qch:\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\t\t\tcase <-time.After(10 * time.Second):\n\t\t\t\t\t}\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t\t// Send as fast as we can.\n\t\t\t\t\tjs.Publish(fmt.Sprintf(\"kv.%d\", rand.Intn(1000)), msg)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\ttime.Sleep(200 * time.Millisecond)\n\n\t// Now let's scale up to an R3.\n\tcfg.Replicas = 3\n\tupdateStream(t, nc, cfg)\n\n\t// Wait for the stream to register the new replicas and have a leader.\n\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster == nil {\n\t\t\treturn fmt.Errorf(\"No cluster yet\")\n\t\t}\n\t\tif si.Cluster.Leader == _EMPTY_ || len(si.Cluster.Replicas) != 2 {\n\t\t\treturn fmt.Errorf(\"Cluster not ready yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tclose(qch)\n\twg.Wait()\n\n\t// Just make sure we can succeed here.\n\tgetMsg(\"kv.22\")\n\n\t// For each non-leader check that the direct sub fires up.\n\t// We just test all, the leader will already have a directSub.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tcheckFor(t, 20*time.Second, 500*time.Millisecond, func() error {\n\t\t\tmset.mu.RLock()\n\t\t\tok := mset.directSub != nil\n\t\t\tmset.mu.RUnlock()\n\t\t\tif ok {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"No directSub yet\")\n\t\t})\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.State.Msgs == uint64(num) {\n\t\tt.Fatalf(\"Expected to see messages increase, got %d\", si.State.Msgs)\n\t}\n\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t// Make sure they are all the same from a state perspective.\n\t\t// Leader will have the expected state.\n\t\tlmset, err := c.streamLeader(\"$G\", \"TEST\").GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\texpected := lmset.state()\n\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tif state := mset.state(); !reflect.DeepEqual(expected, state) {\n\t\t\t\treturn fmt.Errorf(\"Expected %+v, got %+v\", expected, state)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc TestNoRaceJetStreamClusterStreamNamesAndInfosMoreThanAPILimit(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tcreateStream := func(name string) {\n\t\tt.Helper()\n\t\tif _, err := js.AddStream(&nats.StreamConfig{Name: name}); err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tmax := JSApiListLimit\n\tif JSApiNamesLimit > max {\n\t\tmax = JSApiNamesLimit\n\t}\n\tmax += 10\n\n\tfor i := 0; i < max; i++ {\n\t\tname := fmt.Sprintf(\"foo_%d\", i)\n\t\tcreateStream(name)\n\t}\n\n\t// Not using the JS API here because we want to make sure that the\n\t// server returns the proper Total count, but also that it does not\n\t// send more than when the API limit is in one go.\n\tcheck := func(subj string, limit int) {\n\t\tt.Helper()\n\n\t\tnreq := JSApiStreamNamesRequest{}\n\t\tb, _ := json.Marshal(nreq)\n\n\t\tmsg, err := nc.Request(subj, b, 2*time.Second)\n\t\trequire_NoError(t, err)\n\n\t\tswitch subj {\n\t\tcase JSApiStreams:\n\t\t\tnresp := JSApiStreamNamesResponse{}\n\t\t\tjson.Unmarshal(msg.Data, &nresp)\n\t\t\tif n := nresp.ApiPaged.Total; n != max {\n\t\t\t\tt.Fatalf(\"Expected total to be %v, got %v\", max, n)\n\t\t\t}\n\t\t\tif n := nresp.ApiPaged.Limit; n != limit {\n\t\t\t\tt.Fatalf(\"Expected limit to be %v, got %v\", limit, n)\n\t\t\t}\n\t\t\tif n := len(nresp.Streams); n != limit {\n\t\t\t\tt.Fatalf(\"Expected number of streams to be %v, got %v\", limit, n)\n\t\t\t}\n\t\tcase JSApiStreamList:\n\t\t\tnresp := JSApiStreamListResponse{}\n\t\t\tjson.Unmarshal(msg.Data, &nresp)\n\t\t\tif n := nresp.ApiPaged.Total; n != max {\n\t\t\t\tt.Fatalf(\"Expected total to be %v, got %v\", max, n)\n\t\t\t}\n\t\t\tif n := nresp.ApiPaged.Limit; n != limit {\n\t\t\t\tt.Fatalf(\"Expected limit to be %v, got %v\", limit, n)\n\t\t\t}\n\t\t\tif n := len(nresp.Streams); n != limit {\n\t\t\t\tt.Fatalf(\"Expected number of streams to be %v, got %v\", limit, n)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheck(JSApiStreams, JSApiNamesLimit)\n\tcheck(JSApiStreamList, JSApiListLimit)\n}\n\nfunc TestNoRaceJetStreamClusterConsumerListPaging(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomNonLeader()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tcfg := &nats.ConsumerConfig{\n\t\tReplicas: 1,\n\t\tMemoryStorage: true,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t}\n\n\t// create 3000 consumers.\n\tnumConsumers := 3000\n\tfor i := 1; i <= numConsumers; i++ {\n\t\tcfg.Durable = fmt.Sprintf(\"d-%.4d\", i)\n\t\t_, err := js.AddConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Test both names and list operations.\n\n\t// Names\n\treqSubj := fmt.Sprintf(JSApiConsumersT, \"TEST\")\n\tgrabConsumerNames := func(offset int) []string {\n\t\treq := fmt.Sprintf(`{\"offset\":%d}`, offset)\n\t\trespMsg, err := nc.Request(reqSubj, []byte(req), time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar resp JSApiConsumerNamesResponse\n\t\terr = json.Unmarshal(respMsg.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\t// Sanity check that we are actually paging properly around limits.\n\t\tif resp.Limit < len(resp.Consumers) {\n\t\t\tt.Fatalf(\"Expected total limited to %d but got %d\", resp.Limit, len(resp.Consumers))\n\t\t}\n\t\tif resp.Total != numConsumers {\n\t\t\tt.Fatalf(\"Invalid total response: expected %d got %d\", numConsumers, resp.Total)\n\t\t}\n\t\treturn resp.Consumers\n\t}\n\n\tresults := make(map[string]bool)\n\n\tfor offset := 0; len(results) < numConsumers; {\n\t\tconsumers := grabConsumerNames(offset)\n\t\toffset += len(consumers)\n\t\tfor _, name := range consumers {\n\t\t\tif results[name] {\n\t\t\t\tt.Fatalf(\"Found duplicate %q\", name)\n\t\t\t}\n\t\t\tresults[name] = true\n\t\t}\n\t}\n\n\t// List\n\treqSubj = fmt.Sprintf(JSApiConsumerListT, \"TEST\")\n\tgrabConsumerList := func(offset int) []*ConsumerInfo {\n\t\treq := fmt.Sprintf(`{\"offset\":%d}`, offset)\n\t\trespMsg, err := nc.Request(reqSubj, []byte(req), time.Second)\n\t\trequire_NoError(t, err)\n\t\tvar resp JSApiConsumerListResponse\n\t\terr = json.Unmarshal(respMsg.Data, &resp)\n\t\trequire_NoError(t, err)\n\t\t// Sanity check that we are actually paging properly around limits.\n\t\tif resp.Limit < len(resp.Consumers) {\n\t\t\tt.Fatalf(\"Expected total limited to %d but got %d\", resp.Limit, len(resp.Consumers))\n\t\t}\n\t\tif resp.Total != numConsumers {\n\t\t\tt.Fatalf(\"Invalid total response: expected %d got %d\", numConsumers, resp.Total)\n\t\t}\n\t\treturn resp.Consumers\n\t}\n\n\tresults = make(map[string]bool)\n\n\tfor offset := 0; len(results) < numConsumers; {\n\t\tconsumers := grabConsumerList(offset)\n\t\toffset += len(consumers)\n\t\tfor _, ci := range consumers {\n\t\t\tname := ci.Config.Durable\n\t\t\tif results[name] {\n\t\t\t\tt.Fatalf(\"Found duplicate %q\", name)\n\t\t\t}\n\t\t\tresults[name] = true\n\t\t}\n\t}\n\n\tif len(results) != numConsumers {\n\t\tt.Fatalf(\"Received %d / %d consumers\", len(results), numConsumers)\n\t}\n}\n\nfunc TestNoRaceJetStreamFileStoreLargeKVAccessTiming(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tblkSize := uint64(4 * 1024)\n\t// Compensate for slower IO on MacOSX\n\tif runtime.GOOS == \"darwin\" {\n\t\tblkSize *= 4\n\t}\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: storeDir, BlockSize: blkSize, CacheExpire: 30 * time.Second},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"KV.STREAM_NAME.*\"}, Storage: FileStorage, MaxMsgsPer: 1},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\ttmpl := \"KV.STREAM_NAME.%d\"\n\tnkeys, val := 100_000, bytes.Repeat([]byte(\"Z\"), 1024)\n\n\tfor i := 1; i <= nkeys; i++ {\n\t\tsubj := fmt.Sprintf(tmpl, i)\n\t\t_, _, err := fs.StoreMsg(subj, nil, val, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tfirst := fmt.Sprintf(tmpl, 1)\n\tlast := fmt.Sprintf(tmpl, nkeys)\n\n\tstart := time.Now()\n\tsm, err := fs.LoadLastMsg(last, nil)\n\trequire_NoError(t, err)\n\tbase := time.Since(start)\n\n\tif !bytes.Equal(sm.msg, val) {\n\t\tt.Fatalf(\"Retrieved value did not match\")\n\t}\n\n\tstart = time.Now()\n\t_, err = fs.LoadLastMsg(first, nil)\n\trequire_NoError(t, err)\n\tslow := time.Since(start)\n\n\tif base > 100*time.Microsecond || slow > 200*time.Microsecond {\n\t\tt.Fatalf(\"Took too long to look up first key vs last: %v vs %v\", base, slow)\n\t}\n\n\t// time first seq lookup for both as well.\n\t// Base will be first in this case.\n\tfs.mu.Lock()\n\tstart = time.Now()\n\tfs.firstSeqForSubj(first)\n\tbase = time.Since(start)\n\tstart = time.Now()\n\tfs.firstSeqForSubj(last)\n\tslow = time.Since(start)\n\tfs.mu.Unlock()\n\n\tif base > 100*time.Microsecond || slow > 200*time.Microsecond {\n\t\tt.Fatalf(\"Took too long to look up last key by subject vs first: %v vs %v\", base, slow)\n\t}\n}\n\nfunc TestNoRaceJetStreamKVLock(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: \"LOCKS\"})\n\trequire_NoError(t, err)\n\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tvar wg sync.WaitGroup\n\tstart := make(chan bool)\n\n\tvar tracker int64\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\tkv, err := js.KeyValue(\"LOCKS\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\t<-start\n\n\t\t\tfor {\n\t\t\t\tlast, err := kv.Create(\"MY_LOCK\", []byte(\"Z\"))\n\t\t\t\tif err != nil {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-time.After(10 * time.Millisecond):\n\t\t\t\t\t\tcontinue\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif v := atomic.AddInt64(&tracker, 1); v != 1 {\n\t\t\t\t\tt.Logf(\"TRACKER NOT 1 -> %d\\n\", v)\n\t\t\t\t\tcancel()\n\t\t\t\t}\n\n\t\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\t\tif v := atomic.AddInt64(&tracker, -1); v != 0 {\n\t\t\t\t\tt.Logf(\"TRACKER NOT 0 AFTER RELEASE -> %d\\n\", v)\n\t\t\t\t\tcancel()\n\t\t\t\t}\n\n\t\t\t\terr = kv.Delete(\"MY_LOCK\", nats.LastRevision(last))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Could not unlock for last %d: %v\", last, err)\n\t\t\t\t}\n\n\t\t\t\tif ctx.Err() != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tclose(start)\n\twg.Wait()\n}\n\nfunc TestNoRaceJetStreamSuperClusterStreamMoveLongRTT(t *testing.T) {\n\t// Make C2 far away.\n\tgwm := gwProxyMap{\n\t\t\"C2\": &gwProxy{\n\t\t\trtt: 20 * time.Millisecond,\n\t\t\tup: 1 * 1024 * 1024 * 1024, // 1gbit\n\t\t\tdown: 1 * 1024 * 1024 * 1024, // 1gbit\n\t\t},\n\t}\n\tsc := createJetStreamTaggedSuperClusterWithGWProxy(t, gwm)\n\tdefer sc.shutdown()\n\n\tnc, js := jsClientConnect(t, sc.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"chunk.*\"},\n\t\tPlacement: &nats.Placement{Tags: []string{\"cloud:aws\", \"country:us\"}},\n\t\tReplicas: 3,\n\t}\n\n\t// Place a stream in C1.\n\t_, err := js.AddStream(cfg, nats.MaxWait(10*time.Second))\n\trequire_NoError(t, err)\n\n\tchunk := bytes.Repeat([]byte(\"Z\"), 1000*1024) // ~1MB\n\t// 256 MB\n\tfor i := 0; i < 256; i++ {\n\t\tsubj := fmt.Sprintf(\"chunk.%d\", i)\n\t\tjs.PublishAsync(subj, chunk)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// C2, slow RTT.\n\tcfg.Placement = &nats.Placement{Tags: []string{\"cloud:gcp\", \"country:uk\"}}\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 20*time.Second, time.Second, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\", nats.MaxWait(time.Second))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif si.Cluster.Name != \"C2\" {\n\t\t\treturn fmt.Errorf(\"Wrong cluster: %q\", si.Cluster.Name)\n\t\t}\n\t\tif si.Cluster.Leader == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"No leader yet\")\n\t\t} else if !strings.HasPrefix(si.Cluster.Leader, \"C2-\") {\n\t\t\treturn fmt.Errorf(\"Wrong leader: %q\", si.Cluster.Leader)\n\t\t}\n\t\t// Now we want to see that we shrink back to original.\n\t\tif len(si.Cluster.Replicas) != cfg.Replicas-1 {\n\t\t\treturn fmt.Errorf(\"Expected %d replicas, got %d\", cfg.Replicas-1, len(si.Cluster.Replicas))\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// https://github.com/nats-io/nats-server/issues/3455\nfunc TestNoRaceJetStreamConcurrentPullConsumerBatch(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ORDERS.*\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend := int32(100_000)\n\n\tfor i := 0; i < 100_000; i++ {\n\t\tsubj := fmt.Sprintf(\"ORDERS.%d\", i+1)\n\t\tjs.PublishAsync(subj, []byte(\"BUY\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"PROCESSOR\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: 5000,\n\t})\n\trequire_NoError(t, err)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub1, err := js.PullSubscribe(_EMPTY_, _EMPTY_, nats.Bind(\"TEST\", \"PROCESSOR\"))\n\trequire_NoError(t, err)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tsub2, err := js.PullSubscribe(_EMPTY_, _EMPTY_, nats.Bind(\"TEST\", \"PROCESSOR\"))\n\trequire_NoError(t, err)\n\n\tstartCh := make(chan bool)\n\n\tvar received int32\n\n\twg := sync.WaitGroup{}\n\n\tfetchSize := 1000\n\tfetch := func(sub *nats.Subscription) {\n\t\t<-startCh\n\t\tdefer wg.Done()\n\n\t\tfor {\n\t\t\tmsgs, err := sub.Fetch(fetchSize, nats.MaxWait(time.Second))\n\t\t\tif atomic.AddInt32(&received, int32(len(msgs))) >= toSend {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// We should always receive a full batch here if not last competing fetch.\n\t\t\tif err != nil || len(msgs) != fetchSize {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tfor _, m := range msgs {\n\t\t\t\tm.Ack()\n\t\t\t}\n\t\t}\n\t}\n\n\twg.Add(2)\n\n\tgo fetch(sub1)\n\tgo fetch(sub2)\n\n\tclose(startCh)\n\n\twg.Wait()\n\trequire_True(t, received == toSend)\n}\n\nfunc TestNoRaceJetStreamManyPullConsumersNeedAckOptimization(t *testing.T) {\n\t// Uncomment to run. Do not want as part of Travis tests atm.\n\t// Run with cpu and memory profiling to make sure we have improved.\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORDERS\",\n\t\tSubjects: []string{\"ORDERS.*\"},\n\t\tStorage: nats.MemoryStorage,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\ttoSend := 100_000\n\tnumConsumers := 500\n\n\t// Create 500 consumers\n\tfor i := 1; i <= numConsumers; i++ {\n\t\t_, err := js.AddConsumer(\"ORDERS\", &nats.ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"ORDERS_%d\", i),\n\t\t\tFilterSubject: fmt.Sprintf(\"ORDERS.%d\", i),\n\t\t\tAckPolicy: nats.AckAllPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tfor i := 1; i <= toSend; i++ {\n\t\tsubj := fmt.Sprintf(\"ORDERS.%d\", i%numConsumers+1)\n\t\tjs.PublishAsync(subj, []byte(\"HELLO\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsub, err := js.PullSubscribe(\"ORDERS.500\", \"ORDERS_500\")\n\trequire_NoError(t, err)\n\n\tfetchSize := toSend / numConsumers\n\tmsgs, err := sub.Fetch(fetchSize, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\n\tlast := msgs[len(msgs)-1]\n\tlast.AckSync()\n}\n\n// https://github.com/nats-io/nats-server/issues/3499\nfunc TestNoRaceJetStreamDeleteConsumerWithInterestStreamAndHighSeqs(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"log.>\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"c\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Set baseline for time to delete so we can see linear increase as sequence numbers increase.\n\tstart := time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"c\")\n\trequire_NoError(t, err)\n\telapsed := time.Since(start)\n\n\t// Crank up sequence numbers.\n\tmsg := []byte(strings.Repeat(\"ZZZ\", 128))\n\tfor i := 0; i < 5_000_000; i++ {\n\t\tnc.Publish(\"log.Z\", msg)\n\t}\n\tnc.Flush()\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"c\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// We have a bug that spins unecessarily through all the sequences from this consumer's\n\t// ackfloor(0) and the last sequence for the stream. We will detect by looking for the time\n\t// to delete being 100x more. Should be the same since both times no messages exist in the stream.\n\tstart = time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"c\")\n\trequire_NoError(t, err)\n\n\tif e := time.Since(start); e > 100*elapsed {\n\t\tt.Fatalf(\"Consumer delete took too long: %v vs baseline %v\", e, elapsed)\n\t}\n}\n\n// Bug when we encode a timestamp that upon decode causes an error which causes server to panic.\n// This can happen on consumer redelivery since they adjusted timstamps can be in the future, and result\n// in a negative encoding. If that encoding was exactly -1 seconds, would cause decodeConsumerState to fail\n// and the server to panic.\nfunc TestNoRaceEncodeConsumerStateBug(t *testing.T) {\n\tfor i := 0; i < 200_000; i++ {\n\t\t// Pretend we redelivered and updated the timestamp to reflect the new start time for expiration.\n\t\t// The bug will trip when time.Now() rounded to seconds in encode is 1 second below the truncated version\n\t\t// of pending.\n\t\tpending := Pending{Sequence: 1, Timestamp: time.Now().Add(time.Second).UnixNano()}\n\t\tstate := ConsumerState{\n\t\t\tDelivered: SequencePair{Consumer: 1, Stream: 1},\n\t\t\tPending: map[uint64]*Pending{1: &pending},\n\t\t}\n\t\tbuf := encodeConsumerState(&state)\n\t\t_, err := decodeConsumerState(buf)\n\t\trequire_NoError(t, err)\n\t}\n}\n\n// Performance impact on stream ingress with large number of consumers.\nfunc TestNoRaceJetStreamLargeNumConsumersPerfImpact(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Baseline with no consumers.\n\ttoSend := 1_000_000\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt := time.Since(start)\n\tfmt.Printf(\"Base time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Now add in 10 idle consumers.\n\tfor i := 1; i <= 10; i++ {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"d-%d\", i),\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tstart = time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt = time.Since(start)\n\tfmt.Printf(\"\\n10 consumers time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Now add in 90 more idle consumers.\n\tfor i := 11; i <= 100; i++ {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"d-%d\", i),\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tstart = time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt = time.Since(start)\n\tfmt.Printf(\"\\n100 consumers time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n\n\terr = js.PurgeStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Now add in 900 more\n\tfor i := 101; i <= 1000; i++ {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"d-%d\", i),\n\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\tstart = time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"foo\", []byte(\"OK\"))\n\t}\n\t<-js.PublishAsyncComplete()\n\ttt = time.Since(start)\n\tfmt.Printf(\"\\n1000 consumers time is %v\\n\", tt)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(toSend)/tt.Seconds())\n}\n\n// Performance impact on large number of consumers but sparse delivery.\nfunc TestNoRaceJetStreamLargeNumConsumersSparseDelivery(t *testing.T) {\n\tskip(t)\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"ID.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Now add in ~10k consumers on different subjects.\n\tfor i := 3; i <= 10_000; i++ {\n\t\t_, err := js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\tDurable: fmt.Sprintf(\"d-%d\", i),\n\t\t\tFilterSubject: fmt.Sprintf(\"ID.%d\", i),\n\t\t\tAckPolicy: nats.AckNonePolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t}\n\n\ttoSend := 100_000\n\n\t// Bind a consumer to ID.2.\n\tvar received int\n\tdone := make(chan bool)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tmh := func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= toSend {\n\t\t\tclose(done)\n\t\t}\n\t}\n\t_, err = js.Subscribe(\"ID.2\", mh)\n\trequire_NoError(t, err)\n\n\tlast := make(chan bool)\n\t_, err = js.Subscribe(\"ID.1\", func(_ *nats.Msg) { close(last) })\n\trequire_NoError(t, err)\n\n\tnc, _ = jsClientConnect(t, s)\n\tdefer nc.Close()\n\tjs, err = nc.JetStream(nats.PublishAsyncMaxPending(8 * 1024))\n\trequire_NoError(t, err)\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tjs.PublishAsync(\"ID.2\", []byte(\"ok\"))\n\t}\n\t// Check latency for this one message.\n\t// This will show the issue better than throughput which can bypass signal processing.\n\tjs.PublishAsync(\"ID.1\", []byte(\"ok\"))\n\n\tselect {\n\tcase <-done:\n\t\tbreak\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Failed to receive all messages: %d of %d\\n\", received, toSend)\n\t}\n\n\ttt := time.Since(start)\n\tfmt.Printf(\"Took %v to receive %d msgs\\n\", tt, toSend)\n\tfmt.Printf(\"%.0f msgs/s\\n\", float64(toSend)/tt.Seconds())\n\n\tselect {\n\tcase <-last:\n\t\tbreak\n\tcase <-time.After(30 * time.Second):\n\t\tt.Fatalf(\"Failed to receive last message\\n\")\n\t}\n\tlt := time.Since(start)\n\n\tfmt.Printf(\"Took %v to receive last msg\\n\", lt)\n}\n\nfunc TestNoRaceJetStreamEndToEndLatency(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\tvar sent time.Time\n\tvar max time.Duration\n\tnext := make(chan struct{})\n\n\tmh := func(m *nats.Msg) {\n\t\treceived := time.Now()\n\t\ttt := received.Sub(sent)\n\t\tif max == 0 || tt > max {\n\t\t\tmax = tt\n\t\t}\n\t\tnext <- struct{}{}\n\t}\n\tsub, err := js.Subscribe(\"foo\", mh)\n\trequire_NoError(t, err)\n\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\ttoSend := 50_000\n\tfor i := 0; i < toSend; i++ {\n\t\tsent = time.Now()\n\t\tjs.Publish(\"foo\", []byte(\"ok\"))\n\t\t<-next\n\t}\n\tsub.Unsubscribe()\n\n\tif max > 250*time.Millisecond {\n\t\tt.Fatalf(\"Expected max latency to be < 250ms, got %v\", max)\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterEnsureWALCompact(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tDeliverSubject: \"zz\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Force snapshot on stream leader.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tnode := mset.raftNode()\n\trequire_True(t, node != nil)\n\n\terr = node.InstallSnapshot(mset.stateSnapshot(), false)\n\trequire_NoError(t, err)\n\n\t// Now publish more than should be needed to cause an additional snapshot.\n\tns := 75_000\n\tfor i := 0; i <= ns; i++ {\n\t\t_, err := js.Publish(\"foo\", []byte(\"bar\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Grab progress and use that to look into WAL entries.\n\t_, _, applied := node.Progress()\n\t// If ne == ns that means snapshots and compacts were not happening when\n\t// they should have been.\n\tif ne, _ := node.Applied(applied); ne >= uint64(ns) {\n\t\tt.Fatalf(\"Did not snapshot and compact the raft WAL, entries == %d\", ne)\n\t}\n\n\t// Now check consumer.\n\t// Force snapshot on consumerleader.\n\tcl := c.consumerLeader(globalAccountName, \"TEST\", \"dlc\")\n\tmset, err = cl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"dlc\")\n\trequire_True(t, o != nil)\n\n\tnode = o.raftNode()\n\trequire_True(t, node != nil)\n\n\tsnap, err := o.store.EncodedState()\n\trequire_NoError(t, err)\n\terr = node.InstallSnapshot(snap, false)\n\trequire_NoError(t, err)\n\n\treceived, done := 0, make(chan bool, 1)\n\n\tnc.Subscribe(\"zz\", func(m *nats.Msg) {\n\t\treceived++\n\t\tif received >= ns {\n\t\t\tselect {\n\t\t\tcase done <- true:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\tm.Ack()\n\t})\n\n\tselect {\n\tcase <-done:\n\t\treturn\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not received all %d msgs, only %d\", ns, received)\n\t}\n\n\t// Do same trick and check that WAL was compacted.\n\t// Grab progress and use that to look into WAL entries.\n\t_, _, applied = node.Progress()\n\t// If ne == ns that means snapshots and compacts were not happening when\n\t// they should have been.\n\tif ne, _ := node.Applied(applied); ne >= uint64(ns) {\n\t\tt.Fatalf(\"Did not snapshot and compact the raft WAL, entries == %d\", ne)\n\t}\n}\n\nfunc TestNoRaceFileStoreStreamMaxAgePerformance(t *testing.T) {\n\t// Uncomment to run.\n\tskip(t)\n\n\tstoreDir := t.TempDir()\n\tmaxAge := 5 * time.Second\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: storeDir},\n\t\tStreamConfig{Name: \"MA\",\n\t\t\tSubjects: []string{\"foo.*\"},\n\t\t\tMaxAge: maxAge,\n\t\t\tStorage: FileStorage},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Simulate a callback similar to consumers decrementing.\n\tvar mu sync.RWMutex\n\tvar pending int64\n\n\tfs.RegisterStorageUpdates(func(md, bd int64, seq uint64, subj string) {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tpending += md\n\t})\n\n\tstart, num, subj := time.Now(), 0, \"foo.foo\"\n\n\ttimeout := start.Add(maxAge)\n\tfor time.Now().Before(timeout) {\n\t\t// We will store in blocks of 100.\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tnum++\n\t\t}\n\t}\n\telapsed := time.Since(start)\n\tfmt.Printf(\"Took %v to store %d\\n\", elapsed, num)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(num)/elapsed.Seconds())\n\n\t// Now keep running for 2x longer knowing we are expiring messages in the background.\n\t// We want to see the effect on performance.\n\n\tstart = time.Now()\n\ttimeout = start.Add(maxAge * 2)\n\n\tfor time.Now().Before(timeout) {\n\t\t// We will store in blocks of 100.\n\t\tfor i := 0; i < 100; i++ {\n\t\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tnum++\n\t\t}\n\t}\n\telapsed = time.Since(start)\n\tfmt.Printf(\"Took %v to store %d\\n\", elapsed, num)\n\tfmt.Printf(\"%.0f msgs/sec\\n\", float64(num)/elapsed.Seconds())\n}\n\n// FilteredState for \">\" with large interior deletes was very slow.\nfunc TestNoRaceFileStoreFilteredStateWithLargeDeletes(t *testing.T) {\n\tstoreDir := t.TempDir()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: storeDir, BlockSize: 4096},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Storage: FileStorage},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"foo\", []byte(\"Hello World\")\n\n\ttoStore := 500_000\n\tfor i := 0; i < toStore; i++ {\n\t\t_, _, err := fs.StoreMsg(subj, nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Now delete every other one.\n\tfor seq := 2; seq <= toStore; seq += 2 {\n\t\t_, err := fs.RemoveMsg(uint64(seq))\n\t\trequire_NoError(t, err)\n\t}\n\n\truntime.GC()\n\t// Disable to get stable results.\n\tgcp := debug.SetGCPercent(-1)\n\tdefer debug.SetGCPercent(gcp)\n\n\tstart := time.Now()\n\tfss, err := fs.FilteredState(1, _EMPTY_)\n\trequire_NoError(t, err)\n\telapsed := time.Since(start)\n\n\trequire_True(t, fss.Msgs == uint64(toStore/2))\n\trequire_True(t, elapsed < 500*time.Microsecond)\n}\n\n// ConsumerInfo seems to being called quite a bit more than we had anticipated.\n// Under certain circumstances, since we reset num pending, this can be very costly.\n// We will use the fast path to alleviate that performance bottleneck but also make\n// sure we are still being accurate.\nfunc TestNoRaceJetStreamClusterConsumerInfoSpeed(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tc.waitOnLeader()\n\tserver := c.randomNonLeader()\n\n\tnc, js := jsClientConnect(t, server)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// The issue is compounded when we have lots of different subjects captured\n\t// by a terminal fwc. The consumer will have a terminal pwc.\n\t// Here make all subjects unique.\n\n\tsub, err := js.PullSubscribe(\"events.*\", \"DLC\")\n\trequire_NoError(t, err)\n\n\ttoSend := 250_000\n\tfor i := 0; i < toSend; i++ {\n\t\tsubj := fmt.Sprintf(\"events.%d\", i+1)\n\t\tjs.PublishAsync(subj, []byte(\"ok\"))\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckNumPending := func(expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tstart := time.Now()\n\t\t\tci, err := js.ConsumerInfo(\"TEST\", \"DLC\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t// Make sure these are fast now.\n\t\t\tif elapsed := time.Since(start); elapsed > 50*time.Millisecond {\n\t\t\t\treturn fmt.Errorf(\"ConsumerInfo took too long: %v\", elapsed)\n\t\t\t}\n\t\t\t// Make sure pending == expected.\n\t\t\tif ci.NumPending != uint64(expected) {\n\t\t\t\treturn fmt.Errorf(\"Expected %d NumPending, got %d\", expected, ci.NumPending)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Make sure in simple case it is correct.\n\tcheckNumPending(toSend)\n\n\t// Do a few acks.\n\ttoAck := 25\n\tfor _, m := range fetchMsgs(t, sub, 25, time.Second) {\n\t\terr = m.AckSync()\n\t\trequire_NoError(t, err)\n\t}\n\tcheckNumPending(toSend - toAck)\n\n\t// Now do a purge such that we only keep so many.\n\t// We want to make sure we do the right thing here and have correct calculations.\n\ttoKeep := 100_000\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Keep: uint64(toKeep)})\n\trequire_NoError(t, err)\n\n\tcheckNumPending(toKeep)\n}\n\nfunc TestNoRaceJetStreamKVAccountWithServerRestarts(t *testing.T) {\n\t// Uncomment to run. Needs fast machine to not time out on KeyValue lookup.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tnpubs := 10_000\n\tpar := 8\n\titer := 2\n\tnsubjs := 250\n\n\twg := sync.WaitGroup{}\n\tputKeys := func() {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\t\t\tkv, err := js.KeyValue(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\n\t\t\tfor i := 0; i < npubs; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"KEY-%d\", rand.Intn(nsubjs))\n\t\t\t\tif _, err := kv.PutString(subj, \"hello\"); err != nil {\n\t\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\t\tdefer nc.Close()\n\t\t\t\t\tkv, err = js.KeyValue(\"TEST\")\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\trestartServers := func() {\n\t\ttime.Sleep(2 * time.Second)\n\t\t// Rotate through and restart the servers.\n\t\tfor _, server := range c.servers {\n\t\t\tserver.Shutdown()\n\t\t\trestarted := c.restartServer(server)\n\t\t\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\t\t\ths := restarted.healthz(&HealthzOptions{\n\t\t\t\t\tJSMetaOnly: true,\n\t\t\t\t})\n\t\t\t\tif hs.Error != _EMPTY_ {\n\t\t\t\t\treturn errors.New(hs.Error)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t\tc.waitOnLeader()\n\t\tc.waitOnStreamLeader(globalAccountName, \"KV_TEST\")\n\t}\n\n\tfor n := 0; n < iter; n++ {\n\t\tfor i := 0; i < par; i++ {\n\t\t\tputKeys()\n\t\t}\n\t\trestartServers()\n\t}\n\twg.Wait()\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tsi, err := js.StreamInfo(\"KV_TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.NumSubjects == uint64(nsubjs))\n}\n\n// Test for consumer create when the subject cardinality is high and the\n// consumer is filtered with a wildcard that forces linear scans.\n// We have an optimization to use in memory structures in filestore to speed up.\n// Only if asking to scan all (DeliverAll).\nfunc TestNoRaceJetStreamConsumerCreateTimeNumPending(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\tn := 500_000\n\tmsg := bytes.Repeat([]byte(\"X\"), 8*1024)\n\n\tfor i := 0; i < n; i++ {\n\t\tsubj := fmt.Sprintf(\"events.%d\", rand.Intn(100_000))\n\t\tjs.PublishAsync(subj, msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t}\n\n\t// Should stay under 5ms now, but for Travis variability say 50ms.\n\tthreshold := 50 * time.Millisecond\n\n\tstart := time.Now()\n\t_, err = js.PullSubscribe(\"events.*\", \"dlc\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > threshold {\n\t\tt.Skipf(\"Consumer create took longer than expected, %v vs %v\", elapsed, threshold)\n\t}\n\n\tstart = time.Now()\n\t_, err = js.PullSubscribe(\"events.99999\", \"xxx\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > threshold {\n\t\tt.Skipf(\"Consumer create took longer than expected, %v vs %v\", elapsed, threshold)\n\t}\n\n\tstart = time.Now()\n\t_, err = js.PullSubscribe(\">\", \"zzz\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > threshold {\n\t\tt.Skipf(\"Consumer create took longer than expected, %v vs %v\", elapsed, threshold)\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterGhostConsumers(t *testing.T) {\n\tconsumerNotActiveStartInterval = time.Second\n\tconsumerNotActiveMaxInterval = time.Second\n\tdefer func() {\n\t\tconsumerNotActiveStartInterval = defaultConsumerNotActiveStartInterval\n\t\tconsumerNotActiveMaxInterval = defaultConsumerNotActiveMaxInterval\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"GHOST\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"events.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfor j := 0; j < 10; j++ {\n\t\t\trequire_NoError(t, nc.Publish(fmt.Sprintf(\"events.%d.%d\", i, j), []byte(`test`)))\n\t\t}\n\t}\n\n\tfetch := func(id int) {\n\t\tsubject := fmt.Sprintf(\"events.%d.*\", id)\n\t\tsubscription, err := js.PullSubscribe(subject,\n\t\t\t_EMPTY_, // ephemeral consumer\n\t\t\tnats.DeliverAll(),\n\t\t\tnats.ReplayInstant(),\n\t\t\tnats.BindStream(\"TEST\"),\n\t\t\tnats.ConsumerReplicas(1),\n\t\t\tnats.ConsumerMemoryStorage(),\n\t\t)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tdefer subscription.Unsubscribe()\n\n\t\tinfo, err := subscription.ConsumerInfo()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tsubscription.Fetch(int(info.NumPending))\n\t}\n\n\treplay := func(ctx context.Context, id int) {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tfetch(id)\n\t\t\t}\n\t\t}\n\t}\n\n\tctx, cancel := context.WithCancel(context.Background())\n\n\tgo replay(ctx, 0)\n\tgo replay(ctx, 1)\n\tgo replay(ctx, 2)\n\tgo replay(ctx, 3)\n\tgo replay(ctx, 4)\n\tgo replay(ctx, 5)\n\tgo replay(ctx, 6)\n\tgo replay(ctx, 7)\n\tgo replay(ctx, 8)\n\tgo replay(ctx, 9)\n\n\ttime.Sleep(5 * time.Second)\n\n\tfor _, server := range c.servers {\n\t\tserver.Shutdown()\n\t\trestarted := c.restartServer(server)\n\t\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\t\ths := restarted.healthz(&HealthzOptions{\n\t\t\t\tJSMetaOnly: true,\n\t\t\t})\n\t\t\tif hs.Error != _EMPTY_ {\n\t\t\t\treturn errors.New(hs.Error)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\ttime.Sleep(time.Second * 2)\n\t\tgo replay(ctx, 5)\n\t\tgo replay(ctx, 6)\n\t\tgo replay(ctx, 7)\n\t\tgo replay(ctx, 8)\n\t\tgo replay(ctx, 9)\n\t}\n\n\ttime.Sleep(5 * time.Second)\n\tcancel()\n\n\t// Check we don't report missing consumers.\n\tsubj := fmt.Sprintf(JSApiConsumerListT, \"TEST\")\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\t// Request will take at most 4 seconds if some consumers can't be found.\n\t\tm, err := nc.Request(subj, nil, 5*time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tvar resp JSApiConsumerListResponse\n\t\trequire_NoError(t, json.Unmarshal(m.Data, &resp))\n\t\tif len(resp.Missing) == 0 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Still have missing: %+v\", resp.Missing)\n\t})\n\n\t// Also check all servers agree on the available consumer assignments.\n\t// It could be the above check passes, i.e. our meta leader thinks all is okay, but other servers actually drifted.\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tvar previousConsumers []string\n\t\tfor _, s := range c.servers {\n\t\t\tsjs := s.getJetStream()\n\t\t\tsjs.mu.Lock()\n\t\t\tcc := sjs.cluster\n\t\t\tsa := cc.streams[globalAccountName][\"TEST\"]\n\t\t\tvar consumers []string\n\t\t\tfor cName := range sa.consumers {\n\t\t\t\tconsumers = append(consumers, cName)\n\t\t\t}\n\t\t\tsjs.mu.Unlock()\n\t\t\tslices.Sort(consumers)\n\t\t\tif previousConsumers != nil && !slices.Equal(previousConsumers, consumers) {\n\t\t\t\treturn fmt.Errorf(\"Consumer mismatch:\\n- previous: %v\\n- actual : %v\\n\", previousConsumers, consumers)\n\t\t\t}\n\t\t\tpreviousConsumers = consumers\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// This is to test a publish slowdown and general instability experienced in a setup similar to this.\n// We have feeder streams that are all sourced to an aggregate stream. All streams are interest retention.\n// We want to monitor the avg publish time for the sync publishers to the feeder streams, the ingest rate to\n// the aggregate stream, and general health of the consumers on the aggregate stream.\n// Target publish rate is ~2k/s with publish time being ~40-60ms but remaining stable.\n// We can also simulate max redeliveries that create interior deletes in streams.\nfunc TestNoRaceJetStreamClusterF3Setup(t *testing.T) {\n\t// Uncomment to run. Needs to be on a pretty big machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\t// These and the settings below achieve ~60ms pub time on avg and ~2k msgs per sec inbound to the aggregate stream.\n\t// On my machine though.\n\tnp := clusterProxy{\n\t\trtt: 2 * time.Millisecond,\n\t\tup: 1 * 1024 * 1024 * 1024, // 1gbit\n\t\tdown: 1 * 1024 * 1024 * 1024, // 1gbit\n\t}\n\n\t// Test params.\n\tnumSourceStreams := 20\n\tnumConsumersPerSource := 1\n\tnumPullersPerConsumer := 50\n\tnumPublishers := 100\n\tsetHighStartSequence := false\n\tsimulateMaxRedeliveries := false\n\tmaxBadPubTimes := uint32(20)\n\tbadPubThresh := 500 * time.Millisecond\n\ttestTime := 5 * time.Minute // make sure to do --timeout=65m\n\n\tt.Logf(\"Starting Test: Total Test Time %v\", testTime)\n\n\tc := createJetStreamClusterWithNetProxy(t, \"R3S\", 3, &np)\n\tdefer c.shutdown()\n\n\t// Do some quick sanity checking for latency stuff.\n\t{\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"TEST\",\n\t\t\tReplicas: 3,\n\t\t\tSubjects: []string{\"foo\"},\n\t\t\tRetention: nats.InterestPolicy,\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tdefer js.DeleteStream(\"TEST\")\n\n\t\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\t\tnc, js = jsClientConnect(t, sl)\n\t\tdefer nc.Close()\n\t\tstart := time.Now()\n\t\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t\t// This is best case, and with client connection being close to free, this should be at least > rtt\n\t\tif elapsed := time.Since(start); elapsed < np.rtt {\n\t\t\tt.Fatalf(\"Expected publish time to be > %v, got %v\", np.rtt, elapsed)\n\t\t}\n\n\t\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\t\tnc, js = jsClientConnect(t, nl)\n\t\tdefer nc.Close()\n\t\tstart = time.Now()\n\t\t_, err = js.Publish(\"foo\", []byte(\"hello\"))\n\t\trequire_NoError(t, err)\n\t\t// This is worst case, meaning message has to travel to leader, then to fastest replica, then back.\n\t\t// So should be at 3x rtt, so check at least > 2x rtt.\n\t\tif elapsed := time.Since(start); elapsed < 2*np.rtt {\n\t\t\tt.Fatalf(\"Expected publish time to be > %v, got %v\", 2*np.rtt, elapsed)\n\t\t}\n\t}\n\n\t// Setup source streams.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tt.Logf(\"Creating %d Source Streams\", numSourceStreams)\n\n\tvar sources []string\n\twg := sync.WaitGroup{}\n\tfor i := 0; i < numSourceStreams; i++ {\n\t\tsname := fmt.Sprintf(\"EVENT-%s\", nuid.Next())\n\t\tsources = append(sources, sname)\n\t\twg.Add(1)\n\t\tgo func(stream string) {\n\t\t\tdefer wg.Done()\n\t\t\tt.Logf(\" %q\", stream)\n\t\t\tsubj := fmt.Sprintf(\"%s.>\", stream)\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: stream,\n\t\t\t\tSubjects: []string{subj},\n\t\t\t\tReplicas: 3,\n\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tfor j := 0; j < numConsumersPerSource; j++ {\n\t\t\t\tconsumer := fmt.Sprintf(\"C%d\", j)\n\t\t\t\t_, err := js.Subscribe(_EMPTY_, func(msg *nats.Msg) {\n\t\t\t\t\tmsg.Ack()\n\t\t\t\t}, nats.BindStream(stream), nats.Durable(consumer), nats.ManualAck())\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}(sname)\n\t}\n\twg.Wait()\n\n\tvar streamSources []*nats.StreamSource\n\tfor _, src := range sources {\n\t\tstreamSources = append(streamSources, &nats.StreamSource{Name: src})\n\n\t}\n\n\tt.Log(\"Creating Aggregate Stream\")\n\n\t// Now create the aggregate stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"EVENTS\",\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t\tSources: streamSources,\n\t})\n\trequire_NoError(t, err)\n\n\t// Set first sequence to a high number.\n\tif setHighStartSequence {\n\t\trequire_NoError(t, js.PurgeStream(\"EVENTS\", &nats.StreamPurgeRequest{Sequence: 32_000_001}))\n\t}\n\n\t// Now create 2 pull consumers.\n\t_, err = js.PullSubscribe(_EMPTY_, \"C1\",\n\t\tnats.BindStream(\"EVENTS\"),\n\t\tnats.MaxDeliver(1),\n\t\tnats.AckWait(10*time.Second),\n\t\tnats.ManualAck(),\n\t)\n\trequire_NoError(t, err)\n\n\t_, err = js.PullSubscribe(_EMPTY_, \"C2\",\n\t\tnats.BindStream(\"EVENTS\"),\n\t\tnats.MaxDeliver(1),\n\t\tnats.AckWait(10*time.Second),\n\t\tnats.ManualAck(),\n\t)\n\trequire_NoError(t, err)\n\n\tt.Logf(\"Creating %d x 2 Pull Subscribers\", numPullersPerConsumer)\n\n\t// Now create the pullers.\n\tfor _, subName := range []string{\"C1\", \"C2\"} {\n\t\tfor i := 0; i < numPullersPerConsumer; i++ {\n\t\t\tgo func(subName string) {\n\t\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tsub, err := js.PullSubscribe(_EMPTY_, subName,\n\t\t\t\t\tnats.BindStream(\"EVENTS\"),\n\t\t\t\t\tnats.MaxDeliver(1),\n\t\t\t\t\tnats.AckWait(10*time.Second),\n\t\t\t\t\tnats.ManualAck(),\n\t\t\t\t)\n\t\t\t\trequire_NoError(t, err)\n\n\t\t\t\tfor {\n\t\t\t\t\tmsgs, err := sub.Fetch(25, nats.MaxWait(2*time.Second))\n\t\t\t\t\tif err != nil && err != nats.ErrTimeout {\n\t\t\t\t\t\tt.Logf(\"Exiting pull subscriber %q: %v\", subName, err)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\t// Shuffle\n\t\t\t\t\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\n\t\t\t\t\t// Wait for a random interval up to 100ms.\n\t\t\t\t\ttime.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)\n\n\t\t\t\t\tfor _, m := range msgs {\n\t\t\t\t\t\t// If we want to simulate max redeliveries being hit, since not acking\n\t\t\t\t\t\t// once will cause it due to subscriber setup.\n\t\t\t\t\t\t// 100_000 == 0.01%\n\t\t\t\t\t\tif simulateMaxRedeliveries && rand.Intn(100_000) == 0 {\n\t\t\t\t\t\t\tmd, err := m.Metadata()\n\t\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\t\tt.Logf(\"** Skipping Ack: %d **\", md.Sequence.Stream)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tm.Ack()\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(subName)\n\t\t}\n\t}\n\n\t// Now create feeder publishers.\n\teventTypes := []string{\"PAYMENT\", \"SUBMISSION\", \"CANCEL\"}\n\n\tmsg := make([]byte, 2*1024) // 2k payload\n\tcrand.Read(msg)\n\n\t// For tracking pub times.\n\tvar pubs int\n\tvar totalPubTime time.Duration\n\tvar pmu sync.Mutex\n\tlast := time.Now()\n\n\tupdatePubStats := func(elapsed time.Duration) {\n\t\tpmu.Lock()\n\t\tdefer pmu.Unlock()\n\t\t// Reset every 5s\n\t\tif time.Since(last) > 5*time.Second {\n\t\t\tpubs = 0\n\t\t\ttotalPubTime = 0\n\t\t\tlast = time.Now()\n\t\t}\n\t\tpubs++\n\t\ttotalPubTime += elapsed\n\t}\n\tavgPubTime := func() time.Duration {\n\t\tpmu.Lock()\n\t\tnp := pubs\n\t\ttpt := totalPubTime\n\t\tpmu.Unlock()\n\t\treturn tpt / time.Duration(np)\n\t}\n\n\tt.Logf(\"Creating %d Publishers\", numPublishers)\n\n\tvar numLimitsExceeded atomic.Uint32\n\terrCh := make(chan error, 100)\n\n\tfor i := 0; i < numPublishers; i++ {\n\t\tgo func() {\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tfor {\n\t\t\t\t// Grab a random source stream\n\t\t\t\tstream := sources[rand.Intn(len(sources))]\n\t\t\t\t// Grab random event type.\n\t\t\t\tevt := eventTypes[rand.Intn(len(eventTypes))]\n\t\t\t\tsubj := fmt.Sprintf(\"%s.%s\", stream, evt)\n\t\t\t\tstart := time.Now()\n\t\t\t\t_, err := js.Publish(subj, msg)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Exiting publisher: %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\telapsed := time.Since(start)\n\t\t\t\tif elapsed > badPubThresh {\n\t\t\t\t\tt.Logf(\"Publish time took more than expected: %v\", elapsed)\n\t\t\t\t\tnumLimitsExceeded.Add(1)\n\t\t\t\t\tif ne := numLimitsExceeded.Load(); ne > maxBadPubTimes {\n\t\t\t\t\t\terrCh <- fmt.Errorf(\"Too many exceeded times on publish: %d\", ne)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tupdatePubStats(elapsed)\n\t\t\t}\n\t\t}()\n\t}\n\n\tt.Log(\"Creating Monitoring Routine - Data in ~10s\")\n\n\t// Create monitoring routine.\n\tgo func() {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfseq, lseq := uint64(0), uint64(0)\n\t\tfor {\n\t\t\t// Grab consumers\n\t\t\tvar minAckFloor uint64 = math.MaxUint64\n\t\t\tfor _, consumer := range []string{\"C1\", \"C2\"} {\n\t\t\t\tci, err := js.ConsumerInfo(\"EVENTS\", consumer)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Exiting Monitor: %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif lseq > 0 {\n\t\t\t\t\tt.Logf(\"%s:\\n Delivered:\\t%d\\n AckFloor:\\t%d\\n AckPending:\\t%d\\n NumPending:\\t%d\",\n\t\t\t\t\t\tconsumer, ci.Delivered.Stream, ci.AckFloor.Stream, ci.NumAckPending, ci.NumPending)\n\t\t\t\t}\n\t\t\t\tif ci.AckFloor.Stream < minAckFloor {\n\t\t\t\t\tminAckFloor = ci.AckFloor.Stream\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now grab aggregate stream state.\n\t\t\tsi, err := js.StreamInfo(\"EVENTS\")\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Exiting Monitor: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tstate := si.State\n\t\t\tif lseq != 0 {\n\t\t\t\tt.Logf(\"Stream:\\n Msgs: \\t%d\\n First:\\t%d\\n Last: \\t%d\\n Deletes:\\t%d\\n\",\n\t\t\t\t\tstate.Msgs, state.FirstSeq, state.LastSeq, state.NumDeleted)\n\t\t\t\tt.Logf(\"Publish Stats:\\n Msgs/s:\\t%0.2f\\n Avg Pub:\\t%v\\n\\n\", float64(si.State.LastSeq-lseq)/5.0, avgPubTime())\n\t\t\t\tif si.State.FirstSeq < minAckFloor && si.State.FirstSeq == fseq {\n\t\t\t\t\tt.Log(\"Stream first seq < minimum ack floor\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tfseq, lseq = si.State.FirstSeq, si.State.LastSeq\n\t\t\ttime.Sleep(5 * time.Second)\n\t\t}\n\n\t}()\n\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatal(e)\n\tcase <-time.After(testTime):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n}\n\n// Unbalanced stretch cluster.\n// S2 (stream leader) will have a slow path to S1 (via proxy) and S3 (consumer leader) will have a fast path.\n//\n//\t Route Ports\n//\t\t\"S1\": 14622\n//\t\t\"S2\": 15622\n//\t\t\"S3\": 16622\nfunc createStretchUnbalancedCluster(t testing.TB) (c *cluster, np *netProxy) {\n\tt.Helper()\n\n\ttmpl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: \"F3\"\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n\t`\n\t// Do these in order, S1, S2 (proxy) then S3.\n\tc = &cluster{t: t, servers: make([]*Server, 3), opts: make([]*Options, 3), name: \"F3\"}\n\n\t// S1\n\tconf := fmt.Sprintf(tmpl, \"S1\", t.TempDir(), 14622, \"route://127.0.0.1:15622, route://127.0.0.1:16622\")\n\tc.servers[0], c.opts[0] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\t// S2\n\t// Create the proxy first. Connect this to S1. Make it slow, e.g. 5ms RTT.\n\tnp = createNetProxy(1*time.Millisecond, 1024*1024*1024, 1024*1024*1024, \"route://127.0.0.1:14622\", true)\n\troutes := fmt.Sprintf(\"%s, route://127.0.0.1:16622\", np.routeURL())\n\tconf = fmt.Sprintf(tmpl, \"S2\", t.TempDir(), 15622, routes)\n\tc.servers[1], c.opts[1] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\t// S3\n\tconf = fmt.Sprintf(tmpl, \"S3\", t.TempDir(), 16622, \"route://127.0.0.1:14622, route://127.0.0.1:15622\")\n\tc.servers[2], c.opts[2] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\tc.checkClusterFormed()\n\tc.waitOnClusterReady()\n\n\treturn c, np\n}\n\n// We test an interest based stream that has a cluster with a node with asymmetric paths from\n// the stream leader and the consumer leader such that the consumer leader path is fast and\n// replicated acks arrive sooner then the actual message. This path was considered, but also\n// categorized as very rare and was expensive as it tried to forward a new stream msg delete\n// proposal to the original stream leader. It now will deal with the issue locally and not\n// slow down the ingest rate to the stream's publishers.\nfunc TestNoRaceJetStreamClusterDifferentRTTInterestBasedStreamSetup(t *testing.T) {\n\t// Uncomment to run. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\tc, np := createStretchUnbalancedCluster(t)\n\tdefer c.shutdown()\n\tdefer np.stop()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now create the stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"EVENTS\",\n\t\tSubjects: []string{\"EV.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure it's leader is on S2.\n\tsl := c.servers[1]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(globalAccountName, \"EVENTS\")\n\t\tif s := c.streamLeader(globalAccountName, \"EVENTS\"); s != sl {\n\t\t\ts.JetStreamStepdownStream(globalAccountName, \"EVENTS\")\n\t\t\treturn fmt.Errorf(\"Server %s is not stream leader yet\", sl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create the consumer.\n\t_, err = js.PullSubscribe(_EMPTY_, \"C\", nats.BindStream(\"EVENTS\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Make sure the consumer leader is on S3.\n\tcl := c.servers[2]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"C\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"C\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"C\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", cl)\n\t\t}\n\t\treturn nil\n\t})\n\n\tgo func(js nats.JetStream) {\n\t\tsub, err := js.PullSubscribe(_EMPTY_, \"C\", nats.BindStream(\"EVENTS\"), nats.ManualAck())\n\t\trequire_NoError(t, err)\n\n\t\tfor {\n\t\t\tmsgs, err := sub.Fetch(100, nats.MaxWait(2*time.Second))\n\t\t\tif err != nil && err != nats.ErrTimeout {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Shuffle\n\t\t\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\t\t\tfor _, m := range msgs {\n\t\t\t\tm.Ack()\n\t\t\t}\n\t\t}\n\t}(js)\n\n\tnumPublishers := 25\n\tpubThresh := 2 * time.Second\n\tvar maxExceeded atomic.Int64\n\terrCh := make(chan error, numPublishers)\n\twg := sync.WaitGroup{}\n\n\tmsg := make([]byte, 2*1024) // 2k payload\n\tcrand.Read(msg)\n\n\t// Publishers.\n\tfor i := 0; i < numPublishers; i++ {\n\t\twg.Add(1)\n\t\tgo func(iter int) {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Connect to random, the slow ones will be connected to the slow node.\n\t\t\t// But if you connect them all there it will pass.\n\t\t\ts := c.randomServer()\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tfor i := 0; i < 1_000; i++ {\n\t\t\t\tstart := time.Now()\n\t\t\t\t_, err := js.Publish(\"EV.PAID\", msg)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrCh <- fmt.Errorf(\"Publish error: %v\", err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif elapsed := time.Since(start); elapsed > pubThresh {\n\t\t\t\t\terrCh <- fmt.Errorf(\"Publish time exceeded\")\n\t\t\t\t\tif int64(elapsed) > maxExceeded.Load() {\n\t\t\t\t\t\tmaxExceeded.Store(int64(elapsed))\n\t\t\t\t\t}\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatalf(\"%v: threshold is %v, maximum seen: %v\", e, pubThresh, time.Duration(maxExceeded.Load()))\n\tdefault:\n\t}\n}\n\nfunc TestNoRaceJetStreamInterestStreamCheckInterestRaceBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t\tStorage: nats.MemoryStorage,\n\t})\n\trequire_NoError(t, err)\n\n\tnumConsumers := 10\n\tfor i := 0; i < numConsumers; i++ {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err = js.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\t\tm.Ack()\n\t\t}, nats.Durable(fmt.Sprintf(\"C%d\", i)), nats.ManualAck())\n\t\trequire_NoError(t, err)\n\t}\n\n\tnumToSend := 10_000\n\tfor i := 0; i < numToSend; i++ {\n\t\t_, err := js.PublishAsync(\"foo\", nil, nats.StallWait(800*time.Millisecond))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Put this into a function so that the defer lets go of that server's stream lock\n\t// when we have finished checking, otherwise the loop in checkFor() ends up holding\n\t// all of the stream locks across all servers, wedging things.\n\tcheckForServer := func(s *Server) error {\n\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, mset.numConsumers(), numConsumers)\n\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\n\t\tif mset.lseq < uint64(numToSend) {\n\t\t\treturn fmt.Errorf(\"waiting for %d messages in stream (%d so far)\", numToSend, mset.lseq)\n\t\t}\n\n\t\tfor _, o := range mset.consumers {\n\t\t\tstate, err := o.store.State()\n\t\t\trequire_NoError(t, err)\n\t\t\tif state.AckFloor.Stream != uint64(numToSend) {\n\t\t\t\treturn fmt.Errorf(\"Ackfloor not correct yet (%d != %d)\", state.AckFloor.Stream, numToSend)\n\t\t\t}\n\t\t}\n\n\t\tstate := mset.state()\n\t\tif state.Msgs != 0 {\n\t\t\treturn fmt.Errorf(\"too many messages: %d\", state.Msgs)\n\t\t} else if state.FirstSeq != uint64(numToSend+1) {\n\t\t\treturn fmt.Errorf(\"wrong FirstSeq: %d, expected: %d\", state.FirstSeq, numToSend+1)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// Wait til ackfloor is correct for all consumers.\n\tcheckFor(t, 30*time.Second, 250*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif err := checkForServer(s); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterInterestStreamConsistencyAfterRollingRestart(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnumStreams := 200\n\tnumConsumersPer := 5\n\tnumPublishers := 10\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tqch := make(chan bool)\n\n\tvar mm sync.Mutex\n\tackMap := make(map[string]map[uint64][]string)\n\n\taddAckTracking := func(seq uint64, stream, consumer string) {\n\t\tmm.Lock()\n\t\tdefer mm.Unlock()\n\t\tsam := ackMap[stream]\n\t\tif sam == nil {\n\t\t\tsam = make(map[uint64][]string)\n\t\t\tackMap[stream] = sam\n\t\t}\n\t\tsam[seq] = append(sam[seq], consumer)\n\t}\n\n\tdoPullSubscriber := func(stream, consumer, filter string) {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tvar err error\n\t\tvar sub *nats.Subscription\n\t\ttimeout := time.Now().Add(5 * time.Second)\n\t\tfor time.Now().Before(timeout) {\n\t\t\tsub, err = js.PullSubscribe(filter, consumer, nats.BindStream(stream), nats.ManualAck())\n\t\t\tif err == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Logf(\"Error on pull subscriber: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-time.After(500 * time.Millisecond):\n\t\t\t\tmsgs, err := sub.Fetch(100, nats.MaxWait(time.Second))\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Shuffle\n\t\t\t\trand.Shuffle(len(msgs), func(i, j int) { msgs[i], msgs[j] = msgs[j], msgs[i] })\n\t\t\t\tfor _, m := range msgs {\n\t\t\t\t\tmeta, err := m.Metadata()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tm.Ack()\n\t\t\t\t\taddAckTracking(meta.Sequence.Stream, stream, consumer)\n\t\t\t\t\tif meta.NumDelivered > 1 {\n\t\t\t\t\t\tt.Logf(\"Got a msg redelivered %d for sequence %d on %q %q\\n\", meta.NumDelivered, meta.Sequence.Stream, stream, consumer)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase <-qch:\n\t\t\t\tnc.Flush()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// Setup\n\twg := sync.WaitGroup{}\n\tfor i := 0; i < numStreams; i++ {\n\t\twg.Add(1)\n\t\tgo func(stream string) {\n\t\t\tdefer wg.Done()\n\t\t\tsubj := fmt.Sprintf(\"%s.>\", stream)\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: stream,\n\t\t\t\tSubjects: []string{subj},\n\t\t\t\tReplicas: 3,\n\t\t\t\tRetention: nats.InterestPolicy,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\t\t\tfor i := 0; i < numConsumersPer; i++ {\n\t\t\t\tconsumer := fmt.Sprintf(\"C%d\", i)\n\t\t\t\tfilter := fmt.Sprintf(\"%s.%d\", stream, i)\n\t\t\t\t_, err = js.AddConsumer(stream, &nats.ConsumerConfig{\n\t\t\t\t\tDurable: consumer,\n\t\t\t\t\tFilterSubject: filter,\n\t\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\t\tAckWait: 2 * time.Second,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tc.waitOnConsumerLeader(globalAccountName, stream, consumer)\n\t\t\t\tgo doPullSubscriber(stream, consumer, filter)\n\t\t\t}\n\t\t}(fmt.Sprintf(\"A-%d\", i))\n\t}\n\twg.Wait()\n\n\tmsg := make([]byte, 2*1024) // 2k payload\n\tcrand.Read(msg)\n\n\t// Controls if publishing is on or off.\n\tvar pubActive atomic.Bool\n\n\tdoPublish := func() {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-time.After(100 * time.Millisecond):\n\t\t\t\tif pubActive.Load() {\n\t\t\t\t\tfor i := 0; i < numStreams; i++ {\n\t\t\t\t\t\tfor j := 0; j < numConsumersPer; j++ {\n\t\t\t\t\t\t\tsubj := fmt.Sprintf(\"A-%d.%d\", i, j)\n\t\t\t\t\t\t\t// Don't care about errors here for this test.\n\t\t\t\t\t\t\tjs.Publish(subj, msg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase <-qch:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tpubActive.Store(true)\n\n\tfor i := 0; i < numPublishers; i++ {\n\t\tgo doPublish()\n\t}\n\n\t// Let run for a bit.\n\ttime.Sleep(20 * time.Second)\n\n\t// Do a rolling restart.\n\tfor _, s := range c.servers {\n\t\tt.Logf(\"Shutdown %v\\n\", s)\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t\ttime.Sleep(20 * time.Second)\n\t\tt.Logf(\"Restarting %v\\n\", s)\n\t\ts = c.restartServer(s)\n\t\tc.waitOnServerHealthz(s)\n\t}\n\n\t// Let run for a bit longer.\n\ttime.Sleep(10 * time.Second)\n\n\t// Stop pubs.\n\tpubActive.Store(false)\n\n\t// Let settle.\n\ttime.Sleep(10 * time.Second)\n\tclose(qch)\n\ttime.Sleep(20 * time.Second)\n\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tminAckFloor := func(stream string) (uint64, string) {\n\t\tvar maf uint64 = math.MaxUint64\n\t\tvar consumer string\n\t\tfor i := 0; i < numConsumersPer; i++ {\n\t\t\tcname := fmt.Sprintf(\"C%d\", i)\n\t\t\tci, err := js.ConsumerInfo(stream, cname)\n\t\t\trequire_NoError(t, err)\n\t\t\tif ci.AckFloor.Stream < maf {\n\t\t\t\tmaf = ci.AckFloor.Stream\n\t\t\t\tconsumer = cname\n\t\t\t}\n\t\t}\n\t\treturn maf, consumer\n\t}\n\n\tcheckStreamAcks := func(stream string) {\n\t\tmm.Lock()\n\t\tdefer mm.Unlock()\n\t\tif sam := ackMap[stream]; sam != nil {\n\t\t\tfor seq := 1; ; seq++ {\n\t\t\t\tacks := sam[uint64(seq)]\n\t\t\t\tif acks == nil {\n\t\t\t\t\tif sam[uint64(seq+1)] != nil {\n\t\t\t\t\t\tt.Logf(\"Missing an ack on stream %q for sequence %d\\n\", stream, seq)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(acks) > 1 {\n\t\t\t\t\tt.Logf(\"Multiple acks for %d which is not expected: %+v\", seq, acks)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Now check all streams such that their first sequence is equal to the minimum of all consumers.\n\tfor i := 0; i < numStreams; i++ {\n\t\tstream := fmt.Sprintf(\"A-%d\", i)\n\t\tsi, err := js.StreamInfo(stream)\n\t\trequire_NoError(t, err)\n\n\t\tif maf, consumer := minAckFloor(stream); maf > si.State.FirstSeq {\n\t\t\tt.Logf(\"\\nBAD STATE DETECTED FOR %q, CHECKING OTHER SERVERS! ACK %d vs %+v LEADER %v, CL FOR %q %v\\n\",\n\t\t\t\tstream, maf, si.State, c.streamLeader(globalAccountName, stream), consumer, c.consumerLeader(globalAccountName, stream, consumer))\n\n\t\t\tt.Logf(\"TEST ACKS %+v\\n\", ackMap)\n\n\t\t\tcheckStreamAcks(stream)\n\n\t\t\tfor _, s := range c.servers {\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(stream)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tstate := mset.state()\n\t\t\t\tt.Logf(\"Server %v Stream STATE %+v\\n\", s, state)\n\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tif sm, err := mset.store.LoadMsg(state.FirstSeq, &smv); err == nil {\n\t\t\t\t\tt.Logf(\"Subject for msg %d is %q\", state.FirstSeq, sm.subj)\n\t\t\t\t} else {\n\t\t\t\t\tt.Logf(\"Could not retrieve msg for %d: %v\", state.FirstSeq, err)\n\t\t\t\t}\n\n\t\t\t\tif len(mset.preAcks) > 0 {\n\t\t\t\t\tt.Logf(\"%v preAcks %+v\\n\", s, mset.preAcks)\n\t\t\t\t}\n\n\t\t\t\tfor _, o := range mset.consumers {\n\t\t\t\t\tostate, err := o.store.State()\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tt.Logf(\"Consumer STATE for %q is %+v\\n\", o.name, ostate)\n\t\t\t\t}\n\t\t\t}\n\t\t\tt.Fatalf(\"BAD STATE: ACKFLOOR > FIRST %d vs %d\\n\", maf, si.State.FirstSeq)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceFileStoreNumPending(t *testing.T) {\n\t// No need for all permutations here.\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{\n\t\tStoreDir: storeDir,\n\t\tBlockSize: 2 * 1024, // Create many blocks on purpose.\n\t}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"*.*.*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\ttokens := []string{\"foo\", \"bar\", \"baz\"}\n\tgenSubj := func() string {\n\t\treturn fmt.Sprintf(\"%s.%s.%s.%s\",\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t\ttokens[rand.Intn(len(tokens))],\n\t\t)\n\t}\n\n\tfor i := 0; i < 50_000; i++ {\n\t\tsubj := genSubj()\n\t\t_, _, err := fs.StoreMsg(subj, nil, []byte(\"Hello World\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tstate := fs.State()\n\n\t// Scan one by one for sanity check against other calculations.\n\tsanityCheck := func(sseq uint64, filter string) SimpleState {\n\t\tt.Helper()\n\t\tvar ss SimpleState\n\t\tvar smv StoreMsg\n\t\t// For here we know 0 is invalid, set to 1.\n\t\tif sseq == 0 {\n\t\t\tsseq = 1\n\t\t}\n\t\tfor seq := sseq; seq <= state.LastSeq; seq++ {\n\t\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Encountered error %v loading sequence: %d\", err, seq)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif subjectIsSubsetMatch(sm.subj, filter) {\n\t\t\t\tss.Msgs++\n\t\t\t\tss.Last = seq\n\t\t\t\tif ss.First == 0 || seq < ss.First {\n\t\t\t\t\tss.First = seq\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn ss\n\t}\n\n\tcheck := func(sseq uint64, filter string) {\n\t\tt.Helper()\n\t\tnp, lvs, err := fs.NumPending(sseq, filter, false)\n\t\trequire_NoError(t, err)\n\t\tss, err := fs.FilteredState(sseq, filter)\n\t\trequire_NoError(t, err)\n\t\tsss := sanityCheck(sseq, filter)\n\t\tif lvs != state.LastSeq {\n\t\t\tt.Fatalf(\"Expected NumPending to return valid through last of %d but got %d\", state.LastSeq, lvs)\n\t\t}\n\t\tif ss.Msgs != np {\n\t\t\tt.Fatalf(\"NumPending of %d did not match ss.Msgs of %d\", np, ss.Msgs)\n\t\t}\n\t\tif ss != sss {\n\t\t\tt.Fatalf(\"Failed sanity check, expected %+v got %+v\", sss, ss)\n\t\t}\n\t}\n\n\tsanityCheckLastOnly := func(sseq uint64, filter string) SimpleState {\n\t\tt.Helper()\n\t\tvar ss SimpleState\n\t\tvar smv StoreMsg\n\t\t// For here we know 0 is invalid, set to 1.\n\t\tif sseq == 0 {\n\t\t\tsseq = 1\n\t\t}\n\t\tseen := make(map[string]bool)\n\t\tfor seq := state.LastSeq; seq >= sseq; seq-- {\n\t\t\tsm, err := fs.LoadMsg(seq, &smv)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Encountered error %v loading sequence: %d\", err, seq)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !seen[sm.subj] && subjectIsSubsetMatch(sm.subj, filter) {\n\t\t\t\tss.Msgs++\n\t\t\t\tif ss.Last == 0 {\n\t\t\t\t\tss.Last = seq\n\t\t\t\t}\n\t\t\t\tif ss.First == 0 || seq < ss.First {\n\t\t\t\t\tss.First = seq\n\t\t\t\t}\n\t\t\t\tseen[sm.subj] = true\n\t\t\t}\n\t\t}\n\t\treturn ss\n\t}\n\n\tcheckLastOnly := func(sseq uint64, filter string) {\n\t\tt.Helper()\n\t\tnp, lvs, err := fs.NumPending(sseq, filter, true)\n\t\trequire_NoError(t, err)\n\t\tss := sanityCheckLastOnly(sseq, filter)\n\t\tif lvs != state.LastSeq {\n\t\t\tt.Fatalf(\"Expected NumPending to return valid through last of %d but got %d\", state.LastSeq, lvs)\n\t\t}\n\t\tif ss.Msgs != np {\n\t\t\tt.Fatalf(\"NumPending of %d did not match ss.Msgs of %d\", np, ss.Msgs)\n\t\t}\n\t}\n\n\tstartSeqs := []uint64{0, 1, 2, 200, 444, 555, 2222, 8888, 12_345, 28_222, 33_456, 44_400, 49_999}\n\tcheckSubs := []string{\"foo.>\", \"*.bar.>\", \"foo.bar.*.baz\", \"*.bar.>\", \"*.foo.bar.*\", \"foo.foo.bar.baz\"}\n\n\tfor _, filter := range checkSubs {\n\t\tfor _, start := range startSeqs {\n\t\t\tcheck(start, filter)\n\t\t\tcheckLastOnly(start, filter)\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterUnbalancedInterestMultipleConsumers(t *testing.T) {\n\tc, np := createStretchUnbalancedCluster(t)\n\tdefer c.shutdown()\n\tdefer np.stop()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now create the stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"EVENTS\",\n\t\tSubjects: []string{\"EV.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure it's leader is on S2.\n\tsl := c.servers[1]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(globalAccountName, \"EVENTS\")\n\t\tif s := c.streamLeader(globalAccountName, \"EVENTS\"); s != sl {\n\t\t\ts.JetStreamStepdownStream(globalAccountName, \"EVENTS\")\n\t\t\treturn fmt.Errorf(\"Server %s is not stream leader yet\", sl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Create a fast ack consumer.\n\t_, err = js.Subscribe(\"EV.NEW\", func(m *nats.Msg) {\n\t\tm.Ack()\n\t}, nats.Durable(\"C\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Make sure the consumer leader is on S3.\n\tcl := c.servers[2]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"C\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"C\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"C\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", cl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Connect a client directly to the stream leader.\n\tnc, js = jsClientConnect(t, sl)\n\tdefer nc.Close()\n\n\t// Now create a pull subscriber.\n\tsub, err := js.PullSubscribe(\"EV.NEW\", \"D\", nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Make sure this consumer leader is on S1.\n\tcl = c.servers[0]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"D\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"D\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"D\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", cl)\n\t\t}\n\t\treturn nil\n\t})\n\n\tnumToSend := 1000\n\tfor i := 0; i < numToSend; i++ {\n\t\t_, err := js.PublishAsync(\"EV.NEW\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Now make sure we can pull messages since we have not acked.\n\t// The bug is that the acks arrive on S1 faster then the messages but we want to\n\t// make sure we do not remove prematurely.\n\tmsgs, err := sub.Fetch(100, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 100)\n\tfor _, m := range msgs {\n\t\tm.AckSync()\n\t}\n\n\tci, err := js.ConsumerInfo(\"EVENTS\", \"D\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, uint64(numToSend-100))\n\trequire_Equal(t, ci.NumAckPending, 0)\n\trequire_Equal(t, ci.Delivered.Stream, 100)\n\trequire_Equal(t, ci.AckFloor.Stream, 100)\n\n\t// Check stream state on all servers.\n\t// Since acks result in messages to be removed through proposals,\n\t// it could take some time to be reflected in the stream state.\n\tcheckFor(t, 5*time.Second, 500*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"EVENTS\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tstate := mset.state()\n\t\t\tif state.Msgs != 900 {\n\t\t\t\treturn fmt.Errorf(\"expected state.Msgs=900, got %d\", state.Msgs)\n\t\t\t}\n\t\t\tif state.FirstSeq != 101 {\n\t\t\t\treturn fmt.Errorf(\"expected state.FirstSeq=101, got %d\", state.FirstSeq)\n\t\t\t}\n\t\t\tif state.LastSeq != 1000 {\n\t\t\t\treturn fmt.Errorf(\"expected state.LastSeq=1000, got %d\", state.LastSeq)\n\t\t\t}\n\t\t\tif state.Consumers != 2 {\n\t\t\t\treturn fmt.Errorf(\"expected state.Consumers=2, got %d\", state.Consumers)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\tmsgs, err = sub.Fetch(900, nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(msgs), 900)\n\tfor _, m := range msgs {\n\t\tm.AckSync()\n\t}\n\n\t// Let acks propagate.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Check final stream state on all servers.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.GlobalAccount().lookupStream(\"EVENTS\")\n\t\trequire_NoError(t, err)\n\t\tstate := mset.state()\n\t\trequire_Equal(t, state.Msgs, 0)\n\t\trequire_Equal(t, state.FirstSeq, 1001)\n\t\trequire_Equal(t, state.LastSeq, 1000)\n\t\trequire_Equal(t, state.Consumers, 2)\n\t\t// Now check preAcks\n\t\tmset.mu.RLock()\n\t\tnumPreAcks := len(mset.preAcks)\n\t\tmset.mu.RUnlock()\n\t\trequire_Len(t, numPreAcks, 0)\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterUnbalancedInterestMultipleFilteredConsumers(t *testing.T) {\n\tc, np := createStretchUnbalancedCluster(t)\n\tdefer c.shutdown()\n\tdefer np.stop()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now create the stream.\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"EVENTS\",\n\t\tSubjects: []string{\"EV.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure it's leader is on S2.\n\tsl := c.servers[1]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(globalAccountName, \"EVENTS\")\n\t\tif s := c.streamLeader(globalAccountName, \"EVENTS\"); s != sl {\n\t\t\ts.JetStreamStepdownStream(globalAccountName, \"EVENTS\")\n\t\t\treturn fmt.Errorf(\"Server %s is not stream leader yet\", sl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Create a fast ack consumer.\n\t_, err = js.Subscribe(\"EV.NEW\", func(m *nats.Msg) {\n\t\tm.Ack()\n\t}, nats.Durable(\"C\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Make sure the consumer leader is on S3.\n\tcl := c.servers[2]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"C\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"C\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"C\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", cl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Connect a client directly to the stream leader.\n\tnc, js = jsClientConnect(t, sl)\n\tdefer nc.Close()\n\n\t// Now create another fast ack consumer.\n\t_, err = js.Subscribe(\"EV.UPDATED\", func(m *nats.Msg) {\n\t\tm.Ack()\n\t}, nats.Durable(\"D\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Make sure this consumer leader is on S1.\n\tcl = c.servers[0]\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"D\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"D\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"D\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", cl)\n\t\t}\n\t\treturn nil\n\t})\n\n\tnumToSend := 500\n\tfor i := 0; i < numToSend; i++ {\n\t\t_, err := js.PublishAsync(\"EV.NEW\", nil)\n\t\trequire_NoError(t, err)\n\t\t_, err = js.PublishAsync(\"EV.UPDATED\", nil)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Let acks propagate.\n\ttime.Sleep(250 * time.Millisecond)\n\n\tci, err := js.ConsumerInfo(\"EVENTS\", \"D\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, ci.NumPending, 0)\n\trequire_Equal(t, ci.NumAckPending, 0)\n\trequire_Equal(t, ci.Delivered.Consumer, 500)\n\trequire_Equal(t, ci.Delivered.Stream, 1000)\n\trequire_Equal(t, ci.AckFloor.Consumer, 500)\n\trequire_Equal(t, ci.AckFloor.Stream, 1000)\n\n\t// Check final stream state on all servers.\n\tfor _, s := range c.servers {\n\t\tmset, err := s.GlobalAccount().lookupStream(\"EVENTS\")\n\t\trequire_NoError(t, err)\n\t\tstate := mset.state()\n\t\trequire_Equal(t, state.Msgs, 0)\n\t\trequire_Equal(t, state.FirstSeq, 1001)\n\t\trequire_Equal(t, state.LastSeq, 1000)\n\t\trequire_Equal(t, state.Consumers, 2)\n\t\t// Now check preAcks\n\t\tmset.mu.RLock()\n\t\tnumPreAcks := len(mset.preAcks)\n\t\tmset.mu.RUnlock()\n\t\trequire_Len(t, numPreAcks, 0)\n\t}\n}\n\nfunc TestNoRaceParallelStreamAndConsumerCreation(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// stream config.\n\tscfg := &StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\", \"bar\"},\n\t\tMaxMsgs: 10,\n\t\tStorage: FileStorage,\n\t\tReplicas: 1,\n\t}\n\n\t// Will do these direct against the low level API to really make\n\t// sure parallel creation ok.\n\tnp := 1000\n\tstartCh := make(chan bool)\n\terrCh := make(chan error, np)\n\twg := sync.WaitGroup{}\n\twg.Add(np)\n\n\tvar streams sync.Map\n\n\tfor i := 0; i < np; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\tif mset, err := s.GlobalAccount().addStream(scfg); err != nil {\n\t\t\t\tt.Logf(\"Stream create got an error: %v\", err)\n\t\t\t\terrCh <- err\n\t\t\t} else {\n\t\t\t\tstreams.Store(mset, true)\n\t\t\t}\n\t\t}()\n\t}\n\ttime.Sleep(100 * time.Millisecond)\n\tclose(startCh)\n\twg.Wait()\n\n\t// Check for no errors.\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n\n\t// Now make sure we really only created one stream.\n\tvar numStreams int\n\tstreams.Range(func(k, v any) bool {\n\t\tnumStreams++\n\t\treturn true\n\t})\n\tif numStreams > 1 {\n\t\tt.Fatalf(\"Expected only one stream to be really created, got %d out of %d attempts\", numStreams, np)\n\t}\n\n\t// Also make sure we cleanup the inflight entries for streams.\n\tgacc := s.GlobalAccount()\n\t_, jsa, err := gacc.checkForJetStream()\n\trequire_NoError(t, err)\n\tvar numEntries int\n\tjsa.inflight.Range(func(k, v any) bool {\n\t\tnumEntries++\n\t\treturn true\n\t})\n\tif numEntries > 0 {\n\t\tt.Fatalf(\"Expected no inflight entries to be left over, got %d\", numEntries)\n\t}\n\n\t// Now do consumers.\n\tmset, err := gacc.lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcfg := &ConsumerConfig{\n\t\tDeliverSubject: \"to\",\n\t\tName: \"DLC\",\n\t\tAckPolicy: AckExplicit,\n\t}\n\n\tstartCh = make(chan bool)\n\terrCh = make(chan error, np)\n\twg.Add(np)\n\n\tvar consumers sync.Map\n\n\tfor i := 0; i < np; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\tif _, err = mset.addConsumer(cfg); err != nil {\n\t\t\t\tt.Logf(\"Consumer create got an error: %v\", err)\n\t\t\t\terrCh <- err\n\t\t\t} else {\n\t\t\t\tconsumers.Store(mset, true)\n\t\t\t}\n\t\t}()\n\t}\n\ttime.Sleep(100 * time.Millisecond)\n\tclose(startCh)\n\twg.Wait()\n\n\t// Check for no errors.\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n\n\t// Now make sure we really only created one stream.\n\tvar numConsumers int\n\tconsumers.Range(func(k, v any) bool {\n\t\tnumConsumers++\n\t\treturn true\n\t})\n\tif numConsumers > 1 {\n\t\tt.Fatalf(\"Expected only one consumer to be really created, got %d out of %d attempts\", numConsumers, np)\n\t}\n}\n\nfunc TestNoRaceRoutePool(t *testing.T) {\n\tvar dur1 time.Duration\n\tvar dur2 time.Duration\n\n\ttotal := 1_000_000\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize int\n\t}{\n\t\t{\"no pooling\", 0},\n\t\t{\"pooling\", 5},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\tport: -1\n\t\t\taccounts {\n\t\t\t\tA { users: [{user: \"A\", password: \"A\"}] }\n\t\t\t\tB { users: [{user: \"B\", password: \"B\"}] }\n\t\t\t\tC { users: [{user: \"C\", password: \"C\"}] }\n\t\t\t\tD { users: [{user: \"D\", password: \"D\"}] }\n\t\t\t\tE { users: [{user: \"E\", password: \"E\"}] }\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tport: -1\n\t\t\t\tname: \"local\"\n\t\t\t\t%s\n\t\t\t\tpool_size: %d\n\t\t\t}\n\t\t`\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.poolSize)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\t\t\ttest.poolSize)))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(5)\n\n\t\t\tsendAndRecv := func(acc string) (*nats.Conn, *nats.Conn) {\n\t\t\t\tt.Helper()\n\n\t\t\t\ts2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(acc, acc))\n\t\t\t\tcount := 0\n\t\t\t\tnatsSub(t, s2nc, \"foo\", func(_ *nats.Msg) {\n\t\t\t\t\tif count++; count == total {\n\t\t\t\t\t\twg.Done()\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t\tnatsFlush(t, s2nc)\n\n\t\t\t\ts1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(acc, acc))\n\n\t\t\t\tcheckSubInterest(t, s1, acc, \"foo\", time.Second)\n\t\t\t\treturn s2nc, s1nc\n\t\t\t}\n\n\t\t\tvar rcv = [5]*nats.Conn{}\n\t\t\tvar snd = [5]*nats.Conn{}\n\t\t\taccs := []string{\"A\", \"B\", \"C\", \"D\", \"E\"}\n\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\trcv[i], snd[i] = sendAndRecv(accs[i])\n\t\t\t\tdefer rcv[i].Close()\n\t\t\t\tdefer snd[i].Close()\n\t\t\t}\n\n\t\t\tpayload := []byte(\"some message\")\n\t\t\tstart := time.Now()\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\tgo func(idx int) {\n\t\t\t\t\tfor i := 0; i < total; i++ {\n\t\t\t\t\t\tsnd[idx].Publish(\"foo\", payload)\n\t\t\t\t\t}\n\t\t\t\t}(i)\n\t\t\t}\n\n\t\t\twg.Wait()\n\t\t\tdur := time.Since(start)\n\t\t\tif test.poolSize == 0 {\n\t\t\t\tdur1 = dur\n\t\t\t} else {\n\t\t\t\tdur2 = dur\n\t\t\t}\n\t\t})\n\t}\n\tperf1 := float64(total*5) / dur1.Seconds()\n\tt.Logf(\"No pooling: %.0f msgs/sec\", perf1)\n\tperf2 := float64(total*5) / dur2.Seconds()\n\tt.Logf(\"Pooling : %.0f msgs/sec\", perf2)\n\tt.Logf(\"Gain : %.2fx\", perf2/perf1)\n}\n\nfunc testNoRaceRoutePerAccount(t *testing.T, useWildCard bool) {\n\tvar dur1 time.Duration\n\tvar dur2 time.Duration\n\n\taccounts := make([]string, 5)\n\tfor i := 0; i < 5; i++ {\n\t\takp, _ := nkeys.CreateAccount()\n\t\tpub, _ := akp.PublicKey()\n\t\taccounts[i] = pub\n\t}\n\trouteAccs := fmt.Sprintf(\"accounts: [\\\"%s\\\", \\\"%s\\\", \\\"%s\\\", \\\"%s\\\", \\\"%s\\\"]\",\n\t\taccounts[0], accounts[1], accounts[2], accounts[3], accounts[4])\n\n\ttotal := 1_000_000\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tdedicated bool\n\t}{\n\t\t{\"route for all accounts\", false},\n\t\t{\"route per account\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\tserver_name: \"%s\"\n\t\t\tport: -1\n\t\t\taccounts {\n\t\t\t\t%s { users: [{user: \"0\", password: \"0\"}] }\n\t\t\t\t%s { users: [{user: \"1\", password: \"1\"}] }\n\t\t\t\t%s { users: [{user: \"2\", password: \"2\"}] }\n\t\t\t\t%s { users: [{user: \"3\", password: \"3\"}] }\n\t\t\t\t%s { users: [{user: \"4\", password: \"4\"}] }\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tport: -1\n\t\t\t\tname: \"local\"\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t\t}\n\t\t`\n\t\t\tvar racc string\n\t\t\tif test.dedicated {\n\t\t\t\tracc = routeAccs\n\t\t\t} else {\n\t\t\t\tracc = _EMPTY_\n\t\t\t}\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\",\n\t\t\t\taccounts[0], accounts[1], accounts[2], accounts[3],\n\t\t\t\taccounts[4], _EMPTY_, racc)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\",\n\t\t\t\taccounts[0], accounts[1], accounts[2], accounts[3], accounts[4],\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\t\t\tracc)))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(5)\n\n\t\t\tsendAndRecv := func(acc string, user string) (*nats.Conn, *nats.Conn) {\n\t\t\t\tt.Helper()\n\n\t\t\t\ts2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, user))\n\t\t\t\tcount := 0\n\t\t\t\tvar subj string\n\t\t\t\tvar checkSubj string\n\t\t\t\tif useWildCard {\n\t\t\t\t\tsubj, checkSubj = \"foo.*\", \"foo.0\"\n\t\t\t\t} else {\n\t\t\t\t\tsubj, checkSubj = \"foo\", \"foo\"\n\t\t\t\t}\n\t\t\t\tnatsSub(t, s2nc, subj, func(_ *nats.Msg) {\n\t\t\t\t\tif count++; count == total {\n\t\t\t\t\t\twg.Done()\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t\tnatsFlush(t, s2nc)\n\n\t\t\t\ts1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, user))\n\n\t\t\t\tcheckSubInterest(t, s1, acc, checkSubj, time.Second)\n\t\t\t\treturn s2nc, s1nc\n\t\t\t}\n\n\t\t\tvar rcv = [5]*nats.Conn{}\n\t\t\tvar snd = [5]*nats.Conn{}\n\t\t\tusers := []string{\"0\", \"1\", \"2\", \"3\", \"4\"}\n\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\trcv[i], snd[i] = sendAndRecv(accounts[i], users[i])\n\t\t\t\tdefer rcv[i].Close()\n\t\t\t\tdefer snd[i].Close()\n\t\t\t}\n\n\t\t\tpayload := []byte(\"some message\")\n\t\t\tstart := time.Now()\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\tgo func(idx int) {\n\t\t\t\t\tfor i := 0; i < total; i++ {\n\t\t\t\t\t\tvar subj string\n\t\t\t\t\t\tif useWildCard {\n\t\t\t\t\t\t\tsubj = fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tsubj = \"foo\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsnd[idx].Publish(subj, payload)\n\t\t\t\t\t}\n\t\t\t\t}(i)\n\t\t\t}\n\n\t\t\twg.Wait()\n\t\t\tdur := time.Since(start)\n\t\t\tif !test.dedicated {\n\t\t\t\tdur1 = dur\n\t\t\t} else {\n\t\t\t\tdur2 = dur\n\t\t\t}\n\t\t})\n\t}\n\tperf1 := float64(total*5) / dur1.Seconds()\n\tt.Logf(\"Route for all accounts: %.0f msgs/sec\", perf1)\n\tperf2 := float64(total*5) / dur2.Seconds()\n\tt.Logf(\"Route per account : %.0f msgs/sec\", perf2)\n\tt.Logf(\"Gain : %.2fx\", perf2/perf1)\n}\n\nfunc TestNoRaceRoutePerAccount(t *testing.T) {\n\ttestNoRaceRoutePerAccount(t, false)\n}\n\nfunc TestNoRaceRoutePerAccountSubWithWildcard(t *testing.T) {\n\ttestNoRaceRoutePerAccount(t, true)\n}\n\n// This test, which checks that messages are not duplicated when pooling or\n// per-account routes are reloaded, would cause a DATA RACE that is not\n// specific to the changes for pooling/per_account. For this reason, this\n// test is located in the norace_test.go file.\nfunc TestNoRaceRoutePoolAndPerAccountConfigReload(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSizeBefore string\n\t\tpoolSizeAfter string\n\t\taccountsBefore string\n\t\taccountsAfter string\n\t}{\n\t\t{\"from no pool to pool\", _EMPTY_, \"pool_size: 2\", _EMPTY_, _EMPTY_},\n\t\t{\"increase pool size\", \"pool_size: 2\", \"pool_size: 5\", _EMPTY_, _EMPTY_},\n\t\t{\"decrease pool size\", \"pool_size: 5\", \"pool_size: 2\", _EMPTY_, _EMPTY_},\n\t\t{\"from pool to no pool\", \"pool_size: 5\", _EMPTY_, _EMPTY_, _EMPTY_},\n\t\t{\"from no account to account\", _EMPTY_, _EMPTY_, _EMPTY_, \"accounts: [\\\"A\\\"]\"},\n\t\t{\"add account\", _EMPTY_, _EMPTY_, \"accounts: [\\\"B\\\"]\", \"accounts: [\\\"A\\\",\\\"B\\\"]\"},\n\t\t{\"remove account\", _EMPTY_, _EMPTY_, \"accounts: [\\\"A\\\",\\\"B\\\"]\", \"accounts: [\\\"B\\\"]\"},\n\t\t{\"from account to no account\", _EMPTY_, _EMPTY_, \"accounts: [\\\"A\\\"]\", _EMPTY_},\n\t\t{\"increase pool size and add account\", \"pool_size: 2\", \"pool_size: 3\", \"accounts: [\\\"B\\\"]\", \"accounts: [\\\"B\\\",\\\"A\\\"]\"},\n\t\t{\"decrease pool size and remove account\", \"pool_size: 3\", \"pool_size: 2\", \"accounts: [\\\"A\\\",\\\"B\\\"]\", \"accounts: [\\\"B\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttmplA := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"A\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tcluster: {\n\t\t\t\t\tport: -1\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, test.poolSizeBefore, test.accountsBefore)))\n\t\t\tsrva, optsA := RunServerWithConfig(confA)\n\t\t\tdefer srva.Shutdown()\n\n\t\t\ttmplB := `\n\t\t\t\tport: -1\n\t\t\t\tserver_name: \"B\"\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\t\t\tB { users: [{user: b, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tcluster: {\n\t\t\t\t\tport: -1\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(tmplB, test.poolSizeBefore, test.accountsBefore, optsA.Cluster.Port)))\n\t\t\tsrvb, _ := RunServerWithConfig(confB)\n\t\t\tdefer srvb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tncA := natsConnect(t, srva.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\n\t\t\tsub := natsSubSync(t, ncA, \"foo\")\n\t\t\tsub.SetPendingLimits(-1, -1)\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", time.Second)\n\n\t\t\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer ncB.Close()\n\n\t\t\twg := sync.WaitGroup{}\n\t\t\twg.Add(1)\n\t\t\tch := make(chan struct{})\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\n\t\t\t\tfor i := 0; ; i++ {\n\t\t\t\t\tncB.Publish(\"foo\", []byte(fmt.Sprintf(\"%d\", i)))\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ch:\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t\tif i%300 == 0 {\n\t\t\t\t\t\ttime.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tvar l *captureErrorLogger\n\t\t\tif test.accountsBefore != _EMPTY_ && test.accountsAfter == _EMPTY_ {\n\t\t\t\tl = &captureErrorLogger{errCh: make(chan string, 100)}\n\t\t\t\tsrva.SetLogger(l, false, false)\n\t\t\t}\n\n\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(tmplA, test.poolSizeAfter, test.accountsAfter))\n\t\t\ttime.Sleep(125 * time.Millisecond)\n\t\t\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(tmplB, test.poolSizeAfter, test.accountsAfter, optsA.Cluster.Port))\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", time.Second)\n\n\t\t\tif l != nil {\n\t\t\t\t// Errors regarding \"No route for account\" should stop\n\t\t\t\tvar ok bool\n\t\t\t\tfor numErrs := 0; !ok && numErrs < 10; {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase e := <-l.errCh:\n\t\t\t\t\t\tif strings.Contains(e, \"No route for account\") {\n\t\t\t\t\t\t\tnumErrs++\n\t\t\t\t\t\t}\n\t\t\t\t\tcase <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond):\n\t\t\t\t\t\tok = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"Still report of no route for account\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tclose(ch)\n\t\t\twg.Wait()\n\n\t\t\tfor prev := -1; ; {\n\t\t\t\tmsg, err := sub.NextMsg(50 * time.Millisecond)\n\t\t\t\tif err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tcur, _ := strconv.Atoi(string(msg.Data))\n\t\t\t\tif cur <= prev {\n\t\t\t\t\tt.Fatalf(\"Previous was %d, got %d\", prev, cur)\n\t\t\t\t}\n\t\t\t\tprev = cur\n\t\t\t}\n\t\t})\n\t}\n}\n\n// This test ensures that outbound queues don't cause a run on\n// memory when sending something to lots of clients.\nfunc TestNoRaceClientOutboundQueueMemory(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tvar before runtime.MemStats\n\tvar after runtime.MemStats\n\n\tvar err error\n\tclients := make([]*nats.Conn, 50000)\n\twait := &sync.WaitGroup{}\n\twait.Add(len(clients))\n\n\tfor i := 0; i < len(clients); i++ {\n\t\tclients[i], err = nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.InProcessServer(s))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer clients[i].Close()\n\n\t\tclients[i].Subscribe(\"test\", func(m *nats.Msg) {\n\t\t\twait.Done()\n\t\t})\n\t}\n\n\truntime.GC()\n\truntime.ReadMemStats(&before)\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port), nats.InProcessServer(s))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tvar m [48000]byte\n\tif err = nc.Publish(\"test\", m[:]); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\twait.Wait()\n\n\truntime.GC()\n\truntime.ReadMemStats(&after)\n\n\thb, ha := float64(before.HeapAlloc), float64(after.HeapAlloc)\n\tms := float64(len(m))\n\tdiff := float64(ha) - float64(hb)\n\tinc := (diff / float64(hb)) * 100\n\n\tif inc > 10 {\n\t\tt.Logf(\"Message size: %.1fKB\\n\", ms/1024)\n\t\tt.Logf(\"Subscribed clients: %d\\n\", len(clients))\n\t\tt.Logf(\"Heap allocs before: %.1fMB\\n\", hb/1024/1024)\n\t\tt.Logf(\"Heap allocs after: %.1fMB\\n\", ha/1024/1024)\n\t\tt.Logf(\"Heap allocs delta: %.1f%%\\n\", inc)\n\n\t\tt.Fatalf(\"memory increase was %.1f%% (should be <= 10%%)\", inc)\n\t}\n}\n" }, { "path": "server/norace_2_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race && !skip_no_race_tests && !skip_no_race_2_tests\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\tcrand \"crypto/rand\"\n\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// IMPORTANT: Tests in this file are not executed when running with the -race flag.\n// The test name should be prefixed with TestNoRace so we can run only\n// those tests: go test -run=TestNoRace ...\n\nfunc TestNoRaceJetStreamClusterLeafnodeConnectPerf(t *testing.T) {\n\t// Uncomment to run. Needs to be on a big machine. Do not want as part of Travis tests atm.\n\tskip(t)\n\n\ttmpl := strings.Replace(jsClusterAccountsTempl, \"store_dir:\", \"domain: cloud, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"CLOUD\", _EMPTY_, 3, 18033, true)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"STATE\",\n\t\tSubjects: []string{\"STATE.GLOBAL.CELL1.*.>\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\ttmpl = strings.Replace(jsClusterTemplWithSingleFleetLeafNode, \"store_dir:\", \"domain: vehicle, store_dir:\", 1)\n\n\tvar vinSerial int\n\tgenVIN := func() string {\n\t\tvinSerial++\n\t\treturn fmt.Sprintf(\"7PDSGAALXNN%06d\", vinSerial)\n\t}\n\n\tnumVehicles := 500\n\tfor i := 0; i < numVehicles; i++ {\n\t\tstart := time.Now()\n\t\tvin := genVIN()\n\t\tln := c.createLeafNodeWithTemplateNoSystemWithProto(vin, tmpl, \"ws\")\n\t\tnc, js := jsClientConnect(t, ln)\n\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"VEHICLE\",\n\t\t\tSubjects: []string{\"STATE.GLOBAL.LOCAL.>\"},\n\t\t\tSources: []*nats.StreamSource{{\n\t\t\t\tName: \"STATE\",\n\t\t\t\tFilterSubject: fmt.Sprintf(\"STATE.GLOBAL.CELL1.%s.>\", vin),\n\t\t\t\tExternal: &nats.ExternalStream{\n\t\t\t\t\tAPIPrefix: \"$JS.cloud.API\",\n\t\t\t\t\tDeliverPrefix: fmt.Sprintf(\"DELIVER.STATE.GLOBAL.CELL1.%s\", vin),\n\t\t\t\t},\n\t\t\t}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\t// Create the sourced stream.\n\t\tcheckLeafNodeConnectedCount(t, ln, 1)\n\t\tif elapsed := time.Since(start); elapsed > 2*time.Second {\n\t\t\tt.Fatalf(\"Took too long to create leafnode %d connection: %v\", i+1, elapsed)\n\t\t}\n\t\tnc.Close()\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterDifferentRTTInterestBasedStreamPreAck(t *testing.T) {\n\ttmpl := `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\tcluster {\n\t\tname: \"F3\"\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\n\taccounts {\n\t\t$SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] }\n\t}\n\t`\n\n\t// Route Ports\n\t//\t\"S1\": 14622,\n\t//\t\"S2\": 15622,\n\t//\t\"S3\": 16622,\n\n\t// S2 (stream leader) will have a slow path to S1 (via proxy) and S3 (consumer leader) will have a fast path.\n\n\t// Do these in order, S1, S2 (proxy) then S3.\n\tc := &cluster{t: t, servers: make([]*Server, 3), opts: make([]*Options, 3), name: \"F3\"}\n\n\t// S1\n\t// The route connection to S2 must be through a slow proxy\n\tnp12 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, \"route://127.0.0.1:15622\", true)\n\troutes := fmt.Sprintf(\"%s, route://127.0.0.1:16622\", np12.routeURL())\n\tconf := fmt.Sprintf(tmpl, \"S1\", t.TempDir(), 14622, routes)\n\tc.servers[0], c.opts[0] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\t// S2\n\t// The route connection to S1 must be through a slow proxy\n\tnp21 := createNetProxy(10*time.Millisecond, 1024*1024*1024, 1024*1024*1024, \"route://127.0.0.1:14622\", true)\n\troutes = fmt.Sprintf(\"%s, route://127.0.0.1:16622\", np21.routeURL())\n\tconf = fmt.Sprintf(tmpl, \"S2\", t.TempDir(), 15622, routes)\n\tc.servers[1], c.opts[1] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\t// S3\n\tconf = fmt.Sprintf(tmpl, \"S3\", t.TempDir(), 16622, \"route://127.0.0.1:14622, route://127.0.0.1:15622\")\n\tc.servers[2], c.opts[2] = RunServerWithConfig(createConfFile(t, []byte(conf)))\n\n\tc.checkClusterFormed()\n\tc.waitOnClusterReady()\n\tdefer c.shutdown()\n\tdefer np12.stop()\n\tdefer np21.stop()\n\n\tslow := c.servers[0] // Expecting pre-acks here.\n\tsl := c.servers[1] // Stream leader, will publish here.\n\tcl := c.servers[2] // Consumer leader, will consume & ack here.\n\n\tsnc, sjs := jsClientConnect(t, sl)\n\tdefer snc.Close()\n\n\tcnc, cjs := jsClientConnect(t, cl)\n\tdefer cnc.Close()\n\n\t// Now create the stream.\n\t_, err := sjs.AddStream(&nats.StreamConfig{\n\t\tName: \"EVENTS\",\n\t\tSubjects: []string{\"EV.>\"},\n\t\tReplicas: 3,\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure it's leader is on S2.\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnStreamLeader(globalAccountName, \"EVENTS\")\n\t\tif s := c.streamLeader(globalAccountName, \"EVENTS\"); s != sl {\n\t\t\ts.JetStreamStepdownStream(globalAccountName, \"EVENTS\")\n\t\t\treturn fmt.Errorf(\"Server %s is not stream leader yet\", sl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create the consumer.\n\t_, err = sjs.AddConsumer(\"EVENTS\", &nats.ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tDeliverSubject: \"dx\",\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure the consumer leader is on S3.\n\tcheckFor(t, 20*time.Second, 200*time.Millisecond, func() error {\n\t\tc.waitOnConsumerLeader(globalAccountName, \"EVENTS\", \"C\")\n\t\tif s := c.consumerLeader(globalAccountName, \"EVENTS\", \"C\"); s != cl {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"EVENTS\", \"C\")\n\t\t\treturn fmt.Errorf(\"Server %s is not consumer leader yet\", sl)\n\t\t}\n\t\treturn nil\n\t})\n\n\t_, err = cjs.Subscribe(_EMPTY_, func(msg *nats.Msg) {\n\t\tmsg.Ack()\n\t}, nats.BindStream(\"EVENTS\"), nats.Durable(\"C\"), nats.ManualAck())\n\trequire_NoError(t, err)\n\n\t// Publish directly on the stream leader to make it efficient.\n\tfor i := 0; i < 1_000; i++ {\n\t\t_, err := sjs.PublishAsync(\"EV.PAID\", []byte(\"ok\"))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-sjs.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tmset, err := slow.GlobalAccount().lookupStream(\"EVENTS\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tstate := mset.state()\n\t\tif state.LastSeq != 1000 {\n\t\t\treturn fmt.Errorf(\"Haven't received all messages yet (last seq %d)\", state.LastSeq)\n\t\t}\n\t\tmset.mu.RLock()\n\t\tpreAcks := mset.preAcks\n\t\tmset.mu.RUnlock()\n\t\tif preAcks == nil {\n\t\t\treturn fmt.Errorf(\"Expected to have preAcks by now\")\n\t\t}\n\t\tif state.Msgs == 0 {\n\t\t\tmset.mu.RLock()\n\t\t\tlp := len(mset.preAcks)\n\t\t\tmset.mu.RUnlock()\n\t\t\tif lp == 0 {\n\t\t\t\treturn nil\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Expected no preAcks with no msgs, but got %d\", lp)\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Still have %d msgs left\", state.Msgs)\n\t})\n\n}\n\nfunc TestNoRaceCheckAckFloorWithVeryLargeFirstSeqAndNewConsumers(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, _ := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// Make sure to time bound here for the acksync call below.\n\tjs, err := nc.JetStream(nats.MaxWait(200 * time.Millisecond))\n\trequire_NoError(t, err)\n\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"wq-req\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tlargeFirstSeq := uint64(1_200_000_000)\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: largeFirstSeq})\n\trequire_NoError(t, err)\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == largeFirstSeq)\n\n\t// Add a simple request to the stream.\n\tsendStreamMsg(t, nc, \"wq-req\", \"HELP\")\n\n\tsub, err := js.PullSubscribe(\"wq-req\", \"dlc\")\n\trequire_NoError(t, err)\n\n\tmsgs, err := sub.Fetch(1)\n\trequire_NoError(t, err)\n\trequire_True(t, len(msgs) == 1)\n\n\t// The bug is around the checkAckFloor walking the sequences from current ackfloor\n\t// to the first sequence of the stream. We time bound the max wait with the js context\n\t// to 200ms. Since checkAckFloor is spinning and holding up processing of acks this will fail.\n\t// We will short circuit new consumers to fix this one.\n\trequire_NoError(t, msgs[0].AckSync())\n\n\t// Now do again so we move past the new consumer with no ack floor situation.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 2 * largeFirstSeq})\n\trequire_NoError(t, err)\n\tsi, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 2*largeFirstSeq)\n\n\tsendStreamMsg(t, nc, \"wq-req\", \"MORE HELP\")\n\n\t// We check this one directly for this use case.\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\to := mset.lookupConsumer(\"dlc\")\n\trequire_True(t, o != nil)\n\n\t// Purge will move the stream floor by default, so force into the situation where it is back to largeFirstSeq.\n\t// This will not trigger the new consumer logic, but will trigger a walk of the sequence space.\n\t// Fix will be to walk the lesser of the two linear spaces.\n\to.mu.Lock()\n\to.asflr = largeFirstSeq\n\to.mu.Unlock()\n\n\tdone := make(chan bool)\n\tgo func() {\n\t\to.checkAckFloor()\n\t\tdone <- true\n\t}()\n\n\tselect {\n\tcase <-done:\n\t\treturn\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Check ack floor taking too long!\")\n\t}\n}\n\nfunc TestNoRaceReplicatedMirrorWithLargeStartingSequenceOverLeafnode(t *testing.T) {\n\t// Cluster B\n\ttmpl := strings.Replace(jsClusterTempl, \"store_dir:\", \"domain: B, store_dir:\", 1)\n\tc := createJetStreamCluster(t, tmpl, \"B\", _EMPTY_, 3, 22020, true)\n\tdefer c.shutdown()\n\n\t// Cluster A\n\t// Domain is \"A'\n\tlc := c.createLeafNodesWithStartPortAndDomain(\"A\", 3, 22110, \"A\")\n\tdefer lc.shutdown()\n\n\tlc.waitOnClusterReady()\n\n\t// Create a stream on B (HUB/CLOUD) and set its starting sequence very high.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 1_000_000_000})\n\trequire_NoError(t, err)\n\n\t// Send in a small amount of messages.\n\tfor i := 0; i < 1000; i++ {\n\t\tsendStreamMsg(t, nc, \"foo\", \"Hello\")\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 1_000_000_000)\n\n\t// Now try to create a replicated mirror on the leaf cluster.\n\tlnc, ljs := jsClientConnect(t, lc.randomServer())\n\tdefer lnc.Close()\n\n\t_, err = ljs.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tMirror: &nats.StreamSource{\n\t\t\tName: \"TEST\",\n\t\t\tDomain: \"B\",\n\t\t},\n\t})\n\trequire_NoError(t, err)\n\n\t// Make sure we sync quickly.\n\tcheckFor(t, time.Second, 200*time.Millisecond, func() error {\n\t\tsi, err = ljs.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs == 1000 && si.State.FirstSeq == 1_000_000_000 {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Mirror state not correct: %+v\", si.State)\n\t})\n}\n\nfunc TestNoRaceBinaryStreamSnapshotEncodingBasic(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxMsgsPerSubject: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// Set first key\n\tsendStreamMsg(t, nc, \"key:1\", \"hello\")\n\n\t// Set Second key but keep updating it, causing a laggard pattern.\n\tvalue := bytes.Repeat([]byte(\"Z\"), 8*1024)\n\n\tfor i := 0; i <= 1000; i++ {\n\t\t_, err := js.PublishAsync(\"key:2\", value)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Now do more of swiss cheese style.\n\tfor i := 3; i <= 1000; i++ {\n\t\tkey := fmt.Sprintf(\"key:%d\", i)\n\t\t_, err := js.PublishAsync(key, value)\n\t\trequire_NoError(t, err)\n\t\t// Send it twice to create hole right behind it, like swiss cheese.\n\t\t_, err = js.PublishAsync(key, value)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// Make for round numbers for stream state.\n\tsendStreamMsg(t, nc, \"key:2\", \"hello\")\n\tsendStreamMsg(t, nc, \"key:2\", \"world\")\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\trequire_True(t, si.State.FirstSeq == 1)\n\trequire_True(t, si.State.LastSeq == 3000)\n\trequire_True(t, si.State.Msgs == 1000)\n\trequire_True(t, si.State.NumDeleted == 2000)\n\n\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tsnap, err := mset.store.EncodedStreamState(0)\n\trequire_NoError(t, err)\n\n\t// Now decode the snapshot.\n\tss, err := DecodeStreamState(snap)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 3000)\n\trequire_Equal(t, ss.Msgs, 1000)\n\trequire_Equal(t, ss.Deleted.NumDeleted(), 2000)\n}\n\nfunc TestNoRaceFilestoreBinaryStreamSnapshotEncodingLargeGaps(t *testing.T) {\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{\n\t\tStoreDir: storeDir,\n\t\tBlockSize: 512, // Small on purpose to create a lot of blks.\n\t}\n\tfs, err := newFileStore(fcfg, StreamConfig{Name: \"zzz\", Subjects: []string{\"zzz\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tsubj, msg := \"zzz\", bytes.Repeat([]byte(\"X\"), 128)\n\tnumMsgs := 20_000\n\n\tfs.StoreMsg(subj, nil, msg, 0)\n\tfor i := 2; i < numMsgs; i++ {\n\t\tseq, _, err := fs.StoreMsg(subj, nil, nil, 0)\n\t\trequire_NoError(t, err)\n\t\tfs.RemoveMsg(seq)\n\t}\n\tfs.StoreMsg(subj, nil, msg, 0)\n\n\t// The tombstones from above will only be cleaned up when syncing blocks.\n\tfs.syncBlocks()\n\n\tsnap, err := fs.EncodedStreamState(0)\n\trequire_NoError(t, err)\n\trequire_LessThan(t, len(snap), 512)\n\n\t// Now decode the snapshot.\n\tss, err := DecodeStreamState(snap)\n\trequire_NoError(t, err)\n\n\trequire_Equal(t, ss.FirstSeq, 1)\n\trequire_Equal(t, ss.LastSeq, 20_000)\n\trequire_Equal(t, ss.Msgs, 2)\n\trequire_Equal(t, len(ss.Deleted), 1)\n\trequire_Equal(t, ss.Deleted.NumDeleted(), 19_998)\n}\n\nfunc TestNoRaceJetStreamClusterStreamSnapshotCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Client based API\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxMsgsPerSubject: 1,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tmsg := []byte(\"Hello World\")\n\t_, err = js.Publish(\"foo\", msg)\n\trequire_NoError(t, err)\n\n\tfor i := 1; i < 1000; i++ {\n\t\t_, err := js.PublishAsync(\"bar\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsr := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\tsr.Shutdown()\n\n\t// In case we were connected to sr.\n\tnc, js = jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Now create a large gap.\n\tfor i := 0; i < 50_000; i++ {\n\t\t_, err := js.PublishAsync(\"bar\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tsl.JetStreamSnapshotStream(globalAccountName, \"TEST\")\n\n\tsr = c.restartServer(sr)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(sr)\n\tc.waitOnStreamCurrent(sr, globalAccountName, \"TEST\")\n\n\tmset, err := sr.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Make sure it's caught up\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\trequire_Equal(t, state.Msgs, 2)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 51_000)\n\trequire_Equal(t, state.NumDeleted, 51_000-2)\n\n\tsr.Shutdown()\n\n\t_, err = js.Publish(\"baz\", msg)\n\trequire_NoError(t, err)\n\n\tsl.JetStreamSnapshotStream(globalAccountName, \"TEST\")\n\n\tsr = c.restartServer(sr)\n\tc.checkClusterFormed()\n\tc.waitOnServerCurrent(sr)\n\tc.waitOnStreamCurrent(sr, globalAccountName, \"TEST\")\n\n\tmset, err = sr.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.store.FastState(&state)\n\n\trequire_Equal(t, state.Msgs, 3)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 51_001)\n\trequire_Equal(t, state.NumDeleted, 51_001-3)\n}\n\nfunc TestNoRaceStoreStreamEncoderDecoder(t *testing.T) {\n\tcfg := &StreamConfig{\n\t\tName: \"zzz\",\n\t\tSubjects: []string{\"*\"},\n\t\tMaxMsgsPer: 1,\n\t\tStorage: MemoryStorage,\n\t}\n\tms, err := newMemStore(cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*\"}, MaxMsgsPer: 1, Storage: FileStorage},\n\t)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tconst seed = 2222222\n\tmsg := bytes.Repeat([]byte(\"ABC\"), 33) // ~100bytes\n\n\tmaxEncodeTime := 2 * time.Second\n\tmaxEncodeSize := 700 * 1024\n\n\ttest := func(t *testing.T, gs StreamStore) {\n\t\tt.Parallel()\n\t\tprand := rand.New(rand.NewSource(seed))\n\t\ttick := time.NewTicker(time.Second)\n\t\tdefer tick.Stop()\n\t\tdone := time.NewTimer(10 * time.Second)\n\n\t\tfor running := true; running; {\n\t\t\tselect {\n\t\t\tcase <-tick.C:\n\t\t\t\tvar state StreamState\n\t\t\t\tgs.FastState(&state)\n\t\t\t\tif state.NumDeleted == 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tstart := time.Now()\n\t\t\t\tsnap, err := gs.EncodedStreamState(0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\telapsed := time.Since(start)\n\t\t\t\t// Should take <1ms without race but if CI/CD is slow we will give it a bit of room.\n\t\t\t\tif elapsed > maxEncodeTime {\n\t\t\t\t\tt.Logf(\"Encode took longer then expected: %v\", elapsed)\n\t\t\t\t}\n\t\t\t\tif len(snap) > maxEncodeSize {\n\t\t\t\t\tt.Fatalf(\"Expected snapshot size < %v got %v\", friendlyBytes(maxEncodeSize), friendlyBytes(len(snap)))\n\t\t\t\t}\n\t\t\t\tss, err := DecodeStreamState(snap)\n\t\t\t\trequire_True(t, len(ss.Deleted) > 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\tcase <-done.C:\n\t\t\t\trunning = false\n\t\t\tdefault:\n\t\t\t\tkey := strconv.Itoa(prand.Intn(256_000))\n\t\t\t\tgs.StoreMsg(key, nil, msg, 0)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, gs := range []StreamStore{ms, fs} {\n\t\tswitch gs.(type) {\n\t\tcase *memStore:\n\t\t\tt.Run(\"MemStore\", func(t *testing.T) {\n\t\t\t\ttest(t, gs)\n\t\t\t})\n\t\tcase *fileStore:\n\t\t\tt.Run(\"FileStore\", func(t *testing.T) {\n\t\t\t\ttest(t, gs)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterKVWithServerKill(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Setup the KV bucket and use for making assertions.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\t_, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t\tHistory: 10,\n\t})\n\trequire_NoError(t, err)\n\n\t// Total number of keys to range over.\n\tnumKeys := 50\n\n\t// ID is the server id to explicitly connect to.\n\twork := func(ctx context.Context, wg *sync.WaitGroup, id int) {\n\t\tdefer wg.Done()\n\n\t\tnc, js := jsClientConnectEx(t, c.servers[id], []nats.JSOpt{nats.Context(ctx)})\n\t\tdefer nc.Close()\n\n\t\tkv, err := js.KeyValue(\"TEST\")\n\t\trequire_NoError(t, err)\n\n\t\t// 100 messages a second for each single client.\n\t\ttk := time.NewTicker(10 * time.Millisecond)\n\t\tdefer tk.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\n\t\t\tcase <-tk.C:\n\t\t\t\t// Pick a random key within the range.\n\t\t\t\tk := fmt.Sprintf(\"key.%d\", rand.Intn(numKeys))\n\t\t\t\t// Attempt to get a key.\n\t\t\t\te, err := kv.Get(k)\n\t\t\t\t// If found, attempt to update or delete.\n\t\t\t\tif err == nil {\n\t\t\t\t\tif rand.Intn(10) < 3 {\n\t\t\t\t\t\tkv.Delete(k, nats.LastRevision(e.Revision()))\n\t\t\t\t\t} else {\n\t\t\t\t\t\tkv.Update(k, nil, e.Revision())\n\t\t\t\t\t}\n\t\t\t\t} else if errors.Is(err, nats.ErrKeyNotFound) {\n\t\t\t\t\tkv.Create(k, nil)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tvar wg sync.WaitGroup\n\twg.Add(3)\n\n\tgo work(ctx, &wg, 0)\n\tgo work(ctx, &wg, 1)\n\tgo work(ctx, &wg, 2)\n\n\ttime.Sleep(time.Second)\n\n\t// Simulate server stop and restart.\n\tfor i := 0; i < 7; i++ {\n\t\ts := c.randomServer()\n\t\ts.Shutdown()\n\t\tc.waitOnLeader()\n\t\tc.waitOnStreamLeader(globalAccountName, \"KV_TEST\")\n\n\t\t// Wait for a bit and then start the server again.\n\t\ttime.Sleep(time.Duration(rand.Intn(1250)) * time.Millisecond)\n\t\ts = c.restartServer(s)\n\t\tc.waitOnServerCurrent(s)\n\t\tc.waitOnLeader()\n\t\tc.waitOnStreamLeader(globalAccountName, \"KV_TEST\")\n\t\tc.waitOnPeerCount(3)\n\t}\n\n\t// Stop the workload.\n\tcancel()\n\twg.Wait()\n\n\ttype fullState struct {\n\t\tstate StreamState\n\t\tlseq uint64\n\t\tclfs uint64\n\t}\n\n\tgrabState := func(mset *stream) *fullState {\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\t\tvar state StreamState\n\t\tmset.store.FastState(&state)\n\t\treturn &fullState{state, mset.lseq, mset.clfs}\n\t}\n\n\tgrabStore := func(mset *stream) map[string][]uint64 {\n\t\tmset.mu.RLock()\n\t\tstore := mset.store\n\t\tmset.mu.RUnlock()\n\t\tvar state StreamState\n\t\tstore.FastState(&state)\n\t\tstoreMap := make(map[string][]uint64)\n\t\tfor seq := state.FirstSeq; seq <= state.LastSeq; seq++ {\n\t\t\tif sm, err := store.LoadMsg(seq, nil); err == nil {\n\t\t\t\tstoreMap[sm.subj] = append(storeMap[sm.subj], sm.seq)\n\t\t\t}\n\t\t}\n\t\treturn storeMap\n\t}\n\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\t// Current stream leader.\n\t\tsl := c.streamLeader(globalAccountName, \"KV_TEST\")\n\t\tmset, err := sl.GlobalAccount().lookupStream(\"KV_TEST\")\n\t\trequire_NoError(t, err)\n\t\tlstate := grabState(mset)\n\t\tgolden := grabStore(mset)\n\n\t\t// Report messages per server.\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"KV_TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tstate := grabState(mset)\n\t\t\tif !reflect.DeepEqual(state, lstate) {\n\t\t\t\treturn fmt.Errorf(\"Expected follower state\\n%+v\\nto match leader's\\n %+v\", state, lstate)\n\t\t\t}\n\t\t\tsm := grabStore(mset)\n\t\t\tif !reflect.DeepEqual(sm, golden) {\n\t\t\t\tt.Fatalf(\"Expected follower store for %v\\n%+v\\nto match leader's %v\\n %+v\", s, sm, sl, golden)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceFileStoreLargeMsgsAndFirstMatching(t *testing.T) {\n\tsd := t.TempDir()\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: sd, BlockSize: 8 * 1024 * 1024},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\">\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tfor i := 0; i < 150_000; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.bar.%d\", i), nil, nil, 0)\n\t}\n\tfor i := 0; i < 150_000; i++ {\n\t\tfs.StoreMsg(fmt.Sprintf(\"foo.baz.%d\", i), nil, nil, 0)\n\t}\n\trequire_Equal(t, fs.numMsgBlocks(), 2)\n\tfs.mu.RLock()\n\tmb := fs.blks[1]\n\tfs.mu.RUnlock()\n\tfseq := atomic.LoadUint64(&mb.first.seq)\n\t// The -40 leaves enough mb.fss entries to kick in linear scan.\n\tfor seq := fseq; seq < 300_000-40; seq++ {\n\t\tfs.RemoveMsg(uint64(seq))\n\t}\n\tstart := time.Now()\n\tfs.LoadNextMsg(\"*.baz.*\", true, fseq, nil)\n\trequire_True(t, time.Since(start) < 200*time.Microsecond)\n\t// Now remove more to kick into non-linear logic.\n\tfor seq := 300_000 - 40; seq < 300_000; seq++ {\n\t\tfs.RemoveMsg(uint64(seq))\n\t}\n\tstart = time.Now()\n\tfs.LoadNextMsg(\"*.baz.*\", true, fseq, nil)\n\trequire_True(t, time.Since(start) < 200*time.Microsecond)\n}\n\nfunc TestNoRaceWSNoCorruptionWithFrameSizeLimit(t *testing.T) {\n\ttestWSNoCorruptionWithFrameSizeLimit(t, 50000)\n}\n\nfunc TestNoRaceJetStreamAPIDispatchQueuePending(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\t// Setup the KV bucket and use for making assertions.\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*.*\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Queue up 500k messages all with different subjects.\n\t// We want to make num pending for a consumer expensive, so a large subject\n\t// space and wildcards for now does the trick.\n\ttoks := []string{\"foo\", \"bar\", \"baz\"} // for second token.\n\tfor i := 1; i <= 500_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%s.%d\", toks[rand.Intn(len(toks))], i)\n\t\t_, err := js.PublishAsync(subj, nil, nats.StallWait(time.Second))\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// To back up our pending queue we will create lots of filtered, with wildcards, R1 consumers\n\t// from a different server then the one hosting the stream.\n\t// ok to share this connection here.\n\tsldr := c.streamLeader(globalAccountName, \"TEST\")\n\tfor _, s := range c.servers {\n\t\tif s != sldr {\n\t\t\tnc, js = jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\tbreak\n\t\t}\n\t}\n\n\tngr, ncons := 100, 10\n\tstartCh, errCh := make(chan bool), make(chan error, ngr)\n\tvar wg, swg sync.WaitGroup\n\twg.Add(ngr)\n\tswg.Add(ngr)\n\n\t// The wildcard in the filter subject is the key.\n\tcfg := &nats.ConsumerConfig{FilterSubject: \"foo.*.22\"}\n\tvar tt atomic.Int64\n\n\tfor i := 0; i < ngr; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tswg.Done()\n\t\t\t// Make them all fire at once.\n\t\t\t<-startCh\n\n\t\t\tfor i := 0; i < ncons; i++ {\n\t\t\t\tstart := time.Now()\n\t\t\t\tif _, err := js.AddConsumer(\"TEST\", cfg); err != nil {\n\t\t\t\t\terrCh <- err\n\t\t\t\t\tt.Logf(\"Got err creating consumer: %v\", err)\n\t\t\t\t}\n\t\t\t\telapsed := time.Since(start)\n\t\t\t\ttt.Add(int64(elapsed))\n\t\t\t}\n\t\t}()\n\t}\n\tswg.Wait()\n\tclose(startCh)\n\ttime.Sleep(time.Millisecond)\n\tjsz, _ := sldr.Jsz(nil)\n\t// This could be 0 legit, so just log, don't fail.\n\tif jsz.JetStreamStats.API.Inflight == 0 {\n\t\tt.Log(\"Expected a non-zero inflight\")\n\t}\n\twg.Wait()\n\n\tif len(errCh) > 0 {\n\t\tt.Fatalf(\"Expected no errors, got %d: %v\", len(errCh), <-errCh)\n\t}\n}\n\nfunc TestNoRaceJetStreamMirrorAndSourceConsumerFailBackoff(t *testing.T) {\n\towt := srcConsumerWaitTime\n\tsrcConsumerWaitTime = 2 * time.Second\n\tdefer func() { srcConsumerWaitTime = owt }()\n\n\t// Check calculations first.\n\tfor i := 1; i <= 20; i++ {\n\t\tbackoff := calculateRetryBackoff(i)\n\t\tif i < 12 {\n\t\t\trequire_Equal(t, backoff, time.Duration(i)*10*time.Second)\n\t\t} else {\n\t\t\trequire_Equal(t, backoff, retryMaximum)\n\t\t}\n\t}\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*.*\"},\n\t})\n\trequire_NoError(t, err)\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\n\t// Create a mirror.\n\tml := sl\n\t// Make sure not on the same server. Should not happened in general but possible.\n\tfor ml == sl {\n\t\tjs.DeleteStream(\"MIRROR\")\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"MIRROR\",\n\t\t\tMirror: &nats.StreamSource{Name: \"TEST\"},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tml = c.streamLeader(globalAccountName, \"MIRROR\")\n\t}\n\t// Create a source.\n\tsrcl := sl\n\tfor srcl == sl {\n\t\tjs.DeleteStream(\"SOURCE\")\n\t\t_, err = js.AddStream(&nats.StreamConfig{\n\t\t\tName: \"SOURCE\",\n\t\t\tSources: []*nats.StreamSource{{Name: \"TEST\"}},\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tsrcl = c.streamLeader(globalAccountName, \"MIRROR\")\n\t}\n\n\t// Create sub to watch for the consumer create requests.\n\tnc, _ = jsClientConnect(t, ml)\n\tdefer nc.Close()\n\tsub := natsSubSync(t, nc, \"$JS.API.CONSUMER.CREATE.>\")\n\n\t// Kill the server where the source is..\n\tsldr := c.streamLeader(globalAccountName, \"TEST\")\n\tsldr.Shutdown()\n\n\t// Wait for just greater than 5s. We should only see 1 request during this time.\n\ttime.Sleep(6 * time.Second)\n\t// There should have been 2 requests, one for mirror, one for source\n\tn, _, _ := sub.Pending()\n\trequire_Equal(t, n, 2)\n\tvar mreq, sreq int\n\tfor i := 0; i < 2; i++ {\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tif bytes.Contains(msg.Data, []byte(\"$JS.M.\")) {\n\t\t\tmreq++\n\t\t} else if bytes.Contains(msg.Data, []byte(\"$JS.S.\")) {\n\t\t\tsreq++\n\t\t}\n\t}\n\tif mreq != 1 || sreq != 1 {\n\t\tt.Fatalf(\"Consumer create captures invalid: mreq=%v sreq=%v\", mreq, sreq)\n\t}\n\n\t// Now make sure that the fails is set properly.\n\tmset, err := c.streamLeader(globalAccountName, \"MIRROR\").GlobalAccount().lookupStream(\"MIRROR\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tfails := mset.mirror.fails\n\tmset.mu.RUnlock()\n\trequire_Equal(t, fails, 1)\n\n\tmset, err = c.streamLeader(globalAccountName, \"SOURCE\").GlobalAccount().lookupStream(\"SOURCE\")\n\trequire_NoError(t, err)\n\tmset.mu.RLock()\n\tsi := mset.sources[\"TEST > >\"]\n\tmset.mu.RUnlock()\n\trequire_True(t, si != nil)\n\trequire_Equal(t, si.fails, 1)\n}\n\nfunc TestNoRaceJetStreamClusterStreamCatchupLargeInteriorDeletes(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tMaxMsgsPerSubject: 100,\n\t\tReplicas: 1,\n\t}\n\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\tmsg := bytes.Repeat([]byte(\"Z\"), 2*1024)\n\t// We will create lots of interior deletes on our R1 then scale up.\n\t_, err = js.Publish(\"foo.0\", msg)\n\trequire_NoError(t, err)\n\n\t// Create 50k messages randomly from 1-100\n\tfor i := 0; i < 50_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(100)+1)\n\t\tjs.PublishAsync(subj, msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\t// Now create a large gap.\n\tfor i := 0; i < 100_000; i++ {\n\t\tjs.PublishAsync(\"foo.2\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\t// Do 50k random again at end.\n\tfor i := 0; i < 50_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", rand.Intn(100)+1)\n\t\tjs.PublishAsync(subj, msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcfg.Replicas = 2\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Let catchup start.\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\tnl := c.randomNonStreamLeader(globalAccountName, \"TEST\")\n\trequire_True(t, nl != nil)\n\tmset, err := nl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tcheckFor(t, 10*time.Second, 500*time.Millisecond, func() error {\n\t\tstate := mset.state()\n\t\tif state.Msgs == si.State.Msgs {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"Msgs not equal %d vs %d\", state.Msgs, si.State.Msgs)\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterBadRestartsWithHealthzPolling(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.>\"},\n\t\tReplicas: 3,\n\t}\n\t_, err := js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// We will poll healthz at a decent clip and make sure any restart logic works\n\t// correctly with assets coming and going.\n\tch := make(chan struct{})\n\tdefer close(ch)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tcase <-time.After(50 * time.Millisecond):\n\t\t\t\tfor _, s := range c.servers {\n\t\t\t\t\ts.healthz(nil)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tnumConsumers := 500\n\tconsumers := make([]string, 0, numConsumers)\n\n\tvar wg sync.WaitGroup\n\n\tfor i := 0; i < numConsumers; i++ {\n\t\tcname := fmt.Sprintf(\"CONS-%d\", i+1)\n\t\tconsumers = append(consumers, cname)\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.PullSubscribe(\"foo.>\", cname, nats.BindStream(\"TEST\"))\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n\n\t// Make sure all are reported.\n\tc.waitOnAllCurrent()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, _ := s.Jsz(nil)\n\t\t\tif jsz.Consumers != numConsumers {\n\t\t\t\treturn fmt.Errorf(\"%v wrong number of consumers: %d vs %d\", s, jsz.Consumers, numConsumers)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now do same for streams.\n\tnumStreams := 200\n\tstreams := make([]string, 0, numStreams)\n\n\tfor i := 0; i < numStreams; i++ {\n\t\tsname := fmt.Sprintf(\"TEST-%d\", i+1)\n\t\tstreams = append(streams, sname)\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{Name: sname, Replicas: 3})\n\t\t\trequire_NoError(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n\n\t// Make sure all are reported.\n\tc.waitOnAllCurrent()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, _ := s.Jsz(nil)\n\t\t\tif jsz.Streams != numStreams+1 {\n\t\t\t\treturn fmt.Errorf(\"%v wrong number of streams: %d vs %d\", s, jsz.Streams, numStreams+1)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Delete consumers.\n\tfor _, cname := range consumers {\n\t\terr := js.DeleteConsumer(\"TEST\", cname)\n\t\trequire_NoError(t, err)\n\t}\n\t// Make sure reporting goes to zero.\n\tc.waitOnAllCurrent()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, _ := s.Jsz(nil)\n\t\t\tif jsz.Consumers != 0 {\n\t\t\t\treturn fmt.Errorf(\"%v still has %d consumers\", s, jsz.Consumers)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Delete streams\n\tfor _, sname := range streams {\n\t\terr := js.DeleteStream(sname)\n\t\trequire_NoError(t, err)\n\t}\n\terr = js.DeleteStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Make sure reporting goes to zero.\n\tc.waitOnAllCurrent()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tjsz, _ := s.Jsz(nil)\n\t\t\tif jsz.Streams != 0 {\n\t\t\t\treturn fmt.Errorf(\"%v still has %d streams\", s, jsz.Streams)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamKVReplaceWithServerRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\t// Shorten wait time for disconnects.\n\tjs, err := nc.JetStream(nats.MaxWait(time.Second))\n\trequire_NoError(t, err)\n\n\tkv, err := js.CreateKeyValue(&nats.KeyValueConfig{\n\t\tBucket: \"TEST\",\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Manually disable direct get on underlying stream, since this test\n\t// relies on immediate consistency which we can't guarantee with direct gets\n\t// until we provide a solution for this.\n\tsi, err := js.StreamInfo(\"KV_TEST\")\n\trequire_NoError(t, err)\n\tsi.Config.AllowDirect = false\n\t_, err = js.UpdateStream(&si.Config)\n\trequire_NoError(t, err)\n\n\tcreateData := func(n int) []byte {\n\t\tconst letterBytes = \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\"\n\t\tb := make([]byte, n)\n\t\tfor i := range b {\n\t\t\tb[i] = letterBytes[rand.Intn(len(letterBytes))]\n\t\t}\n\t\treturn b\n\t}\n\n\t_, err = kv.Create(\"foo\", createData(160))\n\trequire_NoError(t, err)\n\n\t// Ensure all replicas have applied the key.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\treturn checkState(t, c, globalAccountName, \"KV_TEST\")\n\t})\n\n\tch := make(chan struct{})\n\twg := sync.WaitGroup{}\n\n\t// For counting errors that should not happen.\n\terrCh := make(chan error, 1024)\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tvar lastData []byte\n\t\tvar revision uint64\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\tclose(errCh)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tk, err := kv.Get(\"foo\")\n\t\t\t\tif err == nats.ErrKeyNotFound {\n\t\t\t\t\terrCh <- err\n\t\t\t\t} else if k != nil {\n\t\t\t\t\tif lastData != nil && k.Revision() == revision && !bytes.Equal(lastData, k.Value()) {\n\t\t\t\t\t\terrCh <- fmt.Errorf(\"data loss [%s][rev:%d] expected:[%q] is:[%q]\\n\", \"foo\", revision, lastData, k.Value())\n\t\t\t\t\t}\n\t\t\t\t\tnewData := createData(160)\n\t\t\t\t\tif revision, err = kv.Update(\"foo\", newData, k.Revision()); err == nil {\n\t\t\t\t\t\tlastData = newData\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Wait a short bit.\n\ttime.Sleep(2 * time.Second)\n\tfor _, s := range c.servers {\n\t\ts.Shutdown()\n\t\t// Need to leave servers down for a while to trigger bug properly.\n\t\ttime.Sleep(5 * time.Second)\n\t\ts = c.restartServer(s)\n\t\tc.waitOnServerHealthz(s)\n\t}\n\n\t// Shutdown the go routine above.\n\tclose(ch)\n\t// Wait for it to finish.\n\twg.Wait()\n\n\tif len(errCh) != 0 {\n\t\tfor err := range errCh {\n\t\t\tt.Logf(\"Received err %v during test\", err)\n\t\t}\n\t\tt.Fatalf(\"Encountered errors\")\n\t}\n}\n\nfunc TestNoRaceMemStoreCompactPerformance(t *testing.T) {\n\t//Load MemStore so that it is full\n\tsubj, msg := \"foo\", make([]byte, 1000)\n\tstoredMsgSize := memStoreMsgSize(subj, nil, msg)\n\n\ttoStore := uint64(10_000)\n\ttoStoreOnTop := uint64(1_000)\n\tsetSeqNo := uint64(10_000_000_000)\n\n\texpectedPurge := toStore - 1\n\tmaxBytes := storedMsgSize * toStore\n\n\tms, err := newMemStore(&StreamConfig{Storage: MemoryStorage, MaxBytes: int64(maxBytes)})\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tfor i := uint64(0); i < toStore; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstate := ms.State()\n\trequire_Equal(t, toStore, state.Msgs)\n\trequire_Equal(t, state.Bytes, storedMsgSize*toStore)\n\n\t//1st run: Load additional messages then compact\n\tfor i := uint64(0); i < toStoreOnTop; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstartFirstRun := time.Now()\n\tpurgedFirstRun, _ := ms.Compact(toStore + toStoreOnTop)\n\telapsedFirstRun := time.Since(startFirstRun)\n\trequire_Equal(t, expectedPurge, purgedFirstRun)\n\n\t//set the seq number to a very high value by compacting with a too high seq number\n\tpurgedFull, _ := ms.Compact(setSeqNo)\n\trequire_Equal(t, 1, purgedFull)\n\n\t//2nd run: Compact again\n\tfor i := uint64(0); i < toStore; i++ {\n\t\tms.StoreMsg(subj, nil, msg, 0)\n\t}\n\tstartSecondRun := time.Now()\n\tpurgedSecondRun, _ := ms.Compact(setSeqNo + toStore - 1)\n\telapsedSecondRun := time.Since(startSecondRun)\n\trequire_Equal(t, expectedPurge, purgedSecondRun)\n\n\t//Calculate delta between runs and fail if it is too high\n\trequire_LessThan(t, elapsedSecondRun-elapsedFirstRun, time.Duration(1)*time.Second)\n}\n\nfunc TestNoRaceJetStreamSnapshotsWithSlowAckDontSlowConsumer(t *testing.T) {\n\t// Test takes less time this way.\n\tsnapshotAckTimeout = 500 * time.Millisecond\n\tt.Cleanup(func() {\n\t\tsnapshotAckTimeout = defaultSnapshotAckTimeout\n\t})\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tech := make(chan error)\n\tecb := func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\tif err != nil {\n\t\t\tech <- err\n\t\t}\n\t}\n\tnc, js := jsClientConnect(t, s, nats.ErrorHandler(ecb))\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Put in over 64MB.\n\tmsg, toSend := make([]byte, 1024*1024), 80\n\tcrand.Read(msg)\n\n\tfor i := 0; i < toSend; i++ {\n\t\t_, err := js.Publish(\"foo\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\n\tsreq := &JSApiStreamSnapshotRequest{\n\t\tDeliverSubject: nats.NewInbox(),\n\t\tChunkSize: 1024 * 1024,\n\t}\n\treq, _ := json.Marshal(sreq)\n\trmsg, err := nc.Request(fmt.Sprintf(JSApiStreamSnapshotT, \"TEST\"), req, time.Second)\n\trequire_NoError(t, err)\n\n\tvar resp JSApiStreamSnapshotResponse\n\tjson.Unmarshal(rmsg.Data, &resp)\n\trequire_True(t, resp.Error == nil)\n\n\tdone := make(chan *nats.Msg)\n\tsub, _ := nc.Subscribe(sreq.DeliverSubject, func(m *nats.Msg) {\n\t\t// EOF\n\t\tif len(m.Data) == 0 {\n\t\t\tdone <- m\n\t\t\treturn\n\t\t}\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Check that we do not get disconnected due to slow consumer.\n\tselect {\n\tcase msg := <-done:\n\t\trequire_Equal(t, msg.Header.Get(\"Status\"), \"408\")\n\t\trequire_Equal(t, msg.Header.Get(\"Description\"), \"No Flow Response\")\n\tcase <-ech:\n\t\tt.Fatalf(\"Got disconnected: %v\", err)\n\tcase <-time.After(snapshotAckTimeout * 2):\n\t\tt.Fatalf(\"Should have received EOF with error status\")\n\t}\n}\n\nfunc TestNoRaceJetStreamWQSkippedMsgsOnScaleUp(t *testing.T) {\n\tcheckInterestStateT = 4 * time.Second\n\tcheckInterestStateJ = 1\n\tdefer func() {\n\t\tcheckInterestStateT = defaultCheckInterestStateT\n\t\tcheckInterestStateJ = defaultCheckInterestStateJ\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tconst pre = \"CORE_ENT_DR_OTP_22.\"\n\twcSubj := pre + \">\"\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{wcSubj},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tAllowDirect: true,\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\tcfg := &nats.ConsumerConfig{\n\t\tDurable: \"dlc\",\n\t\tFilterSubject: wcSubj,\n\t\tDeliverPolicy: nats.DeliverAllPolicy,\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxAckPending: 10_000,\n\t\tAckWait: 500 * time.Millisecond,\n\t\tMaxWaiting: 100,\n\t\tMaxRequestExpires: 1050 * time.Millisecond,\n\t}\n\t_, err = js.AddConsumer(\"TEST\", cfg)\n\trequire_NoError(t, err)\n\n\tpdone := make(chan bool)\n\tcdone := make(chan bool)\n\n\t// We will have 51 consumer apps and a producer app. Make sure to wait for\n\t// all go routines to end at the end of the test.\n\twg := sync.WaitGroup{}\n\twg.Add(52)\n\n\t// Publish routine\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\tpublishSubjects := []string{\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682066\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.918886682067\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543211\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543212\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543213\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543214\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543215\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543216\",\n\t\t\t\"CORE_ENT_DR_OTP_22.P.H.TC.10011.1010.916596543217\",\n\t\t}\n\t\t// ~1.7kb\n\t\tmsg := bytes.Repeat([]byte(\"Z\"), 1750)\n\n\t\t// 200 msgs/s\n\t\tst := time.NewTicker(5 * time.Millisecond)\n\t\tdefer st.Stop()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-st.C:\n\t\t\t\tsubj := publishSubjects[rand.Intn(len(publishSubjects))]\n\t\t\t\t_, err = js.Publish(subj, msg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\tcase <-pdone:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\tconsumerApp := func() {\n\t\tdefer wg.Done()\n\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\t_, err := js.ConsumerInfo(\"TEST\", \"dlc\")\n\t\trequire_NoError(t, err)\n\t\t_, err = js.UpdateConsumer(\"TEST\", cfg)\n\t\trequire_NoError(t, err)\n\n\t\tsub, err := js.PullSubscribe(wcSubj, \"dlc\")\n\t\trequire_NoError(t, err)\n\n\t\tst := time.NewTicker(100 * time.Millisecond)\n\t\tdefer st.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-st.C:\n\t\t\t\tmsgs, err := sub.Fetch(1, nats.MaxWait(100*time.Millisecond))\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\trequire_Equal(t, len(msgs), 1)\n\t\t\t\tm := msgs[0]\n\t\t\t\tif rand.Intn(10) == 1 {\n\t\t\t\t\tm.Nak()\n\t\t\t\t} else {\n\t\t\t\t\t// Wait up to 20ms to ack.\n\t\t\t\t\ttime.Sleep(time.Duration(rand.Intn(20)) * time.Millisecond)\n\t\t\t\t\t// This could fail and that is ok, system should recover due to low ack wait.\n\t\t\t\t\tm.Ack()\n\t\t\t\t}\n\t\t\tcase <-cdone:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// Now consumer side single.\n\tgo consumerApp()\n\n\t// Wait for 2s\n\ttime.Sleep(2 * time.Second)\n\n\t// Now spin up 50 more.\n\tfor i := 1; i <= 50; i++ {\n\t\tif i%5 == 0 {\n\t\t\ttime.Sleep(200 * time.Millisecond)\n\t\t}\n\t\tgo consumerApp()\n\t}\n\n\ttimeout := time.Now().Add(8 * time.Second)\n\tfor time.Now().Before(timeout) {\n\t\ttime.Sleep(750 * time.Millisecond)\n\t\tif s := c.consumerLeader(globalAccountName, \"TEST\", \"dlc\"); s != nil {\n\t\t\ts.JetStreamStepdownConsumer(globalAccountName, \"TEST\", \"dlc\")\n\t\t}\n\t}\n\n\t// Close publishers and defer closing consumers.\n\tclose(pdone)\n\tdefer func() {\n\t\tclose(cdone)\n\t\twg.Wait()\n\t}()\n\n\tcheckFor(t, 30*time.Second, 50*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.NumDeleted > 0 || si.State.Msgs > 0 {\n\t\t\treturn fmt.Errorf(\"State not correct: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceConnectionObjectReleased(t *testing.T) {\n\tob1Conf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"B1\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tSYS { users: [{user: sys, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`))\n\tsb1, ob1 := RunServerWithConfig(ob1Conf)\n\tdefer sb1.Shutdown()\n\n\toaConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tSYS { users: [{user: sys, password: pwd}] }\n\t\t}\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"B\"\n\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`, ob1.Gateway.Port)))\n\tsa, oa := RunServerWithConfig(oaConf)\n\tdefer sa.Shutdown()\n\n\twaitForOutboundGateways(t, sa, 1, 2*time.Second)\n\twaitForOutboundGateways(t, sb1, 1, 2*time.Second)\n\n\tob2Conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"B2\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tSYS { users: [{user: sys, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`, ob1.Cluster.Port)))\n\tsb2, _ := RunServerWithConfig(ob2Conf)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\twaitForOutboundGateways(t, sb2, 1, 2*time.Second)\n\twaitForInboundGateways(t, sa, 2, 2*time.Second)\n\n\tleafConf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tserver_name: \"C\"\n\t\taccounts {\n\t\t\tA { users: [{user: a, password: pwd}] }\n\t\t\tSYS { users: [{user: sys, password: pwd}] }\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{ url: \"nats://a:pwd@127.0.0.1:%d\" }\n\t\t\t]\n\t\t}\n\t\tsystem_account: \"SYS\"\n\t`, ob1.LeafNode.Port)))\n\tleaf, _ := RunServerWithConfig(leafConf)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnected(t, leaf)\n\n\t// Start an independent MQTT server to check MQTT client connection.\n\tmo := testMQTTDefaultOptions()\n\tmo.ServerName = \"MQTTServer\"\n\tsm := testMQTTRunServer(t, mo)\n\tdefer testMQTTShutdownServer(sm)\n\n\tmc, mr := testMQTTConnect(t, &mqttConnInfo{cleanSess: true}, mo.MQTT.Host, mo.MQTT.Port)\n\tdefer mc.Close()\n\ttestMQTTCheckConnAck(t, mr, mqttConnAckRCConnectionAccepted, false)\n\n\tnc := natsConnect(t, sb1.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\tdefer nc.Close()\n\tcid, err := nc.GetClientID()\n\trequire_NoError(t, err)\n\tnatsSubSync(t, nc, \"foo\")\n\tnatsFlush(t, nc)\n\n\tncWS := natsConnect(t, fmt.Sprintf(\"ws://a:pwd@127.0.0.1:%d\", oa.Websocket.Port))\n\tdefer ncWS.Close()\n\tcidWS, err := ncWS.GetClientID()\n\trequire_NoError(t, err)\n\n\tvar conns []net.Conn\n\tvar total int\n\tvar ch chan string\n\n\ttrack := func(c *client) {\n\t\ttotal++\n\t\tc.mu.Lock()\n\t\tconns = append(conns, c.nc)\n\t\tc.mu.Unlock()\n\t\truntime.SetFinalizer(c, func(c *client) {\n\t\t\tch <- fmt.Sprintf(\"Server=%s - Kind=%s - Conn=%v\", c.srv, c.kindString(), c)\n\t\t})\n\t}\n\t// Track the connection for the MQTT client\n\tsm.mu.RLock()\n\tfor _, c := range sm.clients {\n\t\ttrack(c)\n\t}\n\tsm.mu.RUnlock()\n\n\t// Track the connection from the NATS client\n\ttrack(sb1.getClient(cid))\n\t// The outbound connection to GW \"A\"\n\ttrack(sb1.getOutboundGatewayConnection(\"A\"))\n\t// The inbound connection from GW \"A\"\n\tvar inGW []*client\n\tsb1.getInboundGatewayConnections(&inGW)\n\ttrack(inGW[0])\n\t// The routes from sb2\n\tsb1.forEachRoute(func(r *client) {\n\t\ttrack(r)\n\t})\n\t// The leaf form \"LEAF\"\n\tsb1.mu.RLock()\n\tfor _, l := range sb1.leafs {\n\t\ttrack(l)\n\t}\n\tsb1.mu.RUnlock()\n\n\t// Now from sb2, the routes to sb1\n\tsb2.forEachRoute(func(r *client) {\n\t\ttrack(r)\n\t})\n\t// The outbound connection to GW \"A\"\n\ttrack(sb2.getOutboundGatewayConnection(\"A\"))\n\n\t// From server \"A\", track the outbound GW\n\ttrack(sa.getOutboundGatewayConnection(\"B\"))\n\tinGW = inGW[:0]\n\t// Track the inbound GW connections\n\tsa.getInboundGatewayConnections(&inGW)\n\tfor _, ig := range inGW {\n\t\ttrack(ig)\n\t}\n\t// Track the websocket client\n\ttrack(sa.getClient(cidWS))\n\n\t// From the LEAF server, the connection to sb1\n\tleaf.mu.RLock()\n\tfor _, l := range leaf.leafs {\n\t\ttrack(l)\n\t}\n\tleaf.mu.RUnlock()\n\n\t// Now close all connections and wait to see if all connections\n\t// with the finalizer set is invoked.\n\tch = make(chan string, total)\n\t// Close the clients and then all other connections to create a disconnect.\n\tnc.Close()\n\tmc.Close()\n\tncWS.Close()\n\tfor _, conn := range conns {\n\t\tconn.Close()\n\t}\n\t// Wait and see if we get them all.\n\ttm := time.NewTimer(10 * time.Second)\n\tdefer tm.Stop()\n\ttk := time.NewTicker(10 * time.Millisecond)\n\tfor clients := make([]string, 0, total); len(clients) < total; {\n\t\tselect {\n\t\tcase <-tk.C:\n\t\t\truntime.GC()\n\t\tcase cs := <-ch:\n\t\t\tclients = append(clients, cs)\n\t\tcase <-tm.C:\n\t\t\t// Don't fail the test since there is no guarantee that\n\t\t\t// finalizers are invoked.\n\t\t\tt.Logf(\"Got %v out of %v finalizers\", len(clients), total)\n\t\t\tslices.Sort(clients)\n\t\t\tfor _, cs := range clients {\n\t\t\t\tt.Logf(\" => %s\", cs)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestNoRaceFileStoreMsgLoadNextMsgMultiPerf(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\t// Put 1k msgs in\n\tfor i := 0; i < 1000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t_, _, err = fs.StoreMsg(subj, nil, []byte(\"ZZZ\"), 0)\n\t\trequire_NoError(t, err)\n\t}\n\n\tvar smv StoreMsg\n\n\t// Now do normal load next with no filter.\n\t// This is baseline.\n\tstart := time.Now()\n\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\tsm, nseq, err := fs.LoadNextMsg(_EMPTY_, false, seq, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\trequire_Equal(t, nseq, seq)\n\t\tseq++\n\t}\n\tbaseline := time.Since(start)\n\tt.Logf(\"Single - No filter %v\", baseline)\n\n\t// Allow some additional skew.\n\tbaseline += time.Millisecond\n\n\t// Now do normal load next with wc filter.\n\tstart = time.Now()\n\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\tsm, nseq, err := fs.LoadNextMsg(\"foo.>\", true, seq, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\trequire_Equal(t, nseq, seq)\n\t\tseq++\n\t}\n\telapsed := time.Since(start)\n\tt.Logf(\"Single - WC filter %v\", elapsed)\n\trequire_LessThan(t, elapsed, 2*baseline)\n\n\t// Now do multi load next with 1 wc entry.\n\tsl := gsl.NewSublist[struct{}]()\n\trequire_NoError(t, sl.Insert(\"foo.>\", struct{}{}))\n\tstart = time.Now()\n\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\tsm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\trequire_Equal(t, nseq, seq)\n\t\tseq++\n\t}\n\telapsed = time.Since(start)\n\tt.Logf(\"Multi - Single WC filter %v\", elapsed)\n\trequire_LessThan(t, elapsed, 2*baseline)\n\n\t// Now do multi load next with 1000 literal subjects.\n\tsl = gsl.NewSublist[struct{}]()\n\tfor i := 0; i < 1000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\trequire_NoError(t, sl.Insert(subj, struct{}{}))\n\t}\n\tstart = time.Now()\n\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\tsm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\trequire_Equal(t, nseq, seq)\n\t\tseq++\n\t}\n\telapsed = time.Since(start)\n\tt.Logf(\"Multi - 1000 filters %v\", elapsed)\n\trequire_LessThan(t, elapsed, 3*baseline)\n}\n\nfunc TestNoRaceWQAndMultiSubjectFilters(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"Z.>\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tstopPubs := make(chan bool)\n\n\tpublish := func(subject string) {\n\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\tdefer nc.Close()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stopPubs:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\t_, _ = js.Publish(subject, []byte(\"hello\"))\n\t\t\t}\n\t\t}\n\t}\n\n\tgo publish(\"Z.foo\")\n\tgo publish(\"Z.bar\")\n\tgo publish(\"Z.baz\")\n\n\t// Cancel pubs after 10s.\n\ttime.AfterFunc(10*time.Second, func() { close(stopPubs) })\n\n\t// Create a consumer\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"zzz\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tAckWait: 5 * time.Second,\n\t\tFilterSubjects: []string{\"Z.foo\", \"Z.bar\", \"Z.baz\"},\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(_EMPTY_, \"zzz\", nats.Bind(\"TEST\", \"zzz\"))\n\trequire_NoError(t, err)\n\n\treceived := make([]uint64, 0, 256_000)\n\tbatchSize := 10\n\n\tfor running := true; running; {\n\t\tmsgs, err := sub.Fetch(batchSize, nats.MaxWait(2*time.Second))\n\t\tif err == nats.ErrTimeout {\n\t\t\trunning = false\n\t\t}\n\t\tfor _, m := range msgs {\n\t\t\tmeta, err := m.Metadata()\n\t\t\trequire_NoError(t, err)\n\t\t\treceived = append(received, meta.Sequence.Stream)\n\t\t\tm.Ack()\n\t\t}\n\t}\n\n\tslices.Sort(received)\n\n\tvar pseq, gaps uint64\n\tfor _, seq := range received {\n\t\tif pseq != 0 && pseq != seq-1 {\n\t\t\tgaps += seq - pseq + 1\n\t\t}\n\t\tpseq = seq\n\t}\n\tsi, err := js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\tif si.State.Msgs != 0 || gaps > 0 {\n\t\tt.Fatalf(\"Orphaned msgs %d with %d gaps detected\", si.State.Msgs, gaps)\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/4957\nfunc TestNoRaceWQAndMultiSubjectFiltersRace(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"Z.>\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tReplicas: 1,\n\t})\n\trequire_NoError(t, err)\n\n\t// The bug would happen when the stream was on same server as meta-leader.\n\t// So make that so.\n\t// Make sure stream leader is on S-1\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif sl == c.leader() {\n\t\t\treturn nil\n\t\t}\n\t\t// Move meta-leader since stream can be R1.\n\t\tsnc, _ := jsClientConnect(t, c.randomServer(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\t\tdefer snc.Close()\n\t\tif _, err := snc.Request(JSApiLeaderStepDown, nil, time.Second); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn fmt.Errorf(\"stream leader on meta-leader\")\n\t})\n\n\tstart := make(chan struct{})\n\tvar done, ready sync.WaitGroup\n\n\t// Create num go routines who will all race to create a consumer with the same filter subject but a different name.\n\tnum := 10\n\tready.Add(num)\n\tdone.Add(num)\n\n\tfor i := 0; i < num; i++ {\n\t\tgo func(n int) {\n\t\t\t// Connect directly to the meta leader but with our own connection.\n\t\t\ts := c.leader()\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\n\t\t\tready.Done()\n\t\t\tdefer done.Done()\n\t\t\t<-start\n\n\t\t\tjs.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\t\t\tName: fmt.Sprintf(\"C-%d\", n),\n\t\t\t\tFilterSubject: \"Z.foo\",\n\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t})\n\t\t}(i)\n\t}\n\n\t// Wait for requestors to be ready\n\tready.Wait()\n\tclose(start)\n\tdone.Wait()\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Consumers != 1 {\n\t\t\treturn fmt.Errorf(\"Consumer count not correct: %d vs 1\", si.State.Consumers)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceFileStoreWriteFullStateUniqueSubjects(t *testing.T) {\n\tfcfg := FileStoreConfig{StoreDir: t.TempDir()}\n\tfs, err := newFileStore(fcfg,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"records.>\"}, Storage: FileStorage, MaxMsgsPer: 1, MaxBytes: 15 * 1024 * 1024 * 1024})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tqch := make(chan struct{})\n\tdefer close(qch)\n\n\tgo func() {\n\t\tconst numThreshold = 1_000_000\n\t\ttick := time.NewTicker(1 * time.Second)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-qch:\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\terr := fs.writeFullState()\n\t\t\t\tvar state StreamState\n\t\t\t\tfs.FastState(&state)\n\t\t\t\tif state.Msgs > numThreshold && err != nil {\n\t\t\t\t\trequire_Error(t, err, errStateTooBig)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tlabels := []string{\"AAAAA\", \"BBBB\", \"CCCC\", \"DD\", \"EEEEE\"}\n\tmsg := []byte(strings.Repeat(\"Z\", 128))\n\n\tfor i := 0; i < 100; i++ {\n\t\tpartA := nuid.Next()\n\t\tfor j := 0; j < 100; j++ {\n\t\t\tpartB := nuid.Next()\n\t\t\tfor k := 0; k < 500; k++ {\n\t\t\t\tpartC := nuid.Next()\n\t\t\t\tpartD := labels[rand.Intn(len(labels)-1)]\n\t\t\t\tsubject := fmt.Sprintf(\"records.%s.%s.%s.%s.%s\", partA, partB, partC, partD, nuid.Next())\n\t\t\t\tstart := time.Now()\n\t\t\t\tfs.StoreMsg(subject, nil, msg, 0)\n\t\t\t\telapsed := time.Since(start)\n\t\t\t\tif elapsed > 500*time.Millisecond {\n\t\t\t\t\tt.Fatalf(\"Slow store for %q: %v\\n\", subject, elapsed)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// Make sure we do write the full state on stop.\n\tfs.Stop()\n\tfi, err := os.Stat(filepath.Join(fcfg.StoreDir, msgDir, streamStreamStateFile))\n\trequire_NoError(t, err)\n\t// ~500MB, could change if we tweak encodings..\n\trequire_True(t, fi.Size() > 500*1024*1024)\n}\n\n// When a catchup takes a long time and the ingest rate is high enough to cause us\n// to drop append entries and move our first past out catchup window due to max bytes or max msgs, etc.\nfunc TestNoRaceLargeStreamCatchups(t *testing.T) {\n\t// This usually takes too long on Travis.\n\tt.Skip()\n\n\tvar jsMaxOutTempl = `\n\tlisten: 127.0.0.1:-1\n\tserver_name: %s\n\tjetstream: {max_file_store: 22GB, store_dir: '%s', max_outstanding_catchup: 128KB}\n\tcluster {\n\t\tname: %s\n\t\tlisten: 127.0.0.1:%d\n\t\troutes = [%s]\n\t}\n\t# For access to system account.\n\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n`\n\n\tc := createJetStreamClusterWithTemplate(t, jsMaxOutTempl, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream(nats.PublishAsyncMaxPending(64 * 1024))\n\trequire_NoError(t, err)\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"Z.>\"},\n\t\tMaxBytes: 4 * 1024 * 1024 * 1024,\n\t\tReplicas: 1, // Start at R1\n\t}\n\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// Load up to a decent size first.\n\tnum, msg := 25_000, bytes.Repeat([]byte(\"Z\"), 256*1024)\n\tfor i := 0; i < num; i++ {\n\t\t_, err := js.PublishAsync(\"Z.Z.Z\", msg)\n\t\trequire_NoError(t, err)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(20 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tvar sl *Server\n\ttime.AfterFunc(time.Second, func() {\n\t\tcfg.Replicas = 3\n\t\t_, err = js.UpdateStream(cfg)\n\t\trequire_NoError(t, err)\n\t\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\t\tsl = c.streamLeader(globalAccountName, \"TEST\")\n\t})\n\n\t// Run for 60 seconds sending new messages at a high rate.\n\ttimeout := time.Now().Add(60 * time.Second)\n\tfor time.Now().Before(timeout) {\n\t\tfor i := 0; i < 5_000; i++ {\n\t\t\t// Not worried about each message, so not checking err here.\n\t\t\t// Just generating high load of new traffic while trying to catch up.\n\t\t\tjs.PublishAsync(\"Z.Z.Z\", msg)\n\t\t}\n\t\t// This will gate us waiting on a response.\n\t\tjs.Publish(\"Z.Z.Z\", msg)\n\t}\n\n\t// Make sure the leader has not changed.\n\trequire_Equal(t, sl, c.streamLeader(globalAccountName, \"TEST\"))\n\n\t// Grab the leader and its state.\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\texpected := mset.state()\n\n\tcheckFor(t, 45*time.Second, time.Second, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tstate := mset.state()\n\t\t\tif !reflect.DeepEqual(expected, state) {\n\t\t\t\treturn fmt.Errorf(\"Follower %v state does not match: %+v vs %+v\", s, state, expected)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceLargeNumDeletesStreamCatchups(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tjs, err := nc.JetStream(nats.PublishAsyncMaxPending(16 * 1024))\n\trequire_NoError(t, err)\n\n\tcfg := &nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t\tReplicas: 1, // Start at R1\n\t}\n\t_, err = js.AddStream(cfg)\n\trequire_NoError(t, err)\n\n\t// We will manipulate the stream at the lower level to achieve large number of interior deletes.\n\t// We will store only 2 msgs, but have 100M deletes in between.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\tmset.mu.Lock()\n\tmset.store.StoreMsg(\"foo\", nil, []byte(\"ok\"), 0)\n\tmset.store.SkipMsgs(2, 1_000_000_000)\n\tmset.store.StoreMsg(\"foo\", nil, []byte(\"ok\"), 0)\n\tmset.store.SkipMsgs(1_000_000_003, 1_000_000_000)\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\tmset.lseq = state.LastSeq\n\tmset.mu.Unlock()\n\n\tcfg.Replicas = 3\n\t_, err = js.UpdateStream(cfg)\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\tmset, err = sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\texpected := mset.state()\n\n\t// This should happen fast and not spin on all interior deletes.\n\tcheckFor(t, 250*time.Millisecond, 50*time.Millisecond, func() error {\n\t\tfor _, s := range c.servers {\n\t\t\tif s == sl {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmset, err := s.GlobalAccount().lookupStream(\"TEST\")\n\t\t\trequire_NoError(t, err)\n\t\t\tstate := mset.state()\n\t\t\t// Ignore LastTime for this test since we send delete range at end.\n\t\t\tstate.LastTime = expected.LastTime\n\t\t\tif !reflect.DeepEqual(expected, state) {\n\t\t\t\treturn fmt.Errorf(\"Follower %v state does not match: %+v vs %+v\", s, state, expected)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamClusterMemoryStreamLastSequenceResetAfterRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\tnumStreams := 250\n\tvar wg sync.WaitGroup\n\twg.Add(numStreams)\n\n\tfor i := 1; i <= numStreams; i++ {\n\t\tgo func(n int) {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: fmt.Sprintf(\"TEST:%d\", n),\n\t\t\t\tStorage: nats.MemoryStorage,\n\t\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d.*\", n)},\n\t\t\t\tReplicas: 3,\n\t\t\t}, nats.MaxWait(30*time.Second))\n\t\t\trequire_NoError(t, err)\n\t\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", n)\n\t\t\tfor i := 0; i < 222; i++ {\n\t\t\t\tjs.Publish(subj, nil)\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\t// Make sure all streams have a snapshot in place to stress the snapshot logic for memory based streams.\n\tfor _, s := range c.servers {\n\t\tfor i := 1; i <= numStreams; i++ {\n\t\t\tstream := fmt.Sprintf(\"TEST:%d\", i)\n\t\t\tmset, err := s.GlobalAccount().lookupStream(stream)\n\t\t\trequire_NoError(t, err)\n\t\t\tnode := mset.raftNode()\n\t\t\trequire_NotNil(t, node)\n\t\t\tnode.InstallSnapshot(mset.stateSnapshot(), false)\n\t\t}\n\t}\n\n\t// Do 5 rolling restarts waiting on healthz in between.\n\tfor i := 0; i < 5; i++ {\n\t\t// Walk the servers and shut each down, and wipe the storage directory.\n\t\tfor _, s := range c.servers {\n\t\t\ts.Shutdown()\n\t\t\ts.WaitForShutdown()\n\t\t\ts = c.restartServer(s)\n\t\t\tc.waitOnServerHealthz(s)\n\t\t\tc.waitOnAllCurrent()\n\t\t\t// Make sure all streams are current after healthz returns ok.\n\t\t\tfor i := 1; i <= numStreams; i++ {\n\t\t\t\tstream := fmt.Sprintf(\"TEST:%d\", i)\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(stream)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tvar state StreamState\n\t\t\t\tcheckFor(t, 30*time.Second, time.Second, func() error {\n\t\t\t\t\tmset.store.FastState(&state)\n\t\t\t\t\tif state.LastSeq != 222 {\n\t\t\t\t\t\treturn fmt.Errorf(\"%v Wrong last sequence %d for %q - State %+v\", s, state.LastSeq, stream, state)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterMemoryWorkQueueLastSequenceResetAfterRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnumStreams := 50\n\tvar wg sync.WaitGroup\n\twg.Add(numStreams)\n\n\tfor i := 1; i <= numStreams; i++ {\n\t\tgo func(n int) {\n\t\t\tdefer wg.Done()\n\n\t\t\tnc, js := jsClientConnect(t, c.randomServer())\n\t\t\tdefer nc.Close()\n\n\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\t\tName: fmt.Sprintf(\"TEST:%d\", n),\n\t\t\t\t\tStorage: nats.MemoryStorage,\n\t\t\t\t\tRetention: nats.WorkQueuePolicy,\n\t\t\t\t\tSubjects: []string{fmt.Sprintf(\"foo.%d.*\", n)},\n\t\t\t\t\tReplicas: 3,\n\t\t\t\t}, nats.MaxWait(time.Second))\n\t\t\t\treturn err\n\t\t\t})\n\n\t\t\tsubj := fmt.Sprintf(\"foo.%d.bar\", n)\n\t\t\tfor i := 0; i < 22; i++ {\n\t\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\t\t_, err := js.Publish(subj, nil)\n\t\t\t\t\treturn err\n\t\t\t\t})\n\t\t\t}\n\t\t\t// Now consume them all as well.\n\t\t\tvar err error\n\t\t\tvar sub *nats.Subscription\n\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\tsub, err = js.PullSubscribe(subj, \"wq\")\n\t\t\t\treturn err\n\t\t\t})\n\n\t\t\tvar msgs []*nats.Msg\n\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\tmsgs, err = sub.Fetch(22, nats.MaxWait(time.Second))\n\t\t\t\treturn err\n\t\t\t})\n\t\t\trequire_Equal(t, len(msgs), 22)\n\t\t\tfor _, m := range msgs {\n\t\t\t\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\t\t\t\treturn m.AckSync()\n\t\t\t\t})\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\t// Do 2 rolling restarts waiting on healthz in between.\n\tfor i := 0; i < 2; i++ {\n\t\t// Walk the servers and shut each down, and wipe the storage directory.\n\t\tfor _, s := range c.servers {\n\t\t\ts.Shutdown()\n\t\t\ts.WaitForShutdown()\n\t\t\ts = c.restartServer(s)\n\t\t\tc.waitOnServerHealthz(s)\n\t\t\tc.waitOnAllCurrent()\n\t\t\t// Make sure all streams are current after healthz returns ok.\n\t\t\tfor i := 1; i <= numStreams; i++ {\n\t\t\t\tstream := fmt.Sprintf(\"TEST:%d\", i)\n\t\t\t\tmset, err := s.GlobalAccount().lookupStream(stream)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tvar state StreamState\n\t\t\t\tcheckFor(t, 20*time.Second, time.Second, func() error {\n\t\t\t\t\tmset.store.FastState(&state)\n\t\t\t\t\tif state.LastSeq != 22 {\n\t\t\t\t\t\treturn fmt.Errorf(\"%v Wrong last sequence %d for %q - State %+v\", s, state.LastSeq, stream, state)\n\t\t\t\t\t}\n\t\t\t\t\tif state.FirstSeq != 23 {\n\t\t\t\t\t\treturn fmt.Errorf(\"%v Wrong first sequence %d for %q - State %+v\", s, state.FirstSeq, stream, state)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterMirrorSkipSequencingBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R5S\", 5)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"ORIGIN\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tMaxMsgsPerSubject: 1,\n\t\tReplicas: 1,\n\t\tStorage: nats.MemoryStorage,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create a mirror with R5 such that it will be much slower than the ORIGIN.\n\t_, err = js.AddStream(&nats.StreamConfig{\n\t\tName: \"M\",\n\t\tReplicas: 5,\n\t\tMirror: &nats.StreamSource{Name: \"ORIGIN\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Connect new directly to ORIGIN\n\ts := c.streamLeader(globalAccountName, \"ORIGIN\")\n\tnc, js = jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t// We are going to send at a high rate and also delete some along the way\n\t// via the max msgs per limit.\n\tfor i := 0; i < 500_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\tjs.PublishAsync(subj, nil)\n\t\t// Create sequence holes every 100k.\n\t\tif i%100_000 == 0 {\n\t\t\tjs.PublishAsync(subj, nil)\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\tcheckFor(t, 20*time.Second, time.Second, func() error {\n\t\tsi, err := js.StreamInfo(\"M\")\n\t\trequire_NoError(t, err)\n\t\tif si.State.Msgs != 500_000 {\n\t\t\treturn fmt.Errorf(\"Expected 1M msgs, got state: %+v\", si.State)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceJetStreamStandaloneDontReplyToAckBeforeProcessingIt(t *testing.T) {\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\t// Client for API requests.\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"WQ\",\n\t\tDiscard: nats.DiscardNew,\n\t\tMaxMsgsPerSubject: 1,\n\t\tDiscardNewPerSubject: true,\n\t\tRetention: nats.WorkQueuePolicy,\n\t\tSubjects: []string{\"queue.>\"},\n\t})\n\trequire_NoError(t, err)\n\n\t// Keep this low since we are going to run as many go routines\n\t// to consume, ack and republish the message.\n\ttotal := 10000\n\t// Populate the queue, one message per subject.\n\tfor i := 0; i < total; i++ {\n\t\tjs.Publish(fmt.Sprintf(\"queue.%d\", i), []byte(\"hello\"))\n\t}\n\n\t_, err = js.AddConsumer(\"WQ\", &nats.ConsumerConfig{\n\t\tDurable: \"cons\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tMaxWaiting: 20000,\n\t\tMaxAckPending: -1,\n\t})\n\trequire_NoError(t, err)\n\n\tsub, err := js.PullSubscribe(\"queue.>\", \"cons\", nats.BindStream(\"WQ\"))\n\trequire_NoError(t, err)\n\n\terrCh := make(chan error, total)\n\n\tvar wg sync.WaitGroup\n\tfor iter := 0; iter < 3; iter++ {\n\t\twg.Add(total)\n\t\tfor i := 0; i < total; i++ {\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tfor {\n\t\t\t\t\tmsgs, err := sub.Fetch(1)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tmsg := msgs[0]\n\t\t\t\t\terr = msg.AckSync()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t_, err = js.Publish(msg.Subject, []byte(\"hello\"))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\terrCh <- err\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t\twg.Wait()\n\t\tselect {\n\t\tcase err := <-errCh:\n\t\t\tt.Fatalf(\"Test failed, first error was: %v\", err)\n\t\tdefault:\n\t\t\t// OK!\n\t\t}\n\t}\n}\n\n// Under certain scenarios an old index.db with a stream that has msg limits set will not restore properly\n// due to an old index.db and compaction after the index.db took place which could lose per subject information.\nfunc TestNoRaceFileStoreMsgLimitsAndOldRecoverState(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\texpectedFirstSeq uint64\n\t\texpectedLastSeq uint64\n\t\texpectedMsgs uint64\n\t\ttransform func(StreamConfig) StreamConfig\n\t}{\n\t\t{\n\t\t\tname: \"MaxMsgsPer\",\n\t\t\texpectedFirstSeq: 10_001,\n\t\t\texpectedLastSeq: 1_010_001,\n\t\t\texpectedMsgs: 1_000_001,\n\t\t\ttransform: func(config StreamConfig) StreamConfig {\n\t\t\t\tconfig.MaxMsgsPer = 1\n\t\t\t\treturn config\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"MaxMsgs\",\n\t\t\texpectedFirstSeq: 10_001,\n\t\t\texpectedLastSeq: 1_010_001,\n\t\t\texpectedMsgs: 1_000_001,\n\t\t\ttransform: func(config StreamConfig) StreamConfig {\n\t\t\t\tconfig.MaxMsgs = 1_000_001\n\t\t\t\treturn config\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"MaxBytes\",\n\t\t\texpectedFirstSeq: 8_624,\n\t\t\texpectedLastSeq: 1_010_001,\n\t\t\texpectedMsgs: 1_001_378,\n\t\t\ttransform: func(config StreamConfig) StreamConfig {\n\t\t\t\tconfig.MaxBytes = 1_065_353_216\n\t\t\t\treturn config\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsd := t.TempDir()\n\t\t\tfs, err := newFileStore(\n\t\t\t\tFileStoreConfig{StoreDir: sd},\n\t\t\t\ttest.transform(StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}),\n\t\t\t)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tmsg := make([]byte, 1024)\n\n\t\t\tfor i := 0; i < 10_000; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t\t}\n\n\t\t\t// This will write the index.db file. We will capture this and use it to replace a new one.\n\t\t\tsfile := filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)\n\t\t\tfs.Stop()\n\t\t\t_, err = os.Stat(sfile)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Read it in and make sure len > 0.\n\t\t\tbuf, err := os.ReadFile(sfile)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, len(buf) > 0)\n\n\t\t\t// Restart\n\t\t\tfs, err = newFileStore(\n\t\t\t\tFileStoreConfig{StoreDir: sd},\n\t\t\t\ttest.transform(StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}),\n\t\t\t)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\t// Put in more messages with wider range. This will compact a bunch of the previous blocks.\n\t\t\tfor i := 0; i < 1_000_001; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\tfs.StoreMsg(subj, nil, msg, 0)\n\t\t\t}\n\n\t\t\tvar ss StreamState\n\t\t\tfs.FastState(&ss)\n\t\t\trequire_Equal(t, ss.FirstSeq, test.expectedFirstSeq)\n\t\t\trequire_Equal(t, ss.LastSeq, test.expectedLastSeq)\n\t\t\trequire_Equal(t, ss.Msgs, test.expectedMsgs)\n\n\t\t\t// Now stop again, but replace index.db with old one.\n\t\t\tfs.Stop()\n\t\t\t// Put back old stream state.\n\t\t\trequire_NoError(t, os.WriteFile(sfile, buf, defaultFilePerms))\n\n\t\t\t// Restart\n\t\t\tfs, err = newFileStore(\n\t\t\t\tFileStoreConfig{StoreDir: sd},\n\t\t\t\ttest.transform(StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}),\n\t\t\t)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\n\t\t\tfs.FastState(&ss)\n\t\t\trequire_Equal(t, ss.FirstSeq, test.expectedFirstSeq)\n\t\t\trequire_Equal(t, ss.LastSeq, test.expectedLastSeq)\n\t\t\trequire_Equal(t, ss.Msgs, test.expectedMsgs)\n\t\t})\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterCheckInterestStatePerformanceWQ(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tRetention: nats.WorkQueuePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Load up a bunch of messages for three different subjects.\n\tmsg := bytes.Repeat([]byte(\"Z\"), 4096)\n\tfor i := 0; i < 100_000; i++ {\n\t\tjs.PublishAsync(\"foo.foo\", msg)\n\t}\n\tfor i := 0; i < 5_000; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", msg)\n\t\tjs.PublishAsync(\"foo.baz\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\t// We will not process this one and leave it as \"offline\".\n\t_, err = js.PullSubscribe(\"foo.foo\", \"A\")\n\trequire_NoError(t, err)\n\tsubB, err := js.PullSubscribe(\"foo.bar\", \"B\")\n\trequire_NoError(t, err)\n\tsubC, err := js.PullSubscribe(\"foo.baz\", \"C\")\n\trequire_NoError(t, err)\n\n\t// Now catch up both B and C but let A simulate being offline of very behind.\n\tfor i := 0; i < 5; i++ {\n\t\tfor _, sub := range []*nats.Subscription{subB, subC} {\n\t\t\tmsgs, err := sub.Fetch(1000)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, len(msgs), 1000)\n\t\t\tfor _, m := range msgs {\n\t\t\t\tm.Ack()\n\t\t\t}\n\t\t}\n\t}\n\t// Let acks process.\n\tnc.Flush()\n\ttime.Sleep(200 * time.Millisecond)\n\n\t// Now test the check checkInterestState() on the stream.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\texpireAllBlks := func() {\n\t\tmset.mu.RLock()\n\t\tfs := mset.store.(*fileStore)\n\t\tmset.mu.RUnlock()\n\t\tfs.mu.RLock()\n\t\tfor _, mb := range fs.blks {\n\t\t\tmb.tryForceExpireCache()\n\t\t}\n\t\tfs.mu.RUnlock()\n\t}\n\n\t// First expire all the blocks.\n\texpireAllBlks()\n\n\tstart := time.Now()\n\tmset.checkInterestState()\n\telapsed := time.Since(start)\n\t// This is actually ~300 microseconds but due to travis and race flags etc.\n\t// Was > 30 ms before fix for comparison, M2 macbook air.\n\trequire_LessThan(t, elapsed, 5*time.Millisecond)\n\n\t// Make sure we set the chkflr correctly.\n\t// The chkflr should be equal to asflr+1 (unless the next message matching the filter is further ahead).\n\t// Otherwise, if chkflr would be set higher a subsequent call to checkInterestState will be ineffective.\n\trequireFloorsEqual := func(o *consumer) {\n\t\tt.Helper()\n\t\trequire_True(t, o != nil)\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\t\t_, seq, err := mset.store.LoadNextMsg(o.cfg.FilterSubject, false, o.asflr+1, nil)\n\t\tif err == ErrStoreEOF {\n\t\t\tseq++\n\t\t}\n\t\trequire_True(t, o.asflr+1 <= seq)\n\t\trequire_Equal(t, o.chkflr, seq)\n\t}\n\n\trequireFloorsEqual(mset.lookupConsumer(\"A\"))\n\trequireFloorsEqual(mset.lookupConsumer(\"B\"))\n\trequireFloorsEqual(mset.lookupConsumer(\"C\"))\n\n\t// Expire all the blocks again.\n\texpireAllBlks()\n\n\t// This checks the chkflr state.\n\tstart = time.Now()\n\tmset.checkInterestState()\n\telapsed = time.Since(start)\n\trequire_LessThan(t, elapsed, 5*time.Millisecond)\n}\n\nfunc TestNoRaceJetStreamClusterCheckInterestStatePerformanceInterest(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo.*\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// We will not process this one and leave it as \"offline\".\n\t_, err = js.PullSubscribe(\"foo.foo\", \"A\")\n\trequire_NoError(t, err)\n\t_, err = js.PullSubscribe(\"foo.*\", \"B\")\n\trequire_NoError(t, err)\n\t// Make subC multi-subject.\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C\",\n\t\tFilterSubjects: []string{\"foo.foo\", \"foo.bar\", \"foo.baz\"},\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t// Load up a bunch of messages for three different subjects.\n\tmsg := bytes.Repeat([]byte(\"Z\"), 4096)\n\tfor i := 0; i < 90_000; i++ {\n\t\tjs.PublishAsync(\"foo.foo\", msg)\n\t}\n\tfor i := 0; i < 5_000; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", msg)\n\t\tjs.PublishAsync(\"foo.baz\", msg)\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\t// This is so we do not asynchronously update our consumer state after we set the state due to notifications\n\t// from new messages for the stream.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Now catch up both B and C but let A simulate being offline of very behind.\n\t// Will do this manually here to speed up tests.\n\tsl := c.streamLeader(globalAccountName, \"TEST\")\n\tmset, err := sl.GlobalAccount().lookupStream(\"TEST\")\n\trequire_NoError(t, err)\n\n\tfor _, cname := range []string{\"B\", \"C\"} {\n\t\to := mset.lookupConsumer(cname)\n\t\to.mu.Lock()\n\t\to.setStoreState(&ConsumerState{\n\t\t\tDelivered: SequencePair{100_000, 100_000},\n\t\t\tAckFloor: SequencePair{100_000, 100_000},\n\t\t})\n\t\to.mu.Unlock()\n\t}\n\n\t// Now test the check checkInterestState() on the stream.\n\tstart := time.Now()\n\tmset.checkInterestState()\n\telapsed := time.Since(start)\n\n\t// Make sure we set the chkflr correctly.\n\tcheckFloor := func(o *consumer) uint64 {\n\t\trequire_True(t, o != nil)\n\t\to.mu.RLock()\n\t\tdefer o.mu.RUnlock()\n\t\treturn o.chkflr\n\t}\n\n\trequire_Equal(t, checkFloor(mset.lookupConsumer(\"A\")), 1)\n\trequire_Equal(t, checkFloor(mset.lookupConsumer(\"B\")), 90_001)\n\trequire_Equal(t, checkFloor(mset.lookupConsumer(\"C\")), 100_001)\n\n\t// This checks the chkflr state. For this test this should be much faster,\n\t// two orders of magnitude then the first time.\n\tstart = time.Now()\n\tmset.checkInterestState()\n\trequire_True(t, time.Since(start) < elapsed/100)\n}\n\nfunc TestNoRaceJetStreamClusterLargeMetaSnapshotTiming(t *testing.T) {\n\t// This test was to show improvements in speed for marshaling the meta layer with lots of assets.\n\t// Move to S2.Encode vs EncodeBetter which is 2x faster and actually better compression.\n\t// Also moved to goccy json which is faster then the default and in my tests now always matches\n\t// the default encoder byte for byte which last time I checked it did not.\n\tt.Skip()\n\n\tc := createJetStreamClusterExplicit(t, \"R3F\", 3)\n\tdefer c.shutdown()\n\n\t// Create 200 streams, each with 500 consumers.\n\tnumStreams := 200\n\tnumConsumers := 500\n\twg := sync.WaitGroup{}\n\twg.Add(numStreams)\n\tfor i := 0; i < numStreams; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\ts := c.randomServer()\n\t\t\tnc, js := jsClientConnect(t, s)\n\t\t\tdefer nc.Close()\n\t\t\tsname := fmt.Sprintf(\"TEST-SNAPSHOT-%d\", i)\n\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t_, err := js.AddStream(&nats.StreamConfig{\n\t\t\t\tName: sname,\n\t\t\t\tSubjects: []string{subj},\n\t\t\t\tReplicas: 3,\n\t\t\t})\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Now consumers.\n\t\t\tfor c := 0; c < numConsumers; c++ {\n\t\t\t\t_, err = js.AddConsumer(sname, &nats.ConsumerConfig{\n\t\t\t\t\tDurable: fmt.Sprintf(\"C-%d\", c),\n\t\t\t\t\tFilterSubject: subj,\n\t\t\t\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\t\t\t\tReplicas: 1,\n\t\t\t\t})\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\n\ts := c.leader()\n\tjs := s.getJetStream()\n\tn := js.getMetaGroup()\n\t// Now let's see how long it takes to create a meta snapshot and how big it is.\n\tstart := time.Now()\n\tsnap, _, _, err := js.metaSnapshot()\n\trequire_NoError(t, err)\n\trequire_NoError(t, n.InstallSnapshot(snap, false))\n\tt.Logf(\"Took %v to snap meta with size of %v\\n\", time.Since(start), friendlyBytes(len(snap)))\n}\n\nfunc TestNoRaceStoreReverseWalkWithDeletesPerf(t *testing.T) {\n\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}, Storage: FileStorage}\n\n\tfs, err := newFileStore(FileStoreConfig{StoreDir: t.TempDir()}, cfg)\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tcfg.Storage = MemoryStorage\n\tms, err := newMemStore(&cfg)\n\trequire_NoError(t, err)\n\tdefer ms.Stop()\n\n\tmsg := []byte(\"Hello\")\n\n\tfor _, store := range []StreamStore{fs, ms} {\n\t\t_, _, err = store.StoreMsg(\"foo.A\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\t\tfor i := 0; i < 1_000_000; i++ {\n\t\t\t_, _, err = store.StoreMsg(\"foo.B\", nil, msg, 0)\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\t_, _, err = store.StoreMsg(\"foo.C\", nil, msg, 0)\n\t\trequire_NoError(t, err)\n\n\t\tvar ss StreamState\n\t\tstore.FastState(&ss)\n\t\trequire_Equal(t, ss.Msgs, 1_000_002)\n\n\t\t// Create a bunch of interior deletes.\n\t\tp, err := store.PurgeEx(\"foo.B\", 1, 0)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, p, 1_000_000)\n\n\t\t// Preload the caches so this test only measures the reverse walk.\n\t\tpreloadCaches := func() {\n\t\t\tif fs, ok := store.(*fileStore); ok {\n\t\t\t\tfs.mu.Lock()\n\t\t\t\tdefer fs.mu.Unlock()\n\t\t\t\tfor _, mb := range fs.blks {\n\t\t\t\t\trequire_NoError(t, mb.loadMsgs())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Now simulate a walk backwards as we currently do when searching for starting sequence numbers in sourced streams.\n\t\tpreloadCaches()\n\t\tstart := time.Now()\n\t\tvar smv StoreMsg\n\t\tfor seq := ss.LastSeq; seq > 0; seq-- {\n\t\t\t_, err := store.LoadMsg(seq, &smv)\n\t\t\tif err == errDeletedMsg || err == ErrStoreMsgNotFound {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\telapsed := time.Since(start)\n\n\t\t// Now use the optimized load prev.\n\t\tpreloadCaches()\n\t\tseq, seen := ss.LastSeq, 0\n\t\tstart = time.Now()\n\t\tfor {\n\t\t\tsm, err := store.LoadPrevMsg(seq, &smv)\n\t\t\tif err == ErrStoreEOF {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\trequire_NoError(t, err)\n\t\t\tseq = sm.seq - 1\n\t\t\tseen++\n\t\t}\n\t\telapsedNew := time.Since(start)\n\t\trequire_Equal(t, seen, 2)\n\n\t\tswitch store.(type) {\n\t\tcase *memStore:\n\t\t\trequire_LessThan(t, elapsedNew, elapsed)\n\t\tcase *fileStore:\n\t\t\t// Bigger gains for filestore.\n\t\t\trequire_LessThan(t, elapsedNew*10, elapsed)\n\t\t}\n\t}\n}\n\ntype fastProdLogger struct {\n\tDummyLogger\n\tgotIt chan struct{}\n}\n\nfunc (l *fastProdLogger) Debugf(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"fast producer\") {\n\t\tselect {\n\t\tcase l.gotIt <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestNoRaceNoFastProducerStall(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tno_fast_producer_stall: %s\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"true\")))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tl := &fastProdLogger{gotIt: make(chan struct{}, 1)}\n\ts.SetLogger(l, true, false)\n\n\tncSlow := natsConnect(t, s.ClientURL())\n\tdefer ncSlow.Close()\n\tnatsSub(t, ncSlow, \"foo\", func(_ *nats.Msg) {})\n\tnatsFlush(t, ncSlow)\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\ttraceSub := natsSubSync(t, nc, \"my.trace.subj\")\n\tnatsFlush(t, nc)\n\n\tncProd := natsConnect(t, s.ClientURL())\n\tdefer ncProd.Close()\n\n\tpayload := make([]byte, 256)\n\n\tcid, err := ncSlow.GetClientID()\n\trequire_NoError(t, err)\n\tc := s.GetClient(cid)\n\trequire_True(t, c != nil)\n\n\twg := sync.WaitGroup{}\n\tpub := func() {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tmsg := nats.NewMsg(\"foo\")\n\t\t\tmsg.Header.Set(MsgTraceDest, traceSub.Subject)\n\t\t\tmsg.Data = payload\n\t\t\tncProd.PublishMsg(msg)\n\t\t}()\n\t}\n\n\tcheckTraceMsg := func(err string) {\n\t\tt.Helper()\n\t\tvar e MsgTraceEvent\n\t\ttraceMsg := natsNexMsg(t, traceSub, time.Second)\n\t\tjson.Unmarshal(traceMsg.Data, &e)\n\t\tegresses := e.Egresses()\n\t\trequire_Equal(t, len(egresses), 1)\n\t\teg := egresses[0]\n\t\trequire_Equal(t, eg.CID, cid)\n\t\tif err != _EMPTY_ {\n\t\t\trequire_Contains(t, eg.Error, err)\n\t\t} else {\n\t\t\trequire_Equal(t, eg.Error, _EMPTY_)\n\t\t}\n\t}\n\n\t// Artificially set a stall channel.\n\tc.mu.Lock()\n\tc.out.stc = make(chan struct{})\n\tc.mu.Unlock()\n\n\t// Publish a message, it should not stall the producer.\n\tpub()\n\t// Now make sure we did not get any fast producer debug statements.\n\tselect {\n\tcase <-l.gotIt:\n\t\tt.Fatal(\"Got debug logs about fast producer\")\n\tcase <-time.After(250 * time.Millisecond):\n\t\t// OK!\n\t}\n\twg.Wait()\n\n\tcheckTraceMsg(errMsgTraceFastProdNoStall)\n\n\t// Now we will conf reload to enable fast producer stalling.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(tmpl, \"false\"))\n\n\t// Publish, this time the prod should be stalled.\n\tpub()\n\tselect {\n\tcase <-l.gotIt:\n\t\t// OK!\n\tcase <-time.After(500 * time.Millisecond):\n\t\tt.Fatal(\"Timed-out waiting for a warning\")\n\t}\n\twg.Wait()\n\n\t// Should have been delivered to the trace subscription.\n\tcheckTraceMsg(_EMPTY_)\n}\n\nfunc TestNoRaceProducerStallLimits(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t`\n\tconf := createConfFile(t, []byte(tmpl))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tncSlow := natsConnect(t, s.ClientURL(), nats.Name(\"slow\"))\n\tdefer ncSlow.Close()\n\tnatsSub(t, ncSlow, \"foo\", func(m *nats.Msg) { m.Respond([]byte(\"42\")) })\n\tnatsFlush(t, ncSlow)\n\n\tncProd := natsConnect(t, s.ClientURL(), nats.Name(\"fast\"))\n\tdefer ncProd.Close()\n\n\tcid, err := ncSlow.GetClientID()\n\trequire_NoError(t, err)\n\tc := s.GetClient(cid)\n\trequire_True(t, c != nil)\n\n\t// Artificially set a stall channel on the subscriber.\n\tc.mu.Lock()\n\tc.out.stc = make(chan struct{})\n\tc.mu.Unlock()\n\n\tstart := time.Now()\n\t_, err = ncProd.Request(\"foo\", []byte(\"HELLO\"), time.Second)\n\telapsed := time.Since(start)\n\trequire_NoError(t, err)\n\n\t// This should have not cleared on its own but should have between min and max pause.\n\trequire_True(t, elapsed >= stallClientMinDuration)\n\trequire_LessThan(t, elapsed, stallClientMaxDuration+5*time.Millisecond)\n\n\t// Now test total maximum by loading up a bunch of requests and measuring the last one.\n\t// Artificially set a stall channel again on the subscriber.\n\tc.mu.Lock()\n\tc.out.stc = make(chan struct{})\n\t// This will prevent us from clearing the stc.\n\tc.out.pb = c.out.mp/4*3 + 100\n\tc.mu.Unlock()\n\n\tfor i := 0; i < 10; i++ {\n\t\terr = ncProd.PublishRequest(\"foo\", \"bar\", []byte(\"HELLO\"))\n\t\trequire_NoError(t, err)\n\t}\n\tstart = time.Now()\n\t_, err = ncProd.Request(\"foo\", []byte(\"HELLO\"), time.Second)\n\telapsed = time.Since(start)\n\trequire_NoError(t, err)\n\n\trequire_True(t, elapsed >= stallTotalAllowed)\n\t// Should always be close to totalAllowed (e.g. 10ms), but if you run a lot of them in one go can bump up\n\t// just past it, hence the Max setting below to avoid a flapper.\n\trequire_LessThan(t, elapsed, stallTotalAllowed+20*time.Millisecond)\n\n\tvar tc *client\n\ts.mu.Lock()\n\tfor _, client := range s.clients {\n\t\tclient.mu.Lock()\n\t\tif client.opts.Name == \"slow\" {\n\t\t\ttc = client\n\t\t\tclient.mu.Unlock()\n\t\t\tbreak\n\t\t}\n\t\tclient.mu.Unlock()\n\t}\n\ts.mu.Unlock()\n\n\t// Verify counter was incremented.\n\ttotalStalls := atomic.LoadInt64(&tc.stalls)\n\tif totalStalls == 0 {\n\t\tt.Fatalf(\"Expected stalls count to be incremented, got: %d\", totalStalls)\n\t}\n\n\t// Verify this shows up in Connz monitoring.\n\tpconns := pollConnz(t, s, 1, \"\", nil)\n\tvar connzStalls int64\n\tfor _, conn := range pconns.Conns {\n\t\tif conn.Cid == tc.cid {\n\t\t\tconnzStalls = conn.Stalls\n\t\t\tbreak\n\t\t}\n\t}\n\tif connzStalls < 1 {\n\t\tt.Fatalf(\"Expected connz stalls count to be incremented, got: %d\", connzStalls)\n\t}\n\tif s.NumStalledClients() < 1 {\n\t\tt.Fatalf(\"Expected total stalled clients to be incremented, got: %d\", s.NumStalledClients())\n\t}\n}\n\nfunc TestNoRaceJetStreamClusterConsumerDeleteInterestPolicyPerf(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Make the first sequence high. We already protect against it but for extra sanity.\n\terr = js.PurgeStream(\"TEST\", &nats.StreamPurgeRequest{Sequence: 100_000_000})\n\trequire_NoError(t, err)\n\n\t// Create 3 consumers. 1 Ack explicit, 1 AckAll and 1 AckNone\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C2\",\n\t\tAckPolicy: nats.AckAllPolicy,\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C3\",\n\t\tAckPolicy: nats.AckNonePolicy,\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 500_000; i++ {\n\t\tjs.PublishAsync(\"foo.bar\", []byte(\"ok\"))\n\n\t\t// Confirm batch.\n\t\tif i%1000 == 0 {\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\texpectedStreamMsgs := func(msgs uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != msgs {\n\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", si.State.Msgs, msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\texpectedStreamMsgs(500_000)\n\n\t// For C1 grab 100 and ack evens.\n\tsub, err := js.PullSubscribe(\"foo.bar\", \"C1\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(100)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 100)\n\tfor _, m := range msgs {\n\t\tmeta, _ := m.Metadata()\n\t\tif meta.Sequence.Stream%2 == 0 {\n\t\t\trequire_NoError(t, m.AckSync())\n\t\t}\n\t}\n\n\t// For C2 grab 500 and ack 100.\n\tsub, err = js.PullSubscribe(\"foo.bar\", \"C2\")\n\trequire_NoError(t, err)\n\tmsgs, err = sub.Fetch(500)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 500)\n\trequire_NoError(t, msgs[99].AckSync())\n\n\t// Simulate stream viewer, get first 10 from C3\n\tsub, err = js.PullSubscribe(\"foo.bar\", \"C3\")\n\trequire_NoError(t, err)\n\tmsgs, err = sub.Fetch(10)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 10)\n\n\texpectedStreamMsgs(499_995)\n\n\t// This test would flake depending on if mset.checkInterestState already ran or not.\n\t// Manually call it here, because consumers don't retry removing messages below their ack floor.\n\tfor _, s := range c.servers {\n\t\tacc, err := s.lookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tmset, err := acc.lookupStream(\"TEST\")\n\t\trequire_NoError(t, err)\n\t\tmset.checkInterestState()\n\t}\n\n\t// Before fix this was in the seconds. All the while the stream is locked.\n\t// This should be short now, but messages might not be cleaned up.\n\tstart := time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"C3\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Deleting AckNone consumer took too long: %v\", elapsed)\n\t}\n\n\texpectedStreamMsgs(499_950)\n\n\t// Now do AckAll\n\tstart = time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"C2\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Deleting AckAll consumer took too long: %v\", elapsed)\n\t}\n\n\texpectedStreamMsgs(499_950)\n\n\t// Now do AckExplicit\n\tstart = time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"C1\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Deleting AckExplicit consumer took too long: %v\", elapsed)\n\t}\n\n\texpectedStreamMsgs(0)\n}\n\nfunc TestNoRaceJetStreamClusterConsumerDeleteInterestPolicyUniqueFiltersPerf(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tRetention: nats.InterestPolicy,\n\t\tSubjects: []string{\"foo.*\"},\n\t\tReplicas: 3,\n\t})\n\trequire_NoError(t, err)\n\n\t// Create 2 consumers. 1 Ack explicit, 1 AckNone\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C0\",\n\t\tAckPolicy: nats.AckExplicitPolicy,\n\t\tFilterSubjects: []string{\"foo.0\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.AddConsumer(\"TEST\", &nats.ConsumerConfig{\n\t\tDurable: \"C1\",\n\t\tAckPolicy: nats.AckNonePolicy,\n\t\tFilterSubjects: []string{\"foo.1\"},\n\t})\n\trequire_NoError(t, err)\n\n\tfor i := 0; i < 500_000; i++ {\n\t\tsubject := fmt.Sprintf(\"foo.%d\", i%2)\n\t\tjs.PublishAsync(subject, []byte(\"ok\"))\n\n\t\t// Confirm batch.\n\t\tif i%1000 == 0 {\n\t\t\tselect {\n\t\t\tcase <-js.PublishAsyncComplete():\n\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\tt.Fatalf(\"Did not receive completion signal\")\n\t\t\t}\n\t\t}\n\t}\n\tselect {\n\tcase <-js.PublishAsyncComplete():\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Did not receive completion signal\")\n\t}\n\n\texpectedStreamMsgs := func(msgs uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tsi, err := js.StreamInfo(\"TEST\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif si.State.Msgs != msgs {\n\t\t\t\treturn fmt.Errorf(\"require uint64 equal, but got: %d != %d\", si.State.Msgs, msgs)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\texpectedStreamMsgs(500_000)\n\n\t// For C0 grab 100 and ack them.\n\tsub, err := js.PullSubscribe(\"foo.0\", \"C0\")\n\trequire_NoError(t, err)\n\tmsgs, err := sub.Fetch(100)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(msgs), 100)\n\tfor _, msg := range msgs {\n\t\trequire_NoError(t, msg.AckSync())\n\t}\n\n\texpectedStreamMsgs(499_900)\n\n\tstart := time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"C1\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Deleting AckNone consumer took too long: %v\", elapsed)\n\t}\n\n\texpectedStreamMsgs(499_800)\n\n\tstart = time.Now()\n\terr = js.DeleteConsumer(\"TEST\", \"C0\")\n\trequire_NoError(t, err)\n\tif elapsed := time.Since(start); elapsed > 100*time.Millisecond {\n\t\tt.Fatalf(\"Deleting AckExplicit consumer took too long: %v\", elapsed)\n\t}\n\texpectedStreamMsgs(0)\n}\n\nfunc TestNoRaceFileStorePurgeExAsyncTombstones(t *testing.T) {\n\tfs, err := newFileStore(\n\t\tFileStoreConfig{StoreDir: t.TempDir()},\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"*.*\"}, Storage: FileStorage})\n\trequire_NoError(t, err)\n\tdefer fs.Stop()\n\n\tmsg := []byte(\"zzz\")\n\n\tfs.StoreMsg(\"foo.A\", nil, msg, 0)\n\tfs.StoreMsg(\"foo.B\", nil, msg, 0)\n\tfor i := 0; i < 500; i++ {\n\t\tfs.StoreMsg(\"foo.C\", nil, msg, 0)\n\t}\n\tfs.StoreMsg(\"foo.D\", nil, msg, 0)\n\n\t// Load all blocks to avoid that being a factor in timing.\n\tfs.mu.RLock()\n\tfor _, mb := range fs.blks {\n\t\tmb.loadMsgs()\n\t}\n\tfs.mu.RUnlock()\n\n\t// Now purge 1 that is not the first message and take note of time.\n\t// Since we are loaded this should mostly be the time to write / flush tombstones.\n\tstart := time.Now()\n\tn, err := fs.PurgeEx(\"foo.B\", 0, 0)\n\telapsed := time.Since(start)\n\trequire_NoError(t, err)\n\trequire_Equal(t, n, 1)\n\n\tstart = time.Now()\n\tn, err = fs.PurgeEx(\"foo.C\", 0, 0)\n\telapsed2 := time.Since(start)\n\trequire_NoError(t, err)\n\trequire_Equal(t, n, 500)\n\n\t// If we are flushing for each tombstone the second elapsed time will be a larger multiple of the single message purge.\n\t// In testing this is like >200x\n\t// With async and flush for all tombstones will be ~30x\n\trequire_True(t, elapsed*50 > elapsed2)\n}\n\n// Chck that we do not leak any go routines from access time optimization.\nfunc TestNoRaceAccessTimeLeakCheck(t *testing.T) {\n\ttime.Sleep(time.Second)\n\tngrp := runtime.NumGoroutine()\n\n\ts := RunBasicJetStreamServer(t)\n\tdefer s.Shutdown()\n\n\tnc, js := jsClientConnect(t, s)\n\tdefer nc.Close()\n\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tSubjects: []string{\"foo\"},\n\t})\n\trequire_NoError(t, err)\n\n\t_, err = js.Publish(\"foo\", []byte(\"Hello World\"))\n\trequire_NoError(t, err)\n\n\t// Close down client and server.\n\tnc.Close()\n\ts.Shutdown()\n\ts.WaitForShutdown()\n\n\tcheckFor(t, 5*time.Second, time.Second, func() error {\n\t\tif ngra := runtime.NumGoroutine(); ngrp != ngra {\n\t\t\treturn fmt.Errorf(\"expected %d, got %d\", ngrp, ngra)\n\t\t}\n\t\treturn nil\n\t})\n}\n" }, { "path": "server/ocsp.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/asn1\"\n\t\"encoding/base64\"\n\t\"encoding/pem\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"golang.org/x/crypto/ocsp\"\n\n\t\"github.com/nats-io/nats-server/v2/server/certidp\"\n\t\"github.com/nats-io/nats-server/v2/server/certstore\"\n)\n\nconst (\n\tdefaultOCSPStoreDir = \"ocsp\"\n\tdefaultOCSPCheckInterval = 24 * time.Hour\n\tminOCSPCheckInterval = 2 * time.Minute\n)\n\ntype OCSPMode uint8\n\nconst (\n\t// OCSPModeAuto staples a status, only if \"status_request\" is set in cert.\n\tOCSPModeAuto OCSPMode = iota\n\n\t// OCSPModeAlways enforces OCSP stapling for certs and shuts down the server in\n\t// case a server is revoked or cannot get OCSP staples.\n\tOCSPModeAlways\n\n\t// OCSPModeNever disables OCSP stapling even if cert has Must-Staple flag.\n\tOCSPModeNever\n\n\t// OCSPModeMust honors the Must-Staple flag from a certificate but also causing shutdown\n\t// in case the certificate has been revoked.\n\tOCSPModeMust\n)\n\n// OCSPMonitor monitors the state of a staple per certificate.\ntype OCSPMonitor struct {\n\tkind string\n\tmu sync.Mutex\n\traw []byte\n\tsrv *Server\n\tcertFile string\n\tresp *ocsp.Response\n\thc *http.Client\n\tstopCh chan struct{}\n\tLeaf *x509.Certificate\n\tIssuer *x509.Certificate\n\n\tshutdownOnRevoke bool\n}\n\nfunc (oc *OCSPMonitor) getNextRun() time.Duration {\n\toc.mu.Lock()\n\tnextUpdate := oc.resp.NextUpdate\n\toc.mu.Unlock()\n\n\tnow := time.Now()\n\tif nextUpdate.IsZero() {\n\t\t// If response is missing NextUpdate, we check the day after.\n\t\t// Technically, if NextUpdate is missing, we can try whenever.\n\t\t// https://tools.ietf.org/html/rfc6960#section-4.2.2.1\n\t\treturn defaultOCSPCheckInterval\n\t}\n\tdur := nextUpdate.Sub(now) / 2\n\n\t// If negative, then wait a couple of minutes before getting another staple.\n\tif dur < 0 {\n\t\treturn minOCSPCheckInterval\n\t}\n\n\treturn dur\n}\n\nfunc (oc *OCSPMonitor) getStatus() ([]byte, *ocsp.Response, error) {\n\traw, resp := oc.getCacheStatus()\n\tif len(raw) > 0 && resp != nil {\n\t\t// Check if the OCSP is still valid.\n\t\tif err := validOCSPResponse(resp); err == nil {\n\t\t\treturn raw, resp, nil\n\t\t}\n\t}\n\tvar err error\n\traw, resp, err = oc.getLocalStatus()\n\tif err == nil {\n\t\treturn raw, resp, nil\n\t}\n\n\treturn oc.getRemoteStatus()\n}\n\nfunc (oc *OCSPMonitor) getCacheStatus() ([]byte, *ocsp.Response) {\n\toc.mu.Lock()\n\tdefer oc.mu.Unlock()\n\treturn oc.raw, oc.resp\n}\n\nfunc (oc *OCSPMonitor) getLocalStatus() ([]byte, *ocsp.Response, error) {\n\topts := oc.srv.getOpts()\n\tstoreDir := opts.StoreDir\n\tif storeDir == _EMPTY_ {\n\t\treturn nil, nil, fmt.Errorf(\"store_dir not set\")\n\t}\n\n\t// This key must be based upon the current full certificate, not the public key,\n\t// so MUST be on the full raw certificate and not an SPKI or other reduced form.\n\tkey := fmt.Sprintf(\"%x\", sha256.Sum256(oc.Leaf.Raw))\n\n\toc.mu.Lock()\n\traw, err := os.ReadFile(filepath.Join(storeDir, defaultOCSPStoreDir, key))\n\toc.mu.Unlock()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tresp, err := ocsp.ParseResponse(raw, oc.Issuer)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"failed to get local status: %w\", err)\n\t}\n\tif err := validOCSPResponse(resp); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// Cache the response.\n\toc.mu.Lock()\n\toc.raw = raw\n\toc.resp = resp\n\toc.mu.Unlock()\n\n\treturn raw, resp, nil\n}\n\nfunc (oc *OCSPMonitor) getRemoteStatus() ([]byte, *ocsp.Response, error) {\n\topts := oc.srv.getOpts()\n\tvar overrideURLs []string\n\tif config := opts.OCSPConfig; config != nil {\n\t\toverrideURLs = config.OverrideURLs\n\t}\n\tgetRequestBytes := func(u string, reqDER []byte, hc *http.Client) ([]byte, error) {\n\t\treqEnc := base64.StdEncoding.EncodeToString(reqDER)\n\t\tu = fmt.Sprintf(\"%s/%s\", u, reqEnc)\n\t\tstart := time.Now()\n\t\tresp, err := hc.Get(u)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\n\t\toc.srv.Debugf(\"Received OCSP response (method=GET, status=%v, url=%s, duration=%.3fs)\",\n\t\t\tresp.StatusCode, u, time.Since(start).Seconds())\n\t\tif resp.StatusCode > 299 {\n\t\t\treturn nil, fmt.Errorf(\"non-ok http status on GET request (reqlen=%d): %d\", len(reqEnc), resp.StatusCode)\n\t\t}\n\t\treturn io.ReadAll(resp.Body)\n\t}\n\tpostRequestBytes := func(u string, body []byte, hc *http.Client) ([]byte, error) {\n\t\threq, err := http.NewRequest(\"POST\", u, bytes.NewReader(body))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\threq.Header.Add(\"Content-Type\", \"application/ocsp-request\")\n\t\threq.Header.Add(\"Accept\", \"application/ocsp-response\")\n\n\t\tstart := time.Now()\n\t\tresp, err := hc.Do(hreq)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\n\t\toc.srv.Debugf(\"Received OCSP response (method=POST, status=%v, url=%s, duration=%.3fs)\",\n\t\t\tresp.StatusCode, u, time.Since(start).Seconds())\n\t\tif resp.StatusCode > 299 {\n\t\t\treturn nil, fmt.Errorf(\"non-ok http status on POST request (reqlen=%d): %d\", len(body), resp.StatusCode)\n\t\t}\n\t\treturn io.ReadAll(resp.Body)\n\t}\n\n\t// Request documentation:\n\t// https://tools.ietf.org/html/rfc6960#appendix-A.1\n\n\treqDER, err := ocsp.CreateRequest(oc.Leaf, oc.Issuer, nil)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tresponders := oc.Leaf.OCSPServer\n\tif len(overrideURLs) > 0 {\n\t\tresponders = overrideURLs\n\t}\n\tif len(responders) == 0 {\n\t\treturn nil, nil, fmt.Errorf(\"no available ocsp servers\")\n\t}\n\n\toc.mu.Lock()\n\thc := oc.hc\n\toc.mu.Unlock()\n\n\tvar raw []byte\n\tfor _, u := range responders {\n\t\tvar postErr, getErr error\n\t\tu = strings.TrimSuffix(u, \"/\")\n\t\t// Prefer to make POST requests first.\n\t\traw, postErr = postRequestBytes(u, reqDER, hc)\n\t\tif postErr == nil {\n\t\t\terr = nil\n\t\t\tbreak\n\t\t} else {\n\t\t\t// Fallback to use a GET request.\n\t\t\traw, getErr = getRequestBytes(u, reqDER, hc)\n\t\t\tif getErr == nil {\n\t\t\t\terr = nil\n\t\t\t\tbreak\n\t\t\t} else {\n\t\t\t\terr = errors.Join(postErr, getErr)\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"exhausted ocsp servers: %w\", err)\n\t}\n\tresp, err := ocsp.ParseResponse(raw, oc.Issuer)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"failed to get remote status: %w\", err)\n\t}\n\tif err := validOCSPResponse(resp); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif storeDir := opts.StoreDir; storeDir != _EMPTY_ {\n\t\tkey := fmt.Sprintf(\"%x\", sha256.Sum256(oc.Leaf.Raw))\n\t\tif err := oc.writeOCSPStatus(storeDir, key, raw); err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"failed to write ocsp status: %w\", err)\n\t\t}\n\t}\n\n\toc.mu.Lock()\n\toc.raw = raw\n\toc.resp = resp\n\toc.mu.Unlock()\n\n\treturn raw, resp, nil\n}\n\nfunc (oc *OCSPMonitor) run() {\n\ts := oc.srv\n\ts.mu.Lock()\n\tquitCh := s.quitCh\n\ts.mu.Unlock()\n\n\tvar doShutdown bool\n\tdefer func() {\n\t\t// Need to decrement before shuting down, otherwise shutdown\n\t\t// would be stuck waiting on grWG to go down to 0.\n\t\ts.grWG.Done()\n\t\tif doShutdown {\n\t\t\ts.Shutdown()\n\t\t}\n\t}()\n\n\toc.mu.Lock()\n\tshutdownOnRevoke := oc.shutdownOnRevoke\n\tcertFile := oc.certFile\n\tstopCh := oc.stopCh\n\tkind := oc.kind\n\toc.mu.Unlock()\n\n\tvar nextRun time.Duration\n\t_, resp, err := oc.getStatus()\n\tif err == nil && resp.Status == ocsp.Good {\n\t\tnextRun = oc.getNextRun()\n\t\tt := resp.NextUpdate.Format(time.RFC3339Nano)\n\t\ts.Noticef(\n\t\t\t\"Found OCSP status for %s certificate at '%s': good, next update %s, checking again in %s\",\n\t\t\tkind, certFile, t, nextRun,\n\t\t)\n\t} else if err == nil && shutdownOnRevoke {\n\t\t// If resp.Status is ocsp.Revoked, ocsp.Unknown, or any other value.\n\t\ts.Errorf(\"Found OCSP status for %s certificate at '%s': %s\", kind, certFile, ocspStatusString(resp.Status))\n\t\tdoShutdown = true\n\t\treturn\n\t}\n\n\tfor {\n\t\t// On reload, if the certificate changes then need to stop this monitor.\n\t\tselect {\n\t\tcase <-time.After(nextRun):\n\t\tcase <-stopCh:\n\t\t\t// In case of reload and have to restart the OCSP stapling monitoring.\n\t\t\treturn\n\t\tcase <-quitCh:\n\t\t\t// Server quit channel.\n\t\t\treturn\n\t\t}\n\t\t_, resp, err := oc.getRemoteStatus()\n\t\tif err != nil {\n\t\t\tnextRun = oc.getNextRun()\n\t\t\ts.Errorf(\"Bad OCSP status update for certificate '%s': %s, trying again in %v\", certFile, err, nextRun)\n\t\t\tcontinue\n\t\t}\n\n\t\tswitch n := resp.Status; n {\n\t\tcase ocsp.Good:\n\t\t\tnextRun = oc.getNextRun()\n\t\t\tt := resp.NextUpdate.Format(time.RFC3339Nano)\n\t\t\ts.Noticef(\n\t\t\t\t\"Received OCSP status for %s certificate '%s': good, next update %s, checking again in %s\",\n\t\t\t\tkind, certFile, t, nextRun,\n\t\t\t)\n\t\t\tcontinue\n\t\tdefault:\n\t\t\ts.Errorf(\"Received OCSP status for %s certificate '%s': %s\", kind, certFile, ocspStatusString(n))\n\t\t\tif shutdownOnRevoke {\n\t\t\t\tdoShutdown = true\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (oc *OCSPMonitor) stop() {\n\toc.mu.Lock()\n\tstopCh := oc.stopCh\n\toc.mu.Unlock()\n\tstopCh <- struct{}{}\n}\n\n// NewOCSPMonitor takes a TLS configuration then wraps it with the callbacks set for OCSP verification\n// along with a monitor that will periodically fetch OCSP staples.\nfunc (srv *Server) NewOCSPMonitor(config *tlsConfigKind) (*tls.Config, *OCSPMonitor, error) {\n\tkind := config.kind\n\ttc := config.tlsConfig\n\ttcOpts := config.tlsOpts\n\topts := srv.getOpts()\n\toc := opts.OCSPConfig\n\n\t// We need to track the CA certificate in case the CA is not present\n\t// in the chain to be able to verify the signature of the OCSP staple.\n\tvar (\n\t\tcertFile string\n\t\tcaFile string\n\t)\n\tif kind == kindStringMap[CLIENT] {\n\t\ttcOpts = opts.tlsConfigOpts\n\t\tif opts.TLSCert != _EMPTY_ {\n\t\t\tcertFile = opts.TLSCert\n\t\t}\n\t\tif opts.TLSCaCert != _EMPTY_ {\n\t\t\tcaFile = opts.TLSCaCert\n\t\t}\n\t}\n\tif tcOpts != nil {\n\t\tcertFile = tcOpts.CertFile\n\t\tcaFile = tcOpts.CaFile\n\t}\n\n\t// NOTE: Currently OCSP Stapling is enabled only for the first certificate found.\n\tvar mon *OCSPMonitor\n\tfor _, currentCert := range tc.Certificates {\n\t\t// Create local copy since this will be used in the GetCertificate callback.\n\t\tcert := currentCert\n\n\t\t// This is normally non-nil, but can still be nil here when in tests\n\t\t// or in some embedded scenarios.\n\t\tif cert.Leaf == nil {\n\t\t\tif len(cert.Certificate) <= 0 {\n\t\t\t\treturn nil, nil, fmt.Errorf(\"no certificate found\")\n\t\t\t}\n\t\t\tvar err error\n\t\t\tcert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, fmt.Errorf(\"error parsing certificate: %v\", err)\n\t\t\t}\n\t\t}\n\t\tvar shutdownOnRevoke bool\n\t\tmustStaple := hasOCSPStatusRequest(cert.Leaf)\n\t\tif oc != nil {\n\t\t\tswitch {\n\t\t\tcase oc.Mode == OCSPModeNever:\n\t\t\t\tif mustStaple {\n\t\t\t\t\tsrv.Warnf(\"Certificate at '%s' has MustStaple but OCSP is disabled\", certFile)\n\t\t\t\t}\n\t\t\t\treturn tc, nil, nil\n\t\t\tcase oc.Mode == OCSPModeAlways:\n\t\t\t\t// Start the monitor for this cert even if it does not have\n\t\t\t\t// the MustStaple flag and shutdown the server in case the\n\t\t\t\t// staple ever gets revoked.\n\t\t\t\tmustStaple = true\n\t\t\t\tshutdownOnRevoke = true\n\t\t\tcase oc.Mode == OCSPModeMust && mustStaple:\n\t\t\t\tshutdownOnRevoke = true\n\t\t\tcase oc.Mode == OCSPModeAuto && !mustStaple:\n\t\t\t\t// \"status_request\" MustStaple flag not set in certificate. No need to do anything.\n\t\t\t\treturn tc, nil, nil\n\t\t\t}\n\t\t}\n\t\tif !mustStaple {\n\t\t\t// No explicit OCSP config and cert does not have MustStaple flag either.\n\t\t\treturn tc, nil, nil\n\t\t}\n\n\t\tif err := srv.setupOCSPStapleStoreDir(); err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\t// TODO: Add OCSP 'responder_cert' option in case CA cert not available.\n\t\tissuer, err := getOCSPIssuer(caFile, cert.Certificate)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tmon = &OCSPMonitor{\n\t\t\tkind: kind,\n\t\t\tsrv: srv,\n\t\t\thc: &http.Client{Timeout: 30 * time.Second},\n\t\t\tshutdownOnRevoke: shutdownOnRevoke,\n\t\t\tcertFile: certFile,\n\t\t\tstopCh: make(chan struct{}, 1),\n\t\t\tLeaf: cert.Leaf,\n\t\t\tIssuer: issuer,\n\t\t}\n\n\t\t// Get the certificate status from the memory, then remote OCSP responder.\n\t\tif _, resp, err := mon.getStatus(); err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"bad OCSP status update for certificate at '%s': %s\", certFile, err)\n\t\t} else if resp != nil && resp.Status != ocsp.Good && shutdownOnRevoke {\n\t\t\treturn nil, nil, fmt.Errorf(\"found existing OCSP status for certificate at '%s': %s\", certFile, ocspStatusString(resp.Status))\n\t\t}\n\n\t\t// Callbacks below will be in charge of returning the certificate instead,\n\t\t// so this has to be nil.\n\t\ttc.Certificates = nil\n\n\t\t// GetCertificate returns a certificate that's presented to a client.\n\t\ttc.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {\n\t\t\tccert := cert\n\t\t\traw, _, err := mon.getStatus()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn &tls.Certificate{\n\t\t\t\tOCSPStaple: raw,\n\t\t\t\tCertificate: ccert.Certificate,\n\t\t\t\tPrivateKey: ccert.PrivateKey,\n\t\t\t\tSupportedSignatureAlgorithms: ccert.SupportedSignatureAlgorithms,\n\t\t\t\tSignedCertificateTimestamps: ccert.SignedCertificateTimestamps,\n\t\t\t\tLeaf: ccert.Leaf,\n\t\t\t}, nil\n\t\t}\n\n\t\t// Check whether need to verify staples from a peer router or gateway connection.\n\t\tswitch kind {\n\t\tcase kindStringMap[ROUTER], kindStringMap[GATEWAY]:\n\t\t\ttc.VerifyConnection = func(s tls.ConnectionState) error {\n\t\t\t\toresp := s.OCSPResponse\n\t\t\t\tif oresp == nil {\n\t\t\t\t\treturn fmt.Errorf(\"%s peer missing OCSP Staple\", kind)\n\t\t\t\t}\n\n\t\t\t\t// Peer connections will verify the response of the staple.\n\t\t\t\tif len(s.VerifiedChains) == 0 {\n\t\t\t\t\treturn fmt.Errorf(\"%s peer missing TLS verified chains\", kind)\n\t\t\t\t}\n\n\t\t\t\tchain := s.VerifiedChains[0]\n\t\t\t\tpeerLeaf := chain[0]\n\t\t\t\tpeerIssuer := certidp.GetLeafIssuerCert(chain, 0)\n\t\t\t\tif peerIssuer == nil {\n\t\t\t\t\treturn fmt.Errorf(\"failed to get issuer certificate for %s peer\", kind)\n\t\t\t\t}\n\n\t\t\t\t// Response signature of issuer or issuer delegate is checked in the library parse\n\t\t\t\tresp, err := ocsp.ParseResponseForCert(oresp, peerLeaf, peerIssuer)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"failed to parse OCSP response from %s peer: %w\", kind, err)\n\t\t\t\t}\n\n\t\t\t\t// If signer was issuer delegate double-check issuer delegate authorization\n\t\t\t\tif resp.Certificate != nil {\n\t\t\t\t\tok := false\n\t\t\t\t\tfor _, eku := range resp.Certificate.ExtKeyUsage {\n\t\t\t\t\t\tif eku == x509.ExtKeyUsageOCSPSigning {\n\t\t\t\t\t\t\tok = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn fmt.Errorf(\"OCSP staple's signer missing authorization by CA to act as OCSP signer\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Check that the OCSP response is effective, take defaults for clockskew and default validity\n\t\t\t\tpeerOpts := certidp.OCSPPeerConfig{ClockSkew: -1, TTLUnsetNextUpdate: -1}\n\t\t\t\tsLog := certidp.Log{Debugf: srv.Debugf}\n\t\t\t\tif !certidp.OCSPResponseCurrent(resp, &peerOpts, &sLog) {\n\t\t\t\t\treturn fmt.Errorf(\"OCSP staple from %s peer not current\", kind)\n\t\t\t\t}\n\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\treturn fmt.Errorf(\"bad status for OCSP Staple from %s peer: %s\", kind, ocspStatusString(resp.Status))\n\t\t\t\t}\n\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\t// When server makes a peer connection, need to also present an OCSP Staple.\n\t\t\ttc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {\n\t\t\t\tccert := cert\n\t\t\t\traw, _, err := mon.getStatus()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\t// NOTE: crypto/tls.sendClientCertificate internally also calls getClientCertificate\n\t\t\t\t// so if for some reason these callbacks are triggered concurrently during a reconnect\n\t\t\t\t// there can be a race. To avoid that, the OCSP monitor lock is used to serialize access\n\t\t\t\t// to the staple which could also change inflight during an update.\n\t\t\t\tmon.mu.Lock()\n\t\t\t\tccert.OCSPStaple = raw\n\t\t\t\tmon.mu.Unlock()\n\n\t\t\t\treturn &ccert, nil\n\t\t\t}\n\t\tdefault:\n\t\t\t// GetClientCertificate returns a certificate that's presented to a server.\n\t\t\ttc.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {\n\t\t\t\treturn &cert, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn tc, mon, nil\n}\n\nfunc (s *Server) setupOCSPStapleStoreDir() error {\n\topts := s.getOpts()\n\tstoreDir := opts.StoreDir\n\tif storeDir == _EMPTY_ {\n\t\treturn nil\n\t}\n\tstoreDir = filepath.Join(storeDir, defaultOCSPStoreDir)\n\tif stat, err := os.Stat(storeDir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(storeDir, defaultDirPerms); err != nil {\n\t\t\treturn fmt.Errorf(\"could not create OCSP storage directory - %v\", err)\n\t\t}\n\t} else if stat == nil || !stat.IsDir() {\n\t\treturn fmt.Errorf(\"OCSP storage directory is not a directory\")\n\t}\n\treturn nil\n}\n\ntype tlsConfigKind struct {\n\ttlsConfig *tls.Config\n\ttlsOpts *TLSConfigOpts\n\tkind string\n\tisLeafSpoke bool\n\tapply func(*tls.Config)\n}\n\nfunc (s *Server) configureOCSP() []*tlsConfigKind {\n\tsopts := s.getOpts()\n\n\tconfigs := make([]*tlsConfigKind, 0)\n\n\tif config := sopts.TLSConfig; config != nil {\n\t\topts := sopts.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[CLIENT],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tif config := sopts.Websocket.TLSConfig; config != nil {\n\t\topts := sopts.Websocket.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[CLIENT],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.Websocket.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tif config := sopts.MQTT.TLSConfig; config != nil {\n\t\topts := sopts.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[CLIENT],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.MQTT.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tif config := sopts.Cluster.TLSConfig; config != nil {\n\t\topts := sopts.Cluster.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[ROUTER],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.Cluster.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tif config := sopts.LeafNode.TLSConfig; config != nil {\n\t\topts := sopts.LeafNode.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[LEAF],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.LeafNode.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tfor _, remote := range sopts.LeafNode.Remotes {\n\t\tif config := remote.TLSConfig; config != nil {\n\t\t\t// Use a copy of the remote here since will be used\n\t\t\t// in the apply func callback below.\n\t\t\tr, opts := remote, remote.tlsConfigOpts\n\t\t\to := &tlsConfigKind{\n\t\t\t\tkind: kindStringMap[LEAF],\n\t\t\t\ttlsConfig: config,\n\t\t\t\ttlsOpts: opts,\n\t\t\t\tisLeafSpoke: true,\n\t\t\t\tapply: func(tc *tls.Config) { r.TLSConfig = tc },\n\t\t\t}\n\t\t\tconfigs = append(configs, o)\n\t\t}\n\t}\n\tif config := sopts.Gateway.TLSConfig; config != nil {\n\t\topts := sopts.Gateway.tlsConfigOpts\n\t\to := &tlsConfigKind{\n\t\t\tkind: kindStringMap[GATEWAY],\n\t\t\ttlsConfig: config,\n\t\t\ttlsOpts: opts,\n\t\t\tapply: func(tc *tls.Config) { sopts.Gateway.TLSConfig = tc },\n\t\t}\n\t\tconfigs = append(configs, o)\n\t}\n\tfor _, remote := range sopts.Gateway.Gateways {\n\t\tif config := remote.TLSConfig; config != nil {\n\t\t\tgw, opts := remote, remote.tlsConfigOpts\n\t\t\to := &tlsConfigKind{\n\t\t\t\tkind: kindStringMap[GATEWAY],\n\t\t\t\ttlsConfig: config,\n\t\t\t\ttlsOpts: opts,\n\t\t\t\tapply: func(tc *tls.Config) { gw.TLSConfig = tc },\n\t\t\t}\n\t\t\tconfigs = append(configs, o)\n\t\t}\n\t}\n\treturn configs\n}\n\nfunc (s *Server) enableOCSP() error {\n\tconfigs := s.configureOCSP()\n\n\tfor _, config := range configs {\n\n\t\t// We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer\n\t\tif config.kind != kindStringMap[LEAF] {\n\t\t\t// OCSP Stapling feature, will also enable tls server peer check for gateway and route peers\n\t\t\ttc, mon, err := s.NewOCSPMonitor(config)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t// Check if an OCSP stapling monitor is required for this certificate.\n\t\t\tif mon != nil {\n\t\t\t\ts.ocsps = append(s.ocsps, mon)\n\n\t\t\t\t// Override the TLS config with one that follows OCSP stapling\n\t\t\t\tconfig.apply(tc)\n\t\t\t}\n\t\t}\n\n\t\t// OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS)\n\t\tif config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] {\n\t\t\ttc, plugged, err := s.plugTLSOCSPPeer(config)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif plugged && tc != nil {\n\t\t\t\ts.ocspPeerVerify = true\n\t\t\t\tconfig.apply(tc)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (s *Server) startOCSPMonitoring() {\n\ts.mu.Lock()\n\tocsps := s.ocsps\n\ts.mu.Unlock()\n\tif ocsps == nil {\n\t\treturn\n\t}\n\tfor _, mon := range ocsps {\n\t\tm := mon\n\t\tm.mu.Lock()\n\t\tkind := m.kind\n\t\tm.mu.Unlock()\n\t\ts.Noticef(\"OCSP Stapling enabled for %s connections\", kind)\n\t\ts.startGoRoutine(func() { m.run() })\n\t}\n}\n\nfunc (s *Server) reloadOCSP() error {\n\tif err := s.setupOCSPStapleStoreDir(); err != nil {\n\t\treturn err\n\t}\n\n\ts.mu.Lock()\n\tocsps := s.ocsps\n\ts.mu.Unlock()\n\n\t// Stop all OCSP Stapling monitors in case there were any running.\n\tfor _, oc := range ocsps {\n\t\toc.stop()\n\t}\n\n\tconfigs := s.configureOCSP()\n\n\t// Restart the monitors under the new configuration.\n\tocspm := make([]*OCSPMonitor, 0)\n\n\t// Reset server's ocspPeerVerify flag to re-detect at least one plugged OCSP peer\n\ts.mu.Lock()\n\ts.ocspPeerVerify = false\n\ts.mu.Unlock()\n\ts.stopOCSPResponseCache()\n\n\tfor _, config := range configs {\n\t\t// We do not staple Leaf Hub and Leaf Spokes, use ocsp_peer\n\t\tif config.kind != kindStringMap[LEAF] {\n\t\t\ttc, mon, err := s.NewOCSPMonitor(config)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t// Check if an OCSP stapling monitor is required for this certificate.\n\t\t\tif mon != nil {\n\t\t\t\tocspm = append(ocspm, mon)\n\n\t\t\t\t// Apply latest TLS configuration after OCSP monitors have started.\n\t\t\t\tdefer config.apply(tc)\n\t\t\t}\n\t\t}\n\n\t\t// OCSP peer check (client mTLS, leaf mTLS, leaf remote TLS)\n\t\tif config.kind == kindStringMap[CLIENT] || config.kind == kindStringMap[LEAF] {\n\t\t\ttc, plugged, err := s.plugTLSOCSPPeer(config)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif plugged && tc != nil {\n\t\t\t\ts.ocspPeerVerify = true\n\t\t\t\tdefer config.apply(tc)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Replace stopped monitors with the new ones.\n\ts.mu.Lock()\n\ts.ocsps = ocspm\n\ts.mu.Unlock()\n\n\t// Dispatch all goroutines once again.\n\ts.startOCSPMonitoring()\n\n\t// Init and restart OCSP responder cache\n\ts.stopOCSPResponseCache()\n\ts.initOCSPResponseCache()\n\ts.startOCSPResponseCache()\n\n\treturn nil\n}\n\nfunc hasOCSPStatusRequest(cert *x509.Certificate) bool {\n\t// OID for id-pe-tlsfeature defined in RFC here:\n\t// https://datatracker.ietf.org/doc/html/rfc7633\n\ttlsFeatures := asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 24}\n\tconst statusRequestExt = 5\n\n\t// Example values:\n\t// * [48 3 2 1 5] - seen when creating own certs locally\n\t// * [30 3 2 1 5] - seen in the wild\n\t// Documentation:\n\t// https://tools.ietf.org/html/rfc6066\n\n\tfor _, ext := range cert.Extensions {\n\t\tif !ext.Id.Equal(tlsFeatures) {\n\t\t\tcontinue\n\t\t}\n\n\t\tvar val []int\n\t\trest, err := asn1.Unmarshal(ext.Value, &val)\n\t\tif err != nil || len(rest) > 0 {\n\t\t\treturn false\n\t\t}\n\n\t\tfor _, n := range val {\n\t\t\tif n == statusRequestExt {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tbreak\n\t}\n\n\treturn false\n}\n\n// writeOCSPStatus writes an OCSP status to a temporary file then moves it to a\n// new path, in an attempt to avoid corrupting existing data.\nfunc (oc *OCSPMonitor) writeOCSPStatus(storeDir, file string, data []byte) error {\n\tstoreDir = filepath.Join(storeDir, defaultOCSPStoreDir)\n\ttmp, err := os.CreateTemp(storeDir, \"tmp-cert-status\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := tmp.Write(data); err != nil {\n\t\ttmp.Close()\n\t\tos.Remove(tmp.Name())\n\t\treturn err\n\t}\n\tif err := tmp.Close(); err != nil {\n\t\treturn err\n\t}\n\n\toc.mu.Lock()\n\terr = os.Rename(tmp.Name(), filepath.Join(storeDir, file))\n\toc.mu.Unlock()\n\tif err != nil {\n\t\tos.Remove(tmp.Name())\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc parseCertPEM(name string) ([]*x509.Certificate, error) {\n\tdata, err := os.ReadFile(name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar pemBytes []byte\n\n\tvar block *pem.Block\n\tfor len(data) != 0 {\n\t\tblock, data = pem.Decode(data)\n\t\tif block == nil {\n\t\t\tbreak\n\t\t}\n\t\tif block.Type != \"CERTIFICATE\" {\n\t\t\treturn nil, fmt.Errorf(\"unexpected PEM certificate type: %s\", block.Type)\n\t\t}\n\n\t\tpemBytes = append(pemBytes, block.Bytes...)\n\t}\n\n\treturn x509.ParseCertificates(pemBytes)\n}\n\n// getOCSPIssuerLocally determines a leaf's issuer from locally configured certificates\nfunc getOCSPIssuerLocally(trustedCAs []*x509.Certificate, certBundle []*x509.Certificate) (*x509.Certificate, error) {\n\tvar vOpts x509.VerifyOptions\n\tvar leaf *x509.Certificate\n\ttrustedCAPool := x509.NewCertPool()\n\n\t// Require Leaf as first cert in bundle\n\tif len(certBundle) > 0 {\n\t\tleaf = certBundle[0]\n\t} else {\n\t\treturn nil, fmt.Errorf(\"invalid ocsp ca configuration\")\n\t}\n\n\t// Allow Issuer to be configured as second cert in bundle\n\tif len(certBundle) > 1 {\n\t\t// The operator may have misconfigured the cert bundle\n\t\tissuerCandidate := certBundle[1]\n\t\terr := issuerCandidate.CheckSignature(leaf.SignatureAlgorithm, leaf.RawTBSCertificate, leaf.Signature)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"invalid issuer configuration: %w\", err)\n\t\t} else {\n\t\t\treturn issuerCandidate, nil\n\t\t}\n\t}\n\n\t// Operator did not provide the Leaf Issuer in cert bundle second position\n\t// so we will attempt to create at least one ordered verified chain from the\n\t// trusted CA pool.\n\n\t// Specify CA trust store to validator; if unset, system trust store used\n\tif len(trustedCAs) > 0 {\n\t\tfor _, ca := range trustedCAs {\n\t\t\ttrustedCAPool.AddCert(ca)\n\t\t}\n\t\tvOpts.Roots = trustedCAPool\n\t}\n\n\treturn certstore.GetLeafIssuer(leaf, vOpts), nil\n}\n\n// getOCSPIssuer determines an issuer certificate from the cert (bundle) or the file-based CA trust store\nfunc getOCSPIssuer(caFile string, chain [][]byte) (*x509.Certificate, error) {\n\tvar issuer *x509.Certificate\n\tvar trustedCAs []*x509.Certificate\n\tvar certBundle []*x509.Certificate\n\tvar err error\n\n\t// FIXME(tgb): extend if pluggable CA store provider added to NATS (i.e. other than PEM file)\n\n\t// Non-system default CA trust store passed\n\tif caFile != _EMPTY_ {\n\t\ttrustedCAs, err = parseCertPEM(caFile)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to parse ca_file: %v\", err)\n\t\t}\n\t}\n\n\t// Specify bundled intermediate CA store\n\tfor _, certBytes := range chain {\n\t\tcert, err := x509.ParseCertificate(certBytes)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to parse cert: %v\", err)\n\t\t}\n\t\tcertBundle = append(certBundle, cert)\n\t}\n\n\tissuer, err = getOCSPIssuerLocally(trustedCAs, certBundle)\n\tif err != nil || issuer == nil {\n\t\treturn nil, fmt.Errorf(\"no issuers found\")\n\t}\n\n\tif !issuer.IsCA {\n\t\treturn nil, fmt.Errorf(\"%s invalid ca basic constraints: is not ca\", issuer.Subject)\n\t}\n\treturn issuer, nil\n}\n\nfunc ocspStatusString(n int) string {\n\tswitch n {\n\tcase ocsp.Good:\n\t\treturn \"good\"\n\tcase ocsp.Revoked:\n\t\treturn \"revoked\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\nfunc validOCSPResponse(r *ocsp.Response) error {\n\t// Time validation not handled by ParseResponse.\n\t// https://tools.ietf.org/html/rfc6960#section-4.2.2.1\n\tif !r.NextUpdate.IsZero() && r.NextUpdate.Before(time.Now()) {\n\t\tt := r.NextUpdate.Format(time.RFC3339Nano)\n\t\treturn fmt.Errorf(\"invalid ocsp NextUpdate, is past time: %s\", t)\n\t}\n\tif r.ThisUpdate.After(time.Now()) {\n\t\tt := r.ThisUpdate.Format(time.RFC3339Nano)\n\t\treturn fmt.Errorf(\"invalid ocsp ThisUpdate, is future time: %s\", t)\n\t}\n\n\treturn nil\n}\n" }, { "path": "server/ocsp_peer.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"golang.org/x/crypto/ocsp\"\n\n\t\"github.com/nats-io/nats-server/v2/server/certidp\"\n)\n\nfunc parseOCSPPeer(v any) (pcfg *certidp.OCSPPeerConfig, retError error) {\n\tvar lt token\n\tdefer convertPanicToError(<, &retError)\n\ttk, v := unwrapValue(v, <)\n\tcm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalPeerOptsConfig, v)}\n\t}\n\tpcfg = certidp.NewOCSPPeerConfig()\n\tretError = nil\n\tfor mk, mv := range cm {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"verify\":\n\t\t\tverify, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.Verify = verify\n\t\tcase \"allowed_clockskew\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, \"unexpected type\")}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, \"unexpected type\")}\n\t\t\t}\n\t\t\tif at >= 0 {\n\t\t\t\tpcfg.ClockSkew = at\n\t\t\t}\n\t\tcase \"ca_timeout\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, \"unexpected type\")}\n\t\t\t}\n\t\t\tif at >= 0 {\n\t\t\t\tpcfg.Timeout = at\n\t\t\t}\n\t\tcase \"cache_ttl_when_next_update_unset\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, \"unexpected type\")}\n\t\t\t}\n\t\t\tif at >= 0 {\n\t\t\t\tpcfg.TTLUnsetNextUpdate = at\n\t\t\t}\n\t\tcase \"warn_only\":\n\t\t\twarnOnly, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.WarnOnly = warnOnly\n\t\tcase \"unknown_is_good\":\n\t\t\tunknownIsGood, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.UnknownIsGood = unknownIsGood\n\t\tcase \"allow_when_ca_unreachable\":\n\t\t\tallowWhenCAUnreachable, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.AllowWhenCAUnreachable = allowWhenCAUnreachable\n\t\tdefault:\n\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldGeneric, mk)}\n\t\t}\n\t}\n\treturn pcfg, nil\n}\n\nfunc peerFromVerifiedChains(chains [][]*x509.Certificate) *x509.Certificate {\n\tif len(chains) == 0 || len(chains[0]) == 0 {\n\t\treturn nil\n\t}\n\treturn chains[0][0]\n}\n\n// plugTLSOCSPPeer will plug the TLS handshake lifecycle for client mTLS connections and Leaf connections\nfunc (s *Server) plugTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) {\n\tif config == nil || config.tlsConfig == nil {\n\t\treturn nil, false, errors.New(certidp.ErrUnableToPlugTLSEmptyConfig)\n\t}\n\tkind := config.kind\n\tisSpoke := config.isLeafSpoke\n\ttcOpts := config.tlsOpts\n\tif tcOpts == nil || tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify {\n\t\treturn nil, false, nil\n\t}\n\ts.Debugf(certidp.DbgPlugTLSForKind, config.kind)\n\t// peer is a tls client\n\tif kind == kindStringMap[CLIENT] || (kind == kindStringMap[LEAF] && !isSpoke) {\n\t\tif !tcOpts.Verify {\n\t\t\treturn nil, false, errors.New(certidp.ErrMTLSRequired)\n\t\t}\n\t\treturn s.plugClientTLSOCSPPeer(config)\n\t}\n\t// peer is a tls server\n\tif kind == kindStringMap[LEAF] && isSpoke {\n\t\treturn s.plugServerTLSOCSPPeer(config)\n\t}\n\treturn nil, false, nil\n}\n\nfunc (s *Server) plugClientTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) {\n\tif config == nil || config.tlsConfig == nil || config.tlsOpts == nil {\n\t\treturn nil, false, errors.New(certidp.ErrUnableToPlugTLSClient)\n\t}\n\ttc := config.tlsConfig\n\ttcOpts := config.tlsOpts\n\tkind := config.kind\n\tif tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify {\n\t\treturn tc, false, nil\n\t}\n\ttc.VerifyConnection = func(cs tls.ConnectionState) error {\n\t\tif !s.tlsClientOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) {\n\t\t\ts.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSClientRejectConnection)\n\t\t\treturn errors.New(certidp.MsgTLSClientRejectConnection)\n\t\t}\n\t\treturn nil\n\t}\n\treturn tc, true, nil\n}\n\nfunc (s *Server) plugServerTLSOCSPPeer(config *tlsConfigKind) (*tls.Config, bool, error) {\n\tif config == nil || config.tlsConfig == nil || config.tlsOpts == nil {\n\t\treturn nil, false, errors.New(certidp.ErrUnableToPlugTLSServer)\n\t}\n\ttc := config.tlsConfig\n\ttcOpts := config.tlsOpts\n\tkind := config.kind\n\tif tcOpts.OCSPPeerConfig == nil || !tcOpts.OCSPPeerConfig.Verify {\n\t\treturn tc, false, nil\n\t}\n\ttc.VerifyConnection = func(cs tls.ConnectionState) error {\n\t\tif !s.tlsServerOCSPValid(cs.VerifiedChains, tcOpts.OCSPPeerConfig) {\n\t\t\ts.sendOCSPPeerRejectEvent(kind, peerFromVerifiedChains(cs.VerifiedChains), certidp.MsgTLSServerRejectConnection)\n\t\t\treturn errors.New(certidp.MsgTLSServerRejectConnection)\n\t\t}\n\t\treturn nil\n\t}\n\treturn tc, true, nil\n}\n\n// tlsServerOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP\n// eligibility. A verified chain is considered OCSP Valid if either none of the links are\n// OCSP eligible, or current \"good\" responses from the CA can be obtained for each eligible link.\n// Upon first OCSP Valid chain found, the Server is deemed OCSP Valid. If none of the chains are\n// OCSP Valid, the Server is deemed OCSP Invalid. A verified self-signed certificate (chain length 1)\n// is also considered OCSP Valid.\nfunc (s *Server) tlsServerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool {\n\ts.Debugf(certidp.DbgNumServerChains, len(chains))\n\treturn s.peerOCSPValid(chains, opts)\n}\n\n// tlsClientOCSPValid evaluates verified chains (post successful TLS handshake) against OCSP\n// eligibility. A verified chain is considered OCSP Valid if either none of the links are\n// OCSP eligible, or current \"good\" responses from the CA can be obtained for each eligible link.\n// Upon first OCSP Valid chain found, the Client is deemed OCSP Valid. If none of the chains are\n// OCSP Valid, the Client is deemed OCSP Invalid. A verified self-signed certificate (chain length 1)\n// is also considered OCSP Valid.\nfunc (s *Server) tlsClientOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool {\n\ts.Debugf(certidp.DbgNumClientChains, len(chains))\n\treturn s.peerOCSPValid(chains, opts)\n}\n\nfunc (s *Server) peerOCSPValid(chains [][]*x509.Certificate, opts *certidp.OCSPPeerConfig) bool {\n\tpeer := peerFromVerifiedChains(chains)\n\tif peer == nil {\n\t\ts.Errorf(certidp.ErrPeerEmptyAutoReject)\n\t\treturn false\n\t}\n\tfor ci, chain := range chains {\n\t\ts.Debugf(certidp.DbgLinksInChain, ci, len(chain))\n\t\t// Self-signed certificate is Client OCSP Valid (no CA)\n\t\tif len(chain) == 1 {\n\t\t\ts.Debugf(certidp.DbgSelfSignedValid, ci)\n\t\t\treturn true\n\t\t}\n\t\t// Check if any of the links in the chain are OCSP eligible\n\t\tchainEligible := false\n\t\tvar eligibleLinks []*certidp.ChainLink\n\t\t// Iterate over links skipping the root cert which is not OCSP eligible (self == issuer)\n\t\tfor linkPos := 0; linkPos < len(chain)-1; linkPos++ {\n\t\t\tcert := chain[linkPos]\n\t\t\tlink := &certidp.ChainLink{\n\t\t\t\tLeaf: cert,\n\t\t\t}\n\t\t\tif certidp.CertOCSPEligible(link) {\n\t\t\t\tchainEligible = true\n\t\t\t\tissuerCert := certidp.GetLeafIssuerCert(chain, linkPos)\n\t\t\t\tif issuerCert == nil {\n\t\t\t\t\t// unexpected chain condition, reject Client as OCSP Invalid\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\tlink.Issuer = issuerCert\n\t\t\t\teligibleLinks = append(eligibleLinks, link)\n\t\t\t}\n\t\t}\n\t\t// A trust-store verified chain that is not OCSP eligible is always OCSP Valid\n\t\tif !chainEligible {\n\t\t\ts.Debugf(certidp.DbgValidNonOCSPChain, ci)\n\t\t\treturn true\n\t\t}\n\t\ts.Debugf(certidp.DbgChainIsOCSPEligible, ci, len(eligibleLinks))\n\t\t// Chain has at least one OCSP eligible link, so check each eligible link;\n\t\t// any link with a !good OCSP response chain OCSP Invalid\n\t\tchainValid := true\n\t\tfor _, link := range eligibleLinks {\n\t\t\t// if option selected, good could reflect either ocsp.Good or ocsp.Unknown\n\t\t\tif badReason, good := s.certOCSPGood(link, opts); !good {\n\t\t\t\ts.Debugf(badReason)\n\t\t\t\ts.sendOCSPPeerChainlinkInvalidEvent(peer, link.Leaf, badReason)\n\t\t\t\tchainValid = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif chainValid {\n\t\t\ts.Debugf(certidp.DbgChainIsOCSPValid, ci)\n\t\t\treturn true\n\t\t}\n\t}\n\t// If we are here, all chains had OCSP eligible links, but none of the chains achieved OCSP valid\n\ts.Debugf(certidp.DbgNoOCSPValidChains)\n\treturn false\n}\n\nfunc (s *Server) certOCSPGood(link *certidp.ChainLink, opts *certidp.OCSPPeerConfig) (string, bool) {\n\tif link == nil || link.Leaf == nil || link.Issuer == nil || link.OCSPWebEndpoints == nil || len(*link.OCSPWebEndpoints) < 1 {\n\t\treturn \"Empty chainlink found\", false\n\t}\n\tvar err error\n\tsLogs := &certidp.Log{\n\t\tDebugf: s.Debugf,\n\t\tNoticef: s.Noticef,\n\t\tWarnf: s.Warnf,\n\t\tErrorf: s.Errorf,\n\t\tTracef: s.Tracef,\n\t}\n\tfingerprint := certidp.GenerateFingerprint(link.Leaf)\n\t// Used for debug/operator only, not match\n\tsubj := certidp.GetSubjectDNForm(link.Leaf)\n\tvar rawResp []byte\n\tvar ocspr *ocsp.Response\n\tvar useCachedResp bool\n\tvar rc = s.ocsprc\n\tvar cachedRevocation bool\n\t// Check our cache before calling out to the CA OCSP responder\n\ts.Debugf(certidp.DbgCheckingCacheForCert, subj, fingerprint)\n\tif rawResp = rc.Get(fingerprint, sLogs); len(rawResp) > 0 {\n\t\t// Signature validation of CA's OCSP response occurs in ParseResponse\n\t\tocspr, err = ocsp.ParseResponse(rawResp, link.Issuer)\n\t\tif err == nil && ocspr != nil {\n\t\t\t// Check if OCSP Response delegation present and if so is valid\n\t\t\tif !certidp.ValidDelegationCheck(link.Issuer, ocspr) {\n\t\t\t\t// Invalid delegation was already in cache, purge it and don't use it\n\t\t\t\ts.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj)\n\t\t\t\trc.Delete(fingerprint, true, sLogs)\n\t\t\t\tgoto AFTERCACHE\n\t\t\t}\n\t\t\tif certidp.OCSPResponseCurrent(ocspr, opts, sLogs) {\n\t\t\t\ts.Debugf(certidp.DbgCurrentResponseCached, certidp.GetStatusAssertionStr(ocspr.Status))\n\t\t\t\tuseCachedResp = true\n\t\t\t} else {\n\t\t\t\t// Cached response is not current, delete it and tidy runtime stats to reflect a miss;\n\t\t\t\t// if preserve_revoked is enabled, the cache will not delete the cached response\n\t\t\t\ts.Debugf(certidp.DbgExpiredResponseCached, certidp.GetStatusAssertionStr(ocspr.Status))\n\t\t\t\trc.Delete(fingerprint, true, sLogs)\n\t\t\t}\n\t\t\t// Regardless of currency, record a cached revocation found in case AllowWhenCAUnreachable is set\n\t\t\tif ocspr.Status == ocsp.Revoked {\n\t\t\t\tcachedRevocation = true\n\t\t\t}\n\t\t} else {\n\t\t\t// Bogus cached assertion, purge it and don't use it\n\t\t\ts.Debugf(certidp.MsgCachedOCSPResponseInvalid, subj, fingerprint)\n\t\t\trc.Delete(fingerprint, true, sLogs)\n\t\t\tgoto AFTERCACHE\n\t\t}\n\t}\nAFTERCACHE:\n\tif !useCachedResp {\n\t\t// CA OCSP responder callout needed\n\t\trawResp, err = certidp.FetchOCSPResponse(link, opts, sLogs)\n\t\tif err != nil || rawResp == nil || len(rawResp) == 0 {\n\t\t\ts.Warnf(certidp.ErrCAResponderCalloutFail, subj, err)\n\t\t\tif opts.WarnOnly {\n\t\t\t\ts.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj)\n\t\t\t\treturn _EMPTY_, true\n\t\t\t}\n\t\t\tif opts.AllowWhenCAUnreachable && !cachedRevocation {\n\t\t\t\t// Link has no cached history of revocation, so allow it to pass\n\t\t\t\ts.Warnf(certidp.MsgAllowWhenCAUnreachableOccurred, subj)\n\t\t\t\treturn _EMPTY_, true\n\t\t\t} else if opts.AllowWhenCAUnreachable {\n\t\t\t\t// Link has cached but expired revocation so reject when CA is unreachable\n\t\t\t\ts.Warnf(certidp.MsgAllowWhenCAUnreachableOccurredCachedRevoke, subj)\n\t\t\t}\n\t\t\treturn certidp.MsgFailedOCSPResponseFetch, false\n\t\t}\n\t\t// Signature validation of CA's OCSP response occurs in ParseResponse\n\t\tocspr, err = ocsp.ParseResponse(rawResp, link.Issuer)\n\t\tif err == nil && ocspr != nil {\n\t\t\t// Check if OCSP Response delegation present and if so is valid\n\t\t\tif !certidp.ValidDelegationCheck(link.Issuer, ocspr) {\n\t\t\t\ts.Warnf(certidp.MsgOCSPResponseDelegationInvalid, subj)\n\t\t\t\tif opts.WarnOnly {\n\t\t\t\t\t// Can't use bogus assertion, but warn-only set so allow link to pass\n\t\t\t\t\ts.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj)\n\t\t\t\t\treturn _EMPTY_, true\n\t\t\t\t}\n\t\t\t\treturn fmt.Sprintf(certidp.MsgOCSPResponseDelegationInvalid, subj), false\n\t\t\t}\n\t\t\tif !certidp.OCSPResponseCurrent(ocspr, opts, sLogs) {\n\t\t\t\ts.Warnf(certidp.ErrNewCAResponseNotCurrent, subj)\n\t\t\t\tif opts.WarnOnly {\n\t\t\t\t\t// Can't use non-effective assertion, but warn-only set so allow link to pass\n\t\t\t\t\ts.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj)\n\t\t\t\t\treturn _EMPTY_, true\n\t\t\t\t}\n\t\t\t\treturn certidp.MsgOCSPResponseNotEffective, false\n\t\t\t}\n\t\t} else {\n\t\t\ts.Errorf(certidp.ErrCAResponseParseFailed, subj, err)\n\t\t\tif opts.WarnOnly {\n\t\t\t\t// Can't use bogus assertion, but warn-only set so allow link to pass\n\t\t\t\ts.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj)\n\t\t\t\treturn _EMPTY_, true\n\t\t\t}\n\t\t\treturn certidp.MsgFailedOCSPResponseParse, false\n\t\t}\n\t\t// cache the valid fetched CA OCSP Response\n\t\trc.Put(fingerprint, ocspr, subj, sLogs)\n\t}\n\n\t// Whether through valid cache response available or newly fetched valid response, now check the status\n\tif ocspr.Status == ocsp.Revoked || (ocspr.Status == ocsp.Unknown && !opts.UnknownIsGood) {\n\t\ts.Warnf(certidp.ErrOCSPInvalidPeerLink, subj, certidp.GetStatusAssertionStr(ocspr.Status))\n\t\tif opts.WarnOnly {\n\t\t\ts.Warnf(certidp.MsgAllowWarnOnlyOccurred, subj)\n\t\t\treturn _EMPTY_, true\n\t\t}\n\t\treturn fmt.Sprintf(certidp.MsgOCSPResponseInvalidStatus, certidp.GetStatusAssertionStr(ocspr.Status)), false\n\t}\n\ts.Debugf(certidp.DbgOCSPValidPeerLink, subj)\n\treturn _EMPTY_, true\n}\n" }, { "path": "server/ocsp_responsecache.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"golang.org/x/crypto/ocsp\"\n\n\t\"github.com/nats-io/nats-server/v2/server/certidp\"\n)\n\nconst (\n\tOCSPResponseCacheDefaultDir = \"_rc_\"\n\tOCSPResponseCacheDefaultFilename = \"cache.json\"\n\tOCSPResponseCacheDefaultTempFilePrefix = \"ocsprc-*\"\n\tOCSPResponseCacheMinimumSaveInterval = 1 * time.Second\n\tOCSPResponseCacheDefaultSaveInterval = 5 * time.Minute\n)\n\ntype OCSPResponseCacheType int\n\nconst (\n\tNONE OCSPResponseCacheType = iota + 1\n\tLOCAL\n)\n\nvar OCSPResponseCacheTypeMap = map[string]OCSPResponseCacheType{\n\t\"none\": NONE,\n\t\"local\": LOCAL,\n}\n\ntype OCSPResponseCacheConfig struct {\n\tType OCSPResponseCacheType\n\tLocalStore string\n\tPreserveRevoked bool\n\tSaveInterval float64\n}\n\nfunc NewOCSPResponseCacheConfig() *OCSPResponseCacheConfig {\n\treturn &OCSPResponseCacheConfig{\n\t\tType: LOCAL,\n\t\tLocalStore: OCSPResponseCacheDefaultDir,\n\t\tPreserveRevoked: false,\n\t\tSaveInterval: OCSPResponseCacheDefaultSaveInterval.Seconds(),\n\t}\n}\n\ntype OCSPResponseCacheStats struct {\n\tResponses int64 `json:\"size\"`\n\tHits int64 `json:\"hits\"`\n\tMisses int64 `json:\"misses\"`\n\tRevokes int64 `json:\"revokes\"`\n\tGoods int64 `json:\"goods\"`\n\tUnknowns int64 `json:\"unknowns\"`\n}\n\ntype OCSPResponseCacheItem struct {\n\tSubject string `json:\"subject,omitempty\"`\n\tCachedAt time.Time `json:\"cached_at\"`\n\tRespStatus certidp.StatusAssertion `json:\"resp_status\"`\n\tRespExpires time.Time `json:\"resp_expires,omitempty\"`\n\tResp []byte `json:\"resp\"`\n}\n\ntype OCSPResponseCache interface {\n\tPut(key string, resp *ocsp.Response, subj string, log *certidp.Log)\n\tGet(key string, log *certidp.Log) []byte\n\tDelete(key string, miss bool, log *certidp.Log)\n\tType() string\n\tStart(s *Server)\n\tStop(s *Server)\n\tOnline() bool\n\tConfig() *OCSPResponseCacheConfig\n\tStats() *OCSPResponseCacheStats\n}\n\n// NoOpCache is a no-op implementation of OCSPResponseCache\ntype NoOpCache struct {\n\tconfig *OCSPResponseCacheConfig\n\tstats *OCSPResponseCacheStats\n\tonline bool\n\tmu *sync.RWMutex\n}\n\nfunc (c *NoOpCache) Put(_ string, _ *ocsp.Response, _ string, _ *certidp.Log) {}\n\nfunc (c *NoOpCache) Get(_ string, _ *certidp.Log) []byte {\n\treturn nil\n}\n\nfunc (c *NoOpCache) Delete(_ string, _ bool, _ *certidp.Log) {}\n\nfunc (c *NoOpCache) Start(_ *Server) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.stats = &OCSPResponseCacheStats{}\n\tc.online = true\n}\n\nfunc (c *NoOpCache) Stop(_ *Server) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.online = false\n}\n\nfunc (c *NoOpCache) Online() bool {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.online\n}\n\nfunc (c *NoOpCache) Type() string {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn \"none\"\n}\n\nfunc (c *NoOpCache) Config() *OCSPResponseCacheConfig {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.config\n}\n\nfunc (c *NoOpCache) Stats() *OCSPResponseCacheStats {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.stats\n}\n\n// LocalCache is a local file implementation of OCSPResponseCache\ntype LocalCache struct {\n\tconfig *OCSPResponseCacheConfig\n\tstats *OCSPResponseCacheStats\n\tonline bool\n\tcache map[string]OCSPResponseCacheItem\n\tmu *sync.RWMutex\n\tsaveInterval time.Duration\n\tdirty bool\n\ttimer *time.Timer\n}\n\n// Put captures a CA OCSP response to the OCSP peer cache indexed by response fingerprint (a hash)\nfunc (c *LocalCache) Put(key string, caResp *ocsp.Response, subj string, log *certidp.Log) {\n\tc.mu.RLock()\n\tif !c.online || caResp == nil || key == \"\" {\n\t\tc.mu.RUnlock()\n\t\treturn\n\t}\n\tc.mu.RUnlock()\n\tlog.Debugf(certidp.DbgCachingResponse, subj, key)\n\trawC, err := c.Compress(caResp.Raw)\n\tif err != nil {\n\t\tlog.Errorf(certidp.ErrResponseCompressFail, key, err)\n\t\treturn\n\t}\n\tlog.Debugf(certidp.DbgAchievedCompression, float64(len(rawC))/float64(len(caResp.Raw)))\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\t// check if we are replacing and do stats\n\titem, ok := c.cache[key]\n\tif ok {\n\t\tc.adjustStats(-1, item.RespStatus)\n\t}\n\titem = OCSPResponseCacheItem{\n\t\tSubject: subj,\n\t\tCachedAt: time.Now().UTC().Round(time.Second),\n\t\tRespStatus: certidp.StatusAssertionIntToVal[caResp.Status],\n\t\tRespExpires: caResp.NextUpdate,\n\t\tResp: rawC,\n\t}\n\tc.cache[key] = item\n\tc.adjustStats(1, item.RespStatus)\n\tc.dirty = true\n}\n\n// Get returns a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash)\nfunc (c *LocalCache) Get(key string, log *certidp.Log) []byte {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\tif !c.online || key == \"\" {\n\t\treturn nil\n\t}\n\tval, ok := c.cache[key]\n\tif ok {\n\t\tatomic.AddInt64(&c.stats.Hits, 1)\n\t\tlog.Debugf(certidp.DbgCacheHit, key)\n\t} else {\n\t\tatomic.AddInt64(&c.stats.Misses, 1)\n\t\tlog.Debugf(certidp.DbgCacheMiss, key)\n\t\treturn nil\n\t}\n\tresp, err := c.Decompress(val.Resp)\n\tif err != nil {\n\t\tlog.Errorf(certidp.ErrResponseDecompressFail, key, err)\n\t\treturn nil\n\t}\n\treturn resp\n}\n\nfunc (c *LocalCache) adjustStatsHitToMiss() {\n\tatomic.AddInt64(&c.stats.Misses, 1)\n\tatomic.AddInt64(&c.stats.Hits, -1)\n}\n\nfunc (c *LocalCache) adjustStats(delta int64, rs certidp.StatusAssertion) {\n\tif delta == 0 {\n\t\treturn\n\t}\n\tatomic.AddInt64(&c.stats.Responses, delta)\n\tswitch rs {\n\tcase ocsp.Good:\n\t\tatomic.AddInt64(&c.stats.Goods, delta)\n\tcase ocsp.Revoked:\n\t\tatomic.AddInt64(&c.stats.Revokes, delta)\n\tcase ocsp.Unknown:\n\t\tatomic.AddInt64(&c.stats.Unknowns, delta)\n\t}\n}\n\n// Delete removes a CA OCSP response from the OCSP peer cache matching the response fingerprint (a hash)\nfunc (c *LocalCache) Delete(key string, wasMiss bool, log *certidp.Log) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif !c.online || key == \"\" || c.config == nil {\n\t\treturn\n\t}\n\titem, ok := c.cache[key]\n\tif !ok {\n\t\treturn\n\t}\n\tif item.RespStatus == ocsp.Revoked && c.config.PreserveRevoked {\n\t\tlog.Debugf(certidp.DbgPreservedRevocation, key)\n\t\tif wasMiss {\n\t\t\tc.adjustStatsHitToMiss()\n\t\t}\n\t\treturn\n\t}\n\tlog.Debugf(certidp.DbgDeletingCacheResponse, key)\n\tdelete(c.cache, key)\n\tc.adjustStats(-1, item.RespStatus)\n\tif wasMiss {\n\t\tc.adjustStatsHitToMiss()\n\t}\n\tc.dirty = true\n}\n\n// Start initializes the configured OCSP peer cache, loads a saved cache from disk (if present), and initializes runtime statistics\nfunc (c *LocalCache) Start(s *Server) {\n\ts.Debugf(certidp.DbgStartingCache)\n\tc.loadCache(s)\n\tc.initStats()\n\tc.mu.Lock()\n\tc.online = true\n\tc.mu.Unlock()\n}\n\nfunc (c *LocalCache) Stop(s *Server) {\n\tc.mu.Lock()\n\ts.Debugf(certidp.DbgStoppingCache)\n\tc.online = false\n\tc.timer.Stop()\n\tc.mu.Unlock()\n\tc.saveCache(s)\n}\n\nfunc (c *LocalCache) Online() bool {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.online\n}\n\nfunc (c *LocalCache) Type() string {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn \"local\"\n}\n\nfunc (c *LocalCache) Config() *OCSPResponseCacheConfig {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.config\n}\n\nfunc (c *LocalCache) Stats() *OCSPResponseCacheStats {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\tif c.stats == nil {\n\t\treturn nil\n\t}\n\tstats := OCSPResponseCacheStats{\n\t\tResponses: c.stats.Responses,\n\t\tHits: c.stats.Hits,\n\t\tMisses: c.stats.Misses,\n\t\tRevokes: c.stats.Revokes,\n\t\tGoods: c.stats.Goods,\n\t\tUnknowns: c.stats.Unknowns,\n\t}\n\treturn &stats\n}\n\nfunc (c *LocalCache) initStats() {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.stats = &OCSPResponseCacheStats{}\n\tc.stats.Hits = 0\n\tc.stats.Misses = 0\n\tc.stats.Responses = int64(len(c.cache))\n\tfor _, resp := range c.cache {\n\t\tswitch resp.RespStatus {\n\t\tcase ocsp.Good:\n\t\t\tc.stats.Goods++\n\t\tcase ocsp.Revoked:\n\t\t\tc.stats.Revokes++\n\t\tcase ocsp.Unknown:\n\t\t\tc.stats.Unknowns++\n\t\t}\n\t}\n}\n\nfunc (c *LocalCache) Compress(buf []byte) ([]byte, error) {\n\tbodyLen := int64(len(buf))\n\tvar output bytes.Buffer\n\twriter := s2.NewWriter(&output)\n\tinput := bytes.NewReader(buf[:bodyLen])\n\tif n, err := io.CopyN(writer, input, bodyLen); err != nil {\n\t\treturn nil, fmt.Errorf(certidp.ErrCannotWriteCompressed, err)\n\t} else if n != bodyLen {\n\t\treturn nil, fmt.Errorf(certidp.ErrTruncatedWrite, n, bodyLen)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\treturn nil, fmt.Errorf(certidp.ErrCannotCloseWriter, err)\n\t}\n\treturn output.Bytes(), nil\n}\n\nfunc (c *LocalCache) Decompress(buf []byte) ([]byte, error) {\n\tbodyLen := int64(len(buf))\n\tinput := bytes.NewReader(buf[:bodyLen])\n\treader := io.NopCloser(s2.NewReader(input))\n\toutput, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(certidp.ErrCannotReadCompressed, err)\n\t}\n\treturn output, reader.Close()\n}\n\nfunc (c *LocalCache) loadCache(s *Server) {\n\td := s.opts.OCSPCacheConfig.LocalStore\n\tif d == _EMPTY_ {\n\t\td = OCSPResponseCacheDefaultDir\n\t}\n\tf := OCSPResponseCacheDefaultFilename\n\tstore, err := filepath.Abs(path.Join(d, f))\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrLoadCacheFail, err)\n\t\treturn\n\t}\n\ts.Debugf(certidp.DbgLoadingCache, store)\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.cache = make(map[string]OCSPResponseCacheItem)\n\tdat, err := os.ReadFile(store)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\ts.Debugf(certidp.DbgNoCacheFound)\n\t\t} else {\n\t\t\ts.Warnf(certidp.ErrLoadCacheFail, err)\n\t\t}\n\t\treturn\n\t}\n\terr = json.Unmarshal(dat, &c.cache)\n\tif err != nil {\n\t\t// make sure clean cache\n\t\tc.cache = make(map[string]OCSPResponseCacheItem)\n\t\ts.Warnf(certidp.ErrLoadCacheFail, err)\n\t\tc.dirty = true\n\t\treturn\n\t}\n\tc.dirty = false\n}\n\nfunc (c *LocalCache) saveCache(s *Server) {\n\tc.mu.RLock()\n\tdirty := c.dirty\n\tc.mu.RUnlock()\n\tif !dirty {\n\t\treturn\n\t}\n\ts.Debugf(certidp.DbgCacheDirtySave)\n\tvar d string\n\tif c.config.LocalStore != _EMPTY_ {\n\t\td = c.config.LocalStore\n\t} else {\n\t\td = OCSPResponseCacheDefaultDir\n\t}\n\tf := OCSPResponseCacheDefaultFilename\n\tstore, err := filepath.Abs(path.Join(d, f))\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\ts.Debugf(certidp.DbgSavingCache, store)\n\tif _, err := os.Stat(d); os.IsNotExist(err) {\n\t\terr = os.Mkdir(d, defaultDirPerms)\n\t\tif err != nil {\n\t\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\t\treturn\n\t\t}\n\t}\n\ttmp, err := os.CreateTemp(d, OCSPResponseCacheDefaultTempFilePrefix)\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\tdefer func() {\n\t\ttmp.Close()\n\t\tos.Remove(tmp.Name())\n\t}() // clean up any temp files\n\n\t// RW lock here because we're going to snapshot the cache to disk and mark as clean if successful\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tdat, err := json.MarshalIndent(c.cache, \"\", \" \")\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\tcacheSize, err := tmp.Write(dat)\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\terr = tmp.Sync()\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\terr = tmp.Close()\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\t// do the final swap and overwrite any old saved peer cache\n\terr = os.Rename(tmp.Name(), store)\n\tif err != nil {\n\t\ts.Errorf(certidp.ErrSaveCacheFail, err)\n\t\treturn\n\t}\n\tc.dirty = false\n\ts.Debugf(certidp.DbgCacheSaved, cacheSize)\n}\n\nvar OCSPResponseCacheUsage = `\nYou may enable OCSP peer response cacheing at server configuration root level:\n\n(If no TLS blocks are configured with OCSP peer verification, ocsp_cache is ignored.)\n\n ...\n # short form enables with defaults\n ocsp_cache: true\n\n # if false or undefined and one or more TLS blocks are configured with OCSP peer verification, \"none\" is implied\n\n # long form includes settable options\n ocsp_cache {\n\n # Cache type (default local)\n type: local\n\n # Cache file directory for local-type cache (default _rc_ in current working directory)\n local_store: \"_rc_\"\n\n # Ignore cache deletes if cached OCSP response is Revoked status (default false)\n preserve_revoked: false\n\n # For local store, interval to save in-memory cache to disk in seconds (default 300 seconds, minimum 1 second)\n save_interval: 300\n }\n ...\n\nNote: Cache of server's own OCSP response (staple) is enabled using the 'ocsp' configuration option.\n`\n\nfunc (s *Server) initOCSPResponseCache() {\n\t// No mTLS OCSP or Leaf OCSP enablements, so no need to init cache\n\ts.mu.RLock()\n\tif !s.ocspPeerVerify {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\ts.mu.RUnlock()\n\tso := s.getOpts()\n\tif so.OCSPCacheConfig == nil {\n\t\tso.OCSPCacheConfig = NewOCSPResponseCacheConfig()\n\t}\n\tvar cc = so.OCSPCacheConfig\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tswitch cc.Type {\n\tcase NONE:\n\t\ts.ocsprc = &NoOpCache{config: cc, online: true, mu: &sync.RWMutex{}}\n\tcase LOCAL:\n\t\tc := &LocalCache{\n\t\t\tconfig: cc,\n\t\t\tonline: false,\n\t\t\tcache: make(map[string]OCSPResponseCacheItem),\n\t\t\tmu: &sync.RWMutex{},\n\t\t\tdirty: false,\n\t\t}\n\t\tc.saveInterval = time.Duration(cc.SaveInterval) * time.Second\n\t\tc.timer = time.AfterFunc(c.saveInterval, func() {\n\t\t\ts.Debugf(certidp.DbgCacheSaveTimerExpired)\n\t\t\tc.saveCache(s)\n\t\t\tc.timer.Reset(c.saveInterval)\n\t\t})\n\t\ts.ocsprc = c\n\tdefault:\n\t\ts.Fatalf(certidp.ErrBadCacheTypeConfig, cc.Type)\n\t}\n}\n\nfunc (s *Server) startOCSPResponseCache() {\n\t// No mTLS OCSP or Leaf OCSP enablements, so no need to start cache\n\ts.mu.RLock()\n\tif !s.ocspPeerVerify || s.ocsprc == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\ts.mu.RUnlock()\n\n\t// Could be heavier operation depending on cache implementation\n\ts.ocsprc.Start(s)\n\tif s.ocsprc.Online() {\n\t\ts.Noticef(certidp.MsgCacheOnline, s.ocsprc.Type())\n\t} else {\n\t\ts.Noticef(certidp.MsgCacheOffline, s.ocsprc.Type())\n\t}\n}\n\nfunc (s *Server) stopOCSPResponseCache() {\n\ts.mu.RLock()\n\tif s.ocsprc == nil {\n\t\ts.mu.RUnlock()\n\t\treturn\n\t}\n\ts.mu.RUnlock()\n\ts.ocsprc.Stop(s)\n}\n\nfunc parseOCSPResponseCache(v any) (pcfg *OCSPResponseCacheConfig, retError error) {\n\tvar lt token\n\tdefer convertPanicToError(<, &retError)\n\ttk, v := unwrapValue(v, <)\n\tcm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrIllegalCacheOptsConfig, v)}\n\t}\n\tpcfg = NewOCSPResponseCacheConfig()\n\tretError = nil\n\tfor mk, mv := range cm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"type\":\n\t\t\tcache, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tcacheType, exists := OCSPResponseCacheTypeMap[strings.ToLower(cache)]\n\t\t\tif !exists {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrUnknownCacheType, cache)}\n\t\t\t}\n\t\t\tpcfg.Type = cacheType\n\t\tcase \"local_store\":\n\t\t\tstore, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.LocalStore = store\n\t\tcase \"preserve_revoked\":\n\t\t\tpreserve, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)}\n\t\t\t}\n\t\t\tpcfg.PreserveRevoked = preserve\n\t\tcase \"save_interval\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingPeerOptFieldTypeConversion, err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldTypeConversion, \"unexpected type\")}\n\t\t\t}\n\t\t\tsi := time.Duration(at) * time.Second\n\t\t\tif si < OCSPResponseCacheMinimumSaveInterval {\n\t\t\t\tsi = OCSPResponseCacheMinimumSaveInterval\n\t\t\t}\n\t\t\tpcfg.SaveInterval = si.Seconds()\n\t\tdefault:\n\t\t\treturn nil, &configErr{tk, fmt.Sprintf(certidp.ErrParsingCacheOptFieldGeneric, mk)}\n\t\t}\n\t}\n\treturn pcfg, nil\n}\n" }, { "path": "server/opts.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"context\"\n\t\"crypto/fips140\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/conf\"\n\t\"github.com/nats-io/nats-server/v2/server/certidp\"\n\t\"github.com/nats-io/nats-server/v2/server/certstore\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nvar allowUnknownTopLevelField = int32(0)\n\n// NoErrOnUnknownFields can be used to change the behavior the processing\n// of a configuration file. By default, an error is reported if unknown\n// fields are found. If `noError` is set to true, no error will be reported\n// if top-level unknown fields are found.\nfunc NoErrOnUnknownFields(noError bool) {\n\tvar val int32\n\tif noError {\n\t\tval = int32(1)\n\t}\n\tatomic.StoreInt32(&allowUnknownTopLevelField, val)\n}\n\n// PinnedCertSet is a set of lower case hex-encoded sha256 of DER encoded SubjectPublicKeyInfo\ntype PinnedCertSet map[string]struct{}\n\n// ClusterOpts are options for clusters.\n// NOTE: This structure is no longer used for monitoring endpoints\n// and json tags are deprecated and may be removed in the future.\ntype ClusterOpts struct {\n\tName string `json:\"-\"`\n\tHost string `json:\"addr,omitempty\"`\n\tPort int `json:\"cluster_port,omitempty\"`\n\tUsername string `json:\"-\"`\n\tPassword string `json:\"-\"`\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"`\n\tPermissions *RoutePermissions `json:\"-\"`\n\tTLSTimeout float64 `json:\"-\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSMap bool `json:\"-\"`\n\tTLSCheckKnownURLs bool `json:\"-\"`\n\tTLSPinnedCerts PinnedCertSet `json:\"-\"`\n\tListenStr string `json:\"-\"`\n\tAdvertise string `json:\"-\"`\n\tNoAdvertise bool `json:\"-\"`\n\tConnectRetries int `json:\"-\"`\n\tConnectBackoff bool `json:\"-\"`\n\tPoolSize int `json:\"-\"`\n\tPinnedAccounts []string `json:\"-\"`\n\tCompression CompressionOpts `json:\"-\"`\n\tPingInterval time.Duration `json:\"-\"`\n\tMaxPingsOut int `json:\"-\"`\n\tWriteDeadline time.Duration `json:\"-\"`\n\tWriteTimeout WriteTimeoutPolicy `json:\"-\"`\n\n\t// Not exported (used in tests)\n\tresolver netResolver\n\t// Snapshot of configured TLS options.\n\ttlsConfigOpts *TLSConfigOpts\n}\n\n// CompressionOpts defines the compression mode and optional configuration.\ntype CompressionOpts struct {\n\tMode string\n\t// If `Mode` is set to CompressionS2Auto, RTTThresholds provides the\n\t// thresholds at which the compression level will go from\n\t// CompressionS2Uncompressed to CompressionS2Fast, CompressionS2Better\n\t// or CompressionS2Best. If a given level is not desired, specify 0\n\t// for this slot. For instance, the slice []{0, 10ms, 20ms} means that\n\t// for any RTT up to 10ms included the compression level will be\n\t// CompressionS2Fast, then from ]10ms..20ms], the level will be selected\n\t// as CompressionS2Better. Anything above 20ms will result in picking\n\t// the CompressionS2Best compression level.\n\tRTTThresholds []time.Duration\n}\n\n// GatewayOpts are options for gateways.\n// NOTE: This structure is no longer used for monitoring endpoints\n// and json tags are deprecated and may be removed in the future.\ntype GatewayOpts struct {\n\tName string `json:\"name\"`\n\tHost string `json:\"addr,omitempty\"`\n\tPort int `json:\"port,omitempty\"`\n\tUsername string `json:\"-\"`\n\tPassword string `json:\"-\"`\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"`\n\tTLSMap bool `json:\"-\"`\n\tTLSCheckKnownURLs bool `json:\"-\"`\n\tTLSPinnedCerts PinnedCertSet `json:\"-\"`\n\tAdvertise string `json:\"advertise,omitempty\"`\n\tConnectRetries int `json:\"connect_retries,omitempty\"`\n\tConnectBackoff bool `json:\"connect_backoff,omitempty\"`\n\tGateways []*RemoteGatewayOpts `json:\"gateways,omitempty\"`\n\tRejectUnknown bool `json:\"reject_unknown,omitempty\"` // config got renamed to reject_unknown_cluster\n\tWriteDeadline time.Duration `json:\"-\"`\n\tWriteTimeout WriteTimeoutPolicy `json:\"-\"`\n\n\t// Not exported, for tests.\n\tresolver netResolver\n\tsendQSubsBufSize int\n\n\t// Snapshot of configured TLS options.\n\ttlsConfigOpts *TLSConfigOpts\n}\n\n// RemoteGatewayOpts are options for connecting to a remote gateway\n// NOTE: This structure is no longer used for monitoring endpoints\n// and json tags are deprecated and may be removed in the future.\ntype RemoteGatewayOpts struct {\n\tName string `json:\"name\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"`\n\tURLs []*url.URL `json:\"urls,omitempty\"`\n\ttlsConfigOpts *TLSConfigOpts\n}\n\n// LeafNodeOpts are options for a given server to accept leaf node connections and/or connect to a remote cluster.\ntype LeafNodeOpts struct {\n\tHost string `json:\"addr,omitempty\"`\n\tPort int `json:\"port,omitempty\"`\n\tUsername string `json:\"-\"`\n\tPassword string `json:\"-\"`\n\tProxyRequired bool `json:\"-\"`\n\tNkey string `json:\"-\"`\n\tAccount string `json:\"-\"`\n\tUsers []*User `json:\"-\"`\n\tAuthTimeout float64 `json:\"auth_timeout,omitempty\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"`\n\tTLSMap bool `json:\"-\"`\n\tTLSPinnedCerts PinnedCertSet `json:\"-\"`\n\t// When set to true, the server will perform the TLS handshake before\n\t// sending the INFO protocol. For remote leafnodes that are not configured\n\t// with a similar option, their connection will fail with some sort\n\t// of timeout or EOF error since they are expecting to receive an\n\t// INFO protocol first.\n\tTLSHandshakeFirst bool `json:\"-\"`\n\t// If TLSHandshakeFirst is true and this value is strictly positive,\n\t// the server will wait for that amount of time for the TLS handshake\n\t// to start before falling back to previous behavior of sending the\n\t// INFO protocol first. It allows for a mix of newer remote leafnodes\n\t// that can require a TLS handshake first, and older that can't.\n\tTLSHandshakeFirstFallback time.Duration `json:\"-\"`\n\tAdvertise string `json:\"-\"`\n\tNoAdvertise bool `json:\"-\"`\n\tReconnectInterval time.Duration `json:\"-\"`\n\tWriteDeadline time.Duration `json:\"-\"`\n\tWriteTimeout WriteTimeoutPolicy `json:\"-\"`\n\n\t// Compression options\n\tCompression CompressionOpts `json:\"-\"`\n\n\t// For solicited connections to other clusters/superclusters.\n\tRemotes []*RemoteLeafOpts `json:\"remotes,omitempty\"`\n\n\t// This is the minimum version that is accepted for remote connections.\n\t// Note that since the server version in the CONNECT protocol was added\n\t// only starting at v2.8.0, any version below that will be rejected\n\t// (since empty version string in CONNECT would fail the \"version at\n\t// least\" test).\n\tMinVersion string\n\n\t// Isolate subject interest from other leafnode connections, preventing\n\t// east-west propagation.\n\tIsolateLeafnodeInterest bool `json:\"-\"`\n\n\t// Not exported, for tests.\n\tresolver netResolver\n\tdialTimeout time.Duration\n\tconnDelay time.Duration\n\n\t// Snapshot of configured TLS options.\n\ttlsConfigOpts *TLSConfigOpts\n}\n\n// SignatureHandler is used to sign a nonce from the server while\n// authenticating with Nkeys. The callback should sign the nonce and\n// return the JWT and the raw signature.\ntype SignatureHandler func([]byte) (string, []byte, error)\n\n// RemoteLeafOpts are options for connecting to a remote server as a leaf node.\ntype RemoteLeafOpts struct {\n\tLocalAccount string `json:\"local_account,omitempty\"`\n\tNoRandomize bool `json:\"-\"`\n\tURLs []*url.URL `json:\"urls,omitempty\"`\n\tCredentials string `json:\"-\"`\n\tNkey string `json:\"-\"`\n\tSignatureCB SignatureHandler `json:\"-\"`\n\tTLS bool `json:\"-\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSTimeout float64 `json:\"tls_timeout,omitempty\"`\n\tTLSHandshakeFirst bool `json:\"-\"`\n\tHub bool `json:\"hub,omitempty\"`\n\tDenyImports []string `json:\"-\"`\n\tDenyExports []string `json:\"-\"`\n\n\t// FirstInfoTimeout is the amount of time the server will wait for the\n\t// initial INFO protocol from the remote server before closing the\n\t// connection.\n\tFirstInfoTimeout time.Duration `json:\"-\"`\n\n\t// Compression options for this remote. Each remote could have a different\n\t// setting and also be different from the LeafNode options.\n\tCompression CompressionOpts `json:\"-\"`\n\n\t// When an URL has the \"ws\" (or \"wss\") scheme, then the server will initiate the\n\t// connection as a websocket connection. By default, the websocket frames will be\n\t// masked (as if this server was a websocket client to the remote server). The\n\t// NoMasking option will change this behavior and will send umasked frames.\n\tWebsocket struct {\n\t\tCompression bool `json:\"-\"`\n\t\tNoMasking bool `json:\"-\"`\n\t}\n\n\t// HTTP Proxy configuration for WebSocket connections\n\tProxy struct {\n\t\t// URL of the HTTP proxy server (e.g., \"http://proxy.example.com:8080\")\n\t\tURL string `json:\"-\"`\n\t\t// Username for proxy authentication\n\t\tUsername string `json:\"-\"`\n\t\t// Password for proxy authentication\n\t\tPassword string `json:\"-\"`\n\t\t// Timeout for proxy connection\n\t\tTimeout time.Duration `json:\"-\"`\n\t}\n\n\ttlsConfigOpts *TLSConfigOpts\n\n\t// If we are clustered and our local account has JetStream, if apps are accessing\n\t// a stream or consumer leader through this LN and it gets dropped, the apps will\n\t// not be able to work. This tells the system to migrate the leaders away from this server.\n\t// This only changes leader for R>1 assets.\n\tJetStreamClusterMigrate bool `json:\"jetstream_cluster_migrate,omitempty\"`\n\n\t// If JetStreamClusterMigrate is set to true, this is the time after which the leader\n\t// will be migrated away from this server if still disconnected.\n\tJetStreamClusterMigrateDelay time.Duration `json:\"jetstream_cluster_migrate_delay,omitempty\"`\n\n\t// LocalIsolation isolates this remote from east-west subject interest originating locally.\n\tLocalIsolation bool `json:\"local_isolation,omitempty\"`\n\n\t// RequestIsolation asks the remote side to isolate us from their east-west subject interest.\n\tRequestIsolation bool `json:\"request_isolation,omitempty\"`\n\n\t// If this is set to true, the connection to this remote will not be solicited.\n\t// During a configuration reload, if this is changed from `false` to `true`, the\n\t// existing connection will be closed and not solicited again (until it is changed\n\t// to `false` again.\n\tDisabled bool `json:\"-\"`\n}\n\n// JSLimitOpts are active limits for the meta cluster\ntype JSLimitOpts struct {\n\tMaxRequestBatch int `json:\"max_request_batch,omitempty\"` // MaxRequestBatch is the maximum amount of updates that can be sent in a batch\n\tMaxAckPending int `json:\"max_ack_pending,omitempty\"` // MaxAckPending is the server limit for maximum amount of outstanding Acks\n\tMaxHAAssets int `json:\"max_ha_assets,omitempty\"` // MaxHAAssets is the maximum of Streams and Consumers that may have more than 1 replica\n\tDuplicates time.Duration `json:\"max_duplicate_window,omitempty\"` // Duplicates is the maximum value for duplicate tracking on Streams\n\tMaxBatchInflightPerStream int `json:\"max_batch_inflight_per_stream,omitempty\"` // MaxBatchInflightPerStream is the maximum amount of open batches per stream\n\tMaxBatchInflightTotal int `json:\"max_batch_inflight_total,omitempty\"` // MaxBatchInflightTotal is the maximum amount of total open batches per server\n\tMaxBatchSize int `json:\"max_batch_size,omitempty\"` // MaxBatchSize is the maximum amount of messages allowed in a batch publish to a Stream\n\tMaxBatchTimeout time.Duration `json:\"max_batch_timeout,omitempty\"` // MaxBatchTimeout is the maximum time to receive the commit message after receiving the first message of a batch\n}\n\ntype JSTpmOpts struct {\n\tKeysFile string\n\tKeyPassword string\n\tSrkPassword string\n\tPcr int\n}\n\n// AuthCallout option used to map external AuthN to NATS based AuthZ.\ntype AuthCallout struct {\n\t// Must be a public account Nkey.\n\tIssuer string\n\t// Account to be used for sending requests.\n\tAccount string\n\t// Users that will bypass auth_callout and be used for the auth service itself.\n\tAuthUsers []string\n\t// XKey is a public xkey for the authorization service.\n\t// This will enable encryption for server requests and the authorization service responses.\n\tXKey string\n\t// AllowedAccounts that will be delegated to the auth service.\n\t// If empty then all accounts will be delegated.\n\tAllowedAccounts []string\n}\n\n// Options block for nats-server.\n// NOTE: This structure is no longer used for monitoring endpoints\n// and json tags are deprecated and may be removed in the future.\ntype Options struct {\n\tConfigFile string `json:\"-\"`\n\tServerName string `json:\"server_name\"`\n\tHost string `json:\"addr\"`\n\tPort int `json:\"port\"`\n\tDontListen bool `json:\"dont_listen\"`\n\tClientAdvertise string `json:\"-\"`\n\tTrace bool `json:\"-\"`\n\tDebug bool `json:\"-\"`\n\tTraceVerbose bool `json:\"-\"`\n\n\t// TraceHeaders if true will only trace message headers, not the payload.\n\tTraceHeaders bool `json:\"-\"`\n\tNoLog bool `json:\"-\"`\n\tNoSigs bool `json:\"-\"`\n\tNoSublistCache bool `json:\"-\"`\n\tNoHeaderSupport bool `json:\"-\"`\n\tDisableShortFirstPing bool `json:\"-\"`\n\tLogtime bool `json:\"-\"`\n\tLogtimeUTC bool `json:\"-\"`\n\tMaxConn int `json:\"max_connections\"`\n\tMaxSubs int `json:\"max_subscriptions,omitempty\"`\n\tMaxSubTokens uint8 `json:\"-\"`\n\tNkeys []*NkeyUser `json:\"-\"`\n\tUsers []*User `json:\"-\"`\n\tAccounts []*Account `json:\"-\"`\n\tNoAuthUser string `json:\"-\"`\n\tDefaultSentinel string `json:\"-\"`\n\tSystemAccount string `json:\"-\"`\n\tNoSystemAccount bool `json:\"-\"`\n\tUsername string `json:\"-\"`\n\tPassword string `json:\"-\"`\n\tProxyRequired bool `json:\"-\"`\n\tProxyProtocol bool `json:\"-\"`\n\tAuthorization string `json:\"-\"`\n\tAuthCallout *AuthCallout `json:\"-\"`\n\tPingInterval time.Duration `json:\"ping_interval\"`\n\tMaxPingsOut int `json:\"ping_max\"`\n\tHTTPHost string `json:\"http_host\"`\n\tHTTPPort int `json:\"http_port\"`\n\tHTTPBasePath string `json:\"http_base_path\"`\n\tHTTPSPort int `json:\"https_port\"`\n\tAuthTimeout float64 `json:\"auth_timeout\"`\n\tMaxControlLine int32 `json:\"max_control_line\"`\n\tMaxPayload int32 `json:\"max_payload\"`\n\tMaxPending int64 `json:\"max_pending\"`\n\tNoFastProducerStall bool `json:\"-\"`\n\tCluster ClusterOpts `json:\"cluster,omitempty\"`\n\tGateway GatewayOpts `json:\"gateway,omitempty\"`\n\tLeafNode LeafNodeOpts `json:\"leaf,omitempty\"`\n\tJetStream bool `json:\"jetstream\"`\n\tNoJetStreamStrict bool `json:\"-\"` // Strict by default.\n\tJetStreamMaxMemory int64 `json:\"-\"`\n\tJetStreamMaxStore int64 `json:\"-\"`\n\tJetStreamDomain string `json:\"-\"`\n\tJetStreamExtHint string `json:\"-\"`\n\tJetStreamKey string `json:\"-\"`\n\tJetStreamOldKey string `json:\"-\"`\n\tJetStreamCipher StoreCipher `json:\"-\"`\n\tJetStreamUniqueTag string\n\tJetStreamLimits JSLimitOpts\n\tJetStreamTpm JSTpmOpts\n\tJetStreamMaxCatchup int64\n\tJetStreamRequestQueueLimit int64\n\tJetStreamInfoQueueLimit int64\n\tJetStreamMetaCompact uint64\n\tJetStreamMetaCompactSize uint64\n\tJetStreamMetaCompactSync bool\n\tStreamMaxBufferedMsgs int `json:\"-\"`\n\tStreamMaxBufferedSize int64 `json:\"-\"`\n\tStoreDir string `json:\"-\"`\n\tSyncInterval time.Duration `json:\"-\"`\n\tSyncAlways bool `json:\"-\"`\n\tJsAccDefaultDomain map[string]string `json:\"-\"` // account to domain name mapping\n\tWebsocket WebsocketOpts `json:\"-\"`\n\tMQTT MQTTOpts `json:\"-\"`\n\tProfPort int `json:\"-\"`\n\tProfBlockRate int `json:\"-\"`\n\tPidFile string `json:\"-\"`\n\tPortsFileDir string `json:\"-\"`\n\tLogFile string `json:\"-\"`\n\tLogSizeLimit int64 `json:\"-\"`\n\tLogMaxFiles int64 `json:\"-\"`\n\tSyslog bool `json:\"-\"`\n\tRemoteSyslog string `json:\"-\"`\n\tRoutes []*url.URL `json:\"-\"`\n\tRoutesStr string `json:\"-\"`\n\tTLSTimeout float64 `json:\"tls_timeout\"`\n\tTLS bool `json:\"-\"`\n\tTLSVerify bool `json:\"-\"`\n\tTLSMap bool `json:\"-\"`\n\tTLSCert string `json:\"-\"`\n\tTLSKey string `json:\"-\"`\n\tTLSCaCert string `json:\"-\"`\n\tTLSConfig *tls.Config `json:\"-\"`\n\tTLSPinnedCerts PinnedCertSet `json:\"-\"`\n\tTLSRateLimit int64 `json:\"-\"`\n\t// When set to true, the server will perform the TLS handshake before\n\t// sending the INFO protocol. For clients that are not configured\n\t// with a similar option, their connection will fail with some sort\n\t// of timeout or EOF error since they are expecting to receive an\n\t// INFO protocol first.\n\tTLSHandshakeFirst bool `json:\"-\"`\n\t// If TLSHandshakeFirst is true and this value is strictly positive,\n\t// the server will wait for that amount of time for the TLS handshake\n\t// to start before falling back to previous behavior of sending the\n\t// INFO protocol first. It allows for a mix of newer clients that can\n\t// require a TLS handshake first, and older clients that can't.\n\tTLSHandshakeFirstFallback time.Duration `json:\"-\"`\n\tAllowNonTLS bool `json:\"-\"`\n\tWriteDeadline time.Duration `json:\"-\"`\n\tWriteTimeout WriteTimeoutPolicy `json:\"-\"`\n\tMaxClosedClients int `json:\"-\"`\n\tLameDuckDuration time.Duration `json:\"-\"`\n\tLameDuckGracePeriod time.Duration `json:\"-\"`\n\n\t// MaxTracedMsgLen is the maximum printable length for traced messages.\n\tMaxTracedMsgLen int `json:\"-\"`\n\n\t// Operating a trusted NATS server\n\tTrustedKeys []string `json:\"-\"`\n\tTrustedOperators []*jwt.OperatorClaims `json:\"-\"`\n\tAccountResolver AccountResolver `json:\"-\"`\n\tAccountResolverTLSConfig *tls.Config `json:\"-\"`\n\n\t// AlwaysEnableNonce will always present a nonce to new connections\n\t// typically used by custom Authentication implementations who embeds\n\t// the server and so not presented as a configuration option\n\tAlwaysEnableNonce bool\n\n\tCustomClientAuthentication Authentication `json:\"-\"`\n\tCustomRouterAuthentication Authentication `json:\"-\"`\n\n\t// CheckConfig configuration file syntax test was successful and exit.\n\tCheckConfig bool `json:\"-\"`\n\n\t// DisableJetStreamBanner will not print the ascii art on startup for JetStream enabled servers\n\tDisableJetStreamBanner bool `json:\"-\"`\n\n\t// ConnectErrorReports specifies the number of failed attempts\n\t// at which point server should report the failure of an initial\n\t// connection to a route, gateway or leaf node.\n\t// See DEFAULT_CONNECT_ERROR_REPORTS for default value.\n\tConnectErrorReports int\n\n\t// ReconnectErrorReports is similar to ConnectErrorReports except\n\t// that this applies to reconnect events.\n\tReconnectErrorReports int\n\n\t// Tags describing the server. They will be included in varz\n\t// and used as a filter criteria for some system requests.\n\tTags jwt.TagList `json:\"-\"`\n\n\t// Metadata describing the server. They will be included in 'Z' responses.\n\tMetadata map[string]string `json:\"-\"`\n\n\t// OCSPConfig enables OCSP Stapling in the server.\n\tOCSPConfig *OCSPConfig\n\ttlsConfigOpts *TLSConfigOpts\n\n\t// Proxies configuration.\n\tProxies *ProxiesConfig\n\n\t// private fields, used to know if bool options are explicitly\n\t// defined in config and/or command line params.\n\tinConfig map[string]bool\n\tinCmdLine map[string]bool\n\n\t// private fields for operator mode\n\toperatorJWT []string\n\tresolverPreloads map[string]string\n\tresolverPinnedAccounts map[string]struct{}\n\n\t// private fields, used for testing\n\tgatewaysSolicitDelay time.Duration\n\toverrideProto int\n\n\t// JetStream\n\tmaxMemSet bool\n\tmaxStoreSet bool\n\tsyncSet bool\n\n\t// OCSP Cache config enables next-gen cache for OCSP features\n\tOCSPCacheConfig *OCSPResponseCacheConfig\n\n\t// Used to mark that we had a top level authorization block.\n\tauthBlockDefined bool\n\n\t// configDigest represents the state of configuration.\n\tconfigDigest string\n}\n\n// WebsocketOpts are options for websocket\ntype WebsocketOpts struct {\n\t// The server will accept websocket client connections on this hostname/IP.\n\tHost string\n\t// The server will accept websocket client connections on this port.\n\tPort int\n\t// The host:port to advertise to websocket clients in the cluster.\n\tAdvertise string\n\n\t// If no user name is provided when a client connects, will default to the\n\t// matching user from the global list of users in `Options.Users`.\n\tNoAuthUser string\n\n\t// Name of the cookie, which if present in WebSocket upgrade headers,\n\t// will be treated as JWT during CONNECT phase as long as\n\t// \"jwt\" specified in the CONNECT options is missing or empty.\n\tJWTCookie string\n\n\t// Name of the cookie, which if present in WebSocket upgrade headers,\n\t// will be treated as Username during CONNECT phase as long as\n\t// \"user\" specified in the CONNECT options is missing or empty.\n\tUsernameCookie string\n\n\t// Name of the cookie, which if present in WebSocket upgrade headers,\n\t// will be treated as Password during CONNECT phase as long as\n\t// \"pass\" specified in the CONNECT options is missing or empty.\n\tPasswordCookie string\n\n\t// Name of the cookie, which if present in WebSocket upgrade headers,\n\t// will be treated as Token during CONNECT phase as long as\n\t// \"auth_token\" specified in the CONNECT options is missing or empty.\n\t// Note that when this is useful for passing a JWT to an cuth callout\n\t// when the server uses delegated authentication (\"operator mode\") or\n\t// when using delegated authentication, but the auth callout validates some\n\t// other JWT or string. Note that this does map to an actual server-wide\n\t// \"auth_token\", note that using it for that purpose is greatly discouraged.\n\tTokenCookie string\n\n\t// Authentication section. If anything is configured in this section,\n\t// it will override the authorization configuration of regular clients.\n\tUsername string\n\tPassword string\n\tToken string\n\n\t// Timeout for the authentication process.\n\tAuthTimeout float64\n\n\t// By default the server will enforce the use of TLS. If no TLS configuration\n\t// is provided, you need to explicitly set NoTLS to true to allow the server\n\t// to start without TLS configuration. Note that if a TLS configuration is\n\t// present, this boolean is ignored and the server will run the Websocket\n\t// server with that TLS configuration.\n\t// Running without TLS is less secure since Websocket clients that use bearer\n\t// tokens will send them in clear. So this should not be used in production.\n\tNoTLS bool\n\n\t// TLS configuration is required.\n\tTLSConfig *tls.Config\n\t// If true, map certificate values for authentication purposes.\n\tTLSMap bool\n\n\t// When present, accepted client certificates (verify/verify_and_map) must be in this list\n\tTLSPinnedCerts PinnedCertSet\n\n\t// If true, the Origin header must match the request's host.\n\tSameOrigin bool\n\n\t// Only origins in this list will be accepted. If empty and\n\t// SameOrigin is false, any origin is accepted.\n\tAllowedOrigins []string\n\n\t// If set to true, the server will negotiate with clients\n\t// if compression can be used. If this is false, no compression\n\t// will be used (both in server and clients) since it has to\n\t// be negotiated between both endpoints\n\tCompression bool\n\n\t// Total time allowed for the server to read the client request\n\t// and write the response back to the client. This include the\n\t// time needed for the TLS Handshake.\n\tHandshakeTimeout time.Duration\n\n\t// How often to send pings to WebSocket clients. When set to a non-zero\n\t// duration, this overrides the default PingInterval for WebSocket connections.\n\t// If not set or zero, the server's default PingInterval will be used.\n\tPingInterval time.Duration\n\n\t// Headers to be added to the upgrade response.\n\t// Useful for adding custom headers like Strict-Transport-Security.\n\tHeaders map[string]string\n\n\t// Snapshot of configured TLS options.\n\ttlsConfigOpts *TLSConfigOpts\n}\n\n// MQTTOpts are options for MQTT\ntype MQTTOpts struct {\n\t// The server will accept MQTT client connections on this hostname/IP.\n\tHost string\n\t// The server will accept MQTT client connections on this port.\n\tPort int\n\n\t// If no user name is provided when a client connects, will default to the\n\t// matching user from the global list of users in `Options.Users`.\n\tNoAuthUser string\n\n\t// Authentication section. If anything is configured in this section,\n\t// it will override the authorization configuration of regular clients.\n\tUsername string\n\tPassword string\n\tToken string\n\n\t// JetStream domain mqtt is supposed to pick up\n\tJsDomain string\n\n\t// Number of replicas for MQTT streams.\n\t// Negative or 0 value means that the server(s) will pick a replica\n\t// number based on the known size of the cluster (but capped at 3).\n\t// Note that if an account was already connected, the stream's replica\n\t// count is not modified. Use the NATS CLI to update the count if desired.\n\tStreamReplicas int\n\n\t// Number of replicas for MQTT consumers.\n\t// Negative or 0 value means that there is no override and the consumer\n\t// will have the same replica factor that the stream it belongs to.\n\t// If a value is specified, it will require to be lower than the stream\n\t// replicas count (lower than StreamReplicas if specified, but also lower\n\t// than the automatic value determined by cluster size).\n\t// Note that existing consumers are not modified.\n\t//\n\t// UPDATE: This is no longer used while messages stream has interest policy retention\n\t// which requires consumer replica count to match the parent stream.\n\tConsumerReplicas int\n\n\t// Indicate if the consumers should be created with memory storage.\n\t// Note that existing consumers are not modified.\n\tConsumerMemoryStorage bool\n\n\t// If specified will have the system auto-cleanup the consumers after being\n\t// inactive for the specified amount of time.\n\tConsumerInactiveThreshold time.Duration\n\n\t// Timeout for the authentication process.\n\tAuthTimeout float64\n\n\t// TLS configuration is required.\n\tTLSConfig *tls.Config\n\t// If true, map certificate values for authentication purposes.\n\tTLSMap bool\n\t// Timeout for the TLS handshake\n\tTLSTimeout float64\n\t// Set of allowable certificates\n\tTLSPinnedCerts PinnedCertSet\n\n\t// AckWait is the amount of time after which a QoS 1 or 2 message sent to a\n\t// client is redelivered as a DUPLICATE if the server has not received the\n\t// PUBACK on the original Packet Identifier. The same value applies to\n\t// PubRel redelivery. The value has to be positive. Zero will cause the\n\t// server to use the default value (30 seconds). Note that changes to this\n\t// option is applied only to new MQTT subscriptions (or sessions for\n\t// PubRels).\n\tAckWait time.Duration\n\n\t// JSAPITimeout defines timeout for JetStream api calls (default is 5 seconds)\n\tJSAPITimeout time.Duration\n\n\t// MaxAckPending is the amount of QoS 1 and 2 messages (combined) the server\n\t// can send to a subscription without receiving any PUBACK for those\n\t// messages. The valid range is [0..65535].\n\t//\n\t// The total of subscriptions' MaxAckPending on a given session cannot\n\t// exceed 65535. Attempting to create a subscription that would bring the\n\t// total above the limit would result in the server returning 0x80 in the\n\t// SUBACK for this subscription.\n\t//\n\t// Due to how the NATS Server handles the MQTT \"#\" wildcard, each\n\t// subscription ending with \"#\" will use 2 times the MaxAckPending value.\n\t// Note that changes to this option is applied only to new subscriptions.\n\tMaxAckPending uint16\n\n\t// Snapshot of configured TLS options.\n\ttlsConfigOpts *TLSConfigOpts\n\n\t// rejectQoS2Pub tells the MQTT client to not accept QoS2 PUBLISH, instead\n\t// error and terminate the connection.\n\trejectQoS2Pub bool\n\n\t// downgradeQOS2Sub tells the MQTT client to downgrade QoS2 SUBSCRIBE\n\t// requests to QoS1.\n\tdowngradeQoS2Sub bool\n}\n\ntype netResolver interface {\n\tLookupHost(ctx context.Context, host string) ([]string, error)\n}\n\n// Clone performs a deep copy of the Options struct, returning a new clone\n// with all values copied.\nfunc (o *Options) Clone() *Options {\n\tif o == nil {\n\t\treturn nil\n\t}\n\tclone := &Options{}\n\t*clone = *o\n\tif o.Users != nil {\n\t\tclone.Users = make([]*User, len(o.Users))\n\t\tfor i, user := range o.Users {\n\t\t\tclone.Users[i] = user.clone()\n\t\t}\n\t}\n\tif o.Nkeys != nil {\n\t\tclone.Nkeys = make([]*NkeyUser, len(o.Nkeys))\n\t\tfor i, nkey := range o.Nkeys {\n\t\t\tclone.Nkeys[i] = nkey.clone()\n\t\t}\n\t}\n\n\tif o.Routes != nil {\n\t\tclone.Routes = deepCopyURLs(o.Routes)\n\t}\n\tif o.TLSConfig != nil {\n\t\tclone.TLSConfig = o.TLSConfig.Clone()\n\t}\n\tif o.Cluster.TLSConfig != nil {\n\t\tclone.Cluster.TLSConfig = o.Cluster.TLSConfig.Clone()\n\t}\n\tif o.Gateway.TLSConfig != nil {\n\t\tclone.Gateway.TLSConfig = o.Gateway.TLSConfig.Clone()\n\t}\n\tif len(o.Gateway.Gateways) > 0 {\n\t\tclone.Gateway.Gateways = make([]*RemoteGatewayOpts, len(o.Gateway.Gateways))\n\t\tfor i, g := range o.Gateway.Gateways {\n\t\t\tclone.Gateway.Gateways[i] = g.clone()\n\t\t}\n\t}\n\t// FIXME(dlc) - clone leaf node stuff.\n\treturn clone\n}\n\nfunc deepCopyURLs(urls []*url.URL) []*url.URL {\n\tif urls == nil {\n\t\treturn nil\n\t}\n\tcurls := make([]*url.URL, len(urls))\n\tfor i, u := range urls {\n\t\tcu := &url.URL{}\n\t\t*cu = *u\n\t\tcurls[i] = cu\n\t}\n\treturn curls\n}\n\n// Configuration file authorization section.\ntype authorization struct {\n\t// Singles\n\tuser string\n\tpass string\n\ttoken string\n\tnkey string\n\tacc string\n\t// If connection must come through proxy\n\tproxyRequired bool\n\t// Multiple Nkeys/Users\n\tnkeys []*NkeyUser\n\tusers []*User\n\ttimeout float64\n\tdefaultPermissions *Permissions\n\t// Auth Callouts\n\tcallout *AuthCallout\n}\n\n// TLSConfigOpts holds the parsed tls config information,\n// used with flag parsing\ntype TLSConfigOpts struct {\n\tCertFile string\n\tKeyFile string\n\tCaFile string\n\tVerify bool\n\tInsecure bool\n\tMap bool\n\tTLSCheckKnownURLs bool\n\tHandshakeFirst bool // Indicate that the TLS handshake should occur first, before sending the INFO protocol.\n\tFallbackDelay time.Duration // Where supported, indicates how long to wait for the handshake before falling back to sending the INFO protocol first.\n\tTimeout float64\n\tRateLimit int64\n\tAllowInsecureCiphers bool\n\tCiphers []uint16\n\tCurvePreferences []tls.CurveID\n\tPinnedCerts PinnedCertSet\n\tCertStore certstore.StoreType\n\tCertMatchBy certstore.MatchByType\n\tCertMatch string\n\tCertMatchSkipInvalid bool\n\tCaCertsMatch []string\n\tOCSPPeerConfig *certidp.OCSPPeerConfig\n\tCertificates []*TLSCertPairOpt\n\tMinVersion uint16\n}\n\n// TLSCertPairOpt are the paths to a certificate and private key.\ntype TLSCertPairOpt struct {\n\tCertFile string\n\tKeyFile string\n}\n\n// OCSPConfig represents the options of OCSP stapling options.\ntype OCSPConfig struct {\n\t// Mode defines the policy for OCSP stapling.\n\tMode OCSPMode\n\n\t// OverrideURLs is the http URL endpoint used to get OCSP staples.\n\tOverrideURLs []string\n}\n\n// ProxiesConfig represents the options of Proxies.\ntype ProxiesConfig struct {\n\tTrusted []*ProxyConfig\n}\n\n// ProxyConfig represents the options of Proxy.\ntype ProxyConfig struct {\n\t// Public key.\n\tKey string\n}\n\nvar tlsUsage = `\nTLS configuration is specified in the tls section of a configuration file:\n\ne.g.\n\n tls {\n cert_file: \"./certs/server-cert.pem\"\n key_file: \"./certs/server-key.pem\"\n ca_file: \"./certs/ca.pem\"\n verify: true\n verify_and_map: true\n\n cipher_suites: [\n \"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\",\n \"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"\n ]\n curve_preferences: [\n \"CurveP256\",\n \"CurveP384\",\n \"CurveP521\"\n ]\n }\n\nAvailable cipher suites include:\n`\n\n// ProcessConfigFile processes a configuration file.\n// FIXME(dlc): A bit hacky\nfunc ProcessConfigFile(configFile string) (*Options, error) {\n\topts := &Options{}\n\tif err := opts.ProcessConfigFile(configFile); err != nil {\n\t\t// If only warnings then continue and return the options.\n\t\tif cerr, ok := err.(*processConfigErr); ok && len(cerr.Errors()) == 0 {\n\t\t\treturn opts, nil\n\t\t}\n\n\t\treturn nil, err\n\t}\n\treturn opts, nil\n}\n\n// token is an item parsed from the configuration.\ntype token interface {\n\tValue() any\n\tLine() int\n\tIsUsedVariable() bool\n\tSourceFile() string\n\tPosition() int\n}\n\n// unwrapValue can be used to get the token and value from an item\n// to be able to report the line number in case of an incorrect\n// configuration.\n// also stores the token in lastToken for use in convertPanicToError\nfunc unwrapValue(v any, lastToken *token) (token, any) {\n\tswitch tk := v.(type) {\n\tcase token:\n\t\tif lastToken != nil {\n\t\t\t*lastToken = tk\n\t\t}\n\t\treturn tk, tk.Value()\n\tdefault:\n\t\treturn nil, v\n\t}\n}\n\n// use in defer to recover from panic and turn it into an error associated with last token\nfunc convertPanicToErrorList(lastToken *token, errors *[]error) {\n\t// only recover if an error can be stored\n\tif errors == nil {\n\t\treturn\n\t} else if err := recover(); err == nil {\n\t\treturn\n\t} else if lastToken != nil && *lastToken != nil {\n\t\t*errors = append(*errors, &configErr{*lastToken, fmt.Sprint(err)})\n\t} else {\n\t\t*errors = append(*errors, fmt.Errorf(\"encountered panic without a token %v\", err))\n\t}\n}\n\n// use in defer to recover from panic and turn it into an error associated with last token\nfunc convertPanicToError(lastToken *token, e *error) {\n\t// only recover if an error can be stored\n\tif e == nil || *e != nil {\n\t\treturn\n\t} else if err := recover(); err == nil {\n\t\treturn\n\t} else if lastToken != nil && *lastToken != nil {\n\t\t*e = &configErr{*lastToken, fmt.Sprint(err)}\n\t} else {\n\t\t*e = fmt.Errorf(\"%v\", err)\n\t}\n}\n\n// configureSystemAccount configures a system account\n// if present in the configuration.\nfunc configureSystemAccount(o *Options, m map[string]any) (retErr error) {\n\tvar lt token\n\tdefer convertPanicToError(<, &retErr)\n\tconfigure := func(v any) error {\n\t\ttk, v := unwrapValue(v, <)\n\t\tsa, ok := v.(string)\n\t\tif !ok {\n\t\t\treturn &configErr{tk, \"system account name must be a string\"}\n\t\t}\n\t\to.SystemAccount = sa\n\t\treturn nil\n\t}\n\n\tif v, ok := m[\"system_account\"]; ok {\n\t\treturn configure(v)\n\t} else if v, ok := m[\"system\"]; ok {\n\t\treturn configure(v)\n\t}\n\n\treturn nil\n}\n\n// ProcessConfigFile updates the Options structure with options\n// present in the given configuration file.\n// This version is convenient if one wants to set some default\n// options and then override them with what is in the config file.\n// For instance, this version allows you to do something such as:\n//\n// opts := &Options{Debug: true}\n// opts.ProcessConfigFile(myConfigFile)\n//\n// If the config file contains \"debug: false\", after this call,\n// opts.Debug would really be false. It would be impossible to\n// achieve that with the non receiver ProcessConfigFile() version,\n// since one would not know after the call if \"debug\" was not present\n// or was present but set to false.\nfunc (o *Options) ProcessConfigFile(configFile string) error {\n\to.ConfigFile = configFile\n\tif configFile == _EMPTY_ {\n\t\treturn nil\n\t}\n\tm, digest, err := conf.ParseFileWithChecksDigest(configFile)\n\tif err != nil {\n\t\treturn err\n\t}\n\to.configDigest = digest\n\n\treturn o.processConfigFile(configFile, m)\n}\n\n// ProcessConfigString is the same as ProcessConfigFile, but expects the\n// contents of the config file to be passed in rather than the file name.\nfunc (o *Options) ProcessConfigString(data string) error {\n\tm, err := conf.ParseWithChecks(data)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn o.processConfigFile(_EMPTY_, m)\n}\n\n// ConfigDigest returns the digest representing the configuration.\nfunc (o *Options) ConfigDigest() string {\n\treturn o.configDigest\n}\n\nfunc (o *Options) processConfigFile(configFile string, m map[string]any) error {\n\t// Collect all errors and warnings and report them all together.\n\terrors := make([]error, 0)\n\twarnings := make([]error, 0)\n\tif len(m) == 0 {\n\t\twarnings = append(warnings, fmt.Errorf(\"%s: config has no values or is empty\", configFile))\n\t}\n\n\t// First check whether a system account has been defined,\n\t// as that is a condition for other features to be enabled.\n\tif err := configureSystemAccount(o, m); err != nil {\n\t\terrors = append(errors, err)\n\t}\n\n\tfor k, v := range m {\n\t\to.processConfigFileLine(k, v, &errors, &warnings)\n\t}\n\n\t// Post-process: check auth callout allowed accounts against configured accounts.\n\tif o.AuthCallout != nil {\n\t\taccounts := make(map[string]struct{})\n\t\tfor _, acc := range o.Accounts {\n\t\t\taccounts[acc.Name] = struct{}{}\n\t\t}\n\n\t\tfor _, acc := range o.AuthCallout.AllowedAccounts {\n\t\t\tif _, ok := accounts[acc]; !ok {\n\t\t\t\terr := &configErr{nil, fmt.Sprintf(\"auth_callout allowed account %q not found in configured accounts\", acc)}\n\t\t\t\terrors = append(errors, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(errors) > 0 || len(warnings) > 0 {\n\t\treturn &processConfigErr{\n\t\t\terrors: errors,\n\t\t\twarnings: warnings,\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (o *Options) processConfigFileLine(k string, v any, errors *[]error, warnings *[]error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tswitch strings.ToLower(k) {\n\tcase \"listen\":\n\t\thp, err := parseListen(v)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\treturn\n\t\t}\n\t\to.Host = hp.host\n\t\to.Port = hp.port\n\tcase \"client_advertise\":\n\t\to.ClientAdvertise = v.(string)\n\tcase \"port\":\n\t\to.Port = int(v.(int64))\n\tcase \"server_name\":\n\t\tsn := v.(string)\n\t\tif strings.Contains(sn, \" \") {\n\t\t\terr := &configErr{tk, ErrServerNameHasSpaces.Error()}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.ServerName = sn\n\tcase \"host\", \"net\":\n\t\to.Host = v.(string)\n\tcase \"debug\":\n\t\to.Debug = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"Debug\", o.Debug)\n\tcase \"trace\":\n\t\to.Trace = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"Trace\", o.Trace)\n\tcase \"trace_verbose\":\n\t\to.TraceVerbose = v.(bool)\n\t\to.Trace = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"TraceVerbose\", o.TraceVerbose)\n\t\ttrackExplicitVal(&o.inConfig, \"Trace\", o.Trace)\n\tcase \"trace_headers\":\n\t\to.TraceHeaders = v.(bool)\n\t\to.Trace = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"TraceHeaders\", o.TraceHeaders)\n\t\ttrackExplicitVal(&o.inConfig, \"Trace\", o.Trace)\n\tcase \"logtime\":\n\t\to.Logtime = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"Logtime\", o.Logtime)\n\tcase \"logtime_utc\":\n\t\to.LogtimeUTC = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"LogtimeUTC\", o.LogtimeUTC)\n\tcase \"mappings\", \"maps\":\n\t\tgacc := NewAccount(globalAccountName)\n\t\to.Accounts = append(o.Accounts, gacc)\n\t\terr := parseAccountMappings(tk, gacc, errors)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"disable_sublist_cache\", \"no_sublist_cache\":\n\t\to.NoSublistCache = v.(bool)\n\tcase \"accounts\":\n\t\terr := parseAccounts(tk, o, errors, warnings)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"default_sentinel\":\n\t\to.DefaultSentinel = v.(string)\n\tcase \"authorization\":\n\t\tauth, err := parseAuthorization(tk, errors, warnings)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.authBlockDefined = true\n\t\to.Username = auth.user\n\t\to.Password = auth.pass\n\t\to.ProxyRequired = auth.proxyRequired\n\t\to.Authorization = auth.token\n\t\to.AuthTimeout = auth.timeout\n\t\to.AuthCallout = auth.callout\n\n\t\tif (auth.user != _EMPTY_ || auth.pass != _EMPTY_) && auth.token != _EMPTY_ {\n\t\t\terr := &configErr{tk, \"Cannot have a user/pass and token\"}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\t// In case parseAccounts() was done first, we need to check for duplicates.\n\t\tunames := setupUsersAndNKeysDuplicateCheckMap(o)\n\t\t// Check for multiple users defined.\n\t\t// Note: auth.users will be != nil as long as `users: []` is present\n\t\t// in the authorization block, even if empty, and will also account for\n\t\t// nkey users. We also check for users/nkeys that may have been already\n\t\t// added in parseAccounts() (which means they will be in unames)\n\t\tif auth.users != nil || len(unames) > 0 {\n\t\t\tif auth.user != _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Can not have a single user/pass and a users array\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif auth.token != _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Can not have a token and a users array\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Now check that if we have users, there is no duplicate, including\n\t\t\t// users that may have been configured in parseAccounts().\n\t\t\tif len(auth.users) > 0 {\n\t\t\t\tfor _, u := range auth.users {\n\t\t\t\t\tif _, ok := unames[u.Username]; ok {\n\t\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Duplicate user %q detected\", u.Username)}\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tunames[u.Username] = struct{}{}\n\t\t\t\t}\n\t\t\t\t// Users may have been added from Accounts parsing, so do an append here\n\t\t\t\to.Users = append(o.Users, auth.users...)\n\t\t\t}\n\t\t}\n\t\t// Check for nkeys\n\t\tif len(auth.nkeys) > 0 {\n\t\t\tfor _, u := range auth.nkeys {\n\t\t\t\tif _, ok := unames[u.Nkey]; ok {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Duplicate nkey %q detected\", u.Nkey)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tunames[u.Nkey] = struct{}{}\n\t\t\t}\n\t\t\t// NKeys may have been added from Accounts parsing, so do an append here\n\t\t\to.Nkeys = append(o.Nkeys, auth.nkeys...)\n\t\t}\n\tcase \"http\":\n\t\thp, err := parseListen(v)\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.HTTPHost = hp.host\n\t\to.HTTPPort = hp.port\n\tcase \"https\":\n\t\thp, err := parseListen(v)\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.HTTPHost = hp.host\n\t\to.HTTPSPort = hp.port\n\tcase \"http_port\", \"monitor_port\":\n\t\to.HTTPPort = int(v.(int64))\n\tcase \"https_port\":\n\t\to.HTTPSPort = int(v.(int64))\n\tcase \"http_base_path\":\n\t\to.HTTPBasePath = v.(string)\n\tcase \"cluster\":\n\t\terr := parseCluster(tk, o, errors, warnings)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"gateway\":\n\t\tif err := parseGateway(tk, o, errors, warnings); err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"leaf\", \"leafnodes\":\n\t\terr := parseLeafNodes(tk, o, errors, warnings)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"store_dir\", \"storedir\":\n\t\t// Check if JetStream configuration is also setting the storage directory.\n\t\tif o.StoreDir != _EMPTY_ {\n\t\t\t*errors = append(*errors, &configErr{tk, \"Duplicate 'store_dir' configuration\"})\n\t\t\treturn\n\t\t}\n\t\to.StoreDir = v.(string)\n\tcase \"jetstream\":\n\t\terr := parseJetStream(tk, o, errors, warnings)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"logfile\", \"log_file\":\n\t\to.LogFile = v.(string)\n\tcase \"logfile_size_limit\", \"log_size_limit\":\n\t\to.LogSizeLimit = v.(int64)\n\tcase \"logfile_max_num\", \"log_max_num\":\n\t\to.LogMaxFiles = v.(int64)\n\tcase \"syslog\":\n\t\to.Syslog = v.(bool)\n\t\ttrackExplicitVal(&o.inConfig, \"Syslog\", o.Syslog)\n\tcase \"remote_syslog\":\n\t\to.RemoteSyslog = v.(string)\n\tcase \"pidfile\", \"pid_file\":\n\t\to.PidFile = v.(string)\n\tcase \"ports_file_dir\":\n\t\to.PortsFileDir = v.(string)\n\tcase \"prof_port\":\n\t\to.ProfPort = int(v.(int64))\n\tcase \"prof_block_rate\":\n\t\to.ProfBlockRate = int(v.(int64))\n\tcase \"max_control_line\":\n\t\tif v.(int64) > 1<<31-1 {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"%s value is too big\", k)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.MaxControlLine = int32(v.(int64))\n\tcase \"max_payload\":\n\t\tif v.(int64) > 1<<31-1 {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"%s value is too big\", k)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.MaxPayload = int32(v.(int64))\n\tcase \"max_pending\":\n\t\to.MaxPending = v.(int64)\n\tcase \"proxy_protocol\":\n\t\to.ProxyProtocol = v.(bool)\n\tcase \"max_connections\", \"max_conn\":\n\t\tif o.MaxConn = int(v.(int64)); o.MaxConn == 0 {\n\t\t\to.MaxConn = -1\n\t\t}\n\tcase \"max_traced_msg_len\":\n\t\to.MaxTracedMsgLen = int(v.(int64))\n\tcase \"max_subscriptions\", \"max_subs\":\n\t\to.MaxSubs = int(v.(int64))\n\tcase \"max_sub_tokens\", \"max_subscription_tokens\":\n\t\tif n := v.(int64); n > math.MaxUint8 {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"%s value is too big\", k)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t} else if n <= 0 {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"%s value can not be negative\", k)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t} else {\n\t\t\to.MaxSubTokens = uint8(n)\n\t\t}\n\tcase \"ping_interval\":\n\t\to.PingInterval = parseDuration(\"ping_interval\", tk, v, errors, warnings)\n\tcase \"ping_max\":\n\t\to.MaxPingsOut = int(v.(int64))\n\tcase \"tls\":\n\t\ttc, err := parseTLS(tk, true)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\tif o.TLSConfig, err = GenTLSConfig(tc); err != nil {\n\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.TLSTimeout = tc.Timeout\n\t\to.TLSMap = tc.Map\n\t\to.TLSPinnedCerts = tc.PinnedCerts\n\t\to.TLSRateLimit = tc.RateLimit\n\t\to.TLSHandshakeFirst = tc.HandshakeFirst\n\t\to.TLSHandshakeFirstFallback = tc.FallbackDelay\n\n\t\t// Need to keep track of path of the original TLS config\n\t\t// and certs path for OCSP Stapling monitoring.\n\t\to.tlsConfigOpts = tc\n\tcase \"ocsp\":\n\t\tswitch vv := v.(type) {\n\t\tcase bool:\n\t\t\tif vv {\n\t\t\t\t// Default is Auto which honors Must Staple status request\n\t\t\t\t// but does not shutdown the server in case it is revoked,\n\t\t\t\t// letting the client choose whether to trust or not the server.\n\t\t\t\to.OCSPConfig = &OCSPConfig{Mode: OCSPModeAuto}\n\t\t\t} else {\n\t\t\t\to.OCSPConfig = &OCSPConfig{Mode: OCSPModeNever}\n\t\t\t}\n\t\tcase map[string]any:\n\t\t\tocsp := &OCSPConfig{Mode: OCSPModeAuto}\n\n\t\t\tfor kk, kv := range vv {\n\t\t\t\t_, v = unwrapValue(kv, &tk)\n\t\t\t\tswitch kk {\n\t\t\t\tcase \"mode\":\n\t\t\t\t\tmode := v.(string)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase strings.EqualFold(mode, \"always\"):\n\t\t\t\t\t\tocsp.Mode = OCSPModeAlways\n\t\t\t\t\tcase strings.EqualFold(mode, \"must\"):\n\t\t\t\t\t\tocsp.Mode = OCSPModeMust\n\t\t\t\t\tcase strings.EqualFold(mode, \"never\"):\n\t\t\t\t\t\tocsp.Mode = OCSPModeNever\n\t\t\t\t\tcase strings.EqualFold(mode, \"auto\"):\n\t\t\t\t\t\tocsp.Mode = OCSPModeAuto\n\t\t\t\t\tdefault:\n\t\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"error parsing ocsp config: unsupported ocsp mode %T\", mode)})\n\t\t\t\t\t}\n\t\t\t\tcase \"urls\":\n\t\t\t\t\turls := v.([]string)\n\t\t\t\t\tocsp.OverrideURLs = urls\n\t\t\t\tcase \"url\":\n\t\t\t\t\turl := v.(string)\n\t\t\t\t\tocsp.OverrideURLs = []string{url}\n\t\t\t\tdefault:\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"error parsing ocsp config: unsupported field %T\", kk)})\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\to.OCSPConfig = ocsp\n\t\tdefault:\n\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"error parsing ocsp config: unsupported type %T\", v)})\n\t\t\treturn\n\t\t}\n\tcase \"allow_non_tls\":\n\t\to.AllowNonTLS = v.(bool)\n\tcase \"write_deadline\":\n\t\to.WriteDeadline = parseDuration(\"write_deadline\", tk, v, errors, warnings)\n\tcase \"write_timeout\":\n\t\to.WriteTimeout = parseWriteDeadlinePolicy(tk, v.(string), errors)\n\tcase \"lame_duck_duration\":\n\t\tdur, err := time.ParseDuration(v.(string))\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing lame_duck_duration: %v\", err)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\tif dur < 30*time.Second {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"invalid lame_duck_duration of %v, minimum is 30 seconds\", dur)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.LameDuckDuration = dur\n\tcase \"lame_duck_grace_period\":\n\t\tdur, err := time.ParseDuration(v.(string))\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing lame_duck_grace_period: %v\", err)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\tif dur < 0 {\n\t\t\terr := &configErr{tk, \"invalid lame_duck_grace_period, needs to be positive\"}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.LameDuckGracePeriod = dur\n\tcase \"operator\", \"operators\", \"roots\", \"root\", \"root_operators\", \"root_operator\":\n\t\topFiles := []string{}\n\t\tswitch v := v.(type) {\n\t\tcase string:\n\t\t\topFiles = append(opFiles, v)\n\t\tcase []string:\n\t\t\topFiles = append(opFiles, v...)\n\t\tcase []any:\n\t\t\tfor _, t := range v {\n\t\t\t\tif token, ok := t.(token); ok {\n\t\t\t\t\tif v, ok := token.Value().(string); ok {\n\t\t\t\t\t\topFiles = append(opFiles, v)\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing operators: unsupported type %T where string is expected\", token)}\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing operators: unsupported type %T\", t)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing operators: unsupported type %T\", v)}\n\t\t\t*errors = append(*errors, err)\n\t\t}\n\t\t// Assume for now these are file names, but they can also be the JWT itself inline.\n\t\to.TrustedOperators = make([]*jwt.OperatorClaims, 0, len(opFiles))\n\t\tfor _, fname := range opFiles {\n\t\t\ttheJWT, opc, err := readOperatorJWT(fname)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing operator JWT: %v\", err)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.operatorJWT = append(o.operatorJWT, theJWT)\n\t\t\to.TrustedOperators = append(o.TrustedOperators, opc)\n\t\t}\n\t\tif len(o.TrustedOperators) == 1 {\n\t\t\t// In case \"resolver\" is defined as well, it takes precedence\n\t\t\tif o.AccountResolver == nil {\n\t\t\t\tif accUrl, err := parseURL(o.TrustedOperators[0].AccountServerURL, \"account resolver\"); err == nil {\n\t\t\t\t\t// nsc automatically appends \"/accounts\" during nsc push\n\t\t\t\t\to.AccountResolver, _ = NewURLAccResolver(accUrl.String() + \"/accounts\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t// In case \"system_account\" is defined as well, it takes precedence\n\t\t\tif o.SystemAccount == _EMPTY_ {\n\t\t\t\to.SystemAccount = o.TrustedOperators[0].SystemAccount\n\t\t\t}\n\t\t}\n\tcase \"resolver\", \"account_resolver\", \"accounts_resolver\":\n\t\tswitch v := v.(type) {\n\t\tcase string:\n\t\t\t// \"resolver\" takes precedence over value obtained from \"operator\".\n\t\t\t// Clear so that parsing errors are not silently ignored.\n\t\t\to.AccountResolver = nil\n\t\t\tmemResolverRe := regexp.MustCompile(`(?i)(MEM|MEMORY)\\s*`)\n\t\t\tresolverRe := regexp.MustCompile(`(?i)(?:URL){1}(?:\\({1}\\s*\"?([^\\s\"]*)\"?\\s*\\){1})?\\s*`)\n\t\t\tif memResolverRe.MatchString(v) {\n\t\t\t\to.AccountResolver = &MemAccResolver{}\n\t\t\t} else if items := resolverRe.FindStringSubmatch(v); len(items) == 2 {\n\t\t\t\turl := items[1]\n\t\t\t\t_, err := parseURL(url, \"account resolver\")\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif ur, err := NewURLAccResolver(url); err != nil {\n\t\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn\n\t\t\t\t} else {\n\t\t\t\t\to.AccountResolver = ur\n\t\t\t\t}\n\t\t\t}\n\t\tcase map[string]any:\n\t\t\tdel := false\n\t\t\thdel := false\n\t\t\thdel_set := false\n\t\t\tdir := _EMPTY_\n\t\t\tdirType := _EMPTY_\n\t\t\tlimit := int64(0)\n\t\t\tttl := time.Duration(0)\n\t\t\tsync := time.Duration(0)\n\t\t\topts := []DirResOption{}\n\t\t\tvar err error\n\t\t\tif v, ok := v[\"dir\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tdir = v.(string)\n\t\t\t}\n\t\t\tif v, ok := v[\"type\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tdirType = v.(string)\n\t\t\t}\n\t\t\tif v, ok := v[\"allow_delete\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tdel = v.(bool)\n\t\t\t}\n\t\t\tif v, ok := v[\"hard_delete\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\thdel_set = true\n\t\t\t\thdel = v.(bool)\n\t\t\t}\n\t\t\tif v, ok := v[\"limit\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tlimit = v.(int64)\n\t\t\t}\n\t\t\tif v, ok := v[\"ttl\"]; ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tttl, err = time.ParseDuration(v.(string))\n\t\t\t}\n\t\t\tif v, ok := v[\"interval\"]; err == nil && ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tsync, err = time.ParseDuration(v.(string))\n\t\t\t}\n\t\t\tif v, ok := v[\"timeout\"]; err == nil && ok {\n\t\t\t\t_, v := unwrapValue(v, <)\n\t\t\t\tvar to time.Duration\n\t\t\t\tif to, err = time.ParseDuration(v.(string)); err == nil {\n\t\t\t\t\topts = append(opts, FetchTimeout(to))\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tcheckDir := func() {\n\t\t\t\tif dir == _EMPTY_ {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"dir has no value and needs to point to a directory\"})\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif info, _ := os.Stat(dir); info != nil && (!info.IsDir() || info.Mode().Perm()&(1<<(uint(7))) == 0) {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"dir needs to point to an accessible directory\"})\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar res AccountResolver\n\t\t\tswitch strings.ToUpper(dirType) {\n\t\t\tcase \"CACHE\":\n\t\t\t\tcheckDir()\n\t\t\t\tif sync != 0 {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"CACHE does not accept sync\"})\n\t\t\t\t}\n\t\t\t\tif del {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"CACHE does not accept allow_delete\"})\n\t\t\t\t}\n\t\t\t\tif hdel_set {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"CACHE does not accept hard_delete\"})\n\t\t\t\t}\n\t\t\t\tres, err = NewCacheDirAccResolver(dir, limit, ttl, opts...)\n\t\t\tcase \"FULL\":\n\t\t\t\tcheckDir()\n\t\t\t\tif ttl != 0 {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"FULL does not accept ttl\"})\n\t\t\t\t}\n\t\t\t\tif hdel_set && !del {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"hard_delete has no effect without delete\"})\n\t\t\t\t}\n\t\t\t\tdelete := NoDelete\n\t\t\t\tif del {\n\t\t\t\t\tif hdel {\n\t\t\t\t\t\tdelete = HardDelete\n\t\t\t\t\t} else {\n\t\t\t\t\t\tdelete = RenameDeleted\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tres, err = NewDirAccResolver(dir, limit, sync, delete, opts...)\n\t\t\tcase \"MEM\", \"MEMORY\":\n\t\t\t\tres = &MemAccResolver{}\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\treturn\n\t\t\t}\n\t\t\to.AccountResolver = res\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing operator resolver, wrong type %T\", v)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\tif o.AccountResolver == nil {\n\t\t\terr := &configErr{tk, \"error parsing account resolver, should be MEM or \" +\n\t\t\t\t\" URL(\\\"url\\\") or a map containing dir and type state=[FULL|CACHE])\"}\n\t\t\t*errors = append(*errors, err)\n\t\t}\n\tcase \"resolver_tls\":\n\t\ttc, err := parseTLS(tk, true)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\ttlsConfig, err := GenTLSConfig(tc)\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.AccountResolverTLSConfig = tlsConfig\n\t\t// GenTLSConfig loads the CA file into ClientCAs, but since this will\n\t\t// be used as a client connection, we need to set RootCAs.\n\t\to.AccountResolverTLSConfig.RootCAs = tlsConfig.ClientCAs\n\tcase \"resolver_preload\":\n\t\tmp, ok := v.(map[string]any)\n\t\tif !ok {\n\t\t\terr := &configErr{tk, \"preload should be a map of account_public_key:account_jwt\"}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.resolverPreloads = make(map[string]string)\n\t\tfor key, val := range mp {\n\t\t\ttk, val = unwrapValue(val, <)\n\t\t\tif jwtstr, ok := val.(string); !ok {\n\t\t\t\t*errors = append(*errors, &configErr{tk, \"preload map value should be a string JWT\"})\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\t// Make sure this is a valid account JWT, that is a config error.\n\t\t\t\t// We will warn of expirations, etc later.\n\t\t\t\tif _, err := jwt.DecodeAccountClaims(jwtstr); err != nil {\n\t\t\t\t\terr := &configErr{tk, \"invalid account JWT\"}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\to.resolverPreloads[key] = jwtstr\n\t\t\t}\n\t\t}\n\tcase \"resolver_pinned_accounts\":\n\t\tswitch v := v.(type) {\n\t\tcase string:\n\t\t\to.resolverPinnedAccounts = map[string]struct{}{v: {}}\n\t\tcase []string:\n\t\t\to.resolverPinnedAccounts = make(map[string]struct{})\n\t\t\tfor _, mv := range v {\n\t\t\t\to.resolverPinnedAccounts[mv] = struct{}{}\n\t\t\t}\n\t\tcase []any:\n\t\t\to.resolverPinnedAccounts = make(map[string]struct{})\n\t\t\tfor _, mv := range v {\n\t\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\t\tif key, ok := mv.(string); ok {\n\t\t\t\t\to.resolverPinnedAccounts[key] = struct{}{}\n\t\t\t\t} else {\n\t\t\t\t\terr := &configErr{tk,\n\t\t\t\t\t\tfmt.Sprintf(\"error parsing resolver_pinned_accounts: unsupported type in array %T\", mv)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing resolver_pinned_accounts: unsupported type %T\", v)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"no_auth_user\":\n\t\to.NoAuthUser = v.(string)\n\tcase \"system_account\", \"system\":\n\t\t// Already processed at the beginning so we just skip them\n\t\t// to not treat them as unknown values.\n\t\treturn\n\tcase \"no_system_account\", \"no_system\", \"no_sys_acc\":\n\t\to.NoSystemAccount = v.(bool)\n\tcase \"no_header_support\":\n\t\to.NoHeaderSupport = v.(bool)\n\tcase \"trusted\", \"trusted_keys\":\n\t\tswitch v := v.(type) {\n\t\tcase string:\n\t\t\to.TrustedKeys = []string{v}\n\t\tcase []string:\n\t\t\to.TrustedKeys = v\n\t\tcase []any:\n\t\t\tkeys := make([]string, 0, len(v))\n\t\t\tfor _, mv := range v {\n\t\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\t\tif key, ok := mv.(string); ok {\n\t\t\t\t\tkeys = append(keys, key)\n\t\t\t\t} else {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing trusted: unsupported type in array %T\", mv)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\to.TrustedKeys = keys\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing trusted: unsupported type %T\", v)}\n\t\t\t*errors = append(*errors, err)\n\t\t}\n\t\t// Do a quick sanity check on keys\n\t\tfor _, key := range o.TrustedKeys {\n\t\t\tif !nkeys.IsValidPublicOperatorKey(key) {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"trust key %q required to be a valid public operator nkey\", key)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\tcase \"connect_error_reports\":\n\t\to.ConnectErrorReports = int(v.(int64))\n\tcase \"reconnect_error_reports\":\n\t\to.ReconnectErrorReports = int(v.(int64))\n\tcase \"websocket\", \"ws\":\n\t\tif err := parseWebsocket(tk, o, errors, warnings); err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"mqtt\":\n\t\tif err := parseMQTT(tk, o, errors, warnings); err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"server_tags\":\n\t\tvar err error\n\t\tswitch v := v.(type) {\n\t\tcase string:\n\t\t\to.Tags.Add(v)\n\t\tcase []string:\n\t\t\to.Tags.Add(v...)\n\t\tcase []any:\n\t\t\tfor _, t := range v {\n\t\t\t\tif token, ok := t.(token); ok {\n\t\t\t\t\tif ts, ok := token.Value().(string); ok {\n\t\t\t\t\t\to.Tags.Add(ts)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t} else {\n\t\t\t\t\t\terr = &configErr{tk, fmt.Sprintf(\"error parsing tags: unsupported type %T where string is expected\", token)}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\terr = &configErr{tk, fmt.Sprintf(\"error parsing tags: unsupported type %T\", t)}\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\tdefault:\n\t\t\terr = &configErr{tk, fmt.Sprintf(\"error parsing tags: unsupported type %T\", v)}\n\t\t}\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"server_metadata\":\n\t\tvar err error\n\t\tswitch v := v.(type) {\n\t\tcase map[string]any:\n\t\t\tfor mk, mv := range v {\n\t\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\t\tif o.Metadata == nil {\n\t\t\t\t\to.Metadata = make(map[string]string)\n\t\t\t\t}\n\t\t\t\to.Metadata[mk] = mv.(string)\n\t\t\t}\n\t\tdefault:\n\t\t\terr = &configErr{tk, fmt.Sprintf(\"error parsing metadata: unsupported type %T\", v)}\n\t\t}\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"default_js_domain\":\n\t\tvv, ok := v.(map[string]any)\n\t\tif !ok {\n\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"error default_js_domain config: unsupported type %T\", v)})\n\t\t\treturn\n\t\t}\n\t\tm := make(map[string]string)\n\t\tfor kk, kv := range vv {\n\t\t\t_, v = unwrapValue(kv, &tk)\n\t\t\tm[kk] = v.(string)\n\t\t}\n\t\to.JsAccDefaultDomain = m\n\tcase \"ocsp_cache\":\n\t\tvar err error\n\t\tswitch vv := v.(type) {\n\t\tcase bool:\n\t\t\tpc := NewOCSPResponseCacheConfig()\n\t\t\tif vv {\n\t\t\t\t// Set enabled\n\t\t\t\tpc.Type = LOCAL\n\t\t\t\to.OCSPCacheConfig = pc\n\t\t\t} else {\n\t\t\t\t// Set disabled (none cache)\n\t\t\t\tpc.Type = NONE\n\t\t\t\to.OCSPCacheConfig = pc\n\t\t\t}\n\t\tcase map[string]any:\n\t\t\tpc, err := parseOCSPResponseCache(v)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\to.OCSPCacheConfig = pc\n\t\tdefault:\n\t\t\terr = &configErr{tk, fmt.Sprintf(\"error parsing tags: unsupported type %T\", v)}\n\t\t}\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\tcase \"no_fast_producer_stall\":\n\t\to.NoFastProducerStall = v.(bool)\n\tcase \"max_closed_clients\":\n\t\to.MaxClosedClients = int(v.(int64))\n\tcase \"proxies\":\n\t\tproxies, err := parseProxies(tk, errors)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn\n\t\t}\n\t\to.Proxies = proxies\n\tdefault:\n\t\tif au := atomic.LoadInt32(&allowUnknownTopLevelField); au == 0 && !tk.IsUsedVariable() {\n\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\tfield: k,\n\t\t\t\tconfigErr: configErr{\n\t\t\t\t\ttoken: tk,\n\t\t\t\t},\n\t\t\t}\n\t\t\t*errors = append(*errors, err)\n\t\t}\n\t}\n}\n\nfunc setupUsersAndNKeysDuplicateCheckMap(o *Options) map[string]struct{} {\n\tunames := make(map[string]struct{}, len(o.Users)+len(o.Nkeys))\n\tfor _, u := range o.Users {\n\t\tunames[u.Username] = struct{}{}\n\t}\n\tfor _, u := range o.Nkeys {\n\t\tunames[u.Nkey] = struct{}{}\n\t}\n\treturn unames\n}\n\nfunc parseDuration(field string, tk token, v any, errors *[]error, warnings *[]error) time.Duration {\n\tif wd, ok := v.(string); ok {\n\t\tif dur, err := time.ParseDuration(wd); err != nil {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing %s: %v\", field, err)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn 0\n\t\t} else {\n\t\t\treturn dur\n\t\t}\n\t} else {\n\t\t// Backward compatible with old type, assume this is the\n\t\t// number of seconds.\n\t\terr := &configWarningErr{\n\t\t\tfield: field,\n\t\t\tconfigErr: configErr{\n\t\t\t\ttoken: tk,\n\t\t\t\treason: field + \" should be converted to a duration\",\n\t\t\t},\n\t\t}\n\t\t*warnings = append(*warnings, err)\n\t\treturn time.Duration(v.(int64)) * time.Second\n\t}\n}\n\nfunc parseWriteDeadlinePolicy(tk token, v string, errors *[]error) WriteTimeoutPolicy {\n\tswitch v {\n\tcase \"default\":\n\t\treturn WriteTimeoutPolicyDefault\n\tcase \"close\":\n\t\treturn WriteTimeoutPolicyClose\n\tcase \"retry\":\n\t\treturn WriteTimeoutPolicyRetry\n\tdefault:\n\t\terr := &configErr{tk, \"write_timeout must be 'default', 'close' or 'retry'\"}\n\t\t*errors = append(*errors, err)\n\t\treturn WriteTimeoutPolicyDefault\n\t}\n}\n\nfunc trackExplicitVal(pm *map[string]bool, name string, val bool) {\n\tm := *pm\n\tif m == nil {\n\t\tm = make(map[string]bool)\n\t\t*pm = m\n\t}\n\tm[name] = val\n}\n\n// hostPort is simple struct to hold parsed listen/addr strings.\ntype hostPort struct {\n\thost string\n\tport int\n}\n\n// parseListen will parse listen option which is replacing host/net and port\nfunc parseListen(v any) (*hostPort, error) {\n\thp := &hostPort{}\n\tswitch vv := v.(type) {\n\t// Only a port\n\tcase int64:\n\t\thp.port = int(vv)\n\tcase string:\n\t\thost, port, err := net.SplitHostPort(vv)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not parse address string %q\", vv)\n\t\t}\n\t\thp.port, err = strconv.Atoi(port)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not parse port %q\", port)\n\t\t}\n\t\thp.host = host\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"expected port or host:port, got %T\", vv)\n\t}\n\treturn hp, nil\n}\n\n// parseCluster will parse the cluster config.\nfunc parseCluster(v any, opts *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tcm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected map to define cluster, got %T\", v)}\n\t}\n\n\tfor mk, mv := range cm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"name\":\n\t\t\tcn := mv.(string)\n\t\t\tif strings.Contains(cn, \" \") {\n\t\t\t\terr := &configErr{tk, ErrClusterNameHasSpaces.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.Cluster.Name = cn\n\t\tcase \"listen\":\n\t\t\thp, err := parseListen(mv)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.Cluster.Host = hp.host\n\t\t\topts.Cluster.Port = hp.port\n\t\tcase \"port\":\n\t\t\topts.Cluster.Port = int(mv.(int64))\n\t\tcase \"host\", \"net\":\n\t\t\topts.Cluster.Host = mv.(string)\n\t\tcase \"authorization\":\n\t\t\tauth, err := parseAuthorization(tk, errors, warnings)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.users != nil {\n\t\t\t\terr := &configErr{tk, \"Cluster authorization does not allow multiple users\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.token != _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Cluster authorization does not support tokens\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.callout != nil {\n\t\t\t\terr := &configErr{tk, \"Cluster authorization does not support callouts\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\topts.Cluster.Username = auth.user\n\t\t\topts.Cluster.Password = auth.pass\n\t\t\topts.Cluster.AuthTimeout = auth.timeout\n\n\t\t\tif auth.defaultPermissions != nil {\n\t\t\t\terr := &configWarningErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\treason: `setting \"permissions\" within cluster authorization block is deprecated`,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*warnings = append(*warnings, err)\n\n\t\t\t\t// Do not set permissions if they were specified in top-level cluster block.\n\t\t\t\tif opts.Cluster.Permissions == nil {\n\t\t\t\t\tsetClusterPermissions(&opts.Cluster, auth.defaultPermissions)\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"routes\":\n\t\t\tra := mv.([]any)\n\t\t\troutes, errs := parseURLs(ra, \"route\", warnings)\n\t\t\tif errs != nil {\n\t\t\t\t*errors = append(*errors, errs...)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.Routes = routes\n\t\tcase \"tls\":\n\t\t\tconfig, tlsopts, err := getTLSConfig(tk)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.Cluster.TLSConfig = config\n\t\t\topts.Cluster.TLSTimeout = tlsopts.Timeout\n\t\t\topts.Cluster.TLSMap = tlsopts.Map\n\t\t\topts.Cluster.TLSPinnedCerts = tlsopts.PinnedCerts\n\t\t\topts.Cluster.TLSCheckKnownURLs = tlsopts.TLSCheckKnownURLs\n\t\t\topts.Cluster.tlsConfigOpts = tlsopts\n\t\tcase \"cluster_advertise\", \"advertise\":\n\t\t\topts.Cluster.Advertise = mv.(string)\n\t\tcase \"no_advertise\":\n\t\t\topts.Cluster.NoAdvertise = mv.(bool)\n\t\t\ttrackExplicitVal(&opts.inConfig, \"Cluster.NoAdvertise\", opts.Cluster.NoAdvertise)\n\t\tcase \"connect_retries\":\n\t\t\topts.Cluster.ConnectRetries = int(mv.(int64))\n\t\tcase \"connect_backoff\":\n\t\t\topts.Cluster.ConnectBackoff = mv.(bool)\n\t\tcase \"permissions\":\n\t\t\tperms, err := parseUserPermissions(mv, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Dynamic response permissions do not make sense here.\n\t\t\tif perms.Response != nil {\n\t\t\t\terr := &configErr{tk, \"Cluster permissions do not support dynamic responses\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// This will possibly override permissions that were define in auth block\n\t\t\tsetClusterPermissions(&opts.Cluster, perms)\n\t\tcase \"pool_size\":\n\t\t\topts.Cluster.PoolSize = int(mv.(int64))\n\t\tcase \"accounts\":\n\t\t\topts.Cluster.PinnedAccounts, _ = parseStringArray(\"accounts\", tk, <, mv, errors)\n\t\tcase \"compression\":\n\t\t\tif err := parseCompression(&opts.Cluster.Compression, CompressionS2Fast, tk, mk, mv); err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase \"ping_interval\":\n\t\t\topts.Cluster.PingInterval = parseDuration(\"ping_interval\", tk, mv, errors, warnings)\n\t\t\tif opts.Cluster.PingInterval > routeMaxPingInterval {\n\t\t\t\t*warnings = append(*warnings, &configErr{tk, fmt.Sprintf(\"Cluster 'ping_interval' will reset to %v which is the max for routes\", routeMaxPingInterval)})\n\t\t\t}\n\t\tcase \"ping_max\":\n\t\t\topts.Cluster.MaxPingsOut = int(mv.(int64))\n\t\tcase \"write_deadline\":\n\t\t\topts.Cluster.WriteDeadline = parseDuration(\"write_deadline\", tk, mv, errors, warnings)\n\t\tcase \"write_timeout\":\n\t\t\topts.Cluster.WriteTimeout = parseWriteDeadlinePolicy(tk, mv.(string), errors)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// The parameter `chosenModeForOn` indicates which compression mode to use\n// when the user selects \"on\" (or enabled, true, etc..). This is because\n// we may have different defaults depending on where the compression is used.\nfunc parseCompression(c *CompressionOpts, chosenModeForOn string, tk token, mk string, mv any) (retErr error) {\n\tvar lt token\n\tdefer convertPanicToError(<, &retErr)\n\n\tswitch mv := mv.(type) {\n\tcase string:\n\t\t// Do not validate here, it will be done in NewServer.\n\t\tc.Mode = mv\n\tcase bool:\n\t\tif mv {\n\t\t\tc.Mode = chosenModeForOn\n\t\t} else {\n\t\t\tc.Mode = CompressionOff\n\t\t}\n\tcase map[string]any:\n\t\tfor mk, mv := range mv {\n\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\tswitch strings.ToLower(mk) {\n\t\t\tcase \"mode\":\n\t\t\t\tc.Mode = mv.(string)\n\t\t\tcase \"rtt_thresholds\", \"thresholds\", \"rtts\", \"rtt\":\n\t\t\t\tfor _, iv := range mv.([]any) {\n\t\t\t\t\t_, mv := unwrapValue(iv, <)\n\t\t\t\t\tdur, err := time.ParseDuration(mv.(string))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn &configErr{tk, err.Error()}\n\t\t\t\t\t}\n\t\t\t\t\tc.RTTThresholds = append(c.RTTThresholds, dur)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"unknown field %q\", mk)}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tdefault:\n\t\treturn &configErr{tk, fmt.Sprintf(\"field %q should be a boolean or a structure, got %T\", mk, mv)}\n\t}\n\treturn nil\n}\n\nfunc parseURLs(a []any, typ string, warnings *[]error) (urls []*url.URL, errors []error) {\n\turls = make([]*url.URL, 0, len(a))\n\tvar lt token\n\tdefer convertPanicToErrorList(<, &errors)\n\n\tdd := make(map[string]bool)\n\n\tfor _, u := range a {\n\t\ttk, u := unwrapValue(u, <)\n\t\tsURL := u.(string)\n\t\tif dd[sURL] {\n\t\t\terr := &configWarningErr{\n\t\t\t\tfield: sURL,\n\t\t\t\tconfigErr: configErr{\n\t\t\t\t\ttoken: tk,\n\t\t\t\t\treason: fmt.Sprintf(\"Duplicate %s entry detected\", typ),\n\t\t\t\t},\n\t\t\t}\n\t\t\t*warnings = append(*warnings, err)\n\t\t\tcontinue\n\t\t}\n\t\tdd[sURL] = true\n\t\turl, err := parseURL(sURL, typ)\n\t\tif err != nil {\n\t\t\terr := &configErr{tk, err.Error()}\n\t\t\terrors = append(errors, err)\n\t\t\tcontinue\n\t\t}\n\t\turls = append(urls, url)\n\t}\n\treturn urls, errors\n}\n\nfunc parseURL(u string, typ string) (*url.URL, error) {\n\turlStr := strings.TrimSpace(u)\n\turl, err := url.Parse(urlStr)\n\tif err != nil {\n\t\t// Security note: if it's not well-formed but still reached us, then we're going to log as-is which might include password information here.\n\t\t// If the URL parses, we don't log the credentials ever, but if it doesn't even parse we don't have a sane way to redact.\n\t\treturn nil, fmt.Errorf(\"error parsing %s url [%q]\", typ, urlStr)\n\t}\n\treturn url, nil\n}\n\nfunc parseGateway(v any, o *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tgm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected gateway to be a map, got %T\", v)}\n\t}\n\tfor mk, mv := range gm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"name\":\n\t\t\tgn := mv.(string)\n\t\t\tif strings.Contains(gn, \" \") {\n\t\t\t\terr := &configErr{tk, ErrGatewayNameHasSpaces.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Gateway.Name = gn\n\t\tcase \"listen\":\n\t\t\thp, err := parseListen(mv)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Gateway.Host = hp.host\n\t\t\to.Gateway.Port = hp.port\n\t\tcase \"port\":\n\t\t\to.Gateway.Port = int(mv.(int64))\n\t\tcase \"host\", \"net\":\n\t\t\to.Gateway.Host = mv.(string)\n\t\tcase \"authorization\":\n\t\t\tauth, err := parseAuthorization(tk, errors, warnings)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.users != nil {\n\t\t\t\t*errors = append(*errors, &configErr{tk, \"Gateway authorization does not allow multiple users\"})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.token != _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Gateway authorization does not support tokens\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif auth.callout != nil {\n\t\t\t\terr := &configErr{tk, \"Gateway authorization does not support callouts\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\to.Gateway.Username = auth.user\n\t\t\to.Gateway.Password = auth.pass\n\t\t\to.Gateway.AuthTimeout = auth.timeout\n\t\tcase \"tls\":\n\t\t\tconfig, tlsopts, err := getTLSConfig(tk)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Gateway.TLSConfig = config\n\t\t\to.Gateway.TLSTimeout = tlsopts.Timeout\n\t\t\to.Gateway.TLSMap = tlsopts.Map\n\t\t\to.Gateway.TLSCheckKnownURLs = tlsopts.TLSCheckKnownURLs\n\t\t\to.Gateway.TLSPinnedCerts = tlsopts.PinnedCerts\n\t\t\to.Gateway.tlsConfigOpts = tlsopts\n\t\tcase \"advertise\":\n\t\t\to.Gateway.Advertise = mv.(string)\n\t\tcase \"connect_retries\":\n\t\t\to.Gateway.ConnectRetries = int(mv.(int64))\n\t\tcase \"connect_backoff\":\n\t\t\to.Gateway.ConnectBackoff = mv.(bool)\n\t\tcase \"gateways\":\n\t\t\tgateways, err := parseGateways(mv, errors, warnings)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to.Gateway.Gateways = gateways\n\t\tcase \"reject_unknown\", \"reject_unknown_cluster\":\n\t\t\to.Gateway.RejectUnknown = mv.(bool)\n\t\tcase \"write_deadline\":\n\t\t\to.Gateway.WriteDeadline = parseDuration(\"write_deadline\", tk, mv, errors, warnings)\n\t\tcase \"write_timeout\":\n\t\t\to.Gateway.WriteTimeout = parseWriteDeadlinePolicy(tk, mv.(string), errors)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nvar dynamicJSAccountLimits = JetStreamAccountLimits{-1, -1, -1, -1, -1, -1, -1, false}\nvar defaultJSAccountTiers = map[string]JetStreamAccountLimits{_EMPTY_: dynamicJSAccountLimits}\n\n// Parses jetstream account limits for an account. Simple setup with boolen is allowed, and we will\n// use dynamic account limits.\nfunc parseJetStreamForAccount(v any, acc *Account, errors *[]error) error {\n\tvar lt token\n\n\ttk, v := unwrapValue(v, <)\n\n\t// Value here can be bool, or string \"enabled\" or a map.\n\tswitch vv := v.(type) {\n\tcase bool:\n\t\tif vv {\n\t\t\tacc.jsLimits = defaultJSAccountTiers\n\t\t}\n\tcase string:\n\t\tswitch strings.ToLower(vv) {\n\t\tcase \"enabled\", \"enable\":\n\t\t\tacc.jsLimits = defaultJSAccountTiers\n\t\tcase \"disabled\", \"disable\":\n\t\t\tacc.jsLimits = nil\n\t\tdefault:\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected 'enabled' or 'disabled' for string value, got '%s'\", vv)}\n\t\t}\n\tcase map[string]any:\n\t\tjsLimits := JetStreamAccountLimits{-1, -1, -1, -1, -1, -1, -1, false}\n\t\tfor mk, mv := range vv {\n\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\tswitch strings.ToLower(mk) {\n\t\t\tcase \"max_memory\", \"max_mem\", \"mem\", \"memory\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxMemory = vv\n\t\t\tcase \"max_store\", \"max_file\", \"max_disk\", \"store\", \"disk\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxStore = vv\n\t\t\tcase \"max_streams\", \"streams\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxStreams = int(vv)\n\t\t\tcase \"max_consumers\", \"consumers\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxConsumers = int(vv)\n\t\t\tcase \"max_bytes_required\", \"max_stream_bytes\", \"max_bytes\":\n\t\t\t\tvv, ok := mv.(bool)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable bool for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxBytesRequired = vv\n\t\t\tcase \"mem_max_stream_bytes\", \"memory_max_stream_bytes\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MemoryMaxStreamBytes = vv\n\t\t\tcase \"disk_max_stream_bytes\", \"store_max_stream_bytes\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.StoreMaxStreamBytes = vv\n\t\t\tcase \"max_ack_pending\":\n\t\t\t\tvv, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tjsLimits.MaxAckPending = int(vv)\n\t\t\tcase \"cluster_traffic\":\n\t\t\t\tvv, ok := mv.(string)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected either 'system' or 'owner' string value for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\tswitch vv {\n\t\t\t\tcase \"system\", _EMPTY_:\n\t\t\t\t\tacc.nrgAccount = _EMPTY_\n\t\t\t\tcase \"owner\":\n\t\t\t\t\tacc.nrgAccount = acc.Name\n\t\t\t\tdefault:\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected 'system' or 'owner' string value for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: mk,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tacc.jsLimits = map[string]JetStreamAccountLimits{_EMPTY_: jsLimits}\n\tdefault:\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected map, bool or string to define JetStream, got %T\", v)}\n\t}\n\treturn nil\n}\n\n// takes in a storage size as either an int or a string and returns an int64 value based on the input.\nfunc getStorageSize(v any) (int64, error) {\n\t_, ok := v.(int64)\n\tif ok {\n\t\treturn v.(int64), nil\n\t}\n\n\ts, ok := v.(string)\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"must be int64 or string\")\n\t}\n\n\tif s == _EMPTY_ {\n\t\treturn 0, nil\n\t}\n\n\tsuffix := s[len(s)-1:]\n\tprefix := s[:len(s)-1]\n\tnum, err := strconv.ParseInt(prefix, 10, 64)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tsuffixMap := map[string]int64{\"K\": 10, \"M\": 20, \"G\": 30, \"T\": 40}\n\n\tmult, ok := suffixMap[suffix]\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"sizes defined as strings must end in K, M, G, T\")\n\t}\n\tnum *= 1 << mult\n\n\treturn num, nil\n}\n\n// Parse enablement of jetstream for a server.\nfunc parseJetStreamLimits(v any, opts *Options, errors *[]error) error {\n\tvar lt token\n\ttk, v := unwrapValue(v, <)\n\n\topts.JetStreamLimits = JSLimitOpts{}\n\n\tvv, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a map to define JetStreamLimits, got %T\", v)}\n\t}\n\tfor mk, mv := range vv {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"max_ack_pending\":\n\t\t\topts.JetStreamLimits.MaxAckPending = int(mv.(int64))\n\t\tcase \"max_ha_assets\":\n\t\t\topts.JetStreamLimits.MaxHAAssets = int(mv.(int64))\n\t\tcase \"max_request_batch\":\n\t\t\topts.JetStreamLimits.MaxRequestBatch = int(mv.(int64))\n\t\tcase \"duplicate_window\":\n\t\t\tvar err error\n\t\t\topts.JetStreamLimits.Duplicates, err = time.ParseDuration(mv.(string))\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\tcase \"batch\":\n\t\t\tif err := parseJetStreamLimitsBatch(tk, opts, errors); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc parseJetStreamLimitsBatch(v any, opts *Options, errors *[]error) error {\n\tvar lt token\n\ttk, v := unwrapValue(v, <)\n\n\tvv, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a map to define batch limits, got %T\", v)}\n\t}\n\tfor mk, mv := range vv {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"max_inflight_per_stream\":\n\t\t\topts.JetStreamLimits.MaxBatchInflightPerStream = int(mv.(int64))\n\t\tcase \"max_inflight_total\":\n\t\t\topts.JetStreamLimits.MaxBatchInflightTotal = int(mv.(int64))\n\t\tcase \"max_msgs\":\n\t\t\topts.JetStreamLimits.MaxBatchSize = int(mv.(int64))\n\t\tcase \"timeout\":\n\t\t\tvar err error\n\t\t\topts.JetStreamLimits.MaxBatchTimeout, err = time.ParseDuration(mv.(string))\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Parse the JetStream TPM options.\nfunc parseJetStreamTPM(v interface{}, opts *Options, errors *[]error) error {\n\tvar lt token\n\ttk, v := unwrapValue(v, <)\n\n\topts.JetStreamTpm = JSTpmOpts{}\n\n\tvv, ok := v.(map[string]interface{})\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a map to define JetStreamLimits, got %T\", v)}\n\t}\n\tfor mk, mv := range vv {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"keys_file\":\n\t\t\topts.JetStreamTpm.KeysFile = mv.(string)\n\t\tcase \"encryption_password\":\n\t\t\topts.JetStreamTpm.KeyPassword = mv.(string)\n\t\tcase \"srk_password\":\n\t\t\topts.JetStreamTpm.SrkPassword = mv.(string)\n\t\tcase \"pcr\":\n\t\t\topts.JetStreamTpm.Pcr = int(mv.(int64))\n\t\tcase \"cipher\":\n\t\t\tif err := setJetStreamEkCipher(opts, mv, tk); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc setJetStreamEkCipher(opts *Options, mv interface{}, tk token) error {\n\tswitch strings.ToLower(mv.(string)) {\n\tcase \"chacha\", \"chachapoly\":\n\t\tif fips140.Enabled() {\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Cipher type %q cannot be used in FIPS-140 mode\", mv)}\n\t\t}\n\t\topts.JetStreamCipher = ChaCha\n\tcase \"aes\":\n\t\topts.JetStreamCipher = AES\n\tdefault:\n\t\treturn &configErr{tk, fmt.Sprintf(\"Unknown cipher type: %q\", mv)}\n\t}\n\treturn nil\n}\n\n// Parse enablement of jetstream for a server.\nfunc parseJetStream(v any, opts *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\n\ttk, v := unwrapValue(v, <)\n\n\t// Value here can be bool, or string \"enabled\" or a map.\n\tswitch vv := v.(type) {\n\tcase bool:\n\t\topts.JetStream = v.(bool)\n\tcase string:\n\t\tswitch strings.ToLower(vv) {\n\t\tcase \"enabled\", \"enable\":\n\t\t\topts.JetStream = true\n\t\tcase \"disabled\", \"disable\":\n\t\t\topts.JetStream = false\n\t\tdefault:\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected 'enabled' or 'disabled' for string value, got '%s'\", vv)}\n\t\t}\n\tcase map[string]any:\n\t\tdoEnable := true\n\t\tfor mk, mv := range vv {\n\t\t\ttk, mv = unwrapValue(mv, <)\n\t\t\tswitch strings.ToLower(mk) {\n\t\t\tcase \"strict\":\n\t\t\t\tif v, ok := mv.(bool); ok {\n\t\t\t\t\topts.NoJetStreamStrict = !v\n\t\t\t\t} else {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected 'true' or 'false' for bool value, got '%s'\", mv)}\n\t\t\t\t}\n\t\t\tcase \"store\", \"store_dir\", \"storedir\":\n\t\t\t\t// StoreDir can be set at the top level as well so have to prevent ambiguous declarations.\n\t\t\t\tif opts.StoreDir != _EMPTY_ {\n\t\t\t\t\treturn &configErr{tk, \"Duplicate 'store_dir' configuration\"}\n\t\t\t\t}\n\t\t\t\topts.StoreDir = mv.(string)\n\t\t\tcase \"sync\", \"sync_interval\":\n\t\t\t\tif v, ok := mv.(string); ok && strings.ToLower(v) == \"always\" {\n\t\t\t\t\topts.SyncInterval = defaultSyncInterval\n\t\t\t\t\topts.SyncAlways = true\n\t\t\t\t} else {\n\t\t\t\t\topts.SyncInterval = parseDuration(mk, tk, mv, errors, warnings)\n\t\t\t\t}\n\t\t\t\topts.syncSet = true\n\t\t\tcase \"max_memory_store\", \"max_mem_store\", \"max_mem\":\n\t\t\t\ts, err := getStorageSize(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"max_mem_store %s\", err)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamMaxMemory = s\n\t\t\t\topts.maxMemSet = true\n\t\t\tcase \"max_file_store\", \"max_file\":\n\t\t\t\ts, err := getStorageSize(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"max_file_store %s\", err)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamMaxStore = s\n\t\t\t\topts.maxStoreSet = true\n\t\t\tcase \"domain\":\n\t\t\t\topts.JetStreamDomain = mv.(string)\n\t\t\tcase \"enable\", \"enabled\":\n\t\t\t\tdoEnable = mv.(bool)\n\t\t\tcase \"key\", \"ek\", \"encryption_key\":\n\t\t\t\topts.JetStreamKey = mv.(string)\n\t\t\tcase \"prev_key\", \"prev_ek\", \"prev_encryption_key\":\n\t\t\t\topts.JetStreamOldKey = mv.(string)\n\t\t\tcase \"cipher\":\n\t\t\t\tif err := setJetStreamEkCipher(opts, mv, tk); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase \"extension_hint\":\n\t\t\t\topts.JetStreamExtHint = mv.(string)\n\t\t\tcase \"limits\":\n\t\t\t\tif err := parseJetStreamLimits(tk, opts, errors); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase \"tpm\":\n\t\t\t\tif err := parseJetStreamTPM(tk, opts, errors); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase \"unique_tag\":\n\t\t\t\topts.JetStreamUniqueTag = strings.ToLower(strings.TrimSpace(mv.(string)))\n\t\t\tcase \"max_outstanding_catchup\":\n\t\t\t\ts, err := getStorageSize(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"%s %s\", strings.ToLower(mk), err)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamMaxCatchup = s\n\t\t\tcase \"max_buffered_size\":\n\t\t\t\ts, err := getStorageSize(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"%s %s\", strings.ToLower(mk), err)}\n\t\t\t\t}\n\t\t\t\topts.StreamMaxBufferedSize = s\n\t\t\tcase \"max_buffered_msgs\":\n\t\t\t\tmlen, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\topts.StreamMaxBufferedMsgs = int(mlen)\n\t\t\tcase \"request_queue_limit\":\n\t\t\t\tlim, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamRequestQueueLimit = lim\n\t\t\tcase \"info_queue_limit\":\n\t\t\t\tlim, ok := mv.(int64)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected a parseable size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamInfoQueueLimit = lim\n\t\t\tcase \"meta_compact\":\n\t\t\t\tthres, ok := mv.(int64)\n\t\t\t\tif !ok || thres < 0 {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected an absolute size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamMetaCompact = uint64(thres)\n\t\t\tcase \"meta_compact_size\":\n\t\t\t\ts, err := getStorageSize(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"%s %s\", strings.ToLower(mk), err)}\n\t\t\t\t}\n\t\t\t\tif s < 0 {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Expected an absolute size for %q, got %v\", mk, mv)}\n\t\t\t\t}\n\t\t\t\topts.JetStreamMetaCompactSize = uint64(s)\n\t\t\tcase \"meta_compact_sync\":\n\t\t\t\topts.JetStreamMetaCompactSync = mv.(bool)\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: mk,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\topts.JetStream = doEnable\n\tdefault:\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected map, bool or string to define JetStream, got %T\", v)}\n\t}\n\n\treturn nil\n}\n\n// parseLeafNodes will parse the leaf node config.\nfunc parseLeafNodes(v any, opts *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tcm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected map to define a leafnode, got %T\", v)}\n\t}\n\n\tfor mk, mv := range cm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"listen\":\n\t\t\thp, err := parseListen(mv)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.LeafNode.Host = hp.host\n\t\t\topts.LeafNode.Port = hp.port\n\t\tcase \"port\":\n\t\t\topts.LeafNode.Port = int(mv.(int64))\n\t\tcase \"host\", \"net\":\n\t\t\topts.LeafNode.Host = mv.(string)\n\t\tcase \"authorization\":\n\t\t\tauth, err := parseLeafAuthorization(tk, errors, warnings)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.LeafNode.Username = auth.user\n\t\t\topts.LeafNode.Password = auth.pass\n\t\t\topts.LeafNode.ProxyRequired = auth.proxyRequired\n\t\t\topts.LeafNode.AuthTimeout = auth.timeout\n\t\t\topts.LeafNode.Account = auth.acc\n\t\t\topts.LeafNode.Users = auth.users\n\t\t\topts.LeafNode.Nkey = auth.nkey\n\t\t\t// Validate user info config for leafnode authorization\n\t\t\tif err := validateLeafNodeAuthOptions(opts); err != nil {\n\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase \"remotes\":\n\t\t\t// Parse the remote options here.\n\t\t\tremotes, err := parseRemoteLeafNodes(tk, errors, warnings)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.LeafNode.Remotes = remotes\n\t\tcase \"reconnect\", \"reconnect_delay\", \"reconnect_interval\":\n\t\t\topts.LeafNode.ReconnectInterval = parseDuration(\"reconnect\", tk, mv, errors, warnings)\n\t\tcase \"tls\":\n\t\t\ttc, err := parseTLS(tk, true)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif opts.LeafNode.TLSConfig, err = GenTLSConfig(tc); err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.LeafNode.TLSTimeout = tc.Timeout\n\t\t\topts.LeafNode.TLSMap = tc.Map\n\t\t\topts.LeafNode.TLSPinnedCerts = tc.PinnedCerts\n\t\t\topts.LeafNode.TLSHandshakeFirst = tc.HandshakeFirst\n\t\t\topts.LeafNode.TLSHandshakeFirstFallback = tc.FallbackDelay\n\t\t\topts.LeafNode.tlsConfigOpts = tc\n\t\tcase \"leafnode_advertise\", \"advertise\":\n\t\t\topts.LeafNode.Advertise = mv.(string)\n\t\tcase \"no_advertise\":\n\t\t\topts.LeafNode.NoAdvertise = mv.(bool)\n\t\t\ttrackExplicitVal(&opts.inConfig, \"LeafNode.NoAdvertise\", opts.LeafNode.NoAdvertise)\n\t\tcase \"min_version\", \"minimum_version\":\n\t\t\tversion := mv.(string)\n\t\t\tif err := checkLeafMinVersionConfig(version); err != nil {\n\t\t\t\terr = &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.LeafNode.MinVersion = version\n\t\tcase \"compression\":\n\t\t\tif err := parseCompression(&opts.LeafNode.Compression, CompressionS2Auto, tk, mk, mv); err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase \"isolate_leafnode_interest\", \"isolate\":\n\t\t\topts.LeafNode.IsolateLeafnodeInterest = mv.(bool)\n\t\tcase \"write_deadline\":\n\t\t\topts.LeafNode.WriteDeadline = parseDuration(\"write_deadline\", tk, mv, errors, warnings)\n\t\tcase \"write_timeout\":\n\t\t\topts.LeafNode.WriteTimeout = parseWriteDeadlinePolicy(tk, mv.(string), errors)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// This is the authorization parser adapter for the leafnode's\n// authorization config.\nfunc parseLeafAuthorization(v any, errors, warnings *[]error) (*authorization, error) {\n\tvar (\n\t\tam map[string]any\n\t\ttk token\n\t\tlt token\n\t\tauth = &authorization{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\t_, v = unwrapValue(v, <)\n\tam = v.(map[string]any)\n\tfor mk, mv := range am {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"user\", \"username\":\n\t\t\tauth.user = mv.(string)\n\t\tcase \"pass\", \"password\":\n\t\t\tauth.pass = mv.(string)\n\t\tcase \"nkey\":\n\t\t\tnk := mv.(string)\n\t\t\tif !nkeys.IsValidPublicUserKey(nk) {\n\t\t\t\t*errors = append(*errors, &configErr{tk, \"Not a valid public nkey for leafnode authorization\"})\n\t\t\t}\n\t\t\tauth.nkey = nk\n\t\tcase \"timeout\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing leafnode authorization config, 'timeout' %s\", err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, \"error parsing leafnode authorization config, 'timeout' wrong type\"}\n\t\t\t}\n\t\t\tif at > (60 * time.Second).Seconds() {\n\t\t\t\treason := fmt.Sprintf(\"timeout of %v (%f seconds) is high, consider keeping it under 60 seconds. possibly caused by unquoted duration; use '1m' instead of 1m, for example\", mv, at)\n\t\t\t\t*warnings = append(*warnings, &configWarningErr{field: mk, configErr: configErr{token: tk, reason: reason}})\n\t\t\t}\n\t\t\tauth.timeout = at\n\t\tcase \"users\":\n\t\t\tusers, err := parseLeafUsers(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tauth.users = users\n\t\tcase \"account\":\n\t\t\tauth.acc = mv.(string)\n\t\tcase \"proxy_required\":\n\t\t\tauth.proxyRequired = mv.(bool)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t}\n\treturn auth, nil\n}\n\n// This is a trimmed down version of parseUsers that is adapted\n// for the users possibly defined in the authorization{} section\n// of leafnodes {}.\nfunc parseLeafUsers(mv any, errors *[]error) ([]*User, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\tusers = []*User{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv = unwrapValue(mv, <)\n\t// Make sure we have an array\n\tuv, ok := mv.([]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected users field to be an array, got %v\", mv)}\n\t}\n\tfor _, u := range uv {\n\t\ttk, u = unwrapValue(u, <)\n\t\t// Check its a map/struct\n\t\tum, ok := u.(map[string]any)\n\t\tif !ok {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected user entry to be a map/struct, got %v\", u)}\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t\tuser := &User{}\n\t\tfor k, v := range um {\n\t\t\ttk, v = unwrapValue(v, <)\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"user\", \"username\":\n\t\t\t\tuser.Username = v.(string)\n\t\t\tcase \"pass\", \"password\":\n\t\t\t\tuser.Password = v.(string)\n\t\t\tcase \"account\":\n\t\t\t\t// We really want to save just the account name here, but\n\t\t\t\t// the User object is *Account. So we create an account object\n\t\t\t\t// but it won't be registered anywhere. The server will just\n\t\t\t\t// use opts.LeafNode.Users[].Account.Name. Alternatively\n\t\t\t\t// we need to create internal objects to store u/p and account\n\t\t\t\t// name and have a server structure to hold that.\n\t\t\t\tuser.Account = NewAccount(v.(string))\n\t\t\tcase \"proxy_required\":\n\t\t\t\tuser.ProxyRequired = v.(bool)\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tusers = append(users, user)\n\t}\n\treturn users, nil\n}\n\nfunc parseRemoteLeafNodes(v any, errors *[]error, warnings *[]error) ([]*RemoteLeafOpts, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\ttk, v := unwrapValue(v, <)\n\tra, ok := v.([]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected remotes field to be an array, got %T\", v)}\n\t}\n\tremotes := make([]*RemoteLeafOpts, 0, len(ra))\n\tfor _, r := range ra {\n\t\ttk, r = unwrapValue(r, <)\n\t\t// Check its a map/struct\n\t\trm, ok := r.(map[string]any)\n\t\tif !ok {\n\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"Expected remote leafnode entry to be a map/struct, got %v\", r)})\n\t\t\tcontinue\n\t\t}\n\t\tremote := &RemoteLeafOpts{}\n\t\tvar proxyToken token\n\t\tfor k, v := range rm {\n\t\t\ttk, v = unwrapValue(v, <)\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"no_randomize\", \"dont_randomize\":\n\t\t\t\tremote.NoRandomize = v.(bool)\n\t\t\tcase \"url\", \"urls\":\n\t\t\t\tswitch v := v.(type) {\n\t\t\t\tcase []any, []string:\n\t\t\t\t\turls, errs := parseURLs(v.([]any), \"leafnode\", warnings)\n\t\t\t\t\tif errs != nil {\n\t\t\t\t\t\t*errors = append(*errors, errs...)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tremote.URLs = urls\n\t\t\t\tcase string:\n\t\t\t\t\turl, err := parseURL(v, \"leafnode\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tremote.URLs = append(remote.URLs, url)\n\t\t\t\tdefault:\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"Expected remote leafnode url to be an array or string, got %v\", v)})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tcase \"account\", \"local\":\n\t\t\t\tremote.LocalAccount = v.(string)\n\t\t\tcase \"creds\", \"credentials\":\n\t\t\t\tp, err := expandPath(v.(string))\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Can't have both creds and nkey\n\t\t\t\tif remote.Nkey != _EMPTY_ {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"Remote leafnode can not have both creds and nkey defined\"})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tremote.Credentials = p\n\t\t\tcase \"nkey\", \"seed\":\n\t\t\t\tnk := v.(string)\n\t\t\t\tif pb, _, err := nkeys.DecodeSeed([]byte(nk)); err != nil || pb != nkeys.PrefixByteUser {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Remote leafnode nkey is not a valid seed: %q\", v)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif remote.Credentials != _EMPTY_ {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, \"Remote leafnode can not have both creds and nkey defined\"})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tremote.Nkey = nk\n\t\t\tcase \"tls\":\n\t\t\t\ttc, err := parseTLS(tk, true)\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif remote.TLSConfig, err = GenTLSConfig(tc); err != nil {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// If ca_file is defined, GenTLSConfig() sets TLSConfig.ClientCAs.\n\t\t\t\t// Set RootCAs since this tls.Config is used when soliciting\n\t\t\t\t// a connection (therefore behaves as a client).\n\t\t\t\tremote.TLSConfig.RootCAs = remote.TLSConfig.ClientCAs\n\t\t\t\tif tc.Timeout > 0 {\n\t\t\t\t\tremote.TLSTimeout = tc.Timeout\n\t\t\t\t} else {\n\t\t\t\t\tremote.TLSTimeout = float64(DEFAULT_LEAF_TLS_TIMEOUT) / float64(time.Second)\n\t\t\t\t}\n\t\t\t\tremote.TLSHandshakeFirst = tc.HandshakeFirst\n\t\t\t\tremote.tlsConfigOpts = tc\n\t\t\tcase \"hub\":\n\t\t\t\tremote.Hub = v.(bool)\n\t\t\tcase \"deny_imports\", \"deny_import\":\n\t\t\t\tsubjects, err := parsePermSubjects(tk, errors)\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tremote.DenyImports = subjects\n\t\t\tcase \"deny_exports\", \"deny_export\":\n\t\t\t\tsubjects, err := parsePermSubjects(tk, errors)\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tremote.DenyExports = subjects\n\t\t\tcase \"ws_compress\", \"ws_compression\", \"websocket_compress\", \"websocket_compression\":\n\t\t\t\tremote.Websocket.Compression = v.(bool)\n\t\t\tcase \"ws_no_masking\", \"websocket_no_masking\":\n\t\t\t\tremote.Websocket.NoMasking = v.(bool)\n\t\t\tcase \"jetstream_cluster_migrate\", \"js_cluster_migrate\":\n\t\t\t\tvar lt token\n\n\t\t\t\ttk, v := unwrapValue(v, <)\n\t\t\t\tswitch vv := v.(type) {\n\t\t\t\tcase bool:\n\t\t\t\t\tremote.JetStreamClusterMigrate = vv\n\t\t\t\tcase map[string]any:\n\t\t\t\t\tremote.JetStreamClusterMigrate = true\n\t\t\t\t\tmigrateConfig, ok := v.(map[string]any)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tval, ok := migrateConfig[\"leader_migrate_delay\"]\n\t\t\t\t\ttk, delay := unwrapValue(val, &tk)\n\t\t\t\t\tif ok {\n\t\t\t\t\t\tremote.JetStreamClusterMigrateDelay = parseDuration(\"leader_migrate_delay\", tk, delay, errors, warnings)\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"Expected boolean or map for jetstream_cluster_migrate, got %T\", v)})\n\t\t\t\t}\n\t\t\tcase \"isolate_leafnode_interest\", \"isolate\":\n\t\t\t\tremote.LocalIsolation = v.(bool)\n\t\t\tcase \"request_isolation\":\n\t\t\t\tremote.RequestIsolation = v.(bool)\n\t\t\tcase \"compression\":\n\t\t\t\tif err := parseCompression(&remote.Compression, CompressionS2Auto, tk, k, v); err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tcase \"first_info_timeout\":\n\t\t\t\tremote.FirstInfoTimeout = parseDuration(k, tk, v, errors, warnings)\n\t\t\tcase \"disabled\":\n\t\t\t\tremote.Disabled = v.(bool)\n\t\t\tcase \"proxy\":\n\t\t\t\tproxyMap, ok := v.(map[string]any)\n\t\t\t\tif !ok {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"Expected proxy to be a map, got %T\", v)})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Capture the token for the \"proxy\" field itself, before the map iteration\n\t\t\t\tproxyToken = tk\n\t\t\t\tfor pk, pv := range proxyMap {\n\t\t\t\t\ttk, pv = unwrapValue(pv, <)\n\t\t\t\t\tswitch strings.ToLower(pk) {\n\t\t\t\t\tcase \"url\":\n\t\t\t\t\t\tremote.Proxy.URL = pv.(string)\n\t\t\t\t\tcase \"username\":\n\t\t\t\t\t\tremote.Proxy.Username = pv.(string)\n\t\t\t\t\tcase \"password\":\n\t\t\t\t\t\tremote.Proxy.Password = pv.(string)\n\t\t\t\t\tcase \"timeout\":\n\t\t\t\t\t\tremote.Proxy.Timeout = parseDuration(\"proxy timeout\", tk, pv, errors, warnings)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\t\t\tfield: pk,\n\t\t\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Use the saved proxy token for validation errors, not the last field token\n\t\tif warns, err := validateLeafNodeProxyOptions(remote); err != nil {\n\t\t\t*errors = append(*errors, &configErr{proxyToken, err.Error()})\n\t\t\tcontinue\n\t\t} else {\n\t\t\t// Add any warnings about proxy configuration\n\t\t\tfor _, warn := range warns {\n\t\t\t\t*warnings = append(*warnings, &configErr{proxyToken, warn})\n\t\t\t}\n\t\t}\n\t\tremotes = append(remotes, remote)\n\t}\n\treturn remotes, nil\n}\n\n// Parse TLS and returns a TLSConfig and TLSTimeout.\n// Used by cluster and gateway parsing.\nfunc getTLSConfig(tk token) (*tls.Config, *TLSConfigOpts, error) {\n\ttc, err := parseTLS(tk, false)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tconfig, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\terr := &configErr{tk, err.Error()}\n\t\treturn nil, nil, err\n\t}\n\t// For clusters/gateways, we will force strict verification. We also act\n\t// as both client and server, so will mirror the rootCA to the\n\t// clientCA pool.\n\tconfig.ClientAuth = tls.RequireAndVerifyClientCert\n\tconfig.RootCAs = config.ClientCAs\n\treturn config, tc, nil\n}\n\nfunc parseGateways(v any, errors *[]error, warnings *[]error) ([]*RemoteGatewayOpts, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\t// Make sure we have an array\n\tga, ok := v.([]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected gateways field to be an array, got %T\", v)}\n\t}\n\tgateways := []*RemoteGatewayOpts{}\n\tfor _, g := range ga {\n\t\ttk, g = unwrapValue(g, <)\n\t\t// Check its a map/struct\n\t\tgm, ok := g.(map[string]any)\n\t\tif !ok {\n\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"Expected gateway entry to be a map/struct, got %v\", g)})\n\t\t\tcontinue\n\t\t}\n\t\tgateway := &RemoteGatewayOpts{}\n\t\tfor k, v := range gm {\n\t\t\ttk, v = unwrapValue(v, <)\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"name\":\n\t\t\t\tgateway.Name = v.(string)\n\t\t\tcase \"tls\":\n\t\t\t\ttls, tlsopts, err := getTLSConfig(tk)\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tgateway.TLSConfig = tls\n\t\t\t\tgateway.TLSTimeout = tlsopts.Timeout\n\t\t\t\tgateway.tlsConfigOpts = tlsopts\n\t\t\tcase \"url\":\n\t\t\t\turl, err := parseURL(v.(string), \"gateway\")\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tgateway.URLs = append(gateway.URLs, url)\n\t\t\tcase \"urls\":\n\t\t\t\turls, errs := parseURLs(v.([]any), \"gateway\", warnings)\n\t\t\t\tif errs != nil {\n\t\t\t\t\t*errors = append(*errors, errs...)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tgateway.URLs = urls\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tgateways = append(gateways, gateway)\n\t}\n\treturn gateways, nil\n}\n\n// Sets cluster's permissions based on given pub/sub permissions,\n// doing the appropriate translation.\nfunc setClusterPermissions(opts *ClusterOpts, perms *Permissions) {\n\t// Import is whether or not we will send a SUB for interest to the other side.\n\t// Export is whether or not we will accept a SUB from the remote for a given subject.\n\t// Both only effect interest registration.\n\t// The parsing sets Import into Publish and Export into Subscribe, convert\n\t// accordingly.\n\topts.Permissions = &RoutePermissions{\n\t\tImport: perms.Publish,\n\t\tExport: perms.Subscribe,\n\t}\n}\n\n// Temp structures to hold account import and export defintions since they need\n// to be processed after being parsed.\ntype export struct {\n\tacc *Account\n\tsub string\n\taccs []string\n\trt ServiceRespType\n\tlat *serviceLatency\n\trthr time.Duration\n\ttPos uint\n\tatrc bool // allow_trace\n}\n\ntype importStream struct {\n\tacc *Account\n\tan string\n\tsub string\n\tto string\n\tpre string\n\tatrc bool // allow_trace\n}\n\ntype importService struct {\n\tacc *Account\n\tan string\n\tsub string\n\tto string\n\tshare bool\n}\n\n// Checks if an account name is reserved.\nfunc isReservedAccount(name string) bool {\n\treturn name == globalAccountName\n}\n\nfunc parseAccountMapDest(v any, tk token, errors *[]error) (*MapDest, *configErr) {\n\t// These should be maps.\n\tmv, ok := v.(map[string]any)\n\tif !ok {\n\t\terr := &configErr{tk, \"Expected an entry for the mapping destination\"}\n\t\t*errors = append(*errors, err)\n\t\treturn nil, err\n\t}\n\n\tmdest := &MapDest{}\n\tvar lt token\n\tvar sw bool\n\n\tfor k, v := range mv {\n\t\ttk, dmv := unwrapValue(v, <)\n\t\tswitch strings.ToLower(k) {\n\t\tcase \"dest\", \"destination\":\n\t\t\tmdest.Subject = dmv.(string)\n\t\tcase \"weight\":\n\t\t\tswitch vv := dmv.(type) {\n\t\t\tcase string:\n\t\t\t\tws := vv\n\t\t\t\tws = strings.TrimSuffix(ws, \"%\")\n\t\t\t\tweight, err := strconv.Atoi(ws)\n\t\t\t\tif err != nil {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Invalid weight %q for mapping destination\", ws)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif weight > 100 || weight < 0 {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Invalid weight %d for mapping destination\", weight)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tmdest.Weight = uint8(weight)\n\t\t\t\tsw = true\n\t\t\tcase int64:\n\t\t\t\tweight := vv\n\t\t\t\tif weight > 100 || weight < 0 {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Invalid weight %d for mapping destination\", weight)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tmdest.Weight = uint8(weight)\n\t\t\t\tsw = true\n\t\t\tdefault:\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown entry type for weight of %v\\n\", vv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\tcase \"cluster\":\n\t\t\tmdest.Cluster = dmv.(string)\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field %q for mapping destination\", k)}\n\t\t\t*errors = append(*errors, err)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif !sw {\n\t\terr := &configErr{tk, fmt.Sprintf(\"Missing weight for mapping destination %q\", mdest.Subject)}\n\t\t*errors = append(*errors, err)\n\t\treturn nil, err\n\t}\n\n\treturn mdest, nil\n}\n\n// parseAccountMappings is called to parse account mappings.\nfunc parseAccountMappings(v any, acc *Account, errors *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tam := v.(map[string]any)\n\tfor subj, mv := range am {\n\t\tif !IsValidSubject(subj) {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"Subject %q is not a valid subject\", subj)}\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t\ttk, v := unwrapValue(mv, <)\n\n\t\tswitch vv := v.(type) {\n\t\tcase string:\n\t\t\tif err := acc.AddMapping(subj, v.(string)); err != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Error adding mapping for %q to %q : %v\", subj, v.(string), err)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase []any:\n\t\t\tvar mappings []*MapDest\n\t\t\tfor _, mv := range v.([]any) {\n\t\t\t\ttk, amv := unwrapValue(mv, <)\n\t\t\t\tmdest, err := parseAccountMapDest(amv, tk, errors)\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tmappings = append(mappings, mdest)\n\t\t\t}\n\n\t\t\t// Now add them in..\n\t\t\tif err := acc.AddWeightedMappings(subj, mappings...); err != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Error adding mapping for %q : %v\", subj, err)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase any:\n\t\t\ttk, amv := unwrapValue(mv, <)\n\t\t\tmdest, err := parseAccountMapDest(amv, tk, errors)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Now add it in..\n\t\t\tif err := acc.AddWeightedMappings(subj, mdest); err != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Error adding mapping for %q : %v\", subj, err)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tdefault:\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown type %T for mapping destination\", vv)}\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// parseAccountLimits is called to parse account limits in a server config.\nfunc parseAccountLimits(mv any, acc *Account, errors *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(mv, <)\n\tam, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected account limits to be a map/struct, got %+v\", v)}\n\t}\n\n\tfor k, v := range am {\n\t\ttk, mv = unwrapValue(v, <)\n\t\tswitch strings.ToLower(k) {\n\t\tcase \"max_connections\", \"max_conn\":\n\t\t\tacc.mconns = int32(mv.(int64))\n\t\tcase \"max_subscriptions\", \"max_subs\":\n\t\t\tacc.msubs = int32(mv.(int64))\n\t\tcase \"max_payload\", \"max_pay\":\n\t\t\tacc.mpay = int32(mv.(int64))\n\t\tcase \"max_leafnodes\", \"max_leafs\":\n\t\t\tacc.mleafs = int32(mv.(int64))\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field %q parsing account limits\", k)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc parseAccountMsgTrace(mv any, topKey string, acc *Account) error {\n\tprocessDest := func(tk token, k string, v any) error {\n\t\ttd, ok := v.(string)\n\t\tif !ok {\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Field %q should be a string, got %T\", k, v)}\n\t\t}\n\t\tif !IsValidPublishSubject(td) {\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Trace destination %q is not valid\", td)}\n\t\t}\n\t\tacc.traceDest = td\n\t\treturn nil\n\t}\n\tprocessSampling := func(tk token, n int) error {\n\t\tif n <= 0 || n > 100 {\n\t\t\treturn &configErr{tk, fmt.Sprintf(\"Ttrace destination sampling value %d is invalid, needs to be [1..100]\", n)}\n\t\t}\n\t\tacc.traceDestSampling = n\n\t\treturn nil\n\t}\n\n\tvar lt token\n\ttk, v := unwrapValue(mv, <)\n\tswitch vv := v.(type) {\n\tcase string:\n\t\treturn processDest(tk, topKey, v)\n\tcase map[string]any:\n\t\tfor k, v := range vv {\n\t\t\ttk, v := unwrapValue(v, <)\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"dest\":\n\t\t\t\tif err := processDest(tk, k, v); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase \"sampling\":\n\t\t\t\tswitch vv := v.(type) {\n\t\t\t\tcase int64:\n\t\t\t\t\tif err := processSampling(tk, int(vv)); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\tcase string:\n\t\t\t\t\ts := strings.TrimSuffix(vv, \"%\")\n\t\t\t\t\tn, err := strconv.Atoi(s)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Invalid trace destination sampling value %q\", vv)}\n\t\t\t\t\t}\n\t\t\t\t\tif err := processSampling(tk, n); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Trace destination sampling field %q should be an integer or a percentage, got %T\", k, v)}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\treturn &configErr{tk, fmt.Sprintf(\"Unknown field %q parsing account message trace map/struct %q\", k, topKey)}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tdefault:\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected account message trace %q to be a string or a map/struct, got %T\", topKey, v)}\n\t}\n\treturn nil\n}\n\n// parseAccounts will parse the different accounts syntax.\nfunc parseAccounts(v any, opts *Options, errors *[]error, warnings *[]error) error {\n\tvar (\n\t\timportStreams []*importStream\n\t\timportServices []*importService\n\t\texportStreams []*export\n\t\texportServices []*export\n\t\tlt token\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tswitch vv := v.(type) {\n\t// Simple array of account names.\n\tcase []any, []string:\n\t\tm := make(map[string]struct{}, len(v.([]any)))\n\t\tfor _, n := range v.([]any) {\n\t\t\ttk, name := unwrapValue(n, <)\n\t\t\tns := name.(string)\n\t\t\t// Check for reserved names.\n\t\t\tif isReservedAccount(ns) {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"%q is a Reserved Account\", ns)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif _, ok := m[ns]; ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Duplicate Account Entry: %s\", ns)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\topts.Accounts = append(opts.Accounts, NewAccount(ns))\n\t\t\tm[ns] = struct{}{}\n\t\t}\n\t// More common map entry\n\tcase map[string]any:\n\t\t// Track users across accounts, must be unique across\n\t\t// accounts and nkeys vs users.\n\t\t// We also want to check for users that may have been added in\n\t\t// parseAuthorization{} if that happened first.\n\t\tuorn := setupUsersAndNKeysDuplicateCheckMap(opts)\n\n\t\tfor aname, mv := range vv {\n\t\t\ttk, amv := unwrapValue(mv, <)\n\n\t\t\t// Skip referenced config vars within the account block.\n\t\t\tif tk.IsUsedVariable() {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// These should be maps.\n\t\t\tmv, ok := amv.(map[string]any)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, \"Expected map entries for accounts\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif isReservedAccount(aname) {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"%q is a Reserved Account\", aname)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar (\n\t\t\t\tusers []*User\n\t\t\t\tnkeyUsr []*NkeyUser\n\t\t\t\tusersTk token\n\t\t\t)\n\t\t\tacc := NewAccount(aname)\n\t\t\topts.Accounts = append(opts.Accounts, acc)\n\n\t\t\tfor k, v := range mv {\n\t\t\t\ttk, mv := unwrapValue(v, <)\n\t\t\t\tswitch strings.ToLower(k) {\n\t\t\t\tcase \"nkey\":\n\t\t\t\t\tnk, ok := mv.(string)\n\t\t\t\t\tif !ok || !nkeys.IsValidPublicAccountKey(nk) {\n\t\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Not a valid public nkey for an account: %q\", mv)}\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tacc.Nkey = nk\n\t\t\t\tcase \"imports\":\n\t\t\t\t\tstreams, services, err := parseAccountImports(tk, acc, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\timportStreams = append(importStreams, streams...)\n\t\t\t\t\timportServices = append(importServices, services...)\n\t\t\t\tcase \"exports\":\n\t\t\t\t\tstreams, services, err := parseAccountExports(tk, acc, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\texportStreams = append(exportStreams, streams...)\n\t\t\t\t\texportServices = append(exportServices, services...)\n\t\t\t\tcase \"jetstream\":\n\t\t\t\t\terr := parseJetStreamForAccount(mv, acc, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\tcase \"users\":\n\t\t\t\t\tvar err error\n\t\t\t\t\tusersTk = tk\n\t\t\t\t\tnkeyUsr, users, err = parseUsers(mv, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\tcase \"default_permissions\":\n\t\t\t\t\tpermissions, err := parseUserPermissions(tk, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tacc.defaultPerms = permissions\n\t\t\t\tcase \"mappings\", \"maps\":\n\t\t\t\t\terr := parseAccountMappings(tk, acc, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\tcase \"limits\":\n\t\t\t\t\terr := parseAccountLimits(tk, acc, errors)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\tcase \"msg_trace\", \"trace_dest\":\n\t\t\t\t\tif err := parseAccountMsgTrace(tk, k, acc); err != nil {\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// If trace destination is set but no sampling, set it to 100%.\n\t\t\t\t\tif acc.traceDest != _EMPTY_ && acc.traceDestSampling == 0 {\n\t\t\t\t\t\tacc.traceDestSampling = 100\n\t\t\t\t\t} else if acc.traceDestSampling > 0 && acc.traceDest == _EMPTY_ {\n\t\t\t\t\t\t// If no trace destination is provided, no trace would be\n\t\t\t\t\t\t// triggered, so if the user set a sampling value expecting\n\t\t\t\t\t\t// something to happen, want and set the value to 0 for good\n\t\t\t\t\t\t// measure.\n\t\t\t\t\t\t*warnings = append(*warnings,\n\t\t\t\t\t\t\t&configErr{tk, \"Trace destination sampling ignored since no destination was set\"})\n\t\t\t\t\t\tacc.traceDestSampling = 0\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t}\n\t\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Report error if there is an authorization{} block\n\t\t\t// with u/p or token and any user defined in accounts{}\n\t\t\tif len(nkeyUsr) > 0 || len(users) > 0 {\n\t\t\t\tif opts.Username != _EMPTY_ {\n\t\t\t\t\terr := &configErr{usersTk, \"Can not have a single user/pass and accounts\"}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif opts.Authorization != _EMPTY_ {\n\t\t\t\t\terr := &configErr{usersTk, \"Can not have a token and accounts\"}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tapplyDefaultPermissions(users, nkeyUsr, acc.defaultPerms)\n\t\t\tfor _, u := range nkeyUsr {\n\t\t\t\tif _, ok := uorn[u.Nkey]; ok {\n\t\t\t\t\terr := &configErr{usersTk, fmt.Sprintf(\"Duplicate nkey %q detected\", u.Nkey)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tuorn[u.Nkey] = struct{}{}\n\t\t\t\tu.Account = acc\n\t\t\t}\n\t\t\topts.Nkeys = append(opts.Nkeys, nkeyUsr...)\n\t\t\tfor _, u := range users {\n\t\t\t\tif _, ok := uorn[u.Username]; ok {\n\t\t\t\t\terr := &configErr{usersTk, fmt.Sprintf(\"Duplicate user %q detected\", u.Username)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tuorn[u.Username] = struct{}{}\n\t\t\t\tu.Account = acc\n\t\t\t}\n\t\t\topts.Users = append(opts.Users, users...)\n\t\t}\n\t}\n\tlt = tk\n\t// Bail already if there are previous errors.\n\tif len(*errors) > 0 {\n\t\treturn nil\n\t}\n\n\t// Parse Imports and Exports here after all accounts defined.\n\t// Do exports first since they need to be defined for imports to succeed\n\t// since we do permissions checks.\n\n\t// Create a lookup map for accounts lookups.\n\tam := make(map[string]*Account, len(opts.Accounts))\n\tfor _, a := range opts.Accounts {\n\t\tam[a.Name] = a\n\t}\n\t// Do stream exports\n\tfor _, stream := range exportStreams {\n\t\t// Make array of accounts if applicable.\n\t\tvar accounts []*Account\n\t\tfor _, an := range stream.accs {\n\t\t\tta := am[an]\n\t\t\tif ta == nil {\n\t\t\t\tmsg := fmt.Sprintf(\"%q account not defined for stream export\", an)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\taccounts = append(accounts, ta)\n\t\t}\n\t\tif err := stream.acc.addStreamExportWithAccountPos(stream.sub, accounts, stream.tPos); err != nil {\n\t\t\tmsg := fmt.Sprintf(\"Error adding stream export %q: %v\", stream.sub, err)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\t}\n\tfor _, service := range exportServices {\n\t\t// Make array of accounts if applicable.\n\t\tvar accounts []*Account\n\t\tfor _, an := range service.accs {\n\t\t\tta := am[an]\n\t\t\tif ta == nil {\n\t\t\t\tmsg := fmt.Sprintf(\"%q account not defined for service export\", an)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\taccounts = append(accounts, ta)\n\t\t}\n\t\tif err := service.acc.addServiceExportWithResponseAndAccountPos(service.sub, service.rt, accounts, service.tPos); err != nil {\n\t\t\tmsg := fmt.Sprintf(\"Error adding service export %q: %v\", service.sub, err)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\n\t\tif service.rthr != 0 {\n\t\t\t// Response threshold was set in options.\n\t\t\tif err := service.acc.SetServiceExportResponseThreshold(service.sub, service.rthr); err != nil {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding service export response threshold for %q: %v\", service.sub, err)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif service.lat != nil {\n\t\t\t// System accounts are on be default so just make sure we have not opted out..\n\t\t\tif opts.NoSystemAccount {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding service latency sampling for %q: %v\", service.sub, ErrNoSysAccount.Error())\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif err := service.acc.TrackServiceExportWithSampling(service.sub, service.lat.subject, int(service.lat.sampling)); err != nil {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding service latency sampling for %q on subject %q: %v\", service.sub, service.lat.subject, err)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif service.atrc {\n\t\t\tif err := service.acc.SetServiceExportAllowTrace(service.sub, true); err != nil {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding allow_trace for %q: %v\", service.sub, err)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\tfor _, stream := range importStreams {\n\t\tta := am[stream.an]\n\t\tif ta == nil {\n\t\t\tmsg := fmt.Sprintf(\"%q account not defined for stream import\", stream.an)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\t\tif stream.pre != _EMPTY_ {\n\t\t\tif err := stream.acc.addStreamImportWithClaim(ta, stream.sub, stream.pre, stream.atrc, nil); err != nil {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding stream import %q: %v\", stream.sub, err)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else {\n\t\t\tif err := stream.acc.addMappedStreamImportWithClaim(ta, stream.sub, stream.to, stream.atrc, nil); err != nil {\n\t\t\t\tmsg := fmt.Sprintf(\"Error adding stream import %q: %v\", stream.sub, err)\n\t\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\tfor _, service := range importServices {\n\t\tta := am[service.an]\n\t\tif ta == nil {\n\t\t\tmsg := fmt.Sprintf(\"%q account not defined for service import\", service.an)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\t\tif service.to == _EMPTY_ {\n\t\t\tservice.to = service.sub\n\t\t}\n\t\tif err := service.acc.AddServiceImport(ta, service.to, service.sub); err != nil {\n\t\t\tmsg := fmt.Sprintf(\"Error adding service import %q: %v\", service.sub, err)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\t\tif err := service.acc.SetServiceImportSharing(ta, service.sub, service.share); err != nil {\n\t\t\tmsg := fmt.Sprintf(\"Error setting service import sharing %q: %v\", service.sub, err)\n\t\t\t*errors = append(*errors, &configErr{tk, msg})\n\t\t\tcontinue\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Parse the account exports\nfunc parseAccountExports(v any, acc *Account, errors *[]error) ([]*export, []*export, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\t// This should be an array of objects/maps.\n\ttk, v := unwrapValue(v, <)\n\tims, ok := v.([]any)\n\tif !ok {\n\t\treturn nil, nil, &configErr{tk, fmt.Sprintf(\"Exports should be an array, got %T\", v)}\n\t}\n\n\tvar services []*export\n\tvar streams []*export\n\n\tfor _, v := range ims {\n\t\t// Should have stream or service\n\t\tstream, service, err := parseExportStreamOrService(v, errors)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t\tif service != nil {\n\t\t\tservice.acc = acc\n\t\t\tservices = append(services, service)\n\t\t}\n\t\tif stream != nil {\n\t\t\tstream.acc = acc\n\t\t\tstreams = append(streams, stream)\n\t\t}\n\t}\n\treturn streams, services, nil\n}\n\n// Parse the account imports\nfunc parseAccountImports(v any, acc *Account, errors *[]error) ([]*importStream, []*importService, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\t// This should be an array of objects/maps.\n\ttk, v := unwrapValue(v, <)\n\tims, ok := v.([]any)\n\tif !ok {\n\t\treturn nil, nil, &configErr{tk, fmt.Sprintf(\"Imports should be an array, got %T\", v)}\n\t}\n\n\tvar services []*importService\n\tvar streams []*importStream\n\tsvcSubjects := map[string][]*importService{}\n\nIMS_LOOP:\n\tfor _, v := range ims {\n\t\t// Should have stream or service\n\t\tstream, service, err := parseImportStreamOrService(v, errors)\n\t\tif err != nil {\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t\tif service != nil {\n\t\t\tsisPerSubj := svcSubjects[service.to]\n\t\t\tfor _, dup := range sisPerSubj {\n\t\t\t\tif dup.an == service.an {\n\t\t\t\t\ttk, _ := unwrapValue(v, <)\n\t\t\t\t\terr := &configErr{tk,\n\t\t\t\t\t\tfmt.Sprintf(\"Duplicate service import subject %q, previously used in import for account %q, subject %q\",\n\t\t\t\t\t\t\tservice.to, dup.an, dup.sub)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue IMS_LOOP\n\t\t\t\t}\n\t\t\t}\n\t\t\tservice.acc = acc\n\t\t\tsisPerSubj = append(sisPerSubj, service)\n\t\t\tsvcSubjects[service.to] = sisPerSubj\n\t\t\tservices = append(services, service)\n\t\t}\n\t\tif stream != nil {\n\t\t\tstream.acc = acc\n\t\t\tstreams = append(streams, stream)\n\t\t}\n\t}\n\treturn streams, services, nil\n}\n\n// Helper to parse an embedded account description for imported services or streams.\nfunc parseAccount(v map[string]any, errors *[]error) (string, string, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\tvar accountName, subject string\n\tfor mk, mv := range v {\n\t\ttk, mv := unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"account\":\n\t\t\taccountName = mv.(string)\n\t\tcase \"subject\":\n\t\t\tsubject = mv.(string)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn accountName, subject, nil\n}\n\n// Parse an export stream or service.\n// e.g.\n// {stream: \"public.>\"} # No accounts means public.\n// {stream: \"synadia.private.>\", accounts: [cncf, natsio]}\n// {service: \"pub.request\"} # No accounts means public.\n// {service: \"pub.special.request\", accounts: [nats.io]}\nfunc parseExportStreamOrService(v any, errors *[]error) (*export, *export, error) {\n\tvar (\n\t\tcurStream *export\n\t\tcurService *export\n\t\taccounts []string\n\t\trt ServiceRespType\n\t\trtSeen bool\n\t\trtToken token\n\t\tlat *serviceLatency\n\t\tthreshSeen bool\n\t\tthresh time.Duration\n\t\tlatToken token\n\t\tlt token\n\t\taccTokPos uint\n\t\tatrc bool\n\t\tatrcSeen bool\n\t\tatrcToken token\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tvv, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn nil, nil, &configErr{tk, fmt.Sprintf(\"Export Items should be a map with type entry, got %T\", v)}\n\t}\n\tfor mk, mv := range vv {\n\t\ttk, mv := unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"stream\":\n\t\t\tif curService != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Detected stream %q but already saw a service\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif rtToken != nil {\n\t\t\t\terr := &configErr{rtToken, \"Detected response directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif latToken != nil {\n\t\t\t\terr := &configErr{latToken, \"Detected latency directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif atrcToken != nil {\n\t\t\t\terr := &configErr{atrcToken, \"Detected allow_trace directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmvs, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected stream name to be string, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurStream = &export{sub: mvs}\n\t\t\tif accounts != nil {\n\t\t\t\tcurStream.accs = accounts\n\t\t\t}\n\t\tcase \"service\":\n\t\t\tif curStream != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Detected service %q but already saw a stream\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmvs, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected service name to be string, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurService = &export{sub: mvs}\n\t\t\tif accounts != nil {\n\t\t\t\tcurService.accs = accounts\n\t\t\t}\n\t\t\tif rtSeen {\n\t\t\t\tcurService.rt = rt\n\t\t\t}\n\t\t\tif lat != nil {\n\t\t\t\tcurService.lat = lat\n\t\t\t}\n\t\t\tif threshSeen {\n\t\t\t\tcurService.rthr = thresh\n\t\t\t}\n\t\t\tif atrcSeen {\n\t\t\t\tcurService.atrc = atrc\n\t\t\t}\n\t\tcase \"response\", \"response_type\":\n\t\t\tif rtSeen {\n\t\t\t\terr := &configErr{tk, \"Duplicate response type definition\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trtSeen = true\n\t\t\trtToken = tk\n\t\t\tmvs, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected response type to be string, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tswitch strings.ToLower(mvs) {\n\t\t\tcase \"single\", \"singleton\":\n\t\t\t\trt = Singleton\n\t\t\tcase \"stream\":\n\t\t\t\trt = Streamed\n\t\t\tcase \"chunk\", \"chunked\":\n\t\t\t\trt = Chunked\n\t\t\tdefault:\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown response type: %q\", mvs)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif curService != nil {\n\t\t\t\tcurService.rt = rt\n\t\t\t}\n\t\t\tif curStream != nil {\n\t\t\t\terr := &configErr{tk, \"Detected response directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\tcase \"threshold\", \"response_threshold\", \"response_max_time\", \"response_time\":\n\t\t\tif threshSeen {\n\t\t\t\terr := &configErr{tk, \"Duplicate response threshold detected\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tthreshSeen = true\n\t\t\tmvs, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected response threshold to be a parseable time duration, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvar err error\n\t\t\tthresh, err = time.ParseDuration(mvs)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected response threshold to be a parseable time duration, got %q\", mvs)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif curService != nil {\n\t\t\t\tcurService.rthr = thresh\n\t\t\t}\n\t\t\tif curStream != nil {\n\t\t\t\terr := &configErr{tk, \"Detected response directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\tcase \"accounts\":\n\t\t\tfor _, iv := range mv.([]any) {\n\t\t\t\t_, mv := unwrapValue(iv, <)\n\t\t\t\taccounts = append(accounts, mv.(string))\n\t\t\t}\n\t\t\tif curStream != nil {\n\t\t\t\tcurStream.accs = accounts\n\t\t\t} else if curService != nil {\n\t\t\t\tcurService.accs = accounts\n\t\t\t}\n\t\tcase \"latency\":\n\t\t\tlatToken = tk\n\t\t\tvar err error\n\t\t\tlat, err = parseServiceLatency(tk, mv)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif curStream != nil {\n\t\t\t\terr = &configErr{tk, \"Detected latency directive on non-service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif curService != nil {\n\t\t\t\tcurService.lat = lat\n\t\t\t}\n\t\tcase \"account_token_position\":\n\t\t\taccTokPos = uint(mv.(int64))\n\t\tcase \"allow_trace\":\n\t\t\tatrcSeen = true\n\t\t\tatrcToken = tk\n\t\t\tatrc = mv.(bool)\n\t\t\tif curStream != nil {\n\t\t\t\t*errors = append(*errors,\n\t\t\t\t\t&configErr{tk, \"Detected allow_trace directive on non-service\"})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif curService != nil {\n\t\t\t\tcurService.atrc = atrc\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\tif curStream != nil {\n\t\tcurStream.tPos = accTokPos\n\t}\n\tif curService != nil {\n\t\tcurService.tPos = accTokPos\n\t}\n\treturn curStream, curService, nil\n}\n\n// parseServiceLatency returns a latency config block.\nfunc parseServiceLatency(root token, v any) (l *serviceLatency, retErr error) {\n\tvar lt token\n\tdefer convertPanicToError(<, &retErr)\n\n\tif subject, ok := v.(string); ok {\n\t\treturn &serviceLatency{\n\t\t\tsubject: subject,\n\t\t\tsampling: DEFAULT_SERVICE_LATENCY_SAMPLING,\n\t\t}, nil\n\t}\n\n\tlatency, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{token: root,\n\t\t\treason: fmt.Sprintf(\"Expected latency entry to be a map/struct or string, got %T\", v)}\n\t}\n\n\tsl := serviceLatency{\n\t\tsampling: DEFAULT_SERVICE_LATENCY_SAMPLING,\n\t}\n\n\t// Read sampling value.\n\tif v, ok := latency[\"sampling\"]; ok {\n\t\ttk, v := unwrapValue(v, <)\n\t\theader := false\n\t\tvar sample int64\n\t\tswitch vv := v.(type) {\n\t\tcase int64:\n\t\t\t// Sample is an int, like 50.\n\t\t\tsample = vv\n\t\tcase string:\n\t\t\t// Sample is a string, like \"50%\".\n\t\t\tif strings.ToLower(strings.TrimSpace(vv)) == \"headers\" {\n\t\t\t\theader = true\n\t\t\t\tsample = 0\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ts := strings.TrimSuffix(vv, \"%\")\n\t\t\tn, err := strconv.Atoi(s)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, &configErr{token: tk,\n\t\t\t\t\treason: fmt.Sprintf(\"Failed to parse latency sample: %v\", err)}\n\t\t\t}\n\t\t\tsample = int64(n)\n\t\tdefault:\n\t\t\treturn nil, &configErr{token: tk,\n\t\t\t\treason: fmt.Sprintf(\"Expected latency sample to be a string or map/struct, got %T\", v)}\n\t\t}\n\t\tif !header {\n\t\t\tif sample < 1 || sample > 100 {\n\t\t\t\treturn nil, &configErr{token: tk,\n\t\t\t\t\treason: ErrBadSampling.Error()}\n\t\t\t}\n\t\t}\n\n\t\tsl.sampling = int8(sample)\n\t}\n\n\t// Read subject value.\n\tv, ok = latency[\"subject\"]\n\tif !ok {\n\t\treturn nil, &configErr{token: root,\n\t\t\treason: \"Latency subject required, but missing\"}\n\t}\n\n\ttk, v := unwrapValue(v, <)\n\tsubject, ok := v.(string)\n\tif !ok {\n\t\treturn nil, &configErr{token: tk,\n\t\t\treason: fmt.Sprintf(\"Expected latency subject to be a string, got %T\", subject)}\n\t}\n\tsl.subject = subject\n\n\treturn &sl, nil\n}\n\n// Parse an import stream or service.\n// e.g.\n// {stream: {account: \"synadia\", subject:\"public.synadia\"}, prefix: \"imports.synadia\"}\n// {stream: {account: \"synadia\", subject:\"synadia.private.*\"}}\n// {service: {account: \"synadia\", subject: \"pub.special.request\"}, to: \"synadia.request\"}\nfunc parseImportStreamOrService(v any, errors *[]error) (*importStream, *importService, error) {\n\tvar (\n\t\tcurStream *importStream\n\t\tcurService *importService\n\t\tpre, to string\n\t\tshare bool\n\t\tlt token\n\t\tatrc bool\n\t\tatrcSeen bool\n\t\tatrcToken token\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv := unwrapValue(v, <)\n\tvv, ok := mv.(map[string]any)\n\tif !ok {\n\t\treturn nil, nil, &configErr{tk, fmt.Sprintf(\"Import Items should be a map with type entry, got %T\", mv)}\n\t}\n\tfor mk, mv := range vv {\n\t\ttk, mv := unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"stream\":\n\t\t\tif curService != nil {\n\t\t\t\terr := &configErr{tk, \"Detected stream but already saw a service\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tac, ok := mv.(map[string]any)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Stream entry should be an account map, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Make sure this is a map with account and subject\n\t\t\taccountName, subject, err := parseAccount(ac, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif accountName == _EMPTY_ || subject == _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Expect an account name and a subject\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurStream = &importStream{an: accountName, sub: subject}\n\t\t\tif to != _EMPTY_ {\n\t\t\t\tcurStream.to = to\n\t\t\t}\n\t\t\tif pre != _EMPTY_ {\n\t\t\t\tcurStream.pre = pre\n\t\t\t}\n\t\t\tif atrcSeen {\n\t\t\t\tcurStream.atrc = atrc\n\t\t\t}\n\t\tcase \"service\":\n\t\t\tif curStream != nil {\n\t\t\t\terr := &configErr{tk, \"Detected service but already saw a stream\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif atrcToken != nil {\n\t\t\t\terr := &configErr{atrcToken, \"Detected allow_trace directive on a non-stream\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tac, ok := mv.(map[string]any)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Service entry should be an account map, got %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Make sure this is a map with account and subject\n\t\t\taccountName, subject, err := parseAccount(ac, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif accountName == _EMPTY_ || subject == _EMPTY_ {\n\t\t\t\terr := &configErr{tk, \"Expect an account name and a subject\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurService = &importService{an: accountName, sub: subject}\n\t\t\tif to != _EMPTY_ {\n\t\t\t\tcurService.to = to\n\t\t\t} else {\n\t\t\t\tcurService.to = subject\n\t\t\t}\n\t\t\tcurService.share = share\n\t\tcase \"prefix\":\n\t\t\tpre = mv.(string)\n\t\t\tif curStream != nil {\n\t\t\t\tcurStream.pre = pre\n\t\t\t}\n\t\tcase \"to\":\n\t\t\tto = mv.(string)\n\t\t\tif curService != nil {\n\t\t\t\tcurService.to = to\n\t\t\t}\n\t\t\tif curStream != nil {\n\t\t\t\tcurStream.to = to\n\t\t\t\tif curStream.pre != _EMPTY_ {\n\t\t\t\t\terr := &configErr{tk, \"Stream import can not have a 'prefix' and a 'to' property\"}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"share\":\n\t\t\tshare = mv.(bool)\n\t\t\tif curService != nil {\n\t\t\t\tcurService.share = share\n\t\t\t}\n\t\tcase \"allow_trace\":\n\t\t\tif curService != nil {\n\t\t\t\terr := &configErr{tk, \"Detected allow_trace directive on a non-stream\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tatrcSeen = true\n\t\t\tatrc = mv.(bool)\n\t\t\tatrcToken = tk\n\t\t\tif curStream != nil {\n\t\t\t\tcurStream.atrc = atrc\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\n\t}\n\treturn curStream, curService, nil\n}\n\n// Apply permission defaults to users/nkeyuser that don't have their own.\nfunc applyDefaultPermissions(users []*User, nkeys []*NkeyUser, defaultP *Permissions) {\n\tif defaultP == nil {\n\t\treturn\n\t}\n\tfor _, user := range users {\n\t\tif user.Permissions == nil {\n\t\t\tuser.Permissions = defaultP\n\t\t}\n\t}\n\tfor _, user := range nkeys {\n\t\tif user.Permissions == nil {\n\t\t\tuser.Permissions = defaultP\n\t\t}\n\t}\n}\n\n// Helper function to parse Authorization configs.\nfunc parseAuthorization(v any, errors, warnings *[]error) (*authorization, error) {\n\tvar (\n\t\tam map[string]any\n\t\ttk token\n\t\tlt token\n\t\tauth = &authorization{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\t_, v = unwrapValue(v, <)\n\tam = v.(map[string]any)\n\tfor mk, mv := range am {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"user\", \"username\":\n\t\t\tauth.user = mv.(string)\n\t\tcase \"pass\", \"password\":\n\t\t\tauth.pass = mv.(string)\n\t\tcase \"token\":\n\t\t\tauth.token = mv.(string)\n\t\tcase \"timeout\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing authorization config, 'timeout' %s\", err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, \"error parsing authorization config, 'timeout' wrong type\"}\n\t\t\t}\n\t\t\tif at > (60 * time.Second).Seconds() {\n\t\t\t\treason := fmt.Sprintf(\"timeout of %v (%f seconds) is high, consider keeping it under 60 seconds. possibly caused by unquoted duration; use '1m' instead of 1m, for example\", mv, at)\n\t\t\t\t*warnings = append(*warnings, &configWarningErr{field: mk, configErr: configErr{token: tk, reason: reason}})\n\t\t\t}\n\t\t\tauth.timeout = at\n\t\tcase \"users\":\n\t\t\tnkeys, users, err := parseUsers(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tauth.users = users\n\t\t\tauth.nkeys = nkeys\n\t\tcase \"default_permission\", \"default_permissions\", \"permissions\":\n\t\t\tpermissions, err := parseUserPermissions(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tauth.defaultPermissions = permissions\n\t\tcase \"auth_callout\", \"auth_hook\":\n\t\t\tif fips140.Enabled() {\n\t\t\t\t*errors = append(*errors, fmt.Errorf(\"'auth_callout' cannot be configured in FIPS-140 mode\"))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tac, err := parseAuthCallout(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tauth.callout = ac\n\t\tcase \"proxy_required\":\n\t\t\tauth.proxyRequired = mv.(bool)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tapplyDefaultPermissions(auth.users, auth.nkeys, auth.defaultPermissions)\n\t}\n\treturn auth, nil\n}\n\n// Helper function to parse multiple users array with optional permissions.\nfunc parseUsers(mv any, errors *[]error) ([]*NkeyUser, []*User, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\tkeys []*NkeyUser\n\t\tusers = []*User{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\ttk, mv = unwrapValue(mv, <)\n\n\t// Make sure we have an array\n\tuv, ok := mv.([]any)\n\tif !ok {\n\t\treturn nil, nil, &configErr{tk, fmt.Sprintf(\"Expected users field to be an array, got %v\", mv)}\n\t}\n\tfor _, u := range uv {\n\t\ttk, u = unwrapValue(u, <)\n\n\t\t// Check its a map/struct\n\t\tum, ok := u.(map[string]any)\n\t\tif !ok {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"Expected user entry to be a map/struct, got %v\", u)}\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tvar (\n\t\t\tuser = &User{}\n\t\t\tnkey = &NkeyUser{}\n\t\t\tperms *Permissions\n\t\t\terr error\n\t\t)\n\t\tfor k, v := range um {\n\t\t\t// Also needs to unwrap first\n\t\t\ttk, v = unwrapValue(v, <)\n\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"nkey\":\n\t\t\t\tnkey.Nkey = v.(string)\n\t\t\tcase \"user\", \"username\":\n\t\t\t\tuser.Username = v.(string)\n\t\t\tcase \"pass\", \"password\":\n\t\t\t\tuser.Password = v.(string)\n\t\t\tcase \"permission\", \"permissions\", \"authorization\":\n\t\t\t\tperms, err = parseUserPermissions(tk, errors)\n\t\t\t\tif err != nil {\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tcase \"allowed_connection_types\", \"connection_types\", \"clients\":\n\t\t\t\tcts := parseAllowedConnectionTypes(tk, <, v, errors)\n\t\t\t\tnkey.AllowedConnectionTypes = cts\n\t\t\t\tuser.AllowedConnectionTypes = cts\n\t\t\tcase \"proxy_required\":\n\t\t\t\tnkey.ProxyRequired = v.(bool)\n\t\t\t\tuser.ProxyRequired = v.(bool)\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Place perms if we have them.\n\t\tif perms != nil {\n\t\t\t// nkey takes precedent.\n\t\t\tif nkey.Nkey != _EMPTY_ {\n\t\t\t\tnkey.Permissions = perms\n\t\t\t} else {\n\t\t\t\tuser.Permissions = perms\n\t\t\t}\n\t\t}\n\n\t\t// Check to make sure we have at least an nkey or username defined.\n\t\tif nkey.Nkey == _EMPTY_ && user.Username == _EMPTY_ {\n\t\t\treturn nil, nil, &configErr{tk, \"User entry requires a user\"}\n\t\t} else if nkey.Nkey != _EMPTY_ {\n\t\t\t// Make sure the nkey a proper public nkey for a user..\n\t\t\tif !nkeys.IsValidPublicUserKey(nkey.Nkey) {\n\t\t\t\treturn nil, nil, &configErr{tk, \"Not a valid public nkey for a user\"}\n\t\t\t}\n\t\t\t// If we have user or password defined here that is an error.\n\t\t\tif user.Username != _EMPTY_ || user.Password != _EMPTY_ {\n\t\t\t\treturn nil, nil, &configErr{tk, \"Nkey users do not take usernames or passwords\"}\n\t\t\t}\n\t\t\tkeys = append(keys, nkey)\n\t\t} else {\n\t\t\tusers = append(users, user)\n\t\t}\n\t}\n\treturn keys, users, nil\n}\n\nfunc parseAllowedConnectionTypes(tk token, lt *token, mv any, errors *[]error) map[string]struct{} {\n\tcts, err := parseStringArray(\"allowed connection types\", tk, lt, mv, errors)\n\t// If error, it has already been added to the `errors` array, simply return\n\tif err != nil {\n\t\treturn nil\n\t}\n\tm, err := convertAllowedConnectionTypes(cts)\n\tif err != nil {\n\t\t*errors = append(*errors, &configErr{tk, err.Error()})\n\t}\n\treturn m\n}\n\n// Helper function to parse auth callouts.\nfunc parseAuthCallout(mv any, errors *[]error) (*AuthCallout, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\tac = &AuthCallout{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv = unwrapValue(mv, <)\n\tpm, ok := mv.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected authorization callout to be a map/struct, got %+v\", mv)}\n\t}\n\tfor k, v := range pm {\n\t\ttk, mv = unwrapValue(v, <)\n\n\t\tswitch strings.ToLower(k) {\n\t\tcase \"issuer\":\n\t\t\tac.Issuer = mv.(string)\n\t\t\tif !nkeys.IsValidPublicAccountKey(ac.Issuer) {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected callout user to be a valid public account nkey, got %q\", ac.Issuer)}\n\t\t\t}\n\t\tcase \"account\", \"acc\":\n\t\t\tac.Account = mv.(string)\n\t\tcase \"auth_users\", \"users\":\n\t\t\taua, ok := mv.([]any)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected auth_users field to be an array, got %T\", v)}\n\t\t\t}\n\t\t\tfor _, uv := range aua {\n\t\t\t\t_, uv = unwrapValue(uv, <)\n\t\t\t\tac.AuthUsers = append(ac.AuthUsers, uv.(string))\n\t\t\t}\n\t\tcase \"xkey\", \"key\":\n\t\t\tac.XKey = mv.(string)\n\t\t\tif !nkeys.IsValidPublicCurveKey(ac.XKey) {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected callout xkey to be a valid public xkey, got %q\", ac.XKey)}\n\t\t\t}\n\t\tcase \"allowed_accounts\":\n\t\t\taua, ok := mv.([]any)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected allowed accounts field to be an array, got %T\", v)}\n\t\t\t}\n\t\t\tfor _, uv := range aua {\n\t\t\t\t_, uv = unwrapValue(uv, <)\n\t\t\t\tac.AllowedAccounts = append(ac.AllowedAccounts, uv.(string))\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field %q parsing authorization callout\", k)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\t// Make sure we have all defined. All fields are required.\n\t// If no account specified, selet $G.\n\tif ac.Account == _EMPTY_ {\n\t\tac.Account = globalAccountName\n\t}\n\tif ac.Issuer == _EMPTY_ {\n\t\treturn nil, &configErr{tk, \"Authorization callouts require an issuer to be specified\"}\n\t}\n\tif len(ac.AuthUsers) == 0 {\n\t\treturn nil, &configErr{tk, \"Authorization callouts require authorized users to be specified\"}\n\t}\n\treturn ac, nil\n}\n\n// Helper function to parse user/account permissions\nfunc parseUserPermissions(mv any, errors *[]error) (*Permissions, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\tp = &Permissions{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv = unwrapValue(mv, <)\n\tpm, ok := mv.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected permissions to be a map/struct, got %+v\", mv)}\n\t}\n\tfor k, v := range pm {\n\t\ttk, mv = unwrapValue(v, <)\n\n\t\tswitch strings.ToLower(k) {\n\t\t// For routes:\n\t\t// Import is Publish\n\t\t// Export is Subscribe\n\t\tcase \"pub\", \"publish\", \"import\":\n\t\t\tperms, err := parseVariablePermissions(mv, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tp.Publish = perms\n\t\tcase \"sub\", \"subscribe\", \"export\":\n\t\t\tperms, err := parseVariablePermissions(mv, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tp.Subscribe = perms\n\t\tcase \"publish_allow_responses\", \"allow_responses\":\n\t\t\trp := &ResponsePermission{\n\t\t\t\tMaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS,\n\t\t\t\tExpires: DEFAULT_ALLOW_RESPONSE_EXPIRATION,\n\t\t\t}\n\t\t\t// Try boolean first\n\t\t\tresponses, ok := mv.(bool)\n\t\t\tif ok {\n\t\t\t\tif responses {\n\t\t\t\t\tp.Response = rp\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tp.Response = parseAllowResponses(v, errors)\n\t\t\t}\n\t\t\tif p.Response != nil {\n\t\t\t\tif p.Publish == nil {\n\t\t\t\t\tp.Publish = &SubjectPermission{}\n\t\t\t\t}\n\t\t\t\tif p.Publish.Allow == nil {\n\t\t\t\t\t// We turn off the blanket allow statement.\n\t\t\t\t\tp.Publish.Allow = []string{}\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field %q parsing permissions\", k)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn p, nil\n}\n\n// Top level parser for authorization configurations.\nfunc parseVariablePermissions(v any, errors *[]error) (*SubjectPermission, error) {\n\tswitch vv := v.(type) {\n\tcase map[string]any:\n\t\t// New style with allow and/or deny properties.\n\t\treturn parseSubjectPermission(vv, errors)\n\tdefault:\n\t\t// Old style\n\t\treturn parseOldPermissionStyle(v, errors)\n\t}\n}\n\n// Helper function to parse subject singletons and/or arrays\nfunc parsePermSubjects(v any, errors *[]error) ([]string, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\n\tvar subjects []string\n\tswitch vv := v.(type) {\n\tcase string:\n\t\tsubjects = append(subjects, vv)\n\tcase []string:\n\t\tsubjects = vv\n\tcase []any:\n\t\tfor _, i := range vv {\n\t\t\ttk, i := unwrapValue(i, <)\n\n\t\t\tsubject, ok := i.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"Subject in permissions array cannot be cast to string\"}\n\t\t\t}\n\t\t\tsubjects = append(subjects, subject)\n\t\t}\n\tdefault:\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"Expected subject permissions to be a subject, or array of subjects, got %T\", v)}\n\t}\n\tif err := checkPermSubjectArray(subjects); err != nil {\n\t\treturn nil, &configErr{tk, err.Error()}\n\t}\n\treturn subjects, nil\n}\n\n// Helper function to parse a ResponsePermission.\nfunc parseAllowResponses(v any, errors *[]error) *ResponsePermission {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\t// Check if this is a map.\n\tpm, ok := v.(map[string]any)\n\tif !ok {\n\t\terr := &configErr{tk, \"error parsing response permissions, expected a boolean or a map\"}\n\t\t*errors = append(*errors, err)\n\t\treturn nil\n\t}\n\n\trp := &ResponsePermission{\n\t\tMaxMsgs: DEFAULT_ALLOW_RESPONSE_MAX_MSGS,\n\t\tExpires: DEFAULT_ALLOW_RESPONSE_EXPIRATION,\n\t}\n\n\tfor k, v := range pm {\n\t\ttk, v = unwrapValue(v, <)\n\t\tswitch strings.ToLower(k) {\n\t\tcase \"max\", \"max_msgs\", \"max_messages\", \"max_responses\":\n\t\t\tmax := int(v.(int64))\n\t\t\t// Negative values are accepted (mean infinite), and 0\n\t\t\t// means default value (set above).\n\t\t\tif max != 0 {\n\t\t\t\trp.MaxMsgs = max\n\t\t\t}\n\t\tcase \"expires\", \"expiration\", \"ttl\":\n\t\t\twd, ok := v.(string)\n\t\t\tif ok {\n\t\t\t\tttl, err := time.ParseDuration(wd)\n\t\t\t\tif err != nil {\n\t\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing expires: %v\", err)}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\t// Negative values are accepted (mean infinite), and 0\n\t\t\t\t// means default value (set above).\n\t\t\t\tif ttl != 0 {\n\t\t\t\t\trp.Expires = ttl\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\terr := &configErr{tk, \"error parsing expires, not a duration string\"}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field %q parsing permissions\", k)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn rp\n}\n\n// Helper function to parse old style authorization configs.\nfunc parseOldPermissionStyle(v any, errors *[]error) (*SubjectPermission, error) {\n\tsubjects, err := parsePermSubjects(v, errors)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &SubjectPermission{Allow: subjects}, nil\n}\n\n// Helper function to parse new style authorization into a SubjectPermission with Allow and Deny.\nfunc parseSubjectPermission(v any, errors *[]error) (*SubjectPermission, error) {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\tm := v.(map[string]any)\n\tif len(m) == 0 {\n\t\treturn nil, nil\n\t}\n\tp := &SubjectPermission{}\n\tfor k, v := range m {\n\t\ttk, _ := unwrapValue(v, <)\n\t\tswitch strings.ToLower(k) {\n\t\tcase \"allow\":\n\t\t\tsubjects, err := parsePermSubjects(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tp.Allow = subjects\n\t\tcase \"deny\":\n\t\t\tsubjects, err := parsePermSubjects(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tp.Deny = subjects\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"Unknown field name %q parsing subject permissions, only 'allow' or 'deny' are permitted\", k)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn p, nil\n}\n\n// Helper function to validate permissions subjects.\nfunc checkPermSubjectArray(sa []string) error {\n\tfor _, s := range sa {\n\t\tif !IsValidSubject(s) {\n\t\t\t// Check here if this is a queue group qualified subject.\n\t\t\telements := strings.Fields(s)\n\t\t\tif len(elements) != 2 {\n\t\t\t\treturn fmt.Errorf(\"subject %q is not a valid subject\", s)\n\t\t\t} else if !IsValidSubject(elements[0]) {\n\t\t\t\treturn fmt.Errorf(\"subject %q is not a valid subject\", elements[0])\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// PrintTLSHelpAndDie prints TLS usage and exits.\nfunc PrintTLSHelpAndDie() {\n\tfmt.Printf(\"%s\", tlsUsage)\n\tfor k := range cipherMap {\n\t\tfmt.Printf(\" %s\\n\", k)\n\t}\n\tfmt.Printf(\"\\nAvailable curve preferences include:\\n\")\n\tfor k := range curvePreferenceMap {\n\t\tfmt.Printf(\" %s\\n\", k)\n\t}\n\tif runtime.GOOS == \"windows\" {\n\t\tfmt.Printf(\"%s\\n\", certstore.Usage)\n\t}\n\tfmt.Printf(\"%s\", certidp.OCSPPeerUsage)\n\tfmt.Printf(\"%s\", OCSPResponseCacheUsage)\n\tos.Exit(0)\n}\n\nfunc parseCipher(cipherName string) (*tls.CipherSuite, error) {\n\tcipher, exists := cipherMap[cipherName]\n\tif !exists {\n\t\treturn nil, fmt.Errorf(\"unrecognized cipher %s\", cipherName)\n\t}\n\treturn cipher, nil\n}\n\nfunc parseCurvePreferences(curveName string) (tls.CurveID, error) {\n\tcurve, exists := curvePreferenceMap[curveName]\n\tif !exists {\n\t\treturn 0, fmt.Errorf(\"unrecognized curve preference %s\", curveName)\n\t}\n\treturn curve, nil\n}\n\nfunc parseTLSVersion(v any) (uint16, error) {\n\tvar tlsVersionNumber uint16\n\tswitch v := v.(type) {\n\tcase string:\n\t\tn, err := tlsVersionFromString(v)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\ttlsVersionNumber = n\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"'min_version' wrong type: %v\", v)\n\t}\n\tif tlsVersionNumber < tls.VersionTLS12 {\n\t\treturn 0, fmt.Errorf(\"unsupported TLS version: %s\", tls.VersionName(tlsVersionNumber))\n\t}\n\treturn tlsVersionNumber, nil\n}\n\n// Helper function to parse TLS configs.\nfunc parseTLS(v any, isClientCtx bool) (t *TLSConfigOpts, retErr error) {\n\tvar (\n\t\ttlsm map[string]any\n\t\ttc = TLSConfigOpts{}\n\t\tlt token\n\t\tics []*tls.CipherSuite // Insecure ciphers found\n\t)\n\tdefer convertPanicToError(<, &retErr)\n\n\ttk, v := unwrapValue(v, <)\n\ttlsm = v.(map[string]any)\n\tfor mk, mv := range tlsm {\n\t\ttk, mv := unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"cert_file\":\n\t\t\tcertFile, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'cert_file' to be filename\"}\n\t\t\t}\n\t\t\ttc.CertFile = certFile\n\t\tcase \"key_file\":\n\t\t\tkeyFile, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'key_file' to be filename\"}\n\t\t\t}\n\t\t\ttc.KeyFile = keyFile\n\t\tcase \"ca_file\":\n\t\t\tcaFile, ok := mv.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'ca_file' to be filename\"}\n\t\t\t}\n\t\t\ttc.CaFile = caFile\n\t\tcase \"insecure\":\n\t\t\tinsecure, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'insecure' to be a boolean\"}\n\t\t\t}\n\t\t\ttc.Insecure = insecure\n\t\tcase \"verify\":\n\t\t\tverify, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'verify' to be a boolean\"}\n\t\t\t}\n\t\t\ttc.Verify = verify\n\t\tcase \"verify_and_map\":\n\t\t\tverify, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'verify_and_map' to be a boolean\"}\n\t\t\t}\n\t\t\tif verify {\n\t\t\t\ttc.Verify = verify\n\t\t\t}\n\t\t\ttc.Map = verify\n\t\tcase \"verify_cert_and_check_known_urls\":\n\t\t\tverify, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'verify_cert_and_check_known_urls' to be a boolean\"}\n\t\t\t}\n\t\t\tif verify && isClientCtx {\n\t\t\t\treturn nil, &configErr{tk, \"verify_cert_and_check_known_urls not supported in this context\"}\n\t\t\t}\n\t\t\tif verify {\n\t\t\t\ttc.Verify = verify\n\t\t\t}\n\t\t\ttc.TLSCheckKnownURLs = verify\n\t\tcase \"allow_insecure_cipher_suites\":\n\t\t\tallow, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'allow_insecure_cipher_suites' to be a boolean\"}\n\t\t\t}\n\t\t\ttc.AllowInsecureCiphers = allow\n\t\tcase \"cipher_suites\":\n\t\t\tra := mv.([]any)\n\t\t\tif len(ra) == 0 {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, 'cipher_suites' cannot be empty\"}\n\t\t\t}\n\t\t\ttc.Ciphers = make([]uint16, 0, len(ra))\n\t\t\tfor _, r := range ra {\n\t\t\t\ttk, r := unwrapValue(r, <)\n\t\t\t\tcipher, err := parseCipher(r.(string))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, err.Error()}\n\t\t\t\t}\n\t\t\t\ttc.Ciphers = append(tc.Ciphers, cipher.ID)\n\t\t\t\tif cipher.Insecure {\n\t\t\t\t\tics = append(ics, cipher)\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"curve_preferences\":\n\t\t\tra := mv.([]any)\n\t\t\tif len(ra) == 0 {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, 'curve_preferences' cannot be empty\"}\n\t\t\t}\n\t\t\ttc.CurvePreferences = make([]tls.CurveID, 0, len(ra))\n\t\t\tfor _, r := range ra {\n\t\t\t\ttk, r := unwrapValue(r, <)\n\t\t\t\tcps, err := parseCurvePreferences(r.(string))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, err.Error()}\n\t\t\t\t}\n\t\t\t\ttc.CurvePreferences = append(tc.CurvePreferences, cps)\n\t\t\t}\n\t\tcase \"timeout\":\n\t\t\tat := float64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\tcase string:\n\t\t\t\td, err := time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing tls config, 'timeout' %s\", err)}\n\t\t\t\t}\n\t\t\t\tat = d.Seconds()\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, 'timeout' wrong type\"}\n\t\t\t}\n\t\t\ttc.Timeout = at\n\t\tcase \"connection_rate_limit\":\n\t\t\tat := int64(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = mv\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, 'connection_rate_limit' wrong type\"}\n\t\t\t}\n\t\t\ttc.RateLimit = at\n\t\tcase \"pinned_certs\":\n\t\t\tra, ok := mv.([]any)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, \"error parsing tls config, expected 'pinned_certs' to be a list of hex-encoded sha256 of DER encoded SubjectPublicKeyInfo\"}\n\t\t\t}\n\t\t\tif len(ra) != 0 {\n\t\t\t\twl := PinnedCertSet{}\n\t\t\t\tre := regexp.MustCompile(\"^[A-Fa-f0-9]{64}$\")\n\t\t\t\tfor _, r := range ra {\n\t\t\t\t\ttk, r := unwrapValue(r, <)\n\t\t\t\t\tentry := strings.ToLower(r.(string))\n\t\t\t\t\tif !re.MatchString(entry) {\n\t\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing tls config, 'pinned_certs' key %s does not look like hex-encoded sha256 of DER encoded SubjectPublicKeyInfo\", entry)}\n\t\t\t\t\t}\n\t\t\t\t\twl[entry] = struct{}{}\n\t\t\t\t}\n\t\t\t\ttc.PinnedCerts = wl\n\t\t\t}\n\t\tcase \"cert_store\":\n\t\t\tcertStore, ok := mv.(string)\n\t\t\tif !ok || certStore == _EMPTY_ {\n\t\t\t\treturn nil, &configErr{tk, certstore.ErrBadCertStoreField.Error()}\n\t\t\t}\n\t\t\tcertStoreType, err := certstore.ParseCertStore(certStore)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, &configErr{tk, err.Error()}\n\t\t\t}\n\t\t\ttc.CertStore = certStoreType\n\t\tcase \"cert_match_by\":\n\t\t\tcertMatchBy, ok := mv.(string)\n\t\t\tif !ok || certMatchBy == _EMPTY_ {\n\t\t\t\treturn nil, &configErr{tk, certstore.ErrBadCertMatchByField.Error()}\n\t\t\t}\n\t\t\tcertMatchByType, err := certstore.ParseCertMatchBy(certMatchBy)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, &configErr{tk, err.Error()}\n\t\t\t}\n\t\t\ttc.CertMatchBy = certMatchByType\n\t\tcase \"cert_match\":\n\t\t\tcertMatch, ok := mv.(string)\n\t\t\tif !ok || certMatch == _EMPTY_ {\n\t\t\t\treturn nil, &configErr{tk, certstore.ErrBadCertMatchField.Error()}\n\t\t\t}\n\t\t\ttc.CertMatch = certMatch\n\t\tcase \"ca_certs_match\":\n\t\t\trv := []string{}\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase string:\n\t\t\t\trv = append(rv, mv)\n\t\t\tcase []string:\n\t\t\t\trv = append(rv, mv...)\n\t\t\tcase []any:\n\t\t\t\tfor _, t := range mv {\n\t\t\t\t\tif token, ok := t.(token); ok {\n\t\t\t\t\t\tif ts, ok := token.Value().(string); ok {\n\t\t\t\t\t\t\trv = append(rv, ts)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing ca_cert_match: unsupported type %T where string is expected\", token)}\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing ca_cert_match: unsupported type %T\", t)}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\ttc.CaCertsMatch = rv\n\t\tcase \"handshake_first\", \"first\", \"immediate\":\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase bool:\n\t\t\t\ttc.HandshakeFirst = mv\n\t\t\tcase string:\n\t\t\t\tswitch strings.ToLower(mv) {\n\t\t\t\tcase \"true\", \"on\":\n\t\t\t\t\ttc.HandshakeFirst = true\n\t\t\t\tcase \"false\", \"off\":\n\t\t\t\t\ttc.HandshakeFirst = false\n\t\t\t\tcase \"auto\", \"auto_fallback\":\n\t\t\t\t\ttc.HandshakeFirst = true\n\t\t\t\t\ttc.FallbackDelay = DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY\n\t\t\t\tdefault:\n\t\t\t\t\t// Check to see if this is a duration.\n\t\t\t\t\tif dur, err := time.ParseDuration(mv); err == nil {\n\t\t\t\t\t\ttc.HandshakeFirst = true\n\t\t\t\t\t\ttc.FallbackDelay = dur\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"field %q's value %q is invalid\", mk, mv)}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"field %q should be a boolean or a string, got %T\", mk, mv)}\n\t\t\t}\n\t\tcase \"cert_match_skip_invalid\":\n\t\t\tcertMatchSkipInvalid, ok := mv.(bool)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, certstore.ErrBadCertMatchSkipInvalidField.Error()}\n\t\t\t}\n\t\t\ttc.CertMatchSkipInvalid = certMatchSkipInvalid\n\t\tcase \"ocsp_peer\":\n\t\t\tswitch vv := mv.(type) {\n\t\t\tcase bool:\n\t\t\t\tpc := certidp.NewOCSPPeerConfig()\n\t\t\t\tif vv {\n\t\t\t\t\t// Set enabled\n\t\t\t\t\tpc.Verify = true\n\t\t\t\t\ttc.OCSPPeerConfig = pc\n\t\t\t\t} else {\n\t\t\t\t\t// Set disabled\n\t\t\t\t\tpc.Verify = false\n\t\t\t\t\ttc.OCSPPeerConfig = pc\n\t\t\t\t}\n\t\t\tcase map[string]any:\n\t\t\t\tpc, err := parseOCSPPeer(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, &configErr{tk, err.Error()}\n\t\t\t\t}\n\t\t\t\ttc.OCSPPeerConfig = pc\n\t\t\tdefault:\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing ocsp peer config: unsupported type %T\", v)}\n\t\t\t}\n\t\tcase \"certs\", \"certificates\":\n\t\t\tcerts, ok := mv.([]any)\n\t\t\tif !ok {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing certificates config: unsupported type %T\", v)}\n\t\t\t}\n\t\t\ttc.Certificates = make([]*TLSCertPairOpt, len(certs))\n\t\t\tfor i, v := range certs {\n\t\t\t\ttk, vv := unwrapValue(v, <)\n\t\t\t\tpair, ok := vv.(map[string]any)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing certificates config: unsupported type %T\", vv)}\n\t\t\t\t}\n\t\t\t\tcertPair := &TLSCertPairOpt{}\n\t\t\t\tfor k, v := range pair {\n\t\t\t\t\ttk, vv = unwrapValue(v, <)\n\t\t\t\t\tfile, ok := vv.(string)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing certificates config: unsupported type %T\", vv)}\n\t\t\t\t\t}\n\t\t\t\t\tswitch k {\n\t\t\t\t\tcase \"cert_file\":\n\t\t\t\t\t\tcertPair.CertFile = file\n\t\t\t\t\tcase \"key_file\":\n\t\t\t\t\t\tcertPair.KeyFile = file\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing tls certs config, unknown field %q\", k)}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif certPair.CertFile == _EMPTY_ || certPair.KeyFile == _EMPTY_ {\n\t\t\t\t\treturn nil, &configErr{tk, \"error parsing certificates config: both 'cert_file' and 'cert_key' options are required\"}\n\t\t\t\t}\n\t\t\t\ttc.Certificates[i] = certPair\n\t\t\t}\n\t\tcase \"min_version\":\n\t\t\tminVersion, err := parseTLSVersion(mv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing tls config: %v\", err)}\n\t\t\t}\n\t\t\ttc.MinVersion = minVersion\n\t\tdefault:\n\t\t\treturn nil, &configErr{tk, fmt.Sprintf(\"error parsing tls config, unknown field %q\", mk)}\n\t\t}\n\t}\n\tif len(tc.Certificates) > 0 && tc.CertFile != _EMPTY_ {\n\t\treturn nil, &configErr{tk, \"error parsing tls config, cannot combine 'cert_file' option with 'certs' option\"}\n\t}\n\n\t// If cipher suites were not specified then use the defaults\n\tif tc.Ciphers == nil {\n\t\ttc.Ciphers = defaultCipherSuites()\n\t}\n\n\t// If curve preferences were not specified, then use the defaults\n\tif tc.CurvePreferences == nil {\n\t\ttc.CurvePreferences = defaultCurvePreferences()\n\t}\n\n\t// If we don't allow insecure ciphers, and yet some were configured, then we\n\t// should error.\n\tif !tc.AllowInsecureCiphers && len(ics) > 0 {\n\t\tnames := make([]string, 0, len(ics))\n\t\tfor _, ic := range ics {\n\t\t\tnames = append(names, ic.Name)\n\t\t}\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"insecure cipher suites configured without 'allow_insecure_cipher_suites' option set: %s\", strings.Join(names, \", \"))}\n\t}\n\n\treturn &tc, nil\n}\n\nfunc parseSimpleAuth(v any, errors *[]error) *authorization {\n\tvar (\n\t\tam map[string]any\n\t\ttk token\n\t\tlt token\n\t\tauth = &authorization{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\t_, v = unwrapValue(v, <)\n\tam = v.(map[string]any)\n\tfor mk, mv := range am {\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"user\", \"username\":\n\t\t\tauth.user = mv.(string)\n\t\tcase \"pass\", \"password\":\n\t\t\tauth.pass = mv.(string)\n\t\tcase \"token\":\n\t\t\tauth.token = mv.(string)\n\t\tcase \"timeout\":\n\t\t\tat := float64(1)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tat = float64(mv)\n\t\t\tcase float64:\n\t\t\t\tat = mv\n\t\t\t}\n\t\t\tauth.timeout = at\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t}\n\treturn auth\n}\n\nfunc parseStringArray(fieldName string, tk token, lt *token, mv any, errors *[]error) ([]string, error) {\n\tswitch mv := mv.(type) {\n\tcase string:\n\t\treturn []string{mv}, nil\n\tcase []any:\n\t\tstrs := make([]string, 0, len(mv))\n\t\tfor _, val := range mv {\n\t\t\ttk, val = unwrapValue(val, lt)\n\t\t\tif str, ok := val.(string); ok {\n\t\t\t\tstrs = append(strs, str)\n\t\t\t} else {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing %s: unsupported type in array %T\", fieldName, val)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\treturn strs, nil\n\tdefault:\n\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing %s: unsupported type %T\", fieldName, mv)}\n\t\t*errors = append(*errors, err)\n\t\treturn nil, err\n\t}\n}\n\nfunc parseWebsocket(v any, o *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tgm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected websocket to be a map, got %T\", v)}\n\t}\n\tfor mk, mv := range gm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"listen\":\n\t\t\thp, err := parseListen(mv)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Websocket.Host = hp.host\n\t\t\to.Websocket.Port = hp.port\n\t\tcase \"port\":\n\t\t\to.Websocket.Port = int(mv.(int64))\n\t\tcase \"host\", \"net\":\n\t\t\to.Websocket.Host = mv.(string)\n\t\tcase \"advertise\":\n\t\t\to.Websocket.Advertise = mv.(string)\n\t\tcase \"no_tls\":\n\t\t\to.Websocket.NoTLS = mv.(bool)\n\t\tcase \"tls\":\n\t\t\ttc, err := parseTLS(tk, true)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif o.Websocket.TLSConfig, err = GenTLSConfig(tc); err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Websocket.TLSMap = tc.Map\n\t\t\to.Websocket.TLSPinnedCerts = tc.PinnedCerts\n\t\t\to.Websocket.tlsConfigOpts = tc\n\t\tcase \"same_origin\":\n\t\t\to.Websocket.SameOrigin = mv.(bool)\n\t\tcase \"allowed_origins\", \"allowed_origin\", \"allow_origins\", \"allow_origin\", \"origins\", \"origin\":\n\t\t\to.Websocket.AllowedOrigins, _ = parseStringArray(\"allowed origins\", tk, <, mv, errors)\n\t\tcase \"handshake_timeout\":\n\t\t\tht := time.Duration(0)\n\t\t\tswitch mv := mv.(type) {\n\t\t\tcase int64:\n\t\t\t\tht = time.Duration(mv) * time.Second\n\t\t\tcase string:\n\t\t\t\tvar err error\n\t\t\t\tht, err = time.ParseDuration(mv)\n\t\t\t\tif err != nil {\n\t\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing handshake timeout: unsupported type %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t\to.Websocket.HandshakeTimeout = ht\n\t\tcase \"compress\", \"compression\":\n\t\t\to.Websocket.Compression = mv.(bool)\n\t\tcase \"authorization\", \"authentication\":\n\t\t\tauth := parseSimpleAuth(tk, errors)\n\t\t\to.Websocket.Username = auth.user\n\t\t\to.Websocket.Password = auth.pass\n\t\t\to.Websocket.Token = auth.token\n\t\t\to.Websocket.AuthTimeout = auth.timeout\n\t\tcase \"jwt_cookie\":\n\t\t\to.Websocket.JWTCookie = mv.(string)\n\t\tcase \"user_cookie\":\n\t\t\to.Websocket.UsernameCookie = mv.(string)\n\t\tcase \"pass_cookie\":\n\t\t\to.Websocket.PasswordCookie = mv.(string)\n\t\tcase \"token_cookie\":\n\t\t\to.Websocket.TokenCookie = mv.(string)\n\t\tcase \"no_auth_user\":\n\t\t\to.Websocket.NoAuthUser = mv.(string)\n\t\tcase \"headers\":\n\t\t\tm, ok := mv.(map[string]any)\n\t\t\tif !ok {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"error parsing headers: unsupported type %T\", mv)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.Websocket.Headers = make(map[string]string)\n\t\t\tfor key, val := range m {\n\t\t\t\ttk, val = unwrapValue(val, <)\n\t\t\t\tif headerValue, ok := val.(string); !ok {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"error parsing header key %s: unsupported type %T\", key, val)})\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\to.Websocket.Headers[key] = headerValue\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"ping_interval\":\n\t\t\to.Websocket.PingInterval = parseDuration(\"ping_interval\", tk, mv, errors, warnings)\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc parseMQTT(v any, o *Options, errors *[]error, warnings *[]error) error {\n\tvar lt token\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, v := unwrapValue(v, <)\n\tgm, ok := v.(map[string]any)\n\tif !ok {\n\t\treturn &configErr{tk, fmt.Sprintf(\"Expected mqtt to be a map, got %T\", v)}\n\t}\n\tfor mk, mv := range gm {\n\t\t// Again, unwrap token value if line check is required.\n\t\ttk, mv = unwrapValue(mv, <)\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"listen\":\n\t\t\thp, err := parseListen(mv)\n\t\t\tif err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.MQTT.Host = hp.host\n\t\t\to.MQTT.Port = hp.port\n\t\tcase \"port\":\n\t\t\to.MQTT.Port = int(mv.(int64))\n\t\tcase \"host\", \"net\":\n\t\t\to.MQTT.Host = mv.(string)\n\t\tcase \"tls\":\n\t\t\ttc, err := parseTLS(tk, true)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif o.MQTT.TLSConfig, err = GenTLSConfig(tc); err != nil {\n\t\t\t\terr := &configErr{tk, err.Error()}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.MQTT.TLSTimeout = tc.Timeout\n\t\t\to.MQTT.TLSMap = tc.Map\n\t\t\to.MQTT.TLSPinnedCerts = tc.PinnedCerts\n\t\t\to.MQTT.tlsConfigOpts = tc\n\t\tcase \"authorization\", \"authentication\":\n\t\t\tauth := parseSimpleAuth(tk, errors)\n\t\t\to.MQTT.Username = auth.user\n\t\t\to.MQTT.Password = auth.pass\n\t\t\to.MQTT.Token = auth.token\n\t\t\to.MQTT.AuthTimeout = auth.timeout\n\t\tcase \"no_auth_user\":\n\t\t\to.MQTT.NoAuthUser = mv.(string)\n\t\tcase \"ack_wait\", \"ackwait\":\n\t\t\to.MQTT.AckWait = parseDuration(\"ack_wait\", tk, mv, errors, warnings)\n\t\tcase \"js_api_timeout\", \"api_timeout\":\n\t\t\to.MQTT.JSAPITimeout = parseDuration(\"js_api_timeout\", tk, mv, errors, warnings)\n\t\tcase \"max_ack_pending\", \"max_pending\", \"max_inflight\":\n\t\t\ttmp := int(mv.(int64))\n\t\t\tif tmp < 0 || tmp > 0xFFFF {\n\t\t\t\terr := &configErr{tk, fmt.Sprintf(\"invalid value %v, should in [0..%d] range\", tmp, 0xFFFF)}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t} else {\n\t\t\t\to.MQTT.MaxAckPending = uint16(tmp)\n\t\t\t}\n\t\tcase \"js_domain\":\n\t\t\to.MQTT.JsDomain = mv.(string)\n\t\tcase \"stream_replicas\":\n\t\t\to.MQTT.StreamReplicas = int(mv.(int64))\n\t\tcase \"consumer_replicas\":\n\t\t\terr := &configWarningErr{\n\t\t\t\tfield: mk,\n\t\t\t\tconfigErr: configErr{\n\t\t\t\t\ttoken: tk,\n\t\t\t\t\treason: `consumer replicas setting ignored in this server version`,\n\t\t\t\t},\n\t\t\t}\n\t\t\t*warnings = append(*warnings, err)\n\t\tcase \"consumer_memory_storage\":\n\t\t\to.MQTT.ConsumerMemoryStorage = mv.(bool)\n\t\tcase \"consumer_inactive_threshold\", \"consumer_auto_cleanup\":\n\t\t\to.MQTT.ConsumerInactiveThreshold = parseDuration(\"consumer_inactive_threshold\", tk, mv, errors, warnings)\n\n\t\tcase \"reject_qos2_publish\":\n\t\t\to.MQTT.rejectQoS2Pub = mv.(bool)\n\t\tcase \"downgrade_qos2_subscribe\":\n\t\t\to.MQTT.downgradeQoS2Sub = mv.(bool)\n\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc parseProxies(mv any, errors *[]error) (*ProxiesConfig, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\tproxies = &ProxiesConfig{}\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv = unwrapValue(mv, <)\n\tpm, ok := mv.(map[string]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"expected proxies to be a map/struct, got %T\", mv)}\n\t}\n\tfor mk, mv := range pm {\n\t\ttk, _ = unwrapValue(mv, <)\n\n\t\tswitch strings.ToLower(mk) {\n\t\tcase \"trusted\":\n\t\t\ttrusted, err := parseProxiesTrusted(tk, errors)\n\t\t\tif err != nil {\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tproxies.Trusted = trusted\n\t\tdefault:\n\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\tfield: mk,\n\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\t*errors = append(*errors, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn proxies, nil\n}\n\nfunc parseProxiesTrusted(mv any, errors *[]error) ([]*ProxyConfig, error) {\n\tvar (\n\t\ttk token\n\t\tlt token\n\t\ttrusted []*ProxyConfig\n\t)\n\tdefer convertPanicToErrorList(<, errors)\n\n\ttk, mv = unwrapValue(mv, <)\n\tta, ok := mv.([]any)\n\tif !ok {\n\t\treturn nil, &configErr{tk, fmt.Sprintf(\"expected proxies' trusted field to be an array, got %T\", mv)}\n\t}\n\tfor _, t := range ta {\n\t\ttk, t = unwrapValue(t, <)\n\t\t// Check its a map/struct\n\t\ttm, ok := t.(map[string]any)\n\t\tif !ok {\n\t\t\terr := &configErr{tk, fmt.Sprintf(\"expected proxies' trusted entry to be a map/struct, got %T\", t)}\n\t\t\t*errors = append(*errors, err)\n\t\t\tcontinue\n\t\t}\n\t\tproxy := &ProxyConfig{}\n\t\tfor k, v := range tm {\n\t\t\ttk, v = unwrapValue(v, <)\n\t\t\tswitch strings.ToLower(k) {\n\t\t\tcase \"key\", \"public_key\":\n\t\t\t\tproxy.Key = v.(string)\n\t\t\t\tif !nkeys.IsValidPublicKey(proxy.Key) {\n\t\t\t\t\t*errors = append(*errors, &configErr{tk, fmt.Sprintf(\"invalid proxy key %q\", proxy.Key)})\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif !tk.IsUsedVariable() {\n\t\t\t\t\terr := &unknownConfigFieldErr{\n\t\t\t\t\t\tfield: k,\n\t\t\t\t\t\tconfigErr: configErr{\n\t\t\t\t\t\t\ttoken: tk,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t\t*errors = append(*errors, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ttrusted = append(trusted, proxy)\n\t}\n\treturn trusted, nil\n}\n\n// GenTLSConfig loads TLS related configuration parameters.\nfunc GenTLSConfig(tc *TLSConfigOpts) (*tls.Config, error) {\n\t// Create the tls.Config from our options before including the certs.\n\t// It will determine the cipher suites that we prefer.\n\t// FIXME(dlc) change if ARM based.\n\tconfig := tls.Config{\n\t\tMinVersion: tls.VersionTLS12,\n\t\tCipherSuites: tc.Ciphers,\n\t\tPreferServerCipherSuites: true,\n\t\tCurvePreferences: tc.CurvePreferences,\n\t\tInsecureSkipVerify: tc.Insecure,\n\t}\n\n\tswitch {\n\tcase tc.CertFile != _EMPTY_ && tc.CertStore != certstore.STOREEMPTY:\n\t\treturn nil, certstore.ErrConflictCertFileAndStore\n\tcase tc.CertFile != _EMPTY_ && tc.KeyFile == _EMPTY_:\n\t\treturn nil, fmt.Errorf(\"missing 'key_file' in TLS configuration\")\n\tcase tc.CertFile == _EMPTY_ && tc.KeyFile != _EMPTY_:\n\t\treturn nil, fmt.Errorf(\"missing 'cert_file' in TLS configuration\")\n\tcase tc.CertFile != _EMPTY_ && tc.KeyFile != _EMPTY_:\n\t\t// Now load in cert and private key\n\t\tcert, err := tls.LoadX509KeyPair(tc.CertFile, tc.KeyFile)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error parsing X509 certificate/key pair: %v\", err)\n\t\t}\n\t\tcert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error parsing certificate: %v\", err)\n\t\t}\n\t\tconfig.Certificates = []tls.Certificate{cert}\n\tcase tc.CertStore != certstore.STOREEMPTY:\n\t\terr := certstore.TLSConfig(tc.CertStore, tc.CertMatchBy, tc.CertMatch, tc.CaCertsMatch, tc.CertMatchSkipInvalid, &config)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase tc.Certificates != nil:\n\t\t// Multiple certificate support.\n\t\tconfig.Certificates = make([]tls.Certificate, len(tc.Certificates))\n\t\tfor i, certPair := range tc.Certificates {\n\t\t\tcert, err := tls.LoadX509KeyPair(certPair.CertFile, certPair.KeyFile)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error parsing X509 certificate/key pair %d/%d: %v\", i+1, len(tc.Certificates), err)\n\t\t\t}\n\t\t\tcert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error parsing certificate %d/%d: %v\", i+1, len(tc.Certificates), err)\n\t\t\t}\n\t\t\tconfig.Certificates[i] = cert\n\t\t}\n\t}\n\n\t// Require client certificates as needed\n\tif tc.Verify {\n\t\tconfig.ClientAuth = tls.RequireAndVerifyClientCert\n\t}\n\t// Add in CAs if applicable.\n\tif tc.CaFile != _EMPTY_ {\n\t\trootPEM, err := os.ReadFile(tc.CaFile)\n\t\tif err != nil || rootPEM == nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tpool := x509.NewCertPool()\n\t\tok := pool.AppendCertsFromPEM(rootPEM)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"failed to parse root ca certificate\")\n\t\t}\n\t\tconfig.ClientCAs = pool\n\t}\n\t// Allow setting TLS minimum version.\n\tif tc.MinVersion > 0 {\n\t\tif tc.MinVersion < tls.VersionTLS12 {\n\t\t\treturn nil, fmt.Errorf(\"unsupported minimum TLS version: %s\", tls.VersionName(tc.MinVersion))\n\t\t}\n\t\tconfig.MinVersion = tc.MinVersion\n\t}\n\n\treturn &config, nil\n}\n\n// MergeOptions will merge two options giving preference to the flagOpts\n// if the item is present.\nfunc MergeOptions(fileOpts, flagOpts *Options) *Options {\n\tif fileOpts == nil {\n\t\treturn flagOpts\n\t}\n\tif flagOpts == nil {\n\t\treturn fileOpts\n\t}\n\t// Merge the two, flagOpts override\n\topts := *fileOpts\n\n\tif flagOpts.Port != 0 {\n\t\topts.Port = flagOpts.Port\n\t}\n\tif flagOpts.Host != _EMPTY_ {\n\t\topts.Host = flagOpts.Host\n\t}\n\tif flagOpts.DontListen {\n\t\topts.DontListen = flagOpts.DontListen\n\t}\n\tif flagOpts.ClientAdvertise != _EMPTY_ {\n\t\topts.ClientAdvertise = flagOpts.ClientAdvertise\n\t}\n\tif flagOpts.Username != _EMPTY_ {\n\t\topts.Username = flagOpts.Username\n\t}\n\tif flagOpts.Password != _EMPTY_ {\n\t\topts.Password = flagOpts.Password\n\t}\n\tif flagOpts.Authorization != _EMPTY_ {\n\t\topts.Authorization = flagOpts.Authorization\n\t}\n\tif flagOpts.HTTPPort != 0 {\n\t\topts.HTTPPort = flagOpts.HTTPPort\n\t}\n\tif flagOpts.HTTPBasePath != _EMPTY_ {\n\t\topts.HTTPBasePath = flagOpts.HTTPBasePath\n\t}\n\tif flagOpts.Debug {\n\t\topts.Debug = true\n\t}\n\tif flagOpts.Trace {\n\t\topts.Trace = true\n\t}\n\tif flagOpts.Logtime {\n\t\topts.Logtime = true\n\t}\n\tif flagOpts.LogFile != _EMPTY_ {\n\t\topts.LogFile = flagOpts.LogFile\n\t}\n\tif flagOpts.PidFile != _EMPTY_ {\n\t\topts.PidFile = flagOpts.PidFile\n\t}\n\tif flagOpts.PortsFileDir != _EMPTY_ {\n\t\topts.PortsFileDir = flagOpts.PortsFileDir\n\t}\n\tif flagOpts.ProfPort != 0 {\n\t\topts.ProfPort = flagOpts.ProfPort\n\t}\n\tif flagOpts.Cluster.ListenStr != _EMPTY_ {\n\t\topts.Cluster.ListenStr = flagOpts.Cluster.ListenStr\n\t}\n\tif flagOpts.Cluster.NoAdvertise {\n\t\topts.Cluster.NoAdvertise = true\n\t}\n\tif flagOpts.Cluster.ConnectRetries != 0 {\n\t\topts.Cluster.ConnectRetries = flagOpts.Cluster.ConnectRetries\n\t}\n\tif flagOpts.Cluster.Advertise != _EMPTY_ {\n\t\topts.Cluster.Advertise = flagOpts.Cluster.Advertise\n\t}\n\tif flagOpts.RoutesStr != _EMPTY_ {\n\t\tmergeRoutes(&opts, flagOpts)\n\t}\n\tif flagOpts.JetStream {\n\t\topts.JetStream = flagOpts.JetStream\n\t}\n\tif flagOpts.StoreDir != _EMPTY_ {\n\t\topts.StoreDir = flagOpts.StoreDir\n\t}\n\treturn &opts\n}\n\n// RoutesFromStr parses route URLs from a string\nfunc RoutesFromStr(routesStr string) []*url.URL {\n\troutes := strings.Split(routesStr, \",\")\n\tif len(routes) == 0 {\n\t\treturn nil\n\t}\n\trouteUrls := []*url.URL{}\n\tfor _, r := range routes {\n\t\tr = strings.TrimSpace(r)\n\t\tu, _ := url.Parse(r)\n\t\trouteUrls = append(routeUrls, u)\n\t}\n\treturn routeUrls\n}\n\n// This will merge the flag routes and override anything that was present.\nfunc mergeRoutes(opts, flagOpts *Options) {\n\trouteUrls := RoutesFromStr(flagOpts.RoutesStr)\n\tif routeUrls == nil {\n\t\treturn\n\t}\n\topts.Routes = routeUrls\n\topts.RoutesStr = flagOpts.RoutesStr\n}\n\nfunc setBaselineOptions(opts *Options) {\n\t// Setup non-standard Go defaults\n\tif opts.Host == _EMPTY_ {\n\t\topts.Host = DEFAULT_HOST\n\t}\n\tif opts.HTTPHost == _EMPTY_ {\n\t\t// Default to same bind from server if left undefined\n\t\topts.HTTPHost = opts.Host\n\t}\n\tif opts.Port == 0 {\n\t\topts.Port = DEFAULT_PORT\n\t} else if opts.Port == RANDOM_PORT {\n\t\t// Choose randomly inside of net.Listen\n\t\topts.Port = 0\n\t}\n\tif opts.MaxConn == 0 {\n\t\topts.MaxConn = DEFAULT_MAX_CONNECTIONS\n\t}\n\tif opts.PingInterval == 0 {\n\t\topts.PingInterval = DEFAULT_PING_INTERVAL\n\t}\n\tif opts.MaxPingsOut == 0 {\n\t\topts.MaxPingsOut = DEFAULT_PING_MAX_OUT\n\t}\n\tif opts.TLSTimeout == 0 {\n\t\topts.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second)\n\t}\n\tif opts.AuthTimeout == 0 {\n\t\topts.AuthTimeout = getDefaultAuthTimeout(opts.TLSConfig, opts.TLSTimeout)\n\t}\n\tif opts.Cluster.Port != 0 || opts.Cluster.ListenStr != _EMPTY_ {\n\t\tif opts.Cluster.Host == _EMPTY_ {\n\t\t\topts.Cluster.Host = DEFAULT_HOST\n\t\t}\n\t\tif opts.Cluster.TLSTimeout == 0 {\n\t\t\topts.Cluster.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second)\n\t\t}\n\t\tif opts.Cluster.AuthTimeout == 0 {\n\t\t\topts.Cluster.AuthTimeout = getDefaultAuthTimeout(opts.Cluster.TLSConfig, opts.Cluster.TLSTimeout)\n\t\t}\n\t\tif opts.Cluster.PoolSize == 0 {\n\t\t\topts.Cluster.PoolSize = DEFAULT_ROUTE_POOL_SIZE\n\t\t}\n\t\t// Unless pooling/accounts are disabled (by PoolSize being set to -1),\n\t\t// check for Cluster.Accounts. Add the system account if not present and\n\t\t// unless we have a configuration that disabled it.\n\t\tif opts.Cluster.PoolSize > 0 {\n\t\t\tsysAccName := opts.SystemAccount\n\t\t\tif sysAccName == _EMPTY_ && !opts.NoSystemAccount {\n\t\t\t\tsysAccName = DEFAULT_SYSTEM_ACCOUNT\n\t\t\t}\n\t\t\tif sysAccName != _EMPTY_ {\n\t\t\t\tvar found bool\n\t\t\t\tfor _, acc := range opts.Cluster.PinnedAccounts {\n\t\t\t\t\tif acc == sysAccName {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\topts.Cluster.PinnedAccounts = append(opts.Cluster.PinnedAccounts, sysAccName)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Default to compression \"accept\", which means that compression is not\n\t\t// initiated, but if the remote selects compression, this server will\n\t\t// use the same.\n\t\tif c := &opts.Cluster.Compression; c.Mode == _EMPTY_ {\n\t\t\tif testDefaultClusterCompression != _EMPTY_ {\n\t\t\t\tc.Mode = testDefaultClusterCompression\n\t\t\t} else {\n\t\t\t\tc.Mode = CompressionAccept\n\t\t\t}\n\t\t}\n\t}\n\tif opts.LeafNode.Port != 0 {\n\t\tif opts.LeafNode.Host == _EMPTY_ {\n\t\t\topts.LeafNode.Host = DEFAULT_HOST\n\t\t}\n\t\tif opts.LeafNode.TLSTimeout == 0 {\n\t\t\topts.LeafNode.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second)\n\t\t}\n\t\tif opts.LeafNode.AuthTimeout == 0 {\n\t\t\topts.LeafNode.AuthTimeout = getDefaultAuthTimeout(opts.LeafNode.TLSConfig, opts.LeafNode.TLSTimeout)\n\t\t}\n\t\t// Default to compression \"s2_auto\".\n\t\tif c := &opts.LeafNode.Compression; c.Mode == _EMPTY_ {\n\t\t\tif testDefaultLeafNodeCompression != _EMPTY_ {\n\t\t\t\tc.Mode = testDefaultLeafNodeCompression\n\t\t\t} else {\n\t\t\t\tc.Mode = CompressionS2Auto\n\t\t\t}\n\t\t}\n\t}\n\t// Set baseline connect port for remotes.\n\tfor _, r := range opts.LeafNode.Remotes {\n\t\tif r != nil {\n\t\t\tfor _, u := range r.URLs {\n\t\t\t\tif u.Port() == _EMPTY_ {\n\t\t\t\t\tu.Host = net.JoinHostPort(u.Host, strconv.Itoa(DEFAULT_LEAFNODE_PORT))\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Default to compression \"s2_auto\".\n\t\t\tif c := &r.Compression; c.Mode == _EMPTY_ {\n\t\t\t\tif testDefaultLeafNodeCompression != _EMPTY_ {\n\t\t\t\t\tc.Mode = testDefaultLeafNodeCompression\n\t\t\t\t} else {\n\t\t\t\t\tc.Mode = CompressionS2Auto\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Set default first info timeout value if not set.\n\t\t\tif r.FirstInfoTimeout <= 0 {\n\t\t\t\tr.FirstInfoTimeout = DEFAULT_LEAFNODE_INFO_WAIT\n\t\t\t}\n\t\t}\n\t}\n\n\t// Set this regardless of opts.LeafNode.Port\n\tif opts.LeafNode.ReconnectInterval == 0 {\n\t\topts.LeafNode.ReconnectInterval = DEFAULT_LEAF_NODE_RECONNECT\n\t}\n\n\tif opts.MaxControlLine == 0 {\n\t\topts.MaxControlLine = MAX_CONTROL_LINE_SIZE\n\t}\n\tif opts.MaxPayload == 0 {\n\t\topts.MaxPayload = MAX_PAYLOAD_SIZE\n\t}\n\tif opts.MaxPending == 0 {\n\t\topts.MaxPending = MAX_PENDING_SIZE\n\t}\n\tif opts.WriteDeadline == time.Duration(0) {\n\t\topts.WriteDeadline = DEFAULT_FLUSH_DEADLINE\n\t}\n\tif opts.MaxClosedClients == 0 {\n\t\topts.MaxClosedClients = DEFAULT_MAX_CLOSED_CLIENTS\n\t}\n\tif opts.LameDuckDuration == 0 {\n\t\topts.LameDuckDuration = DEFAULT_LAME_DUCK_DURATION\n\t}\n\tif opts.LameDuckGracePeriod == 0 {\n\t\topts.LameDuckGracePeriod = DEFAULT_LAME_DUCK_GRACE_PERIOD\n\t}\n\tif opts.Gateway.Port != 0 {\n\t\tif opts.Gateway.Host == _EMPTY_ {\n\t\t\topts.Gateway.Host = DEFAULT_HOST\n\t\t}\n\t\tif opts.Gateway.TLSTimeout == 0 {\n\t\t\topts.Gateway.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second)\n\t\t}\n\t\tif opts.Gateway.AuthTimeout == 0 {\n\t\t\topts.Gateway.AuthTimeout = getDefaultAuthTimeout(opts.Gateway.TLSConfig, opts.Gateway.TLSTimeout)\n\t\t}\n\t}\n\tif opts.ConnectErrorReports == 0 {\n\t\topts.ConnectErrorReports = DEFAULT_CONNECT_ERROR_REPORTS\n\t}\n\tif opts.ReconnectErrorReports == 0 {\n\t\topts.ReconnectErrorReports = DEFAULT_RECONNECT_ERROR_REPORTS\n\t}\n\tif opts.Websocket.Port != 0 {\n\t\tif opts.Websocket.Host == _EMPTY_ {\n\t\t\topts.Websocket.Host = DEFAULT_HOST\n\t\t}\n\t}\n\tif opts.MQTT.Port != 0 {\n\t\tif opts.MQTT.Host == _EMPTY_ {\n\t\t\topts.MQTT.Host = DEFAULT_HOST\n\t\t}\n\t\tif opts.MQTT.TLSTimeout == 0 {\n\t\t\topts.MQTT.TLSTimeout = float64(TLS_TIMEOUT) / float64(time.Second)\n\t\t}\n\t}\n\t// JetStream\n\tif opts.JetStreamMaxMemory == 0 && !opts.maxMemSet {\n\t\topts.JetStreamMaxMemory = -1\n\t}\n\tif opts.JetStreamMaxStore == 0 && !opts.maxStoreSet {\n\t\topts.JetStreamMaxStore = -1\n\t}\n\tif opts.SyncInterval == 0 && !opts.syncSet {\n\t\topts.SyncInterval = defaultSyncInterval\n\t}\n\tif opts.JetStreamRequestQueueLimit <= 0 {\n\t\topts.JetStreamRequestQueueLimit = JSDefaultRequestQueueLimit\n\t}\n\tif opts.JetStreamInfoQueueLimit <= 0 {\n\t\topts.JetStreamInfoQueueLimit = opts.JetStreamRequestQueueLimit\n\t}\n}\n\nfunc getDefaultAuthTimeout(tls *tls.Config, tlsTimeout float64) float64 {\n\tvar authTimeout float64\n\tif tls != nil {\n\t\tauthTimeout = tlsTimeout + 1.0\n\t} else {\n\t\tauthTimeout = float64(AUTH_TIMEOUT / time.Second)\n\t}\n\treturn authTimeout\n}\n\n// ConfigureOptions accepts a flag set and augments it with NATS Server\n// specific flags. On success, an options structure is returned configured\n// based on the selected flags and/or configuration file.\n// The command line options take precedence to the ones in the configuration file.\nfunc ConfigureOptions(fs *flag.FlagSet, args []string, printVersion, printHelp, printTLSHelp func()) (*Options, error) {\n\topts := &Options{}\n\tvar (\n\t\tshowVersion bool\n\t\tshowHelp bool\n\t\tshowTLSHelp bool\n\t\tsignal string\n\t\tconfigFile string\n\t\tdbgAndTrace bool\n\t\ttrcAndVerboseTrc bool\n\t\tdbgAndTrcAndVerboseTrc bool\n\t\terr error\n\t)\n\n\tfs.BoolVar(&showHelp, \"h\", false, \"Show this message.\")\n\tfs.BoolVar(&showHelp, \"help\", false, \"Show this message.\")\n\tfs.IntVar(&opts.Port, \"port\", 0, \"Port to listen on.\")\n\tfs.IntVar(&opts.Port, \"p\", 0, \"Port to listen on.\")\n\tfs.StringVar(&opts.ServerName, \"n\", _EMPTY_, \"Server name.\")\n\tfs.StringVar(&opts.ServerName, \"name\", _EMPTY_, \"Server name.\")\n\tfs.StringVar(&opts.ServerName, \"server_name\", _EMPTY_, \"Server name.\")\n\tfs.StringVar(&opts.Host, \"addr\", _EMPTY_, \"Network host to listen on.\")\n\tfs.StringVar(&opts.Host, \"a\", _EMPTY_, \"Network host to listen on.\")\n\tfs.StringVar(&opts.Host, \"net\", _EMPTY_, \"Network host to listen on.\")\n\tfs.StringVar(&opts.ClientAdvertise, \"client_advertise\", _EMPTY_, \"Client URL to advertise to other servers.\")\n\tfs.BoolVar(&opts.Debug, \"D\", false, \"Enable Debug logging.\")\n\tfs.BoolVar(&opts.Debug, \"debug\", false, \"Enable Debug logging.\")\n\tfs.BoolVar(&opts.Trace, \"V\", false, \"Enable Trace logging.\")\n\tfs.BoolVar(&trcAndVerboseTrc, \"VV\", false, \"Enable Verbose Trace logging. (Traces system account as well)\")\n\tfs.BoolVar(&opts.Trace, \"trace\", false, \"Enable Trace logging.\")\n\tfs.BoolVar(&dbgAndTrace, \"DV\", false, \"Enable Debug and Trace logging.\")\n\tfs.BoolVar(&dbgAndTrcAndVerboseTrc, \"DVV\", false, \"Enable Debug and Verbose Trace logging. (Traces system account as well)\")\n\tfs.BoolVar(&opts.Logtime, \"T\", true, \"Timestamp log entries.\")\n\tfs.BoolVar(&opts.Logtime, \"logtime\", true, \"Timestamp log entries.\")\n\tfs.BoolVar(&opts.LogtimeUTC, \"logtime_utc\", false, \"Timestamps in UTC instead of local timezone.\")\n\tfs.StringVar(&opts.Username, \"user\", _EMPTY_, \"Username required for connection.\")\n\tfs.StringVar(&opts.Password, \"pass\", _EMPTY_, \"Password required for connection.\")\n\tfs.StringVar(&opts.Authorization, \"auth\", _EMPTY_, \"Authorization token required for connection.\")\n\tfs.IntVar(&opts.HTTPPort, \"m\", 0, \"HTTP Port for /varz, /connz endpoints.\")\n\tfs.IntVar(&opts.HTTPPort, \"http_port\", 0, \"HTTP Port for /varz, /connz endpoints.\")\n\tfs.IntVar(&opts.HTTPSPort, \"ms\", 0, \"HTTPS Port for /varz, /connz endpoints.\")\n\tfs.IntVar(&opts.HTTPSPort, \"https_port\", 0, \"HTTPS Port for /varz, /connz endpoints.\")\n\tfs.StringVar(&configFile, \"c\", _EMPTY_, \"Configuration file.\")\n\tfs.StringVar(&configFile, \"config\", _EMPTY_, \"Configuration file.\")\n\tfs.BoolVar(&opts.CheckConfig, \"t\", false, \"Check configuration and exit.\")\n\tfs.StringVar(&signal, \"sl\", \"\", \"Send signal to nats-server process (ldm, stop, quit, term, reopen, reload).\")\n\tfs.StringVar(&signal, \"signal\", \"\", \"Send signal to nats-server process (ldm, stop, quit, term, reopen, reload).\")\n\tfs.StringVar(&opts.PidFile, \"P\", \"\", \"File to store process pid.\")\n\tfs.StringVar(&opts.PidFile, \"pid\", \"\", \"File to store process pid.\")\n\tfs.StringVar(&opts.PortsFileDir, \"ports_file_dir\", \"\", \"Creates a ports file in the specified directory (_.ports).\")\n\tfs.StringVar(&opts.LogFile, \"l\", \"\", \"File to store logging output.\")\n\tfs.StringVar(&opts.LogFile, \"log\", \"\", \"File to store logging output.\")\n\tfs.Int64Var(&opts.LogSizeLimit, \"log_size_limit\", 0, \"Logfile size limit being auto-rotated\")\n\tfs.BoolVar(&opts.Syslog, \"s\", false, \"Enable syslog as log method.\")\n\tfs.BoolVar(&opts.Syslog, \"syslog\", false, \"Enable syslog as log method.\")\n\tfs.StringVar(&opts.RemoteSyslog, \"r\", _EMPTY_, \"Syslog server addr (udp://127.0.0.1:514).\")\n\tfs.StringVar(&opts.RemoteSyslog, \"remote_syslog\", _EMPTY_, \"Syslog server addr (udp://127.0.0.1:514).\")\n\tfs.BoolVar(&showVersion, \"version\", false, \"Print version information.\")\n\tfs.BoolVar(&showVersion, \"v\", false, \"Print version information.\")\n\tfs.IntVar(&opts.ProfPort, \"profile\", 0, \"Profiling HTTP port.\")\n\tfs.StringVar(&opts.RoutesStr, \"routes\", _EMPTY_, \"Routes to actively solicit a connection.\")\n\tfs.StringVar(&opts.Cluster.ListenStr, \"cluster\", _EMPTY_, \"Cluster url from which members can solicit routes.\")\n\tfs.StringVar(&opts.Cluster.ListenStr, \"cluster_listen\", _EMPTY_, \"Cluster url from which members can solicit routes.\")\n\tfs.StringVar(&opts.Cluster.Advertise, \"cluster_advertise\", _EMPTY_, \"Cluster URL to advertise to other servers.\")\n\tfs.BoolVar(&opts.Cluster.NoAdvertise, \"no_advertise\", false, \"Advertise known cluster IPs to clients.\")\n\tfs.IntVar(&opts.Cluster.ConnectRetries, \"connect_retries\", 0, \"For implicit routes, number of connect retries.\")\n\tfs.StringVar(&opts.Cluster.Name, \"cluster_name\", _EMPTY_, \"Cluster Name, if not set one will be dynamically generated.\")\n\tfs.BoolVar(&showTLSHelp, \"help_tls\", false, \"TLS help.\")\n\tfs.BoolVar(&opts.TLS, \"tls\", false, \"Enable TLS.\")\n\tfs.BoolVar(&opts.TLSVerify, \"tlsverify\", false, \"Enable TLS with client verification.\")\n\tfs.StringVar(&opts.TLSCert, \"tlscert\", _EMPTY_, \"Server certificate file.\")\n\tfs.StringVar(&opts.TLSKey, \"tlskey\", _EMPTY_, \"Private key for server certificate.\")\n\tfs.StringVar(&opts.TLSCaCert, \"tlscacert\", _EMPTY_, \"Client certificate CA for verification.\")\n\tfs.IntVar(&opts.MaxTracedMsgLen, \"max_traced_msg_len\", 0, \"Maximum printable length for traced messages. 0 for unlimited.\")\n\tfs.BoolVar(&opts.JetStream, \"js\", false, \"Enable JetStream.\")\n\tfs.BoolVar(&opts.JetStream, \"jetstream\", false, \"Enable JetStream.\")\n\tfs.StringVar(&opts.StoreDir, \"sd\", _EMPTY_, \"Storage directory.\")\n\tfs.StringVar(&opts.StoreDir, \"store_dir\", _EMPTY_, \"Storage directory.\")\n\n\t// The flags definition above set \"default\" values to some of the options.\n\t// Calling Parse() here will override the default options with any value\n\t// specified from the command line. This is ok. We will then update the\n\t// options with the content of the configuration file (if present), and then,\n\t// call Parse() again to override the default+config with command line values.\n\t// Calling Parse() before processing config file is necessary since configFile\n\t// itself is a command line argument, and also Parse() is required in order\n\t// to know if user wants simply to show \"help\" or \"version\", etc...\n\tif err := fs.Parse(args); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif showVersion {\n\t\tprintVersion()\n\t\treturn nil, nil\n\t}\n\n\tif showHelp {\n\t\tprintHelp()\n\t\treturn nil, nil\n\t}\n\n\tif showTLSHelp {\n\t\tprintTLSHelp()\n\t\treturn nil, nil\n\t}\n\n\t// Process args looking for non-flag options,\n\t// 'version' and 'help' only for now\n\tshowVersion, showHelp, err = ProcessCommandLineArgs(fs)\n\tif err != nil {\n\t\treturn nil, err\n\t} else if showVersion {\n\t\tprintVersion()\n\t\treturn nil, nil\n\t} else if showHelp {\n\t\tprintHelp()\n\t\treturn nil, nil\n\t}\n\n\t// Snapshot flag options.\n\tFlagSnapshot = opts.Clone()\n\n\t// Keep track of the boolean flags that were explicitly set with their value.\n\tfs.Visit(func(f *flag.Flag) {\n\t\tswitch f.Name {\n\t\tcase \"DVV\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Debug\", dbgAndTrcAndVerboseTrc)\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Trace\", dbgAndTrcAndVerboseTrc)\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"TraceVerbose\", dbgAndTrcAndVerboseTrc)\n\t\tcase \"DV\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Debug\", dbgAndTrace)\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Trace\", dbgAndTrace)\n\t\tcase \"D\":\n\t\t\tfallthrough\n\t\tcase \"debug\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Debug\", FlagSnapshot.Debug)\n\t\tcase \"VV\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Trace\", trcAndVerboseTrc)\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"TraceVerbose\", trcAndVerboseTrc)\n\t\tcase \"V\":\n\t\t\tfallthrough\n\t\tcase \"trace\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Trace\", FlagSnapshot.Trace)\n\t\tcase \"T\":\n\t\t\tfallthrough\n\t\tcase \"logtime\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Logtime\", FlagSnapshot.Logtime)\n\t\tcase \"s\":\n\t\t\tfallthrough\n\t\tcase \"syslog\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Syslog\", FlagSnapshot.Syslog)\n\t\tcase \"no_advertise\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"Cluster.NoAdvertise\", FlagSnapshot.Cluster.NoAdvertise)\n\t\tcase \"js\":\n\t\t\ttrackExplicitVal(&FlagSnapshot.inCmdLine, \"JetStream\", FlagSnapshot.JetStream)\n\t\t}\n\t})\n\n\t// Process signal control.\n\tif signal != _EMPTY_ {\n\t\tif err := processSignal(signal); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Parse config if given\n\tif configFile != _EMPTY_ {\n\t\t// This will update the options with values from the config file.\n\t\terr := opts.ProcessConfigFile(configFile)\n\t\tif err != nil {\n\t\t\tif opts.CheckConfig {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif cerr, ok := err.(*processConfigErr); !ok || len(cerr.Errors()) != 0 {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// If we get here we only have warnings and can still continue\n\t\t\tfmt.Fprint(os.Stderr, err)\n\t\t} else if opts.CheckConfig {\n\t\t\t// Report configuration file syntax test was successful and exit.\n\t\t\treturn opts, nil\n\t\t}\n\n\t\t// Call this again to override config file options with options from command line.\n\t\t// Note: We don't need to check error here since if there was an error, it would\n\t\t// have been caught the first time this function was called (after setting up the\n\t\t// flags).\n\t\tfs.Parse(args)\n\t} else if opts.CheckConfig {\n\t\treturn nil, fmt.Errorf(\"must specify [-c, --config] option to check configuration file syntax\")\n\t}\n\n\t// Special handling of some flags\n\tvar (\n\t\tflagErr error\n\t\ttlsDisabled bool\n\t\ttlsOverride bool\n\t)\n\tfs.Visit(func(f *flag.Flag) {\n\t\t// short-circuit if an error was encountered\n\t\tif flagErr != nil {\n\t\t\treturn\n\t\t}\n\t\tif strings.HasPrefix(f.Name, \"tls\") {\n\t\t\tif f.Name == \"tls\" {\n\t\t\t\tif !opts.TLS {\n\t\t\t\t\t// User has specified \"-tls=false\", we need to disable TLS\n\t\t\t\t\topts.TLSConfig = nil\n\t\t\t\t\ttlsDisabled = true\n\t\t\t\t\ttlsOverride = false\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\ttlsOverride = true\n\t\t\t} else if !tlsDisabled {\n\t\t\t\ttlsOverride = true\n\t\t\t}\n\t\t} else {\n\t\t\tswitch f.Name {\n\t\t\tcase \"VV\":\n\t\t\t\topts.Trace, opts.TraceVerbose = trcAndVerboseTrc, trcAndVerboseTrc\n\t\t\tcase \"DVV\":\n\t\t\t\topts.Trace, opts.Debug, opts.TraceVerbose = dbgAndTrcAndVerboseTrc, dbgAndTrcAndVerboseTrc, dbgAndTrcAndVerboseTrc\n\t\t\tcase \"DV\":\n\t\t\t\t// Check value to support -DV=false\n\t\t\t\topts.Trace, opts.Debug = dbgAndTrace, dbgAndTrace\n\t\t\tcase \"cluster\", \"cluster_listen\":\n\t\t\t\t// Override cluster config if explicitly set via flags.\n\t\t\t\tflagErr = overrideCluster(opts)\n\t\t\tcase \"routes\":\n\t\t\t\t// Keep in mind that the flag has updated opts.RoutesStr at this point.\n\t\t\t\tif opts.RoutesStr == _EMPTY_ {\n\t\t\t\t\t// Set routes array to nil since routes string is empty\n\t\t\t\t\topts.Routes = nil\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\trouteUrls := RoutesFromStr(opts.RoutesStr)\n\t\t\t\topts.Routes = routeUrls\n\t\t\t}\n\t\t}\n\t})\n\tif flagErr != nil {\n\t\treturn nil, flagErr\n\t}\n\n\t// This will be true if some of the `-tls` params have been set and\n\t// `-tls=false` has not been set.\n\tif tlsOverride {\n\t\tif err := overrideTLS(opts); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// If we don't have cluster defined in the configuration\n\t// file and no cluster listen string override, but we do\n\t// have a routes override, we need to report misconfiguration.\n\tif opts.RoutesStr != _EMPTY_ && opts.Cluster.ListenStr == _EMPTY_ && opts.Cluster.Host == _EMPTY_ && opts.Cluster.Port == 0 {\n\t\treturn nil, errors.New(\"solicited routes require cluster capabilities, e.g. --cluster\")\n\t}\n\n\treturn opts, nil\n}\n\nfunc normalizeBasePath(p string) string {\n\tif len(p) == 0 {\n\t\treturn \"/\"\n\t}\n\t// add leading slash\n\tif p[0] != '/' {\n\t\tp = \"/\" + p\n\t}\n\treturn path.Clean(p)\n}\n\n// overrideTLS is called when at least \"-tls=true\" has been set.\nfunc overrideTLS(opts *Options) error {\n\tif opts.TLSCert == _EMPTY_ {\n\t\treturn errors.New(\"TLS Server certificate must be present and valid\")\n\t}\n\tif opts.TLSKey == _EMPTY_ {\n\t\treturn errors.New(\"TLS Server private key must be present and valid\")\n\t}\n\n\ttc := TLSConfigOpts{}\n\ttc.CertFile = opts.TLSCert\n\ttc.KeyFile = opts.TLSKey\n\ttc.CaFile = opts.TLSCaCert\n\ttc.Verify = opts.TLSVerify\n\ttc.Ciphers = defaultCipherSuites()\n\n\tvar err error\n\topts.TLSConfig, err = GenTLSConfig(&tc)\n\treturn err\n}\n\n// overrideCluster updates Options.Cluster if that flag \"cluster\" (or \"cluster_listen\")\n// has explicitly be set in the command line. If it is set to empty string, it will\n// clear the Cluster options.\nfunc overrideCluster(opts *Options) error {\n\tif opts.Cluster.ListenStr == _EMPTY_ {\n\t\t// This one is enough to disable clustering.\n\t\topts.Cluster.Port = 0\n\t\treturn nil\n\t}\n\t// -1 will fail url.Parse, so if we have -1, change it to\n\t// 0, and then after parse, replace the port with -1 so we get\n\t// automatic port allocation\n\twantsRandom := false\n\tif strings.HasSuffix(opts.Cluster.ListenStr, \":-1\") {\n\t\twantsRandom = true\n\t\tcls := fmt.Sprintf(\"%s:0\", opts.Cluster.ListenStr[0:len(opts.Cluster.ListenStr)-3])\n\t\topts.Cluster.ListenStr = cls\n\t}\n\tclusterURL, err := url.Parse(opts.Cluster.ListenStr)\n\tif err != nil {\n\t\treturn err\n\t}\n\th, p, err := net.SplitHostPort(clusterURL.Host)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif wantsRandom {\n\t\tp = \"-1\"\n\t}\n\topts.Cluster.Host = h\n\t_, err = fmt.Sscan(p, &opts.Cluster.Port)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif clusterURL.User != nil {\n\t\tpass, hasPassword := clusterURL.User.Password()\n\t\tif !hasPassword {\n\t\t\treturn errors.New(\"expected cluster password to be set\")\n\t\t}\n\t\topts.Cluster.Password = pass\n\n\t\tuser := clusterURL.User.Username()\n\t\topts.Cluster.Username = user\n\t} else {\n\t\t// Since we override from flag and there is no user/pwd, make\n\t\t// sure we clear what we may have gotten from config file.\n\t\topts.Cluster.Username = _EMPTY_\n\t\topts.Cluster.Password = _EMPTY_\n\t}\n\n\treturn nil\n}\n\nfunc processSignal(signal string) error {\n\tvar (\n\t\tpid string\n\t\tcommandAndPid = strings.Split(signal, \"=\")\n\t)\n\tif l := len(commandAndPid); l == 2 {\n\t\tpid = maybeReadPidFile(commandAndPid[1])\n\t} else if l > 2 {\n\t\treturn fmt.Errorf(\"invalid signal parameters: %v\", commandAndPid[2:])\n\t}\n\tif err := ProcessSignal(Command(commandAndPid[0]), pid); err != nil {\n\t\treturn err\n\t}\n\tos.Exit(0)\n\treturn nil\n}\n\n// maybeReadPidFile returns a PID or Windows service name obtained via the following method:\n// 1. Try to open a file with path \"pidStr\" (absolute or relative).\n// 2. If such a file exists and can be read, return its contents.\n// 3. Otherwise, return the original \"pidStr\" string.\nfunc maybeReadPidFile(pidStr string) string {\n\tif b, err := os.ReadFile(pidStr); err == nil {\n\t\treturn string(b)\n\t}\n\treturn pidStr\n}\n\nfunc homeDir() (string, error) {\n\tif runtime.GOOS == \"windows\" {\n\t\thomeDrive, homePath := os.Getenv(\"HOMEDRIVE\"), os.Getenv(\"HOMEPATH\")\n\t\tuserProfile := os.Getenv(\"USERPROFILE\")\n\n\t\thome := filepath.Join(homeDrive, homePath)\n\t\tif homeDrive == _EMPTY_ || homePath == _EMPTY_ {\n\t\t\tif userProfile == _EMPTY_ {\n\t\t\t\treturn _EMPTY_, errors.New(\"nats: failed to get home dir, require %HOMEDRIVE% and %HOMEPATH% or %USERPROFILE%\")\n\t\t\t}\n\t\t\thome = userProfile\n\t\t}\n\n\t\treturn home, nil\n\t}\n\n\thome := os.Getenv(\"HOME\")\n\tif home == _EMPTY_ {\n\t\treturn _EMPTY_, errors.New(\"failed to get home dir, require $HOME\")\n\t}\n\treturn home, nil\n}\n\nfunc expandPath(p string) (string, error) {\n\tp = os.ExpandEnv(p)\n\n\tif !strings.HasPrefix(p, \"~\") {\n\t\treturn p, nil\n\t}\n\n\thome, err := homeDir()\n\tif err != nil {\n\t\treturn _EMPTY_, err\n\t}\n\n\treturn filepath.Join(home, p[1:]), nil\n}\n" }, { "path": "server/opts_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/conf\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc checkOptionsEqual(t *testing.T, golden, opts *Options) {\n\tt.Helper()\n\t// Clone them so we can remove private fields that we don't\n\t// want to be compared.\n\tgoldenClone := golden.Clone()\n\tgoldenClone.inConfig, goldenClone.inCmdLine = nil, nil\n\toptsClone := opts.Clone()\n\toptsClone.inConfig, optsClone.inCmdLine = nil, nil\n\tif !reflect.DeepEqual(goldenClone, optsClone) {\n\t\tt.Fatalf(\"Options are incorrect.\\nexpected: %+v\\ngot: %+v\", goldenClone, optsClone)\n\t}\n}\n\nfunc TestDefaultOptions(t *testing.T) {\n\tgolden := &Options{\n\t\tHost: DEFAULT_HOST,\n\t\tPort: DEFAULT_PORT,\n\t\tMaxConn: DEFAULT_MAX_CONNECTIONS,\n\t\tHTTPHost: DEFAULT_HOST,\n\t\tPingInterval: DEFAULT_PING_INTERVAL,\n\t\tMaxPingsOut: DEFAULT_PING_MAX_OUT,\n\t\tTLSTimeout: float64(TLS_TIMEOUT) / float64(time.Second),\n\t\tAuthTimeout: float64(AUTH_TIMEOUT) / float64(time.Second),\n\t\tMaxControlLine: MAX_CONTROL_LINE_SIZE,\n\t\tMaxPayload: MAX_PAYLOAD_SIZE,\n\t\tMaxPending: MAX_PENDING_SIZE,\n\t\tWriteDeadline: DEFAULT_FLUSH_DEADLINE,\n\t\tMaxClosedClients: DEFAULT_MAX_CLOSED_CLIENTS,\n\t\tLameDuckDuration: DEFAULT_LAME_DUCK_DURATION,\n\t\tLameDuckGracePeriod: DEFAULT_LAME_DUCK_GRACE_PERIOD,\n\t\tLeafNode: LeafNodeOpts{\n\t\t\tReconnectInterval: DEFAULT_LEAF_NODE_RECONNECT,\n\t\t},\n\t\tConnectErrorReports: DEFAULT_CONNECT_ERROR_REPORTS,\n\t\tReconnectErrorReports: DEFAULT_RECONNECT_ERROR_REPORTS,\n\t\tMaxTracedMsgLen: 0,\n\t\tJetStreamMaxMemory: -1,\n\t\tJetStreamMaxStore: -1,\n\t\tSyncInterval: 2 * time.Minute,\n\t\tJetStreamRequestQueueLimit: JSDefaultRequestQueueLimit,\n\t\tJetStreamInfoQueueLimit: JSDefaultRequestQueueLimit,\n\t}\n\n\topts := &Options{}\n\tsetBaselineOptions(opts)\n\n\tcheckOptionsEqual(t, golden, opts)\n}\n\nfunc TestOptions_RandomPort(t *testing.T) {\n\topts := &Options{Port: RANDOM_PORT}\n\tsetBaselineOptions(opts)\n\n\tif opts.Port != 0 {\n\t\tt.Fatalf(\"Process of options should have resolved random port to \"+\n\t\t\t\"zero.\\nexpected: %d\\ngot: %d\", 0, opts.Port)\n\t}\n}\n\nfunc TestConfigFile(t *testing.T) {\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/test.conf\",\n\t\tServerName: \"testing_server\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 4242,\n\t\tUsername: \"derek\",\n\t\tPassword: \"porkchop\",\n\t\tAuthTimeout: 1.0,\n\t\tDebug: false,\n\t\tTrace: true,\n\t\tLogtime: false,\n\t\tHTTPPort: 8222,\n\t\tHTTPBasePath: \"/nats\",\n\t\tPidFile: \"/tmp/nats-server/nats-server.pid\",\n\t\tProfPort: 6543,\n\t\tSyslog: true,\n\t\tRemoteSyslog: \"udp://foo.com:33\",\n\t\tMaxControlLine: 2048,\n\t\tMaxPayload: 65536,\n\t\tMaxConn: 100,\n\t\tMaxSubs: 1000,\n\t\tMaxPending: 10000000,\n\t\tPingInterval: 60 * time.Second,\n\t\tMaxPingsOut: 3,\n\t\tWriteDeadline: 3 * time.Second,\n\t\tLameDuckDuration: 4 * time.Minute,\n\t\tConnectErrorReports: 86400,\n\t\tReconnectErrorReports: 5,\n\t\tMetadata: map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"},\n\t\tconfigDigest: \"sha256:a1104db0c8e838096a4f0509ec4d1e7c2c26ff60261ecb8f6a12dde1317872c3\",\n\t\tauthBlockDefined: true,\n\t}\n\n\topts, err := ProcessConfigFile(\"./configs/test.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\tcheckOptionsEqual(t, golden, opts)\n}\n\nfunc TestTLSConfigFile(t *testing.T) {\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/tls.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 4443,\n\t\tUsername: \"derek\",\n\t\tPassword: \"foo\",\n\t\tAuthTimeout: 1.0,\n\t\tTLSTimeout: 2.0,\n\t\tauthBlockDefined: true,\n\t\tconfigDigest: \"sha256:cec986d37d7c09c86916d1ba4990cea1b8dadd49c86f9f782b455b47d07c2ac8\",\n\t}\n\topts, err := ProcessConfigFile(\"./configs/tls.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\ttlsConfig := opts.TLSConfig\n\tif tlsConfig == nil {\n\t\tt.Fatal(\"Expected opts.TLSConfig to be non-nil\")\n\t}\n\topts.TLSConfig = nil\n\topts.tlsConfigOpts = nil\n\tcheckOptionsEqual(t, golden, opts)\n\n\t// Now check TLSConfig a bit more closely\n\t// CipherSuites\n\tciphers := defaultCipherSuites()\n\tif !reflect.DeepEqual(tlsConfig.CipherSuites, ciphers) {\n\t\tt.Fatalf(\"Got incorrect cipher suite list: [%+v]\", tlsConfig.CipherSuites)\n\t}\n\tif tlsConfig.MinVersion != tls.VersionTLS12 {\n\t\tt.Fatalf(\"Expected MinVersion of 1.2 [%v], got [%v]\", tls.VersionTLS12, tlsConfig.MinVersion)\n\t}\n\t//lint:ignore SA1019 We want to retry on a bunch of errors here.\n\tif !tlsConfig.PreferServerCipherSuites { // nolint:staticcheck\n\t\tt.Fatal(\"Expected PreferServerCipherSuites to be true\")\n\t}\n\t// Verify hostname is correct in certificate\n\tif len(tlsConfig.Certificates) != 1 {\n\t\tt.Fatal(\"Expected 1 certificate\")\n\t}\n\tcert := tlsConfig.Certificates[0].Leaf\n\tif err := cert.VerifyHostname(\"127.0.0.1\"); err != nil {\n\t\tt.Fatalf(\"Could not verify hostname in certificate: %v\", err)\n\t}\n\n\t// First make sure that we can't add insecure cipher suites accidentally.\n\t_, err = ProcessConfigFile(\"./configs/tls_insecure_ciphers.conf\")\n\tif err == nil || !strings.Contains(err.Error(), \"insecure\") {\n\t\tt.Fatalf(\"Expected to receive insecure cipher error reading configuration file but didn't\")\n\t}\n\t_, err = ProcessConfigFile(\"./configs/tls_insecure_ciphers_allowed.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\t// Now test adding cipher suites.\n\topts, err = ProcessConfigFile(\"./configs/tls_ciphers.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\ttlsConfig = opts.TLSConfig\n\tif tlsConfig == nil {\n\t\tt.Fatal(\"Expected opts.TLSConfig to be non-nil\")\n\t}\n\n\t// CipherSuites listed in the config - test all of them.\n\tciphers = []uint16{\n\t\ttls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,\n\t\ttls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,\n\t\ttls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,\n\t\ttls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,\n\t\ttls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,\n\t\ttls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,\n\t}\n\n\tif !reflect.DeepEqual(tlsConfig.CipherSuites, ciphers) {\n\t\tt.Fatalf(\"Got incorrect cipher suite list: [%+v]\", tlsConfig.CipherSuites)\n\t}\n\n\t// Test an unrecognized/bad cipher\n\tif _, err := ProcessConfigFile(\"./configs/tls_bad_cipher.conf\"); err == nil {\n\t\tt.Fatal(\"Did not receive an error from a unrecognized cipher\")\n\t}\n\n\t// Test an empty cipher entry in a config file.\n\tif _, err := ProcessConfigFile(\"./configs/tls_empty_cipher.conf\"); err == nil {\n\t\tt.Fatal(\"Did not receive an error from empty cipher_suites\")\n\t}\n\n\t// Test a curve preference from the config.\n\tcurves := []tls.CurveID{\n\t\ttls.CurveP256,\n\t}\n\n\t// test on a file that will load the curve preference defaults\n\topts, err = ProcessConfigFile(\"./configs/tls_ciphers.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\tif !reflect.DeepEqual(opts.TLSConfig.CurvePreferences, defaultCurvePreferences()) {\n\t\tt.Fatalf(\"Got incorrect curve preference list: [%+v]\", tlsConfig.CurvePreferences)\n\t}\n\n\t// Test specifying a single curve preference\n\topts, err = ProcessConfigFile(\"./configs/tls_curve_prefs.conf\")\n\tif err != nil {\n\t\tt.Fatal(\"Did not receive an error from a unrecognized cipher.\")\n\t}\n\n\tif !reflect.DeepEqual(opts.TLSConfig.CurvePreferences, curves) {\n\t\tt.Fatalf(\"Got incorrect cipher suite list: [%+v]\", tlsConfig.CurvePreferences)\n\t}\n\n\t// Test an unrecognized/bad curve preference\n\tif _, err := ProcessConfigFile(\"./configs/tls_bad_curve_prefs.conf\"); err == nil {\n\t\tt.Fatal(\"Did not receive an error from a unrecognized curve preference\")\n\t}\n\t// Test an empty curve preference\n\tif _, err := ProcessConfigFile(\"./configs/tls_empty_curve_prefs.conf\"); err == nil {\n\t\tt.Fatal(\"Did not receive an error from empty curve preferences\")\n\t}\n}\n\nfunc TestMergeOverrides(t *testing.T) {\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/test.conf\",\n\t\tServerName: \"testing_server\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 2222,\n\t\tUsername: \"derek\",\n\t\tPassword: \"porkchop\",\n\t\tAuthTimeout: 1.0,\n\t\tDebug: true,\n\t\tTrace: true,\n\t\tLogtime: false,\n\t\tHTTPPort: DEFAULT_HTTP_PORT,\n\t\tHTTPBasePath: DEFAULT_HTTP_BASE_PATH,\n\t\tPidFile: \"/tmp/nats-server/nats-server.pid\",\n\t\tProfPort: 6789,\n\t\tSyslog: true,\n\t\tRemoteSyslog: \"udp://foo.com:33\",\n\t\tMaxControlLine: 2048,\n\t\tMaxPayload: 65536,\n\t\tMaxConn: 100,\n\t\tMaxSubs: 1000,\n\t\tMaxPending: 10000000,\n\t\tPingInterval: 60 * time.Second,\n\t\tMaxPingsOut: 3,\n\t\tCluster: ClusterOpts{\n\t\t\tNoAdvertise: true,\n\t\t\tConnectRetries: 2,\n\t\t},\n\t\tWriteDeadline: 3 * time.Second,\n\t\tLameDuckDuration: 4 * time.Minute,\n\t\tConnectErrorReports: 86400,\n\t\tReconnectErrorReports: 5,\n\t\tJetStream: true,\n\t\tStoreDir: \"/store/dir\",\n\t\tauthBlockDefined: true,\n\t\tMetadata: map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"},\n\t\tconfigDigest: \"sha256:a1104db0c8e838096a4f0509ec4d1e7c2c26ff60261ecb8f6a12dde1317872c3\",\n\t}\n\tfopts, err := ProcessConfigFile(\"./configs/test.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\t// Overrides via flags\n\topts := &Options{\n\t\tPort: 2222,\n\t\tPassword: \"porkchop\",\n\t\tDebug: true,\n\t\tHTTPPort: DEFAULT_HTTP_PORT,\n\t\tHTTPBasePath: DEFAULT_HTTP_BASE_PATH,\n\t\tProfPort: 6789,\n\t\tCluster: ClusterOpts{\n\t\t\tNoAdvertise: true,\n\t\t\tConnectRetries: 2,\n\t\t},\n\t\tJetStream: true,\n\t\tStoreDir: \"/store/dir\",\n\t}\n\tmerged := MergeOptions(fopts, opts)\n\n\tcheckOptionsEqual(t, golden, merged)\n}\n\nfunc TestRouteFlagOverride(t *testing.T) {\n\trouteFlag := \"nats-route://ruser:top_secret@127.0.0.1:8246\"\n\trurl, _ := url.Parse(routeFlag)\n\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/srv_a.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 7222,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: 7244,\n\t\t\tUsername: \"ruser\",\n\t\t\tPassword: \"top_secret\",\n\t\t\tAuthTimeout: 0.5,\n\t\t},\n\t\tRoutes: []*url.URL{rurl},\n\t\tRoutesStr: routeFlag,\n\t\tconfigDigest: \"sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979\",\n\t}\n\n\tfopts, err := ProcessConfigFile(\"./configs/srv_a.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\t// Overrides via flags\n\topts := &Options{\n\t\tRoutesStr: routeFlag,\n\t}\n\tmerged := MergeOptions(fopts, opts)\n\n\tcheckOptionsEqual(t, golden, merged)\n}\n\nfunc TestClusterFlagsOverride(t *testing.T) {\n\trouteFlag := \"nats-route://ruser:top_secret@127.0.0.1:7246\"\n\trurl, _ := url.Parse(routeFlag)\n\n\t// In this test, we override the cluster listen string. Note that in\n\t// the golden options, the cluster other infos correspond to what\n\t// is recovered from the configuration file, this explains the\n\t// discrepency between ClusterListenStr and the rest.\n\t// The server would then process the ClusterListenStr override and\n\t// correctly override ClusterHost/ClustherPort/etc..\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/srv_a.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 7222,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: 7244,\n\t\t\tListenStr: \"nats://127.0.0.1:8224\",\n\t\t\tUsername: \"ruser\",\n\t\t\tPassword: \"top_secret\",\n\t\t\tAuthTimeout: 0.5,\n\t\t},\n\t\tRoutes: []*url.URL{rurl},\n\t\tconfigDigest: \"sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979\",\n\t}\n\n\tfopts, err := ProcessConfigFile(\"./configs/srv_a.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\t// Overrides via flags\n\topts := &Options{\n\t\tCluster: ClusterOpts{\n\t\t\tListenStr: \"nats://127.0.0.1:8224\",\n\t\t},\n\t}\n\tmerged := MergeOptions(fopts, opts)\n\n\tcheckOptionsEqual(t, golden, merged)\n}\n\nfunc TestRouteFlagOverrideWithMultiple(t *testing.T) {\n\trouteFlag := \"nats-route://ruser:top_secret@127.0.0.1:8246, nats-route://ruser:top_secret@127.0.0.1:8266\"\n\trurls := RoutesFromStr(routeFlag)\n\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/srv_a.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 7222,\n\t\tCluster: ClusterOpts{\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tName: \"abc\",\n\t\t\tPort: 7244,\n\t\t\tUsername: \"ruser\",\n\t\t\tPassword: \"top_secret\",\n\t\t\tAuthTimeout: 0.5,\n\t\t},\n\t\tRoutes: rurls,\n\t\tRoutesStr: routeFlag,\n\t\tconfigDigest: \"sha256:fe3c13f82637723989a9bbd0ad6d064b95d48971666af440d4196d9c0d3af979\",\n\t}\n\n\tfopts, err := ProcessConfigFile(\"./configs/srv_a.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\t// Overrides via flags\n\topts := &Options{\n\t\tRoutesStr: routeFlag,\n\t}\n\tmerged := MergeOptions(fopts, opts)\n\n\tcheckOptionsEqual(t, golden, merged)\n}\n\nfunc TestDynamicPortOnListen(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/listen-1.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tif opts.Port != -1 {\n\t\tt.Fatalf(\"Received incorrect port %v, expected -1\", opts.Port)\n\t}\n\tif opts.HTTPPort != -1 {\n\t\tt.Fatalf(\"Received incorrect monitoring port %v, expected -1\", opts.HTTPPort)\n\t}\n\tif opts.HTTPSPort != -1 {\n\t\tt.Fatalf(\"Received incorrect secure monitoring port %v, expected -1\", opts.HTTPSPort)\n\t}\n}\n\nfunc TestListenConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/listen.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n\n\t// Normal clients\n\thost := \"10.0.1.22\"\n\tport := 4422\n\tmonHost := \"127.0.0.1\"\n\tif opts.Host != host {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, host)\n\t}\n\tif opts.HTTPHost != monHost {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.HTTPHost, monHost)\n\t}\n\tif opts.Port != port {\n\t\tt.Fatalf(\"Received incorrect port %v, expected %v\", opts.Port, port)\n\t}\n\n\t// Clustering\n\tclusterHost := \"127.0.0.1\"\n\tclusterPort := 4244\n\n\tif opts.Cluster.Host != clusterHost {\n\t\tt.Fatalf(\"Received incorrect cluster host %q, expected %q\", opts.Cluster.Host, clusterHost)\n\t}\n\tif opts.Cluster.Port != clusterPort {\n\t\tt.Fatalf(\"Received incorrect cluster port %v, expected %v\", opts.Cluster.Port, clusterPort)\n\t}\n\n\t// HTTP\n\thttpHost := \"127.0.0.1\"\n\thttpPort := 8422\n\n\tif opts.HTTPHost != httpHost {\n\t\tt.Fatalf(\"Received incorrect http host %q, expected %q\", opts.HTTPHost, httpHost)\n\t}\n\tif opts.HTTPPort != httpPort {\n\t\tt.Fatalf(\"Received incorrect http port %v, expected %v\", opts.HTTPPort, httpPort)\n\t}\n\n\t// HTTPS\n\thttpsPort := 9443\n\tif opts.HTTPSPort != httpsPort {\n\t\tt.Fatalf(\"Received incorrect https port %v, expected %v\", opts.HTTPSPort, httpsPort)\n\t}\n}\n\nfunc TestListenPortOnlyConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/listen_port.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n\n\tport := 8922\n\n\tif opts.Host != DEFAULT_HOST {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, DEFAULT_HOST)\n\t}\n\tif opts.HTTPHost != DEFAULT_HOST {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, DEFAULT_HOST)\n\t}\n\tif opts.Port != port {\n\t\tt.Fatalf(\"Received incorrect port %v, expected %v\", opts.Port, port)\n\t}\n}\n\nfunc TestListenPortWithColonConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/listen_port_with_colon.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n\n\tport := 8922\n\n\tif opts.Host != DEFAULT_HOST {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, DEFAULT_HOST)\n\t}\n\tif opts.HTTPHost != DEFAULT_HOST {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, DEFAULT_HOST)\n\t}\n\tif opts.Port != port {\n\t\tt.Fatalf(\"Received incorrect port %v, expected %v\", opts.Port, port)\n\t}\n}\n\nfunc TestListenMonitoringDefault(t *testing.T) {\n\topts := &Options{\n\t\tHost: \"10.0.1.22\",\n\t}\n\tsetBaselineOptions(opts)\n\n\thost := \"10.0.1.22\"\n\tif opts.Host != host {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, host)\n\t}\n\tif opts.HTTPHost != host {\n\t\tt.Fatalf(\"Received incorrect host %q, expected %q\", opts.Host, host)\n\t}\n\tif opts.Port != DEFAULT_PORT {\n\t\tt.Fatalf(\"Received incorrect port %v, expected %v\", opts.Port, DEFAULT_PORT)\n\t}\n}\n\nfunc TestMultipleUsersConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/multiple_users.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n}\n\n// Test highly depends on contents of the config file listed below. Any changes to that file\n// may very well break this test.\nfunc TestAuthorizationConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/authorization.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n\tlu := len(opts.Users)\n\tif lu != 5 {\n\t\tt.Fatalf(\"Expected 5 users, got %d\", lu)\n\t}\n\t// Build a map\n\tmu := make(map[string]*User)\n\tfor _, u := range opts.Users {\n\t\tmu[u.Username] = u\n\t}\n\n\t// Alice\n\talice, ok := mu[\"alice\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Alice\")\n\t}\n\t// Check for permissions details\n\tif alice.Permissions == nil {\n\t\tt.Fatalf(\"Expected Alice's permissions to be non-nil\")\n\t}\n\tif alice.Permissions.Publish == nil {\n\t\tt.Fatalf(\"Expected Alice's publish permissions to be non-nil\")\n\t}\n\tif len(alice.Permissions.Publish.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Alice's publish permissions to have 1 element, got %d\",\n\t\t\tlen(alice.Permissions.Publish.Allow))\n\t}\n\tpubPerm := alice.Permissions.Publish.Allow[0]\n\tif pubPerm != \"*\" {\n\t\tt.Fatalf(\"Expected Alice's publish permissions to be '*', got %q\", pubPerm)\n\t}\n\tif alice.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Alice's subscribe permissions to be non-nil\")\n\t}\n\tif len(alice.Permissions.Subscribe.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Alice's subscribe permissions to have 1 element, got %d\",\n\t\t\tlen(alice.Permissions.Subscribe.Allow))\n\t}\n\tsubPerm := alice.Permissions.Subscribe.Allow[0]\n\tif subPerm != \">\" {\n\t\tt.Fatalf(\"Expected Alice's subscribe permissions to be '>', got %q\", subPerm)\n\t}\n\n\t// Bob\n\tbob, ok := mu[\"bob\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Bob\")\n\t}\n\tif bob.Permissions == nil {\n\t\tt.Fatalf(\"Expected Bob's permissions to be non-nil\")\n\t}\n\n\t// Susan\n\tsusan, ok := mu[\"susan\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Susan\")\n\t}\n\tif susan.Permissions == nil {\n\t\tt.Fatalf(\"Expected Susan's permissions to be non-nil\")\n\t}\n\t// Check susan closely since she inherited the default permissions.\n\tif susan.Permissions == nil {\n\t\tt.Fatalf(\"Expected Susan's permissions to be non-nil\")\n\t}\n\tif susan.Permissions.Publish != nil {\n\t\tt.Fatalf(\"Expected Susan's publish permissions to be nil\")\n\t}\n\tif susan.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Susan's subscribe permissions to be non-nil\")\n\t}\n\tif len(susan.Permissions.Subscribe.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Susan's subscribe permissions to have 1 element, got %d\",\n\t\t\tlen(susan.Permissions.Subscribe.Allow))\n\t}\n\tsubPerm = susan.Permissions.Subscribe.Allow[0]\n\tif subPerm != \"PUBLIC.>\" {\n\t\tt.Fatalf(\"Expected Susan's subscribe permissions to be 'PUBLIC.>', got %q\", subPerm)\n\t}\n\n\t// Service A\n\tsvca, ok := mu[\"svca\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Service A\")\n\t}\n\tif svca.Permissions == nil {\n\t\tt.Fatalf(\"Expected Service A's permissions to be non-nil\")\n\t}\n\tif svca.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Service A's subscribe permissions to be non-nil\")\n\t}\n\tif len(svca.Permissions.Subscribe.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Service A's subscribe permissions to have 1 element, got %d\",\n\t\t\tlen(svca.Permissions.Subscribe.Allow))\n\t}\n\tsubPerm = svca.Permissions.Subscribe.Allow[0]\n\tif subPerm != \"my.service.req\" {\n\t\tt.Fatalf(\"Expected Service A's subscribe permissions to be 'my.service.req', got %q\", subPerm)\n\t}\n\t// We want allow_responses to essentially set deny all, or allow none in this case.\n\tif svca.Permissions.Publish == nil {\n\t\tt.Fatalf(\"Expected Service A's publish permissions to be non-nil\")\n\t}\n\tif len(svca.Permissions.Publish.Allow) != 0 {\n\t\tt.Fatalf(\"Expected Service A's publish permissions to have no elements, got %d\",\n\t\t\tlen(svca.Permissions.Publish.Allow))\n\t}\n\t// We should have a ResponsePermission present with default values.\n\tif svca.Permissions.Response == nil {\n\t\tt.Fatalf(\"Expected Service A's response permissions to be non-nil\")\n\t}\n\tif svca.Permissions.Response.MaxMsgs != DEFAULT_ALLOW_RESPONSE_MAX_MSGS {\n\t\tt.Fatalf(\"Expected Service A's response permissions of max msgs to be %d, got %d\",\n\t\t\tDEFAULT_ALLOW_RESPONSE_MAX_MSGS, svca.Permissions.Response.MaxMsgs,\n\t\t)\n\t}\n\tif svca.Permissions.Response.Expires != DEFAULT_ALLOW_RESPONSE_EXPIRATION {\n\t\tt.Fatalf(\"Expected Service A's response permissions of expiration to be %v, got %v\",\n\t\t\tDEFAULT_ALLOW_RESPONSE_EXPIRATION, svca.Permissions.Response.Expires,\n\t\t)\n\t}\n\n\t// Service B\n\tsvcb, ok := mu[\"svcb\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Service B\")\n\t}\n\tif svcb.Permissions == nil {\n\t\tt.Fatalf(\"Expected Service B's permissions to be non-nil\")\n\t}\n\tif svcb.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Service B's subscribe permissions to be non-nil\")\n\t}\n\tif len(svcb.Permissions.Subscribe.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Service B's subscribe permissions to have 1 element, got %d\",\n\t\t\tlen(svcb.Permissions.Subscribe.Allow))\n\t}\n\tsubPerm = svcb.Permissions.Subscribe.Allow[0]\n\tif subPerm != \"my.service.req\" {\n\t\tt.Fatalf(\"Expected Service B's subscribe permissions to be 'my.service.req', got %q\", subPerm)\n\t}\n\t// We want allow_responses to essentially set deny all, or allow none in this case.\n\tif svcb.Permissions.Publish == nil {\n\t\tt.Fatalf(\"Expected Service B's publish permissions to be non-nil\")\n\t}\n\tif len(svcb.Permissions.Publish.Allow) != 0 {\n\t\tt.Fatalf(\"Expected Service B's publish permissions to have no elements, got %d\",\n\t\t\tlen(svcb.Permissions.Publish.Allow))\n\t}\n\t// We should have a ResponsePermission present with default values.\n\tif svcb.Permissions.Response == nil {\n\t\tt.Fatalf(\"Expected Service B's response permissions to be non-nil\")\n\t}\n\tif svcb.Permissions.Response.MaxMsgs != 10 {\n\t\tt.Fatalf(\"Expected Service B's response permissions of max msgs to be %d, got %d\",\n\t\t\t10, svcb.Permissions.Response.MaxMsgs,\n\t\t)\n\t}\n\tif svcb.Permissions.Response.Expires != time.Minute {\n\t\tt.Fatalf(\"Expected Service B's response permissions of expiration to be %v, got %v\",\n\t\t\ttime.Minute, svcb.Permissions.Response.Expires,\n\t\t)\n\t}\n}\n\n// Test highly depends on contents of the config file listed below. Any changes to that file\n// may very well break this test.\nfunc TestNewStyleAuthorizationConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/new_style_authorization.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tsetBaselineOptions(opts)\n\n\tlu := len(opts.Users)\n\tif lu != 2 {\n\t\tt.Fatalf(\"Expected 2 users, got %d\", lu)\n\t}\n\t// Build a map\n\tmu := make(map[string]*User)\n\tfor _, u := range opts.Users {\n\t\tmu[u.Username] = u\n\t}\n\t// Alice\n\talice, ok := mu[\"alice\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Alice\")\n\t}\n\tif alice.Permissions == nil {\n\t\tt.Fatalf(\"Expected Alice's permissions to be non-nil\")\n\t}\n\n\tif alice.Permissions.Publish == nil {\n\t\tt.Fatalf(\"Expected Alice's publish permissions to be non-nil\")\n\t}\n\tif len(alice.Permissions.Publish.Allow) != 3 {\n\t\tt.Fatalf(\"Expected Alice's allowed publish permissions to have 3 elements, got %d\",\n\t\t\tlen(alice.Permissions.Publish.Allow))\n\t}\n\tpubPerm := alice.Permissions.Publish.Allow[0]\n\tif pubPerm != \"foo\" {\n\t\tt.Fatalf(\"Expected Alice's first allowed publish permission to be 'foo', got %q\", pubPerm)\n\t}\n\tpubPerm = alice.Permissions.Publish.Allow[1]\n\tif pubPerm != \"bar\" {\n\t\tt.Fatalf(\"Expected Alice's second allowed publish permission to be 'bar', got %q\", pubPerm)\n\t}\n\tpubPerm = alice.Permissions.Publish.Allow[2]\n\tif pubPerm != \"baz\" {\n\t\tt.Fatalf(\"Expected Alice's third allowed publish permission to be 'baz', got %q\", pubPerm)\n\t}\n\tif len(alice.Permissions.Publish.Deny) != 0 {\n\t\tt.Fatalf(\"Expected Alice's denied publish permissions to have 0 elements, got %d\",\n\t\t\tlen(alice.Permissions.Publish.Deny))\n\t}\n\n\tif alice.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Alice's subscribe permissions to be non-nil\")\n\t}\n\tif len(alice.Permissions.Subscribe.Allow) != 0 {\n\t\tt.Fatalf(\"Expected Alice's allowed subscribe permissions to have 0 elements, got %d\",\n\t\t\tlen(alice.Permissions.Subscribe.Allow))\n\t}\n\tif len(alice.Permissions.Subscribe.Deny) != 1 {\n\t\tt.Fatalf(\"Expected Alice's denied subscribe permissions to have 1 element, got %d\",\n\t\t\tlen(alice.Permissions.Subscribe.Deny))\n\t}\n\tsubPerm := alice.Permissions.Subscribe.Deny[0]\n\tif subPerm != \"$SYS.>\" {\n\t\tt.Fatalf(\"Expected Alice's only denied subscribe permission to be '$SYS.>', got %q\", subPerm)\n\t}\n\n\t// Bob\n\tbob, ok := mu[\"bob\"]\n\tif !ok {\n\t\tt.Fatalf(\"Expected to see user Bob\")\n\t}\n\tif bob.Permissions == nil {\n\t\tt.Fatalf(\"Expected Bob's permissions to be non-nil\")\n\t}\n\n\tif bob.Permissions.Publish == nil {\n\t\tt.Fatalf(\"Expected Bobs's publish permissions to be non-nil\")\n\t}\n\tif len(bob.Permissions.Publish.Allow) != 1 {\n\t\tt.Fatalf(\"Expected Bob's allowed publish permissions to have 1 element, got %d\",\n\t\t\tlen(bob.Permissions.Publish.Allow))\n\t}\n\tpubPerm = bob.Permissions.Publish.Allow[0]\n\tif pubPerm != \"$SYS.>\" {\n\t\tt.Fatalf(\"Expected Bob's first allowed publish permission to be '$SYS.>', got %q\", pubPerm)\n\t}\n\tif len(bob.Permissions.Publish.Deny) != 0 {\n\t\tt.Fatalf(\"Expected Bob's denied publish permissions to have 0 elements, got %d\",\n\t\t\tlen(bob.Permissions.Publish.Deny))\n\t}\n\n\tif bob.Permissions.Subscribe == nil {\n\t\tt.Fatalf(\"Expected Bob's subscribe permissions to be non-nil\")\n\t}\n\tif len(bob.Permissions.Subscribe.Allow) != 0 {\n\t\tt.Fatalf(\"Expected Bob's allowed subscribe permissions to have 0 elements, got %d\",\n\t\t\tlen(bob.Permissions.Subscribe.Allow))\n\t}\n\tif len(bob.Permissions.Subscribe.Deny) != 3 {\n\t\tt.Fatalf(\"Expected Bobs's denied subscribe permissions to have 3 elements, got %d\",\n\t\t\tlen(bob.Permissions.Subscribe.Deny))\n\t}\n\tsubPerm = bob.Permissions.Subscribe.Deny[0]\n\tif subPerm != \"foo\" {\n\t\tt.Fatalf(\"Expected Bobs's first denied subscribe permission to be 'foo', got %q\", subPerm)\n\t}\n\tsubPerm = bob.Permissions.Subscribe.Deny[1]\n\tif subPerm != \"bar\" {\n\t\tt.Fatalf(\"Expected Bobs's second denied subscribe permission to be 'bar', got %q\", subPerm)\n\t}\n\tsubPerm = bob.Permissions.Subscribe.Deny[2]\n\tif subPerm != \"baz\" {\n\t\tt.Fatalf(\"Expected Bobs's third denied subscribe permission to be 'baz', got %q\", subPerm)\n\t}\n}\n\n// Test new nkey users\nfunc TestNkeyUsersConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n authorization {\n users = [\n {nkey: \"UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV\"}\n {nkey: \"UA3C5TBZYK5GJQJRWPMU6NFY5JNAEVQB2V2TUZFZDHFJFUYVKTTUOFKZ\"}\n ]\n }`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tlu := len(opts.Nkeys)\n\tif lu != 2 {\n\t\tt.Fatalf(\"Expected 2 nkey users, got %d\", lu)\n\t}\n}\n\n// Test pinned certificates\nfunc TestTlsPinnedCertificates(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\ttls {\n\t\tcert_file: \"./configs/certs/server.pem\"\n\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t# Require a client certificate and map user id from certificate\n\t\tverify: true\n\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t}\n\tcluster {\n\t\tport -1\n\t\tname cluster-hub\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t\t}\n\t}\n\tleafnodes {\n\t\tport -1\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t\t}\n\t}\n\tgateway {\n\t\tname: \"A\"\n\t\tport -1\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t\t}\n\t}\n\twebsocket {\n\t\tport -1\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t\t}\n\t}\n\tmqtt {\n\t\tport -1\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\tpinned_certs: [\"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069\",\n\t\t\t\t\"a8f407340dcc719864214b85ed96f98d16cbffa8f509d9fa4ca237b7bb3f9c32\"]\n\t\t}\n\t}`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tcheck := func(set PinnedCertSet) {\n\t\tt.Helper()\n\t\tif l := len(set); l != 2 {\n\t\t\tt.Fatalf(\"Expected 2 pinned certificates, got got %d\", l)\n\t\t}\n\t}\n\n\tcheck(opts.TLSPinnedCerts)\n\tcheck(opts.LeafNode.TLSPinnedCerts)\n\tcheck(opts.Cluster.TLSPinnedCerts)\n\tcheck(opts.MQTT.TLSPinnedCerts)\n\tcheck(opts.Gateway.TLSPinnedCerts)\n\tcheck(opts.Websocket.TLSPinnedCerts)\n}\n\nfunc TestNkeyUsersDefaultPermissionsConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\tauthorization {\n\t\tdefault_permissions = {\n\t\t\tpublish = \"foo\"\n\t\t}\n\t\tusers = [\n\t\t\t{ user: \"user\", password: \"pwd\"}\n\t\t\t{ user: \"other\", password: \"pwd\",\n\t\t\t\tpermissions = {\n\t\t\t\t\tsubscribe = \"bar\"\n\t\t\t\t}\n\t\t\t}\n\t\t\t{ nkey: \"UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV\" }\n\t\t\t{ nkey: \"UA3C5TBZYK5GJQJRWPMU6NFY5JNAEVQB2V2TUZFZDHFJFUYVKTTUOFKZ\",\n\t\t\t\tpermissions = {\n\t\t\t\t\tsubscribe = \"bar\"\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\taccounts {\n\t\tA {\n\t\t\tdefault_permissions = {\n\t\t\t\tpublish = \"foo\"\n\t\t\t}\n\t\t\tusers = [\n\t\t\t\t{ user: \"accuser\", password: \"pwd\"}\n\t\t\t\t{ user: \"accother\", password: \"pwd\",\n\t\t\t\t\tpermissions = {\n\t\t\t\t\t\tsubscribe = \"bar\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t{ nkey: \"UC4YEYJHYKTU4LHROX7UEKEIO5RP5OUWDYXELHWXZOQHZYXHUD44LCRS\" }\n\t\t\t\t{ nkey: \"UDLSDF4UY3YW7JJQCYE6T2D4KFDCH6RGF3R65KHK247G3POJPI27VMQ3\",\n\t\t\t\t\tpermissions = {\n\t\t\t\t\t\tsubscribe = \"bar\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\tcheckPerms := func(permsDef *Permissions, permsNonDef *Permissions) {\n\t\tif permsDef.Publish.Allow[0] != \"foo\" {\n\t\t\tt.Fatal(\"Publish allow foo missing\")\n\t\t} else if permsDef.Subscribe != nil {\n\t\t\tt.Fatal(\"Has unexpected Subscribe permission\")\n\t\t} else if permsNonDef.Subscribe.Allow[0] != \"bar\" {\n\t\t\tt.Fatal(\"Subscribe allow bar missing\")\n\t\t} else if permsNonDef.Publish != nil {\n\t\t\tt.Fatal(\"Has unexpected Publish permission\")\n\t\t}\n\t}\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\n\tfindUsers := func(u1, u2 string) (found []*User) {\n\t\tfind := []string{u1, u2}\n\t\tfor _, f := range find {\n\t\t\tfor _, u := range opts.Users {\n\t\t\t\tif u.Username == f {\n\t\t\t\t\tfound = append(found, u)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n\n\tfindNkeyUsers := func(nk1, nk2 string) (found []*NkeyUser) {\n\t\tfind := []string{nk1, nk2}\n\t\tfor _, f := range find {\n\t\t\tfor _, u := range opts.Nkeys {\n\t\t\t\tif strings.HasPrefix(u.Nkey, f) {\n\t\t\t\t\tfound = append(found, u)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n\n\tif lu := len(opts.Users); lu != 4 {\n\t\tt.Fatalf(\"Expected 4 nkey users, got %d\", lu)\n\t}\n\tfoundU := findUsers(\"user\", \"other\")\n\tcheckPerms(foundU[0].Permissions, foundU[1].Permissions)\n\tfoundU = findUsers(\"accuser\", \"accother\")\n\tcheckPerms(foundU[0].Permissions, foundU[1].Permissions)\n\n\tif lu := len(opts.Nkeys); lu != 4 {\n\t\tt.Fatalf(\"Expected 4 nkey users, got %d\", lu)\n\t}\n\tfoundNk := findNkeyUsers(\"UDK\", \"UA3\")\n\tcheckPerms(foundNk[0].Permissions, foundNk[1].Permissions)\n\tfoundNk = findNkeyUsers(\"UC4\", \"UDL\")\n\tcheckPerms(foundNk[0].Permissions, foundNk[1].Permissions)\n}\n\nfunc TestNkeyUsersWithPermsConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n authorization {\n users = [\n {nkey: \"UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV\",\n permissions = {\n publish = \"$SYS.>\"\n subscribe = { deny = [\"foo\", \"bar\", \"baz\"] }\n }\n }\n ]\n }`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tlu := len(opts.Nkeys)\n\tif lu != 1 {\n\t\tt.Fatalf(\"Expected 1 nkey user, got %d\", lu)\n\t}\n\tnk := opts.Nkeys[0]\n\tif nk.Permissions == nil {\n\t\tt.Fatal(\"Expected to have permissions\")\n\t}\n\tif nk.Permissions.Publish == nil {\n\t\tt.Fatal(\"Expected to have publish permissions\")\n\t}\n\tif nk.Permissions.Publish.Allow[0] != \"$SYS.>\" {\n\t\tt.Fatalf(\"Expected publish to allow \\\"$SYS.>\\\", but got %v\", nk.Permissions.Publish.Allow[0])\n\t}\n\tif nk.Permissions.Subscribe == nil {\n\t\tt.Fatal(\"Expected to have subscribe permissions\")\n\t}\n\tif nk.Permissions.Subscribe.Allow != nil {\n\t\tt.Fatal(\"Expected to have no subscribe allow permissions\")\n\t}\n\tdeny := nk.Permissions.Subscribe.Deny\n\tif deny == nil || len(deny) != 3 ||\n\t\tdeny[0] != \"foo\" || deny[1] != \"bar\" || deny[2] != \"baz\" {\n\t\tt.Fatalf(\"Expected to have subscribe deny permissions, got %v\", deny)\n\t}\n}\n\nfunc TestBadNkeyConfig(t *testing.T) {\n\tconfFileName := \"nkeys_bad.conf\"\n\tcontent := `\n authorization {\n users = [ {nkey: \"Ufoo\"}]\n }`\n\tif err := os.WriteFile(confFileName, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tdefer removeFile(t, confFileName)\n\tif _, err := ProcessConfigFile(confFileName); err == nil {\n\t\tt.Fatalf(\"Expected an error from nkey entry with password\")\n\t}\n}\n\nfunc TestNkeyWithPassConfig(t *testing.T) {\n\tconfFileName := \"nkeys_pass.conf\"\n\tcontent := `\n authorization {\n users = [\n {nkey: \"UDKTV7HZVYJFJN64LLMYQBUR6MTNNYCDC3LAZH4VHURW3GZLL3FULBXV\", pass: \"foo\"}\n ]\n }`\n\tif err := os.WriteFile(confFileName, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tdefer removeFile(t, confFileName)\n\tif _, err := ProcessConfigFile(confFileName); err == nil {\n\t\tt.Fatalf(\"Expected an error from bad nkey entry\")\n\t}\n}\n\nfunc TestTokenWithUserPass(t *testing.T) {\n\tconfFileName := \"test.conf\"\n\tcontent := `\n\tauthorization={\n\t\tuser: user\n\t\tpass: password\n\t\ttoken: $2a$11$whatever\n\t}`\n\tif err := os.WriteFile(confFileName, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tdefer removeFile(t, confFileName)\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n\tif !strings.Contains(err.Error(), \"token\") {\n\t\tt.Fatalf(\"Expected error related to token, got %v\", err)\n\t}\n}\n\nfunc TestTokenWithUsers(t *testing.T) {\n\tconfFileName := \"test.conf\"\n\tcontent := `\n\tauthorization={\n\t\ttoken: $2a$11$whatever\n\t\tusers: [\n\t\t\t{user: test, password: test}\n\t\t]\n\t}`\n\tif err := os.WriteFile(confFileName, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tdefer removeFile(t, confFileName)\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n\tif !strings.Contains(err.Error(), \"token\") {\n\t\tt.Fatalf(\"Expected error related to token, got %v\", err)\n\t}\n}\n\nfunc TestParseWriteDeadline(t *testing.T) {\n\tconfFile := createConfFile(t, []byte(\"write_deadline: \\\"1x\\\"\"))\n\t_, err := ProcessConfigFile(confFile)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n\tif !strings.Contains(err.Error(), \"parsing\") {\n\t\tt.Fatalf(\"Expected error related to parsing, got %v\", err)\n\t}\n\tconfFile = createConfFile(t, []byte(\"write_deadline: \\\"1s\\\"\"))\n\topts, err := ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.WriteDeadline != time.Second {\n\t\tt.Fatalf(\"Expected write_deadline to be 1s, got %v\", opts.WriteDeadline)\n\t}\n\toldStdout := os.Stdout\n\t_, w, _ := os.Pipe()\n\tdefer func() {\n\t\tw.Close()\n\t\tos.Stdout = oldStdout\n\t}()\n\tos.Stdout = w\n\tconfFile = createConfFile(t, []byte(\"write_deadline: 2\"))\n\topts, err = ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.WriteDeadline != 2*time.Second {\n\t\tt.Fatalf(\"Expected write_deadline to be 2s, got %v\", opts.WriteDeadline)\n\t}\n}\n\nfunc TestOptionsClone(t *testing.T) {\n\topts := &Options{\n\t\tConfigFile: \"./configs/test.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 2222,\n\t\tUsername: \"derek\",\n\t\tPassword: \"porkchop\",\n\t\tAuthTimeout: 1.0,\n\t\tDebug: true,\n\t\tTrace: true,\n\t\tLogtime: false,\n\t\tHTTPPort: DEFAULT_HTTP_PORT,\n\t\tHTTPBasePath: DEFAULT_HTTP_BASE_PATH,\n\t\tPidFile: \"/tmp/nats-server/nats-server.pid\",\n\t\tProfPort: 6789,\n\t\tSyslog: true,\n\t\tRemoteSyslog: \"udp://foo.com:33\",\n\t\tMaxControlLine: 2048,\n\t\tMaxPayload: 65536,\n\t\tMaxConn: 100,\n\t\tPingInterval: 60 * time.Second,\n\t\tMaxPingsOut: 3,\n\t\tCluster: ClusterOpts{\n\t\t\tNoAdvertise: true,\n\t\t\tConnectRetries: 2,\n\t\t\tWriteDeadline: 3 * time.Second,\n\t\t},\n\t\tGateway: GatewayOpts{\n\t\t\tName: \"A\",\n\t\t\tGateways: []*RemoteGatewayOpts{\n\t\t\t\t{Name: \"B\", URLs: []*url.URL{{Scheme: \"nats\", Host: \"host:5222\"}}},\n\t\t\t\t{Name: \"C\"},\n\t\t\t},\n\t\t},\n\t\tWriteDeadline: 3 * time.Second,\n\t\tRoutes: []*url.URL{{}},\n\t\tUsers: []*User{{Username: \"foo\", Password: \"bar\"}},\n\t}\n\n\tclone := opts.Clone()\n\n\tif !reflect.DeepEqual(opts, clone) {\n\t\tt.Fatalf(\"Cloned Options are incorrect.\\nexpected: %+v\\ngot: %+v\",\n\t\t\tclone, opts)\n\t}\n\n\tclone.Users[0].Password = \"baz\"\n\tif reflect.DeepEqual(opts, clone) {\n\t\tt.Fatal(\"Expected Options to be different\")\n\t}\n\n\topts.Gateway.Gateways[0].URLs[0] = nil\n\tif reflect.DeepEqual(opts.Gateway.Gateways[0], clone.Gateway.Gateways[0]) {\n\t\tt.Fatal(\"Expected Options to be different\")\n\t}\n\tif clone.Gateway.Gateways[0].URLs[0].Host != \"host:5222\" {\n\t\tt.Fatalf(\"Unexpected URL: %v\", clone.Gateway.Gateways[0].URLs[0])\n\t}\n}\n\nfunc TestOptionsCloneNilLists(t *testing.T) {\n\topts := &Options{}\n\n\tclone := opts.Clone()\n\n\tif clone.Routes != nil {\n\t\tt.Fatalf(\"Expected Routes to be nil, got: %v\", clone.Routes)\n\t}\n\tif clone.Users != nil {\n\t\tt.Fatalf(\"Expected Users to be nil, got: %v\", clone.Users)\n\t}\n}\n\nfunc TestOptionsCloneNil(t *testing.T) {\n\topts := (*Options)(nil)\n\tclone := opts.Clone()\n\tif clone != nil {\n\t\tt.Fatalf(\"Expected nil, got: %+v\", clone)\n\t}\n}\n\nfunc TestEmptyConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"\")\n\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error from empty config, got: %+v\", err)\n\t}\n\n\tif opts.ConfigFile != \"\" {\n\t\tt.Fatalf(\"Expected empty config, got: %+v\", opts)\n\t}\n}\n\nfunc TestMalformedListenAddress(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/malformed_listen_address.conf\")\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error reading config file: got %+v\", opts)\n\t}\n}\n\nfunc TestMalformedClusterAddress(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/malformed_cluster_address.conf\")\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error reading config file: got %+v\", opts)\n\t}\n}\n\nfunc TestPanic(t *testing.T) {\n\tconf := createConfFile(t, []byte(`port: \"this_string_trips_a_panic\"`))\n\topts, err := ProcessConfigFile(conf)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error reading config file: got %+v\", opts)\n\t} else {\n\t\tif !strings.Contains(err.Error(), \":1:0: interface conversion:\") {\n\t\t\tt.Fatalf(\"This was supposed to trip a panic on interface conversion right at the beginning\")\n\t\t}\n\t}\n}\n\nfunc TestMaxClosedClients(t *testing.T) {\n\tconf := createConfFile(t, []byte(`max_closed_clients: 5`))\n\topts, err := ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\trequire_Equal(t, opts.MaxClosedClients, 5)\n}\n\nfunc TestPingIntervalOld(t *testing.T) {\n\tconf := createConfFile(t, []byte(`ping_interval: 5`))\n\topts := &Options{}\n\terr := opts.ProcessConfigFile(conf)\n\tif err == nil {\n\t\tt.Fatalf(\"expected an error\")\n\t}\n\terrTyped, ok := err.(*processConfigErr)\n\tif !ok {\n\t\tt.Fatalf(\"expected an error of type processConfigErr\")\n\t}\n\tif len(errTyped.warnings) != 1 {\n\t\tt.Fatalf(\"expected processConfigErr to have one warning\")\n\t}\n\tif len(errTyped.errors) != 0 {\n\t\tt.Fatalf(\"expected processConfigErr to have no error\")\n\t}\n\tif opts.PingInterval != 5*time.Second {\n\t\tt.Fatalf(\"expected ping interval to be 5 seconds\")\n\t}\n}\n\nfunc TestPingIntervalNew(t *testing.T) {\n\tconf := createConfFile(t, []byte(`ping_interval: \"5m\"`))\n\topts := &Options{}\n\tif err := opts.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"expected no error\")\n\t}\n\tif opts.PingInterval != 5*time.Minute {\n\t\tt.Fatalf(\"expected ping interval to be 5 minutes\")\n\t}\n}\n\nfunc TestOptionsProcessConfigFile(t *testing.T) {\n\t// Create options with default values of Debug and Trace\n\t// that are the opposite of what is in the config file.\n\t// Set another option that is not present in the config file.\n\tlogFileName := \"test.log\"\n\topts := &Options{\n\t\tDebug: true,\n\t\tTrace: false,\n\t\tLogFile: logFileName,\n\t}\n\tconfigFileName := \"./configs/test.conf\"\n\tif err := opts.ProcessConfigFile(configFileName); err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\t// Verify that values are as expected\n\tif opts.ConfigFile != configFileName {\n\t\tt.Fatalf(\"Expected ConfigFile to be set to %q, got %v\", configFileName, opts.ConfigFile)\n\t}\n\tif opts.Debug {\n\t\tt.Fatal(\"Debug option should have been set to false from config file\")\n\t}\n\tif !opts.Trace {\n\t\tt.Fatal(\"Trace option should have been set to true from config file\")\n\t}\n\tif opts.LogFile != logFileName {\n\t\tt.Fatalf(\"Expected LogFile to be %q, got %q\", logFileName, opts.LogFile)\n\t}\n}\n\nfunc TestConfigureOptions(t *testing.T) {\n\t// Options.Configure() will snapshot the flags. This is used by the reload code.\n\t// We need to set it back to nil otherwise it will impact reload tests.\n\tdefer func() { FlagSnapshot = nil }()\n\n\tch := make(chan bool, 1)\n\tcheckPrintInvoked := func() {\n\t\tch <- true\n\t}\n\tusage := func() { panic(\"should not get there\") }\n\tvar fs *flag.FlagSet\n\ttype testPrint struct {\n\t\targs []string\n\t\tversion, help, tlsHelp func()\n\t}\n\ttestFuncs := []testPrint{\n\t\t{[]string{\"-v\"}, checkPrintInvoked, usage, PrintTLSHelpAndDie},\n\t\t{[]string{\"version\"}, checkPrintInvoked, usage, PrintTLSHelpAndDie},\n\t\t{[]string{\"-h\"}, PrintServerAndExit, checkPrintInvoked, PrintTLSHelpAndDie},\n\t\t{[]string{\"help\"}, PrintServerAndExit, checkPrintInvoked, PrintTLSHelpAndDie},\n\t\t{[]string{\"-help_tls\"}, PrintServerAndExit, usage, checkPrintInvoked},\n\t}\n\tfor _, tf := range testFuncs {\n\t\tfs = flag.NewFlagSet(\"test\", flag.ContinueOnError)\n\t\topts, err := ConfigureOptions(fs, tf.args, tf.version, tf.help, tf.tlsHelp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on configure: %v\", err)\n\t\t}\n\t\tif opts != nil {\n\t\t\tt.Fatalf(\"Expected options to be nil, got %v\", opts)\n\t\t}\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatalf(\"Should have invoked print function for args=%v\", tf.args)\n\t\t}\n\t}\n\n\t// Helper function that expect parsing with given args to not produce an error.\n\tmustNotFail := func(args []string) *Options {\n\t\tfs := flag.NewFlagSet(\"test\", flag.ContinueOnError)\n\t\topts, err := ConfigureOptions(fs, args, PrintServerAndExit, fs.Usage, PrintTLSHelpAndDie)\n\t\tif err != nil {\n\t\t\tstackFatalf(t, \"Error on configure: %v\", err)\n\t\t}\n\t\treturn opts\n\t}\n\n\t// Helper function that expect configuration to fail.\n\texpectToFail := func(args []string, errContent ...string) {\n\t\tfs := flag.NewFlagSet(\"test\", flag.ContinueOnError)\n\t\t// Silence the flagSet so that on failure nothing is printed.\n\t\t// (flagSet would print error message about unknown flags, etc..)\n\t\tsilenceOuput := &bytes.Buffer{}\n\t\tfs.SetOutput(silenceOuput)\n\t\topts, err := ConfigureOptions(fs, args, PrintServerAndExit, fs.Usage, PrintTLSHelpAndDie)\n\t\tif opts != nil || err == nil {\n\t\t\tstackFatalf(t, \"Expected no option and an error, got opts=%v and err=%v\", opts, err)\n\t\t}\n\t\tfor _, testErr := range errContent {\n\t\t\tif strings.Contains(err.Error(), testErr) {\n\t\t\t\t// We got the error we wanted.\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tstackFatalf(t, \"Expected errors containing any of those %v, got %v\", errContent, err)\n\t}\n\n\t// Basic test with port number\n\topts := mustNotFail([]string{\"-p\", \"1234\"})\n\tif opts.Port != 1234 {\n\t\tt.Fatalf(\"Expected port to be 1234, got %v\", opts.Port)\n\t}\n\n\t// Should fail because of unknown parameter\n\texpectToFail([]string{\"foo\"}, \"command\")\n\n\t// Should fail because unknown flag\n\texpectToFail([]string{\"-xxx\", \"foo\"}, \"flag\")\n\n\t// Should fail because of config file missing\n\texpectToFail([]string{\"-c\", \"xxx.cfg\"}, \"file\")\n\n\t// Should fail because of too many args for signal command\n\texpectToFail([]string{\"-sl\", \"quit=pid=foo\"}, \"signal\")\n\n\t// Should fail because of invalid pid\n\t// On windows, if not running with admin privileges, you would get access denied.\n\texpectToFail([]string{\"-sl\", \"quit=pid\"}, \"pid\", \"denied\")\n\n\t// The config file set Trace to true.\n\topts = mustNotFail([]string{\"-c\", \"./configs/test.conf\"})\n\tif !opts.Trace {\n\t\tt.Fatal(\"Trace should have been set to true\")\n\t}\n\n\t// The config file set Trace to true, but was overridden by param -V=false\n\topts = mustNotFail([]string{\"-c\", \"./configs/test.conf\", \"-V=false\"})\n\tif opts.Trace {\n\t\tt.Fatal(\"Trace should have been set to false\")\n\t}\n\n\t// The config file set Trace to true, but was overridden by param -DV=false\n\topts = mustNotFail([]string{\"-c\", \"./configs/test.conf\", \"-DV=false\"})\n\tif opts.Debug || opts.Trace {\n\t\tt.Fatal(\"Debug and Trace should have been set to false\")\n\t}\n\n\t// The config file set Trace to true, but was overridden by param -DV\n\topts = mustNotFail([]string{\"-c\", \"./configs/test.conf\", \"-DV\"})\n\tif !opts.Debug || !opts.Trace {\n\t\tt.Fatal(\"Debug and Trace should have been set to true\")\n\t}\n\n\t// This should fail since -cluster is missing\n\texpectedURL, _ := url.Parse(\"nats://127.0.0.1:6223\")\n\texpectToFail([]string{\"-routes\", expectedURL.String()}, \"solicited routes\")\n\n\t// Ensure that we can set cluster and routes from command line\n\topts = mustNotFail([]string{\"-cluster\", \"nats://127.0.0.1:6222\", \"-routes\", expectedURL.String()})\n\tif opts.Cluster.ListenStr != \"nats://127.0.0.1:6222\" {\n\t\tt.Fatalf(\"Unexpected Cluster.ListenStr=%q\", opts.Cluster.ListenStr)\n\t}\n\tif opts.RoutesStr != \"nats://127.0.0.1:6223\" || len(opts.Routes) != 1 || opts.Routes[0].String() != expectedURL.String() {\n\t\tt.Fatalf(\"Unexpected RoutesStr: %q and Routes: %v\", opts.RoutesStr, opts.Routes)\n\t}\n\n\t// Use a config with cluster configuration and explicit route defined.\n\t// Override with empty routes string.\n\topts = mustNotFail([]string{\"-c\", \"./configs/srv_a.conf\", \"-routes\", \"\"})\n\tif opts.RoutesStr != \"\" || len(opts.Routes) != 0 {\n\t\tt.Fatalf(\"Unexpected RoutesStr: %q and Routes: %v\", opts.RoutesStr, opts.Routes)\n\t}\n\n\t// Use a config with cluster configuration and override cluster listen string\n\texpectedURL, _ = url.Parse(\"nats-route://ruser:top_secret@127.0.0.1:7246\")\n\topts = mustNotFail([]string{\"-c\", \"./configs/srv_a.conf\", \"-cluster\", \"nats://ivan:pwd@127.0.0.1:6222\"})\n\tif opts.Cluster.Username != \"ivan\" || opts.Cluster.Password != \"pwd\" || opts.Cluster.Port != 6222 ||\n\t\tlen(opts.Routes) != 1 || opts.Routes[0].String() != expectedURL.String() {\n\t\tt.Fatalf(\"Unexpected Cluster and/or Routes: %#v - %v\", opts.Cluster, opts.Routes)\n\t}\n\n\t// Disable clustering from command line\n\topts = mustNotFail([]string{\"-c\", \"./configs/srv_a.conf\", \"-cluster\", \"\"})\n\tif opts.Cluster.Port != 0 {\n\t\tt.Fatalf(\"Unexpected Cluster: %v\", opts.Cluster)\n\t}\n\n\t// Various erros due to malformed cluster listen string.\n\t// (adding -routes to have more than 1 set flag to check\n\t// that Visit() stops when an error is found).\n\texpectToFail([]string{\"-cluster\", \":\", \"-routes\", \"\"}, \"protocol\")\n\texpectToFail([]string{\"-cluster\", \"nats://127.0.0.1\", \"-routes\", \"\"}, \"port\")\n\texpectToFail([]string{\"-cluster\", \"nats://127.0.0.1:xxx\", \"-routes\", \"\"}, \"invalid port\")\n\texpectToFail([]string{\"-cluster\", \"nats://ivan:127.0.0.1:6222\", \"-routes\", \"\"}, \"colons\")\n\texpectToFail([]string{\"-cluster\", \"nats://ivan@127.0.0.1:6222\", \"-routes\", \"\"}, \"password\")\n\n\t// Override config file's TLS configuration from command line, and completely disable TLS\n\topts = mustNotFail([]string{\"-c\", \"./configs/tls.conf\", \"-tls=false\"})\n\tif opts.TLSConfig != nil || opts.TLS {\n\t\tt.Fatal(\"Expected TLS to be disabled\")\n\t}\n\t// Override config file's TLS configuration from command line, and force TLS verification.\n\t// However, since TLS config has to be regenerated, user need to provide -tlscert and -tlskey too.\n\t// So this should fail.\n\texpectToFail([]string{\"-c\", \"./configs/tls.conf\", \"-tlsverify\"}, \"valid\")\n\n\t// Now same than above, but with all valid params.\n\topts = mustNotFail([]string{\"-c\", \"./configs/tls.conf\", \"-tlsverify\", \"-tlscert\", \"./configs/certs/server.pem\", \"-tlskey\", \"./configs/certs/key.pem\"})\n\tif opts.TLSConfig == nil || !opts.TLSVerify {\n\t\tt.Fatal(\"Expected TLS to be configured and force verification\")\n\t}\n\n\t// Configure TLS, but some TLS params missing\n\texpectToFail([]string{\"-tls\"}, \"valid\")\n\texpectToFail([]string{\"-tls\", \"-tlscert\", \"./configs/certs/server.pem\"}, \"valid\")\n\t// One of the file does not exist\n\texpectToFail([]string{\"-tls\", \"-tlscert\", \"./configs/certs/server.pem\", \"-tlskey\", \"./configs/certs/notfound.pem\"}, \"file\")\n\n\t// Configure TLS and check that this results in a TLSConfig option.\n\topts = mustNotFail([]string{\"-tls\", \"-tlscert\", \"./configs/certs/server.pem\", \"-tlskey\", \"./configs/certs/key.pem\"})\n\tif opts.TLSConfig == nil || !opts.TLS {\n\t\tt.Fatal(\"Expected TLSConfig to be set\")\n\t}\n\t// Check that we use default TLS ciphers\n\tif !reflect.DeepEqual(opts.TLSConfig.CipherSuites, defaultCipherSuites()) {\n\t\tt.Fatalf(\"Default ciphers not set, expected %v, got %v\", defaultCipherSuites(), opts.TLSConfig.CipherSuites)\n\t}\n}\n\nfunc TestClusterPermissionsConfig(t *testing.T) {\n\ttemplate := `\n\t\tcluster {\n\t\t\tport: 1234\n\t\t\t%s\n\t\t\tauthorization {\n\t\t\t\tuser: ivan\n\t\t\t\tpassword: pwd\n\t\t\t\tpermissions {\n\t\t\t\t\timport {\n\t\t\t\t\t\tallow: \"foo\"\n\t\t\t\t\t}\n\t\t\t\t\texport {\n\t\t\t\t\t\tallow: \"bar\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\topts, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tif cerr, ok := err.(*processConfigErr); ok && len(cerr.Errors()) > 0 {\n\t\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t\t}\n\t}\n\tif opts.Cluster.Permissions == nil {\n\t\tt.Fatal(\"Expected cluster permissions to be set\")\n\t}\n\tif opts.Cluster.Permissions.Import == nil {\n\t\tt.Fatal(\"Expected cluster import permissions to be set\")\n\t}\n\tif len(opts.Cluster.Permissions.Import.Allow) != 1 || opts.Cluster.Permissions.Import.Allow[0] != \"foo\" {\n\t\tt.Fatalf(\"Expected cluster import permissions to have %q, got %v\", \"foo\", opts.Cluster.Permissions.Import.Allow)\n\t}\n\tif opts.Cluster.Permissions.Export == nil {\n\t\tt.Fatal(\"Expected cluster export permissions to be set\")\n\t}\n\tif len(opts.Cluster.Permissions.Export.Allow) != 1 || opts.Cluster.Permissions.Export.Allow[0] != \"bar\" {\n\t\tt.Fatalf(\"Expected cluster export permissions to have %q, got %v\", \"bar\", opts.Cluster.Permissions.Export.Allow)\n\t}\n\n\t// Now add permissions in top level cluster and check\n\t// that this is the one that is being used.\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, `\n\t\tpermissions {\n\t\t\timport {\n\t\t\t\tallow: \"baz\"\n\t\t\t}\n\t\t\texport {\n\t\t\t\tallow: \"bat\"\n\t\t\t}\n\t\t}\n\t`)))\n\topts, err = ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\tif opts.Cluster.Permissions == nil {\n\t\tt.Fatal(\"Expected cluster permissions to be set\")\n\t}\n\tif opts.Cluster.Permissions.Import == nil {\n\t\tt.Fatal(\"Expected cluster import permissions to be set\")\n\t}\n\tif len(opts.Cluster.Permissions.Import.Allow) != 1 || opts.Cluster.Permissions.Import.Allow[0] != \"baz\" {\n\t\tt.Fatalf(\"Expected cluster import permissions to have %q, got %v\", \"baz\", opts.Cluster.Permissions.Import.Allow)\n\t}\n\tif opts.Cluster.Permissions.Export == nil {\n\t\tt.Fatal(\"Expected cluster export permissions to be set\")\n\t}\n\tif len(opts.Cluster.Permissions.Export.Allow) != 1 || opts.Cluster.Permissions.Export.Allow[0] != \"bat\" {\n\t\tt.Fatalf(\"Expected cluster export permissions to have %q, got %v\", \"bat\", opts.Cluster.Permissions.Export.Allow)\n\t}\n\n\t// Tests with invalid permissions\n\tinvalidPerms := []string{\n\t\t`permissions: foo`,\n\t\t`permissions {\n\t\t\tunknown_field: \"foo\"\n\t\t}`,\n\t\t`permissions {\n\t\t\timport: [1, 2, 3]\n\t\t}`,\n\t\t`permissions {\n\t\t\timport {\n\t\t\t\tunknown_field: \"foo\"\n\t\t\t}\n\t\t}`,\n\t\t`permissions {\n\t\t\timport {\n\t\t\t\tallow {\n\t\t\t\t\tx: y\n\t\t\t\t}\n\t\t\t}\n\t\t}`,\n\t\t`permissions {\n\t\t\timport {\n\t\t\t\tdeny {\n\t\t\t\t\tx: y\n\t\t\t\t}\n\t\t\t}\n\t\t}`,\n\t\t`permissions {\n\t\t\texport: [1, 2, 3]\n\t\t}`,\n\t\t`permissions {\n\t\t\texport {\n\t\t\t\tunknown_field: \"foo\"\n\t\t\t}\n\t\t}`,\n\t\t`permissions {\n\t\t\texport {\n\t\t\t\tallow {\n\t\t\t\t\tx: y\n\t\t\t\t}\n\t\t\t}\n\t\t}`,\n\t\t`permissions {\n\t\t\texport {\n\t\t\t\tdeny {\n\t\t\t\t\tx: y\n\t\t\t\t}\n\t\t\t}\n\t\t}`,\n\t}\n\tfor _, perms := range invalidPerms {\n\t\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tcluster {\n\t\t\t\tport: 1234\n\t\t\t\t%s\n\t\t\t}\n\t\t`, perms)))\n\t\t_, err := ProcessConfigFile(conf)\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"Expected failure for permissions %s\", perms)\n\t\t}\n\t}\n\n\tfor _, perms := range invalidPerms {\n\t\tconf = createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tcluster {\n\t\t\t\tport: 1234\n\t\t\t\tauthorization {\n\t\t\t\t\tuser: ivan\n\t\t\t\t\tpassword: pwd\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t}\n\t\t`, perms)))\n\t\t_, err := ProcessConfigFile(conf)\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"Expected failure for permissions %s\", perms)\n\t\t}\n\t}\n}\n\nfunc TestParseServiceLatency(t *testing.T) {\n\tcases := []struct {\n\t\tname string\n\t\tconf string\n\t\twant *serviceLatency\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"block with percent sample default value\",\n\t\t\tconf: `system_account = nats.io\n\t\t\taccounts {\n\t\t\t\tnats.io {\n\t\t\t\t\texports [{\n\t\t\t\t\t\tservice: nats.add\n\t\t\t\t\t\tlatency: {\n\t\t\t\t\t\t\tsampling: 100%\n\t\t\t\t\t\t\tsubject: latency.tracking.add\n\t\t\t\t\t\t}\n\t\t\t\t\t}]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\twant: &serviceLatency{\n\t\t\t\tsubject: \"latency.tracking.add\",\n\t\t\t\tsampling: 100,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"block with percent sample nondefault value\",\n\t\t\tconf: `system_account = nats.io\n\t\t\taccounts {\n\t\t\t\tnats.io {\n\t\t\t\t\texports [{\n\t\t\t\t\t\tservice: nats.add\n\t\t\t\t\t\tlatency: {\n\t\t\t\t\t\t\tsampling: 33%\n\t\t\t\t\t\t\tsubject: latency.tracking.add\n\t\t\t\t\t\t}\n\t\t\t\t\t}]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\twant: &serviceLatency{\n\t\t\t\tsubject: \"latency.tracking.add\",\n\t\t\t\tsampling: 33,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"block with number sample nondefault value\",\n\t\t\tconf: `system_account = nats.io\n\t\t\taccounts {\n\t\t\t\tnats.io {\n\t\t\t\t\texports [{\n\t\t\t\t\t\tservice: nats.add\n\t\t\t\t\t\tlatency: {\n\t\t\t\t\t\t\tsampling: 87\n\t\t\t\t\t\t\tsubject: latency.tracking.add\n\t\t\t\t\t\t}\n\t\t\t\t\t}]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\twant: &serviceLatency{\n\t\t\t\tsubject: \"latency.tracking.add\",\n\t\t\t\tsampling: 87,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"field with subject\",\n\t\t\tconf: `system_account = nats.io\n\t\t\taccounts {\n\t\t\t\tnats.io {\n\t\t\t\t\texports [{\n\t\t\t\t\t\tservice: nats.add\n\t\t\t\t\t\tlatency: latency.tracking.add\n\t\t\t\t\t}]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\twant: &serviceLatency{\n\t\t\t\tsubject: \"latency.tracking.add\",\n\t\t\t\tsampling: 100,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"block with missing subject\",\n\t\t\tconf: `system_account = nats.io\n\t\t\taccounts {\n\t\t\t\tnats.io {\n\t\t\t\t\texports [{\n\t\t\t\t\t\tservice: nats.add\n\t\t\t\t\t\tlatency: {\n\t\t\t\t\t\t\tsampling: 87\n\t\t\t\t\t\t}\n\t\t\t\t\t}]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\twantErr: true,\n\t\t},\n\t}\n\n\tfor _, c := range cases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\tf := createConfFile(t, []byte(c.conf))\n\t\t\topts, err := ProcessConfigFile(f)\n\t\t\tswitch {\n\t\t\tcase c.wantErr && err == nil:\n\t\t\t\tt.Fatalf(\"Expected ProcessConfigFile to fail, but didn't\")\n\t\t\tcase c.wantErr && err != nil:\n\t\t\t\t// We wanted an error and got one, test passed.\n\t\t\t\treturn\n\t\t\tcase !c.wantErr && err == nil:\n\t\t\t\t// We didn't want an error and didn't get one, keep going.\n\t\t\t\tbreak\n\t\t\tcase !c.wantErr && err != nil:\n\t\t\t\tt.Fatalf(\"Failed to process config: %v\", err)\n\t\t\t}\n\n\t\t\tif len(opts.Accounts) != 1 {\n\t\t\t\tt.Fatalf(\"Expected accounts to have len %d, got %d\", 1, len(opts.Accounts))\n\t\t\t}\n\t\t\tif len(opts.Accounts[0].exports.services) != 1 {\n\t\t\t\tt.Fatalf(\"Expected export services to have len %d, got %d\", 1, len(opts.Accounts[0].exports.services))\n\t\t\t}\n\t\t\ts, ok := opts.Accounts[0].exports.services[\"nats.add\"]\n\t\t\tif !ok {\n\t\t\t\tt.Fatalf(\"Expected export service nats.add, missing\")\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(s.latency, c.want) {\n\t\t\t\tt.Fatalf(\"Expected latency to be %#v, got %#v\", c.want, s.latency)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseExport(t *testing.T) {\n\tconf := `\n\t\tport: -1\n\t\tsystem_account: sys\n\t\taccounts {\n\t\t\tsys {\n\t\t\t\texports [{\n\t\t\t\t\tstream \"$SYS.SERVER.ACCOUNT.*.CONNS\"\n\t\t\t\t\taccount_token_position 4\n\t\t\t\t}]\n\t\t\t}\n\t\t\taccE {\n\t\t\t\texports [{\n\t\t\t\t\tservice foo.*\n\t\t\t\t\taccount_token_position 2\n\t\t\t\t}]\n\t\t\t\tusers [{\n\t\t\t\t\tuser ue\n\t\t\t\t\tpassword pwd\n\t\t\t\t}],\n\t\t\t}\n\t\t\taccI1 {\n\t\t\t\timports [{\n\t\t\t\t\tservice {\n\t\t\t\t\t\taccount accE\n\t\t\t\t\t\tsubject foo.accI1\n\t\t\t\t\t}\n\t\t\t\t\tto foo\n\t\t\t\t},{\n\t\t\t\t\tstream {\n\t\t\t\t\t\taccount sys\n\t\t\t\t\t\tsubject \"$SYS.SERVER.ACCOUNT.accI1.CONNS\"\n\t\t\t\t\t}\n\t\t\t\t}],\n\t\t\t\tusers [{\n\t\t\t\t\tuser u1\n\t\t\t\t\tpassword pwd\n\t\t\t\t}],\n\t\t\t}\n\t\t\taccI2 {\n\t\t\t\timports [{\n\t\t\t\t\tservice {\n\t\t\t\t\t\taccount accE\n\t\t\t\t\t\tsubject foo.accI2\n\t\t\t\t\t}\n\t\t\t\t\tto foo\n\t\t\t\t},{\n\t\t\t\t\tstream {\n\t\t\t\t\t\taccount sys\n\t\t\t\t\t\tsubject \"$SYS.SERVER.ACCOUNT.accI2.CONNS\"\n\t\t\t\t\t}\n\t\t\t\t}],\n\t\t\t\tusers [{\n\t\t\t\t\tuser u2\n\t\t\t\t\tpassword pwd\n\t\t\t\t}],\n\t\t\t}\n\t\t}`\n\tf := createConfFile(t, []byte(conf))\n\ts, o := RunServerWithConfig(f)\n\tif s == nil {\n\t\tt.Fatal(\"Failed startup\")\n\t}\n\tdefer s.Shutdown()\n\tconnect := func(user string) *nats.Conn {\n\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:pwd@%s:%d\", user, o.Host, o.Port))\n\t\trequire_NoError(t, err)\n\t\treturn nc\n\t}\n\tnc1 := connect(\"u1\")\n\tdefer nc1.Close()\n\tnc2 := connect(\"u2\")\n\tdefer nc2.Close()\n\n\t// Due to the fact that above CONNECT events are generated and sent from\n\t// a system go routine, it is possible that by the time we create the\n\t// subscriptions below, the interest would exist and messages be sent,\n\t// which was causing issues since wg.Done() was called too many times.\n\t// Add a little delay to minimize risk, but also use counter to decide\n\t// when to call wg.Done() to avoid panic due to negative number.\n\ttime.Sleep(100 * time.Millisecond)\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tcount := int32(0)\n\t// We expect a total of 6 messages\n\texpected := int32(6)\n\tsubscribe := func(nc *nats.Conn, subj string) {\n\t\tt.Helper()\n\t\t_, err := nc.Subscribe(subj, func(msg *nats.Msg) {\n\t\t\tif msg.Reply != _EMPTY_ {\n\t\t\t\tmsg.Respond(msg.Data)\n\t\t\t}\n\t\t\tif atomic.AddInt32(&count, 1) == expected {\n\t\t\t\twg.Done()\n\t\t\t}\n\t\t})\n\t\trequire_NoError(t, err)\n\t\tnc.Flush()\n\t}\n\t//Subscribe to CONNS events\n\tsubscribe(nc1, \"$SYS.SERVER.ACCOUNT.accI1.CONNS\")\n\tsubscribe(nc2, \"$SYS.SERVER.ACCOUNT.accI2.CONNS\")\n\t// Trigger 2 CONNS event\n\tnc3 := connect(\"u1\")\n\tnc3.Close()\n\tnc4 := connect(\"u2\")\n\tnc4.Close()\n\t// test service\n\tncE := connect(\"ue\")\n\tdefer ncE.Close()\n\tsubscribe(ncE, \"foo.*\")\n\trequest := func(nc *nats.Conn, msg string) {\n\t\tif m, err := nc.Request(\"foo\", []byte(msg), time.Second); err != nil {\n\t\t\tt.Fatal(\"Failed request \", msg, err)\n\t\t} else if m == nil {\n\t\t\tt.Fatal(\"No response msg\")\n\t\t} else if string(m.Data) != msg {\n\t\t\tt.Fatal(\"Wrong response msg\", string(m.Data))\n\t\t}\n\t}\n\trequest(nc1, \"1\")\n\trequest(nc2, \"1\")\n\twg.Wait()\n}\n\nfunc TestAccountUsersLoadedProperly(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\tauthorization {\n\t\tusers [\n\t\t\t{user: ivan, password: bar}\n\t\t\t{nkey : UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX}\n\t\t]\n\t}\n\taccounts {\n\t\tsynadia {\n\t\t\tusers [\n\t\t\t\t{user: derek, password: foo}\n\t\t\t\t{nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\tcheck := func(t *testing.T) {\n\t\tt.Helper()\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\t\topts := s.getOpts()\n\t\tif n := len(opts.Users); n != 2 {\n\t\t\tt.Fatalf(\"Should have 2 users, got %v\", n)\n\t\t}\n\t\tif n := len(opts.Nkeys); n != 2 {\n\t\t\tt.Fatalf(\"Should have 2 nkeys, got %v\", n)\n\t\t}\n\t}\n\t// Repeat test since issue was with ordering of processing\n\t// of authorization vs accounts that depends on range of a map (after actual parsing)\n\tfor i := 0; i < 20; i++ {\n\t\tcheck(t)\n\t}\n}\n\nfunc TestParsingGateways(t *testing.T) {\n\tcontent := `\n\tgateway {\n\t\tname: \"A\"\n\t\tlisten: \"127.0.0.1:4444\"\n\t\thost: \"127.0.0.1\"\n\t\tport: 4444\n\t\treject_unknown_cluster: true\n\t\tauthorization {\n\t\t\tuser: \"ivan\"\n\t\t\tpassword: \"pwd\"\n\t\t\ttimeout: 2.0\n\t\t}\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\ttimeout: 3.0\n\t\t}\n\t\tadvertise: \"me:1\"\n\t\tconnect_retries: 10\n\t\tconnect_backoff: true\n\t\tgateways: [\n\t\t\t{\n\t\t\t\tname: \"B\"\n\t\t\t\turls: [\"nats://user1:pwd1@host2:5222\", \"nats://user1:pwd1@host3:6222\"]\n\t\t\t}\n\t\t\t{\n\t\t\t\tname: \"C\"\n\t\t\t\turl: \"nats://host4:7222\"\n\t\t\t}\n\t\t]\n\t}\n\t`\n\tfile := \"server_config_gateways.conf\"\n\tif err := os.WriteFile(file, []byte(content), 0600); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tdefer removeFile(t, file)\n\topts, err := ProcessConfigFile(file)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing file: %v\", err)\n\t}\n\n\texpected := &GatewayOpts{\n\t\tName: \"A\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 4444,\n\t\tUsername: \"ivan\",\n\t\tPassword: \"pwd\",\n\t\tAuthTimeout: 2.0,\n\t\tAdvertise: \"me:1\",\n\t\tConnectRetries: 10,\n\t\tConnectBackoff: true,\n\t\tTLSTimeout: 3.0,\n\t\tRejectUnknown: true,\n\t}\n\tu1, _ := url.Parse(\"nats://user1:pwd1@host2:5222\")\n\tu2, _ := url.Parse(\"nats://user1:pwd1@host3:6222\")\n\turls := []*url.URL{u1, u2}\n\tgw := &RemoteGatewayOpts{\n\t\tName: \"B\",\n\t\tURLs: urls,\n\t}\n\texpected.Gateways = append(expected.Gateways, gw)\n\n\tu1, _ = url.Parse(\"nats://host4:7222\")\n\turls = []*url.URL{u1}\n\tgw = &RemoteGatewayOpts{\n\t\tName: \"C\",\n\t\tURLs: urls,\n\t}\n\texpected.Gateways = append(expected.Gateways, gw)\n\n\t// Just make sure that TLSConfig is set.. we have aother test\n\t// to check proper generating TLSConfig from config file...\n\tif opts.Gateway.TLSConfig == nil {\n\t\tt.Fatalf(\"Expected TLSConfig, got none\")\n\t}\n\topts.Gateway.TLSConfig = nil\n\topts.Gateway.tlsConfigOpts = nil\n\tif !reflect.DeepEqual(&opts.Gateway, expected) {\n\t\tt.Fatalf(\"Expected %v, got %v\", expected, opts.Gateway)\n\t}\n}\n\nfunc TestParsingGatewaysErrors(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcontent string\n\t\texpectedErr string\n\t}{\n\t\t{\n\t\t\t\"bad_type\",\n\t\t\t`gateway: \"bad_type\"`,\n\t\t\t\"Expected gateway to be a map\",\n\t\t},\n\t\t{\n\t\t\t\"bad_listen\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\tlisten: \"bad::address\"\n\t\t\t}`,\n\t\t\t\"parse address\",\n\t\t},\n\t\t{\n\t\t\t\"bad_auth\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tusers {\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"be an array\",\n\t\t},\n\t\t{\n\t\t\t\"unknown_field\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\treject_unknown_cluster: true\n\t\t\t\tunknown_field: 1\n\t\t\t}`,\n\t\t\t\"unknown field\",\n\t\t},\n\t\t{\n\t\t\t\"users_not_supported\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tusers [\n\t\t\t\t\t\t{user: alice, password: foo}\n\t\t\t\t\t\t{user: bob, password: bar}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"does not allow multiple users\",\n\t\t},\n\t\t{\n\t\t\t\"tls_error\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: 123\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"to be filename\",\n\t\t},\n\t\t{\n\t\t\t\"tls_gen_error_cert_file_not_found\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/missing.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"certificate/key pair\",\n\t\t},\n\t\t{\n\t\t\t\"tls_gen_error_key_file_not_found\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/missing.pem\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"certificate/key pair\",\n\t\t},\n\t\t{\n\t\t\t\"tls_gen_error_key_file_missing\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t`missing 'key_file' in TLS configuration`,\n\t\t},\n\t\t{\n\t\t\t\"tls_gen_error_cert_file_missing\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t`missing 'cert_file' in TLS configuration`,\n\t\t},\n\t\t{\n\t\t\t\"tls_gen_error_key_file_not_found\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/missing.pem\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"certificate/key pair\",\n\t\t},\n\t\t{\n\t\t\t\"gateways_needs_to_be_an_array\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tgateways {\n\t\t\t\t\tname: \"B\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\t\"Expected gateways field to be an array\",\n\t\t},\n\t\t{\n\t\t\t\"gateways_entry_needs_to_be_a_map\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tgateways [\n\t\t\t\t\t\"g1\", \"g2\"\n\t\t\t\t]\n\t\t\t}`,\n\t\t\t\"Expected gateway entry to be a map\",\n\t\t},\n\t\t{\n\t\t\t\"bad_url\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tgateways [\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"B\"\n\t\t\t\t\t\turl: \"nats://wrong url\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}`,\n\t\t\t\"error parsing gateway url\",\n\t\t},\n\t\t{\n\t\t\t\"bad_urls\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tgateways [\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"B\"\n\t\t\t\t\t\turls: [\"nats://wrong url\", \"nats://host:5222\"]\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}`,\n\t\t\t\"error parsing gateway url\",\n\t\t},\n\t\t{\n\t\t\t\"gateway_tls_error\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\tgateways [\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"B\"\n\t\t\t\t\t\ttls {\n\t\t\t\t\t\t\tcert_file: 123\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}`,\n\t\t\t\"to be filename\",\n\t\t},\n\t\t{\n\t\t\t\"gateway_unknown_field\",\n\t\t\t`gateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tport: -1\n\t\t\t\tgateways [\n\t\t\t\t\t{\n\t\t\t\t\t\tname: \"B\"\n\t\t\t\t\t\tunknown_field: 1\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}`,\n\t\t\t\"unknown field\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tfile := fmt.Sprintf(\"server_config_gateways_%s.conf\", test.name)\n\t\t\tif err := os.WriteFile(file, []byte(test.content), 0600); err != nil {\n\t\t\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t\t\t}\n\t\t\tdefer removeFile(t, file)\n\t\t\t_, err := ProcessConfigFile(file)\n\t\t\tif err == nil {\n\t\t\t\tt.Fatalf(\"Expected to fail, did not. Content:\\n%s\", test.content)\n\t\t\t} else if !strings.Contains(err.Error(), test.expectedErr) {\n\t\t\t\tt.Fatalf(\"Expected error containing %q, got %q, for content:\\n%s\", test.expectedErr, err, test.content)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParsingLeafNodesListener(t *testing.T) {\n\tcontent := `\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:3333\"\n\t\thost: \"127.0.0.1\"\n\t\tport: 3333\n\t\tadvertise: \"me:22\"\n\t\tauthorization {\n\t\t\tuser: \"derek\"\n\t\t\tpassword: \"s3cr3t!\"\n\t\t\ttimeout: 2.2\n\t\t}\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\ttimeout: 3.3\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\topts, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing file: %v\", err)\n\t}\n\n\texpected := &LeafNodeOpts{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 3333,\n\t\tUsername: \"derek\",\n\t\tPassword: \"s3cr3t!\",\n\t\tAuthTimeout: 2.2,\n\t\tAdvertise: \"me:22\",\n\t\tTLSTimeout: 3.3,\n\t}\n\tif opts.LeafNode.TLSConfig == nil {\n\t\tt.Fatalf(\"Expected TLSConfig, got none\")\n\t}\n\tif opts.LeafNode.tlsConfigOpts == nil {\n\t\tt.Fatalf(\"Expected TLSConfig snapshot, got none\")\n\t}\n\topts.LeafNode.TLSConfig = nil\n\topts.LeafNode.tlsConfigOpts = nil\n\tif !reflect.DeepEqual(&opts.LeafNode, expected) {\n\t\tt.Fatalf(\"Expected %v, got %v\", expected, opts.LeafNode)\n\t}\n}\n\nfunc TestParsingLeafNodeRemotes(t *testing.T) {\n\tt.Run(\"parse config file with relative path\", func(t *testing.T) {\n\t\tcontent := `\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:2222\n\t\t\t\t\taccount: foobar // Local Account to bind to..\n\t\t\t\t\tcredentials: \"./my.creds\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\t\tconf := createConfFile(t, []byte(content))\n\t\topts, err := ProcessConfigFile(conf)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error processing file: %v\", err)\n\t\t}\n\t\tif len(opts.LeafNode.Remotes) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 remote, got %d\", len(opts.LeafNode.Remotes))\n\t\t}\n\t\texpected := &RemoteLeafOpts{\n\t\t\tLocalAccount: \"foobar\",\n\t\t\tCredentials: \"./my.creds\",\n\t\t}\n\t\tu, _ := url.Parse(\"nats-leaf://127.0.0.1:2222\")\n\t\texpected.URLs = append(expected.URLs, u)\n\t\tif !reflect.DeepEqual(opts.LeafNode.Remotes[0], expected) {\n\t\t\tt.Fatalf(\"Expected %v, got %v\", expected, opts.LeafNode.Remotes[0])\n\t\t}\n\t})\n\n\tt.Run(\"parse config file with tilde path\", func(t *testing.T) {\n\t\tif runtime.GOOS == \"windows\" {\n\t\t\tt.SkipNow()\n\t\t}\n\n\t\torigHome := os.Getenv(\"HOME\")\n\t\tdefer os.Setenv(\"HOME\", origHome)\n\t\tos.Setenv(\"HOME\", \"/home/foo\")\n\n\t\tcontent := `\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:2222\n\t\t\t\t\taccount: foobar // Local Account to bind to..\n\t\t\t\t\tcredentials: \"~/my.creds\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\t\tconf := createConfFile(t, []byte(content))\n\t\topts, err := ProcessConfigFile(conf)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error processing file: %v\", err)\n\t\t}\n\t\texpected := &RemoteLeafOpts{\n\t\t\tLocalAccount: \"foobar\",\n\t\t\tCredentials: \"/home/foo/my.creds\",\n\t\t}\n\t\tu, _ := url.Parse(\"nats-leaf://127.0.0.1:2222\")\n\t\texpected.URLs = append(expected.URLs, u)\n\t\tif !reflect.DeepEqual(opts.LeafNode.Remotes[0], expected) {\n\t\t\tt.Fatalf(\"Expected %v, got %v\", expected, opts.LeafNode.Remotes[0])\n\t\t}\n\t})\n\n\tt.Run(\"url ordering\", func(t *testing.T) {\n\t\t// 16! possible permutations.\n\t\torderedURLs := make([]string, 0, 16)\n\t\tfor i := 0; i < cap(orderedURLs); i++ {\n\t\t\torderedURLs = append(orderedURLs, fmt.Sprintf(\"nats-leaf://host%d:7422\", i))\n\t\t}\n\t\tconfURLs, err := json.Marshal(orderedURLs)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tcontent := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\tdont_randomize: true\n\t\t\t\t\turls: %[1]s\n\t\t\t\t}\n\t\t\t\t{\n\t\t\t\t\turls: %[1]s\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(content, confURLs)))\n\n\t\ts, _ := RunServerWithConfig(conf)\n\t\tdefer s.Shutdown()\n\n\t\ts.mu.Lock()\n\t\tr1 := s.leafRemoteCfgs[0]\n\t\tr2 := s.leafRemoteCfgs[1]\n\t\ts.mu.Unlock()\n\n\t\tr1.RLock()\n\t\tgotOrdered := r1.urls\n\t\tr1.RUnlock()\n\t\tif got, want := len(gotOrdered), len(orderedURLs); got != want {\n\t\t\tt.Fatalf(\"Unexpected rem0 len URLs, got %d, want %d\", got, want)\n\t\t}\n\n\t\t// These should be IN order.\n\t\tfor i := range orderedURLs {\n\t\t\tif got, want := gotOrdered[i].String(), orderedURLs[i]; got != want {\n\t\t\t\tt.Fatalf(\"Unexpected ordered url, got %s, want %s\", got, want)\n\t\t\t}\n\t\t}\n\n\t\tr2.RLock()\n\t\tgotRandom := r2.urls\n\t\tr2.RUnlock()\n\t\tif got, want := len(gotRandom), len(orderedURLs); got != want {\n\t\t\tt.Fatalf(\"Unexpected rem1 len URLs, got %d, want %d\", got, want)\n\t\t}\n\n\t\t// These should be OUT of order.\n\t\tvar random bool\n\t\tfor i := range orderedURLs {\n\t\t\tif gotRandom[i].String() != orderedURLs[i] {\n\t\t\t\trandom = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !random {\n\t\t\tt.Fatal(\"Expected urls to be random\")\n\t\t}\n\t})\n\n\tt.Run(\"parse config file js_cluster_migrate\", func(t *testing.T) {\n\t\tcontent := `\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:2222\n\t\t\t\t\taccount: foo // Local Account to bind to..\n\t\t\t\t\tcredentials: \"./my.creds\"\n\t\t\t\t\tjs_cluster_migrate: true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:2222\n\t\t\t\t\taccount: bar // Local Account to bind to..\n\t\t\t\t\tcredentials: \"./my.creds\"\n\t\t\t\t\tjs_cluster_migrate: {\n\t\t\t\t\t\tleader_migrate_delay: 30s\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:2222\n\t\t\t\t\taccount: baz // Local Account to bind to..\n\t\t\t\t\tcredentials: \"./my.creds\"\n\t\t\t\t\tjs_cluster_migrate: false\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\t\tconf := createConfFile(t, []byte(content))\n\t\topts, err := ProcessConfigFile(conf)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error processing file: %v\", err)\n\t\t}\n\t\tif len(opts.LeafNode.Remotes) != 3 {\n\t\t\tt.Fatalf(\"Expected 2 remote, got %d\", len(opts.LeafNode.Remotes))\n\t\t}\n\t\tu, _ := url.Parse(\"nats-leaf://127.0.0.1:2222\")\n\t\texpected := []*RemoteLeafOpts{\n\t\t\t{\n\t\t\t\tURLs: []*url.URL{u},\n\t\t\t\tLocalAccount: \"foo\",\n\t\t\t\tCredentials: \"./my.creds\",\n\t\t\t\tJetStreamClusterMigrate: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tURLs: []*url.URL{u},\n\t\t\t\tLocalAccount: \"bar\",\n\t\t\t\tCredentials: \"./my.creds\",\n\t\t\t\tJetStreamClusterMigrate: true,\n\t\t\t\tJetStreamClusterMigrateDelay: 30 * time.Second,\n\t\t\t},\n\t\t\t{\n\t\t\t\tURLs: []*url.URL{u},\n\t\t\t\tLocalAccount: \"baz\",\n\t\t\t\tCredentials: \"./my.creds\",\n\t\t\t\tJetStreamClusterMigrate: false,\n\t\t\t},\n\t\t}\n\t\tif !reflect.DeepEqual(opts.LeafNode.Remotes, expected) {\n\t\t\tt.Fatalf(\"Expected %v, got %v\", expected, opts.LeafNode.Remotes)\n\t\t}\n\t})\n\n}\n\nfunc TestLargeMaxControlLine(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tmax_control_line = 3000000000\n\t`))\n\tif _, err := ProcessConfigFile(confFileName); err == nil {\n\t\tt.Fatalf(\"Expected an error from too large of a max_control_line entry\")\n\t}\n}\n\nfunc TestLargeMaxPayload(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tmax_payload = 3000000000\n\t`))\n\tif _, err := ProcessConfigFile(confFileName); err == nil {\n\t\tt.Fatalf(\"Expected an error from too large of a max_payload entry\")\n\t}\n\n\tconfFileName = createConfFile(t, []byte(`\n\t\tmax_payload = 100000\n\t\tmax_pending = 50000\n\t`))\n\to := LoadConfig(confFileName)\n\ts, err := NewServer(o)\n\tif err == nil || !strings.Contains(err.Error(), \"cannot be higher\") {\n\t\tif s != nil {\n\t\t\ts.Shutdown()\n\t\t}\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n\nfunc TestHandleUnknownTopLevelConfigurationField(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: 1234\n\t\tstreaming {\n\t\t\tid: \"me\"\n\t\t}\n\t`))\n\n\t// Verify that we get an error because of unknown \"streaming\" field.\n\topts := &Options{}\n\tif err := opts.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), \"streaming\") {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n\n\t// Verify that if that is set, we get no error\n\tNoErrOnUnknownFields(true)\n\tdefer NoErrOnUnknownFields(false)\n\n\tif err := opts.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Port != 1234 {\n\t\tt.Fatalf(\"Port was not parsed correctly: %v\", opts.Port)\n\t}\n\n\t// Verify that ignore works only on top level fields.\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(`\n\t\tport: 1234\n\t\tcluster {\n\t\t\tnon_top_level_unknown_field: 123\n\t\t}\n\t\tstreaming {\n\t\t\tid: \"me\"\n\t\t}\n\t`))\n\tif err := opts.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), \"non_top_level\") {\n\t\tt.Fatal(\"Expected error, got none\")\n\t}\n}\n\nfunc TestSublistNoCacheConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n disable_sublist_cache: true\n `))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading config file: %v\", err)\n\t}\n\tif !opts.NoSublistCache {\n\t\tt.Fatalf(\"Expected sublist cache to be disabled\")\n\t}\n}\n\nfunc TestSublistNoCacheConfigOnAccounts(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t listen: \"127.0.0.1:-1\"\n disable_sublist_cache: true\n\n\t accounts {\n\t\tsynadia {\n\t\t\tusers [ {nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E} ]\n\t\t}\n\t\tnats.io {\n\t\t\tusers [ {nkey : UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ]\n\t\t}\n\t }\n\t no_sys_acc = true\n `))\n\n\ts, _ := RunServerWithConfig(confFileName)\n\tdefer s.Shutdown()\n\n\t// Check that all account sublists do not have caching enabled.\n\tta := s.numReservedAccounts() + 2\n\tif la := s.numAccounts(); la != ta {\n\t\tt.Fatalf(\"Expected to have a server with %d active accounts, got %v\", ta, la)\n\t}\n\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\tif acc == nil {\n\t\t\tt.Fatalf(\"Expected non-nil sublist for account\")\n\t\t}\n\t\tif acc.sl.CacheEnabled() {\n\t\t\tt.Fatalf(\"Expected the account sublist to not have caching enabled\")\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc TestParsingResponsePermissions(t *testing.T) {\n\ttemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{\n\t\t\t\t\tuser: ivan\n\t\t\t\t\tpassword: pwd\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\tallow_responses {\n\t\t\t\t\t\t\t%s\n\t\t\t\t\t\t\t%s\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\n\tcheck := func(t *testing.T, conf string, expectedError string, expectedMaxMsgs int, expectedTTL time.Duration) {\n\t\tt.Helper()\n\t\topts, err := ProcessConfigFile(conf)\n\t\tif expectedError != \"\" {\n\t\t\tif err == nil || !strings.Contains(err.Error(), expectedError) {\n\t\t\t\tt.Fatalf(\"Expected error about %q, got %q\", expectedError, err)\n\t\t\t}\n\t\t\t// OK!\n\t\t\treturn\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on process: %v\", err)\n\t\t}\n\t\tu := opts.Users[0]\n\t\tp := u.Permissions.Response\n\t\tif p == nil {\n\t\t\tt.Fatalf(\"Expected response permissions to be set, it was not\")\n\t\t}\n\t\tif n := p.MaxMsgs; n != expectedMaxMsgs {\n\t\t\tt.Fatalf(\"Expected response max msgs to be %v, got %v\", expectedMaxMsgs, n)\n\t\t}\n\t\tif ttl := p.Expires; ttl != expectedTTL {\n\t\t\tt.Fatalf(\"Expected response ttl to be %v, got %v\", expectedTTL, ttl)\n\t\t}\n\t}\n\n\t// Check defaults\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"\", \"\")))\n\tcheck(t, conf, \"\", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 10\", \"\")))\n\tcheck(t, conf, \"\", 10, DEFAULT_ALLOW_RESPONSE_EXPIRATION)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"\", \"ttl: 5s\")))\n\tcheck(t, conf, \"\", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, 5*time.Second)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 0\", \"\")))\n\tcheck(t, conf, \"\", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"\", `ttl: \"0s\"`)))\n\tcheck(t, conf, \"\", DEFAULT_ALLOW_RESPONSE_MAX_MSGS, DEFAULT_ALLOW_RESPONSE_EXPIRATION)\n\n\t// Check normal values\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 10\", `ttl: \"5s\"`)))\n\tcheck(t, conf, \"\", 10, 5*time.Second)\n\n\t// Check negative values ok\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: -1\", `ttl: \"5s\"`)))\n\tcheck(t, conf, \"\", -1, 5*time.Second)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 10\", `ttl: \"-1s\"`)))\n\tcheck(t, conf, \"\", 10, -1*time.Second)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: -1\", `ttl: \"-1s\"`)))\n\tcheck(t, conf, \"\", -1, -1*time.Second)\n\n\t// Check parsing errors\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"unknown_field: 123\", \"\")))\n\tcheck(t, conf, \"Unknown field\", 0, 0)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 10\", \"ttl: 123\")))\n\tcheck(t, conf, \"not a duration string\", 0, 0)\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(template, \"max: 10\", \"ttl: xyz\")))\n\tcheck(t, conf, \"error parsing expires\", 0, 0)\n}\n\nfunc TestExpandPath(t *testing.T) {\n\tif runtime.GOOS == \"windows\" {\n\t\torigUserProfile := os.Getenv(\"USERPROFILE\")\n\t\torigHomeDrive, origHomePath := os.Getenv(\"HOMEDRIVE\"), os.Getenv(\"HOMEPATH\")\n\t\tdefer func() {\n\t\t\tos.Setenv(\"USERPROFILE\", origUserProfile)\n\t\t\tos.Setenv(\"HOMEDRIVE\", origHomeDrive)\n\t\t\tos.Setenv(\"HOMEPATH\", origHomePath)\n\t\t}()\n\n\t\tcases := []struct {\n\t\t\tpath string\n\t\t\tuserProfile string\n\t\t\thomeDrive string\n\t\t\thomePath string\n\n\t\t\twantPath string\n\t\t\twantErr bool\n\t\t}{\n\t\t\t// Missing HOMEDRIVE and HOMEPATH.\n\t\t\t{path: \"/Foo/Bar\", userProfile: `C:\\Foo\\Bar`, wantPath: \"/Foo/Bar\"},\n\t\t\t{path: \"Foo/Bar\", userProfile: `C:\\Foo\\Bar`, wantPath: \"Foo/Bar\"},\n\t\t\t{path: \"~/Fizz\", userProfile: `C:\\Foo\\Bar`, wantPath: `C:\\Foo\\Bar\\Fizz`},\n\t\t\t{path: `${HOMEDRIVE}${HOMEPATH}\\Fizz`, homeDrive: `C:`, homePath: `\\Foo\\Bar`, wantPath: `C:\\Foo\\Bar\\Fizz`},\n\n\t\t\t// Missing USERPROFILE.\n\t\t\t{path: \"~/Fizz\", homeDrive: \"X:\", homePath: `\\Foo\\Bar`, wantPath: `X:\\Foo\\Bar\\Fizz`},\n\n\t\t\t// Set all environment variables. HOMEDRIVE and HOMEPATH take\n\t\t\t// precedence.\n\t\t\t{path: \"~/Fizz\", userProfile: `C:\\Foo\\Bar`,\n\t\t\t\thomeDrive: \"X:\", homePath: `\\Foo\\Bar`, wantPath: `X:\\Foo\\Bar\\Fizz`},\n\n\t\t\t// Missing all environment variables.\n\t\t\t{path: \"~/Fizz\", wantErr: true},\n\t\t}\n\t\tfor i, c := range cases {\n\t\t\tt.Run(fmt.Sprintf(\"windows case %d\", i), func(t *testing.T) {\n\t\t\t\tos.Setenv(\"USERPROFILE\", c.userProfile)\n\t\t\t\tos.Setenv(\"HOMEDRIVE\", c.homeDrive)\n\t\t\t\tos.Setenv(\"HOMEPATH\", c.homePath)\n\n\t\t\t\tgotPath, err := expandPath(c.path)\n\t\t\t\tif !c.wantErr && err != nil {\n\t\t\t\t\tt.Fatalf(\"unexpected error: got=%v; want=%v\", err, nil)\n\t\t\t\t} else if c.wantErr && err == nil {\n\t\t\t\t\tt.Fatalf(\"unexpected success: got=%v; want=%v\", nil, \"err\")\n\t\t\t\t}\n\n\t\t\t\tif gotPath != c.wantPath {\n\t\t\t\t\tt.Fatalf(\"unexpected path: got=%v; want=%v\", gotPath, c.wantPath)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\n\t\treturn\n\t}\n\n\t// Unix tests\n\n\torigHome := os.Getenv(\"HOME\")\n\tdefer os.Setenv(\"HOME\", origHome)\n\n\tcases := []struct {\n\t\tpath string\n\t\thome string\n\t\twantPath string\n\t\twantErr bool\n\t}{\n\t\t{path: \"/foo/bar\", home: \"/fizz/buzz\", wantPath: \"/foo/bar\"},\n\t\t{path: \"foo/bar\", home: \"/fizz/buzz\", wantPath: \"foo/bar\"},\n\t\t{path: \"~/fizz\", home: \"/foo/bar\", wantPath: \"/foo/bar/fizz\"},\n\t\t{path: \"$HOME/fizz\", home: \"/foo/bar\", wantPath: \"/foo/bar/fizz\"},\n\n\t\t// missing HOME env var\n\t\t{path: \"~/fizz\", wantErr: true},\n\t}\n\tfor i, c := range cases {\n\t\tt.Run(fmt.Sprintf(\"unix case %d\", i), func(t *testing.T) {\n\t\t\tos.Setenv(\"HOME\", c.home)\n\n\t\t\tgotPath, err := expandPath(c.path)\n\t\t\tif !c.wantErr && err != nil {\n\t\t\t\tt.Fatalf(\"unexpected error: got=%v; want=%v\", err, nil)\n\t\t\t} else if c.wantErr && err == nil {\n\t\t\t\tt.Fatalf(\"unexpected success: got=%v; want=%v\", nil, \"err\")\n\t\t\t}\n\n\t\t\tif gotPath != c.wantPath {\n\t\t\t\tt.Fatalf(\"unexpected path: got=%v; want=%v\", gotPath, c.wantPath)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNoAuthUserCode(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tno_auth_user: $NO_AUTH_USER\n\n\t\taccounts {\n\t\t\tsynadia {\n\t\t\t\tusers [\n\t\t\t\t\t{user: \"a\", password: \"a\"},\n\t\t\t\t\t{nkey : UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2E},\n\t\t\t\t]\n\t\t\t}\n\t\t\tacc {\n\t\t\t\tusers [\n\t\t\t\t\t{user: \"c\", password: \"c\"}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\t# config for $G\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{user: \"b\", password: \"b\"}\n\t\t\t]\n\t\t}\n\t`))\n\tdefer os.Unsetenv(\"NO_AUTH_USER\")\n\n\tfor _, user := range []string{\"a\", \"b\", \"b\"} {\n\t\tt.Run(user, func(t *testing.T) {\n\t\t\tos.Setenv(\"NO_AUTH_USER\", user)\n\t\t\topts, err := ProcessConfigFile(confFileName)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t\t\t} else {\n\t\t\t\topts.NoLog = true\n\t\t\t\tsrv := RunServer(opts)\n\t\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"couldn't connect %s\", err)\n\t\t\t\t}\n\t\t\t\tnc.Close()\n\t\t\t\tsrv.Shutdown()\n\t\t\t}\n\t\t})\n\t}\n\n\tfor _, badUser := range []string{\"notthere\", \"UBAAQWTW6CG2G6ANGNKB5U2B7HRWHSGMZEZX3AQSAJOQDAUGJD46LD2F\"} {\n\t\tt.Run(badUser, func(t *testing.T) {\n\t\t\tos.Setenv(\"NO_AUTH_USER\", badUser)\n\t\t\topts, err := ProcessConfigFile(confFileName)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t\t\t}\n\t\t\ts, err := NewServer(opts)\n\t\t\tif err != nil {\n\t\t\t\tif !strings.HasPrefix(err.Error(), \"no_auth_user\") {\n\t\t\t\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t\t\t\t}\n\t\t\t\treturn // error looks as expected\n\t\t\t}\n\t\t\ts.Shutdown()\n\t\t\tt.Fatalf(\"Received no error, where no_auth_user error was expected\")\n\t\t})\n\t}\n\n}\n\nconst operatorJwtWithSysAccAndUrlResolver = `\n\tlisten: \"127.0.0.1:-1\"\n\toperator: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJJVEdJNjNCUUszM1VNN1pBSzZWT1RXNUZEU01ESlNQU1pRQ0RMNUlLUzZQTVhBU0ROQ01RIiwiaWF0IjoxNTg5ODM5MjA1LCJpc3MiOiJPQ1k2REUyRVRTTjNVT0RGVFlFWEJaTFFMSTdYNEdTWFI1NE5aQzRCQkxJNlFDVFpVVDY1T0lWTiIsIm5hbWUiOiJPUCIsInN1YiI6Ik9DWTZERTJFVFNOM1VPREZUWUVYQlpMUUxJN1g0R1NYUjU0TlpDNEJCTEk2UUNUWlVUNjVPSVZOIiwidHlwZSI6Im9wZXJhdG9yIiwibmF0cyI6eyJhY2NvdW50X3NlcnZlcl91cmwiOiJodHRwOi8vbG9jYWxob3N0OjgwMDAvand0L3YxIiwib3BlcmF0b3Jfc2VydmljZV91cmxzIjpbIm5hdHM6Ly9sb2NhbGhvc3Q6NDIyMiJdLCJzeXN0ZW1fYWNjb3VudCI6IkFEWjU0N0IyNFdIUExXT0s3VE1MTkJTQTdGUUZYUjZVTTJOWjRISE5JQjdSREZWWlFGT1o0R1FRIn19.3u710KqMLwgXwsMvhxfEp9xzK84XyAZ-4dd6QY0T6hGj8Bw9mS-HcQ7HbvDDNU01S61tNFfpma_JR6LtB3ixBg\n`\n\nfunc TestReadOperatorJWT(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndUrlResolver))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\tif opts.SystemAccount != \"ADZ547B24WHPLWOK7TMLNBSA7FQFXR6UM2NZ4HHNIB7RDFVZQFOZ4GQQ\" {\n\t\tt.Fatalf(\"Expected different SystemAccount: %s\", opts.SystemAccount)\n\t}\n\tif r, ok := opts.AccountResolver.(*URLAccResolver); !ok {\n\t\tt.Fatalf(\"Expected different SystemAccount: %s\", opts.SystemAccount)\n\t} else if r.url != \"http://localhost:8000/jwt/v1/accounts/\" {\n\t\tt.Fatalf(\"Expected different SystemAccount: %s\", r.url)\n\t}\n}\n\nconst operatorJwtList = `\n\tlisten: \"127.0.0.1:-1\"\n system_account = SYSACC\n\toperator: [\n eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJJVEdJNjNCUUszM1VNN1pBSzZWT1RXNUZEU01ESlNQU1pRQ0RMNUlLUzZQTVhBU0ROQ01RIiwiaWF0IjoxNTg5ODM5MjA1LCJpc3MiOiJPQ1k2REUyRVRTTjNVT0RGVFlFWEJaTFFMSTdYNEdTWFI1NE5aQzRCQkxJNlFDVFpVVDY1T0lWTiIsIm5hbWUiOiJPUCIsInN1YiI6Ik9DWTZERTJFVFNOM1VPREZUWUVYQlpMUUxJN1g0R1NYUjU0TlpDNEJCTEk2UUNUWlVUNjVPSVZOIiwidHlwZSI6Im9wZXJhdG9yIiwibmF0cyI6eyJhY2NvdW50X3NlcnZlcl91cmwiOiJodHRwOi8vbG9jYWxob3N0OjgwMDAvand0L3YxIiwib3BlcmF0b3Jfc2VydmljZV91cmxzIjpbIm5hdHM6Ly9sb2NhbGhvc3Q6NDIyMiJdLCJzeXN0ZW1fYWNjb3VudCI6IkFEWjU0N0IyNFdIUExXT0s3VE1MTkJTQTdGUUZYUjZVTTJOWjRISE5JQjdSREZWWlFGT1o0R1FRIn19.3u710KqMLwgXwsMvhxfEp9xzK84XyAZ-4dd6QY0T6hGj8Bw9mS-HcQ7HbvDDNU01S61tNFfpma_JR6LtB3ixBg,\n eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIzTVJCS1BRTU1IUjdOQVFQU080NUlWTlkyMzVMRlQyTEs0WkZFVU1KWU9EWUJXU0RXWlRBIiwiaWF0IjoxNzI2NTYwMjAwLCJpc3MiOiJPQkxPR1VCSVVQSkhGVE00RjRaTE9CR1BMSlBJRjRTR0JDWUVERUtFUVNNWVVaTVFTMkRGTUUyWCIsIm5hbWUiOiJvcDIiLCJzdWIiOiJPQkxPR1VCSVVQSkhGVE00RjRaTE9CR1BMSlBJRjRTR0JDWUVERUtFUVNNWVVaTVFTMkRGTUUyWCIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9ES0xMSTZWWldWNk03V1RaV0I3MjVITE9MVFFRVERLNE5RR1ZFR0Q0Q083SjJMMlVJWk81U0dXIl0sInN5c3RlbV9hY2NvdW50IjoiQUNRVFdWR1NHSFlWWTNSNkQyV01PM1Y2TFYyTUdLNUI3RzQ3RTQzQkhKQjZGUVZZN0VITlRNTUciLCJ0eXBlIjoib3BlcmF0b3IiLCJ2ZXJzaW9uIjoyfX0.8kUmC6CwGLTJSs1zj_blsMpP5b6n2jZhZFNvMPXvJlRyyR5ZbCsxJ442BimaxaiosS8T-IFcZAIphtiOcqhRCg\n ]\n`\n\nfunc TestReadMultipleOperatorJWT(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(operatorJwtList))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\n\trequire_Equal(t, len(opts.TrustedOperators), 2)\n\trequire_Equal(t, opts.TrustedOperators[0].Name, \"OP\")\n\trequire_Equal(t, opts.TrustedOperators[1].Name, \"op2\")\n\trequire_Equal(t, opts.TrustedOperators[0].SystemAccount, \"ADZ547B24WHPLWOK7TMLNBSA7FQFXR6UM2NZ4HHNIB7RDFVZQFOZ4GQQ\")\n\trequire_Equal(t, opts.TrustedOperators[1].SystemAccount, \"ACQTWVGSGHYVY3R6D2WMO3V6LV2MGK5B7G47E43BHJB6FQVY7EHNTMMG\")\n\t// check if system account precedence is correct\n\trequire_Equal(t, opts.SystemAccount, \"SYSACC\")\n\n}\n\n// using memory resolver so this test does not have to start the memory resolver\nconst operatorJwtWithSysAccAndMemResolver = `\n\tlisten: \"127.0.0.1:-1\"\n\t// Operator \"TESTOP\"\n\toperator: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJLRTZRU0tWTU1VWFFKNFZCTDNSNDdGRFlIWElaTDRZSE1INjVIT0k1UjZCNUpPUkxVQlZBIiwiaWF0IjoxNTg5OTE2MzgyLCJpc3MiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsIm5hbWUiOiJURVNUT1AiLCJzdWIiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic3lzdGVtX2FjY291bnQiOiJBRFNQT1lNSFhKTjZKVllRQ0xSWjVYUTVJVU42QTNTMzNYQTROVjRWSDc0NDIzVTdVN1lSNFlWVyJ9fQ.HiyUtlk8kectKHeQHtuqFcjFt0RbYZE_WAqPCcoWlV2IFVdXuOTzShYEMgDmtgvsFG_zxNQOj08Gr6a06ovwBA\n\tresolver: MEMORY\n\tresolver_preload: {\n \t\t// Account \"TESTSYS\"\n \t\tADSPOYMHXJN6JVYQCLRZ5XQ5IUN6A3S33XA4NV4VH74423U7U7YR4YVW: eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiI2WEtYUFZNTjdEVFlBSUE0R1JDWUxXUElSM1ZEM1Q2UVk2RFg3NURHTVFVWkdVWTJSRFNRIiwiaWF0IjoxNTg5OTE2MzIzLCJpc3MiOiJPQVRUVkJYTElVTVRRT1FXVUEySU0zRkdUQlFRSEFHUEZaQTVET05NTlFSUlRQUjYzTERBTDM1WiIsIm5hbWUiOiJURVNUU1lTIiwic3ViIjoiQURTUE9ZTUhYSk42SlZZUUNMUlo1WFE1SVVONkEzUzMzWEE0TlY0Vkg3NDQyM1U3VTdZUjRZVlciLCJ0eXBlIjoiYWNjb3VudCIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJjb25uIjotMSwibGVhZiI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJ3aWxkY2FyZHMiOnRydWV9fX0.vhtWanIrOncdNfg-yO-7L61ccc-yRacvVtEsaIgWBEmW4czlEPhsiF1MkUKG91rtgcbwUf73ZIFEfja5MgFBAQ\n\t}\n`\n\nfunc TestReadOperatorJWTSystemAccountMatch(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndMemResolver+`\n\t\tsystem_account: ADSPOYMHXJN6JVYQCLRZ5XQ5IUN6A3S33XA4NV4VH74423U7U7YR4YVW\n\t`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts, err := NewServer(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts.Shutdown()\n}\n\nfunc TestReadOperatorJWTSystemAccountMismatch(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(operatorJwtWithSysAccAndMemResolver+`\n\t\tsystem_account: ADXJJCDCSRSMCOV25FXQW7R4QOG7R763TVEXBNWJHLBMBGWOJYG5XZBG\n\t`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts, err := NewServer(opts)\n\tif err == nil {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Received no error\")\n\t} else if !strings.Contains(err.Error(), \"system_account in config and operator JWT must be identical\") {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n}\n\nfunc TestReadOperatorAssertVersion(t *testing.T) {\n\tkp, _ := nkeys.CreateOperator()\n\tpk, _ := kp.PublicKey()\n\top := jwt.NewOperatorClaims(pk)\n\top.AssertServerVersion = \"1.2.3\"\n\tjwt, err := op.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(`\n\t\toperator: %s\n\t\tresolver: MEM\n\t`, jwt)))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts, err := NewServer(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts.Shutdown()\n}\n\nfunc TestReadOperatorAssertVersionFail(t *testing.T) {\n\tkp, _ := nkeys.CreateOperator()\n\tpk, _ := kp.PublicKey()\n\top := jwt.NewOperatorClaims(pk)\n\top.AssertServerVersion = \"10.20.30\"\n\tjwt, err := op.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(`\n\t\toperator: %s\n\t\tresolver: MEM\n\t`, jwt)))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t}\n\ts, err := NewServer(opts)\n\tif err == nil {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Received no error\")\n\t} else if !strings.Contains(err.Error(), \"expected major version 10 > server major version\") {\n\t\tt.Fatal(\"expected different error got: \", err)\n\t}\n}\n\nfunc TestClusterNameAndGatewayNameConflict(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tname: A\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tgateway {\n\t\t\tname: B\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`))\n\n\topts, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif err := validateOptions(opts); err != ErrClusterNameConfigConflict {\n\t\tt.Fatalf(\"Expected ErrClusterNameConfigConflict got %v\", err)\n\t}\n}\n\nfunc TestDefaultAuthTimeout(t *testing.T) {\n\topts := DefaultOptions()\n\topts.AuthTimeout = 0\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tsopts := s.getOpts()\n\tif at := time.Duration(sopts.AuthTimeout * float64(time.Second)); at != AUTH_TIMEOUT {\n\t\tt.Fatalf(\"Expected auth timeout to be %v, got %v\", AUTH_TIMEOUT, at)\n\t}\n\ts.Shutdown()\n\n\topts = DefaultOptions()\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\tTimeout: 4.0,\n\t}\n\ttlsConfig, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\topts.TLSConfig = tlsConfig\n\topts.TLSTimeout = tc.Timeout\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tsopts = s.getOpts()\n\tif sopts.AuthTimeout != 5 {\n\t\tt.Fatalf(\"Expected auth timeout to be %v, got %v\", 5, sopts.AuthTimeout)\n\t}\n}\n\nfunc TestQueuePermissions(t *testing.T) {\n\tcfgFmt := `\n\t\tlisten: 127.0.0.1:-1\n\t\tno_auth_user: u\n\t\tauthorization {\n\t\t\tusers [{\n\t\t\t\tuser: u, password: pwd, permissions: { sub: { %s: [\"foo.> *.dev\"] } }\n\t\t\t}]\n\t\t}`\n\terrChan := make(chan error, 1)\n\tdefer close(errChan)\n\tfor _, test := range []struct {\n\t\tpermType string\n\t\tqueue string\n\t\terrExpected bool\n\t}{\n\t\t{\"allow\", \"queue.dev\", false},\n\t\t{\"allow\", \"\", true},\n\t\t{\"allow\", \"bad\", true},\n\t\t{\"deny\", \"\", false},\n\t\t{\"deny\", \"queue.dev\", true},\n\t} {\n\t\tt.Run(test.permType+test.queue, func(t *testing.T) {\n\t\t\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, test.permType)))\n\t\t\topts, err := ProcessConfigFile(confFileName)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected error %s\", err)\n\t\t\t}\n\t\t\topts.NoLog, opts.NoSigs = true, true\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Port),\n\t\t\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\t\t\terrChan <- err\n\t\t\t\t}))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"No error expected: %v\", err)\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tif test.queue == \"\" {\n\t\t\t\tif _, err := nc.Subscribe(\"foo.bar\", func(msg *nats.Msg) {}); err != nil {\n\t\t\t\t\tt.Fatalf(\"no error expected: %v\", err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif _, err := nc.QueueSubscribe(\"foo.bar\", test.queue, func(msg *nats.Msg) {}); err != nil {\n\t\t\t\t\tt.Fatalf(\"no error expected: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t\tselect {\n\t\t\tcase err := <-errChan:\n\t\t\t\tif !test.errExpected {\n\t\t\t\t\tt.Fatalf(\"Expected no error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif !strings.Contains(err.Error(), `Permissions Violation for Subscription to \"foo.bar\"`) {\n\t\t\t\t\tt.Fatalf(\"error %v\", err)\n\t\t\t\t}\n\t\t\tcase <-time.After(150 * time.Millisecond):\n\t\t\t\tif test.errExpected {\n\t\t\t\t\tt.Fatal(\"Expected an error\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\n\t}\n}\n\nfunc TestResolverPinnedAccountsFail(t *testing.T) {\n\tcfgFmt := `\n\t\toperator: %s\n\t\tresolver: URL(foo.bar)\n\t\tresolver_pinned_accounts: [%s]\n\t`\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(cfgFmt, ojwt, \"f\")))\n\tsrv, err := NewServer(LoadConfig(conf))\n\tdefer srv.Shutdown()\n\trequire_Error(t, err)\n\trequire_Contains(t, err.Error(), \" is not a valid public account nkey\")\n\n\tconf = createConfFile(t, []byte(fmt.Sprintf(cfgFmt, ojwt, \"1, x\")))\n\t_, err = ProcessConfigFile(conf)\n\trequire_Error(t, err)\n\trequire_Contains(t, \"parsing resolver_pinned_accounts: unsupported type\")\n}\n\nfunc TestMaxSubTokens(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tmax_sub_tokens: 4\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\terrs := make(chan error, 1)\n\n\tnc.SetErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\terrs <- err\n\t})\n\n\tbad := \"a.b.c.d.e\"\n\t_, err = nc.SubscribeSync(bad)\n\trequire_NoError(t, err)\n\n\tselect {\n\tcase e := <-errs:\n\t\tif !strings.Contains(e.Error(), \"too many tokens\") {\n\t\t\tt.Fatalf(\"Got wrong error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Did not get the permissions error\")\n\t}\n}\n\nfunc TestGetStorageSize(t *testing.T) {\n\ttt := []struct {\n\t\tinput string\n\t\twant int64\n\t\terr bool\n\t}{\n\t\t{\"1K\", 1024, false},\n\t\t{\"1M\", 1048576, false},\n\t\t{\"1G\", 1073741824, false},\n\t\t{\"1T\", 1099511627776, false},\n\t\t{\"1L\", 0, true},\n\t\t{\"TT\", 0, true},\n\t\t{\"\", 0, false},\n\t}\n\n\tfor _, v := range tt {\n\t\tvar testErr bool\n\t\tgot, err := getStorageSize(v.input)\n\t\tif err != nil {\n\t\t\ttestErr = true\n\t\t}\n\n\t\tif got != v.want || v.err != testErr {\n\t\t\tt.Errorf(\"Got: %v, want %v with error: %v\", got, v.want, testErr)\n\t\t}\n\t}\n\n}\n\nfunc TestAuthorizationAndAccountsMisconfigurations(t *testing.T) {\n\t// There is a test called TestConfigCheck but we can't use it\n\t// because the place where the error will be reported will depend\n\t// if the error is found while parsing authorization{} or\n\t// accounts{}, but we can't control the internal parsing of those\n\t// (due to lexer giving back a map and iteration over map is random)\n\t// regardless of the ordering in the configuration file.\n\t// The test is also repeated\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"duplicate users\",\n\t\t\t`\n\t\t\tauthorization = {users = [ {user: \"user1\", pass: \"pwd\"} ] }\n\t\t\taccounts { ACC { users = [ {user: \"user1\"} ] } }\n\t\t\t`,\n\t\t\tfmt.Sprintf(\"Duplicate user %q detected\", \"user1\"),\n\t\t},\n\t\t{\n\t\t\t\"duplicate nkey\",\n\t\t\t`\n\t\t\tauthorization = {users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] }\n\t\t\taccounts { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\tfmt.Sprintf(\"Duplicate nkey %q detected\", \"UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX\"),\n\t\t},\n\t\t{\n\t\t\t\"auth single user and password and accounts users\",\n\t\t\t`\n\t\t\tauthorization = {user: \"user1\", password: \"pwd\"}\n\t\t\taccounts = { ACC { users = [ {user: \"user2\", pass: \"pwd\"} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a single user/pass\",\n\t\t},\n\t\t{\n\t\t\t\"auth single user and password and accounts nkeys\",\n\t\t\t`\n\t\t\tauthorization = {user: \"user1\", password: \"pwd\"}\n\t\t\taccounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a single user/pass\",\n\t\t},\n\t\t{\n\t\t\t\"auth token and accounts users\",\n\t\t\t`\n\t\t\tauthorization = {token: \"my_token\"}\n\t\t\taccounts = { ACC { users = [ {user: \"user2\", pass: \"pwd\"} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a token\",\n\t\t},\n\t\t{\n\t\t\t\"auth token and accounts nkeys\",\n\t\t\t`\n\t\t\tauthorization = {token: \"my_token\"}\n\t\t\taccounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a token\",\n\t\t},\n\t\t{\n\t\t\t\"auth callout allowed accounts\",\n\t\t\t`\n\t\t\taccounts {\n\t\t\t\tAUTH { users = [ {user: \"auth\", password: \"auth\"} ] }\n\t\t\t\tFOO {}\n\t\t\t}\n\t\t\tauthorization {\n\t\t\t\tauth_callout {\n\t\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\t\taccount: AUTH\n\t\t\t\t\tauth_users: [ auth ]\n\t\t\t\t\tallowed_accounts: [ BAR ]\n\t\t\t\t}\n\t\t\t}\n\t\t\t`,\n\t\t\t\"auth_callout allowed account \\\"BAR\\\" not found in configured accounts\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(test.config))\n\t\t\tif _, err := ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %q\", test.err, err.Error())\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestProcessConfigString duplicates the previous test, but uses the (*Options).ProcessConfigString\n// instead of the ProcessConfigFile function.\nfunc TestProcessConfigString(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"duplicate users\",\n\t\t\t`\n\t\t\tauthorization = {users = [ {user: \"user1\", pass: \"pwd\"} ] }\n\t\t\taccounts { ACC { users = [ {user: \"user1\"} ] } }\n\t\t\t`,\n\t\t\tfmt.Sprintf(\"Duplicate user %q detected\", \"user1\"),\n\t\t},\n\t\t{\n\t\t\t\"duplicate nkey\",\n\t\t\t`\n\t\t\tauthorization = {users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] }\n\t\t\taccounts { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\tfmt.Sprintf(\"Duplicate nkey %q detected\", \"UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX\"),\n\t\t},\n\t\t{\n\t\t\t\"auth single user and password and accounts users\",\n\t\t\t`\n\t\t\tauthorization = {user: \"user1\", password: \"pwd\"}\n\t\t\taccounts = { ACC { users = [ {user: \"user2\", pass: \"pwd\"} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a single user/pass\",\n\t\t},\n\t\t{\n\t\t\t\"auth single user and password and accounts nkeys\",\n\t\t\t`\n\t\t\tauthorization = {user: \"user1\", password: \"pwd\"}\n\t\t\taccounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a single user/pass\",\n\t\t},\n\t\t{\n\t\t\t\"auth token and accounts users\",\n\t\t\t`\n\t\t\tauthorization = {token: \"my_token\"}\n\t\t\taccounts = { ACC { users = [ {user: \"user2\", pass: \"pwd\"} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a token\",\n\t\t},\n\t\t{\n\t\t\t\"auth token and accounts nkeys\",\n\t\t\t`\n\t\t\tauthorization = {token: \"my_token\"}\n\t\t\taccounts = { ACC { users = [ {nkey: UC6NLCN7AS34YOJVCYD4PJ3QB7QGLYG5B5IMBT25VW5K4TNUJODM7BOX} ] } }\n\t\t\t`,\n\t\t\t\"Can not have a token\",\n\t\t},\n\t\t{\n\t\t\t\"auth callout allowed accounts\",\n\t\t\t`\n\t\t\taccounts {\n\t\t\t\tAUTH { users = [ {user: \"auth\", password: \"auth\"} ] }\n\t\t\t\tFOO {}\n\t\t\t}\n\t\t\tauthorization {\n\t\t\t\tauth_callout {\n\t\t\t\t\tissuer: \"ABJHLOVMPA4CI6R5KLNGOB4GSLNIY7IOUPAJC4YFNDLQVIOBYQGUWVLA\"\n\t\t\t\t\taccount: AUTH\n\t\t\t\t\tauth_users: [ auth ]\n\t\t\t\t\tallowed_accounts: [ BAR ]\n\t\t\t\t}\n\t\t\t}\n\t\t\t`,\n\t\t\t\"auth_callout allowed account \\\"BAR\\\" not found in configured accounts\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := &Options{}\n\t\t\tif err := opts.ProcessConfigString(test.config); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %q\", test.err, err.Error())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDefaultSentinel(t *testing.T) {\n\td := `\n\t\tdefault_sentinel: \"hello\"\n\t`\n\tconf := createConfFile(t, []byte(d))\n\topts := LoadConfig(conf)\n\trequire_Equal(t, \"hello\", opts.DefaultSentinel)\n\n\t// if we validate, it will fail, we need an operator\n\t// tests verifying run of configured, elsewhere\n\terr := validateOptions(opts)\n\trequire_Error(t, err)\n\trequire_Equal(t, \"default sentinel requires operators and accounts\", err.Error())\n}\n\nfunc TestAuthorizationTimeoutConfigParsing(t *testing.T) {\n\ttype testCase struct {\n\t\tname string\n\t\tconfig string\n\t\texpectParsed float64\n\t\texpectRunning float64\n\t\texpectErrorContains string\n\t}\n\n\tfor _, tc := range []testCase{{\n\t\tname: \"defaults\",\n\t\tconfig: \"authorization {}\",\n\t\texpectParsed: 0,\n\t\texpectRunning: 2,\n\t}, {\n\t\tname: \"explicit zero\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: 0\n\t\t\t}`,\n\t\texpectParsed: 0,\n\t\texpectRunning: 2,\n\t}, {\n\t\tname: \"explicit one\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: 1\n\t\t\t}`,\n\t\texpectParsed: 1,\n\t\texpectRunning: 1,\n\t}, {\n\t\tname: \"garbage\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: random_garbage\n\t\t\t}`,\n\t\texpectErrorContains: `invalid duration \"random_garbage\"`,\n\t}, {\n\t\tname: \"human readable\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: 10s\n\t\t\t}`,\n\t\texpectParsed: 10,\n\t\texpectRunning: 10,\n\t}, {\n\t\tname: \"bare values could be parsed as integers\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: 1m\n\t\t\t}`,\n\t\texpectParsed: 1000000,\n\t\texpectRunning: 1000000,\n\t}, {\n\t\tname: \"but quoted values will be parsed as durations\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: \"1m\"\n\t\t\t}`,\n\t\texpectParsed: 60,\n\t\texpectRunning: 60,\n\t}, {\n\t\tname: \"human readable minutes quoted\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: \"10m5s30ms\"\n\t\t\t}`,\n\t\texpectParsed: 605.03,\n\t\texpectRunning: 605.03,\n\t}, {\n\t\tname: \"floats work\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: 0.091\n\t\t\t}`,\n\t\texpectParsed: .091,\n\t\texpectRunning: .091,\n\t}, {\n\t\tname: \"but no leading digit fails\",\n\t\tconfig: `\n\t\t\tauthorization {\n\t\t\t\ttimeout: .091\n\t\t\t}`,\n\t\texpectErrorContains: \"Floats must start with a digit\",\n\t}} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\topts, err := parseConfigTolerantly(t, tc.config)\n\t\t\tif tc.expectErrorContains != \"\" {\n\t\t\t\tif !strings.Contains(err.Error(), tc.expectErrorContains) {\n\t\t\t\t\tt.Errorf(\"Expected error like %q, got %v\", tc.expectErrorContains, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Error processing config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif opts.AuthTimeout != tc.expectParsed {\n\t\t\t\tt.Errorf(\"Expected Parsed AuthTimeout to be %f, got %f\", tc.expectParsed, opts.AuthTimeout)\n\t\t\t}\n\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tsopts := s.getOpts()\n\t\t\tif sopts.AuthTimeout != tc.expectRunning {\n\t\t\t\tt.Errorf(\"Expected Running AuthTimeout to be %f, got %f\", tc.expectRunning, sopts.AuthTimeout)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafnodeAuthorizationTimeoutConfigParsing(t *testing.T) {\n\ttype testCase struct {\n\t\tname string\n\t\tconfig string\n\t\texpect float64\n\t\texpectErrorContains string\n\t}\n\n\tfor _, tc := range []testCase{{\n\t\tname: \"defaults\",\n\t\tconfig: \"leafnodes { authorization {} }\",\n\t\texpect: 0,\n\t}, {\n\t\tname: \"explicit zero\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 0\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 0,\n\t}, {\n\t\tname: \"explicit one\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 1\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 1,\n\t}, {\n\t\tname: \"garbage\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: random_garbage\n\t\t\t\t}\n\t\t\t}`,\n\t\texpectErrorContains: `invalid duration \"random_garbage\"`,\n\t}, {\n\t\tname: \"human readable\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 10s\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 10,\n\t}, {\n\t\tname: \"bare values could be parsed as integers\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 1m\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 1000000,\n\t}, {\n\t\tname: \"but quoted values will be parsed as durations\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: \"1m\"\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 60,\n\t}, {\n\t\tname: \"human readable minutes quoted\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: \"10m5s30ms\"\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: 605.03,\n\t}, {\n\t\tname: \"floats work\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 0.091\n\t\t\t\t}\n\t\t\t}`,\n\t\texpect: .091,\n\t}, {\n\t\tname: \"but no leading digit fails\",\n\t\tconfig: `\n\t\t\tleafnodes {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: .091\n\t\t\t\t}\n\t\t\t}`,\n\t\texpectErrorContains: \"Floats must start with a digit\",\n\t}} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\topts, err := parseConfigTolerantly(t, tc.config)\n\t\t\tif tc.expectErrorContains != \"\" {\n\t\t\t\tif !strings.Contains(err.Error(), tc.expectErrorContains) {\n\t\t\t\t\tt.Errorf(\"Expected error like %q, got %v\", tc.expectErrorContains, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Error processing config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif opts.LeafNode.AuthTimeout != tc.expect {\n\t\t\t\tt.Errorf(\"Expected Parsed LeafNode AuthTimeout to be %f, got %f\", tc.expect, opts.LeafNode.AuthTimeout)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc parseConfigTolerantly(t *testing.T, data string) (*Options, error) {\n\tt.Helper()\n\n\tm, err := conf.ParseWithChecks(data)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\to := new(Options)\n\tif err = o.processConfigFile(_EMPTY_, m); err != nil {\n\t\tswitch v := err.(type) {\n\t\tcase *processConfigErr:\n\t\t\tif len(v.errors) > 0 {\n\t\t\t\treturn o, err\n\t\t\t}\n\t\t\tfor _, w := range v.warnings {\n\t\t\t\tt.Logf(\"WARNING: %v\", w)\n\t\t\t}\n\t\t\treturn o, nil\n\t\tdefault:\n\t\t\tt.Logf(\"Unexpected error type %T\", v)\n\t\t\treturn o, err\n\t\t}\n\t}\n\n\treturn o, nil\n}\n\nfunc TestOptionsProxyTrustedKeys(t *testing.T) {\n\to := DefaultOptions()\n\to.Proxies = &ProxiesConfig{\n\t\tTrusted: []*ProxyConfig{\n\t\t\t{Key: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"},\n\t\t\t{Key: \"bad1\"},\n\t\t\t{Key: \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"},\n\t\t\t{Key: \"bad2\"},\n\t\t},\n\t}\n\terr := validateOptions(o)\n\trequire_Error(t, err)\n\trequire_Equal(t, \"proxy trusted key \\\"bad1\\\" is invalid\", err.Error())\n\n\to.Proxies = &ProxiesConfig{\n\t\tTrusted: []*ProxyConfig{\n\t\t\t{Key: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"},\n\t\t\t{Key: \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\"},\n\t\t},\n\t}\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\topts := s.getOpts()\n\tfor i, kp := range s.proxiesKeyPairs {\n\t\tpub, err := kp.PublicKey()\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, opts.Proxies.Trusted[i].Key, pub)\n\t}\n}\n\nfunc TestOptionsProxyRequired(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tauthorization {\n\t\t\tuser: user\n\t\t\tpassword: pwd\n\t\t\tproxy_required: true\n\t\t}\n\t`))\n\to, err := ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\trequire_Equal(t, o.Username, \"user\")\n\trequire_Equal(t, o.Password, \"pwd\")\n\trequire_True(t, o.ProxyRequired)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tauthorization {\n\t\t\tusers: [\n\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t{nkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\", proxy_required: true}\n\t\t\t\t{nkey: \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\", proxy_required: false}\n\t\t\t]\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\n\tcheckUsersAndNkeys := func(users []*User, hasNkeys bool, nkeys []*NkeyUser) {\n\t\tt.Helper()\n\t\tvar found bool\n\n\t\trequire_Len(t, len(users), 3)\n\t\tfor _, u := range users {\n\t\t\tswitch u.Username {\n\t\t\tcase \"user1\", \"user3\":\n\t\t\t\trequire_False(t, u.ProxyRequired)\n\t\t\tcase \"user2\":\n\t\t\t\trequire_True(t, u.ProxyRequired)\n\t\t\t\tfound = true\n\t\t\t}\n\t\t}\n\t\trequire_True(t, found)\n\n\t\tif !hasNkeys {\n\t\t\treturn\n\t\t}\n\n\t\tfound = false\n\t\trequire_Len(t, len(nkeys), 2)\n\t\tfor _, u := range nkeys {\n\t\t\tswitch u.Nkey {\n\t\t\tcase \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\":\n\t\t\t\trequire_True(t, u.ProxyRequired)\n\t\t\t\tfound = true\n\t\t\tcase \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\":\n\t\t\t\trequire_False(t, u.ProxyRequired)\n\t\t\t}\n\t\t}\n\t\trequire_True(t, found)\n\t}\n\tcheckUsersAndNkeys(o.Users, true, o.Nkeys)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t\t{nkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\", proxy_required: true}\n\t\t\t\t\t{nkey: \"UD6AYQSOIN2IN5OGC6VQZCR4H3UFMIOXSW6NNS6N53CLJA4PB56CEJJI\", proxy_required: false}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\trequire_Len(t, len(o.Accounts), 1)\n\trequire_Equal(t, o.Accounts[0].Name, \"A\")\n\tcheckUsersAndNkeys(o.Users, true, o.Nkeys)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tuser: user\n\t\t\t\tpassword: pwd\n\t\t\t\tproxy_required: true\n\t\t\t}\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\trequire_Equal(t, o.LeafNode.Username, \"user\")\n\trequire_Equal(t, o.LeafNode.Password, \"pwd\")\n\trequire_True(t, o.LeafNode.ProxyRequired)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tnkey: \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\"\n\t\t\t\tproxy_required: true\n\t\t\t}\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\trequire_Equal(t, o.LeafNode.Nkey, \"UCARKS2E3KVB7YORL2DG34XLT7PUCOL2SVM7YXV6ETHLW6Z46UUJ2VZ3\")\n\trequire_True(t, o.LeafNode.ProxyRequired)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes: {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: user1, password: pwd1}\n\t\t\t\t\t{user: user2, password: pwd2, proxy_required: true}\n\t\t\t\t\t{user: user3, password: pwd3, proxy_required: false}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\to, err = ProcessConfigFile(conf)\n\trequire_NoError(t, err)\n\tcheckUsersAndNkeys(o.LeafNode.Users, false, nil)\n}\n\n// TestNewServerFromConfigFunctionality tests the NewServerFromConfig() function\n// to ensure it properly processes config files and creates servers correctly.\nfunc TestNewServerFromConfigFunctionality(t *testing.T) {\n\t// Test 1: Error handling - invalid configuration\n\tconfFileName := createConfFile(t, []byte(`\n\t\tmax_payload = 3000000000\n\t`))\n\n\topts1 := &Options{\n\t\tConfigFile: confFileName,\n\t}\n\n\t// Should fail due to oversized max_payload (same as TestLargeMaxPayload)\n\tif _, err := NewServerFromConfig(opts1); err == nil {\n\t\tt.Fatalf(\"Expected an error from too large of a max_payload entry\")\n\t}\n\n\t// Test 2: Config validation error - max_pending > max_payload\n\tconfFileName = createConfFile(t, []byte(`\n\t\tmax_payload = 100000\n\t\tmax_pending = 50000\n\t`))\n\n\topts2 := &Options{\n\t\tConfigFile: confFileName,\n\t}\n\n\t// This should trigger validation error (same as TestLargeMaxPayload)\n\tserver, err := NewServerFromConfig(opts2)\n\tif err == nil || !strings.Contains(err.Error(), \"cannot be higher\") {\n\t\tif server != nil {\n\t\t\tserver.Shutdown()\n\t\t}\n\t\tt.Fatalf(\"Expected validation error, got: %v\", err)\n\t}\n}\n\n// TestNewServerFromConfigVsLoadConfig tests that NewServerFromConfig produces\n// equivalent results to the traditional LoadConfig approach.\nfunc TestNewServerFromConfigVsLoadConfig(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tport = 4224\n\t\tmax_payload = 4194304\n\t\tmax_connections = 200\n\t\tping_interval = \"30s\"\n\t`))\n\n\t// Method 1: Using LoadConfig (traditional approach)\n\topts1 := LoadConfig(confFileName)\n\n\t// Method 2: Using NewServerFromConfig (new approach for embedded servers)\n\topts2 := &Options{ConfigFile: confFileName}\n\n\t// Test 1: Both should be able to create servers successfully\n\tserver1, err := NewServer(opts1)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create server with LoadConfig options: %v\", err)\n\t}\n\tserver1.Shutdown()\n\n\tserver2, err := NewServerFromConfig(opts2)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create server with NewServerFromConfig: %v\", err)\n\t}\n\tserver2.Shutdown()\n\n\t// Test 2: Both methods should produce equivalent results - normalize test environment fields\n\t// LoadConfig sets these fields for testing, so we need to match them for fair comparison\n\topts2.NoSigs, opts2.NoLog = true, opts2.LogFile == _EMPTY_\n\n\tcheckOptionsEqual(t, opts1, opts2)\n}\n\nfunc TestWriteDeadlineConfigParsing(t *testing.T) {\n\ttype testCase struct {\n\t\tname string\n\t\tconfig string\n\t\texpect func(t *testing.T, opts *Options)\n\t}\n\n\tfor _, tc := range []testCase{\n\t\t{\n\t\t\tname: \"LeafNode\",\n\t\t\tconfig: `\n\t\t\t\tleafnodes {\n\t\t\t\t\twrite_deadline: 5s\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\trequire_Equal(t, opts.LeafNode.WriteDeadline, 5*time.Second)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"Gateway\",\n\t\t\tconfig: `\n\t\t\t\tgateway {\n\t\t\t\t\twrite_deadline: 6s\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\trequire_Equal(t, opts.Gateway.WriteDeadline, 6*time.Second)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"Cluster\",\n\t\t\tconfig: `\n\t\t\t\tcluster {\n\t\t\t\t\twrite_deadline: 7s\n\t\t\t\t}\n\t\t\t`,\n\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\trequire_Equal(t, opts.Cluster.WriteDeadline, 7*time.Second)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"Global\",\n\t\t\tconfig: `\n\t\t\t\twrite_deadline: 8s\n\t\t\t`,\n\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\trequire_Equal(t, opts.WriteDeadline, 8*time.Second)\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\topts, err := parseConfigTolerantly(t, tc.config)\n\t\t\trequire_NoError(t, err)\n\t\t\ttc.expect(t, opts)\n\t\t})\n\t}\n}\n\nfunc TestWriteTimeoutConfigParsing(t *testing.T) {\n\ttype testCase struct {\n\t\tname string\n\t\tconfig string\n\t\texpect func(t *testing.T, opts *Options)\n\t}\n\n\tfor str, pol := range map[string]WriteTimeoutPolicy{\n\t\t\"default\": WriteTimeoutPolicyDefault,\n\t\t\"retry\": WriteTimeoutPolicyRetry,\n\t\t\"close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tfor _, tc := range []testCase{\n\t\t\t{\n\t\t\t\tname: \"LeafNode\",\n\t\t\t\tconfig: fmt.Sprintf(`\n\t\t\t\t\tleafnodes {\n\t\t\t\t\t\twrite_timeout: %s\n\t\t\t\t\t}\n\t\t\t\t`, str),\n\t\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\t\trequire_Equal(t, opts.LeafNode.WriteTimeout, pol)\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"Gateway\",\n\t\t\t\tconfig: fmt.Sprintf(`\n\t\t\t\t\tgateway {\n\t\t\t\t\t\twrite_timeout: %s\n\t\t\t\t\t}\n\t\t\t\t`, str),\n\t\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\t\trequire_Equal(t, opts.Gateway.WriteTimeout, pol)\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"Cluster\",\n\t\t\t\tconfig: fmt.Sprintf(`\n\t\t\t\t\tcluster {\n\t\t\t\t\t\twrite_timeout: %s\n\t\t\t\t\t}\n\t\t\t\t`, str),\n\t\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\t\trequire_Equal(t, opts.Cluster.WriteTimeout, pol)\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"Global\",\n\t\t\t\tconfig: fmt.Sprintf(`\n\t\t\t\t\twrite_timeout: %s\n\t\t\t\t`, str),\n\t\t\t\texpect: func(t *testing.T, opts *Options) {\n\t\t\t\t\trequire_Equal(t, opts.WriteTimeout, pol)\n\t\t\t\t},\n\t\t\t},\n\t\t} {\n\t\t\tt.Run(fmt.Sprintf(\"%s/%s\", tc.name, str), func(t *testing.T) {\n\t\t\t\topts, err := parseConfigTolerantly(t, tc.config)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\ttc.expect(t, opts)\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestWebsocketPingIntervalConfig(t *testing.T) {\n\t// Test with string format (duration string)\n\tconfFile := createConfFile(t, []byte(`\n\t\twebsocket {\n\t\t\tport: 8080\n\t\t\tping_interval: \"30s\"\n\t\t}\n\t`))\n\topts, err := ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Websocket.PingInterval != 30*time.Second {\n\t\tt.Fatalf(\"Expected websocket ping_interval to be 30s, got %v\", opts.Websocket.PingInterval)\n\t}\n\n\t// Test with integer format (seconds)\n\tconfFile = createConfFile(t, []byte(`\n\t\twebsocket {\n\t\t\tport: 8080\n\t\t\tping_interval: 45\n\t\t}\n\t`))\n\topts, err = ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Websocket.PingInterval != 45*time.Second {\n\t\tt.Fatalf(\"Expected websocket ping_interval to be 45s, got %v\", opts.Websocket.PingInterval)\n\t}\n\n\t// Test with different duration format\n\tconfFile = createConfFile(t, []byte(`\n\t\twebsocket {\n\t\t\tport: 8080\n\t\t\tping_interval: \"2m\"\n\t\t}\n\t`))\n\topts, err = ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Websocket.PingInterval != 2*time.Minute {\n\t\tt.Fatalf(\"Expected websocket ping_interval to be 2m, got %v\", opts.Websocket.PingInterval)\n\t}\n\n\t// Test without ping_interval (should be zero/unset)\n\tconfFile = createConfFile(t, []byte(`\n\t\twebsocket {\n\t\t\tport: 8080\n\t\t}\n\t`))\n\topts, err = ProcessConfigFile(confFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Websocket.PingInterval != 0 {\n\t\tt.Fatalf(\"Expected websocket ping_interval to be 0 (unset), got %v\", opts.Websocket.PingInterval)\n\t}\n}\n\n// Test variables that reference other variables\nfunc TestVarReferencesVar(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tA: 7890\n\t\tB: $A\n\t\tC: $B\n\t\tport: $C\n\t`))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Port != 7890 {\n\t\tt.Fatalf(\"Expected port 7890, found %d\", opts.Port)\n\t}\n}\n\n// A variables that reference an environment variable\nfunc TestVarReferencesEnvVar(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tA: $_TEST_ENV_NATS_PORT_\n\t\tB: $A\n\t\tC: $B\n\t\tport: $C\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_NATS_PORT_\")\n\tos.Setenv(\"_TEST_ENV_NATS_PORT_\", \"7890\")\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Port != 7890 {\n\t\tt.Fatalf(\"Expected port 7890, found %d\", opts.Port)\n\t}\n}\n\n// Test a variable that references itself\nfunc TestVarReferencesSelf(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`A: $A`))\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected var not found error\")\n\t}\n\trequire_Contains(t, err.Error(),\n\t\t\"variable reference for 'A' on line 1 can not be found\")\n}\n\n// An environment variable can't reference a variable\nfunc TestEnvVarReferencesVar(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tP: 8080\n\t\tport: $_TEST_ENV_NATS_PORT_\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_NATS_PORT_\")\n\tos.Setenv(\"_TEST_ENV_NATS_PORT_\", \"$P\")\n\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected var not found error\")\n\t}\n\trequire_Contains(t, err.Error(),\n\t\t\"variable reference for 'P' on line 1 can not be found\")\n}\n\n// Environment variables can reference other environment variables\nfunc TestEnvVarReferencesEnvVar(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tport: $_TEST_ENV_A_\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_A_\")\n\tdefer os.Unsetenv(\"_TEST_ENV_B_\")\n\tdefer os.Unsetenv(\"_TEST_ENV_C_\")\n\n\tos.Setenv(\"_TEST_ENV_A_\", \"$_TEST_ENV_B_\")\n\tos.Setenv(\"_TEST_ENV_B_\", \"$_TEST_ENV_C_\")\n\tos.Setenv(\"_TEST_ENV_C_\", \"7890\")\n\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Port != 7890 {\n\t\tt.Fatalf(\"Expected port 7890, found %d\", opts.Port)\n\t}\n}\n\n// Test an environment variable that references itself\nfunc TestEnvVarReferencesSelf(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tTEST: $_TEST_ENV_\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_\")\n\n\tos.Setenv(\"_TEST_ENV_\", \"$_TEST_ENV_\")\n\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\trequire_Contains(t, err.Error(), \"variable reference cycle\")\n}\n\n// Test an environment variable that references itself through a cycle\nfunc TestEnvVarReferencesSelfCycle(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tTEST: $_TEST_ENV_A_\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_A_\")\n\tdefer os.Unsetenv(\"_TEST_ENV_B_\")\n\tdefer os.Unsetenv(\"_TEST_ENV_C_\")\n\n\tos.Setenv(\"_TEST_ENV_A_\", \"$_TEST_ENV_B_\")\n\tos.Setenv(\"_TEST_ENV_B_\", \"$_TEST_ENV_C_\")\n\tos.Setenv(\"_TEST_ENV_C_\", \"$_TEST_ENV_A_\")\n\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\trequire_Contains(t, err.Error(), \"variable reference cycle\")\n}\n\n// Test can't include from environment variable\nfunc TestEnvVarInclude(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tTEST: $_TEST_ENV_A_\n\t`))\n\n\tdefer os.Unsetenv(\"_TEST_ENV_A_\")\n\n\tos.Setenv(\"_TEST_ENV_A_\", \"include x\")\n\n\t_, err := ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatal(\"Expected an error\")\n\t}\n\trequire_Contains(t, err.Error(), \"Expected a top-level value to end with a new line, comment or EOF\")\n}\n\nfunc TestEnvVarFromIncludedFile(t *testing.T) {\n\tincludeFileName := createConfFile(t, []byte(`\n\t\tTEST_PORT: $_TEST_ENV_PORT_A_\n\t`))\n\n\tconfFileContent := fmt.Sprintf(`\n\t\tinclude \"./%s\"\n\t\tport: $TEST_PORT\n\t`, filepath.Base(includeFileName))\n\n\tdir := filepath.Dir(includeFileName)\n\tconf, err := os.CreateTemp(dir, \"conf-\")\n\trequire_NoError(t, err)\n\tif err := os.WriteFile(conf.Name(), []byte(confFileContent), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing conf file: %v\", err)\n\t}\n\n\tdefer os.Unsetenv(\"_TEST_ENV_PORT_A_\")\n\tdefer os.Unsetenv(\"_TEST_ENV_PORT_B_\")\n\n\tos.Setenv(\"_TEST_ENV_PORT_A_\", \"$_TEST_ENV_PORT_B_\")\n\tos.Setenv(\"_TEST_ENV_PORT_B_\", \"7890\")\n\n\topts, err := ProcessConfigFile(conf.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif opts.Port != 7890 {\n\t\tt.Fatalf(\"Expected port 7890, found %d\", opts.Port)\n\t}\n}\n" }, { "path": "server/parser.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/textproto\"\n)\n\ntype parserState int\ntype parseState struct {\n\tstate parserState\n\top byte\n\tas int\n\tdrop int\n\tpa pubArg\n\targBuf []byte\n\tmsgBuf []byte\n\theader http.Header // access via getHeader\n\tscratch [MAX_CONTROL_LINE_SIZE]byte\n\targsa [MAX_HMSG_ARGS + 1][]byte // pre-allocated args array to avoid per-call heap escape\n}\n\ntype pubArg struct {\n\targ []byte\n\tpacache []byte\n\torigin []byte\n\taccount []byte\n\tsubject []byte\n\tdeliver []byte\n\tmapped []byte\n\treply []byte\n\tszb []byte\n\thdb []byte\n\tqueues [][]byte\n\tsize int\n\thdr int\n\tpsi []*serviceImport\n\ttrace *msgTrace\n\tdelivered bool // Only used for service imports\n}\n\n// Parser constants\nconst (\n\tOP_START parserState = iota\n\tOP_PLUS\n\tOP_PLUS_O\n\tOP_PLUS_OK\n\tOP_MINUS\n\tOP_MINUS_E\n\tOP_MINUS_ER\n\tOP_MINUS_ERR\n\tOP_MINUS_ERR_SPC\n\tMINUS_ERR_ARG\n\tOP_C\n\tOP_CO\n\tOP_CON\n\tOP_CONN\n\tOP_CONNE\n\tOP_CONNEC\n\tOP_CONNECT\n\tCONNECT_ARG\n\tOP_H\n\tOP_HP\n\tOP_HPU\n\tOP_HPUB\n\tOP_HPUB_SPC\n\tHPUB_ARG\n\tOP_HM\n\tOP_HMS\n\tOP_HMSG\n\tOP_HMSG_SPC\n\tHMSG_ARG\n\tOP_P\n\tOP_PU\n\tOP_PUB\n\tOP_PUB_SPC\n\tPUB_ARG\n\tOP_PI\n\tOP_PIN\n\tOP_PING\n\tOP_PO\n\tOP_PON\n\tOP_PONG\n\tMSG_PAYLOAD\n\tMSG_END_R\n\tMSG_END_N\n\tOP_S\n\tOP_SU\n\tOP_SUB\n\tOP_SUB_SPC\n\tSUB_ARG\n\tOP_A\n\tOP_ASUB\n\tOP_ASUB_SPC\n\tASUB_ARG\n\tOP_AUSUB\n\tOP_AUSUB_SPC\n\tAUSUB_ARG\n\tOP_L\n\tOP_LS\n\tOP_R\n\tOP_RS\n\tOP_U\n\tOP_UN\n\tOP_UNS\n\tOP_UNSU\n\tOP_UNSUB\n\tOP_UNSUB_SPC\n\tUNSUB_ARG\n\tOP_M\n\tOP_MS\n\tOP_MSG\n\tOP_MSG_SPC\n\tMSG_ARG\n\tOP_I\n\tOP_IN\n\tOP_INF\n\tOP_INFO\n\tINFO_ARG\n)\n\nfunc (c *client) parse(buf []byte) error {\n\t// Branch out to mqtt clients. c.mqtt is immutable, but should it become\n\t// an issue (say data race detection), we could branch outside in readLoop\n\tif c.isMqtt() {\n\t\treturn c.mqttParse(buf)\n\t}\n\tvar i int\n\tvar b byte\n\tvar lmsg bool\n\n\t// Snapshots\n\tc.mu.Lock()\n\t// Snapshot and then reset when we receive a\n\t// proper CONNECT if needed.\n\tauthSet := c.awaitingAuth()\n\t// Snapshot max control line as well.\n\ts, mcl, trace := c.srv, c.mcl, c.trace\n\tc.mu.Unlock()\n\n\t// Move to loop instead of range syntax to allow jumping of i\n\tfor i = 0; i < len(buf); i++ {\n\t\tb = buf[i]\n\n\t\tswitch c.state {\n\t\tcase OP_START:\n\t\t\tc.op = b\n\t\t\tif b != 'C' && b != 'c' {\n\t\t\t\tif authSet {\n\t\t\t\t\tif s == nil {\n\t\t\t\t\t\tgoto authErr\n\t\t\t\t\t}\n\t\t\t\t\tvar ok bool\n\t\t\t\t\t// Check here for NoAuthUser. If this is set allow non CONNECT protos as our first.\n\t\t\t\t\t// E.g. telnet proto demos.\n\t\t\t\t\tif noAuthUser := s.getOpts().NoAuthUser; noAuthUser != _EMPTY_ {\n\t\t\t\t\t\ts.mu.Lock()\n\t\t\t\t\t\tuser, exists := s.users[noAuthUser]\n\t\t\t\t\t\ts.mu.Unlock()\n\t\t\t\t\t\tif exists {\n\t\t\t\t\t\t\tc.RegisterUser(user)\n\t\t\t\t\t\t\tc.mu.Lock()\n\t\t\t\t\t\t\tc.clearAuthTimer()\n\t\t\t\t\t\t\tc.flags.set(connectReceived)\n\t\t\t\t\t\t\tc.mu.Unlock()\n\t\t\t\t\t\t\tauthSet, ok = false, true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tgoto authErr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// If the connection is a gateway connection, make sure that\n\t\t\t\t// if this is an inbound, it starts with a CONNECT.\n\t\t\t\tif c.kind == GATEWAY && !c.gw.outbound && !c.gw.connected {\n\t\t\t\t\t// Use auth violation since no CONNECT was sent.\n\t\t\t\t\t// It could be a parseErr too.\n\t\t\t\t\tgoto authErr\n\t\t\t\t}\n\t\t\t}\n\t\t\tswitch b {\n\t\t\tcase 'P', 'p':\n\t\t\t\tc.state = OP_P\n\t\t\tcase 'H', 'h':\n\t\t\t\tc.state = OP_H\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_S\n\t\t\tcase 'U', 'u':\n\t\t\t\tc.state = OP_U\n\t\t\tcase 'R', 'r':\n\t\t\t\tif c.kind == CLIENT {\n\t\t\t\t\tgoto parseErr\n\t\t\t\t} else {\n\t\t\t\t\tc.state = OP_R\n\t\t\t\t}\n\t\t\tcase 'L', 'l':\n\t\t\t\tif c.kind != LEAF && c.kind != ROUTER {\n\t\t\t\t\tgoto parseErr\n\t\t\t\t} else {\n\t\t\t\t\tc.state = OP_L\n\t\t\t\t}\n\t\t\tcase 'A', 'a':\n\t\t\t\tif c.kind == CLIENT {\n\t\t\t\t\tgoto parseErr\n\t\t\t\t} else {\n\t\t\t\t\tc.state = OP_A\n\t\t\t\t}\n\t\t\tcase 'C', 'c':\n\t\t\t\tc.state = OP_C\n\t\t\tcase 'I', 'i':\n\t\t\t\tc.state = OP_I\n\t\t\tcase '+':\n\t\t\t\tc.state = OP_PLUS\n\t\t\tcase '-':\n\t\t\t\tc.state = OP_MINUS\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_H:\n\t\t\tswitch b {\n\t\t\tcase 'P', 'p':\n\t\t\t\tc.state = OP_HP\n\t\t\tcase 'M', 'm':\n\t\t\t\tc.state = OP_HM\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HP:\n\t\t\tswitch b {\n\t\t\tcase 'U', 'u':\n\t\t\t\tc.state = OP_HPU\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HPU:\n\t\t\tswitch b {\n\t\t\tcase 'B', 'b':\n\t\t\t\tc.state = OP_HPUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HPUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_HPUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HPUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.pa.hdr = 0\n\t\t\t\tc.state = HPUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase HPUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"HPUB\", arg)\n\t\t\t\t}\n\t\t\t\tvar remaining []byte\n\t\t\t\tif i < len(buf) {\n\t\t\t\t\tremaining = buf[i+1:]\n\t\t\t\t}\n\t\t\t\tif err := c.processHeaderPub(arg, remaining); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD\n\t\t\t\t// If we don't have a saved buffer then jump ahead with\n\t\t\t\t// the index. If this overruns what is left we fall out\n\t\t\t\t// and process split buffer.\n\t\t\t\tif c.msgBuf == nil {\n\t\t\t\t\ti = c.as + c.pa.size - LEN_CR_LF\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_HM:\n\t\t\tswitch b {\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_HMS\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HMS:\n\t\t\tswitch b {\n\t\t\tcase 'G', 'g':\n\t\t\t\tc.state = OP_HMSG\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HMSG:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_HMSG_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_HMSG_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.pa.hdr = 0\n\t\t\t\tc.state = HMSG_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase HMSG_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\tif c.kind == ROUTER || c.kind == GATEWAY {\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"HMSG\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processRoutedHeaderMsgArgs(arg)\n\t\t\t\t} else if c.kind == LEAF {\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"HMSG\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processLeafHeaderMsgArgs(arg)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD\n\n\t\t\t\t// jump ahead with the index. If this overruns\n\t\t\t\t// what is left we fall out and process split\n\t\t\t\t// buffer.\n\t\t\t\ti = c.as + c.pa.size - LEN_CR_LF\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_P:\n\t\t\tswitch b {\n\t\t\tcase 'U', 'u':\n\t\t\t\tc.state = OP_PU\n\t\t\tcase 'I', 'i':\n\t\t\t\tc.state = OP_PI\n\t\t\tcase 'O', 'o':\n\t\t\t\tc.state = OP_PO\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PU:\n\t\t\tswitch b {\n\t\t\tcase 'B', 'b':\n\t\t\t\tc.state = OP_PUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_PUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.pa.hdr = -1\n\t\t\t\tc.state = PUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase PUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"PUB\", arg)\n\t\t\t\t}\n\t\t\t\tif err := c.processPub(arg); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD\n\t\t\t\t// If we don't have a saved buffer then jump ahead with\n\t\t\t\t// the index. If this overruns what is left we fall out\n\t\t\t\t// and process split buffer.\n\t\t\t\tif c.msgBuf == nil {\n\t\t\t\t\ti = c.as + c.pa.size - LEN_CR_LF\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase MSG_PAYLOAD:\n\t\t\tif c.msgBuf != nil {\n\t\t\t\t// copy as much as we can to the buffer and skip ahead.\n\t\t\t\ttoCopy := c.pa.size - len(c.msgBuf)\n\t\t\t\tavail := len(buf) - i\n\t\t\t\tif avail < toCopy {\n\t\t\t\t\ttoCopy = avail\n\t\t\t\t}\n\t\t\t\tif toCopy > 0 {\n\t\t\t\t\tstart := len(c.msgBuf)\n\t\t\t\t\t// This is needed for copy to work.\n\t\t\t\t\tc.msgBuf = c.msgBuf[:start+toCopy]\n\t\t\t\t\tcopy(c.msgBuf[start:], buf[i:i+toCopy])\n\t\t\t\t\t// Update our index\n\t\t\t\t\ti = (i + toCopy) - 1\n\t\t\t\t} else {\n\t\t\t\t\t// Fall back to append if needed.\n\t\t\t\t\tc.msgBuf = append(c.msgBuf, b)\n\t\t\t\t}\n\t\t\t\tif len(c.msgBuf) >= c.pa.size {\n\t\t\t\t\tc.state = MSG_END_R\n\t\t\t\t}\n\t\t\t} else if i-c.as+1 >= c.pa.size {\n\t\t\t\tc.state = MSG_END_R\n\t\t\t}\n\t\tcase MSG_END_R:\n\t\t\tif b != '\\r' {\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\t\tif c.msgBuf != nil {\n\t\t\t\tc.msgBuf = append(c.msgBuf, b)\n\t\t\t}\n\t\t\tc.state = MSG_END_N\n\t\tcase MSG_END_N:\n\t\t\tif b != '\\n' {\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\t\tif c.msgBuf != nil {\n\t\t\t\tc.msgBuf = append(c.msgBuf, b)\n\t\t\t} else {\n\t\t\t\tc.msgBuf = buf[c.as : i+1]\n\t\t\t}\n\n\t\t\tvar mt *msgTrace\n\t\t\tif c.pa.hdr > 0 {\n\t\t\t\tmt = c.initMsgTrace()\n\t\t\t}\n\t\t\t// Check for mappings.\n\t\t\tif (c.kind == CLIENT || c.kind == LEAF) && c.in.flags.isSet(hasMappings) {\n\t\t\t\tchanged := c.selectMappedSubject()\n\t\t\t\tif changed {\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"MAPPING\", []byte(fmt.Sprintf(\"%s -> %s\", c.pa.mapped, c.pa.subject)))\n\t\t\t\t\t}\n\t\t\t\t\t// c.pa.subject is the subject the original is now mapped to.\n\t\t\t\t\tmt.addSubjectMappingEvent(c.pa.subject)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif trace {\n\t\t\t\tc.traceMsg(c.msgBuf)\n\t\t\t}\n\n\t\t\tc.processInboundMsg(c.msgBuf)\n\n\t\t\tmt.sendEvent()\n\t\t\tc.argBuf, c.msgBuf, c.header = nil, nil, nil\n\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\t// Drop all pub args\n\t\t\tc.pa.arg, c.pa.pacache, c.pa.origin, c.pa.account, c.pa.subject, c.pa.mapped = nil, nil, nil, nil, nil, nil\n\t\t\tc.pa.reply, c.pa.hdr, c.pa.size, c.pa.szb, c.pa.hdb, c.pa.queues = nil, -1, 0, nil, nil, nil\n\t\t\tc.pa.trace = nil\n\t\t\tc.pa.delivered = false\n\t\t\tlmsg = false\n\t\tcase OP_A:\n\t\t\tswitch b {\n\t\t\tcase '+':\n\t\t\t\tc.state = OP_ASUB\n\t\t\tcase '-', 'u':\n\t\t\t\tc.state = OP_AUSUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_ASUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_ASUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_ASUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = ASUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase ASUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"A+\", arg)\n\t\t\t\t}\n\t\t\t\tif err := c.processAccountSub(arg); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_AUSUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_AUSUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_AUSUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = AUSUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase AUSUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"A-\", arg)\n\t\t\t\t}\n\t\t\t\tc.processAccountUnsub(arg)\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_S:\n\t\t\tswitch b {\n\t\t\tcase 'U', 'u':\n\t\t\t\tc.state = OP_SU\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_SU:\n\t\t\tswitch b {\n\t\t\tcase 'B', 'b':\n\t\t\t\tc.state = OP_SUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_SUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_SUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_SUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = SUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase SUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tvar err error\n\n\t\t\t\tswitch c.kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"SUB\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.parseSub(arg, false)\n\t\t\t\tcase ROUTER:\n\t\t\t\t\tswitch c.op {\n\t\t\t\t\tcase 'R', 'r':\n\t\t\t\t\t\tif trace {\n\t\t\t\t\t\t\tc.traceInOp(\"RS+\", arg)\n\t\t\t\t\t\t}\n\t\t\t\t\t\terr = c.processRemoteSub(arg, false)\n\t\t\t\t\tcase 'L', 'l':\n\t\t\t\t\t\tif trace {\n\t\t\t\t\t\t\tc.traceInOp(\"LS+\", arg)\n\t\t\t\t\t\t}\n\t\t\t\t\t\terr = c.processRemoteSub(arg, true)\n\t\t\t\t\t}\n\t\t\t\tcase GATEWAY:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"RS+\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processGatewayRSub(arg)\n\t\t\t\tcase LEAF:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"LS+\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processLeafSub(arg)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_L:\n\t\t\tswitch b {\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_LS\n\t\t\tcase 'M', 'm':\n\t\t\t\tc.state = OP_M\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_LS:\n\t\t\tswitch b {\n\t\t\tcase '+':\n\t\t\t\tc.state = OP_SUB\n\t\t\tcase '-':\n\t\t\t\tc.state = OP_UNSUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_R:\n\t\t\tswitch b {\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_RS\n\t\t\tcase 'M', 'm':\n\t\t\t\tc.state = OP_M\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_RS:\n\t\t\tswitch b {\n\t\t\tcase '+':\n\t\t\t\tc.state = OP_SUB\n\t\t\tcase '-':\n\t\t\t\tc.state = OP_UNSUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_U:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_UN\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_UN:\n\t\t\tswitch b {\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_UNS\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_UNS:\n\t\t\tswitch b {\n\t\t\tcase 'U', 'u':\n\t\t\t\tc.state = OP_UNSU\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_UNSU:\n\t\t\tswitch b {\n\t\t\tcase 'B', 'b':\n\t\t\t\tc.state = OP_UNSUB\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_UNSUB:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_UNSUB_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_UNSUB_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = UNSUB_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase UNSUB_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tvar err error\n\n\t\t\t\tswitch c.kind {\n\t\t\t\tcase CLIENT:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"UNSUB\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processUnsub(arg)\n\t\t\t\tcase ROUTER:\n\t\t\t\t\tif trace && c.srv != nil {\n\t\t\t\t\t\tswitch c.op {\n\t\t\t\t\t\tcase 'R', 'r':\n\t\t\t\t\t\t\tc.traceInOp(\"RS-\", arg)\n\t\t\t\t\t\tcase 'L', 'l':\n\t\t\t\t\t\t\tc.traceInOp(\"LS-\", arg)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tleafUnsub := c.op == 'L' || c.op == 'l'\n\t\t\t\t\terr = c.processRemoteUnsub(arg, leafUnsub)\n\t\t\t\tcase GATEWAY:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"RS-\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processGatewayRUnsub(arg)\n\t\t\t\tcase LEAF:\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"LS-\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processLeafUnsub(arg)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_PI:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_PIN\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PIN:\n\t\t\tswitch b {\n\t\t\tcase 'G', 'g':\n\t\t\t\tc.state = OP_PING\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PING:\n\t\t\tswitch b {\n\t\t\tcase '\\n':\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"PING\", nil)\n\t\t\t\t}\n\t\t\t\tc.processPing()\n\t\t\t\tc.drop, c.state = 0, OP_START\n\t\t\t}\n\t\tcase OP_PO:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_PON\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PON:\n\t\t\tswitch b {\n\t\t\tcase 'G', 'g':\n\t\t\t\tc.state = OP_PONG\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PONG:\n\t\t\tswitch b {\n\t\t\tcase '\\n':\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"PONG\", nil)\n\t\t\t\t}\n\t\t\t\tc.processPong()\n\t\t\t\tc.drop, c.state = 0, OP_START\n\t\t\t}\n\t\tcase OP_C:\n\t\t\tswitch b {\n\t\t\tcase 'O', 'o':\n\t\t\t\tc.state = OP_CO\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CO:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_CON\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CON:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_CONN\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CONN:\n\t\t\tswitch b {\n\t\t\tcase 'E', 'e':\n\t\t\t\tc.state = OP_CONNE\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CONNE:\n\t\t\tswitch b {\n\t\t\tcase 'C', 'c':\n\t\t\t\tc.state = OP_CONNEC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CONNEC:\n\t\t\tswitch b {\n\t\t\tcase 'T', 't':\n\t\t\t\tc.state = OP_CONNECT\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_CONNECT:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = CONNECT_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase CONNECT_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif trace {\n\t\t\t\t\tc.traceInOp(\"CONNECT\", removeSecretsFromTrace(arg))\n\t\t\t\t}\n\t\t\t\tif err := c.processConnect(arg); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.state = 0, OP_START\n\t\t\t\t// Reset notion on authSet\n\t\t\t\tc.mu.Lock()\n\t\t\t\tauthSet = c.awaitingAuth()\n\t\t\t\tc.mu.Unlock()\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_M:\n\t\t\tswitch b {\n\t\t\tcase 'S', 's':\n\t\t\t\tc.state = OP_MS\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MS:\n\t\t\tswitch b {\n\t\t\tcase 'G', 'g':\n\t\t\t\tc.state = OP_MSG\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MSG:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_MSG_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MSG_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.pa.hdr = -1\n\t\t\t\tc.state = MSG_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase MSG_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\tif c.kind == ROUTER || c.kind == GATEWAY {\n\t\t\t\t\tswitch c.op {\n\t\t\t\t\tcase 'R', 'r':\n\t\t\t\t\t\tif trace {\n\t\t\t\t\t\t\tc.traceInOp(\"RMSG\", arg)\n\t\t\t\t\t\t}\n\t\t\t\t\t\terr = c.processRoutedMsgArgs(arg)\n\t\t\t\t\tcase 'L', 'l':\n\t\t\t\t\t\tif trace {\n\t\t\t\t\t\t\tc.traceInOp(\"LMSG\", arg)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tlmsg = true\n\t\t\t\t\t\terr = c.processRoutedOriginClusterMsgArgs(arg)\n\t\t\t\t\t}\n\t\t\t\t} else if c.kind == LEAF {\n\t\t\t\t\tif trace {\n\t\t\t\t\t\tc.traceInOp(\"LMSG\", arg)\n\t\t\t\t\t}\n\t\t\t\t\terr = c.processLeafMsgArgs(arg)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD\n\n\t\t\t\t// jump ahead with the index. If this overruns\n\t\t\t\t// what is left we fall out and process split\n\t\t\t\t// buffer.\n\t\t\t\ti = c.as + c.pa.size - LEN_CR_LF\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_I:\n\t\t\tswitch b {\n\t\t\tcase 'N', 'n':\n\t\t\t\tc.state = OP_IN\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_IN:\n\t\t\tswitch b {\n\t\t\tcase 'F', 'f':\n\t\t\t\tc.state = OP_INF\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_INF:\n\t\t\tswitch b {\n\t\t\tcase 'O', 'o':\n\t\t\t\tc.state = OP_INFO\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_INFO:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = INFO_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase INFO_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif err := c.processInfo(arg); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tcase OP_PLUS:\n\t\t\tswitch b {\n\t\t\tcase 'O', 'o':\n\t\t\t\tc.state = OP_PLUS_O\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PLUS_O:\n\t\t\tswitch b {\n\t\t\tcase 'K', 'k':\n\t\t\t\tc.state = OP_PLUS_OK\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_PLUS_OK:\n\t\t\tswitch b {\n\t\t\tcase '\\n':\n\t\t\t\tc.drop, c.state = 0, OP_START\n\t\t\t}\n\t\tcase OP_MINUS:\n\t\t\tswitch b {\n\t\t\tcase 'E', 'e':\n\t\t\t\tc.state = OP_MINUS_E\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MINUS_E:\n\t\t\tswitch b {\n\t\t\tcase 'R', 'r':\n\t\t\t\tc.state = OP_MINUS_ER\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MINUS_ER:\n\t\t\tswitch b {\n\t\t\tcase 'R', 'r':\n\t\t\t\tc.state = OP_MINUS_ERR\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MINUS_ERR:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tc.state = OP_MINUS_ERR_SPC\n\t\t\tdefault:\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\tcase OP_MINUS_ERR_SPC:\n\t\t\tswitch b {\n\t\t\tcase ' ', '\\t':\n\t\t\t\tcontinue\n\t\t\tdefault:\n\t\t\t\tc.state = MINUS_ERR_ARG\n\t\t\t\tc.as = i\n\t\t\t}\n\t\tcase MINUS_ERR_ARG:\n\t\t\tswitch b {\n\t\t\tcase '\\r':\n\t\t\t\tc.drop = 1\n\t\t\tcase '\\n':\n\t\t\t\tvar arg []byte\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\targ = c.argBuf\n\t\t\t\t\tc.argBuf = nil\n\t\t\t\t} else {\n\t\t\t\t\targ = buf[c.as : i-c.drop]\n\t\t\t\t}\n\t\t\t\tif err := c.overMaxControlLineLimit(arg, mcl); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tc.processErr(string(arg))\n\t\t\t\tc.drop, c.as, c.state = 0, i+1, OP_START\n\t\t\tdefault:\n\t\t\t\tif c.argBuf != nil {\n\t\t\t\t\tc.argBuf = append(c.argBuf, b)\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tgoto parseErr\n\t\t}\n\t}\n\n\t// Check for split buffer scenarios for any ARG state.\n\tif c.state == SUB_ARG || c.state == UNSUB_ARG ||\n\t\tc.state == PUB_ARG || c.state == HPUB_ARG ||\n\t\tc.state == ASUB_ARG || c.state == AUSUB_ARG ||\n\t\tc.state == MSG_ARG || c.state == HMSG_ARG ||\n\t\tc.state == MINUS_ERR_ARG || c.state == CONNECT_ARG || c.state == INFO_ARG {\n\n\t\t// Setup a holder buffer to deal with split buffer scenario.\n\t\tif c.argBuf == nil {\n\t\t\tc.argBuf = c.scratch[:0]\n\t\t\tc.argBuf = append(c.argBuf, buf[c.as:i-c.drop]...)\n\t\t}\n\t\t// Check for violations of control line length here. Note that this is not\n\t\t// exact at all but the performance hit is too great to be precise, and\n\t\t// catching here should prevent memory exhaustion attacks.\n\t\tif err := c.overMaxControlLineLimit(c.argBuf, mcl); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Check for split msg\n\tif (c.state == MSG_PAYLOAD || c.state == MSG_END_R || c.state == MSG_END_N) && c.msgBuf == nil {\n\t\t// We need to clone the pubArg if it is still referencing the\n\t\t// read buffer and we are not able to process the msg.\n\n\t\tif c.argBuf == nil {\n\t\t\t// Works also for MSG_ARG, when message comes from ROUTE or GATEWAY.\n\t\t\tif err := c.clonePubArg(lmsg); err != nil {\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\t}\n\n\t\t// If we will overflow the scratch buffer, just create a\n\t\t// new buffer to hold the split message.\n\t\tif c.pa.size > cap(c.scratch)-len(c.argBuf) {\n\t\t\tlrem := len(buf[c.as:])\n\t\t\t// Consider it a protocol error when the remaining payload\n\t\t\t// is larger than the reported size for PUB. It can happen\n\t\t\t// when processing incomplete messages from rogue clients.\n\t\t\tif lrem > c.pa.size+LEN_CR_LF {\n\t\t\t\tgoto parseErr\n\t\t\t}\n\t\t\tc.msgBuf = make([]byte, lrem, c.pa.size+LEN_CR_LF)\n\t\t\tcopy(c.msgBuf, buf[c.as:])\n\t\t} else {\n\t\t\tc.msgBuf = c.scratch[len(c.argBuf):len(c.argBuf)]\n\t\t\tc.msgBuf = append(c.msgBuf, (buf[c.as:])...)\n\t\t}\n\t}\n\n\treturn nil\n\nauthErr:\n\tc.authViolation()\n\treturn ErrAuthentication\n\nparseErr:\n\tc.sendErr(\"Unknown Protocol Operation\")\n\tsnip := protoSnippet(i, PROTO_SNIPPET_SIZE, buf)\n\terr := fmt.Errorf(\"%s parser ERROR, state=%d, i=%d: proto='%s...'\", c.kindString(), c.state, i, snip)\n\treturn err\n}\n\nfunc protoSnippet(start, max int, buf []byte) string {\n\tstop := start + max\n\tbufSize := len(buf)\n\tif start >= bufSize {\n\t\treturn `\"\"`\n\t}\n\tif stop > bufSize {\n\t\tstop = bufSize - 1\n\t}\n\treturn fmt.Sprintf(\"%q\", buf[start:stop])\n}\n\n// Check if the length of buffer `arg` is over the max control line limit `mcl`.\n// If so, an error is sent to the client and the connection is closed.\n// The error ErrMaxControlLine is returned.\nfunc (c *client) overMaxControlLineLimit(arg []byte, mcl int32) error {\n\tif c.kind != CLIENT {\n\t\treturn nil\n\t}\n\tif len(arg) > int(mcl) {\n\t\terr := NewErrorCtx(ErrMaxControlLine, \"State %d, max_control_line %d, Buffer len %d (snip: %s...)\",\n\t\t\tc.state, int(mcl), len(c.argBuf), protoSnippet(0, MAX_CONTROL_LINE_SNIPPET_SIZE, arg))\n\t\tc.sendErr(err.Error())\n\t\tc.closeConnection(MaxControlLineExceeded)\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but\n// we need to hold onto it into the next read.\nfunc (c *client) clonePubArg(lmsg bool) error {\n\t// Just copy and re-process original arg buffer.\n\tc.argBuf = c.scratch[:0]\n\tc.argBuf = append(c.argBuf, c.pa.arg...)\n\n\tswitch c.kind {\n\tcase ROUTER, GATEWAY:\n\t\tif lmsg {\n\t\t\treturn c.processRoutedOriginClusterMsgArgs(c.argBuf)\n\t\t}\n\t\tif c.pa.hdr < 0 {\n\t\t\treturn c.processRoutedMsgArgs(c.argBuf)\n\t\t} else {\n\t\t\treturn c.processRoutedHeaderMsgArgs(c.argBuf)\n\t\t}\n\tcase LEAF:\n\t\tif c.pa.hdr < 0 {\n\t\t\treturn c.processLeafMsgArgs(c.argBuf)\n\t\t} else {\n\t\t\treturn c.processLeafHeaderMsgArgs(c.argBuf)\n\t\t}\n\tdefault:\n\t\tif c.pa.hdr < 0 {\n\t\t\treturn c.processPub(c.argBuf)\n\t\t} else {\n\t\t\treturn c.processHeaderPub(c.argBuf, nil)\n\t\t}\n\t}\n}\n\nfunc (ps *parseState) getHeader() http.Header {\n\tif ps.header == nil {\n\t\tif hdr := ps.pa.hdr; hdr > 0 {\n\t\t\treader := bufio.NewReader(bytes.NewReader(ps.msgBuf[0:hdr]))\n\t\t\ttp := textproto.NewReader(reader)\n\t\t\ttp.ReadLine() // skip over first line, contains version\n\t\t\tif mimeHeader, err := tp.ReadMIMEHeader(); err == nil {\n\t\t\t\tps.header = http.Header(mimeHeader)\n\t\t\t}\n\t\t}\n\t}\n\treturn ps.header\n}\n" }, { "path": "server/parser_fuzz_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc dummyFuzzClient(kind int) *client {\n\tvar r *route\n\tvar gw *gateway\n\tvar lf *leaf\n\n\tswitch kind {\n\tcase ROUTER:\n\t\tr = &route{}\n\tcase GATEWAY:\n\t\tgw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie), outsim: &sync.Map{}}\n\tcase LEAF:\n\t\tlf = &leaf{}\n\t}\n\n\treturn &client{\n\t\tsrv: New(&defaultServerOptions),\n\t\tkind: kind,\n\t\tmsubs: -1,\n\t\tin: readCache{\n\t\t\tresults: make(map[string]*SublistResult),\n\t\t\tpacache: make(map[string]*perAccountCache),\n\t\t},\n\t\tmpay: MAX_PAYLOAD_SIZE,\n\t\tmcl: MAX_CONTROL_LINE_SIZE,\n\t\troute: r,\n\t\tgw: gw,\n\t\tleaf: lf,\n\t}\n}\n\n// FuzzParser performs fuzz testing on the NATS protocol parser implementation.\n// It tests the parser's ability to handle various NATS protocol messages, including\n// partial (chunked) message delivery scenarios that may occur in real-world usage.\nfunc FuzzParser(f *testing.F) {\n\tmsgs := []string{\n\t\t\"PING\\r\\n\",\n\t\t\"PONG\\r\\n\",\n\t\t\"PUB foo 33333\\r\\n\",\n\t\t\"HPUB foo INBOX.22 0 5\\r\\nHELLO\\r\",\n\t\t\"HMSG $foo foo 10 8\\r\\nXXXhello\\r\",\n\t\t\"MSG $foo foo 5\\r\\nhello\\r\",\n\t\t\"SUB foo 1\\r\\nSUB foo 2\\r\\n\",\n\t\t\"UNSUB 1 5\\r\\n\",\n\t\t\"RMSG $G foo.bar | baz 11\\r\\nhello world\\r\",\n\t\t\"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":true,\\\"tls_required\\\":false}\\r\\n\",\n\t}\n\n\tclientKinds := []int{\n\t\tCLIENT,\n\t\tROUTER,\n\t\tGATEWAY,\n\t\tLEAF,\n\t}\n\n\tfor _, ck := range clientKinds {\n\t\tfor _, crp := range msgs {\n\t\t\tf.Add(ck, crp)\n\t\t}\n\t}\n\n\tf.Fuzz(func(t *testing.T, kind int, orig string) {\n\t\tc := dummyFuzzClient(kind)\n\n\t\tdata := []byte(orig)\n\t\thalf := len(data) / 2\n\n\t\tif err := c.parse(data[:half]); err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tif err := c.parse(data[half:]); err != nil {\n\t\t\treturn\n\t\t}\n\t})\n}\n" }, { "path": "server/parser_test.go", "content": "// Copyright 2012-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n)\n\nfunc dummyClient() *client {\n\treturn &client{srv: New(&defaultServerOptions), msubs: -1, mpay: -1, mcl: MAX_CONTROL_LINE_SIZE}\n}\n\nfunc dummyRouteClient() *client {\n\treturn &client{srv: New(&defaultServerOptions), kind: ROUTER}\n}\n\nfunc TestParsePing(t *testing.T) {\n\tc := dummyClient()\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START vs %d\\n\", c.state)\n\t}\n\tping := []byte(\"PING\\r\\n\")\n\terr := c.parse(ping[:1])\n\tif err != nil || c.state != OP_P {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping[1:2])\n\tif err != nil || c.state != OP_PI {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping[2:3])\n\tif err != nil || c.state != OP_PIN {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping[3:4])\n\tif err != nil || c.state != OP_PING {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping[4:5])\n\tif err != nil || c.state != OP_PING {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping[5:6])\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(ping)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\t// Should tolerate spaces\n\tping = []byte(\"PING \\r\")\n\terr = c.parse(ping)\n\tif err != nil || c.state != OP_PING {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tc.state = OP_START\n\tping = []byte(\"PING \\r \\n\")\n\terr = c.parse(ping)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n}\n\nfunc TestParsePong(t *testing.T) {\n\tc := dummyClient()\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START vs %d\\n\", c.state)\n\t}\n\tpong := []byte(\"PONG\\r\\n\")\n\terr := c.parse(pong[:1])\n\tif err != nil || c.state != OP_P {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(pong[1:2])\n\tif err != nil || c.state != OP_PO {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(pong[2:3])\n\tif err != nil || c.state != OP_PON {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(pong[3:4])\n\tif err != nil || c.state != OP_PONG {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(pong[4:5])\n\tif err != nil || c.state != OP_PONG {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(pong[5:6])\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif c.ping.out != 0 {\n\t\tt.Fatalf(\"Unexpected ping.out value: %d vs 0\\n\", c.ping.out)\n\t}\n\terr = c.parse(pong)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif c.ping.out != 0 {\n\t\tt.Fatalf(\"Unexpected ping.out value: %d vs 0\\n\", c.ping.out)\n\t}\n\t// Should tolerate spaces\n\tpong = []byte(\"PONG \\r\")\n\terr = c.parse(pong)\n\tif err != nil || c.state != OP_PONG {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tc.state = OP_START\n\tpong = []byte(\"PONG \\r \\n\")\n\terr = c.parse(pong)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif c.ping.out != 0 {\n\t\tt.Fatalf(\"Unexpected ping.out value: %d vs 0\\n\", c.ping.out)\n\t}\n\n\t// Should be adjusting c.pout (Pings Outstanding): reset to 0\n\tc.state = OP_START\n\tc.ping.out = 10\n\tpong = []byte(\"PONG\\r\\n\")\n\terr = c.parse(pong)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif c.ping.out != 0 {\n\t\tt.Fatalf(\"Unexpected ping.out: %d vs 0\\n\", c.ping.out)\n\t}\n}\n\nfunc TestParseConnect(t *testing.T) {\n\tc := dummyClient()\n\tconnect := []byte(\"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":true,\\\"tls_required\\\":false}\\r\\n\")\n\terr := c.parse(connect)\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\t// Check saved state\n\tif c.as != 8 {\n\t\tt.Fatalf(\"ArgStart state incorrect: 8 vs %d\\n\", c.as)\n\t}\n}\n\nfunc TestParseSub(t *testing.T) {\n\tc := dummyClient()\n\tsub := []byte(\"SUB foo 1\\r\")\n\terr := c.parse(sub)\n\tif err != nil || c.state != SUB_ARG {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\t// Check saved state\n\tif c.as != 4 {\n\t\tt.Fatalf(\"ArgStart state incorrect: 4 vs %d\\n\", c.as)\n\t}\n\tif c.drop != 1 {\n\t\tt.Fatalf(\"Drop state incorrect: 1 vs %d\\n\", c.as)\n\t}\n\tif !bytes.Equal(sub[c.as:], []byte(\"foo 1\\r\")) {\n\t\tt.Fatalf(\"Arg state incorrect: %s\\n\", sub[c.as:])\n\t}\n}\n\nfunc TestParsePub(t *testing.T) {\n\tc := dummyClient()\n\n\tpub := []byte(\"PUB foo 5\\r\\nhello\\r\")\n\terr := c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", string(c.pa.subject))\n\t}\n\tif c.pa.reply != nil {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'nil' vs '%s'\\n\", string(c.pa.reply))\n\t}\n\tif c.pa.size != 5 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 5 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"PUB foo.bar INBOX.22 11\\r\\nhello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", string(c.pa.subject))\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'INBOX.22' vs '%s'\\n\", string(c.pa.reply))\n\t}\n\tif c.pa.size != 11 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 11 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\t// This is the case when data has more bytes than expected by size.\n\tpub = []byte(\"PUB foo.bar 11\\r\\nhello world hello world\\r\")\n\terr = c.parse(pub)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error parsing longer than expected message body\")\n\t}\n\tif c.msgBuf != nil {\n\t\tt.Fatalf(\"Did not expect a c.msgBuf to be non-nil\")\n\t}\n}\n\n// https://www.twistlock.com/labs-blog/finding-dos-vulnerability-nats-go-fuzz-cve-2019-13126/\nfunc TestParsePubSizeOverflow(t *testing.T) {\n\tc := dummyClient()\n\n\tpub := []byte(\"PUB foo 3333333333333333333333333333333333333333333333333333333333333333\\r\\n\")\n\tif err := c.parse(pub); err == nil {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n}\n\nfunc TestParsePubArg(t *testing.T) {\n\tc := dummyClient()\n\n\tfor _, test := range []struct {\n\t\targ string\n\t\tsubject string\n\t\treply string\n\t\tsize int\n\t\tszb string\n\t}{\n\t\t{arg: \"a 2\", subject: \"a\", reply: \"\", size: 2, szb: \"2\"},\n\t\t{arg: \"a 222\", subject: \"a\", reply: \"\", size: 222, szb: \"222\"},\n\t\t{arg: \"foo 22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo 22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo 22 \", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo 22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo 22 \", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo 22 \", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 22 \", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 22 \", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 22 \", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 2222 \", subject: \"foo\", reply: \"bar\", size: 2222, szb: \"2222\"},\n\t\t{arg: \" foo 2222 \", subject: \"foo\", reply: \"\", size: 2222, szb: \"2222\"},\n\t\t{arg: \"a\\t2\", subject: \"a\", reply: \"\", size: 2, szb: \"2\"},\n\t\t{arg: \"a\\t222\", subject: \"a\", reply: \"\", size: 222, szb: \"222\"},\n\t\t{arg: \"foo\\t22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t22\\t\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t\\t\\t22\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t22\\t\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t\\t\\t22\\t\", subject: \"foo\", reply: \"\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\tbar\\t22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\tbar\\t22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\tbar\\t22\\t\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t\\tbar\\t\\t22\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\tbar\\t22\\t\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\t \\tfoo\\t \\t \\tbar\\t \\t22\\t \\t\", subject: \"foo\", reply: \"bar\", size: 22, szb: \"22\"},\n\t\t{arg: \"\\t\\tfoo\\t\\t\\tbar\\t\\t2222\\t\\t\", subject: \"foo\", reply: \"bar\", size: 2222, szb: \"2222\"},\n\t\t{arg: \"\\t \\tfoo\\t \\t \\t\\t\\t2222\\t \\t\", subject: \"foo\", reply: \"\", size: 2222, szb: \"2222\"},\n\t} {\n\t\tt.Run(test.arg, func(t *testing.T) {\n\t\t\tif err := c.processPub([]byte(test.arg)); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.subject, []byte(test.subject)) {\n\t\t\t\tt.Fatalf(\"Mismatched subject: '%s'\\n\", c.pa.subject)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.reply, []byte(test.reply)) {\n\t\t\t\tt.Fatalf(\"Mismatched reply subject: '%s'\\n\", c.pa.reply)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.szb, []byte(test.szb)) {\n\t\t\t\tt.Fatalf(\"Bad size buf: '%s'\\n\", c.pa.szb)\n\t\t\t}\n\t\t\tif c.pa.size != test.size {\n\t\t\t\tt.Fatalf(\"Bad size: %d\\n\", c.pa.size)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParsePubBadSize(t *testing.T) {\n\tc := dummyClient()\n\t// Setup localized max payload\n\tc.mpay = 32768\n\tif err := c.processPub([]byte(\"foo 2222222222222222\")); err == nil {\n\t\tt.Fatalf(\"Expected parse error for size too large\")\n\t}\n}\n\nfunc TestParseHeaderPub(t *testing.T) {\n\tc := dummyClient()\n\tc.headers = true\n\n\thpub := []byte(\"HPUB foo 12 17\\r\\nname:derek\\r\\nHELLO\\r\")\n\tif err := c.parse(hpub); err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\", c.pa.subject)\n\t}\n\tif c.pa.reply != nil {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'nil' vs '%s'\", c.pa.reply)\n\t}\n\tif c.pa.hdr != 12 {\n\t\tt.Fatalf(\"Did not parse msg header size correctly: 12 vs %d\", c.pa.hdr)\n\t}\n\tif !bytes.Equal(c.pa.hdb, []byte(\"12\")) {\n\t\tt.Fatalf(\"Did not parse or capture the header size as bytes correctly: %q\", c.pa.hdb)\n\t}\n\tif c.pa.size != 17 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 17 vs %d\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\thpub = []byte(\"HPUB foo INBOX.22 12 17\\r\\nname:derek\\r\\nHELLO\\r\")\n\tif err := c.parse(hpub); err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'INBOX.22' vs '%s'\", c.pa.reply)\n\t}\n\tif c.pa.hdr != 12 {\n\t\tt.Fatalf(\"Did not parse msg header size correctly: 12 vs %d\", c.pa.hdr)\n\t}\n\tif !bytes.Equal(c.pa.hdb, []byte(\"12\")) {\n\t\tt.Fatalf(\"Did not parse or capture the header size as bytes correctly: %q\", c.pa.hdb)\n\t}\n\tif c.pa.size != 17 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 17 vs %d\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\thpub = []byte(\"HPUB foo INBOX.22 0 5\\r\\nHELLO\\r\")\n\tif err := c.parse(hpub); err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'INBOX.22' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.hdr != 0 {\n\t\tt.Fatalf(\"Did not parse msg header size correctly: 0 vs %d\\n\", c.pa.hdr)\n\t}\n\tif !bytes.Equal(c.pa.hdb, []byte(\"0\")) {\n\t\tt.Fatalf(\"Did not parse or capture the header size as bytes correctly: %q\", c.pa.hdb)\n\t}\n\tif c.pa.size != 5 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 5 vs %d\\n\", c.pa.size)\n\t}\n}\n\nfunc TestParseHeaderPubArg(t *testing.T) {\n\tc := dummyClient()\n\tc.headers = true\n\n\tfor _, test := range []struct {\n\t\targ string\n\t\tsubject string\n\t\treply string\n\t\thdr int\n\t\tsize int\n\t\tszb string\n\t}{\n\t\t{arg: \"a 2 4\", subject: \"a\", reply: \"\", hdr: 2, size: 4, szb: \"4\"},\n\t\t{arg: \"a 22 222\", subject: \"a\", reply: \"\", hdr: 22, size: 222, szb: \"222\"},\n\t\t{arg: \"foo 3 22\", subject: \"foo\", reply: \"\", hdr: 3, size: 22, szb: \"22\"},\n\t\t{arg: \" foo 1 22\", subject: \"foo\", reply: \"\", hdr: 1, size: 22, szb: \"22\"},\n\t\t{arg: \"foo 0 22 \", subject: \"foo\", reply: \"\", hdr: 0, size: 22, szb: \"22\"},\n\t\t{arg: \"foo 0 22\", subject: \"foo\", reply: \"\", hdr: 0, size: 22, szb: \"22\"},\n\t\t{arg: \" foo 1 22 \", subject: \"foo\", reply: \"\", hdr: 1, size: 22, szb: \"22\"},\n\t\t{arg: \" foo 3 22 \", subject: \"foo\", reply: \"\", hdr: 3, size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 1 22\", subject: \"foo\", reply: \"bar\", hdr: 1, size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 11 22\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 11 22 \", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo bar 11 22\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 11 22 \", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 11 22 \", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \" foo bar 22 2222 \", subject: \"foo\", reply: \"bar\", hdr: 22, size: 2222, szb: \"2222\"},\n\t\t{arg: \" foo 1 2222 \", subject: \"foo\", reply: \"\", hdr: 1, size: 2222, szb: \"2222\"},\n\t\t{arg: \"a\\t2\\t22\", subject: \"a\", reply: \"\", hdr: 2, size: 22, szb: \"22\"},\n\t\t{arg: \"a\\t2\\t\\t222\", subject: \"a\", reply: \"\", hdr: 2, size: 222, szb: \"222\"},\n\t\t{arg: \"foo\\t2 22\", subject: \"foo\", reply: \"\", hdr: 2, size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t11\\t 22\", subject: \"foo\", reply: \"\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t11\\t22\\t\", subject: \"foo\", reply: \"\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t\\t\\t11 22\", subject: \"foo\", reply: \"\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t11\\t \\t 22\\t\", subject: \"foo\", reply: \"\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\t\\t\\t11 22\\t\", subject: \"foo\", reply: \"\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\tbar\\t2 22\", subject: \"foo\", reply: \"bar\", hdr: 2, size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\tbar\\t11\\t22\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\tbar\\t11\\t\\t22\\t \", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"foo\\t\\tbar\\t\\t11\\t\\t\\t22\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"\\tfoo\\tbar\\t11\\t22\\t\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"\\t \\tfoo\\t \\t \\tbar\\t \\t11\\t 22\\t \\t\", subject: \"foo\", reply: \"bar\", hdr: 11, size: 22, szb: \"22\"},\n\t\t{arg: \"\\t\\tfoo\\t\\t\\tbar\\t\\t22\\t\\t\\t2222\\t\\t\", subject: \"foo\", reply: \"bar\", hdr: 22, size: 2222, szb: \"2222\"},\n\t\t{arg: \"\\t \\tfoo\\t \\t \\t\\t\\t11\\t\\t 2222\\t \\t\", subject: \"foo\", reply: \"\", hdr: 11, size: 2222, szb: \"2222\"},\n\t} {\n\t\tt.Run(test.arg, func(t *testing.T) {\n\t\t\tif err := c.processHeaderPub([]byte(test.arg), nil); err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.subject, []byte(test.subject)) {\n\t\t\t\tt.Fatalf(\"Mismatched subject: '%s'\\n\", c.pa.subject)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.reply, []byte(test.reply)) {\n\t\t\t\tt.Fatalf(\"Mismatched reply subject: '%s'\\n\", c.pa.reply)\n\t\t\t}\n\t\t\tif !bytes.Equal(c.pa.szb, []byte(test.szb)) {\n\t\t\t\tt.Fatalf(\"Bad size buf: '%s'\\n\", c.pa.szb)\n\t\t\t}\n\t\t\tif c.pa.hdr != test.hdr {\n\t\t\t\tt.Fatalf(\"Bad header size: %d\\n\", c.pa.hdr)\n\t\t\t}\n\t\t\tif c.pa.size != test.size {\n\t\t\t\tt.Fatalf(\"Bad size: %d\\n\", c.pa.size)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseRoutedHeaderMsg(t *testing.T) {\n\tc := dummyRouteClient()\n\tc.route = &route{}\n\n\tpub := []byte(\"HMSG $foo foo 10 8\\r\\nXXXhello\\r\")\n\tif err := c.parse(pub); err == nil {\n\t\tt.Fatalf(\"Expected an error\")\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"HMSG $foo foo 3 8\\r\\nXXXhello\\r\")\n\terr := c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$foo\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$foo' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif c.pa.reply != nil {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'nil' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.hdr != 3 {\n\t\tt.Fatalf(\"Did not parse header size correctly: 3 vs %d\\n\", c.pa.hdr)\n\t}\n\tif c.pa.size != 8 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 8 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"HMSG $G foo.bar INBOX.22 3 14\\r\\nOK:hello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'INBOX.22' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.hdr != 3 {\n\t\tt.Fatalf(\"Did not parse header size correctly: 3 vs %d\\n\", c.pa.hdr)\n\t}\n\tif c.pa.size != 14 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 14 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"HMSG $G foo.bar + reply baz 3 14\\r\\nOK:hello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"reply\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'reply' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif len(c.pa.queues) != 1 {\n\t\tt.Fatalf(\"Expected 1 queue, got %d\", len(c.pa.queues))\n\t}\n\tif !bytes.Equal(c.pa.queues[0], []byte(\"baz\")) {\n\t\tt.Fatalf(\"Did not parse queues correctly: 'baz' vs '%q'\\n\", c.pa.queues[0])\n\t}\n\tif c.pa.hdr != 3 {\n\t\tt.Fatalf(\"Did not parse header size correctly: 3 vs %d\\n\", c.pa.hdr)\n\t}\n\tif c.pa.size != 14 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 14 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"HMSG $G foo.bar | baz 3 14\\r\\nOK:hello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: '' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif len(c.pa.queues) != 1 {\n\t\tt.Fatalf(\"Expected 1 queue, got %d\", len(c.pa.queues))\n\t}\n\tif !bytes.Equal(c.pa.queues[0], []byte(\"baz\")) {\n\t\tt.Fatalf(\"Did not parse queues correctly: 'baz' vs '%q'\\n\", c.pa.queues[0])\n\t}\n\tif c.pa.hdr != 3 {\n\t\tt.Fatalf(\"Did not parse header size correctly: 3 vs %d\\n\", c.pa.hdr)\n\t}\n\tif c.pa.size != 14 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 14 vs %d\\n\", c.pa.size)\n\t}\n}\n\nfunc TestParseRouteMsg(t *testing.T) {\n\tc := dummyRouteClient()\n\tc.route = &route{}\n\n\tpub := []byte(\"MSG $foo foo 5\\r\\nhello\\r\")\n\terr := c.parse(pub)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error, got none\")\n\t}\n\tpub = []byte(\"RMSG $foo foo 5\\r\\nhello\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$foo\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$foo' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif c.pa.reply != nil {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'nil' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.size != 5 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 5 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"RMSG $G foo.bar INBOX.22 11\\r\\nhello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'INBOX.22' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.size != 11 {\n\t\tt.Fatalf(\"Did not parse msg size correctly: 11 vs %d\\n\", c.pa.size)\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"RMSG $G foo.bar + reply baz 11\\r\\nhello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"reply\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: 'reply' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif len(c.pa.queues) != 1 {\n\t\tt.Fatalf(\"Expected 1 queue, got %d\", len(c.pa.queues))\n\t}\n\tif !bytes.Equal(c.pa.queues[0], []byte(\"baz\")) {\n\t\tt.Fatalf(\"Did not parse queues correctly: 'baz' vs '%q'\\n\", c.pa.queues[0])\n\t}\n\n\t// Clear snapshots\n\tc.argBuf, c.msgBuf, c.state = nil, nil, OP_START\n\n\tpub = []byte(\"RMSG $G foo.bar | baz 11\\r\\nhello world\\r\")\n\terr = c.parse(pub)\n\tif err != nil || c.state != MSG_END_N {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\tif !bytes.Equal(c.pa.account, []byte(\"$G\")) {\n\t\tt.Fatalf(\"Did not parse account correctly: '$G' vs '%s'\\n\", c.pa.account)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"Did not parse subject correctly: 'foo' vs '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"\")) {\n\t\tt.Fatalf(\"Did not parse reply correctly: '' vs '%s'\\n\", c.pa.reply)\n\t}\n\tif len(c.pa.queues) != 1 {\n\t\tt.Fatalf(\"Expected 1 queue, got %d\", len(c.pa.queues))\n\t}\n\tif !bytes.Equal(c.pa.queues[0], []byte(\"baz\")) {\n\t\tt.Fatalf(\"Did not parse queues correctly: 'baz' vs '%q'\\n\", c.pa.queues[0])\n\t}\n}\n\nfunc TestParseMsgSpace(t *testing.T) {\n\tc := dummyRouteClient()\n\n\t// Ivan bug he found\n\tif err := c.parse([]byte(\"MSG \\r\\n\")); err == nil {\n\t\tt.Fatalf(\"Expected parse error for MSG \")\n\t}\n\n\tc = dummyClient()\n\n\t// Anything with an M from a client should parse error\n\tif err := c.parse([]byte(\"M\")); err == nil {\n\t\tt.Fatalf(\"Expected parse error for M* from a client\")\n\t}\n}\n\nfunc TestShouldFail(t *testing.T) {\n\twrongProtos := []string{\n\t\t\"xxx\",\n\t\t\"Px\", \"PIx\", \"PINx\", \" PING\",\n\t\t\"POx\", \"PONx\",\n\t\t\"+x\", \"+Ox\",\n\t\t\"-x\", \"-Ex\", \"-ERx\", \"-ERRx\",\n\t\t\"Cx\", \"COx\", \"CONx\", \"CONNx\", \"CONNEx\", \"CONNECx\", \"CONNECx\", \"CONNECT \\r\\n\",\n\t\t\"PUx\", \"PUB foo\\r\\n\", \"PUB \\r\\n\", \"PUB foo bar \\r\\n\",\n\t\t\"PUB foo 2\\r\\nok \\r\\n\", \"PUB foo 2\\r\\nok\\r \\n\",\n\t\t\"Sx\", \"SUx\", \"SUB\\r\\n\", \"SUB \\r\\n\", \"SUB foo\\r\\n\",\n\t\t\"SUB foo bar baz 22\\r\\n\",\n\t\t\"Ux\", \"UNx\", \"UNSx\", \"UNSUx\", \"UNSUBx\", \"UNSUBUNSUB 1\\r\\n\", \"UNSUB_2\\r\\n\",\n\t\t\"UNSUB_UNSUB_UNSUB 2\\r\\n\", \"UNSUB_\\t2\\r\\n\", \"UNSUB\\r\\n\", \"UNSUB \\r\\n\",\n\t\t\"UNSUB \\t \\r\\n\",\n\t\t\"Ix\", \"INx\", \"INFx\", \"INFO \\r\\n\",\n\t}\n\tfor _, proto := range wrongProtos {\n\t\tc := dummyClient()\n\t\tif err := c.parse([]byte(proto)); err == nil {\n\t\t\tt.Fatalf(\"Should have received a parse error for: %v\", proto)\n\t\t}\n\t}\n\n\t// Special case for MSG, type needs to not be client.\n\twrongProtos = []string{\"Mx\", \"MSx\", \"MSGx\", \"MSG \\r\\n\"}\n\tfor _, proto := range wrongProtos {\n\t\tc := dummyClient()\n\t\tc.kind = ROUTER\n\t\tif err := c.parse([]byte(proto)); err == nil {\n\t\t\tt.Fatalf(\"Should have received a parse error for: %v\", proto)\n\t\t}\n\t}\n}\n\nfunc TestProtoSnippet(t *testing.T) {\n\tsample := []byte(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\")\n\n\ttests := []struct {\n\t\tinput int\n\t\texpected string\n\t}{\n\t\t{0, `\"abcdefghijklmnopqrstuvwxyzABCDEF\"`},\n\t\t{1, `\"bcdefghijklmnopqrstuvwxyzABCDEFG\"`},\n\t\t{2, `\"cdefghijklmnopqrstuvwxyzABCDEFGH\"`},\n\t\t{3, `\"defghijklmnopqrstuvwxyzABCDEFGHI\"`},\n\t\t{4, `\"efghijklmnopqrstuvwxyzABCDEFGHIJ\"`},\n\t\t{5, `\"fghijklmnopqrstuvwxyzABCDEFGHIJK\"`},\n\t\t{6, `\"ghijklmnopqrstuvwxyzABCDEFGHIJKL\"`},\n\t\t{7, `\"hijklmnopqrstuvwxyzABCDEFGHIJKLM\"`},\n\t\t{8, `\"ijklmnopqrstuvwxyzABCDEFGHIJKLMN\"`},\n\t\t{9, `\"jklmnopqrstuvwxyzABCDEFGHIJKLMNO\"`},\n\t\t{10, `\"klmnopqrstuvwxyzABCDEFGHIJKLMNOP\"`},\n\t\t{11, `\"lmnopqrstuvwxyzABCDEFGHIJKLMNOPQ\"`},\n\t\t{12, `\"mnopqrstuvwxyzABCDEFGHIJKLMNOPQR\"`},\n\t\t{13, `\"nopqrstuvwxyzABCDEFGHIJKLMNOPQRS\"`},\n\t\t{14, `\"opqrstuvwxyzABCDEFGHIJKLMNOPQRST\"`},\n\t\t{15, `\"pqrstuvwxyzABCDEFGHIJKLMNOPQRSTU\"`},\n\t\t{16, `\"qrstuvwxyzABCDEFGHIJKLMNOPQRSTUV\"`},\n\t\t{17, `\"rstuvwxyzABCDEFGHIJKLMNOPQRSTUVW\"`},\n\t\t{18, `\"stuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\"`},\n\t\t{19, `\"tuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{20, `\"uvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\"`},\n\t\t{21, `\"vwxyzABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{22, `\"wxyzABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{23, `\"xyzABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{24, `\"yzABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{25, `\"zABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{26, `\"ABCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{27, `\"BCDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{28, `\"CDEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{29, `\"DEFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{30, `\"EFGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{31, `\"FGHIJKLMNOPQRSTUVWXY\"`},\n\t\t{32, `\"GHIJKLMNOPQRSTUVWXY\"`},\n\t\t{33, `\"HIJKLMNOPQRSTUVWXY\"`},\n\t\t{34, `\"IJKLMNOPQRSTUVWXY\"`},\n\t\t{35, `\"JKLMNOPQRSTUVWXY\"`},\n\t\t{36, `\"KLMNOPQRSTUVWXY\"`},\n\t\t{37, `\"LMNOPQRSTUVWXY\"`},\n\t\t{38, `\"MNOPQRSTUVWXY\"`},\n\t\t{39, `\"NOPQRSTUVWXY\"`},\n\t\t{40, `\"OPQRSTUVWXY\"`},\n\t\t{41, `\"PQRSTUVWXY\"`},\n\t\t{42, `\"QRSTUVWXY\"`},\n\t\t{43, `\"RSTUVWXY\"`},\n\t\t{44, `\"STUVWXY\"`},\n\t\t{45, `\"TUVWXY\"`},\n\t\t{46, `\"UVWXY\"`},\n\t\t{47, `\"VWXY\"`},\n\t\t{48, `\"WXY\"`},\n\t\t{49, `\"XY\"`},\n\t\t{50, `\"Y\"`},\n\t\t{51, `\"\"`},\n\t\t{52, `\"\"`},\n\t\t{53, `\"\"`},\n\t\t{54, `\"\"`},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot := protoSnippet(tt.input, PROTO_SNIPPET_SIZE, sample)\n\t\tif tt.expected != got {\n\t\t\tt.Errorf(\"Expected protocol snippet to be %s when start=%d but got %s\\n\", tt.expected, tt.input, got)\n\t\t}\n\t}\n}\n\nfunc TestParseOK(t *testing.T) {\n\tc := dummyClient()\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START vs %d\\n\", c.state)\n\t}\n\tokProto := []byte(\"+OK\\r\\n\")\n\terr := c.parse(okProto[:1])\n\tif err != nil || c.state != OP_PLUS {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(okProto[1:2])\n\tif err != nil || c.state != OP_PLUS_O {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(okProto[2:3])\n\tif err != nil || c.state != OP_PLUS_OK {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(okProto[3:4])\n\tif err != nil || c.state != OP_PLUS_OK {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n\terr = c.parse(okProto[4:5])\n\tif err != nil || c.state != OP_START {\n\t\tt.Fatalf(\"Unexpected: %d : %v\\n\", c.state, err)\n\t}\n}\n\nfunc TestMaxControlLine(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tkind int\n\t\tshouldFail bool\n\t}{\n\t\t{\"client\", CLIENT, true},\n\t\t{\"leaf\", LEAF, false},\n\t\t{\"route\", ROUTER, false},\n\t\t{\"gateway\", GATEWAY, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tpub := []byte(\"PUB foo.bar.baz 2\\r\\nok\\r\\n\")\n\n\t\t\tsetupClient := func() *client {\n\t\t\t\tc := dummyClient()\n\t\t\t\tc.setNoReconnect()\n\t\t\t\tc.flags.set(connectReceived)\n\t\t\t\tc.kind = test.kind\n\t\t\t\tswitch test.kind {\n\t\t\t\tcase ROUTER:\n\t\t\t\t\tc.route = &route{}\n\t\t\t\tcase GATEWAY:\n\t\t\t\t\tc.gw = &gateway{outbound: false, connected: true, insim: make(map[string]*insie)}\n\t\t\t\t}\n\t\t\t\tc.mcl = 8\n\t\t\t\treturn c\n\t\t\t}\n\n\t\t\tc := setupClient()\n\t\t\t// First try with a partial:\n\t\t\t// PUB foo.bar.baz 2\\r\\nok\\r\\n\n\t\t\t// .............^\n\t\t\terr := c.parse(pub[:14])\n\t\t\tswitch test.shouldFail {\n\t\t\tcase true:\n\t\t\t\tif !ErrorIs(err, ErrMaxControlLine) {\n\t\t\t\t\tt.Fatalf(\"Expected an error parsing longer than expected control line\")\n\t\t\t\t}\n\t\t\tcase false:\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Should not have failed, got %v\", err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Now with full protocol (no split) and we should still enforce.\n\t\t\tc = setupClient()\n\t\t\terr = c.parse(pub)\n\t\t\tswitch test.shouldFail {\n\t\t\tcase true:\n\t\t\t\tif !ErrorIs(err, ErrMaxControlLine) {\n\t\t\t\t\tt.Fatalf(\"Expected an error parsing longer than expected control line\")\n\t\t\t\t}\n\t\t\tcase false:\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Should not have failed, got %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/ping_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n)\n\nconst PING_CLIENT_PORT = 11228\n\nvar DefaultPingOptions = Options{\n\tHost: \"127.0.0.1\",\n\tPort: PING_CLIENT_PORT,\n\tNoLog: true,\n\tNoSigs: true,\n\tPingInterval: 50 * time.Millisecond,\n}\n\nfunc TestPing(t *testing.T) {\n\to := DefaultPingOptions\n\to.DisableShortFirstPing = true\n\ts := RunServer(&o)\n\tdefer s.Shutdown()\n\n\tc, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", PING_CLIENT_PORT))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer c.Close()\n\tbr := bufio.NewReader(c)\n\t// Wait for INFO\n\tbr.ReadLine()\n\t// Send CONNECT\n\tc.Write([]byte(\"CONNECT {\\\"verbose\\\":false}\\r\\nPING\\r\\n\"))\n\t// Wait for first PONG\n\tbr.ReadLine()\n\t// Wait for PING\n\tstart := time.Now()\n\tfor i := 0; i < 3; i++ {\n\t\tl, _, err := br.ReadLine()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error: %v\", err)\n\t\t}\n\t\tif string(l) != \"PING\" {\n\t\t\tt.Fatalf(\"Expected PING, got %q\", l)\n\t\t}\n\t\tif dur := time.Since(start); dur < 25*time.Millisecond || dur > 75*time.Millisecond {\n\t\t\tt.Fatalf(\"Pings duration off: %v\", dur)\n\t\t}\n\t\tc.Write([]byte(pongProto))\n\t\tstart = time.Now()\n\t}\n}\n" }, { "path": "server/proto.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Inspired by https://github.com/protocolbuffers/protobuf-go/blob/master/encoding/protowire/wire.go\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n)\n\nvar errProtoInsufficient = errors.New(\"insufficient data to read a value\")\nvar errProtoOverflow = errors.New(\"too much data for a value\")\nvar errProtoInvalidFieldNumber = errors.New(\"invalid field number\")\n\nfunc protoScanField(b []byte) (num, typ, size int, err error) {\n\tnum, typ, sizeTag, err := protoScanTag(b)\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\tb = b[sizeTag:]\n\n\tsizeValue, err := protoScanFieldValue(typ, b)\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\treturn num, typ, sizeTag + sizeValue, nil\n}\n\nfunc protoScanTag(b []byte) (num, typ, size int, err error) {\n\ttagint, size, err := protoScanVarint(b)\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\n\t// NOTE: MessageSet allows for larger field numbers than normal.\n\tif (tagint >> 3) > uint64(math.MaxInt32) {\n\t\treturn 0, 0, 0, errProtoInvalidFieldNumber\n\t}\n\tnum = int(tagint >> 3)\n\tif num < 1 {\n\t\treturn 0, 0, 0, errProtoInvalidFieldNumber\n\t}\n\ttyp = int(tagint & 7)\n\n\treturn num, typ, size, nil\n}\n\nfunc protoScanFieldValue(typ int, b []byte) (size int, err error) {\n\tswitch typ {\n\tcase 0:\n\t\t_, size, err = protoScanVarint(b)\n\tcase 5: // fixed32\n\t\tif len(b) < 4 {\n\t\t\treturn 0, errProtoInsufficient\n\t\t}\n\t\tsize = 4\n\tcase 1: // fixed64\n\t\tif len(b) < 8 {\n\t\t\treturn 0, errProtoInsufficient\n\t\t}\n\t\tsize = 8\n\tcase 2: // length-delimited\n\t\tsize, err = protoScanBytes(b)\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"unsupported type: %d\", typ)\n\t}\n\treturn size, err\n}\n\nfunc protoScanVarint(b []byte) (v uint64, size int, err error) {\n\tvar y uint64\n\tif len(b) <= 0 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\tv = uint64(b[0])\n\tif v < 0x80 {\n\t\treturn v, 1, nil\n\t}\n\tv -= 0x80\n\n\tif len(b) <= 1 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[1])\n\tv += y << 7\n\tif y < 0x80 {\n\t\treturn v, 2, nil\n\t}\n\tv -= 0x80 << 7\n\n\tif len(b) <= 2 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[2])\n\tv += y << 14\n\tif y < 0x80 {\n\t\treturn v, 3, nil\n\t}\n\tv -= 0x80 << 14\n\n\tif len(b) <= 3 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[3])\n\tv += y << 21\n\tif y < 0x80 {\n\t\treturn v, 4, nil\n\t}\n\tv -= 0x80 << 21\n\n\tif len(b) <= 4 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[4])\n\tv += y << 28\n\tif y < 0x80 {\n\t\treturn v, 5, nil\n\t}\n\tv -= 0x80 << 28\n\n\tif len(b) <= 5 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[5])\n\tv += y << 35\n\tif y < 0x80 {\n\t\treturn v, 6, nil\n\t}\n\tv -= 0x80 << 35\n\n\tif len(b) <= 6 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[6])\n\tv += y << 42\n\tif y < 0x80 {\n\t\treturn v, 7, nil\n\t}\n\tv -= 0x80 << 42\n\n\tif len(b) <= 7 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[7])\n\tv += y << 49\n\tif y < 0x80 {\n\t\treturn v, 8, nil\n\t}\n\tv -= 0x80 << 49\n\n\tif len(b) <= 8 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[8])\n\tv += y << 56\n\tif y < 0x80 {\n\t\treturn v, 9, nil\n\t}\n\tv -= 0x80 << 56\n\n\tif len(b) <= 9 {\n\t\treturn 0, 0, errProtoInsufficient\n\t}\n\ty = uint64(b[9])\n\tv += y << 63\n\tif y < 2 {\n\t\treturn v, 10, nil\n\t}\n\treturn 0, 0, errProtoOverflow\n}\n\nfunc protoScanBytes(b []byte) (size int, err error) {\n\tl, lenSize, err := protoScanVarint(b)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif l > uint64(len(b[lenSize:])) {\n\t\treturn 0, errProtoInsufficient\n\t}\n\treturn lenSize + int(l), nil\n}\n\nfunc protoEncodeVarint(v uint64) []byte {\n\tb := make([]byte, 0, 10)\n\tswitch {\n\tcase v < 1<<7:\n\t\tb = append(b, byte(v))\n\tcase v < 1<<14:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte(v>>7))\n\tcase v < 1<<21:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte(v>>14))\n\tcase v < 1<<28:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte(v>>21))\n\tcase v < 1<<35:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte(v>>28))\n\tcase v < 1<<42:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte((v>>28)&0x7f|0x80),\n\t\t\tbyte(v>>35))\n\tcase v < 1<<49:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte((v>>28)&0x7f|0x80),\n\t\t\tbyte((v>>35)&0x7f|0x80),\n\t\t\tbyte(v>>42))\n\tcase v < 1<<56:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte((v>>28)&0x7f|0x80),\n\t\t\tbyte((v>>35)&0x7f|0x80),\n\t\t\tbyte((v>>42)&0x7f|0x80),\n\t\t\tbyte(v>>49))\n\tcase v < 1<<63:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte((v>>28)&0x7f|0x80),\n\t\t\tbyte((v>>35)&0x7f|0x80),\n\t\t\tbyte((v>>42)&0x7f|0x80),\n\t\t\tbyte((v>>49)&0x7f|0x80),\n\t\t\tbyte(v>>56))\n\tdefault:\n\t\tb = append(b,\n\t\t\tbyte((v>>0)&0x7f|0x80),\n\t\t\tbyte((v>>7)&0x7f|0x80),\n\t\t\tbyte((v>>14)&0x7f|0x80),\n\t\t\tbyte((v>>21)&0x7f|0x80),\n\t\t\tbyte((v>>28)&0x7f|0x80),\n\t\t\tbyte((v>>35)&0x7f|0x80),\n\t\t\tbyte((v>>42)&0x7f|0x80),\n\t\t\tbyte((v>>49)&0x7f|0x80),\n\t\t\tbyte((v>>56)&0x7f|0x80),\n\t\t\t1)\n\t}\n\treturn b\n}\n" }, { "path": "server/pse/freebsd.txt", "content": "/*\n * Compile and run this as a C program to get the kinfo_proc offsets\n * for your architecture.\n * While FreeBSD works hard at binary-compatibility within an ABI, various\n * we can't say for sure that these are right for _all_ use on a hardware\n * platform. The LP64 ifdef affects the offsets considerably.\n *\n * We use these offsets in hardware-specific files for FreeBSD, to avoid a cgo\n * compilation-time dependency, allowing us to cross-compile for FreeBSD from\n * other hardware platforms, letting us distribute binaries for FreeBSD.\n */\n\n#include \n#include \n#include \n#include \n#include \n\n#define SHOW_OFFSET(FIELD) printf(\" KIP_OFF_%s = %zu\\n\", #FIELD, offsetof(struct kinfo_proc, ki_ ## FIELD))\n\nint main(int argc, char *argv[]) {\n\t/* Uncomment these if you want some extra debugging aids:\n\tSHOW_OFFSET(pid);\n\tSHOW_OFFSET(ppid);\n\tSHOW_OFFSET(uid);\n\t*/\n\tSHOW_OFFSET(size);\n\tSHOW_OFFSET(rssize);\n\tSHOW_OFFSET(pctcpu);\n}\n" }, { "path": "server/pse/pse_darwin.go", "content": "// Copyright 2015-2021 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pse\n\n// On macs after some studying it seems that typical tools like ps and activity monitor report MaxRss and not\n// current RSS. I wrote some C code to pull the real RSS and although it does not go down very often, when it does\n// that is not reflected in the typical tooling one might compare us to, so we can skip cgo and just use rusage imo.\n// We also do not use virtual memory in the upper layers at all, so ok to skip since rusage does not report vss.\n\nimport (\n\t\"math\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n)\n\ntype lastUsage struct {\n\tsync.Mutex\n\tlast time.Time\n\tcpu time.Duration\n\trss int64\n\tpcpu float64\n}\n\n// To hold the last usage and call time.\nvar lu lastUsage\n\nfunc init() {\n\tupdateUsage()\n\tperiodic()\n}\n\n// Get our usage.\nfunc getUsage() (now time.Time, cpu time.Duration, rss int64) {\n\tvar ru syscall.Rusage\n\tsyscall.Getrusage(syscall.RUSAGE_SELF, &ru)\n\tnow = time.Now()\n\tcpu = time.Duration(ru.Utime.Sec)*time.Second + time.Duration(ru.Utime.Usec)*time.Microsecond\n\tcpu += time.Duration(ru.Stime.Sec)*time.Second + time.Duration(ru.Stime.Usec)*time.Microsecond\n\treturn now, cpu, ru.Maxrss\n}\n\n// Update last usage.\n// We need to have a prior sample to compute pcpu.\nfunc updateUsage() (pcpu float64, rss int64) {\n\tlu.Lock()\n\tdefer lu.Unlock()\n\n\tnow, cpu, rss := getUsage()\n\t// Don't skew pcpu by sampling too close to last sample.\n\tif elapsed := now.Sub(lu.last); elapsed < 500*time.Millisecond {\n\t\t// Always update rss.\n\t\tlu.rss = rss\n\t} else {\n\t\ttcpu := float64(cpu - lu.cpu)\n\t\tlu.last, lu.cpu, lu.rss = now, cpu, rss\n\t\t// Want to make this one decimal place and not count on upper layers.\n\t\t// Cores already taken into account via cpu time measurements.\n\t\tlu.pcpu = math.Round(tcpu/float64(elapsed)*1000) / 10\n\t}\n\treturn lu.pcpu, lu.rss\n}\n\n// Sampling function to keep pcpu relevant.\nfunc periodic() {\n\tupdateUsage()\n\ttime.AfterFunc(time.Second, periodic)\n}\n\n// ProcUsage returns CPU and memory usage.\n// Note upper layers do not use virtual memory size, so ok that it is not filled in here.\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\t*pcpu, *rss = updateUsage()\n\treturn nil\n}\n" }, { "path": "server/pse/pse_dragonfly.go", "content": "// Copyright 2015-2023 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Copied from pse_openbsd.go\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n)\n\n// ProcUsage returns CPU usage\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"o\", \"pcpu=,rss=,vsz=\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\t*rss, *vss = -1, -1\n\t\treturn fmt.Errorf(\"ps call failed:%v\", err)\n\t}\n\tfmt.Sscanf(string(out), \"%f %d %d\", pcpu, rss, vss)\n\t*rss *= 1024 // 1k blocks, want bytes.\n\t*vss *= 1024 // 1k blocks, want bytes.\n\treturn nil\n}\n" }, { "path": "server/pse/pse_freebsd_cgo.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build cgo && freebsd\n\npackage pse\n\n/*\n#include \n#include \n#include \n#include \n#include \n\nlong pagetok(long size)\n{\n int pageshift, pagesize;\n\n pagesize = getpagesize();\n pageshift = 0;\n\n while (pagesize > 1) {\n pageshift++;\n pagesize >>= 1;\n }\n\n return (size << pageshift);\n}\n\nint getusage(double *pcpu, unsigned int *rss, unsigned int *vss)\n{\n int mib[4], ret;\n size_t len;\n struct kinfo_proc kp;\n\n len = 4;\n sysctlnametomib(\"kern.proc.pid\", mib, &len);\n\n mib[3] = getpid();\n len = sizeof(kp);\n\n ret = sysctl(mib, 4, &kp, &len, NULL, 0);\n if (ret != 0) {\n return (errno);\n }\n\n *rss = pagetok(kp.ki_rssize);\n *vss = kp.ki_size;\n *pcpu = (double)kp.ki_pctcpu / FSCALE;\n\n return 0;\n}\n\n*/\nimport \"C\"\n\nimport (\n\t\"syscall\"\n)\n\n// This is a placeholder for now.\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tvar r, v C.uint\n\tvar c C.double\n\n\tif ret := C.getusage(&c, &r, &v); ret != 0 {\n\t\treturn syscall.Errno(ret)\n\t}\n\n\t*pcpu = float64(c)\n\t*rss = int64(r)\n\t*vss = int64(v)\n\n\treturn nil\n}\n" }, { "path": "server/pse/pse_freebsd_sysctl.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// There are two FreeBSD implementations; one which uses cgo and should build\n// locally on any FreeBSD, and this one which uses sysctl but needs us to know\n// the offset constants for the fields we care about.\n//\n// The advantage of this one is that without cgo, it is much easier to\n// cross-compile to a target. The official releases are all built with\n// cross-compilation.\n//\n// We've switched the other implementation to include '_cgo' in the filename,\n// to show that it's not the default. This isn't an os or arch build tag,\n// so we have to use explicit build-tags within.\n// If lacking CGO support and targeting an unsupported arch, then before the\n// change you would have a compile failure for not being able to cross-compile.\n// After the change, you have a compile failure for not having the symbols\n// because no source file satisfies them.\n// Thus we are no worse off, and it's now much easier to extend support for\n// non-CGO to new architectures, just by editing this file.\n//\n// To generate for other architectures:\n// 1. Copy `freebsd.txt` to have a .c filename on a box running the target\n// architecture, compile and run it.\n// 2. Update the init() function below to include a case for this architecture\n// 3. Update the build-tags in this file.\n\n//go:build !cgo && freebsd && (amd64 || arm64)\n\npackage pse\n\nimport (\n\t\"encoding/binary\"\n\t\"runtime\"\n\t\"syscall\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// On FreeBSD, to get proc information we read it out of the kernel using a\n// binary sysctl. The endianness of integers is thus explicitly \"host\", rather\n// than little or big endian.\nvar nativeEndian = binary.LittleEndian\n\nvar pageshift int // derived from getpagesize(3) in init() below\n\nvar (\n\t// These are populated in the init function, based on the current architecture.\n\t// (It's less file-count explosion than having one small file for each\n\t// FreeBSD architecture).\n\tKIP_OFF_size int\n\tKIP_OFF_rssize int\n\tKIP_OFF_pctcpu int\n)\n\nfunc init() {\n\tswitch runtime.GOARCH {\n\t// These are the values which come from compiling and running\n\t// freebsd.txt as a C program.\n\t// Most recently validated: 2025-04 with FreeBSD 14.2R in AWS.\n\tcase \"amd64\":\n\t\tKIP_OFF_size = 256\n\t\tKIP_OFF_rssize = 264\n\t\tKIP_OFF_pctcpu = 308\n\tcase \"arm64\":\n\t\tKIP_OFF_size = 256\n\t\tKIP_OFF_rssize = 264\n\t\tKIP_OFF_pctcpu = 308\n\tdefault:\n\t\tpanic(\"code bug: server/pse FreeBSD support missing case for '\" + runtime.GOARCH + \"' but build-tags allowed us to build anyway?\")\n\t}\n\n\t// To get the physical page size, the C library checks two places:\n\t// process ELF auxiliary info, AT_PAGESZ\n\t// as a fallback, the hw.pagesize sysctl\n\t// In looking closely, I found that the Go runtime support is handling\n\t// this for us, and exposing that as syscall.Getpagesize, having checked\n\t// both in the same ways, at process start, so a call to that should return\n\t// a memory value without even a syscall bounce.\n\tpagesize := syscall.Getpagesize()\n\tpageshift = 0\n\tfor pagesize > 1 {\n\t\tpageshift += 1\n\t\tpagesize >>= 1\n\t}\n}\n\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\trawdata, err := unix.SysctlRaw(\"kern.proc.pid\", unix.Getpid())\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tr_vss_bytes := nativeEndian.Uint32(rawdata[KIP_OFF_size:])\n\tr_rss_pages := nativeEndian.Uint32(rawdata[KIP_OFF_rssize:])\n\trss_bytes := r_rss_pages << pageshift\n\n\t// In C: fixpt_t ki_pctcpu\n\t// Doc: %cpu for process during ki_swtime\n\t// fixpt_t is __uint32_t\n\t// usr.bin/top uses pctdouble to convert to a double (float64)\n\t// define pctdouble(p) ((double)(p) / FIXED_PCTCPU)\n\t// FIXED_PCTCPU is _usually_ FSCALE (some architectures are special)\n\t// has:\n\t// #define FSHIFT 11 /* bits to right of fixed binary point */\n\t// #define FSCALE (1< 0 {\n\t\tatomic.StoreInt64(&ipcpu, (total*1000/ticks)/seconds)\n\t}\n\n\ttime.AfterFunc(1*time.Second, periodic)\n}\n\n// ProcUsage returns CPU usage\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tcontents, err := os.ReadFile(procStatFile)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfields := bytes.Fields(contents)\n\n\t// Memory\n\t*rss = (parseInt64(fields[rssPos])) * pageSize\n\t*vss = parseInt64(fields[vssPos])\n\n\t// PCPU\n\t// We track this with periodic sampling, so just load and go.\n\t*pcpu = float64(atomic.LoadInt64(&ipcpu)) / 10.0\n\n\treturn nil\n}\n\n// Ascii numbers 0-9\nconst (\n\tasciiZero = 48\n\tasciiNine = 57\n)\n\n// parseInt64 expects decimal positive numbers. We\n// return -1 to signal error\nfunc parseInt64(d []byte) (n int64) {\n\tif len(d) == 0 {\n\t\treturn -1\n\t}\n\tfor _, dec := range d {\n\t\tif dec < asciiZero || dec > asciiNine {\n\t\t\treturn -1\n\t\t}\n\t\tn = n*10 + (int64(dec) - asciiZero)\n\t}\n\treturn n\n}\n" }, { "path": "server/pse/pse_netbsd.go", "content": "// Copyright 2022 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Copied from pse_openbsd.go\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n)\n\n// ProcUsage returns CPU usage\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"o\", \"pcpu=,rss=,vsz=\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\t*rss, *vss = -1, -1\n\t\treturn fmt.Errorf(\"ps call failed:%v\", err)\n\t}\n\tfmt.Sscanf(string(out), \"%f %d %d\", pcpu, rss, vss)\n\t*rss *= 1024 // 1k blocks, want bytes.\n\t*vss *= 1024 // 1k blocks, want bytes.\n\treturn nil\n}\n" }, { "path": "server/pse/pse_openbsd.go", "content": "// Copyright 2015-2018 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Copied from pse_darwin.go\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n)\n\n// ProcUsage returns CPU usage\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"o\", \"pcpu=,rss=,vsz=\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\t*rss, *vss = -1, -1\n\t\treturn fmt.Errorf(\"ps call failed:%v\", err)\n\t}\n\tfmt.Sscanf(string(out), \"%f %d %d\", pcpu, rss, vss)\n\t*rss *= 1024 // 1k blocks, want bytes.\n\t*vss *= 1024 // 1k blocks, want bytes.\n\treturn nil\n}\n" }, { "path": "server/pse/pse_rumprun.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build rumprun\n\npackage pse\n\n// This is a placeholder for now.\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\t*pcpu = 0.0\n\t*rss = 0\n\t*vss = 0\n\n\treturn nil\n}\n" }, { "path": "server/pse/pse_solaris.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// Copied from pse_openbsd.go\n\n//go:build illumos || solaris\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\n// ProcUsage returns CPU usage\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"-o\", \"pcpu,rss,vsz\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\t*rss, *vss = -1, -1\n\t\treturn fmt.Errorf(\"ps call failed:%v\", err)\n\t}\n\tlines := strings.Split(string(out), \"\\n\")\n\tif len(lines) < 2 {\n\t\t*rss, *vss = -1, -1\n\t\treturn fmt.Errorf(\"no ps output\")\n\t}\n\toutput := lines[1]\n\tfmt.Sscanf(output, \"%f %d %d\", pcpu, rss, vss)\n\t*rss *= 1024 // 1k blocks, want bytes.\n\t*vss *= 1024 // 1k blocks, want bytes.\n\treturn nil\n}\n" }, { "path": "server/pse/pse_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"testing\"\n)\n\nfunc TestPSEmulationCPU(t *testing.T) {\n\tif runtime.GOOS == \"windows\" {\n\t\tt.Skipf(\"Skipping this test on Windows\")\n\t}\n\tvar rss, vss int64\n\tvar pcpu, psPcpu float64\n\n\tdebug.FreeOSMemory()\n\n\t// PS version first\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"o\", \"pcpu=\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to execute ps command: %v\\n\", err)\n\t}\n\n\t// Our internal version\n\tProcUsage(&pcpu, &rss, &vss)\n\n\tfmt.Sscanf(string(out), \"%f\", &psPcpu)\n\n\tif pcpu != psPcpu {\n\t\tdelta := int64(pcpu - psPcpu)\n\t\tif delta < 0 {\n\t\t\tdelta = -delta\n\t\t}\n\t\tif delta > 30 { // 30%?\n\t\t\tt.Fatalf(\"CPUs did not match close enough: %f vs %f\", pcpu, psPcpu)\n\t\t}\n\t}\n}\n\nfunc TestPSEmulationMem(t *testing.T) {\n\tif runtime.GOOS == \"windows\" {\n\t\tt.Skipf(\"Skipping this test on Windows\")\n\t}\n\tvar rss, vss, psRss, psVss int64\n\tvar pcpu float64\n\n\tdebug.FreeOSMemory()\n\n\t// PS version first\n\tpidStr := fmt.Sprintf(\"%d\", os.Getpid())\n\tout, err := exec.Command(\"ps\", \"o\", \"rss=,vsz=\", \"-p\", pidStr).Output()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to execute ps command: %v\\n\", err)\n\t}\n\n\tfmt.Sscanf(string(out), \"%d %d\", &psRss, &psVss)\n\tpsRss *= 1024 // 1k blocks, want bytes.\n\tpsVss *= 1024 // 1k blocks, want bytes.\n\n\tdebug.FreeOSMemory()\n\n\t// Our internal version\n\tProcUsage(&pcpu, &rss, &vss)\n\n\tif rss != psRss {\n\t\tdelta := rss - psRss\n\t\tif delta < 0 {\n\t\t\tdelta = -delta\n\t\t}\n\t\tif delta > 1024*1024 { // 1MB\n\t\t\tt.Fatalf(\"RSSs did not match close enough: %d vs %d\", rss, psRss)\n\t\t}\n\t}\n}\n" }, { "path": "server/pse/pse_wasm.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build wasm\n\npackage pse\n\n// This is a placeholder for now.\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\t*pcpu = 0.0\n\t*rss = 0\n\t*vss = 0\n\n\treturn nil\n}\n" }, { "path": "server/pse/pse_windows.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\nvar (\n\tpdh = windows.NewLazySystemDLL(\"pdh.dll\")\n\twinPdhOpenQuery = pdh.NewProc(\"PdhOpenQuery\")\n\twinPdhAddCounter = pdh.NewProc(\"PdhAddCounterW\")\n\twinPdhCollectQueryData = pdh.NewProc(\"PdhCollectQueryData\")\n\twinPdhGetFormattedCounterValue = pdh.NewProc(\"PdhGetFormattedCounterValue\")\n\twinPdhGetFormattedCounterArray = pdh.NewProc(\"PdhGetFormattedCounterArrayW\")\n)\n\nfunc init() {\n\tif err := pdh.Load(); err != nil {\n\t\tpanic(err)\n\t}\n\tfor _, p := range []*windows.LazyProc{\n\t\twinPdhOpenQuery, winPdhAddCounter, winPdhCollectQueryData,\n\t\twinPdhGetFormattedCounterValue, winPdhGetFormattedCounterArray,\n\t} {\n\t\tif err := p.Find(); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n}\n\n// global performance counter query handle and counters\nvar (\n\tpcHandle PDH_HQUERY\n\tpidCounter, cpuCounter, rssCounter, vssCounter PDH_HCOUNTER\n\tprevCPU float64\n\tprevRss int64\n\tprevVss int64\n\tlastSampleTime time.Time\n\tprocessPid int\n\tpcQueryLock sync.Mutex\n\tinitialSample = true\n)\n\n// maxQuerySize is the number of values to return from a query.\n// It represents the maximum # of servers that can be queried\n// simultaneously running on a machine.\nconst maxQuerySize = 512\n\n// Keep static memory around to reuse; this works best for passing\n// into the pdh API.\nvar counterResults [maxQuerySize]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE\n\n// PDH Types\ntype (\n\tPDH_HQUERY syscall.Handle\n\tPDH_HCOUNTER syscall.Handle\n)\n\n// PDH constants used here\nconst (\n\tPDH_FMT_DOUBLE = 0x00000200\n\tPDH_INVALID_DATA = 0xC0000BC6\n\tPDH_MORE_DATA = 0x800007D2\n)\n\n// PDH_FMT_COUNTERVALUE_DOUBLE - double value\ntype PDH_FMT_COUNTERVALUE_DOUBLE struct {\n\tCStatus uint32\n\tDoubleValue float64\n}\n\n// PDH_FMT_COUNTERVALUE_ITEM_DOUBLE is an array\n// element of a double value\ntype PDH_FMT_COUNTERVALUE_ITEM_DOUBLE struct {\n\tSzName *uint16 // pointer to a string\n\tFmtValue PDH_FMT_COUNTERVALUE_DOUBLE\n}\n\nfunc pdhAddCounter(hQuery PDH_HQUERY, szFullCounterPath string, dwUserData uintptr, phCounter *PDH_HCOUNTER) error {\n\tptxt, _ := syscall.UTF16PtrFromString(szFullCounterPath)\n\tr0, _, _ := winPdhAddCounter.Call(\n\t\tuintptr(hQuery),\n\t\tuintptr(unsafe.Pointer(ptxt)),\n\t\tdwUserData,\n\t\tuintptr(unsafe.Pointer(phCounter)))\n\n\tif r0 != 0 {\n\t\treturn fmt.Errorf(\"pdhAddCounter failed. %d\", r0)\n\t}\n\treturn nil\n}\n\nfunc pdhOpenQuery(datasrc *uint16, userdata uint32, query *PDH_HQUERY) error {\n\tr0, _, _ := syscall.Syscall(winPdhOpenQuery.Addr(), 3, 0, uintptr(userdata), uintptr(unsafe.Pointer(query)))\n\tif r0 != 0 {\n\t\treturn fmt.Errorf(\"pdhOpenQuery failed - %d\", r0)\n\t}\n\treturn nil\n}\n\nfunc pdhCollectQueryData(hQuery PDH_HQUERY) error {\n\tr0, _, _ := winPdhCollectQueryData.Call(uintptr(hQuery))\n\tif r0 != 0 {\n\t\treturn fmt.Errorf(\"pdhCollectQueryData failed - %d\", r0)\n\t}\n\treturn nil\n}\n\n// pdhGetFormattedCounterArrayDouble returns the value of return code\n// rather than error, to easily check return codes\nfunc pdhGetFormattedCounterArrayDouble(hCounter PDH_HCOUNTER, lpdwBufferSize *uint32, lpdwBufferCount *uint32, itemBuffer *PDH_FMT_COUNTERVALUE_ITEM_DOUBLE) uint32 {\n\tret, _, _ := winPdhGetFormattedCounterArray.Call(\n\t\tuintptr(hCounter),\n\t\tuintptr(PDH_FMT_DOUBLE),\n\t\tuintptr(unsafe.Pointer(lpdwBufferSize)),\n\t\tuintptr(unsafe.Pointer(lpdwBufferCount)),\n\t\tuintptr(unsafe.Pointer(itemBuffer)))\n\n\treturn uint32(ret)\n}\n\nfunc getCounterArrayData(counter PDH_HCOUNTER) ([]float64, error) {\n\tvar bufSize uint32\n\tvar bufCount uint32\n\n\t// Retrieving array data requires two calls, the first which\n\t// requires an addressable empty buffer, and sets size fields.\n\t// The second call returns the data.\n\tinitialBuf := make([]PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, 1)\n\tret := pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &initialBuf[0])\n\tif ret == PDH_MORE_DATA {\n\t\t// we'll likely never get here, but be safe.\n\t\tif bufCount > maxQuerySize {\n\t\t\tbufCount = maxQuerySize\n\t\t}\n\t\tret = pdhGetFormattedCounterArrayDouble(counter, &bufSize, &bufCount, &counterResults[0])\n\t\tif ret == 0 {\n\t\t\trv := make([]float64, bufCount)\n\t\t\tfor i := 0; i < int(bufCount); i++ {\n\t\t\t\trv[i] = counterResults[i].FmtValue.DoubleValue\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\tif ret != 0 {\n\t\treturn nil, fmt.Errorf(\"getCounterArrayData failed - %d\", ret)\n\t}\n\n\treturn nil, nil\n}\n\n// getProcessImageName returns the name of the process image, as expected by\n// the performance counter API.\nfunc getProcessImageName() (name string) {\n\tname = filepath.Base(os.Args[0])\n\tname = strings.TrimSuffix(name, \".exe\")\n\treturn\n}\n\n// initialize our counters\nfunc initCounters() (err error) {\n\n\tprocessPid = os.Getpid()\n\t// require an addressible nil pointer\n\tvar source uint16\n\tif err := pdhOpenQuery(&source, 0, &pcHandle); err != nil {\n\t\treturn err\n\t}\n\n\t// setup the performance counters, search for all server instances\n\tname := fmt.Sprintf(\"%s*\", getProcessImageName())\n\tpidQuery := fmt.Sprintf(\"\\\\Process(%s)\\\\ID Process\", name)\n\tcpuQuery := fmt.Sprintf(\"\\\\Process(%s)\\\\%% Processor Time\", name)\n\trssQuery := fmt.Sprintf(\"\\\\Process(%s)\\\\Working Set - Private\", name)\n\tvssQuery := fmt.Sprintf(\"\\\\Process(%s)\\\\Virtual Bytes\", name)\n\n\tif err = pdhAddCounter(pcHandle, pidQuery, 0, &pidCounter); err != nil {\n\t\treturn err\n\t}\n\tif err = pdhAddCounter(pcHandle, cpuQuery, 0, &cpuCounter); err != nil {\n\t\treturn err\n\t}\n\tif err = pdhAddCounter(pcHandle, rssQuery, 0, &rssCounter); err != nil {\n\t\treturn err\n\t}\n\tif err = pdhAddCounter(pcHandle, vssQuery, 0, &vssCounter); err != nil {\n\t\treturn err\n\t}\n\n\t// prime the counters by collecting once, and sleep to get somewhat\n\t// useful information the first request. Counters for the CPU require\n\t// at least two collect calls.\n\tif err = pdhCollectQueryData(pcHandle); err != nil {\n\t\treturn err\n\t}\n\ttime.Sleep(50)\n\n\treturn nil\n}\n\n// ProcUsage returns process CPU and memory statistics\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\tvar err error\n\n\t// For simplicity, protect the entire call.\n\t// Most simultaneous requests will immediately return\n\t// with cached values.\n\tpcQueryLock.Lock()\n\tdefer pcQueryLock.Unlock()\n\n\t// First time through, initialize counters.\n\tif initialSample {\n\t\tif err = initCounters(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tinitialSample = false\n\t} else if time.Since(lastSampleTime) < (2 * time.Second) {\n\t\t// only refresh every two seconds as to minimize impact\n\t\t// on the server.\n\t\t*pcpu = prevCPU\n\t\t*rss = prevRss\n\t\t*vss = prevVss\n\t\treturn nil\n\t}\n\n\t// always save the sample time, even on errors.\n\tdefer func() {\n\t\tlastSampleTime = time.Now()\n\t}()\n\n\t// refresh the performance counter data\n\tif err = pdhCollectQueryData(pcHandle); err != nil {\n\t\treturn err\n\t}\n\n\t// retrieve the data\n\tvar pidAry, cpuAry, rssAry, vssAry []float64\n\tif pidAry, err = getCounterArrayData(pidCounter); err != nil {\n\t\treturn err\n\t}\n\tif cpuAry, err = getCounterArrayData(cpuCounter); err != nil {\n\t\treturn err\n\t}\n\tif rssAry, err = getCounterArrayData(rssCounter); err != nil {\n\t\treturn err\n\t}\n\tif vssAry, err = getCounterArrayData(vssCounter); err != nil {\n\t\treturn err\n\t}\n\t// find the index of the entry for this process\n\tidx := int(-1)\n\tfor i := range pidAry {\n\t\tif int(pidAry[i]) == processPid {\n\t\t\tidx = i\n\t\t\tbreak\n\t\t}\n\t}\n\t// no pid found...\n\tif idx < 0 {\n\t\treturn fmt.Errorf(\"could not find pid in performance counter results\")\n\t}\n\t// assign values from the performance counters\n\t*pcpu = cpuAry[idx]\n\t*rss = int64(rssAry[idx])\n\t*vss = int64(vssAry[idx])\n\n\t// save off cache values\n\tprevCPU = *pcpu\n\tprevRss = *rss\n\tprevVss = *vss\n\n\treturn nil\n}\n" }, { "path": "server/pse/pse_windows_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage pse\n\nimport (\n\t\"fmt\"\n\t\"os/exec\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc checkValues(t *testing.T, pcpu, tPcpu float64, rss, tRss int64) {\n\tif pcpu != tPcpu {\n\t\tdelta := int64(pcpu - tPcpu)\n\t\tif delta < 0 {\n\t\t\tdelta = -delta\n\t\t}\n\t\tif delta > 30 { // 30%?\n\t\t\tt.Fatalf(\"CPUs did not match close enough: %f vs %f\", pcpu, tPcpu)\n\t\t}\n\t}\n\tif rss != tRss {\n\t\tdelta := rss - tRss\n\t\tif delta < 0 {\n\t\t\tdelta = -delta\n\t\t}\n\t\tif delta > 1024*1024 { // 1MB\n\t\t\tt.Fatalf(\"RSSs did not match close enough: %d vs %d\", rss, tRss)\n\t\t}\n\t}\n}\n\nfunc TestPSEmulationWin(t *testing.T) {\n\tvar pcpu, tPcpu float64\n\tvar rss, vss, tRss int64\n\n\truntime.GC()\n\n\tif err := ProcUsage(&pcpu, &rss, &vss); err != nil {\n\t\tt.Fatalf(\"Error: %v\", err)\n\t}\n\n\truntime.GC()\n\n\timageName := getProcessImageName()\n\t// query the counters using typeperf\n\tout, err := exec.Command(\"typeperf.exe\",\n\t\tfmt.Sprintf(\"\\\\Process(%s)\\\\%% Processor Time\", imageName),\n\t\tfmt.Sprintf(\"\\\\Process(%s)\\\\Working Set - Private\", imageName),\n\t\tfmt.Sprintf(\"\\\\Process(%s)\\\\Virtual Bytes\", imageName),\n\t\t\"-sc\", \"1\").Output()\n\tif err != nil {\n\t\tt.Fatal(\"unable to run command\", err)\n\t}\n\n\t// parse out results - refer to comments in procUsage for detail\n\tresults := strings.Split(string(out), \"\\r\\n\")\n\tvalues := strings.Split(results[2], \",\")\n\n\t// parse pcpu\n\ttPcpu, err = strconv.ParseFloat(strings.Trim(values[1], \"\\\"\"), 64)\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to parse percent cpu: %s\", values[1])\n\t}\n\n\t// parse private bytes (rss)\n\tfval, err := strconv.ParseFloat(strings.Trim(values[2], \"\\\"\"), 64)\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to parse private bytes: %s\", values[2])\n\t}\n\ttRss = int64(fval)\n\n\tcheckValues(t, pcpu, tPcpu, rss, tRss)\n\n\truntime.GC()\n\n\t// Again to test caching\n\tif err = ProcUsage(&pcpu, &rss, &vss); err != nil {\n\t\tt.Fatalf(\"Error: %v\", err)\n\t}\n\tcheckValues(t, pcpu, tPcpu, rss, tRss)\n}\n" }, { "path": "server/pse/pse_zos.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build zos\n\npackage pse\n\n// This is a placeholder for now.\nfunc ProcUsage(pcpu *float64, rss, vss *int64) error {\n\t*pcpu = 0.0\n\t*rss = 0\n\t*vss = 0\n\n\treturn nil\n}\n" }, { "path": "server/raft.go", "content": "// Copyright 2020-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"iter\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/antithesishq/antithesis-sdk-go/assert\"\n\t\"github.com/nats-io/nats-server/v2/internal/fastrand\"\n\n\t\"github.com/minio/highwayhash\"\n)\n\ntype RaftNode interface {\n\tPropose(entry []byte) error\n\tProposeMulti(entries []*Entry) error\n\tForwardProposal(entry []byte) error\n\tInstallSnapshot(snap []byte, force bool) error\n\tCreateSnapshotCheckpoint(force bool) (RaftNodeCheckpoint, error)\n\tSendSnapshot(snap []byte) error\n\tNeedSnapshot() bool\n\tApplied(index uint64) (entries uint64, bytes uint64)\n\tProcessed(index uint64, applied uint64) (entries uint64, bytes uint64)\n\tState() RaftState\n\tSize() (entries, bytes uint64)\n\tProgress() (index, commit, applied uint64)\n\tLeader() bool\n\tLeaderSince() *time.Time\n\tQuorum() bool\n\tCurrent() bool\n\tHealthy() bool\n\tTerm() uint64\n\tLeaderless() bool\n\tGroupLeader() string\n\tHadPreviousLeader() bool\n\tStepDown(preferred ...string) error\n\tSetObserver(isObserver bool)\n\tIsObserver() bool\n\tCampaign() error\n\tCampaignImmediately() error\n\tID() string\n\tGroup() string\n\tPeers() []*Peer\n\tProposeKnownPeers(knownPeers []string)\n\tUpdateKnownPeers(knownPeers []string)\n\tProposeAddPeer(peer string) error\n\tProposeRemovePeer(peer string) error\n\tMembershipChangeInProgress() bool\n\tAdjustClusterSize(csz int) error\n\tAdjustBootClusterSize(csz int) error\n\tClusterSize() int\n\tApplyQ() *ipQueue[*CommittedEntry]\n\tPauseApply() error\n\tResumeApply()\n\tDrainAndReplaySnapshot() bool\n\tLeadChangeC() <-chan bool\n\tQuitC() <-chan struct{}\n\tCreated() time.Time\n\tStop()\n\tWaitForStop()\n\tDelete()\n\tIsDeleted() bool\n\tRecreateInternalSubs() error\n\tIsSystemAccount() bool\n\tGetTrafficAccountName() string\n\tGetWriteErr() error\n}\n\n// RaftNodeCheckpoint is used as an alternative to a direct InstallSnapshot.\n// A checkpoint is created from CreateSnapshotCheckpoint and allows installing snapshots asynchronously,\n// as well as loading the last snapshot or entries between the last snapshot and the one we're about to create.\n// Abort can be called to cancel the snapshot installation at any time, or InstallSnapshot to install it.\ntype RaftNodeCheckpoint interface {\n\tLoadLastSnapshot() (snap []byte, err error)\n\tAppendEntriesSeq() iter.Seq2[*appendEntry, error]\n\tAbort()\n\tInstallSnapshot(data []byte) (uint64, error)\n}\n\ntype WAL interface {\n\tType() StorageType\n\tStoreMsg(subj string, hdr, msg []byte, ttl int64) (uint64, int64, error)\n\tLoadMsg(index uint64, sm *StoreMsg) (*StoreMsg, error)\n\tRemoveMsg(index uint64) (bool, error)\n\tCompact(index uint64) (uint64, error)\n\tPurge() (uint64, error)\n\tPurgeEx(subject string, seq, keep uint64) (uint64, error)\n\tTruncate(seq uint64) error\n\tState() StreamState\n\tFastState(*StreamState)\n\tStop() error\n\tDelete(inline bool) error\n}\n\ntype Peer struct {\n\tID string\n\tCurrent bool\n\tLast time.Time\n\tLag uint64\n}\n\ntype RaftState uint8\n\n// Allowable states for a NATS Consensus Group.\nconst (\n\tFollower RaftState = iota\n\tLeader\n\tCandidate\n\tClosed\n)\n\nfunc (state RaftState) String() string {\n\tswitch state {\n\tcase Follower:\n\t\treturn \"FOLLOWER\"\n\tcase Candidate:\n\t\treturn \"CANDIDATE\"\n\tcase Leader:\n\t\treturn \"LEADER\"\n\tcase Closed:\n\t\treturn \"CLOSED\"\n\t}\n\treturn \"UNKNOWN\"\n}\n\ntype raft struct {\n\tsync.RWMutex\n\n\tcreated time.Time // Time that the group was created\n\taccName string // Account name of the asset this raft group is for\n\tacc *Account // Account that NRG traffic will be sent/received in\n\tgroup string // Raft group\n\tsd string // Store directory\n\tid string // Node ID\n\twg sync.WaitGroup // Wait for running goroutines to exit on shutdown\n\n\twal WAL // WAL store (filestore or memstore)\n\twtype StorageType // WAL type, e.g. FileStorage or MemoryStorage\n\tbytes uint64 // Total amount of bytes stored in the WAL. (Saves us from needing to call wal.FastState very often)\n\twerr error // Last write error\n\n\tstate atomic.Int32 // RaftState\n\tleaderState atomic.Bool // Is in (complete) leader state.\n\tleaderSince atomic.Pointer[time.Time] // How long since becoming leader.\n\thh *highwayhash.Digest64 // Highwayhash, used for snapshots\n\tsnapfile string // Snapshot filename\n\n\tcsz int // Cluster size\n\tqn int // Number of nodes needed to establish quorum\n\tpeers map[string]*lps // Other peers in the Raft group\n\n\tremoved map[string]time.Time // Peers that were removed from the group\n\tacks map[uint64]map[string]struct{} // Append entry responses/acks, map of entry index -> peer ID\n\tpae map[uint64]*appendEntry // Pending append entries\n\n\telect *time.Timer // Election timer, normally accessed via electTimer\n\tetlr time.Time // Election timer last reset time, for unit tests only\n\tactive time.Time // Last activity time, i.e. for heartbeats\n\tllqrt time.Time // Last quorum lost time\n\tlsut time.Time // Last scale-up time\n\n\tterm uint64 // The current vote term\n\tpterm uint64 // Previous term from the last snapshot\n\tpindex uint64 // Previous index from the last snapshot\n\tcommit uint64 // Index of the most recent commit\n\tprocessed uint64 // Index of the most recently processed commit\n\tapplied uint64 // Index of the most recently applied commit\n\tpapplied uint64 // First sequence of our log, matches when we last installed a snapshot.\n\n\tmembChangeIndex uint64 // Index of uncommitted membership change entry (0 means no change in progress)\n\n\taflr uint64 // Index when to signal initial messages have been applied after becoming leader. 0 means signaling is disabled.\n\n\tleader string // The ID of the leader\n\tvote string // Our current vote state\n\n\ts *Server // Reference to top-level server\n\tc *client // Internal client for subscriptions\n\tjs *jetStream // JetStream, if running, to see if we are out of resources\n\n\thasleader atomic.Bool // Is there a group leader right now?\n\tpleader atomic.Bool // Has the group ever had a leader?\n\tisSysAcc atomic.Bool // Are we utilizing the system account?\n\n\textSt extensionState // Extension state\n\n\ttrack bool // Whether out of resources checking is enabled.\n\tdflag bool // Debug flag\n\n\tpsubj string // Proposals subject\n\trpsubj string // Remove peers subject\n\tvsubj string // Vote requests subject\n\tvreply string // Vote responses subject\n\tasubj string // Append entries subject\n\tareply string // Append entries responses subject\n\n\tsq *sendq // Send queue for outbound RPC messages\n\taesub *subscription // Subscription for handleAppendEntry callbacks\n\n\twtv []byte // Term and vote to be written\n\twps []byte // Peer state to be written\n\n\tcatchup *catchupState // For when we need to catch up as a follower.\n\tprogress map[string]*ipQueue[uint64] // For leader or server catching up a follower.\n\n\thcommit uint64 // The commit at the time that applies were paused\n\n\tprop *ipQueue[*proposedEntry] // Proposals\n\tentry *ipQueue[*appendEntry] // Append entries\n\tresp *ipQueue[*appendEntryResponse] // Append entries responses\n\tapply *ipQueue[*CommittedEntry] // Apply queue (committed entries to be passed to upper layer)\n\treqs *ipQueue[*voteRequest] // Vote requests\n\tvotes *ipQueue[*voteResponse] // Vote responses\n\tleadc chan bool // Leader changes\n\tquit chan struct{} // Raft group shutdown\n\n\tlxfer bool // Are we doing a leadership transfer?\n\thcbehind bool // Were we falling behind at the last health check? (see: isCurrent)\n\tmaybeLeader bool // The group had a preferred leader. And is maybe already acting as leader prior to scale up.\n\tpaused bool // Whether or not applies are paused\n\tobserver bool // The node is observing, i.e. not able to become leader\n\tinitializing bool // The node is new, and \"empty log\" checks can be temporarily relaxed.\n\tscaleUp bool // The node is part of a scale up, puts us in observer mode until the log contains data.\n\tdeleted bool // If the node was deleted.\n\tsnapshotting bool // Snapshot is in progress.\n}\n\ntype proposedEntry struct {\n\t*Entry\n\treply string // Optional, to respond once proposal handled\n}\n\n// catchupState structure that holds our subscription, and catchup term and index\n// as well as starting term and index and how many updates we have seen.\ntype catchupState struct {\n\tsub *subscription // Subscription that catchup messages will arrive on\n\tcterm uint64 // Catchup term\n\tcindex uint64 // Catchup index\n\tpterm uint64 // Starting term\n\tpindex uint64 // Starting index\n\tactive time.Time // Last time we received a message for this catchup\n\tsignal bool // Whether the EntryCatchup signal was sent.\n}\n\n// lps holds peer state of last time and last index replicated.\ntype lps struct {\n\tts time.Time // Last timestamp\n\tli uint64 // Last index replicated\n\tkp bool // Known peer\n}\n\nconst (\n\tminElectionTimeoutDefault = 4 * time.Second\n\tmaxElectionTimeoutDefault = 9 * time.Second\n\tminCampaignTimeoutDefault = 100 * time.Millisecond\n\tmaxCampaignTimeoutDefault = 8 * minCampaignTimeoutDefault\n\thbIntervalDefault = 1 * time.Second\n\tlostQuorumIntervalDefault = hbIntervalDefault * 10 // 10 seconds\n\tlostQuorumCheckIntervalDefault = hbIntervalDefault * 10 // 10 seconds\n\tobserverModeIntervalDefault = 48 * time.Hour\n\tpeerRemoveTimeoutDefault = 5 * time.Minute\n)\n\nvar (\n\tminElectionTimeout = minElectionTimeoutDefault\n\tmaxElectionTimeout = maxElectionTimeoutDefault\n\tminCampaignTimeout = minCampaignTimeoutDefault\n\tmaxCampaignTimeout = maxCampaignTimeoutDefault\n\thbInterval = hbIntervalDefault\n\tlostQuorumInterval = lostQuorumIntervalDefault\n\tlostQuorumCheck = lostQuorumCheckIntervalDefault\n\tobserverModeInterval = observerModeIntervalDefault\n\tpeerRemoveTimeout = peerRemoveTimeoutDefault\n)\n\ntype RaftConfig struct {\n\tName string\n\tStore string\n\tLog WAL\n\tTrack bool\n\tObserver bool\n\n\t// Recovering must be set for a Raft group that's recovering after a restart, or if it's\n\t// first seen after a catchup from another server. If a server recovers with an empty log,\n\t// we know to protect against data loss.\n\tRecovering bool\n\n\t// ScaleUp identifies the Raft peer set is being scaled up.\n\t// We need to protect against losing state due to the new peers starting with an empty log.\n\t// Therefore, these empty servers can't try to become leader until they at least have _some_ state.\n\tScaleUp bool\n}\n\nvar (\n\terrNotLeader = errors.New(\"raft: not leader\")\n\terrAlreadyLeader = errors.New(\"raft: already leader\")\n\terrNilCfg = errors.New(\"raft: no config given\")\n\terrCorruptPeers = errors.New(\"raft: corrupt peer state\")\n\terrEntryLoadFailed = errors.New(\"raft: could not load entry from WAL\")\n\terrEntryStoreFailed = errors.New(\"raft: could not store entry to WAL\")\n\terrNodeClosed = errors.New(\"raft: node is closed\")\n\terrNodeRemoved = errors.New(\"raft: peer was removed\")\n\terrBadSnapName = errors.New(\"raft: snapshot name could not be parsed\")\n\terrNoSnapAvailable = errors.New(\"raft: no snapshot available\")\n\terrSnapInProgress = errors.New(\"raft: snapshot is already in progress\")\n\terrSnapAborted = errors.New(\"raft: snapshot was aborted\")\n\terrCatchupsRunning = errors.New(\"raft: snapshot can not be installed while catchups running\")\n\terrSnapshotCorrupt = errors.New(\"raft: snapshot corrupt\")\n\terrTooManyPrefs = errors.New(\"raft: stepdown requires at most one preferred new leader\")\n\terrNoPeerState = errors.New(\"raft: no peerstate\")\n\terrAdjustBootCluster = errors.New(\"raft: can not adjust boot peer size on established group\")\n\terrLeaderLen = fmt.Errorf(\"raft: leader should be exactly %d bytes\", idLen)\n\terrTooManyEntries = errors.New(\"raft: append entry can contain a max of 64k entries\")\n\terrBadAppendEntry = errors.New(\"raft: append entry corrupt\")\n\terrNoInternalClient = errors.New(\"raft: no internal client\")\n\terrMembershipChange = errors.New(\"raft: membership change in progress\")\n\terrRemoveLastNode = errors.New(\"raft: cannot remove the last peer\")\n)\n\n// This will bootstrap a raftNode by writing its config into the store directory.\nfunc (s *Server) bootstrapRaftNode(cfg *RaftConfig, knownPeers []string, allPeersKnown bool) error {\n\tif cfg == nil {\n\t\treturn errNilCfg\n\t}\n\t// Check validity of peers if presented.\n\tfor _, p := range knownPeers {\n\t\tif len(p) != idLen {\n\t\t\treturn fmt.Errorf(\"raft: illegal peer: %q\", p)\n\t\t}\n\t}\n\texpected := len(knownPeers)\n\t// We need to adjust this is all peers are not known.\n\tif !allPeersKnown {\n\t\ts.Debugf(\"Determining expected peer size for JetStream meta group\")\n\t\tif expected < 2 {\n\t\t\texpected = 2\n\t\t}\n\t\topts := s.getOpts()\n\t\tnrs := len(opts.Routes)\n\n\t\tcn := s.ClusterName()\n\t\tngwps := 0\n\t\tfor _, gw := range opts.Gateway.Gateways {\n\t\t\t// Ignore our own cluster if specified.\n\t\t\tif gw.Name == cn {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, u := range gw.URLs {\n\t\t\t\thost := u.Hostname()\n\t\t\t\t// If this is an IP just add one.\n\t\t\t\tif net.ParseIP(host) != nil {\n\t\t\t\t\tngwps++\n\t\t\t\t} else {\n\t\t\t\t\taddrs, _ := net.LookupHost(host)\n\t\t\t\t\tngwps += len(addrs)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif expected < nrs+ngwps {\n\t\t\texpected = nrs + ngwps\n\t\t\ts.Debugf(\"Adjusting expected peer set size to %d with %d known\", expected, len(knownPeers))\n\t\t}\n\t}\n\n\t// Check the store directory. If we have a memory based WAL we need to make sure the directory is setup.\n\tif stat, err := os.Stat(cfg.Store); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(cfg.Store, defaultDirPerms); err != nil {\n\t\t\treturn fmt.Errorf(\"raft: could not create storage directory - %v\", err)\n\t\t}\n\t} else if stat == nil || !stat.IsDir() {\n\t\treturn fmt.Errorf(\"raft: storage directory is not a directory\")\n\t}\n\ttmpfile, err := os.CreateTemp(cfg.Store, \"_test_\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"raft: storage directory is not writable\")\n\t}\n\ttmpfile.Close()\n\tos.Remove(tmpfile.Name())\n\n\treturn writePeerState(cfg.Store, &peerState{knownPeers, expected, extUndetermined})\n}\n\n// initRaftNode will initialize the raft node, to be used by startRaftNode or when testing to not run the Go routine.\nfunc (s *Server) initRaftNode(accName string, cfg *RaftConfig, labels pprofLabels) (*raft, error) {\n\trestorePeerState := func(n *raft) error {\n\t\tps, err := readPeerState(cfg.Store)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif ps == nil {\n\t\t\treturn errNoPeerState\n\t\t}\n\t\tn.processPeerState(ps)\n\t\tn.extSt = ps.domainExt\n\t\treturn nil\n\t}\n\n\tif cfg == nil {\n\t\treturn nil, errNilCfg\n\t}\n\ts.mu.RLock()\n\tif s.sys == nil {\n\t\ts.mu.RUnlock()\n\t\treturn nil, ErrNoSysAccount\n\t}\n\thash := s.sys.shash\n\ts.mu.RUnlock()\n\n\tqpfx := fmt.Sprintf(\"[ACC:%s] RAFT '%s' \", accName, cfg.Name)\n\tn := &raft{\n\t\tcreated: time.Now(),\n\t\tid: hash[:idLen],\n\t\tgroup: cfg.Name,\n\t\tsd: cfg.Store,\n\t\twal: cfg.Log,\n\t\twtype: cfg.Log.Type(),\n\t\ttrack: cfg.Track,\n\t\tpeers: make(map[string]*lps),\n\t\tacks: make(map[uint64]map[string]struct{}),\n\t\tpae: make(map[uint64]*appendEntry),\n\t\ts: s,\n\t\tjs: s.getJetStream(),\n\t\tquit: make(chan struct{}),\n\t\treqs: newIPQueue[*voteRequest](s, qpfx+\"vreq\"),\n\t\tvotes: newIPQueue[*voteResponse](s, qpfx+\"vresp\"),\n\t\tprop: newIPQueue[*proposedEntry](s, qpfx+\"entry\"),\n\t\tentry: newIPQueue[*appendEntry](s, qpfx+\"appendEntry\"),\n\t\tresp: newIPQueue[*appendEntryResponse](s, qpfx+\"appendEntryResponse\"),\n\t\tapply: newIPQueue[*CommittedEntry](s, qpfx+\"committedEntry\"),\n\t\taccName: accName,\n\t\tleadc: make(chan bool, 32),\n\t\tobserver: cfg.Observer,\n\t}\n\n\t// Setup our internal subscriptions for proposals, votes and append entries.\n\t// If we fail to do this for some reason then this is fatal — we cannot\n\t// continue setting up or the Raft node may be partially/totally isolated.\n\tif err := n.RecreateInternalSubs(); err != nil {\n\t\tn.shutdown()\n\t\treturn nil, err\n\t}\n\n\tif atomic.LoadInt32(&s.logging.debug) > 0 {\n\t\tn.dflag = true\n\t}\n\n\t// Set up the highwayhash for the snapshots.\n\tkey := sha256.Sum256([]byte(n.group))\n\tn.hh, _ = highwayhash.NewDigest64(key[:])\n\n\t// If we have a term and vote file (tav.idx on the filesystem) then read in\n\t// what we think the term and vote was. It's possible these are out of date\n\t// so a catch-up may be required.\n\tif term, vote, err := n.readTermVote(); err == nil && term > 0 {\n\t\tn.term = term\n\t\tn.vote = vote\n\t}\n\n\t// Can't recover snapshots if memory based since wal will be reset.\n\t// We will inherit from the current leader.\n\tn.papplied = 0\n\tif _, ok := n.wal.(*memStore); ok {\n\t\t_ = os.RemoveAll(filepath.Join(n.sd, snapshotsDir))\n\t} else if err := n.setupLastSnapshot(); err != nil && err != errNoSnapAvailable {\n\t\t// If we failed to recover from the snapshot, then we should surface\n\t\t// the error upwards, otherwise we can complete recovery but have only\n\t\t// a partial view of the world.\n\t\tn.shutdown()\n\t\treturn nil, err\n\t}\n\n\t// We may have restored the peer state from the\n\t// snapshot above. If not, we restore peers from\n\t// the peer state file.\n\tif len(n.peers) == 0 {\n\t\tif err := restorePeerState(n); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Make sure that the snapshots directory exists.\n\tif err := os.MkdirAll(filepath.Join(n.sd, snapshotsDir), defaultDirPerms); err != nil {\n\t\tn.shutdown()\n\t\treturn nil, fmt.Errorf(\"could not create snapshots directory - %v\", err)\n\t}\n\n\ttruncateAndErr := func(index uint64) {\n\t\tif err := n.wal.Truncate(index); err != nil {\n\t\t\tn.setWriteErr(err)\n\t\t}\n\t}\n\n\t// Retrieve the stream state from the WAL. If there are pending append\n\t// entries that were committed but not applied before we last shut down,\n\t// we will try to replay them and process them here.\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\tn.bytes = state.Bytes\n\n\tif state.Msgs > 0 {\n\t\tn.debug(\"Replaying state of %d entries\", state.Msgs)\n\n\t\t// This process will queue up entries on our applied queue but prior to the upper\n\t\t// state machine running. So we will monitor how much we have queued and if we\n\t\t// reach a limit will pause the apply queue and resume inside of run() go routine.\n\t\tconst maxQsz = 32 * 1024 * 1024 // 32MB max\n\n\t\t// It looks like there are entries we have committed but not applied\n\t\t// yet. Replay them.\n\t\tfor index, qsz := state.FirstSeq, 0; index <= state.LastSeq; index++ {\n\t\t\tae, err := n.loadEntry(index)\n\t\t\t// The first entry in our WAL initializes state but must align with our snapshot if we had one.\n\t\t\t// Importantly, check this first, as we might need to truncate the WAL further than the index.\n\t\t\tif index == state.FirstSeq {\n\t\t\t\t// If the entry is missing, corrupt, or doesn't align with the snapshot, truncate the WAL.\n\t\t\t\tif err != nil || ae == nil || ae.pindex != index-1 || n.pindex != ae.pindex {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tn.warn(\"Could not load %d from WAL [%+v]: %v\", index, state, err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tn.warn(\"Misaligned WAL, will truncate\")\n\t\t\t\t\t}\n\t\t\t\t\t// Truncate to the snapshot or beginning if there is none.\n\t\t\t\t\ttruncateAndErr(n.pindex)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tn.pterm, n.pindex = ae.pterm, ae.pindex\n\t\t\t\tif ae.commit > 0 && ae.commit > n.commit {\n\t\t\t\t\tn.commit = ae.commit\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tn.warn(\"Could not load %d from WAL [%+v]: %v\", index, state, err)\n\t\t\t\t// Truncate to the previous correct entry.\n\t\t\t\ttruncateAndErr(index - 1)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif ae.pindex != index-1 {\n\t\t\t\tn.warn(\"Corrupt WAL, will truncate\")\n\t\t\t\t// Truncate to the previous correct entry.\n\t\t\t\ttruncateAndErr(index - 1)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tn.processAppendEntry(ae, nil)\n\t\t\t// Check how much we have queued up so far to determine if we should pause.\n\t\t\tfor _, e := range ae.entries {\n\t\t\t\tqsz += len(e.Data)\n\t\t\t\tif qsz > maxQsz && !n.paused {\n\t\t\t\t\tn.PauseApply()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tn.debug(\"Started (cluster size %d, quorum %d)\", n.csz, n.qn)\n\n\t// Check if we need to start in observer mode due to lame duck status.\n\t// This will stop us from taking on the leader role when we're about to\n\t// shutdown anyway.\n\tif s.isLameDuckMode() {\n\t\tn.debug(\"Will start in observer mode due to lame duck status\")\n\t\tn.SetObserver(true)\n\t}\n\n\t// Set the election timer and lost quorum timers to now, so that we\n\t// won't accidentally trigger either state without knowing the real state\n\t// of the other nodes.\n\tn.Lock()\n\tn.resetElectionTimeout()\n\tn.llqrt = time.Now()\n\n\t// If our log is empty, and we're initializing, relax the \"empty log\" checks temporarily.\n\tif !cfg.Recovering && n.pindex == 0 {\n\t\tn.initializing = true\n\t\t// If we're scaling up and our log is empty, must put ourselves into observer\n\t\t// and wait for data from the leader.\n\t\tif !cfg.Observer && cfg.ScaleUp {\n\t\t\tn.scaleUp = true\n\t\t\tn.setObserverLocked(true, extUndetermined)\n\t\t}\n\t}\n\tn.Unlock()\n\n\t// Register the Raft group.\n\tlabels[\"group\"] = n.group\n\ts.registerRaftNode(n.group, n)\n\n\treturn n, nil\n}\n\n// startRaftNode will start the raft node.\nfunc (s *Server) startRaftNode(accName string, cfg *RaftConfig, labels pprofLabels) (RaftNode, error) {\n\tn, err := s.initRaftNode(accName, cfg, labels)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Start the run goroutine for the Raft state machine.\n\tn.wg.Add(1)\n\ts.startGoRoutine(n.run, labels)\n\n\treturn n, nil\n}\n\n// Returns whether peers within this group claim to support\n// moving NRG traffic into the asset account.\n// Lock must be held.\nfunc (n *raft) checkAccountNRGStatus() bool {\n\tif !n.s.accountNRGAllowed.Load() {\n\t\treturn false\n\t}\n\tenabled := true\n\tfor pn := range n.peers {\n\t\tif si, ok := n.s.nodeToInfo.Load(pn); ok && si != nil {\n\t\t\tenabled = enabled && si.(nodeInfo).accountNRG\n\t\t}\n\t}\n\treturn enabled\n}\n\n// Whether we are using the system account or not.\nfunc (n *raft) IsSystemAccount() bool {\n\treturn n.isSysAcc.Load()\n}\n\n// GetTrafficAccountName returns the account name of the account used for replication traffic.\nfunc (n *raft) GetTrafficAccountName() string {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.acc.GetName()\n}\n\nfunc (n *raft) RecreateInternalSubs() error {\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.recreateInternalSubsLocked()\n}\n\nfunc (n *raft) recreateInternalSubsLocked() error {\n\t// Sanity check for system account, as it can disappear when\n\t// the system is shutting down.\n\tif n.s == nil {\n\t\treturn fmt.Errorf(\"server not found\")\n\t}\n\tn.s.mu.RLock()\n\tsys := n.s.sys\n\tn.s.mu.RUnlock()\n\tif sys == nil {\n\t\treturn fmt.Errorf(\"system account not found\")\n\t}\n\n\t// Default is the system account.\n\tnrgAcc := sys.account\n\tn.isSysAcc.Store(true)\n\n\t// Is account NRG enabled in this account and do all group\n\t// peers claim to also support account NRG?\n\tif n.checkAccountNRGStatus() {\n\t\t// Check whether the account that the asset belongs to\n\t\t// has volunteered a different NRG account.\n\t\ttarget := nrgAcc.Name\n\t\tif a, _ := n.s.lookupAccount(n.accName); a != nil {\n\t\t\ta.mu.RLock()\n\t\t\tif a.js != nil {\n\t\t\t\ttarget = a.nrgAccount\n\t\t\t}\n\t\t\ta.mu.RUnlock()\n\t\t}\n\n\t\t// If the target account exists, then we'll use that.\n\t\tif target != _EMPTY_ {\n\t\t\tif a, _ := n.s.lookupAccount(target); a != nil {\n\t\t\t\tnrgAcc = a\n\t\t\t\tif a != sys.account {\n\t\t\t\t\tn.isSysAcc.Store(false)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif n.aesub != nil && n.acc == nrgAcc {\n\t\t// Subscriptions already exist and the account NRG state\n\t\t// hasn't changed.\n\t\treturn nil\n\t}\n\n\t// Need to cancel any in-progress catch-ups, otherwise the\n\t// inboxes are about to be pulled out from underneath it in\n\t// the next step...\n\tn.cancelCatchup()\n\n\t// If we have an existing client then tear down any existing\n\t// subscriptions and close the internal client.\n\tif c := n.c; c != nil {\n\t\tc.mu.Lock()\n\t\tsubs := make([]*subscription, 0, len(c.subs))\n\t\tfor _, sub := range c.subs {\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t\tc.mu.Unlock()\n\t\tfor _, sub := range subs {\n\t\t\tn.unsubscribe(sub)\n\t\t}\n\t\tc.closeConnection(InternalClient)\n\t}\n\n\tif n.acc != nrgAcc {\n\t\tn.debug(\"Subscribing in '%s'\", nrgAcc.GetName())\n\t}\n\n\tc := n.s.createInternalSystemClient()\n\tc.registerWithAccount(nrgAcc)\n\tif nrgAcc.sq == nil {\n\t\tnrgAcc.sq = n.s.newSendQ(nrgAcc)\n\t}\n\tn.c = c\n\tn.sq = nrgAcc.sq\n\tn.acc = nrgAcc\n\n\t// Recreate any internal subscriptions for voting, append\n\t// entries etc in the new account.\n\treturn n.createInternalSubs()\n}\n\n// outOfResources checks to see if we are out of resources.\nfunc (n *raft) outOfResources() bool {\n\tjs := n.js\n\tif !n.track || js == nil {\n\t\treturn false\n\t}\n\treturn js.limitsExceeded(n.wtype)\n}\n\n// Maps node names back to server names.\nfunc (s *Server) serverNameForNode(node string) string {\n\tif si, ok := s.nodeToInfo.Load(node); ok && si != nil {\n\t\treturn si.(nodeInfo).name\n\t}\n\treturn _EMPTY_\n}\n\n// Maps node names back to cluster names.\nfunc (s *Server) clusterNameForNode(node string) string {\n\tif si, ok := s.nodeToInfo.Load(node); ok && si != nil {\n\t\treturn si.(nodeInfo).cluster\n\t}\n\treturn _EMPTY_\n}\n\n// Registers the Raft node with the server, as it will track all of the Raft\n// nodes.\nfunc (s *Server) registerRaftNode(group string, n RaftNode) {\n\ts.rnMu.Lock()\n\tdefer s.rnMu.Unlock()\n\tif s.raftNodes == nil {\n\t\ts.raftNodes = make(map[string]RaftNode)\n\t}\n\ts.raftNodes[group] = n\n}\n\n// Unregisters the Raft node from the server, i.e. at shutdown.\nfunc (s *Server) unregisterRaftNode(group string) {\n\ts.rnMu.Lock()\n\tdefer s.rnMu.Unlock()\n\tif s.raftNodes != nil {\n\t\tdelete(s.raftNodes, group)\n\t}\n}\n\n// Returns how many Raft nodes are running in this server instance.\nfunc (s *Server) numRaftNodes() int {\n\ts.rnMu.RLock()\n\tdefer s.rnMu.RUnlock()\n\treturn len(s.raftNodes)\n}\n\n// Finds the Raft node for a given Raft group, if any. If there is no Raft node\n// running for this group then it can return nil.\nfunc (s *Server) lookupRaftNode(group string) RaftNode {\n\ts.rnMu.RLock()\n\tdefer s.rnMu.RUnlock()\n\tvar n RaftNode\n\tif s.raftNodes != nil {\n\t\tn = s.raftNodes[group]\n\t}\n\treturn n\n}\n\n// Reloads the debug state for all running Raft nodes. This is necessary when\n// the configuration has been reloaded and the debug log level has changed.\nfunc (s *Server) reloadDebugRaftNodes(debug bool) {\n\tif s == nil {\n\t\treturn\n\t}\n\ts.rnMu.RLock()\n\tfor _, ni := range s.raftNodes {\n\t\tn := ni.(*raft)\n\t\tn.Lock()\n\t\tn.dflag = debug\n\t\tn.Unlock()\n\t}\n\ts.rnMu.RUnlock()\n}\n\n// Requests that all Raft nodes on this server step down and place them into\n// observer mode. This is called when the server is shutting down.\nfunc (s *Server) stepdownRaftNodes() {\n\tif s == nil {\n\t\treturn\n\t}\n\ts.rnMu.RLock()\n\tif len(s.raftNodes) == 0 {\n\t\ts.rnMu.RUnlock()\n\t\treturn\n\t}\n\ts.Debugf(\"Stepping down all leader raft nodes\")\n\tnodes := make([]RaftNode, 0, len(s.raftNodes))\n\tfor _, n := range s.raftNodes {\n\t\tnodes = append(nodes, n)\n\t}\n\ts.rnMu.RUnlock()\n\n\tfor _, node := range nodes {\n\t\tnode.StepDown()\n\t\tnode.SetObserver(true)\n\t}\n}\n\n// Shuts down all Raft nodes on this server. This is called either when the\n// server is either entering lame duck mode, shutting down or when JetStream\n// has been disabled.\nfunc (s *Server) shutdownRaftNodes() {\n\tif s == nil {\n\t\treturn\n\t}\n\ts.rnMu.RLock()\n\tif len(s.raftNodes) == 0 {\n\t\ts.rnMu.RUnlock()\n\t\treturn\n\t}\n\tnodes := make([]RaftNode, 0, len(s.raftNodes))\n\ts.Debugf(\"Shutting down all raft nodes\")\n\tfor _, n := range s.raftNodes {\n\t\tnodes = append(nodes, n)\n\t}\n\ts.rnMu.RUnlock()\n\n\tfor _, node := range nodes {\n\t\tnode.Stop()\n\t}\n}\n\n// Used in lameduck mode to move off the leaders.\n// We also put all nodes in observer mode so new leaders\n// can not be placed on this server.\nfunc (s *Server) transferRaftLeaders() bool {\n\tif s == nil {\n\t\treturn false\n\t}\n\ts.rnMu.RLock()\n\tif len(s.raftNodes) == 0 {\n\t\ts.rnMu.RUnlock()\n\t\treturn false\n\t}\n\tnodes := make([]RaftNode, 0, len(s.raftNodes))\n\tfor _, n := range s.raftNodes {\n\t\tnodes = append(nodes, n)\n\t}\n\ts.rnMu.RUnlock()\n\n\tvar didTransfer bool\n\tfor _, node := range nodes {\n\t\tif err := node.StepDown(); err == nil {\n\t\t\tdidTransfer = true\n\t\t}\n\t\tnode.SetObserver(true)\n\t}\n\treturn didTransfer\n}\n\n// Formal API\n\n// Propose will propose a new entry to the group.\n// This should only be called on the leader.\nfunc (n *raft) Propose(data []byte) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\t// Check state under lock, we might not be leader anymore.\n\tif state := n.State(); state != Leader {\n\t\tn.debug(\"Proposal ignored, not leader (state: %v)\", state)\n\t\treturn errNotLeader\n\t}\n\n\t// Error if we had a previous write error.\n\tif werr := n.werr; werr != nil {\n\t\treturn werr\n\t}\n\tn.prop.push(newProposedEntry(newEntry(EntryNormal, data), _EMPTY_))\n\treturn nil\n}\n\n// ProposeMulti will propose multiple entries at once.\n// This should only be called on the leader.\nfunc (n *raft) ProposeMulti(entries []*Entry) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\t// Check state under lock, we might not be leader anymore.\n\tif state := n.State(); state != Leader {\n\t\tn.debug(\"Multi proposal ignored, not leader (state: %v)\", state)\n\t\treturn errNotLeader\n\t}\n\n\t// Error if we had a previous write error.\n\tif werr := n.werr; werr != nil {\n\t\treturn werr\n\t}\n\tfor _, e := range entries {\n\t\tn.prop.push(newProposedEntry(e, _EMPTY_))\n\t}\n\treturn nil\n}\n\n// ForwardProposal will forward the proposal to the leader if known.\n// If we are the leader this is the same as calling propose.\nfunc (n *raft) ForwardProposal(entry []byte) error {\n\tif n.State() == Leader {\n\t\treturn n.Propose(entry)\n\t}\n\n\t// TODO: Currently we do not set a reply subject, even though we are\n\t// now capable of responding. Do this once enough time has passed,\n\t// i.e. maybe in 2.12.\n\tn.sendRPC(n.psubj, _EMPTY_, entry)\n\treturn nil\n}\n\n// ProposeAddPeer is called to add a peer to the group.\nfunc (n *raft) ProposeAddPeer(peer string) error {\n\tn.RLock()\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader {\n\t\tn.RUnlock()\n\t\treturn errNotLeader\n\t}\n\t// Error if we had a previous write error.\n\tif werr := n.werr; werr != nil {\n\t\tn.RUnlock()\n\t\treturn werr\n\t}\n\tif n.membChangeIndex > 0 {\n\t\tn.RUnlock()\n\t\treturn errMembershipChange\n\t}\n\tprop := n.prop\n\tn.RUnlock()\n\n\tprop.push(newProposedEntry(newEntry(EntryAddPeer, []byte(peer)), _EMPTY_))\n\treturn nil\n}\n\n// ProposeRemovePeer is called to remove a peer from the group.\nfunc (n *raft) ProposeRemovePeer(peer string) error {\n\tn.RLock()\n\n\t// Error if we had a previous write error.\n\tif werr := n.werr; werr != nil {\n\t\tn.RUnlock()\n\t\treturn werr\n\t}\n\n\tif n.State() != Leader {\n\t\tsubj := n.rpsubj\n\t\tn.RUnlock()\n\n\t\t// Forward the proposal to the leader\n\t\tn.sendRPC(subj, _EMPTY_, []byte(peer))\n\t\treturn nil\n\t}\n\n\tif n.membChangeIndex > 0 {\n\t\tn.RUnlock()\n\t\treturn errMembershipChange\n\t}\n\n\tif len(n.peers) <= 1 {\n\t\tn.RUnlock()\n\t\treturn errRemoveLastNode\n\t}\n\n\tprop := n.prop\n\tn.RUnlock()\n\n\tprop.push(newProposedEntry(newEntry(EntryRemovePeer, []byte(peer)), _EMPTY_))\n\treturn nil\n}\n\nfunc (n *raft) MembershipChangeInProgress() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.membChangeIndex > 0\n}\n\n// ClusterSize reports back the total cluster size.\n// This effects quorum etc.\nfunc (n *raft) ClusterSize() int {\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.csz\n}\n\n// AdjustBootClusterSize can be called to adjust the boot cluster size.\n// Will error if called on a group with a leader or a previous leader.\n// This can be helpful in mixed mode.\nfunc (n *raft) AdjustBootClusterSize(csz int) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tif n.leader != noLeader || n.pleader.Load() {\n\t\treturn errAdjustBootCluster\n\t}\n\t// Same floor as bootstrap.\n\tif csz < 2 {\n\t\tcsz = 2\n\t}\n\t// Adjust the cluster size and the number of nodes needed to establish\n\t// a quorum.\n\tn.csz = csz\n\tn.qn = n.csz/2 + 1\n\n\treturn nil\n}\n\n// AdjustClusterSize will change the cluster set size.\n// Must be the leader.\nfunc (n *raft) AdjustClusterSize(csz int) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader {\n\t\treturn errNotLeader\n\t}\n\t// Same floor as bootstrap.\n\tif csz < 2 {\n\t\tcsz = 2\n\t}\n\n\t// Adjust the cluster size and the number of nodes needed to establish\n\t// a quorum.\n\tn.csz = csz\n\tn.qn = n.csz/2 + 1\n\n\tn.sendPeerState()\n\treturn nil\n}\n\n// PauseApply will allow us to pause processing of append entries onto our\n// external apply queue. In effect this means that the upper layer will no longer\n// receive any new entries from the Raft group.\nfunc (n *raft) PauseApply() error {\n\tif n.State() == Leader {\n\t\treturn errAlreadyLeader\n\t}\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.pauseApplyLocked()\n\treturn nil\n}\n\nfunc (n *raft) pauseApplyLocked() {\n\t// If we are currently a candidate make sure we step down.\n\tif n.State() == Candidate {\n\t\tn.stepdownLocked(noLeader)\n\t}\n\n\tn.debug(\"Pausing our apply channel\")\n\tn.paused = true\n\tif n.hcommit < n.commit {\n\t\tn.hcommit = n.commit\n\t}\n\t// Also prevent us from trying to become a leader while paused and catching up.\n\tn.resetElect(observerModeInterval)\n}\n\n// ResumeApply will resume sending applies to the external apply queue. This\n// means that we will start sending new entries to the upper layer.\nfunc (n *raft) ResumeApply() {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tif !n.paused {\n\t\treturn\n\t}\n\n\tn.debug(\"Resuming our apply channel\")\n\n\t// Reset before we start.\n\tn.resetElectionTimeout()\n\n\t// Run catchup..\n\tif n.hcommit > n.commit {\n\t\tn.debug(\"Resuming %d replays\", n.hcommit+1-n.commit)\n\t\tfor index := n.commit + 1; index <= n.hcommit; index++ {\n\t\t\tif err := n.applyCommit(index); err != nil {\n\t\t\t\tn.warn(\"Got error on apply commit during replay: %v\", err)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// We want to unlock here to allow the upper layers to call Applied() without blocking.\n\t\t\tn.Unlock()\n\t\t\t// Give hint to let other Go routines run.\n\t\t\t// Might not be necessary but seems to make it more fine grained interleaving.\n\t\t\truntime.Gosched()\n\t\t\t// Simply re-acquire\n\t\t\tn.Lock()\n\t\t\t// Need to check if we got closed.\n\t\t\tif n.State() == Closed {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// Clear our paused state after we apply.\n\tn.paused = false\n\tn.hcommit = 0\n\n\t// If we had been selected to be the next leader campaign here now that we have resumed.\n\tif n.lxfer {\n\t\tn.xferCampaign()\n\t} else {\n\t\tn.resetElectionTimeout()\n\t}\n}\n\n// DrainAndReplaySnapshot will drain the apply queue and replay the snapshot.\n// Our highest known commit will be preserved by pausing applies. The caller\n// should make sure to call ResumeApply() when handling the snapshot from the\n// queue, which will populate the rest of the committed entries in the queue.\nfunc (n *raft) DrainAndReplaySnapshot() bool {\n\tn.Lock()\n\tdefer n.Unlock()\n\tsnap, err := n.loadLastSnapshot()\n\tif err != nil {\n\t\treturn false\n\t}\n\tn.warn(\"Draining and replaying snapshot\")\n\tn.pauseApplyLocked()\n\tn.apply.drain()\n\t// Cancel after draining, we might have sent EntryCatchup and need to get them the nil entry.\n\tn.cancelCatchup()\n\tn.commit = snap.lastIndex\n\tn.apply.push(newCommittedEntry(n.commit, []*Entry{{EntrySnapshot, snap.data}}))\n\treturn true\n}\n\n// Applied is a callback that must be called by the upper layer when it\n// has successfully applied the committed entries that it received from the\n// apply queue. It will return the number of entries and an estimation of the\n// byte size that could be removed with a snapshot/compact.\nfunc (n *raft) Applied(index uint64) (entries uint64, bytes uint64) {\n\treturn n.Processed(index, index)\n}\n\n// Processed is a callback that must be called by the upper layer when it\n// has processed the committed entries that it received from the apply queue,\n// but it (maybe) hasn't applied all the processed entries yet.\n// Used to indicate a commit was processed, even if it wasn't applied yet and\n// can't be compacted away by a snapshot just yet. Which allows us to try to\n// become leader if we've processed all commits, even if they're not all applied.\nfunc (n *raft) Processed(index uint64, applied uint64) (entries uint64, bytes uint64) {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\t// Ignore if not applicable. This can happen during a reset.\n\tif index > n.commit {\n\t\treturn 0, 0\n\t}\n\n\t// Ignore if already processed.\n\tif index > n.processed {\n\t\tn.processed = index\n\t}\n\n\t// Ignore if already applied.\n\tif applied > index {\n\t\tapplied = index\n\t}\n\tif applied > n.applied {\n\t\tn.applied = applied\n\t}\n\n\t// If it was set, and we reached the minimum processed index, reset and send signal to upper layer.\n\t// We're not waiting for processed AND applied, because applying could take longer.\n\tif n.aflr > 0 && n.processed >= n.aflr {\n\t\tn.aflr = 0\n\t\t// Quick sanity-check to confirm we're still leader.\n\t\t// In which case we must signal, since switchToLeader would not have done so already.\n\t\tif n.State() == Leader {\n\t\t\tif !n.leaderState.Swap(true) {\n\t\t\t\t// Only update timestamp if leader state actually changed.\n\t\t\t\tnowts := time.Now().UTC()\n\t\t\t\tn.leaderSince.Store(&nowts)\n\t\t\t}\n\t\t\tn.updateLeadChange(true)\n\t\t}\n\t}\n\n\t// Calculate the number of entries and estimate the byte size that\n\t// we can now remove with a compaction/snapshot.\n\tif n.applied > n.papplied {\n\t\tentries = n.applied - n.papplied\n\t}\n\tif msgs := n.pindex - n.papplied; msgs > 0 {\n\t\tbytes = entries * n.bytes / msgs\n\t}\n\treturn entries, bytes\n}\n\n// For capturing data needed by snapshot.\ntype snapshot struct {\n\tlastTerm uint64\n\tlastIndex uint64\n\tpeerstate []byte\n\tdata []byte\n}\n\nconst minSnapshotLen = 28\n\n// Encodes a snapshot into a buffer for storage.\n// Lock should be held.\nfunc (n *raft) encodeSnapshot(snap *snapshot) []byte {\n\tif snap == nil {\n\t\treturn nil\n\t}\n\tvar le = binary.LittleEndian\n\tbuf := make([]byte, minSnapshotLen+len(snap.peerstate)+len(snap.data))\n\tle.PutUint64(buf[0:], snap.lastTerm)\n\tle.PutUint64(buf[8:], snap.lastIndex)\n\t// Peer state\n\tle.PutUint32(buf[16:], uint32(len(snap.peerstate)))\n\twi := 20\n\tcopy(buf[wi:], snap.peerstate)\n\twi += len(snap.peerstate)\n\t// data itself.\n\tcopy(buf[wi:], snap.data)\n\twi += len(snap.data)\n\n\t// Now do the hash for the end.\n\tn.hh.Reset()\n\tn.hh.Write(buf[:wi])\n\tvar hb [highwayhash.Size64]byte\n\tchecksum := n.hh.Sum(hb[:0])\n\tcopy(buf[wi:], checksum)\n\twi += len(checksum)\n\treturn buf[:wi]\n}\n\n// SendSnapshot will send the latest snapshot as a normal AE.\n// Should only be used when the upper layers know this is most recent.\n// Used when restoring streams, moving a stream from R1 to R>1, etc.\nfunc (n *raft) SendSnapshot(data []byte) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\t// Don't check if we're leader before sending and storing, this is used on scaleup.\n\tn.sendAppendEntryLocked([]*Entry{{EntrySnapshot, data}}, false)\n\treturn nil\n}\n\n// Used to install a snapshot for the given term and applied index. This will release\n// all of the log entries up to and including index. This should not be called with\n// entries that have been applied to the FSM but have not been applied to the raft state.\nfunc (n *raft) InstallSnapshot(data []byte, force bool) error {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tc, err := n.createSnapshotCheckpointLocked(force)\n\tif err != nil {\n\t\treturn err\n\t}\n\tc.n.debug(\"Installing snapshot of %d bytes [%d:%d]\", len(data), c.term, c.applied)\n\tsnap := &snapshot{\n\t\tlastTerm: c.term,\n\t\tlastIndex: c.applied,\n\t\tpeerstate: c.peerstate,\n\t\tdata: data,\n\t}\n\treturn c.n.installSnapshot(snap)\n}\n\n// Install the snapshot.\n// Lock should be held.\nfunc (n *raft) installSnapshot(snap *snapshot) error {\n\t// Always reset, regardless of success or error.\n\t// This is done even though this doesn't come from a checkpoint. We do this so we can\n\t// interrupt/abort an asynchronously running snapshot (if it exists). Ensures the upper layer\n\t// can't overwrite a snapshot that we installed here with an old asynchronously created one.\n\tdefer func() {\n\t\tn.snapshotting = false\n\t}()\n\n\tsnapDir := filepath.Join(n.sd, snapshotsDir)\n\tsn := fmt.Sprintf(snapFileT, snap.lastTerm, snap.lastIndex)\n\tsfile := filepath.Join(snapDir, sn)\n\n\tif err := writeFileWithSync(sfile, n.encodeSnapshot(snap), defaultFilePerms); err != nil {\n\t\t// We could set write err here, but if this is a temporary situation, too many open files etc.\n\t\t// we want to retry and snapshots are not fatal.\n\t\treturn err\n\t}\n\n\t// Delete our previous snapshot file if it exists.\n\tif n.snapfile != _EMPTY_ && n.snapfile != sfile {\n\t\tos.Remove(n.snapfile)\n\t}\n\t// Remember our latest snapshot file.\n\tn.snapfile = sfile\n\tif _, err := n.wal.Compact(snap.lastIndex + 1); err != nil {\n\t\tn.setWriteErrLocked(err)\n\t\treturn err\n\t}\n\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\tn.papplied = snap.lastIndex\n\tn.bytes = state.Bytes\n\treturn nil\n}\n\n// CreateSnapshotCheckpoint creates a checkpoint to allow installing a snapshot asynchronously.\n// Caller MUST make sure it only ever has one checkpoint handle at most, and either installs or\n// aborts the checkpoint.\n// See also: RaftNodeCheckpoint\nfunc (n *raft) CreateSnapshotCheckpoint(force bool) (RaftNodeCheckpoint, error) {\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.createSnapshotCheckpointLocked(force)\n}\n\nfunc (n *raft) createSnapshotCheckpointLocked(force bool) (*checkpoint, error) {\n\tif n.State() == Closed {\n\t\treturn nil, errNodeClosed\n\t}\n\tif n.snapshotting {\n\t\treturn nil, errSnapInProgress\n\t}\n\n\t// If a write error has occurred already then stop here.\n\tif werr := n.werr; werr != nil {\n\t\treturn nil, werr\n\t}\n\n\t// Check that a catchup isn't already taking place. If it is then we won't\n\t// allow installing snapshots until it is done.\n\t// Unless we're forced to snapshot. We might have been catching up a peer for\n\t// a long period, and this protects our log size from growing indefinitely.\n\tif !force && len(n.progress) > 0 {\n\t\treturn nil, errCatchupsRunning\n\t}\n\n\tif n.applied == 0 {\n\t\tn.debug(\"Not snapshotting as there are no applied entries\")\n\t\treturn nil, errNoSnapAvailable\n\t}\n\n\tvar term uint64\n\tif ae, _ := n.loadEntry(n.applied); ae != nil {\n\t\tterm = ae.term\n\t\tae.returnToPool()\n\t} else {\n\t\tn.debug(\"Not snapshotting as entry %d is not available\", n.applied)\n\t\treturn nil, errNoSnapAvailable\n\t}\n\n\t// Snapshot the current peer state for the current applied index, we'll need it in the snapshot.\n\tpeerstate := encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})\n\tsnapDir := filepath.Join(n.sd, snapshotsDir)\n\tsnapFile := filepath.Join(snapDir, fmt.Sprintf(snapFileT, term, n.applied))\n\n\tn.snapshotting = true\n\tc := &checkpoint{\n\t\tn: n,\n\t\tterm: term,\n\t\tapplied: n.applied,\n\t\tpapplied: n.papplied,\n\t\tsnapFile: snapFile,\n\t\tpeerstate: peerstate,\n\t}\n\treturn c, nil\n}\n\ntype checkpoint struct {\n\tn *raft // Reference to the RaftNode.\n\tterm uint64 // The term of the entry at applied.\n\tapplied uint64 // What applied value the snapshot will represent and what the log can be compacted to.\n\tpapplied uint64 // Previous applied value of the previous snapshot.\n\tsnapFile string // Where the snapshot should be installed.\n\tpeerstate []byte // Encoded peerstate generated when creating this checkpoint.\n}\n\n// LoadLastSnapshot loads the last snapshot from disk when using a RaftNodeCheckpoint.\nfunc (c *checkpoint) LoadLastSnapshot() ([]byte, error) {\n\tc.n.Lock()\n\tdefer c.n.Unlock()\n\tif !c.n.snapshotting {\n\t\t// The checkpoint can be aborted at any time, don't continue if that happened.\n\t\treturn nil, errSnapAborted\n\t}\n\tsnap, err := c.n.loadLastSnapshot()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif snap.lastIndex != c.papplied {\n\t\t// Another snapshot was installed in the meantime. This invalidates our checkpoint.\n\t\treturn nil, errors.New(\"snapshot index mismatch\")\n\t}\n\treturn snap.data, nil\n}\n\n// AppendEntriesSeq allows iterating over entries that can be compacted as part of a snapshot.\nfunc (c *checkpoint) AppendEntriesSeq() iter.Seq2[*appendEntry, error] {\n\treturn func(yield func(*appendEntry, error) bool) {\n\t\tfor index := c.papplied + 1; index <= c.applied; index++ {\n\t\t\tc.n.Lock()\n\t\t\tif !c.n.snapshotting {\n\t\t\t\tc.n.Unlock()\n\t\t\t\t// The checkpoint can be aborted at any time, don't continue if that happened.\n\t\t\t\tyield(nil, errSnapAborted)\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Load entry and yield to the caller while unlocked.\n\t\t\tae, err := c.n.loadEntry(index)\n\t\t\tc.n.Unlock()\n\t\t\tif err != nil {\n\t\t\t\tyield(nil, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tyield(ae, nil)\n\t\t\tae.returnToPool()\n\t\t}\n\t}\n}\n\n// Abort can be called to cancel the snapshot installation at any time.\nfunc (c *checkpoint) Abort() {\n\tc.n.Lock()\n\tdefer c.n.Unlock()\n\tc.n.snapshotting = false\n}\n\n// InstallSnapshot allows asynchronous installation of a snapshot by unlocking when\n// performing operations that don't strictly need to be locked. When the lock is re-acquired\n// n.snapshotting will be checked to ensure we're still meant to.\n// Async snapshots can only be used when using CreateSnapshotCheckpoint.\n// Lock should be held.\nfunc (c *checkpoint) InstallSnapshot(data []byte) (uint64, error) {\n\tn := c.n\n\tn.Lock()\n\tdefer n.Unlock()\n\tif !n.snapshotting {\n\t\t// The checkpoint can be aborted at any time, don't continue if that happened.\n\t\treturn 0, errSnapAborted\n\t}\n\n\t// Always reset, regardless of success or error.\n\tdefer func() {\n\t\tn.snapshotting = false\n\t}()\n\n\tn.debug(\"Installing snapshot of %d bytes [%d:%d]\", len(data), c.term, c.applied)\n\tsnap := &snapshot{\n\t\tlastTerm: c.term,\n\t\tlastIndex: c.applied,\n\t\tpeerstate: c.peerstate,\n\t\tdata: data,\n\t}\n\tencoded := n.encodeSnapshot(snap)\n\n\t// Unlock while writing.\n\tn.Unlock()\n\terr := writeFileWithSync(c.snapFile, encoded, defaultFilePerms)\n\tn.Lock()\n\tif err != nil {\n\t\t// We could set write err here, but if this is a temporary situation, too many open files etc.\n\t\t// we want to retry and snapshots are not fatal.\n\t\treturn 0, err\n\t} else if !n.snapshotting {\n\t\t// The checkpoint can be aborted at any time, don't continue if that happened.\n\t\treturn 0, errSnapAborted\n\t}\n\n\t// Delete our previous snapshot file if it exists.\n\tif n.snapfile != _EMPTY_ && n.snapfile != c.snapFile {\n\t\tos.Remove(n.snapfile)\n\t}\n\t// Remember our latest snapshot file.\n\tn.snapfile = c.snapFile\n\n\t// Unlock while compacting.\n\tn.Unlock()\n\t_, err = n.wal.Compact(snap.lastIndex + 1)\n\tn.Lock()\n\tif err != nil {\n\t\tn.setWriteErrLocked(err)\n\t\treturn 0, err\n\t} else if !n.snapshotting {\n\t\t// The checkpoint can be aborted at any time, don't continue if that happened.\n\t\treturn 0, errSnapAborted\n\t}\n\n\tcompacted := n.bytes\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\tn.papplied = snap.lastIndex\n\tn.bytes = state.Bytes\n\n\t// Expose compacted size.\n\tif n.bytes > compacted {\n\t\tcompacted = 0\n\t} else {\n\t\tcompacted -= n.bytes\n\t}\n\treturn compacted, nil\n}\n\n// NeedSnapshot returns true if it is necessary to try to install a snapshot, i.e.\n// after we have finished recovering/replaying at startup, on a regular interval or\n// as a part of cleaning up when shutting down.\nfunc (n *raft) NeedSnapshot() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.snapfile == _EMPTY_ && n.applied > 1\n}\n\nconst (\n\tsnapshotsDir = \"snapshots\"\n\tsnapFileT = \"snap.%d.%d\"\n)\n\n// termAndIndexFromSnapfile tries to load the snapshot file and returns the term\n// and index from that snapshot.\nfunc termAndIndexFromSnapFile(sn string) (term, index uint64, err error) {\n\tif sn == _EMPTY_ {\n\t\treturn 0, 0, errBadSnapName\n\t}\n\tfn := filepath.Base(sn)\n\tif n, err := fmt.Sscanf(fn, snapFileT, &term, &index); err != nil || n != 2 {\n\t\treturn 0, 0, errBadSnapName\n\t}\n\treturn term, index, nil\n}\n\n// setupLastSnapshot is called at startup to try and recover the last snapshot from\n// the disk if possible. We will try to recover the term, index and commit/applied\n// indices and then notify the upper layer what we found. Compacts the WAL if needed.\nfunc (n *raft) setupLastSnapshot() error {\n\tsnapDir := filepath.Join(n.sd, snapshotsDir)\n\tpsnaps, err := os.ReadDir(snapDir)\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn errNoSnapAvailable\n\t\t}\n\t\treturn err\n\t}\n\n\tvar lterm, lindex uint64\n\tvar latest string\n\tfor _, sf := range psnaps {\n\t\tsfile := filepath.Join(snapDir, sf.Name())\n\t\tvar term, index uint64\n\t\tterm, index, err := termAndIndexFromSnapFile(sf.Name())\n\t\tif err == nil {\n\t\t\tif term > lterm {\n\t\t\t\tlterm, lindex = term, index\n\t\t\t\tlatest = sfile\n\t\t\t} else if term == lterm && index > lindex {\n\t\t\t\tlindex = index\n\t\t\t\tlatest = sfile\n\t\t\t}\n\t\t} else {\n\t\t\t// Clean this up, can't parse the name.\n\t\t\t// TODO(dlc) - We could read in and check actual contents.\n\t\t\tn.debug(\"Removing snapshot, can't parse name: %q\", sf.Name())\n\t\t\tos.Remove(sfile)\n\t\t}\n\t}\n\tif latest == _EMPTY_ {\n\t\treturn nil\n\t}\n\n\t// Set latest snapshot we have.\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tn.snapfile = latest\n\tsnap, err := n.loadLastSnapshot()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// We successfully recovered the last snapshot from the disk.\n\t// Recover state from the snapshot and then notify the upper layer.\n\t// Compact the WAL when we're done if needed.\n\tn.pindex = snap.lastIndex\n\tn.pterm = snap.lastTerm\n\t// Explicitly only set commit, and not applied.\n\t// Applied will move up when the snapshot is actually applied.\n\tn.commit = snap.lastIndex\n\tn.papplied = snap.lastIndex\n\t// Restore the peerState\n\tps, err := decodePeerState(snap.peerstate)\n\tif err != nil {\n\t\treturn err\n\t}\n\tn.processPeerState(ps)\n\tn.extSt = ps.domainExt\n\n\tn.apply.push(newCommittedEntry(n.commit, []*Entry{{EntrySnapshot, snap.data}}))\n\tif _, err := n.wal.Compact(snap.lastIndex + 1); err != nil {\n\t\tn.setWriteErrLocked(err)\n\t\treturn err\n\t}\n\n\t// Now cleanup any old entries. We only do this once we know that the\n\t// latest snapshot was OK.\n\tfor _, sf := range psnaps {\n\t\tif sfile := filepath.Join(snapDir, sf.Name()); sfile != latest {\n\t\t\tn.debug(\"Removing old snapshot: %q\", sfile)\n\t\t\tos.Remove(sfile)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// loadLastSnapshot will load and return our last snapshot.\n// Lock should be held.\nfunc (n *raft) loadLastSnapshot() (*snapshot, error) {\n\tif n.snapfile == _EMPTY_ {\n\t\treturn nil, errNoSnapAvailable\n\t}\n\n\t<-dios\n\tbuf, err := os.ReadFile(n.snapfile)\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\tn.warn(\"Error reading snapshot: %v\", err)\n\t\treturn nil, err\n\t}\n\tif len(buf) < minSnapshotLen {\n\t\tn.warn(\"Snapshot corrupt, too short\")\n\t\treturn nil, errSnapshotCorrupt\n\t}\n\n\t// Check to make sure hash is consistent.\n\thoff := len(buf) - 8\n\tlchk := buf[hoff:]\n\tn.hh.Reset()\n\tn.hh.Write(buf[:hoff])\n\tvar hb [highwayhash.Size64]byte\n\tif !bytes.Equal(lchk[:], n.hh.Sum(hb[:0])) {\n\t\tn.warn(\"Snapshot corrupt, checksums did not match\")\n\t\treturn nil, errSnapshotCorrupt\n\t}\n\n\tvar le = binary.LittleEndian\n\tlps := le.Uint32(buf[16:])\n\tsnap := &snapshot{\n\t\tlastTerm: le.Uint64(buf[0:]),\n\t\tlastIndex: le.Uint64(buf[8:]),\n\t\tpeerstate: buf[20 : 20+lps],\n\t\tdata: buf[20+lps : hoff],\n\t}\n\n\t// We had a bug in 2.9.12 that would allow snapshots on last index of 0.\n\t// Detect that and continue anyway, nothing else we can do about it.\n\tif snap.lastIndex == 0 {\n\t\tn.warn(\"Snapshot with last index 0 is invalid, cleaning up\")\n\t\tos.Remove(n.snapfile)\n\t\tn.snapfile = _EMPTY_\n\t\treturn nil, nil\n\t}\n\n\treturn snap, nil\n}\n\n// Leader returns if we are the leader for our group.\n// We use an atomic here now vs acquiring the read lock.\nfunc (n *raft) Leader() bool {\n\tif n == nil {\n\t\treturn false\n\t}\n\treturn n.leaderState.Load()\n}\n\n// LeaderSince returns how long we have been leader for,\n// if applicable.\nfunc (n *raft) LeaderSince() *time.Time {\n\tif n == nil {\n\t\treturn nil\n\t}\n\treturn n.leaderSince.Load()\n}\n\n// stepdown immediately steps down the Raft node to the\n// follower state. This will take the lock itself.\nfunc (n *raft) stepdown(newLeader string) {\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.stepdownLocked(newLeader)\n}\n\n// stepdownLocked immediately steps down the Raft node to the\n// follower state. This requires the lock is already held.\nfunc (n *raft) stepdownLocked(newLeader string) {\n\tn.debug(\"Stepping down\")\n\tn.switchToFollowerLocked(newLeader)\n}\n\n// isCatchingUp returns true if a catchup is currently taking place.\nfunc (n *raft) isCatchingUp() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.catchup != nil\n}\n\n// isCurrent is called from the healthchecks and returns true if we believe\n// that the upper layer is current with the Raft layer, i.e. that it has applied\n// all of the commits that we have given it.\n// Optionally we can also check whether or not we're making forward progress if we\n// aren't current, in which case this function may block for up to ~10ms to find out.\n// Lock should be held.\nfunc (n *raft) isCurrent(includeForwardProgress bool) bool {\n\t// Check if we are closed.\n\tif n.State() == Closed {\n\t\tn.debug(\"Not current, node is closed\")\n\t\treturn false\n\t}\n\n\t// Check whether we've made progress on any state, 0 is invalid so not healthy.\n\tif n.commit == 0 {\n\t\tn.debug(\"Not current, no commits\")\n\t\treturn false\n\t}\n\n\t// If we were previously logging about falling behind, also log when the problem\n\t// was cleared.\n\tclearBehindState := func() {\n\t\tif n.hcbehind {\n\t\t\tn.warn(\"Health check OK, no longer falling behind\")\n\t\t\tn.hcbehind = false\n\t\t}\n\t}\n\n\t// Make sure we are the leader or we know we have heard from the leader recently.\n\tif n.State() == Leader {\n\t\tclearBehindState()\n\t\treturn true\n\t}\n\n\t// Check to see that we have heard from the current leader lately.\n\tif n.leader != noLeader && n.leader != n.id && n.catchup == nil {\n\t\tokInterval := hbInterval * 2\n\t\tif ps := n.peers[n.leader]; ps == nil || time.Since(ps.ts) > okInterval {\n\t\t\tn.debug(\"Not current, no recent leader contact\")\n\t\t\treturn false\n\t\t}\n\t}\n\tif cs := n.catchup; cs != nil {\n\t\t// We're actively catching up, can't mark current even if commit==applied.\n\t\tn.debug(\"Not current, still catching up pindex=%d, cindex=%d\", n.pindex, cs.cindex)\n\t\treturn false\n\t}\n\n\tif n.paused && n.hcommit > n.commit {\n\t\t// We're currently paused, waiting to be resumed to apply pending commits.\n\t\tn.debug(\"Not current, waiting to resume applies commit=%d, hcommit=%d\", n.commit, n.hcommit)\n\t\treturn false\n\t}\n\n\tif n.commit == n.applied {\n\t\t// At this point if we are current, we can return saying so.\n\t\tclearBehindState()\n\t\treturn true\n\t} else if !includeForwardProgress {\n\t\t// Otherwise, if we aren't allowed to include forward progress\n\t\t// (i.e. we are checking \"current\" instead of \"healthy\") then\n\t\t// give up now.\n\t\treturn false\n\t}\n\n\t// Otherwise, wait for a short period of time and see if we are making any\n\t// forward progress.\n\tif startDelta := n.commit - n.applied; startDelta > 0 {\n\t\tfor i := 0; i < 10; i++ { // 10ms, in 1ms increments\n\t\t\tn.Unlock()\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t\tn.Lock()\n\t\t\tif n.State() == Closed {\n\t\t\t\tn.debug(\"Node closed during health check, returning not current\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif n.commit-n.applied < startDelta {\n\t\t\t\t// The gap is getting smaller, so we're making forward progress.\n\t\t\t\tclearBehindState()\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\n\tn.hcbehind = true\n\tn.warn(\"Falling behind in health check, commit %d != applied %d\", n.commit, n.applied)\n\treturn false\n}\n\n// Current returns if we are the leader for our group or an up to date follower.\nfunc (n *raft) Current() bool {\n\tif n == nil {\n\t\treturn false\n\t}\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.isCurrent(false)\n}\n\n// Healthy returns if we are the leader for our group and nearly up-to-date.\nfunc (n *raft) Healthy() bool {\n\tif n == nil {\n\t\treturn false\n\t}\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.isCurrent(true)\n}\n\n// HadPreviousLeader indicates if this group ever had a leader.\nfunc (n *raft) HadPreviousLeader() bool {\n\treturn n.pleader.Load()\n}\n\n// GroupLeader returns the current leader of the group.\nfunc (n *raft) GroupLeader() string {\n\tif n == nil {\n\t\treturn noLeader\n\t}\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.leader\n}\n\n// Leaderless is a lockless way of finding out if the group has a\n// leader or not. Use instead of GroupLeader in hot paths.\nfunc (n *raft) Leaderless() bool {\n\tif n == nil {\n\t\treturn true\n\t}\n\t// Negated because we want the default state of hasLeader to be\n\t// false until the first setLeader() call.\n\treturn !n.hasleader.Load()\n}\n\n// Guess the best next leader. Stepdown will check more thoroughly.\n// Lock should be held.\nfunc (n *raft) selectNextLeader() string {\n\tnextLeader, hli := noLeader, uint64(0)\n\tfor peer, ps := range n.peers {\n\t\tif peer == n.id || ps.li <= hli {\n\t\t\tcontinue\n\t\t}\n\t\thli = ps.li\n\t\tnextLeader = peer\n\t}\n\treturn nextLeader\n}\n\n// StepDown will have a leader stepdown and optionally do a leader transfer.\nfunc (n *raft) StepDown(preferred ...string) error {\n\tn.Lock()\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader {\n\t\tn.Unlock()\n\t\treturn errNotLeader\n\t}\n\tif len(preferred) > 1 {\n\t\tn.Unlock()\n\t\treturn errTooManyPrefs\n\t}\n\n\tn.debug(\"Being asked to stepdown\")\n\n\t// See if we have up to date followers.\n\tmaybeLeader := noLeader\n\tif len(preferred) > 0 {\n\t\tif preferred[0] != _EMPTY_ {\n\t\t\tmaybeLeader = preferred[0]\n\t\t} else {\n\t\t\tpreferred = nil\n\t\t}\n\t}\n\n\t// Can't pick ourselves.\n\tif maybeLeader == n.id {\n\t\tmaybeLeader = noLeader\n\t\tpreferred = nil\n\t}\n\n\t// If we have a preferred check it first.\n\tif maybeLeader != noLeader {\n\t\tvar isHealthy bool\n\t\tif ps, ok := n.peers[maybeLeader]; ok {\n\t\t\tsi, ok := n.s.nodeToInfo.Load(maybeLeader)\n\t\t\tisHealthy = ok && !si.(nodeInfo).offline && time.Since(ps.ts) < hbInterval*3\n\t\t}\n\t\tif !isHealthy {\n\t\t\tmaybeLeader = noLeader\n\t\t}\n\t}\n\n\t// If we do not have a preferred at this point pick the first healthy one.\n\t// Make sure not ourselves.\n\tif maybeLeader == noLeader {\n\t\tfor peer, ps := range n.peers {\n\t\t\tif peer == n.id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsi, ok := n.s.nodeToInfo.Load(peer)\n\t\t\tisHealthy := ok && !si.(nodeInfo).offline && time.Since(ps.ts) < hbInterval*3\n\t\t\tif isHealthy {\n\t\t\t\tmaybeLeader = peer\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tn.Unlock()\n\n\tif len(preferred) > 0 && maybeLeader == noLeader {\n\t\tn.debug(\"Can not transfer to preferred peer %q\", preferred[0])\n\t}\n\n\t// If we have a new leader selected, transfer over to them.\n\t// Send the append entry directly rather than via the proposals queue,\n\t// as we will switch to follower state immediately and will blow away\n\t// the contents of the proposal queue in the process.\n\tif maybeLeader != noLeader {\n\t\tn.debug(\"Selected %q for new leader, stepping down due to leadership transfer\", maybeLeader)\n\t\tae := newEntry(EntryLeaderTransfer, []byte(maybeLeader))\n\t\tn.sendAppendEntry([]*Entry{ae})\n\t}\n\n\t// Force us to stepdown here.\n\tn.stepdown(noLeader)\n\n\treturn nil\n}\n\n// Campaign will have our node start a leadership vote.\nfunc (n *raft) Campaign() error {\n\tn.Lock()\n\tdefer n.Unlock()\n\treturn n.campaign(randCampaignTimeout())\n}\n\n// CampaignImmediately will have our node start a leadership vote after minimal delay.\nfunc (n *raft) CampaignImmediately() error {\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.maybeLeader = true\n\tn.resetInitializing()\n\treturn n.campaign(minCampaignTimeout / 2)\n}\n\nfunc randCampaignTimeout() time.Duration {\n\tdelta := rand.Int63n(int64(maxCampaignTimeout - minCampaignTimeout))\n\treturn (minCampaignTimeout + time.Duration(delta))\n}\n\n// Campaign will have our node start a leadership vote.\n// Lock should be held.\nfunc (n *raft) campaign(et time.Duration) error {\n\tn.debug(\"Starting campaign\")\n\tif n.State() == Leader {\n\t\treturn errAlreadyLeader\n\t}\n\tn.resetElect(et)\n\treturn nil\n}\n\n// xferCampaign will have our node start an immediate leadership vote.\n// Lock should be held.\nfunc (n *raft) xferCampaign() error {\n\tn.debug(\"Starting transfer campaign\")\n\tif n.State() == Leader {\n\t\tn.lxfer = false\n\t\treturn errAlreadyLeader\n\t}\n\tn.resetElect(10 * time.Millisecond)\n\treturn nil\n}\n\n// State returns the current state for this node.\n// Upper layers should not check State to check if we're Leader, use n.Leader() instead.\nfunc (n *raft) State() RaftState {\n\treturn RaftState(n.state.Load())\n}\n\n// Progress returns the current index, commit and applied values.\nfunc (n *raft) Progress() (index, commit, applied uint64) {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.pindex, n.commit, n.applied\n}\n\n// Size returns number of entries and total bytes for our WAL.\nfunc (n *raft) Size() (entries uint64, bytes uint64) {\n\tn.RLock()\n\tentries = n.pindex - n.papplied\n\tbytes = n.bytes\n\tn.RUnlock()\n\treturn entries, bytes\n}\n\nfunc (n *raft) ID() string {\n\tif n == nil {\n\t\treturn _EMPTY_\n\t}\n\t// Lock not needed as n.id is never changed after creation.\n\treturn n.id\n}\n\nfunc (n *raft) Group() string {\n\t// Lock not needed as n.group is never changed after creation.\n\treturn n.group\n}\n\nfunc (n *raft) Peers() []*Peer {\n\tn.RLock()\n\tdefer n.RUnlock()\n\n\tvar peers []*Peer\n\tfor id, ps := range n.peers {\n\t\tvar current bool\n\t\tvar lag uint64\n\t\tif id == n.id {\n\t\t\t// We are current and have no lag when compared with ourselves.\n\t\t\tcurrent = true\n\t\t} else if n.id == n.leader {\n\t\t\t// We are the leader, we know how many entries this replica has persisted.\n\t\t\t// Lag is determined by how many entries we have quorum on in our log that haven't yet\n\t\t\t// been persisted on the replica. They are current if there's no lag.\n\t\t\t// This will show all peers that are part of quorum as \"current\".\n\t\t\tif n.commit > ps.li {\n\t\t\t\tlag = n.commit - ps.li\n\t\t\t}\n\t\t\tcurrent = lag == 0\n\t\t} else if id == n.leader {\n\t\t\t// This peer is the leader, we don't know our lag, but we can report\n\t\t\t// on whether we've seen the leader recently.\n\t\t\tokInterval := hbInterval * 2\n\t\t\tcurrent = time.Since(ps.ts) <= okInterval\n\t\t} else {\n\t\t\t// The remaining condition is another follower that we're not in contact with.\n\t\t\t// We intentionally leave current and lag as empty.\n\t\t\tcurrent, lag = false, 0\n\t\t}\n\n\t\tp := &Peer{\n\t\t\tID: id,\n\t\t\tCurrent: current,\n\t\t\tLast: ps.ts,\n\t\t\tLag: lag,\n\t\t}\n\t\tpeers = append(peers, p)\n\t}\n\treturn peers\n}\n\n// Update and propose our known set of peers.\nfunc (n *raft) ProposeKnownPeers(knownPeers []string) {\n\tn.Lock()\n\tdefer n.Unlock()\n\t// If we are the leader update and send this update out.\n\tif n.State() != Leader {\n\t\treturn\n\t}\n\tn.updateKnownPeersLocked(knownPeers)\n\tn.sendPeerState()\n}\n\n// Update our known set of peers.\nfunc (n *raft) UpdateKnownPeers(knownPeers []string) {\n\tn.Lock()\n\tn.updateKnownPeersLocked(knownPeers)\n\tn.Unlock()\n}\n\nfunc (n *raft) updateKnownPeersLocked(knownPeers []string) {\n\t// Process like peer state update.\n\tps := &peerState{knownPeers, len(knownPeers), n.extSt}\n\tn.processPeerState(ps)\n}\n\n// ApplyQ returns the apply queue that new commits will be sent to for the\n// upper layer to apply.\nfunc (n *raft) ApplyQ() *ipQueue[*CommittedEntry] { return n.apply }\n\n// LeadChangeC returns the leader change channel, notifying when the Raft\n// leader role has moved.\nfunc (n *raft) LeadChangeC() <-chan bool { return n.leadc }\n\n// QuitC returns the quit channel, notifying when the Raft group has shut down.\nfunc (n *raft) QuitC() <-chan struct{} { return n.quit }\n\nfunc (n *raft) Created() time.Time {\n\t// Lock not needed as n.created is never changed after creation.\n\treturn n.created\n}\n\nfunc (n *raft) Stop() {\n\tn.shutdown()\n}\n\nfunc (n *raft) WaitForStop() {\n\tif n.state.Load() == int32(Closed) {\n\t\tn.wg.Wait()\n\t}\n}\n\nfunc (n *raft) Delete() {\n\tn.shutdown()\n\tn.wg.Wait()\n\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tn.deleted = true\n\tif wal := n.wal; wal != nil {\n\t\twal.Delete(false)\n\t}\n\tos.RemoveAll(n.sd)\n\tn.debug(\"Deleted\")\n}\n\nfunc (n *raft) IsDeleted() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.deleted\n}\n\nfunc (n *raft) shutdown() {\n\t// First call to Stop or Delete should close the quit chan\n\t// to notify the runAs goroutines to stop what they're doing.\n\tif n.state.Swap(int32(Closed)) != int32(Closed) {\n\t\tn.leaderState.Store(false)\n\t\tn.leaderSince.Store(nil)\n\t\tclose(n.quit)\n\t}\n}\n\nconst (\n\traftAllSubj = \"$NRG.>\"\n\traftVoteSubj = \"$NRG.V.%s\"\n\traftAppendSubj = \"$NRG.AE.%s\"\n\traftPropSubj = \"$NRG.P.%s\"\n\traftRemovePeerSubj = \"$NRG.RP.%s\"\n\traftReply = \"$NRG.R.%s\"\n\traftCatchupReply = \"$NRG.CR.%s\"\n)\n\n// Lock should be held (due to use of random generator)\nfunc (n *raft) newCatchupInbox() string {\n\tvar b [replySuffixLen]byte\n\trn := fastrand.Uint64()\n\tfor i, l := 0, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\treturn fmt.Sprintf(raftCatchupReply, b[:])\n}\n\nfunc (n *raft) newInbox() string {\n\tvar b [replySuffixLen]byte\n\trn := fastrand.Uint64()\n\tfor i, l := 0, rn; i < len(b); i++ {\n\t\tb[i] = digits[l%base]\n\t\tl /= base\n\t}\n\treturn fmt.Sprintf(raftReply, b[:])\n}\n\n// Our internal subscribe.\n// Lock should be held.\nfunc (n *raft) subscribe(subject string, cb msgHandler) (*subscription, error) {\n\tif n.c == nil {\n\t\treturn nil, errNoInternalClient\n\t}\n\treturn n.s.systemSubscribe(subject, _EMPTY_, false, n.c, cb)\n}\n\n// Lock should be held.\nfunc (n *raft) unsubscribe(sub *subscription) {\n\tif n.c != nil && sub != nil {\n\t\tn.c.processUnsub(sub.sid)\n\t}\n}\n\n// Lock should be held.\nfunc (n *raft) createInternalSubs() error {\n\tn.vsubj, n.vreply = fmt.Sprintf(raftVoteSubj, n.group), n.newInbox()\n\tn.asubj, n.areply = fmt.Sprintf(raftAppendSubj, n.group), n.newInbox()\n\tn.psubj = fmt.Sprintf(raftPropSubj, n.group)\n\tn.rpsubj = fmt.Sprintf(raftRemovePeerSubj, n.group)\n\n\t// Votes\n\tif _, err := n.subscribe(n.vreply, n.handleVoteResponse); err != nil {\n\t\treturn err\n\t}\n\tif _, err := n.subscribe(n.vsubj, n.handleVoteRequest); err != nil {\n\t\treturn err\n\t}\n\t// AppendEntry\n\tif _, err := n.subscribe(n.areply, n.handleAppendEntryResponse); err != nil {\n\t\treturn err\n\t}\n\tif sub, err := n.subscribe(n.asubj, n.handleAppendEntry); err != nil {\n\t\treturn err\n\t} else {\n\t\tn.aesub = sub\n\t}\n\n\treturn nil\n}\n\nfunc randElectionTimeout() time.Duration {\n\tdelta := rand.Int63n(int64(maxElectionTimeout - minElectionTimeout))\n\treturn (minElectionTimeout + time.Duration(delta))\n}\n\n// Lock should be held.\nfunc (n *raft) resetElectionTimeout() {\n\tn.resetElect(randElectionTimeout())\n}\n\nfunc (n *raft) resetElectionTimeoutWithLock() {\n\tn.resetElectWithLock(randElectionTimeout())\n}\n\n// Lock should be held.\nfunc (n *raft) resetElect(et time.Duration) {\n\tn.etlr = time.Now()\n\tif n.elect == nil {\n\t\tn.elect = time.NewTimer(et)\n\t} else {\n\t\tif !n.elect.Stop() {\n\t\t\tselect {\n\t\t\tcase <-n.elect.C:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\tn.elect.Reset(et)\n\t}\n}\n\nfunc (n *raft) resetElectWithLock(et time.Duration) {\n\tn.Lock()\n\tn.resetElect(et)\n\tn.Unlock()\n}\n\n// run is the top-level runner for the Raft state machine. Depending on the\n// state of the node (leader, follower, candidate, observer), this will call\n// through to other functions. It is expected that this function will run for\n// the entire life of the Raft node once started.\nfunc (n *raft) run() {\n\ts := n.s\n\tdefer s.grWG.Done()\n\tdefer n.wg.Done()\n\n\t// We want to wait for some routing to be enabled, so we will wait for\n\t// at least a route, leaf or gateway connection to be established before\n\t// starting the run loop.\n\tfor gw := s.gateway; ; {\n\t\ts.mu.RLock()\n\t\tready, gwEnabled := s.numRemotes()+len(s.leafs) > 0, gw.enabled\n\t\ts.mu.RUnlock()\n\t\tif !ready && gwEnabled {\n\t\t\tgw.RLock()\n\t\t\tready = len(gw.out)+len(gw.in) > 0\n\t\t\tgw.RUnlock()\n\t\t}\n\t\tif !ready {\n\t\t\tselect {\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(100 * time.Millisecond):\n\t\t\t\ts.RateLimitWarnf(\"Waiting for routing to be established...\")\n\t\t\t}\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// We may have paused adding entries to apply queue, resume here.\n\t// No-op if not paused.\n\tn.ResumeApply()\n\n\t// Send nil entry to signal the upper layers we are done doing replay/restore.\n\tn.apply.push(nil)\n\nrunner:\n\tfor {\n\t\tswitch n.State() {\n\t\tcase Follower:\n\t\t\tn.runAsFollower()\n\t\tcase Candidate:\n\t\t\tn.runAsCandidate()\n\t\tcase Leader:\n\t\t\tn.runAsLeader()\n\t\tcase Closed:\n\t\t\tbreak runner\n\t\t}\n\t}\n\n\t// If we've reached this point then we're shutting down, either because\n\t// the server is stopping or because the Raft group is closing/closed.\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tif c := n.c; c != nil {\n\t\tvar subs []*subscription\n\t\tc.mu.Lock()\n\t\tfor _, sub := range c.subs {\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t\tc.mu.Unlock()\n\t\tfor _, sub := range subs {\n\t\t\tn.unsubscribe(sub)\n\t\t}\n\t\tc.closeConnection(InternalClient)\n\t\tn.c = nil\n\t}\n\n\t// Unregistering ipQueues do not prevent them from push/pop\n\t// just will remove them from the central monitoring map\n\tqueues := []interface {\n\t\tunregister()\n\t\tdrain() int\n\t}{n.reqs, n.votes, n.prop, n.entry, n.resp, n.apply}\n\tfor _, q := range queues {\n\t\tq.drain()\n\t\tq.unregister()\n\t}\n\n\tn.s.unregisterRaftNode(n.group)\n\n\tif wal := n.wal; wal != nil {\n\t\twal.Stop()\n\t}\n\n\tn.debug(\"Shutdown\")\n}\n\nfunc (n *raft) debug(format string, args ...any) {\n\tif n.dflag {\n\t\tnf := fmt.Sprintf(\"RAFT [%s - %s] %s\", n.id, n.group, format)\n\t\tn.s.Debugf(nf, args...)\n\t}\n}\n\nfunc (n *raft) warn(format string, args ...any) {\n\tnf := fmt.Sprintf(\"RAFT [%s - %s] %s\", n.id, n.group, format)\n\tn.s.RateLimitWarnf(nf, args...)\n}\n\nfunc (n *raft) error(format string, args ...any) {\n\tnf := fmt.Sprintf(\"RAFT [%s - %s] %s\", n.id, n.group, format)\n\tn.s.Errorf(nf, args...)\n}\n\nfunc (n *raft) electTimer() *time.Timer {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.elect\n}\n\nfunc (n *raft) IsObserver() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.observer\n}\n\n// Sets the state to observer only.\nfunc (n *raft) SetObserver(isObserver bool) {\n\tn.setObserver(isObserver, extUndetermined)\n}\n\nfunc (n *raft) setObserver(isObserver bool, extSt extensionState) {\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.setObserverLocked(isObserver, extSt)\n}\n\nfunc (n *raft) setObserverLocked(isObserver bool, extSt extensionState) {\n\twasObserver := n.observer\n\tn.observer = isObserver\n\tn.extSt = extSt\n\n\t// If we're leaving observer state then reset the election timer or\n\t// we might end up waiting for up to the observerModeInterval.\n\tif wasObserver && !isObserver {\n\t\tn.resetElect(randElectionTimeout())\n\t}\n}\n\n// processAppendEntries is called by the Raft state machine when there are\n// new append entries to be committed and sent to the upper state machine.\nfunc (n *raft) processAppendEntries() {\n\tcanProcess := true\n\tif n.isClosed() {\n\t\tn.debug(\"AppendEntry not processing inbound, closed\")\n\t\tcanProcess = false\n\t}\n\tif n.outOfResources() {\n\t\tn.debug(\"AppendEntry not processing inbound, no resources\")\n\t\tcanProcess = false\n\t}\n\t// Always pop the entries, but check if we can process them. If we can't\n\t// then the entries are effectively dropped.\n\taes := n.entry.pop()\n\tif canProcess {\n\t\tfor _, ae := range aes {\n\t\t\tn.processAppendEntry(ae, ae.sub)\n\t\t}\n\t}\n\tn.entry.recycle(&aes)\n}\n\n// runAsFollower is called by run and will block for as long as the node is\n// running in the follower state.\nfunc (n *raft) runAsFollower() {\n\tfor n.State() == Follower {\n\t\telect := n.electTimer()\n\n\t\tselect {\n\t\tcase <-n.entry.ch:\n\t\t\t// New append entries have arrived over the network.\n\t\t\tn.processAppendEntries()\n\t\tcase <-n.s.quitCh:\n\t\t\t// The server is shutting down.\n\t\t\treturn\n\t\tcase <-n.quit:\n\t\t\t// The Raft node is shutting down.\n\t\t\treturn\n\t\tcase <-elect.C:\n\t\t\t// The election timer has fired so we think it's time to call an election.\n\t\t\t// If we are out of resources we just want to stay in this state for the moment.\n\t\t\tif n.outOfResources() {\n\t\t\t\tn.resetElectionTimeoutWithLock()\n\t\t\t\tn.debug(\"Not switching to candidate, no resources\")\n\t\t\t} else if n.IsObserver() {\n\t\t\t\tn.resetElectWithLock(observerModeInterval)\n\t\t\t\tn.debug(\"Not switching to candidate, observer only\")\n\t\t\t} else if n.isCatchingUp() {\n\t\t\t\tn.debug(\"Not switching to candidate, catching up\")\n\t\t\t\t// Check to see if our catchup has stalled.\n\t\t\t\tn.Lock()\n\t\t\t\tif n.catchupStalled() {\n\t\t\t\t\tn.cancelCatchup()\n\t\t\t\t}\n\t\t\t\tn.resetElectionTimeout()\n\t\t\t\tn.Unlock()\n\t\t\t} else {\n\t\t\t\tn.switchToCandidate()\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-n.votes.ch:\n\t\t\t// We're receiving votes from the network, probably because we have only\n\t\t\t// just stepped down and they were already in flight. Ignore them.\n\t\t\tn.debug(\"Ignoring old vote response, we have stepped down\")\n\t\t\tn.votes.popOne()\n\t\tcase <-n.resp.ch:\n\t\t\t// Ignore append entry responses received from before the state change.\n\t\t\tn.resp.drain()\n\t\tcase <-n.prop.ch:\n\t\t\t// Ignore proposals received from before the state change.\n\t\t\tn.prop.drain()\n\t\tcase <-n.reqs.ch:\n\t\t\t// We've just received a vote request from the network.\n\t\t\t// Because of drain() it is possible that we get nil from popOne().\n\t\t\tif voteReq, ok := n.reqs.popOne(); ok {\n\t\t\t\tn.processVoteRequest(voteReq)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Pool for CommittedEntry re-use.\nvar cePool = sync.Pool{\n\tNew: func() any {\n\t\treturn &CommittedEntry{}\n\t},\n}\n\n// CommittedEntry is handed back to the user to apply a commit to their upper layer.\ntype CommittedEntry struct {\n\tIndex uint64\n\tEntries []*Entry\n}\n\n// Create a new CommittedEntry. When the returned entry is no longer needed, it\n// should be returned to the pool by calling ReturnToPool.\nfunc newCommittedEntry(index uint64, entries []*Entry) *CommittedEntry {\n\tce := cePool.Get().(*CommittedEntry)\n\tce.Index, ce.Entries = index, entries\n\treturn ce\n}\n\n// ReturnToPool returns the CommittedEntry to the pool, after which point it is\n// no longer safe to reuse.\nfunc (ce *CommittedEntry) ReturnToPool() {\n\tif ce == nil {\n\t\treturn\n\t}\n\tif len(ce.Entries) > 0 {\n\t\tfor _, e := range ce.Entries {\n\t\t\tentryPool.Put(e)\n\t\t}\n\t}\n\tce.Index, ce.Entries = 0, nil\n\tcePool.Put(ce)\n}\n\n// Pool for Entry re-use.\nvar entryPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &Entry{}\n\t},\n}\n\n// Helper to create new entries. When the returned entry is no longer needed, it\n// should be returned to the entryPool pool.\nfunc newEntry(t EntryType, data []byte) *Entry {\n\tentry := entryPool.Get().(*Entry)\n\tentry.Type, entry.Data = t, data\n\treturn entry\n}\n\n// Pool for appendEntry re-use.\nvar aePool = sync.Pool{\n\tNew: func() any {\n\t\treturn &appendEntry{}\n\t},\n}\n\n// appendEntry is the main struct that is used to sync raft peers.\ntype appendEntry struct {\n\tleader string // The leader that this append entry came from.\n\tterm uint64 // The term when this entry was stored.\n\tcommit uint64 // The commit index of the leader when this append entry was sent.\n\tpterm uint64 // The previous term, for checking consistency.\n\tpindex uint64 // The previous commit index, for checking consistency.\n\tentries []*Entry // Entries to process.\n\t// Below fields are for internal use only:\n\tlterm uint64 // The highest term for catchups only, as the leader understands it. (If lterm=0, use term instead)\n\treply string // Reply subject to respond to once committed.\n\tsub *subscription // The subscription that the append entry came in on.\n\tbuf []byte\n}\n\n// Create a new appendEntry.\nfunc newAppendEntry(leader string, term, commit, pterm, pindex uint64, entries []*Entry) *appendEntry {\n\tae := aePool.Get().(*appendEntry)\n\tae.leader, ae.term, ae.commit, ae.pterm, ae.pindex, ae.entries = leader, term, commit, pterm, pindex, entries\n\tae.lterm, ae.reply, ae.sub, ae.buf = 0, _EMPTY_, nil, nil\n\treturn ae\n}\n\n// Will return this append entry, and its interior entries to their respective pools.\nfunc (ae *appendEntry) returnToPool() {\n\tae.entries, ae.buf, ae.sub, ae.reply = nil, nil, nil, _EMPTY_\n\taePool.Put(ae)\n}\n\n// Pool for proposedEntry re-use.\nvar pePool = sync.Pool{\n\tNew: func() any {\n\t\treturn &proposedEntry{}\n\t},\n}\n\n// Create a new proposedEntry.\nfunc newProposedEntry(entry *Entry, reply string) *proposedEntry {\n\tpe := pePool.Get().(*proposedEntry)\n\tpe.Entry, pe.reply = entry, reply\n\treturn pe\n}\n\n// Will return this proosed entry.\nfunc (pe *proposedEntry) returnToPool() {\n\tpe.Entry, pe.reply = nil, _EMPTY_\n\tpePool.Put(pe)\n}\n\ntype EntryType uint8\n\nconst (\n\tEntryNormal EntryType = iota\n\tEntryOldSnapshot\n\tEntryPeerState\n\tEntryAddPeer\n\tEntryRemovePeer\n\tEntryLeaderTransfer\n\tEntrySnapshot\n\t// EntryCatchup signals an internal type used to signal a Raft-level catchup has started.\n\t// After the catchup completes (or is canceled), a nil entry will be sent to signal this.\n\t// This type of entry is purely internal and not transmitted between peers or stored in the log.\n\tEntryCatchup\n)\n\nfunc (t EntryType) String() string {\n\tswitch t {\n\tcase EntryNormal:\n\t\treturn \"Normal\"\n\tcase EntryOldSnapshot:\n\t\treturn \"OldSnapshot\"\n\tcase EntryPeerState:\n\t\treturn \"PeerState\"\n\tcase EntryAddPeer:\n\t\treturn \"AddPeer\"\n\tcase EntryRemovePeer:\n\t\treturn \"RemovePeer\"\n\tcase EntryLeaderTransfer:\n\t\treturn \"LeaderTransfer\"\n\tcase EntrySnapshot:\n\t\treturn \"Snapshot\"\n\t}\n\treturn fmt.Sprintf(\"Unknown [%d]\", uint8(t))\n}\n\ntype Entry struct {\n\tType EntryType\n\tData []byte\n}\n\nfunc (e *Entry) ChangesMembership() bool {\n\tswitch e.Type {\n\tcase EntryAddPeer, EntryRemovePeer:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\nfunc (ae *appendEntry) String() string {\n\treturn fmt.Sprintf(\"&{leader:%s term:%d commit:%d pterm:%d pindex:%d entries: %d}\",\n\t\tae.leader, ae.term, ae.commit, ae.pterm, ae.pindex, len(ae.entries))\n}\n\nconst appendEntryBaseLen = idLen + 4*8 + 2\n\nfunc (ae *appendEntry) encode(b []byte) ([]byte, error) {\n\tif ll := len(ae.leader); ll != idLen && ll != 0 {\n\t\treturn nil, errLeaderLen\n\t}\n\tif len(ae.entries) > math.MaxUint16 {\n\t\treturn nil, errTooManyEntries\n\t}\n\n\tvar elen uint64\n\tfor _, e := range ae.entries {\n\t\t// MaxInt32 instead of MaxUint32 deliberate here to stop int\n\t\t// overflow on 32-bit platforms, still gives us ~2GB limit.\n\t\tulen := uint64(len(e.Data))\n\t\tif ulen > math.MaxInt32 {\n\t\t\treturn nil, errBadAppendEntry\n\t\t}\n\t\telen += ulen + 1 + 4 // 1 is type, 4 is for size.\n\t}\n\t// Uvarint for lterm can be a maximum 10 bytes for a uint64.\n\tvar _lterm [10]byte\n\tlterm := _lterm[:binary.PutUvarint(_lterm[:], ae.lterm)]\n\ttlen := appendEntryBaseLen + elen + uint64(len(lterm))\n\n\tvar buf []byte\n\tif uint64(cap(b)) >= tlen {\n\t\tbuf = b[:idLen]\n\t} else {\n\t\tbuf = make([]byte, idLen, tlen)\n\t}\n\n\tvar le = binary.LittleEndian\n\tcopy(buf[:idLen], ae.leader)\n\tbuf = le.AppendUint64(buf, ae.term)\n\tbuf = le.AppendUint64(buf, ae.commit)\n\tbuf = le.AppendUint64(buf, ae.pterm)\n\tbuf = le.AppendUint64(buf, ae.pindex)\n\tbuf = le.AppendUint16(buf, uint16(len(ae.entries)))\n\tfor _, e := range ae.entries {\n\t\t// The +1 is safe here as we've already checked len(e.Data)\n\t\t// is not greater than MaxInt32, which is less than MaxUint32.\n\t\tbuf = le.AppendUint32(buf, uint32(1+len(e.Data)))\n\t\tbuf = append(buf, byte(e.Type))\n\t\tbuf = append(buf, e.Data...)\n\t}\n\t// This is safe because old nodes will ignore bytes after the\n\t// encoded messages. Nodes that are aware of this will decode\n\t// it correctly.\n\tbuf = append(buf, lterm...)\n\treturn buf, nil\n}\n\n// This can not be used post the wire level callback since we do not copy.\nfunc decodeAppendEntry(msg []byte, sub *subscription, reply string) (*appendEntry, error) {\n\tif len(msg) < appendEntryBaseLen {\n\t\treturn nil, errBadAppendEntry\n\t}\n\n\tvar le = binary.LittleEndian\n\n\tae := newAppendEntry(string(msg[:idLen]), le.Uint64(msg[8:]), le.Uint64(msg[16:]), le.Uint64(msg[24:]), le.Uint64(msg[32:]), nil)\n\tae.reply, ae.sub = reply, sub\n\n\t// Decode Entries.\n\tne, ri := int(le.Uint16(msg[40:])), uint64(42)\n\tfor i, max := 0, uint64(len(msg)); i < ne; i++ {\n\t\tif ri >= max-1 {\n\t\t\treturn nil, errBadAppendEntry\n\t\t}\n\t\tml := uint64(le.Uint32(msg[ri:]))\n\t\tri += 4\n\t\tif ml <= 0 || ri+ml > max {\n\t\t\treturn nil, errBadAppendEntry\n\t\t}\n\t\tentry := newEntry(EntryType(msg[ri]), msg[ri+1:ri+ml])\n\t\tae.entries = append(ae.entries, entry)\n\t\tri += ml\n\t}\n\tif len(msg[ri:]) > 0 {\n\t\tif lterm, n := binary.Uvarint(msg[ri:]); n > 0 {\n\t\t\tae.lterm = lterm\n\t\t}\n\t}\n\tae.buf = msg\n\treturn ae, nil\n}\n\n// Pool for appendEntryResponse re-use.\nvar arPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &appendEntryResponse{}\n\t},\n}\n\n// We want to make sure this does not change from system changing length of syshash.\nconst idLen = 8\nconst appendEntryResponseLen = 24 + 1\n\n// appendEntryResponse is our response to a received appendEntry.\ntype appendEntryResponse struct {\n\tterm uint64\n\tindex uint64\n\tpeer string\n\treply string // internal usage.\n\tsuccess bool\n}\n\n// Create a new appendEntryResponse.\nfunc newAppendEntryResponse(term, index uint64, peer string, success bool) *appendEntryResponse {\n\tar := arPool.Get().(*appendEntryResponse)\n\tar.term, ar.index, ar.peer, ar.success = term, index, peer, success\n\t// Always empty out.\n\tar.reply = _EMPTY_\n\treturn ar\n}\n\nfunc (ar *appendEntryResponse) encode(b []byte) []byte {\n\tvar buf []byte\n\tif cap(b) >= appendEntryResponseLen {\n\t\tbuf = b[:appendEntryResponseLen]\n\t} else {\n\t\tbuf = make([]byte, appendEntryResponseLen)\n\t}\n\tvar le = binary.LittleEndian\n\tle.PutUint64(buf[0:], ar.term)\n\tle.PutUint64(buf[8:], ar.index)\n\tcopy(buf[16:16+idLen], ar.peer)\n\tif ar.success {\n\t\tbuf[24] = 1\n\t} else {\n\t\tbuf[24] = 0\n\t}\n\treturn buf[:appendEntryResponseLen]\n}\n\n// Track all peers we may have ever seen to use an string interns for appendEntryResponse decoding.\nvar peers sync.Map\n\nfunc decodeAppendEntryResponse(msg []byte) *appendEntryResponse {\n\tif len(msg) != appendEntryResponseLen {\n\t\treturn nil\n\t}\n\tvar le = binary.LittleEndian\n\tar := arPool.Get().(*appendEntryResponse)\n\tar.term = le.Uint64(msg[0:])\n\tar.index = le.Uint64(msg[8:])\n\n\tpeer, ok := peers.Load(string(msg[16 : 16+idLen]))\n\tif !ok {\n\t\t// We missed so store inline here.\n\t\tpeer = string(msg[16 : 16+idLen])\n\t\tpeers.Store(peer, peer)\n\t}\n\tar.peer = peer.(string)\n\tar.success = msg[24] == 1\n\treturn ar\n}\n\n// Called when a remove peer proposal has been forwarded\nfunc (n *raft) handleForwardedRemovePeerProposal(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\tn.debug(\"Received forwarded remove peer proposal: %q\", msg)\n\n\tif len(msg) != idLen {\n\t\tn.warn(\"Received invalid peer name for remove proposal: %q\", msg)\n\t\treturn\n\t}\n\n\tn.RLock()\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader || !n.leaderState.Load() {\n\t\tn.debug(\"Ignoring forwarded peer removal proposal, not leader\")\n\t\tn.RUnlock()\n\t\treturn\n\t}\n\t// Error if we had a previous write error.\n\tif werr := n.werr; werr != nil {\n\t\tn.RUnlock()\n\t\treturn\n\t}\n\tif n.membChangeIndex > 0 {\n\t\tn.debug(\"Ignoring forwarded peer removal proposal, membership changing\")\n\t\tn.RUnlock()\n\t\treturn\n\t}\n\tprop := n.prop\n\tn.RUnlock()\n\n\t// Need to copy since this is underlying client/route buffer.\n\tpeer := copyBytes(msg)\n\tprop.push(newProposedEntry(newEntry(EntryRemovePeer, peer), reply))\n}\n\n// Called when a peer has forwarded a proposal.\nfunc (n *raft) handleForwardedProposal(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\t// Need to copy since this is underlying client/route buffer.\n\tmsg = copyBytes(msg)\n\n\tn.RLock()\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader || !n.leaderState.Load() {\n\t\tn.debug(\"Ignoring forwarded proposal, not leader\")\n\t\tn.RUnlock()\n\t\treturn\n\t}\n\tprop, werr := n.prop, n.werr\n\tn.RUnlock()\n\n\t// Ignore if we have had a write error previous.\n\tif werr != nil {\n\t\treturn\n\t}\n\n\tprop.push(newProposedEntry(newEntry(EntryNormal, msg), reply))\n}\n\n// Adds peer with the given id to our membership,\n// and adjusts cluster size and quorum accordingly.\n// Lock should be held.\nfunc (n *raft) addPeer(peer string) {\n\t// If we were on the removed list reverse that here.\n\tif n.removed != nil {\n\t\tdelete(n.removed, peer)\n\t}\n\n\tif lp, ok := n.peers[peer]; !ok {\n\t\t// We are not tracking this one automatically so we need\n\t\t// to bump cluster size.\n\t\tn.peers[peer] = &lps{time.Time{}, 0, true}\n\t} else {\n\t\t// Mark as added.\n\t\tlp.kp = true\n\t}\n\n\t// Adjust cluster size and quorum if needed.\n\tn.adjustClusterSizeAndQuorum()\n\t// Write out our new state.\n\tn.writePeerState(&peerState{n.peerNames(), n.csz, n.extSt})\n}\n\n// Remove the peer with the given id from our membership,\n// and adjusts cluster size and quorum accordingly.\n// Lock should be held.\nfunc (n *raft) removePeer(peer string) {\n\tif n.removed == nil {\n\t\tn.removed = map[string]time.Time{}\n\t}\n\tn.removed[peer] = time.Now()\n\tif _, ok := n.peers[peer]; ok {\n\t\tdelete(n.peers, peer)\n\t\tn.adjustClusterSizeAndQuorum()\n\t\tn.writePeerState(&peerState{n.peerNames(), n.csz, n.extSt})\n\t}\n}\n\n// Build and send appendEntry request for the given entry that changes\n// membership (EntryAddPeer / EntryRemovePeer).\n// Returns true if the entry made it to the WAL and was sent to the followers\nfunc (n *raft) sendMembershipChange(e *Entry) bool {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\t// Only makes sense to call this with entries that change membership.\n\t// Also, ignore if we're already changing membership.\n\tif !e.ChangesMembership() || n.membChangeIndex > 0 {\n\t\treturn false\n\t}\n\n\t// Set to the index where we will store the membership change.\n\t// It needs to be before we send, since if we're cluster size 1 we try to commit immediately.\n\tn.membChangeIndex = n.pindex + 1\n\terr := n.sendAppendEntryLocked([]*Entry{e}, true)\n\tif err != nil {\n\t\tn.membChangeIndex = 0\n\t\treturn false\n\t}\n\n\tif e.Type == EntryAddPeer {\n\t\tn.addPeer(string(e.Data))\n\t}\n\n\tif e.Type == EntryRemovePeer {\n\t\tn.removePeer(string(e.Data))\n\t\tif n.csz == 1 {\n\t\t\tn.tryCommit(n.pindex)\n\t\t\treturn true\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (n *raft) runAsLeader() {\n\tif n.State() == Closed {\n\t\treturn\n\t}\n\n\tn.Lock()\n\tpsubj, rpsubj := n.psubj, n.rpsubj\n\n\t// For forwarded proposals, both normal and remove peer proposals.\n\tfsub, err := n.subscribe(psubj, n.handleForwardedProposal)\n\tif err != nil {\n\t\tn.warn(\"Error subscribing to forwarded proposals: %v\", err)\n\t\tn.stepdownLocked(noLeader)\n\t\tn.Unlock()\n\t\treturn\n\t}\n\trpsub, err := n.subscribe(rpsubj, n.handleForwardedRemovePeerProposal)\n\tif err != nil {\n\t\tn.warn(\"Error subscribing to forwarded remove peer proposals: %v\", err)\n\t\tn.unsubscribe(fsub)\n\t\tn.stepdownLocked(noLeader)\n\t\tn.Unlock()\n\t\treturn\n\t}\n\n\t// Cleanup our subscription when we leave.\n\tdefer func() {\n\t\tn.Lock()\n\t\tn.unsubscribe(fsub)\n\t\tn.unsubscribe(rpsub)\n\t\tn.Unlock()\n\t}()\n\tn.Unlock()\n\n\thb := time.NewTicker(hbInterval)\n\tdefer hb.Stop()\n\n\tlq := time.NewTicker(lostQuorumCheck)\n\tdefer lq.Stop()\n\n\tfor n.State() == Leader {\n\t\tselect {\n\t\tcase <-n.s.quitCh:\n\t\t\treturn\n\t\tcase <-n.quit:\n\t\t\treturn\n\t\tcase <-n.resp.ch:\n\t\t\tars := n.resp.pop()\n\t\t\tfor _, ar := range ars {\n\t\t\t\tn.processAppendEntryResponse(ar)\n\t\t\t}\n\t\t\tn.resp.recycle(&ars)\n\t\tcase <-n.prop.ch:\n\t\t\tconst maxBatch = 256 * 1024\n\t\t\tconst maxEntries = 512\n\t\t\tvar entries []*Entry\n\n\t\t\tes, sz := n.prop.pop(), 0\n\t\t\tfor _, b := range es {\n\t\t\t\tif b.ChangesMembership() {\n\t\t\t\t\tn.sendMembershipChange(b.Entry)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tentries = append(entries, b.Entry)\n\t\t\t\t// Increment size.\n\t\t\t\tsz += len(b.Data) + 1\n\t\t\t\t// If below thresholds go ahead and send.\n\t\t\t\tif sz < maxBatch && len(entries) < maxEntries {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tn.sendAppendEntry(entries)\n\t\t\t\t// Reset our sz and entries.\n\t\t\t\t// We need to re-create `entries` because there is a reference\n\t\t\t\t// to it in the node's pae map.\n\t\t\t\tsz, entries = 0, nil\n\t\t\t}\n\t\t\tif len(entries) > 0 {\n\t\t\t\tn.sendAppendEntry(entries)\n\t\t\t}\n\t\t\t// Respond to any proposals waiting for a confirmation.\n\t\t\tfor _, pe := range es {\n\t\t\t\tif pe.reply != _EMPTY_ {\n\t\t\t\t\tn.sendReply(pe.reply, nil)\n\t\t\t\t}\n\t\t\t\tpe.returnToPool()\n\t\t\t}\n\t\t\tn.prop.recycle(&es)\n\n\t\tcase <-hb.C:\n\t\t\tif n.notActive() {\n\t\t\t\tn.sendHeartbeat()\n\t\t\t}\n\t\tcase <-lq.C:\n\t\t\tif n.lostQuorum() {\n\t\t\t\tn.stepdown(noLeader)\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-n.votes.ch:\n\t\t\t// Because of drain() it is possible that we get nil from popOne().\n\t\t\tvresp, ok := n.votes.popOne()\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif vresp.term > n.Term() {\n\t\t\t\tn.stepdown(noLeader)\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-n.reqs.ch:\n\t\t\t// Because of drain() it is possible that we get nil from popOne().\n\t\t\tif voteReq, ok := n.reqs.popOne(); ok {\n\t\t\t\tn.processVoteRequest(voteReq)\n\t\t\t}\n\t\tcase <-n.entry.ch:\n\t\t\tn.processAppendEntries()\n\t\t}\n\t}\n}\n\n// Quorum reports the quorum status. Will be called on former leaders.\nfunc (n *raft) Quorum() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\n\tnc := 0\n\tfor id, peer := range n.peers {\n\t\tif id == n.id || time.Since(peer.ts) < lostQuorumInterval {\n\t\t\tif nc++; nc >= n.qn {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (n *raft) lostQuorum() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.lostQuorumLocked()\n}\n\nfunc (n *raft) lostQuorumLocked() bool {\n\t// In order to avoid false positives that can happen in heavily loaded systems\n\t// make sure nothing is queued up that we have not processed yet.\n\t// Also make sure we let any scale up actions settle before deciding.\n\tif n.resp.len() != 0 || (!n.lsut.IsZero() && time.Since(n.lsut) < lostQuorumInterval) {\n\t\treturn false\n\t}\n\n\tnc := 0\n\tfor id, peer := range n.peers {\n\t\tif id == n.id || time.Since(peer.ts) < lostQuorumInterval {\n\t\t\tif nc++; nc >= n.qn {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// Check for being not active in terms of sending entries.\n// Used in determining if we need to send a heartbeat.\nfunc (n *raft) notActive() bool {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn time.Since(n.active) > hbInterval\n}\n\n// Return our current term.\nfunc (n *raft) Term() uint64 {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.term\n}\n\n// Lock should be held.\nfunc (n *raft) loadFirstEntry() (ae *appendEntry, err error) {\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\treturn n.loadEntry(state.FirstSeq)\n}\n\nfunc (n *raft) runCatchup(ar *appendEntryResponse, indexUpdatesQ *ipQueue[uint64]) {\n\tn.RLock()\n\ts, reply := n.s, n.areply\n\tpeer, subj, term, pterm, last := ar.peer, ar.reply, n.term, n.pterm, n.pindex\n\tleader := n.State() == Leader // Grab while holding lock, to not race.\n\tn.RUnlock()\n\n\tdefer s.grWG.Done()\n\tdefer arPool.Put(ar)\n\n\tdefer func() {\n\t\tn.Lock()\n\t\tdelete(n.progress, peer)\n\t\tif len(n.progress) == 0 {\n\t\t\tn.progress = nil\n\t\t}\n\t\t// Check if this is a new peer and if so go ahead and propose adding them.\n\t\t_, exists := n.peers[peer]\n\t\tn.Unlock()\n\t\tif !exists {\n\t\t\tn.debug(\"Catchup done for %q, will add into peers\", peer)\n\t\t\tn.ProposeAddPeer(peer)\n\t\t}\n\t\tindexUpdatesQ.unregister()\n\t}()\n\n\tif !leader {\n\t\tn.debug(\"Canceling catchup for %q, not leader anymore\", peer)\n\t\treturn\n\t}\n\tn.debug(\"Running catchup for %q [%d:%d] to [%d:%d]\", peer, ar.term, ar.index, pterm, last)\n\n\tconst maxOutstanding = 2 * 1024 * 1024 // 2MB for now.\n\tnext, total, om := uint64(0), 0, make(map[uint64]int)\n\n\tsendNext := func() bool {\n\t\tfor total <= maxOutstanding {\n\t\t\tnext++\n\t\t\tif next > last {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tae, err := n.loadEntry(next)\n\t\t\tif err != nil {\n\t\t\t\tif err != ErrStoreEOF {\n\t\t\t\t\tn.warn(\"Got an error loading %d index: %v\", next, err)\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t}\n\t\t\t// Re-encode with the lterm if needed\n\t\t\tif ae.lterm != term {\n\t\t\t\tae.lterm = term\n\t\t\t\tif ae.buf, err = ae.encode(ae.buf[:0]); err != nil {\n\t\t\t\t\tn.warn(\"Got an error re-encoding append entry: %v\", err)\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Update our tracking total.\n\t\t\tom[next] = len(ae.buf)\n\t\t\ttotal += len(ae.buf)\n\t\t\tn.sendRPC(subj, reply, ae.buf)\n\t\t}\n\t\treturn false\n\t}\n\n\tconst activityInterval = 2 * time.Second\n\ttimeout := time.NewTimer(activityInterval)\n\tdefer timeout.Stop()\n\n\tstepCheck := time.NewTicker(100 * time.Millisecond)\n\tdefer stepCheck.Stop()\n\n\t// Run as long as we are leader and still not caught up.\n\tfor n.State() == Leader {\n\t\tselect {\n\t\tcase <-n.s.quitCh:\n\t\t\treturn\n\t\tcase <-n.quit:\n\t\t\treturn\n\t\tcase <-stepCheck.C:\n\t\t\tif n.State() != Leader {\n\t\t\t\tn.debug(\"Catching up canceled, no longer leader\")\n\t\t\t\treturn\n\t\t\t}\n\t\tcase <-timeout.C:\n\t\t\tn.debug(\"Catching up for %q stalled\", peer)\n\t\t\treturn\n\t\tcase <-indexUpdatesQ.ch:\n\t\t\tif index, ok := indexUpdatesQ.popOne(); ok {\n\t\t\t\t// Update our activity timer.\n\t\t\t\ttimeout.Reset(activityInterval)\n\t\t\t\t// Update outstanding total.\n\t\t\t\ttotal -= om[index]\n\t\t\t\tdelete(om, index)\n\t\t\t\tif next == 0 {\n\t\t\t\t\tnext = index\n\t\t\t\t}\n\t\t\t\t// Check if we are done.\n\t\t\t\tif index > last || sendNext() {\n\t\t\t\t\tn.debug(\"Finished catching up\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (n *raft) sendSnapshotToFollower(subject string) (uint64, error) {\n\tsnap, err := n.loadLastSnapshot()\n\tif err != nil {\n\t\t// We need to stepdown here when this happens.\n\t\tn.stepdownLocked(noLeader)\n\t\treturn 0, err\n\t}\n\t// Go ahead and send the snapshot and peerstate here as first append entry to the catchup follower.\n\tae := n.buildAppendEntry([]*Entry{{EntrySnapshot, snap.data}, {EntryPeerState, snap.peerstate}})\n\tae.pterm, ae.pindex = snap.lastTerm, snap.lastIndex\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\n\tfpIndex := state.FirstSeq - 1\n\tif snap.lastIndex < fpIndex && state.FirstSeq != 0 {\n\t\tsnap.lastIndex = fpIndex\n\t\tae.pindex = fpIndex\n\t}\n\n\tencoding, err := ae.encode(nil)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn.sendRPC(subject, n.areply, encoding)\n\treturn snap.lastIndex, nil\n}\n\nfunc (n *raft) catchupFollower(ar *appendEntryResponse) {\n\tn.debug(\"Being asked to catch up follower: %q\", ar.peer)\n\tn.Lock()\n\tif n.progress == nil {\n\t\tn.progress = make(map[string]*ipQueue[uint64])\n\t} else if q, ok := n.progress[ar.peer]; ok {\n\t\tn.debug(\"Will cancel existing entry for catching up %q\", ar.peer)\n\t\tdelete(n.progress, ar.peer)\n\t\tq.push(n.pindex)\n\t}\n\n\t// Check to make sure we have this entry.\n\tstart := ar.index + 1\n\tvar state StreamState\n\tn.wal.FastState(&state)\n\n\tif start < state.FirstSeq || (state.Msgs == 0 && start <= state.LastSeq) {\n\t\tn.debug(\"Need to send snapshot to follower\")\n\t\tif lastIndex, err := n.sendSnapshotToFollower(ar.reply); err != nil {\n\t\t\tn.error(\"Error sending snapshot to follower [%s]: %v\", ar.peer, err)\n\t\t\tn.Unlock()\n\t\t\tarPool.Put(ar)\n\t\t\treturn\n\t\t} else {\n\t\t\tstart = lastIndex + 1\n\t\t\t// If no other entries, we can just return here.\n\t\t\tif state.Msgs == 0 || start > state.LastSeq {\n\t\t\t\tn.debug(\"Finished catching up\")\n\t\t\t\tn.Unlock()\n\t\t\t\tarPool.Put(ar)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tn.debug(\"Snapshot sent, reset first catchup entry to %d\", lastIndex)\n\t\t}\n\t}\n\n\tae, err := n.loadEntry(start)\n\tif err != nil {\n\t\tn.warn(\"Request from follower for entry at index [%d] errored for state %+v - %v\", start, state, err)\n\t\tif err == ErrStoreEOF {\n\t\t\t// If we are here we are seeing a request for an item beyond our state, meaning we should stepdown.\n\t\t\tn.stepdownLocked(noLeader)\n\t\t\tn.Unlock()\n\t\t\tarPool.Put(ar)\n\t\t\treturn\n\t\t}\n\t\tae, err = n.loadFirstEntry()\n\t}\n\tif err != nil || ae == nil {\n\t\tn.warn(\"Could not find a starting entry for catchup request: %v\", err)\n\t\t// If we are here we are seeing a request for an item we do not have, meaning we should stepdown.\n\t\t// This is possible on a reset of our WAL but the other side has a snapshot already.\n\t\t// If we do not stepdown this can cycle.\n\t\tn.stepdownLocked(noLeader)\n\t\tn.Unlock()\n\t\tarPool.Put(ar)\n\t\treturn\n\t}\n\tif ae.pindex != ar.index || ae.pterm != ar.term {\n\t\tn.debug(\"Our first entry [%d:%d] does not match request from follower [%d:%d]\", ae.pterm, ae.pindex, ar.term, ar.index)\n\t}\n\t// Create a queue for delivering updates from responses.\n\tindexUpdates := newIPQueue[uint64](n.s, fmt.Sprintf(\"[ACC:%s] RAFT '%s' indexUpdates\", n.accName, n.group))\n\tindexUpdates.push(ae.pindex)\n\tn.progress[ar.peer] = indexUpdates\n\tn.wg.Add(1)\n\tn.Unlock()\n\tn.s.startGoRoutine(func() {\n\t\tdefer n.wg.Done()\n\t\tn.runCatchup(ar, indexUpdates)\n\t})\n}\n\nfunc (n *raft) loadEntry(index uint64) (*appendEntry, error) {\n\tvar smp StoreMsg\n\tsm, err := n.wal.LoadMsg(index, &smp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn decodeAppendEntry(sm.msg, nil, _EMPTY_)\n}\n\n// applyCommit will update our commit index and apply the entry to the apply queue.\n// lock should be held.\nfunc (n *raft) applyCommit(index uint64) error {\n\tif n.State() == Closed {\n\t\treturn errNodeClosed\n\t}\n\tif index <= n.commit {\n\t\tn.debug(\"Ignoring apply commit for %d, already processed\", index)\n\t\treturn nil\n\t}\n\n\tif n.State() == Leader {\n\t\tdelete(n.acks, index)\n\t}\n\n\tae := n.pae[index]\n\tif ae == nil {\n\t\tif index < n.papplied {\n\t\t\treturn nil\n\t\t}\n\t\tvar err error\n\t\tif ae, err = n.loadEntry(index); err != nil {\n\t\t\tif err != ErrStoreClosed && err != ErrStoreEOF {\n\t\t\t\tn.warn(\"Got an error loading %d index: %v - will reset\", index, err)\n\t\t\t\tif n.State() == Leader {\n\t\t\t\t\tn.stepdownLocked(n.selectNextLeader())\n\t\t\t\t}\n\t\t\t\t// Reset and cancel any catchup.\n\t\t\t\tn.resetWAL()\n\t\t\t\tn.cancelCatchup()\n\t\t\t}\n\t\t\treturn errEntryLoadFailed\n\t\t}\n\t} else {\n\t\tdefer delete(n.pae, index)\n\t}\n\n\tn.commit = index\n\tae.buf = nil\n\tvar committed []*Entry\n\n\tdefer func() {\n\t\t// Pass to the upper layers if we have normal entries. It is\n\t\t// entirely possible that 'committed' might be an empty slice here,\n\t\t// which will happen if we've processed updates inline (like peer\n\t\t// states). In which case the upper layer will just call down with\n\t\t// Applied() with no further action.\n\t\tn.apply.push(newCommittedEntry(index, committed))\n\t\t// Place back in the pool.\n\t\tae.returnToPool()\n\t}()\n\n\tfor _, e := range ae.entries {\n\t\tswitch e.Type {\n\t\tcase EntryNormal:\n\t\t\tcommitted = append(committed, e)\n\t\tcase EntryOldSnapshot:\n\t\t\t// For old snapshots in our WAL.\n\t\t\tcommitted = append(committed, newEntry(EntrySnapshot, e.Data))\n\t\tcase EntrySnapshot:\n\t\t\tcommitted = append(committed, e)\n\t\t\t// If we have no snapshot, install the leader's snapshot as our own.\n\t\t\tif len(ae.entries) == 1 && n.snapfile == _EMPTY_ && ae.commit > 0 {\n\t\t\t\tn.installSnapshot(&snapshot{\n\t\t\t\t\tlastTerm: ae.pterm,\n\t\t\t\t\tlastIndex: ae.commit,\n\t\t\t\t\tpeerstate: encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt}),\n\t\t\t\t\tdata: e.Data,\n\t\t\t\t})\n\t\t\t}\n\t\tcase EntryPeerState:\n\t\t\tif n.State() != Leader {\n\t\t\t\tif ps, err := decodePeerState(e.Data); err == nil {\n\t\t\t\t\tn.processPeerState(ps)\n\t\t\t\t}\n\t\t\t}\n\t\tcase EntryAddPeer:\n\t\t\tnewPeer := string(e.Data)\n\t\t\tn.debug(\"Added peer %q\", newPeer)\n\n\t\t\t// Store our peer in our global peer map for all peers.\n\t\t\tpeers.LoadOrStore(newPeer, newPeer)\n\n\t\t\tn.addPeer(newPeer)\n\n\t\t\t// We pass these up as well.\n\t\t\tcommitted = append(committed, e)\n\n\t\t\t// We are done with this membership change\n\t\t\tn.membChangeIndex = 0\n\n\t\tcase EntryRemovePeer:\n\t\t\tpeer := string(e.Data)\n\t\t\tn.debug(\"Removing peer %q\", peer)\n\n\t\t\tn.removePeer(peer)\n\n\t\t\t// Remove from string intern map.\n\t\t\tpeers.Delete(peer)\n\n\t\t\t// We pass these up as well.\n\t\t\tcommitted = append(committed, e)\n\n\t\t\t// We are done with this membership change\n\t\t\tn.membChangeIndex = 0\n\n\t\t\t// If this is us and we are the leader signal the caller\n\t\t\t// to attempt to stepdown.\n\t\t\tif peer == n.id && n.State() == Leader {\n\t\t\t\treturn errNodeRemoved\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Check if there is a quorum for the given index, and if\n// so, commit the corresponding entry.\n// Return true if the index was committed, false otherwise.\n// Lock should be held.\nfunc (n *raft) tryCommit(index uint64) (bool, error) {\n\tacks := len(n.acks[index])\n\t// Count the leader if it's still part of membership\n\tif n.peers[n.ID()] != nil {\n\t\tacks += 1\n\t}\n\tif acks < n.qn {\n\t\treturn false, nil\n\t}\n\t// We have a quorum\n\tfor i := n.commit + 1; i <= index; i++ {\n\t\tif err := n.applyCommit(i); err != nil {\n\t\t\tif err != errNodeClosed && err != errNodeRemoved {\n\t\t\t\tn.error(\"Got an error applying commit for %d: %v\", i, err)\n\t\t\t}\n\t\t\treturn false, err\n\t\t}\n\t}\n\treturn true, nil\n}\n\n// Used to track a success response. Returns true if the\n// response was tracked, false if the response was ignored\n// (the response is old, the index is already committed, ...)\n// Lock should be held.\nfunc (n *raft) trackResponse(ar *appendEntryResponse) bool {\n\t// Check state under lock, we might not be leader anymore.\n\tif n.State() != Leader {\n\t\treturn false\n\t}\n\n\tps := n.peers[ar.peer]\n\n\t// Update peer's last index.\n\tif ps != nil && ar.index > ps.li {\n\t\tps.li = ar.index\n\t}\n\n\t// If we are tracking this peer as a catchup follower, update that here.\n\tif indexUpdateQ := n.progress[ar.peer]; indexUpdateQ != nil {\n\t\tindexUpdateQ.push(ar.index)\n\t}\n\n\t// Ignore items already committed, or skip if this is not about an entry that matches our current term.\n\tif ar.index <= n.commit || ar.term != n.term {\n\t\tassert.AlwaysOrUnreachable(ar.term <= n.term, \"Raft response term mismatch\", map[string]any{\n\t\t\t\"n.accName\": n.accName,\n\t\t\t\"n.group\": n.group,\n\t\t\t\"n.id\": n.id,\n\t\t\t\"n.term\": n.term,\n\t\t\t\"ar.term\": ar.term,\n\t\t})\n\t\treturn false\n\t}\n\n\t// Not a peer, can't count this message towards quorum\n\tif ps == nil {\n\t\treturn false\n\t}\n\n\t// Keep track of the response\n\tresults := n.acks[ar.index]\n\tif results == nil {\n\t\tresults = make(map[string]struct{})\n\t\tn.acks[ar.index] = results\n\t}\n\tresults[ar.peer] = struct{}{}\n\n\treturn true\n}\n\n// Used to adjust cluster size and peer count based on added official peers.\n// lock should be held.\nfunc (n *raft) adjustClusterSizeAndQuorum() {\n\tpcsz, ncsz := n.csz, 0\n\tfor _, peer := range n.peers {\n\t\tif peer.kp {\n\t\t\tncsz++\n\t\t}\n\t}\n\tn.csz = ncsz\n\tn.qn = n.csz/2 + 1\n\n\tif ncsz > pcsz {\n\t\tn.debug(\"Expanding our clustersize: %d -> %d\", pcsz, ncsz)\n\t\tn.lsut = time.Now()\n\t} else if ncsz < pcsz {\n\t\tn.debug(\"Decreasing our clustersize: %d -> %d\", pcsz, ncsz)\n\t\tif n.State() == Leader {\n\t\t\tgo n.sendHeartbeat()\n\t\t}\n\t}\n\tif ncsz != pcsz {\n\t\tn.recreateInternalSubsLocked()\n\t}\n}\n\n// Track interactions with this peer.\nfunc (n *raft) trackPeer(peer string) error {\n\tn.Lock()\n\tvar needPeerAdd, isRemoved bool\n\tvar rts time.Time\n\tif n.removed != nil {\n\t\trts, isRemoved = n.removed[peer]\n\t\t// Removed peers can rejoin after timeout.\n\t\tif isRemoved && time.Since(rts) >= peerRemoveTimeout {\n\t\t\tisRemoved = false\n\t\t}\n\t}\n\tif n.State() == Leader {\n\t\tif lp, ok := n.peers[peer]; !ok || !lp.kp {\n\t\t\t// Check if this peer had been removed previously.\n\t\t\tneedPeerAdd = !isRemoved\n\t\t}\n\t}\n\tif ps := n.peers[peer]; ps != nil {\n\t\tps.ts = time.Now()\n\t}\n\tn.Unlock()\n\n\tif needPeerAdd {\n\t\tn.ProposeAddPeer(peer)\n\t}\n\treturn nil\n}\n\nfunc (n *raft) runAsCandidate() {\n\tn.Lock()\n\t// Drain old responses.\n\tn.votes.drain()\n\tn.Unlock()\n\n\t// Send out our request for votes.\n\tn.requestVote()\n\n\t// We vote for ourselves.\n\tn.votes.push(&voteResponse{term: n.term, peer: n.ID(), granted: true})\n\n\tvotes := map[string]struct{}{}\n\temptyVotes := map[string]struct{}{}\n\n\tfor n.State() == Candidate {\n\t\telect := n.electTimer()\n\t\tselect {\n\t\tcase <-n.entry.ch:\n\t\t\tn.processAppendEntries()\n\t\tcase <-n.resp.ch:\n\t\t\t// Ignore append entry responses received from before the state change.\n\t\t\tn.resp.drain()\n\t\tcase <-n.prop.ch:\n\t\t\t// Ignore proposals received from before the state change.\n\t\t\tn.prop.drain()\n\t\tcase <-n.s.quitCh:\n\t\t\treturn\n\t\tcase <-n.quit:\n\t\t\treturn\n\t\tcase <-elect.C:\n\t\t\tn.switchToCandidate()\n\t\t\treturn\n\t\tcase <-n.votes.ch:\n\t\t\t// Because of drain() it is possible that we get nil from popOne().\n\t\t\tvresp, ok := n.votes.popOne()\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tn.RLock()\n\t\t\tnterm := n.term\n\t\t\tcsz := n.csz\n\t\t\tn.RUnlock()\n\n\t\t\tif vresp.granted && nterm == vresp.term {\n\t\t\t\t// only track peers that would be our followers\n\t\t\t\tn.trackPeer(vresp.peer)\n\t\t\t\tif !vresp.empty {\n\t\t\t\t\tvotes[vresp.peer] = struct{}{}\n\t\t\t\t} else {\n\t\t\t\t\temptyVotes[vresp.peer] = struct{}{}\n\t\t\t\t}\n\t\t\t\tif n.wonElection(len(votes)) {\n\t\t\t\t\t// Become LEADER if we have won and gotten a quorum with everyone we should hear from.\n\t\t\t\t\tn.switchToLeader()\n\t\t\t\t\treturn\n\t\t\t\t} else if len(votes)+len(emptyVotes) == csz {\n\t\t\t\t\t// Become LEADER if we've got voted in by ALL servers.\n\t\t\t\t\t// We couldn't get quorum based on just our normal votes.\n\t\t\t\t\t// But, we have heard from the full cluster, and some servers came up empty.\n\t\t\t\t\t// We know for sure we have the most up-to-date log.\n\t\t\t\t\tn.switchToLeader()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t} else if vresp.term > nterm {\n\t\t\t\t// if we observe a bigger term, we should start over again or risk forming a quorum fully knowing\n\t\t\t\t// someone with a better term exists. This is even the right thing to do if won == true.\n\t\t\t\tn.Lock()\n\t\t\t\tn.debug(\"Stepping down from candidate, detected higher term: %d vs %d\", vresp.term, n.term)\n\t\t\t\tn.term = vresp.term\n\t\t\t\tn.vote = noVote\n\t\t\t\tn.writeTermVote()\n\t\t\t\tn.lxfer = false\n\t\t\t\tn.stepdownLocked(noLeader)\n\t\t\t\tn.Unlock()\n\t\t\t}\n\t\tcase <-n.reqs.ch:\n\t\t\t// Because of drain() it is possible that we get nil from popOne().\n\t\t\tif voteReq, ok := n.reqs.popOne(); ok {\n\t\t\t\tn.processVoteRequest(voteReq)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// handleAppendEntry handles an append entry from the wire. This function\n// is an internal callback from the \"asubj\" append entry subscription.\nfunc (n *raft) handleAppendEntry(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\tmsg = copyBytes(msg)\n\tif ae, err := decodeAppendEntry(msg, sub, reply); err == nil {\n\t\t// Push to the new entry channel. From here one of the worker\n\t\t// goroutines (runAsLeader, runAsFollower, runAsCandidate) will\n\t\t// pick it up.\n\t\tn.entry.push(ae)\n\t} else {\n\t\tn.warn(\"AppendEntry failed to be placed on internal channel: corrupt entry\")\n\t}\n}\n\n// cancelCatchup will stop an in-flight catchup by unsubscribing from the\n// catchup subscription.\n// Lock should be held.\nfunc (n *raft) cancelCatchup() {\n\tn.debug(\"Canceling catchup subscription since we are now up to date\")\n\n\tif n.catchup != nil && n.catchup.sub != nil {\n\t\tn.unsubscribe(n.catchup.sub)\n\t}\n\tn.cancelCatchupSignal()\n\tn.catchup = nil\n}\n\n// catchupStalled will try to determine if we are stalled. This is called\n// on a new entry from our leader.\n// Lock should be held.\nfunc (n *raft) catchupStalled() bool {\n\tif n.catchup == nil {\n\t\treturn false\n\t}\n\tif n.catchup.pindex == n.pindex {\n\t\treturn time.Since(n.catchup.active) > 2*time.Second\n\t}\n\tn.catchup.pindex = n.pindex\n\tn.catchup.active = time.Now()\n\treturn false\n}\n\n// createCatchup will create the state needed to track a catchup as it\n// runs. It then creates a unique inbox for this catchup and subscribes\n// to it. The remote side will stream entries to that subject.\n// Lock should be held.\nfunc (n *raft) createCatchup(ae *appendEntry) string {\n\t// Cleanup any old ones.\n\tif n.catchup != nil && n.catchup.sub != nil {\n\t\tn.unsubscribe(n.catchup.sub)\n\t}\n\t// Snapshot term and index.\n\tn.catchup = &catchupState{\n\t\tcterm: ae.pterm,\n\t\tcindex: ae.pindex,\n\t\tpterm: n.pterm,\n\t\tpindex: n.pindex,\n\t\tactive: time.Now(),\n\t}\n\tinbox := n.newCatchupInbox()\n\tsub, _ := n.subscribe(inbox, n.handleAppendEntry)\n\tn.catchup.sub = sub\n\treturn inbox\n}\n\n// Lock should be held.\nfunc (n *raft) sendCatchupSignal() {\n\tif n.catchup == nil || n.catchup.signal {\n\t\treturn\n\t}\n\tn.catchup.signal = true\n\t// Signal to the upper layer that the following entries are catchup entries, up until the nil guard.\n\tn.apply.push(newCommittedEntry(0, []*Entry{{EntryCatchup, nil}}))\n}\n\n// Lock should be held.\nfunc (n *raft) cancelCatchupSignal() {\n\tif n.catchup == nil || !n.catchup.signal {\n\t\treturn\n\t}\n\t// Send nil entry to signal the upper layers we are done catching up.\n\tn.apply.push(nil)\n}\n\n// Truncate our WAL and reset.\n// Lock should be held.\nfunc (n *raft) truncateWAL(term, index uint64) {\n\tn.debug(\"Truncating and repairing WAL to Term %d Index %d\", term, index)\n\n\tif term == 0 && index == 0 {\n\t\tif n.commit > 0 {\n\t\t\tn.warn(\"Resetting WAL state\")\n\t\t} else {\n\t\t\tn.debug(\"Clearing WAL state (no commits)\")\n\t\t}\n\t}\n\tif index < n.commit {\n\t\tassert.Unreachable(\"WAL truncate lost commits\", map[string]any{\n\t\t\t\"n.accName\": n.accName,\n\t\t\t\"n.group\": n.group,\n\t\t\t\"n.id\": n.id,\n\t\t\t\"term\": term,\n\t\t\t\"index\": index,\n\t\t\t\"commit\": n.commit,\n\t\t\t\"applied\": n.applied,\n\t\t})\n\t}\n\n\tdefer func() {\n\t\t// Check to see if we invalidated any snapshots that might have held state\n\t\t// from the entries we are truncating.\n\t\tif snap, _ := n.loadLastSnapshot(); snap != nil && snap.lastIndex > index {\n\t\t\tos.Remove(n.snapfile)\n\t\t\tn.snapfile = _EMPTY_\n\t\t}\n\t\t// Make sure to reset commit and applied if above\n\t\tif n.commit > n.pindex {\n\t\t\tn.commit = n.pindex\n\t\t}\n\t\tif n.processed > n.commit {\n\t\t\tn.processed = n.commit\n\t\t}\n\t\tif n.applied > n.processed {\n\t\t\tn.applied = n.processed\n\t\t}\n\t\t// Refresh bytes count after truncate.\n\t\tvar state StreamState\n\t\tn.wal.FastState(&state)\n\t\tn.bytes = state.Bytes\n\t}()\n\n\tif err := n.wal.Truncate(index); err != nil {\n\t\tn.warn(\"Error truncating WAL: %v\", err)\n\t\tn.setWriteErrLocked(err)\n\t\treturn\n\t}\n\t// Set after we know we have truncated properly.\n\tn.pterm, n.pindex = term, index\n\n\t// Check if we're truncating an uncommitted membership change.\n\tif n.membChangeIndex > 0 && n.membChangeIndex > index {\n\t\tn.membChangeIndex = 0\n\t}\n}\n\n// Reset our WAL. This is equivalent to truncating all data from the log.\n// Lock should be held.\nfunc (n *raft) resetWAL() {\n\tn.truncateWAL(0, 0)\n}\n\n// Lock should be held\nfunc (n *raft) updateLeader(newLeader string) {\n\twasLeader := n.leader == n.id\n\tn.leader = newLeader\n\tn.hasleader.Store(newLeader != _EMPTY_)\n\tif !n.pleader.Load() && newLeader != noLeader {\n\t\tn.pleader.Store(true)\n\t\t// If we were preferred to become the first leader, but didn't end up successful.\n\t\t// Ensure to call lead change. When scaling from R1 to R3 we've optimized for a scale up\n\t\t// not flipping leader/non-leader/leader status if the leader remains the same. But we need to\n\t\t// correct that if the first leader turns out to be different.\n\t\tif n.maybeLeader {\n\t\t\tn.maybeLeader = false\n\t\t\tif n.id != newLeader {\n\t\t\t\tn.updateLeadChange(false)\n\t\t\t}\n\t\t}\n\t}\n\t// Reset last seen timestamps and indices.\n\t// If we are (or were) the leader we track(ed) everyone, and don't reset.\n\t// But if we're a follower we only track the leader, and reset all others.\n\tif newLeader != n.id && !wasLeader {\n\t\tfor peer, ps := range n.peers {\n\t\t\t// Always reset last replicated index.\n\t\t\tps.li = 0\n\t\t\tif peer == newLeader {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Only reset the last seen timestamp if this peer is not the leader.\n\t\t\tps.ts = time.Time{}\n\t\t}\n\t}\n}\n\n// processAppendEntry will process an appendEntry. This is called either\n// during recovery or from processAppendEntries when there are new entries\n// to be committed.\nfunc (n *raft) processAppendEntry(ae *appendEntry, sub *subscription) {\n\tn.Lock()\n\t// Don't reset here if we have been asked to assume leader position.\n\tif !n.lxfer {\n\t\tn.resetElectionTimeout()\n\t}\n\n\t// Just return if closed or we had previous write error.\n\tif n.State() == Closed || n.werr != nil {\n\t\tn.Unlock()\n\t\treturn\n\t}\n\n\t// Scratch buffer for responses.\n\tvar scratch [appendEntryResponseLen]byte\n\tarbuf := scratch[:]\n\n\t// Grab term from append entry. But if leader explicitly defined its term, use that instead.\n\t// This is required during catchup if the leader catches us up on older items from previous terms.\n\t// While still allowing us to confirm they're matching our highest known term.\n\tlterm := ae.term\n\tif ae.lterm != 0 {\n\t\tlterm = ae.lterm\n\t}\n\n\t// Are we receiving from another leader.\n\tif n.State() == Leader {\n\t\tif lterm >= n.term {\n\t\t\t// If the append entry term is newer than the current term, erase our\n\t\t\t// vote.\n\t\t\tif lterm > n.term {\n\t\t\t\tn.term = lterm\n\t\t\t\tn.vote = noVote\n\t\t\t\tn.writeTermVote()\n\t\t\t} else {\n\t\t\t\tassert.Unreachable(\n\t\t\t\t\t\"Two leaders using the same term\",\n\t\t\t\t\tmap[string]any{\n\t\t\t\t\t\t\"n.accName\": n.accName,\n\t\t\t\t\t\t\"n.group\": n.group,\n\t\t\t\t\t\t\"n.id\": n.id,\n\t\t\t\t\t\t\"n.term\": n.term,\n\t\t\t\t\t\t\"ae.leader\": ae.leader,\n\t\t\t\t\t\t\"ae.term\": ae.term,\n\t\t\t\t\t\t\"ae.lterm\": ae.lterm,\n\t\t\t\t\t})\n\t\t\t}\n\t\t\tn.debug(\"Received append entry from another leader, stepping down to %q\", ae.leader)\n\t\t\tn.stepdownLocked(ae.leader)\n\t\t} else {\n\t\t\t// Let them know we are the leader.\n\t\t\tar := newAppendEntryResponse(n.term, n.pindex, n.id, false)\n\t\t\tn.debug(\"AppendEntry ignoring old term from another leader\")\n\t\t\tn.sendRPC(ae.reply, _EMPTY_, ar.encode(arbuf))\n\t\t\tarPool.Put(ar)\n\t\t\tn.Unlock()\n\t\t\treturn\n\t\t}\n\t}\n\n\t// If we received an append entry as a candidate then it would appear that\n\t// another node has taken on the leader role already, so we should convert\n\t// to a follower of that node instead.\n\tif n.State() == Candidate {\n\t\t// If we have a leader in the current term or higher, we should stepdown,\n\t\t// write the term and vote if the term of the request is higher.\n\t\tif lterm >= n.term {\n\t\t\t// If the append entry term is newer than the current term, erase our\n\t\t\t// vote.\n\t\t\tif lterm > n.term {\n\t\t\t\tn.term = lterm\n\t\t\t\tn.vote = noVote\n\t\t\t\tn.writeTermVote()\n\t\t\t}\n\t\t\tn.debug(\"Received append entry in candidate state from %q, converting to follower\", ae.leader)\n\t\t\tn.stepdownLocked(ae.leader)\n\t\t}\n\t}\n\n\t// Catching up state.\n\tcatchingUp := n.catchup != nil\n\t// Is this a new entry? New entries will be delivered on the append entry\n\t// sub, rather than a catch-up sub.\n\tisNew := sub != nil && sub == n.aesub\n\n\t// If we are/were catching up ignore old catchup subs, but only if catching up from an older server\n\t// that doesn't send the leader term when catching up or if we would truncate as a result.\n\t// We can reject old catchups from newer subs later, just by checking the append entry is on the correct term.\n\tif !isNew && sub != nil && (ae.lterm == 0 || ae.pindex < n.pindex) && (!catchingUp || sub != n.catchup.sub) {\n\t\tn.Unlock()\n\t\tn.debug(\"AppendEntry ignoring old entry from previous catchup\")\n\t\treturn\n\t}\n\n\t// If this term is greater than ours.\n\tif lterm > n.term {\n\t\tn.term = lterm\n\t\tn.vote = noVote\n\t\tif isNew {\n\t\t\tn.writeTermVote()\n\t\t}\n\t\tif n.State() != Follower {\n\t\t\tn.debug(\"Term higher than ours and we are not a follower: %v, stepping down to %q\", n.State(), ae.leader)\n\t\t\tn.stepdownLocked(ae.leader)\n\t\t}\n\t} else if lterm < n.term && sub != nil && (isNew || ae.lterm != 0) {\n\t\t// Anything that's below our expected highest term needs to be rejected.\n\t\t// Unless we're replaying (sub=nil), in which case we'll always continue.\n\t\t// For backward-compatibility we shouldn't reject if we're being caught up by an old server.\n\t\tif !isNew {\n\t\t\tn.debug(\"AppendEntry ignoring old entry from previous catchup\")\n\t\t\tn.Unlock()\n\t\t\treturn\n\t\t}\n\t\tn.debug(\"Rejected AppendEntry from a leader (%s) with term %d which is less than ours\", ae.leader, lterm)\n\t\tar := newAppendEntryResponse(n.term, n.pindex, n.id, false)\n\t\tn.Unlock()\n\t\tn.sendRPC(ae.reply, _EMPTY_, ar.encode(arbuf))\n\t\tarPool.Put(ar)\n\t\treturn\n\t}\n\n\t// Check state if we are catching up.\n\tif catchingUp {\n\t\tif cs := n.catchup; cs != nil && n.pterm >= cs.cterm && n.pindex >= cs.cindex {\n\t\t\t// If we are here we are good, so if we have a catchup pending we can cancel.\n\t\t\tn.cancelCatchup()\n\t\t\t// Reset our notion of catching up.\n\t\t\tcatchingUp = false\n\t\t} else if isNew {\n\t\t\tvar ar *appendEntryResponse\n\t\t\tvar inbox string\n\t\t\t// Check to see if we are stalled. If so recreate our catchup state and resend response.\n\t\t\tif n.catchupStalled() {\n\t\t\t\tn.debug(\"Catchup may be stalled, will request again\")\n\t\t\t\tinbox = n.createCatchup(ae)\n\t\t\t\tar = newAppendEntryResponse(n.pterm, n.pindex, n.id, false)\n\t\t\t}\n\t\t\tn.Unlock()\n\t\t\tif ar != nil {\n\t\t\t\tn.sendRPC(ae.reply, inbox, ar.encode(arbuf))\n\t\t\t\tarPool.Put(ar)\n\t\t\t}\n\t\t\t// Ignore new while catching up or replaying.\n\t\t\treturn\n\t\t}\n\t}\n\n\tif isNew && n.leader != ae.leader && n.State() == Follower {\n\t\tn.debug(\"AppendEntry updating leader to %q\", ae.leader)\n\t\tn.updateLeader(ae.leader)\n\t\tn.writeTermVote()\n\t\tn.resetElectionTimeout()\n\t\tn.updateLeadChange(false)\n\t}\n\n\t// Track leader directly\n\t// But, do so after all consistency checks so we don't track an old leader.\n\tif isNew && ae.leader != noLeader && ae.leader == n.leader {\n\t\tif ps := n.peers[ae.leader]; ps != nil {\n\t\t\tps.ts = time.Now()\n\t\t}\n\t}\n\n\tif ae.pterm != n.pterm || ae.pindex != n.pindex {\n\t\t// Check if this is a lower or equal index than what we were expecting.\n\t\tif ae.pindex <= n.pindex {\n\t\t\tn.debug(\"AppendEntry detected pindex less than/equal to ours: [%d:%d] vs [%d:%d]\", ae.pterm, ae.pindex, n.pterm, n.pindex)\n\t\t\tvar success bool\n\n\t\t\tif ae.pindex < n.commit {\n\t\t\t\t// If we have already committed this entry, just mark success.\n\t\t\t\tsuccess = true\n\t\t\t\tn.debug(\"AppendEntry pindex %d below commit %d, marking success\", ae.pindex, n.commit)\n\t\t\t} else if eae, _ := n.loadEntry(ae.pindex); eae == nil {\n\t\t\t\t// If terms are equal, and we are not catching up, we have simply already processed this message.\n\t\t\t\t// This can happen on server restarts based on timings of snapshots.\n\t\t\t\tif ae.pterm == n.pterm && isNew {\n\t\t\t\t\tsuccess = true\n\t\t\t\t\tn.debug(\"AppendEntry pindex %d already processed, marking success\", ae.pindex)\n\t\t\t\t} else if ae.pindex == n.pindex {\n\t\t\t\t\t// Check if only our terms do not match here.\n\t\t\t\t\t// Make sure pterms match and we take on the leader's.\n\t\t\t\t\t// This prevents constant spinning.\n\t\t\t\t\tn.truncateWAL(ae.pterm, ae.pindex)\n\t\t\t\t} else {\n\t\t\t\t\tsnap, err := n.loadLastSnapshot()\n\t\t\t\t\tif err == nil && snap.lastIndex == ae.pindex && snap.lastTerm == ae.pterm {\n\t\t\t\t\t\t// Entry can't be found, this is normal because we have a snapshot at this index.\n\t\t\t\t\t\t// Truncate back to where we've created the snapshot.\n\t\t\t\t\t\tn.truncateWAL(snap.lastTerm, snap.lastIndex)\n\t\t\t\t\t\t// Only continue if truncation was successful, and we ended up such that we can safely continue.\n\t\t\t\t\t\tif ae.pterm == n.pterm && ae.pindex == n.pindex {\n\t\t\t\t\t\t\tgoto CONTINUE\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Otherwise, something has gone very wrong and we need to reset.\n\t\t\t\t\t\tn.resetWAL()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if eae.term == ae.pterm {\n\t\t\t\t// If terms match we can delete all entries past this one, and then continue storing the current entry.\n\t\t\t\tn.truncateWAL(ae.pterm, ae.pindex)\n\t\t\t\t// Only continue if truncation was successful, and we ended up such that we can safely continue.\n\t\t\t\tif ae.pterm == n.pterm && ae.pindex == n.pindex {\n\t\t\t\t\tgoto CONTINUE\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// If terms mismatched, delete that entry and all others past it.\n\t\t\t\t// Make sure to cancel any catchups in progress.\n\t\t\t\t// Truncate will reset our pterm and pindex. Only do so if we have an entry.\n\t\t\t\tn.truncateWAL(eae.pterm, eae.pindex)\n\t\t\t}\n\t\t\t// Cancel regardless if unsuccessful.\n\t\t\tif !success {\n\t\t\t\tn.cancelCatchup()\n\t\t\t}\n\t\t\t// Intentionally not responding. Otherwise, we could erroneously report \"success\". Reporting\n\t\t\t// non-success is not needed either, and would only be wasting messages.\n\t\t\t// For example, if we got partial catchup, and then the \"real-time\" messages came in very delayed.\n\t\t\t// If we reported \"success\" on those \"real-time\" messages, we'd wrongfully be providing\n\t\t\t// quorum while not having an up-to-date log.\n\t\t\tn.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Check if we are catching up. If we are here we know the leader did not have all of the entries\n\t\t// so make sure this is a snapshot entry. If it is not start the catchup process again since it\n\t\t// means we may have missed additional messages.\n\t\tif catchingUp {\n\t\t\t// This means we already entered into a catchup state but what the leader sent us did not match what we expected.\n\t\t\t// Snapshots and peerstate will always be together when a leader is catching us up in this fashion.\n\t\t\tif len(ae.entries) != 2 || ae.entries[0].Type != EntrySnapshot || ae.entries[1].Type != EntryPeerState {\n\t\t\t\tn.warn(\"Expected first catchup entry to be a snapshot and peerstate, will retry\")\n\t\t\t\tn.cancelCatchup()\n\t\t\t\tn.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif ps, err := decodePeerState(ae.entries[1].Data); err == nil {\n\t\t\t\tn.processPeerState(ps)\n\t\t\t\t// Also need to copy from client's buffer.\n\t\t\t\tae.entries[0].Data = copyBytes(ae.entries[0].Data)\n\t\t\t} else {\n\t\t\t\tn.warn(\"Could not parse snapshot peerstate correctly\")\n\t\t\t\tn.cancelCatchup()\n\t\t\t\tn.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Inherit state from appendEntry with the leader's snapshot.\n\t\t\thadPreviousSnapshot := n.snapfile != _EMPTY_\n\t\t\tn.pindex = ae.pindex\n\t\t\tn.pterm = ae.pterm\n\t\t\tn.commit = ae.pindex\n\n\t\t\tsnap := &snapshot{\n\t\t\t\tlastTerm: n.pterm,\n\t\t\t\tlastIndex: n.pindex,\n\t\t\t\tpeerstate: encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt}),\n\t\t\t\tdata: ae.entries[0].Data,\n\t\t\t}\n\t\t\t// Install the leader's snapshot as our own.\n\t\t\tif err := n.installSnapshot(snap); err != nil {\n\t\t\t\tn.setWriteErrLocked(err)\n\t\t\t\tn.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tn.resetInitializing()\n\n\t\t\tif !hadPreviousSnapshot {\n\t\t\t\t// If the first snapshot we install is received from another server, then we immediately signal\n\t\t\t\t// to the upper-layer it can coalesce catchup entries.\n\t\t\t\tn.sendCatchupSignal()\n\t\t\t}\n\t\t\t// Now send snapshot to upper levels. Only send the snapshot, not the peerstate entry.\n\t\t\tn.apply.push(newCommittedEntry(n.commit, ae.entries[:1]))\n\t\t\tif hadPreviousSnapshot {\n\t\t\t\t// Signal catchup only after we've sent the snapshot. That ensures the upper-layer processes the snapshot\n\t\t\t\t// as-is and can only coalesce other catchup entries after this one.\n\t\t\t\tn.sendCatchupSignal()\n\t\t\t}\n\t\t\tn.Unlock()\n\t\t\treturn\n\t\t}\n\n\t\t// Setup our state for catching up.\n\t\tn.debug(\"AppendEntry did not match [%d:%d] with [%d:%d]\", ae.pterm, ae.pindex, n.pterm, n.pindex)\n\t\tinbox := n.createCatchup(ae)\n\t\tar := newAppendEntryResponse(n.pterm, n.pindex, n.id, false)\n\t\tn.Unlock()\n\t\tn.sendRPC(ae.reply, inbox, ar.encode(arbuf))\n\t\tarPool.Put(ar)\n\t\treturn\n\t}\n\nCONTINUE:\n\t// Save to our WAL if we have entries.\n\tif ae.shouldStore() {\n\t\t// Only store if an original which will have sub != nil\n\t\tif sub != nil {\n\t\t\tif err := n.storeToWAL(ae); err != nil {\n\t\t\t\tif err != ErrStoreClosed {\n\t\t\t\t\tn.warn(\"Error storing entry to WAL: %v\", err)\n\t\t\t\t}\n\t\t\t\tn.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tn.cachePendingEntry(ae)\n\t\t\tn.resetInitializing()\n\t\t} else {\n\t\t\t// This is a replay on startup so just take the appendEntry version.\n\t\t\tn.pterm = ae.term\n\t\t\tn.pindex = ae.pindex + 1\n\t\t}\n\t}\n\n\t// Check to see if we have any related entries to process here.\n\tfor _, e := range ae.entries {\n\t\tswitch e.Type {\n\t\tcase EntryLeaderTransfer:\n\t\t\t// Only process these if they are new, so no replays or catchups.\n\t\t\tif isNew {\n\t\t\t\tmaybeLeader := string(e.Data)\n\t\t\t\t// This is us. We need to check if we can become the leader.\n\t\t\t\tif maybeLeader == n.id {\n\t\t\t\t\t// If not an observer and not paused we are good to go.\n\t\t\t\t\tif !n.observer && !n.paused {\n\t\t\t\t\t\tn.lxfer = true\n\t\t\t\t\t\tn.xferCampaign()\n\t\t\t\t\t} else if n.paused {\n\t\t\t\t\t\t// Here we can become a leader but need to wait for resume of the apply queue.\n\t\t\t\t\t\tn.lxfer = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase EntryAddPeer:\n\t\t\t// When receiving or restoring, mark membership as changing.\n\t\t\t// Set to the index where this entry was stored (pindex is now this entry's index)\n\t\t\tn.membChangeIndex = n.pindex\n\t\t\tif newPeer := string(e.Data); len(newPeer) == idLen {\n\t\t\t\t// Track directly, but wait for commit to be official\n\t\t\t\tif _, ok := n.peers[newPeer]; !ok {\n\t\t\t\t\tn.peers[newPeer] = &lps{time.Time{}, 0, false}\n\t\t\t\t}\n\t\t\t\t// Store our peer in our global peer map for all peers.\n\t\t\t\tpeers.LoadOrStore(newPeer, newPeer)\n\t\t\t}\n\t\tcase EntryRemovePeer:\n\t\t\t// When receiving or restoring, mark membership as changing.\n\t\t\t// Set to the index where this entry was stored (pindex is now this entry's index)\n\t\t\tn.membChangeIndex = n.pindex\n\t\t}\n\t}\n\n\t// Make a copy of these values, as the AppendEntry might be cached and returned to the pool in applyCommit.\n\taeCommit := ae.commit\n\taeReply := ae.reply\n\n\t// Apply anything we need here.\n\tif aeCommit > n.commit {\n\t\t// If we're catching up, we might need to signal that it's okay to potentially coalesce entries from here.\n\t\tif catchingUp {\n\t\t\tn.sendCatchupSignal()\n\t\t}\n\t\tif n.paused {\n\t\t\tn.hcommit = aeCommit\n\t\t\tn.debug(\"Paused, not applying %d\", aeCommit)\n\t\t} else {\n\t\t\tfor index := n.commit + 1; index <= aeCommit; index++ {\n\t\t\t\tif err := n.applyCommit(index); err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Only ever respond to new entries.\n\t// Never respond to catchup messages, because providing quorum based on this is unsafe.\n\t// The only way for the leader to receive \"success\" MUST be through this path.\n\tvar ar *appendEntryResponse\n\tif sub != nil && isNew {\n\t\tar = newAppendEntryResponse(n.pterm, n.pindex, n.id, true)\n\t}\n\tn.Unlock()\n\n\t// Success. Send our response.\n\tif ar != nil {\n\t\tn.sendRPC(aeReply, _EMPTY_, ar.encode(arbuf))\n\t\tarPool.Put(ar)\n\t}\n}\n\n// resetInitializing resets the notion of initializing.\n// If we were scaling up, also leaves observer mode.\n// Lock should be held.\nfunc (n *raft) resetInitializing() {\n\tn.initializing = false\n\tif n.scaleUp {\n\t\tn.scaleUp = false\n\t\tn.setObserverLocked(false, extUndetermined)\n\t}\n}\n\n// processPeerState is called when a peer state entry is received\n// over the wire or when we're updating known peers.\n// Lock should be held.\nfunc (n *raft) processPeerState(ps *peerState) {\n\t// Update our version of peers to that of the leader. Calculate\n\t// the number of nodes needed to establish a quorum.\n\tn.csz = ps.clusterSize\n\tn.qn = n.csz/2 + 1\n\n\told := n.peers\n\tn.peers = make(map[string]*lps)\n\tfor _, peer := range ps.knownPeers {\n\t\tif lp := old[peer]; lp != nil {\n\t\t\tlp.kp = true\n\t\t\tn.peers[peer] = lp\n\t\t} else {\n\t\t\tn.peers[peer] = &lps{time.Time{}, 0, true}\n\t\t}\n\t}\n\tn.debug(\"Update peers from leader to %+v\", n.peers)\n\tn.writePeerState(ps)\n}\n\n// processAppendEntryResponse is called when we receive an append entry\n// response from another node. They will send a confirmation to tell us\n// whether they successfully committed the entry or not.\nfunc (n *raft) processAppendEntryResponse(ar *appendEntryResponse) {\n\tn.trackPeer(ar.peer)\n\n\tif ar.success {\n\t\t// The remote node successfully committed the append entry.\n\t\t// They agree with our leadership and are happy with the state of the log.\n\t\t// In this case ar.term was populated with the remote's pterm. If this matches\n\t\t// our term, we can use it to check for quorum and up our commit.\n\t\tvar err error\n\t\tvar committed bool\n\n\t\tn.Lock()\n\t\tif n.trackResponse(ar) {\n\t\t\tcommitted, err = n.tryCommit(ar.index)\n\t\t}\n\t\tn.Unlock()\n\n\t\t// Leader was peer-removed. Attempt a step-down to\n\t\t// a new leader before shutting down.\n\t\tif err == errNodeRemoved {\n\t\t\tn.StepDown()\n\t\t\tn.Stop()\n\t\t}\n\n\t\t// Send a heartbeat if there is no other message lined\n\t\t// up, so that followers can apply without waiting for\n\t\t// the next message.\n\t\tif committed && n.prop.len() == 0 {\n\t\t\tn.sendHeartbeat()\n\t\t}\n\n\t\tarPool.Put(ar)\n\t} else if ar.reply != _EMPTY_ {\n\t\t// The remote node didn't commit the append entry, and they believe they\n\t\t// are behind and have specified a reply subject, so let's try to catch them up.\n\t\t// In this case ar.term was populated with the remote's pterm.\n\t\tn.catchupFollower(ar)\n\t} else if ar.term > n.term {\n\t\t// The remote node didn't commit the append entry, it looks like\n\t\t// they are on a newer term than we are. Step down.\n\t\t// In this case ar.term was populated with the remote's term.\n\t\tn.Lock()\n\t\tn.term = ar.term\n\t\tn.vote = noVote\n\t\tn.writeTermVote()\n\t\tn.warn(\"Detected another leader with higher term, will stepdown\")\n\t\tn.stepdownLocked(noLeader)\n\t\tn.Unlock()\n\t\tarPool.Put(ar)\n\t} else {\n\t\t// Ignore, but return back to pool.\n\t\tarPool.Put(ar)\n\t}\n}\n\n// handleAppendEntryResponse processes responses to append entries.\nfunc (n *raft) handleAppendEntryResponse(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tar := decodeAppendEntryResponse(msg)\n\tar.reply = reply\n\tn.resp.push(ar)\n}\n\nfunc (n *raft) buildAppendEntry(entries []*Entry) *appendEntry {\n\treturn newAppendEntry(n.id, n.term, n.commit, n.pterm, n.pindex, entries)\n}\n\n// Determine if we should store an entry. This stops us from storing\n// heartbeat messages.\nfunc (ae *appendEntry) shouldStore() bool {\n\treturn ae != nil && len(ae.entries) > 0\n}\n\n// Store our append entry to our WAL.\n// lock should be held.\nfunc (n *raft) storeToWAL(ae *appendEntry) error {\n\tif ae == nil {\n\t\treturn fmt.Errorf(\"raft: Missing append entry for storage\")\n\t}\n\tif n.werr != nil {\n\t\treturn n.werr\n\t}\n\n\tseq, _, err := n.wal.StoreMsg(_EMPTY_, nil, ae.buf, 0)\n\tif err != nil {\n\t\tn.setWriteErrLocked(err)\n\t\treturn err\n\t}\n\n\t// Sanity checking for now.\n\tif index := ae.pindex + 1; index != seq {\n\t\tn.warn(\"Wrong index, ae is %+v, index stored was %d, n.pindex is %d, will reset\", ae, seq, n.pindex)\n\t\tif n.State() == Leader {\n\t\t\tn.stepdownLocked(n.selectNextLeader())\n\t\t}\n\t\t// Reset and cancel any catchup.\n\t\tn.resetWAL()\n\t\tn.cancelCatchup()\n\t\treturn errEntryStoreFailed\n\t}\n\n\tvar sz uint64\n\tif n.wtype == FileStorage {\n\t\tsz = fileStoreMsgSize(_EMPTY_, nil, ae.buf)\n\t} else {\n\t\tsz = memStoreMsgSize(_EMPTY_, nil, ae.buf)\n\t}\n\tn.bytes += sz\n\tn.pterm = ae.term\n\tn.pindex = seq\n\treturn nil\n}\n\nconst (\n\tpaeDropThreshold = 20_000\n\tpaeWarnThreshold = 10_000\n\tpaeWarnModulo = 5_000\n)\n\nfunc (n *raft) sendAppendEntry(entries []*Entry) {\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.sendAppendEntryLocked(entries, true)\n}\n\n// Returns nil if an appendEntry was appended to our WAL and sent to followers,\n// an error otherwise.\nfunc (n *raft) sendAppendEntryLocked(entries []*Entry, checkLeader bool) error {\n\t// Safeguard against sending an append entry right after a stepdown from a different goroutine.\n\t// Specifically done while holding the lock to not race.\n\tif checkLeader && n.State() != Leader {\n\t\tn.debug(\"Not sending append entry, not leader\")\n\t\treturn errNotLeader\n\t}\n\tae := n.buildAppendEntry(entries)\n\n\tvar err error\n\tvar scratch [1024]byte\n\tae.buf, err = ae.encode(scratch[:])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// If we have entries store this in our wal.\n\tshouldStore := ae.shouldStore()\n\tif shouldStore {\n\t\tif err := n.storeToWAL(ae); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tn.active = time.Now()\n\t\tn.cachePendingEntry(ae)\n\t}\n\tn.sendRPC(n.asubj, n.areply, ae.buf)\n\tif !shouldStore {\n\t\tae.returnToPool()\n\t}\n\tif n.csz == 1 {\n\t\tn.tryCommit(n.pindex)\n\t}\n\treturn nil\n}\n\n// cachePendingEntry saves append entries in memory for faster processing during applyCommit.\n// Only save so many however to avoid memory bloat.\nfunc (n *raft) cachePendingEntry(ae *appendEntry) {\n\tif l := len(n.pae); l < paeDropThreshold {\n\t\tn.pae[n.pindex], l = ae, l+1\n\t\tif l >= paeWarnThreshold && l%paeWarnModulo == 0 {\n\t\t\tn.warn(\"%d append entries pending\", len(n.pae))\n\t\t}\n\t} else {\n\t\t// Invalidate cache entry at this index, we might have\n\t\t// stored it previously with a different value.\n\t\tdelete(n.pae, n.pindex)\n\t}\n}\n\ntype extensionState uint16\n\nconst (\n\textUndetermined = extensionState(iota)\n\textExtended\n\textNotExtended\n)\n\ntype peerState struct {\n\tknownPeers []string\n\tclusterSize int\n\tdomainExt extensionState\n}\n\nfunc peerStateBufSize(ps *peerState) int {\n\treturn 4 + 4 + (idLen * len(ps.knownPeers)) + 2\n}\n\nfunc encodePeerState(ps *peerState) []byte {\n\tvar le = binary.LittleEndian\n\tbuf := make([]byte, peerStateBufSize(ps))\n\tle.PutUint32(buf[0:], uint32(ps.clusterSize))\n\tle.PutUint32(buf[4:], uint32(len(ps.knownPeers)))\n\twi := 8\n\tfor _, peer := range ps.knownPeers {\n\t\tcopy(buf[wi:], peer)\n\t\twi += idLen\n\t}\n\tle.PutUint16(buf[wi:], uint16(ps.domainExt))\n\treturn buf\n}\n\nfunc decodePeerState(buf []byte) (*peerState, error) {\n\tif len(buf) < 8 {\n\t\treturn nil, errCorruptPeers\n\t}\n\tvar le = binary.LittleEndian\n\tps := &peerState{clusterSize: int(le.Uint32(buf[0:]))}\n\texpectedPeers := int(le.Uint32(buf[4:]))\n\tbuf = buf[8:]\n\tri := 0\n\tfor i, n := 0, expectedPeers; i < n && ri < len(buf); i++ {\n\t\tps.knownPeers = append(ps.knownPeers, string(buf[ri:ri+idLen]))\n\t\tri += idLen\n\t}\n\tif len(ps.knownPeers) != expectedPeers {\n\t\treturn nil, errCorruptPeers\n\t}\n\tif len(buf[ri:]) >= 2 {\n\t\tps.domainExt = extensionState(le.Uint16(buf[ri:]))\n\t}\n\treturn ps, nil\n}\n\n// Lock should be held.\nfunc (n *raft) peerNames() []string {\n\tvar peers []string\n\tfor name, peer := range n.peers {\n\t\tif peer.kp {\n\t\t\tpeers = append(peers, name)\n\t\t}\n\t}\n\treturn peers\n}\n\nfunc (n *raft) currentPeerState() *peerState {\n\tn.RLock()\n\tps := n.currentPeerStateLocked()\n\tn.RUnlock()\n\treturn ps\n}\n\nfunc (n *raft) currentPeerStateLocked() *peerState {\n\treturn &peerState{n.peerNames(), n.csz, n.extSt}\n}\n\n// sendPeerState will send our current peer state to the cluster.\n// Lock should be held.\nfunc (n *raft) sendPeerState() {\n\tn.sendAppendEntryLocked([]*Entry{{EntryPeerState, encodePeerState(n.currentPeerStateLocked())}}, true)\n}\n\n// Send a heartbeat.\nfunc (n *raft) sendHeartbeat() {\n\tn.sendAppendEntry(nil)\n}\n\ntype voteRequest struct {\n\tterm uint64\n\tlastTerm uint64\n\tlastIndex uint64\n\tcandidate string\n\t// internal only.\n\treply string\n}\n\nconst voteRequestLen = 24 + idLen\n\nfunc (vr *voteRequest) encode() []byte {\n\tvar buf [voteRequestLen]byte\n\tvar le = binary.LittleEndian\n\tle.PutUint64(buf[0:], vr.term)\n\tle.PutUint64(buf[8:], vr.lastTerm)\n\tle.PutUint64(buf[16:], vr.lastIndex)\n\tcopy(buf[24:24+idLen], vr.candidate)\n\n\treturn buf[:voteRequestLen]\n}\n\nfunc decodeVoteRequest(msg []byte, reply string) *voteRequest {\n\tif len(msg) != voteRequestLen {\n\t\treturn nil\n\t}\n\n\tvar le = binary.LittleEndian\n\treturn &voteRequest{\n\t\tterm: le.Uint64(msg[0:]),\n\t\tlastTerm: le.Uint64(msg[8:]),\n\t\tlastIndex: le.Uint64(msg[16:]),\n\t\tcandidate: string(copyBytes(msg[24 : 24+idLen])),\n\t\treply: reply,\n\t}\n}\n\nconst peerStateFile = \"peers.idx\"\n\n// Lock should be held.\nfunc (n *raft) writePeerState(ps *peerState) {\n\tpse := encodePeerState(ps)\n\tif bytes.Equal(n.wps, pse) {\n\t\treturn\n\t}\n\t// Stamp latest and write the peer state file.\n\tn.wps = pse\n\tif err := writePeerState(n.sd, ps); err != nil && !n.isClosed() {\n\t\tn.setWriteErrLocked(err)\n\t\tn.warn(\"Error writing peer state file for %q: %v\", n.group, err)\n\t}\n}\n\n// Writes out our peer state outside of a specific raft context.\nfunc writePeerState(sd string, ps *peerState) error {\n\tpsf := filepath.Join(sd, peerStateFile)\n\tif _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\treturn writeFileWithSync(psf, encodePeerState(ps), defaultFilePerms)\n}\n\nfunc readPeerState(sd string) (ps *peerState, err error) {\n\t<-dios\n\tbuf, err := os.ReadFile(filepath.Join(sd, peerStateFile))\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn decodePeerState(buf)\n}\n\nconst termVoteFile = \"tav.idx\"\nconst termLen = 8 // uint64\nconst termVoteLen = idLen + termLen\n\n// Writes out our term & vote outside of a specific raft context.\nfunc writeTermVote(sd string, wtv []byte) error {\n\tpsf := filepath.Join(sd, termVoteFile)\n\tif _, err := os.Stat(psf); err != nil && !os.IsNotExist(err) {\n\t\treturn err\n\t}\n\treturn writeFileWithSync(psf, wtv, defaultFilePerms)\n}\n\n// readTermVote will read the largest term and who we voted from to stable storage.\n// Lock should be held.\nfunc (n *raft) readTermVote() (term uint64, voted string, err error) {\n\t<-dios\n\tbuf, err := os.ReadFile(filepath.Join(n.sd, termVoteFile))\n\tdios <- struct{}{}\n\n\tif err != nil {\n\t\treturn 0, noVote, err\n\t}\n\tif len(buf) < termLen {\n\t\t// Not enough bytes for the uint64 below, so avoid a panic.\n\t\treturn 0, noVote, nil\n\t}\n\tvar le = binary.LittleEndian\n\tterm = le.Uint64(buf[0:])\n\tif len(buf) < termVoteLen {\n\t\treturn term, noVote, nil\n\t}\n\tvoted = string(buf[8:])\n\treturn term, voted, nil\n}\n\n// Lock should be held.\nfunc (n *raft) setWriteErrLocked(err error) {\n\t// Check if we are closed already.\n\tif n.State() == Closed {\n\t\treturn\n\t}\n\t// Ignore if already set.\n\tif n.werr == err || err == nil {\n\t\treturn\n\t}\n\t// Ignore non-write errors.\n\tif err == ErrStoreClosed ||\n\t\terr == ErrStoreEOF ||\n\t\terr == ErrStoreMsgNotFound ||\n\t\terr == errNoPending ||\n\t\terr == errPartialCache {\n\t\treturn\n\t}\n\t// If this is a not found report but do not disable.\n\tif os.IsNotExist(err) {\n\t\tn.warn(\"Resource not found: %v\", err)\n\t\treturn\n\t}\n\tn.error(\"Critical write error: %v\", err)\n\tn.werr = err\n\tn.shutdown()\n\tassert.Unreachable(\"Raft encountered write error\", map[string]any{\n\t\t\"n.accName\": n.accName,\n\t\t\"n.group\": n.group,\n\t\t\"n.id\": n.id,\n\t\t\"err\": err,\n\t})\n\n\tif isPermissionError(err) {\n\t\tgo n.s.handleWritePermissionError()\n\t}\n\n\tif isOutOfSpaceErr(err) {\n\t\t// For now since this can be happening all under the covers, we will call up and disable JetStream.\n\t\tgo n.s.handleOutOfSpace(nil)\n\t}\n}\n\n// Helper to check if we are closed when we do not hold a lock already.\nfunc (n *raft) isClosed() bool {\n\treturn n.State() == Closed\n}\n\n// GetWriteErr returns the write error (if any).\nfunc (n *raft) GetWriteErr() error {\n\tn.RLock()\n\tdefer n.RUnlock()\n\treturn n.werr\n}\n\n// Capture our write error if any and hold.\nfunc (n *raft) setWriteErr(err error) {\n\tn.Lock()\n\tdefer n.Unlock()\n\tn.setWriteErrLocked(err)\n}\n\n// writeTermVote will record the largest term and who we voted for to stable storage.\n// Lock should be held.\nfunc (n *raft) writeTermVote() {\n\tvar buf [termVoteLen]byte\n\tvar le = binary.LittleEndian\n\tle.PutUint64(buf[0:], n.term)\n\tcopy(buf[8:], n.vote)\n\tb := buf[:8+len(n.vote)]\n\n\t// If the term and vote hasn't changed then don't rewrite to disk.\n\tif bytes.Equal(n.wtv, b) {\n\t\treturn\n\t}\n\t// Stamp latest and write the term & vote file.\n\tn.wtv = b\n\tif err := writeTermVote(n.sd, n.wtv); err != nil && !n.isClosed() {\n\t\t// Clear wtv since we failed.\n\t\tn.wtv = nil\n\t\tn.setWriteErrLocked(err)\n\t\tn.warn(\"Error writing term and vote file for %q: %v\", n.group, err)\n\t}\n}\n\n// voteResponse is a response to a vote request.\ntype voteResponse struct {\n\tterm uint64\n\tpeer string\n\tgranted bool\n\tempty bool // \"Empty vote\", whether this peer's log is empty.\n}\n\nconst voteResponseLen = 8 + 8 + 1\n\nfunc (vr *voteResponse) encode() []byte {\n\tvar buf [voteResponseLen]byte\n\tvar le = binary.LittleEndian\n\tle.PutUint64(buf[0:], vr.term)\n\tcopy(buf[8:], vr.peer)\n\tif vr.granted {\n\t\tbuf[16] |= 1\n\t}\n\tif vr.empty {\n\t\tbuf[16] |= 2\n\t}\n\treturn buf[:voteResponseLen]\n}\n\nfunc decodeVoteResponse(msg []byte) *voteResponse {\n\tif len(msg) != voteResponseLen {\n\t\treturn nil\n\t}\n\tvar le = binary.LittleEndian\n\tvr := &voteResponse{term: le.Uint64(msg[0:]), peer: string(msg[8:16])}\n\tvr.granted = msg[16]&1 != 0\n\tvr.empty = msg[16]&2 != 0\n\treturn vr\n}\n\nfunc (n *raft) handleVoteResponse(sub *subscription, c *client, _ *Account, _, reply string, msg []byte) {\n\tvr := decodeVoteResponse(msg)\n\tn.debug(\"Received a voteResponse %+v\", vr)\n\tif vr == nil {\n\t\tn.error(\"Received malformed vote response for %q\", n.group)\n\t\treturn\n\t}\n\n\tif state := n.State(); state != Candidate && state != Leader {\n\t\tn.debug(\"Ignoring old vote response, we have stepped down\")\n\t\treturn\n\t}\n\n\tn.votes.push(vr)\n}\n\nfunc (n *raft) processVoteRequest(vr *voteRequest) error {\n\t// To simplify calling code, we can possibly pass `nil` to this function.\n\t// If that is the case, does not consider it an error.\n\tif vr == nil {\n\t\treturn nil\n\t}\n\tn.debug(\"Received a voteRequest %+v\", vr)\n\n\tn.Lock()\n\n\tvresp := &voteResponse{n.term, n.id, false, n.pindex == 0}\n\tdefer n.debug(\"Sending a voteResponse %+v -> %q\", vresp, vr.reply)\n\n\t// Ignore if we are newer. This is important so that we don't accidentally process\n\t// votes from a previous term if they were still in flight somewhere.\n\tif vr.term < n.term {\n\t\tn.Unlock()\n\t\tn.sendReply(vr.reply, vresp.encode())\n\t\treturn nil\n\t}\n\n\t// If this is a higher term go ahead and stepdown.\n\tif vr.term > n.term {\n\t\tif n.State() != Follower {\n\t\t\tn.debug(\"Stepping down from %s, detected higher term: %d vs %d\",\n\t\t\t\tstrings.ToLower(n.State().String()), vr.term, n.term)\n\t\t\tn.stepdownLocked(noLeader)\n\t\t}\n\t\tn.cancelCatchup()\n\t\tn.term = vr.term\n\t\tn.vote = noVote\n\t\tn.writeTermVote()\n\t}\n\n\t// Only way we get to yes is through here.\n\tvoteOk := n.vote == noVote || n.vote == vr.candidate\n\n\t// If we have an empty log, but are initializing.\n\tif voteOk && vresp.empty && n.initializing {\n\t\t// Reset notion of having an empty log if we're voting during initialization/scale up.\n\t\t// Ensures they only need quorum, and not need to hear from all servers.\n\t\tvresp.empty = false\n\t}\n\n\t// Other server's log needs to be equal or more up-to-date than ours.\n\tif voteOk && (vr.lastTerm > n.pterm || vr.lastTerm == n.pterm && vr.lastIndex >= n.pindex) {\n\t\tvresp.granted = true\n\t\tn.term = vr.term\n\t\tn.vote = vr.candidate\n\t\tn.writeTermVote()\n\t\tn.resetElectionTimeout()\n\t} else if n.vote == noVote && n.State() != Candidate {\n\t\t// We have a more up-to-date log, and haven't voted yet.\n\t\t// Start campaigning earlier, but only if not candidate already, as that would short-circuit us.\n\t\tn.resetElect(randCampaignTimeout())\n\t}\n\n\t// Term might have changed, make sure response has the most current\n\tvresp.term = n.term\n\n\tn.Unlock()\n\n\tn.sendReply(vr.reply, vresp.encode())\n\n\treturn nil\n}\n\nfunc (n *raft) handleVoteRequest(sub *subscription, c *client, _ *Account, subject, reply string, msg []byte) {\n\tvr := decodeVoteRequest(msg, reply)\n\tif vr == nil {\n\t\tn.error(\"Received malformed vote request for %q\", n.group)\n\t\treturn\n\t}\n\tn.reqs.push(vr)\n}\n\nfunc (n *raft) requestVote() {\n\tn.Lock()\n\tif n.State() != Candidate {\n\t\tn.Unlock()\n\t\treturn\n\t}\n\tn.vote = n.id\n\tn.writeTermVote()\n\tvr := voteRequest{n.term, n.pterm, n.pindex, n.id, _EMPTY_}\n\tsubj, reply := n.vsubj, n.vreply\n\tn.Unlock()\n\n\tn.debug(\"Sending out voteRequest %+v\", vr)\n\n\t// Now send it out.\n\tn.sendRPC(subj, reply, vr.encode())\n}\n\nfunc (n *raft) sendRPC(subject, reply string, msg []byte) {\n\tif n.sq != nil {\n\t\tn.sq.send(subject, reply, nil, msg)\n\t}\n}\n\nfunc (n *raft) sendReply(subject string, msg []byte) {\n\tif n.sq != nil {\n\t\tn.sq.send(subject, _EMPTY_, nil, msg)\n\t}\n}\n\nfunc (n *raft) wonElection(votes int) bool {\n\treturn votes >= n.quorumNeeded()\n}\n\n// Return the quorum size for a given cluster config.\nfunc (n *raft) quorumNeeded() int {\n\tn.RLock()\n\tqn := n.qn\n\tn.RUnlock()\n\treturn qn\n}\n\n// Lock should be held.\nfunc (n *raft) updateLeadChange(isLeader bool) {\n\t// We don't care about values that have not been consumed (transitory states),\n\t// so we dequeue any state that is pending and push the new one.\n\tfor {\n\t\tselect {\n\t\tcase n.leadc <- isLeader:\n\t\t\treturn\n\t\tdefault:\n\t\t\tselect {\n\t\t\tcase <-n.leadc:\n\t\t\tdefault:\n\t\t\t\t// May have been consumed by the \"reader\" go routine, so go back\n\t\t\t\t// to the top of the loop and try to send again.\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (n *raft) switchState(state RaftState) bool {\nretry:\n\tpstate := n.State()\n\tif pstate == Closed {\n\t\treturn false\n\t}\n\n\t// Set our state. If something else has changed our state\n\t// then retry, this will either be a Stop or Delete call.\n\tif !n.state.CompareAndSwap(int32(pstate), int32(state)) {\n\t\tgoto retry\n\t}\n\n\t// Reset the election timer.\n\tn.resetElectionTimeout()\n\n\tvar leadChange bool\n\tif pstate == Leader && state != Leader {\n\t\tleadChange = true\n\t\tn.updateLeadChange(false)\n\t\t// Drain the append entry response and proposal queues.\n\t\tn.resp.drain()\n\t\tn.prop.drain()\n\t} else if state == Leader && pstate != Leader {\n\t\t// Don't updateLeadChange here, it will be done in switchToLeader or after initial messages are applied.\n\t\tleadChange = true\n\t\tif len(n.pae) > 0 {\n\t\t\tn.pae = make(map[uint64]*appendEntry)\n\t\t}\n\t}\n\n\tn.writeTermVote()\n\treturn leadChange\n}\n\nconst (\n\tnoLeader = _EMPTY_\n\tnoVote = _EMPTY_\n)\n\nfunc (n *raft) switchToFollower(leader string) {\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tn.switchToFollowerLocked(leader)\n}\n\nfunc (n *raft) switchToFollowerLocked(leader string) {\n\tif n.State() == Closed {\n\t\treturn\n\t}\n\n\tn.debug(\"Switching to follower\")\n\n\tn.aflr = 0\n\tn.leaderState.Store(false)\n\tn.leaderSince.Store(nil)\n\tn.lxfer = false\n\n\t// Reset acks, we can't assume acks from a previous term are still valid in another term.\n\tif len(n.acks) > 0 {\n\t\tn.acks = make(map[uint64]map[string]struct{})\n\t}\n\tn.updateLeader(leader)\n\tn.switchState(Follower)\n}\n\nfunc (n *raft) switchToCandidate() {\n\tif n.State() == Closed {\n\t\treturn\n\t}\n\n\tn.Lock()\n\tdefer n.Unlock()\n\n\t// If we are catching up or are in observer mode we can not switch.\n\t// Avoid petitioning to become leader if we're behind on applies.\n\tif n.observer || n.paused || n.processed < n.commit {\n\t\tn.resetElect(minElectionTimeout / 4)\n\t\treturn\n\t}\n\n\tif n.State() != Candidate {\n\t\tn.debug(\"Switching to candidate\")\n\t} else {\n\t\tif n.lostQuorumLocked() && time.Since(n.llqrt) > 20*time.Second {\n\t\t\t// We signal to the upper layers such that can alert on quorum lost.\n\t\t\tn.updateLeadChange(false)\n\t\t\tn.llqrt = time.Now()\n\t\t}\n\t}\n\t// Increment the term.\n\tn.term++\n\tn.vote = noVote\n\t// Clear current Leader.\n\tn.updateLeader(noLeader)\n\tn.switchState(Candidate)\n}\n\nfunc (n *raft) switchToLeader() {\n\tif n.State() == Closed {\n\t\treturn\n\t}\n\n\tn.Lock()\n\tdefer n.Unlock()\n\n\tn.debug(\"Switching to leader\")\n\n\tn.lxfer = false\n\tn.updateLeader(n.id)\n\tn.switchState(Leader)\n\n\t// To send out our initial peer state.\n\t// In our implementation this is equivalent to sending a NOOP-entry upon becoming leader.\n\t// Wait for this message (and potentially more) to be applied.\n\t// It's important to wait signaling we're leader if we're not up-to-date yet, as that\n\t// would mean we're in a consistent state compared with the previous leader.\n\tn.sendPeerState()\n\tn.aflr = n.pindex\n}\n" }, { "path": "server/raft_chain_of_blocks_helpers_test.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Raft Chain of Blocks (RCOB) is a replicated state machine on top of Raft used to test the latter.\n// Each value (\"block\") proposed to raft is a simple array of bytes (stateless, can be small or large).\n// Each replica state consists of a hash of all blocks delivered so far.\n// This makes it easy to check consistency over time.\n// If all replicas deliver the same blocks in the same order (the whole point of Raft!), then the N-th hash on each\n// replica should be identical. This invariant is easy to verify in test settings, because if at any point a replica\n// delivered (hashed) a different value, then all subsequent hashes will diverge from the rest.\n\npackage server\n\nimport (\n\t\"encoding\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"hash\"\n\t\"hash/crc32\"\n\t\"math/rand\"\n\t\"sync\"\n)\n\n// Static options (but may be useful to tweak when debugging)\nvar RCOBOptions = struct {\n\tverbose bool\n\t// Proposed block [data] size is random between 1 and maxBlockSize\n\tmaxBlockSize int\n\t// If set to true, the state machine will not take snapshots while in recovery\n\t// (corresponding to the 'ready' state machine member variable)\n\tsafeSnapshots bool\n}{\n\tfalse,\n\t10240,\n\ttrue,\n}\n\n// Simple state machine on top of Raft.\n// Each value delivered is hashed, N-th value hash should match on all replicas or something went wrong.\n// If even a single value in the chain of hash differs on one replica, hashes will diverge.\ntype RCOBStateMachine struct {\n\tsync.Mutex\n\ts *Server\n\tn RaftNode\n\tcfg *RaftConfig\n\twg sync.WaitGroup\n\tleader bool\n\tproposalSequence uint64\n\trng *rand.Rand\n\thash hash.Hash\n\tblocksApplied uint64\n\tblocksAppliedSinceSnapshot uint64\n\tstopped bool\n\tready bool\n}\n\nfunc (sm *RCOBStateMachine) waitGroup() *sync.WaitGroup {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\treturn &sm.wg\n}\n\n// RCOBBlock is the structure of values replicated through Raft.\n// Block data is a random array of bytes.\n// Additional proposer metadata is present for debugging and for further ordering invariant checks (but not included\n// in the hash).\ntype RCOBBlock struct {\n\tProposer string\n\tProposerSequence uint64\n\tData []byte\n}\n\n// logDebug is for fine-grained event logging, useful for debugging but default off\nfunc (sm *RCOBStateMachine) logDebug(format string, args ...any) {\n\tif RCOBOptions.verbose {\n\t\tfmt.Printf(\"[\"+sm.s.Name()+\" (\"+sm.n.ID()+\")] \"+format+\"\\n\", args...)\n\t}\n}\n\nfunc (sm *RCOBStateMachine) server() *Server {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\treturn sm.s\n}\n\nfunc (sm *RCOBStateMachine) node() RaftNode {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\treturn sm.n\n}\n\nfunc (sm *RCOBStateMachine) propose(data []byte) {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\tif !sm.ready {\n\t\tsm.logDebug(\"Refusing to propose during recovery\")\n\t}\n\terr := sm.n.ForwardProposal(data)\n\tif err != nil {\n\t\tsm.logDebug(\"block proposal error: %s\", err)\n\t}\n}\n\nfunc (sm *RCOBStateMachine) applyEntry(ce *CommittedEntry) {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\tif ce == nil {\n\t\t// A nil entry signals that the previous recovery backlog is over\n\t\tsm.logDebug(\"Recovery complete\")\n\t\tsm.ready = true\n\t\treturn\n\t}\n\tsm.logDebug(\"Apply entries #%d (%d entries)\", ce.Index, len(ce.Entries))\n\tfor _, entry := range ce.Entries {\n\t\tif entry.Type == EntryNormal {\n\t\t\tsm.applyBlock(entry.Data)\n\t\t} else if entry.Type == EntrySnapshot {\n\t\t\tsm.loadSnapshot(entry.Data)\n\t\t} else if entry.Type == EntryCatchup {\n\t\t\t// Ignore.\n\t\t} else {\n\t\t\tpanic(fmt.Sprintf(\"[%s] unknown entry type: %s\", sm.s.Name(), entry.Type))\n\t\t}\n\t}\n\t// Signal to the node that entries were applied\n\tsm.n.Applied(ce.Index)\n}\n\nfunc (sm *RCOBStateMachine) leaderChange(isLeader bool) {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\tif sm.leader && !isLeader {\n\t\tsm.logDebug(\"Leader change: no longer leader\")\n\t} else if sm.leader && isLeader {\n\t\tsm.logDebug(\"Elected leader while already leader\")\n\t} else if !sm.leader && isLeader {\n\t\tsm.logDebug(\"Leader change: i am leader\")\n\t} else {\n\t\tsm.logDebug(\"Leader change\")\n\t}\n\tsm.leader = isLeader\n\tif isLeader != sm.n.Leader() {\n\t\tsm.logDebug(\"⚠️ Leader state out of sync with underlying node\")\n\t}\n}\n\nfunc (sm *RCOBStateMachine) stop() {\n\tn, wg := sm.node(), sm.waitGroup()\n\tn.Stop()\n\tn.WaitForStop()\n\twg.Wait()\n\n\t// Clear state, on restart it will be recovered from snapshot or peers\n\tsm.Lock()\n\tdefer sm.Unlock()\n\n\tsm.stopped = true\n\tsm.blocksApplied = 0\n\tsm.hash.Reset()\n\tsm.leader = false\n\tsm.logDebug(\"Stopped\")\n}\n\nfunc (sm *RCOBStateMachine) restart() {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\n\tsm.logDebug(\"Restarting\")\n\n\tsm.stopped = false\n\tsm.ready = false\n\tif sm.n.State() != Closed {\n\t\treturn\n\t}\n\n\t// The filestore is stopped as well, so need to extract the parts to recreate it.\n\trn := sm.n.(*raft)\n\tfs := rn.wal.(*fileStore)\n\n\tvar err error\n\tsm.cfg.Log, err = newFileStore(fs.fcfg, fs.cfg.StreamConfig)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tsm.n, err = sm.s.startRaftNode(globalAccountName, sm.cfg, pprofLabels{})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\t// Finally restart the driver.\n\tgo smLoop(sm)\n}\n\nfunc (sm *RCOBStateMachine) proposeBlock() {\n\t// Keep track how many blocks this replica proposed\n\tsm.proposalSequence += 1\n\n\t// Create a block\n\tblockSize := 1 + sm.rng.Intn(RCOBOptions.maxBlockSize)\n\tblock := RCOBBlock{\n\t\tProposer: sm.s.Name(),\n\t\tProposerSequence: sm.proposalSequence,\n\t\tData: make([]byte, blockSize),\n\t}\n\n\t// Data is random bytes\n\tsm.rng.Read(block.Data)\n\n\t// Serialize block to JSON\n\tblockData, err := json.Marshal(block)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"serialization error: %s\", err))\n\t}\n\n\tsm.logDebug(\n\t\t\"Proposing block <%s, %d, [%dB]>\",\n\t\tblock.Proposer,\n\t\tblock.ProposerSequence,\n\t\tlen(block.Data),\n\t)\n\n\t// Propose block (may fail, and that's ok)\n\tsm.propose(blockData)\n}\n\nfunc (sm *RCOBStateMachine) applyBlock(data []byte) {\n\t// Deserialize block\n\tvar block RCOBBlock\n\terr := json.Unmarshal(data, &block)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"deserialization error: %s\", err))\n\t}\n\n\tsm.logDebug(\"Applying block <%s, %d>\", block.Proposer, block.ProposerSequence)\n\n\t// Hash the data on top of the existing running hash\n\tn, err := sm.hash.Write(block.Data)\n\tif n != len(block.Data) {\n\t\tpanic(fmt.Sprintf(\"unexpected checksum written %d data block size: %d\", n, len(block.Data)))\n\t} else if err != nil {\n\t\tpanic(fmt.Sprintf(\"checksum error: %s\", err))\n\t}\n\n\t// Track numbers of blocks applied\n\tsm.blocksApplied += 1\n\tsm.blocksAppliedSinceSnapshot += 1\n\n\tsm.logDebug(\"Hash after %d blocks: %X \", sm.blocksApplied, sm.hash.Sum(nil))\n}\n\nfunc (sm *RCOBStateMachine) getCurrentHash() (bool, uint64, string) {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\n\t// Return running, the number of blocks applied and the current running hash\n\treturn !sm.stopped, sm.blocksApplied, fmt.Sprintf(\"%X\", sm.hash.Sum(nil))\n}\n\n// RCOBSnapshot structure for RCOB snapshots\n// Consists of the hash (32b) after BlocksCount blocks were hashed.\n// TODO: Could start storing last N blocks to make snapshot larger and exercise more code paths.\n// (a big chunk of random data would also do the trick)\ntype RCOBSnapshot struct {\n\tSourceNode string\n\tHashData []byte\n\tBlocksCount uint64\n}\n\n// createSnapshot asks the state machine to create a snapshot.\n// The request may be ignored if no blocks were applied since the last snapshot.\nfunc (sm *RCOBStateMachine) createSnapshot() {\n\tsm.Lock()\n\tdefer sm.Unlock()\n\n\tif sm.blocksAppliedSinceSnapshot == 0 {\n\t\tsm.logDebug(\"Skip snapshot, no new entries\")\n\t\treturn\n\t}\n\n\tif RCOBOptions.safeSnapshots && !sm.ready {\n\t\tsm.logDebug(\"Skip snapshot, still recovering\")\n\t\treturn\n\t}\n\n\tsm.logDebug(\n\t\t\"Snapshot (with %d blocks applied, %d since last snapshot)\",\n\t\tsm.blocksApplied,\n\t\tsm.blocksAppliedSinceSnapshot,\n\t)\n\n\t// Serialize the internal state of the hash block\n\tserializedHash, err := sm.hash.(encoding.BinaryMarshaler).MarshalBinary()\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"failed to marshal hash: %s\", err))\n\t}\n\n\t// Create snapshot\n\tsnapshot := RCOBSnapshot{\n\t\tSourceNode: fmt.Sprintf(\"%s (%s)\", sm.s.Name(), sm.n.ID()),\n\t\tHashData: serializedHash,\n\t\tBlocksCount: sm.blocksApplied,\n\t}\n\n\t// Serialize snapshot as JSON\n\tsnapshotData, err := json.Marshal(snapshot)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"failed to marshal snapshot: %s\", err))\n\t}\n\n\t// InstallSnapshot is actually \"save the snapshot\", which is an operation delegated to the node\n\terr = sm.n.InstallSnapshot(snapshotData, false)\n\tif err != nil {\n\t\tsm.logDebug(\"failed to snapshot: %s\", err)\n\t\treturn\n\t}\n\n\t// Reset counter since last snapshot\n\tsm.blocksAppliedSinceSnapshot = 0\n}\n\nfunc (sm *RCOBStateMachine) loadSnapshot(data []byte) {\n\t// Deserialize snapshot from JSON\n\tvar snapshot RCOBSnapshot\n\terr := json.Unmarshal(data, &snapshot)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"failed to unmarshal snapshot: %s\", err))\n\t}\n\n\tsm.logDebug(\n\t\t\"Applying snapshot (created by %s) taken after %d blocks\",\n\t\tsnapshot.SourceNode,\n\t\tsnapshot.BlocksCount,\n\t)\n\n\t// Load internal hash state\n\terr = sm.hash.(encoding.BinaryUnmarshaler).UnmarshalBinary(snapshot.HashData)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"failed to unmarshal hash data: %s\", err))\n\t}\n\n\t// Load block counter\n\tsm.blocksApplied = snapshot.BlocksCount\n\n\t// Reset number of blocks applied since snapshot\n\tsm.blocksAppliedSinceSnapshot = 0\n\n\tsm.logDebug(\"Hash after snapshot with %d blocks: %X \", sm.blocksApplied, sm.hash.Sum(nil))\n}\n\n// Factory function to create state machine on top of the given server/node\nfunc newRaftChainStateMachine(s *Server, cfg *RaftConfig, n RaftNode) stateMachine {\n\t// Create RNG seed based on server name and node id\n\tvar seed int64\n\tfor _, c := range []byte(s.Name()) {\n\t\tseed += int64(c)\n\t}\n\tfor _, c := range []byte(n.ID()) {\n\t\tseed += int64(c)\n\t}\n\trng := rand.New(rand.NewSource(seed))\n\n\t// Initialize empty hash block\n\thashBlock := crc32.NewIEEE()\n\n\treturn &RCOBStateMachine{s: s, n: n, cfg: cfg, rng: rng, hash: hashBlock}\n}\n" }, { "path": "server/raft_helpers_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Do not exlude this file with the !skip_js_tests since those helpers\n// are also used by MQTT.\n\npackage server\n\nimport (\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype stateMachine interface {\n\tserver() *Server\n\tnode() RaftNode\n\twaitGroup() *sync.WaitGroup\n\t// This will call forward as needed so can be called on any node.\n\tpropose(data []byte)\n\t// When entries have been committed and can be applied.\n\tapplyEntry(ce *CommittedEntry)\n\t// When a leader change happens.\n\tleaderChange(isLeader bool)\n\t// Stop the raft group.\n\tstop()\n\t// Restart\n\trestart()\n}\n\n// Factory function needed for constructor.\ntype smFactory func(s *Server, cfg *RaftConfig, node RaftNode) stateMachine\n\ntype smGroup []stateMachine\n\n// Leader of the group.\nfunc (sg smGroup) leader() stateMachine {\n\tfor _, sm := range sg {\n\t\tif sm.node().Leader() {\n\t\t\treturn sm\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (sg smGroup) followers() []stateMachine {\n\tvar f []stateMachine\n\tfor _, sm := range sg {\n\t\tif sm.node().Leader() {\n\t\t\tcontinue\n\t\t}\n\t\tf = append(f, sm)\n\t}\n\treturn f\n}\n\n// Wait on a leader to be elected.\nfunc (sg smGroup) waitOnLeader() stateMachine {\n\texpires := time.Now().Add(10 * time.Second)\n\tfor time.Now().Before(expires) {\n\t\tfor _, sm := range sg {\n\t\t\tif sm.node().Leader() {\n\t\t\t\treturn sm\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\treturn nil\n}\n\n// Pick a random member.\nfunc (sg smGroup) randomMember() stateMachine {\n\treturn sg[rand.Intn(len(sg))]\n}\n\n// Return a non-leader\nfunc (sg smGroup) nonLeader() stateMachine {\n\tfor _, sm := range sg {\n\t\tif !sm.node().Leader() {\n\t\t\treturn sm\n\t\t}\n\t}\n\treturn nil\n}\n\n// Take out the lock on all nodes.\nfunc (sg smGroup) lockAll() {\n\tfor _, sm := range sg {\n\t\tsm.node().(*raft).Lock()\n\t}\n}\n\n// Release the lock on all nodes.\nfunc (sg smGroup) unlockAll() {\n\tfor _, sm := range sg {\n\t\tsm.node().(*raft).Unlock()\n\t}\n}\n\n// Acquire the lock on all follower nodes.\nfunc (sg smGroup) lockFollowers() []stateMachine {\n\tvar locked []stateMachine\n\tfor _, sm := range sg {\n\t\tif !sm.node().Leader() {\n\t\t\tlocked = append(locked, sm)\n\t\t\tsm.node().(*raft).Lock()\n\t\t}\n\t}\n\treturn locked[:]\n}\n\n// Create a raft group and place on numMembers servers at random.\n// Filestore based.\nfunc (c *cluster) createRaftGroup(name string, numMembers int, smf smFactory) smGroup {\n\treturn c.createRaftGroupEx(name, numMembers, smf, FileStorage)\n}\n\nfunc (c *cluster) createMemRaftGroup(name string, numMembers int, smf smFactory) smGroup {\n\treturn c.createRaftGroupEx(name, numMembers, smf, MemoryStorage)\n}\n\nfunc (c *cluster) createRaftGroupEx(name string, numMembers int, smf smFactory, st StorageType) smGroup {\n\tc.t.Helper()\n\tif numMembers > len(c.servers) {\n\t\tc.t.Fatalf(\"Members > Peers: %d vs %d\", numMembers, len(c.servers))\n\t}\n\tservers := append([]*Server{}, c.servers...)\n\trand.Shuffle(len(servers), func(i, j int) { servers[i], servers[j] = servers[j], servers[i] })\n\treturn c.createRaftGroupWithPeers(name, servers[:numMembers], smf, st)\n}\n\nfunc (c *cluster) createWAL(name string, st StorageType) WAL {\n\tc.t.Helper()\n\tvar err error\n\tvar store WAL\n\tif st == FileStorage {\n\t\tstore, err = newFileStore(\n\t\t\tFileStoreConfig{\n\t\t\t\tStoreDir: c.t.TempDir(),\n\t\t\t\tBlockSize: defaultMediumBlockSize,\n\t\t\t\tAsyncFlush: false,\n\t\t\t\tSyncInterval: 5 * time.Minute},\n\t\t\tStreamConfig{\n\t\t\t\tName: name,\n\t\t\t\tStorage: FileStorage})\n\t} else {\n\t\tstore, err = newMemStore(\n\t\t\t&StreamConfig{\n\t\t\t\tName: name,\n\t\t\t\tStorage: MemoryStorage})\n\t}\n\trequire_NoError(c.t, err)\n\treturn store\n}\n\nfunc serverPeerNames(servers []*Server) []string {\n\tvar peers []string\n\n\tfor _, s := range servers {\n\t\t// generate peer names.\n\t\ts.mu.RLock()\n\t\tpeers = append(peers, s.sys.shash)\n\t\ts.mu.RUnlock()\n\t}\n\n\treturn peers\n}\n\nfunc (c *cluster) createStateMachine(s *Server, cfg *RaftConfig, peers []string, smf smFactory) stateMachine {\n\ts.bootstrapRaftNode(cfg, peers, true)\n\tn, err := s.startRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(c.t, err)\n\tsm := smf(s, cfg, n)\n\tgo smLoop(sm)\n\treturn sm\n}\n\nfunc (c *cluster) createRaftGroupWithPeers(name string, servers []*Server, smf smFactory, st StorageType) smGroup {\n\tc.t.Helper()\n\n\tvar sg smGroup\n\tpeers := serverPeerNames(servers)\n\n\tfor _, s := range servers {\n\t\tcfg := &RaftConfig{\n\t\t\tName: name,\n\t\t\tStore: c.t.TempDir(),\n\t\t\tLog: c.createWAL(name, st)}\n\t\tsg = append(sg, c.createStateMachine(s, cfg, peers, smf))\n\t}\n\treturn sg\n}\n\nfunc (c *cluster) addNodeEx(name string, smf smFactory, st StorageType) stateMachine {\n\tc.t.Helper()\n\n\tserver := c.addInNewServer()\n\n\tcfg := &RaftConfig{\n\t\tName: name,\n\t\tStore: c.t.TempDir(),\n\t\tLog: c.createWAL(name, st)}\n\n\tpeers := serverPeerNames(c.servers)\n\treturn c.createStateMachine(server, cfg, peers, smf)\n}\n\nfunc (c *cluster) addRaftNode(name string, smf smFactory) stateMachine {\n\treturn c.addNodeEx(name, smf, FileStorage)\n}\n\nfunc (c *cluster) addMemRaftNode(name string, smf smFactory) stateMachine {\n\treturn c.addNodeEx(name, smf, MemoryStorage)\n}\n\n// Driver program for the state machine.\n// Should be run in its own go routine.\nfunc smLoop(sm stateMachine) {\n\ts, n, wg := sm.server(), sm.node(), sm.waitGroup()\n\tqch, lch, aq := n.QuitC(), n.LeadChangeC(), n.ApplyQ()\n\n\t// Wait group used to allow waiting until we exit from here.\n\twg.Add(1)\n\tdefer wg.Done()\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-aq.ch:\n\t\t\tces := aq.pop()\n\t\t\tfor _, ce := range ces {\n\t\t\t\tsm.applyEntry(ce)\n\t\t\t}\n\t\t\taq.recycle(&ces)\n\n\t\tcase isLeader := <-lch:\n\t\t\tsm.leaderChange(isLeader)\n\t\t}\n\t}\n}\n\n// Simple implementation of a replicated state.\n// The adder state just sums up int64 values.\ntype stateAdder struct {\n\tsync.Mutex\n\ts *Server\n\tn RaftNode\n\twg sync.WaitGroup\n\tcfg *RaftConfig\n\tsum int64\n\tlch chan bool\n}\n\n// Simple getters for server and the raft node.\nfunc (a *stateAdder) server() *Server {\n\ta.Lock()\n\tdefer a.Unlock()\n\treturn a.s\n}\n\nfunc (a *stateAdder) node() RaftNode {\n\ta.Lock()\n\tdefer a.Unlock()\n\treturn a.n\n}\n\nfunc (a *stateAdder) waitGroup() *sync.WaitGroup {\n\ta.Lock()\n\tdefer a.Unlock()\n\treturn &a.wg\n}\n\nfunc (a *stateAdder) propose(data []byte) {\n\t// Don't hold state machine lock as we could deadlock if the node was locked as part of the test.\n\tn := a.node()\n\tn.ForwardProposal(data)\n}\n\nfunc (a *stateAdder) applyEntry(ce *CommittedEntry) {\n\ta.Lock()\n\tif ce == nil {\n\t\t// This means initial state is done/replayed.\n\t\ta.Unlock()\n\t\treturn\n\t}\n\tfor _, e := range ce.Entries {\n\t\tif e.Type == EntryNormal {\n\t\t\tdelta, _ := binary.Varint(e.Data)\n\t\t\ta.sum += delta\n\t\t} else if e.Type == EntrySnapshot {\n\t\t\ta.sum, _ = binary.Varint(e.Data)\n\t\t}\n\t}\n\t// Update applied.\n\t// But don't hold state machine lock as we could deadlock if the node was locked as part of the test.\n\tn := a.n\n\ta.Unlock()\n\tn.Applied(ce.Index)\n}\n\nfunc (a *stateAdder) leaderChange(isLeader bool) {\n\tselect {\n\tcase a.lch <- isLeader:\n\tdefault:\n\t}\n}\n\n// Adder specific to change the total.\nfunc (a *stateAdder) proposeDelta(delta int64) {\n\tdata := make([]byte, binary.MaxVarintLen64)\n\tn := binary.PutVarint(data, int64(delta))\n\ta.propose(data[:n])\n}\n\n// Stop the group.\nfunc (a *stateAdder) stop() {\n\tn, wg := a.node(), a.waitGroup()\n\tn.Stop()\n\tn.WaitForStop()\n\twg.Wait()\n}\n\n// Restart the group\nfunc (a *stateAdder) restart() {\n\ta.Lock()\n\tdefer a.Unlock()\n\n\tif a.n.State() != Closed {\n\t\treturn\n\t}\n\n\t// The filestore is stopped as well, so need to extract the parts to recreate it.\n\trn := a.n.(*raft)\n\tvar err error\n\n\tswitch rn.wal.(type) {\n\tcase *fileStore:\n\t\tfs := rn.wal.(*fileStore)\n\t\ta.cfg.Log, err = newFileStore(fs.fcfg, fs.cfg.StreamConfig)\n\tcase *memStore:\n\t\tms := rn.wal.(*memStore)\n\t\ta.cfg.Log, err = newMemStore(&ms.cfg)\n\t}\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Must reset in-memory state.\n\t// A real restart would not preserve it, but more importantly we have no way to detect if we\n\t// already applied an entry. So, the sum must only be updated based on append entries or snapshots.\n\ta.sum = 0\n\n\ta.n, err = a.s.startRaftNode(globalAccountName, a.cfg, pprofLabels{})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\t// Finally restart the driver.\n\tgo smLoop(a)\n}\n\n// Total for the adder state machine.\nfunc (a *stateAdder) total() int64 {\n\ta.Lock()\n\tdefer a.Unlock()\n\treturn a.sum\n}\n\n// Install a snapshot.\nfunc (a *stateAdder) snapshot(t *testing.T) {\n\t// Don't hold state machine lock as we could deadlock if the node was locked as part of the test.\n\ta.Lock()\n\tsum := a.sum\n\trn := a.n\n\ta.Unlock()\n\n\tdata := make([]byte, binary.MaxVarintLen64)\n\tn := binary.PutVarint(data, sum)\n\tsnap := data[:n]\n\trequire_NoError(t, rn.InstallSnapshot(snap, false))\n}\n\n// Helper to wait for a certain state.\nfunc (rg smGroup) waitOnTotal(t *testing.T, expected int64) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\tvar err error\n\t\tfor _, sm := range rg {\n\t\t\tif sm.node().State() == Closed {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tasm := sm.(*stateAdder)\n\t\t\tif total := asm.total(); total != expected {\n\t\t\t\terr = errors.Join(err, fmt.Errorf(\"Adder on %v has wrong total: %d vs %d\",\n\t\t\t\t\tasm.server(), total, expected))\n\t\t\t}\n\t\t}\n\t\treturn err\n\t})\n}\n\n// Factory function.\nfunc newStateAdder(s *Server, cfg *RaftConfig, n RaftNode) stateMachine {\n\treturn &stateAdder{s: s, n: n, cfg: cfg, lch: make(chan bool, 1)}\n}\n\nfunc initSingleMemRaftNode(t *testing.T) (*raft, func()) {\n\tt.Helper()\n\tn, c := initSingleMemRaftNodeWithCluster(t)\n\tcleanup := func() {\n\t\tc.shutdown()\n\t}\n\treturn n, cleanup\n}\n\nfunc initSingleMemRaftNodeWithCluster(t *testing.T) (*raft, *cluster) {\n\tt.Helper()\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\ts := c.servers[0] // RunBasicJetStreamServer not available\n\n\tms, err := newMemStore(&StreamConfig{Name: \"TEST\", Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tcfg := &RaftConfig{Name: \"TEST\", Store: t.TempDir(), Log: ms}\n\n\tid := s.sys.shash[:idLen]\n\terr = s.bootstrapRaftNode(cfg, []string{id}, true)\n\trequire_NoError(t, err)\n\tn, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\treturn n, c\n}\n\n// Encode an AppendEntry.\n// An AppendEntry is encoded into a buffer and that's stored into the WAL.\n// This is a helper function to generate that buffer.\nfunc encode(t *testing.T, ae *appendEntry) *appendEntry {\n\tt.Helper()\n\tbuf, err := ae.encode(nil)\n\trequire_NoError(t, err)\n\tae.buf = buf\n\treturn ae\n}\n" }, { "path": "server/raft_test.go", "content": "// Copyright 2021-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestNRGSimple(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\t// Do several state transitions.\n\trg.randomMember().(*stateAdder).proposeDelta(22)\n\trg.randomMember().(*stateAdder).proposeDelta(-11)\n\trg.randomMember().(*stateAdder).proposeDelta(-10)\n\t// Wait for all members to have the correct state.\n\trg.waitOnTotal(t, 1)\n}\n\nfunc TestNRGSnapshotAndRestart(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\tlsm := rg.waitOnLeader()\n\trequire_NotNil(t, lsm)\n\tleader := lsm.(*stateAdder)\n\n\tvar expectedTotal int64\n\n\tsm := rg.nonLeader().(*stateAdder)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tdelta := rand.Int63n(222)\n\t\texpectedTotal += delta\n\t\tleader.proposeDelta(delta)\n\n\t\tif i == 250 {\n\t\t\t// Let some things catchup.\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\t// Snapshot leader and stop and snapshot a member.\n\t\t\tleader.snapshot(t)\n\t\t\tsm.snapshot(t)\n\t\t\tsm.stop()\n\t\t}\n\t}\n\t// Restart.\n\tsm.restart()\n\t// Wait for all members to have the correct state.\n\trg.waitOnTotal(t, expectedTotal)\n}\n\nfunc TestNRGAppendEntryEncode(t *testing.T) {\n\tae := &appendEntry{\n\t\tterm: 1,\n\t\tpindex: 0,\n\t}\n\n\t// Test leader should be _EMPTY_ or exactly idLen long\n\tae.leader = \"foo_bar_baz\"\n\t_, err := ae.encode(nil)\n\trequire_Error(t, err, errLeaderLen)\n\n\t// Empty ok (noLeader)\n\tae.leader = noLeader // _EMPTY_\n\t_, err = ae.encode(nil)\n\trequire_NoError(t, err)\n\n\tae.leader = \"DEREK123\"\n\t_, err = ae.encode(nil)\n\trequire_NoError(t, err)\n\n\t// Buffer reuse\n\tvar rawSmall [32]byte\n\tvar rawBigger [64]byte\n\n\tb := rawSmall[:]\n\tae.encode(b)\n\tif b[0] != 0 {\n\t\tt.Fatalf(\"Expected arg buffer to not be used\")\n\t}\n\tb = rawBigger[:]\n\tae.encode(b)\n\tif b[0] == 0 {\n\t\tt.Fatalf(\"Expected arg buffer to be used\")\n\t}\n\n\t// Test max number of entries.\n\tfor i := 0; i < math.MaxUint16+1; i++ {\n\t\tae.entries = append(ae.entries, &Entry{EntryNormal, nil})\n\t}\n\t_, err = ae.encode(b)\n\trequire_Error(t, err, errTooManyEntries)\n}\n\nfunc TestNRGAppendEntryDecode(t *testing.T) {\n\tae := &appendEntry{\n\t\tleader: \"12345678\",\n\t\tterm: 1,\n\t\tpindex: 0,\n\t}\n\tfor i := 0; i < math.MaxUint16; i++ {\n\t\tae.entries = append(ae.entries, &Entry{EntryNormal, nil})\n\t}\n\tbuf, err := ae.encode(nil)\n\trequire_NoError(t, err)\n\n\t// Truncate buffer first.\n\tshort := buf[0 : len(buf)-1025]\n\t_, err = decodeAppendEntry(short, nil, _EMPTY_)\n\trequire_Error(t, err, errBadAppendEntry)\n\n\tfor i := 0; i < 100; i++ {\n\t\tb := copyBytes(buf)\n\t\t// modifying the header (idx < 42) will not result in an error by decodeAppendEntry\n\t\tbi := 42 + rand.Intn(len(b)-42)\n\t\tif b[bi] != 0 && bi != 40 {\n\t\t\tb[bi] = 0\n\t\t\t_, err = decodeAppendEntry(b, nil, _EMPTY_)\n\t\t\trequire_Error(t, err, errBadAppendEntry)\n\t\t}\n\t}\n}\n\nfunc TestNRGRecoverFromFollowingNoLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Find out what term we are on.\n\tterm := rg.leader().node().Term()\n\n\t// Start by pausing all of the nodes. This will stop them from\n\t// processing new entries.\n\tfor _, n := range rg {\n\t\tn.node().PauseApply()\n\t}\n\n\t// Now drain all of the ApplyQ entries from them, which will stop\n\t// them from automatically trying to follow a previous leader if\n\t// they happened to have received an apply entry from one. Then\n\t// we're going to force them into a state where they are all\n\t// followers but they don't have a leader.\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\trn.ApplyQ().drain()\n\t\trn.switchToFollower(noLeader)\n\t}\n\n\t// Resume the nodes.\n\tfor _, n := range rg {\n\t\tn.node().ResumeApply()\n\t}\n\n\t// Wait a while. The nodes should notice that they haven't heard\n\t// from a leader lately and will switch to voting. After an\n\t// election we should find a new leader and be on a new term.\n\trg.waitOnLeader()\n\trequire_True(t, rg.leader() != nil)\n\trequire_NotEqual(t, rg.leader().node().Term(), term)\n}\n\nfunc TestNRGInlineStepdown(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// When StepDown() completes, we should not be the leader. Before,\n\t// this would not be guaranteed as the stepdown could be processed\n\t// some time later.\n\tn := rg.leader().node().(*raft)\n\trequire_NoError(t, n.StepDown())\n\trequire_NotEqual(t, n.State(), Leader)\n}\n\nfunc TestNRGObserverMode(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Put all of the followers into observer mode. In this state\n\t// they will not participate in an election but they will continue\n\t// to apply incoming commits.\n\tfor _, n := range rg {\n\t\tif n.node().Leader() {\n\t\t\tcontinue\n\t\t}\n\t\tn.node().SetObserver(true)\n\t}\n\n\t// Propose a change from the leader.\n\tadder := rg.leader().(*stateAdder)\n\tadder.proposeDelta(1)\n\tadder.proposeDelta(2)\n\tadder.proposeDelta(3)\n\n\t// Wait for the followers to apply it.\n\trg.waitOnTotal(t, 6)\n\n\t// Confirm the followers are still just observers and weren't\n\t// reset out of that state for some reason.\n\tfor _, n := range rg {\n\t\tif n.node().Leader() {\n\t\t\tcontinue\n\t\t}\n\t\trequire_True(t, n.node().IsObserver())\n\t}\n}\n\nfunc TestNRGAEFromOldLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Listen out for catchup requests.\n\tch := make(chan *nats.Msg, 16)\n\t_, err := nc.ChanSubscribe(fmt.Sprintf(raftCatchupReply, \">\"), ch)\n\trequire_NoError(t, err)\n\n\t// Start next term so that we can reuse term 1 in the next step.\n\tleader := rg.leader().node().(*raft)\n\tleader.StepDown()\n\ttime.Sleep(time.Millisecond * 100)\n\trg.waitOnLeader()\n\trequire_Equal(t, leader.Term(), 2)\n\tleader = rg.leader().node().(*raft)\n\n\t// Send an append entry with an outdated term. Beforehand, doing\n\t// so would have caused a WAL reset and then would have triggered\n\t// a Raft-level catchup.\n\tae := &appendEntry{\n\t\tterm: 1,\n\t\tpindex: 0,\n\t\tleader: leader.id,\n\t\treply: nc.NewRespInbox(),\n\t}\n\tpayload, err := ae.encode(nil)\n\trequire_NoError(t, err)\n\tresp, err := nc.Request(leader.asubj, payload, time.Second)\n\trequire_NoError(t, err)\n\n\t// Wait for the response, the server should have rejected it.\n\tar := decodeAppendEntryResponse(resp.Data)\n\trequire_NotNil(t, ar)\n\trequire_Equal(t, ar.success, false)\n\n\t// No catchup should happen at this point because no reset should\n\t// have happened.\n\trequire_NoChanRead(t, ch, time.Second*2)\n}\n\n// TestNRGSimpleElection tests that a simple election succeeds. It is\n// simple because the group hasn't processed any entries and hasn't\n// suffered any interruptions of any kind, therefore there should be\n// no way that the conditions for granting the votes can fail.\nfunc TestNRGSimpleElection(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 9)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 9, newStateAdder)\n\trg.waitOnLeader()\n\n\tvoteReqs := make(chan *nats.Msg, 1)\n\tvoteResps := make(chan *nats.Msg, len(rg)-1)\n\n\t// Keep a record of the term when we started.\n\tleader := rg.leader().node().(*raft)\n\tstartTerm := leader.term\n\n\t// Subscribe to the vote request subject, this should be the\n\t// same across all nodes in the group.\n\t_, err := nc.ChanSubscribe(leader.vsubj, voteReqs)\n\trequire_NoError(t, err)\n\n\t// Subscribe to all of the vote response inboxes for all nodes\n\t// in the Raft group, as they can differ.\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\t_, err = nc.ChanSubscribe(rn.vreply, voteResps)\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Step down, this will start a new voting session.\n\trequire_NoError(t, rg.leader().node().StepDown())\n\n\t// Wait for a vote request to come in.\n\tmsg := require_ChanRead(t, voteReqs, time.Second)\n\tvr := decodeVoteRequest(msg.Data, msg.Reply)\n\trequire_True(t, vr != nil)\n\trequire_NotEqual(t, vr.candidate, _EMPTY_)\n\n\t// The leader should have bumped their term in order to start\n\t// an election.\n\trequire_Equal(t, vr.term, startTerm+1)\n\trequire_Equal(t, vr.lastTerm, startTerm)\n\n\t// Wait for all of the vote responses to come in. There should\n\t// be as many vote responses as there are followers.\n\tfor i := 0; i < len(rg)-1; i++ {\n\t\tmsg := require_ChanRead(t, voteResps, time.Second)\n\t\tre := decodeVoteResponse(msg.Data)\n\t\trequire_True(t, re != nil)\n\n\t\t// Ignore old vote responses that could be in-flight.\n\t\tif re.term < vr.term {\n\t\t\ti--\n\t\t\tcontinue\n\t\t}\n\n\t\t// The vote should have been granted.\n\t\trequire_Equal(t, re.granted, true)\n\n\t\t// The node granted the vote, therefore the term in the vote\n\t\t// response should have advanced as well.\n\t\trequire_Equal(t, re.term, vr.term)\n\t\trequire_Equal(t, re.term, startTerm+1)\n\t}\n\n\t// Everyone in the group should have voted for our candidate\n\t// and arrived at the term from the vote request.\n\trg.lockAll()\n\tdefer rg.unlockAll()\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\trequire_Equal(t, rn.term, vr.term)\n\t\trequire_Equal(t, rn.term, startTerm+1)\n\t\trequire_Equal(t, rn.vote, vr.candidate)\n\t}\n}\n\n// TestNRGLeaderTransfer verifies that a Raft group correctly transfers\n// leadership to a chosen preferred node when the current leader steps down.\nfunc TestNRGLeaderTransfer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createMemRaftGroup(\"Test\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader()\n\tsub, err := nc.SubscribeSync(leader.node().(*raft).asubj)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\tpreferredID := rg.nonLeader().node().ID()\n\tleader.node().StepDown(preferredID)\n\tnewLeader := rg.waitOnLeader()\n\n\trequire_Equal(t, newLeader.node().ID(), preferredID)\n\n\t// Expect to see a EntryLeader message\n\tcheckFor(t, time.Second, 0, func() (err error) {\n\t\tmsg, err := sub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tae, err := decodeAppendEntry(msg.Data, nil, msg.Reply)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif len(ae.entries) == 1 {\n\t\t\te := ae.entries[0]\n\t\t\tif e.Type == EntryLeaderTransfer && string(e.Data) == preferredID {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\treturn fmt.Errorf(\"Expect EntryLeaderTransfer\")\n\t})\n}\n\nfunc TestNRGSwitchStateClearsQueues(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\ts := c.servers[0] // RunBasicJetStreamServer not available\n\n\tn := &raft{\n\t\tprop: newIPQueue[*proposedEntry](s, \"prop\"),\n\t\tresp: newIPQueue[*appendEntryResponse](s, \"resp\"),\n\t\tleadc: make(chan bool, 1), // for switchState\n\t\tsd: t.TempDir(),\n\t}\n\tn.state.Store(int32(Leader))\n\trequire_Equal(t, n.prop.len(), 0)\n\trequire_Equal(t, n.resp.len(), 0)\n\n\tn.prop.push(&proposedEntry{&Entry{}, _EMPTY_})\n\tn.resp.push(&appendEntryResponse{})\n\trequire_Equal(t, n.prop.len(), 1)\n\trequire_Equal(t, n.resp.len(), 1)\n\n\tn.switchState(Follower)\n\trequire_Equal(t, n.prop.len(), 0)\n\trequire_Equal(t, n.resp.len(), 0)\n}\n\nfunc TestNRGStepDownOnSameTermDoesntClearVote(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tlsm := rg.leader().(*stateAdder)\n\tleader := lsm.node().(*raft)\n\tfollower := rg.nonLeader().node().(*raft)\n\n\t// Make sure we handle the leader change notification from above.\n\trequire_ChanRead(t, lsm.lch, time.Second)\n\n\t// Subscribe to the append entry subject.\n\tsub, err := nc.SubscribeSync(leader.asubj)\n\trequire_NoError(t, err)\n\n\t// Get the first append entry that we receive.\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\trequire_NoError(t, sub.Unsubscribe())\n\n\t// We're going to modify the append entry that we received so that\n\t// we can send it again with modifications.\n\tae, err := decodeAppendEntry(msg.Data, nil, msg.Reply)\n\trequire_NoError(t, err)\n\n\t// First of all we're going to try sending an append entry that\n\t// has an old term and a fake leader.\n\tmsg.Reply = follower.areply\n\tae.leader = follower.id\n\tae.term = leader.term - 1\n\tmsg.Data, err = ae.encode(msg.Data[:0])\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.PublishMsg(msg))\n\n\t// Because the term was old, the fake leader shouldn't matter as\n\t// the current leader should ignore it.\n\trequire_NoChanRead(t, lsm.lch, time.Second)\n\n\t// Now we're going to send it on the same term that the current leader\n\t// is on. What we expect to happen is that the leader will step down\n\t// but it *shouldn't* clear the vote.\n\tae.term = leader.term\n\tmsg.Data, err = ae.encode(msg.Data[:0])\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.PublishMsg(msg))\n\n\t// Wait for the leader transition and ensure that the vote wasn't\n\t// cleared.\n\trequire_ChanRead(t, lsm.lch, time.Second)\n\trequire_NotEqual(t, leader.vote, noVote)\n}\n\nfunc TestNRGUnsuccessfulVoteRequestDoesntResetElectionTimer(t *testing.T) {\n\t// This test relies on nodes not hitting their election timer too often,\n\t// otherwise the step later where we capture the election time before and\n\t// after the failed vote request will flake.\n\torigMinTimeout, origMaxTimeout, origHBInterval := minElectionTimeout, maxElectionTimeout, hbInterval\n\tminElectionTimeout, maxElectionTimeout, hbInterval = time.Second*5, time.Second*10, time.Second*10\n\tdefer func() {\n\t\tminElectionTimeout, maxElectionTimeout, hbInterval = origMinTimeout, origMaxTimeout, origHBInterval\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\n\t// Because the election timer is quite high, we want to kick a node into\n\t// campaigning before it naturally needs to, otherwise the test takes a\n\t// long time just to pick a leader.\n\tfor _, n := range rg {\n\t\tn.node().Campaign()\n\t\tbreak\n\t}\n\trg.waitOnLeader()\n\trgLeader := rg.leader()\n\tleader := rgLeader.node().(*raft)\n\tfollower := rg.nonLeader().node().(*raft)\n\n\t// Send one message to ensure heartbeats are not sent during the remainder of this test.\n\trgLeader.(*stateAdder).proposeDelta(1)\n\trg.waitOnTotal(t, 1)\n\n\t// Set up a new inbox for the vote responses to go to.\n\tvsubj, vreply := leader.vsubj, nc.NewInbox()\n\tch := make(chan *nats.Msg, 3)\n\t_, err := nc.ChanSubscribe(vreply, ch)\n\trequire_NoError(t, err)\n\n\t// Keep a track of the last time the election timer was reset before this.\n\t// Also build up a vote request that's obviously behind so that the other\n\t// nodes should not do anything with it. All locks are taken at the same\n\t// time so that it guarantees that both the leader and the follower aren't\n\t// operating at the time we take the etlr snapshots.\n\trg.lockAll()\n\tleader.resetElect(maxElectionTimeout)\n\tfollower.resetElect(maxElectionTimeout)\n\tleaderOriginal := leader.etlr\n\tfollowerOriginal := follower.etlr\n\tvr := &voteRequest{\n\t\tterm: follower.term - 1,\n\t\tlastTerm: follower.term - 1,\n\t\tlastIndex: 0,\n\t\tcandidate: follower.id,\n\t}\n\trg.unlockAll()\n\n\t// Now send a vote request that's obviously behind.\n\trequire_NoError(t, nc.PublishMsg(&nats.Msg{\n\t\tSubject: vsubj,\n\t\tReply: vreply,\n\t\tData: vr.encode(),\n\t}))\n\n\t// Wait for everyone to respond.\n\trequire_ChanRead(t, ch, time.Second)\n\trequire_ChanRead(t, ch, time.Second)\n\trequire_ChanRead(t, ch, time.Second)\n\n\t// Neither the leader nor our chosen follower should have updated their\n\t// election timer as a result of this.\n\trg.lockAll()\n\tleaderEqual := leaderOriginal.Equal(leader.etlr)\n\tfollowerEqual := followerOriginal.Equal(follower.etlr)\n\trg.unlockAll()\n\trequire_True(t, leaderEqual)\n\trequire_True(t, followerEqual)\n}\n\nfunc TestNRGUnsuccessfulVoteRequestCampaignEarly(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tn.etlr = time.Time{}\n\n\t// Simple case: we are follower and vote for a candidate.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 1, lastTerm: 0, lastIndex: 0, candidate: nats0}))\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.vote, nats0)\n\trequire_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it voted.\n\tn.etlr = time.Time{}\n\n\t// We are follower and deny vote for outdated candidate.\n\tn.pterm, n.pindex = 1, 100\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 2, lastTerm: 1, lastIndex: 2, candidate: nats0}))\n\trequire_Equal(t, n.term, 2)\n\trequire_Equal(t, n.vote, noVote)\n\trequire_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it starts campaigning.\n\tn.etlr = time.Time{}\n\n\t// Switch to candidate.\n\tn.pterm, n.pindex = 2, 200\n\tn.switchToCandidate()\n\trequire_Equal(t, n.term, 3)\n\trequire_Equal(t, n.State(), Candidate)\n\trequire_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state.\n\tn.etlr = time.Time{}\n\n\t// We are candidate and deny vote for outdated candidate. But they were on a more recent term, restart campaign.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 4, lastTerm: 1, lastIndex: 2, candidate: nats0}))\n\trequire_Equal(t, n.term, 4)\n\trequire_Equal(t, n.vote, noVote)\n\trequire_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it restarts campaigning.\n\tn.etlr = time.Time{}\n\n\t// Switch to candidate.\n\tn.pterm, n.pindex = 4, 400\n\tn.switchToCandidate()\n\trequire_Equal(t, n.term, 5)\n\trequire_Equal(t, n.State(), Candidate)\n\trequire_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state.\n\tn.etlr = time.Time{}\n\n\t// We are candidate and deny vote for outdated candidate. Don't start campaigning early.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 5, lastTerm: 1, lastIndex: 2, candidate: nats0}))\n\trequire_Equal(t, n.term, 5)\n\trequire_Equal(t, n.vote, noVote)\n\t// Election timer must NOT be updated as that would mean another candidate that we don't vote\n\t// for can short-circuit us by making us restart elections, denying us the ability to become leader.\n\trequire_Equal(t, n.etlr, time.Time{})\n}\n\nfunc TestNRGInvalidTAVDoesntPanic(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\tleader := rg.waitOnLeader()\n\trequire_NotNil(t, leader)\n\n\t// Mangle the TAV file to a short length (less than uint64).\n\ttav := filepath.Join(leader.node().(*raft).sd, termVoteFile)\n\trequire_NoError(t, os.WriteFile(tav, []byte{1, 2, 3, 4}, 0644))\n\n\t// Restart the node.\n\tleader.stop()\n\tleader.restart()\n\n\t// Before the fix, a crash would have happened before this point.\n\tc.waitOnAllCurrent()\n}\n\nfunc TestNRGAssumeHighTermAfterCandidateIsolation(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Bump the term up on one of the follower nodes by a considerable\n\t// amount and force it into the candidate state. This is what happens\n\t// after a period of time in isolation.\n\tfollower := rg.nonLeader().node().(*raft)\n\tfollower.Lock()\n\tfollower.term += 100\n\tfollower.switchState(Candidate)\n\tfollower.Unlock()\n\n\tfollower.requestVote()\n\ttime.Sleep(time.Millisecond * 100)\n\n\t// The candidate will shortly send a vote request. When that happens,\n\t// the rest of the nodes in the cluster should move up to that term,\n\t// even though they will not grant the vote.\n\tnterm := follower.term\n\tfor _, n := range rg {\n\t\trequire_Equal(t, n.node().Term(), nterm)\n\t}\n\n\t// Have the leader send out a proposal, which will force the candidate\n\t// back into follower state.\n\trg.waitOnLeader()\n\trg.leader().(*stateAdder).proposeDelta(1)\n\trg.waitOnTotal(t, 1)\n\n\t// The candidate should have switched to a follower on a term equal to\n\t// or newer than the candidate had.\n\tfor _, n := range rg {\n\t\trequire_NotEqual(t, n.node().State(), Candidate)\n\t\trequire_True(t, n.node().Term() >= nterm)\n\t}\n}\n\n// Test to make sure this does not cause us to truncate our wal or enter catchup state.\nfunc TestNRGHeartbeatOnLeaderChange(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tfor i := 0; i < 10; i++ {\n\t\t// Restart the leader.\n\t\tleader := rg.leader().(*stateAdder)\n\t\tleader.proposeDelta(22)\n\t\tleader.proposeDelta(-11)\n\t\tleader.proposeDelta(-10)\n\t\t// Must observe forward progress, so each iteration will check +1 total.\n\t\trg.waitOnTotal(t, int64(i+1))\n\t\tleader.stop()\n\t\tleader.restart()\n\t\trg.waitOnLeader()\n\t}\n}\n\nfunc TestNRGElectionTimerAfterObserver(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tfor _, n := range rg {\n\t\tn.node().SetObserver(true)\n\t}\n\n\ttime.Sleep(maxElectionTimeout)\n\tbefore := time.Now()\n\n\tfor _, n := range rg {\n\t\tn.node().SetObserver(false)\n\t}\n\n\ttime.Sleep(maxCampaignTimeout)\n\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\trn.RLock()\n\t\tetlr := rn.etlr\n\t\trn.RUnlock()\n\t\trequire_True(t, etlr.After(before))\n\t}\n}\n\nfunc TestNRGSystemClientCleanupFromAccount(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\ts := c.randomServer()\n\tsacc := s.SystemAccount()\n\n\tnumClients := func() int {\n\t\tsacc.mu.RLock()\n\t\tdefer sacc.mu.RUnlock()\n\t\treturn len(sacc.clients)\n\t}\n\n\tstart := numClients()\n\n\tvar all []smGroup\n\tfor i := 0; i < 5; i++ {\n\t\trgName := fmt.Sprintf(\"TEST-%d\", i)\n\t\trg := c.createRaftGroup(rgName, 3, newStateAdder)\n\t\tall = append(all, rg)\n\t\trg.waitOnLeader()\n\t}\n\tfor _, rg := range all {\n\t\tfor _, sm := range rg {\n\t\t\tsm.node().Stop()\n\t\t}\n\t\tfor _, sm := range rg {\n\t\t\tsm.node().WaitForStop()\n\t\t}\n\t}\n\tfinish := numClients()\n\trequire_Equal(t, start, finish)\n}\n\nfunc TestNRGCandidateDoesntRevertTermAfterOldAE(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Bump the term up a few times.\n\tfor i := 0; i < 3; i++ {\n\t\trg.leader().node().StepDown()\n\t\ttime.Sleep(time.Millisecond * 50) // Needed because stepdowns not synchronous\n\t\trg.waitOnLeader()\n\t}\n\n\tleader := rg.leader().node().(*raft)\n\tfollower := rg.nonLeader().node().(*raft)\n\n\t// Sanity check that we are where we expect to be.\n\trequire_Equal(t, leader.term, 4)\n\trequire_Equal(t, follower.term, 4)\n\n\t// At this point the active term is 4 and pterm is 4, force the\n\t// term up to 9. This won't bump the pterm.\n\trg.lockAll()\n\tfor _, n := range rg {\n\t\tn.node().(*raft).term += 5\n\t}\n\trg.unlockAll()\n\n\t// Build an AE that has a term newer than the pterm but older than\n\t// the term. Give it to the follower in candidate state.\n\tae := newAppendEntry(leader.id, 6, leader.commit, leader.pterm, leader.pindex, nil)\n\tfollower.switchToCandidate()\n\tfollower.processAppendEntry(ae, nil)\n\n\t// The candidate must not have reverted back to term 6.\n\trequire_NotEqual(t, follower.term, 6)\n}\n\nfunc TestNRGTermDoesntRollBackToPtermOnCatchup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\t// Propose some entries so that we have entries in the log that have pterm 1.\n\tlsm := rg.leader().(*stateAdder)\n\tfor i := 0; i < 5; i++ {\n\t\tlsm.proposeDelta(1)\n\t\trg.waitOnTotal(t, int64(i)+1)\n\t}\n\n\t// Check that everyone is where they are supposed to be.\n\trg.lockAll()\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\trequire_Equal(t, rn.term, 1)\n\t\trequire_Equal(t, rn.pterm, 1)\n\t\trequire_Equal(t, rn.pindex, 6)\n\t}\n\trg.unlockAll()\n\n\t// Force a stepdown so that we move up to term 2.\n\trg.leader().node().(*raft).switchToFollower(noLeader)\n\trg.waitOnLeader()\n\tleader := rg.leader().node().(*raft)\n\n\t// Now make sure everyone has moved up to term 2. Additionally we're\n\t// going to prune back the follower logs to term 1 as this is what will\n\t// create the right conditions for the catchup.\n\trg.lockAll()\n\tfor _, n := range rg {\n\t\trn := n.node().(*raft)\n\t\trequire_Equal(t, rn.term, 2)\n\n\t\tif !rn.Leader() {\n\t\t\trn.truncateWAL(1, 6)\n\t\t\trequire_Equal(t, rn.term, 2) // rn.term must stay the same\n\t\t\trequire_Equal(t, rn.pterm, 1)\n\t\t\trequire_Equal(t, rn.pindex, 6)\n\t\t}\n\t}\n\t// This will make followers run a catchup.\n\tae := newAppendEntry(leader.id, leader.term, leader.commit, leader.pterm, leader.pindex, nil)\n\trg.unlockAll()\n\n\tarInbox := nc.NewRespInbox()\n\tarCh := make(chan *nats.Msg, 2)\n\t_, err := nc.ChanSubscribe(arInbox, arCh)\n\trequire_NoError(t, err)\n\n\t// Ensure the subscription is known by the server we're connected to,\n\t// but also by the other servers in the cluster.\n\trequire_NoError(t, nc.Flush())\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// In order to trip this condition, we need to send an append entry that\n\t// will trick the followers into running a catchup. In the process they\n\t// were setting the term back to pterm which is incorrect.\n\tb, err := ae.encode(nil)\n\trequire_NoError(t, err)\n\trequire_NoError(t, nc.PublishMsg(&nats.Msg{\n\t\tSubject: fmt.Sprintf(raftAppendSubj, \"TEST\"),\n\t\tReply: arInbox,\n\t\tData: b,\n\t}))\n\n\t// Wait for both followers to respond to the append entry and then verify\n\t// that none of the nodes should have reverted back to term 1.\n\trequire_ChanRead(t, arCh, time.Second*5) // First follower\n\trequire_ChanRead(t, arCh, time.Second*5) // Second follower\n\tfor _, n := range rg {\n\t\trequire_NotEqual(t, n.node().Term(), 1)\n\t}\n}\n\nfunc TestNRGNoResetOnAppendEntryResponse(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\tc.waitOnAllCurrent()\n\n\tleader := rg.leader().node().(*raft)\n\tfollower := rg.nonLeader().node().(*raft)\n\tlsm := rg.leader().(*stateAdder)\n\tisLeader := require_ChanRead(t, lsm.lch, 5*time.Second)\n\trequire_True(t, isLeader)\n\n\t// Subscribe for append entries that aren't heartbeats and respond to\n\t// each of them as though it's a non-success and with a higher term.\n\t// The higher term in this case is what would cause the leader previously\n\t// to reset the entire log which it shouldn't do.\n\t_, err := nc.Subscribe(fmt.Sprintf(raftAppendSubj, \"TEST\"), func(msg *nats.Msg) {\n\t\tif ae, err := decodeAppendEntry(msg.Data, nil, msg.Reply); err == nil && len(ae.entries) > 0 {\n\t\t\tar := newAppendEntryResponse(ae.term+1, ae.commit, follower.id, false)\n\t\t\trequire_NoError(t, msg.Respond(ar.encode(nil)))\n\t\t}\n\t})\n\trequire_NoError(t, err)\n\n\tc.waitOnAllCurrent()\n\n\t// Temporarily prevent followers to respond\n\tlocked := rg.lockFollowers()\n\n\t// Generate an append entry that the subscriber above can respond to.\n\tlsm.proposeDelta(5)\n\n\t// Expect the leader to step down\n\tcheckFor(t, 10*time.Second, 0, func() error {\n\t\tisLeader = require_ChanRead(t, lsm.lch, 5*time.Second)\n\t\tif isLeader {\n\t\t\treturn fmt.Errorf(\"Node is still leader!\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Unlock the followers\n\tfor _, l := range locked {\n\t\tl.node().(*raft).Unlock()\n\t}\n\n\t// The was-leader should now have stepped down, make sure that it\n\t// didn't blow away its log in the process.\n\trg.lockAll()\n\tdefer rg.unlockAll()\n\trequire_Equal(t, leader.State(), Follower)\n\trequire_NotEqual(t, leader.pterm, 0)\n\trequire_NotEqual(t, leader.pindex, 0)\n}\n\nfunc TestNRGCandidateDontStepdownDueToLeaderOfPreviousTerm(t *testing.T) {\n\t// This test relies on nodes not hitting their election timer too often.\n\torigMinTimeout, origMaxTimeout, origHBInterval := minElectionTimeout, maxElectionTimeout, hbInterval\n\tminElectionTimeout, maxElectionTimeout, hbInterval = time.Second*5, time.Second*10, time.Second\n\tdefer func() {\n\t\tminElectionTimeout, maxElectionTimeout, hbInterval = origMinTimeout, origMaxTimeout, origHBInterval\n\t}()\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\tc.waitOnLeader()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tvar (\n\t\tcandidatePterm uint64 = 50\n\t\tcandidatePindex uint64 = 70\n\t\tcandidateTerm uint64 = 100\n\t)\n\n\t// Create a candidate that has received entries while they were a follower in a previous term\n\tcandidate := rg.nonLeader().node().(*raft)\n\tcandidate.Lock()\n\tcandidate.switchState(Candidate)\n\tcandidate.pterm = candidatePterm\n\tcandidate.pindex = candidatePindex\n\tcandidate.term = candidateTerm\n\tcandidate.Unlock()\n\n\t// Leader term is behind candidate\n\tleader := rg.leader().node().(*raft)\n\tleader.Lock()\n\tleader.term = candidatePterm\n\tleader.pterm = candidatePterm\n\tleader.pindex = candidatePindex\n\tleader.Unlock()\n\n\t// Subscribe to the append entry subject.\n\tsub, err := nc.SubscribeSync(leader.asubj)\n\trequire_NoError(t, err)\n\n\t// Get the first append entry that we receive, should be heartbeat from leader of prev term\n\tmsg, err := sub.NextMsg(5 * time.Second)\n\trequire_NoError(t, err)\n\n\t// Stop nodes from progressing so we can check state\n\trg.lockAll()\n\tdefer rg.unlockAll()\n\n\t// Decode the append entry\n\tae, err := decodeAppendEntry(msg.Data, nil, msg.Reply)\n\trequire_NoError(t, err)\n\n\t// Check that the append entry is from the leader\n\trequire_Equal(t, ae.leader, leader.id)\n\t// Check that it came from the leader before it updated its term with the response from the candidate\n\trequire_Equal(t, ae.term, candidatePterm)\n\n\t// Check that the candidate hasn't stepped down\n\trequire_Equal(t, candidate.State(), Candidate)\n\t// Check that the candidate's term is still ahead of the leader's term\n\trequire_True(t, candidate.term > ae.term)\n}\n\nfunc TestNRGRemoveLeaderPeerDeadlockBug(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tn := rg.leader().node().(*raft)\n\tleader := n.ID()\n\n\t// Propose to remove the leader as a peer. Will lead to a deadlock with bug.\n\trequire_NoError(t, n.ProposeRemovePeer(leader))\n\trg.waitOnLeader()\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tnl := n.GroupLeader()\n\t\tif nl != leader {\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.New(\"Leader has not moved\")\n\t})\n}\n\nfunc TestNRGWALEntryWithoutQuorumMustTruncate(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\tmodify func(rg smGroup)\n\t}{\n\t\t{\n\t\t\t// state equals, only need to remove the entry\n\t\t\ttitle: \"equal\",\n\t\t\tmodify: func(rg smGroup) {},\n\t\t},\n\t\t{\n\t\t\t// state diverged, need to replace the entry\n\t\t\ttitle: \"diverged\",\n\t\t\tmodify: func(rg smGroup) {\n\t\t\t\trg.leader().(*stateAdder).proposeDelta(11)\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\t\t\trg.waitOnLeader()\n\n\t\t\tvar err error\n\t\t\tvar scratch [1024]byte\n\n\t\t\t// Simulate leader storing an AppendEntry in WAL but being hard killed before it can propose to its peers.\n\t\t\tn := rg.leader().node().(*raft)\n\t\t\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\t\t\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\t\t\tn.Lock()\n\t\t\tae := n.buildAppendEntry(entries)\n\t\t\tae.buf, err = ae.encode(scratch[:])\n\t\t\trequire_NoError(t, err)\n\t\t\terr = n.storeToWAL(ae)\n\t\t\tn.Unlock()\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Stop the leader so it moves to another one.\n\t\t\tn.shutdown()\n\n\t\t\t// Wait for another leader to be picked\n\t\t\trg.waitOnLeader()\n\n\t\t\t// Make a modification, specific to this test.\n\t\t\ttest.modify(rg)\n\n\t\t\t// Restart the previous leader that contains the stored AppendEntry without quorum.\n\t\t\tfor _, a := range rg {\n\t\t\t\tif a.node().ID() == n.ID() {\n\t\t\t\t\tsa := a.(*stateAdder)\n\t\t\t\t\tsa.restart()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// The previous leader's WAL should truncate to remove the AppendEntry only it has.\n\t\t\t// Eventually all WALs for all peers must match.\n\t\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() error {\n\t\t\t\tvar expected []*appendEntry\n\t\t\t\tfor _, a := range rg {\n\t\t\t\t\tan := a.node().(*raft)\n\t\t\t\t\tvar state StreamState\n\t\t\t\t\tan.wal.FastState(&state)\n\t\t\t\t\tif len(expected) > 0 && int(state.LastSeq-state.FirstSeq+1) != len(expected) {\n\t\t\t\t\t\treturn fmt.Errorf(\"WAL is different: too many entries\")\n\t\t\t\t\t}\n\t\t\t\t\t// Loop over all entries in the WAL, checking if the contents for all RAFT nodes are equal.\n\t\t\t\t\tfor index := state.FirstSeq; index <= state.LastSeq; index++ {\n\t\t\t\t\t\tae, err := an.loadEntry(index)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tae.buf = nil // ... as we'll deeply check everything else in the AE.\n\t\t\t\t\t\tae.lterm = 0 // ... as lterm can differ if one node caught up another.\n\t\t\t\t\t\tseq := int(index)\n\t\t\t\t\t\tif len(expected) < seq {\n\t\t\t\t\t\t\texpected = append(expected, ae)\n\t\t\t\t\t\t} else if !reflect.DeepEqual(expected[seq-1], ae) {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"WAL is different: stored AEs differ\")\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestNRGTermNoDecreaseAfterWALReset(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tl := rg.leader().node().(*raft)\n\tl.Lock()\n\tl.term = 20\n\tl.Unlock()\n\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\tl.Lock()\n\tae := l.buildAppendEntry(entries)\n\tl.Unlock()\n\n\tfor _, f := range rg {\n\t\tif f.node().ID() != l.ID() {\n\t\t\tfn := f.node().(*raft)\n\t\t\tfn.processAppendEntry(ae, fn.aesub)\n\t\t\trequire_Equal(t, fn.term, 20) // Follower's term gets upped as expected.\n\t\t}\n\t}\n\n\t// Lower the term, simulating the followers receiving a message from an old term/leader.\n\tae.term = 3\n\tfor _, f := range rg {\n\t\tif f.node().ID() != l.ID() {\n\t\t\tfn := f.node().(*raft)\n\t\t\tfn.processAppendEntry(ae, fn.aesub)\n\t\t\trequire_Equal(t, fn.term, 20) // Follower should reject and the term stays the same.\n\n\t\t\tfn.Lock()\n\t\t\tfn.resetWAL()\n\t\t\tfn.Unlock()\n\t\t\tfn.processAppendEntry(ae, fn.aesub)\n\t\t\trequire_Equal(t, fn.term, 20) // Follower should reject again, even after reset, term stays the same.\n\t\t}\n\t}\n}\n\nfunc TestNRGPendingAppendEntryCacheInvalidation(t *testing.T) {\n\tfor _, test := range []struct {\n\t\ttitle string\n\t\tentries int\n\t}{\n\t\t{title: \"empty\", entries: 1},\n\t\t{title: \"at limit\", entries: paeDropThreshold},\n\t\t{title: \"full\", entries: paeDropThreshold + 1},\n\t} {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\t\tdefer c.shutdown()\n\n\t\t\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\t\t\trg.waitOnLeader()\n\t\t\tl := rg.leader()\n\n\t\t\tl.(*stateAdder).proposeDelta(1)\n\t\t\trg.waitOnTotal(t, 1)\n\n\t\t\t// Fill up the cache with N entries.\n\t\t\t// The contents don't matter as they should never be applied.\n\t\t\trg.lockAll()\n\t\t\tfor _, s := range rg {\n\t\t\t\tn := s.node().(*raft)\n\t\t\t\tfor i := 0; i < test.entries; i++ {\n\t\t\t\t\tn.pae[n.pindex+uint64(1+i)] = newAppendEntry(\"\", 0, 0, 0, 0, nil)\n\t\t\t\t}\n\t\t\t}\n\t\t\trg.unlockAll()\n\n\t\t\tl.(*stateAdder).proposeDelta(1)\n\t\t\trg.waitOnTotal(t, 2)\n\t\t})\n\t}\n}\n\nfunc TestNRGCatchupDoesNotTruncateUncommittedEntriesWithQuorum(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\t// Timeline, for first leader\n\taeInitial := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\taeUncommitted := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeNoQuorum := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries})\n\n\t// Timeline, after leader change\n\taeMissed := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 2, entries: entries})\n\taeCatchupTrigger := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 3, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 4, entries: nil})\n\taeHeartbeat3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 4, pterm: 2, pindex: 4, entries: nil})\n\n\t// Initial case is simple, just store the entry.\n\tn.processAppendEntry(aeInitial, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Heartbeat, makes sure commit moves up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\n\t// We get one entry that has quorum (but we don't know that yet), so it stays uncommitted for a bit.\n\tn.processAppendEntry(aeUncommitted, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We get one entry that has NO quorum (but we don't know that yet).\n\tn.processAppendEntry(aeNoQuorum, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 3)\n\tentry, err = n.loadEntry(3)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We've just had a leader election, and we missed one message from the previous leader.\n\t// We should truncate the last message.\n\tn.processAppendEntry(aeCatchupTrigger, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\trequire_True(t, n.catchup == nil)\n\n\t// We get a heartbeat that prompts us to catchup.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\trequire_Equal(t, n.commit, 1) // Commit should not change, as we missed an item.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 1) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 2) // n.pindex\n\n\t// We now notice the leader indicated a different entry at the (no quorum) index, should save that.\n\tn.processAppendEntry(aeMissed, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 3)\n\trequire_True(t, n.catchup != nil)\n\n\t// We now get the entry that initially triggered us to catchup, it should be added.\n\tn.processAppendEntry(aeCatchupTrigger, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 4)\n\trequire_True(t, n.catchup != nil)\n\tentry, err = n.loadEntry(4)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats1)\n\n\t// Heartbeat, makes sure we commit (and reset catchup, as we're now up-to-date).\n\tn.processAppendEntry(aeHeartbeat3, n.aesub)\n\trequire_Equal(t, n.commit, 4)\n\trequire_True(t, n.catchup == nil)\n}\n\nfunc TestNRGCatchupCanTruncateMultipleEntriesWithoutQuorum(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\t// Timeline, for first leader\n\taeInitial := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\taeNoQuorum1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeNoQuorum2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries})\n\n\t// Timeline, after leader change\n\taeMissed1 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeMissed2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 2, entries: entries})\n\taeCatchupTrigger := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 2, pindex: 3, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 4, entries: nil})\n\taeHeartbeat3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 4, pterm: 2, pindex: 4, entries: nil})\n\n\t// Initial case is simple, just store the entry.\n\tn.processAppendEntry(aeInitial, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Heartbeat, makes sure commit moves up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\n\t// We get one entry that has NO quorum (but we don't know that yet).\n\tn.processAppendEntry(aeNoQuorum1, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We get another entry that has NO quorum (but we don't know that yet).\n\tn.processAppendEntry(aeNoQuorum2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 3)\n\tentry, err = n.loadEntry(3)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We've just had a leader election, and we missed messages from the previous leader.\n\t// We should truncate the last message.\n\tn.processAppendEntry(aeCatchupTrigger, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\trequire_True(t, n.catchup == nil)\n\n\t// We get a heartbeat that prompts us to catchup.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\trequire_Equal(t, n.commit, 1) // Commit should not change, as we missed an item.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 1) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 2) // n.pindex\n\n\t// We now notice the leader indicated a different entry at the (no quorum) index. We should truncate again.\n\tn.processAppendEntry(aeMissed2, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\trequire_True(t, n.catchup == nil)\n\n\t// We get a heartbeat that prompts us to catchup.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\trequire_Equal(t, n.commit, 1) // Commit should not change, as we missed an item.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 1) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 1) // n.pindex\n\n\t// We now get caught up with the missed messages.\n\tn.processAppendEntry(aeMissed1, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\trequire_True(t, n.catchup != nil)\n\n\tn.processAppendEntry(aeMissed2, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 3)\n\trequire_True(t, n.catchup != nil)\n\n\t// We now get the entry that initially triggered us to catchup, it should be added.\n\tn.processAppendEntry(aeCatchupTrigger, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 4)\n\trequire_True(t, n.catchup != nil)\n\tentry, err = n.loadEntry(4)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats1)\n\n\t// Heartbeat, makes sure we commit (and reset catchup, as we're now up-to-date).\n\tn.processAppendEntry(aeHeartbeat3, n.aesub)\n\trequire_Equal(t, n.commit, 4)\n\trequire_True(t, n.catchup == nil)\n}\n\nfunc TestNRGCatchupDoesNotTruncateCommittedEntriesDuringRedelivery(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\n\t// Initial case is simple, just store the entry.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Deliver a message.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Heartbeat, makes sure commit moves up.\n\tn.processAppendEntry(aeHeartbeat1, n.aesub)\n\trequire_Equal(t, n.commit, 2)\n\n\t// Deliver another message.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 3)\n\tentry, err = n.loadEntry(3)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Simulate receiving an old entry as a redelivery. We should not truncate as that lowers our commit.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.commit, 2)\n\n\t// Heartbeat, makes sure we commit.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.commit, 3)\n}\n\nfunc TestNRGCatchupFromNewLeaderWithIncorrectPterm(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\n\t// Heartbeat, triggers catchup.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 0) // Commit should not change, as we missed an item.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\n\t// First catchup message has the incorrect pterm, stop catchup and re-trigger later with the correct pterm.\n\tn.processAppendEntry(aeMsg, n.catchup.sub)\n\trequire_True(t, n.catchup == nil)\n\trequire_Equal(t, n.pterm, 1)\n\trequire_Equal(t, n.pindex, 0)\n\n\t// Heartbeat, triggers catchup.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 0) // Commit should not change, as we missed an item.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 1) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\n\t// Now we get the message again and can continue to store it.\n\tn.processAppendEntry(aeMsg, n.catchup.sub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Now heartbeat is able to commit the entry.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n}\n\nfunc TestNRGDontRemoveSnapshotIfTruncateToApplied(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\n\t// Initial case is simple, just store the entry.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Heartbeat, makes sure commit moves up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.pterm, 1)\n\n\t// Simulate upper layer calling down to apply.\n\tn.Applied(1)\n\n\t// Install snapshot and check it exists.\n\terr = n.InstallSnapshot(nil, false)\n\trequire_NoError(t, err)\n\n\tsnapshots := path.Join(n.sd, snapshotsDir)\n\tfiles, err := os.ReadDir(snapshots)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(files), 1)\n\n\t// Truncate and check snapshot is kept.\n\tn.truncateWAL(n.pterm, n.applied)\n\n\tfiles, err = os.ReadDir(snapshots)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(files), 1)\n}\n\nfunc TestNRGSnapshotAndTruncateToApplied(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.Unlock()\n\n\t// Timeline, other leader\n\taeMsg1 := encode(t, &appendEntry{leader: nats1, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats1, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Timeline, we temporarily became leader\n\taeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 2, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 2, commit: 3, pterm: 1, pindex: 3, entries: entries})\n\n\t// Timeline, old leader is back.\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats1, term: 3, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\n\t// Simply receive first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats1)\n\n\t// Receive second message, which commits the first message.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats1)\n\n\t// Simulate upper layer calling down to apply.\n\tn.Applied(1)\n\n\t// Send heartbeat, which commits the second message.\n\tn.switchToLeader()\n\tn.term = aeHeartbeat1.term\n\tn.processAppendEntryResponse(&appendEntryResponse{\n\t\tterm: aeHeartbeat1.term,\n\t\tindex: aeHeartbeat1.pindex,\n\t\tpeer: nats1,\n\t\treply: _EMPTY_,\n\t\tsuccess: true,\n\t})\n\trequire_Equal(t, n.commit, 3)\n\n\t// Simulate upper layer calling down to apply.\n\tn.Applied(3)\n\n\t// Install snapshot and check it exists.\n\terr = n.InstallSnapshot(nil, false)\n\trequire_NoError(t, err)\n\n\tsnapshots := path.Join(n.sd, snapshotsDir)\n\tfiles, err := os.ReadDir(snapshots)\n\trequire_NoError(t, err)\n\trequire_Equal(t, len(files), 1)\n\trequire_Equal(t, n.wal.State().Msgs, 0)\n\n\t// Store a third message, it stays uncommitted.\n\trequire_NoError(t, n.storeToWAL(aeMsg3))\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err = n.loadEntry(4)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Receive heartbeat from new leader, should not lose commits.\n\tn.stepdown(noLeader)\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.wal.State().Msgs, 0)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.applied, 3)\n}\n\nfunc TestNRGIgnoreDoubleSnapshot(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Simply receive first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\n\t// Heartbeat moves commit up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\n\t// Manually call back down to applied.\n\tn.Applied(1)\n\n\t// Second message just for upping the pterm.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.pterm, 2)\n\trequire_Equal(t, n.term, 2)\n\n\t// Snapshot, and confirm state.\n\terr := n.InstallSnapshot(nil, false)\n\trequire_NoError(t, err)\n\tsnap, err := n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_Equal(t, snap.lastTerm, 1)\n\trequire_Equal(t, snap.lastIndex, 1)\n\n\t// Snapshot again, should not overwrite previous snapshot.\n\terr = n.InstallSnapshot(nil, false)\n\trequire_Error(t, err, errNoSnapAvailable)\n\tsnap, err = n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_Equal(t, snap.lastTerm, 1)\n\trequire_Equal(t, snap.lastIndex, 1)\n}\n\nfunc TestNRGDontSwitchToCandidateWithInflightSnapshot(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample snapshot entry, the content doesn't matter.\n\tsnapshotEntries := []*Entry{\n\t\tnewEntry(EntrySnapshot, nil),\n\t\tnewEntry(EntryPeerState, encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})),\n\t}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeTriggerCatchup := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\taeCatchupSnapshot := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: snapshotEntries})\n\n\t// Switch follower into catchup.\n\tn.processAppendEntry(aeTriggerCatchup, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\n\t// Follower receives a snapshot, marking a snapshot as inflight as the apply queue is async.\n\tn.processAppendEntry(aeCatchupSnapshot, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 1)\n\n\t// Try to switch to candidate, it should be blocked since the snapshot is not processed yet.\n\tn.switchToCandidate()\n\trequire_Equal(t, n.State(), Follower)\n\n\t// Simulate snapshot being processed by the upper layer.\n\tn.Applied(1)\n\n\t// Retry becoming candidate, snapshot is processed so can now do so.\n\tn.switchToCandidate()\n\trequire_Equal(t, n.State(), Candidate)\n}\n\nfunc TestNRGDontSwitchToCandidateWithMultipleInflightSnapshots(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample snapshot entry, the content doesn't matter.\n\tsnapshotEntries := []*Entry{\n\t\tnewEntry(EntrySnapshot, nil),\n\t\tnewEntry(EntryPeerState, encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})),\n\t}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeSnapshot1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: snapshotEntries})\n\taeSnapshot2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: snapshotEntries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil})\n\n\t// Simulate snapshots being sent to us.\n\tn.processAppendEntry(aeSnapshot1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Equal(t, n.applied, 0)\n\n\tn.processAppendEntry(aeSnapshot2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.applied, 0)\n\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 2)\n\trequire_Equal(t, n.applied, 0)\n\n\tfor i := uint64(1); i <= 2; i++ {\n\t\t// Try to switch to candidate, it should be blocked since the snapshot is not processed yet.\n\t\tn.switchToCandidate()\n\t\trequire_Equal(t, n.State(), Follower)\n\n\t\t// Simulate snapshot being processed by the upper layer.\n\t\tn.Applied(i)\n\t}\n\n\t// Retry becoming candidate, all snapshots processed so can now do so.\n\tn.switchToCandidate()\n\trequire_Equal(t, n.State(), Candidate)\n}\n\nfunc TestNRGRecoverPindexPtermOnlyIfLogNotEmpty(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tgn := rg[0].(*stateAdder)\n\trn := rg[0].node().(*raft)\n\n\t// Delete the msgs and snapshots, leaving the only remaining trace\n\t// of the term in the TAV file.\n\tstore := filepath.Join(gn.cfg.Store)\n\trequire_NoError(t, rn.wal.Truncate(0))\n\trequire_NoError(t, os.RemoveAll(filepath.Join(store, \"msgs\")))\n\trequire_NoError(t, os.RemoveAll(filepath.Join(store, \"snapshots\")))\n\n\tfor _, gn := range rg {\n\t\tgn.stop()\n\t}\n\trg[0].restart()\n\trn = rg[0].node().(*raft)\n\n\t// Both should be zero as, without any snapshots or log entries,\n\t// the log is considered empty and therefore we account as such.\n\trequire_Equal(t, rn.pterm, 0)\n\trequire_Equal(t, rn.pindex, 0)\n}\n\nfunc TestNRGCancelCatchupWhenDetectingHigherTermDuringVoteRequest(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline.\n\taeCatchupTrigger := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\n\t// Truncate to simulate we missed one message and need to catchup.\n\tn.processAppendEntry(aeCatchupTrigger, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\n\t// Process first message as part of the catchup.\n\tcatchupSub := n.catchup.sub\n\tn.processAppendEntry(aeMsg1, catchupSub)\n\trequire_True(t, n.catchup != nil)\n\n\t// Receiving a vote request should cancel our catchup.\n\t// Otherwise, we could receive catchup messages after this that provides the previous leader with quorum.\n\t// If the new leader doesn't have these entries, the previous leader would desync since it would commit them.\n\terr := n.processVoteRequest(&voteRequest{2, 1, 1, nats0, \"reply\"})\n\trequire_NoError(t, err)\n\trequire_True(t, n.catchup == nil)\n}\n\nfunc TestNRGMultipleStopsDontPanic(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tdefer func() {\n\t\tp := recover()\n\t\trequire_True(t, p == nil)\n\t}()\n\n\tfor i := 0; i < 10; i++ {\n\t\tn.Stop()\n\t}\n}\n\nfunc TestNRGTruncateDownToCommitted(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\t// Timeline, we are leader\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Timeline, after leader change\n\taeMsg3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats1, term: 2, commit: 2, pterm: 2, pindex: 2, entries: nil})\n\n\t// Simply receive first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Receive second message, which commits the first message.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We receive an entry from another leader, should truncate down to commit / remove the second message.\n\t// After doing so, we should also be able to immediately store the message after.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats1)\n\n\t// Heartbeat moves commit up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 2)\n}\n\ntype mockWALTruncateAlwaysFails struct {\n\tWAL\n}\n\nfunc (m mockWALTruncateAlwaysFails) Truncate(seq uint64) error {\n\treturn errors.New(\"test: truncate always fails\")\n}\n\nfunc TestNRGTruncateDownToCommittedWhenTruncateFails(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tn.Lock()\n\tn.wal = mockWALTruncateAlwaysFails{n.wal}\n\tn.Unlock()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\t// Timeline, we are leader\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Timeline, after leader change\n\taeMsg3 := encode(t, &appendEntry{leader: nats1, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Simply receive first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Equal(t, n.wal.State().Msgs, 1)\n\tentry, err := n.loadEntry(1)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// Receive second message, which commits the first message.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.wal.State().Msgs, 2)\n\tentry, err = n.loadEntry(2)\n\trequire_NoError(t, err)\n\trequire_Equal(t, entry.leader, nats0)\n\n\t// We receive an entry from another leader, should truncate down to commit / remove the second message.\n\t// But, truncation fails so should register that and not change pindex/pterm.\n\tbindex, bterm := n.pindex, n.pterm\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Error(t, n.werr, errors.New(\"test: truncate always fails\"))\n\trequire_Equal(t, bindex, n.pindex)\n\trequire_Equal(t, bterm, n.pterm)\n}\n\nfunc TestNRGForwardProposalResponse(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, _ := jsClientConnect(t, c.leader(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tn := rg.nonLeader().node().(*raft)\n\tpsubj := n.psubj\n\n\tdata := make([]byte, binary.MaxVarintLen64)\n\tdn := binary.PutVarint(data, int64(123))\n\n\t_, err := nc.Request(psubj, data[:dn], time.Second*5)\n\trequire_NoError(t, err)\n\n\trg.waitOnTotal(t, 123)\n}\n\nfunc TestNRGMemoryWALEmptiesSnapshotsDir(t *testing.T) {\n\tn, c := initSingleMemRaftNodeWithCluster(t)\n\tdefer c.shutdown()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\n\t// Simply receive first message.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\n\t// Heartbeat moves commit up.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.commit, 1)\n\n\t// Manually call back down to applied, and then snapshot.\n\tn.Applied(1)\n\terr := n.InstallSnapshot(nil, false)\n\trequire_NoError(t, err)\n\n\t// Stop current node and restart it.\n\tn.Stop()\n\tn.WaitForStop()\n\n\ts := c.servers[0]\n\tms, err := newMemStore(&StreamConfig{Name: \"TEST\", Storage: MemoryStorage})\n\trequire_NoError(t, err)\n\tcfg := &RaftConfig{Name: \"TEST\", Store: n.sd, Log: ms}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Since the WAL is in-memory, the snapshots dir should've been emptied upon restart.\n\tfiles, err := os.ReadDir(filepath.Join(n.sd, snapshotsDir))\n\trequire_NoError(t, err)\n\trequire_Len(t, len(files), 0)\n}\n\nfunc TestNRGHealthCheckWaitForCatchup(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.Unlock()\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\n\t// Switch follower into catchup.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\trequire_Equal(t, n.catchup.cterm, aeHeartbeat.term)\n\trequire_Equal(t, n.catchup.cindex, aeHeartbeat.pindex)\n\n\t// Catchup first message.\n\tn.processAppendEntry(aeMsg1, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// Catchup second message.\n\tn.processAppendEntry(aeMsg2, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\trequire_False(t, n.Healthy())\n\n\t// If we apply the entry sooner than we receive the next catchup message,\n\t// should not mark as healthy since we're still in catchup.\n\tn.Applied(1)\n\trequire_False(t, n.Healthy())\n\n\t// Catchup third message.\n\tn.processAppendEntry(aeMsg3, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 2)\n\tn.Applied(2)\n\trequire_False(t, n.Healthy())\n\n\t// Heartbeat stops catchup.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_True(t, n.catchup == nil)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_False(t, n.Healthy())\n\n\t// Still need to wait for the last entry to be applied.\n\tn.Applied(3)\n\trequire_True(t, n.Healthy())\n}\n\nfunc TestNRGHealthCheckWaitForDoubleCatchup(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.Unlock()\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\n\t// Switch follower into catchup.\n\tn.processAppendEntry(aeHeartbeat1, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\trequire_Equal(t, n.catchup.cterm, aeHeartbeat1.term)\n\trequire_Equal(t, n.catchup.cindex, aeHeartbeat1.pindex)\n\n\t// Catchup first message.\n\tn.processAppendEntry(aeMsg1, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// We miss this message, since we're catching up.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// We also miss the heartbeat, since we're catching up.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// Catchup second message, this will stop catchup.\n\tn.processAppendEntry(aeMsg2, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_False(t, n.Healthy())\n\n\t// We expect to still be in catchup, waiting for a heartbeat or new append entry to reset.\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.cterm, aeHeartbeat1.term)\n\trequire_Equal(t, n.catchup.cindex, aeHeartbeat1.pindex)\n\n\t// We now get a 'future' heartbeat, should restart catchup.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 1) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 2) // n.pindex\n\trequire_Equal(t, n.catchup.cterm, aeHeartbeat2.term)\n\trequire_Equal(t, n.catchup.cindex, aeHeartbeat2.pindex)\n\trequire_False(t, n.Healthy())\n\n\t// Catchup third message.\n\tn.processAppendEntry(aeMsg3, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 2)\n\tn.Applied(2)\n\trequire_False(t, n.Healthy())\n\n\t// Heartbeat stops catchup.\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_True(t, n.catchup == nil)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_False(t, n.Healthy())\n\n\t// Still need to wait for the last entry to be applied.\n\tn.Applied(3)\n\trequire_True(t, n.Healthy())\n}\n\nfunc TestNRGHealthCheckWaitForPendingCommitsWhenPaused(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.Unlock()\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: nil})\n\n\t// Process first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// Process second message, moves commit up.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_False(t, n.Healthy())\n\n\t// We're healthy once we've applied the first message.\n\tn.Applied(1)\n\trequire_True(t, n.Healthy())\n\n\t// If we're paused we still are healthy if there are no pending commits.\n\terr := n.PauseApply()\n\trequire_NoError(t, err)\n\trequire_True(t, n.Healthy())\n\n\t// Heartbeat marks second message to be committed.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_False(t, n.Healthy())\n\n\t// Resuming apply commits the message.\n\tn.ResumeApply()\n\trequire_NoError(t, err)\n\trequire_False(t, n.Healthy())\n\n\t// But still waiting for it to be applied before marking healthy.\n\tn.Applied(2)\n\trequire_True(t, n.Healthy())\n}\n\nfunc TestNRGAppendEntryCanEstablishQuorumAfterLeaderChange(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.Unlock()\n\n\t// Timeline\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeatResponse := &appendEntryResponse{term: 1, index: 2, peer: nats0, success: true}\n\n\t// Process first message.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.aflr, 0)\n\n\t// Simulate becoming leader, and not knowing if the stored entry has quorum and can be committed.\n\t// Switching to leader should send a heartbeat.\n\tn.switchToLeader()\n\trequire_Equal(t, n.aflr, 2)\n\trequire_Equal(t, n.commit, 0)\n\n\t// We simulate receiving the successful heartbeat response here. It should move the commit up.\n\tn.processAppendEntryResponse(aeHeartbeatResponse)\n\trequire_Equal(t, n.commit, 2)\n\trequire_Equal(t, n.aflr, 2)\n\n\t// Once the entry is applied, it should reset the applied floor.\n\tn.Applied(2)\n\trequire_Equal(t, n.aflr, 0)\n}\n\nfunc TestNRGQuorumAccounting(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats2 := \"cnrtt3eg\" // \"nats-2\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.addPeer(nats2)\n\tn.Unlock()\n\n\t// Timeline\n\taeHeartbeat1Response := &appendEntryResponse{term: 1, index: 1, peer: nats1, success: true}\n\taeHeartbeat2Response := &appendEntryResponse{term: 1, index: 1, peer: nats2, success: true}\n\n\t// Adjust cluster size, so we need at least 2 responses from other servers to establish quorum.\n\trequire_NoError(t, n.AdjustBootClusterSize(5))\n\trequire_Equal(t, n.csz, 5)\n\trequire_Equal(t, n.qn, 3)\n\n\t// Switch this node to leader which sends an entry.\n\tn.switchToLeader()\n\tn.term = 1\n\trequire_Equal(t, n.pindex, 1)\n\n\t// The first response MUST NOT indicate quorum has been reached.\n\tn.processAppendEntryResponse(aeHeartbeat1Response)\n\trequire_Equal(t, n.commit, 0)\n\n\t// The second response means we have reached quorum and can move commit up.\n\tn.processAppendEntryResponse(aeHeartbeat2Response)\n\trequire_Equal(t, n.commit, 1)\n}\n\nfunc TestNRGRevalidateQuorumAfterLeaderChange(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats2 := \"cnrtt3eg\" // \"nats-2\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.addPeer(nats2)\n\tn.Unlock()\n\n\t// Timeline\n\taeHeartbeat1Response := &appendEntryResponse{term: 1, index: 1, peer: nats1, success: true}\n\taeHeartbeat2Response := &appendEntryResponse{term: 6, index: 2, peer: nats2, success: true}\n\n\t// Adjust cluster size, so we need at least 2 responses from other servers to establish quorum.\n\trequire_NoError(t, n.AdjustBootClusterSize(5))\n\trequire_Equal(t, n.csz, 5)\n\trequire_Equal(t, n.qn, 3)\n\n\t// Switch this node to leader which sends an entry.\n\tn.term++\n\tn.switchToLeader()\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// We have one server that signals the message was stored. The leader will add 1 to the acks count.\n\tn.processAppendEntryResponse(aeHeartbeat1Response)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Len(t, len(n.acks), 1)\n\n\t// We stepdown now and don't know if we will have quorum on the first entry.\n\tn.stepdown(noLeader)\n\n\t// Let's assume there are a bunch of leader elections now, data being added to the log, being truncated, etc.\n\t// We don't know what happened, maybe we were partitioned, but we can't know for sure if the first entry has quorum.\n\n\t// We now become leader again.\n\tn.term = 6\n\tn.switchToLeader()\n\trequire_Equal(t, n.term, 6)\n\n\t// We now receive a successful response from another server saying they have stored it.\n\t// Anything can have happened to the replica that said success before, we can't assume that's still valid.\n\t// So our commit must stay the same and we restart counting for quorum.\n\tn.processAppendEntryResponse(aeHeartbeat2Response)\n\trequire_Equal(t, n.commit, 0)\n\trequire_Len(t, len(n.acks), 1)\n}\n\nfunc TestNRGSignalLeadChangeFalseIfCampaignImmediately(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\tswitchNode func(n *raft)\n\t}{\n\t\t{\n\t\t\ttitle: \"Follower\",\n\t\t},\n\t\t{\n\t\t\ttitle: \"Candidate\",\n\t\t\tswitchNode: func(n *raft) {\n\t\t\t\tn.switchToCandidate()\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"Leader\",\n\t\t\tswitchNode: func(n *raft) {\n\t\t\t\tn.switchToCandidate()\n\t\t\t\tn.switchToLeader()\n\t\t\t\tselect {\n\t\t\t\tcase <-n.LeadChangeC():\n\t\t\t\t\tt.Error(\"Expected no leadChange signal\")\n\t\t\t\tdefault:\n\t\t\t\t\t// Expecting no signal yet.\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.title, func(t *testing.T) {\n\t\t\tn, cleanup := initSingleMemRaftNode(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\t\t\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\t\t\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\t\t\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\t\t\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\n\t\t\t// Campaigning immediately signals we're the preferred leader.\n\t\t\trequire_NoError(t, n.CampaignImmediately())\n\t\t\tif test.switchNode != nil {\n\t\t\t\ttest.switchNode(n)\n\t\t\t}\n\n\t\t\tn.processAppendEntry(aeMsg1, n.aesub)\n\n\t\t\tselect {\n\t\t\tcase isLeader := <-n.LeadChangeC():\n\t\t\t\trequire_False(t, isLeader)\n\t\t\tdefault:\n\t\t\t\tt.Error(\"Expected leadChange signal\")\n\t\t\t}\n\t\t\trequire_Equal(t, n.State(), Follower)\n\t\t\trequire_Equal(t, n.leader, nats0)\n\t\t\trequire_Equal(t, n.term, 1)\n\t\t})\n\t}\n}\n\nfunc TestNRGCatchupDontCountTowardQuorum(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\taeReply := \"$TEST\"\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(aeReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Timeline\n\taeMissedMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries, reply: aeReply})\n\tae := appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries, reply: aeReply}\n\taeCatchupTrigger := encode(t, &ae)\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 2, entries: nil, reply: aeReply})\n\n\t// Simulate we missed all messages up to this point.\n\tn.processAppendEntry(aeCatchupTrigger, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\trequire_Equal(t, n.catchup.cterm, ae.pterm)\n\trequire_Equal(t, n.catchup.cindex, ae.pindex)\n\n\t// Should reply we require catchup.\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tar := decodeAppendEntryResponse(msg.Data)\n\trequire_Equal(t, ar.index, 0)\n\trequire_False(t, ar.success)\n\trequire_True(t, strings.HasPrefix(msg.Reply, \"$NRG.CR\"))\n\n\t// Should NEVER respond to catchup messages.\n\tn.processAppendEntry(aeMissedMsg, n.catchup.sub)\n\t_, err = sub.NextMsg(time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\tn.processAppendEntry(aeCatchupTrigger, n.catchup.sub)\n\t_, err = sub.NextMsg(time.Second)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\t// Now we've received all messages, stop catchup, and respond success to new message.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tar = decodeAppendEntryResponse(msg.Data)\n\trequire_Equal(t, ar.index, aeHeartbeat.pindex)\n\trequire_True(t, ar.success)\n\trequire_Equal(t, msg.Reply, _EMPTY_)\n}\n\nfunc TestNRGIgnoreTrackResponseWhenNotLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tn.Lock()\n\tn.addPeer(nats1)\n\t// Keep quorum at 2 so the initial leader NOOP does not auto-commit.\n\tn.lsut = time.Time{}\n\tn.Unlock()\n\n\t// Switch this node to leader which sends an entry.\n\tn.term++\n\tn.switchToLeader()\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.pterm, 1)\n\trequire_Equal(t, n.commit, 0)\n\n\t// Step down\n\tn.stepdown(noLeader)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.pterm, 1)\n\trequire_Equal(t, n.commit, 0)\n\n\t// Normally would commit the entry, but since we're not leader anymore we should ignore it.\n\tn.trackResponse(&appendEntryResponse{1, 1, \"peer\", _EMPTY_, true})\n\trequire_Equal(t, n.commit, 0)\n}\n\nfunc TestNRGRejectNewAppendEntryFromPreviousLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t// Accept first message because it equals our term.\n\tn.term = 1\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pterm, 1)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// We are part of the successful vote for a new leader under a new term.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 5, lastTerm: 1, lastIndex: 2}))\n\n\t// Must reject entry from a previous term.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pterm, 1)\n\trequire_Equal(t, n.pindex, 1)\n}\n\nfunc TestNRGRejectAppendEntryDuringCatchupFromPreviousLeader(t *testing.T) {\n\ttest := func(t *testing.T, isCatchingUp, oldBehavior bool) {\n\t\tn, cleanup := initSingleMemRaftNode(t)\n\t\tdefer cleanup()\n\n\t\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\t\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\t\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\t\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t\t// Timeline\n\t\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\t\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\n\t\t// Accept first message because it equals our term.\n\t\tn.term = 1\n\t\tn.processAppendEntry(aeMsg2, n.aesub)\n\t\trequire_True(t, n.catchup != nil)\n\t\trequire_Equal(t, n.catchup.pterm, 0) // n.pterm\n\t\trequire_Equal(t, n.catchup.pindex, 0) // n.pindex\n\t\trequire_Equal(t, n.catchup.cterm, aeMsg2.pterm)\n\t\trequire_Equal(t, n.catchup.cindex, aeMsg2.pindex)\n\n\t\t// Under the new behavior the term of the leader that's doing the catchup is included.\n\t\tif !oldBehavior {\n\t\t\taeMsg1.lterm = 1\n\t\t\taeMsg2.lterm = 1\n\t\t}\n\n\t\t// First catchup message is accepted.\n\t\tcatchup := n.catchup\n\t\tn.processAppendEntry(aeMsg1, catchup.sub)\n\t\trequire_Equal(t, n.pterm, 1)\n\t\trequire_Equal(t, n.pindex, 1)\n\n\t\t// We are part of the successful vote for a new leader under a new term.\n\t\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 5, lastTerm: 1, lastIndex: 2}))\n\n\t\t// Voting cancels catchup. For testing, revert that so we can test\n\t\t// what a catchup message after upping the term does.\n\t\tnsub := n.aesub\n\t\tif isCatchingUp {\n\t\t\tn.catchup = catchup\n\t\t\tnsub = catchup.sub\n\t\t}\n\n\t\t// Now send the second catchup entry.\n\t\tn.processAppendEntry(aeMsg2, nsub)\n\t\trequire_Equal(t, n.pterm, 1)\n\n\t\t// Under the old behavior this entry is wrongly accepted.\n\t\t// A new server will also know to be backward-compatible if being caught up by an old server.\n\t\tif isCatchingUp && oldBehavior {\n\t\t\trequire_Equal(t, n.pindex, 2)\n\t\t} else {\n\t\t\t// Under the new behavior the entry is correctly accepted.\n\t\t\trequire_Equal(t, n.pindex, 1)\n\t\t}\n\t}\n\n\tfor _, isCatchingUp := range []bool{false, true} {\n\t\tfor _, oldBehavior := range []bool{false, true} {\n\t\t\ttitle := \"new-entry\"\n\t\t\tif isCatchingUp {\n\t\t\t\ttitle = \"catchup\"\n\t\t\t}\n\t\t\tif oldBehavior {\n\t\t\t\ttitle += \"-backward-compatible\"\n\t\t\t}\n\t\t\tt.Run(title, func(t *testing.T) { test(t, isCatchingUp, oldBehavior) })\n\t\t}\n\t}\n}\n\nfunc TestNRGDontRejectAppendEntryFromReplay(t *testing.T) {\n\ttest := func(t *testing.T, restart bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\t\trg.waitOnLeader()\n\n\t\t// Stop all servers except the leader.\n\t\tl := rg.leader().(*stateAdder)\n\t\trs := rg.nonLeader()\n\t\tfor _, sm := range rg {\n\t\t\tif sm != l {\n\t\t\t\tsm.stop()\n\t\t\t}\n\t\t}\n\n\t\t// Propose a new entry to the leader and confirm it's stored in its log.\n\t\tpindex, _, _ := l.node().Progress()\n\t\tl.proposeDelta(10)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif index, _, _ := l.node().Progress(); index == pindex {\n\t\t\t\treturn errors.New(\"proposal not stored yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Optionally restart the leader, which will require replay of the entries in the log.\n\t\tif restart {\n\t\t\tl.stop()\n\t\t\tl.restart()\n\t\t}\n\n\t\t// Shutdown server should be caught up.\n\t\trs.restart()\n\n\t\t// Wait for both online servers to have applied the delta.\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() (err error) {\n\t\t\tfor _, sm := range rg {\n\t\t\t\tif sm != l && sm != rs {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif total := sm.(*stateAdder).total(); total != 10 {\n\t\t\t\t\terr = errors.Join(err, fmt.Errorf(\"expected 10 total, got: %d\", total))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t}\n\n\tt.Run(\"no-restart\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"with-restart\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestNRGSimpleCatchup(t *testing.T) {\n\ttest := func(t *testing.T, leaderChange bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\t\trg.waitOnLeader()\n\n\t\t// Shutdown a single group, we'll start it later for catchup.\n\t\tnl := rg.nonLeader()\n\t\tnl.stop()\n\n\t\tl := rg.leader().(*stateAdder)\n\t\tl.proposeDelta(10)\n\n\t\t// Wait for remaining servers to have applied the delta.\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() (err error) {\n\t\t\tfor _, sm := range rg {\n\t\t\t\ttotal := sm.(*stateAdder).total()\n\t\t\t\tif sm == nl && total != 0 {\n\t\t\t\t\terr = errors.Join(err, errors.New(\"expected shutdown server to not get data\"))\n\t\t\t\t} else if sm != nl && total != 10 {\n\t\t\t\t\terr = errors.Join(err, fmt.Errorf(\"expected 10 total, got: %d\", total))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\n\t\tif leaderChange {\n\t\t\trequire_NoError(t, l.node().StepDown())\n\t\t}\n\n\t\t// Shutdown server should be caught up.\n\t\tnl.restart()\n\t\trg.waitOnTotal(t, 10)\n\t}\n\n\tt.Run(\"same-leader\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"change-leader\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestNRGSnapshotCatchup(t *testing.T) {\n\ttest := func(t *testing.T, restart bool) {\n\t\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\t\tdefer c.shutdown()\n\n\t\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\t\trg.waitOnLeader()\n\n\t\tl := rg.leader().(*stateAdder)\n\t\tvar s1 stateMachine\n\t\tvar s2 stateMachine\n\t\tfor _, sm := range rg {\n\t\t\tif sm == l {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif s1 == nil {\n\t\t\t\ts1 = sm\n\t\t\t} else {\n\t\t\t\ts2 = sm\n\t\t\t}\n\t\t}\n\n\t\t// Stop one non-leader server.\n\t\ts1.stop()\n\n\t\t// Wait for both online servers to have applied the delta.\n\t\tl.proposeDelta(10)\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() (err error) {\n\t\t\tfor _, sm := range rg {\n\t\t\t\tif sm == s1 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif total := sm.(*stateAdder).total(); total != 10 {\n\t\t\t\t\terr = errors.Join(err, fmt.Errorf(\"expected 10 total, got: %d\", total))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\n\t\t// Shutdown last non-leader server.\n\t\ts2.stop()\n\n\t\t// Snapshot so outdated server needs to catchup based on it.\n\t\tl.snapshot(t)\n\n\t\t// Propose a new entry to the leader and confirm it's stored in its log.\n\t\tpindex, _, _ := l.node().Progress()\n\t\tl.proposeDelta(10)\n\t\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t\tif index, _, _ := l.node().Progress(); index == pindex {\n\t\t\t\treturn errors.New(\"proposal not stored yet\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Optionally restart the leader, which will require replay of the entries in the log.\n\t\tif restart {\n\t\t\tl.stop()\n\t\t\tl.restart()\n\t\t}\n\n\t\t// Shutdown server should be caught up.\n\t\ts1.restart()\n\n\t\t// Wait for both online servers to have applied the delta.\n\t\tcheckFor(t, 5*time.Second, 200*time.Millisecond, func() (err error) {\n\t\t\tfor _, sm := range rg {\n\t\t\t\tif sm == s2 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif total := sm.(*stateAdder).total(); total != 20 {\n\t\t\t\t\terr = errors.Join(err, fmt.Errorf(\"expected 20 total, got: %d\", total))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t})\n\t}\n\n\tt.Run(\"no-restart\", func(t *testing.T) { test(t, false) })\n\tt.Run(\"with-restart\", func(t *testing.T) { test(t, true) })\n}\n\nfunc TestNRGSnapshotRecovery(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 0, pindex: 0, entries: entries})\n\n\t// Store one entry.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.applied, 0)\n\n\t// Apply it.\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Install the snapshot.\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\n\t// Restoring the snapshot should not up applied, because the apply queue is async.\n\tn.pindex, n.commit, n.processed, n.applied = 0, 0, 0, 0\n\tn.setupLastSnapshot()\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.processed, 0)\n\trequire_Equal(t, n.applied, 0)\n}\n\nfunc TestNRGKeepRunningOnServerShutdown(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tn.RLock()\n\ts := n.s\n\twal := n.wal.(*memStore)\n\tn.RUnlock()\n\n\tn.wg.Add(1)\n\ts.startGoRoutine(n.run, nil)\n\n\ts.running.Store(false)\n\ttime.Sleep(time.Second)\n\n\twal.mu.RLock()\n\tmsgs := wal.msgs\n\twal.mu.RUnlock()\n\trequire_NotNil(t, msgs)\n\n\tn.Stop()\n\tn.WaitForStop()\n\n\twal.mu.RLock()\n\tmsgs = wal.msgs\n\twal.mu.RUnlock()\n\trequire_True(t, msgs == nil)\n}\n\nfunc TestNRGVoteResponseEncoding(t *testing.T) {\n\tvr := &voteResponse{term: 1, peer: \"S1Nunr6R\"}\n\tvr.granted, vr.empty = false, false\n\tres := vr.encode()\n\trequire_Equal(t, res[16], 0)\n\trequire_True(t, reflect.DeepEqual(decodeVoteResponse(res), vr))\n\n\tvr.granted, vr.empty = true, false\n\tres = vr.encode()\n\trequire_Equal(t, res[16], 1)\n\trequire_True(t, reflect.DeepEqual(decodeVoteResponse(res), vr))\n\n\tvr.granted, vr.empty = false, true\n\tres = vr.encode()\n\trequire_Equal(t, res[16], 2)\n\trequire_True(t, reflect.DeepEqual(decodeVoteResponse(res), vr))\n\n\tvr.granted, vr.empty = true, true\n\tres = vr.encode()\n\trequire_Equal(t, res[16], 3)\n\trequire_True(t, reflect.DeepEqual(decodeVoteResponse(res), vr))\n}\n\nfunc TestNRGInitializeAndScaleUp(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 0, pindex: 0, entries: entries})\n\n\trequire_True(t, n.initializing)\n\tn.scaleUp = true\n\tn.SetObserver(true)\n\trequire_Equal(t, n.term, 0)\n\n\tvoteReply := \"$TEST\"\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(voteReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Votes on a new leader, and resets notion of \"empty vote\" to ease getting quorum.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 1, candidate: nats0, reply: voteReply}))\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.vote, nats0)\n\trequire_True(t, n.initializing)\n\trequire_True(t, n.scaleUp)\n\trequire_True(t, n.observer)\n\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvr := decodeVoteResponse(msg.Data)\n\trequire_True(t, vr.granted)\n\trequire_False(t, vr.empty)\n\n\t// Processing an append entry resets scale up and puts us out of observer mode.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.initializing)\n\trequire_False(t, n.scaleUp)\n\trequire_False(t, n.observer)\n\n\t// Simulate a reset.\n\tn.resetWAL()\n\trequire_Equal(t, n.pindex, 0)\n\n\t// Vote after a WAL reset must be an \"empty vote\".\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 2, candidate: \"_random_\", reply: voteReply}))\n\trequire_Equal(t, n.term, 2)\n\trequire_Equal(t, n.vote, \"_random_\")\n\n\t// Vote is still granted, but now marked as \"empty\".\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvr = decodeVoteResponse(msg.Data)\n\trequire_True(t, vr.granted)\n\trequire_True(t, vr.empty)\n\n\t// Reset to check if snapshot on catchup can also reset this.\n\tn.initializing = true\n\tn.scaleUp = true\n\tn.SetObserver(true)\n\tsnapshotEntries := []*Entry{\n\t\tnewEntry(EntrySnapshot, nil),\n\t\tnewEntry(EntryPeerState, encodePeerState(&peerState{n.peerNames(), n.csz, n.extSt})),\n\t}\n\taeSnapshot := encode(t, &appendEntry{leader: nats0, term: 2, commit: 1, pterm: 1, pindex: 1, entries: snapshotEntries})\n\tn.createCatchup(aeSnapshot)\n\tn.processAppendEntry(aeSnapshot, n.catchup.sub)\n\trequire_False(t, n.initializing)\n\trequire_False(t, n.scaleUp)\n\trequire_False(t, n.observer)\n\n\t// Simulate a reset.\n\tn.resetWAL()\n\trequire_Equal(t, n.pindex, 0)\n\n\t// Vote when initializing but not scaling up, must also NOT be an \"empty vote\".\n\tn.initializing = true\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 3, candidate: \"_random_\", reply: voteReply}))\n\trequire_Equal(t, n.term, 3)\n\trequire_Equal(t, n.vote, \"_random_\")\n\n\t// Vote is still granted, but now marked as \"empty\".\n\tmsg, err = sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tvr = decodeVoteResponse(msg.Data)\n\trequire_True(t, vr.granted)\n\trequire_False(t, vr.empty)\n}\n\nfunc TestNRGReplayOnSnapshotSameTerm(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries})\n\n\t// Process the first append entry.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// Commit and apply.\n\trequire_NoError(t, n.applyCommit(1))\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Install snapshot.\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\tsnap, err := n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_Equal(t, snap.lastIndex, 1)\n\n\t// Process other messages.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\n\t// Replay the append entry that matches our snapshot.\n\t// This can happen as a repeated entry, or a delayed append entry after having already received it in a catchup.\n\t// Should be recognized as a replay with the same term, marked as success, and not truncate.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n}\n\nfunc TestNRGReplayOnSnapshotDifferentTerm(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 0, pindex: 0, entries: entries, lterm: 2})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 2, commit: 1, pterm: 1, pindex: 1, entries: entries, lterm: 2})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 2, commit: 1, pterm: 2, pindex: 2, entries: entries, lterm: 2})\n\n\t// Process the first append entry.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// Commit and apply.\n\trequire_NoError(t, n.applyCommit(1))\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Install snapshot.\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\tsnap, err := n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_Equal(t, snap.lastIndex, 1)\n\n\t// Reset applied to simulate having received the snapshot from\n\t// another leader, and we didn't apply yet since it's async.\n\tn.applied = 0\n\n\t// Process other messages.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\n\t// Replay the append entry that matches our snapshot.\n\t// This can happen as a repeated entry, or a delayed append entry after having already received it in a catchup.\n\t// Should be recognized as truncating back to the installed snapshot, not reset the WAL fully.\n\t// Since all is aligned after truncation, should also be able to apply the entry.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\n\t// Should now also be able to apply the third entry.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n}\n\nfunc TestNRGSizeAndApplied(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: []*Entry{newEntry(EntryNormal, esm)}})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: []*Entry{newEntry(EntryNormal, esm)}})\n\n\tvar (\n\t\tentries uint64\n\t\tbytes uint64\n\t)\n\n\t// Initially our WAL is empty.\n\tentries, bytes = n.Size()\n\trequire_Equal(t, entries, 0)\n\trequire_Equal(t, bytes, 0)\n\n\t// Store the first append entry.\n\trequire_NoError(t, n.storeToWAL(aeMsg1))\n\tentries, bytes = n.Size()\n\trequire_Equal(t, entries, 1)\n\trequire_Equal(t, bytes, 105)\n\n\t// Store the second append entry.\n\trequire_NoError(t, n.storeToWAL(aeMsg2))\n\tentries, bytes = n.Size()\n\trequire_Equal(t, entries, 2)\n\trequire_Equal(t, bytes, 210)\n\n\t// Applying should return what part of the WAL can be compacted.\n\tn.commit = 1\n\tentries, bytes = n.Applied(1)\n\trequire_Equal(t, entries, 1)\n\trequire_Equal(t, bytes, 105)\n\n\t// After applying all should return our whole WAL can be compacted.\n\tn.commit = 2\n\tentries, bytes = n.Applied(2)\n\trequire_Equal(t, entries, 2)\n\trequire_Equal(t, bytes, 210)\n\n\t// Installing a snapshot should properly correct n.papplied and n.bytes\n\tn.applied = 1 // Reset just for testing.\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\trequire_Equal(t, n.papplied, 1)\n\trequire_Equal(t, n.bytes, 105)\n\tentries, bytes = n.Size()\n\trequire_Equal(t, entries, 1)\n\trequire_Equal(t, bytes, 105)\n\n\tentries, bytes = n.Applied(2)\n\trequire_Equal(t, entries, 1)\n\trequire_Equal(t, bytes, 105)\n\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\trequire_Equal(t, n.papplied, 2)\n\trequire_Equal(t, n.bytes, 0)\n\tentries, bytes = n.Size()\n\trequire_Equal(t, entries, 0)\n\trequire_Equal(t, bytes, 0)\n}\n\nfunc TestNRGIgnoreEntryAfterCanceledCatchup(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\n\tcsub := n.catchup.sub\n\tn.cancelCatchup()\n\n\t// Catchup was canceled, a message on this canceled catchup should not be stored.\n\tn.processAppendEntry(aeMsg1, csub)\n\trequire_Equal(t, n.pindex, 0)\n}\n\nfunc TestNRGDelayedMessagesAfterCatchupDontCountTowardQuorum(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\taeReply := \"$TEST\"\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries, reply: aeReply})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries, reply: aeReply})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 2, entries: entries, reply: aeReply})\n\n\t// Triggers catchup.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_True(t, n.catchup != nil)\n\n\t// Catchup runs partially.\n\tn.processAppendEntry(aeMsg1, n.catchup.sub)\n\tn.processAppendEntry(aeMsg2, n.catchup.sub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(aeReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// We now receive delayed \"real-time\" messages.\n\t// The first message needs to be a copy, because we've committed it before and returned it to the pool.\n\taeMsg1Copy := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries, reply: aeReply})\n\tn.processAppendEntry(aeMsg1Copy, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\t// Should NOT reply \"success\", otherwise we would wrongfully provide quorum while not having an up-to-date log.\n\t_, err = sub.NextMsg(500 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\t// Should NOT reply \"success\", otherwise we would wrongfully provide quorum while not having an up-to-date log.\n\t_, err = sub.NextMsg(500 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\t// Should reply \"success\", this is the latest message.\n\tmsg, err := sub.NextMsg(500 * time.Millisecond)\n\trequire_NoError(t, err)\n\tar := decodeAppendEntryResponse(msg.Data)\n\trequire_Equal(t, ar.index, 3)\n\trequire_True(t, ar.success)\n\trequire_Equal(t, msg.Reply, _EMPTY_)\n}\n\nfunc TestNRGStepdownWithHighestTermDuringCatchup(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, lterm: 10, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, lterm: 20, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\n\t// Need to store the message, stepdown, and up term.\n\tn.switchToCandidate()\n\trequire_Equal(t, n.term, 1)\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.term, 10)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// Need to store the message, stepdown, and up term.\n\tn.switchToLeader()\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.term, 20)\n\trequire_Equal(t, n.pindex, 2)\n}\n\nfunc TestNRGTruncateOnStartup(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\ts := c.servers[0] // RunBasicJetStreamServer not available\n\tdefer c.shutdown()\n\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{StoreDir: storeDir, BlockSize: defaultMediumBlockSize, AsyncFlush: false, srv: s}\n\tscfg := StreamConfig{Name: \"RAFT\", Storage: FileStorage}\n\tfs, err := newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\n\tcfg := &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\n\terr = s.bootstrapRaftNode(cfg, nil, false)\n\trequire_NoError(t, err)\n\tn, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\n\t// Store two messages the normal way.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\n\tstate := n.wal.State()\n\trequire_Equal(t, state.Msgs, 2)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 2)\n\trequire_Equal(t, state.NumDeleted, 0)\n\n\t// Simulate a truncation that's only performed halfway, and we got hard killed at some point.\n\tremoved, err := n.wal.RemoveMsg(2)\n\trequire_True(t, removed)\n\trequire_NoError(t, err)\n\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 2)\n\trequire_Equal(t, state.NumDeleted, 1)\n\n\t// Restart.\n\tn.Stop()\n\tn.WaitForStop()\n\trequire_NoError(t, fs.Stop())\n\tfs, err = newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\tcfg = &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Should truncate the WAL on startup, the message was removed.\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 1)\n\trequire_Equal(t, state.NumDeleted, 0)\n\n\t// Store an invalid append entry manually.\n\taeMsg2.pindex = 0\n\taeMsg2 = encode(t, aeMsg2)\n\t_, _, err = n.wal.StoreMsg(_EMPTY_, nil, aeMsg2.buf, 0)\n\trequire_NoError(t, err)\n\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 2)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 2)\n\trequire_Equal(t, state.NumDeleted, 0)\n\n\t// Restart.\n\tn.Stop()\n\tn.WaitForStop()\n\trequire_NoError(t, fs.Stop())\n\tfs, err = newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\tcfg = &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Should truncate the WAL on startup, the append entry is invalid.\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 1)\n\trequire_Equal(t, state.FirstSeq, 1)\n\trequire_Equal(t, state.LastSeq, 1)\n\trequire_Equal(t, state.NumDeleted, 0)\n}\n\nfunc TestNRGLeaderCatchupHandling(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 2, entries: entries})\n\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\n\tn.switchToLeader()\n\n\tcatchupReply := \"$TEST\"\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(catchupReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Simulate a follower that's up-to-date with only the first message.\n\tn.catchupFollower(&appendEntryResponse{success: false, term: 1, index: 1, reply: catchupReply})\n\n\t// Should receive all messages the leader knows up to this point.\n\tmsg, err := sub.NextMsg(500 * time.Millisecond)\n\trequire_NoError(t, err)\n\tae, err := decodeAppendEntry(msg.Data, nil, _EMPTY_)\n\trequire_NoError(t, err)\n\trequire_Equal(t, ae.pterm, 1)\n\trequire_Equal(t, ae.pindex, 1)\n\n\tmsg, err = sub.NextMsg(500 * time.Millisecond)\n\trequire_NoError(t, err)\n\tae, err = decodeAppendEntry(msg.Data, nil, _EMPTY_)\n\trequire_NoError(t, err)\n\trequire_Equal(t, ae.pterm, 1)\n\trequire_Equal(t, ae.pindex, 2)\n}\n\nfunc TestNRGNewEntriesFromOldLeaderResetsWALDuringCatchup(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, lterm: 20, term: 20, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, lterm: 20, term: 20, commit: 0, pterm: 20, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, lterm: 20, term: 20, commit: 0, pterm: 20, pindex: 2, entries: entries})\n\n\taeReply := \"$TEST\"\n\taeMsg1Fork := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries, reply: aeReply})\n\taeMsg2Fork := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\n\t// Trigger a catchup.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\tvalidateCatchup := func() {\n\t\tt.Helper()\n\t\trequire_True(t, n.catchup != nil)\n\t\trequire_Equal(t, n.catchup.cterm, 20)\n\t\trequire_Equal(t, n.catchup.cindex, 1)\n\t}\n\tvalidateCatchup()\n\n\t// Catchup the first missed entry.\n\tcsub := n.catchup.sub\n\tn.processAppendEntry(aeMsg1, csub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.pterm, 20)\n\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(aeReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Would previously stall the catchup and restart it with a previous leader.\n\tn.catchup.pindex = aeMsg1.pindex + 1\n\tn.catchup.active = time.Time{}\n\tn.processAppendEntry(aeMsg1Fork, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.pterm, 20)\n\tvalidateCatchup()\n\n\t// Should reply we have a higher term, prompting the server to step down.\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tar := decodeAppendEntryResponse(msg.Data)\n\trequire_False(t, ar.success)\n\trequire_Equal(t, ar.index, 1)\n\trequire_Equal(t, ar.term, 20)\n\n\t// Would previously reset the WAL.\n\tn.processAppendEntry(aeMsg2Fork, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.pterm, 20)\n\tvalidateCatchup()\n\n\t// Now the catchup should continue, undisturbed by an old leader sending append entries.\n\tn.processAppendEntry(aeMsg2, csub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.pterm, 20)\n\trequire_True(t, n.catchup == nil)\n\n\t// A remaining catchup entry can still be ingested, even if the catchup state itself is gone.\n\tn.processAppendEntry(aeMsg3, csub)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.pterm, 20)\n}\n\nfunc TestNRGProcessed(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 2, entries: entries})\n\taeMsg4 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 4, pterm: 1, pindex: 3, entries: entries})\n\taeMsg5 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 5, pterm: 1, pindex: 4, entries: entries})\n\n\t// Store three entries.\n\tfor i, aeMsg := range []*appendEntry{aeMsg1, aeMsg2, aeMsg3} {\n\t\tn.processAppendEntry(aeMsg, n.aesub)\n\t\trequire_Equal(t, n.pindex, uint64(i+1))\n\t\trequire_Equal(t, n.commit, uint64(i+1))\n\t\trequire_Equal(t, n.processed, 0)\n\t\trequire_Equal(t, n.applied, 0)\n\t}\n\n\t// n.Processed can move both n.processed and n.applied up, with different values.\n\tn.Processed(1, 0)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.processed, 1)\n\trequire_Equal(t, n.applied, 0)\n\n\tn.Processed(2, 1)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.processed, 2)\n\trequire_Equal(t, n.applied, 1)\n\n\t// n.Applied moves both n.processed and n.applied up to the same value.\n\tn.Applied(3)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.processed, 3)\n\trequire_Equal(t, n.applied, 3)\n\n\t// Store the remaining messages.\n\tfor i, aeMsg := range []*appendEntry{aeMsg4, aeMsg5} {\n\t\tn.processAppendEntry(aeMsg, n.aesub)\n\t\trequire_Equal(t, n.pindex, uint64(i+4))\n\t\trequire_Equal(t, n.commit, uint64(i+4))\n\t\trequire_Equal(t, n.processed, 3)\n\t\trequire_Equal(t, n.applied, 3)\n\t}\n\n\t// An invalid processed call with a higher applied should be clamped back down to the processed index.\n\tn.Processed(4, 5)\n\trequire_Equal(t, n.pindex, 5)\n\trequire_Equal(t, n.commit, 5)\n\trequire_Equal(t, n.processed, 4)\n\trequire_Equal(t, n.applied, 4)\n}\n\nfunc TestNRGSendAppendEntryNotLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\trequire_Equal(t, n.State(), Follower)\n\trequire_Equal(t, n.pindex, 0)\n\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(n.asubj)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// sendAppendEntry acquires the lock by itself, so it must also protect against us not being leader anymore.\n\tn.sendAppendEntry([]*Entry{newEntry(EntryNormal, nil)})\n\n\t// We're a follower, so we should not be able to send or store a message.\n\trequire_Equal(t, n.pindex, 0)\n\tmsg, err := sub.NextMsg(250 * time.Millisecond)\n\trequire_Error(t, err, nats.ErrTimeout)\n\trequire_True(t, msg == nil)\n}\n\nfunc TestNRGDrainAndReplaySnapshot(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 2, entries: entries})\n\taeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: nil})\n\taeMsg4 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 4, pterm: 1, pindex: 4, entries: nil})\n\n\t// Stage some entries as normal.\n\trequire_Len(t, n.apply.len(), 0)\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\tn.processAppendEntry(aeHeartbeat1, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Len(t, n.apply.len(), 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.hcommit, 0)\n\n\t// Just a sanity-check, if we have no snapshot then this should fail.\n\trequire_False(t, n.DrainAndReplaySnapshot())\n\trequire_Len(t, n.apply.len(), 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.hcommit, 0)\n\n\t// Simulate this server processing a snapshot that requires upper layer catchup.\n\t// This catchup timed out and we would then call into DrainAndReplaySnapshot.\n\tsnap := []byte(\"snapshot\")\n\tn.Applied(1)\n\trequire_NoError(t, n.InstallSnapshot(snap, false))\n\n\trequire_True(t, n.DrainAndReplaySnapshot())\n\trequire_True(t, n.paused)\n\trequire_Len(t, n.apply.len(), 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.hcommit, 3)\n\n\t// Simulate snapshot processing being successful and restoring the apply queue when resuming.\n\tn.ResumeApply()\n\trequire_False(t, n.paused)\n\trequire_Len(t, n.apply.len(), 3)\n\trequire_Equal(t, n.commit, 3)\n\trequire_Equal(t, n.hcommit, 0)\n\n\t// Now simulate another case where the snapshot processing times out multiple times.\n\trequire_True(t, n.DrainAndReplaySnapshot())\n\trequire_True(t, n.paused)\n\trequire_Len(t, n.apply.len(), 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.hcommit, 3)\n\n\t// Could receive new messages in the meantime and need to keep tracking the highest known commit properly.\n\tn.processAppendEntry(aeMsg4, n.aesub)\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.pindex, 4)\n\trequire_True(t, n.paused)\n\trequire_Len(t, n.apply.len(), 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.hcommit, 4)\n\n\t// Replaying again should preserve the highest known commit.\n\trequire_True(t, n.DrainAndReplaySnapshot())\n\trequire_True(t, n.paused)\n\trequire_Len(t, n.apply.len(), 1)\n\trequire_Equal(t, n.commit, 1)\n\trequire_Equal(t, n.hcommit, 4)\n\n\t// Resume applies, and ensure correct state.\n\tn.ResumeApply()\n\trequire_False(t, n.paused)\n\trequire_Len(t, n.apply.len(), 4)\n\trequire_Equal(t, n.commit, 4)\n\trequire_Equal(t, n.hcommit, 0)\n}\n\nfunc TestNRGTrackPeerActive(t *testing.T) {\n\t// The leader should track timestamps for all peers.\n\t// Each follower should only track the leader, otherwise we would get outdated timestamps.\n\tcheckLastSeen := func(peers map[string]RaftzGroupPeer) {\n\t\tfor _, peer := range peers {\n\t\t\tif peer.LastSeen == _EMPTY_ {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\telapsed, err := time.ParseDuration(peer.LastSeen)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_LessThan(t, elapsed, time.Second)\n\t\t}\n\t}\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tml := c.leader()\n\trs := c.randomNonLeader()\n\tvar preferred *Server\n\tfor _, s := range c.servers {\n\t\tif s == ml || s == rs {\n\t\t\tcontinue\n\t\t}\n\t\tpreferred = s\n\t\tbreak\n\t}\n\trequire_NotNil(t, preferred)\n\n\ttime.Sleep(2 * time.Second)\n\tbefore := (*rs.Raftz(&RaftzOptions{}))[DEFAULT_SYSTEM_ACCOUNT][defaultMetaGroupName].Peers\n\tcheckLastSeen(before)\n\n\tjs := ml.getJetStream()\n\tn := js.getMetaGroup()\n\trequire_NoError(t, n.StepDown(preferred.NodeName()))\n\n\ttime.Sleep(2 * time.Second)\n\tafter := (*rs.Raftz(&RaftzOptions{}))[DEFAULT_SYSTEM_ACCOUNT][defaultMetaGroupName].Peers\n\tcheckLastSeen(after)\n}\n\nfunc TestNRGLostQuorum(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tn.Lock()\n\tn.addPeer(nats1)\n\t// Ignore scale-up grace period so this\n\t// test focuses on lost-quorum tracking.\n\tn.lsut = time.Time{}\n\tn.Unlock()\n\n\trequire_Equal(t, n.State(), Follower)\n\trequire_False(t, n.Quorum())\n\trequire_True(t, n.lostQuorum())\n\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// Respond to a vote request.\n\tsub, err := nc.Subscribe(n.vsubj, func(m *nats.Msg) {\n\t\treq := decodeVoteRequest(m.Data, m.Reply)\n\t\tresp := voteResponse{term: req.term, peer: nats0, granted: true}\n\t\tm.Respond(resp.encode())\n\t})\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Switch to candidate and make sure we properly track the peer as active.\n\tn.switchToCandidate()\n\trequire_Equal(t, n.State(), Candidate)\n\trequire_False(t, n.Quorum())\n\trequire_True(t, n.lostQuorum())\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.Unlock()\n\n\tn.runAsCandidate()\n\trequire_Equal(t, n.State(), Leader)\n\trequire_True(t, n.Quorum())\n\trequire_False(t, n.lostQuorum())\n}\n\nfunc TestNRGParallelCatchupRollback(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\taeReply := \"$TEST\"\n\tnc, err := nats.Connect(n.s.ClientURL(), nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(aeReply)\n\trequire_NoError(t, err)\n\tdefer sub.Drain()\n\trequire_NoError(t, nc.Flush())\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, lterm: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, lterm: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 2, entries: nil, reply: aeReply})\n\n\t// Trigger a catchup.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 0)\n\trequire_NotNil(t, n.catchup)\n\trequire_Equal(t, n.catchup.cterm, aeMsg2.pterm)\n\trequire_Equal(t, n.catchup.cindex, aeMsg2.pindex)\n\tcsub := n.catchup.sub\n\n\t// Receive the missed messages.\n\tn.processAppendEntry(aeMsg1, csub)\n\trequire_Equal(t, n.pindex, 1)\n\tn.processAppendEntry(aeMsg2, csub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_True(t, n.catchup == nil)\n\n\t// Should respond to the heartbeat and allow the leader to commit.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\tmsg, err := sub.NextMsg(time.Second)\n\trequire_NoError(t, err)\n\tar := decodeAppendEntryResponse(msg.Data)\n\trequire_NotNil(t, ar)\n\trequire_True(t, ar.success)\n\trequire_Equal(t, ar.term, aeHeartbeat.term)\n\trequire_Equal(t, ar.index, aeHeartbeat.pindex)\n\n\t// Now replay a message that was already received as a catchup entry.\n\t// Likely due to running multiple catchups in parallel.\n\t// Since our WAL is already ahead, we should not truncate based on this.\n\tn.processAppendEntry(aeMsg1, csub)\n\trequire_Equal(t, n.pindex, 2)\n}\n\nfunc TestNRGReportLeaderAfterNoopEntry(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\trequire_Equal(t, n.State(), Follower)\n\trequire_Equal(t, n.term, 0)\n\trequire_False(t, n.Leader())\n\n\tn.switchToCandidate()\n\trequire_Equal(t, n.State(), Candidate)\n\trequire_Equal(t, n.term, 1)\n\trequire_False(t, n.Leader())\n\n\t// Switching to leader will put us into Leader state,\n\t// but we're not necessarily an up-to-date leader yet.\n\tn.switchToLeader()\n\trequire_Equal(t, n.State(), Leader)\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.pindex, 1) // Should've sent a NOOP-entry to establish leadership.\n\trequire_Equal(t, n.applied, 0)\n\trequire_False(t, n.Leader())\n\n\t// Once we commit and apply the final entry, we should starting to report we're leader.\n\tn.commit = 1\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\trequire_True(t, n.Leader())\n}\n\nfunc TestNRGSendSnapshotInstallsSnapshot(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.addPeer(nats1)\n\tn.Unlock()\n\n\trequire_Equal(t, n.pindex, 0)\n\trequire_Equal(t, n.snapfile, _EMPTY_)\n\n\t// Switch to candidate, to become leader.\n\trequire_Equal(t, n.term, 0)\n\tn.switchToCandidate()\n\trequire_Equal(t, n.term, 1)\n\n\t// When switching to leader a NOOP-entry is sent.\n\tn.switchToLeader()\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.snapfile, _EMPTY_)\n\trequire_NoError(t, n.applyCommit(1))\n\trequire_Equal(t, n.snapfile, _EMPTY_)\n\n\t// On scaleup, we send a snapshot.\n\trequire_NoError(t, n.SendSnapshot([]byte(\"snapshot_data\")))\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.snapfile, _EMPTY_)\n\n\t// When applying the entry, the sent snapshot should be installed.\n\trequire_NoError(t, n.applyCommit(2))\n\trequire_NotEqual(t, n.snapfile, _EMPTY_)\n\n\tsnap, err := n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_NotNil(t, snap)\n\trequire_Equal(t, snap.lastTerm, 1)\n\trequire_Equal(t, snap.lastIndex, 1)\n\trequire_Equal(t, string(snap.data), \"snapshot_data\")\n\n\t// Draining and replaying the snapshot should work.\n\trequire_True(t, n.DrainAndReplaySnapshot())\n}\n\nfunc TestNRGQuorumAfterLeaderStepdown(t *testing.T) {\n\torigMinTimeout, origMaxTimeout, origHBInterval := minElectionTimeout, maxElectionTimeout, hbInterval\n\tminElectionTimeout, maxElectionTimeout, hbInterval = minElectionTimeoutDefault, maxElectionTimeoutDefault, hbIntervalDefault\n\tdefer func() {\n\t\tminElectionTimeout, maxElectionTimeout, hbInterval = origMinTimeout, origMaxTimeout, origHBInterval\n\t}()\n\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\tn.Lock()\n\tn.addPeer(nats0)\n\tn.addPeer(nats1)\n\tn.Unlock()\n\n\t// Become leader.\n\tn.switchToCandidate()\n\tn.switchToLeader()\n\n\t// Should not report having quorum, unless at least 1 peer is seen.\n\trequire_False(t, n.Quorum())\n\tn.s.nodeToInfo.Store(nats0, nodeInfo{})\n\trequire_NoError(t, n.trackPeer(nats0))\n\trequire_True(t, n.Quorum())\n\tn.s.nodeToInfo.Store(nats1, nodeInfo{})\n\trequire_NoError(t, n.trackPeer(nats1))\n\trequire_True(t, n.Quorum())\n\trequire_Len(t, len(n.peers), 3)\n\tfor _, ps := range n.peers {\n\t\tps.kp = true\n\t}\n\n\t// If we hand off leadership to another server, we should\n\t// still be reporting we have quorum.\n\trequire_NoError(t, n.StepDown())\n\trequire_True(t, n.Quorum())\n\trequire_Len(t, len(n.peers), 3)\n\tfor peer, ps := range n.peers {\n\t\tif peer == n.id {\n\t\t\tcontinue\n\t\t}\n\t\t// All peer timestamps should be preserved.\n\t\trequire_False(t, ps.ts.IsZero())\n\t}\n\n\t// After our new leader comes online, we should still have quorum,\n\t// but the other follower's timestamp should be cleared.\n\tn.updateLeader(nats0)\n\trequire_Equal(t, n.leader, nats0)\n\trequire_True(t, n.Quorum())\n\trequire_Len(t, len(n.peers), 3)\n\tfor peer, ps := range n.peers {\n\t\tif peer == n.id {\n\t\t\tcontinue\n\t\t}\n\t\tif peer == nats0 {\n\t\t\trequire_False(t, ps.ts.IsZero()) // Leader is preserved.\n\t\t} else {\n\t\t\trequire_True(t, ps.ts.IsZero()) // Other follower is cleared.\n\t\t}\n\t}\n}\n\nfunc TestNRGNoLogResetOnCorruptedSendToFollower(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 2)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 2, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader().(*stateAdder)\n\tfollower := rg.nonLeader().(*stateAdder)\n\n\tleaderrg := leader.node().(*raft)\n\tfollowerrg := follower.node().(*raft)\n\n\tfor i := range int64(10) {\n\t\tleader.proposeDelta(i + 1)\n\t\ttime.Sleep(50 * time.Millisecond) // ... for multiple AEs.\n\t}\n\n\t// Snapshot and compact.\n\trg.waitOnTotal(t, 55)\n\tleader.snapshot(t)\n\n\t// Above snapshot should have resulted in compaction of the WAL.\n\tvar ss StreamState\n\tleaderrg.wal.FastState(&ss)\n\trequire_Equal(t, ss.Msgs, 0)\n\n\t// Stop the follower, we'll flatten their state so they have to\n\t// request a snapshot.\n\trequire_NoError(t, followerrg.wal.Truncate(0))\n\tfollower.stop()\n\n\t// Now we're going to subtly corrupt the snapshot on the leader.\n\tstat, err := os.Stat(leaderrg.snapfile)\n\trequire_NoError(t, err)\n\trequire_NoError(t, os.Truncate(leaderrg.snapfile, stat.Size()-1))\n\n\t// Now we'll bring the follower back. It should request a snapshot\n\t// from the leader. Previously this would have caused the leader to\n\t// blow away the entire log, but in reality we will probably just\n\t// install a new snapshot at some point soon anyway.\n\tfollower.restart()\n\tc.waitOnAllCurrent()\n\tleaderrg.wal.FastState(&ss)\n\trequire_NotEqual(t, ss.LastSeq, 0)\n}\n\nfunc TestNRGTruncateLogWithMisalignedSnapshotGap(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\ts := c.servers[0] // RunBasicJetStreamServer not available\n\tdefer c.shutdown()\n\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{StoreDir: storeDir, BlockSize: defaultMediumBlockSize, AsyncFlush: false, srv: s}\n\tscfg := StreamConfig{Name: \"RAFT\", Storage: FileStorage}\n\tfs, err := newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\n\tcfg := &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\n\terr = s.bootstrapRaftNode(cfg, nil, false)\n\trequire_NoError(t, err)\n\tn, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\n\tfor i, ae := range []*appendEntry{aeMsg1, aeMsg2, aeMsg3} {\n\t\tn.processAppendEntry(ae, n.aesub)\n\t\trequire_Equal(t, n.pindex, uint64(i+1))\n\t}\n\n\t// Manually call back down to applied, and then snapshot.\n\tn.Applied(1)\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\n\tstate := n.wal.State()\n\trequire_Equal(t, state.FirstSeq, 2)\n\trequire_Equal(t, state.LastSeq, 3)\n\n\t// Manually compact the WAL further so it doesn't align anymore with the snapshot.\n\t_, err = n.wal.Compact(3)\n\trequire_NoError(t, err)\n\tstate = n.wal.State()\n\trequire_Equal(t, state.FirstSeq, 3)\n\trequire_Equal(t, state.LastSeq, 3)\n\n\t// Restart.\n\tn.Stop()\n\tn.WaitForStop()\n\trequire_NoError(t, fs.Stop())\n\tfs, err = newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\tcfg = &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// The gap between the snapshot and the WAL should be detected, and the WAL should truncate.\n\t// Can't continue normally with missing entries.\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 0)\n\trequire_Equal(t, state.FirstSeq, 2)\n\trequire_Equal(t, state.LastSeq, 1)\n\trequire_Equal(t, n.pindex, 1)\n\n\t// Should be able to re-populate the missing messages.\n\t// Need to re-encode them, though, since they would have been returned to the pool before.\n\taeMsg2 = encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 = encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\tfor i, ae := range []*appendEntry{aeMsg2, aeMsg3} {\n\t\tn.processAppendEntry(ae, n.aesub)\n\t\trequire_Equal(t, n.pindex, uint64(i+2))\n\t}\n}\n\nfunc TestNRGTruncateLogWithMissingSnapshot(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\ts := c.servers[0] // RunBasicJetStreamServer not available\n\tdefer c.shutdown()\n\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{StoreDir: storeDir, BlockSize: defaultMediumBlockSize, AsyncFlush: false, srv: s}\n\tscfg := StreamConfig{Name: \"RAFT\", Storage: FileStorage}\n\tfs, err := newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\n\tcfg := &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\n\terr = s.bootstrapRaftNode(cfg, nil, false)\n\trequire_NoError(t, err)\n\tn, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\n\tfor i, ae := range []*appendEntry{aeMsg1, aeMsg2, aeMsg3} {\n\t\tn.processAppendEntry(ae, n.aesub)\n\t\trequire_Equal(t, n.pindex, uint64(i+1))\n\t}\n\n\t// Manually simulate a snapshot being made, only compacting but not actually installing a snapshot.\n\t_, err = n.wal.Compact(2)\n\trequire_NoError(t, err)\n\n\tstate := n.wal.State()\n\trequire_Equal(t, state.FirstSeq, 2)\n\trequire_Equal(t, state.LastSeq, 3)\n\n\t// Restart.\n\tn.Stop()\n\tn.WaitForStop()\n\trequire_NoError(t, fs.Stop())\n\tfs, err = newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\tcfg = &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// The gap between the snapshot and the WAL should be detected, and the WAL should truncate.\n\t// Can't continue normally with missing entries.\n\tstate = n.wal.State()\n\trequire_Equal(t, state.Msgs, 0)\n\trequire_Equal(t, state.FirstSeq, 0)\n\trequire_Equal(t, state.LastSeq, 0)\n\trequire_Equal(t, n.pindex, 0)\n}\n\n// This is a RaftChainOfBlocks test where a block is proposed and then we wait for all replicas to apply it before\n// proposing the next one.\n// The test may fail if:\n// - Replicas hash diverge\n// - One replica never applies the N-th block applied by the rest\n// - The given number of blocks cannot be applied within some amount of time\nfunc TestNRGChainOfBlocksRunInLockstep(t *testing.T) {\n\tconst iterations = 50\n\tconst applyTimeout = 3 * time.Second\n\tconst testTimeout = iterations * time.Second\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newRaftChainStateMachine)\n\trg.waitOnLeader()\n\n\ttestTimer := time.NewTimer(testTimeout)\n\n\tfor iteration := uint64(1); iteration <= iterations; iteration++ {\n\t\tselect {\n\t\tcase <-testTimer.C:\n\t\t\tt.Fatalf(\"Timeout, completed %d/%d iterations\", iteration-1, iterations)\n\t\tdefault:\n\t\t\t// Continue\n\t\t}\n\n\t\t// Propose the next block (this test assumes the proposal always goes through)\n\t\trg.randomMember().(*RCOBStateMachine).proposeBlock()\n\n\t\tvar currentHash string\n\n\t\t// Wait on participants to converge\n\t\tcheckFor(t, applyTimeout, 500*time.Millisecond, func() error {\n\t\t\tvar previousNodeName string\n\t\t\tvar previousNodeHash string\n\t\t\tfor _, sm := range rg {\n\t\t\t\tstateMachine := sm.(*RCOBStateMachine)\n\t\t\t\tnodeName := fmt.Sprintf(\n\t\t\t\t\t\"%s/%s\",\n\t\t\t\t\tstateMachine.server().Name(),\n\t\t\t\t\tstateMachine.node().ID(),\n\t\t\t\t)\n\n\t\t\t\trunning, blocksCount, hash := stateMachine.getCurrentHash()\n\t\t\t\t// All nodes always running\n\t\t\t\tif !running {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s is not running\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Node is behind\n\t\t\t\tif blocksCount != iteration {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s applied %d blocks out of %d expected\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\titeration,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Make sure hash is not empty\n\t\t\t\tif hash == \"\" {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s has empty hash after applying %d blocks\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Check against previous node hash, unless this is the first node and we don't have anyone to compare\n\t\t\t\tif previousNodeHash != \"\" && previousNodeHash != hash {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"hash mismatch after %d blocks: %s hash: %s != %s hash: %s\",\n\t\t\t\t\t\titeration,\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\thash,\n\t\t\t\t\t\tpreviousNodeName,\n\t\t\t\t\t\tpreviousNodeHash,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Set node name and hash for next node to compare against\n\t\t\t\tpreviousNodeName, previousNodeHash = nodeName, hash\n\n\t\t\t}\n\t\t\t// All replicas applied the last block and their hashes match\n\t\t\tcurrentHash = previousNodeHash\n\t\t\treturn nil\n\t\t})\n\t\tif RCOBOptions.verbose {\n\t\t\tt.Logf(\n\t\t\t\t\"Verified chain hash %s for %d/%d nodes after %d/%d iterations\",\n\t\t\t\tcurrentHash,\n\t\t\t\tlen(rg),\n\t\t\t\tlen(rg),\n\t\t\t\titeration,\n\t\t\t\titerations,\n\t\t\t)\n\t\t}\n\t}\n}\n\n// This is a RaftChainOfBlocks test where one of the replicas is stopped before proposing a short burst of blocks.\n// Upon resuming the replica, we check it is able to catch up to the rest.\n// The test may fail if:\n// - Replicas hash diverge\n// - One replica never applies the N-th block applied by the rest\n// - The given number of blocks cannot be applied within some amount of time\nfunc TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) {\n\tconst iterations = 50\n\tconst blocksPerIteration = 3\n\tconst applyTimeout = 3 * time.Second\n\tconst testTimeout = 2 * iterations * time.Second\n\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newRaftChainStateMachine)\n\trg.waitOnLeader()\n\n\ttestTimer := time.NewTimer(testTimeout)\n\thighestBlockSeen := uint64(0)\n\n\tfor iteration := uint64(1); iteration <= iterations; iteration++ {\n\n\t\tselect {\n\t\tcase <-testTimer.C:\n\t\t\tt.Fatalf(\"Timeout, completed %d/%d iterations\", iteration-1, iterations)\n\t\tdefault:\n\t\t\t// Continue\n\t\t}\n\n\t\t// Stop a random node\n\t\tstoppedNode := rg.randomMember()\n\t\tleader := stoppedNode.node().Leader()\n\t\tstoppedNode.stop()\n\n\t\t// Snapshot a random (non-stopped) node\n\t\tsnapshotNode := rg.randomMember()\n\t\tfor snapshotNode == stoppedNode {\n\t\t\tsnapshotNode = rg.randomMember()\n\t\t}\n\t\tsnapshotNode.(*RCOBStateMachine).createSnapshot()\n\n\t\tif RCOBOptions.verbose {\n\t\t\tt.Logf(\n\t\t\t\t\"Iteration %d/%d: stopping node: %s/%s (leader: %v)\",\n\t\t\t\titeration,\n\t\t\t\titerations,\n\t\t\t\tstoppedNode.server().Name(),\n\t\t\t\tstoppedNode.node().ID(),\n\t\t\t\tleader,\n\t\t\t)\n\t\t}\n\n\t\t// Propose some new blocks\n\t\trg.waitOnLeader()\n\t\tfor i := 0; i < blocksPerIteration; i++ {\n\t\t\tproposer := rg.randomMember()\n\t\t\t// Pick again if we randomly chose the stopped node\n\t\t\tfor proposer == stoppedNode {\n\t\t\t\tproposer = rg.randomMember()\n\t\t\t}\n\n\t\t\tproposer.(*RCOBStateMachine).proposeBlock()\n\t\t}\n\n\t\t// Restart the stopped node\n\t\tstoppedNode.restart()\n\n\t\t// Wait on participants to converge\n\t\texpectedBlocks := iteration * blocksPerIteration\n\t\tvar currentHash string\n\t\tcheckFor(t, applyTimeout, 250*time.Millisecond, func() error {\n\t\t\tvar previousNodeName string\n\t\t\tvar previousNodeHash string\n\t\t\tfor _, sm := range rg {\n\t\t\t\tstateMachine := sm.(*RCOBStateMachine)\n\t\t\t\tnodeName := fmt.Sprintf(\n\t\t\t\t\t\"%s/%s\",\n\t\t\t\t\tstateMachine.server().Name(),\n\t\t\t\t\tstateMachine.node().ID(),\n\t\t\t\t)\n\t\t\t\trunning, blocksCount, currentHash := stateMachine.getCurrentHash()\n\t\t\t\t// Track the highest block seen by any replica\n\t\t\t\tif blocksCount > highestBlockSeen {\n\t\t\t\t\thighestBlockSeen = blocksCount\n\t\t\t\t\t// Must check all replicas again\n\t\t\t\t\treturn fmt.Errorf(\"updated highest block to %d (%s)\", highestBlockSeen, nodeName)\n\t\t\t\t}\n\t\t\t\t// All nodes should be running\n\t\t\t\tif !running {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s not running\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Node is behind\n\t\t\t\tif blocksCount != expectedBlocks {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s applied %d blocks out of %d expected\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t\texpectedBlocks,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Make sure hash is not empty\n\t\t\t\tif currentHash == \"\" {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"node %s has empty hash after applying %d blocks\",\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\tblocksCount,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Check against previous node hash, unless this is the first node to be checked\n\t\t\t\tif previousNodeHash != \"\" && previousNodeHash != currentHash {\n\t\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\t\"hash mismatch after %d blocks: %s hash: %s != %s hash: %s\",\n\t\t\t\t\t\texpectedBlocks,\n\t\t\t\t\t\tnodeName,\n\t\t\t\t\t\tcurrentHash,\n\t\t\t\t\t\tpreviousNodeName,\n\t\t\t\t\t\tpreviousNodeHash,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\t// Set node name and hash for next node to compare against\n\t\t\t\tpreviousNodeName, previousNodeHash = nodeName, currentHash\n\t\t\t}\n\t\t\t// All is well\n\t\t\tcurrentHash = previousNodeHash\n\t\t\treturn nil\n\t\t})\n\n\t\tif RCOBOptions.verbose {\n\t\t\tt.Logf(\n\t\t\t\t\"Verified chain hash %s for %d/%d nodes after %d blocks, %d/%d iterations (%d lost proposals)\",\n\t\t\t\tcurrentHash,\n\t\t\t\tlen(rg),\n\t\t\t\tlen(rg),\n\t\t\t\thighestBlockSeen,\n\t\t\t\titeration,\n\t\t\t\titerations,\n\t\t\t\texpectedBlocks-highestBlockSeen,\n\t\t\t)\n\t\t}\n\t}\n}\n\nfunc TestNRGProposeRemovePeer(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tn := rg.leader().node()\n\trg.waitOnLeader()\n\n\tpeerId := rg.nonLeader().node().ID()\n\trequire_NoError(t, n.ProposeRemovePeer(peerId))\n\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tvar err error\n\t\tfor _, r := range rg {\n\t\t\tif len(r.node().Peers()) != 2 {\n\t\t\t\terr = errors.New(\"has not removed peer\")\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc TestNRGProposeRemovePeerConcurrent(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tn := rg.leader().node()\n\n\tlocked := rg.lockFollowers()\n\n\t// Attempt to remove the first follower, should succeed.\n\terr := n.ProposeRemovePeer(locked[0].node().ID())\n\trequire_NoError(t, err)\n\n\t// Check that membership change is in progress.\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\tif n.MembershipChangeInProgress() {\n\t\t\treturn nil\n\t\t} else {\n\t\t\treturn errors.New(\"membership not in progress\")\n\t\t}\n\t})\n\n\t// Attempt to remove the second follower, should fail.\n\terr = n.ProposeRemovePeer(locked[1].node().ID())\n\trequire_Error(t, err, errMembershipChange)\n\n\tfor _, l := range locked {\n\t\tl.node().(*raft).Unlock()\n\t}\n\n\t// Expect only one peer removal to succeed\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tvar err error\n\t\tfor _, r := range rg {\n\t\t\tif len(r.node().Peers()) != 2 {\n\t\t\t\terr = errors.New(\"has not removed peer\")\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t}\n\t\treturn err\n\t})\n}\n\nfunc TestNRGUncommittedMembershipChangeOnNewLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats2 := \"cnrtt3eg\" // \"nats-2\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.addPeer(nats2)\n\tn.Unlock()\n\n\tentries := []*Entry{newEntry(EntryRemovePeer, []byte(nats2))}\n\taeRemovePeer := encode(t, &appendEntry{leader: nats1, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\n\t// plant a EntryRemovePeer in the log\n\tn.processAppendEntry(aeRemovePeer, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// become the new leader\n\tn.term = 2\n\tn.switchToLeader()\n\n\terr := n.ProposeRemovePeer(nats1)\n\trequire_Error(t, err, errMembershipChange)\n}\n\nfunc TestNRGUncommittedMembershipChangeGetsTruncated(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\taeMsg := encode(t, &appendEntry{leader: nats1, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\tentries = []*Entry{newEntry(EntryAddPeer, []byte(nats1))}\n\taeAddPeer := encode(t, &appendEntry{leader: nats1, term: 1, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\n\t// Set up a membership change.\n\tn.processAppendEntry(aeMsg, n.aesub)\n\tn.processAppendEntry(aeAddPeer, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_True(t, n.MembershipChangeInProgress())\n\trequire_Equal(t, n.membChangeIndex, 2)\n\n\t// If the entry containing the membership change isn't truncated, it should remain in progress.\n\tn.truncateWAL(n.pterm, n.pindex)\n\trequire_True(t, n.MembershipChangeInProgress())\n\trequire_Equal(t, n.membChangeIndex, 2)\n\n\t// If the entry IS truncated, then it shouldn't be in progress anymore.\n\tn.truncateWAL(n.pterm, n.pindex-1)\n\trequire_False(t, n.MembershipChangeInProgress())\n\trequire_Equal(t, n.membChangeIndex, 0)\n}\n\nfunc TestNRGUncommittedMembershipChangeOnNewLeaderForwardedRemovePeerProposal(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats2 := \"cnrtt3eg\" // \"nats-2\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.addPeer(nats2)\n\tn.Unlock()\n\n\tentries := []*Entry{newEntry(EntryRemovePeer, []byte(nats2))}\n\taeRemovePeer := encode(t, &appendEntry{leader: nats1, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\n\t// plant a EntryRemovePeer in the log\n\tn.processAppendEntry(aeRemovePeer, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_False(t, n.Healthy())\n\n\t// become the new leader\n\tn.term = 2\n\tn.switchToLeader()\n\tn.leaderState.Store(true)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tn.runAsLeader()\n\t}()\n\tdefer wg.Wait()\n\n\tn.RLock()\n\trpsubj := n.rpsubj\n\tn.RUnlock()\n\n\tnc, _ := jsClientConnect(t, n.s, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t// Forward a peer-remove proposal to the new leader.\n\tbefore, _, _ := n.Progress()\n\trequire_True(t, n.MembershipChangeInProgress())\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t_, err := nc.Request(rpsubj, []byte(nats1), 100*time.Millisecond)\n\t\t// Wait for the server to be subscribed and either respond or time out.\n\t\tif err == nil || errors.Is(err, nats.ErrTimeout) {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t})\n\t// The forwarded peer-remove should not be accepted.\n\ttime.Sleep(200 * time.Millisecond)\n\tafter, _, _ := n.Progress()\n\trequire_Equal(t, before, after)\n\trequire_True(t, n.MembershipChangeInProgress())\n}\n\nfunc TestNRGIgnoreForwardedProposalIfNotCaughtUpLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats1 := \"yrzKKRBu\" // \"nats-1\"\n\tnats2 := \"cnrtt3eg\" // \"nats-2\"\n\n\tn.Lock()\n\tn.addPeer(nats1)\n\tn.addPeer(nats2)\n\tn.Unlock()\n\n\tn.term = 1\n\tn.switchToLeader()\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tn.runAsLeader()\n\t}()\n\tdefer wg.Wait()\n\n\tn.RLock()\n\t// Although this server is the leader, mark it as not caught up yet.\n\tn.leaderState.Store(false)\n\tpsubj := n.psubj\n\tn.RUnlock()\n\n\tnc, _ := jsClientConnect(t, n.s, nats.UserInfo(\"admin\", \"s3cr3t!\"))\n\tdefer nc.Close()\n\n\t// Forward a normal proposal to the new leader.\n\tbefore, _, _ := n.Progress()\n\tcheckFor(t, 2*time.Second, 200*time.Millisecond, func() error {\n\t\t_, err := nc.Request(psubj, nil, 100*time.Millisecond)\n\t\t// Wait for the server to be subscribed and either respond or time out.\n\t\tif err == nil || errors.Is(err, nats.ErrTimeout) {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t})\n\t// The forwarded proposal should not be accepted.\n\ttime.Sleep(200 * time.Millisecond)\n\tafter, _, _ := n.Progress()\n\trequire_Equal(t, before, after)\n}\n\nfunc TestNRGProposeRemovePeerQuorum(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader().node()\n\tfollowers := rg.followers()\n\n\trequire_True(t, len(followers) == 2)\n\n\t// Block follower 0 and remove follower 1\n\tfollowers[0].node().(*raft).Lock()\n\terr := leader.ProposeRemovePeer(followers[1].node().ID())\n\trequire_NoError(t, err)\n\n\t// Should not be able to make progress\n\ttime.Sleep(time.Second)\n\trequire_True(t, leader.MembershipChangeInProgress())\n\n\t// Unlock the other follower and expect the membership\n\t// change to eventually finish\n\tfollowers[0].node().(*raft).Unlock()\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tif leader.MembershipChangeInProgress() {\n\t\t\treturn errors.New(\"membership still in progress\")\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t})\n}\n\n// Test outline:\n// - In a R3 cluster, PeerRemove the leader and block on of\n// the followers.\n// - Verify that the membership change can't make progress,\n// the leader should not count its own ack towards quorum.\n// - Release the previously blocked follower and expect the\n// membership change to take place.\nfunc TestNRGProposeRemovePeerLeader(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader().node()\n\tfollowers := rg.followers()\n\tleaderID := leader.ID()\n\trequire_True(t, len(followers) == 2)\n\n\t// Block follower 0 and remove the leader\n\tfollowers[0].node().(*raft).Lock()\n\terr := leader.ProposeRemovePeer(leader.ID())\n\trequire_NoError(t, err)\n\n\t// Should not be able to make progress\n\ttime.Sleep(time.Second)\n\trequire_True(t, leader.MembershipChangeInProgress())\n\n\t// Unlock the follower and expect the membership\n\t// change to eventually finish\n\tfollowers[0].node().(*raft).Unlock()\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tif leader.MembershipChangeInProgress() {\n\t\t\treturn errors.New(\"membership still in progress\")\n\t\t} else {\n\t\t\treturn nil\n\t\t}\n\t})\n\n\t// Old leader steps down\n\tcheckFor(t, 10*time.Second, 200*time.Millisecond, func() error {\n\t\tnewLeader := rg.waitOnLeader()\n\t\tif newLeader.node().ID() == leaderID {\n\t\t\treturn errors.New(\"leader has not changed yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tnewLeader := rg.waitOnLeader()\n\trequire_Equal(t, leader.State(), Closed)\n\trequire_NotEqual(t, leader.ID(), newLeader.node().ID())\n\trequire_Equal(t, len(newLeader.node().Peers()), 2)\n\trequire_False(t, newLeader.node().MembershipChangeInProgress())\n}\n\nfunc TestNRGProposeRemovePeerAll(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader()\n\tfollowers := rg.followers()\n\trequire_Equal(t, len(followers), 2)\n\n\tpeers := leader.node().Peers()\n\trequire_Equal(t, len(peers), 3)\n\tfor i, follower := range followers {\n\t\trequire_NoError(t, leader.node().ProposeRemovePeer(follower.node().ID()))\n\t\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\t\tif peers = leader.node().Peers(); len(peers) == 2-i {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn errors.New(\"membership still in progress\")\n\t\t})\n\t}\n\n\tpeers = leader.node().Peers()\n\tleaderID := leader.node().ID()\n\n\t// The leader is the only one left...\n\trequire_Equal(t, len(peers), 1)\n\trequire_Equal(t, peers[0].ID, leaderID)\n\t// and we can't remove it\n\trequire_Error(t, leader.node().ProposeRemovePeer(leaderID), errRemoveLastNode)\n}\n\nfunc TestNRGLeaderResurrectsRemovedPeers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader()\n\tfollowers := rg.followers()\n\trequire_Equal(t, len(followers), 2)\n\n\t// Remove one follower\n\trequire_NoError(t, leader.node().ProposeRemovePeer(followers[0].node().ID()))\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif peers := leader.node().Peers(); len(peers) == 2 {\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.New(\"membership still in progress\")\n\t})\n\n\t// Stop the leader and restart it.\n\t// If bug is present: the leader resurrects the previously removed peer.\n\tleader.stop()\n\tfollowers[1].stop()\n\n\tleader.restart()\n\trequire_Equal(t, len(leader.node().Peers()), 2)\n\n\tfollowers[1].restart()\n\trequire_Equal(t, len(leader.node().Peers()), 2)\n}\n\nfunc TestNRGAddPeers(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\tleader := rg.waitOnLeader()\n\n\trequire_Equal(t, leader.node().ClusterSize(), 3)\n\n\tfor range 6 {\n\t\trg = append(rg, c.addMemRaftNode(\"TEST\", newStateAdder))\n\t}\n\n\tcheckFor(t, 1*time.Second, 10*time.Millisecond, func() error {\n\t\tif leader.node().ClusterSize() != 9 {\n\t\t\treturn errors.New(\"node additions still in progress\")\n\t\t}\n\t\treturn nil\n\t})\n\n\trequire_Equal(t, leader.node().ClusterSize(), 9)\n}\n\nfunc TestNRGDisjointMajorities(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\trg.waitOnLeader()\n\n\tleader := rg.leader()\n\tfollowers := rg.followers()\n\trequire_Equal(t, len(followers), 2)\n\n\t// Lock all followers, a majority.\n\tlocked := rg.lockFollowers()\n\trequire_Equal(t, len(locked), 2)\n\n\tdefer func() {\n\t\tfor _, l := range locked {\n\t\t\tl.node().(*raft).Unlock()\n\t\t}\n\t}()\n\n\t// Add one node (cluster size is 4)\n\tc.addMemRaftNode(\"TEST\", newStateAdder)\n\n\tcheckFor(t, 1*time.Second, 10*time.Millisecond, func() error {\n\t\tif leader.node().ClusterSize() != 4 {\n\t\t\treturn errors.New(\"node addition still in progress\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Attempt another node addition\n\tc.addMemRaftNode(\"TEST\", newStateAdder)\n\n\t// If bug is present:\n\t// The leader is able to form a majority because it would\n\t// add peers immediately, and readjust cluster size and\n\t// quorum only after committing EntryAddPeer. This allowed\n\t// the leader to commit the entries using only newly added\n\t// nodes (the original followers are locked and not\n\t// acknowledging the EntryAddPeer proposals!)\n\t// This should not never happen. In a real scenario the\n\t// followers could be partitioned, and they actually have\n\t// a majority... so they could diverge and we end up with\n\t// two different histories.\n\t//\n\t// Here wait a little bit, and check that the leader is\n\t// unable to make any progess.\n\ttime.Sleep(time.Second)\n\n\trequire_Equal(t, leader.node().ClusterSize(), 4)\n\trequire_Equal(t, leader.node().MembershipChangeInProgress(), true)\n}\n\nfunc TestNRGAppendEntryResurrectsLeader(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tS2 := \"z3WIzPtj\" // S-2\n\n\tn.addPeer(S2)\n\n\trequire_Equal(t, len(n.peers), 2)\n\trequire_Equal(t, n.ClusterSize(), 2)\n\n\t// PeerRemove S2\n\tentries := []*Entry{newEntry(EntryRemovePeer, []byte(S2))}\n\taeRemovePeer := encode(t, &appendEntry{\n\t\tleader: S2, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\tn.processAppendEntry(aeRemovePeer, n.aesub)\n\n\t// Heartbeat commits the PeerRemove\n\taeHeartBeat := encode(t, &appendEntry{\n\t\tleader: S2, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\tn.processAppendEntry(aeHeartBeat, n.aesub)\n\n\trequire_Equal(t, len(n.peers), 1)\n\trequire_Equal(t, n.ClusterSize(), 1)\n\n\t// If bug is present: receiving a appendEntry from the old leader\n\t// will resurrect it. In practice, this has been observed with\n\t// LeaderTransfer entries, but any type of entry will do...\n\taeHeartBeat2 := encode(t, &appendEntry{\n\t\tleader: S2, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\tn.processAppendEntry(aeHeartBeat2, n.aesub)\n\n\t// Expect the cluster size to be unchanged\n\trequire_Equal(t, len(n.peers), 1)\n\trequire_Equal(t, n.ClusterSize(), 1)\n}\n\nfunc TestNRGSingleNodeElection(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createMemRaftGroup(\"TEST\", 3, newStateAdder)\n\n\t// Remove the cluster leader, and then again\n\tfor range 2 {\n\t\tleader := rg.waitOnLeader().node()\n\t\trequire_NoError(t, leader.ProposeRemovePeer(leader.ID()))\n\t\tcheckFor(t, 1*time.Second, 10*time.Millisecond, func() error {\n\t\t\tif leader.State() == Leader {\n\t\t\t\treturn errors.New(\"Removed node is still leader\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\trequire_False(t, leader.MembershipChangeInProgress())\n\t}\n\n\t// The remaining follower must be able to become leader\n\t// on its own\n\tnewLeader := rg.waitOnLeader()\n\trequire_Equal(t, len(newLeader.node().Peers()), 1)\n\trequire_Equal(t, newLeader.node().ClusterSize(), 1)\n\trequire_False(t, newLeader.node().MembershipChangeInProgress())\n\n\tadder := newLeader.(*stateAdder)\n\tadder.proposeDelta(1)\n\tadder.proposeDelta(10)\n\tadder.proposeDelta(100)\n\n\trg.waitOnTotal(t, 111)\n\n\t// Add two nodes back\n\trg = append(rg, c.addMemRaftNode(\"TEST\", newStateAdder))\n\trg = append(rg, c.addMemRaftNode(\"TEST\", newStateAdder))\n\n\tcheckFor(t, 1*time.Second, 10*time.Millisecond, func() error {\n\t\tif newLeader.node().ClusterSize() != 3 {\n\t\t\treturn errors.New(\"node additions still in progress\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckFor(t, 1*time.Second, 10*time.Millisecond, func() error {\n\t\tif newLeader.node().MembershipChangeInProgress() {\n\t\t\treturn errors.New(\"membership still in progress\")\n\t\t}\n\t\treturn nil\n\t})\n\n\trg.waitOnTotal(t, 111)\n\trequire_Equal(t, newLeader.node().ClusterSize(), 3)\n\trequire_False(t, newLeader.node().MembershipChangeInProgress())\n}\n\nfunc TestNRGMustNotResetVoteOnStepDownOrLeaderTransfer(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Vote for a new leader.\n\trequire_NoError(t, n.processVoteRequest(&voteRequest{term: 1, candidate: nats0}))\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.vote, nats0)\n\n\t// Stepping down as leader must NOT reset the vote info.\n\tn.state.Store(int32(Leader))\n\trequire_NoError(t, n.StepDown())\n\trequire_Equal(t, n.vote, nats0)\n\n\t// A leader transfer must NOT reset the vote info.\n\t// This is automatically cleared once the intended leader starts a new election with a higher term.\n\tentries := []*Entry{newEntry(EntryLeaderTransfer, []byte(nats0))}\n\taeLeaderTransfer := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\tn.processAppendEntry(aeLeaderTransfer, n.aesub)\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.vote, nats0)\n}\n\nfunc TestNRGInstallSnapshotFromCheckpoint(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: entries})\n\taeMsg3 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 2, pterm: 1, pindex: 2, entries: entries})\n\taeMsg4 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 3, pterm: 1, pindex: 3, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 4, pterm: 1, pindex: 4, entries: nil})\n\n\t// Process the first set of messages.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 2)\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Start the first checkpoint, creating the first snapshot.\n\tc, err := n.CreateSnapshotCheckpoint(false)\n\trequire_NoError(t, err)\n\n\t// There's no previous snapshot.\n\tbuf, err := c.LoadLastSnapshot()\n\trequire_Error(t, err, errNoSnapAvailable)\n\trequire_True(t, buf == nil)\n\n\t// Since only 1 entry was applied, we should be able to retrieve that here.\n\tvar count int\n\tfor ae, err := range c.AppendEntriesSeq() {\n\t\tcount++\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ae.pindex, 0)\n\t}\n\trequire_Equal(t, count, 1)\n\n\t// Install the snapshot and confirm we can retrieve it.\n\t_, err = c.InstallSnapshot([]byte(\"old\"))\n\trequire_NoError(t, err)\n\tsnap, err := n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(snap.data, []byte(\"old\")))\n\n\t// Checkpoint was installed, so all should now abort.\n\t_, err = c.LoadLastSnapshot()\n\trequire_Error(t, err, errSnapAborted)\n\tcount = 0\n\tfor _, err = range c.AppendEntriesSeq() {\n\t\tcount++\n\t\trequire_Error(t, err, errSnapAborted)\n\t}\n\trequire_Equal(t, count, 1)\n\t_, err = c.InstallSnapshot(nil)\n\trequire_Error(t, err, errSnapAborted)\n\n\t// Process the next set of messages, we'll be able to compact two append entries now.\n\tn.processAppendEntry(aeMsg3, n.aesub)\n\trequire_Equal(t, n.pindex, 3)\n\trequire_Equal(t, n.commit, 2)\n\tn.processAppendEntry(aeMsg4, n.aesub)\n\trequire_Equal(t, n.pindex, 4)\n\trequire_Equal(t, n.commit, 3)\n\tn.Applied(3)\n\trequire_Equal(t, n.applied, 3)\n\n\t// Check a couple edge cases now, a second checkpoint should fail noting a snapshot is already in progress.\n\tc, err = n.CreateSnapshotCheckpoint(false)\n\trequire_NoError(t, err)\n\t_, err = n.CreateSnapshotCheckpoint(false)\n\trequire_Error(t, err, errSnapInProgress)\n\n\t// Abort checkpoint, so all should now abort.\n\tc.Abort()\n\t_, err = c.LoadLastSnapshot()\n\trequire_Error(t, err, errSnapAborted)\n\tcount = 0\n\tfor _, err = range c.AppendEntriesSeq() {\n\t\tcount++\n\t\trequire_Error(t, err, errSnapAborted)\n\t}\n\trequire_Equal(t, count, 1) // Must always iterate at least once to retrieve the error.\n\t_, err = c.InstallSnapshot(nil)\n\trequire_Error(t, err, errSnapAborted)\n\n\t// Start the second checkpoint, creating the second snapshot.\n\tc, err = n.CreateSnapshotCheckpoint(false)\n\trequire_NoError(t, err)\n\n\t// Should load the previous snapshot.\n\tbuf, err = c.LoadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(buf, []byte(\"old\")))\n\n\t// Iterate over the two append entries part of this new snapshot.\n\tcount = 0\n\tfor ae, err := range c.AppendEntriesSeq() {\n\t\tcount++\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ae.pindex, uint64(count))\n\t}\n\trequire_Equal(t, count, 2)\n\n\t// Install the snapshot and confirm we can retrieve it.\n\t_, err = c.InstallSnapshot([]byte(\"new\"))\n\trequire_NoError(t, err)\n\tsnap, err = n.loadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(snap.data, []byte(\"new\")))\n\n\t// Commit and apply the last entry to test for the final edge case.\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.pindex, 4)\n\trequire_Equal(t, n.commit, 4)\n\tn.Applied(4)\n\trequire_Equal(t, n.applied, 4)\n\n\t// Start an async snapshot.\n\tc, err = n.CreateSnapshotCheckpoint(false)\n\trequire_NoError(t, err)\n\n\t// Installing a snapshot normally should be denied, as one is already in progress.\n\trequire_Error(t, n.InstallSnapshot(nil, false), errSnapInProgress)\n\n\t// The checkpoint should still work successfully.\n\tbuf, err = c.LoadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(buf, []byte(\"new\")))\n\n\t// However, if we install using the internal API, this should be allowed and abort our checkpoint.\n\t// This API will be used as a result of receiving a snapshot as part of catchup, so it's really important\n\t// we aren't allowed to replace this newest snapshot with an older snapshot from an earlier checkpoint.\n\tsnap.lastIndex++\n\trequire_NoError(t, n.installSnapshot(snap))\n\n\t// Check it's aborted.\n\t_, err = c.LoadLastSnapshot()\n\trequire_Error(t, err, errSnapAborted)\n\n\t// Check if the last snapshot was updated from underneath of us.\n\tn.Lock()\n\tn.snapshotting = true\n\tn.Unlock()\n\t_, err = c.LoadLastSnapshot()\n\trequire_Error(t, err, errors.New(\"snapshot index mismatch\"))\n}\n\nfunc TestNRGInstallSnapshotForce(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\n\tn.processAppendEntry(aeMsg, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\tn.processAppendEntry(aeHeartbeat, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Block snapshots unless they're forced to skip in-progress catchups.\n\tn.progress = make(map[string]*ipQueue[uint64])\n\tn.progress[\"blockSnapshots\"] = newIPQueue[uint64](n.s, \"blockSnapshots\")\n\n\trequire_Error(t, n.InstallSnapshot(nil, false), errCatchupsRunning)\n\trequire_Equal(t, n.papplied, 0)\n\n\trequire_NoError(t, n.InstallSnapshot(nil, true))\n\trequire_Equal(t, n.papplied, 1)\n}\n\nfunc TestNRGInstallSnapshotFromCheckpointAfterTruncateToSnapshot(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\t// Timeline\n\taeMsg1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\taeHeartbeat1 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 1, pterm: 1, pindex: 1, entries: nil})\n\taeMsg2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 100, pterm: 1, pindex: 100, entries: entries})\n\taeHeartbeat2 := encode(t, &appendEntry{leader: nats0, term: 1, commit: 101, pterm: 1, pindex: 101, entries: nil})\n\n\t// Process the first message.\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 0)\n\tn.processAppendEntry(aeHeartbeat1, n.aesub)\n\trequire_Equal(t, n.pindex, 1)\n\trequire_Equal(t, n.commit, 1)\n\tn.Applied(1)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Install a snapshot as normal to set up papplied.\n\trequire_NoError(t, n.InstallSnapshot(nil, false))\n\trequire_Equal(t, n.papplied, 1)\n\n\t// Simulate being caught up from a leader with a snapshot.\n\tsnap := &snapshot{\n\t\tlastTerm: 1,\n\t\tlastIndex: 100,\n\t\tpeerstate: encodePeerState(n.currentPeerState()),\n\t\tdata: []byte(\"catchup\"),\n\t}\n\tn.pterm = snap.lastTerm\n\tn.pindex = snap.lastIndex\n\tn.commit = snap.lastIndex\n\trequire_NoError(t, n.installSnapshot(snap))\n\trequire_Equal(t, n.papplied, 100)\n\trequire_Equal(t, n.applied, 1)\n\n\t// Simulate uncommitted entries being truncated up to the above snapshot.\n\tn.truncateWAL(1, 100)\n\trequire_Equal(t, n.papplied, 100)\n\n\t// We haven't applied anything since the previous snapshot, so should error.\n\t_, err := n.CreateSnapshotCheckpoint(false)\n\trequire_Error(t, err, errNoSnapAvailable)\n\n\t// We have only applied the previous snapshot, so should still error as there's nothing (new) to snapshot.\n\tn.Applied(100)\n\trequire_Equal(t, n.applied, 100)\n\t_, err = n.CreateSnapshotCheckpoint(false)\n\trequire_Error(t, err, errNoSnapAvailable)\n\n\t// Process a second message such that we can snapshot.\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\trequire_Equal(t, n.pindex, 101)\n\trequire_Equal(t, n.commit, 100)\n\tn.processAppendEntry(aeHeartbeat2, n.aesub)\n\trequire_Equal(t, n.pindex, 101)\n\trequire_Equal(t, n.commit, 101)\n\tn.Applied(101)\n\trequire_Equal(t, n.applied, 101)\n\n\t// The checkpoint should function, load the previous/catchup snapshot and load above entry.\n\tc, err := n.CreateSnapshotCheckpoint(false)\n\trequire_NoError(t, err)\n\n\tbuf, err := c.LoadLastSnapshot()\n\trequire_NoError(t, err)\n\trequire_True(t, bytes.Equal(buf, []byte(\"catchup\")))\n\n\tvar count int\n\tfor ae, err := range c.AppendEntriesSeq() {\n\t\tcount++\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, ae.pindex, 100)\n\t}\n\trequire_Equal(t, count, 1)\n}\n\nfunc TestNRGSwitchToCandidateResetsVote(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\tnats0 := \"S1Nunr6R\" // \"nats-0\"\n\n\trequire_Equal(t, n.term, 0)\n\tn.vote = nats0\n\tn.switchToCandidate()\n\n\t// Confirm the term was incremented and the vote reset.\n\trequire_Equal(t, n.term, 1)\n\trequire_Equal(t, n.vote, _EMPTY_)\n\n\t// Confirm this was written to disk.\n\tterm, voted, err := n.readTermVote()\n\trequire_NoError(t, err)\n\trequire_Equal(t, term, 1)\n\trequire_Equal(t, voted, _EMPTY_)\n}\n\nfunc TestNRGInitSingleMemRaftNodeDefaults(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\trequire_Equal(t, n.ID(), \"esFhDys3\")\n\trequire_Equal(t, len(n.Peers()), 1)\n\trequire_Equal(t, n.Peers()[0].ID, \"esFhDys3\")\n\trequire_Equal(t, n.ClusterSize(), 1)\n\trequire_True(t, n.Quorum())\n}\n\nfunc TestNRGReplayAddPeerKeepsClusterSize(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\ts := c.servers[0]\n\tdefer c.shutdown()\n\n\tstoreDir := t.TempDir()\n\tfcfg := FileStoreConfig{\n\t\tStoreDir: storeDir,\n\t\tBlockSize: defaultMediumBlockSize,\n\t\tAsyncFlush: false,\n\t\tsrv: s,\n\t}\n\tscfg := StreamConfig{Name: \"RAFT\", Storage: FileStorage}\n\tfs, err := newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\n\tcfg := &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\n\t// Seed peer state as a 3-node group.\n\ts.mu.RLock()\n\tself := s.sys.shash[:idLen]\n\ts.mu.RUnlock()\n\tpeerA, peerB := \"yrzKKRBu\", \"cnrtt3eg\"\n\trequire_NoError(t, s.bootstrapRaftNode(cfg, []string{self, peerA, peerB}, true))\n\n\tn, err := s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\trequire_Equal(t, n.ClusterSize(), 3)\n\n\t// Store a normal entry and a EntryAddPeer and commit them.\n\t// The latter triggered the bug at restart.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\taeMsg1 := encode(t, &appendEntry{\n\t\tleader: self,\n\t\tterm: 2,\n\t\tcommit: 0,\n\t\tpterm: 0,\n\t\tpindex: 0,\n\t\tentries: []*Entry{newEntry(EntryNormal, esm)},\n\t})\n\taeMsg2 := encode(t, &appendEntry{\n\t\tleader: self,\n\t\tterm: 2,\n\t\tcommit: 2,\n\t\tpterm: 2,\n\t\tpindex: 1,\n\t\tentries: []*Entry{newEntry(EntryAddPeer, []byte(peerA))},\n\t})\n\tn.processAppendEntry(aeMsg1, n.aesub)\n\tn.processAppendEntry(aeMsg2, n.aesub)\n\n\t// Restart from disk, forcing replay of the two WAL entries.\n\tn.Stop()\n\tn.WaitForStop()\n\tfs.Stop()\n\n\tfs, err = newFileStore(fcfg, scfg)\n\trequire_NoError(t, err)\n\tcfg = &RaftConfig{Name: \"TEST\", Store: storeDir, Log: fs}\n\tn, err = s.initRaftNode(globalAccountName, cfg, pprofLabels{})\n\trequire_NoError(t, err)\n\n\t// If bug is present, cluster size is 1 instead of 3.\n\t// EntryAddPeer would increases the initial cluster\n\t// of size of 0 to 1. initRaftNode() would then add\n\t// the remaining peers from peer state file, but would\n\t// not adjust cluster size and quorum accordingly.\n\t// This could lead to erroneously create singleton\n\t// clusters after restart.\n\trequire_Equal(t, n.ClusterSize(), 3)\n\trequire_Equal(t, n.qn, 2)\n\n\tn.Stop()\n\tn.WaitForStop()\n\tfs.Stop()\n}\n\nfunc TestNRGTrackPeerLag(t *testing.T) {\n\tc := createJetStreamClusterExplicit(t, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\trg := c.createRaftGroup(\"TEST\", 3, newStateAdder)\n\tlsm := rg.waitOnLeader()\n\trequire_NotNil(t, lsm)\n\tls := lsm.(*stateAdder)\n\n\tcheckPeerData := func() {\n\t\tfor _, sm := range rg {\n\t\t\trn := sm.node().(*raft)\n\t\t\trn.RLock()\n\t\t\tleader := rn.leader\n\t\t\tisLeader := RaftState(rn.state.Load()) == Leader\n\t\t\tfor id, ps := range rn.peers {\n\t\t\t\t// If this peer is the leader (but not ourselves), then these values need to be populated.\n\t\t\t\tif isLeader && id != rn.id {\n\t\t\t\t\tif li := ps.li; li == 0 {\n\t\t\t\t\t\trn.RUnlock()\n\t\t\t\t\t\tt.Fatal(\"require last replicated index to be set\")\n\t\t\t\t\t}\n\t\t\t\t\tif ts := ps.ts; ts.IsZero() {\n\t\t\t\t\t\trn.RUnlock()\n\t\t\t\t\t\tt.Fatal(\"require last seen timestamp to be set\")\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif li := ps.li; li != 0 {\n\t\t\t\t\trn.RUnlock()\n\t\t\t\t\tt.Fatalf(\"require equal, but got: %v != %v\", ps.li, 0)\n\t\t\t\t}\n\t\t\t\tif ts := ps.ts; id != rn.leader && !ts.IsZero() {\n\t\t\t\t\trn.RUnlock()\n\t\t\t\t\tt.Fatalf(\"require zero ts, but got: %v\", ts)\n\t\t\t\t}\n\t\t\t}\n\t\t\trn.RUnlock()\n\n\t\t\t// This test expects no lag and all nodes as current (except for followers as seen by other followers).\n\t\t\tfor _, ps := range rn.Peers() {\n\t\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\t\trequire_Equal(t, ps.Current, ps.ID == rn.id || ps.ID == leader || isLeader)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor i := range 5 {\n\t\tls.proposeDelta(10)\n\t\trg.waitOnTotal(t, int64(10*(i+1)))\n\t}\n\tcheckPeerData()\n\n\trequire_NoError(t, lsm.node().StepDown())\n\tlsm = rg.waitOnLeader()\n\trequire_NotNil(t, lsm)\n\tls = lsm.(*stateAdder)\n\n\tfor i := range 5 {\n\t\tls.proposeDelta(10)\n\t\trg.waitOnTotal(t, int64(50+10*(i+1)))\n\t}\n\tcheckPeerData()\n}\n\nfunc TestNRGPeersResponse(t *testing.T) {\n\t// As a leader.\n\trn := &raft{\n\t\tid: \"me\",\n\t\tleader: \"me\",\n\t\tpeers: map[string]*lps{\n\t\t\t\"me\": {li: 0},\n\t\t\t\"peer1\": {li: 10},\n\t\t\t\"peer2\": {li: 3},\n\t\t\t\"peer3\": {li: 0},\n\t\t},\n\t\tpindex: 10,\n\t\tcommit: 10,\n\t\tapplied: 10,\n\t}\n\trn.state.Store(int32(Leader))\n\tfor _, ps := range rn.Peers() {\n\t\tswitch ps.ID {\n\t\tcase \"me\":\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_True(t, ps.Current)\n\t\tcase \"peer1\":\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_True(t, ps.Current)\n\t\tcase \"peer2\":\n\t\t\trequire_Equal(t, ps.Lag, 7) // We persisted 10 entries, they have only 3.\n\t\t\trequire_False(t, ps.Current)\n\t\tcase \"peer3\":\n\t\t\trequire_Equal(t, ps.Lag, 10) // We persisted 10 entries, they have none.\n\t\t\trequire_False(t, ps.Current)\n\t\tdefault:\n\t\t\tt.Fatalf(\"unexpected peer ID: %s\", ps.ID)\n\t\t}\n\t}\n\n\t// As a follower.\n\trn = &raft{\n\t\tid: \"me\",\n\t\tleader: \"leader\",\n\t\tpeers: map[string]*lps{\n\t\t\t\"me\": {li: 0},\n\t\t\t\"leader\": {li: 0},\n\t\t\t\"peer\": {li: 0},\n\t\t},\n\t}\n\trn.state.Store(int32(Follower))\n\tfor _, ps := range rn.Peers() {\n\t\tswitch ps.ID {\n\t\tcase \"me\":\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_True(t, ps.Current)\n\t\tcase \"leader\":\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_False(t, ps.Current)\n\t\tdefault:\n\t\t\t// As a follower ourselves, we don't know about the state of other followers.\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_False(t, ps.Current)\n\t\t}\n\t}\n\n\t// If we've heard from the leader recently, we report it as current.\n\trn.peers[\"leader\"].ts = time.Now()\n\tfor _, ps := range rn.Peers() {\n\t\tif ps.ID == \"leader\" {\n\t\t\trequire_Equal(t, ps.Lag, 0)\n\t\t\trequire_True(t, ps.Current)\n\t\t}\n\t}\n}\n\nfunc TestNRGOnlyCommitIfCurrentTerm(t *testing.T) {\n\tn, cleanup := initSingleMemRaftNode(t)\n\tdefer cleanup()\n\n\t// Create a sample entry, the content doesn't matter, just that it's stored.\n\tesm := encodeStreamMsgAllowCompress(\"foo\", \"_INBOX.foo\", nil, nil, 0, 0, true)\n\tentries := []*Entry{newEntry(EntryNormal, esm)}\n\n\ts1 := getHash(\"S-1\")\n\ts2 := getHash(\"S-2\")\n\ts3 := getHash(\"S-3\")\n\n\tn.addPeer(s2)\n\tn.addPeer(s3)\n\trequire_Len(t, len(n.Peers()), 3)\n\n\t// The below timeline describes a test ensuring a leader doesn't commit entries from previous terms.\n\n\t// This server became leader twice and stored two entries.\n\taeMsg1 := encode(t, &appendEntry{leader: s1, term: 1, commit: 0, pterm: 0, pindex: 0, entries: entries})\n\trequire_NoError(t, n.storeToWAL(aeMsg1))\n\taeMsg2 := encode(t, &appendEntry{leader: s1, term: 2, commit: 0, pterm: 1, pindex: 1, entries: entries})\n\trequire_NoError(t, n.storeToWAL(aeMsg2))\n\n\t// Another server became leader for term 3 without us knowing and stored a different entry at index 2.\n\t// We now become leader again and store a new entry at index 3, but unaware of the above term 3 entry.\n\taeMsg3 := encode(t, &appendEntry{leader: s1, term: 4, commit: 0, pterm: 2, pindex: 2, entries: entries})\n\trequire_NoError(t, n.storeToWAL(aeMsg3))\n\n\t// Set up leader state for below response handling.\n\tn.switchToLeader()\n\tn.term = 4\n\trequire_Equal(t, n.commit, 0)\n\n\t// When we get quorum on the first two entries this doesn't count toward upping the commit yet.\n\tn.processAppendEntryResponse(&appendEntryResponse{term: 1, index: 1, peer: s2, success: true})\n\trequire_Equal(t, n.commit, 0)\n\tn.processAppendEntryResponse(&appendEntryResponse{term: 2, index: 2, peer: s2, success: true})\n\trequire_Equal(t, n.commit, 0)\n\t// Only once we receive quorum on an entry of our current term do we count this toward upping the commit.\n\t// If we wouldn't have waited to up our commit until now, the server that was leader during term 3\n\t// could still have overwritten the entry at index 2, resulting in a desync.\n\tn.processAppendEntryResponse(&appendEntryResponse{term: 4, index: 3, peer: s2, success: true})\n\trequire_Equal(t, n.commit, 3)\n}\n" }, { "path": "server/rate_counter.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"sync\"\n\t\"time\"\n)\n\ntype rateCounter struct {\n\tlimit int64\n\tcount int64\n\tblocked uint64\n\tend time.Time\n\tinterval time.Duration\n\tmu sync.Mutex\n}\n\nfunc newRateCounter(limit int64) *rateCounter {\n\treturn &rateCounter{\n\t\tlimit: limit,\n\t\tinterval: time.Second,\n\t}\n}\n\nfunc (r *rateCounter) allow() bool {\n\tnow := time.Now()\n\n\tr.mu.Lock()\n\n\tif now.After(r.end) {\n\t\tr.count = 0\n\t\tr.end = now.Add(r.interval)\n\t} else {\n\t\tr.count++\n\t}\n\tallow := r.count < r.limit\n\tif !allow {\n\t\tr.blocked++\n\t}\n\n\tr.mu.Unlock()\n\n\treturn allow\n}\n\nfunc (r *rateCounter) countBlocked() uint64 {\n\tr.mu.Lock()\n\tblocked := r.blocked\n\tr.blocked = 0\n\tr.mu.Unlock()\n\n\treturn blocked\n}\n" }, { "path": "server/rate_counter_test.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRateCounter(t *testing.T) {\n\tcounter := newRateCounter(10)\n\tcounter.interval = 100 * time.Millisecond\n\n\tvar i int\n\tfor i = 0; i < 10; i++ {\n\t\tif !counter.allow() {\n\t\t\tt.Errorf(\"counter should allow (iteration %d)\", i)\n\t\t}\n\t}\n\tfor i = 0; i < 5; i++ {\n\t\tif counter.allow() {\n\t\t\tt.Errorf(\"counter should not allow (iteration %d)\", i)\n\t\t}\n\t}\n\n\tblocked := counter.countBlocked()\n\tif blocked != 5 {\n\t\tt.Errorf(\"Expected blocked = 5, got %d\", blocked)\n\t}\n\n\tblocked = counter.countBlocked()\n\tif blocked != 0 {\n\t\tt.Errorf(\"Expected blocked = 0, got %d\", blocked)\n\t}\n\n\ttime.Sleep(150 * time.Millisecond)\n\n\tif !counter.allow() {\n\t\tt.Errorf(\"Expected true after current time window expired\")\n\t}\n}\n" }, { "path": "server/reload.go", "content": "// Copyright 2017-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"cmp\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// FlagSnapshot captures the server options as specified by CLI flags at\n// startup. This should not be modified once the server has started.\nvar FlagSnapshot *Options\n\ntype reloadContext struct {\n\toldClusterPerms *RoutePermissions\n}\n\n// option is a hot-swappable configuration setting.\ntype option interface {\n\t// Apply the server option.\n\tApply(server *Server)\n\n\t// IsLoggingChange indicates if this option requires reloading the logger.\n\tIsLoggingChange() bool\n\n\t// IsTraceLevelChange indicates if this option requires reloading cached trace level.\n\t// Clients store trace level separately.\n\tIsTraceLevelChange() bool\n\n\t// IsAuthChange indicates if this option requires reloading authorization.\n\tIsAuthChange() bool\n\n\t// IsTLSChange indicates if this option requires reloading TLS.\n\tIsTLSChange() bool\n\n\t// IsClusterPermsChange indicates if this option requires reloading\n\t// cluster permissions.\n\tIsClusterPermsChange() bool\n\n\t// IsClusterPoolSizeOrAccountsChange indicates if this option requires\n\t// special handling for changes in cluster's pool size or accounts list.\n\tIsClusterPoolSizeOrAccountsChange() bool\n\n\t// IsJetStreamChange inidicates a change in the servers config for JetStream.\n\t// Account changes will be handled separately in reloadAuthorization.\n\tIsJetStreamChange() bool\n\n\t// Indicates a change in the server that requires publishing the server's statz\n\tIsStatszChange() bool\n}\n\n// noopOption is a base struct that provides default no-op behaviors.\ntype noopOption struct{}\n\nfunc (n noopOption) IsLoggingChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsTraceLevelChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsAuthChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsTLSChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsClusterPermsChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsClusterPoolSizeOrAccountsChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsJetStreamChange() bool {\n\treturn false\n}\n\nfunc (n noopOption) IsStatszChange() bool {\n\treturn false\n}\n\n// loggingOption is a base struct that provides default option behaviors for\n// logging-related options.\ntype loggingOption struct {\n\tnoopOption\n}\n\nfunc (l loggingOption) IsLoggingChange() bool {\n\treturn true\n}\n\n// traceLevelOption is a base struct that provides default option behaviors for\n// tracelevel-related options.\ntype traceLevelOption struct {\n\tloggingOption\n}\n\nfunc (l traceLevelOption) IsTraceLevelChange() bool {\n\treturn true\n}\n\n// traceOption implements the option interface for the `trace` setting.\ntype traceOption struct {\n\ttraceLevelOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (t *traceOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: trace = %v\", t.newValue)\n}\n\n// traceVersboseOption implements the option interface for the `trace_verbose` setting.\ntype traceVerboseOption struct {\n\ttraceLevelOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (t *traceVerboseOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: trace_verbose = %v\", t.newValue)\n}\n\n// traceHeadersOption implements the option interface for the `trace_headers` setting.\ntype traceHeadersOption struct {\n\ttraceLevelOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (t *traceHeadersOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: trace_headers = %v\", t.newValue)\n}\n\n// debugOption implements the option interface for the `debug` setting.\ntype debugOption struct {\n\tloggingOption\n\tnewValue bool\n}\n\n// Apply is mostly a no-op because logging will be reloaded after options are applied.\n// However we will kick the raft nodes if they exist to reload.\nfunc (d *debugOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: debug = %v\", d.newValue)\n\tserver.reloadDebugRaftNodes(d.newValue)\n}\n\n// logtimeOption implements the option interface for the `logtime` setting.\ntype logtimeOption struct {\n\tloggingOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (l *logtimeOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: logtime = %v\", l.newValue)\n}\n\n// logtimeUTCOption implements the option interface for the `logtime_utc` setting.\ntype logtimeUTCOption struct {\n\tloggingOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (l *logtimeUTCOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: logtime_utc = %v\", l.newValue)\n}\n\n// logfileOption implements the option interface for the `log_file` setting.\ntype logfileOption struct {\n\tloggingOption\n\tnewValue string\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (l *logfileOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: log_file = %v\", l.newValue)\n}\n\n// syslogOption implements the option interface for the `syslog` setting.\ntype syslogOption struct {\n\tloggingOption\n\tnewValue bool\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (s *syslogOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: syslog = %v\", s.newValue)\n}\n\n// remoteSyslogOption implements the option interface for the `remote_syslog`\n// setting.\ntype remoteSyslogOption struct {\n\tloggingOption\n\tnewValue string\n}\n\n// Apply is a no-op because logging will be reloaded after options are applied.\nfunc (r *remoteSyslogOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: remote_syslog = %v\", r.newValue)\n}\n\n// tlsOption implements the option interface for the `tls` setting.\ntype tlsOption struct {\n\tnoopOption\n\tnewValue *tls.Config\n}\n\n// Apply the tls change.\nfunc (t *tlsOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\ttlsRequired := t.newValue != nil\n\tserver.info.TLSRequired = tlsRequired && !server.getOpts().AllowNonTLS\n\tmessage := \"disabled\"\n\tif tlsRequired {\n\t\tserver.info.TLSVerify = (t.newValue.ClientAuth == tls.RequireAndVerifyClientCert)\n\t\tmessage = \"enabled\"\n\t}\n\tserver.mu.Unlock()\n\tserver.Noticef(\"Reloaded: tls = %s\", message)\n}\n\nfunc (t *tlsOption) IsTLSChange() bool {\n\treturn true\n}\n\n// tlsTimeoutOption implements the option interface for the tls `timeout`\n// setting.\ntype tlsTimeoutOption struct {\n\tnoopOption\n\tnewValue float64\n}\n\n// Apply is a no-op because the timeout will be reloaded after options are\n// applied.\nfunc (t *tlsTimeoutOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: tls timeout = %v\", t.newValue)\n}\n\n// tlsPinnedCertOption implements the option interface for the tls `pinned_certs` setting.\ntype tlsPinnedCertOption struct {\n\tnoopOption\n\tnewValue PinnedCertSet\n}\n\n// Apply is a no-op because the pinned certs will be reloaded after options are applied.\nfunc (t *tlsPinnedCertOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: %d pinned_certs\", len(t.newValue))\n}\n\n// tlsHandshakeFirst implements the option interface for the tls `handshake first` setting.\ntype tlsHandshakeFirst struct {\n\tnoopOption\n\tnewValue bool\n}\n\n// Apply is a no-op because the timeout will be reloaded after options are applied.\nfunc (t *tlsHandshakeFirst) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: Client TLS handshake first: %v\", t.newValue)\n}\n\n// tlsHandshakeFirstFallback implements the option interface for the tls `handshake first fallback delay` setting.\ntype tlsHandshakeFirstFallback struct {\n\tnoopOption\n\tnewValue time.Duration\n}\n\n// Apply is a no-op because the timeout will be reloaded after options are applied.\nfunc (t *tlsHandshakeFirstFallback) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: Client TLS handshake first fallback delay: %v\", t.newValue)\n}\n\n// authOption is a base struct that provides default option behaviors.\ntype authOption struct {\n\tnoopOption\n}\n\nfunc (o authOption) IsAuthChange() bool {\n\treturn true\n}\n\n// usernameOption implements the option interface for the `username` setting.\ntype usernameOption struct {\n\tauthOption\n}\n\n// Apply is a no-op because authorization will be reloaded after options are\n// applied.\nfunc (u *usernameOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization username\")\n}\n\n// passwordOption implements the option interface for the `password` setting.\ntype passwordOption struct {\n\tauthOption\n}\n\n// Apply is a no-op because authorization will be reloaded after options are\n// applied.\nfunc (p *passwordOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization password\")\n}\n\n// authorizationOption implements the option interface for the `token`\n// authorization setting.\ntype authorizationOption struct {\n\tauthOption\n}\n\n// Apply is a no-op because authorization will be reloaded after options are\n// applied.\nfunc (a *authorizationOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization token\")\n}\n\n// authTimeoutOption implements the option interface for the authorization\n// `timeout` setting.\ntype authTimeoutOption struct {\n\tnoopOption // Not authOption because this is a no-op; will be reloaded with options.\n\tnewValue float64\n}\n\n// Apply is a no-op because the timeout will be reloaded after options are\n// applied.\nfunc (a *authTimeoutOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization timeout = %v\", a.newValue)\n}\n\n// tagsOption implements the option interface for the `tags` setting.\ntype tagsOption struct {\n\tnoopOption // Not authOption because this is a no-op; will be reloaded with options.\n}\n\nfunc (u *tagsOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: tags\")\n}\n\nfunc (u *tagsOption) IsStatszChange() bool {\n\treturn true\n}\n\n// metadataOption implements the option interface for the `metadata` setting.\ntype metadataOption struct {\n\tnoopOption // Not authOption because this is a no-op; will be reloaded with options.\n}\n\nfunc (u *metadataOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: metadata\")\n}\n\nfunc (u *metadataOption) IsStatszChange() bool {\n\treturn true\n}\n\n// usersOption implements the option interface for the authorization `users`\n// setting.\ntype usersOption struct {\n\tauthOption\n}\n\nfunc (u *usersOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization users\")\n}\n\n// nkeysOption implements the option interface for the authorization `users`\n// setting.\ntype nkeysOption struct {\n\tauthOption\n}\n\nfunc (u *nkeysOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: authorization nkey users\")\n}\n\n// clusterOption implements the option interface for the `cluster` setting.\ntype clusterOption struct {\n\tauthOption\n\tnewValue ClusterOpts\n\tpermsChanged bool\n\taccsAdded []string\n\taccsRemoved []string\n\tpoolSizeChanged bool\n\tcompressChanged bool\n}\n\n// Apply the cluster change.\nfunc (c *clusterOption) Apply(s *Server) {\n\t// TODO: support enabling/disabling clustering.\n\ts.mu.Lock()\n\ttlsRequired := c.newValue.TLSConfig != nil\n\ts.routeInfo.TLSRequired = tlsRequired\n\ts.routeInfo.TLSVerify = tlsRequired\n\ts.routeInfo.AuthRequired = c.newValue.Username != \"\"\n\tif c.newValue.NoAdvertise {\n\t\ts.routeInfo.ClientConnectURLs = nil\n\t\ts.routeInfo.WSConnectURLs = nil\n\t} else {\n\t\ts.routeInfo.ClientConnectURLs = s.clientConnectURLs\n\t\ts.routeInfo.WSConnectURLs = s.websocket.connectURLs\n\t}\n\ts.setRouteInfoHostPortAndIP()\n\tvar routes []*client\n\tif c.compressChanged {\n\t\tco := &s.getOpts().Cluster.Compression\n\t\tnewMode := co.Mode\n\t\ts.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\t// Skip routes that are \"not supported\" (because they will never do\n\t\t\t// compression) or the routes that have already the new compression\n\t\t\t// mode.\n\t\t\tif r.route.compression == CompressionNotSupported || r.route.compression == newMode {\n\t\t\t\tr.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// We need to close the route if it had compression \"off\" or the new\n\t\t\t// mode is compression \"off\", or if the new mode is \"accept\", because\n\t\t\t// these require negotiation.\n\t\t\tif r.route.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept {\n\t\t\t\troutes = append(routes, r)\n\t\t\t} else if newMode == CompressionS2Auto {\n\t\t\t\t// If the mode is \"s2_auto\", we need to check if there is really\n\t\t\t\t// need to change, and at any rate, we want to save the actual\n\t\t\t\t// compression level here, not s2_auto.\n\t\t\t\tr.updateS2AutoCompressionLevel(co, &r.route.compression)\n\t\t\t} else {\n\t\t\t\t// Simply change the compression writer\n\t\t\t\tr.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...)\n\t\t\t\tr.route.compression = newMode\n\t\t\t}\n\t\t\tr.mu.Unlock()\n\t\t})\n\t}\n\ts.mu.Unlock()\n\tif c.newValue.Name != \"\" && c.newValue.Name != s.ClusterName() {\n\t\ts.setClusterName(c.newValue.Name)\n\t}\n\tfor _, r := range routes {\n\t\tr.closeConnection(ClientClosed)\n\t}\n\ts.Noticef(\"Reloaded: cluster\")\n\tif tlsRequired && c.newValue.TLSConfig.InsecureSkipVerify {\n\t\ts.Warnf(clusterTLSInsecureWarning)\n\t}\n}\n\nfunc (c *clusterOption) IsClusterPermsChange() bool {\n\treturn c.permsChanged\n}\n\nfunc (c *clusterOption) IsClusterPoolSizeOrAccountsChange() bool {\n\treturn c.poolSizeChanged || len(c.accsAdded) > 0 || len(c.accsRemoved) > 0\n}\n\nfunc (c *clusterOption) diffPoolAndAccounts(old *ClusterOpts) {\n\tc.poolSizeChanged = c.newValue.PoolSize != old.PoolSize\naddLoop:\n\tfor _, na := range c.newValue.PinnedAccounts {\n\t\tfor _, oa := range old.PinnedAccounts {\n\t\t\tif na == oa {\n\t\t\t\tcontinue addLoop\n\t\t\t}\n\t\t}\n\t\tc.accsAdded = append(c.accsAdded, na)\n\t}\nremoveLoop:\n\tfor _, oa := range old.PinnedAccounts {\n\t\tfor _, na := range c.newValue.PinnedAccounts {\n\t\t\tif oa == na {\n\t\t\t\tcontinue removeLoop\n\t\t\t}\n\t\t}\n\t\tc.accsRemoved = append(c.accsRemoved, oa)\n\t}\n}\n\n// routesOption implements the option interface for the cluster `routes`\n// setting.\ntype routesOption struct {\n\tnoopOption\n\tadd []*url.URL\n\tremove []*url.URL\n}\n\n// Apply the route changes by adding and removing the necessary routes.\nfunc (r *routesOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\troutes := make([]*client, server.numRoutes())\n\ti := 0\n\tserver.forEachRoute(func(r *client) {\n\t\troutes[i] = r\n\t\ti++\n\t})\n\t// If there was a change, notify monitoring code that it should\n\t// update the route URLs if /varz endpoint is inspected.\n\tif len(r.add)+len(r.remove) > 0 {\n\t\tserver.varzUpdateRouteURLs = true\n\t}\n\tserver.mu.Unlock()\n\n\t// Remove routes.\n\tfor _, remove := range r.remove {\n\t\tfor _, client := range routes {\n\t\t\tvar url *url.URL\n\t\t\tclient.mu.Lock()\n\t\t\tif client.route != nil {\n\t\t\t\turl = client.route.url\n\t\t\t}\n\t\t\tclient.mu.Unlock()\n\t\t\tif url != nil && urlsAreEqual(url, remove) {\n\t\t\t\t// Do not attempt to reconnect when route is removed.\n\t\t\t\tclient.setNoReconnect()\n\t\t\t\tclient.closeConnection(RouteRemoved)\n\t\t\t\tserver.Noticef(\"Removed route %v\", remove)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Add routes.\n\tserver.mu.Lock()\n\tserver.solicitRoutes(r.add, server.getOpts().Cluster.PinnedAccounts)\n\tserver.mu.Unlock()\n\n\tserver.Noticef(\"Reloaded: cluster routes\")\n}\n\n// maxConnOption implements the option interface for the `max_connections`\n// setting.\ntype maxConnOption struct {\n\tnoopOption\n\tnewValue int\n}\n\n// Apply the max connections change by closing random connections til we are\n// below the limit if necessary.\nfunc (m *maxConnOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\tclients := make([]*client, 0, len(server.clients))\n\t// Map iteration is random, which allows us to close random connections.\n\tfor _, client := range server.clients {\n\t\tif isInternalClient(client.kind) {\n\t\t\tcontinue\n\t\t}\n\t\tclients = append(clients, client)\n\t}\n\tserver.mu.Unlock()\n\n\tif newc := max(0, m.newValue); len(clients) > newc {\n\t\t// Close connections til we are within the limit.\n\t\tvar (\n\t\t\tnumClose = len(clients) - newc\n\t\t\tclosed = 0\n\t\t)\n\t\tfor _, client := range clients {\n\t\t\tclient.maxConnExceeded()\n\t\t\tclosed++\n\t\t\tif closed >= numClose {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tserver.Noticef(\"Closed %d connections to fall within max_connections\", closed)\n\t}\n\tserver.Noticef(\"Reloaded: max_connections = %v\", m.newValue)\n}\n\n// pidFileOption implements the option interface for the `pid_file` setting.\ntype pidFileOption struct {\n\tnoopOption\n\tnewValue string\n}\n\n// Apply the setting by logging the pid to the new file.\nfunc (p *pidFileOption) Apply(server *Server) {\n\tif p.newValue == \"\" {\n\t\treturn\n\t}\n\tif err := server.logPid(); err != nil {\n\t\tserver.Errorf(\"Failed to write pidfile: %v\", err)\n\t}\n\tserver.Noticef(\"Reloaded: pid_file = %v\", p.newValue)\n}\n\n// portsFileDirOption implements the option interface for the `portFileDir` setting.\ntype portsFileDirOption struct {\n\tnoopOption\n\toldValue string\n\tnewValue string\n}\n\nfunc (p *portsFileDirOption) Apply(server *Server) {\n\tserver.deletePortsFile(p.oldValue)\n\tserver.logPorts()\n\tserver.Noticef(\"Reloaded: ports_file_dir = %v\", p.newValue)\n}\n\n// maxControlLineOption implements the option interface for the\n// `max_control_line` setting.\ntype maxControlLineOption struct {\n\tnoopOption\n\tnewValue int32\n}\n\n// Apply the setting by updating each client.\nfunc (m *maxControlLineOption) Apply(server *Server) {\n\tmcl := int32(m.newValue)\n\tserver.mu.Lock()\n\tfor _, client := range server.clients {\n\t\tatomic.StoreInt32(&client.mcl, mcl)\n\t}\n\tserver.mu.Unlock()\n\tserver.Noticef(\"Reloaded: max_control_line = %d\", mcl)\n}\n\n// maxPayloadOption implements the option interface for the `max_payload`\n// setting.\ntype maxPayloadOption struct {\n\tnoopOption\n\tnewValue int32\n}\n\n// Apply the setting by updating the server info and each client.\nfunc (m *maxPayloadOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\tserver.info.MaxPayload = m.newValue\n\tfor _, client := range server.clients {\n\t\tatomic.StoreInt32(&client.mpay, int32(m.newValue))\n\t}\n\tserver.mu.Unlock()\n\tserver.Noticef(\"Reloaded: max_payload = %d\", m.newValue)\n}\n\n// pingIntervalOption implements the option interface for the `ping_interval`\n// setting.\ntype pingIntervalOption struct {\n\tnoopOption\n\tnewValue time.Duration\n}\n\n// Apply is a no-op because the ping interval will be reloaded after options\n// are applied.\nfunc (p *pingIntervalOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: ping_interval = %s\", p.newValue)\n}\n\n// maxPingsOutOption implements the option interface for the `ping_max`\n// setting.\ntype maxPingsOutOption struct {\n\tnoopOption\n\tnewValue int\n}\n\n// Apply is a no-op because the ping interval will be reloaded after options\n// are applied.\nfunc (m *maxPingsOutOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: ping_max = %d\", m.newValue)\n}\n\n// writeDeadlineOption implements the option interface for the `write_deadline`\n// setting.\ntype writeDeadlineOption struct {\n\tnoopOption\n\tnewValue time.Duration\n}\n\n// Apply is a no-op because the write deadline will be reloaded after options\n// are applied.\nfunc (w *writeDeadlineOption) Apply(server *Server) {\n\tserver.Noticef(\"Reloaded: write_deadline = %s\", w.newValue)\n}\n\n// clientAdvertiseOption implements the option interface for the `client_advertise` setting.\ntype clientAdvertiseOption struct {\n\tnoopOption\n\tnewValue string\n}\n\n// Apply the setting by updating the server info and regenerate the infoJSON byte array.\nfunc (c *clientAdvertiseOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\tserver.setInfoHostPort()\n\tserver.mu.Unlock()\n\tserver.Noticef(\"Reload: client_advertise = %s\", c.newValue)\n}\n\n// accountsOption implements the option interface.\n// Ensure that authorization code is executed if any change in accounts\ntype accountsOption struct {\n\tauthOption\n}\n\n// Apply is a no-op. Changes will be applied in reloadAuthorization\nfunc (a *accountsOption) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: accounts\")\n}\n\n// For changes to a server's config.\ntype jetStreamOption struct {\n\tnoopOption\n\tnewValue bool\n}\n\nfunc (a *jetStreamOption) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: JetStream\")\n}\n\nfunc (jso jetStreamOption) IsJetStreamChange() bool {\n\treturn true\n}\n\nfunc (jso jetStreamOption) IsStatszChange() bool {\n\treturn true\n}\n\ntype defaultSentinelOption struct {\n\tnoopOption\n\tnewValue string\n}\n\nfunc (so *defaultSentinelOption) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: default_sentinel = %s\", so.newValue)\n}\n\ntype ocspOption struct {\n\ttlsOption\n\tnewValue *OCSPConfig\n}\n\nfunc (a *ocspOption) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: OCSP\")\n}\n\ntype ocspResponseCacheOption struct {\n\ttlsOption\n\tnewValue *OCSPResponseCacheConfig\n}\n\nfunc (a *ocspResponseCacheOption) Apply(s *Server) {\n\ts.Noticef(\"Reloaded OCSP peer cache\")\n}\n\n// connectErrorReports implements the option interface for the `connect_error_reports`\n// setting.\ntype connectErrorReports struct {\n\tnoopOption\n\tnewValue int\n}\n\n// Apply is a no-op because the value will be reloaded after options are applied.\nfunc (c *connectErrorReports) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: connect_error_reports = %v\", c.newValue)\n}\n\n// connectErrorReports implements the option interface for the `connect_error_reports`\n// setting.\ntype reconnectErrorReports struct {\n\tnoopOption\n\tnewValue int\n}\n\n// Apply is a no-op because the value will be reloaded after options are applied.\nfunc (r *reconnectErrorReports) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: reconnect_error_reports = %v\", r.newValue)\n}\n\n// maxTracedMsgLenOption implements the option interface for the `max_traced_msg_len` setting.\ntype maxTracedMsgLenOption struct {\n\tnoopOption\n\tnewValue int\n}\n\n// Apply the setting by updating the maximum traced message length.\nfunc (m *maxTracedMsgLenOption) Apply(server *Server) {\n\tserver.mu.Lock()\n\tdefer server.mu.Unlock()\n\tserver.opts.MaxTracedMsgLen = m.newValue\n\tserver.Noticef(\"Reloaded: max_traced_msg_len = %d\", m.newValue)\n}\n\ntype mqttAckWaitReload struct {\n\tnoopOption\n\tnewValue time.Duration\n}\n\nfunc (o *mqttAckWaitReload) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: MQTT ack_wait = %v\", o.newValue)\n}\n\ntype mqttMaxAckPendingReload struct {\n\tnoopOption\n\tnewValue uint16\n}\n\nfunc (o *mqttMaxAckPendingReload) Apply(s *Server) {\n\ts.mqttUpdateMaxAckPending(o.newValue)\n\ts.Noticef(\"Reloaded: MQTT max_ack_pending = %v\", o.newValue)\n}\n\ntype mqttStreamReplicasReload struct {\n\tnoopOption\n\tnewValue int\n}\n\nfunc (o *mqttStreamReplicasReload) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: MQTT stream_replicas = %v\", o.newValue)\n}\n\ntype mqttConsumerReplicasReload struct {\n\tnoopOption\n\tnewValue int\n}\n\nfunc (o *mqttConsumerReplicasReload) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: MQTT consumer_replicas = %v\", o.newValue)\n}\n\ntype mqttConsumerMemoryStorageReload struct {\n\tnoopOption\n\tnewValue bool\n}\n\nfunc (o *mqttConsumerMemoryStorageReload) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: MQTT consumer_memory_storage = %v\", o.newValue)\n}\n\ntype mqttInactiveThresholdReload struct {\n\tnoopOption\n\tnewValue time.Duration\n}\n\nfunc (o *mqttInactiveThresholdReload) Apply(s *Server) {\n\ts.Noticef(\"Reloaded: MQTT consumer_inactive_threshold = %v\", o.newValue)\n}\n\ntype profBlockRateReload struct {\n\tnoopOption\n\tnewValue int\n}\n\nfunc (o *profBlockRateReload) Apply(s *Server) {\n\ts.setBlockProfileRate(o.newValue)\n\ts.Noticef(\"Reloaded: prof_block_rate = %v\", o.newValue)\n}\n\ntype leafNodeOption struct {\n\tnoopOption\n\ttlsFirstChanged bool\n\tcompressionChanged bool\n\tdisabledChanged bool\n}\n\nfunc (l *leafNodeOption) Apply(s *Server) {\n\topts := s.getOpts()\n\tif l.tlsFirstChanged {\n\t\ts.Noticef(\"Reloaded: LeafNode TLS HandshakeFirst value is: %v\", opts.LeafNode.TLSHandshakeFirst)\n\t\ts.Noticef(\"Reloaded: LeafNode TLS HandshakeFirstFallback value is: %v\", opts.LeafNode.TLSHandshakeFirstFallback)\n\t\tfor _, r := range opts.LeafNode.Remotes {\n\t\t\ts.Noticef(\"Reloaded: LeafNode Remote to %v TLS HandshakeFirst value is: %v\", r.URLs, r.TLSHandshakeFirst)\n\t\t}\n\t}\n\tif l.compressionChanged || l.disabledChanged {\n\t\tvar leafs []*client\n\t\tvar solicit []*leafNodeCfg\n\t\tacceptSideCompOpts := &opts.LeafNode.Compression\n\n\t\ts.mu.RLock()\n\t\t// First, update our internal leaf remote configurations with the new\n\t\t// compress options.\n\t\t// Since changing the remotes (as in adding/removing) is currently not\n\t\t// supported, we know that we should have the same number in Options\n\t\t// than in leafRemoteCfgs, but to be sure, use the max size.\n\t\tmax := len(opts.LeafNode.Remotes)\n\t\tif l := len(s.leafRemoteCfgs); l < max {\n\t\t\tmax = l\n\t\t}\n\t\tfor i := range max {\n\t\t\tlr := s.leafRemoteCfgs[i]\n\t\t\tor := opts.LeafNode.Remotes[i]\n\t\t\tlr.Lock()\n\t\t\tlr.Compression = or.Compression\n\t\t\tif lr.Disabled && !or.Disabled {\n\t\t\t\tsolicit = append(solicit, lr)\n\t\t\t}\n\t\t\tlr.Disabled = or.Disabled\n\t\t\tlr.Unlock()\n\t\t}\n\n\t\tfor _, l := range s.leafs {\n\t\t\tvar co *CompressionOpts\n\n\t\t\tl.mu.Lock()\n\t\t\tif r := l.leaf.remote; r != nil {\n\t\t\t\t// If newly marked as disabled, collect and ignore the rest.\n\t\t\t\tif r.Disabled {\n\t\t\t\t\tl.flags.set(noReconnect)\n\t\t\t\t\tleafs = append(leafs, l)\n\t\t\t\t\tl.mu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tco = &r.Compression\n\t\t\t} else {\n\t\t\t\tco = acceptSideCompOpts\n\t\t\t}\n\t\t\tnewMode := co.Mode\n\t\t\t// Skip leaf connections that are \"not supported\" (because they\n\t\t\t// will never do compression) or the ones that have already the\n\t\t\t// new compression mode.\n\t\t\tif l.leaf.compression == CompressionNotSupported || l.leaf.compression == newMode {\n\t\t\t\tl.mu.Unlock()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// We need to close the connections if it had compression \"off\" or the new\n\t\t\t// mode is compression \"off\", or if the new mode is \"accept\", because\n\t\t\t// these require negotiation.\n\t\t\tif l.leaf.compression == CompressionOff || newMode == CompressionOff || newMode == CompressionAccept {\n\t\t\t\tleafs = append(leafs, l)\n\t\t\t} else if newMode == CompressionS2Auto {\n\t\t\t\t// If the mode is \"s2_auto\", we need to check if there is really\n\t\t\t\t// need to change, and at any rate, we want to save the actual\n\t\t\t\t// compression level here, not s2_auto.\n\t\t\t\tl.updateS2AutoCompressionLevel(co, &l.leaf.compression)\n\t\t\t} else {\n\t\t\t\t// Simply change the compression writer\n\t\t\t\tl.out.cw = s2.NewWriter(nil, s2WriterOptions(newMode)...)\n\t\t\t\tl.leaf.compression = newMode\n\t\t\t}\n\t\t\tl.mu.Unlock()\n\t\t}\n\t\ts.mu.RUnlock()\n\t\t// Close the connections for which negotiation is required, or that\n\t\t// have been disabled.\n\t\tfor _, l := range leafs {\n\t\t\tl.closeConnection(ClientClosed)\n\t\t}\n\t\tif l.compressionChanged {\n\t\t\ts.Noticef(\"Reloaded: LeafNode compression settings\")\n\t\t}\n\t\tif l.disabledChanged {\n\t\t\tif len(leafs) > 0 {\n\t\t\t\ts.Noticef(\"Reloaded: LeafNode(s) disabled\")\n\t\t\t}\n\t\t\tif len(solicit) > 0 {\n\t\t\t\tfor _, remote := range solicit {\n\t\t\t\t\ts.startGoRoutine(func() { s.connectToRemoteLeafNode(remote, true) })\n\t\t\t\t}\n\t\t\t\ts.Noticef(\"Reloaded: LeafNode(s) enabled\")\n\t\t\t}\n\t\t}\n\t}\n}\n\ntype noFastProdStallReload struct {\n\tnoopOption\n\tnoStall bool\n}\n\nfunc (l *noFastProdStallReload) Apply(s *Server) {\n\tvar not string\n\tif l.noStall {\n\t\tnot = \"not \"\n\t}\n\ts.Noticef(\"Reloaded: fast producers will %sbe stalled\", not)\n}\n\n// Compares options and disconnects clients that are no longer listed in pinned certs. Lock must not be held.\nfunc (s *Server) recheckPinnedCerts(curOpts *Options, newOpts *Options) {\n\ts.mu.Lock()\n\tdisconnectClients := []*client{}\n\tprotoToPinned := map[int]PinnedCertSet{}\n\tif !reflect.DeepEqual(newOpts.TLSPinnedCerts, curOpts.TLSPinnedCerts) {\n\t\tprotoToPinned[NATS] = curOpts.TLSPinnedCerts\n\t}\n\tif !reflect.DeepEqual(newOpts.MQTT.TLSPinnedCerts, curOpts.MQTT.TLSPinnedCerts) {\n\t\tprotoToPinned[MQTT] = curOpts.MQTT.TLSPinnedCerts\n\t}\n\tif !reflect.DeepEqual(newOpts.Websocket.TLSPinnedCerts, curOpts.Websocket.TLSPinnedCerts) {\n\t\tprotoToPinned[WS] = curOpts.Websocket.TLSPinnedCerts\n\t}\n\tfor _, c := range s.clients {\n\t\tif c.kind != CLIENT {\n\t\t\tcontinue\n\t\t}\n\t\tif pinned, ok := protoToPinned[c.clientType()]; ok {\n\t\t\tif !c.matchesPinnedCert(pinned) {\n\t\t\t\tdisconnectClients = append(disconnectClients, c)\n\t\t\t}\n\t\t}\n\t}\n\tcheckClients := func(kind int, clients map[uint64]*client, set PinnedCertSet) {\n\t\tfor _, c := range clients {\n\t\t\tif c.kind == kind && !c.matchesPinnedCert(set) {\n\t\t\t\tdisconnectClients = append(disconnectClients, c)\n\t\t\t}\n\t\t}\n\t}\n\tif !reflect.DeepEqual(newOpts.LeafNode.TLSPinnedCerts, curOpts.LeafNode.TLSPinnedCerts) {\n\t\tcheckClients(LEAF, s.leafs, newOpts.LeafNode.TLSPinnedCerts)\n\t}\n\tif !reflect.DeepEqual(newOpts.Cluster.TLSPinnedCerts, curOpts.Cluster.TLSPinnedCerts) {\n\t\ts.forEachRoute(func(c *client) {\n\t\t\tif !c.matchesPinnedCert(newOpts.Cluster.TLSPinnedCerts) {\n\t\t\t\tdisconnectClients = append(disconnectClients, c)\n\t\t\t}\n\t\t})\n\t}\n\tif s.gateway.enabled && reflect.DeepEqual(newOpts.Gateway.TLSPinnedCerts, curOpts.Gateway.TLSPinnedCerts) {\n\t\tgw := s.gateway\n\t\tgw.RLock()\n\t\tfor _, c := range gw.out {\n\t\t\tif !c.matchesPinnedCert(newOpts.Gateway.TLSPinnedCerts) {\n\t\t\t\tdisconnectClients = append(disconnectClients, c)\n\t\t\t}\n\t\t}\n\t\tcheckClients(GATEWAY, gw.in, newOpts.Gateway.TLSPinnedCerts)\n\t\tgw.RUnlock()\n\t}\n\ts.mu.Unlock()\n\tif len(disconnectClients) > 0 {\n\t\ts.Noticef(\"Disconnect %d clients due to pinned certs reload\", len(disconnectClients))\n\t\tfor _, c := range disconnectClients {\n\t\t\tc.closeConnection(TLSHandshakeError)\n\t\t}\n\t}\n}\n\ntype proxiesReload struct {\n\tnoopOption\n\tadd []string\n\tdel []string\n}\n\nfunc (p *proxiesReload) Apply(s *Server) {\n\tvar clients []*client\n\ts.mu.Lock()\n\tfor _, k := range p.del {\n\t\tcc := s.proxiedConns[k]\n\t\tdelete(s.proxiedConns, k)\n\t\tif len(cc) > 0 {\n\t\t\tfor _, c := range cc {\n\t\t\t\tclients = append(clients, c)\n\t\t\t}\n\t\t}\n\t}\n\ts.processProxiesTrustedKeys()\n\ts.mu.Unlock()\n\tif len(p.del) > 0 {\n\t\tfor _, c := range clients {\n\t\t\tc.setAuthError(ErrAuthProxyNotTrusted)\n\t\t\tc.authViolation()\n\t\t}\n\t\ts.Noticef(\"Reloaded: proxies trusted keys %q were removed\", p.del)\n\t}\n\tif len(p.add) > 0 {\n\t\ts.Noticef(\"Reloaded: proxies trusted keys %q were added\", p.add)\n\t}\n}\n\n// Reload reads the current configuration file and calls out to ReloadOptions\n// to apply the changes. This returns an error if the server was not started\n// with a config file or an option which doesn't support hot-swapping was changed.\nfunc (s *Server) Reload() error {\n\ts.mu.Lock()\n\tconfigFile := s.configFile\n\ts.mu.Unlock()\n\tif configFile == \"\" {\n\t\treturn errors.New(\"can only reload config when a file is provided using -c or --config\")\n\t}\n\n\tnewOpts, err := ProcessConfigFile(configFile)\n\tif err != nil {\n\t\t// TODO: Dump previous good config to a .bak file?\n\t\treturn err\n\t}\n\treturn s.ReloadOptions(newOpts)\n}\n\n// ReloadOptions applies any supported options from the provided Options\n// type. This returns an error if an option which doesn't support\n// hot-swapping was changed.\n// The provided Options type should not be re-used afterwards.\n// Either use Options.Clone() to pass a copy, or make a new one.\nfunc (s *Server) ReloadOptions(newOpts *Options) error {\n\ts.reloadMu.Lock()\n\tdefer s.reloadMu.Unlock()\n\n\ts.mu.Lock()\n\n\tcurOpts := s.getOpts()\n\n\t// Wipe trusted keys if needed when we have an operator.\n\tif len(curOpts.TrustedOperators) > 0 && len(curOpts.TrustedKeys) > 0 {\n\t\tcurOpts.TrustedKeys = nil\n\t}\n\n\tclientOrgPort := curOpts.Port\n\tclusterOrgPort := curOpts.Cluster.Port\n\tgatewayOrgPort := curOpts.Gateway.Port\n\tleafnodesOrgPort := curOpts.LeafNode.Port\n\twebsocketOrgPort := curOpts.Websocket.Port\n\tmqttOrgPort := curOpts.MQTT.Port\n\n\ts.mu.Unlock()\n\n\t// In case \"-cluster ...\" was provided through the command line, this will\n\t// properly set the Cluster.Host/Port etc...\n\tif l := curOpts.Cluster.ListenStr; l != _EMPTY_ {\n\t\tnewOpts.Cluster.ListenStr = l\n\t\toverrideCluster(newOpts)\n\t}\n\n\t// Apply flags over config file settings.\n\tnewOpts = MergeOptions(newOpts, FlagSnapshot)\n\n\t// Need more processing for boolean flags...\n\tif FlagSnapshot != nil {\n\t\tapplyBoolFlags(newOpts, FlagSnapshot)\n\t}\n\n\tsetBaselineOptions(newOpts)\n\n\t// setBaselineOptions sets Port to 0 if set to -1 (RANDOM port)\n\t// If that's the case, set it to the saved value when the accept loop was\n\t// created.\n\tif newOpts.Port == 0 {\n\t\tnewOpts.Port = clientOrgPort\n\t}\n\t// We don't do that for cluster, so check against -1.\n\tif newOpts.Cluster.Port == -1 {\n\t\tnewOpts.Cluster.Port = clusterOrgPort\n\t}\n\tif newOpts.Gateway.Port == -1 {\n\t\tnewOpts.Gateway.Port = gatewayOrgPort\n\t}\n\tif newOpts.LeafNode.Port == -1 {\n\t\tnewOpts.LeafNode.Port = leafnodesOrgPort\n\t}\n\tif newOpts.Websocket.Port == -1 {\n\t\tnewOpts.Websocket.Port = websocketOrgPort\n\t}\n\tif newOpts.MQTT.Port == -1 {\n\t\tnewOpts.MQTT.Port = mqttOrgPort\n\t}\n\n\tif err := s.reloadOptions(curOpts, newOpts); err != nil {\n\t\treturn err\n\t}\n\n\ts.recheckPinnedCerts(curOpts, newOpts)\n\n\ts.mu.Lock()\n\ts.configTime = time.Now().UTC()\n\ts.updateVarzConfigReloadableFields(s.varz)\n\ts.mu.Unlock()\n\treturn nil\n}\nfunc applyBoolFlags(newOpts, flagOpts *Options) {\n\t// Reset fields that may have been set to `true` in\n\t// MergeOptions() when some of the flags default to `true`\n\t// but have not been explicitly set and therefore value\n\t// from config file should take precedence.\n\tfor name, val := range newOpts.inConfig {\n\t\tf := reflect.ValueOf(newOpts).Elem()\n\t\tnames := strings.Split(name, \".\")\n\t\tfor _, name := range names {\n\t\t\tf = f.FieldByName(name)\n\t\t}\n\t\tf.SetBool(val)\n\t}\n\t// Now apply value (true or false) from flags that have\n\t// been explicitly set in command line\n\tfor name, val := range flagOpts.inCmdLine {\n\t\tf := reflect.ValueOf(newOpts).Elem()\n\t\tnames := strings.Split(name, \".\")\n\t\tfor _, name := range names {\n\t\t\tf = f.FieldByName(name)\n\t\t}\n\t\tf.SetBool(val)\n\t}\n}\n\n// reloadOptions reloads the server config with the provided options. If an\n// option that doesn't support hot-swapping is changed, this returns an error.\nfunc (s *Server) reloadOptions(curOpts, newOpts *Options) error {\n\t// Apply to the new options some of the options that may have been set\n\t// that can't be configured in the config file (this can happen in\n\t// applications starting NATS Server programmatically).\n\tnewOpts.CustomClientAuthentication = curOpts.CustomClientAuthentication\n\tnewOpts.CustomRouterAuthentication = curOpts.CustomRouterAuthentication\n\n\tchanged, err := s.diffOptions(newOpts)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif len(changed) != 0 {\n\t\tif err := validateOptions(newOpts); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Create a context that is used to pass special info that we may need\n\t// while applying the new options.\n\tctx := reloadContext{oldClusterPerms: curOpts.Cluster.Permissions}\n\ts.setOpts(newOpts)\n\ts.applyOptions(&ctx, changed)\n\treturn nil\n}\n\n// For the purpose of comparing, impose a order on slice data types where order does not matter\nfunc imposeOrder(value any) error {\n\tswitch value := value.(type) {\n\tcase []*Account:\n\t\tslices.SortFunc(value, func(i, j *Account) int { return cmp.Compare(i.Name, j.Name) })\n\t\tfor _, a := range value {\n\t\t\tslices.SortFunc(a.imports.streams, func(i, j *streamImport) int { return cmp.Compare(i.acc.Name, j.acc.Name) })\n\t\t}\n\tcase []*User:\n\t\tslices.SortFunc(value, func(i, j *User) int { return cmp.Compare(i.Username, j.Username) })\n\tcase []*NkeyUser:\n\t\tslices.SortFunc(value, func(i, j *NkeyUser) int { return cmp.Compare(i.Nkey, j.Nkey) })\n\tcase []*url.URL:\n\t\tslices.SortFunc(value, func(i, j *url.URL) int { return cmp.Compare(i.String(), j.String()) })\n\tcase []string:\n\t\tslices.Sort(value)\n\tcase []*jwt.OperatorClaims:\n\t\tslices.SortFunc(value, func(i, j *jwt.OperatorClaims) int { return cmp.Compare(i.Issuer, j.Issuer) })\n\tcase GatewayOpts:\n\t\tslices.SortFunc(value.Gateways, func(i, j *RemoteGatewayOpts) int { return cmp.Compare(i.Name, j.Name) })\n\tcase WebsocketOpts:\n\t\tslices.Sort(value.AllowedOrigins)\n\tcase string, bool, uint8, uint16, uint64, int, int32, int64, time.Duration, float64, nil, LeafNodeOpts, ClusterOpts, *tls.Config, PinnedCertSet,\n\t\t*URLAccResolver, *MemAccResolver, *DirAccResolver, *CacheDirAccResolver, Authentication, MQTTOpts, jwt.TagList,\n\t\t*OCSPConfig, map[string]string, JSLimitOpts, StoreCipher, *OCSPResponseCacheConfig, *ProxiesConfig, WriteTimeoutPolicy:\n\t\t// explicitly skipped types\n\tcase *AuthCallout:\n\tcase JSTpmOpts:\n\tdefault:\n\t\t// this will fail during unit tests\n\t\treturn fmt.Errorf(\"OnReload, sort or explicitly skip type: %s\",\n\t\t\treflect.TypeOf(value))\n\t}\n\treturn nil\n}\n\n// diffOptions returns a slice containing options which have been changed. If\n// an option that doesn't support hot-swapping is changed, this returns an\n// error.\nfunc (s *Server) diffOptions(newOpts *Options) ([]option, error) {\n\tvar (\n\t\toldConfig = reflect.ValueOf(s.getOpts()).Elem()\n\t\tnewConfig = reflect.ValueOf(newOpts).Elem()\n\t\tdiffOpts = []option{}\n\n\t\t// Need to keep track of whether JS is being disabled\n\t\t// to prevent changing limits at runtime.\n\t\tjsEnabled = s.JetStreamEnabled()\n\t\tdisableJS bool\n\t\tjsMemLimitsChanged bool\n\t\tjsFileLimitsChanged bool\n\t\tjsStoreDirChanged bool\n\t)\n\tfor i := 0; i < oldConfig.NumField(); i++ {\n\t\tfield := oldConfig.Type().Field(i)\n\t\t// field.PkgPath is empty for exported fields, and is not for unexported ones.\n\t\t// We skip the unexported fields.\n\t\tif field.PkgPath != _EMPTY_ {\n\t\t\tcontinue\n\t\t}\n\t\tvar (\n\t\t\toldValue = oldConfig.Field(i).Interface()\n\t\t\tnewValue = newConfig.Field(i).Interface()\n\t\t)\n\t\tif err := imposeOrder(oldValue); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif err := imposeOrder(newValue); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\toptName := strings.ToLower(field.Name)\n\t\t// accounts and users (referencing accounts) will always differ as accounts\n\t\t// contain internal state, say locks etc..., so we don't bother here.\n\t\t// This also avoids races with atomic stats counters\n\t\tif optName != \"accounts\" && optName != \"users\" {\n\t\t\tif changed := !reflect.DeepEqual(oldValue, newValue); !changed {\n\t\t\t\t// Check to make sure we are running JetStream if we think we should be.\n\t\t\t\tif optName == \"jetstream\" && newValue.(bool) {\n\t\t\t\t\tif !jsEnabled {\n\t\t\t\t\t\tdiffOpts = append(diffOpts, &jetStreamOption{newValue: true})\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tswitch optName {\n\t\tcase \"traceverbose\":\n\t\t\tdiffOpts = append(diffOpts, &traceVerboseOption{newValue: newValue.(bool)})\n\t\tcase \"traceheaders\":\n\t\t\tdiffOpts = append(diffOpts, &traceHeadersOption{newValue: newValue.(bool)})\n\t\tcase \"trace\":\n\t\t\tdiffOpts = append(diffOpts, &traceOption{newValue: newValue.(bool)})\n\t\tcase \"debug\":\n\t\t\tdiffOpts = append(diffOpts, &debugOption{newValue: newValue.(bool)})\n\t\tcase \"logtime\":\n\t\t\tdiffOpts = append(diffOpts, &logtimeOption{newValue: newValue.(bool)})\n\t\tcase \"logtimeutc\":\n\t\t\tdiffOpts = append(diffOpts, &logtimeUTCOption{newValue: newValue.(bool)})\n\t\tcase \"logfile\":\n\t\t\tdiffOpts = append(diffOpts, &logfileOption{newValue: newValue.(string)})\n\t\tcase \"syslog\":\n\t\t\tdiffOpts = append(diffOpts, &syslogOption{newValue: newValue.(bool)})\n\t\tcase \"remotesyslog\":\n\t\t\tdiffOpts = append(diffOpts, &remoteSyslogOption{newValue: newValue.(string)})\n\t\tcase \"tlsconfig\":\n\t\t\tdiffOpts = append(diffOpts, &tlsOption{newValue: newValue.(*tls.Config)})\n\t\tcase \"tlstimeout\":\n\t\t\tdiffOpts = append(diffOpts, &tlsTimeoutOption{newValue: newValue.(float64)})\n\t\tcase \"tlspinnedcerts\":\n\t\t\tdiffOpts = append(diffOpts, &tlsPinnedCertOption{newValue: newValue.(PinnedCertSet)})\n\t\tcase \"tlshandshakefirst\":\n\t\t\tdiffOpts = append(diffOpts, &tlsHandshakeFirst{newValue: newValue.(bool)})\n\t\tcase \"tlshandshakefirstfallback\":\n\t\t\tdiffOpts = append(diffOpts, &tlsHandshakeFirstFallback{newValue: newValue.(time.Duration)})\n\t\tcase \"username\":\n\t\t\tdiffOpts = append(diffOpts, &usernameOption{})\n\t\tcase \"password\":\n\t\t\tdiffOpts = append(diffOpts, &passwordOption{})\n\t\tcase \"tags\":\n\t\t\tdiffOpts = append(diffOpts, &tagsOption{})\n\t\tcase \"metadata\":\n\t\t\tdiffOpts = append(diffOpts, &metadataOption{})\n\t\tcase \"authorization\":\n\t\t\tdiffOpts = append(diffOpts, &authorizationOption{})\n\t\tcase \"authtimeout\":\n\t\t\tdiffOpts = append(diffOpts, &authTimeoutOption{newValue: newValue.(float64)})\n\t\tcase \"users\":\n\t\t\tdiffOpts = append(diffOpts, &usersOption{})\n\t\tcase \"nkeys\":\n\t\t\tdiffOpts = append(diffOpts, &nkeysOption{})\n\t\tcase \"cluster\":\n\t\t\tnewClusterOpts := newValue.(ClusterOpts)\n\t\t\toldClusterOpts := oldValue.(ClusterOpts)\n\t\t\tif err := validateClusterOpts(oldClusterOpts, newClusterOpts); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tco := &clusterOption{\n\t\t\t\tnewValue: newClusterOpts,\n\t\t\t\tpermsChanged: !reflect.DeepEqual(newClusterOpts.Permissions, oldClusterOpts.Permissions),\n\t\t\t\tcompressChanged: !compressOptsEqual(&oldClusterOpts.Compression, &newClusterOpts.Compression),\n\t\t\t}\n\t\t\tco.diffPoolAndAccounts(&oldClusterOpts)\n\t\t\t// If there are added accounts, first make sure that we can look them up.\n\t\t\t// If we can't let's fail the reload.\n\t\t\tfor _, acc := range co.accsAdded {\n\t\t\t\tif _, err := s.LookupAccount(acc); err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"unable to add account %q to the list of dedicated routes: %v\", acc, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If pool_size has been set to negative (but was not before), then let's\n\t\t\t// add the system account to the list of removed accounts (we don't have\n\t\t\t// to check if already there, duplicates are ok in that case).\n\t\t\tif newClusterOpts.PoolSize < 0 && oldClusterOpts.PoolSize >= 0 {\n\t\t\t\tif sys := s.SystemAccount(); sys != nil {\n\t\t\t\t\tco.accsRemoved = append(co.accsRemoved, sys.GetName())\n\t\t\t\t}\n\t\t\t}\n\t\t\tdiffOpts = append(diffOpts, co)\n\t\tcase \"routes\":\n\t\t\tadd, remove := diffRoutes(oldValue.([]*url.URL), newValue.([]*url.URL))\n\t\t\tdiffOpts = append(diffOpts, &routesOption{add: add, remove: remove})\n\t\tcase \"maxconn\":\n\t\t\tdiffOpts = append(diffOpts, &maxConnOption{newValue: newValue.(int)})\n\t\tcase \"pidfile\":\n\t\t\tdiffOpts = append(diffOpts, &pidFileOption{newValue: newValue.(string)})\n\t\tcase \"portsfiledir\":\n\t\t\tdiffOpts = append(diffOpts, &portsFileDirOption{newValue: newValue.(string), oldValue: oldValue.(string)})\n\t\tcase \"maxcontrolline\":\n\t\t\tdiffOpts = append(diffOpts, &maxControlLineOption{newValue: newValue.(int32)})\n\t\tcase \"maxpayload\":\n\t\t\tdiffOpts = append(diffOpts, &maxPayloadOption{newValue: newValue.(int32)})\n\t\tcase \"pinginterval\":\n\t\t\tdiffOpts = append(diffOpts, &pingIntervalOption{newValue: newValue.(time.Duration)})\n\t\tcase \"maxpingsout\":\n\t\t\tdiffOpts = append(diffOpts, &maxPingsOutOption{newValue: newValue.(int)})\n\t\tcase \"writedeadline\":\n\t\t\tdiffOpts = append(diffOpts, &writeDeadlineOption{newValue: newValue.(time.Duration)})\n\t\tcase \"clientadvertise\":\n\t\t\tcliAdv := newValue.(string)\n\t\t\tif cliAdv != \"\" {\n\t\t\t\t// Validate ClientAdvertise syntax\n\t\t\t\tif _, _, err := parseHostPort(cliAdv, 0); err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"invalid ClientAdvertise value of %s, err=%v\", cliAdv, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tdiffOpts = append(diffOpts, &clientAdvertiseOption{newValue: cliAdv})\n\t\tcase \"accounts\":\n\t\t\tdiffOpts = append(diffOpts, &accountsOption{})\n\t\tcase \"resolver\", \"accountresolver\", \"accountsresolver\":\n\t\t\t// We can't move from no resolver to one. So check for that.\n\t\t\tif (oldValue == nil && newValue != nil) ||\n\t\t\t\t(oldValue != nil && newValue == nil) {\n\t\t\t\treturn nil, fmt.Errorf(\"config reload does not support moving to or from an account resolver\")\n\t\t\t}\n\t\t\tdiffOpts = append(diffOpts, &accountsOption{})\n\t\tcase \"accountresolvertlsconfig\":\n\t\t\tdiffOpts = append(diffOpts, &accountsOption{})\n\t\tcase \"gateway\":\n\t\t\t// Not supported for now, but report warning if configuration of gateway\n\t\t\t// is actually changed so that user knows that it won't take effect.\n\n\t\t\t// Any deep-equal is likely to fail for when there is a TLSConfig. so\n\t\t\t// remove for the test.\n\t\t\ttmpOld := oldValue.(GatewayOpts)\n\t\t\ttmpNew := newValue.(GatewayOpts)\n\t\t\ttmpOld.TLSConfig = nil\n\t\t\ttmpNew.TLSConfig = nil\n\t\t\ttmpOld.tlsConfigOpts = nil\n\t\t\ttmpNew.tlsConfigOpts = nil\n\n\t\t\t// Need to do the same for remote gateways' TLS configs.\n\t\t\t// But we can't just set remotes' TLSConfig to nil otherwise this\n\t\t\t// would lose the real TLS configuration.\n\t\t\ttmpOld.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpOld.Gateways)\n\t\t\ttmpNew.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpNew.Gateways)\n\n\t\t\t// If there is really a change prevents reload.\n\t\t\tif !reflect.DeepEqual(tmpOld, tmpNew) {\n\t\t\t\t// See TODO(ik) note below about printing old/new values.\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\t\tcase \"leafnode\":\n\t\t\t// Similar to gateways\n\t\t\ttmpOld := oldValue.(LeafNodeOpts)\n\t\t\ttmpNew := newValue.(LeafNodeOpts)\n\t\t\ttmpOld.TLSConfig = nil\n\t\t\ttmpNew.TLSConfig = nil\n\t\t\ttmpOld.tlsConfigOpts = nil\n\t\t\ttmpNew.tlsConfigOpts = nil\n\t\t\t// We will allow TLSHandshakeFirst to be config reloaded. First,\n\t\t\t// we just want to detect if there was a change in the leafnodes{}\n\t\t\t// block, and if not, we will check the remotes.\n\t\t\thandshakeFirstChanged := tmpOld.TLSHandshakeFirst != tmpNew.TLSHandshakeFirst ||\n\t\t\t\ttmpOld.TLSHandshakeFirstFallback != tmpNew.TLSHandshakeFirstFallback\n\t\t\t// If changed, set them (in the temporary variables) to false so that the\n\t\t\t// rest of the comparison does not fail.\n\t\t\tif handshakeFirstChanged {\n\t\t\t\ttmpOld.TLSHandshakeFirst, tmpNew.TLSHandshakeFirst = false, false\n\t\t\t\ttmpOld.TLSHandshakeFirstFallback, tmpNew.TLSHandshakeFirstFallback = 0, 0\n\t\t\t} else if len(tmpOld.Remotes) == len(tmpNew.Remotes) {\n\t\t\t\t// Since we don't support changes in the remotes, we will do a\n\t\t\t\t// simple pass to see if there was a change of this field.\n\t\t\t\tfor i := 0; i < len(tmpOld.Remotes); i++ {\n\t\t\t\t\tif tmpOld.Remotes[i].TLSHandshakeFirst != tmpNew.Remotes[i].TLSHandshakeFirst {\n\t\t\t\t\t\thandshakeFirstChanged = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// We also support config reload for compression. Check if it changed before\n\t\t\t// blanking them out for the deep-equal check at the end.\n\t\t\tcompressionChanged := !compressOptsEqual(&tmpOld.Compression, &tmpNew.Compression)\n\t\t\tif compressionChanged {\n\t\t\t\ttmpOld.Compression, tmpNew.Compression = CompressionOpts{}, CompressionOpts{}\n\t\t\t} else if len(tmpOld.Remotes) == len(tmpNew.Remotes) {\n\t\t\t\t// Same that for tls first check, do the remotes now.\n\t\t\t\tfor i := range len(tmpOld.Remotes) {\n\t\t\t\t\tif !compressOptsEqual(&tmpOld.Remotes[i].Compression, &tmpNew.Remotes[i].Compression) {\n\t\t\t\t\t\tcompressionChanged = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Check if the \"disabled\" option of each remote has changed.\n\t\t\tvar disabledChanged bool\n\t\t\tfor i := range len(tmpOld.Remotes) {\n\t\t\t\tif tmpOld.Remotes[i].Disabled != tmpNew.Remotes[i].Disabled {\n\t\t\t\t\tdisabledChanged = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Need to do the same for remote leafnodes' TLS configs.\n\t\t\t// But we can't just set remotes' TLSConfig to nil otherwise this\n\t\t\t// would lose the real TLS configuration.\n\t\t\ttmpOld.Remotes = copyRemoteLNConfigForReloadCompare(tmpOld.Remotes)\n\t\t\ttmpNew.Remotes = copyRemoteLNConfigForReloadCompare(tmpNew.Remotes)\n\n\t\t\t// Special check for leafnode remotes changes which are not supported right now.\n\t\t\tleafRemotesChanged := func(a, b LeafNodeOpts) bool {\n\t\t\t\tif len(a.Remotes) != len(b.Remotes) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\n\t\t\t\t// Check whether all remotes URLs are still the same.\n\t\t\t\tfor _, oldRemote := range a.Remotes {\n\t\t\t\t\tvar found bool\n\n\t\t\t\t\tif oldRemote.LocalAccount == _EMPTY_ {\n\t\t\t\t\t\toldRemote.LocalAccount = globalAccountName\n\t\t\t\t\t}\n\n\t\t\t\t\tfor _, newRemote := range b.Remotes {\n\t\t\t\t\t\t// Bind to global account in case not defined.\n\t\t\t\t\t\tif newRemote.LocalAccount == _EMPTY_ {\n\t\t\t\t\t\t\tnewRemote.LocalAccount = globalAccountName\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif reflect.DeepEqual(oldRemote, newRemote) {\n\t\t\t\t\t\t\tfound = true\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif !found {\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\t// First check whether remotes changed at all. If they did not,\n\t\t\t// skip them in the complete equal check.\n\t\t\tif !leafRemotesChanged(tmpOld, tmpNew) {\n\t\t\t\ttmpOld.Remotes = nil\n\t\t\t\ttmpNew.Remotes = nil\n\t\t\t}\n\n\t\t\t// Special check for auth users to detect changes.\n\t\t\t// If anything is off will fall through and fail below.\n\t\t\t// If we detect they are semantically the same we nil them out\n\t\t\t// to pass the check below.\n\t\t\tif tmpOld.Users != nil || tmpNew.Users != nil {\n\t\t\t\tif len(tmpOld.Users) == len(tmpNew.Users) {\n\t\t\t\t\toua := make(map[string]*User, len(tmpOld.Users))\n\t\t\t\t\tnua := make(map[string]*User, len(tmpOld.Users))\n\t\t\t\t\tfor _, u := range tmpOld.Users {\n\t\t\t\t\t\toua[u.Username] = u\n\t\t\t\t\t}\n\t\t\t\t\tfor _, u := range tmpNew.Users {\n\t\t\t\t\t\tnua[u.Username] = u\n\t\t\t\t\t}\n\t\t\t\t\tsame := true\n\t\t\t\t\tfor uname, u := range oua {\n\t\t\t\t\t\t// If we can not find new one with same name, drop through to fail.\n\t\t\t\t\t\tnu, ok := nua[uname]\n\t\t\t\t\t\tif !ok {\n\t\t\t\t\t\t\tsame = false\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// If username or password or account different break.\n\t\t\t\t\t\tif u.Username != nu.Username || u.Password != nu.Password || u.Account.GetName() != nu.Account.GetName() {\n\t\t\t\t\t\t\tsame = false\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// We can nil out here.\n\t\t\t\t\tif same {\n\t\t\t\t\t\ttmpOld.Users, tmpNew.Users = nil, nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// If there is really a change prevents reload.\n\t\t\tif !reflect.DeepEqual(tmpOld, tmpNew) {\n\t\t\t\t// See TODO(ik) note below about printing old/new values.\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\n\t\t\tdiffOpts = append(diffOpts, &leafNodeOption{\n\t\t\t\ttlsFirstChanged: handshakeFirstChanged,\n\t\t\t\tcompressionChanged: compressionChanged,\n\t\t\t\tdisabledChanged: disabledChanged,\n\t\t\t})\n\t\tcase \"jetstream\":\n\t\t\tnew := newValue.(bool)\n\t\t\told := oldValue.(bool)\n\t\t\tif new != old {\n\t\t\t\tdiffOpts = append(diffOpts, &jetStreamOption{newValue: new})\n\t\t\t}\n\n\t\t\t// Mark whether JS will be disabled.\n\t\t\tdisableJS = !new\n\t\tcase \"storedir\":\n\t\t\tnew := newValue.(string)\n\t\t\told := oldValue.(string)\n\t\t\tmodified := new != old\n\n\t\t\t// Check whether JS is being disabled and/or storage dir attempted to change.\n\t\t\tif jsEnabled && modified {\n\t\t\t\tif new == _EMPTY_ {\n\t\t\t\t\t// This means that either JS is being disabled or it is using an temp dir.\n\t\t\t\t\t// Allow the change but error in case JS was not disabled.\n\t\t\t\t\tjsStoreDirChanged = true\n\t\t\t\t} else {\n\t\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream storage directory\")\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"jetstreammaxmemory\", \"jetstreammaxstore\":\n\t\t\told := oldValue.(int64)\n\t\t\tnew := newValue.(int64)\n\n\t\t\t// Check whether JS is being disabled and/or limits are being changed.\n\t\t\tvar (\n\t\t\t\tmodified = new != old\n\t\t\t\tfromUnset = old == -1\n\t\t\t\tfromSet = !fromUnset\n\t\t\t\ttoUnset = new == -1\n\t\t\t\ttoSet = !toUnset\n\t\t\t)\n\t\t\tif jsEnabled && modified {\n\t\t\t\t// Cannot change limits from dynamic storage at runtime.\n\t\t\t\tswitch {\n\t\t\t\tcase fromSet && toUnset:\n\t\t\t\t\t// Limits changed but it may mean that JS is being disabled,\n\t\t\t\t\t// keep track of the change and error in case it is not.\n\t\t\t\t\tswitch optName {\n\t\t\t\t\tcase \"jetstreammaxmemory\":\n\t\t\t\t\t\tjsMemLimitsChanged = true\n\t\t\t\t\tcase \"jetstreammaxstore\":\n\t\t\t\t\t\tjsFileLimitsChanged = true\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream max memory and store\")\n\t\t\t\t\t}\n\t\t\t\tcase fromUnset && toSet:\n\t\t\t\t\t// Prevent changing from dynamic max memory / file at runtime.\n\t\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream dynamic max memory and store\")\n\t\t\t\tdefault:\n\t\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream max memory and store\")\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"jetstreammetacompact\", \"jetstreammetacompactsize\", \"jetstreammetacompactsync\":\n\t\t\t// Allowed at runtime but monitorCluster looks at s.opts directly, so no further work needed here.\n\t\tcase \"websocket\":\n\t\t\t// Similar to gateways\n\t\t\ttmpOld := oldValue.(WebsocketOpts)\n\t\t\ttmpNew := newValue.(WebsocketOpts)\n\t\t\ttmpOld.TLSConfig, tmpOld.tlsConfigOpts = nil, nil\n\t\t\ttmpNew.TLSConfig, tmpNew.tlsConfigOpts = nil, nil\n\t\t\t// If there is really a change prevents reload.\n\t\t\tif !reflect.DeepEqual(tmpOld, tmpNew) {\n\t\t\t\t// See TODO(ik) note below about printing old/new values.\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\t\tcase \"mqtt\":\n\t\t\tdiffOpts = append(diffOpts, &mqttAckWaitReload{newValue: newValue.(MQTTOpts).AckWait})\n\t\t\tdiffOpts = append(diffOpts, &mqttMaxAckPendingReload{newValue: newValue.(MQTTOpts).MaxAckPending})\n\t\t\tdiffOpts = append(diffOpts, &mqttStreamReplicasReload{newValue: newValue.(MQTTOpts).StreamReplicas})\n\t\t\tdiffOpts = append(diffOpts, &mqttConsumerReplicasReload{newValue: newValue.(MQTTOpts).ConsumerReplicas})\n\t\t\tdiffOpts = append(diffOpts, &mqttConsumerMemoryStorageReload{newValue: newValue.(MQTTOpts).ConsumerMemoryStorage})\n\t\t\tdiffOpts = append(diffOpts, &mqttInactiveThresholdReload{newValue: newValue.(MQTTOpts).ConsumerInactiveThreshold})\n\n\t\t\t// Nil out/set to 0 the options that we allow to be reloaded so that\n\t\t\t// we only fail reload if some that we don't support are changed.\n\t\t\ttmpOld := oldValue.(MQTTOpts)\n\t\t\ttmpNew := newValue.(MQTTOpts)\n\t\t\ttmpOld.TLSConfig, tmpOld.tlsConfigOpts, tmpOld.AckWait, tmpOld.MaxAckPending, tmpOld.StreamReplicas, tmpOld.ConsumerReplicas, tmpOld.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false\n\t\t\ttmpOld.ConsumerInactiveThreshold = 0\n\t\t\ttmpNew.TLSConfig, tmpNew.tlsConfigOpts, tmpNew.AckWait, tmpNew.MaxAckPending, tmpNew.StreamReplicas, tmpNew.ConsumerReplicas, tmpNew.ConsumerMemoryStorage = nil, nil, 0, 0, 0, 0, false\n\t\t\ttmpNew.ConsumerInactiveThreshold = 0\n\n\t\t\tif !reflect.DeepEqual(tmpOld, tmpNew) {\n\t\t\t\t// See TODO(ik) note below about printing old/new values.\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\t\t\ttmpNew.AckWait = newValue.(MQTTOpts).AckWait\n\t\t\ttmpNew.MaxAckPending = newValue.(MQTTOpts).MaxAckPending\n\t\t\ttmpNew.StreamReplicas = newValue.(MQTTOpts).StreamReplicas\n\t\t\ttmpNew.ConsumerReplicas = newValue.(MQTTOpts).ConsumerReplicas\n\t\t\ttmpNew.ConsumerMemoryStorage = newValue.(MQTTOpts).ConsumerMemoryStorage\n\t\t\ttmpNew.ConsumerInactiveThreshold = newValue.(MQTTOpts).ConsumerInactiveThreshold\n\t\tcase \"connecterrorreports\":\n\t\t\tdiffOpts = append(diffOpts, &connectErrorReports{newValue: newValue.(int)})\n\t\tcase \"reconnecterrorreports\":\n\t\t\tdiffOpts = append(diffOpts, &reconnectErrorReports{newValue: newValue.(int)})\n\t\tcase \"nolog\", \"nosigs\":\n\t\t\t// Ignore NoLog and NoSigs options since they are not parsed and only used in\n\t\t\t// testing.\n\t\t\tcontinue\n\t\tcase \"disableshortfirstping\":\n\t\t\tnewOpts.DisableShortFirstPing = oldValue.(bool)\n\t\t\tcontinue\n\t\tcase \"maxtracedmsglen\":\n\t\t\tdiffOpts = append(diffOpts, &maxTracedMsgLenOption{newValue: newValue.(int)})\n\t\tcase \"port\":\n\t\t\t// check to see if newValue == 0 and continue if so.\n\t\t\tif newValue == 0 {\n\t\t\t\t// ignore RANDOM_PORT\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfallthrough\n\t\tcase \"noauthuser\":\n\t\t\tif oldValue != _EMPTY_ && newValue == _EMPTY_ {\n\t\t\t\tfor _, user := range newOpts.Users {\n\t\t\t\t\tif user.Username == oldValue {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\t\tcase \"defaultsentinel\":\n\t\t\tdiffOpts = append(diffOpts, &defaultSentinelOption{newValue: newValue.(string)})\n\t\tcase \"systemaccount\":\n\t\t\tif oldValue != DEFAULT_SYSTEM_ACCOUNT || newValue != _EMPTY_ {\n\t\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\t\tfield.Name, oldValue, newValue)\n\t\t\t}\n\t\tcase \"ocspconfig\":\n\t\t\tdiffOpts = append(diffOpts, &ocspOption{newValue: newValue.(*OCSPConfig)})\n\t\tcase \"ocspcacheconfig\":\n\t\t\tdiffOpts = append(diffOpts, &ocspResponseCacheOption{newValue: newValue.(*OCSPResponseCacheConfig)})\n\t\tcase \"profblockrate\":\n\t\t\tnew := newValue.(int)\n\t\t\told := oldValue.(int)\n\t\t\tif new != old {\n\t\t\t\tdiffOpts = append(diffOpts, &profBlockRateReload{newValue: new})\n\t\t\t}\n\t\tcase \"configdigest\":\n\t\t\t// skip changes in config digest, this is handled already while\n\t\t\t// processing the config.\n\t\t\tcontinue\n\t\tcase \"nofastproducerstall\":\n\t\t\tdiffOpts = append(diffOpts, &noFastProdStallReload{noStall: newValue.(bool)})\n\t\tcase \"proxies\":\n\t\t\tnew := newValue.(*ProxiesConfig)\n\t\t\told := oldValue.(*ProxiesConfig)\n\t\t\tif add, del := diffProxiesTrustedKeys(old.Trusted, new.Trusted); len(add) > 0 || len(del) > 0 {\n\t\t\t\tdiffOpts = append(diffOpts, &proxiesReload{add: add, del: del})\n\t\t\t}\n\t\tdefault:\n\t\t\t// TODO(ik): Implement String() on those options to have a nice print.\n\t\t\t// %v is difficult to figure what's what, %+v print private fields and\n\t\t\t// would print passwords. Tried json.Marshal but it is too verbose for\n\t\t\t// the URL array.\n\n\t\t\t// Bail out if attempting to reload any unsupported options.\n\t\t\treturn nil, fmt.Errorf(\"config reload not supported for %s: old=%v, new=%v\",\n\t\t\t\tfield.Name, oldValue, newValue)\n\t\t}\n\t}\n\n\t// If not disabling JS but limits have changed then it is an error.\n\tif !disableJS {\n\t\tif jsMemLimitsChanged || jsFileLimitsChanged {\n\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream max memory and max store\")\n\t\t}\n\t\tif jsStoreDirChanged {\n\t\t\treturn nil, fmt.Errorf(\"config reload not supported for jetstream storage dir\")\n\t\t}\n\t}\n\n\treturn diffOpts, nil\n}\n\nfunc copyRemoteGWConfigsWithoutTLSConfig(current []*RemoteGatewayOpts) []*RemoteGatewayOpts {\n\tl := len(current)\n\tif l == 0 {\n\t\treturn nil\n\t}\n\trgws := make([]*RemoteGatewayOpts, 0, l)\n\tfor _, rcfg := range current {\n\t\tcp := *rcfg\n\t\tcp.TLSConfig = nil\n\t\tcp.tlsConfigOpts = nil\n\t\trgws = append(rgws, &cp)\n\t}\n\treturn rgws\n}\n\nfunc copyRemoteLNConfigForReloadCompare(current []*RemoteLeafOpts) []*RemoteLeafOpts {\n\tl := len(current)\n\tif l == 0 {\n\t\treturn nil\n\t}\n\trlns := make([]*RemoteLeafOpts, 0, l)\n\tfor _, rcfg := range current {\n\t\tcp := *rcfg\n\t\tcp.TLSConfig = nil\n\t\tcp.tlsConfigOpts = nil\n\t\tcp.TLSHandshakeFirst = false\n\t\t// This is set only when processing a CONNECT, so reset here so that we\n\t\t// don't fail the DeepEqual comparison.\n\t\tcp.TLS = false\n\t\t// For now, remove DenyImports/Exports since those get modified at runtime\n\t\t// to add JS APIs.\n\t\tcp.DenyImports, cp.DenyExports = nil, nil\n\t\t// Remove compression mode\n\t\tcp.Compression = CompressionOpts{}\n\t\t// Reset disabled status\n\t\tcp.Disabled = false\n\t\trlns = append(rlns, &cp)\n\t}\n\treturn rlns\n}\n\nfunc (s *Server) applyOptions(ctx *reloadContext, opts []option) {\n\tvar (\n\t\treloadLogging = false\n\t\treloadAuth = false\n\t\treloadClusterPerms = false\n\t\treloadClientTrcLvl = false\n\t\treloadJetstream = false\n\t\tjsEnabled = false\n\t\tisStatszChange = false\n\t\tco *clusterOption\n\t)\n\tfor _, opt := range opts {\n\t\topt.Apply(s)\n\t\tif opt.IsLoggingChange() {\n\t\t\treloadLogging = true\n\t\t}\n\t\tif opt.IsTraceLevelChange() {\n\t\t\treloadClientTrcLvl = true\n\t\t}\n\t\tif opt.IsAuthChange() {\n\t\t\treloadAuth = true\n\t\t}\n\t\tif opt.IsClusterPoolSizeOrAccountsChange() {\n\t\t\tco = opt.(*clusterOption)\n\t\t}\n\t\tif opt.IsClusterPermsChange() {\n\t\t\treloadClusterPerms = true\n\t\t}\n\t\tif opt.IsJetStreamChange() {\n\t\t\treloadJetstream = true\n\t\t\tjsEnabled = opt.(*jetStreamOption).newValue\n\t\t}\n\t\tif opt.IsStatszChange() {\n\t\t\tisStatszChange = true\n\t\t}\n\t}\n\n\tif reloadLogging {\n\t\ts.ConfigureLogger()\n\t}\n\tif reloadClientTrcLvl {\n\t\ts.reloadClientTraceLevel()\n\t}\n\tif reloadAuth {\n\t\ts.reloadAuthorization()\n\t}\n\tif reloadClusterPerms {\n\t\ts.reloadClusterPermissions(ctx.oldClusterPerms)\n\t}\n\tnewOpts := s.getOpts()\n\t// If we need to reload cluster pool/per-account, then co will be not nil\n\tif co != nil {\n\t\ts.reloadClusterPoolAndAccounts(co, newOpts)\n\t}\n\tif reloadJetstream {\n\t\tif !jsEnabled {\n\t\t\ts.DisableJetStream()\n\t\t} else if !s.JetStreamEnabled() {\n\t\t\tif err := s.restartJetStream(); err != nil {\n\t\t\t\ts.Warnf(\"Can't start JetStream: %v\", err)\n\t\t\t}\n\t\t}\n\t\t// Make sure to reset the internal loop's version of JS.\n\t\ts.resetInternalLoopInfo()\n\t}\n\tif isStatszChange {\n\t\ts.sendStatszUpdate()\n\t}\n\n\t// For remote gateways and leafnodes, make sure that their TLS configuration\n\t// is updated (since the config is \"captured\" early and changes would otherwise\n\t// not be visible).\n\tif s.gateway.enabled {\n\t\ts.gateway.updateRemotesTLSConfig(newOpts)\n\t}\n\tif len(newOpts.LeafNode.Remotes) > 0 {\n\t\ts.updateRemoteLeafNodesTLSConfig(newOpts)\n\t}\n\n\t// Always restart OCSP monitoring on reload.\n\tif err := s.reloadOCSP(); err != nil {\n\t\ts.Warnf(\"Can't restart OCSP features: %v\", err)\n\t}\n\tvar cd string\n\tif newOpts.configDigest != \"\" {\n\t\tcd = fmt.Sprintf(\"(%s)\", newOpts.configDigest)\n\t}\n\ts.Noticef(\"Reloaded server configuration %s\", cd)\n}\n\n// This will send a reset to the internal send loop.\nfunc (s *Server) resetInternalLoopInfo() {\n\tvar resetCh chan struct{}\n\ts.mu.Lock()\n\tif s.sys != nil {\n\t\t// can't hold the lock as go routine reading it may be waiting for lock as well\n\t\tresetCh = s.sys.resetCh\n\t}\n\ts.mu.Unlock()\n\n\tif resetCh != nil {\n\t\tresetCh <- struct{}{}\n\t}\n}\n\n// Update all cached debug and trace settings for every client\nfunc (s *Server) reloadClientTraceLevel() {\n\topts := s.getOpts()\n\n\tif opts.NoLog {\n\t\treturn\n\t}\n\n\t// Create a list of all clients.\n\t// Update their trace level when not holding server or gateway lock\n\n\ts.mu.Lock()\n\tclientCnt := 1 + len(s.clients) + len(s.grTmpClients) + s.numRoutes() + len(s.leafs)\n\ts.mu.Unlock()\n\n\ts.gateway.RLock()\n\tclientCnt += len(s.gateway.in) + len(s.gateway.outo)\n\ts.gateway.RUnlock()\n\n\tclients := make([]*client, 0, clientCnt)\n\n\ts.mu.Lock()\n\tif s.eventsEnabled() {\n\t\tclients = append(clients, s.sys.client)\n\t}\n\n\tcMaps := []map[uint64]*client{s.clients, s.grTmpClients, s.leafs}\n\tfor _, m := range cMaps {\n\t\tfor _, c := range m {\n\t\t\tclients = append(clients, c)\n\t\t}\n\t}\n\ts.forEachRoute(func(c *client) {\n\t\tclients = append(clients, c)\n\t})\n\ts.mu.Unlock()\n\n\ts.gateway.RLock()\n\tfor _, c := range s.gateway.in {\n\t\tclients = append(clients, c)\n\t}\n\tclients = append(clients, s.gateway.outo...)\n\ts.gateway.RUnlock()\n\n\tfor _, c := range clients {\n\t\t// client.trace is commonly read while holding the lock\n\t\tc.mu.Lock()\n\t\tc.setTraceLevel()\n\t\tc.mu.Unlock()\n\t}\n}\n\n// reloadAuthorization reconfigures the server authorization settings,\n// disconnects any clients who are no longer authorized, and removes any\n// unauthorized subscriptions.\nfunc (s *Server) reloadAuthorization() {\n\t// This map will contain the names of accounts that have their streams\n\t// import configuration changed.\n\tvar awcsti map[string]struct{}\n\tcheckJetStream := false\n\topts := s.getOpts()\n\ts.mu.Lock()\n\n\tdeletedAccounts := make(map[string]*Account)\n\n\t// This can not be changed for now so ok to check server's trustedKeys unlocked.\n\t// If plain configured accounts, process here.\n\tif s.trustedKeys == nil {\n\t\t// Make a map of the configured account names so we figure out the accounts\n\t\t// that should be removed later on.\n\t\tconfigAccs := make(map[string]struct{}, len(opts.Accounts))\n\t\tfor _, acc := range opts.Accounts {\n\t\t\tconfigAccs[acc.GetName()] = struct{}{}\n\t\t}\n\t\t// Now range over existing accounts and keep track of the ones deleted\n\t\t// so some cleanup can be made after releasing the server lock.\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tan, acc := k.(string), v.(*Account)\n\t\t\t// Exclude default and system account from this test since those\n\t\t\t// may not actually be in opts.Accounts.\n\t\t\tif an == DEFAULT_GLOBAL_ACCOUNT || an == DEFAULT_SYSTEM_ACCOUNT {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\t// Check check if existing account is still in opts.Accounts.\n\t\t\tif _, ok := configAccs[an]; !ok {\n\t\t\t\tdeletedAccounts[an] = acc\n\t\t\t\ts.accounts.Delete(k)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\t// This will update existing and add new ones.\n\t\tawcsti, _ = s.configureAccounts(true)\n\t\ts.configureAuthorization()\n\t\t// Double check any JetStream configs.\n\t\tcheckJetStream = s.getJetStream() != nil\n\t} else if opts.AccountResolver != nil {\n\t\ts.configureResolver()\n\t\tif _, ok := s.accResolver.(*MemAccResolver); ok {\n\t\t\t// Check preloads so we can issue warnings etc if needed.\n\t\t\ts.checkResolvePreloads()\n\t\t\t// With a memory resolver we want to do something similar to configured accounts.\n\t\t\t// We will walk the accounts and delete them if they are no longer present via fetch.\n\t\t\t// If they are present we will force a claim update to process changes.\n\t\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\t\tacc := v.(*Account)\n\t\t\t\t// Skip global account.\n\t\t\t\tif acc == s.gacc {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\taccName := acc.GetName()\n\t\t\t\t// Release server lock for following actions\n\t\t\t\ts.mu.Unlock()\n\t\t\t\taccClaims, claimJWT, _ := s.fetchAccountClaims(accName)\n\t\t\t\tif accClaims != nil {\n\t\t\t\t\tif err := s.updateAccountWithClaimJWT(acc, claimJWT); err != nil {\n\t\t\t\t\t\ts.Noticef(\"Reloaded: deleting account [bad claims]: %q\", accName)\n\t\t\t\t\t\ts.accounts.Delete(k)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\ts.Noticef(\"Reloaded: deleting account [removed]: %q\", accName)\n\t\t\t\t\ts.accounts.Delete(k)\n\t\t\t\t}\n\t\t\t\t// Regrab server lock.\n\t\t\t\ts.mu.Lock()\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t}\n\n\tvar (\n\t\tcclientsa [64]*client\n\t\tcclients = cclientsa[:0]\n\t\tclientsa [64]*client\n\t\tclients = clientsa[:0]\n\t\troutesa [64]*client\n\t\troutes = routesa[:0]\n\t)\n\n\t// Gather clients that changed accounts. We will close them and they\n\t// will reconnect, doing the right thing.\n\tfor _, client := range s.clients {\n\t\tif s.clientHasMovedToDifferentAccount(client) {\n\t\t\tcclients = append(cclients, client)\n\t\t} else {\n\t\t\tclients = append(clients, client)\n\t\t}\n\t}\n\ts.forEachRoute(func(route *client) {\n\t\troutes = append(routes, route)\n\t})\n\t// Check here for any system/internal clients which will not be in the servers map of normal clients.\n\tif s.sys != nil && s.sys.account != nil && !opts.NoSystemAccount {\n\t\ts.accounts.Store(s.sys.account.Name, s.sys.account)\n\t}\n\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\tacc.mu.RLock()\n\t\t// Check for sysclients accounting, ignore the system account.\n\t\tif acc.sysclients > 0 && (s.sys == nil || s.sys.account != acc) {\n\t\t\tfor c := range acc.clients {\n\t\t\t\tif c.kind != CLIENT && c.kind != LEAF {\n\t\t\t\t\tclients = append(clients, c)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tacc.mu.RUnlock()\n\t\treturn true\n\t})\n\n\tvar resetCh chan struct{}\n\tif s.sys != nil {\n\t\t// can't hold the lock as go routine reading it may be waiting for lock as well\n\t\tresetCh = s.sys.resetCh\n\t}\n\ts.mu.Unlock()\n\n\t// Clear some timers and remove service import subs for deleted accounts.\n\tfor _, acc := range deletedAccounts {\n\t\tacc.mu.Lock()\n\t\tclearTimer(&acc.etmr)\n\t\tclearTimer(&acc.ctmr)\n\t\tfor _, se := range acc.exports.services {\n\t\t\tse.clearResponseThresholdTimer()\n\t\t}\n\t\tacc.mu.Unlock()\n\t\tacc.removeAllServiceImportSubs()\n\t}\n\n\tif resetCh != nil {\n\t\tresetCh <- struct{}{}\n\t}\n\n\t// Close clients that have moved accounts\n\tfor _, client := range cclients {\n\t\tclient.closeConnection(ClientClosed)\n\t}\n\n\tfor _, c := range clients {\n\t\t// Disconnect any unauthorized clients.\n\t\t// Ignore internal clients.\n\t\tif (c.kind == CLIENT || c.kind == LEAF) && !s.isClientAuthorized(c) {\n\t\t\tc.authViolation()\n\t\t\tcontinue\n\t\t}\n\t\t// Check to make sure account is correct.\n\t\tc.swapAccountAfterReload()\n\t\t// Remove any unauthorized subscriptions and check for account imports.\n\t\tc.processSubsOnConfigReload(awcsti)\n\t}\n\n\tfor _, route := range routes {\n\t\t// Disconnect any unauthorized routes.\n\t\t// Do this only for routes that were accepted, not initiated\n\t\t// because in the later case, we don't have the user name/password\n\t\t// of the remote server.\n\t\tif !route.isSolicitedRoute() && !s.isRouterAuthorized(route) {\n\t\t\troute.setNoReconnect()\n\t\t\troute.authViolation()\n\t\t}\n\t}\n\n\tif res := s.AccountResolver(); res != nil {\n\t\tres.Reload()\n\t}\n\n\t// We will double check all JetStream configs on a reload.\n\tif checkJetStream {\n\t\tif err := s.enableJetStreamAccounts(); err != nil {\n\t\t\ts.Errorf(err.Error())\n\t\t}\n\t}\n\n\t// Check that publish retained messages sources are still allowed to publish.\n\t// Do this after dealing with JetStream.\n\ts.mqttCheckPubRetainedPerms()\n}\n\n// Returns true if given client current account has changed (or user\n// no longer exist) in the new config, false if the user did not\n// change accounts.\n// Server lock is held on entry.\nfunc (s *Server) clientHasMovedToDifferentAccount(c *client) bool {\n\tvar (\n\t\tnu *NkeyUser\n\t\tu *User\n\t)\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif c.opts.Nkey != _EMPTY_ {\n\t\tif s.nkeys != nil {\n\t\t\tnu = s.nkeys[c.opts.Nkey]\n\t\t}\n\t} else if c.opts.Username != _EMPTY_ {\n\t\tif s.users != nil {\n\t\t\tu = s.users[c.opts.Username]\n\t\t}\n\t} else {\n\t\treturn false\n\t}\n\t// Get the current account name\n\tvar curAccName string\n\tif c.acc != nil {\n\t\tcurAccName = c.acc.Name\n\t}\n\tif nu != nil && nu.Account != nil {\n\t\treturn curAccName != nu.Account.Name\n\t} else if u != nil && u.Account != nil {\n\t\treturn curAccName != u.Account.Name\n\t}\n\t// user/nkey no longer exists.\n\treturn true\n}\n\n// reloadClusterPermissions reconfigures the cluster's permssions\n// and set the permissions to all existing routes, sending an\n// update INFO protocol so that remote can resend their local\n// subs if needed, and sending local subs matching cluster's\n// import subjects.\nfunc (s *Server) reloadClusterPermissions(oldPerms *RoutePermissions) {\n\ts.mu.Lock()\n\tnewPerms := s.getOpts().Cluster.Permissions\n\troutes := make(map[uint64]*client, s.numRoutes())\n\t// Get all connected routes\n\ts.forEachRoute(func(route *client) {\n\t\troute.mu.Lock()\n\t\troutes[route.cid] = route\n\t\troute.mu.Unlock()\n\t})\n\t// If new permissions is nil, then clear routeInfo import/export\n\tif newPerms == nil {\n\t\ts.routeInfo.Import = nil\n\t\ts.routeInfo.Export = nil\n\t} else {\n\t\ts.routeInfo.Import = newPerms.Import\n\t\ts.routeInfo.Export = newPerms.Export\n\t}\n\tinfoJSON := generateInfoJSON(&s.routeInfo)\n\ts.mu.Unlock()\n\n\t// Close connections for routes that don't understand async INFO.\n\tfor _, route := range routes {\n\t\troute.mu.Lock()\n\t\tclose := route.opts.Protocol < RouteProtoInfo\n\t\tcid := route.cid\n\t\troute.mu.Unlock()\n\t\tif close {\n\t\t\troute.closeConnection(RouteRemoved)\n\t\t\tdelete(routes, cid)\n\t\t}\n\t}\n\n\t// If there are no route left, we are done\n\tif len(routes) == 0 {\n\t\treturn\n\t}\n\n\t// Fake clients to test cluster permissions\n\toldPermsTester := &client{}\n\toldPermsTester.setRoutePermissions(oldPerms)\n\tnewPermsTester := &client{}\n\tnewPermsTester.setRoutePermissions(newPerms)\n\n\tvar (\n\t\t_localSubs [4096]*subscription\n\t\tsubsNeedSUB = map[*client][]*subscription{}\n\t\tsubsNeedUNSUB = map[*client][]*subscription{}\n\t\tdeleteRoutedSubs []*subscription\n\t)\n\n\tgetRouteForAccount := func(accName string, poolIdx int) *client {\n\t\tfor _, r := range routes {\n\t\t\tr.mu.Lock()\n\t\t\tok := (poolIdx >= 0 && poolIdx == r.route.poolIdx) || (string(r.route.accName) == accName) || r.route.noPool\n\t\t\tr.mu.Unlock()\n\t\t\tif ok {\n\t\t\t\treturn r\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\t// First set the new permissions on all routes.\n\tfor _, route := range routes {\n\t\troute.mu.Lock()\n\t\troute.setRoutePermissions(newPerms)\n\t\troute.mu.Unlock()\n\t}\n\n\t// Then, go over all accounts and gather local subscriptions that need to be\n\t// sent over as SUB or removed as UNSUB, and routed subscriptions that need\n\t// to be dropped due to export permissions.\n\ts.accounts.Range(func(_, v any) bool {\n\t\tacc := v.(*Account)\n\t\tacc.mu.RLock()\n\t\taccName, sl, poolIdx := acc.Name, acc.sl, acc.routePoolIdx\n\t\tacc.mu.RUnlock()\n\t\t// Get the route handling this account. If no route or sublist, bail out.\n\t\troute := getRouteForAccount(accName, poolIdx)\n\t\tif route == nil || sl == nil {\n\t\t\treturn true\n\t\t}\n\t\tlocalSubs := _localSubs[:0]\n\t\tsl.localSubs(&localSubs, false)\n\n\t\t// Go through all local subscriptions\n\t\tfor _, sub := range localSubs {\n\t\t\t// Get all subs that can now be imported\n\t\t\tsubj := string(sub.subject)\n\t\t\tcouldImportThen := oldPermsTester.canImport(subj)\n\t\t\tcanImportNow := newPermsTester.canImport(subj)\n\t\t\tif canImportNow {\n\t\t\t\t// If we could not before, then will need to send a SUB protocol.\n\t\t\t\tif !couldImportThen {\n\t\t\t\t\tsubsNeedSUB[route] = append(subsNeedSUB[route], sub)\n\t\t\t\t}\n\t\t\t} else if couldImportThen {\n\t\t\t\t// We were previously able to import this sub, but now\n\t\t\t\t// we can't so we need to send an UNSUB protocol\n\t\t\t\tsubsNeedUNSUB[route] = append(subsNeedUNSUB[route], sub)\n\t\t\t}\n\t\t}\n\t\tdeleteRoutedSubs = deleteRoutedSubs[:0]\n\t\troute.mu.Lock()\n\t\tpa, _, hasSubType := route.getRoutedSubKeyInfo()\n\t\tfor key, sub := range route.subs {\n\t\t\t// If this is not a pinned-account route, we need to get the\n\t\t\t// account name from the key to see if we collect this sub.\n\t\t\tif !pa {\n\t\t\t\tif an := getAccNameFromRoutedSubKey(sub, key, hasSubType); an != accName {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\t// If we can't export, we need to drop the subscriptions that\n\t\t\t// we have on behalf of this route.\n\t\t\t// Need to make a string cast here since canExport call sl.Match()\n\t\t\tsubj := string(sub.subject)\n\t\t\tif !route.canExport(subj) {\n\t\t\t\t// We can use bytesToString() here.\n\t\t\t\tdelete(route.subs, bytesToString(sub.sid))\n\t\t\t\tdeleteRoutedSubs = append(deleteRoutedSubs, sub)\n\t\t\t}\n\t\t}\n\t\troute.mu.Unlock()\n\t\t// Remove as a batch all the subs that we have removed from each route.\n\t\tsl.RemoveBatch(deleteRoutedSubs)\n\t\treturn true\n\t})\n\n\t// Send an update INFO, which will allow remote server to show\n\t// our current route config in monitoring and resend subscriptions\n\t// that we now possibly allow with a change of Export permissions.\n\tfor _, route := range routes {\n\t\troute.mu.Lock()\n\t\troute.enqueueProto(infoJSON)\n\t\t// Now send SUB and UNSUB protocols as needed.\n\t\tif subs, ok := subsNeedSUB[route]; ok && len(subs) > 0 {\n\t\t\troute.sendRouteSubProtos(subs, false, nil)\n\t\t}\n\t\tif unsubs, ok := subsNeedUNSUB[route]; ok && len(unsubs) > 0 {\n\t\t\troute.sendRouteUnSubProtos(unsubs, false, nil)\n\t\t}\n\t\troute.mu.Unlock()\n\t}\n}\n\nfunc (s *Server) reloadClusterPoolAndAccounts(co *clusterOption, opts *Options) {\n\ts.mu.Lock()\n\t// Prevent adding new routes until we are ready to do so.\n\ts.routesReject = true\n\tvar ch chan struct{}\n\t// For accounts that have been added to the list of dedicated routes,\n\t// send a protocol to their current assigned routes to allow the\n\t// other side to prepare for the changes.\n\tif len(co.accsAdded) > 0 {\n\t\tprotosSent := 0\n\t\ts.accAddedReqID = nuid.Next()\n\t\tfor _, an := range co.accsAdded {\n\t\t\tif s.accRoutes == nil {\n\t\t\t\ts.accRoutes = make(map[string]map[string]*client)\n\t\t\t}\n\t\t\t// In case a config reload was first done on another server,\n\t\t\t// we may have already switched this account to a dedicated route.\n\t\t\t// But we still want to send the protocol over the routes that\n\t\t\t// would have otherwise handled it.\n\t\t\tif _, ok := s.accRoutes[an]; !ok {\n\t\t\t\ts.accRoutes[an] = make(map[string]*client)\n\t\t\t}\n\t\t\tif a, ok := s.accounts.Load(an); ok {\n\t\t\t\tacc := a.(*Account)\n\t\t\t\tacc.mu.Lock()\n\t\t\t\tsl := acc.sl\n\t\t\t\t// Get the current route pool index before calling setRouteInfo.\n\t\t\t\trpi := acc.routePoolIdx\n\t\t\t\t// Switch to per-account route if not already done.\n\t\t\t\tif rpi >= 0 {\n\t\t\t\t\ts.setRouteInfo(acc)\n\t\t\t\t} else {\n\t\t\t\t\t// If it was transitioning, make sure we set it to the state\n\t\t\t\t\t// that indicates that it has a dedicated route\n\t\t\t\t\tif rpi == accTransitioningToDedicatedRoute {\n\t\t\t\t\t\tacc.routePoolIdx = accDedicatedRoute\n\t\t\t\t\t}\n\t\t\t\t\t// Otherwise get the route pool index it would have been before\n\t\t\t\t\t// the move so we can send the protocol to those routes.\n\t\t\t\t\trpi = computeRoutePoolIdx(s.routesPoolSize, acc.Name)\n\t\t\t\t}\n\t\t\t\tacc.mu.Unlock()\n\t\t\t\t// Generate the INFO protocol to send indicating that this account\n\t\t\t\t// is being moved to a dedicated route.\n\t\t\t\tri := Info{\n\t\t\t\t\tRoutePoolSize: s.routesPoolSize,\n\t\t\t\t\tRouteAccount: an,\n\t\t\t\t\tRouteAccReqID: s.accAddedReqID,\n\t\t\t\t}\n\t\t\t\tproto := generateInfoJSON(&ri)\n\t\t\t\t// Since v2.11.0, we support remotes with a different pool size\n\t\t\t\t// (for rolling upgrades), so we need to use the remote route\n\t\t\t\t// pool index (based on the remote configured pool size) since\n\t\t\t\t// the remote subscriptions will be attached to the route at\n\t\t\t\t// that index, not at our account's route pool index. However,\n\t\t\t\t// we are going to send the protocol through the route that\n\t\t\t\t// handles this account from our pool size perspective (that\n\t\t\t\t// would be the route at index `rpi`).\n\t\t\t\tremoveSubsAndSendProto := func(r *client, doSubs, doProto bool) {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tdefer r.mu.Unlock()\n\t\t\t\t\t// Exclude routes to servers that don't support pooling.\n\t\t\t\t\tif r.route.noPool {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif doSubs {\n\t\t\t\t\t\tif subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 {\n\t\t\t\t\t\t\tsl.RemoveBatch(subs)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif doProto {\n\t\t\t\t\t\tr.enqueueProto(proto)\n\t\t\t\t\t\tprotosSent++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor remote, conns := range s.routes {\n\t\t\t\t\tr := conns[rpi]\n\t\t\t\t\t// The route connection at this index is currently not up,\n\t\t\t\t\t// so we won't be able to send the protocol, so move to the\n\t\t\t\t\t// next remote.\n\t\t\t\t\tif r == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tdoSubs := true\n\t\t\t\t\t// Check the remote's route pool size and if different than\n\t\t\t\t\t// ours, remove the subs on that other route.\n\t\t\t\t\tremotePoolSize, ok := s.remoteRoutePoolSize[remote]\n\t\t\t\t\tif ok && remotePoolSize != s.routesPoolSize {\n\t\t\t\t\t\t// This is the remote's route pool index for this account\n\t\t\t\t\t\trrpi := computeRoutePoolIdx(remotePoolSize, an)\n\t\t\t\t\t\tif rr := conns[rrpi]; rr != nil {\n\t\t\t\t\t\t\tremoveSubsAndSendProto(rr, true, false)\n\t\t\t\t\t\t\t// Indicate that we have already remove the subs.\n\t\t\t\t\t\t\tdoSubs = false\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Now send the protocol from the route that handles the\n\t\t\t\t\t// account from this server perspective.\n\t\t\t\t\tremoveSubsAndSendProto(r, doSubs, true)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif protosSent > 0 {\n\t\t\ts.accAddedCh = make(chan struct{}, protosSent)\n\t\t\tch = s.accAddedCh\n\t\t}\n\t}\n\t// Collect routes that need to be closed.\n\troutes := make(map[*client]struct{})\n\t// Collect the per-account routes that need to be closed.\n\tif len(co.accsRemoved) > 0 {\n\t\tfor _, an := range co.accsRemoved {\n\t\t\tif remotes, ok := s.accRoutes[an]; ok && remotes != nil {\n\t\t\t\tfor _, r := range remotes {\n\t\t\t\t\tif r != nil {\n\t\t\t\t\t\tr.setNoReconnect()\n\t\t\t\t\t\troutes[r] = struct{}{}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// If the pool size has changed, we need to close all pooled routes.\n\tif co.poolSizeChanged {\n\t\ts.forEachNonPerAccountRoute(func(r *client) {\n\t\t\troutes[r] = struct{}{}\n\t\t})\n\t}\n\t// If there are routes to close, we need to release the server lock.\n\t// Same if we need to wait on responses from the remotes when\n\t// processing new per-account routes.\n\tif len(routes) > 0 || len(ch) > 0 {\n\t\ts.mu.Unlock()\n\n\t\tfor done := false; !done && len(ch) > 0; {\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\tcase <-time.After(2 * time.Second):\n\t\t\t\ts.Warnf(\"Timed out waiting for confirmation from all routes regarding per-account routes changes\")\n\t\t\t\tdone = true\n\t\t\t}\n\t\t}\n\n\t\tfor r := range routes {\n\t\t\tr.closeConnection(RouteRemoved)\n\t\t}\n\n\t\ts.mu.Lock()\n\t}\n\t// Clear the accAddedCh/ReqID fields in case they were set.\n\ts.accAddedReqID, s.accAddedCh = _EMPTY_, nil\n\t// Now that per-account routes that needed to be closed are closed,\n\t// remove them from s.accRoutes. Doing so before would prevent\n\t// removeRoute() to do proper cleanup because the route would not\n\t// be found in s.accRoutes.\n\tfor _, an := range co.accsRemoved {\n\t\tdelete(s.accRoutes, an)\n\t\t// Do not lookup and call setRouteInfo() on the accounts here.\n\t\t// We need first to set the new s.routesPoolSize value and\n\t\t// anyway, there is no need to do here if the pool size has\n\t\t// changed (since it will be called for all accounts).\n\t}\n\t// We have already added the accounts to s.accRoutes that needed to\n\t// be added.\n\n\t// We should always have at least the system account with a dedicated route,\n\t// but in case we have a configuration that disables pooling and without\n\t// a system account, possibly set the accRoutes to nil.\n\tif len(opts.Cluster.PinnedAccounts) == 0 {\n\t\ts.accRoutes = nil\n\t}\n\t// Now deal with pool size updates.\n\tif ps := opts.Cluster.PoolSize; ps > 0 {\n\t\ts.routesPoolSize = ps\n\t\ts.routeInfo.RoutePoolSize = ps\n\t} else {\n\t\ts.routesPoolSize = 1\n\t\ts.routeInfo.RoutePoolSize = 0\n\t}\n\t// If the pool size has changed, we need to recompute all accounts' route\n\t// pool index. Note that the added/removed accounts will be reset there\n\t// too, but that's ok (we could use a map to exclude them, but not worth it).\n\tif co.poolSizeChanged {\n\t\ts.accounts.Range(func(_, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\tacc.mu.Lock()\n\t\t\ts.setRouteInfo(acc)\n\t\t\tacc.mu.Unlock()\n\t\t\treturn true\n\t\t})\n\t} else if len(co.accsRemoved) > 0 {\n\t\t// For accounts that no longer have a dedicated route, we need to send\n\t\t// the subsriptions on the existing pooled routes for those accounts.\n\t\tfor _, an := range co.accsRemoved {\n\t\t\tif a, ok := s.accounts.Load(an); ok {\n\t\t\t\tacc := a.(*Account)\n\t\t\t\tacc.mu.Lock()\n\t\t\t\t// First call this which will assign a new route pool index.\n\t\t\t\ts.setRouteInfo(acc)\n\t\t\t\t// Get the value so we can send the subscriptions interest\n\t\t\t\t// on all routes with this pool index.\n\t\t\t\trpi := acc.routePoolIdx\n\t\t\t\tacc.mu.Unlock()\n\t\t\t\ts.forEachRouteIdx(rpi, func(r *client) bool {\n\t\t\t\t\t// We have the guarantee that if the route exists, it\n\t\t\t\t\t// is not a new one that would have been created when\n\t\t\t\t\t// we released the server lock if some routes needed\n\t\t\t\t\t// to be closed, because we have set s.routesReject\n\t\t\t\t\t// to `true` at the top of this function.\n\t\t\t\t\ts.sendSubsToRoute(r, rpi, an)\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\t// Allow routes to be accepted now.\n\ts.routesReject = false\n\t// If there is a pool size change or added accounts, solicit routes now.\n\tif co.poolSizeChanged || len(co.accsAdded) > 0 {\n\t\ts.solicitRoutes(opts.Routes, co.accsAdded)\n\t}\n\ts.mu.Unlock()\n}\n\n// validateClusterOpts ensures the new ClusterOpts does not change some of the\n// fields that do not support reload.\nfunc validateClusterOpts(old, new ClusterOpts) error {\n\tif old.Host != new.Host {\n\t\treturn fmt.Errorf(\"config reload not supported for cluster host: old=%s, new=%s\",\n\t\t\told.Host, new.Host)\n\t}\n\tif old.Port != new.Port {\n\t\treturn fmt.Errorf(\"config reload not supported for cluster port: old=%d, new=%d\",\n\t\t\told.Port, new.Port)\n\t}\n\t// Validate Cluster.Advertise syntax\n\tif new.Advertise != \"\" {\n\t\tif _, _, err := parseHostPort(new.Advertise, 0); err != nil {\n\t\t\treturn fmt.Errorf(\"invalid Cluster.Advertise value of %s, err=%v\", new.Advertise, err)\n\t\t}\n\t}\n\treturn nil\n}\n\n// diffRoutes diffs the old routes and the new routes and returns the ones that\n// should be added and removed from the server.\nfunc diffRoutes(old, new []*url.URL) (add, remove []*url.URL) {\n\t// Find routes to remove.\nremoveLoop:\n\tfor _, oldRoute := range old {\n\t\tfor _, newRoute := range new {\n\t\t\tif urlsAreEqual(oldRoute, newRoute) {\n\t\t\t\tcontinue removeLoop\n\t\t\t}\n\t\t}\n\t\tremove = append(remove, oldRoute)\n\t}\n\n\t// Find routes to add.\naddLoop:\n\tfor _, newRoute := range new {\n\t\tfor _, oldRoute := range old {\n\t\t\tif urlsAreEqual(oldRoute, newRoute) {\n\t\t\t\tcontinue addLoop\n\t\t\t}\n\t\t}\n\t\tadd = append(add, newRoute)\n\t}\n\n\treturn add, remove\n}\n\nfunc diffProxiesTrustedKeys(old, new []*ProxyConfig) ([]string, []string) {\n\tvar add []string\n\tvar del []string\n\t// Both \"old\" and \"new\" lists should be small...\n\tfor _, op := range old {\n\t\tif !slices.ContainsFunc(new, func(pc *ProxyConfig) bool {\n\t\t\treturn pc.Key == op.Key\n\t\t}) {\n\t\t\tdel = append(del, op.Key)\n\t\t}\n\t}\n\tfor _, np := range new {\n\t\tif !slices.ContainsFunc(old, func(pc *ProxyConfig) bool {\n\t\t\treturn pc.Key == np.Key\n\t\t}) {\n\t\t\tadd = append(add, np.Key)\n\t\t}\n\t}\n\treturn add, del\n}\n" }, { "path": "server/reload_test.go", "content": "// Copyright 2017-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc newServerWithConfig(t *testing.T, configFile string) (*Server, *Options, string) {\n\tt.Helper()\n\tcontent, err := os.ReadFile(configFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error loading file: %v\", err)\n\t}\n\treturn newServerWithContent(t, content)\n}\n\nfunc newServerWithContent(t *testing.T, content []byte) (*Server, *Options, string) {\n\tt.Helper()\n\topts, tmpFile := newOptionsFromContent(t, content)\n\treturn New(opts), opts, tmpFile\n}\n\nfunc newOptionsFromContent(t *testing.T, content []byte) (*Options, string) {\n\tt.Helper()\n\ttmpFile := createConfFile(t, content)\n\topts, err := ProcessConfigFile(tmpFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoSigs = true\n\treturn opts, tmpFile\n}\n\nfunc createConfFile(t testing.TB, content []byte) string {\n\tt.Helper()\n\tconf := createTempFile(t, _EMPTY_)\n\tfName := conf.Name()\n\tconf.Close()\n\tif err := os.WriteFile(fName, content, 0666); err != nil {\n\t\tt.Fatalf(\"Error writing conf file: %v\", err)\n\t}\n\treturn fName\n}\n\nfunc runReloadServerWithConfig(t *testing.T, configFile string) (*Server, *Options, string) {\n\tt.Helper()\n\tcontent, err := os.ReadFile(configFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error loading file: %v\", err)\n\t}\n\treturn runReloadServerWithContent(t, content)\n}\n\nfunc runReloadServerWithContent(t *testing.T, content []byte) (*Server, *Options, string) {\n\tt.Helper()\n\topts, tmpFile := newOptionsFromContent(t, content)\n\topts.NoLog = true\n\topts.NoSigs = true\n\ts := RunServer(opts)\n\treturn s, opts, tmpFile\n}\n\nfunc changeCurrentConfigContent(t *testing.T, curConfig, newConfig string) {\n\tt.Helper()\n\tcontent, err := os.ReadFile(newConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Error loading file: %v\", err)\n\t}\n\tchangeCurrentConfigContentWithNewContent(t, curConfig, content)\n}\n\nfunc changeCurrentConfigContentWithNewContent(t *testing.T, curConfig string, content []byte) {\n\tt.Helper()\n\tif err := os.WriteFile(curConfig, content, 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n}\n\n// Ensure Reload returns an error when attempting to reload a server that did\n// not start with a config file.\nfunc TestConfigReloadNoConfigFile(t *testing.T) {\n\tserver := New(&Options{NoSigs: true})\n\tloaded := server.ConfigTime()\n\tif server.Reload() == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n\tif reloaded := server.ConfigTime(); reloaded != loaded {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected: %s\\ngot: %s\", loaded, reloaded)\n\t}\n}\n\n// Ensure Reload returns an error when attempting to change an option which\n// does not support reloading.\nfunc TestConfigReloadUnsupported(t *testing.T) {\n\tserver, _, config := newServerWithConfig(t, \"./configs/reload/test.conf\")\n\tdefer server.Shutdown()\n\n\tloaded := server.ConfigTime()\n\n\tgolden := &Options{\n\t\tConfigFile: config,\n\t\tHost: \"0.0.0.0\",\n\t\tPort: 2233,\n\t\tAuthTimeout: float64(AUTH_TIMEOUT / time.Second),\n\t\tDebug: false,\n\t\tTrace: false,\n\t\tLogtime: false,\n\t\tMaxControlLine: 4096,\n\t\tMaxPayload: 1048576,\n\t\tMaxConn: 65536,\n\t\tPingInterval: 2 * time.Minute,\n\t\tMaxPingsOut: 2,\n\t\tWriteDeadline: 10 * time.Second,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: -1,\n\t\t},\n\t\tNoSigs: true,\n\t\tconfigDigest: \"sha256:3c5c4141f56274bcfa801f2d7326ec64d5eabddd9e6d4f9a06ed167315a57f55\",\n\t}\n\tsetBaselineOptions(golden)\n\n\tcheckOptionsEqual(t, golden, server.getOpts())\n\n\t// Change config file to bad config.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/reload_unsupported.conf\")\n\n\t// This should fail because `cluster` host cannot be changed.\n\tif err := server.Reload(); err == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n\n\t// Ensure config didn't change.\n\tcheckOptionsEqual(t, golden, server.getOpts())\n\n\tif reloaded := server.ConfigTime(); reloaded != loaded {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected: %s\\ngot: %s\", loaded, reloaded)\n\t}\n}\n\n// This checks that if we change an option that does not support hot-swapping\n// we get an error. Using `listen` for now (test may need to be updated if\n// server is changed to support change of listen spec).\nfunc TestConfigReloadUnsupportedHotSwapping(t *testing.T) {\n\tserver, _, config := newServerWithContent(t, []byte(\"listen: 127.0.0.1:-1\"))\n\tdefer server.Shutdown()\n\n\tloaded := server.ConfigTime()\n\n\ttime.Sleep(time.Millisecond)\n\n\t// Change config file with unsupported option hot-swap\n\tchangeCurrentConfigContentWithNewContent(t, config, []byte(\"listen: 127.0.0.1:9999\"))\n\n\t// This should fail because `listen` host cannot be changed.\n\tif err := server.Reload(); err == nil || !strings.Contains(err.Error(), \"not supported\") {\n\t\tt.Fatalf(\"Expected Reload to return a not supported error, got %v\", err)\n\t}\n\n\tif reloaded := server.ConfigTime(); reloaded != loaded {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected: %s\\ngot: %s\", loaded, reloaded)\n\t}\n}\n\n// Ensure Reload returns an error when reloading from a bad config file.\nfunc TestConfigReloadInvalidConfig(t *testing.T) {\n\tserver, _, config := newServerWithConfig(t, \"./configs/reload/test.conf\")\n\tdefer server.Shutdown()\n\n\tloaded := server.ConfigTime()\n\n\tgolden := &Options{\n\t\tConfigFile: config,\n\t\tHost: \"0.0.0.0\",\n\t\tPort: 2233,\n\t\tAuthTimeout: float64(AUTH_TIMEOUT / time.Second),\n\t\tDebug: false,\n\t\tTrace: false,\n\t\tLogtime: false,\n\t\tMaxControlLine: 4096,\n\t\tMaxPayload: 1048576,\n\t\tMaxConn: 65536,\n\t\tPingInterval: 2 * time.Minute,\n\t\tMaxPingsOut: 2,\n\t\tWriteDeadline: 10 * time.Second,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: -1,\n\t\t},\n\t\tNoSigs: true,\n\t\tconfigDigest: \"sha256:3c5c4141f56274bcfa801f2d7326ec64d5eabddd9e6d4f9a06ed167315a57f55\",\n\t}\n\tsetBaselineOptions(golden)\n\n\tcheckOptionsEqual(t, golden, server.getOpts())\n\n\t// Change config file to bad config.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/invalid.conf\")\n\n\t// This should fail because the new config should not parse.\n\tif err := server.Reload(); err == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n\n\t// Ensure config didn't change.\n\tcheckOptionsEqual(t, golden, server.getOpts())\n\n\tif reloaded := server.ConfigTime(); reloaded != loaded {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected: %s\\ngot: %s\", loaded, reloaded)\n\t}\n}\n\n// Ensure Reload returns nil and the config is changed on success.\nfunc TestConfigReload(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/test.conf\")\n\tdefer removeFile(t, \"nats-server.pid\")\n\tdefer removeFile(t, \"nats-server.log\")\n\tdefer server.Shutdown()\n\n\tvar content []byte\n\tif runtime.GOOS != \"windows\" {\n\t\tcontent = []byte(`\n\t\t\tremote_syslog: \"udp://127.0.0.1:514\" # change on reload\n\t\t\tsyslog: true # enable on reload\n\t\t`)\n\t}\n\tplatformConf := filepath.Join(filepath.Dir(config), \"platform.conf\")\n\tif err := os.WriteFile(platformConf, content, 0666); err != nil {\n\t\tt.Fatalf(\"Unable to write config file: %v\", err)\n\t}\n\n\tloaded := server.ConfigTime()\n\n\tgolden := &Options{\n\t\tConfigFile: config,\n\t\tHost: \"0.0.0.0\",\n\t\tPort: 2233,\n\t\tAuthTimeout: float64(AUTH_TIMEOUT / time.Second),\n\t\tDebug: false,\n\t\tTrace: false,\n\t\tNoLog: true,\n\t\tLogtime: false,\n\t\tMaxControlLine: 4096,\n\t\tMaxPayload: 1048576,\n\t\tMaxConn: 65536,\n\t\tPingInterval: 2 * time.Minute,\n\t\tMaxPingsOut: 2,\n\t\tWriteDeadline: 10 * time.Second,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: server.ClusterAddr().Port,\n\t\t},\n\t\tNoSigs: true,\n\t\tconfigDigest: \"sha256:3c5c4141f56274bcfa801f2d7326ec64d5eabddd9e6d4f9a06ed167315a57f55\",\n\t}\n\tsetBaselineOptions(golden)\n\n\tcheckOptionsEqual(t, golden, opts)\n\n\t// Change config file to new config.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/reload.conf\")\n\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure config changed.\n\tupdated := server.getOpts()\n\tif !updated.Trace {\n\t\tt.Fatal(\"Expected Trace to be true\")\n\t}\n\tif !updated.Debug {\n\t\tt.Fatal(\"Expected Debug to be true\")\n\t}\n\tif !updated.Logtime {\n\t\tt.Fatal(\"Expected Logtime to be true\")\n\t}\n\tif !updated.LogtimeUTC {\n\t\tt.Fatal(\"Expected LogtimeUTC to be true\")\n\t}\n\tif runtime.GOOS != \"windows\" {\n\t\tif !updated.Syslog {\n\t\t\tt.Fatal(\"Expected Syslog to be true\")\n\t\t}\n\t\tif updated.RemoteSyslog != \"udp://127.0.0.1:514\" {\n\t\t\tt.Fatalf(\"RemoteSyslog is incorrect.\\nexpected: udp://127.0.0.1:514\\ngot: %s\", updated.RemoteSyslog)\n\t\t}\n\t}\n\tif updated.LogFile != \"nats-server.log\" {\n\t\tt.Fatalf(\"LogFile is incorrect.\\nexpected: nats-server.log\\ngot: %s\", updated.LogFile)\n\t}\n\tif updated.TLSConfig == nil {\n\t\tt.Fatal(\"Expected TLSConfig to be non-nil\")\n\t}\n\tif !server.info.TLSRequired {\n\t\tt.Fatal(\"Expected TLSRequired to be true\")\n\t}\n\tif !server.info.TLSVerify {\n\t\tt.Fatal(\"Expected TLSVerify to be true\")\n\t}\n\tif updated.Username != \"tyler\" {\n\t\tt.Fatalf(\"Username is incorrect.\\nexpected: tyler\\ngot: %s\", updated.Username)\n\t}\n\tif updated.Password != \"T0pS3cr3t\" {\n\t\tt.Fatalf(\"Password is incorrect.\\nexpected: T0pS3cr3t\\ngot: %s\", updated.Password)\n\t}\n\tif updated.AuthTimeout != 2 {\n\t\tt.Fatalf(\"AuthTimeout is incorrect.\\nexpected: 2\\ngot: %f\", updated.AuthTimeout)\n\t}\n\tif !server.info.AuthRequired {\n\t\tt.Fatal(\"Expected AuthRequired to be true\")\n\t}\n\tif !updated.Cluster.NoAdvertise {\n\t\tt.Fatal(\"Expected NoAdvertise to be true\")\n\t}\n\tif updated.Cluster.PingInterval != 20*time.Second {\n\t\tt.Fatalf(\"Cluster PingInterval is incorrect.\\nexpected: 20s\\ngot: %v\", updated.Cluster.PingInterval)\n\t}\n\tif updated.Cluster.MaxPingsOut != 8 {\n\t\tt.Fatalf(\"Cluster MaxPingsOut is incorrect.\\nexpected: 6\\ngot: %v\", updated.Cluster.MaxPingsOut)\n\t}\n\tif updated.PidFile != \"nats-server.pid\" {\n\t\tt.Fatalf(\"PidFile is incorrect.\\nexpected: nats-server.pid\\ngot: %s\", updated.PidFile)\n\t}\n\tif updated.MaxControlLine != 512 {\n\t\tt.Fatalf(\"MaxControlLine is incorrect.\\nexpected: 512\\ngot: %d\", updated.MaxControlLine)\n\t}\n\tif updated.PingInterval != 5*time.Second {\n\t\tt.Fatalf(\"PingInterval is incorrect.\\nexpected 5s\\ngot: %s\", updated.PingInterval)\n\t}\n\tif updated.MaxPingsOut != 1 {\n\t\tt.Fatalf(\"MaxPingsOut is incorrect.\\nexpected 1\\ngot: %d\", updated.MaxPingsOut)\n\t}\n\tif updated.WriteDeadline != 3*time.Second {\n\t\tt.Fatalf(\"WriteDeadline is incorrect.\\nexpected 3s\\ngot: %s\", updated.WriteDeadline)\n\t}\n\tif updated.MaxPayload != 1024 {\n\t\tt.Fatalf(\"MaxPayload is incorrect.\\nexpected 1024\\ngot: %d\", updated.MaxPayload)\n\t}\n\texpectedMetadata := map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"}\n\tif !reflect.DeepEqual(expectedMetadata, updated.Metadata) {\n\t\tt.Fatalf(\"Metadata is incorrect.\\nexpected: %v\\ngot: %v\", expectedMetadata, updated.Metadata)\n\t}\n\n\tif reloaded := server.ConfigTime(); !reloaded.After(loaded) {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected greater than: %s\\ngot: %s\", loaded, reloaded)\n\t}\n}\n\n// Ensure Reload supports TLS config changes. Test this by starting a server\n// with TLS enabled, connect to it to verify, reload config using a different\n// key pair and client verification enabled, ensure reconnect fails, then\n// ensure reconnect succeeds when the client provides a cert.\nfunc TestConfigReloadRotateTLS(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/tls_test.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\n\tnc, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\t// Rotate cert and enable client verification.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/tls_verify_test.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when client presents cert.\n\tcert := nats.ClientCert(\"./configs/certs/cert.new.pem\", \"./configs/certs/key.new.pem\")\n\tconn, err := nats.Connect(addr, cert, nats.RootCAs(\"./configs/certs/cert.new.pem\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the original connection can still publish/receive.\n\tif err := nc.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tnc.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n}\n\n// Ensure Reload supports enabling TLS. Test this by starting a server without\n// TLS enabled, connect to it to verify, reload config with TLS enabled, ensure\n// reconnect fails, then ensure reconnect succeeds when using secure.\nfunc TestConfigReloadEnableTLS(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tnc.Close()\n\n\t// Enable TLS.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/tls_test.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting is OK (we need to skip server cert verification since\n\t// the library is not doing that by default now).\n\tnc, err = nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tnc.Close()\n}\n\n// Ensure Reload supports disabling TLS. Test this by starting a server with\n// TLS enabled, connect to it to verify, reload config with TLS disabled,\n// ensure reconnect fails, then ensure reconnect succeeds when connecting\n// without secure.\nfunc TestConfigReloadDisableTLS(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/tls_test.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tnc.Close()\n\n\t// Disable TLS.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/basic.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when not using secure.\n\tnc, err = nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tnc.Close()\n}\n\nfunc TestConfigReloadRotateTLSMultiCert(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/tls_multi_cert_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\n\trawCerts := make(chan []byte, 3)\n\tnc, err := nats.Connect(addr, nats.Secure(&tls.Config{\n\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\trawCerts <- s.PeerCertificates[0].Raw\n\t\t\treturn nil\n\t\t},\n\t\tInsecureSkipVerify: true,\n\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\t// Rotate cert and enable client verification.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/tls_multi_cert_2.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.Secure(&tls.Config{InsecureSkipVerify: true})); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when client presents cert.\n\tcert := nats.ClientCert(\"../test/configs/certs/client-cert.pem\", \"../test/configs/certs/client-key.pem\")\n\tconn, err := nats.Connect(addr, cert, nats.RootCAs(\"../test/configs/certs/ca.pem\"), nats.Secure(&tls.Config{\n\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\trawCerts <- s.PeerCertificates[0].Raw\n\t\t\treturn nil\n\t\t},\n\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the original connection can still publish/receive.\n\tif err := nc.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tnc.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\t// Rotate cert and disable client verification.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/tls_multi_cert_3.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\tnc, err = nats.Connect(addr, nats.Secure(&tls.Config{\n\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\trawCerts <- s.PeerCertificates[0].Raw\n\t\t\treturn nil\n\t\t},\n\t\tInsecureSkipVerify: true,\n\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tsub, err = nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\tcertA := <-rawCerts\n\tcertB := <-rawCerts\n\tcertC := <-rawCerts\n\tif !bytes.Equal(certA, certB) {\n\t\tt.Error(\"Expected the same cert\")\n\t}\n\tif bytes.Equal(certB, certC) {\n\t\tt.Error(\"Expected a different cert\")\n\t}\n}\n\nfunc TestConfigReloadDefaultSentinel(t *testing.T) {\n\tvar err error\n\tpreload := make(map[string]string)\n\n\t_, sysPub, sysAC := NewJwtAccountClaim(\"SYS\")\n\tpreload[sysPub], err = sysAC.Encode(oKp)\n\trequire_NoError(t, err)\n\n\taKP, aPub, aAC := NewJwtAccountClaim(\"A\")\n\tpreload[aPub], err = aAC.Encode(oKp)\n\trequire_NoError(t, err)\n\n\tpreloadConfig, err := json.MarshalIndent(preload, \"\", \" \")\n\trequire_NoError(t, err)\n\n\tuKP, err := nkeys.CreateUser()\n\trequire_NoError(t, err)\n\tuPub, err := uKP.PublicKey()\n\trequire_NoError(t, err)\n\tuc := jwt.NewUserClaims(uPub)\n\tuc.BearerToken = true\n\tuc.Name = \"sentinel\"\n\tsentinelToken, err := uc.Encode(aKP)\n\trequire_NoError(t, err)\n\tcontent := func() []byte {\n\t\treturn []byte(fmt.Sprintf(`\n listen: 127.0.0.1:4747\n operator: %s\n system_account: %s\n resolver: MEM\n resolver_preload: %s\n\t\t\tdefault_sentinel: %s\n`, ojwt, sysPub, preloadConfig, sentinelToken))\n\t}\n\n\tserver, opts, config := runReloadServerWithContent(t, content())\n\tdefer server.Shutdown()\n\trequire_Equal(t, opts.DefaultSentinel, sentinelToken)\n\n\tuc.Name = \"sentinel-updated\"\n\tsentinelToken, err = uc.Encode(aKP)\n\trequire_NoError(t, err)\n\tchangeCurrentConfigContentWithNewContent(t, config, content())\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\topts = server.getOpts()\n\trequire_Equal(t, opts.DefaultSentinel, sentinelToken)\n}\n\n// Ensure Reload supports single user authentication config changes. Test this\n// by starting a server with authentication enabled, connect to it to verify,\n// reload config using a different username/password, ensure reconnect fails,\n// then ensure reconnect succeeds when using the correct credentials.\nfunc TestConfigReloadRotateUserAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/single_user_authentication_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.UserInfo(\"tyler\", \"T0pS3cr3t\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tdisconnected := make(chan struct{}, 1)\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tdisconnected <- struct{}{}\n\t})\n\n\t// Change user credentials.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/single_user_authentication_2.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.UserInfo(\"tyler\", \"T0pS3cr3t\")); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.UserInfo(\"derek\", \"passw0rd\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\t// Note that it is possible that client gets EOF and not able to\n\t// process async error, so don't fail if we don't get it.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\t// Give it up to 1 sec.\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n}\n\n// Ensure Reload supports enabling single user authentication. Test this by\n// starting a server with authentication disabled, connect to it to verify,\n// reload config using with a username/password, ensure reconnect fails, then\n// ensure reconnect succeeds when using the correct credentials.\nfunc TestConfigReloadEnableUserAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tdisconnected := make(chan struct{}, 1)\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tdisconnected <- struct{}{}\n\t})\n\n\t// Enable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/single_user_authentication_1.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.UserInfo(\"tyler\", \"T0pS3cr3t\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\t// Note that it is possible that client gets EOF and not able to\n\t// process async error, so don't fail if we don't get it.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n}\n\n// Ensure Reload supports disabling single user authentication. Test this by\n// starting a server with authentication enabled, connect to it to verify,\n// reload config using with authentication disabled, then ensure connecting\n// with no credentials succeeds.\nfunc TestConfigReloadDisableUserAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/single_user_authentication_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.UserInfo(\"tyler\", \"T0pS3cr3t\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tt.Fatalf(\"Client received an unexpected error: %v\", err)\n\t})\n\n\t// Disable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/basic.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting succeeds with no credentials.\n\tconn, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n}\n\n// Ensure Reload supports token authentication config changes. Test this by\n// starting a server with token authentication enabled, connect to it to\n// verify, reload config using a different token, ensure reconnect fails, then\n// ensure reconnect succeeds when using the correct token.\nfunc TestConfigReloadRotateTokenAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/token_authentication_1.conf\")\n\tdefer server.Shutdown()\n\n\tdisconnected := make(chan struct{})\n\tasyncErr := make(chan error)\n\teh := func(nc *nats.Conn, sub *nats.Subscription, err error) { asyncErr <- err }\n\tdh := func(*nats.Conn) { disconnected <- struct{}{} }\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.Token(\"T0pS3cr3t\"), nats.ErrorHandler(eh), nats.DisconnectHandler(dh))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Change authentication token.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/token_authentication_2.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.Token(\"T0pS3cr3t\")); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.Token(\"passw0rd\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected authorization error\")\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n}\n\n// Ensure Reload supports enabling token authentication. Test this by starting\n// a server with authentication disabled, connect to it to verify, reload\n// config using with a token, ensure reconnect fails, then ensure reconnect\n// succeeds when using the correct token.\nfunc TestConfigReloadEnableTokenAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tdisconnected := make(chan struct{}, 1)\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tdisconnected <- struct{}{}\n\t})\n\n\t// Enable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/token_authentication_1.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.Token(\"T0pS3cr3t\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\t// Note that it is possible that client gets EOF and not able to\n\t// process async error, so don't fail if we don't get it.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n}\n\n// Ensure Reload supports disabling single token authentication. Test this by\n// starting a server with authentication enabled, connect to it to verify,\n// reload config using with authentication disabled, then ensure connecting\n// with no token succeeds.\nfunc TestConfigReloadDisableTokenAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/token_authentication_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.Token(\"T0pS3cr3t\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tt.Fatalf(\"Client received an unexpected error: %v\", err)\n\t})\n\n\t// Disable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/basic.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting succeeds with no credentials.\n\tconn, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n}\n\n// Ensure Reload supports users authentication config changes. Test this by\n// starting a server with users authentication enabled, connect to it to\n// verify, reload config using a different user, ensure reconnect fails, then\n// ensure reconnect succeeds when using the correct credentials.\nfunc TestConfigReloadRotateUsersAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/multiple_users_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tdisconnected := make(chan struct{}, 1)\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tdisconnected <- struct{}{}\n\t})\n\n\t// These credentials won't change.\n\tnc2, err := nats.Connect(addr, nats.UserInfo(\"bob\", \"bar\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc2.Close()\n\tsub, err := nc2.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\t// Change users credentials.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/multiple_users_2.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"foo\")); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"baz\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\t// Note that it is possible that client gets EOF and not able to\n\t// process async error, so don't fail if we don't get it.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n\n\t// Ensure the connection using unchanged credentials can still\n\t// publish/receive.\n\tif err := nc2.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tnc2.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n}\n\n// Ensure Reload supports enabling users authentication. Test this by starting\n// a server with authentication disabled, connect to it to verify, reload\n// config using with users, ensure reconnect fails, then ensure reconnect\n// succeeds when using the correct credentials.\nfunc TestConfigReloadEnableUsersAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tdisconnected := make(chan struct{}, 1)\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tdisconnected <- struct{}{}\n\t})\n\n\t// Enable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/multiple_users_1.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting fails.\n\tif _, err := nats.Connect(addr); err == nil {\n\t\tt.Fatal(\"Expected connect to fail\")\n\t}\n\n\t// Ensure connecting succeeds when using new credentials.\n\tconn, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n\n\t// Ensure the previous connection received an authorization error.\n\t// Note that it is possible that client gets EOF and not able to\n\t// process async error, so don't fail if we don't get it.\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t}\n\n\t// Ensure the previous connection was disconnected.\n\tselect {\n\tcase <-disconnected:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n}\n\n// Ensure Reload supports disabling users authentication. Test this by starting\n// a server with authentication enabled, connect to it to verify,\n// reload config using with authentication disabled, then ensure connecting\n// with no credentials succeeds.\nfunc TestConfigReloadDisableUsersAuthentication(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/multiple_users_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tt.Fatalf(\"Client received an unexpected error: %v\", err)\n\t})\n\n\t// Disable authentication.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/basic.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure connecting succeeds with no credentials.\n\tconn, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tconn.Close()\n}\n\n// Ensure Reload supports changing permissions. Test this by starting a server\n// with a user configured with certain permissions, test publish and subscribe,\n// reload config with new permissions, ensure the previous subscription was\n// closed and publishes fail, then ensure the new permissions succeed.\nfunc TestConfigReloadChangePermissions(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/authorization_1.conf\")\n\tdefer server.Shutdown()\n\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr, nats.UserInfo(\"bob\", \"bar\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\t// Ensure we can publish and receive messages as a sanity check.\n\tsub, err := nc.SubscribeSync(\"_INBOX.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tnc.Flush()\n\n\tconn, err := nats.Connect(addr, nats.UserInfo(\"alice\", \"foo\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\tsub2, err := conn.SubscribeSync(\"req.foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tif err := conn.Publish(\"_INBOX.foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\tif err := nc.Publish(\"req.foo\", []byte(\"world\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tnc.Flush()\n\n\tmsg, err = sub2.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"world\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"world\"), msg.Data)\n\t}\n\n\t// Susan will subscribe to two subjects, both will succeed but a send to foo.bar should not succeed\n\t// however PUBLIC.foo should.\n\tsconn, err := nats.Connect(addr, nats.UserInfo(\"susan\", \"baz\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer sconn.Close()\n\n\tasyncErr2 := make(chan error, 1)\n\tsconn.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr2 <- err\n\t})\n\n\tfooSub, err := sconn.SubscribeSync(\"foo.*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tsconn.Flush()\n\n\t// Publishing from bob on foo.bar should not come through.\n\tif err := conn.Publish(\"foo.bar\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\t_, err = fooSub.NextMsg(100 * time.Millisecond)\n\tif err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Received a message we shouldn't have\")\n\t}\n\n\tpubSub, err := sconn.SubscribeSync(\"PUBLIC.*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tsconn.Flush()\n\n\tselect {\n\tcase err := <-asyncErr2:\n\t\tt.Fatalf(\"Received unexpected error for susan: %v\", err)\n\tdefault:\n\t}\n\n\t// This should work ok with original config.\n\tif err := conn.Publish(\"PUBLIC.foo\", []byte(\"hello monkey\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\tmsg, err = pubSub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello monkey\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %q\\ngot: %q\", \"hello monkey\", msg.Data)\n\t}\n\n\t///////////////////////////////////////////\n\t// Change permissions.\n\t///////////////////////////////////////////\n\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/authorization_2.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure we receive an error for the subscription that is no longer authorized.\n\t// In this test, since connection is not closed by the server,\n\t// the client must receive an -ERR\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif !strings.Contains(strings.ToLower(err.Error()), \"permissions violation for subscription to \\\"_inbox.>\\\"\") {\n\t\t\tt.Fatalf(\"Expected permissions violation error, got %v\", err)\n\t\t}\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected permissions violation error\")\n\t}\n\n\t// Ensure we receive an error when publishing to req.foo and we no longer\n\t// receive messages on _INBOX.>.\n\tif err := nc.Publish(\"req.foo\", []byte(\"hola\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tnc.Flush()\n\tif err := conn.Publish(\"_INBOX.foo\", []byte(\"mundo\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif !strings.Contains(strings.ToLower(err.Error()), \"permissions violation for publish to \\\"req.foo\\\"\") {\n\t\t\tt.Fatalf(\"Expected permissions violation error, got %v\", err)\n\t\t}\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected permissions violation error\")\n\t}\n\n\tqueued, _, err := sub2.Pending()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to get pending messaged: %v\", err)\n\t}\n\tif queued != 0 {\n\t\tt.Fatalf(\"Pending is incorrect.\\nexpected: 0\\ngot: %d\", queued)\n\t}\n\n\tqueued, _, err = sub.Pending()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to get pending messaged: %v\", err)\n\t}\n\tif queued != 0 {\n\t\tt.Fatalf(\"Pending is incorrect.\\nexpected: 0\\ngot: %d\", queued)\n\t}\n\n\t// Ensure we can publish to _INBOX.foo.bar and subscribe to _INBOX.foo.>.\n\tsub, err = nc.SubscribeSync(\"_INBOX.foo.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tnc.Flush()\n\tif err := nc.Publish(\"_INBOX.foo.bar\", []byte(\"testing\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tnc.Flush()\n\tmsg, err = sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"testing\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"testing\"), msg.Data)\n\t}\n\n\tselect {\n\tcase err := <-asyncErr:\n\t\tt.Fatalf(\"Received unexpected error: %v\", err)\n\tdefault:\n\t}\n\n\t// Now check susan again.\n\t//\n\t// This worked ok with original config but should not deliver a message now.\n\tif err := conn.Publish(\"PUBLIC.foo\", []byte(\"hello monkey\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\t_, err = pubSub.NextMsg(100 * time.Millisecond)\n\tif err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Received a message we shouldn't have\")\n\t}\n\n\t// Now check foo.bar, which did not work before but should work now..\n\tif err := conn.Publish(\"foo.bar\", []byte(\"hello?\")); err != nil {\n\t\tt.Fatalf(\"Error publishing message: %v\", err)\n\t}\n\tconn.Flush()\n\n\tmsg, err = fooSub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello?\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %q\\ngot: %q\", \"hello?\", msg.Data)\n\t}\n\n\t// Once last check for no errors.\n\tsconn.Flush()\n\n\tselect {\n\tcase err := <-asyncErr2:\n\t\tt.Fatalf(\"Received unexpected error for susan: %v\", err)\n\tdefault:\n\t}\n}\n\n// Ensure Reload returns an error when attempting to change cluster address\n// host.\nfunc TestConfigReloadClusterHostUnsupported(t *testing.T) {\n\tserver, _, config := runReloadServerWithConfig(t, \"./configs/reload/srv_a_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Attempt to change cluster listen host.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/srv_c_1.conf\")\n\n\t// This should fail because cluster address cannot be changed.\n\tif err := server.Reload(); err == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n}\n\n// Ensure Reload returns an error when attempting to change cluster address\n// port.\nfunc TestConfigReloadClusterPortUnsupported(t *testing.T) {\n\tserver, _, config := runReloadServerWithConfig(t, \"./configs/reload/srv_a_1.conf\")\n\tdefer server.Shutdown()\n\n\t// Attempt to change cluster listen port.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/srv_b_1.conf\")\n\n\t// This should fail because cluster address cannot be changed.\n\tif err := server.Reload(); err == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n}\n\n// Ensure Reload supports enabling route authorization. Test this by starting\n// two servers in a cluster without authorization, ensuring messages flow\n// between them, then reloading with authorization and ensuring messages no\n// longer flow until reloading with the correct credentials.\nfunc TestConfigReloadEnableClusterAuthorization(t *testing.T) {\n\tsrvb, srvbOpts, srvbConfig := runReloadServerWithConfig(t, \"./configs/reload/srv_b_1.conf\")\n\tdefer srvb.Shutdown()\n\n\tsrva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, \"./configs/reload/srv_a_1.conf\")\n\tdefer srva.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\tsrvaAddr := fmt.Sprintf(\"nats://%s:%d\", srvaOpts.Host, srvaOpts.Port)\n\tsrvaConn, err := nats.Connect(srvaAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvaConn.Close()\n\tsub, err := srvaConn.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\tif err := srvaConn.Flush(); err != nil {\n\t\tt.Fatalf(\"Error flushing: %v\", err)\n\t}\n\n\tsrvbAddr := fmt.Sprintf(\"nats://%s:%d\", srvbOpts.Host, srvbOpts.Port)\n\tsrvbConn, err := nats.Connect(srvbAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvbConn.Close()\n\n\tif numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, numRoutes)\n\t}\n\n\t// Ensure messages flow through the cluster as a sanity check.\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\t// Enable route authorization.\n\tchangeCurrentConfigContent(t, srvbConfig, \"./configs/reload/srv_b_2.conf\")\n\tif err := srvb.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\tcheckNumRoutes(t, srvb, 0)\n\n\t// Ensure messages no longer flow through the cluster.\n\tfor i := 0; i < 5; i++ {\n\t\tif err := srvbConn.Publish(\"foo\", []byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t\t}\n\t\tsrvbConn.Flush()\n\t}\n\tif _, err := sub.NextMsg(50 * time.Millisecond); err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected ErrTimeout, got %v\", err)\n\t}\n\n\t// Reload Server A with correct route credentials.\n\tchangeCurrentConfigContent(t, srvaConfig, \"./configs/reload/srv_a_2.conf\")\n\tif err := srva.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\tcheckClusterFormed(t, srva, srvb)\n\n\tif numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, numRoutes)\n\t}\n\n\t// Ensure messages flow through the cluster now.\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hola\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err = sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hola\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hola\"), msg.Data)\n\t}\n}\n\n// Ensure Reload supports disabling route authorization. Test this by starting\n// two servers in a cluster with authorization, ensuring messages flow\n// between them, then reloading without authorization and ensuring messages\n// still flow.\nfunc TestConfigReloadDisableClusterAuthorization(t *testing.T) {\n\tsrvb, srvbOpts, srvbConfig := runReloadServerWithConfig(t, \"./configs/reload/srv_b_2.conf\")\n\tdefer srvb.Shutdown()\n\n\tsrva, srvaOpts, _ := runReloadServerWithConfig(t, \"./configs/reload/srv_a_2.conf\")\n\tdefer srva.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\tsrvaAddr := fmt.Sprintf(\"nats://%s:%d\", srvaOpts.Host, srvaOpts.Port)\n\tsrvaConn, err := nats.Connect(srvaAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvaConn.Close()\n\n\tsub, err := srvaConn.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\tif err := srvaConn.Flush(); err != nil {\n\t\tt.Fatalf(\"Error flushing: %v\", err)\n\t}\n\n\tsrvbAddr := fmt.Sprintf(\"nats://%s:%d\", srvbOpts.Host, srvbOpts.Port)\n\tsrvbConn, err := nats.Connect(srvbAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvbConn.Close()\n\n\tif numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, numRoutes)\n\t}\n\n\t// Ensure messages flow through the cluster as a sanity check.\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\t// Disable route authorization.\n\tchangeCurrentConfigContent(t, srvbConfig, \"./configs/reload/srv_b_1.conf\")\n\tif err := srvb.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\tif numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, numRoutes)\n\t}\n\n\t// Ensure messages still flow through the cluster.\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hola\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err = sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hola\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hola\"), msg.Data)\n\t}\n}\n\n// Ensure Reload supports changing cluster routes. Test this by starting\n// two servers in a cluster, ensuring messages flow between them, then\n// reloading with a different route and ensuring messages flow through the new\n// cluster.\nfunc TestConfigReloadClusterRoutes(t *testing.T) {\n\tsrvb, srvbOpts, _ := runReloadServerWithConfig(t, \"./configs/reload/srv_b_1.conf\")\n\tdefer srvb.Shutdown()\n\n\tsrva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, \"./configs/reload/srv_a_1.conf\")\n\tdefer srva.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\tsrvcOpts, err := ProcessConfigFile(\"./configs/reload/srv_c_1.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\tsrvcOpts.NoLog = true\n\tsrvcOpts.NoSigs = true\n\n\tsrvc := RunServer(srvcOpts)\n\tdefer srvc.Shutdown()\n\n\tsrvaAddr := fmt.Sprintf(\"nats://%s:%d\", srvaOpts.Host, srvaOpts.Port)\n\tsrvaConn, err := nats.Connect(srvaAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvaConn.Close()\n\n\tsub, err := srvaConn.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\tif err := srvaConn.Flush(); err != nil {\n\t\tt.Fatalf(\"Error flushing: %v\", err)\n\t}\n\n\tsrvbAddr := fmt.Sprintf(\"nats://%s:%d\", srvbOpts.Host, srvbOpts.Port)\n\tsrvbConn, err := nats.Connect(srvbAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvbConn.Close()\n\n\tif numRoutes := srvb.NumRoutes(); numRoutes != DEFAULT_ROUTE_POOL_SIZE {\n\t\tt.Fatalf(\"Expected %d route, got %d\", DEFAULT_ROUTE_POOL_SIZE, numRoutes)\n\t}\n\n\t// Ensure consumer on srvA is propagated to srvB\n\tcheckExpectedSubs(t, 1, srvb)\n\n\t// Ensure messages flow through the cluster as a sanity check.\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err := sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\t// Reload cluster routes.\n\tchangeCurrentConfigContent(t, srvaConfig, \"./configs/reload/srv_a_3.conf\")\n\tif err := srva.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Kill old route server.\n\tsrvbConn.Close()\n\tsrvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvc)\n\n\tsrvcAddr := fmt.Sprintf(\"nats://%s:%d\", srvcOpts.Host, srvcOpts.Port)\n\tsrvcConn, err := nats.Connect(srvcAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvcConn.Close()\n\n\t// Ensure messages flow through the new cluster.\n\tfor i := 0; i < 5; i++ {\n\t\tif err := srvcConn.Publish(\"foo\", []byte(\"hola\")); err != nil {\n\t\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t\t}\n\t\tsrvcConn.Flush()\n\t}\n\tmsg, err = sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hola\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hola\"), msg.Data)\n\t}\n}\n\n// Ensure Reload supports removing a solicited route. In this case from A->B\n// Test this by starting two servers in a cluster, ensuring messages flow between them.\n// Then stop server B, and have server A continue to try to connect. Reload A with a config\n// that removes the route and make sure it does not connect to server B when its restarted.\nfunc TestConfigReloadClusterRemoveSolicitedRoutes(t *testing.T) {\n\tsrvb, srvbOpts := RunServerWithConfig(\"./configs/reload/srv_b_1.conf\")\n\tdefer srvb.Shutdown()\n\n\tsrva, srvaOpts, srvaConfig := runReloadServerWithConfig(t, \"./configs/reload/srv_a_1.conf\")\n\tdefer srva.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\tsrvaAddr := fmt.Sprintf(\"nats://%s:%d\", srvaOpts.Host, srvaOpts.Port)\n\tsrvaConn, err := nats.Connect(srvaAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvaConn.Close()\n\tsub, err := srvaConn.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\tif err := srvaConn.Flush(); err != nil {\n\t\tt.Fatalf(\"Error flushing: %v\", err)\n\t}\n\tcheckExpectedSubs(t, 1, srvb)\n\n\tsrvbAddr := fmt.Sprintf(\"nats://%s:%d\", srvbOpts.Host, srvbOpts.Port)\n\tsrvbConn, err := nats.Connect(srvbAddr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer srvbConn.Close()\n\n\tif err := srvbConn.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tsrvbConn.Flush()\n\tmsg, err := sub.NextMsg(5 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\tif string(msg.Data) != \"hello\" {\n\t\tt.Fatalf(\"Msg is incorrect.\\nexpected: %+v\\ngot: %+v\", []byte(\"hello\"), msg.Data)\n\t}\n\n\t// Now stop server B.\n\tsrvb.Shutdown()\n\n\t// Wait til route is dropped.\n\tcheckNumRoutes(t, srva, 0)\n\n\t// Now change config for server A to not solicit a route to server B.\n\tchangeCurrentConfigContent(t, srvaConfig, \"./configs/reload/srv_a_4.conf\")\n\tif err := srva.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Restart server B.\n\tsrvb, _ = RunServerWithConfig(\"./configs/reload/srv_b_1.conf\")\n\tdefer srvb.Shutdown()\n\n\t// We should not have a cluster formed here.\n\tnumRoutes := 0\n\tdeadline := time.Now().Add(2 * DEFAULT_ROUTE_RECONNECT)\n\tfor time.Now().Before(deadline) {\n\t\tif numRoutes = srva.NumRoutes(); numRoutes != 0 {\n\t\t\tbreak\n\t\t} else {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t}\n\t}\n\tif numRoutes != 0 {\n\t\tt.Fatalf(\"Expected 0 routes for server A, got %d\", numRoutes)\n\t}\n}\n\nfunc reloadUpdateConfig(t *testing.T, s *Server, conf, content string) {\n\tt.Helper()\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n}\n\nfunc TestConfigReloadClusterAdvertise(t *testing.T) {\n\ts, _, conf := runReloadServerWithContent(t, []byte(`\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tcluster: {\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t}\n\t`))\n\tdefer s.Shutdown()\n\n\torgClusterPort := s.ClusterAddr().Port\n\n\tverify := func(expectedHost string, expectedPort int, expectedIP string) {\n\t\ts.mu.Lock()\n\t\trouteInfo := s.routeInfo\n\t\trij := generateInfoJSON(&routeInfo)\n\t\ts.mu.Unlock()\n\t\tif routeInfo.Host != expectedHost || routeInfo.Port != expectedPort || routeInfo.IP != expectedIP {\n\t\t\tt.Fatalf(\"Expected host/port/IP to be %s:%v, %q, got %s:%d, %q\",\n\t\t\t\texpectedHost, expectedPort, expectedIP, routeInfo.Host, routeInfo.Port, routeInfo.IP)\n\t\t}\n\t\trouteInfoJSON := Info{}\n\t\terr := json.Unmarshal(rij[5:], &routeInfoJSON) // Skip \"INFO \"\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on Unmarshal: %v\", err)\n\t\t}\n\t\t// Check that server routeInfoJSON was updated too\n\t\tif !reflect.DeepEqual(routeInfo, routeInfoJSON) {\n\t\t\tt.Fatalf(\"Expected routeInfoJSON to be %+v, got %+v\", routeInfo, routeInfoJSON)\n\t\t}\n\t}\n\n\t// Update config with cluster_advertise\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tcluster_advertise: \"me:1\"\n\t}\n\t`)\n\tverify(\"me\", 1, \"nats-route://me:1/\")\n\n\t// Update config with cluster_advertise (no port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tcluster_advertise: \"me\"\n\t}\n\t`)\n\tverify(\"me\", orgClusterPort, fmt.Sprintf(\"nats-route://me:%d/\", orgClusterPort))\n\n\t// Update config with cluster_advertise (-1 port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tcluster_advertise: \"me:-1\"\n\t}\n\t`)\n\tverify(\"me\", orgClusterPort, fmt.Sprintf(\"nats-route://me:%d/\", orgClusterPort))\n\n\t// Update to remove cluster_advertise\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t}\n\t`)\n\tverify(\"0.0.0.0\", orgClusterPort, \"\")\n}\n\nfunc TestConfigReloadClusterNoAdvertise(t *testing.T) {\n\ts, _, conf := runReloadServerWithContent(t, []byte(`\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tclient_advertise: \"me:1\"\n\t\tcluster: {\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t}\n\t`))\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\tccurls := s.routeInfo.ClientConnectURLs\n\ts.mu.Unlock()\n\tif len(ccurls) != 1 && ccurls[0] != \"me:1\" {\n\t\tt.Fatalf(\"Unexpected routeInfo.ClientConnectURLS: %v\", ccurls)\n\t}\n\n\t// Update config with no_advertise\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tclient_advertise: \"me:1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tno_advertise: true\n\t}\n\t`)\n\n\ts.mu.Lock()\n\tccurls = s.routeInfo.ClientConnectURLs\n\ts.mu.Unlock()\n\tif len(ccurls) != 0 {\n\t\tt.Fatalf(\"Unexpected routeInfo.ClientConnectURLS: %v\", ccurls)\n\t}\n\n\t// Update config with cluster_advertise (no port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tclient_advertise: \"me:1\"\n\tcluster: {\n\t\tlisten: \"0.0.0.0:-1\"\n\t}\n\t`)\n\ts.mu.Lock()\n\tccurls = s.routeInfo.ClientConnectURLs\n\ts.mu.Unlock()\n\tif len(ccurls) != 1 && ccurls[0] != \"me:1\" {\n\t\tt.Fatalf(\"Unexpected routeInfo.ClientConnectURLS: %v\", ccurls)\n\t}\n}\n\nfunc TestConfigReloadClusterName(t *testing.T) {\n\ts, _, conf := runReloadServerWithContent(t, []byte(`\n\t\tlisten: \"0.0.0.0:-1\"\n\t\tcluster: {\n\t\t\tname: \"abc\"\n\t\t\tlisten: \"0.0.0.0:-1\"\n\t\t}\n\t`))\n\tdefer s.Shutdown()\n\n\t// Update config with a new cluster name.\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tcluster: {\n\t\tname: \"xyz\"\n\t\tlisten: \"0.0.0.0:-1\"\n\t}\n\t`)\n\n\tif s.ClusterName() != \"xyz\" {\n\t\tt.Fatalf(\"Expected update clustername of \\\"xyz\\\", got %q\", s.ClusterName())\n\t}\n}\n\nfunc TestConfigReloadMaxSubsUnsupported(t *testing.T) {\n\ts, _, conf := runReloadServerWithContent(t, []byte(`\n\t\tport: -1\n\t\tmax_subs: 1\n\t\t`))\n\tdefer s.Shutdown()\n\n\tif err := os.WriteFile(conf, []byte(`max_subs: 10`), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\tif err := s.Reload(); err == nil {\n\t\tt.Fatal(\"Expected Reload to return an error\")\n\t}\n}\n\nfunc TestConfigReloadClientAdvertise(t *testing.T) {\n\ts, _, conf := runReloadServerWithContent(t, []byte(`listen: \"0.0.0.0:-1\"`))\n\tdefer s.Shutdown()\n\n\torgPort := s.Addr().(*net.TCPAddr).Port\n\n\tverify := func(expectedHost string, expectedPort int) {\n\t\ts.mu.Lock()\n\t\tinfo := s.info\n\t\ts.mu.Unlock()\n\t\tif info.Host != expectedHost || info.Port != expectedPort {\n\t\t\tstackFatalf(t, \"Expected host/port to be %s:%d, got %s:%d\",\n\t\t\t\texpectedHost, expectedPort, info.Host, info.Port)\n\t\t}\n\t}\n\n\t// Update config with ClientAdvertise (port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tclient_advertise: \"me:1\"\n\t`)\n\tverify(\"me\", 1)\n\n\t// Update config with ClientAdvertise (no port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tclient_advertise: \"me\"\n\t`)\n\tverify(\"me\", orgPort)\n\n\t// Update config with ClientAdvertise (-1 port specified)\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"0.0.0.0:-1\"\n\tclient_advertise: \"me:-1\"\n\t`)\n\tverify(\"me\", orgPort)\n\n\t// Now remove ClientAdvertise to check that original values\n\t// are restored.\n\treloadUpdateConfig(t, s, conf, `listen: \"0.0.0.0:-1\"`)\n\tverify(\"0.0.0.0\", orgPort)\n}\n\n// Ensure Reload supports changing the max connections. Test this by starting a\n// server with no max connections, connecting two clients, reloading with a\n// max connections of one, and ensuring one client is disconnected.\nfunc TestConfigReloadMaxConnections(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Make two connections.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc1, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc1.Close()\n\tclosed := make(chan struct{}, 1)\n\tnc1.SetDisconnectHandler(func(*nats.Conn) {\n\t\tclosed <- struct{}{}\n\t})\n\tnc2, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc2.Close()\n\tnc2.SetDisconnectHandler(func(*nats.Conn) {\n\t\tclosed <- struct{}{}\n\t})\n\n\tif numClients := server.NumClients(); numClients != 2 {\n\t\tt.Fatalf(\"Expected 2 clients, got %d\", numClients)\n\t}\n\n\t// Set max connections to one.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/max_connections.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure one connection was closed.\n\tselect {\n\tcase <-closed:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected to be disconnected\")\n\t}\n\n\tcheckClientsCount(t, server, 1)\n\n\t// Ensure new connections fail.\n\t_, err = nats.Connect(addr)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error on connect\")\n\t}\n}\n\n// Ensure Reload supports refusing all connections. Test this by starting a\n// server with no max connections, connecting two clients, reloading with a\n// max connections of one, and ensuring one client is disconnected.\nfunc TestConfigReloadMaxConnectionsPreventAll(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Make two connections.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc1, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc1.Close()\n\tclosed := make(chan struct{}, 1)\n\tnc1.SetDisconnectHandler(func(*nats.Conn) {\n\t\tclosed <- struct{}{}\n\t})\n\tnc2, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc2.Close()\n\tnc2.SetDisconnectHandler(func(*nats.Conn) {\n\t\tclosed <- struct{}{}\n\t})\n\n\tif numClients := server.NumClients(); numClients != 2 {\n\t\tt.Fatalf(\"Expected 2 clients, got %d\", numClients)\n\t}\n\n\t// Set max connections to one.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/max_connections_refuse_all.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure one connection was closed.\n\tselect {\n\tcase <-closed:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected to be disconnected\")\n\t}\n\n\tcheckClientsCount(t, server, 0)\n\n\t// Ensure new connections fail.\n\t_, err = nats.Connect(addr)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error on connect\")\n\t}\n}\n\n// Ensure reload supports changing the max payload size. Test this by starting\n// a server with the default size limit, ensuring publishes work, reloading\n// with a restrictive limit, and ensuring publishing an oversized message fails\n// and disconnects the client.\nfunc TestConfigReloadMaxPayload(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\tclosed := make(chan struct{})\n\tnc.SetDisconnectHandler(func(*nats.Conn) {\n\t\tclosed <- struct{}{}\n\t})\n\n\tconn, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer conn.Close()\n\tsub, err := conn.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tconn.Flush()\n\n\t// Ensure we can publish as a sanity check.\n\tif err := nc.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tnc.Flush()\n\t_, err = sub.NextMsg(2 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error receiving message: %v\", err)\n\t}\n\n\t// Set max payload to one.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/max_payload.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Ensure oversized messages don't get delivered and the client is\n\t// disconnected.\n\tif err := nc.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error publishing: %v\", err)\n\t}\n\tnc.Flush()\n\t_, err = sub.NextMsg(20 * time.Millisecond)\n\tif err != nats.ErrTimeout {\n\t\tt.Fatalf(\"Expected ErrTimeout, got: %v\", err)\n\t}\n\n\tselect {\n\tcase <-closed:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Expected to be disconnected\")\n\t}\n}\n\n// Ensure reload supports rotating out files. Test this by starting\n// a server with log and pid files, reloading new ones, then check that\n// we can rename and delete the old log/pid files.\nfunc TestConfigReloadRotateFiles(t *testing.T) {\n\tserver, _, config := runReloadServerWithConfig(t, \"./configs/reload/file_rotate.conf\")\n\tdefer func() {\n\t\tremoveFile(t, \"log1.txt\")\n\t\tremoveFile(t, \"nats-server1.pid\")\n\t}()\n\tdefer server.Shutdown()\n\n\t// Configure the logger to enable actual logging\n\topts := server.getOpts()\n\topts.NoLog = false\n\tserver.ConfigureLogger()\n\n\t// Load a config that renames the files.\n\tchangeCurrentConfigContent(t, config, \"./configs/reload/file_rotate1.conf\")\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Error reloading config: %v\", err)\n\t}\n\n\t// Make sure the new files exist.\n\tif _, err := os.Stat(\"log1.txt\"); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Error reloading config, no new file: %v\", err)\n\t}\n\tif _, err := os.Stat(\"nats-server1.pid\"); os.IsNotExist(err) {\n\t\tt.Fatalf(\"Error reloading config, no new file: %v\", err)\n\t}\n\n\t// Check that old file can be renamed.\n\tif err := os.Rename(\"log.txt\", \"log_old.txt\"); err != nil {\n\t\tt.Fatalf(\"Error reloading config, cannot rename file: %v\", err)\n\t}\n\tif err := os.Rename(\"nats-server.pid\", \"nats-server_old.pid\"); err != nil {\n\t\tt.Fatalf(\"Error reloading config, cannot rename file: %v\", err)\n\t}\n\n\t// Check that the old files can be removed after rename.\n\tremoveFile(t, \"log_old.txt\")\n\tremoveFile(t, \"nats-server_old.pid\")\n}\n\nfunc TestConfigReloadClusterWorks(t *testing.T) {\n\tconfBTemplate := `\n\t\tlisten: -1\n\t\tcluster: {\n\t\t\tlisten: 127.0.0.1:7244\n\t\t\tauthorization {\n\t\t\t\tuser: ruser\n\t\t\t\tpassword: pwd\n\t\t\t\ttimeout: %d\n\t\t\t}\n\t\t\troutes = [\n\t\t\t\tnats-route://ruser:pwd@127.0.0.1:7246\n\t\t\t]\n\t\t}`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, 3)))\n\n\tconfATemplate := `\n\t\tlisten: -1\n\t\tcluster: {\n\t\t\tlisten: 127.0.0.1:7246\n\t\t\tauthorization {\n\t\t\t\tuser: ruser\n\t\t\t\tpassword: pwd\n\t\t\t\ttimeout: %d\n\t\t\t}\n\t\t\troutes = [\n\t\t\t\tnats-route://ruser:pwd@127.0.0.1:7244\n\t\t\t]\n\t\t}`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, 3)))\n\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\n\tsrva, _ := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\t// Wait for the cluster to form and capture the connection IDs of each route\n\tcheckClusterFormed(t, srva, srvb)\n\n\tgetCID := func(s *Server) uint64 {\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\tif r := getFirstRoute(s); r != nil {\n\t\t\treturn r.cid\n\t\t}\n\t\treturn 0\n\t}\n\tacid := getCID(srva)\n\tbcid := getCID(srvb)\n\n\t// Update auth timeout to force a check of the connected route auth\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, 5))\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, 5))\n\n\t// Wait a little bit to ensure that there is no issue with connection\n\t// breaking at this point (this was an issue before).\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Cluster should still exist\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Check that routes were not re-created\n\tnewacid := getCID(srva)\n\tnewbcid := getCID(srvb)\n\n\tif newacid != acid {\n\t\tt.Fatalf(\"Expected server A route ID to be %v, got %v\", acid, newacid)\n\t}\n\tif newbcid != bcid {\n\t\tt.Fatalf(\"Expected server B route ID to be %v, got %v\", bcid, newbcid)\n\t}\n}\n\nfunc TestConfigReloadClusterPerms(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\timport {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t\texport {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `\"foo\"`, `\"foo\"`)))\n\tsrva, _ := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBTemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\timport {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t\texport {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t}\n\t\t\troutes = [\n\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t]\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `\"foo\"`, `\"foo\"`, srva.ClusterAddr().Port)))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Create a connection on A\n\tnca, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", srva.Addr().(*net.TCPAddr).Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nca.Close()\n\t// Create a subscription on \"foo\" and \"bar\", only \"foo\" will be also on server B.\n\tsubFooOnA, err := nca.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tsubBarOnA, err := nca.SubscribeSync(\"bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\t// Connect on B and do the same\n\tncb, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvb.Addr().(*net.TCPAddr).Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncb.Close()\n\t// Create a subscription on \"foo\" and \"bar\", only \"foo\" will be also on server B.\n\tsubFooOnB, err := ncb.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tsubBarOnB, err := ncb.SubscribeSync(\"bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\t// Check subscriptions on each server. There should be 3 on each server,\n\t// foo and bar locally and foo from remote server.\n\tcheckExpectedSubs(t, 3, srva, srvb)\n\n\tsendMsg := func(t *testing.T, subj string, nc *nats.Conn) {\n\t\tt.Helper()\n\t\tif err := nc.Publish(subj, []byte(\"msg\")); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\tcheckSub := func(t *testing.T, sub *nats.Subscription, shouldReceive bool) {\n\t\tt.Helper()\n\t\t_, err := sub.NextMsg(100 * time.Millisecond)\n\t\tif shouldReceive && err != nil {\n\t\t\tt.Fatalf(\"Expected message on %q, got %v\", sub.Subject, err)\n\t\t} else if !shouldReceive && err == nil {\n\t\t\tt.Fatalf(\"Expected no message on %q, got one\", sub.Subject)\n\t\t}\n\t}\n\n\t// Produce from A and check received on both sides\n\tsendMsg(t, \"foo\", nca)\n\tcheckSub(t, subFooOnA, true)\n\tcheckSub(t, subFooOnB, true)\n\t// Now from B:\n\tsendMsg(t, \"foo\", ncb)\n\tcheckSub(t, subFooOnA, true)\n\tcheckSub(t, subFooOnB, true)\n\n\t// Publish on bar from A and make sure only local sub receives\n\tsendMsg(t, \"bar\", nca)\n\tcheckSub(t, subBarOnA, true)\n\tcheckSub(t, subBarOnB, false)\n\n\t// Publish on bar from B and make sure only local sub receives\n\tsendMsg(t, \"bar\", ncb)\n\tcheckSub(t, subBarOnA, false)\n\tcheckSub(t, subBarOnB, true)\n\n\t// We will now both import/export foo and bar. Start with reloading A.\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`, `[\"foo\", \"bar\"]`))\n\n\t// Since B has not been updated yet, the state should remain the same,\n\t// that is 3 subs on each server.\n\tcheckExpectedSubs(t, 3, srva, srvb)\n\n\t// Now update and reload B. Add \"baz\" for another test down below\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `[\"foo\", \"bar\", \"baz\"]`, `[\"foo\", \"bar\", \"baz\"]`, srva.ClusterAddr().Port))\n\n\t// Now 4 on each server\n\tcheckExpectedSubs(t, 4, srva, srvb)\n\n\t// Make sure that we can receive all messages\n\tsendMsg(t, \"foo\", nca)\n\tcheckSub(t, subFooOnA, true)\n\tcheckSub(t, subFooOnB, true)\n\tsendMsg(t, \"foo\", ncb)\n\tcheckSub(t, subFooOnA, true)\n\tcheckSub(t, subFooOnB, true)\n\n\tsendMsg(t, \"bar\", nca)\n\tcheckSub(t, subBarOnA, true)\n\tcheckSub(t, subBarOnB, true)\n\tsendMsg(t, \"bar\", ncb)\n\tcheckSub(t, subBarOnA, true)\n\tcheckSub(t, subBarOnB, true)\n\n\t// Create subscription on baz on server B.\n\tsubBazOnB, err := ncb.SubscribeSync(\"baz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\t// Check subscriptions count\n\tcheckExpectedSubs(t, 5, srvb)\n\tcheckExpectedSubs(t, 4, srva)\n\n\tsendMsg(t, \"baz\", nca)\n\tcheckSub(t, subBazOnB, false)\n\tsendMsg(t, \"baz\", ncb)\n\tcheckSub(t, subBazOnB, true)\n\n\t// Test UNSUB by denying something that was previously imported\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"foo\"`, `[\"foo\", \"bar\"]`))\n\t// Since A no longer imports \"bar\", we should have one less subscription\n\t// on B (B will have received an UNSUB for bar)\n\tcheckExpectedSubs(t, 4, srvb)\n\t// A, however, should still have same number of subs.\n\tcheckExpectedSubs(t, 4, srva)\n\n\t// Remove all permissions from A.\n\treloadUpdateConfig(t, srva, confA, `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t\tno_sys_acc: true\n\t`)\n\t// Server A should now have baz sub\n\tcheckExpectedSubs(t, 5, srvb)\n\tcheckExpectedSubs(t, 5, srva)\n\n\tsendMsg(t, \"baz\", nca)\n\tcheckSub(t, subBazOnB, true)\n\tsendMsg(t, \"baz\", ncb)\n\tcheckSub(t, subBazOnB, true)\n\n\t// Finally, remove permissions from B\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes = [\n\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t]\n\t\t}\n\t\tno_sys_acc: true\n\t`, srva.ClusterAddr().Port))\n\t// Check expected subscriptions count.\n\tcheckExpectedSubs(t, 5, srvb)\n\tcheckExpectedSubs(t, 5, srva)\n}\n\nfunc TestConfigReloadClusterPermsImport(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\timport: {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`)))\n\tsrva, _ := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBTemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes = [\n\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t]\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, srva.ClusterAddr().Port)))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Create a connection on A\n\tnca, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", srva.Addr().(*net.TCPAddr).Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nca.Close()\n\t// Create a subscription on \"foo\" and \"bar\"\n\tif _, err := nca.SubscribeSync(\"foo\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif _, err := nca.SubscribeSync(\"bar\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\t// Drop foo\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`))\n\tcheckExpectedSubs(t, 2, srva)\n\tcheckExpectedSubs(t, 1, srvb)\n\n\t// Add it back\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`))\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\t// Empty Import means implicit allow\n\treloadUpdateConfig(t, srva, confA, `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\texport: \">\"\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`)\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\tconfATemplate = `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\timport: {\n\t\t\t\t\tallow: [\"foo\", \"bar\"]\n\t\t\t\t\tdeny: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\t// Now deny all:\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`))\n\tcheckExpectedSubs(t, 2, srva)\n\tcheckExpectedSubs(t, 0, srvb)\n\n\t// Drop foo from the deny list\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`))\n\tcheckExpectedSubs(t, 2, srva)\n\tcheckExpectedSubs(t, 1, srvb)\n}\n\nfunc TestConfigReloadClusterPermsExport(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\texport: {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`)))\n\tsrva, _ := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBTemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes = [\n\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t]\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, srva.ClusterAddr().Port)))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Create a connection on B\n\tncb, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvb.Addr().(*net.TCPAddr).Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncb.Close()\n\t// Create a subscription on \"foo\" and \"bar\"\n\tif _, err := ncb.SubscribeSync(\"foo\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif _, err := ncb.SubscribeSync(\"bar\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\t// Drop foo\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`))\n\tcheckExpectedSubs(t, 2, srvb)\n\tcheckExpectedSubs(t, 1, srva)\n\n\t// Add it back\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`))\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\t// Empty Export means implicit allow\n\treloadUpdateConfig(t, srva, confA, `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\timport: \">\"\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`)\n\tcheckExpectedSubs(t, 2, srva, srvb)\n\n\tconfATemplate = `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\texport: {\n\t\t\t\t\tallow: [\"foo\", \"bar\"]\n\t\t\t\t\tdeny: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\t// Now deny all:\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`))\n\tcheckExpectedSubs(t, 0, srva)\n\tcheckExpectedSubs(t, 2, srvb)\n\n\t// Drop foo from the deny list\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`))\n\tcheckExpectedSubs(t, 1, srva)\n\tcheckExpectedSubs(t, 2, srvb)\n}\n\nfunc TestConfigReloadClusterPermsOldServer(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tpermissions {\n\t\t\t\texport: {\n\t\t\t\t\tallow: %s\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`)))\n\tsrva, _ := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\toptsB := DefaultOptions()\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srva.ClusterAddr().Port))\n\t// Make server B behave like an old server\n\toptsB.overrideProto = setServerProtoForTest(RouteProtoZero)\n\tsrvb := RunServer(optsB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Get the route's connection ID\n\tgetRouteRID := func() uint64 {\n\t\trid := uint64(0)\n\t\tsrvb.mu.Lock()\n\t\tif r := getFirstRoute(srvb); r != nil {\n\t\t\tr.mu.Lock()\n\t\t\trid = r.cid\n\t\t\tr.mu.Unlock()\n\t\t}\n\t\tsrvb.mu.Unlock()\n\t\treturn rid\n\t}\n\torgRID := getRouteRID()\n\n\t// Cause a config reload on A\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`))\n\n\t// Check that new route gets created\n\tcheck := func(t *testing.T) {\n\t\tt.Helper()\n\t\tcheckFor(t, 3*time.Second, 15*time.Millisecond, func() error {\n\t\t\tif rid := getRouteRID(); rid == orgRID {\n\t\t\t\treturn fmt.Errorf(\"Route does not seem to have been recreated\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheck(t)\n\n\t// Save the current value\n\torgRID = getRouteRID()\n\n\t// Add another server that supports INFO updates\n\n\toptsC := DefaultOptions()\n\toptsC.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srva.ClusterAddr().Port))\n\tsrvc := RunServer(optsC)\n\tdefer srvc.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\t// Cause a config reload on A\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"foo\"`))\n\t// Check that new route gets created\n\tcheck(t)\n}\n\nfunc TestConfigReloadAccountUsers(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tsynadia {\n\t\t\tusers = [\n\t\t\t\t{user: derek, password: derek}\n\t\t\t\t{user: foo, password: foo}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers = [\n\t\t\t\t{user: ivan, password: ivan}\n\t\t\t\t{user: bar, password: bar}\n\t\t\t]\n\t\t}\n\t\tacc_deleted_after_reload {\n\t\t\tusers = [\n\t\t\t\t{user: gone, password: soon}\n\t\t\t\t{user: baz, password: baz}\n\t\t\t\t{user: bat, password: bat}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Connect as exisiting users, should work.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://derek:derek@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tch := make(chan bool, 2)\n\tcb := func(_ *nats.Conn) {\n\t\tch <- true\n\t}\n\tnc2, err := nats.Connect(\n\t\tfmt.Sprintf(\"nats://ivan:ivan@%s:%d\", opts.Host, opts.Port),\n\t\tnats.NoReconnect(),\n\t\tnats.ClosedHandler(cb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\tnc3, err := nats.Connect(\n\t\tfmt.Sprintf(\"nats://gone:soon@%s:%d\", opts.Host, opts.Port),\n\t\tnats.NoReconnect(),\n\t\tnats.ClosedHandler(cb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc3.Close()\n\t// These users will be moved from an account to another (to a specific or to global account)\n\t// We will create subscriptions to ensure that they are moved to proper sublists too.\n\trch := make(chan bool, 4)\n\trcb := func(_ *nats.Conn) {\n\t\trch <- true\n\t}\n\tnc4, err := nats.Connect(fmt.Sprintf(\"nats://foo:foo@%s:%d\", opts.Host, opts.Port),\n\t\tnats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc4.Close()\n\tif _, err := nc4.SubscribeSync(\"foo\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc5, err := nats.Connect(fmt.Sprintf(\"nats://bar:bar@%s:%d\", opts.Host, opts.Port),\n\t\tnats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc5.Close()\n\tif _, err := nc5.SubscribeSync(\"bar\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc6, err := nats.Connect(fmt.Sprintf(\"nats://baz:baz@%s:%d\", opts.Host, opts.Port),\n\t\tnats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc6.Close()\n\tif _, err := nc6.SubscribeSync(\"baz\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc7, err := nats.Connect(fmt.Sprintf(\"nats://bat:bat@%s:%d\", opts.Host, opts.Port),\n\t\tnats.ReconnectWait(50*time.Millisecond), nats.ReconnectHandler(rcb))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc7.Close()\n\tif _, err := nc7.SubscribeSync(\"bat\"); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\t// confirm subscriptions before and after reload.\n\tvar expectedSubs uint32 = 5\n\tsAcc, err := s.LookupAccount(\"synadia\")\n\trequire_NoError(t, err)\n\tsAcc.mu.RLock()\n\tn := sAcc.sl.Count()\n\tsAcc.mu.RUnlock()\n\tif n != expectedSubs {\n\t\tt.Errorf(\"Synadia account should have %d sub, got %v\", expectedSubs, n)\n\t}\n\tnAcc, err := s.LookupAccount(\"nats.io\")\n\trequire_NoError(t, err)\n\tnAcc.mu.RLock()\n\tn = nAcc.sl.Count()\n\tnAcc.mu.RUnlock()\n\tif n != expectedSubs {\n\t\tt.Errorf(\"Nats.io account should have %d sub, got %v\", expectedSubs, n)\n\t}\n\n\t// Remove user from account and whole account\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\tauthorization {\n\t\tusers = [\n\t\t\t{user: foo, password: foo}\n\t\t\t{user: baz, password: baz}\n\t\t]\n\t}\n\taccounts {\n\t\tsynadia {\n\t\t\tusers = [\n\t\t\t\t{user: derek, password: derek}\n\t\t\t\t{user: bar, password: bar}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers = [\n\t\t\t\t{user: bat, password: bat}\n\t\t\t]\n\t\t}\n\t}\n\t`)\n\t// nc2 and nc3 should be closed\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"Did not get the closed callback\")\n\t}\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"Did not get the closed callback\")\n\t}\n\t// And first connection should still be connected\n\tif !nc.IsConnected() {\n\t\tt.Fatal(\"First connection should still be connected\")\n\t}\n\n\t// Old account should be gone\n\tif _, err := s.LookupAccount(\"acc_deleted_after_reload\"); err == nil {\n\t\tt.Fatal(\"old account should be gone\")\n\t}\n\n\t// Check subscriptions. Since most of the users have been\n\t// moving accounts, make sure we account for the reconnect\n\tfor i := 0; i < 4; i++ {\n\t\tif err := wait(rch); err != nil {\n\t\t\tt.Fatal(\"Did not get the reconnect cb\")\n\t\t}\n\t}\n\t// Still need to do the tests in a checkFor() because clients\n\t// being reconnected does not mean that resent of subscriptions\n\t// has already been processed.\n\tcheckFor(t, 2*time.Second, 100*time.Millisecond, func() error {\n\t\tgAcc, err := s.LookupAccount(globalAccountName)\n\t\trequire_NoError(t, err)\n\t\tgAcc.mu.RLock()\n\t\tn := gAcc.sl.Count()\n\t\tfooMatch := gAcc.sl.Match(\"foo\")\n\t\tbazMatch := gAcc.sl.Match(\"baz\")\n\t\tgAcc.mu.RUnlock()\n\t\t// The number of subscriptions should be 4 ($SYS.REQ.USER.INFO,\n\t\t// $SYS.REQ.ACCOUNT.PING.CONNZ, $SYS.REQ.ACCOUNT.PING.STATZ,\n\t\t// $SYS.REQ.SERVER.PING.CONNZ) + 2 (foo and baz)\n\t\tif n != 6 {\n\t\t\treturn fmt.Errorf(\"Global account should have 6 subs, got %v\", n)\n\t\t}\n\t\tif len(fooMatch.psubs) != 1 {\n\t\t\treturn fmt.Errorf(\"Global account should have foo sub\")\n\t\t}\n\t\tif len(bazMatch.psubs) != 1 {\n\t\t\treturn fmt.Errorf(\"Global account should have baz sub\")\n\t\t}\n\n\t\tsAcc, err := s.LookupAccount(\"synadia\")\n\t\trequire_NoError(t, err)\n\t\tsAcc.mu.RLock()\n\t\tn = sAcc.sl.Count()\n\t\tbarMatch := sAcc.sl.Match(\"bar\")\n\n\t\tsAcc.mu.RUnlock()\n\t\tif n != expectedSubs {\n\t\t\treturn fmt.Errorf(\"Synadia account should have %d sub, got %v\", expectedSubs, n)\n\t\t}\n\t\tif len(barMatch.psubs) != 1 {\n\t\t\treturn fmt.Errorf(\"Synadia account should have bar sub\")\n\t\t}\n\n\t\tnAcc, err := s.LookupAccount(\"nats.io\")\n\t\trequire_NoError(t, err)\n\t\tnAcc.mu.RLock()\n\t\tn = nAcc.sl.Count()\n\t\tbatMatch := nAcc.sl.Match(\"bat\")\n\t\tnAcc.mu.RUnlock()\n\t\tif n != expectedSubs {\n\t\t\treturn fmt.Errorf(\"Nats.io account should have %d sub, got %v\", expectedSubs, n)\n\t\t}\n\t\tif len(batMatch.psubs) != 1 {\n\t\t\treturn fmt.Errorf(\"Synadia account should have bar sub\")\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestConfigReloadAccountWithNoChanges(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n system_account: sys\n\taccounts {\n\t\tA {\n\t\t\tusers = [{ user: a }]\n\t\t}\n\t\tB {\n\t\t\tusers = [{ user: b }]\n\t\t}\n\t\tC {\n\t\t\tusers = [{ user: c }]\n\t\t}\n\t\tsys {\n\t\t\tusers = [{ user: sys }]\n\t\t}\n\t}\n\t`))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tncA, err := nats.Connect(fmt.Sprintf(\"nats://a:@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncA.Close()\n\n\t// Confirm default service imports are ok.\n\tcheckSubs := func(t *testing.T) {\n\t\tresp, err := ncA.Request(\"$SYS.REQ.ACCOUNT.PING.CONNZ\", nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif resp == nil || !strings.Contains(string(resp.Data), `\"num_connections\":1`) {\n\t\t\tt.Fatal(\"unexpected data in connz response\")\n\t\t}\n\t\tresp, err = ncA.Request(\"$SYS.REQ.SERVER.PING.CONNZ\", nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif resp == nil || !strings.Contains(string(resp.Data), `\"num_connections\":1`) {\n\t\t\tt.Fatal(\"unexpected data in connz response\")\n\t\t}\n\t\tresp, err = ncA.Request(\"$SYS.REQ.ACCOUNT.PING.STATZ\", nil, time.Second)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif resp == nil || !strings.Contains(string(resp.Data), `\"conns\":1`) {\n\t\t\tt.Fatal(\"unexpected data in connz response\")\n\t\t}\n\t}\n\tcheckSubs(t)\n\tbefore := s.NumSubscriptions()\n\ts.Reload()\n\tafter := s.NumSubscriptions()\n\tif before != after {\n\t\tt.Errorf(\"Number of subscriptions changed after reload: %d -> %d\", before, after)\n\t}\n\n\t// Confirm this still works after a reload...\n\tcheckSubs(t)\n\tbefore = s.NumSubscriptions()\n\ts.Reload()\n\tafter = s.NumSubscriptions()\n\tif before != after {\n\t\tt.Errorf(\"Number of subscriptions changed after reload: %d -> %d\", before, after)\n\t}\n\n\t// Do another extra reload just in case.\n\tcheckSubs(t)\n\tbefore = s.NumSubscriptions()\n\ts.Reload()\n\tafter = s.NumSubscriptions()\n\tif before != after {\n\t\tt.Errorf(\"Number of subscriptions changed after reload: %d -> %d\", before, after)\n\t}\n}\n\nfunc TestConfigReloadAccountNKeyUsers(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tsynadia {\n\t\t\tusers = [\n\t\t\t\t# Derek\n\t\t\t\t{nkey : UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers = [\n\t\t\t\t# Ivan\n\t\t\t\t{nkey : UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tsynadia, _ := s.LookupAccount(\"synadia\")\n\tnats, _ := s.LookupAccount(\"nats.io\")\n\n\tseed1 := []byte(\"SUAPM67TC4RHQLKBX55NIQXSMATZDOZK6FNEOSS36CAYA7F7TY66LP4BOM\")\n\tseed2 := []byte(\"SUAIS5JPX4X4GJ7EIIJEQ56DH2GWPYJRPWN5XJEDENJOZHCBLI7SEPUQDE\")\n\n\tkp, _ := nkeys.FromSeed(seed1)\n\tpubKey, _ := kp.PublicKey()\n\n\tc, cr, l := newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\tvar info nonceInfo\n\tif err := json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err := kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs := fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n\tif c.acc != synadia {\n\t\tt.Fatalf(\"Expected the nkey client's account to match 'synadia', got %v\", c.acc)\n\t}\n\n\t// Now nats account nkey user.\n\tkp, _ = nkeys.FromSeed(seed2)\n\tpubKey, _ = kp.PublicKey()\n\n\tc, cr, l = newClientForServer(s)\n\tdefer c.close()\n\t// Check for Nonce\n\terr = json.Unmarshal([]byte(l[5:]), &info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non-empty nonce with nkeys defined\")\n\t}\n\tsigraw, err = kp.Sign([]byte(info.Nonce))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed signing nonce: %v\", err)\n\t}\n\tsig = base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t// PING needed to flush the +OK to us.\n\tcs = fmt.Sprintf(\"CONNECT {\\\"nkey\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", pubKey, sig)\n\tc.parseAsync(cs)\n\tl, _ = cr.ReadString('\\n')\n\tif !strings.HasPrefix(l, \"+OK\") {\n\t\tt.Fatalf(\"Expected an OK, got: %v\", l)\n\t}\n\tif c.acc != nats {\n\t\tt.Fatalf(\"Expected the nkey client's account to match 'nats', got %v\", c.acc)\n\t}\n\n\t// Remove user from account and whole account\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\tauthorization {\n\t\tusers = [\n\t\t\t# Ivan\n\t\t\t{nkey : UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH}\n\t\t]\n\t}\n\taccounts {\n\t\tnats.io {\n\t\t\tusers = [\n\t\t\t\t# Derek\n\t\t\t\t{nkey : UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR}\n\t\t\t]\n\t\t}\n\t}\n\t`)\n\n\ts.mu.Lock()\n\tnkeys := s.nkeys\n\tglobalAcc := s.gacc\n\ts.mu.Unlock()\n\n\tif n := len(nkeys); n != 2 {\n\t\tt.Fatalf(\"NKeys map should have 2 users, got %v\", n)\n\t}\n\tderek := nkeys[\"UCNGL4W5QX66CFX6A6DCBVDH5VOHMI7B2UZZU7TXAUQQSI2JPHULCKBR\"]\n\tif derek == nil {\n\t\tt.Fatal(\"NKey for user Derek not found\")\n\t}\n\tif derek.Account == nil || derek.Account.Name != \"nats.io\" {\n\t\tt.Fatalf(\"Invalid account for user Derek: %#v\", derek.Account)\n\t}\n\tivan := nkeys[\"UDPGQVFIWZ7Q5UH4I5E6DBCZULQS6VTVBG6CYBD7JV3G3N2GMQOMNAUH\"]\n\tif ivan == nil {\n\t\tt.Fatal(\"NKey for user Ivan not found\")\n\t}\n\tif ivan.Account != globalAcc {\n\t\tt.Fatalf(\"Invalid account for user Ivan: %#v\", ivan.Account)\n\t}\n\tif _, err := s.LookupAccount(\"synadia\"); err == nil {\n\t\tt.Fatal(\"Account Synadia should have been removed\")\n\t}\n}\n\nfunc TestConfigReloadAccountStreamsImportExport(t *testing.T) {\n\ttemplate := `\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tsynadia {\n\t\t\tusers [{user: derek, password: foo}]\n\t\t\texports = [\n\t\t\t\t{stream: \"private.>\", accounts: [nats.io]}\n\t\t\t\t{stream: %s}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers [\n\t\t\t\t{user: ivan, password: bar, permissions: {subscribe: {deny: %s}}}\n\t\t\t]\n\t\t\timports = [\n\t\t\t\t{stream: {account: \"synadia\", subject: %s}}\n\t\t\t\t{stream: {account: \"synadia\", subject: \"private.natsio.*\"}, prefix: %s}\n\t\t\t]\n\t\t}\n\t}\n\tno_sys_acc: true\n\t`\n\t// synadia account exports \"private.>\" to nats.io\n\t// synadia account exports \"foo.*\"\n\t// user ivan denies subscription on \"xxx\"\n\t// nats.io account imports \"foo.*\" from synadia\n\t// nats.io account imports \"private.natsio.*\" from synadia with prefix \"ivan\"\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, `\"foo.*\"`, `\"xxx\"`, `\"foo.*\"`, `\"ivan\"`)))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tderek, err := nats.Connect(fmt.Sprintf(\"nats://derek:foo@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer derek.Close()\n\tcheckClientsCount(t, s, 1)\n\n\tch := make(chan bool, 1)\n\tivan, err := nats.Connect(fmt.Sprintf(\"nats://ivan:bar@%s:%d\", opts.Host, opts.Port),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tif strings.Contains(strings.ToLower(err.Error()), \"permissions violation\") {\n\t\t\t\tch <- true\n\t\t\t}\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ivan.Close()\n\tcheckClientsCount(t, s, 2)\n\n\tsubscribe := func(t *testing.T, nc *nats.Conn, subj string) *nats.Subscription {\n\t\tt.Helper()\n\t\ts, err := nc.SubscribeSync(subj)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\treturn s\n\t}\n\n\tsubFooBar := subscribe(t, ivan, \"foo.bar\")\n\tsubFooBaz := subscribe(t, ivan, \"foo.baz\")\n\tsubFooBat := subscribe(t, ivan, \"foo.bat\")\n\tsubPriv := subscribe(t, ivan, \"ivan.private.natsio.*\")\n\tivan.Flush()\n\n\tpublish := func(t *testing.T, nc *nats.Conn, subj string) {\n\t\tt.Helper()\n\t\tif err := nc.Publish(subj, []byte(\"hello\")); err != nil {\n\t\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t\t}\n\t}\n\n\tnextMsg := func(t *testing.T, sub *nats.Subscription, expected bool) {\n\t\tt.Helper()\n\t\tdur := 100 * time.Millisecond\n\t\tif expected {\n\t\t\tdur = time.Second\n\t\t}\n\t\t_, err := sub.NextMsg(dur)\n\t\tif expected && err != nil {\n\t\t\tt.Fatalf(\"Expected a message on %s, got %v\", sub.Subject, err)\n\t\t} else if !expected && err != nats.ErrTimeout {\n\t\t\tt.Fatalf(\"Expected a timeout on %s, got %v\", sub.Subject, err)\n\t\t}\n\t}\n\n\t// Checks the derek's user sublist for presence of given subject\n\t// interest. Boolean says if interest is expected or not.\n\tcheckSublist := func(t *testing.T, subject string, shouldBeThere bool) {\n\t\tt.Helper()\n\t\tdcli := s.getClient(1)\n\t\tdcli.mu.Lock()\n\t\tr := dcli.acc.sl.Match(subject)\n\t\tdcli.mu.Unlock()\n\t\tif shouldBeThere && len(r.psubs) != 1 {\n\t\t\tt.Fatalf(\"%s should have 1 match in derek's sublist, got %v\", subject, len(r.psubs))\n\t\t} else if !shouldBeThere && len(r.psubs) > 0 {\n\t\t\tt.Fatalf(\"%s should not be in derek's sublist\", subject)\n\t\t}\n\t}\n\n\t// Publish on all subjects and the subs should receive and\n\t// subjects should be in sublist\n\tpublish(t, derek, \"foo.bar\")\n\tnextMsg(t, subFooBar, true)\n\tcheckSublist(t, \"foo.bar\", true)\n\n\tpublish(t, derek, \"foo.baz\")\n\tnextMsg(t, subFooBaz, true)\n\tcheckSublist(t, \"foo.baz\", true)\n\n\tpublish(t, derek, \"foo.bat\")\n\tnextMsg(t, subFooBat, true)\n\tcheckSublist(t, \"foo.bat\", true)\n\n\tpublish(t, derek, \"private.natsio.foo\")\n\tnextMsg(t, subPriv, true)\n\tcheckSublist(t, \"private.natsio.foo\", true)\n\n\t// Also make sure that intra-account subscription works OK\n\tivanSub := subscribe(t, ivan, \"ivan.sub\")\n\tpublish(t, ivan, \"ivan.sub\")\n\tnextMsg(t, ivanSub, true)\n\tderekSub := subscribe(t, derek, \"derek.sub\")\n\tpublish(t, derek, \"derek.sub\")\n\tnextMsg(t, derekSub, true)\n\n\t// synadia account exports \"private.>\" to nats.io\n\t// synadia account exports \"foo.*\"\n\t// user ivan denies subscription on \"foo.bat\"\n\t// nats.io account imports \"foo.baz\" from synadia\n\t// nats.io account imports \"private.natsio.*\" from synadia with prefix \"yyyy\"\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template, `\"foo.*\"`, `\"foo.bat\"`, `\"foo.baz\"`, `\"yyyy\"`))\n\n\t// Sub on foo.bar should now fail to receive\n\tpublish(t, derek, \"foo.bar\")\n\tnextMsg(t, subFooBar, false)\n\tcheckSublist(t, \"foo.bar\", false)\n\t// But foo.baz should be received\n\tpublish(t, derek, \"foo.baz\")\n\tnextMsg(t, subFooBaz, true)\n\tcheckSublist(t, \"foo.baz\", true)\n\t// Due to permissions, foo.bat should not\n\tpublish(t, derek, \"foo.bat\")\n\tnextMsg(t, subFooBat, false)\n\tcheckSublist(t, \"foo.bat\", false)\n\t// Prefix changed, so should not be received\n\tpublish(t, derek, \"private.natsio.foo\")\n\tnextMsg(t, subPriv, false)\n\tcheckSublist(t, \"private.natsio.foo\", false)\n\n\t// Wait for client notification of permissions error\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"Did not the permissions error\")\n\t}\n\n\tpublish(t, ivan, \"ivan.sub\")\n\tnextMsg(t, ivanSub, true)\n\tpublish(t, derek, \"derek.sub\")\n\tnextMsg(t, derekSub, true)\n\n\t// Change export so that foo.* is no longer exported\n\t// synadia account exports \"private.>\" to nats.io\n\t// synadia account exports \"xxx\"\n\t// user ivan denies subscription on \"foo.bat\"\n\t// nats.io account imports \"xxx\" from synadia\n\t// nats.io account imports \"private.natsio.*\" from synadia with prefix \"ivan\"\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template, `\"xxx\"`, `\"foo.bat\"`, `\"xxx\"`, `\"ivan\"`))\n\n\tpublish(t, derek, \"foo.bar\")\n\tnextMsg(t, subFooBar, false)\n\tcheckSublist(t, \"foo.bar\", false)\n\n\tpublish(t, derek, \"foo.baz\")\n\tnextMsg(t, subFooBaz, false)\n\tcheckSublist(t, \"foo.baz\", false)\n\n\tpublish(t, derek, \"foo.bat\")\n\tnextMsg(t, subFooBat, false)\n\tcheckSublist(t, \"foo.bat\", false)\n\n\t// Prefix changed back, so should receive\n\tpublish(t, derek, \"private.natsio.foo\")\n\tnextMsg(t, subPriv, true)\n\tcheckSublist(t, \"private.natsio.foo\", true)\n\n\tpublish(t, ivan, \"ivan.sub\")\n\tnextMsg(t, ivanSub, true)\n\tpublish(t, derek, \"derek.sub\")\n\tnextMsg(t, derekSub, true)\n}\n\nfunc TestConfigReloadAccountServicesImportExport(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tsynadia {\n\t\t\tusers [{user: derek, password: foo}]\n\t\t\texports = [\n\t\t\t\t{service: \"pub.request\"}\n\t\t\t\t{service: \"pub.special.request\", accounts: [nats.io]}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers [{user: ivan, password: bar}]\n\t\t\timports = [\n\t\t\t\t{service: {account: \"synadia\", subject: \"pub.special.request\"}, to: \"foo\"}\n\t\t\t\t{service: {account: \"synadia\", subject: \"pub.request\"}, to: \"bar\"}\n\t\t\t]\n\t\t}\n\t}\n\tcluster {\n\t\tname: \"abc\"\n\t\tport: -1\n\t}\n\t`))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\topts2 := DefaultOptions()\n\topts2.Cluster.Name = \"abc\"\n\topts2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", opts.Cluster.Port))\n\ts2 := RunServer(opts2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s, s2)\n\n\tderek, err := nats.Connect(fmt.Sprintf(\"nats://derek:foo@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer derek.Close()\n\tcheckClientsCount(t, s, 1)\n\n\tivan, err := nats.Connect(fmt.Sprintf(\"nats://ivan:bar@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ivan.Close()\n\tcheckClientsCount(t, s, 2)\n\n\tif _, err := derek.Subscribe(\"pub.special.request\", func(m *nats.Msg) {\n\t\tderek.Publish(m.Reply, []byte(\"reply1\"))\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif _, err := derek.Subscribe(\"pub.request\", func(m *nats.Msg) {\n\t\tderek.Publish(m.Reply, []byte(\"reply2\"))\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif _, err := derek.Subscribe(\"pub.special.request.new\", func(m *nats.Msg) {\n\t\tderek.Publish(m.Reply, []byte(\"reply3\"))\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\t// Also create one that will be used for intra-account communication\n\tif _, err := derek.Subscribe(\"derek.sub\", func(m *nats.Msg) {\n\t\tderek.Publish(m.Reply, []byte(\"private\"))\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tderek.Flush()\n\n\t// Create an intra-account sub for ivan too\n\tif _, err := ivan.Subscribe(\"ivan.sub\", func(m *nats.Msg) {\n\t\tivan.Publish(m.Reply, []byte(\"private\"))\n\t}); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\t// This subscription is just to make sure that we can update\n\t// route map without locking issues during reload.\n\tnatsSubSync(t, ivan, \"bar\")\n\n\treq := func(t *testing.T, nc *nats.Conn, subj string, reply string) {\n\t\tt.Helper()\n\t\tvar timeout time.Duration\n\t\tif reply != \"\" {\n\t\t\ttimeout = time.Second\n\t\t} else {\n\t\t\ttimeout = 100 * time.Millisecond\n\t\t}\n\t\tmsg, err := nc.Request(subj, []byte(\"request\"), timeout)\n\t\tif reply != \"\" {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Expected reply %s on subject %s, got %v\", reply, subj, err)\n\t\t\t}\n\t\t\tif string(msg.Data) != reply {\n\t\t\t\tt.Fatalf(\"Expected reply %s on subject %s, got %s\", reply, subj, msg.Data)\n\t\t\t}\n\t\t} else if err != nats.ErrTimeout && err != nats.ErrNoResponders {\n\t\t\tt.Fatalf(\"Expected timeout on subject %s, got %v\", subj, err)\n\t\t}\n\t}\n\n\treq(t, ivan, \"foo\", \"reply1\")\n\treq(t, ivan, \"bar\", \"reply2\")\n\t// This not exported/imported, so should timeout\n\treq(t, ivan, \"baz\", \"\")\n\n\t// Check intra-account communication\n\treq(t, ivan, \"ivan.sub\", \"private\")\n\treq(t, derek, \"derek.sub\", \"private\")\n\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tsynadia {\n\t\t\tusers [{user: derek, password: foo}]\n\t\t\texports = [\n\t\t\t\t{service: \"pub.request\"}\n\t\t\t\t{service: \"pub.special.request\", accounts: [nats.io]}\n\t\t\t\t{service: \"pub.special.request.new\", accounts: [nats.io]}\n\t\t\t]\n\t\t}\n\t\tnats.io {\n\t\t\tusers [{user: ivan, password: bar}]\n\t\t\timports = [\n\t\t\t\t{service: {account: \"synadia\", subject: \"pub.special.request\"}, to: \"foo\"}\n\t\t\t\t{service: {account: \"synadia\", subject: \"pub.special.request.new\"}, to: \"baz\"}\n\t\t\t]\n\t\t}\n\t}\n\tcluster {\n\t\tname: \"abc\"\n\t\tport: -1\n\t}\n\t`)\n\t// This still should work\n\treq(t, ivan, \"foo\", \"reply1\")\n\t// This should not\n\treq(t, ivan, \"bar\", \"\")\n\t// This now should work\n\treq(t, ivan, \"baz\", \"reply3\")\n\n\t// Check intra-account communication\n\treq(t, ivan, \"ivan.sub\", \"private\")\n\treq(t, derek, \"derek.sub\", \"private\")\n}\n\n// As of now, config reload does not support changes for gateways.\n// However, ensure that if a gateway is defined, one can still\n// do reload as long as we don't change the gateway spec.\nfunc TestConfigReloadNotPreventedByGateways(t *testing.T) {\n\tconfTemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\t%s\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/server.pem\"\n\t\t\t\tkey_file: \"configs/certs/key.pem\"\n\t\t\t\ttimeout: %s\n\t\t\t}\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"B\"\n\t\t\t\t\turl: \"nats://localhost:8888\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, \"\", \"5\")))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Cause reload with adding a param that is supported\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(confTemplate, \"max_payload: 100000\", \"5\"))\n\n\t// Now update gateway, should fail to reload.\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, \"max_payload: 100000\", \"3\")))\n\tif err := s.Reload(); err == nil || !strings.Contains(err.Error(), \"not supported for Gateway\") {\n\t\tt.Fatalf(\"Expected Reload to return a not supported error, got %v\", err)\n\t}\n}\n\nfunc TestConfigReloadBoolFlags(t *testing.T) {\n\tdefer func() { FlagSnapshot = nil }()\n\n\tlogfile := filepath.Join(t.TempDir(), \"logtime.log\")\n\ttemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tlogfile: \"%s\"\n\t\t%s\n\t`\n\n\tvar opts *Options\n\tvar err error\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcontent string\n\t\tcmdLine []string\n\t\texpected bool\n\t\tval func() bool\n\t}{\n\t\t// Logtime\n\t\t{\n\t\t\t\"logtime_not_in_config_no_override\",\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_not_in_config_override_short_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-T\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_not_in_config_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-logtime\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_false_in_config_no_override\",\n\t\t\t\"logtime: false\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_false_in_config_override_short_true\",\n\t\t\t\"logtime: false\",\n\t\t\t[]string{\"-T\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_false_in_config_override_true\",\n\t\t\t\"logtime: false\",\n\t\t\t[]string{\"-logtime\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_true_in_config_no_override\",\n\t\t\t\"logtime: true\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_true_in_config_override_short_false\",\n\t\t\t\"logtime: true\",\n\t\t\t[]string{\"-T=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t{\n\t\t\t\"logtime_true_in_config_override_false\",\n\t\t\t\"logtime: true\",\n\t\t\t[]string{\"-logtime=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Logtime },\n\t\t},\n\t\t// Debug\n\t\t{\n\t\t\t\"debug_not_in_config_no_override\",\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_not_in_config_override_short_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-D\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_not_in_config_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-debug\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_false_in_config_no_override\",\n\t\t\t\"debug: false\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_false_in_config_override_short_true\",\n\t\t\t\"debug: false\",\n\t\t\t[]string{\"-D\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_false_in_config_override_true\",\n\t\t\t\"debug: false\",\n\t\t\t[]string{\"-debug\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_true_in_config_no_override\",\n\t\t\t\"debug: true\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_true_in_config_override_short_false\",\n\t\t\t\"debug: true\",\n\t\t\t[]string{\"-D=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t{\n\t\t\t\"debug_true_in_config_override_false\",\n\t\t\t\"debug: true\",\n\t\t\t[]string{\"-debug=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Debug },\n\t\t},\n\t\t// Trace\n\t\t{\n\t\t\t\"trace_not_in_config_no_override\",\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_not_in_config_override_short_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-V\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_not_in_config_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-trace\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_false_in_config_no_override\",\n\t\t\t\"trace: false\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_false_in_config_override_short_true\",\n\t\t\t\"trace: false\",\n\t\t\t[]string{\"-V\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_false_in_config_override_true\",\n\t\t\t\"trace: false\",\n\t\t\t[]string{\"-trace\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_true_in_config_no_override\",\n\t\t\t\"trace: true\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_true_in_config_override_short_false\",\n\t\t\t\"trace: true\",\n\t\t\t[]string{\"-V=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_true_in_config_override_false\",\n\t\t\t\"trace: true\",\n\t\t\t[]string{\"-trace=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Trace },\n\t\t},\n\t\t// Syslog\n\t\t{\n\t\t\t\"syslog_not_in_config_no_override\",\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_not_in_config_override_short_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-s\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_not_in_config_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-syslog\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_false_in_config_no_override\",\n\t\t\t\"syslog: false\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_false_in_config_override_short_true\",\n\t\t\t\"syslog: false\",\n\t\t\t[]string{\"-s\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_false_in_config_override_true\",\n\t\t\t\"syslog: false\",\n\t\t\t[]string{\"-syslog\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_true_in_config_no_override\",\n\t\t\t\"syslog: true\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_true_in_config_override_short_false\",\n\t\t\t\"syslog: true\",\n\t\t\t[]string{\"-s=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t{\n\t\t\t\"syslog_true_in_config_override_false\",\n\t\t\t\"syslog: true\",\n\t\t\t[]string{\"-syslog=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t// Cluster.NoAdvertise\n\t\t{\n\t\t\t\"cluster_no_advertise_not_in_config_no_override\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t}`,\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Cluster.NoAdvertise },\n\t\t},\n\t\t{\n\t\t\t\"cluster_no_advertise_not_in_config_override_true\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t}`,\n\t\t\t[]string{\"-no_advertise\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Cluster.NoAdvertise },\n\t\t},\n\t\t{\n\t\t\t\"cluster_no_advertise_false_in_config_no_override\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t\tno_advertise: false\n\t\t\t}`,\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Cluster.NoAdvertise },\n\t\t},\n\t\t{\n\t\t\t\"cluster_no_advertise_false_in_config_override_true\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t\tno_advertise: false\n\t\t\t}`,\n\t\t\t[]string{\"-no_advertise\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Cluster.NoAdvertise },\n\t\t},\n\t\t{\n\t\t\t\"cluster_no_advertise_true_in_config_no_override\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t\tno_advertise: true\n\t\t\t}`,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Cluster.NoAdvertise },\n\t\t},\n\t\t{\n\t\t\t\"cluster_no_advertise_true_in_config_override_false\",\n\t\t\t`cluster {\n\t\t\t\tport: -1\n\t\t\t\tno_advertise: true\n\t\t\t}`,\n\t\t\t[]string{\"-no_advertise=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Syslog },\n\t\t},\n\t\t// -DV override\n\t\t{\n\t\t\t\"debug_trace_not_in_config_dv_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"-DV\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug && opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"debug_trace_false_in_config_dv_override_true\",\n\t\t\t`debug: false\n\t\t trace: false\n\t\t\t`,\n\t\t\t[]string{\"-DV\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Debug && opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"debug_trace_true_in_config_dv_override_false\",\n\t\t\t`debug: true\n\t\t trace: true\n\t\t\t`,\n\t\t\t[]string{\"-DV=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.Debug && opts.Trace },\n\t\t},\n\t\t{\n\t\t\t\"trace_verbose_true_in_config_override_true\",\n\t\t\t`trace_verbose: true\n\t\t\t`,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.Trace && opts.TraceVerbose },\n\t\t},\n\t\t{\n\t\t\t\"trace_verbose_true_in_config_override_false\",\n\t\t\t`trace_verbose: true\n\t\t\t`,\n\t\t\t[]string{\"--VV=false\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return !opts.TraceVerbose },\n\t\t},\n\t\t{\n\t\t\t\"trace_verbose_true_in_config_override_false\",\n\t\t\t`trace_verbose: false\n\t\t\t`,\n\t\t\t[]string{\"--VV=true\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.TraceVerbose },\n\t\t},\n\t\t// --js override\n\t\t{\n\t\t\t\"jetstream_not_in_config_no_override\",\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t\t{\n\t\t\t\"jetstream_not_in_config_override_true\",\n\t\t\t\"\",\n\t\t\t[]string{\"--js\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t\t{\n\t\t\t\"jetstream_false_in_config_no_override\",\n\t\t\t\"jetstream: false\",\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t\t{\n\t\t\t\"jetstream_false_in_config_override_true\",\n\t\t\t\"jetstream: false\",\n\t\t\t[]string{\"--js\"},\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t\t{\n\t\t\t\"jetstream_true_in_config_no_override\",\n\t\t\t\"jetstream: true\",\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t\t{\n\t\t\t\"jetstream_true_in_config_override_false\",\n\t\t\t\"jetstream: true\",\n\t\t\t[]string{\"--js=false\"},\n\t\t\tfalse,\n\t\t\tfunc() bool { return opts.JetStream },\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := fmt.Sprintf(template, logfile, test.content)\n\t\t\tconf := createConfFile(t, []byte(content))\n\n\t\t\tfs := flag.NewFlagSet(\"test\", flag.ContinueOnError)\n\t\t\tvar args []string\n\t\t\targs = append(args, \"-c\", conf)\n\t\t\tif test.cmdLine != nil {\n\t\t\t\targs = append(args, test.cmdLine...)\n\t\t\t}\n\t\t\topts, err = ConfigureOptions(fs, args, nil, nil, nil)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error processing config: %v\", err)\n\t\t\t}\n\t\t\topts.NoSigs = true\n\t\t\ts := RunServer(opts)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tif test.val() != test.expected {\n\t\t\t\tt.Fatalf(\"Expected to be set to %v, got %v\", test.expected, test.val())\n\t\t\t}\n\t\t\t// Do a config reload with a modified config file so that s.Reload()\n\t\t\t// actually does something (otherwise it would not because config\n\t\t\t// digest would not have changed). We could alternatively change\n\t\t\t// s.opts.configDigest to the empty string.\n\t\t\treloadUpdateConfig(t, s, conf, content+`\n\t\t\t\tmax_connections: 1000\n\t\t\t`)\n\t\t\t// Have `opts` now point to the new options after the Reload()\n\t\t\topts = s.getOpts()\n\t\t\tif test.val() != test.expected {\n\t\t\t\tt.Fatalf(\"Expected to be set to %v, got %v\", test.expected, test.val())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigReloadMaxControlLineWithClients(t *testing.T) {\n\tserver, opts, config := runReloadServerWithConfig(t, \"./configs/reload/basic.conf\")\n\tdefer server.Shutdown()\n\n\t// Ensure we can connect as a sanity check.\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, server.Addr().(*net.TCPAddr).Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Now grab server's internal client that matches.\n\tcid, _ := nc.GetClientID()\n\tc := server.getClient(cid)\n\tif c == nil {\n\t\tt.Fatalf(\"Could not look up internal client\")\n\t}\n\n\t// Check that we have the correct mcl snapshotted into the connected client.\n\tgetMcl := func(c *client) int32 {\n\t\tc.mu.Lock()\n\t\tdefer c.mu.Unlock()\n\t\treturn c.mcl\n\t}\n\tif mcl := getMcl(c); mcl != opts.MaxControlLine {\n\t\tt.Fatalf(\"Expected snapshot in client for mcl to be same as opts.MaxControlLine, got %d vs %d\",\n\t\t\tmcl, opts.MaxControlLine)\n\t}\n\n\tchangeCurrentConfigContentWithNewContent(t, config, []byte(\"listen: 127.0.0.1:-1; max_control_line: 222\"))\n\tif err := server.Reload(); err != nil {\n\t\tt.Fatalf(\"Expected Reload to succeed, got %v\", err)\n\t}\n\n\t// Refresh properly.\n\topts = server.getOpts()\n\n\tif mcl := getMcl(c); mcl != opts.MaxControlLine {\n\t\tt.Fatalf(\"Expected snapshot in client for mcl to be same as new opts.MaxControlLine, got %d vs %d\",\n\t\t\tmcl, opts.MaxControlLine)\n\t}\n}\n\ntype testCustomAuth struct{}\n\nfunc (ca *testCustomAuth) Check(c ClientAuthentication) bool { return true }\n\nfunc TestConfigReloadIgnoreCustomAuth(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t`))\n\topts := LoadConfig(conf)\n\n\tca := &testCustomAuth{}\n\topts.CustomClientAuthentication = ca\n\topts.CustomRouterAuthentication = ca\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\tif s.getOpts().CustomClientAuthentication != ca || s.getOpts().CustomRouterAuthentication != ca {\n\t\tt.Fatalf(\"Custom auth missing\")\n\t}\n}\n\nfunc TestConfigReloadLeafNodeRandomPort(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\tlnPortBefore := s.leafNodeListener.Addr().(*net.TCPAddr).Port\n\ts.mu.Unlock()\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\ts.mu.Lock()\n\tlnPortAfter := s.leafNodeListener.Addr().(*net.TCPAddr).Port\n\ts.mu.Unlock()\n\n\tif lnPortBefore != lnPortAfter {\n\t\tt.Fatalf(\"Expected leafnodes listen port to be same, was %v is now %v\", lnPortBefore, lnPortAfter)\n\t}\n}\n\nfunc TestConfigReloadLeafNodeWithTLS(t *testing.T) {\n\ttemplate := `\n\t\tport: -1\n\t\t%s\n\t\tleaf {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls: {\n\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\t\ttimeout: 3\n\t\t\t}\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tu, err := url.Parse(fmt.Sprintf(\"nats://localhost:%d\", o1.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating url: %v\", err)\n\t}\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"%s\"\n\t\t\t\t\ttls {\n\t\t\t\t\t\tca_file: \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\t\t\t\tcert_file: \"../test/configs/certs/tlsauth/client.pem\"\n\t\t\t\t\t\tkey_file: \"../test/configs/certs/tlsauth/client-key.pem\"\n\t\t\t\t\t\ttimeout: 2\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, u.String())))\n\to2, err := ProcessConfigFile(conf2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\to2.NoLog, o2.NoSigs = true, true\n\to2.LeafNode.resolver = &testLoopbackResolver{}\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckFor(t, 3*time.Second, 15*time.Millisecond, func() error {\n\t\tif n := s1.NumLeafNodes(); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 leaf node, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\tchangeCurrentConfigContentWithNewContent(t, conf1, []byte(fmt.Sprintf(template, \"debug: false\")))\n\n\tif err := s1.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n}\n\nfunc TestConfigReloadLeafNodeWithRemotesNoChanges(t *testing.T) {\n\ttemplate := `\n port: -1\n cluster {\n port: -1\n name: \"%s\"\n }\n leaf {\n remotes [\n {\n urls: [\n \"nats://127.0.0.1:1234\",\n \"nats://127.0.0.1:1235\",\n \"nats://127.0.0.1:1236\",\n \"nats://127.0.0.1:1237\",\n \"nats://127.0.0.1:1238\",\n \"nats://127.0.0.1:1239\",\n ]\n }\n ]\n }\n\t`\n\tconfig := fmt.Sprintf(template, \"A\")\n\tconf := createConfFile(t, []byte(config))\n\to, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\to.NoLog, o.NoSigs = true, false\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tconfig = fmt.Sprintf(template, \"B\")\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(config))\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n}\n\nfunc TestConfigReloadAndVarz(t *testing.T) {\n\ttemplate := `\n\t\tport: -1\n\t\t%s\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\tinitConfigTime := s.configTime\n\ts.mu.Unlock()\n\n\tv, _ := s.Varz(nil)\n\tif !v.ConfigLoadTime.Equal(initConfigTime) {\n\t\tt.Fatalf(\"ConfigLoadTime should be %v, got %v\", initConfigTime, v.ConfigLoadTime)\n\t}\n\tif v.MaxConn != DEFAULT_MAX_CONNECTIONS {\n\t\tt.Fatalf(\"MaxConn should be %v, got %v\", DEFAULT_MAX_CONNECTIONS, v.MaxConn)\n\t}\n\tgot := v.ConfigDigest\n\texpected := \"sha256:7ea4cc5c6864139d814ce940a40ee6546b7ac5285eec3c390a4ce11e34dc1102\"\n\tif got != expected {\n\t\tt.Fatalf(\"got: %v, expected: %v\", got, expected)\n\t}\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, \"max_connections: 10\")))\n\n\t// Make sure we wait a bit so config load time has a chance to change.\n\ttime.Sleep(15 * time.Millisecond)\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\tv, _ = s.Varz(nil)\n\tif v.ConfigLoadTime.Equal(initConfigTime) {\n\t\tt.Fatalf(\"ConfigLoadTime should be different from %v\", initConfigTime)\n\t}\n\tif v.MaxConn != 10 {\n\t\tt.Fatalf(\"MaxConn should be 10, got %v\", v.MaxConn)\n\t}\n\tgot = v.ConfigDigest\n\texpected = \"sha256:508a26309068f62c3022aa8951e712ed8ff5dd6f2360f727ad8242a2a233176e\"\n\tif got != expected {\n\t\tt.Fatalf(\"got: %v, expected: %v\", got, expected)\n\t}\n}\n\nfunc TestConfigReloadConnectErrReports(t *testing.T) {\n\ttemplate := `\n\t\tport: -1\n\t\t%s\n\t\t%s\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template, \"\", \"\")))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\topts := s.getOpts()\n\tif cer := opts.ConnectErrorReports; cer != DEFAULT_CONNECT_ERROR_REPORTS {\n\t\tt.Fatalf(\"Expected ConnectErrorReports to be %v, got %v\", DEFAULT_CONNECT_ERROR_REPORTS, cer)\n\t}\n\tif rer := opts.ReconnectErrorReports; rer != DEFAULT_RECONNECT_ERROR_REPORTS {\n\t\tt.Fatalf(\"Expected ReconnectErrorReports to be %v, got %v\", DEFAULT_RECONNECT_ERROR_REPORTS, rer)\n\t}\n\n\tchangeCurrentConfigContentWithNewContent(t, conf,\n\t\t[]byte(fmt.Sprintf(template, \"connect_error_reports: 2\", \"reconnect_error_reports: 3\")))\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\topts = s.getOpts()\n\tif cer := opts.ConnectErrorReports; cer != 2 {\n\t\tt.Fatalf(\"Expected ConnectErrorReports to be %v, got %v\", 2, cer)\n\t}\n\tif rer := opts.ReconnectErrorReports; rer != 3 {\n\t\tt.Fatalf(\"Expected ReconnectErrorReports to be %v, got %v\", 3, rer)\n\t}\n}\n\nfunc TestConfigReloadAuthDoesNotBreakRouteInterest(t *testing.T) {\n\ts, opts := RunServerWithConfig(\"./configs/seed_tls.conf\")\n\tdefer s.Shutdown()\n\n\t// Create client and sub interest on seed server.\n\turlSeed := fmt.Sprintf(\"nats://%s:%d/\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(urlSeed)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tch := make(chan bool)\n\tnc.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc.Flush()\n\n\t// Use this to check for message.\n\tcheckForMsg := func() {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t\t}\n\t}\n\n\t// Create second server and form cluster. We will send from here.\n\turlRoute := fmt.Sprintf(\"nats://%s:%d\", opts.Cluster.Host, opts.Cluster.Port)\n\toptsA := nextServerOpts(opts)\n\toptsA.Routes = RoutesFromStr(urlRoute)\n\n\tsa := RunServer(optsA)\n\tdefer sa.Shutdown()\n\n\tcheckClusterFormed(t, s, sa)\n\tcheckSubInterest(t, sa, globalAccountName, \"foo\", time.Second)\n\n\t// Create second client and send message from this one. Interest should be here.\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\tnc2, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Check that we can send messages.\n\tnc2.Publish(\"foo\", nil)\n\tcheckForMsg()\n\n\t// Now shutdown nc2 and srvA.\n\tnc2.Close()\n\tsa.Shutdown()\n\n\t// Now force reload on seed server of auth.\n\ts.reloadAuthorization()\n\n\t// Restart both server A and client 2.\n\tsa = RunServer(optsA)\n\tdefer sa.Shutdown()\n\n\tcheckClusterFormed(t, s, sa)\n\tcheckSubInterest(t, sa, globalAccountName, \"foo\", time.Second)\n\n\tnc2, err = nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Check that we can still send messages.\n\tnc2.Publish(\"foo\", nil)\n\tcheckForMsg()\n}\n\nfunc TestConfigReloadAccountResolverTLSConfig(t *testing.T) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tajwt, err := nac.Encode(kp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tpub, _ := kp.PublicKey()\n\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t}\n\ttlsConfig, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\tts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Write([]byte(ajwt))\n\t}))\n\tts.TLS = tlsConfig\n\tts.StartTLS()\n\tdefer ts.Close()\n\t// Set a dummy logger to prevent tls bad certificate output to stderr.\n\tts.Config.ErrorLog = log.New(&bytes.Buffer{}, \"\", 0)\n\n\tconfTemplate := `\n\t\t\t\tlisten: -1\n\t\t\t\ttrusted_keys: %s\n\t\t\t\tresolver: URL(\"%s/ngs/v1/accounts/jwt/\")\n\t\t\t\t%s\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, `\n\t\tresolver_tls {\n\t\t\tcert_file: \"../test/configs/certs/client-cert.pem\"\n\t\t\tkey_file: \"../test/configs/certs/client-key.pem\"\n\t\t\tca_file: \"../test/configs/certs/ca.pem\"\n\t\t}\n\t`)))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, \"\")))\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\tif _, err := s.LookupAccount(apub); err == nil {\n\t\tt.Fatal(\"Expected error during lookup, did not get one\")\n\t}\n\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(confTemplate, pub, ts.URL, `\n\t\tresolver_tls {\n\t\t\tinsecure: true\n\t\t}\n\t`)))\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t}\n\n\tacc, err := s.LookupAccount(apub)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during lookup: %v\", err)\n\t}\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to receive an account\")\n\t}\n\tif acc.Name != apub {\n\t\tt.Fatalf(\"Account name did not match claim key\")\n\t}\n}\n\nfunc TestConfigReloadLogging(t *testing.T) {\n\t// This test basically starts a server and causes it's configuration to be reloaded 3 times.\n\t// Each time, a new log file is created and trace levels are turned, off - on - off.\n\n\t// At the end of the test, all 3 log files are inspected for certain traces.\n\tcountMatches := func(log []byte, stmts ...string) int {\n\t\tmatchCnt := 0\n\t\tfor _, stmt := range stmts {\n\t\t\tif strings.Contains(string(log), stmt) {\n\t\t\t\tmatchCnt++\n\t\t\t}\n\t\t}\n\t\treturn matchCnt\n\t}\n\n\ttraces := []string{\"[TRC]\", \"[DBG]\", \"SYSTEM\", \"MSG_PAYLOAD\", \"$SYS.SERVER.ACCOUNT\"}\n\n\tdidTrace := func(log []byte) bool {\n\t\treturn countMatches(log, \"[INF] Reloaded server configuration\") == 1\n\t}\n\n\ttracingAbsent := func(log []byte) bool {\n\t\treturn countMatches(log, traces...) == 0 && didTrace(log)\n\t}\n\n\ttracingPresent := func(log []byte) bool {\n\t\treturn len(traces) == countMatches(log, traces...) && didTrace(log)\n\t}\n\n\tcheck := func(filename string, valid func([]byte) bool) {\n\t\tt.Helper()\n\t\tlog, err := os.ReadFile(filename)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading log file %s: %v\\n\", filename, err)\n\t\t}\n\t\tif !valid(log) {\n\t\t\tt.Fatalf(\"%s is not valid: %s\", filename, log)\n\t\t}\n\t\t//t.Logf(\"%s contains: %s\\n\", filename, log)\n\t}\n\n\t// common configuration setting up system accounts. trace_verbose needs this to cause traces\n\tcommonCfg := `\n\t\tport: -1\n\t\tsystem_account: sys\n\t\taccounts {\n\t\t sys { users = [ {user: sys, pass: \"\" } ] }\n\t\t nats.io: { users = [ { user : bar, pass: \"pwd\" } ] }\n\t\t}\n\t`\n\n\tconf := createConfFile(t, []byte(commonCfg))\n\n\tdefer removeFile(t, \"off-pre.log\")\n\tdefer removeFile(t, \"on.log\")\n\tdefer removeFile(t, \"off-post.log\")\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\treload := func(change string) {\n\t\tt.Helper()\n\t\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(commonCfg+`\n\t\t\t`+change+`\n\t\t`))\n\n\t\tif err := s.Reload(); err != nil {\n\t\t\tt.Fatalf(\"Error during reload: %v\", err)\n\t\t}\n\t}\n\n\ttraffic := func(cnt int) {\n\t\tt.Helper()\n\t\t// Create client and sub interest on server and create traffic\n\t\turlSeed := fmt.Sprintf(\"nats://bar:pwd@%s:%d/\", opts.Host, opts.Port)\n\t\tnc, err := nats.Connect(urlSeed)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tmsgs := make(chan *nats.Msg, 1)\n\t\tdefer close(msgs)\n\n\t\tsub, err := nc.ChanSubscribe(\"foo\", msgs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating subscriber: %v\\n\", err)\n\t\t}\n\n\t\tnc.Flush()\n\n\t\tfor i := 0; i < cnt; i++ {\n\t\t\tif err := nc.Publish(\"foo\", []byte(\"bar\")); err == nil {\n\t\t\t\t<-msgs\n\t\t\t}\n\t\t}\n\n\t\tsub.Unsubscribe()\n\t\tnc.Close()\n\t}\n\n\treload(\"log_file: off-pre.log\")\n\n\ttraffic(10) // generate NO trace/debug entries in off-pre.log\n\n\treload(`\n\t\tlog_file: on.log\n\t\tdebug: true\n\t\ttrace_verbose: true\n\t`)\n\n\ttraffic(10) // generate trace/debug entries in on.log\n\n\treload(`\n\t\tlog_file: off-post.log\n\t\tdebug: false\n\t\ttrace_verbose: false\n\t`)\n\n\ttraffic(10) // generate trace/debug entries in off-post.log\n\n\t// check resulting log files for expected content\n\tcheck(\"off-pre.log\", tracingAbsent)\n\tcheck(\"on.log\", tracingPresent)\n\tcheck(\"off-post.log\", tracingAbsent)\n}\n\nfunc TestConfigReloadValidate(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tno_auth_user: a\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{user: \"a\", password: \"a\"},\n\t\t\t\t{user: \"b\", password: \"b\"}\n\t\t\t]\n\t\t}\n\t`))\n\tsrv, _ := RunServerWithConfig(confFileName)\n\tif srv == nil {\n\t\tt.Fatal(\"Server did not start\")\n\t}\n\t// Induce error by removing the user no_auth_user points to\n\tchangeCurrentConfigContentWithNewContent(t, confFileName, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tno_auth_user: a\n\t\tauthorization {\n\t\t\tusers [\n\t\t\t\t{user: \"b\", password: \"b\"}\n\t\t\t]\n\t\t}\n\t`))\n\tif err := srv.Reload(); err == nil {\n\t\tt.Fatal(\"Expected error on reload, got none\")\n\t} else if strings.HasPrefix(err.Error(), \" no_auth_user:\") {\n\t\tt.Logf(\"Expected no_auth_user error, got different one %s\", err)\n\t}\n\tsrv.Shutdown()\n}\n\nfunc TestConfigReloadAccounts(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\tsystem_account: SYS\n\taccounts {\n\t\tSYS {\n\t\t\tusers = [\n\t\t\t\t{user: sys, password: pwd}\n\t\t\t]\n\t\t}\n\t\tACC {\n\t\t\tusers = [\n\t\t\t\t{user: usr, password: pwd}\n\t\t\t]\n\t\t}\n\t\tacc_deleted_after_reload_will_trigger_reload_of_all_accounts {\n\t\t\tusers = [\n\t\t\t\t{user: notused, password: soon}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\turlSys := fmt.Sprintf(\"nats://sys:pwd@%s:%d\", o.Host, o.Port)\n\turlUsr := fmt.Sprintf(\"nats://usr:pwd@%s:%d\", o.Host, o.Port)\n\toldAcci, ok := s.accounts.Load(\"SYS\")\n\tif !ok {\n\t\tt.Fatal(\"No SYS account\")\n\t}\n\toldAcc := oldAcci.(*Account)\n\n\ttestSrvState := func(oldAcc *Account) {\n\t\tt.Helper()\n\t\tsysAcc := s.SystemAccount()\n\t\ts.mu.Lock()\n\t\tdefer s.mu.Unlock()\n\t\tif s.sys == nil || sysAcc == nil {\n\t\t\tt.Fatal(\"Expected sys.account to be non-nil\")\n\t\t}\n\t\tif sysAcc.Name != \"SYS\" {\n\t\t\tt.Fatal(\"Found wrong sys.account\")\n\t\t}\n\t\tif s.opts.SystemAccount != \"SYS\" {\n\t\t\tt.Fatal(\"Found wrong sys.account\")\n\t\t}\n\t\tai, ok := s.accounts.Load(s.opts.SystemAccount)\n\t\tif !ok {\n\t\t\tt.Fatalf(\"System account %q not found in s.accounts map\", s.opts.SystemAccount)\n\t\t}\n\t\tacc := ai.(*Account)\n\t\tif acc != oldAcc {\n\t\t\tt.Fatalf(\"System account pointer was changed during reload, was %p now %p\", oldAcc, acc)\n\t\t}\n\t\tif s.sys.client == nil {\n\t\t\tt.Fatal(\"Expected sys.client to be non-nil\")\n\t\t}\n\t\ts.sys.client.mu.Lock()\n\t\tdefer s.sys.client.mu.Unlock()\n\t\tif s.sys.client.acc.Name != \"SYS\" {\n\t\t\tt.Fatal(\"Found wrong sys.account\")\n\t\t}\n\t\tif s.sys.client.echo {\n\t\t\tt.Fatal(\"Internal clients should always have echo false\")\n\t\t}\n\t\ts.sys.account.mu.Lock()\n\t\tif _, ok := s.sys.account.clients[s.sys.client]; !ok {\n\t\t\ts.sys.account.mu.Unlock()\n\t\t\tt.Fatal(\"internal client not present\")\n\t\t}\n\t\ts.sys.account.mu.Unlock()\n\t}\n\n\t// Below tests use connection names so that they can be checked for.\n\t// The test subscribes to ACC only. This avoids receiving own messages.\n\tsubscribe := func(name string) (*nats.Conn, *nats.Subscription, *nats.Subscription) {\n\t\tt.Helper()\n\t\tc, err := nats.Connect(urlSys, nats.Name(name))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tsubCon, err := c.SubscribeSync(\"$SYS.ACCOUNT.ACC.CONNECT\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe CONNECT: %v\", err)\n\t\t}\n\t\tsubDis, err := c.SubscribeSync(\"$SYS.ACCOUNT.ACC.DISCONNECT\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe DISCONNECT: %v\", err)\n\t\t}\n\t\tc.Flush()\n\t\treturn c, subCon, subDis\n\t}\n\trecv := func(name string, sub *nats.Subscription) {\n\t\tt.Helper()\n\t\tif msg, err := sub.NextMsg(1 * time.Second); err != nil {\n\t\t\tt.Fatalf(\"%s Error on next: %v\", name, err)\n\t\t} else {\n\t\t\tcMsg := ConnectEventMsg{}\n\t\t\tjson.Unmarshal(msg.Data, &cMsg)\n\t\t\tif cMsg.Client.Name != name {\n\t\t\t\tt.Fatalf(\"%s wrong message: %s\", name, string(msg.Data))\n\t\t\t}\n\t\t}\n\t}\n\ttriggerSysEvent := func(name string, subs []*nats.Subscription) {\n\t\tt.Helper()\n\t\tncs1, err := nats.Connect(urlUsr, nats.Name(name))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tncs1.Close()\n\t\tfor _, sub := range subs {\n\t\t\trecv(name, sub)\n\t\t\t// Make sure they are empty.\n\t\t\tif pending, _, _ := sub.Pending(); pending != 0 {\n\t\t\t\tt.Fatalf(\"Expected no pending, got %d for %+v\", pending, sub)\n\t\t\t}\n\t\t}\n\t}\n\n\ttestSrvState(oldAcc)\n\tc1, s1C, s1D := subscribe(\"SYS1\")\n\tdefer c1.Close()\n\tdefer s1C.Unsubscribe()\n\tdefer s1D.Unsubscribe()\n\ttriggerSysEvent(\"BEFORE1\", []*nats.Subscription{s1C, s1D})\n\ttriggerSysEvent(\"BEFORE2\", []*nats.Subscription{s1C, s1D})\n\n\t// Remove account to trigger account reload\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\tsystem_account: SYS\n\taccounts {\n\t\tSYS {\n\t\t\tusers = [\n\t\t\t\t{user: sys, password: pwd}\n\t\t\t]\n\t\t}\n\t\tACC {\n\t\t\tusers = [\n\t\t\t\t{user: usr, password: pwd}\n\t\t\t]\n\t\t}\n\t}\n\t`)\n\n\ttestSrvState(oldAcc)\n\tc2, s2C, s2D := subscribe(\"SYS2\")\n\tdefer c2.Close()\n\tdefer s2C.Unsubscribe()\n\tdefer s2D.Unsubscribe()\n\t// test new and existing subscriptions\n\ttriggerSysEvent(\"AFTER1\", []*nats.Subscription{s1C, s1D, s2C, s2D})\n\ttriggerSysEvent(\"AFTER2\", []*nats.Subscription{s1C, s1D, s2C, s2D})\n}\n\nfunc TestConfigReloadDefaultSystemAccount(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tACC {\n\t\t\tusers = [\n\t\t\t\t{user: usr, password: pwd}\n\t\t\t]\n\t\t}\n\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tsysAcc := s.SystemAccount()\n\tif sysAcc == nil {\n\t\tt.Fatalf(\"Expected system account to be present\")\n\t}\n\tnumSubs := sysAcc.TotalSubs()\n\n\tsname := sysAcc.GetName()\n\ttestInAccounts := func() {\n\t\tt.Helper()\n\t\tvar found bool\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\tacc := v.(*Account)\n\t\t\tif acc.GetName() == sname {\n\t\t\t\tfound = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif !found {\n\t\t\tt.Fatalf(\"System account not found in accounts list\")\n\t\t}\n\t}\n\ttestInAccounts()\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Unexpected error reloading: %v\", err)\n\t}\n\n\tsysAcc = s.SystemAccount()\n\tif sysAcc == nil {\n\t\tt.Fatalf(\"Expected system account to still be present\")\n\t}\n\tif sysAcc.TotalSubs() != numSubs {\n\t\tt.Fatalf(\"Expected %d subs, got %d\", numSubs, sysAcc.TotalSubs())\n\t}\n\ttestInAccounts()\n}\n\nfunc TestConfigReloadAccountMappings(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tACC {\n\t\t\tusers = [{user: usr, password: pwd}]\n\t\t\tmappings = { foo: bar }\n\t\t}\n\t}\n\t`))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tACC {\n\t\t\tusers = [{user: usr, password: pwd}]\n\t\t\tmappings = { foo: baz }\n\t\t}\n\t}\n\t`)\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://usr:pwd@%s:%d\", opts.Host, opts.Port))\n\tdefer nc.Close()\n\n\tfsub, _ := nc.SubscribeSync(\"foo\")\n\tsub, _ := nc.SubscribeSync(\"baz\")\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending := func(sub *nats.Subscription, expected int) {\n\t\tt.Helper()\n\t\tif n, _, _ := sub.Pending(); n != expected {\n\t\t\tt.Fatalf(\"Expected %d msgs for %q, but got %d\", expected, sub.Subject, n)\n\t\t}\n\t}\n\tcheckPending(fsub, 0)\n\tcheckPending(sub, 1)\n\n\t// Drain it off\n\tif _, err := sub.NextMsg(2 * time.Second); err != nil {\n\t\tt.Fatalf(\"Error receiving msg: %v\", err)\n\t}\n\n\treloadUpdateConfig(t, s, conf, `\n\tlisten: \"127.0.0.1:-1\"\n\taccounts {\n\t\tACC {\n\t\t\tusers = [{user: usr, password: pwd}]\n\t\t}\n\t}\n\t`)\n\n\tnc.Publish(\"foo\", nil)\n\tnc.Flush()\n\n\tcheckPending(fsub, 1)\n\tcheckPending(sub, 0)\n}\n\nfunc TestConfigReloadWithSysAccountOnly(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\t$SYS {\n\t\t\t\tusers = [{user: \"system\",pass: \"password\"}, {user: \"system2\",pass: \"password2\"}]\n\t\t\t}\n\t\t}\n\t`))\n\tdefer os.Remove(conf)\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tdch := make(chan struct{}, 1)\n\tnc := natsConnect(t,\n\t\ts.ClientURL(),\n\t\tnats.DisconnectErrHandler(func(_ *nats.Conn, _ error) {\n\t\t\tdch <- struct{}{}\n\t\t}),\n\t\tnats.NoCallbacksAfterClientClose())\n\tdefer nc.Close()\n\n\t// Just reload...\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\t// Make sure we did not get disconnected\n\tselect {\n\tcase <-dch:\n\t\tt.Fatal(\"Got disconnected!\")\n\tcase <-time.After(500 * time.Millisecond):\n\t\t// ok\n\t}\n}\n\nfunc TestConfigReloadRouteImportPermissionsWithAccounts(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize string\n\t\taccounts string\n\t}{\n\t\t{\"regular\", \"pool_size: -1\", _EMPTY_},\n\t\t{\"pooling\", \"pool_size: 5\", _EMPTY_},\n\t\t{\"per-account\", _EMPTY_, \"accounts: [\\\"A\\\"]\"},\n\t\t{\"pool and per-account\", \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"},\n\t} {\n\t\tt.Run(\"import \"+test.name, func(t *testing.T) {\n\t\t\tconfATemplate := `\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\timport {\n\t\t\t\t\t\t\tallow: %s\n\t\t\t\t\t\t}\n\t\t\t\t\t\texport {\n\t\t\t\t\t\t\tallow: \">\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `\"foo\"`, test.poolSize, test.accounts)))\n\t\t\tsrva, optsA := RunServerWithConfig(confA)\n\t\t\tdefer srva.Shutdown()\n\n\t\t\tconfBTemplate := `\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\timport {\n\t\t\t\t\t\t\tallow: %s\n\t\t\t\t\t\t}\n\t\t\t\t\t\texport {\n\t\t\t\t\t\t\tallow: \">\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t\t]\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `\"foo\"`, optsA.Cluster.Port, test.poolSize, test.accounts)))\n\t\t\tsrvb, _ := RunServerWithConfig(confB)\n\t\t\tdefer srvb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tncA := natsConnect(t, srva.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\n\t\t\tsub1Foo := natsSubSync(t, ncA, \"foo\")\n\t\t\tsub2Foo := natsSubSync(t, ncA, \"foo\")\n\n\t\t\tsub1Bar := natsSubSync(t, ncA, \"bar\")\n\t\t\tsub2Bar := natsSubSync(t, ncA, \"bar\")\n\n\t\t\tnatsFlush(t, ncA)\n\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubNoInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncB.Close()\n\n\t\t\tcheck := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tif expected {\n\t\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t\t\t} else {\n\t\t\t\t\tif msg, err := sub.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have gotten the message, got %s/%s\", msg.Subject, msg.Data)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Should receive on \"foo\"\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo1\"))\n\t\t\tcheck(sub1Foo, true)\n\t\t\tcheck(sub2Foo, true)\n\n\t\t\t// But not on \"bar\"\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar1\"))\n\t\t\tcheck(sub1Bar, false)\n\t\t\tcheck(sub2Bar, false)\n\n\t\t\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"bar\"`, test.poolSize, test.accounts))\n\t\t\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `\"bar\"`, optsA.Cluster.Port, test.poolSize, test.accounts))\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tcheckSubNoInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\t// Should not receive on foo\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo2\"))\n\t\t\tcheck(sub1Foo, false)\n\t\t\tcheck(sub2Foo, false)\n\n\t\t\t// Should be able to receive on bar\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar2\"))\n\t\t\tcheck(sub1Bar, true)\n\t\t\tcheck(sub2Bar, true)\n\n\t\t\t// Restore \"foo\"\n\t\t\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"foo\"`, test.poolSize, test.accounts))\n\t\t\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `\"foo\"`, optsA.Cluster.Port, test.poolSize, test.accounts))\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubNoInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\t// Should receive on \"foo\"\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo3\"))\n\t\t\tcheck(sub1Foo, true)\n\t\t\tcheck(sub2Foo, true)\n\t\t\t// But make sure there are no more than what we expect\n\t\t\tcheck(sub1Foo, false)\n\t\t\tcheck(sub2Foo, false)\n\n\t\t\t// And now \"bar\" should fail\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar3\"))\n\t\t\tcheck(sub1Bar, false)\n\t\t\tcheck(sub2Bar, false)\n\t\t})\n\t}\n\t// Check export now\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize string\n\t\taccounts string\n\t}{\n\t\t{\"regular\", \"pool_size: -1\", _EMPTY_},\n\t\t{\"pooling\", \"pool_size: 5\", _EMPTY_},\n\t\t{\"per-account\", _EMPTY_, \"accounts: [\\\"A\\\"]\"},\n\t\t{\"pool and per-account\", \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"},\n\t} {\n\t\tt.Run(\"export \"+test.name, func(t *testing.T) {\n\t\t\tconfATemplate := `\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\timport {\n\t\t\t\t\t\t\tallow: \">\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\texport {\n\t\t\t\t\t\t\tallow: %s\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, `\"foo\"`, test.poolSize, test.accounts)))\n\t\t\tsrva, optsA := RunServerWithConfig(confA)\n\t\t\tdefer srva.Shutdown()\n\n\t\t\tconfBTemplate := `\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\timport {\n\t\t\t\t\t\t\tallow: \">\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\texport {\n\t\t\t\t\t\t\tallow: %s\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t\t]\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, `\"foo\"`, optsA.Cluster.Port, test.poolSize, test.accounts)))\n\t\t\tsrvb, _ := RunServerWithConfig(confB)\n\t\t\tdefer srvb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tncA := natsConnect(t, srva.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\n\t\t\tsub1Foo := natsSubSync(t, ncA, \"foo\")\n\t\t\tsub2Foo := natsSubSync(t, ncA, \"foo\")\n\n\t\t\tsub1Bar := natsSubSync(t, ncA, \"bar\")\n\t\t\tsub2Bar := natsSubSync(t, ncA, \"bar\")\n\n\t\t\tnatsFlush(t, ncA)\n\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubNoInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncB.Close()\n\n\t\t\tcheck := func(sub *nats.Subscription, expected bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tif expected {\n\t\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t\t\t} else {\n\t\t\t\t\tif msg, err := sub.NextMsg(50 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not have gotten the message, got %s/%s\", msg.Subject, msg.Data)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Should receive on \"foo\"\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo1\"))\n\t\t\tcheck(sub1Foo, true)\n\t\t\tcheck(sub2Foo, true)\n\n\t\t\t// But not on \"bar\"\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar1\"))\n\t\t\tcheck(sub1Bar, false)\n\t\t\tcheck(sub2Bar, false)\n\n\t\t\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `[\"foo\", \"bar\"]`, test.poolSize, test.accounts))\n\t\t\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `[\"foo\", \"bar\"]`, optsA.Cluster.Port, test.poolSize, test.accounts))\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\t// Should receive on foo and bar\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo2\"))\n\t\t\tcheck(sub1Foo, true)\n\t\t\tcheck(sub2Foo, true)\n\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar2\"))\n\t\t\tcheck(sub1Bar, true)\n\t\t\tcheck(sub2Bar, true)\n\n\t\t\t// Remove \"bar\"\n\t\t\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, `\"foo\"`, test.poolSize, test.accounts))\n\t\t\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBTemplate, `\"foo\"`, optsA.Cluster.Port, test.poolSize, test.accounts))\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\t\t\tcheckSubNoInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\n\t\t\t// Should receive on \"foo\"\n\t\t\tnatsPub(t, ncB, \"foo\", []byte(\"foo3\"))\n\t\t\tcheck(sub1Foo, true)\n\t\t\tcheck(sub2Foo, true)\n\t\t\t// But make sure there are no more than what we expect\n\t\t\tcheck(sub1Foo, false)\n\t\t\tcheck(sub2Foo, false)\n\n\t\t\t// And now \"bar\" should fail\n\t\t\tnatsPub(t, ncB, \"bar\", []byte(\"bar3\"))\n\t\t\tcheck(sub1Bar, false)\n\t\t\tcheck(sub2Bar, false)\n\t\t})\n\t}\n}\n\nfunc TestConfigReloadRoutePoolAndPerAccount(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\")))\n\tsrva, optsA := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBCTemplate := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tname: \"local\"\n\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t]\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\")))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\")))\n\tsrvc, _ := RunServerWithConfig(confC)\n\tdefer srvc.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\t// We will also create subscriptions for accounts A, B and C on all sides\n\t// just to make sure that interest is properly propagated after a reload.\n\t// The conns slices will contain connections for accounts A on srva, srvb,\n\t// srvc, then B on srva, srvb, etc.. and the subs slices will contain\n\t// subscriptions for account A on foo on srva, bar on srvb, baz on srvc,\n\t// then for account B on foo on srva, etc...\n\tvar conns []*nats.Conn\n\tvar subs []*nats.Subscription\n\tfor _, user := range []string{\"user1\", \"user2\", \"user3\"} {\n\t\tnc := natsConnect(t, srva.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc.Close()\n\t\tconns = append(conns, nc)\n\t\tsub := natsSubSync(t, nc, \"foo\")\n\t\tsubs = append(subs, sub)\n\t\tnc = natsConnect(t, srvb.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc.Close()\n\t\tconns = append(conns, nc)\n\t\tsub = natsSubSync(t, nc, \"bar\")\n\t\tsubs = append(subs, sub)\n\t\tnc = natsConnect(t, srvc.ClientURL(), nats.UserInfo(user, \"pwd\"))\n\t\tdefer nc.Close()\n\t\tconns = append(conns, nc)\n\t\tsub = natsSubSync(t, nc, \"baz\")\n\t\tsubs = append(subs, sub)\n\t}\n\n\tcheckCluster := func() {\n\t\tt.Helper()\n\t\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\t\tfor _, acc := range []string{\"A\", \"B\", \"C\"} {\n\t\t\t// On server A, there should be interest for bar/baz\n\t\t\tcheckSubInterest(t, srva, acc, \"bar\", 2*time.Second)\n\t\t\tcheckSubInterest(t, srva, acc, \"baz\", 2*time.Second)\n\t\t\t// On serer B, there should be interest on foo/baz\n\t\t\tcheckSubInterest(t, srvb, acc, \"foo\", 2*time.Second)\n\t\t\tcheckSubInterest(t, srvb, acc, \"baz\", 2*time.Second)\n\t\t\t// And on server C, interest on foo/bar\n\t\t\tcheckSubInterest(t, srvc, acc, \"foo\", 2*time.Second)\n\t\t\tcheckSubInterest(t, srvc, acc, \"bar\", 2*time.Second)\n\t\t}\n\t}\n\tcheckCluster()\n\n\tgetAccRouteID := func(acc string) uint64 {\n\t\ts := srva\n\t\tvar id uint64\n\t\tsrvbId := srvb.ID()\n\t\ts.mu.RLock()\n\t\tif remotes, ok := s.accRoutes[acc]; ok {\n\t\t\t// For this test, we will take a single remote, say srvb\n\t\t\tif r := remotes[srvbId]; r != nil {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tif string(r.route.accName) == acc {\n\t\t\t\t\tid = r.cid\n\t\t\t\t}\n\t\t\t\tr.mu.Unlock()\n\t\t\t}\n\t\t}\n\t\ts.mu.RUnlock()\n\t\treturn id\n\t}\n\t// Capture the route for account \"A\"\n\traid := getAccRouteID(\"A\")\n\tif raid == 0 {\n\t\tt.Fatal(\"Did not find route for account A\")\n\t}\n\n\tgetRouteIDForAcc := func(acc string) uint64 {\n\t\ts := srva\n\t\ta, _ := s.LookupAccount(acc)\n\t\tif a == nil {\n\t\t\treturn 0\n\t\t}\n\t\ta.mu.RLock()\n\t\tpidx := a.routePoolIdx\n\t\ta.mu.RUnlock()\n\t\tvar id uint64\n\t\ts.mu.RLock()\n\t\t// For this test, we will take a single remote, say srvb\n\t\tsrvbId := srvb.ID()\n\t\tif conns, ok := s.routes[srvbId]; ok {\n\t\t\tif r := conns[pidx]; r != nil {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tid = r.cid\n\t\t\t\tr.mu.Unlock()\n\t\t\t}\n\t\t}\n\t\ts.mu.RUnlock()\n\t\treturn id\n\t}\n\trbid := getRouteIDForAcc(\"B\")\n\tif rbid == 0 {\n\t\tt.Fatal(\"Did not find route for account B\")\n\t}\n\trcid := getRouteIDForAcc(\"C\")\n\tif rcid == 0 {\n\t\tt.Fatal(\"Did not find route for account C\")\n\t}\n\trdid := getRouteIDForAcc(\"D\")\n\tif rdid == 0 {\n\t\tt.Fatal(\"Did not find route for account D\")\n\t}\n\n\tsendAndRecv := func(msg string) {\n\t\tt.Helper()\n\t\tfor accIdx := 0; accIdx < 9; accIdx += 3 {\n\t\t\tnatsPub(t, conns[accIdx], \"bar\", []byte(msg))\n\t\t\tm := natsNexMsg(t, subs[accIdx+1], time.Second)\n\t\t\tcheckMsg := func(m *nats.Msg, subj string) {\n\t\t\t\tt.Helper()\n\t\t\t\tif string(m.Data) != msg {\n\t\t\t\t\tt.Fatalf(\"For accIdx=%v, subject %q, expected message %q, got %q\", accIdx, subj, msg, m.Data)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckMsg(m, \"bar\")\n\t\t\tnatsPub(t, conns[accIdx+1], \"baz\", []byte(msg))\n\t\t\tm = natsNexMsg(t, subs[accIdx+2], time.Second)\n\t\t\tcheckMsg(m, \"baz\")\n\t\t\tnatsPub(t, conns[accIdx+2], \"foo\", []byte(msg))\n\t\t\tm = natsNexMsg(t, subs[accIdx], time.Second)\n\t\t\tcheckMsg(m, \"foo\")\n\t\t}\n\t}\n\tsendAndRecv(\"0\")\n\n\t// Now add accounts \"B\" and \"D\" and do a config reload.\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 3\", \"accounts: [\\\"A\\\",\\\"B\\\",\\\"D\\\"]\"))\n\n\t// Even before reloading srvb and srvc, we should already have per-account\n\t// routes for accounts B and D being established. The accounts routePoolIdx\n\t// should be marked as transitioning.\n\tcheckAccPoolIdx := func(s *Server, acc string, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\ts.mu.RLock()\n\t\t\tdefer s.mu.RUnlock()\n\t\t\tif a, ok := s.accounts.Load(acc); ok {\n\t\t\t\tacc := a.(*Account)\n\t\t\t\tacc.mu.RLock()\n\t\t\t\trpi := acc.routePoolIdx\n\t\t\t\tacc.mu.RUnlock()\n\t\t\t\tif rpi != expected {\n\t\t\t\t\treturn fmt.Errorf(\"Server %q - Account %q routePoolIdx should be %v, but is %v\", s, acc, expected, rpi)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Server %q - Account %q not found\", s, acc)\n\t\t})\n\t}\n\tcheckRoutePerAccAlreadyEstablished := func(s *Server, acc string) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\ts.mu.RLock()\n\t\t\tdefer s.mu.RUnlock()\n\t\t\tif _, ok := s.accRoutes[acc]; !ok {\n\t\t\t\treturn fmt.Errorf(\"Route for account %q still not established\", acc)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tcheckAccPoolIdx(s, acc, accTransitioningToDedicatedRoute)\n\t}\n\t// Check srvb and srvc for both accounts.\n\tfor _, s := range []*Server{srvb, srvc} {\n\t\tfor _, acc := range []string{\"B\", \"D\"} {\n\t\t\tcheckRoutePerAccAlreadyEstablished(s, acc)\n\t\t}\n\t}\n\t// On srva, the accounts should already have their routePoolIdx set to\n\t// the accDedicatedRoute value.\n\tfor _, acc := range []string{\"B\", \"D\"} {\n\t\tcheckAccPoolIdx(srva, acc, accDedicatedRoute)\n\t}\n\t// Now reload the other servers\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\",\\\"B\\\",\\\"D\\\"]\"))\n\treloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\",\\\"B\\\",\\\"D\\\"]\"))\n\n\tcheckCluster()\n\t// Now check that the accounts B and D are no longer transitioning\n\tfor _, s := range []*Server{srva, srvb, srvc} {\n\t\tfor _, acc := range []string{\"B\", \"D\"} {\n\t\t\tcheckAccPoolIdx(s, acc, accDedicatedRoute)\n\t\t}\n\t}\n\n\tcheckRouteForADidNotChange := func() {\n\t\tt.Helper()\n\t\tif id := getAccRouteID(\"A\"); id != raid {\n\t\t\tt.Fatalf(\"Route id for account 'A' was %d, is now %d\", raid, id)\n\t\t}\n\t}\n\t// Verify that the route for account \"A\" did not change.\n\tcheckRouteForADidNotChange()\n\n\t// Verify that account \"B\" has now its own route\n\tif id := getAccRouteID(\"B\"); id == 0 {\n\t\tt.Fatal(\"Did not find route for account B\")\n\t}\n\t// Same for \"D\".\n\tif id := getAccRouteID(\"D\"); id == 0 {\n\t\tt.Fatal(\"Did not find route for account D\")\n\t}\n\n\tcheckRouteStillPresent := func(id uint64) {\n\t\tt.Helper()\n\t\tsrva.mu.RLock()\n\t\tdefer srva.mu.RUnlock()\n\t\tsrvbId := srvb.ID()\n\t\tfor _, r := range srva.routes[srvbId] {\n\t\t\tif r != nil {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tfound := r.cid == id\n\t\t\t\tr.mu.Unlock()\n\t\t\t\tif found {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tt.Fatalf(\"Route id %v has been disconnected\", id)\n\t}\n\t// Verify that routes that were dealing with \"B\", and \"D\" were not disconnected.\n\t// Of course, since \"C\" was not involved, that route should still be present too.\n\tcheckRouteStillPresent(rbid)\n\tcheckRouteStillPresent(rcid)\n\tcheckRouteStillPresent(rdid)\n\n\tsendAndRecv(\"1\")\n\n\t// Now remove \"B\" and \"D\" and verify that route for \"A\" did not change.\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"))\n\n\tcheckCluster()\n\n\t// Verify that the route for account \"A\" did not change.\n\tcheckRouteForADidNotChange()\n\n\t// Verify that there is no dedicated route for account \"B\"\n\tif id := getAccRouteID(\"B\"); id != 0 {\n\t\tt.Fatal(\"Should not have found a route for account B\")\n\t}\n\t// It should instead be in one of the pooled route, and same\n\t// than it was before.\n\tif id := getRouteIDForAcc(\"B\"); id != rbid {\n\t\tt.Fatalf(\"Account B's route was %d, it is now %d\", rbid, id)\n\t}\n\t// Same for \"D\"\n\tif id := getAccRouteID(\"D\"); id != 0 {\n\t\tt.Fatal(\"Should not have found a route for account D\")\n\t}\n\tif id := getRouteIDForAcc(\"D\"); id != rdid {\n\t\tt.Fatalf(\"Account D's route was %d, it is now %d\", rdid, id)\n\t}\n\n\tsendAndRecv(\"2\")\n\n\t// Finally, change pool size and make sure that routes handling B, C and D\n\t// were disconnected/reconnected, and that A did not change.\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 5\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 5\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"pool_size: 5\", \"accounts: [\\\"A\\\"]\"))\n\n\tcheckCluster()\n\n\tcheckRouteForADidNotChange()\n\n\tcheckRouteDisconnected := func(acc string, oldID uint64) {\n\t\tt.Helper()\n\t\tif id := getRouteIDForAcc(acc); id == oldID {\n\t\t\tt.Fatalf(\"Route that was handling account %q did not change\", acc)\n\t\t}\n\t}\n\tcheckRouteDisconnected(\"B\", rbid)\n\tcheckRouteDisconnected(\"C\", rcid)\n\tcheckRouteDisconnected(\"D\", rdid)\n\n\tsendAndRecv(\"3\")\n\n\t// Now check that there were no duplicates and that all subs have 0 pending messages.\n\tfor i, sub := range subs {\n\t\tif n, _, _ := sub.Pending(); n != 0 {\n\t\t\tt.Fatalf(\"Expected 0 pending messages, got %v for accIdx=%d sub=%q\", n, i, sub.Subject)\n\t\t}\n\t}\n}\n\nfunc TestConfigReloadRoutePoolAndPerAccountNoPanicIfFirstAdded(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tpool_size: 2\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconf1 := createConfFile(t, fmt.Appendf(nil, tmpl, \"A\", _EMPTY_, _EMPTY_))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troute := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, fmt.Appendf(nil, tmpl, \"B\", _EMPTY_, route))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", \"accounts:[\\\"A\\\"]\", _EMPTY_))\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", \"accounts:[\\\"A\\\"]\", route))\n\n\ttime.Sleep(50 * time.Millisecond)\n\tcheckClusterFormed(t, s1, s2)\n}\n\nfunc TestConfigReloadRoutePoolCannotBeDisabledIfAccountsPresent(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"user2\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", \"accounts: [\\\"A\\\"]\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", \"accounts: [\\\"A\\\"]\", _EMPTY_,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\terr := os.WriteFile(conf1, []byte(fmt.Sprintf(tmpl, \"A\", \"accounts: [\\\"A\\\"]\", \"pool_size: -1\", _EMPTY_)), 0666)\n\trequire_NoError(t, err)\n\tif err := s1.Reload(); err == nil || !strings.Contains(err.Error(), \"accounts\") {\n\t\tt.Fatalf(\"Expected error regarding presence of accounts, got %v\", err)\n\t}\n\n\t// Now remove the accounts too and reload, this should work\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"pool_size: -1\", _EMPTY_))\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", _EMPTY_, \"pool_size: -1\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)))\n\tcheckClusterFormed(t, s1, s2)\n\n\tncs2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncs2.Close()\n\tsub := natsSubSync(t, ncs2, \"foo\")\n\tcheckSubInterest(t, s1, \"A\", \"foo\", time.Second)\n\n\tncs1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncs1.Close()\n\tnatsPub(t, ncs1, \"foo\", []byte(\"hello\"))\n\tnatsNexMsg(t, sub, time.Second)\n\n\t// Wait a bit and make sure there are no duplicates\n\ttime.Sleep(50 * time.Millisecond)\n\tif n, _, _ := sub.Pending(); n != 0 {\n\t\tt.Fatalf(\"Expected no pending messages, got %v\", n)\n\t}\n\n\t// Finally, verify that the system account is no longer bound to\n\t// a dedicated route. For that matter, s.accRoutes should be nil.\n\tfor _, s := range []*Server{s1, s2} {\n\t\tsys := s.SystemAccount()\n\t\tif sys == nil {\n\t\t\tt.Fatal(\"No system account found\")\n\t\t}\n\t\tsys.mu.RLock()\n\t\trpi := sys.routePoolIdx\n\t\tsys.mu.RUnlock()\n\t\tif rpi != 0 {\n\t\t\tt.Fatalf(\"Server %q - expected account's routePoolIdx to be 0, got %v\", s, rpi)\n\t\t}\n\t\ts.mu.RLock()\n\t\tarNil := s.accRoutes == nil\n\t\ts.mu.RUnlock()\n\t\tif !arNil {\n\t\t\tt.Fatalf(\"Server %q - accRoutes expected to be nil, it was not\", s)\n\t\t}\n\t}\n}\n\nfunc TestConfigReloadRoutePoolAndPerAccountWithOlderServer(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, \"pool_size: 3\", _EMPTY_)))\n\tsrva, optsA := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBCTemplate := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tname: \"local\"\n\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t]\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", _EMPTY_)))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"pool_size: -1\", _EMPTY_)))\n\tsrvc, _ := RunServerWithConfig(confC)\n\tdefer srvc.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\t// Create a connection and sub on B and C\n\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncB.Close()\n\tsubB := natsSubSync(t, ncB, \"foo\")\n\n\tncC := natsConnect(t, srvc.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncC.Close()\n\tsubC := natsSubSync(t, ncC, \"bar\")\n\n\t// Check that on server B, there is interest on \"bar\" for account A\n\t// (coming from server C), and on server C, there is interest on \"foo\"\n\t// for account A (coming from server B).\n\tcheckCluster := func() {\n\t\tt.Helper()\n\t\tcheckClusterFormed(t, srva, srvb, srvc)\n\t\tcheckSubInterest(t, srvb, \"A\", \"bar\", 2*time.Second)\n\t\tcheckSubInterest(t, srvc, \"A\", \"foo\", 2*time.Second)\n\t}\n\tcheckCluster()\n\n\tsendAndRecv := func(msg string) {\n\t\tt.Helper()\n\t\tnatsPub(t, ncB, \"bar\", []byte(msg))\n\t\tif m := natsNexMsg(t, subC, time.Second); string(m.Data) != msg {\n\t\t\tt.Fatalf(\"Expected message %q on %q, got %q\", msg, \"bar\", m.Data)\n\t\t}\n\t\tnatsPub(t, ncC, \"foo\", []byte(msg))\n\t\tif m := natsNexMsg(t, subB, time.Second); string(m.Data) != msg {\n\t\t\tt.Fatalf(\"Expected message %q on %q, got %q\", msg, \"foo\", m.Data)\n\t\t}\n\t}\n\tsendAndRecv(\"0\")\n\n\t// Now add account \"A\" and do a config reload. We do this only on\n\t// server srva and srb since server C really does not change.\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", \"accounts: [\\\"A\\\"]\"))\n\tcheckCluster()\n\tsendAndRecv(\"1\")\n\n\t// Remove \"A\" from the accounts list\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 3\", _EMPTY_))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", _EMPTY_))\n\tcheckCluster()\n\tsendAndRecv(\"2\")\n\n\t// Change the pool size\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 5\", _EMPTY_))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 5\", _EMPTY_))\n\tcheckCluster()\n\tsendAndRecv(\"3\")\n\n\t// Add account \"A\" and change the pool size\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 4\", \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 4\", \"accounts: [\\\"A\\\"]\"))\n\tcheckCluster()\n\tsendAndRecv(\"4\")\n\n\t// Remove account \"A\" and change the pool size\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"pool_size: 3\", _EMPTY_))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"pool_size: 3\", _EMPTY_))\n\tcheckCluster()\n\tsendAndRecv(\"5\")\n}\n\nfunc TestConfigReloadRoutePoolAndPerAccountNoDuplicateSub(t *testing.T) {\n\tconfATemplate := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tpool_size: 3\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, _EMPTY_)))\n\tsrva, optsA := RunServerWithConfig(confA)\n\tdefer srva.Shutdown()\n\n\tconfBCTemplate := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\tname: \"local\"\n\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t]\n\t\t\t\tpool_size: 3\n\t\t\t\t%s\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, _EMPTY_)))\n\tsrvb, _ := RunServerWithConfig(confB)\n\tdefer srvb.Shutdown()\n\tconfC := createConfFile(t, []byte(fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, _EMPTY_)))\n\tsrvc, _ := RunServerWithConfig(confC)\n\tdefer srvc.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\tncC := natsConnect(t, srvc.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncC.Close()\n\n\tch := make(chan struct{})\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tvar subs []*nats.Subscription\n\tgo func() {\n\t\tdefer wg.Done()\n\t\t// Limit the number of subscriptions. From experimentation, the issue would\n\t\t// arise around subscriptions ~700.\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tif sub, err := ncC.SubscribeSync(fmt.Sprintf(\"foo.%d\", i)); err == nil {\n\t\t\t\tsubs = append(subs, sub)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-ch:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tif i%100 == 0 {\n\t\t\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Wait a tiny bit before doing the configuration reload.\n\ttime.Sleep(100 * time.Millisecond)\n\n\treloadUpdateConfig(t, srva, confA, fmt.Sprintf(confATemplate, \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvb, confB, fmt.Sprintf(confBCTemplate, \"B\", optsA.Cluster.Port, \"accounts: [\\\"A\\\"]\"))\n\treloadUpdateConfig(t, srvc, confC, fmt.Sprintf(confBCTemplate, \"C\", optsA.Cluster.Port, \"accounts: [\\\"A\\\"]\"))\n\n\tcheckClusterFormed(t, srva, srvb, srvc)\n\n\tclose(ch)\n\twg.Wait()\n\n\tfor _, sub := range subs {\n\t\tcheckSubInterest(t, srvb, \"A\", sub.Subject, 500*time.Millisecond)\n\t}\n\n\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\tdefer ncB.Close()\n\n\tfor _, sub := range subs {\n\t\tnatsPub(t, ncB, sub.Subject, []byte(\"hello\"))\n\t}\n\n\t// Now make sure that there is only 1 pending message for each sub.\n\t// Wait a bit to give a chance to duplicate messages to arrive if\n\t// there was a bug that would lead to a sub on each route (the pooled\n\t// and the per-account)\n\ttime.Sleep(250 * time.Millisecond)\n\tfor _, sub := range subs {\n\t\tif n, _, _ := sub.Pending(); n != 1 {\n\t\t\tt.Fatalf(\"Expected only 1 message for subscription on %q, got %v\", sub.Subject, n)\n\t\t}\n\t}\n}\n\nfunc TestConfigReloadGlobalAccountWithMappingAndJetStream(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tjetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'}\n\n\t\tmappings {\n\t\t\tsubj.orig: subj.mapped.before.reload\n\t\t}\n\n\t\tleaf {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\n\t\tcluster {\n\t\t\tname: %s\n\t\t\tlisten: 127.0.0.1:%d\n\t\t\troutes = [%s]\n\t\t}\n\n\t\t# For access to system account.\n\t\taccounts { $SYS { users = [ { user: \"admin\", pass: \"s3cr3t!\" } ] } }\n\t`\n\tc := createJetStreamClusterWithTemplate(t, tmpl, \"R3S\", 3)\n\tdefer c.shutdown()\n\n\tnc, js := jsClientConnect(t, c.randomServer())\n\tdefer nc.Close()\n\n\t// Verify that mapping works\n\tcheckMapping := func(expectedSubj string) {\n\t\tt.Helper()\n\t\tsub := natsSubSync(t, nc, \"subj.>\")\n\t\tdefer sub.Unsubscribe()\n\t\tnatsPub(t, nc, \"subj.orig\", nil)\n\t\tmsg := natsNexMsg(t, sub, time.Second)\n\t\tif msg.Subject != expectedSubj {\n\t\t\tt.Fatalf(\"Expected subject to have been mapped to %q, got %q\", expectedSubj, msg.Subject)\n\t\t}\n\t}\n\tcheckMapping(\"subj.mapped.before.reload\")\n\n\t// Create a stream and check that we can get the INFO\n\t_, err := js.AddStream(&nats.StreamConfig{\n\t\tName: \"TEST\",\n\t\tReplicas: 3,\n\t\tSubjects: []string{\"foo\"},\n\t\tRetention: nats.InterestPolicy,\n\t})\n\trequire_NoError(t, err)\n\tc.waitOnStreamLeader(globalAccountName, \"TEST\")\n\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n\n\t// Change mapping on all servers and issue reload\n\tfor i, s := range c.servers {\n\t\topts := c.opts[i]\n\t\tcontent, err := os.ReadFile(opts.ConfigFile)\n\t\trequire_NoError(t, err)\n\t\treloadUpdateConfig(t, s, opts.ConfigFile, strings.Replace(string(content), \"subj.mapped.before.reload\", \"subj.mapped.after.reload\", 1))\n\t}\n\t// Make sure the cluster is still formed\n\tcheckClusterFormed(t, c.servers...)\n\t// Now repeat the test for the subject mapping and stream info\n\tcheckMapping(\"subj.mapped.after.reload\")\n\t_, err = js.StreamInfo(\"TEST\")\n\trequire_NoError(t, err)\n}\n\nfunc TestConfigReloadRouteCompression(t *testing.T) {\n\torg := testDefaultClusterCompression\n\ttestDefaultClusterCompression = _EMPTY_\n\tdefer func() { testDefaultClusterCompression = org }()\n\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troutes := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", routes, _EMPTY_)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\t// Run a 3rd server but make it as if it was an old server. We want to\n\t// make sure that reload of s1 and s2 will not affect routes from s3 to\n\t// s1/s2 because these do not support compression.\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"C\", routes, \"compression: \\\"not supported\\\"\")))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// Collect routes' cid from servers so we can check if routes are\n\t// recreated when they should and are not when they should not.\n\tcollect := func(s *Server) map[uint64]struct{} {\n\t\tm := make(map[uint64]struct{})\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\ts.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\tm[r.cid] = struct{}{}\n\t\t\tr.mu.Unlock()\n\t\t})\n\t\treturn m\n\t}\n\ts1RouteIDs := collect(s1)\n\ts2RouteIDs := collect(s2)\n\ts3ID := s3.ID()\n\n\tservers := []*Server{s1, s2}\n\tcheckCompMode := func(s1Expected, s2Expected string, shouldBeNew bool) {\n\t\tt.Helper()\n\t\t// We wait a bit to make sure that we have routes closed before\n\t\t// checking that the cluster has (re)formed.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\t// First, make sure that the cluster is formed\n\t\tcheckClusterFormed(t, s1, s2, s3)\n\t\t// Then check that all routes are with the expected mode. We need to\n\t\t// possibly wait a bit since there is negotiation going on.\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tfor _, s := range servers {\n\t\t\t\tvar err error\n\t\t\t\ts.mu.RLock()\n\t\t\t\ts.forEachRoute(func(r *client) {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tvar exp string\n\t\t\t\t\tvar m map[uint64]struct{}\n\t\t\t\t\tif r.route.remoteID == s3ID {\n\t\t\t\t\t\texp = CompressionNotSupported\n\t\t\t\t\t} else if s == s1 {\n\t\t\t\t\t\texp = s1Expected\n\t\t\t\t\t} else {\n\t\t\t\t\t\texp = s2Expected\n\t\t\t\t\t}\n\t\t\t\t\tif s == s1 {\n\t\t\t\t\t\tm = s1RouteIDs\n\t\t\t\t\t} else {\n\t\t\t\t\t\tm = s2RouteIDs\n\t\t\t\t\t}\n\t\t\t\t\t_, present := m[r.cid]\n\t\t\t\t\tcm := r.route.compression\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\tif cm != exp {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected route %v for server %s to have compression mode %q, got %q\", r, s, exp, cm)\n\t\t\t\t\t}\n\t\t\t\t\tsbn := shouldBeNew\n\t\t\t\t\tif exp == CompressionNotSupported {\n\t\t\t\t\t\t// Override for routes to s3\n\t\t\t\t\t\tsbn = false\n\t\t\t\t\t}\n\t\t\t\t\tif sbn && present {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected route %v for server %s to be a new route, but it was already present\", r, s)\n\t\t\t\t\t} else if !sbn && !present {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected route %v for server %s to not be new\", r, s)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t\ts.mu.RUnlock()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\ts1RouteIDs = collect(s1)\n\t\t\ts2RouteIDs = collect(s2)\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Since both started without any compression setting, we default to\n\t// \"accept\" which means that a server can accept/switch to compression\n\t// but not initiate compression, so they should both be \"off\"\n\tcheckCompMode(CompressionOff, CompressionOff, false)\n\n\t// Now reload s1 with \"on\" (s2_fast), since s2 is *configured* with \"accept\",\n\t// they should both be CompressionS2Fast, even before we reload s2.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: on\"))\n\tcheckCompMode(CompressionS2Fast, CompressionS2Fast, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, \"compression: on\"))\n\tcheckCompMode(CompressionS2Fast, CompressionS2Fast, false)\n\n\t// Move on with \"better\"\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: s2_better\"))\n\t// s1 should be at \"better\", but s2 still at \"fast\"\n\tcheckCompMode(CompressionS2Better, CompressionS2Fast, false)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, \"compression: s2_better\"))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, false)\n\n\t// Move to \"best\"\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: s2_best\"))\n\tcheckCompMode(CompressionS2Best, CompressionS2Better, false)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, \"compression: s2_best\"))\n\tcheckCompMode(CompressionS2Best, CompressionS2Best, false)\n\n\t// Now turn off\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: off\"))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, \"compression: off\"))\n\tcheckCompMode(CompressionOff, CompressionOff, false)\n\n\t// When \"off\" (and not \"accept\"), enabling 1 is not enough, the reload\n\t// has to be done on both to take effect.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: s2_better\"))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, \"compression: s2_better\"))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, true)\n\n\t// Try now to have different ones\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: s2_best\"))\n\t// S1 should be \"best\" but S2 should have stayed at \"better\"\n\tcheckCompMode(CompressionS2Best, CompressionS2Better, false)\n\n\t// If we remove the compression setting, it defaults to \"accept\", which\n\t// in that case we want to have a negotiation and use the remote's compression\n\t// level. So connections should be re-created.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, true)\n\n\t// To avoid flapping, add a little sleep here to make sure we have things\n\t// settled before reloading s2.\n\ttime.Sleep(100 * time.Millisecond)\n\t// And if we do the same with s2, then we will end-up with no compression.\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", routes, _EMPTY_))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n}\n\nfunc TestConfigReloadRouteCompressionS2Auto(t *testing.T) {\n\t// This test checks s2_auto specific behavior. It makes sure that we update\n\t// only if the rtt_thresholds and current RTT value warrants a change and\n\t// also that we actually save in c.route.compression the actual compression\n\t// level (not s2_auto).\n\ttmpl1 := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: -1\n\t\t\tcompression: {mode: s2_auto, rtt_thresholds: [%s]}\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, \"50ms, 100ms, 150ms\")))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: -1\n\t\t\tcompression: s2_fast\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t`, o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tgetCompInfo := func() (string, io.Writer) {\n\t\tvar cm string\n\t\tvar cw io.Writer\n\t\ts1.mu.RLock()\n\t\t// There should be only 1 route...\n\t\ts1.forEachRemote(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\tcm = r.route.compression\n\t\t\tcw = r.out.cw\n\t\t\tr.mu.Unlock()\n\t\t})\n\t\ts1.mu.RUnlock()\n\t\treturn cm, cw\n\t}\n\t// Capture the s2 writer from s1 to s2\n\tcm, cw := getCompInfo()\n\n\t// We do a reload but really the mode is still s2_auto (even if the current\n\t// compression level may be \"uncompressed\", \"better\", etc.. so we don't\n\t// expect the writer to have changed.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"100ms, 200ms, 300ms\"))\n\tif ncm, ncw := getCompInfo(); ncm != cm || ncw != cw {\n\t\tt.Fatalf(\"Expected compression info to have stayed the same, was %q - %p, got %q - %p\", cm, cw, ncm, ncw)\n\t}\n}\n\nfunc TestConfigReloadLeafNodeCompression(t *testing.T) {\n\torg := testDefaultLeafNodeCompression\n\ttestDefaultLeafNodeCompression = _EMPTY_\n\tdefer func() { testDefaultLeafNodeCompression = org }()\n\n\ttmpl1 := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, \"compression: accept\")))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tport := o1.LeafNode.Port\n\n\ttmpl2 := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, \"B\", port, \"compression: accept\")))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\t// Run a 3rd server but make it as if it was an old server. We want to\n\t// make sure that reload of s1 and s2 will not affect leafnodes from s3 to\n\t// s1/s2 because these do not support compression.\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl2, \"C\", port, \"compression: \\\"not supported\\\"\")))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\tcheckLeafNodeConnected(t, s3)\n\tcheckLeafNodeConnectedCount(t, s1, 2)\n\n\t// Collect leafnodes' cid from servers so we can check if connections are\n\t// recreated when they should and are not when they should not.\n\tcollect := func(s *Server) map[uint64]struct{} {\n\t\tm := make(map[uint64]struct{})\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\tfor _, l := range s.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tm[l.cid] = struct{}{}\n\t\t\tl.mu.Unlock()\n\t\t}\n\t\treturn m\n\t}\n\ts1LeafNodeIDs := collect(s1)\n\ts2LeafNodeIDs := collect(s2)\n\n\tservers := []*Server{s1, s2}\n\tcheckCompMode := func(s1Expected, s2Expected string, shouldBeNew bool) {\n\t\tt.Helper()\n\t\t// We wait a bit to make sure that we have leaf connections closed\n\t\t// before checking that they are properly reconnected.\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tcheckLeafNodeConnected(t, s2)\n\t\tcheckLeafNodeConnected(t, s3)\n\t\tcheckLeafNodeConnectedCount(t, s1, 2)\n\t\t// Check that all leafnodes are with the expected mode. We need to\n\t\t// possibly wait a bit since there is negotiation going on.\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tfor _, s := range servers {\n\t\t\t\tvar err error\n\t\t\t\ts.mu.RLock()\n\t\t\t\tfor _, l := range s.leafs {\n\t\t\t\t\tl.mu.Lock()\n\t\t\t\t\tvar exp string\n\t\t\t\t\tvar m map[uint64]struct{}\n\t\t\t\t\tif l.leaf.remoteServer == \"C\" {\n\t\t\t\t\t\texp = CompressionNotSupported\n\t\t\t\t\t} else if s == s1 {\n\t\t\t\t\t\texp = s1Expected\n\t\t\t\t\t} else {\n\t\t\t\t\t\texp = s2Expected\n\t\t\t\t\t}\n\t\t\t\t\tif s == s1 {\n\t\t\t\t\t\tm = s1LeafNodeIDs\n\t\t\t\t\t} else {\n\t\t\t\t\t\tm = s2LeafNodeIDs\n\t\t\t\t\t}\n\t\t\t\t\t_, present := m[l.cid]\n\t\t\t\t\tcm := l.leaf.compression\n\t\t\t\t\tl.mu.Unlock()\n\t\t\t\t\tif cm != exp {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected leaf %v for server %s to have compression mode %q, got %q\", l, s, exp, cm)\n\t\t\t\t\t}\n\t\t\t\t\tsbn := shouldBeNew\n\t\t\t\t\tif exp == CompressionNotSupported {\n\t\t\t\t\t\t// Override for routes to s3\n\t\t\t\t\t\tsbn = false\n\t\t\t\t\t}\n\t\t\t\t\tif sbn && present {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected leaf %v for server %s to be a new leaf, but it was already present\", l, s)\n\t\t\t\t\t} else if !sbn && !present {\n\t\t\t\t\t\terr = fmt.Errorf(\"Expected leaf %v for server %s to not be new\", l, s)\n\t\t\t\t\t}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.mu.RUnlock()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\ts1LeafNodeIDs = collect(s1)\n\t\t\ts2LeafNodeIDs = collect(s2)\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Since both started with compression \"accept\", they should both be set to \"off\"\n\tcheckCompMode(CompressionOff, CompressionOff, false)\n\n\t// Now reload s1 with \"on\" (s2_auto), since s2 is *configured* with \"accept\",\n\t// s1 should be \"uncompressed\" (due to low RTT), and s2 is in that case set\n\t// to s2_fast.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: on\"))\n\tcheckCompMode(CompressionS2Uncompressed, CompressionS2Fast, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: on\"))\n\tcheckCompMode(CompressionS2Uncompressed, CompressionS2Uncompressed, false)\n\n\t// Move on with \"better\"\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: s2_better\"))\n\t// s1 should be at \"better\", but s2 still at \"uncompressed\"\n\tcheckCompMode(CompressionS2Better, CompressionS2Uncompressed, false)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: s2_better\"))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, false)\n\n\t// Move to \"best\"\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: s2_best\"))\n\tcheckCompMode(CompressionS2Best, CompressionS2Better, false)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: s2_best\"))\n\tcheckCompMode(CompressionS2Best, CompressionS2Best, false)\n\n\t// Now turn off\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: off\"))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: off\"))\n\tcheckCompMode(CompressionOff, CompressionOff, false)\n\n\t// When \"off\" (and not \"accept\"), enabling 1 is not enough, the reload\n\t// has to be done on both to take effect.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: s2_better\"))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n\t// Now reload s2\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: s2_better\"))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, true)\n\n\t// Try now to have different ones\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: s2_best\"))\n\t// S1 should be \"best\" but S2 should have stayed at \"better\"\n\tcheckCompMode(CompressionS2Best, CompressionS2Better, false)\n\n\t// Change the setting to \"accept\", which in that case we want to have a\n\t// negotiation and use the remote's compression level. So connections\n\t// should be re-created.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"compression: accept\"))\n\tcheckCompMode(CompressionS2Better, CompressionS2Better, true)\n\n\t// To avoid flapping, add a little sleep here to make sure we have things\n\t// settled before reloading s2.\n\ttime.Sleep(100 * time.Millisecond)\n\t// And if we do the same with s2, then we will end-up with no compression.\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, \"compression: accept\"))\n\tcheckCompMode(CompressionOff, CompressionOff, true)\n\n\t// Now remove completely and we should default to s2_auto, which means that\n\t// s1 should be at \"uncompressed\" and s2 to \"fast\".\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, _EMPTY_))\n\tcheckCompMode(CompressionS2Uncompressed, CompressionS2Fast, true)\n\n\t// Now with s2, both will be \"uncompressed\"\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl2, \"B\", port, _EMPTY_))\n\tcheckCompMode(CompressionS2Uncompressed, CompressionS2Uncompressed, false)\n}\n\nfunc TestConfigReloadLeafNodeCompressionS2Auto(t *testing.T) {\n\t// This test checks s2_auto specific behavior. It makes sure that we update\n\t// only if the rtt_thresholds and current RTT value warrants a change and\n\t// also that we actually save in c.leaf.compression the actual compression\n\t// level (not s2_auto).\n\ttmpl1 := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tcompression: {mode: s2_auto, rtt_thresholds: [%s]}\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl1, \"50ms, 100ms, 150ms\")))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tleafnodes {\n\t\t\tremotes [{ url: \"nats://127.0.0.1:%d\", compression: s2_fast}]\n\t\t}\n\t`, o1.LeafNode.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckLeafNodeConnected(t, s2)\n\n\tgetCompInfo := func() (string, io.Writer) {\n\t\tvar cm string\n\t\tvar cw io.Writer\n\t\ts1.mu.RLock()\n\t\t// There should be only 1 leaf...\n\t\tfor _, l := range s1.leafs {\n\t\t\tl.mu.Lock()\n\t\t\tcm = l.leaf.compression\n\t\t\tcw = l.out.cw\n\t\t\tl.mu.Unlock()\n\t\t}\n\t\ts1.mu.RUnlock()\n\t\treturn cm, cw\n\t}\n\t// Capture the s2 writer from s1 to s2\n\tcm, cw := getCompInfo()\n\n\t// We do a reload but really the mode is still s2_auto (even if the current\n\t// compression level may be \"uncompressed\", \"better\", etc.. so we don't\n\t// expect the writer to have changed.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl1, \"100ms, 200ms, 300ms\"))\n\tif ncm, ncw := getCompInfo(); ncm != cm || ncw != cw {\n\t\tt.Fatalf(\"Expected compression info to have stayed the same, was %q - %p, got %q - %p\", cm, cw, ncm, ncw)\n\t}\n}\n\nfunc TestConfigReloadNoPanicOnShutdown(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tjetstream: true\n\t\taccounts {\n\t\t\tA {\n\t\t\t\tusers: [{user: A, password: pwd}]\n\t\t\t\t%s\n\t\t\t}\n\t\t\tB {\n\t\t\t\tusers: [{user: B, password: pwd}]\n\t\t\t}\n\t\t}\n\t`\n\tfor i := 0; i < 50; i++ {\n\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\t\ts, _ := RunServerWithConfig(conf)\n\t\t// Don't use a defer s.Shutdown() here since it would prevent the panic\n\t\t// to be reported (but the test would still fail because of a runtime timeout).\n\n\t\terr := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, \"jetstream: true\")), 0666)\n\t\trequire_NoError(t, err)\n\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\ts.Shutdown()\n\t\t}()\n\n\t\ttime.Sleep(8 * time.Millisecond)\n\t\terr = s.Reload()\n\t\trequire_NoError(t, err)\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "server/ring.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\n// We wrap to hold onto optional items for /connz.\ntype closedClient struct {\n\tConnInfo\n\tsubs []SubDetail\n\tuser string\n\tacc string\n}\n\n// Fixed sized ringbuffer for closed connections.\ntype closedRingBuffer struct {\n\ttotal uint64\n\tconns []*closedClient\n}\n\n// Create a new ring buffer with at most max items.\nfunc newClosedRingBuffer(max int) *closedRingBuffer {\n\trb := &closedRingBuffer{}\n\trb.conns = make([]*closedClient, max)\n\treturn rb\n}\n\n// Adds in a new closed connection. If there is no more room,\n// remove the oldest.\nfunc (rb *closedRingBuffer) append(cc *closedClient) {\n\trb.conns[rb.next()] = cc\n\trb.total++\n}\n\nfunc (rb *closedRingBuffer) next() int {\n\treturn int(rb.total % uint64(cap(rb.conns)))\n}\n\nfunc (rb *closedRingBuffer) len() int {\n\tif rb.total > uint64(cap(rb.conns)) {\n\t\treturn cap(rb.conns)\n\t}\n\treturn int(rb.total)\n}\n\nfunc (rb *closedRingBuffer) totalConns() uint64 {\n\treturn rb.total\n}\n\n// This will return a sorted copy of the list which recipient can\n// modify. If the contents of the client itself need to be modified,\n// meaning swapping in any optional items, a copy should be made. We\n// could introduce a new lock and hold that but since we return this\n// list inside monitor which allows programatic access, we do not\n// know when it would be done.\nfunc (rb *closedRingBuffer) closedClients() []*closedClient {\n\tdup := make([]*closedClient, rb.len())\n\thead := rb.next()\n\tif rb.total <= uint64(cap(rb.conns)) || head == 0 {\n\t\tcopy(dup, rb.conns[:rb.len()])\n\t} else {\n\t\tfp := rb.conns[head:]\n\t\tsp := rb.conns[:head]\n\t\tcopy(dup, fp)\n\t\tcopy(dup[len(fp):], sp)\n\t}\n\treturn dup\n}\n" }, { "path": "server/ring_test.go", "content": "// Copyright 2018 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestRBAppendAndLenAndTotal(t *testing.T) {\n\trb := newClosedRingBuffer(10)\n\tfor i := 0; i < 5; i++ {\n\t\trb.append(&closedClient{})\n\t}\n\tif rbl := rb.len(); rbl != 5 {\n\t\tt.Fatalf(\"Expected len of 5, got %d\", rbl)\n\t}\n\tif rbt := rb.totalConns(); rbt != 5 {\n\t\tt.Fatalf(\"Expected total of 5, got %d\", rbt)\n\t}\n\tfor i := 0; i < 25; i++ {\n\t\trb.append(&closedClient{})\n\t}\n\tif rbl := rb.len(); rbl != 10 {\n\t\tt.Fatalf(\"Expected len of 10, got %d\", rbl)\n\t}\n\tif rbt := rb.totalConns(); rbt != 30 {\n\t\tt.Fatalf(\"Expected total of 30, got %d\", rbt)\n\t}\n}\n\nfunc (cc *closedClient) String() string {\n\treturn cc.user\n}\n\nfunc TestRBclosedClients(t *testing.T) {\n\trb := newClosedRingBuffer(10)\n\n\tvar ui int\n\taddConn := func() {\n\t\tui++\n\t\trb.append(&closedClient{user: fmt.Sprintf(\"%d\", ui)})\n\t}\n\n\tmax := 100\n\tmaster := make([]*closedClient, 0, max)\n\tfor i := 1; i <= max; i++ {\n\t\tmaster = append(master, &closedClient{user: fmt.Sprintf(\"%d\", i)})\n\t}\n\n\ttestList := func(i int) {\n\t\tccs := rb.closedClients()\n\t\tstart := int(rb.totalConns()) - len(ccs)\n\t\tms := master[start : start+len(ccs)]\n\t\tif !reflect.DeepEqual(ccs, ms) {\n\t\t\tt.Fatalf(\"test %d: List result did not match master: %+v vs %+v\", i, ccs, ms)\n\t\t}\n\t}\n\n\tfor i := 0; i < max; i++ {\n\t\taddConn()\n\t\ttestList(i)\n\t}\n}\n" }, { "path": "server/route.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/s2\"\n)\n\n// RouteType designates the router type\ntype RouteType int\n\n// Type of Route\nconst (\n\t// This route we learned from speaking to other routes.\n\tImplicit RouteType = iota\n\t// This route was explicitly configured.\n\tExplicit\n)\n\n// Include the space for the proto\nvar (\n\taSubBytes = []byte{'A', '+', ' '}\n\taUnsubBytes = []byte{'A', '-', ' '}\n\trSubBytes = []byte{'R', 'S', '+', ' '}\n\trUnsubBytes = []byte{'R', 'S', '-', ' '}\n\tlSubBytes = []byte{'L', 'S', '+', ' '}\n\tlUnsubBytes = []byte{'L', 'S', '-', ' '}\n)\n\ntype route struct {\n\tremoteID string\n\tremoteName string\n\tdidSolicit bool\n\tretry bool\n\tlnoc bool\n\tlnocu bool\n\trouteType RouteType\n\turl *url.URL\n\tauthRequired bool\n\ttlsRequired bool\n\tjetstream bool\n\tconnectURLs []string\n\twsConnURLs []string\n\tgatewayURL string\n\tleafnodeURL string\n\thash string\n\tidHash string\n\t// Location of the route in the slice: s.routes[remoteID][]*client.\n\t// Initialized to -1 on creation, as to indicate that it is not\n\t// added to the list.\n\tpoolIdx int\n\t// If this is set, it means that the route is dedicated for this\n\t// account and the account name will not be included in protocols.\n\taccName []byte\n\t// This is set to true if this is a route connection to an old\n\t// server or a server that has pooling completely disabled.\n\tnoPool bool\n\t// Selected compression mode, which may be different from the\n\t// server configured mode.\n\tcompression string\n\t// Transient value used to set the Info.GossipMode when initiating\n\t// an implicit route and sending to the remote.\n\tgossipMode byte\n\t// This will be set in case of pooling so that a route can trigger\n\t// the creation of the next after receiving a PONG, ensuring\n\t// that authentication did not fail.\n\tstartNewRoute *routeInfo\n}\n\n// This contains the information required to create a new route.\ntype routeInfo struct {\n\turl *url.URL\n\trtype RouteType\n\tgossipMode byte\n}\n\n// Do not change the values/order since they are exchanged between servers.\nconst (\n\tgossipDefault = byte(iota)\n\tgossipDisabled\n\tgossipOverride\n)\n\ntype connectInfo struct {\n\tEcho bool `json:\"echo\"`\n\tVerbose bool `json:\"verbose\"`\n\tPedantic bool `json:\"pedantic\"`\n\tUser string `json:\"user,omitempty\"`\n\tPass string `json:\"pass,omitempty\"`\n\tTLS bool `json:\"tls_required\"`\n\tHeaders bool `json:\"headers\"`\n\tName string `json:\"name\"`\n\tCluster string `json:\"cluster\"`\n\tDynamic bool `json:\"cluster_dynamic,omitempty\"`\n\tLNOC bool `json:\"lnoc,omitempty\"`\n\tLNOCU bool `json:\"lnocu,omitempty\"` // Support for LS- with origin cluster name\n\tGateway string `json:\"gateway,omitempty\"`\n}\n\n// Route protocol constants\nconst (\n\tConProto = \"CONNECT %s\" + _CRLF_\n\tInfoProto = \"INFO %s\" + _CRLF_\n)\n\nconst (\n\t// Warning when user configures cluster TLS insecure\n\tclusterTLSInsecureWarning = \"TLS certificate chain and hostname of solicited routes will not be verified. DO NOT USE IN PRODUCTION!\"\n\n\t// The default ping interval is set to 2 minutes, which is fine for client\n\t// connections, etc.. but for route compression, the CompressionS2Auto\n\t// mode uses RTT measurements (ping/pong) to decide which compression level\n\t// to use, we want the interval to not be that high.\n\tdefaultRouteMaxPingInterval = 30 * time.Second\n)\n\n// Can be changed for tests\nvar (\n\trouteConnectDelay = DEFAULT_ROUTE_CONNECT\n\trouteConnectMaxDelay = DEFAULT_ROUTE_CONNECT_MAX\n\trouteMaxPingInterval = defaultRouteMaxPingInterval\n)\n\n// removeReplySub is called when we trip the max on remoteReply subs.\nfunc (c *client) removeReplySub(sub *subscription) {\n\tif sub == nil {\n\t\treturn\n\t}\n\t// Lookup the account based on sub.sid.\n\tif i := bytes.Index(sub.sid, []byte(\" \")); i > 0 {\n\t\t// First part of SID for route is account name.\n\t\tif v, ok := c.srv.accounts.Load(bytesToString(sub.sid[:i])); ok {\n\t\t\t(v.(*Account)).sl.Remove(sub)\n\t\t}\n\t\tc.mu.Lock()\n\t\tdelete(c.subs, bytesToString(sub.sid))\n\t\tc.mu.Unlock()\n\t}\n}\n\nfunc (c *client) processAccountSub(arg []byte) error {\n\tif c.kind == GATEWAY {\n\t\treturn c.processGatewayAccountSub(string(arg))\n\t}\n\treturn nil\n}\n\nfunc (c *client) processAccountUnsub(arg []byte) {\n\tif c.kind == GATEWAY {\n\t\tc.processGatewayAccountUnsub(string(arg))\n\t}\n}\n\n// Process an inbound LMSG specification from the remote route. This means\n// we have an origin cluster and we force header semantics.\nfunc (c *client) processRoutedOriginClusterMsgArgs(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\targs := c.argsa[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tvar an []byte\n\tif c.kind == ROUTER {\n\t\tif an = c.route.accName; len(an) > 0 && len(args) > 2 {\n\t\t\targs = append(args[:2], args[1:]...)\n\t\t\targs[1] = an\n\t\t}\n\t}\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 0, 1, 2, 3, 4:\n\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Parse Error: '%s'\", args)\n\tcase 5:\n\t\tc.pa.reply = nil\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[3]\n\t\tc.pa.hdr = parseSize(args[3])\n\t\tc.pa.szb = args[4]\n\t\tc.pa.size = parseSize(args[4])\n\tcase 6:\n\t\tc.pa.reply = args[3]\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[4]\n\t\tc.pa.hdr = parseSize(args[4])\n\t\tc.pa.szb = args[5]\n\t\tc.pa.size = parseSize(args[5])\n\tdefault:\n\t\t// args[2] is our reply indicator. Should be + or | normally.\n\t\tif len(args[3]) != 1 {\n\t\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'\", args[3])\n\t\t}\n\t\tswitch args[3][0] {\n\t\tcase '+':\n\t\t\tc.pa.reply = args[4]\n\t\tcase '|':\n\t\t\tc.pa.reply = nil\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Bad or Missing Reply Indicator: '%s'\", args[3])\n\t\t}\n\n\t\t// Grab header size.\n\t\tc.pa.hdb = args[len(args)-2]\n\t\tc.pa.hdr = parseSize(c.pa.hdb)\n\n\t\t// Grab size.\n\t\tc.pa.szb = args[len(args)-1]\n\t\tc.pa.size = parseSize(c.pa.szb)\n\n\t\t// Grab queue names.\n\t\tif c.pa.reply != nil {\n\t\t\tc.pa.queues = args[5 : len(args)-2]\n\t\t} else {\n\t\t\tc.pa.queues = args[4 : len(args)-2]\n\t\t}\n\t}\n\tif c.pa.hdr < 0 {\n\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Bad or Missing Header Size: '%s'\", arg)\n\t}\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Bad or Missing Size: '%s'\", args)\n\t}\n\tif c.pa.hdr > c.pa.size {\n\t\treturn fmt.Errorf(\"processRoutedOriginClusterMsgArgs Header Size larger then TotalSize: '%s'\", arg)\n\t}\n\n\t// Common ones processed after check for arg length\n\tc.pa.origin = args[0]\n\tc.pa.account = args[1]\n\tc.pa.subject = args[2]\n\tif len(an) > 0 {\n\t\tc.pa.pacache = c.pa.subject\n\t} else {\n\t\tc.pa.pacache = arg[len(args[0])+1 : len(args[0])+len(args[1])+len(args[2])+2]\n\t}\n\treturn nil\n}\n\n// Process an inbound HMSG specification from the remote route.\nfunc (c *client) processRoutedHeaderMsgArgs(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\targs := c.argsa[:0]\n\tvar an []byte\n\tif c.kind == ROUTER {\n\t\tif an = c.route.accName; len(an) > 0 {\n\t\t\targs = append(args, an)\n\t\t}\n\t}\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 0, 1, 2, 3:\n\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Parse Error: '%s'\", args)\n\tcase 4:\n\t\tc.pa.reply = nil\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[2]\n\t\tc.pa.hdr = parseSize(args[2])\n\t\tc.pa.szb = args[3]\n\t\tc.pa.size = parseSize(args[3])\n\tcase 5:\n\t\tc.pa.reply = args[2]\n\t\tc.pa.queues = nil\n\t\tc.pa.hdb = args[3]\n\t\tc.pa.hdr = parseSize(args[3])\n\t\tc.pa.szb = args[4]\n\t\tc.pa.size = parseSize(args[4])\n\tdefault:\n\t\t// args[2] is our reply indicator. Should be + or | normally.\n\t\tif len(args[2]) != 1 {\n\t\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'\", args[2])\n\t\t}\n\t\tswitch args[2][0] {\n\t\tcase '+':\n\t\t\tc.pa.reply = args[3]\n\t\tcase '|':\n\t\t\tc.pa.reply = nil\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Bad or Missing Reply Indicator: '%s'\", args[2])\n\t\t}\n\n\t\t// Grab header size.\n\t\tc.pa.hdb = args[len(args)-2]\n\t\tc.pa.hdr = parseSize(c.pa.hdb)\n\n\t\t// Grab size.\n\t\tc.pa.szb = args[len(args)-1]\n\t\tc.pa.size = parseSize(c.pa.szb)\n\n\t\t// Grab queue names.\n\t\tif c.pa.reply != nil {\n\t\t\tc.pa.queues = args[4 : len(args)-2]\n\t\t} else {\n\t\t\tc.pa.queues = args[3 : len(args)-2]\n\t\t}\n\t}\n\tif c.pa.hdr < 0 {\n\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Bad or Missing Header Size: '%s'\", arg)\n\t}\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Bad or Missing Size: '%s'\", args)\n\t}\n\tif c.pa.hdr > c.pa.size {\n\t\treturn fmt.Errorf(\"processRoutedHeaderMsgArgs Header Size larger then TotalSize: '%s'\", arg)\n\t}\n\n\t// Common ones processed after check for arg length\n\tc.pa.account = args[0]\n\tc.pa.subject = args[1]\n\tif len(an) > 0 {\n\t\tc.pa.pacache = c.pa.subject\n\t} else {\n\t\tc.pa.pacache = arg[:len(args[0])+len(args[1])+1]\n\t}\n\treturn nil\n}\n\n// Process an inbound RMSG or LMSG specification from the remote route.\nfunc (c *client) processRoutedMsgArgs(arg []byte) error {\n\t// Unroll splitArgs to avoid runtime/heap issues\n\targs := c.argsa[:0]\n\tvar an []byte\n\tif c.kind == ROUTER {\n\t\tif an = c.route.accName; len(an) > 0 {\n\t\t\targs = append(args, an)\n\t\t}\n\t}\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tc.pa.arg = arg\n\tswitch len(args) {\n\tcase 0, 1, 2:\n\t\treturn fmt.Errorf(\"processRoutedMsgArgs Parse Error: '%s'\", args)\n\tcase 3:\n\t\tc.pa.reply = nil\n\t\tc.pa.queues = nil\n\t\tc.pa.szb = args[2]\n\t\tc.pa.size = parseSize(args[2])\n\tcase 4:\n\t\tc.pa.reply = args[2]\n\t\tc.pa.queues = nil\n\t\tc.pa.szb = args[3]\n\t\tc.pa.size = parseSize(args[3])\n\tdefault:\n\t\t// args[2] is our reply indicator. Should be + or | normally.\n\t\tif len(args[2]) != 1 {\n\t\t\treturn fmt.Errorf(\"processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'\", args[2])\n\t\t}\n\t\tswitch args[2][0] {\n\t\tcase '+':\n\t\t\tc.pa.reply = args[3]\n\t\tcase '|':\n\t\t\tc.pa.reply = nil\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'\", args[2])\n\t\t}\n\t\t// Grab size.\n\t\tc.pa.szb = args[len(args)-1]\n\t\tc.pa.size = parseSize(c.pa.szb)\n\n\t\t// Grab queue names.\n\t\tif c.pa.reply != nil {\n\t\t\tc.pa.queues = args[4 : len(args)-1]\n\t\t} else {\n\t\t\tc.pa.queues = args[3 : len(args)-1]\n\t\t}\n\t}\n\tif c.pa.size < 0 {\n\t\treturn fmt.Errorf(\"processRoutedMsgArgs Bad or Missing Size: '%s'\", args)\n\t}\n\n\t// Common ones processed after check for arg length\n\tc.pa.account = args[0]\n\tc.pa.subject = args[1]\n\tif len(an) > 0 {\n\t\tc.pa.pacache = c.pa.subject\n\t} else {\n\t\tc.pa.pacache = arg[:len(args[0])+len(args[1])+1]\n\t}\n\treturn nil\n}\n\n// processInboundRoutedMsg is called to process an inbound msg from a route.\nfunc (c *client) processInboundRoutedMsg(msg []byte) {\n\t// Update statistics\n\tc.in.msgs++\n\t// The msg includes the CR_LF, so pull back out for accounting.\n\tsize := len(msg) - LEN_CR_LF\n\tc.in.bytes += int32(size)\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\n\t// Mostly under testing scenarios.\n\tif c.srv == nil {\n\t\treturn\n\t}\n\n\t// If the subject (c.pa.subject) has the gateway prefix, this function will handle it.\n\tif c.handleGatewayReply(msg) {\n\t\t// We are done here.\n\t\treturn\n\t}\n\n\tacc, r := c.getAccAndResultFromCache()\n\tif acc == nil {\n\t\tc.Debugf(\"Unknown account %q for routed message on subject: %q\", c.pa.account, c.pa.subject)\n\t\treturn\n\t}\n\n\tacc.stats.Lock()\n\tacc.stats.inMsgs++\n\tacc.stats.inBytes += int64(size)\n\tacc.stats.rt.inMsgs++\n\tacc.stats.rt.inBytes += int64(size)\n\tacc.stats.Unlock()\n\n\t// Check for no interest, short circuit if so.\n\t// This is the fanout scale.\n\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\tc.processMsgResults(acc, r, msg, nil, c.pa.subject, c.pa.reply, pmrNoFlag)\n\t}\n}\n\n// Lock should be held entering here.\nfunc (c *client) sendRouteConnect(clusterName string, tlsRequired bool) error {\n\tvar user, pass string\n\tif userInfo := c.route.url.User; userInfo != nil {\n\t\tuser = userInfo.Username()\n\t\tpass, _ = userInfo.Password()\n\t}\n\ts := c.srv\n\tcinfo := connectInfo{\n\t\tEcho: true,\n\t\tVerbose: false,\n\t\tPedantic: false,\n\t\tUser: user,\n\t\tPass: pass,\n\t\tTLS: tlsRequired,\n\t\tName: s.info.ID,\n\t\tHeaders: s.supportsHeaders(),\n\t\tCluster: clusterName,\n\t\tDynamic: s.isClusterNameDynamic(),\n\t\tLNOC: true,\n\t}\n\n\tb, err := json.Marshal(cinfo)\n\tif err != nil {\n\t\tc.Errorf(\"Error marshaling CONNECT to route: %v\\n\", err)\n\t\treturn err\n\t}\n\tc.enqueueProto([]byte(fmt.Sprintf(ConProto, b)))\n\treturn nil\n}\n\n// Returns a route pool index for this account based on the given pool size.\n// If `poolSize` is smaller or equal to 1, the returned value will always\n// be 0, regardless of the account name. If not, the returned value will\n// be in the range [0..poolSize-1]. The value for a given account name\n// is constant and same on all servers (given the same `poolSize` value).\nfunc computeRoutePoolIdx(poolSize int, an string) int {\n\tif poolSize <= 1 {\n\t\treturn 0\n\t}\n\th := fnv.New32a()\n\th.Write([]byte(an))\n\tsum32 := h.Sum32()\n\treturn int((sum32 % uint32(poolSize)))\n}\n\n// Process the info message if we are a route.\nfunc (c *client) processRouteInfo(info *Info) {\n\n\tsupportsHeaders := c.srv.supportsHeaders()\n\tclusterName := c.srv.ClusterName()\n\tsrvName := c.srv.Name()\n\n\tc.mu.Lock()\n\t// Connection can be closed at any time (by auth timeout, etc).\n\t// Does not make sense to continue here if connection is gone.\n\tif c.route == nil || c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn\n\t}\n\n\ts := c.srv\n\n\t// Detect route to self.\n\tif info.ID == s.info.ID {\n\t\t// Need to set this so that the close does the right thing\n\t\tc.route.remoteID = info.ID\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(DuplicateRoute)\n\t\treturn\n\t}\n\n\t// Detect if we have a mismatch of cluster names.\n\tif info.Cluster != \"\" && info.Cluster != clusterName {\n\t\tc.mu.Unlock()\n\t\t// If we are dynamic we may update our cluster name.\n\t\t// Use other if remote is non dynamic or their name is \"bigger\"\n\t\tif s.isClusterNameDynamic() && (!info.Dynamic || (strings.Compare(clusterName, info.Cluster) < 0)) {\n\t\t\ts.setClusterName(info.Cluster)\n\t\t\ts.removeAllRoutesExcept(info.ID)\n\t\t\tc.mu.Lock()\n\t\t} else {\n\t\t\tc.closeConnection(ClusterNameConflict)\n\t\t\treturn\n\t\t}\n\t}\n\n\topts := s.getOpts()\n\n\tdidSolicit := c.route.didSolicit\n\n\t// If this is an async INFO from an existing route...\n\tif c.flags.isSet(infoReceived) {\n\t\tremoteID := c.route.remoteID\n\n\t\t// Check if this is an INFO about adding a per-account route during\n\t\t// a configuration reload.\n\t\tif info.RouteAccReqID != _EMPTY_ {\n\t\t\tc.mu.Unlock()\n\n\t\t\t// If there is an account name, then the remote server is telling\n\t\t\t// us that this account will now have its dedicated route.\n\t\t\tif an := info.RouteAccount; an != _EMPTY_ {\n\t\t\t\tacc, err := s.LookupAccount(an)\n\t\t\t\tif err != nil {\n\t\t\t\t\ts.Errorf(\"Error looking up account %q: %v\", an, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\ts.mu.Lock()\n\t\t\t\t// If running without system account and adding a dedicated\n\t\t\t\t// route for an account for the first time, it could be that\n\t\t\t\t// the map is nil. If so, create it.\n\t\t\t\tif s.accRoutes == nil {\n\t\t\t\t\ts.accRoutes = make(map[string]map[string]*client)\n\t\t\t\t}\n\t\t\t\tif _, ok := s.accRoutes[an]; !ok {\n\t\t\t\t\ts.accRoutes[an] = make(map[string]*client)\n\t\t\t\t}\n\t\t\t\tacc.mu.Lock()\n\t\t\t\tsl := acc.sl\n\t\t\t\trpi := acc.routePoolIdx\n\t\t\t\t// Make sure that the account was not already switched.\n\t\t\t\tif rpi >= 0 {\n\t\t\t\t\ts.setRouteInfo(acc)\n\t\t\t\t\t// Change the route pool index to indicate that this\n\t\t\t\t\t// account is actually transitioning. This will be used\n\t\t\t\t\t// to suppress possible remote subscription interest coming\n\t\t\t\t\t// in while the transition is happening.\n\t\t\t\t\tacc.routePoolIdx = accTransitioningToDedicatedRoute\n\t\t\t\t}\n\t\t\t\tacc.mu.Unlock()\n\t\t\t\t// Since v2.11.0, we support remotes with a different pool size\n\t\t\t\t// (for rolling upgrades), so we need to use the remote route\n\t\t\t\t// pool index (based on the remote configured pool size) since\n\t\t\t\t// the remote subscriptions will be attached to the route at\n\t\t\t\t// that index, not at our account's route pool index. But we\n\t\t\t\t// need to compute only if rpi is negative or the pool sizes\n\t\t\t\t// are different.\n\t\t\t\tif rpi <= 0 || info.RoutePoolSize != s.routesPoolSize {\n\t\t\t\t\trpi = computeRoutePoolIdx(info.RoutePoolSize, an)\n\t\t\t\t}\n\t\t\t\t// Go over each remote's route at pool index `rpi` and remove\n\t\t\t\t// remote subs for this account.\n\t\t\t\ts.forEachRouteIdx(rpi, func(r *client) bool {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\t// Exclude routes to servers that don't support pooling.\n\t\t\t\t\tif !r.route.noPool {\n\t\t\t\t\t\tif subs := r.removeRemoteSubsForAcc(an); len(subs) > 0 {\n\t\t\t\t\t\t\tsl.RemoveBatch(subs)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t\t// Respond to the remote by clearing the RouteAccount field.\n\t\t\t\tinfo.RouteAccount = _EMPTY_\n\t\t\t\tproto := generateInfoJSON(info)\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.enqueueProto(proto)\n\t\t\t\tc.mu.Unlock()\n\t\t\t\ts.mu.Unlock()\n\t\t\t} else {\n\t\t\t\t// If no account name is specified, this is a response from the\n\t\t\t\t// remote. Simply send to the communication channel, if the\n\t\t\t\t// request ID matches the current one.\n\t\t\t\ts.mu.Lock()\n\t\t\t\tif info.RouteAccReqID == s.accAddedReqID && s.accAddedCh != nil {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase s.accAddedCh <- struct{}{}:\n\t\t\t\t\tdefault:\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.mu.Unlock()\n\t\t\t}\n\t\t\t// In both cases, we are done here.\n\t\t\treturn\n\t\t}\n\n\t\t// Check if this is an INFO for gateways...\n\t\tif info.Gateway != _EMPTY_ {\n\t\t\tc.mu.Unlock()\n\t\t\t// If this server has no gateway configured, report error and return.\n\t\t\tif !s.gateway.enabled {\n\t\t\t\t// FIXME: Should this be a Fatalf()?\n\t\t\t\ts.Errorf(\"Received information about gateway %q from %s, but gateway is not configured\",\n\t\t\t\t\tinfo.Gateway, remoteID)\n\t\t\t\treturn\n\t\t\t}\n\t\t\ts.processGatewayInfoFromRoute(info, remoteID)\n\t\t\treturn\n\t\t}\n\n\t\t// We receive an INFO from a server that informs us about another server,\n\t\t// so the info.ID in the INFO protocol does not match the ID of this route.\n\t\tif remoteID != _EMPTY_ && remoteID != info.ID {\n\t\t\t// We want to know if the existing route supports pooling/pinned-account\n\t\t\t// or not when processing the implicit route.\n\t\t\tnoPool := c.route.noPool\n\t\t\tc.mu.Unlock()\n\n\t\t\t// Process this implicit route. We will check that it is not an explicit\n\t\t\t// route and/or that it has not been connected already.\n\t\t\ts.processImplicitRoute(info, noPool)\n\t\t\treturn\n\t\t}\n\n\t\tvar connectURLs []string\n\t\tvar wsConnectURLs []string\n\t\tvar updateRoutePerms bool\n\n\t\t// If we are notified that the remote is going into LDM mode, capture route's connectURLs.\n\t\tif info.LameDuckMode {\n\t\t\tconnectURLs = c.route.connectURLs\n\t\t\twsConnectURLs = c.route.wsConnURLs\n\t\t} else {\n\t\t\t// Update only if we detect a difference\n\t\t\tupdateRoutePerms = !reflect.DeepEqual(c.opts.Import, info.Import) || !reflect.DeepEqual(c.opts.Export, info.Export)\n\t\t}\n\t\tc.mu.Unlock()\n\n\t\tif updateRoutePerms {\n\t\t\ts.updateRemoteRoutePerms(c, info)\n\t\t}\n\n\t\t// If the remote is going into LDM and there are client connect URLs\n\t\t// associated with this route and we are allowed to advertise, remove\n\t\t// those URLs and update our clients.\n\t\tif (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise {\n\t\t\ts.mu.Lock()\n\t\t\ts.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs)\n\t\t\ts.mu.Unlock()\n\t\t}\n\t\treturn\n\t}\n\n\t// Check if remote has same server name than this server.\n\tif !didSolicit && info.Name == srvName {\n\t\tc.mu.Unlock()\n\t\t// This is now an error and we close the connection. We need unique names for JetStream clustering.\n\t\tc.Errorf(\"Remote server has a duplicate name: %q\", info.Name)\n\t\tc.closeConnection(DuplicateServerName)\n\t\treturn\n\t}\n\n\tvar sendDelayedInfo bool\n\n\t// First INFO, check if this server is configured for compression because\n\t// if that is the case, we need to negotiate it with the remote server.\n\tif needsCompression(opts.Cluster.Compression.Mode) {\n\t\taccName := bytesToString(c.route.accName)\n\t\t// If we did not yet negotiate...\n\t\tcompNeg := c.flags.isSet(compressionNegotiated)\n\t\tif !compNeg {\n\t\t\t// Prevent from getting back here.\n\t\t\tc.flags.set(compressionNegotiated)\n\t\t\t// Release client lock since following function will need server lock.\n\t\t\tc.mu.Unlock()\n\t\t\tcompress, err := s.negotiateRouteCompression(c, didSolicit, accName, info.Compression, opts)\n\t\t\tif err != nil {\n\t\t\t\tc.sendErrAndErr(err.Error())\n\t\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif compress {\n\t\t\t\t// Done for now, will get back another INFO protocol...\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// No compression because one side does not want/can't, so proceed.\n\t\t\tc.mu.Lock()\n\t\t\t// Check that the connection did not close if the lock was released.\n\t\t\tif c.isClosed() {\n\t\t\t\tc.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// We can set the ping timer after we just negotiated compression above,\n\t\t// or for solicited routes if we already negotiated.\n\t\tif !compNeg || didSolicit {\n\t\t\tc.setFirstPingTimer()\n\t\t}\n\t\t// When compression is configured, we delay the initial INFO for any\n\t\t// solicited route. So we need to send the delayed INFO simply based\n\t\t// on the didSolicit boolean.\n\t\tsendDelayedInfo = didSolicit\n\t} else {\n\t\t// Coming from an old server, the Compression field would be the empty\n\t\t// string. For servers that are configured with CompressionNotSupported,\n\t\t// this makes them behave as old servers.\n\t\tif info.Compression == _EMPTY_ || opts.Cluster.Compression.Mode == CompressionNotSupported {\n\t\t\tc.route.compression = CompressionNotSupported\n\t\t} else {\n\t\t\tc.route.compression = CompressionOff\n\t\t}\n\t\t// When compression is not configured, we delay the initial INFO only\n\t\t// for solicited pooled routes, so use the same check that we did when\n\t\t// we decided to delay in createRoute().\n\t\tsendDelayedInfo = didSolicit && routeShouldDelayInfo(bytesToString(c.route.accName), opts)\n\t}\n\n\t// Mark that the INFO protocol has been received, so we can detect updates.\n\tc.flags.set(infoReceived)\n\n\t// Get the route's proto version. It will be used to check if the connection\n\t// supports certain features, such as message tracing.\n\tc.opts.Protocol = info.Proto\n\n\t// Headers\n\tc.headers = supportsHeaders && info.Headers\n\n\t// Copy over important information.\n\tc.route.remoteID = info.ID\n\tc.route.authRequired = info.AuthRequired\n\tc.route.tlsRequired = info.TLSRequired\n\tc.route.gatewayURL = info.GatewayURL\n\tc.route.remoteName = info.Name\n\tc.route.lnoc = info.LNOC\n\tc.route.lnocu = info.LNOCU\n\tc.route.jetstream = info.JetStream\n\n\t// When sent through route INFO, if the field is set, it should be of size 1.\n\tif len(info.LeafNodeURLs) == 1 {\n\t\tc.route.leafnodeURL = info.LeafNodeURLs[0]\n\t}\n\t// Compute the hash of this route based on remote server name\n\tc.route.hash = getHash(info.Name)\n\t// Same with remote server ID (used for GW mapped replies routing).\n\t// Use getGWHash since we don't use the same hash len for that\n\t// for backward compatibility.\n\tc.route.idHash = string(getGWHash(info.ID))\n\n\t// Copy over permissions as well.\n\tc.opts.Import = info.Import\n\tc.opts.Export = info.Export\n\n\t// If we do not know this route's URL, construct one on the fly\n\t// from the information provided.\n\tif c.route.url == nil {\n\t\t// Add in the URL from host and port\n\t\thp := net.JoinHostPort(info.Host, strconv.Itoa(info.Port))\n\t\turl, err := url.Parse(fmt.Sprintf(\"nats-route://%s/\", hp))\n\t\tif err != nil {\n\t\t\tc.Errorf(\"Error parsing URL from INFO: %v\\n\", err)\n\t\t\tc.mu.Unlock()\n\t\t\tc.closeConnection(ParseError)\n\t\t\treturn\n\t\t}\n\t\tc.route.url = url\n\t}\n\t// The incoming INFO from the route will have IP set\n\t// if it has Cluster.Advertise. In that case, use that\n\t// otherwise construct it from the remote TCP address.\n\tif info.IP == _EMPTY_ {\n\t\t// Need to get the remote IP address.\n\t\tswitch conn := c.nc.(type) {\n\t\tcase *net.TCPConn, *tls.Conn:\n\t\t\taddr := conn.RemoteAddr().(*net.TCPAddr)\n\t\t\tinfo.IP = fmt.Sprintf(\"nats-route://%s/\", net.JoinHostPort(addr.IP.String(),\n\t\t\t\tstrconv.Itoa(info.Port)))\n\t\tdefault:\n\t\t\tinfo.IP = c.route.url.String()\n\t\t}\n\t}\n\t// For accounts that are configured to have their own route:\n\t// If this is a solicited route, we already have c.route.accName set in createRoute.\n\t// For non solicited route (the accept side), we will set the account name that\n\t// is present in the INFO protocol.\n\tif didSolicit && len(c.route.accName) > 0 {\n\t\t// Set it in the info.RouteAccount so that addRoute can use that\n\t\t// and we properly gossip that this is a route for an account.\n\t\tinfo.RouteAccount = string(c.route.accName)\n\t} else if !didSolicit && info.RouteAccount != _EMPTY_ {\n\t\tc.route.accName = []byte(info.RouteAccount)\n\t}\n\taccName := string(c.route.accName)\n\n\t// Capture the noGossip value and reset it here.\n\tgossipMode := c.route.gossipMode\n\tc.route.gossipMode = 0\n\n\t// Check to see if we have this remote already registered.\n\t// This can happen when both servers have routes to each other.\n\tc.mu.Unlock()\n\n\tif added := s.addRoute(c, didSolicit, sendDelayedInfo, gossipMode, info, accName); added {\n\t\tif accName != _EMPTY_ {\n\t\t\tc.Debugf(\"Registering remote route %q for account %q\", info.ID, accName)\n\t\t} else {\n\t\t\tc.Debugf(\"Registering remote route %q\", info.ID)\n\t\t}\n\t} else {\n\t\tc.Debugf(\"Detected duplicate remote route %q\", info.ID)\n\t\tc.closeConnection(DuplicateRoute)\n\t}\n}\n\nfunc (s *Server) negotiateRouteCompression(c *client, didSolicit bool, accName, infoCompression string, opts *Options) (bool, error) {\n\t// Negotiate the appropriate compression mode (or no compression)\n\tcm, err := selectCompressionMode(opts.Cluster.Compression.Mode, infoCompression)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tc.mu.Lock()\n\t// For \"auto\" mode, set the initial compression mode based on RTT\n\tif cm == CompressionS2Auto {\n\t\tif c.rttStart.IsZero() {\n\t\t\tc.rtt = computeRTT(c.start)\n\t\t}\n\t\tcm = selectS2AutoModeBasedOnRTT(c.rtt, opts.Cluster.Compression.RTTThresholds)\n\t}\n\t// Keep track of the negotiated compression mode.\n\tc.route.compression = cm\n\tc.mu.Unlock()\n\n\t// If we end-up doing compression...\n\tif needsCompression(cm) {\n\t\t// Generate an INFO with the chosen compression mode.\n\t\ts.mu.Lock()\n\t\tinfoProto := s.generateRouteInitialInfoJSON(accName, cm, 0, gossipDefault)\n\t\ts.mu.Unlock()\n\n\t\t// If we solicited, then send this INFO protocol BEFORE switching\n\t\t// to compression writer. However, if we did not, we send it after.\n\t\tc.mu.Lock()\n\t\tif didSolicit {\n\t\t\tc.enqueueProto(infoProto)\n\t\t\t// Make sure it is completely flushed (the pending bytes goes to\n\t\t\t// 0) before proceeding.\n\t\t\tfor c.out.pb > 0 && !c.isClosed() {\n\t\t\t\tc.flushOutbound()\n\t\t\t}\n\t\t}\n\t\t// This is to notify the readLoop that it should switch to a\n\t\t// (de)compression reader.\n\t\tc.in.flags.set(switchToCompression)\n\t\t// Create the compress writer before queueing the INFO protocol for\n\t\t// a route that did not solicit. It will make sure that that proto\n\t\t// is sent with compression on.\n\t\tc.out.cw = s2.NewWriter(nil, s2WriterOptions(cm)...)\n\t\tif !didSolicit {\n\t\t\tc.enqueueProto(infoProto)\n\t\t}\n\t\t// We can now set the ping timer.\n\t\tc.setFirstPingTimer()\n\t\tc.mu.Unlock()\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\n// Possibly sends local subscriptions interest to this route\n// based on changes in the remote's Export permissions.\nfunc (s *Server) updateRemoteRoutePerms(c *client, info *Info) {\n\tc.mu.Lock()\n\t// Interested only on Export permissions for the remote server.\n\t// Create \"fake\" clients that we will use to check permissions\n\t// using the old permissions...\n\toldPerms := &RoutePermissions{Export: c.opts.Export}\n\toldPermsTester := &client{}\n\toldPermsTester.setRoutePermissions(oldPerms)\n\t// and the new ones.\n\tnewPerms := &RoutePermissions{Export: info.Export}\n\tnewPermsTester := &client{}\n\tnewPermsTester.setRoutePermissions(newPerms)\n\n\tc.opts.Import = info.Import\n\tc.opts.Export = info.Export\n\n\trouteAcc, poolIdx, noPool := bytesToString(c.route.accName), c.route.poolIdx, c.route.noPool\n\tc.mu.Unlock()\n\n\tvar (\n\t\t_localSubs [4096]*subscription\n\t\t_allSubs [4096]*subscription\n\t\tallSubs = _allSubs[:0]\n\t)\n\n\ts.accounts.Range(func(_, v any) bool {\n\t\tacc := v.(*Account)\n\t\tacc.mu.RLock()\n\t\taccName, sl, accPoolIdx := acc.Name, acc.sl, acc.routePoolIdx\n\t\tacc.mu.RUnlock()\n\n\t\t// Do this only for accounts handled by this route\n\t\tif (accPoolIdx >= 0 && accPoolIdx == poolIdx) || (routeAcc == accName) || noPool {\n\t\t\tlocalSubs := _localSubs[:0]\n\t\t\tsl.localSubs(&localSubs, false)\n\t\t\tif len(localSubs) > 0 {\n\t\t\t\tallSubs = append(allSubs, localSubs...)\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\tif len(allSubs) == 0 {\n\t\treturn\n\t}\n\n\tc.mu.Lock()\n\tc.sendRouteSubProtos(allSubs, false, func(sub *subscription) bool {\n\t\tsubj := string(sub.subject)\n\t\t// If the remote can now export but could not before, and this server can import this\n\t\t// subject, then send SUB protocol.\n\t\tif newPermsTester.canExport(subj) && !oldPermsTester.canExport(subj) && c.canImport(subj) {\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t})\n\tc.mu.Unlock()\n}\n\n// sendAsyncInfoToClients sends an INFO protocol to all\n// connected clients that accept async INFO updates.\n// The server lock is held on entry.\nfunc (s *Server) sendAsyncInfoToClients(regCli, wsCli bool) {\n\t// If there are no clients supporting async INFO protocols, we are done.\n\t// Also don't send if we are shutting down...\n\tif s.cproto == 0 || s.isShuttingDown() {\n\t\treturn\n\t}\n\tinfo := s.copyInfo()\n\n\tfor _, c := range s.clients {\n\t\tc.mu.Lock()\n\t\t// Here, we are going to send only to the clients that are fully\n\t\t// registered (server has received CONNECT and first PING). For\n\t\t// clients that are not at this stage, this will happen in the\n\t\t// processing of the first PING (see client.processPing)\n\t\tif ((regCli && !c.isWebsocket()) || (wsCli && c.isWebsocket())) &&\n\t\t\tc.opts.Protocol >= ClientProtoInfo &&\n\t\t\tc.flags.isSet(firstPongSent) {\n\t\t\t// sendInfo takes care of checking if the connection is still\n\t\t\t// valid or not, so don't duplicate tests here.\n\t\t\tc.enqueueProto(c.generateClientInfoJSON(info, true))\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n}\n\n// This will process implicit route information received from another server.\n// We will check to see if we have configured or are already connected,\n// and if so we will ignore. Otherwise we will attempt to connect.\nfunc (s *Server) processImplicitRoute(info *Info, routeNoPool bool) {\n\tremoteID := info.ID\n\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\t// Don't connect to ourself\n\tif remoteID == s.info.ID {\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Check if this route already exists\n\tif accName := info.RouteAccount; accName != _EMPTY_ {\n\t\t// If we don't support pooling/pinned account, bail.\n\t\tif opts.Cluster.PoolSize <= 0 {\n\t\t\treturn\n\t\t}\n\t\tif remotes, ok := s.accRoutes[accName]; ok {\n\t\t\tif r := remotes[remoteID]; r != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t} else if _, exists := s.routes[remoteID]; exists {\n\t\treturn\n\t}\n\t// Check if we have this route as a configured route\n\tif s.hasThisRouteConfigured(info) {\n\t\treturn\n\t}\n\n\t// Initiate the connection, using info.IP instead of info.URL here...\n\tr, err := url.Parse(info.IP)\n\tif err != nil {\n\t\ts.Errorf(\"Error parsing URL from INFO: %v\\n\", err)\n\t\treturn\n\t}\n\n\tif info.AuthRequired {\n\t\tr.User = url.UserPassword(opts.Cluster.Username, opts.Cluster.Password)\n\t}\n\ts.startGoRoutine(func() { s.connectToRoute(r, Implicit, true, info.GossipMode, info.RouteAccount) })\n\t// If we are processing an implicit route from a route that does not\n\t// support pooling/pinned-accounts, we won't receive an INFO for each of\n\t// the pinned-accounts that we would normally receive. In that case, just\n\t// initiate routes for all our configured pinned accounts.\n\tif routeNoPool && info.RouteAccount == _EMPTY_ && len(opts.Cluster.PinnedAccounts) > 0 {\n\t\t// Copy since we are going to pass as closure to a go routine.\n\t\trURL := r\n\t\tfor _, an := range opts.Cluster.PinnedAccounts {\n\t\t\taccName := an\n\t\t\ts.startGoRoutine(func() { s.connectToRoute(rURL, Implicit, true, info.GossipMode, accName) })\n\t\t}\n\t}\n}\n\n// hasThisRouteConfigured returns true if info.Host:info.Port is present\n// in the server's opts.Routes, false otherwise.\n// Server lock is assumed to be held by caller.\nfunc (s *Server) hasThisRouteConfigured(info *Info) bool {\n\troutes := s.getOpts().Routes\n\tif len(routes) == 0 {\n\t\treturn false\n\t}\n\t// This could possibly be a 0.0.0.0 host so we will also construct a second\n\t// url with the host section of the `info.IP` (if present).\n\tsPort := strconv.Itoa(info.Port)\n\turlOne := strings.ToLower(net.JoinHostPort(info.Host, sPort))\n\tvar urlTwo string\n\tif info.IP != _EMPTY_ {\n\t\tif u, _ := url.Parse(info.IP); u != nil {\n\t\t\turlTwo = strings.ToLower(net.JoinHostPort(u.Hostname(), sPort))\n\t\t\t// Ignore if same than the first\n\t\t\tif urlTwo == urlOne {\n\t\t\t\turlTwo = _EMPTY_\n\t\t\t}\n\t\t}\n\t}\n\tfor _, ri := range routes {\n\t\trHost := strings.ToLower(ri.Host)\n\t\tif rHost == urlOne {\n\t\t\treturn true\n\t\t}\n\t\tif urlTwo != _EMPTY_ && rHost == urlTwo {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// forwardNewRouteInfoToKnownServers possibly sends the INFO protocol of the\n// new route to all routes known by this server. In turn, each server will\n// contact this new route.\n// Server lock held on entry.\nfunc (s *Server) forwardNewRouteInfoToKnownServers(info *Info, rtype RouteType, didSolicit bool, localGossipMode byte) {\n\t// Determine if this connection is resulting from a gossip notification.\n\tfromGossip := didSolicit && rtype == Implicit\n\t// If from gossip (but we are not overriding it) or if the remote disabled gossip, bail out.\n\tif (fromGossip && localGossipMode != gossipOverride) || info.GossipMode == gossipDisabled {\n\t\treturn\n\t}\n\n\t// Note: nonce is not used in routes.\n\t// That being said, the info we get is the initial INFO which\n\t// contains a nonce, but we now forward this to existing routes,\n\t// so clear it now.\n\tinfo.Nonce = _EMPTY_\n\n\tvar (\n\t\tinfoGMDefault []byte\n\t\tinfoGMDisabled []byte\n\t\tinfoGMOverride []byte\n\t)\n\n\tgenerateJSON := func(gm byte) []byte {\n\t\tinfo.GossipMode = gm\n\t\tb, _ := json.Marshal(info)\n\t\treturn []byte(fmt.Sprintf(InfoProto, b))\n\t}\n\n\tgetJSON := func(r *client) []byte {\n\t\tif (!didSolicit && r.route.routeType == Explicit) || (didSolicit && rtype == Explicit) {\n\t\t\tif infoGMOverride == nil {\n\t\t\t\tinfoGMOverride = generateJSON(gossipOverride)\n\t\t\t}\n\t\t\treturn infoGMOverride\n\t\t} else if !didSolicit {\n\t\t\tif infoGMDisabled == nil {\n\t\t\t\tinfoGMDisabled = generateJSON(gossipDisabled)\n\t\t\t}\n\t\t\treturn infoGMDisabled\n\t\t}\n\t\tif infoGMDefault == nil {\n\t\t\tinfoGMDefault = generateJSON(0)\n\t\t}\n\t\treturn infoGMDefault\n\t}\n\n\tvar accRemotes map[string]*client\n\tpinnedAccount := info.RouteAccount != _EMPTY_\n\t// If this is for a pinned account, we will try to send the gossip\n\t// through our pinned account routes, but fall back to the other\n\t// routes in case we don't have one for a given remote.\n\tif pinnedAccount {\n\t\tvar ok bool\n\t\tif accRemotes, ok = s.accRoutes[info.RouteAccount]; ok {\n\t\t\tfor remoteID, r := range accRemotes {\n\t\t\t\tif r == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tr.mu.Lock()\n\t\t\t\t// Do not send to a remote that does not support pooling/pinned-accounts.\n\t\t\t\tif remoteID != info.ID && !r.route.noPool {\n\t\t\t\t\tr.enqueueProto(getJSON(r))\n\t\t\t\t}\n\t\t\t\tr.mu.Unlock()\n\t\t\t}\n\t\t}\n\t}\n\n\ts.forEachRemote(func(r *client) {\n\t\tr.mu.Lock()\n\t\tremoteID := r.route.remoteID\n\t\tif pinnedAccount {\n\t\t\tif _, processed := accRemotes[remoteID]; processed {\n\t\t\t\tr.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// If this is a new route for a given account, do not send to a server\n\t\t// that does not support pooling/pinned-accounts.\n\t\tif remoteID != info.ID && (!pinnedAccount || !r.route.noPool) {\n\t\t\tr.enqueueProto(getJSON(r))\n\t\t}\n\t\tr.mu.Unlock()\n\t})\n}\n\n// canImport is whether or not we will send a SUB for interest to the other side.\n// This is for ROUTER connections only.\n// Lock is held on entry.\nfunc (c *client) canImport(subject string) bool {\n\t// Use pubAllowed() since this checks Publish permissions which\n\t// is what Import maps to.\n\treturn c.pubAllowedFullCheck(subject, false, true)\n}\n\n// canExport is whether or not we will accept a SUB from the remote for a given subject.\n// This is for ROUTER connections only.\n// Lock is held on entry\nfunc (c *client) canExport(subject string) bool {\n\t// Use canSubscribe() since this checks Subscribe permissions which\n\t// is what Export maps to.\n\treturn c.canSubscribe(subject)\n}\n\n// Initialize or reset cluster's permissions.\n// This is for ROUTER connections only.\n// Client lock is held on entry\nfunc (c *client) setRoutePermissions(perms *RoutePermissions) {\n\t// Reset if some were set\n\tif perms == nil {\n\t\tc.perms = nil\n\t\tc.mperms = nil\n\t\treturn\n\t}\n\t// Convert route permissions to user permissions.\n\t// The Import permission is mapped to Publish\n\t// and Export permission is mapped to Subscribe.\n\t// For meaning of Import/Export, see canImport and canExport.\n\tp := &Permissions{\n\t\tPublish: perms.Import,\n\t\tSubscribe: perms.Export,\n\t}\n\tc.setPermissions(p)\n}\n\n// Type used to hold a list of subs on a per account basis.\ntype asubs struct {\n\tacc *Account\n\tsubs []*subscription\n}\n\n// Returns the account name from the subscription's key.\n// This is invoked knowing that the key contains an account name, so for a sub\n// that is not from a pinned-account route.\n// The `keyHasSubType` boolean indicates that the key starts with the indicator\n// for leaf or regular routed subscriptions.\nfunc getAccNameFromRoutedSubKey(sub *subscription, key string, keyHasSubType bool) string {\n\tvar accIdx int\n\tif keyHasSubType {\n\t\t// Start after the sub type indicator.\n\t\taccIdx = 1\n\t\t// But if there is an origin, bump its index.\n\t\tif len(sub.origin) > 0 {\n\t\t\taccIdx = 2\n\t\t}\n\t}\n\treturn strings.Fields(key)[accIdx]\n}\n\n// Returns if the route is dedicated to an account, its name, and a boolean\n// that indicates if this route uses the routed subscription indicator at\n// the beginning of the subscription key.\n// Lock held on entry.\nfunc (c *client) getRoutedSubKeyInfo() (bool, string, bool) {\n\tvar accName string\n\tif an := c.route.accName; len(an) > 0 {\n\t\taccName = string(an)\n\t}\n\treturn accName != _EMPTY_, accName, c.route.lnocu\n}\n\n// removeRemoteSubs will walk the subs and remove them from the appropriate account.\nfunc (c *client) removeRemoteSubs() {\n\t// We need to gather these on a per account basis.\n\t// FIXME(dlc) - We should be smarter about this..\n\tas := map[string]*asubs{}\n\tc.mu.Lock()\n\tsrv := c.srv\n\tsubs := c.subs\n\tc.subs = nil\n\tpa, accountName, hasSubType := c.getRoutedSubKeyInfo()\n\tc.mu.Unlock()\n\n\tfor key, sub := range subs {\n\t\tc.mu.Lock()\n\t\tsub.max = 0\n\t\tc.mu.Unlock()\n\t\t// If not a pinned-account route, we need to find the account\n\t\t// name from the sub's key.\n\t\tif !pa {\n\t\t\taccountName = getAccNameFromRoutedSubKey(sub, key, hasSubType)\n\t\t}\n\t\tase := as[accountName]\n\t\tif ase == nil {\n\t\t\tif v, ok := srv.accounts.Load(accountName); ok {\n\t\t\t\tase = &asubs{acc: v.(*Account), subs: []*subscription{sub}}\n\t\t\t\tas[accountName] = ase\n\t\t\t} else {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else {\n\t\t\tase.subs = append(ase.subs, sub)\n\t\t}\n\t\tdelta := int32(1)\n\t\tif len(sub.queue) > 0 {\n\t\t\tdelta = sub.qw\n\t\t}\n\t\tif srv.gateway.enabled {\n\t\t\tsrv.gatewayUpdateSubInterest(accountName, sub, -delta)\n\t\t}\n\t\tase.acc.updateLeafNodes(sub, -delta)\n\t}\n\n\t// Now remove the subs by batch for each account sublist.\n\tfor _, ase := range as {\n\t\tc.Debugf(\"Removing %d subscriptions for account %q\", len(ase.subs), ase.acc.Name)\n\t\tase.acc.mu.Lock()\n\t\tase.acc.sl.RemoveBatch(ase.subs)\n\t\tase.acc.mu.Unlock()\n\t}\n}\n\n// Removes (and returns) the subscriptions from this route's subscriptions map\n// that belong to the given account.\n// Lock is held on entry\nfunc (c *client) removeRemoteSubsForAcc(name string) []*subscription {\n\tvar subs []*subscription\n\t_, _, hasSubType := c.getRoutedSubKeyInfo()\n\tfor key, sub := range c.subs {\n\t\tan := getAccNameFromRoutedSubKey(sub, key, hasSubType)\n\t\tif an == name {\n\t\t\tsub.max = 0\n\t\t\tsubs = append(subs, sub)\n\t\t\tdelete(c.subs, key)\n\t\t}\n\t}\n\treturn subs\n}\n\nfunc (c *client) parseUnsubProto(arg []byte, accInProto, hasOrigin bool) ([]byte, string, []byte, []byte, error) {\n\t// Indicate any activity, so pub and sub or unsubs.\n\tc.in.subs++\n\n\targs := splitArg(arg)\n\n\tvar (\n\t\torigin []byte\n\t\taccountName string\n\t\tqueue []byte\n\t\tsubjIdx int\n\t)\n\t// If `hasOrigin` is true, then it means this is a LS- with origin in proto.\n\tif hasOrigin {\n\t\t// We would not be here if there was not at least 1 field.\n\t\torigin = args[0]\n\t\tsubjIdx = 1\n\t}\n\t// If there is an account in the protocol, bump the subject index.\n\tif accInProto {\n\t\tsubjIdx++\n\t}\n\n\tswitch len(args) {\n\tcase subjIdx + 1:\n\tcase subjIdx + 2:\n\t\tqueue = args[subjIdx+1]\n\tdefault:\n\t\treturn nil, _EMPTY_, nil, nil, fmt.Errorf(\"parse error: '%s'\", arg)\n\t}\n\tif accInProto {\n\t\t// If there is an account in the protocol, it is before the subject.\n\t\taccountName = string(args[subjIdx-1])\n\t}\n\treturn origin, accountName, args[subjIdx], queue, nil\n}\n\n// Indicates no more interest in the given account/subject for the remote side.\nfunc (c *client) processRemoteUnsub(arg []byte, leafUnsub bool) (err error) {\n\tsrv := c.srv\n\tif srv == nil {\n\t\treturn nil\n\t}\n\n\tvar accountName string\n\t// Assume the account will be in the protocol.\n\taccInProto := true\n\n\tc.mu.Lock()\n\toriginSupport := c.route.lnocu\n\tif c.route != nil && len(c.route.accName) > 0 {\n\t\taccountName, accInProto = string(c.route.accName), false\n\t}\n\tc.mu.Unlock()\n\n\thasOrigin := leafUnsub && originSupport\n\t_, accNameFromProto, subject, _, err := c.parseUnsubProto(arg, accInProto, hasOrigin)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"processRemoteUnsub %s\", err.Error())\n\t}\n\tif accInProto {\n\t\taccountName = accNameFromProto\n\t}\n\t// Lookup the account\n\tvar acc *Account\n\tif v, ok := srv.accounts.Load(accountName); ok {\n\t\tacc = v.(*Account)\n\t} else {\n\t\tc.Debugf(\"Unknown account %q for subject %q\", accountName, subject)\n\t\treturn nil\n\t}\n\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t_keya := [128]byte{}\n\t_key := _keya[:0]\n\n\tvar key string\n\tif !originSupport {\n\t\t// If it is an LS- or RS-, we use the protocol as-is as the key.\n\t\tkey = bytesToString(arg)\n\t} else {\n\t\t// We need to prefix with the sub type.\n\t\tif leafUnsub {\n\t\t\t_key = append(_key, keyRoutedLeafSubByte)\n\t\t} else {\n\t\t\t_key = append(_key, keyRoutedSubByte)\n\t\t}\n\t\t_key = append(_key, ' ')\n\t\t_key = append(_key, arg...)\n\t\tkey = bytesToString(_key)\n\t}\n\tdelta := int32(1)\n\tsub, ok := c.subs[key]\n\tif ok {\n\t\tdelete(c.subs, key)\n\t\tacc.sl.Remove(sub)\n\t\tif len(sub.queue) > 0 {\n\t\t\tdelta = sub.qw\n\t\t}\n\t}\n\tc.mu.Unlock()\n\n\t// Update gateways and leaf nodes only if the subscription was found.\n\tif ok {\n\t\tif srv.gateway.enabled {\n\t\t\tsrv.gatewayUpdateSubInterest(accountName, sub, -delta)\n\t\t}\n\n\t\t// Now check on leafnode updates.\n\t\tacc.updateLeafNodes(sub, -delta)\n\t}\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\treturn nil\n}\n\nfunc (c *client) processRemoteSub(argo []byte, hasOrigin bool) (err error) {\n\t// Indicate activity.\n\tc.in.subs++\n\n\tsrv := c.srv\n\tif srv == nil {\n\t\treturn nil\n\t}\n\n\t// We copy `argo` to not reference the read buffer. However, we will\n\t// prefix with a code that says if the remote sub is for a leaf\n\t// (hasOrigin == true) or not to prevent key collisions. Imagine:\n\t// \"RS+ foo bar baz 1\\r\\n\" => \"foo bar baz\" (a routed queue sub)\n\t// \"LS+ foo bar baz\\r\\n\" => \"foo bar baz\" (a route leaf sub on \"baz\",\n\t// for account \"bar\" with origin \"foo\").\n\t//\n\t// The sub.sid/key will be set respectively to \"R foo bar baz\" and\n\t// \"L foo bar baz\".\n\t//\n\t// We also no longer add the account if it was not present (due to\n\t// pinned-account route) since there is no need really.\n\t//\n\t// For routes to older server, we will still create the \"arg\" with\n\t// the above layout, but we will create the sub.sid/key as before,\n\t// that is, not including the origin for LS+ because older server\n\t// only send LS- without origin, so we would not be able to find\n\t// the sub in the map.\n\tc.mu.Lock()\n\taccountName := string(c.route.accName)\n\toldStyle := !c.route.lnocu\n\tc.mu.Unlock()\n\n\t// Indicate if the account name should be in the protocol. It would be the\n\t// case if accountName is empty.\n\taccInProto := accountName == _EMPTY_\n\n\t// Copy so we do not reference a potentially large buffer.\n\t// Add 2 more bytes for the routed sub type.\n\targ := make([]byte, 0, 2+len(argo))\n\tif hasOrigin {\n\t\targ = append(arg, keyRoutedLeafSubByte)\n\t} else {\n\t\targ = append(arg, keyRoutedSubByte)\n\t}\n\targ = append(arg, ' ')\n\targ = append(arg, argo...)\n\n\t// Now split to get all fields. Unroll splitArgs to avoid runtime/heap issues.\n\ta := [MAX_RSUB_ARGS][]byte{}\n\targs := a[:0]\n\tstart := -1\n\tfor i, b := range arg {\n\t\tswitch b {\n\t\tcase ' ', '\\t', '\\r', '\\n':\n\t\t\tif start >= 0 {\n\t\t\t\targs = append(args, arg[start:i])\n\t\t\t\tstart = -1\n\t\t\t}\n\t\tdefault:\n\t\t\tif start < 0 {\n\t\t\t\tstart = i\n\t\t\t}\n\t\t}\n\t}\n\tif start >= 0 {\n\t\targs = append(args, arg[start:])\n\t}\n\n\tdelta := int32(1)\n\tsub := &subscription{client: c}\n\n\t// There will always be at least a subject, but its location will depend\n\t// on if there is an origin, an account name, etc.. Since we know that\n\t// we have added the sub type indicator as the first field, the subject\n\t// position will be at minimum at index 1.\n\tsubjIdx := 1\n\tif hasOrigin {\n\t\tsubjIdx++\n\t}\n\tif accInProto {\n\t\tsubjIdx++\n\t}\n\tswitch len(args) {\n\tcase subjIdx + 1:\n\t\tsub.queue = nil\n\tcase subjIdx + 3:\n\t\tsub.queue = args[subjIdx+1]\n\t\tsub.qw = int32(parseSize(args[subjIdx+2]))\n\t\t// TODO: (ik) We should have a non empty queue name and a queue\n\t\t// weight >= 1. For 2.11, we may want to return an error if that\n\t\t// is not the case, but for now just overwrite `delta` if queue\n\t\t// weight is greater than 1 (it is possible after a reconnect/\n\t\t// server restart to receive a queue weight > 1 for a new sub).\n\t\tif sub.qw > 1 {\n\t\t\tdelta = sub.qw\n\t\t}\n\tdefault:\n\t\treturn fmt.Errorf(\"processRemoteSub Parse Error: '%s'\", arg)\n\t}\n\t// We know that the number of fields is correct. So we can access args[] based\n\t// on where we expect the fields to be.\n\n\t// If there is an origin, it will be at index 1.\n\tif hasOrigin {\n\t\tsub.origin = args[1]\n\t}\n\t// For subject, use subjIdx.\n\tsub.subject = args[subjIdx]\n\t// If the account name is in the protocol, it will be before the subject.\n\tif accInProto {\n\t\taccountName = bytesToString(args[subjIdx-1])\n\t}\n\t// Now set the sub.sid from the arg slice. However, we will have a different\n\t// one if we use the origin or not.\n\tstart = 0\n\tend := len(arg)\n\tif sub.queue != nil {\n\t\t// Remove the ' ' from the arg length.\n\t\tend -= 1 + len(args[subjIdx+2])\n\t}\n\tif oldStyle {\n\t\t// We will start at the account (if present) or at the subject.\n\t\t// We first skip the \"R \" or \"L \"\n\t\tstart = 2\n\t\t// And if there is an origin skip that.\n\t\tif hasOrigin {\n\t\t\tstart += len(sub.origin) + 1\n\t\t}\n\t\t// Here we are pointing at the account (if present), or at the subject.\n\t}\n\tsub.sid = arg[start:end]\n\n\t// Lookup account while avoiding fetch.\n\t// A slow fetch delays subsequent remote messages. It also avoids the expired check (see below).\n\t// With all but memory resolver lookup can be delayed or fail.\n\t// It is also possible that the account can't be resolved yet.\n\t// This does not apply to the memory resolver.\n\t// When used, perform the fetch.\n\tstaticResolver := true\n\tif res := srv.AccountResolver(); res != nil {\n\t\tif _, ok := res.(*MemAccResolver); !ok {\n\t\t\tstaticResolver = false\n\t\t}\n\t}\n\tvar acc *Account\n\tif staticResolver {\n\t\tacc, _ = srv.LookupAccount(accountName)\n\t} else if v, ok := srv.accounts.Load(accountName); ok {\n\t\tacc = v.(*Account)\n\t}\n\tif acc == nil {\n\t\t// if the option of retrieving accounts later exists, create an expired one.\n\t\t// When a client comes along, expiration will prevent it from being used,\n\t\t// cause a fetch and update the account to what is should be.\n\t\tif staticResolver {\n\t\t\tc.Errorf(\"Unknown account %q for remote subject %q\", accountName, sub.subject)\n\t\t\treturn\n\t\t}\n\t\tc.Debugf(\"Unknown account %q for remote subject %q\", accountName, sub.subject)\n\n\t\tvar isNew bool\n\t\tif acc, isNew = srv.LookupOrRegisterAccount(accountName); isNew {\n\t\t\tacc.mu.Lock()\n\t\t\tacc.expired.Store(true)\n\t\t\tacc.incomplete = true\n\t\t\tacc.mu.Unlock()\n\t\t}\n\t}\n\n\tc.mu.Lock()\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t// Check permissions if applicable.\n\tif c.perms != nil && !c.canExport(string(sub.subject)) {\n\t\tc.mu.Unlock()\n\t\tc.Debugf(\"Can not export %q, ignoring remote subscription request\", sub.subject)\n\t\treturn nil\n\t}\n\n\t// Check if we have a maximum on the number of subscriptions.\n\tif c.subsAtLimit() {\n\t\tc.mu.Unlock()\n\t\tc.maxSubsExceeded()\n\t\treturn nil\n\t}\n\n\tacc.mu.RLock()\n\t// For routes (this can be called by leafnodes), check if the account is\n\t// transitioning (from pool to dedicated route) and this route is not a\n\t// per-account route (route.poolIdx >= 0). If so, ignore this subscription.\n\t// Exclude \"no pool\" routes from this check.\n\tif c.kind == ROUTER && !c.route.noPool &&\n\t\tacc.routePoolIdx == accTransitioningToDedicatedRoute && c.route.poolIdx >= 0 {\n\t\tacc.mu.RUnlock()\n\t\tc.mu.Unlock()\n\t\t// Do not return an error, which would cause the connection to be closed.\n\t\treturn nil\n\t}\n\tsl := acc.sl\n\tacc.mu.RUnlock()\n\n\t// We use the sub.sid for the key of the c.subs map.\n\tkey := bytesToString(sub.sid)\n\tosub := c.subs[key]\n\tif osub == nil {\n\t\tc.subs[key] = sub\n\t\t// Now place into the account sl.\n\t\tif err = sl.Insert(sub); err != nil {\n\t\t\tdelete(c.subs, key)\n\t\t\tc.mu.Unlock()\n\t\t\tc.Errorf(\"Could not insert subscription: %v\", err)\n\t\t\tc.sendErr(\"Invalid Subscription\")\n\t\t\treturn nil\n\t\t}\n\t} else if sub.queue != nil {\n\t\t// For a queue we need to update the weight.\n\t\tdelta = sub.qw - atomic.LoadInt32(&osub.qw)\n\t\tatomic.StoreInt32(&osub.qw, sub.qw)\n\t\tsl.UpdateRemoteQSub(osub)\n\t}\n\tc.mu.Unlock()\n\n\tif srv.gateway.enabled {\n\t\tsrv.gatewayUpdateSubInterest(acc.Name, sub, delta)\n\t}\n\n\t// Now check on leafnode updates.\n\tacc.updateLeafNodes(sub, delta)\n\n\tif c.opts.Verbose {\n\t\tc.sendOK()\n\t}\n\n\treturn nil\n}\n\n// Lock is held on entry\nfunc (c *client) addRouteSubOrUnsubProtoToBuf(buf []byte, accName string, sub *subscription, isSubProto bool) []byte {\n\t// If we have an origin cluster and the other side supports leafnode origin clusters\n\t// send an LS+/LS- version instead.\n\tif len(sub.origin) > 0 && c.route.lnoc {\n\t\tif isSubProto {\n\t\t\tbuf = append(buf, lSubBytes...)\n\t\t\tbuf = append(buf, sub.origin...)\n\t\t\tbuf = append(buf, ' ')\n\t\t} else {\n\t\t\tbuf = append(buf, lUnsubBytes...)\n\t\t\tif c.route.lnocu {\n\t\t\t\tbuf = append(buf, sub.origin...)\n\t\t\t\tbuf = append(buf, ' ')\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif isSubProto {\n\t\t\tbuf = append(buf, rSubBytes...)\n\t\t} else {\n\t\t\tbuf = append(buf, rUnsubBytes...)\n\t\t}\n\t}\n\tif len(c.route.accName) == 0 {\n\t\tbuf = append(buf, accName...)\n\t\tbuf = append(buf, ' ')\n\t}\n\tbuf = append(buf, sub.subject...)\n\tif len(sub.queue) > 0 {\n\t\tbuf = append(buf, ' ')\n\t\tbuf = append(buf, sub.queue...)\n\t\t// Send our weight if we are a sub proto\n\t\tif isSubProto {\n\t\t\tbuf = append(buf, ' ')\n\t\t\tvar b [12]byte\n\t\t\tvar i = len(b)\n\t\t\tfor l := sub.qw; l > 0; l /= 10 {\n\t\t\t\ti--\n\t\t\t\tb[i] = digits[l%10]\n\t\t\t}\n\t\t\tbuf = append(buf, b[i:]...)\n\t\t}\n\t}\n\tbuf = append(buf, CR_LF...)\n\treturn buf\n}\n\n// sendSubsToRoute will send over our subject interest to\n// the remote side. For each account we will send the\n// complete interest for all subjects, both normal as a binary\n// and queue group weights.\n//\n// Server lock held on entry.\nfunc (s *Server) sendSubsToRoute(route *client, idx int, account string) {\n\tvar noPool bool\n\tif idx >= 0 {\n\t\t// We need to check if this route is \"no_pool\" in which case we\n\t\t// need to select all accounts.\n\t\troute.mu.Lock()\n\t\tnoPool = route.route.noPool\n\t\troute.mu.Unlock()\n\t}\n\t// Estimated size of all protocols. It does not have to be accurate at all.\n\tvar eSize int\n\testimateProtosSize := func(a *Account, addAccountName bool) {\n\t\tif ns := len(a.rm); ns > 0 {\n\t\t\tvar accSize int\n\t\t\tif addAccountName {\n\t\t\t\taccSize = len(a.Name) + 1\n\t\t\t}\n\t\t\t// Proto looks like: \"RS+ [ ][ ]\\r\\n\"\n\t\t\teSize += ns * (len(rSubBytes) + 1 + accSize)\n\t\t\tfor key := range a.rm {\n\t\t\t\t// Key contains \"[ ]\"\n\t\t\t\teSize += len(key)\n\t\t\t\t// In case this is a queue, just add some bytes for the queue weight.\n\t\t\t\t// If we want to be accurate, would have to check if \"key\" has a space,\n\t\t\t\t// if so, then figure out how many bytes we need to represent the weight.\n\t\t\t\teSize += 5\n\t\t\t}\n\t\t}\n\t}\n\t// Send over our account subscriptions.\n\taccs := make([]*Account, 0, 1024)\n\tif idx < 0 || account != _EMPTY_ {\n\t\tif ai, ok := s.accounts.Load(account); ok {\n\t\t\ta := ai.(*Account)\n\t\t\ta.mu.RLock()\n\t\t\t// Estimate size and add account name in protocol if idx is not -1\n\t\t\testimateProtosSize(a, idx >= 0)\n\t\t\taccs = append(accs, a)\n\t\t\ta.mu.RUnlock()\n\t\t}\n\t} else {\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\ta := v.(*Account)\n\t\t\ta.mu.RLock()\n\t\t\t// We are here for regular or pooled routes (not per-account).\n\t\t\t// So we collect all accounts whose routePoolIdx matches the\n\t\t\t// one for this route, or only the account provided, or all\n\t\t\t// accounts if dealing with a \"no pool\" route.\n\t\t\tif a.routePoolIdx == idx || noPool {\n\t\t\t\testimateProtosSize(a, true)\n\t\t\t\taccs = append(accs, a)\n\t\t\t}\n\t\t\ta.mu.RUnlock()\n\t\t\treturn true\n\t\t})\n\t}\n\n\tbuf := make([]byte, 0, eSize)\n\n\troute.mu.Lock()\n\tfor _, a := range accs {\n\t\ta.mu.RLock()\n\t\tfor key, n := range a.rm {\n\t\t\tvar origin, qn []byte\n\t\t\ts := strings.Fields(key)\n\t\t\t// Subject will always be the second field (index 1).\n\t\t\tsubj := stringToBytes(s[1])\n\t\t\t// Check if the key is for a leaf (will be field 0).\n\t\t\tforLeaf := s[0] == keyRoutedLeafSub\n\t\t\t// For queue, if not for a leaf, we need 3 fields \"R foo bar\",\n\t\t\t// but if for a leaf, we need 4 fields \"L foo bar leaf_origin\".\n\t\t\tif l := len(s); (!forLeaf && l == 3) || (forLeaf && l == 4) {\n\t\t\t\tqn = stringToBytes(s[2])\n\t\t\t}\n\t\t\tif forLeaf {\n\t\t\t\t// The leaf origin will be the last field.\n\t\t\t\torigin = stringToBytes(s[len(s)-1])\n\t\t\t}\n\t\t\t// s[1] is the subject and already as a string, so use that\n\t\t\t// instead of converting back `subj` to a string.\n\t\t\tif !route.canImport(s[1]) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsub := subscription{origin: origin, subject: subj, queue: qn, qw: n}\n\t\t\tbuf = route.addRouteSubOrUnsubProtoToBuf(buf, a.Name, &sub, true)\n\t\t}\n\t\ta.mu.RUnlock()\n\t}\n\tif len(buf) > 0 {\n\t\troute.enqueueProto(buf)\n\t\troute.Debugf(\"Sent local subscriptions to route\")\n\t}\n\troute.mu.Unlock()\n}\n\n// Sends SUBs protocols for the given subscriptions. If a filter is specified, it is\n// invoked for each subscription. If the filter returns false, the subscription is skipped.\n// This function may release the route's lock due to flushing of outbound data. A boolean\n// is returned to indicate if the connection has been closed during this call.\n// Lock is held on entry.\nfunc (c *client) sendRouteSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) {\n\tc.sendRouteSubOrUnSubProtos(subs, true, trace, filter)\n}\n\n// Sends UNSUBs protocols for the given subscriptions. If a filter is specified, it is\n// invoked for each subscription. If the filter returns false, the subscription is skipped.\n// This function may release the route's lock due to flushing of outbound data. A boolean\n// is returned to indicate if the connection has been closed during this call.\n// Lock is held on entry.\nfunc (c *client) sendRouteUnSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) {\n\tc.sendRouteSubOrUnSubProtos(subs, false, trace, filter)\n}\n\n// Low-level function that sends RS+ or RS- protocols for the given subscriptions.\n// This can now also send LS+ and LS- for origin cluster based leafnode subscriptions for cluster no-echo.\n// Use sendRouteSubProtos or sendRouteUnSubProtos instead for clarity.\n// Lock is held on entry.\nfunc (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto, trace bool, filter func(sub *subscription) bool) {\n\tvar (\n\t\t_buf [1024]byte\n\t\tbuf = _buf[:0]\n\t)\n\n\tfor _, sub := range subs {\n\t\tif filter != nil && !filter(sub) {\n\t\t\tcontinue\n\t\t}\n\t\t// Determine the account. If sub has an ImportMap entry, use that, otherwise scoped to\n\t\t// client. Default to global if all else fails.\n\t\tvar accName string\n\t\tif sub.client != nil && sub.client != c {\n\t\t\tsub.client.mu.Lock()\n\t\t}\n\t\tif sub.im != nil {\n\t\t\taccName = sub.im.acc.Name\n\t\t} else if sub.client != nil && sub.client.acc != nil {\n\t\t\taccName = sub.client.acc.Name\n\t\t} else {\n\t\t\tc.Debugf(\"Falling back to default account for sending subs\")\n\t\t\taccName = globalAccountName\n\t\t}\n\t\tif sub.client != nil && sub.client != c {\n\t\t\tsub.client.mu.Unlock()\n\t\t}\n\n\t\tas := len(buf)\n\t\tbuf = c.addRouteSubOrUnsubProtoToBuf(buf, accName, sub, isSubProto)\n\t\tif trace {\n\t\t\tc.traceOutOp(\"\", buf[as:len(buf)-LEN_CR_LF])\n\t\t}\n\t}\n\tc.enqueueProto(buf)\n}\n\nfunc (s *Server) createRoute(conn net.Conn, rURL *url.URL, rtype RouteType, gossipMode byte, accName string) *client {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tdidSolicit := rURL != nil\n\tr := &route{routeType: rtype, didSolicit: didSolicit, poolIdx: -1, gossipMode: gossipMode}\n\n\tc := &client{srv: s, nc: conn, opts: ClientOpts{}, kind: ROUTER, msubs: -1, mpay: -1, route: r, start: time.Now()}\n\n\t// Is the server configured for compression?\n\tcompressionConfigured := needsCompression(opts.Cluster.Compression.Mode)\n\n\tvar infoJSON []byte\n\t// Grab server variables and generates route INFO Json. Note that we set\n\t// and reset some of s.routeInfo fields when that happens, so we need\n\t// the server write lock.\n\ts.mu.Lock()\n\t// If we are creating a pooled connection and this is the server soliciting\n\t// the connection, we will delay sending the INFO after we have processed\n\t// the incoming INFO from the remote. Also delay if configured for compression.\n\tdelayInfo := didSolicit && (compressionConfigured || routeShouldDelayInfo(accName, opts))\n\tif !delayInfo {\n\t\tinfoJSON = s.generateRouteInitialInfoJSON(accName, opts.Cluster.Compression.Mode, 0, gossipMode)\n\t}\n\tauthRequired := s.routeInfo.AuthRequired\n\ttlsRequired := s.routeInfo.TLSRequired\n\tclusterName := s.info.Cluster\n\ttlsName := s.routeTLSName\n\ts.mu.Unlock()\n\n\t// Grab lock\n\tc.mu.Lock()\n\n\t// Initialize\n\tc.initClient()\n\n\tif didSolicit {\n\t\t// Do this before the TLS code, otherwise, in case of failure\n\t\t// and if route is explicit, it would try to reconnect to 'nil'...\n\t\tr.url = rURL\n\t\tr.accName = []byte(accName)\n\t} else {\n\t\tc.flags.set(expectConnect)\n\t}\n\n\t// Check for TLS\n\tif tlsRequired {\n\t\ttlsConfig := opts.Cluster.TLSConfig\n\t\tif didSolicit {\n\t\t\t// Copy off the config to add in ServerName if we need to.\n\t\t\ttlsConfig = tlsConfig.Clone()\n\t\t}\n\t\t// Perform (server or client side) TLS handshake.\n\t\tif resetTLSName, err := c.doTLSHandshake(\"route\", didSolicit, rURL, tlsConfig, tlsName, opts.Cluster.TLSTimeout, opts.Cluster.TLSPinnedCerts); err != nil {\n\t\t\tc.mu.Unlock()\n\t\t\tif resetTLSName {\n\t\t\t\ts.mu.Lock()\n\t\t\t\ts.routeTLSName = _EMPTY_\n\t\t\t\ts.mu.Unlock()\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t// Do final client initialization\n\n\t// Initialize the per-account cache.\n\tc.in.pacache = make(map[string]*perAccountCache)\n\tif didSolicit {\n\t\t// Set permissions associated with the route user (if applicable).\n\t\t// No lock needed since we are already under client lock.\n\t\tc.setRoutePermissions(opts.Cluster.Permissions)\n\t}\n\n\t// We can't safely send the pings until we have negotiated compression\n\t// with the remote, but we want to protect against a connection that\n\t// does not perform the handshake. We will start a timer that will close\n\t// the connection as stale based on the ping interval and max out values,\n\t// but without actually sending pings.\n\tif compressionConfigured {\n\t\tpingInterval := opts.PingInterval\n\t\tpingMax := opts.MaxPingsOut\n\t\tif opts.Cluster.PingInterval > 0 {\n\t\t\tpingInterval = opts.Cluster.PingInterval\n\t\t}\n\t\tif opts.Cluster.MaxPingsOut > 0 {\n\t\t\tpingMax = opts.MaxPingsOut\n\t\t}\n\t\tc.watchForStaleConnection(adjustPingInterval(ROUTER, pingInterval), pingMax)\n\t} else {\n\t\t// Set the Ping timer\n\t\tc.setFirstPingTimer()\n\t}\n\n\t// For routes, the \"client\" is added to s.routes only when processing\n\t// the INFO protocol, that is much later.\n\t// In the meantime, if the server shutsdown, there would be no reference\n\t// to the client (connection) to be closed, leaving this readLoop\n\t// uinterrupted, causing the Shutdown() to wait indefinitively.\n\t// We need to store the client in a special map, under a special lock.\n\tif !s.addToTempClients(c.cid, c) {\n\t\tc.mu.Unlock()\n\t\tc.setNoReconnect()\n\t\tc.closeConnection(ServerShutdown)\n\t\treturn nil\n\t}\n\n\t// Check for Auth required state for incoming connections.\n\t// Make sure to do this before spinning up readLoop.\n\tif authRequired && !didSolicit {\n\t\tttl := secondsToDuration(opts.Cluster.AuthTimeout)\n\t\tc.setAuthTimer(ttl)\n\t}\n\n\t// Spin up the read loop.\n\ts.startGoRoutine(func() { c.readLoop(nil) })\n\n\t// Spin up the write loop.\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\tif tlsRequired {\n\t\tc.Debugf(\"TLS handshake complete\")\n\t\tcs := c.nc.(*tls.Conn).ConnectionState()\n\t\tc.Debugf(\"TLS version %s, cipher suite %s\", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite))\n\t}\n\n\t// Queue Connect proto if we solicited the connection.\n\tif didSolicit {\n\t\tc.Debugf(\"Route connect msg sent\")\n\t\tif err := c.sendRouteConnect(clusterName, tlsRequired); err != nil {\n\t\t\tc.mu.Unlock()\n\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif !delayInfo {\n\t\t// Send our info to the other side.\n\t\t// Our new version requires dynamic information for accounts and a nonce.\n\t\tc.enqueueProto(infoJSON)\n\t}\n\tc.mu.Unlock()\n\n\tc.Noticef(\"Route connection created\")\n\treturn c\n}\n\nfunc routeShouldDelayInfo(accName string, opts *Options) bool {\n\treturn accName == _EMPTY_ && opts.Cluster.PoolSize >= 1\n}\n\n// Generates a nonce and set some route info's fields before marshal'ing into JSON.\n// To be used only when a route is created (to send the initial INFO protocol).\n//\n// Server lock held on entry.\nfunc (s *Server) generateRouteInitialInfoJSON(accName, compression string, poolIdx int, gossipMode byte) []byte {\n\t// New proto wants a nonce (although not used in routes, that is, not signed in CONNECT)\n\tvar raw [nonceLen]byte\n\tnonce := raw[:]\n\ts.generateNonce(nonce)\n\tri := &s.routeInfo\n\t// Override compression with s2_auto instead of actual compression level.\n\tif s.getOpts().Cluster.Compression.Mode == CompressionS2Auto {\n\t\tcompression = CompressionS2Auto\n\t}\n\tri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression, ri.GossipMode = string(nonce), accName, poolIdx, compression, gossipMode\n\tinfoJSON := generateInfoJSON(&s.routeInfo)\n\t// Clear now that it has been serialized. Will prevent nonce to be included in async INFO that we may send.\n\t// Same for some other fields.\n\tri.Nonce, ri.RouteAccount, ri.RoutePoolIdx, ri.Compression, ri.GossipMode = _EMPTY_, _EMPTY_, 0, _EMPTY_, 0\n\treturn infoJSON\n}\n\nconst (\n\t_CRLF_ = \"\\r\\n\"\n\t_EMPTY_ = \"\"\n)\n\nfunc (s *Server) addRoute(c *client, didSolicit, sendDelayedInfo bool, gossipMode byte, info *Info, accName string) bool {\n\tid := info.ID\n\n\tvar acc *Account\n\tif accName != _EMPTY_ {\n\t\tvar err error\n\t\tacc, err = s.LookupAccount(accName)\n\t\tif err != nil {\n\t\t\tc.sendErrAndErr(fmt.Sprintf(\"Unable to lookup account %q: %v\", accName, err))\n\t\t\tc.closeConnection(MissingAccount)\n\t\t\treturn false\n\t\t}\n\t}\n\n\ts.mu.Lock()\n\tif !s.isRunning() || s.routesReject {\n\t\ts.mu.Unlock()\n\t\treturn false\n\t}\n\tvar invProtoErr string\n\n\topts := s.getOpts()\n\n\t// Assume we are in pool mode if info.RoutePoolSize is set. We may disable\n\t// in some cases.\n\tpool := info.RoutePoolSize > 0\n\t// This is used to prevent a server with pooling to constantly trying\n\t// to connect to a server with no pooling (for instance old server) after\n\t// the first connection is established.\n\tvar noReconnectForOldServer bool\n\n\t// To allow rolling updates, we now allow servers with different pool sizes\n\t// so we will use as the effective pool size here, the max between our\n\t// configured size and the size we receive in the info protocol.\n\teffectivePoolSize := max(s.routesPoolSize, info.RoutePoolSize)\n\n\t// If the remote is an old server, info.RoutePoolSize will be 0, or if\n\t// this server's Cluster.PoolSize is negative, we will behave as an old\n\t// server and need to handle things differently.\n\tif info.RoutePoolSize <= 0 || opts.Cluster.PoolSize < 0 {\n\t\tif accName != _EMPTY_ {\n\t\t\tinvProtoErr = fmt.Sprintf(\"Not possible to have a dedicated route for account %q between those servers\", accName)\n\t\t\t// In this case, make sure this route does not attempt to reconnect\n\t\t\tc.setNoReconnect()\n\t\t} else {\n\t\t\t// We will accept, but treat this remote has \"no pool\"\n\t\t\tpool, noReconnectForOldServer = false, true\n\t\t\tc.mu.Lock()\n\t\t\tc.route.poolIdx = 0\n\t\t\tc.route.noPool = true\n\t\t\tc.mu.Unlock()\n\t\t\t// Keep track of number of routes like that. We will use that when\n\t\t\t// sending subscriptions over routes.\n\t\t\ts.routesNoPool++\n\t\t}\n\t} else if didSolicit {\n\t\t// For solicited route, the incoming's RoutePoolIdx should not be set.\n\t\tif info.RoutePoolIdx != 0 {\n\t\t\tinvProtoErr = fmt.Sprintf(\"Route pool index should not be set but is set to %v\", info.RoutePoolIdx)\n\t\t}\n\t} else if info.RoutePoolIdx < 0 || info.RoutePoolIdx >= effectivePoolSize {\n\t\t// For non solicited routes, if the remote sends a RoutePoolIdx, make\n\t\t// sure it is a valid one (in range of the pool size).\n\t\tinvProtoErr = fmt.Sprintf(\"Invalid route pool index: %v - pool size is %v\", info.RoutePoolIdx, info.RoutePoolSize)\n\t}\n\tif invProtoErr != _EMPTY_ {\n\t\ts.mu.Unlock()\n\t\tc.sendErrAndErr(invProtoErr)\n\t\tc.closeConnection(ProtocolViolation)\n\t\treturn false\n\t}\n\t// If accName is set, we are dealing with a per-account connection.\n\tif accName != _EMPTY_ {\n\t\t// When an account has its own route, it will be an error if the given\n\t\t// account name is not found in s.accRoutes map.\n\t\tconns, exists := s.accRoutes[accName]\n\t\tif !exists {\n\t\t\ts.mu.Unlock()\n\t\t\tc.sendErrAndErr(fmt.Sprintf(\"No route for account %q\", accName))\n\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\treturn false\n\t\t}\n\t\tremote, exists := conns[id]\n\t\tif !exists {\n\t\t\tconns[id] = c\n\t\t\tc.mu.Lock()\n\t\t\tidHash := c.route.idHash\n\t\t\tcid := c.cid\n\t\t\trtype := c.route.routeType\n\t\t\tif sendDelayedInfo {\n\t\t\t\tcm := compressionModeForInfoProtocol(&opts.Cluster.Compression, c.route.compression)\n\t\t\t\tc.enqueueProto(s.generateRouteInitialInfoJSON(accName, cm, 0, gossipMode))\n\t\t\t}\n\t\t\tif c.last.IsZero() {\n\t\t\t\tc.last = time.Now()\n\t\t\t}\n\t\t\tif acc != nil {\n\t\t\t\tc.acc = acc\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\n\t\t\t// Store this route with key being the route id hash + account name\n\t\t\ts.storeRouteByHash(idHash+accName, c)\n\n\t\t\t// Now that we have registered the route, we can remove from the temp map.\n\t\t\ts.removeFromTempClients(cid)\n\n\t\t\t// We don't need to send if the only route is the one we just accepted.\n\t\t\tif len(conns) > 1 {\n\t\t\t\ts.forwardNewRouteInfoToKnownServers(info, rtype, didSolicit, gossipMode)\n\t\t\t}\n\n\t\t\t// Send subscription interest\n\t\t\ts.sendSubsToRoute(c, -1, accName)\n\t\t} else {\n\t\t\thandleDuplicateRoute(remote, c, true)\n\t\t}\n\t\ts.mu.Unlock()\n\t\treturn !exists\n\t}\n\tvar remote *client\n\t// That will be the position of the connection in the slice, we initialize\n\t// to -1 to indicate that no space was found.\n\tidx := -1\n\t// This will be the size (or number of connections) in a given slice.\n\tsz := 0\n\t// Check if we know about the remote server\n\tconns, exists := s.routes[id]\n\tif !exists {\n\t\t// Now, create a slice for route connections of the size of the pool\n\t\t// or 1 when not in pool mode.\n\t\tconns = make([]*client, effectivePoolSize)\n\t\t// Track this slice for this remote server.\n\t\ts.routes[id] = conns\n\t\t// Set the index to info.RoutePoolIdx because if this is a solicited\n\t\t// route, this value will be 0, which is what we want, otherwise, we\n\t\t// will use whatever index the remote has chosen.\n\t\tidx = info.RoutePoolIdx\n\t} else if pool {\n\t\t// The remote could have done a config reload and increased the pool size.\n\t\t// It will close the connections before soliciting again, however, if\n\t\t// on this side, one of the route is not yet fully removed, but the\n\t\t// first one is, it would accept the new connection (with a greater pool\n\t\t// size) and we would not go through the phase of `!exists` above creating\n\t\t// the slice with the right size. So we need to check here and add new empty\n\t\t// entries to complete the effective pool size.\n\t\tif n := effectivePoolSize - len(conns); n > 0 {\n\t\t\tfor range n {\n\t\t\t\tconns = append(conns, nil)\n\t\t\t}\n\t\t\ts.routes[id] = conns\n\t\t}\n\t\t// The remote was found. If this is a non solicited route, we will place\n\t\t// the connection in the pool at the index given by info.RoutePoolIdx.\n\t\t// But if there is already one, close this incoming connection as a\n\t\t// duplicate.\n\t\tif !didSolicit {\n\t\t\tidx = info.RoutePoolIdx\n\t\t\tif remote = conns[idx]; remote != nil {\n\t\t\t\thandleDuplicateRoute(remote, c, false)\n\t\t\t\ts.mu.Unlock()\n\t\t\t\treturn false\n\t\t\t}\n\t\t\t// Look if there is a solicited route in the pool. If there is one,\n\t\t\t// they should all be, so stop at the first.\n\t\t\tif url, rtype, hasSolicited := hasSolicitedRoute(conns); hasSolicited {\n\t\t\t\tupgradeRouteToSolicited(c, url, rtype)\n\t\t\t}\n\t\t} else {\n\t\t\t// If we solicit, upgrade to solicited all non-solicited routes that\n\t\t\t// we may have registered.\n\t\t\tc.mu.Lock()\n\t\t\turl := c.route.url\n\t\t\trtype := c.route.routeType\n\t\t\tc.mu.Unlock()\n\t\t\tfor _, r := range conns {\n\t\t\t\tupgradeRouteToSolicited(r, url, rtype)\n\t\t\t}\n\t\t}\n\t\t// For all cases (solicited and not) we need to count how many connections\n\t\t// we already have, and for solicited route, we will find a free spot in\n\t\t// the slice.\n\t\tfor i, r := range conns {\n\t\t\tif idx == -1 && r == nil {\n\t\t\t\tidx = i\n\t\t\t} else if r != nil {\n\t\t\t\tsz++\n\t\t\t}\n\t\t}\n\t} else {\n\t\tremote = conns[0]\n\t}\n\t// If there is a spot, idx will be greater or equal to 0.\n\tif idx >= 0 {\n\t\tc.mu.Lock()\n\t\tc.route.connectURLs = info.ClientConnectURLs\n\t\tc.route.wsConnURLs = info.WSConnectURLs\n\t\tc.route.poolIdx = idx\n\t\trtype := c.route.routeType\n\t\tcid := c.cid\n\t\tidHash := c.route.idHash\n\t\trHash := c.route.hash\n\t\trn := c.route.remoteName\n\t\turl := c.route.url\n\t\tif sendDelayedInfo {\n\t\t\tcm := compressionModeForInfoProtocol(&opts.Cluster.Compression, c.route.compression)\n\t\t\tc.enqueueProto(s.generateRouteInitialInfoJSON(_EMPTY_, cm, idx, gossipMode))\n\t\t}\n\t\tif c.last.IsZero() {\n\t\t\tc.last = time.Now()\n\t\t}\n\t\tc.mu.Unlock()\n\n\t\t// With pooling, we keep track of the remote's configured route pool size.\n\t\t// We do so when adding the connection in the first slot, not when `sz == 1`\n\t\t// because there could be situations where we have old connections that have\n\t\t// not yet been removed and so we would not have `sz == `. However, we will\n\t\t// always have the condition where we are adding the new connection at `idx==0`\n\t\t// so use that as the condition to store the remote pool size.\n\t\tif pool && idx == 0 {\n\t\t\tif s.remoteRoutePoolSize == nil {\n\t\t\t\ts.remoteRoutePoolSize = make(map[string]int)\n\t\t\t}\n\t\t\ts.remoteRoutePoolSize[id] = info.RoutePoolSize\n\t\t}\n\n\t\t// Add to the slice and bump the count of connections for this remote\n\t\tconns[idx] = c\n\t\tsz++\n\t\t// This boolean will indicate that we are registering the only\n\t\t// connection in non pooled situation or we stored the very first\n\t\t// connection for a given remote server.\n\t\tdoOnce := !pool || sz == 1\n\t\tif doOnce {\n\t\t\t// check to be consistent and future proof. but will be same domain\n\t\t\tif s.sameDomain(info.Domain) {\n\t\t\t\ts.nodeToInfo.Store(rHash, nodeInfo{\n\t\t\t\t\tname: rn,\n\t\t\t\t\tversion: s.info.Version,\n\t\t\t\t\tcluster: s.info.Cluster,\n\t\t\t\t\tdomain: info.Domain,\n\t\t\t\t\tid: id,\n\t\t\t\t\ttags: nil,\n\t\t\t\t\tcfg: nil,\n\t\t\t\t\tstats: nil,\n\t\t\t\t\toffline: false,\n\t\t\t\t\tjs: info.JetStream,\n\t\t\t\t\tbinarySnapshots: true, // Updated default to true. Versions 2.10.0+ support it.\n\t\t\t\t\taccountNRG: false,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\n\t\t// Store this route using the hash as the key\n\t\tif pool {\n\t\t\tidHash += strconv.Itoa(idx)\n\t\t}\n\t\ts.storeRouteByHash(idHash, c)\n\n\t\t// Now that we have registered the route, we can remove from the temp map.\n\t\ts.removeFromTempClients(cid)\n\n\t\tif doOnce {\n\t\t\t// If the INFO contains a Gateway URL, add it to the list for our cluster.\n\t\t\tif info.GatewayURL != _EMPTY_ && s.addGatewayURL(info.GatewayURL) {\n\t\t\t\ts.sendAsyncGatewayInfo()\n\t\t\t}\n\n\t\t\t// We don't need to send if the only route is the one we just accepted.\n\t\t\tif len(s.routes) > 1 {\n\t\t\t\ts.forwardNewRouteInfoToKnownServers(info, rtype, didSolicit, gossipMode)\n\t\t\t}\n\n\t\t\t// Send info about the known gateways to this route.\n\t\t\ts.sendGatewayConfigsToRoute(c)\n\n\t\t\t// Unless disabled, possibly update the server's INFO protocol\n\t\t\t// and send to clients that know how to handle async INFOs.\n\t\t\tif !opts.Cluster.NoAdvertise {\n\t\t\t\ts.addConnectURLsAndSendINFOToClients(info.ClientConnectURLs, info.WSConnectURLs)\n\t\t\t}\n\n\t\t\t// Add the remote's leafnodeURL to our list of URLs and send the update\n\t\t\t// to all LN connections. (Note that when coming from a route, LeafNodeURLs\n\t\t\t// is an array of size 1 max).\n\t\t\tif len(info.LeafNodeURLs) == 1 && s.addLeafNodeURL(info.LeafNodeURLs[0]) {\n\t\t\t\ts.sendAsyncLeafNodeInfo()\n\t\t\t}\n\t\t}\n\n\t\t// Send the subscriptions interest.\n\t\ts.sendSubsToRoute(c, idx, _EMPTY_)\n\n\t\t// In pool mode, if we did not yet reach the cap, try to connect a new connection,\n\t\t// but do so only after receiving the first PONG to our PING, which will ensure\n\t\t// that we have proper authentication.\n\t\tif pool && didSolicit && sz != effectivePoolSize {\n\t\t\tc.mu.Lock()\n\t\t\tc.route.startNewRoute = &routeInfo{\n\t\t\t\turl: url,\n\t\t\t\trtype: rtype,\n\t\t\t\tgossipMode: gossipMode,\n\t\t\t}\n\t\t\tc.sendPing()\n\t\t\tc.mu.Unlock()\n\t\t}\n\t}\n\ts.mu.Unlock()\n\tif pool {\n\t\tif idx == -1 {\n\t\t\t// Was full, so need to close connection\n\t\t\tc.Debugf(\"Route pool size reached, closing extra connection to %q\", id)\n\t\t\thandleDuplicateRoute(nil, c, true)\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\t// This is for non-pool mode at this point.\n\tif exists {\n\t\thandleDuplicateRoute(remote, c, noReconnectForOldServer)\n\t}\n\n\treturn !exists\n}\n\nfunc hasSolicitedRoute(conns []*client) (*url.URL, RouteType, bool) {\n\tvar url *url.URL\n\tvar rtype RouteType\n\tfor _, r := range conns {\n\t\tif r == nil {\n\t\t\tcontinue\n\t\t}\n\t\tr.mu.Lock()\n\t\tif r.route.didSolicit {\n\t\t\turl = r.route.url\n\t\t\trtype = r.route.routeType\n\t\t}\n\t\tr.mu.Unlock()\n\t\tif url != nil {\n\t\t\treturn url, rtype, true\n\t\t}\n\t}\n\treturn nil, 0, false\n}\n\nfunc upgradeRouteToSolicited(r *client, url *url.URL, rtype RouteType) {\n\tif r == nil {\n\t\treturn\n\t}\n\tr.mu.Lock()\n\tif !r.route.didSolicit {\n\t\tr.route.didSolicit = true\n\t\tr.route.url = url\n\t}\n\tif rtype == Explicit {\n\t\tr.route.routeType = Explicit\n\t}\n\tr.mu.Unlock()\n}\n\nfunc handleDuplicateRoute(remote, c *client, setNoReconnect bool) {\n\t// We used to clear some fields when closing a duplicate connection\n\t// to prevent sending INFO protocols for the remotes to update\n\t// their leafnode/gateway URLs. This is no longer needed since\n\t// removeRoute() now does the right thing of doing that only when\n\t// the closed connection was an added route connection.\n\tc.mu.Lock()\n\tdidSolicit := c.route.didSolicit\n\turl := c.route.url\n\trtype := c.route.routeType\n\tif setNoReconnect {\n\t\tc.flags.set(noReconnect)\n\t}\n\tc.mu.Unlock()\n\n\tif remote == nil {\n\t\treturn\n\t}\n\n\tremote.mu.Lock()\n\tif didSolicit && !remote.route.didSolicit {\n\t\tremote.route.didSolicit = true\n\t\tremote.route.url = url\n\t}\n\t// The extra route might be an configured explicit route\n\t// so keep the state that the remote was configured.\n\tif rtype == Explicit {\n\t\tremote.route.routeType = rtype\n\t}\n\t// This is to mitigate the issue where both sides add the route\n\t// on the opposite connection, and therefore end-up with both\n\t// connections being dropped.\n\tremote.route.retry = true\n\tremote.mu.Unlock()\n}\n\n// Import filter check.\nfunc (c *client) importFilter(sub *subscription) bool {\n\tif c.perms == nil {\n\t\treturn true\n\t}\n\treturn c.canImport(string(sub.subject))\n}\n\n// updateRouteSubscriptionMap will make sure to update the route map for the subscription. Will\n// also forward to all routes if needed.\nfunc (s *Server) updateRouteSubscriptionMap(acc *Account, sub *subscription, delta int32) {\n\tif acc == nil || sub == nil {\n\t\treturn\n\t}\n\n\t// We only store state on local subs for transmission across all other routes.\n\tif sub.client == nil || sub.client.kind == ROUTER || sub.client.kind == GATEWAY {\n\t\treturn\n\t}\n\n\tif sub.si {\n\t\treturn\n\t}\n\n\t// Copy to hold outside acc lock.\n\tvar n int32\n\tvar ok bool\n\n\tisq := len(sub.queue) > 0\n\n\taccLock := func() {\n\t\t// Not required for code correctness, but helps reduce the number of\n\t\t// updates sent to the routes when processing high number of concurrent\n\t\t// queue subscriptions updates (sub/unsub).\n\t\t// See https://github.com/nats-io/nats-server/pull/1126 for more details.\n\t\tif isq {\n\t\t\tacc.sqmu.Lock()\n\t\t}\n\t\tacc.mu.Lock()\n\t}\n\taccUnlock := func() {\n\t\tacc.mu.Unlock()\n\t\tif isq {\n\t\t\tacc.sqmu.Unlock()\n\t\t}\n\t}\n\n\taccLock()\n\n\t// This is non-nil when we know we are in cluster mode.\n\trm, lqws := acc.rm, acc.lqws\n\tif rm == nil {\n\t\taccUnlock()\n\t\treturn\n\t}\n\n\t// Create the subscription key which will prevent collisions between regular\n\t// and leaf routed subscriptions. See keyFromSubWithOrigin() for details.\n\tkey := keyFromSubWithOrigin(sub)\n\n\t// Decide whether we need to send an update out to all the routes.\n\tupdate := isq\n\n\t// This is where we do update to account. For queues we need to take\n\t// special care that this order of updates is same as what is sent out\n\t// over routes.\n\tif n, ok = rm[key]; ok {\n\t\tn += delta\n\t\tif n <= 0 {\n\t\t\tdelete(rm, key)\n\t\t\tif isq {\n\t\t\t\tdelete(lqws, key)\n\t\t\t}\n\t\t\tupdate = true // Update for deleting (N->0)\n\t\t} else {\n\t\t\trm[key] = n\n\t\t}\n\t} else if delta > 0 {\n\t\tn = delta\n\t\trm[key] = delta\n\t\tupdate = true // Adding a new entry for normal sub means update (0->1)\n\t}\n\n\taccUnlock()\n\n\tif !update {\n\t\treturn\n\t}\n\n\t// If we are sending a queue sub, make a copy and place in the queue weight.\n\t// FIXME(dlc) - We can be smarter here and avoid copying and acquiring the lock.\n\tif isq {\n\t\tsub.client.mu.Lock()\n\t\tnsub := *sub\n\t\tsub.client.mu.Unlock()\n\t\tnsub.qw = n\n\t\tsub = &nsub\n\t}\n\n\t// We need to send out this update. Gather routes\n\tvar _routes [32]*client\n\troutes := _routes[:0]\n\n\ts.mu.RLock()\n\t// The account's routePoolIdx field is set/updated under the server lock\n\t// (but also the account's lock). So we don't need to acquire the account's\n\t// lock here to get the value.\n\tif poolIdx := acc.routePoolIdx; poolIdx < 0 {\n\t\tif conns, ok := s.accRoutes[acc.Name]; ok {\n\t\t\tfor _, r := range conns {\n\t\t\t\troutes = append(routes, r)\n\t\t\t}\n\t\t}\n\t\tif s.routesNoPool > 0 {\n\t\t\t// We also need to look for \"no pool\" remotes (that is, routes to older\n\t\t\t// servers or servers that have explicitly disabled pooling).\n\t\t\ts.forEachRemote(func(r *client) {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tif r.route.noPool {\n\t\t\t\t\troutes = append(routes, r)\n\t\t\t\t}\n\t\t\t\tr.mu.Unlock()\n\t\t\t})\n\t\t}\n\t} else {\n\t\t// We can't use s.forEachRouteIdx here since we want to check/get the\n\t\t// \"no pool\" route ONLY if we don't find a route at the given `poolIdx`.\n\t\tfor _, conns := range s.routes {\n\t\t\tif r := conns[poolIdx]; r != nil {\n\t\t\t\troutes = append(routes, r)\n\t\t\t} else if s.routesNoPool > 0 {\n\t\t\t\t// Check if we have a \"no pool\" route at index 0, and if so, it\n\t\t\t\t// means that for this remote, we have a single connection because\n\t\t\t\t// that server does not have pooling.\n\t\t\t\tif r := conns[0]; r != nil {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tif r.route.noPool {\n\t\t\t\t\t\troutes = append(routes, r)\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\ttrace := atomic.LoadInt32(&s.logging.trace) == 1\n\ts.mu.RUnlock()\n\n\t// If we are a queue subscriber we need to make sure our updates are serialized from\n\t// potential multiple connections. We want to make sure that the order above is preserved\n\t// here but not necessarily all updates need to be sent. We need to block and recheck the\n\t// n count with the lock held through sending here. We will suppress duplicate sends of same qw.\n\tif isq {\n\t\t// However, we can't hold the acc.mu lock since we allow client.mu.Lock -> acc.mu.Lock\n\t\t// but not the opposite. So use a dedicated lock while holding the route's lock.\n\t\tacc.sqmu.Lock()\n\t\tdefer acc.sqmu.Unlock()\n\n\t\tacc.mu.Lock()\n\t\tn = rm[key]\n\t\tsub.qw = n\n\t\t// Check the last sent weight here. If same, then someone\n\t\t// beat us to it and we can just return here. Otherwise update\n\t\tif ls, ok := lqws[key]; ok && ls == n {\n\t\t\tacc.mu.Unlock()\n\t\t\treturn\n\t\t} else if n > 0 {\n\t\t\tlqws[key] = n\n\t\t}\n\t\tacc.mu.Unlock()\n\t}\n\n\t// Snapshot into array\n\tsubs := []*subscription{sub}\n\n\t// Deliver to all routes.\n\tfor _, route := range routes {\n\t\troute.mu.Lock()\n\t\t// Note that queue unsubs where n > 0 are still\n\t\t// subscribes with a smaller weight.\n\t\troute.sendRouteSubOrUnSubProtos(subs, n > 0, trace, route.importFilter)\n\t\troute.mu.Unlock()\n\t}\n}\n\n// This starts the route accept loop in a go routine, unless it\n// is detected that the server has already been shutdown.\n// It will also start soliciting explicit routes.\nfunc (s *Server) startRouteAcceptLoop() {\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Snapshot server options.\n\tport := opts.Cluster.Port\n\n\tif port == -1 {\n\t\tport = 0\n\t}\n\n\t// This requires lock, so do this outside of may block.\n\tclusterName := s.ClusterName()\n\n\ts.mu.Lock()\n\ts.Noticef(\"Cluster name is %s\", clusterName)\n\tif s.isClusterNameDynamic() {\n\t\ts.Warnf(\"Cluster name was dynamically generated, consider setting one\")\n\t}\n\n\thp := net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(port))\n\tl, e := natsListen(\"tcp\", hp)\n\ts.routeListenerErr = e\n\tif e != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Error listening on router port: %d - %v\", opts.Cluster.Port, e)\n\t\treturn\n\t}\n\ts.Noticef(\"Listening for route connections on %s\",\n\t\tnet.JoinHostPort(opts.Cluster.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port)))\n\n\t// Check for TLSConfig\n\ttlsReq := opts.Cluster.TLSConfig != nil\n\tinfo := Info{\n\t\tID: s.info.ID,\n\t\tName: s.info.Name,\n\t\tVersion: s.info.Version,\n\t\tGoVersion: runtime.Version(),\n\t\tAuthRequired: false,\n\t\tTLSRequired: tlsReq,\n\t\tTLSVerify: tlsReq,\n\t\tMaxPayload: s.info.MaxPayload,\n\t\tJetStream: s.info.JetStream,\n\t\tProto: s.getServerProto(),\n\t\tGatewayURL: s.getGatewayURL(),\n\t\tHeaders: s.supportsHeaders(),\n\t\tCluster: s.info.Cluster,\n\t\tDomain: s.info.Domain,\n\t\tDynamic: s.isClusterNameDynamic(),\n\t\tLNOC: true,\n\t\tLNOCU: true,\n\t}\n\t// For tests that want to simulate old servers, do not set the compression\n\t// on the INFO protocol if configured with CompressionNotSupported.\n\tif cm := opts.Cluster.Compression.Mode; cm != CompressionNotSupported {\n\t\tinfo.Compression = cm\n\t}\n\tif ps := opts.Cluster.PoolSize; ps > 0 {\n\t\tinfo.RoutePoolSize = ps\n\t}\n\t// Set this if only if advertise is not disabled\n\tif !opts.Cluster.NoAdvertise {\n\t\tinfo.ClientConnectURLs = s.clientConnectURLs\n\t\tinfo.WSConnectURLs = s.websocket.connectURLs\n\t}\n\t// If we have selected a random port...\n\tif port == 0 {\n\t\t// Write resolved port back to options.\n\t\topts.Cluster.Port = l.Addr().(*net.TCPAddr).Port\n\t}\n\t// Check for Auth items\n\tif opts.Cluster.Username != \"\" {\n\t\tinfo.AuthRequired = true\n\t}\n\t// Check for permissions.\n\tif opts.Cluster.Permissions != nil {\n\t\tinfo.Import = opts.Cluster.Permissions.Import\n\t\tinfo.Export = opts.Cluster.Permissions.Export\n\t}\n\t// If this server has a LeafNode accept loop, s.leafNodeInfo.IP is,\n\t// at this point, set to the host:port for the leafnode accept URL,\n\t// taking into account possible advertise setting. Use the LeafNodeURLs\n\t// and set this server's leafnode accept URL. This will be sent to\n\t// routed servers.\n\tif !opts.LeafNode.NoAdvertise && s.leafNodeInfo.IP != _EMPTY_ {\n\t\tinfo.LeafNodeURLs = []string{s.leafNodeInfo.IP}\n\t}\n\ts.routeInfo = info\n\t// Possibly override Host/Port and set IP based on Cluster.Advertise\n\tif err := s.setRouteInfoHostPortAndIP(); err != nil {\n\t\ts.Fatalf(\"Error setting route INFO with Cluster.Advertise value of %s, err=%v\", opts.Cluster.Advertise, err)\n\t\tl.Close()\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\t// Setup state that can enable shutdown\n\ts.routeListener = l\n\t// Warn if using Cluster.Insecure\n\tif tlsReq && opts.Cluster.TLSConfig.InsecureSkipVerify {\n\t\ts.Warnf(clusterTLSInsecureWarning)\n\t}\n\n\t// Now that we have the port, keep track of all ip:port that resolve to this server.\n\tif interfaceAddr, err := net.InterfaceAddrs(); err == nil {\n\t\tvar localIPs []string\n\t\tfor i := 0; i < len(interfaceAddr); i++ {\n\t\t\tinterfaceIP, _, _ := net.ParseCIDR(interfaceAddr[i].String())\n\t\t\tipStr := interfaceIP.String()\n\t\t\tif net.ParseIP(ipStr) != nil {\n\t\t\t\tlocalIPs = append(localIPs, ipStr)\n\t\t\t}\n\t\t}\n\t\tvar portStr = strconv.FormatInt(int64(s.routeInfo.Port), 10)\n\t\tfor _, ip := range localIPs {\n\t\t\tipPort := net.JoinHostPort(ip, portStr)\n\t\t\ts.routesToSelf[ipPort] = struct{}{}\n\t\t}\n\t}\n\n\t// Start the accept loop in a different go routine.\n\tgo s.acceptConnections(l, \"Route\", func(conn net.Conn) { s.createRoute(conn, nil, Implicit, gossipDefault, _EMPTY_) }, nil)\n\n\t// Solicit Routes if applicable. This will not block.\n\ts.solicitRoutes(opts.Routes, opts.Cluster.PinnedAccounts)\n\n\ts.mu.Unlock()\n}\n\n// Similar to setInfoHostPortAndGenerateJSON, but for routeInfo.\nfunc (s *Server) setRouteInfoHostPortAndIP() error {\n\topts := s.getOpts()\n\tif opts.Cluster.Advertise != _EMPTY_ {\n\t\tadvHost, advPort, err := parseHostPort(opts.Cluster.Advertise, opts.Cluster.Port)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ts.routeInfo.Host = advHost\n\t\ts.routeInfo.Port = advPort\n\t\ts.routeInfo.IP = fmt.Sprintf(\"nats-route://%s/\", net.JoinHostPort(advHost, strconv.Itoa(advPort)))\n\t} else {\n\t\ts.routeInfo.Host = opts.Cluster.Host\n\t\ts.routeInfo.Port = opts.Cluster.Port\n\t\ts.routeInfo.IP = \"\"\n\t}\n\treturn nil\n}\n\n// StartRouting will start the accept loop on the cluster host:port\n// and will actively try to connect to listed routes.\nfunc (s *Server) StartRouting(clientListenReady chan struct{}) {\n\tdefer s.grWG.Done()\n\n\t// Wait for the client and leafnode listen ports to be opened,\n\t// and the possible ephemeral ports to be selected.\n\t<-clientListenReady\n\n\t// Start the accept loop and solicitation of explicit routes (if applicable)\n\ts.startRouteAcceptLoop()\n\n}\n\nfunc (s *Server) reConnectToRoute(rURL *url.URL, rtype RouteType, accName string) {\n\t// If A connects to B, and B to A (regardless if explicit or\n\t// implicit - due to auto-discovery), and if each server first\n\t// registers the route on the opposite TCP connection, the\n\t// two connections will end-up being closed.\n\t// Add some random delay to reduce risk of repeated failures.\n\tdelay := time.Duration(rand.Intn(100)) * time.Millisecond\n\tif rtype == Explicit {\n\t\tdelay += DEFAULT_ROUTE_RECONNECT\n\t}\n\tselect {\n\tcase <-time.After(delay):\n\tcase <-s.quitCh:\n\t\ts.grWG.Done()\n\t\treturn\n\t}\n\ts.connectToRoute(rURL, rtype, false, gossipDefault, accName)\n}\n\n// Checks to make sure the route is still valid.\nfunc (s *Server) routeStillValid(rURL *url.URL) bool {\n\tfor _, ri := range s.getOpts().Routes {\n\t\tif urlsAreEqual(ri, rURL) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (s *Server) connectToRoute(rURL *url.URL, rtype RouteType, firstConnect bool, gossipMode byte, accName string) {\n\tdefer s.grWG.Done()\n\tif rURL == nil {\n\t\treturn\n\t}\n\t// For explicit routes, we will try to connect until we succeed. For implicit\n\t// we will try only based on the number of ConnectRetries optin.\n\ttryForEver := rtype == Explicit\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tconst connErrFmt = \"Error trying to connect to route (attempt %v): %v\"\n\n\ts.mu.RLock()\n\tresolver := s.routeResolver\n\texcludedAddresses := s.routesToSelf\n\ts.mu.RUnlock()\n\n\tattemptDelay := routeConnectDelay\n\tfor attempts := 0; s.isRunning(); {\n\t\tif tryForEver {\n\t\t\tif !s.routeStillValid(rURL) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif accName != _EMPTY_ {\n\t\t\t\ts.mu.RLock()\n\t\t\t\t_, valid := s.accRoutes[accName]\n\t\t\t\ts.mu.RUnlock()\n\t\t\t\tif !valid {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tvar conn net.Conn\n\t\taddress, err := s.getRandomIP(resolver, rURL.Host, excludedAddresses)\n\t\tif err == errNoIPAvail {\n\t\t\t// This is ok, we are done.\n\t\t\treturn\n\t\t}\n\t\tif err == nil {\n\t\t\ts.Debugf(\"Trying to connect to route on %s (%s)\", rURL.Host, address)\n\t\t\tconn, err = natsDialTimeout(\"tcp\", address, DEFAULT_ROUTE_DIAL)\n\t\t}\n\t\tif err != nil {\n\t\t\tattempts++\n\t\t\tif s.shouldReportConnectErr(firstConnect, attempts) {\n\t\t\t\ts.Errorf(connErrFmt, attempts, err)\n\t\t\t} else {\n\t\t\t\ts.Debugf(connErrFmt, attempts, err)\n\t\t\t}\n\t\t\tif !tryForEver {\n\t\t\t\tif opts.Cluster.ConnectRetries <= 0 {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif attempts > opts.Cluster.ConnectRetries {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(attemptDelay):\n\t\t\t\tif opts.Cluster.ConnectBackoff {\n\t\t\t\t\t// Use exponential backoff for connection attempts.\n\t\t\t\t\tattemptDelay *= 2\n\t\t\t\t\tif attemptDelay > routeConnectMaxDelay {\n\t\t\t\t\t\tattemptDelay = routeConnectMaxDelay\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif tryForEver && !s.routeStillValid(rURL) {\n\t\t\tconn.Close()\n\t\t\treturn\n\t\t}\n\n\t\t// We have a route connection here.\n\t\t// Go ahead and create it and exit this func.\n\t\ts.createRoute(conn, rURL, rtype, gossipMode, accName)\n\t\treturn\n\t}\n}\n\nfunc (c *client) isSolicitedRoute() bool {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.kind == ROUTER && c.route != nil && c.route.didSolicit\n}\n\n// Save the first hostname found in route URLs. This will be used in gossip mode\n// when trying to create a TLS connection by setting the tlsConfig.ServerName.\n// Lock is held on entry\nfunc (s *Server) saveRouteTLSName(routes []*url.URL) {\n\tfor _, u := range routes {\n\t\tif s.routeTLSName == _EMPTY_ && net.ParseIP(u.Hostname()) == nil {\n\t\t\ts.routeTLSName = u.Hostname()\n\t\t}\n\t}\n}\n\n// Start connection process to provided routes. Each route connection will\n// be started in a dedicated go routine.\n// Lock is held on entry\nfunc (s *Server) solicitRoutes(routes []*url.URL, accounts []string) {\n\ts.saveRouteTLSName(routes)\n\tfor _, r := range routes {\n\t\troute := r\n\t\ts.startGoRoutine(func() { s.connectToRoute(route, Explicit, true, gossipDefault, _EMPTY_) })\n\t}\n\t// Now go over possible per-account routes and create them.\n\tfor _, an := range accounts {\n\t\tfor _, r := range routes {\n\t\t\troute, accName := r, an\n\t\t\ts.startGoRoutine(func() { s.connectToRoute(route, Explicit, true, gossipDefault, accName) })\n\t\t}\n\t}\n}\n\nfunc (c *client) processRouteConnect(srv *Server, arg []byte, lang string) error {\n\t// Way to detect clients that incorrectly connect to the route listen\n\t// port. Client provide Lang in the CONNECT protocol while ROUTEs don't.\n\tif lang != \"\" {\n\t\tc.sendErrAndErr(ErrClientConnectedToRoutePort.Error())\n\t\tc.closeConnection(WrongPort)\n\t\treturn ErrClientConnectedToRoutePort\n\t}\n\t// Unmarshal as a route connect protocol\n\tproto := &connectInfo{}\n\n\tif err := json.Unmarshal(arg, proto); err != nil {\n\t\treturn err\n\t}\n\t// Reject if this has Gateway which means that it would be from a gateway\n\t// connection that incorrectly connects to the Route port.\n\tif proto.Gateway != \"\" {\n\t\terrTxt := fmt.Sprintf(\"Rejecting connection from gateway %q on the Route port\", proto.Gateway)\n\t\tc.Errorf(errTxt)\n\t\tc.sendErr(errTxt)\n\t\tc.closeConnection(WrongGateway)\n\t\treturn ErrWrongGateway\n\t}\n\n\tif srv == nil {\n\t\treturn ErrServerNotRunning\n\t}\n\n\tperms := srv.getOpts().Cluster.Permissions\n\tclusterName := srv.ClusterName()\n\n\t// If we have a cluster name set, make sure it matches ours.\n\tif proto.Cluster != clusterName {\n\t\tshouldReject := true\n\t\t// If we have a dynamic name we will do additional checks.\n\t\tif srv.isClusterNameDynamic() {\n\t\t\tif !proto.Dynamic || strings.Compare(clusterName, proto.Cluster) < 0 {\n\t\t\t\t// We will take on their name since theirs is configured or higher then ours.\n\t\t\t\tsrv.setClusterName(proto.Cluster)\n\t\t\t\tif !proto.Dynamic {\n\t\t\t\t\tsrv.optsMu.Lock()\n\t\t\t\t\tsrv.opts.Cluster.Name = proto.Cluster\n\t\t\t\t\tsrv.optsMu.Unlock()\n\t\t\t\t}\n\t\t\t\tc.mu.Lock()\n\t\t\t\tremoteID := c.opts.Name\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tsrv.removeAllRoutesExcept(remoteID)\n\t\t\t\tshouldReject = false\n\t\t\t}\n\t\t}\n\t\tif shouldReject {\n\t\t\terrTxt := fmt.Sprintf(\"Rejecting connection, cluster name %q does not match %q\", proto.Cluster, clusterName)\n\t\t\tc.Errorf(errTxt)\n\t\t\tc.sendErr(errTxt)\n\t\t\tc.closeConnection(ClusterNameConflict)\n\t\t\treturn ErrClusterNameRemoteConflict\n\t\t}\n\t}\n\n\tsupportsHeaders := c.srv.supportsHeaders()\n\n\t// Grab connection name of remote route.\n\tc.mu.Lock()\n\tc.route.remoteID = c.opts.Name\n\tc.route.lnoc = proto.LNOC\n\tc.route.lnocu = proto.LNOCU\n\tc.setRoutePermissions(perms)\n\tc.headers = supportsHeaders && proto.Headers\n\tc.mu.Unlock()\n\treturn nil\n}\n\n// Called when we update our cluster name during negotiations with remotes.\nfunc (s *Server) removeAllRoutesExcept(remoteID string) {\n\ts.mu.Lock()\n\troutes := make([]*client, 0, s.numRoutes())\n\tfor rID, conns := range s.routes {\n\t\tif rID == remoteID {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, r := range conns {\n\t\t\tif r != nil {\n\t\t\t\troutes = append(routes, r)\n\t\t\t}\n\t\t}\n\t}\n\tfor _, conns := range s.accRoutes {\n\t\tfor rID, r := range conns {\n\t\t\tif rID == remoteID {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\troutes = append(routes, r)\n\t\t}\n\t}\n\ts.mu.Unlock()\n\n\tfor _, r := range routes {\n\t\tr.closeConnection(ClusterNameConflict)\n\t}\n}\n\nfunc (s *Server) removeRoute(c *client) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\tvar (\n\t\trID string\n\t\tlnURL string\n\t\tgwURL string\n\t\tidHash string\n\t\taccName string\n\t\tpoolIdx = -1\n\t\tconnectURLs []string\n\t\twsConnectURLs []string\n\t\topts = s.getOpts()\n\t\trURL *url.URL\n\t\tnoPool bool\n\t\trtype RouteType\n\t)\n\tc.mu.Lock()\n\tcid := c.cid\n\tr := c.route\n\tif r != nil {\n\t\trID = r.remoteID\n\t\tlnURL = r.leafnodeURL\n\t\tidHash = r.idHash\n\t\tgwURL = r.gatewayURL\n\t\tpoolIdx = r.poolIdx\n\t\taccName = bytesToString(r.accName)\n\t\tif r.noPool {\n\t\t\ts.routesNoPool--\n\t\t\tnoPool = true\n\t\t}\n\t\tconnectURLs = r.connectURLs\n\t\twsConnectURLs = r.wsConnURLs\n\t\trURL = r.url\n\t\trtype = r.routeType\n\t}\n\tc.mu.Unlock()\n\tif accName != _EMPTY_ {\n\t\tif conns, ok := s.accRoutes[accName]; ok {\n\t\t\tif r := conns[rID]; r == c {\n\t\t\t\ts.removeRouteByHash(idHash + accName)\n\t\t\t\tdelete(conns, rID)\n\t\t\t\t// Do not remove or set to nil when all remotes have been\n\t\t\t\t// removed from the map. The configured accounts must always\n\t\t\t\t// be in the accRoutes map and addRoute expects \"conns\" map\n\t\t\t\t// to be created.\n\t\t\t}\n\t\t}\n\t}\n\t// If this is still -1, it means that it was not added to the routes\n\t// so simply remove from temp clients and we are done.\n\tif poolIdx == -1 || accName != _EMPTY_ {\n\t\ts.removeFromTempClients(cid)\n\t\treturn\n\t}\n\tif conns, ok := s.routes[rID]; ok {\n\t\t// If this route was not the one stored, simply remove from the\n\t\t// temporary map and be done.\n\t\tif conns[poolIdx] != c {\n\t\t\ts.removeFromTempClients(cid)\n\t\t\treturn\n\t\t}\n\t\tconns[poolIdx] = nil\n\t\t// Now check if this was the last connection to be removed.\n\t\tempty := true\n\t\tfor _, c := range conns {\n\t\t\tif c != nil {\n\t\t\t\tempty = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// This was the last route for this remote. Remove the remote entry\n\t\t// and possibly send some async INFO protocols regarding gateway\n\t\t// and leafnode URLs.\n\t\tif empty {\n\t\t\tdelete(s.routes, rID)\n\n\t\t\t// Since this is the last route for this remote, possibly update\n\t\t\t// the client connect URLs and send an update to connected\n\t\t\t// clients.\n\t\t\tif (len(connectURLs) > 0 || len(wsConnectURLs) > 0) && !opts.Cluster.NoAdvertise {\n\t\t\t\ts.removeConnectURLsAndSendINFOToClients(connectURLs, wsConnectURLs)\n\t\t\t}\n\t\t\t// Remove the remote's gateway URL from our list and\n\t\t\t// send update to inbound Gateway connections.\n\t\t\tif gwURL != _EMPTY_ && s.removeGatewayURL(gwURL) {\n\t\t\t\ts.sendAsyncGatewayInfo()\n\t\t\t}\n\t\t\t// Remove the remote's leafNode URL from\n\t\t\t// our list and send update to LN connections.\n\t\t\tif lnURL != _EMPTY_ && s.removeLeafNodeURL(lnURL) {\n\t\t\t\ts.sendAsyncLeafNodeInfo()\n\t\t\t}\n\t\t\t// We can remove the configured route pool size of this remote.\n\t\t\tdelete(s.remoteRoutePoolSize, rID)\n\t\t\t// If this server has pooling/pinned accounts and the route for\n\t\t\t// this remote was a \"no pool\" route, attempt to reconnect.\n\t\t\tif noPool {\n\t\t\t\tif s.routesPoolSize > 1 {\n\t\t\t\t\ts.startGoRoutine(func() { s.connectToRoute(rURL, rtype, true, gossipDefault, _EMPTY_) })\n\t\t\t\t}\n\t\t\t\tif len(opts.Cluster.PinnedAccounts) > 0 {\n\t\t\t\t\tfor _, an := range opts.Cluster.PinnedAccounts {\n\t\t\t\t\t\taccName := an\n\t\t\t\t\t\ts.startGoRoutine(func() { s.connectToRoute(rURL, rtype, true, gossipDefault, accName) })\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// This is for gateway code. Remove this route from a map that uses\n\t\t// the route hash in combination with the pool index as the key.\n\t\tif s.routesPoolSize > 1 {\n\t\t\tidHash += strconv.Itoa(poolIdx)\n\t\t}\n\t\ts.removeRouteByHash(idHash)\n\t}\n\ts.removeFromTempClients(cid)\n}\n\nfunc (s *Server) isDuplicateServerName(name string) bool {\n\tif name == _EMPTY_ {\n\t\treturn false\n\t}\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\n\tif s.info.Name == name {\n\t\treturn true\n\t}\n\tfor _, conns := range s.routes {\n\t\tfor _, r := range conns {\n\t\t\tif r != nil {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tduplicate := r.route.remoteName == name\n\t\t\t\tr.mu.Unlock()\n\t\t\t\tif duplicate {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Goes over each non-nil route connection for all remote servers\n// and invokes the function `f`. It does not go over per-account\n// routes.\n// Server lock is held on entry.\nfunc (s *Server) forEachNonPerAccountRoute(f func(r *client)) {\n\tfor _, conns := range s.routes {\n\t\tfor _, r := range conns {\n\t\t\tif r != nil {\n\t\t\t\tf(r)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Goes over each non-nil route connection for all remote servers\n// and invokes the function `f`. This also includes the per-account\n// routes.\n// Server lock is held on entry.\nfunc (s *Server) forEachRoute(f func(r *client)) {\n\ts.forEachNonPerAccountRoute(f)\n\tfor _, conns := range s.accRoutes {\n\t\tfor _, r := range conns {\n\t\t\tf(r)\n\t\t}\n\t}\n}\n\n// Goes over each non-nil route connection at the given pool index\n// location in the slice and invokes the function `f`. If the\n// callback returns `true`, this function moves to the next remote.\n// Otherwise, the iteration over removes stops.\n// This does not include per-account routes.\n// Server lock is held on entry.\nfunc (s *Server) forEachRouteIdx(idx int, f func(r *client) bool) {\n\tfor _, conns := range s.routes {\n\t\tif r := conns[idx]; r != nil {\n\t\t\tif !f(r) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Goes over each remote and for the first non nil route connection,\n// invokes the function `f`.\n// Server lock is held on entry.\nfunc (s *Server) forEachRemote(f func(r *client)) {\n\tfor _, conns := range s.routes {\n\t\tfor _, r := range conns {\n\t\t\tif r != nil {\n\t\t\t\tf(r)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "server/routes_test.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc init() {\n\trouteConnectDelay = 15 * time.Millisecond\n\trouteConnectMaxDelay = 15 * time.Millisecond\n}\n\nfunc checkNumRoutes(t *testing.T, s *Server, expected int) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 15*time.Millisecond, func() error {\n\t\tif nr := s.NumRoutes(); nr != expected {\n\t\t\treturn fmt.Errorf(\"Expected %v routes, got %v\", expected, nr)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkSubInterest(t *testing.T, s *Server, accName, subject string, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tacc, err := s.LookupAccount(accName)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error looking up account %q: %v\", accName, err)\n\t\t}\n\t\tif acc.SubscriptionInterest(subject) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"no subscription interest for account %q on %q\", accName, subject)\n\t})\n}\n\nfunc checkSubNoInterest(t *testing.T, s *Server, accName, subject string, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tacc, err := s.LookupAccount(accName)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error looking up account %q: %v\", accName, err)\n\t\t}\n\t\tif acc.SubscriptionInterest(subject) {\n\t\t\treturn fmt.Errorf(\"unexpected subscription interest for account %q on %q\", accName, subject)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestRouteConfig(t *testing.T) {\n\topts, err := ProcessConfigFile(\"./configs/cluster.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Received an error reading route config file: %v\\n\", err)\n\t}\n\n\tgolden := &Options{\n\t\tConfigFile: \"./configs/cluster.conf\",\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 4242,\n\t\tUsername: \"derek\",\n\t\tPassword: \"porkchop\",\n\t\tAuthTimeout: 1.0,\n\t\tCluster: ClusterOpts{\n\t\t\tName: \"abc\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: 4244,\n\t\t\tUsername: \"route_user\",\n\t\t\tPassword: \"top_secret\",\n\t\t\tAuthTimeout: 1.0,\n\t\t\tNoAdvertise: true,\n\t\t\tConnectRetries: 2,\n\t\t\tConnectBackoff: true,\n\t\t},\n\t\tPidFile: \"/tmp/nats-server/nats_cluster_test.pid\",\n\t\tconfigDigest: \"sha256:b2f6d54063e43a85f5a09d198204f044bc154f9f9ccfb84e47d5762091aec487\",\n\t\tauthBlockDefined: true,\n\t}\n\n\t// Setup URLs\n\tr1, _ := url.Parse(\"nats-route://foo:bar@127.0.0.1:4245\")\n\tr2, _ := url.Parse(\"nats-route://foo:bar@127.0.0.1:4246\")\n\n\tgolden.Routes = []*url.URL{r1, r2}\n\n\tcheckOptionsEqual(t, golden, opts)\n}\n\nfunc TestClusterAdvertise(t *testing.T) {\n\tlst, err := natsListen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error starting listener: %v\", err)\n\t}\n\tch := make(chan error)\n\tgo func() {\n\t\tc, err := lst.Accept()\n\t\tif err != nil {\n\t\t\tch <- err\n\t\t\treturn\n\t\t}\n\t\tc.Close()\n\t\tch <- nil\n\t}()\n\n\toptsA, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\toptsA.NoSigs, optsA.NoLog = true, true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvARouteURL := fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, srvA.ClusterAddr().Port)\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(srvARouteURL)\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for these 2 to connect to each other\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Now start server C that connects to A. A should ask B to connect to C,\n\t// based on C's URL. But since C configures a Cluster.Advertise, it will connect\n\t// to our listener.\n\toptsC := nextServerOpts(optsB)\n\toptsC.Cluster.Advertise = lst.Addr().String()\n\toptsC.ClientAdvertise = \"me:1\"\n\toptsC.Routes = RoutesFromStr(srvARouteURL)\n\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\tselect {\n\tcase e := <-ch:\n\t\tif e != nil {\n\t\t\tt.Fatalf(\"Error: %v\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Test timed out\")\n\t}\n}\n\nfunc TestClusterAdvertiseErrorOnStartup(t *testing.T) {\n\topts := DefaultOptions()\n\t// Set invalid address\n\topts.Cluster.Advertise = \"addr:::123\"\n\ttestFatalErrorOnStart(t, opts, \"Cluster.Advertise\")\n}\n\nfunc TestClientAdvertise(t *testing.T) {\n\toptsA, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\toptsA.NoSigs, optsA.NoLog = true, true\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\toptsB.ClientAdvertise = \"me:1\"\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tds := nc.DiscoveredServers()\n\t\tif len(ds) == 1 {\n\t\t\tif ds[0] == \"nats://me:1\" {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn fmt.Errorf(\"Did not get expected discovered servers: %v\", nc.DiscoveredServers())\n\t})\n}\n\nfunc TestServerRoutesWithClients(t *testing.T) {\n\toptsA, err := ProcessConfigFile(\"./configs/srv_a.conf\")\n\trequire_NoError(t, err)\n\toptsB, err := ProcessConfigFile(\"./configs/srv_b.conf\")\n\trequire_NoError(t, err)\n\n\toptsA.NoSigs, optsA.NoLog = true, true\n\toptsB.NoSigs, optsB.NoLog = true, true\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\tch := make(chan bool)\n\tsub, _ := nc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc1.QueueSubscribe(\"foo\", \"bar\", func(m *nats.Msg) {})\n\tnc1.Publish(\"foo\", []byte(\"Hello\"))\n\t// Wait for message\n\t<-ch\n\tsub.Unsubscribe()\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for route to form.\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\tnc2.Flush()\n}\n\nfunc TestServerRoutesWithAuthAndBCrypt(t *testing.T) {\n\toptsA, err := ProcessConfigFile(\"./configs/srv_a_bcrypt.conf\")\n\trequire_NoError(t, err)\n\toptsB, err := ProcessConfigFile(\"./configs/srv_b_bcrypt.conf\")\n\trequire_NoError(t, err)\n\n\toptsA.NoSigs, optsA.NoLog = true, true\n\toptsB.NoSigs, optsB.NoLog = true, true\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Wait for route to form.\n\tcheckClusterFormed(t, srvA, srvB)\n\n\turlA := fmt.Sprintf(\"nats://%s:%s@%s:%d/\", optsA.Username, optsA.Password, optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%s@%s:%d/\", optsB.Username, optsB.Password, optsB.Host, optsB.Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Test that we are connected.\n\tch := make(chan bool)\n\tsub, err := nc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating subscription: %v\\n\", err)\n\t}\n\tnc1.Flush()\n\tdefer sub.Unsubscribe()\n\n\tcheckSubInterest(t, srvB, globalAccountName, \"foo\", time.Second)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\tnc2.Flush()\n\n\t// Wait for message\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t}\n}\n\n// Helper function to check that a cluster is formed\nfunc checkClusterFormed(t testing.TB, servers ...*Server) {\n\tt.Helper()\n\tvar _enr [8]int\n\tenr := _enr[:0]\n\tfor _, a := range servers {\n\t\tif a.getOpts().Cluster.PoolSize < 0 {\n\t\t\tenr = append(enr, len(servers)-1)\n\t\t} else {\n\t\t\ta.mu.RLock()\n\t\t\tnr := a.routesPoolSize + len(a.accRoutes)\n\t\t\ta.mu.RUnlock()\n\t\t\ttotal := 0\n\t\t\tfor _, b := range servers {\n\t\t\t\tif a == b {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tbo := b.getOpts()\n\t\t\t\tif ps := bo.Cluster.PoolSize; ps < 0 {\n\t\t\t\t\ttotal++\n\t\t\t\t} else {\n\t\t\t\t\tbps := ps + len(bo.Cluster.PinnedAccounts)\n\t\t\t\t\ttotal += max(nr, bps)\n\t\t\t\t}\n\t\t\t}\n\t\t\tenr = append(enr, total)\n\t\t}\n\t}\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tfor i, s := range servers {\n\t\t\tif numRoutes := s.NumRoutes(); numRoutes != enr[i] {\n\t\t\t\treturn fmt.Errorf(\"Expected %d routes for server %q, got %d\", enr[i], s, numRoutes)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Helper function to generate next opts to make sure no port conflicts etc.\nfunc nextServerOpts(opts *Options) *Options {\n\tnopts := *opts\n\tnopts.Port = -1\n\tnopts.Cluster.Port = -1\n\tnopts.HTTPPort = -1\n\tif nopts.Gateway.Name != \"\" {\n\t\tnopts.Gateway.Port = -1\n\t}\n\tnopts.ServerName = \"\"\n\treturn &nopts\n}\n\nfunc TestSeedSolicitWorks(t *testing.T) {\n\toptsSeed, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\n\toptsSeed.NoSigs, optsSeed.NoLog = true, true\n\toptsSeed.NoSystemAccount = true\n\n\tsrvSeed := RunServer(optsSeed)\n\tdefer srvSeed.Shutdown()\n\n\toptsA := nextServerOpts(optsSeed)\n\toptsA.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host,\n\t\tsrvSeed.ClusterAddr().Port))\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, srvA.ClusterAddr().Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Test that we are connected.\n\tch := make(chan bool)\n\tnc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc1.Flush()\n\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host,\n\t\tsrvSeed.ClusterAddr().Port))\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, srvB.ClusterAddr().Port)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckClusterFormed(t, srvSeed, srvA, srvB)\n\tcheckExpectedSubs(t, 1, srvB)\n\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t}\n}\n\nfunc TestTLSSeedSolicitWorks(t *testing.T) {\n\toptsSeed, err := ProcessConfigFile(\"./configs/seed_tls.conf\")\n\trequire_NoError(t, err)\n\n\toptsSeed.NoSigs, optsSeed.NoLog = true, true\n\toptsSeed.NoSystemAccount = true\n\n\tsrvSeed := RunServer(optsSeed)\n\tdefer srvSeed.Shutdown()\n\n\tseedRouteURL := fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host,\n\t\tsrvSeed.ClusterAddr().Port)\n\toptsA := nextServerOpts(optsSeed)\n\toptsA.Routes = RoutesFromStr(seedRouteURL)\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, srvA.Addr().(*net.TCPAddr).Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Test that we are connected.\n\tch := make(chan bool)\n\tnc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc1.Flush()\n\n\toptsB := nextServerOpts(optsA)\n\toptsB.Routes = RoutesFromStr(seedRouteURL)\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, srvB.Addr().(*net.TCPAddr).Port)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckClusterFormed(t, srvSeed, srvA, srvB)\n\tcheckExpectedSubs(t, 1, srvB)\n\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t}\n}\n\nfunc TestChainedSolicitWorks(t *testing.T) {\n\toptsSeed, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\n\toptsSeed.NoSigs, optsSeed.NoLog = true, true\n\toptsSeed.NoSystemAccount = true\n\n\tsrvSeed := RunServer(optsSeed)\n\tdefer srvSeed.Shutdown()\n\n\tseedRouteURL := fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host,\n\t\tsrvSeed.ClusterAddr().Port)\n\toptsA := nextServerOpts(optsSeed)\n\toptsA.Routes = RoutesFromStr(seedRouteURL)\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\turlSeed := fmt.Sprintf(\"nats://%s:%d/\", optsSeed.Host, srvA.Addr().(*net.TCPAddr).Port)\n\n\tnc1, err := nats.Connect(urlSeed)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Test that we are connected.\n\tch := make(chan bool)\n\tnc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc1.Flush()\n\n\toptsB := nextServerOpts(optsA)\n\t// Server B connects to A\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host,\n\t\tsrvA.ClusterAddr().Port))\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, srvB.Addr().(*net.TCPAddr).Port)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckClusterFormed(t, srvSeed, srvA, srvB)\n\tcheckExpectedSubs(t, 1, srvB)\n\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t}\n}\n\n// Helper function to check that a server (or list of servers) have the\n// expected number of subscriptions.\nfunc checkExpectedSubs(t *testing.T, expected int, servers ...*Server) {\n\tt.Helper()\n\tcheckFor(t, 4*time.Second, 10*time.Millisecond, func() error {\n\t\tfor _, s := range servers {\n\t\t\tif numSubs := int(s.NumSubscriptions()); numSubs != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d subscriptions for server %q, got %d\", expected, s.ID(), numSubs)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestTLSChainedSolicitWorks(t *testing.T) {\n\toptsSeed, err := ProcessConfigFile(\"./configs/seed_tls.conf\")\n\trequire_NoError(t, err)\n\n\toptsSeed.NoSigs, optsSeed.NoLog = true, true\n\toptsSeed.NoSystemAccount = true\n\n\tsrvSeed := RunServer(optsSeed)\n\tdefer srvSeed.Shutdown()\n\n\turlSeedRoute := fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host,\n\t\tsrvSeed.ClusterAddr().Port)\n\toptsA := nextServerOpts(optsSeed)\n\toptsA.Routes = RoutesFromStr(urlSeedRoute)\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\turlSeed := fmt.Sprintf(\"nats://%s:%d/\", optsSeed.Host, srvSeed.Addr().(*net.TCPAddr).Port)\n\n\tnc1, err := nats.Connect(urlSeed)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Test that we are connected.\n\tch := make(chan bool)\n\tnc1.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc1.Flush()\n\n\toptsB := nextServerOpts(optsA)\n\t// Server B connects to A\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host,\n\t\tsrvA.ClusterAddr().Port))\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvSeed, srvA, srvB)\n\tcheckExpectedSubs(t, 1, srvA, srvB)\n\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, srvB.Addr().(*net.TCPAddr).Port)\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t}\n}\n\nfunc TestRouteTLSHandshakeError(t *testing.T) {\n\toptsSeed, err := ProcessConfigFile(\"./configs/seed_tls.conf\")\n\trequire_NoError(t, err)\n\toptsSeed.NoLog = true\n\toptsSeed.NoSigs = true\n\tsrvSeed := RunServer(optsSeed)\n\tdefer srvSeed.Shutdown()\n\n\topts := DefaultOptions()\n\topts.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host, optsSeed.Cluster.Port))\n\n\tsrv := RunServer(opts)\n\tdefer srv.Shutdown()\n\n\ttime.Sleep(500 * time.Millisecond)\n\n\tcheckNumRoutes(t, srv, 0)\n}\n\nfunc TestBlockedShutdownOnRouteAcceptLoopFailure(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Cluster.Host = \"x.x.x.x\"\n\topts.Cluster.Port = 7222\n\n\ts := New(opts)\n\ts.Start()\n\t// Wait a second\n\ttime.Sleep(time.Second)\n\tch := make(chan bool)\n\tgo func() {\n\t\ts.Shutdown()\n\t\tch <- true\n\t}()\n\n\ttimeout := time.NewTimer(5 * time.Second)\n\tselect {\n\tcase <-ch:\n\t\treturn\n\tcase <-timeout.C:\n\t\tt.Fatal(\"Shutdown did not complete\")\n\t}\n}\n\nfunc TestRouteUseIPv6(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Cluster.Host = \"::\"\n\topts.Cluster.Port = 6222\n\n\t// I believe that there is no IPv6 support on Travis...\n\t// Regardless, cannot have this test fail simply because IPv6 is disabled\n\t// on the host.\n\thp := net.JoinHostPort(opts.Cluster.Host, strconv.Itoa(opts.Cluster.Port))\n\t_, err := net.ResolveTCPAddr(\"tcp\", hp)\n\tif err != nil {\n\t\tt.Skipf(\"Skipping this test since there is no IPv6 support on this host: %v\", err)\n\t}\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\trouteUp := false\n\ttimeout := time.Now().Add(5 * time.Second)\n\tfor time.Now().Before(timeout) && !routeUp {\n\t\t// We know that the server is local and listening to\n\t\t// all IPv6 interfaces. Try connect using IPv6 loopback.\n\t\tif conn, err := net.Dial(\"tcp\", \"[::1]:6222\"); err != nil {\n\t\t\t// Travis seem to have the server actually listening to 0.0.0.0,\n\t\t\t// so try with 127.0.0.1\n\t\t\tif conn, err := net.Dial(\"tcp\", \"127.0.0.1:6222\"); err != nil {\n\t\t\t\ttime.Sleep(time.Second)\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tconn.Close()\n\t\t\t}\n\t\t} else {\n\t\t\tconn.Close()\n\t\t}\n\t\trouteUp = true\n\t}\n\tif !routeUp {\n\t\tt.Fatal(\"Server failed to start route accept loop\")\n\t}\n}\n\nfunc TestClientConnectToRoutePort(t *testing.T) {\n\topts := DefaultOptions()\n\n\t// Since client will first connect to the route listen port, set the\n\t// cluster's Host to 127.0.0.1 so it works on Windows too, since on\n\t// Windows, a client can't use 0.0.0.0 in a connect.\n\topts.Cluster.Host = \"127.0.0.1\"\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Cluster.Host, s.ClusterAddr().Port)\n\tclientURL := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t// When connecting to the ROUTE port, the client library will receive the\n\t// CLIENT port in the INFO protocol. This URL is added to the client's pool\n\t// and will be tried after the initial connect failure. So all those\n\t// nats.Connect() should succeed.\n\t// The only reason for a failure would be if there are too many FDs in time-wait\n\t// which would delay the creation of TCP connection. So keep the total of\n\t// attempts rather small.\n\ttotal := 10\n\tfor i := 0; i < total; i++ {\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tif nc.ConnectedUrl() != clientURL {\n\t\t\tt.Fatalf(\"Expected client to be connected to %v, got %v\", clientURL, nc.ConnectedUrl())\n\t\t}\n\t}\n\n\ts.Shutdown()\n\t// Try again with NoAdvertise and this time, the client should fail to connect.\n\topts.Cluster.NoAdvertise = true\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfor i := 0; i < total; i++ {\n\t\tnc, err := nats.Connect(url)\n\t\tif err == nil {\n\t\t\tnc.Close()\n\t\t\tt.Fatal(\"Expected error on connect, got none\")\n\t\t}\n\t}\n}\n\ntype checkDuplicateRouteLogger struct {\n\tDummyLogger\n\tgotDuplicate bool\n}\n\nfunc (l *checkDuplicateRouteLogger) Debugf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"duplicate remote route\") {\n\t\tl.gotDuplicate = true\n\t}\n}\n\nfunc TestRoutesToEachOther(t *testing.T) {\n\toptsA := DefaultOptions()\n\toptsA.Cluster.Port = 7246\n\toptsA.Routes = RoutesFromStr(\"nats://127.0.0.1:7247\")\n\n\toptsB := DefaultOptions()\n\toptsB.Cluster.Port = 7247\n\toptsB.Routes = RoutesFromStr(\"nats://127.0.0.1:7246\")\n\n\tsrvALogger := &checkDuplicateRouteLogger{}\n\tsrvA := New(optsA)\n\tsrvA.SetLogger(srvALogger, true, false)\n\tdefer srvA.Shutdown()\n\n\tsrvBLogger := &checkDuplicateRouteLogger{}\n\tsrvB := New(optsB)\n\tsrvB.SetLogger(srvBLogger, true, false)\n\tdefer srvB.Shutdown()\n\n\tgo srvA.Start()\n\tgo srvB.Start()\n\n\tstart := time.Now()\n\tcheckClusterFormed(t, srvA, srvB)\n\tend := time.Now()\n\n\tsrvALogger.Lock()\n\tgotIt := srvALogger.gotDuplicate\n\tsrvALogger.Unlock()\n\tif !gotIt {\n\t\tsrvBLogger.Lock()\n\t\tgotIt = srvBLogger.gotDuplicate\n\t\tsrvBLogger.Unlock()\n\t}\n\tif gotIt {\n\t\tdur := end.Sub(start)\n\t\t// It should not take too long to have a successful connection\n\t\t// between the 2 servers.\n\t\tif dur > 5*time.Second {\n\t\t\tt.Logf(\"Cluster formed, but took a long time: %v\", dur)\n\t\t}\n\t} else {\n\t\tt.Log(\"Was not able to get duplicate route this time!\")\n\t}\n}\n\nfunc wait(ch chan bool) error {\n\tselect {\n\tcase <-ch:\n\t\treturn nil\n\tcase <-time.After(5 * time.Second):\n\t}\n\treturn fmt.Errorf(\"timeout\")\n}\n\nfunc TestServerPoolUpdatedWhenRouteGoesAway(t *testing.T) {\n\ts1Opts := DefaultOptions()\n\ts1Opts.ServerName = \"A\"\n\ts1Opts.Host = \"127.0.0.1\"\n\ts1Opts.Port = 4222\n\ts1Opts.Cluster.Host = \"127.0.0.1\"\n\ts1Opts.Cluster.Port = 6222\n\ts1Opts.Routes = RoutesFromStr(\"nats://127.0.0.1:6223,nats://127.0.0.1:6224\")\n\ts1 := RunServer(s1Opts)\n\tdefer s1.Shutdown()\n\n\ts1Url := \"nats://127.0.0.1:4222\"\n\ts2Url := \"nats://127.0.0.1:4223\"\n\ts3Url := \"nats://127.0.0.1:4224\"\n\n\tch := make(chan bool, 1)\n\tchch := make(chan bool, 1)\n\tconnHandler := func(_ *nats.Conn) {\n\t\tchch <- true\n\t}\n\tnc, err := nats.Connect(s1Url,\n\t\tnats.ReconnectWait(50*time.Millisecond),\n\t\tnats.ReconnectHandler(connHandler),\n\t\tnats.DiscoveredServersHandler(func(_ *nats.Conn) {\n\t\t\tch <- true\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect\")\n\t}\n\tdefer nc.Close()\n\n\ts2Opts := DefaultOptions()\n\ts2Opts.ServerName = \"B\"\n\ts2Opts.Host = \"127.0.0.1\"\n\ts2Opts.Port = s1Opts.Port + 1\n\ts2Opts.Cluster.Host = \"127.0.0.1\"\n\ts2Opts.Cluster.Port = 6223\n\ts2Opts.Routes = RoutesFromStr(\"nats://127.0.0.1:6222,nats://127.0.0.1:6224\")\n\ts2 := RunServer(s2Opts)\n\tdefer s2.Shutdown()\n\n\t// Wait to be notified\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"New server callback was not invoked\")\n\t}\n\n\tcheckPool := func(expected []string) {\n\t\tt.Helper()\n\t\t// Don't use discovered here, but Servers to have the full list.\n\t\t// Also, there may be cases where the mesh is not formed yet,\n\t\t// so try again on failure.\n\t\tcheckFor(t, 5*time.Second, 50*time.Millisecond, func() error {\n\t\t\tds := nc.Servers()\n\t\t\tif len(ds) == len(expected) {\n\t\t\t\tm := make(map[string]struct{}, len(ds))\n\t\t\t\tfor _, url := range ds {\n\t\t\t\t\tm[url] = struct{}{}\n\t\t\t\t}\n\t\t\t\tok := true\n\t\t\t\tfor _, url := range expected {\n\t\t\t\t\tif _, present := m[url]; !present {\n\t\t\t\t\t\tok = false\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif ok {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Expected %v, got %v\", expected, ds)\n\t\t})\n\t}\n\t// Verify that we now know about s2\n\tcheckPool([]string{s1Url, s2Url})\n\n\ts3Opts := DefaultOptions()\n\ts3Opts.ServerName = \"C\"\n\ts3Opts.Host = \"127.0.0.1\"\n\ts3Opts.Port = s2Opts.Port + 1\n\ts3Opts.Cluster.Host = \"127.0.0.1\"\n\ts3Opts.Cluster.Port = 6224\n\ts3Opts.Routes = RoutesFromStr(\"nats://127.0.0.1:6222,nats://127.0.0.1:6223\")\n\ts3 := RunServer(s3Opts)\n\tdefer s3.Shutdown()\n\n\t// Wait to be notified\n\tif err := wait(ch); err != nil {\n\t\tt.Fatal(\"New server callback was not invoked\")\n\t}\n\t// Verify that we now know about s3\n\tcheckPool([]string{s1Url, s2Url, s3Url})\n\n\t// Stop s1. Since this was passed to the Connect() call, this one should\n\t// still be present.\n\ts1.Shutdown()\n\t// Wait for reconnect\n\tif err := wait(chch); err != nil {\n\t\tt.Fatal(\"Reconnect handler not invoked\")\n\t}\n\tcheckPool([]string{s1Url, s2Url, s3Url})\n\n\t// Check the server we reconnected to.\n\treConnectedTo := nc.ConnectedUrl()\n\texpected := []string{s1Url}\n\tif reConnectedTo == s2Url {\n\t\ts2.Shutdown()\n\t\texpected = append(expected, s3Url)\n\t} else if reConnectedTo == s3Url {\n\t\ts3.Shutdown()\n\t\texpected = append(expected, s2Url)\n\t} else {\n\t\tt.Fatalf(\"Unexpected server client has reconnected to: %v\", reConnectedTo)\n\t}\n\t// Wait for reconnect\n\tif err := wait(chch); err != nil {\n\t\tt.Fatal(\"Reconnect handler not invoked\")\n\t}\n\t// The implicit server that we just shutdown should have been removed from the pool\n\tcheckPool(expected)\n\tnc.Close()\n}\n\nfunc TestRouteFailedConnRemovedFromTmpMap(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize int\n\t}{\n\t\t{\"no pooling\", -1},\n\t\t{\"pool 1\", 1},\n\t\t{\"pool 3\", 3},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toptsA, err := ProcessConfigFile(\"./configs/srv_a.conf\")\n\t\t\trequire_NoError(t, err)\n\t\t\toptsA.NoSigs, optsA.NoLog = true, true\n\t\t\toptsA.Cluster.PoolSize = test.poolSize\n\n\t\t\toptsB, err := ProcessConfigFile(\"./configs/srv_b.conf\")\n\t\t\trequire_NoError(t, err)\n\t\t\toptsB.NoSigs, optsB.NoLog = true, true\n\t\t\toptsB.Cluster.PoolSize = test.poolSize\n\n\t\t\tsrvA := New(optsA)\n\t\t\tdefer srvA.Shutdown()\n\t\t\tsrvB := New(optsB)\n\t\t\tdefer srvB.Shutdown()\n\n\t\t\t// Start this way to increase chance of having the two connect\n\t\t\t// to each other at the same time. This will cause one of the\n\t\t\t// route to be dropped.\n\t\t\twg := &sync.WaitGroup{}\n\t\t\twg.Add(2)\n\t\t\tgo func() {\n\t\t\t\tsrvA.Start()\n\t\t\t\twg.Done()\n\t\t\t}()\n\t\t\tgo func() {\n\t\t\t\tsrvB.Start()\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\tcheckClusterFormed(t, srvA, srvB)\n\n\t\t\t// Ensure that maps are empty\n\t\t\tcheckMap := func(s *Server) {\n\t\t\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\ts.grMu.Lock()\n\t\t\t\t\tl := len(s.grTmpClients)\n\t\t\t\t\ts.grMu.Unlock()\n\t\t\t\t\tif l != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"grTmpClients map should be empty, got %v\", l)\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t}\n\t\t\tcheckMap(srvA)\n\t\t\tcheckMap(srvB)\n\n\t\t\tsrvB.Shutdown()\n\t\t\tsrvA.Shutdown()\n\t\t\twg.Wait()\n\t\t})\n\t}\n}\n\nfunc getFirstRoute(s *Server) *client {\n\tfor _, conns := range s.routes {\n\t\tfor _, r := range conns {\n\t\t\tif r != nil {\n\t\t\t\treturn r\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc TestRoutePermsAppliedOnInboundAndOutboundRoute(t *testing.T) {\n\n\tperms := &RoutePermissions{\n\t\tImport: &SubjectPermission{\n\t\t\tAllow: []string{\"imp.foo\"},\n\t\t\tDeny: []string{\"imp.bar\"},\n\t\t},\n\t\tExport: &SubjectPermission{\n\t\t\tAllow: []string{\"exp.foo\"},\n\t\t\tDeny: []string{\"exp.bar\"},\n\t\t},\n\t}\n\n\toptsA, err := ProcessConfigFile(\"./configs/seed.conf\")\n\trequire_NoError(t, err)\n\toptsA.NoLog = true\n\toptsA.NoSigs = true\n\toptsA.Cluster.Permissions = perms\n\tsrva := RunServer(optsA)\n\tdefer srva.Shutdown()\n\n\toptsB := DefaultOptions()\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\tsrvb := RunServer(optsB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Ensure permission is properly set\n\tcheck := func(t *testing.T, s *Server) {\n\t\tt.Helper()\n\t\tvar route *client\n\t\ts.mu.Lock()\n\t\troute = getFirstRoute(s)\n\t\ts.mu.Unlock()\n\t\troute.mu.Lock()\n\t\tperms := route.perms\n\t\troute.mu.Unlock()\n\t\tif perms == nil {\n\t\t\tt.Fatal(\"Expected perms to be set\")\n\t\t}\n\t\tif perms.pub.allow == nil || perms.pub.allow.Count() != 1 {\n\t\t\tt.Fatal(\"unexpected pub allow perms\")\n\t\t}\n\t\tif r := perms.pub.allow.Match(\"imp.foo\"); len(r.psubs) != 1 {\n\t\t\tt.Fatal(\"unexpected pub allow match\")\n\t\t}\n\t\tif perms.pub.deny == nil || perms.pub.deny.Count() != 1 {\n\t\t\tt.Fatal(\"unexpected pub deny perms\")\n\t\t}\n\t\tif r := perms.pub.deny.Match(\"imp.bar\"); len(r.psubs) != 1 {\n\t\t\tt.Fatal(\"unexpected pub deny match\")\n\t\t}\n\t\tif perms.sub.allow == nil || perms.sub.allow.Count() != 1 {\n\t\t\tt.Fatal(\"unexpected sub allow perms\")\n\t\t}\n\t\tif r := perms.sub.allow.Match(\"exp.foo\"); len(r.psubs) != 1 {\n\t\t\tt.Fatal(\"unexpected sub allow match\")\n\t\t}\n\t\tif perms.sub.deny == nil || perms.sub.deny.Count() != 1 {\n\t\t\tt.Fatal(\"unexpected sub deny perms\")\n\t\t}\n\t\tif r := perms.sub.deny.Match(\"exp.bar\"); len(r.psubs) != 1 {\n\t\t\tt.Fatal(\"unexpected sub deny match\")\n\t\t}\n\t}\n\n\t// First check when permissions are set on the server accepting the route connection\n\tcheck(t, srva)\n\n\tsrvb.Shutdown()\n\tsrva.Shutdown()\n\n\toptsA.Cluster.Permissions = nil\n\toptsB.Cluster.Permissions = perms\n\n\tsrva = RunServer(optsA)\n\tdefer srva.Shutdown()\n\n\tsrvb = RunServer(optsB)\n\tdefer srvb.Shutdown()\n\n\tcheckClusterFormed(t, srva, srvb)\n\n\t// Now check for permissions set on server initiating the route connection\n\tcheck(t, srvb)\n}\n\nfunc TestRouteSendLocalSubsWithLowMaxPending(t *testing.T) {\n\toptsA := DefaultOptions()\n\toptsA.MaxPayload = 1024\n\toptsA.MaxPending = 1024\n\toptsA.NoSystemAccount = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnumSubs := 1000\n\tfor i := 0; i < numSubs; i++ {\n\t\tsubj := fmt.Sprintf(\"fo.bar.%d\", i)\n\t\tnc.Subscribe(subj, func(_ *nats.Msg) {})\n\t}\n\tcheckExpectedSubs(t, numSubs, srvA)\n\n\t// Now create a route between B and A\n\toptsB := DefaultOptions()\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsA.Cluster.Host, optsA.Cluster.Port))\n\toptsB.NoSystemAccount = true\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Check that all subs have been sent ok\n\tcheckExpectedSubs(t, numSubs, srvA, srvB)\n}\n\nfunc TestRouteNoCrashOnAddingSubToRoute(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tnumRoutes := routeTargetInit + 2\n\ttotal := int32(numRoutes)\n\tcount := int32(0)\n\tch := make(chan bool, 1)\n\tcb := func(_ *nats.Msg) {\n\t\tif n := atomic.AddInt32(&count, 1); n == total {\n\t\t\tch <- true\n\t\t}\n\t}\n\n\tvar servers []*Server\n\tservers = append(servers, s)\n\n\tseedURL := fmt.Sprintf(\"nats://%s:%d\", opts.Cluster.Host, opts.Cluster.Port)\n\tfor i := 0; i < numRoutes; i++ {\n\t\tropts := DefaultOptions()\n\t\tropts.Routes = RoutesFromStr(seedURL)\n\t\trs := RunServer(ropts)\n\t\tdefer rs.Shutdown()\n\t\tservers = append(servers, rs)\n\n\t\t// Confirm routes are active before clients connect.\n\t\tfor _, srv := range servers {\n\t\t\trz, err := srv.Routez(nil)\n\t\t\trequire_NoError(t, err)\n\t\t\tfor i, route := range rz.Routes {\n\t\t\t\tif route.LastActivity.IsZero() {\n\t\t\t\t\tt.Errorf(\"Expected LastActivity to be valid (%d)\", i)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Create a sub on each routed server.\n\t\tnc := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", ropts.Host, ropts.Port))\n\t\tdefer nc.Close()\n\t\tnatsSub(t, nc, \"foo\", cb)\n\t}\n\tcheckClusterFormed(t, servers...)\n\n\t// Make sure all subs are registered in s.\n\tgacc := s.globalAccount()\n\tgacc.mu.RLock()\n\tsl := gacc.sl\n\tgacc.mu.RUnlock()\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tvar _subs [64]*subscription\n\t\tsubs := _subs[:0]\n\t\tsl.All(&subs)\n\t\tvar ts int\n\t\tfor _, sub := range subs {\n\t\t\tif string(sub.subject) == \"foo\" {\n\t\t\t\tts++\n\t\t\t}\n\t\t}\n\t\tif ts != int(numRoutes) {\n\t\t\treturn fmt.Errorf(\"Not all %d routed subs were registered: %d\", numRoutes, ts)\n\t\t}\n\t\treturn nil\n\t})\n\n\tpubNC := natsConnect(t, fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port))\n\tdefer pubNC.Close()\n\tnatsPub(t, pubNC, \"foo\", []byte(\"hello world!\"))\n\n\twaitCh(t, ch, \"Did not get all messages\")\n}\n\nfunc TestRouteRTT(t *testing.T) {\n\tob := DefaultOptions()\n\tob.PingInterval = 15 * time.Millisecond\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\toa := DefaultOptions()\n\toa.PingInterval = 15 * time.Millisecond\n\toa.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ob.Cluster.Host, ob.Cluster.Port))\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tcheckClusterFormed(t, sa, sb)\n\n\tcheckRTT := func(t *testing.T, s *Server) time.Duration {\n\t\tt.Helper()\n\t\tvar route *client\n\t\ts.mu.Lock()\n\t\troute = getFirstRoute(s)\n\t\ts.mu.Unlock()\n\n\t\tvar rtt time.Duration\n\t\tcheckFor(t, 2*firstPingInterval, 15*time.Millisecond, func() error {\n\t\t\troute.mu.Lock()\n\t\t\trtt = route.rtt\n\t\t\troute.mu.Unlock()\n\t\t\tif rtt == 0 {\n\t\t\t\treturn fmt.Errorf(\"RTT not tracked\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\treturn rtt\n\t}\n\n\tprevA := checkRTT(t, sa)\n\tprevB := checkRTT(t, sb)\n\n\tcheckUpdated := func(t *testing.T, s *Server, prev time.Duration) {\n\t\tt.Helper()\n\t\tattempts := 0\n\t\ttimeout := time.Now().Add(2 * firstPingInterval)\n\t\tfor time.Now().Before(timeout) {\n\t\t\tif rtt := checkRTT(t, s); rtt != prev {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tattempts++\n\t\t\tif attempts == 5 {\n\t\t\t\t// If could be that we are very unlucky\n\t\t\t\t// and the RTT is constant. So override\n\t\t\t\t// the route's RTT to 0 to see if it gets\n\t\t\t\t// updated.\n\t\t\t\ts.mu.Lock()\n\t\t\t\tif r := getFirstRoute(s); r != nil {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tr.rtt = 0\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t}\n\t\t\t\ts.mu.Unlock()\n\t\t\t}\n\t\t\ttime.Sleep(15 * time.Millisecond)\n\t\t}\n\t\tt.Fatalf(\"RTT probably not updated\")\n\t}\n\tcheckUpdated(t, sa, prevA)\n\tcheckUpdated(t, sb, prevB)\n\n\tsa.Shutdown()\n\tsb.Shutdown()\n\n\t// Now check that initial RTT is computed prior to first PingInterval\n\t// Get new options to avoid possible race changing the ping interval.\n\tob = DefaultOptions()\n\tob.PingInterval = time.Minute\n\tsb = RunServer(ob)\n\tdefer sb.Shutdown()\n\n\toa = DefaultOptions()\n\toa.PingInterval = time.Minute\n\toa.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ob.Cluster.Host, ob.Cluster.Port))\n\tsa = RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tcheckClusterFormed(t, sa, sb)\n\tcheckRTT(t, sa)\n\tcheckRTT(t, sb)\n}\n\nfunc TestRouteCloseTLSConnection(t *testing.T) {\n\topts := DefaultOptions()\n\topts.DisableShortFirstPing = true\n\topts.Cluster.Name = \"A\"\n\topts.Cluster.Host = \"127.0.0.1\"\n\topts.Cluster.Port = -1\n\topts.Cluster.TLSTimeout = 100\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server.pem\",\n\t\tKeyFile: \"./configs/certs/key.pem\",\n\t\tInsecure: true,\n\t}\n\ttlsConf, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\topts.Cluster.TLSConfig = tlsConf\n\topts.NoLog = true\n\topts.NoSigs = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tendpoint := net.JoinHostPort(opts.Cluster.Host, fmt.Sprintf(\"%d\", opts.Cluster.Port))\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error on dial: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\ttlsConn := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})\n\tdefer tlsConn.Close()\n\tif err := tlsConn.Handshake(); err != nil {\n\t\tt.Fatalf(\"Unexpected error during handshake: %v\", err)\n\t}\n\tconnectOp := []byte(\"CONNECT {\\\"name\\\":\\\"route\\\",\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"tls_required\\\":true,\\\"cluster\\\":\\\"A\\\"}\\r\\n\")\n\tif _, err := tlsConn.Write(connectOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tinfoOp := []byte(\"INFO {\\\"server_id\\\":\\\"route\\\",\\\"tls_required\\\":true}\\r\\n\")\n\tif _, err := tlsConn.Write(infoOp); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing CONNECT: %v\", err)\n\t}\n\tif _, err := tlsConn.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\tt.Fatalf(\"Unexpected error writing PING: %v\", err)\n\t}\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif s.NumRoutes() != 1 {\n\t\t\treturn fmt.Errorf(\"No route registered yet\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Get route connection\n\tvar route *client\n\ts.mu.Lock()\n\troute = getFirstRoute(s)\n\ts.mu.Unlock()\n\t// Fill the buffer. We want to timeout on write so that nc.Close()\n\t// would block due to a write that cannot complete.\n\tbuf := make([]byte, 64*1024)\n\tdone := false\n\tfor !done {\n\t\troute.nc.SetWriteDeadline(time.Now().Add(time.Second))\n\t\tif _, err := route.nc.Write(buf); err != nil {\n\t\t\tdone = true\n\t\t}\n\t\troute.nc.SetWriteDeadline(time.Time{})\n\t}\n\tch := make(chan bool)\n\tgo func() {\n\t\tselect {\n\t\tcase <-ch:\n\t\t\treturn\n\t\tcase <-time.After(3 * time.Second):\n\t\t\tfmt.Println(\"!!!! closeConnection is blocked, test will hang !!!\")\n\t\t\treturn\n\t\t}\n\t}()\n\t// Close the route\n\troute.closeConnection(SlowConsumerWriteDeadline)\n\tch <- true\n}\n\nfunc TestRouteClusterNameConflictBetweenStaticAndDynamic(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.Cluster.Name = \"AAAAAAAAAAAAAAAAAAAA\" // make it alphabetically the \"smallest\"\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.Cluster.Name = \"\" // intentional, let it be assigned dynamically\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n}\n\ntype testRouteResolver struct{}\n\nfunc (r *testRouteResolver) LookupHost(ctx context.Context, host string) ([]string, error) {\n\treturn []string{\"127.0.0.1\", \"other.host.in.cluster\"}, nil\n}\n\ntype routeHostLookupLogger struct {\n\tDummyLogger\n\terrCh chan string\n\tch chan bool\n\tcount int\n}\n\nfunc (l *routeHostLookupLogger) Debugf(format string, v ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"127.0.0.1:1234\") {\n\t\tl.errCh <- msg\n\t} else if strings.Contains(msg, \"other.host.in.cluster\") {\n\t\tif l.count++; l.count == 10 {\n\t\t\tl.ch <- true\n\t\t}\n\t}\n}\n\nfunc TestRouteIPResolutionAndRouteToSelf(t *testing.T) {\n\to := DefaultOptions()\n\to.Cluster.Port = 1234\n\tr := &testRouteResolver{}\n\to.Cluster.resolver = r\n\to.Routes = RoutesFromStr(\"nats://routehost:1234\")\n\to.Debug = true\n\to.NoLog = false\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tdefer s.Shutdown()\n\tl := &routeHostLookupLogger{errCh: make(chan string, 1), ch: make(chan bool, 1)}\n\ts.SetLogger(l, true, true)\n\ts.Start()\n\tif err := s.readyForConnections(time.Second); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tselect {\n\tcase e := <-l.errCh:\n\t\tt.Fatalf(\"Unexpected trace: %q\", e)\n\tcase <-l.ch:\n\t\t// Ok\n\t\treturn\n\t}\n}\n\nfunc TestRouteDuplicateServerName(t *testing.T) {\n\to := DefaultOptions()\n\to.ServerName = \"A\"\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 1)}\n\ts.SetLogger(l, false, false)\n\n\to2 := DefaultOptions()\n\t// Set the same server name on purpose\n\to2.ServerName = \"A\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\t// This is an error now so can't wait on cluster formed.\n\tselect {\n\tcase w := <-l.errCh:\n\t\tif !strings.Contains(w, \"Remote server has a duplicate name\") {\n\t\t\tt.Fatalf(\"Expected warning about same name, got %q\", w)\n\t\t}\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"Should have gotten a warning regarding duplicate server name\")\n\t}\n}\n\nfunc TestRouteLockReleasedOnTLSFailure(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.Cluster.Name = \"abc\"\n\to1.Cluster.Host = \"127.0.0.1\"\n\to1.Cluster.Port = -1\n\to1.Cluster.TLSTimeout = 0.25\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server.pem\",\n\t\tKeyFile: \"./configs/certs/key.pem\",\n\t\tInsecure: true,\n\t}\n\ttlsConf, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\to1.Cluster.TLSConfig = tlsConf\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts1.SetLogger(l, false, false)\n\n\to2 := DefaultOptions()\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tselect {\n\tcase err := <-l.errCh:\n\t\tif !strings.Contains(err, \"TLS\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(time.Second):\n\t}\n\n\ts2.Shutdown()\n\n\t// Wait for longer than the TLS timeout and check that tlsTimeout is not stuck\n\ttime.Sleep(500 * time.Millisecond)\n\n\tbuf := make([]byte, 10000)\n\tn := runtime.Stack(buf, true)\n\tif bytes.Contains(buf[:n], []byte(\"tlsTimeout\")) {\n\t\tt.Fatal(\"Seem connection lock was not released\")\n\t}\n}\n\ntype localhostResolver struct{}\n\nfunc (r *localhostResolver) LookupHost(ctx context.Context, host string) ([]string, error) {\n\treturn []string{\"127.0.0.1\"}, nil\n}\n\nfunc TestTLSRoutesCertificateImplicitAllowPass(t *testing.T) {\n\ttestTLSRoutesCertificateImplicitAllow(t, true)\n}\n\nfunc TestTLSRoutesCertificateImplicitAllowFail(t *testing.T) {\n\ttestTLSRoutesCertificateImplicitAllow(t, false)\n}\n\nfunc testTLSRoutesCertificateImplicitAllow(t *testing.T, pass bool) {\n\t// Base config for the servers\n\tcfg := createTempFile(t, \"cfg\")\n\tcfg.WriteString(fmt.Sprintf(`\n\t\tcluster {\n\t\t tls {\n\t\t\tcert_file = \"../test/configs/certs/tlsauth/server.pem\"\n\t\t\tkey_file = \"../test/configs/certs/tlsauth/server-key.pem\"\n\t\t\tca_file = \"../test/configs/certs/tlsauth/ca.pem\"\n\t\t\tverify_cert_and_check_known_urls = true\n\t\t\tinsecure = %t\n\t\t\ttimeout = 1\n\t\t }\n\t\t}\n\t`, !pass)) // set insecure to skip verification on the outgoing end\n\tif err := cfg.Sync(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcfg.Close()\n\n\toptsA := LoadConfig(cfg.Name())\n\toptsB := LoadConfig(cfg.Name())\n\n\trouteURLs := \"nats://localhost:9935, nats://localhost:9936\"\n\tif !pass {\n\t\trouteURLs = \"nats://127.0.0.1:9935, nats://127.0.0.1:9936\"\n\t}\n\toptsA.Host = \"127.0.0.1\"\n\toptsA.Port = 9335\n\toptsA.Cluster.Name = \"xyz\"\n\toptsA.Cluster.Host = optsA.Host\n\toptsA.Cluster.Port = 9935\n\toptsA.Cluster.resolver = &localhostResolver{}\n\toptsA.Routes = RoutesFromStr(routeURLs)\n\toptsA.NoSystemAccount = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB.Host = \"127.0.0.1\"\n\toptsB.Port = 9336\n\toptsB.Cluster.Name = \"xyz\"\n\toptsB.Cluster.Host = optsB.Host\n\toptsB.Cluster.Port = 9936\n\toptsB.Cluster.resolver = &localhostResolver{}\n\toptsB.Routes = RoutesFromStr(routeURLs)\n\toptsB.NoSystemAccount = true\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tif pass {\n\t\tcheckNumRoutes(t, srvA, DEFAULT_ROUTE_POOL_SIZE)\n\t\tcheckNumRoutes(t, srvB, DEFAULT_ROUTE_POOL_SIZE)\n\t} else {\n\t\ttime.Sleep(1 * time.Second) // the fail case uses the IP, so a short wait is sufficient\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tif srvA.NumRoutes() != 0 || srvB.NumRoutes() != 0 {\n\t\t\t\treturn fmt.Errorf(\"No route connection expected\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n}\n\nfunc TestSubjectRenameViaJetStreamAck(t *testing.T) {\n\ts := RunRandClientPortServer(t)\n\tdefer s.Shutdown()\n\terrChan := make(chan error)\n\tdefer close(errChan)\n\tncPub := natsConnect(t, s.ClientURL(), nats.UserInfo(\"client\", \"pwd\"),\n\t\tnats.ErrorHandler(func(conn *nats.Conn, s *nats.Subscription, err error) {\n\t\t\terrChan <- err\n\t\t}))\n\tdefer ncPub.Close()\n\trequire_NoError(t, ncPub.PublishRequest(\"SVC.ALLOWED\", \"$JS.ACK.whatever@ADMIN\", nil))\n\tselect {\n\tcase err := <-errChan:\n\t\trequire_Contains(t, err.Error(), \"Permissions Violation for Publish with Reply of\")\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Expected error\")\n\t}\n}\n\nfunc TestClusterQueueGroupWeightTrackingLeak(t *testing.T) {\n\to := DefaultOptions()\n\to.ServerName = \"A\"\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.ServerName = \"B\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\t// Create a queue subscription\n\tsub := natsQueueSubSync(t, nc, \"foo\", \"bar\")\n\n\t// Check on s0 that we have the proper queue weight info\n\tacc := s.GlobalAccount()\n\n\tcheck := func(present bool, expected int32) {\n\t\tt.Helper()\n\t\tsub := subscription{subject: []byte(\"foo\"), queue: []byte(\"bar\")}\n\t\tkey := keyFromSubWithOrigin(&sub)\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tacc.mu.RLock()\n\t\t\tv, ok := acc.lqws[key]\n\t\t\tacc.mu.RUnlock()\n\t\t\tif present {\n\t\t\t\tif !ok {\n\t\t\t\t\treturn fmt.Errorf(\"the key is not present\")\n\t\t\t\t}\n\t\t\t\tif v != expected {\n\t\t\t\t\treturn fmt.Errorf(\"lqws doest not contain expected value of %v: %v\", expected, v)\n\t\t\t\t}\n\t\t\t} else if ok {\n\t\t\t\treturn fmt.Errorf(\"the key is present with value %v and should not be\", v)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheck(true, 1)\n\n\t// Now unsub, and it should be removed, not just be 0\n\tsub.Unsubscribe()\n\tcheck(false, 0)\n\n\t// Still make sure that the subject interest is gone from both servers.\n\tcheckSubGone := func(s *Server) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tacc := s.GlobalAccount()\n\t\t\tacc.mu.RLock()\n\t\t\tres := acc.sl.Match(\"foo\")\n\t\t\tacc.mu.RUnlock()\n\t\t\tif res != nil && len(res.qsubs) > 0 {\n\t\t\t\treturn fmt.Errorf(\"Found queue sub on foo for server %v\", s)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckSubGone(s)\n\tcheckSubGone(s2)\n}\n\ntype testRouteReconnectLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *testRouteReconnectLogger) Debugf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"Trying to connect to route\") {\n\t\tselect {\n\t\tcase l.ch <- msg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestRouteSolicitedReconnectsEvenIfImplicit(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.ServerName = \"A\"\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.ServerName = \"B\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\t// Not strictly required to reconnect, but if the reconnect were to fail for any reason\n\t// then the server would retry only once and then stops. So set it to some higher value\n\t// and then we will check that the server does not try more than that.\n\to2.Cluster.ConnectRetries = 3\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\to3 := DefaultOptions()\n\to3.ServerName = \"C\"\n\to3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to3.Cluster.ConnectRetries = 3\n\ts3 := RunServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\ts2.mu.Lock()\n\ts2.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\t// Close the route between S2 and S3 (that do not have explicit route to each other)\n\t\tif r.route.remoteID == s3.ID() {\n\t\t\tr.nc.Close()\n\t\t}\n\t\tr.mu.Unlock()\n\t})\n\ts2.mu.Unlock()\n\t// Wait a bit to make sure that we don't check for cluster formed too soon (need to make\n\t// sure that connection is really removed and reconnect mechanism starts).\n\ttime.Sleep(500 * time.Millisecond)\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// Now shutdown server 3 and make sure that s2 stops trying to reconnect to s3 at one point\n\tl := &testRouteReconnectLogger{ch: make(chan string, 10)}\n\ts2.SetLogger(l, true, false)\n\ts3.Shutdown()\n\t// S2 should retry ConnectRetries+1 times and then stop\n\t// Take into account default route pool size and system account dedicated route\n\tfor i := 0; i < (DEFAULT_ROUTE_POOL_SIZE+1)*(o2.Cluster.ConnectRetries+1); i++ {\n\t\tselect {\n\t\tcase <-l.ch:\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Did not attempt to reconnect\")\n\t\t}\n\t}\n\t// Now it should have stopped (in tests, reconnect delay is down to 15ms, so we don't need\n\t// to wait for too long).\n\tselect {\n\tcase msg := <-l.ch:\n\t\tt.Fatalf(\"Unexpected attempt to reconnect: %s\", msg)\n\tcase <-time.After(50 * time.Millisecond):\n\t\t// OK\n\t}\n}\n\nfunc TestRouteReconnectExponentialBackoff(t *testing.T) {\n\toRouteConnectDelay := routeConnectDelay\n\toRouteConnectMaxDelay := routeConnectMaxDelay\n\trouteConnectDelay = 500 * time.Millisecond\n\trouteConnectMaxDelay = 2 * time.Second\n\tdefer func() {\n\t\trouteConnectDelay = oRouteConnectDelay\n\t\trouteConnectMaxDelay = oRouteConnectMaxDelay\n\t}()\n\n\to1 := DefaultOptions()\n\to1.ServerName = \"A\"\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.ServerName = \"B\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to2.Cluster.ConnectRetries = 3\n\to2.Cluster.ConnectBackoff = true\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now shutdown server 1 and make sure that s2 stops trying to reconnect to s1 at one point\n\tl := &testRouteReconnectLogger{ch: make(chan string, 10)}\n\ts2.SetLogger(l, true, false)\n\n\t// Remove initial delay before reconnect, and allow for some skew.\n\tnow := time.Now().Add(DEFAULT_ROUTE_RECONNECT).Add(-100 * time.Millisecond)\n\tvar delay time.Duration\n\n\t// S2 should retry ConnectRetries+1 times and then stop\n\t// Take into account default route pool size and system account dedicated route\n\ts1.Shutdown()\n\tfor i := 0; i < (DEFAULT_ROUTE_POOL_SIZE+1)*(o2.Cluster.ConnectRetries+1); i++ {\n\t\tselect {\n\t\tcase <-l.ch:\n\t\t\tif since := time.Since(now); since < delay {\n\t\t\t\tt.Fatalf(\"Expected delay to take %v, took %v\", delay, since)\n\t\t\t}\n\t\t\tif i > 0 && (i+1)%(DEFAULT_ROUTE_POOL_SIZE+1) == 0 {\n\t\t\t\tif delay == 0 {\n\t\t\t\t\tdelay = routeConnectDelay\n\t\t\t\t} else {\n\t\t\t\t\tdelay *= 2\n\t\t\t\t}\n\t\t\t\tif delay > routeConnectMaxDelay {\n\t\t\t\t\tdelay = routeConnectMaxDelay\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-time.After(routeConnectMaxDelay + time.Second):\n\t\t\tt.Fatal(\"Did not attempt to reconnect\")\n\t\t}\n\t}\n}\n\nfunc TestRouteSaveTLSName(t *testing.T) {\n\tc1Conf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"abc\"\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\t\t\ttls {\n\t\t\t\tcert_file: '../test/configs/certs/server-noip.pem'\n\t\t\t\tkey_file: '../test/configs/certs/server-key-noip.pem'\n\t\t\t\tca_file: '../test/configs/certs/ca.pem'\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(c1Conf)\n\tdefer s1.Shutdown()\n\n\ttmpl := `\n\tport: -1\n\tcluster {\n\t\tname: \"abc\"\n\t\tport: -1\n\t\tpool_size: -1\n\t\troutes: [\"nats://%s:%d\"]\n\t\ttls {\n\t\t\tcert_file: '../test/configs/certs/server-noip.pem'\n\t\t\tkey_file: '../test/configs/certs/server-key-noip.pem'\n\t\t\tca_file: '../test/configs/certs/ca.pem'\n\t\t}\n\t}\n\t`\n\tc2And3Conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"localhost\", o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(c2And3Conf)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\ts3, _ := RunServerWithConfig(c2And3Conf)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\treloadUpdateConfig(t, s2, c2And3Conf, fmt.Sprintf(tmpl, \"127.0.0.1\", o1.Cluster.Port))\n\n\ts2.mu.RLock()\n\ts2.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tif r.route.routeType == Implicit {\n\t\t\tr.nc.Close()\n\t\t}\n\t\tr.mu.Unlock()\n\t})\n\ts2.mu.RUnlock()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// Set a logger to capture errors trying to connect after clearing\n\t// the routeTLSName and causing a disconnect\n\tl := &captureErrorLogger{errCh: make(chan string, 1)}\n\ts2.SetLogger(l, false, false)\n\n\tvar gotIt bool\n\tfor i := 0; !gotIt && i < 5; i++ {\n\t\ts2.mu.Lock()\n\t\ts2.routeTLSName = _EMPTY_\n\t\ts2.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\tif r.route.routeType == Implicit {\n\t\t\t\tr.nc.Close()\n\t\t\t}\n\t\t\tr.mu.Unlock()\n\t\t})\n\t\ts2.mu.Unlock()\n\t\tselect {\n\t\tcase <-l.errCh:\n\t\t\tgotIt = true\n\t\tcase <-time.After(time.Second):\n\t\t\t// Try again\n\t\t}\n\t}\n\tif !gotIt {\n\t\tt.Fatal(\"Did not get the handshake error\")\n\t}\n\n\t// Now get back to localhost in config and reload config and\n\t// it should start to work again.\n\treloadUpdateConfig(t, s2, c2And3Conf, fmt.Sprintf(tmpl, \"localhost\", o1.Cluster.Port))\n\tcheckClusterFormed(t, s1, s2, s3)\n}\n\nfunc TestRoutePoolAndPerAccountErrors(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tport: -1\n\t\t\taccounts: [\"abc\", \"def\", \"abc\"]\n\t\t}\n\t`))\n\to := LoadConfig(conf)\n\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), \"duplicate\") {\n\t\tt.Fatalf(\"Expected error about duplicate, got %v\", err)\n\t}\n\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tabc { users: [{user:abc, password: pwd}] }\n\t\t\tdef { users: [{user:def, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\taccounts: [\"abc\"]\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\ts1.SetLogger(l, false, false)\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\taccounts {\n\t\t\tabc { users: [{user:abc, password: pwd}] }\n\t\t\tdef { users: [{user:def, password: pwd}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\taccounts: [\"def\"]\n\t\t}\n\t`, o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase e := <-l.errCh:\n\t\t\tif !strings.Contains(e, \"No route for account \\\"def\\\"\") {\n\t\t\t\tt.Fatalf(\"Expected error about no route for account, got %v\", e)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Did not get expected error regarding no route for account\")\n\t\t}\n\t\ttime.Sleep(DEFAULT_ROUTE_RECONNECT + 100*time.Millisecond)\n\t}\n}\n\nfunc TestRoutePool(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"a\"}] }\n\t\t\tB { users: [{user: \"b\", password: \"b\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\tpool_size: 2\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckRoutePoolIdx := func(s *Server, accName string, expected int) {\n\t\tt.Helper()\n\t\ta, err := s.LookupAccount(accName)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, a != nil)\n\t\ta.mu.RLock()\n\t\trpi := a.routePoolIdx\n\t\ta.mu.RUnlock()\n\t\trequire_True(t, rpi == expected)\n\t}\n\tcheckRoutePoolIdx(s1, \"A\", 0)\n\tcheckRoutePoolIdx(s2, \"A\", 0)\n\tcheckRoutePoolIdx(s2, \"B\", 1)\n\tcheckRoutePoolIdx(s2, \"B\", 1)\n\n\tsendAndRecv := func(acc, user, pwd string) {\n\t\tt.Helper()\n\t\ts2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, pwd))\n\t\tdefer s2nc.Close()\n\n\t\tsub := natsSubSync(t, s2nc, \"foo\")\n\t\tnatsFlush(t, s2nc)\n\n\t\ts1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, pwd))\n\t\tdefer s1nc.Close()\n\n\t\tcheckSubInterest(t, s1, acc, \"foo\", time.Second)\n\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tnatsPub(t, s1nc, \"foo\", []byte(\"hello\"))\n\t\t}\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t}\n\t\t// Make sure we don't receive more\n\t\tif msg, err := sub.NextMsg(150 * time.Millisecond); err == nil {\n\t\t\tt.Fatalf(\"Unexpected message: %+v\", msg)\n\t\t}\n\t}\n\n\tsendAndRecv(\"A\", \"a\", \"a\")\n\tsendAndRecv(\"B\", \"b\", \"b\")\n\n\tcheckStats := func(s *Server, isOut bool) {\n\t\tt.Helper()\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\tfor _, conns := range s.routes {\n\t\t\tfor i, r := range conns {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tif isOut {\n\t\t\t\t\tif v := r.stats.outMsgs; v < 1000 {\n\t\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\t\tt.Fatalf(\"Expected at least 1000 in out msgs for route %v, got %v\", i+1, v)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif v := r.stats.inMsgs; v < 1000 {\n\t\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\t\tt.Fatalf(\"Expected at least 1000 in msgs for route %v, got %v\", i+1, v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tr.mu.Unlock()\n\t\t\t}\n\t\t}\n\t}\n\tcheckStats(s1, true)\n\tcheckStats(s2, false)\n\n\tdisconnectRoute := func(s *Server, idx int) {\n\t\tt.Helper()\n\t\tattempts := 0\n\tTRY_AGAIN:\n\t\ts.mu.RLock()\n\t\tfor _, conns := range s.routes {\n\t\t\tfor i, r := range conns {\n\t\t\t\tif i != idx {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif r != nil {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tnc := r.nc\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\tif nc == nil {\n\t\t\t\t\t\ts.mu.RUnlock()\n\t\t\t\t\t\tif attempts++; attempts < 10 {\n\t\t\t\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\t\t\t\tgoto TRY_AGAIN\n\t\t\t\t\t\t}\n\t\t\t\t\t\tt.Fatalf(\"Route %v net.Conn is nil\", i)\n\t\t\t\t\t}\n\t\t\t\t\tnc.Close()\n\t\t\t\t} else {\n\t\t\t\t\ts.mu.RUnlock()\n\t\t\t\t\tif attempts++; attempts < 10 {\n\t\t\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\t\t\tgoto TRY_AGAIN\n\t\t\t\t\t}\n\t\t\t\t\tt.Fatalf(\"Route %v connection is nil\", i)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ts.mu.RUnlock()\n\t\ttime.Sleep(250 * time.Millisecond)\n\t\tcheckClusterFormed(t, s1, s2)\n\t}\n\tdisconnectRoute(s1, 0)\n\tdisconnectRoute(s2, 1)\n}\n\nfunc TestRoutePoolConnectRace(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize int\n\t}{\n\t\t{\"no pool\", -1},\n\t\t{\"pool size 1\", 1},\n\t\t{\"pool size 5\", 5},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// This test will have each server point to each other and that is causing\n\t\t\t// each one to attempt to connect routes to each other which should lead\n\t\t\t// to connections needing to be dropped. We make sure that there is still\n\t\t\t// resolution and there is the expected number of routes.\n\t\t\tcreateSrv := func(name string, port int) *Server {\n\t\t\t\to := DefaultOptions()\n\t\t\t\to.Port = -1\n\t\t\t\to.ServerName = name\n\t\t\t\to.Cluster.PoolSize = test.poolSize\n\t\t\t\to.Cluster.Name = \"local\"\n\t\t\t\to.Cluster.Port = port\n\t\t\t\to.Routes = RoutesFromStr(\"nats://127.0.0.1:1234,nats://127.0.0.1:1235,nats://127.0.0.1:1236\")\n\t\t\t\ts, err := NewServer(o)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn s\n\t\t\t}\n\t\t\ts1 := createSrv(\"A\", 1234)\n\t\t\ts2 := createSrv(\"B\", 1235)\n\t\t\ts3 := createSrv(\"C\", 1236)\n\n\t\t\tl := &captureDebugLogger{dbgCh: make(chan string, 100)}\n\t\t\ts1.SetLogger(l, true, false)\n\n\t\t\tservers := []*Server{s1, s2, s3}\n\n\t\t\tfor _, s := range servers {\n\t\t\t\tgo s.Start()\n\t\t\t\tdefer s.Shutdown()\n\t\t\t}\n\n\t\t\tcheckClusterFormed(t, s1, s2, s3)\n\n\t\t\tfor done, duplicate := false, 0; !done; {\n\t\t\t\tselect {\n\t\t\t\tcase e := <-l.dbgCh:\n\t\t\t\t\tif strings.Contains(e, \"duplicate\") {\n\t\t\t\t\t\tif duplicate++; duplicate > 20 {\n\t\t\t\t\t\t\tt.Fatalf(\"Routes are constantly reconnecting: %v\", e)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tcase <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond):\n\t\t\t\t\t// More than reconnect and some, and no reconnect, so we are good.\n\t\t\t\t\tdone = true\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Also, check that they all report as solicited and configured in monitoring.\n\t\t\tfor _, s := range servers {\n\t\t\t\troutes, err := s.Routez(nil)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tfor _, r := range routes.Routes {\n\t\t\t\t\tif !r.DidSolicit {\n\t\t\t\t\t\tt.Fatalf(\"All routes should have been marked as solicited, this one was not: %+v\", r)\n\t\t\t\t\t}\n\t\t\t\t\tif !r.IsConfigured {\n\t\t\t\t\t\tt.Fatalf(\"All routes should have been marked as configured, this one was not: %+v\", r)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, s := range servers {\n\t\t\t\ts.Shutdown()\n\t\t\t\ts.WaitForShutdown()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRoutePoolRouteStoredSameIndexBothSides(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"a\"}] }\n\t\t\tB { users: [{user: \"b\", password: \"b\"}] }\n\t\t\tC { users: [{user: \"c\", password: \"c\"}] }\n\t\t\tD { users: [{user: \"d\", password: \"d\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\tpool_size: 4\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tfor i := 0; i < 20; i++ {\n\t\tcheckClusterFormed(t, s1, s2)\n\n\t\tcollect := func(s *Server, checkRemoteAddr bool) []string {\n\t\t\taddrs := make([]string, 0, 4)\n\t\t\ts.mu.RLock()\n\t\t\ts.forEachRoute(func(r *client) {\n\t\t\t\tvar addr string\n\t\t\t\tr.mu.Lock()\n\t\t\t\tif r.nc != nil {\n\t\t\t\t\tif checkRemoteAddr {\n\t\t\t\t\t\taddr = r.nc.RemoteAddr().String()\n\t\t\t\t\t} else {\n\t\t\t\t\t\taddr = r.nc.LocalAddr().String()\n\t\t\t\t\t}\n\t\t\t\t\taddrs = append(addrs, addr)\n\t\t\t\t}\n\t\t\t\tr.mu.Unlock()\n\t\t\t})\n\t\t\ts.mu.RUnlock()\n\t\t\treturn addrs\n\t\t}\n\n\t\taddrsS1 := collect(s1, true)\n\t\taddrsS2 := collect(s2, false)\n\t\tif len(addrsS1) != 4 || len(addrsS2) != 4 {\n\t\t\t// It could be that connections were not ready (r.nc is nil in collect())\n\t\t\t// if that is the case, try again.\n\t\t\ti--\n\t\t\tcontinue\n\t\t}\n\n\t\tif !reflect.DeepEqual(addrsS1, addrsS2) {\n\t\t\tt.Fatalf(\"Connections not stored at same index:\\ns1=%v\\ns2=%v\", addrsS1, addrsS2)\n\t\t}\n\n\t\ts1.mu.RLock()\n\t\ts1.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\tif r.nc != nil {\n\t\t\t\tr.nc.Close()\n\t\t\t}\n\t\t\tr.mu.Unlock()\n\t\t})\n\t\ts1.mu.RUnlock()\n\t}\n}\n\nfunc TestRoutePoolSizeDifferentOnEachServer(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t\tC { users: [{user: \"C\", password: \"pwd\"}] }\n\t\t\tD { users: [{user: \"D\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\tpool_size: %d\n\t\t\taccounts: [%s]\n\t\t}\n\t\tno_sys_acc: true\n\t`\n\tconf1 := createConfFile(t, fmt.Appendf(nil, tmpl, \"S1\", _EMPTY_, 3, `\"A\"`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, fmt.Appendf(nil, tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), 2, `\"A\"`))\n\ts2, o2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tconf3 := createConfFile(t, fmt.Appendf(nil, tmpl, \"S3\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), 4, `\"A\"`))\n\ts3, o3 := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\t// Since each one has a different configured pool size, we are not going\n\t// to use checkClusterFormed, but use a low level checFor here.\n\tservers := []*Server{s1, s2, s3}\n\n\t// Both S2 and S3 solicit connections to S1, and with the order the servers\n\t// are started, S2 will implicitly connect to S3. With that in mind, the\n\t// expected number of routes for each server will be as such:\n\t// S1: when S2 solicits 2+1 routes, it will detect that S1 is configured\n\t// for 3+1, so S1<->S2 will have the max of the two, which is 4 routes,\n\t// then when S3 solicits its 4+1 routes, the total will be 4+5=9 routes.\n\t// S2: it solicits 2+1 but since S1 is configured for 3+1, it will have\n\t// 3+1 routes to S1, and when being told to connect to S3, it will start\n\t// with 2+1 but since S3 is configured for 4+1, it will have that many\n\t// connections, so total is 4+5=9 too.\n\t// S3: it solicits 4+1 routes to S1, so S1 although configured for 3+1\n\t// will accept the 4+1, and when S2 implicitly connects to S3, it will\n\t// force S2 to create 4+1 connections too. So here the total is 5+5=10.\n\texpected := []int{9, 9, 10}\n\tcheckCluster := func() {\n\t\tt.Helper()\n\t\t// In most case, we will do a config reload and want to give time\n\t\t// for connections to be closed before checking.\n\t\ttime.Sleep(50 * time.Millisecond)\n\t\tfor i, s := range servers {\n\t\t\tif !s.isRunning() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcheckFor(t, 10*time.Second, 25*time.Millisecond, func() error {\n\t\t\t\tif numRoutes := s.NumRoutes(); numRoutes != expected[i] {\n\t\t\t\t\treturn fmt.Errorf(\"Expected %d routes for server %q, got %d\", expected[i], s, numRoutes)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}\n\t}\n\tcheckCluster()\n\n\t// We will create a subscription per account per server.\n\taccs := []string{\"A\", \"B\", \"C\", \"D\"}\n\tvar aconns [][]*nats.Conn\n\tvar asubs [][]*nats.Subscription\n\tfor _, acc := range accs {\n\t\tvar subs []*nats.Subscription\n\t\tvar conns []*nats.Conn\n\t\tfor _, s := range servers {\n\t\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(acc, \"pwd\"))\n\t\t\tdefer nc.Close()\n\t\t\tconns = append(conns, nc)\n\t\t\tsub := natsSubSync(t, nc, \"foo\")\n\t\t\tsubs = append(subs, sub)\n\t\t}\n\t\tasubs = append(asubs, subs)\n\t\taconns = append(aconns, conns)\n\t}\n\n\t// Now we will check that a message produced on each account from\n\t// each server reaches all subs.\n\tcheckRecv := func(payload string) {\n\t\tt.Helper()\n\t\t// We first to check the interest on all servers for each account\n\t\tfor _, s := range servers {\n\t\t\tfor _, acc := range accs {\n\t\t\t\tcheckSubInterest(t, s, acc, \"foo\", time.Second)\n\t\t\t}\n\t\t}\n\n\t\t// Now from each server, and each account, send a message and check\n\t\t// that all subs for this account receive the message.\n\t\tfor _, s := range servers {\n\t\t\tfor i, acc := range accs {\n\t\t\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(acc, \"pwd\"))\n\t\t\t\tdefer nc.Close()\n\n\t\t\t\tnatsPub(t, nc, \"foo\", []byte(payload))\n\t\t\t\tfor j, sub := range asubs[i] {\n\t\t\t\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Producer on %q - account=%q - sub on server %q - err=%v\",\n\t\t\t\t\t\t\ts, acc, servers[j], err)\n\t\t\t\t\t}\n\t\t\t\t\t// Wait a tiny bit and check that there is not a duplicate\n\t\t\t\t\tif msg, err := sub.NextMsg(10 * time.Millisecond); err == nil {\n\t\t\t\t\t\tt.Fatalf(\"Producer on %q - account=%q - sub on server %q received a duplicate msg=%s\",\n\t\t\t\t\t\t\ts, acc, servers[j], msg.Data)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tcheckRecv(\"hello1\")\n\n\t// Now that the 3 servers are up and we know their route port, we will\n\t// update the config of each server so that they have routes that point\n\t// to each other (so each server will solicit) and for routes to be\n\t// recreated, then test again.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"S1\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o2.Cluster.Port, o3.Cluster.Port), 3, `\"A\"`))\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port, o3.Cluster.Port), 2, `\"A\"`))\n\treloadUpdateConfig(t, s3, conf3, fmt.Sprintf(tmpl, \"S3\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port, o2.Cluster.Port), 4, `\"A\"`))\n\n\ttime.Sleep(50 * time.Millisecond)\n\n\t// Collect all route TCP connections so we close them all as fast as possible\n\tvar rcs []net.Conn\n\tfor _, s := range servers {\n\t\ts.mu.RLock()\n\t\ts.forEachRoute(func(r *client) {\n\t\t\tr.mu.Lock()\n\t\t\trcs = append(rcs, r.nc)\n\t\t\tr.mu.Unlock()\n\t\t})\n\t\ts.mu.RUnlock()\n\t}\n\t// Close all connections\n\tfor _, c := range rcs {\n\t\tc.Close()\n\t}\n\n\tcheckCluster()\n\tcheckRecv(\"hello2\")\n\n\t// Now we will make account \"B\" have a dedicated route, and remove the one for \"A\"\n\t// Now upgrade one of the account to a dedicated route.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"S1\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o2.Cluster.Port, o3.Cluster.Port), 3, `\"B\"`))\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port, o3.Cluster.Port), 2, `\"B\"`))\n\treloadUpdateConfig(t, s3, conf3, fmt.Sprintf(tmpl, \"S3\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\",\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port, o2.Cluster.Port), 4, `\"B\"`))\n\n\tcheckCluster()\n\tcheckRecv(\"hello3\")\n\n\t// Since we are going to shutdown and not restart S3,\n\t// close client connections to this server.\n\tfor _, conns := range aconns {\n\t\tconns[2].Close()\n\t}\n\t// Get S3 server ID and shut it down.\n\ts3ID := s3.ID()\n\ts3.Shutdown()\n\n\t// Wait for routes to be gone from S1 and S2.\n\texpected = []int{4, 4}\n\tservers = servers[:2]\n\tcheckCluster()\n\n\t// Check that both S1 and S2 have cleaned-up the remoteRoutePoolSize for S3.\n\tfor _, s := range servers {\n\t\ts.mu.RLock()\n\t\trps, ok := s.remoteRoutePoolSize[s3ID]\n\t\ts.mu.RUnlock()\n\t\tif ok {\n\t\t\tt.Fatalf(\"On server %q, found remote pool size of %v for S3\", s, rps)\n\t\t}\n\t}\n\n\ts1.Shutdown()\n\ts2.Shutdown()\n\n\t// Now start 2 servers with different pool size.\n\tconf1 = createConfFile(t, fmt.Appendf(nil, tmpl, \"S1\", _EMPTY_, 3, `\"A\"`))\n\ts1, o1 = RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 = createConfFile(t, fmt.Appendf(nil, tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), 2, `\"A\"`))\n\ts2, _ = RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tservers = []*Server{s1, s2}\n\texpected = []int{4, 4}\n\tcheckCluster()\n\n\t// Now check that S1 and S2 have the proper remoteRoutePoolSize to each other.\n\texpectedRPS := func(s *Server, otherID string, expected int) {\n\t\tt.Helper()\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\trps, ok := s.remoteRoutePoolSize[otherID]\n\t\trequire_True(t, ok)\n\t\trequire_Equal(t, rps, expected)\n\t}\n\ts2ID := s2.ID()\n\texpectedRPS(s1, s2ID, 2)\n\texpectedRPS(s2, s1.ID(), 3)\n\n\t// We will now config reload S2 with an increased pool size, but we want\n\t// to have S1 in a condition where the update does not go through the\n\t// process of re-creating the connections slice completely, which could\n\t// happen when the config reload reconnect from S2 to S1 happens in a way\n\t// that the earlier connections in the slice are replaced before the later\n\t// are, which prevents the situation in removeRoute() where all slots are\n\t// empty. To do so with certainty, we are going to replace the last\n\t// connection in s1's routes with a \"fake\" one that is not going to close\n\t// when s2 reloads.\n\ts1.mu.Lock()\n\tconns, ok := s1.routes[s2ID]\n\tif ok {\n\t\trc := conns[2]\n\t\tconns[2] = &client{kind: ROUTER, route: &route{}}\n\t\t// Empty the slot in a bit and close the old connection.\n\t\ttime.AfterFunc(150*time.Millisecond, func() {\n\t\t\ts1.mu.Lock()\n\t\t\tconns, ok := s1.routes[s2ID]\n\t\t\tif ok {\n\t\t\t\tconns[2] = nil\n\t\t\t}\n\t\t\ts1.mu.Unlock()\n\t\t\trc.closeConnection(ReadError)\n\t\t})\n\t}\n\ts1.mu.Unlock()\n\trequire_True(t, ok)\n\n\t// Increase the pool size.\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), 4, `\"A\"`))\n\texpected = []int{5, 5}\n\tcheckCluster()\n\n\texpectedRPS(s1, s2ID, 4)\n\texpectedRPS(s2, s1.ID(), 3)\n\n\t// Try to decrease now...\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"S2\",\n\t\tfmt.Sprintf(\"routes:[\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), 2, `\"A\"`))\n\texpected = []int{4, 4}\n\tcheckCluster()\n\n\texpectedRPS(s1, s2ID, 2)\n\texpectedRPS(s2, s1.ID(), 3)\n}\n\ntype captureRMsgTrace struct {\n\tDummyLogger\n\tsync.Mutex\n\ttraces *bytes.Buffer\n\tout []string\n}\n\nfunc (l *captureRMsgTrace) Tracef(format string, args ...any) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"[RMSG \") {\n\t\tl.traces.WriteString(msg)\n\t\tl.out = append(l.out, msg)\n\t}\n}\n\nfunc TestRoutePerAccount(t *testing.T) {\n\n\takp1, _ := nkeys.CreateAccount()\n\tacc1, _ := akp1.PublicKey()\n\n\takp2, _ := nkeys.CreateAccount()\n\tacc2, _ := akp2.PublicKey()\n\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\t%s { users: [{user: \"a\", password: \"a\"}] }\n\t\t\t%s { users: [{user: \"b\", password: \"b\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\taccounts: [\"%s\"]\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, acc1, acc2, _EMPTY_, acc2)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl,\n\t\tacc1, acc2,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\tacc2)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tl := &captureRMsgTrace{traces: &bytes.Buffer{}}\n\ts2.SetLogger(l, false, true)\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tdisconnectRoute := func(s *Server) {\n\t\tt.Helper()\n\t\tattempts := 0\n\tTRY_AGAIN:\n\t\ts.mu.RLock()\n\t\tif conns, ok := s.accRoutes[acc2]; ok {\n\t\t\tfor _, r := range conns {\n\t\t\t\tif r != nil {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tnc := r.nc\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\tif nc == nil {\n\t\t\t\t\t\ts.mu.RUnlock()\n\t\t\t\t\t\tif attempts++; attempts < 10 {\n\t\t\t\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\t\t\t\tgoto TRY_AGAIN\n\t\t\t\t\t\t}\n\t\t\t\t\t\tt.Fatal(\"Route net.Conn is nil\")\n\t\t\t\t\t}\n\t\t\t\t\tnc.Close()\n\t\t\t\t} else {\n\t\t\t\t\ts.mu.RUnlock()\n\t\t\t\t\tif attempts++; attempts < 10 {\n\t\t\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\t\t\tgoto TRY_AGAIN\n\t\t\t\t\t}\n\t\t\t\t\tt.Fatal(\"Route connection is nil\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ts.mu.RUnlock()\n\t\ttime.Sleep(250 * time.Millisecond)\n\t\tcheckClusterFormed(t, s1, s2)\n\t}\n\tdisconnectRoute(s1)\n\tdisconnectRoute(s2)\n\n\tsendAndRecv := func(acc, user, pwd string) {\n\t\tt.Helper()\n\t\ts2nc := natsConnect(t, s2.ClientURL(), nats.UserInfo(user, pwd))\n\t\tdefer s2nc.Close()\n\n\t\tsub := natsSubSync(t, s2nc, \"foo\")\n\t\tnatsFlush(t, s2nc)\n\n\t\ts1nc := natsConnect(t, s1.ClientURL(), nats.UserInfo(user, pwd))\n\t\tdefer s1nc.Close()\n\n\t\tcheckSubInterest(t, s1, acc, \"foo\", time.Second)\n\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tnatsPub(t, s1nc, \"foo\", []byte(\"hello\"))\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t}\n\t\t// Make sure we don't receive more\n\t\tif msg, err := sub.NextMsg(150 * time.Millisecond); err == nil {\n\t\t\tt.Fatalf(\"Unexpected message: %+v\", msg)\n\t\t}\n\t}\n\n\tsendAndRecv(acc1, \"a\", \"a\")\n\tsendAndRecv(acc2, \"b\", \"b\")\n\n\tl.Lock()\n\ttraces := l.traces.String()\n\tout := append([]string(nil), l.out...)\n\tl.Unlock()\n\t// We should not have any \"[RMSG \"\n\tif strings.Contains(traces, fmt.Sprintf(\"[RMSG %s\", acc2)) {\n\t\tvar outStr string\n\t\tfor _, l := range out {\n\t\t\toutStr += l + \"\\r\\n\"\n\t\t}\n\t\tt.Fatalf(\"Should not have included account %q in protocol, got:\\n%s\", acc2, outStr)\n\t}\n}\n\nfunc TestRoutePerAccountImplicit(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"a\"}] }\n\t\t\tB { users: [{user: \"b\", password: \"b\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\taccounts: [\"A\"]\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2And3 := createConfFile(t, []byte(fmt.Sprintf(tmpl,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2And3)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\ts3, _ := RunServerWithConfig(conf2And3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// On s3, close the per-account routes from s2\n\ts3.mu.RLock()\n\tfor _, conns := range s3.accRoutes {\n\t\tfor rem, r := range conns {\n\t\t\tif rem != s2.ID() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tr.mu.Lock()\n\t\t\tif r.nc != nil {\n\t\t\t\tr.nc.Close()\n\t\t\t}\n\t\t\tr.mu.Unlock()\n\t\t}\n\t}\n\ts3.mu.RUnlock()\n\t// Wait a bit to make sure there is a disconnect, then check the cluster is ok\n\ttime.Sleep(250 * time.Millisecond)\n\tcheckClusterFormed(t, s1, s2, s3)\n}\n\nfunc TestRoutePerAccountDefaultForSysAccount(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", password: \"a\"}] }\n\t\t\tB { users: [{user: \"b\", password: \"b\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\t%s\n\t\t\t%s\n\t\t}\n\t\t%s\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\taccounts string\n\t\tsysAcc string\n\t\tnoSysAcc bool\n\t}{\n\t\t{\"default sys no accounts\", _EMPTY_, _EMPTY_, false},\n\t\t{\"default sys in accounts\", \"accounts: [\\\"$SYS\\\"]\", _EMPTY_, false},\n\t\t{\"default sys with other accounts\", \"accounts: [\\\"A\\\",\\\"$SYS\\\"]\", _EMPTY_, false},\n\t\t{\"explicit sys no accounts\", _EMPTY_, \"system_account: B\", false},\n\t\t{\"explicit sys in accounts\", \"accounts: [\\\"B\\\"]\", \"system_account: B\", false},\n\t\t{\"explicit sys with other accounts\", \"accounts: [\\\"B\\\",\\\"A\\\"]\", \"system_account: B\", false},\n\t\t{\"no system account no accounts\", _EMPTY_, \"no_sys_acc: true\", true},\n\t\t{\"no system account with accounts\", \"accounts: [\\\"A\\\"]\", \"no_sys_acc: true\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.accounts,\n\t\t\t\t_EMPTY_, test.sysAcc)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_, test.accounts,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), test.sysAcc)))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\tcheckSysAccRoute := func(s *Server) {\n\t\t\t\tt.Helper()\n\t\t\t\tvar name string\n\t\t\t\tacc := s.SystemAccount()\n\t\t\t\tif test.noSysAcc {\n\t\t\t\t\tif acc != nil {\n\t\t\t\t\t\tt.Fatalf(\"Should not be any system account, got %q\", acc.GetName())\n\t\t\t\t\t}\n\t\t\t\t\t// We will check that there is no accRoutes for the default\n\t\t\t\t\t// system account name\n\t\t\t\t\tname = DEFAULT_SYSTEM_ACCOUNT\n\t\t\t\t} else {\n\t\t\t\t\tacc.mu.RLock()\n\t\t\t\t\tpi := acc.routePoolIdx\n\t\t\t\t\tname = acc.Name\n\t\t\t\t\tacc.mu.RUnlock()\n\t\t\t\t\tif pi != -1 {\n\t\t\t\t\t\tt.Fatalf(\"System account %q should have route pool index==-1, got %v\", name, pi)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.mu.RLock()\n\t\t\t\t_, ok := s.accRoutes[name]\n\t\t\t\ts.mu.RUnlock()\n\t\t\t\tif test.noSysAcc {\n\t\t\t\t\tif ok {\n\t\t\t\t\t\tt.Fatalf(\"System account %q should not have its own route, since NoSystemAccount was specified\", name)\n\t\t\t\t\t}\n\t\t\t\t} else if !ok {\n\t\t\t\t\tt.Fatalf(\"System account %q should be present in accRoutes, it was not\", name)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckSysAccRoute(s1)\n\t\t\tcheckSysAccRoute(s2)\n\n\t\t\t// Check that this is still the case after a config reload\n\t\t\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"pool_size: 4\", test.accounts,\n\t\t\t\t_EMPTY_, test.sysAcc))\n\t\t\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"pool_size: 4\", test.accounts,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port), test.sysAcc))\n\n\t\t\tcheckSysAccRoute(s1)\n\t\t\tcheckSysAccRoute(s2)\n\t\t})\n\t}\n}\n\nfunc TestRoutePerAccountConnectRace(t *testing.T) {\n\t// This test will have each server point to each other and that is causing\n\t// each one to attempt to connect routes to each other which should lead\n\t// to connections needing to be dropped. We make sure that there is still\n\t// resolution and there is the expected number of routes.\n\tcreateSrv := func(name string, port int) *Server {\n\t\to := DefaultOptions()\n\t\to.Port = -1\n\t\to.ServerName = name\n\t\to.Accounts = []*Account{NewAccount(\"A\")}\n\t\to.NoSystemAccount = true\n\t\to.Cluster.PoolSize = 1\n\t\to.Cluster.PinnedAccounts = []string{\"A\"}\n\t\to.Cluster.Name = \"local\"\n\t\to.Cluster.Port = port\n\t\to.Routes = RoutesFromStr(\"nats://127.0.0.1:1234,nats://127.0.0.1:1235,nats://127.0.0.1:1236\")\n\t\ts, err := NewServer(o)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t\t}\n\t\treturn s\n\t}\n\ts1 := createSrv(\"A\", 1234)\n\ts2 := createSrv(\"B\", 1235)\n\ts3 := createSrv(\"C\", 1236)\n\n\tl := &captureDebugLogger{dbgCh: make(chan string, 100)}\n\ts1.SetLogger(l, true, false)\n\n\tservers := []*Server{s1, s2, s3}\n\n\tfor _, s := range servers {\n\t\tgo s.Start()\n\t\tdefer s.Shutdown()\n\t}\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tfor done, duplicate := false, 0; !done; {\n\t\tselect {\n\t\tcase e := <-l.dbgCh:\n\t\t\tif strings.Contains(e, \"duplicate\") {\n\t\t\t\tif duplicate++; duplicate > 10 {\n\t\t\t\t\tt.Fatalf(\"Routes are constantly reconnecting: %v\", e)\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-time.After(DEFAULT_ROUTE_RECONNECT + 250*time.Millisecond):\n\t\t\t// More than reconnect and some, and no reconnect, so we are good.\n\t\t\tdone = true\n\t\t}\n\t}\n\n\t// Also, check that they all report as solicited and configured in monitoring.\n\tfor _, s := range servers {\n\t\troutes, err := s.Routez(nil)\n\t\trequire_NoError(t, err)\n\t\tfor _, r := range routes.Routes {\n\t\t\tif !r.DidSolicit {\n\t\t\t\tt.Fatalf(\"All routes should have been marked as solicited, this one was not: %+v\", r)\n\t\t\t}\n\t\t\tif !r.IsConfigured {\n\t\t\t\tt.Fatalf(\"All routes should have been marked as configured, this one was not: %+v\", r)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, s := range servers {\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t}\n}\n\nfunc TestRoutePerAccountGossipWorks(t *testing.T) {\n\ttmplA := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: %d\n\t\t\tname: \"local\"\n\t\t\taccounts: [\"A\"]\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\ttmplBC := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"B\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now connect s3 to s1 and make sure that s2 connects properly to s3.\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"C\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\ts3.Shutdown()\n\ts2.Shutdown()\n\ts1.Shutdown()\n\n\t// Slightly different version where s2 is connecting to s1, while s1\n\t// connects to s3 (and s3 does not solicit connections).\n\n\tconf1 = createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, _EMPTY_)))\n\ts1, o1 = RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 = createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"B\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ = RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Start s3 first that will simply accept connections\n\tconf3 = createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"C\", _EMPTY_)))\n\ts3, o3 := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\t// Now config reload s1 so that it points to s3.\n\treloadUpdateConfig(t, s1, conf1,\n\t\tfmt.Sprintf(tmplA, o1.Cluster.Port, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o3.Cluster.Port)))\n\n\tcheckClusterFormed(t, s1, s2, s3)\n}\n\nfunc TestRoutePerAccountGossipWorksWithOldServerNotSeed(t *testing.T) {\n\ttmplA := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: %d\n\t\t\tname: \"local\"\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\t// Here, server \"B\" will have no pooling/accounts.\n\ttmplB := `\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\tpool_size: -1\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmplB, o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tl := &captureErrorLogger{errCh: make(chan string, 100)}\n\ts2.SetLogger(l, false, false)\n\n\t// Now connect s3 to s1. Server s1 should not gossip to s2 information\n\t// about pinned-account routes or extra routes from the pool.\n\ttmplC := `\n\t\tport: -1\n\t\tserver_name: \"C\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmplC, o1.Cluster.Port)))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// We should not have had s2 try to create dedicated routes for \"A\" or \"$SYS\"\n\ttm := time.NewTimer(time.Second)\n\tdefer tm.Stop()\n\tfor {\n\t\tselect {\n\t\tcase err := <-l.errCh:\n\t\t\tif strings.Contains(err, \"dedicated route\") {\n\t\t\t\tt.Fatalf(\"Server s2 should not have tried to create a dedicated route: %s\", err)\n\t\t\t}\n\t\tcase <-tm.C:\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestRoutePerAccountGossipWorksWithOldServerSeed(t *testing.T) {\n\ttmplA := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: %d\n\t\t\tname: \"local\"\n\t\t\tpool_size: %d\n\t\t\t%s\n\t\t}\n\t`\n\t// Start with s1 being an \"old\" server, which does not support pooling/pinned-accounts.\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, -1, -1, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\ttmplBC := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: 3\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t\taccounts: [\"A\"]\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"B\", o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now connect s3 to s1 and make sure that s2 connects properly to s3.\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmplBC, \"C\", o1.Cluster.Port)))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tcheckRoutes := func(s *Server, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tif nr := s.NumRoutes(); nr != expected {\n\t\t\t\treturn fmt.Errorf(\"Server %q should have %v routes, got %v\", s.Name(), expected, nr)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Since s1 has no pooling/pinned-accounts, there should be only 2 routes,\n\t// one to s2 and one to s3.\n\tcheckRoutes(s1, 2)\n\t// s2 and s3 should have 1 route to s1 and 3(pool)+\"A\"+\"$SYS\" == 6\n\tcheckRoutes(s2, 6)\n\tcheckRoutes(s3, 6)\n\n\ts1.Shutdown()\n\n\t// The server s1 will now support pooling and accounts pinning.\n\t// Restart the server s1 with the same cluster port otherwise\n\t// s2/s3 would not be able to reconnect.\n\tconf1 = createConfFile(t, []byte(fmt.Sprintf(tmplA, o1.Cluster.Port, 3, \"accounts: [\\\"A\\\"]\")))\n\ts1, _ = RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\t// Make sure reconnect occurs. We should now have 5 routes.\n\tcheckClusterFormed(t, s1, s2, s3)\n\t// Now all servers should have 3+2 to each other, so 10 total.\n\tcheckRoutes(s1, 10)\n\tcheckRoutes(s2, 10)\n\tcheckRoutes(s3, 10)\n}\n\nfunc TestRoutePoolPerAccountSubUnsubProtoParsing(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\textra string\n\t}{\n\t\t{\"regular\", _EMPTY_},\n\t\t{\"pooling\", \"pool_size: 5\"},\n\t\t{\"per-account\", \"accounts: [\\\"A\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconfATemplate := `\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(confATemplate, test.extra)))\n\t\t\tsrva, optsA := RunServerWithConfig(confA)\n\t\t\tdefer srva.Shutdown()\n\n\t\t\tconfBTemplate := `\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA { users: [{user: \"user1\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\troutes = [\n\t\t\t\t\t\t\"nats://127.0.0.1:%d\"\n\t\t\t\t\t]\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(confBTemplate, optsA.Cluster.Port, test.extra)))\n\t\t\tsrvb, _ := RunServerWithConfig(confB)\n\t\t\tdefer srvb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tncA := natsConnect(t, srva.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\tvar sub *nats.Subscription\n\t\t\t\tif i == 0 {\n\t\t\t\t\tsub = natsSubSync(t, ncA, \"foo\")\n\t\t\t\t} else {\n\t\t\t\t\tsub = natsQueueSubSync(t, ncA, \"foo\", \"bar\")\n\t\t\t\t}\n\n\t\t\t\tcheckSubInterest(t, srvb, \"A\", \"foo\", 2*time.Second)\n\n\t\t\t\tcheckSubs := func(s *Server, queue, expected bool) {\n\t\t\t\t\tt.Helper()\n\t\t\t\t\tacc, err := s.LookupAccount(\"A\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Error looking account: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\t\t\tacc.mu.RLock()\n\t\t\t\t\t\tres := acc.sl.Match(\"foo\")\n\t\t\t\t\t\tacc.mu.RUnlock()\n\t\t\t\t\t\tif expected {\n\t\t\t\t\t\t\tif queue && (len(res.qsubs) == 0 || len(res.psubs) != 0) {\n\t\t\t\t\t\t\t\treturn fmt.Errorf(\"Expected queue sub, did not find it\")\n\t\t\t\t\t\t\t} else if !queue && (len(res.psubs) == 0 || len(res.qsubs) != 0) {\n\t\t\t\t\t\t\t\treturn fmt.Errorf(\"Expected psub, did not find it\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else if len(res.psubs)+len(res.qsubs) != 0 {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"Unexpected subscription: %+v\", res)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t}\n\n\t\t\t\tcheckSubs(srva, i == 1, true)\n\t\t\t\tcheckSubs(srvb, i == 1, true)\n\n\t\t\t\tsub.Unsubscribe()\n\t\t\t\tnatsFlush(t, ncA)\n\n\t\t\t\tcheckSubs(srva, i == 1, false)\n\t\t\t\tcheckSubs(srvb, i == 1, false)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRoutePoolPerAccountStreamImport(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\troute string\n\t}{\n\t\t{\"regular\", _EMPTY_},\n\t\t{\"pooled\", \"pool_size: 5\"},\n\t\t{\"one per account\", \"accounts: [\\\"A\\\"]\"},\n\t\t{\"both per account\", \"accounts: [\\\"A\\\", \\\"B\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttmplA := `\n\t\t\t\tserver_name: \"A\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA {\n\t\t\t\t\t\tusers: [{user: \"user1\", password: \"pwd\"}]\n\t\t\t\t\t\texports: [{stream: \"foo\"}]\n\t\t\t\t\t}\n\t\t\t\t\tB {\n\t\t\t\t\t\tusers: [{user: \"user2\", password: \"pwd\"}]\n\t\t\t\t\t\timports: [{stream: {subject: \"foo\", account: \"A\"}}]\n\t\t\t\t\t}\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfA := createConfFile(t, []byte(fmt.Sprintf(tmplA, test.route)))\n\t\t\tsrva, optsA := RunServerWithConfig(confA)\n\t\t\tdefer srva.Shutdown()\n\n\t\t\ttmplB := `\n\t\t\t\tserver_name: \"B\"\n\t\t\t\tport: -1\n\t\t\t\taccounts {\n\t\t\t\t\tA {\n\t\t\t\t\t\tusers: [{user: \"user1\", password: \"pwd\"}]\n\t\t\t\t\t\texports: [{stream: \"foo\"}]\n\t\t\t\t\t}\n\t\t\t\t\tB {\n\t\t\t\t\t\tusers: [{user: \"user2\", password: \"pwd\"}]\n\t\t\t\t\t\timports: [{stream: {subject: \"foo\", account: \"A\"}}]\n\t\t\t\t\t}\n\t\t\t\t\tC { users: [{user: \"user3\", password: \"pwd\"}] }\n\t\t\t\t\tD { users: [{user: \"user4\", password: \"pwd\"}] }\n\t\t\t\t}\n\t\t\t\tcluster {\n\t\t\t\t\tname: \"local\"\n\t\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`\n\t\t\tconfB := createConfFile(t, []byte(fmt.Sprintf(tmplB,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", optsA.Cluster.Port),\n\t\t\t\ttest.route)))\n\t\t\tsrvb, _ := RunServerWithConfig(confB)\n\t\t\tdefer srvb.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srva, srvb)\n\n\t\t\tncB := natsConnect(t, srvb.ClientURL(), nats.UserInfo(\"user2\", \"pwd\"))\n\t\t\tdefer ncB.Close()\n\n\t\t\tsub := natsSubSync(t, ncB, \"foo\")\n\n\t\t\tcheckSubInterest(t, srva, \"B\", \"foo\", time.Second)\n\t\t\tcheckSubInterest(t, srva, \"A\", \"foo\", time.Second)\n\n\t\t\tncA := natsConnect(t, srva.ClientURL(), nats.UserInfo(\"user1\", \"pwd\"))\n\t\t\tdefer ncA.Close()\n\n\t\t\tnatsPub(t, ncA, \"foo\", []byte(\"hello\"))\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\n\t\t\tnatsUnsub(t, sub)\n\t\t\tnatsFlush(t, ncB)\n\n\t\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\t\tfor _, acc := range []string{\"A\", \"B\"} {\n\t\t\t\t\ta, err := srva.LookupAccount(acc)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\ta.mu.RLock()\n\t\t\t\t\tr := a.sl.Match(\"foo\")\n\t\t\t\t\ta.mu.RUnlock()\n\t\t\t\t\tif len(r.psubs) != 0 {\n\t\t\t\t\t\treturn fmt.Errorf(\"Subscription not unsubscribed\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestRoutePoolAndPerAccountWithServiceLatencyNoDataRace(t *testing.T) {\n\t// For this test, we want the system (SYS) and SERVICE accounts to be bound\n\t// to different routes. So the names and pool size have been chosen accordingly.\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolStr string\n\t}{\n\t\t{\"pool\", \"pool_size: 5\"},\n\t\t{\"per account\", \"accounts: [\\\"SYS\\\", \\\"SERVICE\\\", \\\"REQUESTOR\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttmpl := `\n\t\t\tport: -1\n\t\t\taccounts {\n\t\t\t\tSYS {\n\t\t\t\t\tusers [{user: \"sys\", password: \"pwd\"}]\n\t\t\t\t}\n\t\t\t\tSERVICE {\n\t\t\t\t\tusers [{user: \"svc\", password: \"pwd\"}]\n\t\t\t\t\texports = [\n\t\t\t\t\t\t{service: \"req.*\", latency: {subject: \"results\"}}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t\tREQUESTOR {\n\t\t\t\t\tusers [{user: \"req\", password: \"pwd\"}]\n\t\t\t\t\timports = [\n\t\t\t\t\t\t{service: {account: \"SERVICE\", subject: \"req.echo\"}, to: \"request\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t\tsystem_account: \"SYS\"\n\t\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tport: -1\n\t\t\t\t%s\n\t\t\t\t%s\n\t\t\t}\n\t\t`\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.poolStr, _EMPTY_)))\n\t\t\ts1, opts1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, test.poolStr,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", opts1.Cluster.Port))))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\t// Create service provider.\n\t\t\tnc := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"svc\", \"pwd\"))\n\t\t\tdefer nc.Close()\n\n\t\t\t// The service listener.\n\t\t\tnatsSub(t, nc, \"req.echo\", func(msg *nats.Msg) {\n\t\t\t\tmsg.Respond(msg.Data)\n\t\t\t})\n\n\t\t\t// Listen for metrics\n\t\t\trsub := natsSubSync(t, nc, \"results\")\n\t\t\tnatsFlush(t, nc)\n\t\t\tcheckSubInterest(t, s2, \"SERVICE\", \"results\", time.Second)\n\n\t\t\t// Create second client and send request from this one.\n\t\t\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"req\", \"pwd\"))\n\t\t\tdefer nc2.Close()\n\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\t// Send the request.\n\t\t\t\t_, err := nc2.Request(\"request\", []byte(\"hello\"), time.Second)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// Get the latency result\n\t\t\t\tnatsNexMsg(t, rsub, time.Second)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRouteParseOriginClusterMsgArgs(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tracc bool\n\t\targs string\n\t\tpacache string\n\t\treply string\n\t\tqueues [][]byte\n\t}{\n\t\t{true, \"ORIGIN foo 12 345\\r\\n\", \"MY_ACCOUNT foo\", _EMPTY_, nil},\n\t\t{true, \"ORIGIN foo bar 12 345\\r\\n\", \"MY_ACCOUNT foo\", \"bar\", nil},\n\t\t{true, \"ORIGIN foo + bar queue1 queue2 12 345\\r\\n\", \"MY_ACCOUNT foo\", \"bar\", [][]byte{[]byte(\"queue1\"), []byte(\"queue2\")}},\n\t\t{true, \"ORIGIN foo | queue1 queue2 12 345\\r\\n\", \"MY_ACCOUNT foo\", _EMPTY_, [][]byte{[]byte(\"queue1\"), []byte(\"queue2\")}},\n\n\t\t{false, \"ORIGIN MY_ACCOUNT foo 12 345\\r\\n\", \"MY_ACCOUNT foo\", _EMPTY_, nil},\n\t\t{false, \"ORIGIN MY_ACCOUNT foo bar 12 345\\r\\n\", \"MY_ACCOUNT foo\", \"bar\", nil},\n\t\t{false, \"ORIGIN MY_ACCOUNT foo + bar queue1 queue2 12 345\\r\\n\", \"MY_ACCOUNT foo\", \"bar\", [][]byte{[]byte(\"queue1\"), []byte(\"queue2\")}},\n\t\t{false, \"ORIGIN MY_ACCOUNT foo | queue1 queue2 12 345\\r\\n\", \"MY_ACCOUNT foo\", _EMPTY_, [][]byte{[]byte(\"queue1\"), []byte(\"queue2\")}},\n\t} {\n\t\tt.Run(test.args, func(t *testing.T) {\n\t\t\tc := &client{kind: ROUTER, route: &route{}}\n\t\t\tif test.racc {\n\t\t\t\tc.route.accName = []byte(\"MY_ACCOUNT\")\n\t\t\t}\n\t\t\tif err := c.processRoutedOriginClusterMsgArgs([]byte(test.args)); err != nil {\n\t\t\t\tt.Fatalf(\"Error processing: %v\", err)\n\t\t\t}\n\t\t\tif string(c.pa.origin) != \"ORIGIN\" {\n\t\t\t\tt.Fatalf(\"Invalid origin: %q\", c.pa.origin)\n\t\t\t}\n\t\t\tif string(c.pa.account) != \"MY_ACCOUNT\" {\n\t\t\t\tt.Fatalf(\"Invalid account: %q\", c.pa.account)\n\t\t\t}\n\t\t\tif string(c.pa.subject) != \"foo\" {\n\t\t\t\tt.Fatalf(\"Invalid subject: %q\", c.pa.subject)\n\t\t\t}\n\t\t\tif string(c.pa.reply) != test.reply {\n\t\t\t\tt.Fatalf(\"Invalid reply: %q\", c.pa.reply)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(c.pa.queues, test.queues) {\n\t\t\t\tt.Fatalf(\"Invalid queues: %v\", c.pa.queues)\n\t\t\t}\n\t\t\tif c.pa.hdr != 12 {\n\t\t\t\tt.Fatalf(\"Invalid header size: %v\", c.pa.hdr)\n\t\t\t}\n\t\t\tif c.pa.size != 345 {\n\t\t\t\tt.Fatalf(\"Invalid size: %v\", c.pa.size)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRoutePoolAndPerAccountOperatorMode(t *testing.T) {\n\t_, spub := createKey(t)\n\tsysClaim := jwt.NewAccountClaims(spub)\n\tsysClaim.Name = \"SYS\"\n\tsysJwt := encodeClaim(t, sysClaim, spub)\n\n\takp, apub := createKey(t)\n\tclaima := jwt.NewAccountClaims(apub)\n\tajwt := encodeClaim(t, claima, apub)\n\n\tbkp, bpub := createKey(t)\n\tclaimb := jwt.NewAccountClaims(bpub)\n\tbjwt := encodeClaim(t, claimb, bpub)\n\n\tckp, cpub := createKey(t)\n\tclaimc := jwt.NewAccountClaims(cpub)\n\tcjwt := encodeClaim(t, claimc, cpub)\n\n\t_, dpub := createKey(t)\n\n\tbasePath := \"/ngs/v1/accounts/jwt/\"\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == basePath {\n\t\t\tw.Write([]byte(\"ok\"))\n\t\t} else if strings.HasSuffix(r.URL.Path, spub) {\n\t\t\tw.Write([]byte(sysJwt))\n\t\t} else if strings.HasSuffix(r.URL.Path, apub) {\n\t\t\tw.Write([]byte(ajwt))\n\t\t} else if strings.HasSuffix(r.URL.Path, bpub) {\n\t\t\tw.Write([]byte(bjwt))\n\t\t} else if strings.HasSuffix(r.URL.Path, cpub) {\n\t\t\tw.Write([]byte(cjwt))\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\toperator := fmt.Sprintf(`\n\t\toperator: %s\n\t\tsystem_account: %s\n\t\tresolver: URL(\"%s%s\")\n\t`, ojwt, spub, ts.URL, basePath)\n\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: %s\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\taccounts: [\"` + apub + `\"%s]\n\t\t}\n\t` + operator\n\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\t_EMPTY_)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckRoute := func(s *Server, acc string, perAccount bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\ts.mu.RLock()\n\t\t\t_, ok := s.accRoutes[acc]\n\t\t\ts.mu.RUnlock()\n\t\t\tif perAccount && !ok {\n\t\t\t\treturn fmt.Errorf(\"No dedicated route for account %q on server %q\", acc, s)\n\t\t\t} else if !perAccount && ok {\n\t\t\t\treturn fmt.Errorf(\"Dedicated route for account %q on server %q\", acc, s)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\t// Route for accounts \"apub\" and \"spub\" should be a dedicated route\n\tcheckRoute(s1, apub, true)\n\tcheckRoute(s2, apub, true)\n\tcheckRoute(s1, spub, true)\n\tcheckRoute(s2, spub, true)\n\t// Route for account \"bpub\" should not\n\tcheckRoute(s1, bpub, false)\n\tcheckRoute(s2, bpub, false)\n\n\tcheckComm := func(acc string, kp nkeys.KeyPair, subj string) {\n\t\tt.Helper()\n\t\tusr := createUserCreds(t, nil, kp)\n\t\tncAs2 := natsConnect(t, s2.ClientURL(), usr)\n\t\tdefer ncAs2.Close()\n\t\tsub := natsSubSync(t, ncAs2, subj)\n\t\tcheckSubInterest(t, s1, acc, subj, time.Second)\n\n\t\tncAs1 := natsConnect(t, s1.ClientURL(), usr)\n\t\tdefer ncAs1.Close()\n\t\tnatsPub(t, ncAs1, subj, nil)\n\t\tnatsNexMsg(t, sub, time.Second)\n\t}\n\tcheckComm(apub, akp, \"foo\")\n\tcheckComm(bpub, bkp, \"bar\")\n\n\t// Add account \"bpub\" in accounts doing a configuration reload\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, fmt.Sprintf(\",\\\"%s\\\"\", bpub)))\n\t// Already the route should be moved to a dedicated route, even\n\t// before doing the config reload on s2.\n\tcheckRoute(s1, bpub, true)\n\tcheckRoute(s2, bpub, true)\n\t// Account \"apub\" should still have its dedicated route\n\tcheckRoute(s1, apub, true)\n\tcheckRoute(s2, apub, true)\n\t// So the system account\n\tcheckRoute(s1, spub, true)\n\tcheckRoute(s2, spub, true)\n\t// Let's complete the config reload on srvb\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\tfmt.Sprintf(\",\\\"%s\\\"\", bpub)))\n\tcheckClusterFormed(t, s1, s2)\n\t// Check communication on account bpub again.\n\tcheckComm(bpub, bkp, \"baz\")\n\n\t// Now add with config reload an account that has not been used yet (cpub).\n\t// We will also remove account bpub from the account list.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", _EMPTY_, fmt.Sprintf(\",\\\"%s\\\"\", cpub)))\n\t// Again, check before reloading s2.\n\tcheckRoute(s1, cpub, true)\n\tcheckRoute(s2, cpub, true)\n\t// Now reload s2 and do other checks.\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port),\n\t\tfmt.Sprintf(\",\\\"%s\\\"\", cpub)))\n\tcheckClusterFormed(t, s1, s2)\n\tcheckRoute(s1, bpub, false)\n\tcheckRoute(s2, bpub, false)\n\tcheckComm(cpub, ckp, \"bat\")\n\n\t// Finally, let's try to add an account that the account server rejects.\n\terr := os.WriteFile(conf1, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, fmt.Sprintf(\",\\\"%s\\\",\\\"%s\\\"\", cpub, dpub))), 0666)\n\trequire_NoError(t, err)\n\tif err := s1.Reload(); err == nil || !strings.Contains(err.Error(), dpub) {\n\t\tt.Fatalf(\"Expected error about not being able to lookup this account, got %q\", err)\n\t}\n}\n\nfunc TestRoutePoolAndPerAccountWithOlderServer(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: 5\n\t\t\taccounts: [\"A\"]\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\",\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\t// Have s3 explicitly disable pooling (to behave as an old server)\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tserver_name: \"C\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"A\", password: \"pwd\"}] }\n\t\t\tB { users: [{user: \"B\", password: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: -1\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t`, o1.Cluster.Port)))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tcheck := func(acc, subj string, subSrv, pubSrv1, pubSrv2 *Server) {\n\t\tt.Helper()\n\n\t\tncSub := natsConnect(t, subSrv.ClientURL(), nats.UserInfo(acc, \"pwd\"))\n\t\tdefer ncSub.Close()\n\t\tsub := natsSubSync(t, ncSub, subj)\n\n\t\tcheckSubInterest(t, pubSrv1, acc, subj, time.Second)\n\t\tcheckSubInterest(t, pubSrv2, acc, subj, time.Second)\n\n\t\tpub := func(s *Server) {\n\t\t\tt.Helper()\n\t\t\tnc := natsConnect(t, s.ClientURL(), nats.UserInfo(acc, \"pwd\"))\n\t\t\tdefer nc.Close()\n\n\t\t\tnatsPub(t, nc, subj, []byte(\"hello\"))\n\t\t\tnatsNexMsg(t, sub, time.Second)\n\t\t}\n\t\tpub(pubSrv1)\n\t\tpub(pubSrv2)\n\t}\n\tcheck(\"A\", \"subj1\", s1, s2, s3)\n\tcheck(\"A\", \"subj2\", s2, s1, s3)\n\tcheck(\"A\", \"subj3\", s3, s1, s2)\n\tcheck(\"B\", \"subj4\", s1, s2, s3)\n\tcheck(\"B\", \"subj5\", s2, s1, s3)\n\tcheck(\"B\", \"subj6\", s3, s1, s2)\n}\n\ntype testDuplicateRouteLogger struct {\n\tDummyLogger\n\tch chan struct{}\n\tcount int\n}\n\nfunc (l *testDuplicateRouteLogger) Noticef(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif !strings.Contains(msg, DuplicateRoute.String()) {\n\t\treturn\n\t}\n\tselect {\n\tcase l.ch <- struct{}{}:\n\tdefault:\n\t}\n\tl.Mutex.Lock()\n\tl.count++\n\tl.Mutex.Unlock()\n}\n\n// This test will make sure that a server with pooling does not\n// keep trying to connect to a non pooled (for instance old) server.\n// Will also make sure that if the old server is simply accepting\n// connections, and restarted, the server with pooling will connect.\nfunc TestRoutePoolWithOlderServerConnectAndReconnect(t *testing.T) {\n\ttmplA := `\n\t\tport: -1\n\t\tserver_name: \"A\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tpool_size: 3\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmplA, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tl := &testDuplicateRouteLogger{ch: make(chan struct{}, 50)}\n\ts1.SetLogger(l, false, false)\n\n\ttmplB := `\n\t\tport: -1\n\t\tserver_name: \"B\"\n\t\tcluster {\n\t\t\tport: %d\n\t\t\tname: \"local\"\n\t\t\tpool_size: -1\n\t\t\t%s\n\t\t}\n\t`\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmplB, -1,\n\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\ts2, o2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Now reload configuration of s1 to point to s2.\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmplA, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o2.Cluster.Port)))\n\tcheckClusterFormed(t, s1, s2)\n\n\t// We could get some, but it should settle.\n\tcheckRepeatConnect := func() {\n\t\tt.Helper()\n\t\ttm := time.NewTimer(4 * routeConnectDelay)\n\t\tvar last time.Time\n\t\tfor done := false; !done; {\n\t\t\tselect {\n\t\t\tcase <-l.ch:\n\t\t\t\tlast = time.Now()\n\t\t\tcase <-tm.C:\n\t\t\t\tdone = true\n\t\t\t}\n\t\t}\n\t\tif dur := time.Since(last); dur <= routeConnectDelay {\n\t\t\tt.Fatalf(\"Still attempted to connect %v ago\", dur)\n\t\t}\n\t}\n\tcheckRepeatConnect()\n\n\t// Now shutdown s2 and restart it without active route to s1.\n\t// Check that cluster can still be formed: that is, s1 is\n\t// still trying to connect to s2.\n\ts2.Shutdown()\n\t// Wait for more than a regular reconnect delay attempt.\n\t// Note that in test it is set to 15ms.\n\ttime.Sleep(50 * time.Millisecond)\n\t// Restart the server s2 with the same cluster port otherwise\n\t// s1 would not be able to reconnect.\n\tconf2 = createConfFile(t, []byte(fmt.Sprintf(tmplB, o2.Cluster.Port, _EMPTY_)))\n\ts2, _ = RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\t// Make sure reconnect occurs\n\tcheckClusterFormed(t, s1, s2)\n\t// And again, make sure there is no repeat-connect\n\tcheckRepeatConnect()\n}\n\nfunc TestRoutePoolBadAuthNoRunawayCreateRoute(t *testing.T) {\n\tconf1 := createConfFile(t, []byte(`\n\t\tserver_name: \"S1\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tpool_size: 4\n\t\t\tauthorization {\n\t\t\t\tuser: \"correct\"\n\t\t\t\tpassword: \"correct\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tl := &captureErrorLogger{errCh: make(chan string, 100)}\n\ts1.SetLogger(l, false, false)\n\n\ttmpl := `\n\t\tserver_name: \"S2\"\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tpool_size: 5\n\t\t\troutes: [\"nats://%s@127.0.0.1:%d\"]\n\t\t}\n\t`\n\tconf2 := createConfFile(t, fmt.Appendf(nil, tmpl, \"incorrect:incorrect\", o1.Cluster.Port))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tdeadline := time.Now().Add(2 * time.Second)\n\tvar errors int\n\tfor time.Now().Before(deadline) {\n\t\tselect {\n\t\tcase <-l.errCh:\n\t\t\terrors++\n\t\tdefault:\n\t\t}\n\t}\n\t// We should not get that many errors now. In the past, we would get more\n\t// than 200 for the 2 sec wait.\n\tif errors > 10 {\n\t\tt.Fatalf(\"Unexpected number of errors: %v\", errors)\n\t}\n\n\t// Reload with proper credentials.\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"correct:correct\", o1.Cluster.Port))\n\t// Ensure we can connect.\n\tcheckClusterFormed(t, s1, s2)\n}\n\nfunc TestRouteCompressionOptions(t *testing.T) {\n\torg := testDefaultClusterCompression\n\ttestDefaultClusterCompression = _EMPTY_\n\tdefer func() { testDefaultClusterCompression = org }()\n\n\ttmpl := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tcompression: %s\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmode string\n\t\trttVals []int\n\t\texpected string\n\t\trtts []time.Duration\n\t}{\n\t\t{\"boolean enabled\", \"true\", nil, CompressionS2Fast, nil},\n\t\t{\"string enabled\", \"enabled\", nil, CompressionS2Fast, nil},\n\t\t{\"string EnaBled\", \"EnaBled\", nil, CompressionS2Fast, nil},\n\t\t{\"string on\", \"on\", nil, CompressionS2Fast, nil},\n\t\t{\"string ON\", \"ON\", nil, CompressionS2Fast, nil},\n\t\t{\"string fast\", \"fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string Fast\", \"Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_fast\", \"s2_fast\", nil, CompressionS2Fast, nil},\n\t\t{\"string s2_Fast\", \"s2_Fast\", nil, CompressionS2Fast, nil},\n\t\t{\"boolean disabled\", \"false\", nil, CompressionOff, nil},\n\t\t{\"string disabled\", \"disabled\", nil, CompressionOff, nil},\n\t\t{\"string DisableD\", \"DisableD\", nil, CompressionOff, nil},\n\t\t{\"string off\", \"off\", nil, CompressionOff, nil},\n\t\t{\"string OFF\", \"OFF\", nil, CompressionOff, nil},\n\t\t{\"better\", \"better\", nil, CompressionS2Better, nil},\n\t\t{\"Better\", \"Better\", nil, CompressionS2Better, nil},\n\t\t{\"s2_better\", \"s2_better\", nil, CompressionS2Better, nil},\n\t\t{\"S2_BETTER\", \"S2_BETTER\", nil, CompressionS2Better, nil},\n\t\t{\"best\", \"best\", nil, CompressionS2Best, nil},\n\t\t{\"BEST\", \"BEST\", nil, CompressionS2Best, nil},\n\t\t{\"s2_best\", \"s2_best\", nil, CompressionS2Best, nil},\n\t\t{\"S2_BEST\", \"S2_BEST\", nil, CompressionS2Best, nil},\n\t\t{\"auto no rtts\", \"auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"s2_auto no rtts\", \"s2_auto\", nil, CompressionS2Auto, defaultCompressionS2AutoRTTThresholds},\n\t\t{\"auto\", \"{mode: auto, rtt_thresholds: [%s]}\", []int{1}, CompressionS2Auto, []time.Duration{time.Millisecond}},\n\t\t{\"Auto\", \"{Mode: Auto, thresholds: [%s]}\", []int{1, 2}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond}},\n\t\t{\"s2_auto\", \"{mode: s2_auto, thresholds: [%s]}\", []int{1, 2, 3}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}},\n\t\t{\"s2_AUTO\", \"{mode: s2_AUTO, thresholds: [%s]}\", []int{1, 2, 3, 4}, CompressionS2Auto, []time.Duration{time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond, 4 * time.Millisecond}},\n\t\t{\"s2_auto:-10,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{-10, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,15\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 15}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 15 * time.Millisecond}},\n\t\t{\"s2_auto:0,5,10\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 5, 10}, CompressionS2Auto, []time.Duration{0, 5 * time.Millisecond, 10 * time.Millisecond}},\n\t\t{\"s2_auto:5,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{5, 10, 0, 20}, CompressionS2Auto, []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 10, 0, 20}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,0,0,20\", \"{mode: s2_auto, thresholds: [%s]}\", []int{0, 0, 0, 20}, CompressionS2Auto, []time.Duration{0, 0, 0, 20 * time.Millisecond}},\n\t\t{\"s2_auto:0,10,0,0\", \"{mode: s2_auto, rtt_thresholds: [%s]}\", []int{0, 10, 0, 0}, CompressionS2Auto, []time.Duration{0, 10 * time.Millisecond}},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar val string\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\tvar rtts string\n\t\t\t\tfor i, v := range test.rttVals {\n\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\trtts += \", \"\n\t\t\t\t\t}\n\t\t\t\t\trtts += fmt.Sprintf(\"%dms\", v)\n\t\t\t\t}\n\t\t\t\tval = fmt.Sprintf(test.mode, rtts)\n\t\t\t} else {\n\t\t\t\tval = test.mode\n\t\t\t}\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, val)))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tif cm := o.Cluster.Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, o.Cluster.Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, o.Cluster.Compression.RTTThresholds)\n\t\t\t}\n\t\t\ts.Shutdown()\n\n\t\t\to.Cluster.Port = -1\n\t\t\to.Cluster.Compression.Mode = test.mode\n\t\t\tif len(test.rttVals) > 0 {\n\t\t\t\to.Cluster.Compression.Mode = CompressionS2Auto\n\t\t\t\to.Cluster.Compression.RTTThresholds = o.Cluster.Compression.RTTThresholds[:0]\n\t\t\t\tfor _, v := range test.rttVals {\n\t\t\t\t\to.Cluster.Compression.RTTThresholds = append(o.Cluster.Compression.RTTThresholds, time.Duration(v)*time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts = RunServer(o)\n\t\t\tdefer s.Shutdown()\n\t\t\tif cm := o.Cluster.Compression.Mode; cm != test.expected {\n\t\t\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", test.expected, cm)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.rtts, o.Cluster.Compression.RTTThresholds) {\n\t\t\t\tt.Fatalf(\"Expected RTT tresholds to be %+v, got %+v\", test.rtts, o.Cluster.Compression.RTTThresholds)\n\t\t\t}\n\t\t})\n\t}\n\t// Test that with no compression specified, we default to \"accept\"\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tif cm := o.Cluster.Compression.Mode; cm != CompressionAccept {\n\t\tt.Fatalf(\"Expected compression value to be %q, got %q\", CompressionAccept, cm)\n\t}\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmode string\n\t\trtts []time.Duration\n\t\terr string\n\t}{\n\t\t{\"unsupported mode\", \"gzip\", nil, \"unsupported\"},\n\t\t{\"not ascending order\", \"s2_auto\", []time.Duration{\n\t\t\t5 * time.Millisecond,\n\t\t\t10 * time.Millisecond,\n\t\t\t2 * time.Millisecond,\n\t\t}, \"ascending\"},\n\t\t{\"too many thresholds\", \"s2_auto\", []time.Duration{\n\t\t\t5 * time.Millisecond,\n\t\t\t10 * time.Millisecond,\n\t\t\t20 * time.Millisecond,\n\t\t\t40 * time.Millisecond,\n\t\t\t60 * time.Millisecond,\n\t\t}, \"more than 4\"},\n\t\t{\"all 0\", \"s2_auto\", []time.Duration{0, 0, 0, 0}, \"at least one\"},\n\t\t{\"single 0\", \"s2_auto\", []time.Duration{0}, \"at least one\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.Cluster.Port = -1\n\t\t\to.Cluster.Compression = CompressionOpts{test.mode, test.rtts}\n\t\t\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype testConnSentBytes struct {\n\tnet.Conn\n\tsync.RWMutex\n\tsent int\n}\n\nfunc (c *testConnSentBytes) Write(p []byte) (int, error) {\n\tn, err := c.Conn.Write(p)\n\tc.Lock()\n\tc.sent += n\n\tc.Unlock()\n\treturn n, err\n}\n\nfunc TestRouteCompression(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\taccounts {\n\t\t\tA { users: [{user: \"a\", pass: \"pwd\"}] }\n\t\t}\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\tcompression: true\n\t\t\tpool_size: %d\n\t\t\t%s\n\t\t\t%s\n\t\t}\n\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpoolSize int\n\t\taccounts string\n\t}{\n\t\t{\"no pooling\", -1, _EMPTY_},\n\t\t{\"pooling\", 3, _EMPTY_},\n\t\t{\"per account\", 1, \"accounts: [\\\"A\\\"]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"S1\", test.poolSize, test.accounts, _EMPTY_)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"S2\", test.poolSize, test.accounts,\n\t\t\t\tfmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\ts1.mu.RLock()\n\t\t\ts1.forEachRoute(func(r *client) {\n\t\t\t\tr.mu.Lock()\n\t\t\t\tr.nc = &testConnSentBytes{Conn: r.nc}\n\t\t\t\tr.mu.Unlock()\n\t\t\t})\n\t\t\ts1.mu.RUnlock()\n\n\t\t\tnc2 := natsConnect(t, s2.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer nc2.Close()\n\t\t\tsub := natsSubSync(t, nc2, \"foo\")\n\t\t\tnatsFlush(t, nc2)\n\t\t\tcheckSubInterest(t, s1, \"A\", \"foo\", time.Second)\n\n\t\t\tnc1 := natsConnect(t, s1.ClientURL(), nats.UserInfo(\"a\", \"pwd\"))\n\t\t\tdefer nc1.Close()\n\n\t\t\tvar payloads [][]byte\n\t\t\tcount := 26\n\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\tn := rand.Intn(2048) + 1\n\t\t\t\tp := make([]byte, n)\n\t\t\t\tfor j := 0; j < n; j++ {\n\t\t\t\t\tp[j] = byte(i) + 'A'\n\t\t\t\t}\n\t\t\t\tpayloads = append(payloads, p)\n\t\t\t\tnatsPub(t, nc1, \"foo\", p)\n\t\t\t}\n\n\t\t\ttotalPayloadSize := 0\n\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\tm := natsNexMsg(t, sub, time.Second)\n\t\t\t\tif !bytes.Equal(m.Data, payloads[i]) {\n\t\t\t\t\tt.Fatalf(\"Expected payload %q - got %q\", payloads[i], m.Data)\n\t\t\t\t}\n\t\t\t\ttotalPayloadSize += len(m.Data)\n\t\t\t}\n\n\t\t\t// Also check that the route stats shows that compression likely occurred\n\t\t\tvar out int\n\t\t\ts1.mu.RLock()\n\t\t\tif len(test.accounts) > 0 {\n\t\t\t\trems := s1.accRoutes[\"A\"]\n\t\t\t\tif rems == nil {\n\t\t\t\t\tt.Fatal(\"Did not find route for account\")\n\t\t\t\t}\n\t\t\t\tfor _, r := range rems {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tif r.nc != nil {\n\t\t\t\t\t\tnc := r.nc.(*testConnSentBytes)\n\t\t\t\t\t\tnc.RLock()\n\t\t\t\t\t\tout = nc.sent\n\t\t\t\t\t\tnc.RUnlock()\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tai, _ := s1.accounts.Load(\"A\")\n\t\t\t\tacc := ai.(*Account)\n\t\t\t\tacc.mu.RLock()\n\t\t\t\tpi := acc.routePoolIdx\n\t\t\t\tacc.mu.RUnlock()\n\t\t\t\ts1.forEachRouteIdx(pi, func(r *client) bool {\n\t\t\t\t\tr.mu.Lock()\n\t\t\t\t\tif r.nc != nil {\n\t\t\t\t\t\tnc := r.nc.(*testConnSentBytes)\n\t\t\t\t\t\tnc.RLock()\n\t\t\t\t\t\tout = nc.sent\n\t\t\t\t\t\tnc.RUnlock()\n\t\t\t\t\t}\n\t\t\t\t\tr.mu.Unlock()\n\t\t\t\t\treturn false\n\t\t\t\t})\n\t\t\t}\n\t\t\ts1.mu.RUnlock()\n\t\t\t// Should at least be smaller than totalPayloadSize, use 20%.\n\t\t\tlimit := totalPayloadSize * 80 / 100\n\t\t\tif int(out) > limit {\n\t\t\t\tt.Fatalf(\"Expected s1's outBytes to be less than %v, got %v\", limit, out)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRouteCompressionMatrixModes(t *testing.T) {\n\ttmpl := `\n\t\t\tport: -1\n\t\t\tserver_name: \"%s\"\n\t\t\tcluster {\n\t\t\t\tname: \"local\"\n\t\t\t\tport: -1\n\t\t\t\tcompression: %s\n\t\t\t\tpool_size: -1\n\t\t\t\t%s\n\t\t\t}\n\t\t`\n\tfor _, test := range []struct {\n\t\tname string\n\t\ts1 string\n\t\ts2 string\n\t\ts1Expected string\n\t\ts2Expected string\n\t}{\n\t\t{\"off off\", \"off\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"off accept\", \"off\", \"accept\", CompressionOff, CompressionOff},\n\t\t{\"off on\", \"off\", \"on\", CompressionOff, CompressionOff},\n\t\t{\"off better\", \"off\", \"better\", CompressionOff, CompressionOff},\n\t\t{\"off best\", \"off\", \"best\", CompressionOff, CompressionOff},\n\n\t\t{\"accept off\", \"accept\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"accept accept\", \"accept\", \"accept\", CompressionOff, CompressionOff},\n\t\t{\"accept on\", \"accept\", \"on\", CompressionS2Fast, CompressionS2Fast},\n\t\t{\"accept better\", \"accept\", \"better\", CompressionS2Better, CompressionS2Better},\n\t\t{\"accept best\", \"accept\", \"best\", CompressionS2Best, CompressionS2Best},\n\n\t\t{\"on off\", \"on\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"on accept\", \"on\", \"accept\", CompressionS2Fast, CompressionS2Fast},\n\t\t{\"on on\", \"on\", \"on\", CompressionS2Fast, CompressionS2Fast},\n\t\t{\"on better\", \"on\", \"better\", CompressionS2Fast, CompressionS2Better},\n\t\t{\"on best\", \"on\", \"best\", CompressionS2Fast, CompressionS2Best},\n\n\t\t{\"better off\", \"better\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"better accept\", \"better\", \"accept\", CompressionS2Better, CompressionS2Better},\n\t\t{\"better on\", \"better\", \"on\", CompressionS2Better, CompressionS2Fast},\n\t\t{\"better better\", \"better\", \"better\", CompressionS2Better, CompressionS2Better},\n\t\t{\"better best\", \"better\", \"best\", CompressionS2Better, CompressionS2Best},\n\n\t\t{\"best off\", \"best\", \"off\", CompressionOff, CompressionOff},\n\t\t{\"best accept\", \"best\", \"accept\", CompressionS2Best, CompressionS2Best},\n\t\t{\"best on\", \"best\", \"on\", CompressionS2Best, CompressionS2Fast},\n\t\t{\"best better\", \"best\", \"better\", CompressionS2Best, CompressionS2Better},\n\t\t{\"best best\", \"best\", \"best\", CompressionS2Best, CompressionS2Best},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", test.s1, _EMPTY_)))\n\t\t\ts1, o1 := RunServerWithConfig(conf1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", test.s2, fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port))))\n\t\t\ts2, _ := RunServerWithConfig(conf2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\tnc1 := natsConnect(t, s1.ClientURL())\n\t\t\tdefer nc1.Close()\n\n\t\t\tnc2 := natsConnect(t, s2.ClientURL())\n\t\t\tdefer nc2.Close()\n\n\t\t\tpayload := make([]byte, 128)\n\t\t\tcheck := func(ncp, ncs *nats.Conn, subj string, s *Server) {\n\t\t\t\tt.Helper()\n\t\t\t\tsub := natsSubSync(t, ncs, subj)\n\t\t\t\tcheckSubInterest(t, s, globalAccountName, subj, time.Second)\n\t\t\t\tnatsPub(t, ncp, subj, payload)\n\t\t\t\tnatsNexMsg(t, sub, time.Second)\n\n\t\t\t\tfor _, srv := range []*Server{s1, s2} {\n\t\t\t\t\trz, err := srv.Routez(nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tvar expected string\n\t\t\t\t\tif srv == s1 {\n\t\t\t\t\t\texpected = test.s1Expected\n\t\t\t\t\t} else {\n\t\t\t\t\t\texpected = test.s2Expected\n\t\t\t\t\t}\n\t\t\t\t\tif cm := rz.Routes[0].Compression; cm != expected {\n\t\t\t\t\t\tt.Fatalf(\"Server %s - expected compression %q, got %q\", srv, expected, cm)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheck(nc1, nc2, \"foo\", s1)\n\t\t\tcheck(nc2, nc1, \"bar\", s2)\n\t\t})\n\t}\n}\n\nfunc TestRouteCompressionWithOlderServer(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, \"compression: \\\"on\\\"\")))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troutes := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", routes, \"compression: \\\"not supported\\\"\")))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Make sure that s1 route's compression is \"off\"\n\ts1.mu.RLock()\n\ts1.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tcm := r.route.compression\n\t\tr.mu.Unlock()\n\t\tif cm != CompressionNotSupported {\n\t\t\ts1.mu.RUnlock()\n\t\t\tt.Fatalf(\"Compression should be %q, got %q\", CompressionNotSupported, cm)\n\t\t}\n\t})\n\ts1.mu.RUnlock()\n}\n\nfunc TestRouteCompressionImplicitRoute(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\t%s\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", _EMPTY_, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troutes := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", routes, \"compression: \\\"fast\\\"\")))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"C\", routes, \"compression: \\\"best\\\"\")))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tcheckComp := func(s *Server, remoteID, expected string) {\n\t\tt.Helper()\n\t\ts.mu.RLock()\n\t\tdefer s.mu.RUnlock()\n\t\tvar err error\n\t\ts.forEachRoute(func(r *client) {\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tvar cm string\n\t\t\tok := true\n\t\t\tr.mu.Lock()\n\t\t\tif r.route.remoteID == remoteID {\n\t\t\t\tcm = r.route.compression\n\t\t\t\tok = cm == expected\n\t\t\t}\n\t\t\tr.mu.Unlock()\n\t\t\tif !ok {\n\t\t\t\terr = fmt.Errorf(\"Server %q - expected route to %q to use compression %q, got %q\",\n\t\t\t\t\ts, remoteID, expected, cm)\n\t\t\t}\n\t\t})\n\t}\n\tcheckComp(s1, s2.ID(), CompressionS2Fast)\n\tcheckComp(s1, s3.ID(), CompressionS2Best)\n\tcheckComp(s2, s1.ID(), CompressionS2Fast)\n\tcheckComp(s2, s3.ID(), CompressionS2Best)\n\tcheckComp(s3, s1.ID(), CompressionS2Best)\n\tcheckComp(s3, s2.ID(), CompressionS2Best)\n}\n\nfunc TestRouteCompressionAuto(t *testing.T) {\n\ttmpl := `\n\t\tport: -1\n\t\tserver_name: \"%s\"\n\t\tping_interval: \"%s\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"local\"\n\t\t\tcompression: %s\n\t\t\t%s\n\t\t}\n\t`\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"A\", \"10s\", \"s2_fast\", _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\t// Start with 0ms RTT\n\tnp := createNetProxy(0, 1024*1024*1024, 1024*1024*1024, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port), true)\n\troutes := fmt.Sprintf(\"routes: [\\\"%s\\\"]\", np.routeURL())\n\n\trtts := \"{mode: s2_auto, rtt_thresholds: [100ms, 200ms, 300ms]}\"\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"B\", \"500ms\", rtts, routes)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\tdefer np.stop()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckComp := func(expected string) {\n\t\tt.Helper()\n\t\tcheckFor(t, 4*time.Second, 50*time.Millisecond, func() error {\n\t\t\ts2.mu.RLock()\n\t\t\tdefer s2.mu.RUnlock()\n\t\t\tif n := s2.numRoutes(); n != 4 {\n\t\t\t\treturn fmt.Errorf(\"Cluster not formed properly, got %v routes\", n)\n\t\t\t}\n\t\t\tvar err error\n\t\t\ts2.forEachRoute(func(r *client) {\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tr.mu.Lock()\n\t\t\t\tcm := r.route.compression\n\t\t\t\tr.mu.Unlock()\n\t\t\t\tif cm != expected {\n\t\t\t\t\terr = fmt.Errorf(\"Route %v compression mode expected to be %q, got %q\", r, expected, cm)\n\t\t\t\t}\n\t\t\t})\n\t\t\treturn err\n\t\t})\n\t}\n\tcheckComp(CompressionS2Uncompressed)\n\n\t// Change the proxy RTT and we should get compression \"fast\"\n\tnp.updateRTT(150 * time.Millisecond)\n\tcheckComp(CompressionS2Fast)\n\n\t// Now 250ms, and get \"better\"\n\tnp.updateRTT(250 * time.Millisecond)\n\tcheckComp(CompressionS2Better)\n\n\t// Above 350 and we should get \"best\"\n\tnp.updateRTT(350 * time.Millisecond)\n\tcheckComp(CompressionS2Best)\n\n\t// Down to 1ms and again should get \"uncompressed\"\n\tnp.updateRTT(1 * time.Millisecond)\n\tcheckComp(CompressionS2Uncompressed)\n\n\t// Do a config reload with disabling uncompressed\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", \"500ms\", \"{mode: s2_auto, rtt_thresholds: [0ms, 100ms, 0ms, 300ms]}\", routes))\n\t// Change the RTT back down to 1ms, but we should not go uncompressed,\n\t// we should have \"fast\" compression.\n\tnp.updateRTT(1 * time.Millisecond)\n\tcheckComp(CompressionS2Fast)\n\t// Now bump to 150ms and we should be using \"best\", not the \"better\" mode\n\tnp.updateRTT(150 * time.Millisecond)\n\tcheckComp(CompressionS2Best)\n\t// Try 400ms and we should still be using \"best\"\n\tnp.updateRTT(400 * time.Millisecond)\n\tcheckComp(CompressionS2Best)\n\n\t// Try other variations\n\treloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, \"B\", \"500ms\", \"{mode: s2_auto, rtt_thresholds: [50ms, 150ms, 0ms, 0ms]}\", routes))\n\tnp.updateRTT(0 * time.Millisecond)\n\tcheckComp(CompressionS2Uncompressed)\n\tnp.updateRTT(100 * time.Millisecond)\n\tcheckComp(CompressionS2Fast)\n\t// Since we expect the same compression level, just wait before doing\n\t// the update and the next check.\n\ttime.Sleep(100 * time.Millisecond)\n\tnp.updateRTT(250 * time.Millisecond)\n\tcheckComp(CompressionS2Fast)\n\n\t// Now disable compression on s1\n\treloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, \"A\", \"10s\", \"off\", _EMPTY_))\n\t// Wait a bit to make sure we don't check for cluster too soon since\n\t// we expect a disconnect.\n\ttime.Sleep(100 * time.Millisecond)\n\tcheckClusterFormed(t, s1, s2)\n\t// Now change the RTT values in the proxy.\n\tnp.updateRTT(0 * time.Millisecond)\n\t// Now check that s2 also shows as \"off\". Wait for some ping intervals.\n\ttime.Sleep(200 * time.Millisecond)\n\tcheckComp(CompressionOff)\n}\n\nfunc TestRoutePings(t *testing.T) {\n\trouteMaxPingInterval = 50 * time.Millisecond\n\tdefer func() { routeMaxPingInterval = defaultRouteMaxPingInterval }()\n\n\to1 := DefaultOptions()\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tch := make(chan struct{}, 1)\n\ts1.mu.RLock()\n\ts1.forEachRemote(func(r *client) {\n\t\tr.mu.Lock()\n\t\tr.nc = &capturePingConn{r.nc, ch}\n\t\tr.mu.Unlock()\n\t})\n\ts1.mu.RUnlock()\n\n\tfor i := 0; i < 5; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\tt.Fatalf(\"Did not send PING\")\n\t\t}\n\t}\n}\n\nfunc TestRouteCustomPing(t *testing.T) {\n\tpingInterval := 50 * time.Millisecond\n\to1 := DefaultOptions()\n\to1.Cluster.PingInterval = pingInterval\n\to1.Cluster.MaxPingsOut = 2\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.Cluster.PingInterval = pingInterval\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tch := make(chan struct{}, 1)\n\ts1.mu.RLock()\n\ts1.forEachRemote(func(r *client) {\n\t\tr.mu.Lock()\n\t\tr.nc = &capturePingConn{r.nc, ch}\n\t\tr.mu.Unlock()\n\t})\n\ts1.mu.RUnlock()\n\n\tfor i := 0; i < 5; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\tt.Fatalf(\"Did not send PING\")\n\t\t}\n\t}\n}\n\nfunc TestRouteNoLeakOnSlowConsumer(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.Cluster.PoolSize = -1\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.Cluster.PoolSize = -1\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// For any route connections on the first server, drop the write\n\t// deadline down and then get the client to try sending something.\n\t// This should result in an effectively immediate write timeout,\n\t// which will surface as a slow consumer.\n\ts1.mu.Lock()\n\tfor _, cl := range s1.routes {\n\t\tfor _, c := range cl {\n\t\t\tc.mu.Lock()\n\t\t\tc.out.wdl = time.Nanosecond\n\t\t\tc.mu.Unlock()\n\t\t\tc.sendRTTPing()\n\t\t}\n\t}\n\ts1.mu.Unlock()\n\n\t// By now the routes should have gone down, so check that there\n\t// aren't any routes listed still.\n\tcheckFor(t, time.Millisecond*500, time.Millisecond*25, func() error {\n\t\tif nc := s1.NumRoutes(); nc != 0 {\n\t\t\treturn fmt.Errorf(\"Server 1 should have no route connections, got %v\", nc)\n\t\t}\n\t\tif nc := s2.NumRoutes(); nc != 0 {\n\t\t\treturn fmt.Errorf(\"Server 2 should have no route connections, got %v\", nc)\n\t\t}\n\t\treturn nil\n\t})\n\tvar got, expected int64\n\tgot = s1.NumSlowConsumers()\n\texpected = 1\n\tif got != expected {\n\t\tt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t}\n\tgot = int64(s1.NumSlowConsumersRoutes())\n\tif got != expected {\n\t\tt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t}\n\tgot = int64(s1.NumSlowConsumersClients())\n\texpected = 0\n\tif got != expected {\n\t\tt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t}\n\tvarz, err := s1.Varz(nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif varz.SlowConsumersStats.Clients != 0 {\n\t\tt.Error(\"Expected no slow consumer clients\")\n\t}\n\tif varz.SlowConsumersStats.Routes != 1 {\n\t\tt.Error(\"Expected a slow consumer route\")\n\t}\n}\n\nfunc TestRouteNoAppSubLeakOnSlowConsumer(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.MaxPending = 1024\n\to1.MaxPayload = int32(o1.MaxPending)\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2 := DefaultOptions()\n\to2.MaxPending = 1024\n\to2.MaxPayload = int32(o2.MaxPending)\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tcheckSub := func(expected bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tsubsz, err := s1.Subsz(&SubszOptions{Subscriptions: true, Account: globalAccountName})\n\t\t\trequire_NoError(t, err)\n\t\t\tfor _, sub := range subsz.Subs {\n\t\t\t\tif sub.Subject == \"foo\" {\n\t\t\t\t\tif expected {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\treturn fmt.Errorf(\"Subscription should not have been found: %+v\", sub)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif expected {\n\t\t\t\treturn fmt.Errorf(\"Subscription on `foo` not found\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckRoutedSub := func(expected bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\troutez, err := s2.Routez(&RoutezOptions{Subscriptions: true})\n\t\t\trequire_NoError(t, err)\n\t\t\tfor _, route := range routez.Routes {\n\t\t\t\tif route.Account != _EMPTY_ {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif len(route.Subs) == 1 && route.Subs[0] == \"foo\" {\n\t\t\t\t\tif expected {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\treturn fmt.Errorf(\"Subscription should not have been found: %+v\", route.Subs)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif expected {\n\t\t\t\treturn fmt.Errorf(\"Did not find `foo` subscription\")\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckClosed := func(cid uint64) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 50*time.Millisecond, func() error {\n\t\t\tconnz, err := s1.Connz(&ConnzOptions{State: ConnClosed, CID: cid, Subscriptions: true})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\trequire_Len(t, len(connz.Conns), 1)\n\t\t\tconn := connz.Conns[0]\n\t\t\trequire_Equal(t, conn.Reason, SlowConsumerPendingBytes.String())\n\t\t\tsubs := conn.Subs\n\t\t\trequire_Len(t, len(subs), 1)\n\t\t\trequire_Equal[string](t, subs[0], \"foo\")\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tfor i := 0; i < 5; i++ {\n\t\tnc := natsConnect(t, s1.ClientURL())\n\t\tdefer nc.Close()\n\n\t\tnatsSubSync(t, nc, \"foo\")\n\t\tnatsFlush(t, nc)\n\n\t\tcheckSub(true)\n\t\tcheckRoutedSub(true)\n\n\t\tcid, err := nc.GetClientID()\n\t\trequire_NoError(t, err)\n\t\tc := s1.getClient(cid)\n\t\tpayload := make([]byte, 2048)\n\t\tc.mu.Lock()\n\t\tc.queueOutbound([]byte(fmt.Sprintf(\"MSG foo 1 2048\\r\\n%s\\r\\n\", payload)))\n\t\tclosed := c.isClosed()\n\t\tc.mu.Unlock()\n\n\t\trequire_True(t, closed)\n\t\tcheckSub(false)\n\t\tcheckRoutedSub(false)\n\t\tcheckClosed(cid)\n\n\t\tnc.Close()\n\t}\n}\n\nfunc TestRouteSlowConsumerRecover(t *testing.T) {\n\to1 := DefaultOptions()\n\to1.Cluster.PoolSize = -1\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\trtt := 1500 * time.Nanosecond\n\tupRate := 1024 * 1024\n\tdownRate := 128 * 1024\n\tnp := createNetProxy(rtt, upRate, downRate, fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port), true)\n\tdefer np.stop()\n\n\to2 := DefaultOptions()\n\to2.Cluster.PoolSize = -1\n\to2.Routes = RoutesFromStr(np.routeURL())\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tchangeWriteDeadline := func(s *Server, duration time.Duration) {\n\t\ts.mu.Lock()\n\t\tfor _, cl := range s.routes {\n\t\t\tfor _, c := range cl {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tc.out.wdl = duration\n\t\t\t\tc.mu.Unlock()\n\t\t\t}\n\t\t}\n\t\ts.mu.Unlock()\n\t}\n\thasSlowConsumerRoutes := func(s *Server) bool {\n\t\tvar sc bool\n\t\ts.mu.Lock()\n\tLoop:\n\t\tfor _, cl := range s.routes {\n\t\t\tfor _, c := range cl {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tsc = c.flags.isSet(isSlowConsumer)\n\t\t\t\tc.mu.Unlock()\n\t\t\t\tif sc {\n\t\t\t\t\tbreak Loop\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ts.mu.Unlock()\n\t\treturn sc\n\t}\n\n\t// Start with a shorter write deadline to cause errors\n\t// then bump it again later to let it recover.\n\tchangeWriteDeadline(s1, 1*time.Second)\n\n\tncA, err := nats.Connect(s1.Addr().String())\n\trequire_NoError(t, err)\n\tdefer ncA.Close()\n\n\tncB, err := nats.Connect(s2.Addr().String())\n\trequire_NoError(t, err)\n\tdefer ncB.Close()\n\n\tvar wg sync.WaitGroup\n\tncB.Subscribe(\"test\", func(*nats.Msg) {\n\t\tncB.Close()\n\t})\n\tncB.Flush()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 800*time.Millisecond)\n\tdefer cancel()\n\n\tgo func() {\n\t\tvar total int\n\t\tpayload := bytes.Repeat([]byte(\"A\"), 132*1024)\n\t\tfor range time.NewTicker(30 * time.Millisecond).C {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tncA.Publish(\"test\", payload)\n\t\t\tncA.Flush()\n\t\t\ttotal++\n\t\t}\n\t}()\n\twg.Add(1)\n\n\tcheckFor(t, 20*time.Second, 2*time.Millisecond, func() error {\n\t\tif s1.NumRoutes() < 1 {\n\t\t\treturn fmt.Errorf(\"No routes connected\")\n\t\t}\n\t\tif !hasSlowConsumerRoutes(s1) {\n\t\t\tif s1.NumSlowConsumersRoutes() > 0 {\n\t\t\t\t// In case it has recovered already.\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Expected Slow Consumer routes\")\n\t\t}\n\t\treturn nil\n\t})\n\tcancel()\n\tchangeWriteDeadline(s1, 5*time.Second)\n\tnp.updateRTT(0)\n\tcheckFor(t, 20*time.Second, 10*time.Millisecond, func() error {\n\t\tif s1.NumRoutes() < 1 {\n\t\t\treturn fmt.Errorf(\"No routes connected\")\n\t\t}\n\t\tif hasSlowConsumerRoutes(s1) {\n\t\t\treturn fmt.Errorf(\"Expected Slow Consumer routes to recover\")\n\t\t}\n\t\treturn nil\n\t})\n\n\tcheckFor(t, 20*time.Second, 100*time.Millisecond, func() error {\n\t\tvar got, expected int64\n\t\tgot = int64(s1.NumSlowConsumersRoutes())\n\t\texpected = 1\n\t\tif got != expected {\n\t\t\treturn fmt.Errorf(\"got: %d, expected: %d\", got, expected)\n\t\t}\n\t\treturn nil\n\t})\n\twg.Wait()\n}\n\nfunc TestRouteNoLeakOnAuthTimeout(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Cluster.Username = \"foo\"\n\topts.Cluster.Password = \"bar\"\n\topts.AuthTimeout = 0.01 // Deliberately short timeout\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tc, err := net.Dial(\"tcp\", net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Cluster.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer c.Close()\n\n\tcr := bufio.NewReader(c)\n\n\t// Wait for INFO...\n\tline, _, _ := cr.ReadLine()\n\tvar info serverInfo\n\tif err = json.Unmarshal(line[5:], &info); err != nil {\n\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t}\n\n\t// Wait out the clock so we hit the auth timeout\n\ttime.Sleep(secondsToDuration(opts.AuthTimeout) * 2)\n\tline, _, _ = cr.ReadLine()\n\tif string(line) != \"-ERR 'Authentication Timeout'\" {\n\t\tt.Fatalf(\"Expected '-ERR 'Authentication Timeout'' but got %q\", line)\n\t}\n\n\t// There shouldn't be a route entry as we didn't set up.\n\tif nc := s.NumRoutes(); nc != 0 {\n\t\tt.Fatalf(\"Server should have no route connections, got %v\", nc)\n\t}\n}\n\nfunc TestRouteNoRaceOnClusterNameNegotiation(t *testing.T) {\n\t// Running the test 5 times was consistently producing the race.\n\tfor i := 0; i < 5; i++ {\n\t\to1 := DefaultOptions()\n\t\t// Set this cluster name as dynamic\n\t\to1.Cluster.Name = _EMPTY_\n\t\t// Increase number of routes and pinned accounts to increase\n\t\t// the number of processRouteConnect() happening in parallel\n\t\t// to produce the race.\n\t\to1.Cluster.PoolSize = 5\n\t\to1.Cluster.PinnedAccounts = []string{\"A\", \"B\", \"C\", \"D\"}\n\t\to1.Accounts = []*Account{NewAccount(\"A\"), NewAccount(\"B\"), NewAccount(\"C\"), NewAccount(\"D\")}\n\t\ts1 := RunServer(o1)\n\t\tdefer s1.Shutdown()\n\n\t\troute := RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\n\t\to2 := DefaultOptions()\n\t\t// Set this one as explicit. Use name that is likely to be \"higher\"\n\t\t// than the dynamic one.\n\t\to2.Cluster.Name = \"ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ\"\n\t\to2.Cluster.PoolSize = 5\n\t\to2.Cluster.PinnedAccounts = []string{\"A\", \"B\", \"C\", \"D\"}\n\t\to2.Accounts = []*Account{NewAccount(\"A\"), NewAccount(\"B\"), NewAccount(\"C\"), NewAccount(\"D\")}\n\t\to2.Routes = route\n\t\ts2 := RunServer(o2)\n\t\tdefer s2.Shutdown()\n\t\tcheckClusterFormed(t, s1, s2)\n\t\ts2.Shutdown()\n\t\ts1.Shutdown()\n\t}\n}\n\nfunc TestRouteImplicitNotTooManyDuplicates(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpooling bool\n\t\tcompression bool\n\t}{\n\t\t{\"no pooling-no compression\", false, false},\n\t\t{\"no pooling-compression\", false, true},\n\t\t{\"pooling-no compression\", true, false},\n\t\t{\"pooling-compression\", true, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := DefaultOptions()\n\t\t\to.ServerName = \"SEED\"\n\t\t\tif !test.pooling {\n\t\t\t\to.Cluster.PoolSize = -1\n\t\t\t}\n\t\t\tif !test.compression {\n\t\t\t\to.Cluster.Compression.Mode = CompressionOff\n\t\t\t}\n\t\t\tseed := RunServer(o)\n\t\t\tdefer seed.Shutdown()\n\n\t\t\tdl := &testDuplicateRouteLogger{}\n\n\t\t\tservers := make([]*Server, 0, 10)\n\t\t\tfor i := 0; i < cap(servers); i++ {\n\t\t\t\tio := DefaultOptions()\n\t\t\t\tio.ServerName = fmt.Sprintf(\"IMPLICIT_%d\", i+1)\n\t\t\t\tio.NoLog = false\n\t\t\t\tio.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o.Cluster.Port))\n\t\t\t\tif !test.pooling {\n\t\t\t\t\tio.Cluster.PoolSize = -1\n\t\t\t\t}\n\t\t\t\tif !test.compression {\n\t\t\t\t\tio.Cluster.Compression.Mode = CompressionOff\n\t\t\t\t}\n\t\t\t\tis, err := NewServer(io)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\t// Will do defer of shutdown later.\n\t\t\t\tis.SetLogger(dl, true, false)\n\t\t\t\tis.Start()\n\t\t\t\tservers = append(servers, is)\n\t\t\t}\n\n\t\t\tallServers := make([]*Server, 0, len(servers)+1)\n\t\t\tallServers = append(allServers, seed)\n\t\t\tallServers = append(allServers, servers...)\n\n\t\t\t// Let's make sure that we wait for each server to be ready.\n\t\t\tfor _, s := range allServers {\n\t\t\t\tif !s.ReadyForConnections(2 * time.Second) {\n\t\t\t\t\tt.Fatalf(\"Server %q is not ready for connections\", s)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Do the defer of shutdown of all servers this way instead of individual\n\t\t\t// defers when starting them. It takes less time for the servers to shutdown\n\t\t\t// this way.\n\t\t\tdefer func() {\n\t\t\t\tfor _, s := range allServers {\n\t\t\t\t\ts.Shutdown()\n\t\t\t\t}\n\t\t\t}()\n\t\t\tcheckClusterFormed(t, allServers...)\n\n\t\t\tdl.Mutex.Lock()\n\t\t\tcount := dl.count\n\t\t\tdl.Mutex.Unlock()\n\t\t\t// Getting duplicates should not be considered fatal, it is an optimization\n\t\t\t// to reduce the occurrences of those. But to make sure we don't have a\n\t\t\t// regression, we will fail the test if we get say more than 20 or so (\n\t\t\t// without the code change, we would get more than 500 of duplicates).\n\t\t\tif count > 20 {\n\t\t\t\tt.Fatalf(\"Got more duplicates than anticipated: %v\", count)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRouteImplicitJoinsSeparateGroups(t *testing.T) {\n\t// The test TestRouteImplicitNotTooManyDuplicates makes sure that we do\n\t// not have too many duplicate routes cases when processing implicit routes.\n\t// This test is to ensure that the code changes to reduce the number\n\t// of duplicate routes does not prevent the formation of the cluster\n\t// with the given setup (which is admittedly not good since a disconnect\n\t// between some routes would not result in a reconnect leading to a full mesh).\n\t// Still, original code was able to create the original full mesh, so we want\n\t// to make sure that this is still possible.\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpooling bool\n\t\tcompression bool\n\t}{\n\t\t{\"no pooling-no compression\", false, false},\n\t\t{\"no pooling-compression\", false, true},\n\t\t{\"pooling-no compression\", true, false},\n\t\t{\"pooling-compression\", true, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsetOpts := func(o *Options) {\n\t\t\t\tif !test.pooling {\n\t\t\t\t\to.Cluster.PoolSize = -1\n\t\t\t\t}\n\t\t\t\tif !test.compression {\n\t\t\t\t\to.Cluster.Compression.Mode = CompressionOff\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Create a cluster s1/s2/s3\n\t\t\to1 := DefaultOptions()\n\t\t\to1.ServerName = \"S1\"\n\t\t\tsetOpts(o1)\n\t\t\ts1 := RunServer(o1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\to2 := DefaultOptions()\n\t\t\to2.ServerName = \"S2\"\n\t\t\tsetOpts(o2)\n\t\t\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\t\t\ts2 := RunServer(o2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\ttmpl := `\n\t\t\t\tserver_name: \"S3\"\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\tcluster {\n\t\t\t\t\tname: \"abc\"\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\t%s\n\t\t\t\t\t%s\n\t\t\t\t\troutes: [\"nats://127.0.0.1:%d\"%s]\n\t\t\t\t}\n\t\t\t`\n\t\t\tvar poolCfg string\n\t\t\tvar compressionCfg string\n\t\t\tif !test.pooling {\n\t\t\t\tpoolCfg = \"pool_size: -1\"\n\t\t\t}\n\t\t\tif !test.compression {\n\t\t\t\tcompressionCfg = \"compression: off\"\n\t\t\t}\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, poolCfg, compressionCfg, o1.Cluster.Port, _EMPTY_)))\n\t\t\ts3, _ := RunServerWithConfig(conf)\n\t\t\tdefer s3.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2, s3)\n\n\t\t\t// Now s4 and s5 connected to each other, but not linked to s1/s2/s3\n\t\t\to4 := DefaultOptions()\n\t\t\to4.ServerName = \"S4\"\n\t\t\tsetOpts(o4)\n\t\t\ts4 := RunServer(o4)\n\t\t\tdefer s4.Shutdown()\n\n\t\t\to5 := DefaultOptions()\n\t\t\to5.ServerName = \"S5\"\n\t\t\tsetOpts(o5)\n\t\t\to5.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o4.Cluster.Port))\n\t\t\ts5 := RunServer(o5)\n\t\t\tdefer s5.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s4, s5)\n\n\t\t\t// Now add a route from s3 to s4 and make sure that we have a full mesh.\n\t\t\trouteToS4 := fmt.Sprintf(`, \"nats://127.0.0.1:%d\"`, o4.Cluster.Port)\n\t\t\treloadUpdateConfig(t, s3, conf, fmt.Sprintf(tmpl, poolCfg, compressionCfg, o1.Cluster.Port, routeToS4))\n\n\t\t\tcheckClusterFormed(t, s1, s2, s3, s4, s5)\n\t\t})\n\t}\n}\n\nfunc TestRouteConfigureWriteDeadline(t *testing.T) {\n\to1, o2 := DefaultOptions(), DefaultOptions()\n\n\to1.Cluster.WriteDeadline = 5 * time.Second\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\to2.Cluster.WriteDeadline = 6 * time.Second\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\ts1.mu.RLock()\n\ts2.mu.RLock()\n\tdefer s1.mu.RUnlock()\n\tdefer s2.mu.RUnlock()\n\n\ts1.forEachRoute(func(r *client) {\n\t\trequire_Equal(t, r.out.wdl, 5*time.Second)\n\t})\n\n\ts2.forEachRoute(func(r *client) {\n\t\trequire_Equal(t, r.out.wdl, 6*time.Second)\n\t})\n}\n\nfunc TestRouteConfigureWriteTimeoutPolicy(t *testing.T) {\n\tfor name, policy := range map[string]WriteTimeoutPolicy{\n\t\t\"Default\": WriteTimeoutPolicyDefault,\n\t\t\"Retry\": WriteTimeoutPolicyRetry,\n\t\t\"Close\": WriteTimeoutPolicyClose,\n\t} {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\to1 := testDefaultOptionsForGateway(\"B\")\n\t\t\to1.Gateway.WriteTimeout = policy\n\t\t\ts1 := runGatewayServer(o1)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\to2 := testGatewayOptionsFromToWithServers(t, \"A\", \"B\", s1)\n\t\t\ts2 := runGatewayServer(o2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, s2, 1, time.Second)\n\t\t\twaitForInboundGateways(t, s1, 1, time.Second)\n\t\t\twaitForOutboundGateways(t, s1, 1, time.Second)\n\n\t\t\ts1.mu.RLock()\n\t\t\tdefer s1.mu.RUnlock()\n\n\t\t\ts1.forEachRoute(func(r *client) {\n\t\t\t\tif policy == WriteTimeoutPolicyDefault {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, WriteTimeoutPolicyRetry)\n\t\t\t\t} else {\n\t\t\t\t\trequire_Equal(t, r.out.wtp, policy)\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestRoutePoolFirstPongBlocksChain(t *testing.T) {\n\t// Use a very short ping interval so the timer fires quickly, before we (the mock server) send our INFO.\n\torgMaxPing := routeMaxPingInterval\n\trouteMaxPingInterval = 200 * time.Millisecond\n\tdefer func() { routeMaxPingInterval = orgMaxPing }()\n\n\t// Start a TCP listener acting as a mock route server. S2 will solicit a connection to this listener.\n\tmockListener, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\trequire_NoError(t, err)\n\tdefer mockListener.Close()\n\n\tmockPort := mockListener.Addr().(*net.TCPAddr).Port\n\n\to2 := DefaultOptions()\n\to2.ServerName = \"S2\"\n\to2.Cluster.Name = \"local\"\n\to2.Cluster.PoolSize = 3\n\to2.Cluster.Compression.Mode = CompressionOff\n\to2.Cluster.MaxPingsOut = 10\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", mockPort))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tvar ready sync.WaitGroup\n\tvar keepAlive sync.WaitGroup\n\tmockListener.(*net.TCPListener).SetDeadline(time.Now().Add(5 * time.Second))\n\thandleConnection := func() {\n\t\tconn, err := mockListener.Accept()\n\t\tif err != nil {\n\t\t\tready.Done()\n\t\t\trequire_NoError(t, err)\n\t\t}\n\t\tdefer conn.Close()\n\t\tbr := bufio.NewReader(conn)\n\n\t\t// Ensure we call these before conn.Close() above.\n\t\t// We have to wait for both connections to be ready and then wait for the test to finish.\n\t\tdefer keepAlive.Wait()\n\t\tdefer ready.Done()\n\n\t\t// S2 sends CONNECT immediately (solicited route).\n\t\t// Read it but do NOT send our INFO yet — we want S2's timer to fire first.\n\t\trequire_NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second)))\n\n\t\t// Read S2's CONNECT line.\n\t\tline, err := br.ReadString('\\n')\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, strings.HasPrefix(line, \"CONNECT\"))\n\n\t\t// Wait for S2's timer PING to arrive.\n\t\t// The readLoop on S2 is blocked waiting for our INFO, so the\n\t\t// timer goroutine enqueues the PING on S2's outbound buffer.\n\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\tif _, err = conn.Write([]byte(\"PING\\r\\n\")); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif line, err = br.ReadString('\\n'); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tt.Logf(\"S2 received: %q\", line)\n\t\t\tif !strings.HasPrefix(line, \"PING\") {\n\t\t\t\treturn fmt.Errorf(\"expected PING, got %q\", line)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\n\t\t// Respond with PONG — this sets firstPong on S2's route connection\n\t\t// BEFORE addRoute has run (since we haven't sent INFO yet).\n\t\t_, err = conn.Write([]byte(\"PONG\\r\\n\"))\n\t\trequire_NoError(t, err)\n\n\t\t// NOW send our INFO. This triggers S2's processRouteInfo → addRoute,\n\t\t// which sets startNewRoute and sends another PING.\n\t\tmockInfo := Info{\n\t\t\tID: \"MOCK_SERVER_ID\",\n\t\t\tName: \"mock-server\",\n\t\t\tHost: \"127.0.0.1\",\n\t\t\tPort: mockPort,\n\t\t\tCluster: \"local\",\n\t\t\tHeaders: true,\n\t\t\tProto: 1,\n\t\t\tRoutePoolSize: 3,\n\t\t}\n\t\tinfoJSON, err := json.Marshal(mockInfo)\n\t\trequire_NoError(t, err)\n\t\t_, err = fmt.Fprintf(conn, \"INFO %s\\r\\n\", infoJSON)\n\t\trequire_NoError(t, err)\n\n\t\t// Read S2's delayed INFO (sent during addRoute) + subscription data + PING.\n\t\t// We need to consume everything up to and including the PING.\n\t\tfor {\n\t\t\tline, err = br.ReadString('\\n')\n\t\t\trequire_NoError(t, err)\n\t\t\tif strings.HasPrefix(line, \"PING\") {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// Respond to addRoute's PING with PONG.\n\t\t_, err = conn.Write([]byte(\"PONG\\r\\n\"))\n\t\trequire_NoError(t, err)\n\t}\n\n\t// Keep the below connections alive until the test is done.\n\t// This allows us to check that S2 creates the next pool connection.\n\tkeepAlive.Add(1)\n\tdefer keepAlive.Done()\n\n\tready.Add(2)\n\tfor range 2 {\n\t\tgo handleConnection()\n\t}\n\tready.Wait()\n\n\t// Check if S2 attempts to create a second pool connection by\n\t// trying to accept another connection on our mock listener.\n\tsecondConnCh := make(chan struct{}, 1)\n\tgo func() {\n\t\tmockListener.(*net.TCPListener).SetDeadline(time.Now().Add(2 * time.Second))\n\t\tif c2, err := mockListener.Accept(); err == nil {\n\t\t\tc2.Close()\n\t\t\tsecondConnCh <- struct{}{}\n\t\t}\n\t}()\n\n\tselect {\n\tcase <-secondConnCh:\n\t\t// Good — S2 tried to create the next pool connection.\n\t\t// The fix works: startNewRoute was consumed despite firstPong.\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"S2 did not attempt to create next pool connection; \" +\n\t\t\t\"firstPong blocked startNewRoute consumption, pool chain is broken\")\n\t}\n}\n\n// Benchmarks for message arg processing functions to measure heap allocations.\n// These functions parse incoming protocol messages and split arguments.\n\nfunc BenchmarkProcessRoutedMsgArgs(b *testing.B) {\n\t// RMSG format: account subject [reply] size\n\targ := []byte(\"$G foo.bar _INBOX.xxx 1024\")\n\tc := &client{kind: ROUTER, route: &route{}}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processRoutedMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessRoutedHeaderMsgArgs(b *testing.B) {\n\t// HMSG format: account subject [reply] headerSize totalSize\n\targ := []byte(\"$G foo.bar 12 1024\")\n\tc := &client{kind: ROUTER, route: &route{}}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processRoutedHeaderMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessRoutedOriginClusterMsgArgs(b *testing.B) {\n\t// Origin cluster HMSG format: origin account subject [reply] headerSize totalSize\n\targ := []byte(\"ORIGIN MY_ACCOUNT foo.bar 12 345\")\n\tc := &client{kind: ROUTER, route: &route{}}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processRoutedOriginClusterMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessLeafMsgArgs(b *testing.B) {\n\t// LMSG format: subject [reply] size\n\targ := []byte(\"foo.bar _INBOX.xxx 1024\")\n\tc := &client{kind: LEAF}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processLeafMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessLeafHeaderMsgArgs(b *testing.B) {\n\t// Leaf HMSG format: subject headerSize totalSize\n\targ := []byte(\"foo.bar 12 1024\")\n\tc := &client{kind: LEAF}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processLeafHeaderMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\n// Benchmarks with queue subscribers to exercise the larger arg paths.\n\nfunc BenchmarkProcessRoutedMsgArgs_Queues(b *testing.B) {\n\t// RMSG format with queues: account subject replyIndicator reply queue1 queue2 size\n\targ := []byte(\"$G foo.bar + _INBOX.xxx queue1 queue2 1024\")\n\tc := &client{kind: ROUTER, route: &route{}}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processRoutedMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessRoutedHeaderMsgArgs_Queues(b *testing.B) {\n\t// HMSG format with queues: account subject replyIndicator reply queue1 queue2 headerSize totalSize\n\targ := []byte(\"$G foo.bar + _INBOX.xxx queue1 queue2 12 1024\")\n\tc := &client{kind: ROUTER, route: &route{}}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processRoutedHeaderMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkProcessLeafMsgArgs_Queues(b *testing.B) {\n\t// LMSG format with queues: subject replyIndicator reply queue1 queue2 size\n\targ := []byte(\"foo.bar + _INBOX.xxx queue1 queue2 1024\")\n\tc := &client{kind: LEAF}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor b.Loop() {\n\t\tif err := c.processLeafMsgArgs(arg); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n" }, { "path": "server/scheduler.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"io\"\n\t\"math\"\n\t\"slices\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/thw\"\n)\n\n// Error for when we try to decode a binary-encoded message schedule with an unknown version number.\nvar ErrMsgScheduleInvalidVersion = errors.New(\"msg scheduling: encoded version not known\")\n\nconst (\n\theaderLen = 17 // 1 byte magic + 2x uint64s\n)\n\ntype MsgScheduling struct {\n\trun func()\n\tttls *thw.HashWheel\n\ttimer *time.Timer\n\trunning bool\n\tdeadline int64\n\tschedules map[string]*MsgSchedule\n\tseqToSubj map[uint64]string\n\tinflight map[string]struct{}\n}\n\ntype MsgSchedule struct {\n\tseq uint64\n\tts int64\n}\n\nfunc newMsgScheduling(run func()) *MsgScheduling {\n\treturn &MsgScheduling{\n\t\trun: run,\n\t\tttls: thw.NewHashWheel(),\n\t\tschedules: make(map[string]*MsgSchedule),\n\t\tseqToSubj: make(map[uint64]string),\n\t\tinflight: make(map[string]struct{}),\n\t}\n}\n\nfunc (ms *MsgScheduling) add(seq uint64, subj string, ts int64) {\n\tms.init(seq, subj, ts)\n\tms.resetTimer()\n}\n\nfunc (ms *MsgScheduling) init(seq uint64, subj string, ts int64) {\n\tif sched, ok := ms.schedules[subj]; ok {\n\t\tdelete(ms.seqToSubj, sched.seq)\n\t\t// Remove and add separately, since they'll have different sequences.\n\t\tms.ttls.Remove(sched.seq, sched.ts)\n\t\tms.ttls.Add(seq, ts)\n\t\tsched.ts, sched.seq = ts, seq\n\t} else {\n\t\tms.ttls.Add(seq, ts)\n\t\tms.schedules[subj] = &MsgSchedule{seq: seq, ts: ts}\n\t}\n\tms.seqToSubj[seq] = subj\n\tdelete(ms.inflight, subj)\n}\n\nfunc (ms *MsgScheduling) update(subj string, ts int64) {\n\tif sched, ok := ms.schedules[subj]; ok {\n\t\t// Remove and add separately, it's for the same sequence, but if replicated\n\t\t// this server could not know the previous timestamp yet.\n\t\tms.ttls.Remove(sched.seq, sched.ts)\n\t\tms.ttls.Add(sched.seq, ts)\n\t\tsched.ts = ts\n\t\tdelete(ms.inflight, subj)\n\t\tms.resetTimer()\n\t}\n}\n\nfunc (ms *MsgScheduling) markInflight(subj string) {\n\tif _, ok := ms.schedules[subj]; ok {\n\t\tms.inflight[subj] = struct{}{}\n\t}\n}\n\nfunc (ms *MsgScheduling) isInflight(subj string) bool {\n\t_, ok := ms.inflight[subj]\n\treturn ok\n}\n\nfunc (ms *MsgScheduling) remove(seq uint64) {\n\tif subj, ok := ms.seqToSubj[seq]; ok {\n\t\tdelete(ms.seqToSubj, seq)\n\t\tdelete(ms.schedules, subj)\n\t}\n}\n\nfunc (ms *MsgScheduling) removeSubject(subj string) {\n\tif sched, ok := ms.schedules[subj]; ok {\n\t\tms.ttls.Remove(sched.seq, sched.ts)\n\t\tdelete(ms.schedules, subj)\n\t\tdelete(ms.seqToSubj, sched.seq)\n\t}\n}\n\nfunc (ms *MsgScheduling) clearInflight() {\n\tms.inflight = make(map[string]struct{})\n}\n\nfunc (ms *MsgScheduling) resetTimer() {\n\t// If we're already scheduling messages, it will make sure to reset.\n\t// Don't trigger again, as that could result in many expire goroutines.\n\tif ms.running {\n\t\treturn\n\t}\n\n\tnext := ms.ttls.GetNextExpiration(math.MaxInt64)\n\tif next == math.MaxInt64 {\n\t\tclearTimer(&ms.timer)\n\t\treturn\n\t}\n\tfireIn := time.Until(time.Unix(0, next))\n\n\t// Make sure we aren't firing too often either way, otherwise we can\n\t// negatively impact stream ingest performance.\n\tif fireIn < 250*time.Millisecond {\n\t\tfireIn = 250 * time.Millisecond\n\t}\n\n\t// If we want to kick the timer to run later than what was assigned before, don't reset it.\n\t// Otherwise, we could get in a situation where the timer is continuously reset, and it never runs.\n\tdeadline := time.Now().UnixNano() + fireIn.Nanoseconds()\n\tif ms.deadline > 0 && deadline > ms.deadline {\n\t\treturn\n\t}\n\n\tms.deadline = deadline\n\tif ms.timer != nil {\n\t\tms.timer.Reset(fireIn)\n\t} else {\n\t\tms.timer = time.AfterFunc(fireIn, ms.run)\n\t}\n}\n\nfunc (ms *MsgScheduling) getScheduledMessages(loadMsg func(seq uint64, smv *StoreMsg) *StoreMsg, loadLast func(subj string, smv *StoreMsg) *StoreMsg) []*inMsg {\n\tvar (\n\t\tsmv StoreMsg\n\t\tsm *StoreMsg\n\t\tmsgs []*inMsg\n\t)\n\tms.ttls.ExpireTasks(func(seq uint64, ts int64) bool {\n\t\t// Need to grab the message for the specified sequence, and check\n\t\t// if it hasn't been removed in the meantime.\n\t\tsm = loadMsg(seq, &smv)\n\t\tif sm != nil {\n\t\t\t// If already inflight, don't duplicate a scheduled message. The stream could\n\t\t\t// be replicated and the scheduled message could take some time to propagate.\n\t\t\tsubj := sm.subj\n\t\t\tif ms.isInflight(subj) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\t// Validate the contents are correct if not, we just remove it from THW.\n\t\t\tpattern := bytesToString(sliceHeader(JSSchedulePattern, sm.hdr))\n\t\t\tif pattern == _EMPTY_ {\n\t\t\t\tms.remove(seq)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\ttz := bytesToString(sliceHeader(JSScheduleTimeZone, sm.hdr))\n\t\t\tnext, repeat, ok := parseMsgSchedule(pattern, tz, ts)\n\t\t\tif !ok {\n\t\t\t\tms.remove(seq)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tttl, ok := getMessageScheduleTTL(sm.hdr)\n\t\t\tif !ok {\n\t\t\t\tms.remove(seq)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\ttarget := getMessageScheduleTarget(sm.hdr)\n\t\t\tif target == _EMPTY_ {\n\t\t\t\tms.remove(seq)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tsource := getMessageScheduleSource(sm.hdr)\n\t\t\tif source != _EMPTY_ {\n\t\t\t\tif sm = loadLast(source, &smv); sm == nil {\n\t\t\t\t\tms.remove(seq)\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Copy, as this is retrieved directly from storage, and we'll need to keep hold of this for some time.\n\t\t\t// And in the case of headers, we'll copy all of them, but make changes.\n\t\t\thdr, msg := copyBytes(sm.hdr), copyBytes(sm.msg)\n\n\t\t\t// Strip headers specific to the schedule.\n\t\t\thdr = removeHeaderIfPresent(hdr, JSSchedulePattern)\n\t\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Schedule-\")\n\t\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Expected-\")\n\t\t\thdr = removeHeaderIfPresent(hdr, JSMsgId)\n\t\t\thdr = removeHeaderIfPresent(hdr, JSMessageTTL)\n\t\t\thdr = removeHeaderIfPresent(hdr, JSMsgRollup)\n\n\t\t\t// Add headers for the scheduled message.\n\t\t\thdr = genHeader(hdr, JSScheduler, subj)\n\t\t\tif !repeat {\n\t\t\t\thdr = genHeader(hdr, JSScheduleNext, JSScheduleNextPurge) // Purge the schedule message itself.\n\t\t\t} else {\n\t\t\t\thdr = genHeader(hdr, JSScheduleNext, next.Format(time.RFC3339)) // Next time the schedule fires.\n\t\t\t}\n\t\t\tif ttl != _EMPTY_ {\n\t\t\t\thdr = genHeader(hdr, JSMessageTTL, ttl)\n\t\t\t}\n\t\t\tmsgs = append(msgs, &inMsg{seq: seq, subj: target, hdr: hdr, msg: msg})\n\t\t\tms.markInflight(subj)\n\t\t\treturn false\n\t\t}\n\t\tms.remove(seq)\n\t\treturn true\n\t})\n\t// THW is unordered, so must sort by sequence.\n\tslices.SortFunc(msgs, func(a, b *inMsg) int {\n\t\tif a.seq == b.seq {\n\t\t\treturn 0\n\t\t} else if a.seq < b.seq {\n\t\t\treturn -1\n\t\t} else {\n\t\t\treturn 1\n\t\t}\n\t})\n\treturn msgs\n}\n\n// encode writes out the contents of the schedule into a binary snapshot\n// and returns it. The high seq number is included in the snapshot and will\n// be returned on decode.\nfunc (ms *MsgScheduling) encode(highSeq uint64) []byte {\n\tcount := uint64(len(ms.schedules))\n\tb := make([]byte, 0, headerLen+(count*(2*binary.MaxVarintLen64)))\n\tb = append(b, 1) // Magic version\n\tb = binary.LittleEndian.AppendUint64(b, count) // Entry count\n\tb = binary.LittleEndian.AppendUint64(b, highSeq) // Stamp\n\tfor subj, sched := range ms.schedules {\n\t\tslen := min(uint64(len(subj)), math.MaxUint16)\n\t\tb = binary.LittleEndian.AppendUint16(b, uint16(slen))\n\t\tb = append(b, subj[:slen]...)\n\t\tb = binary.AppendVarint(b, sched.ts)\n\t\tb = binary.AppendUvarint(b, sched.seq)\n\t}\n\treturn b\n}\n\n// decode snapshots a binary-encoded schedule and replaces the contents of this\n// schedule with them. Returns the high seq number from the snapshot.\nfunc (ms *MsgScheduling) decode(b []byte) (uint64, error) {\n\tif len(b) < headerLen {\n\t\treturn 0, io.ErrShortBuffer\n\t}\n\tif b[0] != 1 {\n\t\treturn 0, ErrMsgScheduleInvalidVersion\n\t}\n\n\tcount := binary.LittleEndian.Uint64(b[1:])\n\tstamp := binary.LittleEndian.Uint64(b[9:])\n\tb = b[headerLen:]\n\tfor i := uint64(0); i < count; i++ {\n\t\tsl := int(binary.LittleEndian.Uint16(b))\n\t\tb = b[2:]\n\t\tif len(b) < sl {\n\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t}\n\t\tsubj := string(b[:sl])\n\t\tb = b[sl:]\n\t\tts, tn := binary.Varint(b)\n\t\tif tn < 0 {\n\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t}\n\t\tseq, vn := binary.Uvarint(b[tn:])\n\t\tif vn < 0 {\n\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t}\n\t\tms.init(seq, subj, ts)\n\t\tb = b[tn+vn:]\n\t}\n\treturn stamp, nil\n}\n\n// parseMsgSchedule parses a message schedule pattern and returns the time\n// to fire, whether it is a repeating schedule, and whether the pattern was valid.\nfunc parseMsgSchedule(pattern string, tz string, ts int64) (time.Time, bool, bool) {\n\tif pattern == _EMPTY_ {\n\t\treturn time.Time{}, false, true\n\t}\n\t// Exact time.\n\tif strings.HasPrefix(pattern, \"@at \") {\n\t\t// Time zone is not supported for @at.\n\t\tif tz != _EMPTY_ {\n\t\t\treturn time.Time{}, false, false\n\t\t}\n\t\tt, err := time.Parse(time.RFC3339, pattern[4:])\n\t\treturn t, false, err == nil\n\t}\n\t// Repeating on a simple interval.\n\tif strings.HasPrefix(pattern, \"@every \") {\n\t\t// Time zone is not supported for @every.\n\t\tif tz != _EMPTY_ {\n\t\t\treturn time.Time{}, false, false\n\t\t}\n\t\tdur, err := time.ParseDuration(pattern[7:])\n\t\tif err != nil {\n\t\t\treturn time.Time{}, false, false\n\t\t}\n\t\t// Only allow intervals of at least a second.\n\t\tif dur.Seconds() < 1 {\n\t\t\treturn time.Time{}, false, false\n\t\t}\n\t\t// If this schedule would trigger multiple times, for example after a restart, skip ahead and only fire once.\n\t\tnext := time.Unix(0, ts).UTC().Round(time.Second).Add(dur)\n\t\tif now := time.Now().UTC(); next.Before(now) {\n\t\t\tnext = now.Round(time.Second).Add(dur)\n\t\t}\n\t\treturn next, true, true\n\t}\n\n\t// Predefined schedules for cron.\n\tswitch pattern {\n\tcase \"@yearly\", \"@annually\":\n\t\tpattern = \"0 0 0 1 1 *\"\n\tcase \"@monthly\":\n\t\tpattern = \"0 0 0 1 * *\"\n\tcase \"@weekly\":\n\t\tpattern = \"0 0 0 * * 0\"\n\tcase \"@daily\", \"@midnight\":\n\t\tpattern = \"0 0 0 * * *\"\n\tcase \"@hourly\":\n\t\tpattern = \"0 0 * * * *\"\n\t}\n\n\t// Parse the cron pattern.\n\tnext, err := parseCron(pattern, tz, ts)\n\tif err != nil {\n\t\treturn time.Time{}, false, false\n\t}\n\t// If this schedule would trigger multiple times, for example after a restart, skip ahead and only fire once.\n\tif now := time.Now().UTC(); next.Before(now) {\n\t\tts = now.Round(time.Second).UnixNano()\n\t\tnext, err = parseCron(pattern, tz, ts)\n\t\tif err != nil {\n\t\t\treturn time.Time{}, false, false\n\t\t}\n\t}\n\treturn next, true, true\n}\n" }, { "path": "server/sdm.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"time\"\n)\n\n// SDMMeta holds pending/proposed data for subject delete markers or message removals.\ntype SDMMeta struct {\n\ttotals map[string]uint64\n\tpending map[uint64]SDMBySeq\n}\n\n// SDMBySeq holds data for a message with a specific sequence.\ntype SDMBySeq struct {\n\tlast bool // Whether the message for this sequence was the last for this subject.\n\tts int64 // Last timestamp we proposed a removal/sdm.\n}\n\nfunc newSDMMeta() *SDMMeta {\n\treturn &SDMMeta{\n\t\ttotals: make(map[string]uint64, 1),\n\t\tpending: make(map[uint64]SDMBySeq, 1),\n\t}\n}\n\n// isSubjectDeleteMarker returns whether the headers indicate this message is a subject delete marker.\n// Either it's a usual marker with JSMarkerReason, or it's a KV Purge marker as the KVOperation.\nfunc isSubjectDeleteMarker(hdr []byte) bool {\n\treturn len(sliceHeader(JSMarkerReason, hdr)) != 0 || bytes.Equal(sliceHeader(KVOperation, hdr), KVOperationValuePurge)\n}\n\n// empty clears all data.\nfunc (sdm *SDMMeta) empty() {\n\tif sdm == nil {\n\t\treturn\n\t}\n\tclear(sdm.totals)\n\tclear(sdm.pending)\n}\n\n// trackPending caches the given seq and subj and whether it's the last message for that subject.\nfunc (sdm *SDMMeta) trackPending(seq uint64, subj string, last bool) bool {\n\tif p, ok := sdm.pending[seq]; ok {\n\t\treturn p.last\n\t}\n\tsdm.pending[seq] = SDMBySeq{last, time.Now().UnixNano()}\n\tsdm.totals[subj]++\n\treturn last\n}\n\n// removeSeqAndSubject clears the seq and subj from the cache.\nfunc (sdm *SDMMeta) removeSeqAndSubject(seq uint64, subj string) {\n\tif sdm == nil {\n\t\treturn\n\t}\n\tif _, ok := sdm.pending[seq]; ok {\n\t\tdelete(sdm.pending, seq)\n\t\tif msgs, ok := sdm.totals[subj]; ok {\n\t\t\tif msgs <= 1 {\n\t\t\t\tdelete(sdm.totals, subj)\n\t\t\t} else {\n\t\t\t\tsdm.totals[subj] = msgs - 1\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "server/sendq.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"strconv\"\n\t\"sync\"\n)\n\ntype outMsg struct {\n\tsubj string\n\trply string\n\thdr []byte\n\tmsg []byte\n}\n\ntype sendq struct {\n\tmu sync.Mutex\n\tq *ipQueue[*outMsg]\n\ts *Server\n\ta *Account\n}\n\nfunc (s *Server) newSendQ(acc *Account) *sendq {\n\tsq := &sendq{s: s, q: newIPQueue[*outMsg](s, \"SendQ\"), a: acc}\n\ts.startGoRoutine(sq.internalLoop)\n\treturn sq\n}\n\nfunc (sq *sendq) internalLoop() {\n\tsq.mu.Lock()\n\ts, q := sq.s, sq.q\n\tsq.mu.Unlock()\n\n\tdefer s.grWG.Done()\n\n\tc := s.createInternalSystemClient()\n\tc.registerWithAccount(sq.a)\n\tc.noIcb = true\n\n\tdefer c.closeConnection(ClientClosed)\n\n\t// To optimize for not converting a string to a []byte slice.\n\tvar (\n\t\tsubj [256]byte\n\t\trply [256]byte\n\t\tszb [10]byte\n\t\thdb [10]byte\n\t\t_msg [4096]byte\n\t\tmsg = _msg[:0]\n\t)\n\n\tfor s.isRunning() {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-q.ch:\n\t\t\tpms := q.pop()\n\t\t\tfor _, pm := range pms {\n\t\t\t\tc.pa.subject = append(subj[:0], pm.subj...)\n\t\t\t\tc.pa.size = len(pm.msg) + len(pm.hdr)\n\t\t\t\tc.pa.szb = append(szb[:0], strconv.Itoa(c.pa.size)...)\n\t\t\t\tif len(pm.rply) > 0 {\n\t\t\t\t\tc.pa.reply = append(rply[:0], pm.rply...)\n\t\t\t\t} else {\n\t\t\t\t\tc.pa.reply = nil\n\t\t\t\t}\n\t\t\t\tmsg = msg[:0]\n\t\t\t\tif len(pm.hdr) > 0 {\n\t\t\t\t\tc.pa.hdr = len(pm.hdr)\n\t\t\t\t\tc.pa.hdb = append(hdb[:0], strconv.Itoa(c.pa.hdr)...)\n\t\t\t\t\tmsg = append(msg, pm.hdr...)\n\t\t\t\t\tmsg = append(msg, pm.msg...)\n\t\t\t\t\tmsg = append(msg, _CRLF_...)\n\t\t\t\t} else {\n\t\t\t\t\tc.pa.hdr = -1\n\t\t\t\t\tc.pa.hdb = nil\n\t\t\t\t\tmsg = append(msg, pm.msg...)\n\t\t\t\t\tmsg = append(msg, _CRLF_...)\n\t\t\t\t}\n\t\t\t\tc.processInboundClientMsg(msg)\n\t\t\t\tc.pa.szb = nil\n\t\t\t\toutMsgPool.Put(pm)\n\t\t\t}\n\t\t\t// TODO: should this be in the for-loop instead?\n\t\t\tc.flushClients(0)\n\t\t\tq.recycle(&pms)\n\t\t}\n\t}\n}\n\nvar outMsgPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &outMsg{}\n\t},\n}\n\nfunc (sq *sendq) send(subj, rply string, hdr, msg []byte) {\n\tif sq == nil {\n\t\treturn\n\t}\n\tout := outMsgPool.Get().(*outMsg)\n\tout.subj, out.rply = subj, rply\n\tout.hdr = append(out.hdr[:0], hdr...)\n\tout.msg = append(out.msg[:0], msg...)\n\tsq.q.push(out)\n}\n" }, { "path": "server/server.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/fips140\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"runtime/pprof\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t// Allow dynamic profiling.\n\t_ \"net/http/pprof\"\n\n\t\"expvar\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/logger\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nconst (\n\t// Interval for the first PING for non client connections.\n\tfirstPingInterval = time.Second\n\n\t// This is for the first ping for client connections.\n\tfirstClientPingInterval = 2 * time.Second\n)\n\n// These are protocol versions sent between server connections: ROUTER, LEAF and\n// GATEWAY. We may have protocol versions that have a meaning only for a certain\n// type of connections, but we don't have to have separate enums for that.\n// However, it is CRITICAL to not change the order of those constants since they\n// are exchanged between servers. When adding a new protocol version, add to the\n// end of the list, don't try to group them by connection types.\nconst (\n\t// RouteProtoZero is the original Route protocol from 2009.\n\t// http://nats.io/documentation/internals/nats-protocol/\n\tRouteProtoZero = iota\n\t// RouteProtoInfo signals a route can receive more then the original INFO block.\n\t// This can be used to update remote cluster permissions, etc...\n\tRouteProtoInfo\n\t// RouteProtoV2 is the new route/cluster protocol that provides account support.\n\tRouteProtoV2\n\t// MsgTraceProto indicates that this server understands distributed message tracing.\n\tMsgTraceProto\n)\n\n// Will return the latest server-to-server protocol versions, unless the\n// option to override it is set.\nfunc (s *Server) getServerProto() int {\n\topts := s.getOpts()\n\t// Initialize with the latest protocol version.\n\tproto := MsgTraceProto\n\t// For tests, we want to be able to make this server behave\n\t// as an older server so check this option to see if we should override.\n\tif opts.overrideProto < 0 {\n\t\t// The option overrideProto is set to 0 by default (when creating an\n\t\t// Options structure). Since this is the same value than the original\n\t\t// proto RouteProtoZero, tests call setServerProtoForTest() with the\n\t\t// desired protocol level, which sets it as negative value equal to:\n\t\t// (wantedProto + 1) * -1. Here we compute back the real value.\n\t\tproto = (opts.overrideProto * -1) - 1\n\t}\n\treturn proto\n}\n\n// Used by tests.\nfunc setServerProtoForTest(wantedProto int) int {\n\treturn (wantedProto + 1) * -1\n}\n\n// Info is the information sent to clients, routes, gateways, and leaf nodes,\n// to help them understand information about this server.\ntype Info struct {\n\tID string `json:\"server_id\"`\n\tName string `json:\"server_name\"`\n\tVersion string `json:\"version\"`\n\tProto int `json:\"proto\"`\n\tGitCommit string `json:\"git_commit,omitempty\"`\n\tGoVersion string `json:\"go\"`\n\tHost string `json:\"host\"`\n\tPort int `json:\"port\"`\n\tHeaders bool `json:\"headers\"`\n\tAuthRequired bool `json:\"auth_required,omitempty\"`\n\tTLSRequired bool `json:\"tls_required,omitempty\"`\n\tTLSVerify bool `json:\"tls_verify,omitempty\"`\n\tTLSAvailable bool `json:\"tls_available,omitempty\"`\n\tMaxPayload int32 `json:\"max_payload\"`\n\tJetStream bool `json:\"jetstream,omitempty\"`\n\tIP string `json:\"ip,omitempty\"`\n\tCID uint64 `json:\"client_id,omitempty\"`\n\tClientIP string `json:\"client_ip,omitempty\"`\n\tNonce string `json:\"nonce,omitempty\"`\n\tCluster string `json:\"cluster,omitempty\"`\n\tDynamic bool `json:\"cluster_dynamic,omitempty\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tClientConnectURLs []string `json:\"connect_urls,omitempty\"` // Contains URLs a client can connect to.\n\tWSConnectURLs []string `json:\"ws_connect_urls,omitempty\"` // Contains URLs a ws client can connect to.\n\tLameDuckMode bool `json:\"ldm,omitempty\"`\n\tCompression string `json:\"compression,omitempty\"`\n\tConnectInfo bool `json:\"connect_info,omitempty\"` // When true this is the server INFO response to CONNECT\n\tRemoteAccount string `json:\"remote_account,omitempty\"` // Lets the client or leafnode side know the remote account that they bind to.\n\tIsSystemAccount bool `json:\"acc_is_sys,omitempty\"` // Indicates if the account is a system account.\n\tJSApiLevel int `json:\"api_lvl,omitempty\"`\n\n\t// Route Specific\n\tImport *SubjectPermission `json:\"import,omitempty\"`\n\tExport *SubjectPermission `json:\"export,omitempty\"`\n\tLNOC bool `json:\"lnoc,omitempty\"`\n\tLNOCU bool `json:\"lnocu,omitempty\"`\n\tInfoOnConnect bool `json:\"info_on_connect,omitempty\"` // When true the server will respond to CONNECT with an INFO\n\tRoutePoolSize int `json:\"route_pool_size,omitempty\"`\n\tRoutePoolIdx int `json:\"route_pool_idx,omitempty\"`\n\tRouteAccount string `json:\"route_account,omitempty\"`\n\tRouteAccReqID string `json:\"route_acc_add_reqid,omitempty\"`\n\tGossipMode byte `json:\"gossip_mode,omitempty\"`\n\n\t// Gateways Specific\n\tGateway string `json:\"gateway,omitempty\"` // Name of the origin Gateway (sent by gateway's INFO)\n\tGatewayURLs []string `json:\"gateway_urls,omitempty\"` // Gateway URLs in the originating cluster (sent by gateway's INFO)\n\tGatewayURL string `json:\"gateway_url,omitempty\"` // Gateway URL on that server (sent by route's INFO)\n\tGatewayCmd byte `json:\"gateway_cmd,omitempty\"` // Command code for the receiving server to know what to do\n\tGatewayCmdPayload []byte `json:\"gateway_cmd_payload,omitempty\"` // Command payload when needed\n\tGatewayNRP bool `json:\"gateway_nrp,omitempty\"` // Uses new $GNR. prefix for mapped replies\n\tGatewayIOM bool `json:\"gateway_iom,omitempty\"` // Indicate that all accounts will be switched to InterestOnly mode \"right away\"\n\n\t// LeafNode Specific\n\tLeafNodeURLs []string `json:\"leafnode_urls,omitempty\"` // LeafNode URLs that the server can reconnect to.\n\n\tXKey string `json:\"xkey,omitempty\"` // Public server's x25519 key.\n}\n\n// Server is our main struct.\ntype Server struct {\n\t// Fields accessed with atomic operations need to be 64-bit aligned\n\tgcid uint64\n\t// How often user logon fails due to the issuer account not being pinned.\n\tpinnedAccFail uint64\n\tstats\n\tscStats\n\tstaleStats\n\tmu sync.RWMutex\n\treloadMu sync.RWMutex // Write-locked when a config reload is taking place ONLY\n\tkp nkeys.KeyPair\n\txkp nkeys.KeyPair\n\txpub string\n\tinfo Info\n\tconfigFile string\n\toptsMu sync.RWMutex\n\topts *Options\n\trunning atomic.Bool\n\tshutdown atomic.Bool\n\tlistener net.Listener\n\tlistenerErr error\n\tgacc *Account\n\tsys *internal\n\tsysAcc atomic.Pointer[Account]\n\tjs atomic.Pointer[jetStream]\n\tisMetaLeader atomic.Bool\n\tjsClustered atomic.Bool\n\taccounts sync.Map\n\ttmpAccounts sync.Map // Temporarily stores accounts that are being built\n\tactiveAccounts int32\n\taccResolver AccountResolver\n\tclients map[uint64]*client\n\troutes map[string][]*client\n\tremoteRoutePoolSize map[string]int // Map for remote's configure route pool size\n\troutesPoolSize int // Configured pool size\n\troutesReject bool // During reload, we may want to reject adding routes until some conditions are met\n\troutesNoPool int // Number of routes that don't use pooling (connecting to older server for instance)\n\taccRoutes map[string]map[string]*client // Key is account name, value is key=remoteID/value=route connection\n\taccRouteByHash sync.Map // Key is account name, value is nil or a pool index\n\taccAddedCh chan struct{}\n\taccAddedReqID string\n\tleafs map[uint64]*client\n\tusers map[string]*User\n\tnkeys map[string]*NkeyUser\n\ttotalClients uint64\n\tclosed *closedRingBuffer\n\tdone chan bool\n\tstart time.Time\n\thttp net.Listener\n\thttpHandler http.Handler\n\thttpBasePath string\n\tprofiler net.Listener\n\thttpReqStats map[string]uint64\n\trouteListener net.Listener\n\trouteListenerErr error\n\trouteInfo Info\n\trouteResolver netResolver\n\troutesToSelf map[string]struct{}\n\trouteTLSName string\n\tleafNodeListener net.Listener\n\tleafNodeListenerErr error\n\tleafNodeInfo Info\n\tleafNodeInfoJSON []byte\n\tleafURLsMap refCountedUrlSet\n\tleafNodeOpts struct {\n\t\tresolver netResolver\n\t\tdialTimeout time.Duration\n\t}\n\tleafRemoteCfgs []*leafNodeCfg\n\tleafRemoteAccounts sync.Map\n\tleafNodeEnabled bool\n\tleafDisableConnect bool // Used in test only\n\tleafNoCluster bool // Indicate that this server has only remotes and no cluster defined\n\n\tquitCh chan struct{}\n\tstartupComplete chan struct{}\n\tshutdownComplete chan struct{}\n\n\t// Tracking Go routines\n\tgrMu sync.Mutex\n\tgrTmpClients map[uint64]*client\n\tgrRunning bool\n\tgrWG sync.WaitGroup // to wait on various go routines\n\n\tcproto int64 // number of clients supporting async INFO\n\tconfigTime time.Time // last time config was loaded\n\n\tlogging struct {\n\t\tsync.RWMutex\n\t\tlogger Logger\n\t\ttrace int32\n\t\tdebug int32\n\t\ttraceSysAcc int32\n\t}\n\n\tclientConnectURLs []string\n\n\t// Used internally for quick look-ups.\n\tclientConnectURLsMap refCountedUrlSet\n\n\t// For Gateways\n\tgatewayListener net.Listener // Accept listener\n\tgatewayListenerErr error\n\tgateway *srvGateway\n\n\t// Used by tests to check that http.Servers do\n\t// not set any timeout.\n\tmonitoringServer *http.Server\n\tprofilingServer *http.Server\n\n\t// LameDuck mode\n\tldm bool\n\tldmCh chan bool\n\n\t// Trusted public operator keys.\n\ttrustedKeys []string\n\t// map of trusted keys to operator setting StrictSigningKeyUsage\n\tstrictSigningKeyUsage map[string]struct{}\n\n\t// We use this to minimize mem copies for requests to monitoring\n\t// endpoint /varz (when it comes from http).\n\tvarzMu sync.Mutex\n\tvarz *Varz\n\t// This is set during a config reload if we detect that we have\n\t// added/removed routes. The monitoring code then check that\n\t// to know if it should update the cluster's URLs array.\n\tvarzUpdateRouteURLs bool\n\n\t// Keeps a sublist of of subscriptions attached to leafnode connections\n\t// for the $GNR.*.*.*.> subject so that a server can send back a mapped\n\t// gateway reply.\n\tgwLeafSubs *Sublist\n\n\t// Used for expiration of mapped GW replies\n\tgwrm struct {\n\t\tw int32\n\t\tch chan time.Duration\n\t\tm sync.Map\n\t}\n\n\t// For eventIDs\n\teventIds *nuid.NUID\n\n\t// Websocket structure\n\twebsocket srvWebsocket\n\n\t// MQTT structure\n\tmqtt srvMQTT\n\n\t// OCSP monitoring\n\tocsps []*OCSPMonitor\n\n\t// OCSP peer verification (at least one TLS block)\n\tocspPeerVerify bool\n\n\t// OCSP response cache\n\tocsprc OCSPResponseCache\n\n\t// exporting account name the importer experienced issues with\n\tincompleteAccExporterMap sync.Map\n\n\t// Holds cluster name under different lock for mapping\n\tcnMu sync.RWMutex\n\tcn string\n\n\t// For registering raft nodes with the server.\n\trnMu sync.RWMutex\n\traftNodes map[string]RaftNode\n\n\t// For mapping from a raft node name back to a server name and cluster. Node has to be in the same domain.\n\tnodeToInfo sync.Map\n\n\t// For out of resources to not log errors too fast.\n\trerrMu sync.Mutex\n\trerrLast time.Time\n\n\tconnRateCounter *rateCounter\n\n\t// If there is a system account configured, to still support the $G account,\n\t// the server will create a fake user and add it to the list of users.\n\t// Keep track of what that user name is for config reload purposes.\n\tsysAccOnlyNoAuthUser string\n\n\t// IPQueues map\n\tipQueues sync.Map\n\n\t// To limit logging frequency\n\trateLimitLogging sync.Map\n\trateLimitLoggingCh chan time.Duration\n\n\t// Total outstanding catchup bytes in flight.\n\tgcbMu sync.RWMutex\n\tgcbOut int64\n\tgcbOutMax int64 // Taken from JetStreamMaxCatchup or defaultMaxTotalCatchupOutBytes\n\t// A global chanel to kick out stalled catchup sequences.\n\tgcbKick chan struct{}\n\n\t// Total outbound syncRequests\n\tsyncOutSem chan struct{}\n\n\t// Queue to process JS API requests that come from routes (or gateways)\n\tjsAPIRoutedReqs *ipQueue[*jsAPIRoutedReq]\n\tjsAPIRoutedInfoReqs *ipQueue[*jsAPIRoutedReq]\n\n\t// Delayed API responses.\n\tdelayedAPIResponses *ipQueue[*delayedAPIResponse]\n\n\t// Whether moving NRG traffic into accounts is permitted on this server.\n\t// Controls whether or not the account NRG capability is set in statsz.\n\t// Currently used by unit tests to simulate nodes not supporting account NRG.\n\taccountNRGAllowed atomic.Bool\n\n\t// List of proxies trusted keys in `KeyPair` form so we can do signature\n\t// verification when processing incoming proxy connections.\n\tproxiesKeyPairs []nkeys.KeyPair\n\tproxiedConns map[string]map[uint64]*client\n}\n\n// For tracking JS nodes.\ntype nodeInfo struct {\n\tname string\n\tversion string\n\tcluster string\n\tdomain string\n\tid string\n\ttags jwt.TagList\n\tcfg *JetStreamConfig\n\tstats *JetStreamStats\n\toffline bool\n\tjs bool\n\tbinarySnapshots bool\n\taccountNRG bool\n}\n\ntype stats struct {\n\tinMsgs int64\n\toutMsgs int64\n\tinBytes int64\n\toutBytes int64\n\tslowConsumers int64\n\tstaleConnections int64\n\tstalls int64\n}\n\n// scStats includes the total and per connection counters of Slow Consumers.\ntype scStats struct {\n\tclients atomic.Uint64\n\troutes atomic.Uint64\n\tleafs atomic.Uint64\n\tgateways atomic.Uint64\n}\n\n// staleStats includes the total and per connection counters of Stale Connections.\ntype staleStats struct {\n\tclients atomic.Uint64\n\troutes atomic.Uint64\n\tleafs atomic.Uint64\n\tgateways atomic.Uint64\n}\n\n// This is used by tests so we can run all server tests with a default route\n// or leafnode compression mode. For instance:\n// go test -race -v ./server -cluster_compression=fast\nvar (\n\ttestDefaultClusterCompression string\n\ttestDefaultLeafNodeCompression string\n)\n\n// Compression modes.\nconst (\n\tCompressionNotSupported = \"not supported\"\n\tCompressionOff = \"off\"\n\tCompressionAccept = \"accept\"\n\tCompressionS2Auto = \"s2_auto\"\n\tCompressionS2Uncompressed = \"s2_uncompressed\"\n\tCompressionS2Fast = \"s2_fast\"\n\tCompressionS2Better = \"s2_better\"\n\tCompressionS2Best = \"s2_best\"\n)\n\n// defaultCompressionS2AutoRTTThresholds is the default of RTT thresholds for\n// the CompressionS2Auto mode.\nvar defaultCompressionS2AutoRTTThresholds = []time.Duration{\n\t// [0..10ms] -> CompressionS2Uncompressed\n\t10 * time.Millisecond,\n\t// ]10ms..50ms] -> CompressionS2Fast\n\t50 * time.Millisecond,\n\t// ]50ms..100ms] -> CompressionS2Better\n\t100 * time.Millisecond,\n\t// ]100ms..] -> CompressionS2Best\n}\n\n// For a given user provided string, matches to one of the compression mode\n// constant and updates the provided string to that constant. Returns an\n// error if the provided compression mode is not known.\n// The parameter `chosenModeForOn` indicates which compression mode to use\n// when the user selects \"on\" (or enabled, true, etc..). This is because\n// we may have different defaults depending on where the compression is used.\nfunc validateAndNormalizeCompressionOption(c *CompressionOpts, chosenModeForOn string) error {\n\tif c == nil {\n\t\treturn nil\n\t}\n\tcmtl := strings.ToLower(c.Mode)\n\t// First, check for the \"on\" case so that we set to the default compression\n\t// mode for that. The other switch/case will finish setup if needed (for\n\t// instance if the default mode is s2Auto).\n\tswitch cmtl {\n\tcase \"on\", \"enabled\", \"true\":\n\t\tcmtl = chosenModeForOn\n\tdefault:\n\t}\n\t// Check (again) with the proper mode.\n\tswitch cmtl {\n\tcase \"not supported\", \"not_supported\":\n\t\tc.Mode = CompressionNotSupported\n\tcase \"disabled\", \"off\", \"false\":\n\t\tc.Mode = CompressionOff\n\tcase \"accept\":\n\t\tc.Mode = CompressionAccept\n\tcase \"auto\", \"s2_auto\":\n\t\tvar rtts []time.Duration\n\t\tif len(c.RTTThresholds) == 0 {\n\t\t\trtts = defaultCompressionS2AutoRTTThresholds\n\t\t} else {\n\t\t\tfor _, n := range c.RTTThresholds {\n\t\t\t\t// Do not error on negative, but simply set to 0\n\t\t\t\tif n < 0 {\n\t\t\t\t\tn = 0\n\t\t\t\t}\n\t\t\t\t// Make sure they are properly ordered. However, it is possible\n\t\t\t\t// to have a \"0\" anywhere in the list to indicate that this\n\t\t\t\t// compression level should not be used.\n\t\t\t\tif l := len(rtts); l > 0 && n != 0 {\n\t\t\t\t\tfor _, v := range rtts {\n\t\t\t\t\t\tif n < v {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"RTT threshold values %v should be in ascending order\", c.RTTThresholds)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\trtts = append(rtts, n)\n\t\t\t}\n\t\t\tif len(rtts) > 0 {\n\t\t\t\t// Trim 0 that are at the end.\n\t\t\t\tstop := -1\n\t\t\t\tfor i := len(rtts) - 1; i >= 0; i-- {\n\t\t\t\t\tif rtts[i] != 0 {\n\t\t\t\t\t\tstop = i\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\trtts = rtts[:stop+1]\n\t\t\t}\n\t\t\tif len(rtts) > 4 {\n\t\t\t\t// There should be at most values for \"uncompressed\", \"fast\",\n\t\t\t\t// \"better\" and \"best\" (when some 0 are present).\n\t\t\t\treturn fmt.Errorf(\"compression mode %q should have no more than 4 RTT thresholds: %v\", c.Mode, c.RTTThresholds)\n\t\t\t} else if len(rtts) == 0 {\n\t\t\t\t// But there should be at least 1 if the user provided the slice.\n\t\t\t\t// We would be here only if it was provided by say with values\n\t\t\t\t// being a single or all zeros.\n\t\t\t\treturn fmt.Errorf(\"compression mode %q requires at least one RTT threshold\", c.Mode)\n\t\t\t}\n\t\t}\n\t\tc.Mode = CompressionS2Auto\n\t\tc.RTTThresholds = rtts\n\tcase \"fast\", \"s2_fast\":\n\t\tc.Mode = CompressionS2Fast\n\tcase \"better\", \"s2_better\":\n\t\tc.Mode = CompressionS2Better\n\tcase \"best\", \"s2_best\":\n\t\tc.Mode = CompressionS2Best\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported compression mode %q\", c.Mode)\n\t}\n\treturn nil\n}\n\n// Returns `true` if the compression mode `m` indicates that the server\n// will negotiate compression with the remote server, `false` otherwise.\n// Note that the provided compression mode is assumed to have been\n// normalized and validated.\nfunc needsCompression(m string) bool {\n\treturn m != _EMPTY_ && m != CompressionOff && m != CompressionNotSupported\n}\n\n// Compression is asymmetric, meaning that one side can have a different\n// compression level than the other. However, we need to check for cases\n// when this server `scm` or the remote `rcm` do not support compression\n// (say older server, or test to make it behave as it is not), or have\n// the compression off.\n// Note that `scm` is assumed to not be \"off\" or \"not supported\".\nfunc selectCompressionMode(scm, rcm string) (mode string, err error) {\n\tif rcm == CompressionNotSupported || rcm == _EMPTY_ {\n\t\treturn CompressionNotSupported, nil\n\t}\n\tswitch rcm {\n\tcase CompressionOff:\n\t\t// If the remote explicitly disables compression, then we won't\n\t\t// use compression.\n\t\treturn CompressionOff, nil\n\tcase CompressionAccept:\n\t\t// If the remote is ok with compression (but is not initiating it),\n\t\t// and if we too are in this mode, then it means no compression.\n\t\tif scm == CompressionAccept {\n\t\t\treturn CompressionOff, nil\n\t\t}\n\t\t// Otherwise use our compression mode.\n\t\treturn scm, nil\n\tcase CompressionS2Auto, CompressionS2Uncompressed, CompressionS2Fast, CompressionS2Better, CompressionS2Best:\n\t\t// This case is here to make sure that if we don't recognize a\n\t\t// compression setting, we error out.\n\t\tif scm == CompressionAccept {\n\t\t\t// If our compression mode is \"accept\", then we will use the remote\n\t\t\t// compression mode, except if it is \"auto\", in which case we will\n\t\t\t// default to \"fast\". This is not a configuration (auto in one\n\t\t\t// side and accept in the other) that would be recommended.\n\t\t\tif rcm == CompressionS2Auto {\n\t\t\t\treturn CompressionS2Fast, nil\n\t\t\t}\n\t\t\t// Use their compression mode.\n\t\t\treturn rcm, nil\n\t\t}\n\t\t// Otherwise use our compression mode.\n\t\treturn scm, nil\n\tdefault:\n\t\treturn _EMPTY_, fmt.Errorf(\"unsupported route compression mode %q\", rcm)\n\t}\n}\n\n// If the configured compression mode is \"auto\" then will return that,\n// otherwise will return the given `cm` compression mode.\nfunc compressionModeForInfoProtocol(co *CompressionOpts, cm string) string {\n\tif co.Mode == CompressionS2Auto {\n\t\treturn CompressionS2Auto\n\t}\n\treturn cm\n}\n\n// Given a connection RTT and a list of thresholds durations, this\n// function will return an S2 compression level such as \"uncompressed\",\n// \"fast\", \"better\" or \"best\". For instance, with the following slice:\n// [5ms, 10ms, 15ms, 20ms], a RTT of up to 5ms will result\n// in the compression level \"uncompressed\", ]5ms..10ms] will result in\n// \"fast\" compression, etc..\n// However, the 0 value allows for disabling of some compression levels.\n// For instance, the following slice: [0, 0, 20, 30] means that a RTT of\n// [0..20ms] would result in the \"better\" compression - effectively disabling\n// the use of \"uncompressed\" and \"fast\", then anything above 20ms would\n// result in the use of \"best\" level (the 30 in the list has no effect\n// and the list could have been simplified to [0, 0, 20]).\nfunc selectS2AutoModeBasedOnRTT(rtt time.Duration, rttThresholds []time.Duration) string {\n\tvar idx int\n\tvar found bool\n\tfor i, d := range rttThresholds {\n\t\tif rtt <= d {\n\t\t\tidx = i\n\t\t\tfound = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !found {\n\t\t// If we did not find but we have all levels, then use \"best\",\n\t\t// otherwise use the last one in array.\n\t\tif l := len(rttThresholds); l >= 3 {\n\t\t\tidx = 3\n\t\t} else {\n\t\t\tidx = l - 1\n\t\t}\n\t}\n\tswitch idx {\n\tcase 0:\n\t\treturn CompressionS2Uncompressed\n\tcase 1:\n\t\treturn CompressionS2Fast\n\tcase 2:\n\t\treturn CompressionS2Better\n\t}\n\treturn CompressionS2Best\n}\n\nfunc compressOptsEqual(c1, c2 *CompressionOpts) bool {\n\tif c1 == c2 {\n\t\treturn true\n\t}\n\tif (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) {\n\t\treturn false\n\t}\n\tif c1.Mode != c2.Mode {\n\t\treturn false\n\t}\n\t// For s2_auto, if one has an empty RTTThresholds, it is equivalent\n\t// to the defaultCompressionS2AutoRTTThresholds array, so compare with that.\n\tif c1.Mode == CompressionS2Auto {\n\t\tif len(c1.RTTThresholds) == 0 && !reflect.DeepEqual(c2.RTTThresholds, defaultCompressionS2AutoRTTThresholds) {\n\t\t\treturn false\n\t\t}\n\t\tif len(c2.RTTThresholds) == 0 && !reflect.DeepEqual(c1.RTTThresholds, defaultCompressionS2AutoRTTThresholds) {\n\t\t\treturn false\n\t\t}\n\t\tif !reflect.DeepEqual(c1.RTTThresholds, c2.RTTThresholds) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Returns an array of s2 WriterOption based on the route compression mode.\n// So far we return a single option, but this way we can call s2.NewWriter()\n// with a nil []s2.WriterOption, but not with a nil s2.WriterOption, so\n// this is more versatile.\nfunc s2WriterOptions(cm string) []s2.WriterOption {\n\t_opts := [2]s2.WriterOption{}\n\topts := append(\n\t\t_opts[:0],\n\t\ts2.WriterConcurrency(1), // Stop asynchronous flushing in separate goroutines\n\t)\n\tswitch cm {\n\tcase CompressionS2Uncompressed:\n\t\treturn append(opts, s2.WriterUncompressed())\n\tcase CompressionS2Best:\n\t\treturn append(opts, s2.WriterBestCompression())\n\tcase CompressionS2Better:\n\t\treturn append(opts, s2.WriterBetterCompression())\n\tdefault:\n\t\treturn nil\n\t}\n}\n\n// New will setup a new server struct after parsing the options.\n// DEPRECATED: Use NewServer(opts)\nfunc New(opts *Options) *Server {\n\ts, _ := NewServer(opts)\n\treturn s\n}\n\nfunc NewServerFromConfig(opts *Options) (*Server, error) {\n\tif opts.ConfigFile != _EMPTY_ && opts.configDigest == \"\" {\n\t\tif err := opts.ProcessConfigFile(opts.ConfigFile); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn NewServer(opts)\n}\n\n// NewServer will setup a new server struct after parsing the options.\n// Could return an error if options can not be validated.\n// The provided Options type should not be re-used afterwards.\n// Either use Options.Clone() to pass a copy, or make a new one.\nfunc NewServer(opts *Options) (*Server, error) {\n\tsetBaselineOptions(opts)\n\n\t// Process TLS options, including whether we require client certificates.\n\ttlsReq := opts.TLSConfig != nil\n\tverify := (tlsReq && opts.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert)\n\n\t// Create our server's nkey identity.\n\tkp, _ := nkeys.CreateServer()\n\tpub, _ := kp.PublicKey()\n\n\t// Create an xkey for encrypting messages from this server.\n\tvar xkp nkeys.KeyPair\n\tvar xpub string\n\tif !fips140.Enabled() {\n\t\txkp, _ = nkeys.CreateCurveKeys()\n\t\txpub, _ = xkp.PublicKey()\n\t}\n\n\tserverName := pub\n\tif opts.ServerName != _EMPTY_ {\n\t\tserverName = opts.ServerName\n\t}\n\n\thttpBasePath := normalizeBasePath(opts.HTTPBasePath)\n\n\t// Validate some options. This is here because we cannot assume that\n\t// server will always be started with configuration parsing (that could\n\t// report issues). Its options can be (incorrectly) set by hand when\n\t// server is embedded. If there is an error, return nil.\n\tif err := validateOptions(opts); err != nil {\n\t\treturn nil, err\n\t}\n\n\tinfo := Info{\n\t\tID: pub,\n\t\tXKey: xpub,\n\t\tVersion: VERSION,\n\t\tProto: PROTO,\n\t\tGitCommit: gitCommit,\n\t\tGoVersion: runtime.Version(),\n\t\tName: serverName,\n\t\tHost: opts.Host,\n\t\tPort: opts.Port,\n\t\tAuthRequired: false,\n\t\tTLSRequired: tlsReq && !opts.AllowNonTLS,\n\t\tTLSVerify: verify,\n\t\tMaxPayload: opts.MaxPayload,\n\t\tJetStream: opts.JetStream,\n\t\tHeaders: !opts.NoHeaderSupport,\n\t\tCluster: opts.Cluster.Name,\n\t\tDomain: opts.JetStreamDomain,\n\t\tJSApiLevel: JSApiLevel,\n\t}\n\n\tif tlsReq && !info.TLSRequired {\n\t\tinfo.TLSAvailable = true\n\t}\n\n\tnow := time.Now()\n\n\ts := &Server{\n\t\tkp: kp,\n\t\txkp: xkp,\n\t\txpub: xpub,\n\t\tconfigFile: opts.ConfigFile,\n\t\tinfo: info,\n\t\topts: opts,\n\t\tdone: make(chan bool, 1),\n\t\tstart: now,\n\t\tconfigTime: now,\n\t\tgwLeafSubs: NewSublistWithCache(),\n\t\thttpBasePath: httpBasePath,\n\t\teventIds: nuid.New(),\n\t\troutesToSelf: make(map[string]struct{}),\n\t\thttpReqStats: make(map[string]uint64), // Used to track HTTP requests\n\t\trateLimitLoggingCh: make(chan time.Duration, 1),\n\t\tleafNodeEnabled: opts.LeafNode.Port != 0 || len(opts.LeafNode.Remotes) > 0,\n\t\tsyncOutSem: make(chan struct{}, maxConcurrentSyncRequests),\n\t}\n\n\t// Delayed API response queue. Create regardless if JetStream is configured\n\t// or not (since it can be enabled/disabled with config reload, we want this\n\t// queue to exist at all times).\n\ts.delayedAPIResponses = newIPQueue[*delayedAPIResponse](s, \"delayed API responses\")\n\n\t// By default we'll allow account NRG.\n\ts.accountNRGAllowed.Store(true)\n\n\t// Fill up the maximum in flight syncRequests for this server.\n\t// Used in JetStream catchup semantics.\n\tfor i := 0; i < maxConcurrentSyncRequests; i++ {\n\t\ts.syncOutSem <- struct{}{}\n\t}\n\n\tif opts.TLSRateLimit > 0 {\n\t\ts.connRateCounter = newRateCounter(opts.tlsConfigOpts.RateLimit)\n\t}\n\n\t// Trusted root operator keys.\n\tif !s.processTrustedKeys() {\n\t\treturn nil, fmt.Errorf(\"Error processing trusted operator keys\")\n\t}\n\n\t// If we have solicited leafnodes but no clustering and no clustername.\n\t// However we may need a stable clustername so use the server name.\n\tif len(opts.LeafNode.Remotes) > 0 && opts.Cluster.Port == 0 && opts.Cluster.Name == _EMPTY_ {\n\t\ts.leafNoCluster = true\n\t\topts.Cluster.Name = opts.ServerName\n\t}\n\n\tif opts.Cluster.Name != _EMPTY_ {\n\t\t// Also place into mapping cn with cnMu lock.\n\t\ts.cnMu.Lock()\n\t\ts.cn = opts.Cluster.Name\n\t\ts.cnMu.Unlock()\n\t}\n\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\n\t// If there are proxies trusted public keys in the configuration\n\t// this will fill create the corresponding list of nkeys.KeyPair\n\t// that we can use for signature verification.\n\ts.processProxiesTrustedKeys()\n\n\t// Place ourselves in the JetStream nodeInfo if needed.\n\tif opts.JetStream {\n\t\tourNode := getHash(serverName)\n\t\ts.nodeToInfo.Store(ourNode, nodeInfo{\n\t\t\tname: serverName,\n\t\t\tversion: VERSION,\n\t\t\tcluster: opts.Cluster.Name,\n\t\t\tdomain: opts.JetStreamDomain,\n\t\t\tid: info.ID,\n\t\t\ttags: opts.Tags,\n\t\t\tcfg: &JetStreamConfig{MaxMemory: opts.JetStreamMaxMemory, MaxStore: opts.JetStreamMaxStore, CompressOK: true},\n\t\t\tstats: nil,\n\t\t\toffline: false,\n\t\t\tjs: true,\n\t\t\tbinarySnapshots: true,\n\t\t\taccountNRG: true,\n\t\t})\n\t}\n\n\ts.routeResolver = opts.Cluster.resolver\n\tif s.routeResolver == nil {\n\t\ts.routeResolver = net.DefaultResolver\n\t}\n\n\t// Used internally for quick look-ups.\n\ts.clientConnectURLsMap = make(refCountedUrlSet)\n\ts.websocket.connectURLsMap = make(refCountedUrlSet)\n\ts.leafURLsMap = make(refCountedUrlSet)\n\n\t// Ensure that non-exported options (used in tests) are properly set.\n\ts.setLeafNodeNonExportedOptions()\n\n\t// Setup OCSP Stapling and OCSP Peer. This will abort server from starting if there\n\t// are no valid staples and OCSP Stapling policy is set to Always or MustStaple.\n\tif err := s.enableOCSP(); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Call this even if there is no gateway defined. It will\n\t// initialize the structure so we don't have to check for\n\t// it to be nil or not in various places in the code.\n\tif err := s.newGateway(opts); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// If we have a cluster definition but do not have a cluster name, create one.\n\tif opts.Cluster.Port != 0 && opts.Cluster.Name == _EMPTY_ {\n\t\ts.info.Cluster = nuid.Next()\n\t} else if opts.Cluster.Name != _EMPTY_ {\n\t\t// Likewise here if we have a cluster name set.\n\t\ts.info.Cluster = opts.Cluster.Name\n\t}\n\n\t// This is normally done in the AcceptLoop, once the\n\t// listener has been created (possibly with random port),\n\t// but since some tests may expect the INFO to be properly\n\t// set after New(), let's do it now.\n\ts.setInfoHostPort()\n\n\t// For tracking clients\n\ts.clients = make(map[uint64]*client)\n\n\t// For tracking closed clients.\n\ts.closed = newClosedRingBuffer(opts.MaxClosedClients)\n\n\t// For tracking connections that are not yet registered\n\t// in s.routes, but for which readLoop has started.\n\ts.grTmpClients = make(map[uint64]*client)\n\n\t// For tracking routes and their remote ids\n\ts.initRouteStructures(opts)\n\n\t// For tracking leaf nodes.\n\ts.leafs = make(map[uint64]*client)\n\n\t// Used to kick out all go routines possibly waiting on server\n\t// to shutdown.\n\ts.quitCh = make(chan struct{})\n\n\t// Closed when startup is complete. ReadyForConnections() will block on\n\t// this before checking the presence of listening sockets.\n\ts.startupComplete = make(chan struct{})\n\n\t// Closed when Shutdown() is complete. Allows WaitForShutdown() to block\n\t// waiting for complete shutdown.\n\ts.shutdownComplete = make(chan struct{})\n\n\t// Check for configured account resolvers.\n\tif err := s.configureResolver(); err != nil {\n\t\treturn nil, err\n\t}\n\t// If there is an URL account resolver, do basic test to see if anyone is home.\n\tif ar := opts.AccountResolver; ar != nil {\n\t\tif ur, ok := ar.(*URLAccResolver); ok {\n\t\t\tif _, err := ur.Fetch(_EMPTY_); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\t// For other resolver:\n\t// In operator mode, when the account resolver depends on an external system and\n\t// the system account can't be fetched, inject a temporary one.\n\tif ar := s.accResolver; len(opts.TrustedOperators) == 1 && ar != nil &&\n\t\topts.SystemAccount != _EMPTY_ && opts.SystemAccount != DEFAULT_SYSTEM_ACCOUNT {\n\t\tif _, ok := ar.(*MemAccResolver); !ok {\n\t\t\ts.mu.Unlock()\n\t\t\tvar a *Account\n\t\t\t// perform direct lookup to avoid warning trace\n\t\t\tif _, err := fetchAccount(ar, opts.SystemAccount); err == nil {\n\t\t\t\ta, _ = s.lookupAccount(opts.SystemAccount)\n\t\t\t}\n\t\t\ts.mu.Lock()\n\t\t\tif a == nil {\n\t\t\t\tsac := NewAccount(opts.SystemAccount)\n\t\t\t\tsac.Issuer = opts.TrustedOperators[0].Issuer\n\t\t\t\tsac.signingKeys = map[string]jwt.Scope{}\n\t\t\t\tsac.signingKeys[opts.SystemAccount] = nil\n\t\t\t\ts.registerAccountNoLock(sac)\n\t\t\t}\n\t\t}\n\t}\n\n\t// For tracking accounts\n\tif _, err := s.configureAccounts(false); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Used to setup Authorization.\n\ts.configureAuthorization()\n\n\t// Start signal handler\n\ts.handleSignals()\n\n\treturn s, nil\n}\n\n// Initializes route structures based on pooling and/or per-account routes.\n//\n// Server lock is held on entry\nfunc (s *Server) initRouteStructures(opts *Options) {\n\ts.routes = make(map[string][]*client)\n\tif ps := opts.Cluster.PoolSize; ps > 0 {\n\t\ts.routesPoolSize = ps\n\t} else {\n\t\ts.routesPoolSize = 1\n\t}\n\t// If we have per-account routes, we create accRoutes and initialize it\n\t// with nil values. The presence of an account as the key will allow us\n\t// to know if a given account is supposed to have dedicated routes.\n\tif l := len(opts.Cluster.PinnedAccounts); l > 0 {\n\t\ts.accRoutes = make(map[string]map[string]*client, l)\n\t\tfor _, acc := range opts.Cluster.PinnedAccounts {\n\t\t\ts.accRoutes[acc] = make(map[string]*client)\n\t\t}\n\t}\n}\n\nfunc (s *Server) logRejectedTLSConns() {\n\tdefer s.grWG.Done()\n\tt := time.NewTicker(time.Second)\n\tdefer t.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-t.C:\n\t\t\tblocked := s.connRateCounter.countBlocked()\n\t\t\tif blocked > 0 {\n\t\t\t\ts.Warnf(\"Rejected %d connections due to TLS rate limiting\", blocked)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// clusterName returns our cluster name which could be dynamic.\nfunc (s *Server) ClusterName() string {\n\ts.mu.RLock()\n\tcn := s.info.Cluster\n\ts.mu.RUnlock()\n\treturn cn\n}\n\n// Grabs cluster name with cluster name specific lock.\nfunc (s *Server) cachedClusterName() string {\n\ts.cnMu.RLock()\n\tcn := s.cn\n\ts.cnMu.RUnlock()\n\treturn cn\n}\n\n// setClusterName will update the cluster name for this server.\nfunc (s *Server) setClusterName(name string) {\n\ts.mu.Lock()\n\tvar resetCh chan struct{}\n\tif s.sys != nil && s.info.Cluster != name {\n\t\t// can't hold the lock as go routine reading it may be waiting for lock as well\n\t\tresetCh = s.sys.resetCh\n\t}\n\ts.info.Cluster = name\n\ts.routeInfo.Cluster = name\n\n\t// Need to close solicited leaf nodes. The close has to be done outside of the server lock.\n\tvar leafs []*client\n\tfor _, c := range s.leafs {\n\t\tc.mu.Lock()\n\t\tif c.leaf != nil && c.leaf.remote != nil {\n\t\t\tleafs = append(leafs, c)\n\t\t}\n\t\tc.mu.Unlock()\n\t}\n\ts.mu.Unlock()\n\n\t// Also place into mapping cn with cnMu lock.\n\ts.cnMu.Lock()\n\ts.cn = name\n\ts.cnMu.Unlock()\n\n\tfor _, l := range leafs {\n\t\tl.closeConnection(ClusterNameConflict)\n\t}\n\tif resetCh != nil {\n\t\tresetCh <- struct{}{}\n\t}\n\ts.Noticef(\"Cluster name updated to %s\", name)\n}\n\n// Return whether the cluster name is dynamic.\nfunc (s *Server) isClusterNameDynamic() bool {\n\t// We need to lock the whole \"Cluster.Name\" check and not use s.getOpts()\n\t// because otherwise this could cause a data race with setting the name in\n\t// route.go's processRouteConnect().\n\ts.optsMu.RLock()\n\tdynamic := s.opts.Cluster.Name == _EMPTY_\n\ts.optsMu.RUnlock()\n\treturn dynamic\n}\n\n// Returns our configured serverName.\nfunc (s *Server) serverName() string {\n\treturn s.getOpts().ServerName\n}\n\n// ClientURL returns the URL used to connect clients.\n// Helpful in tests and with in-process servers using a random client port (-1).\nfunc (s *Server) ClientURL() string {\n\t// FIXME(dlc) - should we add in user and pass if defined single?\n\topts := s.getOpts()\n\tvar u url.URL\n\tu.Scheme = \"nats\"\n\tif opts.TLSConfig != nil {\n\t\tu.Scheme = \"tls\"\n\t}\n\tu.Host = net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port))\n\treturn u.String()\n}\n\n// WebsocketURL returns the URL used to connect websocket clients.\n// Helpful in tests and with in-process servers using a random websocket port (-1).\nfunc (s *Server) WebsocketURL() string {\n\topts := s.getOpts()\n\tvar u url.URL\n\tu.Scheme = \"ws\"\n\tif opts.Websocket.TLSConfig != nil {\n\t\tu.Scheme = \"wss\"\n\t}\n\tu.Host = net.JoinHostPort(opts.Websocket.Host, fmt.Sprintf(\"%d\", opts.Websocket.Port))\n\treturn u.String()\n}\n\nfunc validateCluster(o *Options) error {\n\tif o.Cluster.Name != _EMPTY_ && strings.Contains(o.Cluster.Name, \" \") {\n\t\treturn ErrClusterNameHasSpaces\n\t}\n\tif o.Cluster.Compression.Mode != _EMPTY_ {\n\t\tif err := validateAndNormalizeCompressionOption(&o.Cluster.Compression, CompressionS2Fast); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := validatePinnedCerts(o.Cluster.TLSPinnedCerts); err != nil {\n\t\treturn fmt.Errorf(\"cluster: %v\", err)\n\t}\n\t// Check that cluster name if defined matches any gateway name.\n\t// Note that we have already verified that the gateway name does not have spaces.\n\tif o.Gateway.Name != _EMPTY_ && o.Gateway.Name != o.Cluster.Name {\n\t\tif o.Cluster.Name != _EMPTY_ {\n\t\t\treturn ErrClusterNameConfigConflict\n\t\t}\n\t\t// Set this here so we do not consider it dynamic.\n\t\to.Cluster.Name = o.Gateway.Name\n\t}\n\tif l := len(o.Cluster.PinnedAccounts); l > 0 {\n\t\tif o.Cluster.PoolSize < 0 {\n\t\t\treturn fmt.Errorf(\"pool_size cannot be negative if pinned accounts are specified\")\n\t\t}\n\t\tm := make(map[string]struct{}, l)\n\t\tfor _, a := range o.Cluster.PinnedAccounts {\n\t\t\tif _, exists := m[a]; exists {\n\t\t\t\treturn fmt.Errorf(\"found duplicate account name %q in pinned accounts list %q\", a, o.Cluster.PinnedAccounts)\n\t\t\t}\n\t\t\tm[a] = struct{}{}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validatePinnedCerts(pinned PinnedCertSet) error {\n\tre := regexp.MustCompile(\"^[a-f0-9]{64}$\")\n\tfor certId := range pinned {\n\t\tentry := strings.ToLower(certId)\n\t\tif !re.MatchString(entry) {\n\t\t\treturn fmt.Errorf(\"error parsing 'pinned_certs' key %s does not look like lower case hex-encoded sha256 of DER encoded SubjectPublicKeyInfo\", entry)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validateOptions(o *Options) error {\n\tif o.LameDuckDuration > 0 && o.LameDuckGracePeriod >= o.LameDuckDuration {\n\t\treturn fmt.Errorf(\"lame duck grace period (%v) should be strictly lower than lame duck duration (%v)\",\n\t\t\to.LameDuckGracePeriod, o.LameDuckDuration)\n\t}\n\tif int64(o.MaxPayload) > o.MaxPending {\n\t\treturn fmt.Errorf(\"max_payload (%v) cannot be higher than max_pending (%v)\",\n\t\t\to.MaxPayload, o.MaxPending)\n\t}\n\tif o.ServerName != _EMPTY_ && strings.Contains(o.ServerName, \" \") {\n\t\treturn errors.New(\"server name cannot contain spaces\")\n\t}\n\t// Check that the trust configuration is correct.\n\tif err := validateTrustedOperators(o); err != nil {\n\t\treturn err\n\t}\n\t// Check on leaf nodes which will require a system\n\t// account when gateways are also configured.\n\tif err := validateLeafNode(o); err != nil {\n\t\treturn err\n\t}\n\t// Check that authentication is properly configured.\n\tif err := validateAuth(o); err != nil {\n\t\treturn err\n\t}\n\t// Check that proxies is properly configured.\n\tif err := validateProxies(o); err != nil {\n\t\treturn err\n\t}\n\t// Check that gateway is properly configured. Returns no error\n\t// if there is no gateway defined.\n\tif err := validateGatewayOptions(o); err != nil {\n\t\treturn err\n\t}\n\t// Check that cluster name if defined matches any gateway name.\n\tif err := validateCluster(o); err != nil {\n\t\treturn err\n\t}\n\tif err := validateMQTTOptions(o); err != nil {\n\t\treturn err\n\t}\n\tif err := validateJetStreamOptions(o); err != nil {\n\t\treturn err\n\t}\n\t// Finally check websocket options.\n\treturn validateWebsocketOptions(o)\n}\n\nfunc (s *Server) getOpts() *Options {\n\ts.optsMu.RLock()\n\topts := s.opts\n\ts.optsMu.RUnlock()\n\treturn opts\n}\n\nfunc (s *Server) setOpts(opts *Options) {\n\ts.optsMu.Lock()\n\ts.opts = opts\n\ts.optsMu.Unlock()\n}\n\nfunc (s *Server) globalAccount() *Account {\n\ts.mu.RLock()\n\tgacc := s.gacc\n\ts.mu.RUnlock()\n\treturn gacc\n}\n\n// Used to setup or update Accounts.\n// Returns a map that indicates which accounts have had their stream imports\n// changed (in case of an update in configuration reload).\n// Lock is held upon entry, but will be released/reacquired in this function.\nfunc (s *Server) configureAccounts(reloading bool) (map[string]struct{}, error) {\n\tawcsti := make(map[string]struct{})\n\n\t// Create the global account.\n\tif s.gacc == nil {\n\t\ts.gacc = NewAccount(globalAccountName)\n\t\ts.registerAccountNoLock(s.gacc)\n\t}\n\n\topts := s.getOpts()\n\n\t// We need to track service imports since we can not swap them out (unsub and re-sub)\n\t// until the proper server struct accounts have been swapped in properly. Doing it in\n\t// place could lead to data loss or server panic since account under new si has no real\n\t// account and hence no sublist, so will panic on inbound message.\n\tsiMap := make(map[*Account][][]byte)\n\n\t// Check opts and walk through them. We need to copy them here\n\t// so that we do not keep a real one sitting in the options.\n\tfor _, acc := range opts.Accounts {\n\t\tvar a *Account\n\t\tcreate := true\n\t\t// For the global account, we want to skip the reload process\n\t\t// and fall back into the \"create\" case which will in that\n\t\t// case really be just an update (shallowCopy will make sure\n\t\t// that mappings are copied over).\n\t\tif reloading && acc.Name != globalAccountName {\n\t\t\tif ai, ok := s.accounts.Load(acc.Name); ok {\n\t\t\t\ta = ai.(*Account)\n\t\t\t\t// Before updating the account, check if stream imports have changed.\n\t\t\t\tif !a.checkStreamImportsEqual(acc) {\n\t\t\t\t\tawcsti[acc.Name] = struct{}{}\n\t\t\t\t}\n\t\t\t\ta.mu.Lock()\n\t\t\t\t// Collect the sids for the service imports since we are going to\n\t\t\t\t// replace with new ones.\n\t\t\t\tvar sids [][]byte\n\t\t\t\tfor _, sis := range a.imports.services {\n\t\t\t\t\tfor _, si := range sis {\n\t\t\t\t\t\tif si.sid != nil {\n\t\t\t\t\t\t\tsids = append(sids, si.sid)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Setup to process later if needed.\n\t\t\t\tif len(sids) > 0 || len(acc.imports.services) > 0 {\n\t\t\t\t\tsiMap[a] = sids\n\t\t\t\t}\n\n\t\t\t\t// Now reset all export/imports fields since they are going to be\n\t\t\t\t// filled in shallowCopy()\n\t\t\t\ta.imports.streams, a.imports.services = nil, nil\n\t\t\t\ta.exports.streams, a.exports.services = nil, nil\n\t\t\t\t// We call shallowCopy from the account `acc` (the one in Options)\n\t\t\t\t// and pass `a` (our existing account) to get it updated.\n\t\t\t\tacc.shallowCopy(a)\n\t\t\t\ta.mu.Unlock()\n\t\t\t\tcreate = false\n\t\t\t}\n\t\t}\n\t\t// Track old mappings if global account.\n\t\tvar oldGMappings []*mapping\n\t\tif create {\n\t\t\tif acc.Name == globalAccountName {\n\t\t\t\ta = s.gacc\n\t\t\t\ta.mu.Lock()\n\t\t\t\toldGMappings = append(oldGMappings, a.mappings...)\n\t\t\t\ta.mu.Unlock()\n\t\t\t} else {\n\t\t\t\ta = NewAccount(acc.Name)\n\t\t\t}\n\t\t\t// Locking matters in the case of an update of the global account\n\t\t\ta.mu.Lock()\n\t\t\tacc.shallowCopy(a)\n\t\t\ta.mu.Unlock()\n\t\t\t// Will be a no-op in case of the global account since it is already registered.\n\t\t\ts.registerAccountNoLock(a)\n\t\t}\n\n\t\t// The `acc` account is stored in options, not in the server, and these can be cleared.\n\t\tacc.sl, acc.clients, acc.mappings = nil, nil, nil\n\n\t\t// Check here if we have been reloaded and we have a global account with mappings that may have changed.\n\t\t// If we have leafnodes they need to be updated.\n\t\tif reloading && a == s.gacc {\n\t\t\ta.mu.Lock()\n\t\t\tmappings := make(map[string]*mapping)\n\t\t\tif len(a.mappings) > 0 && a.nleafs > 0 {\n\t\t\t\tfor _, em := range a.mappings {\n\t\t\t\t\tmappings[em.src] = em\n\t\t\t\t}\n\t\t\t}\n\t\t\ta.mu.Unlock()\n\t\t\tif len(mappings) > 0 || len(oldGMappings) > 0 {\n\t\t\t\ta.lmu.RLock()\n\t\t\t\tfor _, lc := range a.lleafs {\n\t\t\t\t\tfor _, em := range mappings {\n\t\t\t\t\t\tlc.forceAddToSmap(em.src)\n\t\t\t\t\t}\n\t\t\t\t\t// Remove any old ones if needed.\n\t\t\t\t\tfor _, em := range oldGMappings {\n\t\t\t\t\t\t// Only remove if not in the new ones.\n\t\t\t\t\t\tif _, ok := mappings[em.src]; !ok {\n\t\t\t\t\t\t\tlc.forceRemoveFromSmap(em.src)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ta.lmu.RUnlock()\n\t\t\t}\n\t\t}\n\n\t\t// If we see an account defined using $SYS we will make sure that is set as system account.\n\t\tif acc.Name == DEFAULT_SYSTEM_ACCOUNT && opts.SystemAccount == _EMPTY_ {\n\t\t\topts.SystemAccount = DEFAULT_SYSTEM_ACCOUNT\n\t\t}\n\t}\n\n\t// Now that we have this we need to remap any referenced accounts in\n\t// import or export maps to the new ones.\n\tswapApproved := func(ea *exportAuth) {\n\t\tfor sub, a := range ea.approved {\n\t\t\tvar acc *Account\n\t\t\tif v, ok := s.accounts.Load(a.Name); ok {\n\t\t\t\tacc = v.(*Account)\n\t\t\t}\n\t\t\tea.approved[sub] = acc\n\t\t}\n\t}\n\tvar numAccounts int\n\ts.accounts.Range(func(k, v any) bool {\n\t\tnumAccounts++\n\t\tacc := v.(*Account)\n\t\tacc.mu.Lock()\n\t\t// Exports\n\t\tfor _, se := range acc.exports.streams {\n\t\t\tif se != nil {\n\t\t\t\tswapApproved(&se.exportAuth)\n\t\t\t}\n\t\t}\n\t\tfor _, se := range acc.exports.services {\n\t\t\tif se != nil {\n\t\t\t\t// Swap over the bound account for service exports.\n\t\t\t\tif se.acc != nil {\n\t\t\t\t\tif v, ok := s.accounts.Load(se.acc.Name); ok {\n\t\t\t\t\t\tse.acc = v.(*Account)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tswapApproved(&se.exportAuth)\n\t\t\t}\n\t\t}\n\t\t// Imports\n\t\tfor _, si := range acc.imports.streams {\n\t\t\tif v, ok := s.accounts.Load(si.acc.Name); ok {\n\t\t\t\tsi.acc = v.(*Account)\n\t\t\t}\n\t\t}\n\t\tfor _, sis := range acc.imports.services {\n\t\t\tfor _, si := range sis {\n\t\t\t\tif v, ok := s.accounts.Load(si.acc.Name); ok {\n\t\t\t\t\tsi.acc = v.(*Account)\n\n\t\t\t\t\t// It is possible to allow for latency tracking inside your\n\t\t\t\t\t// own account, so lock only when not the same account.\n\t\t\t\t\tif si.acc == acc {\n\t\t\t\t\t\tsi.se = si.acc.getServiceExport(si.to)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tsi.acc.mu.RLock()\n\t\t\t\t\tsi.se = si.acc.getServiceExport(si.to)\n\t\t\t\t\tsi.acc.mu.RUnlock()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Make sure the subs are running, but only if not reloading.\n\t\tif len(acc.imports.services) > 0 && acc.ic == nil && !reloading {\n\t\t\tacc.ic = s.createInternalAccountClient()\n\t\t\tacc.ic.acc = acc\n\t\t\t// Need to release locks to invoke this function.\n\t\t\tacc.mu.Unlock()\n\t\t\ts.mu.Unlock()\n\t\t\tacc.addAllServiceImportSubs()\n\t\t\ts.mu.Lock()\n\t\t\tacc.mu.Lock()\n\t\t}\n\t\tacc.updated = time.Now()\n\t\tacc.mu.Unlock()\n\t\treturn true\n\t})\n\n\t// Check if we need to process service imports pending from above.\n\t// This processing needs to be after we swap in the real accounts above.\n\tfor acc, sids := range siMap {\n\t\tc := acc.ic\n\t\tfor _, sid := range sids {\n\t\t\tc.processUnsub(sid)\n\t\t}\n\t\tacc.addAllServiceImportSubs()\n\t\ts.mu.Unlock()\n\t\ts.registerSystemImports(acc)\n\t\ts.mu.Lock()\n\t}\n\n\t// Set the system account if it was configured.\n\t// Otherwise create a default one.\n\tif opts.SystemAccount != _EMPTY_ {\n\t\t// Lock may be acquired in lookupAccount, so release to call lookupAccount.\n\t\ts.mu.Unlock()\n\t\tacc, err := s.lookupAccount(opts.SystemAccount)\n\t\ts.mu.Lock()\n\t\tif err == nil && s.sys != nil && acc != s.sys.account {\n\t\t\t// sys.account.clients (including internal client)/respmap/etc... are transferred separately\n\t\t\ts.sys.account = acc\n\t\t\ts.sysAcc.Store(acc)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn awcsti, fmt.Errorf(\"error resolving system account: %v\", err)\n\t\t}\n\n\t\t// If we have defined a system account here check to see if its just us and the $G account.\n\t\t// We would do this to add user/pass to the system account. If this is the case add in\n\t\t// no-auth-user for $G.\n\t\t// Only do this if non-operator mode and we did not have an authorization block defined.\n\t\tif len(opts.TrustedOperators) == 0 && numAccounts == 2 && opts.NoAuthUser == _EMPTY_ && !opts.authBlockDefined {\n\t\t\t// If we come here from config reload, let's not recreate the fake user name otherwise\n\t\t\t// it will cause currently clients to be disconnected.\n\t\t\tuname := s.sysAccOnlyNoAuthUser\n\t\t\tif uname == _EMPTY_ {\n\t\t\t\t// Create a unique name so we do not collide.\n\t\t\t\tvar b [8]byte\n\t\t\t\trn := rand.Int63()\n\t\t\t\tfor i, l := 0, rn; i < len(b); i++ {\n\t\t\t\t\tb[i] = digits[l%base]\n\t\t\t\t\tl /= base\n\t\t\t\t}\n\t\t\t\tuname = fmt.Sprintf(\"nats-%s\", b[:])\n\t\t\t\ts.sysAccOnlyNoAuthUser = uname\n\t\t\t}\n\t\t\topts.Users = append(opts.Users, &User{Username: uname, Password: uname[6:], Account: s.gacc})\n\t\t\topts.NoAuthUser = uname\n\t\t}\n\t}\n\n\t// Add any required exports from system account.\n\tif s.sys != nil {\n\t\tsysAcc := s.sys.account\n\t\ts.mu.Unlock()\n\t\ts.addSystemAccountExports(sysAcc)\n\t\ts.mu.Lock()\n\t}\n\n\treturn awcsti, nil\n}\n\n// Setup the account resolver. For memory resolver, make sure the JWTs are\n// properly formed but do not enforce expiration etc.\n// Lock is held on entry, but may be released/reacquired during this call.\nfunc (s *Server) configureResolver() error {\n\topts := s.getOpts()\n\ts.accResolver = opts.AccountResolver\n\tif opts.AccountResolver != nil {\n\t\t// For URL resolver, set the TLSConfig if specified.\n\t\tif opts.AccountResolverTLSConfig != nil {\n\t\t\tif ar, ok := opts.AccountResolver.(*URLAccResolver); ok {\n\t\t\t\tif t, ok := ar.c.Transport.(*http.Transport); ok {\n\t\t\t\t\tt.CloseIdleConnections()\n\t\t\t\t\tt.TLSClientConfig = opts.AccountResolverTLSConfig.Clone()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(opts.resolverPreloads) > 0 {\n\t\t\t// Lock ordering is account resolver -> server, so we need to release\n\t\t\t// the lock and reacquire it when done with account resolver's calls.\n\t\t\tar := s.accResolver\n\t\t\ts.mu.Unlock()\n\t\t\tdefer s.mu.Lock()\n\t\t\tif ar.IsReadOnly() {\n\t\t\t\treturn fmt.Errorf(\"resolver preloads only available for writeable resolver types MEM/DIR/CACHE_DIR\")\n\t\t\t}\n\t\t\tfor k, v := range opts.resolverPreloads {\n\t\t\t\t_, err := jwt.DecodeAccountClaims(v)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"preload account error for %q: %v\", k, err)\n\t\t\t\t}\n\t\t\t\tar.Store(k, v)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// This will check preloads for validation issues.\nfunc (s *Server) checkResolvePreloads() {\n\topts := s.getOpts()\n\t// We can just check the read-only opts versions here, that way we do not need\n\t// to grab server lock or access s.accResolver.\n\tfor k, v := range opts.resolverPreloads {\n\t\tclaims, err := jwt.DecodeAccountClaims(v)\n\t\tif err != nil {\n\t\t\ts.Errorf(\"Preloaded account [%s] not valid\", k)\n\t\t\tcontinue\n\t\t}\n\t\t// Check if it is expired.\n\t\tvr := jwt.CreateValidationResults()\n\t\tclaims.Validate(vr)\n\t\tif vr.IsBlocking(true) {\n\t\t\ts.Warnf(\"Account [%s] has validation issues:\", k)\n\t\t\tfor _, v := range vr.Issues {\n\t\t\t\ts.Warnf(\" - %s\", v.Description)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Determines if we are in pre NATS 2.0 setup with no accounts.\nfunc (s *Server) globalAccountOnly() bool {\n\tvar hasOthers bool\n\n\tif s.trustedKeys != nil {\n\t\treturn false\n\t}\n\n\ts.mu.RLock()\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\t// Ignore global and system\n\t\tif acc == s.gacc || (s.sys != nil && acc == s.sys.account) {\n\t\t\treturn true\n\t\t}\n\t\thasOthers = true\n\t\treturn false\n\t})\n\ts.mu.RUnlock()\n\n\treturn !hasOthers\n}\n\n// Determines if this server is in standalone mode, meaning no routes or gateways.\nfunc (s *Server) standAloneMode() bool {\n\topts := s.getOpts()\n\treturn opts.Cluster.Port == 0 && opts.Gateway.Port == 0\n}\n\nfunc (s *Server) configuredRoutes() int {\n\treturn len(s.getOpts().Routes)\n}\n\n// activePeers is used in bootstrapping raft groups like the JetStream meta controller.\nfunc (s *Server) ActivePeers() (peers []string) {\n\ts.nodeToInfo.Range(func(k, v any) bool {\n\t\tsi := v.(nodeInfo)\n\t\tif !si.offline {\n\t\t\tpeers = append(peers, k.(string))\n\t\t}\n\t\treturn true\n\t})\n\treturn peers\n}\n\n// isTrustedIssuer will check that the issuer is a trusted public key.\n// This is used to make sure an account was signed by a trusted operator.\nfunc (s *Server) isTrustedIssuer(issuer string) bool {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\t// If we are not running in trusted mode and there is no issuer, that is ok.\n\tif s.trustedKeys == nil && issuer == _EMPTY_ {\n\t\treturn true\n\t}\n\tfor _, tk := range s.trustedKeys {\n\t\tif tk == issuer {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// processTrustedKeys will process binary stamped and\n// options-based trusted nkeys. Returns success.\nfunc (s *Server) processTrustedKeys() bool {\n\ts.strictSigningKeyUsage = map[string]struct{}{}\n\topts := s.getOpts()\n\tif trustedKeys != _EMPTY_ && !s.initStampedTrustedKeys() {\n\t\treturn false\n\t} else if opts.TrustedKeys != nil {\n\t\tfor _, key := range opts.TrustedKeys {\n\t\t\tif !nkeys.IsValidPublicOperatorKey(key) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\ts.trustedKeys = append([]string(nil), opts.TrustedKeys...)\n\t\tfor _, claim := range opts.TrustedOperators {\n\t\t\tif !claim.StrictSigningKeyUsage {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, key := range claim.SigningKeys {\n\t\t\t\ts.strictSigningKeyUsage[key] = struct{}{}\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// checkTrustedKeyString will check that the string is a valid array\n// of public operator nkeys.\nfunc checkTrustedKeyString(keys string) []string {\n\ttks := strings.Fields(keys)\n\tif len(tks) == 0 {\n\t\treturn nil\n\t}\n\t// Walk all the keys and make sure they are valid.\n\tfor _, key := range tks {\n\t\tif !nkeys.IsValidPublicOperatorKey(key) {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn tks\n}\n\n// initStampedTrustedKeys will check the stamped trusted keys\n// and will set the server field 'trustedKeys'. Returns whether\n// it succeeded or not.\nfunc (s *Server) initStampedTrustedKeys() bool {\n\t// Check to see if we have an override in options, which will cause us to fail.\n\tif len(s.getOpts().TrustedKeys) > 0 {\n\t\treturn false\n\t}\n\ttks := checkTrustedKeyString(trustedKeys)\n\tif len(tks) == 0 {\n\t\treturn false\n\t}\n\ts.trustedKeys = tks\n\treturn true\n}\n\n// PrintAndDie is exported for access in other packages.\nfunc PrintAndDie(msg string) {\n\tfmt.Fprintln(os.Stderr, msg)\n\tos.Exit(1)\n}\n\n// PrintServerAndExit will print our version and exit.\nfunc PrintServerAndExit() {\n\tfmt.Printf(\"nats-server: v%s\\n\", VERSION)\n\tos.Exit(0)\n}\n\n// ProcessCommandLineArgs takes the command line arguments\n// validating and setting flags for handling in case any\n// sub command was present.\nfunc ProcessCommandLineArgs(cmd *flag.FlagSet) (showVersion bool, showHelp bool, err error) {\n\tif len(cmd.Args()) > 0 {\n\t\targ := cmd.Args()[0]\n\t\tswitch strings.ToLower(arg) {\n\t\tcase \"version\":\n\t\t\treturn true, false, nil\n\t\tcase \"help\":\n\t\t\treturn false, true, nil\n\t\tdefault:\n\t\t\treturn false, false, fmt.Errorf(\"unrecognized command: %q\", arg)\n\t\t}\n\t}\n\n\treturn false, false, nil\n}\n\n// Public version.\nfunc (s *Server) Running() bool {\n\treturn s.isRunning()\n}\n\n// Protected check on running state\nfunc (s *Server) isRunning() bool {\n\treturn s.running.Load()\n}\n\nfunc (s *Server) logPid() error {\n\tpidStr := strconv.Itoa(os.Getpid())\n\treturn os.WriteFile(s.getOpts().PidFile, []byte(pidStr), defaultFilePerms)\n}\n\n// numReservedAccounts will return the number of reserved accounts configured in the server.\n// Currently this is 1, one for the global default account.\nfunc (s *Server) numReservedAccounts() int {\n\treturn 1\n}\n\n// NumActiveAccounts reports number of active accounts on this server.\nfunc (s *Server) NumActiveAccounts() int32 {\n\treturn atomic.LoadInt32(&s.activeAccounts)\n}\n\n// incActiveAccounts() just adds one under lock.\nfunc (s *Server) incActiveAccounts() {\n\tatomic.AddInt32(&s.activeAccounts, 1)\n}\n\n// decActiveAccounts() just subtracts one under lock.\nfunc (s *Server) decActiveAccounts() {\n\tatomic.AddInt32(&s.activeAccounts, -1)\n}\n\n// This should be used for testing only. Will be slow since we have to\n// range over all accounts in the sync.Map to count.\nfunc (s *Server) numAccounts() int {\n\tcount := 0\n\ts.mu.RLock()\n\ts.accounts.Range(func(k, v any) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\ts.mu.RUnlock()\n\treturn count\n}\n\n// NumLoadedAccounts returns the number of loaded accounts.\nfunc (s *Server) NumLoadedAccounts() int {\n\treturn s.numAccounts()\n}\n\n// LookupOrRegisterAccount will return the given account if known or create a new entry.\nfunc (s *Server) LookupOrRegisterAccount(name string) (account *Account, isNew bool) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif v, ok := s.accounts.Load(name); ok {\n\t\treturn v.(*Account), false\n\t}\n\tacc := NewAccount(name)\n\ts.registerAccountNoLock(acc)\n\treturn acc, true\n}\n\n// RegisterAccount will register an account. The account must be new\n// or this call will fail.\nfunc (s *Server) RegisterAccount(name string) (*Account, error) {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tif _, ok := s.accounts.Load(name); ok {\n\t\treturn nil, ErrAccountExists\n\t}\n\tacc := NewAccount(name)\n\ts.registerAccountNoLock(acc)\n\treturn acc, nil\n}\n\n// SetSystemAccount will set the internal system account.\n// If root operators are present it will also check validity.\nfunc (s *Server) SetSystemAccount(accName string) error {\n\t// Lookup from sync.Map first.\n\tif v, ok := s.accounts.Load(accName); ok {\n\t\treturn s.setSystemAccount(v.(*Account))\n\t}\n\n\t// If we are here we do not have local knowledge of this account.\n\t// Do this one by hand to return more useful error.\n\tac, jwt, err := s.fetchAccountClaims(accName)\n\tif err != nil {\n\t\treturn err\n\t}\n\tacc := s.buildInternalAccount(ac)\n\tacc.claimJWT = jwt\n\t// Due to race, we need to make sure that we are not\n\t// registering twice.\n\tif racc := s.registerAccount(acc); racc != nil {\n\t\treturn nil\n\t}\n\treturn s.setSystemAccount(acc)\n}\n\n// SystemAccount returns the system account if set.\nfunc (s *Server) SystemAccount() *Account {\n\treturn s.sysAcc.Load()\n}\n\n// GlobalAccount returns the global account.\n// Default clients will use the global account.\nfunc (s *Server) GlobalAccount() *Account {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.gacc\n}\n\n// SetDefaultSystemAccount will create a default system account if one is not present.\nfunc (s *Server) SetDefaultSystemAccount() error {\n\tif _, isNew := s.LookupOrRegisterAccount(DEFAULT_SYSTEM_ACCOUNT); !isNew {\n\t\treturn nil\n\t}\n\ts.Debugf(\"Created system account: %q\", DEFAULT_SYSTEM_ACCOUNT)\n\treturn s.SetSystemAccount(DEFAULT_SYSTEM_ACCOUNT)\n}\n\n// Assign a system account. Should only be called once.\n// This sets up a server to send and receive messages from\n// inside the server itself.\nfunc (s *Server) setSystemAccount(acc *Account) error {\n\tif acc == nil {\n\t\treturn ErrMissingAccount\n\t}\n\t// Don't try to fix this here.\n\tif acc.IsExpired() {\n\t\treturn ErrAccountExpired\n\t}\n\t// If we are running with trusted keys for an operator\n\t// make sure we check the account is legit.\n\tif !s.isTrustedIssuer(acc.Issuer) {\n\t\treturn ErrAccountValidation\n\t}\n\n\ts.mu.Lock()\n\n\tif s.sys != nil {\n\t\ts.mu.Unlock()\n\t\treturn ErrAccountExists\n\t}\n\n\t// This is here in an attempt to quiet the race detector and not have to place\n\t// locks on fast path for inbound messages and checking service imports.\n\tacc.mu.Lock()\n\tif acc.imports.services == nil {\n\t\tacc.imports.services = make(map[string][]*serviceImport)\n\t}\n\tacc.mu.Unlock()\n\n\ts.sys = &internal{\n\t\taccount: acc,\n\t\tclient: s.createInternalSystemClient(),\n\t\tseq: 1,\n\t\tsid: 1,\n\t\tservers: make(map[string]*serverUpdate),\n\t\treplies: make(map[string]msgHandler),\n\t\tsendq: newIPQueue[*pubMsg](s, \"System sendQ\"),\n\t\trecvq: newIPQueue[*inSysMsg](s, \"System recvQ\"),\n\t\trecvqp: newIPQueue[*inSysMsg](s, \"System recvQ Pings\"),\n\t\tresetCh: make(chan struct{}),\n\t\tsq: s.newSendQ(acc),\n\t\tstatsz: statsHBInterval,\n\t\torphMax: 5 * eventsHBInterval,\n\t\tchkOrph: 3 * eventsHBInterval,\n\t}\n\trecvq, recvqp := s.sys.recvq, s.sys.recvqp\n\ts.sys.wg.Add(1)\n\ts.mu.Unlock()\n\n\t// Store in atomic for fast lookup.\n\ts.sysAcc.Store(acc)\n\n\t// Register with the account.\n\ts.sys.client.registerWithAccount(acc)\n\n\ts.addSystemAccountExports(acc)\n\n\t// Start our internal loop to serialize outbound messages.\n\t// We do our own wg here since we will stop first during shutdown.\n\tgo s.internalSendLoop(&s.sys.wg)\n\n\t// Start the internal loop for inbound messages.\n\tgo s.internalReceiveLoop(recvq)\n\t// Start the internal loop for inbound STATSZ/Ping messages.\n\tgo s.internalReceiveLoop(recvqp)\n\n\t// Start up our general subscriptions\n\ts.initEventTracking()\n\n\t// Track for dead remote servers.\n\ts.wrapChk(s.startRemoteServerSweepTimer)()\n\n\t// Send out statsz updates periodically.\n\ts.wrapChk(s.startStatszTimer)()\n\n\t// If we have existing accounts make sure we enable account tracking.\n\ts.mu.Lock()\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\ts.enableAccountTracking(acc)\n\t\treturn true\n\t})\n\ts.mu.Unlock()\n\n\treturn nil\n}\n\n// Creates an internal system client.\nfunc (s *Server) createInternalSystemClient() *client {\n\treturn s.createInternalClient(SYSTEM)\n}\n\n// Creates an internal jetstream client.\nfunc (s *Server) createInternalJetStreamClient() *client {\n\treturn s.createInternalClient(JETSTREAM)\n}\n\n// Creates an internal client for Account.\nfunc (s *Server) createInternalAccountClient() *client {\n\treturn s.createInternalClient(ACCOUNT)\n}\n\n// Internal clients. kind should be SYSTEM, JETSTREAM or ACCOUNT\nfunc (s *Server) createInternalClient(kind int) *client {\n\tif !isInternalClient(kind) {\n\t\treturn nil\n\t}\n\tnow := time.Now()\n\tc := &client{srv: s, kind: kind, opts: internalOpts, msubs: -1, mpay: -1, start: now, last: now}\n\tc.initClient()\n\tc.echo = false\n\tc.headers = true\n\tc.flags.set(noReconnect)\n\treturn c\n}\n\n// Determine if accounts should track subscriptions for\n// efficient propagation.\n// Lock should be held on entry.\nfunc (s *Server) shouldTrackSubscriptions() bool {\n\topts := s.getOpts()\n\treturn (opts.Cluster.Port != 0 || opts.Gateway.Port != 0)\n}\n\n// Invokes registerAccountNoLock under the protection of the server lock.\n// That is, server lock is acquired/released in this function.\n// See registerAccountNoLock for comment on returned value.\nfunc (s *Server) registerAccount(acc *Account) *Account {\n\ts.mu.Lock()\n\tracc := s.registerAccountNoLock(acc)\n\ts.mu.Unlock()\n\treturn racc\n}\n\n// Helper to set the sublist based on preferences.\nfunc (s *Server) setAccountSublist(acc *Account) {\n\tif acc != nil && acc.sl == nil {\n\t\topts := s.getOpts()\n\t\tif opts != nil && opts.NoSublistCache {\n\t\t\tacc.sl = NewSublistNoCache()\n\t\t} else {\n\t\t\tacc.sl = NewSublistWithCache()\n\t\t}\n\t}\n}\n\n// Registers an account in the server.\n// Due to some locking considerations, we may end-up trying\n// to register the same account twice. This function will\n// then return the already registered account.\n// Lock should be held on entry.\nfunc (s *Server) registerAccountNoLock(acc *Account) *Account {\n\t// We are under the server lock. Lookup from map, if present\n\t// return existing account.\n\tif a, _ := s.accounts.Load(acc.Name); a != nil {\n\t\ts.tmpAccounts.Delete(acc.Name)\n\t\treturn a.(*Account)\n\t}\n\t// Finish account setup and store.\n\ts.setAccountSublist(acc)\n\n\tacc.mu.Lock()\n\ts.setRouteInfo(acc)\n\tif acc.clients == nil {\n\t\tacc.clients = make(map[*client]struct{})\n\t}\n\n\t// If we are capable of routing we will track subscription\n\t// information for efficient interest propagation.\n\t// During config reload, it is possible that account was\n\t// already created (global account), so use locking and\n\t// make sure we create only if needed.\n\t// TODO(dlc)- Double check that we need this for GWs.\n\tif acc.rm == nil && s.opts != nil && s.shouldTrackSubscriptions() {\n\t\tacc.rm = make(map[string]int32)\n\t\tacc.lqws = make(map[string]int32)\n\t}\n\tacc.srv = s\n\tacc.updated = time.Now()\n\taccName := acc.Name\n\tjsEnabled := len(acc.jsLimits) > 0\n\tacc.mu.Unlock()\n\n\tif opts := s.getOpts(); opts != nil && len(opts.JsAccDefaultDomain) > 0 {\n\t\tif defDomain, ok := opts.JsAccDefaultDomain[accName]; ok {\n\t\t\tif jsEnabled {\n\t\t\t\ts.Warnf(\"Skipping Default Domain %q, set for JetStream enabled account %q\", defDomain, accName)\n\t\t\t} else if defDomain != _EMPTY_ {\n\t\t\t\tfor src, dest := range generateJSMappingTable(defDomain) {\n\t\t\t\t\t// flip src and dest around so the domain is inserted\n\t\t\t\t\ts.Noticef(\"Adding default domain mapping %q -> %q to account %q %p\", dest, src, accName, acc)\n\t\t\t\t\tif err := acc.AddMapping(dest, src); err != nil {\n\t\t\t\t\t\ts.Errorf(\"Error adding JetStream default domain mapping: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\ts.accounts.Store(acc.Name, acc)\n\ts.tmpAccounts.Delete(acc.Name)\n\ts.enableAccountTracking(acc)\n\n\t// Can not have server lock here.\n\ts.mu.Unlock()\n\ts.registerSystemImports(acc)\n\t// Starting 2.9.0, we are phasing out the optimistic mode, so change\n\t// the account to interest-only mode (except if instructed not to do\n\t// it in some tests).\n\tif s.gateway.enabled && !gwDoNotForceInterestOnlyMode {\n\t\ts.switchAccountToInterestMode(acc.GetName())\n\t}\n\ts.mu.Lock()\n\n\treturn nil\n}\n\n// Sets the account's routePoolIdx depending on presence or not of\n// pooling or per-account routes. Also updates a map used by\n// gateway code to retrieve a route based on some route hash.\n//\n// Both Server and Account lock held on entry.\nfunc (s *Server) setRouteInfo(acc *Account) {\n\t// If there is a dedicated route configured for this account\n\tif _, ok := s.accRoutes[acc.Name]; ok {\n\t\t// We want the account name to be in the map, but we don't\n\t\t// need a value (we could store empty string)\n\t\ts.accRouteByHash.Store(acc.Name, nil)\n\t\t// Set the route pool index to -1 so that it is easy when\n\t\t// ranging over accounts to exclude those accounts when\n\t\t// trying to get accounts for a given pool index.\n\t\tacc.routePoolIdx = accDedicatedRoute\n\t} else {\n\t\t// If pool size more than 1, we will compute a hash code and\n\t\t// use modulo to assign to an index of the pool slice. For 1\n\t\t// and below, all accounts will be bound to the single connection\n\t\t// at index 0.\n\t\tacc.routePoolIdx = computeRoutePoolIdx(s.routesPoolSize, acc.Name)\n\t\tif s.routesPoolSize > 1 {\n\t\t\ts.accRouteByHash.Store(acc.Name, acc.routePoolIdx)\n\t\t}\n\t}\n}\n\n// lookupAccount is a function to return the account structure\n// associated with an account name.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) lookupAccount(name string) (*Account, error) {\n\treturn s.lookupOrFetchAccount(name, true)\n}\n\n// lookupOrFetchAccount is a function to return the account structure\n// associated with an account name.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) lookupOrFetchAccount(name string, fetch bool) (*Account, error) {\n\tvar acc *Account\n\tif v, ok := s.accounts.Load(name); ok {\n\t\tacc = v.(*Account)\n\t}\n\tif acc != nil {\n\t\t// If we are expired and we have a resolver, then\n\t\t// return the latest information from the resolver.\n\t\tif acc.IsExpired() {\n\t\t\ts.Debugf(\"Requested account [%s] has expired\", name)\n\t\t\tif s.AccountResolver() != nil && fetch {\n\t\t\t\tif err := s.updateAccount(acc); err != nil {\n\t\t\t\t\t// This error could mask expired, so just return expired here.\n\t\t\t\t\treturn nil, ErrAccountExpired\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn nil, ErrAccountExpired\n\t\t\t}\n\t\t}\n\t\treturn acc, nil\n\t}\n\t// If we have a resolver see if it can fetch the account.\n\tif s.AccountResolver() == nil || !fetch {\n\t\treturn nil, ErrMissingAccount\n\t}\n\treturn s.fetchAccount(name)\n}\n\n// LookupAccount is a public function to return the account structure\n// associated with name.\nfunc (s *Server) LookupAccount(name string) (*Account, error) {\n\treturn s.lookupAccount(name)\n}\n\n// This will fetch new claims and if found update the account with new claims.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) updateAccount(acc *Account) error {\n\tacc.mu.RLock()\n\t// TODO(dlc) - Make configurable\n\tif !acc.incomplete && time.Since(acc.updated) < time.Second {\n\t\tacc.mu.RUnlock()\n\t\ts.Debugf(\"Requested account update for [%s] ignored, too soon\", acc.Name)\n\t\treturn ErrAccountResolverUpdateTooSoon\n\t}\n\tacc.mu.RUnlock()\n\tclaimJWT, err := s.fetchRawAccountClaims(acc.Name)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn s.updateAccountWithClaimJWT(acc, claimJWT)\n}\n\n// updateAccountWithClaimJWT will check and apply the claim update.\n// Lock MUST NOT be held upon entry.\nfunc (s *Server) updateAccountWithClaimJWT(acc *Account, claimJWT string) error {\n\tif acc == nil {\n\t\treturn ErrMissingAccount\n\t}\n\tacc.mu.RLock()\n\tsameClaim := acc.claimJWT != _EMPTY_ && acc.claimJWT == claimJWT && !acc.incomplete\n\tacc.mu.RUnlock()\n\tif sameClaim {\n\t\ts.Debugf(\"Requested account update for [%s], same claims detected\", acc.Name)\n\t\treturn nil\n\t}\n\taccClaims, _, err := s.verifyAccountClaims(claimJWT)\n\tif err == nil && accClaims != nil {\n\t\tacc.mu.Lock()\n\t\t// if an account is updated with a different operator signing key, we want to\n\t\t// show a consistent issuer.\n\t\tacc.Issuer = accClaims.Issuer\n\t\tif acc.Name != accClaims.Subject {\n\t\t\tacc.mu.Unlock()\n\t\t\treturn ErrAccountValidation\n\t\t}\n\t\tacc.mu.Unlock()\n\t\ts.UpdateAccountClaims(acc, accClaims)\n\t\tacc.mu.Lock()\n\t\t// needs to be set after update completed.\n\t\t// This causes concurrent calls to return with sameClaim=true if the change is effective.\n\t\tacc.claimJWT = claimJWT\n\t\tacc.mu.Unlock()\n\t\treturn nil\n\t}\n\treturn err\n}\n\n// fetchRawAccountClaims will grab raw account claims iff we have a resolver.\n// Lock is NOT held upon entry.\nfunc (s *Server) fetchRawAccountClaims(name string) (string, error) {\n\taccResolver := s.AccountResolver()\n\tif accResolver == nil {\n\t\treturn _EMPTY_, ErrNoAccountResolver\n\t}\n\t// Need to do actual Fetch\n\tstart := time.Now()\n\tclaimJWT, err := fetchAccount(accResolver, name)\n\tfetchTime := time.Since(start)\n\tif fetchTime > time.Second {\n\t\ts.Warnf(\"Account [%s] fetch took %v\", name, fetchTime)\n\t} else {\n\t\ts.Debugf(\"Account [%s] fetch took %v\", name, fetchTime)\n\t}\n\tif err != nil {\n\t\ts.Warnf(\"Account fetch failed: %v\", err)\n\t\treturn \"\", err\n\t}\n\treturn claimJWT, nil\n}\n\n// fetchAccountClaims will attempt to fetch new claims if a resolver is present.\n// Lock is NOT held upon entry.\nfunc (s *Server) fetchAccountClaims(name string) (*jwt.AccountClaims, string, error) {\n\tclaimJWT, err := s.fetchRawAccountClaims(name)\n\tif err != nil {\n\t\treturn nil, _EMPTY_, err\n\t}\n\tvar claim *jwt.AccountClaims\n\tclaim, claimJWT, err = s.verifyAccountClaims(claimJWT)\n\tif claim != nil && claim.Subject != name {\n\t\treturn nil, _EMPTY_, ErrAccountValidation\n\t}\n\treturn claim, claimJWT, err\n}\n\n// verifyAccountClaims will decode and validate any account claims.\nfunc (s *Server) verifyAccountClaims(claimJWT string) (*jwt.AccountClaims, string, error) {\n\taccClaims, err := jwt.DecodeAccountClaims(claimJWT)\n\tif err != nil {\n\t\treturn nil, _EMPTY_, err\n\t}\n\tif !s.isTrustedIssuer(accClaims.Issuer) {\n\t\treturn nil, _EMPTY_, ErrAccountValidation\n\t}\n\tvr := jwt.CreateValidationResults()\n\taccClaims.Validate(vr)\n\tif vr.IsBlocking(true) {\n\t\treturn nil, _EMPTY_, ErrAccountValidation\n\t}\n\treturn accClaims, claimJWT, nil\n}\n\n// This will fetch an account from a resolver if defined.\n// Lock is NOT held upon entry.\nfunc (s *Server) fetchAccount(name string) (*Account, error) {\n\taccClaims, claimJWT, err := s.fetchAccountClaims(name)\n\tif accClaims == nil {\n\t\treturn nil, err\n\t}\n\tacc := s.buildInternalAccount(accClaims)\n\t// Due to possible race, if registerAccount() returns a non\n\t// nil account, it means the same account was already\n\t// registered and we should use this one.\n\tif racc := s.registerAccount(acc); racc != nil {\n\t\t// Update with the new claims in case they are new.\n\t\tif err = s.updateAccountWithClaimJWT(racc, claimJWT); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn racc, nil\n\t}\n\t// The sub imports may have been setup but will not have had their\n\t// subscriptions properly setup. Do that here.\n\tvar needImportSubs bool\n\n\tacc.mu.Lock()\n\tacc.claimJWT = claimJWT\n\tif len(acc.imports.services) > 0 {\n\t\tif acc.ic == nil {\n\t\t\tacc.ic = s.createInternalAccountClient()\n\t\t\tacc.ic.acc = acc\n\t\t}\n\t\tneedImportSubs = true\n\t}\n\tacc.mu.Unlock()\n\n\t// Do these outside the lock.\n\tif needImportSubs {\n\t\tacc.addAllServiceImportSubs()\n\t}\n\n\treturn acc, nil\n}\n\n// Start up the server, this will not block.\n//\n// WaitForShutdown can be used to block and wait for the server to shutdown properly if needed\n// after calling s.Shutdown()\nfunc (s *Server) Start() {\n\ts.Noticef(\"Starting nats-server\")\n\n\tgc := gitCommit\n\tif gc == _EMPTY_ {\n\t\tgc = \"not set\"\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Capture if this server is a leaf that has no cluster, so we don't\n\t// display the cluster name if that is the case.\n\ts.mu.RLock()\n\tleafNoCluster := s.leafNoCluster\n\ts.mu.RUnlock()\n\n\tvar clusterName string\n\tif !leafNoCluster {\n\t\tclusterName = s.ClusterName()\n\t}\n\n\ts.Noticef(\" Version: %s\", VERSION)\n\ts.Noticef(\" Git: [%s]\", gc)\n\ts.Debugf(\" Go build: %s\", s.info.GoVersion)\n\tif clusterName != _EMPTY_ {\n\t\ts.Noticef(\" Cluster: %s\", clusterName)\n\t}\n\ts.Noticef(\" Name: %s\", s.info.Name)\n\tif opts.JetStream {\n\t\ts.Noticef(\" Node: %s\", getHash(s.info.Name))\n\t}\n\ts.Noticef(\" ID: %s\", s.info.ID)\n\n\tdefer s.Noticef(\"Server is ready\")\n\n\t// Check for insecure configurations.\n\ts.checkAuthforWarnings()\n\n\t// Avoid RACE between Start() and Shutdown()\n\ts.running.Store(true)\n\ts.mu.Lock()\n\t// Update leafNodeEnabled in case options have changed post NewServer()\n\t// and before Start() (we should not be able to allow that, but server has\n\t// direct reference to user-provided options - at least before a Reload() is\n\t// performed.\n\ts.leafNodeEnabled = opts.LeafNode.Port != 0 || len(opts.LeafNode.Remotes) > 0\n\ts.mu.Unlock()\n\n\ts.grMu.Lock()\n\ts.grRunning = true\n\ts.grMu.Unlock()\n\n\ts.startRateLimitLogExpiration()\n\n\t// Pprof http endpoint for the profiler.\n\tif opts.ProfPort != 0 {\n\t\ts.StartProfiler()\n\t} else {\n\t\t// It's still possible to access this profile via a SYS endpoint, so set\n\t\t// this anyway. (Otherwise StartProfiler would have called it.)\n\t\ts.setBlockProfileRate(opts.ProfBlockRate)\n\t}\n\n\tif opts.ConfigFile != _EMPTY_ {\n\t\tvar cd string\n\t\tif opts.configDigest != \"\" {\n\t\t\tcd = fmt.Sprintf(\"(%s)\", opts.configDigest)\n\t\t}\n\t\ts.Noticef(\"Using configuration file: %s %s\", opts.ConfigFile, cd)\n\t}\n\n\thasOperators := len(opts.TrustedOperators) > 0\n\tif hasOperators {\n\t\ts.Noticef(\"Trusted Operators\")\n\t}\n\tfor _, opc := range opts.TrustedOperators {\n\t\ts.Noticef(\" System : %q\", opc.Audience)\n\t\ts.Noticef(\" Operator: %q\", opc.Name)\n\t\ts.Noticef(\" Issued : %v\", time.Unix(opc.IssuedAt, 0))\n\t\tswitch opc.Expires {\n\t\tcase 0:\n\t\t\ts.Noticef(\" Expires : Never\")\n\t\tdefault:\n\t\t\ts.Noticef(\" Expires : %v\", time.Unix(opc.Expires, 0))\n\t\t}\n\t}\n\tif hasOperators && opts.SystemAccount == _EMPTY_ {\n\t\ts.Warnf(\"Trusted Operators should utilize a System Account\")\n\t}\n\tif opts.MaxPayload > MAX_PAYLOAD_MAX_SIZE {\n\t\ts.Warnf(\"Maximum payloads over %v are generally discouraged and could lead to poor performance\",\n\t\t\tfriendlyBytes(int64(MAX_PAYLOAD_MAX_SIZE)))\n\t}\n\n\tif len(opts.JsAccDefaultDomain) > 0 {\n\t\ts.Warnf(\"The option `default_js_domain` is a temporary backwards compatibility measure and will be removed\")\n\t}\n\n\t// If we have a memory resolver, check the accounts here for validation exceptions.\n\t// This allows them to be logged right away vs when they are accessed via a client.\n\tif hasOperators && len(opts.resolverPreloads) > 0 {\n\t\ts.checkResolvePreloads()\n\t}\n\n\t// Log the pid to a file.\n\tif opts.PidFile != _EMPTY_ {\n\t\tif err := s.logPid(); err != nil {\n\t\t\ts.Fatalf(\"Could not write pidfile: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Setup system account which will start the eventing stack.\n\tif sa := opts.SystemAccount; sa != _EMPTY_ {\n\t\tif err := s.SetSystemAccount(sa); err != nil {\n\t\t\ts.Fatalf(\"Can't set system account: %v\", err)\n\t\t\treturn\n\t\t}\n\t} else if !opts.NoSystemAccount {\n\t\t// We will create a default system account here.\n\t\ts.SetDefaultSystemAccount()\n\t}\n\n\t// Start monitoring before enabling other subsystems of the\n\t// server to be able to monitor during startup.\n\tif err := s.StartMonitoring(); err != nil {\n\t\ts.Fatalf(\"Can't start monitoring: %v\", err)\n\t\treturn\n\t}\n\n\t// Start up resolver machinery.\n\tif ar := s.AccountResolver(); ar != nil {\n\t\tif err := ar.Start(s); err != nil {\n\t\t\ts.Fatalf(\"Could not start resolver: %v\", err)\n\t\t\treturn\n\t\t}\n\t\t// In operator mode, when the account resolver depends on an external system and\n\t\t// the system account is the bootstrapping account, start fetching it.\n\t\tif len(opts.TrustedOperators) == 1 && opts.SystemAccount != _EMPTY_ && opts.SystemAccount != DEFAULT_SYSTEM_ACCOUNT {\n\t\t\topts := s.getOpts()\n\t\t\t_, isMemResolver := ar.(*MemAccResolver)\n\t\t\tif v, ok := s.accounts.Load(opts.SystemAccount); !isMemResolver && ok && v.(*Account).claimJWT == _EMPTY_ {\n\t\t\t\ts.Noticef(\"Using bootstrapping system account\")\n\t\t\t\ts.startGoRoutine(func() {\n\t\t\t\t\tdefer s.grWG.Done()\n\t\t\t\t\tt := time.NewTicker(time.Second)\n\t\t\t\t\tdefer t.Stop()\n\t\t\t\t\tfor {\n\t\t\t\t\t\tselect {\n\t\t\t\t\t\tcase <-s.quitCh:\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\tcase <-t.C:\n\t\t\t\t\t\t\tsacc := s.SystemAccount()\n\t\t\t\t\t\t\tif claimJWT, err := fetchAccount(ar, opts.SystemAccount); err != nil {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t} else if err = s.updateAccountWithClaimJWT(sacc, claimJWT); err != nil {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\ts.Noticef(\"System account fetched and updated\")\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\n\t// Start expiration of mapped GW replies, regardless if\n\t// this server is configured with gateway or not.\n\ts.startGWReplyMapExpiration()\n\n\t// Check if JetStream has been enabled. This needs to be after\n\t// the system account setup above. JetStream will create its\n\t// own system account if one is not present.\n\tif opts.JetStream {\n\t\t// Make sure someone is not trying to enable on the system account.\n\t\tif sa := s.SystemAccount(); sa != nil && len(sa.jsLimits) > 0 {\n\t\t\ts.Fatalf(\"Not allowed to enable JetStream on the system account\")\n\t\t}\n\t\tcfg := &JetStreamConfig{\n\t\t\tStoreDir: opts.StoreDir,\n\t\t\tSyncInterval: opts.SyncInterval,\n\t\t\tSyncAlways: opts.SyncAlways,\n\t\t\tStrict: !opts.NoJetStreamStrict,\n\t\t\tMaxMemory: opts.JetStreamMaxMemory,\n\t\t\tMaxStore: opts.JetStreamMaxStore,\n\t\t\tDomain: opts.JetStreamDomain,\n\t\t\tCompressOK: true,\n\t\t\tUniqueTag: opts.JetStreamUniqueTag,\n\t\t}\n\t\tif err := s.EnableJetStream(cfg); err != nil {\n\t\t\ts.Fatalf(\"Can't start JetStream: %v\", err)\n\t\t\treturn\n\t\t}\n\t} else {\n\t\t// Check to see if any configured accounts have JetStream enabled.\n\t\tsa, ga := s.SystemAccount(), s.GlobalAccount()\n\t\tvar hasSys, hasGlobal bool\n\t\tvar total int\n\n\t\ts.accounts.Range(func(k, v any) bool {\n\t\t\ttotal++\n\t\t\tacc := v.(*Account)\n\t\t\tif acc == sa {\n\t\t\t\thasSys = true\n\t\t\t} else if acc == ga {\n\t\t\t\thasGlobal = true\n\t\t\t}\n\t\t\tacc.mu.RLock()\n\t\t\thasJs := len(acc.jsLimits) > 0\n\t\t\tacc.mu.RUnlock()\n\t\t\tif hasJs {\n\t\t\t\ts.checkJetStreamExports()\n\t\t\t\tacc.enableAllJetStreamServiceImportsAndMappings()\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\t// If we only have the system account and the global account and we are not standalone,\n\t\t// go ahead and enable JS on $G in case we are in simple mixed mode setup.\n\t\tif total == 2 && hasSys && hasGlobal && !s.standAloneMode() {\n\t\t\tga.mu.Lock()\n\t\t\tga.jsLimits = map[string]JetStreamAccountLimits{\n\t\t\t\t_EMPTY_: dynamicJSAccountLimits,\n\t\t\t}\n\t\t\tga.mu.Unlock()\n\t\t\ts.checkJetStreamExports()\n\t\t\tga.enableAllJetStreamServiceImportsAndMappings()\n\t\t}\n\t}\n\n\t// Delayed API response handling. Start regardless of JetStream being\n\t// currently configured or not (since it can be enabled/disabled with\n\t// configuration reload).\n\ts.startGoRoutine(s.delayedAPIResponder)\n\n\t// Start OCSP Stapling monitoring for TLS certificates if enabled. Hook TLS handshake for\n\t// OCSP check on peers (LEAF and CLIENT kind) if enabled.\n\ts.startOCSPMonitoring()\n\n\t// Configure OCSP Response Cache for peer OCSP checks if enabled.\n\ts.initOCSPResponseCache()\n\n\t// Start up gateway if needed. Do this before starting the routes, because\n\t// we want to resolve the gateway host:port so that this information can\n\t// be sent to other routes.\n\tif opts.Gateway.Port != 0 {\n\t\ts.startGateways()\n\t}\n\n\t// Start websocket server if needed. Do this before starting the routes, and\n\t// leaf node because we want to resolve the gateway host:port so that this\n\t// information can be sent to other routes.\n\tif opts.Websocket.Port != 0 {\n\t\ts.startWebsocketServer()\n\t}\n\n\t// Start up listen if we want to accept leaf node connections.\n\tif opts.LeafNode.Port != 0 {\n\t\t// Will resolve or assign the advertise address for the leafnode listener.\n\t\t// We need that in StartRouting().\n\t\ts.startLeafNodeAcceptLoop()\n\t}\n\n\t// Solicit remote servers for leaf node connections.\n\tif len(opts.LeafNode.Remotes) > 0 {\n\t\ts.solicitLeafNodeRemotes(opts.LeafNode.Remotes)\n\t}\n\n\t// TODO (ik): I wanted to refactor this by starting the client\n\t// accept loop first, that is, it would resolve listen spec\n\t// in place, but start the accept-for-loop in a different go\n\t// routine. This would get rid of the synchronization between\n\t// this function and StartRouting, which I also would have wanted\n\t// to refactor, but both AcceptLoop() and StartRouting() have\n\t// been exported and not sure if that would break users using them.\n\t// We could mark them as deprecated and remove in a release or two...\n\n\t// The Routing routine needs to wait for the client listen\n\t// port to be opened and potential ephemeral port selected.\n\tclientListenReady := make(chan struct{})\n\n\t// MQTT\n\tif opts.MQTT.Port != 0 {\n\t\ts.startMQTT()\n\t}\n\n\t// Start up routing as well if needed.\n\tif opts.Cluster.Port != 0 {\n\t\ts.startGoRoutine(func() {\n\t\t\ts.StartRouting(clientListenReady)\n\t\t})\n\t}\n\n\tif opts.PortsFileDir != _EMPTY_ {\n\t\ts.logPorts()\n\t}\n\n\tif opts.TLSRateLimit > 0 {\n\t\ts.startGoRoutine(s.logRejectedTLSConns)\n\t}\n\n\t// We've finished starting up.\n\tclose(s.startupComplete)\n\n\t// Wait for clients.\n\tif !opts.DontListen {\n\t\ts.AcceptLoop(clientListenReady)\n\t}\n\n\t// Bring OSCP Response cache online after accept loop started in anticipation of NATS-enabled cache types\n\ts.startOCSPResponseCache()\n}\n\nfunc (s *Server) isShuttingDown() bool {\n\treturn s.shutdown.Load()\n}\n\n// Shutdown will shutdown the server instance by kicking out the AcceptLoop\n// and closing all associated clients.\nfunc (s *Server) Shutdown() {\n\tif s == nil {\n\t\treturn\n\t}\n\t// This is for JetStream R1 Pull Consumers to allow signaling\n\t// that pending pull requests are invalid.\n\ts.signalPullConsumers()\n\n\t// Transfer off any raft nodes that we are a leader by stepping them down.\n\ts.stepdownRaftNodes()\n\n\t// Shutdown the eventing system as needed.\n\t// This is done first to send out any messages for\n\t// account status. We will also clean up any\n\t// eventing items associated with accounts.\n\ts.shutdownEventing()\n\n\t// Prevent issues with multiple calls.\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\ts.mu.Lock()\n\ts.Noticef(\"Initiating Shutdown...\")\n\n\taccRes := s.accResolver\n\n\topts := s.getOpts()\n\n\ts.shutdown.Store(true)\n\ts.running.Store(false)\n\ts.grMu.Lock()\n\ts.grRunning = false\n\ts.grMu.Unlock()\n\ts.mu.Unlock()\n\n\tif accRes != nil {\n\t\taccRes.Close()\n\t}\n\n\t// Now check and shutdown jetstream.\n\ts.shutdownJetStream()\n\n\t// Now shutdown the nodes\n\ts.shutdownRaftNodes()\n\n\ts.mu.Lock()\n\tconns := make(map[uint64]*client)\n\n\t// Copy off the clients\n\tfor i, c := range s.clients {\n\t\tconns[i] = c\n\t}\n\t// Copy off the connections that are not yet registered\n\t// in s.routes, but for which the readLoop has started\n\ts.grMu.Lock()\n\tfor i, c := range s.grTmpClients {\n\t\tconns[i] = c\n\t}\n\ts.grMu.Unlock()\n\t// Copy off the routes\n\ts.forEachRoute(func(r *client) {\n\t\tr.mu.Lock()\n\t\tconns[r.cid] = r\n\t\tr.mu.Unlock()\n\t})\n\t// Copy off the gateways\n\ts.getAllGatewayConnections(conns)\n\n\t// Copy off the leaf nodes\n\tfor i, c := range s.leafs {\n\t\tconns[i] = c\n\t}\n\n\t// Number of done channel responses we expect.\n\tdoneExpected := 0\n\n\t// Kick client AcceptLoop()\n\tif s.listener != nil {\n\t\tdoneExpected++\n\t\ts.listener.Close()\n\t\ts.listener = nil\n\t}\n\n\t// Kick websocket server\n\tdoneExpected += s.closeWebsocketServer()\n\n\t// Kick MQTT accept loop\n\tif s.mqtt.listener != nil {\n\t\tdoneExpected++\n\t\ts.mqtt.listener.Close()\n\t\ts.mqtt.listener = nil\n\t}\n\n\t// Kick leafnodes AcceptLoop()\n\tif s.leafNodeListener != nil {\n\t\tdoneExpected++\n\t\ts.leafNodeListener.Close()\n\t\ts.leafNodeListener = nil\n\t}\n\n\t// Kick route AcceptLoop()\n\tif s.routeListener != nil {\n\t\tdoneExpected++\n\t\ts.routeListener.Close()\n\t\ts.routeListener = nil\n\t}\n\n\t// Kick Gateway AcceptLoop()\n\tif s.gatewayListener != nil {\n\t\tdoneExpected++\n\t\ts.gatewayListener.Close()\n\t\ts.gatewayListener = nil\n\t}\n\n\t// Kick HTTP monitoring if its running\n\tif s.http != nil {\n\t\tdoneExpected++\n\t\ts.http.Close()\n\t\ts.http = nil\n\t}\n\n\t// Kick Profiling if its running\n\tif s.profiler != nil {\n\t\tdoneExpected++\n\t\ts.profiler.Close()\n\t}\n\n\ts.mu.Unlock()\n\n\t// Release go routines that wait on that channel\n\tclose(s.quitCh)\n\n\t// Close client and route connections\n\tfor _, c := range conns {\n\t\tc.setNoReconnect()\n\t\tc.closeConnection(ServerShutdown)\n\t}\n\n\t// Block until the accept loops exit\n\tfor doneExpected > 0 {\n\t\t<-s.done\n\t\tdoneExpected--\n\t}\n\n\t// Wait for go routines to be done.\n\ts.grWG.Wait()\n\n\tif opts.PortsFileDir != _EMPTY_ {\n\t\ts.deletePortsFile(opts.PortsFileDir)\n\t}\n\n\ts.Noticef(\"Server Exiting..\")\n\n\t// Stop OCSP Response Cache\n\tif s.ocsprc != nil {\n\t\ts.ocsprc.Stop(s)\n\t}\n\n\t// Close logger if applicable. It allows tests on Windows\n\t// to be able to do proper cleanup (delete log file).\n\ts.logging.RLock()\n\tlog := s.logging.logger\n\ts.logging.RUnlock()\n\tif log != nil {\n\t\tif l, ok := log.(*logger.Logger); ok {\n\t\t\tl.Close()\n\t\t}\n\t}\n\t// Notify that the shutdown is complete\n\tclose(s.shutdownComplete)\n}\n\n// Close the websocket server if running. If so, returns 1, else 0.\n// Server lock held on entry.\nfunc (s *Server) closeWebsocketServer() int {\n\tws := &s.websocket\n\tws.mu.Lock()\n\ths := ws.server\n\tif hs != nil {\n\t\tws.server = nil\n\t\tws.listener = nil\n\t}\n\tws.mu.Unlock()\n\tif hs != nil {\n\t\ths.Close()\n\t\treturn 1\n\t}\n\treturn 0\n}\n\n// WaitForShutdown will block until the server has been fully shutdown.\nfunc (s *Server) WaitForShutdown() {\n\t<-s.shutdownComplete\n}\n\n// AcceptLoop is exported for easier testing.\nfunc (s *Server) AcceptLoop(clr chan struct{}) {\n\t// If we were to exit before the listener is setup properly,\n\t// make sure we close the channel.\n\tdefer func() {\n\t\tif clr != nil {\n\t\t\tclose(clr)\n\t\t}\n\t}()\n\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Setup state that can enable shutdown\n\ts.mu.Lock()\n\thp := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))\n\tl, e := s.getServerListener(hp)\n\ts.listenerErr = e\n\tif e != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Error listening on port: %s, %q\", hp, e)\n\t\treturn\n\t}\n\ts.Noticef(\"Listening for client connections on %s\",\n\t\tnet.JoinHostPort(opts.Host, strconv.Itoa(l.Addr().(*net.TCPAddr).Port)))\n\n\t// Alert if PROXY protocol is enabled\n\tif opts.ProxyProtocol {\n\t\ts.Noticef(\"PROXY protocol enabled for client connections\")\n\t}\n\n\t// Alert of TLS enabled.\n\tif opts.TLSConfig != nil {\n\t\ts.Noticef(\"TLS required for client connections\")\n\t\tif opts.TLSHandshakeFirst && opts.TLSHandshakeFirstFallback == 0 {\n\t\t\ts.Warnf(\"Clients that are not using \\\"TLS Handshake First\\\" option will fail to connect\")\n\t\t}\n\t}\n\n\t// If server was started with RANDOM_PORT (-1), opts.Port would be equal\n\t// to 0 at the beginning this function. So we need to get the actual port\n\tif opts.Port == 0 {\n\t\t// Write resolved port back to options.\n\t\topts.Port = l.Addr().(*net.TCPAddr).Port\n\t}\n\n\t// Now that port has been set (if it was set to RANDOM), set the\n\t// server's info Host/Port with either values from Options or\n\t// ClientAdvertise.\n\tif err := s.setInfoHostPort(); err != nil {\n\t\ts.Fatalf(\"Error setting server INFO with ClientAdvertise value of %s, err=%v\", opts.ClientAdvertise, err)\n\t\tl.Close()\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\t// Keep track of client connect URLs. We may need them later.\n\ts.clientConnectURLs = s.getClientConnectURLs()\n\ts.listener = l\n\n\tgo s.acceptConnections(l, \"Client\", func(conn net.Conn) { s.createClient(conn) },\n\t\tfunc(_ error) bool {\n\t\t\tif s.isLameDuckMode() {\n\t\t\t\t// Signal that we are not accepting new clients\n\t\t\t\ts.ldmCh <- true\n\t\t\t\t// Now wait for the Shutdown...\n\t\t\t\t<-s.quitCh\n\t\t\t\treturn true\n\t\t\t}\n\t\t\treturn false\n\t\t})\n\ts.mu.Unlock()\n\n\t// Let the caller know that we are ready\n\tclose(clr)\n\tclr = nil\n}\n\n// getServerListener returns a network listener for the given host-port address.\n// If the Server already has an active listener (s.listener), it returns that listener\n// along with any previous error (s.listenerErr). Otherwise, it creates and returns\n// a new TCP listener on the specified address using natsListen.\nfunc (s *Server) getServerListener(hp string) (net.Listener, error) {\n\tif s.listener != nil {\n\t\treturn s.listener, s.listenerErr\n\t}\n\n\treturn natsListen(\"tcp\", hp)\n}\n\n// InProcessConn returns an in-process connection to the server,\n// avoiding the need to use a TCP listener for local connectivity\n// within the same process. This can be used regardless of the\n// state of the DontListen option.\nfunc (s *Server) InProcessConn() (net.Conn, error) {\n\tpl, pr := net.Pipe()\n\tif !s.startGoRoutine(func() {\n\t\ts.createClientInProcess(pl)\n\t\ts.grWG.Done()\n\t}) {\n\t\tpl.Close()\n\t\tpr.Close()\n\t\treturn nil, fmt.Errorf(\"failed to create connection\")\n\t}\n\treturn pr, nil\n}\n\nfunc (s *Server) acceptConnections(l net.Listener, acceptName string, createFunc func(conn net.Conn), errFunc func(err error) bool) {\n\ttmpDelay := ACCEPT_MIN_SLEEP\n\n\tfor {\n\t\tconn, err := l.Accept()\n\t\tif err != nil {\n\t\t\tif errFunc != nil && errFunc(err) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif tmpDelay = s.acceptError(acceptName, err, tmpDelay); tmpDelay < 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\ttmpDelay = ACCEPT_MIN_SLEEP\n\t\tif !s.startGoRoutine(func() {\n\t\t\ts.reloadMu.RLock()\n\t\t\tcreateFunc(conn)\n\t\t\ts.reloadMu.RUnlock()\n\t\t\ts.grWG.Done()\n\t\t}) {\n\t\t\tconn.Close()\n\t\t}\n\t}\n\ts.Debugf(acceptName + \" accept loop exiting..\")\n\ts.done <- true\n}\n\n// This function sets the server's info Host/Port based on server Options.\n// Note that this function may be called during config reload, this is why\n// Host/Port may be reset to original Options if the ClientAdvertise option\n// is not set (since it may have previously been).\nfunc (s *Server) setInfoHostPort() error {\n\t// When this function is called, opts.Port is set to the actual listen\n\t// port (if option was originally set to RANDOM), even during a config\n\t// reload. So use of s.opts.Port is safe.\n\topts := s.getOpts()\n\tif opts.ClientAdvertise != _EMPTY_ {\n\t\th, p, err := parseHostPort(opts.ClientAdvertise, opts.Port)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ts.info.Host = h\n\t\ts.info.Port = p\n\t} else {\n\t\ts.info.Host = opts.Host\n\t\ts.info.Port = opts.Port\n\t}\n\treturn nil\n}\n\n// StartProfiler is called to enable dynamic profiling.\nfunc (s *Server) StartProfiler() {\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tport := opts.ProfPort\n\n\t// Check for Random Port\n\tif port == -1 {\n\t\tport = 0\n\t}\n\n\ts.mu.Lock()\n\thp := net.JoinHostPort(opts.Host, strconv.Itoa(port))\n\tl, err := net.Listen(\"tcp\", hp)\n\n\tif err != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"error starting profiler: %s\", err)\n\t\treturn\n\t}\n\ts.Noticef(\"profiling port: %d\", l.Addr().(*net.TCPAddr).Port)\n\n\tsrv := &http.Server{\n\t\tAddr: hp,\n\t\tHandler: http.DefaultServeMux,\n\t\tMaxHeaderBytes: 1 << 20,\n\t\tReadTimeout: time.Second * 5,\n\t}\n\ts.profiler = l\n\ts.profilingServer = srv\n\n\ts.setBlockProfileRate(opts.ProfBlockRate)\n\n\tgo func() {\n\t\t// if this errors out, it's probably because the server is being shutdown\n\t\terr := srv.Serve(l)\n\t\tif err != nil {\n\t\t\tif !s.isShuttingDown() {\n\t\t\t\ts.Fatalf(\"error starting profiler: %s\", err)\n\t\t\t}\n\t\t}\n\t\tsrv.Close()\n\t\ts.done <- true\n\t}()\n\ts.mu.Unlock()\n}\n\nfunc (s *Server) setBlockProfileRate(rate int) {\n\t// Passing i ProfBlockRate <= 0 here will disable or > 0 will enable.\n\truntime.SetBlockProfileRate(rate)\n\n\tif rate > 0 {\n\t\ts.Warnf(\"Block profiling is enabled (rate %d), this may have a performance impact\", rate)\n\t}\n}\n\n// StartHTTPMonitoring will enable the HTTP monitoring port.\n// DEPRECATED: Should use StartMonitoring.\nfunc (s *Server) StartHTTPMonitoring() {\n\ts.startMonitoring(false)\n}\n\n// StartHTTPSMonitoring will enable the HTTPS monitoring port.\n// DEPRECATED: Should use StartMonitoring.\nfunc (s *Server) StartHTTPSMonitoring() {\n\ts.startMonitoring(true)\n}\n\n// StartMonitoring starts the HTTP or HTTPs server if needed.\nfunc (s *Server) StartMonitoring() error {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\t// Specifying both HTTP and HTTPS ports is a misconfiguration\n\tif opts.HTTPPort != 0 && opts.HTTPSPort != 0 {\n\t\treturn fmt.Errorf(\"can't specify both HTTP (%v) and HTTPs (%v) ports\", opts.HTTPPort, opts.HTTPSPort)\n\t}\n\tvar err error\n\tif opts.HTTPPort != 0 {\n\t\terr = s.startMonitoring(false)\n\t} else if opts.HTTPSPort != 0 {\n\t\tif opts.TLSConfig == nil {\n\t\t\treturn fmt.Errorf(\"TLS cert and key required for HTTPS\")\n\t\t}\n\t\terr = s.startMonitoring(true)\n\t}\n\treturn err\n}\n\n// HTTP endpoints\nconst (\n\tRootPath = \"/\"\n\tVarzPath = \"/varz\"\n\tConnzPath = \"/connz\"\n\tRoutezPath = \"/routez\"\n\tGatewayzPath = \"/gatewayz\"\n\tLeafzPath = \"/leafz\"\n\tSubszPath = \"/subsz\"\n\tStackszPath = \"/stacksz\"\n\tAccountzPath = \"/accountz\"\n\tAccountStatzPath = \"/accstatz\"\n\tJszPath = \"/jsz\"\n\tHealthzPath = \"/healthz\"\n\tIPQueuesPath = \"/ipqueuesz\"\n\tRaftzPath = \"/raftz\"\n\tExpvarzPath = \"/debug/vars\"\n)\n\nfunc (s *Server) basePath(p string) string {\n\treturn path.Join(s.httpBasePath, p)\n}\n\ntype captureHTTPServerLog struct {\n\ts *Server\n\tprefix string\n}\n\nfunc (cl *captureHTTPServerLog) Write(p []byte) (int, error) {\n\tvar buf [128]byte\n\tvar b = buf[:0]\n\n\tb = append(b, []byte(cl.prefix)...)\n\toffset := 0\n\tif bytes.HasPrefix(p, []byte(\"http:\")) {\n\t\toffset = 6\n\t}\n\tb = append(b, p[offset:]...)\n\tcl.s.Errorf(string(b))\n\treturn len(p), nil\n}\n\n// The TLS configuration is passed to the listener when the monitoring\n// \"server\" is setup. That prevents TLS configuration updates on reload\n// from being used. By setting this function in tls.Config.GetConfigForClient\n// we instruct the TLS handshake to ask for the tls configuration to be\n// used for a specific client. We don't care which client, we always use\n// the same TLS configuration.\nfunc (s *Server) getMonitoringTLSConfig(_ *tls.ClientHelloInfo) (*tls.Config, error) {\n\topts := s.getOpts()\n\ttc := opts.TLSConfig.Clone()\n\ttc.ClientAuth = tls.NoClientCert\n\treturn tc, nil\n}\n\n// Start the monitoring server\nfunc (s *Server) startMonitoring(secure bool) error {\n\tif s.isShuttingDown() {\n\t\treturn nil\n\t}\n\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tvar (\n\t\thp string\n\t\terr error\n\t\thttpListener net.Listener\n\t\tport int\n\t)\n\n\tmonitorProtocol := \"http\"\n\n\tif secure {\n\t\tmonitorProtocol += \"s\"\n\t\tport = opts.HTTPSPort\n\t\tif port == -1 {\n\t\t\tport = 0\n\t\t}\n\t\thp = net.JoinHostPort(opts.HTTPHost, strconv.Itoa(port))\n\t\tconfig := opts.TLSConfig.Clone()\n\t\tif !s.ocspPeerVerify {\n\t\t\tconfig.GetConfigForClient = s.getMonitoringTLSConfig\n\t\t\tconfig.ClientAuth = tls.NoClientCert\n\t\t}\n\t\thttpListener, err = tls.Listen(\"tcp\", hp, config)\n\n\t} else {\n\t\tport = opts.HTTPPort\n\t\tif port == -1 {\n\t\t\tport = 0\n\t\t}\n\t\thp = net.JoinHostPort(opts.HTTPHost, strconv.Itoa(port))\n\t\thttpListener, err = net.Listen(\"tcp\", hp)\n\t}\n\n\tif err != nil {\n\t\treturn fmt.Errorf(\"can't listen to the monitor port: %v\", err)\n\t}\n\n\trport := httpListener.Addr().(*net.TCPAddr).Port\n\ts.Noticef(\"Starting %s monitor on %s\", monitorProtocol, net.JoinHostPort(opts.HTTPHost, strconv.Itoa(rport)))\n\n\tmux := http.NewServeMux()\n\n\t// Root\n\tmux.HandleFunc(s.basePath(RootPath), s.HandleRoot)\n\t// Varz\n\tmux.HandleFunc(s.basePath(VarzPath), s.HandleVarz)\n\t// Connz\n\tmux.HandleFunc(s.basePath(ConnzPath), s.HandleConnz)\n\t// Routez\n\tmux.HandleFunc(s.basePath(RoutezPath), s.HandleRoutez)\n\t// Gatewayz\n\tmux.HandleFunc(s.basePath(GatewayzPath), s.HandleGatewayz)\n\t// Leafz\n\tmux.HandleFunc(s.basePath(LeafzPath), s.HandleLeafz)\n\t// Subz\n\tmux.HandleFunc(s.basePath(SubszPath), s.HandleSubsz)\n\t// Subz alias for backwards compatibility\n\tmux.HandleFunc(s.basePath(\"/subscriptionsz\"), s.HandleSubsz)\n\t// Stacksz\n\tmux.HandleFunc(s.basePath(StackszPath), s.HandleStacksz)\n\t// Accountz\n\tmux.HandleFunc(s.basePath(AccountzPath), s.HandleAccountz)\n\t// Accstatz\n\tmux.HandleFunc(s.basePath(AccountStatzPath), s.HandleAccountStatz)\n\t// Jsz\n\tmux.HandleFunc(s.basePath(JszPath), s.HandleJsz)\n\t// Healthz\n\tmux.HandleFunc(s.basePath(HealthzPath), s.HandleHealthz)\n\t// IPQueuesz\n\tmux.HandleFunc(s.basePath(IPQueuesPath), s.HandleIPQueuesz)\n\t// Raftz\n\tmux.HandleFunc(s.basePath(RaftzPath), s.HandleRaftz)\n\t// Expvarz\n\tmux.Handle(s.basePath(ExpvarzPath), expvar.Handler())\n\n\t// Do not set a WriteTimeout because it could cause cURL/browser\n\t// to return empty response or unable to display page if the\n\t// server needs more time to build the response.\n\tsrv := &http.Server{\n\t\tAddr: hp,\n\t\tHandler: mux,\n\t\tMaxHeaderBytes: 1 << 20,\n\t\tErrorLog: log.New(&captureHTTPServerLog{s, \"monitoring: \"}, _EMPTY_, 0),\n\t\tReadHeaderTimeout: time.Second * 5,\n\t}\n\ts.mu.Lock()\n\ts.http = httpListener\n\ts.httpHandler = mux\n\ts.monitoringServer = srv\n\ts.mu.Unlock()\n\n\tgo func() {\n\t\tif err := srv.Serve(httpListener); err != nil {\n\t\t\tif !s.isShuttingDown() {\n\t\t\t\ts.Fatalf(\"Error starting monitor on %q: %v\", hp, err)\n\t\t\t}\n\t\t}\n\t\tsrv.Close()\n\t\ts.mu.Lock()\n\t\ts.httpHandler = nil\n\t\ts.mu.Unlock()\n\t\ts.done <- true\n\t}()\n\n\treturn nil\n}\n\n// HTTPHandler returns the http.Handler object used to handle monitoring\n// endpoints. It will return nil if the server is not configured for\n// monitoring, or if the server has not been started yet (Server.Start()).\nfunc (s *Server) HTTPHandler() http.Handler {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\treturn s.httpHandler\n}\n\n// Perform a conditional deep copy due to reference nature of [Client|WS]ConnectURLs.\n// If updates are made to Info, this function should be consulted and updated.\n// Assume lock is held.\nfunc (s *Server) copyInfo() Info {\n\tinfo := s.info\n\tif len(info.ClientConnectURLs) > 0 {\n\t\tinfo.ClientConnectURLs = append([]string(nil), s.info.ClientConnectURLs...)\n\t}\n\tif len(info.WSConnectURLs) > 0 {\n\t\tinfo.WSConnectURLs = append([]string(nil), s.info.WSConnectURLs...)\n\t}\n\treturn info\n}\n\n// tlsMixConn is used when we can receive both TLS and non-TLS connections on same port.\ntype tlsMixConn struct {\n\tnet.Conn\n\tpre *bytes.Buffer\n}\n\n// Read for our mixed multi-reader.\nfunc (c *tlsMixConn) Read(b []byte) (int, error) {\n\tif c.pre != nil {\n\t\tn, err := c.pre.Read(b)\n\t\tif c.pre.Len() == 0 {\n\t\t\tc.pre = nil\n\t\t}\n\t\treturn n, err\n\t}\n\treturn c.Conn.Read(b)\n}\n\nfunc (s *Server) createClient(conn net.Conn) *client {\n\treturn s.createClientEx(conn, false)\n}\n\nfunc (s *Server) createClientInProcess(conn net.Conn) *client {\n\treturn s.createClientEx(conn, true)\n}\n\nfunc (s *Server) createClientEx(conn net.Conn, inProcess bool) *client {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\tmaxPay := int32(opts.MaxPayload)\n\tmaxSubs := int32(opts.MaxSubs)\n\t// For system, maxSubs of 0 means unlimited, so re-adjust here.\n\tif maxSubs == 0 {\n\t\tmaxSubs = -1\n\t}\n\tnow := time.Now()\n\n\tc := &client{\n\t\tsrv: s,\n\t\tnc: conn,\n\t\topts: defaultOpts,\n\t\tmpay: maxPay,\n\t\tmsubs: maxSubs,\n\t\tstart: now,\n\t\tlast: now,\n\t\tiproc: inProcess,\n\t}\n\n\tc.registerWithAccount(s.globalAccount())\n\n\tvar info Info\n\tvar authRequired bool\n\n\ts.mu.Lock()\n\t// Grab JSON info string\n\tinfo = s.copyInfo()\n\tif s.nonceRequired() {\n\t\t// Nonce handling\n\t\tvar raw [nonceLen]byte\n\t\tnonce := raw[:]\n\t\ts.generateNonce(nonce)\n\t\tinfo.Nonce = string(nonce)\n\t}\n\tc.nonce = []byte(info.Nonce)\n\tauthRequired = info.AuthRequired\n\n\t// Check to see if we have auth_required set but we also have a no_auth_user.\n\t// If so set back to false.\n\tif info.AuthRequired && opts.NoAuthUser != _EMPTY_ && opts.NoAuthUser != s.sysAccOnlyNoAuthUser {\n\t\tinfo.AuthRequired = false\n\t}\n\n\t// Check to see if this is an in-process connection with tls_required.\n\t// If so, set as not required, but available.\n\tif inProcess && info.TLSRequired {\n\t\tinfo.TLSRequired = false\n\t\tinfo.TLSAvailable = true\n\t}\n\n\ts.totalClients++\n\ts.mu.Unlock()\n\n\t// Grab lock\n\tc.mu.Lock()\n\tif authRequired {\n\t\tc.flags.set(expectConnect)\n\t}\n\n\t// Initialize\n\tc.initClient()\n\n\tc.Debugf(\"Client connection created\")\n\n\t// Save info.TLSRequired value since we may neeed to change it back and forth.\n\torgInfoTLSReq := info.TLSRequired\n\n\tvar tlsFirstFallback time.Duration\n\t// Check if we should do TLS first.\n\ttlsFirst := opts.TLSConfig != nil && opts.TLSHandshakeFirst\n\tif tlsFirst {\n\t\t// Make sure info.TLSRequired is set to true (it could be false\n\t\t// if AllowNonTLS is enabled).\n\t\tinfo.TLSRequired = true\n\t\t// Get the fallback delay value if applicable.\n\t\tif f := opts.TLSHandshakeFirstFallback; f > 0 {\n\t\t\ttlsFirstFallback = f\n\t\t} else if inProcess {\n\t\t\t// For in-process connection, we will always have a fallback\n\t\t\t// delay. It allows support for non-TLS, TLS and \"TLS First\"\n\t\t\t// in-process clients to successfully connect.\n\t\t\ttlsFirstFallback = DEFAULT_TLS_HANDSHAKE_FIRST_FALLBACK_DELAY\n\t\t}\n\t}\n\n\t// Decide if we are going to require TLS or not and generate INFO json.\n\t// If we have ProxyProtocol enabled then we won't include the client\n\t// IP in the initial INFO, as that would leak the proxy IP itself.\n\t// In that case we'll send another INFO after the client introduces itself.\n\ttlsRequired := info.TLSRequired\n\tinfoBytes := c.generateClientInfoJSON(info, !opts.ProxyProtocol)\n\n\t// Send our information, except if TLS and TLSHandshakeFirst is requested.\n\tif !tlsFirst {\n\t\t// Need to be sent in place since writeLoop cannot be started until\n\t\t// TLS handshake is done (if applicable).\n\t\tc.sendProtoNow(infoBytes)\n\t}\n\n\t// Unlock to register\n\tc.mu.Unlock()\n\n\t// Register with the server.\n\ts.mu.Lock()\n\t// If server is not running, Shutdown() may have already gathered the\n\t// list of connections to close. It won't contain this one, so we need\n\t// to bail out now otherwise the readLoop started down there would not\n\t// be interrupted. Skip also if in lame duck mode.\n\tif !s.isRunning() || s.ldm {\n\t\t// There are some tests that create a server but don't start it,\n\t\t// and use \"async\" clients and perform the parsing manually. Such\n\t\t// clients would branch here (since server is not running). However,\n\t\t// when a server was really running and has been shutdown, we must\n\t\t// close this connection.\n\t\tif s.isShuttingDown() {\n\t\t\tconn.Close()\n\t\t}\n\t\ts.mu.Unlock()\n\t\treturn c\n\t}\n\n\t// If there is a max connections specified, check that adding\n\t// this new client would not push us over the max\n\tif opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) {\n\t\ts.mu.Unlock()\n\t\tc.maxConnExceeded()\n\t\treturn nil\n\t}\n\ts.clients[c.cid] = c\n\n\ts.mu.Unlock()\n\n\t// Re-Grab lock\n\tc.mu.Lock()\n\n\tisClosed := c.isClosed()\n\tvar pre []byte\n\t// We need first to check for \"TLS First\" fallback delay.\n\tif !isClosed && tlsFirstFallback > 0 {\n\t\t// We wait and see if we are getting any data. Since we did not send\n\t\t// the INFO protocol yet, only clients that use TLS first should be\n\t\t// sending data (the TLS handshake). We don't really check the content:\n\t\t// if it is a rogue agent and not an actual client performing the\n\t\t// TLS handshake, the error will be detected when performing the\n\t\t// handshake on our side.\n\t\tpre = make([]byte, 4)\n\t\tc.nc.SetReadDeadline(time.Now().Add(tlsFirstFallback))\n\t\tn, _ := io.ReadFull(c.nc, pre[:])\n\t\tc.nc.SetReadDeadline(time.Time{})\n\t\t// If we get any data (regardless of possible timeout), we will proceed\n\t\t// with the TLS handshake.\n\t\tif n > 0 {\n\t\t\tpre = pre[:n]\n\t\t} else {\n\t\t\t// We did not get anything so we will send the INFO protocol.\n\t\t\tpre = nil\n\n\t\t\t// Restore the original info.TLSRequired value if it is\n\t\t\t// different that the current value and regenerate infoBytes.\n\t\t\tif orgInfoTLSReq != info.TLSRequired {\n\t\t\t\tinfo.TLSRequired = orgInfoTLSReq\n\t\t\t\tinfoBytes = c.generateClientInfoJSON(info, !opts.ProxyProtocol)\n\t\t\t}\n\t\t\tc.sendProtoNow(infoBytes)\n\t\t\t// Set the boolean to false for the rest of the function.\n\t\t\ttlsFirst = false\n\t\t\t// Check closed status again\n\t\t\tisClosed = c.isClosed()\n\t\t}\n\t}\n\t// If we have both TLS and non-TLS allowed we need to see which\n\t// one the client wants. We'll always allow this for in-process\n\t// connections.\n\tif !isClosed && !tlsFirst && opts.TLSConfig != nil && (inProcess || opts.AllowNonTLS) {\n\t\tpre = make([]byte, 6) // Minimum 6 bytes for proxy proto in next step.\n\t\tc.nc.SetReadDeadline(time.Now().Add(secondsToDuration(opts.TLSTimeout)))\n\t\tn, _ := io.ReadFull(c.nc, pre[:])\n\t\tc.nc.SetReadDeadline(time.Time{})\n\t\tpre = pre[:n]\n\t\tif n > 0 && pre[0] == 0x16 {\n\t\t\ttlsRequired = true\n\t\t} else {\n\t\t\ttlsRequired = false\n\t\t}\n\t}\n\n\t// Check for proxy protocol if enabled.\n\tif !isClosed && !tlsRequired && opts.ProxyProtocol {\n\t\tif len(pre) == 0 {\n\t\t\t// There has been no pre-read yet, do so so we can work out\n\t\t\t// if the client is trying to negotiate PROXY.\n\t\t\tpre = make([]byte, 6)\n\t\t\tc.nc.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout))\n\t\t\tn, _ := io.ReadFull(c.nc, pre)\n\t\t\tc.nc.SetReadDeadline(time.Time{})\n\t\t\tpre = pre[:n]\n\t\t}\n\t\tconn = &tlsMixConn{conn, bytes.NewBuffer(pre)}\n\t\taddr, proxyPre, err := readProxyProtoHeader(conn)\n\t\tif err != nil && err != errProxyProtoUnrecognized {\n\t\t\t// err != errProxyProtoUnrecognized implies that we detected a proxy\n\t\t\t// protocol header but we failed to parse it, so don't continue.\n\t\t\tc.mu.Unlock()\n\t\t\ts.Warnf(\"Error reading PROXY protocol header from %s: %v\", conn.RemoteAddr(), err)\n\t\t\tc.closeConnection(ProtocolViolation)\n\t\t\treturn nil\n\t\t}\n\t\t// If addr is nil, it was a LOCAL/UNKNOWN command (health check)\n\t\t// Use the connection as-is\n\t\tif addr != nil {\n\t\t\tc.nc = &proxyConn{\n\t\t\t\tConn: conn,\n\t\t\t\tremoteAddr: addr,\n\t\t\t}\n\t\t\t// These were set already by initClient, override them.\n\t\t\tc.host = addr.srcIP.String()\n\t\t\tc.port = addr.srcPort\n\t\t}\n\t\t// At this point, err is either:\n\t\t// - nil => we parsed the proxy protocol header successfully\n\t\t// - errProxyProtoUnrecognized => we didn't detect proxy protocol at all\n\t\t// We only clear the pre-read if we successfully read the protocol header\n\t\t// so that the next step doesn't re-read it. Otherwise we have to assume\n\t\t// that it's a non-proxied connection and we want the pre-read to remain\n\t\t// for the next step.\n\t\tif err == nil {\n\t\t\tpre = proxyPre\n\t\t}\n\t\t// Because we have ProxyProtocol enabled, our earlier INFO message didn't\n\t\t// include the client_ip. If we need to send it again then we will include\n\t\t// it, but sending it here immediately can confuse clients who have just\n\t\t// PING'd.\n\t\tinfoBytes = c.generateClientInfoJSON(info, true)\n\t}\n\n\t// Check for TLS\n\tif !isClosed && tlsRequired {\n\t\tif s.connRateCounter != nil && !s.connRateCounter.allow() {\n\t\t\tc.mu.Unlock()\n\t\t\tc.sendErr(\"Connection throttling is active. Please try again later.\")\n\t\t\tc.closeConnection(MaxConnectionsExceeded)\n\t\t\treturn nil\n\t\t}\n\n\t\t// If we have a prebuffer create a multi-reader.\n\t\tif len(pre) > 0 {\n\t\t\tc.nc = &tlsMixConn{c.nc, bytes.NewBuffer(pre)}\n\t\t\t// Clear pre so it is not parsed.\n\t\t\tpre = nil\n\t\t}\n\t\t// Performs server-side TLS handshake.\n\t\tif err := c.doTLSServerHandshake(_EMPTY_, opts.TLSConfig, opts.TLSTimeout, opts.TLSPinnedCerts); err != nil {\n\t\t\tc.mu.Unlock()\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t// Now, send the INFO if it was delayed\n\tif !isClosed && tlsFirst {\n\t\tc.flags.set(didTLSFirst)\n\t\tc.sendProtoNow(infoBytes)\n\t\t// Check closed status\n\t\tisClosed = c.isClosed()\n\t}\n\n\t// Connection could have been closed while sending the INFO proto.\n\tif isClosed {\n\t\tc.mu.Unlock()\n\t\t// We need to call closeConnection() to make sure that proper cleanup is done.\n\t\tc.closeConnection(WriteError)\n\t\treturn nil\n\t}\n\n\t// Check for Auth. We schedule this timer after the TLS handshake to avoid\n\t// the race where the timer fires during the handshake and causes the\n\t// server to write bad data to the socket. See issue #432.\n\tif authRequired {\n\t\tc.setAuthTimer(secondsToDuration(opts.AuthTimeout))\n\t}\n\n\t// Do final client initialization\n\n\t// Set the Ping timer. Will be reset once connect was received.\n\tc.setPingTimer()\n\n\t// Spin up the read loop.\n\ts.startGoRoutine(func() { c.readLoop(pre) })\n\n\t// Spin up the write loop.\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\tif tlsRequired {\n\t\tc.Debugf(\"TLS handshake complete\")\n\t\tcs := c.nc.(*tls.Conn).ConnectionState()\n\t\tc.Debugf(\"TLS version %s, cipher suite %s\", tlsVersion(cs.Version), tls.CipherSuiteName(cs.CipherSuite))\n\t}\n\n\tc.mu.Unlock()\n\n\treturn c\n}\n\n// This will save off a closed client in a ring buffer such that\n// /connz can inspect. Useful for debugging, etc.\nfunc (s *Server) saveClosedClient(c *client, nc net.Conn, subs map[string]*subscription, reason ClosedState) {\n\tnow := time.Now()\n\n\ts.accountDisconnectEvent(c, now, reason.String())\n\n\tc.mu.Lock()\n\n\tcc := &closedClient{}\n\tcc.fill(c, nc, now, false)\n\t// Note that cc.fill is using len(c.subs), which may have been set to nil by now,\n\t// so replace cc.NumSubs with len(subs).\n\tcc.NumSubs = uint32(len(subs))\n\tcc.Stop = &now\n\tcc.Reason = reason.String()\n\n\t// Do subs, do not place by default in main ConnInfo\n\tif len(subs) > 0 {\n\t\tcc.subs = make([]SubDetail, 0, len(subs))\n\t\tfor _, sub := range subs {\n\t\t\tcc.subs = append(cc.subs, newSubDetail(sub))\n\t\t}\n\t}\n\t// Hold user as well.\n\tcc.user = c.getRawAuthUser()\n\t// Hold account name if not the global account.\n\tif c.acc != nil && c.acc.Name != globalAccountName {\n\t\tcc.acc = c.acc.Name\n\t}\n\tcc.JWT = c.opts.JWT\n\tcc.IssuerKey = issuerForClient(c)\n\tcc.Tags = c.tags\n\tcc.NameTag = c.nameTag\n\tc.mu.Unlock()\n\n\t// Place in the ring buffer\n\ts.mu.Lock()\n\tif s.closed != nil {\n\t\ts.closed.append(cc)\n\t}\n\ts.mu.Unlock()\n}\n\n// Adds to the list of client and websocket clients connect URLs.\n// If there was a change, an INFO protocol is sent to registered clients\n// that support async INFO protocols.\n// Server lock held on entry.\nfunc (s *Server) addConnectURLsAndSendINFOToClients(curls, wsurls []string) {\n\ts.updateServerINFOAndSendINFOToClients(curls, wsurls, true)\n}\n\n// Removes from the list of client and websocket clients connect URLs.\n// If there was a change, an INFO protocol is sent to registered clients\n// that support async INFO protocols.\n// Server lock held on entry.\nfunc (s *Server) removeConnectURLsAndSendINFOToClients(curls, wsurls []string) {\n\ts.updateServerINFOAndSendINFOToClients(curls, wsurls, false)\n}\n\n// Updates the list of client and websocket clients connect URLs and if any change\n// sends an async INFO update to clients that support it.\n// Server lock held on entry.\nfunc (s *Server) updateServerINFOAndSendINFOToClients(curls, wsurls []string, add bool) {\n\tremove := !add\n\t// Will return true if we need alter the server's Info object.\n\tupdateMap := func(urls []string, m refCountedUrlSet) bool {\n\t\twasUpdated := false\n\t\tfor _, url := range urls {\n\t\t\tif add && m.addUrl(url) {\n\t\t\t\twasUpdated = true\n\t\t\t} else if remove && m.removeUrl(url) {\n\t\t\t\twasUpdated = true\n\t\t\t}\n\t\t}\n\t\treturn wasUpdated\n\t}\n\tcliUpdated := updateMap(curls, s.clientConnectURLsMap)\n\twsUpdated := updateMap(wsurls, s.websocket.connectURLsMap)\n\n\tupdateInfo := func(infoURLs *[]string, urls []string, m refCountedUrlSet) {\n\t\t// Recreate the info's slice from the map\n\t\t*infoURLs = (*infoURLs)[:0]\n\t\t// Add this server client connect ULRs first...\n\t\t*infoURLs = append(*infoURLs, urls...)\n\t\t// Then the ones from the map\n\t\tfor url := range m {\n\t\t\t*infoURLs = append(*infoURLs, url)\n\t\t}\n\t}\n\tif cliUpdated {\n\t\tupdateInfo(&s.info.ClientConnectURLs, s.clientConnectURLs, s.clientConnectURLsMap)\n\t}\n\tif wsUpdated {\n\t\tupdateInfo(&s.info.WSConnectURLs, s.websocket.connectURLs, s.websocket.connectURLsMap)\n\t}\n\tif cliUpdated || wsUpdated {\n\t\t// Send to all registered clients that support async INFO protocols.\n\t\ts.sendAsyncInfoToClients(cliUpdated, wsUpdated)\n\t}\n}\n\n// Handle closing down a connection when the handshake has timedout.\nfunc tlsTimeout(c *client, conn *tls.Conn) {\n\tc.mu.Lock()\n\tclosed := c.isClosed()\n\tc.mu.Unlock()\n\t// Check if already closed\n\tif closed {\n\t\treturn\n\t}\n\tcs := conn.ConnectionState()\n\tif !cs.HandshakeComplete {\n\t\tc.Errorf(\"TLS handshake timeout\")\n\t\tc.sendErr(\"Secure Connection - TLS Required\")\n\t\tc.closeConnection(TLSHandshakeError)\n\t}\n}\n\n// Seems silly we have to write these\nfunc tlsVersion(ver uint16) string {\n\tswitch ver {\n\tcase tls.VersionTLS10:\n\t\treturn \"1.0\"\n\tcase tls.VersionTLS11:\n\t\treturn \"1.1\"\n\tcase tls.VersionTLS12:\n\t\treturn \"1.2\"\n\tcase tls.VersionTLS13:\n\t\treturn \"1.3\"\n\t}\n\treturn fmt.Sprintf(\"Unknown [0x%x]\", ver)\n}\n\nfunc tlsVersionFromString(ver string) (uint16, error) {\n\tswitch ver {\n\tcase \"1.0\":\n\t\treturn tls.VersionTLS10, nil\n\tcase \"1.1\":\n\t\treturn tls.VersionTLS11, nil\n\tcase \"1.2\":\n\t\treturn tls.VersionTLS12, nil\n\tcase \"1.3\":\n\t\treturn tls.VersionTLS13, nil\n\t}\n\treturn 0, fmt.Errorf(\"unknown version: %v\", ver)\n}\n\n// Remove a client or route from our internal accounting.\nfunc (s *Server) removeClient(c *client) {\n\t// kind is immutable, so can check without lock\n\tswitch c.kind {\n\tcase CLIENT:\n\t\tc.mu.Lock()\n\t\tcid := c.cid\n\t\tupdateProtoInfoCount := false\n\t\tif c.kind == CLIENT && c.opts.Protocol >= ClientProtoInfo {\n\t\t\tupdateProtoInfoCount = true\n\t\t}\n\t\tproxyKey := c.proxyKey\n\t\tc.mu.Unlock()\n\n\t\ts.mu.Lock()\n\t\tdelete(s.clients, cid)\n\t\tif updateProtoInfoCount {\n\t\t\ts.cproto--\n\t\t}\n\t\tif proxyKey != _EMPTY_ {\n\t\t\ts.removeProxiedConn(proxyKey, cid)\n\t\t}\n\t\ts.mu.Unlock()\n\tcase ROUTER:\n\t\ts.removeRoute(c)\n\tcase GATEWAY:\n\t\ts.removeRemoteGatewayConnection(c)\n\tcase LEAF:\n\t\ts.removeLeafNodeConnection(c)\n\t}\n}\n\n// Remove the connection with id `cid` from the map of connections\n// under the public key `key` of the trusted proxies.\n//\n// Server lock must be held on entry.\nfunc (s *Server) removeProxiedConn(key string, cid uint64) {\n\tconns := s.proxiedConns[key]\n\tdelete(conns, cid)\n\tif len(conns) == 0 {\n\t\tdelete(s.proxiedConns, key)\n\t}\n}\n\nfunc (s *Server) removeFromTempClients(cid uint64) {\n\ts.grMu.Lock()\n\tdelete(s.grTmpClients, cid)\n\ts.grMu.Unlock()\n}\n\nfunc (s *Server) addToTempClients(cid uint64, c *client) bool {\n\tadded := false\n\ts.grMu.Lock()\n\tif s.grRunning {\n\t\ts.grTmpClients[cid] = c\n\t\tadded = true\n\t}\n\ts.grMu.Unlock()\n\treturn added\n}\n\n/////////////////////////////////////////////////////////////////\n// These are some helpers for accounting in functional tests.\n/////////////////////////////////////////////////////////////////\n\n// NumRoutes will report the number of registered routes.\nfunc (s *Server) NumRoutes() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.numRoutes()\n}\n\n// numRoutes will report the number of registered routes.\n// Server lock held on entry\nfunc (s *Server) numRoutes() int {\n\tvar nr int\n\ts.forEachRoute(func(c *client) {\n\t\tnr++\n\t})\n\treturn nr\n}\n\n// NumRemotes will report number of registered remotes.\nfunc (s *Server) NumRemotes() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.numRemotes()\n}\n\n// numRemotes will report number of registered remotes.\n// Server lock held on entry\nfunc (s *Server) numRemotes() int {\n\treturn len(s.routes)\n}\n\n// NumLeafNodes will report number of leaf node connections.\nfunc (s *Server) NumLeafNodes() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn len(s.leafs)\n}\n\n// NumClients will report the number of registered clients.\nfunc (s *Server) NumClients() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn len(s.clients)\n}\n\n// GetClient will return the client associated with cid.\nfunc (s *Server) GetClient(cid uint64) *client {\n\treturn s.getClient(cid)\n}\n\n// getClient will return the client associated with cid.\nfunc (s *Server) getClient(cid uint64) *client {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.clients[cid]\n}\n\n// GetLeafNode returns the leafnode associated with the cid.\nfunc (s *Server) GetLeafNode(cid uint64) *client {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.leafs[cid]\n}\n\n// NumSubscriptions will report how many subscriptions are active.\nfunc (s *Server) NumSubscriptions() uint32 {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.numSubscriptions()\n}\n\n// numSubscriptions will report how many subscriptions are active.\n// Lock should be held.\nfunc (s *Server) numSubscriptions() uint32 {\n\tvar subs int\n\ts.accounts.Range(func(k, v any) bool {\n\t\tacc := v.(*Account)\n\t\tsubs += acc.TotalSubs()\n\t\treturn true\n\t})\n\treturn uint32(subs)\n}\n\n// NumSlowConsumers will report the number of slow consumers.\nfunc (s *Server) NumSlowConsumers() int64 {\n\treturn atomic.LoadInt64(&s.slowConsumers)\n}\n\n// NumStalledClients will report the total number of times clients have been stalled.\nfunc (s *Server) NumStalledClients() int64 {\n\treturn atomic.LoadInt64(&s.stalls)\n}\n\n// NumSlowConsumersClients will report the number of slow consumers clients.\nfunc (s *Server) NumSlowConsumersClients() uint64 {\n\treturn s.scStats.clients.Load()\n}\n\n// NumSlowConsumersRoutes will report the number of slow consumers routes.\nfunc (s *Server) NumSlowConsumersRoutes() uint64 {\n\treturn s.scStats.routes.Load()\n}\n\n// NumSlowConsumersGateways will report the number of slow consumers leafs.\nfunc (s *Server) NumSlowConsumersGateways() uint64 {\n\treturn s.scStats.gateways.Load()\n}\n\n// NumSlowConsumersLeafs will report the number of slow consumers leafs.\nfunc (s *Server) NumSlowConsumersLeafs() uint64 {\n\treturn s.scStats.leafs.Load()\n}\n\n// NumStaleConnections will report the number of stale connections.\nfunc (s *Server) NumStaleConnections() int64 {\n\treturn atomic.LoadInt64(&s.staleConnections)\n}\n\n// NumStaleConnectionsClients will report the number of stale client connections.\nfunc (s *Server) NumStaleConnectionsClients() uint64 {\n\treturn s.staleStats.clients.Load()\n}\n\n// NumStaleConnectionsRoutes will report the number of stale route connections.\nfunc (s *Server) NumStaleConnectionsRoutes() uint64 {\n\treturn s.staleStats.routes.Load()\n}\n\n// NumStaleConnectionsGateways will report the number of stale gateway connections.\nfunc (s *Server) NumStaleConnectionsGateways() uint64 {\n\treturn s.staleStats.gateways.Load()\n}\n\n// NumStaleConnectionsLeafs will report the number of stale leaf connections.\nfunc (s *Server) NumStaleConnectionsLeafs() uint64 {\n\treturn s.staleStats.leafs.Load()\n}\n\n// ConfigTime will report the last time the server configuration was loaded.\nfunc (s *Server) ConfigTime() time.Time {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.configTime\n}\n\n// Addr will return the net.Addr object for the current listener.\nfunc (s *Server) Addr() net.Addr {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif s.listener == nil {\n\t\treturn nil\n\t}\n\treturn s.listener.Addr()\n}\n\n// MonitorAddr will return the net.Addr object for the monitoring listener.\nfunc (s *Server) MonitorAddr() *net.TCPAddr {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif s.http == nil {\n\t\treturn nil\n\t}\n\treturn s.http.Addr().(*net.TCPAddr)\n}\n\n// ClusterAddr returns the net.Addr object for the route listener.\nfunc (s *Server) ClusterAddr() *net.TCPAddr {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif s.routeListener == nil {\n\t\treturn nil\n\t}\n\treturn s.routeListener.Addr().(*net.TCPAddr)\n}\n\n// ProfilerAddr returns the net.Addr object for the profiler listener.\nfunc (s *Server) ProfilerAddr() *net.TCPAddr {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\tif s.profiler == nil {\n\t\treturn nil\n\t}\n\treturn s.profiler.Addr().(*net.TCPAddr)\n}\n\nfunc (s *Server) readyForConnections(d time.Duration) error {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\n\ttype info struct {\n\t\tok bool\n\t\terr error\n\t}\n\tchk := make(map[string]info)\n\n\tend := time.Now().Add(d)\n\tfor time.Now().Before(end) {\n\t\ts.mu.RLock()\n\t\tchk[\"server\"] = info{ok: s.listener != nil || opts.DontListen, err: s.listenerErr}\n\t\tchk[\"route\"] = info{ok: (opts.Cluster.Port == 0 || s.routeListener != nil), err: s.routeListenerErr}\n\t\tchk[\"gateway\"] = info{ok: (opts.Gateway.Name == _EMPTY_ || s.gatewayListener != nil), err: s.gatewayListenerErr}\n\t\tchk[\"leafnode\"] = info{ok: (opts.LeafNode.Port == 0 || s.leafNodeListener != nil), err: s.leafNodeListenerErr}\n\t\tchk[\"websocket\"] = info{ok: (opts.Websocket.Port == 0 || s.websocket.listener != nil), err: s.websocket.listenerErr}\n\t\tchk[\"mqtt\"] = info{ok: (opts.MQTT.Port == 0 || s.mqtt.listener != nil), err: s.mqtt.listenerErr}\n\t\ts.mu.RUnlock()\n\n\t\tvar numOK int\n\t\tfor _, inf := range chk {\n\t\t\tif inf.ok {\n\t\t\t\tnumOK++\n\t\t\t}\n\t\t}\n\t\tif numOK == len(chk) {\n\t\t\t// In the case of DontListen option (no accept loop), we still want\n\t\t\t// to make sure that Start() has done all the work, so we wait on\n\t\t\t// that.\n\t\t\tif opts.DontListen {\n\t\t\t\tselect {\n\t\t\t\tcase <-s.startupComplete:\n\t\t\t\tcase <-time.After(d):\n\t\t\t\t\treturn fmt.Errorf(\"failed to be ready for connections after %s: startup did not complete\", d)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tif d > 25*time.Millisecond {\n\t\t\ttime.Sleep(25 * time.Millisecond)\n\t\t}\n\t}\n\n\tfailed := make([]string, 0, len(chk))\n\tfor name, inf := range chk {\n\t\tif inf.ok && inf.err != nil {\n\t\t\tfailed = append(failed, fmt.Sprintf(\"%s(ok, but %s)\", name, inf.err))\n\t\t}\n\t\tif !inf.ok && inf.err == nil {\n\t\t\tfailed = append(failed, name)\n\t\t}\n\t\tif !inf.ok && inf.err != nil {\n\t\t\tfailed = append(failed, fmt.Sprintf(\"%s(%s)\", name, inf.err))\n\t\t}\n\t}\n\n\treturn fmt.Errorf(\n\t\t\"failed to be ready for connections after %s: %s\",\n\t\td, strings.Join(failed, \", \"),\n\t)\n}\n\n// ReadyForConnections returns `true` if the server is ready to accept clients\n// and, if routing is enabled, route connections. If after the duration\n// `dur` the server is still not ready, returns `false`.\nfunc (s *Server) ReadyForConnections(dur time.Duration) bool {\n\treturn s.readyForConnections(dur) == nil\n}\n\n// Quick utility to function to tell if the server supports headers.\nfunc (s *Server) supportsHeaders() bool {\n\tif s == nil {\n\t\treturn false\n\t}\n\treturn !(s.getOpts().NoHeaderSupport)\n}\n\n// ID returns the server's ID\nfunc (s *Server) ID() string {\n\treturn s.info.ID\n}\n\n// NodeName returns the node name for this server.\nfunc (s *Server) NodeName() string {\n\treturn getHash(s.info.Name)\n}\n\n// Name returns the server's name. This will be the same as the ID if it was not set.\nfunc (s *Server) Name() string {\n\treturn s.info.Name\n}\n\nfunc (s *Server) String() string {\n\treturn s.info.Name\n}\n\ntype pprofLabels map[string]string\n\nfunc setGoRoutineLabels(tags ...pprofLabels) {\n\tvar labels []string\n\tfor _, m := range tags {\n\t\tfor k, v := range m {\n\t\t\tlabels = append(labels, k, v)\n\t\t}\n\t}\n\tif len(labels) > 0 {\n\t\tpprof.SetGoroutineLabels(\n\t\t\tpprof.WithLabels(context.Background(), pprof.Labels(labels...)),\n\t\t)\n\t}\n}\n\nfunc (s *Server) startGoRoutine(f func(), tags ...pprofLabels) bool {\n\tvar started bool\n\ts.grMu.Lock()\n\tdefer s.grMu.Unlock()\n\tif s.grRunning {\n\t\ts.grWG.Add(1)\n\t\tgo func() {\n\t\t\tsetGoRoutineLabels(tags...)\n\t\t\tf()\n\t\t}()\n\t\tstarted = true\n\t}\n\treturn started\n}\n\nfunc (s *Server) numClosedConns() int {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.closed.len()\n}\n\nfunc (s *Server) totalClosedConns() uint64 {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.closed.totalConns()\n}\n\nfunc (s *Server) closedClients() []*closedClient {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.closed.closedClients()\n}\n\n// getClientConnectURLs returns suitable URLs for clients to connect to the listen\n// port based on the server options' Host and Port. If the Host corresponds to\n// \"any\" interfaces, this call returns the list of resolved IP addresses.\n// If ClientAdvertise is set, returns the client advertise host and port.\n// The server lock is assumed held on entry.\nfunc (s *Server) getClientConnectURLs() []string {\n\t// Snapshot server options.\n\topts := s.getOpts()\n\t// Ignore error here since we know that if there is client advertise, the\n\t// parseHostPort is correct because we did it right before calling this\n\t// function in Server.New().\n\turls, _ := s.getConnectURLs(opts.ClientAdvertise, opts.Host, opts.Port)\n\treturn urls\n}\n\n// Generic version that will return an array of URLs based on the given\n// advertise, host and port values.\nfunc (s *Server) getConnectURLs(advertise, host string, port int) ([]string, error) {\n\turls := make([]string, 0, 1)\n\n\t// short circuit if advertise is set\n\tif advertise != \"\" {\n\t\th, p, err := parseHostPort(advertise, port)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\turls = append(urls, net.JoinHostPort(h, strconv.Itoa(p)))\n\t} else {\n\t\tsPort := strconv.Itoa(port)\n\t\t_, ips, err := s.getNonLocalIPsIfHostIsIPAny(host, true)\n\t\tfor _, ip := range ips {\n\t\t\turls = append(urls, net.JoinHostPort(ip, sPort))\n\t\t}\n\t\tif err != nil || len(urls) == 0 {\n\t\t\t// We are here if s.opts.Host is not \"0.0.0.0\" nor \"::\", or if for some\n\t\t\t// reason we could not add any URL in the loop above.\n\t\t\t// We had a case where a Windows VM was hosed and would have err == nil\n\t\t\t// and not add any address in the array in the loop above, and we\n\t\t\t// ended-up returning 0.0.0.0, which is problematic for Windows clients.\n\t\t\t// Check for 0.0.0.0 or :: specifically, and ignore if that's the case.\n\t\t\tif host == \"0.0.0.0\" || host == \"::\" {\n\t\t\t\ts.Errorf(\"Address %q can not be resolved properly\", host)\n\t\t\t} else {\n\t\t\t\turls = append(urls, net.JoinHostPort(host, sPort))\n\t\t\t}\n\t\t}\n\t}\n\treturn urls, nil\n}\n\n// Returns an array of non local IPs if the provided host is\n// 0.0.0.0 or ::. It returns the first resolved if `all` is\n// false.\n// The boolean indicate if the provided host was 0.0.0.0 (or ::)\n// so that if the returned array is empty caller can decide\n// what to do next.\nfunc (s *Server) getNonLocalIPsIfHostIsIPAny(host string, all bool) (bool, []string, error) {\n\tip := net.ParseIP(host)\n\t// If this is not an IP, we are done\n\tif ip == nil {\n\t\treturn false, nil, nil\n\t}\n\t// If this is not 0.0.0.0 or :: we have nothing to do.\n\tif !ip.IsUnspecified() {\n\t\treturn false, nil, nil\n\t}\n\ts.Debugf(\"Get non local IPs for %q\", host)\n\tvar ips []string\n\tifaces, _ := net.Interfaces()\n\tfor _, i := range ifaces {\n\t\taddrs, _ := i.Addrs()\n\t\tfor _, addr := range addrs {\n\t\t\tswitch v := addr.(type) {\n\t\t\tcase *net.IPNet:\n\t\t\t\tip = v.IP\n\t\t\tcase *net.IPAddr:\n\t\t\t\tip = v.IP\n\t\t\t}\n\t\t\tipStr := ip.String()\n\t\t\t// Skip non global unicast addresses\n\t\t\tif !ip.IsGlobalUnicast() || ip.IsUnspecified() {\n\t\t\t\tip = nil\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ts.Debugf(\" ip=%s\", ipStr)\n\t\t\tips = append(ips, ipStr)\n\t\t\tif !all {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn true, ips, nil\n}\n\n// if the ip is not specified, attempt to resolve it\nfunc resolveHostPorts(addr net.Listener) []string {\n\thostPorts := make([]string, 0)\n\thp := addr.Addr().(*net.TCPAddr)\n\tport := strconv.Itoa(hp.Port)\n\tif hp.IP.IsUnspecified() {\n\t\tvar ip net.IP\n\t\tifaces, _ := net.Interfaces()\n\t\tfor _, i := range ifaces {\n\t\t\taddrs, _ := i.Addrs()\n\t\t\tfor _, addr := range addrs {\n\t\t\t\tswitch v := addr.(type) {\n\t\t\t\tcase *net.IPNet:\n\t\t\t\t\tip = v.IP\n\t\t\t\t\thostPorts = append(hostPorts, net.JoinHostPort(ip.String(), port))\n\t\t\t\tcase *net.IPAddr:\n\t\t\t\t\tip = v.IP\n\t\t\t\t\thostPorts = append(hostPorts, net.JoinHostPort(ip.String(), port))\n\t\t\t\tdefault:\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\thostPorts = append(hostPorts, net.JoinHostPort(hp.IP.String(), port))\n\t}\n\treturn hostPorts\n}\n\n// format the address of a net.Listener with a protocol\nfunc formatURL(protocol string, addr net.Listener) []string {\n\thostports := resolveHostPorts(addr)\n\tfor i, hp := range hostports {\n\t\thostports[i] = fmt.Sprintf(\"%s://%s\", protocol, hp)\n\t}\n\treturn hostports\n}\n\n// Ports describes URLs that the server can be contacted in\ntype Ports struct {\n\tNats []string `json:\"nats,omitempty\"`\n\tMonitoring []string `json:\"monitoring,omitempty\"`\n\tCluster []string `json:\"cluster,omitempty\"`\n\tProfile []string `json:\"profile,omitempty\"`\n\tWebSocket []string `json:\"websocket,omitempty\"`\n}\n\n// PortsInfo attempts to resolve all the ports. If after maxWait the ports are not\n// resolved, it returns nil. Otherwise it returns a Ports struct\n// describing ports where the server can be contacted\nfunc (s *Server) PortsInfo(maxWait time.Duration) *Ports {\n\tif s.readyForListeners(maxWait) {\n\t\topts := s.getOpts()\n\n\t\ts.mu.RLock()\n\t\ttls := s.info.TLSRequired\n\t\tlistener := s.listener\n\t\thttpListener := s.http\n\t\tclusterListener := s.routeListener\n\t\tprofileListener := s.profiler\n\t\twsListener := s.websocket.listener\n\t\twss := s.websocket.tls\n\t\ts.mu.RUnlock()\n\n\t\tports := Ports{}\n\n\t\tif listener != nil {\n\t\t\tnatsProto := \"nats\"\n\t\t\tif tls {\n\t\t\t\tnatsProto = \"tls\"\n\t\t\t}\n\t\t\tports.Nats = formatURL(natsProto, listener)\n\t\t}\n\n\t\tif httpListener != nil {\n\t\t\tmonProto := \"http\"\n\t\t\tif opts.HTTPSPort != 0 {\n\t\t\t\tmonProto = \"https\"\n\t\t\t}\n\t\t\tports.Monitoring = formatURL(monProto, httpListener)\n\t\t}\n\n\t\tif clusterListener != nil {\n\t\t\tclusterProto := \"nats\"\n\t\t\tif opts.Cluster.TLSConfig != nil {\n\t\t\t\tclusterProto = \"tls\"\n\t\t\t}\n\t\t\tports.Cluster = formatURL(clusterProto, clusterListener)\n\t\t}\n\n\t\tif profileListener != nil {\n\t\t\tports.Profile = formatURL(\"http\", profileListener)\n\t\t}\n\n\t\tif wsListener != nil {\n\t\t\tprotocol := wsSchemePrefix\n\t\t\tif wss {\n\t\t\t\tprotocol = wsSchemePrefixTLS\n\t\t\t}\n\t\t\tports.WebSocket = formatURL(protocol, wsListener)\n\t\t}\n\n\t\treturn &ports\n\t}\n\n\treturn nil\n}\n\n// Returns the portsFile. If a non-empty dirHint is provided, the dirHint\n// path is used instead of the server option value\nfunc (s *Server) portFile(dirHint string) string {\n\tdirname := s.getOpts().PortsFileDir\n\tif dirHint != \"\" {\n\t\tdirname = dirHint\n\t}\n\tif dirname == _EMPTY_ {\n\t\treturn _EMPTY_\n\t}\n\treturn filepath.Join(dirname, fmt.Sprintf(\"%s_%d.ports\", filepath.Base(os.Args[0]), os.Getpid()))\n}\n\n// Delete the ports file. If a non-empty dirHint is provided, the dirHint\n// path is used instead of the server option value\nfunc (s *Server) deletePortsFile(hintDir string) {\n\tportsFile := s.portFile(hintDir)\n\tif portsFile != \"\" {\n\t\tif err := os.Remove(portsFile); err != nil {\n\t\t\ts.Errorf(\"Error cleaning up ports file %s: %v\", portsFile, err)\n\t\t}\n\t}\n}\n\n// Writes a file with a serialized Ports to the specified ports_file_dir.\n// The name of the file is `exename_pid.ports`, typically nats-server_pid.ports.\n// if ports file is not set, this function has no effect\nfunc (s *Server) logPorts() {\n\topts := s.getOpts()\n\tportsFile := s.portFile(opts.PortsFileDir)\n\tif portsFile != _EMPTY_ {\n\t\tgo func() {\n\t\t\tinfo := s.PortsInfo(5 * time.Second)\n\t\t\tif info == nil {\n\t\t\t\ts.Errorf(\"Unable to resolve the ports in the specified time\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdata, err := json.Marshal(info)\n\t\t\tif err != nil {\n\t\t\t\ts.Errorf(\"Error marshaling ports file: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err := os.WriteFile(portsFile, data, 0666); err != nil {\n\t\t\t\ts.Errorf(\"Error writing ports file (%s): %v\", portsFile, err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t}()\n\t}\n}\n\n// waits until a calculated list of listeners is resolved or a timeout\nfunc (s *Server) readyForListeners(dur time.Duration) bool {\n\tend := time.Now().Add(dur)\n\tfor time.Now().Before(end) {\n\t\ts.mu.RLock()\n\t\tlisteners := s.serviceListeners()\n\t\ts.mu.RUnlock()\n\t\tif len(listeners) == 0 {\n\t\t\treturn false\n\t\t}\n\n\t\tok := true\n\t\tfor _, l := range listeners {\n\t\t\tif l == nil {\n\t\t\t\tok = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif ok {\n\t\t\treturn true\n\t\t}\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn false\n\t\tcase <-time.After(25 * time.Millisecond):\n\t\t\t// continue - unable to select from quit - we are still running\n\t\t}\n\t}\n\treturn false\n}\n\n// returns a list of listeners that are intended for the process\n// if the entry is nil, the interface is yet to be resolved\nfunc (s *Server) serviceListeners() []net.Listener {\n\tlisteners := make([]net.Listener, 0)\n\topts := s.getOpts()\n\tlisteners = append(listeners, s.listener)\n\tif opts.Cluster.Port != 0 {\n\t\tlisteners = append(listeners, s.routeListener)\n\t}\n\tif opts.HTTPPort != 0 || opts.HTTPSPort != 0 {\n\t\tlisteners = append(listeners, s.http)\n\t}\n\tif opts.ProfPort != 0 {\n\t\tlisteners = append(listeners, s.profiler)\n\t}\n\tif opts.Websocket.Port != 0 {\n\t\tlisteners = append(listeners, s.websocket.listener)\n\t}\n\treturn listeners\n}\n\n// Returns true if in lame duck mode.\nfunc (s *Server) isLameDuckMode() bool {\n\ts.mu.RLock()\n\tdefer s.mu.RUnlock()\n\treturn s.ldm\n}\n\n// LameDuckShutdown will perform a lame duck shutdown of NATS, whereby\n// the client listener is closed, existing client connections are\n// kicked, Raft leaderships are transferred, JetStream is shutdown\n// and then finally shutdown the the NATS Server itself.\n// This function blocks and will not return until the NATS Server\n// has completed the entire shutdown operation.\nfunc (s *Server) LameDuckShutdown() {\n\ts.lameDuckMode()\n}\n\n// This function will close the client listener then close the clients\n// at some interval to avoid a reconnect storm.\n// We will also transfer any raft leaders and shutdown JetStream.\nfunc (s *Server) lameDuckMode() {\n\ts.mu.Lock()\n\t// Check if there is actually anything to do\n\tif s.isShuttingDown() || s.ldm || s.listener == nil {\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\ts.Noticef(\"Entering lame duck mode, stop accepting new clients\")\n\ts.ldm = true\n\ts.sendLDMShutdownEventLocked()\n\texpected := 1\n\ts.listener.Close()\n\ts.listener = nil\n\texpected += s.closeWebsocketServer()\n\ts.ldmCh = make(chan bool, expected)\n\topts := s.getOpts()\n\tgp := opts.LameDuckGracePeriod\n\t// For tests, we want the grace period to be in some cases bigger\n\t// than the ldm duration, so to by-pass the validateOptions() check,\n\t// we use negative number and flip it here.\n\tif gp < 0 {\n\t\tgp *= -1\n\t}\n\ts.mu.Unlock()\n\n\t// If we are running any raftNodes transfer leaders.\n\tif hadTransfers := s.transferRaftLeaders(); hadTransfers {\n\t\t// They will transfer leadership quickly, but wait here for a second.\n\t\tselect {\n\t\tcase <-time.After(time.Second):\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Now check and shutdown jetstream.\n\ts.shutdownJetStream()\n\n\t// Now shutdown the nodes\n\ts.shutdownRaftNodes()\n\n\t// Wait for accept loops to be done to make sure that no new\n\t// client can connect\n\tfor i := 0; i < expected; i++ {\n\t\t<-s.ldmCh\n\t}\n\n\ts.mu.Lock()\n\t// Need to recheck few things\n\tif s.isShuttingDown() || len(s.clients) == 0 {\n\t\ts.mu.Unlock()\n\t\t// If there is no client, we need to call Shutdown() to complete\n\t\t// the LDMode. If server has been shutdown while lock was released,\n\t\t// calling Shutdown() should be no-op.\n\t\ts.Shutdown()\n\t\treturn\n\t}\n\tdur := int64(opts.LameDuckDuration)\n\tdur -= int64(gp)\n\tif dur <= 0 {\n\t\tdur = int64(time.Second)\n\t}\n\tnumClients := int64(len(s.clients))\n\tbatch := 1\n\t// Sleep interval between each client connection close.\n\tvar si int64\n\tif numClients != 0 {\n\t\tsi = dur / numClients\n\t}\n\tif si < 1 {\n\t\t// Should not happen (except in test with very small LD duration), but\n\t\t// if there are too many clients, batch the number of close and\n\t\t// use a tiny sleep interval that will result in yield likely.\n\t\tsi = 1\n\t\tbatch = int(numClients / dur)\n\t} else if si > int64(time.Second) {\n\t\t// Conversely, there is no need to sleep too long between clients\n\t\t// and spread say 10 clients for the 2min duration. Sleeping no\n\t\t// more than 1sec.\n\t\tsi = int64(time.Second)\n\t}\n\n\t// Now capture all clients\n\tclients := make([]*client, 0, len(s.clients))\n\tfor _, client := range s.clients {\n\t\tclients = append(clients, client)\n\t}\n\t// Now that we know that no new client can be accepted,\n\t// send INFO to routes and clients to notify this state.\n\ts.sendLDMToRoutes()\n\ts.sendLDMToClients()\n\ts.mu.Unlock()\n\n\tt := time.NewTimer(gp)\n\t// Delay start of closing of client connections in case\n\t// we have several servers that we want to signal to enter LD mode\n\t// and not have their client reconnect to each other.\n\tselect {\n\tcase <-t.C:\n\t\ts.Noticef(\"Closing existing clients\")\n\tcase <-s.quitCh:\n\t\tt.Stop()\n\t\treturn\n\t}\n\tfor i, client := range clients {\n\t\tclient.closeConnection(ServerShutdown)\n\t\tif i == len(clients)-1 {\n\t\t\tbreak\n\t\t}\n\t\tif batch == 1 || i%batch == 0 {\n\t\t\t// We pick a random interval which will be at least si/2\n\t\t\tv := rand.Int63n(si)\n\t\t\tif v < si/2 {\n\t\t\t\tv = si / 2\n\t\t\t}\n\t\t\tt.Reset(time.Duration(v))\n\t\t\t// Sleep for given interval or bail out if kicked by Shutdown().\n\t\t\tselect {\n\t\t\tcase <-t.C:\n\t\t\tcase <-s.quitCh:\n\t\t\t\tt.Stop()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\ts.Shutdown()\n\ts.WaitForShutdown()\n}\n\n// Send an INFO update to routes with the indication that this server is in LDM mode.\n// Server lock is held on entry.\nfunc (s *Server) sendLDMToRoutes() {\n\ts.routeInfo.LameDuckMode = true\n\tinfoJSON := generateInfoJSON(&s.routeInfo)\n\ts.forEachRemote(func(r *client) {\n\t\tr.mu.Lock()\n\t\tr.enqueueProto(infoJSON)\n\t\tr.mu.Unlock()\n\t})\n\t// Clear now so that we notify only once, should we have to send other INFOs.\n\ts.routeInfo.LameDuckMode = false\n}\n\n// Send an INFO update to clients with the indication that this server is in\n// LDM mode and with only URLs of other nodes.\n// Server lock is held on entry.\nfunc (s *Server) sendLDMToClients() {\n\ts.info.LameDuckMode = true\n\t// Clear this so that if there are further updates, we don't send our URLs.\n\ts.clientConnectURLs = s.clientConnectURLs[:0]\n\tif s.websocket.connectURLs != nil {\n\t\ts.websocket.connectURLs = s.websocket.connectURLs[:0]\n\t}\n\t// Reset content first.\n\ts.info.ClientConnectURLs = s.info.ClientConnectURLs[:0]\n\ts.info.WSConnectURLs = s.info.WSConnectURLs[:0]\n\t// Only add the other nodes if we are allowed to.\n\tif !s.getOpts().Cluster.NoAdvertise {\n\t\tfor url := range s.clientConnectURLsMap {\n\t\t\ts.info.ClientConnectURLs = append(s.info.ClientConnectURLs, url)\n\t\t}\n\t\tfor url := range s.websocket.connectURLsMap {\n\t\t\ts.info.WSConnectURLs = append(s.info.WSConnectURLs, url)\n\t\t}\n\t}\n\t// Send to all registered clients that support async INFO protocols.\n\ts.sendAsyncInfoToClients(true, true)\n\t// We now clear the info.LameDuckMode flag so that if there are\n\t// cluster updates and we send the INFO, we don't have the boolean\n\t// set which would cause multiple LDM notifications to clients.\n\ts.info.LameDuckMode = false\n}\n\n// If given error is a net.Error and is temporary, sleeps for the given\n// delay and double it, but cap it to ACCEPT_MAX_SLEEP. The sleep is\n// interrupted if the server is shutdown.\n// An error message is displayed depending on the type of error.\n// Returns the new (or unchanged) delay, or a negative value if the\n// server has been or is being shutdown.\nfunc (s *Server) acceptError(acceptName string, err error, tmpDelay time.Duration) time.Duration {\n\tif !s.isRunning() {\n\t\treturn -1\n\t}\n\t//lint:ignore SA1019 We want to retry on a bunch of errors here.\n\tif ne, ok := err.(net.Error); ok && ne.Temporary() { // nolint:staticcheck\n\t\ts.Errorf(\"Temporary %s Accept Error(%v), sleeping %dms\", acceptName, ne, tmpDelay/time.Millisecond)\n\t\tselect {\n\t\tcase <-time.After(tmpDelay):\n\t\tcase <-s.quitCh:\n\t\t\treturn -1\n\t\t}\n\t\ttmpDelay *= 2\n\t\tif tmpDelay > ACCEPT_MAX_SLEEP {\n\t\t\ttmpDelay = ACCEPT_MAX_SLEEP\n\t\t}\n\t} else {\n\t\ts.Errorf(\"%s Accept error: %v\", acceptName, err)\n\t}\n\treturn tmpDelay\n}\n\nvar errNoIPAvail = errors.New(\"no IP available\")\n\nfunc (s *Server) getRandomIP(resolver netResolver, url string, excludedAddresses map[string]struct{}) (string, error) {\n\thost, port, err := net.SplitHostPort(url)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\t// If already an IP, skip.\n\tif net.ParseIP(host) != nil {\n\t\treturn url, nil\n\t}\n\tips, err := resolver.LookupHost(context.Background(), host)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"lookup for host %q: %v\", host, err)\n\t}\n\tif len(excludedAddresses) > 0 {\n\t\tfor i := 0; i < len(ips); i++ {\n\t\t\tip := ips[i]\n\t\t\taddr := net.JoinHostPort(ip, port)\n\t\t\tif _, excluded := excludedAddresses[addr]; excluded {\n\t\t\t\tif len(ips) == 1 {\n\t\t\t\t\tips = nil\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tips[i] = ips[len(ips)-1]\n\t\t\t\tips = ips[:len(ips)-1]\n\t\t\t\ti--\n\t\t\t}\n\t\t}\n\t\tif len(ips) == 0 {\n\t\t\treturn \"\", errNoIPAvail\n\t\t}\n\t}\n\tvar address string\n\tif len(ips) == 0 {\n\t\ts.Warnf(\"Unable to get IP for %s, will try with %s: %v\", host, url, err)\n\t\taddress = url\n\t} else {\n\t\tvar ip string\n\t\tif len(ips) == 1 {\n\t\t\tip = ips[0]\n\t\t} else {\n\t\t\tip = ips[rand.Int31n(int32(len(ips)))]\n\t\t}\n\t\t// add the port\n\t\taddress = net.JoinHostPort(ip, port)\n\t}\n\treturn address, nil\n}\n\n// Returns true for the first attempt and depending on the nature\n// of the attempt (first connect or a reconnect), when the number\n// of attempts is equal to the configured report attempts.\nfunc (s *Server) shouldReportConnectErr(firstConnect bool, attempts int) bool {\n\topts := s.getOpts()\n\tif firstConnect {\n\t\tif attempts == 1 || attempts%opts.ConnectErrorReports == 0 {\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\tif attempts == 1 || attempts%opts.ReconnectErrorReports == 0 {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (s *Server) updateRemoteSubscription(acc *Account, sub *subscription, delta int32) {\n\ts.updateRouteSubscriptionMap(acc, sub, delta)\n\tif s.gateway.enabled {\n\t\ts.gatewayUpdateSubInterest(acc.Name, sub, delta)\n\t}\n\n\tacc.updateLeafNodes(sub, delta)\n}\n\nfunc (s *Server) startRateLimitLogExpiration() {\n\tinterval := time.Second\n\ts.startGoRoutine(func() {\n\t\tdefer s.grWG.Done()\n\n\t\tticker := time.NewTicker(time.Second)\n\t\tdefer ticker.Stop()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\tcase interval = <-s.rateLimitLoggingCh:\n\t\t\t\tticker.Reset(interval)\n\t\t\tcase <-ticker.C:\n\t\t\t\ts.rateLimitLogging.Range(func(k, v any) bool {\n\t\t\t\t\tstart := v.(time.Time)\n\t\t\t\t\tif time.Since(start) >= interval {\n\t\t\t\t\t\ts.rateLimitLogging.Delete(k)\n\t\t\t\t\t}\n\t\t\t\t\treturn true\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc (s *Server) changeRateLimitLogInterval(d time.Duration) {\n\tif d <= 0 {\n\t\treturn\n\t}\n\tselect {\n\tcase s.rateLimitLoggingCh <- d:\n\tdefault:\n\t}\n}\n\n// DisconnectClientByID disconnects a client by connection ID\nfunc (s *Server) DisconnectClientByID(id uint64) error {\n\tif s == nil {\n\t\treturn ErrServerNotRunning\n\t}\n\tif client := s.getClient(id); client != nil {\n\t\tclient.closeConnection(Kicked)\n\t\treturn nil\n\t} else if client = s.GetLeafNode(id); client != nil {\n\t\tclient.closeConnection(Kicked)\n\t\treturn nil\n\t}\n\treturn errors.New(\"no such client or leafnode id\")\n}\n\n// LDMClientByID sends a Lame Duck Mode info message to a client by connection ID\nfunc (s *Server) LDMClientByID(id uint64) error {\n\tif s == nil {\n\t\treturn ErrServerNotRunning\n\t}\n\ts.mu.RLock()\n\tc := s.clients[id]\n\tif c == nil {\n\t\ts.mu.RUnlock()\n\t\treturn errors.New(\"no such client id\")\n\t}\n\tinfo := s.copyInfo()\n\tinfo.LameDuckMode = true\n\ts.mu.RUnlock()\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif c.opts.Protocol >= ClientProtoInfo && c.flags.isSet(firstPongSent) {\n\t\t// sendInfo takes care of checking if the connection is still\n\t\t// valid or not, so don't duplicate tests here.\n\t\tc.Debugf(\"Sending Lame Duck Mode info to client\")\n\t\tc.enqueueProto(c.generateClientInfoJSON(info, true))\n\t\treturn nil\n\t} else {\n\t\treturn errors.New(\"client does not support Lame Duck Mode or is not ready to receive the notification\")\n\t}\n}\n" }, { "path": "server/server_fuzz_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n)\n\nconst (\n\ttlsHandshake byte = 22\n\ttlsClientHello byte = 1\n)\n\ntype ClientHelloInjector struct {\n\tsock io.ReadWriteCloser\n\ttlsVersion uint16\n\tbuf []byte\n}\n\nfunc NewClientHelloInjector(s io.ReadWriteCloser, tlsVer uint16, b []byte) *ClientHelloInjector {\n\treturn &ClientHelloInjector{\n\t\tsock: s,\n\t\ttlsVersion: tlsVer,\n\t\tbuf: b,\n\t}\n}\n\nfunc (i *ClientHelloInjector) inject(b []byte) []byte {\n\tif !(b != nil && b[0] == tlsHandshake) {\n\t\treturn b\n\t}\n\n\thsLen := (uint16(b[3]) << 8) + uint16(b[4])\n\n\tif !(hsLen > 0 && b[5] == tlsClientHello) {\n\t\treturn b\n\t}\n\n\t// fuzz tls version in client hello\n\tb[9] = uint8(i.tlsVersion >> 8)\n\tb[10] = uint8(i.tlsVersion & 0xFF)\n\n\t// Go to begin of random opaque\n\toffset := 11\n\n\trandomOpaque := b[offset : offset+32]\n\n\tcopy(randomOpaque, i.buf)\n\n\toffset += 32\n\n\tsessionIDLen := b[offset]\n\n\t// Skip session id len + opaque\n\toffset += 1 + int(sessionIDLen)\n\n\tcypherSuiteLen := (uint16(b[offset]) << 8) + uint16(b[offset+1])\n\n\t// Skip cypherSuiteLen\n\toffset += 2\n\n\tcupherSuites := b[offset : offset+int(cypherSuiteLen)]\n\n\t// Leave unchanged if i.cypherSuites empty\n\tcopy(cupherSuites, i.buf)\n\n\t// Skip cypherSuites\n\toffset += int(cypherSuiteLen)\n\n\t// Skip CompressionMethod\n\toffset += 2\n\n\textensionsLen := (uint16(b[offset]) << 8) + uint16(b[offset+1])\n\n\t// Skip extensions length\n\toffset += 2\n\n\t// Extensions slice. Stub for future use\n\t_ = b[offset : offset+int(extensionsLen)]\n\n\treturn b\n}\n\nfunc (i *ClientHelloInjector) Write(b []byte) (int, error) {\n\treturn i.sock.Write(i.inject(b))\n}\n\nfunc (i *ClientHelloInjector) Read(b []byte) (int, error) {\n\treturn i.sock.Read(b)\n}\n\nfunc (i *ClientHelloInjector) Close() error {\n\treturn i.sock.Close()\n}\n\ntype FakeSocket struct {\n\tsockName string\n\tbuf []byte\n\tdata chan []byte\n\tdone chan struct{}\n}\n\nfunc NewFakeSocket(name string, capacity int) *FakeSocket {\n\treturn &FakeSocket{\n\t\tsockName: name,\n\t\tdata: make(chan []byte, capacity),\n\t\tdone: make(chan struct{}),\n\t}\n}\n\nfunc (s *FakeSocket) Write(b []byte) (int, error) {\n\tselect {\n\tcase s.data <- b:\n\t\treturn len(b), nil\n\tcase <-s.done:\n\t\treturn 0, net.ErrClosed\n\t}\n}\n\nfunc (s *FakeSocket) readChunk(b []byte) (int, error) {\n\tn := copy(b, s.buf)\n\ts.buf = s.buf[n:]\n\treturn n, nil\n}\n\nfunc (s *FakeSocket) Read(b []byte) (int, error) {\n\tif len(s.buf) > 0 {\n\t\treturn s.readChunk(b)\n\t}\n\n\tselect {\n\tcase buf, ok := <-s.data:\n\t\tif !ok {\n\t\t\treturn 0, nil\n\t\t}\n\t\ts.buf = buf\n\t\treturn s.readChunk(b)\n\tcase <-s.done:\n\t\treturn 0, nil\n\t}\n}\n\nfunc (s *FakeSocket) Close() error {\n\tclose(s.done)\n\treturn nil\n}\n\ntype FakeConn struct {\n\tlocal io.ReadWriteCloser\n\tremote io.ReadWriteCloser\n}\n\nfunc NewFakeConn(loc io.ReadWriteCloser, rem io.ReadWriteCloser) *FakeConn {\n\treturn &FakeConn{\n\t\tlocal: loc,\n\t\tremote: rem,\n\t}\n}\n\nfunc (c *FakeConn) Read(b []byte) (int, error) {\n\treturn c.local.Read(b)\n}\n\nfunc (c *FakeConn) Write(b []byte) (int, error) {\n\treturn c.remote.Write(b)\n}\n\nfunc (c *FakeConn) Close() error {\n\treturn c.local.Close()\n}\n\nfunc (c *FakeConn) LocalAddr() net.Addr {\n\treturn &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: \"\"}\n}\n\nfunc (c *FakeConn) RemoteAddr() net.Addr {\n\treturn &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: \"\"}\n}\n\nfunc (c *FakeConn) SetDeadline(t time.Time) error {\n\treturn nil\n}\n\nfunc (c *FakeConn) SetReadDeadline(t time.Time) error {\n\treturn nil\n}\n\nfunc (c *FakeConn) SetWriteDeadline(t time.Time) error {\n\treturn nil\n}\n\ntype FakeListener struct {\n\tch chan *FakeConn\n\tacceptErr error\n}\n\nfunc NewFakeListener() *FakeListener {\n\treturn &FakeListener{\n\t\tch: make(chan *FakeConn),\n\t\tacceptErr: nil,\n\t}\n}\n\nfunc (ln *FakeListener) Accept() (c net.Conn, err error) {\n\treturn <-ln.ch, ln.acceptErr\n}\n\nfunc (ln *FakeListener) Close() error {\n\tln.acceptErr = io.EOF\n\tclose(ln.ch)\n\treturn nil\n}\n\nfunc (ln *FakeListener) Addr() net.Addr {\n\treturn &net.TCPAddr{IP: net.IP{127, 0, 0, 1}, Port: 4222, Zone: \"\"}\n}\n\nfunc getTlsVersion(useTls13 bool) uint16 {\n\tif useTls13 {\n\t\treturn tls.VersionTLS13\n\t}\n\n\treturn tls.VersionTLS12\n}\n\nfunc corruptCert(crt []byte, i uint16) []byte {\n\tcrt[int(i)%len(crt)] ^= 0xFF\n\treturn crt\n}\n\nfunc runServerWithListener(ln net.Listener, opts *Options) *Server {\n\tif opts == nil {\n\t\topts = DefaultOptions()\n\t}\n\ts, err := NewServer(opts)\n\tif err != nil || s == nil {\n\t\tpanic(fmt.Sprintf(\"No NATS Server object returned: %v\", err))\n\t}\n\n\tif !opts.NoLog {\n\t\ts.ConfigureLogger()\n\t}\n\n\ts.listener = ln\n\ts.listenerErr = nil\n\n\t// Run server in Go routine.\n\ts.Start()\n\n\t// Wait for accept loop(s) to be started\n\tif err := s.readyForConnections(10 * time.Second); err != nil {\n\t\tpanic(err)\n\t}\n\treturn s\n}\n\ntype MathRandReader byte\n\nfunc (m MathRandReader) Read(buf []byte) (int, error) {\n\tfor i := range buf {\n\t\tbuf[i] = byte(m)\n\t}\n\treturn len(buf), nil\n}\n\n// FuzzServerTLS performs fuzz testing of the NATS server's TLS handshake implementation.\n// It verifies the server's ability to handle various TLS connection scenarios, including:\n// - Different TLS versions (1.2 and 1.3)\n// - Malformed/mutated client certificates\n// - Corrupted TLS handshake data\n// - Edge cases in the TLS negotiation process\n//\n// Test Setup:\n// - Configures a server with mutual TLS authentication using test certificates\n// - Creates a client with configurable TLS parameters\n// - Uses fake network connections to inject test cases\n//\n// Fuzzing Parameters:\n// - useTls13: Boolean flag to test TLS 1.3 (true) or TLS 1.2 (false)\n// - tlsVer: TLS version number to use in ClientHello\n// - buf: Additional bytes to inject into ClientHello message\n// - corruptCertOffset: Position to corrupt in client certificate (MaxUint16 = no corruption)\n//\n// Expectations\n// The server should either:\n// - Successfully complete the TLS handshake and protocol exchange, or\n// - Cleanly reject invalid connections without crashing\nfunc FuzzServerTLS(f *testing.F) {\n\tsrvTc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/tlsauth/server.pem\",\n\t\tKeyFile: \"../test/configs/certs/tlsauth/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/tlsauth/ca.pem\",\n\t\tInsecure: false,\n\t\tVerify: true,\n\t}\n\n\tsrvTlsCfg, err := GenTLSConfig(srvTc)\n\tif err != nil {\n\t\tf.Fatalf(\"Error generating server tls config: %v\", err)\n\t}\n\n\topts := &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: 4222,\n\t\tNoLog: true,\n\t\tNoSigs: true,\n\t\tDebug: true,\n\t\tTrace: true,\n\t\tTLSHandshakeFirst: true,\n\t\tAllowNonTLS: false,\n\t\tJetStream: false,\n\t\tTLSConfig: srvTlsCfg,\n\t\tCheckConfig: false,\n\t}\n\n\tclientCerts, err := tls.LoadX509KeyPair(\"../test/configs/certs/tlsauth/client.pem\", \"../test/configs/certs/tlsauth/client-key.pem\")\n\tif err != nil {\n\t\tf.Fatalf(\"client1 certificate load error: %s\", err)\n\t}\n\n\tclientTlsCfg := &tls.Config{\n\t\tInsecureSkipVerify: true,\n\t\tRand: MathRandReader(0),\n\t\tTime: func() time.Time { return time.Date(2025, 1, 1, 1, 1, 1, 1, nil) },\n\t}\n\n\ttlsVer := uint16(0x0303)\n\n\tcorpuses := []struct {\n\t\tuseTls13 bool\n\t\tclientHelloTlsVer uint16\n\t\tbuf []byte\n\t\tcorruptCertOffset uint16\n\t}{\n\t\t{useTls13: false, clientHelloTlsVer: tlsVer, buf: []byte{}, corruptCertOffset: math.MaxUint16},\n\t\t{useTls13: true, clientHelloTlsVer: tlsVer, buf: []byte{}, corruptCertOffset: math.MaxUint16},\n\t}\n\n\tfor _, crp := range corpuses {\n\t\tf.Add(crp.useTls13, crp.clientHelloTlsVer, crp.buf, crp.corruptCertOffset)\n\t}\n\n\tf.Fuzz(func(t *testing.T, useTls13 bool, tlsVer uint16, buf []byte, corruptCertOffset uint16) {\n\t\tln := NewFakeListener()\n\t\ts := runServerWithListener(ln, opts.Clone())\n\t\tdefer s.Shutdown()\n\n\t\tclientSocket := NewFakeSocket(\"CLIENT\", 8)\n\t\tserverSocket := NewClientHelloInjector(NewFakeSocket(\"SERVER\", 8), tlsVer, buf)\n\n\t\tclientConn := NewFakeConn(clientSocket, serverSocket)\n\t\tserverConn := NewFakeConn(serverSocket, clientSocket)\n\n\t\t// Connect to server\n\t\tln.ch <- serverConn\n\n\t\ttlsVersion := getTlsVersion(useTls13)\n\n\t\ttlsCfg := clientTlsCfg.Clone()\n\t\ttlsCfg.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {\n\t\t\tif corruptCertOffset == math.MaxUint16 {\n\t\t\t\tt.Log(\"Leave certificate unchanged\")\n\t\t\t\treturn &clientCerts, nil\n\t\t\t}\n\n\t\t\torigCert := clientCerts.Certificate[0]\n\t\t\tnewCert := make([]byte, len(origCert))\n\t\t\tcopy(newCert, origCert)\n\n\t\t\tnewTlsCerts := clientCerts\n\t\t\tnewTlsCerts.Certificate[0] = corruptCert(newCert, corruptCertOffset)\n\n\t\t\treturn &newTlsCerts, nil\n\t\t}\n\t\ttlsCfg.MaxVersion = tlsVersion\n\t\ttlsCfg.MinVersion = tlsVersion\n\n\t\ttlsClientConn := tls.Client(clientConn, tlsCfg)\n\t\tdefer tlsClientConn.Close()\n\n\t\tif err := tlsClientConn.Handshake(); err != nil {\n\t\t\tt.Logf(\"Handshake error: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\tbr := bufio.NewReaderSize(tlsClientConn, 128)\n\t\tif _, err := br.ReadString('\\n'); err != nil {\n\t\t\tt.Logf(\"Unexpected error reading INFO message: %v\", err)\n\t\t\treturn\n\t\t}\n\t})\n}\n" }, { "path": "server/server_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/antithesis\"\n\tsrvlog \"github.com/nats-io/nats-server/v2/logger\"\n)\n\nfunc checkForErr(totalWait, sleepDur time.Duration, f func() error) error {\n\ttimeout := time.Now().Add(totalWait)\n\tvar err error\n\tfor time.Now().Before(timeout) {\n\t\terr = f()\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\t\ttime.Sleep(sleepDur)\n\t}\n\treturn err\n}\n\nfunc checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) {\n\tt.Helper()\n\terr := checkForErr(totalWait, sleepDur, f)\n\tif err != nil {\n\t\tantithesis.AssertUnreachable(t, \"Timeout in checkFor\", nil)\n\t\tt.Fatal(err.Error())\n\t}\n}\n\nfunc DefaultOptions() *Options {\n\treturn &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tHTTPPort: -1,\n\t\tCluster: ClusterOpts{Port: -1, Name: \"abc\"},\n\t\tNoLog: true,\n\t\tNoSigs: true,\n\t\tDebug: true,\n\t\tTrace: true,\n\t}\n}\n\n// New Go Routine based server\nfunc RunServer(opts *Options) *Server {\n\tif opts == nil {\n\t\topts = DefaultOptions()\n\t}\n\ts, err := NewServer(opts)\n\tif err != nil || s == nil {\n\t\tpanic(fmt.Sprintf(\"No NATS Server object returned: %v\", err))\n\t}\n\n\tif !opts.NoLog {\n\t\ts.ConfigureLogger()\n\t}\n\n\tif ll := os.Getenv(\"NATS_LOGGING\"); ll != \"\" {\n\t\tlog := srvlog.NewTestLogger(fmt.Sprintf(\"[%s] | \", s), true)\n\t\tdebug := ll == \"debug\" || ll == \"trace\"\n\t\ttrace := ll == \"trace\"\n\t\ts.SetLoggerV2(log, debug, trace, false)\n\t}\n\n\t// Run server in Go routine.\n\ts.Start()\n\n\t// Wait for accept loop(s) to be started\n\tif err := s.readyForConnections(10 * time.Second); err != nil {\n\t\tpanic(err)\n\t}\n\treturn s\n}\n\n// LoadConfig loads a configuration from a filename\nfunc LoadConfig(configFile string) (opts *Options) {\n\topts, err := ProcessConfigFile(configFile)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Error processing configuration file: %v\", err))\n\t}\n\topts.NoSigs, opts.NoLog = true, opts.LogFile == _EMPTY_\n\treturn\n}\n\n// RunServerWithConfig starts a new Go routine based server with a configuration file.\nfunc RunServerWithConfig(configFile string) (srv *Server, opts *Options) {\n\topts = LoadConfig(configFile)\n\tsrv = RunServer(opts)\n\treturn\n}\n\nfunc TestSemanticVersion(t *testing.T) {\n\tif !semVerRe.MatchString(VERSION) {\n\t\tt.Fatalf(\"Version (%s) is not a valid SemVer string\", VERSION)\n\t}\n}\n\nfunc TestVersionMatchesTag(t *testing.T) {\n\ttag := os.Getenv(\"TRAVIS_TAG\")\n\t// Travis started to return '' when no tag is set. Support both now.\n\tif tag == \"\" || tag == \"''\" {\n\t\tt.SkipNow()\n\t}\n\t// We expect a tag of the form vX.Y.Z. If that's not the case,\n\t// we need someone to have a look. So fail if first letter is not\n\t// a `v`\n\tif tag[0] != 'v' {\n\t\tt.Fatalf(\"Expect tag to start with `v`, tag is: %s\", tag)\n\t}\n\t// Strip the `v` from the tag for the version comparison.\n\tif VERSION != tag[1:] {\n\t\tt.Fatalf(\"Version (%s) does not match tag (%s)\", VERSION, tag[1:])\n\t}\n\t// Check that the version dynamically set via ldflags matches the version\n\t// from the server previous to releasing.\n\tif serverVersion == _EMPTY_ {\n\t\tt.Fatal(\"Version missing in ldflags\")\n\t}\n\t// Unlike VERSION constant, serverVersion is prefixed with a 'v'\n\t// since it should be the same as the git tag.\n\texpected := \"v\" + VERSION\n\tif serverVersion != _EMPTY_ && expected != serverVersion {\n\t\tt.Fatalf(\"Version (%s) does not match ldflags version (%s)\", expected, serverVersion)\n\t}\n}\n\nfunc TestStartProfiler(t *testing.T) {\n\ts := New(DefaultOptions())\n\ts.StartProfiler()\n\ts.mu.Lock()\n\ts.profiler.Close()\n\ts.mu.Unlock()\n}\n\nfunc TestStartupAndShutdown(t *testing.T) {\n\topts := DefaultOptions()\n\topts.NoSystemAccount = true\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tif !s.isRunning() {\n\t\tt.Fatal(\"Could not run server\")\n\t}\n\n\t// Debug stuff.\n\tnumRoutes := s.NumRoutes()\n\tif numRoutes != 0 {\n\t\tt.Fatalf(\"Expected numRoutes to be 0 vs %d\\n\", numRoutes)\n\t}\n\n\tnumRemotes := s.NumRemotes()\n\tif numRemotes != 0 {\n\t\tt.Fatalf(\"Expected numRemotes to be 0 vs %d\\n\", numRemotes)\n\t}\n\n\tnumClients := s.NumClients()\n\tif numClients != 0 && numClients != 1 {\n\t\tt.Fatalf(\"Expected numClients to be 1 or 0 vs %d\\n\", numClients)\n\t}\n\n\tnumSubscriptions := s.NumSubscriptions()\n\tif numSubscriptions != 0 {\n\t\tt.Fatalf(\"Expected numSubscriptions to be 0 vs %d\\n\", numSubscriptions)\n\t}\n}\n\nfunc TestTLSVersions(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tvalue uint16\n\t\texpected string\n\t}{\n\t\t{\"1.0\", tls.VersionTLS10, \"1.0\"},\n\t\t{\"1.1\", tls.VersionTLS11, \"1.1\"},\n\t\t{\"1.2\", tls.VersionTLS12, \"1.2\"},\n\t\t{\"1.3\", tls.VersionTLS13, \"1.3\"},\n\t\t{\"unknown\", 0x999, \"Unknown [0x999]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif v := tlsVersion(test.value); v != test.expected {\n\t\t\t\tt.Fatalf(\"Expected value 0x%x to be %q, got %q\", test.value, test.expected, v)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTLSMinVersionConfig(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \t\"../test/configs/certs/server-cert.pem\"\n\t\t\tkey_file: \t\"../test/configs/certs/server-key.pem\"\n\t\t\ttimeout: \t1\n\t\t\tmin_version: \t%s\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, `\"1.3\"`)))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tconnect := func(t *testing.T, tlsConf *tls.Config, expectedErr error) {\n\t\tt.Helper()\n\t\topts := []nats.Option{}\n\t\tif tlsConf != nil {\n\t\t\topts = append(opts, nats.Secure(tlsConf))\n\t\t}\n\t\topts = append(opts, nats.RootCAs(\"../test/configs/certs/ca.pem\"))\n\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", o.Port), opts...)\n\t\tif err == nil {\n\t\t\tdefer nc.Close()\n\t\t}\n\t\tif expectedErr == nil {\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t} else if err == nil || err.Error() != expectedErr.Error() {\n\t\t\tnc.Close()\n\t\t\tt.Fatalf(\"Expected error %v, got: %v\", expectedErr, err)\n\t\t}\n\t}\n\n\t// Cannot connect with client requiring a lower minimum TLS Version.\n\tconnect(t, &tls.Config{\n\t\tMaxVersion: tls.VersionTLS12,\n\t}, errors.New(`remote error: tls: protocol version not supported`))\n\n\t// Should connect since matching minimum TLS version.\n\tconnect(t, &tls.Config{\n\t\tMinVersion: tls.VersionTLS13,\n\t}, nil)\n\n\t// Reloading with invalid values should fail.\n\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `\"1.0\"`)), 0666); err != nil {\n\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t}\n\tif err := s.Reload(); err == nil {\n\t\tt.Fatalf(\"Expected reload to fail: %v\", err)\n\t}\n\n\t// Reloading with original values and no changes should be ok.\n\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `\"1.3\"`)), 0666); err != nil {\n\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Unexpected error reloading TLS version: %v\", err)\n\t}\n\n\t// Reloading with a new minimum lower version.\n\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `\"1.2\"`)), 0666); err != nil {\n\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Unexpected error reloading: %v\", err)\n\t}\n\n\t// Should connect since now matching minimum TLS version.\n\tconnect(t, &tls.Config{\n\t\tMaxVersion: tls.VersionTLS12,\n\t}, nil)\n\tconnect(t, &tls.Config{\n\t\tMinVersion: tls.VersionTLS13,\n\t}, nil)\n\n\t// Setting unsupported TLS versions\n\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, `\"1.4\"`)), 0666); err != nil {\n\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t}\n\tif err := s.Reload(); err == nil || !strings.Contains(err.Error(), `unknown version: 1.4`) {\n\t\tt.Fatalf(\"Unexpected error reloading: %v\", err)\n\t}\n\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\tTimeout: 4.0,\n\t\tMinVersion: tls.VersionTLS11,\n\t}\n\t_, err := GenTLSConfig(tc)\n\tif err == nil || err.Error() != `unsupported minimum TLS version: TLS 1.1` {\n\t\tt.Fatalf(\"Expected error generating TLS config: %v\", err)\n\t}\n}\n\nfunc TestTLSCipher(t *testing.T) {\n\trequire_Equal(t, tls.CipherSuiteName(0x0005), \"TLS_RSA_WITH_RC4_128_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0x000a), \"TLS_RSA_WITH_3DES_EDE_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0x002f), \"TLS_RSA_WITH_AES_128_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0x0035), \"TLS_RSA_WITH_AES_256_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc007), \"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc009), \"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc00a), \"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc011), \"TLS_ECDHE_RSA_WITH_RC4_128_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc012), \"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc013), \"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc014), \"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc02f), \"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc02b), \"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc030), \"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\")\n\trequire_Equal(t, tls.CipherSuiteName(0xc02c), \"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\")\n\trequire_Equal(t, tls.CipherSuiteName(0x1301), \"TLS_AES_128_GCM_SHA256\")\n\trequire_Equal(t, tls.CipherSuiteName(0x1302), \"TLS_AES_256_GCM_SHA384\")\n\trequire_Equal(t, tls.CipherSuiteName(0x1303), \"TLS_CHACHA20_POLY1305_SHA256\")\n\trequire_Equal(t, tls.CipherSuiteName(0x9999), \"0x9999\")\n}\n\nfunc TestGetConnectURLs(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = 4222\n\n\tvar globalIP net.IP\n\n\tcheckGlobalConnectURLs := func() {\n\t\ts := New(opts)\n\t\tdefer s.Shutdown()\n\n\t\ts.mu.Lock()\n\t\turls := s.getClientConnectURLs()\n\t\ts.mu.Unlock()\n\t\tif len(urls) == 0 {\n\t\t\tt.Fatalf(\"Expected to get a list of urls, got none for listen addr: %v\", opts.Host)\n\t\t}\n\t\tfor _, u := range urls {\n\t\t\ttcpaddr, err := net.ResolveTCPAddr(\"tcp\", u)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error resolving: %v\", err)\n\t\t\t}\n\t\t\tip := tcpaddr.IP\n\t\t\tif !ip.IsGlobalUnicast() {\n\t\t\t\tt.Fatalf(\"IP %v is not global\", ip.String())\n\t\t\t}\n\t\t\tif ip.IsUnspecified() {\n\t\t\t\tt.Fatalf(\"IP %v is unspecified\", ip.String())\n\t\t\t}\n\t\t\taddr := strings.TrimSuffix(u, \":4222\")\n\t\t\tif addr == opts.Host {\n\t\t\t\tt.Fatalf(\"Returned url is not right: %v\", u)\n\t\t\t}\n\t\t\tif globalIP == nil {\n\t\t\t\tglobalIP = ip\n\t\t\t}\n\t\t}\n\t}\n\n\tlistenAddrs := []string{\"0.0.0.0\", \"::\"}\n\tfor _, listenAddr := range listenAddrs {\n\t\topts.Host = listenAddr\n\t\tcheckGlobalConnectURLs()\n\t}\n\n\tcheckConnectURLsHasOnlyOne := func() {\n\t\ts := New(opts)\n\t\tdefer s.Shutdown()\n\n\t\ts.mu.Lock()\n\t\turls := s.getClientConnectURLs()\n\t\ts.mu.Unlock()\n\t\tif len(urls) != 1 {\n\t\t\tt.Fatalf(\"Expected one URL, got %v\", urls)\n\t\t}\n\t\ttcpaddr, err := net.ResolveTCPAddr(\"tcp\", urls[0])\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error resolving: %v\", err)\n\t\t}\n\t\tip := tcpaddr.IP\n\t\tif ip.String() != opts.Host {\n\t\t\tt.Fatalf(\"Expected connect URL to be %v, got %v\", opts.Host, ip.String())\n\t\t}\n\t}\n\n\tsingleConnectReturned := []string{\"127.0.0.1\", \"::1\"}\n\tif globalIP != nil {\n\t\tsingleConnectReturned = append(singleConnectReturned, globalIP.String())\n\t}\n\tfor _, listenAddr := range singleConnectReturned {\n\t\topts.Host = listenAddr\n\t\tcheckConnectURLsHasOnlyOne()\n\t}\n}\n\nfunc TestInfoServerNameDefaultsToPK(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = 4222\n\topts.ClientAdvertise = \"nats.example.com\"\n\ts := New(opts)\n\tdefer s.Shutdown()\n\n\tif s.info.Name != s.info.ID {\n\t\tt.Fatalf(\"server info hostname is incorrect, got: '%v' expected: '%v'\", s.info.Name, s.info.ID)\n\t}\n}\n\nfunc TestInfoServerNameIsSettable(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = 4222\n\topts.ClientAdvertise = \"nats.example.com\"\n\topts.ServerName = \"test_server_name\"\n\ts := New(opts)\n\tdefer s.Shutdown()\n\n\tif s.info.Name != \"test_server_name\" {\n\t\tt.Fatalf(\"server info hostname is incorrect, got: '%v' expected: 'test_server_name'\", s.info.Name)\n\t}\n}\n\nfunc TestClientAdvertiseConnectURL(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Port = 4222\n\topts.ClientAdvertise = \"nats.example.com\"\n\ts := New(opts)\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\turls := s.getClientConnectURLs()\n\ts.mu.Unlock()\n\tif len(urls) != 1 {\n\t\tt.Fatalf(\"Expected to get one url, got none: %v with ClientAdvertise %v\",\n\t\t\topts.Host, opts.ClientAdvertise)\n\t}\n\tif urls[0] != \"nats.example.com:4222\" {\n\t\tt.Fatalf(\"Expected to get '%s', got: '%v'\", \"nats.example.com:4222\", urls[0])\n\t}\n\ts.Shutdown()\n\n\topts.ClientAdvertise = \"nats.example.com:7777\"\n\ts = New(opts)\n\ts.mu.Lock()\n\turls = s.getClientConnectURLs()\n\ts.mu.Unlock()\n\tif len(urls) != 1 {\n\t\tt.Fatalf(\"Expected to get one url, got none: %v with ClientAdvertise %v\",\n\t\t\topts.Host, opts.ClientAdvertise)\n\t}\n\tif urls[0] != \"nats.example.com:7777\" {\n\t\tt.Fatalf(\"Expected 'nats.example.com:7777', got: '%v'\", urls[0])\n\t}\n\tif s.info.Host != \"nats.example.com\" {\n\t\tt.Fatalf(\"Expected host to be set to nats.example.com\")\n\t}\n\tif s.info.Port != 7777 {\n\t\tt.Fatalf(\"Expected port to be set to 7777\")\n\t}\n\ts.Shutdown()\n\n\topts = DefaultOptions()\n\topts.Port = 0\n\topts.ClientAdvertise = \"nats.example.com:7777\"\n\ts = New(opts)\n\tif s.info.Host != \"nats.example.com\" && s.info.Port != 7777 {\n\t\tt.Fatalf(\"Expected Client Advertise Host:Port to be nats.example.com:7777, got: %s:%d\",\n\t\t\ts.info.Host, s.info.Port)\n\t}\n\ts.Shutdown()\n}\n\nfunc TestClientAdvertiseInCluster(t *testing.T) {\n\toptsA := DefaultOptions()\n\toptsA.ClientAdvertise = \"srvA:4222\"\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tnc := natsConnect(t, srvA.ClientURL())\n\tdefer nc.Close()\n\n\toptsB := DefaultOptions()\n\toptsB.ClientAdvertise = \"srvBC:4222\"\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Cluster.Port))\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tcheckURLs := func(expected string) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tsrvs := nc.DiscoveredServers()\n\t\t\tfor _, u := range srvs {\n\t\t\t\tif u == expected {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"Url %q not found in %q\", expected, srvs)\n\t\t})\n\t}\n\tcheckURLs(\"nats://srvBC:4222\")\n\n\toptsC := DefaultOptions()\n\toptsC.ClientAdvertise = \"srvBC:4222\"\n\toptsC.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Cluster.Port))\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB, srvC)\n\tcheckURLs(\"nats://srvBC:4222\")\n\n\tsrvB.Shutdown()\n\tcheckNumRoutes(t, srvA, DEFAULT_ROUTE_POOL_SIZE+1)\n\tcheckURLs(\"nats://srvBC:4222\")\n}\n\nfunc TestClientAdvertiseErrorOnStartup(t *testing.T) {\n\topts := DefaultOptions()\n\t// Set invalid address\n\topts.ClientAdvertise = \"addr:::123\"\n\ttestFatalErrorOnStart(t, opts, \"ClientAdvertise\")\n}\n\nfunc TestNoDeadlockOnStartFailure(t *testing.T) {\n\topts := DefaultOptions()\n\topts.Host = \"x.x.x.x\" // bad host\n\topts.Port = 4222\n\topts.HTTPHost = opts.Host\n\topts.Cluster.Host = \"127.0.0.1\"\n\topts.Cluster.Port = -1\n\topts.ProfPort = -1\n\ts := New(opts)\n\n\t// This should return since it should fail to start a listener\n\t// on x.x.x.x:4222\n\tch := make(chan struct{})\n\tgo func() {\n\t\ts.Start()\n\t\tclose(ch)\n\t}()\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Start() should have returned due to failure to start listener\")\n\t}\n\n\t// We should be able to shutdown\n\ts.Shutdown()\n}\n\nfunc TestMaxConnections(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxConn = 1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc2, err := nats.Connect(addr)\n\tif err == nil {\n\t\tnc2.Close()\n\t\tt.Fatal(\"Expected connection to fail\")\n\t}\n}\n\nfunc TestMaxConnectionsPreventsAll(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxConn = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t_, err := nats.Connect(addr)\n\trequire_Error(t, err)\n}\n\nfunc TestMaxSubscriptions(t *testing.T) {\n\topts := DefaultOptions()\n\topts.MaxSubs = 10\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := nc.Subscribe(fmt.Sprintf(\"foo.%d\", i), func(*nats.Msg) {})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error subscribing: %v\\n\", err)\n\t\t}\n\t}\n\t// This should cause the error.\n\tnc.Subscribe(\"foo.22\", func(*nats.Msg) {})\n\tnc.Flush()\n\tif err := nc.LastError(); err == nil {\n\t\tt.Fatal(\"Expected an error but got none\\n\")\n\t}\n}\n\nfunc TestProcessCommandLineArgs(t *testing.T) {\n\tvar host string\n\tvar port int\n\tcmd := flag.NewFlagSet(\"nats-server\", flag.ExitOnError)\n\tcmd.StringVar(&host, \"a\", \"0.0.0.0\", \"Host.\")\n\tcmd.IntVar(&port, \"p\", 4222, \"Port.\")\n\n\tcmd.Parse([]string{\"-a\", \"127.0.0.1\", \"-p\", \"9090\"})\n\tshowVersion, showHelp, err := ProcessCommandLineArgs(cmd)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no errors, got: %s\", err)\n\t}\n\tif showVersion || showHelp {\n\t\tt.Errorf(\"Expected not having to handle subcommands\")\n\t}\n\n\tcmd.Parse([]string{\"version\"})\n\tshowVersion, showHelp, err = ProcessCommandLineArgs(cmd)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no errors, got: %s\", err)\n\t}\n\tif !showVersion {\n\t\tt.Errorf(\"Expected having to handle version command\")\n\t}\n\tif showHelp {\n\t\tt.Errorf(\"Expected not having to handle help command\")\n\t}\n\n\tcmd.Parse([]string{\"help\"})\n\tshowVersion, showHelp, err = ProcessCommandLineArgs(cmd)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no errors, got: %s\", err)\n\t}\n\tif showVersion {\n\t\tt.Errorf(\"Expected not having to handle version command\")\n\t}\n\tif !showHelp {\n\t\tt.Errorf(\"Expected having to handle help command\")\n\t}\n\n\tcmd.Parse([]string{\"foo\", \"-p\", \"9090\"})\n\t_, _, err = ProcessCommandLineArgs(cmd)\n\tif err == nil {\n\t\tt.Errorf(\"Expected an error handling the command arguments\")\n\t}\n}\n\nfunc TestRandomPorts(t *testing.T) {\n\topts := DefaultOptions()\n\topts.HTTPPort = -1\n\topts.Port = -1\n\ts := RunServer(opts)\n\n\tdefer s.Shutdown()\n\n\tif s.Addr() == nil || s.Addr().(*net.TCPAddr).Port <= 0 {\n\t\tt.Fatal(\"Should have dynamically assigned server port.\")\n\t}\n\n\tif s.Addr() == nil || s.Addr().(*net.TCPAddr).Port == 4222 {\n\t\tt.Fatal(\"Should not have dynamically assigned default port: 4222.\")\n\t}\n\n\tif s.MonitorAddr() == nil || s.MonitorAddr().Port <= 0 {\n\t\tt.Fatal(\"Should have dynamically assigned monitoring port.\")\n\t}\n\n}\n\nfunc TestNilMonitoringPort(t *testing.T) {\n\topts := DefaultOptions()\n\topts.HTTPPort = 0\n\topts.HTTPSPort = 0\n\ts := RunServer(opts)\n\n\tdefer s.Shutdown()\n\n\tif s.MonitorAddr() != nil {\n\t\tt.Fatal(\"HttpAddr should be nil.\")\n\t}\n}\n\ntype DummyAuth struct {\n\tt *testing.T\n\tneedNonce bool\n\tdeadline time.Time\n\tregister bool\n}\n\nfunc (d *DummyAuth) Check(c ClientAuthentication) bool {\n\tif d.needNonce && len(c.GetNonce()) == 0 {\n\t\td.t.Fatalf(\"Expected a nonce but received none\")\n\t} else if !d.needNonce && len(c.GetNonce()) > 0 {\n\t\td.t.Fatalf(\"Received a nonce when none was expected\")\n\t}\n\n\tif c.GetOpts().Username != \"valid\" {\n\t\treturn false\n\t}\n\n\tif !d.register {\n\t\treturn true\n\t}\n\n\tu := &User{\n\t\tUsername: c.GetOpts().Username,\n\t\tConnectionDeadline: d.deadline,\n\t}\n\tc.RegisterUser(u)\n\n\treturn true\n}\n\nfunc TestCustomClientAuthentication(t *testing.T) {\n\ttestAuth := func(t *testing.T, nonce bool) {\n\t\tclientAuth := &DummyAuth{t: t, needNonce: nonce}\n\n\t\topts := DefaultOptions()\n\t\topts.CustomClientAuthentication = clientAuth\n\t\topts.AlwaysEnableNonce = nonce\n\n\t\ts := RunServer(opts)\n\t\tdefer s.Shutdown()\n\n\t\taddr := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t\tnc, err := nats.Connect(addr, nats.UserInfo(\"valid\", \"\"))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected client to connect, got: %s\", err)\n\t\t}\n\t\tnc.Close()\n\t\tif _, err := nats.Connect(addr, nats.UserInfo(\"invalid\", \"\")); err == nil {\n\t\t\tt.Fatal(\"Expected client to fail to connect\")\n\t\t}\n\t}\n\n\tt.Run(\"with nonce\", func(t *testing.T) { testAuth(t, true) })\n\tt.Run(\"without nonce\", func(t *testing.T) { testAuth(t, false) })\n}\n\nfunc TestCustomRouterAuthentication(t *testing.T) {\n\topts := DefaultOptions()\n\topts.CustomRouterAuthentication = &DummyAuth{}\n\topts.Cluster.Host = \"127.0.0.1\"\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\tclusterPort := s.ClusterAddr().Port\n\n\topts2 := DefaultOptions()\n\topts2.Cluster.Host = \"127.0.0.1\"\n\topts2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://invalid@127.0.0.1:%d\", clusterPort))\n\ts2 := RunServer(opts2)\n\tdefer s2.Shutdown()\n\n\t// s2 will attempt to connect to s, which should reject.\n\t// Keep in mind that s2 will try again...\n\ttime.Sleep(50 * time.Millisecond)\n\tcheckNumRoutes(t, s2, 0)\n\n\topts3 := DefaultOptions()\n\topts3.Cluster.Host = \"127.0.0.1\"\n\topts3.Routes = RoutesFromStr(fmt.Sprintf(\"nats://valid@127.0.0.1:%d\", clusterPort))\n\ts3 := RunServer(opts3)\n\tdefer s3.Shutdown()\n\tcheckClusterFormed(t, s, s3)\n\t// Default pool size + 1 for system account\n\tcheckNumRoutes(t, s3, DEFAULT_ROUTE_POOL_SIZE+1)\n}\n\nfunc TestMonitoringNoTimeout(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\ts.mu.Lock()\n\tsrv := s.monitoringServer\n\ts.mu.Unlock()\n\n\tif srv == nil {\n\t\tt.Fatalf(\"Monitoring server not set\")\n\t}\n\tif srv.ReadTimeout != 0 {\n\t\tt.Fatalf(\"ReadTimeout should not be set, was set to %v\", srv.ReadTimeout)\n\t}\n\tif srv.WriteTimeout != 0 {\n\t\tt.Fatalf(\"WriteTimeout should not be set, was set to %v\", srv.WriteTimeout)\n\t}\n}\n\nfunc TestProfilingNoTimeout(t *testing.T) {\n\topts := DefaultOptions()\n\topts.ProfPort = -1\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tpaddr := s.ProfilerAddr()\n\tif paddr == nil {\n\t\tt.Fatalf(\"Profiler not started\")\n\t}\n\tpport := paddr.Port\n\tif pport <= 0 {\n\t\tt.Fatalf(\"Expected profiler port to be set, got %v\", pport)\n\t}\n\ts.mu.Lock()\n\tsrv := s.profilingServer\n\ts.mu.Unlock()\n\n\tif srv == nil {\n\t\tt.Fatalf(\"Profiling server not set\")\n\t}\n\tif srv.ReadTimeout != time.Second*5 {\n\t\tt.Fatalf(\"ReadTimeout should not be set, was set to %v\", srv.ReadTimeout)\n\t}\n\tif srv.WriteTimeout != 0 {\n\t\tt.Fatalf(\"WriteTimeout should not be set, was set to %v\", srv.WriteTimeout)\n\t}\n}\n\nfunc TestLameDuckOptionsValidation(t *testing.T) {\n\to := DefaultOptions()\n\to.LameDuckDuration = 5 * time.Second\n\to.LameDuckGracePeriod = 10 * time.Second\n\ts, err := NewServer(o)\n\tif s != nil {\n\t\ts.Shutdown()\n\t}\n\tif err == nil || !strings.Contains(err.Error(), \"should be strictly lower\") {\n\t\tt.Fatalf(\"Expected error saying that ldm grace period should be lower than ldm duration, got %v\", err)\n\t}\n}\n\nfunc testSetLDMGracePeriod(o *Options, val time.Duration) {\n\t// For tests, we set the grace period as a negative value\n\t// so we can have a grace period bigger than the total duration.\n\t// When validating options, we would not be able to run the\n\t// server without this trick.\n\to.LameDuckGracePeriod = val * -1\n}\n\nfunc TestLameDuckMode(t *testing.T) {\n\toptsA := DefaultOptions()\n\ttestSetLDMGracePeriod(optsA, time.Nanosecond)\n\toptsA.Cluster.Host = \"127.0.0.1\"\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\t// Check that if there is no client, server is shutdown\n\tsrvA.lameDuckMode()\n\tif !srvA.isShuttingDown() {\n\t\tt.Fatalf(\"Server should have shutdown\")\n\t}\n\n\toptsA.LameDuckDuration = 10 * time.Nanosecond\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB := DefaultOptions()\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\ttotal := 50\n\tconnectClients := func() []*nats.Conn {\n\t\tncs := make([]*nats.Conn, 0, total)\n\t\tfor i := 0; i < total; i++ {\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port),\n\t\t\t\tnats.ReconnectWait(50*time.Millisecond))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t\t}\n\t\t\tncs = append(ncs, nc)\n\t\t}\n\t\treturn ncs\n\t}\n\tstopClientsAndSrvB := func(ncs []*nats.Conn) {\n\t\tfor _, nc := range ncs {\n\t\t\tnc.Close()\n\t\t}\n\t\tsrvB.Shutdown()\n\t}\n\n\tncs := connectClients()\n\n\tcheckClientsCount(t, srvA, total)\n\tcheckClientsCount(t, srvB, 0)\n\n\tstart := time.Now()\n\tsrvA.lameDuckMode()\n\t// Make sure that nothing bad happens if called twice\n\tsrvA.lameDuckMode()\n\t// Wait that shutdown completes\n\telapsed := time.Since(start)\n\t// It should have taken more than the allotted time of 10ms since we had 50 clients.\n\tif elapsed <= optsA.LameDuckDuration {\n\t\tt.Fatalf(\"Expected to take more than %v, got %v\", optsA.LameDuckDuration, elapsed)\n\t}\n\n\tcheckClientsCount(t, srvA, 0)\n\tcheckClientsCount(t, srvB, total)\n\n\t// Check closed status on server A\n\t// Connections are saved in go routines, so although we have evaluated the number\n\t// of connections in the server A to be 0, the polling of connection closed may\n\t// need a bit more time.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tcz := pollConnz(t, srvA, 1, \"\", &ConnzOptions{State: ConnClosed})\n\t\tif n := len(cz.Conns); n != total {\n\t\t\treturn fmt.Errorf(\"expected %v closed connections, got %v\", total, n)\n\t\t}\n\t\treturn nil\n\t})\n\tcz := pollConnz(t, srvA, 1, \"\", &ConnzOptions{State: ConnClosed})\n\tif n := len(cz.Conns); n != total {\n\t\tt.Fatalf(\"Expected %v closed connections, got %v\", total, n)\n\t}\n\tfor _, c := range cz.Conns {\n\t\tcheckReason(t, c.Reason, ServerShutdown)\n\t}\n\n\tstopClientsAndSrvB(ncs)\n\n\toptsA.LameDuckDuration = time.Second\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tncs = connectClients()\n\n\tcheckClientsCount(t, srvA, total)\n\tcheckClientsCount(t, srvB, 0)\n\n\tstart = time.Now()\n\tgo srvA.lameDuckMode()\n\t// Check that while in lameDuckMode, it is not possible to connect\n\t// to the server. Wait to be in LD mode first\n\tcheckFor(t, 500*time.Millisecond, 15*time.Millisecond, func() error {\n\t\tsrvA.mu.Lock()\n\t\tldm := srvA.ldm\n\t\tsrvA.mu.Unlock()\n\t\tif !ldm {\n\t\t\treturn fmt.Errorf(\"Did not reach lame duck mode\")\n\t\t}\n\t\treturn nil\n\t})\n\tif _, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port)); err != nats.ErrNoServers {\n\t\tt.Fatalf(\"Expected %v, got %v\", nats.ErrNoServers, err)\n\t}\n\tsrvA.grWG.Wait()\n\telapsed = time.Since(start)\n\n\tcheckClientsCount(t, srvA, 0)\n\tcheckClientsCount(t, srvB, total)\n\n\tif elapsed > time.Duration(float64(optsA.LameDuckDuration)*1.1) {\n\t\tt.Fatalf(\"Expected to not take more than %v, got %v\", optsA.LameDuckDuration, elapsed)\n\t}\n\n\tstopClientsAndSrvB(ncs)\n\n\t// Now check that we can shutdown server while in LD mode.\n\toptsA.LameDuckDuration = 60 * time.Second\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tncs = connectClients()\n\n\tcheckClientsCount(t, srvA, total)\n\tcheckClientsCount(t, srvB, 0)\n\n\tstart = time.Now()\n\tgo srvA.lameDuckMode()\n\ttime.Sleep(100 * time.Millisecond)\n\tsrvA.Shutdown()\n\telapsed = time.Since(start)\n\t// Make sure that it did not take that long\n\tif elapsed > time.Second {\n\t\tt.Fatalf(\"Took too long: %v\", elapsed)\n\t}\n\tcheckClientsCount(t, srvA, 0)\n\tcheckClientsCount(t, srvB, total)\n\n\tstopClientsAndSrvB(ncs)\n\n\t// Now test that we introduce delay before starting closing client connections.\n\t// This allow to \"signal\" multiple servers and avoid their clients to reconnect\n\t// to a server that is going to be going in LD mode.\n\ttestSetLDMGracePeriod(optsA, 100*time.Millisecond)\n\toptsA.LameDuckDuration = 10 * time.Millisecond\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\ttestSetLDMGracePeriod(optsB, 100*time.Millisecond)\n\toptsB.LameDuckDuration = 10 * time.Millisecond\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\toptsC := DefaultOptions()\n\toptsC.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\ttestSetLDMGracePeriod(optsC, 100*time.Millisecond)\n\toptsC.LameDuckGracePeriod = -100 * time.Millisecond\n\toptsC.LameDuckDuration = 10 * time.Millisecond\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB, srvC)\n\n\trt := int32(0)\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port),\n\t\tnats.ReconnectWait(15*time.Millisecond),\n\t\tnats.ReconnectHandler(func(*nats.Conn) {\n\t\t\tatomic.AddInt32(&rt, 1)\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tgo srvA.lameDuckMode()\n\t// Wait a bit, but less than lameDuckModeInitialDelay that we set in this\n\t// test to 100ms.\n\ttime.Sleep(30 * time.Millisecond)\n\tgo srvB.lameDuckMode()\n\n\tsrvA.grWG.Wait()\n\tsrvB.grWG.Wait()\n\tcheckClientsCount(t, srvC, 1)\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tif n := atomic.LoadInt32(&rt); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected client to reconnect only once, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLameDuckModeInfo(t *testing.T) {\n\toptsA := testWSOptions()\n\toptsA.Cluster.Name = \"abc\"\n\toptsA.Cluster.Host = \"127.0.0.1\"\n\toptsA.Cluster.Port = -1\n\t// Ensure that initial delay is set very high so that we can\n\t// check that some events occur as expected before the client\n\t// is disconnected.\n\ttestSetLDMGracePeriod(optsA, 5*time.Second)\n\toptsA.LameDuckDuration = 50 * time.Millisecond\n\toptsA.DisableShortFirstPing = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tcurla := fmt.Sprintf(\"127.0.0.1:%d\", optsA.Port)\n\twscurla := fmt.Sprintf(\"127.0.0.1:%d\", optsA.Websocket.Port)\n\tc, err := net.Dial(\"tcp\", curla)\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer c.Close()\n\tclient := bufio.NewReaderSize(c, maxBufSize)\n\n\twsconn, wsclient := testWSCreateClient(t, false, false, optsA.Websocket.Host, optsA.Websocket.Port)\n\tdefer wsconn.Close()\n\n\tgetInfo := func(ws bool) *serverInfo {\n\t\tt.Helper()\n\t\tvar l string\n\t\tvar err error\n\t\tif ws {\n\t\t\tl = string(testWSReadFrame(t, wsclient))\n\t\t} else {\n\t\t\tl, err = client.ReadString('\\n')\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error receiving info from server: %v\\n\", err)\n\t\t\t}\n\t\t}\n\t\tvar info serverInfo\n\t\tif err = json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\t\tt.Fatalf(\"Could not parse INFO json: %v\\n\", err)\n\t\t}\n\t\treturn &info\n\t}\n\n\tgetInfo(false)\n\tc.Write([]byte(\"CONNECT {\\\"protocol\\\":1,\\\"verbose\\\":false}\\r\\nPING\\r\\n\"))\n\t// Consume both the first PONG and INFO in response to the Connect.\n\tclient.ReadString('\\n')\n\tclient.ReadString('\\n')\n\n\toptsB := testWSOptions()\n\toptsB.Cluster.Name = \"abc\"\n\toptsB.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tcheckConnectURLs := func(expected [][]string) *serverInfo {\n\t\tt.Helper()\n\t\tvar si *serverInfo\n\t\tfor i, ws := range []bool{false, true} {\n\t\t\tslices.Sort(expected[i])\n\t\t\tsi = getInfo(ws)\n\t\t\tslices.Sort(si.ConnectURLs)\n\t\t\tif !reflect.DeepEqual(expected[i], si.ConnectURLs) {\n\t\t\t\tt.Fatalf(\"Expected %q, got %q\", expected, si.ConnectURLs)\n\t\t\t}\n\t\t}\n\t\treturn si\n\t}\n\n\tcurlb := fmt.Sprintf(\"127.0.0.1:%d\", optsB.Port)\n\twscurlb := fmt.Sprintf(\"127.0.0.1:%d\", optsB.Websocket.Port)\n\texpected := [][]string{{curla, curlb}, {wscurla, wscurlb}}\n\tcheckConnectURLs(expected)\n\n\toptsC := testWSOptions()\n\ttestSetLDMGracePeriod(optsA, 5*time.Second)\n\toptsC.Cluster.Name = \"abc\"\n\toptsC.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB, srvC)\n\n\tcurlc := fmt.Sprintf(\"127.0.0.1:%d\", optsC.Port)\n\twscurlc := fmt.Sprintf(\"127.0.0.1:%d\", optsC.Websocket.Port)\n\texpected = [][]string{{curla, curlb, curlc}, {wscurla, wscurlb, wscurlc}}\n\tcheckConnectURLs(expected)\n\n\toptsD := testWSOptions()\n\toptsD.Cluster.Name = \"abc\"\n\toptsD.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", srvA.ClusterAddr().Port))\n\tsrvD := RunServer(optsD)\n\tdefer srvD.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB, srvC, srvD)\n\n\tcurld := fmt.Sprintf(\"127.0.0.1:%d\", optsD.Port)\n\twscurld := fmt.Sprintf(\"127.0.0.1:%d\", optsD.Websocket.Port)\n\texpected = [][]string{{curla, curlb, curlc, curld}, {wscurla, wscurlb, wscurlc, wscurld}}\n\tcheckConnectURLs(expected)\n\n\t// Now lame duck server A and C. We should have client connected to A\n\t// receive info that A is in LDM without A's URL, but also receive\n\t// an update with C's URL gone.\n\t// But first we need to create a client to C because otherwise the\n\t// LDM signal will just shut it down because it would have no client.\n\tnc, err := nats.Connect(srvC.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnc.Flush()\n\n\tstart := time.Now()\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tsrvA.lameDuckMode()\n\t}()\n\n\texpected = [][]string{{curlb, curlc, curld}, {wscurlb, wscurlc, wscurld}}\n\tsi := checkConnectURLs(expected)\n\tif !si.LameDuckMode {\n\t\tt.Fatal(\"Expected LameDuckMode to be true, it was not\")\n\t}\n\n\t// Start LDM for server C. This should send an update to A\n\t// which in turn should remove C from the list of URLs and\n\t// update its client.\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tsrvC.lameDuckMode()\n\t}()\n\n\texpected = [][]string{{curlb, curld}, {wscurlb, wscurld}}\n\tsi = checkConnectURLs(expected)\n\t// This update should not say that it is LDM.\n\tif si.LameDuckMode {\n\t\tt.Fatal(\"Expected LameDuckMode to be false, it was true\")\n\t}\n\n\t// Now shutdown D, and we also should get an update.\n\tsrvD.Shutdown()\n\n\texpected = [][]string{{curlb}, {wscurlb}}\n\tsi = checkConnectURLs(expected)\n\t// This update should not say that it is LDM.\n\tif si.LameDuckMode {\n\t\tt.Fatal(\"Expected LameDuckMode to be false, it was true\")\n\t}\n\tif time.Since(start) > 2*time.Second {\n\t\tt.Fatalf(\"Did not get the expected events prior of server A and C shutting down\")\n\t}\n\n\t// Now explicitly shutdown srvA. When a server shutdown, it closes all its\n\t// connections. For routes, it means that it is going to remove the remote's\n\t// URL from its map. We want to make sure that in that case, server does not\n\t// actually send an updated INFO to its clients.\n\tsrvA.Shutdown()\n\n\t// Expect nothing to be received on the client connection.\n\tif l, err := client.ReadString('\\n'); err == nil {\n\t\tt.Fatalf(\"Expected connection to fail, instead got %q\", l)\n\t}\n\n\tc.Close()\n\tnc.Close()\n\t// Don't need to wait for actual disconnect of clients.\n\tsrvC.Shutdown()\n\twg.Wait()\n}\n\nfunc TestServerValidateGatewaysOptions(t *testing.T) {\n\tbaseOpt := testDefaultOptionsForGateway(\"A\")\n\tu, _ := url.Parse(\"host:5222\")\n\tg := &RemoteGatewayOpts{\n\t\tURLs: []*url.URL{u},\n\t}\n\tbaseOpt.Gateway.Gateways = append(baseOpt.Gateway.Gateways, g)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\texpectedErr string\n\t}{\n\t\t{\n\t\t\tname: \"gateway_has_no_name\",\n\t\t\topts: func() *Options {\n\t\t\t\to := baseOpt.Clone()\n\t\t\t\to.Gateway.Name = \"\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\texpectedErr: \"has no name\",\n\t\t},\n\t\t{\n\t\t\tname: \"gateway_has_no_port\",\n\t\t\topts: func() *Options {\n\t\t\t\to := baseOpt.Clone()\n\t\t\t\to.Gateway.Port = 0\n\t\t\t\treturn o\n\t\t\t},\n\t\t\texpectedErr: \"no port specified\",\n\t\t},\n\t\t{\n\t\t\tname: \"gateway_dst_has_no_name\",\n\t\t\topts: func() *Options {\n\t\t\t\to := baseOpt.Clone()\n\t\t\t\treturn o\n\t\t\t},\n\t\t\texpectedErr: \"has no name\",\n\t\t},\n\t\t{\n\t\t\tname: \"gateway_dst_urls_is_nil\",\n\t\t\topts: func() *Options {\n\t\t\t\to := baseOpt.Clone()\n\t\t\t\to.Gateway.Gateways[0].Name = \"B\"\n\t\t\t\to.Gateway.Gateways[0].URLs = nil\n\t\t\t\treturn o\n\t\t\t},\n\t\t\texpectedErr: \"has no URL\",\n\t\t},\n\t\t{\n\t\t\tname: \"gateway_dst_urls_is_empty\",\n\t\t\topts: func() *Options {\n\t\t\t\to := baseOpt.Clone()\n\t\t\t\to.Gateway.Gateways[0].Name = \"B\"\n\t\t\t\to.Gateway.Gateways[0].URLs = []*url.URL{}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\texpectedErr: \"has no URL\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif err := validateOptions(test.opts()); err == nil || !strings.Contains(err.Error(), test.expectedErr) {\n\t\t\t\tt.Fatalf(\"Expected error about %q, got %v\", test.expectedErr, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAcceptError(t *testing.T) {\n\to := DefaultOptions()\n\ts := New(o)\n\ts.running.Store(true)\n\tdefer s.Shutdown()\n\torgDelay := time.Hour\n\tdelay := s.acceptError(\"Test\", fmt.Errorf(\"any error\"), orgDelay)\n\tif delay != orgDelay {\n\t\tt.Fatalf(\"With this type of error, delay should have stayed same, got %v\", delay)\n\t}\n\n\t// Create any net.Error and make it a temporary\n\tne := &net.DNSError{IsTemporary: true}\n\torgDelay = 10 * time.Millisecond\n\tdelay = s.acceptError(\"Test\", ne, orgDelay)\n\tif delay != 2*orgDelay {\n\t\tt.Fatalf(\"Expected delay to double, got %v\", delay)\n\t}\n\t// Now check the max\n\torgDelay = 60 * ACCEPT_MAX_SLEEP / 100\n\tdelay = s.acceptError(\"Test\", ne, orgDelay)\n\tif delay != ACCEPT_MAX_SLEEP {\n\t\tt.Fatalf(\"Expected delay to double, got %v\", delay)\n\t}\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tstart := time.Now()\n\tgo func() {\n\t\ts.acceptError(\"Test\", ne, orgDelay)\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\t// This should kick out the sleep in acceptError\n\ts.Shutdown()\n\tif dur := time.Since(start); dur >= ACCEPT_MAX_SLEEP {\n\t\tt.Fatalf(\"Shutdown took too long: %v\", dur)\n\t}\n\twg.Wait()\n\tif d := s.acceptError(\"Test\", ne, orgDelay); d >= 0 {\n\t\tt.Fatalf(\"Expected delay to be negative, got %v\", d)\n\t}\n}\n\nfunc TestServerShutdownDuringStart(t *testing.T) {\n\to := DefaultOptions()\n\to.ServerName = \"server\"\n\to.DisableShortFirstPing = true\n\to.Accounts = []*Account{NewAccount(\"$SYS\")}\n\to.SystemAccount = \"$SYS\"\n\to.Cluster.Name = \"abc\"\n\to.Cluster.Host = \"127.0.0.1\"\n\to.Cluster.Port = -1\n\to.Gateway.Name = \"abc\"\n\to.Gateway.Host = \"127.0.0.1\"\n\to.Gateway.Port = -1\n\to.LeafNode.Host = \"127.0.0.1\"\n\to.LeafNode.Port = -1\n\to.Websocket.Host = \"127.0.0.1\"\n\to.Websocket.Port = -1\n\to.Websocket.HandshakeTimeout = 1\n\to.Websocket.NoTLS = true\n\to.MQTT.Host = \"127.0.0.1\"\n\to.MQTT.Port = -1\n\n\t// We are going to test that if the server is shutdown\n\t// while Start() runs (in this case, before), we don't\n\t// start the listeners and therefore leave accept loops\n\t// hanging.\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\ts.Shutdown()\n\n\t// Start() should not block, but just in case, start in\n\t// different go routine.\n\tch := make(chan struct{}, 1)\n\tgo func() {\n\t\ts.Start()\n\t\tclose(ch)\n\t}()\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Start appear to have blocked after server was shutdown\")\n\t}\n\t// Now make sure that none of the listeners have been created\n\tlisteners := []string{}\n\ts.mu.Lock()\n\tif s.listener != nil {\n\t\tlisteners = append(listeners, \"client\")\n\t}\n\tif s.routeListener != nil {\n\t\tlisteners = append(listeners, \"route\")\n\t}\n\tif s.gatewayListener != nil {\n\t\tlisteners = append(listeners, \"gateway\")\n\t}\n\tif s.leafNodeListener != nil {\n\t\tlisteners = append(listeners, \"leafnode\")\n\t}\n\tif s.websocket.listener != nil {\n\t\tlisteners = append(listeners, \"websocket\")\n\t}\n\tif s.mqtt.listener != nil {\n\t\tlisteners = append(listeners, \"mqtt\")\n\t}\n\ts.mu.Unlock()\n\tif len(listeners) > 0 {\n\t\tlst := \"\"\n\t\tfor i, l := range listeners {\n\t\t\tif i > 0 {\n\t\t\t\tlst += \", \"\n\t\t\t}\n\t\t\tlst += l\n\t\t}\n\t\tt.Fatalf(\"Following listeners have been created: %s\", lst)\n\t}\n}\n\ntype myDummyDNSResolver struct {\n\tips []string\n\terr error\n}\n\nfunc (r *myDummyDNSResolver) LookupHost(ctx context.Context, host string) ([]string, error) {\n\tif r.err != nil {\n\t\treturn nil, r.err\n\t}\n\treturn r.ips, nil\n}\n\nfunc TestGetRandomIP(t *testing.T) {\n\ts := &Server{}\n\tresolver := &myDummyDNSResolver{}\n\t// no port...\n\tif _, err := s.getRandomIP(resolver, \"noport\", nil); err == nil || !strings.Contains(err.Error(), \"port\") {\n\t\tt.Fatalf(\"Expected error about port missing, got %v\", err)\n\t}\n\tresolver.err = fmt.Errorf(\"on purpose\")\n\tif _, err := s.getRandomIP(resolver, \"localhost:4222\", nil); err == nil || !strings.Contains(err.Error(), \"on purpose\") {\n\t\tt.Fatalf(\"Expected error about no port, got %v\", err)\n\t}\n\tresolver.err = nil\n\ta, err := s.getRandomIP(resolver, \"localhost:4222\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif a != \"localhost:4222\" {\n\t\tt.Fatalf(\"Expected address to be %q, got %q\", \"localhost:4222\", a)\n\t}\n\tresolver.ips = []string{\"1.2.3.4\"}\n\ta, err = s.getRandomIP(resolver, \"localhost:4222\", nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif a != \"1.2.3.4:4222\" {\n\t\tt.Fatalf(\"Expected address to be %q, got %q\", \"1.2.3.4:4222\", a)\n\t}\n\t// Check for randomness\n\tresolver.ips = []string{\"1.2.3.4\", \"2.2.3.4\", \"3.2.3.4\"}\n\tdist := [3]int{}\n\tfor i := 0; i < 100; i++ {\n\t\tip, err := s.getRandomIP(resolver, \"localhost:4222\", nil)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tv := int(ip[0]-'0') - 1\n\t\tdist[v]++\n\t}\n\tlow := 20\n\thigh := 47\n\tfor i, d := range dist {\n\t\tif d == 0 || d == 100 {\n\t\t\tt.Fatalf(\"Unexpected distribution for ip %v, got %v\", i, d)\n\t\t} else if d < low || d > high {\n\t\t\tt.Logf(\"Warning: out of expected range [%v,%v] for ip %v, got %v\", low, high, i, d)\n\t\t}\n\t}\n\n\t// Check IP exclusions\n\texcludedIPs := map[string]struct{}{\"1.2.3.4:4222\": {}}\n\tfor i := 0; i < 100; i++ {\n\t\tip, err := s.getRandomIP(resolver, \"localhost:4222\", excludedIPs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ip[0] == '1' {\n\t\t\tt.Fatalf(\"Should not have returned this ip: %q\", ip)\n\t\t}\n\t}\n\texcludedIPs[\"2.2.3.4:4222\"] = struct{}{}\n\tfor i := 0; i < 100; i++ {\n\t\tip, err := s.getRandomIP(resolver, \"localhost:4222\", excludedIPs)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif ip[0] != '3' {\n\t\t\tt.Fatalf(\"Should only have returned '3.2.3.4', got returned %q\", ip)\n\t\t}\n\t}\n\texcludedIPs[\"3.2.3.4:4222\"] = struct{}{}\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := s.getRandomIP(resolver, \"localhost:4222\", excludedIPs); err != errNoIPAvail {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\t// Now check that exclusion takes into account the port number.\n\tresolver.ips = []string{\"127.0.0.1\"}\n\texcludedIPs = map[string]struct{}{\"127.0.0.1:4222\": {}}\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err := s.getRandomIP(resolver, \"localhost:4223\", excludedIPs); err == errNoIPAvail {\n\t\t\tt.Fatal(\"Should not have failed\")\n\t\t}\n\t}\n}\n\ntype shortWriteConn struct {\n\tnet.Conn\n}\n\nfunc (swc *shortWriteConn) Write(b []byte) (int, error) {\n\t// Limit the write to 10 bytes at a time.\n\tshort := false\n\tmax := len(b)\n\tif max > 10 {\n\t\tmax = 10\n\t\tshort = true\n\t}\n\tn, err := swc.Conn.Write(b[:max])\n\tif err == nil && short {\n\t\treturn n, io.ErrShortWrite\n\t}\n\treturn n, err\n}\n\nfunc TestClientWriteLoopStall(t *testing.T) {\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\terrCh := make(chan error, 1)\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url,\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, e error) {\n\t\t\tselect {\n\t\t\tcase errCh <- e:\n\t\t\tdefault:\n\t\t\t}\n\t\t}))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tnc.Flush()\n\tcid, _ := nc.GetClientID()\n\n\tsender, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer sender.Close()\n\n\tc := s.getClient(cid)\n\tc.mu.Lock()\n\tc.nc = &shortWriteConn{Conn: c.nc}\n\tc.mu.Unlock()\n\n\tsender.Publish(\"foo\", make([]byte, 100))\n\n\tif _, err := sub.NextMsg(3 * time.Second); err != nil {\n\t\tt.Fatalf(\"WriteLoop has stalled!\")\n\t}\n\n\t// Make sure that we did not get any async error\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatalf(\"Got error: %v\", e)\n\tcase <-time.After(250 * time.Millisecond):\n\t}\n}\n\nfunc TestInsecureSkipVerifyWarning(t *testing.T) {\n\tcheckWarnReported := func(t *testing.T, o *Options, expectedWarn string) {\n\t\tt.Helper()\n\t\ts, err := NewServer(o)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on new server: %v\", err)\n\t\t}\n\t\tl := &captureWarnLogger{warn: make(chan string, 1)}\n\t\ts.SetLogger(l, false, false)\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\ts.Start()\n\t\t\twg.Done()\n\t\t}()\n\t\tif err := s.readyForConnections(time.Second); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tselect {\n\t\tcase w := <-l.warn:\n\t\t\tif !strings.Contains(w, expectedWarn) {\n\t\t\t\tt.Fatalf(\"Expected warning %q, got %q\", expectedWarn, w)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Did not get warning %q\", expectedWarn)\n\t\t}\n\t\ts.Shutdown()\n\t\twg.Wait()\n\t}\n\n\ttc := &TLSConfigOpts{}\n\ttc.CertFile = \"../test/configs/certs/server-cert.pem\"\n\ttc.KeyFile = \"../test/configs/certs/server-key.pem\"\n\ttc.CaFile = \"../test/configs/certs/ca.pem\"\n\ttc.Insecure = true\n\tconfig, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t}\n\n\to := DefaultOptions()\n\to.Cluster.Name = \"A\"\n\to.Cluster.Port = -1\n\to.Cluster.TLSConfig = config.Clone()\n\tcheckWarnReported(t, o, clusterTLSInsecureWarning)\n\n\t// Remove the route setting\n\to.Cluster.Port = 0\n\to.Cluster.TLSConfig = nil\n\n\t// Configure LeafNode with no TLS in the main block first, but only with remotes.\n\to.LeafNode.Port = -1\n\trurl, _ := url.Parse(\"nats://127.0.0.1:1234\")\n\to.LeafNode.Remotes = []*RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{rurl},\n\t\t\tTLSConfig: config.Clone(),\n\t\t},\n\t}\n\tcheckWarnReported(t, o, leafnodeTLSInsecureWarning)\n\n\t// Now add to main block.\n\to.LeafNode.TLSConfig = config.Clone()\n\tcheckWarnReported(t, o, leafnodeTLSInsecureWarning)\n\n\t// Now remove remote and check warning still reported\n\to.LeafNode.Remotes = nil\n\tcheckWarnReported(t, o, leafnodeTLSInsecureWarning)\n\n\t// Remove the LN setting\n\to.LeafNode.Port = 0\n\to.LeafNode.TLSConfig = nil\n\n\t// Configure GW with no TLS in main block first, but only with remotes\n\to.Gateway.Name = \"A\"\n\to.Gateway.Host = \"127.0.0.1\"\n\to.Gateway.Port = -1\n\to.Gateway.Gateways = []*RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tURLs: []*url.URL{rurl},\n\t\t\tTLSConfig: config.Clone(),\n\t\t},\n\t}\n\tcheckWarnReported(t, o, gatewayTLSInsecureWarning)\n\n\t// Now add to main block.\n\to.Gateway.TLSConfig = config.Clone()\n\tcheckWarnReported(t, o, gatewayTLSInsecureWarning)\n\n\t// Now remove remote and check warning still reported\n\to.Gateway.Gateways = nil\n\tcheckWarnReported(t, o, gatewayTLSInsecureWarning)\n}\n\nfunc TestConnectErrorReports(t *testing.T) {\n\t// On Windows, an attempt to connect to a port that has no listener will\n\t// take whatever timeout specified in DialTimeout() before failing.\n\t// So skip for now.\n\tif runtime.GOOS == \"windows\" {\n\t\tt.Skip()\n\t}\n\t// Check that default report attempts is as expected\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tif ra := s.getOpts().ConnectErrorReports; ra != DEFAULT_CONNECT_ERROR_REPORTS {\n\t\tt.Fatalf(\"Expected default value to be %v, got %v\", DEFAULT_CONNECT_ERROR_REPORTS, ra)\n\t}\n\n\ttmpFile := createTempFile(t, \"\")\n\tlog := tmpFile.Name()\n\ttmpFile.Close()\n\n\tremoteURLs := RoutesFromStr(\"nats://127.0.0.1:1234\")\n\n\topts = DefaultOptions()\n\topts.ConnectErrorReports = 3\n\topts.Cluster.Port = -1\n\topts.Routes = remoteURLs\n\topts.NoLog = false\n\topts.LogFile = log\n\topts.Logtime = true\n\topts.Debug = true\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tcontent, err := os.ReadFile(log)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Error reading log file: %v\", err)\n\t\t\t}\n\t\t\tpresent := bytes.Contains(content, []byte(fmt.Sprintf(\"%s (attempt %d)\", txt, attempt)))\n\t\t\tif shouldBeThere && !present {\n\t\t\t\treturn fmt.Errorf(\"Did not find expected log statement (%s) for attempt %d: %s\", txt, attempt, content)\n\t\t\t} else if !shouldBeThere && present {\n\t\t\t\treturn fmt.Errorf(\"Log statement (%s) for attempt %d should not be present: %s\", txt, attempt, content)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\ttype testConnect struct {\n\t\tname string\n\t\tattempt int\n\t\terrExpected bool\n\t}\n\tfor _, test := range []testConnect{\n\t\t{\"route_attempt_1\", 1, true},\n\t\t{\"route_attempt_2\", 2, false},\n\t\t{\"route_attempt_3\", 3, true},\n\t\t{\"route_attempt_4\", 4, false},\n\t\t{\"route_attempt_6\", 6, true},\n\t\t{\"route_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tcheckContent(t, \"[DBG] Error trying to connect to route\", test.attempt, debugExpected)\n\t\t\tcheckContent(t, \"[ERR] Error trying to connect to route\", test.attempt, test.errExpected)\n\t\t})\n\t}\n\n\ts.Shutdown()\n\tremoveFile(t, log)\n\n\t// Now try with leaf nodes\n\topts.Cluster.Port = 0\n\topts.Routes = nil\n\topts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{remoteURLs[0]}}}\n\topts.LeafNode.ReconnectInterval = 15 * time.Millisecond\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tcontent, err := os.ReadFile(log)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Error reading log file: %v\", err)\n\t\t\t}\n\t\t\tpresent := bytes.Contains(content, []byte(fmt.Sprintf(\"%s %q (attempt %d)\", txt, host, attempt)))\n\t\t\tif shouldBeThere && !present {\n\t\t\t\treturn fmt.Errorf(\"Did not find expected log statement (%s %q) for attempt %d: %s\", txt, host, attempt, content)\n\t\t\t} else if !shouldBeThere && present {\n\t\t\t\treturn fmt.Errorf(\"Log statement (%s %q) for attempt %d should not be present: %s\", txt, host, attempt, content)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tfor _, test := range []testConnect{\n\t\t{\"leafnode_attempt_1\", 1, true},\n\t\t{\"leafnode_attempt_2\", 2, false},\n\t\t{\"leafnode_attempt_3\", 3, true},\n\t\t{\"leafnode_attempt_4\", 4, false},\n\t\t{\"leafnode_attempt_6\", 6, true},\n\t\t{\"leafnode_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tcheckLeafContent(t, \"[DBG] Error trying to connect as leafnode to remote server\", remoteURLs[0].Host, test.attempt, debugExpected)\n\t\t\tcheckLeafContent(t, \"[ERR] Error trying to connect as leafnode to remote server\", remoteURLs[0].Host, test.attempt, test.errExpected)\n\t\t})\n\t}\n\n\ts.Shutdown()\n\tremoveFile(t, log)\n\n\t// Now try with gateways\n\topts.LeafNode.Remotes = nil\n\topts.Cluster.Name = \"A\"\n\topts.Gateway.Name = \"A\"\n\topts.Gateway.Port = -1\n\topts.Gateway.Gateways = []*RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tURLs: remoteURLs,\n\t\t},\n\t}\n\topts.gatewaysSolicitDelay = 15 * time.Millisecond\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []testConnect{\n\t\t{\"gateway_attempt_1\", 1, true},\n\t\t{\"gateway_attempt_2\", 2, false},\n\t\t{\"gateway_attempt_3\", 3, true},\n\t\t{\"gateway_attempt_4\", 4, false},\n\t\t{\"gateway_attempt_6\", 6, true},\n\t\t{\"gateway_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tinfoExpected := test.errExpected\n\t\t\t// For gateways, we also check our notice that we attempt to connect\n\t\t\tcheckContent(t, \"[DBG] Connecting to explicit gateway \\\"B\\\" (127.0.0.1:1234) at 127.0.0.1:1234\", test.attempt, debugExpected)\n\t\t\tcheckContent(t, \"[INF] Connecting to explicit gateway \\\"B\\\" (127.0.0.1:1234) at 127.0.0.1:1234\", test.attempt, infoExpected)\n\t\t\tcheckContent(t, \"[DBG] Error connecting to explicit gateway \\\"B\\\" (127.0.0.1:1234) at 127.0.0.1:1234\", test.attempt, debugExpected)\n\t\t\tcheckContent(t, \"[ERR] Error connecting to explicit gateway \\\"B\\\" (127.0.0.1:1234) at 127.0.0.1:1234\", test.attempt, test.errExpected)\n\t\t})\n\t}\n}\n\nfunc TestReconnectErrorReports(t *testing.T) {\n\t// On Windows, an attempt to connect to a port that has no listener will\n\t// take whatever timeout specified in DialTimeout() before failing.\n\t// So skip for now.\n\tif runtime.GOOS == \"windows\" {\n\t\tt.Skip()\n\t}\n\t// Check that default report attempts is as expected\n\topts := DefaultOptions()\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tif ra := s.getOpts().ReconnectErrorReports; ra != DEFAULT_RECONNECT_ERROR_REPORTS {\n\t\tt.Fatalf(\"Expected default value to be %v, got %v\", DEFAULT_RECONNECT_ERROR_REPORTS, ra)\n\t}\n\n\ttmpFile := createTempFile(t, \"\")\n\tlog := tmpFile.Name()\n\ttmpFile.Close()\n\n\tcsOpts := DefaultOptions()\n\tcsOpts.Cluster.Port = -1\n\tcs := RunServer(csOpts)\n\tdefer cs.Shutdown()\n\n\topts = DefaultOptions()\n\topts.ReconnectErrorReports = 3\n\topts.Cluster.Port = -1\n\topts.Routes = RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", cs.ClusterAddr().Port))\n\topts.NoLog = false\n\topts.LogFile = log\n\topts.Logtime = true\n\topts.Debug = true\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Wait for cluster to be formed\n\tcheckClusterFormed(t, s, cs)\n\n\t// Now shutdown the server s connected to.\n\tcs.Shutdown()\n\n\t// Specifically for route test, wait at least reconnect interval before checking logs\n\ttime.Sleep(DEFAULT_ROUTE_RECONNECT)\n\n\tcheckContent := func(t *testing.T, txt string, attempt int, shouldBeThere bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tcontent, err := os.ReadFile(log)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Error reading log file: %v\", err)\n\t\t\t}\n\t\t\tpresent := bytes.Contains(content, []byte(fmt.Sprintf(\"%s (attempt %d)\", txt, attempt)))\n\t\t\tif shouldBeThere && !present {\n\t\t\t\treturn fmt.Errorf(\"Did not find expected log statement (%s) for attempt %d: %s\", txt, attempt, content)\n\t\t\t} else if !shouldBeThere && present {\n\t\t\t\treturn fmt.Errorf(\"Log statement (%s) for attempt %d should not be present: %s\", txt, attempt, content)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\ttype testConnect struct {\n\t\tname string\n\t\tattempt int\n\t\terrExpected bool\n\t}\n\tfor _, test := range []testConnect{\n\t\t{\"route_attempt_1\", 1, true},\n\t\t{\"route_attempt_2\", 2, false},\n\t\t{\"route_attempt_3\", 3, true},\n\t\t{\"route_attempt_4\", 4, false},\n\t\t{\"route_attempt_6\", 6, true},\n\t\t{\"route_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tcheckContent(t, \"[DBG] Error trying to connect to route\", test.attempt, debugExpected)\n\t\t\tcheckContent(t, \"[ERR] Error trying to connect to route\", test.attempt, test.errExpected)\n\t\t})\n\t}\n\n\ts.Shutdown()\n\tremoveFile(t, log)\n\n\t// Now try with leaf nodes\n\tcsOpts.Cluster.Port = 0\n\tcsOpts.Cluster.Name = _EMPTY_\n\tcsOpts.LeafNode.Host = \"127.0.0.1\"\n\tcsOpts.LeafNode.Port = -1\n\n\tcs = RunServer(csOpts)\n\tdefer cs.Shutdown()\n\n\topts.Cluster.Port = 0\n\topts.Cluster.Name = _EMPTY_\n\topts.Routes = nil\n\tu, _ := url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", csOpts.LeafNode.Port))\n\topts.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{u}}}\n\topts.LeafNode.ReconnectInterval = 15 * time.Millisecond\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Now shutdown the server s is connected to\n\tcs.Shutdown()\n\n\tcheckLeafContent := func(t *testing.T, txt, host string, attempt int, shouldBeThere bool) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\tcontent, err := os.ReadFile(log)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Error reading log file: %v\", err)\n\t\t\t}\n\t\t\tpresent := bytes.Contains(content, []byte(fmt.Sprintf(\"%s %q (attempt %d)\", txt, host, attempt)))\n\t\t\tif shouldBeThere && !present {\n\t\t\t\treturn fmt.Errorf(\"Did not find expected log statement (%s %q) for attempt %d: %s\", txt, host, attempt, content)\n\t\t\t} else if !shouldBeThere && present {\n\t\t\t\treturn fmt.Errorf(\"Log statement (%s %q) for attempt %d should not be present: %s\", txt, host, attempt, content)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tfor _, test := range []testConnect{\n\t\t{\"leafnode_attempt_1\", 1, true},\n\t\t{\"leafnode_attempt_2\", 2, false},\n\t\t{\"leafnode_attempt_3\", 3, true},\n\t\t{\"leafnode_attempt_4\", 4, false},\n\t\t{\"leafnode_attempt_6\", 6, true},\n\t\t{\"leafnode_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tcheckLeafContent(t, \"[DBG] Error trying to connect as leafnode to remote server\", u.Host, test.attempt, debugExpected)\n\t\t\tcheckLeafContent(t, \"[ERR] Error trying to connect as leafnode to remote server\", u.Host, test.attempt, test.errExpected)\n\t\t})\n\t}\n\n\ts.Shutdown()\n\tremoveFile(t, log)\n\n\t// Now try with gateways\n\tcsOpts.LeafNode.Port = 0\n\tcsOpts.Cluster.Name = \"B\"\n\tcsOpts.Gateway.Name = \"B\"\n\tcsOpts.Gateway.Port = -1\n\tcs = RunServer(csOpts)\n\n\topts.LeafNode.Remotes = nil\n\topts.Cluster.Name = \"A\"\n\topts.Gateway.Name = \"A\"\n\topts.Gateway.Port = -1\n\tremoteGWPort := cs.GatewayAddr().Port\n\tu, _ = url.Parse(fmt.Sprintf(\"nats://127.0.0.1:%d\", remoteGWPort))\n\topts.Gateway.Gateways = []*RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tURLs: []*url.URL{u},\n\t\t},\n\t}\n\topts.gatewaysSolicitDelay = 15 * time.Millisecond\n\ts = RunServer(opts)\n\tdefer s.Shutdown()\n\n\twaitForOutboundGateways(t, s, 1, 2*time.Second)\n\twaitForInboundGateways(t, s, 1, 2*time.Second)\n\n\t// Now stop server s is connecting to\n\tcs.Shutdown()\n\n\tconnTxt := fmt.Sprintf(\"Connecting to explicit gateway \\\"B\\\" (127.0.0.1:%d) at 127.0.0.1:%d\", remoteGWPort, remoteGWPort)\n\tdbgConnTxt := fmt.Sprintf(\"[DBG] %s\", connTxt)\n\tinfConnTxt := fmt.Sprintf(\"[INF] %s\", connTxt)\n\n\terrTxt := fmt.Sprintf(\"Error connecting to explicit gateway \\\"B\\\" (127.0.0.1:%d) at 127.0.0.1:%d\", remoteGWPort, remoteGWPort)\n\tdbgErrTxt := fmt.Sprintf(\"[DBG] %s\", errTxt)\n\terrErrTxt := fmt.Sprintf(\"[ERR] %s\", errTxt)\n\n\tfor _, test := range []testConnect{\n\t\t{\"gateway_attempt_1\", 1, true},\n\t\t{\"gateway_attempt_2\", 2, false},\n\t\t{\"gateway_attempt_3\", 3, true},\n\t\t{\"gateway_attempt_4\", 4, false},\n\t\t{\"gateway_attempt_6\", 6, true},\n\t\t{\"gateway_attempt_7\", 7, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdebugExpected := !test.errExpected\n\t\t\tinfoExpected := test.errExpected\n\t\t\t// For gateways, we also check our notice that we attempt to connect\n\t\t\tcheckContent(t, dbgConnTxt, test.attempt, debugExpected)\n\t\t\tcheckContent(t, infConnTxt, test.attempt, infoExpected)\n\t\t\tcheckContent(t, dbgErrTxt, test.attempt, debugExpected)\n\t\t\tcheckContent(t, errErrTxt, test.attempt, test.errExpected)\n\t\t})\n\t}\n}\n\nfunc TestServerLogsConfigurationFile(t *testing.T) {\n\tfile := createTempFile(t, \"nats_server_log_\")\n\tfile.Close()\n\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\tport: -1\n\tlogfile: '%s'\n\t`, file.Name())))\n\n\to := LoadConfig(conf)\n\to.ConfigFile = file.Name()\n\to.NoLog = false\n\ts := RunServer(o)\n\ts.Shutdown()\n\n\tlog, err := os.ReadFile(file.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading log file: %v\", err)\n\t}\n\tif !bytes.Contains(log, []byte(fmt.Sprintf(\"Using configuration file: %s\", file.Name()))) {\n\t\tt.Fatalf(\"Config file location was not reported in log: %s\", log)\n\t}\n}\n\nfunc TestServerRateLimitLogging(t *testing.T) {\n\ts := RunServer(DefaultOptions())\n\tdefer s.Shutdown()\n\n\ts.changeRateLimitLogInterval(100 * time.Millisecond)\n\n\tl := &captureWarnLogger{warn: make(chan string, 100)}\n\ts.SetLogger(l, false, false)\n\n\ts.RateLimitWarnf(\"Warning number 1\")\n\ts.RateLimitWarnf(\"Warning number 2\")\n\ts.rateLimitFormatWarnf(\"warning value %d\", 1)\n\ts.RateLimitWarnf(\"Warning number 1\")\n\ts.RateLimitWarnf(\"Warning number 2\")\n\ts.rateLimitFormatWarnf(\"warning value %d\", 2)\n\n\tcheckLog := func(c1, c2 *client) {\n\t\tt.Helper()\n\n\t\tnb1 := \"Warning number 1\"\n\t\tnb2 := \"Warning number 2\"\n\t\tnbv := \"warning value\"\n\t\tgotOne := 0\n\t\tgotTwo := 0\n\t\tgotFormat := 0\n\t\tfor done := false; !done; {\n\t\t\tselect {\n\t\t\tcase w := <-l.warn:\n\t\t\t\tif strings.Contains(w, nb1) {\n\t\t\t\t\tgotOne++\n\t\t\t\t} else if strings.Contains(w, nb2) {\n\t\t\t\t\tgotTwo++\n\t\t\t\t} else if strings.Contains(w, nbv) {\n\t\t\t\t\tgotFormat++\n\t\t\t\t}\n\t\t\tcase <-time.After(150 * time.Millisecond):\n\t\t\t\tdone = true\n\t\t\t}\n\t\t}\n\t\tif gotOne != 1 {\n\t\t\tt.Fatalf(\"Should have had only 1 warning for nb1, got %v\", gotOne)\n\t\t}\n\t\tif gotTwo != 1 {\n\t\t\tt.Fatalf(\"Should have had only 1 warning for nb2, got %v\", gotTwo)\n\t\t}\n\t\tif gotFormat != 1 {\n\t\t\tt.Fatalf(\"Should have had only 1 warning for format, got %v\", gotFormat)\n\t\t}\n\n\t\t// Wait for more than the expiration interval\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\tif c1 == nil {\n\t\t\ts.RateLimitWarnf(\"%s\", nb1)\n\t\t\ts.rateLimitFormatWarnf(\"warning value %d\", 1)\n\t\t} else {\n\t\t\tc1.RateLimitWarnf(\"%s\", nb1)\n\t\t\tc2.RateLimitWarnf(\"%s\", nb1)\n\t\t\tc1.rateLimitFormatWarnf(\"warning value %d\", 1)\n\t\t}\n\t\tgotOne = 0\n\t\tgotFormat = 0\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase w := <-l.warn:\n\t\t\t\tif strings.Contains(w, nb1) {\n\t\t\t\t\tgotOne++\n\t\t\t\t} else if strings.Contains(w, nbv) {\n\t\t\t\t\tgotFormat++\n\t\t\t\t}\n\t\t\tcase <-time.After(200 * time.Millisecond):\n\t\t\t\tif gotOne == 0 {\n\t\t\t\t\tt.Fatalf(\"Warning was still suppressed\")\n\t\t\t\t} else if gotOne > 1 {\n\t\t\t\t\tt.Fatalf(\"Should have had only 1 warning for nb1, got %v\", gotOne)\n\t\t\t\t} else if gotFormat == 0 {\n\t\t\t\t\tt.Fatalf(\"Warning was still suppressed\")\n\t\t\t\t} else if gotFormat > 1 {\n\t\t\t\t\tt.Fatalf(\"Should have had only 1 warning for format, got %v\", gotFormat)\n\t\t\t\t} else {\n\t\t\t\t\t// OK! we are done\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckLog(nil, nil)\n\n\tnc1 := natsConnect(t, s.ClientURL(), nats.Name(\"c1\"))\n\tdefer nc1.Close()\n\tnc2 := natsConnect(t, s.ClientURL(), nats.Name(\"c2\"))\n\tdefer nc2.Close()\n\n\tvar c1 *client\n\tvar c2 *client\n\ts.mu.Lock()\n\tfor _, cli := range s.clients {\n\t\tcli.mu.Lock()\n\t\tswitch cli.opts.Name {\n\t\tcase \"c1\":\n\t\t\tc1 = cli\n\t\tcase \"c2\":\n\t\t\tc2 = cli\n\t\t}\n\t\tcli.mu.Unlock()\n\t\tif c1 != nil && c2 != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\ts.mu.Unlock()\n\tif c1 == nil || c2 == nil {\n\t\tt.Fatal(\"Did not find the clients\")\n\t}\n\n\t// Wait for more than the expiration interval\n\ttime.Sleep(200 * time.Millisecond)\n\n\tc1.RateLimitWarnf(\"Warning number 1\")\n\tc1.RateLimitWarnf(\"Warning number 2\")\n\tc1.rateLimitFormatWarnf(\"warning value %d\", 1)\n\tc2.RateLimitWarnf(\"Warning number 1\")\n\tc2.RateLimitWarnf(\"Warning number 2\")\n\tc2.rateLimitFormatWarnf(\"warning value %d\", 2)\n\n\tcheckLog(c1, c2)\n}\n\n// https://github.com/nats-io/nats-server/discussions/4535\nfunc TestServerAuthBlockAndSysAccounts(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s-test\n\t\tauthorization {\n\t\t\tusers = [ { user: \"u\", password: \"pass\"} ]\n\t\t}\n\t\taccounts {\n\t\t\t$SYS: { users: [ { user: admin, password: pwd } ] }\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// This should work of course.\n\tnc, err := nats.Connect(s.ClientURL(), nats.UserInfo(\"u\", \"pass\"))\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n\n\t// This should not.\n\t_, err = nats.Connect(s.ClientURL())\n\trequire_Error(t, err, nats.ErrAuthorization, errors.New(\"nats: Authorization Violation\"))\n}\n\n// https://github.com/nats-io/nats-server/issues/5396\nfunc TestServerConfigLastLineComments(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t{\n\t\t\"listen\": \"0.0.0.0:4222\"\n\t}\n\t# wibble\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// This should work of course.\n\tnc, err := nats.Connect(s.ClientURL())\n\trequire_NoError(t, err)\n\tdefer nc.Close()\n}\n\nfunc TestServerClusterAndGatewayNameNoSpace(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"my server\"\n\t`))\n\t_, err := ProcessConfigFile(conf)\n\trequire_Error(t, err, ErrServerNameHasSpaces)\n\n\to := DefaultOptions()\n\to.ServerName = \"my server\"\n\t_, err = NewServer(o)\n\trequire_Error(t, err, ErrServerNameHasSpaces)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"myserver\"\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tname: \"my cluster\"\n\t\t}\n\t`))\n\t_, err = ProcessConfigFile(conf)\n\trequire_Error(t, err, ErrClusterNameHasSpaces)\n\n\to = DefaultOptions()\n\to.Cluster.Name = \"my cluster\"\n\to.Cluster.Port = -1\n\t_, err = NewServer(o)\n\trequire_Error(t, err, ErrClusterNameHasSpaces)\n\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tserver_name: \"myserver\"\n\t\tgateway {\n\t\t\tport: -1\n\t\t\tname: \"my gateway\"\n\t\t}\n\t`))\n\t_, err = ProcessConfigFile(conf)\n\trequire_Error(t, err, ErrGatewayNameHasSpaces)\n\n\to = DefaultOptions()\n\to.Cluster.Name = _EMPTY_\n\to.Cluster.Port = 0\n\to.Gateway.Name = \"my gateway\"\n\to.Gateway.Port = -1\n\t_, err = NewServer(o)\n\trequire_Error(t, err, ErrGatewayNameHasSpaces)\n}\n\nfunc TestServerClientURL(t *testing.T) {\n\tfor host, expected := range map[string]string{\n\t\t\"host.com\": \"nats://host.com:12345\",\n\t\t\"1.2.3.4\": \"nats://1.2.3.4:12345\",\n\t\t\"2000::1\": \"nats://[2000::1]:12345\",\n\t} {\n\t\to := DefaultOptions()\n\t\to.Host = host\n\t\to.Port = 12345\n\t\ts, err := NewServer(o)\n\t\trequire_NoError(t, err)\n\t\trequire_Equal(t, s.ClientURL(), expected)\n\t}\n}\n\n// This is a test that guards against using goccy/go-json.\n// At least until it's fully compatible with std encoding/json, and we've thoroughly tested it.\n// This is just one bug (at the time of writing) that results in a panic.\n// https://github.com/goccy/go-json/issues/519\nfunc TestServerJsonMarshalNestedStructsPanic(t *testing.T) {\n\ttype Item struct {\n\t\tA string `json:\"a\"`\n\t\tB string `json:\"b,omitempty\"`\n\t}\n\n\ttype Detail struct {\n\t\tI Item `json:\"i\"`\n\t}\n\n\ttype Body struct {\n\t\tPayload *Detail `json:\"p,omitempty\"`\n\t}\n\n\tb, err := json.Marshal(Body{Payload: &Detail{I: Item{A: \"a\", B: \"b\"}}})\n\trequire_NoError(t, err)\n\trequire_Equal(t, string(b), \"{\\\"p\\\":{\\\"i\\\":{\\\"a\\\":\\\"a\\\",\\\"b\\\":\\\"b\\\"}}}\")\n}\n\nfunc TestBuildinfoFormatRevision(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\trevision string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"Git-like longer version\",\n\t\t\trevision: \"abc123def456789\",\n\t\t\texpected: \"abc123d\",\n\t\t},\n\t\t{\n\t\t\tname: \"Git-like exactly 7 chars\",\n\t\t\trevision: \"abc123d\",\n\t\t\texpected: \"abc123d\",\n\t\t},\n\t\t{\n\t\t\tname: \"SVN shorter revision\",\n\t\t\trevision: \"1234\",\n\t\t\texpected: \"1234\",\n\t\t},\n\t\t{\n\t\t\tname: \"SVN single digit\",\n\t\t\trevision: \"5\",\n\t\t\texpected: \"5\",\n\t\t},\n\t\t{\n\t\t\tname: \"Empty revision\",\n\t\t\trevision: \"\",\n\t\t\texpected: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"6 character revision\",\n\t\t\trevision: \"abc123\",\n\t\t\texpected: \"abc123\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := formatRevision(tt.revision)\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"formatRevision(%q) = %q, expected %q\", tt.revision, result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "server/service.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage server\n\n// Run starts the NATS server. This wrapper function allows Windows to add a\n// hook for running NATS as a service.\nfunc Run(server *Server) error {\n\tserver.Start()\n\treturn nil\n}\n\n// isWindowsService indicates if NATS is running as a Windows service.\nfunc isWindowsService() bool {\n\treturn false\n}\n" }, { "path": "server/service_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage server\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRun(t *testing.T) {\n\tvar (\n\t\ts = New(DefaultOptions())\n\t\tstarted = make(chan error, 1)\n\t\terrC = make(chan error, 1)\n\t)\n\tgo func() {\n\t\terrC <- Run(s)\n\t}()\n\tgo func() {\n\t\tif err := s.readyForConnections(time.Second); err != nil {\n\t\t\tstarted <- err\n\t\t\treturn\n\t\t}\n\t\ts.Shutdown()\n\t\tclose(started)\n\t}()\n\n\tselect {\n\tcase err := <-errC:\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatal(\"Timed out\")\n\t}\n\tif err := <-started; err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n}\n" }, { "path": "server/service_windows.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"os\"\n\t\"time\"\n\n\t\"golang.org/x/sys/windows/svc\"\n)\n\nconst (\n\treopenLogCode = 128\n\treopenLogCmd = svc.Cmd(reopenLogCode)\n\tldmCode = 129\n\tldmCmd = svc.Cmd(ldmCode)\n\tacceptReopenLog = svc.Accepted(reopenLogCode)\n)\n\nvar serviceName = \"nats-server\"\n\n// SetServiceName allows setting a different service name\nfunc SetServiceName(name string) {\n\tserviceName = name\n}\n\n// winServiceWrapper implements the svc.Handler interface for implementing\n// nats-server as a Windows service.\ntype winServiceWrapper struct {\n\tserver *Server\n}\n\nvar dockerized = false\n\nfunc init() {\n\tif v, exists := os.LookupEnv(\"NATS_DOCKERIZED\"); exists && v == \"1\" {\n\t\tdockerized = true\n\t}\n}\n\n// Execute will be called by the package code at the start of\n// the service, and the service will exit once Execute completes.\n// Inside Execute you must read service change requests from r and\n// act accordingly. You must keep service control manager up to date\n// about state of your service by writing into s as required.\n// args contains service name followed by argument strings passed\n// to the service.\n// You can provide service exit code in exitCode return parameter,\n// with 0 being \"no error\". You can also indicate if exit code,\n// if any, is service specific or not by using svcSpecificEC\n// parameter.\nfunc (w *winServiceWrapper) Execute(args []string, changes <-chan svc.ChangeRequest,\n\tstatus chan<- svc.Status) (bool, uint32) {\n\n\tstatus <- svc.Status{State: svc.StartPending}\n\tgo w.server.Start()\n\n\tvar startupDelay = 10 * time.Second\n\tif v, exists := os.LookupEnv(\"NATS_STARTUP_DELAY\"); exists {\n\t\tif delay, err := time.ParseDuration(v); err == nil {\n\t\t\tstartupDelay = delay\n\t\t} else {\n\t\t\tw.server.Errorf(\"Failed to parse \\\"%v\\\" as a duration for startup: %s\", v, err)\n\t\t}\n\t}\n\t// Wait for accept loop(s) to be started\n\tif !w.server.ReadyForConnections(startupDelay) {\n\t\t// Failed to start.\n\t\treturn false, 1\n\t}\n\n\tstatus <- svc.Status{\n\t\tState: svc.Running,\n\t\tAccepts: svc.AcceptStop | svc.AcceptShutdown | svc.AcceptParamChange | acceptReopenLog,\n\t}\n\nloop:\n\tfor {\n\t\tselect {\n\t\tcase change, ok := <-changes:\n\t\t\tif !ok {\n\t\t\t\tbreak loop\n\t\t\t}\n\t\t\tswitch change.Cmd {\n\t\t\tcase svc.Interrogate:\n\t\t\t\tstatus <- change.CurrentStatus\n\t\t\tcase svc.Stop, svc.Shutdown:\n\t\t\t\tw.server.Shutdown()\n\t\t\t\tbreak loop\n\t\t\tcase reopenLogCmd:\n\t\t\t\t// File log re-open for rotating file logs.\n\t\t\t\tw.server.ReOpenLogFile()\n\t\t\tcase ldmCmd:\n\t\t\t\tgo w.server.lameDuckMode()\n\t\t\tcase svc.ParamChange:\n\t\t\t\tif err := w.server.Reload(); err != nil {\n\t\t\t\t\tw.server.Errorf(\"Failed to reload server configuration: %s\", err)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tw.server.Debugf(\"Unexpected control request: %v\", change.Cmd)\n\t\t\t}\n\t\tcase <-w.server.quitCh:\n\t\t\tbreak loop\n\t\t}\n\t}\n\n\tstatus <- svc.Status{State: svc.StopPending}\n\treturn false, 0\n}\n\n// Run starts the NATS server as a Windows service.\nfunc Run(server *Server) error {\n\tif dockerized {\n\t\tserver.Start()\n\t\treturn nil\n\t}\n\tisWindowsService, err := svc.IsWindowsService()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !isWindowsService {\n\t\tserver.Start()\n\t\treturn nil\n\t}\n\treturn svc.Run(serviceName, &winServiceWrapper{server})\n}\n\n// isWindowsService indicates if NATS is running as a Windows service.\nfunc isWindowsService() bool {\n\tif dockerized {\n\t\treturn false\n\t}\n\tisWindowsService, _ := svc.IsWindowsService()\n\treturn isWindowsService\n}\n" }, { "path": "server/service_windows_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"golang.org/x/sys/windows/svc\"\n)\n\n// TestWinServiceWrapper reproduces the tests for service,\n// with extra complication for windows API\nfunc TestWinServiceWrapper(t *testing.T) {\n\t/*\n\t\tSince the windows API can't be tested through just the Run func,\n\t\ta basic mock checks the intractions of the server, as happens in svc.Run.\n\t\tThis test checks :\n\t\t- that the service fails to start within an unreasonable timeframe (serverC)\n\t\t- that the service signals its state correctly to windows with svc.StartPending (mockC)\n\t\t- that no other signal is sent to the windows service API(mockC)\n\t*/\n\tvar (\n\t\twsw = &winServiceWrapper{New(DefaultOptions())}\n\t\targs = make([]string, 0)\n\t\t// Size of 1 should be enough but don't want to block if the test fails.\n\t\tserverC = make(chan error, 10)\n\t\tmockC = make(chan error, 10)\n\t\tchanges = make(chan svc.ChangeRequest)\n\t\tstatus = make(chan svc.Status)\n\t)\n\tt.Setenv(\"NATS_STARTUP_DELAY\", \"1ns\") // purposely small\n\t// prepare mock expectations\n\twsm := &winSvcMock{status: status}\n\twsm.Expect(svc.StartPending)\n\n\tgo func() {\n\t\tmockC <- wsm.Listen(250 * time.Millisecond)\n\t}()\n\n\tgo func() {\n\t\tvar err error\n\t\t_, exitCode := wsw.Execute(args, changes, status)\n\t\t// We expect an error...\n\t\tif exitCode == 0 {\n\t\t\terr = errors.New(\"Should have exitCode != 0\")\n\t\t}\n\t\tserverC <- err\n\t}()\n\n\tcheckErr := func(c chan error, txt string) {\n\t\tselect {\n\t\tcase err := <-c:\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"%s: %v\", txt, err)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Test timed out\")\n\t\t}\n\t}\n\tcheckErr(mockC, \"windows.svc mock\")\n\tcheckErr(serverC, \"server behavior\")\n\n\twsw.server.Shutdown()\n}\n\nfunc TestWinServiceLDMExit(t *testing.T) {\n\tvar (\n\t\twsw = &winServiceWrapper{New(DefaultOptions())}\n\t\targs = make([]string, 0)\n\t\t// Size of 1 should be enough but don't want to block if the test fails.\n\t\tserverC = make(chan error, 10)\n\t\tmockC = make(chan error, 10)\n\t\tchanges = make(chan svc.ChangeRequest, 1)\n\t\tstatus = make(chan svc.Status)\n\t)\n\t// prepare mock expectations\n\twsm := &winSvcMock{status: status}\n\twsm.Expect(svc.StartPending)\n\t// Expect that we will be running\n\twsm.Expect(svc.Running)\n\t// Sending the LDM signal\n\tchanges <- svc.ChangeRequest{Cmd: svc.Cmd(ldmCode)}\n\t// Expect to be stopping\n\twsm.Expect(svc.StopPending)\n\n\tgo func() {\n\t\t// Duration long enough for the test to complete, but not too long\n\t\t// to wait for nothing.\n\t\tmockC <- wsm.Listen(time.Second)\n\t}()\n\n\tgo func() {\n\t\tvar err error\n\t\tif _, exitCode := wsw.Execute(args, changes, status); exitCode != 0 {\n\t\t\terr = fmt.Errorf(\"exited with %v\", exitCode)\n\t\t}\n\t\tserverC <- err\n\t}()\n\n\tcheckErr := func(c chan error, txt string) {\n\t\tselect {\n\t\tcase err := <-c:\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"%s: %v\", txt, err)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Test timed out\")\n\t\t}\n\t}\n\tcheckErr(mockC, \"windows.svc mock\")\n\tcheckErr(serverC, \"server behavior\")\n\n\t// At this point, we have proven already that the LDM signal did\n\t// end the NATS server's Execute() loop because we received the\n\t// svc.StopPending status. Still, we will verify that we get\n\t// notified that the server has shutdown.\n\tch := make(chan struct{}, 1)\n\tgo func() {\n\t\twsw.server.WaitForShutdown()\n\t\tch <- struct{}{}\n\t}()\n\tselect {\n\tcase <-ch:\n\t\t// OK\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Did not finish shutdown\")\n\t}\n}\n\n// winSvcMock mocks part of the golang.org/x/sys/windows/svc\n// execution stack, listening to svc.Status on its chan.\ntype winSvcMock struct {\n\tstatus chan svc.Status\n\texpectedSt []svc.State\n}\n\n// Expect allows to prepare a winSvcMock to receive a specific type of StatusMessage\nfunc (w *winSvcMock) Expect(st svc.State) {\n\tw.expectedSt = append(w.expectedSt, st)\n}\n\n// Listen is the mock's mainloop, expects messages to comply with previous Expect().\nfunc (w *winSvcMock) Listen(dur time.Duration) error {\n\ttimeout := time.NewTimer(dur)\n\tdefer timeout.Stop()\n\tfor _, state := range w.expectedSt {\n\t\tselect {\n\t\tcase status := <-w.status:\n\t\t\tif status.State != state {\n\t\t\t\treturn fmt.Errorf(\"message to winsock: expected %v, got %v\", state, status.State)\n\t\t\t}\n\t\tcase <-timeout.C:\n\t\t\treturn errors.New(\"Mock timed out\")\n\t\t}\n\t}\n\tselect {\n\tcase <-timeout.C:\n\t\treturn nil\n\tcase st := <-w.status:\n\t\treturn fmt.Errorf(\"extra message to winsock: got %v\", st)\n\n\t}\n}\n" }, { "path": "server/signal.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows && !wasm\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n)\n\nvar processName = \"nats-server\"\n\n// SetProcessName allows to change the expected name of the process.\nfunc SetProcessName(name string) {\n\tprocessName = name\n}\n\n// Signal Handling\nfunc (s *Server) handleSignals() {\n\tif s.getOpts().NoSigs {\n\t\treturn\n\t}\n\tc := make(chan os.Signal, 1)\n\n\tsignal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase sig := <-c:\n\t\t\t\ts.Noticef(\"Trapped %q signal\", sig)\n\t\t\t\tswitch sig {\n\t\t\t\tcase syscall.SIGINT:\n\t\t\t\t\ts.Shutdown()\n\t\t\t\t\ts.WaitForShutdown()\n\t\t\t\t\tos.Exit(0)\n\t\t\t\tcase syscall.SIGTERM:\n\t\t\t\t\t// Shutdown unless graceful shutdown already in progress.\n\t\t\t\t\ts.mu.Lock()\n\t\t\t\t\tldm := s.ldm\n\t\t\t\t\ts.mu.Unlock()\n\n\t\t\t\t\tif !ldm {\n\t\t\t\t\t\ts.Shutdown()\n\t\t\t\t\t\ts.WaitForShutdown()\n\t\t\t\t\t\tos.Exit(0)\n\t\t\t\t\t}\n\t\t\t\tcase syscall.SIGUSR1:\n\t\t\t\t\t// File log re-open for rotating file logs.\n\t\t\t\t\ts.ReOpenLogFile()\n\t\t\t\tcase syscall.SIGUSR2:\n\t\t\t\t\tgo s.lameDuckMode()\n\t\t\t\tcase syscall.SIGHUP:\n\t\t\t\t\t// Config reload.\n\t\t\t\t\tif err := s.Reload(); err != nil {\n\t\t\t\t\t\ts.Errorf(\"Failed to reload server configuration: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// ProcessSignal sends the given signal command to the given process. If pidStr\n// is empty, this will send the signal to the single running instance of\n// nats-server. If multiple instances are running, pidStr can be a globular\n// expression ending with '*'. This returns an error if the given process is\n// not running or the command is invalid.\nfunc ProcessSignal(command Command, pidExpr string) error {\n\tvar (\n\t\terr error\n\t\terrStr string\n\t\tpids = make([]int, 1)\n\t\tpidStr = strings.TrimSuffix(pidExpr, \"*\")\n\t\tisGlob = strings.HasSuffix(pidExpr, \"*\")\n\t)\n\n\t// Validate input if given\n\tif pidStr != \"\" {\n\t\tif pids[0], err = strconv.Atoi(pidStr); err != nil {\n\t\t\treturn fmt.Errorf(\"invalid pid: %s\", pidStr)\n\t\t}\n\t}\n\t// Gather all PIDs unless the input is specific\n\tif pidStr == \"\" || isGlob {\n\t\tif pids, err = resolvePids(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Multiple instances are running and the input is not an expression\n\tif len(pids) > 1 && !isGlob {\n\t\terrStr = fmt.Sprintf(\"multiple %s processes running:\", processName)\n\t\tfor _, p := range pids {\n\t\t\terrStr += fmt.Sprintf(\"\\n%d\", p)\n\t\t}\n\t\treturn errors.New(errStr)\n\t}\n\t// No instances are running\n\tif len(pids) == 0 {\n\t\treturn fmt.Errorf(\"no %s processes running\", processName)\n\t}\n\n\tvar signum syscall.Signal\n\tif signum, err = CommandToSignal(command); err != nil {\n\t\treturn err\n\t}\n\n\tfor _, pid := range pids {\n\t\tif _pidStr := strconv.Itoa(pid); _pidStr != pidStr && pidStr != \"\" {\n\t\t\tif !isGlob || !strings.HasPrefix(_pidStr, pidStr) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif err = kill(pid, signum); err != nil {\n\t\t\terrStr += fmt.Sprintf(\"\\nsignal %q %d: %s\", command, pid, err)\n\t\t}\n\t}\n\tif errStr != \"\" {\n\t\treturn errors.New(errStr)\n\t}\n\treturn nil\n}\n\n// Translates a command to a signal number\nfunc CommandToSignal(command Command) (syscall.Signal, error) {\n\tswitch command {\n\tcase CommandStop:\n\t\treturn syscall.SIGKILL, nil\n\tcase CommandQuit:\n\t\treturn syscall.SIGINT, nil\n\tcase CommandReopen:\n\t\treturn syscall.SIGUSR1, nil\n\tcase CommandReload:\n\t\treturn syscall.SIGHUP, nil\n\tcase commandLDMode:\n\t\treturn syscall.SIGUSR2, nil\n\tcase commandTerm:\n\t\treturn syscall.SIGTERM, nil\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"unknown signal %q\", command)\n\t}\n}\n\n// resolvePids returns the pids for all running nats-server processes.\nfunc resolvePids() ([]int, error) {\n\t// If pgrep isn't available, this will just bail out and the user will be\n\t// required to specify a pid.\n\toutput, err := pgrep()\n\tif err != nil {\n\t\tswitch err.(type) {\n\t\tcase *exec.ExitError:\n\t\t\t// ExitError indicates non-zero exit code, meaning no processes\n\t\t\t// found.\n\t\t\tbreak\n\t\tdefault:\n\t\t\treturn nil, errors.New(\"unable to resolve pid, try providing one\")\n\t\t}\n\t}\n\tvar (\n\t\tmyPid = os.Getpid()\n\t\tpidStrs = strings.Split(string(output), \"\\n\")\n\t\tpids = make([]int, 0, len(pidStrs))\n\t)\n\tfor _, pidStr := range pidStrs {\n\t\tif pidStr == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tpid, err := strconv.Atoi(pidStr)\n\t\tif err != nil {\n\t\t\treturn nil, errors.New(\"unable to resolve pid, try providing one\")\n\t\t}\n\t\t// Ignore the current process.\n\t\tif pid == myPid {\n\t\t\tcontinue\n\t\t}\n\t\tpids = append(pids, pid)\n\t}\n\treturn pids, nil\n}\n\nvar kill = func(pid int, signal syscall.Signal) error {\n\treturn syscall.Kill(pid, signal)\n}\n\nvar pgrep = func() ([]byte, error) {\n\treturn exec.Command(\"pgrep\", processName).Output()\n}\n" }, { "path": "server/signal_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/logger\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestSignalToReOpenLogFile(t *testing.T) {\n\tlogFile := filepath.Join(t.TempDir(), \"test.log\")\n\topts := &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tNoSigs: false,\n\t\tLogFile: logFile,\n\t}\n\ts := RunServer(opts)\n\tdefer s.SetLogger(nil, false, false)\n\tdefer s.Shutdown()\n\n\t// Set the file log\n\tfileLog := logger.NewFileLogger(s.opts.LogFile, s.opts.Logtime, s.opts.Debug, s.opts.Trace, true, logger.LogUTC(s.opts.LogtimeUTC))\n\ts.SetLogger(fileLog, false, false)\n\n\t// Add a trace\n\texpectedStr := \"This is a Notice\"\n\ts.Noticef(expectedStr)\n\tbuf, err := os.ReadFile(logFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\tif !strings.Contains(string(buf), expectedStr) {\n\t\tt.Fatalf(\"Expected log to contain %q, got %q\", expectedStr, string(buf))\n\t}\n\t// Rename the file\n\tif err := os.Rename(logFile, logFile+\".bak\"); err != nil {\n\t\tt.Fatalf(\"Unable to rename file: %v\", err)\n\t}\n\t// This should cause file to be reopened.\n\tsyscall.Kill(syscall.Getpid(), syscall.SIGUSR1)\n\t// Wait a bit for action to be performed\n\ttime.Sleep(500 * time.Millisecond)\n\tbuf, err = os.ReadFile(logFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading file: %v\", err)\n\t}\n\texpectedStr = \"File log re-opened\"\n\tif !strings.Contains(string(buf), expectedStr) {\n\t\tt.Fatalf(\"Expected log to contain %q, got %q\", expectedStr, string(buf))\n\t}\n}\n\nfunc TestSignalToReloadConfig(t *testing.T) {\n\ttmpl := `\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t\tA: { users: [ { user: %s, password: foo } ] }\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"foo\")))\n\topts, err := ProcessConfigFile(conf)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoLog = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Check that the reload time does not change when there are no changes.\n\tloaded := s.ConfigTime()\n\ttime.Sleep(250 * time.Millisecond)\n\tsyscall.Kill(syscall.Getpid(), syscall.SIGHUP)\n\ttime.Sleep(250 * time.Millisecond)\n\tif reloaded := s.ConfigTime(); reloaded.Equal(loaded) {\n\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected reload time to change: %s\\ngot: %s\", loaded, reloaded)\n\t}\n\n\t// Repeat test to make sure that server services signals more than once...\n\tfor i := 0; i < 2; i++ {\n\t\tloaded := s.ConfigTime()\n\t\tuser := fmt.Sprintf(\"foo:%d\", i)\n\t\tif err := os.WriteFile(conf, []byte(fmt.Sprintf(tmpl, user)), 0666); err != nil {\n\t\t\tt.Fatalf(\"Error creating config file: %v\", err)\n\t\t}\n\t\t// Wait a bit to ensure ConfigTime changes.\n\t\ttime.Sleep(5 * time.Millisecond)\n\n\t\t// This should cause config to be reloaded.\n\t\tsyscall.Kill(syscall.Getpid(), syscall.SIGHUP)\n\t\t// Wait a bit for action to be performed\n\t\ttime.Sleep(500 * time.Millisecond)\n\n\t\tif reloaded := s.ConfigTime(); !reloaded.After(loaded) {\n\t\t\tt.Fatalf(\"ConfigTime is incorrect.\\nexpected greater than: %s\\ngot: %s\", loaded, reloaded)\n\t\t}\n\t}\n}\n\nfunc TestProcessSignalNoProcesses(t *testing.T) {\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn nil, &exec.ExitError{}\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"no nats-server processes running\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalMultipleProcesses(t *testing.T) {\n\tpid := os.Getpid()\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn []byte(fmt.Sprintf(\"123\\n456\\n%d\\n\", pid)), nil\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"multiple nats-server processes running:\\n123\\n456\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalMultipleProcessesGlob(t *testing.T) {\n\tpid := os.Getpid()\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn []byte(fmt.Sprintf(\"123\\n456\\n%d\\n\", pid)), nil\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"*\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\tlines := strings.Split(err.Error(), \"\\n\")\n\trequire_Len(t, len(lines), 3)\n\trequire_Equal(t, lines[0], \"\") // Empty line comes first\n\trequire_True(t, strings.HasPrefix(lines[1], \"signal \\\"stop\\\" 123:\"))\n\trequire_True(t, strings.HasPrefix(lines[2], \"signal \\\"stop\\\" 456:\"))\n}\n\nfunc TestProcessSignalMultipleProcessesGlobPartial(t *testing.T) {\n\tpid := os.Getpid()\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn []byte(fmt.Sprintf(\"123\\n124\\n456\\n%d\\n\", pid)), nil\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"12*\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\tlines := strings.Split(err.Error(), \"\\n\")\n\trequire_Len(t, len(lines), 3)\n\trequire_Equal(t, lines[0], \"\") // Empty line comes first\n\trequire_True(t, strings.HasPrefix(lines[1], \"signal \\\"stop\\\" 123:\"))\n\trequire_True(t, strings.HasPrefix(lines[2], \"signal \\\"stop\\\" 124:\"))\n}\n\nfunc TestProcessSignalPgrepError(t *testing.T) {\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn nil, errors.New(\"error\")\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"unable to resolve pid, try providing one\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalPgrepMangled(t *testing.T) {\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn []byte(\"12x\"), nil\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\n\terr := ProcessSignal(CommandStop, \"\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"unable to resolve pid, try providing one\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalResolveSingleProcess(t *testing.T) {\n\tpid := os.Getpid()\n\tpgrepBefore := pgrep\n\tpgrep = func() ([]byte, error) {\n\t\treturn []byte(fmt.Sprintf(\"123\\n%d\\n\", pid)), nil\n\t}\n\tdefer func() {\n\t\tpgrep = pgrepBefore\n\t}()\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGKILL {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: killed\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(CommandStop, \"\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalInvalidCommand(t *testing.T) {\n\terr := ProcessSignal(Command(\"invalid\"), \"123\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"unknown signal \\\"invalid\\\"\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalInvalidPid(t *testing.T) {\n\terr := ProcessSignal(CommandStop, \"abc\")\n\tif err == nil {\n\t\tt.Fatal(\"Expected error\")\n\t}\n\texpectedStr := \"invalid pid: abc\"\n\tif err.Error() != expectedStr {\n\t\tt.Fatalf(\"Error is incorrect.\\nexpected: %s\\ngot: %s\", expectedStr, err.Error())\n\t}\n}\n\nfunc TestProcessSignalQuitProcess(t *testing.T) {\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGINT {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: interrupt\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(CommandQuit, \"123\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalTermProcess(t *testing.T) {\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGTERM {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: interrupt\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(commandTerm, \"123\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalReopenProcess(t *testing.T) {\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGUSR1 {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: user defined signal 1\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(CommandReopen, \"123\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalReloadProcess(t *testing.T) {\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGHUP {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: hangup\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(CommandReload, \"123\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalLameDuckMode(t *testing.T) {\n\tkillBefore := kill\n\tcalled := false\n\tkill = func(pid int, signal syscall.Signal) error {\n\t\tcalled = true\n\t\tif pid != 123 {\n\t\t\tt.Fatalf(\"pid is incorrect.\\nexpected: 123\\ngot: %d\", pid)\n\t\t}\n\t\tif signal != syscall.SIGUSR2 {\n\t\t\tt.Fatalf(\"signal is incorrect.\\nexpected: sigusr2\\ngot: %v\", signal)\n\t\t}\n\t\treturn nil\n\t}\n\tdefer func() {\n\t\tkill = killBefore\n\t}()\n\n\tif err := ProcessSignal(commandLDMode, \"123\"); err != nil {\n\t\tt.Fatalf(\"ProcessSignal failed: %v\", err)\n\t}\n\n\tif !called {\n\t\tt.Fatal(\"Expected kill to be called\")\n\t}\n}\n\nfunc TestProcessSignalTermDuringLameDuckMode(t *testing.T) {\n\topts := &Options{\n\t\tHost: \"127.0.0.1\",\n\t\tPort: -1,\n\t\tNoSigs: false,\n\t\tNoLog: true,\n\t\tLameDuckDuration: 2 * time.Second,\n\t\tLameDuckGracePeriod: 1 * time.Second,\n\t}\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Create single NATS Connection which will cause the server\n\t// to delay the shutdown.\n\tdoneCh := make(chan struct{})\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port),\n\t\tnats.DisconnectHandler(func(*nats.Conn) {\n\t\t\tclose(doneCh)\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Trigger lame duck based shutdown.\n\tgo s.lameDuckMode()\n\n\t// Wait for client to be disconnected.\n\tselect {\n\tcase <-doneCh:\n\t\tbreak\n\tcase <-time.After(3 * time.Second):\n\t\tt.Fatalf(\"Timed out waiting for client to disconnect\")\n\t}\n\n\t// Termination signal should not cause server to shutdown\n\t// while in lame duck mode already.\n\tsyscall.Kill(syscall.Getpid(), syscall.SIGTERM)\n\n\t// Wait for server shutdown due to lame duck shutdown.\n\ttimeoutCh := make(chan error)\n\ttimer := time.AfterFunc(3*time.Second, func() {\n\t\ttimeoutCh <- errors.New(\"Timed out waiting for server shutdown\")\n\t})\n\tfor range time.NewTicker(1 * time.Millisecond).C {\n\t\tselect {\n\t\tcase err := <-timeoutCh:\n\t\t\tt.Fatal(err)\n\t\tdefault:\n\t\t}\n\n\t\tif !s.isRunning() {\n\t\t\ttimer.Stop()\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestSignalInterruptHasSuccessfulExit(t *testing.T) {\n\tif os.Getenv(\"IN_TEST\") == \"1\" {\n\t\ts := RunServer(&Options{})\n\t\tdefer s.Shutdown()\n\t\trequire_NoError(t, syscall.Kill(syscall.Getpid(), syscall.SIGINT))\n\t\ts.WaitForShutdown()\n\t\treturn\n\t}\n\t// To check for successful/0 exit code, need execute as separate process.\n\tcmd := exec.Command(os.Args[0], \"-test.run=TestSignalInterruptHasSuccessfulExit\")\n\tcmd.Env = append(os.Environ(), \"IN_TEST=1\")\n\terr := cmd.Run()\n\trequire_NoError(t, err)\n}\n\nfunc TestSignalTermHasSuccessfulExit(t *testing.T) {\n\tif os.Getenv(\"IN_TEST\") == \"1\" {\n\t\ts := RunServer(&Options{})\n\t\tdefer s.Shutdown()\n\t\trequire_NoError(t, syscall.Kill(syscall.Getpid(), syscall.SIGTERM))\n\t\ts.WaitForShutdown()\n\t\treturn\n\t}\n\t// To check for successful/0 exit code, need execute as separate process.\n\tcmd := exec.Command(os.Args[0], \"-test.run=TestSignalTermHasSuccessfulExit\")\n\tcmd.Env = append(os.Environ(), \"IN_TEST=1\")\n\terr := cmd.Run()\n\trequire_NoError(t, err)\n}\n" }, { "path": "server/signal_wasm.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build wasm\n\npackage server\n\nfunc (s *Server) handleSignals() {\n\n}\n\nfunc ProcessSignal(command Command, service string) error {\n\treturn nil\n}\n" }, { "path": "server/signal_windows.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/signal\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"golang.org/x/sys/windows/svc\"\n\t\"golang.org/x/sys/windows/svc/mgr\"\n)\n\n// Signal Handling\nfunc (s *Server) handleSignals() {\n\tif s.getOpts().NoSigs {\n\t\treturn\n\t}\n\tc := make(chan os.Signal, 1)\n\n\tsignal.Notify(c, os.Interrupt, syscall.SIGTERM)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase sig := <-c:\n\t\t\t\ts.Debugf(\"Trapped %q signal\", sig)\n\t\t\t\ts.Shutdown()\n\t\t\t\tos.Exit(0)\n\t\t\tcase <-s.quitCh:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// ProcessSignal sends the given signal command to the running nats-server service.\n// If service is empty, this signals the \"nats-server\" service. This returns an\n// error is the given service is not running or the command is invalid.\nfunc ProcessSignal(command Command, service string) error {\n\tif service == \"\" {\n\t\tservice = serviceName\n\t}\n\n\tm, err := mgr.Connect()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer m.Disconnect()\n\n\ts, err := m.OpenService(service)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"could not access service: %v\", err)\n\t}\n\tdefer s.Close()\n\n\tvar (\n\t\tcmd svc.Cmd\n\t\tto svc.State\n\t)\n\n\tswitch command {\n\tcase CommandStop, CommandQuit:\n\t\tcmd = svc.Stop\n\t\tto = svc.Stopped\n\tcase CommandReopen:\n\t\tcmd = reopenLogCmd\n\t\tto = svc.Running\n\tcase CommandReload:\n\t\tcmd = svc.ParamChange\n\t\tto = svc.Running\n\tcase commandLDMode:\n\t\tcmd = ldmCmd\n\t\tto = svc.Running\n\tdefault:\n\t\treturn fmt.Errorf(\"unknown signal %q\", command)\n\t}\n\n\tstatus, err := s.Control(cmd)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"could not send control=%d: %v\", cmd, err)\n\t}\n\n\ttimeout := time.Now().Add(10 * time.Second)\n\tfor status.State != to {\n\t\tif timeout.Before(time.Now()) {\n\t\t\treturn fmt.Errorf(\"timeout waiting for service to go to state=%d\", to)\n\t\t}\n\t\ttime.Sleep(300 * time.Millisecond)\n\t\tstatus, err = s.Query()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not retrieve service status: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "server/split_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"net\"\n\t\"testing\"\n)\n\nfunc TestSplitBufferSubOp(t *testing.T) {\n\tcli, trash := net.Pipe()\n\tdefer cli.Close()\n\tdefer trash.Close()\n\n\ts := &Server{gacc: NewAccount(globalAccountName)}\n\tif err := s.newGateway(DefaultOptions()); err != nil {\n\t\tt.Fatalf(\"Error creating gateways: %v\", err)\n\t}\n\ts.registerAccount(s.gacc)\n\tc := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), nc: cli}\n\n\tsubop := []byte(\"SUB foo 1\\r\\n\")\n\tsubop1 := subop[:6]\n\tsubop2 := subop[6:]\n\n\tif err := c.parse(subop1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != SUB_ARG {\n\t\tt.Fatalf(\"Expected SUB_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(subop2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START state vs %d\\n\", c.state)\n\t}\n\tr := s.gacc.sl.Match(\"foo\")\n\tif r == nil || len(r.psubs) != 1 {\n\t\tt.Fatalf(\"Did not match subscription properly: %+v\\n\", r)\n\t}\n\tsub := r.psubs[0]\n\tif !bytes.Equal(sub.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Subject did not match expected 'foo' : '%s'\\n\", sub.subject)\n\t}\n\tif !bytes.Equal(sub.sid, []byte(\"1\")) {\n\t\tt.Fatalf(\"Sid did not match expected '1' : '%s'\\n\", sub.sid)\n\t}\n\tif sub.queue != nil {\n\t\tt.Fatalf(\"Received a non-nil queue: '%s'\\n\", sub.queue)\n\t}\n}\n\nfunc TestSplitBufferUnsubOp(t *testing.T) {\n\ts := &Server{gacc: NewAccount(globalAccountName), gateway: &srvGateway{}}\n\ts.registerAccount(s.gacc)\n\tc := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\n\tsubop := []byte(\"SUB foo 1024\\r\\n\")\n\tif err := c.parse(subop); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START state vs %d\\n\", c.state)\n\t}\n\n\tunsubop := []byte(\"UNSUB 1024\\r\\n\")\n\tunsubop1 := unsubop[:8]\n\tunsubop2 := unsubop[8:]\n\n\tif err := c.parse(unsubop1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != UNSUB_ARG {\n\t\tt.Fatalf(\"Expected UNSUB_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(unsubop2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START state vs %d\\n\", c.state)\n\t}\n\tr := s.gacc.sl.Match(\"foo\")\n\tif r != nil && len(r.psubs) != 0 {\n\t\tt.Fatalf(\"Should be no subscriptions in results: %+v\\n\", r)\n\t}\n}\n\nfunc TestSplitBufferPubOp(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tpub := []byte(\"PUB foo.bar INBOX.22 11\\r\\nhello world\\r\")\n\tpub1 := pub[:2]\n\tpub2 := pub[2:9]\n\tpub3 := pub[9:15]\n\tpub4 := pub[15:22]\n\tpub5 := pub[22:25]\n\tpub6 := pub[25:33]\n\tpub7 := pub[33:]\n\n\tif err := c.parse(pub1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_PU {\n\t\tt.Fatalf(\"Expected OP_PU state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != PUB_ARG {\n\t\tt.Fatalf(\"Expected OP_PU state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub3); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != PUB_ARG {\n\t\tt.Fatalf(\"Expected OP_PU state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub4); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != PUB_ARG {\n\t\tt.Fatalf(\"Expected PUB_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub5); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\n\t// Check c.pa\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"PUB arg subject incorrect: '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"INBOX.22\")) {\n\t\tt.Fatalf(\"PUB arg reply subject incorrect: '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.size != 11 {\n\t\tt.Fatalf(\"PUB arg msg size incorrect: %d\\n\", c.pa.size)\n\t}\n\tif err := c.parse(pub6); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub7); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_END_N {\n\t\tt.Fatalf(\"Expected MSG_END_N state vs %d\\n\", c.state)\n\t}\n}\n\nfunc TestSplitBufferPubOp2(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tpub := []byte(\"PUB foo.bar INBOX.22 11\\r\\nhello world\\r\\n\")\n\tpub1 := pub[:30]\n\tpub2 := pub[30:]\n\n\tif err := c.parse(pub1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(pub2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START state vs %d\\n\", c.state)\n\t}\n}\n\nfunc TestSplitBufferPubOp3(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tpubAll := []byte(\"PUB foo bar 11\\r\\nhello world\\r\\n\")\n\tpub := pubAll[:16]\n\n\tif err := c.parse(pub); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Unexpected subject: '%s' vs '%s'\\n\", c.pa.subject, \"foo\")\n\t}\n\n\t// Simulate next read of network, make sure pub state is saved\n\t// until msg payload has cleared.\n\tcopy(pubAll, \"XXXXXXXXXXXXXXXX\")\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Unexpected subject: '%s' vs '%s'\\n\", c.pa.subject, \"foo\")\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"bar\")) {\n\t\tt.Fatalf(\"Unexpected reply: '%s' vs '%s'\\n\", c.pa.reply, \"bar\")\n\t}\n\tif !bytes.Equal(c.pa.szb, []byte(\"11\")) {\n\t\tt.Fatalf(\"Unexpected size bytes: '%s' vs '%s'\\n\", c.pa.szb, \"11\")\n\t}\n}\n\nfunc TestSplitBufferPubOp4(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tpubAll := []byte(\"PUB foo 11\\r\\nhello world\\r\\n\")\n\tpub := pubAll[:12]\n\n\tif err := c.parse(pub); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Unexpected subject: '%s' vs '%s'\\n\", c.pa.subject, \"foo\")\n\t}\n\n\t// Simulate next read of network, make sure pub state is saved\n\t// until msg payload has cleared.\n\tcopy(pubAll, \"XXXXXXXXXXXX\")\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\tt.Fatalf(\"Unexpected subject: '%s' vs '%s'\\n\", c.pa.subject, \"foo\")\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"\")) {\n\t\tt.Fatalf(\"Unexpected reply: '%s' vs '%s'\\n\", c.pa.reply, \"\")\n\t}\n\tif !bytes.Equal(c.pa.szb, []byte(\"11\")) {\n\t\tt.Fatalf(\"Unexpected size bytes: '%s' vs '%s'\\n\", c.pa.szb, \"11\")\n\t}\n}\n\nfunc TestSplitBufferPubOp5(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tpubAll := []byte(\"PUB foo 11\\r\\nhello world\\r\\n\")\n\n\t// Splits need to be on MSG_END_R now too, so make sure we check that.\n\t// Split between \\r and \\n\n\tpub := pubAll[:len(pubAll)-1]\n\n\tif err := c.parse(pub); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.msgBuf == nil {\n\t\tt.Fatalf(\"msgBuf should not be nil!\\n\")\n\t}\n\tif !bytes.Equal(c.msgBuf, []byte(\"hello world\\r\")) {\n\t\tt.Fatalf(\"c.msgBuf did not snaphot the msg\")\n\t}\n}\n\nfunc TestSplitConnectArg(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\tconnectAll := []byte(\"CONNECT {\\\"verbose\\\":false,\\\"tls_required\\\":false,\" +\n\t\t\"\\\"user\\\":\\\"test\\\",\\\"pedantic\\\":true,\\\"pass\\\":\\\"pass\\\"}\\r\\n\")\n\n\targJSON := connectAll[8:]\n\n\tc1 := connectAll[:5]\n\tc2 := connectAll[5:22]\n\tc3 := connectAll[22 : len(connectAll)-2]\n\tc4 := connectAll[len(connectAll)-2:]\n\n\tif err := c.parse(c1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Unexpected argBug placeholder.\\n\")\n\t}\n\n\tif err := c.parse(c2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.argBuf == nil {\n\t\tt.Fatalf(\"Expected argBug to not be nil.\\n\")\n\t}\n\tif !bytes.Equal(c.argBuf, argJSON[:14]) {\n\t\tt.Fatalf(\"argBuf not correct, received %q, wanted %q\\n\", argJSON[:14], c.argBuf)\n\t}\n\n\tif err := c.parse(c3); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.argBuf == nil {\n\t\tt.Fatalf(\"Expected argBug to not be nil.\\n\")\n\t}\n\tif !bytes.Equal(c.argBuf, argJSON[:len(argJSON)-2]) {\n\t\tt.Fatalf(\"argBuf not correct, received %q, wanted %q\\n\",\n\t\t\targJSON[:len(argJSON)-2], c.argBuf)\n\t}\n\n\tif err := c.parse(c4); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Unexpected argBuf placeholder.\\n\")\n\t}\n}\n\nfunc TestSplitDanglingArgBuf(t *testing.T) {\n\ts := New(&defaultServerOptions)\n\tc := &client{srv: s, acc: s.gacc, msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription)}\n\n\t// We test to make sure we do not dangle any argBufs after processing\n\t// since that could lead to performance issues.\n\n\t// SUB\n\tsubop := []byte(\"SUB foo 1\\r\\n\")\n\tc.parse(subop[:6])\n\tc.parse(subop[6:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// UNSUB\n\tunsubop := []byte(\"UNSUB 1024\\r\\n\")\n\tc.parse(unsubop[:8])\n\tc.parse(unsubop[8:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// PUB\n\tpubop := []byte(\"PUB foo.bar INBOX.22 11\\r\\nhello world\\r\\n\")\n\tc.parse(pubop[:22])\n\tc.parse(pubop[22:25])\n\tif c.argBuf == nil {\n\t\tt.Fatal(\"Expected a non-nil argBuf!\")\n\t}\n\tc.parse(pubop[25:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// MINUS_ERR\n\terrop := []byte(\"-ERR Too Long\\r\\n\")\n\tc.parse(errop[:8])\n\tc.parse(errop[8:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// CONNECT_ARG\n\tconnop := []byte(\"CONNECT {\\\"verbose\\\":false,\\\"tls_required\\\":false,\" +\n\t\t\"\\\"user\\\":\\\"test\\\",\\\"pedantic\\\":true,\\\"pass\\\":\\\"pass\\\"}\\r\\n\")\n\tc.parse(connop[:22])\n\tc.parse(connop[22:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// INFO_ARG\n\tinfoop := []byte(\"INFO {\\\"server_id\\\":\\\"id\\\"}\\r\\n\")\n\tc.parse(infoop[:8])\n\tc.parse(infoop[8:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected c.argBuf to be nil: %q\\n\", c.argBuf)\n\t}\n\n\t// MSG (the client has to be a ROUTE)\n\tc = &client{msubs: -1, mpay: -1, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}}\n\tmsgop := []byte(\"RMSG $foo foo 5\\r\\nhello\\r\\n\")\n\tc.parse(msgop[:5])\n\tc.parse(msgop[5:10])\n\tif c.argBuf == nil {\n\t\tt.Fatal(\"Expected a non-nil argBuf\")\n\t}\n\tif string(c.argBuf) != \"$foo \" {\n\t\tt.Fatalf(\"Expected argBuf to be \\\"$foo \\\", got %q\", string(c.argBuf))\n\t}\n\tc.parse(msgop[10:])\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected argBuf to be nil: %q\", c.argBuf)\n\t}\n\tif c.msgBuf != nil {\n\t\tt.Fatalf(\"Expected msgBuf to be nil: %q\", c.msgBuf)\n\t}\n\n\tc.state = OP_START\n\t// Parse up-to somewhere in the middle of the payload.\n\t// Verify that we have saved the MSG_ARG info\n\tc.parse(msgop[:23])\n\tif c.argBuf == nil {\n\t\tt.Fatal(\"Expected a non-nil argBuf\")\n\t}\n\tif string(c.pa.account) != \"$foo\" {\n\t\tt.Fatalf(\"Expected account to be \\\"$foo\\\", got %q\", c.pa.account)\n\t}\n\tif string(c.pa.subject) != \"foo\" {\n\t\tt.Fatalf(\"Expected subject to be \\\"foo\\\", got %q\", c.pa.subject)\n\t}\n\tif string(c.pa.reply) != \"\" {\n\t\tt.Fatalf(\"Expected reply to be \\\"\\\", got %q\", c.pa.reply)\n\t}\n\tif c.pa.size != 5 {\n\t\tt.Fatalf(\"Expected sid to 5, got %v\", c.pa.size)\n\t}\n\t// msg buffer should be\n\tif c.msgBuf == nil || string(c.msgBuf) != \"hello\\r\" {\n\t\tt.Fatalf(\"Expected msgBuf to be \\\"hello\\r\\\", got %q\", c.msgBuf)\n\t}\n\tc.parse(msgop[23:])\n\t// At the end, we should have cleaned-up both arg and msg buffers.\n\tif c.argBuf != nil {\n\t\tt.Fatalf(\"Expected argBuf to be nil: %q\", c.argBuf)\n\t}\n\tif c.msgBuf != nil {\n\t\tt.Fatalf(\"Expected msgBuf to be nil: %q\", c.msgBuf)\n\t}\n}\n\nfunc TestSplitRoutedMsgArg(t *testing.T) {\n\t_, c, _ := setupClient()\n\tdefer c.close()\n\t// Allow parser to process RMSG\n\tc.kind = ROUTER\n\tc.route = &route{}\n\n\tb := make([]byte, 1024)\n\n\tcopy(b, []byte(\"RMSG $G hello.world 6040\\r\\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"))\n\tc.parse(b)\n\n\tcopy(b, []byte(\"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\\r\\n\"))\n\tc.parse(b)\n\n\twantAccount := \"$G\"\n\twantSubject := \"hello.world\"\n\twantSzb := \"6040\"\n\n\tif string(c.pa.account) != wantAccount {\n\t\tt.Fatalf(\"Incorrect account: want %q, got %q\", wantAccount, c.pa.account)\n\t}\n\tif string(c.pa.subject) != wantSubject {\n\t\tt.Fatalf(\"Incorrect subject: want %q, got %q\", wantSubject, c.pa.subject)\n\t}\n\tif string(c.pa.szb) != wantSzb {\n\t\tt.Fatalf(\"Incorrect szb: want %q, got %q\", wantSzb, c.pa.szb)\n\t}\n}\n\nfunc TestSplitBufferMsgOp(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: ROUTER, route: &route{}}\n\tmsg := []byte(\"RMSG $G foo.bar _INBOX.22 11\\r\\nhello world\\r\")\n\tmsg1 := msg[:2]\n\tmsg2 := msg[2:9]\n\tmsg3 := msg[9:15]\n\tmsg4 := msg[15:22]\n\tmsg5 := msg[22:25]\n\tmsg6 := msg[25:37]\n\tmsg7 := msg[37:40]\n\tmsg8 := msg[40:]\n\n\tif err := c.parse(msg1); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_M {\n\t\tt.Fatalf(\"Expected OP_M state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg2); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_ARG {\n\t\tt.Fatalf(\"Expected MSG_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg3); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_ARG {\n\t\tt.Fatalf(\"Expected MSG_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg4); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_ARG {\n\t\tt.Fatalf(\"Expected MSG_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg5); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_ARG {\n\t\tt.Fatalf(\"Expected MSG_ARG state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg6); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\n\t// Check c.pa\n\tif !bytes.Equal(c.pa.subject, []byte(\"foo.bar\")) {\n\t\tt.Fatalf(\"MSG arg subject incorrect: '%s'\\n\", c.pa.subject)\n\t}\n\tif !bytes.Equal(c.pa.reply, []byte(\"_INBOX.22\")) {\n\t\tt.Fatalf(\"MSG arg reply subject incorrect: '%s'\\n\", c.pa.reply)\n\t}\n\tif c.pa.size != 11 {\n\t\tt.Fatalf(\"MSG arg msg size incorrect: %d\\n\", c.pa.size)\n\t}\n\tif err := c.parse(msg7); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\tif err := c.parse(msg8); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_END_N {\n\t\tt.Fatalf(\"Expected MSG_END_N state vs %d\\n\", c.state)\n\t}\n}\n\nfunc TestSplitBufferLeafMsgArg(t *testing.T) {\n\tc := &client{msubs: -1, mpay: -1, mcl: 1024, subs: make(map[string]*subscription), kind: LEAF}\n\tmsg := []byte(\"LMSG foo + bar baz 11\\r\\n\")\n\tif err := c.parse(msg); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_PAYLOAD {\n\t\tt.Fatalf(\"Expected MSG_PAYLOAD state vs %d\\n\", c.state)\n\t}\n\tcheckPA := func(t *testing.T) {\n\t\tt.Helper()\n\t\tif !bytes.Equal(c.pa.subject, []byte(\"foo\")) {\n\t\t\tt.Fatalf(\"Expected subject to be %q, got %q\", \"foo\", c.pa.subject)\n\t\t}\n\t\tif !bytes.Equal(c.pa.reply, []byte(\"bar\")) {\n\t\t\tt.Fatalf(\"Expected reply to be %q, got %q\", \"bar\", c.pa.reply)\n\t\t}\n\t\tif n := len(c.pa.queues); n != 1 {\n\t\t\tt.Fatalf(\"Expected 1 queue, got %v\", n)\n\t\t}\n\t\tif !bytes.Equal(c.pa.queues[0], []byte(\"baz\")) {\n\t\t\tt.Fatalf(\"Expected queues to be %q, got %q\", \"baz\", c.pa.queues)\n\t\t}\n\t}\n\tcheckPA(t)\n\n\t// overwrite msg with payload\n\tn := copy(msg, []byte(\"fffffffffff\"))\n\tif err := c.parse(msg[:n]); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != MSG_END_R {\n\t\tt.Fatalf(\"Expected MSG_END_R state vs %d\\n\", c.state)\n\t}\n\tcheckPA(t)\n\n\t// Finish processing\n\tcopy(msg, []byte(\"\\r\\n\"))\n\tif err := c.parse(msg[:2]); err != nil {\n\t\tt.Fatalf(\"Unexpected parse error: %v\\n\", err)\n\t}\n\tif c.state != OP_START {\n\t\tt.Fatalf(\"Expected OP_START state vs %d\\n\", c.state)\n\t}\n\tif c.pa.subject != nil || c.pa.reply != nil || c.pa.queues != nil || c.pa.size != 0 ||\n\t\tc.pa.szb != nil || c.pa.arg != nil {\n\t\tt.Fatalf(\"parser state not cleaned-up properly: %+v\", c.pa)\n\t}\n}\n" }, { "path": "server/store.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"github.com/nats-io/nats-server/v2/server/avl\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n)\n\n// StorageType determines how messages are stored for retention.\ntype StorageType int\n\nconst (\n\t// File specifies on disk, designated by the JetStream config StoreDir.\n\tFileStorage = StorageType(22)\n\t// MemoryStorage specifies in memory only.\n\tMemoryStorage = StorageType(33)\n)\n\nvar (\n\t// ErrStoreClosed is returned when the store has been closed\n\tErrStoreClosed = errors.New(\"store is closed\")\n\t// ErrStoreMsgNotFound when message was not found but was expected to be.\n\tErrStoreMsgNotFound = errors.New(\"no message found\")\n\t// ErrStoreEOF is returned when message seq is greater than the last sequence.\n\tErrStoreEOF = errors.New(\"stream store EOF\")\n\t// ErrMaxMsgs is returned when we have discard new as a policy and we reached the message limit.\n\tErrMaxMsgs = errors.New(\"maximum messages exceeded\")\n\t// ErrMaxBytes is returned when we have discard new as a policy and we reached the bytes limit.\n\tErrMaxBytes = errors.New(\"maximum bytes exceeded\")\n\t// ErrMaxMsgsPerSubject is returned when we have discard new as a policy and we reached the message limit per subject.\n\tErrMaxMsgsPerSubject = errors.New(\"maximum messages per subject exceeded\")\n\t// ErrStoreSnapshotInProgress is returned when RemoveMsg or EraseMsg is called\n\t// while a snapshot is in progress.\n\tErrStoreSnapshotInProgress = errors.New(\"snapshot in progress\")\n\t// ErrMsgTooLarge is returned when a message is considered too large.\n\tErrMsgTooLarge = errors.New(\"message too large\")\n\t// ErrStoreWrongType is for when you access the wrong storage type.\n\tErrStoreWrongType = errors.New(\"wrong storage type\")\n\t// ErrNoAckPolicy is returned when trying to update a consumer's acks with no ack policy.\n\tErrNoAckPolicy = errors.New(\"ack policy is none\")\n\t// ErrSequenceMismatch is returned when storing a raw message and the expected sequence is wrong.\n\tErrSequenceMismatch = errors.New(\"expected sequence does not match store\")\n\t// ErrCorruptStreamState\n\tErrCorruptStreamState = errors.New(\"stream state snapshot is corrupt\")\n\t// ErrTooManyResults\n\tErrTooManyResults = errors.New(\"too many matching results for request\")\n\t// ErrStoreOldUpdate is returned when a consumer update is older than the current state.\n\tErrStoreOldUpdate = errors.New(\"old update ignored\")\n)\n\n// StoreMsg is the stored message format for messages that are retained by the Store layer.\ntype StoreMsg struct {\n\tsubj string\n\thdr []byte\n\tmsg []byte\n\tbuf []byte\n\tseq uint64\n\tts int64\n}\n\n// Used to call back into the upper layers to report on changes in storage resources.\n// For the cases where its a single message we will also supply sequence number and subject.\ntype StorageUpdateHandler func(msgs, bytes int64, seq uint64, subj string)\n\n// Used to call back into the upper layers to remove a message.\ntype StorageRemoveMsgHandler func(seq uint64)\n\n// Used to call back into the upper layers to process a JetStream message.\n// Will propose the message if the stream is replicated.\ntype ProcessJetStreamMsgHandler func(*inMsg)\n\ntype StreamStore interface {\n\tStoreMsg(subject string, hdr, msg []byte, ttl int64) (uint64, int64, error)\n\tStoreRawMsg(subject string, hdr, msg []byte, seq uint64, ts int64, ttl int64, discardNewCheck bool) error\n\tSkipMsg(seq uint64) (uint64, error)\n\tSkipMsgs(seq uint64, num uint64) error\n\tFlushAllPending() error\n\tLoadMsg(seq uint64, sm *StoreMsg) (*StoreMsg, error)\n\tLoadNextMsg(filter string, wc bool, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error)\n\tLoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error)\n\tLoadLastMsg(subject string, sm *StoreMsg) (*StoreMsg, error)\n\tLoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error)\n\tLoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error)\n\tRemoveMsg(seq uint64) (bool, error)\n\tEraseMsg(seq uint64) (bool, error)\n\tPurge() (uint64, error)\n\tPurgeEx(subject string, seq, keep uint64) (uint64, error)\n\tCompact(seq uint64) (uint64, error)\n\tTruncate(seq uint64) error\n\tGetSeqFromTime(t time.Time) uint64\n\tFilteredState(seq uint64, subject string) (SimpleState, error)\n\tSubjectsState(filterSubject string) map[string]SimpleState\n\tSubjectsTotals(filterSubject string) map[string]uint64\n\tAllLastSeqs() ([]uint64, error)\n\tMultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error)\n\tSubjectForSeq(seq uint64) (string, error)\n\tNumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error)\n\tNumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error)\n\tState() StreamState\n\tFastState(*StreamState)\n\tEncodedStreamState(failed uint64) (enc []byte, err error)\n\tSyncDeleted(dbs DeleteBlocks) error\n\tType() StorageType\n\tRegisterStorageUpdates(StorageUpdateHandler)\n\tRegisterStorageRemoveMsg(StorageRemoveMsgHandler)\n\tRegisterProcessJetStreamMsg(ProcessJetStreamMsgHandler)\n\tUpdateConfig(cfg *StreamConfig) error\n\tDelete(inline bool) error\n\tStop() error\n\tConsumerStore(name string, created time.Time, cfg *ConsumerConfig) (ConsumerStore, error)\n\tAddConsumer(o ConsumerStore) error\n\tRemoveConsumer(o ConsumerStore) error\n\tSnapshot(deadline time.Duration, includeConsumers, checkMsgs bool) (*SnapshotResult, error)\n\tUtilization() (total, reported uint64, err error)\n\tResetState()\n}\n\n// RetentionPolicy determines how messages in a set are retained.\ntype RetentionPolicy int\n\nconst (\n\t// LimitsPolicy (default) means that messages are retained until any given limit is reached.\n\t// This could be one of MaxMsgs, MaxBytes, or MaxAge.\n\tLimitsPolicy RetentionPolicy = iota\n\t// InterestPolicy specifies that when all known consumers have acknowledged a message it can be removed.\n\tInterestPolicy\n\t// WorkQueuePolicy specifies that when the first worker or subscriber acknowledges the message it can be removed.\n\tWorkQueuePolicy\n)\n\n// Discard Policy determines how we proceed when limits of messages or bytes are hit. The default, DicscardOld will\n// remove older messages. DiscardNew will fail to store the new message.\ntype DiscardPolicy int\n\nconst (\n\t// DiscardOld will remove older messages to return to the limits.\n\tDiscardOld = iota\n\t// DiscardNew will error on a StoreMsg call\n\tDiscardNew\n)\n\n// StreamState is information about the given stream.\ntype StreamState struct {\n\tMsgs uint64 `json:\"messages\"`\n\tBytes uint64 `json:\"bytes\"`\n\tFirstSeq uint64 `json:\"first_seq\"`\n\tFirstTime time.Time `json:\"first_ts\"`\n\tLastSeq uint64 `json:\"last_seq\"`\n\tLastTime time.Time `json:\"last_ts\"`\n\tNumSubjects int `json:\"num_subjects,omitempty\"`\n\tSubjects map[string]uint64 `json:\"subjects,omitempty\"`\n\tNumDeleted int `json:\"num_deleted,omitempty\"`\n\tDeleted []uint64 `json:\"deleted,omitempty\"`\n\tLost *LostStreamData `json:\"lost,omitempty\"`\n\tConsumers int `json:\"consumer_count\"`\n}\n\n// SimpleState for filtered subject specific state.\ntype SimpleState struct {\n\tMsgs uint64 `json:\"messages\"`\n\tFirst uint64 `json:\"first_seq\"`\n\tLast uint64 `json:\"last_seq\"`\n\n\t// Internal usage for when the first needs to be updated before use.\n\tfirstNeedsUpdate bool\n\t// Internal usage for when the last needs to be updated before use.\n\tlastNeedsUpdate bool\n}\n\n// LostStreamData indicates msgs that have been lost.\ntype LostStreamData struct {\n\tMsgs []uint64 `json:\"msgs\"`\n\tBytes uint64 `json:\"bytes\"`\n}\n\n// SnapshotResult contains information about the snapshot.\ntype SnapshotResult struct {\n\tReader io.ReadCloser\n\tState StreamState\n\terrCh chan string\n}\n\nconst (\n\t// Magic is used to identify stream state encodings.\n\tstreamStateMagic = uint8(42)\n\t// Version\n\tstreamStateVersion = uint8(1)\n\t// Magic / Identifier for run length encodings.\n\trunLengthMagic = uint8(33)\n\t// Magic / Identifier for AVL seqsets.\n\tseqSetMagic = uint8(22)\n)\n\n// Interface for DeleteBlock.\n// These will be of three types:\n// 1. AVL seqsets.\n// 2. Run length encoding of a deleted range.\n// 3. Legacy []uint64\ntype DeleteBlock interface {\n\tState() (first, last, num uint64)\n\tRange(f func(uint64) bool)\n}\n\ntype DeleteBlocks []DeleteBlock\n\n// StreamReplicatedState represents what is encoded in a binary stream snapshot used\n// for stream replication in an NRG.\ntype StreamReplicatedState struct {\n\tMsgs uint64\n\tBytes uint64\n\tFirstSeq uint64\n\tLastSeq uint64\n\tFailed uint64\n\tDeleted DeleteBlocks\n}\n\n// Determine if this is an encoded stream state.\nfunc IsEncodedStreamState(buf []byte) bool {\n\treturn len(buf) >= hdrLen && buf[0] == streamStateMagic && buf[1] == streamStateVersion\n}\n\nvar ErrBadStreamStateEncoding = errors.New(\"bad stream state encoding\")\n\nfunc DecodeStreamState(buf []byte) (*StreamReplicatedState, error) {\n\tss := &StreamReplicatedState{}\n\tif len(buf) < hdrLen || buf[0] != streamStateMagic || buf[1] != streamStateVersion {\n\t\treturn nil, ErrBadStreamStateEncoding\n\t}\n\tvar bi = hdrLen\n\n\treadU64 := func() uint64 {\n\t\tif bi < 0 || bi >= len(buf) {\n\t\t\tbi = -1\n\t\t\treturn 0\n\t\t}\n\t\tnum, n := binary.Uvarint(buf[bi:])\n\t\tif n <= 0 {\n\t\t\tbi = -1\n\t\t\treturn 0\n\t\t}\n\t\tbi += n\n\t\treturn num\n\t}\n\n\tparserFailed := func() bool {\n\t\treturn bi < 0\n\t}\n\n\tss.Msgs = readU64()\n\tss.Bytes = readU64()\n\tss.FirstSeq = readU64()\n\tss.LastSeq = readU64()\n\tss.Failed = readU64()\n\n\tif parserFailed() {\n\t\treturn nil, ErrCorruptStreamState\n\t}\n\n\tif numDeleted := readU64(); numDeleted > 0 {\n\t\t// If we have some deleted blocks.\n\t\tfor l := len(buf); l > bi; {\n\t\t\tswitch buf[bi] {\n\t\t\tcase seqSetMagic:\n\t\t\t\tdmap, n, err := avl.Decode(buf[bi:])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, ErrCorruptStreamState\n\t\t\t\t}\n\t\t\t\tbi += n\n\t\t\t\tss.Deleted = append(ss.Deleted, dmap)\n\t\t\tcase runLengthMagic:\n\t\t\t\tbi++\n\t\t\t\tvar rl DeleteRange\n\t\t\t\trl.First = readU64()\n\t\t\t\trl.Num = readU64()\n\t\t\t\tif parserFailed() {\n\t\t\t\t\treturn nil, ErrCorruptStreamState\n\t\t\t\t}\n\t\t\t\tss.Deleted = append(ss.Deleted, &rl)\n\t\t\tdefault:\n\t\t\t\treturn nil, ErrCorruptStreamState\n\t\t\t}\n\t\t}\n\t}\n\n\treturn ss, nil\n}\n\n// DeleteRange is a run length encoded delete range.\ntype DeleteRange struct {\n\tFirst uint64\n\tNum uint64\n}\n\nfunc (dr *DeleteRange) State() (first, last, num uint64) {\n\tdeletesAfterFirst := dr.Num\n\tif deletesAfterFirst > 0 {\n\t\tdeletesAfterFirst--\n\t}\n\treturn dr.First, dr.First + deletesAfterFirst, dr.Num\n}\n\n// Range will range over all the deleted sequences represented by this block.\nfunc (dr *DeleteRange) Range(f func(uint64) bool) {\n\tfor seq := dr.First; seq < dr.First+dr.Num; seq++ {\n\t\tif !f(seq) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Legacy []uint64\ntype DeleteSlice []uint64\n\nfunc (ds DeleteSlice) State() (first, last, num uint64) {\n\tif len(ds) == 0 {\n\t\treturn 0, 0, 0\n\t}\n\treturn ds[0], ds[len(ds)-1], uint64(len(ds))\n}\n\n// Range will range over all the deleted sequences represented by this []uint64.\nfunc (ds DeleteSlice) Range(f func(uint64) bool) {\n\tfor _, seq := range ds {\n\t\tif !f(seq) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (dbs DeleteBlocks) NumDeleted() (total uint64) {\n\tfor _, db := range dbs {\n\t\t_, _, num := db.State()\n\t\ttotal += num\n\t}\n\treturn total\n}\n\n// ConsumerStore stores state on consumers for streams.\ntype ConsumerStore interface {\n\tSetStarting(sseq uint64) error\n\tUpdateStarting(sseq uint64)\n\tReset(sseq uint64) error\n\tHasState() bool\n\tUpdateDelivered(dseq, sseq, dc uint64, ts int64) error\n\tUpdateAcks(dseq, sseq uint64) error\n\tUpdateConfig(cfg *ConsumerConfig) error\n\tUpdate(*ConsumerState) error\n\tForceUpdate(*ConsumerState) error\n\tState() (*ConsumerState, error)\n\tBorrowState() (*ConsumerState, error)\n\tEncodedState() ([]byte, error)\n\tType() StorageType\n\tStop() error\n\tDelete() error\n\tStreamDelete() error\n}\n\n// SequencePair has both the consumer and the stream sequence. They point to same message.\ntype SequencePair struct {\n\tConsumer uint64 `json:\"consumer_seq\"`\n\tStream uint64 `json:\"stream_seq\"`\n}\n\n// ConsumerState represents a stored state for a consumer.\ntype ConsumerState struct {\n\t// Delivered keeps track of last delivered sequence numbers for both the stream and the consumer.\n\tDelivered SequencePair `json:\"delivered\"`\n\t// AckFloor keeps track of the ack floors for both the stream and the consumer.\n\tAckFloor SequencePair `json:\"ack_floor\"`\n\t// These are both in stream sequence context.\n\t// Pending is for all messages pending and the timestamp for the delivered time.\n\t// This will only be present when the AckPolicy is ExplicitAck.\n\tPending map[uint64]*Pending `json:\"pending,omitempty\"`\n\t// This is for messages that have been redelivered, so count > 1.\n\tRedelivered map[uint64]uint64 `json:\"redelivered,omitempty\"`\n}\n\n// Encode consumer state.\nfunc encodeConsumerState(state *ConsumerState) []byte {\n\tvar hdr [seqsHdrSize]byte\n\tvar buf []byte\n\n\tmaxSize := seqsHdrSize\n\tif lp := len(state.Pending); lp > 0 {\n\t\tmaxSize += lp*(3*binary.MaxVarintLen64) + binary.MaxVarintLen64\n\t}\n\tif lr := len(state.Redelivered); lr > 0 {\n\t\tmaxSize += lr*(2*binary.MaxVarintLen64) + binary.MaxVarintLen64\n\t}\n\tif maxSize == seqsHdrSize {\n\t\tbuf = hdr[:seqsHdrSize]\n\t} else {\n\t\tbuf = make([]byte, maxSize)\n\t}\n\n\t// Write header\n\tbuf[0] = magic\n\tbuf[1] = 2\n\n\tn := hdrLen\n\tn += binary.PutUvarint(buf[n:], state.AckFloor.Consumer)\n\tn += binary.PutUvarint(buf[n:], state.AckFloor.Stream)\n\tn += binary.PutUvarint(buf[n:], state.Delivered.Consumer)\n\tn += binary.PutUvarint(buf[n:], state.Delivered.Stream)\n\tn += binary.PutUvarint(buf[n:], uint64(len(state.Pending)))\n\n\tasflr := state.AckFloor.Stream\n\tadflr := state.AckFloor.Consumer\n\n\t// These are optional, but always write len. This is to avoid a truncate inline.\n\tif len(state.Pending) > 0 {\n\t\t// To save space we will use now rounded to seconds to be our base timestamp.\n\t\tmints := time.Now().Round(time.Second).Unix()\n\t\t// Write minimum timestamp we found from above.\n\t\tn += binary.PutVarint(buf[n:], mints)\n\n\t\tfor k, v := range state.Pending {\n\t\t\tn += binary.PutUvarint(buf[n:], k-asflr)\n\t\t\tn += binary.PutUvarint(buf[n:], v.Sequence-adflr)\n\t\t\t// Downsample to seconds to save on space.\n\t\t\t// Subsecond resolution not needed for recovery etc.\n\t\t\tts := v.Timestamp / int64(time.Second)\n\t\t\tn += binary.PutVarint(buf[n:], mints-ts)\n\t\t}\n\t}\n\n\t// We always write the redelivered len.\n\tn += binary.PutUvarint(buf[n:], uint64(len(state.Redelivered)))\n\n\t// We expect these to be small.\n\tif len(state.Redelivered) > 0 {\n\t\tfor k, v := range state.Redelivered {\n\t\t\tn += binary.PutUvarint(buf[n:], k-asflr)\n\t\t\tn += binary.PutUvarint(buf[n:], v)\n\t\t}\n\t}\n\n\treturn buf[:n]\n}\n\n// Represents a pending message for explicit ack or ack all.\n// Sequence is the original consumer sequence.\ntype Pending struct {\n\tSequence uint64\n\tTimestamp int64\n}\n\nconst (\n\tlimitsPolicyJSONString = `\"limits\"`\n\tinterestPolicyJSONString = `\"interest\"`\n\tworkQueuePolicyJSONString = `\"workqueue\"`\n)\n\nvar (\n\tlimitsPolicyJSONBytes = []byte(limitsPolicyJSONString)\n\tinterestPolicyJSONBytes = []byte(interestPolicyJSONString)\n\tworkQueuePolicyJSONBytes = []byte(workQueuePolicyJSONString)\n)\n\nfunc (rp RetentionPolicy) String() string {\n\tswitch rp {\n\tcase LimitsPolicy:\n\t\treturn \"Limits\"\n\tcase InterestPolicy:\n\t\treturn \"Interest\"\n\tcase WorkQueuePolicy:\n\t\treturn \"WorkQueue\"\n\tdefault:\n\t\treturn \"Unknown Retention Policy\"\n\t}\n}\n\nfunc (rp RetentionPolicy) MarshalJSON() ([]byte, error) {\n\tswitch rp {\n\tcase LimitsPolicy:\n\t\treturn limitsPolicyJSONBytes, nil\n\tcase InterestPolicy:\n\t\treturn interestPolicyJSONBytes, nil\n\tcase WorkQueuePolicy:\n\t\treturn workQueuePolicyJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", rp)\n\t}\n}\n\nfunc (rp *RetentionPolicy) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase limitsPolicyJSONString:\n\t\t*rp = LimitsPolicy\n\tcase interestPolicyJSONString:\n\t\t*rp = InterestPolicy\n\tcase workQueuePolicyJSONString:\n\t\t*rp = WorkQueuePolicy\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\nfunc (dp DiscardPolicy) String() string {\n\tswitch dp {\n\tcase DiscardOld:\n\t\treturn \"DiscardOld\"\n\tcase DiscardNew:\n\t\treturn \"DiscardNew\"\n\tdefault:\n\t\treturn \"Unknown Discard Policy\"\n\t}\n}\n\nfunc (dp DiscardPolicy) MarshalJSON() ([]byte, error) {\n\tswitch dp {\n\tcase DiscardOld:\n\t\treturn []byte(`\"old\"`), nil\n\tcase DiscardNew:\n\t\treturn []byte(`\"new\"`), nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", dp)\n\t}\n}\n\nfunc (dp *DiscardPolicy) UnmarshalJSON(data []byte) error {\n\tswitch strings.ToLower(string(data)) {\n\tcase `\"old\"`:\n\t\t*dp = DiscardOld\n\tcase `\"new\"`:\n\t\t*dp = DiscardNew\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\nconst (\n\tmemoryStorageJSONString = `\"memory\"`\n\tfileStorageJSONString = `\"file\"`\n)\n\nvar (\n\tmemoryStorageJSONBytes = []byte(memoryStorageJSONString)\n\tfileStorageJSONBytes = []byte(fileStorageJSONString)\n)\n\nfunc (st StorageType) String() string {\n\tswitch st {\n\tcase MemoryStorage:\n\t\treturn \"Memory\"\n\tcase FileStorage:\n\t\treturn \"File\"\n\tdefault:\n\t\treturn \"Unknown Storage Type\"\n\t}\n}\n\nfunc (st StorageType) MarshalJSON() ([]byte, error) {\n\tswitch st {\n\tcase MemoryStorage:\n\t\treturn memoryStorageJSONBytes, nil\n\tcase FileStorage:\n\t\treturn fileStorageJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", st)\n\t}\n}\n\nfunc (st *StorageType) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase memoryStorageJSONString:\n\t\t*st = MemoryStorage\n\tcase fileStorageJSONString:\n\t\t*st = FileStorage\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\nconst (\n\tackNonePolicyJSONString = `\"none\"`\n\tackAllPolicyJSONString = `\"all\"`\n\tackExplicitPolicyJSONString = `\"explicit\"`\n)\n\nvar (\n\tackNonePolicyJSONBytes = []byte(ackNonePolicyJSONString)\n\tackAllPolicyJSONBytes = []byte(ackAllPolicyJSONString)\n\tackExplicitPolicyJSONBytes = []byte(ackExplicitPolicyJSONString)\n)\n\nfunc (ap AckPolicy) MarshalJSON() ([]byte, error) {\n\tswitch ap {\n\tcase AckNone:\n\t\treturn ackNonePolicyJSONBytes, nil\n\tcase AckAll:\n\t\treturn ackAllPolicyJSONBytes, nil\n\tcase AckExplicit:\n\t\treturn ackExplicitPolicyJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", ap)\n\t}\n}\n\nfunc (ap *AckPolicy) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase ackNonePolicyJSONString:\n\t\t*ap = AckNone\n\tcase ackAllPolicyJSONString:\n\t\t*ap = AckAll\n\tcase ackExplicitPolicyJSONString:\n\t\t*ap = AckExplicit\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\nconst (\n\treplayInstantPolicyJSONString = `\"instant\"`\n\treplayOriginalPolicyJSONString = `\"original\"`\n)\n\nvar (\n\treplayInstantPolicyJSONBytes = []byte(replayInstantPolicyJSONString)\n\treplayOriginalPolicyJSONBytes = []byte(replayOriginalPolicyJSONString)\n)\n\nfunc (rp ReplayPolicy) MarshalJSON() ([]byte, error) {\n\tswitch rp {\n\tcase ReplayInstant:\n\t\treturn replayInstantPolicyJSONBytes, nil\n\tcase ReplayOriginal:\n\t\treturn replayOriginalPolicyJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", rp)\n\t}\n}\n\nfunc (rp *ReplayPolicy) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase replayInstantPolicyJSONString:\n\t\t*rp = ReplayInstant\n\tcase replayOriginalPolicyJSONString:\n\t\t*rp = ReplayOriginal\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\nconst (\n\tdeliverAllPolicyJSONString = `\"all\"`\n\tdeliverLastPolicyJSONString = `\"last\"`\n\tdeliverNewPolicyJSONString = `\"new\"`\n\tdeliverByStartSequenceJSONString = `\"by_start_sequence\"`\n\tdeliverByStartTimeJSONString = `\"by_start_time\"`\n\tdeliverLastPerPolicyJSONString = `\"last_per_subject\"`\n\tdeliverUndefinedJSONString = `\"undefined\"`\n)\n\nvar (\n\tdeliverAllPolicyJSONBytes = []byte(deliverAllPolicyJSONString)\n\tdeliverLastPolicyJSONBytes = []byte(deliverLastPolicyJSONString)\n\tdeliverNewPolicyJSONBytes = []byte(deliverNewPolicyJSONString)\n\tdeliverByStartSequenceJSONBytes = []byte(deliverByStartSequenceJSONString)\n\tdeliverByStartTimeJSONBytes = []byte(deliverByStartTimeJSONString)\n\tdeliverLastPerPolicyJSONBytes = []byte(deliverLastPerPolicyJSONString)\n\tdeliverUndefinedJSONBytes = []byte(deliverUndefinedJSONString)\n)\n\nfunc (p *DeliverPolicy) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase deliverAllPolicyJSONString, deliverUndefinedJSONString:\n\t\t*p = DeliverAll\n\tcase deliverLastPolicyJSONString:\n\t\t*p = DeliverLast\n\tcase deliverLastPerPolicyJSONString:\n\t\t*p = DeliverLastPerSubject\n\tcase deliverNewPolicyJSONString:\n\t\t*p = DeliverNew\n\tcase deliverByStartSequenceJSONString:\n\t\t*p = DeliverByStartSequence\n\tcase deliverByStartTimeJSONString:\n\t\t*p = DeliverByStartTime\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\n\treturn nil\n}\n\nfunc (p DeliverPolicy) MarshalJSON() ([]byte, error) {\n\tswitch p {\n\tcase DeliverAll:\n\t\treturn deliverAllPolicyJSONBytes, nil\n\tcase DeliverLast:\n\t\treturn deliverLastPolicyJSONBytes, nil\n\tcase DeliverLastPerSubject:\n\t\treturn deliverLastPerPolicyJSONBytes, nil\n\tcase DeliverNew:\n\t\treturn deliverNewPolicyJSONBytes, nil\n\tcase DeliverByStartSequence:\n\t\treturn deliverByStartSequenceJSONBytes, nil\n\tcase DeliverByStartTime:\n\t\treturn deliverByStartTimeJSONBytes, nil\n\tdefault:\n\t\treturn deliverUndefinedJSONBytes, nil\n\t}\n}\n\nfunc isOutOfSpaceErr(err error) bool {\n\treturn err != nil && (strings.Contains(err.Error(), \"no space left\"))\n}\n\n// For when our upper layer catchup detects its missing messages from the beginning of the stream.\nvar errFirstSequenceMismatch = errors.New(\"first sequence mismatch\")\n\nfunc isClusterResetErr(err error) bool {\n\treturn err == errLastSeqMismatch || err == ErrStoreEOF || err == errFirstSequenceMismatch || errors.Is(err, errCatchupAbortedNoLeader) || err == errCatchupTooManyRetries\n}\n\n// Copy all fields.\nfunc (smo *StoreMsg) copy(sm *StoreMsg) {\n\tif sm.buf != nil {\n\t\tsm.buf = sm.buf[:0]\n\t}\n\tsm.buf = append(sm.buf, smo.buf...)\n\t// We set cap on header in case someone wants to expand it.\n\tsm.hdr, sm.msg = sm.buf[:len(smo.hdr):len(smo.hdr)], sm.buf[len(smo.hdr):]\n\tsm.subj, sm.seq, sm.ts = smo.subj, smo.seq, smo.ts\n}\n\n// Clear all fields except underlying buffer but reset that if present to [:0].\nfunc (sm *StoreMsg) clear() {\n\tif sm == nil {\n\t\treturn\n\t}\n\t*sm = StoreMsg{_EMPTY_, nil, nil, sm.buf, 0, 0}\n\tif len(sm.buf) > 0 {\n\t\tsm.buf = sm.buf[:0]\n\t}\n}\n\n// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer.\n// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same.\nfunc bytesToString(b []byte) string {\n\tif len(b) == 0 {\n\t\treturn _EMPTY_\n\t}\n\tp := unsafe.SliceData(b)\n\treturn unsafe.String(p, len(b))\n}\n\n// Same in reverse. Used less often.\nfunc stringToBytes(s string) []byte {\n\tif len(s) == 0 {\n\t\treturn nil\n\t}\n\tp := unsafe.StringData(s)\n\tb := unsafe.Slice(p, len(s))\n\treturn b\n}\n\n// Forces a copy of a string, for use in the case that you might have been passed a value when bytesToString was used,\n// but now you need a separate copy of it to store for longer-term use.\nfunc copyString(s string) string {\n\tb := make([]byte, len(s))\n\tcopy(b, s)\n\treturn bytesToString(b)\n}\n\nfunc isPermissionError(err error) bool {\n\treturn err != nil && os.IsPermission(err)\n}\n" }, { "path": "server/store_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !skip_store_tests\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n)\n\nfunc testAllStoreAllPermutations(t *testing.T, compressionAndEncryption bool, cfg StreamConfig, fn func(t *testing.T, fs StreamStore)) {\n\tt.Run(\"Memory\", func(t *testing.T) {\n\t\tcfg.Storage = MemoryStorage\n\t\tfs, err := newMemStore(&cfg)\n\t\trequire_NoError(t, err)\n\t\tdefer fs.Stop()\n\t\tfn(t, fs)\n\t})\n\tt.Run(\"File\", func(t *testing.T) {\n\t\tcfg.Storage = FileStorage\n\t\tif compressionAndEncryption {\n\t\t\ttestFileStoreAllPermutations(t, func(t *testing.T, fcfg FileStoreConfig) {\n\t\t\t\tfs, err := newFileStore(fcfg, cfg)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tdefer fs.Stop()\n\t\t\t\tfn(t, fs)\n\t\t\t})\n\t\t} else {\n\t\t\tfs, err := newFileStore(FileStoreConfig{\n\t\t\t\tStoreDir: t.TempDir(),\n\t\t\t}, cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\tdefer fs.Stop()\n\t\t\tfn(t, fs)\n\t\t}\n\t})\n}\n\nfunc TestStoreMsgLoadNextMsgMulti(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t// Put 1k msgs in\n\t\t\tfor i := 0; i < 1000; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\tfs.StoreMsg(subj, nil, []byte(\"ZZZ\"), 0)\n\t\t\t}\n\n\t\t\tvar smv StoreMsg\n\t\t\t// Do multi load next with 1 wc entry.\n\t\t\tsl := gsl.NewSublist[struct{}]()\n\t\t\tsl.Insert(\"foo.>\", struct{}{})\n\t\t\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\t\t\tsm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\t\t\trequire_Equal(t, nseq, seq)\n\t\t\t\tseq++\n\t\t\t}\n\n\t\t\t// Now do multi load next with 1000 literal subjects.\n\t\t\tsl = gsl.NewSublist[struct{}]()\n\t\t\tfor i := 0; i < 1000; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i)\n\t\t\t\tsl.Insert(subj, struct{}{})\n\t\t\t}\n\t\t\tfor i, seq := 0, uint64(1); i < 1000; i++ {\n\t\t\t\tsm, nseq, err := fs.LoadNextMsgMulti(sl, seq, &smv)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", i))\n\t\t\t\trequire_Equal(t, nseq, seq)\n\t\t\t\tseq++\n\t\t\t}\n\n\t\t\t// Check that we can pull out 3 individuals.\n\t\t\tsl = gsl.NewSublist[struct{}]()\n\t\t\tsl.Insert(\"foo.2\", struct{}{})\n\t\t\tsl.Insert(\"foo.222\", struct{}{})\n\t\t\tsl.Insert(\"foo.999\", struct{}{})\n\t\t\tsm, seq, err := fs.LoadNextMsgMulti(sl, 1, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.subj, \"foo.2\")\n\t\t\trequire_Equal(t, seq, 3)\n\t\t\tsm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.subj, \"foo.222\")\n\t\t\trequire_Equal(t, seq, 223)\n\t\t\tsm, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.subj, \"foo.999\")\n\t\t\trequire_Equal(t, seq, 1000)\n\t\t\t_, seq, err = fs.LoadNextMsgMulti(sl, seq+1, &smv)\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Equal(t, seq, 1000)\n\t\t},\n\t)\n}\n\nfunc TestStoreLoadNextMsgWildcardStartBeforeFirstMatch(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"bar.*\", \"foo.*\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t// Fill non-matching subjects first so the first wildcard match starts\n\t\t\t// strictly after the requested start sequence.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tsubj := fmt.Sprintf(\"bar.%d\", i)\n\t\t\t\t_, _, err := fs.StoreMsg(subj, nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tseq, _, err := fs.StoreMsg(\"foo.1\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, seq, uint64(101))\n\n\t\t\tvar smv StoreMsg\n\t\t\tsm, nseq, err := fs.LoadNextMsg(\"foo.*\", true, 1, &smv)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.subj, \"foo.1\")\n\t\t\trequire_Equal(t, nseq, uint64(101))\n\n\t\t\t_, nseq, err = fs.LoadNextMsg(\"foo.*\", true, nseq+1, &smv)\n\t\t\trequire_Error(t, err)\n\t\t\trequire_Equal(t, nseq, uint64(101))\n\t\t},\n\t)\n}\n\nfunc TestStoreDeleteSlice(t *testing.T) {\n\tds := DeleteSlice{2}\n\tvar deletes []uint64\n\tds.Range(func(seq uint64) bool {\n\t\tdeletes = append(deletes, seq)\n\t\treturn true\n\t})\n\trequire_Len(t, len(deletes), 1)\n\trequire_Equal(t, deletes[0], 2)\n\n\tfirst, last, num := ds.State()\n\trequire_Equal(t, first, 2)\n\trequire_Equal(t, last, 2)\n\trequire_Equal(t, num, 1)\n}\n\nfunc TestStoreDeleteRange(t *testing.T) {\n\tdr := DeleteRange{First: 2, Num: 1}\n\tvar deletes []uint64\n\tdr.Range(func(seq uint64) bool {\n\t\tdeletes = append(deletes, seq)\n\t\treturn true\n\t})\n\trequire_Len(t, len(deletes), 1)\n\trequire_Equal(t, deletes[0], 2)\n\n\tfirst, last, num := dr.State()\n\trequire_Equal(t, first, 2)\n\trequire_Equal(t, last, 2)\n\trequire_Equal(t, num, 1)\n}\n\nfunc TestStoreSubjectStateConsistency(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tgetSubjectState := func() SimpleState {\n\t\t\t\tt.Helper()\n\t\t\t\tss := fs.SubjectsState(\"foo\")\n\t\t\t\treturn ss[\"foo\"]\n\t\t\t}\n\t\t\tvar smp StoreMsg\n\t\t\texpectFirstSeq := func(eseq uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tsm, _, err := fs.LoadNextMsg(\"foo\", false, 0, &smp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.seq, eseq)\n\t\t\t}\n\t\t\texpectLastSeq := func(eseq uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tsm, err := fs.LoadLastMsg(\"foo\", &smp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.seq, eseq)\n\t\t\t}\n\n\t\t\t// Publish an initial batch of messages.\n\t\t\tfor i := 0; i < 4; i++ {\n\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Expect 4 msgs, with first=1, last=4.\n\t\t\tss := getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 4)\n\t\t\trequire_Equal(t, ss.First, 1)\n\t\t\texpectFirstSeq(1)\n\t\t\trequire_Equal(t, ss.Last, 4)\n\t\t\texpectLastSeq(4)\n\n\t\t\t// Remove first message, ss.First is lazy so will only mark ss.firstNeedsUpdate.\n\t\t\tremoved, err := fs.RemoveMsg(1)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// Will update first, so corrects to seq 2.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 3)\n\t\t\trequire_Equal(t, ss.First, 2)\n\t\t\texpectFirstSeq(2)\n\t\t\trequire_Equal(t, ss.Last, 4)\n\t\t\texpectLastSeq(4)\n\n\t\t\t// Remove last message, ss.Last is lazy so will only mark ss.lastNeedsUpdate.\n\t\t\tremoved, err = fs.RemoveMsg(4)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// Will update last, so corrects to 3.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 2)\n\t\t\trequire_Equal(t, ss.First, 2)\n\t\t\texpectFirstSeq(2)\n\t\t\trequire_Equal(t, ss.Last, 3)\n\t\t\texpectLastSeq(3)\n\n\t\t\t// Remove first message again.\n\t\t\tremoved, err = fs.RemoveMsg(2)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// Since we only have one message left, must update ss.First and ensure ss.Last equals.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 3)\n\t\t\texpectFirstSeq(3)\n\t\t\trequire_Equal(t, ss.Last, 3)\n\t\t\texpectLastSeq(3)\n\n\t\t\t// Publish some more messages so we can test another scenario.\n\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\t// Just check the state is complete again.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 4)\n\t\t\trequire_Equal(t, ss.First, 3)\n\t\t\texpectFirstSeq(3)\n\t\t\trequire_Equal(t, ss.Last, 7)\n\t\t\texpectLastSeq(7)\n\n\t\t\t// Remove last sequence, ss.Last is lazy so doesn't get updated.\n\t\t\tremoved, err = fs.RemoveMsg(7)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// Remove first sequence, ss.First is lazy so doesn't get updated.\n\t\t\tremoved, err = fs.RemoveMsg(3)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// Remove (now) first sequence. Both ss.First and ss.Last are lazy and both need to be recalculated later.\n\t\t\tremoved, err = fs.RemoveMsg(5)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\n\t\t\t// ss.First and ss.Last should both be recalculated and equal each other.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 6)\n\t\t\texpectFirstSeq(6)\n\t\t\trequire_Equal(t, ss.Last, 6)\n\t\t\texpectLastSeq(6)\n\n\t\t\t// We store a new message for ss.Last and remove it after, which marks it to be recalculated.\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tremoved, err = fs.RemoveMsg(8)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_True(t, removed)\n\t\t\t// This will be the new ss.Last message, so reset ss.lastNeedsUpdate\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// ss.First should remain the same, but ss.Last should equal the last message.\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 2)\n\t\t\trequire_Equal(t, ss.First, 6)\n\t\t\texpectFirstSeq(6)\n\t\t\trequire_Equal(t, ss.Last, 9)\n\t\t\texpectLastSeq(9)\n\t\t},\n\t)\n}\n\nfunc TestStoreSubjectStateConsistencyOptimization(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tfillMsgs := func(c int) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor i := 0; i < c; i++ {\n\t\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tremoveMsgs := func(seqs ...uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tfor _, seq := range seqs {\n\t\t\t\t\tremoved, err := fs.RemoveMsg(seq)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_True(t, removed)\n\t\t\t\t}\n\t\t\t}\n\t\t\tgetSubjectState := func() (ss *SimpleState) {\n\t\t\t\tt.Helper()\n\t\t\t\tif f, ok := fs.(*fileStore); ok {\n\t\t\t\t\tss, ok = f.lmb.fss.Find([]byte(\"foo\"))\n\t\t\t\t\trequire_True(t, ok)\n\t\t\t\t} else if ms, ok := fs.(*memStore); ok {\n\t\t\t\t\tss, ok = ms.fss.Find([]byte(\"foo\"))\n\t\t\t\t\trequire_True(t, ok)\n\t\t\t\t} else {\n\t\t\t\t\tt.Fatal(\"Store not supported\")\n\t\t\t\t}\n\t\t\t\treturn ss\n\t\t\t}\n\t\t\tvar smp StoreMsg\n\t\t\texpectSeq := func(seq uint64) {\n\t\t\t\tt.Helper()\n\t\t\t\tsm, _, err := fs.LoadNextMsg(\"foo\", false, 0, &smp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.seq, seq)\n\t\t\t\tsm, err = fs.LoadLastMsg(\"foo\", &smp)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.seq, seq)\n\t\t\t}\n\n\t\t\t// results in ss.Last, ss.First is marked lazy (when we hit ss.Msgs-1==1).\n\t\t\tfillMsgs(3)\n\t\t\tremoveMsgs(2, 1)\n\t\t\tss := getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 3)\n\t\t\trequire_Equal(t, ss.Last, 3)\n\t\t\trequire_False(t, ss.firstNeedsUpdate)\n\t\t\trequire_False(t, ss.lastNeedsUpdate)\n\t\t\texpectSeq(3)\n\n\t\t\t// ss.First is marked lazy first, then ss.Last is marked lazy (when we hit ss.Msgs-1==1).\n\t\t\tfillMsgs(2)\n\t\t\tremoveMsgs(3, 5)\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 3)\n\t\t\trequire_Equal(t, ss.Last, 5)\n\t\t\trequire_True(t, ss.firstNeedsUpdate)\n\t\t\trequire_True(t, ss.lastNeedsUpdate)\n\t\t\texpectSeq(4)\n\n\t\t\t// ss.Last is marked lazy first, then ss.First is marked lazy (when we hit ss.Msgs-1==1).\n\t\t\tfillMsgs(2)\n\t\t\tremoveMsgs(7, 4)\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 4)\n\t\t\trequire_Equal(t, ss.Last, 7)\n\t\t\trequire_True(t, ss.firstNeedsUpdate)\n\t\t\trequire_True(t, ss.lastNeedsUpdate)\n\t\t\texpectSeq(6)\n\n\t\t\t// ss.Msgs=1, results in ss.First, ss.Last is marked lazy (when we hit ss.Msgs-1==1).\n\t\t\tfillMsgs(2)\n\t\t\tremoveMsgs(9, 8)\n\t\t\tss = getSubjectState()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.First, 6)\n\t\t\trequire_Equal(t, ss.Last, 6)\n\t\t\trequire_False(t, ss.firstNeedsUpdate)\n\t\t\trequire_False(t, ss.lastNeedsUpdate)\n\t\t\texpectSeq(6)\n\t\t},\n\t)\n}\n\nfunc TestStoreMaxMsgsPerUpdateBug(t *testing.T) {\n\tconfig := func() StreamConfig {\n\t\treturn StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, MaxMsgsPer: 0}\n\t}\n\ttestAllStoreAllPermutations(\n\t\tt, false, config(),\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tfor i := 0; i < 5; i++ {\n\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\n\t\t\tss := fs.State()\n\t\t\trequire_Equal(t, ss.Msgs, 5)\n\t\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\t\trequire_Equal(t, ss.LastSeq, 5)\n\n\t\t\t// Update max messages per-subject from 0 (infinite) to 1.\n\t\t\t// Since the per-subject limit was not specified before, messages should be removed upon config update.\n\t\t\tcfg := config()\n\t\t\tif _, ok := fs.(*fileStore); ok {\n\t\t\t\tcfg.Storage = FileStorage\n\t\t\t} else {\n\t\t\t\tcfg.Storage = MemoryStorage\n\t\t\t}\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t\terr := fs.UpdateConfig(&cfg)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Only one message should remain.\n\t\t\tss = fs.State()\n\t\t\trequire_Equal(t, ss.Msgs, 1)\n\t\t\trequire_Equal(t, ss.FirstSeq, 5)\n\t\t\trequire_Equal(t, ss.LastSeq, 5)\n\n\t\t\t// Update max messages per-subject from 0 (infinite) to an invalid value (< -1).\n\t\t\tcfg.MaxMsgsPer = -2\n\t\t\terr = fs.UpdateConfig(&cfg)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, cfg.MaxMsgsPer, -1)\n\t\t},\n\t)\n}\n\nfunc TestStoreCompactCleansUpDmap(t *testing.T) {\n\tconfig := func() StreamConfig {\n\t\treturn StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, MaxMsgsPer: 0}\n\t}\n\tfor cseq := uint64(2); cseq <= 4; cseq++ {\n\t\tt.Run(fmt.Sprintf(\"Compact(%d)\", cseq), func(t *testing.T) {\n\t\t\ttestAllStoreAllPermutations(\n\t\t\t\tt, false, config(),\n\t\t\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t\t\tdmapEntries := func() int {\n\t\t\t\t\t\tif fss, ok := fs.(*fileStore); ok {\n\t\t\t\t\t\t\treturn fss.dmapEntries()\n\t\t\t\t\t\t} else if mss, ok := fs.(*memStore); ok {\n\t\t\t\t\t\t\tmss.mu.RLock()\n\t\t\t\t\t\t\tdefer mss.mu.RUnlock()\n\t\t\t\t\t\t\treturn mss.dmap.Size()\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\treturn 0\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Publish messages, should have no interior deletes.\n\t\t\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t}\n\t\t\t\t\trequire_Len(t, dmapEntries(), 0)\n\n\t\t\t\t\t// Removing one message in the middle should be an interior delete.\n\t\t\t\t\t_, err := fs.RemoveMsg(2)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, dmapEntries(), 1)\n\n\t\t\t\t\t// Compacting must always clean up the interior delete.\n\t\t\t\t\t_, err = fs.Compact(cseq)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, dmapEntries(), 0)\n\n\t\t\t\t\t// Validate first/last sequence.\n\t\t\t\t\tstate := fs.State()\n\t\t\t\t\tfseq := uint64(3)\n\t\t\t\t\tif fseq < cseq {\n\t\t\t\t\t\tfseq = cseq\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, state.FirstSeq, fseq)\n\t\t\t\t\trequire_Equal(t, state.LastSeq, 3)\n\t\t\t\t})\n\t\t})\n\t}\n}\n\nfunc TestStoreTruncateCleansUpDmap(t *testing.T) {\n\tconfig := func() StreamConfig {\n\t\treturn StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}, MaxMsgsPer: 0}\n\t}\n\tfor tseq := uint64(0); tseq <= 1; tseq++ {\n\t\tt.Run(fmt.Sprintf(\"Truncate(%d)\", tseq), func(t *testing.T) {\n\t\t\ttestAllStoreAllPermutations(\n\t\t\t\tt, false, config(),\n\t\t\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t\t\tdmapEntries := func() int {\n\t\t\t\t\t\tif fss, ok := fs.(*fileStore); ok {\n\t\t\t\t\t\t\treturn fss.dmapEntries()\n\t\t\t\t\t\t} else if mss, ok := fs.(*memStore); ok {\n\t\t\t\t\t\t\tmss.mu.RLock()\n\t\t\t\t\t\t\tdefer mss.mu.RUnlock()\n\t\t\t\t\t\t\treturn mss.dmap.Size()\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\treturn 0\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Publish messages, should have no interior deletes.\n\t\t\t\t\tfor i := 0; i < 3; i++ {\n\t\t\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t}\n\t\t\t\t\trequire_Len(t, dmapEntries(), 0)\n\n\t\t\t\t\t// Removing one message in the middle should be an interior delete.\n\t\t\t\t\t_, err := fs.RemoveMsg(2)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, dmapEntries(), 1)\n\n\t\t\t\t\t// Truncating must always clean up the interior delete.\n\t\t\t\t\terr = fs.Truncate(tseq)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_Len(t, dmapEntries(), 0)\n\n\t\t\t\t\t// Validate first/last sequence.\n\t\t\t\t\tstate := fs.State()\n\t\t\t\t\tfseq := uint64(1)\n\t\t\t\t\tif fseq > tseq {\n\t\t\t\t\t\tfseq = tseq\n\t\t\t\t\t}\n\t\t\t\t\trequire_Equal(t, state.FirstSeq, fseq)\n\t\t\t\t\trequire_Equal(t, state.LastSeq, tseq)\n\t\t\t\t})\n\t\t})\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/6709\nfunc TestStorePurgeExZero(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t// Simple purge all.\n\t\t\t_, err := fs.Purge()\n\t\t\trequire_NoError(t, err)\n\t\t\tss := fs.State()\n\t\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\t\trequire_Equal(t, ss.LastSeq, 0)\n\n\t\t\t// PurgeEx(seq=0) must be equal.\n\t\t\t_, err = fs.PurgeEx(_EMPTY_, 0, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tss = fs.State()\n\t\t\trequire_Equal(t, ss.FirstSeq, 1)\n\t\t\trequire_Equal(t, ss.LastSeq, 0)\n\t\t},\n\t)\n}\n\nfunc TestStoreUpdateConfigTTLState(t *testing.T) {\n\tconfig := func() StreamConfig {\n\t\treturn StreamConfig{Name: \"TEST\", Subjects: []string{\"foo\"}}\n\t}\n\ttestAllStoreAllPermutations(\n\t\tt, false, config(),\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tcfg := config()\n\t\t\tswitch fs.(type) {\n\t\t\tcase *fileStore:\n\t\t\t\tcfg.Storage = FileStorage\n\t\t\tcase *memStore:\n\t\t\t\tcfg.Storage = MemoryStorage\n\t\t\t}\n\n\t\t\t// TTLs disabled at this point so this message should survive.\n\t\t\tseq, _, err := fs.StoreMsg(\"foo\", nil, nil, 1)\n\t\t\trequire_NoError(t, err)\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t_, err = fs.LoadMsg(seq, nil)\n\t\t\trequire_NoError(t, err)\n\n\t\t\t// Now enable TTLs.\n\t\t\tcfg.AllowMsgTTL = true\n\t\t\trequire_NoError(t, fs.UpdateConfig(&cfg))\n\n\t\t\t// TTLs enabled at this point so this message should be cleaned up.\n\t\t\tseq, _, err = fs.StoreMsg(\"foo\", nil, nil, 1)\n\t\t\trequire_NoError(t, err)\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t_, err = fs.LoadMsg(seq, nil)\n\t\t\trequire_Error(t, err)\n\n\t\t\t// Now disable TTLs again.\n\t\t\tcfg.AllowMsgTTL = false\n\t\t\trequire_NoError(t, fs.UpdateConfig(&cfg))\n\n\t\t\t// TTLs disabled again so this message should survive.\n\t\t\tseq, _, err = fs.StoreMsg(\"foo\", nil, nil, 1)\n\t\t\trequire_NoError(t, err)\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t_, err = fs.LoadMsg(seq, nil)\n\t\t\trequire_NoError(t, err)\n\t\t},\n\t)\n}\n\nfunc TestStoreStreamInteriorDeleteAccounting(t *testing.T) {\n\ttests := []struct {\n\t\ttitle string\n\t\taction func(s StreamStore, lseq uint64)\n\t}{\n\t\t{\n\t\t\ttitle: \"TruncateWithRemove\",\n\t\t\taction: func(s StreamStore, lseq uint64) {\n\t\t\t\tseq, _, err := s.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, seq, lseq)\n\t\t\t\tremoved, err := s.RemoveMsg(lseq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\t\t\t\trequire_NoError(t, s.Truncate(lseq))\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"TruncateWithErase\",\n\t\t\taction: func(s StreamStore, lseq uint64) {\n\t\t\t\tseq, _, err := s.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, seq, lseq)\n\t\t\t\tremoved, err := s.EraseMsg(lseq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_True(t, removed)\n\t\t\t\trequire_NoError(t, s.Truncate(lseq))\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"TruncateWithTombstone\",\n\t\t\taction: func(s StreamStore, lseq uint64) {\n\t\t\t\tseq, _, err := s.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, seq, lseq)\n\t\t\t\tif fs, ok := s.(*fileStore); ok {\n\t\t\t\t\tremoved, err := fs.removeMsg(lseq, false, false, true)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_True(t, removed)\n\t\t\t\t} else {\n\t\t\t\t\tremoved, err := s.RemoveMsg(lseq)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\trequire_True(t, removed)\n\t\t\t\t}\n\t\t\t\trequire_NoError(t, s.Truncate(lseq))\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"SkipMsg\",\n\t\t\taction: func(s StreamStore, lseq uint64) {\n\t\t\t\ts.SkipMsg(0)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttitle: \"SkipMsgs\",\n\t\t\taction: func(s StreamStore, lseq uint64) {\n\t\t\t\trequire_NoError(t, s.SkipMsgs(lseq, 1))\n\t\t\t},\n\t\t},\n\t}\n\tfor _, empty := range []bool{false, true} {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(fmt.Sprintf(\"Empty=%v/%s\", empty, test.title), func(t *testing.T) {\n\t\t\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}}\n\t\t\t\ttestAllStoreAllPermutations(t, true, cfg, func(t *testing.T, s StreamStore) {\n\t\t\t\t\tvar err error\n\t\t\t\t\tvar lseq uint64\n\t\t\t\t\tif !empty {\n\t\t\t\t\t\tlseq, _, err = s.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\t\trequire_Equal(t, lseq, 1)\n\t\t\t\t\t}\n\t\t\t\t\tlseq++\n\n\t\t\t\t\ttest.action(s, lseq)\n\n\t\t\t\t\t// Confirm state as baseline.\n\t\t\t\t\tbefore := s.State()\n\t\t\t\t\tif empty {\n\t\t\t\t\t\trequire_Equal(t, before.Msgs, 0)\n\t\t\t\t\t\trequire_Equal(t, before.FirstSeq, 2)\n\t\t\t\t\t\trequire_Equal(t, before.LastSeq, 1)\n\t\t\t\t\t} else {\n\t\t\t\t\t\trequire_Equal(t, before.Msgs, 1)\n\t\t\t\t\t\trequire_Equal(t, before.FirstSeq, 1)\n\t\t\t\t\t\trequire_Equal(t, before.LastSeq, 2)\n\t\t\t\t\t}\n\n\t\t\t\t\tvar fs *fileStore\n\t\t\t\t\tvar ok bool\n\t\t\t\t\tif fs, ok = s.(*fileStore); !ok {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tcfg.Storage = FileStorage\n\t\t\t\t\tfcfg := fs.fcfg\n\t\t\t\t\tcreated := time.Time{}\n\n\t\t\t\t\t// Restart should equal state.\n\t\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer fs.Stop()\n\n\t\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Stop and remove stream state file.\n\t\t\t\t\trequire_NoError(t, fs.Stop())\n\t\t\t\t\trequire_NoError(t, os.Remove(filepath.Join(fs.fcfg.StoreDir, msgDir, streamStreamStateFile)))\n\n\t\t\t\t\t// Recovering based on blocks should result in the same state.\n\t\t\t\t\tfs, err = newFileStoreWithCreated(fcfg, cfg, created, prf(&fcfg), nil)\n\t\t\t\t\trequire_NoError(t, err)\n\t\t\t\t\tdefer fs.Stop()\n\n\t\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Rebuilding state must also result in the same state.\n\t\t\t\t\tfs.rebuildState(nil)\n\t\t\t\t\tif state := fs.State(); !reflect.DeepEqual(state, before) {\n\t\t\t\t\t\tt.Fatalf(\"Expected state of:\\n%+v, got:\\n%+v\", before, state)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestStoreMsgLoadPrevMsgMulti(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo.*\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\t// Put 1k msgs in\n\t\t\tfor i := range 1000 {\n\t\t\t\tsubj := fmt.Sprintf(\"foo.%d\", i+1)\n\t\t\t\tfs.StoreMsg(subj, nil, []byte(\"ZZZ\"), 0)\n\t\t\t}\n\n\t\t\tvar sm StoreMsg\n\t\t\tvar count int\n\t\t\tvar state StreamState\n\t\t\tfs.FastState(&state)\n\n\t\t\tsl := gsl.NewSimpleSublist()\n\t\t\tsl.Insert(\"foo.5\", struct{}{})\n\t\t\tsl.Insert(\"foo.15\", struct{}{})\n\t\t\tsl.Insert(\"foo.105\", struct{}{})\n\n\t\t\tfor seq := state.LastSeq; seq > 5; seq-- {\n\t\t\t\tvar err error\n\t\t\t\t_, seq, err = fs.LoadPrevMsgMulti(sl, seq, &sm)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\trequire_Equal(t, sm.subj, fmt.Sprintf(\"foo.%d\", sm.seq))\n\t\t\t\tcount++\n\t\t\t}\n\n\t\t\t_, _, err := fs.LoadPrevMsgMulti(sl, 4, &sm)\n\t\t\trequire_Error(t, err, ErrStoreEOF)\n\t\t\trequire_Equal(t, count, 3)\n\t\t},\n\t)\n}\n\nfunc TestStoreDiscardNew(t *testing.T) {\n\ttest := func(t *testing.T, updateConfig func(cfg *StreamConfig), expectedErr error) {\n\t\tcfg := StreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}, Discard: DiscardNew}\n\t\tupdateConfig(&cfg)\n\t\ttestAllStoreAllPermutations(t, false, cfg, func(t *testing.T, fs StreamStore) {\n\t\t\tts := time.Now().UnixNano()\n\t\t\texpectedSeq := uint64(1)\n\t\t\trequireState := func() {\n\t\t\t\tt.Helper()\n\t\t\t\tstate := fs.State()\n\t\t\t\trequire_Equal(t, state.Msgs, 1)\n\t\t\t\trequire_Equal(t, state.FirstSeq, expectedSeq)\n\t\t\t\trequire_Equal(t, state.LastSeq, expectedSeq)\n\t\t\t}\n\n\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\n\t\t\terr = fs.StoreRawMsg(\"foo\", nil, nil, 0, ts, 0, true)\n\t\t\tif expectedErr == nil {\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\texpectedSeq++\n\t\t\t} else {\n\t\t\t\trequire_Equal(t, err, expectedErr)\n\t\t\t}\n\t\t\trequireState()\n\n\t\t\t// For a clustered stream DiscardNew should only be enforced by the stream leader.\n\t\t\t// Followers MUST always accept data that they've received from the leader,\n\t\t\t// otherwise we risk stream desync if some servers decide to reject.\n\t\t\terr = fs.StoreRawMsg(\"foo\", nil, nil, 0, ts, 0, false)\n\t\t\trequire_NoError(t, err)\n\t\t\texpectedSeq++\n\n\t\t\t// Since DiscardNew we must only add to the stream, and not act like\n\t\t\t// DiscardOld based on MaxMsgs/MaxBytes limits, unless MaxMsgsPer is set.\n\t\t\tif cfg.MaxMsgsPer > 0 {\n\t\t\t\trequireState()\n\t\t\t} else {\n\t\t\t\tstate := fs.State()\n\t\t\t\trequire_Equal(t, state.Msgs, 2)\n\t\t\t\trequire_Equal(t, state.FirstSeq, expectedSeq-1)\n\t\t\t\trequire_Equal(t, state.LastSeq, expectedSeq)\n\t\t\t}\n\t\t})\n\t}\n\n\tt.Run(\"MaxMsgs\", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxMsgs = 1 }, ErrMaxMsgs) })\n\tt.Run(\"MaxBytes\", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxBytes = 33 }, ErrMaxBytes) })\n\tt.Run(\"MaxMsgsPer\", func(t *testing.T) { test(t, func(cfg *StreamConfig) { cfg.MaxMsgsPer = 1 }, nil) })\n\tt.Run(\"MaxMsgsPer_DiscardNewPer\", func(t *testing.T) {\n\t\ttest(t, func(cfg *StreamConfig) {\n\t\t\tcfg.DiscardNewPer = true\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t}, ErrMaxMsgsPerSubject)\n\t})\n\tt.Run(\"MaxMsgsPer_MaxMsgs\", func(t *testing.T) {\n\t\ttest(t, func(cfg *StreamConfig) {\n\t\t\t// Without DiscardNewPerSubject we can replace the message with the new one if it fits (it will for this test).\n\t\t\tcfg.MaxMsgs = 1\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t}, nil)\n\t})\n\tt.Run(\"MaxMsgsPer_MaxBytes\", func(t *testing.T) {\n\t\ttest(t, func(cfg *StreamConfig) {\n\t\t\t// Without DiscardNewPerSubject we can replace the message with the new one if it fits (it will for this test).\n\t\t\tcfg.MaxBytes = 33\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t}, nil)\n\t})\n\tt.Run(\"MaxMsgsPer_MaxMsgs_DiscardNewPer\", func(t *testing.T) {\n\t\ttest(t, func(cfg *StreamConfig) {\n\t\t\tcfg.DiscardNewPer = true\n\t\t\tcfg.MaxMsgs = 1\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t}, ErrMaxMsgsPerSubject)\n\t})\n\tt.Run(\"MaxMsgsPer_MaxBytes_DiscardNewPer\", func(t *testing.T) {\n\t\ttest(t, func(cfg *StreamConfig) {\n\t\t\tcfg.DiscardNewPer = true\n\t\t\tcfg.MaxBytes = 33\n\t\t\tcfg.MaxMsgsPer = 1\n\t\t}, ErrMaxMsgsPerSubject)\n\t})\n}\n\nfunc TestStoreGetSeqFromTimeWithInteriorDeletesGap(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tvar start int64\n\t\t\tfor i := range 10 {\n\t\t\t\t_, ts, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif i == 1 {\n\t\t\t\t\tstart = ts\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Create a delete gap to prove a simple binary search between sequences\n\t\t\t// does not work, and deletes need to be accounted for. A simple binary search\n\t\t\t// will hit the deleted sequences and then return the last sequence.\n\t\t\tfor seq := uint64(4); seq <= 7; seq++ {\n\t\t\t\t_, err := fs.RemoveMsg(seq)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tts := time.Unix(0, start).UTC()\n\t\t\trequire_Equal(t, fs.GetSeqFromTime(ts), 2)\n\t\t},\n\t)\n}\n\nfunc TestStoreGetSeqFromTimeWithTrailingDeletes(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tvar start int64\n\t\t\tfor i := range 3 {\n\t\t\t\t_, ts, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t\tif i == 1 {\n\t\t\t\t\tstart = ts\n\t\t\t\t}\n\t\t\t}\n\t\t\t_, err := fs.RemoveMsg(3)\n\t\t\trequire_NoError(t, err)\n\t\t\tts := time.Unix(0, start).UTC()\n\t\t\trequire_Equal(t, fs.GetSeqFromTime(ts), 2)\n\t\t},\n\t)\n}\n\nfunc TestFileStoreMultiLastSeqsAndLoadLastMsgWithLazySubjectState(t *testing.T) {\n\ttestAllStoreAllPermutations(\n\t\tt, false,\n\t\tStreamConfig{Name: \"zzz\", Subjects: []string{\"foo\"}},\n\t\tfunc(t *testing.T, fs StreamStore) {\n\t\t\tfor range 3 {\n\t\t\t\t_, _, err := fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\t\trequire_NoError(t, err)\n\t\t\t}\n\t\t\tseqs, err := fs.MultiLastSeqs([]string{\"foo\"}, 0, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, len(seqs), 1)\n\t\t\trequire_Equal(t, seqs[0], 3)\n\n\t\t\t_, err = fs.RemoveMsg(3)\n\t\t\trequire_NoError(t, err)\n\t\t\tseqs, err = fs.MultiLastSeqs([]string{\"foo\"}, 0, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, len(seqs), 1)\n\t\t\trequire_Equal(t, seqs[0], 2)\n\n\t\t\t_, _, err = fs.StoreMsg(\"foo\", nil, nil, 0)\n\t\t\trequire_NoError(t, err)\n\t\t\tsm, err := fs.LoadLastMsg(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.seq, 4)\n\n\t\t\t_, err = fs.RemoveMsg(4)\n\t\t\trequire_NoError(t, err)\n\t\t\tsm, err = fs.LoadLastMsg(\"foo\", nil)\n\t\t\trequire_NoError(t, err)\n\t\t\trequire_Equal(t, sm.seq, 2)\n\t\t},\n\t)\n}\n" }, { "path": "server/stream.go", "content": "// Copyright 2019-2026 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"archive/tar\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"math/big\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/antithesishq/antithesis-sdk-go/assert\"\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// StreamConfigRequest is used to create or update a stream.\ntype StreamConfigRequest struct {\n\tStreamConfig\n\t// This is not part of the StreamConfig, because its scoped to request,\n\t// and not to the stream itself.\n\tPedantic bool `json:\"pedantic,omitempty\"`\n}\n\n// StreamConfig will determine the name, subjects and retention policy\n// for a given stream. If subjects is empty the name will be used.\ntype StreamConfig struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description,omitempty\"`\n\tSubjects []string `json:\"subjects,omitempty\"`\n\tRetention RetentionPolicy `json:\"retention\"`\n\tMaxConsumers int `json:\"max_consumers\"`\n\tMaxMsgs int64 `json:\"max_msgs\"`\n\tMaxBytes int64 `json:\"max_bytes\"`\n\tMaxAge time.Duration `json:\"max_age\"`\n\tMaxMsgsPer int64 `json:\"max_msgs_per_subject\"`\n\tMaxMsgSize int32 `json:\"max_msg_size,omitempty\"`\n\tDiscard DiscardPolicy `json:\"discard\"`\n\tStorage StorageType `json:\"storage\"`\n\tReplicas int `json:\"num_replicas\"`\n\tNoAck bool `json:\"no_ack,omitempty\"`\n\tDuplicates time.Duration `json:\"duplicate_window,omitempty\"`\n\tPlacement *Placement `json:\"placement,omitempty\"`\n\tMirror *StreamSource `json:\"mirror,omitempty\"`\n\tSources []*StreamSource `json:\"sources,omitempty\"`\n\tCompression StoreCompression `json:\"compression\"`\n\tFirstSeq uint64 `json:\"first_seq,omitempty\"`\n\n\t// Allow applying a subject transform to incoming messages before doing anything else\n\tSubjectTransform *SubjectTransformConfig `json:\"subject_transform,omitempty\"`\n\n\t// Allow republish of the message after being sequenced and stored.\n\tRePublish *RePublish `json:\"republish,omitempty\"`\n\n\t// Allow higher performance, direct access to get individual messages. E.g. KeyValue\n\tAllowDirect bool `json:\"allow_direct\"`\n\t// Allow higher performance and unified direct access for mirrors as well.\n\tMirrorDirect bool `json:\"mirror_direct\"`\n\n\t// Allow KV like semantics to also discard new on a per subject basis\n\tDiscardNewPer bool `json:\"discard_new_per_subject,omitempty\"`\n\n\t// Optional qualifiers. These can not be modified after set to true.\n\n\t// Sealed will seal a stream so no messages can get out or in.\n\tSealed bool `json:\"sealed\"`\n\t// DenyDelete will restrict the ability to delete messages.\n\tDenyDelete bool `json:\"deny_delete\"`\n\t// DenyPurge will restrict the ability to purge messages.\n\tDenyPurge bool `json:\"deny_purge\"`\n\t// AllowRollup allows messages to be placed into the system and purge\n\t// all older messages using a special msg header.\n\tAllowRollup bool `json:\"allow_rollup_hdrs\"`\n\n\t// The following defaults will apply to consumers when created against\n\t// this stream, unless overridden manually.\n\t// TODO(nat): Can/should we name these better?\n\tConsumerLimits StreamConsumerLimits `json:\"consumer_limits\"`\n\n\t// AllowMsgTTL allows header initiated per-message TTLs. If disabled,\n\t// then the `NATS-TTL` header will be ignored.\n\tAllowMsgTTL bool `json:\"allow_msg_ttl\"`\n\n\t// SubjectDeleteMarkerTTL sets the TTL of delete marker messages left behind by\n\t// subject delete markers.\n\tSubjectDeleteMarkerTTL time.Duration `json:\"subject_delete_marker_ttl,omitempty\"`\n\n\t// AllowMsgCounter allows a stream to use (only) counter CRDTs.\n\tAllowMsgCounter bool `json:\"allow_msg_counter,omitempty\"`\n\n\t// AllowAtomicPublish allows atomic batch publishing into the stream.\n\tAllowAtomicPublish bool `json:\"allow_atomic,omitempty\"`\n\n\t// AllowMsgSchedules allows the scheduling of messages.\n\tAllowMsgSchedules bool `json:\"allow_msg_schedules,omitempty\"`\n\n\t// PersistMode allows to opt-in to different persistence mode settings.\n\tPersistMode PersistModeType `json:\"persist_mode,omitempty\"`\n\n\t// AllowBatchPublish allows fast batch publishing into the stream.\n\tAllowBatchPublish bool `json:\"allow_batched,omitempty\"`\n\n\t// Metadata is additional metadata for the Stream.\n\tMetadata map[string]string `json:\"metadata,omitempty\"`\n}\n\n// clone performs a deep copy of the StreamConfig struct, returning a new clone with\n// all values copied.\nfunc (cfg *StreamConfig) clone() *StreamConfig {\n\tclone := *cfg\n\tif cfg.Placement != nil {\n\t\tplacement := *cfg.Placement\n\t\tclone.Placement = &placement\n\t}\n\tif cfg.Mirror != nil {\n\t\tmirror := *cfg.Mirror\n\t\tclone.Mirror = &mirror\n\t}\n\tif len(cfg.Sources) > 0 {\n\t\tclone.Sources = make([]*StreamSource, len(cfg.Sources))\n\t\tfor i, cfgSource := range cfg.Sources {\n\t\t\tsource := *cfgSource\n\t\t\tclone.Sources[i] = &source\n\t\t}\n\t}\n\tif cfg.SubjectTransform != nil {\n\t\ttransform := *cfg.SubjectTransform\n\t\tclone.SubjectTransform = &transform\n\t}\n\tif cfg.RePublish != nil {\n\t\trePublish := *cfg.RePublish\n\t\tclone.RePublish = &rePublish\n\t}\n\tif cfg.Metadata != nil {\n\t\tclone.Metadata = make(map[string]string, len(cfg.Metadata))\n\t\tfor k, v := range cfg.Metadata {\n\t\t\tclone.Metadata[k] = v\n\t\t}\n\t}\n\treturn &clone\n}\n\ntype StreamConsumerLimits struct {\n\tInactiveThreshold time.Duration `json:\"inactive_threshold,omitempty\"`\n\tMaxAckPending int `json:\"max_ack_pending,omitempty\"`\n}\n\n// SubjectTransformConfig is for applying a subject transform (to matching messages) before doing anything else when a new message is received\ntype SubjectTransformConfig struct {\n\tSource string `json:\"src\"`\n\tDestination string `json:\"dest\"`\n}\n\n// RePublish is for republishing messages once committed to a stream.\ntype RePublish struct {\n\tSource string `json:\"src,omitempty\"`\n\tDestination string `json:\"dest\"`\n\tHeadersOnly bool `json:\"headers_only,omitempty\"`\n}\n\n// PersistModeType determines what persistence mode the stream uses.\ntype PersistModeType int\n\nconst (\n\t// DefaultPersistMode specifies the default persist mode. Writes to the stream will immediately be flushed.\n\t// The publish acknowledgement will be sent after the persisting completes.\n\tDefaultPersistMode = PersistModeType(iota)\n\t// AsyncPersistMode specifies writes to the stream will be flushed asynchronously.\n\t// The publish acknowledgement may be sent before the persisting completes.\n\t// This means writes could be lost if they weren't flushed prior to a hard kill of the server.\n\tAsyncPersistMode\n)\n\nconst (\n\tdefaultPersistModeJSONString = `\"default\"`\n\tasyncPersistModeJSONString = `\"async\"`\n)\n\nvar (\n\tdefaultPersistModeJSONBytes = []byte(defaultPersistModeJSONString)\n\tasyncPersistModeJSONBytes = []byte(asyncPersistModeJSONString)\n)\n\nfunc (wc PersistModeType) String() string {\n\tswitch wc {\n\tcase DefaultPersistMode:\n\t\treturn \"Default\"\n\tcase AsyncPersistMode:\n\t\treturn \"Async\"\n\tdefault:\n\t\treturn \"Unknown Persist Mode Type\"\n\t}\n}\n\nfunc (wc PersistModeType) MarshalJSON() ([]byte, error) {\n\tswitch wc {\n\tcase DefaultPersistMode:\n\t\treturn defaultPersistModeJSONBytes, nil\n\tcase AsyncPersistMode:\n\t\treturn asyncPersistModeJSONBytes, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"can not marshal %v\", wc)\n\t}\n}\n\nfunc (wc *PersistModeType) UnmarshalJSON(data []byte) error {\n\tswitch string(data) {\n\tcase defaultPersistModeJSONString, `\"\"`:\n\t\t*wc = DefaultPersistMode\n\tcase asyncPersistModeJSONString:\n\t\t*wc = AsyncPersistMode\n\tdefault:\n\t\treturn fmt.Errorf(\"can not unmarshal %q\", data)\n\t}\n\treturn nil\n}\n\n// JSPubAckResponse is a formal response to a publish operation.\ntype JSPubAckResponse struct {\n\tError *ApiError `json:\"error,omitempty\"`\n\t*PubAck\n}\n\n// ToError checks if the response has a error and if it does converts it to an error\n// avoiding the pitfalls described by https://yourbasic.org/golang/gotcha-why-nil-error-not-equal-nil/\nfunc (r *JSPubAckResponse) ToError() error {\n\tif r.Error == nil {\n\t\treturn nil\n\t}\n\treturn r.Error\n}\n\n// PubAck is the detail you get back from a publish to a stream that was successful.\n// e.g. +OK {\"stream\": \"Orders\", \"seq\": 22}\ntype PubAck struct {\n\tStream string `json:\"stream\"`\n\tSequence uint64 `json:\"seq\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tDuplicate bool `json:\"duplicate,omitempty\"`\n\tValue string `json:\"val,omitempty\"`\n\tBatchId string `json:\"batch,omitempty\"`\n\tBatchSize int `json:\"count,omitempty\"`\n}\n\n// CounterValue is the body of a message when used as a counter.\n// e.g. {\"val\":\"123\"}\ntype CounterValue struct {\n\tValue string `json:\"val\"`\n}\n\n// CounterSources is the body of the Nats-Counter-Sources header.\n// e.g. {\"stream\":{\"subject\":\"123\"}}\ntype CounterSources map[string]map[string]string\n\n// BatchFlowAck is used for flow control when fast batch publishing into a stream.\n// This message is vital to handling acknowledgements and flow control.\n// These may technically be lost without the client receiving it. The client can retrieve\n// these by using the \"ping\" operation if it's expecting acks but not receiving any.\ntype BatchFlowAck struct {\n\t// Type: \"ack\"\n\tType string `json:\"type\"`\n\t// Sequence is the sequence of the message that triggered the ack.\n\t// If \"gap: fail\" this means the messages up to and including Sequence were persisted.\n\t// If \"gap: ok\" this means _some_ of the messages up to and including Sequence were persisted.\n\t// But there could have been gaps.\n\tSequence uint64 `json:\"seq\"`\n\t// Messages indicates acknowledgements will be sent every N messages.\n\tMessages uint16 `json:\"msgs\"`\n}\n\nfunc (ack BatchFlowAck) MarshalJSON() []byte {\n\ttype Alias BatchFlowAck\n\ta := Alias(ack)\n\ta.Type = \"ack\"\n\tbuf, _ := json.Marshal(a)\n\treturn buf\n}\n\n// BatchFlowGap is used for reporting gaps when fast batch publishing into a stream.\n// This message is purely informational and could technically be lost without the client receiving it.\ntype BatchFlowGap struct {\n\t// Type: \"gap\"\n\tType string `json:\"type\"`\n\t// ExpectedLastSequence is the sequence expected to be received next.\n\t// Messages starting from ExpectedLastSequence up to (but not including) CurrentSequence were lost.\n\tExpectedLastSequence uint64 `json:\"last_seq\"`\n\t// CurrentSequence is the sequence of the message that just came in and detected the gap.\n\tCurrentSequence uint64 `json:\"seq\"`\n}\n\nfunc (gap BatchFlowGap) MarshalJSON() []byte {\n\ttype Alias BatchFlowGap\n\ta := Alias(gap)\n\ta.Type = \"gap\"\n\tbuf, _ := json.Marshal(a)\n\treturn buf\n}\n\n// BatchFlowErr is used for reporting errors when fast batch publishing into a stream.\n// This message is purely informational and could technically be lost without the client receiving it.\ntype BatchFlowErr struct {\n\t// Type: \"err\"\n\tType string `json:\"type\"`\n\t// Sequence is the sequence of the message that triggered the error.\n\t// There are no (relative) guarantees whatsoever about whether the messages up to this sequence were persisted.\n\t// Such guarantees require the use of \"gap: fail\" and listening for BatchFlowAck and PubAck.\n\tSequence uint64 `json:\"seq\"`\n\t// Error is used to return the error for the Sequence.\n\tError *ApiError `json:\"error\"`\n}\n\nfunc (err BatchFlowErr) MarshalJSON() []byte {\n\ttype Alias BatchFlowErr\n\ta := Alias(err)\n\ta.Type = \"err\"\n\tbuf, _ := json.Marshal(a)\n\treturn buf\n}\n\n// StreamInfo shows config and current state for this stream.\ntype StreamInfo struct {\n\tConfig StreamConfig `json:\"config\"`\n\tCreated time.Time `json:\"created\"`\n\tState StreamState `json:\"state\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tCluster *ClusterInfo `json:\"cluster,omitempty\"`\n\tMirror *StreamSourceInfo `json:\"mirror,omitempty\"`\n\tSources []*StreamSourceInfo `json:\"sources,omitempty\"`\n\tAlternates []StreamAlternate `json:\"alternates,omitempty\"`\n\t// TimeStamp indicates when the info was gathered\n\tTimeStamp time.Time `json:\"ts\"`\n}\n\n// streamInfoClusterResponse is a response used in a cluster to communicate the stream info\n// back to the meta leader as part of a stream list request.\ntype streamInfoClusterResponse struct {\n\tStreamInfo\n\tOfflineReason string `json:\"offline_reason,omitempty\"` // Reporting when a stream is offline.\n}\n\ntype StreamAlternate struct {\n\tName string `json:\"name\"`\n\tDomain string `json:\"domain,omitempty\"`\n\tCluster string `json:\"cluster\"`\n}\n\n// ClusterInfo shows information about the underlying set of servers\n// that make up the stream or consumer.\ntype ClusterInfo struct {\n\tName string `json:\"name,omitempty\"`\n\tRaftGroup string `json:\"raft_group,omitempty\"`\n\tLeader string `json:\"leader,omitempty\"`\n\tLeaderSince *time.Time `json:\"leader_since,omitempty\"`\n\tSystemAcc bool `json:\"system_account,omitempty\"`\n\tTrafficAcc string `json:\"traffic_account,omitempty\"`\n\tReplicas []*PeerInfo `json:\"replicas,omitempty\"`\n}\n\n// PeerInfo shows information about all the peers in the cluster that\n// are supporting the stream or consumer.\ntype PeerInfo struct {\n\tName string `json:\"name\"` // Name is the unique name for the peer\n\tCurrent bool `json:\"current\"` // Current indicates if it was seen recently and fully caught up\n\tOffline bool `json:\"offline,omitempty\"` // Offline indicates if it has not been seen recently\n\tActive time.Duration `json:\"active\"` // Active is the timestamp it was last active\n\tLag uint64 `json:\"lag,omitempty\"` // Lag is how many operations behind it is\n\tPeer string `json:\"peer\"` // Peer is the unique ID for the peer\n\t// For migrations.\n\tcluster string\n}\n\n// StreamSourceInfo shows information about an upstream stream source.\ntype StreamSourceInfo struct {\n\tName string `json:\"name\"`\n\tExternal *ExternalStream `json:\"external,omitempty\"`\n\tLag uint64 `json:\"lag\"`\n\tActive time.Duration `json:\"active\"`\n\tError *ApiError `json:\"error,omitempty\"`\n\tFilterSubject string `json:\"filter_subject,omitempty\"`\n\tSubjectTransforms []SubjectTransformConfig `json:\"subject_transforms,omitempty\"`\n}\n\n// StreamSource dictates how streams can source from other streams.\ntype StreamSource struct {\n\tName string `json:\"name\"`\n\tOptStartSeq uint64 `json:\"opt_start_seq,omitempty\"`\n\tOptStartTime *time.Time `json:\"opt_start_time,omitempty\"`\n\tFilterSubject string `json:\"filter_subject,omitempty\"`\n\tSubjectTransforms []SubjectTransformConfig `json:\"subject_transforms,omitempty\"`\n\tExternal *ExternalStream `json:\"external,omitempty\"`\n\n\t// Internal\n\tiname string // For indexing when stream names are the same for multiple sources.\n}\n\n// ExternalStream allows you to qualify access to a stream source in another account or domain.\ntype ExternalStream struct {\n\tApiPrefix string `json:\"api\"`\n\tDeliverPrefix string `json:\"deliver\"`\n}\n\n// Will return the domain for this external stream.\nfunc (ext *ExternalStream) Domain() string {\n\tif ext == nil || ext.ApiPrefix == _EMPTY_ {\n\t\treturn _EMPTY_\n\t}\n\treturn tokenAt(ext.ApiPrefix, 2)\n}\n\n// For managing stream ingest.\nconst (\n\tstreamDefaultMaxQueueMsgs = 100_000\n\tstreamDefaultMaxQueueBytes = 128 * 1024 * 1024\n)\n\n// For managing stream batches.\nconst (\n\tstreamDefaultMaxBatchTimeout = 10 * time.Second\n\t// Atomic batches.\n\tstreamDefaultMaxAtomicBatchInflightPerStream = 50\n\tstreamDefaultMaxAtomicBatchInflightTotal = 1000\n\tstreamDefaultMaxAtomicBatchSize = 1000\n\t// Fast batches.\n\tstreamDefaultMaxFastBatchInflightPerStream = 1000\n\tstreamDefaultMaxFastBatchInflightTotal = 50_000\n)\n\nvar (\n\tstreamMaxBatchTimeout = streamDefaultMaxBatchTimeout\n\t// Atomic batches.\n\tstreamMaxAtomicBatchInflightPerStream = streamDefaultMaxAtomicBatchInflightPerStream\n\tstreamMaxAtomicBatchInflightTotal = streamDefaultMaxAtomicBatchInflightTotal\n\tstreamMaxAtomicBatchSize = streamDefaultMaxAtomicBatchSize\n\t// Fast batches.\n\tstreamMaxFastBatchInflightPerStream = streamDefaultMaxFastBatchInflightPerStream\n\tstreamMaxFastBatchInflightTotal = streamDefaultMaxFastBatchInflightTotal\n)\n\n// Stream is a jetstream stream of messages. When we receive a message internally destined\n// for a Stream we will direct link from the client to this structure.\ntype stream struct {\n\tmu sync.RWMutex // Read/write lock for the stream.\n\tjs *jetStream // The internal *jetStream for the account.\n\tjsa *jsAccount // The JetStream account-level information.\n\tacc *Account // The account this stream is defined in.\n\tsrv *Server // The server we are running in.\n\tclient *client // The internal JetStream client.\n\tsysc *client // The internal JetStream system client.\n\n\t// The current last subscription ID for the subscriptions through `client`.\n\t// Those subscriptions are for the subjects filters being listened to and captured by the stream.\n\tsid atomic.Uint64\n\n\tpubAck []byte // The template (prefix) to generate the pubAck responses for this stream quickly.\n\toutq *jsOutQ // Queue of *jsPubMsg for sending messages.\n\tmsgs *ipQueue[*inMsg] // Intra-process queue for the ingress of messages.\n\tgets *ipQueue[*directGetReq] // Intra-process queue for the direct get requests.\n\tstore StreamStore // The storage for this stream.\n\tackq *ipQueue[uint64] // Intra-process queue for acks.\n\tlseq uint64 // The sequence number of the last message stored in the stream.\n\tlmsgId string // The de-duplication message ID of the last message stored in the stream.\n\tconsumers map[string]*consumer // The consumers for this stream.\n\tnumFilter int // The number of filtered consumers.\n\tcfg StreamConfig // The stream's config.\n\tcfgMu sync.RWMutex // Config mutex used to solve some races with consumer code\n\tcreated time.Time // Time the stream was created.\n\tstype StorageType // The storage type.\n\ttier string // The tier is the number of replicas for the stream (e.g. \"R1\" or \"R3\").\n\tddMu sync.Mutex // Lock for dedupe state.\n\tddmap map[string]*ddentry // The dedupe map.\n\tddarr []*ddentry // The dedupe array.\n\tddindex int // The dedupe index.\n\tddtmr *time.Timer // The dedupe timer.\n\tqch chan struct{} // The quit channel.\n\tmqch chan struct{} // The monitor's quit channel.\n\tactive bool // Indicates that there are active internal subscriptions (for the subject filters)\n\t// and/or mirror/sources consumers are scheduled to be established or already started.\n\tclosed atomic.Bool // Set to true when stop() is called on the stream.\n\tcisrun atomic.Bool // Indicates one checkInterestState is already running.\n\n\t// Mirror\n\tmirror *sourceInfo\n\tmirrorConsumerSetup *time.Timer\n\n\t// Sources\n\tsources map[string]*sourceInfo\n\tsourceSetupSchedules map[string]*time.Timer\n\tsourcesConsumerSetup *time.Timer\n\tsmsgs *ipQueue[*inMsg] // Intra-process queue for all incoming sourced messages.\n\n\t// Indicates we have direct consumers.\n\tdirects int\n\n\t// For input subject transform.\n\titr *subjectTransform\n\n\t// For republishing.\n\ttr *subjectTransform\n\n\t// For processing consumers without main stream lock.\n\tclsMu sync.RWMutex\n\tcList []*consumer // Consumer list.\n\tsch chan struct{} // Channel to signal consumers.\n\tsigq *ipQueue[*cMsg] // Intra-process queue for the messages to signal to the consumers.\n\tcsl *gsl.GenericSublist[*consumer] // Consumer subscription list.\n\n\t// Leader will store seq/msgTrace in clustering mode. Used in applyStreamEntries\n\t// to know if trace event should be sent after processing.\n\tmt map[uint64]*msgTrace\n\n\t// For non limits policy streams when they process an ack before the actual msg.\n\t// Can happen in stretch clusters, multi-cloud, or during catchup for a restarted server.\n\tpreAcks map[uint64]map[*consumer]struct{}\n\n\t// TODO(dlc) - Hide everything below behind two pointers.\n\t// Clustered mode.\n\tsa *streamAssignment // What the meta controller uses to assign streams to peers.\n\tnode RaftNode // Our RAFT node for the stream's group.\n\tcatchup atomic.Bool // Used to signal we are in catchup mode.\n\tcatchups map[string]uint64 // The number of messages that need to be caught per peer.\n\tsyncSub *subscription // Internal subscription for sync messages (on \"$JSC.SYNC\").\n\tinfoSub *subscription // Internal subscription for stream info requests.\n\tclMu sync.Mutex // The mutex for clseq and clfs.\n\tclseq uint64 // The current last seq being proposed to the NRG layer.\n\tclfs uint64 // The count (offset) of the number of failed NRG sequences used to compute clseq.\n\tlqsent time.Time // The time at which the last lost quorum advisory was sent. Used to rate limit.\n\tuch chan struct{} // The channel to signal updates to the monitor routine.\n\tinMonitor bool // True if the monitor routine has been started.\n\twerr error // If a write error was encountered, and if so what error.\n\n\tinflight map[string]*inflightSubjectRunningTotal // Inflight message sizes per subject.\n\tclusteredCounterTotal map[string]*msgCounterRunningTotal // Inflight counter totals.\n\texpectedPerSubjectSequence map[uint64]string // Inflight 'expected per subject' subjects per clseq.\n\texpectedPerSubjectInProcess map[string]struct{} // Current 'expected per subject' subjects in process.\n\n\t// Direct get subscription.\n\tdirectLeaderSub *subscription\n\tdirectSub *subscription\n\tlastBySub *subscription\n\tmirrorDirectSub *subscription // Mirrors only.\n\tmirrorLastBySub *subscription // Mirrors only.\n\n\tmonitorWg sync.WaitGroup // Wait group for the monitor routine.\n\n\t// If standalone/single-server, the offline reason needs to be stored directly in the stream.\n\t// Otherwise, if clustered it will be part of the stream assignment.\n\tofflineReason string\n\n\tbatches *batching // Inflight batches prior to committing them.\n\tbatchApply *batchApply // State to check for batch completeness before applying it.\n}\n\n// inflightSubjectRunningTotal stores a running total of inflight messages for a specific subject.\ntype inflightSubjectRunningTotal struct {\n\tbytes uint64 // Running total of inflight bytes for inflight messages.\n\tops uint64 // Inflight operations, i.e. inflight messages for this subject. If this reaches zero, we can remove the running total.\n}\n\n// msgCounterRunningTotal stores a running total and a number of inflight\n// but not yet applied clustered proposals/operations for this counter.\ntype msgCounterRunningTotal struct {\n\ttotal *big.Int // Running total.\n\tsources CounterSources // Last seen counter sources.\n\tops uint64 // Inflight operations. If this reaches zero, we can remove the running total.\n}\n\ntype sourceInfo struct {\n\tname string // The name of the stream being sourced.\n\tiname string // The unique index name of this particular source.\n\tcname string // The name of the current consumer for this source.\n\tsub *subscription // The subscription to the consumer.\n\n\tmsgs *ipQueue[*inMsg] // Intra-process queue for incoming messages.\n\tsseq uint64 // Last stream message sequence number seen from the source.\n\tdseq uint64 // Last delivery (i.e. consumer's) sequence number.\n\tlag uint64 // 0 or number of messages pending (as last reported by the consumer) - 1.\n\terr *ApiError // The API error that caused the last consumer setup to fail.\n\tfails int // The number of times trying to setup the consumer failed.\n\tlast atomic.Int64 // Time the consumer was created or of last message it received.\n\tlreq time.Time // The last time setupMirrorConsumer/setupSourceConsumer was called.\n\tqch chan struct{} // Quit channel.\n\tsip bool // Setup in progress.\n\twg sync.WaitGroup // WaitGroup for the consumer's go routine.\n\tsf string // The subject filter.\n\tsfs []string // The subject filters.\n\ttrs []*subjectTransform // The subject transforms.\n}\n\n// For mirrors and direct get\nconst (\n\tdgetGroup = sysGroup\n\tdgetCaughtUpThresh = 10\n)\n\n// Headers for published messages.\nconst (\n\tJSMsgId = \"Nats-Msg-Id\"\n\tJSExpectedStream = \"Nats-Expected-Stream\"\n\tJSExpectedLastSeq = \"Nats-Expected-Last-Sequence\"\n\tJSExpectedLastSubjSeq = \"Nats-Expected-Last-Subject-Sequence\"\n\tJSExpectedLastSubjSeqSubj = \"Nats-Expected-Last-Subject-Sequence-Subject\"\n\tJSExpectedLastMsgId = \"Nats-Expected-Last-Msg-Id\"\n\tJSStreamSource = \"Nats-Stream-Source\"\n\tJSLastConsumerSeq = \"Nats-Last-Consumer\"\n\tJSLastStreamSeq = \"Nats-Last-Stream\"\n\tJSConsumerStalled = \"Nats-Consumer-Stalled\"\n\tJSMsgRollup = \"Nats-Rollup\"\n\tJSMsgSize = \"Nats-Msg-Size\"\n\tJSResponseType = \"Nats-Response-Type\"\n\tJSMessageTTL = \"Nats-TTL\"\n\tJSMarkerReason = \"Nats-Marker-Reason\"\n\tJSMessageIncr = \"Nats-Incr\"\n\tJSMessageCounterSources = \"Nats-Counter-Sources\"\n\tJSBatchId = \"Nats-Batch-Id\"\n\tJSBatchSeq = \"Nats-Batch-Sequence\"\n\tJSBatchCommit = \"Nats-Batch-Commit\"\n\tJSSchedulePattern = \"Nats-Schedule\"\n\tJSScheduleTimeZone = \"Nats-Schedule-Time-Zone\"\n\tJSScheduleTTL = \"Nats-Schedule-TTL\"\n\tJSScheduleTarget = \"Nats-Schedule-Target\"\n\tJSScheduleSource = \"Nats-Schedule-Source\"\n)\n\n// Headers for published KV messages.\nvar (\n\tKVOperation = \"KV-Operation\"\n\tKVOperationValuePurge = []byte(\"PURGE\")\n)\n\n// Headers for scheduled messages.\nconst (\n\tJSScheduler = \"Nats-Scheduler\"\n\tJSScheduleNext = \"Nats-Schedule-Next\"\n\tJSScheduleNextPurge = \"purge\" // If it's a non-repeating/delayed message, the schedule is purged.\n)\n\n// Headers for republished messages and direct get responses.\nconst (\n\tJSStream = \"Nats-Stream\"\n\tJSSequence = \"Nats-Sequence\"\n\tJSTimeStamp = \"Nats-Time-Stamp\"\n\tJSSubject = \"Nats-Subject\"\n\tJSLastSequence = \"Nats-Last-Sequence\" // The sequence number of the precedent message either republished or in a batch of responses. 0 in the first message.\n\tJSNumPending = \"Nats-Num-Pending\" // Number of messages pending in the multi/batched get response\n\tJSUpToSequence = \"Nats-UpTo-Sequence\"\n)\n\n// Rollups, can be subject only or all messages.\nconst (\n\tJSMsgRollupSubject = \"sub\"\n\tJSMsgRollupAll = \"all\"\n)\n\n// Applied limits in the Nats-Applied-Limit header.\nconst (\n\tJSMarkerReasonMaxAge = \"MaxAge\"\n\tJSMarkerReasonPurge = \"Purge\"\n\tJSMarkerReasonRemove = \"Remove\"\n)\n\nconst (\n\tjsCreateResponse = \"create\"\n)\n\n// Dedupe entry\ntype ddentry struct {\n\tid string // The unique message ID provided by the client.\n\tseq uint64 // The sequence number of the message.\n\tts int64 // The timestamp of the message.\n}\n\n// Replicas Range\nconst StreamMaxReplicas = 5\n\n// AddStream adds a stream for the given account.\nfunc (a *Account) addStream(config *StreamConfig) (*stream, error) {\n\treturn a.addStreamWithAssignment(config, nil, nil, false, false)\n}\n\n// recoverStream recovers a stream from disk for the given account.\nfunc (a *Account) recoverStream(config *StreamConfig) (*stream, error) {\n\treturn a.addStreamWithAssignment(config, nil, nil, false, true)\n}\n\n// AddStreamWithStore adds a stream for the given account with custome store config options.\nfunc (a *Account) addStreamWithStore(config *StreamConfig, fsConfig *FileStoreConfig) (*stream, error) {\n\treturn a.addStreamWithAssignment(config, fsConfig, nil, false, false)\n}\n\nfunc (a *Account) addStreamPedantic(config *StreamConfig, pedantic bool) (*stream, error) {\n\treturn a.addStreamWithAssignment(config, nil, nil, pedantic, false)\n}\n\nfunc (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileStoreConfig, sa *streamAssignment, pedantic, recovering bool) (*stream, error) {\n\ts, jsa, err := a.checkForJetStream()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// If we do not have the stream currently assigned to us in cluster mode we will proceed but warn.\n\t// This can happen on startup with restored state where on meta replay we still do not have\n\t// the assignment. Running in single server mode this always returns true.\n\tif !jsa.streamAssigned(config.Name) {\n\t\ts.Debugf(\"Stream '%s > %s' does not seem to be assigned to this server\", a.Name, config.Name)\n\t}\n\n\t// Sensible defaults.\n\tccfg, apiErr := s.checkStreamCfg(config, a, pedantic)\n\tif apiErr != nil {\n\t\treturn nil, apiErr\n\t}\n\tcfg := &ccfg\n\n\tsingleServerMode := !s.JetStreamIsClustered() && s.standAloneMode()\n\tif singleServerMode && cfg.Replicas > 1 {\n\t\treturn nil, ApiErrors[JSStreamReplicasNotSupportedErr]\n\t}\n\n\t// Make sure we are ok when these are done in parallel.\n\t// We used to call Add(1) in the \"else\" clause of the \"if loaded\"\n\t// statement. This caused a data race because it was possible\n\t// that one go routine stores (with count==0) and another routine\n\t// gets \"loaded==true\" and calls wg.Wait() while the other routine\n\t// then calls wg.Add(1). It also could mean that two routines execute\n\t// the rest of the code concurrently.\n\tswg := &sync.WaitGroup{}\n\tswg.Add(1)\n\tv, loaded := jsa.inflight.LoadOrStore(cfg.Name, swg)\n\twg := v.(*sync.WaitGroup)\n\tif loaded {\n\t\twg.Wait()\n\t\t// This waitgroup is \"thrown away\" (since there was an existing one).\n\t\tswg.Done()\n\t} else {\n\t\tdefer func() {\n\t\t\tjsa.inflight.Delete(cfg.Name)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// Note that isClustered will be false during recovery, even if we're part of a cluster. It shouldn't be used then.\n\tjs, isClustered := jsa.jetStreamAndClustered()\n\tjsa.mu.Lock()\n\tif mset, ok := jsa.streams[cfg.Name]; ok {\n\t\tjsa.mu.Unlock()\n\t\t// Check to see if configs are same.\n\t\tocfg := mset.config()\n\n\t\t// set the index name on cfg since it would not contain a value for iname while the return from mset.config() does to ensure the DeepEqual works\n\t\tfor _, s := range cfg.Sources {\n\t\t\ts.setIndexName()\n\t\t}\n\n\t\t// Hold lock, because we'll be reading from and writing to a shared object.\n\t\tjs.mu.Lock()\n\t\tcopyStreamMetadata(cfg, &ocfg)\n\t\tdeepEqual := reflect.DeepEqual(cfg, &ocfg)\n\t\tjs.mu.Unlock()\n\t\tif deepEqual {\n\t\t\tif sa != nil {\n\t\t\t\tmset.setStreamAssignment(sa)\n\t\t\t}\n\t\t\treturn mset, nil\n\t\t} else {\n\t\t\treturn nil, ApiErrors[JSStreamNameExistErr]\n\t\t}\n\t}\n\tjsa.usageMu.RLock()\n\tselected, tier, hasTier := jsa.selectLimits(cfg.Replicas)\n\tjsa.usageMu.RUnlock()\n\n\tif !hasTier {\n\t\tjsa.mu.Unlock()\n\t\treturn nil, NewJSNoLimitsError()\n\t}\n\n\t// Skip if we're recovering.\n\tif !recovering {\n\t\treserved := int64(0)\n\t\tif !isClustered {\n\t\t\treserved = jsa.tieredReservation(tier, cfg)\n\t\t}\n\t\tjsa.mu.Unlock()\n\t\tjs.mu.RLock()\n\t\tif isClustered {\n\t\t\t_, reserved = js.tieredStreamAndReservationCount(a.Name, tier, cfg)\n\t\t}\n\t\tif err := js.checkAllLimits(&selected, cfg, reserved, 0); err != nil {\n\t\t\tjs.mu.RUnlock()\n\t\t\treturn nil, err\n\t\t}\n\t\tjs.mu.RUnlock()\n\t\tjsa.mu.Lock()\n\t}\n\n\t// If mirror, check if the transforms (if any) are valid.\n\tif cfg.Mirror != nil {\n\t\tif len(cfg.Mirror.SubjectTransforms) == 0 {\n\t\t\tif cfg.Mirror.FilterSubject != _EMPTY_ && !IsValidSubject(cfg.Mirror.FilterSubject) {\n\t\t\t\tjsa.mu.Unlock()\n\t\t\t\treturn nil, fmt.Errorf(\"subject filter '%s' for the mirror %w\", cfg.Mirror.FilterSubject, ErrBadSubject)\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, st := range cfg.Mirror.SubjectTransforms {\n\t\t\t\tif st.Source != _EMPTY_ && !IsValidSubject(st.Source) {\n\t\t\t\t\tjsa.mu.Unlock()\n\t\t\t\t\treturn nil, fmt.Errorf(\"invalid subject transform source '%s' for the mirror: %w\", st.Source, ErrBadSubject)\n\t\t\t\t}\n\t\t\t\t// check the transform, if any, is valid\n\t\t\t\tif st.Destination != _EMPTY_ {\n\t\t\t\t\tif _, err = NewSubjectTransform(st.Source, st.Destination); err != nil {\n\t\t\t\t\t\tjsa.mu.Unlock()\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"subject transform from '%s' to '%s' for the mirror: %w\", st.Source, st.Destination, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Setup our internal indexed names here for sources and check if the transforms (if any) are valid.\n\tfor _, ssi := range cfg.Sources {\n\t\tif len(ssi.SubjectTransforms) == 0 {\n\t\t\t// check the filter, if any, is valid\n\t\t\tif ssi.FilterSubject != _EMPTY_ && !IsValidSubject(ssi.FilterSubject) {\n\t\t\t\tjsa.mu.Unlock()\n\t\t\t\treturn nil, fmt.Errorf(\"subject filter '%s' for the source: %w\", ssi.FilterSubject, ErrBadSubject)\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, st := range ssi.SubjectTransforms {\n\t\t\t\tif st.Source != _EMPTY_ && !IsValidSubject(st.Source) {\n\t\t\t\t\tjsa.mu.Unlock()\n\t\t\t\t\treturn nil, fmt.Errorf(\"subject filter '%s' for the source: %w\", st.Source, ErrBadSubject)\n\t\t\t\t}\n\t\t\t\t// check the transform, if any, is valid\n\t\t\t\tif st.Destination != _EMPTY_ {\n\t\t\t\t\tif _, err = NewSubjectTransform(st.Source, st.Destination); err != nil {\n\t\t\t\t\t\tjsa.mu.Unlock()\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"subject transform from '%s' to '%s' for the source: %w\", st.Source, st.Destination, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check for overlapping subjects with other streams.\n\t// These are not allowed for now.\n\tif jsa.subjectsOverlap(cfg.Subjects, nil) {\n\t\tjsa.mu.Unlock()\n\t\treturn nil, NewJSStreamSubjectOverlapError()\n\t}\n\n\t// Setup the internal clients.\n\tc := s.createInternalJetStreamClient()\n\tic := s.createInternalJetStreamClient()\n\n\t// Work out the stream ingest limits.\n\tmlen := s.opts.StreamMaxBufferedMsgs\n\tmsz := uint64(s.opts.StreamMaxBufferedSize)\n\tif mlen == 0 {\n\t\tmlen = streamDefaultMaxQueueMsgs\n\t}\n\tif msz == 0 {\n\t\tmsz = streamDefaultMaxQueueBytes\n\t}\n\n\tqpfx := fmt.Sprintf(\"[ACC:%s] stream '%s' \", a.Name, config.Name)\n\tmset := &stream{\n\t\tacc: a,\n\t\tjsa: jsa,\n\t\tcfg: *cfg,\n\t\tjs: js,\n\t\tsrv: s,\n\t\tclient: c,\n\t\tsysc: ic,\n\t\ttier: tier,\n\t\tstype: cfg.Storage,\n\t\tconsumers: make(map[string]*consumer),\n\t\tmsgs: newIPQueue[*inMsg](s, qpfx+\"messages\",\n\t\t\tipqSizeCalculation(func(msg *inMsg) uint64 {\n\t\t\t\treturn uint64(len(msg.hdr) + len(msg.msg) + len(msg.rply) + len(msg.subj))\n\t\t\t}),\n\t\t\tipqLimitByLen[*inMsg](mlen),\n\t\t\tipqLimitBySize[*inMsg](msz),\n\t\t),\n\t\tgets: newIPQueue[*directGetReq](s, qpfx+\"direct gets\"),\n\t\tqch: make(chan struct{}),\n\t\tmqch: make(chan struct{}),\n\t\tuch: make(chan struct{}, 4),\n\t\tsch: make(chan struct{}, 1),\n\t\tcreated: time.Now().UTC(),\n\t}\n\n\t// Add created timestamp used for the store, must match that of the stream assignment if it exists.\n\tif sa != nil {\n\t\t// The following assignment does not require mutex\n\t\t// protection: sa.Created is immutable.\n\t\tmset.created = sa.Created\n\t}\n\n\t// Start our signaling routine to process consumers.\n\tmset.sigq = newIPQueue[*cMsg](s, qpfx+\"obs\") // of *cMsg\n\tgo mset.signalConsumersLoop()\n\n\t// For no-ack consumers when we are interest retention.\n\tif cfg.Retention != LimitsPolicy {\n\t\tmset.ackq = newIPQueue[uint64](s, qpfx+\"acks\")\n\t}\n\n\t// Check for input subject transform\n\tif cfg.SubjectTransform != nil {\n\t\ttr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination)\n\t\tif err != nil {\n\t\t\tjsa.mu.Unlock()\n\t\t\treturn nil, fmt.Errorf(\"stream subject transform from '%s' to '%s': %w\", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err)\n\t\t}\n\t\tmset.itr = tr\n\t}\n\n\t// Check for RePublish.\n\tif cfg.RePublish != nil {\n\t\ttr, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination)\n\t\tif err != nil {\n\t\t\tjsa.mu.Unlock()\n\t\t\treturn nil, fmt.Errorf(\"stream republish transform from '%s' to '%s': %w\", cfg.RePublish.Source, cfg.RePublish.Destination, err)\n\t\t}\n\t\t// Assign our transform for republishing.\n\t\tmset.tr = tr\n\t}\n\tstoreDir := filepath.Join(jsa.storeDir, streamsDir, cfg.Name)\n\tjsa.mu.Unlock()\n\n\t// Bind to the user account.\n\tc.registerWithAccount(a)\n\t// Bind to the system account.\n\tic.registerWithAccount(s.SystemAccount())\n\n\t// Create the appropriate storage\n\tfsCfg := fsConfig\n\tif fsCfg == nil {\n\t\tfsCfg = &FileStoreConfig{}\n\t\t// If we are file based and not explicitly configured\n\t\t// we may be able to auto-tune based on max msgs or bytes.\n\t\tif cfg.Storage == FileStorage {\n\t\t\tmset.autoTuneFileStorageBlockSize(fsCfg)\n\t\t}\n\t}\n\tfsCfg.StoreDir = storeDir\n\t// Grab configured sync interval.\n\tfsCfg.SyncInterval = s.getOpts().SyncInterval\n\tfsCfg.SyncAlways = s.getOpts().SyncAlways\n\tfsCfg.Compression = config.Compression\n\t// Async flushing is only allowed if the stream has a sync log backing it.\n\tfsCfg.AsyncFlush = !fsCfg.SyncAlways && config.Replicas > 1\n\n\t// Async persist mode opts in to async flushing,\n\t// sync always would also be disabled if it was configured.\n\tif config.PersistMode == AsyncPersistMode {\n\t\tfsCfg.SyncAlways = false\n\t\tfsCfg.AsyncFlush = true\n\t}\n\tif err := mset.setupStore(fsCfg); err != nil {\n\t\tmset.stop(true, false)\n\t\treturn nil, NewJSStreamStoreFailedError(err)\n\t}\n\n\t// Create our pubAck template here. Better than json marshal each time on success.\n\tif domain := s.getOpts().JetStreamDomain; domain != _EMPTY_ {\n\t\tmset.pubAck = fmt.Appendf(nil, \"{%q:%q,%q:%q,%q:\", \"stream\", cfg.Name, \"domain\", domain, \"seq\")\n\t} else {\n\t\tmset.pubAck = fmt.Appendf(nil, \"{%q:%q,%q:\", \"stream\", cfg.Name, \"seq\")\n\t}\n\tend := len(mset.pubAck)\n\tmset.pubAck = mset.pubAck[:end:end]\n\n\t// Set our known last sequence.\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\n\t// Possible race with consumer.setLeader during recovery.\n\tmset.mu.Lock()\n\tmset.lseq = state.LastSeq\n\n\t// Ensure dedupe state is loaded.\n\tmset.ddMu.Lock()\n\tmset.rebuildDedupe()\n\tmset.ddMu.Unlock()\n\tmset.mu.Unlock()\n\n\t// Set our stream assignment if in clustered mode.\n\treserveResources := true\n\tif sa != nil {\n\t\tmset.setStreamAssignment(sa)\n\n\t\t// If the stream is resetting we must not double-account resources, they were already accounted for.\n\t\tjs.mu.Lock()\n\t\tif sa.resetting {\n\t\t\treserveResources, sa.resetting = false, false\n\t\t}\n\t\tjs.mu.Unlock()\n\t}\n\n\t// Setup our internal send go routine.\n\tmset.setupSendCapabilities()\n\n\t// Reserve resources if MaxBytes present.\n\tif reserveResources {\n\t\tmset.js.reserveStreamResources(&mset.cfg)\n\t}\n\n\t// Call directly to set leader if not in clustered mode.\n\t// This can be called though before we actually setup clustering, so check both.\n\tif singleServerMode {\n\t\tif err := mset.setLeader(true); err != nil {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// This is always true in single server mode.\n\tif mset.IsLeader() {\n\t\t// Send advisory.\n\t\tvar suppress bool\n\t\tif !s.standAloneMode() && sa == nil {\n\t\t\tif cfg.Replicas > 1 {\n\t\t\t\tsuppress = true\n\t\t\t}\n\t\t} else if sa != nil {\n\t\t\tsuppress = sa.responded\n\t\t}\n\t\tif !suppress {\n\t\t\tmset.sendCreateAdvisory()\n\t\t}\n\t}\n\n\t// Register with our account last.\n\tjsa.mu.Lock()\n\tjsa.streams[cfg.Name] = mset\n\tjsa.mu.Unlock()\n\n\treturn mset, nil\n}\n\n// Composes the index name. Contains the stream name, subject filter, and transform destination\n// when the stream is external we will use the api prefix as part of the index name\n// (as the same stream name could be used in multiple JS domains)\nfunc (ssi *StreamSource) composeIName() string {\n\tvar iName = ssi.Name\n\n\tif ssi.External != nil {\n\t\tiName = iName + \":\" + getHash(ssi.External.ApiPrefix)\n\t}\n\n\tsource := ssi.FilterSubject\n\tdestination := fwcs\n\n\tif len(ssi.SubjectTransforms) == 0 {\n\t\t// normalize filter and destination in case they are empty\n\t\tif source == _EMPTY_ {\n\t\t\tsource = fwcs\n\t\t}\n\t\tif destination == _EMPTY_ {\n\t\t\tdestination = fwcs\n\t\t}\n\t} else {\n\t\tvar sources, destinations []string\n\n\t\tfor _, tr := range ssi.SubjectTransforms {\n\t\t\ttrsrc, trdest := tr.Source, tr.Destination\n\t\t\tif trsrc == _EMPTY_ {\n\t\t\t\ttrsrc = fwcs\n\t\t\t}\n\t\t\tif trdest == _EMPTY_ {\n\t\t\t\ttrdest = fwcs\n\t\t\t}\n\t\t\tsources = append(sources, trsrc)\n\t\t\tdestinations = append(destinations, trdest)\n\t\t}\n\t\tsource = strings.Join(sources, \"\\f\")\n\t\tdestination = strings.Join(destinations, \"\\f\")\n\t}\n\n\treturn strings.Join([]string{iName, source, destination}, \" \")\n}\n\n// Sets the index name.\nfunc (ssi *StreamSource) setIndexName() {\n\tssi.iname = ssi.composeIName()\n}\n\nfunc (mset *stream) streamAssignment() *streamAssignment {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.sa\n}\n\nfunc (mset *stream) setStreamAssignment(sa *streamAssignment) {\n\tvar node RaftNode\n\tvar peers []string\n\n\tmset.mu.RLock()\n\tjs := mset.js\n\tmset.mu.RUnlock()\n\n\tif js != nil {\n\t\tjs.mu.RLock()\n\t\tif sa.Group != nil {\n\t\t\tnode = sa.Group.node\n\t\t\tpeers = sa.Group.Peers\n\t\t}\n\t\tjs.mu.RUnlock()\n\t}\n\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\n\tmset.sa = sa\n\tif sa == nil {\n\t\treturn\n\t}\n\n\t// Set our node.\n\tmset.node = node\n\tif mset.node != nil {\n\t\tmset.node.UpdateKnownPeers(peers)\n\t}\n\n\t// Setup our info sub here as well for all stream members. This is now by design.\n\tif mset.infoSub == nil {\n\t\tisubj := fmt.Sprintf(clusterStreamInfoT, mset.jsa.acc(), mset.cfg.Name)\n\t\t// Note below the way we subscribe here is so that we can send requests to ourselves.\n\t\tmset.infoSub, _ = mset.srv.systemSubscribe(isubj, _EMPTY_, false, mset.sysc, mset.handleClusterStreamInfoRequest)\n\t}\n\n\t// Trigger update chan.\n\tselect {\n\tcase mset.uch <- struct{}{}:\n\tdefault:\n\t}\n}\n\nfunc (mset *stream) monitorQuitC() <-chan struct{} {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\t// Recreate if a prior monitor routine was stopped.\n\tif mset.mqch == nil {\n\t\tmset.mqch = make(chan struct{})\n\t}\n\treturn mset.mqch\n}\n\n// signalMonitorQuit signals to exit the monitor loop. If there's no Raft node,\n// this will be the only way to stop the monitor goroutine.\nfunc (mset *stream) signalMonitorQuit() {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tif mset.mqch != nil {\n\t\tclose(mset.mqch)\n\t\tmset.mqch = nil\n\t}\n}\n\nfunc (mset *stream) updateC() <-chan struct{} {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.uch\n}\n\n// IsLeader will return if we are the current leader.\nfunc (mset *stream) IsLeader() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.isLeader()\n}\n\n// Lock should be held.\nfunc (mset *stream) isLeader() bool {\n\tif mset.isClustered() {\n\t\treturn mset.node.Leader()\n\t}\n\treturn true\n}\n\n// isLeaderNodeState should NOT be used normally, use isLeader instead.\n// Returns whether the node thinks it is the leader, regardless of whether applies are up-to-date yet\n// (unlike isLeader, which requires applies to be caught up).\n// May be used to respond to clients after a leader change, when applying entries from a former leader.\n// Lock should be held.\nfunc (mset *stream) isLeaderNodeState() bool {\n\tif mset.isClustered() {\n\t\treturn mset.node.State() == Leader\n\t}\n\treturn true\n}\n\n// TODO(dlc) - Check to see if we can accept being the leader or we should step down.\nfunc (mset *stream) setLeader(isLeader bool) error {\n\tmset.mu.Lock()\n\t// If we are here we have a change in leader status.\n\tif isLeader {\n\t\t// Make sure we are listening for sync requests.\n\t\t// TODO(dlc) - Original design was that all in sync members of the group would do DQ.\n\t\tif mset.isClustered() {\n\t\t\tmset.startClusterSubs()\n\t\t}\n\n\t\t// Setup subscriptions if we were not already the leader.\n\t\tif err := mset.subscribeToStream(); err != nil {\n\t\t\tif mset.isClustered() {\n\t\t\t\t// Stepdown since we have an error.\n\t\t\t\tmset.node.StepDown()\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\n\t\t// Reset any inflight fast batches. We were likely a follower before and need\n\t\t// to send an ack to the publishers so they know we're still there.\n\t\tif mset.batches != nil {\n\t\t\tmset.batches.mu.Lock()\n\t\t\tfor batchId, b := range mset.batches.fast {\n\t\t\t\tmset.batches.fastBatchReset(mset, batchId, b)\n\t\t\t}\n\t\t\tmset.batches.mu.Unlock()\n\t\t}\n\t} else {\n\t\t// cancel timer to create the source consumers if not fired yet\n\t\tif mset.sourcesConsumerSetup != nil {\n\t\t\tmset.sourcesConsumerSetup.Stop()\n\t\t\tmset.sourcesConsumerSetup = nil\n\t\t} else {\n\t\t\t// Stop any source consumers\n\t\t\tmset.stopSourceConsumers()\n\t\t}\n\n\t\t// Stop responding to sync requests.\n\t\tmset.stopClusterSubs()\n\t\t// Unsubscribe from direct stream.\n\t\tmset.unsubscribeToStream(false, false)\n\t\t// Clear catchup state\n\t\tmset.clearAllCatchupPeers()\n\t}\n\tmset.store.ResetState()\n\tmset.mu.Unlock()\n\n\t// If we are interest based make sure to check consumers.\n\t// This is to make sure we process any outstanding acks.\n\tmset.checkInterestState()\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) startClusterSubs() {\n\tif mset.syncSub == nil {\n\t\tmset.syncSub, _ = mset.srv.systemSubscribe(mset.sa.Sync, _EMPTY_, false, mset.sysc, mset.handleClusterSyncRequest)\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) stopClusterSubs() {\n\tif mset.syncSub != nil {\n\t\tmset.srv.sysUnsubscribe(mset.syncSub)\n\t\tmset.syncSub = nil\n\t}\n}\n\n// account gets the account for this stream.\nfunc (mset *stream) account() *Account {\n\treturn mset.accountLocked(true)\n}\n\nfunc (mset *stream) accountLocked(needLock bool) *Account {\n\tif needLock {\n\t\tmset.mu.RLock()\n\t}\n\tjsa := mset.jsa\n\tif needLock {\n\t\tmset.mu.RUnlock()\n\t}\n\tif jsa == nil {\n\t\treturn nil\n\t}\n\treturn jsa.acc()\n}\n\n// Helper to determine the max msg size for this stream if file based.\nfunc (mset *stream) maxMsgSize() uint64 {\n\tmaxMsgSize := mset.cfg.MaxMsgSize\n\tif maxMsgSize <= 0 {\n\t\t// Pull from the account.\n\t\tif mset.jsa != nil {\n\t\t\tif acc := mset.jsa.acc(); acc != nil {\n\t\t\t\tacc.mu.RLock()\n\t\t\t\tmaxMsgSize = acc.mpay\n\t\t\t\tacc.mu.RUnlock()\n\t\t\t}\n\t\t}\n\t\t// If all else fails use default.\n\t\tif maxMsgSize <= 0 {\n\t\t\tmaxMsgSize = MAX_PAYLOAD_SIZE\n\t\t}\n\t}\n\t// Now determine an estimation for the subjects etc.\n\tmaxSubject := -1\n\tfor _, subj := range mset.cfg.Subjects {\n\t\tif subjectIsLiteral(subj) {\n\t\t\tif len(subj) > maxSubject {\n\t\t\t\tmaxSubject = len(subj)\n\t\t\t}\n\t\t}\n\t}\n\tif maxSubject < 0 {\n\t\tconst defaultMaxSubject = 256\n\t\tmaxSubject = defaultMaxSubject\n\t}\n\t// filestore will add in estimates for record headers, etc.\n\treturn fileStoreMsgSizeEstimate(maxSubject, int(maxMsgSize))\n}\n\n// If we are file based and the file storage config was not explicitly set\n// we can autotune block sizes to better match. Our target will be to store 125%\n// of the theoretical limit. We will round up to nearest 100 bytes as well.\nfunc (mset *stream) autoTuneFileStorageBlockSize(fsCfg *FileStoreConfig) {\n\tvar totalEstSize uint64\n\n\t// MaxBytes will take precedence for now.\n\tif mset.cfg.MaxBytes > 0 {\n\t\ttotalEstSize = uint64(mset.cfg.MaxBytes)\n\t} else if mset.cfg.MaxMsgs > 0 {\n\t\t// Determine max message size to estimate.\n\t\ttotalEstSize = mset.maxMsgSize() * uint64(mset.cfg.MaxMsgs)\n\t} else if mset.cfg.MaxMsgsPer > 0 {\n\t\tfsCfg.BlockSize = uint64(defaultKVBlockSize)\n\t\treturn\n\t} else {\n\t\t// If nothing set will let underlying filestore determine blkSize.\n\t\treturn\n\t}\n\n\tblkSize := (totalEstSize / 4) + 1 // (25% overhead)\n\t// Round up to nearest 100\n\tif m := blkSize % 100; m != 0 {\n\t\tblkSize += 100 - m\n\t}\n\tif blkSize <= FileStoreMinBlkSize {\n\t\tblkSize = FileStoreMinBlkSize\n\t} else if blkSize >= FileStoreMaxBlkSize {\n\t\tblkSize = FileStoreMaxBlkSize\n\t} else {\n\t\tblkSize = defaultMediumBlockSize\n\t}\n\tfsCfg.BlockSize = uint64(blkSize)\n}\n\n// rebuildDedupe will rebuild any dedupe structures needed after recovery of a stream.\n// Will be called lazily to avoid penalizing startup times.\n// TODO(dlc) - Might be good to know if this should be checked at all for streams with no\n// headers and msgId in them. Would need signaling from the storage layer.\n// mset.mu and mset.ddMu locks should be held.\nfunc (mset *stream) rebuildDedupe() {\n\tduplicates := mset.cfg.Duplicates\n\tif duplicates <= 0 {\n\t\treturn\n\t}\n\n\t// We have some messages. Lookup starting sequence by duplicate time window.\n\tsseq := mset.store.GetSeqFromTime(time.Now().Add(-duplicates))\n\tif sseq == 0 {\n\t\treturn\n\t}\n\n\tvar smv StoreMsg\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\n\tfor seq := sseq; seq <= state.LastSeq; seq++ {\n\t\tsm, err := mset.store.LoadMsg(seq, &smv)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tvar msgId string\n\t\tif len(sm.hdr) > 0 {\n\t\t\tif msgId = getMsgId(sm.hdr); msgId != _EMPTY_ {\n\t\t\t\tmset.storeMsgIdLocked(&ddentry{msgId, sm.seq, sm.ts})\n\t\t\t}\n\t\t}\n\t\tif seq == state.LastSeq {\n\t\t\tmset.lmsgId = msgId\n\t\t}\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) lastSeqAndCLFS() (uint64, uint64) {\n\treturn mset.lseq, mset.getCLFS()\n}\n\nfunc (mset *stream) getCLFS() uint64 {\n\tif mset == nil {\n\t\treturn 0\n\t}\n\tmset.clMu.Lock()\n\tdefer mset.clMu.Unlock()\n\treturn mset.clfs\n}\n\nfunc (mset *stream) setCLFS(clfs uint64) {\n\tmset.clMu.Lock()\n\tmset.clfs = clfs\n\tmset.clMu.Unlock()\n}\n\nfunc (mset *stream) lastSeq() uint64 {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.lseq\n}\n\n// Set last seq.\n// Write lock should be held.\nfunc (mset *stream) setLastSeq(lseq uint64) {\n\tmset.lseq = lseq\n}\n\nfunc (mset *stream) sendCreateAdvisory() {\n\tmset.mu.RLock()\n\tname := mset.cfg.Name\n\toutq := mset.outq\n\tsrv := mset.srv\n\tmset.mu.RUnlock()\n\n\tif outq == nil {\n\t\treturn\n\t}\n\n\t// finally send an event that this stream was created\n\tm := JSStreamActionAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamActionAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: name,\n\t\tAction: CreateEvent,\n\t\tDomain: srv.getOpts().JetStreamDomain,\n\t}\n\n\tj, err := json.Marshal(m)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tsubj := JSAdvisoryStreamCreatedPre + \".\" + name\n\toutq.sendMsg(subj, j)\n}\n\nfunc (mset *stream) sendDeleteAdvisoryLocked() {\n\tif mset.outq == nil {\n\t\treturn\n\t}\n\n\tm := JSStreamActionAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamActionAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: mset.cfg.Name,\n\t\tAction: DeleteEvent,\n\t\tDomain: mset.srv.getOpts().JetStreamDomain,\n\t}\n\n\tj, err := json.Marshal(m)\n\tif err == nil {\n\t\tsubj := JSAdvisoryStreamDeletedPre + \".\" + mset.cfg.Name\n\t\tmset.outq.sendMsg(subj, j)\n\t}\n}\n\nfunc (mset *stream) sendUpdateAdvisoryLocked() {\n\tif mset.outq == nil {\n\t\treturn\n\t}\n\n\tm := JSStreamActionAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamActionAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: mset.cfg.Name,\n\t\tAction: ModifyEvent,\n\t\tDomain: mset.srv.getOpts().JetStreamDomain,\n\t}\n\n\tj, err := json.Marshal(m)\n\tif err == nil {\n\t\tsubj := JSAdvisoryStreamUpdatedPre + \".\" + mset.cfg.Name\n\t\tmset.outq.sendMsg(subj, j)\n\t}\n}\n\nfunc (mset *stream) sendStreamBatchAbandonedAdvisory(batchId string, reason BatchAbandonReason) {\n\tif mset == nil {\n\t\treturn\n\t}\n\ts := mset.srv\n\tstream, acc := mset.name(), mset.account()\n\tsubj := JSAdvisoryStreamBatchAbandonedPre + \".\" + stream\n\tadv := &JSStreamBatchAbandonedAdvisory{\n\t\tTypedEvent: TypedEvent{\n\t\t\tType: JSStreamBatchAbandonedAdvisoryType,\n\t\t\tID: nuid.Next(),\n\t\t\tTime: time.Now().UTC(),\n\t\t},\n\t\tStream: stream,\n\t\tDomain: s.getOpts().JetStreamDomain,\n\t\tBatchId: batchId,\n\t\tReason: reason,\n\t}\n\n\t// Send to the user's account if not the system account.\n\tif acc != s.SystemAccount() {\n\t\ts.publishAdvisory(acc, subj, adv)\n\t}\n\t// Now do system level one. Place account info in adv, and nil account means system.\n\tadv.Account = acc.GetName()\n\ts.publishAdvisory(nil, subj, adv)\n}\n\n// Created returns created time.\nfunc (mset *stream) createdTime() time.Time {\n\tmset.mu.RLock()\n\tcreated := mset.created\n\tmset.mu.RUnlock()\n\treturn created\n}\n\n// Internal to allow creation time to be restored.\nfunc (mset *stream) setCreatedTime(created time.Time) {\n\tmset.mu.Lock()\n\tmset.created = created\n\tmset.mu.Unlock()\n}\n\n// subjectsOverlap to see if these subjects overlap with existing subjects.\n// Use only for non-clustered JetStream\n// RLock minimum should be held.\nfunc (jsa *jsAccount) subjectsOverlap(subjects []string, self *stream) bool {\n\tfor _, mset := range jsa.streams {\n\t\tif self != nil && mset == self {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, subj := range mset.cfg.Subjects {\n\t\t\tfor _, tsubj := range subjects {\n\t\t\t\tif SubjectsCollide(tsubj, subj) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// StreamDefaultDuplicatesWindow default duplicates window.\nconst StreamDefaultDuplicatesWindow = 2 * time.Minute\n\nfunc (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic bool) (StreamConfig, *ApiError) {\n\tlim := &s.getOpts().JetStreamLimits\n\n\tif config == nil {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration invalid\"))\n\t}\n\tif !isValidName(config.Name) {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream name is required and can not contain '.', '*', '>'\"))\n\t}\n\tif len(config.Name) > JSMaxNameLen {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream name is too long, maximum allowed is %d\", JSMaxNameLen))\n\t}\n\tif len(config.Description) > JSMaxDescriptionLen {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream description is too long, maximum allowed is %d\", JSMaxDescriptionLen))\n\t}\n\n\tvar metadataLen int\n\tfor k, v := range config.Metadata {\n\t\tmetadataLen += len(k) + len(v)\n\t}\n\tif metadataLen > JSMaxMetadataLen {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream metadata exceeds maximum size of %d bytes\", JSMaxMetadataLen))\n\t}\n\n\tcfg := *config\n\n\tif _, err := cfg.Retention.MarshalJSON(); err != nil {\n\t\treturn cfg, NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid retention\"))\n\t}\n\tif _, err := cfg.Discard.MarshalJSON(); err != nil {\n\t\treturn cfg, NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid discard policy\"))\n\t}\n\tif _, err := cfg.Compression.MarshalJSON(); err != nil {\n\t\treturn cfg, NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid compression\"))\n\t}\n\n\t// Make file the default.\n\tif cfg.Storage == 0 {\n\t\tcfg.Storage = FileStorage\n\t}\n\tif _, err := cfg.Storage.MarshalJSON(); err != nil {\n\t\treturn cfg, NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid storage type\"))\n\t}\n\n\tif cfg.Replicas == 0 {\n\t\tcfg.Replicas = 1\n\t}\n\tif cfg.Replicas > StreamMaxReplicas {\n\t\treturn cfg, NewJSStreamInvalidConfigError(fmt.Errorf(\"maximum replicas is %d\", StreamMaxReplicas))\n\t}\n\tif cfg.Replicas < 0 {\n\t\treturn cfg, NewJSReplicasCountCannotBeNegativeError()\n\t}\n\tif cfg.MaxMsgs == 0 || cfg.MaxMsgs < -1 {\n\t\tif pedantic && cfg.MaxMsgs < -1 {\n\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"max_msgs must be set to -1\"))\n\t\t}\n\t\tcfg.MaxMsgs = -1\n\t}\n\tif cfg.MaxMsgsPer == 0 || cfg.MaxMsgsPer < -1 {\n\t\tif pedantic && cfg.MaxMsgsPer < -1 {\n\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"max_msgs_per_subject must be set to -1\"))\n\t\t}\n\t\tcfg.MaxMsgsPer = -1\n\t}\n\tif cfg.MaxBytes == 0 || cfg.MaxBytes < -1 {\n\t\tif pedantic && cfg.MaxBytes < -1 {\n\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"max_bytes must be set to -1\"))\n\t\t}\n\t\tcfg.MaxBytes = -1\n\t}\n\tif cfg.MaxMsgSize == 0 || cfg.MaxMsgSize < -1 {\n\t\tif pedantic && cfg.MaxMsgSize < -1 {\n\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"max_msg_size must be set to -1\"))\n\t\t}\n\t\tcfg.MaxMsgSize = -1\n\t}\n\tif cfg.MaxConsumers == 0 || cfg.MaxConsumers < -1 {\n\t\tif pedantic && cfg.MaxConsumers < -1 {\n\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"max_consumers must be set to -1\"))\n\t\t}\n\t\tcfg.MaxConsumers = -1\n\t}\n\tif cfg.MaxAge < 0 {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"max age can not be negative\"))\n\t}\n\tif cfg.MaxAge != 0 && cfg.MaxAge < 100*time.Millisecond {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"max age needs to be >= 100ms\"))\n\t}\n\n\tif cfg.Duplicates == 0 && cfg.Mirror == nil && len(cfg.Sources) == 0 {\n\t\tmaxWindow := StreamDefaultDuplicatesWindow\n\t\tif lim.Duplicates > 0 && maxWindow > lim.Duplicates {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"duplicate window limits are higher than current limits\"))\n\t\t\t}\n\t\t\tmaxWindow = lim.Duplicates\n\t\t}\n\t\tif cfg.MaxAge != 0 && cfg.MaxAge < maxWindow {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"duplicate window cannot be bigger than max age\"))\n\t\t\t}\n\t\t\tcfg.Duplicates = cfg.MaxAge\n\t\t} else {\n\t\t\tcfg.Duplicates = maxWindow\n\t\t}\n\t}\n\tif cfg.Duplicates < 0 {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"duplicates window can not be negative\"))\n\t}\n\t// Check that duplicates is not larger then age if set.\n\tif cfg.MaxAge != 0 && cfg.Duplicates > cfg.MaxAge {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"duplicates window can not be larger then max age\"))\n\t}\n\tif lim.Duplicates > 0 && cfg.Duplicates > lim.Duplicates {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"duplicates window can not be larger then server limit of %v\",\n\t\t\tlim.Duplicates.String()))\n\t}\n\tif cfg.Duplicates > 0 && cfg.Duplicates < 100*time.Millisecond {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"duplicates window needs to be >= 100ms\"))\n\t}\n\n\tif cfg.DenyPurge && cfg.AllowRollup {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"roll-ups require the purge permission\"))\n\t}\n\n\t// Counter is not compatible with some settings.\n\tif cfg.AllowMsgCounter {\n\t\tif cfg.Discard == DiscardNew {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream cannot use discard new\"))\n\t\t}\n\t\tif cfg.AllowMsgTTL {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream cannot use message TTLs\"))\n\t\t}\n\t\tif cfg.Retention != LimitsPolicy {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"counter stream can only use limits retention\"))\n\t\t}\n\t}\n\n\t// Check for new discard new per subject, we require the discard policy to also be new.\n\tif cfg.DiscardNewPer {\n\t\tif cfg.Discard != DiscardNew {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"discard new per subject requires discard new policy to be set\"))\n\t\t}\n\t\tif cfg.MaxMsgsPer <= 0 {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"discard new per subject requires max msgs per subject > 0\"))\n\t\t}\n\t}\n\n\tif cfg.SubjectDeleteMarkerTTL > 0 {\n\t\tif cfg.SubjectDeleteMarkerTTL < time.Second {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject delete marker TTL must be at least 1 second\"))\n\t\t}\n\t\tif !cfg.AllowMsgTTL {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject delete marker cannot be set if message TTLs are disabled\"))\n\t\t\t}\n\t\t\tcfg.AllowMsgTTL = true\n\t\t}\n\t\tif !cfg.AllowRollup {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject delete marker cannot be set if roll-ups are disabled\"))\n\t\t\t}\n\t\t\tcfg.AllowRollup, cfg.DenyPurge = true, false\n\t\t}\n\t} else if cfg.SubjectDeleteMarkerTTL < 0 {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject delete marker TTL must not be negative\"))\n\t}\n\n\tif cfg.AllowMsgSchedules {\n\t\tif !cfg.AllowRollup {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"message scheduling cannot be set if roll-ups are disabled\"))\n\t\t\t}\n\t\t\tcfg.AllowRollup, cfg.DenyPurge = true, false\n\t\t}\n\t}\n\n\tif cfg.PersistMode == AsyncPersistMode {\n\t\tif cfg.Storage != FileStorage {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is only supported on file storage\"))\n\t\t}\n\t\tif cfg.Replicas > 1 {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is not supported on replicated streams\"))\n\t\t}\n\t\tif cfg.AllowAtomicPublish {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"async persist mode is not supported with atomic batch publish\"))\n\t\t}\n\t}\n\n\tgetStream := func(streamName string) (bool, StreamConfig) {\n\t\tvar exists bool\n\t\tvar cfg StreamConfig\n\t\tif s.JetStreamIsClustered() {\n\t\t\tif js, _ := s.getJetStreamCluster(); js != nil {\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tif sa := js.streamAssignment(acc.Name, streamName); sa != nil {\n\t\t\t\t\tcfg = *sa.Config.clone()\n\t\t\t\t\texists = true\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t}\n\t\t} else if mset, err := acc.lookupStream(streamName); err == nil {\n\t\t\tcfg = mset.cfg\n\t\t\texists = true\n\t\t}\n\t\treturn exists, cfg\n\t}\n\n\thasStream := func(streamName string) (bool, int32, []string) {\n\t\texists, cfg := getStream(streamName)\n\t\treturn exists, cfg.MaxMsgSize, cfg.Subjects\n\t}\n\n\tvar streamSubs []string\n\tvar deliveryPrefixes []string\n\tvar apiPrefixes []string\n\n\t// Do some pre-checking for mirror config to avoid cycles in clustered mode.\n\tif cfg.Mirror != nil {\n\t\tif cfg.FirstSeq > 0 {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithFirstSeqError()\n\t\t}\n\t\tif len(cfg.Subjects) > 0 {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithSubjectsError()\n\t\t}\n\t\tif len(cfg.Sources) > 0 {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithSourcesError()\n\t\t}\n\t\tif cfg.AllowMsgCounter {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithCountersError()\n\t\t}\n\t\tif cfg.AllowAtomicPublish {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithAtomicPublishError()\n\t\t}\n\t\tif cfg.AllowBatchPublish {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithBatchPublishError()\n\t\t}\n\t\tif cfg.AllowMsgSchedules {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithMsgSchedulesError()\n\t\t}\n\t\tif cfg.Mirror.FilterSubject != _EMPTY_ && len(cfg.Mirror.SubjectTransforms) != 0 {\n\t\t\treturn StreamConfig{}, NewJSMirrorMultipleFiltersNotAllowedError()\n\t\t}\n\t\tif cfg.SubjectDeleteMarkerTTL > 0 {\n\t\t\t// Delete markers cannot be configured on a mirror as it would result in new\n\t\t\t// tombstones which would use up sequence numbers, diverging from the origin\n\t\t\t// stream.\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject delete markers forbidden on mirrors\"))\n\t\t}\n\t\t// Check subject filters overlap.\n\t\tfor outer, tr := range cfg.Mirror.SubjectTransforms {\n\t\t\tif tr.Source != _EMPTY_ && !IsValidSubject(tr.Source) {\n\t\t\t\treturn StreamConfig{}, NewJSMirrorInvalidSubjectFilterError(fmt.Errorf(\"%w %s\", ErrBadSubject, tr.Source))\n\t\t\t}\n\n\t\t\terr := ValidateMapping(tr.Source, tr.Destination)\n\t\t\tif err != nil {\n\t\t\t\treturn StreamConfig{}, NewJSMirrorInvalidTransformDestinationError(err)\n\t\t\t}\n\n\t\t\tfor inner, innertr := range cfg.Mirror.SubjectTransforms {\n\t\t\t\tif inner != outer && SubjectsCollide(tr.Source, innertr.Source) {\n\t\t\t\t\treturn StreamConfig{}, NewJSMirrorOverlappingSubjectFiltersError()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Do not perform checks if External is provided, as it could lead to\n\t\t// checking against itself (if sourced stream name is the same on different JetStream)\n\t\tif cfg.Mirror.External == nil {\n\t\t\tif !isValidName(cfg.Mirror.Name) {\n\t\t\t\treturn StreamConfig{}, NewJSMirrorInvalidStreamNameError()\n\t\t\t}\n\t\t\t// We do not require other stream to exist anymore, but if we can see it check payloads.\n\t\t\texists, maxMsgSize, subs := hasStream(cfg.Mirror.Name)\n\t\t\tif len(subs) > 0 {\n\t\t\t\tstreamSubs = append(streamSubs, subs...)\n\t\t\t}\n\t\t\tif exists {\n\t\t\t\tif cfg.MaxMsgSize > 0 && maxMsgSize > 0 && cfg.MaxMsgSize < maxMsgSize {\n\t\t\t\t\treturn StreamConfig{}, NewJSMirrorMaxMessageSizeTooBigError()\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Determine if we are inheriting direct gets.\n\t\t\tif exists, ocfg := getStream(cfg.Mirror.Name); exists {\n\t\t\t\tif pedantic && cfg.MirrorDirect != ocfg.AllowDirect {\n\t\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"origin stream has direct get set, mirror has it disabled\"))\n\t\t\t\t}\n\t\t\t\tcfg.MirrorDirect = ocfg.AllowDirect\n\t\t\t} else if js := s.getJetStream(); js != nil && js.isClustered() {\n\t\t\t\t// Could not find it here. If we are clustered we can look it up.\n\t\t\t\tjs.mu.RLock()\n\t\t\t\tif cc := js.cluster; cc != nil {\n\t\t\t\t\tif as := cc.streams[acc.Name]; as != nil {\n\t\t\t\t\t\tif sa := as[cfg.Mirror.Name]; sa != nil {\n\t\t\t\t\t\t\tif pedantic && cfg.MirrorDirect != sa.Config.AllowDirect {\n\t\t\t\t\t\t\t\tjs.mu.RUnlock()\n\t\t\t\t\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"origin stream has direct get set, mirror has it disabled\"))\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcfg.MirrorDirect = sa.Config.AllowDirect\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tjs.mu.RUnlock()\n\t\t\t}\n\t\t} else {\n\t\t\tif cfg.Mirror.External.DeliverPrefix != _EMPTY_ {\n\t\t\t\tdeliveryPrefixes = append(deliveryPrefixes, cfg.Mirror.External.DeliverPrefix)\n\t\t\t}\n\n\t\t\tif cfg.Mirror.External.ApiPrefix != _EMPTY_ {\n\t\t\t\tapiPrefixes = append(apiPrefixes, cfg.Mirror.External.ApiPrefix)\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(cfg.Sources) > 0 {\n\t\tif cfg.AllowMsgSchedules {\n\t\t\treturn StreamConfig{}, NewJSSourceWithMsgSchedulesError()\n\t\t}\n\t}\n\n\t// check sources for duplicates\n\tvar iNames = make(map[string]struct{})\n\tfor _, src := range cfg.Sources {\n\t\tif src == nil || !isValidName(src.Name) {\n\t\t\treturn StreamConfig{}, NewJSSourceInvalidStreamNameError()\n\t\t}\n\t\tif _, ok := iNames[src.composeIName()]; !ok {\n\t\t\tiNames[src.composeIName()] = struct{}{}\n\t\t} else {\n\t\t\treturn StreamConfig{}, NewJSSourceDuplicateDetectedError()\n\t\t}\n\n\t\tif src.FilterSubject != _EMPTY_ && len(src.SubjectTransforms) != 0 {\n\t\t\treturn StreamConfig{}, NewJSSourceMultipleFiltersNotAllowedError()\n\t\t}\n\n\t\tfor _, tr := range src.SubjectTransforms {\n\t\t\tif tr.Source != _EMPTY_ && !IsValidSubject(tr.Source) {\n\t\t\t\treturn StreamConfig{}, NewJSSourceInvalidSubjectFilterError(fmt.Errorf(\"%w %s\", ErrBadSubject, tr.Source))\n\t\t\t}\n\t\t\terr := ValidateMapping(tr.Source, tr.Destination)\n\t\t\tif err != nil {\n\t\t\t\treturn StreamConfig{}, NewJSSourceInvalidTransformDestinationError(err)\n\t\t\t}\n\t\t}\n\n\t\t// Check subject filters overlap.\n\t\tfor outer, tr := range src.SubjectTransforms {\n\t\t\tfor inner, innertr := range src.SubjectTransforms {\n\t\t\t\tif inner != outer && subjectIsSubsetMatch(tr.Source, innertr.Source) {\n\t\t\t\t\treturn StreamConfig{}, NewJSSourceOverlappingSubjectFiltersError()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Do not perform checks if External is provided, as it could lead to\n\t\t// checking against itself (if sourced stream name is the same on different JetStream)\n\t\tif src.External == nil {\n\t\t\texists, maxMsgSize, subs := hasStream(src.Name)\n\t\t\tif len(subs) > 0 {\n\t\t\t\tstreamSubs = append(streamSubs, subs...)\n\t\t\t}\n\t\t\tif exists {\n\t\t\t\tif cfg.MaxMsgSize > 0 && maxMsgSize > 0 && cfg.MaxMsgSize < maxMsgSize {\n\t\t\t\t\treturn StreamConfig{}, NewJSSourceMaxMessageSizeTooBigError()\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t} else {\n\t\t\tif src.External.DeliverPrefix != _EMPTY_ {\n\t\t\t\tdeliveryPrefixes = append(deliveryPrefixes, src.External.DeliverPrefix)\n\t\t\t}\n\t\t\tif src.External.ApiPrefix != _EMPTY_ {\n\t\t\t\tapiPrefixes = append(apiPrefixes, src.External.ApiPrefix)\n\t\t\t}\n\t\t}\n\t}\n\n\t// check prefix overlap with subjects\n\tfor _, pfx := range deliveryPrefixes {\n\t\tif !IsValidPublishSubject(pfx) {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidExternalDeliverySubjError(pfx)\n\t\t}\n\t\tfor _, sub := range streamSubs {\n\t\t\tif SubjectsCollide(sub, fmt.Sprintf(\"%s.%s\", pfx, sub)) {\n\t\t\t\treturn StreamConfig{}, NewJSStreamExternalDelPrefixOverlapsError(pfx, sub)\n\t\t\t}\n\t\t}\n\t}\n\t// check if api prefixes overlap\n\tfor _, apiPfx := range apiPrefixes {\n\t\tif !IsValidPublishSubject(apiPfx) {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(\n\t\t\t\tfmt.Errorf(\"stream external api prefix %q must be a valid subject without wildcards\", apiPfx))\n\t\t}\n\t\tif SubjectsCollide(apiPfx, JSApiPrefix) {\n\t\t\treturn StreamConfig{}, NewJSStreamExternalApiOverlapError(apiPfx, JSApiPrefix)\n\t\t}\n\t}\n\n\t// cycle check for source cycle\n\ttoVisit := []*StreamConfig{&cfg}\n\tvisited := make(map[string]struct{})\n\toverlaps := func(subjects []string, filter string) bool {\n\t\tif filter == _EMPTY_ {\n\t\t\treturn true\n\t\t}\n\t\tfor _, subject := range subjects {\n\t\t\tif SubjectsCollide(subject, filter) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\n\tfor len(toVisit) > 0 {\n\t\tcfg := toVisit[0]\n\t\ttoVisit = toVisit[1:]\n\t\tvisited[cfg.Name] = struct{}{}\n\t\tfor _, src := range cfg.Sources {\n\t\t\tif src.External != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// We can detect a cycle between streams, but let's double check that the\n\t\t\t// subjects actually form a cycle.\n\t\t\tif _, ok := visited[src.Name]; ok {\n\t\t\t\tif overlaps(cfg.Subjects, src.FilterSubject) {\n\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(errors.New(\"detected cycle\"))\n\t\t\t\t}\n\t\t\t} else if exists, cfg := getStream(src.Name); exists {\n\t\t\t\ttoVisit = append(toVisit, &cfg)\n\t\t\t}\n\t\t}\n\t\t// Avoid cycles hiding behind mirrors\n\t\tif m := cfg.Mirror; m != nil {\n\t\t\tif m.External == nil {\n\t\t\t\tif _, ok := visited[m.Name]; ok {\n\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(errors.New(\"detected cycle\"))\n\t\t\t\t}\n\t\t\t\tif exists, cfg := getStream(m.Name); exists {\n\t\t\t\t\ttoVisit = append(toVisit, &cfg)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(cfg.Subjects) == 0 {\n\t\tif cfg.Mirror == nil && len(cfg.Sources) == 0 {\n\t\t\tcfg.Subjects = append(cfg.Subjects, cfg.Name)\n\t\t}\n\t} else {\n\t\tif cfg.Mirror != nil {\n\t\t\treturn StreamConfig{}, NewJSMirrorWithSubjectsError()\n\t\t}\n\n\t\t// Check for literal duplication of subject interest in config\n\t\t// and no overlap with any JS or SYS API subject space.\n\t\tdset := make(map[string]struct{}, len(cfg.Subjects))\n\t\tfor i, subj := range cfg.Subjects {\n\t\t\t// Make sure the subject is valid. Check this first.\n\t\t\tif !IsValidSubject(subj) {\n\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"invalid subject\"))\n\t\t\t}\n\t\t\tif _, ok := dset[subj]; ok {\n\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"duplicate subjects detected\"))\n\t\t\t}\n\t\t\t// Check for trying to capture everything.\n\t\t\tif subj == fwcs {\n\t\t\t\tif !cfg.NoAck {\n\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"capturing all subjects requires no-ack to be true\"))\n\t\t\t\t}\n\t\t\t\t// Capturing everything also will require R1.\n\t\t\t\tif cfg.Replicas != 1 {\n\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"capturing all subjects requires replicas of 1\"))\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Also check to make sure we do not overlap with our $JS API subjects.\n\t\t\tif !cfg.NoAck {\n\t\t\t\tfor _, namespace := range []string{\"$JS.>\", \"$JSC.>\", \"$NRG.>\"} {\n\t\t\t\t\tif SubjectsCollide(subj, namespace) {\n\t\t\t\t\t\t// We allow an exception for $JS.EVENT.> since these could have been created in the past.\n\t\t\t\t\t\tif !subjectIsSubsetMatch(subj, \"$JS.EVENT.>\") {\n\t\t\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subjects that overlap with jetstream api require no-ack to be true\"))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif SubjectsCollide(subj, \"$SYS.>\") {\n\t\t\t\t\tif !subjectIsSubsetMatch(subj, \"$SYS.ACCOUNT.>\") {\n\t\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subjects that overlap with system api require no-ack to be true\"))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now check if we have multiple subjects that we do not overlap ourselves\n\t\t\t// which would cause duplicate entries (assuming no MsgID).\n\t\t\tfor _, tsubj := range cfg.Subjects[i+1:] {\n\t\t\t\tif SubjectsCollide(tsubj, subj) {\n\t\t\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"subject %q overlaps with %q\", subj, tsubj))\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Mark for duplicate check.\n\t\t\tdset[subj] = struct{}{}\n\t\t}\n\t}\n\n\tif len(cfg.Subjects) == 0 && len(cfg.Sources) == 0 && cfg.Mirror == nil {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(\n\t\t\tfmt.Errorf(\"stream needs at least one configured subject or be a source/mirror\"))\n\t}\n\n\t// Check for MaxBytes required and it's limit\n\tif required, limit := acc.maxBytesLimits(&cfg); required && cfg.MaxBytes <= 0 {\n\t\treturn StreamConfig{}, NewJSStreamMaxBytesRequiredError()\n\t} else if limit > 0 && cfg.MaxBytes > limit {\n\t\treturn StreamConfig{}, NewJSStreamMaxStreamBytesExceededError()\n\t}\n\n\t// Check the subject transform if any\n\tif cfg.SubjectTransform != nil {\n\t\tif cfg.SubjectTransform.Source != _EMPTY_ && !IsValidSubject(cfg.SubjectTransform.Source) {\n\t\t\treturn StreamConfig{}, NewJSStreamTransformInvalidSourceError(fmt.Errorf(\"%w %s\", ErrBadSubject, cfg.SubjectTransform.Source))\n\t\t}\n\n\t\terr := ValidateMapping(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination)\n\t\tif err != nil {\n\t\t\treturn StreamConfig{}, NewJSStreamTransformInvalidDestinationError(err)\n\t\t}\n\t}\n\n\t// If we have a republish directive check if we can create a transform here.\n\tif cfg.RePublish != nil {\n\t\t// Check to make sure source is a valid subset of the subjects we have.\n\t\t// Also make sure it does not form a cycle.\n\t\t// Empty same as all.\n\t\tif cfg.RePublish.Source == _EMPTY_ {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"republish source can not be empty\"))\n\t\t\t}\n\t\t\tcfg.RePublish.Source = fwcs\n\t\t}\n\t\t// A RePublish from '>' to '>' could be used, normally this would form a cycle with the stream subjects.\n\t\t// But if this aligns to a different subject based on the transform, we allow it still.\n\t\t// The RePublish will be implicit based on the transform, but only if the transform's source\n\t\t// is the only stream subject.\n\t\tif cfg.RePublish.Destination == fwcs && cfg.RePublish.Source == fwcs && cfg.SubjectTransform != nil &&\n\t\t\tlen(cfg.Subjects) == 1 && cfg.SubjectTransform.Source == cfg.Subjects[0] {\n\t\t\tif pedantic {\n\t\t\t\treturn StreamConfig{}, NewJSPedanticError(fmt.Errorf(\"implicit republish based on subject transform\"))\n\t\t\t}\n\t\t\t// RePublish all messages with the transformed subject.\n\t\t\tcfg.RePublish.Source, cfg.RePublish.Destination = cfg.SubjectTransform.Destination, cfg.SubjectTransform.Destination\n\t\t}\n\t\tvar formsCycle bool\n\t\tfor _, subj := range cfg.Subjects {\n\t\t\tif SubjectsCollide(cfg.RePublish.Destination, subj) {\n\t\t\t\tformsCycle = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif formsCycle {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for republish destination forms a cycle\"))\n\t\t}\n\t\tif _, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination); err != nil {\n\t\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration for republish with transform from '%s' to '%s' not valid\", cfg.RePublish.Source, cfg.RePublish.Destination))\n\t\t}\n\t}\n\n\t// Remove placement if it's an empty object.\n\tif cfg.Placement != nil && reflect.DeepEqual(cfg.Placement, &Placement{}) {\n\t\tcfg.Placement = nil\n\t}\n\t// For now don't allow preferred server in placement.\n\tif cfg.Placement != nil && cfg.Placement.Preferred != _EMPTY_ {\n\t\treturn StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf(\"preferred server not permitted in placement\"))\n\t}\n\n\treturn cfg, nil\n}\n\n// Config returns the stream's configuration.\nfunc (mset *stream) config() StreamConfig {\n\tmset.cfgMu.RLock()\n\tdefer mset.cfgMu.RUnlock()\n\treturn mset.cfg\n}\n\nfunc (mset *stream) fileStoreConfig() (FileStoreConfig, error) {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tfs, ok := mset.store.(*fileStore)\n\tif !ok {\n\t\treturn FileStoreConfig{}, ErrStoreWrongType\n\t}\n\treturn fs.fileStoreConfig(), nil\n}\n\n// Do not hold jsAccount or jetStream lock\nfunc (jsa *jsAccount) configUpdateCheck(old, new *StreamConfig, s *Server, pedantic bool) (*StreamConfig, error) {\n\tcfg, apiErr := s.checkStreamCfg(new, jsa.acc(), pedantic)\n\tif apiErr != nil {\n\t\treturn nil, apiErr\n\t}\n\n\t// Name must match.\n\tif cfg.Name != old.Name {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration name must match original\"))\n\t}\n\t// Can't change storage types.\n\tif cfg.Storage != old.Storage {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change storage type\"))\n\t}\n\t// Can only change retention from limits to interest or back, not to/from work queue for now.\n\tif cfg.Retention != old.Retention {\n\t\tif old.Retention == WorkQueuePolicy || cfg.Retention == WorkQueuePolicy {\n\t\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change retention policy to/from workqueue\"))\n\t\t}\n\t}\n\t// Can not change from true to false.\n\tif !cfg.Sealed && old.Sealed {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not unseal a sealed stream\"))\n\t}\n\t// Can not change from true to false.\n\tif !cfg.DenyDelete && old.DenyDelete {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not cancel deny message deletes\"))\n\t}\n\t// Can not change from true to false.\n\tif !cfg.DenyPurge && old.DenyPurge {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not cancel deny purge\"))\n\t}\n\t// Check for mirror changes which are not allowed.\n\t// We will allow removing the mirror config to \"promote\" the mirror to a normal stream.\n\tif cfg.Mirror != nil && !reflect.DeepEqual(cfg.Mirror, old.Mirror) {\n\t\treturn nil, NewJSStreamMirrorNotUpdatableError()\n\t}\n\n\t// Check on new discard new per subject.\n\tif cfg.DiscardNewPer {\n\t\tif cfg.Discard != DiscardNew {\n\t\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"discard new per subject requires discard new policy to be set\"))\n\t\t}\n\t\tif cfg.MaxMsgsPer <= 0 {\n\t\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"discard new per subject requires max msgs per subject > 0\"))\n\t\t}\n\t}\n\n\t// Check on the allowed message TTL status.\n\tif old.AllowMsgTTL && !cfg.AllowMsgTTL {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"message TTL status can not be disabled\"))\n\t}\n\n\t// Can't change counter setting.\n\tif cfg.AllowMsgCounter != old.AllowMsgCounter {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change message counter setting\"))\n\t}\n\n\t// Can't disable message schedules setting.\n\tif old.AllowMsgSchedules && !cfg.AllowMsgSchedules {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"message schedules can not be disabled\"))\n\t}\n\n\tif old.PersistMode != cfg.PersistMode {\n\t\treturn nil, NewJSStreamInvalidConfigError(fmt.Errorf(\"stream configuration update can not change persist mode\"))\n\t}\n\n\t// Do some adjustments for being sealed.\n\t// Pedantic mode will allow those changes to be made, as they are deterministic and important to get a sealed stream.\n\tif cfg.Sealed {\n\t\tcfg.MaxAge = 0\n\t\tcfg.Discard = DiscardNew\n\t\tcfg.DenyDelete, cfg.DenyPurge = true, true\n\t\tcfg.AllowRollup = false\n\t}\n\n\t// Check limits. We need some extra handling to allow updating MaxBytes.\n\n\t// First, let's calculate the difference between the new and old MaxBytes.\n\tmaxBytesDiff := max(cfg.MaxBytes, 0) - max(old.MaxBytes, 0)\n\tif maxBytesDiff < 0 {\n\t\t// If we're updating to a lower MaxBytes (maxBytesDiff is negative),\n\t\t// then set to zero so checkBytesLimits doesn't set addBytes to 1.\n\t\tmaxBytesDiff = 0\n\t}\n\t// If maxBytesDiff == 0, then that means MaxBytes didn't change.\n\t// If maxBytesDiff > 0, then we want to reserve additional bytes.\n\n\t// Save the user configured MaxBytes.\n\tnewMaxBytes := cfg.MaxBytes\n\tmaxBytesOffset := int64(0)\n\n\t// We temporarily set cfg.MaxBytes to maxBytesDiff because checkAllLimits\n\t// adds cfg.MaxBytes to the current reserved limit and checks if we've gone\n\t// over. However, we don't want an addition cfg.MaxBytes, we only want to\n\t// reserve the difference between the new and the old values.\n\tcfg.MaxBytes = maxBytesDiff\n\n\t// Check limits.\n\tjs, isClustered := jsa.jetStreamAndClustered()\n\tjsa.mu.RLock()\n\tacc := jsa.account\n\tjsa.usageMu.RLock()\n\tselected, tier, hasTier := jsa.selectLimits(cfg.Replicas)\n\tif !hasTier && old.Replicas != cfg.Replicas {\n\t\tselected, tier, hasTier = jsa.selectLimits(old.Replicas)\n\t}\n\tjsa.usageMu.RUnlock()\n\treserved := int64(0)\n\tif !isClustered {\n\t\treserved = jsa.tieredReservation(tier, &cfg)\n\t}\n\tjsa.mu.RUnlock()\n\tif !hasTier {\n\t\treturn nil, NewJSNoLimitsError()\n\t}\n\tjs.mu.RLock()\n\tdefer js.mu.RUnlock()\n\tif isClustered {\n\t\t_, reserved = js.tieredStreamAndReservationCount(acc.Name, tier, &cfg)\n\t}\n\t// reservation does not account for this stream, hence add the old value\n\tif tier == _EMPTY_ && old.Replicas > 1 {\n\t\treserved += old.MaxBytes * int64(old.Replicas)\n\t} else {\n\t\treserved += old.MaxBytes\n\t}\n\tif err := js.checkAllLimits(&selected, &cfg, reserved, maxBytesOffset); err != nil {\n\t\treturn nil, err\n\t}\n\t// Restore the user configured MaxBytes.\n\tcfg.MaxBytes = newMaxBytes\n\treturn &cfg, nil\n}\n\n// Update will allow certain configuration properties of an existing stream to be updated.\nfunc (mset *stream) update(config *StreamConfig) error {\n\treturn mset.updateWithAdvisory(config, true, false)\n}\n\nfunc (mset *stream) updatePedantic(config *StreamConfig, pedantic bool) error {\n\treturn mset.updateWithAdvisory(config, true, pedantic)\n}\n\n// Update will allow certain configuration properties of an existing stream to be updated.\nfunc (mset *stream) updateWithAdvisory(config *StreamConfig, sendAdvisory bool, pedantic bool) error {\n\t_, jsa, err := mset.acc.checkForJetStream()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmset.mu.RLock()\n\tocfg := mset.cfg\n\ts := mset.srv\n\tmset.mu.RUnlock()\n\n\tcfg, err := mset.jsa.configUpdateCheck(&ocfg, config, s, pedantic)\n\tif err != nil {\n\t\treturn NewJSStreamInvalidConfigError(err, Unless(err))\n\t}\n\n\t// In the event that some of the stream-level limits have changed, yell appropriately\n\t// if any of the consumers exceed that limit.\n\tupdateLimits := ocfg.ConsumerLimits.InactiveThreshold != cfg.ConsumerLimits.InactiveThreshold ||\n\t\tocfg.ConsumerLimits.MaxAckPending != cfg.ConsumerLimits.MaxAckPending\n\tif updateLimits {\n\t\tvar errorConsumers []string\n\t\tconsumers := map[string]*ConsumerConfig{}\n\t\tif mset.js.isClustered() {\n\t\t\tfor _, c := range mset.sa.consumers {\n\t\t\t\tconsumers[c.Name] = c.Config\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, c := range mset.consumers {\n\t\t\t\tconsumers[c.name] = &c.cfg\n\t\t\t}\n\t\t}\n\t\tfor name, ccfg := range consumers {\n\t\t\tif ccfg.InactiveThreshold > cfg.ConsumerLimits.InactiveThreshold ||\n\t\t\t\tccfg.MaxAckPending > cfg.ConsumerLimits.MaxAckPending {\n\t\t\t\terrorConsumers = append(errorConsumers, name)\n\t\t\t}\n\t\t}\n\t\tif len(errorConsumers) > 0 {\n\t\t\t// TODO(nat): Return a parsable error so that we can surface something\n\t\t\t// sensible through the JS API.\n\t\t\treturn fmt.Errorf(\"change to limits violates consumers: %s\", strings.Join(errorConsumers, \", \"))\n\t\t}\n\t}\n\n\tjsa.mu.RLock()\n\tif jsa.subjectsOverlap(cfg.Subjects, mset) {\n\t\tjsa.mu.RUnlock()\n\t\treturn NewJSStreamSubjectOverlapError()\n\t}\n\tjsa.mu.RUnlock()\n\n\tmset.mu.Lock()\n\tif mset.isLeader() {\n\t\t// Check for mirror promotion.\n\t\tif ocfg.Mirror != nil && cfg.Mirror == nil {\n\t\t\tmset.cancelMirrorConsumer()\n\t\t\tmset.mirror = nil\n\t\t}\n\n\t\t// Now check for subject interest differences.\n\t\tcurrent := make(map[string]struct{}, len(ocfg.Subjects))\n\t\tfor _, s := range ocfg.Subjects {\n\t\t\tcurrent[s] = struct{}{}\n\t\t}\n\t\t// Update config with new values. The store update will enforce any stricter limits.\n\n\t\t// Now walk new subjects. All of these need to be added, but we will check\n\t\t// the originals first, since if it is in there we can skip, already added.\n\t\tfor _, s := range cfg.Subjects {\n\t\t\tif _, ok := current[s]; !ok {\n\t\t\t\tif _, err := mset.subscribeInternal(s, mset.processInboundJetStreamMsg); err != nil {\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tdelete(current, s)\n\t\t}\n\t\t// What is left in current needs to be deleted.\n\t\tfor s := range current {\n\t\t\tif err := mset.unsubscribeInternal(s); err != nil {\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\t// Check for the Duplicates\n\t\tmset.ddMu.Lock()\n\t\tif cfg.Duplicates != ocfg.Duplicates && mset.ddtmr != nil {\n\t\t\t// Let it fire right away, it will adjust properly on purge.\n\t\t\tmset.ddtmr.Reset(time.Microsecond)\n\t\t}\n\t\tmset.ddMu.Unlock()\n\n\t\t// Check for Sources.\n\t\tif len(cfg.Sources) > 0 || len(ocfg.Sources) > 0 {\n\t\t\tcurrentIName := make(map[string]struct{})\n\t\t\tneedsStartingSeqNum := make(map[string]struct{})\n\n\t\t\tfor _, s := range ocfg.Sources {\n\t\t\t\tcurrentIName[s.iname] = struct{}{}\n\t\t\t}\n\t\t\tfor _, s := range cfg.Sources {\n\t\t\t\ts.setIndexName()\n\t\t\t\tif _, ok := currentIName[s.iname]; !ok {\n\t\t\t\t\t// new source\n\t\t\t\t\tif mset.sources == nil {\n\t\t\t\t\t\tmset.sources = make(map[string]*sourceInfo)\n\t\t\t\t\t}\n\t\t\t\t\tmset.cfg.Sources = append(mset.cfg.Sources, s)\n\n\t\t\t\t\tvar si *sourceInfo\n\n\t\t\t\t\tif len(s.SubjectTransforms) == 0 {\n\t\t\t\t\t\tsi = &sourceInfo{name: s.Name, iname: s.iname, sf: s.FilterSubject}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsi = &sourceInfo{name: s.Name, iname: s.iname}\n\t\t\t\t\t\tsi.trs = make([]*subjectTransform, len(s.SubjectTransforms))\n\t\t\t\t\t\tsi.sfs = make([]string, len(s.SubjectTransforms))\n\t\t\t\t\t\tfor i := range s.SubjectTransforms {\n\t\t\t\t\t\t\t// err can be ignored as already validated in config check\n\t\t\t\t\t\t\tsi.sfs[i] = s.SubjectTransforms[i].Source\n\t\t\t\t\t\t\tvar err error\n\t\t\t\t\t\t\tsi.trs[i], err = NewSubjectTransform(s.SubjectTransforms[i].Source, s.SubjectTransforms[i].Destination)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\t\t\treturn fmt.Errorf(\"unable to get subject transform for source: %v\", err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tmset.sources[s.iname] = si\n\t\t\t\t\tneedsStartingSeqNum[s.iname] = struct{}{}\n\t\t\t\t} else {\n\t\t\t\t\t// source already exists\n\t\t\t\t\tdelete(currentIName, s.iname)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// What is left in currentIName needs to be deleted.\n\t\t\tfor iName := range currentIName {\n\t\t\t\tmset.cancelSourceConsumer(iName)\n\t\t\t\tdelete(mset.sources, iName)\n\t\t\t}\n\t\t\tneededCopy := make(map[string]struct{}, len(needsStartingSeqNum))\n\t\t\tfor iName := range needsStartingSeqNum {\n\t\t\t\tneededCopy[iName] = struct{}{}\n\t\t\t}\n\t\t\tmset.setStartingSequenceForSources(needsStartingSeqNum)\n\t\t\tfor iName := range neededCopy {\n\t\t\t\tmset.setupSourceConsumer(iName, mset.sources[iName].sseq+1, time.Time{})\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check for a change in allow direct/mirror status.\n\t// These will run on all members, so just update as appropriate here.\n\t// We do make sure we are caught up under monitorStream() during initial startup.\n\tif cfg.AllowDirect != ocfg.AllowDirect {\n\t\tif cfg.AllowDirect {\n\t\t\tmset.subscribeToDirect()\n\t\t} else {\n\t\t\tmset.unsubscribeToDirect()\n\t\t}\n\t}\n\tif cfg.MirrorDirect != ocfg.MirrorDirect {\n\t\tif cfg.MirrorDirect {\n\t\t\tmset.subscribeToMirrorDirect()\n\t\t} else {\n\t\t\tmset.unsubscribeToMirrorDirect()\n\t\t}\n\t}\n\n\t// Check for changes to RePublish.\n\tif cfg.RePublish != nil {\n\t\t// Empty same as all.\n\t\tif cfg.RePublish.Source == _EMPTY_ {\n\t\t\tcfg.RePublish.Source = fwcs\n\t\t}\n\t\tif cfg.RePublish.Destination == _EMPTY_ {\n\t\t\tcfg.RePublish.Destination = fwcs\n\t\t}\n\t\ttr, err := NewSubjectTransform(cfg.RePublish.Source, cfg.RePublish.Destination)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn fmt.Errorf(\"stream configuration for republish from '%s' to '%s': %w\", cfg.RePublish.Source, cfg.RePublish.Destination, err)\n\t\t}\n\t\t// Assign our transform for republishing.\n\t\tmset.tr = tr\n\t} else {\n\t\tmset.tr = nil\n\t}\n\n\t// Check for changes to subject transform\n\tif ocfg.SubjectTransform == nil && cfg.SubjectTransform != nil {\n\t\ttr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn fmt.Errorf(\"stream configuration for subject transform from '%s' to '%s': %w\", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err)\n\t\t}\n\t\tmset.itr = tr\n\t} else if ocfg.SubjectTransform != nil && cfg.SubjectTransform != nil &&\n\t\t(ocfg.SubjectTransform.Source != cfg.SubjectTransform.Source || ocfg.SubjectTransform.Destination != cfg.SubjectTransform.Destination) {\n\t\ttr, err := NewSubjectTransform(cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn fmt.Errorf(\"stream configuration for subject transform from '%s' to '%s': %w\", cfg.SubjectTransform.Source, cfg.SubjectTransform.Destination, err)\n\t\t}\n\t\tmset.itr = tr\n\t} else if ocfg.SubjectTransform != nil && cfg.SubjectTransform == nil {\n\t\tmset.itr = nil\n\t}\n\n\tjs := mset.js\n\n\tif targetTier := tierName(cfg.Replicas); mset.tier != targetTier {\n\t\t// In cases such as R1->R3, only one update is needed\n\t\tjsa.usageMu.RLock()\n\t\t_, ok := jsa.limits[targetTier]\n\t\tjsa.usageMu.RUnlock()\n\t\tif ok {\n\t\t\t// error never set\n\t\t\t_, reported, _ := mset.store.Utilization()\n\t\t\tjsa.updateUsage(mset.tier, mset.stype, -int64(reported))\n\t\t\tjsa.updateUsage(targetTier, mset.stype, int64(reported))\n\t\t\tmset.tier = targetTier\n\t\t}\n\t\t// else in case the new tier does not exist (say on move), keep the old tier around\n\t\t// a subsequent update to an existing tier will then move from existing past tier to existing new tier\n\t}\n\n\tif mset.isLeader() && mset.sa != nil && ocfg.Retention != cfg.Retention && cfg.Retention == InterestPolicy {\n\t\t// Before we can update the retention policy for the consumer, we need\n\t\t// the replica count of all consumers to match the stream.\n\t\tfor _, c := range mset.sa.consumers {\n\t\t\tif c.Config.Replicas > 0 && c.Config.Replicas != cfg.Replicas {\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn fmt.Errorf(\"consumer %q replica count must be %d\", c.Name, cfg.Replicas)\n\t\t\t}\n\t\t}\n\t}\n\n\t// If atomic publish is disabled, delete any in-progress batches.\n\tif !cfg.AllowAtomicPublish {\n\t\tmset.deleteAtomicBatches(false)\n\t\tmset.deleteBatchApplyState()\n\t}\n\t// If fast batch publish is disabled, delete any in-progress batches.\n\tif !cfg.AllowBatchPublish {\n\t\tmset.deleteFastBatches()\n\t}\n\tif !cfg.AllowAtomicPublish && !cfg.AllowBatchPublish {\n\t\tmset.batches = nil\n\t}\n\n\t// Now update config and store's version of our config.\n\t// Although we are under the stream write lock, we will also assign the new\n\t// configuration under mset.cfgMu lock. This is so that in places where\n\t// mset.mu cannot be acquired (like many cases in consumer.go where code\n\t// is under the consumer's lock), and the stream's configuration needs to\n\t// be inspected, one can use mset.cfgMu's read lock to do that safely.\n\tmset.cfgMu.Lock()\n\tmset.cfg = *cfg\n\tmset.cfgMu.Unlock()\n\n\t// If we're changing retention and haven't errored because of consumer\n\t// replicas by now, whip through and update the consumer retention.\n\tif ocfg.Retention != cfg.Retention {\n\t\ttoUpdate := make([]*consumer, 0, len(mset.consumers))\n\t\tfor _, c := range mset.consumers {\n\t\t\ttoUpdate = append(toUpdate, c)\n\t\t}\n\t\tvar ss StreamState\n\t\tmset.store.FastState(&ss)\n\t\tmset.mu.Unlock()\n\t\tfor _, c := range toUpdate {\n\t\t\tc.mu.Lock()\n\t\t\tc.retention = cfg.Retention\n\t\t\tc.mu.Unlock()\n\t\t\tif c.retention == InterestPolicy {\n\t\t\t\t// If we're switching to interest, force a check of the\n\t\t\t\t// interest of existing stream messages.\n\t\t\t\tc.checkStateForInterestStream(&ss)\n\t\t\t}\n\t\t}\n\t\tmset.mu.Lock()\n\t}\n\n\t// If we are the leader never suppress update advisory, simply send.\n\tif mset.isLeader() && sendAdvisory {\n\t\tmset.sendUpdateAdvisoryLocked()\n\t}\n\tmset.mu.Unlock()\n\n\tif js != nil {\n\t\tmaxBytesDiff := max(cfg.MaxBytes, 0) - max(ocfg.MaxBytes, 0)\n\t\tif maxBytesDiff > 0 {\n\t\t\t// Reserve the difference\n\t\t\tjs.reserveStreamResources(&StreamConfig{\n\t\t\t\tMaxBytes: maxBytesDiff,\n\t\t\t\tStorage: cfg.Storage,\n\t\t\t})\n\t\t} else if maxBytesDiff < 0 {\n\t\t\t// Release the difference\n\t\t\tjs.releaseStreamResources(&StreamConfig{\n\t\t\t\tMaxBytes: -maxBytesDiff,\n\t\t\t\tStorage: ocfg.Storage,\n\t\t\t})\n\t\t}\n\t}\n\n\tmset.store.UpdateConfig(cfg)\n\n\treturn nil\n}\n\n// Small helper to return the Name field from mset.cfg, protected by\n// the mset.cfgMu mutex. This is simply because we have several places\n// in consumer.go where we need it.\nfunc (mset *stream) getCfgName() string {\n\tmset.cfgMu.RLock()\n\tdefer mset.cfgMu.RUnlock()\n\treturn mset.cfg.Name\n}\n\n// Purge will remove all messages from the stream and underlying store based on the request.\nfunc (mset *stream) purge(preq *JSApiStreamPurgeRequest) (purged uint64, err error) {\n\treturn mset.purgeLocked(preq, true)\n}\n\nfunc (mset *stream) purgeLocked(preq *JSApiStreamPurgeRequest, needLock bool) (purged uint64, err error) {\n\tif needLock {\n\t\tmset.mu.Lock()\n\t\tdefer mset.mu.Unlock()\n\t}\n\tif mset.closed.Load() {\n\t\treturn 0, errStreamClosed\n\t}\n\tif mset.cfg.Sealed {\n\t\treturn 0, errors.New(\"sealed stream\")\n\t}\n\tstore, mlseq := mset.store, mset.lseq\n\n\tif preq != nil {\n\t\tpurged, err = mset.store.PurgeEx(preq.Subject, preq.Sequence, preq.Keep)\n\t} else {\n\t\tpurged, err = mset.store.Purge()\n\t}\n\tif err != nil {\n\t\treturn purged, err\n\t}\n\n\t// Grab our stream state.\n\tvar state StreamState\n\tstore.FastState(&state)\n\tfseq, lseq := state.FirstSeq, state.LastSeq\n\n\t// Check if our last has moved past what our original last sequence was, if so reset.\n\tif lseq > mlseq {\n\t\tmset.setLastSeq(lseq)\n\t}\n\n\t// Clear any pending acks below first seq.\n\tmset.clearAllPreAcksBelowFloor(fseq)\n\n\t// Purge consumers.\n\t// Check for filtered purge.\n\tif preq != nil && preq.Subject != _EMPTY_ {\n\t\tss, err := store.FilteredState(fseq, preq.Subject)\n\t\tif err != nil {\n\t\t\treturn purged, err\n\t\t}\n\t\tfseq = ss.First\n\t}\n\n\t// Take a copy of cList to avoid o.purge() potentially taking the stream lock and\n\t// violating the lock ordering.\n\tmset.clsMu.RLock()\n\tcList := slices.Clone(mset.cList)\n\tmset.clsMu.RUnlock()\n\tfor _, o := range cList {\n\t\tstart := fseq\n\t\to.mu.RLock()\n\t\t// we update consumer sequences if:\n\t\t// no subject was specified, we can purge all consumers sequences\n\t\tdoPurge := preq == nil ||\n\t\t\tpreq.Subject == _EMPTY_ ||\n\t\t\t// consumer filter subject is equal to purged subject\n\t\t\t// or consumer filter subject is subset of purged subject,\n\t\t\t// but not the other way around.\n\t\t\to.isEqualOrSubsetMatch(preq.Subject)\n\t\t// Check if a consumer has a wider subject space than what we purged\n\t\tvar isWider bool\n\t\tif !doPurge && preq != nil && o.isFilteredMatch(preq.Subject) {\n\t\t\tdoPurge, isWider = true, true\n\t\t\tstart = state.FirstSeq\n\t\t}\n\t\to.mu.RUnlock()\n\t\tif doPurge {\n\t\t\to.purge(start, lseq, isWider)\n\t\t}\n\t}\n\n\treturn purged, nil\n}\n\n// RemoveMsg will remove a message from a stream.\n// FIXME(dlc) - Should pick one and be consistent.\nfunc (mset *stream) removeMsg(seq uint64) (bool, error) {\n\treturn mset.deleteMsg(seq)\n}\n\n// DeleteMsg will remove a message from a stream.\nfunc (mset *stream) deleteMsg(seq uint64) (bool, error) {\n\tif mset.closed.Load() {\n\t\treturn false, errStreamClosed\n\t}\n\tremoved, err := mset.store.RemoveMsg(seq)\n\tif err != nil {\n\t\treturn removed, err\n\t}\n\tmset.mu.Lock()\n\tmset.clearAllPreAcks(seq)\n\tmset.mu.Unlock()\n\treturn removed, err\n}\n\n// EraseMsg will securely remove a message and rewrite the data with random data.\nfunc (mset *stream) eraseMsg(seq uint64) (bool, error) {\n\tif mset.closed.Load() {\n\t\treturn false, errStreamClosed\n\t}\n\tremoved, err := mset.store.EraseMsg(seq)\n\tif err != nil {\n\t\treturn removed, err\n\t}\n\tmset.mu.Lock()\n\tmset.clearAllPreAcks(seq)\n\tmset.mu.Unlock()\n\treturn removed, err\n}\n\n// Are we a mirror?\nfunc (mset *stream) isMirror() bool {\n\tmset.cfgMu.RLock()\n\tdefer mset.cfgMu.RUnlock()\n\treturn mset.cfg.Mirror != nil\n}\n\nfunc (mset *stream) sourcesInfo() (sis []*StreamSourceInfo) {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\tsis = make([]*StreamSourceInfo, 0, len(mset.sources))\n\tfor _, si := range mset.sources {\n\t\tsis = append(sis, mset.sourceInfo(si))\n\t}\n\treturn sis\n}\n\n// Lock should be held\nfunc (mset *stream) sourceInfo(si *sourceInfo) *StreamSourceInfo {\n\tif si == nil {\n\t\treturn nil\n\t}\n\n\tvar ssi = StreamSourceInfo{Name: si.name, Lag: si.lag, Error: si.err, FilterSubject: si.sf}\n\n\ttrConfigs := make([]SubjectTransformConfig, len(si.sfs))\n\tfor i := range si.sfs {\n\t\tvar destination string\n\t\tif si.trs[i] != nil {\n\t\t\tdestination = si.trs[i].dest\n\t\t}\n\t\ttrConfigs[i] = SubjectTransformConfig{si.sfs[i], destination}\n\t}\n\n\tssi.SubjectTransforms = trConfigs\n\n\t// If we have not heard from the source, set Active to -1.\n\tif last := si.last.Load(); last == 0 {\n\t\tssi.Active = -1\n\t} else {\n\t\tssi.Active = time.Since(time.Unix(0, last))\n\t}\n\n\tvar ext *ExternalStream\n\tif mset.cfg.Mirror != nil {\n\t\text = mset.cfg.Mirror.External\n\t} else if ss := mset.streamSource(si.iname); ss != nil && ss.External != nil {\n\t\text = ss.External\n\t}\n\tif ext != nil {\n\t\tssi.External = &ExternalStream{\n\t\t\tApiPrefix: ext.ApiPrefix,\n\t\t\tDeliverPrefix: ext.DeliverPrefix,\n\t\t}\n\t}\n\treturn &ssi\n}\n\n// Return our source info for our mirror.\nfunc (mset *stream) mirrorInfo() *StreamSourceInfo {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.sourceInfo(mset.mirror)\n}\n\n// retryDisconnectedSyncConsumers() will check if we have any disconnected\n// sync consumers for either mirror or a source and will reset and retry to connect.\nfunc (mset *stream) retryDisconnectedSyncConsumers() {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\n\t// Only applicable if we are the stream leader.\n\tif !mset.isLeader() {\n\t\treturn\n\t}\n\n\tshouldRetry := func(si *sourceInfo) bool {\n\t\tif si != nil && (si.sip || si.sub == nil || (si.sub.client != nil && si.sub.client.isClosed())) {\n\t\t\t// Need to reset\n\t\t\tsi.fails, si.sip = 0, false\n\t\t\tmset.cancelSourceInfo(si)\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\t// Check mirrors first.\n\tif si := mset.mirror; si != nil {\n\t\tif shouldRetry(si) {\n\t\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t\t}\n\t} else {\n\t\tfor _, si := range mset.sources {\n\t\t\tif shouldRetry(si) {\n\t\t\t\tmset.setupSourceConsumer(si.iname, si.sseq+1, time.Time{})\n\t\t\t}\n\t\t}\n\t}\n}\n\nconst (\n\t// Our consumer HB interval.\n\tsourceHealthHB = 1 * time.Second\n\t// How often we check and our stalled interval.\n\tsourceHealthCheckInterval = 10 * time.Second\n)\n\n// Will run as a Go routine to process mirror consumer messages.\nfunc (mset *stream) processMirrorMsgs(mirror *sourceInfo, ready *sync.WaitGroup) {\n\ts := mset.srv\n\tdefer func() {\n\t\tmirror.wg.Done()\n\t\ts.grWG.Done()\n\t}()\n\n\t// Grab stream quit channel.\n\tmset.mu.Lock()\n\tmsgs, qch, siqch := mirror.msgs, mset.qch, mirror.qch\n\t// If the mirror was already canceled before we got here, exit early.\n\tif siqch == nil {\n\t\tmset.mu.Unlock()\n\t\tready.Done()\n\t\treturn\n\t}\n\t// Set the last seen as now so that we don't fail at the first check.\n\tmirror.last.Store(time.Now().UnixNano())\n\tmset.mu.Unlock()\n\n\t// Signal the caller that we have captured the above fields.\n\tready.Done()\n\n\t// Make sure we have valid ipq for msgs.\n\tif msgs == nil {\n\t\tmset.mu.Lock()\n\t\tmset.cancelMirrorConsumer()\n\t\tmset.mu.Unlock()\n\t\treturn\n\t}\n\n\tt := time.NewTicker(sourceHealthCheckInterval)\n\tdefer t.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-siqch:\n\t\t\treturn\n\t\tcase <-msgs.ch:\n\t\t\tims := msgs.pop()\n\t\t\tfor _, im := range ims {\n\t\t\t\tif !mset.processInboundMirrorMsg(im) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tim.returnToPool()\n\t\t\t}\n\t\t\tmsgs.recycle(&ims)\n\t\tcase <-t.C:\n\t\t\tmset.mu.RLock()\n\t\t\tvar stalled bool\n\t\t\tif mset.mirror != nil {\n\t\t\t\tstalled = time.Since(time.Unix(0, mset.mirror.last.Load())) > sourceHealthCheckInterval\n\t\t\t}\n\t\t\tisLeader := mset.isLeader()\n\t\t\tmset.mu.RUnlock()\n\t\t\t// No longer leader.\n\t\t\tif !isLeader {\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.cancelMirrorConsumer()\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// We are stalled.\n\t\t\tif stalled {\n\t\t\t\tmset.retryMirrorConsumer()\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Checks that the message is from our current direct consumer. We can not depend on sub comparison\n// since cross account imports break.\nfunc (si *sourceInfo) isCurrentSub(reply string) bool {\n\treturn si.cname != _EMPTY_ && strings.HasPrefix(reply, jsAckPre) && si.cname == tokenAt(reply, 4)\n}\n\n// processInboundMirrorMsg handles processing messages bound for a stream.\nfunc (mset *stream) processInboundMirrorMsg(m *inMsg) bool {\n\tmset.mu.Lock()\n\tif mset.mirror == nil {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\tif !mset.isLeader() {\n\t\tmset.cancelMirrorConsumer()\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\tisControl := m.isControlMsg()\n\n\t// Ignore from old subscriptions.\n\t// The reason we can not just compare subs is that on cross account imports they will not match.\n\tif !mset.mirror.isCurrentSub(m.rply) && !isControl {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Check for heartbeats and flow control messages.\n\tif isControl {\n\t\tvar needsRetry bool\n\t\t// Flow controls have reply subjects.\n\t\tif m.rply != _EMPTY_ {\n\t\t\tmset.handleFlowControl(m)\n\t\t} else {\n\t\t\t// For idle heartbeats make sure we did not miss anything and check if we are considered stalled.\n\t\t\tif ldseq := parseInt64(getHeader(JSLastConsumerSeq, m.hdr)); ldseq > 0 && uint64(ldseq) != mset.mirror.dseq {\n\t\t\t\tneedsRetry = true\n\t\t\t} else if fcReply := getHeader(JSConsumerStalled, m.hdr); len(fcReply) > 0 {\n\t\t\t\t// Other side thinks we are stalled, so send flow control reply.\n\t\t\t\tmset.outq.sendMsg(string(fcReply), nil)\n\t\t\t}\n\t\t}\n\t\tmset.mu.Unlock()\n\t\tif needsRetry {\n\t\t\tmset.retryMirrorConsumer()\n\t\t}\n\t\treturn !needsRetry\n\t}\n\n\tsseq, dseq, dc, ts, pending := replyInfo(m.rply)\n\n\tif dc > 1 {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Mirror info tracking.\n\tolag, osseq, odseq := mset.mirror.lag, mset.mirror.sseq, mset.mirror.dseq\n\tif sseq == mset.mirror.sseq+1 {\n\t\tmset.mirror.dseq = dseq\n\t\tmset.mirror.sseq++\n\t} else if sseq <= mset.mirror.sseq {\n\t\t// Ignore older messages.\n\t\tmset.mu.Unlock()\n\t\treturn true\n\t} else if mset.mirror.cname == _EMPTY_ {\n\t\tmset.mirror.cname = tokenAt(m.rply, 4)\n\t\tmset.mirror.dseq, mset.mirror.sseq = dseq, sseq\n\t} else {\n\t\t// If the deliver sequence matches then the upstream stream has expired or deleted messages.\n\t\tif dseq == mset.mirror.dseq+1 {\n\t\t\tmset.skipMsgs(mset.mirror.sseq+1, sseq-1)\n\t\t\tmset.mirror.dseq++\n\t\t\tmset.mirror.sseq = sseq\n\t\t} else {\n\t\t\tmset.mu.Unlock()\n\t\t\tmset.retryMirrorConsumer()\n\t\t\treturn false\n\t\t}\n\t}\n\n\tif pending == 0 {\n\t\tmset.mirror.lag = 0\n\t} else {\n\t\tmset.mirror.lag = pending - 1\n\t}\n\n\t// Check if we allow mirror direct here. If so check they we have mostly caught up.\n\t// The reason we do not require 0 is if the source is active we may always be slightly behind.\n\tif mset.cfg.MirrorDirect && mset.mirrorDirectSub == nil && pending < dgetCaughtUpThresh {\n\t\tif err := mset.subscribeToMirrorDirect(); err != nil {\n\t\t\t// Disable since we had problems above.\n\t\t\tmset.cfg.MirrorDirect = false\n\t\t}\n\t}\n\n\t// Do the subject transform if there's one\n\tif len(mset.mirror.trs) > 0 {\n\t\tfor _, tr := range mset.mirror.trs {\n\t\t\tif tr == nil {\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\ttsubj, err := tr.Match(m.subj)\n\t\t\t\tif err == nil {\n\t\t\t\t\tm.subj = tsubj\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\ts, js, stype := mset.srv, mset.js, mset.cfg.Storage\n\tnode := mset.node\n\tmset.mu.Unlock()\n\n\tvar err error\n\tif node != nil {\n\t\tif js.limitsExceeded(stype) {\n\t\t\ts.resourcesExceededError(stype)\n\t\t\terr = ApiErrors[JSInsufficientResourcesErr]\n\t\t} else {\n\t\t\terr = node.Propose(encodeStreamMsg(m.subj, _EMPTY_, m.hdr, m.msg, sseq-1, ts, true))\n\t\t}\n\t} else {\n\t\terr = mset.processJetStreamMsg(m.subj, _EMPTY_, m.hdr, m.msg, sseq-1, ts, nil, true, true)\n\t}\n\tif err != nil {\n\t\tif strings.Contains(err.Error(), \"no space left\") {\n\t\t\ts.Errorf(\"JetStream out of space, will be DISABLED\")\n\t\t\ts.DisableJetStream()\n\t\t\treturn false\n\t\t}\n\t\tif err != errLastSeqMismatch {\n\t\t\tmset.mu.RLock()\n\t\t\taccName, sname := mset.acc.Name, mset.cfg.Name\n\t\t\tmset.mu.RUnlock()\n\t\t\ts.RateLimitWarnf(\"Error processing inbound mirror message for '%s' > '%s': %v\",\n\t\t\t\taccName, sname, err)\n\t\t} else {\n\t\t\t// We may have missed messages, restart.\n\t\t\tif sseq <= mset.lastSeq() {\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.mirror.lag = olag\n\t\t\t\tmset.mirror.sseq = osseq\n\t\t\t\tmset.mirror.dseq = odseq\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn false\n\t\t\t} else {\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.mirror.dseq = odseq\n\t\t\t\tmset.mirror.sseq = osseq\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\tmset.retryMirrorConsumer()\n\t\t\t}\n\t\t}\n\t}\n\treturn err == nil\n}\n\nfunc (mset *stream) setMirrorErr(err *ApiError) {\n\tmset.mu.Lock()\n\tif mset.mirror != nil {\n\t\tmset.mirror.err = err\n\t}\n\tmset.mu.Unlock()\n}\n\n// Cancels a mirror consumer.\n//\n// Lock held on entry\nfunc (mset *stream) cancelMirrorConsumer() {\n\tif mset.mirror == nil {\n\t\treturn\n\t}\n\tmset.cancelSourceInfo(mset.mirror)\n}\n\n// Similar to setupMirrorConsumer except that it will print a debug statement\n// indicating that there is a retry.\n//\n// Lock is acquired in this function\nfunc (mset *stream) retryMirrorConsumer() error {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tmset.srv.Debugf(\"Retrying mirror consumer for '%s > %s'\", mset.acc.Name, mset.cfg.Name)\n\tmset.cancelMirrorConsumer()\n\treturn mset.setupMirrorConsumer()\n}\n\n// Lock should be held.\nfunc (mset *stream) skipMsgs(start, end uint64) {\n\tnode, store := mset.node, mset.store\n\t// If we are not clustered we can short circuit now with store.SkipMsgs\n\tif node == nil {\n\t\tstore.SkipMsgs(start, end-start+1)\n\t\tmset.lseq = end\n\t\treturn\n\t}\n\n\t// FIXME (dlc) - We should allow proposals of DeleteRange, but would need to make sure all peers support.\n\t// With syncRequest was easy to add bool into request.\n\tvar entries []*Entry\n\tfor seq := start; seq <= end; seq++ {\n\t\tentries = append(entries, newEntry(EntryNormal, encodeStreamMsg(_EMPTY_, _EMPTY_, nil, nil, seq-1, 0, false)))\n\t\t// So a single message does not get too big.\n\t\tif len(entries) > 10_000 {\n\t\t\tnode.ProposeMulti(entries)\n\t\t\t// We need to re-create `entries` because there is a reference\n\t\t\t// to it in the node's pae map.\n\t\t\tentries = entries[:0]\n\t\t}\n\t}\n\t// Send all at once.\n\tif len(entries) > 0 {\n\t\tnode.ProposeMulti(entries)\n\t}\n}\n\nconst (\n\t// Base retry backoff duration.\n\tretryBackOff = 5 * time.Second\n\t// Maximum amount we will wait.\n\tretryMaximum = 2 * time.Minute\n)\n\n// Calculate our backoff based on number of failures.\nfunc calculateRetryBackoff(fails int) time.Duration {\n\tbackoff := time.Duration(retryBackOff) * time.Duration(fails*2)\n\tif backoff > retryMaximum {\n\t\tbackoff = retryMaximum\n\t}\n\treturn backoff\n}\n\n// This will schedule a call to setupMirrorConsumer, taking into account the last\n// time it was retried and determine the soonest setupMirrorConsumer can be called\n// without tripping the sourceConsumerRetryThreshold. We will also take into account\n// number of failures and will back off our retries.\n// The mset.mirror pointer has been verified to be not nil by the caller.\n//\n// Lock held on entry\nfunc (mset *stream) scheduleSetupMirrorConsumerRetry() {\n\t// We are trying to figure out how soon we can retry. setupMirrorConsumer will reject\n\t// a retry if last was done less than \"sourceConsumerRetryThreshold\" ago.\n\tnext := sourceConsumerRetryThreshold - time.Since(mset.mirror.lreq)\n\tif next < 0 {\n\t\t// It means that we have passed the threshold and so we are ready to go.\n\t\tnext = 0\n\t}\n\t// Take into account failures here.\n\tnext += calculateRetryBackoff(mset.mirror.fails)\n\n\t// Add some jitter.\n\tnext += time.Duration(rand.Intn(int(100*time.Millisecond))) + 100*time.Millisecond\n\n\tstopAndClearTimer(&mset.mirrorConsumerSetup)\n\tmset.mirrorConsumerSetup = time.AfterFunc(next, func() {\n\t\tmset.mu.Lock()\n\t\tmset.setupMirrorConsumer()\n\t\tmset.mu.Unlock()\n\t})\n}\n\n// How long we wait for a response from a consumer create request for a source or mirror.\nvar srcConsumerWaitTime = 30 * time.Second\n\n// Setup our mirror consumer.\n// Lock should be held.\nfunc (mset *stream) setupMirrorConsumer() error {\n\tif mset.closed.Load() {\n\t\treturn errStreamClosed\n\t}\n\tif mset.outq == nil {\n\t\treturn errors.New(\"outq required\")\n\t}\n\t// We use to prevent update of a mirror configuration in cluster\n\t// mode but not in standalone. This is now fixed. However, without\n\t// rejecting the update, it could be that if the source stream was\n\t// removed and then later the mirrored stream config changed to\n\t// remove mirror configuration, this function would panic when\n\t// accessing mset.cfg.Mirror fields. Adding this protection in case\n\t// we allow in the future the mirror config to be changed (removed).\n\tif mset.cfg.Mirror == nil {\n\t\treturn errors.New(\"invalid mirror configuration\")\n\t}\n\n\t// If this is the first time\n\tif mset.mirror == nil {\n\t\tmset.mirror = &sourceInfo{name: mset.cfg.Mirror.Name}\n\t} else {\n\t\tmset.cancelSourceInfo(mset.mirror)\n\t\tmset.mirror.sseq = mset.lseq\n\t}\n\n\t// If we are no longer the leader stop trying.\n\tif !mset.isLeader() {\n\t\treturn nil\n\t}\n\n\tmirror := mset.mirror\n\n\t// We want to throttle here in terms of how fast we request new consumers,\n\t// or if the previous is still in progress.\n\tif last := time.Since(mirror.lreq); last < sourceConsumerRetryThreshold || mirror.sip {\n\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t\treturn nil\n\t}\n\tmirror.lreq = time.Now()\n\n\t// Determine subjects etc.\n\tvar deliverSubject string\n\text := mset.cfg.Mirror.External\n\n\tif ext != nil && ext.DeliverPrefix != _EMPTY_ {\n\t\tdeliverSubject = strings.ReplaceAll(ext.DeliverPrefix+syncSubject(\".M\"), \"..\", \".\")\n\t} else {\n\t\tdeliverSubject = syncSubject(\"$JS.M\")\n\t}\n\n\t// Now send off request to create/update our consumer. This will be all API based even in single server mode.\n\t// We calculate durable names apriori so we do not need to save them off.\n\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\n\treq := &CreateConsumerRequest{\n\t\tStream: mset.cfg.Mirror.Name,\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: deliverSubject,\n\t\t\tDeliverPolicy: DeliverByStartSequence,\n\t\t\tOptStartSeq: state.LastSeq + 1,\n\t\t\tAckPolicy: AckNone,\n\t\t\tAckWait: 22 * time.Hour,\n\t\t\tMaxDeliver: 1,\n\t\t\tHeartbeat: sourceHealthHB,\n\t\t\tFlowControl: true,\n\t\t\tDirect: true,\n\t\t\tInactiveThreshold: sourceHealthCheckInterval,\n\t\t},\n\t}\n\n\t// Only use start optionals on first time.\n\tif state.Msgs == 0 && state.FirstSeq == 0 {\n\t\treq.Config.OptStartSeq = 0\n\t\tif mset.cfg.Mirror.OptStartSeq > 0 {\n\t\t\treq.Config.OptStartSeq = mset.cfg.Mirror.OptStartSeq\n\t\t} else if mset.cfg.Mirror.OptStartTime != nil {\n\t\t\treq.Config.OptStartTime = mset.cfg.Mirror.OptStartTime\n\t\t\treq.Config.DeliverPolicy = DeliverByStartTime\n\t\t}\n\t}\n\tif req.Config.OptStartSeq == 0 && req.Config.OptStartTime == nil {\n\t\t// If starting out and lastSeq is 0.\n\t\treq.Config.DeliverPolicy = DeliverAll\n\t}\n\n\t// Filters\n\tif mset.cfg.Mirror.FilterSubject != _EMPTY_ {\n\t\treq.Config.FilterSubject = mset.cfg.Mirror.FilterSubject\n\t\tmirror.sf = mset.cfg.Mirror.FilterSubject\n\t}\n\n\tif lst := len(mset.cfg.Mirror.SubjectTransforms); lst > 0 {\n\t\tsfs := make([]string, lst)\n\t\ttrs := make([]*subjectTransform, lst)\n\n\t\tfor i, tr := range mset.cfg.Mirror.SubjectTransforms {\n\t\t\t// will not fail as already checked before that the transform will work\n\t\t\tsubjectTransform, err := NewSubjectTransform(tr.Source, tr.Destination)\n\t\t\tif err != nil {\n\t\t\t\tmset.srv.Errorf(\"Unable to get transform for mirror consumer: %v\", err)\n\t\t\t}\n\t\t\tsfs[i] = tr.Source\n\t\t\ttrs[i] = subjectTransform\n\t\t}\n\t\tmirror.sfs = sfs\n\t\tmirror.trs = trs\n\t\t// If there was no explicit FilterSubject defined and we have a single\n\t\t// subject transform, use Config.FilterSubject instead of FilterSubjects\n\t\t// so that we can use the extended consumer create API down below.\n\t\tif req.Config.FilterSubject == _EMPTY_ && len(sfs) == 1 {\n\t\t\treq.Config.FilterSubject = sfs[0]\n\t\t} else {\n\t\t\treq.Config.FilterSubjects = sfs\n\t\t}\n\t}\n\n\trespCh := make(chan *JSApiConsumerCreateResponse, 1)\n\treply := infoReplySubject()\n\tcrSub, err := mset.subscribeInternal(reply, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\t\tmset.unsubscribe(sub)\n\t\t_, msg := c.msgParts(rmsg)\n\n\t\tvar ccr JSApiConsumerCreateResponse\n\t\tif err := json.Unmarshal(msg, &ccr); err != nil {\n\t\t\tc.Warnf(\"JetStream bad mirror consumer create response: %q\", msg)\n\t\t\tmset.setMirrorErr(ApiErrors[JSInvalidJSONErr])\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase respCh <- &ccr:\n\t\tdefault:\n\t\t}\n\t})\n\tif err != nil {\n\t\tmirror.err = NewJSMirrorConsumerSetupFailedError(err, Unless(err))\n\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t\treturn nil\n\t}\n\n\tvar subject string\n\tif req.Config.FilterSubject != _EMPTY_ {\n\t\treq.Config.Name = fmt.Sprintf(\"mirror-%s\", createConsumerName())\n\t\tsubject = fmt.Sprintf(JSApiConsumerCreateExT, mset.cfg.Mirror.Name, req.Config.Name, req.Config.FilterSubject)\n\t} else {\n\t\tsubject = fmt.Sprintf(JSApiConsumerCreateT, mset.cfg.Mirror.Name)\n\t}\n\tif ext != nil {\n\t\tsubject = strings.Replace(subject, JSApiPrefix, ext.ApiPrefix, 1)\n\t\tsubject = strings.ReplaceAll(subject, \"..\", \".\")\n\t}\n\n\t// Marshal now that we are done with `req`.\n\tb, _ := json.Marshal(req)\n\n\t// Reset\n\tmirror.msgs = nil\n\tmirror.err = nil\n\tmirror.sip = true\n\n\t// Send the consumer create request\n\tmset.outq.send(newJSPubMsg(subject, _EMPTY_, reply, nil, b, nil, 0))\n\n\tgo func() {\n\n\t\tvar retry bool\n\t\tdefer func() {\n\t\t\tmset.mu.Lock()\n\t\t\t// Check that this is still valid and if so, clear the \"setup in progress\" flag.\n\t\t\tif mset.mirror != nil {\n\t\t\t\tmset.mirror.sip = false\n\t\t\t\t// If we need to retry, schedule now\n\t\t\t\t// If sub is not nil means we re-established somewhere else so do not re-attempt here.\n\t\t\t\tif retry && mset.mirror.sub == nil {\n\t\t\t\t\tmset.mirror.fails++\n\t\t\t\t\t// Cancel here since we can not do anything with this consumer at this point.\n\t\t\t\t\tmset.cancelSourceInfo(mset.mirror)\n\t\t\t\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t\t\t\t} else {\n\t\t\t\t\t// Clear on success.\n\t\t\t\t\tmset.mirror.fails = 0\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t}()\n\n\t\t// Wait for previous processMirrorMsgs go routine to be completely done.\n\t\t// If none is running, this will not block.\n\t\tmset.mu.Lock()\n\t\tif mset.mirror == nil {\n\t\t\t// Mirror config has been removed.\n\t\t\tmset.mu.Unlock()\n\t\t\treturn\n\t\t} else {\n\t\t\twg := &mset.mirror.wg\n\t\t\tmset.mu.Unlock()\n\t\t\twg.Wait()\n\t\t}\n\n\t\tselect {\n\t\tcase ccr := <-respCh:\n\t\t\tmset.mu.Lock()\n\t\t\t// Mirror config has been removed.\n\t\t\tif mset.mirror == nil {\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tready := sync.WaitGroup{}\n\t\t\tmirror := mset.mirror\n\t\t\tmirror.err = nil\n\t\t\tif ccr.Error != nil || ccr.ConsumerInfo == nil {\n\t\t\t\tmset.srv.Warnf(\"JetStream error response for create mirror consumer: %+v\", ccr.Error)\n\t\t\t\tmirror.err = ccr.Error\n\t\t\t\t// Let's retry as soon as possible, but we are gated by sourceConsumerRetryThreshold\n\t\t\t\tretry = true\n\t\t\t\tmset.mu.Unlock()\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\t// Setup actual subscription to process messages from our source.\n\t\t\t\tqname := fmt.Sprintf(\"[ACC:%s] stream mirror '%s' of '%s' msgs\", mset.acc.Name, mset.cfg.Name, mset.cfg.Mirror.Name)\n\t\t\t\t// Create a new queue each time\n\t\t\t\tmirror.msgs = newIPQueue[*inMsg](mset.srv, qname)\n\t\t\t\tmsgs := mirror.msgs\n\t\t\t\tsub, err := mset.subscribeInternal(deliverSubject, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\t\t\t\t\thdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy.\n\t\t\t\t\tif len(hdr) > 0 {\n\t\t\t\t\t\t// Remove any Nats-Expected- headers as we don't want to validate them.\n\t\t\t\t\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Expected-\")\n\t\t\t\t\t\t// Remove any Nats-Batch- headers, batching is not supported when mirroring.\n\t\t\t\t\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Batch-\")\n\t\t\t\t\t}\n\t\t\t\t\tmset.queueInbound(msgs, subject, reply, hdr, msg, nil, nil)\n\t\t\t\t\tmirror.last.Store(time.Now().UnixNano())\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\tmirror.err = NewJSMirrorConsumerSetupFailedError(err, Unless(err))\n\t\t\t\t\tretry = true\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\t// Save our sub.\n\t\t\t\tmirror.sub = sub\n\n\t\t\t\t// When an upstream stream expires messages or in general has messages that we want\n\t\t\t\t// that are no longer available we need to adjust here.\n\t\t\t\tvar state StreamState\n\t\t\t\tmset.store.FastState(&state)\n\n\t\t\t\t// Check if we need to skip messages.\n\t\t\t\tif state.LastSeq != ccr.ConsumerInfo.Delivered.Stream {\n\t\t\t\t\t// Check to see if delivered is past our last and we have no msgs. This will help the\n\t\t\t\t\t// case when mirroring a stream that has a very high starting sequence number.\n\t\t\t\t\tif state.Msgs == 0 && ccr.ConsumerInfo.Delivered.Stream > state.LastSeq {\n\t\t\t\t\t\tmset.store.PurgeEx(_EMPTY_, ccr.ConsumerInfo.Delivered.Stream+1, 0)\n\t\t\t\t\t\tmset.lseq = ccr.ConsumerInfo.Delivered.Stream\n\t\t\t\t\t} else {\n\t\t\t\t\t\tmset.skipMsgs(state.LastSeq+1, ccr.ConsumerInfo.Delivered.Stream)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Capture consumer name.\n\t\t\t\tmirror.cname = ccr.ConsumerInfo.Name\n\t\t\t\tmirror.dseq = 0\n\t\t\t\tmirror.sseq = ccr.ConsumerInfo.Delivered.Stream\n\t\t\t\tmirror.qch = make(chan struct{})\n\t\t\t\tmirror.wg.Add(1)\n\t\t\t\tready.Add(1)\n\t\t\t\tif !mset.srv.startGoRoutine(\n\t\t\t\t\tfunc() { mset.processMirrorMsgs(mirror, &ready) },\n\t\t\t\t\tpprofLabels{\n\t\t\t\t\t\t\"type\": \"mirror\",\n\t\t\t\t\t\t\"account\": mset.acc.Name,\n\t\t\t\t\t\t\"stream\": mset.cfg.Name,\n\t\t\t\t\t\t\"consumer\": mirror.cname,\n\t\t\t\t\t},\n\t\t\t\t) {\n\t\t\t\t\tmirror.wg.Done()\n\t\t\t\t\tready.Done()\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t\tready.Wait()\n\t\tcase <-time.After(srcConsumerWaitTime):\n\t\t\tmset.unsubscribe(crSub)\n\t\t\t// We already waited 30 seconds, let's retry now.\n\t\t\tretry = true\n\t\t}\n\t}()\n\n\treturn nil\n}\n\nfunc (mset *stream) streamSource(iname string) *StreamSource {\n\tfor _, ssi := range mset.cfg.Sources {\n\t\tif ssi.iname == iname {\n\t\t\treturn ssi\n\t\t}\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) retrySourceConsumerAtSeq(iName string, seq uint64) {\n\ts := mset.srv\n\n\ts.Debugf(\"Retrying source consumer for '%s > %s'\", mset.acc.Name, mset.cfg.Name)\n\n\t// setupSourceConsumer will check that the source is still configured.\n\tmset.setupSourceConsumer(iName, seq, time.Time{})\n}\n\n// Lock should be held.\nfunc (mset *stream) cancelSourceConsumer(iname string) {\n\tif si := mset.sources[iname]; si != nil {\n\t\tmset.cancelSourceInfo(si)\n\t\tsi.sseq, si.dseq = 0, 0\n\t}\n}\n\n// The `si` has been verified to be not nil. The sourceInfo's sub will\n// be unsubscribed and set to nil (if not already done) and the\n// cname will be reset. The message processing's go routine quit channel\n// will be closed if still opened.\n//\n// Lock should be held\nfunc (mset *stream) cancelSourceInfo(si *sourceInfo) {\n\tif si.sub != nil {\n\t\tmset.unsubscribe(si.sub)\n\t\tsi.sub = nil\n\t}\n\tmset.removeInternalConsumer(si)\n\tif si.qch != nil {\n\t\tclose(si.qch)\n\t\tsi.qch = nil\n\t}\n\tif si.msgs != nil {\n\t\tsi.msgs.drain()\n\t\tsi.msgs.unregister()\n\t}\n\t// If we have a schedule setup go ahead and delete that.\n\tif t := mset.sourceSetupSchedules[si.iname]; t != nil {\n\t\tt.Stop()\n\t\tdelete(mset.sourceSetupSchedules, si.iname)\n\t}\n}\n\nconst sourceConsumerRetryThreshold = 2 * time.Second\n\n// This is the main function to call when needing to setup a new consumer for the source.\n// It actually only does the scheduling of the execution of trySetupSourceConsumer in order to implement retry backoff\n// and throttle the number of requests.\n// Lock should be held.\nfunc (mset *stream) setupSourceConsumer(iname string, seq uint64, startTime time.Time) {\n\tif mset.sourceSetupSchedules == nil {\n\t\tmset.sourceSetupSchedules = map[string]*time.Timer{}\n\t}\n\n\tif _, ok := mset.sourceSetupSchedules[iname]; ok {\n\t\t// If there is already a timer scheduled, we don't need to do anything.\n\t\treturn\n\t}\n\n\tsi := mset.sources[iname]\n\tif si == nil || si.sip { // if sourceInfo was removed or setup is in progress, nothing to do\n\t\treturn\n\t}\n\n\t// First calculate the delay until the next time we can\n\tvar scheduleDelay time.Duration\n\n\tif !si.lreq.IsZero() { // it's not the very first time we are called, compute the delay\n\t\t// We want to throttle here in terms of how fast we request new consumers\n\t\tif sinceLast := time.Since(si.lreq); sinceLast < sourceConsumerRetryThreshold {\n\t\t\tscheduleDelay = sourceConsumerRetryThreshold - sinceLast\n\t\t}\n\t\t// Is it a retry? If so, add a backoff\n\t\tif si.fails > 0 {\n\t\t\tscheduleDelay += calculateRetryBackoff(si.fails)\n\t\t}\n\t}\n\n\t// Always add some jitter\n\tscheduleDelay += time.Duration(rand.Intn(int(100*time.Millisecond))) + 100*time.Millisecond\n\n\t// Schedule the call to trySetupSourceConsumer\n\tmset.sourceSetupSchedules[iname] = time.AfterFunc(scheduleDelay, func() {\n\t\tmset.mu.Lock()\n\t\tdefer mset.mu.Unlock()\n\n\t\tdelete(mset.sourceSetupSchedules, iname)\n\t\tmset.trySetupSourceConsumer(iname, seq, startTime)\n\t})\n}\n\n// This is where we will actually try to create a new consumer for the source\n// Lock should be held.\nfunc (mset *stream) trySetupSourceConsumer(iname string, seq uint64, startTime time.Time) {\n\t// Ignore if closed or not leader.\n\tif mset.closed.Load() || !mset.isLeader() {\n\t\treturn\n\t}\n\n\tsi := mset.sources[iname]\n\tif si == nil {\n\t\treturn\n\t}\n\n\t// Cancel previous instance if applicable\n\tmset.cancelSourceInfo(si)\n\n\tssi := mset.streamSource(iname)\n\tif ssi == nil {\n\t\treturn\n\t}\n\n\tsi.lreq = time.Now()\n\n\t// Determine subjects etc.\n\tvar deliverSubject string\n\text := ssi.External\n\n\tif ext != nil && ext.DeliverPrefix != _EMPTY_ {\n\t\tdeliverSubject = strings.ReplaceAll(ext.DeliverPrefix+syncSubject(\".S\"), \"..\", \".\")\n\t} else {\n\t\tdeliverSubject = syncSubject(\"$JS.S\")\n\t}\n\n\treq := &CreateConsumerRequest{\n\t\tStream: si.name,\n\t\tConfig: ConsumerConfig{\n\t\t\tDeliverSubject: deliverSubject,\n\t\t\tAckPolicy: AckNone,\n\t\t\tAckWait: 22 * time.Hour,\n\t\t\tMaxDeliver: 1,\n\t\t\tHeartbeat: sourceHealthHB,\n\t\t\tFlowControl: true,\n\t\t\tDirect: true,\n\t\t\tInactiveThreshold: sourceHealthCheckInterval,\n\t\t},\n\t}\n\n\t// If starting, check any configs.\n\tif !startTime.IsZero() && seq > 1 {\n\t\treq.Config.OptStartTime = &startTime\n\t\treq.Config.DeliverPolicy = DeliverByStartTime\n\t} else if seq <= 1 {\n\t\tif ssi.OptStartSeq > 0 {\n\t\t\treq.Config.OptStartSeq = ssi.OptStartSeq\n\t\t\treq.Config.DeliverPolicy = DeliverByStartSequence\n\t\t} else {\n\t\t\t// We have not recovered state so check that configured time is less that our first seq time.\n\t\t\tvar state StreamState\n\t\t\tmset.store.FastState(&state)\n\t\t\tif ssi.OptStartTime != nil {\n\t\t\t\tif !state.LastTime.IsZero() && ssi.OptStartTime.Before(state.LastTime) {\n\t\t\t\t\treq.Config.OptStartTime = &state.LastTime\n\t\t\t\t} else {\n\t\t\t\t\treq.Config.OptStartTime = ssi.OptStartTime\n\t\t\t\t}\n\t\t\t\treq.Config.DeliverPolicy = DeliverByStartTime\n\t\t\t} else if state.FirstSeq > 1 && !state.LastTime.IsZero() {\n\t\t\t\treq.Config.OptStartTime = &state.LastTime\n\t\t\t\treq.Config.DeliverPolicy = DeliverByStartTime\n\t\t\t}\n\t\t}\n\n\t} else {\n\t\treq.Config.OptStartSeq = seq\n\t\treq.Config.DeliverPolicy = DeliverByStartSequence\n\t}\n\t// Filters\n\tif ssi.FilterSubject != _EMPTY_ {\n\t\treq.Config.FilterSubject = ssi.FilterSubject\n\t}\n\n\tvar filterSubjects []string\n\tfor _, tr := range ssi.SubjectTransforms {\n\t\tfilterSubjects = append(filterSubjects, tr.Source)\n\t}\n\treq.Config.FilterSubjects = filterSubjects\n\n\trespCh := make(chan *JSApiConsumerCreateResponse, 1)\n\treply := infoReplySubject()\n\tcrSub, err := mset.subscribeInternal(reply, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\t\tmset.unsubscribe(sub)\n\t\t_, msg := c.msgParts(rmsg)\n\t\tvar ccr JSApiConsumerCreateResponse\n\t\tif err := json.Unmarshal(msg, &ccr); err != nil {\n\t\t\tc.Warnf(\"JetStream bad source consumer create response: %q\", msg)\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase respCh <- &ccr:\n\t\tdefault:\n\t\t}\n\t})\n\tif err != nil {\n\t\tsi.err = NewJSSourceConsumerSetupFailedError(err, Unless(err))\n\t\tmset.setupSourceConsumer(iname, seq, startTime)\n\t\treturn\n\t}\n\n\tvar subject string\n\tif req.Config.FilterSubject != _EMPTY_ {\n\t\treq.Config.Name = fmt.Sprintf(\"src-%s\", createConsumerName())\n\t\tsubject = fmt.Sprintf(JSApiConsumerCreateExT, si.name, req.Config.Name, req.Config.FilterSubject)\n\t} else if len(req.Config.FilterSubjects) == 1 {\n\t\treq.Config.Name = fmt.Sprintf(\"src-%s\", createConsumerName())\n\t\t// It is necessary to switch to using FilterSubject here as the extended consumer\n\t\t// create API checks for it, so as to not accidentally allow multiple filtered subjects.\n\t\treq.Config.FilterSubject = req.Config.FilterSubjects[0]\n\t\treq.Config.FilterSubjects = nil\n\t\tsubject = fmt.Sprintf(JSApiConsumerCreateExT, si.name, req.Config.Name, req.Config.FilterSubject)\n\t} else {\n\t\tsubject = fmt.Sprintf(JSApiConsumerCreateT, si.name)\n\t}\n\tif ext != nil {\n\t\tsubject = strings.Replace(subject, JSApiPrefix, ext.ApiPrefix, 1)\n\t\tsubject = strings.ReplaceAll(subject, \"..\", \".\")\n\t}\n\n\t// Marshal request.\n\tb, _ := json.Marshal(req)\n\n\t// Reset\n\tsi.msgs = nil\n\tsi.err = nil\n\tsi.sip = true\n\n\t// Send the consumer create request\n\tmset.outq.send(newJSPubMsg(subject, _EMPTY_, reply, nil, b, nil, 0))\n\n\tgo func() {\n\n\t\tvar retry bool\n\t\tdefer func() {\n\t\t\tmset.mu.Lock()\n\t\t\t// Check that this is still valid and if so, clear the \"setup in progress\" flag.\n\t\t\tif si := mset.sources[iname]; si != nil {\n\t\t\t\tsi.sip = false\n\t\t\t\t// If we need to retry, schedule now\n\t\t\t\t// If sub is not nil means we re-established somewhere else so do not re-attempt here.\n\t\t\t\tif retry && si.sub == nil {\n\t\t\t\t\tsi.fails++\n\t\t\t\t\t// Cancel here since we can not do anything with this consumer at this point.\n\t\t\t\t\tmset.cancelSourceInfo(si)\n\t\t\t\t\tmset.setupSourceConsumer(iname, seq, startTime)\n\t\t\t\t} else {\n\t\t\t\t\t// Clear on success.\n\t\t\t\t\tsi.fails = 0\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\t}()\n\n\t\tselect {\n\t\tcase ccr := <-respCh:\n\t\t\tmset.mu.Lock()\n\t\t\t// Check that it has not been removed or canceled (si.sub would be nil)\n\t\t\tif si := mset.sources[iname]; si != nil {\n\t\t\t\tsi.err = nil\n\t\t\t\tif ccr.Error != nil || ccr.ConsumerInfo == nil {\n\t\t\t\t\t// Note: this warning can happen a few times when starting up the server when sourcing streams are\n\t\t\t\t\t// defined, this is normal as the streams are re-created in no particular order and it is possible\n\t\t\t\t\t// that a stream sourcing another could come up before all of its sources have been recreated.\n\t\t\t\t\tmset.srv.Warnf(\"JetStream error response for stream %s create source consumer %s: %+v\", mset.cfg.Name, si.name, ccr.Error)\n\t\t\t\t\tsi.err = ccr.Error\n\t\t\t\t\t// Let's retry as soon as possible, but we are gated by sourceConsumerRetryThreshold\n\t\t\t\t\tretry = true\n\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t} else {\n\t\t\t\t\t// Check if our shared msg queue and go routine is running or not.\n\t\t\t\t\tif mset.smsgs == nil {\n\t\t\t\t\t\tqname := fmt.Sprintf(\"[ACC:%s] stream sources '%s' msgs\", mset.acc.Name, mset.cfg.Name)\n\t\t\t\t\t\tmset.smsgs = newIPQueue[*inMsg](mset.srv, qname)\n\t\t\t\t\t\tmset.srv.startGoRoutine(func() { mset.processAllSourceMsgs() },\n\t\t\t\t\t\t\tpprofLabels{\n\t\t\t\t\t\t\t\t\"type\": \"source\",\n\t\t\t\t\t\t\t\t\"account\": mset.acc.Name,\n\t\t\t\t\t\t\t\t\"stream\": mset.cfg.Name,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\n\t\t\t\t\t// Setup actual subscription to process messages from our source.\n\t\t\t\t\tif si.sseq != ccr.ConsumerInfo.Delivered.Stream {\n\t\t\t\t\t\tsi.sseq = ccr.ConsumerInfo.Delivered.Stream + 1\n\t\t\t\t\t}\n\t\t\t\t\t// Capture consumer name.\n\t\t\t\t\tsi.cname = ccr.ConsumerInfo.Name\n\n\t\t\t\t\t// Do not set si.sseq to seq here. si.sseq will be set in processInboundSourceMsg\n\t\t\t\t\tsi.dseq = 0\n\t\t\t\t\tsi.qch = make(chan struct{})\n\t\t\t\t\t// Set the last seen as now so that we don't fail at the first check.\n\t\t\t\t\tsi.last.Store(time.Now().UnixNano())\n\n\t\t\t\t\tmsgs := mset.smsgs\n\t\t\t\t\tsub, err := mset.subscribeInternal(deliverSubject, func(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\t\t\t\t\t\thdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy.\n\t\t\t\t\t\tmset.queueInbound(msgs, subject, reply, hdr, msg, si, nil)\n\t\t\t\t\t\tsi.last.Store(time.Now().UnixNano())\n\t\t\t\t\t})\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tsi.err = NewJSSourceConsumerSetupFailedError(err, Unless(err))\n\t\t\t\t\t\tretry = true\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\t// Save our sub.\n\t\t\t\t\tsi.sub = sub\n\t\t\t\t}\n\t\t\t}\n\t\t\tmset.mu.Unlock()\n\t\tcase <-time.After(srcConsumerWaitTime):\n\t\t\tmset.unsubscribe(crSub)\n\t\t\t// We already waited 30 seconds, let's retry now.\n\t\t\tretry = true\n\t\t}\n\t}()\n}\n\n// This will process all inbound source msgs.\n// We mux them into one go routine to avoid lock contention and high cpu and thread thrashing.\n// TODO(dlc) make this more then one and pin sources to one of a group.\nfunc (mset *stream) processAllSourceMsgs() {\n\ts := mset.srv\n\tdefer s.grWG.Done()\n\n\tmset.mu.RLock()\n\tmsgs, qch := mset.smsgs, mset.qch\n\tmset.mu.RUnlock()\n\n\tt := time.NewTicker(sourceHealthCheckInterval)\n\tdefer t.Stop()\n\n\t// When we detect we are no longer leader, we will cleanup.\n\t// Should always return right after this is called.\n\tcleanUp := func() {\n\t\tmset.mu.Lock()\n\t\tdefer mset.mu.Unlock()\n\t\tfor _, si := range mset.sources {\n\t\t\tmset.cancelSourceConsumer(si.iname)\n\t\t}\n\t\tmset.smsgs.drain()\n\t\tmset.smsgs.unregister()\n\t\tmset.smsgs = nil\n\t}\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-msgs.ch:\n\t\t\tims := msgs.pop()\n\t\t\tfor _, im := range ims {\n\t\t\t\tif !mset.processInboundSourceMsg(im.si, im) {\n\t\t\t\t\t// If we are no longer leader bail.\n\t\t\t\t\tif !mset.IsLeader() {\n\t\t\t\t\t\tmsgs.recycle(&ims)\n\t\t\t\t\t\tcleanUp()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tim.returnToPool()\n\t\t\t}\n\t\t\tmsgs.recycle(&ims)\n\t\tcase <-t.C:\n\t\t\t// If we are no longer leader bail.\n\t\t\tif !mset.IsLeader() {\n\t\t\t\tcleanUp()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Check health of all sources.\n\t\t\tvar stalled []*sourceInfo\n\t\t\tmset.mu.RLock()\n\t\t\tfor _, si := range mset.sources {\n\t\t\t\tif time.Since(time.Unix(0, si.last.Load())) > sourceHealthCheckInterval {\n\t\t\t\t\tstalled = append(stalled, si)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnumSources := len(mset.sources)\n\t\t\tmset.mu.RUnlock()\n\n\t\t\t// This can happen on an update when no longer have sources.\n\t\t\tif numSources == 0 {\n\t\t\t\tcleanUp()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// We do not want to block here so do in separate Go routine.\n\t\t\tif len(stalled) > 0 {\n\t\t\t\tgo func() {\n\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t\tdefer mset.mu.Unlock()\n\t\t\t\t\tfor _, si := range stalled {\n\t\t\t\t\t\tmset.setupSourceConsumer(si.iname, si.sseq+1, time.Time{})\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\t\t}\n\t}\n}\n\n// isControlMsg determines if this is a control message.\nfunc (m *inMsg) isControlMsg() bool {\n\treturn len(m.msg) == 0 && len(m.hdr) > 0 && bytes.HasPrefix(m.hdr, []byte(\"NATS/1.0 100 \"))\n}\n\n// Sends a reply to a flow control request.\n// Lock should be held.\nfunc (mset *stream) sendFlowControlReply(reply string) {\n\tif mset.isLeader() && mset.outq != nil {\n\t\tmset.outq.sendMsg(reply, nil)\n\t}\n}\n\n// handleFlowControl will properly handle flow control messages for both R==1 and R>1.\n// Lock should be held.\nfunc (mset *stream) handleFlowControl(m *inMsg) {\n\t// If we are clustered we will send the flow control message through the replication stack.\n\tif mset.isClustered() {\n\t\tmset.node.Propose(encodeStreamMsg(_EMPTY_, m.rply, m.hdr, nil, 0, 0, false))\n\t} else {\n\t\tmset.outq.sendMsg(m.rply, nil)\n\t}\n}\n\n// processInboundSourceMsg handles processing other stream messages bound for this stream.\nfunc (mset *stream) processInboundSourceMsg(si *sourceInfo, m *inMsg) bool {\n\tmset.mu.Lock()\n\t// If we are no longer the leader cancel this subscriber.\n\tif !mset.isLeader() {\n\t\tmset.cancelSourceConsumer(si.iname)\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\tisControl := m.isControlMsg()\n\n\t// Ignore from old subscriptions.\n\tif !si.isCurrentSub(m.rply) && !isControl {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Check for heartbeats and flow control messages.\n\tif isControl {\n\t\tvar needsRetry bool\n\t\t// Flow controls have reply subjects.\n\t\tif m.rply != _EMPTY_ {\n\t\t\tmset.handleFlowControl(m)\n\t\t} else {\n\t\t\t// For idle heartbeats make sure we did not miss anything.\n\t\t\tif ldseq := parseInt64(getHeader(JSLastConsumerSeq, m.hdr)); ldseq > 0 && uint64(ldseq) != si.dseq {\n\t\t\t\tneedsRetry = true\n\t\t\t\tmset.retrySourceConsumerAtSeq(si.iname, si.sseq+1)\n\t\t\t} else if fcReply := getHeader(JSConsumerStalled, m.hdr); len(fcReply) > 0 {\n\t\t\t\t// Other side thinks we are stalled, so send flow control reply.\n\t\t\t\tmset.outq.sendMsg(string(fcReply), nil)\n\t\t\t}\n\t\t}\n\t\tmset.mu.Unlock()\n\t\treturn !needsRetry\n\t}\n\n\tsseq, dseq, dc, _, pending := replyInfo(m.rply)\n\n\tif dc > 1 {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// Tracking is done here.\n\tif dseq == si.dseq+1 {\n\t\tsi.dseq++\n\t\tsi.sseq = sseq\n\t} else if dseq > si.dseq {\n\t\tif si.cname == _EMPTY_ {\n\t\t\tsi.cname = tokenAt(m.rply, 4)\n\t\t\tsi.dseq, si.sseq = dseq, sseq\n\t\t} else {\n\t\t\tmset.retrySourceConsumerAtSeq(si.iname, si.sseq+1)\n\t\t\tmset.mu.Unlock()\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\tif pending == 0 {\n\t\tsi.lag = 0\n\t} else {\n\t\tsi.lag = pending - 1\n\t}\n\tnode := mset.node\n\tmset.mu.Unlock()\n\n\thdr, msg := m.hdr, m.msg\n\n\t// If we are daisy chained here make sure to remove the original one.\n\tif len(hdr) > 0 {\n\t\thdr = removeHeaderIfPresent(hdr, JSStreamSource)\n\t\t// Remove any Nats-Expected- headers as we don't want to validate them.\n\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Expected-\")\n\t\t// Remove any Nats-Batch- headers, batching is not supported when sourcing.\n\t\thdr = removeHeaderIfPrefixPresent(hdr, \"Nats-Batch-\")\n\t}\n\t// Hold onto the origin reply which has all the metadata.\n\thdr = genHeader(hdr, JSStreamSource, si.genSourceHeader(m.subj, m.rply))\n\n\t// Do the subject transform for the source if there's one\n\tif len(si.trs) > 0 {\n\t\tfor _, tr := range si.trs {\n\t\t\tif tr == nil {\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\ttsubj, err := tr.Match(m.subj)\n\t\t\t\tif err == nil {\n\t\t\t\t\tm.subj = tsubj\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tvar err error\n\t// If we are clustered we need to propose this message to the underlying raft group.\n\tif node != nil {\n\t\terr = mset.processClusteredInboundMsg(m.subj, _EMPTY_, hdr, msg, nil, true)\n\t} else {\n\t\terr = mset.processJetStreamMsg(m.subj, _EMPTY_, hdr, msg, 0, 0, nil, true, true)\n\t}\n\tif err != nil {\n\t\ts := mset.srv\n\t\tif strings.Contains(err.Error(), \"no space left\") {\n\t\t\ts.Errorf(\"JetStream out of space, will be DISABLED\")\n\t\t\ts.DisableJetStream()\n\t\t} else {\n\t\t\tmset.mu.RLock()\n\t\t\taccName, sname, iName := mset.acc.Name, mset.cfg.Name, si.iname\n\t\t\tmset.mu.RUnlock()\n\t\t\t// Can happen temporarily all the time during normal operations when the sourcing stream is discard new\n\t\t\t// (example use case is for sourcing into a work queue)\n\t\t\t// TODO - Maybe improve sourcing to WQ with limit and new to use flow control rather than re-creating the consumer.\n\t\t\tif errors.Is(err, ErrMaxMsgs) || errors.Is(err, ErrMaxBytes) || errors.Is(err, ErrMaxMsgsPerSubject) {\n\t\t\t\t// Do not need to do a full retry that includes finding the last sequence in the stream\n\t\t\t\t// for that source. Just re-create starting with the seq we couldn't store instead.\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.retrySourceConsumerAtSeq(iName, si.sseq)\n\t\t\t\tmset.mu.Unlock()\n\t\t\t} else {\n\t\t\t\t// Log some warning for errors other than errLastSeqMismatch.\n\t\t\t\tif !errors.Is(err, errLastSeqMismatch) && !errors.Is(err, errMsgIdDuplicate) {\n\t\t\t\t\ts.RateLimitWarnf(\"Error processing inbound source %q for '%s' > '%s': %v\",\n\t\t\t\t\t\tiName, accName, sname, err)\n\t\t\t\t}\n\t\t\t\t// Retry in all type of errors we do not want to skip if we are still leader.\n\t\t\t\tif mset.isLeader() {\n\t\t\t\t\tif !errors.Is(err, errMsgIdDuplicate) {\n\t\t\t\t\t\t// This will make sure the source is still in mset.sources map,\n\t\t\t\t\t\t// find the last sequence and then call setupSourceConsumer.\n\t\t\t\t\t\tiNameMap := map[string]struct{}{iName: {}}\n\t\t\t\t\t\tmset.setStartingSequenceForSources(iNameMap)\n\t\t\t\t\t\tmset.mu.Lock()\n\t\t\t\t\t\tmset.retrySourceConsumerAtSeq(iName, si.sseq+1)\n\t\t\t\t\t\tmset.mu.Unlock()\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// skipping the message but keep processing the rest of the batch\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\n\treturn true\n}\n\n// Generate a new (2.10) style source header (stream name, sequence number, source filter, source destination transform).\nfunc (si *sourceInfo) genSourceHeader(orig, reply string) string {\n\tvar b strings.Builder\n\tiNameParts := strings.Split(si.iname, \" \")\n\n\tb.WriteString(iNameParts[0])\n\tb.WriteByte(' ')\n\t// Grab sequence as text here from reply subject.\n\tvar tsa [expectedNumReplyTokens]string\n\tstart, tokens := 0, tsa[:0]\n\tfor i := 0; i < len(reply); i++ {\n\t\tif reply[i] == btsep {\n\t\t\ttokens, start = append(tokens, reply[start:i]), i+1\n\t\t}\n\t}\n\ttokens = append(tokens, reply[start:])\n\tseq := \"1\" // Default\n\tif len(tokens) == expectedNumReplyTokens && tokens[0] == \"$JS\" && tokens[1] == \"ACK\" {\n\t\tseq = tokens[5]\n\t}\n\tb.WriteString(seq)\n\n\tb.WriteByte(' ')\n\tb.WriteString(iNameParts[1])\n\tb.WriteByte(' ')\n\tb.WriteString(iNameParts[2])\n\tb.WriteByte(' ')\n\tb.WriteString(orig)\n\treturn b.String()\n}\n\n// Original version of header that stored ack reply direct.\nfunc streamAndSeqFromAckReply(reply string) (string, string, uint64) {\n\ttsa := [expectedNumReplyTokens]string{}\n\tstart, tokens := 0, tsa[:0]\n\tfor i := 0; i < len(reply); i++ {\n\t\tif reply[i] == btsep {\n\t\t\ttokens, start = append(tokens, reply[start:i]), i+1\n\t\t}\n\t}\n\ttokens = append(tokens, reply[start:])\n\tif len(tokens) != expectedNumReplyTokens || tokens[0] != \"$JS\" || tokens[1] != \"ACK\" {\n\t\treturn _EMPTY_, _EMPTY_, 0\n\t}\n\treturn tokens[2], _EMPTY_, uint64(parseAckReplyNum(tokens[5]))\n}\n\n// Extract the stream name, the source index name and the message sequence number from the source header.\n// Uses the filter and transform arguments to provide backwards compatibility\nfunc streamAndSeq(shdr string) (string, string, uint64) {\n\tif strings.HasPrefix(shdr, jsAckPre) {\n\t\treturn streamAndSeqFromAckReply(shdr)\n\t}\n\t// New version which is stream index name sequence\n\tfields := strings.Split(shdr, \" \")\n\tnFields := len(fields)\n\n\tif nFields != 2 && nFields <= 3 {\n\t\treturn _EMPTY_, _EMPTY_, 0\n\t}\n\n\tif nFields >= 4 {\n\t\treturn fields[0], strings.Join([]string{fields[0], fields[2], fields[3]}, \" \"), uint64(parseAckReplyNum(fields[1]))\n\t} else {\n\t\treturn fields[0], _EMPTY_, uint64(parseAckReplyNum(fields[1]))\n\t}\n\n}\n\n// Lock should be held.\nfunc (mset *stream) setStartingSequenceForSources(iNames map[string]struct{}) {\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\n\t// Do not reset sseq here so we can remember when purge/expiration happens.\n\tif state.Msgs == 0 {\n\t\tfor iName := range iNames {\n\t\t\tsi := mset.sources[iName]\n\t\t\tif si == nil {\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tsi.dseq = 0\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n\n\t// From the provided list of sources, we build a sublist that contains\n\t// the interested filters (including transforms). As we figure out the\n\t// starting sequence for each source, we will eliminate the source from\n\t// the map and then refresh the sublist, which in turn makes the sublist\n\t// ideally more specific. This allows LoadPrevMsgsMulti to work most\n\t// effectively.\n\t// Because this is a SimpleSublist we can't just remove the entries per\n\t// source so we have no other option but to rebuild it from scratch, but\n\t// this is cheap enough to do so not the end of the world.\n\tvar sl *gsl.SimpleSublist\n\trefreshSublist := func() {\n\t\tsl = gsl.NewSimpleSublist()\n\t\tfor iName := range iNames {\n\t\t\tsi := mset.sources[iName]\n\t\t\tif si == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif si.sf == _EMPTY_ {\n\t\t\t\tsl.Insert(fwcs, struct{}{})\n\t\t\t} else {\n\t\t\t\tsl.Insert(si.sf, struct{}{})\n\t\t\t}\n\t\t\tfor _, sf := range si.sfs {\n\t\t\t\tif sf == _EMPTY_ {\n\t\t\t\t\tsl.Insert(fwcs, struct{}{})\n\t\t\t\t} else {\n\t\t\t\t\tsl.Insert(sf, struct{}{})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\trefreshSublist()\n\n\tvar smv StoreMsg\n\tfor last := state.LastSeq; ; {\n\t\tsm, seq, err := mset.store.LoadPrevMsgMulti(sl, last, &smv)\n\t\tif err == ErrStoreEOF || err != nil {\n\t\t\tbreak\n\t\t}\n\t\tlast = seq - 1\n\t\tif len(sm.hdr) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tss := sliceHeader(JSStreamSource, sm.hdr)\n\t\tif len(ss) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tstreamName, indexName, sseq := streamAndSeq(bytesToString(ss))\n\t\tif _, ok := iNames[indexName]; ok {\n\t\t\tsi := mset.sources[indexName]\n\t\t\tsi.sseq = sseq\n\t\t\tsi.dseq = 0\n\t\t\tdelete(iNames, indexName)\n\t\t\trefreshSublist()\n\t\t} else if indexName == _EMPTY_ && streamName != _EMPTY_ {\n\t\t\tfor iName := range iNames {\n\t\t\t\t// TODO streamSource is a linear walk, to optimize later\n\t\t\t\tif si := mset.sources[iName]; si != nil && streamName == si.name ||\n\t\t\t\t\t(mset.streamSource(iName).External != nil && streamName == si.name+\":\"+getHash(mset.streamSource(iName).External.ApiPrefix)) {\n\t\t\t\t\tsi.sseq = sseq\n\t\t\t\t\tsi.dseq = 0\n\t\t\t\t\tdelete(iNames, iName)\n\t\t\t\t\trefreshSublist()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(iNames) == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// Resets the SourceInfo for all the sources\n// lock should be held.\nfunc (mset *stream) resetSourceInfo() {\n\t// Reset if needed.\n\tmset.stopSourceConsumers()\n\tmset.sources = make(map[string]*sourceInfo)\n\n\tfor _, ssi := range mset.cfg.Sources {\n\t\tif ssi.iname == _EMPTY_ {\n\t\t\tssi.setIndexName()\n\t\t}\n\n\t\tvar si *sourceInfo\n\n\t\tif len(ssi.SubjectTransforms) == 0 {\n\t\t\tsi = &sourceInfo{name: ssi.Name, iname: ssi.iname, sf: ssi.FilterSubject}\n\t\t} else {\n\t\t\tsfs := make([]string, len(ssi.SubjectTransforms))\n\t\t\ttrs := make([]*subjectTransform, len(ssi.SubjectTransforms))\n\t\t\tfor i, str := range ssi.SubjectTransforms {\n\t\t\t\ttr, err := NewSubjectTransform(str.Source, str.Destination)\n\t\t\t\tif err != nil {\n\t\t\t\t\tmset.srv.Errorf(\"Unable to get subject transform for source: %v\", err)\n\t\t\t\t}\n\t\t\t\tsfs[i] = str.Source\n\t\t\t\ttrs[i] = tr\n\t\t\t}\n\t\t\tsi = &sourceInfo{name: ssi.Name, iname: ssi.iname, sfs: sfs, trs: trs}\n\t\t}\n\t\tmset.sources[ssi.iname] = si\n\t}\n}\n\n// This will do a reverse scan on startup or leader election\n// searching for the starting sequence number.\n// This can be slow in degenerative cases.\n// Lock should be held.\nfunc (mset *stream) startingSequenceForSources() {\n\tif len(mset.cfg.Sources) == 0 {\n\t\treturn\n\t}\n\n\t// Always reset here.\n\tmset.resetSourceInfo()\n\n\tvar state StreamState\n\tmset.store.FastState(&state)\n\n\t// Bail if no messages, meaning no context.\n\tif state.Msgs == 0 {\n\t\treturn\n\t}\n\n\t// For short circuiting return.\n\texpected := len(mset.cfg.Sources)\n\tseqs := make(map[string]uint64)\n\n\t// Stamp our si seq records on the way out.\n\tdefer func() {\n\t\tfor sname, seq := range seqs {\n\t\t\t// Ignore if not set.\n\t\t\tif seq == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif si := mset.sources[sname]; si != nil {\n\t\t\t\tsi.sseq = seq\n\t\t\t\tsi.dseq = 0\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Generate a list of sources and, from that, a sublist that contains\n\t// the interested filters (including transforms). As we figure out the\n\t// starting sequence for each source, we will eliminate the source from\n\t// the map and then refresh the sublist, which in turn makes the sublist\n\t// ideally more specific. This allows LoadPrevMsgsMulti to work most\n\t// effectively.\n\t// Because this is a SimpleSublist we can't just remove the entries per\n\t// source so we have no other option but to rebuild it from scratch, but\n\t// this is cheap enough to do so not the end of the world.\n\tsources := map[string]*StreamSource{}\n\tfor _, src := range mset.cfg.Sources {\n\t\tsources[src.composeIName()] = src\n\t}\n\tvar sl *gsl.SimpleSublist\n\trefreshSublist := func() {\n\t\tsl = gsl.NewSimpleSublist()\n\t\tfor _, src := range sources {\n\t\t\tif src.FilterSubject == _EMPTY_ {\n\t\t\t\tsl.Insert(fwcs, struct{}{})\n\t\t\t} else {\n\t\t\t\tsl.Insert(src.FilterSubject, struct{}{})\n\t\t\t}\n\t\t\tfor _, tr := range src.SubjectTransforms {\n\t\t\t\tif tr.Destination == _EMPTY_ {\n\t\t\t\t\tsl.Insert(fwcs, struct{}{})\n\t\t\t\t} else {\n\t\t\t\t\tsl.Insert(tr.Destination, struct{}{})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\trefreshSublist()\n\n\tupdate := func(iName string, seq uint64) {\n\t\t// Only update active in case we have older ones in here that got configured out.\n\t\tif si := mset.sources[iName]; si != nil {\n\t\t\tif _, ok := seqs[iName]; !ok {\n\t\t\t\tseqs[iName] = seq\n\t\t\t\tdelete(sources, iName)\n\t\t\t\trefreshSublist()\n\t\t\t}\n\t\t}\n\t}\n\n\tvar smv StoreMsg\n\tfor last := state.LastSeq; ; {\n\t\tsm, seq, err := mset.store.LoadPrevMsgMulti(sl, last, &smv)\n\t\tif err == ErrStoreEOF || err != nil {\n\t\t\tbreak\n\t\t}\n\t\tlast = seq - 1\n\t\tif len(sm.hdr) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tss := sliceHeader(JSStreamSource, sm.hdr)\n\t\tif len(ss) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tstreamName, iName, sseq := streamAndSeq(bytesToString(ss))\n\t\tif iName == _EMPTY_ { // Pre-2.10 message header means it's a match for any source using that stream name\n\t\t\tfor _, ssi := range mset.cfg.Sources {\n\t\t\t\tif streamName == ssi.Name || (ssi.External != nil && streamName == ssi.Name+\":\"+getHash(ssi.External.ApiPrefix)) {\n\t\t\t\t\tupdate(ssi.iname, sseq)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tupdate(iName, sseq)\n\t\t}\n\t\tif len(seqs) == expected {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Setup our source consumers.\n// Lock should be held.\nfunc (mset *stream) setupSourceConsumers() error {\n\tif mset.outq == nil {\n\t\treturn errors.New(\"outq required\")\n\t}\n\t// Reset if needed.\n\tfor _, si := range mset.sources {\n\t\tif si.sub != nil {\n\t\t\tmset.cancelSourceConsumer(si.iname)\n\t\t}\n\t}\n\n\t// If we are no longer the leader, give up\n\tif !mset.isLeader() {\n\t\treturn nil\n\t}\n\n\tmset.startingSequenceForSources()\n\n\t// Setup our consumers at the proper starting position.\n\tfor _, ssi := range mset.cfg.Sources {\n\t\tif si := mset.sources[ssi.iname]; si != nil {\n\t\t\tmset.setupSourceConsumer(ssi.iname, si.sseq+1, time.Time{})\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Will create internal subscriptions for the stream.\n// Lock should be held.\nfunc (mset *stream) subscribeToStream() error {\n\tif mset.active {\n\t\treturn nil\n\t}\n\tfor _, subject := range mset.cfg.Subjects {\n\t\tif _, err := mset.subscribeInternal(subject, mset.processInboundJetStreamMsg); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Check if we need to setup mirroring.\n\tif mset.cfg.Mirror != nil {\n\t\t// setup the initial mirror sourceInfo\n\t\tmset.mirror = &sourceInfo{name: mset.cfg.Mirror.Name}\n\t\tsfs := make([]string, len(mset.cfg.Mirror.SubjectTransforms))\n\t\ttrs := make([]*subjectTransform, len(mset.cfg.Mirror.SubjectTransforms))\n\n\t\tfor i, tr := range mset.cfg.Mirror.SubjectTransforms {\n\t\t\t// will not fail as already checked before that the transform will work\n\t\t\tsubjectTransform, err := NewSubjectTransform(tr.Source, tr.Destination)\n\t\t\tif err != nil {\n\t\t\t\tmset.srv.Errorf(\"Unable to get transform for mirror consumer: %v\", err)\n\t\t\t}\n\n\t\t\tsfs[i] = tr.Source\n\t\t\ttrs[i] = subjectTransform\n\t\t}\n\t\tmset.mirror.sfs = sfs\n\t\tmset.mirror.trs = trs\n\t\t// delay the actual mirror consumer creation for after a delay\n\t\tmset.scheduleSetupMirrorConsumerRetry()\n\t} else if len(mset.cfg.Sources) > 0 && mset.sourcesConsumerSetup == nil {\n\t\t// Setup the initial source infos for the sources\n\t\tmset.resetSourceInfo()\n\t\t// Delay the actual source consumer(s) creation(s) for after a delay if a replicated stream.\n\t\t// If it's an R1, this is done at startup and we will do inline.\n\t\tif mset.cfg.Replicas == 1 {\n\t\t\tmset.setupSourceConsumers()\n\t\t} else {\n\t\t\tmset.sourcesConsumerSetup = time.AfterFunc(time.Duration(rand.Intn(int(500*time.Millisecond)))+100*time.Millisecond, func() {\n\t\t\t\tmset.mu.Lock()\n\t\t\t\tmset.setupSourceConsumers()\n\t\t\t\tmset.mu.Unlock()\n\t\t\t})\n\t\t}\n\t}\n\t// Check for direct get access.\n\t// We spin up followers for clustered streams in monitorStream().\n\tif mset.cfg.AllowDirect {\n\t\tif err := mset.subscribeToDirect(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tmset.active = true\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) subscribeToDirect() error {\n\t// We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis.\n\tif mset.directSub == nil {\n\t\tdsubj := fmt.Sprintf(JSDirectMsgGetT, mset.cfg.Name)\n\t\tif sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetRequest); err == nil {\n\t\t\tmset.directSub = sub\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Now the one that will have subject appended past stream name.\n\tif mset.lastBySub == nil {\n\t\tdsubj := fmt.Sprintf(JSDirectGetLastBySubjectT, mset.cfg.Name, fwcs)\n\t\t// We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis.\n\t\tif sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetLastBySubjectRequest); err == nil {\n\t\t\tmset.lastBySub = sub\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) unsubscribeToDirect() {\n\tif mset.directSub != nil {\n\t\tmset.unsubscribe(mset.directSub)\n\t\tmset.directSub = nil\n\t}\n\tif mset.lastBySub != nil {\n\t\tmset.unsubscribe(mset.lastBySub)\n\t\tmset.lastBySub = nil\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) subscribeToMirrorDirect() error {\n\tif mset.cfg.Mirror == nil {\n\t\treturn nil\n\t}\n\tmirrorName := mset.cfg.Mirror.Name\n\n\t// We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis.\n\tif mset.mirrorDirectSub == nil {\n\t\tdsubj := fmt.Sprintf(JSDirectMsgGetT, mirrorName)\n\t\t// We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis.\n\t\tif sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetRequest); err == nil {\n\t\t\tmset.mirrorDirectSub = sub\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Now the one that will have subject appended past stream name.\n\tif mset.mirrorLastBySub == nil {\n\t\tdsubj := fmt.Sprintf(JSDirectGetLastBySubjectT, mirrorName, fwcs)\n\t\t// We will make this listen on a queue group by default, which can allow mirrors to participate on opt-in basis.\n\t\tif sub, err := mset.queueSubscribeInternal(dsubj, dgetGroup, mset.processDirectGetLastBySubjectRequest); err == nil {\n\t\t\tmset.mirrorLastBySub = sub\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) unsubscribeToMirrorDirect() {\n\tif mset.mirrorDirectSub != nil {\n\t\tmset.unsubscribe(mset.mirrorDirectSub)\n\t\tmset.mirrorDirectSub = nil\n\t}\n\tif mset.mirrorLastBySub != nil {\n\t\tmset.unsubscribe(mset.mirrorLastBySub)\n\t\tmset.mirrorLastBySub = nil\n\t}\n}\n\n// Stop our source consumers.\n// Lock should be held.\nfunc (mset *stream) stopSourceConsumers() {\n\tfor _, si := range mset.sources {\n\t\tmset.cancelSourceInfo(si)\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) removeInternalConsumer(si *sourceInfo) {\n\tif si == nil || si.cname == _EMPTY_ {\n\t\treturn\n\t}\n\tsi.cname = _EMPTY_\n}\n\n// Will unsubscribe from the stream.\n// Lock should be held.\nfunc (mset *stream) unsubscribeToStream(stopping, shuttingDown bool) error {\n\tfor _, subject := range mset.cfg.Subjects {\n\t\tmset.unsubscribeInternal(subject)\n\t}\n\tif mset.mirror != nil {\n\t\tmset.cancelSourceInfo(mset.mirror)\n\t\tmset.mirror = nil\n\t}\n\n\tif len(mset.sources) > 0 {\n\t\tmset.stopSourceConsumers()\n\t}\n\t// Clear batching state.\n\tmset.deleteAtomicBatches(shuttingDown)\n\tif stopping || shuttingDown {\n\t\tmset.deleteFastBatches()\n\t}\n\tif mset.batches != nil {\n\t\tmset.batches.mu.Lock()\n\t\treset := len(mset.batches.atomic) == 0 && len(mset.batches.fast) == 0\n\t\tmset.batches.mu.Unlock()\n\t\tif reset {\n\t\t\tmset.batches = nil\n\t\t}\n\t}\n\n\tif stopping {\n\t\t// In case we had a direct get subscriptions.\n\t\tmset.unsubscribeToDirect()\n\t\tmset.unsubscribeToMirrorDirect()\n\t}\n\n\tif mset.directLeaderSub == nil {\n\t\t// Always unsubscribe the leader sub.\n\t\tmset.unsubscribe(mset.directLeaderSub)\n\t\tmset.directLeaderSub = nil\n\t}\n\n\tmset.active = false\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) deleteAtomicBatches(shuttingDown bool) {\n\tif mset.batches != nil {\n\t\tmset.batches.mu.Lock()\n\t\tfor batchId, b := range mset.batches.atomic {\n\t\t\t// If shutting down, do fixup during startup. In-memory batches don't require manual cleanup.\n\t\t\tif shuttingDown {\n\t\t\t\tb.stopLocked()\n\t\t\t} else {\n\t\t\t\tb.cleanupLocked(batchId, mset.batches)\n\t\t\t}\n\t\t}\n\t\tmset.batches.atomic = nil\n\t\tmset.batches.mu.Unlock()\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) deleteBatchApplyState() {\n\tif batch := mset.batchApply; batch != nil {\n\t\t// Need to return entries (if any) to the pool.\n\t\tfor _, bce := range batch.entries {\n\t\t\tbce.ReturnToPool()\n\t\t}\n\t\tmset.batchApply = nil\n\t}\n}\n\n// Lock should be held.\nfunc (mset *stream) deleteFastBatches() {\n\tif mset.batches != nil {\n\t\tmset.batches.mu.Lock()\n\t\tfor batchId, b := range mset.batches.fast {\n\t\t\tb.cleanupLocked(batchId, mset.batches)\n\t\t}\n\t\tmset.batches.fast = nil\n\t\tmset.batches.mu.Unlock()\n\t}\n}\n\n// Lock does NOT need to be held, we set the client on setup and never change it at this point.\nfunc (mset *stream) subscribeInternal(subject string, cb msgHandler) (*subscription, error) {\n\tif mset.closed.Load() {\n\t\treturn nil, errStreamClosed\n\t}\n\tif cb == nil {\n\t\treturn nil, errInvalidMsgHandler\n\t}\n\tc := mset.client\n\tsid := int(mset.sid.Add(1))\n\t// Now create the subscription\n\treturn c.processSub([]byte(subject), nil, []byte(strconv.Itoa(sid)), cb, false)\n}\n\n// Lock does NOT need to be held, we set the client on setup and never change it at this point.\nfunc (mset *stream) queueSubscribeInternal(subject, group string, cb msgHandler) (*subscription, error) {\n\tif mset.closed.Load() {\n\t\treturn nil, errStreamClosed\n\t}\n\tif cb == nil {\n\t\treturn nil, errInvalidMsgHandler\n\t}\n\tc := mset.client\n\tsid := int(mset.sid.Add(1))\n\t// Now create the subscription\n\treturn c.processSub([]byte(subject), []byte(group), []byte(strconv.Itoa(sid)), cb, false)\n}\n\n// This will unsubscribe us from the exact subject given.\n// We do not currently track the subs so do not have the sid.\n// This should be called only on an update.\n// Lock does NOT need to be held, we set the client on setup and never change it at this point.\nfunc (mset *stream) unsubscribeInternal(subject string) error {\n\tif mset.closed.Load() {\n\t\treturn errStreamClosed\n\t}\n\tc := mset.client\n\tvar sid []byte\n\tc.mu.Lock()\n\tfor _, sub := range c.subs {\n\t\tif subject == string(sub.subject) {\n\t\t\tsid = sub.sid\n\t\t\tbreak\n\t\t}\n\t}\n\tc.mu.Unlock()\n\n\tif sid != nil {\n\t\treturn c.processUnsub(sid)\n\t}\n\treturn nil\n}\n\n// Lock should be held.\nfunc (mset *stream) unsubscribe(sub *subscription) {\n\tif sub == nil || mset.closed.Load() {\n\t\treturn\n\t}\n\tmset.client.processUnsub(sub.sid)\n}\n\nfunc (mset *stream) setupStore(fsCfg *FileStoreConfig) error {\n\tmset.mu.Lock()\n\tswitch mset.cfg.Storage {\n\tcase MemoryStorage:\n\t\tms, err := newMemStore(&mset.cfg)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t\tmset.store = ms\n\tcase FileStorage:\n\t\ts := mset.srv\n\t\tprf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name)\n\t\tif prf != nil {\n\t\t\t// We are encrypted here, fill in correct cipher selection.\n\t\t\tfsCfg.Cipher = s.getOpts().JetStreamCipher\n\t\t}\n\t\toldprf := s.jsKeyGen(s.getOpts().JetStreamOldKey, mset.acc.Name)\n\t\tcfg := *fsCfg\n\t\tcfg.srv = s\n\t\tfs, err := newFileStoreWithCreated(cfg, mset.cfg, mset.created, prf, oldprf)\n\t\tif err != nil {\n\t\t\tmset.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\t\tmset.store = fs\n\t}\n\t// This will fire the callback but we do not require the lock since md will be 0 here.\n\tmset.store.RegisterStorageUpdates(mset.storeUpdates)\n\tmset.store.RegisterStorageRemoveMsg(func(seq uint64) {\n\t\tif mset.IsClustered() {\n\t\t\tif mset.IsLeader() {\n\t\t\t\tmset.mu.RLock()\n\t\t\t\tmd := streamMsgDelete{Seq: seq, NoErase: true, Stream: mset.cfg.Name}\n\t\t\t\tmset.node.Propose(encodeMsgDelete(&md))\n\t\t\t\tmset.mu.RUnlock()\n\t\t\t}\n\t\t} else {\n\t\t\tmset.removeMsg(seq)\n\t\t}\n\t})\n\tmset.store.RegisterProcessJetStreamMsg(func(im *inMsg) {\n\t\tif mset.IsClustered() {\n\t\t\tif mset.IsLeader() {\n\t\t\t\tmset.processClusteredInboundMsg(im.subj, im.rply, im.hdr, im.msg, im.mt, false)\n\t\t\t}\n\t\t} else {\n\t\t\tmset.processJetStreamMsg(im.subj, im.rply, im.hdr, im.msg, 0, 0, im.mt, false, true)\n\t\t}\n\t})\n\tmset.mu.Unlock()\n\n\treturn nil\n}\n\n// Called for any updates to the underlying stream. We pass through the bytes to the\n// jetstream account. We do local processing for stream pending for consumers, but only\n// for removals.\n// Lock should not be held.\nfunc (mset *stream) storeUpdates(md, bd int64, seq uint64, subj string) {\n\t// If we have a single negative update then we will process our consumers for stream pending.\n\t// Purge and Store handled separately inside individual calls.\n\tif md == -1 && seq > 0 && subj != _EMPTY_ {\n\t\t// We use our consumer list mutex here instead of the main stream lock since it may be held already.\n\t\tmset.clsMu.RLock()\n\t\tif mset.csl != nil {\n\t\t\tmset.csl.Match(subj, func(o *consumer) {\n\t\t\t\to.decStreamPending(seq, subj)\n\t\t\t})\n\t\t} else {\n\t\t\tfor _, o := range mset.cList {\n\t\t\t\to.decStreamPending(seq, subj)\n\t\t\t}\n\t\t}\n\t\tmset.clsMu.RUnlock()\n\t} else if md < 0 {\n\t\t// Batch decrements we need to force consumers to re-calculate num pending.\n\t\tmset.clsMu.RLock()\n\t\tfor _, o := range mset.cList {\n\t\t\to.streamNumPendingLocked()\n\t\t}\n\t\tmset.clsMu.RUnlock()\n\t}\n\n\tif mset.jsa != nil {\n\t\tmset.jsa.updateUsage(mset.tier, mset.stype, bd)\n\t}\n}\n\n// NumMsgIds returns the number of message ids being tracked for duplicate suppression.\nfunc (mset *stream) numMsgIds() int {\n\tmset.ddMu.Lock()\n\tdefer mset.ddMu.Unlock()\n\treturn len(mset.ddmap)\n}\n\n// checkMsgId will process and check for duplicates.\n// mset.ddMu lock should be held.\nfunc (mset *stream) checkMsgId(id string) *ddentry {\n\tif id == _EMPTY_ || len(mset.ddmap) == 0 {\n\t\treturn nil\n\t}\n\treturn mset.ddmap[id]\n}\n\n// Will purge the entries that are past the window.\n// Should be called from a timer.\nfunc (mset *stream) purgeMsgIds() {\n\tnow := time.Now().UnixNano()\n\tmset.cfgMu.RLock()\n\ttmrNext := mset.cfg.Duplicates\n\tmset.cfgMu.RUnlock()\n\twindow := int64(tmrNext)\n\n\tmset.ddMu.Lock()\n\tdefer mset.ddMu.Unlock()\n\n\tfor i, dde := range mset.ddarr[mset.ddindex:] {\n\t\tif now-dde.ts >= window {\n\t\t\tdelete(mset.ddmap, dde.id)\n\t\t} else {\n\t\t\tmset.ddindex += i\n\t\t\t// Check if we should garbage collect here if we are 1/3 total size.\n\t\t\tif cap(mset.ddarr) > 3*(len(mset.ddarr)-mset.ddindex) {\n\t\t\t\tmset.ddarr = append([]*ddentry(nil), mset.ddarr[mset.ddindex:]...)\n\t\t\t\tmset.ddindex = 0\n\t\t\t}\n\t\t\ttmrNext = time.Duration(window - (now - dde.ts))\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(mset.ddmap) > 0 {\n\t\t// Make sure to not fire too quick\n\t\tconst minFire = 50 * time.Millisecond\n\t\tif tmrNext < minFire {\n\t\t\ttmrNext = minFire\n\t\t}\n\t\tif mset.ddtmr != nil {\n\t\t\tmset.ddtmr.Reset(tmrNext)\n\t\t} else {\n\t\t\tmset.ddtmr = time.AfterFunc(tmrNext, mset.purgeMsgIds)\n\t\t}\n\t} else {\n\t\tif mset.ddtmr != nil {\n\t\t\tmset.ddtmr.Stop()\n\t\t\tmset.ddtmr = nil\n\t\t}\n\t\tmset.ddmap = nil\n\t\tmset.ddarr = nil\n\t\tmset.ddindex = 0\n\t}\n}\n\n// storeMsgId will store the message id for duplicate detection.\nfunc (mset *stream) storeMsgId(dde *ddentry) {\n\tmset.ddMu.Lock()\n\tdefer mset.ddMu.Unlock()\n\tmset.storeMsgIdLocked(dde)\n}\n\n// storeMsgIdLocked will store the message id for duplicate detection.\n// mset.ddMu lock should be held.\nfunc (mset *stream) storeMsgIdLocked(dde *ddentry) {\n\t// Zero means disabled.\n\tif mset.cfg.Duplicates <= 0 {\n\t\treturn\n\t}\n\n\tif mset.ddmap == nil {\n\t\tmset.ddmap = make(map[string]*ddentry)\n\t}\n\tmset.ddmap[dde.id] = dde\n\tmset.ddarr = append(mset.ddarr, dde)\n\tif mset.ddtmr == nil {\n\t\tmset.ddtmr = time.AfterFunc(mset.cfg.Duplicates, mset.purgeMsgIds)\n\t}\n}\n\n// Fast lookup of msgId.\nfunc getMsgId(hdr []byte) string {\n\treturn string(getHeader(JSMsgId, hdr))\n}\n\n// Fast lookup of expected last msgId.\nfunc getExpectedLastMsgId(hdr []byte) string {\n\treturn string(getHeader(JSExpectedLastMsgId, hdr))\n}\n\n// Fast lookup of expected stream.\nfunc getExpectedStream(hdr []byte) string {\n\treturn string(getHeader(JSExpectedStream, hdr))\n}\n\n// Fast lookup of expected last sequence.\nfunc getExpectedLastSeq(hdr []byte) (uint64, bool) {\n\tbseq := sliceHeader(JSExpectedLastSeq, hdr)\n\tif len(bseq) == 0 {\n\t\treturn 0, false\n\t}\n\treturn uint64(parseInt64(bseq)), true\n}\n\n// Fast lookup of rollups.\nfunc getRollup(hdr []byte) string {\n\tr := getHeader(JSMsgRollup, hdr)\n\tif len(r) == 0 {\n\t\treturn _EMPTY_\n\t}\n\treturn strings.ToLower(string(r))\n}\n\n// Fast lookup of expected stream sequence per subject.\nfunc getExpectedLastSeqPerSubject(hdr []byte) (uint64, bool) {\n\tbseq := sliceHeader(JSExpectedLastSubjSeq, hdr)\n\tif len(bseq) == 0 {\n\t\treturn 0, false\n\t}\n\treturn uint64(parseInt64(bseq)), true\n}\n\n// Fast lookup of expected subject for the expected stream sequence per subject.\nfunc getExpectedLastSeqPerSubjectForSubject(hdr []byte) string {\n\treturn bytesToString(sliceHeader(JSExpectedLastSubjSeqSubj, hdr))\n}\n\n// Fast lookup of the message TTL from headers:\n// - Positive return value: duration in seconds.\n// - Zero return value: no TTL or parse error.\n// - Negative return value: never expires.\nfunc getMessageTTL(hdr []byte) (int64, error) {\n\tttl := getHeader(JSMessageTTL, hdr)\n\tif len(ttl) == 0 {\n\t\treturn 0, nil\n\t}\n\treturn parseMessageTTL(bytesToString(ttl))\n}\n\n// - Positive return value: duration in seconds.\n// - Zero return value: no TTL or parse error.\n// - Negative return value: never expires.\nfunc parseMessageTTL(ttl string) (int64, error) {\n\tif strings.ToLower(ttl) == \"never\" {\n\t\treturn -1, nil\n\t}\n\tdur, err := time.ParseDuration(ttl)\n\tif err == nil {\n\t\tif dur < time.Second {\n\t\t\treturn 0, NewJSMessageTTLInvalidError()\n\t\t}\n\t\treturn int64(dur.Seconds()), nil\n\t}\n\tt := parseInt64(stringToBytes(ttl))\n\tif t < 0 {\n\t\t// This probably means a parse failure, hence why\n\t\t// we have a special case \"never\" for returning -1.\n\t\t// Otherwise we can't know if it's a genuine TTL\n\t\t// that says never expire or if it's a parse error.\n\t\treturn 0, NewJSMessageTTLInvalidError()\n\t}\n\treturn t, nil\n}\n\n// Fast lookup of the message Incr from headers.\n// Return includes the value or nil, and success.\nfunc getMessageIncr(hdr []byte) (*big.Int, bool) {\n\tincr := sliceHeader(JSMessageIncr, hdr)\n\tif len(incr) == 0 {\n\t\treturn nil, true\n\t}\n\tvar v big.Int\n\treturn v.SetString(bytesToString(incr), 10)\n}\n\n// Fast lookup of message schedule.\nfunc getMessageSchedule(hdr []byte) (time.Time, bool) {\n\tif len(hdr) == 0 {\n\t\treturn time.Time{}, true\n\t}\n\treturn nextMessageSchedule(hdr, time.Now().UTC().UnixNano())\n}\n\n// Fast lookup and calculation of next message schedule.\nfunc nextMessageSchedule(hdr []byte, ts int64) (time.Time, bool) {\n\tif len(hdr) == 0 {\n\t\treturn time.Time{}, true\n\t}\n\tval := bytesToString(sliceHeader(JSSchedulePattern, hdr))\n\ttz := bytesToString(sliceHeader(JSScheduleTimeZone, hdr))\n\tschedule, _, ok := parseMsgSchedule(val, tz, ts)\n\treturn schedule, ok\n}\n\n// Fast lookup of the message schedule TTL from headers.\n// The TTL is confirmed to be valid, but the raw TTL string is returned.\nfunc getMessageScheduleTTL(hdr []byte) (string, bool) {\n\tttl := getHeader(JSScheduleTTL, hdr)\n\tif len(ttl) == 0 {\n\t\treturn _EMPTY_, true\n\t}\n\tif _, err := parseMessageTTL(bytesToString(ttl)); err != nil {\n\t\treturn _EMPTY_, false\n\t}\n\treturn string(ttl), true\n}\n\n// Fast lookup of message schedule target.\nfunc getMessageScheduleTarget(hdr []byte) string {\n\tif len(hdr) == 0 {\n\t\treturn _EMPTY_\n\t}\n\treturn string(getHeader(JSScheduleTarget, hdr))\n}\n\n// Fast lookup of message schedule source.\nfunc getMessageScheduleSource(hdr []byte) string {\n\tif len(hdr) == 0 {\n\t\treturn _EMPTY_\n\t}\n\treturn string(getHeader(JSScheduleSource, hdr))\n}\n\n// Fast lookup of message scheduler.\nfunc getMessageScheduler(hdr []byte) string {\n\tif len(hdr) == 0 {\n\t\treturn _EMPTY_\n\t}\n\treturn string(getHeader(JSScheduler, hdr))\n}\n\n// Fast lookup of batch ID.\nfunc getBatchId(hdr []byte) string {\n\tif len(hdr) == 0 {\n\t\treturn _EMPTY_\n\t}\n\tif atomicBatchId := sliceHeader(JSBatchId, hdr); atomicBatchId != nil {\n\t\treturn string(atomicBatchId)\n\t}\n\treturn _EMPTY_\n}\n\ntype FastBatch struct {\n\tid string\n\tseq uint64\n\tflow uint16\n\tping bool\n\tgapOk bool\n\tcommit bool\n\tcommitEob bool\n}\n\nconst (\n\tFastBatchSuffix = \".$FI\"\n\tFastBatchGapFail = \"fail\"\n\tFastBatchGapOk = \"ok\"\n)\n\nconst (\n\tFastBatchOpStart = iota\n\tFastBatchOpAppend\n\tFastBatchOpCommit\n\tFastBatchOpCommitEob\n\tFastBatchOpPing\n)\n\nvar fastBatchPool sync.Pool\n\nfunc getFastBatchFromPool() *FastBatch {\n\tidx := fastBatchPool.Get()\n\tif idx != nil {\n\t\treturn idx.(*FastBatch)\n\t}\n\treturn new(FastBatch)\n}\n\nfunc (b *FastBatch) returnToPool() {\n\tif b == nil {\n\t\treturn\n\t}\n\t// Nil out all values.\n\t*b = FastBatch{}\n\tfastBatchPool.Put(b)\n}\n\n// getFastBatch gets fast batch info from the reply subject in the form:\n// ......$FI\nfunc getFastBatch(reply string, hdr []byte) (*FastBatch, bool) {\n\tlreply := len(reply)\n\tif lreply <= 4 || reply[lreply-4:] != FastBatchSuffix {\n\t\tif !isServiceReply(stringToBytes(reply)) {\n\t\t\treturn nil, false\n\t\t}\n\t\t// If account imports/exports are used, the reply might be internal.\n\t\t// Check the client header for the original reply subject.\n\t\tci := sliceHeader(ClientInfoHdr, hdr)\n\t\tif ci == nil {\n\t\t\treturn nil, false\n\t\t}\n\t\tvar cis ClientInfo\n\t\tif err := json.Unmarshal(ci, &cis); err != nil || cis.Reply == _EMPTY_ {\n\t\t\treturn nil, false\n\t\t}\n\t\treply = cis.Reply\n\t\tlreply = len(reply)\n\t\tif lreply <= 4 || reply[lreply-4:] != FastBatchSuffix {\n\t\t\treturn nil, false\n\t\t}\n\t}\n\n\tn := lreply - 4 // Move to just before the dot\n\to := strings.LastIndexByte(reply[:n], '.')\n\tif o == -1 {\n\t\treturn nil, true\n\t}\n\t// Batch operation.\n\tops := reply[o+1 : n]\n\top := parseInt64(stringToBytes(ops))\n\tif op < FastBatchOpStart || op > FastBatchOpPing {\n\t\treturn nil, true\n\t}\n\n\tb := getFastBatchFromPool()\n\tb.ping = op == FastBatchOpPing\n\tb.commitEob = op == FastBatchOpCommitEob\n\tb.commit = b.commitEob || op == FastBatchOpCommit\n\tp := o\n\n\t// Batch seq.\n\tif o = strings.LastIndexByte(reply[:o], '.'); o == -1 {\n\t\treturn nil, true\n\t}\n\ta := parseInt64(stringToBytes(reply[o+1 : p]))\n\tif a < 1 {\n\t\treturn nil, true\n\t}\n\tb.seq = uint64(a)\n\tp = o\n\tif b.seq <= 0 {\n\t\treturn nil, true\n\t}\n\tif op == FastBatchOpStart && b.seq != 1 {\n\t\treturn nil, true\n\t} else if op == FastBatchOpAppend && b.seq <= 1 {\n\t\treturn nil, true\n\t}\n\n\t// Gap mode.\n\tif o = strings.LastIndexByte(reply[:o], '.'); o == -1 {\n\t\treturn nil, true\n\t}\n\tgapMode := reply[o+1 : p]\n\tif gapMode != FastBatchGapFail && gapMode != FastBatchGapOk {\n\t\treturn nil, true // Not recognized.\n\t}\n\tb.gapOk = gapMode == FastBatchGapOk\n\tp = o\n\n\t// Ack flow.\n\tif o = strings.LastIndexByte(reply[:o], '.'); o == -1 {\n\t\treturn nil, true\n\t}\n\ta = parseInt64(stringToBytes(reply[o+1 : p]))\n\tif a <= 0 {\n\t\ta = 10\n\t} else if a > math.MaxUint16 {\n\t\ta = math.MaxUint16\n\t}\n\tb.flow = uint16(a)\n\tp = o\n\n\t// Batch id.\n\tif o = strings.LastIndexByte(reply[:o], '.'); o == -1 {\n\t\treturn nil, true\n\t}\n\tb.id = reply[o+1 : p]\n\treturn b, false\n}\n\n// Fast lookup of batch sequence.\nfunc getBatchSequence(hdr []byte) (uint64, bool) {\n\tbseq := sliceHeader(JSBatchSeq, hdr)\n\tif len(bseq) == 0 {\n\t\treturn 0, false\n\t}\n\treturn uint64(parseInt64(bseq)), true\n}\n\n// Signal if we are clustered. Will acquire rlock.\nfunc (mset *stream) IsClustered() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.isClustered()\n}\n\n// Lock should be held.\nfunc (mset *stream) isClustered() bool {\n\treturn mset.node != nil\n}\n\n// Used if we have to queue things internally to avoid the route/gw path.\ntype inMsg struct {\n\tsubj string\n\trply string\n\thdr []byte\n\tmsg []byte\n\tsi *sourceInfo\n\tmt *msgTrace\n\tseq uint64 // seq that can be optionally used for sorting out-of-band.\n}\n\nvar inMsgPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &inMsg{}\n\t},\n}\n\nfunc (im *inMsg) returnToPool() {\n\tim.subj, im.rply, im.hdr, im.msg, im.si, im.mt = _EMPTY_, _EMPTY_, nil, nil, nil, nil\n\tinMsgPool.Put(im)\n}\n\nfunc (mset *stream) queueInbound(ib *ipQueue[*inMsg], subj, rply string, hdr, msg []byte, si *sourceInfo, mt *msgTrace) {\n\tim := inMsgPool.Get().(*inMsg)\n\tim.subj, im.rply, im.hdr, im.msg, im.si, im.mt = subj, rply, hdr, msg, si, mt\n\tif _, err := ib.push(im); err != nil {\n\t\tim.returnToPool()\n\t\tstreamName := mset.cfg.Name\n\t\tmset.srv.RateLimitWarnf(\"Dropping messages due to excessive stream ingest rate on '%s' > '%s': %s\", mset.acc.Name, streamName, err)\n\t\tif rply != _EMPTY_ {\n\t\t\thdr := []byte(\"NATS/1.0 429 Too Many Requests\\r\\n\\r\\n\")\n\t\t\tb, _ := json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: streamName}, Error: NewJSStreamTooManyRequestsError()})\n\t\t\tmset.outq.send(newJSPubMsg(rply, _EMPTY_, _EMPTY_, hdr, b, nil, 0))\n\t\t}\n\t}\n}\n\nvar dgPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &directGetReq{}\n\t},\n}\n\n// For when we need to not inline the request.\ntype directGetReq struct {\n\t// Copy of this is correct for this.\n\treq JSApiMsgGetRequest\n\treply string\n}\n\n// processDirectGetRequest handles direct get request for stream messages.\nfunc (mset *stream) processDirectGetRequest(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif len(reply) == 0 {\n\t\treturn\n\t}\n\thdr, msg := c.msgParts(rmsg)\n\tif errorOnRequiredApiLevel(hdr) {\n\t\thdr := []byte(\"NATS/1.0 412 Required Api Level\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\tif len(msg) == 0 {\n\t\thdr := []byte(\"NATS/1.0 408 Empty Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\tvar req JSApiMsgGetRequest\n\terr := json.Unmarshal(msg, &req)\n\tif err != nil {\n\t\thdr := []byte(\"NATS/1.0 408 Malformed Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\t// Check if nothing set.\n\tif req.Seq == 0 && req.LastFor == _EMPTY_ && req.NextFor == _EMPTY_ && len(req.MultiLastFor) == 0 && req.StartTime == nil {\n\t\thdr := []byte(\"NATS/1.0 408 Empty Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\t// Check we don't have conflicting options set.\n\t// We do not allow batch mode for lastFor requests.\n\tif (req.Seq > 0 && req.LastFor != _EMPTY_) ||\n\t\t(req.Seq > 0 && req.StartTime != nil) ||\n\t\t(req.StartTime != nil && req.LastFor != _EMPTY_) ||\n\t\t(req.LastFor != _EMPTY_ && req.NextFor != _EMPTY_) ||\n\t\t(req.LastFor != _EMPTY_ && req.Batch > 0) ||\n\t\t(req.LastFor != _EMPTY_ && len(req.MultiLastFor) > 0) ||\n\t\t(req.NextFor != _EMPTY_ && len(req.MultiLastFor) > 0) ||\n\t\t(req.UpToSeq > 0 && req.UpToTime != nil) {\n\t\thdr := []byte(\"NATS/1.0 408 Bad Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\n\tinlineOk := c.kind != ROUTER && c.kind != GATEWAY && c.kind != LEAF\n\tif !inlineOk {\n\t\tdg := dgPool.Get().(*directGetReq)\n\t\tdg.req, dg.reply = req, reply\n\t\tmset.gets.push(dg)\n\t} else {\n\t\tmset.getDirectRequest(&req, reply)\n\t}\n}\n\n// This is for direct get by last subject which is part of the subject itself.\nfunc (mset *stream) processDirectGetLastBySubjectRequest(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\tif len(reply) == 0 {\n\t\treturn\n\t}\n\thdr, msg := c.msgParts(rmsg)\n\tif errorOnRequiredApiLevel(hdr) {\n\t\thdr := []byte(\"NATS/1.0 412 Required Api Level\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\n\tvar req JSApiMsgGetRequest\n\tif len(msg) > 0 {\n\t\terr := json.Unmarshal(msg, &req)\n\t\tif err != nil {\n\t\t\thdr := []byte(\"NATS/1.0 408 Malformed Request\\r\\n\\r\\n\")\n\t\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Extract the key.\n\tvar key string\n\tfor i, n := 0, 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif n == 4 {\n\t\t\t\tif start := i + 1; start < len(subject) {\n\t\t\t\t\tkey = subject[i+1:]\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tn++\n\t\t}\n\t}\n\tif len(key) == 0 {\n\t\thdr := []byte(\"NATS/1.0 408 Bad Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\n\t// Error if request fields are provided that don't make sense for this endpoint.\n\tswitch {\n\tcase req.LastFor != _EMPTY_, req.NextFor != _EMPTY_,\n\t\tlen(req.MultiLastFor) > 0, req.Seq != 0, req.Batch != 0,\n\t\treq.StartTime != nil, req.UpToSeq != 0, req.UpToTime != nil:\n\t\thdr := []byte(\"NATS/1.0 408 Bad Request\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\tdefault:\n\t\treq.LastFor = key\n\t}\n\n\tinlineOk := c.kind != ROUTER && c.kind != GATEWAY && c.kind != LEAF\n\tif !inlineOk {\n\t\tdg := dgPool.Get().(*directGetReq)\n\t\tdg.req, dg.reply = req, reply\n\t\tmset.gets.push(dg)\n\t} else {\n\t\tmset.getDirectRequest(&req, reply)\n\t}\n}\n\n// For direct get batch and multi requests.\nconst (\n\tdg = \"NATS/1.0\\r\\nNats-Stream: %s\\r\\nNats-Subject: %s\\r\\nNats-Sequence: %d\\r\\nNats-Time-Stamp: %s\\r\\n\\r\\n\"\n\tdgb = \"NATS/1.0\\r\\nNats-Stream: %s\\r\\nNats-Subject: %s\\r\\nNats-Sequence: %d\\r\\nNats-Time-Stamp: %s\\r\\nNats-Num-Pending: %d\\r\\nNats-Last-Sequence: %d\\r\\n\\r\\n\"\n\teob = \"NATS/1.0 204 EOB\\r\\nNats-Num-Pending: %d\\r\\nNats-Last-Sequence: %d\\r\\n\\r\\n\"\n\teobm = \"NATS/1.0 204 EOB\\r\\nNats-Num-Pending: %d\\r\\nNats-Last-Sequence: %d\\r\\nNats-UpTo-Sequence: %d\\r\\n\\r\\n\"\n)\n\n// Handle a multi request.\nfunc (mset *stream) getDirectMulti(req *JSApiMsgGetRequest, reply string) {\n\t// TODO(dlc) - Make configurable?\n\tconst maxAllowedResponses = 1024\n\n\t// Ensure this read request is isolated and doesn't interleave with writes.\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\n\t// Grab store and name.\n\tstore, name, s := mset.store, mset.cfg.Name, mset.srv\n\n\t// Grab MaxBytes\n\tmb := req.MaxBytes\n\tif mb == 0 && s != nil {\n\t\t// Fill in with the server's MaxPending.\n\t\tmb = int(s.opts.MaxPending)\n\t}\n\n\tupToSeq := req.UpToSeq\n\t// If we have UpToTime set get the proper sequence.\n\tif req.UpToTime != nil {\n\t\tupToSeq = store.GetSeqFromTime((*req.UpToTime).UTC())\n\t\t// Avoid selecting a first sequence that will take us to before the stream first\n\t\t// sequence, otherwise we can return messages after the supplied UpToTime.\n\t\tif upToSeq <= mset.state().FirstSeq {\n\t\t\thdr := []byte(\"NATS/1.0 404 No Results\\r\\n\\r\\n\")\n\t\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\treturn\n\t\t}\n\t\t// We need to back off one since this is used to determine start sequence normally,\n\t\t// whereas here we want it to be the ceiling.\n\t\tupToSeq--\n\t}\n\t// If not set, set to the last sequence and remember that for EOB.\n\tif upToSeq == 0 {\n\t\tvar state StreamState\n\t\tmset.store.FastState(&state)\n\t\tupToSeq = state.LastSeq\n\t}\n\n\tseqs, err := store.MultiLastSeqs(req.MultiLastFor, upToSeq, maxAllowedResponses)\n\tif err != nil {\n\t\tvar hdr []byte\n\t\tif err == ErrTooManyResults {\n\t\t\thdr = []byte(\"NATS/1.0 413 Too Many Results\\r\\n\\r\\n\")\n\t\t} else {\n\t\t\thdr = []byte(fmt.Sprintf(\"NATS/1.0 500 %v\\r\\n\\r\\n\", err))\n\t\t}\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\tif len(seqs) == 0 {\n\t\thdr := []byte(\"NATS/1.0 404 No Results\\r\\n\\r\\n\")\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\treturn\n\t}\n\n\tnp, lseq, sentBytes, sent := uint64(len(seqs)), uint64(0), 0, 0\n\tfor _, seq := range seqs {\n\t\tif seq < req.Seq {\n\t\t\tif np > 0 {\n\t\t\t\tnp--\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tvar svp StoreMsg\n\t\tsm, err := store.LoadMsg(seq, &svp)\n\t\tif err != nil {\n\t\t\thdr := []byte(\"NATS/1.0 404 Message Not Found\\r\\n\\r\\n\")\n\t\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\treturn\n\t\t}\n\n\t\thdr := sm.hdr\n\t\tts := time.Unix(0, sm.ts).UTC()\n\n\t\t// Decrement num pending. This is an optimization, and we do not continue to look it up for these operations.\n\t\tif np > 0 {\n\t\t\tnp--\n\t\t}\n\t\tif len(hdr) == 0 {\n\t\t\thdr = fmt.Appendf(nil, dgb, name, sm.subj, sm.seq, ts.Format(time.RFC3339Nano), np, lseq)\n\t\t} else {\n\t\t\thdr = copyBytes(hdr)\n\t\t\thdr = genHeader(hdr, JSStream, name)\n\t\t\thdr = genHeader(hdr, JSSubject, sm.subj)\n\t\t\thdr = genHeader(hdr, JSSequence, strconv.FormatUint(sm.seq, 10))\n\t\t\thdr = genHeader(hdr, JSTimeStamp, ts.Format(time.RFC3339Nano))\n\t\t\thdr = genHeader(hdr, JSNumPending, strconv.FormatUint(np, 10))\n\t\t\thdr = genHeader(hdr, JSLastSequence, strconv.FormatUint(lseq, 10))\n\t\t}\n\t\t// Track our lseq\n\t\tlseq = sm.seq\n\t\t// Send out our message.\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, sm.msg, nil, 0))\n\t\t// Check if we have exceeded max bytes.\n\t\tsentBytes += len(sm.subj) + len(sm.hdr) + len(sm.msg)\n\t\tif sentBytes >= mb {\n\t\t\tbreak\n\t\t}\n\t\tsent++\n\t\tif req.Batch > 0 && sent >= req.Batch {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Send out EOB\n\thdr := fmt.Appendf(nil, eobm, np, lseq, upToSeq)\n\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n}\n\n// Do actual work on a direct msg request.\n// This could be called in a Go routine if we are inline for a non-client connection.\nfunc (mset *stream) getDirectRequest(req *JSApiMsgGetRequest, reply string) {\n\t// Handle multi in separate function.\n\tif len(req.MultiLastFor) > 0 {\n\t\tmset.getDirectMulti(req, reply)\n\t\treturn\n\t}\n\n\t// Ensure this read request is isolated and doesn't interleave with writes.\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\n\tstore, name, s := mset.store, mset.cfg.Name, mset.srv\n\n\tvar seq uint64\n\t// Lookup start seq if AsOfTime is set.\n\tif req.StartTime != nil {\n\t\tseq = store.GetSeqFromTime(*req.StartTime)\n\t} else {\n\t\tseq = req.Seq\n\t}\n\n\twc := subjectHasWildcard(req.NextFor)\n\t// For tracking num pending if we are batch.\n\tvar np, lseq, validThrough uint64\n\tvar isBatchRequest bool\n\tbatch := req.Batch\n\tif batch == 0 {\n\t\tbatch = 1\n\t} else {\n\t\t// This is a batch request, capture initial numPending.\n\t\tisBatchRequest = true\n\t\tvar err error\n\t\tif np, validThrough, err = store.NumPending(seq, req.NextFor, false); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Grab MaxBytes\n\tmb := req.MaxBytes\n\tif mb == 0 && s != nil {\n\t\t// Fill in with the server's MaxPending.\n\t\tmb = int(s.opts.MaxPending)\n\t}\n\t// Track what we have sent.\n\tvar sentBytes int\n\n\t// Loop over batch, which defaults to 1.\n\tfor i := 0; i < batch; i++ {\n\t\tvar (\n\t\t\tsvp StoreMsg\n\t\t\tsm *StoreMsg\n\t\t\terr error\n\t\t)\n\t\tif seq > 0 && req.NextFor == _EMPTY_ {\n\t\t\t// Only do direct lookup for a non batch.\n\t\t\tif i == 0 && !isBatchRequest {\n\t\t\t\tsm, err = store.LoadMsg(seq, &svp)\n\t\t\t} else {\n\t\t\t\t// We want to use load next with fwcs to step over deleted msgs.\n\t\t\t\tsm, seq, err = store.LoadNextMsg(fwcs, true, seq, &svp)\n\t\t\t}\n\t\t\t// Bump for next loop if applicable.\n\t\t\tseq++\n\t\t} else if req.NextFor != _EMPTY_ {\n\t\t\tsm, seq, err = store.LoadNextMsg(req.NextFor, wc, seq, &svp)\n\t\t\tseq++\n\t\t} else {\n\t\t\t// Batch is not applicable here, this is checked before we get here.\n\t\t\tsm, err = store.LoadLastMsg(req.LastFor, &svp)\n\t\t}\n\t\tif err != nil {\n\t\t\t// For batches, if we stop early we want to do EOB logic below.\n\t\t\tif batch > 1 && i > 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\thdr := []byte(\"NATS/1.0 404 Message Not Found\\r\\n\\r\\n\")\n\t\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t\t\treturn\n\t\t}\n\n\t\tts := time.Unix(0, sm.ts).UTC()\n\t\tvar hdr []byte\n\t\tif !req.NoHeaders {\n\t\t\thdr = sm.hdr\n\t\t\tif isBatchRequest {\n\t\t\t\t// Decrement num pending. This is an optimization, and we do not continue to look it up for these operations.\n\t\t\t\tif np > 0 {\n\t\t\t\t\tnp--\n\t\t\t\t}\n\t\t\t\tif len(hdr) == 0 {\n\t\t\t\t\thdr = fmt.Appendf(nil, dgb, name, sm.subj, sm.seq, ts.Format(time.RFC3339Nano), np, lseq)\n\t\t\t\t} else {\n\t\t\t\t\thdr = copyBytes(hdr)\n\t\t\t\t\thdr = genHeader(hdr, JSStream, name)\n\t\t\t\t\thdr = genHeader(hdr, JSSubject, sm.subj)\n\t\t\t\t\thdr = genHeader(hdr, JSSequence, strconv.FormatUint(sm.seq, 10))\n\t\t\t\t\thdr = genHeader(hdr, JSTimeStamp, ts.Format(time.RFC3339Nano))\n\t\t\t\t\thdr = genHeader(hdr, JSNumPending, strconv.FormatUint(np, 10))\n\t\t\t\t\thdr = genHeader(hdr, JSLastSequence, strconv.FormatUint(lseq, 10))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif len(hdr) == 0 {\n\t\t\t\t\thdr = fmt.Appendf(nil, dg, name, sm.subj, sm.seq, ts.Format(time.RFC3339Nano))\n\t\t\t\t} else {\n\t\t\t\t\thdr = copyBytes(hdr)\n\t\t\t\t\thdr = genHeader(hdr, JSStream, name)\n\t\t\t\t\thdr = genHeader(hdr, JSSubject, sm.subj)\n\t\t\t\t\thdr = genHeader(hdr, JSSequence, strconv.FormatUint(sm.seq, 10))\n\t\t\t\t\thdr = genHeader(hdr, JSTimeStamp, ts.Format(time.RFC3339Nano))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Track our lseq\n\t\tlseq = sm.seq\n\t\t// Send out our message.\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, sm.msg, nil, 0))\n\t\t// Check if we have exceeded max bytes.\n\t\tsentBytes += len(sm.subj) + len(sm.hdr) + len(sm.msg)\n\t\tif sentBytes >= mb {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// If batch was requested send EOB.\n\tif isBatchRequest {\n\t\t// Update if the stream's last sequence has moved past our validThrough.\n\t\tif mset.lseq > validThrough {\n\t\t\tvar err error\n\t\t\tif np, _, err = store.NumPending(seq, req.NextFor, false); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\thdr := fmt.Appendf(nil, eob, np, lseq)\n\t\tmset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))\n\t}\n}\n\n// processInboundJetStreamMsg handles processing messages bound for a stream.\nfunc (mset *stream) processInboundJetStreamMsg(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {\n\thdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy.\n\tif mt, traceOnly := c.isMsgTraceEnabled(); mt != nil {\n\t\t// If message is delivered, we need to disable the message trace headers\n\t\t// to prevent a trace event to be generated when a stored message\n\t\t// is delivered to a consumer and routed.\n\t\tif !traceOnly {\n\t\t\thdr = setHeader(MsgTraceDest, MsgTraceDestDisabled, hdr)\n\t\t}\n\t\t// This will add the jetstream event while in the client read loop.\n\t\t// Since the event will be updated in a different go routine, the\n\t\t// tracing object will have a separate reference to the JS trace\n\t\t// object.\n\t\tmt.addJetStreamEvent(mset.name())\n\t}\n\tmset.queueInbound(mset.msgs, subject, reply, hdr, msg, nil, c.pa.trace)\n}\n\nvar (\n\terrLastSeqMismatch = errors.New(\"last sequence mismatch\")\n\terrMsgIdDuplicate = errors.New(\"msgid is duplicate\")\n\terrStreamClosed = errors.New(\"stream closed\")\n\terrInvalidMsgHandler = errors.New(\"undefined message handler\")\n\terrStreamMismatch = errors.New(\"expected stream does not match\")\n\terrMsgTTLDisabled = errors.New(\"message TTL disabled\")\n)\n\n// processJetStreamMsg is where we try to actually process the stream msg.\nfunc (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, mt *msgTrace, sourced bool, needLock bool) error {\n\treturn mset.processJetStreamMsgWithBatch(subject, reply, hdr, msg, lseq, ts, mt, sourced, needLock, nil)\n}\n\nfunc (mset *stream) processJetStreamMsgWithBatch(subject, reply string, hdr, msg []byte, lseq uint64, ts int64, mt *msgTrace, sourced bool, needLock bool, fastBatch *FastBatch) (retErr error) {\n\tif mt != nil {\n\t\t// Only the leader/standalone will have mt!=nil. On exit, send the\n\t\t// message trace event.\n\t\tdefer func() {\n\t\t\tmt.sendEventFromJetStream(retErr)\n\t\t}()\n\t}\n\n\tif mset == nil || mset.closed.Load() {\n\t\treturn errStreamClosed\n\t}\n\n\t// Hold lock while storing the message, and potentially purging as part of a rollup.\n\t// If we're writing an atomic batch of multiple messages, the lock is already held.\n\tif needLock {\n\t\tmset.mu.Lock()\n\t\tdefer mset.mu.Unlock()\n\t}\n\n\ts, store := mset.srv, mset.store\n\n\ttraceOnly := mt.traceOnly()\n\tbumpCLFS := func() {\n\t\t// Do not bump if tracing and not doing message delivery.\n\t\tif traceOnly {\n\t\t\treturn\n\t\t}\n\t\tmset.clMu.Lock()\n\t\tmset.clfs++\n\t\tmset.clMu.Unlock()\n\t}\n\n\t// Apply the input subject transform if any\n\tif mset.itr != nil {\n\t\tts, err := mset.itr.Match(subject)\n\t\tif err == nil {\n\t\t\t// no filtering: if the subject doesn't map the source of the transform, don't change it\n\t\t\tsubject = ts\n\t\t}\n\t}\n\n\tvar accName string\n\tif mset.acc != nil {\n\t\taccName = mset.acc.Name\n\t}\n\n\tjs, jsa, doAck := mset.js, mset.jsa, !mset.cfg.NoAck\n\tname, stype := mset.cfg.Name, mset.cfg.Storage\n\tmaxMsgSize := int(mset.cfg.MaxMsgSize)\n\tnumConsumers := len(mset.consumers)\n\tinterestRetention := mset.cfg.Retention == InterestPolicy\n\tallowMsgCounter, allowMsgSchedules := mset.cfg.AllowMsgCounter, mset.cfg.AllowMsgSchedules\n\t// Snapshot if we are the leader and if we can respond.\n\tisLeader, isSealed := mset.isLeaderNodeState(), mset.cfg.Sealed\n\tisClustered := mset.isClustered()\n\tcanConsistencyCheck := !isClustered || traceOnly\n\tcanRespond := doAck && len(reply) > 0 && isLeader\n\toutq := mset.outq\n\n\tvar resp = &JSPubAckResponse{}\n\n\tvar (\n\t\tbatchId string\n\t\tbatchSeq uint64\n\t)\n\t// Populate batch details.\n\tif fastBatch != nil {\n\t\t// For R1 we can reuse without regenerating.\n\t\tbatchId, batchSeq = fastBatch.id, fastBatch.seq\n\t\t// Disable consistency checking if this was already done\n\t\t// earlier as part of the batch consistency check.\n\t\tcanConsistencyCheck = traceOnly\n\t} else if fastBatch, _ = getFastBatch(reply, hdr); fastBatch != nil {\n\t\tdefer fastBatch.returnToPool()\n\t\tbatchId, batchSeq = fastBatch.id, fastBatch.seq\n\t\t// Disable consistency checking if this was already done\n\t\t// earlier as part of the batch consistency check.\n\t\tcanConsistencyCheck = traceOnly\n\t}\n\tif len(hdr) > 0 && batchId == _EMPTY_ {\n\t\tif batchId = getBatchId(hdr); batchId != _EMPTY_ {\n\t\t\tbatchSeq, _ = getBatchSequence(hdr)\n\t\t\t// Disable consistency checking if this was already done\n\t\t\t// earlier as part of the batch consistency check.\n\t\t\tcanConsistencyCheck = traceOnly\n\t\t}\n\t}\n\n\t// Bail here if sealed.\n\tif canConsistencyCheck && isSealed {\n\t\tif canRespond && outq != nil {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = ApiErrors[JSStreamSealedErr]\n\t\t\tb, _ := json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, b)\n\t\t}\n\t\treturn ApiErrors[JSStreamSealedErr]\n\t}\n\n\tvar buf [256]byte\n\tpubAck := append(buf[:0], mset.pubAck...)\n\n\t// If this is a non-clustered msg and we are not considered active, meaning no active subscription, do not process.\n\tif lseq == 0 && ts == 0 && !mset.active {\n\t\treturn nil\n\t}\n\n\t// For clustering the lower layers will pass our expected lseq. If it is present check for that here.\n\tvar clfs uint64\n\tif lseq > 0 {\n\t\tclfs = mset.getCLFS()\n\t\tif lseq != (mset.lseq + clfs) {\n\t\t\tisMisMatch := true\n\t\t\t// We may be able to recover here if we have no state whatsoever, or we are a mirror.\n\t\t\t// See if we have to adjust our starting sequence.\n\t\t\tif mset.lseq == 0 || mset.cfg.Mirror != nil {\n\t\t\t\tvar state StreamState\n\t\t\t\tmset.store.FastState(&state)\n\t\t\t\tif state.FirstSeq == 0 {\n\t\t\t\t\tif _, err := mset.store.Compact(lseq + 1); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tmset.lseq = lseq\n\t\t\t\t\tisMisMatch = false\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Really is a mismatch.\n\t\t\tif isMisMatch {\n\t\t\t\tif canRespond && outq != nil {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = ApiErrors[JSStreamSequenceNotMatchErr]\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn errLastSeqMismatch\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we have received this message across an account we may have request information attached.\n\t// For now remove. TODO(dlc) - Should this be opt-in or opt-out?\n\tif len(hdr) > 0 {\n\t\thdr = removeHeaderIfPresent(hdr, ClientInfoHdr)\n\t}\n\n\t// Process additional msg headers if still present.\n\tvar msgId string\n\tvar incr *big.Int\n\tvar rollupSub, rollupAll bool\n\n\tif len(hdr) > 0 {\n\t\t// Certain checks have already been performed if in clustered mode, so only check if not.\n\t\t// Note, for cluster mode but with message tracing (without message delivery), we need\n\t\t// to do this check here since it was not done in processClusteredInboundMsg().\n\t\tif canConsistencyCheck {\n\t\t\t// Counter increments.\n\t\t\t// Only supported on counter streams, and payload must be empty (if not coming from a source).\n\t\t\tvar ok bool\n\t\t\tif incr, ok = getMessageIncr(hdr); !ok {\n\t\t\t\tapiErr := NewJSMessageIncrInvalidError()\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn apiErr\n\t\t\t} else if incr != nil && !sourced {\n\t\t\t\t// Only do checks if the message isn't sourced. Otherwise, we need to store verbatim.\n\t\t\t\tif !allowMsgCounter {\n\t\t\t\t\tapiErr := NewJSMessageIncrDisabledError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else if len(msg) > 0 {\n\t\t\t\t\tapiErr := NewJSMessageIncrPayloadError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else {\n\t\t\t\t\t// Check for incompatible headers.\n\t\t\t\t\tvar doErr bool\n\t\t\t\t\tif getRollup(hdr) != _EMPTY_ ||\n\t\t\t\t\t\tgetExpectedStream(hdr) != _EMPTY_ ||\n\t\t\t\t\t\tgetExpectedLastMsgId(hdr) != _EMPTY_ ||\n\t\t\t\t\t\tgetExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ {\n\t\t\t\t\t\tdoErr = true\n\t\t\t\t\t} else if _, ok := getExpectedLastSeq(hdr); ok {\n\t\t\t\t\t\tdoErr = true\n\t\t\t\t\t} else if _, ok := getExpectedLastSeqPerSubject(hdr); ok {\n\t\t\t\t\t\tdoErr = true\n\t\t\t\t\t}\n\n\t\t\t\t\tif doErr {\n\t\t\t\t\t\tapiErr := NewJSMessageIncrInvalidError()\n\t\t\t\t\t\tif canRespond {\n\t\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn apiErr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Expected stream.\n\t\t\tif sname := getExpectedStream(hdr); sname != _EMPTY_ && sname != name {\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSStreamNotMatchError()\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn errStreamMismatch\n\t\t\t}\n\n\t\t\t// TTL'd messages are rejected entirely if TTLs are not enabled on the stream.\n\t\t\t// Shouldn't happen in clustered mode since we should have already caught this\n\t\t\t// in processClusteredInboundMsg, but needed here for non-clustered etc.\n\t\t\tif ttl, _ := getMessageTTL(hdr); !sourced && ttl != 0 && !mset.cfg.AllowMsgTTL {\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSMessageTTLDisabledError()\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn errMsgTTLDisabled\n\t\t\t}\n\n\t\t\t// Expected last sequence per subject.\n\t\t\tif seq, exists := getExpectedLastSeqPerSubject(hdr); exists {\n\t\t\t\t// Allow override of the subject used for the check.\n\t\t\t\tseqSubj := subject\n\t\t\t\tif optSubj := getExpectedLastSeqPerSubjectForSubject(hdr); optSubj != _EMPTY_ {\n\t\t\t\t\tseqSubj = optSubj\n\t\t\t\t}\n\n\t\t\t\t// TODO(dlc) - We could make a new store func that does this all in one.\n\t\t\t\tvar smv StoreMsg\n\t\t\t\tvar fseq uint64\n\t\t\t\tsm, err := store.LoadLastMsg(seqSubj, &smv)\n\t\t\t\tif sm != nil {\n\t\t\t\t\tfseq = sm.seq\n\t\t\t\t}\n\t\t\t\tif err == ErrStoreMsgNotFound && seq == 0 {\n\t\t\t\t\tfseq, err = 0, nil\n\t\t\t\t}\n\t\t\t\tif err != nil || fseq != seq {\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = NewJSStreamWrongLastSequenceError(fseq)\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn fmt.Errorf(\"last sequence by subject mismatch: %d vs %d\", seq, fseq)\n\t\t\t\t}\n\t\t\t} else if getExpectedLastSeqPerSubjectForSubject(hdr) != _EMPTY_ {\n\t\t\t\tapiErr := NewJSStreamExpectedLastSeqPerSubjectInvalidError()\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn apiErr\n\t\t\t}\n\n\t\t\t// Expected last sequence.\n\t\t\tif seq, exists := getExpectedLastSeq(hdr); exists && seq != mset.lseq {\n\t\t\t\tmlseq := mset.lseq\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSStreamWrongLastSequenceError(mlseq)\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"last sequence mismatch: %d vs %d\", seq, mlseq)\n\t\t\t}\n\n\t\t\t// Message scheduling.\n\t\t\tif schedule, ok := getMessageSchedule(hdr); !ok {\n\t\t\t\tapiErr := NewJSMessageSchedulesPatternInvalidError()\n\t\t\t\tif !allowMsgSchedules {\n\t\t\t\t\tapiErr = NewJSMessageSchedulesDisabledError()\n\t\t\t\t}\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn apiErr\n\t\t\t} else if !schedule.IsZero() {\n\t\t\t\tif !allowMsgSchedules {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesDisabledError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else if scheduleTtl, ok := getMessageScheduleTTL(hdr); !ok {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesTTLInvalidError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else if scheduleTtl != _EMPTY_ && !mset.cfg.AllowMsgTTL {\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = NewJSMessageTTLDisabledError()\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn errMsgTTLDisabled\n\t\t\t\t} else if scheduleTarget := getMessageScheduleTarget(hdr); scheduleTarget == _EMPTY_ ||\n\t\t\t\t\t!IsValidPublishSubject(scheduleTarget) || SubjectsCollide(scheduleTarget, subject) {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesTargetInvalidError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else if scheduleSource := getMessageScheduleSource(hdr); scheduleSource != _EMPTY_ &&\n\t\t\t\t\t(scheduleSource == scheduleTarget || scheduleSource == subject || !IsValidPublishSubject(scheduleSource)) {\n\t\t\t\t\tapiErr := NewJSMessageSchedulesSourceInvalidError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t} else {\n\t\t\t\t\tmatch := slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {\n\t\t\t\t\t\treturn SubjectsCollide(subj, scheduleTarget)\n\t\t\t\t\t})\n\t\t\t\t\tif !match {\n\t\t\t\t\t\tapiErr := NewJSMessageSchedulesTargetInvalidError()\n\t\t\t\t\t\tif canRespond {\n\t\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn apiErr\n\t\t\t\t\t}\n\n\t\t\t\t\t// Add a rollup sub header if it doesn't already exist.\n\t\t\t\t\t// Otherwise, it must exist already as a rollup on the subject.\n\t\t\t\t\tif rollup := getRollup(hdr); rollup == _EMPTY_ {\n\t\t\t\t\t\thdr = genHeader(hdr, JSMsgRollup, JSMsgRollupSubject)\n\t\t\t\t\t} else if rollup != JSMsgRollupSubject {\n\t\t\t\t\t\tapiErr := NewJSMessageSchedulesRollupInvalidError()\n\t\t\t\t\t\tif canRespond {\n\t\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn apiErr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Dedupe detection. This is done at the cluster level for dedupe detection above the\n\t\t// lower layers. But we still need to pull out the msgId.\n\t\tif msgId = getMsgId(hdr); msgId != _EMPTY_ {\n\t\t\t// Do real check only if not clustered or traceOnly flag is set.\n\t\t\tif canConsistencyCheck {\n\t\t\t\tvar seq uint64\n\t\t\t\tmset.ddMu.Lock()\n\t\t\t\tdde := mset.checkMsgId(msgId)\n\t\t\t\tif dde != nil {\n\t\t\t\t\tseq = dde.seq\n\t\t\t\t}\n\t\t\t\tmset.ddMu.Unlock()\n\t\t\t\tif seq > 0 {\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresponse := append(pubAck, strconv.FormatUint(seq, 10)...)\n\t\t\t\t\t\tresponse = append(response, \",\\\"duplicate\\\": true}\"...)\n\t\t\t\t\t\toutq.sendMsg(reply, response)\n\t\t\t\t\t}\n\t\t\t\t\treturn errMsgIdDuplicate\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Expected last msgId.\n\t\tif lmsgId := getExpectedLastMsgId(hdr); lmsgId != _EMPTY_ {\n\t\t\tif lmsgId != mset.lmsgId {\n\t\t\t\tlast := mset.lmsgId\n\t\t\t\tbumpCLFS()\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSStreamWrongLastMsgIDError(last)\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"last msgid mismatch: %q vs %q\", lmsgId, last)\n\t\t\t}\n\t\t}\n\t\t// Check for any rollups.\n\t\tif rollup := getRollup(hdr); rollup != _EMPTY_ {\n\t\t\tif canConsistencyCheck && (!mset.cfg.AllowRollup || mset.cfg.DenyPurge) {\n\t\t\t\terr := errors.New(\"rollup not permitted\")\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSStreamRollupFailedError(err)\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tswitch rollup {\n\t\t\tcase JSMsgRollupSubject:\n\t\t\t\trollupSub = true\n\t\t\tcase JSMsgRollupAll:\n\t\t\t\trollupAll = true\n\t\t\tdefault:\n\t\t\t\tif canConsistencyCheck {\n\t\t\t\t\terr := fmt.Errorf(\"rollup value invalid: %q\", rollup)\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = NewJSStreamRollupFailedError(err)\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif canConsistencyCheck && incr == nil && allowMsgCounter {\n\t\tapiErr := NewJSMessageIncrMissingError()\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = apiErr\n\t\t\tb, _ := json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, b)\n\t\t}\n\t\treturn apiErr\n\t}\n\n\t// Response Ack.\n\tvar (\n\t\tresponse []byte\n\t\tseq uint64\n\t\terr error\n\t)\n\n\t// Apply increment for counter.\n\t// But only if it's allowed for this stream. This can happen when we store verbatim for a sourced stream.\n\tif canConsistencyCheck && incr != nil && allowMsgCounter {\n\t\tvar initial big.Int\n\t\tvar sources CounterSources\n\t\tvar smv StoreMsg\n\t\tsm, err := store.LoadLastMsg(subject, &smv)\n\t\tif err == nil && sm != nil {\n\t\t\tvar val CounterValue\n\t\t\t// Return an error if the counter is broken somehow.\n\t\t\tif json.Unmarshal(sm.msg, &val) != nil {\n\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn apiErr\n\t\t\t}\n\t\t\tif ncs := sliceHeader(JSMessageCounterSources, sm.hdr); len(ncs) > 0 {\n\t\t\t\tif err := json.Unmarshal(ncs, &sources); err != nil {\n\t\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\t\tif canRespond {\n\t\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t\t}\n\t\t\t\t\treturn apiErr\n\t\t\t\t}\n\t\t\t}\n\t\t\tinitial.SetString(val.Value, 10)\n\t\t}\n\t\tsrchdr := sliceHeader(JSStreamSource, hdr)\n\t\tif len(srchdr) > 0 {\n\t\t\t// This is a sourced message, so we can't apply Nats-Incr but\n\t\t\t// instead should just update the source count header.\n\t\t\tfields := strings.Split(string(srchdr), \" \")\n\t\t\torigStream := fields[0]\n\t\t\torigSubj := subject\n\t\t\tif len(fields) >= 5 {\n\t\t\t\torigSubj = fields[4]\n\t\t\t}\n\t\t\tvar val CounterValue\n\t\t\tif json.Unmarshal(msg, &val) != nil {\n\t\t\t\tapiErr := NewJSMessageCounterBrokenError()\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = apiErr\n\t\t\t\t\tb, _ := json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, b)\n\t\t\t\t}\n\t\t\t\treturn apiErr\n\t\t\t}\n\t\t\tvar sourced big.Int\n\t\t\tsourced.SetString(val.Value, 10)\n\t\t\tif sources == nil {\n\t\t\t\tsources = map[string]map[string]string{}\n\t\t\t}\n\t\t\tif _, ok := sources[origStream]; !ok {\n\t\t\t\tsources[origStream] = map[string]string{}\n\t\t\t}\n\t\t\tprevVal := sources[origStream][origSubj]\n\t\t\tsources[origStream][origSubj] = sourced.String()\n\t\t\t// We will also replace the Nats-Incr header with the diff\n\t\t\t// between our last value from this source and this one, so\n\t\t\t// that the arithmetic is always correct.\n\t\t\tvar previous big.Int\n\t\t\tprevious.SetString(prevVal, 10)\n\t\t\tincr.Sub(&sourced, &previous)\n\t\t\thdr = setHeader(JSMessageIncr, incr.String(), hdr)\n\t\t}\n\t\t// Now make the change.\n\t\tinitial.Add(&initial, incr)\n\t\t// Generate the new payload.\n\t\tvar _msg [128]byte\n\t\tmsg = fmt.Appendf(_msg[:0], \"{%q:%q}\", \"val\", initial.String())\n\t\t// Write the updated source count headers.\n\t\tif len(sources) > 0 {\n\t\t\tnhdr, err := json.Marshal(sources)\n\t\t\tif err != nil {\n\t\t\t\tif canRespond {\n\t\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\t\tresp.Error = NewJSMessageCounterBrokenError()\n\t\t\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\t\t\toutq.sendMsg(reply, response)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\thdr = setHeader(JSMessageCounterSources, string(nhdr), hdr)\n\t\t}\n\n\t\t// Check to see if we are over the max msg size.\n\t\t// Subtract to prevent against overflows.\n\t\tmaxPayload := int64(mset.srv.getOpts().MaxPayload)\n\t\thdrLen, msgLen := int64(len(hdr)), int64(len(msg))\n\t\tif hdrLen > maxPayload || msgLen > maxPayload-hdrLen {\n\t\t\tif canRespond {\n\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\tresp.Error = NewJSStreamMessageExceedsMaximumError()\n\t\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\t\toutq.sendMsg(reply, response)\n\t\t\t}\n\t\t\treturn ErrMaxPayload\n\t\t}\n\t}\n\n\t// Check to see if we are over the max msg size.\n\t// Subtract to prevent against overflows.\n\tif canConsistencyCheck && maxMsgSize >= 0 && (len(hdr) > maxMsgSize || len(msg) > maxMsgSize-len(hdr)) {\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = NewJSStreamMessageExceedsMaximumError()\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\treturn ErrMaxPayload\n\t}\n\n\tif canConsistencyCheck && len(hdr) > math.MaxUint16 {\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = NewJSStreamHeaderExceedsMaximumError()\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\treturn ErrMaxPayload\n\t}\n\n\t// Check to see if we have exceeded our limits.\n\t// Don't error and log/stepdown if we're tracing when clustered.\n\tif !isClustered && js.limitsExceeded(stype) {\n\t\ts.resourcesExceededError(stype)\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = NewJSInsufficientResourcesError()\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\t// Stepdown regardless.\n\t\tif node := mset.node; node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn NewJSInsufficientResourcesError()\n\t}\n\n\tvar noInterest bool\n\n\t// If we are interest based retention and have no consumers then we can skip.\n\tif interestRetention {\n\t\tmset.clsMu.RLock()\n\t\tnoInterest = numConsumers == 0 || mset.csl == nil || !mset.csl.HasInterest(subject)\n\t\tmset.clsMu.RUnlock()\n\t}\n\n\t// Grab timestamp if not already set.\n\tif ts == 0 && lseq > 0 {\n\t\tts = time.Now().UnixNano()\n\t}\n\n\tmt.updateJetStreamEvent(subject, noInterest)\n\tif traceOnly {\n\t\treturn nil\n\t}\n\n\t// Skip msg here.\n\tif noInterest {\n\t\tmset.lseq, _ = store.SkipMsg(0)\n\t\tmset.lmsgId = msgId\n\t\t// If we have a msgId make sure to save.\n\t\tif msgId != _EMPTY_ {\n\t\t\tmset.storeMsgId(&ddentry{msgId, mset.lseq, ts})\n\t\t}\n\t\t// If using fast batch publish, we occasionally send flow control messages.\n\t\t// And, we need to ensure a PubAck is sent if the commit happens through EOB.\n\t\tif fastBatch != nil {\n\t\t\tif mset.batches == nil {\n\t\t\t\tmset.batches = &batching{}\n\t\t\t}\n\t\t\tmset.batches.mu.Lock()\n\t\t\t// Check full leader state so we only send the client an update once we're caught up.\n\t\t\tcommit := mset.batches.fastBatchRegisterSequences(mset, reply, mset.lseq, mset.isLeader(), fastBatch)\n\t\t\tmset.batches.mu.Unlock()\n\t\t\tif !commit {\n\t\t\t\treply = _EMPTY_\n\t\t\t\tcanRespond = false\n\t\t\t}\n\t\t}\n\t\tif canRespond {\n\t\t\tresponse = append(pubAck, strconv.FormatUint(mset.lseq, 10)...)\n\t\t\tif batchId != _EMPTY_ {\n\t\t\t\tresponse = append(response, fmt.Sprintf(\",\\\"batch\\\":%q,\\\"count\\\":%d}\", batchId, batchSeq)...)\n\t\t\t} else {\n\t\t\t\tresponse = append(response, '}')\n\t\t\t}\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Republish state if needed.\n\tvar tsubj string\n\tvar tlseq uint64\n\tvar thdrsOnly bool\n\tif mset.tr != nil {\n\t\ttsubj, _ = mset.tr.Match(subject)\n\t\tif mset.cfg.RePublish != nil {\n\t\t\tthdrsOnly = mset.cfg.RePublish.HeadersOnly\n\t\t}\n\t}\n\trepublish := tsubj != _EMPTY_ && isLeader\n\n\t// If we are republishing grab last sequence for this exact subject. Aids in gap detection for lightweight clients.\n\tif republish {\n\t\tvar smv StoreMsg\n\t\tif sm, _ := store.LoadLastMsg(subject, &smv); sm != nil {\n\t\t\ttlseq = sm.seq\n\t\t}\n\t}\n\n\t// If clustered this was already checked and we do not want to check here and possibly introduce skew.\n\t// Don't error and log if we're tracing when clustered.\n\tif !isClustered {\n\t\tif exceeded, err := jsa.wouldExceedLimits(stype, mset.tier, mset.cfg.Replicas, subject, hdr, msg); exceeded {\n\t\t\tif err == nil {\n\t\t\t\terr = NewJSAccountResourcesExceededError()\n\t\t\t}\n\t\t\ts.RateLimitWarnf(\"JetStream %s resource limits exceeded for account: %q\", strings.ToLower(stype.String()), accName)\n\t\t\tif canRespond {\n\t\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\t\tresp.Error = err\n\t\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\t\toutq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0))\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Find the message TTL if any.\n\tttl, err := getMessageTTL(hdr)\n\tif err != nil {\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = NewJSMessageTTLInvalidError()\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, nil, response, nil, 0))\n\t\t}\n\t\treturn err\n\t}\n\n\t// If subject delete markers are used, ensure message TTL is that at minimum.\n\t// Otherwise, subject delete markers could be missed if one already exists for this subject.\n\t// MaxMsgsPer=1 is an exception, because we'll only ever have one message.\n\tif ttl > 0 && mset.cfg.SubjectDeleteMarkerTTL > 0 && mset.cfg.MaxMsgsPer != 1 {\n\t\tif minTtl := int64(mset.cfg.SubjectDeleteMarkerTTL.Seconds()); ttl < minTtl {\n\t\t\tttl = minTtl\n\t\t\thdr = removeHeaderIfPresent(hdr, JSMessageTTL)\n\t\t\thdr = genHeader(hdr, JSMessageTTL, strconv.FormatInt(ttl, 10))\n\t\t}\n\t}\n\n\t// Store actual msg.\n\tif lseq == 0 && ts == 0 {\n\t\tseq, ts, err = store.StoreMsg(subject, hdr, msg, ttl)\n\t} else {\n\t\t// Make sure to take into account any message assignments that we had to skip (clfs).\n\t\tseq = lseq + 1 - clfs\n\t\terr = store.StoreRawMsg(subject, hdr, msg, seq, ts, ttl, canConsistencyCheck)\n\t}\n\n\tif err != nil {\n\t\tif isPermissionError(err) {\n\t\t\t// messages in block cache could be lost in the worst case.\n\t\t\t// In the clustered mode it is very highly unlikely as a result of replication.\n\t\t\tgo mset.srv.DisableJetStream()\n\t\t\tmset.srv.Warnf(\"Filesystem permission denied while writing msg, disabling JetStream: %v\", err)\n\t\t\treturn err\n\t\t}\n\n\t\tswitch err {\n\t\tcase ErrMaxMsgs, ErrMaxBytes, ErrMaxMsgsPerSubject, ErrMsgTooLarge:\n\t\t\ts.RateLimitDebugf(\"JetStream failed to store a msg on stream '%s > %s': %v\", accName, name, err)\n\t\tcase ErrStoreClosed:\n\t\tdefault:\n\t\t\t// We don't want to respond back to the user, and definitely not up CLFS either.\n\t\t\t// This was likely an IO issue, so only log and return the error. This will stop\n\t\t\t// the stream if it was replicated.\n\t\t\ts.RateLimitErrorf(\"JetStream failed to store a msg on stream '%s > %s': %v\", accName, name, err)\n\t\t\tmset.setWriteErrLocked(err)\n\t\t\treturn err\n\t\t}\n\n\t\t// If we did not succeed increment clfs in case we are clustered.\n\t\tbumpCLFS()\n\t\tif canRespond {\n\t\t\tresp.PubAck = &PubAck{Stream: name}\n\t\t\tresp.Error = NewJSStreamStoreFailedError(err, Unless(err))\n\t\t\tresponse, _ = json.Marshal(resp)\n\t\t\toutq.sendMsg(reply, response)\n\t\t}\n\t\treturn err\n\t}\n\n\t// Check for preAcks and the need to clear it.\n\tif mset.hasAllPreAcks(seq, subject) {\n\t\tmset.clearAllPreAcks(seq)\n\t\t// If we're clustered and the stream leader, we can now propose deleting this message.\n\t\t// We still store it below, so we remain properly synchronized with our followers.\n\t\t// If this proposal fails, we retry out-of-band.\n\t\tif isClustered && isLeader {\n\t\t\tmd := streamMsgDelete{Seq: seq, NoErase: true, Stream: mset.cfg.Name}\n\t\t\t_ = mset.node.Propose(encodeMsgDelete(&md))\n\t\t}\n\t}\n\n\t// If here we succeeded in storing the message.\n\tmset.lmsgId = msgId\n\tmset.lseq = seq\n\n\t// If we have a msgId make sure to save.\n\t// This will replace our estimate from the cluster layer if we are clustered.\n\tif msgId != _EMPTY_ {\n\t\tmset.ddMu.Lock()\n\t\tif isClustered && isLeader && mset.ddmap != nil {\n\t\t\tif dde := mset.ddmap[msgId]; dde != nil {\n\t\t\t\tdde.seq, dde.ts = seq, ts\n\t\t\t} else {\n\t\t\t\tmset.storeMsgIdLocked(&ddentry{msgId, seq, ts})\n\t\t\t}\n\t\t} else {\n\t\t\t// R1 or not leader..\n\t\t\tmset.storeMsgIdLocked(&ddentry{msgId, seq, ts})\n\t\t}\n\t\tmset.ddMu.Unlock()\n\t}\n\n\t// No errors, this is the normal path.\n\tif rollupSub {\n\t\tif _, err = mset.purgeLocked(&JSApiStreamPurgeRequest{Subject: subject, Keep: 1}, false); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if rollupAll {\n\t\tif _, err = mset.purgeLocked(&JSApiStreamPurgeRequest{Keep: 1}, false); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if scheduleNext := sliceHeader(JSScheduleNext, hdr); len(scheduleNext) > 0 && bytesToString(scheduleNext) == JSScheduleNextPurge {\n\t\t// Purge the message schedule.\n\t\tscheduler := getMessageScheduler(hdr)\n\t\tif scheduler != _EMPTY_ {\n\t\t\tif _, err = mset.purgeLocked(&JSApiStreamPurgeRequest{Subject: scheduler}, false); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check for republish.\n\tif republish {\n\t\tconst ht = \"NATS/1.0\\r\\nNats-Stream: %s\\r\\nNats-Subject: %s\\r\\nNats-Sequence: %d\\r\\nNats-Time-Stamp: %s\\r\\nNats-Last-Sequence: %d\\r\\n\\r\\n\"\n\t\tconst htho = \"NATS/1.0\\r\\nNats-Stream: %s\\r\\nNats-Subject: %s\\r\\nNats-Sequence: %d\\r\\nNats-Time-Stamp: %s\\r\\nNats-Last-Sequence: %d\\r\\nNats-Msg-Size: %d\\r\\n\\r\\n\"\n\t\t// When adding to existing headers, will use the fmt.Append version so this skips the headers from above.\n\t\tconst hoff = 10\n\n\t\ttsStr := time.Unix(0, ts).UTC().Format(time.RFC3339Nano)\n\t\tvar rpMsg []byte\n\t\tif len(hdr) == 0 {\n\t\t\tif !thdrsOnly {\n\t\t\t\thdr = fmt.Appendf(nil, ht, name, subject, seq, tsStr, tlseq)\n\t\t\t\trpMsg = copyBytes(msg)\n\t\t\t} else {\n\t\t\t\thdr = fmt.Appendf(nil, htho, name, subject, seq, tsStr, tlseq, len(msg))\n\t\t\t}\n\t\t} else {\n\t\t\t// use hdr[:end:end] to make sure as we add we copy the original hdr.\n\t\t\tend := len(hdr) - LEN_CR_LF\n\t\t\tif !thdrsOnly {\n\t\t\t\thdr = fmt.Appendf(hdr[:end:end], ht[hoff:], name, subject, seq, tsStr, tlseq)\n\t\t\t\trpMsg = copyBytes(msg)\n\t\t\t} else {\n\t\t\t\thdr = fmt.Appendf(hdr[:end:end], htho[hoff:], name, subject, seq, tsStr, tlseq, len(msg))\n\t\t\t}\n\t\t}\n\t\toutq.send(newJSPubMsg(tsubj, _EMPTY_, _EMPTY_, hdr, rpMsg, nil, seq))\n\t}\n\n\t// If using fast batch publish, we occasionally send flow control messages.\n\t// And, we need to ensure a PubAck is sent if the commit happens through EOB.\n\tif fastBatch != nil {\n\t\tif mset.batches == nil {\n\t\t\tmset.batches = &batching{}\n\t\t}\n\t\tmset.batches.mu.Lock()\n\t\t// Check full leader state so we only send the client an update once we're caught up.\n\t\tcommit := mset.batches.fastBatchRegisterSequences(mset, reply, mset.lseq, mset.isLeader(), fastBatch)\n\t\tmset.batches.mu.Unlock()\n\t\tif !commit {\n\t\t\treply = _EMPTY_\n\t\t\tcanRespond = false\n\t\t}\n\t}\n\n\t// Send response here.\n\tif canRespond {\n\t\tresponse = append(pubAck, strconv.FormatUint(seq, 10)...)\n\t\tif batchId != _EMPTY_ {\n\t\t\tresponse = append(response, fmt.Sprintf(\",\\\"batch\\\":%q,\\\"count\\\":%d}\", batchId, batchSeq)...)\n\t\t} else if allowMsgCounter {\n\t\t\tvar counter CounterValue\n\t\t\tjson.Unmarshal(msg, &counter)\n\t\t\tresponse = append(response, fmt.Sprintf(\",\\\"val\\\":%q}\", counter.Value)...)\n\t\t} else {\n\t\t\tresponse = append(response, '}')\n\t\t}\n\t\toutq.sendMsg(reply, response)\n\t}\n\n\t// Signal consumers for new messages.\n\tif numConsumers > 0 {\n\t\tmset.sigq.push(newCMsg(subject, seq))\n\t\tselect {\n\t\tcase mset.sch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// processJetStreamAtomicBatchMsg processes a JetStream message that's part of an atomic batch publish.\n// Handles constraints around the batch, storing messages, doing consistency checks, and performing the commit.\nfunc (mset *stream) processJetStreamAtomicBatchMsg(batchId, subject, reply string, hdr, msg []byte, mt *msgTrace) (retErr error) {\n\tmset.mu.RLock()\n\tcanRespond := !mset.cfg.NoAck && len(reply) > 0\n\tname, stype := mset.cfg.Name, mset.cfg.Storage\n\tdiscard, discardNewPer, maxMsgs, maxMsgsPer, maxBytes := mset.cfg.Discard, mset.cfg.DiscardNewPer, mset.cfg.MaxMsgs, mset.cfg.MaxMsgsPer, mset.cfg.MaxBytes\n\ts, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node\n\tmaxMsgSize, lseq := int(mset.cfg.MaxMsgSize), mset.lseq\n\tisLeader, isClustered, isSealed, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules, allowAtomicPublish := mset.isLeader(), mset.isClustered(), mset.cfg.Sealed, mset.cfg.AllowRollup, mset.cfg.DenyPurge, mset.cfg.AllowMsgTTL, mset.cfg.AllowMsgCounter, mset.cfg.AllowMsgSchedules, mset.cfg.AllowAtomicPublish\n\tmset.mu.RUnlock()\n\n\t// If message tracing (with message delivery), we will need to send the\n\t// event on exit in case there was an error (if message was not proposed).\n\t// Otherwise, the event will be sent from processJetStreamMsg when\n\t// invoked by the leader (from applyStreamEntries).\n\tif mt != nil {\n\t\tdefer func() {\n\t\t\tif retErr != nil {\n\t\t\t\tmt.sendEventFromJetStream(retErr)\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Check that we are the leader. This can be false if we have scaled up from an R1 that had inbound queued messages.\n\tif !isLeader {\n\t\treturn NewJSClusterNotLeaderError()\n\t}\n\n\trespondError := func(apiErr *ApiError) error {\n\t\tif canRespond {\n\t\t\tbuf, _ := json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: name}, Error: apiErr})\n\t\t\toutq.sendMsg(reply, buf)\n\t\t}\n\t\treturn apiErr\n\t}\n\n\t// Bail here if sealed.\n\tif isSealed {\n\t\treturn respondError(NewJSStreamSealedError())\n\t}\n\n\t// Check here pre-emptively if we have exceeded this server limits.\n\tif js.limitsExceeded(stype) {\n\t\ts.resourcesExceededError(stype)\n\t\t// Stepdown regardless.\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn respondError(NewJSInsufficientResourcesError())\n\t}\n\n\t// Check here pre-emptively if we have exceeded our account limits.\n\tif exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded {\n\t\tif err == nil {\n\t\t\terr = NewJSAccountResourcesExceededError()\n\t\t}\n\t\ts.RateLimitWarnf(\"JetStream account limits exceeded for '%s': %s\", jsa.acc().GetName(), err.Error())\n\t\treturn respondError(err)\n\t}\n\n\t// Check msgSize if we have a limit set there. Again this works if it goes through but better to be pre-emptive.\n\t// Subtract to prevent against overflows.\n\tif maxMsgSize >= 0 && (len(hdr) > maxMsgSize || len(msg) > maxMsgSize-len(hdr)) {\n\t\terr := fmt.Errorf(\"JetStream message size exceeds limits for '%s > %s'\", jsa.acc().Name, mset.cfg.Name)\n\t\ts.RateLimitWarnf(\"%s\", err.Error())\n\t\t_ = respondError(NewJSStreamMessageExceedsMaximumError())\n\t\treturn err\n\t}\n\n\tif !allowAtomicPublish {\n\t\treturn respondError(NewJSAtomicPublishDisabledError())\n\t}\n\n\t// Batch ID is too long.\n\tif len(batchId) > 64 {\n\t\treturn respondError(NewJSAtomicPublishInvalidBatchIDError())\n\t}\n\n\tbatchSeq, exists := getBatchSequence(hdr)\n\tif !exists {\n\t\treturn respondError(NewJSAtomicPublishMissingSeqError())\n\t}\n\n\tmset.mu.Lock()\n\tif mset.batches == nil {\n\t\tmset.batches = &batching{}\n\t}\n\tbatches := mset.batches\n\tmset.mu.Unlock()\n\n\trespondIncompleteBatch := func() error {\n\t\treturn respondError(NewJSAtomicPublishIncompleteBatchError())\n\t}\n\n\t// Get batch.\n\tbatches.mu.Lock()\n\tb, ok := batches.atomic[batchId]\n\tif !ok {\n\t\tif batchSeq != 1 {\n\t\t\tbatches.mu.Unlock()\n\t\t\tmaxBatchSize := streamMaxAtomicBatchSize\n\t\t\topts := s.getOpts()\n\t\t\tif opts.JetStreamLimits.MaxBatchSize > 0 {\n\t\t\t\tmaxBatchSize = opts.JetStreamLimits.MaxBatchSize\n\t\t\t}\n\t\t\tif batchSeq > uint64(maxBatchSize) {\n\t\t\t\terr := NewJSAtomicPublishTooLargeBatchError(maxBatchSize)\n\t\t\t\treturn respondError(err)\n\t\t\t}\n\t\t\treturn respondIncompleteBatch()\n\t\t}\n\n\t\t// Limits.\n\t\tmaxInflightPerStream := streamMaxAtomicBatchInflightPerStream\n\t\tmaxInflightTotal := streamMaxAtomicBatchInflightTotal\n\t\topts := s.getOpts()\n\t\tif opts.JetStreamLimits.MaxBatchInflightPerStream > 0 {\n\t\t\tmaxInflightPerStream = opts.JetStreamLimits.MaxBatchInflightPerStream\n\t\t}\n\t\tif opts.JetStreamLimits.MaxBatchInflightTotal > 0 {\n\t\t\tmaxInflightTotal = opts.JetStreamLimits.MaxBatchInflightTotal\n\t\t}\n\n\t\t// Confirm we can facilitate an additional batch.\n\t\tif len(batches.atomic)+1 > maxInflightPerStream {\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSAtomicPublishTooManyInflightError())\n\t\t}\n\n\t\t// Confirm we'll not exceed the server limit.\n\t\tif globalInflightAtomicBatches.Add(1) > int64(maxInflightTotal) {\n\t\t\tglobalInflightAtomicBatches.Add(-1)\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSAtomicPublishTooManyInflightError())\n\t\t}\n\n\t\tvar err error\n\t\tb, err = batches.newAtomicBatch(mset, batchId)\n\t\tif err != nil {\n\t\t\tglobalInflightAtomicBatches.Add(-1)\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondIncompleteBatch()\n\t\t}\n\t\tif batches.atomic == nil {\n\t\t\tbatches.atomic = make(map[string]*atomicBatch, 1)\n\t\t}\n\t\tbatches.atomic[batchId] = b\n\t}\n\n\tvar commit, commitEob bool\n\tif c := sliceHeader(JSBatchCommit, hdr); c != nil {\n\t\tcommitEob = bytes.Equal(c, []byte(\"eob\"))\n\t\t// Reject the batch if the commit is not recognized.\n\t\tif !commitEob && !bytes.Equal(c, []byte(\"1\")) {\n\t\t\tb.cleanupLocked(batchId, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\terr := NewJSAtomicPublishInvalidBatchCommitError()\n\t\t\treturn respondError(err)\n\t\t}\n\t\tcommit = true\n\t}\n\n\t// The required API level can have the batch be rejected. But the header is always removed.\n\tif len(sliceHeader(JSRequiredApiLevel, hdr)) != 0 {\n\t\tif errorOnRequiredApiLevel(hdr) {\n\t\t\tb.cleanupLocked(batchId, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\terr := NewJSRequiredApiLevelError()\n\t\t\treturn respondError(err)\n\t\t}\n\t\thdr = removeHeaderIfPresent(hdr, JSRequiredApiLevel)\n\t}\n\n\t// If cleanup has already happened, we can't continue.\n\tcleanup := !b.resetCleanupTimer(mset)\n\n\t// Detect gaps.\n\tb.lseq++\n\tif b.lseq != batchSeq || cleanup {\n\t\tb.cleanupLocked(batchId, batches)\n\t\tbatches.mu.Unlock()\n\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, BatchIncomplete)\n\t\treturn respondIncompleteBatch()\n\t}\n\n\t// Confirm the batch doesn't exceed the allowed size.\n\tmaxSize := streamMaxAtomicBatchSize\n\tif maxBatchSize := s.getOpts().JetStreamLimits.MaxBatchSize; maxBatchSize > 0 {\n\t\tmaxSize = maxBatchSize\n\t}\n\tif batchSeq > uint64(maxSize) {\n\t\tb.cleanupLocked(batchId, batches)\n\t\tbatches.mu.Unlock()\n\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, BatchLarge)\n\t\terr := NewJSAtomicPublishTooLargeBatchError(maxSize)\n\t\treturn respondError(err)\n\t}\n\n\t// Persist, but optimize if we're committing because we already know last.\n\t// If the underlying store is file-based we need to persist everything to survive hard kills, because we're R1.\n\tif !commit || b.store.Type() == FileStorage {\n\t\tseq, _, err := b.store.StoreMsg(subject, hdr, msg, 0)\n\t\tif err != nil || seq != batchSeq {\n\t\t\tb.cleanupLocked(batchId, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, BatchIncomplete)\n\t\t\treturn respondIncompleteBatch()\n\t\t}\n\t}\n\tif !commit {\n\t\tbatches.mu.Unlock()\n\t\t// Send empty ack to let them know we've persisted the data prior to commit.\n\t\tif canRespond {\n\t\t\toutq.sendMsg(reply, nil)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Ensure the batch is prepared for the commit and will not be cleaned up while committing.\n\tif abandonReason := b.readyForCommit(); abandonReason != nil {\n\t\t// Don't do cleanup, this is already done.\n\t\tbatches.mu.Unlock()\n\t\tmset.sendStreamBatchAbandonedAdvisory(batchId, *abandonReason)\n\t\treturn respondIncompleteBatch()\n\t}\n\n\t// Proceed with proposing the batch.\n\tts := time.Now().UnixNano()\n\n\t// Need to hold this lock even if we're not clustered, because we'll\n\t// be accessing state that requires this lock (even if it's empty).\n\tmset.clMu.Lock()\n\n\t// If not clustered, set up the last sequence.\n\tif !isClustered {\n\t\tmset.clseq, mset.clfs = mset.lseq, 0\n\t}\n\n\t// We only use mset.clseq for clustering and in case we run ahead of actual commits.\n\t// Check if we need to set initial value here\n\tif isClustered && (mset.clseq == 0 || mset.clseq < lseq+mset.clfs) {\n\t\tlseq = recalculateClusteredSeq(mset)\n\t}\n\n\trollback := func(seq uint64) {\n\t\tif isClustered {\n\t\t\t// Only need to move the clustered sequence back if the batch fails to commit.\n\t\t\t// Other changes were staged but not applied, so this is the only thing we need to do.\n\t\t\tmset.clseq -= seq - 1\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\n\terrorOnUnsupported := func(seq uint64, header string) *ApiError {\n\t\tapiErr := NewJSAtomicPublishUnsupportedHeaderBatchError(header)\n\t\trollback(seq)\n\t\tb.cleanupLocked(batchId, batches)\n\t\tbatches.mu.Unlock()\n\t\t_ = respondError(apiErr)\n\t\treturn apiErr\n\t}\n\n\tvar (\n\t\tentries []*Entry\n\t\tbsubj string\n\t\tbhdr []byte\n\t\tbmsg []byte\n\t\tapiErr *ApiError\n\t\terr error\n\t\tsmv StoreMsg\n\t\tsm *StoreMsg\n\t\tsz int\n\t)\n\n\t// If the commit ends with an \"End Of Batch\" message, we don't store this.\n\tif commitEob {\n\t\tbatchSeq--\n\t}\n\n\tdiff := &batchStagedDiff{}\n\tfor seq := uint64(1); seq <= batchSeq; seq++ {\n\t\tif seq == batchSeq && !commitEob && b.store.Type() != FileStorage {\n\t\t\tbsubj, bhdr, bmsg = subject, hdr, msg\n\t\t} else if sm, err = b.store.LoadMsg(seq, &smv); sm != nil && err == nil {\n\t\t\tbsubj, bhdr, bmsg = sm.subj, sm.hdr, sm.msg\n\t\t} else {\n\t\t\trollback(seq)\n\t\t\tb.cleanupLocked(batchId, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondIncompleteBatch()\n\t\t}\n\n\t\t// Reject unsupported headers.\n\t\tif getExpectedLastMsgId(bhdr) != _EMPTY_ {\n\t\t\treturn errorOnUnsupported(seq, JSExpectedLastMsgId)\n\t\t}\n\n\t\tif bhdr, bmsg, _, apiErr, err = checkMsgHeadersPreClusteredProposal(diff, mset, bsubj, bhdr, bmsg, false, name, jsa, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules, discard, discardNewPer, maxMsgSize, maxMsgs, maxMsgsPer, maxBytes); err != nil {\n\t\t\trollback(seq)\n\t\t\tb.cleanupLocked(batchId, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\t_ = respondError(apiErr)\n\t\t\treturn err\n\t\t}\n\n\t\tif isClustered {\n\t\t\tvar _reply string\n\t\t\tisCommit := seq == batchSeq\n\t\t\tif isCommit {\n\t\t\t\t_reply = reply\n\t\t\t\t// If committed by EOB, the last message must get the normal commit header.\n\t\t\t\tif commitEob {\n\t\t\t\t\tbhdr = genHeader(bhdr, JSBatchCommit, \"1\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tesm := encodeStreamMsgAllowCompressAndBatch(bsubj, _reply, bhdr, bmsg, mset.clseq, ts, false, batchId, seq, isCommit)\n\t\t\tentries = append(entries, newEntry(EntryNormal, esm))\n\t\t\tsz += len(esm)\n\t\t}\n\t\tmset.clseq++\n\t}\n\n\t// Commit batch.\n\tif !isClustered {\n\t\t// Reset, we only used this to do the batching checks.\n\t\tmset.clseq, mset.clfs = 0, 0\n\t\tmset.clMu.Unlock()\n\n\t\t// Ensure the whole batch is fully isolated, and reads\n\t\t// can only happen after the full batch is committed.\n\t\tmset.mu.Lock()\n\t\tfor seq := uint64(1); seq <= batchSeq; seq++ {\n\t\t\tif seq == batchSeq && !commitEob && b.store.Type() != FileStorage {\n\t\t\t\tbsubj, bhdr, bmsg = subject, hdr, msg\n\t\t\t} else if sm, err = b.store.LoadMsg(seq, &smv); sm != nil && err == nil {\n\t\t\t\tbsubj, bhdr, bmsg = sm.subj, sm.hdr, sm.msg\n\t\t\t} else {\n\t\t\t\t// Should not happen, we've already checked this message existed while doing consistency checks.\n\t\t\t\t// We'll just exit here, the batch is already inconsistent without the message at this sequence.\n\t\t\t\t// No use in trying to still store the rest.\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tvar _reply string\n\t\t\tif seq == batchSeq {\n\t\t\t\t_reply = reply\n\t\t\t\t// If committed by EOB, the last message must get the normal commit header.\n\t\t\t\tif commitEob {\n\t\t\t\t\tbhdr = genHeader(bhdr, JSBatchCommit, \"1\")\n\t\t\t\t}\n\t\t\t}\n\t\t\t_ = mset.processJetStreamMsg(bsubj, _reply, bhdr, bmsg, 0, 0, mt, false, false)\n\t\t}\n\t\tmset.mu.Unlock()\n\t} else {\n\t\t// Do a single multi proposal. This ensures we get to push all entries to the proposal queue in-order\n\t\t// and not interleaved with other proposals.\n\t\tdiff.commit(mset)\n\t\t_ = node.ProposeMulti(entries)\n\t\t// The proposal can fail, but we always account for trying.\n\t\tmset.trackReplicationTraffic(node, sz, r)\n\n\t\t// Check to see if we are being overrun.\n\t\t// TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured.\n\t\tif mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold {\n\t\t\tlerr := fmt.Errorf(\"JetStream stream '%s > %s' has high message lag\", jsa.acc().Name, name)\n\t\t\ts.RateLimitWarnf(\"%s\", lerr.Error())\n\t\t}\n\t\tmset.clMu.Unlock()\n\t}\n\tb.cleanupLocked(batchId, batches)\n\tbatches.mu.Unlock()\n\treturn nil\n}\n\n// processJetStreamFastBatchMsg processes a JetStream message that's part of an atomic batch publish.\n// Handles constraints around the batch, storing messages, doing consistency checks, and performing the commit.\nfunc (mset *stream) processJetStreamFastBatchMsg(batch *FastBatch, subject, reply string, hdr, msg []byte, mt *msgTrace) (retErr error) {\n\tmset.mu.RLock()\n\tcanRespond := !mset.cfg.NoAck && len(reply) > 0\n\tname, stype := mset.cfg.Name, mset.cfg.Storage\n\tdiscard, discardNewPer, maxMsgs, maxMsgsPer, maxBytes := mset.cfg.Discard, mset.cfg.DiscardNewPer, mset.cfg.MaxMsgs, mset.cfg.MaxMsgsPer, mset.cfg.MaxBytes\n\ts, js, jsa, st, r, tierName, outq, node := mset.srv, mset.js, mset.jsa, mset.cfg.Storage, mset.cfg.Replicas, mset.tier, mset.outq, mset.node\n\tmaxMsgSize, lseq := int(mset.cfg.MaxMsgSize), mset.lseq\n\tisLeader, isClustered, isSealed, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules, allowBatchPublish := mset.isLeader(), mset.isClustered(), mset.cfg.Sealed, mset.cfg.AllowRollup, mset.cfg.DenyPurge, mset.cfg.AllowMsgTTL, mset.cfg.AllowMsgCounter, mset.cfg.AllowMsgSchedules, mset.cfg.AllowBatchPublish\n\tmset.mu.RUnlock()\n\n\t// If message tracing (with message delivery), we will need to send the\n\t// event on exit in case there was an error (if message was not proposed).\n\t// Otherwise, the event will be sent from processJetStreamMsg when\n\t// invoked by the leader (from applyStreamEntries).\n\tif mt != nil {\n\t\tdefer func() {\n\t\t\tif retErr != nil {\n\t\t\t\tmt.sendEventFromJetStream(retErr)\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Check that we are the leader. This can be false if we have scaled up from an R1 that had inbound queued messages.\n\tif !isLeader {\n\t\treturn NewJSClusterNotLeaderError()\n\t}\n\n\trespondError := func(apiErr *ApiError) error {\n\t\tif canRespond {\n\t\t\tbuf, _ := json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: name}, Error: apiErr})\n\t\t\toutq.sendMsg(reply, buf)\n\t\t}\n\t\treturn apiErr\n\t}\n\n\t// Bail here if sealed.\n\tif isSealed {\n\t\treturn respondError(NewJSStreamSealedError())\n\t}\n\n\t// Check here pre-emptively if we have exceeded this server limits.\n\tif js.limitsExceeded(stype) {\n\t\ts.resourcesExceededError(stype)\n\t\t// Stepdown regardless.\n\t\tif node := mset.raftNode(); node != nil {\n\t\t\tnode.StepDown()\n\t\t}\n\t\treturn respondError(NewJSInsufficientResourcesError())\n\t}\n\n\t// Check here pre-emptively if we have exceeded our account limits.\n\tif exceeded, err := jsa.wouldExceedLimits(st, tierName, r, subject, hdr, msg); exceeded {\n\t\tif err == nil {\n\t\t\terr = NewJSAccountResourcesExceededError()\n\t\t}\n\t\ts.RateLimitWarnf(\"JetStream account limits exceeded for '%s': %s\", jsa.acc().GetName(), err.Error())\n\t\treturn respondError(err)\n\t}\n\n\t// Check msgSize if we have a limit set there. Again this works if it goes through but better to be pre-emptive.\n\t// Subtract to prevent against overflows.\n\tif maxMsgSize >= 0 && (len(hdr) > maxMsgSize || len(msg) > maxMsgSize-len(hdr)) {\n\t\terr := fmt.Errorf(\"JetStream message size exceeds limits for '%s > %s'\", jsa.acc().Name, mset.cfg.Name)\n\t\ts.RateLimitWarnf(\"%s\", err.Error())\n\t\t_ = respondError(NewJSStreamMessageExceedsMaximumError())\n\t\treturn err\n\t}\n\n\tif !allowBatchPublish {\n\t\treturn respondError(NewJSBatchPublishDisabledError())\n\t}\n\n\tif batch == nil {\n\t\treturn respondError(NewJSBatchPublishInvalidPatternError())\n\t}\n\n\t// Batch ID is too long.\n\tif len(batch.id) > 64 {\n\t\treturn respondError(NewJSBatchPublishInvalidBatchIDError())\n\t}\n\n\tmset.mu.Lock()\n\tif mset.batches == nil {\n\t\tmset.batches = &batching{}\n\t}\n\tbatches := mset.batches\n\tmset.mu.Unlock()\n\n\t// Get batch.\n\tbatches.mu.Lock()\n\tb, ok := batches.fast[batch.id]\n\tif !ok {\n\t\tif batch.seq != 1 {\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSBatchPublishUnknownBatchIDError())\n\t\t}\n\n\t\t// Limits.\n\t\tmaxInflightPerStream := streamMaxFastBatchInflightPerStream\n\t\tmaxInflightTotal := streamMaxFastBatchInflightTotal\n\t\topts := s.getOpts()\n\t\tif opts.JetStreamLimits.MaxBatchInflightPerStream > 0 {\n\t\t\tmaxInflightPerStream = opts.JetStreamLimits.MaxBatchInflightPerStream\n\t\t}\n\t\tif opts.JetStreamLimits.MaxBatchInflightTotal > 0 {\n\t\t\tmaxInflightTotal = opts.JetStreamLimits.MaxBatchInflightTotal\n\t\t}\n\n\t\t// Confirm we can facilitate an additional batch.\n\t\tif len(batches.fast)+1 > maxInflightPerStream {\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSBatchPublishTooManyInflightError())\n\t\t}\n\n\t\t// Confirm we'll not exceed the server limit.\n\t\tif globalInflightFastBatches.Add(1) > int64(maxInflightTotal) {\n\t\t\tglobalInflightFastBatches.Add(-1)\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSBatchPublishTooManyInflightError())\n\t\t}\n\n\t\t// We'll need a copy as we'll use it as a key and later for cleanup.\n\t\tbatchId := copyString(batch.id)\n\t\tb = batches.newFastBatch(mset, batchId, batch.gapOk, batch.flow)\n\t\tif batches.fast == nil {\n\t\t\tbatches.fast = make(map[string]*fastBatch, 1)\n\t\t}\n\t\tbatches.fast[batchId] = b\n\t}\n\n\t// The required API level can have the batch be rejected. But the header is always removed.\n\tif len(sliceHeader(JSRequiredApiLevel, hdr)) != 0 {\n\t\tif errorOnRequiredApiLevel(hdr) {\n\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn respondError(NewJSRequiredApiLevelError())\n\t\t}\n\t\thdr = removeHeaderIfPresent(hdr, JSRequiredApiLevel)\n\t}\n\n\t// Fast publishing resets the cleanup timer.\n\t// If cleanup has already happened, we can't continue.\n\tcleanup := !b.resetCleanupTimer(mset)\n\n\t// A ping operation confirms we've received a minimum amount of data and resends ack messages.\n\tif batch.ping {\n\t\tsendFlowControl := true\n\t\t// Detect a gap or if the batch was cleaned up in the meantime.\n\t\tif batch.seq > b.lseq || cleanup {\n\t\t\t// If a gap is detected, we always report about it.\n\t\t\tbuf := BatchFlowGap{ExpectedLastSequence: b.lseq + 1, CurrentSequence: batch.seq + 1}.MarshalJSON()\n\t\t\toutq.sendMsg(reply, buf)\n\t\t\t// If the gap is okay, we can continue without rejecting.\n\t\t\tif b.gapOk && !cleanup {\n\t\t\t\tb.lseq = batch.seq\n\t\t\t\tif b.pending == 0 {\n\t\t\t\t\tb.pseq = b.lseq\n\t\t\t\t}\n\t\t\t\tsendFlowControl = !b.checkFlowControl(mset, reply, batches)\n\t\t\t} else if cleanup = batches.fastBatchCommit(b, batch.id, mset, reply); cleanup {\n\t\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\t\tsendFlowControl = false\n\t\t\t}\n\t\t}\n\t\tif sendFlowControl {\n\t\t\tb.sendFlowControl(b.fseq, mset, reply)\n\t\t}\n\t\tbatches.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t// If the batch is committing, due to an error, we can't add more messages.\n\t// We simply skip, since the client will be waiting for the PubAck.\n\tif b.commit {\n\t\t// MUST NOT clean up, that will happen when the commit completes.\n\t\tbatches.mu.Unlock()\n\t\treturn nil\n\t}\n\n\t// Detect gaps.\n\tb.lseq++\n\tif b.lseq != batch.seq || cleanup {\n\t\t// If a gap is detected, we always report about it.\n\t\tbuf := BatchFlowGap{ExpectedLastSequence: b.lseq - 1, CurrentSequence: batch.seq}.MarshalJSON()\n\t\toutq.sendMsg(reply, buf)\n\t\t// If the gap is okay, we can continue without rejecting.\n\t\tif b.gapOk && !cleanup {\n\t\t\tb.lseq = batch.seq\n\t\t} else {\n\t\t\t// Revert, since we incremented for the gap check.\n\t\t\tb.lseq--\n\t\t\tif cleanup = batches.fastBatchCommit(b, batch.id, mset, reply); cleanup {\n\t\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\t}\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif batch.commit {\n\t\tif batch.commitEob {\n\t\t\t// Revert, since we incremented for the gap check.\n\t\t\tb.lseq--\n\t\t\t// If there is none pending, correct the persisted sequence as we need to commit below.\n\t\t\tif b.pending == 0 {\n\t\t\t\tb.pseq = b.lseq\n\t\t\t}\n\t\t}\n\t\t// We'll try to immediately send a PubAck if we can.\n\t\t// Only possible if EOB is used and the last message was already persisted\n\t\t// Otherwise, this sets up the commit for the last message we're about to propose.\n\t\tcleanup = batches.fastBatchCommit(b, batch.id, mset, reply)\n\t\tif batch.commitEob {\n\t\t\tif cleanup {\n\t\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\t}\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t// The first message in the batch responds with the settings used for flow control.\n\t// If committing immediately, we only send the PubAck.\n\tif batch.seq == 1 && canRespond && !batch.commit {\n\t\tbuf := BatchFlowAck{Sequence: 0, Messages: b.ackMessages}.MarshalJSON()\n\t\toutq.sendMsg(reply, buf)\n\t}\n\n\t// Proceed with proposing this message.\n\n\t// We only use mset.clseq for clustering and in case we run ahead of actual commits.\n\t// Check if we need to set initial value here\n\tmset.clMu.Lock()\n\tif mset.clseq == 0 || mset.clseq < lseq+mset.clfs {\n\t\tlseq = recalculateClusteredSeq(mset)\n\t}\n\n\tvar (\n\t\tdseq uint64\n\t\tapiErr *ApiError\n\t\terr error\n\t)\n\tdiff := &batchStagedDiff{}\n\tif hdr, msg, dseq, apiErr, err = checkMsgHeadersPreClusteredProposal(diff, mset, subject, hdr, msg, false, name, jsa, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules, discard, discardNewPer, maxMsgSize, maxMsgs, maxMsgsPer, maxBytes); err != nil {\n\t\tmset.clMu.Unlock()\n\n\t\t// If the message is a duplicate, and we have no pending messages, we should check if we need to\n\t\t// send the flow control message here.\n\t\tif err == errMsgIdDuplicate {\n\t\t\tif b.pending == 0 {\n\t\t\t\tb.pseq = batch.seq\n\t\t\t\tb.checkFlowControl(mset, reply, batches)\n\t\t\t}\n\t\t\tif !batch.commit {\n\t\t\t\t// Otherwise, just skip.\n\t\t\t\tbatches.mu.Unlock()\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\t// If a batch immediately errors, we send the same response as we would a normal publish.\n\t\tif !batch.gapOk && b.lseq == 1 {\n\t\t\tvar response []byte\n\t\t\tif err == errMsgIdDuplicate && dseq > 0 {\n\t\t\t\tvar buf [256]byte\n\t\t\t\tresponse = append(buf[:0], mset.pubAck...)\n\t\t\t\tresponse = append(response, strconv.FormatUint(dseq, 10)...)\n\t\t\t\tresponse = append(response, fmt.Sprintf(\",\\\"duplicate\\\": true,\\\"batch\\\":%q,\\\"count\\\":%d}\", batch.id, batch.seq)...)\n\t\t\t} else {\n\t\t\t\tresponse, _ = json.Marshal(&JSPubAckResponse{PubAck: &PubAck{Stream: name}, Error: apiErr})\n\t\t\t}\n\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t\tbatches.mu.Unlock()\n\t\t\toutq.sendMsg(reply, response)\n\t\t\treturn err\n\t\t}\n\n\t\t// We always return the error to the client, unless it's a duplicate.\n\t\tif err != errMsgIdDuplicate {\n\t\t\tbuf := BatchFlowErr{Sequence: batch.seq, Error: apiErr}.MarshalJSON()\n\t\t\toutq.sendMsg(reply, buf)\n\t\t}\n\n\t\t// If gaps are okay, we just allow them to continue.\n\t\tif batch.gapOk {\n\t\t\tbatches.mu.Unlock()\n\t\t\treturn err\n\t\t}\n\n\t\t// Revert the last sequence, we might be able to immediately return the PubAck as part of the commit.\n\t\t// Otherwise, the batch is cleaned up automatically later.\n\t\tif err != errMsgIdDuplicate {\n\t\t\tb.lseq--\n\t\t}\n\t\tif cleanup = batches.fastBatchCommit(b, batch.id, mset, reply); cleanup {\n\t\t\tb.cleanupLocked(batch.id, batches)\n\t\t}\n\t\tbatches.mu.Unlock()\n\t\treturn err\n\t}\n\tb.pending++\n\tbatches.mu.Unlock()\n\tif !isClustered {\n\t\tmset.clMu.Unlock()\n\t\treturn mset.processJetStreamMsgWithBatch(subject, reply, hdr, msg, 0, 0, mt, false, true, batch)\n\t}\n\tcommitSingleMsg(diff, mset, subject, reply, hdr, msg, name, jsa, mt, node, r, lseq)\n\tmset.clMu.Unlock()\n\treturn nil\n}\n\n// Used to signal inbound message to registered consumers.\ntype cMsg struct {\n\tseq uint64\n\tsubj string\n}\n\n// Pool to recycle consumer bound msgs.\nvar cMsgPool sync.Pool\n\n// Used to queue up consumer bound msgs for signaling.\nfunc newCMsg(subj string, seq uint64) *cMsg {\n\tvar m *cMsg\n\tcm := cMsgPool.Get()\n\tif cm != nil {\n\t\tm = cm.(*cMsg)\n\t} else {\n\t\tm = new(cMsg)\n\t}\n\tm.subj, m.seq = subj, seq\n\n\treturn m\n}\n\nfunc (m *cMsg) returnToPool() {\n\tif m == nil {\n\t\treturn\n\t}\n\tm.subj, m.seq = _EMPTY_, 0\n\tcMsgPool.Put(m)\n}\n\n// Go routine to signal consumers.\n// Offloaded from stream msg processing.\nfunc (mset *stream) signalConsumersLoop() {\n\tmset.mu.RLock()\n\ts, qch, sch, msgs := mset.srv, mset.qch, mset.sch, mset.sigq\n\tmset.mu.RUnlock()\n\n\tfor {\n\t\tselect {\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-sch:\n\t\t\tcms := msgs.pop()\n\t\t\tfor _, m := range cms {\n\t\t\t\tseq, subj := m.seq, m.subj\n\t\t\t\tm.returnToPool()\n\t\t\t\t// Signal all appropriate consumers.\n\t\t\t\tmset.signalConsumers(subj, seq)\n\t\t\t}\n\t\t\tmsgs.recycle(&cms)\n\t\t}\n\t}\n}\n\n// This will update and signal all consumers that match.\nfunc (mset *stream) signalConsumers(subj string, seq uint64) {\n\tmset.clsMu.RLock()\n\tdefer mset.clsMu.RUnlock()\n\tcsl := mset.csl\n\tif csl == nil {\n\t\treturn\n\t}\n\tcsl.Match(subj, func(o *consumer) {\n\t\to.processStreamSignal(seq)\n\t})\n}\n\n// Internal message for use by jetstream subsystem.\ntype jsPubMsg struct {\n\tdsubj string // Subject to send to, e.g. _INBOX.xxx\n\treply string\n\tStoreMsg\n\to *consumer\n}\n\nvar jsPubMsgPool = sync.Pool{\n\tNew: func() any {\n\t\treturn &jsPubMsg{}\n\t},\n}\n\nfunc newJSPubMsg(dsubj, subj, reply string, hdr, msg []byte, o *consumer, seq uint64) *jsPubMsg {\n\tm := getJSPubMsgFromPool()\n\tif m.buf == nil {\n\t\tm.buf = make([]byte, 0, len(hdr)+len(msg))\n\t}\n\tbuf := append(m.buf[:0], hdr...)\n\tbuf = append(buf, msg...)\n\thdr = buf[:len(hdr):len(hdr)]\n\tmsg = buf[len(hdr):]\n\t// When getting something from a pool it is critical that all fields are\n\t// initialized. Doing this way guarantees that if someone adds a field to\n\t// the structure, the compiler will fail the build if this line is not updated.\n\t(*m) = jsPubMsg{dsubj, reply, StoreMsg{subj, hdr, msg, buf, seq, 0}, o}\n\treturn m\n}\n\n// Gets a jsPubMsg from the pool.\nfunc getJSPubMsgFromPool() *jsPubMsg {\n\treturn jsPubMsgPool.Get().(*jsPubMsg)\n}\n\nfunc (pm *jsPubMsg) returnToPool() {\n\tif pm == nil {\n\t\treturn\n\t}\n\tpm.subj, pm.dsubj, pm.reply, pm.hdr, pm.msg, pm.o = _EMPTY_, _EMPTY_, _EMPTY_, nil, nil, nil\n\tif len(pm.buf) > 0 {\n\t\tpm.buf = pm.buf[:0]\n\t}\n\tjsPubMsgPool.Put(pm)\n}\n\nfunc (pm *jsPubMsg) size() int {\n\tif pm == nil {\n\t\treturn 0\n\t}\n\treturn len(pm.dsubj) + len(pm.reply) + len(pm.hdr) + len(pm.msg)\n}\n\n// Queue of *jsPubMsg for sending internal system messages.\ntype jsOutQ struct {\n\t*ipQueue[*jsPubMsg]\n}\n\nfunc (q *jsOutQ) sendMsg(subj string, msg []byte) {\n\tif q != nil {\n\t\tq.send(newJSPubMsg(subj, _EMPTY_, _EMPTY_, nil, msg, nil, 0))\n\t}\n}\n\nfunc (q *jsOutQ) send(msg *jsPubMsg) {\n\tif q == nil || msg == nil {\n\t\treturn\n\t}\n\tq.push(msg)\n}\n\nfunc (q *jsOutQ) unregister() {\n\tif q == nil {\n\t\treturn\n\t}\n\tq.ipQueue.unregister()\n}\n\n// StoredMsg is for raw access to messages in a stream.\ntype StoredMsg struct {\n\tSubject string `json:\"subject\"`\n\tSequence uint64 `json:\"seq\"`\n\tHeader []byte `json:\"hdrs,omitempty\"`\n\tData []byte `json:\"data,omitempty\"`\n\tTime time.Time `json:\"time\"`\n}\n\n// This is similar to system semantics but did not want to overload the single system sendq,\n// or require system account when doing simple setup with jetstream.\nfunc (mset *stream) setupSendCapabilities() {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tif mset.outq != nil {\n\t\treturn\n\t}\n\tqname := fmt.Sprintf(\"[ACC:%s] stream '%s' sendQ\", mset.acc.Name, mset.cfg.Name)\n\tmset.outq = &jsOutQ{newIPQueue[*jsPubMsg](mset.srv, qname)}\n\tgo mset.internalLoop()\n}\n\n// Returns the associated account name.\nfunc (mset *stream) accName() string {\n\tif mset == nil {\n\t\treturn _EMPTY_\n\t}\n\tmset.mu.RLock()\n\tacc := mset.acc\n\tmset.mu.RUnlock()\n\treturn acc.Name\n}\n\n// Name returns the stream name.\nfunc (mset *stream) name() string {\n\treturn mset.nameLocked(true)\n}\n\nfunc (mset *stream) nameLocked(needLock bool) string {\n\tif mset == nil {\n\t\treturn _EMPTY_\n\t}\n\tif needLock {\n\t\tmset.mu.RLock()\n\t\tdefer mset.mu.RUnlock()\n\t}\n\treturn mset.cfg.Name\n}\n\nfunc (mset *stream) internalLoop() {\n\tmset.mu.RLock()\n\tsetGoRoutineLabels(pprofLabels{\n\t\t\"account\": mset.acc.Name,\n\t\t\"stream\": mset.cfg.Name,\n\t})\n\ts := mset.srv\n\tc := s.createInternalJetStreamClient()\n\tc.registerWithAccount(mset.acc)\n\tdefer c.closeConnection(ClientClosed)\n\toutq, qch, msgs, gets := mset.outq, mset.qch, mset.msgs, mset.gets\n\n\t// For the ack msgs queue for interest retention.\n\tvar (\n\t\tamch chan struct{}\n\t\tackq *ipQueue[uint64]\n\t)\n\tif mset.ackq != nil {\n\t\tackq, amch = mset.ackq, mset.ackq.ch\n\t}\n\tmset.mu.RUnlock()\n\n\t// Raw scratch buffer.\n\t// This should be rarely used now so can be smaller.\n\tvar _r [1024]byte\n\n\t// To optimize for not converting a string to a []byte slice.\n\tvar (\n\t\tsubj [256]byte\n\t\tdsubj [256]byte\n\t\trply [256]byte\n\t\tszb [10]byte\n\t\thdb [10]byte\n\t)\n\n\tfor {\n\t\tselect {\n\t\tcase <-outq.ch:\n\t\t\tpms := outq.pop()\n\t\t\tfor _, pm := range pms {\n\t\t\t\tc.pa.subject = append(dsubj[:0], pm.dsubj...)\n\t\t\t\tc.pa.deliver = append(subj[:0], pm.subj...)\n\t\t\t\tc.pa.size = len(pm.msg) + len(pm.hdr)\n\t\t\t\tc.pa.szb = append(szb[:0], strconv.Itoa(c.pa.size)...)\n\t\t\t\tif len(pm.reply) > 0 {\n\t\t\t\t\tc.pa.reply = append(rply[:0], pm.reply...)\n\t\t\t\t} else {\n\t\t\t\t\tc.pa.reply = nil\n\t\t\t\t}\n\n\t\t\t\t// If we have an underlying buf that is the wire contents for hdr + msg, else construct on the fly.\n\t\t\t\tvar msg []byte\n\t\t\t\tif len(pm.buf) > 0 {\n\t\t\t\t\tmsg = pm.buf\n\t\t\t\t} else {\n\t\t\t\t\tif len(pm.hdr) > 0 {\n\t\t\t\t\t\tmsg = pm.hdr\n\t\t\t\t\t\tif len(pm.msg) > 0 {\n\t\t\t\t\t\t\tmsg = _r[:0]\n\t\t\t\t\t\t\tmsg = append(msg, pm.hdr...)\n\t\t\t\t\t\t\tmsg = append(msg, pm.msg...)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if len(pm.msg) > 0 {\n\t\t\t\t\t\t// We own this now from a low level buffer perspective so can use directly here.\n\t\t\t\t\t\tmsg = pm.msg\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif len(pm.hdr) > 0 {\n\t\t\t\t\tc.pa.hdr = len(pm.hdr)\n\t\t\t\t\tc.pa.hdb = []byte(strconv.Itoa(c.pa.hdr))\n\t\t\t\t\tc.pa.hdb = append(hdb[:0], strconv.Itoa(c.pa.hdr)...)\n\t\t\t\t} else {\n\t\t\t\t\tc.pa.hdr = -1\n\t\t\t\t\tc.pa.hdb = nil\n\t\t\t\t}\n\n\t\t\t\tmsg = append(msg, _CRLF_...)\n\n\t\t\t\tdidDeliver, _ := c.processInboundClientMsg(msg)\n\t\t\t\tc.pa.szb, c.pa.subject, c.pa.deliver = nil, nil, nil\n\n\t\t\t\t// Check to see if this is a delivery for a consumer and\n\t\t\t\t// we failed to deliver the message. If so alert the consumer.\n\t\t\t\tif pm.o != nil && pm.seq > 0 && !didDeliver {\n\t\t\t\t\tpm.o.didNotDeliver(pm.seq, pm.dsubj)\n\t\t\t\t}\n\t\t\t\tpm.returnToPool()\n\t\t\t}\n\t\t\t// TODO: Move in the for-loop?\n\t\t\tc.flushClients(0)\n\t\t\toutq.recycle(&pms)\n\t\tcase <-msgs.ch:\n\t\t\t// This can possibly change now so needs to be checked here.\n\t\t\tisClustered := mset.IsClustered()\n\t\t\tims := msgs.pop()\n\t\t\tfor _, im := range ims {\n\t\t\t\t// If we are clustered we need to propose this message to the underlying raft group.\n\t\t\t\tif batch, err := getFastBatch(im.rply, im.hdr); batch != nil || err {\n\t\t\t\t\tmset.processJetStreamFastBatchMsg(batch, im.subj, im.rply, im.hdr, im.msg, im.mt)\n\t\t\t\t\tbatch.returnToPool()\n\t\t\t\t} else if batchId := getBatchId(im.hdr); batchId != _EMPTY_ {\n\t\t\t\t\tmset.processJetStreamAtomicBatchMsg(batchId, im.subj, im.rply, im.hdr, im.msg, im.mt)\n\t\t\t\t} else if isClustered {\n\t\t\t\t\tmset.processClusteredInboundMsg(im.subj, im.rply, im.hdr, im.msg, im.mt, false)\n\t\t\t\t} else {\n\t\t\t\t\tmset.processJetStreamMsg(im.subj, im.rply, im.hdr, im.msg, 0, 0, im.mt, false, true)\n\t\t\t\t}\n\t\t\t\tim.returnToPool()\n\t\t\t}\n\t\t\tmsgs.recycle(&ims)\n\t\tcase <-gets.ch:\n\t\t\tdgs := gets.pop()\n\t\t\tfor _, dg := range dgs {\n\t\t\t\tmset.getDirectRequest(&dg.req, dg.reply)\n\t\t\t\tdgPool.Put(dg)\n\t\t\t}\n\t\t\tgets.recycle(&dgs)\n\n\t\tcase <-amch:\n\t\t\tseqs := ackq.pop()\n\t\t\tfor _, seq := range seqs {\n\t\t\t\tmset.ackMsg(nil, seq)\n\t\t\t}\n\t\t\tackq.recycle(&seqs)\n\t\tcase <-qch:\n\t\t\treturn\n\t\tcase <-s.quitCh:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Used to break consumers out of their monitorConsumer go routines.\nfunc (mset *stream) resetAndWaitOnConsumers() {\n\tmset.mu.RLock()\n\tconsumers := make([]*consumer, 0, len(mset.consumers))\n\tfor _, o := range mset.consumers {\n\t\tconsumers = append(consumers, o)\n\t}\n\tmset.mu.RUnlock()\n\n\tfor _, o := range consumers {\n\t\tif node := o.raftNode(); node != nil {\n\t\t\tnode.StepDown()\n\t\t\tnode.Stop()\n\t\t}\n\t\tif o.isMonitorRunning() {\n\t\t\to.signalMonitorQuit()\n\t\t\to.monitorWg.Wait()\n\t\t}\n\t}\n}\n\n// Internal function to delete a stream.\nfunc (mset *stream) delete() error {\n\tif mset == nil {\n\t\treturn nil\n\t}\n\treturn mset.stop(true, true)\n}\n\n// Internal function to stop or delete the stream.\nfunc (mset *stream) stop(deleteFlag, advisory bool) error {\n\tmset.mu.RLock()\n\tjs, jsa, name, offlineReason := mset.js, mset.jsa, mset.cfg.Name, mset.offlineReason\n\tmset.mu.RUnlock()\n\n\tif jsa == nil {\n\t\treturn NewJSNotEnabledForAccountError()\n\t}\n\n\tisShuttingDown := js.isShuttingDown()\n\n\t// Remove from our account map first.\n\tjsa.mu.Lock()\n\t// Preserve in the account if it's marked offline, to have it remain queryable.\n\tif deleteFlag || offlineReason == _EMPTY_ {\n\t\tdelete(jsa.streams, name)\n\t}\n\taccName := jsa.account.Name\n\tjsa.mu.Unlock()\n\n\t// Kick monitor and collect consumers first.\n\tmset.mu.Lock()\n\n\t// Mark closed.\n\tmset.closed.Store(true)\n\n\t// Signal to the monitor loop.\n\t// Can't use qch here.\n\tif mset.mqch != nil {\n\t\tclose(mset.mqch)\n\t\tmset.mqch = nil\n\t}\n\n\t// Stop responding to sync requests.\n\tmset.stopClusterSubs()\n\t// Unsubscribe from direct stream.\n\tmset.unsubscribeToStream(true, isShuttingDown)\n\n\t// Our info sub if we spun it up.\n\tif mset.infoSub != nil {\n\t\tmset.srv.sysUnsubscribe(mset.infoSub)\n\t\tmset.infoSub = nil\n\t}\n\n\t// Clean up consumers.\n\tvar obs []*consumer\n\tfor _, o := range mset.consumers {\n\t\tobs = append(obs, o)\n\t}\n\t// Preserve the consumers if it's marked offline, to have them remain queryable.\n\tif deleteFlag || offlineReason == _EMPTY_ {\n\t\tmset.clsMu.Lock()\n\t\tmset.consumers, mset.cList, mset.csl = nil, nil, nil\n\t\tmset.clsMu.Unlock()\n\t}\n\n\t// Check if we are a mirror.\n\tif mset.mirror != nil && mset.mirror.sub != nil {\n\t\tmset.unsubscribe(mset.mirror.sub)\n\t\tmset.mirror.sub = nil\n\t\tmset.removeInternalConsumer(mset.mirror)\n\t}\n\t// Now check for sources.\n\tif len(mset.sources) > 0 {\n\t\tfor _, si := range mset.sources {\n\t\t\tmset.cancelSourceConsumer(si.iname)\n\t\t}\n\t}\n\tmset.mu.Unlock()\n\n\tfor _, o := range obs {\n\t\tif !o.isClosed() {\n\t\t\t// Third flag says do not broadcast a signal.\n\t\t\t// TODO(dlc) - If we have an err here we don't want to stop\n\t\t\t// but should we log?\n\t\t\to.stopWithFlags(deleteFlag, deleteFlag, false, advisory)\n\t\t\tif !isShuttingDown {\n\t\t\t\to.signalMonitorQuit()\n\t\t\t\to.monitorWg.Wait()\n\t\t\t}\n\t\t}\n\t}\n\n\tmset.mu.Lock()\n\t// Send stream delete advisory after the consumers.\n\tif deleteFlag && advisory {\n\t\tmset.sendDeleteAdvisoryLocked()\n\t}\n\n\t// Quit channel, do this after sending the delete advisory\n\tif mset.qch != nil {\n\t\tclose(mset.qch)\n\t\tmset.qch = nil\n\t}\n\n\t// Cluster cleanup\n\tvar sa *streamAssignment\n\tif n := mset.node; n != nil {\n\t\tif deleteFlag {\n\t\t\tn.Delete()\n\t\t\tsa = mset.sa\n\t\t} else if !isShuttingDown {\n\t\t\t// Stop Raft, unless JetStream is already shutting down, in which case they'll be stopped separately.\n\t\t\tn.Stop()\n\t\t}\n\t}\n\n\t// Cleanup duplicate timer if running.\n\tmset.ddMu.Lock()\n\tif mset.ddtmr != nil {\n\t\tmset.ddtmr.Stop()\n\t\tmset.ddtmr = nil\n\t\tmset.ddmap = nil\n\t\tmset.ddarr = nil\n\t\tmset.ddindex = 0\n\t}\n\tmset.ddMu.Unlock()\n\n\tsysc := mset.sysc\n\tmset.sysc = nil\n\n\tif deleteFlag {\n\t\t// Unregistering ipQueues do not prevent them from push/pop\n\t\t// just will remove them from the central monitoring map\n\t\tmset.msgs.unregister()\n\t\tmset.ackq.unregister()\n\t\tmset.outq.unregister()\n\t\tmset.sigq.unregister()\n\t\tmset.smsgs.unregister()\n\t}\n\n\t// Snapshot store.\n\tstore := mset.store\n\tc := mset.client\n\n\t// Clustered cleanup.\n\tmset.mu.Unlock()\n\n\t// Check if the stream assignment has the group node specified.\n\t// We need this cleared for if the stream gets reassigned here.\n\tif sa != nil {\n\t\tjs.mu.Lock()\n\t\tif sa.Group != nil {\n\t\t\tsa.Group.node = nil\n\t\t}\n\t\tjs.mu.Unlock()\n\t}\n\n\tif c != nil {\n\t\tc.closeConnection(ClientClosed)\n\t}\n\n\tif sysc != nil {\n\t\tsysc.closeConnection(ClientClosed)\n\t}\n\n\tif deleteFlag {\n\t\t// cleanup directories after the stream\n\t\taccDir := filepath.Join(js.config.StoreDir, accName)\n\t\tif store != nil {\n\t\t\t// Ignore errors.\n\t\t\tstore.Delete(false)\n\t\t} else {\n\t\t\tstreamDir := filepath.Join(accDir, streamsDir)\n\t\t\tos.RemoveAll(filepath.Join(streamDir, name))\n\t\t}\n\t\t// Release any resources.\n\t\tjs.releaseStreamResources(&mset.cfg)\n\t\t// Do cleanup in separate go routine similar to how fs will use purge here..\n\t\tgo func() {\n\t\t\t// no op if not empty\n\t\t\tos.Remove(filepath.Join(accDir, streamsDir))\n\t\t\tos.Remove(accDir)\n\t\t}()\n\t} else if store != nil {\n\t\t// Ignore errors.\n\t\tstore.Stop()\n\t}\n\n\treturn nil\n}\n\nfunc (mset *stream) getMsg(seq uint64) (*StoredMsg, error) {\n\tvar smv StoreMsg\n\tsm, err := mset.store.LoadMsg(seq, &smv)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// This only used in tests directly so no need to pool etc.\n\treturn &StoredMsg{\n\t\tSubject: sm.subj,\n\t\tSequence: sm.seq,\n\t\tHeader: sm.hdr,\n\t\tData: sm.msg,\n\t\tTime: time.Unix(0, sm.ts).UTC(),\n\t}, nil\n}\n\n// getConsumers will return a copy of all the current consumers for this stream.\nfunc (mset *stream) getConsumers() []*consumer {\n\tmset.clsMu.RLock()\n\tdefer mset.clsMu.RUnlock()\n\treturn append([]*consumer(nil), mset.cList...)\n}\n\n// Lock should be held for this one.\nfunc (mset *stream) numPublicConsumers() int {\n\treturn len(mset.consumers) - mset.directs\n}\n\n// This returns all consumers that are not DIRECT.\nfunc (mset *stream) getPublicConsumers() []*consumer {\n\tmset.clsMu.RLock()\n\tdefer mset.clsMu.RUnlock()\n\n\tvar obs []*consumer\n\tfor _, o := range mset.cList {\n\t\tif !o.cfg.Direct {\n\t\t\tobs = append(obs, o)\n\t\t}\n\t}\n\treturn obs\n}\n\n// This returns all consumers that are DIRECT.\nfunc (mset *stream) getDirectConsumers() []*consumer {\n\tmset.clsMu.RLock()\n\tdefer mset.clsMu.RUnlock()\n\n\tvar obs []*consumer\n\tfor _, o := range mset.cList {\n\t\tif o.cfg.Direct {\n\t\t\tobs = append(obs, o)\n\t\t}\n\t}\n\treturn obs\n}\n\n// 2 minutes plus up to 30s jitter.\nconst (\n\tdefaultCheckInterestStateT = 2 * time.Minute\n\tdefaultCheckInterestStateJ = 30\n)\n\nvar (\n\tcheckInterestStateT = defaultCheckInterestStateT // Interval\n\tcheckInterestStateJ = defaultCheckInterestStateJ // Jitter (secs)\n)\n\n// Will check for interest retention and make sure messages\n// that have been acked are processed and removed.\n// This will check the ack floors of all consumers, and adjust our first sequence accordingly.\nfunc (mset *stream) checkInterestState() {\n\tif mset == nil || !mset.isInterestRetention() {\n\t\t// If we are limits based nothing to do.\n\t\treturn\n\t}\n\n\t// Ensure only one of these runs at the same time.\n\tif !mset.cisrun.CompareAndSwap(false, true) {\n\t\treturn\n\t}\n\tdefer mset.cisrun.Store(false)\n\n\tvar ss StreamState\n\tmset.store.FastState(&ss)\n\n\tasflr := uint64(math.MaxUint64)\n\tfor _, o := range mset.getConsumers() {\n\t\to.checkStateForInterestStream(&ss)\n\t\to.mu.RLock()\n\t\tchkflr := o.chkflr\n\t\to.mu.RUnlock()\n\t\tasflr = min(asflr, chkflr)\n\t}\n\n\tmset.cfgMu.RLock()\n\trp := mset.cfg.Retention\n\tmset.cfgMu.RUnlock()\n\t// Remove as many messages from the \"head\" of the stream if there's no interest anymore.\n\tif rp == InterestPolicy && asflr != math.MaxUint64 {\n\t\tmset.store.Compact(asflr)\n\t}\n}\n\nfunc (mset *stream) isInterestRetention() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.cfg.Retention != LimitsPolicy\n}\n\n// NumConsumers reports on number of active consumers for this stream.\nfunc (mset *stream) numConsumers() int {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn len(mset.consumers)\n}\n\n// Lock should be held.\nfunc (mset *stream) setConsumer(o *consumer) {\n\tmset.consumers[o.name] = o\n\tif len(o.subjf) > 0 {\n\t\tmset.numFilter++\n\t}\n\tif o.cfg.Direct {\n\t\tmset.directs++\n\t}\n\t// Now update consumers list as well\n\tmset.clsMu.Lock()\n\tmset.cList = append(mset.cList, o)\n\tif mset.csl == nil {\n\t\tmset.csl = gsl.NewSublist[*consumer]()\n\t}\n\tfor _, sub := range o.signalSubs() {\n\t\tmset.csl.Insert(sub, o)\n\t}\n\tmset.clsMu.Unlock()\n}\n\n// Lock should be held.\nfunc (mset *stream) removeConsumer(o *consumer) {\n\tif o.cfg.FilterSubject != _EMPTY_ && mset.numFilter > 0 {\n\t\tmset.numFilter--\n\t}\n\tif o.cfg.Direct && mset.directs > 0 {\n\t\tmset.directs--\n\t}\n\tif mset.consumers != nil {\n\t\tdelete(mset.consumers, o.name)\n\t\t// Now update consumers list as well\n\t\tmset.clsMu.Lock()\n\t\tfor i, ol := range mset.cList {\n\t\t\tif ol == o {\n\t\t\t\tmset.cList = append(mset.cList[:i], mset.cList[i+1:]...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// Always remove from the leader sublist.\n\t\tif mset.csl != nil {\n\t\t\tfor _, sub := range o.signalSubs() {\n\t\t\t\tmset.csl.Remove(sub, o)\n\t\t\t}\n\t\t}\n\t\tmset.clsMu.Unlock()\n\t}\n}\n\n// swapSigSubs will update signal Subs for a new subject filter.\n// consumer lock should not be held.\nfunc (mset *stream) swapSigSubs(o *consumer, newFilters []string) {\n\tmset.clsMu.Lock()\n\to.mu.Lock()\n\n\tif o.closed || o.mset == nil {\n\t\to.mu.Unlock()\n\t\tmset.clsMu.Unlock()\n\t\treturn\n\t}\n\n\tif o.sigSubs != nil {\n\t\tif mset.csl != nil {\n\t\t\tfor _, sub := range o.sigSubs {\n\t\t\t\tmset.csl.Remove(sub, o)\n\t\t\t}\n\t\t}\n\t\to.sigSubs = nil\n\t}\n\n\tif mset.csl == nil {\n\t\tmset.csl = gsl.NewSublist[*consumer]()\n\t}\n\t// If no filters are present, add fwcs to sublist for that consumer.\n\tif newFilters == nil {\n\t\tmset.csl.Insert(fwcs, o)\n\t\to.sigSubs = append(o.sigSubs, fwcs)\n\t} else {\n\t\t// If there are filters, add their subjects to sublist.\n\t\tfor _, filter := range newFilters {\n\t\t\tmset.csl.Insert(filter, o)\n\t\t\to.sigSubs = append(o.sigSubs, filter)\n\t\t}\n\t}\n\to.mu.Unlock()\n\tmset.clsMu.Unlock()\n\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\n\tif mset.numFilter > 0 && len(o.subjf) > 0 {\n\t\tmset.numFilter--\n\t}\n\tif len(newFilters) > 0 {\n\t\tmset.numFilter++\n\t}\n}\n\n// lookupConsumer will retrieve a consumer by name.\nfunc (mset *stream) lookupConsumer(name string) *consumer {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.consumers[name]\n}\n\nfunc (mset *stream) numDirectConsumers() (num int) {\n\tmset.clsMu.RLock()\n\tdefer mset.clsMu.RUnlock()\n\n\t// Consumers that are direct are not recorded at the store level.\n\tfor _, o := range mset.cList {\n\t\to.mu.RLock()\n\t\tif o.cfg.Direct {\n\t\t\tnum++\n\t\t}\n\t\to.mu.RUnlock()\n\t}\n\treturn num\n}\n\n// State will return the current state for this stream.\nfunc (mset *stream) state() StreamState {\n\treturn mset.stateWithDetail(false)\n}\n\nfunc (mset *stream) stateWithDetail(details bool) StreamState {\n\t// mset.store does not change once set, so ok to reference here directly.\n\t// We do this elsewhere as well.\n\tstore := mset.store\n\tif store == nil {\n\t\treturn StreamState{}\n\t}\n\n\t// Currently rely on store for details.\n\tif details {\n\t\treturn store.State()\n\t}\n\t// Here we do the fast version.\n\tvar state StreamState\n\tstore.FastState(&state)\n\treturn state\n}\n\nfunc (mset *stream) Store() StreamStore {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.store\n}\n\n// Determines if the new proposed partition is unique amongst all consumers.\n// Lock should be held.\nfunc (mset *stream) partitionUnique(name string, partitions []string) bool {\n\tfor _, partition := range partitions {\n\t\tfor n, o := range mset.consumers {\n\t\t\t// Skip the consumer being checked.\n\t\t\tif n == name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\to.mu.RLock()\n\t\t\tif o.subjf == nil {\n\t\t\t\to.mu.RUnlock()\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tfor _, filter := range o.subjf {\n\t\t\t\tif SubjectsCollide(partition, filter.subject) {\n\t\t\t\t\to.mu.RUnlock()\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\to.mu.RUnlock()\n\t\t}\n\t}\n\treturn true\n}\n\n// Lock should be held.\nfunc (mset *stream) potentialFilteredConsumers() bool {\n\tnumSubjects := len(mset.cfg.Subjects)\n\tif len(mset.consumers) == 0 || numSubjects == 0 {\n\t\treturn false\n\t}\n\tif numSubjects > 1 || subjectHasWildcard(mset.cfg.Subjects[0]) {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Check if there is no interest in this sequence number across our consumers.\n// The consumer passed is optional if we are processing the ack for that consumer.\n// Write lock should be held.\nfunc (mset *stream) noInterest(seq uint64, obs *consumer) bool {\n\treturn !mset.checkForInterest(seq, obs)\n}\n\n// Check if there is no interest in this sequence number and subject across our consumers.\n// The consumer passed is optional if we are processing the ack for that consumer.\n// Write lock should be held.\nfunc (mset *stream) noInterestWithSubject(seq uint64, subj string, obs *consumer) bool {\n\treturn !mset.checkForInterestWithSubject(seq, subj, obs)\n}\n\n// Write lock should be held here for the stream to avoid race conditions on state.\nfunc (mset *stream) checkForInterest(seq uint64, obs *consumer) bool {\n\tvar subj string\n\tif mset.potentialFilteredConsumers() {\n\t\tpmsg := getJSPubMsgFromPool()\n\t\tdefer pmsg.returnToPool()\n\t\tsm, err := mset.store.LoadMsg(seq, &pmsg.StoreMsg)\n\t\tif err != nil {\n\t\t\tif err == ErrStoreEOF {\n\t\t\t\t// Register this as a preAck.\n\t\t\t\tmset.registerPreAck(obs, seq)\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tmset.clearAllPreAcks(seq)\n\t\t\treturn false\n\t\t}\n\t\tsubj = sm.subj\n\t}\n\treturn mset.checkForInterestWithSubject(seq, subj, obs)\n}\n\n// Checks for interest given a sequence and subject.\nfunc (mset *stream) checkForInterestWithSubject(seq uint64, subj string, obs *consumer) bool {\n\tfor _, o := range mset.consumers {\n\t\t// If this is us or we have a registered preAck for this consumer continue inspecting.\n\t\tif o == obs || mset.hasPreAck(o, seq) {\n\t\t\tcontinue\n\t\t}\n\t\t// Check if we need an ack.\n\t\tif o.needAck(seq, subj) {\n\t\t\treturn true\n\t\t}\n\t}\n\tmset.clearAllPreAcks(seq)\n\treturn false\n}\n\n// Check if we have a pre-registered ack for this sequence.\n// Write lock should be held.\nfunc (mset *stream) hasPreAck(o *consumer, seq uint64) bool {\n\tif o == nil || len(mset.preAcks) == 0 {\n\t\treturn false\n\t}\n\tconsumers := mset.preAcks[seq]\n\tif len(consumers) == 0 {\n\t\treturn false\n\t}\n\t_, found := consumers[o]\n\treturn found\n}\n\n// Check if we have all consumers pre-acked for this sequence and subject.\n// Write lock should be held.\nfunc (mset *stream) hasAllPreAcks(seq uint64, subj string) bool {\n\tif len(mset.preAcks) == 0 || len(mset.preAcks[seq]) == 0 {\n\t\treturn false\n\t}\n\t// Since these can be filtered and mutually exclusive,\n\t// if we have some preAcks we need to check all interest here.\n\treturn mset.noInterestWithSubject(seq, subj, nil)\n}\n\n// Check if we have all consumers pre-acked.\n// Write lock should be held.\nfunc (mset *stream) clearAllPreAcks(seq uint64) {\n\tdelete(mset.preAcks, seq)\n}\n\n// Clear all preAcks below floor.\n// Write lock should be held.\nfunc (mset *stream) clearAllPreAcksBelowFloor(floor uint64) {\n\tfor seq := range mset.preAcks {\n\t\tif seq < floor {\n\t\t\tdelete(mset.preAcks, seq)\n\t\t}\n\t}\n}\n\n// This will register an ack for a consumer if it arrives before the actual message.\nfunc (mset *stream) registerPreAckLock(o *consumer, seq uint64) {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tmset.registerPreAck(o, seq)\n}\n\n// This will register an ack for a consumer if it arrives before\n// the actual message.\n// Write lock should be held.\nfunc (mset *stream) registerPreAck(o *consumer, seq uint64) {\n\tif o == nil {\n\t\treturn\n\t}\n\tif mset.preAcks == nil {\n\t\tmset.preAcks = make(map[uint64]map[*consumer]struct{})\n\t}\n\tif mset.preAcks[seq] == nil {\n\t\tmset.preAcks[seq] = make(map[*consumer]struct{})\n\t}\n\tmset.preAcks[seq][o] = struct{}{}\n}\n\n// This will clear an ack for a consumer.\n// Write lock should be held.\nfunc (mset *stream) clearPreAck(o *consumer, seq uint64) {\n\tif o == nil || len(mset.preAcks) == 0 {\n\t\treturn\n\t}\n\tif consumers := mset.preAcks[seq]; len(consumers) > 0 {\n\t\tdelete(consumers, o)\n\t\tif len(consumers) == 0 {\n\t\t\tdelete(mset.preAcks, seq)\n\t\t}\n\t}\n}\n\n// ackMsg is called into from a consumer when we have a WorkQueue or Interest Retention Policy.\n// Returns whether the message at seq was removed as a result of the ACK.\n// (Or should be removed in the case of clustered streams, since it requires a message delete proposal)\nfunc (mset *stream) ackMsg(o *consumer, seq uint64) bool {\n\tif seq == 0 {\n\t\treturn false\n\t}\n\n\t// Don't make this RLock(). We need to have only 1 running at a time to gauge interest across all consumers.\n\tmset.mu.Lock()\n\tif mset.closed.Load() || mset.cfg.Retention == LimitsPolicy {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\tstore := mset.store\n\tvar state StreamState\n\tstore.FastState(&state)\n\n\t// If this has arrived before we have processed the message itself.\n\tif seq > state.LastSeq {\n\t\tmset.registerPreAck(o, seq)\n\t\tmset.mu.Unlock()\n\t\t// We have not removed the message, but should still signal so we could retry later\n\t\t// since we potentially need to remove it then.\n\t\treturn true\n\t}\n\n\t// Always clear pre-ack if here.\n\tmset.clearPreAck(o, seq)\n\n\t// Make sure this sequence is not below our first sequence.\n\tif seq < state.FirstSeq {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\t// If there's no interest left on this message for all consumers, we can remove it.\n\tshouldRemove := mset.noInterest(seq, nil)\n\n\t// If nothing else to do.\n\tif !shouldRemove {\n\t\tmset.mu.Unlock()\n\t\treturn false\n\t}\n\n\tif !mset.isClustered() {\n\t\tmset.mu.Unlock()\n\t\t// If we are here we should attempt to remove.\n\t\tif _, err := store.RemoveMsg(seq); err == ErrStoreEOF {\n\t\t\t// This should not happen, but being pedantic.\n\t\t\tmset.registerPreAckLock(o, seq)\n\t\t}\n\t\treturn true\n\t}\n\n\t// Only propose message deletion to the stream if we're the leader, otherwise followers would also propose.\n\t// We must be the stream leader, since we are the only one that can guarantee message ordering and ack handling.\n\t// Either we've stored the message, and we know for sure all consumers have acked the message.\n\t// Or, we've not stored the message yet (rare), and all consumers have registered as pre-acks,\n\t// then we do the message delete proposal after we've stored the message instead.\n\t// Except for a Direct AckNone consumer, as that has a nil consumer here, we still forward the delete proposal.\n\tif o != nil && !mset.isLeader() {\n\t\t// Currently, interest-based streams can race on \"no interest\" because consumer creates/updates go over\n\t\t// the meta layer and published messages go over the stream layer. Some servers could then either store\n\t\t// or not store some initial set of messages that gained new interest. To get the stream back in sync,\n\t\t// we allow moving the first sequence up.\n\t\t// TODO(mvv): later on only the stream leader should determine \"no interest\"\n\t\tinterestRaiseFirst := mset.cfg.Retention == InterestPolicy && seq == state.FirstSeq\n\t\tmset.mu.Unlock()\n\t\tif interestRaiseFirst {\n\t\t\tif _, err := store.RemoveMsg(seq); err == ErrStoreEOF {\n\t\t\t\t// This should not happen, but being pedantic.\n\t\t\t\tmset.registerPreAckLock(o, seq)\n\t\t\t}\n\t\t}\n\t\t// Must still mark as removal if follower. If we become leader later, we must be able to retry the proposal.\n\t\treturn true\n\t}\n\n\tmd := streamMsgDelete{Seq: seq, NoErase: true, Stream: mset.cfg.Name}\n\t// Directly proposes if stream leader, otherwise forwards it.\n\tmset.node.ForwardProposal(encodeMsgDelete(&md))\n\tmset.mu.Unlock()\n\treturn true\n}\n\n// Snapshot creates a snapshot for the stream and possibly consumers.\nfunc (mset *stream) snapshot(deadline time.Duration, checkMsgs, includeConsumers bool) (*SnapshotResult, error) {\n\tif mset.closed.Load() {\n\t\treturn nil, errStreamClosed\n\t}\n\tstore := mset.store\n\treturn store.Snapshot(deadline, checkMsgs, includeConsumers)\n}\n\nconst snapsDir = \"__snapshots__\"\n\n// RestoreStream will restore a stream from a snapshot.\nfunc (a *Account) RestoreStream(ncfg *StreamConfig, r io.Reader) (*stream, error) {\n\tif ncfg == nil {\n\t\treturn nil, errors.New(\"nil config on stream restore\")\n\t}\n\n\ts, jsa, err := a.checkForJetStream()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tjs := jsa.js\n\tif js == nil {\n\t\treturn nil, NewJSNotEnabledForAccountError()\n\t}\n\n\tcfg, apiErr := s.checkStreamCfg(ncfg, a, false)\n\tif apiErr != nil {\n\t\treturn nil, apiErr\n\t}\n\n\tsd := filepath.Join(jsa.storeDir, snapsDir)\n\tif _, err := os.Stat(sd); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(sd, defaultDirPerms); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not create snapshots directory - %v\", err)\n\t\t}\n\t}\n\tsdir, err := os.MkdirTemp(sd, \"snap-\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif _, err := os.Stat(sdir); os.IsNotExist(err) {\n\t\tif err := os.MkdirAll(sdir, defaultDirPerms); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not create snapshots directory - %v\", err)\n\t\t}\n\t}\n\tdefer os.RemoveAll(sdir)\n\n\tlogAndReturnError := func() error {\n\t\ta.mu.RLock()\n\t\terr := fmt.Errorf(\"unexpected content (account=%s)\", a.Name)\n\t\tif a.srv != nil {\n\t\t\ta.srv.Errorf(\"Stream restore failed due to %v\", err)\n\t\t}\n\t\ta.mu.RUnlock()\n\t\treturn err\n\t}\n\tsdirCheck := filepath.Clean(sdir) + string(os.PathSeparator)\n\n\t_, isClustered := jsa.jetStreamAndClustered()\n\tjsa.usageMu.RLock()\n\tselected, tier, hasTier := jsa.selectLimits(cfg.Replicas)\n\tjsa.usageMu.RUnlock()\n\treserved := int64(0)\n\tif hasTier {\n\t\tif isClustered {\n\t\t\tjs.mu.RLock()\n\t\t\t_, reserved = js.tieredStreamAndReservationCount(a.Name, tier, &cfg)\n\t\t\tjs.mu.RUnlock()\n\t\t} else {\n\t\t\treserved = jsa.tieredReservation(tier, &cfg)\n\t\t}\n\t}\n\n\tvar bc int64\n\ttr := tar.NewReader(s2.NewReader(r))\n\tfor {\n\t\thdr, err := tr.Next()\n\t\tif err == io.EOF {\n\t\t\tbreak // End of snapshot\n\t\t}\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif hdr.Typeflag != tar.TypeReg {\n\t\t\treturn nil, logAndReturnError()\n\t\t}\n\t\tbc += hdr.Size\n\t\tjs.mu.RLock()\n\t\terr = js.checkAllLimits(&selected, &cfg, reserved, bc)\n\t\tjs.mu.RUnlock()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfpath := filepath.Join(sdir, filepath.Clean(hdr.Name))\n\t\tif !strings.HasPrefix(fpath, sdirCheck) {\n\t\t\treturn nil, logAndReturnError()\n\t\t}\n\t\tos.MkdirAll(filepath.Dir(fpath), defaultDirPerms)\n\t\tfd, err := os.OpenFile(fpath, os.O_CREATE|os.O_RDWR, 0600)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t_, err = io.Copy(fd, tr)\n\t\tfd.Close()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Check metadata.\n\t// The cfg passed in will be the new identity for the stream.\n\tvar fcfg FileStreamInfo\n\tb, err := os.ReadFile(filepath.Join(sdir, JetStreamMetaFile))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := json.Unmarshal(b, &fcfg); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Check to make sure names match.\n\tif fcfg.Name != cfg.Name {\n\t\treturn nil, errors.New(\"stream names do not match\")\n\t}\n\n\t// See if this stream already exists.\n\tif _, err := a.lookupStream(cfg.Name); err == nil {\n\t\treturn nil, NewJSStreamNameExistRestoreFailedError()\n\t}\n\t// Move into the correct place here.\n\tndir := filepath.Join(jsa.storeDir, streamsDir, cfg.Name)\n\t// Remove old one if for some reason it is still here.\n\tif _, err := os.Stat(ndir); err == nil {\n\t\tos.RemoveAll(ndir)\n\t}\n\t// Make sure our destination streams directory exists.\n\tif err := os.MkdirAll(filepath.Join(jsa.storeDir, streamsDir), defaultDirPerms); err != nil {\n\t\treturn nil, err\n\t}\n\t// Move into new location.\n\tif err := os.Rename(sdir, ndir); err != nil {\n\t\treturn nil, err\n\t}\n\n\tmset, err := a.addStream(&cfg)\n\tif err != nil {\n\t\t// Make sure to clean up after ourselves here.\n\t\tos.RemoveAll(ndir)\n\t\treturn nil, err\n\t}\n\tif !fcfg.Created.IsZero() {\n\t\tmset.setCreatedTime(fcfg.Created)\n\t}\n\n\t// Make sure we do an update if the configs have changed.\n\tif !reflect.DeepEqual(fcfg.StreamConfig, cfg) {\n\t\tif err := mset.update(&cfg); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Now do consumers.\n\todir := filepath.Join(ndir, consumerDir)\n\tofis, _ := os.ReadDir(odir)\n\tfor _, ofi := range ofis {\n\t\tmetafile := filepath.Join(odir, ofi.Name(), JetStreamMetaFile)\n\t\tmetasum := filepath.Join(odir, ofi.Name(), JetStreamMetaFileSum)\n\t\tif _, err := os.Stat(metafile); os.IsNotExist(err) {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t\tbuf, err := os.ReadFile(metafile)\n\t\tif err != nil {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t\tif _, err := os.Stat(metasum); os.IsNotExist(err) {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t\tvar cfg FileConsumerInfo\n\t\tif err := json.Unmarshal(buf, &cfg); err != nil {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t\tisEphemeral := !isDurableConsumer(&cfg.ConsumerConfig)\n\t\tif isEphemeral {\n\t\t\t// This is an ephermal consumer and this could fail on restart until\n\t\t\t// the consumer can reconnect. We will create it as a durable and switch it.\n\t\t\tcfg.ConsumerConfig.Durable = ofi.Name()\n\t\t}\n\t\tobs, err := mset.addConsumer(&cfg.ConsumerConfig)\n\t\tif err != nil {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t\tif isEphemeral {\n\t\t\tobs.switchToEphemeral()\n\t\t}\n\t\tif !cfg.Created.IsZero() {\n\t\t\tobs.setCreatedTime(cfg.Created)\n\t\t}\n\t\tobs.mu.Lock()\n\t\terr = obs.readStoredState()\n\t\tobs.mu.Unlock()\n\t\tif err != nil {\n\t\t\tmset.stop(true, false)\n\t\t\treturn nil, fmt.Errorf(\"error restoring consumer [%q]: %v\", ofi.Name(), err)\n\t\t}\n\t}\n\treturn mset, nil\n}\n\n// This is to check for dangling messages on interest retention streams. Only called on account enable.\n// Issue https://github.com/nats-io/nats-server/issues/3612\nfunc (mset *stream) checkForOrphanMsgs() {\n\tmset.mu.RLock()\n\tconsumers := make([]*consumer, 0, len(mset.consumers))\n\tfor _, o := range mset.consumers {\n\t\tconsumers = append(consumers, o)\n\t}\n\taccName, stream := mset.acc.Name, mset.cfg.Name\n\n\tvar ss StreamState\n\tmset.store.FastState(&ss)\n\tmset.mu.RUnlock()\n\n\tfor _, o := range consumers {\n\t\tif err := o.checkStateForInterestStream(&ss); err == errAckFloorHigherThanLastSeq {\n\t\t\to.mu.RLock()\n\t\t\ts, consumer := o.srv, o.name\n\t\t\tstate, _ := o.store.State()\n\t\t\tasflr := state.AckFloor.Stream\n\t\t\to.mu.RUnlock()\n\t\t\t// Warn about stream state vs our ack floor.\n\t\t\ts.RateLimitWarnf(\"Detected consumer '%s > %s > %s' ack floor %d is ahead of stream's last sequence %d\",\n\t\t\t\taccName, stream, consumer, asflr, ss.LastSeq)\n\t\t}\n\t}\n}\n\n// Check on startup to make sure that consumers replication matches us.\n// Interest retention requires replication matches.\nfunc (mset *stream) checkConsumerReplication() {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\n\tif mset.cfg.Retention != InterestPolicy {\n\t\treturn\n\t}\n\n\ts, acc := mset.srv, mset.acc\n\tfor _, o := range mset.consumers {\n\t\to.mu.RLock()\n\t\t// Consumer replicas 0 can be a legit config for the replicas and we will inherit from the stream\n\t\t// when this is the case.\n\t\tif mset.cfg.Replicas != o.cfg.Replicas && o.cfg.Replicas != 0 {\n\t\t\ts.Errorf(\"consumer '%s > %s > %s' MUST match replication (%d vs %d) of stream with interest policy\",\n\t\t\t\tacc, mset.cfg.Name, o.cfg.Name, mset.cfg.Replicas, o.cfg.Replicas)\n\t\t}\n\t\to.mu.RUnlock()\n\t}\n}\n\n// Will check if we are running in the monitor already and if not set the appropriate flag.\nfunc (mset *stream) checkInMonitor() bool {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\n\tif mset.inMonitor {\n\t\treturn true\n\t}\n\tmset.inMonitor = true\n\treturn false\n}\n\n// Clear us being in the monitor routine.\nfunc (mset *stream) clearMonitorRunning() {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tmset.inMonitor = false\n\n\t// We've stopped running the monitor goroutine, now also do additional cleanup.\n\tmset.deleteBatchApplyState()\n}\n\n// Check if our monitor is running.\nfunc (mset *stream) isMonitorRunning() bool {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.inMonitor\n}\n\n// setWriteErr stores the write error in the stream.\nfunc (mset *stream) setWriteErr(err error) {\n\tmset.mu.Lock()\n\tdefer mset.mu.Unlock()\n\tmset.setWriteErrLocked(err)\n}\n\nfunc (mset *stream) setWriteErrLocked(err error) {\n\tif mset.werr != nil {\n\t\treturn\n\t}\n\t// Ignore non-write errors.\n\tif err == ErrStoreClosed {\n\t\treturn\n\t}\n\tmset.srv.Errorf(\"JetStream stream '%s > %s' critical write error: %v\", mset.acc.Name, mset.cfg.Name, err)\n\tmset.werr = err\n\tassert.Unreachable(\"Stream encountered write error\", map[string]any{\n\t\t\"account\": mset.acc.Name,\n\t\t\"stream\": mset.cfg.Name,\n\t\t\"err\": err,\n\t})\n\n\t// If stream is replicated, put it in observer mode to make sure another server can pick it up.\n\tif node := mset.node; node != nil {\n\t\tnode.StepDown()\n\t\tnode.SetObserver(true)\n\t}\n}\n\n// getWriteErr returns the write error stored in the stream (if any).\nfunc (mset *stream) getWriteErr() error {\n\tmset.mu.RLock()\n\tdefer mset.mu.RUnlock()\n\treturn mset.werr\n}\n\n// Adjust accounting for sent messages as part of replication.\nfunc (mset *stream) trackReplicationTraffic(node RaftNode, sz int, r int) {\n\t// If we are using the system account for NRG, add in the extra sent msgs and bytes to our account\n\t// so that the end user / account owner has visibility.\n\tif node.IsSystemAccount() && mset.acc != nil && r > 1 {\n\t\toutMsgs := int64(r - 1)\n\t\toutBytes := int64(sz * (r - 1))\n\n\t\tmset.acc.stats.Lock()\n\t\tmset.acc.stats.outMsgs += outMsgs\n\t\tmset.acc.stats.outBytes += outBytes\n\t\tmset.acc.stats.rt.outMsgs += outMsgs\n\t\tmset.acc.stats.rt.outBytes += outBytes\n\t\tmset.acc.stats.Unlock()\n\t}\n}\n" }, { "path": "server/stree/dump.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n)\n\n// For dumping out a text representation of a tree.\nfunc (t *SubjectTree[T]) Dump(w io.Writer) {\n\tt.dump(w, t.root, 0)\n\tfmt.Fprintln(w)\n}\n\n// Will dump out a node.\nfunc (t *SubjectTree[T]) dump(w io.Writer, n node, depth int) {\n\tif n == nil {\n\t\tfmt.Fprintf(w, \"EMPTY\\n\")\n\t\treturn\n\t}\n\tif n.isLeaf() {\n\t\tleaf := n.(*leaf[T])\n\t\tfmt.Fprintf(w, \"%s LEAF: Suffix: %q Value: %+v\\n\", dumpPre(depth), leaf.suffix, leaf.value)\n\t\tn = nil\n\t} else {\n\t\t// We are a node type here, grab meta portion.\n\t\tbn := n.base()\n\t\tfmt.Fprintf(w, \"%s %s Prefix: %q\\n\", dumpPre(depth), n.kind(), bn.prefix)\n\t\tdepth++\n\t\tn.iter(func(n node) bool {\n\t\t\tt.dump(w, n, depth)\n\t\t\treturn true\n\t\t})\n\t}\n}\n\n// For individual node/leaf dumps.\nfunc (n *leaf[T]) kind() string { return \"LEAF\" }\nfunc (n *node4) kind() string { return \"NODE4\" }\nfunc (n *node10) kind() string { return \"NODE10\" }\nfunc (n *node16) kind() string { return \"NODE16\" }\nfunc (n *node48) kind() string { return \"NODE48\" }\nfunc (n *node256) kind() string { return \"NODE256\" }\n\n// Calculates the indendation, etc.\nfunc dumpPre(depth int) string {\n\tif depth == 0 {\n\t\treturn \"-- \"\n\t} else {\n\t\tvar b strings.Builder\n\t\tfor i := 0; i < depth; i++ {\n\t\t\tb.WriteString(\" \")\n\t\t}\n\t\tb.WriteString(\"|__ \")\n\t\treturn b.String()\n\t}\n}\n" }, { "path": "server/stree/helper_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/antithesis\"\n)\n\nfunc require_True(t testing.TB, b bool) {\n\tt.Helper()\n\tif !b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_True check\", nil)\n\t\tt.Fatalf(\"require true, but got false\")\n\t}\n}\n\nfunc require_False(t testing.TB, b bool) {\n\tt.Helper()\n\tif b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_False check\", nil)\n\t\tt.Fatalf(\"require false, but got true\")\n\t}\n}\n\nfunc require_NoError(t testing.TB, err error) {\n\tt.Helper()\n\tif err != nil {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_NoError check\", map[string]any{\n\t\t\t\"error\": err.Error(),\n\t\t})\n\t\tt.Fatalf(\"require no error, but got: %v\", err)\n\t}\n}\n\nfunc require_Equal[T comparable](t testing.TB, a, b T) {\n\tt.Helper()\n\tif a != b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Equal check\", nil)\n\t\tt.Fatalf(\"require %T equal, but got: %v != %v\", a, a, b)\n\t}\n}\n\nfunc require_Len(t testing.TB, a, b int) {\n\tt.Helper()\n\tif a != b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Len check\", nil)\n\t\tt.Fatalf(\"require len, but got: %v != %v\", a, b)\n\t}\n}\n" }, { "path": "server/stree/leaf.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\t\"bytes\"\n)\n\n// Leaf node\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype leaf[T any] struct {\n\tvalue T\n\t// This could be the whole subject, but most likely just the suffix portion.\n\t// We will only store the suffix here and assume all prior prefix paths have\n\t// been checked once we arrive at this leafnode.\n\tsuffix []byte\n}\n\nfunc newLeaf[T any](suffix []byte, value T) *leaf[T] {\n\treturn &leaf[T]{value, copyBytes(suffix)}\n}\n\nfunc (n *leaf[T]) isLeaf() bool { return true }\nfunc (n *leaf[T]) base() *meta { return nil }\nfunc (n *leaf[T]) match(subject []byte) bool { return bytes.Equal(subject, n.suffix) }\nfunc (n *leaf[T]) setSuffix(suffix []byte) { n.suffix = copyBytes(suffix) }\nfunc (n *leaf[T]) isFull() bool { return true }\nfunc (n *leaf[T]) matchParts(parts [][]byte) ([][]byte, bool) { return matchParts(parts, n.suffix) }\nfunc (n *leaf[T]) iter(f func(node) bool) {}\nfunc (n *leaf[T]) children() []node { return nil }\nfunc (n *leaf[T]) numChildren() uint16 { return 0 }\nfunc (n *leaf[T]) path() []byte { return n.suffix }\n\n// Not applicable to leafs and should not be called, so panic if we do.\nfunc (n *leaf[T]) setPrefix(pre []byte) { panic(\"setPrefix called on leaf\") }\nfunc (n *leaf[T]) addChild(_ byte, _ node) { panic(\"addChild called on leaf\") }\nfunc (n *leaf[T]) findChild(_ byte) *node { panic(\"findChild called on leaf\") }\nfunc (n *leaf[T]) grow() node { panic(\"grow called on leaf\") }\nfunc (n *leaf[T]) deleteChild(_ byte) { panic(\"deleteChild called on leaf\") }\nfunc (n *leaf[T]) shrink() node { panic(\"shrink called on leaf\") }\n" }, { "path": "server/stree/node.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Internal node interface.\ntype node interface {\n\tisLeaf() bool\n\tbase() *meta\n\tsetPrefix(pre []byte)\n\taddChild(c byte, n node)\n\tfindChild(c byte) *node\n\tdeleteChild(c byte)\n\tisFull() bool\n\tgrow() node\n\tshrink() node\n\tmatchParts(parts [][]byte) ([][]byte, bool)\n\tkind() string\n\titer(f func(node) bool)\n\tchildren() []node\n\tnumChildren() uint16\n\tpath() []byte\n}\n\ntype meta struct {\n\tprefix []byte\n\tsize uint16\n}\n\nfunc (n *meta) isLeaf() bool { return false }\nfunc (n *meta) base() *meta { return n }\n\nfunc (n *meta) setPrefix(pre []byte) {\n\tn.prefix = append([]byte(nil), pre...)\n}\n\nfunc (n *meta) numChildren() uint16 { return n.size }\nfunc (n *meta) path() []byte { return n.prefix }\n\n// Will match parts against our prefix.\nfunc (n *meta) matchParts(parts [][]byte) ([][]byte, bool) {\n\treturn matchParts(parts, n.prefix)\n}\n" }, { "path": "server/stree/node10.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Node with 10 children\n// This node size is for the particular case that a part of the subject is numeric\n// in nature, i.e. it only needs to satisfy the range 0-9 without wasting bytes\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype node10 struct {\n\tchild [10]node\n\tmeta\n\tkey [10]byte\n}\n\nfunc newNode10(prefix []byte) *node10 {\n\tnn := &node10{}\n\tnn.setPrefix(prefix)\n\treturn nn\n}\n\n// Currently we do not keep node10 sorted or use bitfields for traversal so just add to the end.\n// TODO(dlc) - We should revisit here with more detailed benchmarks.\nfunc (n *node10) addChild(c byte, nn node) {\n\tif n.size >= 10 {\n\t\tpanic(\"node10 full!\")\n\t}\n\tn.key[n.size] = c\n\tn.child[n.size] = nn\n\tn.size++\n}\n\nfunc (n *node10) findChild(c byte) *node {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\treturn &n.child[i]\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (n *node10) isFull() bool { return n.size >= 10 }\n\nfunc (n *node10) grow() node {\n\tnn := newNode16(n.prefix)\n\tfor i := 0; i < 10; i++ {\n\t\tnn.addChild(n.key[i], n.child[i])\n\t}\n\treturn nn\n}\n\n// Deletes a child from the node.\nfunc (n *node10) deleteChild(c byte) {\n\tfor i, last := uint16(0), n.size-1; i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\t// Unsorted so just swap in last one here, else nil if last.\n\t\t\tif i < last {\n\t\t\t\tn.key[i] = n.key[last]\n\t\t\t\tn.child[i] = n.child[last]\n\t\t\t\tn.key[last] = 0\n\t\t\t\tn.child[last] = nil\n\t\t\t} else {\n\t\t\t\tn.key[i] = 0\n\t\t\t\tn.child[i] = nil\n\t\t\t}\n\t\t\tn.size--\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Shrink if needed and return new node, otherwise return nil.\nfunc (n *node10) shrink() node {\n\tif n.size > 4 {\n\t\treturn nil\n\t}\n\tnn := newNode4(nil)\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tnn.addChild(n.key[i], n.child[i])\n\t}\n\treturn nn\n}\n\n// Iterate over all children calling func f.\nfunc (n *node10) iter(f func(node) bool) {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif !f(n.child[i]) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Return our children as a slice.\nfunc (n *node10) children() []node {\n\treturn n.child[:n.size]\n}\n" }, { "path": "server/stree/node16.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Node with 16 children\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype node16 struct {\n\tchild [16]node\n\tmeta\n\tkey [16]byte\n}\n\nfunc newNode16(prefix []byte) *node16 {\n\tnn := &node16{}\n\tnn.setPrefix(prefix)\n\treturn nn\n}\n\n// Currently we do not keep node16 sorted or use bitfields for traversal so just add to the end.\n// TODO(dlc) - We should revisit here with more detailed benchmarks.\nfunc (n *node16) addChild(c byte, nn node) {\n\tif n.size >= 16 {\n\t\tpanic(\"node16 full!\")\n\t}\n\tn.key[n.size] = c\n\tn.child[n.size] = nn\n\tn.size++\n}\n\nfunc (n *node16) findChild(c byte) *node {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\treturn &n.child[i]\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (n *node16) isFull() bool { return n.size >= 16 }\n\nfunc (n *node16) grow() node {\n\tnn := newNode48(n.prefix)\n\tfor i := 0; i < 16; i++ {\n\t\tnn.addChild(n.key[i], n.child[i])\n\t}\n\treturn nn\n}\n\n// Deletes a child from the node.\nfunc (n *node16) deleteChild(c byte) {\n\tfor i, last := uint16(0), n.size-1; i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\t// Unsorted so just swap in last one here, else nil if last.\n\t\t\tif i < last {\n\t\t\t\tn.key[i] = n.key[last]\n\t\t\t\tn.child[i] = n.child[last]\n\t\t\t\tn.key[last] = 0\n\t\t\t\tn.child[last] = nil\n\t\t\t} else {\n\t\t\t\tn.key[i] = 0\n\t\t\t\tn.child[i] = nil\n\t\t\t}\n\t\t\tn.size--\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Shrink if needed and return new node, otherwise return nil.\nfunc (n *node16) shrink() node {\n\tif n.size > 10 {\n\t\treturn nil\n\t}\n\tnn := newNode10(nil)\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tnn.addChild(n.key[i], n.child[i])\n\t}\n\treturn nn\n}\n\n// Iterate over all children calling func f.\nfunc (n *node16) iter(f func(node) bool) {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif !f(n.child[i]) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Return our children as a slice.\nfunc (n *node16) children() []node {\n\treturn n.child[:n.size]\n}\n" }, { "path": "server/stree/node256.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Node with 256 children\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype node256 struct {\n\tchild [256]node\n\tmeta\n}\n\nfunc newNode256(prefix []byte) *node256 {\n\tnn := &node256{}\n\tnn.setPrefix(prefix)\n\treturn nn\n}\n\nfunc (n *node256) addChild(c byte, nn node) {\n\tn.child[c] = nn\n\tn.size++\n}\n\nfunc (n *node256) findChild(c byte) *node {\n\tif n.child[c] != nil {\n\t\treturn &n.child[c]\n\t}\n\treturn nil\n}\n\nfunc (n *node256) isFull() bool { return false }\nfunc (n *node256) grow() node { panic(\"grow can not be called on node256\") }\n\n// Deletes a child from the node.\nfunc (n *node256) deleteChild(c byte) {\n\tif n.child[c] != nil {\n\t\tn.child[c] = nil\n\t\tn.size--\n\t}\n}\n\n// Shrink if needed and return new node, otherwise return nil.\nfunc (n *node256) shrink() node {\n\tif n.size > 48 {\n\t\treturn nil\n\t}\n\tnn := newNode48(nil)\n\tfor c, child := range n.child {\n\t\tif child != nil {\n\t\t\tnn.addChild(byte(c), n.child[c])\n\t\t}\n\t}\n\treturn nn\n}\n\n// Iterate over all children calling func f.\nfunc (n *node256) iter(f func(node) bool) {\n\tfor i := 0; i < 256; i++ {\n\t\tif n.child[i] != nil {\n\t\t\tif !f(n.child[i]) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Return our children as a slice.\nfunc (n *node256) children() []node {\n\treturn n.child[:256]\n}\n" }, { "path": "server/stree/node4.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Node with 4 children\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype node4 struct {\n\tchild [4]node\n\tmeta\n\tkey [4]byte\n}\n\nfunc newNode4(prefix []byte) *node4 {\n\tnn := &node4{}\n\tnn.setPrefix(prefix)\n\treturn nn\n}\n\n// Currently we do not need to keep sorted for traversal so just add to the end.\nfunc (n *node4) addChild(c byte, nn node) {\n\tif n.size >= 4 {\n\t\tpanic(\"node4 full!\")\n\t}\n\tn.key[n.size] = c\n\tn.child[n.size] = nn\n\tn.size++\n}\n\nfunc (n *node4) findChild(c byte) *node {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\treturn &n.child[i]\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (n *node4) isFull() bool { return n.size >= 4 }\n\nfunc (n *node4) grow() node {\n\tnn := newNode10(n.prefix)\n\tfor i := 0; i < 4; i++ {\n\t\tnn.addChild(n.key[i], n.child[i])\n\t}\n\treturn nn\n}\n\n// Deletes a child from the node.\nfunc (n *node4) deleteChild(c byte) {\n\tfor i, last := uint16(0), n.size-1; i < n.size; i++ {\n\t\tif n.key[i] == c {\n\t\t\t// Unsorted so just swap in last one here, else nil if last.\n\t\t\tif i < last {\n\t\t\t\tn.key[i] = n.key[last]\n\t\t\t\tn.child[i] = n.child[last]\n\t\t\t\tn.key[last] = 0\n\t\t\t\tn.child[last] = nil\n\t\t\t} else {\n\t\t\t\tn.key[i] = 0\n\t\t\t\tn.child[i] = nil\n\t\t\t}\n\t\t\tn.size--\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Shrink if needed and return new node, otherwise return nil.\nfunc (n *node4) shrink() node {\n\tif n.size == 1 {\n\t\treturn n.child[0]\n\t}\n\treturn nil\n}\n\n// Iterate over all children calling func f.\nfunc (n *node4) iter(f func(node) bool) {\n\tfor i := uint16(0); i < n.size; i++ {\n\t\tif !f(n.child[i]) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Return our children as a slice.\nfunc (n *node4) children() []node {\n\treturn n.child[:n.size]\n}\n" }, { "path": "server/stree/node48.go", "content": "// Copyright 2023-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// Node with 48 children\n// Memory saving vs node256 comes from the fact that the child array is 16 bytes\n// per `node` entry, so node256's 256*16=4096 vs node48's 256+(48*16)=1024\n// Note that `key` is effectively 1-indexed, as 0 means no entry, so offset by 1\n// Order of struct fields for best memory alignment (as per govet/fieldalignment)\ntype node48 struct {\n\tchild [48]node\n\tmeta\n\tkey [256]byte\n}\n\nfunc newNode48(prefix []byte) *node48 {\n\tnn := &node48{}\n\tnn.setPrefix(prefix)\n\treturn nn\n}\n\nfunc (n *node48) addChild(c byte, nn node) {\n\tif n.size >= 48 {\n\t\tpanic(\"node48 full!\")\n\t}\n\tn.child[n.size] = nn\n\tn.key[c] = byte(n.size + 1) // 1-indexed\n\tn.size++\n}\n\nfunc (n *node48) findChild(c byte) *node {\n\ti := n.key[c]\n\tif i == 0 {\n\t\treturn nil\n\t}\n\treturn &n.child[i-1]\n}\n\nfunc (n *node48) isFull() bool { return n.size >= 48 }\n\nfunc (n *node48) grow() node {\n\tnn := newNode256(n.prefix)\n\tfor c := 0; c < len(n.key); c++ {\n\t\tif i := n.key[byte(c)]; i > 0 {\n\t\t\tnn.addChild(byte(c), n.child[i-1])\n\t\t}\n\t}\n\treturn nn\n}\n\n// Deletes a child from the node.\nfunc (n *node48) deleteChild(c byte) {\n\ti := n.key[c]\n\tif i == 0 {\n\t\treturn\n\t}\n\ti-- // Adjust for 1-indexing\n\tlast := byte(n.size - 1)\n\tif i < last {\n\t\tn.child[i] = n.child[last]\n\t\tfor ic := 0; ic < len(n.key); ic++ {\n\t\t\tif n.key[byte(ic)] == last+1 {\n\t\t\t\tn.key[byte(ic)] = i + 1\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tn.child[last] = nil\n\tn.key[c] = 0\n\tn.size--\n}\n\n// Shrink if needed and return new node, otherwise return nil.\nfunc (n *node48) shrink() node {\n\tif n.size > 16 {\n\t\treturn nil\n\t}\n\tnn := newNode16(nil)\n\tfor c := 0; c < len(n.key); c++ {\n\t\tif i := n.key[byte(c)]; i > 0 {\n\t\t\tnn.addChild(byte(c), n.child[i-1])\n\t\t}\n\t}\n\treturn nn\n}\n\n// Iterate over all children calling func f.\nfunc (n *node48) iter(f func(node) bool) {\n\tfor _, c := range n.child {\n\t\tif c != nil && !f(c) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Return our children as a slice.\nfunc (n *node48) children() []node {\n\treturn n.child[:n.size]\n}\n" }, { "path": "server/stree/parts.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\t\"bytes\"\n)\n\n// genParts will break a filter subject up into parts.\n// We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'.\n// We do not care about other tokens per se, just parts that are separated by wildcards with an optional end fwc.\nfunc genParts(filter []byte, parts [][]byte) [][]byte {\n\tvar start int\n\tfor i, e := 0, len(filter)-1; i < len(filter); i++ {\n\t\tif filter[i] == tsep {\n\t\t\t// See if next token is pwc. Either internal or end pwc.\n\t\t\tif i < e && filter[i+1] == pwc && (i+2 <= e && filter[i+2] == tsep || i+1 == e) {\n\t\t\t\tif i > start {\n\t\t\t\t\tparts = append(parts, filter[start:i+1])\n\t\t\t\t}\n\t\t\t\tparts = append(parts, filter[i+1:i+2])\n\t\t\t\ti++ // Skip pwc\n\t\t\t\tif i+2 <= e {\n\t\t\t\t\ti++ // Skip next tsep from next part too.\n\t\t\t\t}\n\t\t\t\tstart = i + 1\n\t\t\t} else if i < e && filter[i+1] == fwc && i+1 == e {\n\t\t\t\tif i > start {\n\t\t\t\t\tparts = append(parts, filter[start:i+1])\n\t\t\t\t}\n\t\t\t\tparts = append(parts, filter[i+1:i+2])\n\t\t\t\ti++ // Skip fwc\n\t\t\t\tstart = i + 1\n\t\t\t}\n\t\t} else if filter[i] == pwc || filter[i] == fwc {\n\t\t\t// Wildcard must be at the start or preceded by tsep.\n\t\t\tif prev := i - 1; prev >= 0 && filter[prev] != tsep {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Wildcard must be at the end or followed by tsep.\n\t\t\tif next := i + 1; next == e || next < e && filter[next] != tsep {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Full wildcard must be terminal.\n\t\t\tif filter[i] == fwc && i < e {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// We start with a pwc or fwc.\n\t\t\tparts = append(parts, filter[i:i+1])\n\t\t\tif i+1 <= e {\n\t\t\t\ti++ // Skip next tsep from next part too.\n\t\t\t}\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\tif start < len(filter) {\n\t\t// Check to see if we need to eat a leading tsep.\n\t\tif filter[start] == tsep {\n\t\t\tstart++\n\t\t}\n\t\tparts = append(parts, filter[start:])\n\t}\n\treturn parts\n}\n\n// Match our parts against a fragment, which could be prefix for nodes or a suffix for leafs.\nfunc matchParts(parts [][]byte, frag []byte) ([][]byte, bool) {\n\tlf := len(frag)\n\tif lf == 0 {\n\t\treturn parts, true\n\t}\n\n\tvar si int\n\tlpi := len(parts) - 1\n\n\tfor i, part := range parts {\n\t\tif si >= lf {\n\t\t\treturn parts[i:], true\n\t\t}\n\t\tlp := len(part)\n\t\t// Check for pwc or fwc place holders.\n\t\tif lp == 1 {\n\t\t\tif part[0] == pwc {\n\t\t\t\tindex := bytes.IndexByte(frag[si:], tsep)\n\t\t\t\t// We are trying to match pwc and did not find our tsep.\n\t\t\t\t// Will need to move to next node from caller.\n\t\t\t\tif index < 0 {\n\t\t\t\t\tif i == lpi {\n\t\t\t\t\t\treturn nil, true\n\t\t\t\t\t}\n\t\t\t\t\treturn parts[i:], true\n\t\t\t\t}\n\t\t\t\tsi += index + 1\n\t\t\t\tcontinue\n\t\t\t} else if part[0] == fwc {\n\t\t\t\t// If we are here we should be good.\n\t\t\t\treturn nil, true\n\t\t\t}\n\t\t}\n\t\tend := min(si+lp, lf)\n\t\t// If part is bigger then the remaining fragment, adjust to a portion on the part.\n\t\tif si+lp > end {\n\t\t\t// Frag is smaller then part itself.\n\t\t\tpart = part[:end-si]\n\t\t}\n\t\tif !bytes.Equal(part, frag[si:end]) {\n\t\t\treturn parts, false\n\t\t}\n\t\t// If we still have a portion of the fragment left, update and continue.\n\t\tif end < lf {\n\t\t\tsi = end\n\t\t\tcontinue\n\t\t}\n\t\t// If we matched a partial, do not move past current part\n\t\t// but update the part to what was consumed. This allows upper layers to continue.\n\t\tif end < si+lp {\n\t\t\tif end >= lf {\n\t\t\t\t// Create a copy before modifying. Reuse slice capacity available at the\n\t\t\t\t// end of the parts slice, since this saves us additional allocations.\n\t\t\t\tlp := len(parts)\n\t\t\t\tparts = append(parts[lp:], parts[:lp]...)\n\t\t\t\tparts[i] = parts[i][lf-si:]\n\t\t\t} else {\n\t\t\t\ti++\n\t\t\t}\n\t\t\treturn parts[i:], true\n\t\t}\n\t\tif i == lpi {\n\t\t\treturn nil, true\n\t\t}\n\t\t// If we are here we are not the last part which means we have a wildcard\n\t\t// gap, so we need to match anything up to next tsep.\n\t\tsi += len(part)\n\t}\n\treturn parts, false\n}\n" }, { "path": "server/stree/stree.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\t\"bytes\"\n\t\"slices\"\n\t\"unsafe\"\n\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n)\n\n// SubjectTree is an adaptive radix trie (ART) for storing subject information on literal subjects.\n// Will use dynamic nodes, path compression and lazy expansion.\n// The reason this exists is to not only save some memory in our filestore but to greatly optimize matching\n// a wildcard subject to certain members, e.g. consumer NumPending calculations.\ntype SubjectTree[T any] struct {\n\troot node\n\tsize int\n}\n\n// NewSubjectTree creates a new SubjectTree with values T.\nfunc NewSubjectTree[T any]() *SubjectTree[T] {\n\treturn &SubjectTree[T]{}\n}\n\n// Size returns the number of elements stored.\nfunc (t *SubjectTree[T]) Size() int {\n\tif t == nil {\n\t\treturn 0\n\t}\n\treturn t.size\n}\n\n// Will empty out the tree, or if tree is nil create a new one.\nfunc (t *SubjectTree[T]) Empty() *SubjectTree[T] {\n\tif t == nil {\n\t\treturn NewSubjectTree[T]()\n\t}\n\tt.root, t.size = nil, 0\n\treturn t\n}\n\n// Insert a value into the tree. Will return if the value was updated and if so the old value.\nfunc (t *SubjectTree[T]) Insert(subject []byte, value T) (*T, bool) {\n\tif t == nil {\n\t\treturn nil, false\n\t}\n\n\t// Make sure we never insert anything with a noPivot byte.\n\tif bytes.IndexByte(subject, noPivot) >= 0 {\n\t\treturn nil, false\n\t}\n\n\told, updated := t.insert(&t.root, subject, value, 0)\n\tif !updated {\n\t\tt.size++\n\t}\n\treturn old, updated\n}\n\n// Find will find the value and return it or false if it was not found.\nfunc (t *SubjectTree[T]) Find(subject []byte) (*T, bool) {\n\tif t == nil {\n\t\treturn nil, false\n\t}\n\n\tvar si int\n\tfor n := t.root; n != nil; {\n\t\tif n.isLeaf() {\n\t\t\tif ln := n.(*leaf[T]); ln.match(subject[si:]) {\n\t\t\t\treturn &ln.value, true\n\t\t\t}\n\t\t\treturn nil, false\n\t\t}\n\t\t// We are a node type here, grab meta portion.\n\t\tif bn := n.base(); len(bn.prefix) > 0 {\n\t\t\tend := min(si+len(bn.prefix), len(subject))\n\t\t\tif !bytes.Equal(subject[si:end], bn.prefix) {\n\t\t\t\treturn nil, false\n\t\t\t}\n\t\t\t// Increment our subject index.\n\t\t\tsi += len(bn.prefix)\n\t\t}\n\t\tif an := n.findChild(pivot(subject, si)); an != nil {\n\t\t\tn = *an\n\t\t} else {\n\t\t\treturn nil, false\n\t\t}\n\t}\n\treturn nil, false\n}\n\n// Delete will delete the item and return its value, or not found if it did not exist.\nfunc (t *SubjectTree[T]) Delete(subject []byte) (*T, bool) {\n\tif t == nil {\n\t\treturn nil, false\n\t}\n\n\tval, deleted := t.delete(&t.root, subject, 0)\n\tif deleted {\n\t\tt.size--\n\t}\n\treturn val, deleted\n}\n\n// Match will match against a subject that can have wildcards and invoke the callback func for each matched value.\nfunc (t *SubjectTree[T]) Match(filter []byte, cb func(subject []byte, val *T)) {\n\tif t == nil || t.root == nil || len(filter) == 0 || cb == nil {\n\t\treturn\n\t}\n\t// We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'.\n\tvar raw [16][]byte\n\tparts := genParts(filter, raw[:0])\n\tvar _pre [256]byte\n\tt.match(t.root, parts, _pre[:0], func(subject []byte, val *T) bool {\n\t\tcb(subject, val)\n\t\treturn true\n\t})\n}\n\n// MatchUntil will match against a subject that can have wildcards and invoke\n// the callback func for each matched value.\n// Returning false from the callback will stop matching immediately.\n// Returns true if matching ran to completion, false if callback stopped it early.\nfunc (t *SubjectTree[T]) MatchUntil(filter []byte, cb func(subject []byte, val *T) bool) bool {\n\tif t == nil || t.root == nil || len(filter) == 0 || cb == nil {\n\t\treturn true\n\t}\n\t// We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'.\n\tvar raw [16][]byte\n\tparts := genParts(filter, raw[:0])\n\tvar _pre [256]byte\n\treturn t.match(t.root, parts, _pre[:0], cb)\n}\n\n// IterOrdered will walk all entries in the SubjectTree lexicographically. The callback can return false to terminate the walk.\nfunc (t *SubjectTree[T]) IterOrdered(cb func(subject []byte, val *T) bool) {\n\tif t == nil || t.root == nil {\n\t\treturn\n\t}\n\tvar _pre [256]byte\n\tt.iter(t.root, _pre[:0], true, cb)\n}\n\n// IterFast will walk all entries in the SubjectTree with no guarantees of ordering. The callback can return false to terminate the walk.\nfunc (t *SubjectTree[T]) IterFast(cb func(subject []byte, val *T) bool) {\n\tif t == nil || t.root == nil {\n\t\treturn\n\t}\n\tvar _pre [256]byte\n\tt.iter(t.root, _pre[:0], false, cb)\n}\n\n// Internal methods\n\n// Internal call to insert that can be recursive.\nfunc (t *SubjectTree[T]) insert(np *node, subject []byte, value T, si int) (*T, bool) {\n\tn := *np\n\tif n == nil {\n\t\t*np = newLeaf(subject, value)\n\t\treturn nil, false\n\t}\n\tif n.isLeaf() {\n\t\tln := n.(*leaf[T])\n\t\tif ln.match(subject[si:]) {\n\t\t\t// Replace with new value.\n\t\t\told := ln.value\n\t\t\tln.value = value\n\t\t\treturn &old, true\n\t\t}\n\t\t// Here we need to split this leaf.\n\t\tcpi := commonPrefixLen(ln.suffix, subject[si:])\n\t\tnn := newNode4(subject[si : si+cpi])\n\t\tln.setSuffix(ln.suffix[cpi:])\n\t\tsi += cpi\n\t\t// Make sure we have different pivot, normally this will be the case unless we have overflowing prefixes.\n\t\tif p := pivot(ln.suffix, 0); cpi > 0 && si < len(subject) && p == subject[si] {\n\t\t\t// We need to split the original leaf. Recursively call into insert.\n\t\t\tt.insert(np, subject, value, si)\n\t\t\t// Now add the update version of *np as a child to the new node4.\n\t\t\tnn.addChild(p, *np)\n\t\t} else {\n\t\t\t// Can just add this new leaf as a sibling.\n\t\t\tnl := newLeaf(subject[si:], value)\n\t\t\tnn.addChild(pivot(nl.suffix, 0), nl)\n\t\t\t// Add back original.\n\t\t\tnn.addChild(pivot(ln.suffix, 0), ln)\n\t\t}\n\t\t*np = nn\n\t\treturn nil, false\n\t}\n\n\t// Non-leaf nodes.\n\tbn := n.base()\n\tif len(bn.prefix) > 0 {\n\t\tcpi := commonPrefixLen(bn.prefix, subject[si:])\n\t\tif pli := len(bn.prefix); cpi >= pli {\n\t\t\t// Move past this node. We look for an existing child node to recurse into.\n\t\t\t// If one does not exist we can create a new leaf node.\n\t\t\tsi += pli\n\t\t\tif nn := n.findChild(pivot(subject, si)); nn != nil {\n\t\t\t\treturn t.insert(nn, subject, value, si)\n\t\t\t}\n\t\t\tif n.isFull() {\n\t\t\t\tn = n.grow()\n\t\t\t\t*np = n\n\t\t\t}\n\t\t\tn.addChild(pivot(subject, si), newLeaf(subject[si:], value))\n\t\t\treturn nil, false\n\t\t} else {\n\t\t\t// We did not match the prefix completely here.\n\t\t\t// Calculate new prefix for this node.\n\t\t\tprefix := subject[si : si+cpi]\n\t\t\tsi += len(prefix)\n\t\t\t// We will insert a new node4 and attach our current node below after adjusting prefix.\n\t\t\tnn := newNode4(prefix)\n\t\t\t// Shift the prefix for our original node.\n\t\t\tn.setPrefix(bn.prefix[cpi:])\n\t\t\tnn.addChild(pivot(bn.prefix[:], 0), n)\n\t\t\t// Add in our new leaf.\n\t\t\tnn.addChild(pivot(subject[si:], 0), newLeaf(subject[si:], value))\n\t\t\t// Update our node reference.\n\t\t\t*np = nn\n\t\t}\n\t} else {\n\t\tif nn := n.findChild(pivot(subject, si)); nn != nil {\n\t\t\treturn t.insert(nn, subject, value, si)\n\t\t}\n\t\t// No prefix and no matched child, so add in new leafnode as needed.\n\t\tif n.isFull() {\n\t\t\tn = n.grow()\n\t\t\t*np = n\n\t\t}\n\t\tn.addChild(pivot(subject, si), newLeaf(subject[si:], value))\n\t}\n\n\treturn nil, false\n}\n\n// internal function to recursively find the leaf to delete. Will do compaction if the item is found and removed.\nfunc (t *SubjectTree[T]) delete(np *node, subject []byte, si int) (*T, bool) {\n\tif t == nil || np == nil || *np == nil || len(subject) == 0 {\n\t\treturn nil, false\n\t}\n\tn := *np\n\tif n.isLeaf() {\n\t\tln := n.(*leaf[T])\n\t\tif ln.match(subject[si:]) {\n\t\t\t*np = nil\n\t\t\treturn &ln.value, true\n\t\t}\n\t\treturn nil, false\n\t}\n\t// Not a leaf node.\n\tif bn := n.base(); len(bn.prefix) > 0 {\n\t\t// subject could be shorter and would panic on bad index into subject slice.\n\t\tif len(subject) < si+len(bn.prefix) {\n\t\t\treturn nil, false\n\t\t}\n\t\tif !bytes.Equal(subject[si:si+len(bn.prefix)], bn.prefix) {\n\t\t\treturn nil, false\n\t\t}\n\t\t// Increment our subject index.\n\t\tsi += len(bn.prefix)\n\t}\n\tp := pivot(subject, si)\n\tnna := n.findChild(p)\n\tif nna == nil {\n\t\treturn nil, false\n\t}\n\tnn := *nna\n\tif nn.isLeaf() {\n\t\tln := nn.(*leaf[T])\n\t\tif ln.match(subject[si:]) {\n\t\t\tn.deleteChild(p)\n\n\t\t\tif sn := n.shrink(); sn != nil {\n\t\t\t\tbn := n.base()\n\t\t\t\t// Make sure to set cap so we force an append to copy below.\n\t\t\t\tpre := bn.prefix[:len(bn.prefix):len(bn.prefix)]\n\t\t\t\t// Need to fix up prefixes/suffixes.\n\t\t\t\tif sn.isLeaf() {\n\t\t\t\t\tln := sn.(*leaf[T])\n\t\t\t\t\t// Make sure to set cap so we force an append to copy.\n\t\t\t\t\tln.suffix = append(pre, ln.suffix...)\n\t\t\t\t} else {\n\t\t\t\t\t// We are a node here, we need to add in the old prefix.\n\t\t\t\t\tif len(pre) > 0 {\n\t\t\t\t\t\tbsn := sn.base()\n\t\t\t\t\t\tsn.setPrefix(append(pre, bsn.prefix...))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t*np = sn\n\t\t\t}\n\n\t\t\treturn &ln.value, true\n\t\t}\n\t\treturn nil, false\n\t}\n\treturn t.delete(nna, subject, si)\n}\n\n// Internal function which can be called recursively to match all leaf nodes to a given filter subject which\n// once here has been decomposed to parts. These parts only care about wildcards, both pwc and fwc.\n// Returns false if the callback requested to stop matching.\nfunc (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subject []byte, val *T) bool) bool {\n\t// Capture if we are sitting on a terminal fwc.\n\tvar hasFWC bool\n\tif lp := len(parts); lp > 0 && len(parts[lp-1]) > 0 && parts[lp-1][0] == fwc {\n\t\thasFWC = true\n\t}\n\n\tfor n != nil {\n\t\tnparts, matched := n.matchParts(parts)\n\t\t// Check if we did not match.\n\t\tif !matched {\n\t\t\treturn true\n\t\t}\n\t\t// We have matched here. If we are a leaf and have exhausted all parts or he have a FWC fire callback.\n\t\tif n.isLeaf() {\n\t\t\tif len(nparts) == 0 || (hasFWC && len(nparts) == 1) {\n\t\t\t\tln := n.(*leaf[T])\n\t\t\t\tif !cb(append(pre, ln.suffix...), &ln.value) {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t\t// We have normal nodes here.\n\t\t// We need to append our prefix\n\t\tbn := n.base()\n\t\tif len(bn.prefix) > 0 {\n\t\t\t// Note that this append may reallocate, but it doesn't modify \"pre\" at the \"match\" callsite.\n\t\t\tpre = append(pre, bn.prefix...)\n\t\t}\n\n\t\t// Check our remaining parts.\n\t\tif len(nparts) == 0 && !hasFWC {\n\t\t\t// We are a node with no parts left and we are not looking at a fwc.\n\t\t\t// We could have a leafnode with no suffix which would be a match.\n\t\t\t// We could also have a terminal pwc. Check for those here.\n\t\t\tvar hasTermPWC bool\n\t\t\tif lp := len(parts); lp > 0 && len(parts[lp-1]) == 1 && parts[lp-1][0] == pwc {\n\t\t\t\t// If we are sitting on a terminal pwc, put the pwc back and continue.\n\t\t\t\tnparts = parts[len(parts)-1:]\n\t\t\t\thasTermPWC = true\n\t\t\t}\n\t\t\tfor _, cn := range n.children() {\n\t\t\t\tif cn == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif cn.isLeaf() {\n\t\t\t\t\tln := cn.(*leaf[T])\n\t\t\t\t\tif len(ln.suffix) == 0 {\n\t\t\t\t\t\tif !cb(append(pre, ln.suffix...), &ln.value) {\n\t\t\t\t\t\t\treturn false\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if hasTermPWC && bytes.IndexByte(ln.suffix, tsep) < 0 {\n\t\t\t\t\t\tif !cb(append(pre, ln.suffix...), &ln.value) {\n\t\t\t\t\t\t\treturn false\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else if hasTermPWC {\n\t\t\t\t\t// We have terminal pwc so call into match again with the child node.\n\t\t\t\t\tif !t.match(cn, nparts, pre, cb) {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Return regardless.\n\t\t\treturn true\n\t\t}\n\t\t// If we are sitting on a terminal fwc, put back and continue.\n\t\tif hasFWC && len(nparts) == 0 {\n\t\t\tnparts = parts[len(parts)-1:]\n\t\t}\n\n\t\t// Here we are a node type with a partial match.\n\t\t// Check if the first part is a wildcard.\n\t\tfp := nparts[0]\n\t\tp := pivot(fp, 0)\n\t\t// Check if we have a pwc/fwc part here. This will cause us to iterate.\n\t\tif len(fp) == 1 && (p == pwc || p == fwc) {\n\t\t\t// We need to iterate over all children here for the current node\n\t\t\t// to see if we match further down.\n\t\t\tfor _, cn := range n.children() {\n\t\t\t\tif cn != nil {\n\t\t\t\t\tif !t.match(cn, nparts, pre, cb) {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t\t// Here we have normal traversal, so find the next child.\n\t\tnn := n.findChild(p)\n\t\tif nn == nil {\n\t\t\treturn true\n\t\t}\n\t\tn, parts = *nn, nparts\n\t}\n\treturn true\n}\n\n// Internal iter function to walk nodes in lexicographical order.\nfunc (t *SubjectTree[T]) iter(n node, pre []byte, ordered bool, cb func(subject []byte, val *T) bool) bool {\n\tif n.isLeaf() {\n\t\tln := n.(*leaf[T])\n\t\treturn cb(append(pre, ln.suffix...), &ln.value)\n\t}\n\t// We are normal node here.\n\tbn := n.base()\n\t// Note that this append may reallocate, but it doesn't modify \"pre\" at the \"iter\" callsite.\n\tpre = append(pre, bn.prefix...)\n\t// Not everything requires lexicographical sorting, so support a fast path for iterating in\n\t// whatever order the stree has things stored instead.\n\tif !ordered {\n\t\tfor _, cn := range n.children() {\n\t\t\tif cn == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !t.iter(cn, pre, false, cb) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}\n\t// Collect nodes since unsorted.\n\tvar _nodes [256]node\n\tnodes := _nodes[:0]\n\tfor _, cn := range n.children() {\n\t\tif cn != nil {\n\t\t\tnodes = append(nodes, cn)\n\t\t}\n\t}\n\t// Now sort.\n\tslices.SortStableFunc(nodes, func(a, b node) int { return bytes.Compare(a.path(), b.path()) })\n\t// Now walk the nodes in order and call into next iter.\n\tfor i := range nodes {\n\t\tif !t.iter(nodes[i], pre, true, cb) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// LazyIntersect iterates the smaller of the two provided subject trees and\n// looks for matching entries in the other. It is lazy in that it does not\n// aggressively optimize against repeated walks, but is considerably faster\n// in most cases than intersecting against a potentially large sublist.\nfunc LazyIntersect[TL, TR any](tl *SubjectTree[TL], tr *SubjectTree[TR], cb func([]byte, *TL, *TR)) {\n\tif tl == nil || tr == nil || tl.root == nil || tr.root == nil {\n\t\treturn\n\t}\n\t// Iterate over the smaller tree to reduce the number of rounds.\n\tif tl.Size() <= tr.Size() {\n\t\ttl.IterFast(func(key []byte, v1 *TL) bool {\n\t\t\tif v2, ok := tr.Find(key); ok {\n\t\t\t\tcb(key, v1, v2)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t} else {\n\t\ttr.IterFast(func(key []byte, v2 *TR) bool {\n\t\t\tif v1, ok := tl.Find(key); ok {\n\t\t\t\tcb(key, v1, v2)\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t}\n}\n\n// IntersectGSL will match all items in the given subject tree that\n// have interest expressed in the given sublist. The callback will only be called\n// once for each subject, regardless of overlapping subscriptions in the sublist.\nfunc IntersectGSL[T any, SL comparable](t *SubjectTree[T], sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T)) {\n\tif t == nil || t.root == nil || sl == nil {\n\t\treturn\n\t}\n\tvar _pre [256]byte\n\t_intersectGSL(t.root, _pre[:0], sl, cb)\n}\n\nfunc _intersectGSL[T any, SL comparable](n node, pre []byte, sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T)) {\n\tif n.isLeaf() {\n\t\tln := n.(*leaf[T])\n\t\tsubj := append(pre, ln.suffix...)\n\t\tif sl.HasInterest(bytesToString(subj)) {\n\t\t\tcb(subj, &ln.value)\n\t\t}\n\t\treturn\n\t}\n\tbn := n.base()\n\tpre = append(pre, bn.prefix...)\n\tfor _, cn := range n.children() {\n\t\tif cn == nil {\n\t\t\tcontinue\n\t\t}\n\t\tsubj := append(pre, cn.path()...)\n\t\tif !hasInterestForTokens(sl, subj, len(pre)) {\n\t\t\tcontinue\n\t\t}\n\t\t_intersectGSL(cn, pre, sl, cb)\n\t}\n}\n\n// The subject tree can return partial tokens so we need to check starting interest\n// only from whole tokens when we encounter a tsep.\nfunc hasInterestForTokens[SL comparable](sl *gsl.GenericSublist[SL], subj []byte, since int) bool {\n\tfor i := since; i < len(subj); i++ {\n\t\tif subj[i] == tsep {\n\t\t\tif !sl.HasInterestStartingIn(bytesToString(subj[:i])) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer.\n// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same.\nfunc bytesToString(b []byte) string {\n\tif len(b) == 0 {\n\t\treturn \"\"\n\t}\n\tp := unsafe.SliceData(b)\n\treturn unsafe.String(p, len(b))\n}\n" }, { "path": "server/stree/stree_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\nimport (\n\tcrand \"crypto/rand\"\n\t\"encoding/hex\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server/gsl\"\n)\n\n// Print Results: go test -v --args --results\n// For some benchmarks.\nvar runResults = flag.Bool(\"results\", false, \"Enable Results Tests\")\n\nfunc TestSubjectTreeBasics(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\trequire_Equal(t, st.Size(), 0)\n\t// Single leaf\n\told, updated := st.Insert(b(\"foo.bar.baz\"), 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\trequire_Equal(t, st.Size(), 1)\n\t// Find shouldn't work with a wildcard.\n\t_, found := st.Find(b(\"foo.bar.*\"))\n\trequire_False(t, found)\n\t// But it should with a literal. Find with single leaf.\n\tv, found := st.Find(b(\"foo.bar.baz\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t// Update single leaf\n\told, updated = st.Insert(b(\"foo.bar.baz\"), 33)\n\trequire_True(t, old != nil)\n\trequire_Equal(t, *old, 22)\n\trequire_True(t, updated)\n\trequire_Equal(t, st.Size(), 1)\n\t// Split the tree\n\told, updated = st.Insert(b(\"foo.bar\"), 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\trequire_Equal(t, st.Size(), 2)\n\t// Now we have node4 -> leaf*2\n\tv, found = st.Find(b(\"foo.bar\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t// Make sure we can still retrieve the original after the split.\n\tv, found = st.Find(b(\"foo.bar.baz\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 33)\n}\n\nfunc TestSubjectTreeNodeGrow(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tfor i := 0; i < 4; i++ {\n\t\tsubj := b(fmt.Sprintf(\"foo.bar.%c\", 'A'+i))\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t}\n\t// We have filled a node4.\n\t_, ok := st.root.(*node4)\n\trequire_True(t, ok)\n\t// This one will trigger us to grow.\n\told, updated := st.Insert(b(\"foo.bar.E\"), 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\t_, ok = st.root.(*node10)\n\trequire_True(t, ok)\n\tfor i := 5; i < 10; i++ {\n\t\tsubj := b(fmt.Sprintf(\"foo.bar.%c\", 'A'+i))\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t}\n\t// This one will trigger us to grow.\n\told, updated = st.Insert(b(\"foo.bar.K\"), 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\t// We have filled a node10.\n\t_, ok = st.root.(*node16)\n\trequire_True(t, ok)\n\tfor i := 11; i < 16; i++ {\n\t\tsubj := b(fmt.Sprintf(\"foo.bar.%c\", 'A'+i))\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t}\n\t// This one will trigger us to grow.\n\told, updated = st.Insert(b(\"foo.bar.Q\"), 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\t_, ok = st.root.(*node48)\n\trequire_True(t, ok)\n\t// Fill the node48.\n\tfor i := 17; i < 48; i++ {\n\t\tsubj := b(fmt.Sprintf(\"foo.bar.%c\", 'A'+i))\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t}\n\t// This one will trigger us to grow.\n\tsubj := b(fmt.Sprintf(\"foo.bar.%c\", 'A'+49))\n\told, updated = st.Insert(subj, 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\t_, ok = st.root.(*node256)\n\trequire_True(t, ok)\n}\n\nfunc TestSubjectTreeNodePrefixMismatch(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 11)\n\tst.Insert(b(\"foo.bar.B\"), 22)\n\tst.Insert(b(\"foo.bar.C\"), 33)\n\t// Grab current root. Split below will cause update.\n\tor := st.root\n\t// This one will force a split of the node\n\tst.Insert(b(\"foo.foo.A\"), 44)\n\trequire_True(t, or != st.root)\n\t// Now make sure we can retrieve correctly.\n\tv, found := st.Find(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 11)\n\tv, found = st.Find(b(\"foo.bar.B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\tv, found = st.Find(b(\"foo.bar.C\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 33)\n\tv, found = st.Find(b(\"foo.foo.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 44)\n}\n\nfunc TestSubjectTreeNodeDelete(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 22)\n\tv, found := st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\trequire_Equal(t, st.root, nil)\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_False(t, found)\n\trequire_Equal(t, v, nil)\n\tv, found = st.Find(b(\"foo.foo.A\"))\n\trequire_False(t, found)\n\trequire_Equal(t, v, nil)\n\t// Kick to a node4.\n\tst.Insert(b(\"foo.bar.A\"), 11)\n\tst.Insert(b(\"foo.bar.B\"), 22)\n\tst.Insert(b(\"foo.bar.C\"), 33)\n\t// Make sure we can delete and that we shrink back to leaf.\n\tv, found = st.Delete(b(\"foo.bar.C\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 33)\n\tv, found = st.Delete(b(\"foo.bar.B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t// We should have shrunk here.\n\trequire_True(t, st.root.isLeaf())\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 11)\n\trequire_Equal(t, st.root, nil)\n\t// Now pop up to a node10 and make sure we can shrink back down.\n\tfor i := 0; i < 5; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.bar.%c\", 'A'+i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\t_, ok := st.root.(*node10)\n\trequire_True(t, ok)\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t_, ok = st.root.(*node4)\n\trequire_True(t, ok)\n\t// Now pop up to node16\n\tfor i := 0; i < 11; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.bar.%c\", 'A'+i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\t_, ok = st.root.(*node16)\n\trequire_True(t, ok)\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t_, ok = st.root.(*node10)\n\trequire_True(t, ok)\n\tv, found = st.Find(b(\"foo.bar.B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t// Now pop up to node48\n\tst = NewSubjectTree[int]()\n\tfor i := 0; i < 17; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.bar.%c\", 'A'+i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\t_, ok = st.root.(*node48)\n\trequire_True(t, ok)\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t_, ok = st.root.(*node16)\n\trequire_True(t, ok)\n\tv, found = st.Find(b(\"foo.bar.B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t// Now pop up to node256\n\tst = NewSubjectTree[int]()\n\tfor i := 0; i < 49; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.bar.%c\", 'A'+i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\t_, ok = st.root.(*node256)\n\trequire_True(t, ok)\n\tv, found = st.Delete(b(\"foo.bar.A\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\t_, ok = st.root.(*node48)\n\trequire_True(t, ok)\n\tv, found = st.Find(b(\"foo.bar.B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n}\n\nfunc TestSubjectTreeNodesAndPaths(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tcheck := func(subj string) {\n\t\tt.Helper()\n\t\tv, found := st.Find(b(subj))\n\t\trequire_True(t, found)\n\t\trequire_Equal(t, *v, 22)\n\t}\n\tst.Insert(b(\"foo.bar.A\"), 22)\n\tst.Insert(b(\"foo.bar.B\"), 22)\n\tst.Insert(b(\"foo.bar.C\"), 22)\n\tst.Insert(b(\"foo.bar\"), 22)\n\tcheck(\"foo.bar.A\")\n\tcheck(\"foo.bar.B\")\n\tcheck(\"foo.bar.C\")\n\tcheck(\"foo.bar\")\n\t// This will do several things in terms of shrinking and pruning,\n\t// want to make sure it gets prefix correct for new top node4.\n\tst.Delete(b(\"foo.bar\"))\n\tcheck(\"foo.bar.A\")\n\tcheck(\"foo.bar.B\")\n\tcheck(\"foo.bar.C\")\n}\n\n// Check that we are constructing a proper tree with complex insert patterns.\nfunc TestSubjectTreeConstruction(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\tcheckNode := func(an *node, kind string, pors string, numChildren uint16) {\n\t\tt.Helper()\n\t\trequire_True(t, an != nil)\n\t\tn := *an\n\t\trequire_True(t, n != nil)\n\t\trequire_Equal(t, n.kind(), kind)\n\t\trequire_Equal(t, pors, string(n.path()))\n\t\trequire_Equal(t, numChildren, n.numChildren())\n\t}\n\n\tcheckNode(&st.root, \"NODE4\", \"foo.ba\", 2)\n\tnn := st.root.findChild('r')\n\tcheckNode(nn, \"NODE4\", \"r\", 2)\n\tcheckNode((*nn).findChild(noPivot), \"LEAF\", \"\", 0)\n\trnn := (*nn).findChild('.')\n\tcheckNode(rnn, \"NODE4\", \".\", 3)\n\tcheckNode((*rnn).findChild('A'), \"LEAF\", \"A\", 0)\n\tcheckNode((*rnn).findChild('B'), \"LEAF\", \"B\", 0)\n\tcheckNode((*rnn).findChild('C'), \"LEAF\", \"C\", 0)\n\tznn := st.root.findChild('z')\n\tcheckNode(znn, \"NODE4\", \"z.\", 3)\n\tcheckNode((*znn).findChild('A'), \"LEAF\", \"A\", 0)\n\tcheckNode((*znn).findChild('B'), \"LEAF\", \"B\", 0)\n\tcheckNode((*znn).findChild('C'), \"LEAF\", \"C\", 0)\n\t// Use st.Dump() if you want a tree print out.\n\n\t// Now delete \"foo.bar\" and make sure put ourselves back together properly.\n\tv, found := st.Delete(b(\"foo.bar\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 42)\n\n\tcheckNode(&st.root, \"NODE4\", \"foo.ba\", 2)\n\tnn = st.root.findChild('r')\n\tcheckNode(nn, \"NODE4\", \"r.\", 3)\n\tcheckNode((*nn).findChild('A'), \"LEAF\", \"A\", 0)\n\tcheckNode((*nn).findChild('B'), \"LEAF\", \"B\", 0)\n\tcheckNode((*nn).findChild('C'), \"LEAF\", \"C\", 0)\n\tznn = st.root.findChild('z')\n\tcheckNode(znn, \"NODE4\", \"z.\", 3)\n\tcheckNode((*znn).findChild('A'), \"LEAF\", \"A\", 0)\n\tcheckNode((*znn).findChild('B'), \"LEAF\", \"B\", 0)\n\tcheckNode((*znn).findChild('C'), \"LEAF\", \"C\", 0)\n}\n\nfunc match(t *testing.T, st *SubjectTree[int], filter string, expected int) {\n\tt.Helper()\n\tvar matches []int\n\tst.Match(b(filter), func(_ []byte, v *int) {\n\t\tmatches = append(matches, *v)\n\t})\n\trequire_Equal(t, expected, len(matches))\n}\n\nfunc TestSubjectTreeMatchLeafOnly(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.baz.A\"), 1)\n\n\t// Check all placements of pwc in token space.\n\tmatch(t, st, \"foo.bar.*.A\", 1)\n\tmatch(t, st, \"foo.*.baz.A\", 1)\n\tmatch(t, st, \"foo.*.*.A\", 1)\n\tmatch(t, st, \"foo.*.*.*\", 1)\n\tmatch(t, st, \"*.*.*.*\", 1)\n\t// Now check fwc.\n\tmatch(t, st, \">\", 1)\n\tmatch(t, st, \"foo.>\", 1)\n\tmatch(t, st, \"foo.*.>\", 1)\n\tmatch(t, st, \"foo.bar.>\", 1)\n\tmatch(t, st, \"foo.bar.*.>\", 1)\n\n\t// Check partials so they do not trigger on leafs.\n\tmatch(t, st, \"foo.bar.baz\", 0)\n}\n\nfunc TestSubjectTreeMatchNodes(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\n\t// Test literals.\n\tmatch(t, st, \"foo.bar.A\", 1)\n\tmatch(t, st, \"foo.baz.A\", 1)\n\tmatch(t, st, \"foo.bar\", 0)\n\t// Test internal pwc\n\tmatch(t, st, \"foo.*.A\", 2)\n\t// Test terminal pwc\n\tmatch(t, st, \"foo.bar.*\", 3)\n\tmatch(t, st, \"foo.baz.*\", 3)\n\t// Check fwc\n\tmatch(t, st, \">\", 6)\n\tmatch(t, st, \"foo.>\", 6)\n\tmatch(t, st, \"foo.bar.>\", 3)\n\tmatch(t, st, \"foo.baz.>\", 3)\n\t// Make sure we do not have false positives on prefix matches.\n\tmatch(t, st, \"foo.ba\", 0)\n\n\t// Now add in \"foo.bar\" to make a more complex tree construction\n\t// and re-test.\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\t// Test literals.\n\tmatch(t, st, \"foo.bar.A\", 1)\n\tmatch(t, st, \"foo.baz.A\", 1)\n\tmatch(t, st, \"foo.bar\", 1)\n\t// Test internal pwc\n\tmatch(t, st, \"foo.*.A\", 2)\n\t// Test terminal pwc\n\tmatch(t, st, \"foo.bar.*\", 3)\n\tmatch(t, st, \"foo.baz.*\", 3)\n\t// Check fwc\n\tmatch(t, st, \">\", 7)\n\tmatch(t, st, \"foo.>\", 7)\n\tmatch(t, st, \"foo.bar.>\", 3)\n\tmatch(t, st, \"foo.baz.>\", 3)\n}\n\nfunc matchUntilCount[T any](st *SubjectTree[T], filter string, stopAfter int) (int, bool) {\n\tvar n int\n\tcompleted := st.MatchUntil(b(filter), func(_ []byte, _ *T) bool {\n\t\tn++\n\t\treturn n < stopAfter\n\t})\n\treturn n, completed\n}\n\nfunc TestSubjectTreeMatchUntil(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\t// Ensure early stop terminates traversal.\n\tcount, completed := matchUntilCount(st, \"foo.>\", 3)\n\trequire_Equal(t, count, 3)\n\trequire_False(t, completed)\n\n\t// Match completes\n\tcount, completed = matchUntilCount(st, \"foo.bar\", 3)\n\trequire_Equal(t, count, 1)\n\trequire_True(t, completed)\n\n\tcount, completed = matchUntilCount(st, \"foo.baz.*\", 4)\n\trequire_Equal(t, count, 3)\n\trequire_True(t, completed)\n}\n\nfunc TestSubjectTreeNoPrefix(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tfor i := 0; i < 26; i++ {\n\t\tsubj := b(fmt.Sprintf(\"%c\", 'A'+i))\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t}\n\tn, ok := st.root.(*node48)\n\trequire_True(t, ok)\n\trequire_Equal(t, n.numChildren(), 26)\n\tv, found := st.Delete(b(\"B\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\trequire_Equal(t, n.numChildren(), 25)\n\tv, found = st.Delete(b(\"Z\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 22)\n\trequire_Equal(t, n.numChildren(), 24)\n}\n\nfunc TestSubjectTreePartialTerminalWildcardBugMatch(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-A\"), 5)\n\tst.Insert(b(\"STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-B\"), 1)\n\tst.Insert(b(\"STATE.GLOBAL.CELL1.7PDSGAALXNN000010.PROPERTY-C\"), 2)\n\tmatch(t, st, \"STATE.GLOBAL.CELL1.7PDSGAALXNN000010.*\", 3)\n}\n\nfunc TestSubjectTreeMatchSubjectParam(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\tcheckValMap := map[string]int{\n\t\t\"foo.bar.A\": 1,\n\t\t\"foo.bar.B\": 2,\n\t\t\"foo.bar.C\": 3,\n\t\t\"foo.baz.A\": 11,\n\t\t\"foo.baz.B\": 22,\n\t\t\"foo.baz.C\": 33,\n\t\t\"foo.bar\": 42,\n\t}\n\t// Make sure we get a proper subject parameter and it matches our value properly.\n\tst.Match([]byte(\">\"), func(subject []byte, v *int) {\n\t\tif expected, ok := checkValMap[string(subject)]; !ok {\n\t\t\tt.Fatalf(\"Unexpected subject parameter: %q\", subject)\n\t\t} else if expected != *v {\n\t\t\tt.Fatalf(\"Expected %q to have value of %d, but got %d\", subject, expected, *v)\n\t\t}\n\t})\n}\n\nfunc TestSubjectTreeMatchRandomDoublePWC(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tfor i := 1; i <= 10_000; i++ {\n\t\tsubj := fmt.Sprintf(\"foo.%d.%d\", rand.Intn(20)+1, i)\n\t\tst.Insert(b(subj), 42)\n\t}\n\tmatch(t, st, \"foo.*.*\", 10_000)\n\n\t// Check with pwc and short interior token.\n\tseen, verified := 0, 0\n\tst.Match(b(\"*.2.*\"), func(_ []byte, _ *int) {\n\t\tseen++\n\t})\n\t// Now check via walk to make sure we are right.\n\tst.IterOrdered(func(subject []byte, v *int) bool {\n\t\ttokens := strings.Split(string(subject), \".\")\n\t\trequire_Equal(t, len(tokens), 3)\n\t\tif tokens[1] == \"2\" {\n\t\t\tverified++\n\t\t}\n\t\treturn true\n\t})\n\trequire_Equal(t, seen, verified)\n\n\tseen, verified = 0, 0\n\tst.Match(b(\"*.*.222\"), func(_ []byte, _ *int) {\n\t\tseen++\n\t})\n\tst.IterOrdered(func(subject []byte, v *int) bool {\n\t\ttokens := strings.Split(string(subject), \".\")\n\t\trequire_Equal(t, len(tokens), 3)\n\t\tif tokens[2] == \"222\" {\n\t\t\tverified++\n\t\t}\n\t\treturn true\n\t})\n\trequire_Equal(t, seen, verified)\n}\n\nfunc TestSubjectTreeIterOrdered(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\tcheckValMap := map[string]int{\n\t\t\"foo.bar.A\": 1,\n\t\t\"foo.bar.B\": 2,\n\t\t\"foo.bar.C\": 3,\n\t\t\"foo.baz.A\": 11,\n\t\t\"foo.baz.B\": 22,\n\t\t\"foo.baz.C\": 33,\n\t\t\"foo.bar\": 42,\n\t}\n\tcheckOrder := []string{\n\t\t\"foo.bar\",\n\t\t\"foo.bar.A\",\n\t\t\"foo.bar.B\",\n\t\t\"foo.bar.C\",\n\t\t\"foo.baz.A\",\n\t\t\"foo.baz.B\",\n\t\t\"foo.baz.C\",\n\t}\n\tvar received int\n\twalk := func(subject []byte, v *int) bool {\n\t\tif expected := checkOrder[received]; expected != string(subject) {\n\t\t\tt.Fatalf(\"Expected %q for %d item returned, got %q\", expected, received, subject)\n\t\t}\n\t\treceived++\n\t\trequire_True(t, v != nil)\n\t\tif expected := checkValMap[string(subject)]; expected != *v {\n\t\t\tt.Fatalf(\"Expected %q to have value of %d, but got %d\", subject, expected, *v)\n\t\t}\n\t\treturn true\n\t}\n\t// Kick in the iter.\n\tst.IterOrdered(walk)\n\trequire_Equal(t, received, len(checkOrder))\n\n\t// Make sure we can terminate properly.\n\treceived = 0\n\tst.IterOrdered(func(subject []byte, v *int) bool {\n\t\treceived++\n\t\treturn received != 4\n\t})\n\trequire_Equal(t, received, 4)\n}\n\nfunc TestSubjectTreeIterFast(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 2)\n\tst.Insert(b(\"foo.bar.C\"), 3)\n\tst.Insert(b(\"foo.baz.A\"), 11)\n\tst.Insert(b(\"foo.baz.B\"), 22)\n\tst.Insert(b(\"foo.baz.C\"), 33)\n\tst.Insert(b(\"foo.bar\"), 42)\n\n\tcheckValMap := map[string]int{\n\t\t\"foo.bar.A\": 1,\n\t\t\"foo.bar.B\": 2,\n\t\t\"foo.bar.C\": 3,\n\t\t\"foo.baz.A\": 11,\n\t\t\"foo.baz.B\": 22,\n\t\t\"foo.baz.C\": 33,\n\t\t\"foo.bar\": 42,\n\t}\n\tvar received int\n\twalk := func(subject []byte, v *int) bool {\n\t\treceived++\n\t\trequire_True(t, v != nil)\n\t\tif expected := checkValMap[string(subject)]; expected != *v {\n\t\t\tt.Fatalf(\"Expected %q to have value of %d, but got %d\", subject, expected, *v)\n\t\t}\n\t\treturn true\n\t}\n\t// Kick in the iter.\n\tst.IterFast(walk)\n\trequire_Equal(t, received, len(checkValMap))\n\n\t// Make sure we can terminate properly.\n\treceived = 0\n\tst.IterFast(func(subject []byte, v *int) bool {\n\t\treceived++\n\t\treturn received != 4\n\t})\n\trequire_Equal(t, received, 4)\n}\n\nfunc TestSubjectTreeInsertSamePivotBug(t *testing.T) {\n\ttestSubjects := [][]byte{\n\t\t[]byte(\"0d00.2abbb82c1d.6e16.fa7f85470e.3e46\"),\n\t\t[]byte(\"534b12.3486c17249.4dde0666\"),\n\t\t[]byte(\"6f26aabd.920ee3.d4d3.5ffc69f6\"),\n\t\t[]byte(\"8850.ade3b74c31.aa533f77.9f59.a4bd8415.b3ed7b4111\"),\n\t\t[]byte(\"5a75047dcb.5548e845b6.76024a34.14d5b3.80c426.51db871c3a\"),\n\t\t[]byte(\"825fa8acfc.5331.00caf8bbbd.107c4b.c291.126d1d010e\"),\n\t}\n\tst := NewSubjectTree[int]()\n\tfor _, subj := range testSubjects {\n\t\told, updated := st.Insert(subj, 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t\tif _, found := st.Find(subj); !found {\n\t\t\tt.Fatalf(\"Could not find subject %q which should be findable\", subj)\n\t\t}\n\t}\n}\n\nfunc TestSubjectTreeMatchTsepSecondThenPartialPartBug(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.xxxxx.foo1234.zz\"), 22)\n\tst.Insert(b(\"foo.yyy.foo123.zz\"), 22)\n\tst.Insert(b(\"foo.yyybar789.zz\"), 22)\n\tst.Insert(b(\"foo.yyy.foo12345.zz\"), 22)\n\tst.Insert(b(\"foo.yyy.foo12345.yy\"), 22)\n\tst.Insert(b(\"foo.yyy.foo123456789.zz\"), 22)\n\tmatch(t, st, \"foo.*.foo123456789.*\", 1)\n\tmatch(t, st, \"foo.*.*.zzz.foo.>\", 0)\n}\n\nfunc TestSubjectTreeMatchMultipleWildcardBasic(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"A.B.C.D.0.G.H.I.0\"), 22)\n\tst.Insert(b(\"A.B.C.D.1.G.H.I.0\"), 22)\n\tmatch(t, st, \"A.B.*.D.1.*.*.I.0\", 1)\n}\n\nfunc TestSubjectTreeMatchInvalidWildcard(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.123\"), 22)\n\tst.Insert(b(\"one.two.three.four.five\"), 22)\n\tst.Insert(b(\"'*.123\"), 22)\n\tst.Insert(b(\"bar\"), 22)\n\tmatch(t, st, \"invalid.>\", 0)\n\tmatch(t, st, \"foo.>.bar\", 0)\n\tmatch(t, st, \">\", 4)\n\tmatch(t, st, `'*.*`, 1)\n\tmatch(t, st, `'*.*.*'`, 0)\n\t// None of these should match.\n\tmatch(t, st, \"`>`\", 0)\n\tmatch(t, st, `\">\"`, 0)\n\tmatch(t, st, `'>'`, 0)\n\tmatch(t, st, `'*.>'`, 0)\n\tmatch(t, st, `'*.>.`, 0)\n\tmatch(t, st, \"`invalid.>`\", 0)\n\tmatch(t, st, `'*.*'`, 0)\n}\n\nfunc TestSubjectTreeRandomTrackEntries(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tsmap := make(map[string]struct{}, 1000)\n\n\t// Make sure all added items can be found.\n\tcheck := func() {\n\t\tt.Helper()\n\t\tfor subj := range smap {\n\t\t\tif _, found := st.Find(b(subj)); !found {\n\t\t\t\tt.Fatalf(\"Could not find subject %q which should be findable\", subj)\n\t\t\t}\n\t\t}\n\t}\n\n\tbuf := make([]byte, 10)\n\tfor i := 0; i < 1000; i++ {\n\t\tvar sb strings.Builder\n\t\t// 1-6 tokens.\n\t\tnumTokens := rand.Intn(6) + 1\n\t\tfor i := 0; i < numTokens; i++ {\n\t\t\ttlen := rand.Intn(4) + 2\n\t\t\ttok := buf[:tlen]\n\t\t\tcrand.Read(tok)\n\t\t\tsb.WriteString(hex.EncodeToString(tok))\n\t\t\tif i != numTokens-1 {\n\t\t\t\tsb.WriteString(\".\")\n\t\t\t}\n\t\t}\n\t\tsubj := sb.String()\n\t\t// Avoid dupes since will cause check to fail after we delete messages.\n\t\tif _, ok := smap[subj]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tsmap[subj] = struct{}{}\n\t\told, updated := st.Insert(b(subj), 22)\n\t\trequire_True(t, old == nil)\n\t\trequire_False(t, updated)\n\t\trequire_Equal(t, st.Size(), len(smap))\n\t\tcheck()\n\t}\n}\n\n// Needs to be longer then internal node prefix, which currently is 24.\nfunc TestSubjectTreeLongTokens(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"a1.aaaaaaaaaaaaaaaaaaaaaa0\"), 1)\n\tst.Insert(b(\"a2.0\"), 2)\n\tst.Insert(b(\"a1.aaaaaaaaaaaaaaaaaaaaaa1\"), 3)\n\tst.Insert(b(\"a2.1\"), 4)\n\t// Simulate purge of a2.>\n\t// This required to show bug.\n\tst.Delete(b(\"a2.0\"))\n\tst.Delete(b(\"a2.1\"))\n\trequire_Equal(t, st.Size(), 2)\n\tv, found := st.Find(b(\"a1.aaaaaaaaaaaaaaaaaaaaaa0\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 1)\n\tv, found = st.Find(b(\"a1.aaaaaaaaaaaaaaaaaaaaaa1\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 3)\n}\n\nfunc b(s string) []byte {\n\treturn []byte(s)\n}\n\nfunc TestSubjectTreeMatchAllPerf(t *testing.T) {\n\tif !*runResults {\n\t\tt.Skip()\n\t}\n\tst := NewSubjectTree[int]()\n\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"subj.%d.%d\", rand.Intn(100)+1, i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\n\tfor _, f := range [][]byte{\n\t\t[]byte(\">\"),\n\t\t[]byte(\"subj.>\"),\n\t\t[]byte(\"subj.*.*\"),\n\t\t[]byte(\"*.*.*\"),\n\t\t[]byte(\"subj.1.*\"),\n\t\t[]byte(\"subj.1.>\"),\n\t\t[]byte(\"subj.*.1\"),\n\t\t[]byte(\"*.*.1\"),\n\t} {\n\t\tstart := time.Now()\n\t\tcount := 0\n\t\tst.Match(f, func(_ []byte, _ *int) {\n\t\t\tcount++\n\t\t})\n\t\tt.Logf(\"Match %q took %s and matched %d entries\", f, time.Since(start), count)\n\t}\n}\n\nfunc TestSubjectTreeIterPerf(t *testing.T) {\n\tif !*runResults {\n\t\tt.Skip()\n\t}\n\tst := NewSubjectTree[int]()\n\n\tfor i := 0; i < 1_000_000; i++ {\n\t\tsubj := fmt.Sprintf(\"subj.%d.%d\", rand.Intn(100)+1, i)\n\t\tst.Insert(b(subj), 22)\n\t}\n\n\tstart := time.Now()\n\tcount := 0\n\tst.IterOrdered(func(_ []byte, _ *int) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\tt.Logf(\"Iter took %s and matched %d entries\", time.Since(start), count)\n}\n\nfunc TestSubjectTreeNode48(t *testing.T) {\n\tvar a, b, c leaf[int]\n\tvar n node48\n\n\tn.addChild('A', &a)\n\trequire_Equal(t, n.key['A'], 1)\n\trequire_True(t, n.child[0] != nil)\n\trequire_Equal(t, n.child[0].(*leaf[int]), &a)\n\trequire_Equal(t, len(n.children()), 1)\n\n\tchild := n.findChild('A')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, (*child).(*leaf[int]), &a)\n\n\tn.addChild('B', &b)\n\trequire_Equal(t, n.key['B'], 2)\n\trequire_True(t, n.child[1] != nil)\n\trequire_Equal(t, n.child[1].(*leaf[int]), &b)\n\trequire_Equal(t, len(n.children()), 2)\n\n\tchild = n.findChild('B')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, (*child).(*leaf[int]), &b)\n\n\tn.addChild('C', &c)\n\trequire_Equal(t, n.key['C'], 3)\n\trequire_True(t, n.child[2] != nil)\n\trequire_Equal(t, n.child[2].(*leaf[int]), &c)\n\trequire_Equal(t, len(n.children()), 3)\n\n\tchild = n.findChild('C')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, (*child).(*leaf[int]), &c)\n\n\tn.deleteChild('A')\n\trequire_Equal(t, len(n.children()), 2)\n\trequire_Equal(t, n.key['A'], 0) // Now deleted\n\trequire_Equal(t, n.key['B'], 2) // Untouched\n\trequire_Equal(t, n.key['C'], 1) // Where A was\n\n\tchild = n.findChild('A')\n\trequire_Equal(t, child, nil)\n\trequire_True(t, n.child[0] != nil)\n\trequire_Equal(t, n.child[0].(*leaf[int]), &c)\n\n\tchild = n.findChild('B')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, (*child).(*leaf[int]), &b)\n\trequire_True(t, n.child[1] != nil)\n\trequire_Equal(t, n.child[1].(*leaf[int]), &b)\n\n\tchild = n.findChild('C')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, (*child).(*leaf[int]), &c)\n\trequire_True(t, n.child[2] == nil)\n\n\tvar gotB, gotC bool\n\tvar iterations int\n\tn.iter(func(n node) bool {\n\t\titerations++\n\t\tif gb, ok := n.(*leaf[int]); ok && &b == gb {\n\t\t\tgotB = true\n\t\t}\n\t\tif gc, ok := n.(*leaf[int]); ok && &c == gc {\n\t\t\tgotC = true\n\t\t}\n\t\treturn true\n\t})\n\trequire_Equal(t, iterations, 2)\n\trequire_True(t, gotB)\n\trequire_True(t, gotC)\n\n\t// Check for off-by-one on byte 255 as found by staticcheck, see\n\t// https://github.com/nats-io/nats-server/pull/5826.\n\tn.addChild(255, &c)\n\trequire_Equal(t, n.key[255], 3)\n\tgrown := n.grow().(*node256)\n\trequire_True(t, grown.findChild(255) != nil)\n\tshrunk := n.shrink().(*node16)\n\trequire_True(t, shrunk.findChild(255) != nil)\n}\n\nfunc TestSubjectTreeMatchNoCallbackDupe(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tst.Insert(b(\"foo.bar.A\"), 1)\n\tst.Insert(b(\"foo.bar.B\"), 1)\n\tst.Insert(b(\"foo.bar.C\"), 1)\n\tst.Insert(b(\"foo.bar.>\"), 1)\n\n\tfor _, f := range [][]byte{\n\t\t[]byte(\">\"),\n\t\t[]byte(\"foo.>\"),\n\t\t[]byte(\"foo.bar.>\"),\n\t} {\n\t\tseen := map[string]struct{}{}\n\t\tst.Match(f, func(bsubj []byte, _ *int) {\n\t\t\tsubj := string(bsubj)\n\t\t\tif _, ok := seen[subj]; ok {\n\t\t\t\tt.Logf(\"Match callback was called twice for %q\", subj)\n\t\t\t}\n\t\t\tseen[subj] = struct{}{}\n\t\t})\n\t}\n}\n\nfunc TestSubjectTreeNilNoPanic(t *testing.T) {\n\tvar st *SubjectTree[int]\n\tst.Match([]byte(\"foo\"), func(_ []byte, _ *int) {})\n\t_, found := st.Find([]byte(\"foo\"))\n\trequire_False(t, found)\n\t_, found = st.Delete([]byte(\"foo\"))\n\trequire_False(t, found)\n\t_, found = st.Insert([]byte(\"foo\"), 22)\n\trequire_False(t, found)\n}\n\n// This bug requires the trailing suffix contain repeating nulls \\x00\n// and the second subject be longer with more nulls.\nfunc TestSubjectTreeInsertLongerLeafSuffixWithTrailingNulls(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tsubj := []byte(\"foo.bar.baz_\")\n\t// add in 10 nulls.\n\tfor i := 0; i < 10; i++ {\n\t\tsubj = append(subj, 0)\n\t}\n\n\tst.Insert(subj, 1)\n\t// add in 10 more nulls.\n\tsubj2 := subj\n\tfor i := 0; i < 10; i++ {\n\t\tsubj2 = append(subj, 0)\n\t}\n\tst.Insert(subj2, 2)\n\n\t// Make sure we can look them up.\n\tv, found := st.Find(subj)\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 1)\n\tv, found = st.Find(subj2)\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 2)\n}\n\n// Make sure the system does not insert any subject with the noPivot (DEL) in it.\nfunc TestSubjectTreeInsertWithNoPivot(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\tsubj := []byte(\"foo.bar.baz.\")\n\tsubj = append(subj, noPivot)\n\told, updated := st.Insert(subj, 22)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\trequire_Equal(t, st.Size(), 0)\n}\n\n// Make sure we don't panic when checking for fwc.\nfunc TestSubjectTreeMatchHasFWCNoPanic(t *testing.T) {\n\tdefer func() {\n\t\tp := recover()\n\t\trequire_True(t, p == nil)\n\t}()\n\tst := NewSubjectTree[int]()\n\tsubj := []byte(\"foo\")\n\tst.Insert(subj, 1)\n\tst.Match([]byte(\".\"), func(subject []byte, val *int) {})\n}\n\nfunc TestSubjectTreeLazyIntersect(t *testing.T) {\n\tst1 := NewSubjectTree[int]()\n\tst2 := NewSubjectTree[int]()\n\n\t// Should cause an intersection.\n\tst1.Insert([]byte(\"foo.bar\"), 1)\n\tst2.Insert([]byte(\"foo.bar\"), 1)\n\n\t// Should cause an intersection.\n\tst1.Insert([]byte(\"foo.bar.baz.qux\"), 1)\n\tst2.Insert([]byte(\"foo.bar.baz.qux\"), 1)\n\n\t// Should not cause any intersections.\n\tst1.Insert([]byte(\"bar\"), 1)\n\tst2.Insert([]byte(\"baz\"), 1)\n\tst1.Insert([]byte(\"a.b.c\"), 1)\n\tst2.Insert([]byte(\"a.b.d\"), 1)\n\tst1.Insert([]byte(\"a.b.ee\"), 1)\n\tst2.Insert([]byte(\"a.b.e\"), 1)\n\tst1.Insert([]byte(\"bb.c.d\"), 1)\n\tst2.Insert([]byte(\"b.c.d\"), 1)\n\tst2.Insert([]byte(\"foo.bar.baz.qux.alice\"), 1)\n\tst2.Insert([]byte(\"foo.bar.baz.qux.bob\"), 1)\n\n\tintersected := map[string]int{}\n\tLazyIntersect(st1, st2, func(key []byte, val1, val2 *int) {\n\t\tintersected[string(key)]++\n\t})\n\trequire_Equal(t, len(intersected), 2)\n\trequire_Equal(t, intersected[\"foo.bar\"], 1)\n\trequire_Equal(t, intersected[\"foo.bar.baz.qux\"], 1)\n}\n\nfunc TestSubjectTreeGSLIntersection(t *testing.T) {\n\tst := NewSubjectTree[struct{}]()\n\tst.Insert([]byte(\"one.two.three.four\"), struct{}{})\n\tst.Insert([]byte(\"one.two.three.five\"), struct{}{})\n\tst.Insert([]byte(\"one.two.six\"), struct{}{})\n\tst.Insert([]byte(\"one.two.seven\"), struct{}{})\n\tst.Insert([]byte(\"eight.nine\"), struct{}{})\n\tst.Insert([]byte(\"stream.A\"), struct{}{})\n\tst.Insert([]byte(\"stream.A.child\"), struct{}{})\n\n\trequire_NoDuplicates := func(t *testing.T, got map[string]int) {\n\t\tt.Helper()\n\t\tfor _, c := range got {\n\t\t\trequire_Equal(t, c, 1)\n\t\t}\n\t}\n\n\tt.Run(\"Literals\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.two.six\", 11))\n\t\trequire_NoError(t, sl.Insert(\"eight.nine\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWC\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.two.*.*\", 11))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCOverlapping\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.two.*.four\", 11))\n\t\trequire_NoError(t, sl.Insert(\"one.two.*.*\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCAll\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"*.*\", 11))\n\t\trequire_NoError(t, sl.Insert(\"*.*.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"*.*.*.*\", 33))\n\t\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\t\trequire_True(t, sl.HasInterest(\"foo.bar.baz\"))\n\t\trequire_True(t, sl.HasInterest(\"foo.bar.baz.qux\"))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 7)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"FWC\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.>\", 11))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 4)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"FWCOverlapping\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.two.three.four\", 11))\n\t\trequire_NoError(t, sl.Insert(\"one.>\", 22))\n\t\trequire_NoError(t, sl.Insert(\">\", 33))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 7)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"FWCExtended\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.>\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.A\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtended\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.*.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.A\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"*.A.child\", 22))\n\t\trequire_NoError(t, sl.Insert(\"stream.*.child\", 22))\n\t\trequire_NoError(t, sl.Insert(\"stream.A.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"stream.*.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"*.A.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"*.*.child\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 1)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive2\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"*.A.child\", 22))\n\t\trequire_NoError(t, sl.Insert(\"stream.A.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"*.A.*\", 22))\n\t\trequire_NoError(t, sl.Insert(\"*.*.child\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 1)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive3\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.*.*\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 1)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive4\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.*.>\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 1)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive5\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"*.*\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 3)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive6\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.*\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCExtendedAggressive7\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A\", 11))\n\t\trequire_NoError(t, sl.Insert(\"*.*.*\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 4)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"FWCAll\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\">\", 11))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 7)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"NoMatch\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one\", 11))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 0)\n\t})\n\n\tt.Run(\"NoMatches\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one\", 11))\n\t\trequire_NoError(t, sl.Insert(\"eight\", 22))\n\t\trequire_NoError(t, sl.Insert(\"ten\", 33))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 0)\n\t})\n\n\tt.Run(\"NoMatchPartial\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"stream.A.not-child\", 11))\n\t\trequire_NoError(t, sl.Insert(\"stream.A.child.>\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 0)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCDoesntHideLiteral\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.*.six\", 11))\n\t\trequire_NoError(t, sl.Insert(\"one.two.seven\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"PWCDoesntHideLiteral2\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.*.*.four\", 11))\n\t\trequire_NoError(t, sl.Insert(\"one.*.*.five\", 22))\n\t\trequire_NoError(t, sl.Insert(\"one.*.three\", 33))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"DisjointWildcardPositions\", func(t *testing.T) {\n\t\t// Test case documenting bitmask approach limitation:\n\t\t// When patterns have wildcards at different positions (not strict subsets),\n\t\t// the bitmask considers earlier-position wildcards as \"less specific\"\n\t\t// which can cause some matches to be skipped.\n\t\t// e.g., *.two.three.four (wildcard at pos 0) vs one.two.*.* (wildcards at pos 2,3)\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.two.*.*\", 33))\n\t\trequire_NoError(t, sl.Insert(\"*.two.three.four\", 55))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\t// Due to bitmask comparison, one.two.*.* may be skipped in favor of *.two.three.four\n\t\t// This results in only one.two.three.four being matched, missing one.two.three.five\n\t\trequire_Len(t, len(got), 2)\n\t\trequire_NoDuplicates(t, got)\n\t})\n\n\tt.Run(\"LiteralMatchWithOverlappingPWCs\", func(t *testing.T) {\n\t\tgot := map[string]int{}\n\t\tsl := gsl.NewSublist[int]()\n\t\trequire_NoError(t, sl.Insert(\"one.*.four.five\", 11))\n\t\trequire_NoError(t, sl.Insert(\"one.two.three.four\", 22))\n\t\tIntersectGSL(st, sl, func(subj []byte, entry *struct{}) {\n\t\t\tgot[string(subj)]++\n\t\t})\n\t\trequire_Len(t, len(got), 1)\n\t\trequire_NoDuplicates(t, got)\n\t})\n}\n\nfunc TestSubjectTreeDeleteShortSubjectNoPanic(t *testing.T) {\n\tdefer func() {\n\t\tp := recover()\n\t\trequire_True(t, p == nil)\n\t}()\n\n\tst := NewSubjectTree[int]()\n\n\tst.Insert(b(\"foo.bar.baz\"), 1)\n\tst.Insert(b(\"foo.bar.qux\"), 2)\n\n\tv, found := st.Delete(b(\"foo.bar\"))\n\trequire_False(t, found)\n\trequire_Equal(t, v, nil)\n\tv, found = st.Find(b(\"foo.bar.baz\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 1)\n\tv, found = st.Find(b(\"foo.bar.qux\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 2)\n}\n\nfunc TestSubjectTreeEmpty(t *testing.T) {\n\t// Test Empty on nil tree\n\tvar st *SubjectTree[int]\n\tst2 := st.Empty()\n\trequire_True(t, st2 != nil)\n\trequire_Equal(t, st2.Size(), 0)\n\n\t// Test Empty on new tree\n\tst = NewSubjectTree[int]()\n\trequire_Equal(t, st.Size(), 0)\n\tst2 = st.Empty()\n\trequire_True(t, st2 == st) // Should return same instance\n\trequire_Equal(t, st2.Size(), 0)\n\n\t// Test Empty on tree with data\n\tst.Insert(b(\"foo.bar\"), 1)\n\tst.Insert(b(\"foo.baz\"), 2)\n\tst.Insert(b(\"bar.baz\"), 3)\n\trequire_Equal(t, st.Size(), 3)\n\n\t// Empty should clear everything\n\tst2 = st.Empty()\n\trequire_True(t, st2 == st) // Should return same instance\n\trequire_Equal(t, st.Size(), 0)\n\trequire_True(t, st.root == nil)\n\n\t// Verify we can't find old entries\n\t_, found := st.Find(b(\"foo.bar\"))\n\trequire_False(t, found)\n\t_, found = st.Find(b(\"foo.baz\"))\n\trequire_False(t, found)\n\t_, found = st.Find(b(\"bar.baz\"))\n\trequire_False(t, found)\n\n\t// Verify we can insert new entries after Empty\n\told, updated := st.Insert(b(\"new.entry\"), 42)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\trequire_Equal(t, st.Size(), 1)\n\n\tv, found := st.Find(b(\"new.entry\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 42)\n}\n\nfunc TestSubjectTreeLazyIntersectComprehensive(t *testing.T) {\n\t// Test with nil trees\n\tvar st1 *SubjectTree[int]\n\tvar st2 *SubjectTree[string]\n\tcount := 0\n\tLazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) {\n\t\tcount++\n\t})\n\trequire_Equal(t, count, 0)\n\n\t// Test with one nil tree\n\tst1 = NewSubjectTree[int]()\n\tst1.Insert(b(\"foo\"), 1)\n\tLazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) {\n\t\tcount++\n\t})\n\trequire_Equal(t, count, 0)\n\n\t// Test with empty trees\n\tst2 = NewSubjectTree[string]()\n\tLazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) {\n\t\tcount++\n\t})\n\trequire_Equal(t, count, 0)\n\n\t// Test with different value types\n\tst1 = NewSubjectTree[int]()\n\tst2 = NewSubjectTree[string]()\n\n\t// Add some intersecting keys\n\tst1.Insert(b(\"foo.bar\"), 42)\n\tst2.Insert(b(\"foo.bar\"), \"hello\")\n\tst1.Insert(b(\"baz.qux\"), 100)\n\tst2.Insert(b(\"baz.qux\"), \"world\")\n\n\t// Add non-intersecting keys\n\tst1.Insert(b(\"only.in.st1\"), 1)\n\tst2.Insert(b(\"only.in.st2\"), \"two\")\n\n\tresults := make(map[string]struct {\n\t\tv1 int\n\t\tv2 string\n\t})\n\n\tLazyIntersect(st1, st2, func(key []byte, v1 *int, v2 *string) {\n\t\tresults[string(key)] = struct {\n\t\t\tv1 int\n\t\t\tv2 string\n\t\t}{*v1, *v2}\n\t})\n\n\trequire_Equal(t, len(results), 2)\n\trequire_Equal(t, results[\"foo.bar\"].v1, 42)\n\trequire_Equal(t, results[\"foo.bar\"].v2, \"hello\")\n\trequire_Equal(t, results[\"baz.qux\"].v1, 100)\n\trequire_Equal(t, results[\"baz.qux\"].v2, \"world\")\n\n\t// Test that it iterates over smaller tree\n\t// Create a large tree and a small tree\n\tlarge := NewSubjectTree[int]()\n\tsmall := NewSubjectTree[int]()\n\n\t// Large tree has many entries\n\tfor i := 0; i < 100; i++ {\n\t\tlarge.Insert([]byte(fmt.Sprintf(\"large.%d\", i)), i)\n\t}\n\t// Small tree has few entries with some overlap\n\tsmall.Insert(b(\"large.5\"), 500)\n\tsmall.Insert(b(\"large.10\"), 1000)\n\tsmall.Insert(b(\"large.50\"), 5000)\n\tsmall.Insert(b(\"small.only\"), 999)\n\n\tintersectCount := 0\n\tLazyIntersect(large, small, func(key []byte, v1 *int, v2 *int) {\n\t\tintersectCount++\n\t\t// Verify we get the correct values\n\t\tswitch string(key) {\n\t\tcase \"large.5\":\n\t\t\trequire_Equal(t, *v1, 5)\n\t\t\trequire_Equal(t, *v2, 500)\n\t\tcase \"large.10\":\n\t\t\trequire_Equal(t, *v1, 10)\n\t\t\trequire_Equal(t, *v2, 1000)\n\t\tcase \"large.50\":\n\t\t\trequire_Equal(t, *v1, 50)\n\t\t\trequire_Equal(t, *v2, 5000)\n\t\tdefault:\n\t\t\tt.Fatalf(\"Unexpected key: %s\", key)\n\t\t}\n\t})\n\trequire_Equal(t, intersectCount, 3)\n\n\t// Test with complex subjects (multiple levels)\n\tst3 := NewSubjectTree[int]()\n\tst4 := NewSubjectTree[int]()\n\n\t// Deep nesting\n\tst3.Insert(b(\"a.b.c.d.e.f.g\"), 1)\n\tst4.Insert(b(\"a.b.c.d.e.f.g\"), 2)\n\n\t// Partial matches (should not intersect)\n\tst3.Insert(b(\"a.b.c.d\"), 3)\n\tst4.Insert(b(\"a.b.c.d.e\"), 4)\n\n\t// Same prefix different suffix\n\tst3.Insert(b(\"prefix.suffix1\"), 5)\n\tst4.Insert(b(\"prefix.suffix2\"), 6)\n\n\tintersections := 0\n\tLazyIntersect(st3, st4, func(key []byte, v1 *int, v2 *int) {\n\t\tintersections++\n\t\trequire_Equal(t, string(key), \"a.b.c.d.e.f.g\")\n\t\trequire_Equal(t, *v1, 1)\n\t\trequire_Equal(t, *v2, 2)\n\t})\n\trequire_Equal(t, intersections, 1)\n}\n\nfunc TestNode256Operations(t *testing.T) {\n\t// Test node256 creation and basic operations\n\tn := newNode256(b(\"prefix\"))\n\trequire_False(t, n.isFull()) // node256 is never full\n\n\t// Test findChild when child doesn't exist\n\tchild := n.findChild('a')\n\trequire_True(t, child == nil)\n\n\t// Add a child and find it\n\tleaf := newLeaf(b(\"suffix\"), 42)\n\tn.addChild('a', leaf)\n\tchild = n.findChild('a')\n\trequire_True(t, child != nil)\n\trequire_Equal(t, n.size, uint16(1))\n\n\t// Test iter function\n\titerCount := 0\n\tn.iter(func(node) bool {\n\t\titerCount++\n\t\treturn true\n\t})\n\trequire_Equal(t, iterCount, 1)\n\n\t// Test iter with early termination\n\tn.addChild('b', newLeaf(b(\"suffix2\"), 43))\n\tn.addChild('c', newLeaf(b(\"suffix3\"), 44))\n\titerCount = 0\n\tn.iter(func(node) bool {\n\t\titerCount++\n\t\treturn false // Stop after first\n\t})\n\trequire_Equal(t, iterCount, 1)\n\n\t// Test children() method\n\tchildren := n.children()\n\trequire_Equal(t, len(children), 256)\n\n\t// Test that grow() panics\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\trequire_Equal(t, r, \"grow can not be called on node256\")\n\t\t} else {\n\t\t\tt.Fatal(\"grow() should panic on node256\")\n\t\t}\n\t}()\n\tn.grow()\n}\n\nfunc TestNode256Shrink(t *testing.T) {\n\t// To get a node256, we need to go through the progression:\n\t// node4 -> node10 -> node16 -> node48 -> node256\n\t// We need at least 49 children to get to node256\n\n\t// Create nodes directly to test node256 shrinking\n\tn256 := newNode256(b(\"prefix\"))\n\n\t// Add 49 children\n\tfor i := 0; i < 49; i++ {\n\t\tn256.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t}\n\trequire_Equal(t, n256.size, uint16(49))\n\n\t// Shrink should not happen yet (> 48 children)\n\tshrunk := n256.shrink()\n\trequire_True(t, shrunk == nil)\n\n\t// Delete one to get to 48 children\n\tn256.deleteChild(0)\n\trequire_Equal(t, n256.size, uint16(48))\n\n\t// Now shrink should return a node48\n\tshrunk = n256.shrink()\n\trequire_True(t, shrunk != nil)\n\t_, isNode48 := shrunk.(*node48)\n\trequire_True(t, isNode48)\n\n\t// Verify the shrunk node has all remaining children\n\tfor i := 1; i < 49; i++ {\n\t\tchild := shrunk.findChild(byte(i))\n\t\trequire_True(t, child != nil)\n\t}\n}\n\nfunc TestLeafPanicMethods(t *testing.T) {\n\tleaf := newLeaf(b(\"test\"), 42)\n\n\t// Test setPrefix panic\n\tt.Run(\"setPrefix\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"setPrefix called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"setPrefix should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.setPrefix(b(\"prefix\"))\n\t})\n\n\t// Test addChild panic\n\tt.Run(\"addChild\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"addChild called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"addChild should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.addChild('a', nil)\n\t})\n\n\t// Test findChild panic\n\tt.Run(\"findChild\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"findChild called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"findChild should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.findChild('a')\n\t})\n\n\t// Test grow panic\n\tt.Run(\"grow\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"grow called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"grow should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.grow()\n\t})\n\n\t// Test deleteChild panic\n\tt.Run(\"deleteChild\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"deleteChild called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"deleteChild should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.deleteChild('a')\n\t})\n\n\t// Test shrink panic\n\tt.Run(\"shrink\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"shrink called on leaf\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"shrink should panic on leaf\")\n\t\t\t}\n\t\t}()\n\t\tleaf.shrink()\n\t})\n\n\t// Test other leaf methods that should work\n\trequire_True(t, leaf.isFull())\n\trequire_True(t, leaf.base() == nil)\n\trequire_Equal(t, leaf.numChildren(), uint16(0))\n\trequire_True(t, leaf.children() == nil)\n\n\t// Test iter (should do nothing)\n\tcalled := false\n\tleaf.iter(func(n node) bool {\n\t\tcalled = true\n\t\treturn true\n\t})\n\trequire_False(t, called)\n}\n\nfunc TestSizeOnNilTree(t *testing.T) {\n\tvar st *SubjectTree[int]\n\trequire_Equal(t, st.Size(), 0)\n}\n\nfunc TestFindEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Test Find with empty subject at root level\n\tst.Insert(b(\"foo.bar.baz\"), 1)\n\tst.Insert(b(\"foo\"), 2)\n\n\t// This should create a tree structure, now test finding with edge cases\n\tv, found := st.Find(b(\"\"))\n\trequire_False(t, found)\n\trequire_True(t, v == nil)\n}\n\nfunc TestNodeIterMethods(t *testing.T) {\n\t// Test node4 iter\n\tn4 := newNode4(b(\"prefix\"))\n\tn4.addChild('a', newLeaf(b(\"1\"), 1))\n\tn4.addChild('b', newLeaf(b(\"2\"), 2))\n\n\tcount := 0\n\tn4.iter(func(n node) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 2)\n\n\t// Test early termination\n\tcount = 0\n\tn4.iter(func(n node) bool {\n\t\tcount++\n\t\treturn false\n\t})\n\trequire_Equal(t, count, 1)\n\n\t// Test node10 iter\n\tn10 := newNode10(b(\"prefix\"))\n\tfor i := 0; i < 5; i++ {\n\t\tn10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i))\n\t}\n\n\tcount = 0\n\tn10.iter(func(n node) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 5)\n\n\t// Test node16 iter\n\tn16 := newNode16(b(\"prefix\"))\n\tfor i := 0; i < 8; i++ {\n\t\tn16.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i))\n\t}\n\n\tcount = 0\n\tn16.iter(func(n node) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 8)\n}\n\nfunc TestIterOrderedAndIterFastNilRoot(t *testing.T) {\n\t// Test IterOrdered with nil root\n\tst := NewSubjectTree[int]()\n\tcount := 0\n\tst.IterOrdered(func(subject []byte, val *int) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 0)\n\n\t// Test IterFast with nil root\n\tcount = 0\n\tst.IterFast(func(subject []byte, val *int) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 0)\n}\n\nfunc TestNodeAddChildPanic(t *testing.T) {\n\t// Test node4 addChild panic when full\n\tn4 := newNode4(b(\"prefix\"))\n\tn4.addChild('a', newLeaf(b(\"1\"), 1))\n\tn4.addChild('b', newLeaf(b(\"2\"), 2))\n\tn4.addChild('c', newLeaf(b(\"3\"), 3))\n\tn4.addChild('d', newLeaf(b(\"4\"), 4))\n\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\trequire_Equal(t, r, \"node4 full!\")\n\t\t} else {\n\t\t\tt.Fatal(\"addChild should panic when node4 is full\")\n\t\t}\n\t}()\n\tn4.addChild('e', newLeaf(b(\"5\"), 5))\n}\n\nfunc TestNodeAddChildPanicOthers(t *testing.T) {\n\t// Test node10 addChild panic when full\n\tt.Run(\"node10\", func(t *testing.T) {\n\t\tn10 := newNode10(b(\"prefix\"))\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tn10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i))\n\t\t}\n\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"node10 full!\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"addChild should panic when node10 is full\")\n\t\t\t}\n\t\t}()\n\t\tn10.addChild('k', newLeaf(b(\"11\"), 11))\n\t})\n\n\t// Test node16 addChild panic when full\n\tt.Run(\"node16\", func(t *testing.T) {\n\t\tn16 := newNode16(b(\"prefix\"))\n\t\tfor i := 0; i < 16; i++ {\n\t\t\tn16.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t\t}\n\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"node16 full!\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"addChild should panic when node16 is full\")\n\t\t\t}\n\t\t}()\n\t\tn16.addChild(16, newLeaf(b(\"16\"), 16))\n\t})\n\n\t// Test node48 addChild panic when full\n\tt.Run(\"node48\", func(t *testing.T) {\n\t\tn48 := newNode48(b(\"prefix\"))\n\t\tfor i := 0; i < 48; i++ {\n\t\t\tn48.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t\t}\n\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\trequire_Equal(t, r, \"node48 full!\")\n\t\t\t} else {\n\t\t\t\tt.Fatal(\"addChild should panic when node48 is full\")\n\t\t\t}\n\t\t}()\n\t\tn48.addChild(48, newLeaf(b(\"48\"), 48))\n\t})\n}\n\nfunc TestNodeDeleteChildNotFound(t *testing.T) {\n\t// Test node10 deleteChild when child doesn't exist\n\tn10 := newNode10(b(\"prefix\"))\n\tn10.addChild('a', newLeaf(b(\"1\"), 1))\n\tn10.addChild('b', newLeaf(b(\"2\"), 2))\n\n\t// Try to delete non-existent child\n\tn10.deleteChild('z')\n\trequire_Equal(t, n10.size, uint16(2)) // Size should remain unchanged\n\n\t// Test node16 deleteChild when child doesn't exist\n\tn16 := newNode16(b(\"prefix\"))\n\tn16.addChild('a', newLeaf(b(\"1\"), 1))\n\tn16.addChild('b', newLeaf(b(\"2\"), 2))\n\n\tn16.deleteChild('z')\n\trequire_Equal(t, n16.size, uint16(2))\n\n\t// Test node48 deleteChild when child doesn't exist\n\tn48 := newNode48(b(\"prefix\"))\n\tn48.addChild(0, newLeaf(b(\"1\"), 1))\n\tn48.addChild(1, newLeaf(b(\"2\"), 2))\n\n\tn48.deleteChild(255)\n\trequire_Equal(t, n48.size, uint16(2))\n}\n\nfunc TestNodeShrinkNotNeeded(t *testing.T) {\n\t// Test node10 shrink when not needed (has more than 4 children)\n\tn10 := newNode10(b(\"prefix\"))\n\tfor i := 0; i < 5; i++ {\n\t\tn10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i))\n\t}\n\n\tshrunk := n10.shrink()\n\trequire_True(t, shrunk == nil) // Should not shrink\n\n\t// Test node16 shrink when not needed (has more than 10 children)\n\tn16 := newNode16(b(\"prefix\"))\n\tfor i := 0; i < 11; i++ {\n\t\tn16.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t}\n\n\tshrunk = n16.shrink()\n\trequire_True(t, shrunk == nil) // Should not shrink\n}\n\nfunc TestNode48IterEarlyTermination(t *testing.T) {\n\tn48 := newNode48(b(\"prefix\"))\n\tfor i := 0; i < 10; i++ {\n\t\tn48.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t}\n\n\tcount := 0\n\tn48.iter(func(n node) bool {\n\t\tcount++\n\t\treturn false // Stop immediately\n\t})\n\trequire_Equal(t, count, 1)\n}\n\nfunc TestNode10And16IterEarlyTermination(t *testing.T) {\n\t// Test node10 early termination\n\tn10 := newNode10(b(\"prefix\"))\n\tfor i := 0; i < 5; i++ {\n\t\tn10.addChild(byte('a'+i), newLeaf([]byte{byte('0' + i)}, i))\n\t}\n\n\tcount := 0\n\tn10.iter(func(n node) bool {\n\t\tcount++\n\t\treturn count < 2 // Stop after 2\n\t})\n\trequire_Equal(t, count, 2)\n\n\t// Test node16 early termination\n\tn16 := newNode16(b(\"prefix\"))\n\tfor i := 0; i < 8; i++ {\n\t\tn16.addChild(byte(i), newLeaf([]byte{byte(i)}, i))\n\t}\n\n\tcount = 0\n\tn16.iter(func(n node) bool {\n\t\tcount++\n\t\treturn count < 3 // Stop after 3\n\t})\n\trequire_Equal(t, count, 3)\n}\n\nfunc TestMatchPartsEdgeCases(t *testing.T) {\n\t// Test the edge case in matchParts that's not covered\n\t// This is the case where we have a part that needs to be copied and modified\n\n\t// Create a complex filter that will trigger the edge case\n\tfilter := b(\"foo.*.bar.>\")\n\tparts := genParts(filter, nil)\n\n\t// Test with a fragment that will cause partial matching\n\tfrag := b(\"foo.test\")\n\tremaining, matched := matchParts(parts, frag)\n\trequire_True(t, matched)\n\trequire_True(t, len(remaining) > 0)\n}\n\nfunc TestInsertEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Test inserting with noPivot byte (should fail)\n\told, updated := st.Insert([]byte(\"foo\\x7fbar\"), 1)\n\trequire_True(t, old == nil)\n\trequire_False(t, updated)\n\trequire_Equal(t, st.Size(), 0) // Should not insert\n\n\t// Test the edge case where we need to split with same pivot\n\tst = NewSubjectTree[int]()\n\t// This case tests subjects that cause the same pivot after split\n\t// Both subjects share prefix \"a\" and have same pivot \".\" after split\n\tst.Insert(b(\"a.b\"), 1)\n\t// Now insert one that will cause the split with same pivot\n\tst.Insert(b(\"a.c\"), 2)\n\n\trequire_Equal(t, st.Size(), 2)\n}\n\nfunc TestDeleteEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Test delete on empty tree\n\tval, deleted := st.Delete(b(\"foo\"))\n\trequire_False(t, deleted)\n\trequire_True(t, val == nil)\n\n\t// Test delete with empty subject\n\tst.Insert(b(\"foo\"), 1)\n\tval, deleted = st.Delete(b(\"\"))\n\trequire_False(t, deleted)\n\trequire_True(t, val == nil)\n\n\t// Test delete with subject shorter than prefix\n\tst = NewSubjectTree[int]()\n\tst.Insert(b(\"verylongprefix.suffix\"), 1)\n\tst.Insert(b(\"verylongprefix.suffix2\"), 2)\n\tval, deleted = st.Delete(b(\"very\"))\n\trequire_False(t, deleted)\n\trequire_True(t, val == nil)\n}\n\nfunc TestMatchEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Test match with nil callback\n\tst.Insert(b(\"foo.bar\"), 1)\n\tst.Match(b(\"foo.*\"), nil) // Should not panic\n\n\t// Test match with empty filter\n\tcount := 0\n\tst.Match(b(\"\"), func(subject []byte, val *int) {\n\t\tcount++\n\t})\n\trequire_Equal(t, count, 0)\n}\n\nfunc TestIterEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Add multiple subjects to create a complex tree\n\tst.Insert(b(\"a.b.c\"), 1)\n\tst.Insert(b(\"a.b.d\"), 2)\n\tst.Insert(b(\"a.c.d\"), 3)\n\tst.Insert(b(\"b.c.d\"), 4)\n\n\t// Test iter with early termination at different points\n\tcount := 0\n\tst.iter(st.root, nil, false, func(subject []byte, val *int) bool {\n\t\tcount++\n\t\treturn count < 2\n\t})\n\trequire_Equal(t, count, 2)\n}\n\nfunc TestLeafIter(t *testing.T) {\n\t// Test that leaf iter does nothing (it's a no-op)\n\tleaf := newLeaf(b(\"test\"), 42)\n\tcalled := false\n\n\t// Call iter with a function that would set called to true\n\tleaf.iter(func(n node) bool {\n\t\tcalled = true\n\t\treturn true\n\t})\n\trequire_False(t, called) // Should never be called since leaf.iter is a no-op\n\n\t// Call iter again with a function that returns false\n\tleaf.iter(func(n node) bool {\n\t\tcalled = true\n\t\treturn false\n\t})\n\trequire_False(t, called) // Still should never be called\n\n\t// Verify the leaf itself is not affected\n\trequire_True(t, leaf.match(b(\"test\")))\n\trequire_Equal(t, leaf.value, 42)\n\n\t// Also test through the node interface to ensure coverage\n\tvar n node = leaf\n\tcalled = false\n\tn.iter(func(child node) bool {\n\t\tcalled = true\n\t\treturn true\n\t})\n\trequire_False(t, called) // Still should never be called\n}\n\nfunc TestDeleteChildEdgeCasesMore(t *testing.T) {\n\t// Test the edge case in node10 deleteChild where we don't swap (last element)\n\tn10 := newNode10(b(\"prefix\"))\n\tn10.addChild('a', newLeaf(b(\"1\"), 1))\n\tn10.addChild('b', newLeaf(b(\"2\"), 2))\n\tn10.addChild('c', newLeaf(b(\"3\"), 3))\n\n\t// Delete the last child\n\tn10.deleteChild('c')\n\trequire_Equal(t, n10.size, uint16(2))\n\n\t// Test the edge case in node16 deleteChild where we don't swap (last element)\n\tn16 := newNode16(b(\"prefix\"))\n\tn16.addChild('a', newLeaf(b(\"1\"), 1))\n\tn16.addChild('b', newLeaf(b(\"2\"), 2))\n\tn16.addChild('c', newLeaf(b(\"3\"), 3))\n\n\t// Delete the last child\n\tn16.deleteChild('c')\n\trequire_Equal(t, n16.size, uint16(2))\n}\n\nfunc TestMatchPartsMoreEdgeCases(t *testing.T) {\n\t// Test the remaining 2.6% of matchParts\n\t// Case where frag is empty\n\tparts := genParts(b(\"foo.*\"), nil)\n\tremaining, matched := matchParts(parts, b(\"\"))\n\trequire_True(t, matched)\n\trequire_Equal(t, len(remaining), len(parts))\n}\n\nfunc TestInsertComplexEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Test the recursive insert case with same pivot\n\t// This requires a very specific setup\n\t// First, create a tree structure that will trigger the recursive path\n\tst.Insert(b(\"a\"), 1)\n\tst.Insert(b(\"aa\"), 2) // This will create a split\n\n\t// Now insert something that has the same pivot after split\n\tst.Insert(b(\"aaa\"), 3) // This should trigger the recursive insert path\n\n\trequire_Equal(t, st.Size(), 3)\n\n\t// Verify all values can be found\n\tv, found := st.Find(b(\"a\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 1)\n\n\tv, found = st.Find(b(\"aa\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 2)\n\n\tv, found = st.Find(b(\"aaa\"))\n\trequire_True(t, found)\n\trequire_Equal(t, *v, 3)\n}\n\nfunc TestDeleteNilNodePointer(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\t// Test delete with nil node\n\tvar n node\n\tval, deleted := st.delete(&n, b(\"foo\"), 0)\n\trequire_False(t, deleted)\n\trequire_True(t, val == nil)\n}\n\nfunc TestMatchComplexEdgeCases(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Build a complex tree to test the 2.2% uncovered in match\n\tst.Insert(b(\"foo.bar.baz\"), 1)\n\tst.Insert(b(\"foo.bar.qux\"), 2)\n\tst.Insert(b(\"foo.baz.bar\"), 3)\n\tst.Insert(b(\"bar.foo.baz\"), 4)\n\n\t// Test with terminal fwc but no remaining parts\n\tcount := 0\n\tst.Match(b(\"foo.bar.>\"), func(subject []byte, val *int) {\n\t\tcount++\n\t})\n\trequire_Equal(t, count, 2)\n}\n\nfunc TestIterComplexTree(t *testing.T) {\n\tst := NewSubjectTree[int]()\n\n\t// Build a deeper tree to test the remaining iter cases\n\tfor i := 0; i < 20; i++ {\n\t\tst.Insert([]byte(fmt.Sprintf(\"level1.level2.level3.item%d\", i)), i)\n\t}\n\n\t// This should create multiple node types and test more paths\n\tcount := 0\n\tst.IterOrdered(func(subject []byte, val *int) bool {\n\t\tcount++\n\t\treturn true\n\t})\n\trequire_Equal(t, count, 20)\n}\n" }, { "path": "server/stree/util.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stree\n\n// For subject matching.\nconst (\n\tpwc = '*'\n\tfwc = '>'\n\ttsep = '.'\n)\n\n// Determine index of common prefix. No match at all is 0, etc.\nfunc commonPrefixLen(s1, s2 []byte) int {\n\tlimit := min(len(s1), len(s2))\n\tvar i int\n\tfor ; i < limit; i++ {\n\t\tif s1[i] != s2[i] {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn i\n}\n\n// Helper to copy bytes.\nfunc copyBytes(src []byte) []byte {\n\tif len(src) == 0 {\n\t\treturn nil\n\t}\n\tdst := make([]byte, len(src))\n\tcopy(dst, src)\n\treturn dst\n}\n\ntype position interface{ int | uint16 }\n\n// No pivot available.\nconst noPivot = byte(127)\n\n// Can return 127 (DEL) if we have all the subject as prefixes.\n// We used to use 0, but when that was in the subject would cause infinite recursion in some situations.\nfunc pivot[N position](subject []byte, pos N) byte {\n\tif int(pos) >= len(subject) {\n\t\treturn noPivot\n\t}\n\treturn subject[pos]\n}\n" }, { "path": "server/subject_fuzz_test.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport \"testing\"\n\n// FuzzSubjectsCollide performs fuzz testing on the NATS subject collision detection logic.\n// It verifies the behavior of the SubjectsCollide function which determines if two NATS\n// subjects/subscriptions could potentially overlap in the NATS pub-sub system.\nfunc FuzzSubjectsCollide(f *testing.F) {\n\tcorpuses := []struct {\n\t\ts1 string\n\t\ts2 string\n\t}{\n\t\t// SubjectsCollide true\n\t\t{s1: \"\", s2: \"\"},\n\t\t{s1: \"a\", s2: \"a\"},\n\t\t{s1: \"a.b.c\", s2: \"a.b.c\"},\n\t\t{s1: \"$JS.b.c\", s2: \"$JS.b.c\"},\n\t\t{s1: \"a.b.c\", s2: \"a.*.c\"},\n\t\t{s1: \"a.b.*\", s2: \"a.*.c\"},\n\t\t{s1: \"aaa.bbb.ccc\", s2: \"aaa.bbb.ccc\"},\n\t\t{s1: \"aaa.*.ccc\", s2: \"*.bbb.ccc\"},\n\t\t{s1: \"*\", s2: \"*\"},\n\t\t{s1: \"**\", s2: \"*\"},\n\t\t{s1: \"\", s2: \">\"},\n\t\t{s1: \">\", s2: \">\"},\n\t\t{s1: \">>\", s2: \">\"},\n\t\t{s1: \"a\", s2: \">\"},\n\t\t{s1: \"a.b.c\", s2: \">\"},\n\t\t{s1: \"a.b.c.>\", s2: \"a.b.>\"},\n\t\t{s1: \"a.b.c.d.*\", s2: \"a.b.c.*.e\"},\n\t\t{s1: \"a.*.*.d.>\", s2: \"a.bbb.ccc.*.e\"},\n\n\t\t// SubjectsCollide false\n\t\t{s1: \"a\", s2: \"\"},\n\t\t{s1: \"a.b\", s2: \"b.a\"},\n\t\t{s1: \"a.bbbbb.*.d\", s2: \"a.b.>\"},\n\t\t{s1: \"a.b\", s2: \"a.b.c\"},\n\t\t{s1: \"a.b.c\", s2: \"a.b\"},\n\t\t{s1: \"a.b\", s2: \"\"},\n\t\t{s1: \"a.*.*.d.e.>\", s2: \"a.bbb.ccc.*.e\"},\n\t}\n\n\tfor _, crp := range corpuses {\n\t\tf.Add(crp.s1, crp.s2)\n\t}\n\n\tf.Fuzz(func(t *testing.T, s1, s2 string) {\n\t\tif !IsValidSubject(s1) {\n\t\t\treturn\n\t\t}\n\n\t\tif !IsValidSubject(s2) {\n\t\t\treturn\n\t\t}\n\n\t\tSubjectsCollide(s1, s2)\n\t})\n}\n" }, { "path": "server/subject_transform.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"math\"\n\t\"math/rand\"\n\t\"regexp\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n// Subject mapping and transform setups.\nvar (\n\tcommaSeparatorRegEx = regexp.MustCompile(`,\\s*`)\n\tpartitionMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[pP]artition\\s*\\((.*)\\)\\s*}}`)\n\twildcardMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[wW]ildcard\\s*\\((.*)\\)\\s*}}`)\n\tsplitFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[sS]plit[fF]rom[lL]eft\\s*\\((.*)\\)\\s*}}`)\n\tsplitFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[sS]plit[fF]rom[rR]ight\\s*\\((.*)\\)\\s*}}`)\n\tsliceFromLeftMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[sS]lice[fF]rom[lL]eft\\s*\\((.*)\\)\\s*}}`)\n\tsliceFromRightMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[sS]lice[fF]rom[rR]ight\\s*\\((.*)\\)\\s*}}`)\n\tsplitMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[sS]plit\\s*\\((.*)\\)\\s*}}`)\n\tleftMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[lL]eft\\s*\\((.*)\\)\\s*}}`)\n\trightMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[rR]ight\\s*\\((.*)\\)\\s*}}`)\n\trandomMappingFunctionRegEx = regexp.MustCompile(`{{\\s*[rR]andom\\s*\\((.*)\\)\\s*}}`)\n)\n\n// Enum for the subject mapping subjectTransform function types\nconst (\n\tNoTransform int16 = iota\n\tBadTransform\n\tPartition\n\tWildcard\n\tSplitFromLeft\n\tSplitFromRight\n\tSliceFromLeft\n\tSliceFromRight\n\tSplit\n\tLeft\n\tRight\n\tRandom\n)\n\n// Transforms for arbitrarily mapping subjects from one to another for maps, tees and filters.\n// These can also be used for proper mapping on wildcard exports/imports.\n// These will be grouped and caching and locking are assumed to be in the upper layers.\ntype subjectTransform struct {\n\tsrc, dest string\n\tdtoks []string // destination tokens\n\tstoks []string // source tokens\n\tdtokmftypes []int16 // destination token mapping function types\n\tdtokmftokindexesargs [][]int // destination token mapping function array of source token index arguments\n\tdtokmfintargs []int32 // destination token mapping function int32 arguments\n\tdtokmfstringargs []string // destination token mapping function string arguments\n}\n\n// SubjectTransformer transforms subjects using mappings\n//\n// This API is not part of the public API and not subject to SemVer protections\ntype SubjectTransformer interface {\n\t// TODO(dlc) - We could add in client here to allow for things like foo -> foo.$ACCOUNT\n\tMatch(string) (string, error)\n\tTransformSubject(subject string) string\n\tTransformTokenizedSubject(tokens []string) string\n}\n\nfunc NewSubjectTransformWithStrict(src, dest string, strict bool) (*subjectTransform, error) {\n\t// strict = true for import subject mappings that need to be reversible\n\t// (meaning can only use the Wildcard function and must use all the pwcs that are present in the source)\n\t// No source given is equivalent to the source being \">\"\n\n\tif dest == _EMPTY_ {\n\t\treturn nil, nil\n\t}\n\n\tif src == _EMPTY_ {\n\t\tsrc = fwcs\n\t}\n\n\t// Both entries need to be valid subjects.\n\tsv, stokens, npwcs, hasFwc := subjectInfo(src)\n\tdv, dtokens, dnpwcs, dHasFwc := subjectInfo(dest)\n\n\t// Make sure both are valid, match fwc if present and there are no pwcs in the dest subject.\n\tif !sv || !dv || dnpwcs > 0 || hasFwc != dHasFwc {\n\t\treturn nil, ErrBadSubject\n\t}\n\n\tvar dtokMappingFunctionTypes []int16\n\tvar dtokMappingFunctionTokenIndexes [][]int\n\tvar dtokMappingFunctionIntArgs []int32\n\tvar dtokMappingFunctionStringArgs []string\n\n\t// If the src has partial wildcards then the dest needs to have the token place markers.\n\tif npwcs > 0 || hasFwc {\n\t\t// We need to count to make sure that the dest has token holders for the pwcs.\n\t\tsti := make(map[int]int)\n\t\tfor i, token := range stokens {\n\t\t\tif len(token) == 1 && token[0] == pwc {\n\t\t\t\tsti[len(sti)+1] = i\n\t\t\t}\n\t\t}\n\n\t\tnphs := 0\n\t\tfor _, token := range dtokens {\n\t\t\ttranformType, transformArgWildcardIndexes, transfomArgInt, transformArgString, err := indexPlaceHolders(token)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif strict {\n\t\t\t\tif tranformType != NoTransform && tranformType != Wildcard {\n\t\t\t\t\treturn nil, &mappingDestinationErr{token, ErrMappingDestinationNotSupportedForImport}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif tranformType == NoTransform {\n\t\t\t\tdtokMappingFunctionTypes = append(dtokMappingFunctionTypes, NoTransform)\n\t\t\t\tdtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{-1})\n\t\t\t\tdtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, -1)\n\t\t\t\tdtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_)\n\t\t\t} else if tranformType == Random {\n\t\t\t\tdtokMappingFunctionTypes = append(dtokMappingFunctionTypes, Random)\n\t\t\t\tdtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{})\n\t\t\t\tdtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt)\n\t\t\t\tdtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_)\n\t\t\t} else {\n\t\t\t\tnphs += len(transformArgWildcardIndexes)\n\t\t\t\t// Now build up our runtime mapping from dest to source tokens.\n\t\t\t\tvar stis []int\n\t\t\t\tfor _, wildcardIndex := range transformArgWildcardIndexes {\n\t\t\t\t\tif wildcardIndex > npwcs {\n\t\t\t\t\t\treturn nil, &mappingDestinationErr{fmt.Sprintf(\"%s: [%d]\", token, wildcardIndex), ErrMappingDestinationIndexOutOfRange}\n\t\t\t\t\t}\n\t\t\t\t\tstis = append(stis, sti[wildcardIndex])\n\t\t\t\t}\n\t\t\t\tdtokMappingFunctionTypes = append(dtokMappingFunctionTypes, tranformType)\n\t\t\t\tdtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, stis)\n\t\t\t\tdtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt)\n\t\t\t\tdtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, transformArgString)\n\n\t\t\t}\n\t\t}\n\t\tif strict && nphs < npwcs {\n\t\t\t// not all wildcards are being used in the destination\n\t\t\treturn nil, &mappingDestinationErr{dest, ErrMappingDestinationNotUsingAllWildcards}\n\t\t}\n\t} else {\n\t\t// no wildcards used in the source: check that no transform functions are used in the destination\n\t\tfor _, token := range dtokens {\n\t\t\ttranformType, _, transfomArgInt, _, err := indexPlaceHolders(token)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif tranformType == NoTransform {\n\t\t\t\tdtokMappingFunctionTypes = append(dtokMappingFunctionTypes, NoTransform)\n\t\t\t\tdtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{-1})\n\t\t\t\tdtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, -1)\n\t\t\t\tdtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_)\n\t\t\t} else if tranformType == Random || tranformType == Partition {\n\t\t\t\tdtokMappingFunctionTypes = append(dtokMappingFunctionTypes, tranformType)\n\t\t\t\tdtokMappingFunctionTokenIndexes = append(dtokMappingFunctionTokenIndexes, []int{})\n\t\t\t\tdtokMappingFunctionIntArgs = append(dtokMappingFunctionIntArgs, transfomArgInt)\n\t\t\t\tdtokMappingFunctionStringArgs = append(dtokMappingFunctionStringArgs, _EMPTY_)\n\t\t\t} else {\n\t\t\t\treturn nil, &mappingDestinationErr{token, ErrMappingDestinationIndexOutOfRange}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn &subjectTransform{\n\t\tsrc: src,\n\t\tdest: dest,\n\t\tdtoks: dtokens,\n\t\tstoks: stokens,\n\t\tdtokmftypes: dtokMappingFunctionTypes,\n\t\tdtokmftokindexesargs: dtokMappingFunctionTokenIndexes,\n\t\tdtokmfintargs: dtokMappingFunctionIntArgs,\n\t\tdtokmfstringargs: dtokMappingFunctionStringArgs,\n\t}, nil\n}\n\nfunc NewSubjectTransform(src, dest string) (*subjectTransform, error) {\n\treturn NewSubjectTransformWithStrict(src, dest, false)\n}\n\nfunc NewSubjectTransformStrict(src, dest string) (*subjectTransform, error) {\n\treturn NewSubjectTransformWithStrict(src, dest, true)\n}\n\nfunc getMappingFunctionArgs(functionRegEx *regexp.Regexp, token string) []string {\n\tcommandStrings := functionRegEx.FindStringSubmatch(token)\n\tif len(commandStrings) > 1 {\n\t\treturn commaSeparatorRegEx.Split(commandStrings[1], -1)\n\t}\n\treturn nil\n}\n\n// Helper for mapping functions that take a wildcard index and an integer as arguments\nfunc transformIndexIntArgsHelper(token string, args []string, transformType int16) (int16, []int, int32, string, error) {\n\tif len(args) < 2 {\n\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs}\n\t}\n\tif len(args) > 2 {\n\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs}\n\t}\n\ti, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\tif err != nil {\n\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t}\n\tmappingFunctionIntArg, err := strconv.ParseInt(strings.Trim(args[1], \" \"), 10, 32)\n\tif err != nil {\n\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t}\n\n\treturn transformType, []int{i}, int32(mappingFunctionIntArg), _EMPTY_, nil\n}\n\n// Helper to ingest and index the subjectTransform destination token (e.g. $x or {{}}) in the token\n// returns a transformation type, and three function arguments: an array of source subject token indexes,\n// and a single number (e.g. number of partitions, or a slice size), and a string (e.g.a split delimiter)\nfunc indexPlaceHolders(token string) (int16, []int, int32, string, error) {\n\tlength := len(token)\n\tif length > 1 {\n\t\t// old $1, $2, etc... mapping format still supported to maintain backwards compatibility\n\t\tif token[0] == '$' { // simple non-partition mapping\n\t\t\ttp, err := strconv.Atoi(token[1:])\n\t\t\tif err != nil {\n\t\t\t\t// other things rely on tokens starting with $ so not an error just leave it as is\n\t\t\t\treturn NoTransform, []int{-1}, -1, _EMPTY_, nil\n\t\t\t}\n\t\t\treturn Wildcard, []int{tp}, -1, _EMPTY_, nil\n\t\t}\n\n\t\t// New 'mustache' style mapping\n\t\tif length > 4 && token[0] == '{' && token[1] == '{' && token[length-2] == '}' && token[length-1] == '}' {\n\t\t\t// wildcard(wildcard token index) (equivalent to $)\n\t\t\targs := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\tif len(args) == 1 && args[0] == _EMPTY_ {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs}\n\t\t\t\t}\n\t\t\t\tif len(args) == 1 {\n\t\t\t\t\ttokenIndex, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t\t}\n\t\t\t\t\treturn Wildcard, []int{tokenIndex}, -1, _EMPTY_, nil\n\t\t\t\t} else {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// partition(number of partitions, token1, token2, ...)\n\t\t\targs = getMappingFunctionArgs(partitionMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\tif len(args) < 1 {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs}\n\t\t\t\t}\n\t\t\t\tif len(args) == 1 {\n\t\t\t\t\tmappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\t\t\t\t\tif err != nil || mappingFunctionIntArg > math.MaxInt32 {\n\t\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t\t}\n\t\t\t\t\treturn Partition, []int{}, int32(mappingFunctionIntArg), _EMPTY_, nil\n\t\t\t\t}\n\t\t\t\tif len(args) >= 2 {\n\t\t\t\t\tmappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\t\t\t\t\tif err != nil || mappingFunctionIntArg > math.MaxInt32 {\n\t\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t\t}\n\t\t\t\t\tvar numPositions = len(args[1:])\n\t\t\t\t\ttokenIndexes := make([]int, numPositions)\n\t\t\t\t\tfor ti, t := range args[1:] {\n\t\t\t\t\t\ti, err := strconv.Atoi(strings.Trim(t, \" \"))\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t\t\t}\n\t\t\t\t\t\ttokenIndexes[ti] = i\n\t\t\t\t\t}\n\n\t\t\t\t\treturn Partition, tokenIndexes, int32(mappingFunctionIntArg), _EMPTY_, nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// SplitFromLeft(token, position)\n\t\t\targs = getMappingFunctionArgs(splitFromLeftMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, SplitFromLeft)\n\t\t\t}\n\n\t\t\t// SplitFromRight(token, position)\n\t\t\targs = getMappingFunctionArgs(splitFromRightMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, SplitFromRight)\n\t\t\t}\n\n\t\t\t// SliceFromLeft(token, position)\n\t\t\targs = getMappingFunctionArgs(sliceFromLeftMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, SliceFromLeft)\n\t\t\t}\n\n\t\t\t// SliceFromRight(token, position)\n\t\t\targs = getMappingFunctionArgs(sliceFromRightMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, SliceFromRight)\n\t\t\t}\n\n\t\t\t// Right(token, length)\n\t\t\targs = getMappingFunctionArgs(rightMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, Right)\n\t\t\t}\n\n\t\t\t// Left(token, length)\n\t\t\targs = getMappingFunctionArgs(leftMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\treturn transformIndexIntArgsHelper(token, args, Left)\n\t\t\t}\n\n\t\t\t// split(token, deliminator)\n\t\t\targs = getMappingFunctionArgs(splitMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\tif len(args) < 2 {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs}\n\t\t\t\t}\n\t\t\t\tif len(args) > 2 {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationTooManyArgs}\n\t\t\t\t}\n\t\t\t\ti, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t}\n\t\t\t\tif strings.Contains(args[1], \" \") || strings.Contains(args[1], tsep) {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token: token, err: ErrMappingDestinationInvalidArg}\n\t\t\t\t}\n\n\t\t\t\treturn Split, []int{i}, -1, args[1], nil\n\t\t\t}\n\n\t\t\t// Random(max)\n\t\t\targs = getMappingFunctionArgs(randomMappingFunctionRegEx, token)\n\t\t\tif args != nil {\n\t\t\t\tif len(args) != 1 {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationNotEnoughArgs}\n\t\t\t\t}\n\t\t\t\tmappingFunctionIntArg, err := strconv.Atoi(strings.Trim(args[0], \" \"))\n\t\t\t\tif err != nil || mappingFunctionIntArg > math.MaxInt32 {\n\t\t\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrMappingDestinationInvalidArg}\n\t\t\t\t}\n\t\t\t\treturn Random, []int{}, int32(mappingFunctionIntArg), _EMPTY_, nil\n\t\t\t}\n\n\t\t\treturn BadTransform, []int{}, -1, _EMPTY_, &mappingDestinationErr{token, ErrUnknownMappingDestinationFunction}\n\t\t}\n\t}\n\treturn NoTransform, []int{-1}, -1, _EMPTY_, nil\n}\n\n// Helper function to tokenize subjects with partial wildcards into formal transform destinations.\n// e.g. \"foo.*.*\" -> \"foo.$1.$2\"\nfunc transformTokenize(subject string) string {\n\t// We need to make the appropriate markers for the wildcards etc.\n\ti := 1\n\tvar nda []string\n\tfor token := range strings.SplitSeq(subject, tsep) {\n\t\tif token == pwcs {\n\t\t\tnda = append(nda, fmt.Sprintf(\"$%d\", i))\n\t\t\ti++\n\t\t} else {\n\t\t\tnda = append(nda, token)\n\t\t}\n\t}\n\treturn strings.Join(nda, tsep)\n}\n\n// Helper function to go from transform destination to a subject with partial wildcards and ordered list of placeholders\n// E.g.:\n//\n//\t\t\"bar\" -> \"bar\", []\n//\t\t\"foo.$2.$1\" -> \"foo.*.*\", [\"$2\",\"$1\"]\n//\t \"foo.{{wildcard(2)}}.{{wildcard(1)}}\" -> \"foo.*.*\", [\"{{wildcard(2)}}\",\"{{wildcard(1)}}\"]\nfunc transformUntokenize(subject string) (string, []string) {\n\tvar phs []string\n\tvar nda []string\n\n\tfor token := range strings.SplitSeq(subject, tsep) {\n\t\tif args := getMappingFunctionArgs(wildcardMappingFunctionRegEx, token); (len(token) > 1 && token[0] == '$' && token[1] >= '1' && token[1] <= '9') || (len(args) == 1 && args[0] != _EMPTY_) {\n\t\t\tphs = append(phs, token)\n\t\t\tnda = append(nda, pwcs)\n\t\t} else {\n\t\t\tnda = append(nda, token)\n\t\t}\n\t}\n\treturn strings.Join(nda, tsep), phs\n}\n\nfunc tokenizeSubject(subject string) []string {\n\t// Tokenize the subject.\n\ttsa := [32]string{}\n\ttts := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttts = append(tts, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttts = append(tts, subject[start:])\n\treturn tts\n}\n\n// Match will take a literal published subject that is associated with a client and will match and subjectTransform\n// the subject if possible.\n//\n// This API is not part of the public API and not subject to SemVer protections\nfunc (tr *subjectTransform) Match(subject string) (string, error) {\n\t// Special case: matches any and no no-op subjectTransform. May not be legal config for some features\n\t// but specific validations made at subjectTransform create time\n\tif (tr.src == fwcs || tr.src == _EMPTY_) && (tr.dest == fwcs || tr.dest == _EMPTY_) {\n\t\treturn subject, nil\n\t}\n\n\ttts := tokenizeSubject(subject)\n\n\t// TODO(jnm): optimization -> not sure this is actually needed but was there in initial code\n\tif !isValidLiteralSubject(slices.Values(tts)) {\n\t\treturn _EMPTY_, ErrBadSubject\n\t}\n\n\tif (tr.src == _EMPTY_ || tr.src == fwcs) || isSubsetMatch(tts, tr.src) {\n\t\treturn tr.TransformTokenizedSubject(tts), nil\n\t}\n\treturn _EMPTY_, ErrNoTransforms\n}\n\n// TransformSubject transforms a subject\n//\n// This API is not part of the public API and not subject to SemVer protection\nfunc (tr *subjectTransform) TransformSubject(subject string) string {\n\treturn tr.TransformTokenizedSubject(tokenizeSubject(subject))\n}\n\nfunc (tr *subjectTransform) getRandomPartition(ceiling int) string {\n\t// Avoid an integer divide by zero panic below.\n\tif ceiling == 0 {\n\t\treturn \"0\"\n\t}\n\n\treturn strconv.Itoa(int(rand.Int31()) % ceiling)\n}\n\nfunc (tr *subjectTransform) getHashPartition(key []byte, numBuckets int) string {\n\t// Avoid an integer divide by zero panic below.\n\tif numBuckets == 0 {\n\t\treturn \"0\"\n\t}\n\n\th := fnv.New32a()\n\t_, _ = h.Write(key)\n\n\treturn strconv.Itoa(int(h.Sum32() % uint32(numBuckets)))\n}\n\n// Do a subjectTransform on the subject to the dest subject.\nfunc (tr *subjectTransform) TransformTokenizedSubject(tokens []string) string {\n\tif len(tr.dtokmftypes) == 0 {\n\t\treturn tr.dest\n\t}\n\n\tvar b strings.Builder\n\n\t// We need to walk destination tokens and create the mapped subject pulling tokens or mapping functions\n\tli := len(tr.dtokmftypes) - 1\n\tfor i, mfType := range tr.dtokmftypes {\n\t\tif mfType == NoTransform {\n\t\t\t// Break if fwc\n\t\t\tif len(tr.dtoks[i]) == 1 && tr.dtoks[i][0] == fwc {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tb.WriteString(tr.dtoks[i])\n\t\t} else {\n\t\t\tswitch mfType {\n\t\t\tcase Partition:\n\t\t\t\tvar (\n\t\t\t\t\t_buffer [64]byte\n\t\t\t\t\tkeyForHashing = _buffer[:0]\n\t\t\t\t)\n\t\t\t\tif len(tr.dtokmftokindexesargs[i]) > 0 {\n\t\t\t\t\t// When token positions are specified.\n\t\t\t\t\tfor _, sourceToken := range tr.dtokmftokindexesargs[i] {\n\t\t\t\t\t\tkeyForHashing = append(keyForHashing, []byte(tokens[sourceToken])...)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// When using the shorthand partition(n).\n\t\t\t\t\tkeyForHashing = append(keyForHashing, strings.Join(tokens, \".\")...)\n\t\t\t\t}\n\t\t\t\tb.WriteString(tr.getHashPartition(keyForHashing, int(tr.dtokmfintargs[i])))\n\t\t\tcase Wildcard: // simple substitution\n\t\t\t\tswitch {\n\t\t\t\tcase len(tr.dtokmftokindexesargs) < i:\n\t\t\t\t\tbreak\n\t\t\t\tcase len(tr.dtokmftokindexesargs[i]) < 1:\n\t\t\t\t\tbreak\n\t\t\t\tcase len(tokens) <= tr.dtokmftokindexesargs[i][0]:\n\t\t\t\t\tbreak\n\t\t\t\tdefault:\n\t\t\t\t\tb.WriteString(tokens[tr.dtokmftokindexesargs[i][0]])\n\t\t\t\t}\n\t\t\tcase SplitFromLeft:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tposition := int(tr.dtokmfintargs[i])\n\t\t\t\tif position > 0 && position < sourceTokenLen {\n\t\t\t\t\tb.WriteString(sourceToken[:position])\n\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\tb.WriteString(sourceToken[position:])\n\t\t\t\t} else { // too small to split at the requested position: don't split\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase SplitFromRight:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tposition := int(tr.dtokmfintargs[i])\n\t\t\t\tif position > 0 && position < sourceTokenLen {\n\t\t\t\t\tb.WriteString(sourceToken[:sourceTokenLen-position])\n\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\tb.WriteString(sourceToken[sourceTokenLen-position:])\n\t\t\t\t} else { // too small to split at the requested position: don't split\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase SliceFromLeft:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tsliceSize := int(tr.dtokmfintargs[i])\n\t\t\t\tif sliceSize > 0 && sliceSize < sourceTokenLen {\n\t\t\t\t\tfor i := 0; i+sliceSize <= sourceTokenLen; i += sliceSize {\n\t\t\t\t\t\tif i != 0 {\n\t\t\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tb.WriteString(sourceToken[i : i+sliceSize])\n\t\t\t\t\t\tif i+sliceSize != sourceTokenLen && i+sliceSize+sliceSize > sourceTokenLen {\n\t\t\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\t\t\tb.WriteString(sourceToken[i+sliceSize:])\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else { // too small to slice at the requested size: don't slice\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase SliceFromRight:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tsliceSize := int(tr.dtokmfintargs[i])\n\t\t\t\tif sliceSize > 0 && sliceSize < sourceTokenLen {\n\t\t\t\t\tremainder := sourceTokenLen % sliceSize\n\t\t\t\t\tif remainder > 0 {\n\t\t\t\t\t\tb.WriteString(sourceToken[:remainder])\n\t\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\t}\n\t\t\t\t\tfor i := remainder; i+sliceSize <= sourceTokenLen; i += sliceSize {\n\t\t\t\t\t\tb.WriteString(sourceToken[i : i+sliceSize])\n\t\t\t\t\t\tif i+sliceSize < sourceTokenLen {\n\t\t\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else { // too small to slice at the requested size: don't slice\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase Split:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsplits := strings.Split(sourceToken, tr.dtokmfstringargs[i])\n\t\t\t\tfor j, split := range splits {\n\t\t\t\t\tif split != _EMPTY_ {\n\t\t\t\t\t\tb.WriteString(split)\n\t\t\t\t\t}\n\t\t\t\t\tif j < len(splits)-1 && splits[j+1] != _EMPTY_ && !(j == 0 && split == _EMPTY_) {\n\t\t\t\t\t\tb.WriteString(tsep)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase Left:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tsliceSize := int(tr.dtokmfintargs[i])\n\t\t\t\tif sliceSize > 0 && sliceSize < sourceTokenLen {\n\t\t\t\t\tb.WriteString(sourceToken[0:sliceSize])\n\t\t\t\t} else { // too small to slice at the requested size: don't slice\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase Right:\n\t\t\t\tsourceToken := tokens[tr.dtokmftokindexesargs[i][0]]\n\t\t\t\tsourceTokenLen := len(sourceToken)\n\t\t\t\tsliceSize := int(tr.dtokmfintargs[i])\n\t\t\t\tif sliceSize > 0 && sliceSize < sourceTokenLen {\n\t\t\t\t\tb.WriteString(sourceToken[sourceTokenLen-sliceSize : sourceTokenLen])\n\t\t\t\t} else { // too small to slice at the requested size: don't slice\n\t\t\t\t\tb.WriteString(sourceToken)\n\t\t\t\t}\n\t\t\tcase Random:\n\t\t\t\tb.WriteString(tr.getRandomPartition(int(tr.dtokmfintargs[i])))\n\t\t\t}\n\t\t}\n\n\t\tif i < li {\n\t\t\tb.WriteByte(btsep)\n\t\t}\n\t}\n\n\t// We may have more source tokens available. This happens with \">\".\n\tif tr.dtoks[len(tr.dtoks)-1] == fwcs {\n\t\tfor sli, i := len(tokens)-1, len(tr.stoks)-1; i < len(tokens); i++ {\n\t\t\tb.WriteString(tokens[i])\n\t\t\tif i < sli {\n\t\t\t\tb.WriteByte(btsep)\n\t\t\t}\n\t\t}\n\t}\n\treturn b.String()\n}\n\n// Reverse a subjectTransform.\nfunc (tr *subjectTransform) reverse() *subjectTransform {\n\tif len(tr.dtokmftokindexesargs) == 0 {\n\t\trtr, _ := NewSubjectTransformStrict(tr.dest, tr.src)\n\t\treturn rtr\n\t}\n\t// If we are here we need to dynamically get the correct reverse\n\t// of this subjectTransform.\n\tnsrc, phs := transformUntokenize(tr.dest)\n\tvar nda []string\n\tfor _, token := range tr.stoks {\n\t\tif token == pwcs {\n\t\t\tif len(phs) == 0 {\n\t\t\t\t// TODO(dlc) - Should not happen\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tnda = append(nda, phs[0])\n\t\t\tphs = phs[1:]\n\t\t} else {\n\t\t\tnda = append(nda, token)\n\t\t}\n\t}\n\tndest := strings.Join(nda, tsep)\n\trtr, _ := NewSubjectTransformStrict(nsrc, ndest)\n\treturn rtr\n}\n\n// Will share relevant info regarding the subject.\n// Returns valid, tokens, num pwcs, has fwc.\nfunc subjectInfo(subject string) (bool, []string, int, bool) {\n\tif subject == \"\" {\n\t\treturn false, nil, 0, false\n\t}\n\tnpwcs := 0\n\tsfwc := false\n\ttokens := strings.Split(subject, tsep)\n\tfor _, t := range tokens {\n\t\tif len(t) == 0 || sfwc {\n\t\t\treturn false, nil, 0, false\n\t\t}\n\t\tif len(t) > 1 {\n\t\t\tcontinue\n\t\t}\n\t\tswitch t[0] {\n\t\tcase fwc:\n\t\t\tsfwc = true\n\t\tcase pwc:\n\t\t\tnpwcs++\n\t\t}\n\t}\n\treturn true, tokens, npwcs, sfwc\n}\n" }, { "path": "server/subject_transform_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"slices\"\n\t\"testing\"\n)\n\nfunc TestPlaceHolderIndex(t *testing.T) {\n\ttestString := \"$1\"\n\ttransformType, indexes, nbPartitions, _, err := indexPlaceHolders(testString)\n\tvar position int32\n\n\tif err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 1 || nbPartitions != -1 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{partition(10,1,2,3)}}\"\n\n\ttransformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{ Partition (10,1,2,3) }}\"\n\n\ttransformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != Partition || !reflect.DeepEqual(indexes, []int{1, 2, 3}) || nbPartitions != 10 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{wildcard(2)}}\"\n\ttransformType, indexes, nbPartitions, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != Wildcard || len(indexes) != 1 || indexes[0] != 2 || nbPartitions != -1 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{SplitFromLeft(2,1)}}\"\n\ttransformType, indexes, position, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != SplitFromLeft || len(indexes) != 1 || indexes[0] != 2 || position != 1 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{SplitFromRight(3,2)}}\"\n\ttransformType, indexes, position, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != SplitFromRight || len(indexes) != 1 || indexes[0] != 3 || position != 2 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{SliceFromLeft(2,2)}}\"\n\ttransformType, indexes, sliceSize, _, err := indexPlaceHolders(testString)\n\n\tif err != nil || transformType != SliceFromLeft || len(indexes) != 1 || indexes[0] != 2 || sliceSize != 2 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{Left(3,2)}}\"\n\ttransformType, indexes, position, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != Left || len(indexes) != 1 || indexes[0] != 3 || position != 2 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n\n\ttestString = \"{{Right(3,2)}}\"\n\ttransformType, indexes, position, _, err = indexPlaceHolders(testString)\n\n\tif err != nil || transformType != Right || len(indexes) != 1 || indexes[0] != 3 || position != 2 {\n\t\tt.Fatalf(\"Error parsing %s\", testString)\n\t}\n}\n\nfunc TestSubjectTransformHelpers(t *testing.T) {\n\tequals := func(a, b []string) bool {\n\t\tif len(a) != len(b) {\n\t\t\treturn false\n\t\t}\n\t\tfor i, v := range a {\n\t\t\tif v != b[i] {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\t}\n\n\tfilter, placeHolders := transformUntokenize(\"bar\")\n\tif filter != \"bar\" || len(placeHolders) != 0 {\n\t\tt.Fatalf(\"transformUntokenize for not returning expected result\")\n\t}\n\n\tfilter, placeHolders = transformUntokenize(\"foo.$2.$1\")\n\tif filter != \"foo.*.*\" || !equals(placeHolders, []string{\"$2\", \"$1\"}) {\n\t\tt.Fatalf(\"transformUntokenize for not returning expected result\")\n\t}\n\n\tfilter, placeHolders = transformUntokenize(\"foo.{{wildcard(2)}}.{{wildcard(1)}}\")\n\tif filter != \"foo.*.*\" || !equals(placeHolders, []string{\"{{wildcard(2)}}\", \"{{wildcard(1)}}\"}) {\n\t\tt.Fatalf(\"transformUntokenize for not returning expected result\")\n\t}\n\n\tnewReversibleTransform := func(src, dest string) *subjectTransform {\n\t\ttr, err := NewSubjectTransformStrict(src, dest)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error getting reversible transform: %s to %s\", src, dest)\n\t\t}\n\t\treturn tr\n\t}\n\n\ttr := newReversibleTransform(\"foo.*.*\", \"bar.$2.{{Wildcard(1)}}\")\n\tsubject := \"foo.b.a\"\n\ttransformed := tr.TransformSubject(subject)\n\treverse := tr.reverse()\n\tif reverse.TransformSubject(transformed) != subject {\n\t\tt.Fatal(\"Reversed transform subject not matching\")\n\t}\n}\n\nfunc TestSubjectTransforms(t *testing.T) {\n\tshouldErr := func(src, dest string, strict bool) {\n\t\tt.Helper()\n\t\tif _, err := NewSubjectTransformWithStrict(src, dest, strict); err != ErrBadSubject && !errors.Is(err, ErrInvalidMappingDestination) {\n\t\t\tt.Fatalf(\"Did not get an error for src=%q and dest=%q\", src, dest)\n\t\t}\n\t}\n\n\t// Must be valid subjects.\n\tshouldErr(\"foo..\", \"bar\", false)\n\n\t// Wildcards are allowed in src, but must be matched by token placements on the other side.\n\t// e.g. foo.* -> bar.$1.\n\t// Need to have as many pwcs as placements on other side\n\n\tshouldErr(\"foo.*\", \"bar.*\", false)\n\tshouldErr(\"foo.*\", \"bar.$2\", false) // Bad pwc token identifier\n\tshouldErr(\"foo.*\", \"bar.$1.>\", false) // fwcs have to match.\n\tshouldErr(\"foo.>\", \"bar.baz\", false) // fwcs have to match.\n\tshouldErr(\"foo.*.*\", \"bar.$2\", true) // Must place all pwcs.\n\tshouldErr(\"foo.*\", \"foo.$foo\", true) // invalid $ value\n\tshouldErr(\"foo.*\", \"bar.{{Partition(2,1)}}\", true) // can only use Wildcard function (and old-style $x) in import transform\n\tshouldErr(\"foo.*\", \"foo.{{wildcard(2)}}\", false) // Mapping function being passed an out of range wildcard index\n\tshouldErr(\"foo.*\", \"foo.{{unimplemented(1)}}\", false) // Mapping trying to use an unknown mapping function\n\tshouldErr(\"foo.*\", \"foo.{{partition()}}\", false) // Not enough arguments passed to the mapping function\n\tshouldErr(\"foo.*\", \"foo.{{random()}}\", false) // Not enough arguments passed to the random function\n\tshouldErr(\"foo.*\", \"foo.{{wildcard(foo)}}\", false) // Invalid argument passed to the mapping function\n\tshouldErr(\"foo.*\", \"foo.{{wildcard()}}\", false) // Not enough arguments passed to the mapping function\n\tshouldErr(\"foo.*\", \"foo.{{wildcard(1,2)}}\", false) // Too many arguments passed to the mapping function\n\tshouldErr(\"foo.*\", \"foo.{{ wildcard5) }}\", false) // Bad mapping function\n\tshouldErr(\"foo.*\", \"foo.{{splitLeft(2,2}}\", false) // arg out of range\n\tshouldErr(\"foo\", \"bla.{{wildcard(1)}}\", false) // arg out of range with no wildcard in the source\n\n\tshouldErr(\"foo.*\", fmt.Sprintf(\"foo.{{partition(%d)}}\", math.MaxInt32+1), false) // Larger than int32\n\tshouldErr(\"foo.*\", fmt.Sprintf(\"foo.{{random(%d)}}\", math.MaxInt32+1), false) // Larger than int32\n\n\tshouldBeOK := func(src, dest string, strict bool) *subjectTransform {\n\t\tt.Helper()\n\t\ttr, err := NewSubjectTransformWithStrict(src, dest, strict)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Got an error %v for src=%q and dest=%q\", err, src, dest)\n\t\t}\n\t\treturn tr\n\t}\n\n\tshouldBeOK(\"foo.*\", \"bar.{{Wildcard(1)}}\", true)\n\n\tshouldBeOK(\"foo.*.*\", \"bar.$2\", false) // don't have to use all pwcs.\n\tshouldBeOK(\"foo.*.*\", \"bar.{{wildcard(1)}}\", false) // don't have to use all pwcs.\n\tshouldBeOK(\"foo.*.*\", \"bar.{{partition(1)}}\", false)\n\tshouldBeOK(\"foo.*.*\", \"bar.{{random(5)}}\", false)\n\tshouldBeOK(\"foo\", \"bar\", false)\n\tshouldBeOK(\"foo.*.bar.*.baz\", \"req.$2.$1\", false)\n\tshouldBeOK(\"baz.>\", \"mybaz.>\", false)\n\tshouldBeOK(\"*\", \"{{splitfromleft(1,1)}}\", false)\n\tshouldBeOK(\"\", \"prefix.>\", false)\n\tshouldBeOK(\"*.*\", \"{{partition(10,1,2)}}\", false)\n\tshouldBeOK(\"foo.*.*\", \"foo.{{wildcard(1)}}.{{wildcard(2)}}.{{partition(5,1,2)}}\", false)\n\n\tshouldBeOK(\"foo.*\", fmt.Sprintf(\"foo.{{partition(%d)}}\", math.MaxInt32), false) // Exactly int32\n\tshouldBeOK(\"foo.*\", fmt.Sprintf(\"foo.{{random(%d)}}\", math.MaxInt32), false) // Exactly int32\n\tshouldBeOK(\"foo.bar\", fmt.Sprintf(\"foo.{{random(%d)}}\", math.MaxInt32), false) // Exactly int32\n\n\tshouldMatch := func(src, dest, sample string, expected ...string) {\n\t\tt.Helper()\n\t\ttr := shouldBeOK(src, dest, false)\n\t\tif tr != nil {\n\t\t\ts, err := tr.Match(sample)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Got an error %v when expecting a match for %q to %q\", err, sample, expected)\n\t\t\t}\n\t\t\tif !slices.Contains(expected, s) {\n\t\t\t\tt.Fatalf(\"Dest does not match what was expected. Got %q, expected %q\", s, expected)\n\t\t\t}\n\t\t}\n\t}\n\n\tshouldMatch(\"\", \"prefix.>\", \"foo\", \"prefix.foo\")\n\tshouldMatch(\"foo\", \"\", \"foo\", \"foo\")\n\tshouldMatch(\"foo\", \"bar\", \"foo\", \"bar\")\n\tshouldMatch(\"foo.*.bar.*.baz\", \"req.$2.$1\", \"foo.A.bar.B.baz\", \"req.B.A\")\n\tshouldMatch(\"foo.*.bar.*.baz\", \"req.{{wildcard(2)}}.{{wildcard(1)}}\", \"foo.A.bar.B.baz\", \"req.B.A\")\n\tshouldMatch(\"baz.>\", \"my.pre.>\", \"baz.1.2.3\", \"my.pre.1.2.3\")\n\tshouldMatch(\"baz.>\", \"foo.bar.>\", \"baz.1.2.3\", \"foo.bar.1.2.3\")\n\tshouldMatch(\"*\", \"foo.bar.$1\", \"foo\", \"foo.bar.foo\")\n\tshouldMatch(\"*\", \"{{splitfromleft(1,3)}}\", \"12345\", \"123.45\")\n\tshouldMatch(\"*\", \"{{SplitFromRight(1,3)}}\", \"12345\", \"12.345\")\n\tshouldMatch(\"*\", \"{{SliceFromLeft(1,3)}}\", \"1234567890\", \"123.456.789.0\")\n\tshouldMatch(\"*\", \"{{SliceFromRight(1,3)}}\", \"1234567890\", \"1.234.567.890\")\n\tshouldMatch(\"*\", \"{{split(1,-)}}\", \"-abc-def--ghi-\", \"abc.def.ghi\")\n\tshouldMatch(\"*\", \"{{split(1,-)}}\", \"abc-def--ghi-\", \"abc.def.ghi\")\n\tshouldMatch(\"*.*\", \"{{split(2,-)}}.{{splitfromleft(1,2)}}\", \"foo.-abc-def--ghij-\", \"abc.def.ghij.fo.o\") // combo + checks split for multiple instance of deliminator and deliminator being at the start or end\n\tshouldMatch(\"*\", \"{{right(1,1)}}\", \"1234\", \"4\")\n\tshouldMatch(\"*\", \"{{right(1,3)}}\", \"1234\", \"234\")\n\tshouldMatch(\"*\", \"{{right(1,6)}}\", \"1234\", \"1234\")\n\tshouldMatch(\"*\", \"{{left(1,1)}}\", \"1234\", \"1\")\n\tshouldMatch(\"*\", \"{{left(1,3)}}\", \"1234\", \"123\")\n\tshouldMatch(\"*\", \"{{left(1,6)}}\", \"1234\", \"1234\")\n\tshouldMatch(\"*\", \"bar.{{partition(0)}}\", \"baz\", \"bar.0\")\n\tshouldMatch(\"*\", \"bar.{{partition(10, 0)}}\", \"foo\", \"bar.3\")\n\tshouldMatch(\"*.*\", \"bar.{{partition(10)}}\", \"foo.bar\", \"bar.6\")\n\tshouldMatch(\"*\", \"bar.{{partition(10)}}\", \"foo\", \"bar.3\")\n\tshouldMatch(\"*\", \"bar.{{partition(10)}}\", \"baz\", \"bar.0\")\n\tshouldMatch(\"*\", \"bar.{{partition(10)}}\", \"qux\", \"bar.9\")\n\tshouldMatch(\"*\", \"bar.{{random(0)}}\", \"qux\", \"bar.0\")\n\tfor range 100 {\n\t\tshouldMatch(\"*\", \"bar.{{random(6)}}\", \"qux\", \"bar.0\", \"bar.1\", \"bar.2\", \"bar.3\", \"bar.4\", \"bar.5\")\n\t}\n\tshouldBeOK(\"foo.bar\", \"baz.{{partition(10)}}\", false)\n\tshouldMatch(\"foo.bar\", \"baz.{{partition(10)}}\", \"foo.bar\", \"baz.6\")\n\tshouldMatch(\"foo.baz\", \"qux.{{partition(10)}}\", \"foo.baz\", \"qux.4\")\n\tshouldMatch(\"test.subject\", \"result.{{partition(5)}}\", \"test.subject\", \"result.0\")\n}\n\nfunc TestSubjectTransformDoesntPanicTransformingMissingToken(t *testing.T) {\n\tdefer func() {\n\t\tp := recover()\n\t\trequire_True(t, p == nil)\n\t}()\n\n\ttr, err := NewSubjectTransform(\"foo.*\", \"one.two.{{wildcard(1)}}\")\n\trequire_NoError(t, err)\n\trequire_Equal(t, tr.TransformTokenizedSubject([]string{\"foo\"}), \"one.two.\")\n}\n" }, { "path": "server/sublist.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"iter\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"unicode/utf8\"\n)\n\n// Sublist is a routing mechanism to handle subject distribution and\n// provides a facility to match subjects from published messages to\n// interested subscribers. Subscribers can have wildcard subjects to\n// match multiple published subjects.\n\n// Common byte variables for wildcards and token separator.\nconst (\n\tpwc = '*'\n\tpwcs = \"*\"\n\tfwc = '>'\n\tfwcs = \">\"\n\ttsep = \".\"\n\tbtsep = '.'\n)\n\n// Sublist related errors\nvar (\n\tErrInvalidSubject = errors.New(\"sublist: invalid subject\")\n\tErrNotFound = errors.New(\"sublist: no matches found\")\n\tErrNilChan = errors.New(\"sublist: nil channel\")\n\tErrAlreadyRegistered = errors.New(\"sublist: notification already registered\")\n)\n\nconst (\n\t// cacheMax is used to bound limit the frontend cache\n\tslCacheMax = 1024\n\t// If we run a sweeper we will drain to this count.\n\tslCacheSweep = 256\n\t// plistMin is our lower bounds to create a fast plist for Match.\n\tplistMin = 256\n)\n\n// SublistResult is a result structure better optimized for queue subs.\ntype SublistResult struct {\n\tpsubs []*subscription\n\tqsubs [][]*subscription // don't make this a map, too expensive to iterate\n}\n\n// A Sublist stores and efficiently retrieves subscriptions.\ntype Sublist struct {\n\tsync.RWMutex\n\tgenid uint64\n\tmatches uint64\n\tcacheHits uint64\n\tinserts uint64\n\tremoves uint64\n\troot *level\n\tcache map[string]*SublistResult\n\tccSweep int32\n\tnotify *notifyMaps\n\tcount uint32\n}\n\n// notifyMaps holds maps of arrays of channels for notifications\n// on a change of interest.\ntype notifyMaps struct {\n\tinsert map[string][]chan<- bool\n\tremove map[string][]chan<- bool\n}\n\n// A node contains subscriptions and a pointer to the next level.\ntype node struct {\n\tnext *level\n\tpsubs map[*subscription]struct{}\n\tqsubs map[string]map[*subscription]struct{}\n\tplist []*subscription\n}\n\n// A level represents a group of nodes and special pointers to\n// wildcard nodes.\ntype level struct {\n\tnodes map[string]*node\n\tpwc, fwc *node\n}\n\n// Create a new default node.\nfunc newNode() *node {\n\treturn &node{psubs: make(map[*subscription]struct{})}\n}\n\n// Create a new default level.\nfunc newLevel() *level {\n\treturn &level{nodes: make(map[string]*node)}\n}\n\n// In general caching is recommended however in some extreme cases where\n// interest changes are high, suppressing the cache can help.\n// https://github.com/nats-io/nats-server/issues/941\n// FIXME(dlc) - should be more dynamic at some point based on cache thrashing.\n\n// NewSublist will create a default sublist with caching enabled per the flag.\nfunc NewSublist(enableCache bool) *Sublist {\n\tif enableCache {\n\t\treturn &Sublist{root: newLevel(), cache: make(map[string]*SublistResult)}\n\t}\n\treturn &Sublist{root: newLevel()}\n}\n\n// NewSublistWithCache will create a default sublist with caching enabled.\nfunc NewSublistWithCache() *Sublist {\n\treturn NewSublist(true)\n}\n\n// NewSublistNoCache will create a default sublist with caching disabled.\nfunc NewSublistNoCache() *Sublist {\n\treturn NewSublist(false)\n}\n\n// CacheEnabled returns whether or not caching is enabled for this sublist.\nfunc (s *Sublist) CacheEnabled() bool {\n\ts.RLock()\n\tenabled := s.cache != nil\n\ts.RUnlock()\n\treturn enabled\n}\n\n// RegisterNotification will register for notifications when interest for the given\n// subject changes. The subject must be a literal publish type subject.\n// The notification is true for when the first interest for a subject is inserted,\n// and false when all interest in the subject is removed. Note that this interest\n// needs to be exact and that wildcards will not trigger the notifications. The sublist\n// will not block when trying to send the notification. Its up to the caller to make\n// sure the channel send will not block.\nfunc (s *Sublist) RegisterNotification(subject string, notify chan<- bool) error {\n\treturn s.registerNotification(subject, _EMPTY_, notify)\n}\n\nfunc (s *Sublist) RegisterQueueNotification(subject, queue string, notify chan<- bool) error {\n\treturn s.registerNotification(subject, queue, notify)\n}\n\nfunc (s *Sublist) registerNotification(subject, queue string, notify chan<- bool) error {\n\tif subjectHasWildcard(subject) {\n\t\treturn ErrInvalidSubject\n\t}\n\tif notify == nil {\n\t\treturn ErrNilChan\n\t}\n\n\tvar hasInterest bool\n\tr := s.Match(subject)\n\n\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\tif queue == _EMPTY_ {\n\t\t\tfor _, sub := range r.psubs {\n\t\t\t\tif string(sub.subject) == subject {\n\t\t\t\t\thasInterest = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tfor _, qsub := range r.qsubs {\n\t\t\t\tqs := qsub[0]\n\t\t\t\tif string(qs.subject) == subject && string(qs.queue) == queue {\n\t\t\t\t\thasInterest = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tkey := keyFromSubjectAndQueue(subject, queue)\n\tvar err error\n\n\ts.Lock()\n\tif s.notify == nil {\n\t\ts.notify = ¬ifyMaps{\n\t\t\tinsert: make(map[string][]chan<- bool),\n\t\t\tremove: make(map[string][]chan<- bool),\n\t\t}\n\t}\n\t// Check which list to add us to.\n\tif hasInterest {\n\t\terr = s.addRemoveNotify(key, notify)\n\t} else {\n\t\terr = s.addInsertNotify(key, notify)\n\t}\n\ts.Unlock()\n\n\tif err == nil {\n\t\tsendNotification(notify, hasInterest)\n\t}\n\treturn err\n}\n\n// Lock should be held.\nfunc chkAndRemove(key string, notify chan<- bool, ms map[string][]chan<- bool) bool {\n\tchs := ms[key]\n\tfor i, ch := range chs {\n\t\tif ch == notify {\n\t\t\tchs[i] = chs[len(chs)-1]\n\t\t\tchs = chs[:len(chs)-1]\n\t\t\tif len(chs) == 0 {\n\t\t\t\tdelete(ms, key)\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (s *Sublist) ClearNotification(subject string, notify chan<- bool) bool {\n\treturn s.clearNotification(subject, _EMPTY_, notify)\n}\n\nfunc (s *Sublist) ClearQueueNotification(subject, queue string, notify chan<- bool) bool {\n\treturn s.clearNotification(subject, queue, notify)\n}\n\nfunc (s *Sublist) clearNotification(subject, queue string, notify chan<- bool) bool {\n\ts.Lock()\n\tif s.notify == nil {\n\t\ts.Unlock()\n\t\treturn false\n\t}\n\tkey := keyFromSubjectAndQueue(subject, queue)\n\t// Check both, start with remove.\n\tdidRemove := chkAndRemove(key, notify, s.notify.remove)\n\tdidRemove = didRemove || chkAndRemove(key, notify, s.notify.insert)\n\t// Check if everything is gone\n\tif len(s.notify.remove)+len(s.notify.insert) == 0 {\n\t\ts.notify = nil\n\t}\n\ts.Unlock()\n\treturn didRemove\n}\n\nfunc sendNotification(ch chan<- bool, hasInterest bool) {\n\tselect {\n\tcase ch <- hasInterest:\n\tdefault:\n\t}\n}\n\n// Add a new channel for notification in insert map.\n// Write lock should be held.\nfunc (s *Sublist) addInsertNotify(subject string, notify chan<- bool) error {\n\treturn s.addNotify(s.notify.insert, subject, notify)\n}\n\n// Add a new channel for notification in removal map.\n// Write lock should be held.\nfunc (s *Sublist) addRemoveNotify(subject string, notify chan<- bool) error {\n\treturn s.addNotify(s.notify.remove, subject, notify)\n}\n\n// Add a new channel for notification.\n// Write lock should be held.\nfunc (s *Sublist) addNotify(m map[string][]chan<- bool, subject string, notify chan<- bool) error {\n\tchs := m[subject]\n\tif len(chs) > 0 {\n\t\t// Check to see if this chan is already registered.\n\t\tfor _, ch := range chs {\n\t\t\tif ch == notify {\n\t\t\t\treturn ErrAlreadyRegistered\n\t\t\t}\n\t\t}\n\t}\n\n\tm[subject] = append(chs, notify)\n\treturn nil\n}\n\n// To generate a key from subject and queue. We just add spc.\nfunc keyFromSubjectAndQueue(subject, queue string) string {\n\tif len(queue) == 0 {\n\t\treturn subject\n\t}\n\tvar sb strings.Builder\n\tsb.WriteString(subject)\n\tsb.WriteString(\" \")\n\tsb.WriteString(queue)\n\treturn sb.String()\n}\n\n// chkForInsertNotification will check to see if we need to notify on this subject.\n// Write lock should be held.\nfunc (s *Sublist) chkForInsertNotification(subject, queue string) {\n\tkey := keyFromSubjectAndQueue(subject, queue)\n\n\t// All notify subjects are also literal so just do a hash lookup here.\n\tif chs := s.notify.insert[key]; len(chs) > 0 {\n\t\tfor _, ch := range chs {\n\t\t\tsendNotification(ch, true)\n\t\t}\n\t\t// Move from the insert map to the remove map.\n\t\ts.notify.remove[key] = append(s.notify.remove[key], chs...)\n\t\tdelete(s.notify.insert, key)\n\t}\n}\n\n// chkForRemoveNotification will check to see if we need to notify on this subject.\n// Write lock should be held.\nfunc (s *Sublist) chkForRemoveNotification(subject, queue string) {\n\tkey := keyFromSubjectAndQueue(subject, queue)\n\tif chs := s.notify.remove[key]; len(chs) > 0 {\n\t\t// We need to always check that we have no interest anymore.\n\t\tvar hasInterest bool\n\t\tr := s.matchNoLock(subject)\n\n\t\tif len(r.psubs)+len(r.qsubs) > 0 {\n\t\t\tif queue == _EMPTY_ {\n\t\t\t\tfor _, sub := range r.psubs {\n\t\t\t\t\tif string(sub.subject) == subject {\n\t\t\t\t\t\thasInterest = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor _, qsub := range r.qsubs {\n\t\t\t\t\tqs := qsub[0]\n\t\t\t\t\tif string(qs.subject) == subject && string(qs.queue) == queue {\n\t\t\t\t\t\thasInterest = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif !hasInterest {\n\t\t\tfor _, ch := range chs {\n\t\t\t\tsendNotification(ch, false)\n\t\t\t}\n\t\t\t// Move from the remove map to the insert map.\n\t\t\ts.notify.insert[key] = append(s.notify.insert[key], chs...)\n\t\t\tdelete(s.notify.remove, key)\n\t\t}\n\t}\n}\n\n// Insert adds a subscription into the sublist\nfunc (s *Sublist) Insert(sub *subscription) error {\n\t// copy the subject since we hold this and this might be part of a large byte slice.\n\tsubject := string(sub.subject)\n\n\ts.Lock()\n\n\tvar sfwc, haswc, isnew bool\n\tvar n *node\n\tl := s.root\n\n\tfor t := range strings.SplitSeq(subject, tsep) {\n\t\tlt := len(t)\n\t\tif lt == 0 || sfwc {\n\t\t\ts.Unlock()\n\t\t\treturn ErrInvalidSubject\n\t\t}\n\n\t\tif lt > 1 {\n\t\t\tn = l.nodes[t]\n\t\t} else {\n\t\t\tswitch t[0] {\n\t\t\tcase pwc:\n\t\t\t\tn = l.pwc\n\t\t\t\thaswc = true\n\t\t\tcase fwc:\n\t\t\t\tn = l.fwc\n\t\t\t\thaswc, sfwc = true, true\n\t\t\tdefault:\n\t\t\t\tn = l.nodes[t]\n\t\t\t}\n\t\t}\n\t\tif n == nil {\n\t\t\tn = newNode()\n\t\t\tif lt > 1 {\n\t\t\t\tl.nodes[t] = n\n\t\t\t} else {\n\t\t\t\tswitch t[0] {\n\t\t\t\tcase pwc:\n\t\t\t\t\tl.pwc = n\n\t\t\t\tcase fwc:\n\t\t\t\t\tl.fwc = n\n\t\t\t\tdefault:\n\t\t\t\t\tl.nodes[t] = n\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif n.next == nil {\n\t\t\tn.next = newLevel()\n\t\t}\n\t\tl = n.next\n\t}\n\tif sub.queue == nil {\n\t\tn.psubs[sub] = struct{}{}\n\t\tisnew = len(n.psubs) == 1\n\t\tif n.plist != nil {\n\t\t\tn.plist = append(n.plist, sub)\n\t\t} else if len(n.psubs) > plistMin {\n\t\t\tn.plist = make([]*subscription, 0, len(n.psubs))\n\t\t\t// Populate\n\t\t\tfor psub := range n.psubs {\n\t\t\t\tn.plist = append(n.plist, psub)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif n.qsubs == nil {\n\t\t\tn.qsubs = make(map[string]map[*subscription]struct{})\n\t\t}\n\t\tqname := string(sub.queue)\n\t\t// This is a queue subscription\n\t\tsubs, ok := n.qsubs[qname]\n\t\tif !ok {\n\t\t\tsubs = make(map[*subscription]struct{})\n\t\t\tn.qsubs[qname] = subs\n\t\t\tisnew = true\n\t\t}\n\t\tsubs[sub] = struct{}{}\n\t}\n\n\ts.count++\n\ts.inserts++\n\n\ts.addToCache(subject, sub)\n\tatomic.AddUint64(&s.genid, 1)\n\n\tif s.notify != nil && isnew && !haswc && len(s.notify.insert) > 0 {\n\t\ts.chkForInsertNotification(subject, string(sub.queue))\n\t}\n\ts.Unlock()\n\n\treturn nil\n}\n\n// Deep copy\nfunc copyResult(r *SublistResult) *SublistResult {\n\tnr := &SublistResult{}\n\tnr.psubs = append([]*subscription(nil), r.psubs...)\n\tfor _, qr := range r.qsubs {\n\t\tnqr := append([]*subscription(nil), qr...)\n\t\tnr.qsubs = append(nr.qsubs, nqr)\n\t}\n\treturn nr\n}\n\n// Adds a new sub to an existing result.\nfunc (r *SublistResult) addSubToResult(sub *subscription) *SublistResult {\n\t// Copy since others may have a reference.\n\tnr := copyResult(r)\n\tif sub.queue == nil {\n\t\tnr.psubs = append(nr.psubs, sub)\n\t} else {\n\t\tif i := findQSlot(sub.queue, nr.qsubs); i >= 0 {\n\t\t\tnr.qsubs[i] = append(nr.qsubs[i], sub)\n\t\t} else {\n\t\t\tnr.qsubs = append(nr.qsubs, []*subscription{sub})\n\t\t}\n\t}\n\treturn nr\n}\n\n// addToCache will add the new entry to the existing cache\n// entries if needed. Assumes write lock is held.\n// Assumes write lock is held.\nfunc (s *Sublist) addToCache(subject string, sub *subscription) {\n\tif s.cache == nil {\n\t\treturn\n\t}\n\t// If literal we can direct match.\n\tif subjectIsLiteral(subject) {\n\t\tif r := s.cache[subject]; r != nil {\n\t\t\ts.cache[subject] = r.addSubToResult(sub)\n\t\t}\n\t\treturn\n\t}\n\tfor key, r := range s.cache {\n\t\tif matchLiteral(key, subject) {\n\t\t\ts.cache[key] = r.addSubToResult(sub)\n\t\t}\n\t}\n}\n\n// removeFromCache will remove the sub from any active cache entries.\n// Assumes write lock is held.\nfunc (s *Sublist) removeFromCache(subject string) {\n\tif s.cache == nil {\n\t\treturn\n\t}\n\t// If literal we can direct match.\n\tif subjectIsLiteral(subject) {\n\t\tdelete(s.cache, subject)\n\t\treturn\n\t}\n\t// Wildcard here.\n\tfor key := range s.cache {\n\t\tif matchLiteral(key, subject) {\n\t\t\tdelete(s.cache, key)\n\t\t}\n\t}\n}\n\n// a place holder for an empty result.\nvar emptyResult = &SublistResult{}\n\n// Match will match all entries to the literal subject.\n// It will return a set of results for both normal and queue subscribers.\nfunc (s *Sublist) Match(subject string) *SublistResult {\n\treturn s.match(subject, true, false)\n}\n\n// MatchBytes will match all entries to the literal subject.\n// It will return a set of results for both normal and queue subscribers.\nfunc (s *Sublist) MatchBytes(subject []byte) *SublistResult {\n\treturn s.match(bytesToString(subject), true, true)\n}\n\n// HasInterest will return whether or not there is any interest in the subject.\n// In cases where more detail is not required, this may be faster than Match.\nfunc (s *Sublist) HasInterest(subject string) bool {\n\treturn s.hasInterest(subject, true, nil, nil)\n}\n\n// NumInterest will return the number of subs/qsubs interested in the subject.\n// In cases where more detail is not required, this may be faster than Match.\nfunc (s *Sublist) NumInterest(subject string) (np, nq int) {\n\ts.hasInterest(subject, true, &np, &nq)\n\treturn\n}\n\nfunc (s *Sublist) matchNoLock(subject string) *SublistResult {\n\treturn s.match(subject, false, false)\n}\n\nfunc (s *Sublist) match(subject string, doLock bool, doCopyOnCache bool) *SublistResult {\n\tatomic.AddUint64(&s.matches, 1)\n\n\t// Check cache first.\n\tif doLock {\n\t\ts.RLock()\n\t}\n\tcacheEnabled := s.cache != nil\n\tr, ok := s.cache[subject]\n\tif doLock {\n\t\ts.RUnlock()\n\t}\n\tif ok {\n\t\tatomic.AddUint64(&s.cacheHits, 1)\n\t\treturn r\n\t}\n\n\ttsa := [32]string{}\n\ttokens := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif i-start == 0 {\n\t\t\t\treturn emptyResult\n\t\t\t}\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\tif start >= len(subject) {\n\t\treturn emptyResult\n\t}\n\ttokens = append(tokens, subject[start:])\n\n\t// FIXME(dlc) - Make shared pool between sublist and client readLoop?\n\tresult := &SublistResult{}\n\n\t// Get result from the main structure and place into the shared cache.\n\t// Hold the read lock to avoid race between match and store.\n\tvar n int\n\n\tif doLock {\n\t\tif cacheEnabled {\n\t\t\ts.Lock()\n\t\t} else {\n\t\t\ts.RLock()\n\t\t}\n\t}\n\n\tmatchLevel(s.root, tokens, result)\n\t// Check for empty result.\n\tif len(result.psubs) == 0 && len(result.qsubs) == 0 {\n\t\tresult = emptyResult\n\t}\n\tif cacheEnabled {\n\t\tif doCopyOnCache {\n\t\t\tsubject = copyString(subject)\n\t\t}\n\t\ts.cache[subject] = result\n\t\tn = len(s.cache)\n\t}\n\tif doLock {\n\t\tif cacheEnabled {\n\t\t\ts.Unlock()\n\t\t} else {\n\t\t\ts.RUnlock()\n\t\t}\n\t}\n\n\t// Reduce the cache count if we have exceeded our set maximum.\n\tif cacheEnabled && n > slCacheMax && atomic.CompareAndSwapInt32(&s.ccSweep, 0, 1) {\n\t\tgo s.reduceCacheCount()\n\t}\n\n\treturn result\n}\n\nfunc (s *Sublist) hasInterest(subject string, doLock bool, np, nq *int) bool {\n\t// Check cache first.\n\tif doLock {\n\t\ts.RLock()\n\t}\n\tvar matched bool\n\tif s.cache != nil {\n\t\tif r, ok := s.cache[subject]; ok {\n\t\t\tif np != nil && nq != nil {\n\t\t\t\t*np += len(r.psubs)\n\t\t\t\tfor _, qsub := range r.qsubs {\n\t\t\t\t\t*nq += len(qsub)\n\t\t\t\t}\n\t\t\t}\n\t\t\tmatched = len(r.psubs)+len(r.qsubs) > 0\n\t\t}\n\t}\n\tif doLock {\n\t\ts.RUnlock()\n\t}\n\tif matched {\n\t\tatomic.AddUint64(&s.cacheHits, 1)\n\t\treturn true\n\t}\n\n\ttsa := [32]string{}\n\ttokens := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif i-start == 0 {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\tif start >= len(subject) {\n\t\treturn false\n\t}\n\ttokens = append(tokens, subject[start:])\n\n\tif doLock {\n\t\ts.RLock()\n\t\tdefer s.RUnlock()\n\t}\n\treturn matchLevelForAny(s.root, tokens, np, nq)\n}\n\n// Remove entries in the cache until we are under the maximum.\n// TODO(dlc) this could be smarter now that its not inline.\nfunc (s *Sublist) reduceCacheCount() {\n\tdefer atomic.StoreInt32(&s.ccSweep, 0)\n\t// If we are over the cache limit randomly drop until under the limit.\n\ts.Lock()\n\tfor key := range s.cache {\n\t\tdelete(s.cache, key)\n\t\tif len(s.cache) <= slCacheSweep {\n\t\t\tbreak\n\t\t}\n\t}\n\ts.Unlock()\n}\n\n// Helper function for auto-expanding remote qsubs.\nfunc isRemoteQSub(sub *subscription) bool {\n\treturn sub != nil && sub.queue != nil && sub.client != nil && (sub.client.kind == ROUTER || sub.client.kind == LEAF)\n}\n\n// UpdateRemoteQSub should be called when we update the weight of an existing\n// remote queue sub.\nfunc (s *Sublist) UpdateRemoteQSub(sub *subscription) {\n\t// We could search to make sure we find it, but probably not worth\n\t// it unless we are thrashing the cache. Just remove from our L2 and update\n\t// the genid so L1 will be flushed.\n\ts.Lock()\n\ts.removeFromCache(string(sub.subject))\n\tatomic.AddUint64(&s.genid, 1)\n\ts.Unlock()\n}\n\n// This will add in a node's results to the total results.\nfunc addNodeToResults(n *node, results *SublistResult) {\n\t// Normal subscriptions\n\tif n.plist != nil {\n\t\tresults.psubs = append(results.psubs, n.plist...)\n\t} else {\n\t\tfor psub := range n.psubs {\n\t\t\tresults.psubs = append(results.psubs, psub)\n\t\t}\n\t}\n\t// Queue subscriptions\n\tfor qname, qr := range n.qsubs {\n\t\tif len(qr) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\t// Need to find matching list in results\n\t\tvar i int\n\t\tif i = findQSlot([]byte(qname), results.qsubs); i < 0 {\n\t\t\ti = len(results.qsubs)\n\t\t\tnqsub := make([]*subscription, 0, len(qr))\n\t\t\tresults.qsubs = append(results.qsubs, nqsub)\n\t\t}\n\t\tfor sub := range qr {\n\t\t\tif isRemoteQSub(sub) {\n\t\t\t\tns := atomic.LoadInt32(&sub.qw)\n\t\t\t\t// Shadow these subscriptions\n\t\t\t\tfor n := 0; n < int(ns); n++ {\n\t\t\t\t\tresults.qsubs[i] = append(results.qsubs[i], sub)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tresults.qsubs[i] = append(results.qsubs[i], sub)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// We do not use a map here since we want iteration to be past when\n// processing publishes in L1 on client. So we need to walk sequentially\n// for now. Keep an eye on this in case we start getting large number of\n// different queue subscribers for the same subject.\nfunc findQSlot(queue []byte, qsl [][]*subscription) int {\n\tif queue == nil {\n\t\treturn -1\n\t}\n\tfor i, qr := range qsl {\n\t\tif len(qr) > 0 && bytes.Equal(queue, qr[0].queue) {\n\t\t\treturn i\n\t\t}\n\t}\n\treturn -1\n}\n\n// matchLevel is used to recursively descend into the trie.\nfunc matchLevel(l *level, toks []string, results *SublistResult) {\n\tvar pwc, n *node\n\tfor i, t := range toks {\n\t\tif l == nil {\n\t\t\treturn\n\t\t}\n\t\tif l.fwc != nil {\n\t\t\taddNodeToResults(l.fwc, results)\n\t\t}\n\t\tif pwc = l.pwc; pwc != nil {\n\t\t\tmatchLevel(pwc.next, toks[i+1:], results)\n\t\t}\n\t\tn = l.nodes[t]\n\t\tif n != nil {\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\tif n != nil {\n\t\taddNodeToResults(n, results)\n\t}\n\tif pwc != nil {\n\t\taddNodeToResults(pwc, results)\n\t}\n}\n\nfunc matchLevelForAny(l *level, toks []string, np, nq *int) bool {\n\tvar pwc, n *node\n\tfor i, t := range toks {\n\t\tif l == nil {\n\t\t\treturn false\n\t\t}\n\t\tif l.fwc != nil {\n\t\t\tif np != nil && nq != nil {\n\t\t\t\t*np += len(l.fwc.psubs)\n\t\t\t\tfor _, qsub := range l.fwc.qsubs {\n\t\t\t\t\t*nq += len(qsub)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t\tif pwc = l.pwc; pwc != nil {\n\t\t\tif match := matchLevelForAny(pwc.next, toks[i+1:], np, nq); match {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tn = l.nodes[t]\n\t\tif n != nil {\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\tif n != nil {\n\t\tif np != nil && nq != nil {\n\t\t\t*np += len(n.psubs)\n\t\t\tfor _, qsub := range n.qsubs {\n\t\t\t\t*nq += len(qsub)\n\t\t\t}\n\t\t}\n\t\tif len(n.plist) > 0 || len(n.psubs) > 0 || len(n.qsubs) > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\tif pwc != nil {\n\t\tif np != nil && nq != nil {\n\t\t\t*np += len(pwc.psubs)\n\t\t\tfor _, qsub := range pwc.qsubs {\n\t\t\t\t*nq += len(qsub)\n\t\t\t}\n\t\t}\n\t\treturn len(pwc.plist) > 0 || len(pwc.psubs) > 0 || len(pwc.qsubs) > 0\n\t}\n\treturn false\n}\n\n// lnt is used to track descent into levels for a removal for pruning.\ntype lnt struct {\n\tl *level\n\tn *node\n\tt string\n}\n\n// Raw low level remove, can do batches with lock held outside.\nfunc (s *Sublist) remove(sub *subscription, shouldLock bool, doCacheUpdates bool) error {\n\tsubject := string(sub.subject)\n\n\tif shouldLock {\n\t\ts.Lock()\n\t\tdefer s.Unlock()\n\t}\n\n\tvar sfwc, haswc bool\n\tvar n *node\n\tl := s.root\n\n\t// Track levels for pruning\n\tvar lnts [32]lnt\n\tlevels := lnts[:0]\n\n\tfor t := range strings.SplitSeq(subject, tsep) {\n\t\tlt := len(t)\n\t\tif lt == 0 || sfwc {\n\t\t\treturn ErrInvalidSubject\n\t\t}\n\t\tif l == nil {\n\t\t\treturn ErrNotFound\n\t\t}\n\t\tif lt > 1 {\n\t\t\tn = l.nodes[t]\n\t\t} else {\n\t\t\tswitch t[0] {\n\t\t\tcase pwc:\n\t\t\t\tn = l.pwc\n\t\t\t\thaswc = true\n\t\t\tcase fwc:\n\t\t\t\tn = l.fwc\n\t\t\t\thaswc, sfwc = true, true\n\t\t\tdefault:\n\t\t\t\tn = l.nodes[t]\n\t\t\t}\n\t\t}\n\t\tif n != nil {\n\t\t\tlevels = append(levels, lnt{l, n, t})\n\t\t\tl = n.next\n\t\t} else {\n\t\t\tl = nil\n\t\t}\n\t}\n\tremoved, last := s.removeFromNode(n, sub)\n\tif !removed {\n\t\treturn ErrNotFound\n\t}\n\n\ts.count--\n\ts.removes++\n\n\tfor i := len(levels) - 1; i >= 0; i-- {\n\t\tl, n, t := levels[i].l, levels[i].n, levels[i].t\n\t\tif n.isEmpty() {\n\t\t\tl.pruneNode(n, t)\n\t\t}\n\t}\n\tif doCacheUpdates {\n\t\ts.removeFromCache(subject)\n\t\tatomic.AddUint64(&s.genid, 1)\n\t}\n\n\tif s.notify != nil && last && !haswc && len(s.notify.remove) > 0 {\n\t\ts.chkForRemoveNotification(subject, string(sub.queue))\n\t}\n\n\treturn nil\n}\n\n// Remove will remove a subscription.\nfunc (s *Sublist) Remove(sub *subscription) error {\n\treturn s.remove(sub, true, true)\n}\n\n// RemoveBatch will remove a list of subscriptions.\nfunc (s *Sublist) RemoveBatch(subs []*subscription) error {\n\tif len(subs) == 0 {\n\t\treturn nil\n\t}\n\n\ts.Lock()\n\tdefer s.Unlock()\n\n\t// TODO(dlc) - We could try to be smarter here for a client going away but the account\n\t// has a large number of subscriptions compared to this client. Quick and dirty testing\n\t// though said just disabling all the time best for now.\n\n\t// Turn off our cache if enabled.\n\twasEnabled := s.cache != nil\n\ts.cache = nil\n\t// We will try to remove all subscriptions but will report the first that caused\n\t// an error. In other words, we don't bail out at the first error which would\n\t// possibly leave a bunch of subscriptions that could have been removed.\n\tvar err error\n\tfor _, sub := range subs {\n\t\tif lerr := s.remove(sub, false, false); lerr != nil && err == nil {\n\t\t\terr = lerr\n\t\t}\n\t}\n\t// Turn caching back on here.\n\tatomic.AddUint64(&s.genid, 1)\n\tif wasEnabled {\n\t\ts.cache = make(map[string]*SublistResult)\n\t}\n\treturn err\n}\n\n// pruneNode is used to prune an empty node from the tree.\nfunc (l *level) pruneNode(n *node, t string) {\n\tif n == nil {\n\t\treturn\n\t}\n\tif n == l.fwc {\n\t\tl.fwc = nil\n\t} else if n == l.pwc {\n\t\tl.pwc = nil\n\t} else {\n\t\tdelete(l.nodes, t)\n\t}\n}\n\n// isEmpty will test if the node has any entries. Used\n// in pruning.\nfunc (n *node) isEmpty() bool {\n\tif len(n.psubs) == 0 && len(n.qsubs) == 0 {\n\t\tif n.next == nil || n.next.numNodes() == 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Return the number of nodes for the given level.\nfunc (l *level) numNodes() int {\n\tif l == nil {\n\t\treturn 0\n\t}\n\tnum := len(l.nodes)\n\tif l.pwc != nil {\n\t\tnum++\n\t}\n\tif l.fwc != nil {\n\t\tnum++\n\t}\n\treturn num\n}\n\n// Remove the sub for the given node.\nfunc (s *Sublist) removeFromNode(n *node, sub *subscription) (found, last bool) {\n\tif n == nil {\n\t\treturn false, true\n\t}\n\tif sub.queue == nil {\n\t\t_, found = n.psubs[sub]\n\t\tdelete(n.psubs, sub)\n\t\tif found && n.plist != nil {\n\t\t\t// This will brute force remove the plist to perform\n\t\t\t// correct behavior. Will get re-populated on a call\n\t\t\t// to Match as needed.\n\t\t\tn.plist = nil\n\t\t}\n\t\treturn found, len(n.psubs) == 0\n\t}\n\n\t// We have a queue group subscription here\n\tqsub := n.qsubs[string(sub.queue)]\n\t_, found = qsub[sub]\n\tdelete(qsub, sub)\n\tif len(qsub) == 0 {\n\t\t// This is the last queue subscription interest when len(qsub) == 0, not\n\t\t// when n.qsubs is empty.\n\t\tlast = true\n\t\tdelete(n.qsubs, string(sub.queue))\n\t}\n\treturn found, last\n}\n\n// Count returns the number of subscriptions.\nfunc (s *Sublist) Count() uint32 {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.count\n}\n\n// CacheCount returns the number of result sets in the cache.\nfunc (s *Sublist) CacheCount() int {\n\ts.RLock()\n\tcc := len(s.cache)\n\ts.RUnlock()\n\treturn cc\n}\n\n// SublistStats are public stats for the sublist\ntype SublistStats struct {\n\tNumSubs uint32 `json:\"num_subscriptions\"`\n\tNumCache uint32 `json:\"num_cache\"`\n\tNumInserts uint64 `json:\"num_inserts\"`\n\tNumRemoves uint64 `json:\"num_removes\"`\n\tNumMatches uint64 `json:\"num_matches\"`\n\tCacheHitRate float64 `json:\"cache_hit_rate\"`\n\tMaxFanout uint32 `json:\"max_fanout\"`\n\tAvgFanout float64 `json:\"avg_fanout\"`\n\ttotFanout int\n\tcacheCnt int\n\tcacheHits uint64\n}\n\nfunc (s *SublistStats) add(stat *SublistStats) {\n\ts.NumSubs += stat.NumSubs\n\ts.NumCache += stat.NumCache\n\ts.NumInserts += stat.NumInserts\n\ts.NumRemoves += stat.NumRemoves\n\ts.NumMatches += stat.NumMatches\n\ts.cacheHits += stat.cacheHits\n\tif s.MaxFanout < stat.MaxFanout {\n\t\ts.MaxFanout = stat.MaxFanout\n\t}\n\n\t// ignore slStats.AvgFanout, collect the values\n\t// it's based on instead\n\ts.totFanout += stat.totFanout\n\ts.cacheCnt += stat.cacheCnt\n\tif s.totFanout > 0 {\n\t\ts.AvgFanout = float64(s.totFanout) / float64(s.cacheCnt)\n\t}\n\tif s.NumMatches > 0 {\n\t\ts.CacheHitRate = float64(s.cacheHits) / float64(s.NumMatches)\n\t}\n}\n\n// Stats will return a stats structure for the current state.\nfunc (s *Sublist) Stats() *SublistStats {\n\tst := &SublistStats{}\n\n\ts.RLock()\n\tcache := s.cache\n\tcc := len(s.cache)\n\tst.NumSubs = s.count\n\tst.NumInserts = s.inserts\n\tst.NumRemoves = s.removes\n\ts.RUnlock()\n\n\tst.NumCache = uint32(cc)\n\tst.NumMatches = atomic.LoadUint64(&s.matches)\n\tst.cacheHits = atomic.LoadUint64(&s.cacheHits)\n\tif st.NumMatches > 0 {\n\t\tst.CacheHitRate = float64(st.cacheHits) / float64(st.NumMatches)\n\t}\n\n\t// whip through cache for fanout stats, this can be off if cache is full and doing evictions.\n\t// If this is called frequently, which it should not be, this could hurt performance.\n\tif cache != nil {\n\t\ttot, max, clen := 0, 0, 0\n\t\ts.RLock()\n\t\tfor _, r := range s.cache {\n\t\t\tclen++\n\t\t\tl := len(r.psubs) + len(r.qsubs)\n\t\t\ttot += l\n\t\t\tif l > max {\n\t\t\t\tmax = l\n\t\t\t}\n\t\t}\n\t\ts.RUnlock()\n\t\tst.totFanout = tot\n\t\tst.cacheCnt = clen\n\t\tst.MaxFanout = uint32(max)\n\t\tif tot > 0 {\n\t\t\tst.AvgFanout = float64(tot) / float64(clen)\n\t\t}\n\t}\n\treturn st\n}\n\n// numLevels will return the maximum number of levels\n// contained in the Sublist tree.\nfunc (s *Sublist) numLevels() int {\n\treturn visitLevel(s.root, 0)\n}\n\n// visitLevel is used to descend the Sublist tree structure\n// recursively.\nfunc visitLevel(l *level, depth int) int {\n\tif l == nil || l.numNodes() == 0 {\n\t\treturn depth\n\t}\n\n\tdepth++\n\tmaxDepth := depth\n\n\tfor _, n := range l.nodes {\n\t\tif n == nil {\n\t\t\tcontinue\n\t\t}\n\t\tnewDepth := visitLevel(n.next, depth)\n\t\tif newDepth > maxDepth {\n\t\t\tmaxDepth = newDepth\n\t\t}\n\t}\n\tif l.pwc != nil {\n\t\tpwcDepth := visitLevel(l.pwc.next, depth)\n\t\tif pwcDepth > maxDepth {\n\t\t\tmaxDepth = pwcDepth\n\t\t}\n\t}\n\tif l.fwc != nil {\n\t\tfwcDepth := visitLevel(l.fwc.next, depth)\n\t\tif fwcDepth > maxDepth {\n\t\t\tmaxDepth = fwcDepth\n\t\t}\n\t}\n\treturn maxDepth\n}\n\n// Determine if a subject has any wildcard tokens.\nfunc subjectHasWildcard(subject string) bool {\n\t// This one exits earlier then !subjectIsLiteral(subject)\n\tfor i, c := range subject {\n\t\tif c == pwc || c == fwc {\n\t\t\tif (i == 0 || subject[i-1] == btsep) &&\n\t\t\t\t(i+1 == len(subject) || subject[i+1] == btsep) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Determine if the subject has any wildcards. Fast version, does not check for\n// valid subject. Used in caching layer.\nfunc subjectIsLiteral(subject string) bool {\n\tfor i, c := range subject {\n\t\tif c == pwc || c == fwc {\n\t\t\tif (i == 0 || subject[i-1] == btsep) &&\n\t\t\t\t(i+1 == len(subject) || subject[i+1] == btsep) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\treturn true\n}\n\n// IsValidPublishSubject returns true if a subject is valid and a literal, false otherwise\nfunc IsValidPublishSubject(subject string) bool {\n\treturn IsValidSubject(subject) && subjectIsLiteral(subject)\n}\n\n// IsValidSubject returns true if a subject is valid, false otherwise\nfunc IsValidSubject(subject string) bool {\n\treturn isValidSubject(subject, false)\n}\n\nfunc isValidSubject(subject string, checkRunes bool) bool {\n\tif subject == _EMPTY_ {\n\t\treturn false\n\t}\n\tif checkRunes {\n\t\t// Check if we have embedded nulls.\n\t\tif bytes.IndexByte(stringToBytes(subject), 0) >= 0 {\n\t\t\treturn false\n\t\t}\n\t\t// Since casting to a string will always produce valid UTF-8, we need to look for replacement runes.\n\t\t// This signals something is off or corrupt.\n\t\tfor _, r := range subject {\n\t\t\tif r == utf8.RuneError {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\tsfwc := false\n\tfor t := range strings.SplitSeq(subject, tsep) {\n\t\tlength := len(t)\n\t\tif length == 0 || sfwc {\n\t\t\treturn false\n\t\t}\n\t\tif length > 1 {\n\t\t\tif strings.ContainsAny(t, \"\\t\\n\\f\\r \") {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tswitch t[0] {\n\t\tcase fwc:\n\t\t\tsfwc = true\n\t\tcase ' ', '\\t', '\\n', '\\r', '\\f':\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// IsValidLiteralSubject returns true if a subject is valid and literal (no wildcards), false otherwise\nfunc IsValidLiteralSubject(subject string) bool {\n\treturn isValidLiteralSubject(strings.SplitSeq(subject, tsep))\n}\n\n// isValidLiteralSubject returns true if the tokens are valid and literal (no wildcards), false otherwise\nfunc isValidLiteralSubject(tokens iter.Seq[string]) bool {\n\tfor t := range tokens {\n\t\tif len(t) == 0 {\n\t\t\treturn false\n\t\t}\n\t\tif len(t) > 1 {\n\t\t\tcontinue\n\t\t}\n\t\tswitch t[0] {\n\t\tcase pwc, fwc:\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// ValidateMapping returns nil error if the subject is a valid subject mapping destination subject\nfunc ValidateMapping(src string, dest string) error {\n\tif dest == _EMPTY_ {\n\t\treturn nil\n\t}\n\tsfwc := false\n\tfor t := range strings.SplitSeq(dest, tsep) {\n\t\tlength := len(t)\n\t\tif length == 0 || sfwc {\n\t\t\treturn &mappingDestinationErr{t, ErrInvalidMappingDestinationSubject}\n\t\t}\n\n\t\t// if it looks like it contains a mapping function, it should be a valid mapping function\n\t\tif length > 4 && t[0] == '{' && t[1] == '{' && t[length-2] == '}' && t[length-1] == '}' {\n\t\t\tif !partitionMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!wildcardMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!splitFromLeftMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!splitFromRightMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!sliceFromLeftMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!sliceFromRightMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!splitMappingFunctionRegEx.MatchString(t) &&\n\t\t\t\t!randomMappingFunctionRegEx.MatchString(t) {\n\t\t\t\treturn &mappingDestinationErr{t, ErrUnknownMappingDestinationFunction}\n\t\t\t} else {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif length == 1 && t[0] == fwc {\n\t\t\tsfwc = true\n\t\t} else if strings.ContainsAny(t, \"\\t\\n\\f\\r \") {\n\t\t\treturn ErrInvalidMappingDestinationSubject\n\t\t}\n\t}\n\n\t// Finally, verify that the transform can actually be created from the source and destination\n\t_, err := NewSubjectTransform(src, dest)\n\treturn err\n}\n\n// Will check tokens and report back if the have any partial or full wildcards.\nfunc analyzeTokens(tokens []string) (hasPWC, hasFWC bool) {\n\tfor _, t := range tokens {\n\t\tif lt := len(t); lt == 0 || lt > 1 {\n\t\t\tcontinue\n\t\t}\n\t\tswitch t[0] {\n\t\tcase pwc:\n\t\t\thasPWC = true\n\t\tcase fwc:\n\t\t\thasFWC = true\n\t\t}\n\t}\n\treturn\n}\n\n// Check on a token basis if they could match.\nfunc tokensCanMatch(t1, t2 string) bool {\n\tif len(t1) == 0 || len(t2) == 0 {\n\t\treturn false\n\t}\n\tt1c, t2c := t1[0], t2[0]\n\tif t1c == pwc || t2c == pwc || t1c == fwc || t2c == fwc {\n\t\treturn true\n\t}\n\treturn t1 == t2\n}\n\n// SubjectsCollide will determine if two subjects could both match a single literal subject.\nfunc SubjectsCollide(subj1, subj2 string) bool {\n\tif subj1 == subj2 {\n\t\treturn true\n\t}\n\ttsa, tsb := [32]string{}, [32]string{}\n\ttoks1 := tokenizeSubjectIntoSlice(tsa[:0], subj1)\n\ttoks2 := tokenizeSubjectIntoSlice(tsb[:0], subj2)\n\tpwc1, fwc1 := analyzeTokens(toks1)\n\tpwc2, fwc2 := analyzeTokens(toks2)\n\t// if both literal just string compare.\n\tl1, l2 := !(pwc1 || fwc1), !(pwc2 || fwc2)\n\tif l1 && l2 {\n\t\treturn subj1 == subj2\n\t}\n\t// So one or both have wildcards. If one is literal than we can do subset matching.\n\tif l1 && !l2 {\n\t\treturn isSubsetMatchTokenized(toks1, toks2)\n\t} else if l2 && !l1 {\n\t\treturn isSubsetMatchTokenized(toks2, toks1)\n\t}\n\t// Both have wildcards.\n\t// If they only have partials then the lengths must match.\n\tif !fwc1 && !fwc2 && len(toks1) != len(toks2) {\n\t\treturn false\n\t}\n\tif lt1, lt2 := len(toks1), len(toks2); lt1 != lt2 {\n\t\t// If the shorter one only has partials then these will not collide.\n\t\tif lt1 < lt2 && !fwc1 || lt2 < lt1 && !fwc2 {\n\t\t\treturn false\n\t\t}\n\t}\n\n\tstop := len(toks1)\n\tif len(toks2) < stop {\n\t\tstop = len(toks2)\n\t}\n\n\t// We look for reasons to say no.\n\tfor i := 0; i < stop; i++ {\n\t\tt1, t2 := toks1[i], toks2[i]\n\t\tif !tokensCanMatch(t1, t2) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Returns number of tokens in the subject.\nfunc numTokens(subject string) int {\n\tvar numTokens int\n\tif len(subject) == 0 {\n\t\treturn 0\n\t}\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tnumTokens++\n\t\t}\n\t}\n\treturn numTokens + 1\n}\n\n// Fast way to return an indexed token.\n// This is one based, so first token is TokenAt(subject, 1)\nfunc tokenAt(subject string, index uint8) string {\n\tti, start := uint8(1), 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\tif ti == index {\n\t\t\t\treturn subject[start:i]\n\t\t\t}\n\t\t\tstart = i + 1\n\t\t\tti++\n\t\t}\n\t}\n\tif ti == index {\n\t\treturn subject[start:]\n\t}\n\treturn _EMPTY_\n}\n\n// use similar to append. meaning, the updated slice will be returned\nfunc tokenizeSubjectIntoSlice(tts []string, subject string) []string {\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttts = append(tts, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttts = append(tts, subject[start:])\n\treturn tts\n}\n\n// SubjectMatchesFilter returns true if the subject matches the provided\n// filter or false otherwise.\nfunc SubjectMatchesFilter(subject, filter string) bool {\n\treturn subjectIsSubsetMatch(subject, filter)\n}\n\n// Calls into the function isSubsetMatch()\nfunc subjectIsSubsetMatch(subject, test string) bool {\n\ttsa := [32]string{}\n\ttts := tokenizeSubjectIntoSlice(tsa[:0], subject)\n\treturn isSubsetMatch(tts, test)\n}\n\n// This will test a subject as an array of tokens against a test subject\n// Calls into the function isSubsetMatchTokenized\nfunc isSubsetMatch(tokens []string, test string) bool {\n\ttsa := [32]string{}\n\ttts := tokenizeSubjectIntoSlice(tsa[:0], test)\n\treturn isSubsetMatchTokenized(tokens, tts)\n}\n\n// This will test a subject as an array of tokens against a test subject (also encoded as array of tokens)\n// and determine if the tokens are matched. Both test subject and tokens\n// may contain wildcards. So foo.* is a subset match of [\">\", \"*.*\", \"foo.*\"],\n// but not of foo.bar, etc.\nfunc isSubsetMatchTokenized(tokens, test []string) bool {\n\t// Walk the target tokens\n\tfor i, t2 := range test {\n\t\tif i >= len(tokens) {\n\t\t\treturn false\n\t\t}\n\t\tl := len(t2)\n\t\tif l == 0 {\n\t\t\treturn false\n\t\t}\n\t\tif t2[0] == fwc && l == 1 {\n\t\t\treturn true\n\t\t}\n\t\tt1 := tokens[i]\n\n\t\tl = len(t1)\n\t\tif l == 0 || t1[0] == fwc && l == 1 {\n\t\t\treturn false\n\t\t}\n\n\t\tif t1[0] == pwc && len(t1) == 1 {\n\t\t\tm := t2[0] == pwc && len(t2) == 1\n\t\t\tif !m {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif i >= len(test) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif t2[0] != pwc && strings.Compare(t1, t2) != 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn len(tokens) == len(test)\n}\n\n// matchLiteral is used to test literal subjects, those that do not have any\n// wildcards, with a target subject. This is used in the cache layer.\nfunc matchLiteral(literal, subject string) bool {\n\tli := 0\n\tll := len(literal)\n\tls := len(subject)\n\tfor i := 0; i < ls; i++ {\n\t\tif li >= ll {\n\t\t\treturn false\n\t\t}\n\t\t// This function has been optimized for speed.\n\t\t// For instance, do not set b:=subject[i] here since\n\t\t// we may bump `i` in this loop to avoid `continue` or\n\t\t// skipping common test in a particular test.\n\t\t// Run Benchmark_SublistMatchLiteral before making any change.\n\t\tswitch subject[i] {\n\t\tcase pwc:\n\t\t\t// NOTE: This is not testing validity of a subject, instead ensures\n\t\t\t// that wildcards are treated as such if they follow some basic rules,\n\t\t\t// namely that they are a token on their own.\n\t\t\tif i == 0 || subject[i-1] == btsep {\n\t\t\t\tif i == ls-1 {\n\t\t\t\t\t// There is no more token in the subject after this wildcard.\n\t\t\t\t\t// Skip token in literal and expect to not find a separator.\n\t\t\t\t\tfor {\n\t\t\t\t\t\t// End of literal, this is a match.\n\t\t\t\t\t\tif li >= ll {\n\t\t\t\t\t\t\treturn true\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Presence of separator, this can't be a match.\n\t\t\t\t\t\tif literal[li] == btsep {\n\t\t\t\t\t\t\treturn false\n\t\t\t\t\t\t}\n\t\t\t\t\t\tli++\n\t\t\t\t\t}\n\t\t\t\t} else if subject[i+1] == btsep {\n\t\t\t\t\t// There is another token in the subject after this wildcard.\n\t\t\t\t\t// Skip token in literal and expect to get a separator.\n\t\t\t\t\tfor {\n\t\t\t\t\t\t// We found the end of the literal before finding a separator,\n\t\t\t\t\t\t// this can't be a match.\n\t\t\t\t\t\tif li >= ll {\n\t\t\t\t\t\t\treturn false\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif literal[li] == btsep {\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\tli++\n\t\t\t\t\t}\n\t\t\t\t\t// Bump `i` since we know there is a `.` following, we are\n\t\t\t\t\t// safe. The common test below is going to check `.` with `.`\n\t\t\t\t\t// which is good. A `continue` here is too costly.\n\t\t\t\t\ti++\n\t\t\t\t}\n\t\t\t}\n\t\tcase fwc:\n\t\t\t// For `>` to be a wildcard, it means being the only or last character\n\t\t\t// in the string preceded by a `.`\n\t\t\tif (i == 0 || subject[i-1] == btsep) && i == ls-1 {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tif subject[i] != literal[li] {\n\t\t\treturn false\n\t\t}\n\t\tli++\n\t}\n\t// Make sure we have processed all of the literal's chars..\n\treturn li >= ll\n}\n\nfunc addLocalSub(sub *subscription, subs *[]*subscription, includeLeafHubs bool) {\n\tif sub != nil && sub.client != nil {\n\t\tkind := sub.client.kind\n\t\tif kind == CLIENT || kind == SYSTEM || kind == JETSTREAM || kind == ACCOUNT ||\n\t\t\t(includeLeafHubs && sub.client.isHubLeafNode() /* implied kind==LEAF */) {\n\t\t\t*subs = append(*subs, sub)\n\t\t}\n\t}\n}\n\nfunc (s *Sublist) addNodeToSubs(n *node, subs *[]*subscription, includeLeafHubs bool) {\n\t// Normal subscriptions\n\tif n.plist != nil {\n\t\tfor _, sub := range n.plist {\n\t\t\taddLocalSub(sub, subs, includeLeafHubs)\n\t\t}\n\t} else {\n\t\tfor sub := range n.psubs {\n\t\t\taddLocalSub(sub, subs, includeLeafHubs)\n\t\t}\n\t}\n\t// Queue subscriptions\n\tfor _, qr := range n.qsubs {\n\t\tfor sub := range qr {\n\t\t\taddLocalSub(sub, subs, includeLeafHubs)\n\t\t}\n\t}\n}\n\nfunc (s *Sublist) collectLocalSubs(l *level, subs *[]*subscription, includeLeafHubs bool) {\n\tfor _, n := range l.nodes {\n\t\ts.addNodeToSubs(n, subs, includeLeafHubs)\n\t\ts.collectLocalSubs(n.next, subs, includeLeafHubs)\n\t}\n\tif l.pwc != nil {\n\t\ts.addNodeToSubs(l.pwc, subs, includeLeafHubs)\n\t\ts.collectLocalSubs(l.pwc.next, subs, includeLeafHubs)\n\t}\n\tif l.fwc != nil {\n\t\ts.addNodeToSubs(l.fwc, subs, includeLeafHubs)\n\t\ts.collectLocalSubs(l.fwc.next, subs, includeLeafHubs)\n\t}\n}\n\n// Return all local client subscriptions. Use the supplied slice.\nfunc (s *Sublist) localSubs(subs *[]*subscription, includeLeafHubs bool) {\n\ts.RLock()\n\ts.collectLocalSubs(s.root, subs, includeLeafHubs)\n\ts.RUnlock()\n}\n\n// All is used to collect all subscriptions.\nfunc (s *Sublist) All(subs *[]*subscription) {\n\ts.RLock()\n\ts.collectAllSubs(s.root, subs)\n\ts.RUnlock()\n}\n\nfunc (s *Sublist) addAllNodeToSubs(n *node, subs *[]*subscription) {\n\t// Normal subscriptions\n\tif n.plist != nil {\n\t\t*subs = append(*subs, n.plist...)\n\t} else {\n\t\tfor sub := range n.psubs {\n\t\t\t*subs = append(*subs, sub)\n\t\t}\n\t}\n\t// Queue subscriptions\n\tfor _, qr := range n.qsubs {\n\t\tfor sub := range qr {\n\t\t\t*subs = append(*subs, sub)\n\t\t}\n\t}\n}\n\nfunc (s *Sublist) collectAllSubs(l *level, subs *[]*subscription) {\n\tfor _, n := range l.nodes {\n\t\ts.addAllNodeToSubs(n, subs)\n\t\ts.collectAllSubs(n.next, subs)\n\t}\n\tif l.pwc != nil {\n\t\ts.addAllNodeToSubs(l.pwc, subs)\n\t\ts.collectAllSubs(l.pwc.next, subs)\n\t}\n\tif l.fwc != nil {\n\t\ts.addAllNodeToSubs(l.fwc, subs)\n\t\ts.collectAllSubs(l.fwc.next, subs)\n\t}\n}\n\n// For a given subject (which may contain wildcards), this call returns all\n// subscriptions that would match that subject. For instance, suppose that\n// the sublist contains: foo.bar, foo.bar.baz and foo.baz, ReverseMatch(\"foo.*\")\n// would return foo.bar and foo.baz.\n// This is used in situations where the sublist is likely to contain only\n// literals and one wants to get all the subjects that would have been a match\n// to a subscription on `subject`.\nfunc (s *Sublist) ReverseMatch(subject string) *SublistResult {\n\ttsa := [32]string{}\n\ttokens := tsa[:0]\n\tstart := 0\n\tfor i := 0; i < len(subject); i++ {\n\t\tif subject[i] == btsep {\n\t\t\ttokens = append(tokens, subject[start:i])\n\t\t\tstart = i + 1\n\t\t}\n\t}\n\ttokens = append(tokens, subject[start:])\n\n\tresult := &SublistResult{}\n\n\ts.RLock()\n\treverseMatchLevel(s.root, tokens, nil, result)\n\t// Check for empty result.\n\tif len(result.psubs) == 0 && len(result.qsubs) == 0 {\n\t\tresult = emptyResult\n\t}\n\ts.RUnlock()\n\n\treturn result\n}\n\nfunc reverseMatchLevel(l *level, toks []string, n *node, results *SublistResult) {\n\tif l == nil {\n\t\treturn\n\t}\n\tfor i, t := range toks {\n\t\tif len(t) == 1 {\n\t\t\tif t[0] == fwc {\n\t\t\t\tgetAllNodes(l, results)\n\t\t\t\treturn\n\t\t\t} else if t[0] == pwc {\n\t\t\t\tfor _, n := range l.nodes {\n\t\t\t\t\treverseMatchLevel(n.next, toks[i+1:], n, results)\n\t\t\t\t}\n\t\t\t\tif l.pwc != nil {\n\t\t\t\t\treverseMatchLevel(l.pwc.next, toks[i+1:], n, results)\n\t\t\t\t}\n\t\t\t\tif l.fwc != nil {\n\t\t\t\t\tgetAllNodes(l, results)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\t// If the sub tree has a fwc at this position, match as well.\n\t\tif l.fwc != nil {\n\t\t\tgetAllNodes(l, results)\n\t\t\treturn\n\t\t} else if l.pwc != nil {\n\t\t\treverseMatchLevel(l.pwc.next, toks[i+1:], n, results)\n\t\t}\n\t\tn = l.nodes[t]\n\t\tif n == nil {\n\t\t\tbreak\n\t\t}\n\t\tl = n.next\n\t}\n\tif n != nil {\n\t\taddNodeToResults(n, results)\n\t}\n}\n\nfunc getAllNodes(l *level, results *SublistResult) {\n\tif l == nil {\n\t\treturn\n\t}\n\tif l.pwc != nil {\n\t\taddNodeToResults(l.pwc, results)\n\t}\n\tif l.fwc != nil {\n\t\taddNodeToResults(l.fwc, results)\n\t}\n\tfor _, n := range l.nodes {\n\t\taddNodeToResults(n, results)\n\t\tgetAllNodes(n.next, results)\n\t}\n}\n" }, { "path": "server/sublist_test.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc stackFatalf(t *testing.T, f string, args ...any) {\n\tlines := make([]string, 0, 32)\n\tmsg := fmt.Sprintf(f, args...)\n\tlines = append(lines, msg)\n\n\t// Generate the Stack of callers: Skip us and verify* frames.\n\tfor i := 2; true; i++ {\n\t\t_, file, line, ok := runtime.Caller(i)\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tmsg := fmt.Sprintf(\"%d - %s:%d\", i, file, line)\n\t\tlines = append(lines, msg)\n\t}\n\tt.Fatalf(\"%s\", strings.Join(lines, \"\\n\"))\n}\n\nfunc verifyCount(s *Sublist, count uint32, t *testing.T) {\n\tt.Helper()\n\tif s.Count() != count {\n\t\tt.Fatalf(\"Count is %d, should be %d\", s.Count(), count)\n\t}\n}\n\nfunc verifyLen(r []*subscription, l int, t *testing.T) {\n\tt.Helper()\n\tif len(r) != l {\n\t\tt.Fatalf(\"Results len is %d, should be %d\", len(r), l)\n\t}\n}\n\nfunc verifyQLen(r [][]*subscription, l int, t *testing.T) {\n\tt.Helper()\n\tif len(r) != l {\n\t\tt.Fatalf(\"Queue Results len is %d, should be %d\", len(r), l)\n\t}\n}\n\nfunc verifyNumLevels(s *Sublist, expected int, t *testing.T) {\n\tt.Helper()\n\tdl := s.numLevels()\n\tif dl != expected {\n\t\tt.Fatalf(\"NumLevels is %d, should be %d\", dl, expected)\n\t}\n}\n\nfunc verifyQMember(qsubs [][]*subscription, val *subscription, t *testing.T) {\n\tt.Helper()\n\tverifyMember(qsubs[findQSlot(val.queue, qsubs)], val, t)\n}\n\nfunc verifyMember(r []*subscription, val *subscription, t *testing.T) {\n\tt.Helper()\n\tfor _, v := range r {\n\t\tif v == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif v == val {\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatalf(\"Subscription (%p) for [%s : %s] not found in results\", val, val.subject, val.queue)\n}\n\n// Helpers to generate test subscriptions.\nfunc newSub(subject string) *subscription {\n\tc := &client{kind: CLIENT}\n\treturn &subscription{client: c, subject: []byte(subject)}\n}\n\nfunc newQSub(subject, queue string) *subscription {\n\tif queue != \"\" {\n\t\treturn &subscription{subject: []byte(subject), queue: []byte(queue)}\n\t}\n\treturn newSub(subject)\n}\n\nfunc newRemoteQSub(subject, queue string, num int32) *subscription {\n\tif queue != \"\" {\n\t\tc := &client{kind: ROUTER}\n\t\treturn &subscription{client: c, subject: []byte(subject), queue: []byte(queue), qw: num}\n\t}\n\treturn newSub(subject)\n}\n\nfunc TestSublistInit(t *testing.T) {\n\ts := NewSublistWithCache()\n\tverifyCount(s, 0, t)\n}\n\nfunc TestSublistInsertCount(t *testing.T) {\n\ttestSublistInsertCount(t, NewSublistWithCache())\n}\n\nfunc TestSublistInsertCountNoCache(t *testing.T) {\n\ttestSublistInsertCount(t, NewSublistNoCache())\n}\n\nfunc testSublistInsertCount(t *testing.T, s *Sublist) {\n\ts.Insert(newSub(\"foo\"))\n\ts.Insert(newSub(\"bar\"))\n\ts.Insert(newSub(\"foo.bar\"))\n\tverifyCount(s, 3, t)\n}\n\nfunc TestSublistSimple(t *testing.T) {\n\ttestSublistSimple(t, NewSublistWithCache())\n}\n\nfunc TestSublistSimpleNoCache(t *testing.T) {\n\ttestSublistSimple(t, NewSublistNoCache())\n}\n\nfunc testSublistSimple(t *testing.T, s *Sublist) {\n\tsubject := \"foo\"\n\tsub := newSub(subject)\n\ts.Insert(sub)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub, t)\n}\n\nfunc TestSublistSimpleMultiTokens(t *testing.T) {\n\ttestSublistSimpleMultiTokens(t, NewSublistWithCache())\n}\n\nfunc TestSublistSimpleMultiTokensNoCache(t *testing.T) {\n\ttestSublistSimpleMultiTokens(t, NewSublistNoCache())\n}\n\nfunc testSublistSimpleMultiTokens(t *testing.T, s *Sublist) {\n\tsubject := \"foo.bar.baz\"\n\tsub := newSub(subject)\n\ts.Insert(sub)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub, t)\n}\n\nfunc TestSublistPartialWildcard(t *testing.T) {\n\ttestSublistPartialWildcard(t, NewSublistWithCache())\n}\n\nfunc TestSublistPartialWildcardNoCache(t *testing.T) {\n\ttestSublistPartialWildcard(t, NewSublistNoCache())\n}\n\nfunc testSublistPartialWildcard(t *testing.T, s *Sublist) {\n\tlsub := newSub(\"a.b.c\")\n\tpsub := newSub(\"a.*.c\")\n\ts.Insert(lsub)\n\ts.Insert(psub)\n\tr := s.Match(\"a.b.c\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, lsub, t)\n\tverifyMember(r.psubs, psub, t)\n}\n\nfunc TestSublistPartialWildcardAtEnd(t *testing.T) {\n\ttestSublistPartialWildcardAtEnd(t, NewSublistWithCache())\n}\n\nfunc TestSublistPartialWildcardAtEndNoCache(t *testing.T) {\n\ttestSublistPartialWildcardAtEnd(t, NewSublistNoCache())\n}\n\nfunc testSublistPartialWildcardAtEnd(t *testing.T, s *Sublist) {\n\tlsub := newSub(\"a.b.c\")\n\tpsub := newSub(\"a.b.*\")\n\ts.Insert(lsub)\n\ts.Insert(psub)\n\tr := s.Match(\"a.b.c\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, lsub, t)\n\tverifyMember(r.psubs, psub, t)\n}\n\nfunc TestSublistFullWildcard(t *testing.T) {\n\ttestSublistFullWildcard(t, NewSublistWithCache())\n}\n\nfunc TestSublistFullWildcardNoCache(t *testing.T) {\n\ttestSublistFullWildcard(t, NewSublistNoCache())\n}\n\nfunc testSublistFullWildcard(t *testing.T, s *Sublist) {\n\tlsub := newSub(\"a.b.c\")\n\tfsub := newSub(\"a.>\")\n\ts.Insert(lsub)\n\ts.Insert(fsub)\n\tr := s.Match(\"a.b.c\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, lsub, t)\n\tverifyMember(r.psubs, fsub, t)\n\n\tr = s.Match(\"a.>\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, fsub, t)\n}\n\nfunc TestSublistRemove(t *testing.T) {\n\ttestSublistRemove(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveNoCache(t *testing.T) {\n\ttestSublistRemove(t, NewSublistNoCache())\n}\n\nfunc testSublistRemove(t *testing.T, s *Sublist) {\n\tsubject := \"a.b.c.d\"\n\tsub := newSub(subject)\n\ts.Insert(sub)\n\tverifyCount(s, 1, t)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\ts.Remove(newSub(\"a.b.c\"))\n\tverifyCount(s, 1, t)\n\ts.Remove(sub)\n\tverifyCount(s, 0, t)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n}\n\nfunc TestSublistRemoveWildcard(t *testing.T) {\n\ttestSublistRemoveWildcard(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveWildcardNoCache(t *testing.T) {\n\ttestSublistRemoveWildcard(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoveWildcard(t *testing.T, s *Sublist) {\n\tsubject := \"a.b.c.d\"\n\tsub := newSub(subject)\n\tpsub := newSub(\"a.b.*.d\")\n\tfsub := newSub(\"a.b.>\")\n\ts.Insert(sub)\n\ts.Insert(psub)\n\ts.Insert(fsub)\n\tverifyCount(s, 3, t)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 3, t)\n\ts.Remove(sub)\n\tverifyCount(s, 2, t)\n\ts.Remove(fsub)\n\tverifyCount(s, 1, t)\n\ts.Remove(psub)\n\tverifyCount(s, 0, t)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n}\n\nfunc TestSublistRemoveCleanup(t *testing.T) {\n\ttestSublistRemoveCleanup(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveCleanupNoCache(t *testing.T) {\n\ttestSublistRemoveCleanup(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoveCleanup(t *testing.T, s *Sublist) {\n\tliteral := \"a.b.c.d.e.f\"\n\tdepth := len(strings.Split(literal, tsep))\n\tsub := newSub(literal)\n\tverifyNumLevels(s, 0, t)\n\ts.Insert(sub)\n\tverifyNumLevels(s, depth, t)\n\ts.Remove(sub)\n\tverifyNumLevels(s, 0, t)\n}\n\nfunc TestSublistRemoveCleanupWildcards(t *testing.T) {\n\ttestSublistRemoveCleanupWildcards(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveCleanupWildcardsNoCache(t *testing.T) {\n\ttestSublistRemoveCleanupWildcards(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoveCleanupWildcards(t *testing.T, s *Sublist) {\n\tsubject := \"a.b.*.d.e.>\"\n\tdepth := len(strings.Split(subject, tsep))\n\tsub := newSub(subject)\n\tverifyNumLevels(s, 0, t)\n\ts.Insert(sub)\n\tverifyNumLevels(s, depth, t)\n\ts.Remove(sub)\n\tverifyNumLevels(s, 0, t)\n}\n\nfunc TestSublistRemoveWithLargeSubs(t *testing.T) {\n\ttestSublistRemoveWithLargeSubs(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveWithLargeSubsNoCache(t *testing.T) {\n\ttestSublistRemoveWithLargeSubs(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoveWithLargeSubs(t *testing.T, s *Sublist) {\n\tsubject := \"foo\"\n\tfor i := 0; i < plistMin*2; i++ {\n\t\tsub := newSub(subject)\n\t\ts.Insert(sub)\n\t}\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, plistMin*2, t)\n\t// Remove one that is in the middle\n\ts.Remove(r.psubs[plistMin])\n\t// Remove first one\n\ts.Remove(r.psubs[0])\n\t// Remove last one\n\ts.Remove(r.psubs[len(r.psubs)-1])\n\t// Check len again\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, plistMin*2-3, t)\n}\n\nfunc TestSublistInvalidSubjectsInsert(t *testing.T) {\n\ttestSublistInvalidSubjectsInsert(t, NewSublistWithCache())\n}\n\nfunc TestSublistInvalidSubjectsInsertNoCache(t *testing.T) {\n\ttestSublistInvalidSubjectsInsert(t, NewSublistNoCache())\n}\n\nfunc TestSublistNoCacheRemoveBatch(t *testing.T) {\n\ts := NewSublistNoCache()\n\ts.Insert(newSub(\"foo\"))\n\tsub := newSub(\"bar\")\n\ts.Insert(sub)\n\ts.RemoveBatch([]*subscription{sub})\n\t// Now test that this did not turn on cache\n\tfor i := 0; i < 10; i++ {\n\t\ts.Match(\"foo\")\n\t}\n\tif s.CacheEnabled() {\n\t\tt.Fatalf(\"Cache should not be enabled\")\n\t}\n}\n\nfunc TestSublistRemoveBatchWithError(t *testing.T) {\n\ts := NewSublistNoCache()\n\tsub1 := newSub(\"foo\")\n\tsub2 := newSub(\"bar\")\n\tsub3 := newSub(\"baz\")\n\ts.Insert(sub1)\n\ts.Insert(sub2)\n\ts.Insert(sub3)\n\tsubNotPresent := newSub(\"not.inserted\")\n\t// Try to remove all subs, but include the sub that has not been inserted.\n\terr := s.RemoveBatch([]*subscription{subNotPresent, sub1, sub3})\n\t// We expect an error to be returned, but sub1,2 and 3 to have been removed.\n\trequire_Error(t, err, ErrNotFound)\n\t// Make sure that we have only sub2 present\n\tverifyCount(s, 1, t)\n\tr := s.Match(\"bar\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub2, t)\n\tr = s.Match(\"foo\")\n\tverifyLen(r.psubs, 0, t)\n\tr = s.Match(\"baz\")\n\tverifyLen(r.psubs, 0, t)\n}\n\nfunc testSublistInvalidSubjectsInsert(t *testing.T, s *Sublist) {\n\t// Insert, or subscriptions, can have wildcards, but not empty tokens,\n\t// and can not have a FWC that is not the terminal token.\n\n\t// beginning empty token\n\tif err := s.Insert(newSub(\".foo\")); err != ErrInvalidSubject {\n\t\tt.Fatal(\"Expected invalid subject error\")\n\t}\n\n\t// trailing empty token\n\tif err := s.Insert(newSub(\"foo.\")); err != ErrInvalidSubject {\n\t\tt.Fatal(\"Expected invalid subject error\")\n\t}\n\t// empty middle token\n\tif err := s.Insert(newSub(\"foo..bar\")); err != ErrInvalidSubject {\n\t\tt.Fatal(\"Expected invalid subject error\")\n\t}\n\t// empty middle token #2\n\tif err := s.Insert(newSub(\"foo.bar..baz\")); err != ErrInvalidSubject {\n\t\tt.Fatal(\"Expected invalid subject error\")\n\t}\n\t// fwc not terminal\n\tif err := s.Insert(newSub(\"foo.>.bar\")); err != ErrInvalidSubject {\n\t\tt.Fatal(\"Expected invalid subject error\")\n\t}\n}\n\nfunc TestSublistCache(t *testing.T) {\n\ts := NewSublistWithCache()\n\n\t// Test add a remove logistics\n\tsubject := \"a.b.c.d\"\n\tsub := newSub(subject)\n\tpsub := newSub(\"a.b.*.d\")\n\tfsub := newSub(\"a.b.>\")\n\ts.Insert(sub)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\ts.Insert(psub)\n\ts.Insert(fsub)\n\tverifyCount(s, 3, t)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 3, t)\n\ts.Remove(sub)\n\tverifyCount(s, 2, t)\n\ts.Remove(fsub)\n\tverifyCount(s, 1, t)\n\ts.Remove(psub)\n\tverifyCount(s, 0, t)\n\n\t// Check that cache is now empty\n\tif cc := s.CacheCount(); cc != 0 {\n\t\tt.Fatalf(\"Cache should be zero, got %d\\n\", cc)\n\t}\n\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\n\tfor i := 0; i < 2*slCacheMax; i++ {\n\t\ts.Match(fmt.Sprintf(\"foo-%d\\n\", i))\n\t}\n\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tif cc := s.CacheCount(); cc > slCacheMax {\n\t\t\treturn fmt.Errorf(\"Cache should be constrained by cacheMax, got %d for current count\", cc)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Test that adding to a wildcard properly adds to the cache.\n\ts = NewSublistWithCache()\n\ts.Insert(newSub(\"foo.*\"))\n\ts.Insert(newSub(\"foo.bar\"))\n\tr = s.Match(\"foo.baz\")\n\tverifyLen(r.psubs, 1, t)\n\tr = s.Match(\"foo.bar\")\n\tverifyLen(r.psubs, 2, t)\n\ts.Insert(newSub(\"foo.>\"))\n\tr = s.Match(\"foo.bar\")\n\tverifyLen(r.psubs, 3, t)\n}\n\nfunc TestSublistBasicQueueResults(t *testing.T) {\n\ttestSublistBasicQueueResults(t, NewSublistWithCache())\n}\n\nfunc TestSublistBasicQueueResultsNoCache(t *testing.T) {\n\ttestSublistBasicQueueResults(t, NewSublistNoCache())\n}\n\nfunc testSublistBasicQueueResults(t *testing.T, s *Sublist) {\n\t// Test some basics\n\tsubject := \"foo\"\n\tsub := newSub(subject)\n\tsub1 := newQSub(subject, \"bar\")\n\tsub2 := newQSub(subject, \"baz\")\n\n\ts.Insert(sub1)\n\tr := s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 1, t)\n\tverifyLen(r.qsubs[0], 1, t)\n\tverifyQMember(r.qsubs, sub1, t)\n\n\ts.Insert(sub2)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 2, t)\n\tverifyLen(r.qsubs[0], 1, t)\n\tverifyLen(r.qsubs[1], 1, t)\n\tverifyQMember(r.qsubs, sub1, t)\n\tverifyQMember(r.qsubs, sub2, t)\n\n\ts.Insert(sub)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\tverifyQLen(r.qsubs, 2, t)\n\tverifyLen(r.qsubs[0], 1, t)\n\tverifyLen(r.qsubs[1], 1, t)\n\tverifyQMember(r.qsubs, sub1, t)\n\tverifyQMember(r.qsubs, sub2, t)\n\tverifyMember(r.psubs, sub, t)\n\n\tsub3 := newQSub(subject, \"bar\")\n\tsub4 := newQSub(subject, \"baz\")\n\n\ts.Insert(sub3)\n\ts.Insert(sub4)\n\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 1, t)\n\tverifyQLen(r.qsubs, 2, t)\n\tverifyLen(r.qsubs[0], 2, t)\n\tverifyLen(r.qsubs[1], 2, t)\n\tverifyQMember(r.qsubs, sub1, t)\n\tverifyQMember(r.qsubs, sub2, t)\n\tverifyQMember(r.qsubs, sub3, t)\n\tverifyQMember(r.qsubs, sub4, t)\n\tverifyMember(r.psubs, sub, t)\n\n\t// Now removal\n\ts.Remove(sub)\n\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 2, t)\n\tverifyLen(r.qsubs[0], 2, t)\n\tverifyLen(r.qsubs[1], 2, t)\n\tverifyQMember(r.qsubs, sub1, t)\n\tverifyQMember(r.qsubs, sub2, t)\n\n\ts.Remove(sub1)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 2, t)\n\tverifyLen(r.qsubs[findQSlot(sub1.queue, r.qsubs)], 1, t)\n\tverifyLen(r.qsubs[findQSlot(sub2.queue, r.qsubs)], 2, t)\n\tverifyQMember(r.qsubs, sub2, t)\n\tverifyQMember(r.qsubs, sub3, t)\n\tverifyQMember(r.qsubs, sub4, t)\n\n\ts.Remove(sub3) // Last one\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 1, t)\n\tverifyLen(r.qsubs[0], 2, t) // this is sub2/baz now\n\tverifyQMember(r.qsubs, sub2, t)\n\n\ts.Remove(sub2)\n\ts.Remove(sub4)\n\tr = s.Match(subject)\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 0, t)\n}\n\nfunc checkBool(b, expected bool, t *testing.T) {\n\tt.Helper()\n\tif b != expected {\n\t\tt.Fatalf(\"Expected %v, but got %v\\n\", expected, b)\n\t}\n}\n\nfunc checkError(err, expected error, t *testing.T) {\n\tt.Helper()\n\tif err != expected && err != nil && !errors.Is(err, expected) {\n\t\tt.Fatalf(\"Expected %v, but got %v\\n\", expected, err)\n\t}\n}\n\nfunc TestSublistValidLiteralSubjects(t *testing.T) {\n\tcheckBool(IsValidLiteralSubject(\"foo\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\".foo\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\"foo..bar\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.bar.*\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.bar.>\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\"*\"), false, t)\n\tcheckBool(IsValidLiteralSubject(\">\"), false, t)\n\t// The followings have widlcards characters but are not\n\t// considered as such because they are not individual tokens.\n\tcheckBool(IsValidLiteralSubject(\"foo*\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo**\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.**\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo*bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.*bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo*.bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"*bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo>\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo>>\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.>>\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo>bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo.>bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\"foo>.bar\"), true, t)\n\tcheckBool(IsValidLiteralSubject(\">bar\"), true, t)\n}\n\nfunc TestSublistValidSubjects(t *testing.T) {\n\tcheckBool(IsValidSubject(\".\"), false, t)\n\tcheckBool(IsValidSubject(\".foo\"), false, t)\n\tcheckBool(IsValidSubject(\"foo.\"), false, t)\n\tcheckBool(IsValidSubject(\"foo..bar\"), false, t)\n\tcheckBool(IsValidSubject(\">.bar\"), false, t)\n\tcheckBool(IsValidSubject(\"foo.>.bar\"), false, t)\n\tcheckBool(IsValidSubject(\"foo\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.bar.*\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.bar.>\"), true, t)\n\tcheckBool(IsValidSubject(\"*\"), true, t)\n\tcheckBool(IsValidSubject(\">\"), true, t)\n\tcheckBool(IsValidSubject(\"foo*\"), true, t)\n\tcheckBool(IsValidSubject(\"foo**\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.**\"), true, t)\n\tcheckBool(IsValidSubject(\"foo*bar\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.*bar\"), true, t)\n\tcheckBool(IsValidSubject(\"foo*.bar\"), true, t)\n\tcheckBool(IsValidSubject(\"*bar\"), true, t)\n\tcheckBool(IsValidSubject(\"foo>\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.>>\"), true, t)\n\tcheckBool(IsValidSubject(\"foo>bar\"), true, t)\n\tcheckBool(IsValidSubject(\"foo.>bar\"), true, t)\n\tcheckBool(IsValidSubject(\"foo>.bar\"), true, t)\n\tcheckBool(IsValidSubject(\">bar\"), true, t)\n\n\t// Check for embedded nulls.\n\tsubj := []byte(\"foo.bar.baz.\")\n\tsubj = append(subj, 0)\n\tcheckBool(isValidSubject(string(subj), true), false, t)\n}\n\nfunc TestSublistMatchLiterals(t *testing.T) {\n\tcheckBool(matchLiteral(\"foo\", \"foo\"), true, t)\n\tcheckBool(matchLiteral(\"foo\", \"bar\"), false, t)\n\tcheckBool(matchLiteral(\"foo\", \"*\"), true, t)\n\tcheckBool(matchLiteral(\"foo\", \">\"), true, t)\n\tcheckBool(matchLiteral(\"foo.bar\", \">\"), true, t)\n\tcheckBool(matchLiteral(\"foo.bar\", \"foo.>\"), true, t)\n\tcheckBool(matchLiteral(\"foo.bar\", \"bar.>\"), false, t)\n\tcheckBool(matchLiteral(\"stats.test.22\", \"stats.>\"), true, t)\n\tcheckBool(matchLiteral(\"stats.test.22\", \"stats.*.*\"), true, t)\n\tcheckBool(matchLiteral(\"foo.bar\", \"foo\"), false, t)\n\tcheckBool(matchLiteral(\"stats.test.foos\", \"stats.test.foos\"), true, t)\n\tcheckBool(matchLiteral(\"stats.test.foos\", \"stats.test.foo\"), false, t)\n\tcheckBool(matchLiteral(\"stats.test\", \"stats.test.*\"), false, t)\n\tcheckBool(matchLiteral(\"stats.test.foos\", \"stats.*\"), false, t)\n\tcheckBool(matchLiteral(\"stats.test.foos\", \"stats.*.*.foos\"), false, t)\n\n\t// These are cases where wildcards characters should not be considered\n\t// wildcards since they do not follow the rules of wildcards.\n\tcheckBool(matchLiteral(\"*bar\", \"*bar\"), true, t)\n\tcheckBool(matchLiteral(\"foo*\", \"foo*\"), true, t)\n\tcheckBool(matchLiteral(\"foo*bar\", \"foo*bar\"), true, t)\n\tcheckBool(matchLiteral(\"foo.***.bar\", \"foo.***.bar\"), true, t)\n\tcheckBool(matchLiteral(\">bar\", \">bar\"), true, t)\n\tcheckBool(matchLiteral(\"foo>\", \"foo>\"), true, t)\n\tcheckBool(matchLiteral(\"foo>bar\", \"foo>bar\"), true, t)\n\tcheckBool(matchLiteral(\"foo.>>>.bar\", \"foo.>>>.bar\"), true, t)\n}\n\nfunc TestSubjectIsLiteral(t *testing.T) {\n\tcheckBool(subjectIsLiteral(\"foo\"), true, t)\n\tcheckBool(subjectIsLiteral(\"foo.bar\"), true, t)\n\tcheckBool(subjectIsLiteral(\"foo*.bar\"), true, t)\n\tcheckBool(subjectIsLiteral(\"*\"), false, t)\n\tcheckBool(subjectIsLiteral(\">\"), false, t)\n\tcheckBool(subjectIsLiteral(\"foo.*\"), false, t)\n\tcheckBool(subjectIsLiteral(\"foo.>\"), false, t)\n\tcheckBool(subjectIsLiteral(\"foo.*.>\"), false, t)\n\tcheckBool(subjectIsLiteral(\"foo.*.bar\"), false, t)\n\tcheckBool(subjectIsLiteral(\"foo.bar.>\"), false, t)\n}\n\nfunc TestValidateDestinationSubject(t *testing.T) {\n\tcheckError(ValidateMapping(\"bar\", \"foo\"), nil, t)\n\tcheckError(ValidateMapping(\"foo\", \"foo.bar\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"{{wildcard(1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.>\", \">\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.*\", \"bar.{{wildcard(1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.>\", \"bar.>\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.*.>\", \"bar.{{wildcard(1)}}.>\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.*.bar\", \"bar.{{wildcard(1)}}.foo\"), nil, t)\n\tcheckError(ValidateMapping(\"foo.bar.>\", \"foo.bar.foo.>\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"foo.{{wildcard(1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"foo.{{ wildcard(1) }}\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"foo.{{wildcard( 1 )}}\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"foo.{{partition(2,1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"*.*\", \"foo.{{SplitFromLeft(2,1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"*.*\", \"foo.{{SplitFromRight(2,1)}}\"), nil, t)\n\tcheckError(ValidateMapping(\"*\", \"foo.{{unknown(1)}}\"), ErrInvalidMappingDestination, t)\n\tcheckError(ValidateMapping(\"foo\", \"foo..}\"), ErrInvalidMappingDestination, t)\n\tcheckError(ValidateMapping(\"foo\", \"foo. bar}\"), ErrInvalidMappingDestinationSubject, t)\n}\n\nfunc TestSubjectToken(t *testing.T) {\n\tcheckToken := func(token, expected string) {\n\t\tt.Helper()\n\t\tif token != expected {\n\t\t\tt.Fatalf(\"Expected token of %q, got %q\", expected, token)\n\t\t}\n\t}\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 0), \"\")\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 1), \"foo\")\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 2), \"bar\")\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 3), \"baz\")\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 4), \"*\")\n\tcheckToken(tokenAt(\"foo.bar.baz.*\", 5), \"\")\n}\n\nfunc TestSublistBadSubjectOnRemove(t *testing.T) {\n\ttestSublistBadSubjectOnRemove(t, NewSublistWithCache())\n}\n\nfunc TestSublistBadSubjectOnRemoveNoCache(t *testing.T) {\n\ttestSublistBadSubjectOnRemove(t, NewSublistNoCache())\n}\n\nfunc testSublistBadSubjectOnRemove(t *testing.T, s *Sublist) {\n\tbad := \"a.b..d\"\n\tsub := newSub(bad)\n\n\tif err := s.Insert(sub); err != ErrInvalidSubject {\n\t\tt.Fatalf(\"Expected ErrInvalidSubject, got %v\\n\", err)\n\t}\n\n\tif err := s.Remove(sub); err != ErrInvalidSubject {\n\t\tt.Fatalf(\"Expected ErrInvalidSubject, got %v\\n\", err)\n\t}\n\n\tbadfwc := \"a.>.b\"\n\tif err := s.Remove(newSub(badfwc)); err != ErrInvalidSubject {\n\t\tt.Fatalf(\"Expected ErrInvalidSubject, got %v\\n\", err)\n\t}\n}\n\n// This is from bug report #18\nfunc TestSublistTwoTokenPubMatchSingleTokenSub(t *testing.T) {\n\ttestSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistWithCache())\n}\n\nfunc TestSublistTwoTokenPubMatchSingleTokenSubNoCache(t *testing.T) {\n\ttestSublistTwoTokenPubMatchSingleTokenSub(t, NewSublistNoCache())\n}\n\nfunc testSublistTwoTokenPubMatchSingleTokenSub(t *testing.T, s *Sublist) {\n\tsub := newSub(\"foo\")\n\ts.Insert(sub)\n\tr := s.Match(\"foo\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub, t)\n\tr = s.Match(\"foo.bar\")\n\tverifyLen(r.psubs, 0, t)\n}\n\nfunc TestSublistInsertWithWildcardsAsLiterals(t *testing.T) {\n\ttestSublistInsertWithWildcardsAsLiterals(t, NewSublistWithCache())\n}\n\nfunc TestSublistInsertWithWildcardsAsLiteralsNoCache(t *testing.T) {\n\ttestSublistInsertWithWildcardsAsLiterals(t, NewSublistNoCache())\n}\n\nfunc testSublistInsertWithWildcardsAsLiterals(t *testing.T, s *Sublist) {\n\tsubjects := []string{\"foo.*-\", \"foo.>-\"}\n\tfor _, subject := range subjects {\n\t\tsub := newSub(subject)\n\t\ts.Insert(sub)\n\t\t// Should find no match\n\t\tr := s.Match(\"foo.bar\")\n\t\tverifyLen(r.psubs, 0, t)\n\t\t// Should find a match\n\t\tr = s.Match(subject)\n\t\tverifyLen(r.psubs, 1, t)\n\t}\n}\n\nfunc TestSublistRemoveWithWildcardsAsLiterals(t *testing.T) {\n\ttestSublistRemoveWithWildcardsAsLiterals(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoveWithWildcardsAsLiteralsNoCache(t *testing.T) {\n\ttestSublistRemoveWithWildcardsAsLiterals(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoveWithWildcardsAsLiterals(t *testing.T, s *Sublist) {\n\tsubjects := []string{\"foo.*-\", \"foo.>-\"}\n\tfor _, subject := range subjects {\n\t\tsub := newSub(subject)\n\t\ts.Insert(sub)\n\t\t// Should find no match\n\t\trsub := newSub(\"foo.bar\")\n\t\ts.Remove(rsub)\n\t\tif c := s.Count(); c != 1 {\n\t\t\tt.Fatalf(\"Expected sublist to still contain sub, got %v\", c)\n\t\t}\n\t\ts.Remove(sub)\n\t\tif c := s.Count(); c != 0 {\n\t\t\tt.Fatalf(\"Expected sublist to be empty, got %v\", c)\n\t\t}\n\t}\n}\n\nfunc TestSublistRaceOnRemove(t *testing.T) {\n\ttestSublistRaceOnRemove(t, NewSublistWithCache())\n}\n\nfunc TestSublistRaceOnRemoveNoCache(t *testing.T) {\n\ttestSublistRaceOnRemove(t, NewSublistNoCache())\n}\n\nfunc testSublistRaceOnRemove(t *testing.T, s *Sublist) {\n\tvar (\n\t\ttotal = 100\n\t\tsubs = make(map[int]*subscription, total) // use map for randomness\n\t)\n\tfor i := 0; i < total; i++ {\n\t\tsub := newQSub(\"foo\", \"bar\")\n\t\tsubs[i] = sub\n\t}\n\n\tfor i := 0; i < 2; i++ {\n\t\tfor _, sub := range subs {\n\t\t\ts.Insert(sub)\n\t\t}\n\t\t// Call Match() once or twice, to make sure we get from cache\n\t\tif i == 1 {\n\t\t\ts.Match(\"foo\")\n\t\t}\n\t\t// This will be from cache when i==1\n\t\tr := s.Match(\"foo\")\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tfor _, sub := range subs {\n\t\t\t\ts.Remove(sub)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t\tfor _, qsub := range r.qsubs {\n\t\t\tfor i := 0; i < len(qsub); i++ {\n\t\t\t\tsub := qsub[i]\n\t\t\t\tif string(sub.queue) != \"bar\" {\n\t\t\t\t\tt.Fatalf(\"Queue name should be bar, got %s\", qsub[i].queue)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\twg.Wait()\n\t}\n\n\t// Repeat tests with regular subs\n\tfor i := 0; i < total; i++ {\n\t\tsub := newSub(\"foo\")\n\t\tsubs[i] = sub\n\t}\n\n\tfor i := 0; i < 2; i++ {\n\t\tfor _, sub := range subs {\n\t\t\ts.Insert(sub)\n\t\t}\n\t\t// Call Match() once or twice, to make sure we get from cache\n\t\tif i == 1 {\n\t\t\ts.Match(\"foo\")\n\t\t}\n\t\t// This will be from cache when i==1\n\t\tr := s.Match(\"foo\")\n\t\twg := sync.WaitGroup{}\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tfor _, sub := range subs {\n\t\t\t\ts.Remove(sub)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t\tfor i := 0; i < len(r.psubs); i++ {\n\t\t\tsub := r.psubs[i]\n\t\t\tif string(sub.subject) != \"foo\" {\n\t\t\t\tt.Fatalf(\"Subject should be foo, got %s\", sub.subject)\n\t\t\t}\n\t\t}\n\t\twg.Wait()\n\t}\n}\n\nfunc TestSublistRaceOnInsert(t *testing.T) {\n\ttestSublistRaceOnInsert(t, NewSublistWithCache())\n}\n\nfunc TestSublistRaceOnInsertNoCache(t *testing.T) {\n\ttestSublistRaceOnInsert(t, NewSublistNoCache())\n}\n\nfunc testSublistRaceOnInsert(t *testing.T, s *Sublist) {\n\tvar (\n\t\ttotal = 100\n\t\tsubs = make(map[int]*subscription, total) // use map for randomness\n\t\twg sync.WaitGroup\n\t)\n\tfor i := 0; i < total; i++ {\n\t\tsub := newQSub(\"foo\", \"bar\")\n\t\tsubs[i] = sub\n\t}\n\twg.Add(1)\n\tgo func() {\n\t\tfor _, sub := range subs {\n\t\t\ts.Insert(sub)\n\t\t}\n\t\twg.Done()\n\t}()\n\tfor i := 0; i < 1000; i++ {\n\t\tr := s.Match(\"foo\")\n\t\tfor _, qsubs := range r.qsubs {\n\t\t\tfor _, qsub := range qsubs {\n\t\t\t\tif string(qsub.queue) != \"bar\" {\n\t\t\t\t\tt.Fatalf(\"Expected queue name to be bar, got %v\", string(qsub.queue))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n\n\t// Repeat the test with plain subs\n\tfor i := 0; i < total; i++ {\n\t\tsub := newSub(\"foo\")\n\t\tsubs[i] = sub\n\t}\n\twg.Add(1)\n\tgo func() {\n\t\tfor _, sub := range subs {\n\t\t\ts.Insert(sub)\n\t\t}\n\t\twg.Done()\n\t}()\n\tfor i := 0; i < 1000; i++ {\n\t\tr := s.Match(\"foo\")\n\t\tfor _, sub := range r.psubs {\n\t\t\tif string(sub.subject) != \"foo\" {\n\t\t\t\tt.Fatalf(\"Expected subject to be foo, got %v\", string(sub.subject))\n\t\t\t}\n\t\t}\n\t}\n\twg.Wait()\n}\n\nfunc TestSublistRaceOnMatch(t *testing.T) {\n\ts := NewSublistNoCache()\n\ts.Insert(newQSub(\"foo.*\", \"workers\"))\n\ts.Insert(newQSub(\"foo.bar\", \"workers\"))\n\ts.Insert(newSub(\"foo.*\"))\n\ts.Insert(newSub(\"foo.bar\"))\n\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\terrCh := make(chan error, 2)\n\tf := func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tr := s.Match(\"foo.bar\")\n\t\t\tfor _, sub := range r.psubs {\n\t\t\t\tif !strings.HasPrefix(string(sub.subject), \"foo.\") {\n\t\t\t\t\terrCh <- fmt.Errorf(\"Wrong subject: %s\", sub.subject)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, qsub := range r.qsubs {\n\t\t\t\tfor _, sub := range qsub {\n\t\t\t\t\tif string(sub.queue) != \"workers\" {\n\t\t\t\t\t\terrCh <- fmt.Errorf(\"Wrong queue name: %s\", sub.queue)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tgo f()\n\tgo f()\n\twg.Wait()\n\tselect {\n\tcase e := <-errCh:\n\t\tt.Fatal(e.Error())\n\tdefault:\n\t}\n}\n\n// Remote subscriptions for queue subscribers will be weighted such that a single subscription\n// is received, but represents all of the queue subscribers on the remote side.\nfunc TestSublistRemoteQueueSubscriptions(t *testing.T) {\n\ttestSublistRemoteQueueSubscriptions(t, NewSublistWithCache())\n}\n\nfunc TestSublistRemoteQueueSubscriptionsNoCache(t *testing.T) {\n\ttestSublistRemoteQueueSubscriptions(t, NewSublistNoCache())\n}\n\nfunc testSublistRemoteQueueSubscriptions(t *testing.T, s *Sublist) {\n\t// Normals\n\ts1 := newQSub(\"foo\", \"bar\")\n\ts2 := newQSub(\"foo\", \"bar\")\n\ts.Insert(s1)\n\ts.Insert(s2)\n\n\t// Now do weighted remotes.\n\trs1 := newRemoteQSub(\"foo\", \"bar\", 10)\n\ts.Insert(rs1)\n\trs2 := newRemoteQSub(\"foo\", \"bar\", 10)\n\ts.Insert(rs2)\n\n\t// These are just shadowed in results, so should appear as 4 subs.\n\tverifyCount(s, 4, t)\n\n\tr := s.Match(\"foo\")\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 1, t)\n\tverifyLen(r.qsubs[0], 22, t)\n\n\ts.Remove(s1)\n\ts.Remove(rs1)\n\n\tverifyCount(s, 2, t)\n\n\t// Now make sure our shadowed results are correct after a removal.\n\tr = s.Match(\"foo\")\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 1, t)\n\tverifyLen(r.qsubs[0], 11, t)\n\n\t// Now do an update to an existing remote sub to update its weight.\n\trs2.qw = 1\n\ts.UpdateRemoteQSub(rs2)\n\n\t// Results should reflect new weight.\n\tr = s.Match(\"foo\")\n\tverifyLen(r.psubs, 0, t)\n\tverifyQLen(r.qsubs, 1, t)\n\tverifyLen(r.qsubs[0], 2, t)\n}\n\nfunc TestSublistSharedEmptyResult(t *testing.T) {\n\ts := NewSublistWithCache()\n\tr1 := s.Match(\"foo\")\n\tverifyLen(r1.psubs, 0, t)\n\tverifyQLen(r1.qsubs, 0, t)\n\n\tr2 := s.Match(\"bar\")\n\tverifyLen(r2.psubs, 0, t)\n\tverifyQLen(r2.qsubs, 0, t)\n\n\tif r1 != r2 {\n\t\tt.Fatalf(\"Expected empty result to be a shared result set\")\n\t}\n}\n\nfunc TestSublistNoCacheStats(t *testing.T) {\n\ts := NewSublistNoCache()\n\ts.Insert(newSub(\"foo\"))\n\ts.Insert(newSub(\"bar\"))\n\ts.Insert(newSub(\"baz\"))\n\ts.Insert(newSub(\"foo.bar.baz\"))\n\ts.Match(\"a.b.c\")\n\ts.Match(\"bar\")\n\tstats := s.Stats()\n\tif stats.NumCache != 0 {\n\t\tt.Fatalf(\"Expected 0 for NumCache stat, got %d\", stats.NumCache)\n\t}\n}\n\nfunc TestSublistAll(t *testing.T) {\n\ts := NewSublistNoCache()\n\tsubs := []*subscription{\n\t\tnewSub(\"foo.bar.baz\"),\n\t\tnewSub(\"foo\"),\n\t\tnewSub(\"baz\"),\n\t}\n\t// alter client's kind\n\tsubs[0].client.kind = LEAF\n\tfor _, sub := range subs {\n\t\ts.Insert(sub)\n\t}\n\n\tvar buf [32]*subscription\n\toutput := buf[:0]\n\ts.All(&output)\n\tif len(output) != len(subs) {\n\t\tt.Fatalf(\"Expected %d for All, got %d\", len(subs), len(output))\n\t}\n}\n\nfunc TestIsSubsetMatch(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tsubject string\n\t\ttest string\n\t\tresult bool\n\t}{\n\t\t{\"foo.bar\", \"foo.bar\", true},\n\t\t{\"foo.*\", \">\", true},\n\t\t{\"foo.*\", \"*.*\", true},\n\t\t{\"foo.*\", \"foo.*\", true},\n\t\t{\"foo.*\", \"foo.bar\", false},\n\t\t{\"foo.>\", \">\", true},\n\t\t{\"foo.>\", \"*.>\", true},\n\t\t{\"foo.>\", \"foo.>\", true},\n\t\t{\"foo.>\", \"foo.bar\", false},\n\t\t{\"foo..bar\", \"foo.*\", false}, // Bad subject, we return false\n\t\t{\"foo.*\", \"foo..bar\", false}, // Bad subject, we return false\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tif res := subjectIsSubsetMatch(test.subject, test.test); res != test.result {\n\t\t\t\tt.Fatalf(\"Subject %q subset match of %q, should be %v, got %v\",\n\t\t\t\t\ttest.test, test.subject, test.result, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSublistRegisterInterestNotification(t *testing.T) {\n\ts := NewSublistWithCache()\n\tch := make(chan bool, 1)\n\n\texpectErr := func(subject string) {\n\t\tif err := s.RegisterNotification(subject, ch); err != ErrInvalidSubject {\n\t\t\tt.Fatalf(\"Expected err, got %v\", err)\n\t\t}\n\t}\n\n\t// Test that we require a literal subject.\n\texpectErr(\"foo.*\")\n\texpectErr(\">\")\n\n\t// Chan needs to be non-nil\n\tif err := s.RegisterNotification(\"foo\", nil); err != ErrNilChan {\n\t\tt.Fatalf(\"Expected err, got %v\", err)\n\t}\n\n\t// Clearing one that is not there will return false.\n\tif s.ClearNotification(\"foo\", ch) {\n\t\tt.Fatalf(\"Expected to return false on non-existent notification entry\")\n\t}\n\n\t// This should work properly.\n\tif err := s.RegisterNotification(\"foo\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ttt := time.NewTimer(time.Second)\n\texpectBoolWithCh := func(ch chan bool, b bool) {\n\t\tt.Helper()\n\t\ttt.Reset(time.Second)\n\t\tdefer tt.Stop()\n\t\tselect {\n\t\tcase v := <-ch:\n\t\t\tif v != b {\n\t\t\t\tt.Fatalf(\"Expected %v, got %v\", b, v)\n\t\t\t}\n\t\tcase <-tt.C:\n\t\t\tt.Fatalf(\"Timeout waiting for expected value\")\n\t\t}\n\t}\n\texpectBool := func(b bool) {\n\t\tt.Helper()\n\t\texpectBoolWithCh(ch, b)\n\t}\n\texpectFalse := func() {\n\t\tt.Helper()\n\t\texpectBool(false)\n\t}\n\texpectTrue := func() {\n\t\tt.Helper()\n\t\texpectBool(true)\n\t}\n\texpectNone := func() {\n\t\tt.Helper()\n\t\tif lch := len(ch); lch != 0 {\n\t\t\tt.Fatalf(\"Expected no notifications, had %d and first was %v\", lch, <-ch)\n\t\t}\n\t}\n\texpectOneWithCh := func(ch chan bool) {\n\t\tt.Helper()\n\t\tif len(ch) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 notification\")\n\t\t}\n\t}\n\texpectOne := func() {\n\t\tt.Helper()\n\t\texpectOneWithCh(ch)\n\t}\n\n\texpectOne()\n\texpectFalse()\n\tsub := newSub(\"foo\")\n\ts.Insert(sub)\n\texpectTrue()\n\n\tsub2 := newSub(\"foo\")\n\ts.Insert(sub2)\n\texpectNone()\n\n\tif err := s.RegisterNotification(\"bar\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectFalse()\n\n\tsub3 := newSub(\"foo\")\n\ts.Insert(sub3)\n\texpectNone()\n\n\t// Now remove literals.\n\ts.Remove(sub)\n\texpectNone()\n\ts.Remove(sub2)\n\texpectNone()\n\ts.Remove(sub3)\n\texpectFalse()\n\n\tif err := s.RegisterNotification(\"test.node\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectOne()\n\texpectFalse()\n\n\ttnSub1 := newSub(\"test.node.already.exist\")\n\ts.Insert(tnSub1)\n\texpectNone()\n\n\ttnSub2 := newSub(\"test.node\")\n\ts.Insert(tnSub2)\n\texpectTrue()\n\n\ttnSub3 := newSub(\"test.node\")\n\ts.Insert(tnSub3)\n\texpectNone()\n\n\ts.Remove(tnSub1)\n\texpectNone()\n\ts.Remove(tnSub2)\n\texpectNone()\n\ts.Remove(tnSub3)\n\texpectFalse()\n\n\tif !s.ClearNotification(\"test.node\", ch) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\n\tsub4 := newSub(\"bar\")\n\ts.Insert(sub4)\n\texpectTrue()\n\n\tif !s.ClearNotification(\"bar\", ch) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\ts.RLock()\n\tlnr := len(s.notify.remove)\n\ts.RUnlock()\n\tif lnr != 0 {\n\t\tt.Fatalf(\"Expected zero entries for remove notify, got %d\", lnr)\n\t}\n\tif !s.ClearNotification(\"foo\", ch) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\ts.RLock()\n\tnotifyMap := s.notify\n\ts.RUnlock()\n\tif notifyMap != nil {\n\t\tt.Fatalf(\"Expected the notify map to be nil\")\n\t}\n\n\t// Let's do some wildcard checks.\n\t// Wildcards will not trigger interest.\n\tsubpwc := newSub(\"*\")\n\ts.Insert(subpwc)\n\texpectNone()\n\n\tif err := s.RegisterNotification(\"foo\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectFalse()\n\n\ts.Insert(sub)\n\texpectTrue()\n\n\ts.Remove(sub)\n\texpectFalse()\n\n\ts.Remove(subpwc)\n\texpectNone()\n\n\tsubfwc := newSub(\">\")\n\ts.Insert(subfwc)\n\texpectNone()\n\n\ts.Insert(subpwc)\n\texpectNone()\n\n\ts.Remove(subpwc)\n\texpectNone()\n\n\ts.Remove(subfwc)\n\texpectNone()\n\n\t// Test batch\n\tsubs := []*subscription{sub, sub2, sub3, sub4, subpwc, subfwc}\n\tfor _, sub := range subs {\n\t\ts.Insert(sub)\n\t}\n\texpectTrue()\n\n\ts.RemoveBatch(subs)\n\texpectOne()\n\texpectFalse()\n\n\t// Test queue subs\n\t// We know make sure that if you have qualified a queue group it has to match, etc.\n\t// Also if you do not specify one they will not trigger.\n\tqsub := newQSub(\"foo.bar.baz\", \"1\")\n\ts.Insert(qsub)\n\texpectNone()\n\n\tif err := s.RegisterNotification(\"foo.bar.baz\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectFalse()\n\n\twcqsub := newQSub(\"foo.bar.>\", \"1\")\n\ts.Insert(wcqsub)\n\texpectNone()\n\n\ts.Remove(qsub)\n\texpectNone()\n\n\ts.Remove(wcqsub)\n\texpectNone()\n\n\ts.Insert(wcqsub)\n\texpectNone()\n\n\tif err := s.RegisterQueueNotification(\"queue.test.node\", \"q22\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectOne()\n\texpectFalse()\n\n\tqsub1 := newQSub(\"queue.test.node.already.exist\", \"queue\")\n\ts.Insert(qsub1)\n\texpectNone()\n\n\tqsub2 := newQSub(\"queue.test.node\", \"q22\")\n\ts.Insert(qsub2)\n\texpectTrue()\n\n\tqsub3 := newQSub(\"queue.test.node\", \"otherqueue\")\n\ts.Insert(qsub3)\n\texpectNone()\n\n\tqsub4 := newQSub(\"queue.different.node\", \"q22\")\n\ts.Insert(qsub4)\n\texpectNone()\n\n\tqsub5 := newQSub(\"queue.test.node\", \"q22\")\n\ts.Insert(qsub5)\n\texpectNone()\n\n\ts.Remove(qsub3)\n\texpectNone()\n\ts.Remove(qsub1)\n\texpectNone()\n\ts.Remove(qsub2)\n\texpectNone()\n\ts.Remove(qsub4)\n\texpectNone()\n\ts.Remove(qsub5)\n\texpectFalse()\n\n\tif !s.ClearQueueNotification(\"queue.test.node\", \"q22\", ch) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\n\tif err := s.RegisterQueueNotification(\"some.subject\", \"queue1\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectOne()\n\texpectFalse()\n\n\tqsub1 = newQSub(\"some.subject\", \"queue1\")\n\ts.Insert(qsub1)\n\texpectTrue()\n\n\t// Create a second channel for this other queue\n\tch2 := make(chan bool, 1)\n\tif err := s.RegisterQueueNotification(\"some.subject\", \"queue2\", ch2); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\texpectOneWithCh(ch2)\n\texpectBoolWithCh(ch2, false)\n\n\tqsub2 = newQSub(\"some.subject\", \"queue2\")\n\ts.Insert(qsub2)\n\texpectBoolWithCh(ch2, true)\n\n\t// But we should not get notification on queue1\n\texpectNone()\n\n\ts.Remove(qsub1)\n\texpectFalse()\n\ts.Remove(qsub2)\n\texpectBoolWithCh(ch2, false)\n\n\tif !s.ClearQueueNotification(\"some.subject\", \"queue1\", ch) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\tif !s.ClearQueueNotification(\"some.subject\", \"queue2\", ch2) {\n\t\tt.Fatalf(\"Expected to return true\")\n\t}\n\n\t// Test non-blocking notifications.\n\tif err := s.RegisterNotification(\"bar\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tif err := s.RegisterNotification(\"baz\", ch); err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\ts.Insert(newSub(\"baz\"))\n\ts.Insert(newSub(\"bar\"))\n\ts.Insert(subpwc)\n\texpectOne()\n\texpectFalse()\n}\n\nfunc TestSublistReverseMatch(t *testing.T) {\n\ts := NewSublistWithCache()\n\tfooSub := newSub(\"foo\")\n\tbarSub := newSub(\"bar\")\n\tfooBarSub := newSub(\"foo.bar\")\n\tfooBazSub := newSub(\"foo.baz\")\n\tfooBarBazSub := newSub(\"foo.bar.baz\")\n\ts.Insert(fooSub)\n\ts.Insert(barSub)\n\ts.Insert(fooBarSub)\n\ts.Insert(fooBazSub)\n\ts.Insert(fooBarBazSub)\n\n\tr := s.ReverseMatch(\"foo\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, fooSub, t)\n\n\tr = s.ReverseMatch(\"bar\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, barSub, t)\n\n\tr = s.ReverseMatch(\"*\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, fooSub, t)\n\tverifyMember(r.psubs, barSub, t)\n\n\tr = s.ReverseMatch(\"baz\")\n\tverifyLen(r.psubs, 0, t)\n\n\tr = s.ReverseMatch(\"foo.*\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, fooBarSub, t)\n\tverifyMember(r.psubs, fooBazSub, t)\n\n\tr = s.ReverseMatch(\"*.*\")\n\tverifyLen(r.psubs, 2, t)\n\tverifyMember(r.psubs, fooBarSub, t)\n\tverifyMember(r.psubs, fooBazSub, t)\n\n\tr = s.ReverseMatch(\"*.bar\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, fooBarSub, t)\n\n\tr = s.ReverseMatch(\"*.baz\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, fooBazSub, t)\n\n\tr = s.ReverseMatch(\"bar.*\")\n\tverifyLen(r.psubs, 0, t)\n\n\tr = s.ReverseMatch(\"*.bat\")\n\tverifyLen(r.psubs, 0, t)\n\n\tr = s.ReverseMatch(\"foo.>\")\n\tverifyLen(r.psubs, 3, t)\n\tverifyMember(r.psubs, fooBarSub, t)\n\tverifyMember(r.psubs, fooBazSub, t)\n\tverifyMember(r.psubs, fooBarBazSub, t)\n\n\tr = s.ReverseMatch(\">\")\n\tverifyLen(r.psubs, 5, t)\n\tverifyMember(r.psubs, fooSub, t)\n\tverifyMember(r.psubs, barSub, t)\n\tverifyMember(r.psubs, fooBarSub, t)\n\tverifyMember(r.psubs, fooBazSub, t)\n\tverifyMember(r.psubs, fooBarBazSub, t)\n}\n\nfunc TestSublistReverseMatchWider(t *testing.T) {\n\ts := NewSublistWithCache()\n\tsub := newSub(\"uplink.*.*.>\")\n\ts.Insert(sub)\n\n\tr := s.ReverseMatch(\"uplink.1.*.*.>\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub, t)\n\n\tr = s.ReverseMatch(\"uplink.1.2.3.>\")\n\tverifyLen(r.psubs, 1, t)\n\tverifyMember(r.psubs, sub, t)\n}\n\nfunc TestSublistMatchWithEmptyTokens(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcache bool\n\t}{\n\t\t{\"cache\", true},\n\t\t{\"no cache\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsl := NewSublist(true)\n\t\t\tsub1 := newSub(\">\")\n\t\t\tsub2 := newQSub(\">\", \"queue\")\n\t\t\tsl.Insert(sub1)\n\t\t\tsl.Insert(sub2)\n\n\t\t\tfor _, subj := range []string{\".foo\", \"..foo\", \"foo..\", \"foo.\", \"foo..bar\", \"foo...bar\"} {\n\t\t\t\tt.Run(subj, func(t *testing.T) {\n\t\t\t\t\tr := sl.Match(subj)\n\t\t\t\t\tverifyLen(r.psubs, 0, t)\n\t\t\t\t\tverifyQLen(r.qsubs, 0, t)\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSublistSubjectCollide(t *testing.T) {\n\trequire_False(t, SubjectsCollide(\"foo.*\", \"foo.*.bar.>\"))\n\trequire_False(t, SubjectsCollide(\"foo.*.bar.>\", \"foo.*\"))\n\trequire_True(t, SubjectsCollide(\"foo.*\", \"foo.foo\"))\n\trequire_True(t, SubjectsCollide(\"foo.*\", \"*.foo\"))\n\trequire_True(t, SubjectsCollide(\"foo.bar.>\", \"*.bar.foo\"))\n}\n\nfunc TestSublistAddCacheHitRate(t *testing.T) {\n\tsl1 := NewSublistWithCache()\n\tfooSub := newSub(\"foo\")\n\tsl1.Insert(fooSub)\n\tfor i := 0; i < 4; i++ {\n\t\tsl1.Match(\"foo\")\n\t}\n\tstats1 := sl1.Stats()\n\trequire_True(t, stats1.CacheHitRate == 0.75)\n\n\tsl2 := NewSublistWithCache()\n\tbarSub := newSub(\"bar\")\n\tsl2.Insert(barSub)\n\tfor i := 0; i < 4; i++ {\n\t\tsl2.Match(\"bar\")\n\t}\n\tstats2 := sl2.Stats()\n\trequire_True(t, stats2.CacheHitRate == 0.75)\n\n\tts := &SublistStats{}\n\tts.add(stats1)\n\tts.add(stats2)\n\trequire_True(t, ts.CacheHitRate == 0.75)\n}\n\n// -- Benchmarks Setup --\n\nvar benchSublistSubs []*subscription\nvar benchSublistSl = NewSublistWithCache()\n\n// https://github.com/golang/go/issues/31859\nfunc TestMain(m *testing.M) {\n\tflag.StringVar(&testDefaultClusterCompression, \"cluster_compression\", _EMPTY_, \"Test with this compression level as the default\")\n\tflag.StringVar(&testDefaultLeafNodeCompression, \"leafnode_compression\", _EMPTY_, \"Test with this compression level as the default\")\n\tflag.Parse()\n\tinitSublist := false\n\tflag.Visit(func(f *flag.Flag) {\n\t\tif f.Name == \"test.bench\" {\n\t\t\tinitSublist = true\n\t\t}\n\t})\n\tif initSublist {\n\t\tbenchSublistSubs = make([]*subscription, 0, 256*1024)\n\t\ttoks := []string{\"synadia\", \"nats\", \"jetstream\", \"nkeys\", \"jwt\", \"deny\", \"auth\", \"drain\"}\n\t\tsubsInit(\"\", toks)\n\t\tfor i := 0; i < len(benchSublistSubs); i++ {\n\t\t\tbenchSublistSl.Insert(benchSublistSubs[i])\n\t\t}\n\t\taddWildcards()\n\t}\n\tos.Exit(m.Run())\n}\n\nfunc TestSublistHasInterest(t *testing.T) {\n\tsl := NewSublistWithCache()\n\tfooSub := newSub(\"foo\")\n\tsl.Insert(fooSub)\n\n\t// Expect to find that \"foo\" matches but \"bar\" doesn't.\n\t// At this point nothing should be in the cache.\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"bar\"))\n\trequire_Equal(t, sl.cacheHits, 0)\n\n\t// Now call Match(), which will populate the cache.\n\tsl.Match(\"foo\")\n\trequire_Equal(t, sl.cacheHits, 0)\n\n\t// Future calls to HasInterest() should hit the cache now.\n\tfor i := uint64(1); i <= 5; i++ {\n\t\trequire_True(t, sl.HasInterest(\"foo\"))\n\t\trequire_Equal(t, sl.cacheHits, i)\n\t}\n\n\t// Call Match on a subject we know there is no match.\n\tsl.Match(\"bar\")\n\trequire_False(t, sl.HasInterest(\"bar\"))\n\n\t// Remove fooSub and check interest again\n\tsl.Remove(fooSub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\n\t// Try with some wildcards\n\tsub := newSub(\"foo.*\")\n\tsl.Insert(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove sub, there should be no interest\n\tsl.Remove(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\tsub = newSub(\"foo.>\")\n\tsl.Insert(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\tsl.Remove(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\tsub = newSub(\"*.>\")\n\tsl.Insert(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.baz\"))\n\tsl.Remove(sub)\n\n\tsub = newSub(\"*.bar\")\n\tsl.Insert(sub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.baz\"))\n\tsl.Remove(sub)\n\n\tsub = newSub(\"*\")\n\tsl.Insert(sub)\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\tsl.Remove(sub)\n\n\t// Try with queues now.\n\tqsub := newQSub(\"foo\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\n\tqsub2 := newQSub(\"foo\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\n\t// Remove last.\n\tsl.Remove(qsub2)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\n\t// With wildcards now\n\tqsub = newQSub(\"foo.*\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Add another queue to the group\n\tqsub2 = newQSub(\"foo.*\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove last\n\tsl.Remove(qsub2)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\tqsub = newQSub(\"foo.>\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Add another queue to the group\n\tqsub2 = newQSub(\"foo.>\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\t// Remove last\n\tsl.Remove(qsub2)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar.baz\"))\n\n\tqsub = newQSub(\"*.>\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_True(t, sl.HasInterest(\"foo.baz\"))\n\tsl.Remove(qsub)\n\n\tqsub = newQSub(\"*.bar\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_False(t, sl.HasInterest(\"foo\"))\n\trequire_True(t, sl.HasInterest(\"foo.bar\"))\n\trequire_False(t, sl.HasInterest(\"foo.baz\"))\n\tsl.Remove(qsub)\n\n\tqsub = newQSub(\"*\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_True(t, sl.HasInterest(\"foo\"))\n\trequire_False(t, sl.HasInterest(\"foo.bar\"))\n\tsl.Remove(qsub)\n}\n\nfunc TestSublistHasInterestOverlapping(t *testing.T) {\n\tsl := NewSublistWithCache()\n\trequire_NoError(t, sl.Insert(newSub(\"stream.A.child\")))\n\trequire_NoError(t, sl.Insert(newSub(\"stream.*\")))\n\trequire_True(t, sl.HasInterest(\"stream.A.child\"))\n\trequire_True(t, sl.HasInterest(\"stream.A\"))\n}\n\nfunc TestSublistNumInterest(t *testing.T) {\n\tsl := NewSublistWithCache()\n\tfooSub := newSub(\"foo\")\n\tsl.Insert(fooSub)\n\n\trequire_NumInterest := func(t *testing.T, subj string, wnp, wnq int) {\n\t\tt.Helper()\n\t\tnp, nq := sl.NumInterest(subj)\n\t\trequire_Equal(t, np, wnp)\n\t\trequire_Equal(t, nq, wnq)\n\t}\n\n\t// Expect to find that \"foo\" matches but \"bar\" doesn't.\n\t// At this point nothing should be in the cache.\n\trequire_NumInterest(t, \"foo\", 1, 0)\n\trequire_NumInterest(t, \"bar\", 0, 0)\n\trequire_Equal(t, sl.cacheHits, 0)\n\n\t// Now call Match(), which will populate the cache.\n\tsl.Match(\"foo\")\n\trequire_Equal(t, sl.cacheHits, 0)\n\n\t// Future calls to HasInterest() should hit the cache now.\n\tfor i := uint64(1); i <= 5; i++ {\n\t\trequire_NumInterest(t, \"foo\", 1, 0)\n\t\trequire_Equal(t, sl.cacheHits, i)\n\t}\n\n\t// Call Match on a subject we know there is no match.\n\tsl.Match(\"bar\")\n\trequire_NumInterest(t, \"bar\", 0, 0)\n\n\t// Remove fooSub and check interest again\n\tsl.Remove(fooSub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\n\t// Try with some wildcards\n\tsub := newSub(\"foo.*\")\n\tsl.Insert(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 1, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// Remove sub, there should be no interest\n\tsl.Remove(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\tsub = newSub(\"foo.>\")\n\tsl.Insert(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 1, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 1, 0)\n\n\tsl.Remove(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\tsub = newSub(\"*.>\")\n\tsl.Insert(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 1, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 1, 0)\n\tsl.Remove(sub)\n\n\tsub = newSub(\"*.bar\")\n\tsl.Insert(sub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 1, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\tsl.Remove(sub)\n\n\tsub = newSub(\"*\")\n\tsl.Insert(sub)\n\trequire_NumInterest(t, \"foo\", 1, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\tsl.Remove(sub)\n\n\t// Try with queues now.\n\tqsub := newQSub(\"foo\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\tqsub2 := newQSub(\"foo\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\t// Add a second qsub to the second queue group\n\tqsub3 := newQSub(\"foo\", \"baz\")\n\tsl.Insert(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 3)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\t// Remove second\n\tsl.Remove(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\t// Remove last.\n\tsl.Remove(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\n\t// With wildcards now\n\tqsub = newQSub(\"foo.*\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// Add another queue to the group\n\tqsub2 = newQSub(\"foo.*\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\tqsub3 = newQSub(\"foo.*\", \"baz\")\n\tsl.Insert(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 3)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// Remove second\n\tsl.Remove(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// Remove last\n\tsl.Remove(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\t// With > wildcard\n\tqsub = newQSub(\"foo.>\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 1)\n\n\t// Add another queue to the group\n\tqsub2 = newQSub(\"foo.>\", \"baz\")\n\tsl.Insert(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 2)\n\n\t// Add another queue to second group.\n\tqsub3 = newQSub(\"foo.>\", \"baz\")\n\tsl.Insert(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 3)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 3)\n\n\t// Remove first queue\n\tsl.Remove(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 2)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 2)\n\n\t// Remove second\n\tsl.Remove(qsub2)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 1)\n\n\t// Remove last\n\tsl.Remove(qsub3)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\n\tqsub = newQSub(\"*.>\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 1)\n\tsl.Remove(qsub)\n\n\tqsub = newQSub(\"*.bar\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 0)\n\trequire_NumInterest(t, \"foo.bar\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar.baz\", 0, 0)\n\tsl.Remove(qsub)\n\n\tqsub = newQSub(\"*\", \"bar\")\n\tsl.Insert(qsub)\n\trequire_NumInterest(t, \"foo\", 0, 1)\n\trequire_NumInterest(t, \"foo.bar\", 0, 0)\n\tsl.Remove(qsub)\n}\n\nfunc subsInit(pre string, toks []string) {\n\tvar sub string\n\tfor _, t := range toks {\n\t\tif len(pre) > 0 {\n\t\t\tsub = pre + tsep + t\n\t\t} else {\n\t\t\tsub = t\n\t\t}\n\t\tbenchSublistSubs = append(benchSublistSubs, newSub(sub))\n\t\tif len(strings.Split(sub, tsep)) < 5 {\n\t\t\tsubsInit(sub, toks)\n\t\t}\n\t}\n}\n\nfunc addWildcards() {\n\tbenchSublistSl.Insert(newSub(\"cloud.>\"))\n\tbenchSublistSl.Insert(newSub(\"cloud.nats.component.>\"))\n\tbenchSublistSl.Insert(newSub(\"cloud.*.*.nkeys.*\"))\n}\n\n// -- Benchmarks Setup End --\n\nfunc Benchmark______________________SublistInsert(b *testing.B) {\n\ts := NewSublistWithCache()\n\tfor i, l := 0, len(benchSublistSubs); i < b.N; i++ {\n\t\tindex := i % l\n\t\ts.Insert(benchSublistSubs[index])\n\t}\n}\n\nfunc Benchmark_______________SublistInsertNoCache(b *testing.B) {\n\ts := NewSublistNoCache()\n\tfor i, l := 0, len(benchSublistSubs); i < b.N; i++ {\n\t\tindex := i % l\n\t\ts.Insert(benchSublistSubs[index])\n\t}\n}\n\nfunc benchSublistTokens(b *testing.B, tokens string) {\n\tfor i := 0; i < b.N; i++ {\n\t\tbenchSublistSl.Match(tokens)\n\t}\n}\n\nfunc Benchmark____________SublistMatchSingleToken(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia\")\n}\n\nfunc Benchmark______________SublistMatchTwoTokens(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia.nats\")\n}\n\nfunc Benchmark____________SublistMatchThreeTokens(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia.nats.jetstream\")\n}\n\nfunc Benchmark_____________SublistMatchFourTokens(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia.nats.jetstream.nkeys\")\n}\n\nfunc Benchmark_SublistMatchFourTokensSingleResult(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia.nats.jetstream.nkeys\")\n}\n\nfunc Benchmark_SublistMatchFourTokensMultiResults(b *testing.B) {\n\tbenchSublistTokens(b, \"cloud.nats.component.router\")\n}\n\nfunc Benchmark_______SublistMissOnLastTokenOfFive(b *testing.B) {\n\tbenchSublistTokens(b, \"synadia.nats.jetstream.nkeys.ZZZZ\")\n}\n\nfunc multiRead(b *testing.B, num int) {\n\tvar swg, fwg sync.WaitGroup\n\tswg.Add(num)\n\tfwg.Add(num)\n\ts := \"synadia.nats.jetstream.nkeys\"\n\tfor i := 0; i < num; i++ {\n\t\tgo func() {\n\t\t\tswg.Done()\n\t\t\tswg.Wait()\n\t\t\tn := b.N / num\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tbenchSublistSl.Match(s)\n\t\t\t}\n\t\t\tfwg.Done()\n\t\t}()\n\t}\n\tswg.Wait()\n\tb.ResetTimer()\n\tfwg.Wait()\n}\n\nfunc Benchmark____________Sublist10XMultipleReads(b *testing.B) {\n\tmultiRead(b, 10)\n}\n\nfunc Benchmark___________Sublist100XMultipleReads(b *testing.B) {\n\tmultiRead(b, 100)\n}\n\nfunc Benchmark__________Sublist1000XMultipleReads(b *testing.B) {\n\tmultiRead(b, 1000)\n}\n\nfunc Benchmark________________SublistMatchLiteral(b *testing.B) {\n\tcachedSubj := \"foo.foo.foo.foo.foo.foo.foo.foo.foo.foo\"\n\tsubjects := []string{\n\t\t\"foo.foo.foo.foo.foo.foo.foo.foo.foo.foo\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.foo.>\",\n\t\t\"foo.foo.foo.>\",\n\t\t\"foo.foo.>\",\n\t\t\"foo.>\",\n\t\t\">\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.foo.foo.*\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.foo.*.*\",\n\t\t\"foo.foo.foo.foo.foo.foo.foo.*.*.*\",\n\t\t\"foo.foo.foo.foo.foo.foo.*.*.*.*\",\n\t\t\"foo.foo.foo.foo.foo.*.*.*.*.*\",\n\t\t\"foo.foo.foo.foo.*.*.*.*.*.*\",\n\t\t\"foo.foo.foo.*.*.*.*.*.*.*\",\n\t\t\"foo.foo.*.*.*.*.*.*.*.*\",\n\t\t\"foo.*.*.*.*.*.*.*.*.*\",\n\t\t\"*.*.*.*.*.*.*.*.*.*\",\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, subject := range subjects {\n\t\t\tif !matchLiteral(cachedSubj, subject) {\n\t\t\t\tb.Fatalf(\"Subject %q no match with %q\", cachedSubj, subject)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc Benchmark_____SublistMatch10kSubsWithNoCache(b *testing.B) {\n\tvar nsubs = 512\n\ts := NewSublistNoCache()\n\tsubject := \"foo\"\n\tfor i := 0; i < nsubs; i++ {\n\t\ts.Insert(newSub(subject))\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tr := s.Match(subject)\n\t\tif len(r.psubs) != nsubs {\n\t\t\tb.Fatalf(\"Results len is %d, should be %d\", len(r.psubs), nsubs)\n\t\t}\n\t}\n}\n\nfunc removeTest(b *testing.B, singleSubject, doBatch bool, qgroup string) {\n\ts := NewSublistWithCache()\n\tsubject := \"foo\"\n\n\tsubs := make([]*subscription, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tvar sub *subscription\n\t\tif singleSubject {\n\t\t\tsub = newQSub(subject, qgroup)\n\t\t} else {\n\t\t\tsub = newQSub(fmt.Sprintf(\"%s.%d\\n\", subject, i), qgroup)\n\t\t}\n\t\ts.Insert(sub)\n\t\tsubs = append(subs, sub)\n\t}\n\n\t// Actual test on Remove\n\tb.ResetTimer()\n\tif doBatch {\n\t\ts.RemoveBatch(subs)\n\t} else {\n\t\tfor _, sub := range subs {\n\t\t\ts.Remove(sub)\n\t\t}\n\t}\n}\n\nfunc Benchmark__________SublistRemove1TokenSingle(b *testing.B) {\n\tremoveTest(b, true, false, \"\")\n}\n\nfunc Benchmark___________SublistRemove1TokenBatch(b *testing.B) {\n\tremoveTest(b, true, true, \"\")\n}\n\nfunc Benchmark_________SublistRemove2TokensSingle(b *testing.B) {\n\tremoveTest(b, false, false, \"\")\n}\n\nfunc Benchmark__________SublistRemove2TokensBatch(b *testing.B) {\n\tremoveTest(b, false, true, \"\")\n}\n\nfunc Benchmark________SublistRemove1TokenQGSingle(b *testing.B) {\n\tremoveTest(b, true, false, \"bar\")\n}\n\nfunc Benchmark_________SublistRemove1TokenQGBatch(b *testing.B) {\n\tremoveTest(b, true, true, \"bar\")\n}\n\nfunc removeMultiTest(b *testing.B, singleSubject, doBatch bool) {\n\ts := NewSublistWithCache()\n\tsubject := \"foo\"\n\tvar swg, fwg sync.WaitGroup\n\tswg.Add(b.N)\n\tfwg.Add(b.N)\n\n\t// We will have b.N go routines each with 1k subscriptions.\n\tsc := 1000\n\n\tfor i := 0; i < b.N; i++ {\n\t\tgo func() {\n\t\t\tsubs := make([]*subscription, 0, sc)\n\t\t\tfor n := 0; n < sc; n++ {\n\t\t\t\tvar sub *subscription\n\t\t\t\tif singleSubject {\n\t\t\t\t\tsub = newSub(subject)\n\t\t\t\t} else {\n\t\t\t\t\tsub = newSub(fmt.Sprintf(\"%s.%d\\n\", subject, n))\n\t\t\t\t}\n\t\t\t\ts.Insert(sub)\n\t\t\t\tsubs = append(subs, sub)\n\t\t\t}\n\t\t\t// Wait to start test\n\t\t\tswg.Done()\n\t\t\tswg.Wait()\n\t\t\t// Actual test on Remove\n\t\t\tif doBatch {\n\t\t\t\ts.RemoveBatch(subs)\n\t\t\t} else {\n\t\t\t\tfor _, sub := range subs {\n\t\t\t\t\ts.Remove(sub)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfwg.Done()\n\t\t}()\n\t}\n\tswg.Wait()\n\tb.ResetTimer()\n\tfwg.Wait()\n}\n\n// Check contention rates for remove from multiple Go routines.\n// Reason for BatchRemove.\nfunc Benchmark_________SublistRemove1kSingleMulti(b *testing.B) {\n\tremoveMultiTest(b, true, false)\n}\n\n// Batch version\nfunc Benchmark__________SublistRemove1kBatchMulti(b *testing.B) {\n\tremoveMultiTest(b, true, true)\n}\n\nfunc Benchmark__SublistRemove1kSingle2TokensMulti(b *testing.B) {\n\tremoveMultiTest(b, false, false)\n}\n\n// Batch version\nfunc Benchmark___SublistRemove1kBatch2TokensMulti(b *testing.B) {\n\tremoveMultiTest(b, false, true)\n}\n\n// Cache contention tests\nfunc cacheContentionTest(b *testing.B, numMatchers, numAdders, numRemovers int) {\n\tvar swg, fwg, mwg sync.WaitGroup\n\ttotal := numMatchers + numAdders + numRemovers\n\tswg.Add(total)\n\tfwg.Add(total)\n\tmwg.Add(numMatchers)\n\n\tmu := sync.RWMutex{}\n\tsubs := make([]*subscription, 0, 8192)\n\n\tquitCh := make(chan struct{})\n\n\t// Set up a new sublist. subjects will be foo.bar.baz.N\n\ts := NewSublistWithCache()\n\tmu.Lock()\n\tfor i := 0; i < 10000; i++ {\n\t\tsub := newSub(fmt.Sprintf(\"foo.bar.baz.%d\", i))\n\t\ts.Insert(sub)\n\t\tsubs = append(subs, sub)\n\t}\n\tmu.Unlock()\n\n\t// Now warm up the cache\n\tfor i := 0; i < slCacheMax; i++ {\n\t\ts.Match(fmt.Sprintf(\"foo.bar.baz.%d\", i))\n\t}\n\n\t// Setup go routines.\n\n\t// Adders\n\tfor i := 0; i < numAdders; i++ {\n\t\tgo func() {\n\t\t\tswg.Done()\n\t\t\tswg.Wait()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-quitCh:\n\t\t\t\t\tfwg.Done()\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\tnext := len(subs)\n\t\t\t\t\tsubj := \"foo.bar.baz.\" + strconv.FormatInt(int64(next), 10)\n\t\t\t\t\tsub := newSub(subj)\n\t\t\t\t\tsubs = append(subs, sub)\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\ts.Insert(sub)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Removers\n\tfor i := 0; i < numRemovers; i++ {\n\t\tgo func() {\n\t\t\tprand := rand.New(rand.NewSource(time.Now().UnixNano()))\n\t\t\tswg.Done()\n\t\t\tswg.Wait()\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-quitCh:\n\t\t\t\t\tfwg.Done()\n\t\t\t\t\treturn\n\t\t\t\tdefault:\n\t\t\t\t\tmu.RLock()\n\t\t\t\t\tlh := len(subs) - 1\n\t\t\t\t\tindex := prand.Intn(lh)\n\t\t\t\t\tsub := subs[index]\n\t\t\t\t\tmu.RUnlock()\n\t\t\t\t\ts.Remove(sub)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Matchers\n\tfor i := 0; i < numMatchers; i++ {\n\t\tgo func() {\n\t\t\tid := nuid.New()\n\t\t\tswg.Done()\n\t\t\tswg.Wait()\n\n\t\t\t// We will miss on purpose to blow the cache.\n\t\t\tn := b.N / numMatchers\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tsubj := \"foo.bar.baz.\" + id.Next()\n\t\t\t\ts.Match(subj)\n\t\t\t}\n\t\t\tmwg.Done()\n\t\t\tfwg.Done()\n\t\t}()\n\t}\n\n\tswg.Wait()\n\tb.ResetTimer()\n\tmwg.Wait()\n\tb.StopTimer()\n\tclose(quitCh)\n\tfwg.Wait()\n}\n\nfunc Benchmark____SublistCacheContention10M10A10R(b *testing.B) {\n\tcacheContentionTest(b, 10, 10, 10)\n}\n\nfunc Benchmark_SublistCacheContention100M100A100R(b *testing.B) {\n\tcacheContentionTest(b, 100, 100, 100)\n}\n\nfunc Benchmark____SublistCacheContention1kM1kA1kR(b *testing.B) {\n\tcacheContentionTest(b, 1024, 1024, 1024)\n}\n\nfunc Benchmark_SublistCacheContention10kM10kA10kR(b *testing.B) {\n\tcacheContentionTest(b, 10*1024, 10*1024, 10*1024)\n}\n\nfunc Benchmark______________IsValidLiteralSubject(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tIsValidLiteralSubject(\"foo.bar.baz.22\")\n\t}\n}\n\nfunc Benchmark___________________subjectIsLiteral(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tsubjectIsLiteral(\"foo.bar.baz.22\")\n\t}\n}\n\n// Test to determine the fastest way to match a filter\n// with wildcards against a literal subject.\nfunc Benchmark_SubjectFilterMatchers(b *testing.B) {\n\tcases := []struct {\n\t\tname string\n\t\tsubject string\n\t\tfilter string\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tname: \"short_single_wc_match\",\n\t\t\tsubject: \"foo.baz.12345\",\n\t\t\tfilter: \"foo.baz.*\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"short_mixed_wc_match\",\n\t\t\tsubject: \"foo.baz.12345\",\n\t\t\tfilter: \"foo.*.>\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"long_single_wc_match\",\n\t\t\tsubject: \"foo.alpha.beta.gamma.delta.epsilon.zeta.12345\",\n\t\t\tfilter: \"foo.alpha.beta.gamma.delta.epsilon.zeta.*\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"long_many_wc_late_mismatch\",\n\t\t\tsubject: \"foo.alpha.beta.gamma.delta.epsilon.zeta.12345\",\n\t\t\tfilter: \"foo.*.*.*.*.*.*.99999\",\n\t\t\twant: false,\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tb.Run(tc.name, func(b *testing.B) {\n\t\t\tb.Run(\"subjectIsSubsetMatch\", func(b *testing.B) {\n\t\t\t\tvar matched bool\n\t\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t\tmatched = subjectIsSubsetMatch(tc.subject, tc.filter)\n\t\t\t\t}\n\t\t\t\tif matched != tc.want {\n\t\t\t\t\tb.Fatalf(\"unexpected result: got %v, want %v\", matched, tc.want)\n\t\t\t\t}\n\t\t\t})\n\t\t\tb.Run(\"matchLiteral\", func(b *testing.B) {\n\t\t\t\tvar matched bool\n\t\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t\tmatched = matchLiteral(tc.subject, tc.filter)\n\t\t\t\t}\n\t\t\t\tif matched != tc.want {\n\t\t\t\t\tb.Fatalf(\"unexpected result: got %v, want %v\", matched, tc.want)\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n" }, { "path": "server/sysmem/mem_bsd.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build freebsd || openbsd || dragonfly || netbsd\n\npackage sysmem\n\nfunc Memory() int64 {\n\treturn sysctlInt64(\"hw.physmem\")\n}\n" }, { "path": "server/sysmem/mem_darwin.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin\n\npackage sysmem\n\nfunc Memory() int64 {\n\treturn sysctlInt64(\"hw.memsize\")\n}\n" }, { "path": "server/sysmem/mem_linux.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build linux\n\npackage sysmem\n\nimport \"syscall\"\n\nfunc Memory() int64 {\n\tvar info syscall.Sysinfo_t\n\terr := syscall.Sysinfo(&info)\n\tif err != nil {\n\t\treturn 0\n\t}\n\treturn int64(info.Totalram) * int64(info.Unit)\n}\n" }, { "path": "server/sysmem/mem_solaris.go", "content": "// Copyright 2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build illumos || solaris\n\npackage sysmem\n\nimport (\n\t\"golang.org/x/sys/unix\"\n)\n\nconst (\n\t_SC_PHYS_PAGES = 500\n\t_SC_PAGESIZE = 11\n)\n\nfunc Memory() int64 {\n\tpages, err := unix.Sysconf(_SC_PHYS_PAGES)\n\tif err != nil {\n\t\treturn 0\n\t}\n\tpageSize, err := unix.Sysconf(_SC_PAGESIZE)\n\tif err != nil {\n\t\treturn 0\n\t}\n\treturn int64(pages) * int64(pageSize)\n}\n" }, { "path": "server/sysmem/mem_wasm.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build wasm\n\npackage sysmem\n\nfunc Memory() int64 {\n\t// TODO: We don't know the system memory\n\treturn 0\n}\n" }, { "path": "server/sysmem/mem_windows.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage sysmem\n\nimport (\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\nvar winKernel32 = windows.NewLazySystemDLL(\"kernel32.dll\")\nvar winGlobalMemoryStatusEx = winKernel32.NewProc(\"GlobalMemoryStatusEx\")\n\nfunc init() {\n\tif err := winKernel32.Load(); err != nil {\n\t\tpanic(err)\n\t}\n\tif err := winGlobalMemoryStatusEx.Find(); err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex\ntype _memoryStatusEx struct {\n\tdwLength uint32\n\tdwMemoryLoad uint32\n\tullTotalPhys uint64\n\tunused [6]uint64 // ignore rest of struct\n}\n\nfunc Memory() int64 {\n\tmsx := &_memoryStatusEx{dwLength: 64}\n\tres, _, _ := winGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))\n\tif res == 0 {\n\t\treturn 0\n\t}\n\treturn int64(msx.ullTotalPhys)\n}\n" }, { "path": "server/sysmem/mem_zos.go", "content": "// Copyright 2022-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build zos\n\npackage sysmem\n\nfunc Memory() int64 {\n\t// TODO: We don't know the system memory\n\treturn 0\n}\n" }, { "path": "server/sysmem/sysctl.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || freebsd || openbsd || dragonfly || netbsd\n\npackage sysmem\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nfunc sysctlInt64(name string) int64 {\n\ts, err := syscall.Sysctl(name)\n\tif err != nil {\n\t\treturn 0\n\t}\n\t// Make sure it's 8 bytes when we do the cast below.\n\t// We were getting fatal error: checkptr: converted pointer straddles multiple allocations in go 1.22.1 on darwin.\n\tvar b [8]byte\n\tcopy(b[:], s)\n\treturn *(*int64)(unsafe.Pointer(&b[0]))\n}\n" }, { "path": "server/test_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"cmp\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/antithesis\"\n)\n\n// DefaultTestOptions are default options for the unit tests.\nvar DefaultTestOptions = Options{\n\tHost: \"127.0.0.1\",\n\tPort: 4222,\n\tNoLog: true,\n\tNoSigs: true,\n\tMaxControlLine: 4096,\n\tDisableShortFirstPing: true,\n\tNoJetStreamStrict: false,\n}\n\nfunc testDefaultClusterOptionsForLeafNodes() *Options {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.Cluster.Host = o.Host\n\to.Cluster.Port = -1\n\to.Gateway.Host = o.Host\n\to.Gateway.Port = -1\n\to.LeafNode.Host = o.Host\n\to.LeafNode.Port = -1\n\treturn &o\n}\n\nfunc RunRandClientPortServer(t *testing.T) *Server {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.StoreDir = t.TempDir()\n\treturn RunServer(&opts)\n}\n\nfunc require_True(t testing.TB, b bool) {\n\tt.Helper()\n\tif !b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_True check\", nil)\n\t\tt.Fatalf(\"require true, but got false\")\n\t}\n}\n\nfunc require_False(t testing.TB, b bool) {\n\tt.Helper()\n\tif b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_False check\", nil)\n\t\tt.Fatalf(\"require false, but got true\")\n\t}\n}\n\nfunc require_NoError(t testing.TB, err error) {\n\tt.Helper()\n\tif err != nil {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_NoError check\", map[string]any{\n\t\t\t\"error\": err.Error(),\n\t\t})\n\t\tt.Fatalf(\"require no error, but got: %v\", err)\n\t}\n}\n\n// Must be used in a defer call.\nfunc require_NoPanic(t testing.TB) {\n\tt.Helper()\n\tif err := recover(); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc require_NotNil[T any](t testing.TB, vs ...T) {\n\tt.Helper()\n\tfor _, v := range vs {\n\t\tr := reflect.ValueOf(v)\n\t\tswitch k := r.Kind(); k {\n\t\tcase reflect.Ptr, reflect.Interface, reflect.Slice,\n\t\t\treflect.Map, reflect.Chan, reflect.Func:\n\t\t\tif r.IsNil() {\n\t\t\t\tantithesis.AssertUnreachable(t, \"Failed require_NotNil check\", nil)\n\t\t\t\tt.Fatalf(\"require not nil, but got nil\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc require_Contains(t *testing.T, s string, subStrs ...string) {\n\tt.Helper()\n\tfor _, subStr := range subStrs {\n\t\tif !strings.Contains(s, subStr) {\n\t\t\tantithesis.AssertUnreachable(t, \"Failed require_Contains check\", map[string]any{\n\t\t\t\t\"string\": s,\n\t\t\t\t\"sub_strings\": subStr,\n\t\t\t})\n\t\t\tt.Fatalf(\"require %q to be contained in %q\", subStr, s)\n\t\t}\n\t}\n}\n\nfunc require_Error(t testing.TB, err error, expected ...error) {\n\tt.Helper()\n\tif err == nil {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Error check (nil error)\", nil)\n\t\tt.Fatalf(\"require error, but got none\")\n\t}\n\tif len(expected) == 0 {\n\t\treturn\n\t}\n\t// Try to strip nats prefix from Go library if present.\n\tconst natsErrPre = \"nats: \"\n\teStr := err.Error()\n\tif strings.HasPrefix(eStr, natsErrPre) {\n\t\teStr = strings.Replace(eStr, natsErrPre, _EMPTY_, 1)\n\t}\n\n\tfor _, e := range expected {\n\t\tif err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tantithesis.AssertUnreachable(t, \"Failed require_Error check (unexpected error)\", map[string]any{\n\t\t\"error\": err.Error(),\n\t})\n\tt.Fatalf(\"Expected one of %v, got '%v'\", expected, err)\n}\n\nfunc require_Equal[T comparable](t testing.TB, a, b T) {\n\tt.Helper()\n\tif a != b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Equal check\", nil)\n\t\tt.Fatalf(\"require %T equal, but got: %v != %v\", a, a, b)\n\t}\n}\n\nfunc require_NotEqual[T comparable](t testing.TB, a, b T) {\n\tt.Helper()\n\tif a == b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_NotEqual check\", nil)\n\t\tt.Fatalf(\"require %T not equal, but got: %v == %v\", a, a, b)\n\t}\n}\n\nfunc require_Len(t testing.TB, a, b int) {\n\tt.Helper()\n\tif a != b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_Len check\", nil)\n\t\tt.Fatalf(\"require len, but got: %v != %v\", a, b)\n\t}\n}\n\nfunc require_LessThan[T cmp.Ordered](t *testing.T, a, b T) {\n\tt.Helper()\n\tif a >= b {\n\t\tantithesis.AssertUnreachable(t, \"Failed require_LessThan check\", nil)\n\t\tt.Fatalf(\"require %v to be less than %v\", a, b)\n\t}\n}\n\nfunc require_ChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) T {\n\tt.Helper()\n\tselect {\n\tcase v := <-ch:\n\t\treturn v\n\tcase <-time.After(timeout):\n\t\tantithesis.AssertUnreachable(t, \"Failed require_ChanRead check\", nil)\n\t\tt.Fatalf(\"require read from channel within %v but didn't get anything\", timeout)\n\t}\n\tpanic(\"this shouldn't be possible\")\n}\n\nfunc require_NoChanRead[T any](t *testing.T, ch chan T, timeout time.Duration) {\n\tt.Helper()\n\tselect {\n\tcase <-ch:\n\t\tantithesis.AssertUnreachable(t, \"Failed require_NoChanRead check\", nil)\n\t\tt.Fatalf(\"require no read from channel within %v but got something\", timeout)\n\tcase <-time.After(timeout):\n\t}\n}\n\nfunc checkNatsError(t *testing.T, e *ApiError, id ErrorIdentifier) {\n\tt.Helper()\n\tae, ok := ApiErrors[id]\n\tif !ok {\n\t\tt.Fatalf(\"Unknown error ID identifier: %d\", id)\n\t}\n\n\tif e.ErrCode != ae.ErrCode {\n\t\tt.Fatalf(\"Did not get NATS Error %d: %+v\", e.ErrCode, e)\n\t}\n}\n\n// Creates a full cluster with numServers and given name and makes sure its well formed.\n// Will have Gateways and Leaf Node connections active.\nfunc createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster {\n\tt.Helper()\n\treturn createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...)\n}\n\n// Creates a cluster and optionally additional accounts and users.\n// Will have Gateways and Leaf Node connections active.\nfunc createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster {\n\tt.Helper()\n\n\tif clusterName == \"\" || numServers < 1 {\n\t\tt.Fatalf(\"Bad params\")\n\t}\n\n\t// Setup some accounts and users.\n\t// $SYS is always the system account. And we have default FOO and BAR accounts, as well\n\t// as DLC and NGS which do a service import.\n\tcreateAccountsAndUsers := func() ([]*Account, []*User) {\n\t\tif !doAccounts {\n\t\t\treturn []*Account{NewAccount(\"$SYS\")}, nil\n\t\t}\n\n\t\tsys := NewAccount(\"$SYS\")\n\t\tngs := NewAccount(\"NGS\")\n\t\tdlc := NewAccount(\"DLC\")\n\t\tfoo := NewAccount(\"FOO\")\n\t\tbar := NewAccount(\"BAR\")\n\n\t\taccounts := []*Account{sys, foo, bar, ngs, dlc}\n\n\t\tngs.AddServiceExport(\"ngs.usage.*\", nil)\n\t\tdlc.AddServiceImport(ngs, \"ngs.usage\", \"ngs.usage.dlc\")\n\n\t\t// Setup users\n\t\tusers := []*User{\n\t\t\t{Username: \"dlc\", Password: \"pass\", Permissions: nil, Account: dlc},\n\t\t\t{Username: \"ngs\", Password: \"pass\", Permissions: nil, Account: ngs},\n\t\t\t{Username: \"foo\", Password: \"pass\", Permissions: nil, Account: foo},\n\t\t\t{Username: \"bar\", Password: \"pass\", Permissions: nil, Account: bar},\n\t\t\t{Username: \"sys\", Password: \"pass\", Permissions: nil, Account: sys},\n\t\t}\n\t\treturn accounts, users\n\t}\n\n\tbindGlobal := func(s *Server) {\n\t\tngs, err := s.LookupAccount(\"NGS\")\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\t// Bind global to service import\n\t\tgacc, _ := s.LookupAccount(\"$G\")\n\t\tgacc.AddServiceImport(ngs, \"ngs.usage\", \"ngs.usage.$G\")\n\t}\n\n\t// If we are going to connect to another cluster set that up now for options.\n\tvar gws []*RemoteGatewayOpts\n\tfor _, c := range connectTo {\n\t\t// Gateways autodiscover here too, so just need one address from the set.\n\t\tgwAddr := fmt.Sprintf(\"nats-gw://%s:%d\", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port)\n\t\tgwurl, _ := url.Parse(gwAddr)\n\t\tgws = append(gws, &RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}})\n\t}\n\n\t// Make the GWs form faster for the tests.\n\tSetGatewaysSolicitDelay(gwSolicit)\n\tdefer ResetGatewaysSolicitDelay()\n\n\t// Create seed first.\n\to := testDefaultClusterOptionsForLeafNodes()\n\to.Gateway.Name = clusterName\n\to.Gateway.Gateways = gws\n\t// All of these need system accounts.\n\to.Accounts, o.Users = createAccountsAndUsers()\n\to.SystemAccount = \"$SYS\"\n\to.ServerName = fmt.Sprintf(\"%s1\", clusterName)\n\t// Run the server\n\ts := RunServer(o)\n\tbindGlobal(s)\n\n\tc := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName}\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\n\t// For connecting to seed server above.\n\trouteAddr := fmt.Sprintf(\"nats-route://%s:%d\", o.Cluster.Host, o.Cluster.Port)\n\trurl, _ := url.Parse(routeAddr)\n\troutes := []*url.URL{rurl}\n\n\tfor i := 1; i < numServers; i++ {\n\t\to := testDefaultClusterOptionsForLeafNodes()\n\t\to.Gateway.Name = clusterName\n\t\to.Gateway.Gateways = gws\n\t\to.Routes = routes\n\t\t// All of these need system accounts.\n\t\to.Accounts, o.Users = createAccountsAndUsers()\n\t\to.SystemAccount = \"$SYS\"\n\t\to.ServerName = fmt.Sprintf(\"%s%d\", clusterName, i+1)\n\t\ts := RunServer(o)\n\t\tbindGlobal(s)\n\n\t\tc.servers = append(c.servers, s)\n\t\tc.opts = append(c.opts, o)\n\t}\n\tcheckClusterFormed(t, c.servers...)\n\n\tif waitOnGWs {\n\t\t// Wait on gateway connections if we were asked to connect to other gateways.\n\t\tif numGWs := len(connectTo); numGWs > 0 {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\twaitForOutboundGateways(t, s, numGWs, 2*time.Second)\n\t\t\t}\n\t\t}\n\t}\n\tc.t = t\n\treturn c\n}\n\nfunc (c *cluster) shutdown() {\n\tif c == nil {\n\t\treturn\n\t}\n\t// Stop any proxies.\n\tfor _, np := range c.nproxies {\n\t\tnp.stop()\n\t}\n\t// Shutdown and cleanup servers.\n\tfor i, s := range c.servers {\n\t\tsd := s.StoreDir()\n\t\ts.Shutdown()\n\t\ts.WaitForShutdown()\n\t\tif cf := c.opts[i].ConfigFile; cf != _EMPTY_ {\n\t\t\tos.Remove(cf)\n\t\t}\n\t\tif sd != _EMPTY_ {\n\t\t\tsd = strings.TrimSuffix(sd, JetStreamStoreDir)\n\t\t\tos.RemoveAll(sd)\n\t\t}\n\t}\n}\n\nfunc shutdownCluster(c *cluster) {\n\tc.shutdown()\n}\n\nfunc (c *cluster) randomServer() *Server {\n\treturn c.randomServerFromCluster(c.name)\n}\n\nfunc (c *cluster) randomServerFromCluster(cname string) *Server {\n\t// Since these can be randomly shutdown in certain tests make sure they are running first.\n\t// Copy our servers list and shuffle then walk looking for first running server.\n\tcs := append(c.servers[:0:0], c.servers...)\n\trand.Shuffle(len(cs), func(i, j int) { cs[i], cs[j] = cs[j], cs[i] })\n\n\tfor _, s := range cs {\n\t\tif s.Running() && s.ClusterName() == cname {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc runSolicitLeafServer(lso *Options) (*Server, *Options) {\n\treturn runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://%s:%d\", lso.LeafNode.Host, lso.LeafNode.Port))\n}\n\nfunc runSolicitLeafServerToURL(surl string) (*Server, *Options) {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.NoSystemAccount = true\n\trurl, _ := url.Parse(surl)\n\to.LeafNode.Remotes = []*RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\to.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\treturn RunServer(&o), &o\n}\n\nfunc skipIfBuildkite(t *testing.T) {\n\tif os.Getenv(\"BUILDKITE\") == \"true\" {\n\t\tt.Skip(\"skipping test on Buildkite CI\")\n\t}\n}\n" }, { "path": "server/thw/helper_test.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage thw\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc require_NoError(t *testing.T, err error) {\n\tt.Helper()\n\tif err != nil {\n\t\tt.Fatalf(\"require no error, but got: %v\", err)\n\t}\n}\n\nfunc require_Error(t *testing.T, err error, expected ...error) {\n\tt.Helper()\n\tif err == nil {\n\t\tt.Fatalf(\"require error, but got none\")\n\t}\n\tif len(expected) == 0 {\n\t\treturn\n\t}\n\teStr := err.Error()\n\tfor _, e := range expected {\n\t\tif err == e || strings.Contains(eStr, e.Error()) || strings.Contains(e.Error(), eStr) {\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatalf(\"Expected one of %v, got '%v'\", expected, err)\n}\n\nfunc require_True(t *testing.T, b bool) {\n\tt.Helper()\n\tif !b {\n\t\tt.Fatalf(\"require true, but got false\")\n\t}\n}\n\nfunc require_Equal[T comparable](t *testing.T, a, b T) {\n\tt.Helper()\n\tif a != b {\n\t\tt.Fatalf(\"require %T equal, but got: %v != %v\", a, a, b)\n\t}\n}\n" }, { "path": "server/thw/thw.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage thw\n\nimport (\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"io\"\n\t\"math\"\n\t\"time\"\n)\n\n// Error for when we can not locate a task for removal or updates.\nvar ErrTaskNotFound = errors.New(\"thw: task not found\")\n\n// Error for when we try to decode a binary-encoded THW with an unknown version number.\nvar ErrInvalidVersion = errors.New(\"thw: encoded version not known\")\n\nconst (\n\ttickDuration = int64(time.Second) // Tick duration in nanoseconds.\n\twheelBits = 12 // 2^12 = 4096 slots.\n\twheelSize = 1 << wheelBits // Number of slots in the wheel.\n\twheelMask = wheelSize - 1 // Mask for calculating position.\n\theaderLen = 17 // 1 byte magic + 2x uint64s\n)\n\n// slot represents a single slot in the wheel.\ntype slot struct {\n\tentries map[uint64]int64 // Map of sequence to expires.\n\tlowest int64 // Lowest expiration time in this slot.\n}\n\n// HashWheel represents the timing wheel.\ntype HashWheel struct {\n\twheel []*slot // Array of slots.\n\tlowest int64 // Track the lowest expiration time across all slots.\n\tcount uint64 // How many entries are present?\n}\n\n// HashWheelEntry represents a single entry in the wheel.\ntype HashWheelEntry struct {\n\tSeq uint64\n\tExpires int64\n}\n\n// NewHashWheel initializes a new HashWheel.\nfunc NewHashWheel() *HashWheel {\n\treturn &HashWheel{\n\t\twheel: make([]*slot, wheelSize),\n\t\tlowest: math.MaxInt64,\n\t}\n}\n\n// getPosition calculates the slot position for a given expiration time.\nfunc (hw *HashWheel) getPosition(expires int64) int64 {\n\treturn (expires / tickDuration) & wheelMask\n}\n\n// newSlot creates a new slot.\nfunc newSlot() *slot {\n\treturn &slot{\n\t\tentries: make(map[uint64]int64),\n\t\tlowest: math.MaxInt64,\n\t}\n}\n\n// Add schedules a new timer task.\nfunc (hw *HashWheel) Add(seq uint64, expires int64) error {\n\tpos := hw.getPosition(expires)\n\t// Initialize the slot lazily.\n\tif hw.wheel[pos] == nil {\n\t\thw.wheel[pos] = newSlot()\n\t}\n\tif _, ok := hw.wheel[pos].entries[seq]; !ok {\n\t\thw.count++\n\t}\n\thw.wheel[pos].entries[seq] = expires\n\n\t// Update slot's lowest expiration if this is earlier.\n\tif expires < hw.wheel[pos].lowest {\n\t\thw.wheel[pos].lowest = expires\n\t\t// Update global lowest if this is now the earliest.\n\t\tif expires < hw.lowest {\n\t\t\thw.lowest = expires\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Remove removes a timer task.\nfunc (hw *HashWheel) Remove(seq uint64, expires int64) error {\n\tpos := hw.getPosition(expires)\n\ts := hw.wheel[pos]\n\tif s == nil {\n\t\treturn ErrTaskNotFound\n\t}\n\tif _, exists := s.entries[seq]; !exists {\n\t\treturn ErrTaskNotFound\n\t}\n\tdelete(s.entries, seq)\n\thw.count--\n\n\t// If the slot is empty, we can set it to nil to free memory.\n\tif len(s.entries) == 0 {\n\t\thw.wheel[pos] = nil\n\t}\n\treturn nil\n}\n\n// Update updates the expiration time of an existing timer task.\nfunc (hw *HashWheel) Update(seq uint64, oldExpires int64, newExpires int64) error {\n\t// Remove from old position.\n\tif err := hw.Remove(seq, oldExpires); err != nil {\n\t\treturn err\n\t}\n\t// Add to new position.\n\treturn hw.Add(seq, newExpires)\n}\n\n// ExpireTasks processes all expired tasks using a callback, but only expires a task if the callback returns true.\nfunc (hw *HashWheel) ExpireTasks(callback func(seq uint64, expires int64) bool) {\n\tnow := time.Now().UnixNano()\n\thw.expireTasks(now, callback)\n}\n\nfunc (hw *HashWheel) expireTasks(ts int64, callback func(seq uint64, expires int64) bool) {\n\t// Quick return if nothing is expired.\n\tif hw.lowest > ts {\n\t\treturn\n\t}\n\n\tglobalLowest := int64(math.MaxInt64)\n\tfor pos, s := range hw.wheel {\n\t\t// Skip s if nothing to expire.\n\t\tif s == nil || s.lowest > ts {\n\t\t\tif s != nil && s.lowest < globalLowest {\n\t\t\t\tglobalLowest = s.lowest\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Track new lowest while processing expirations\n\t\tslotLowest := int64(math.MaxInt64)\n\t\tfor seq, expires := range s.entries {\n\t\t\tif expires <= ts && callback(seq, expires) {\n\t\t\t\tdelete(s.entries, seq)\n\t\t\t\thw.count--\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif expires < slotLowest {\n\t\t\t\tslotLowest = expires\n\t\t\t}\n\t\t}\n\n\t\t// Nil out if we are empty.\n\t\tif len(s.entries) == 0 {\n\t\t\thw.wheel[pos] = nil\n\t\t} else {\n\t\t\ts.lowest = slotLowest\n\t\t\tif slotLowest < globalLowest {\n\t\t\t\tglobalLowest = slotLowest\n\t\t\t}\n\t\t}\n\t}\n\thw.lowest = globalLowest\n}\n\n// GetNextExpiration returns the earliest expiration time before the given time.\n// Returns math.MaxInt64 if no expirations exist before the specified time.\nfunc (hw *HashWheel) GetNextExpiration(before int64) int64 {\n\tif hw.lowest < before {\n\t\treturn hw.lowest\n\t}\n\treturn math.MaxInt64\n}\n\n// Count returns the amount of tasks in the THW.\nfunc (hw *HashWheel) Count() uint64 {\n\treturn hw.count\n}\n\n// Encode writes out the contents of the THW into a binary snapshot\n// and returns it. The high seq number is included in the snapshot and will\n// be returned on decode.\nfunc (hw *HashWheel) Encode(highSeq uint64) []byte {\n\tb := make([]byte, 0, headerLen+(hw.count*(2*binary.MaxVarintLen64)))\n\tb = append(b, 1) // Magic version\n\tb = binary.LittleEndian.AppendUint64(b, hw.count) // Entry count\n\tb = binary.LittleEndian.AppendUint64(b, highSeq) // Stamp\n\tfor _, slot := range hw.wheel {\n\t\tif slot == nil || slot.entries == nil {\n\t\t\tcontinue\n\t\t}\n\t\tfor v, ts := range slot.entries {\n\t\t\tb = binary.AppendVarint(b, ts)\n\t\t\tb = binary.AppendUvarint(b, v)\n\t\t}\n\t}\n\treturn b\n}\n\n// Decode snapshots a binary-encoded THW and replaces the contents of this\n// THW with them. Returns the high seq number from the snapshot.\nfunc (hw *HashWheel) Decode(b []byte) (uint64, error) {\n\tif len(b) < headerLen {\n\t\treturn 0, io.ErrShortBuffer\n\t}\n\tif b[0] != 1 {\n\t\treturn 0, ErrInvalidVersion\n\t}\n\thw.wheel = make([]*slot, wheelSize)\n\thw.lowest = math.MaxInt64\n\tcount := binary.LittleEndian.Uint64(b[1:])\n\tstamp := binary.LittleEndian.Uint64(b[9:])\n\tb = b[headerLen:]\n\tfor i := uint64(0); i < count; i++ {\n\t\tts, tn := binary.Varint(b)\n\t\tif tn < 0 {\n\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t}\n\t\tv, vn := binary.Uvarint(b[tn:])\n\t\tif vn < 0 {\n\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t}\n\t\thw.Add(v, ts)\n\t\tb = b[tn+vn:]\n\t}\n\treturn stamp, nil\n}\n" }, { "path": "server/thw/thw_test.go", "content": "// Copyright 2024-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage thw\n\nimport (\n\t\"math\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestHashWheelBasics(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Add a sequence.\n\tseq, expires := uint64(1), now.Add(5*time.Second).UnixNano()\n\trequire_NoError(t, hw.Add(seq, expires))\n\trequire_Equal(t, hw.count, 1)\n\n\t// Try to remove non-existent sequence.\n\trequire_Error(t, hw.Remove(999, expires), ErrTaskNotFound)\n\trequire_Equal(t, hw.count, 1)\n\n\t// Remove the sequence properly.\n\trequire_NoError(t, hw.Remove(seq, expires))\n\trequire_Equal(t, hw.count, 0)\n\n\t// Verify it's gone.\n\trequire_Error(t, hw.Remove(seq, expires), ErrTaskNotFound)\n\trequire_Equal(t, hw.count, 0)\n}\n\nfunc TestHashWheelUpdate(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\toldExpires := now.Add(5 * time.Second).UnixNano()\n\tnewExpires := now.Add(10 * time.Second).UnixNano()\n\n\t// Add initial sequence.\n\trequire_NoError(t, hw.Add(1, oldExpires))\n\trequire_Equal(t, hw.count, 1)\n\n\t// Update expiration.\n\trequire_NoError(t, hw.Update(1, oldExpires, newExpires))\n\trequire_Equal(t, hw.count, 1)\n\n\t// Verify old expiration is gone.\n\trequire_Error(t, hw.Remove(1, oldExpires), ErrTaskNotFound)\n\trequire_Equal(t, hw.count, 1)\n\n\t// Verify new expiration exists\n\trequire_NoError(t, hw.Remove(1, newExpires))\n\trequire_Equal(t, hw.count, 0)\n}\n\nfunc TestHashWheelExpiration(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Add sequences with different expiration times.\n\tseqs := map[uint64]int64{\n\t\t1: now.Add(-1 * time.Second).UnixNano(), // Already expired\n\t\t2: now.Add(1 * time.Second).UnixNano(), // Expires soon\n\t\t3: now.Add(10 * time.Second).UnixNano(), // Expires later\n\t\t4: now.Add(60 * time.Second).UnixNano(), // Expires much later\n\t}\n\n\tfor seq, expires := range seqs {\n\t\trequire_NoError(t, hw.Add(seq, expires))\n\t}\n\trequire_Equal(t, hw.count, uint64(len(seqs)))\n\n\t// Process expired tasks.\n\texpired := make(map[uint64]bool)\n\thw.ExpireTasks(func(seq uint64, expires int64) bool {\n\t\texpired[seq] = true\n\t\treturn true\n\t})\n\n\t// Verify only sequence 1 expired.\n\trequire_Equal(t, len(expired), 1)\n\trequire_True(t, expired[1])\n\trequire_Equal(t, hw.count, 3)\n}\n\nfunc TestHashWheelManualExpiration(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now().UnixNano()\n\n\tfor seq := uint64(1); seq <= 4; seq++ {\n\t\trequire_NoError(t, hw.Add(seq, now))\n\t}\n\trequire_Equal(t, hw.count, 4)\n\n\t// Loop over expired multiple times, but without removing them.\n\texpired := make(map[uint64]uint64)\n\tfor i := uint64(0); i <= 1; i++ {\n\t\thw.ExpireTasks(func(seq uint64, expires int64) bool {\n\t\t\texpired[seq]++\n\t\t\treturn false\n\t\t})\n\n\t\trequire_Equal(t, len(expired), 4)\n\t\trequire_Equal(t, expired[1], 1+i)\n\t\trequire_Equal(t, expired[2], 1+i)\n\t\trequire_Equal(t, expired[3], 1+i)\n\t\trequire_Equal(t, expired[4], 1+i)\n\t\trequire_Equal(t, hw.count, 4)\n\t}\n\n\t// Only remove even sequences.\n\tfor i := uint64(0); i <= 1; i++ {\n\t\thw.ExpireTasks(func(seq uint64, expires int64) bool {\n\t\t\texpired[seq]++\n\t\t\treturn seq%2 == 0\n\t\t})\n\n\t\t// Verify even sequences are removed.\n\t\trequire_Equal(t, expired[1], 3+i)\n\t\trequire_Equal(t, expired[2], 3)\n\t\trequire_Equal(t, expired[3], 3+i)\n\t\trequire_Equal(t, expired[4], 3)\n\t\trequire_Equal(t, hw.count, 2)\n\t}\n\n\t// Manually remove last items.\n\trequire_NoError(t, hw.Remove(1, now))\n\trequire_NoError(t, hw.Remove(3, now))\n\trequire_Equal(t, hw.count, 0)\n}\n\nfunc TestHashWheelExpirationLargerThanWheel(t *testing.T) {\n\thw := NewHashWheel()\n\n\t// Add sequences such that they can be expired immediately.\n\tseqs := map[uint64]int64{\n\t\t1: 0,\n\t\t2: int64(time.Second),\n\t}\n\tfor seq, expires := range seqs {\n\t\trequire_NoError(t, hw.Add(seq, expires))\n\t}\n\trequire_Equal(t, hw.count, 2)\n\n\t// Pick a timestamp such that the expiration needs to wrap around the whole wheel.\n\tnow := int64(time.Second) * wheelMask\n\n\t// Process expired tasks.\n\texpired := make(map[uint64]bool)\n\thw.expireTasks(now, func(seq uint64, expires int64) bool {\n\t\texpired[seq] = true\n\t\treturn true\n\t})\n\n\t// Verify both sequences are expired.\n\trequire_Equal(t, len(expired), 2)\n\trequire_Equal(t, hw.count, 0)\n}\n\nfunc TestHashWheelNextExpiration(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Add sequences with different expiration times.\n\tseqs := map[uint64]int64{\n\t\t1: now.Add(5 * time.Second).UnixNano(),\n\t\t2: now.Add(3 * time.Second).UnixNano(), // Earliest\n\t\t3: now.Add(10 * time.Second).UnixNano(),\n\t}\n\n\tfor seq, expires := range seqs {\n\t\trequire_NoError(t, hw.Add(seq, expires))\n\t}\n\trequire_Equal(t, hw.count, uint64(len(seqs)))\n\n\t// Test GetNextExpiration.\n\tnextExternalTick := now.Add(6 * time.Second).UnixNano()\n\t// Should return sequence 2's expiration\n\trequire_Equal(t, hw.GetNextExpiration(nextExternalTick), seqs[2])\n\n\t// Test with empty wheel.\n\tempty := NewHashWheel()\n\trequire_Equal(t, empty.GetNextExpiration(now.Add(1*time.Second).UnixNano()), math.MaxInt64)\n}\n\nfunc TestHashWheelStress(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Add many sequences.\n\tnumSequences := 100_000\n\tfor seq := 0; seq < numSequences; seq++ {\n\t\texpires := now.Add(time.Duration(seq) * time.Second).UnixNano()\n\t\trequire_NoError(t, hw.Add(uint64(seq), expires))\n\t}\n\n\t// Update many sequences.\n\tfor seq := 0; seq < numSequences; seq += 2 { // Update every other sequence\n\t\toldExpires := now.Add(time.Duration(seq) * time.Second).UnixNano()\n\t\tnewExpires := now.Add(time.Duration(seq+numSequences) * time.Second).UnixNano()\n\t\trequire_NoError(t, hw.Update(uint64(seq), oldExpires, newExpires))\n\t}\n\n\t// Remove many sequences.\n\tfor seq := 1; seq < numSequences; seq += 2 { // Remove odd-numbered sequences\n\t\texpires := now.Add(time.Duration(seq) * time.Second).UnixNano()\n\t\trequire_NoError(t, hw.Remove(uint64(seq), expires))\n\t}\n}\n\nfunc TestHashWheelEncodeDecode(t *testing.T) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Add many sequences.\n\tnumSequences := 100_000\n\tfor seq := 0; seq < numSequences; seq++ {\n\t\texpires := now.Add(time.Duration(seq) * time.Second).UnixNano()\n\t\trequire_NoError(t, hw.Add(uint64(seq), expires))\n\t}\n\n\tb := hw.Encode(12345)\n\trequire_True(t, len(b) > 17) // Bigger than just the header\n\n\tnhw := NewHashWheel()\n\tstamp, err := nhw.Decode(b)\n\trequire_NoError(t, err)\n\trequire_Equal(t, stamp, 12345)\n\trequire_Equal(t, hw.GetNextExpiration(math.MaxInt64), nhw.GetNextExpiration(math.MaxInt64))\n\n\tfor s, slot := range hw.wheel {\n\t\tnslot := nhw.wheel[s]\n\t\trequire_Equal(t, slot.lowest, nslot.lowest)\n\t\trequire_Equal(t, len(slot.entries), len(nslot.entries))\n\t\tfor v, ts := range slot.entries {\n\t\t\tnts, ok := nslot.entries[v]\n\t\t\trequire_True(t, ok)\n\t\t\trequire_Equal(t, ts, nts)\n\t\t}\n\t}\n}\n\n// Benchmarks\n\nfunc BenchmarkHashWheel_Add(b *testing.B) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Create different ranges for expires to spread across slots\n\texpires := make([]int64, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\texpires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano()\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Add(uint64(i), expires[i])\n\t}\n}\n\nfunc BenchmarkHashWheel_Add_SameSlot(b *testing.B) {\n\thw := NewHashWheel()\n\texpires := time.Now().Add(time.Second).UnixNano()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Add(uint64(i), expires)\n\t}\n}\n\nfunc BenchmarkHashWheel_Update(b *testing.B) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// First add N items\n\texpires := make([]int64, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\texpires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano()\n\t\thw.Add(uint64(i), expires[i])\n\t}\n\n\tnewExpires := now.Add(2 * time.Hour).UnixNano()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Update(uint64(i), expires[i], newExpires)\n\t}\n}\n\nfunc BenchmarkHashWheel_Update_SameSlot(b *testing.B) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\toldExpires := now.Add(time.Second).UnixNano()\n\tnewExpires := now.Add(2 * time.Second).UnixNano()\n\n\t// First add N items\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Add(uint64(i), oldExpires)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Update(uint64(i), oldExpires, newExpires)\n\t}\n}\n\n// Benchmark memory allocation\nfunc BenchmarkHashWheel_Add_Memory(b *testing.B) {\n\tb.ReportAllocs()\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\texpires := make([]int64, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\texpires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano()\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Add(uint64(i), expires[i])\n\t}\n}\n\n// Large scale test\nfunc BenchmarkHashWheel_LargeScale(b *testing.B) {\n\thw := NewHashWheel()\n\tnow := time.Now()\n\n\t// Pre-populate with 100k items\n\tfor i := 0; i < 100_000; i++ {\n\t\texpires := now.Add(time.Duration(i%3600) * time.Second).UnixNano()\n\t\thw.Add(uint64(i), expires)\n\t}\n\n\texpires := make([]int64, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\texpires[i] = now.Add(time.Duration(i%3600) * time.Second).UnixNano()\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\thw.Add(uint64(i+100000), expires[i])\n\t}\n}\n" }, { "path": "server/tpm/js_ek_tpm_other.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n\npackage tpm\n\nimport \"fmt\"\n\n// LoadJetStreamEncryptionKeyFromTPM here is a stub for unsupported platforms.\nfunc LoadJetStreamEncryptionKeyFromTPM(srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) {\n\treturn \"\", fmt.Errorf(\"TPM functionality is not supported on this platform\")\n}\n" }, { "path": "server/tpm/js_ek_tpm_test.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage tpm\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc getTempFile(t *testing.T) string {\n\treturn t.TempDir() + \"/jskeys.json\"\n}\n\nfunc TestLoadJetStreamEncryptionKeyFromTPM(t *testing.T) {\n\ttestFile := getTempFile(t)\n\ttype args struct {\n\t\tsrkPassword string\n\t\tjsKeyFile string\n\t\tjsKeyPassword string\n\t\tpcr int\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\tclear bool\n\t\twantErr bool\n\t}{\n\t\t{\"TestLoadJetStreamEncryptionKeyFromTPM-Load\", args{\"\", testFile, \"password\", 22}, true, false},\n\t\t{\"TestLoadJetStreamEncryptionKeyFromTPM-Read\", args{\"\", testFile, \"password\", 22}, false, false},\n\t\t{\"TestLoadJetStreamEncryptionKeyFromTPM-BadPass\", args{\"\", testFile, \"badpass\", 22}, false, true},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif tt.clear {\n\t\t\t\tos.Remove(tt.args.jsKeyFile)\n\t\t\t}\n\t\t\t_, err := LoadJetStreamEncryptionKeyFromTPM(tt.args.srkPassword, tt.args.jsKeyFile, tt.args.jsKeyPassword, tt.args.pcr)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"LoadJetStreamEncryptionKeyFromTPM() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestLoadJetStreamEncryptionKeyFromTPMBasic tests the basic functionality.\n// The first pass will create the keys and generate the js encryption key.\n// the second pass will read the keys from disk, decrypt with the TPM (unseal),\n// and return the same key.\nfunc TestLoadJetStreamEncryptionKeyFromTPMBasic(t *testing.T) {\n\ttestFile := getTempFile(t)\n\n\t// Create the key file.\n\tkey1, err := LoadJetStreamEncryptionKeyFromTPM(\"\", testFile, \"password\", 22)\n\tif err != nil {\n\t\tt.Errorf(\"LoadJetStreamEncryptionKeyFromTPM() failed: %v\", err)\n\t}\n\n\t// Now obtain the newly generated key from the file.\n\tkey2, err := LoadJetStreamEncryptionKeyFromTPM(\"\", testFile, \"password\", 22)\n\tif err != nil {\n\t\tt.Errorf(\"LoadJetStreamEncryptionKeyFromTPM() failed: %v\", err)\n\t}\n\tif key1 != key2 {\n\t\tt.Errorf(\"Keys should match\")\n\t}\n}\n" }, { "path": "server/tpm/js_ek_tpm_windows.go", "content": "// Copyright 2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n\npackage tpm\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/google/go-tpm/legacy/tpm2\"\n\t\"github.com/google/go-tpm/tpmutil\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nvar (\n\t// Version of the NATS TPM JS implmentation\n\tJsKeyTPMVersion = 1\n)\n\n// How this works:\n// Create a Storage Root Key (SRK) in the TPM.\n// If existing JS Encryption keys do not exist on disk.\n// \t - Create a JetStream encryption key (js key) and seal it to the SRK\n// using a provided js encryption key password.\n// \t - Save the public and private blobs to a file on disk.\n// - Return the new js encryption key (the private portion of the nkey)\n// Otherwise (keys exist on disk)\n// - Read the public and private blobs from disk\n// - Load them into the TPM\n// - Unseal the js key using the TPM, and the provided js encryption keys password.\n//\n// Note: a SRK password for the SRK is supported but not tested here.\n\n// Gets/Regenerates the Storage Root Key (SRK) from the TPM. Caller MUST flush this handle when done.\nfunc regenerateSRK(rwc io.ReadWriteCloser, srkPassword string) (tpmutil.Handle, error) {\n\t// Default EK template defined in:\n\t// https://trustedcomputinggroup.org/wp-content/uploads/Credential_Profile_EK_V2.0_R14_published.pdf\n\t// Shared SRK template based off of EK template and specified in:\n\t// https://trustedcomputinggroup.org/wp-content/uploads/TCG-TPM-v2.0-Provisioning-Guidance-Published-v1r1.pdf\n\tsrkTemplate := tpm2.Public{\n\t\tType: tpm2.AlgRSA,\n\t\tNameAlg: tpm2.AlgSHA256,\n\t\tAttributes: tpm2.FlagFixedTPM | tpm2.FlagFixedParent | tpm2.FlagSensitiveDataOrigin | tpm2.FlagUserWithAuth | tpm2.FlagRestricted | tpm2.FlagDecrypt | tpm2.FlagNoDA,\n\t\tAuthPolicy: nil,\n\t\t// We must use RSA 2048 for the intel TSS2 stack\n\t\tRSAParameters: &tpm2.RSAParams{\n\t\t\tSymmetric: &tpm2.SymScheme{\n\t\t\t\tAlg: tpm2.AlgAES,\n\t\t\t\tKeyBits: 128,\n\t\t\t\tMode: tpm2.AlgCFB,\n\t\t\t},\n\t\t\tKeyBits: 2048,\n\t\t\tModulusRaw: make([]byte, 256),\n\t\t},\n\t}\n\t// Create the parent key against which to seal the data\n\tsrkHandle, _, err := tpm2.CreatePrimary(rwc, tpm2.HandleOwner, tpm2.PCRSelection{}, \"\", srkPassword, srkTemplate)\n\treturn srkHandle, err\n}\n\ntype natsTPMPersistedKeys struct {\n\tVersion int `json:\"version\"`\n\tPrivateKey []byte `json:\"private_key\"`\n\tPublicKey []byte `json:\"public_key\"`\n}\n\n// Writes the private and public blobs to disk in a single file. If the directory does\n// not exist, it will be created. If the file already exists it will be overwritten.\nfunc writeTPMKeysToFile(filename string, privateBlob []byte, publicBlob []byte) error {\n\tkeyDir := filepath.Dir(filename)\n\tif err := os.MkdirAll(keyDir, 0750); err != nil {\n\t\treturn fmt.Errorf(\"unable to create/access directory %q: %v\", keyDir, err)\n\t}\n\n\t// Create a new set of persisted keys. Note that the private key doesn't necessarily\n\t// need to be protected as the TPM password is required to use unseal, although it's\n\t// a good idea to put this in a secure location accessible to the server.\n\ttpmKeys := natsTPMPersistedKeys{\n\t\tVersion: JsKeyTPMVersion,\n\t\tPrivateKey: make([]byte, base64.StdEncoding.EncodedLen(len(privateBlob))),\n\t\tPublicKey: make([]byte, base64.StdEncoding.EncodedLen(len(publicBlob))),\n\t}\n\tbase64.StdEncoding.Encode(tpmKeys.PrivateKey, privateBlob)\n\tbase64.StdEncoding.Encode(tpmKeys.PublicKey, publicBlob)\n\t// Convert to JSON\n\tkeysJSON, err := json.Marshal(tpmKeys)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to marshal keys to JSON: %v\", err)\n\t}\n\t// Write the JSON to a file\n\tif err := os.WriteFile(filename, keysJSON, 0640); err != nil {\n\t\treturn fmt.Errorf(\"unable to write keys file to %q: %v\", filename, err)\n\t}\n\treturn nil\n}\n\n// Reads the private and public blobs from a single file. If the file does not exist,\n// or the file cannot be read and the keys decoded, an error is returned.\nfunc readTPMKeysFromFile(filename string) ([]byte, []byte, error) {\n\tkeysJSON, err := os.ReadFile(filename)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tvar tpmKeys natsTPMPersistedKeys\n\tif err := json.Unmarshal(keysJSON, &tpmKeys); err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"unable to unmarshal TPM file keys JSON from %s: %v\", filename, err)\n\t}\n\n\t// Placeholder for future-proofing. Here is where we would\n\t// check the current version against tpmKeys.Version and\n\t// handle any changes.\n\n\t// Base64 decode the private and public blobs.\n\tprivateBlob := make([]byte, base64.StdEncoding.DecodedLen(len(tpmKeys.PrivateKey)))\n\tpublicBlob := make([]byte, base64.StdEncoding.DecodedLen(len(tpmKeys.PublicKey)))\n\tprn, err := base64.StdEncoding.Decode(privateBlob, tpmKeys.PrivateKey)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"unable to decode privateBlob from base64: %v\", err)\n\t}\n\tpun, err := base64.StdEncoding.Decode(publicBlob, tpmKeys.PublicKey)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"unable to decode publicBlob from base64: %v\", err)\n\t}\n\treturn publicBlob[:pun], privateBlob[:prn], nil\n}\n\n// Creates a new JetStream encryption key, seals it to the TPM, and saves the public and\n// private blobs to disk in a JSON encoded file. The key is returned as a string.\nfunc createAndSealJsEncryptionKey(rwc io.ReadWriteCloser, srkHandle tpmutil.Handle, srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) {\n\t// Get the authorization policy that will protect the data to be sealed\n\tsessHandle, policy, err := policyPCRPasswordSession(rwc, pcr)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to get policy: %v\", err)\n\t}\n\tif err := tpm2.FlushContext(rwc, sessHandle); err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to flush session: %v\", err)\n\t}\n\t// Seal the data to the parent key and the policy\n\tuser, err := nkeys.CreateUser()\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to create seed: %v\", err)\n\t}\n\t// We'll use the seed to represent the encryption key.\n\tjsStoreKey, err := user.Seed()\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to get seed: %v\", err)\n\t}\n\tprivateArea, publicArea, err := tpm2.Seal(rwc, srkHandle, srkPassword, jsKeyPassword, policy, jsStoreKey)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to seal data: %v\", err)\n\t}\n\terr = writeTPMKeysToFile(jsKeyFile, privateArea, publicArea)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to write key file: %v\", err)\n\t}\n\treturn string(jsStoreKey), nil\n}\n\n// Unseals the JetStream encryption key from the TPM with the provided keys.\n// The key is returned as a string.\nfunc unsealJsEncrpytionKey(rwc io.ReadWriteCloser, pcr int, srkHandle tpmutil.Handle, srkPassword, objectPassword string, publicBlob, privateBlob []byte) (string, error) {\n\t// Load the public/private blobs into the TPM for decryption.\n\tobjectHandle, _, err := tpm2.Load(rwc, srkHandle, srkPassword, publicBlob, privateBlob)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to load data: %v\", err)\n\t}\n\tdefer tpm2.FlushContext(rwc, objectHandle)\n\n\t// Create the authorization session with TPM.\n\tsessHandle, _, err := policyPCRPasswordSession(rwc, pcr)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to get auth session: %v\", err)\n\t}\n\tdefer func() {\n\t\ttpm2.FlushContext(rwc, sessHandle)\n\t}()\n\t// Unseal the data we've loaded into the TPM with the object (js key) password.\n\tunsealedData, err := tpm2.UnsealWithSession(rwc, sessHandle, objectHandle, objectPassword)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to unseal data: %v\", err)\n\t}\n\treturn string(unsealedData), nil\n}\n\n// Returns session handle and policy digest.\nfunc policyPCRPasswordSession(rwc io.ReadWriteCloser, pcr int) (sessHandle tpmutil.Handle, policy []byte, retErr error) {\n\tsessHandle, _, err := tpm2.StartAuthSession(\n\t\trwc,\n\t\ttpm2.HandleNull, /*tpmKey*/\n\t\ttpm2.HandleNull, /*bindKey*/\n\t\tmake([]byte, 16), /*nonceCaller*/\n\t\tnil, /*secret*/\n\t\ttpm2.SessionPolicy,\n\t\ttpm2.AlgNull,\n\t\ttpm2.AlgSHA256)\n\tif err != nil {\n\t\treturn tpm2.HandleNull, nil, fmt.Errorf(\"unable to start session: %v\", err)\n\t}\n\tdefer func() {\n\t\tif sessHandle != tpm2.HandleNull && err != nil {\n\t\t\tif err := tpm2.FlushContext(rwc, sessHandle); err != nil {\n\t\t\t\tretErr = fmt.Errorf(\"%v\\nunable to flush session: %v\", retErr, err)\n\t\t\t}\n\t\t}\n\t}()\n\n\tpcrSelection := tpm2.PCRSelection{\n\t\tHash: tpm2.AlgSHA256,\n\t\tPCRs: []int{pcr},\n\t}\n\tif err := tpm2.PolicyPCR(rwc, sessHandle, nil, pcrSelection); err != nil {\n\t\treturn sessHandle, nil, fmt.Errorf(\"unable to bind PCRs to auth policy: %v\", err)\n\t}\n\tif err := tpm2.PolicyPassword(rwc, sessHandle); err != nil {\n\t\treturn sessHandle, nil, fmt.Errorf(\"unable to require password for auth policy: %v\", err)\n\t}\n\tpolicy, err = tpm2.PolicyGetDigest(rwc, sessHandle)\n\tif err != nil {\n\t\treturn sessHandle, nil, fmt.Errorf(\"unable to get policy digest: %v\", err)\n\t}\n\treturn sessHandle, policy, nil\n}\n\n// LoadJetStreamEncryptionKeyFromTPM loads the JetStream encryption key from the TPM.\n// If the keyfile does not exist, a key will be created and sealed. Public and private blobs\n// used to decrypt the key in future sessions will be saved to disk in the file provided.\n// The key will be unsealed and returned only with the correct password and PCR value.\nfunc LoadJetStreamEncryptionKeyFromTPM(srkPassword, jsKeyFile, jsKeyPassword string, pcr int) (string, error) {\n\trwc, err := tpm2.OpenTPM()\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"could not open the TPM: %v\", err)\n\t}\n\tdefer rwc.Close()\n\n\t// Load the key from the TPM\n\tsrkHandle, err := regenerateSRK(rwc, srkPassword)\n\tdefer func() {\n\t\ttpm2.FlushContext(rwc, srkHandle)\n\t}()\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to regenerate SRK from the TPM: %v\", err)\n\t}\n\t// Read the keys from the key file. If the filed doesn't exist it means we need to create\n\t// a new js encrytpion key.\n\tpublicBlob, privateBlob, err := readTPMKeysFromFile(jsKeyFile)\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\tjsek, err := createAndSealJsEncryptionKey(rwc, srkHandle, srkPassword, jsKeyFile, jsKeyPassword, pcr)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", fmt.Errorf(\"unable to generate new key from the TPM: %v\", err)\n\t\t\t}\n\t\t\t// we've created and sealed the JS Encryption key, now we just return it.\n\t\t\treturn jsek, nil\n\t\t}\n\t\treturn \"\", fmt.Errorf(\"unable to load key from TPM: %v\", err)\n\t}\n\n\t// Unseal the JetStream encryption key using the TPM.\n\tjsek, err := unsealJsEncrpytionKey(rwc, pcr, srkHandle, srkPassword, jsKeyPassword, publicBlob, privateBlob)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to unseal key from the TPM: %v\", err)\n\t}\n\treturn jsek, nil\n}\n" }, { "path": "server/trust_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n)\n\nconst (\n\tt1 = \"OBYEOZQ46VZMFMNETBAW2H6VGDSOBLP67VUEZJ5LPR3PIZBWWRIY4UI4\"\n\tt2 = \"OAHC7NGAHG3YVPTD6QOUFZGPM2OMU6EOS67O2VHBUOA6BJLPTWFHGLKU\"\n)\n\nfunc TestStampedTrustedKeys(t *testing.T) {\n\topts := DefaultOptions()\n\tdefer func() { trustedKeys = \"\" }()\n\n\t// Set this to a bad key. We require valid operator public keys.\n\ttrustedKeys = \"bad\"\n\tif s := New(opts); s != nil {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Expected a bad trustedKeys to return nil server\")\n\t}\n\n\ttrustedKeys = t1\n\ts := New(opts)\n\tif s == nil {\n\t\tt.Fatalf(\"Expected non-nil server\")\n\t}\n\tif len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 {\n\t\tt.Fatalf(\"Trusted Nkeys not setup properly\")\n\t}\n\ttrustedKeys = strings.Join([]string{t1, t2}, \" \")\n\tif s = New(opts); s == nil {\n\t\tt.Fatalf(\"Expected non-nil server\")\n\t}\n\tif len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 {\n\t\tt.Fatalf(\"Trusted Nkeys not setup properly\")\n\t}\n\n\topts.TrustedKeys = []string{\"OVERRIDE ME\"}\n\tif s = New(opts); s != nil {\n\t\tt.Fatalf(\"Expected opts.TrustedKeys to return nil server\")\n\t}\n}\n\nfunc TestTrustedKeysOptions(t *testing.T) {\n\ttrustedKeys = \"\"\n\topts := DefaultOptions()\n\topts.TrustedKeys = []string{\"bad\"}\n\tif s := New(opts); s != nil {\n\t\ts.Shutdown()\n\t\tt.Fatalf(\"Expected a bad opts.TrustedKeys to return nil server\")\n\t}\n\topts.TrustedKeys = []string{t1}\n\ts := New(opts)\n\tif s == nil {\n\t\tt.Fatalf(\"Expected non-nil server\")\n\t}\n\tif len(s.trustedKeys) != 1 || s.trustedKeys[0] != t1 {\n\t\tt.Fatalf(\"Trusted Nkeys not setup properly via options\")\n\t}\n\topts.TrustedKeys = []string{t1, t2}\n\tif s = New(opts); s == nil {\n\t\tt.Fatalf(\"Expected non-nil server\")\n\t}\n\tif len(s.trustedKeys) != 2 || s.trustedKeys[0] != t1 || s.trustedKeys[1] != t2 {\n\t\tt.Fatalf(\"Trusted Nkeys not setup properly via options\")\n\t}\n}\n\nfunc TestTrustConfigOption(t *testing.T) {\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(\"trusted = %q\", t1)))\n\topts, err := ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing config: %v\", err)\n\t}\n\tif l := len(opts.TrustedKeys); l != 1 {\n\t\tt.Fatalf(\"Expected 1 trusted key, got %d\", l)\n\t}\n\tif opts.TrustedKeys[0] != t1 {\n\t\tt.Fatalf(\"Expected trusted key to be %q, got %q\", t1, opts.TrustedKeys[0])\n\t}\n\n\tconfFileName = createConfFile(t, []byte(fmt.Sprintf(\"trusted = [%q, %q]\", t1, t2)))\n\topts, err = ProcessConfigFile(confFileName)\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing config: %v\", err)\n\t}\n\tif l := len(opts.TrustedKeys); l != 2 {\n\t\tt.Fatalf(\"Expected 2 trusted key, got %d\", l)\n\t}\n\tif opts.TrustedKeys[0] != t1 {\n\t\tt.Fatalf(\"Expected trusted key to be %q, got %q\", t1, opts.TrustedKeys[0])\n\t}\n\tif opts.TrustedKeys[1] != t2 {\n\t\tt.Fatalf(\"Expected trusted key to be %q, got %q\", t2, opts.TrustedKeys[1])\n\t}\n\n\t// Now do a bad one.\n\tconfFileName = createConfFile(t, []byte(fmt.Sprintf(\"trusted = [%q, %q]\", t1, \"bad\")))\n\t_, err = ProcessConfigFile(confFileName)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error parsing trust keys with a bad key\")\n\t}\n}\n" }, { "path": "server/util.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// This map is used to store URLs string as the key with a reference count as\n// the value. This is used to handle gossiped URLs such as connect_urls, etc..\ntype refCountedUrlSet map[string]int\n\n// Ascii numbers 0-9\nconst (\n\tasciiZero = 48\n\tasciiNine = 57\n)\n\nfunc versionComponents(version string) (major, minor, patch int, err error) {\n\tm := semVerRe.FindStringSubmatch(version)\n\tif len(m) == 0 {\n\t\treturn 0, 0, 0, errors.New(\"invalid semver\")\n\t}\n\tmajor, err = strconv.Atoi(m[1])\n\tif err != nil {\n\t\treturn -1, -1, -1, err\n\t}\n\tminor, err = strconv.Atoi(m[2])\n\tif err != nil {\n\t\treturn -1, -1, -1, err\n\t}\n\tpatch, err = strconv.Atoi(m[3])\n\tif err != nil {\n\t\treturn -1, -1, -1, err\n\t}\n\treturn major, minor, patch, err\n}\n\nfunc versionAtLeastCheckError(version string, emajor, eminor, epatch int) (bool, error) {\n\tmajor, minor, patch, err := versionComponents(version)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tif major > emajor ||\n\t\t(major == emajor && minor > eminor) ||\n\t\t(major == emajor && minor == eminor && patch >= epatch) {\n\t\treturn true, nil\n\t}\n\treturn false, err\n}\n\nfunc versionAtLeast(version string, emajor, eminor, epatch int) bool {\n\tres, _ := versionAtLeastCheckError(version, emajor, eminor, epatch)\n\treturn res\n}\n\n// parseSize expects decimal positive numbers. We\n// return -1 to signal error.\nfunc parseSize(d []byte) (n int) {\n\tconst maxParseSizeLen = 9 //999M\n\n\tl := len(d)\n\tif l == 0 || l > maxParseSizeLen {\n\t\treturn -1\n\t}\n\tvar (\n\t\ti int\n\t\tdec byte\n\t)\n\n\t// Note: Use `goto` here to avoid for loop in order\n\t// to have the function be inlined.\n\t// See: https://github.com/golang/go/issues/14768\nloop:\n\tdec = d[i]\n\tif dec < asciiZero || dec > asciiNine {\n\t\treturn -1\n\t}\n\tn = n*10 + (int(dec) - asciiZero)\n\n\ti++\n\tif i < l {\n\t\tgoto loop\n\t}\n\treturn n\n}\n\n// parseInt64 expects decimal positive numbers. We\n// return -1 to signal error\nfunc parseInt64(d []byte) (n int64) {\n\tif len(d) == 0 {\n\t\treturn -1\n\t}\n\tfor _, dec := range d {\n\t\tif dec < asciiZero || dec > asciiNine {\n\t\t\treturn -1\n\t\t}\n\t\tn = n*10 + (int64(dec) - asciiZero)\n\t}\n\treturn n\n}\n\n// Helper to move from float seconds to time.Duration\nfunc secondsToDuration(seconds float64) time.Duration {\n\tttl := seconds * float64(time.Second)\n\treturn time.Duration(ttl)\n}\n\n// Parse a host/port string with a default port to use\n// if none (or 0 or -1) is specified in `hostPort` string.\nfunc parseHostPort(hostPort string, defaultPort int) (host string, port int, err error) {\n\tif hostPort != \"\" {\n\t\thost, sPort, err := net.SplitHostPort(hostPort)\n\t\tif ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, \"missing port\") {\n\t\t\t// try appending the current port\n\t\t\thost, sPort, err = net.SplitHostPort(fmt.Sprintf(\"%s:%d\", hostPort, defaultPort))\n\t\t}\n\t\tif err != nil {\n\t\t\treturn \"\", -1, err\n\t\t}\n\t\tport, err = strconv.Atoi(strings.TrimSpace(sPort))\n\t\tif err != nil {\n\t\t\treturn \"\", -1, err\n\t\t}\n\t\tif port == 0 || port == -1 {\n\t\t\tport = defaultPort\n\t\t}\n\t\treturn strings.TrimSpace(host), port, nil\n\t}\n\treturn \"\", -1, errors.New(\"no hostport specified\")\n}\n\n// Returns true if URL u1 represents the same URL than u2,\n// false otherwise.\nfunc urlsAreEqual(u1, u2 *url.URL) bool {\n\treturn reflect.DeepEqual(u1, u2)\n}\n\n// comma produces a string form of the given number in base 10 with\n// commas after every three orders of magnitude.\n//\n// e.g. comma(834142) -> 834,142\n//\n// This function was copied from the github.com/dustin/go-humanize\n// package (MIT License) and is Copyright Dustin Sallings \nfunc comma(v int64) string {\n\tsign := \"\"\n\n\t// Min int64 can't be negated to a usable value, so it has to be special cased.\n\tif v == math.MinInt64 {\n\t\treturn \"-9,223,372,036,854,775,808\"\n\t}\n\n\tif v < 0 {\n\t\tsign = \"-\"\n\t\tv = 0 - v\n\t}\n\n\tparts := []string{\"\", \"\", \"\", \"\", \"\", \"\", \"\"}\n\tj := len(parts) - 1\n\n\tfor v > 999 {\n\t\tparts[j] = strconv.FormatInt(v%1000, 10)\n\t\tswitch len(parts[j]) {\n\t\tcase 2:\n\t\t\tparts[j] = \"0\" + parts[j]\n\t\tcase 1:\n\t\t\tparts[j] = \"00\" + parts[j]\n\t\t}\n\t\tv = v / 1000\n\t\tj--\n\t}\n\tparts[j] = strconv.Itoa(int(v))\n\treturn sign + strings.Join(parts[j:], \",\")\n}\n\n// Adds urlStr to the given map. If the string was already present, simply\n// bumps the reference count.\n// Returns true only if it was added for the first time.\nfunc (m refCountedUrlSet) addUrl(urlStr string) bool {\n\tm[urlStr]++\n\treturn m[urlStr] == 1\n}\n\n// Removes urlStr from the given map. If the string is not present, nothing\n// is done and false is returned.\n// If the string was present, its reference count is decreased. Returns true\n// if this was the last reference, false otherwise.\nfunc (m refCountedUrlSet) removeUrl(urlStr string) bool {\n\tremoved := false\n\tif ref, ok := m[urlStr]; ok {\n\t\tif ref == 1 {\n\t\t\tremoved = true\n\t\t\tdelete(m, urlStr)\n\t\t} else {\n\t\t\tm[urlStr]--\n\t\t}\n\t}\n\treturn removed\n}\n\n// Returns the unique URLs in this map as a slice\nfunc (m refCountedUrlSet) getAsStringSlice() []string {\n\ta := make([]string, 0, len(m))\n\tfor u := range m {\n\t\ta = append(a, u)\n\t}\n\treturn a\n}\n\n// natsListenConfig provides a common configuration to match the one used by\n// net.Listen() but with our own defaults.\n// Go 1.13 introduced default-on TCP keepalives with aggressive timings and\n// there's no sane portable way in Go with stdlib to split the initial timer\n// from the retry timer. Linux/BSD defaults are 2hrs/75s and Go sets both\n// to 15s; the issue re making them indepedently tunable has been open since\n// 2014 and this code here is being written in 2020.\n// The NATS protocol has its own L7 PING/PONG keepalive system and the Go\n// defaults are inappropriate for IoT deployment scenarios.\n// Replace any NATS-protocol calls to net.Listen(...) with\n// natsListenConfig.Listen(ctx,...) or use natsListen(); leave calls for HTTP\n// monitoring, etc, on the default.\nvar natsListenConfig = &net.ListenConfig{\n\tKeepAlive: -1,\n}\n\n// natsListen() is the same as net.Listen() except that TCP keepalives are\n// disabled (to match Go's behavior before Go 1.13).\nfunc natsListen(network, address string) (net.Listener, error) {\n\treturn natsListenConfig.Listen(context.Background(), network, address)\n}\n\n// natsDialTimeout is the same as net.DialTimeout() except the TCP keepalives\n// are disabled (to match Go's behavior before Go 1.13).\nfunc natsDialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {\n\td := net.Dialer{\n\t\tTimeout: timeout,\n\t\tKeepAlive: -1,\n\t}\n\treturn d.Dial(network, address)\n}\n\n// redactURLList() returns a copy of a list of URL pointers where each item\n// in the list will either be the same pointer if the URL does not contain a\n// password, or to a new object if there is a password.\n// The intended use-case is for logging lists of URLs safely.\nfunc redactURLList(unredacted []*url.URL) []*url.URL {\n\tr := make([]*url.URL, len(unredacted))\n\t// In the common case of no passwords, if we don't let the new object leave\n\t// this function then GC should be easier.\n\tneedCopy := false\n\tfor i := range unredacted {\n\t\tif unredacted[i] == nil {\n\t\t\tr[i] = nil\n\t\t\tcontinue\n\t\t}\n\t\tif _, has := unredacted[i].User.Password(); !has {\n\t\t\tr[i] = unredacted[i]\n\t\t\tcontinue\n\t\t}\n\t\tneedCopy = true\n\t\tru := *unredacted[i]\n\t\tru.User = url.UserPassword(ru.User.Username(), \"xxxxx\")\n\t\tr[i] = &ru\n\t}\n\tif needCopy {\n\t\treturn r\n\t}\n\treturn unredacted\n}\n\n// redactURLString() attempts to redact a URL string.\nfunc redactURLString(raw string) string {\n\tif !strings.ContainsRune(raw, '@') {\n\t\treturn raw\n\t}\n\tu, err := url.Parse(raw)\n\tif err != nil {\n\t\treturn raw\n\t}\n\treturn u.Redacted()\n}\n\n// getURLsAsString returns a slice of u.Host from the given slice of url.URL's\nfunc getURLsAsString(urls []*url.URL) []string {\n\ta := make([]string, 0, len(urls))\n\tfor _, u := range urls {\n\t\ta = append(a, u.Host)\n\t}\n\treturn a\n}\n\n// copyBytes make a new slice of the same size as `src` and copy its content.\n// If `src` is nil or its length is 0, then this returns `nil`\nfunc copyBytes(src []byte) []byte {\n\tif len(src) == 0 {\n\t\treturn nil\n\t}\n\tdst := make([]byte, len(src))\n\tcopy(dst, src)\n\treturn dst\n}\n\n// copyStrings make a new slice of the same size than `src` and copy its content.\n// If `src` is nil, then this returns `nil`\nfunc copyStrings(src []string) []string {\n\tif src == nil {\n\t\treturn nil\n\t}\n\tdst := make([]string, len(src))\n\tcopy(dst, src)\n\treturn dst\n}\n\n// Returns a byte slice for the INFO protocol.\nfunc generateInfoJSON(info *Info) []byte {\n\tb, _ := json.Marshal(info)\n\tpcs := [][]byte{[]byte(\"INFO\"), b, []byte(CR_LF)}\n\treturn bytes.Join(pcs, []byte(\" \"))\n}\n\n// parallelTaskQueue starts a number of goroutines and returns a channel\n// which functions can be sent to for queued parallel execution. The\n// goroutines will stop running when the returned channel is closed and\n// all queued tasks have completed. The passed in mp limits concurrency,\n// or a value <= 0 will default to GOMAXPROCS.\nfunc parallelTaskQueue(mp int) chan<- func() {\n\tif rmp := runtime.GOMAXPROCS(-1); mp <= 0 {\n\t\tmp = rmp\n\t} else {\n\t\tmp = max(rmp, mp)\n\t}\n\ttq := make(chan func(), mp)\n\tfor range mp {\n\t\tgo func() {\n\t\t\tfor fn := range tq {\n\t\t\t\tfn()\n\t\t\t}\n\t\t}()\n\t}\n\treturn tq\n}\n" }, { "path": "server/util_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"math\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestParseSize(t *testing.T) {\n\tif parseSize(nil) != -1 {\n\t\tt.Fatal(\"Should error on nil byte slice\")\n\t}\n\tn := []byte(\"12345678\")\n\tif pn := parseSize(n); pn != 12345678 {\n\t\tt.Fatalf(\"Did not parse %q correctly, res=%d\", n, pn)\n\t}\n\n\tn = []byte(\"12345invalid678\")\n\tif pn := parseSize(n); pn != -1 {\n\t\tt.Fatalf(\"Should error on %q, res=%d\", n, pn)\n\t}\n}\n\nfunc TestParseSInt64(t *testing.T) {\n\tif parseInt64(nil) != -1 {\n\t\tt.Fatal(\"Should error on nil byte slice\")\n\t}\n\tn := []byte(\"12345678\")\n\tif pn := parseInt64(n); pn != 12345678 {\n\t\tt.Fatalf(\"Did not parse %q correctly, res=%d\", n, pn)\n\t}\n\n\tn = []byte(\"12345invalid678\")\n\tif pn := parseInt64(n); pn != -1 {\n\t\tt.Fatalf(\"Should error on %q, res=%d\", n, pn)\n\t}\n}\n\nfunc TestParseHostPort(t *testing.T) {\n\tcheck := func(hostPort string, defaultPort int, expectedHost string, expectedPort int, expectedErr bool) {\n\t\th, p, err := parseHostPort(hostPort, defaultPort)\n\t\tif expectedErr {\n\t\t\tif err == nil {\n\t\t\t\tstackFatalf(t, \"Expected an error, did not get one\")\n\t\t\t}\n\t\t\t// expected error, so we are done\n\t\t\treturn\n\t\t}\n\t\tif !expectedErr && err != nil {\n\t\t\tstackFatalf(t, \"Unexpected error: %v\", err)\n\t\t}\n\t\tif expectedHost != h {\n\t\t\tstackFatalf(t, \"Expected host %q, got %q\", expectedHost, h)\n\t\t}\n\t\tif expectedPort != p {\n\t\t\tstackFatalf(t, \"Expected port %d, got %d\", expectedPort, p)\n\t\t}\n\t}\n\tcheck(\"addr:1234\", 5678, \"addr\", 1234, false)\n\tcheck(\" addr:1234 \", 5678, \"addr\", 1234, false)\n\tcheck(\" addr : 1234 \", 5678, \"addr\", 1234, false)\n\tcheck(\"addr\", 5678, \"addr\", 5678, false)\n\tcheck(\" addr \", 5678, \"addr\", 5678, false)\n\tcheck(\"addr:-1\", 5678, \"addr\", 5678, false)\n\tcheck(\" addr:-1 \", 5678, \"addr\", 5678, false)\n\tcheck(\" addr : -1 \", 5678, \"addr\", 5678, false)\n\tcheck(\"addr:0\", 5678, \"addr\", 5678, false)\n\tcheck(\" addr:0 \", 5678, \"addr\", 5678, false)\n\tcheck(\" addr : 0 \", 5678, \"addr\", 5678, false)\n\tcheck(\"addr:addr\", 0, \"\", 0, true)\n\tcheck(\"addr:::1234\", 0, \"\", 0, true)\n\tcheck(\"\", 0, \"\", 0, true)\n}\n\nfunc TestURLsAreEqual(t *testing.T) {\n\tcheck := func(t *testing.T, u1Str, u2Str string, expectedSame bool) {\n\t\tt.Helper()\n\t\tu1, err := url.Parse(u1Str)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error parsing url %q: %v\", u1Str, err)\n\t\t}\n\t\tu2, err := url.Parse(u2Str)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error parsing url %q: %v\", u2Str, err)\n\t\t}\n\t\tsame := urlsAreEqual(u1, u2)\n\t\tif expectedSame && !same {\n\t\t\tt.Fatalf(\"Expected %v and %v to be the same, they were not\", u1, u2)\n\t\t} else if !expectedSame && same {\n\t\t\tt.Fatalf(\"Expected %v and %v to be different, they were not\", u1, u2)\n\t\t}\n\t}\n\tcheck(t, \"nats://localhost:4222\", \"nats://localhost:4222\", true)\n\tcheck(t, \"nats://ivan:pwd@localhost:4222\", \"nats://ivan:pwd@localhost:4222\", true)\n\tcheck(t, \"nats://ivan@localhost:4222\", \"nats://ivan@localhost:4222\", true)\n\tcheck(t, \"nats://ivan:@localhost:4222\", \"nats://ivan:@localhost:4222\", true)\n\tcheck(t, \"nats://host1:4222\", \"nats://host2:4222\", false)\n}\n\nfunc TestComma(t *testing.T) {\n\ttype testList []struct {\n\t\tname, got, exp string\n\t}\n\n\tl := testList{\n\t\t{\"0\", comma(0), \"0\"},\n\t\t{\"10\", comma(10), \"10\"},\n\t\t{\"100\", comma(100), \"100\"},\n\t\t{\"1,000\", comma(1000), \"1,000\"},\n\t\t{\"10,000\", comma(10000), \"10,000\"},\n\t\t{\"100,000\", comma(100000), \"100,000\"},\n\t\t{\"10,000,000\", comma(10000000), \"10,000,000\"},\n\t\t{\"10,100,000\", comma(10100000), \"10,100,000\"},\n\t\t{\"10,010,000\", comma(10010000), \"10,010,000\"},\n\t\t{\"10,001,000\", comma(10001000), \"10,001,000\"},\n\t\t{\"123,456,789\", comma(123456789), \"123,456,789\"},\n\t\t{\"maxint\", comma(9.223372e+18), \"9,223,372,000,000,000,000\"},\n\t\t{\"math.maxint\", comma(math.MaxInt64), \"9,223,372,036,854,775,807\"},\n\t\t{\"math.minint\", comma(math.MinInt64), \"-9,223,372,036,854,775,808\"},\n\t\t{\"minint\", comma(-9.223372e+18), \"-9,223,372,000,000,000,000\"},\n\t\t{\"-123,456,789\", comma(-123456789), \"-123,456,789\"},\n\t\t{\"-10,100,000\", comma(-10100000), \"-10,100,000\"},\n\t\t{\"-10,010,000\", comma(-10010000), \"-10,010,000\"},\n\t\t{\"-10,001,000\", comma(-10001000), \"-10,001,000\"},\n\t\t{\"-10,000,000\", comma(-10000000), \"-10,000,000\"},\n\t\t{\"-100,000\", comma(-100000), \"-100,000\"},\n\t\t{\"-10,000\", comma(-10000), \"-10,000\"},\n\t\t{\"-1,000\", comma(-1000), \"-1,000\"},\n\t\t{\"-100\", comma(-100), \"-100\"},\n\t\t{\"-10\", comma(-10), \"-10\"},\n\t}\n\n\tfailed := false\n\tfor _, test := range l {\n\t\tif test.got != test.exp {\n\t\t\tt.Errorf(\"On %v, expected '%v', but got '%v'\",\n\t\t\t\ttest.name, test.exp, test.got)\n\t\t\tfailed = true\n\t\t}\n\t}\n\tif failed {\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestURLRedaction(t *testing.T) {\n\tredactionFromTo := []struct {\n\t\tFull string\n\t\tSafe string\n\t}{\n\t\t{\"nats://foo:bar@example.org\", \"nats://foo:xxxxx@example.org\"},\n\t\t{\"nats://foo@example.org\", \"nats://foo@example.org\"},\n\t\t{\"nats://example.org\", \"nats://example.org\"},\n\t\t{\"nats://example.org/foo?bar=1\", \"nats://example.org/foo?bar=1\"},\n\t}\n\tvar err error\n\tlistFull := make([]*url.URL, len(redactionFromTo))\n\tlistSafe := make([]*url.URL, len(redactionFromTo))\n\tfor i := range redactionFromTo {\n\t\tr := redactURLString(redactionFromTo[i].Full)\n\t\tif r != redactionFromTo[i].Safe {\n\t\t\tt.Fatalf(\"Redacting URL [index %d] %q, expected %q got %q\", i, redactionFromTo[i].Full, redactionFromTo[i].Safe, r)\n\t\t}\n\t\tif listFull[i], err = url.Parse(redactionFromTo[i].Full); err != nil {\n\t\t\tt.Fatalf(\"Redacting URL index %d parse Full failed: %v\", i, err)\n\t\t}\n\t\tif listSafe[i], err = url.Parse(redactionFromTo[i].Safe); err != nil {\n\t\t\tt.Fatalf(\"Redacting URL index %d parse Safe failed: %v\", i, err)\n\t\t}\n\t}\n\tresults := redactURLList(listFull)\n\tif !reflect.DeepEqual(results, listSafe) {\n\t\tt.Fatalf(\"Redacting URL list did not compare equal, even after each URL did\")\n\t}\n}\n\nfunc TestVersionAtLeast(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tversion string\n\t\tmajor int\n\t\tminor int\n\t\tupdate int\n\t\tresult bool\n\t}{\n\t\t{\"2.0.0-beta\", 1, 9, 9, true},\n\t\t{\"2.0.0\", 1, 99, 9, true},\n\t\t{\"2.2.0\", 2, 1, 9, true},\n\t\t{\"2.2.2\", 2, 2, 2, true},\n\t\t{\"2.2.2\", 2, 2, 3, false},\n\t\t{\"2.2.2\", 2, 3, 2, false},\n\t\t{\"2.2.2\", 3, 2, 2, false},\n\t\t{\"2.22.2\", 3, 0, 0, false},\n\t\t{\"2.2.22\", 2, 3, 0, false},\n\t\t{\"bad.version\", 1, 2, 3, false},\n\t} {\n\t\tt.Run(_EMPTY_, func(t *testing.T) {\n\t\t\tif res := versionAtLeast(test.version, test.major, test.minor, test.update); res != test.result {\n\t\t\t\tt.Fatalf(\"For check version %q at least %d.%d.%d result should have been %v, got %v\",\n\t\t\t\t\ttest.version, test.major, test.minor, test.update, test.result, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkParseInt(b *testing.B) {\n\tb.SetBytes(1)\n\tn := \"12345678\"\n\tfor i := 0; i < b.N; i++ {\n\t\tstrconv.ParseInt(n, 10, 0)\n\t}\n}\n\nfunc BenchmarkParseSize(b *testing.B) {\n\tb.SetBytes(1)\n\tn := []byte(\"12345678\")\n\tfor i := 0; i < b.N; i++ {\n\t\tparseSize(n)\n\t}\n}\n\nfunc deferUnlock(mu *sync.Mutex) {\n\tmu.Lock()\n\tdefer mu.Unlock()\n\t// see noDeferUnlock\n\tif false {\n\t\treturn\n\t}\n}\n\nfunc BenchmarkDeferMutex(b *testing.B) {\n\tvar mu sync.Mutex\n\tb.SetBytes(1)\n\tfor i := 0; i < b.N; i++ {\n\t\tdeferUnlock(&mu)\n\t}\n}\n\nfunc noDeferUnlock(mu *sync.Mutex) {\n\tmu.Lock()\n\t// prevent staticcheck warning about empty critical section\n\tif false {\n\t\treturn\n\t}\n\tmu.Unlock()\n}\n\nfunc BenchmarkNoDeferMutex(b *testing.B) {\n\tvar mu sync.Mutex\n\tb.SetBytes(1)\n\tfor i := 0; i < b.N; i++ {\n\t\tnoDeferUnlock(&mu)\n\t}\n}\n\nfunc createTestSub() *subscription {\n\treturn &subscription{\n\t\tsubject: []byte(\"foo\"),\n\t\tqueue: []byte(\"bar\"),\n\t\tsid: []byte(\"22\"),\n\t}\n}\n\nfunc BenchmarkArrayRand(b *testing.B) {\n\tb.StopTimer()\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\t// Create an array of 10 items\n\tsubs := []*subscription{}\n\tfor i := 0; i < 10; i++ {\n\t\tsubs = append(subs, createTestSub())\n\t}\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tindex := r.Intn(len(subs))\n\t\t_ = subs[index]\n\t}\n}\n\nfunc BenchmarkMapRange(b *testing.B) {\n\tb.StopTimer()\n\t// Create an map of 10 items\n\tsubs := map[int]*subscription{}\n\tfor i := 0; i < 10; i++ {\n\t\tsubs[i] = createTestSub()\n\t}\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfor range subs {\n\t\t\tbreak\n\t\t}\n\t}\n}\n" }, { "path": "server/websocket.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bytes\"\n\tcrand \"crypto/rand\"\n\t\"crypto/sha1\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\tmrand \"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/klauspost/compress/flate\"\n)\n\ntype wsOpCode int\n\nconst (\n\t// From https://tools.ietf.org/html/rfc6455#section-5.2\n\twsTextMessage = wsOpCode(1)\n\twsBinaryMessage = wsOpCode(2)\n\twsCloseMessage = wsOpCode(8)\n\twsPingMessage = wsOpCode(9)\n\twsPongMessage = wsOpCode(10)\n\n\twsFinalBit = 1 << 7\n\twsRsv1Bit = 1 << 6 // Used for compression, from https://tools.ietf.org/html/rfc7692#section-6\n\twsRsv2Bit = 1 << 5\n\twsRsv3Bit = 1 << 4\n\n\twsMaskBit = 1 << 7\n\n\twsContinuationFrame = 0\n\twsMaxFrameHeaderSize = 14 // Since LeafNode may need to behave as a client\n\twsMaxControlPayloadSize = 125\n\twsFrameSizeForBrowsers = 4096 // From experiment, webrowsers behave better with limited frame size\n\twsCompressThreshold = 64 // Don't compress for small buffer(s)\n\twsCloseSatusSize = 2\n\n\t// From https://tools.ietf.org/html/rfc6455#section-11.7\n\twsCloseStatusNormalClosure = 1000\n\twsCloseStatusGoingAway = 1001\n\twsCloseStatusProtocolError = 1002\n\twsCloseStatusUnsupportedData = 1003\n\twsCloseStatusNoStatusReceived = 1005\n\twsCloseStatusInvalidPayloadData = 1007\n\twsCloseStatusPolicyViolation = 1008\n\twsCloseStatusMessageTooBig = 1009\n\twsCloseStatusInternalSrvError = 1011\n\twsCloseStatusTLSHandshake = 1015\n\n\twsFirstFrame = true\n\twsContFrame = false\n\twsFinalFrame = true\n\twsUncompressedFrame = false\n\n\twsSchemePrefix = \"ws\"\n\twsSchemePrefixTLS = \"wss\"\n\n\twsNoMaskingHeader = \"Nats-No-Masking\"\n\twsNoMaskingValue = \"true\"\n\twsXForwardedForHeader = \"X-Forwarded-For\"\n\twsNoMaskingFullResponse = wsNoMaskingHeader + \": \" + wsNoMaskingValue + CR_LF\n\twsPMCExtension = \"permessage-deflate\" // per-message compression\n\twsPMCSrvNoCtx = \"server_no_context_takeover\"\n\twsPMCCliNoCtx = \"client_no_context_takeover\"\n\twsPMCReqHeaderValue = wsPMCExtension + \"; \" + wsPMCSrvNoCtx + \"; \" + wsPMCCliNoCtx\n\twsPMCFullResponse = \"Sec-WebSocket-Extensions: \" + wsPMCExtension + \"; \" + wsPMCSrvNoCtx + \"; \" + wsPMCCliNoCtx + _CRLF_\n\twsSecProto = \"Sec-Websocket-Protocol\"\n\twsMQTTSecProtoVal = \"mqtt\"\n\twsMQTTSecProto = wsSecProto + \": \" + wsMQTTSecProtoVal + CR_LF\n)\n\nvar decompressorPool sync.Pool\nvar compressLastBlock = []byte{0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff}\n\n// From https://tools.ietf.org/html/rfc6455#section-1.3\nvar wsGUID = []byte(\"258EAFA5-E914-47DA-95CA-C5AB0DC85B11\")\n\n// Test can enable this so that server does not support \"no-masking\" requests.\nvar wsTestRejectNoMasking = false\n\ntype websocket struct {\n\tframes net.Buffers\n\tfs int64\n\tcloseMsg []byte\n\tcompress bool\n\tcloseSent bool\n\tbrowser bool\n\tnocompfrag bool // No fragment for compressed frames\n\tmaskread bool\n\tmaskwrite bool\n\tcompressor *flate.Writer\n\tcookieJwt string\n\tcookieUsername string\n\tcookiePassword string\n\tcookieToken string\n\tclientIP string\n}\n\ntype srvWebsocket struct {\n\tmu sync.RWMutex\n\tserver *http.Server\n\tlistener net.Listener\n\tlistenerErr error\n\tallowedOrigins map[string][]*allowedOrigin // host will be the key\n\tsameOrigin bool\n\tconnectURLs []string\n\tconnectURLsMap refCountedUrlSet\n\tauthOverride bool // indicate if there is auth override in websocket config\n\trawHeaders string // raw headers to be used in the upgrade response.\n\n\t// These are immutable and can be accessed without lock.\n\t// This is the case when generating the client INFO.\n\ttls bool // True if TLS is required (TLSConfig is specified).\n\thost string // Host/IP the webserver is listening on (shortcut to opts.Websocket.Host).\n\tport int // Port the webserver is listening on. This is after an ephemeral port may have been selected (shortcut to opts.Websocket.Port).\n}\n\ntype allowedOrigin struct {\n\tscheme string\n\tport string\n}\n\ntype wsUpgradeResult struct {\n\tconn net.Conn\n\tws *websocket\n\tkind int\n}\n\ntype wsReadInfo struct {\n\trem uint64\n\tfs bool\n\tff bool\n\tfc bool\n\tmask bool // Incoming leafnode connections may not have masking.\n\tmkpos byte\n\tmkey [4]byte\n\tcbufs [][]byte\n\tcoff int\n\tcsz uint64\n}\n\nfunc (r *wsReadInfo) init() {\n\tr.fs, r.ff = true, true\n}\n\nfunc (r *wsReadInfo) resetCompressedState() {\n\tr.fs = true\n\tr.ff = true\n\tr.fc = false\n\tr.rem = 0\n\tr.cbufs = nil\n\tr.coff = 0\n\tr.csz = 0\n}\n\n// Returns a slice containing `needed` bytes from the given buffer `buf`\n// starting at position `pos`, and possibly read from the given reader `r`.\n// When bytes are present in `buf`, the `pos` is incremented by the number\n// of bytes found up to `needed` and the new position is returned. If not\n// enough bytes are found, the bytes found in `buf` are copied to the returned\n// slice and the remaning bytes are read from `r`.\nfunc wsGet(r io.Reader, buf []byte, pos, needed uint64) ([]byte, uint64, error) {\n\tavail := uint64(len(buf)) - pos\n\tif avail >= needed {\n\t\treturn buf[pos : pos+needed], pos + needed, nil\n\t}\n\tb := make([]byte, needed)\n\tstart := uint64(copy(b, buf[pos:]))\n\tfor start != needed {\n\t\tn, err := r.Read(b[start:cap(b)])\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\tstart += uint64(n)\n\t}\n\treturn b, pos + avail, nil\n}\n\n// Returns true if this connection is from a Websocket client.\n// Lock held on entry.\nfunc (c *client) isWebsocket() bool {\n\treturn c.ws != nil\n}\n\n// Returns a slice of byte slices corresponding to payload of websocket frames.\n// The byte slice `buf` is filled with bytes from the connection's read loop.\n// This function will decode the frame headers and unmask the payload(s).\n// It is possible that the returned slices point to the given `buf` slice, so\n// `buf` should not be overwritten until the returned slices have been parsed.\n//\n// Client lock MUST NOT be held on entry.\nfunc (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, error) {\n\tvar (\n\t\tbufs [][]byte\n\t\ttmpBuf []byte\n\t\terr error\n\t\tpos uint64\n\t\tmax = uint64(len(buf))\n\t\tmpay = int(atomic.LoadInt32(&c.mpay))\n\t)\n\tif mpay <= 0 {\n\t\tmpay = MAX_PAYLOAD_SIZE\n\t}\n\tfor pos != max {\n\t\tif r.fs {\n\t\t\tb0 := buf[pos]\n\t\t\tframeType := wsOpCode(b0 & 0xF)\n\t\t\tfinal := b0&wsFinalBit != 0\n\t\t\tcompressed := b0&wsRsv1Bit != 0\n\t\t\tif b0&(wsRsv2Bit|wsRsv3Bit) != 0 {\n\t\t\t\treturn bufs, c.wsHandleProtocolError(\"RSV2 and RSV3 must be clear\")\n\t\t\t}\n\t\t\tif compressed && !c.ws.compress {\n\t\t\t\treturn bufs, c.wsHandleProtocolError(\"compressed frame received without negotiated permessage-deflate\")\n\t\t\t}\n\t\t\tpos++\n\n\t\t\ttmpBuf, pos, err = wsGet(ior, buf, pos, 1)\n\t\t\tif err != nil {\n\t\t\t\treturn bufs, err\n\t\t\t}\n\t\t\tb1 := tmpBuf[0]\n\n\t\t\t// Clients MUST set the mask bit. If not set, reject.\n\t\t\t// However, LEAF by default will not have masking, unless they are forced to, by configuration.\n\t\t\tif r.mask && b1&wsMaskBit == 0 {\n\t\t\t\treturn bufs, c.wsHandleProtocolError(\"mask bit missing\")\n\t\t\t}\n\n\t\t\t// Store size in case it is < 125\n\t\t\tr.rem = uint64(b1 & 0x7F)\n\n\t\t\tswitch frameType {\n\t\t\tcase wsPingMessage, wsPongMessage, wsCloseMessage:\n\t\t\t\tif r.rem > wsMaxControlPayloadSize {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\n\t\t\t\t\t\tfmt.Sprintf(\"control frame length bigger than maximum allowed of %v bytes\",\n\t\t\t\t\t\t\twsMaxControlPayloadSize))\n\t\t\t\t}\n\t\t\t\tif !final {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\"control frame does not have final bit set\")\n\t\t\t\t}\n\t\t\t\tif compressed {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\"control frame must not be compressed\")\n\t\t\t\t}\n\t\t\tcase wsTextMessage, wsBinaryMessage:\n\t\t\t\tif !r.ff {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\"new message started before final frame for previous message was received\")\n\t\t\t\t}\n\t\t\t\tr.ff = final\n\t\t\t\tr.fc = compressed\n\t\t\tcase wsContinuationFrame:\n\t\t\t\t// Compressed bit must be only set in the first frame\n\t\t\t\tif r.ff || compressed {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\"invalid continuation frame\")\n\t\t\t\t}\n\t\t\t\tr.ff = final\n\t\t\tdefault:\n\t\t\t\treturn bufs, c.wsHandleProtocolError(fmt.Sprintf(\"unknown opcode %v\", frameType))\n\t\t\t}\n\n\t\t\tswitch r.rem {\n\t\t\tcase 126:\n\t\t\t\ttmpBuf, pos, err = wsGet(ior, buf, pos, 2)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn bufs, err\n\t\t\t\t}\n\t\t\t\tr.rem = uint64(binary.BigEndian.Uint16(tmpBuf))\n\t\t\tcase 127:\n\t\t\t\ttmpBuf, pos, err = wsGet(ior, buf, pos, 8)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn bufs, err\n\t\t\t\t}\n\t\t\t\tif r.rem = binary.BigEndian.Uint64(tmpBuf); r.rem&(uint64(1)<<63) != 0 {\n\t\t\t\t\treturn bufs, c.wsHandleProtocolError(\"invalid 64-bit payload length\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif r.mask {\n\t\t\t\t// Read masking key\n\t\t\t\ttmpBuf, pos, err = wsGet(ior, buf, pos, 4)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn bufs, err\n\t\t\t\t}\n\t\t\t\tcopy(r.mkey[:], tmpBuf)\n\t\t\t\tr.mkpos = 0\n\t\t\t}\n\n\t\t\t// Handle control messages in place...\n\t\t\tif wsIsControlFrame(frameType) {\n\t\t\t\tpos, err = c.wsHandleControlFrame(r, frameType, ior, buf, pos)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn bufs, err\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Done with the frame header\n\t\t\tr.fs = false\n\t\t}\n\t\tif pos < max {\n\t\t\tvar b []byte\n\t\t\tvar n uint64\n\n\t\t\tn = r.rem\n\t\t\tif pos+n > max {\n\t\t\t\tn = max - pos\n\t\t\t}\n\t\t\tb = buf[pos : pos+n]\n\t\t\tpos += n\n\t\t\tr.rem -= n\n\t\t\t// If needed, unmask the buffer\n\t\t\tif r.mask {\n\t\t\t\tr.unmask(b)\n\t\t\t}\n\t\t\taddToBufs := true\n\t\t\t// Handle compressed message\n\t\t\tif r.fc {\n\t\t\t\t// Assume that we may have continuation frames or not the full payload.\n\t\t\t\taddToBufs = false\n\t\t\t\tif r.csz+uint64(len(b)) > uint64(mpay) {\n\t\t\t\t\tr.resetCompressedState()\n\t\t\t\t\treturn bufs, ErrMaxPayload\n\t\t\t\t}\n\t\t\t\t// Make a copy of the buffer before adding it to the list\n\t\t\t\t// of compressed fragments.\n\t\t\t\tr.cbufs = append(r.cbufs, append([]byte(nil), b...))\n\t\t\t\tr.csz += uint64(len(b))\n\t\t\t\t// When we have the final frame and we have read the full payload,\n\t\t\t\t// we can decompress it.\n\t\t\t\tif r.ff && r.rem == 0 {\n\t\t\t\t\tb, err = r.decompress(mpay)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tr.resetCompressedState()\n\t\t\t\t\t\treturn bufs, err\n\t\t\t\t\t}\n\t\t\t\t\tr.fc = false\n\t\t\t\t\t// Now we can add to `bufs`\n\t\t\t\t\taddToBufs = true\n\t\t\t\t}\n\t\t\t}\n\t\t\t// For non compressed frames, or when we have decompressed the\n\t\t\t// whole message.\n\t\t\tif addToBufs {\n\t\t\t\tbufs = append(bufs, b)\n\t\t\t}\n\t\t\t// If payload has been fully read, then indicate that next\n\t\t\t// is the start of a frame.\n\t\t\tif r.rem == 0 {\n\t\t\t\tr.fs = true\n\t\t\t}\n\t\t}\n\t}\n\treturn bufs, nil\n}\n\nfunc (r *wsReadInfo) Read(dst []byte) (int, error) {\n\tif len(dst) == 0 {\n\t\treturn 0, nil\n\t}\n\tif len(r.cbufs) == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tcopied := 0\n\trem := len(dst)\n\tfor buf := r.cbufs[0]; buf != nil && rem > 0; {\n\t\tn := len(buf[r.coff:])\n\t\tif n > rem {\n\t\t\tn = rem\n\t\t}\n\t\tcopy(dst[copied:], buf[r.coff:r.coff+n])\n\t\tcopied += n\n\t\trem -= n\n\t\tr.coff += n\n\t\tbuf = r.nextCBuf()\n\t}\n\treturn copied, nil\n}\n\nfunc (r *wsReadInfo) nextCBuf() []byte {\n\t// We still have remaining data in the first buffer\n\tif r.coff != len(r.cbufs[0]) {\n\t\treturn r.cbufs[0]\n\t}\n\t// We read the full first buffer. Reset offset.\n\tr.coff = 0\n\t// We were at the last buffer, so we are done.\n\tif len(r.cbufs) == 1 {\n\t\tr.cbufs = nil\n\t\treturn nil\n\t}\n\t// Here we move to the next buffer.\n\tr.cbufs = r.cbufs[1:]\n\treturn r.cbufs[0]\n}\n\nfunc (r *wsReadInfo) ReadByte() (byte, error) {\n\tfor len(r.cbufs) > 0 && len(r.cbufs[0]) == 0 {\n\t\tr.nextCBuf()\n\t}\n\tif len(r.cbufs) == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tb := r.cbufs[0][r.coff]\n\tr.coff++\n\tr.nextCBuf()\n\treturn b, nil\n}\n\n// decompress decompresses the collected buffers.\n// The size of the decompressed buffer will be limited to the `mpay` value.\n// If, while decompressing, the resulting uncompressed buffer exceeds this\n// limit, the decompression stops and an empty buffer and the ErrMaxPayload\n// error are returned.\nfunc (r *wsReadInfo) decompress(mpay int) ([]byte, error) {\n\t// If not limit is specified, use the default maximum payload size.\n\tif mpay <= 0 {\n\t\tmpay = MAX_PAYLOAD_SIZE\n\t}\n\tr.coff = 0\n\t// As per https://tools.ietf.org/html/rfc7692#section-7.2.2\n\t// add 0x00, 0x00, 0xff, 0xff and then a final block so that flate reader\n\t// does not report unexpected EOF.\n\tr.cbufs = append(r.cbufs, compressLastBlock)\n\t// Get a decompressor from the pool and bind it to this object (wsReadInfo)\n\t// that provides Read() and ReadByte() APIs that will consume the compressed\n\t// buffers (r.cbufs).\n\td, _ := decompressorPool.Get().(io.ReadCloser)\n\tif d == nil {\n\t\td = flate.NewReader(r)\n\t} else {\n\t\td.(flate.Resetter).Reset(r, nil)\n\t}\n\t// Use a LimitedReader to limit the decompressed size.\n\t// We use \"limit+1\" bytes for \"N\" so we can detect if the limit is exceeded.\n\tlr := io.LimitedReader{R: d, N: int64(mpay + 1)}\n\tb, err := io.ReadAll(&lr)\n\tif err == nil && len(b) > mpay {\n\t\t// Decompressed data exceeds the maximum payload size.\n\t\tb, err = nil, ErrMaxPayload\n\t}\n\tlr.R = nil\n\tdecompressorPool.Put(d)\n\t// Now reset the compressed buffers list.\n\tr.cbufs = nil\n\tr.coff = 0\n\tr.csz = 0\n\treturn b, err\n}\n\n// Handles the PING, PONG and CLOSE websocket control frames.\n//\n// Client lock MUST NOT be held on entry.\nfunc (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.Reader, buf []byte, pos uint64) (uint64, error) {\n\tvar payload []byte\n\tvar err error\n\n\tif r.rem > 0 {\n\t\tpayload, pos, err = wsGet(nc, buf, pos, r.rem)\n\t\tif err != nil {\n\t\t\treturn pos, err\n\t\t}\n\t\tif r.mask {\n\t\t\tr.unmask(payload)\n\t\t}\n\t\tr.rem = 0\n\t}\n\tswitch frameType {\n\tcase wsCloseMessage:\n\t\tstatus := wsCloseStatusNoStatusReceived\n\t\tvar body string\n\t\tlp := len(payload)\n\t\tif lp == 1 {\n\t\t\treturn pos, c.wsHandleProtocolError(\"close frame payload cannot be 1 byte\")\n\t\t}\n\t\t// If there is a payload, the status is represented as a 2-byte\n\t\t// unsigned integer (in network byte order). Then, there may be an\n\t\t// optional body.\n\t\thasStatus, hasBody := lp >= wsCloseSatusSize, lp > wsCloseSatusSize\n\t\tif hasStatus {\n\t\t\t// Decode the status\n\t\t\tstatus = int(binary.BigEndian.Uint16(payload[:wsCloseSatusSize]))\n\t\t\tif !wsIsValidCloseStatus(status) {\n\t\t\t\treturn pos, c.wsHandleProtocolError(fmt.Sprintf(\"invalid close status code %v\", status))\n\t\t\t}\n\t\t\t// Now if there is a body, capture it and make sure this is a valid UTF-8.\n\t\t\tif hasBody {\n\t\t\t\tbody = string(payload[wsCloseSatusSize:])\n\t\t\t\tif !utf8.ValidString(body) {\n\t\t\t\t\t// https://tools.ietf.org/html/rfc6455#section-5.5.1\n\t\t\t\t\t// If body is present, it must be a valid utf8\n\t\t\t\t\tstatus = wsCloseStatusInvalidPayloadData\n\t\t\t\t\tbody = \"invalid utf8 body in close frame\"\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// If the status indicates that nothing was received, then we don't\n\t\t// send anything back.\n\t\t// From https://datatracker.ietf.org/doc/html/rfc6455#section-7.4\n\t\t// it says that code 1005 is a reserved value and MUST NOT be set as a\n\t\t// status code in a Close control frame by an endpoint. It is\n\t\t// designated for use in applications expecting a status code to indicate\n\t\t// that no status code was actually present.\n\t\tvar clm []byte\n\t\tif status != wsCloseStatusNoStatusReceived {\n\t\t\tclm = wsCreateCloseMessage(status, body)\n\t\t}\n\t\tc.wsEnqueueControlMessage(wsCloseMessage, clm)\n\t\tif len(clm) > 0 {\n\t\t\tnbPoolPut(clm) // wsEnqueueControlMessage has taken a copy.\n\t\t}\n\t\t// Return io.EOF so that readLoop will close the connection as ClientClosed\n\t\t// after processing pending buffers.\n\t\treturn pos, io.EOF\n\tcase wsPingMessage:\n\t\tc.wsEnqueueControlMessage(wsPongMessage, payload)\n\tcase wsPongMessage:\n\t\t// Nothing to do..\n\t}\n\treturn pos, nil\n}\n\n// Unmask the given slice.\nfunc (r *wsReadInfo) unmask(buf []byte) {\n\tp := int(r.mkpos)\n\tif len(buf) < 16 {\n\t\tfor i := 0; i < len(buf); i++ {\n\t\t\tbuf[i] ^= r.mkey[p&3]\n\t\t\tp++\n\t\t}\n\t\tr.mkpos = byte(p & 3)\n\t\treturn\n\t}\n\tvar k [8]byte\n\tfor i := 0; i < 8; i++ {\n\t\tk[i] = r.mkey[(p+i)&3]\n\t}\n\tkm := binary.BigEndian.Uint64(k[:])\n\tn := (len(buf) / 8) * 8\n\tfor i := 0; i < n; i += 8 {\n\t\ttmp := binary.BigEndian.Uint64(buf[i : i+8])\n\t\ttmp ^= km\n\t\tbinary.BigEndian.PutUint64(buf[i:], tmp)\n\t}\n\tbuf = buf[n:]\n\tfor i := 0; i < len(buf); i++ {\n\t\tbuf[i] ^= r.mkey[p&3]\n\t\tp++\n\t}\n\tr.mkpos = byte(p & 3)\n}\n\n// Returns true if the op code corresponds to a control frame.\nfunc wsIsControlFrame(frameType wsOpCode) bool {\n\treturn frameType >= wsCloseMessage\n}\n\n// Create the frame header.\n// Encodes the frame type and optional compression flag, and the size of the payload.\nfunc wsCreateFrameHeader(useMasking, compressed bool, frameType wsOpCode, l int) ([]byte, []byte) {\n\tfh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize]\n\tn, key := wsFillFrameHeader(fh, useMasking, wsFirstFrame, wsFinalFrame, compressed, frameType, l)\n\treturn fh[:n], key\n}\n\nfunc wsFillFrameHeader(fh []byte, useMasking, first, final, compressed bool, frameType wsOpCode, l int) (int, []byte) {\n\tvar n int\n\tvar b byte\n\tif first {\n\t\tb = byte(frameType)\n\t}\n\tif final {\n\t\tb |= wsFinalBit\n\t}\n\tif compressed {\n\t\tb |= wsRsv1Bit\n\t}\n\tb1 := byte(0)\n\tif useMasking {\n\t\tb1 |= wsMaskBit\n\t}\n\tswitch {\n\tcase l <= 125:\n\t\tn = 2\n\t\tfh[0] = b\n\t\tfh[1] = b1 | byte(l)\n\tcase l < 65536:\n\t\tn = 4\n\t\tfh[0] = b\n\t\tfh[1] = b1 | 126\n\t\tbinary.BigEndian.PutUint16(fh[2:], uint16(l))\n\tdefault:\n\t\tn = 10\n\t\tfh[0] = b\n\t\tfh[1] = b1 | 127\n\t\tbinary.BigEndian.PutUint64(fh[2:], uint64(l))\n\t}\n\tvar key []byte\n\tif useMasking {\n\t\tvar keyBuf [4]byte\n\t\tif _, err := io.ReadFull(crand.Reader, keyBuf[:4]); err != nil {\n\t\t\tkv := mrand.Int31()\n\t\t\tbinary.LittleEndian.PutUint32(keyBuf[:4], uint32(kv))\n\t\t}\n\t\tcopy(fh[n:], keyBuf[:4])\n\t\tkey = fh[n : n+4]\n\t\tn += 4\n\t}\n\treturn n, key\n}\n\n// Invokes wsEnqueueControlMessageLocked under client lock.\n//\n// Client lock MUST NOT be held on entry\nfunc (c *client) wsEnqueueControlMessage(controlMsg wsOpCode, payload []byte) {\n\tc.mu.Lock()\n\tc.wsEnqueueControlMessageLocked(controlMsg, payload)\n\tc.mu.Unlock()\n}\n\n// Mask the buffer with the given key\nfunc wsMaskBuf(key, buf []byte) {\n\tfor i := 0; i < len(buf); i++ {\n\t\tbuf[i] ^= key[i&3]\n\t}\n}\n\n// Mask the buffers, as if they were contiguous, with the given key\nfunc wsMaskBufs(key []byte, bufs [][]byte) {\n\tpos := 0\n\tfor i := 0; i < len(bufs); i++ {\n\t\tbuf := bufs[i]\n\t\tfor j := 0; j < len(buf); j++ {\n\t\t\tbuf[j] ^= key[pos&3]\n\t\t\tpos++\n\t\t}\n\t}\n}\n\n// Enqueues a websocket control message.\n// If the control message is a wsCloseMessage, then marks this client\n// has having sent the close message (since only one should be sent).\n// This will prevent the generic closeConnection() to enqueue one.\n//\n// Client lock held on entry.\nfunc (c *client) wsEnqueueControlMessageLocked(controlMsg wsOpCode, payload []byte) {\n\t// Control messages are never compressed and their size will be\n\t// less than wsMaxControlPayloadSize, which means the frame header\n\t// will be only 2 or 6 bytes.\n\tuseMasking := c.ws.maskwrite\n\tsz := 2\n\tif useMasking {\n\t\tsz += 4\n\t}\n\tcm := nbPoolGet(sz + len(payload))\n\tcm = cm[:cap(cm)]\n\tn, key := wsFillFrameHeader(cm, useMasking, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, controlMsg, len(payload))\n\tcm = cm[:n]\n\t// Note that payload is optional.\n\tif len(payload) > 0 {\n\t\tcm = append(cm, payload...)\n\t\tif useMasking {\n\t\t\twsMaskBuf(key, cm[n:])\n\t\t}\n\t}\n\tc.out.pb += int64(len(cm))\n\tif controlMsg == wsCloseMessage {\n\t\t// We can't add the close message to the frames buffers\n\t\t// now. It will be done on a flushOutbound() when there\n\t\t// are no more pending buffers to send.\n\t\tc.ws.closeSent = true\n\t\tc.ws.closeMsg = cm\n\t} else {\n\t\tc.ws.frames = append(c.ws.frames, cm)\n\t\tc.ws.fs += int64(len(cm))\n\t}\n\tc.flushSignal()\n}\n\n// Enqueues a websocket close message with a status mapped from the given `reason`.\n//\n// Client lock held on entry\nfunc (c *client) wsEnqueueCloseMessage(reason ClosedState) {\n\tvar status int\n\tswitch reason {\n\tcase ClientClosed:\n\t\tstatus = wsCloseStatusNormalClosure\n\tcase AuthenticationTimeout, AuthenticationViolation, SlowConsumerPendingBytes, SlowConsumerWriteDeadline,\n\t\tMaxAccountConnectionsExceeded, MaxConnectionsExceeded, MaxControlLineExceeded, MaxSubscriptionsExceeded,\n\t\tMissingAccount, AuthenticationExpired, Revocation:\n\t\tstatus = wsCloseStatusPolicyViolation\n\tcase TLSHandshakeError:\n\t\tstatus = wsCloseStatusTLSHandshake\n\tcase ParseError, ProtocolViolation, BadClientProtocolVersion:\n\t\tstatus = wsCloseStatusProtocolError\n\tcase MaxPayloadExceeded:\n\t\tstatus = wsCloseStatusMessageTooBig\n\tcase WriteError, ReadError, StaleConnection, ServerShutdown:\n\t\t// We used to have WriteError, ReadError and StaleConnection result in\n\t\t// code 1006, which the spec says that it must not be used to set the\n\t\t// status in the close message. So using this one instead.\n\t\tstatus = wsCloseStatusGoingAway\n\tdefault:\n\t\tstatus = wsCloseStatusInternalSrvError\n\t}\n\tbody := wsCreateCloseMessage(status, reason.String())\n\tc.wsEnqueueControlMessageLocked(wsCloseMessage, body)\n\tnbPoolPut(body) // wsEnqueueControlMessageLocked has taken a copy.\n}\n\n// Create and then enqueue a close message with a protocol error and the\n// given message. This is invoked when parsing websocket frames.\n//\n// Lock MUST NOT be held on entry.\nfunc (c *client) wsHandleProtocolError(message string) error {\n\tbuf := wsCreateCloseMessage(wsCloseStatusProtocolError, message)\n\tc.wsEnqueueControlMessage(wsCloseMessage, buf)\n\tnbPoolPut(buf) // wsEnqueueControlMessage has taken a copy.\n\treturn errors.New(message)\n}\n\nfunc wsIsValidCloseStatus(code int) bool {\n\tswitch code {\n\tcase wsCloseStatusNoStatusReceived, 1004, 1006, wsCloseStatusTLSHandshake:\n\t\treturn false\n\t}\n\tif code < 1000 || code >= 5000 {\n\t\treturn false\n\t}\n\t// 1016-2999 are currently reserved.\n\tif code >= 1016 && code <= 2999 {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Create a close message with the given `status` and `body`.\n// If the `body` is more than the maximum allows control frame payload size,\n// it is truncated and \"...\" is added at the end (as a hint that message\n// is not complete).\nfunc wsCreateCloseMessage(status int, body string) []byte {\n\t// Since a control message payload is limited in size, we\n\t// will limit the text and add trailing \"...\" if truncated.\n\t// The body of a Close Message must be preceded with 2 bytes,\n\t// so take that into account for limiting the body length.\n\tif len(body) > wsMaxControlPayloadSize-2 {\n\t\tbody = body[:wsMaxControlPayloadSize-5]\n\t\tbody += \"...\"\n\t}\n\tbuf := nbPoolGet(2 + len(body))[:2+len(body)]\n\t// We need to have a 2 byte unsigned int that represents the error status code\n\t// https://tools.ietf.org/html/rfc6455#section-5.5.1\n\tbinary.BigEndian.PutUint16(buf[:2], uint16(status))\n\tcopy(buf[2:], []byte(body))\n\treturn buf\n}\n\n// Process websocket client handshake. On success, returns the raw net.Conn that\n// will be used to create a *client object.\n// Invoked from the HTTP server listening on websocket port.\nfunc (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeResult, error) {\n\tkind := CLIENT\n\tif r.URL != nil {\n\t\tep := r.URL.EscapedPath()\n\t\tif strings.HasSuffix(ep, leafNodeWSPath) {\n\t\t\tkind = LEAF\n\t\t} else if strings.HasSuffix(ep, mqttWSPath) {\n\t\t\tkind = MQTT\n\t\t}\n\t}\n\n\topts := s.getOpts()\n\n\t// From https://tools.ietf.org/html/rfc6455#section-4.2.1\n\t// Point 1.\n\tif r.Method != \"GET\" {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusMethodNotAllowed, \"request method must be GET\")\n\t}\n\t// Point 2.\n\tif r.Host == _EMPTY_ {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"'Host' missing in request\")\n\t}\n\t// Point 3.\n\tif !wsHeaderContains(r.Header, \"Upgrade\", \"websocket\") {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"invalid value for header 'Upgrade'\")\n\t}\n\t// Point 4.\n\tif !wsHeaderContains(r.Header, \"Connection\", \"Upgrade\") {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"invalid value for header 'Connection'\")\n\t}\n\t// Point 5.\n\tkey := r.Header.Get(\"Sec-Websocket-Key\")\n\tif key == _EMPTY_ {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"key missing\")\n\t}\n\tdecoded, err := base64.StdEncoding.DecodeString(key)\n\tif err != nil || len(decoded) != 16 {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"invalid websocket key\")\n\t}\n\t// Point 6.\n\tif !wsHeaderContains(r.Header, \"Sec-Websocket-Version\", \"13\") {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"invalid version\")\n\t}\n\t// Others are optional\n\t// Point 7.\n\tif err := s.websocket.checkOrigin(r); err != nil {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusForbidden, fmt.Sprintf(\"origin not allowed: %v\", err))\n\t}\n\t// Point 8.\n\t// We don't have protocols, so ignore.\n\t// Point 9.\n\t// Extensions, only support for compression at the moment\n\tcompress := opts.Websocket.Compression\n\tif compress {\n\t\t// Simply check if permessage-deflate extension is present.\n\t\tcompress, _ = wsPMCExtensionSupport(r.Header, true)\n\t}\n\t// We will do masking if asked (unless we reject for tests)\n\tnoMasking := r.Header.Get(wsNoMaskingHeader) == wsNoMaskingValue && !wsTestRejectNoMasking\n\n\th, ok := w.(http.Hijacker)\n\tif !ok {\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"websocket upgrade not supported\")\n\t}\n\tconn, brw, err := h.Hijack()\n\tif err != nil {\n\t\tif conn != nil {\n\t\t\tconn.Close()\n\t\t}\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusInternalServerError, err.Error())\n\t}\n\tif brw.Reader.Buffered() > 0 {\n\t\tconn.Close()\n\t\treturn nil, wsReturnHTTPError(w, r, http.StatusBadRequest, \"client sent data before handshake is complete\")\n\t}\n\n\tvar buf [1024]byte\n\tp := buf[:0]\n\n\t// From https://tools.ietf.org/html/rfc6455#section-4.2.2\n\tp = append(p, \"HTTP/1.1 101 Switching Protocols\\r\\nUpgrade: websocket\\r\\nConnection: Upgrade\\r\\nSec-WebSocket-Accept: \"...)\n\tp = append(p, wsAcceptKey(key)...)\n\tp = append(p, _CRLF_...)\n\tif compress {\n\t\tp = append(p, wsPMCFullResponse...)\n\t}\n\tif noMasking {\n\t\tp = append(p, wsNoMaskingFullResponse...)\n\t}\n\tif kind == MQTT {\n\t\tp = append(p, wsMQTTSecProto...)\n\t}\n\tif s.websocket.rawHeaders != _EMPTY_ {\n\t\tp = append(p, s.websocket.rawHeaders...)\n\t}\n\tp = append(p, _CRLF_...)\n\n\tif _, err = conn.Write(p); err != nil {\n\t\tconn.Close()\n\t\treturn nil, err\n\t}\n\t// If there was a deadline set for the handshake, clear it now.\n\tif opts.Websocket.HandshakeTimeout > 0 {\n\t\tconn.SetDeadline(time.Time{})\n\t}\n\t// Server always expect \"clients\" to send masked payload, unless the option\n\t// \"no-masking\" has been enabled.\n\tws := &websocket{compress: compress, maskread: !noMasking}\n\n\t// Check for X-Forwarded-For header\n\tif cips, ok := r.Header[wsXForwardedForHeader]; ok {\n\t\tif len(cips) > 0 {\n\t\t\tcip := cips[0]\n\t\t\tif net.ParseIP(cip) != nil {\n\t\t\t\tws.clientIP = cip\n\t\t\t}\n\t\t}\n\t}\n\n\tif kind == CLIENT || kind == MQTT {\n\t\t// Indicate if this is likely coming from a browser.\n\t\tif ua := r.Header.Get(\"User-Agent\"); ua != _EMPTY_ && strings.HasPrefix(ua, \"Mozilla/\") {\n\t\t\tws.browser = true\n\t\t\t// Disable fragmentation of compressed frames for Safari browsers.\n\t\t\t// Unfortunately, you could be running Chrome on macOS and this\n\t\t\t// string will contain \"Safari/\" (along \"Chrome/\"). However, what\n\t\t\t// I have found is that actual Safari browser also have \"Version/\".\n\t\t\t// So make the combination of the two.\n\t\t\tws.nocompfrag = ws.compress && strings.Contains(ua, \"Version/\") && strings.Contains(ua, \"Safari/\")\n\t\t}\n\n\t\tif cookies := r.Cookies(); len(cookies) > 0 {\n\t\t\tows := &opts.Websocket\n\t\t\tfor _, c := range cookies {\n\t\t\t\tif ows.JWTCookie == c.Name {\n\t\t\t\t\tws.cookieJwt = c.Value\n\t\t\t\t} else if ows.UsernameCookie == c.Name {\n\t\t\t\t\tws.cookieUsername = c.Value\n\t\t\t\t} else if ows.PasswordCookie == c.Name {\n\t\t\t\t\tws.cookiePassword = c.Value\n\t\t\t\t} else if ows.TokenCookie == c.Name {\n\t\t\t\t\tws.cookieToken = c.Value\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn &wsUpgradeResult{conn: conn, ws: ws, kind: kind}, nil\n}\n\n// Returns true if the header named `name` contains a token with value `value`.\nfunc wsHeaderContains(header http.Header, name string, value string) bool {\n\tfor _, s := range header[name] {\n\t\ttokens := strings.Split(s, \",\")\n\t\tfor _, t := range tokens {\n\t\t\tt = strings.Trim(t, \" \\t\")\n\t\t\tif strings.EqualFold(t, value) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc wsPMCExtensionSupport(header http.Header, checkPMCOnly bool) (bool, bool) {\n\tfor _, extensionList := range header[\"Sec-Websocket-Extensions\"] {\n\t\textensions := strings.Split(extensionList, \",\")\n\t\tfor _, extension := range extensions {\n\t\t\textension = strings.Trim(extension, \" \\t\")\n\t\t\tparams := strings.Split(extension, \";\")\n\t\t\tfor i, p := range params {\n\t\t\t\tp = strings.Trim(p, \" \\t\")\n\t\t\t\tif strings.EqualFold(p, wsPMCExtension) {\n\t\t\t\t\tif checkPMCOnly {\n\t\t\t\t\t\treturn true, false\n\t\t\t\t\t}\n\t\t\t\t\tvar snc bool\n\t\t\t\t\tvar cnc bool\n\t\t\t\t\tfor j := i + 1; j < len(params); j++ {\n\t\t\t\t\t\tp = params[j]\n\t\t\t\t\t\tp = strings.Trim(p, \" \\t\")\n\t\t\t\t\t\tif strings.EqualFold(p, wsPMCSrvNoCtx) {\n\t\t\t\t\t\t\tsnc = true\n\t\t\t\t\t\t} else if strings.EqualFold(p, wsPMCCliNoCtx) {\n\t\t\t\t\t\t\tcnc = true\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif snc && cnc {\n\t\t\t\t\t\t\treturn true, true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\treturn true, false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn false, false\n}\n\n// Send an HTTP error with the given `status` to the given http response writer `w`.\n// Return an error created based on the `reason` string.\nfunc wsReturnHTTPError(w http.ResponseWriter, r *http.Request, status int, reason string) error {\n\terr := fmt.Errorf(\"%s - websocket handshake error: %s\", r.RemoteAddr, reason)\n\tw.Header().Set(\"Sec-Websocket-Version\", \"13\")\n\thttp.Error(w, http.StatusText(status), status)\n\treturn err\n}\n\n// If the server is configured to accept any origin, then this function returns\n// `nil` without checking if the Origin is present and valid. This is also\n// the case if the request does not have the Origin header.\n// Otherwise, this will check that the Origin matches the same origin or\n// any origin in the allowed list.\nfunc (w *srvWebsocket) checkOrigin(r *http.Request) error {\n\tw.mu.RLock()\n\tcheckSame := w.sameOrigin\n\tlistEmpty := len(w.allowedOrigins) == 0\n\tw.mu.RUnlock()\n\tif !checkSame && listEmpty {\n\t\treturn nil\n\t}\n\torigin := r.Header.Get(\"Origin\")\n\tif origin == _EMPTY_ {\n\t\torigin = r.Header.Get(\"Sec-Websocket-Origin\")\n\t}\n\t// If the header is not present, we will accept.\n\t// From https://datatracker.ietf.org/doc/html/rfc6455#section-1.6\n\t// \"Naturally, when the WebSocket Protocol is used by a dedicated client\n\t// directly (i.e., not from a web page through a web browser), the origin\n\t// model is not useful, as the client can provide any arbitrary origin string.\"\n\tif origin == _EMPTY_ {\n\t\treturn nil\n\t}\n\tu, err := url.ParseRequestURI(origin)\n\tif err != nil {\n\t\treturn err\n\t}\n\toh, op, err := wsGetHostAndPort(u.Scheme == \"https\", u.Host)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// If checking same origin, compare with the http's request's Host.\n\tif checkSame {\n\t\trh, rp, err := wsGetHostAndPort(r.TLS != nil, r.Host)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trs := \"http\"\n\t\tif r.TLS != nil {\n\t\t\trs = \"https\"\n\t\t}\n\t\tif oh != rh || op != rp || !strings.EqualFold(u.Scheme, rs) {\n\t\t\treturn errors.New(\"not same origin\")\n\t\t}\n\t\t// I guess it is possible to have cases where one wants to check\n\t\t// same origin, but also that the origin is in the allowed list.\n\t\t// So continue with the next check.\n\t}\n\tif !listEmpty {\n\t\tw.mu.RLock()\n\t\torigins := w.allowedOrigins[oh]\n\t\tw.mu.RUnlock()\n\t\tvar allowed bool\n\t\tfor _, ao := range origins {\n\t\t\tif u.Scheme == ao.scheme && op == ao.port {\n\t\t\t\tallowed = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !allowed {\n\t\t\treturn errors.New(\"not in the allowed list\")\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc wsGetHostAndPort(tls bool, hostport string) (string, string, error) {\n\thost, port, err := net.SplitHostPort(hostport)\n\tif err != nil {\n\t\t// If error is missing port, then use defaults based on the scheme\n\t\tif ae, ok := err.(*net.AddrError); ok && strings.Contains(ae.Err, \"missing port\") {\n\t\t\terr = nil\n\t\t\thost = hostport\n\t\t\tif tls {\n\t\t\t\tport = \"443\"\n\t\t\t} else {\n\t\t\t\tport = \"80\"\n\t\t\t}\n\t\t}\n\t}\n\treturn strings.ToLower(host), port, err\n}\n\n// Concatenate the key sent by the client with the GUID, then computes the SHA1 hash\n// and returns it as a based64 encoded string.\nfunc wsAcceptKey(key string) string {\n\th := sha1.New()\n\th.Write([]byte(key))\n\th.Write(wsGUID)\n\treturn base64.StdEncoding.EncodeToString(h.Sum(nil))\n}\n\nfunc wsMakeChallengeKey() (string, error) {\n\tp := make([]byte, 16)\n\tif _, err := io.ReadFull(crand.Reader, p); err != nil {\n\t\treturn _EMPTY_, err\n\t}\n\treturn base64.StdEncoding.EncodeToString(p), nil\n}\n\n// Validate the websocket related options.\nfunc validateWebsocketOptions(o *Options) error {\n\two := &o.Websocket\n\t// If no port is defined, we don't care about other options\n\tif wo.Port == 0 {\n\t\treturn nil\n\t}\n\t// Enforce TLS... unless NoTLS is set to true.\n\tif wo.TLSConfig == nil && !wo.NoTLS {\n\t\treturn errors.New(\"websocket requires TLS configuration\")\n\t}\n\t// Make sure that allowed origins, if specified, can be parsed.\n\tfor _, ao := range wo.AllowedOrigins {\n\t\tu, err := url.ParseRequestURI(ao)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"unable to parse allowed origin: %v\", err)\n\t\t}\n\t\tif u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\t\treturn fmt.Errorf(\"unable to parse allowed origin %q: allowed origins must be absolute URLs with http or https scheme\", ao)\n\t\t}\n\t\tif u.Host == _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"unable to parse allowed origin %q: host is required\", ao)\n\t\t}\n\t\tif _, _, err := wsGetHostAndPort(u.Scheme == \"https\", u.Host); err != nil {\n\t\t\treturn fmt.Errorf(\"unable to parse allowed origin: %v\", err)\n\t\t}\n\t}\n\t// If there is a NoAuthUser, we need to have Users defined and\n\t// the user to be present.\n\tif wo.NoAuthUser != _EMPTY_ {\n\t\tif err := validateNoAuthUser(o, wo.NoAuthUser); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Token/Username not possible if there are users/nkeys\n\tif len(o.Users) > 0 || len(o.Nkeys) > 0 {\n\t\tif wo.Username != _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"websocket authentication username not compatible with presence of users/nkeys\")\n\t\t}\n\t\tif wo.Token != _EMPTY_ {\n\t\t\treturn fmt.Errorf(\"websocket authentication token not compatible with presence of users/nkeys\")\n\t\t}\n\t}\n\t// Using JWT requires Trusted Keys\n\tif wo.JWTCookie != _EMPTY_ {\n\t\tif len(o.TrustedOperators) == 0 && len(o.TrustedKeys) == 0 {\n\t\t\treturn fmt.Errorf(\"trusted operators or trusted keys configuration is required for JWT authentication via cookie %q\", wo.JWTCookie)\n\t\t}\n\t}\n\tif err := validatePinnedCerts(wo.TLSPinnedCerts); err != nil {\n\t\treturn fmt.Errorf(\"websocket: %v\", err)\n\t}\n\n\t// Check for invalid headers here.\n\tfor key := range wo.Headers {\n\t\tk := strings.ToLower(key)\n\t\tswitch k {\n\t\tcase \"host\",\n\t\t\t\"content-length\",\n\t\t\t\"connection\",\n\t\t\t\"upgrade\",\n\t\t\t\"nats-no-masking\":\n\t\t\treturn fmt.Errorf(\"websocket: invalid header %q not allowed\", key)\n\t\t}\n\n\t\tif strings.HasPrefix(k, \"sec-websocket-\") {\n\t\t\treturn fmt.Errorf(\"websocket: invalid header %q, \\\"Sec-WebSocket-\\\" prefix not allowed\", key)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Creates or updates the existing map\nfunc (s *Server) wsSetOriginOptions(o *WebsocketOpts) {\n\tws := &s.websocket\n\tws.mu.Lock()\n\tdefer ws.mu.Unlock()\n\t// Copy over the option's same origin boolean\n\tws.sameOrigin = o.SameOrigin\n\t// Reset the map. Will help for config reload if/when we support it.\n\tws.allowedOrigins = nil\n\tif o.AllowedOrigins == nil {\n\t\treturn\n\t}\n\tfor _, ao := range o.AllowedOrigins {\n\t\t// We have previously checked (during options validation) that the urls\n\t\t// are parseable, but if we get an error, report and skip.\n\t\tu, err := url.ParseRequestURI(ao)\n\t\tif err != nil {\n\t\t\ts.Errorf(\"error parsing allowed origin: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\th, p, _ := wsGetHostAndPort(u.Scheme == \"https\", u.Host)\n\t\tif ws.allowedOrigins == nil {\n\t\t\tws.allowedOrigins = make(map[string][]*allowedOrigin, len(o.AllowedOrigins))\n\t\t}\n\t\tws.allowedOrigins[h] = append(ws.allowedOrigins[h], &allowedOrigin{scheme: u.Scheme, port: p})\n\t}\n}\n\n// Calculate the raw headers for websocket upgrade response.\nfunc (s *Server) wsSetHeadersOptions(o *WebsocketOpts) {\n\tvar sb strings.Builder\n\tfor k, v := range o.Headers {\n\t\tsb.WriteString(k)\n\t\tsb.WriteString(\": \")\n\t\tsb.WriteString(v)\n\t\tsb.WriteString(_CRLF_)\n\t}\n\tws := &s.websocket\n\tws.mu.Lock()\n\tdefer ws.mu.Unlock()\n\tws.rawHeaders = sb.String()\n}\n\n// Given the websocket options, we check if any auth configuration\n// has been provided. If so, possibly create users/nkey users and\n// store them in s.websocket.users/nkeys.\n// Also update a boolean that indicates if auth is required for\n// websocket clients.\n// Server lock is held on entry.\nfunc (s *Server) wsConfigAuth(opts *WebsocketOpts) {\n\tws := &s.websocket\n\t// If any of those is specified, we consider that there is an override.\n\tws.authOverride = opts.Username != _EMPTY_ || opts.Token != _EMPTY_ || opts.NoAuthUser != _EMPTY_\n}\n\nfunc (s *Server) startWebsocketServer() {\n\tif s.isShuttingDown() {\n\t\treturn\n\t}\n\n\tsopts := s.getOpts()\n\to := &sopts.Websocket\n\n\ts.wsSetOriginOptions(o)\n\ts.wsSetHeadersOptions(o)\n\n\tvar hl net.Listener\n\tvar proto string\n\tvar err error\n\n\tport := o.Port\n\tif port == -1 {\n\t\tport = 0\n\t}\n\thp := net.JoinHostPort(o.Host, strconv.Itoa(port))\n\n\t// We are enforcing (when validating the options) the use of TLS, but the\n\t// code was originally supporting both modes. The reason for TLS only is\n\t// that we expect users to send JWTs with bearer tokens and we want to\n\t// avoid the possibility of it being \"intercepted\".\n\n\ts.mu.Lock()\n\t// Do not check o.NoTLS here. If a TLS configuration is available, use it,\n\t// regardless of NoTLS. If we don't have a TLS config, it means that the\n\t// user has configured NoTLS because otherwise the server would have failed\n\t// to start due to options validation.\n\tvar config *tls.Config\n\tif o.TLSConfig != nil {\n\t\tproto = wsSchemePrefixTLS\n\t\tconfig = o.TLSConfig.Clone()\n\t\tconfig.GetConfigForClient = s.wsGetTLSConfig\n\t} else {\n\t\tproto = wsSchemePrefix\n\t}\n\thl, err = natsListen(\"tcp\", hp)\n\ts.websocket.listenerErr = err\n\tif err != nil {\n\t\ts.mu.Unlock()\n\t\ts.Fatalf(\"Unable to listen for websocket connections: %v\", err)\n\t\treturn\n\t}\n\tif config != nil {\n\t\thl = tls.NewListener(hl, config)\n\t}\n\tif port == 0 {\n\t\to.Port = hl.Addr().(*net.TCPAddr).Port\n\t}\n\ts.Noticef(\"Listening for websocket clients on %s://%s:%d\", proto, o.Host, o.Port)\n\tif proto == wsSchemePrefix {\n\t\ts.Warnf(\"Websocket not configured with TLS. DO NOT USE IN PRODUCTION!\")\n\t}\n\n\t// These 3 are immutable and will be accessed without lock by the client\n\t// when generating/sending the INFO protocols.\n\ts.websocket.tls = proto == wsSchemePrefixTLS\n\ts.websocket.host, s.websocket.port = o.Host, o.Port\n\n\t// This will be updated when/if the cluster changes.\n\ts.websocket.connectURLs, err = s.getConnectURLs(o.Advertise, o.Host, o.Port)\n\tif err != nil {\n\t\ts.Fatalf(\"Unable to get websocket connect URLs: %v\", err)\n\t\thl.Close()\n\t\ts.mu.Unlock()\n\t\treturn\n\t}\n\thasLeaf := sopts.LeafNode.Port != 0\n\tmux := http.NewServeMux()\n\tmux.HandleFunc(\"/\", func(w http.ResponseWriter, r *http.Request) {\n\t\tres, err := s.wsUpgrade(w, r)\n\t\tif err != nil {\n\t\t\ts.Errorf(err.Error())\n\t\t\treturn\n\t\t}\n\t\tswitch res.kind {\n\t\tcase CLIENT:\n\t\t\ts.createWSClient(res.conn, res.ws)\n\t\tcase MQTT:\n\t\t\ts.createMQTTClient(res.conn, res.ws)\n\t\tcase LEAF:\n\t\t\tif !hasLeaf {\n\t\t\t\ts.Errorf(\"Not configured to accept leaf node connections\")\n\t\t\t\t// Silently close for now. If we want to send an error back, we would\n\t\t\t\t// need to create the leafnode client anyway, so that is handling websocket\n\t\t\t\t// frames, then send the error to the remote.\n\t\t\t\tres.conn.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\ts.createLeafNode(res.conn, nil, nil, res.ws)\n\t\t}\n\t})\n\ths := &http.Server{\n\t\tAddr: hp,\n\t\tHandler: mux,\n\t\tReadTimeout: o.HandshakeTimeout,\n\t\tErrorLog: log.New(&captureHTTPServerLog{s, \"websocket: \"}, _EMPTY_, 0),\n\t}\n\ts.websocket.mu.Lock()\n\ts.websocket.server = hs\n\ts.websocket.listener = hl\n\ts.websocket.mu.Unlock()\n\tgo func() {\n\t\tif err := hs.Serve(hl); err != http.ErrServerClosed {\n\t\t\ts.Fatalf(\"websocket listener error: %v\", err)\n\t\t}\n\t\tif s.isLameDuckMode() {\n\t\t\t// Signal that we are not accepting new clients\n\t\t\ts.ldmCh <- true\n\t\t\t// Now wait for the Shutdown...\n\t\t\t<-s.quitCh\n\t\t\treturn\n\t\t}\n\t\ts.done <- true\n\t}()\n\ts.mu.Unlock()\n}\n\n// The TLS configuration is passed to the listener when the websocket\n// \"server\" is setup. That prevents TLS configuration updates on reload\n// from being used. By setting this function in tls.Config.GetConfigForClient\n// we instruct the TLS handshake to ask for the tls configuration to be\n// used for a specific client. We don't care which client, we always use\n// the same TLS configuration.\nfunc (s *Server) wsGetTLSConfig(_ *tls.ClientHelloInfo) (*tls.Config, error) {\n\topts := s.getOpts()\n\treturn opts.Websocket.TLSConfig, nil\n}\n\n// This is similar to createClient() but has some modifications\n// specific to handle websocket clients.\n// The comments have been kept to minimum to reduce code size.\n// Check createClient() for more details.\nfunc (s *Server) createWSClient(conn net.Conn, ws *websocket) *client {\n\topts := s.getOpts()\n\n\tmaxPay := int32(opts.MaxPayload)\n\tmaxSubs := int32(opts.MaxSubs)\n\tif maxSubs == 0 {\n\t\tmaxSubs = -1\n\t}\n\tnow := time.Now().UTC()\n\n\tc := &client{srv: s, nc: conn, opts: defaultOpts, mpay: maxPay, msubs: maxSubs, start: now, last: now, ws: ws}\n\n\tc.registerWithAccount(s.globalAccount())\n\n\tvar info Info\n\tvar authRequired bool\n\n\ts.mu.Lock()\n\tinfo = s.copyInfo()\n\t// Check auth, override if applicable.\n\tif !info.AuthRequired {\n\t\t// Set info.AuthRequired since this is what is sent to the client.\n\t\tinfo.AuthRequired = s.websocket.authOverride\n\t}\n\tif s.nonceRequired() {\n\t\tvar raw [nonceLen]byte\n\t\tnonce := raw[:]\n\t\ts.generateNonce(nonce)\n\t\tinfo.Nonce = string(nonce)\n\t}\n\tc.nonce = []byte(info.Nonce)\n\tauthRequired = info.AuthRequired\n\n\ts.totalClients++\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\tif authRequired {\n\t\tc.flags.set(expectConnect)\n\t}\n\tc.initClient()\n\tc.Debugf(\"Client connection created\")\n\tc.sendProtoNow(c.generateClientInfoJSON(info, true))\n\tc.mu.Unlock()\n\n\ts.mu.Lock()\n\tif !s.isRunning() || s.ldm {\n\t\tif s.isShuttingDown() {\n\t\t\tconn.Close()\n\t\t}\n\t\ts.mu.Unlock()\n\t\treturn c\n\t}\n\n\tif opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) {\n\t\ts.mu.Unlock()\n\t\tc.maxConnExceeded()\n\t\treturn nil\n\t}\n\ts.clients[c.cid] = c\n\ts.mu.Unlock()\n\n\tc.mu.Lock()\n\t// Websocket clients do TLS in the websocket http server.\n\t// So no TLS initiation here...\n\tif _, ok := conn.(*tls.Conn); ok {\n\t\tc.flags.set(handshakeComplete)\n\t}\n\n\tif c.isClosed() {\n\t\tc.mu.Unlock()\n\t\tc.closeConnection(WriteError)\n\t\treturn nil\n\t}\n\n\tif authRequired {\n\t\ttimeout := opts.AuthTimeout\n\t\t// Possibly override with Websocket specific value.\n\t\tif opts.Websocket.AuthTimeout != 0 {\n\t\t\ttimeout = opts.Websocket.AuthTimeout\n\t\t}\n\t\tc.setAuthTimer(secondsToDuration(timeout))\n\t}\n\n\tc.setPingTimer()\n\n\ts.startGoRoutine(func() { c.readLoop(nil) })\n\ts.startGoRoutine(func() { c.writeLoop() })\n\n\tc.mu.Unlock()\n\n\treturn c\n}\n\nfunc (c *client) wsCollapsePtoNB() (net.Buffers, int64) {\n\tnb := c.out.nb\n\tvar mfs int\n\tvar usz int\n\tif c.ws.browser {\n\t\tmfs = wsFrameSizeForBrowsers\n\t}\n\tmask := c.ws.maskwrite\n\t// Start with possible already framed buffers (that we could have\n\t// got from partials or control messages such as ws pings or pongs).\n\tbufs := c.ws.frames\n\tcompress := c.ws.compress\n\tif compress && len(nb) > 0 {\n\t\t// First, make sure we don't compress for very small cumulative buffers.\n\t\tfor _, b := range nb {\n\t\t\tusz += len(b)\n\t\t}\n\t\tif usz <= wsCompressThreshold {\n\t\t\tcompress = false\n\t\t\tif cp := c.ws.compressor; cp != nil {\n\t\t\t\tcp.Reset(nil)\n\t\t\t}\n\t\t}\n\t}\n\tif compress && len(nb) > 0 {\n\t\t// Overwrite mfs if this connection does not support fragmented compressed frames.\n\t\tif mfs > 0 && c.ws.nocompfrag {\n\t\t\tmfs = 0\n\t\t}\n\t\tbuf := bytes.NewBuffer(nbPoolGet(usz))\n\t\tcp := c.ws.compressor\n\t\tif cp == nil {\n\t\t\tc.ws.compressor, _ = flate.NewWriter(buf, flate.BestSpeed)\n\t\t\tcp = c.ws.compressor\n\t\t} else {\n\t\t\tcp.Reset(buf)\n\t\t}\n\t\tvar csz int\n\t\tfor i, b := range nb {\n\t\t\tfor len(b) > 0 {\n\t\t\t\tn, err := cp.Write(b)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Whatever this error is, it'll be handled by the cp.Flush()\n\t\t\t\t\t// call below, as the same error will be returned there.\n\t\t\t\t\t// Let the outer loop return all the buffers back to the pool\n\t\t\t\t\t// and fall through naturally.\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tb = b[n:]\n\t\t\t}\n\t\t\t// Use original slice since capacity will change to zero\n\t\t\t// in the loop after consuming the buffer, which will make\n\t\t\t// nbPoolPut discard it.\n\t\t\tnbPoolPut(nb[i])\n\t\t}\n\t\tif err := cp.Flush(); err != nil {\n\t\t\tc.Errorf(\"Error during compression: %v\", err)\n\t\t\tc.markConnAsClosed(WriteError)\n\t\t\tcp.Reset(nil)\n\t\t\treturn nil, 0\n\t\t}\n\t\tb := buf.Bytes()\n\t\tp := b[:len(b)-4]\n\t\tif mfs > 0 && len(p) > mfs {\n\t\t\tfor first, final := true, false; len(p) > 0; first = false {\n\t\t\t\tlp := len(p)\n\t\t\t\tif lp > mfs {\n\t\t\t\t\tlp = mfs\n\t\t\t\t} else {\n\t\t\t\t\tfinal = true\n\t\t\t\t}\n\t\t\t\t// Only the first frame should be marked as compressed, so pass\n\t\t\t\t// `first` for the compressed boolean.\n\t\t\t\tfh := nbPoolGet(wsMaxFrameHeaderSize)[:wsMaxFrameHeaderSize]\n\t\t\t\tn, key := wsFillFrameHeader(fh, mask, first, final, first, wsBinaryMessage, lp)\n\t\t\t\tif mask {\n\t\t\t\t\twsMaskBuf(key, p[:lp])\n\t\t\t\t}\n\t\t\t\tbufs = append(bufs, fh[:n], p[:lp])\n\t\t\t\tcsz += n + lp\n\t\t\t\tp = p[lp:]\n\t\t\t}\n\t\t} else {\n\t\t\tol := len(p)\n\t\t\th, key := wsCreateFrameHeader(mask, true, wsBinaryMessage, ol)\n\t\t\tif mask {\n\t\t\t\twsMaskBuf(key, p)\n\t\t\t}\n\t\t\tif ol > 0 {\n\t\t\t\tbufs = append(bufs, h, p)\n\t\t\t}\n\t\t\tcsz = len(h) + ol\n\t\t}\n\t\t// Make sure that the compressor no longer holds a reference to\n\t\t// the bytes.Buffer, so that the underlying memory gets cleaned\n\t\t// up after flushOutbound/flushAndClose. For this to be safe, we\n\t\t// always cp.Reset(...) before reusing the compressor again.\n\t\tcp.Reset(nil)\n\t\t// Add to pb the compressed data size (including headers), but\n\t\t// remove the original uncompressed data size that was added\n\t\t// during the queueing.\n\t\tc.out.pb += int64(csz) - int64(usz)\n\t\tc.ws.fs += int64(csz)\n\t} else if len(nb) > 0 {\n\t\tvar total int\n\t\tif mfs > 0 {\n\t\t\t// We are limiting the frame size.\n\t\t\tstartFrame := func() int {\n\t\t\t\tbufs = append(bufs, nbPoolGet(wsMaxFrameHeaderSize))\n\t\t\t\treturn len(bufs) - 1\n\t\t\t}\n\t\t\tendFrame := func(idx, size int) {\n\t\t\t\tbufs[idx] = bufs[idx][:wsMaxFrameHeaderSize]\n\t\t\t\tn, key := wsFillFrameHeader(bufs[idx], mask, wsFirstFrame, wsFinalFrame, wsUncompressedFrame, wsBinaryMessage, size)\n\t\t\t\tbufs[idx] = bufs[idx][:n]\n\t\t\t\tc.out.pb += int64(n)\n\t\t\t\tc.ws.fs += int64(n + size)\n\t\t\t\tif mask {\n\t\t\t\t\twsMaskBufs(key, bufs[idx+1:])\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfhIdx := startFrame()\n\t\t\tfor i := 0; i < len(nb); i++ {\n\t\t\t\tb := nb[i]\n\t\t\t\tif total+len(b) <= mfs {\n\t\t\t\t\tbuf := nbPoolGet(len(b))\n\t\t\t\t\tbufs = append(bufs, append(buf, b...))\n\t\t\t\t\ttotal += len(b)\n\t\t\t\t\tnbPoolPut(nb[i])\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor len(b) > 0 {\n\t\t\t\t\tendStart := total != 0\n\t\t\t\t\tif endStart {\n\t\t\t\t\t\tendFrame(fhIdx, total)\n\t\t\t\t\t}\n\t\t\t\t\ttotal = len(b)\n\t\t\t\t\tif total >= mfs {\n\t\t\t\t\t\ttotal = mfs\n\t\t\t\t\t}\n\t\t\t\t\tif endStart {\n\t\t\t\t\t\tfhIdx = startFrame()\n\t\t\t\t\t}\n\t\t\t\t\tbuf := nbPoolGet(total)\n\t\t\t\t\tbufs = append(bufs, append(buf, b[:total]...))\n\t\t\t\t\tb = b[total:]\n\t\t\t\t}\n\t\t\t\tnbPoolPut(nb[i]) // No longer needed as copied into smaller frames.\n\t\t\t}\n\t\t\tif total > 0 {\n\t\t\t\tendFrame(fhIdx, total)\n\t\t\t}\n\t\t} else {\n\t\t\t// If there is no limit on the frame size, create a single frame for\n\t\t\t// all pending buffers.\n\t\t\tfor _, b := range nb {\n\t\t\t\ttotal += len(b)\n\t\t\t}\n\t\t\twsfh, key := wsCreateFrameHeader(mask, false, wsBinaryMessage, total)\n\t\t\tc.out.pb += int64(len(wsfh))\n\t\t\tbufs = append(bufs, wsfh)\n\t\t\tidx := len(bufs)\n\t\t\tbufs = append(bufs, nb...)\n\t\t\tif mask {\n\t\t\t\twsMaskBufs(key, bufs[idx:])\n\t\t\t}\n\t\t\tc.ws.fs += int64(len(wsfh) + total)\n\t\t}\n\t}\n\tif len(c.ws.closeMsg) > 0 {\n\t\tbufs = append(bufs, c.ws.closeMsg)\n\t\tc.ws.fs += int64(len(c.ws.closeMsg))\n\t\tc.ws.closeMsg = nil\n\t\tc.ws.compressor = nil\n\t}\n\tc.ws.frames = nil\n\treturn bufs, c.ws.fs\n}\n\nfunc isWSURL(u *url.URL) bool {\n\treturn strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefix)\n}\n\nfunc isWSSURL(u *url.URL) bool {\n\treturn strings.HasPrefix(strings.ToLower(u.Scheme), wsSchemePrefixTLS)\n}\n" }, { "path": "server/websocket_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/klauspost/compress/flate\"\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\ntype testReader struct {\n\tbuf []byte\n\tpos int\n\tmax int\n\terr error\n}\n\nfunc (tr *testReader) Read(p []byte) (int, error) {\n\tif tr.err != nil {\n\t\treturn 0, tr.err\n\t}\n\tn := len(tr.buf) - tr.pos\n\tif n == 0 {\n\t\treturn 0, nil\n\t}\n\tif n > len(p) {\n\t\tn = len(p)\n\t}\n\tif tr.max > 0 && n > tr.max {\n\t\tn = tr.max\n\t}\n\tcopy(p, tr.buf[tr.pos:tr.pos+n])\n\ttr.pos += n\n\treturn n, nil\n}\n\nfunc TestWSGet(t *testing.T) {\n\trb := []byte(\"012345\")\n\n\ttr := &testReader{buf: []byte(\"6789\")}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpos uint64\n\t\tneeded uint64\n\t\tnewpos uint64\n\t\ttrmax int\n\t\tresult string\n\t\treterr bool\n\t}{\n\t\t{\"fromrb1\", 0, 3, 3, 4, \"012\", false}, // Partial from read buffer\n\t\t{\"fromrb2\", 3, 2, 5, 4, \"34\", false}, // Partial from read buffer\n\t\t{\"fromrb3\", 5, 1, 6, 4, \"5\", false}, // Partial from read buffer\n\t\t{\"fromtr1\", 4, 4, 6, 4, \"4567\", false}, // Partial from read buffer + some of ioReader\n\t\t{\"fromtr2\", 4, 6, 6, 4, \"456789\", false}, // Partial from read buffer + all of ioReader\n\t\t{\"fromtr3\", 4, 6, 6, 2, \"456789\", false}, // Partial from read buffer + all of ioReader with several reads\n\t\t{\"fromtr4\", 4, 6, 6, 2, \"\", true}, // ioReader returns error\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttr.pos = 0\n\t\t\ttr.max = test.trmax\n\t\t\tif test.reterr {\n\t\t\t\ttr.err = fmt.Errorf(\"on purpose\")\n\t\t\t}\n\t\t\tres, np, err := wsGet(tr, rb, test.pos, test.needed)\n\t\t\tif test.reterr {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatalf(\"Expected error, got none\")\n\t\t\t\t}\n\t\t\t\tif err.Error() != \"on purpose\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif np != 0 || res != nil {\n\t\t\t\t\tt.Fatalf(\"Unexpected returned values: res=%v n=%v\", res, np)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on get: %v\", err)\n\t\t\t}\n\t\t\tif np != test.newpos {\n\t\t\t\tt.Fatalf(\"Expected pos=%v, got %v\", test.newpos, np)\n\t\t\t}\n\t\t\tif string(res) != test.result {\n\t\t\t\tt.Fatalf(\"Invalid returned content: %s\", res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSIsControlFrame(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcode wsOpCode\n\t\tisControl bool\n\t}{\n\t\t{\"binary\", wsBinaryMessage, false},\n\t\t{\"text\", wsTextMessage, false},\n\t\t{\"ping\", wsPingMessage, true},\n\t\t{\"pong\", wsPongMessage, true},\n\t\t{\"close\", wsCloseMessage, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tif res := wsIsControlFrame(test.code); res != test.isControl {\n\t\t\t\tt.Fatalf(\"Expected %q isControl to be %v, got %v\", test.name, test.isControl, res)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testWSSimpleMask(key, buf []byte) {\n\tfor i := 0; i < len(buf); i++ {\n\t\tbuf[i] ^= key[i&3]\n\t}\n}\n\nfunc TestWSUnmask(t *testing.T) {\n\tkey := []byte{1, 2, 3, 4}\n\torgBuf := []byte(\"this is a clear text\")\n\n\tmask := func() []byte {\n\t\tt.Helper()\n\t\tbuf := append([]byte(nil), orgBuf...)\n\t\ttestWSSimpleMask(key, buf)\n\t\t// First ensure that the content is masked.\n\t\tif bytes.Equal(buf, orgBuf) {\n\t\t\tt.Fatalf(\"Masking did not do anything: %q\", buf)\n\t\t}\n\t\treturn buf\n\t}\n\n\tri := &wsReadInfo{mask: true}\n\tri.init()\n\tcopy(ri.mkey[:], key)\n\n\tbuf := mask()\n\t// Unmask in one call\n\tri.unmask(buf)\n\tif !bytes.Equal(buf, orgBuf) {\n\t\tt.Fatalf(\"Unmask error, expected %q, got %q\", orgBuf, buf)\n\t}\n\n\t// Unmask in multiple calls\n\tbuf = mask()\n\tri.mkpos = 0\n\tri.unmask(buf[:3])\n\tri.unmask(buf[3:11])\n\tri.unmask(buf[11:])\n\tif !bytes.Equal(buf, orgBuf) {\n\t\tt.Fatalf(\"Unmask error, expected %q, got %q\", orgBuf, buf)\n\t}\n}\n\nfunc TestWSCreateCloseMessage(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tstatus int\n\t\tpsize int\n\t\ttruncated bool\n\t}{\n\t\t{\"fits\", wsCloseStatusInternalSrvError, 10, false},\n\t\t{\"truncated\", wsCloseStatusProtocolError, wsMaxControlPayloadSize + 10, true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tpayload := make([]byte, test.psize)\n\t\t\tfor i := 0; i < len(payload); i++ {\n\t\t\t\tpayload[i] = byte('A' + (i % 26))\n\t\t\t}\n\t\t\tres := wsCreateCloseMessage(test.status, string(payload))\n\t\t\tif status := binary.BigEndian.Uint16(res[:2]); int(status) != test.status {\n\t\t\t\tt.Fatalf(\"Expected status to be %v, got %v\", test.status, status)\n\t\t\t}\n\t\t\tpsize := len(res) - 2\n\t\t\tif !test.truncated {\n\t\t\t\tif int(psize) != test.psize {\n\t\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", test.psize, psize)\n\t\t\t\t}\n\t\t\t\tif !bytes.Equal(res[2:], payload) {\n\t\t\t\t\tt.Fatalf(\"Unexpected result: %q\", res[2:])\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// Since the payload of a close message contains a 2 byte status, the\n\t\t\t// actual max text size will be wsMaxControlPayloadSize-2\n\t\t\tif int(psize) != wsMaxControlPayloadSize-2 {\n\t\t\t\tt.Fatalf(\"Expected size to be capped to %v, got %v\", wsMaxControlPayloadSize-2, psize)\n\t\t\t}\n\t\t\tif string(res[len(res)-3:]) != \"...\" {\n\t\t\t\tt.Fatalf(\"Expected res to have `...` at the end, got %q\", res[4:])\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSCreateFrameHeader(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tframeType wsOpCode\n\t\tcompressed bool\n\t\tlen int\n\t}{\n\t\t{\"uncompressed 10\", wsBinaryMessage, false, 10},\n\t\t{\"uncompressed 600\", wsTextMessage, false, 600},\n\t\t{\"uncompressed 100000\", wsTextMessage, false, 100000},\n\t\t{\"compressed 10\", wsBinaryMessage, true, 10},\n\t\t{\"compressed 600\", wsBinaryMessage, true, 600},\n\t\t{\"compressed 100000\", wsTextMessage, true, 100000},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tres, _ := wsCreateFrameHeader(false, test.compressed, test.frameType, test.len)\n\t\t\t// The server is always sending the message has a single frame,\n\t\t\t// so the \"final\" bit should be set.\n\t\t\texpected := byte(test.frameType) | wsFinalBit\n\t\t\tif test.compressed {\n\t\t\t\texpected |= wsRsv1Bit\n\t\t\t}\n\t\t\tif b := res[0]; b != expected {\n\t\t\t\tt.Fatalf(\"Expected first byte to be %v, got %v\", expected, b)\n\t\t\t}\n\t\t\tswitch {\n\t\t\tcase test.len <= 125:\n\t\t\t\tif len(res) != 2 {\n\t\t\t\t\tt.Fatalf(\"Frame len should be 2, got %v\", len(res))\n\t\t\t\t}\n\t\t\t\tif res[1] != byte(test.len) {\n\t\t\t\t\tt.Fatalf(\"Expected len to be in second byte and be %v, got %v\", test.len, res[1])\n\t\t\t\t}\n\t\t\tcase test.len < 65536:\n\t\t\t\t// 1+1+2\n\t\t\t\tif len(res) != 4 {\n\t\t\t\t\tt.Fatalf(\"Frame len should be 4, got %v\", len(res))\n\t\t\t\t}\n\t\t\t\tif res[1] != 126 {\n\t\t\t\t\tt.Fatalf(\"Second byte value should be 126, got %v\", res[1])\n\t\t\t\t}\n\t\t\t\tif rl := binary.BigEndian.Uint16(res[2:]); int(rl) != test.len {\n\t\t\t\t\tt.Fatalf(\"Expected len to be %v, got %v\", test.len, rl)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\t// 1+1+8\n\t\t\t\tif len(res) != 10 {\n\t\t\t\t\tt.Fatalf(\"Frame len should be 10, got %v\", len(res))\n\t\t\t\t}\n\t\t\t\tif res[1] != 127 {\n\t\t\t\t\tt.Fatalf(\"Second byte value should be 127, got %v\", res[1])\n\t\t\t\t}\n\t\t\t\tif rl := binary.BigEndian.Uint64(res[2:]); int(rl) != test.len {\n\t\t\t\t\tt.Fatalf(\"Expected len to be %v, got %v\", test.len, rl)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testWSCreateClientMsg(frameType wsOpCode, frameNum int, final, compressed bool, payload []byte) []byte {\n\tif compressed {\n\t\tbuf := &bytes.Buffer{}\n\t\tcompressor, _ := flate.NewWriter(buf, 1)\n\t\tcompressor.Write(payload)\n\t\tcompressor.Flush()\n\t\tpayload = buf.Bytes()\n\t\t// The last 4 bytes are dropped\n\t\tpayload = payload[:len(payload)-4]\n\t}\n\tframe := make([]byte, 14+len(payload))\n\tif frameNum == 1 {\n\t\tframe[0] = byte(frameType)\n\t}\n\tif final {\n\t\tframe[0] |= wsFinalBit\n\t}\n\tif compressed {\n\t\tframe[0] |= wsRsv1Bit\n\t}\n\tpos := 1\n\tlenPayload := len(payload)\n\tswitch {\n\tcase lenPayload <= 125:\n\t\tframe[pos] = byte(lenPayload) | wsMaskBit\n\t\tpos++\n\tcase lenPayload < 65536:\n\t\tframe[pos] = 126 | wsMaskBit\n\t\tbinary.BigEndian.PutUint16(frame[2:], uint16(lenPayload))\n\t\tpos += 3\n\tdefault:\n\t\tframe[1] = 127 | wsMaskBit\n\t\tbinary.BigEndian.PutUint64(frame[2:], uint64(lenPayload))\n\t\tpos += 9\n\t}\n\tkey := []byte{1, 2, 3, 4}\n\tcopy(frame[pos:], key)\n\tpos += 4\n\tcopy(frame[pos:], payload)\n\ttestWSSimpleMask(key, frame[pos:])\n\tpos += lenPayload\n\treturn frame[:pos]\n}\n\nfunc testWSSetupForRead() (*client, *wsReadInfo, *testReader) {\n\tri := &wsReadInfo{mask: true}\n\tri.init()\n\ttr := &testReader{}\n\topts := DefaultOptions()\n\topts.MaxPending = MAX_PENDING_SIZE\n\ts := &Server{opts: opts}\n\tc := &client{srv: s, ws: &websocket{}}\n\tc.initClient()\n\treturn c, ri, tr\n}\n\nfunc TestWSReadUncompressedFrames(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\t// Create 2 WS messages\n\tpl1 := []byte(\"first message\")\n\twsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, pl1)\n\tpl2 := []byte(\"second message\")\n\twsmsg2 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, pl2)\n\t// Add both in single buffer\n\torgrb := append([]byte(nil), wsmsg1...)\n\torgrb = append(orgrb, wsmsg2...)\n\n\trb := append([]byte(nil), orgrb...)\n\tbufs, err := c.wsRead(ri, tr, rb)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 2 {\n\t\tt.Fatalf(\"Expected 2 buffers, got %v\", n)\n\t}\n\tif !bytes.Equal(bufs[0], pl1) {\n\t\tt.Fatalf(\"Unexpected content for buffer 1: %s\", bufs[0])\n\t}\n\tif !bytes.Equal(bufs[1], pl2) {\n\t\tt.Fatalf(\"Unexpected content for buffer 2: %s\", bufs[1])\n\t}\n\n\t// Now reset and try with the read buffer not containing full ws frame\n\tc, ri, tr = testWSSetupForRead()\n\trb = append([]byte(nil), orgrb...)\n\t// Frame is 1+1+4+'first message'. So say we pass with rb of 11 bytes,\n\t// then we should get \"first\"\n\tbufs, err = c.wsRead(ri, tr, rb[:11])\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 1 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif string(bufs[0]) != \"first\" {\n\t\tt.Fatalf(\"Unexpected content: %q\", bufs[0])\n\t}\n\t// Call again with more data..\n\tbufs, err = c.wsRead(ri, tr, rb[11:32])\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 2 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif string(bufs[0]) != \" message\" {\n\t\tt.Fatalf(\"Unexpected content: %q\", bufs[0])\n\t}\n\tif string(bufs[1]) != \"second \" {\n\t\tt.Fatalf(\"Unexpected content: %q\", bufs[1])\n\t}\n\t// Call with the rest\n\tbufs, err = c.wsRead(ri, tr, rb[32:])\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 1 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif string(bufs[0]) != \"message\" {\n\t\tt.Fatalf(\"Unexpected content: %q\", bufs[0])\n\t}\n}\n\nfunc TestWSReadCompressedFrames(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tc.ws.compress = true\n\tuncompressed := []byte(\"this is the uncompress data\")\n\twsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, uncompressed)\n\trb := append([]byte(nil), wsmsg1...)\n\t// Call with some but not all of the payload\n\tbufs, err := c.wsRead(ri, tr, rb[:10])\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\t// Call with the rest, only then should we get the uncompressed data.\n\tbufs, err = c.wsRead(ri, tr, rb[10:])\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 1 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif !bytes.Equal(bufs[0], uncompressed) {\n\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t}\n\t// Stress the fact that we use a pool and want to make sure\n\t// that if we get a decompressor from the pool, it is properly reset\n\t// with the buffer to decompress.\n\t// Since we unmask the read buffer, reset it now and fill it\n\t// with 10 compressed frames.\n\trb = nil\n\tfor i := 0; i < 10; i++ {\n\t\trb = append(rb, wsmsg1...)\n\t}\n\tbufs, err = c.wsRead(ri, tr, rb)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 10 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\n\t// Compress a message and send it in several frames.\n\tbuf := &bytes.Buffer{}\n\tcompressor, _ := flate.NewWriter(buf, 1)\n\tcompressor.Write(uncompressed)\n\tcompressor.Flush()\n\tcompressed := buf.Bytes()\n\t// The last 4 bytes are dropped\n\tcompressed = compressed[:len(compressed)-4]\n\tncomp := 10\n\tfrag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, compressed[:ncomp])\n\tfrag1[0] |= wsRsv1Bit\n\tfrag2 := testWSCreateClientMsg(wsBinaryMessage, 2, true, false, compressed[ncomp:])\n\trb = append([]byte(nil), frag1...)\n\trb = append(rb, frag2...)\n\tbufs, err = c.wsRead(ri, tr, rb)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 1 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif !bytes.Equal(bufs[0], uncompressed) {\n\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t}\n}\n\nfunc TestWSReadCompressedFrameCorrupted(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tc.ws.compress = true\n\tuncompressed := []byte(\"this is the uncompress data\")\n\twsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, uncompressed)\n\tcopy(wsmsg1[10:], []byte{1, 2, 3, 4})\n\trb := append([]byte(nil), wsmsg1...)\n\tbufs, err := c.wsRead(ri, tr, rb)\n\tif err == nil || !strings.Contains(err.Error(), \"corrupt\") {\n\t\tt.Fatalf(\"Expected error about corrupted data, got %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Expected no buffer, got %v\", n)\n\t}\n}\n\nfunc TestWSReadVariousFrameSizes(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsize int\n\t}{\n\t\t{\"tiny\", 100},\n\t\t{\"medium\", 1000},\n\t\t{\"large\", 70000},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, ri, tr := testWSSetupForRead()\n\t\t\tuncompressed := make([]byte, test.size)\n\t\t\tfor i := 0; i < len(uncompressed); i++ {\n\t\t\t\tuncompressed[i] = 'A' + byte(i%26)\n\t\t\t}\n\t\t\twsmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, uncompressed)\n\t\t\trb := append([]byte(nil), wsmsg1...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif n := len(bufs); n != 1 {\n\t\t\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t\t\t}\n\t\t\tif !bytes.Equal(bufs[0], uncompressed) {\n\t\t\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSReadFragmentedFrames(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tpayloads := []string{\"first\", \"second\", \"third\"}\n\tvar rb []byte\n\tfor i := 0; i < len(payloads); i++ {\n\t\tfinal := i == len(payloads)-1\n\t\tfrag := testWSCreateClientMsg(wsBinaryMessage, i+1, final, false, []byte(payloads[i]))\n\t\trb = append(rb, frag...)\n\t}\n\tbufs, err := c.wsRead(ri, tr, rb)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 3 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tfor i, expected := range payloads {\n\t\tif string(bufs[i]) != expected {\n\t\t\tt.Fatalf(\"Unexpected content for buf=%v: %s\", i, bufs[i])\n\t\t}\n\t}\n}\n\nfunc TestWSReadPartialFrameHeaderAtEndOfReadBuffer(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tmsg1 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"msg1\"))\n\tmsg2 := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"msg2\"))\n\trb := append([]byte(nil), msg1...)\n\trb = append(rb, msg2...)\n\t// We will pass the first frame + the first byte of the next frame.\n\trbl := rb[:len(msg1)+1]\n\t// Make the io reader return the rest of the frame\n\ttr.buf = rb[len(msg1)+1:]\n\tbufs, err := c.wsRead(ri, tr, rbl)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 1 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\t// We should not have asked to the io reader more than what is needed for reading\n\t// the frame header. Since we had already the first byte in the read buffer,\n\t// tr.pos should be 1(size)+4(key)=5\n\tif tr.pos != 5 {\n\t\tt.Fatalf(\"Expected reader pos to be 5, got %v\", tr.pos)\n\t}\n}\n\nfunc TestWSReadPingFrame(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpayload []byte\n\t}{\n\t\t{\"without payload\", nil},\n\t\t{\"with payload\", []byte(\"optional payload\")},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, ri, tr := testWSSetupForRead()\n\t\t\tping := testWSCreateClientMsg(wsPingMessage, 1, true, false, test.payload)\n\t\t\trb := append([]byte(nil), ping...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif n := len(bufs); n != 0 {\n\t\t\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t\t\t}\n\t\t\t// A PONG should have been queued with the payload of the ping\n\t\t\tc.mu.Lock()\n\t\t\tnb, _ := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif n := len(nb); n == 0 {\n\t\t\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t\t\t}\n\t\t\tif expected := 2 + len(test.payload); expected != len(nb[0]) {\n\t\t\t\tt.Fatalf(\"Expected buffer to be %v bytes long, got %v\", expected, len(nb[0]))\n\t\t\t}\n\t\t\tb := nb[0][0]\n\t\t\tif b&wsFinalBit == 0 {\n\t\t\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t\t\t}\n\t\t\tif b&byte(wsPongMessage) == 0 {\n\t\t\t\tt.Fatalf(\"Should have been a PONG, it wasn't: %v\", b)\n\t\t\t}\n\t\t\tif len(test.payload) > 0 {\n\t\t\t\tif !bytes.Equal(nb[0][2:], test.payload) {\n\t\t\t\t\tt.Fatalf(\"Unexpected content: %s\", nb[0][2:])\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSReadPongFrame(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpayload []byte\n\t}{\n\t\t{\"without payload\", nil},\n\t\t{\"with payload\", []byte(\"optional payload\")},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, ri, tr := testWSSetupForRead()\n\t\t\tpong := testWSCreateClientMsg(wsPongMessage, 1, true, false, test.payload)\n\t\t\trb := append([]byte(nil), pong...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif n := len(bufs); n != 0 {\n\t\t\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t\t\t}\n\t\t\t// Nothing should be sent...\n\t\t\tc.mu.Lock()\n\t\t\tnb, _ := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif n := len(nb); n != 0 {\n\t\t\t\tt.Fatalf(\"Expected no buffer, got %v\", n)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSReadCloseFrame(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnoStatus bool\n\t\tpayload []byte\n\t}{\n\t\t{\"status without payload\", false, nil},\n\t\t{\"status with payload\", false, []byte(\"optional payload\")},\n\t\t{\"no status no payload\", true, nil},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, ri, tr := testWSSetupForRead()\n\t\t\tvar payload []byte\n\t\t\tif !test.noStatus {\n\t\t\t\t// a close message has a status in 2 bytes + optional payload\n\t\t\t\tpayload = make([]byte, 2+len(test.payload))\n\t\t\t\tbinary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure)\n\t\t\t\tif len(test.payload) > 0 {\n\t\t\t\t\tcopy(payload[2:], test.payload)\n\t\t\t\t}\n\t\t\t}\n\t\t\tclose := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload)\n\t\t\t// Have a normal frame prior to close to make sure that wsRead returns\n\t\t\t// the normal frame along with io.EOF to indicate that wsCloseMessage was received.\n\t\t\tmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"msg\"))\n\t\t\trb := append([]byte(nil), msg...)\n\t\t\trb = append(rb, close...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb)\n\t\t\t// It is expected that wsRead returns io.EOF on processing a close.\n\t\t\tif err != io.EOF {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif n := len(bufs); n != 1 {\n\t\t\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t\t\t}\n\t\t\tif string(bufs[0]) != \"msg\" {\n\t\t\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t\t\t}\n\t\t\t// A CLOSE should have been queued with the payload of the original close message.\n\t\t\tc.mu.Lock()\n\t\t\tnb, _ := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif n := len(nb); n == 0 {\n\t\t\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t\t\t}\n\t\t\tif test.noStatus {\n\t\t\t\tif expected := 2; expected != len(nb[0]) {\n\t\t\t\t\tt.Fatalf(\"Expected buffer to be %v bytes long, got %v\", expected, len(nb[0]))\n\t\t\t\t}\n\t\t\t} else if expected := 2 + 2 + len(test.payload); expected != len(nb[0]) {\n\t\t\t\tt.Fatalf(\"Expected buffer to be %v bytes long, got %v\", expected, len(nb[0]))\n\t\t\t}\n\t\t\tb := nb[0][0]\n\t\t\tif b&wsFinalBit == 0 {\n\t\t\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t\t\t}\n\t\t\tif b&byte(wsCloseMessage) == 0 {\n\t\t\t\tt.Fatalf(\"Should have been a CLOSE, it wasn't: %v\", b)\n\t\t\t}\n\t\t\tif !test.noStatus {\n\t\t\t\tif status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusNormalClosure {\n\t\t\t\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusNormalClosure, status)\n\t\t\t\t}\n\t\t\t\tif len(test.payload) > 0 {\n\t\t\t\t\tif !bytes.Equal(nb[0][4:], test.payload) {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected content: %s\", nb[0][4:])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSReadControlFrameBetweebFragmentedFrames(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tfrag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, []byte(\"first\"))\n\tfrag2 := testWSCreateClientMsg(wsBinaryMessage, 2, true, false, []byte(\"second\"))\n\tctrl := testWSCreateClientMsg(wsPongMessage, 1, true, false, nil)\n\trb := append([]byte(nil), frag1...)\n\trb = append(rb, ctrl...)\n\trb = append(rb, frag2...)\n\tbufs, err := c.wsRead(ri, tr, rb)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 2 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tif string(bufs[0]) != \"first\" {\n\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t}\n\tif string(bufs[1]) != \"second\" {\n\t\tt.Fatalf(\"Unexpected content: %s\", bufs[1])\n\t}\n}\n\nfunc TestWSCloseFrameWithPartialOrInvalid(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\t// a close message has a status in 2 bytes + optional payload\n\tpayloadTxt := []byte(\"hello\")\n\tpayload := make([]byte, 2+len(payloadTxt))\n\tbinary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure)\n\tcopy(payload[2:], payloadTxt)\n\tcloseMsg := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload)\n\n\t// We will pass to wsRead a buffer of small capacity that contains\n\t// only 1 byte.\n\tcloseFirtByte := []byte{closeMsg[0]}\n\t// Make the io reader return the rest of the frame\n\ttr.buf = closeMsg[1:]\n\tbufs, err := c.wsRead(ri, tr, closeFirtByte[:])\n\t// It is expected that wsRead returns io.EOF on processing a close.\n\tif err != io.EOF {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\t// A CLOSE should have been queued with the payload of the original close message.\n\tc.mu.Lock()\n\tnb, _ := c.collapsePtoNB()\n\tc.mu.Unlock()\n\tif n := len(nb); n == 0 {\n\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t}\n\tif expected := 2 + 2 + len(payloadTxt); expected != len(nb[0]) {\n\t\tt.Fatalf(\"Expected buffer to be %v bytes long, got %v\", expected, len(nb[0]))\n\t}\n\tb := nb[0][0]\n\tif b&wsFinalBit == 0 {\n\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t}\n\tif b&byte(wsCloseMessage) == 0 {\n\t\tt.Fatalf(\"Should have been a CLOSE, it wasn't: %v\", b)\n\t}\n\tif status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusNormalClosure {\n\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusNormalClosure, status)\n\t}\n\tif !bytes.Equal(nb[0][4:], payloadTxt) {\n\t\tt.Fatalf(\"Unexpected content: %s\", nb[0][4:])\n\t}\n\n\t// Now test close with invalid status size (1 instead of 2 bytes)\n\tc, ri, tr = testWSSetupForRead()\n\tbinary.BigEndian.PutUint16(payload, wsCloseStatusNormalClosure)\n\tcloseMsg = testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload[:1])\n\n\t// We will pass to wsRead a buffer of small capacity that contains\n\t// only 1 byte.\n\tcloseFirtByte = []byte{closeMsg[0]}\n\t// Make the io reader return the rest of the frame\n\ttr.buf = closeMsg[1:]\n\tbufs, err = c.wsRead(ri, tr, closeFirtByte[:])\n\tif err == nil || !strings.Contains(err.Error(), \"close frame payload cannot be 1 byte\") {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tc.mu.Lock()\n\tnb, _ = c.collapsePtoNB()\n\tc.mu.Unlock()\n\tif n := len(nb); n == 0 {\n\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t}\n\tif len(nb[0]) < 4 {\n\t\tt.Fatalf(\"Expected buffer to be at least 4 bytes long, got %v\", len(nb[0]))\n\t}\n\tb = nb[0][0]\n\tif b&wsFinalBit == 0 {\n\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t}\n\tif b&byte(wsCloseMessage) == 0 {\n\t\tt.Fatalf(\"Should have been a CLOSE, it wasn't: %v\", b)\n\t}\n\tif status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusProtocolError {\n\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusProtocolError, status)\n\t}\n\n\t// Now test close with invalid status code.\n\tc, ri, tr = testWSSetupForRead()\n\tpayload = make([]byte, 2)\n\tbinary.BigEndian.PutUint16(payload, wsCloseStatusNoStatusReceived)\n\tcloseMsg = testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload)\n\tcloseFirtByte = []byte{closeMsg[0]}\n\ttr.buf = closeMsg[1:]\n\tbufs, err = c.wsRead(ri, tr, closeFirtByte[:])\n\tif err == nil || !strings.Contains(err.Error(), \"invalid close status code\") {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tc.mu.Lock()\n\tnb, _ = c.collapsePtoNB()\n\tc.mu.Unlock()\n\tif n := len(nb); n == 0 {\n\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t}\n\tif len(nb[0]) < 4 {\n\t\tt.Fatalf(\"Expected buffer to be at least 4 bytes long, got %v\", len(nb[0]))\n\t}\n\tif status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusProtocolError {\n\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusProtocolError, status)\n\t}\n}\n\nfunc TestWSReadGetErrors(t *testing.T) {\n\ttr := &testReader{err: fmt.Errorf(\"on purpose\")}\n\tfor _, test := range []struct {\n\t\tlenPayload int\n\t\trbextra int\n\t}{\n\t\t{10, 1},\n\t\t{10, 3},\n\t\t{200, 1},\n\t\t{200, 2},\n\t\t{200, 5},\n\t\t{70000, 1},\n\t\t{70000, 5},\n\t\t{70000, 13},\n\t} {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tc, ri, _ := testWSSetupForRead()\n\t\t\tmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"msg\"))\n\t\t\tframe := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, make([]byte, test.lenPayload))\n\t\t\trb := append([]byte(nil), msg...)\n\t\t\trb = append(rb, frame...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb[:len(msg)+test.rbextra])\n\t\t\tif err == nil || err.Error() != \"on purpose\" {\n\t\t\t\tt.Fatalf(\"Expected 'on purpose' error, got %v\", err)\n\t\t\t}\n\t\t\tif n := len(bufs); n != 1 {\n\t\t\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t\t\t}\n\t\t\tif string(bufs[0]) != \"msg\" {\n\t\t\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSHandleControlFrameErrors(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\ttr.err = fmt.Errorf(\"on purpose\")\n\n\t// a close message has a status in 2 bytes + optional payload\n\ttext := []byte(\"this is a close message\")\n\tpayload := make([]byte, 2+len(text))\n\tbinary.BigEndian.PutUint16(payload[:2], wsCloseStatusNormalClosure)\n\tcopy(payload[2:], text)\n\tctrl := testWSCreateClientMsg(wsCloseMessage, 1, true, false, payload)\n\n\tbufs, err := c.wsRead(ri, tr, ctrl[:len(ctrl)-4])\n\tif err == nil || err.Error() != \"on purpose\" {\n\t\tt.Fatalf(\"Expected 'on purpose' error, got %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\n\t// Alter the content of close message. It is supposed to be valid utf-8.\n\tc, ri, tr = testWSSetupForRead()\n\tcp := append([]byte(nil), payload...)\n\tcp[10] = 0xF1\n\tctrl = testWSCreateClientMsg(wsCloseMessage, 1, true, false, cp)\n\tbufs, err = c.wsRead(ri, tr, ctrl)\n\t// We should still receive an EOF but the message enqueued to the client\n\t// should contain wsCloseStatusInvalidPayloadData and the error about invalid utf8\n\tif err != io.EOF {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif n := len(bufs); n != 0 {\n\t\tt.Fatalf(\"Unexpected buffer returned: %v\", n)\n\t}\n\tc.mu.Lock()\n\tnb, _ := c.collapsePtoNB()\n\tc.mu.Unlock()\n\tif n := len(nb); n == 0 {\n\t\tt.Fatalf(\"Expected buffers, got %v\", n)\n\t}\n\tb := nb[0][0]\n\tif b&wsFinalBit == 0 {\n\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t}\n\tif b&byte(wsCloseMessage) == 0 {\n\t\tt.Fatalf(\"Should have been a CLOSE, it wasn't: %v\", b)\n\t}\n\tif status := binary.BigEndian.Uint16(nb[0][2:4]); status != wsCloseStatusInvalidPayloadData {\n\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusInvalidPayloadData, status)\n\t}\n\tif !bytes.Contains(nb[0][4:], []byte(\"utf8\")) {\n\t\tt.Fatalf(\"Unexpected content: %s\", nb[0][4:])\n\t}\n}\n\nfunc TestWSReadErrors(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tcframe func() []byte\n\t\terr string\n\t\tnbufs int\n\t\tsetup func(*client)\n\t\tverify func(*testing.T, *wsReadInfo)\n\t}{\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"hello\"))\n\t\t\t\tmsg[1] &= ^byte(wsMaskBit)\n\t\t\t\treturn msg\n\t\t\t},\n\t\t\terr: \"mask bit missing\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\treturn testWSCreateClientMsg(wsPingMessage, 1, true, false, make([]byte, 200))\n\t\t\t},\n\t\t\terr: \"control frame length bigger than maximum allowed\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\treturn testWSCreateClientMsg(wsPingMessage, 1, false, false, []byte(\"hello\"))\n\t\t\t},\n\t\t\terr: \"control frame does not have final bit set\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tfrag1 := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, []byte(\"frag1\"))\n\t\t\t\tnewMsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"new message\"))\n\t\t\t\tall := append([]byte(nil), frag1...)\n\t\t\t\tall = append(all, newMsg...)\n\t\t\t\treturn all\n\t\t\t},\n\t\t\terr: \"new message started before final frame for previous message was received\", nbufs: 2,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tframe := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"frame\"))\n\t\t\t\tfrag := testWSCreateClientMsg(wsBinaryMessage, 2, false, false, []byte(\"continuation\"))\n\t\t\t\tall := append([]byte(nil), frame...)\n\t\t\t\tall = append(all, frag...)\n\t\t\t\treturn all\n\t\t\t},\n\t\t\terr: \"invalid continuation frame\", nbufs: 2,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\treturn testWSCreateClientMsg(wsBinaryMessage, 2, false, false, []byte(\"frame\"))\n\t\t\t},\n\t\t\terr: \"invalid continuation frame\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\treturn testWSCreateClientMsg(11, 1, false, false, []byte(\"hello\"))\n\t\t\t},\n\t\t\terr: \"unknown opcode\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, nil)\n\t\t\t\tmsg = append(msg, 0, 0, 0, 0)\n\t\t\t\tmsg[1] = 127 | wsMaskBit\n\t\t\t\tbinary.BigEndian.PutUint64(msg[2:], uint64(1)<<63)\n\t\t\t\treturn msg\n\t\t\t},\n\t\t\terr: \"invalid 64-bit payload length\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\treturn testWSCreateClientMsg(wsBinaryMessage, 1, true, true, []byte(\"compressed\"))\n\t\t\t},\n\t\t\terr: \"compressed frame received without negotiated permessage-deflate\", nbufs: 1,\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tframe := testWSCreateClientMsg(wsBinaryMessage, 1, false, false, make([]byte, 32))\n\t\t\t\tframe[0] |= wsRsv1Bit\n\t\t\t\treturn frame\n\t\t\t},\n\t\t\terr: ErrMaxPayload.Error(), nbufs: 1,\n\t\t\tsetup: func(c *client) {\n\t\t\t\tatomic.StoreInt32(&c.mpay, 16)\n\t\t\t\tc.ws.compress = true\n\t\t\t},\n\t\t\tverify: func(t *testing.T, ri *wsReadInfo) {\n\t\t\t\tif !ri.fs {\n\t\t\t\t\tt.Fatal(\"Expected r.fs=true after compressed payload rejection\")\n\t\t\t\t}\n\t\t\t\tif ri.fc {\n\t\t\t\t\tt.Fatal(\"Expected r.fc=false after compressed payload rejection\")\n\t\t\t\t}\n\t\t\t\tif !ri.ff {\n\t\t\t\t\tt.Fatal(\"Expected r.ff=true after compressed payload rejection\")\n\t\t\t\t}\n\t\t\t\tif ri.cbufs != nil || ri.coff != 0 || ri.csz != 0 || ri.rem != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected compressed state reset, got cbufs=%v coff=%d csz=%d rem=%d\", ri.cbufs != nil, ri.coff, ri.csz, ri.rem)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tcframe: func() []byte {\n\t\t\t\tpayload := bytes.Repeat([]byte{0}, 2048)\n\t\t\t\tbuf := &bytes.Buffer{}\n\t\t\t\tcompressor, _ := flate.NewWriter(buf, 1)\n\t\t\t\tcompressor.Write(payload)\n\t\t\t\tcompressor.Flush()\n\t\t\t\tcompressed := buf.Bytes()\n\t\t\t\tcompressed = compressed[:len(compressed)-4]\n\t\t\t\tframe := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, compressed)\n\t\t\t\tframe[0] |= wsRsv1Bit\n\t\t\t\treturn frame\n\t\t\t},\n\t\t\terr: ErrMaxPayload.Error(), nbufs: 1,\n\t\t\tsetup: func(c *client) {\n\t\t\t\tatomic.StoreInt32(&c.mpay, 16)\n\t\t\t\tc.ws.compress = true\n\t\t\t},\n\t\t\tverify: func(t *testing.T, ri *wsReadInfo) {\n\t\t\t\tif !ri.fs {\n\t\t\t\t\tt.Fatal(\"Expected r.fs=true after decompression error\")\n\t\t\t\t}\n\t\t\t\tif ri.fc {\n\t\t\t\t\tt.Fatal(\"Expected r.fc=false after decompression error\")\n\t\t\t\t}\n\t\t\t\tif !ri.ff {\n\t\t\t\t\tt.Fatal(\"Expected r.ff=true after decompression error\")\n\t\t\t\t}\n\t\t\t\tif ri.cbufs != nil || ri.coff != 0 || ri.csz != 0 || ri.rem != 0 {\n\t\t\t\t\tt.Fatalf(\"Expected compressed state reset, got cbufs=%v coff=%d csz=%d rem=%d\", ri.cbufs != nil, ri.coff, ri.csz, ri.rem)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.err, func(t *testing.T) {\n\t\t\tc, ri, tr := testWSSetupForRead()\n\t\t\tif test.setup != nil {\n\t\t\t\ttest.setup(c)\n\t\t\t}\n\t\t\t// Add a valid message first\n\t\t\tmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"hello\"))\n\t\t\t// Then add the bad frame\n\t\t\tbad := test.cframe()\n\t\t\t// Add them both to a read buffer\n\t\t\trb := append([]byte(nil), msg...)\n\t\t\trb = append(rb, bad...)\n\t\t\tbufs, err := c.wsRead(ri, tr, rb)\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Expected error to contain %q, got %q\", test.err, err.Error())\n\t\t\t}\n\t\t\tif n := len(bufs); n != test.nbufs {\n\t\t\t\tt.Fatalf(\"Unexpected number of buffers: %v\", n)\n\t\t\t}\n\t\t\tif string(bufs[0]) != \"hello\" {\n\t\t\t\tt.Fatalf(\"Unexpected content: %s\", bufs[0])\n\t\t\t}\n\t\t\tif test.verify != nil {\n\t\t\t\ttest.verify(t, ri)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSReadHugePayloadLenDoesNotPanic(t *testing.T) {\n\tc, ri, tr := testWSSetupForRead()\n\tdefer require_NoPanic(t)\n\n\trb := make([]byte, 14)\n\trb[0] = byte(wsBinaryMessage) | wsFinalBit\n\trb[1] = 127 | wsMaskBit\n\tbinary.BigEndian.PutUint64(rb[2:], ^uint64(0))\n\tcopy(rb[10:], []byte{1, 2, 3, 4})\n\n\t_, err := c.wsRead(ri, tr, rb)\n\trequire_Error(t, err, errors.New(\"invalid 64-bit payload length\"))\n}\n\nfunc TestWSReadByteWithEmptyCompressedBufferDoesNotPanic(t *testing.T) {\n\tr := &wsReadInfo{cbufs: [][]byte{{}}}\n\tdefer require_NoPanic(t)\n\n\t_, err := r.ReadByte()\n\trequire_Error(t, err, io.EOF)\n}\n\nfunc TestWSEnqueueCloseMsg(t *testing.T) {\n\tfor _, test := range []struct {\n\t\treason ClosedState\n\t\tstatus int\n\t}{\n\t\t{ClientClosed, wsCloseStatusNormalClosure},\n\t\t{AuthenticationTimeout, wsCloseStatusPolicyViolation},\n\t\t{AuthenticationViolation, wsCloseStatusPolicyViolation},\n\t\t{SlowConsumerPendingBytes, wsCloseStatusPolicyViolation},\n\t\t{SlowConsumerWriteDeadline, wsCloseStatusPolicyViolation},\n\t\t{MaxAccountConnectionsExceeded, wsCloseStatusPolicyViolation},\n\t\t{MaxConnectionsExceeded, wsCloseStatusPolicyViolation},\n\t\t{MaxControlLineExceeded, wsCloseStatusPolicyViolation},\n\t\t{MaxSubscriptionsExceeded, wsCloseStatusPolicyViolation},\n\t\t{MissingAccount, wsCloseStatusPolicyViolation},\n\t\t{AuthenticationExpired, wsCloseStatusPolicyViolation},\n\t\t{Revocation, wsCloseStatusPolicyViolation},\n\t\t{TLSHandshakeError, wsCloseStatusTLSHandshake},\n\t\t{ParseError, wsCloseStatusProtocolError},\n\t\t{ProtocolViolation, wsCloseStatusProtocolError},\n\t\t{BadClientProtocolVersion, wsCloseStatusProtocolError},\n\t\t{MaxPayloadExceeded, wsCloseStatusMessageTooBig},\n\t\t{ServerShutdown, wsCloseStatusGoingAway},\n\t\t{WriteError, wsCloseStatusGoingAway},\n\t\t{ReadError, wsCloseStatusGoingAway},\n\t\t{StaleConnection, wsCloseStatusGoingAway},\n\t\t{ClosedState(254), wsCloseStatusInternalSrvError},\n\t} {\n\t\tt.Run(test.reason.String(), func(t *testing.T) {\n\t\t\tc, _, _ := testWSSetupForRead()\n\t\t\tc.wsEnqueueCloseMessage(test.reason)\n\t\t\tc.mu.Lock()\n\t\t\tnb, _ := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif n := len(nb); n != 1 {\n\t\t\t\tt.Fatalf(\"Expected 1 buffer, got %v\", n)\n\t\t\t}\n\t\t\tb := nb[0][0]\n\t\t\tif b&wsFinalBit == 0 {\n\t\t\t\tt.Fatalf(\"Control frame should have been the final flag, it was not set: %v\", b)\n\t\t\t}\n\t\t\tif b&byte(wsCloseMessage) == 0 {\n\t\t\t\tt.Fatalf(\"Should have been a CLOSE, it wasn't: %v\", b)\n\t\t\t}\n\t\t\tif status := binary.BigEndian.Uint16(nb[0][2:4]); int(status) != test.status {\n\t\t\t\tt.Fatalf(\"Expected status to be %v, got %v\", test.status, status)\n\t\t\t}\n\t\t\tif string(nb[0][4:]) != test.reason.String() {\n\t\t\t\tt.Fatalf(\"Unexpected content: %s\", nb[0][4:])\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype testResponseWriter struct {\n\thttp.ResponseWriter\n\tbuf bytes.Buffer\n\theaders http.Header\n\terr error\n\tbrw *bufio.ReadWriter\n\tconn *testWSFakeNetConn\n}\n\nfunc (trw *testResponseWriter) Write(p []byte) (int, error) {\n\treturn trw.buf.Write(p)\n}\n\nfunc (trw *testResponseWriter) WriteHeader(status int) {\n\ttrw.buf.WriteString(fmt.Sprintf(\"%v\", status))\n}\n\nfunc (trw *testResponseWriter) Header() http.Header {\n\tif trw.headers == nil {\n\t\ttrw.headers = make(http.Header)\n\t}\n\treturn trw.headers\n}\n\ntype testNoHijackResponseWriter struct {\n\tbuf bytes.Buffer\n\theaders http.Header\n}\n\nfunc (trw *testNoHijackResponseWriter) Write(p []byte) (int, error) {\n\treturn trw.buf.Write(p)\n}\n\nfunc (trw *testNoHijackResponseWriter) WriteHeader(status int) {\n\ttrw.buf.WriteString(fmt.Sprintf(\"%v\", status))\n}\n\nfunc (trw *testNoHijackResponseWriter) Header() http.Header {\n\tif trw.headers == nil {\n\t\ttrw.headers = make(http.Header)\n\t}\n\treturn trw.headers\n}\n\ntype testWSFakeNetConn struct {\n\tnet.Conn\n\twbuf bytes.Buffer\n\terr error\n\twsOpened bool\n\tisClosed bool\n\tdeadlineCleared bool\n}\n\nfunc (c *testWSFakeNetConn) Write(p []byte) (int, error) {\n\tif c.err != nil {\n\t\treturn 0, c.err\n\t}\n\treturn c.wbuf.Write(p)\n}\n\nfunc (c *testWSFakeNetConn) SetDeadline(t time.Time) error {\n\tif t.IsZero() {\n\t\tc.deadlineCleared = true\n\t}\n\treturn nil\n}\n\nfunc (c *testWSFakeNetConn) Close() error {\n\tc.isClosed = true\n\treturn nil\n}\n\nfunc (trw *testResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {\n\tif trw.conn == nil {\n\t\ttrw.conn = &testWSFakeNetConn{}\n\t}\n\ttrw.conn.wsOpened = true\n\tif trw.brw == nil {\n\t\ttrw.brw = bufio.NewReadWriter(bufio.NewReader(trw.conn), bufio.NewWriter(trw.conn))\n\t}\n\treturn trw.conn, trw.brw, trw.err\n}\n\nfunc testWSOptions() *Options {\n\topts := DefaultOptions()\n\topts.DisableShortFirstPing = true\n\topts.Websocket.Host = \"127.0.0.1\"\n\topts.Websocket.Port = -1\n\topts.NoSystemAccount = true\n\tvar err error\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server.pem\",\n\t\tKeyFile: \"./configs/certs/key.pem\",\n\t}\n\topts.Websocket.TLSConfig, err = GenTLSConfig(tc)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn opts\n}\n\nfunc testWSCreateValidReq() *http.Request {\n\treq := &http.Request{\n\t\tMethod: \"GET\",\n\t\tHost: \"localhost\",\n\t\tProto: \"HTTP/1.1\",\n\t}\n\treq.Header = make(http.Header)\n\treq.Header.Set(\"Upgrade\", \"websocket\")\n\treq.Header.Set(\"Connection\", \"Upgrade\")\n\treq.Header.Set(\"Sec-Websocket-Key\", \"dGhlIHNhbXBsZSBub25jZQ==\")\n\treq.Header.Set(\"Sec-Websocket-Version\", \"13\")\n\treturn req\n}\n\nfunc TestWSCheckOrigin(t *testing.T) {\n\tnotSameOrigin := false\n\tsameOrigin := true\n\tallowedListEmpty := []string{}\n\tsomeList := []string{\"http://host1.com\", \"http://host2.com:1234\"}\n\tsameHostMultiScheme := []string{\"http://host3.com\", \"https://host3.com\"}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsameOrigin bool\n\t\torigins []string\n\t\treqHost string\n\t\treqTLS bool\n\t\torigin string\n\t\terr string\n\t}{\n\t\t{\"any\", notSameOrigin, allowedListEmpty, \"\", false, \"http://any.host.com\", \"\"},\n\t\t{\"same origin ok\", sameOrigin, allowedListEmpty, \"host.com\", false, \"http://host.com:80\", \"\"},\n\t\t{\"same origin bad host\", sameOrigin, allowedListEmpty, \"host.com\", false, \"http://other.host.com\", \"not same origin\"},\n\t\t{\"same origin bad port\", sameOrigin, allowedListEmpty, \"host.com\", false, \"http://host.com:81\", \"not same origin\"},\n\t\t{\"same origin bad scheme explicit port\", sameOrigin, allowedListEmpty, \"host.com:443\", true, \"http://host.com:443\", \"not same origin\"},\n\t\t{\"same origin bad scheme\", sameOrigin, allowedListEmpty, \"host.com\", true, \"http://host.com\", \"not same origin\"},\n\t\t{\"same origin bad uri\", sameOrigin, allowedListEmpty, \"host.com\", false, \"@@@://invalid:url:1234\", \"invalid URI\"},\n\t\t{\"same origin bad url\", sameOrigin, allowedListEmpty, \"host.com\", false, \"http://invalid:url:1234\", \"too many colons\"},\n\t\t{\"same origin bad req host\", sameOrigin, allowedListEmpty, \"invalid:url:1234\", false, \"http://host.com\", \"too many colons\"},\n\t\t{\"no origin same origin ignored\", sameOrigin, allowedListEmpty, \"\", false, \"\", \"\"},\n\t\t{\"no origin list ignored\", sameOrigin, someList, \"\", false, \"\", \"\"},\n\t\t{\"no origin same origin and list ignored\", sameOrigin, someList, \"\", false, \"\", \"\"},\n\t\t{\"allowed from list\", notSameOrigin, someList, \"\", false, \"http://host2.com:1234\", \"\"},\n\t\t{\"allowed with different path\", notSameOrigin, someList, \"\", false, \"http://host1.com/some/path\", \"\"},\n\t\t{\"allowed from list same host http\", notSameOrigin, sameHostMultiScheme, \"\", false, \"http://host3.com\", \"\"},\n\t\t{\"allowed from list same host https\", notSameOrigin, sameHostMultiScheme, \"\", false, \"https://host3.com\", \"\"},\n\t\t{\"list bad port\", notSameOrigin, someList, \"\", false, \"http://host1.com:1234\", \"not in the allowed list\"},\n\t\t{\"list bad scheme\", notSameOrigin, someList, \"\", false, \"https://host2.com:1234\", \"not in the allowed list\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := DefaultOptions()\n\t\t\topts.Websocket.SameOrigin = test.sameOrigin\n\t\t\topts.Websocket.AllowedOrigins = test.origins\n\t\t\ts := &Server{opts: opts}\n\t\t\ts.wsSetOriginOptions(&opts.Websocket)\n\n\t\t\treq := testWSCreateValidReq()\n\t\t\treq.Host = test.reqHost\n\t\t\tif test.reqTLS {\n\t\t\t\treq.TLS = &tls.ConnectionState{}\n\t\t\t}\n\t\t\tif test.origin != \"\" {\n\t\t\t\treq.Header.Set(\"Origin\", test.origin)\n\t\t\t}\n\t\t\terr := s.websocket.checkOrigin(req)\n\t\t\tif test.err == \"\" && err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else if test.err != \"\" && (err == nil || !strings.Contains(err.Error(), test.err)) {\n\t\t\t\tt.Fatalf(\"Expected error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSUpgradeValidationErrors(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsetup func() (*Options, *testResponseWriter, *http.Request)\n\t\terr string\n\t\tstatus int\n\t}{\n\t\t{\n\t\t\t\"bad method\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Method = \"POST\"\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"must be GET\",\n\t\t\thttp.StatusMethodNotAllowed,\n\t\t},\n\t\t{\n\t\t\t\"no host\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Host = \"\"\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"'Host' missing in request\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"invalid upgrade header\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Del(\"Upgrade\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid value for header 'Upgrade'\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"invalid connection header\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Del(\"Connection\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid value for header 'Connection'\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"no key\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Del(\"Sec-Websocket-Key\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"key missing\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"empty key\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Set(\"Sec-Websocket-Key\", \"\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"key missing\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"invalid key encoding\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Set(\"Sec-Websocket-Key\", \"%%%\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid websocket key\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"invalid key length\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Set(\"Sec-Websocket-Key\", base64.StdEncoding.EncodeToString([]byte(\"short\")))\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid websocket key\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"missing version\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Del(\"Sec-Websocket-Version\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid version\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"wrong version\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Set(\"Sec-Websocket-Version\", \"99\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"invalid version\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t\t{\n\t\t\t\"origin\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\topts.Websocket.SameOrigin = true\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treq.Header.Set(\"Origin\", \"http://bad.host.com\")\n\t\t\t\treturn opts, nil, req\n\t\t\t},\n\t\t\t\"origin not allowed\",\n\t\t\thttp.StatusForbidden,\n\t\t},\n\t\t{\n\t\t\t\"hijack error\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\trw := &testResponseWriter{err: fmt.Errorf(\"on purpose\")}\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treturn opts, rw, req\n\t\t\t},\n\t\t\t\"on purpose\",\n\t\t\thttp.StatusInternalServerError,\n\t\t},\n\t\t{\n\t\t\t\"hijack buffered data\",\n\t\t\tfunc() (*Options, *testResponseWriter, *http.Request) {\n\t\t\t\topts := testWSOptions()\n\t\t\t\tbuf := &bytes.Buffer{}\n\t\t\t\tbuf.WriteString(\"some data\")\n\t\t\t\trw := &testResponseWriter{\n\t\t\t\t\tconn: &testWSFakeNetConn{},\n\t\t\t\t\tbrw: bufio.NewReadWriter(bufio.NewReader(buf), bufio.NewWriter(nil)),\n\t\t\t\t}\n\t\t\t\ttmp := [1]byte{}\n\t\t\t\tio.ReadAtLeast(rw.brw, tmp[:1], 1)\n\t\t\t\treq := testWSCreateValidReq()\n\t\t\t\treturn opts, rw, req\n\t\t\t},\n\t\t\t\"client sent data before handshake is complete\",\n\t\t\thttp.StatusBadRequest,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts, rw, req := test.setup()\n\t\t\tif rw == nil {\n\t\t\t\trw = &testResponseWriter{}\n\t\t\t}\n\t\t\ts := &Server{opts: opts}\n\t\t\ts.wsSetOriginOptions(&opts.Websocket)\n\t\t\tres, err := s.wsUpgrade(rw, req)\n\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\tt.Fatalf(\"Should get error %q, got %v\", test.err, err)\n\t\t\t}\n\t\t\tif res != nil {\n\t\t\t\tt.Fatalf(\"Should not have returned a result, got %v\", res)\n\t\t\t}\n\t\t\texpected := fmt.Sprintf(\"%v%s\\n\", test.status, http.StatusText(test.status))\n\t\t\tif got := rw.buf.String(); got != expected {\n\t\t\t\tt.Fatalf(\"Expected %q got %q\", expected, got)\n\t\t\t}\n\t\t\t// Check that if the connection was opened, it is now closed.\n\t\t\tif rw.conn != nil && rw.conn.wsOpened && !rw.conn.isClosed {\n\t\t\t\tt.Fatal(\"Connection was opened, but has not been closed\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSUpgradeResponseWriteError(t *testing.T) {\n\topts := testWSOptions()\n\ts := &Server{opts: opts}\n\texpectedErr := errors.New(\"on purpose\")\n\trw := &testResponseWriter{\n\t\tconn: &testWSFakeNetConn{err: expectedErr},\n\t}\n\treq := testWSCreateValidReq()\n\tres, err := s.wsUpgrade(rw, req)\n\tif err != expectedErr {\n\t\tt.Fatalf(\"Should get error %q, got %v\", expectedErr.Error(), err)\n\t}\n\tif res != nil {\n\t\tt.Fatalf(\"Should not have returned a result, got %v\", res)\n\t}\n\tif !rw.conn.isClosed {\n\t\tt.Fatal(\"Connection should have been closed\")\n\t}\n}\n\nfunc TestWSUpgradeNoHijacker(t *testing.T) {\n\topts := testWSOptions()\n\ts := &Server{opts: opts}\n\trw := &testNoHijackResponseWriter{}\n\treq := testWSCreateValidReq()\n\tres, err := s.wsUpgrade(rw, req)\n\tif err == nil || !strings.Contains(err.Error(), \"websocket upgrade not supported\") {\n\t\tt.Fatalf(\"Should get error %q, got %v\", \"websocket upgrade not supported\", err)\n\t}\n\tif res != nil {\n\t\tt.Fatalf(\"Should not have returned a result, got %v\", res)\n\t}\n\texpected := fmt.Sprintf(\"%v%s\\n\", http.StatusBadRequest, http.StatusText(http.StatusBadRequest))\n\tif got := rw.buf.String(); got != expected {\n\t\tt.Fatalf(\"Expected %q got %q\", expected, got)\n\t}\n}\n\nfunc TestWSUpgradeConnDeadline(t *testing.T) {\n\topts := testWSOptions()\n\topts.Websocket.HandshakeTimeout = time.Second\n\ts := &Server{opts: opts}\n\trw := &testResponseWriter{}\n\treq := testWSCreateValidReq()\n\tres, err := s.wsUpgrade(rw, req)\n\tif res == nil || err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif rw.conn.isClosed {\n\t\tt.Fatal(\"Connection should NOT have been closed\")\n\t}\n\tif !rw.conn.deadlineCleared {\n\t\tt.Fatal(\"Connection deadline should have been cleared after handshake\")\n\t}\n}\n\nfunc TestWSUpgradeWithEmptyXForwardedForSliceDoesNotPanic(t *testing.T) {\n\topts := testWSOptions()\n\ts := &Server{opts: opts}\n\trw := &testResponseWriter{}\n\treq := testWSCreateValidReq()\n\treq.Header[wsXForwardedForHeader] = []string{}\n\tdefer require_NoPanic(t)\n\n\tres, err := s.wsUpgrade(rw, req)\n\trequire_NoError(t, err)\n\trequire_NotNil(t, res)\n\trequire_NotNil(t, res.ws)\n\trequire_Equal(t, res.ws.clientIP, _EMPTY_)\n}\n\nfunc TestWSCompressNegotiation(t *testing.T) {\n\t// No compression on the server, but client asks\n\topts := testWSOptions()\n\ts := &Server{opts: opts}\n\trw := &testResponseWriter{}\n\treq := testWSCreateValidReq()\n\treq.Header.Set(\"Sec-Websocket-Extensions\", \"permessage-deflate\")\n\tres, err := s.wsUpgrade(rw, req)\n\tif res == nil || err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// The http response should not contain \"permessage-deflate\"\n\toutput := rw.conn.wbuf.String()\n\tif strings.Contains(output, \"permessage-deflate\") {\n\t\tt.Fatalf(\"Compression disabled in server so response to client should not contain extension, got %s\", output)\n\t}\n\n\t// Option in the server and client, so compression should be negotiated.\n\ts.opts.Websocket.Compression = true\n\trw = &testResponseWriter{}\n\tres, err = s.wsUpgrade(rw, req)\n\tif res == nil || err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// The http response should not contain \"permessage-deflate\"\n\toutput = rw.conn.wbuf.String()\n\tif !strings.Contains(output, \"permessage-deflate\") {\n\t\tt.Fatalf(\"Compression in server and client request, so response should contain extension, got %s\", output)\n\t}\n\n\t// Option in server but not asked by the client, so response should not contain \"permessage-deflate\"\n\trw = &testResponseWriter{}\n\treq.Header.Del(\"Sec-Websocket-Extensions\")\n\tres, err = s.wsUpgrade(rw, req)\n\tif res == nil || err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\t// The http response should not contain \"permessage-deflate\"\n\toutput = rw.conn.wbuf.String()\n\tif strings.Contains(output, \"permessage-deflate\") {\n\t\tt.Fatalf(\"Compression in server but not in client, so response to client should not contain extension, got %s\", output)\n\t}\n}\n\nfunc TestWSSetHeader(t *testing.T) {\n\topts := testWSOptions()\n\topts.Websocket.Headers = map[string]string{\n\t\t\"X-Header\": \"some-value\",\n\t\t\"X-Another-Header\": \"another-value\",\n\t}\n\ts := &Server{opts: opts}\n\ts.wsSetHeadersOptions(&opts.Websocket)\n\trw := &testResponseWriter{}\n\treq := testWSCreateValidReq()\n\tres, err := s.wsUpgrade(rw, req)\n\tif res == nil || err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\n\tbuf := bufio.NewReader(&rw.conn.wbuf)\n\tresp, err := http.ReadResponse(buf, req)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading request: %v\", err)\n\t}\n\tdefer resp.Body.Close()\n\n\t// Check that the response is a 101\n\tif resp.StatusCode != http.StatusSwitchingProtocols {\n\t\tt.Fatalf(\"Expected 101, got %v\", resp.StatusCode)\n\t}\n\n\theaders := resp.Header.Clone()\n\n\t// Compare all the headers\n\tfor k, v := range opts.Websocket.Headers {\n\t\tif got := headers.Get(k); got != v {\n\t\t\tt.Fatalf(\"Expected %q for header %q, got %q\", v, k, got)\n\t\t}\n\t\theaders.Del(k)\n\t}\n\n\t// Check remain headers\n\tfor k, v := range map[string]string{\n\t\t\"Upgrade\": \"websocket\",\n\t\t\"Connection\": \"Upgrade\",\n\t\t\"Sec-Websocket-Accept\": wsAcceptKey(req.Header.Get(\"Sec-Websocket-Key\")),\n\t} {\n\t\tif got := headers.Get(k); got != v {\n\t\t\tt.Fatalf(\"Expected %q for header %q, got %q\", v, k, got)\n\t\t}\n\t\theaders.Del(k)\n\t}\n\n\t// Check that we have no more headers\n\tif len(headers) > 0 {\n\t\tt.Fatalf(\"Unexpected headers: %v\", headers)\n\t}\n}\n\nfunc TestWSParseOptions(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcontent string\n\t\tcheckOpt func(*WebsocketOpts) error\n\t\terr string\n\t}{\n\t\t// Negative tests\n\t\t{\"bad type\", \"websocket: []\", nil, \"to be a map\"},\n\t\t{\"bad listen\", \"websocket: { listen: [] }\", nil, \"port or host:port\"},\n\t\t{\"bad port\", `websocket: { port: \"abc\" }`, nil, \"not int64\"},\n\t\t{\"bad host\", `websocket: { host: 123 }`, nil, \"not string\"},\n\t\t{\"bad advertise type\", `websocket: { advertise: 123 }`, nil, \"not string\"},\n\t\t{\"bad tls\", `websocket: { tls: 123 }`, nil, \"not map[string]interface {}\"},\n\t\t{\"bad same origin\", `websocket: { same_origin: \"abc\" }`, nil, \"not bool\"},\n\t\t{\"bad allowed origins type\", `websocket: { allowed_origins: {} }`, nil, \"unsupported type\"},\n\t\t{\"bad allowed origins values\", `websocket: { allowed_origins: [ {} ] }`, nil, \"unsupported type in array\"},\n\t\t{\"bad handshake timeout type\", `websocket: { handshake_timeout: [] }`, nil, \"unsupported type\"},\n\t\t{\"bad handshake timeout duration\", `websocket: { handshake_timeout: \"abc\" }`, nil, \"invalid duration\"},\n\t\t{\"bad header type\", `websocket: { headers: 123 }`, nil, \"unsupported type\"},\n\t\t{\"bad header type\", `websocket: { headers: [] }`, nil, \"unsupported type\"},\n\t\t{\"bad header value\", `websocket: { headers: { \"key\": 123 } }`, nil, \"unsupported type\"},\n\t\t{\"unknown field\", `websocket: { this_does_not_exist: 123 }`, nil, \"unknown\"},\n\t\t// Positive tests\n\t\t{\"listen port only\", `websocket { listen: 1234 }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected 1234, got %v\", wo.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"listen host and port\", `websocket { listen: \"localhost:1234\" }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.Host != \"localhost\" || wo.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected localhost:1234, got %v:%v\", wo.Host, wo.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"host\", `websocket { host: \"localhost\" }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.Host != \"localhost\" {\n\t\t\t\treturn fmt.Errorf(\"expected localhost, got %v\", wo.Host)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"port\", `websocket { port: 1234 }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.Port != 1234 {\n\t\t\t\treturn fmt.Errorf(\"expected 1234, got %v\", wo.Port)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"advertise\", `websocket { advertise: \"host:1234\" }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.Advertise != \"host:1234\" {\n\t\t\t\treturn fmt.Errorf(\"expected %q, got %q\", \"host:1234\", wo.Advertise)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"same origin\", `websocket { same_origin: true }`, func(wo *WebsocketOpts) error {\n\t\t\tif !wo.SameOrigin {\n\t\t\t\treturn fmt.Errorf(\"expected same_origin==true, got %v\", wo.SameOrigin)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"allowed origins one only\", `websocket { allowed_origins: \"https://host.com/\" }`, func(wo *WebsocketOpts) error {\n\t\t\texpected := []string{\"https://host.com/\"}\n\t\t\tif !reflect.DeepEqual(wo.AllowedOrigins, expected) {\n\t\t\t\treturn fmt.Errorf(\"expected allowed origins to be %q, got %q\", expected, wo.AllowedOrigins)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"allowed origins array\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tallowed_origins: [\n\t\t\t\t\t\"https://host1.com/\"\n\t\t\t\t\t\"https://host2.com/\"\n\t\t\t\t]\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\texpected := []string{\"https://host1.com/\", \"https://host2.com/\"}\n\t\t\t\tif !reflect.DeepEqual(wo.AllowedOrigins, expected) {\n\t\t\t\t\treturn fmt.Errorf(\"expected allowed origins to be %q, got %q\", expected, wo.AllowedOrigins)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"handshake timeout in whole seconds\", `websocket { handshake_timeout: 3 }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.HandshakeTimeout != 3*time.Second {\n\t\t\t\treturn fmt.Errorf(\"expected handshake to be 3s, got %v\", wo.HandshakeTimeout)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"handshake timeout n duration\", `websocket { handshake_timeout: \"4s\" }`, func(wo *WebsocketOpts) error {\n\t\t\tif wo.HandshakeTimeout != 4*time.Second {\n\t\t\t\treturn fmt.Errorf(\"expected handshake to be 4s, got %v\", wo.HandshakeTimeout)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, \"\"},\n\t\t{\"tls config\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/key.pem\"\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif wo.TLSConfig == nil {\n\t\t\t\t\treturn fmt.Errorf(\"TLSConfig should have been set\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"compression\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tcompression: true\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif !wo.Compression {\n\t\t\t\t\treturn fmt.Errorf(\"Compression should have been set\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"jwt cookie\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tjwt_cookie: \"jwtcookie\"\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif wo.JWTCookie != \"jwtcookie\" {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid JWTCookie value: %q\", wo.JWTCookie)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"no auth user\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tno_auth_user: \"noauthuser\"\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif wo.NoAuthUser != \"noauthuser\" {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid NoAuthUser value: %q\", wo.NoAuthUser)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"auth block\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tauthorization {\n\t\t\t\t\tuser: \"webuser\"\n\t\t\t\t\tpassword: \"pwd\"\n\t\t\t\t\ttoken: \"token\"\n\t\t\t\t\ttimeout: 2.0\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif wo.Username != \"webuser\" || wo.Password != \"pwd\" || wo.Token != \"token\" || wo.AuthTimeout != 2.0 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid auth block: %+v\", wo)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"auth timeout as int\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\tauthorization {\n\t\t\t\t\ttimeout: 2\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif wo.AuthTimeout != 2.0 {\n\t\t\t\t\treturn fmt.Errorf(\"Invalid auth timeout: %v\", wo.AuthTimeout)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t\t{\"headers block\",\n\t\t\t`\n\t\t\twebsocket {\n\t\t\t\theaders {\n\t\t\t\t\t\"X-Header\": \"some-value\"\n\t\t\t\t\t\"X-Another-Header\": \"another-value\"\n\t\t\t\t}\n\t\t\t}\n\t\t\t`, func(wo *WebsocketOpts) error {\n\t\t\t\tif len(wo.Headers) != 2 {\n\t\t\t\t\treturn fmt.Errorf(\"Expected 2 headers, got %v\", len(wo.Headers))\n\t\t\t\t}\n\n\t\t\t\tfor k, v := range map[string]string{\n\t\t\t\t\t\"X-Header\": \"some-value\",\n\t\t\t\t\t\"X-Another-Header\": \"another-value\",\n\t\t\t\t} {\n\t\t\t\t\tif got, ok := wo.Headers[k]; !ok || got != v {\n\t\t\t\t\t\treturn fmt.Errorf(\"Invalid value for %q: %q\", k, got)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}, \"\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(test.content))\n\t\t\to, err := ProcessConfigFile(conf)\n\t\t\tif test.err != _EMPTY_ {\n\t\t\t\tif err == nil || !strings.Contains(err.Error(), test.err) {\n\t\t\t\t\tt.Fatalf(\"For content: %q, expected error about %q, got %v\", test.content, test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else if err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error for content %q: %v\", test.content, err)\n\t\t\t}\n\t\t\tif err := test.checkOpt(&o.Websocket); err != nil {\n\t\t\t\tt.Fatalf(\"Incorrect option for content %q: %v\", test.content, err.Error())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSValidateOptions(t *testing.T) {\n\tnwso := DefaultOptions()\n\twso := testWSOptions()\n\tfor _, test := range []struct {\n\t\tname string\n\t\tgetOpts func() *Options\n\t\terr string\n\t}{\n\t\t{\"websocket disabled\", func() *Options { return nwso.Clone() }, \"\"},\n\t\t{\"no tls\", func() *Options { o := wso.Clone(); o.Websocket.TLSConfig = nil; return o }, \"requires TLS configuration\"},\n\t\t{\"bad url in allowed list\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.AllowedOrigins = []string{\"http://this:is:bad:url\"}\n\t\t\treturn o\n\t\t}, \"unable to parse\"},\n\t\t{\"allowed origin must be absolute URL\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.AllowedOrigins = []string{\"foo\"}\n\t\t\treturn o\n\t\t}, \"unable to parse\"},\n\t\t{\"allowed origin scheme must be http or https\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.AllowedOrigins = []string{\"ftp://host.com\"}\n\t\t\treturn o\n\t\t}, \"must be absolute URLs with http or https scheme\"},\n\t\t{\"missing trusted configuration\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.JWTCookie = \"jwt\"\n\t\t\treturn o\n\t\t}, \"keys configuration is required\"},\n\t\t{\"websocket username not allowed if users specified\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Nkeys = []*NkeyUser{{Nkey: \"abc\"}}\n\t\t\to.Websocket.Username = \"b\"\n\t\t\to.Websocket.Password = \"pwd\"\n\t\t\treturn o\n\t\t}, \"websocket authentication username not compatible with presence of users/nkeys\"},\n\t\t{\"websocket token not allowed if users specified\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Nkeys = []*NkeyUser{{Nkey: \"abc\"}}\n\t\t\to.Websocket.Token = \"mytoken\"\n\t\t\treturn o\n\t\t}, \"websocket authentication token not compatible with presence of users/nkeys\"},\n\t\t{\"headers with sec-websocket- prefix not allowed\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Sec-WebSocket-Key\": \"123\"}\n\t\t\treturn o\n\t\t}, `invalid header \"Sec-WebSocket-Key\", \"Sec-WebSocket-\" prefix not allowed`},\n\t\t{\"header with host\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Host\": \"http://localhost:8080\"}\n\t\t\treturn o\n\t\t}, `websocket: invalid header \"Host\" not allowed`},\n\t\t{\"header with content-length\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Content-Length\": \"0\"}\n\t\t\treturn o\n\t\t}, `websocket: invalid header \"Content-Length\" not allowed`},\n\t\t{\"header with connection\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Connection\": \"Upgrade\"}\n\t\t\treturn o\n\t\t}, `websocket: invalid header \"Connection\" not allowed`},\n\t\t{\"header with upgrade\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Upgrade\": \"websocket\"}\n\t\t\treturn o\n\t\t}, `websocket: invalid header \"Upgrade\" not allowed`},\n\t\t{\"header with Nats-No-Masking\", func() *Options {\n\t\t\to := wso.Clone()\n\t\t\to.Websocket.Headers = map[string]string{\"Nats-No-Masking\": \"false\"}\n\t\t\treturn o\n\t\t}, `websocket: invalid header \"Nats-No-Masking\" not allowed`},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\terr := validateWebsocketOptions(test.getOpts())\n\t\t\tif test.err == \"\" && err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t} else if test.err != \"\" && (err == nil || !strings.Contains(err.Error(), test.err)) {\n\t\t\t\tt.Fatalf(\"Expected error to contain %q, got %v\", test.err, err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSSetOriginOptions(t *testing.T) {\n\to := testWSOptions()\n\tfor _, test := range []struct {\n\t\tcontent string\n\t\terr string\n\t}{\n\t\t{\"@@@://host.com/\", \"invalid URI\"},\n\t\t{\"http://this:is:bad:url/\", \"invalid port\"},\n\t} {\n\t\tt.Run(test.err, func(t *testing.T) {\n\t\t\to.Websocket.AllowedOrigins = []string{test.content}\n\t\t\ts := &Server{}\n\t\t\tl := &captureErrorLogger{errCh: make(chan string, 1)}\n\t\t\ts.SetLogger(l, false, false)\n\t\t\ts.wsSetOriginOptions(&o.Websocket)\n\t\t\tselect {\n\t\t\tcase e := <-l.errCh:\n\t\t\t\tif !strings.Contains(e, test.err) {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t\t\t}\n\t\t\tcase <-time.After(50 * time.Millisecond):\n\t\t\t\tt.Fatalf(\"Did not get the error\")\n\t\t\t}\n\n\t\t})\n\t}\n}\n\ntype captureFatalLogger struct {\n\tDummyLogger\n\tfatalCh chan string\n}\n\nfunc (l *captureFatalLogger) Fatalf(format string, v ...any) {\n\tselect {\n\tcase l.fatalCh <- fmt.Sprintf(format, v...):\n\tdefault:\n\t}\n}\n\nfunc TestWSFailureToStartServer(t *testing.T) {\n\t// Create a listener to use a port\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error listening: %v\", err)\n\t}\n\tdefer l.Close()\n\n\to := testWSOptions()\n\t// Make sure we don't have unnecessary listen ports opened.\n\to.HTTPPort = 0\n\to.Cluster.Port = 0\n\to.Gateway.Name = \"\"\n\to.Gateway.Port = 0\n\to.LeafNode.Port = 0\n\to.Websocket.Port = l.Addr().(*net.TCPAddr).Port\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating server: %v\", err)\n\t}\n\tdefer s.Shutdown()\n\tlogger := &captureFatalLogger{fatalCh: make(chan string, 1)}\n\ts.SetLogger(logger, false, false)\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\ts.Start()\n\t\twg.Done()\n\t}()\n\n\tselect {\n\tcase e := <-logger.fatalCh:\n\t\tif !strings.Contains(e, \"Unable to listen\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Should have reported a fatal error\")\n\t}\n\t// Since this is a test and the process does not actually\n\t// exit on Fatal error, wait for the client port to be\n\t// ready so when we shutdown we don't leave the accept\n\t// loop hanging.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\ts.mu.Lock()\n\t\tready := s.listener != nil\n\t\ts.mu.Unlock()\n\t\tif !ready {\n\t\t\treturn fmt.Errorf(\"client accept loop not started yet\")\n\t\t}\n\t\treturn nil\n\t})\n\ts.Shutdown()\n\twg.Wait()\n}\n\nfunc TestWSAbnormalFailureOfWebServer(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\tlogger := &captureFatalLogger{fatalCh: make(chan string, 1)}\n\ts.SetLogger(logger, false, false)\n\n\t// Now close the WS listener to cause a WebServer error\n\ts.mu.Lock()\n\ts.websocket.listener.Close()\n\ts.mu.Unlock()\n\n\tselect {\n\tcase e := <-logger.fatalCh:\n\t\tif !strings.Contains(e, \"websocket listener error\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Should have reported a fatal error\")\n\t}\n}\n\ntype testWSClientOptions struct {\n\tcompress, web bool\n\thost string\n\tport int\n\textraHeaders map[string][]string\n\tnoTLS bool\n\tpath string\n\textraResponseHeaders map[string]string\n}\n\nfunc testNewWSClient(t testing.TB, o testWSClientOptions) (net.Conn, *bufio.Reader, []byte) {\n\tt.Helper()\n\tc, br, info, err := testNewWSClientWithError(t, o)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn c, br, info\n}\n\nfunc testNewWSClientWithError(t testing.TB, o testWSClientOptions) (net.Conn, *bufio.Reader, []byte, error) {\n\taddr := net.JoinHostPort(o.host, fmt.Sprintf(\"%d\", o.port))\n\twsc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\treturn nil, nil, nil, fmt.Errorf(\"Error creating ws connection: %v\", err)\n\t}\n\tif !o.noTLS {\n\t\twsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true})\n\t\twsc.SetDeadline(time.Now().Add(time.Second))\n\t\tif err := wsc.(*tls.Conn).Handshake(); err != nil {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"Error during handshake: %v\", err)\n\t\t}\n\t\twsc.SetDeadline(time.Time{})\n\t}\n\treq := testWSCreateValidReq()\n\tif o.compress {\n\t\treq.Header.Set(\"Sec-Websocket-Extensions\", \"permessage-deflate\")\n\t}\n\tif o.web {\n\t\treq.Header.Set(\"User-Agent\", \"Mozilla/5.0\")\n\t}\n\tif len(o.extraHeaders) > 0 {\n\t\tfor hdr, values := range o.extraHeaders {\n\t\t\tif len(values) == 0 {\n\t\t\t\treq.Header.Set(hdr, _EMPTY_)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treq.Header.Set(hdr, values[0])\n\t\t\tfor i := 1; i < len(values); i++ {\n\t\t\t\treq.Header.Add(hdr, values[i])\n\t\t\t}\n\t\t}\n\t}\n\treq.URL, _ = url.Parse(\"wss://\" + addr + o.path)\n\tif err := req.Write(wsc); err != nil {\n\t\treturn nil, nil, nil, fmt.Errorf(\"Error sending request: %v\", err)\n\t}\n\tbr := bufio.NewReader(wsc)\n\tresp, err := http.ReadResponse(br, req)\n\tif err != nil {\n\t\treturn nil, nil, nil, fmt.Errorf(\"Error reading response: %v\", err)\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != http.StatusSwitchingProtocols {\n\t\treturn nil, nil, nil, fmt.Errorf(\"Expected response status %v, got %v\", http.StatusSwitchingProtocols, resp.StatusCode)\n\t}\n\tfor k, v := range o.extraResponseHeaders {\n\t\tif value := resp.Header.Get(k); value != v {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"Expected header %q to be %q, got %q\", k, v, value)\n\t\t}\n\t}\n\tvar info []byte\n\tif o.path == mqttWSPath {\n\t\tif v := resp.Header[wsSecProto]; len(v) != 1 || v[0] != wsMQTTSecProtoVal {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"No mqtt protocol in header: %v\", resp.Header)\n\t\t}\n\t} else {\n\t\t// Wait for the INFO\n\t\tinfo = testWSReadFrame(t, br)\n\t\tif !bytes.HasPrefix(info, []byte(\"INFO {\")) {\n\t\t\treturn nil, nil, nil, fmt.Errorf(\"Expected INFO, got %s\", info)\n\t\t}\n\t}\n\treturn wsc, br, info, nil\n}\n\ntype testClaimsOptions struct {\n\tnac *jwt.AccountClaims\n\tnuc *jwt.UserClaims\n\tconnectRequest any\n\tdontSign bool\n\texpectAnswer string\n}\n\nfunc testWSWithClaims(t *testing.T, s *Server, o testWSClientOptions, tclm testClaimsOptions) (kp nkeys.KeyPair, conn net.Conn, rdr *bufio.Reader, auth_was_required bool) {\n\tt.Helper()\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tif tclm.nac == nil {\n\t\ttclm.nac = jwt.NewAccountClaims(apub)\n\t} else {\n\t\ttclm.nac.Subject = apub\n\t}\n\tajwt, err := tclm.nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\tif tclm.nuc == nil {\n\t\ttclm.nuc = jwt.NewUserClaims(pub)\n\t} else {\n\t\ttclm.nuc.Subject = pub\n\t}\n\tjwt, err := tclm.nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\taddAccountToMemResolver(s, apub, ajwt)\n\n\tc, cr, l := testNewWSClient(t, o)\n\n\tvar info struct {\n\t\tNonce string `json:\"nonce,omitempty\"`\n\t\tAuthRequired bool `json:\"auth_required,omitempty\"`\n\t}\n\n\tif err := json.Unmarshal([]byte(l[5:]), &info); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif info.AuthRequired {\n\t\tcs := \"\"\n\t\tif tclm.connectRequest != nil {\n\t\t\tcustomReq, err := json.Marshal(tclm.connectRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\t// PING needed to flush the +OK/-ERR to us.\n\t\t\tcs = fmt.Sprintf(\"CONNECT %v\\r\\nPING\\r\\n\", string(customReq))\n\t\t} else if !tclm.dontSign {\n\t\t\t// Sign Nonce\n\t\t\tsigraw, _ := nkp.Sign([]byte(info.Nonce))\n\t\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\t\t\tcs = fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"sig\\\":\\\"%s\\\",\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt, sig)\n\t\t} else {\n\t\t\tcs = fmt.Sprintf(\"CONNECT {\\\"jwt\\\":%q,\\\"verbose\\\":true,\\\"pedantic\\\":true}\\r\\nPING\\r\\n\", jwt)\n\t\t}\n\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(cs))\n\t\tc.Write(wsmsg)\n\t\tl = testWSReadFrame(t, cr)\n\t\tif !strings.HasPrefix(string(l), tclm.expectAnswer) {\n\t\t\tt.Fatalf(\"Expected %q, got %q\", tclm.expectAnswer, l)\n\t\t}\n\t}\n\treturn akp, c, cr, info.AuthRequired\n}\n\nfunc setupAddTrusted(o *Options) {\n\tkp, _ := nkeys.FromSeed(oSeed)\n\tpub, _ := kp.PublicKey()\n\to.TrustedKeys = []string{pub}\n}\n\nfunc setupAddCookie(o *Options) {\n\to.Websocket.JWTCookie = \"jwt\"\n}\n\nfunc testWSCreateClientGetInfo(t testing.TB, compress, web bool, host string, port int, cookies ...string) (net.Conn, *bufio.Reader, []byte) {\n\tt.Helper()\n\topts := testWSClientOptions{\n\t\tcompress: compress,\n\t\tweb: web,\n\t\thost: host,\n\t\tport: port,\n\t}\n\n\tif len(cookies) > 0 {\n\t\topts.extraHeaders = map[string][]string{}\n\t\topts.extraHeaders[\"Cookie\"] = cookies\n\t}\n\treturn testNewWSClient(t, opts)\n}\n\nfunc testWSCreateClient(t testing.TB, compress, web bool, host string, port int) (net.Conn, *bufio.Reader) {\n\twsc, br, _ := testWSCreateClientGetInfo(t, compress, web, host, port)\n\t// Send CONNECT and PING\n\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, []byte(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1}\\r\\nPING\\r\\n\"))\n\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t}\n\t// Wait for the PONG\n\tif msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\tt.Fatalf(\"Expected PONG, got %s\", msg)\n\t}\n\treturn wsc, br\n}\n\nfunc testWSReadFrame(t testing.TB, br *bufio.Reader) []byte {\n\tt.Helper()\n\tfh := [2]byte{}\n\tif _, err := io.ReadAtLeast(br, fh[:2], 2); err != nil {\n\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t}\n\tfc := fh[0]&wsRsv1Bit != 0\n\tsb := fh[1]\n\tsize := 0\n\tswitch {\n\tcase sb <= 125:\n\t\tsize = int(sb)\n\tcase sb == 126:\n\t\ttmp := [2]byte{}\n\t\tif _, err := io.ReadAtLeast(br, tmp[:2], 2); err != nil {\n\t\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t\t}\n\t\tsize = int(binary.BigEndian.Uint16(tmp[:2]))\n\tcase sb == 127:\n\t\ttmp := [8]byte{}\n\t\tif _, err := io.ReadAtLeast(br, tmp[:8], 8); err != nil {\n\t\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t\t}\n\t\tsize = int(binary.BigEndian.Uint64(tmp[:8]))\n\t}\n\tbuf := make([]byte, size)\n\tif _, err := io.ReadAtLeast(br, buf, size); err != nil {\n\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t}\n\tif !fc {\n\t\treturn buf\n\t}\n\tbuf = append(buf, 0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff)\n\tdbr := bytes.NewBuffer(buf)\n\td := flate.NewReader(dbr)\n\tuncompressed, err := io.ReadAll(d)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t}\n\treturn uncompressed\n}\n\nfunc TestWSPubSub(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tcompression bool\n\t}{\n\t\t{\"no compression\", false},\n\t\t{\"compression\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testWSOptions()\n\t\t\tif test.compression {\n\t\t\t\to.Websocket.Compression = true\n\t\t\t}\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Create a regular client to subscribe\n\t\t\tnc := natsConnect(t, s.ClientURL())\n\t\t\tdefer nc.Close()\n\t\t\tnsub := natsSubSync(t, nc, \"foo\")\n\t\t\tcheckExpectedSubs(t, 1, s)\n\n\t\t\t// Now create a WS client and send a message on \"foo\"\n\t\t\twsc, br := testWSCreateClient(t, test.compression, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\t// Send a WS message for \"PUB foo 2\\r\\nok\\r\\n\"\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"PUB foo 7\\r\\nfrom ws\\r\\n\"))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\n\t\t\t// Now check that message is received\n\t\t\tmsg := natsNexMsg(t, nsub, time.Second)\n\t\t\tif string(msg.Data) != \"from ws\" {\n\t\t\t\tt.Fatalf(\"Expected message to be %q, got %q\", \"ok\", string(msg.Data))\n\t\t\t}\n\n\t\t\t// Now do reverse, create a subscription on WS client on bar\n\t\t\twsmsg = testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"SUB bar 1\\r\\n\"))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending subscription: %v\", err)\n\t\t\t}\n\t\t\t// Wait for it to be registered on server\n\t\t\tcheckExpectedSubs(t, 2, s)\n\t\t\t// Now publish from NATS connection and verify received on WS client\n\t\t\tnatsPub(t, nc, \"bar\", []byte(\"from nats\"))\n\t\t\tnatsFlush(t, nc)\n\n\t\t\t// Check for the \"from nats\" message...\n\t\t\t// Set some deadline so we are not stuck forever on failure\n\t\t\twsc.SetReadDeadline(time.Now().Add(10 * time.Second))\n\t\t\tok := 0\n\t\t\tfor {\n\t\t\t\tline, _, err := br.ReadLine()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error reading: %v\", err)\n\t\t\t\t}\n\t\t\t\t// Note that this works even in compression test because those\n\t\t\t\t// texts are likely not to be compressed, but compression code is\n\t\t\t\t// still executed.\n\t\t\t\tif ok == 0 && bytes.Contains(line, []byte(\"MSG bar 1 9\")) {\n\t\t\t\t\tok = 1\n\t\t\t\t\tcontinue\n\t\t\t\t} else if ok == 1 && bytes.Contains(line, []byte(\"from nats\")) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSTLSConnection(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddr := net.JoinHostPort(o.Websocket.Host, fmt.Sprintf(\"%d\", o.Websocket.Port))\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuseTLS bool\n\t\tstatus int\n\t}{\n\t\t{\"client uses TLS\", true, http.StatusSwitchingProtocols},\n\t\t{\"client does not use TLS\", false, http.StatusBadRequest},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\twsc, err := net.Dial(\"tcp\", addr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t\t\t}\n\t\t\tdefer wsc.Close()\n\t\t\tif test.useTLS {\n\t\t\t\twsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true})\n\t\t\t\tif err := wsc.(*tls.Conn).Handshake(); err != nil {\n\t\t\t\t\tt.Fatalf(\"Error during handshake: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treq := testWSCreateValidReq()\n\t\t\tvar scheme string\n\t\t\tif test.useTLS {\n\t\t\t\tscheme = \"s\"\n\t\t\t}\n\t\t\treq.URL, _ = url.Parse(\"ws\" + scheme + \"://\" + addr)\n\t\t\tif err := req.Write(wsc); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t\t\t}\n\t\t\tbr := bufio.NewReader(wsc)\n\t\t\tresp, err := http.ReadResponse(br, req)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading response: %v\", err)\n\t\t\t}\n\t\t\tdefer resp.Body.Close()\n\t\t\tif resp.StatusCode != test.status {\n\t\t\t\tt.Fatalf(\"Expected status %v, got %v\", test.status, resp.StatusCode)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSTLSVerifyClientCert(t *testing.T) {\n\to := testWSOptions()\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/server-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/server-key.pem\",\n\t\tCaFile: \"../test/configs/certs/ca.pem\",\n\t\tVerify: true,\n\t}\n\ttlsc, err := GenTLSConfig(tc)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating tls config: %v\", err)\n\t}\n\to.Websocket.TLSConfig = tlsc\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddr := net.JoinHostPort(o.Websocket.Host, fmt.Sprintf(\"%d\", o.Websocket.Port))\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tprovideCert bool\n\t}{\n\t\t{\"client provides cert\", true},\n\t\t{\"client does not provide cert\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\twsc, err := net.Dial(\"tcp\", addr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t\t\t}\n\t\t\tdefer wsc.Close()\n\t\t\ttlsc := &tls.Config{}\n\t\t\tif test.provideCert {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/client-cert.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/client-key.pem\",\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\ttlsc, err = GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\ttlsc.InsecureSkipVerify = true\n\t\t\twsc = tls.Client(wsc, tlsc)\n\t\t\tif err := wsc.(*tls.Conn).Handshake(); err != nil {\n\t\t\t\tt.Fatalf(\"Error during handshake: %v\", err)\n\t\t\t}\n\t\t\treq := testWSCreateValidReq()\n\t\t\treq.URL, _ = url.Parse(\"wss://\" + addr)\n\t\t\tif err := req.Write(wsc); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t\t\t}\n\t\t\tbr := bufio.NewReader(wsc)\n\t\t\tresp, err := http.ReadResponse(br, req)\n\t\t\tif resp != nil {\n\t\t\t\tresp.Body.Close()\n\t\t\t}\n\t\t\tif !test.provideCert {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected error, did not get one\")\n\t\t\t\t} else if !strings.Contains(err.Error(), \"bad certificate\") && !strings.Contains(err.Error(), \"certificate required\") {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif resp.StatusCode != http.StatusSwitchingProtocols {\n\t\t\t\tt.Fatalf(\"Expected status %v, got %v\", http.StatusSwitchingProtocols, resp.StatusCode)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc testCreateAllowedConnectionTypes(list []string) map[string]struct{} {\n\tif len(list) == 0 {\n\t\treturn nil\n\t}\n\tm := make(map[string]struct{}, len(list))\n\tfor _, l := range list {\n\t\tm[l] = struct{}{}\n\t}\n\treturn m\n}\n\nfunc TestWSTLSVerifyAndMap(t *testing.T) {\n\taccName := \"MyAccount\"\n\tacc := NewAccount(accName)\n\tcertUserName := \"CN=example.com,OU=NATS.io\"\n\tusers := []*User{{Username: certUserName, Account: acc}}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tfiltering bool\n\t\tprovideCert bool\n\t}{\n\t\t{\"no filtering, client provides cert\", false, true},\n\t\t{\"no filtering, client does not provide cert\", false, false},\n\t\t{\"filtering, client provides cert\", true, true},\n\t\t{\"filtering, client does not provide cert\", true, false},\n\t\t{\"no users override, client provides cert\", false, true},\n\t\t{\"no users override, client does not provide cert\", false, false},\n\t\t{\"users override, client provides cert\", true, true},\n\t\t{\"users override, client does not provide cert\", true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testWSOptions()\n\t\t\to.Accounts = []*Account{acc}\n\t\t\to.Users = users\n\t\t\tif test.filtering {\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket})\n\t\t\t}\n\t\t\ttc := &TLSConfigOpts{\n\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/server.pem\",\n\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/server-key.pem\",\n\t\t\t\tCaFile: \"../test/configs/certs/tlsauth/ca.pem\",\n\t\t\t\tVerify: true,\n\t\t\t}\n\t\t\ttlsc, err := GenTLSConfig(tc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating tls config: %v\", err)\n\t\t\t}\n\t\t\to.Websocket.TLSConfig = tlsc\n\t\t\to.Websocket.TLSMap = true\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\taddr := net.JoinHostPort(o.Websocket.Host, fmt.Sprintf(\"%d\", o.Websocket.Port))\n\t\t\twsc, err := net.Dial(\"tcp\", addr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t\t\t}\n\t\t\tdefer wsc.Close()\n\t\t\ttlscc := &tls.Config{}\n\t\t\tif test.provideCert {\n\t\t\t\ttc := &TLSConfigOpts{\n\t\t\t\t\tCertFile: \"../test/configs/certs/tlsauth/client.pem\",\n\t\t\t\t\tKeyFile: \"../test/configs/certs/tlsauth/client-key.pem\",\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\ttlscc, err = GenTLSConfig(tc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error generating tls config: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\ttlscc.InsecureSkipVerify = true\n\t\t\twsc = tls.Client(wsc, tlscc)\n\t\t\tif err := wsc.(*tls.Conn).Handshake(); err != nil {\n\t\t\t\tt.Fatalf(\"Error during handshake: %v\", err)\n\t\t\t}\n\t\t\treq := testWSCreateValidReq()\n\t\t\treq.URL, _ = url.Parse(\"wss://\" + addr)\n\t\t\tif err := req.Write(wsc); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t\t\t}\n\t\t\tbr := bufio.NewReader(wsc)\n\t\t\tresp, err := http.ReadResponse(br, req)\n\t\t\tif resp != nil {\n\t\t\t\tresp.Body.Close()\n\t\t\t}\n\t\t\tif !test.provideCert {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatal(\"Expected error, did not get one\")\n\t\t\t\t} else if !strings.Contains(err.Error(), \"bad certificate\") && !strings.Contains(err.Error(), \"certificate required\") {\n\t\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif resp.StatusCode != http.StatusSwitchingProtocols {\n\t\t\t\tt.Fatalf(\"Expected status %v, got %v\", http.StatusSwitchingProtocols, resp.StatusCode)\n\t\t\t}\n\t\t\t// Wait for the INFO\n\t\t\tl := testWSReadFrame(t, br)\n\t\t\tif !bytes.HasPrefix(l, []byte(\"INFO {\")) {\n\t\t\t\tt.Fatalf(\"Expected INFO, got %s\", l)\n\t\t\t}\n\t\t\tvar info serverInfo\n\t\t\tif err := json.Unmarshal(l[5:], &info); err != nil {\n\t\t\t\tt.Fatalf(\"Unable to unmarshal info: %v\", err)\n\t\t\t}\n\t\t\t// Send CONNECT and PING\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1}\\r\\nPING\\r\\n\"))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\t// Wait for the PONG\n\t\t\tif msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected PONG, got %s\", msg)\n\t\t\t}\n\n\t\t\tvar uname string\n\t\t\tvar accname string\n\t\t\tc := s.getClient(info.CID)\n\t\t\tif c != nil {\n\t\t\t\tc.mu.Lock()\n\t\t\t\tuname = c.opts.Username\n\t\t\t\tif c.acc != nil {\n\t\t\t\t\taccname = c.acc.GetName()\n\t\t\t\t}\n\t\t\t\tc.mu.Unlock()\n\t\t\t}\n\t\t\tif uname != certUserName {\n\t\t\t\tt.Fatalf(\"Expected username %q, got %q\", certUserName, uname)\n\t\t\t}\n\t\t\tif accname != accName {\n\t\t\t\tt.Fatalf(\"Expected account %q, got %v\", accName, accname)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSHandshakeTimeout(t *testing.T) {\n\to := testWSOptions()\n\to.Websocket.HandshakeTimeout = time.Millisecond\n\ttc := &TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server.pem\",\n\t\tKeyFile: \"./configs/certs/key.pem\",\n\t}\n\to.Websocket.TLSConfig, _ = GenTLSConfig(tc)\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlogger := &captureErrorLogger{errCh: make(chan string, 1)}\n\ts.SetLogger(logger, false, false)\n\n\taddr := net.JoinHostPort(o.Websocket.Host, fmt.Sprintf(\"%d\", o.Websocket.Port))\n\twsc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t}\n\tdefer wsc.Close()\n\n\t// Delay the handshake\n\twsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true})\n\ttime.Sleep(20 * time.Millisecond)\n\t// We expect error since the server should have cut us off\n\tif err := wsc.(*tls.Conn).Handshake(); err == nil {\n\t\tt.Fatal(\"Expected error during handshake\")\n\t}\n\n\t// Check that server logs error\n\tselect {\n\tcase e := <-logger.errCh:\n\t\t// Check that log starts with \"websocket: \"\n\t\tif !strings.HasPrefix(e, \"websocket: \") {\n\t\t\tt.Fatalf(\"Wrong log line start: %s\", e)\n\t\t}\n\t\tif !strings.Contains(e, \"timeout\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Should have timed-out\")\n\t}\n}\n\nfunc TestWSServerReportUpgradeFailure(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tlogger := &captureErrorLogger{errCh: make(chan string, 1)}\n\ts.SetLogger(logger, false, false)\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.Websocket.Port)\n\treq := testWSCreateValidReq()\n\treq.URL, _ = url.Parse(\"wss://\" + addr)\n\n\twsc, err := net.Dial(\"tcp\", addr)\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating ws connection: %v\", err)\n\t}\n\tdefer wsc.Close()\n\twsc = tls.Client(wsc, &tls.Config{InsecureSkipVerify: true})\n\tif err := wsc.(*tls.Conn).Handshake(); err != nil {\n\t\tt.Fatalf(\"Error during handshake: %v\", err)\n\t}\n\t// Remove a required field from the request to have it fail\n\treq.Header.Del(\"Connection\")\n\t// Send the request\n\tif err := req.Write(wsc); err != nil {\n\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t}\n\tbr := bufio.NewReader(wsc)\n\tresp, err := http.ReadResponse(br, req)\n\tif err != nil {\n\t\tt.Fatalf(\"Error reading response: %v\", err)\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != http.StatusBadRequest {\n\t\tt.Fatalf(\"Expected status %v, got %v\", http.StatusBadRequest, resp.StatusCode)\n\t}\n\n\t// Check that server logs error\n\tselect {\n\tcase e := <-logger.errCh:\n\t\tif !strings.Contains(e, \"invalid value for header 'Connection'\") {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", e)\n\t\t}\n\t\t// The client IP's local should be printed as a remote from server perspective.\n\t\tclientIP := wsc.LocalAddr().String()\n\t\tif !strings.HasPrefix(e, clientIP) {\n\t\t\tt.Fatalf(\"IP should have been logged, it was not: %v\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Should have timed-out\")\n\t}\n}\n\nfunc TestWSCloseMsgSendOnConnectionClose(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\twsc, br := testWSCreateClient(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\tdefer wsc.Close()\n\n\tcheckClientsCount(t, s, 1)\n\tvar c *client\n\ts.mu.Lock()\n\tfor _, cli := range s.clients {\n\t\tc = cli\n\t\tbreak\n\t}\n\ts.mu.Unlock()\n\n\tc.closeConnection(ProtocolViolation)\n\tmsg := testWSReadFrame(t, br)\n\tif len(msg) < 2 {\n\t\tt.Fatalf(\"Should have 2 bytes to represent the status, got %v\", msg)\n\t}\n\tif sc := int(binary.BigEndian.Uint16(msg[:2])); sc != wsCloseStatusProtocolError {\n\t\tt.Fatalf(\"Expected status to be %v, got %v\", wsCloseStatusProtocolError, sc)\n\t}\n\texpectedPayload := ProtocolViolation.String()\n\tif p := string(msg[2:]); p != expectedPayload {\n\t\tt.Fatalf(\"Expected payload to be %q, got %q\", expectedPayload, p)\n\t}\n}\n\nfunc TestWSAdvertise(t *testing.T) {\n\to := testWSOptions()\n\to.Cluster.Port = 0\n\to.HTTPPort = 0\n\to.Websocket.Advertise = \"xxx:host:yyy\"\n\ts, err := NewServer(o)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer s.Shutdown()\n\tl := &captureFatalLogger{fatalCh: make(chan string, 1)}\n\ts.SetLogger(l, false, false)\n\ts.Start()\n\tselect {\n\tcase e := <-l.fatalCh:\n\t\tif !strings.Contains(e, \"Unable to get websocket connect URLs\") {\n\t\t\tt.Fatalf(\"Unexpected error: %q\", e)\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"Should have failed to start\")\n\t}\n\ts.Shutdown()\n\n\to1 := testWSOptions()\n\to1.Websocket.Advertise = \"host1:1234\"\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\twsc, br := testWSCreateClient(t, false, false, o1.Websocket.Host, o1.Websocket.Port)\n\tdefer wsc.Close()\n\n\to2 := testWSOptions()\n\to2.Websocket.Advertise = \"host2:5678\"\n\to2.Routes = RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", o1.Cluster.Host, o1.Cluster.Port))\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckInfo := func(expected []string) {\n\t\tt.Helper()\n\t\tinfob := testWSReadFrame(t, br)\n\t\tinfo := &Info{}\n\t\tjson.Unmarshal(infob[5:], info)\n\t\tif n := len(info.ClientConnectURLs); n != len(expected) {\n\t\t\tt.Fatalf(\"Unexpected info: %+v\", info)\n\t\t}\n\t\tgood := 0\n\t\tfor _, u := range info.ClientConnectURLs {\n\t\t\tfor _, eu := range expected {\n\t\t\t\tif u == eu {\n\t\t\t\t\tgood++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif good != len(expected) {\n\t\t\tt.Fatalf(\"Unexpected connect urls: %q\", info.ClientConnectURLs)\n\t\t}\n\t}\n\tcheckInfo([]string{\"host1:1234\", \"host2:5678\"})\n\n\t// Now shutdown s2 and expect another INFO\n\ts2.Shutdown()\n\tcheckInfo([]string{\"host1:1234\"})\n\n\t// Restart with another advertise and check that it gets updated\n\to2.Websocket.Advertise = \"host3:9012\"\n\ts2 = RunServer(o2)\n\tdefer s2.Shutdown()\n\tcheckInfo([]string{\"host1:1234\", \"host3:9012\"})\n}\n\nfunc TestWSFrameOutbound(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmaskingWrite bool\n\t}{\n\t\t{\"no write masking\", false},\n\t\t{\"write masking\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, _, _ := testWSSetupForRead()\n\t\t\tc.ws.maskwrite = test.maskingWrite\n\n\t\t\tgetKey := func(buf []byte) []byte {\n\t\t\t\treturn buf[len(buf)-4:]\n\t\t\t}\n\n\t\t\tvar bufs net.Buffers\n\t\t\tbufs = append(bufs, []byte(\"this \"))\n\t\t\tbufs = append(bufs, []byte(\"is \"))\n\t\t\tbufs = append(bufs, []byte(\"a \"))\n\t\t\tbufs = append(bufs, []byte(\"set \"))\n\t\t\tbufs = append(bufs, []byte(\"of \"))\n\t\t\tbufs = append(bufs, []byte(\"buffers\"))\n\t\t\ten := 2\n\t\t\tfor _, b := range bufs {\n\t\t\t\ten += len(b)\n\t\t\t}\n\t\t\tif test.maskingWrite {\n\t\t\t\ten += 4\n\t\t\t}\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, n := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif n != int64(en) {\n\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", en, n)\n\t\t\t}\n\t\t\tif eb := 1 + len(bufs); eb != len(res) {\n\t\t\t\tt.Fatalf(\"Expected %v buffers, got %v\", eb, len(res))\n\t\t\t}\n\t\t\tvar ob []byte\n\t\t\tfor i := 1; i < len(res); i++ {\n\t\t\t\tob = append(ob, res[i]...)\n\t\t\t}\n\t\t\tif test.maskingWrite {\n\t\t\t\twsMaskBuf(getKey(res[0]), ob)\n\t\t\t}\n\t\t\tif !bytes.Equal(ob, []byte(\"this is a set of buffers\")) {\n\t\t\t\tt.Fatalf(\"Unexpected outbound: %q\", ob)\n\t\t\t}\n\n\t\t\tbufs = nil\n\t\t\tc.out.pb = 0\n\t\t\tc.ws.fs = 0\n\t\t\tc.ws.frames = nil\n\t\t\tc.ws.browser = true\n\t\t\tbufs = append(bufs, []byte(\"some smaller \"))\n\t\t\tbufs = append(bufs, []byte(\"buffers\"))\n\t\t\tbufs = append(bufs, make([]byte, wsFrameSizeForBrowsers+10))\n\t\t\tbufs = append(bufs, []byte(\"then some more\"))\n\t\t\ten = 2 + len(bufs[0]) + len(bufs[1])\n\t\t\ten += 4 + len(bufs[2]) - 10\n\t\t\ten += 2 + len(bufs[3]) + 10\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, n = c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif test.maskingWrite {\n\t\t\t\ten += 3 * 4\n\t\t\t}\n\t\t\tif n != int64(en) {\n\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", en, n)\n\t\t\t}\n\t\t\tif len(res) != 8 {\n\t\t\t\tt.Fatalf(\"Unexpected number of outbound buffers: %v\", len(res))\n\t\t\t}\n\t\t\tif len(res[4]) != wsFrameSizeForBrowsers {\n\t\t\t\tt.Fatalf(\"Big frame should have been limited to %v, got %v\", wsFrameSizeForBrowsers, len(res[4]))\n\t\t\t}\n\t\t\tif len(res[6]) != 10 {\n\t\t\t\tt.Fatalf(\"Frame 6 should have the partial of 10 bytes, got %v\", len(res[6]))\n\t\t\t}\n\t\t\tif test.maskingWrite {\n\t\t\t\tb := &bytes.Buffer{}\n\t\t\t\tkey := getKey(res[0])\n\t\t\t\tb.Write(res[1])\n\t\t\t\tb.Write(res[2])\n\t\t\t\tud := b.Bytes()\n\t\t\t\twsMaskBuf(key, ud)\n\t\t\t\tif string(ud) != \"some smaller buffers\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected result: %q\", ud)\n\t\t\t\t}\n\n\t\t\t\tb.Reset()\n\t\t\t\tkey = getKey(res[3])\n\t\t\t\tb.Write(res[4])\n\t\t\t\tud = b.Bytes()\n\t\t\t\twsMaskBuf(key, ud)\n\t\t\t\tfor i := 0; i < len(ud); i++ {\n\t\t\t\t\tif ud[i] != 0 {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected result: %v\", ud)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tb.Reset()\n\t\t\t\tkey = getKey(res[5])\n\t\t\t\tb.Write(res[6])\n\t\t\t\tb.Write(res[7])\n\t\t\t\tud = b.Bytes()\n\t\t\t\twsMaskBuf(key, ud)\n\t\t\t\tfor i := 0; i < len(ud[:10]); i++ {\n\t\t\t\t\tif ud[i] != 0 {\n\t\t\t\t\t\tt.Fatalf(\"Unexpected result: %v\", ud[:10])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif string(ud[10:]) != \"then some more\" {\n\t\t\t\t\tt.Fatalf(\"Unexpected result: %q\", ud[10:])\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tbufs = nil\n\t\t\tc.out.pb = 0\n\t\t\tc.ws.fs = 0\n\t\t\tc.ws.frames = nil\n\t\t\tc.ws.browser = true\n\t\t\tbufs = append(bufs, []byte(\"some smaller \"))\n\t\t\tbufs = append(bufs, []byte(\"buffers\"))\n\t\t\t// Have one of the exact max size\n\t\t\tbufs = append(bufs, make([]byte, wsFrameSizeForBrowsers))\n\t\t\tbufs = append(bufs, []byte(\"then some more\"))\n\t\t\ten = 2 + len(bufs[0]) + len(bufs[1])\n\t\t\ten += 4 + len(bufs[2])\n\t\t\ten += 2 + len(bufs[3])\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, n = c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif test.maskingWrite {\n\t\t\t\ten += 3 * 4\n\t\t\t}\n\t\t\tif n != int64(en) {\n\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", en, n)\n\t\t\t}\n\t\t\tif len(res) != 7 {\n\t\t\t\tt.Fatalf(\"Unexpected number of outbound buffers: %v\", len(res))\n\t\t\t}\n\t\t\tif len(res[4]) != wsFrameSizeForBrowsers {\n\t\t\t\tt.Fatalf(\"Big frame should have been limited to %v, got %v\", wsFrameSizeForBrowsers, len(res[4]))\n\t\t\t}\n\t\t\tif test.maskingWrite {\n\t\t\t\tkey := getKey(res[5])\n\t\t\t\twsMaskBuf(key, res[6])\n\t\t\t}\n\t\t\tif string(res[6]) != \"then some more\" {\n\t\t\t\tt.Fatalf(\"Frame 6 incorrect: %q\", res[6])\n\t\t\t}\n\n\t\t\tbufs = nil\n\t\t\tc.out.pb = 0\n\t\t\tc.ws.fs = 0\n\t\t\tc.ws.frames = nil\n\t\t\tc.ws.browser = true\n\t\t\tbufs = append(bufs, []byte(\"some smaller \"))\n\t\t\tbufs = append(bufs, []byte(\"buffers\"))\n\t\t\t// Have one of the exact max size, and last in the list\n\t\t\tbufs = append(bufs, make([]byte, wsFrameSizeForBrowsers))\n\t\t\ten = 2 + len(bufs[0]) + len(bufs[1])\n\t\t\ten += 4 + len(bufs[2])\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, n = c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif test.maskingWrite {\n\t\t\t\ten += 2 * 4\n\t\t\t}\n\t\t\tif n != int64(en) {\n\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", en, n)\n\t\t\t}\n\t\t\tif len(res) != 5 {\n\t\t\t\tt.Fatalf(\"Unexpected number of outbound buffers: %v\", len(res))\n\t\t\t}\n\t\t\tif len(res[4]) != wsFrameSizeForBrowsers {\n\t\t\t\tt.Fatalf(\"Big frame should have been limited to %v, got %v\", wsFrameSizeForBrowsers, len(res[4]))\n\t\t\t}\n\n\t\t\tbufs = nil\n\t\t\tc.out.pb = 0\n\t\t\tc.ws.fs = 0\n\t\t\tc.ws.frames = nil\n\t\t\tc.ws.browser = true\n\t\t\tbufs = append(bufs, []byte(\"some smaller buffer\"))\n\t\t\tbufs = append(bufs, make([]byte, wsFrameSizeForBrowsers-5))\n\t\t\tbufs = append(bufs, []byte(\"then some more\"))\n\t\t\ten = 2 + len(bufs[0])\n\t\t\ten += 4 + len(bufs[1])\n\t\t\ten += 2 + len(bufs[2])\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, n = c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif test.maskingWrite {\n\t\t\t\ten += 3 * 4\n\t\t\t}\n\t\t\tif n != int64(en) {\n\t\t\t\tt.Fatalf(\"Expected size to be %v, got %v\", en, n)\n\t\t\t}\n\t\t\tif len(res) != 6 {\n\t\t\t\tt.Fatalf(\"Unexpected number of outbound buffers: %v\", len(res))\n\t\t\t}\n\t\t\tif len(res[3]) != wsFrameSizeForBrowsers-5 {\n\t\t\t\tt.Fatalf(\"Big frame should have been limited to %v, got %v\", wsFrameSizeForBrowsers, len(res[4]))\n\t\t\t}\n\t\t\tif test.maskingWrite {\n\t\t\t\tkey := getKey(res[4])\n\t\t\t\twsMaskBuf(key, res[5])\n\t\t\t}\n\t\t\tif string(res[5]) != \"then some more\" {\n\t\t\t\tt.Fatalf(\"Frame 6 incorrect %q\", res[5])\n\t\t\t}\n\n\t\t\tbufs = nil\n\t\t\tc.out.pb = 0\n\t\t\tc.ws.fs = 0\n\t\t\tc.ws.frames = nil\n\t\t\tc.ws.browser = true\n\t\t\tbufs = append(bufs, make([]byte, wsFrameSizeForBrowsers+100))\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = bufs\n\t\t\tres, _ = c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\t\t\tif len(res) != 4 {\n\t\t\t\tt.Fatalf(\"Unexpected number of frames: %v\", len(res))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSWebrowserClient(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\twsc, br := testWSCreateClient(t, false, true, o.Websocket.Host, o.Websocket.Port)\n\tdefer wsc.Close()\n\n\tcheckClientsCount(t, s, 1)\n\tvar c *client\n\ts.mu.Lock()\n\tfor _, cli := range s.clients {\n\t\tc = cli\n\t\tbreak\n\t}\n\ts.mu.Unlock()\n\n\tproto := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"SUB foo 1\\r\\nPING\\r\\n\"))\n\twsc.Write(proto)\n\tif res := testWSReadFrame(t, br); !bytes.Equal(res, []byte(pongProto)) {\n\t\tt.Fatalf(\"Expected PONG back\")\n\t}\n\n\tc.mu.Lock()\n\tok := c.isWebsocket() && c.ws.browser == true\n\tc.mu.Unlock()\n\tif !ok {\n\t\tt.Fatalf(\"Client is not marked as webrowser client\")\n\t}\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\t// Send a big message and check that it is received in smaller frames\n\tpsize := 204813\n\tnc.Publish(\"foo\", make([]byte, psize))\n\tnc.Flush()\n\n\trsize := psize + len(fmt.Sprintf(\"MSG foo %d\\r\\n\\r\\n\", psize))\n\tnframes := 0\n\tfor total := 0; total < rsize; nframes++ {\n\t\tres := testWSReadFrame(t, br)\n\t\ttotal += len(res)\n\t}\n\tif expected := psize / wsFrameSizeForBrowsers; expected > nframes {\n\t\tt.Fatalf(\"Expected %v frames, got %v\", expected, nframes)\n\t}\n}\n\ntype testWSWrappedConn struct {\n\tnet.Conn\n\tmu sync.RWMutex\n\tbuf *bytes.Buffer\n\tpartial bool\n}\n\nfunc (wc *testWSWrappedConn) Write(p []byte) (int, error) {\n\twc.mu.Lock()\n\tdefer wc.mu.Unlock()\n\tvar err error\n\tn := len(p)\n\tif wc.partial && n > 10 {\n\t\tn = 10\n\t\terr = io.ErrShortWrite\n\t}\n\tp = p[:n]\n\twc.buf.Write(p)\n\twc.Conn.Write(p)\n\treturn n, err\n}\n\nfunc TestWSCompressionBasic(t *testing.T) {\n\tpayload := \"This is the content of a message that will be compresseddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd.\"\n\tmsgProto := fmt.Sprintf(\"MSG foo 1 %d\\r\\n%s\\r\\n\", len(payload), payload)\n\tcbuf := &bytes.Buffer{}\n\tcompressor, err := flate.NewWriter(cbuf, flate.BestSpeed)\n\trequire_NoError(t, err)\n\tcompressor.Write([]byte(msgProto))\n\tcompressor.Flush()\n\tcompressed := cbuf.Bytes()\n\t// The last 4 bytes are dropped\n\tcompressed = compressed[:len(compressed)-4]\n\n\to := testWSOptions()\n\to.Websocket.Compression = true\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tc, br := testWSCreateClient(t, true, false, o.Websocket.Host, o.Websocket.Port)\n\tdefer c.Close()\n\n\tproto := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, []byte(\"SUB foo 1\\r\\nPING\\r\\n\"))\n\tc.Write(proto)\n\tl := testWSReadFrame(t, br)\n\tif !bytes.Equal(l, []byte(pongProto)) {\n\t\tt.Fatalf(\"Expected PONG, got %q\", l)\n\t}\n\n\tvar wc *testWSWrappedConn\n\ts.mu.RLock()\n\tfor _, c := range s.clients {\n\t\tc.mu.Lock()\n\t\twc = &testWSWrappedConn{Conn: c.nc, buf: &bytes.Buffer{}}\n\t\tc.nc = wc\n\t\tc.mu.Unlock()\n\t}\n\ts.mu.RUnlock()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\tnatsPub(t, nc, \"foo\", []byte(payload))\n\n\tres := &bytes.Buffer{}\n\tfor total := 0; total < len(msgProto); {\n\t\tl := testWSReadFrame(t, br)\n\t\tn, _ := res.Write(l)\n\t\ttotal += n\n\t}\n\tif !bytes.Equal([]byte(msgProto), res.Bytes()) {\n\t\tt.Fatalf(\"Unexpected result: %q\", res)\n\t}\n\n\t// Now check the wrapped connection buffer to check that data was actually compressed.\n\twc.mu.RLock()\n\tres = wc.buf\n\twc.mu.RUnlock()\n\tif bytes.Contains(res.Bytes(), []byte(payload)) {\n\t\tt.Fatalf(\"Looks like frame was not compressed: %q\", res.Bytes())\n\t}\n\theader := res.Bytes()[:2]\n\tbody := res.Bytes()[2:]\n\texpectedB0 := byte(wsBinaryMessage) | wsFinalBit | wsRsv1Bit\n\texpectedPS := len(compressed)\n\texpectedB1 := byte(expectedPS)\n\n\tif b := header[0]; b != expectedB0 {\n\t\tt.Fatalf(\"Expected first byte to be %v, got %v\", expectedB0, b)\n\t}\n\tif b := header[1]; b != expectedB1 {\n\t\tt.Fatalf(\"Expected second byte to be %v, got %v\", expectedB1, b)\n\t}\n\tif len(body) != expectedPS {\n\t\tt.Fatalf(\"Expected payload length to be %v, got %v\", expectedPS, len(body))\n\t}\n\tif !bytes.Equal(body, compressed) {\n\t\tt.Fatalf(\"Unexpected compress body: %q\", body)\n\t}\n\n\twc.mu.Lock()\n\twc.buf.Reset()\n\twc.mu.Unlock()\n\n\tpayload = \"small\"\n\tnatsPub(t, nc, \"foo\", []byte(payload))\n\tmsgProto = fmt.Sprintf(\"MSG foo 1 %d\\r\\n%s\\r\\n\", len(payload), payload)\n\tres = &bytes.Buffer{}\n\tfor total := 0; total < len(msgProto); {\n\t\tl := testWSReadFrame(t, br)\n\t\tn, _ := res.Write(l)\n\t\ttotal += n\n\t}\n\tif !bytes.Equal([]byte(msgProto), res.Bytes()) {\n\t\tt.Fatalf(\"Unexpected result: %q\", res)\n\t}\n\twc.mu.RLock()\n\tres = wc.buf\n\twc.mu.RUnlock()\n\tif !bytes.HasSuffix(res.Bytes(), []byte(msgProto)) {\n\t\tt.Fatalf(\"Looks like frame was compressed: %q\", res.Bytes())\n\t}\n}\n\nfunc TestWSCompressionWithPartialWrite(t *testing.T) {\n\tpayload := \"This is the content of a message that will be compresseddddddddddddddddddddd.\"\n\tmsgProto := fmt.Sprintf(\"MSG foo 1 %d\\r\\n%s\\r\\n\", len(payload), payload)\n\n\to := testWSOptions()\n\to.Websocket.Compression = true\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tc, br := testWSCreateClient(t, true, false, o.Websocket.Host, o.Websocket.Port)\n\tdefer c.Close()\n\n\tproto := testWSCreateClientMsg(wsBinaryMessage, 1, true, true, []byte(\"SUB foo 1\\r\\nPING\\r\\n\"))\n\tc.Write(proto)\n\tl := testWSReadFrame(t, br)\n\tif !bytes.Equal(l, []byte(pongProto)) {\n\t\tt.Fatalf(\"Expected PONG, got %q\", l)\n\t}\n\n\tpingPayload := []byte(\"my ping\")\n\tpingFromWSClient := testWSCreateClientMsg(wsPingMessage, 1, true, false, pingPayload)\n\n\tvar wc *testWSWrappedConn\n\tvar ws *client\n\ts.mu.Lock()\n\tfor _, c := range s.clients {\n\t\tws = c\n\t\tc.mu.Lock()\n\t\twc = &testWSWrappedConn{\n\t\t\tConn: c.nc,\n\t\t\tbuf: &bytes.Buffer{},\n\t\t}\n\t\tc.nc = wc\n\t\tc.mu.Unlock()\n\t\tbreak\n\t}\n\ts.mu.Unlock()\n\n\twc.mu.Lock()\n\twc.partial = true\n\twc.mu.Unlock()\n\n\tnc := natsConnect(t, s.ClientURL())\n\tdefer nc.Close()\n\n\texpected := &bytes.Buffer{}\n\tfor i := 0; i < 10; i++ {\n\t\tif i > 0 {\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t}\n\t\texpected.Write([]byte(msgProto))\n\t\tnatsPub(t, nc, \"foo\", []byte(payload))\n\t\tif i == 1 {\n\t\t\tc.Write(pingFromWSClient)\n\t\t}\n\t}\n\n\tvar gotPingResponse bool\n\tres := &bytes.Buffer{}\n\tfor total := 0; total < 10*len(msgProto); {\n\t\tl := testWSReadFrame(t, br)\n\t\tif bytes.Equal(l, pingPayload) {\n\t\t\tgotPingResponse = true\n\t\t} else {\n\t\t\tn, _ := res.Write(l)\n\t\t\ttotal += n\n\t\t}\n\t}\n\tif !bytes.Equal(expected.Bytes(), res.Bytes()) {\n\t\tt.Fatalf(\"Unexpected result: %q\", res)\n\t}\n\tif !gotPingResponse {\n\t\tt.Fatal(\"Did not get the ping response\")\n\t}\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tws.mu.Lock()\n\t\tpb := ws.out.pb\n\t\twf := ws.ws.frames\n\t\tfs := ws.ws.fs\n\t\tws.mu.Unlock()\n\t\tif pb != 0 || len(wf) != 0 || fs != 0 {\n\t\t\treturn fmt.Errorf(\"Expected pb, wf and fs to be 0, got %v, %v, %v\", pb, wf, fs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestWSCompressionFrameSizeLimit(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmaskWrite bool\n\t\tnoLimit bool\n\t}{\n\t\t{\"no write masking\", false, false},\n\t\t{\"write masking\", true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := testWSOptions()\n\t\t\topts.MaxPending = MAX_PENDING_SIZE\n\t\t\ts := &Server{opts: opts}\n\t\t\tc := &client{srv: s, ws: &websocket{compress: true, browser: true, nocompfrag: test.noLimit, maskwrite: test.maskWrite}}\n\t\t\tc.initClient()\n\n\t\t\tuncompressedPayload := make([]byte, 2*wsFrameSizeForBrowsers)\n\t\t\tfor i := 0; i < len(uncompressedPayload); i++ {\n\t\t\t\tuncompressedPayload[i] = byte(rand.Intn(256))\n\t\t\t}\n\n\t\t\tc.mu.Lock()\n\t\t\tc.out.nb = append(net.Buffers(nil), uncompressedPayload)\n\t\t\tnb, _ := c.collapsePtoNB()\n\t\t\tc.mu.Unlock()\n\n\t\t\tif test.noLimit && len(nb) != 2 {\n\t\t\t\tt.Fatalf(\"There should be only 2 buffers, the header and payload, got %v\", len(nb))\n\t\t\t}\n\n\t\t\tbb := &bytes.Buffer{}\n\t\t\tvar key []byte\n\t\t\tfor i, b := range nb {\n\t\t\t\tif !test.noLimit {\n\t\t\t\t\t// frame header buffer are always very small. The payload should not be more\n\t\t\t\t\t// than 10 bytes since that is what we passed as the limit.\n\t\t\t\t\tif len(b) > wsFrameSizeForBrowsers {\n\t\t\t\t\t\tt.Fatalf(\"Frame size too big: %v (%q)\", len(b), b)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif test.maskWrite {\n\t\t\t\t\tif i%2 == 0 {\n\t\t\t\t\t\tkey = b[len(b)-4:]\n\t\t\t\t\t} else {\n\t\t\t\t\t\twsMaskBuf(key, b)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Check frame headers for the proper formatting.\n\t\t\t\tif i%2 == 0 {\n\t\t\t\t\t// Only the first frame should have the compress bit set.\n\t\t\t\t\tif b[0]&wsRsv1Bit != 0 {\n\t\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\t\tt.Fatalf(\"Compressed bit should not be in continuation frame\")\n\t\t\t\t\t\t}\n\t\t\t\t\t} else if i == 0 {\n\t\t\t\t\t\tt.Fatalf(\"Compressed bit missing\")\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif test.noLimit {\n\t\t\t\t\t\t// Since the payload is likely not well compressed, we are expecting\n\t\t\t\t\t\t// the length to be > wsFrameSizeForBrowsers\n\t\t\t\t\t\tif len(b) <= wsFrameSizeForBrowsers {\n\t\t\t\t\t\t\tt.Fatalf(\"Expected frame to be bigger, got %v\", len(b))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// Collect the payload\n\t\t\t\t\tbb.Write(b)\n\t\t\t\t}\n\t\t\t}\n\t\t\tbuf := bb.Bytes()\n\t\t\tbuf = append(buf, 0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff)\n\t\t\tdbr := bytes.NewBuffer(buf)\n\t\t\td := flate.NewReader(dbr)\n\t\t\tuncompressed, err := io.ReadAll(d)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading frame: %v\", err)\n\t\t\t}\n\t\t\tif !bytes.Equal(uncompressed, uncompressedPayload) {\n\t\t\t\tt.Fatalf(\"Unexpected uncomressed data: %q\", uncompressed)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSCompressionPoolBufferRecycling(t *testing.T) {\n\topts := testWSOptions()\n\topts.MaxPending = MAX_PENDING_SIZE\n\ts := &Server{opts: opts}\n\tc := &client{srv: s, ws: &websocket{compress: true}}\n\tc.initClient()\n\n\t// Use a payload larger than wsCompressThreshold (64 bytes)\n\t// to trigger the compression path.\n\tpayload := make([]byte, 256)\n\tfor i := range payload {\n\t\t// Semi-random to be compressible.\n\t\tpayload[i] = byte(i % 251)\n\t}\n\n\t// Warm up: populate the pool and initialize the compressor.\n\tnbSlice := make(net.Buffers, 1)\n\tfor i := 0; i < 10; i++ {\n\t\tc.mu.Lock()\n\t\tdata := nbPoolGet(len(payload))\n\t\tdata = append(data, payload...)\n\t\tnbSlice[0] = data\n\t\tc.out.nb = nbSlice\n\t\tc.out.pb = int64(len(payload))\n\t\tc.ws.fs = 0\n\t\tbufs, _ := c.collapsePtoNB()\n\t\tfor _, buf := range bufs {\n\t\t\tnbPoolPut(buf)\n\t\t}\n\t\tc.out.nb = nil\n\t\tc.mu.Unlock()\n\t}\n\n\tallocs := testing.AllocsPerRun(500, func() {\n\t\tc.mu.Lock()\n\t\tdata := nbPoolGet(len(payload))\n\t\tdata = append(data, payload...)\n\t\tnbSlice[0] = data\n\t\tc.out.nb = nbSlice\n\t\tc.out.pb = int64(len(payload))\n\t\tc.ws.fs = 0\n\t\tbufs, _ := c.collapsePtoNB()\n\t\tfor _, buf := range bufs {\n\t\t\tnbPoolPut(buf)\n\t\t}\n\t\tc.out.nb = nil\n\t\tc.mu.Unlock()\n\t})\n\tif allocs > 2 {\n\t\tt.Fatalf(\"Too many allocs per iteration (%.1f); pool buffers are likely being leaked\", allocs)\n\t}\n}\n\nfunc TestWSBasicAuth(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\tuser string\n\t\tpass string\n\t\terr string\n\t\tcookies []string\n\t}{\n\t\t{\n\t\t\t\"top level auth, no override, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"websocket\", \"client\", \"-ERR 'Authorization Violation'\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, no override, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", \"\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, ws auth, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.Username = \"websocket\"\n\t\t\t\to.Websocket.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", \"-ERR 'Authorization Violation'\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, ws auth, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.Username = \"websocket\"\n\t\t\t\to.Websocket.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"websocket\", \"client\", \"\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, ws override, wrong u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\to.Websocket.Username = \"websocket\"\n\t\t\t\to.Websocket.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"normal\", \"client\", \"-ERR 'Authorization Violation'\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"top level auth, ws override, correct u/p\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Username = \"normal\"\n\t\t\t\to.Password = \"client\"\n\t\t\t\to.Websocket.Username = \"websocket\"\n\t\t\t\to.Websocket.Password = \"client\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"websocket\", \"client\", \"\",\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"username/password from cookies\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.UsernameCookie = \"un\"\n\t\t\t\to.Websocket.PasswordCookie = \"pw\"\n\t\t\t\to.Username = \"me\"\n\t\t\t\to.Password = \"s3cr3t!\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"\", \"\", \"\",\n\t\t\t[]string{\"un=me\", \"pw=s3cr3t!\"},\n\t\t},\n\t\t{\n\t\t\t\"bad username/ good password from cookies\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.UsernameCookie = \"un\"\n\t\t\t\to.Websocket.PasswordCookie = \"pw\"\n\t\t\t\to.Username = \"me\"\n\t\t\t\to.Password = \"s3cr3t!\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"\", \"\", \"-ERR 'Authorization Violation\",\n\t\t\t[]string{\"un=m\", \"pw=s3cr3t!\"},\n\t\t},\n\t\t{\n\t\t\t\"good username/ bad password from cookies\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.UsernameCookie = \"un\"\n\t\t\t\to.Websocket.PasswordCookie = \"pw\"\n\t\t\t\to.Username = \"me\"\n\t\t\t\to.Password = \"s3cr3t!\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"\", \"\", \"-ERR 'Authorization Violation\",\n\t\t\t[]string{\"un=me\", \"pw=hi!\"},\n\t\t},\n\t\t{\n\t\t\t\"token from cookie\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.TokenCookie = \"tok\"\n\t\t\t\to.Authorization = \"l3tm31n!\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"\", \"\", \"\",\n\t\t\t[]string{\"tok=l3tm31n!\"},\n\t\t},\n\t\t{\n\t\t\t\"bad token from cookie\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.TokenCookie = \"tok\"\n\t\t\t\to.Authorization = \"l3tm31n!\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"\", \"\", \"-ERR 'Authorization Violation\",\n\t\t\t[]string{\"tok=hello!\"},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port, test.cookies...)\n\t\t\tdefer wsc.Close()\n\n\t\t\tconnectProto := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\",\n\t\t\t\ttest.user, test.pass)\n\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif test.err == \"\" && !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected to receive PONG, got %q\", msg)\n\t\t\t} else if test.err != \"\" && !bytes.HasPrefix(msg, []byte(test.err)) {\n\t\t\t\tt.Fatalf(\"Expected to receive %q, got %q\", test.err, msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSAuthTimeout(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tat float64\n\t\twat float64\n\t\terr string\n\t}{\n\t\t{\"use top-level auth timeout\", 10.0, 0.0, \"\"},\n\t\t{\"use websocket auth timeout\", 10.0, 0.05, \"-ERR 'Authentication Timeout'\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testWSOptions()\n\t\t\to.AuthTimeout = test.at\n\t\t\to.Websocket.Username = \"websocket\"\n\t\t\to.Websocket.Password = \"client\"\n\t\t\to.Websocket.AuthTimeout = test.wat\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, l := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\tvar info serverInfo\n\t\t\tjson.Unmarshal([]byte(l[5:]), &info)\n\t\t\t// Make sure that we are told that auth is required.\n\t\t\tif !info.AuthRequired {\n\t\t\t\tt.Fatalf(\"Expected auth required, was not: %q\", l)\n\t\t\t}\n\t\t\tstart := time.Now()\n\t\t\t// Wait before sending connect\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\tconnectProto := \"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"user\\\":\\\"websocket\\\",\\\"pass\\\":\\\"client\\\"}\\r\\nPING\\r\\n\"\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif test.err != \"\" && !bytes.HasPrefix(msg, []byte(test.err)) {\n\t\t\t\tt.Fatalf(\"Expected to receive %q error, got %q\", test.err, msg)\n\t\t\t} else if test.err == \"\" && !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Unexpected error: %q\", msg)\n\t\t\t}\n\t\t\tif dur := time.Since(start); dur > time.Second {\n\t\t\t\tt.Fatalf(\"Too long to get timeout error: %v\", dur)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSTokenAuth(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\ttoken string\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"top level auth, no override, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Authorization = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"badtoken\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"top level auth, no override, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Authorization = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"goodtoken\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, ws auth, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.Token = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"badtoken\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"no top level auth, ws auth, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Websocket.Token = \"goodtoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"goodtoken\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"top level auth, ws override, wrong token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Authorization = \"clienttoken\"\n\t\t\t\to.Websocket.Token = \"websockettoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"clienttoken\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"top level auth, ws override, correct token\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Authorization = \"clienttoken\"\n\t\t\t\to.Websocket.Token = \"websockettoken\"\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"websockettoken\", \"\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\tconnectProto := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"auth_token\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\",\n\t\t\t\ttest.token)\n\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif test.err == \"\" && !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected to receive PONG, got %q\", msg)\n\t\t\t} else if test.err != \"\" && !bytes.HasPrefix(msg, []byte(test.err)) {\n\t\t\t\tt.Fatalf(\"Expected to receive %q, got %q\", test.err, msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSBindToProperAccount(t *testing.T) {\n\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\taccounts {\n\t\t\ta {\n\t\t\t\tusers [\n\t\t\t\t\t{user: a, password: pwd, allowed_connection_types: [\"%s\", \"%s\"]}\n\t\t\t\t]\n\t\t\t}\n\t\t\tb {\n\t\t\t\tusers [\n\t\t\t\t\t{user: b, password: pwd}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t`, jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeWebsocket)))) // on purpose use lower case to ensure that it is converted.\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc := natsConnect(t, fmt.Sprintf(\"nats://a:pwd@127.0.0.1:%d\", o.Port))\n\tdefer nc.Close()\n\n\tsub := natsSubSync(t, nc, \"foo\")\n\n\twsc, br, _ := testNewWSClient(t, testWSClientOptions{host: o.Websocket.Host, port: o.Websocket.Port, noTLS: true})\n\t// Send CONNECT and PING\n\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false,\n\t\t[]byte(fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\", \"a\", \"pwd\")))\n\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t}\n\t// Wait for the PONG\n\tif msg := testWSReadFrame(t, br); !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\tt.Fatalf(\"Expected PONG, got %s\", msg)\n\t}\n\n\twsmsg = testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"PUB foo 7\\r\\nfrom ws\\r\\n\"))\n\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t}\n\n\tnatsNexMsg(t, sub, time.Second)\n}\n\nfunc TestWSUsersAuth(t *testing.T) {\n\tusers := []*User{{Username: \"user\", Password: \"pwd\"}}\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\tuser string\n\t\tpass string\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"no filtering, wrong user\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Users = users\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"wronguser\", \"pwd\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"no filtering, correct user\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Users = users\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"filering, user not allowed\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Users = users\n\t\t\t\t// Only allowed for regular clients\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"filtering, user allowed\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Users = users\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"pwd\", \"\",\n\t\t},\n\t\t{\n\t\t\t\"filtering, wrong password\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Users = users\n\t\t\t\to.Users[0].AllowedConnectionTypes = testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket})\n\t\t\t\treturn o\n\t\t\t},\n\t\t\t\"user\", \"badpassword\", \"-ERR 'Authorization Violation'\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, _ := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\tconnectProto := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\",\n\t\t\t\ttest.user, test.pass)\n\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif test.err == \"\" && !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected to receive PONG, got %q\", msg)\n\t\t\t} else if test.err != \"\" && !bytes.HasPrefix(msg, []byte(test.err)) {\n\t\t\t\tt.Fatalf(\"Expected to receive %q, got %q\", test.err, msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSNoAuthUserValidation(t *testing.T) {\n\to := testWSOptions()\n\to.Users = []*User{{Username: \"user\", Password: \"pwd\"}}\n\t// Should fail because it is not part of o.Users.\n\to.Websocket.NoAuthUser = \"notfound\"\n\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), \"not present as user\") {\n\t\tt.Fatalf(\"Expected error saying not present as user, got %v\", err)\n\t}\n\t// Set a valid no auth user for global options, but still should fail because\n\t// of o.Websocket.NoAuthUser\n\to.NoAuthUser = \"user\"\n\to.Websocket.NoAuthUser = \"notfound\"\n\tif _, err := NewServer(o); err == nil || !strings.Contains(err.Error(), \"not present as user\") {\n\t\tt.Fatalf(\"Expected error saying not present as user, got %v\", err)\n\t}\n}\n\nfunc TestWSNoAuthUser(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\toverride bool\n\t\tuseAuth bool\n\t\texpectedUser string\n\t\texpectedAcc string\n\t}{\n\t\t{\"no override, no user provided\", false, false, \"noauth\", \"normal\"},\n\t\t{\"no override, user povided\", false, true, \"user\", \"normal\"},\n\t\t{\"override, no user provided\", true, false, \"wsnoauth\", \"websocket\"},\n\t\t{\"override, user provided\", true, true, \"wsuser\", \"websocket\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := testWSOptions()\n\t\t\tnormalAcc := NewAccount(\"normal\")\n\t\t\twebsocketAcc := NewAccount(\"websocket\")\n\t\t\to.Accounts = []*Account{normalAcc, websocketAcc}\n\t\t\to.Users = []*User{\n\t\t\t\t{Username: \"noauth\", Password: \"pwd\", Account: normalAcc},\n\t\t\t\t{Username: \"user\", Password: \"pwd\", Account: normalAcc},\n\t\t\t\t{Username: \"wsnoauth\", Password: \"pwd\", Account: websocketAcc},\n\t\t\t\t{Username: \"wsuser\", Password: \"pwd\", Account: websocketAcc},\n\t\t\t}\n\t\t\to.NoAuthUser = \"noauth\"\n\t\t\tif test.override {\n\t\t\t\to.Websocket.NoAuthUser = \"wsnoauth\"\n\t\t\t}\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, l := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\tvar info serverInfo\n\t\t\tjson.Unmarshal([]byte(l[5:]), &info)\n\n\t\t\tvar connectProto string\n\t\t\tif test.useAuth {\n\t\t\t\tconnectProto = fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"pwd\\\"}\\r\\nPING\\r\\n\",\n\t\t\t\t\ttest.expectedUser)\n\t\t\t} else {\n\t\t\t\tconnectProto = \"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1}\\r\\nPING\\r\\n\"\n\t\t\t}\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Unexpected error: %q\", msg)\n\t\t\t}\n\n\t\t\tc := s.getClient(info.CID)\n\t\t\tc.mu.Lock()\n\t\t\tuname := c.opts.Username\n\t\t\taname := c.acc.GetName()\n\t\t\tc.mu.Unlock()\n\t\t\tif uname != test.expectedUser {\n\t\t\t\tt.Fatalf(\"Expected selected user to be %q, got %q\", test.expectedUser, uname)\n\t\t\t}\n\t\t\tif aname != test.expectedAcc {\n\t\t\t\tt.Fatalf(\"Expected selected account to be %q, got %q\", test.expectedAcc, aname)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSNkeyAuth(t *testing.T) {\n\tnkp, _ := nkeys.CreateUser()\n\tpub, _ := nkp.PublicKey()\n\n\twsnkp, _ := nkeys.CreateUser()\n\twspub, _ := wsnkp.PublicKey()\n\n\tbadkp, _ := nkeys.CreateUser()\n\tbadpub, _ := badkp.PublicKey()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\topts func() *Options\n\t\tnkey string\n\t\tkp nkeys.KeyPair\n\t\terr string\n\t}{\n\t\t{\n\t\t\t\"no filtering, wrong nkey\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Nkeys = []*NkeyUser{{Nkey: pub}}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\tbadpub, badkp, \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"no filtering, correct nkey\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Nkeys = []*NkeyUser{{Nkey: pub}}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\tpub, nkp, \"\",\n\t\t},\n\t\t{\n\t\t\t\"filtering, nkey not allowed\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Nkeys = []*NkeyUser{\n\t\t\t\t\t{\n\t\t\t\t\t\tNkey: pub,\n\t\t\t\t\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard}),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tNkey: wspub,\n\t\t\t\t\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeWebsocket}),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\tpub, nkp, \"-ERR 'Authorization Violation'\",\n\t\t},\n\t\t{\n\t\t\t\"filtering, correct nkey\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Nkeys = []*NkeyUser{\n\t\t\t\t\t{Nkey: pub},\n\t\t\t\t\t{\n\t\t\t\t\t\tNkey: wspub,\n\t\t\t\t\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\twspub, wsnkp, \"\",\n\t\t},\n\t\t{\n\t\t\t\"filtering, wrong nkey\",\n\t\t\tfunc() *Options {\n\t\t\t\to := testWSOptions()\n\t\t\t\to.Nkeys = []*NkeyUser{\n\t\t\t\t\t{\n\t\t\t\t\t\tNkey: wspub,\n\t\t\t\t\t\tAllowedConnectionTypes: testCreateAllowedConnectionTypes([]string{jwt.ConnectionTypeStandard, jwt.ConnectionTypeWebsocket}),\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\treturn o\n\t\t\t},\n\t\t\tbadpub, badkp, \"-ERR 'Authorization Violation'\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\to := test.opts()\n\t\t\ts := RunServer(o)\n\t\t\tdefer s.Shutdown()\n\n\t\t\twsc, br, infoMsg := testWSCreateClientGetInfo(t, false, false, o.Websocket.Host, o.Websocket.Port)\n\t\t\tdefer wsc.Close()\n\n\t\t\t// Sign Nonce\n\t\t\tvar info nonceInfo\n\t\t\tjson.Unmarshal([]byte(infoMsg[5:]), &info)\n\t\t\tsigraw, _ := test.kp.Sign([]byte(info.Nonce))\n\t\t\tsig := base64.RawURLEncoding.EncodeToString(sigraw)\n\n\t\t\tconnectProto := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1,\\\"nkey\\\":\\\"%s\\\",\\\"sig\\\":\\\"%s\\\"}\\r\\nPING\\r\\n\", test.nkey, sig)\n\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(connectProto))\n\t\t\tif _, err := wsc.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\tmsg := testWSReadFrame(t, br)\n\t\t\tif test.err == \"\" && !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected to receive PONG, got %q\", msg)\n\t\t\t} else if test.err != \"\" && !bytes.HasPrefix(msg, []byte(test.err)) {\n\t\t\t\tt.Fatalf(\"Expected to receive %q, got %q\", test.err, msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWSSetHeaderServer(t *testing.T) {\n\to := testWSOptions()\n\to.Websocket.Headers = map[string]string{\n\t\t\"X-Custom-Header\": \"custom-value\",\n\t}\n\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\topts := testWSClientOptions{\n\t\thost: o.Websocket.Host,\n\t\tport: o.Websocket.Port,\n\t\textraResponseHeaders: o.Websocket.Headers,\n\t}\n\n\tc, _, _ := testNewWSClient(t, opts)\n\tdefer c.Close()\n}\n\nfunc TestWSJWTWithAllowedConnectionTypes(t *testing.T) {\n\to := testWSOptions()\n\tsetupAddTrusted(o)\n\ts := RunServer(o)\n\tbuildMemAccResolver(s)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconnectionTypes []string\n\t\texpectedAnswer string\n\t}{\n\t\t{\"not allowed\", []string{jwt.ConnectionTypeStandard}, \"-ERR\"},\n\t\t{\"allowed\", []string{jwt.ConnectionTypeStandard, strings.ToLower(jwt.ConnectionTypeWebsocket)}, \"+OK\"},\n\t\t{\"allowed with unknown\", []string{jwt.ConnectionTypeWebsocket, \"SomeNewType\"}, \"+OK\"},\n\t\t{\"not allowed with unknown\", []string{\"SomeNewType\"}, \"-ERR\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tnuc := newJWTTestUserClaims()\n\t\t\tnuc.AllowedConnectionTypes = test.connectionTypes\n\t\t\tclaimOpt := testClaimsOptions{\n\t\t\t\tnuc: nuc,\n\t\t\t\texpectAnswer: test.expectedAnswer,\n\t\t\t}\n\t\t\t_, c, _, _ := testWSWithClaims(t, s, testWSClientOptions{host: o.Websocket.Host, port: o.Websocket.Port}, claimOpt)\n\t\t\tc.Close()\n\t\t})\n\t}\n}\n\nfunc TestWSJWTCookieUser(t *testing.T) {\n\tnucSigFunc := func() *jwt.UserClaims { return newJWTTestUserClaims() }\n\tnucBearerFunc := func() *jwt.UserClaims {\n\t\tret := newJWTTestUserClaims()\n\t\tret.BearerToken = true\n\t\treturn ret\n\t}\n\n\to := testWSOptions()\n\tsetupAddTrusted(o)\n\tsetupAddCookie(o)\n\ts := RunServer(o)\n\tbuildMemAccResolver(s)\n\tdefer s.Shutdown()\n\n\tgenJwt := func(t *testing.T, nuc *jwt.UserClaims) string {\n\t\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t\takp, _ := nkeys.CreateAccount()\n\t\tapub, _ := akp.PublicKey()\n\n\t\tnac := jwt.NewAccountClaims(apub)\n\t\tajwt, err := nac.Encode(okp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t\t}\n\n\t\tnkp, _ := nkeys.CreateUser()\n\t\tpub, _ := nkp.PublicKey()\n\t\tnuc.Subject = pub\n\t\tjwt, err := nuc.Encode(akp)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t\t}\n\t\taddAccountToMemResolver(s, apub, ajwt)\n\t\treturn jwt\n\t}\n\n\tcliOpts := testWSClientOptions{\n\t\thost: o.Websocket.Host,\n\t\tport: o.Websocket.Port,\n\t}\n\tfor _, test := range []struct {\n\t\tname string\n\t\tnuc *jwt.UserClaims\n\t\topts func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions)\n\t\texpectAnswer string\n\t}{\n\t\t{\n\t\t\tname: \"protocol auth, non-bearer key, with signature\",\n\t\t\tnuc: nucSigFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\treturn cliOpts, testClaimsOptions{nuc: claims}\n\t\t\t},\n\t\t\texpectAnswer: \"+OK\",\n\t\t},\n\t\t{\n\t\t\tname: \"protocol auth, non-bearer key, w/o required signature\",\n\t\t\tnuc: nucSigFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\treturn cliOpts, testClaimsOptions{nuc: claims, dontSign: true}\n\t\t\t},\n\t\t\texpectAnswer: \"-ERR\",\n\t\t},\n\t\t{\n\t\t\tname: \"protocol auth, bearer key, w/o signature\",\n\t\t\tnuc: nucBearerFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\treturn cliOpts, testClaimsOptions{nuc: claims, dontSign: true}\n\t\t\t},\n\t\t\texpectAnswer: \"+OK\",\n\t\t},\n\t\t{\n\t\t\tname: \"cookie auth, non-bearer key, protocol auth fail\",\n\t\t\tnuc: nucSigFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\tco := cliOpts\n\t\t\t\tco.extraHeaders = map[string][]string{}\n\t\t\t\tco.extraHeaders[\"Cookie\"] = []string{o.Websocket.JWTCookie + \"=\" + genJwt(t, claims)}\n\t\t\t\treturn co, testClaimsOptions{connectRequest: struct{}{}}\n\t\t\t},\n\t\t\texpectAnswer: \"-ERR\",\n\t\t},\n\t\t{\n\t\t\tname: \"cookie auth, bearer key, protocol auth success with implied cookie jwt\",\n\t\t\tnuc: nucBearerFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\tco := cliOpts\n\t\t\t\tco.extraHeaders = map[string][]string{}\n\t\t\t\tco.extraHeaders[\"Cookie\"] = []string{o.Websocket.JWTCookie + \"=\" + genJwt(t, claims)}\n\t\t\t\treturn co, testClaimsOptions{connectRequest: struct{}{}}\n\t\t\t},\n\t\t\texpectAnswer: \"+OK\",\n\t\t},\n\t\t{\n\t\t\tname: \"cookie auth, non-bearer key, protocol auth success via override jwt in CONNECT opts\",\n\t\t\tnuc: nucSigFunc(),\n\t\t\topts: func(t *testing.T, claims *jwt.UserClaims) (testWSClientOptions, testClaimsOptions) {\n\t\t\t\tco := cliOpts\n\t\t\t\tco.extraHeaders = map[string][]string{}\n\t\t\t\tco.extraHeaders[\"Cookie\"] = []string{o.Websocket.JWTCookie + \"=\" + genJwt(t, claims)}\n\t\t\t\treturn co, testClaimsOptions{nuc: nucBearerFunc()}\n\t\t\t},\n\t\t\texpectAnswer: \"+OK\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcliOpt, claimOpt := test.opts(t, test.nuc)\n\t\t\tclaimOpt.expectAnswer = test.expectAnswer\n\t\t\t_, c, _, _ := testWSWithClaims(t, s, cliOpt, claimOpt)\n\t\t\tc.Close()\n\t\t})\n\t}\n\ts.Shutdown()\n}\n\nfunc TestWSReloadTLSConfig(t *testing.T) {\n\ttlsBlock := `\n\t\ttls {\n\t\t\tcert_file: '%s'\n\t\t\tkey_file: '%s'\n\t\t\tca_file: '../test/configs/certs/ca.pem'\n\t\t\tverify: %v\n\t\t}\n\t`\n\ttemplate := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t%s\n\t\t\tno_tls: %v\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(template,\n\t\tfmt.Sprintf(tlsBlock,\n\t\t\t\"../test/configs/certs/server-noip.pem\",\n\t\t\t\"../test/configs/certs/server-key-noip.pem\",\n\t\t\tfalse), false)))\n\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\taddr := fmt.Sprintf(\"127.0.0.1:%d\", o.Websocket.Port)\n\n\tcheck := func(tlsConfig *tls.Config, handshakeFail bool, errTxt string) {\n\t\tt.Helper()\n\n\t\twsc, err := net.Dial(\"tcp\", addr)\n\t\trequire_NoError(t, err)\n\t\tdefer wsc.Close()\n\n\t\twsc = tls.Client(wsc, tlsConfig)\n\t\terr = wsc.(*tls.Conn).Handshake()\n\t\tif handshakeFail {\n\t\t\trequire_True(t, err != nil)\n\t\t\trequire_Contains(t, err.Error(), errTxt)\n\t\t\treturn\n\t\t}\n\t\trequire_NoError(t, err)\n\n\t\treq := testWSCreateValidReq()\n\t\treq.URL, _ = url.Parse(wsSchemePrefixTLS + \"://\" + addr)\n\t\terr = req.Write(wsc)\n\t\trequire_NoError(t, err)\n\n\t\tbr := bufio.NewReader(wsc)\n\t\tresp, err := http.ReadResponse(br, req)\n\t\tif errTxt == _EMPTY_ {\n\t\t\trequire_NoError(t, err)\n\t\t} else {\n\t\t\trequire_True(t, err != nil)\n\t\t\trequire_Contains(t, err.Error(), errTxt)\n\t\t\treturn\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tl := testWSReadFrame(t, br)\n\t\trequire_True(t, bytes.HasPrefix(l, []byte(\"INFO {\")))\n\t\tvar info Info\n\t\terr = json.Unmarshal(l[5:], &info)\n\t\trequire_NoError(t, err)\n\t\trequire_True(t, info.TLSAvailable)\n\t\trequire_True(t, info.TLSRequired)\n\t\trequire_Equal[string](t, info.Host, \"127.0.0.1\")\n\t\trequire_Equal[int](t, info.Port, o.Websocket.Port)\n\t}\n\n\ttc := &TLSConfigOpts{CaFile: \"../test/configs/certs/ca.pem\"}\n\ttlsConfig, err := GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\ttlsConfig.ServerName = \"127.0.0.1\"\n\ttlsConfig.RootCAs = tlsConfig.ClientCAs\n\ttlsConfig.ClientCAs = nil\n\n\t// Handshake should fail with error regarding SANs\n\tcheck(tlsConfig.Clone(), true, \"SAN\")\n\n\t// Replace certs with ones that allow IP.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template,\n\t\tfmt.Sprintf(tlsBlock,\n\t\t\t\"../test/configs/certs/server-cert.pem\",\n\t\t\t\"../test/configs/certs/server-key.pem\",\n\t\t\tfalse), false))\n\n\t// Connection should succeed\n\tcheck(tlsConfig.Clone(), false, _EMPTY_)\n\n\t// Udpate config to require client cert.\n\treloadUpdateConfig(t, s, conf, fmt.Sprintf(template,\n\t\tfmt.Sprintf(tlsBlock,\n\t\t\t\"../test/configs/certs/server-cert.pem\",\n\t\t\t\"../test/configs/certs/server-key.pem\",\n\t\t\ttrue), false))\n\n\t// Connection should fail saying that a tls cert is required\n\tcheck(tlsConfig.Clone(), false, \"required\")\n\n\t// Add a client cert\n\ttc = &TLSConfigOpts{\n\t\tCertFile: \"../test/configs/certs/client-cert.pem\",\n\t\tKeyFile: \"../test/configs/certs/client-key.pem\",\n\t}\n\ttlsConfig, err = GenTLSConfig(tc)\n\trequire_NoError(t, err)\n\ttlsConfig.InsecureSkipVerify = true\n\n\t// Connection should succeed\n\tcheck(tlsConfig.Clone(), false, _EMPTY_)\n\n\t// Removing the tls{} block but with no_tls still false should fail\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, _EMPTY_, false)))\n\terr = s.Reload()\n\trequire_True(t, err != nil)\n\trequire_Contains(t, err.Error(), \"TLS configuration\")\n\n\t// We should still be able to connect a TLS client\n\tcheck(tlsConfig.Clone(), false, _EMPTY_)\n\n\t// Now remove the tls{} block and set no_tls: true and that should fail\n\t// since this is not supported.\n\tchangeCurrentConfigContentWithNewContent(t, conf, []byte(fmt.Sprintf(template, _EMPTY_, true)))\n\terr = s.Reload()\n\trequire_True(t, err != nil)\n\trequire_Contains(t, err.Error(), \"not supported\")\n\n\t// We should still be able to connect a TLS client\n\tcheck(tlsConfig.Clone(), false, _EMPTY_)\n}\n\ntype captureClientConnectedLogger struct {\n\tDummyLogger\n\tch chan string\n}\n\nfunc (l *captureClientConnectedLogger) Debugf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif !strings.Contains(msg, \"Client connection created\") {\n\t\treturn\n\t}\n\tselect {\n\tcase l.ch <- msg:\n\tdefault:\n\t}\n}\n\nfunc TestWSXForwardedFor(t *testing.T) {\n\to := testWSOptions()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tl := &captureClientConnectedLogger{ch: make(chan string, 1)}\n\ts.SetLogger(l, true, false)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\theaders func() map[string][]string\n\t\tuseHdrValue bool\n\t\texpectedValue string\n\t}{\n\t\t{\"nil map\", func() map[string][]string {\n\t\t\treturn nil\n\t\t}, false, _EMPTY_},\n\t\t{\"empty map\", func() map[string][]string {\n\t\t\treturn make(map[string][]string)\n\t\t}, false, _EMPTY_},\n\t\t{\"header present empty value\", func() map[string][]string {\n\t\t\tm := make(map[string][]string)\n\t\t\tm[wsXForwardedForHeader] = []string{}\n\t\t\treturn m\n\t\t}, false, _EMPTY_},\n\t\t{\"header present invalid IP\", func() map[string][]string {\n\t\t\tm := make(map[string][]string)\n\t\t\tm[wsXForwardedForHeader] = []string{\"not a valid IP\"}\n\t\t\treturn m\n\t\t}, false, _EMPTY_},\n\t\t{\"header present one IP\", func() map[string][]string {\n\t\t\tm := make(map[string][]string)\n\t\t\tm[wsXForwardedForHeader] = []string{\"1.2.3.4\"}\n\t\t\treturn m\n\t\t}, true, \"1.2.3.4\"},\n\t\t{\"header present multiple IPs\", func() map[string][]string {\n\t\t\tm := make(map[string][]string)\n\t\t\tm[wsXForwardedForHeader] = []string{\"1.2.3.4\", \"5.6.7.8\"}\n\t\t\treturn m\n\t\t}, true, \"1.2.3.4\"},\n\t\t{\"header present IPv6\", func() map[string][]string {\n\t\t\tm := make(map[string][]string)\n\t\t\tm[wsXForwardedForHeader] = []string{\"::1\"}\n\t\t\treturn m\n\t\t}, true, \"[::1]\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tc, r, _ := testNewWSClient(t, testWSClientOptions{\n\t\t\t\thost: o.Websocket.Host,\n\t\t\t\tport: o.Websocket.Port,\n\t\t\t\textraHeaders: test.headers(),\n\t\t\t})\n\t\t\tdefer c.Close()\n\t\t\t// Send CONNECT and PING\n\t\t\twsmsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"CONNECT {\\\"verbose\\\":false,\\\"protocol\\\":1}\\r\\nPING\\r\\n\"))\n\t\t\tif _, err := c.Write(wsmsg); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\t\t\t// Wait for the PONG\n\t\t\tif msg := testWSReadFrame(t, r); !bytes.HasPrefix(msg, []byte(\"PONG\\r\\n\")) {\n\t\t\t\tt.Fatalf(\"Expected PONG, got %s\", msg)\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase d := <-l.ch:\n\t\t\t\tipAndSlash := fmt.Sprintf(\"%s/\", test.expectedValue)\n\t\t\t\tif test.useHdrValue {\n\t\t\t\t\tif !strings.HasPrefix(d, ipAndSlash) {\n\t\t\t\t\t\tt.Fatalf(\"Expected debug statement to start with: %q, got %q\", ipAndSlash, d)\n\t\t\t\t\t}\n\t\t\t\t} else if strings.HasPrefix(d, ipAndSlash) {\n\t\t\t\t\tt.Fatalf(\"Unexpected debug statement: %q\", d)\n\t\t\t\t}\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatal(\"Did not get connect debug statement\")\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype partialWriteConn struct {\n\tnet.Conn\n}\n\nfunc (c *partialWriteConn) Write(b []byte) (int, error) {\n\tmax := len(b)\n\tif max > 0 {\n\t\tmax = rand.Intn(max)\n\t\tif max == 0 {\n\t\t\tmax = 1\n\t\t}\n\t}\n\tn, err := c.Conn.Write(b[:max])\n\tif err == nil && max != len(b) {\n\t\terr = io.ErrShortWrite\n\t}\n\treturn n, err\n}\n\nfunc TestWSWithPartialWrite(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\twebsocket {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tno_tls: true\n\t\t}\n\t`))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc1 := natsConnect(t, s.WebsocketURL())\n\tdefer nc1.Close()\n\n\tsub := natsSubSync(t, nc1, \"foo\")\n\tsub.SetPendingLimits(-1, -1)\n\tnatsFlush(t, nc1)\n\n\tnc2 := natsConnect(t, s.WebsocketURL())\n\tdefer nc2.Close()\n\n\t// Replace websocket connections with ones that will produce short writes.\n\ts.mu.RLock()\n\tfor _, c := range s.clients {\n\t\tc.mu.Lock()\n\t\tc.nc = &partialWriteConn{Conn: c.nc}\n\t\tc.mu.Unlock()\n\t}\n\ts.mu.RUnlock()\n\n\tvar msgs [][]byte\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := make([]byte, rand.Intn(10000)+10)\n\t\tfor j := 0; j < len(msg); j++ {\n\t\t\tmsg[j] = byte('A' + j%26)\n\t\t}\n\t\tmsgs = append(msgs, msg)\n\t\tnatsPub(t, nc2, \"foo\", msg)\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\trmsg := natsNexMsg(t, sub, time.Second)\n\t\tif !bytes.Equal(msgs[i], rmsg.Data) {\n\t\t\tt.Fatalf(\"Expected message %q, got %q\", msgs[i], rmsg.Data)\n\t\t}\n\t}\n}\n\nfunc testWSNoCorruptionWithFrameSizeLimit(t *testing.T, total int) {\n\ttmpl := `\n listen: \"127.0.0.1:-1\"\n cluster {\n name: \"local\"\n port: -1\n %s\n }\n websocket {\n listen: \"127.0.0.1:-1\"\n no_tls: true\n }\n `\n\tconf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, _EMPTY_)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\troutes := fmt.Sprintf(\"routes: [\\\"nats://127.0.0.1:%d\\\"]\", o1.Cluster.Port)\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, routes)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, routes)))\n\ts3, _ := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\tnc3 := natsConnect(t, s3.WebsocketURL())\n\tdefer nc3.Close()\n\n\tnc2 := natsConnect(t, s2.WebsocketURL())\n\tdefer nc2.Close()\n\n\tpayload := make([]byte, 2*wsFrameSizeForBrowsers+123)\n\tfor i := 0; i < len(payload); i++ {\n\t\tpayload[i] = 'A' + byte(i%26)\n\t}\n\terrCh := make(chan error, 1)\n\tdoneCh := make(chan struct{}, 1)\n\tcount := int32(0)\n\n\tcreateSub := func(nc *nats.Conn) {\n\t\tsub := natsSub(t, nc, \"foo\", func(m *nats.Msg) {\n\t\t\tif !bytes.Equal(m.Data, payload) {\n\t\t\t\tstop := len(m.Data)\n\t\t\t\tif l := len(payload); l < stop {\n\t\t\t\t\tstop = l\n\t\t\t\t}\n\t\t\t\tstart := 0\n\t\t\t\tfor i := 0; i < stop; i++ {\n\t\t\t\t\tif m.Data[i] != payload[i] {\n\t\t\t\t\t\tstart = i\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif stop-start > 20 {\n\t\t\t\t\tstop = start + 20\n\t\t\t\t}\n\t\t\t\tselect {\n\t\t\t\tcase errCh <- fmt.Errorf(\"Invalid message: [%d bytes same]%s[...]\", start, m.Data[start:stop]):\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif n := atomic.AddInt32(&count, 1); int(n) == 2*total {\n\t\t\t\tdoneCh <- struct{}{}\n\t\t\t}\n\t\t})\n\t\tsub.SetPendingLimits(-1, -1)\n\t}\n\tcreateSub(nc2)\n\tcreateSub(nc3)\n\n\tcheckSubInterest(t, s1, globalAccountName, \"foo\", time.Second)\n\n\tnc1 := natsConnect(t, s1.WebsocketURL())\n\tdefer nc1.Close()\n\tnatsFlush(t, nc1)\n\n\t// Change websocket connections to force a max frame size.\n\tfor _, s := range []*Server{s1, s2, s3} {\n\t\ts.mu.RLock()\n\t\tfor _, c := range s.clients {\n\t\t\tc.mu.Lock()\n\t\t\tif c.ws != nil {\n\t\t\t\tc.ws.browser = true\n\t\t\t}\n\t\t\tc.mu.Unlock()\n\t\t}\n\t\ts.mu.RUnlock()\n\t}\n\n\tfor i := 0; i < total; i++ {\n\t\tnatsPub(t, nc1, \"foo\", payload)\n\t\tif i%100 == 0 {\n\t\t\tselect {\n\t\t\tcase err := <-errCh:\n\t\t\t\tt.Fatalf(\"Error: %v\", err)\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}\n\tselect {\n\tcase err := <-errCh:\n\t\tt.Fatalf(\"Error: %v\", err)\n\tcase <-doneCh:\n\t\treturn\n\tcase <-time.After(10 * time.Second):\n\t\tt.Fatalf(\"Test timed out\")\n\t}\n}\n\nfunc TestWSNoCorruptionWithFrameSizeLimit(t *testing.T) {\n\ttestWSNoCorruptionWithFrameSizeLimit(t, 1000)\n}\n\nfunc TestWSDecompressLimit(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tmpayCfg string\n\t}{\n\t\t{\"not explicitly configured\", _EMPTY_},\n\t\t{\"explicit high\", \"max_payload: 2097152\"},\n\t\t{\"explicit low\", \"max_payload: 4096\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, fmt.Appendf(nil, `\n\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\twebsocket {\n\t\t\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\t\t\tno_tls: true\n\t\t\t\t\t\tcompression: true\n\t\t\t\t\t}\n\t\t\t\t\t%s\n\t\t\t\t`, test.mpayCfg))\n\t\t\ts, o := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tl := &captureErrorLogger{errCh: make(chan string, 10)}\n\t\t\ts.SetLogger(l, false, false)\n\n\t\t\t// Create a client that will use compression.\n\t\t\twsc, br, _ := testNewWSClient(t, testWSClientOptions{\n\t\t\t\tcompress: true,\n\t\t\t\thost: o.Websocket.Host,\n\t\t\t\tport: o.Websocket.Port,\n\t\t\t\tnoTLS: true,\n\t\t\t})\n\t\t\t// We will hand-craft a frame that would use a 10MB of uncompressed zeros\n\t\t\t// that should compress really small.\n\t\t\tbuf := &bytes.Buffer{}\n\t\t\tcompressor, _ := flate.NewWriter(buf, 1)\n\t\t\tchunk := make([]byte, 1024*1024)\n\t\t\t// Compress the equivalent of 100MB of data. We do by chunks to limit\n\t\t\t// memory usage here.\n\t\t\tfor range 100 {\n\t\t\t\tcompressor.Write(chunk)\n\t\t\t}\n\t\t\tcompressor.Flush()\n\t\t\tpayload := buf.Bytes()\n\t\t\t// The last 4 bytes are dropped\n\t\t\tpayload = payload[:len(payload)-4]\n\t\t\tlenPayload := len(payload)\n\t\t\tframe := make([]byte, 14+lenPayload)\n\t\t\tframe[0] = byte(wsBinaryMessage)\n\t\t\tframe[0] |= wsFinalBit\n\t\t\tframe[0] |= wsRsv1Bit\n\t\t\tpos := 1\n\t\t\tswitch {\n\t\t\tcase lenPayload <= 125:\n\t\t\t\tframe[pos] = byte(lenPayload) | wsMaskBit\n\t\t\t\tpos++\n\t\t\tcase lenPayload < 65536:\n\t\t\t\tframe[pos] = 126 | wsMaskBit\n\t\t\t\tbinary.BigEndian.PutUint16(frame[2:], uint16(lenPayload))\n\t\t\t\tpos += 3\n\t\t\tdefault:\n\t\t\t\tframe[1] = 127 | wsMaskBit\n\t\t\t\tbinary.BigEndian.PutUint64(frame[2:], uint64(lenPayload))\n\t\t\t\tpos += 9\n\t\t\t}\n\t\t\tkey := []byte{1, 2, 3, 4}\n\t\t\tcopy(frame[pos:], key)\n\t\t\tpos += 4\n\t\t\tcopy(frame[pos:], payload)\n\t\t\ttestWSSimpleMask(key, frame[pos:])\n\t\t\tpos += lenPayload\n\t\t\ttoSend := frame[:pos]\n\t\t\tif _, err := wsc.Write(toSend); err != nil {\n\t\t\t\tt.Fatalf(\"Error sending message: %v\", err)\n\t\t\t}\n\n\t\t\t// We should have been disconnected.\n\t\t\trbuf := make([]byte, 1024)\n\t\t\t_, err := br.Read(rbuf)\n\t\t\trequire_Error(t, err)\n\n\t\t\tselect {\n\t\t\tcase err := <-l.errCh:\n\t\t\t\tif !strings.Contains(err, ErrMaxPayload.Error()) {\n\t\t\t\t\tt.Fatalf(\"Expected %s error, got %s\", ErrMaxPayload, err)\n\t\t\t\t}\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\tt.Fatal(\"Did not get the expected error\")\n\t\t\t}\n\t\t})\n\t}\n}\n\n// ==================================================================\n// = Benchmark tests\n// ==================================================================\n\nconst testWSBenchSubject = \"a\"\n\nvar ch = []byte(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$#%^&*()\")\n\nfunc sizedString(sz int) string {\n\tb := make([]byte, sz)\n\tfor i := range b {\n\t\tb[i] = ch[rand.Intn(len(ch))]\n\t}\n\treturn string(b)\n}\n\nfunc sizedStringForCompression(sz int) string {\n\tb := make([]byte, sz)\n\tc := byte(0)\n\ts := 0\n\tfor i := range b {\n\t\tif s%20 == 0 {\n\t\t\tc = ch[rand.Intn(len(ch))]\n\t\t}\n\t\tb[i] = c\n\t}\n\treturn string(b)\n}\n\nfunc testWSFlushConn(b *testing.B, compress bool, c net.Conn, br *bufio.Reader) {\n\tbuf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, []byte(pingProto))\n\tc.Write(buf)\n\tc.SetReadDeadline(time.Now().Add(5 * time.Second))\n\tres := testWSReadFrame(b, br)\n\tc.SetReadDeadline(time.Time{})\n\tif !bytes.HasPrefix(res, []byte(pongProto)) {\n\t\tb.Fatalf(\"Failed read of PONG: %s\\n\", res)\n\t}\n}\n\nfunc wsBenchPub(b *testing.B, numPubs int, compress bool, payload string) {\n\tb.StopTimer()\n\topts := testWSOptions()\n\topts.Websocket.Compression = compress\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\textra := 0\n\tpubProto := []byte(fmt.Sprintf(\"PUB %s %d\\r\\n%s\\r\\n\", testWSBenchSubject, len(payload), payload))\n\tsingleOpBuf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, pubProto)\n\n\t// Simulate client that would buffer messages before framing/sending.\n\t// Figure out how many we can fit in one frame based on b.N and length of pubProto\n\tconst bufSize = 32768\n\ttmpa := [bufSize]byte{}\n\ttmp := tmpa[:0]\n\tpb := 0\n\tfor i := 0; i < b.N; i++ {\n\t\ttmp = append(tmp, pubProto...)\n\t\tpb++\n\t\tif len(tmp) >= bufSize {\n\t\t\tbreak\n\t\t}\n\t}\n\tsendBuf := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress, tmp)\n\tn := b.N / pb\n\textra = b.N - (n * pb)\n\n\twg := sync.WaitGroup{}\n\twg.Add(numPubs)\n\n\ttype pub struct {\n\t\tc net.Conn\n\t\tbr *bufio.Reader\n\t\tbw *bufio.Writer\n\t}\n\tvar pubs []pub\n\tfor i := 0; i < numPubs; i++ {\n\t\twsc, br := testWSCreateClient(b, compress, false, opts.Websocket.Host, opts.Websocket.Port)\n\t\tdefer wsc.Close()\n\t\tbw := bufio.NewWriterSize(wsc, bufSize)\n\t\tpubs = append(pubs, pub{wsc, br, bw})\n\t}\n\n\t// Average the amount of bytes sent by iteration\n\tavg := len(sendBuf) / pb\n\tif extra > 0 {\n\t\tavg += len(singleOpBuf)\n\t\tavg /= 2\n\t}\n\tb.SetBytes(int64(numPubs * avg))\n\tb.StartTimer()\n\n\tfor i := 0; i < numPubs; i++ {\n\t\tp := pubs[i]\n\t\tgo func(p pub) {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\tp.bw.Write(sendBuf)\n\t\t\t}\n\t\t\tfor i := 0; i < extra; i++ {\n\t\t\t\tp.bw.Write(singleOpBuf)\n\t\t\t}\n\t\t\tp.bw.Flush()\n\t\t\ttestWSFlushConn(b, compress, p.c, p.br)\n\t\t}(p)\n\t}\n\twg.Wait()\n\tb.StopTimer()\n}\n\nfunc Benchmark_WS_Pubx1_CN_____0b(b *testing.B) {\n\twsBenchPub(b, 1, false, \"\")\n}\n\nfunc Benchmark_WS_Pubx1_CY_____0b(b *testing.B) {\n\twsBenchPub(b, 1, true, \"\")\n}\n\nfunc Benchmark_WS_Pubx1_CN___128b(b *testing.B) {\n\ts := sizedString(128)\n\twsBenchPub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Pubx1_CY___128b(b *testing.B) {\n\ts := sizedStringForCompression(128)\n\twsBenchPub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Pubx1_CN__1024b(b *testing.B) {\n\ts := sizedString(1024)\n\twsBenchPub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Pubx1_CY__1024b(b *testing.B) {\n\ts := sizedStringForCompression(1024)\n\twsBenchPub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Pubx1_CN__4096b(b *testing.B) {\n\ts := sizedString(4 * 1024)\n\twsBenchPub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Pubx1_CY__4096b(b *testing.B) {\n\ts := sizedStringForCompression(4 * 1024)\n\twsBenchPub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Pubx1_CN__8192b(b *testing.B) {\n\ts := sizedString(8 * 1024)\n\twsBenchPub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Pubx1_CY__8192b(b *testing.B) {\n\ts := sizedStringForCompression(8 * 1024)\n\twsBenchPub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Pubx1_CN_32768b(b *testing.B) {\n\ts := sizedString(32 * 1024)\n\twsBenchPub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Pubx1_CY_32768b(b *testing.B) {\n\ts := sizedStringForCompression(32 * 1024)\n\twsBenchPub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Pubx5_CN_____0b(b *testing.B) {\n\twsBenchPub(b, 5, false, \"\")\n}\n\nfunc Benchmark_WS_Pubx5_CY_____0b(b *testing.B) {\n\twsBenchPub(b, 5, true, \"\")\n}\n\nfunc Benchmark_WS_Pubx5_CN___128b(b *testing.B) {\n\ts := sizedString(128)\n\twsBenchPub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Pubx5_CY___128b(b *testing.B) {\n\ts := sizedStringForCompression(128)\n\twsBenchPub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Pubx5_CN__1024b(b *testing.B) {\n\ts := sizedString(1024)\n\twsBenchPub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Pubx5_CY__1024b(b *testing.B) {\n\ts := sizedStringForCompression(1024)\n\twsBenchPub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Pubx5_CN__4096b(b *testing.B) {\n\ts := sizedString(4 * 1024)\n\twsBenchPub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Pubx5_CY__4096b(b *testing.B) {\n\ts := sizedStringForCompression(4 * 1024)\n\twsBenchPub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Pubx5_CN__8192b(b *testing.B) {\n\ts := sizedString(8 * 1024)\n\twsBenchPub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Pubx5_CY__8192b(b *testing.B) {\n\ts := sizedStringForCompression(8 * 1024)\n\twsBenchPub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Pubx5_CN_32768b(b *testing.B) {\n\ts := sizedString(32 * 1024)\n\twsBenchPub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Pubx5_CY_32768b(b *testing.B) {\n\ts := sizedStringForCompression(32 * 1024)\n\twsBenchPub(b, 5, true, s)\n}\n\nfunc wsBenchSub(b *testing.B, numSubs int, compress bool, payload string) {\n\tb.StopTimer()\n\topts := testWSOptions()\n\topts.Websocket.Compression = compress\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tvar subs []*bufio.Reader\n\tfor i := 0; i < numSubs; i++ {\n\t\twsc, br := testWSCreateClient(b, compress, false, opts.Websocket.Host, opts.Websocket.Port)\n\t\tdefer wsc.Close()\n\t\tsubProto := testWSCreateClientMsg(wsBinaryMessage, 1, true, compress,\n\t\t\t[]byte(fmt.Sprintf(\"SUB %s 1\\r\\nPING\\r\\n\", testWSBenchSubject)))\n\t\twsc.Write(subProto)\n\t\t// Waiting for PONG\n\t\ttestWSReadFrame(b, br)\n\t\tsubs = append(subs, br)\n\t}\n\n\twg := sync.WaitGroup{}\n\twg.Add(numSubs)\n\n\t// Use regular NATS client to publish messages\n\tnc := natsConnect(b, s.ClientURL())\n\tdefer nc.Close()\n\n\tb.StartTimer()\n\n\tfor i := 0; i < numSubs; i++ {\n\t\tbr := subs[i]\n\t\tgo func(br *bufio.Reader) {\n\t\t\tdefer wg.Done()\n\t\t\tfor count := 0; count < b.N; {\n\t\t\t\tmsgs := testWSReadFrame(b, br)\n\t\t\t\tcount += bytes.Count(msgs, []byte(\"MSG \"))\n\t\t\t}\n\t\t}(br)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tnatsPub(b, nc, testWSBenchSubject, []byte(payload))\n\t}\n\twg.Wait()\n\tb.StopTimer()\n}\n\nfunc Benchmark_WS_Subx1_CN_____0b(b *testing.B) {\n\twsBenchSub(b, 1, false, \"\")\n}\n\nfunc Benchmark_WS_Subx1_CY_____0b(b *testing.B) {\n\twsBenchSub(b, 1, true, \"\")\n}\n\nfunc Benchmark_WS_Subx1_CN___128b(b *testing.B) {\n\ts := sizedString(128)\n\twsBenchSub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Subx1_CY___128b(b *testing.B) {\n\ts := sizedStringForCompression(128)\n\twsBenchSub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Subx1_CN__1024b(b *testing.B) {\n\ts := sizedString(1024)\n\twsBenchSub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Subx1_CY__1024b(b *testing.B) {\n\ts := sizedStringForCompression(1024)\n\twsBenchSub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Subx1_CN__4096b(b *testing.B) {\n\ts := sizedString(4096)\n\twsBenchSub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Subx1_CY__4096b(b *testing.B) {\n\ts := sizedStringForCompression(4096)\n\twsBenchSub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Subx1_CN__8192b(b *testing.B) {\n\ts := sizedString(8192)\n\twsBenchSub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Subx1_CY__8192b(b *testing.B) {\n\ts := sizedStringForCompression(8192)\n\twsBenchSub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Subx1_CN_32768b(b *testing.B) {\n\ts := sizedString(32768)\n\twsBenchSub(b, 1, false, s)\n}\n\nfunc Benchmark_WS_Subx1_CY_32768b(b *testing.B) {\n\ts := sizedStringForCompression(32768)\n\twsBenchSub(b, 1, true, s)\n}\n\nfunc Benchmark_WS_Subx5_CN_____0b(b *testing.B) {\n\twsBenchSub(b, 5, false, \"\")\n}\n\nfunc Benchmark_WS_Subx5_CY_____0b(b *testing.B) {\n\twsBenchSub(b, 5, true, \"\")\n}\n\nfunc Benchmark_WS_Subx5_CN___128b(b *testing.B) {\n\ts := sizedString(128)\n\twsBenchSub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Subx5_CY___128b(b *testing.B) {\n\ts := sizedStringForCompression(128)\n\twsBenchSub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Subx5_CN__1024b(b *testing.B) {\n\ts := sizedString(1024)\n\twsBenchSub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Subx5_CY__1024b(b *testing.B) {\n\ts := sizedStringForCompression(1024)\n\twsBenchSub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Subx5_CN__4096b(b *testing.B) {\n\ts := sizedString(4096)\n\twsBenchSub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Subx5_CY__4096b(b *testing.B) {\n\ts := sizedStringForCompression(4096)\n\twsBenchSub(b, 5, true, s)\n}\n\nfunc TestWebsocketPingInterval(t *testing.T) {\n\topts := testWSOptions()\n\topts.Websocket.PingInterval = 200 * time.Millisecond\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\twsc, br := testWSCreateClient(t, false, false, opts.Websocket.Host, opts.Websocket.Port)\n\tdefer wsc.Close()\n\n\tpingCount := 0\n\tdeadline := time.Now().Add(1 * time.Second)\n\n\tfor time.Now().Before(deadline) {\n\t\twsc.SetReadDeadline(time.Now().Add(1 * time.Second))\n\n\t\tmsg := testWSReadFrame(t, br)\n\t\tif bytes.Contains(msg, []byte(\"PING\\r\\n\")) {\n\t\t\tpingCount++\n\t\t\tpongMsg := testWSCreateClientMsg(wsBinaryMessage, 1, true, false, []byte(\"PONG\\r\\n\"))\n\t\t\twsc.Write(pongMsg)\n\t\t}\n\t}\n\tif pingCount < 2 {\n\t\tt.Fatalf(\"Expected at least 2 PINGs, got %d\", pingCount)\n\t}\n}\n\nfunc Benchmark_WS_Subx5_CN__8192b(b *testing.B) {\n\ts := sizedString(8192)\n\twsBenchSub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Subx5_CY__8192b(b *testing.B) {\n\ts := sizedStringForCompression(8192)\n\twsBenchSub(b, 5, true, s)\n}\n\nfunc Benchmark_WS_Subx5_CN_32768b(b *testing.B) {\n\ts := sizedString(32768)\n\twsBenchSub(b, 5, false, s)\n}\n\nfunc Benchmark_WS_Subx5_CY_32768b(b *testing.B) {\n\ts := sizedStringForCompression(32768)\n\twsBenchSub(b, 5, true, s)\n}\n" }, { "path": "test/accounts_cycles_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestAccountCycleService(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n\n\tconf = createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: help, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: *, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n\n\tconf = createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: help, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: C } } ]\n\t\t }\n\t\t C {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: *, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n}\n\nfunc TestAccountCycleStream(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { stream: strm } ]\n\t\t\timports [ { stream { subject: strm, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { stream: strm } ]\n\t\t\timports [ { stream { subject: strm, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cyclic import, got none\")\n\t}\n}\n\nfunc TestAccountCycleStreamWithMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { stream: * } ]\n\t\t\timports [ { stream { subject: bar, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { stream: bar } ]\n\t\t\timports [ { stream { subject: foo, account: A }, to: bar } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cyclic import, got none\")\n\t}\n}\n\nfunc TestAccountCycleNonCycleStreamWithMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { stream: foo } ]\n\t\t\timports [ { stream { subject: bar, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { stream: bar } ]\n\t\t\timports [ { stream { subject: baz, account: C }, to: bar } ]\n\t\t }\n\t\t C {\n\t\t exports [ { stream: baz } ]\n\t\t\timports [ { stream { subject: foo, account: A }, to: bar } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"Expected no error but got %s\", err)\n\t}\n}\n\nfunc TestAccountCycleServiceCycleWithMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: a } ]\n\t\t\timports [ { service { subject: b, account: B }, to: a } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: b } ]\n\t\t\timports [ { service { subject: a, account: A }, to: b } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n}\n\nfunc TestAccountCycleServiceNonCycle(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: help, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: nohelp, account: C } } ]\n\t\t }\n\t\t C {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: *, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\n\tif _, err := server.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"Expected no error but got %s\", err)\n\t}\n}\n\nfunc TestAccountCycleServiceNonCycleChain(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: B } } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: C } } ]\n\t\t }\n\t\t C {\n\t\t exports [ { service: help } ]\n\t\t\timports [ { service { subject: help, account: D } } ]\n\t\t }\n\t\t D {\n\t\t exports [ { service: help } ]\n\t\t }\n\t\t}\n\t`))\n\n\tif _, err := server.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"Expected no error but got %s\", err)\n\t}\n}\n\n// bug: https://github.com/nats-io/nats-server/issues/1769\nfunc TestServiceImportReplyMatchCycle(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t A {\n\t\t\tusers: [{user: d, pass: x}]\n\t\t\timports [ {service: {account: B, subject: \">\" }}]\n\t\t }\n\t\t B {\n\t\t\tusers: [{user: x, pass: x}]\n\t\t exports [ { service: \">\" } ]\n\t\t }\n\t\t}\n\t\tno_auth_user: d\n\t`))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc1 := clientConnectToServerWithUP(t, opts, \"x\", \"x\")\n\tdefer nc1.Close()\n\n\tmsg := []byte(\"HELLO\")\n\t_, err := nc1.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(msg)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Ensure the subscription is known by the server we're connected to.\n\terr = nc1.Flush()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcheckSubInterest(t, s, \"B\", \"foo\", time.Second)\n\n\tnc2 := clientConnectToServer(t, s)\n\tdefer nc2.Close()\n\n\tresp, err := nc2.Request(\"foo\", nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp == nil || string(resp.Data) != string(msg) {\n\t\tt.Fatalf(\"Wrong or empty response\")\n\t}\n}\n\nfunc TestServiceImportReplyMatchCycleMultiHops(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\taccounts {\n\t\t A {\n\t\t\tusers: [{user: d, pass: x}]\n\t\t\timports [ {service: {account: B, subject: \">\" }}]\n\t\t }\n\t\t B {\n\t\t exports [ { service: \">\" } ]\n\t\t\timports [ {service: {account: C, subject: \">\" }}]\n\t\t }\n\t\t C {\n\t\t\tusers: [{user: x, pass: x}]\n\t\t exports [ { service: \">\" } ]\n\t\t }\n\t\t}\n\t\tno_auth_user: d\n\t`))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc1 := clientConnectToServerWithUP(t, opts, \"x\", \"x\")\n\tdefer nc1.Close()\n\n\tmsg := []byte(\"HELLO\")\n\t_, err := nc1.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(msg)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Ensure the subscription is known by the server we're connected to.\n\terr = nc1.Flush()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcheckSubInterest(t, s, \"B\", \"foo\", time.Second)\n\n\tnc2 := clientConnectToServer(t, s)\n\tdefer nc2.Close()\n\n\tresp, err := nc2.Request(\"foo\", nil, time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp == nil || string(resp.Data) != string(msg) {\n\t\tt.Fatalf(\"Wrong or empty response\")\n\t}\n}\n\n// Go's stack are infinite sans memory, but not call depth. However its good to limit.\nfunc TestAccountCycleDepthLimit(t *testing.T) {\n\tvar last *server.Account\n\tchainLen := server.MaxAccountCycleSearchDepth + 1\n\n\t// Services\n\tfor i := 1; i <= chainLen; i++ {\n\t\tacc := server.NewAccount(fmt.Sprintf(\"ACC-%d\", i))\n\t\tif err := acc.AddServiceExport(\"*\", nil); err != nil {\n\t\t\tt.Fatalf(\"Error adding service export to '*': %v\", err)\n\t\t}\n\t\tif last != nil {\n\t\t\terr := acc.AddServiceImport(last, \"foo\", \"foo\")\n\t\t\tswitch i {\n\t\t\tcase chainLen:\n\t\t\t\tif err != server.ErrCycleSearchDepth {\n\t\t\t\t\tt.Fatalf(\"Expected last import to fail with '%v', but got '%v'\", server.ErrCycleSearchDepth, err)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error adding service import to 'foo': %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tlast = acc\n\t}\n\n\tlast = nil\n\n\t// Streams\n\tfor i := 1; i <= chainLen; i++ {\n\t\tacc := server.NewAccount(fmt.Sprintf(\"ACC-%d\", i))\n\t\tif err := acc.AddStreamExport(\"foo\", nil); err != nil {\n\t\t\tt.Fatalf(\"Error adding stream export to '*': %v\", err)\n\t\t}\n\t\tif last != nil {\n\t\t\terr := acc.AddStreamImport(last, \"foo\", \"\")\n\t\t\tswitch i {\n\t\t\tcase chainLen:\n\t\t\t\tif err != server.ErrCycleSearchDepth {\n\t\t\t\t\tt.Fatalf(\"Expected last import to fail with '%v', but got '%v'\", server.ErrCycleSearchDepth, err)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error adding stream import to 'foo': %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tlast = acc\n\t}\n}\n\n// Test token and partition subject mapping within an account\nfunc TestAccountSubjectMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tmappings = {\n \t\t\"foo.*.*\" : \"foo.$1.{{wildcard(2)}}.{{partition(10,1,2)}}\"\n\t\t}\n\t`))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc1 := clientConnectToServer(t, s)\n\tdefer nc1.Close()\n\n\tnumMessages := 100\n\tsubjectsReceived := make(chan string)\n\n\tmsg := []byte(\"HELLO\")\n\tsub1, err := nc1.Subscribe(\"foo.*.*.*\", func(m *nats.Msg) {\n\t\tsubjectsReceived <- m.Subject\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsub1.AutoUnsubscribe(numMessages * 2)\n\tnc1.Flush()\n\n\tnc2 := clientConnectToServer(t, s)\n\tdefer nc2.Close()\n\n\t// publish numMessages with an increasing id (should map to partition numbers with the range of 10 partitions) - twice\n\tfor j := 0; j < 2; j++ {\n\t\tfor i := 0; i < numMessages; i++ {\n\t\t\terr = nc2.Publish(fmt.Sprintf(\"foo.%d.%d\", i, numMessages-i), msg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// verify all the partition numbers are in the expected range\n\tpartitionsReceived := make([]int, numMessages)\n\n\tfor i := 0; i < numMessages; i++ {\n\t\tvar subject string\n\t\tselect {\n\t\tcase subject = <-subjectsReceived:\n\t\tcase <-time.After(5 * time.Second):\n\t\t\tt.Fatal(\"Timed out waiting for messages\")\n\t\t}\n\t\tsTokens := strings.Split(subject, \".\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tt1, _ := strconv.Atoi(sTokens[1])\n\t\tt2, _ := strconv.Atoi(sTokens[2])\n\t\tpartitionsReceived[i], err = strconv.Atoi(sTokens[3])\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\n\t\tif partitionsReceived[i] > 9 || partitionsReceived[i] < 0 || t1 != i || t2 != numMessages-i {\n\t\t\tt.Fatalf(\"Error received unexpected %d.%d to partition %d\", t1, t2, partitionsReceived[i])\n\t\t}\n\t}\n\n\t// verify hashing is deterministic by checking it produces the same exact result twice\n\tfor i := 0; i < numMessages; i++ {\n\t\tsubject := <-subjectsReceived\n\t\tpartitionNumber, err := strconv.Atoi(strings.Split(subject, \".\")[3])\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif partitionsReceived[i] != partitionNumber {\n\t\t\tt.Fatalf(\"Error: same id mapped to two different partitions\")\n\t\t}\n\t}\n}\n\n// test token subject mapping within an account\n// Alice imports from Bob with subject mapping\nfunc TestAccountImportSubjectMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n port: -1\n accounts {\n A {\n users: [{user: a, pass: x}]\n imports [ {stream: {account: B, subject: \"foo.*.*\"}, to : \"foo.$1.{{wildcard(2)}}\"}]\n }\n B {\n users: [{user: b, pass x}]\n exports [ { stream: \">\" } ]\n }\n }\n\t`))\n\n\ts, opts := RunServerWithConfig(conf)\n\n\tdefer s.Shutdown()\n\tncA := clientConnectToServerWithUP(t, opts, \"a\", \"x\")\n\tdefer ncA.Close()\n\n\tnumMessages := 100\n\tsubjectsReceived := make(chan string)\n\n\tmsg := []byte(\"HELLO\")\n\tsub1, err := ncA.Subscribe(\"foo.*.*\", func(m *nats.Msg) {\n\t\tsubjectsReceived <- m.Subject\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tsub1.AutoUnsubscribe(numMessages)\n\tncA.Flush()\n\n\tncB := clientConnectToServerWithUP(t, opts, \"b\", \"x\")\n\tdefer ncB.Close()\n\n\t// publish numMessages with an increasing id\n\n\tfor i := 0; i < numMessages; i++ {\n\t\terr = ncB.Publish(fmt.Sprintf(\"foo.%d.%d\", i, numMessages-i), msg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t}\n\n\tfor i := 0; i < numMessages; i++ {\n\t\tvar subject string\n\t\tselect {\n\t\tcase subject = <-subjectsReceived:\n\t\tcase <-time.After(1 * time.Second):\n\t\t\tt.Fatal(\"Timed out waiting for messages\")\n\t\t}\n\t\tsTokens := strings.Split(subject, \".\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tt1, _ := strconv.Atoi(sTokens[1])\n\t\tt2, _ := strconv.Atoi(sTokens[2])\n\n\t\tif t1 != i || t2 != numMessages-i {\n\t\t\tt.Fatalf(\"Error received unexpected %d.%d\", t1, t2)\n\t\t}\n\t}\n}\n\n// bug: https://github.com/nats-io/nats-server/issues/1789\nfunc TestAccountCycleWithRenaming(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: * } ]\n\t\t\timports [ { service { subject: foo, account: B }, to: foo } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: foo } ]\n\t\t\timports [ { service { subject: *, account: A }, to: \"$1\" } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n\n\tconf = createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { stream: * } ]\n\t\t\timports [ { stream { subject: foo, account: B }, to: foo } ]\n\t\t }\n\t\t B {\n\t\t exports [ { stream: foo } ]\n\t\t\timports [ { stream { subject: *, account: A }, to: \"$1\" } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected an error on cycle service import, got none\")\n\t}\n}\n\n// https://github.com/nats-io/nats-server/issues/5752\nfunc TestAccountCycleFalsePositiveSubjectMapping(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { service: \"a.*\" } ]\n\t\t\timports [ { service { subject: \"a.*\", account: B }, to: \"b.*\" } ]\n\t\t }\n\t\t B {\n\t\t exports [ { service: \"a.*\" } ]\n\t\t\timports [ { service { subject: \"a.foo\", account: A }, to: \"c.foo\" } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err != nil {\n\t\tt.Fatalf(\"Expected no errors on cycle service import, got error\")\n\t}\n}\n\n// TestAccountCycleServiceMissedDueToFromMutation reproduces a bug where\n// checkServiceImportsForCycles mutates the `from` loop variable via\n// `from = si.from`, affecting subsequent map iterations non-deterministically.\n//\n// Cycle: E -> A -> D -> E\n// A also has a non-cyclic import from C. When C's map key is iterated first,\n// `from` is narrowed from \"*\" to \"narrow\", causing D's import to be skipped\n// (SubjectsCollide(\"narrow\",\"other\") is false), and the cycle is missed.\nfunc TestAccountCycleServiceMissedDueToFromMutation(t *testing.T) {\n\t// Run multiple times because Go map iteration order is non-deterministic.\n\t// The bug only manifests when the non-cyclic import is visited first.\n\tfor i := 0; i < 100; i++ {\n\t\tconf := createConfFile(t, []byte(`\n\t\t\taccounts {\n\t\t\t A {\n\t\t\t exports [ { service: * } ]\n\t\t\t imports [\n\t\t\t { service { subject: narrow, account: C } }\n\t\t\t { service { subject: other, account: D } }\n\t\t\t ]\n\t\t\t }\n\t\t\t C {\n\t\t\t exports [ { service: narrow } ]\n\t\t\t }\n\t\t\t D {\n\t\t\t exports [ { service: other } ]\n\t\t\t imports [ { service { subject: other, account: E } } ]\n\t\t\t }\n\t\t\t E {\n\t\t\t exports [ { service: * } ]\n\t\t\t imports [ { service { subject: *, account: A } } ]\n\t\t\t }\n\t\t\t}\n\t\t`))\n\t\t_, err := server.ProcessConfigFile(conf)\n\t\tif err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\t\tt.Fatalf(\"Iteration %d: Expected cycle detection error for E->A->D->E, got: %v\", i, err)\n\t\t}\n\t}\n}\n\n// TestAccountCycleStreamMissedDueToToMutation reproduces a bug where the `to`\n// parameter is mutated inside the loop in checkStreamImportsForCycles.\n// Stream imports use a slice (deterministic order), so listing the narrowing\n// import first guarantees the bug triggers every time.\n//\n// Cycle: E → A → D → E (via subject \"other\")\n// A also imports \"narrow\" from C (leaf, no cycle) declared first.\n// After processing the first import, `to` is narrowed from \"*\" to \"narrow\",\n// causing SubjectsCollide(\"narrow\",\"other\") to return false and miss the cycle.\nfunc TestAccountCycleStreamMissedDueToToMutation(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t A {\n\t\t exports [ { stream: * } ]\n\t\t imports [\n\t\t { stream { subject: narrow, account: C }, to: narrow }\n\t\t { stream { subject: other, account: D }, to: other }\n\t\t ]\n\t\t }\n\t\t C {\n\t\t exports [ { stream: narrow } ]\n\t\t }\n\t\t D {\n\t\t exports [ { stream: other } ]\n\t\t imports [ { stream { subject: other, account: E }, to: other } ]\n\t\t }\n\t\t E {\n\t\t exports [ { stream: * } ]\n\t\t imports [ { stream { subject: *, account: A } } ]\n\t\t }\n\t\t}\n\t`))\n\tif _, err := server.ProcessConfigFile(conf); err == nil || !strings.Contains(err.Error(), server.ErrImportFormsCycle.Error()) {\n\t\tt.Fatalf(\"Expected cycle detection error for E->A->D->E, got: %v\", err)\n\t}\n}\n\nfunc clientConnectToServer(t *testing.T, s *server.Server) *nats.Conn {\n\tt.Helper()\n\tnc, err := nats.Connect(s.ClientURL(),\n\t\tnats.Name(\"JS-TEST\"),\n\t\tnats.ReconnectWait(5*time.Millisecond),\n\t\tnats.MaxReconnects(-1))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc clientConnectToServerWithUP(t *testing.T, opts *server.Options, user, pass string) *nats.Conn {\n\tcurl := fmt.Sprintf(\"nats://%s:%s@%s:%d\", user, pass, opts.Host, opts.Port)\n\tnc, err := nats.Connect(curl, nats.Name(\"JS-UP-TEST\"), nats.ReconnectWait(5*time.Millisecond), nats.MaxReconnects(-1))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create client: %v\", err)\n\t}\n\treturn nc\n}\n" }, { "path": "test/auth_test.go", "content": "// Copyright 2012-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc doAuthConnect(t tLogger, c net.Conn, token, user, pass string) {\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":true,\\\"auth_token\\\":\\\"%s\\\",\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\n\", token, user, pass)\n\tsendProto(t, c, cs)\n}\n\nfunc testInfoForAuth(t tLogger, infojs []byte) bool {\n\tvar sinfo server.Info\n\terr := json.Unmarshal(infojs, &sinfo)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not unmarshal INFO json: %v\\n\", err)\n\t}\n\treturn sinfo.AuthRequired\n}\n\nfunc expectAuthRequired(t tLogger, c net.Conn) {\n\tbuf := expectResult(t, c, infoRe)\n\tinfojs := infoRe.FindAllSubmatch(buf, 1)[0][1]\n\tif !testInfoForAuth(t, infojs) {\n\t\tt.Fatalf(\"Expected server to require authorization: '%s'\", infojs)\n\t}\n}\n\n////////////////////////////////////////////////////////////\n// The authorization token version\n////////////////////////////////////////////////////////////\n\nconst AUTH_PORT = 10422\nconst AUTH_TOKEN = \"_YZZ22_\"\n\nfunc runAuthServerWithToken() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = AUTH_PORT\n\topts.Authorization = AUTH_TOKEN\n\treturn RunServer(&opts)\n}\n\nfunc TestNoAuthClient(t *testing.T) {\n\ts := runAuthServerWithToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"\", \"\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestAuthClientBadToken(t *testing.T) {\n\ts := runAuthServerWithToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"ZZZ\", \"\", \"\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestAuthClientNoConnect(t *testing.T) {\n\ts := runAuthServerWithToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\t// This is timing dependent..\n\ttime.Sleep(server.AUTH_TIMEOUT)\n\texpectResult(t, c, errRe)\n}\n\nfunc TestAuthClientGoodConnect(t *testing.T) {\n\ts := runAuthServerWithToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, AUTH_TOKEN, \"\", \"\")\n\texpectResult(t, c, okRe)\n}\n\nfunc TestAuthClientFailOnEverythingElse(t *testing.T) {\n\ts := runAuthServerWithToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tsendProto(t, c, \"PUB foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n}\n\n////////////////////////////////////////////////////////////\n// The username/password version\n////////////////////////////////////////////////////////////\n\nconst AUTH_USER = \"derek\"\nconst AUTH_PASS = \"foobar\"\n\nfunc runAuthServerWithUserPass() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = AUTH_PORT\n\topts.Username = AUTH_USER\n\topts.Password = AUTH_PASS\n\treturn RunServer(&opts)\n}\n\nfunc TestNoUserOrPasswordClient(t *testing.T) {\n\ts := runAuthServerWithUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"\", \"\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestBadUserClient(t *testing.T) {\n\ts := runAuthServerWithUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"derekzz\", AUTH_PASS)\n\texpectResult(t, c, errRe)\n}\n\nfunc TestBadPasswordClient(t *testing.T) {\n\ts := runAuthServerWithUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", AUTH_USER, \"ZZ\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestPasswordClientGoodConnect(t *testing.T) {\n\ts := runAuthServerWithUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", AUTH_USER, AUTH_PASS)\n\texpectResult(t, c, okRe)\n}\n\n////////////////////////////////////////////////////////////\n// The bcrypt username/password version\n////////////////////////////////////////////////////////////\n\n// Generated with nats server passwd (Cost 4 because of cost of --race, default is 11)\nconst BCRYPT_AUTH_PASS = \"IW@$6v(y1(t@fhPDvf!5^%\"\nconst BCRYPT_AUTH_HASH = \"$2a$04$Q.CgCP2Sl9pkcTXEZHazaeMwPaAkSHk7AI51HkyMt5iJQQyUA4qxq\"\n\nfunc runAuthServerWithBcryptUserPass() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = AUTH_PORT\n\topts.Username = AUTH_USER\n\topts.Password = BCRYPT_AUTH_HASH\n\treturn RunServer(&opts)\n}\n\nfunc TestBadBcryptPassword(t *testing.T) {\n\ts := runAuthServerWithBcryptUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", AUTH_USER, BCRYPT_AUTH_HASH)\n\texpectResult(t, c, errRe)\n}\n\nfunc TestGoodBcryptPassword(t *testing.T) {\n\ts := runAuthServerWithBcryptUserPass()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", AUTH_USER, BCRYPT_AUTH_PASS)\n\texpectResult(t, c, okRe)\n}\n\n////////////////////////////////////////////////////////////\n// The bcrypt authorization token version\n////////////////////////////////////////////////////////////\n\nconst BCRYPT_AUTH_TOKEN = \"0uhJOSr3GW7xvHvtd^K6pa\"\nconst BCRYPT_AUTH_TOKEN_HASH = \"$2a$04$u5ZClXpcjHgpfc61Ee0VKuwI1K3vTC4zq7SjphjnlHMeb1Llkb5Y6\"\n\nfunc runAuthServerWithBcryptToken() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = AUTH_PORT\n\topts.Authorization = BCRYPT_AUTH_TOKEN_HASH\n\treturn RunServer(&opts)\n}\n\nfunc TestBadBcryptToken(t *testing.T) {\n\ts := runAuthServerWithBcryptToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, BCRYPT_AUTH_TOKEN_HASH, \"\", \"\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestGoodBcryptToken(t *testing.T) {\n\ts := runAuthServerWithBcryptToken()\n\tdefer s.Shutdown()\n\tc := createClientConn(t, \"127.0.0.1\", AUTH_PORT)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, BCRYPT_AUTH_TOKEN, \"\", \"\")\n\texpectResult(t, c, okRe)\n}\n" }, { "path": "test/bench_results.txt", "content": "2017 iMac Pro 3Ghz (Turbo 4Ghz) 10-Core Skylake\nOSX High Sierra 10.13.2\n\n===================\nGo version go1.9.2\n===================\n\nBenchmark_____Pub0b_Payload-20 \t30000000\t 55.1 ns/op\t 199.78 MB/s\nBenchmark_____Pub8b_Payload-20 \t30000000\t 55.8 ns/op\t 340.21 MB/s\nBenchmark____Pub32b_Payload-20 \t20000000\t 63.4 ns/op\t 694.34 MB/s\nBenchmark___Pub128B_Payload-20 \t20000000\t 79.8 ns/op\t1766.47 MB/s\nBenchmark___Pub256B_Payload-20 \t20000000\t 98.1 ns/op\t2741.51 MB/s\nBenchmark_____Pub1K_Payload-20 \t 5000000\t 283 ns/op\t3660.72 MB/s\nBenchmark_____Pub4K_Payload-20 \t 1000000\t 1395 ns/op\t2945.30 MB/s\nBenchmark_____Pub8K_Payload-20 \t 500000\t 2846 ns/op\t2882.35 MB/s\nBenchmark_AuthPub0b_Payload-20 \t10000000\t 126 ns/op\t 86.82 MB/s\nBenchmark____________PubSub-20 \t10000000\t 135 ns/op\nBenchmark____PubSubTwoConns-20 \t10000000\t 136 ns/op\nBenchmark____PubTwoQueueSub-20 \t10000000\t 152 ns/op\nBenchmark___PubFourQueueSub-20 \t10000000\t 152 ns/op\nBenchmark__PubEightQueueSub-20 \t10000000\t 152 ns/op\nBenchmark___RoutedPubSub_0b-20 \t 5000000\t 385 ns/op\nBenchmark___RoutedPubSub_1K-20 \t 1000000\t 1076 ns/op\nBenchmark_RoutedPubSub_100K-20 \t 20000\t 78501 ns/op\n\n\n2015 iMac5k 4Ghz i7 Haswell\nOSX El Capitan 10.11.3\n\n===================\nGo version go1.6\n===================\n\nBenchmark____PubNo_Payload-8\t20000000\t 88.6 ns/op\t 124.11 MB/s\nBenchmark____Pub8b_Payload-8\t20000000\t 89.8 ns/op\t 211.63 MB/s\nBenchmark___Pub32b_Payload-8\t20000000\t 97.3 ns/op\t 452.20 MB/s\nBenchmark__Pub256B_Payload-8\t10000000\t 129 ns/op\t2078.43 MB/s\nBenchmark____Pub1K_Payload-8\t 5000000\t 216 ns/op\t4791.00 MB/s\nBenchmark____Pub4K_Payload-8\t 1000000\t 1123 ns/op\t3657.53 MB/s\nBenchmark____Pub8K_Payload-8\t 500000\t 2309 ns/op\t3553.09 MB/s\nBenchmark___________PubSub-8\t10000000\t 210 ns/op\nBenchmark___PubSubTwoConns-8\t10000000\t 205 ns/op\nBenchmark___PubTwoQueueSub-8\t10000000\t 231 ns/op\nBenchmark__PubFourQueueSub-8\t10000000\t 233 ns/op\nBenchmark_PubEightQueueSub-8\t 5000000\t 231 ns/op\n\nOSX Yosemite 10.10.5\n\n===================\nGo version go1.4.2\n===================\n\nBenchmark___PubNo_Payload\t10000000\t 133 ns/op\t 82.44 MB/s\nBenchmark___Pub8b_Payload\t10000000\t 135 ns/op\t 140.27 MB/s\nBenchmark__Pub32b_Payload\t10000000\t 147 ns/op\t 297.56 MB/s\nBenchmark_Pub256B_Payload\t10000000\t 211 ns/op\t1273.82 MB/s\nBenchmark___Pub1K_Payload\t 3000000\t 447 ns/op\t2321.55 MB/s\nBenchmark___Pub4K_Payload\t 1000000\t 1677 ns/op\t2450.43 MB/s\nBenchmark___Pub8K_Payload\t 300000\t 3670 ns/op\t2235.80 MB/s\nBenchmark__________PubSub\t 5000000\t 263 ns/op\nBenchmark__PubSubTwoConns\t 5000000\t 268 ns/op\nBenchmark__PubTwoQueueSub\t 2000000\t 936 ns/op\nBenchmark_PubFourQueueSub\t 1000000\t 1103 ns/op\n\n===================\nGo version go1.5.0\n===================\n\nBenchmark___PubNo_Payload-8\t10000000\t 122 ns/op\t 89.94 MB/s\nBenchmark___Pub8b_Payload-8\t10000000\t 124 ns/op\t 152.72 MB/s\nBenchmark__Pub32b_Payload-8\t10000000\t 135 ns/op\t 325.73 MB/s\nBenchmark_Pub256B_Payload-8\t10000000\t 159 ns/op\t1685.78 MB/s\nBenchmark___Pub1K_Payload-8\t 5000000\t 256 ns/op\t4047.90 MB/s\nBenchmark___Pub4K_Payload-8\t 1000000\t 1164 ns/op\t3530.77 MB/s\nBenchmark___Pub8K_Payload-8\t 500000\t 2444 ns/op\t3357.34 MB/s\nBenchmark__________PubSub-8\t 5000000\t 254 ns/op\nBenchmark__PubSubTwoConns-8\t 5000000\t 245 ns/op\nBenchmark__PubTwoQueueSub-8\t 2000000\t 845 ns/op\nBenchmark_PubFourQueueSub-8\t 1000000\t 1004 ns/op\n" }, { "path": "test/bench_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Please note that these tests will stress a system and they need generous\n// amounts of CPU, Memory and network sockets. Make sure the 'open files'\n// setting for your platform is at least 8192. On linux and MacOSX you can\n// do this via 'ulimit -n 8192'\n//\n\npackage test\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nconst PERF_PORT = 8422\n\n// For Go routine based server.\nfunc runBenchServer() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = PERF_PORT\n\topts.DisableShortFirstPing = true\n\treturn RunServer(&opts)\n}\n\nconst defaultRecBufSize = 32768\nconst defaultSendBufSize = 32768\n\nfunc flushConnection(b *testing.B, c net.Conn) {\n\tbuf := make([]byte, 32)\n\tc.Write([]byte(\"PING\\r\\n\"))\n\tc.SetReadDeadline(time.Now().Add(5 * time.Second))\n\tn, err := c.Read(buf)\n\tc.SetReadDeadline(time.Time{})\n\tif err != nil {\n\t\tb.Fatalf(\"Failed read: %v\\n\", err)\n\t}\n\tif n != 6 && buf[0] != 'P' && buf[1] != 'O' {\n\t\tb.Fatalf(\"Failed read of PONG: %s\\n\", buf)\n\t}\n}\n\nfunc benchPub(b *testing.B, subject, payload string) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(fmt.Sprintf(\"PUB %s %d\\r\\n%s\\r\\n\", subject, len(payload), payload))\n\tb.SetBytes(int64(len(sendOp)))\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\tbw.Flush()\n\tflushConnection(b, c)\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nvar ch = []byte(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@$#%^&*()\")\n\nfunc sizedBytes(sz int) []byte {\n\tb := make([]byte, sz)\n\tfor i := range b {\n\t\tb[i] = ch[rand.Intn(len(ch))]\n\t}\n\treturn b\n}\n\nfunc sizedString(sz int) string {\n\treturn string(sizedBytes(sz))\n}\n\n// Publish subject for pub benchmarks.\nvar psub = \"a\"\n\nfunc Benchmark______Pub0b_Payload(b *testing.B) {\n\tbenchPub(b, psub, \"\")\n}\n\nfunc Benchmark______Pub8b_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(8)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark_____Pub32b_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(32)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark____Pub128B_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(128)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark____Pub256B_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(256)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark______Pub1K_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(1024)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark______Pub4K_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(4 * 1024)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark______Pub8K_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(8 * 1024)\n\tbenchPub(b, psub, s)\n}\n\nfunc Benchmark_____Pub32K_Payload(b *testing.B) {\n\tb.StopTimer()\n\ts := sizedString(32 * 1024)\n\tbenchPub(b, psub, s)\n}\n\nfunc drainConnection(b *testing.B, c net.Conn, ch chan bool, expected int) {\n\tbuf := make([]byte, defaultRecBufSize)\n\tbytes := 0\n\n\tfor {\n\t\tc.SetReadDeadline(time.Now().Add(30 * time.Second))\n\t\tn, err := c.Read(buf)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"Error on read: %v\\n\", err)\n\t\t\tbreak\n\t\t}\n\t\tbytes += n\n\t\tif bytes >= expected {\n\t\t\tbreak\n\t\t}\n\t}\n\tif bytes != expected {\n\t\tb.Errorf(\"Did not receive all bytes: %d vs %d\\n\", bytes, expected)\n\t}\n\tch <- true\n}\n\n// Benchmark the authorization code path.\nfunc Benchmark__AuthPub0b_Payload(b *testing.B) {\n\tb.StopTimer()\n\n\tsrv, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\topts.DisableShortFirstPing = true\n\tdefer srv.Shutdown()\n\n\tc := createClientConn(b, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(b, c)\n\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\n\", \"bench\", DefaultPass)\n\tsendProto(b, c, cs)\n\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB a 0\\r\\n\\r\\n\")\n\tb.SetBytes(int64(len(sendOp)))\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\tbw.Flush()\n\tflushConnection(b, c)\n\tb.StopTimer()\n}\n\nfunc Benchmark_____________PubSub(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tsendProto(b, c, \"SUB foo 1\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc Benchmark_____PubSubTwoConns(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\n\tc2 := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c2)\n\tsendProto(b, c2, \"SUB foo 1\\r\\n\")\n\tflushConnection(b, c2)\n\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c2, ch, expected)\n\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\tc2.Close()\n\ts.Shutdown()\n}\n\nfunc benchDefaultOptionsForAccounts() *server.Options {\n\to := DefaultTestOptions\n\to.Host = \"127.0.0.1\"\n\to.Port = -1\n\to.Cluster.Host = o.Host\n\to.Cluster.Port = -1\n\to.DisableShortFirstPing = true\n\tfooAcc := server.NewAccount(\"$foo\")\n\tfooAcc.AddStreamExport(\"foo\", nil)\n\tbarAcc := server.NewAccount(\"$bar\")\n\tbarAcc.AddStreamImport(fooAcc, \"foo\", \"\")\n\to.Accounts = []*server.Account{fooAcc, barAcc}\n\n\treturn &o\n}\n\nfunc createClientWithAccount(b *testing.B, account, host string, port int) net.Conn {\n\tc := createClientConn(b, host, port)\n\tcheckInfoMsg(b, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"account\\\":%q}\\r\\n\", false, false, false, account)\n\tsendProto(b, c, cs)\n\treturn c\n}\n\nfunc benchOptionsForServiceImports() *server.Options {\n\to := DefaultTestOptions\n\to.Host = \"127.0.0.1\"\n\to.Port = -1\n\to.DisableShortFirstPing = true\n\tfoo := server.NewAccount(\"$foo\")\n\tbar := server.NewAccount(\"$bar\")\n\to.Accounts = []*server.Account{foo, bar}\n\n\treturn &o\n}\n\nfunc addServiceImports(b *testing.B, s *server.Server) {\n\t// Add a bunch of service exports with wildcards, similar to JS.\n\tvar exports = []string{\n\t\tserver.JSApiAccountInfo,\n\t\tserver.JSApiStreamCreate,\n\t\tserver.JSApiStreamUpdate,\n\t\tserver.JSApiStreams,\n\t\tserver.JSApiStreamInfo,\n\t\tserver.JSApiStreamDelete,\n\t\tserver.JSApiStreamPurge,\n\t\tserver.JSApiMsgDelete,\n\t\tserver.JSApiConsumerCreate,\n\t\tserver.JSApiConsumers,\n\t\tserver.JSApiConsumerInfo,\n\t\tserver.JSApiConsumerDelete,\n\t}\n\tfoo, _ := s.LookupAccount(\"$foo\")\n\tbar, _ := s.LookupAccount(\"$bar\")\n\n\tfor _, export := range exports {\n\t\tif err := bar.AddServiceExport(export, nil); err != nil {\n\t\t\tb.Fatalf(\"Could not add service export: %v\", err)\n\t\t}\n\t\tif err := foo.AddServiceImport(bar, export, \"\"); err != nil {\n\t\t\tb.Fatalf(\"Could not add service import: %v\", err)\n\t\t}\n\t}\n}\n\nfunc Benchmark__PubServiceImports(b *testing.B) {\n\to := benchOptionsForServiceImports()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\taddServiceImports(b, s)\n\n\tc := createClientWithAccount(b, \"$foo\", o.Host, o.Port)\n\tdefer c.Close()\n\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\tb.StopTimer()\n}\n\nfunc Benchmark___PubSubAccsImport(b *testing.B) {\n\to := benchDefaultOptionsForAccounts()\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tpub := createClientWithAccount(b, \"$foo\", o.Host, o.Port)\n\tdefer pub.Close()\n\n\tsub := createClientWithAccount(b, \"$bar\", o.Host, o.Port)\n\tdefer sub.Close()\n\n\tsendProto(b, sub, \"SUB foo 1\\r\\n\")\n\tflushConnection(b, sub)\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, sub, ch, expected)\n\n\tbw := bufio.NewWriterSize(pub, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\tb.StopTimer()\n}\n\nfunc Benchmark_____PubTwoQueueSub(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tsendProto(b, c, \"SUB foo group1 1\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 2\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Fatalf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc Benchmark____PubFourQueueSub(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tsendProto(b, c, \"SUB foo group1 1\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 2\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 3\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 4\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Fatalf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc Benchmark___PubEightQueueSub(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tsendProto(b, c, \"SUB foo group1 1\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 2\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 3\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 4\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 5\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 6\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 7\\r\\n\")\n\tsendProto(b, c, \"SUB foo group1 8\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Fatalf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\ts.Shutdown()\n}\n\nfunc Benchmark_PubSub512kTwoConns(b *testing.B) {\n\tb.StopTimer()\n\ts := runBenchServer()\n\tc := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c)\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\n\tc2 := createClientConn(b, \"127.0.0.1\", PERF_PORT)\n\tdoDefaultConnect(b, c2)\n\tsendProto(b, c2, \"SUB foo 1\\r\\n\")\n\tflushConnection(b, c2)\n\n\tsz := 1024 * 512\n\tpayload := sizedString(sz)\n\n\tsendOp := []byte(fmt.Sprintf(\"PUB foo %d\\r\\n%s\\r\\n\", sz, payload))\n\tch := make(chan bool)\n\n\texpected := len(fmt.Sprintf(\"MSG foo 1 %d\\r\\n%s\\r\\n\", sz, payload)) * b.N\n\tgo drainConnection(b, c2, ch, expected)\n\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbw.Write(sendOp)\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tc.Close()\n\tc2.Close()\n\ts.Shutdown()\n}\n\nfunc Benchmark__DenyMsgNoWCPubSub(b *testing.B) {\n\ts, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\topts.DisableShortFirstPing = true\n\tdefer s.Shutdown()\n\n\tc := createClientConn(b, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\texpectAuthRequired(b, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\n\", \"bench-deny\", DefaultPass)\n\tsendProto(b, c, cs)\n\n\tsendProto(b, c, \"SUB foo 1\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\t// To not count defer cleanup of client and server.\n\tb.StopTimer()\n}\n\nfunc Benchmark_DenyMsgYesWCPubSub(b *testing.B) {\n\ts, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\topts.DisableShortFirstPing = true\n\tdefer s.Shutdown()\n\n\tc := createClientConn(b, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\texpectAuthRequired(b, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":false,\\\"pedantic\\\":false,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\"}\\r\\n\", \"bench-deny\", DefaultPass)\n\tsendProto(b, c, cs)\n\n\tsendProto(b, c, \"SUB * 1\\r\\n\")\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(\"PUB foo 2\\r\\nok\\r\\n\")\n\tch := make(chan bool)\n\texpected := len(\"MSG foo 1 2\\r\\nok\\r\\n\") * b.N\n\tgo drainConnection(b, c, ch, expected)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\t// To not count defer cleanup of client and server.\n\tb.StopTimer()\n}\n\nfunc routePubSub(b *testing.B, size int) {\n\tb.StopTimer()\n\n\ts1, o1 := RunServerWithConfig(\"./configs/srv_a.conf\")\n\to1.DisableShortFirstPing = true\n\tdefer s1.Shutdown()\n\ts2, o2 := RunServerWithConfig(\"./configs/srv_b.conf\")\n\to2.DisableShortFirstPing = true\n\tdefer s2.Shutdown()\n\n\tsub := createClientConn(b, o1.Host, o1.Port)\n\tdoDefaultConnect(b, sub)\n\tsendProto(b, sub, \"SUB foo 1\\r\\n\")\n\tflushConnection(b, sub)\n\n\tpayload := sizedString(size)\n\n\tpub := createClientConn(b, o2.Host, o2.Port)\n\tdoDefaultConnect(b, pub)\n\tbw := bufio.NewWriterSize(pub, defaultSendBufSize)\n\n\tch := make(chan bool)\n\tsendOp := []byte(fmt.Sprintf(\"PUB foo %d\\r\\n%s\\r\\n\", len(payload), payload))\n\texpected := len(fmt.Sprintf(\"MSG foo 1 %d\\r\\n%s\\r\\n\", len(payload), payload)) * b.N\n\tgo drainConnection(b, sub, ch, expected)\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n\tpub.Close()\n\tsub.Close()\n}\n\nfunc Benchmark____RoutedPubSub_0b(b *testing.B) {\n\troutePubSub(b, 2)\n}\n\nfunc Benchmark____RoutedPubSub_1K(b *testing.B) {\n\troutePubSub(b, 1024)\n}\n\nfunc Benchmark__RoutedPubSub_100K(b *testing.B) {\n\troutePubSub(b, 100*1024)\n}\n\nfunc routeQueue(b *testing.B, numQueueSubs, size int) {\n\ts1, o1 := RunServerWithConfig(\"./configs/srv_a.conf\")\n\to1.DisableShortFirstPing = true\n\tdefer s1.Shutdown()\n\ts2, o2 := RunServerWithConfig(\"./configs/srv_b.conf\")\n\to2.DisableShortFirstPing = true\n\tdefer s2.Shutdown()\n\n\tsub := createClientConn(b, o1.Host, o1.Port)\n\tdefer sub.Close()\n\tdoDefaultConnect(b, sub)\n\tfor i := 0; i < numQueueSubs; i++ {\n\t\tsendProto(b, sub, fmt.Sprintf(\"SUB foo bar %d\\r\\n\", 100+i))\n\t}\n\tflushConnection(b, sub)\n\n\tpayload := sizedString(size)\n\n\tpub := createClientConn(b, o2.Host, o2.Port)\n\tdefer pub.Close()\n\tdoDefaultConnect(b, pub)\n\tbw := bufio.NewWriterSize(pub, defaultSendBufSize)\n\n\tch := make(chan bool)\n\tsendOp := []byte(fmt.Sprintf(\"PUB foo %d\\r\\n%s\\r\\n\", len(payload), payload))\n\texpected := len(fmt.Sprintf(\"MSG foo 100 %d\\r\\n%s\\r\\n\", len(payload), payload)) * b.N\n\tgo drainConnection(b, sub, ch, expected)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connection to be drained\n\t<-ch\n\n\tb.StopTimer()\n}\n\nfunc Benchmark____Routed2QueueSub(b *testing.B) {\n\trouteQueue(b, 2, 2)\n}\n\nfunc Benchmark____Routed4QueueSub(b *testing.B) {\n\trouteQueue(b, 4, 2)\n}\n\nfunc Benchmark____Routed8QueueSub(b *testing.B) {\n\trouteQueue(b, 8, 2)\n}\n\nfunc Benchmark___Routed16QueueSub(b *testing.B) {\n\trouteQueue(b, 16, 2)\n}\n\nfunc doS2CompressBench(b *testing.B, compress string) {\n\tb.StopTimer()\n\tconf1 := createConfFile(b, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\t\t\tcompression: %s\n\t\t}\n\t`, compress)))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(b, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"local\"\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\t\t\tcompression: %s\n\t\t\troutes: [\"nats://127.0.0.1:%d\"]\n\t\t}\n\t`, compress, o1.Cluster.Port)))\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(b, s1, s2)\n\n\tnc2, err := nats.Connect(s2.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tch := make(chan struct{}, 1)\n\tvar count int\n\tnc2.Subscribe(\"foo\", func(_ *nats.Msg) {\n\t\tif count++; count == b.N {\n\t\t\tselect {\n\t\t\tcase ch <- struct{}{}:\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t})\n\n\tcheckSubInterest(b, s1, \"$G\", \"foo\", time.Second)\n\n\tnc1, err := nats.Connect(s1.ClientURL())\n\tif err != nil {\n\t\tb.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// This one is easily compressible.\n\tpayload1 := make([]byte, 128)\n\t// Make it random so that compression code has more to do.\n\tpayload2 := make([]byte, 256)\n\tfor i := 0; i < len(payload); i++ {\n\t\tpayload2[i] = byte(rand.Intn(26) + 'A')\n\t}\n\tb.StartTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tif i%2 == 0 {\n\t\t\tnc1.Publish(\"foo\", payload1)\n\t\t} else {\n\t\t\tnc1.Publish(\"foo\", payload2)\n\t\t}\n\t}\n\n\tselect {\n\tcase <-ch:\n\t\treturn\n\tcase <-time.After(10 * time.Second):\n\t\tb.Fatal(\"Timeout waiting to receive all messages\")\n\t}\n}\n\nfunc Benchmark____________RouteCompressOff(b *testing.B) {\n\tdoS2CompressBench(b, server.CompressionOff)\n}\n\nfunc Benchmark_RouteCompressS2Uncompressed(b *testing.B) {\n\tdoS2CompressBench(b, server.CompressionS2Uncompressed)\n}\n\nfunc Benchmark_________RouteCompressS2Fast(b *testing.B) {\n\tdoS2CompressBench(b, server.CompressionS2Fast)\n}\n\nfunc Benchmark_______RouteCompressS2Better(b *testing.B) {\n\tdoS2CompressBench(b, server.CompressionS2Better)\n}\n\nfunc Benchmark_________RouteCompressS2Best(b *testing.B) {\n\tdoS2CompressBench(b, server.CompressionS2Best)\n}\n\nfunc doFanout(b *testing.B, numServers, numConnections, subsPerConnection int, subject, payload string) {\n\tvar s1, s2 *server.Server\n\tvar o1, o2 *server.Options\n\n\tswitch numServers {\n\tcase 1:\n\t\ts1, o1 = RunServerWithConfig(\"./configs/srv_a.conf\")\n\t\to1.DisableShortFirstPing = true\n\t\tdefer s1.Shutdown()\n\t\ts2, o2 = s1, o1\n\tcase 2:\n\t\ts1, o1 = RunServerWithConfig(\"./configs/srv_a.conf\")\n\t\to1.DisableShortFirstPing = true\n\t\tdefer s1.Shutdown()\n\t\ts2, o2 = RunServerWithConfig(\"./configs/srv_b.conf\")\n\t\to2.DisableShortFirstPing = true\n\t\tdefer s2.Shutdown()\n\tdefault:\n\t\tb.Fatalf(\"%d servers not supported for this test\\n\", numServers)\n\t}\n\n\t// To get a consistent length sid in MSG sent to us for drainConnection.\n\tvar sidFloor int\n\tswitch {\n\tcase subsPerConnection <= 100:\n\t\tsidFloor = 100\n\tcase subsPerConnection <= 1000:\n\t\tsidFloor = 1000\n\tcase subsPerConnection <= 10000:\n\t\tsidFloor = 10000\n\tdefault:\n\t\tb.Fatalf(\"Unsupported SubsPerConnection argument of %d\\n\", subsPerConnection)\n\t}\n\n\tmsgOp := fmt.Sprintf(\"MSG %s %d %d\\r\\n%s\\r\\n\", subject, sidFloor, len(payload), payload)\n\texpected := len(msgOp) * subsPerConnection * b.N\n\n\t// Client connections and subscriptions.\n\tclients := make([]chan bool, 0, numConnections)\n\tfor i := 0; i < numConnections; i++ {\n\t\tc := createClientConn(b, o2.Host, o2.Port)\n\t\tdoDefaultConnect(b, c)\n\t\tdefer c.Close()\n\n\t\tch := make(chan bool)\n\t\tclients = append(clients, ch)\n\n\t\tfor s := 0; s < subsPerConnection; s++ {\n\t\t\tsubOp := fmt.Sprintf(\"SUB %s %d\\r\\n\", subject, sidFloor+s)\n\t\t\tsendProto(b, c, subOp)\n\t\t}\n\t\tflushConnection(b, c)\n\t\tgo drainConnection(b, c, ch, expected)\n\t}\n\t// Publish Connection\n\tc := createClientConn(b, o1.Host, o1.Port)\n\tdoDefaultConnect(b, c)\n\tflushConnection(b, c)\n\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tsendOp := []byte(fmt.Sprintf(\"PUB %s %d\\r\\n%s\\r\\n\", subject, len(payload), payload))\n\n\tb.SetBytes(int64(len(sendOp) + (len(msgOp) * numConnections * subsPerConnection)))\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := bw.Write(sendOp)\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"Received error on PUB write: %v\\n\", err)\n\t\t}\n\t}\n\terr := bw.Flush()\n\tif err != nil {\n\t\tb.Fatalf(\"Received error on FLUSH write: %v\\n\", err)\n\t}\n\n\t// Wait for connections to be drained\n\tfor i := 0; i < numConnections; i++ {\n\t\t<-clients[i]\n\t}\n\tb.StopTimer()\n}\n\nvar sub = \"x\"\nvar payload = \"12345678\"\n\nfunc Benchmark______FanOut_8x1x10(b *testing.B) {\n\tdoFanout(b, 1, 1, 10, sub, payload)\n}\n\nfunc Benchmark_____FanOut_8x1x100(b *testing.B) {\n\tdoFanout(b, 1, 1, 100, sub, payload)\n}\n\nfunc Benchmark____FanOut_8x10x100(b *testing.B) {\n\tdoFanout(b, 1, 10, 100, sub, payload)\n}\n\nfunc Benchmark___FanOut_8x10x1000(b *testing.B) {\n\tdoFanout(b, 1, 10, 1000, sub, payload)\n}\n\nfunc Benchmark___FanOut_8x100x100(b *testing.B) {\n\tdoFanout(b, 1, 100, 100, sub, payload)\n}\n\nfunc Benchmark__FanOut_8x100x1000(b *testing.B) {\n\tdoFanout(b, 1, 100, 1000, sub, payload)\n}\n\nfunc Benchmark__FanOut_8x10x10000(b *testing.B) {\n\tdoFanout(b, 1, 10, 10000, sub, payload)\n}\n\nfunc Benchmark___FanOut_8x500x100(b *testing.B) {\n\tdoFanout(b, 1, 500, 100, sub, payload)\n}\n\nfunc Benchmark___FanOut_128x1x100(b *testing.B) {\n\tdoFanout(b, 1, 1, 100, sub, sizedString(128))\n}\n\nfunc Benchmark__FanOut_128x10x100(b *testing.B) {\n\tdoFanout(b, 1, 10, 100, sub, sizedString(128))\n}\n\nfunc Benchmark_FanOut_128x10x1000(b *testing.B) {\n\tdoFanout(b, 1, 10, 1000, sub, sizedString(128))\n}\n\nfunc Benchmark_FanOut_128x100x100(b *testing.B) {\n\tdoFanout(b, 1, 100, 100, sub, sizedString(128))\n}\n\nfunc BenchmarkFanOut_128x100x1000(b *testing.B) {\n\tdoFanout(b, 1, 100, 1000, sub, sizedString(128))\n}\n\nfunc BenchmarkFanOut_128x10x10000(b *testing.B) {\n\tdoFanout(b, 1, 10, 10000, sub, sizedString(128))\n}\n\nfunc BenchmarkFanOut__128x500x100(b *testing.B) {\n\tdoFanout(b, 1, 500, 100, sub, sizedString(128))\n}\n\nfunc Benchmark_FanOut_512x100x100(b *testing.B) {\n\tdoFanout(b, 1, 100, 100, sub, sizedString(512))\n}\n\nfunc Benchmark__FanOut_512x100x1k(b *testing.B) {\n\tdoFanout(b, 1, 100, 1000, sub, sizedString(512))\n}\n\nfunc Benchmark____FanOut_1kx10x1k(b *testing.B) {\n\tdoFanout(b, 1, 10, 1000, sub, sizedString(1024))\n}\n\nfunc Benchmark__FanOut_1kx100x100(b *testing.B) {\n\tdoFanout(b, 1, 100, 100, sub, sizedString(1024))\n}\n\nfunc Benchmark_____RFanOut_8x1x10(b *testing.B) {\n\tdoFanout(b, 2, 1, 10, sub, payload)\n}\n\nfunc Benchmark____RFanOut_8x1x100(b *testing.B) {\n\tdoFanout(b, 2, 1, 100, sub, payload)\n}\n\nfunc Benchmark___RFanOut_8x10x100(b *testing.B) {\n\tdoFanout(b, 2, 10, 100, sub, payload)\n}\n\nfunc Benchmark__RFanOut_8x10x1000(b *testing.B) {\n\tdoFanout(b, 2, 10, 1000, sub, payload)\n}\n\nfunc Benchmark__RFanOut_8x100x100(b *testing.B) {\n\tdoFanout(b, 2, 100, 100, sub, payload)\n}\n\nfunc Benchmark_RFanOut_8x100x1000(b *testing.B) {\n\tdoFanout(b, 2, 100, 1000, sub, payload)\n}\n\nfunc Benchmark_RFanOut_8x10x10000(b *testing.B) {\n\tdoFanout(b, 2, 10, 10000, sub, payload)\n}\n\nfunc Benchmark_RFanOut_1kx10x1000(b *testing.B) {\n\tdoFanout(b, 2, 10, 1000, sub, sizedString(1024))\n}\n\nfunc doFanIn(b *testing.B, numServers, numPublishers, numSubscribers int, subject, payload string) {\n\tb.Helper()\n\tif b.N < numPublishers {\n\t\treturn\n\t}\n\t// Don't check for number of subscribers being lower than the number of publishers.\n\t// We also use this bench to show the performance impact of increased number of publishers,\n\t// and for those tests, the number of publishers will start at 1 and increase to 10,\n\t// while the number of subscribers will always be 3.\n\tif numSubscribers > 10 {\n\t\tb.Fatalf(\"numSubscribers should be <= 10\")\n\t}\n\n\tvar s1, s2 *server.Server\n\tvar o1, o2 *server.Options\n\n\tswitch numServers {\n\tcase 1:\n\t\ts1, o1 = RunServerWithConfig(\"./configs/srv_a.conf\")\n\t\to1.DisableShortFirstPing = true\n\t\tdefer s1.Shutdown()\n\t\ts2, o2 = s1, o1\n\tcase 2:\n\t\ts1, o1 = RunServerWithConfig(\"./configs/srv_a.conf\")\n\t\to1.DisableShortFirstPing = true\n\t\tdefer s1.Shutdown()\n\t\ts2, o2 = RunServerWithConfig(\"./configs/srv_b.conf\")\n\t\to2.DisableShortFirstPing = true\n\t\tdefer s2.Shutdown()\n\tdefault:\n\t\tb.Fatalf(\"%d servers not supported for this test\\n\", numServers)\n\t}\n\n\tmsgOp := fmt.Sprintf(\"MSG %s %d %d\\r\\n%s\\r\\n\", subject, 9, len(payload), payload)\n\tl := b.N / numPublishers\n\texpected := len(msgOp) * l * numPublishers\n\n\t// Client connections and subscriptions. For fan in these are smaller then numPublishers.\n\tclients := make([]chan bool, 0, numSubscribers)\n\tfor i := 0; i < numSubscribers; i++ {\n\t\tc := createClientConn(b, o2.Host, o2.Port)\n\t\tdoDefaultConnect(b, c)\n\t\tdefer c.Close()\n\n\t\tch := make(chan bool)\n\t\tclients = append(clients, ch)\n\n\t\tsubOp := fmt.Sprintf(\"SUB %s %d\\r\\n\", subject, i)\n\t\tsendProto(b, c, subOp)\n\t\tflushConnection(b, c)\n\t\tgo drainConnection(b, c, ch, expected)\n\t}\n\n\tsendOp := []byte(fmt.Sprintf(\"PUB %s %d\\r\\n%s\\r\\n\", subject, len(payload), payload))\n\tstartCh := make(chan bool)\n\n\tpubLoop := func(c net.Conn, ch chan bool) {\n\t\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\n\t\t// Signal we are ready\n\t\tclose(ch)\n\n\t\t// Wait to start up actual sends.\n\t\t<-startCh\n\n\t\tfor i := 0; i < l; i++ {\n\t\t\t_, err := bw.Write(sendOp)\n\t\t\tif err != nil {\n\t\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\terr := bw.Flush()\n\t\tif err != nil {\n\t\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t\t\treturn\n\t\t}\n\t}\n\n\t// Publish Connections SPINUP\n\tfor i := 0; i < numPublishers; i++ {\n\t\tc := createClientConn(b, o1.Host, o1.Port)\n\t\tdoDefaultConnect(b, c)\n\t\tflushConnection(b, c)\n\t\tch := make(chan bool)\n\n\t\tgo pubLoop(c, ch)\n\t\t<-ch\n\t}\n\n\tb.SetBytes(int64(len(sendOp) + len(msgOp)))\n\tb.ResetTimer()\n\n\t// Closing this will start all publishers at once (roughly)\n\tclose(startCh)\n\n\t// Wait for connections to be drained\n\tfor i := 0; i < len(clients); i++ {\n\t\t<-clients[i]\n\t}\n\tb.StopTimer()\n}\n\nfunc Benchmark_____FanIn_1kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(1024))\n}\n\nfunc Benchmark_____FanIn_4kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(4096))\n}\n\nfunc Benchmark_____FanIn_8kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(8192))\n}\n\nfunc Benchmark____FanIn_16kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(16384))\n}\n\nfunc Benchmark____FanIn_64kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(65536))\n}\n\nfunc Benchmark___FanIn_128kx100x1(b *testing.B) {\n\tdoFanIn(b, 1, 100, 1, sub, sizedString(65536*2))\n}\n\nfunc Benchmark___BumpPubCount_1x3(b *testing.B) {\n\tdoFanIn(b, 1, 1, 3, sub, sizedString(128))\n}\n\nfunc Benchmark___BumpPubCount_2x3(b *testing.B) {\n\tdoFanIn(b, 1, 2, 3, sub, sizedString(128))\n}\n\nfunc Benchmark___BumpPubCount_5x3(b *testing.B) {\n\tdoFanIn(b, 1, 5, 3, sub, sizedString(128))\n}\n\nfunc Benchmark__BumpPubCount_10x3(b *testing.B) {\n\tdoFanIn(b, 1, 10, 3, sub, sizedString(128))\n}\n\nfunc testDefaultBenchOptionsForGateway(name string) *server.Options {\n\topts := testDefaultOptionsForGateway(name)\n\topts.DisableShortFirstPing = true\n\treturn opts\n}\n\nfunc gatewaysBench(b *testing.B, optimisticMode bool, payload string, numPublishers int, subInterest bool) {\n\tb.Helper()\n\tif b.N < numPublishers {\n\t\treturn\n\t}\n\n\tob := testDefaultBenchOptionsForGateway(\"B\")\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tserver.SetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tgwbURL, err := url.Parse(fmt.Sprintf(\"nats://%s:%d\", ob.Gateway.Host, ob.Gateway.Port))\n\tif err != nil {\n\t\tb.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\toa := testDefaultBenchOptionsForGateway(\"A\")\n\toa.Gateway.Gateways = []*server.RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tURLs: []*url.URL{gwbURL},\n\t\t},\n\t}\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tsub := createClientConn(b, ob.Host, ob.Port)\n\tdefer sub.Close()\n\tdoDefaultConnect(b, sub)\n\tsendProto(b, sub, \"SUB end.test 1\\r\\n\")\n\tif subInterest {\n\t\tsendProto(b, sub, \"SUB foo 2\\r\\n\")\n\t}\n\tflushConnection(b, sub)\n\n\t// If not optimisticMode, make B switch GW connection\n\t// to interest mode only\n\tif !optimisticMode {\n\t\tpub := createClientConn(b, oa.Host, oa.Port)\n\t\tdoDefaultConnect(b, pub)\n\t\t// has to be more that defaultGatewayMaxRUnsubBeforeSwitch\n\t\tfor i := 0; i < 2000; i++ {\n\t\t\tsendProto(b, pub, fmt.Sprintf(\"PUB reject.me.%d 2\\r\\nok\\r\\n\", i+1))\n\t\t}\n\t\tflushConnection(b, pub)\n\t\tpub.Close()\n\t}\n\n\tch := make(chan bool)\n\tvar msgOp string\n\tvar expected int\n\tl := b.N / numPublishers\n\tif subInterest {\n\t\tmsgOp = fmt.Sprintf(\"MSG foo 2 %d\\r\\n%s\\r\\n\", len(payload), payload)\n\t\texpected = len(msgOp) * l * numPublishers\n\t}\n\t// Last message sent to end.test\n\tlastMsg := \"MSG end.test 1 2\\r\\nok\\r\\n\"\n\texpected += len(lastMsg) * numPublishers\n\tgo drainConnection(b, sub, ch, expected)\n\n\tsendOp := []byte(fmt.Sprintf(\"PUB foo %d\\r\\n%s\\r\\n\", len(payload), payload))\n\tstartCh := make(chan bool)\n\n\tlastMsgSendOp := []byte(\"PUB end.test 2\\r\\nok\\r\\n\")\n\n\tpubLoop := func(c net.Conn, ch chan bool) {\n\t\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\n\t\t// Signal we are ready\n\t\tclose(ch)\n\n\t\t// Wait to start up actual sends.\n\t\t<-startCh\n\n\t\tfor i := 0; i < l; i++ {\n\t\t\tif _, err := bw.Write(sendOp); err != nil {\n\t\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tif _, err := bw.Write(lastMsgSendOp); err != nil {\n\t\t\tb.Errorf(\"Received error on PUB write: %v\\n\", err)\n\t\t\treturn\n\t\t}\n\t\tif err := bw.Flush(); err != nil {\n\t\t\tb.Errorf(\"Received error on FLUSH write: %v\\n\", err)\n\t\t\treturn\n\t\t}\n\t\tflushConnection(b, c)\n\t}\n\n\t// Publish Connections SPINUP\n\tfor i := 0; i < numPublishers; i++ {\n\t\tc := createClientConn(b, oa.Host, oa.Port)\n\t\tdefer c.Close()\n\t\tdoDefaultConnect(b, c)\n\t\tflushConnection(b, c)\n\t\tch := make(chan bool)\n\n\t\tgo pubLoop(c, ch)\n\t\t<-ch\n\t}\n\n\t// To report the number of bytes:\n\t// from publisher to server on cluster A:\n\tnumBytes := len(sendOp)\n\tif subInterest {\n\t\t// from server in cluster A to server on cluster B:\n\t\t// RMSG $G foo \\r\\n\n\t\tnumBytes += len(\"RMSG $G foo xxxx \") + len(payload) + 2\n\n\t\t// From server in cluster B to sub:\n\t\tnumBytes += len(msgOp)\n\t}\n\tb.SetBytes(int64(numBytes))\n\tb.ResetTimer()\n\n\t// Closing this will start all publishers at once (roughly)\n\tclose(startCh)\n\n\t// Wait for end of test\n\t<-ch\n\n\tb.StopTimer()\n}\n\nfunc Benchmark____GWs_Opt_1kx01x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(1024), 1, false)\n}\n\nfunc Benchmark____GWs_Opt_2kx01x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(2048), 1, false)\n}\n\nfunc Benchmark____GWs_Opt_4kx01x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(4096), 1, false)\n}\n\nfunc Benchmark____GWs_Opt_1kx10x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(1024), 10, false)\n}\n\nfunc Benchmark____GWs_Opt_2kx10x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(2048), 10, false)\n}\n\nfunc Benchmark____GWs_Opt_4kx10x0(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(4096), 10, false)\n}\n\nfunc Benchmark____GWs_Opt_1kx01x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(1024), 1, true)\n}\n\nfunc Benchmark____GWs_Opt_2kx01x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(2048), 1, true)\n}\n\nfunc Benchmark____GWs_Opt_4kx01x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(4096), 1, true)\n}\n\nfunc Benchmark____GWs_Opt_1kx10x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(1024), 10, true)\n}\n\nfunc Benchmark____GWs_Opt_2kx10x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(2048), 10, true)\n}\n\nfunc Benchmark____GWs_Opt_4kx10x1(b *testing.B) {\n\tgatewaysBench(b, true, sizedString(4096), 10, true)\n}\n\nfunc Benchmark____GWs_Int_1kx01x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(1024), 1, false)\n}\n\nfunc Benchmark____GWs_Int_2kx01x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(2048), 1, false)\n}\n\nfunc Benchmark____GWs_Int_4kx01x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(4096), 1, false)\n}\n\nfunc Benchmark____GWs_Int_1kx10x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(1024), 10, false)\n}\n\nfunc Benchmark____GWs_Int_2kx10x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(2048), 10, false)\n}\n\nfunc Benchmark____GWs_Int_4kx10x0(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(4096), 10, false)\n}\n\nfunc Benchmark____GWs_Int_1kx01x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(1024), 1, true)\n}\n\nfunc Benchmark____GWs_Int_2kx01x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(2048), 1, true)\n}\n\nfunc Benchmark____GWs_Int_4kx01x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(4096), 1, true)\n}\n\nfunc Benchmark____GWs_Int_1kx10x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(1024), 10, true)\n}\n\nfunc Benchmark____GWs_Int_2kx10x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(2048), 10, true)\n}\n\nfunc Benchmark____GWs_Int_4kx10x1(b *testing.B) {\n\tgatewaysBench(b, false, sizedString(4096), 10, true)\n}\n\n// This bench only sends the requests to verify impact\n// of reply mapping in GW code.\nfunc gatewaySendRequestsBench(b *testing.B, singleReplySub bool) {\n\tserver.SetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tob := testDefaultBenchOptionsForGateway(\"B\")\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tgwbURL, err := url.Parse(fmt.Sprintf(\"nats://%s:%d\", ob.Gateway.Host, ob.Gateway.Port))\n\tif err != nil {\n\t\tb.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\toa := testDefaultBenchOptionsForGateway(\"A\")\n\toa.Gateway.Gateways = []*server.RemoteGatewayOpts{\n\t\t{\n\t\t\tName: \"B\",\n\t\t\tURLs: []*url.URL{gwbURL},\n\t\t},\n\t}\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tsub := createClientConn(b, ob.Host, ob.Port)\n\tdefer sub.Close()\n\tdoDefaultConnect(b, sub)\n\tsendProto(b, sub, \"SUB foo 1\\r\\n\")\n\tflushConnection(b, sub)\n\n\tlenMsg := len(\"MSG foo reply.xxxxxxxxxx 1 2\\r\\nok\\r\\n\")\n\texpected := b.N * lenMsg\n\tch := make(chan bool, 1)\n\tgo drainConnection(b, sub, ch, expected)\n\n\tc := createClientConn(b, oa.Host, oa.Port)\n\tdefer c.Close()\n\tdoDefaultConnect(b, c)\n\tflushConnection(b, c)\n\n\t// From pub to server in cluster A:\n\tnumBytes := len(\"PUB foo reply.0123456789 2\\r\\nok\\r\\n\")\n\tif !singleReplySub {\n\t\t// Add the preceding SUB\n\t\tnumBytes += len(\"SUB reply.0123456789 0123456789\\r\\n\")\n\t\t// And UNSUB...\n\t\tnumBytes += len(\"UNSUB 0123456789\\r\\n\")\n\t}\n\t// From server in cluster A to cluster B\n\tnumBytes += len(\"RMSG $G foo reply.0123456789 2\\r\\nok\\r\\n\")\n\t// If mapping of reply...\n\tif !singleReplySub {\n\t\t// the mapping uses about 24 more bytes. So add them\n\t\t// for RMSG from server to server.\n\t\tnumBytes += 24\n\t}\n\t// From server in cluster B to sub\n\tnumBytes += lenMsg\n\tb.SetBytes(int64(numBytes))\n\n\tbw := bufio.NewWriterSize(c, defaultSendBufSize)\n\tvar subStr string\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tif !singleReplySub {\n\t\t\tsubStr = fmt.Sprintf(\"SUB reply.%010d %010d\\r\\n\", i+1, i+1)\n\t\t}\n\t\tbw.Write([]byte(fmt.Sprintf(\"%sPUB foo reply.%010d 2\\r\\nok\\r\\n\", subStr, i+1)))\n\t\t// Simulate that we are doing actual request/reply and therefore\n\t\t// unsub'ing the subs on the reply subject.\n\t\tif !singleReplySub && i > 1000 {\n\t\t\tbw.Write([]byte(fmt.Sprintf(\"UNSUB %010d\\r\\n\", (i - 1000))))\n\t\t}\n\t}\n\tbw.Flush()\n\tflushConnection(b, c)\n\n\t<-ch\n}\n\nfunc Benchmark__GWs_Reqs_1_SubAll(b *testing.B) {\n\tgatewaySendRequestsBench(b, true)\n}\n\nfunc Benchmark__GWs_Reqs_1SubEach(b *testing.B) {\n\tgatewaySendRequestsBench(b, false)\n}\n" }, { "path": "test/client_auth_test.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc TestMultipleUserAuth(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/multi_user.conf\")\n\tdefer srv.Shutdown()\n\n\tif opts.Users == nil {\n\t\tt.Fatal(\"Expected a user array that is not nil\")\n\t}\n\tif len(opts.Users) != 2 {\n\t\tt.Fatal(\"Expected a user array that had 2 users\")\n\t}\n\n\t// Test first user\n\turl := fmt.Sprintf(\"nats://%s:%s@%s:%d/\",\n\t\topts.Users[0].Username,\n\t\topts.Users[0].Password,\n\t\topts.Host, opts.Port)\n\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a successful connect, got %v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tif !nc.AuthRequired() {\n\t\tt.Fatal(\"Expected auth to be required for the server\")\n\t}\n\n\t// Test second user\n\turl = fmt.Sprintf(\"nats://%s:%s@%s:%d/\",\n\t\topts.Users[1].Username,\n\t\topts.Users[1].Password,\n\t\topts.Host, opts.Port)\n\n\tnc, err = nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a successful connect, got %v\\n\", err)\n\t}\n\tdefer nc.Close()\n}\n\n// Resolves to \"test\"\nconst testToken = \"$2a$05$3sSWEVA1eMCbV0hWavDjXOx.ClBjI6u1CuUdLqf22cbJjXsnzz8/.\"\n\nfunc TestTokenInConfig(t *testing.T) {\n\tcontent := `\n\tlisten: 127.0.0.1:4567\n\tauthorization={\n\t\ttoken: ` + testToken + `\n\t\ttimeout: 5\n\t}`\n\tconfFile := createConfFile(t, []byte(content))\n\tif err := os.WriteFile(confFile, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config file: %v\", err)\n\t}\n\ts, opts := RunServerWithConfig(confFile)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://test@%s:%d/\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a successful connect, got %v\\n\", err)\n\t}\n\tdefer nc.Close()\n\tif !nc.AuthRequired() {\n\t\tt.Fatal(\"Expected auth to be required for the server\")\n\t}\n}\n\nfunc TestClientConnectInfo(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsys string\n\t\thasSys bool\n\t\tprotocol int\n\t}{\n\t\t{\"async info support with explicit system account\", \"system_account: SYS\", true, server.ClientProtoInfo},\n\t\t{\"async info support without explicit system account\", \"\", false, server.ClientProtoInfo},\n\t\t{\"no async info support with explicit system account\", \"system_account: SYS\", true, server.ClientProtoZero},\n\t\t{\"no async info support without explicit system account\", \"\", false, server.ClientProtoZero},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\t\t\t\taccounts {\n\t\t\t\t\tSYS: { users: [{ user: sys, password: pwd}] }\n\t\t\t\t\tA: { users: [{ user: a, password: pwd}] }\n\t\t\t\t\tB: { users: [{ user: b, password: pwd}] }\n\t\t\t\t}\n\t\t\t`, test.sys)))\n\t\t\thub, oHub := RunServerWithConfig(conf)\n\t\t\tdefer hub.Shutdown()\n\n\t\t\tcheckInfoOnConnect := func(user, acc string, isSys bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tcc := createClientConn(t, oHub.Host, oHub.Port)\n\t\t\t\tdefer cc.Close()\n\n\t\t\t\tcheckInfoMsg(t, cc)\n\t\t\t\tsendProto(t, cc, fmt.Sprintf(\"CONNECT {\\\"user\\\":%q,\\\"pass\\\":\\\"pwd\\\",\\\"pedantic\\\":false,\\\"verbose\\\":false,\\\"protocol\\\":%d}\\r\\nPING\\r\\n\", user, test.protocol))\n\t\t\t\t// Since the PONG and INFO may be receive as a single TCP read,\n\t\t\t\t// make sure we consume the pong alone here.\n\t\t\t\tpongBuf := make([]byte, len(\"PONG\\r\\n\"))\n\t\t\t\tcc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\t\tn, err := cc.Read(pongBuf)\n\t\t\t\tcc.SetReadDeadline(time.Time{})\n\t\t\t\tif n <= 0 && err != nil {\n\t\t\t\t\tt.Fatalf(\"Error reading from conn: %v\\n\", err)\n\t\t\t\t}\n\t\t\t\tif !pongRe.Match(pongBuf) {\n\t\t\t\t\tt.Fatalf(\"Response did not match expected: \\n\\tReceived:'%q'\\n\\tExpected:'%s'\\n\", pongBuf, pongRe)\n\t\t\t\t}\n\t\t\t\tif test.protocol < server.ClientProtoInfo {\n\t\t\t\t\texpectNothing(t, cc)\n\t\t\t\t} else {\n\t\t\t\t\tinfo := checkInfoMsg(t, cc)\n\t\t\t\t\tif !info.ConnectInfo {\n\t\t\t\t\t\tt.Fatal(\"Expected ConnectInfo to be true\")\n\t\t\t\t\t}\n\t\t\t\t\tif an := info.RemoteAccount; an != acc {\n\t\t\t\t\t\tt.Fatalf(\"Expected account %q, got %q\", acc, info.RemoteAccount)\n\t\t\t\t\t}\n\t\t\t\t\tif ais := info.IsSystemAccount; ais != isSys {\n\t\t\t\t\t\tt.Fatalf(\"Expected IsSystemAccount to be %v, got %v\", isSys, ais)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcheckInfoOnConnect(\"a\", \"A\", false)\n\t\t\tcheckInfoOnConnect(\"sys\", \"SYS\", test.hasSys)\n\t\t\tcheckInfoOnConnect(\"b\", \"B\", false)\n\t\t})\n\t}\n}\n\ntype captureProxiesReloadLogger struct {\n\tdummyLogger\n\tch chan string\n}\n\nfunc (l *captureProxiesReloadLogger) Noticef(format string, args ...any) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif strings.Contains(msg, \"proxies trusted keys\") {\n\t\tl.ch <- msg\n\t}\n}\n\nfunc TestProxyKeyVerification(t *testing.T) {\n\tu1, _ := nkeys.CreateUser()\n\tu1Pub, _ := u1.PublicKey()\n\n\tu2, _ := nkeys.CreateUser()\n\tu2Pub, _ := u2.PublicKey()\n\n\tu3, _ := nkeys.CreateUser()\n\tu3Pub, _ := u3.PublicKey()\n\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tproxies {\n\t\t\ttrusted [\n\t\t\t\t{key: %q}\n\t\t\t\t{key: %q}\n\t\t\t]\n\t\t}\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`\n\tconf := createConfFile(t, fmt.Appendf(nil, tmpl, u1Pub, u2Pub))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tvarz, err := s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting varz: %v\", err)\n\t}\n\tvproxies := varz.Proxies\n\tif vproxies == nil {\n\t\tt.Fatal(\"Expected proxies to be not nil\")\n\t}\n\ttrusted := vproxies.Trusted\n\tif len(trusted) != 2 {\n\t\tt.Fatalf(\"Expected 2 trusted proxies, got %+v\", trusted)\n\t}\n\tif k := trusted[0].Key; k != u1Pub {\n\t\tt.Fatalf(\"Expected key to be %q, got %q\", u1Pub, k)\n\t}\n\tif k := trusted[1].Key; k != u2Pub {\n\t\tt.Fatalf(\"Expected key to be %q, got %q\", u2Pub, k)\n\t}\n\n\tvar currentCID uint64\n\tconnect := func(t *testing.T, kp nkeys.KeyPair, bsig bool, port int, isClient bool, user string) (expectFun, net.Conn) {\n\t\tc := createClientConn(t, \"127.0.0.1\", port)\n\t\texpect := expectCommand(t, c)\n\t\tinfo := checkInfoMsg(t, c)\n\t\tcurrentCID = info.CID\n\t\tif info.Nonce == \"\" {\n\t\t\tc.Close()\n\t\t\tt.Fatalf(\"Expected nonce from INFO, got %+v\", info)\n\t\t}\n\t\tif info.JSApiLevel != server.JSApiLevel {\n\t\t\tc.Close()\n\t\t\tt.Fatalf(\"Expected JSApiLevel to be set to %v, got %v\", server.JSApiLevel, info.JSApiLevel)\n\t\t}\n\t\tsig, err := kp.Sign([]byte(info.Nonce))\n\t\tif err != nil {\n\t\t\tc.Close()\n\t\t\tt.Fatalf(\"Error signing nonce %q: %v\", info.Nonce, err)\n\t\t}\n\t\tencodedSig := make([]byte, base64.RawURLEncoding.EncodedLen(len(sig)))\n\t\tbase64.RawURLEncoding.Encode(encodedSig, sig)\n\t\tif bsig {\n\t\t\tencodedSig[0] = '*'\n\t\t}\n\t\tsendProto(t, c,\n\t\t\tfmt.Sprintf(\"CONNECT {\\\"pedantic\\\":false,\\\"verbose\\\":false,\\\"user\\\":%q,\\\"pass\\\":\\\"pwd\\\",\\\"proxy_sig\\\":\\\"%s\\\"}\\r\\n\",\n\t\t\t\tuser, string(encodedSig)))\n\t\tif isClient {\n\t\t\tsendProto(t, c, \"PING\\r\\n\")\n\t\t}\n\t\treturn expect, c\n\t}\n\n\tcheckClosedState := func(t *testing.T, cid uint64, reason server.ClosedState) {\n\t\t// Closed connection getting in the closed state is done in a go routine\n\t\t// so make sure we wait for it.\n\t\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\t\tconnz, err := s.Connz(&server.ConnzOptions{\n\t\t\t\tCID: cid,\n\t\t\t\tState: server.ConnClosed,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"Error getting connz for closed connection %v: %v\", cid, err)\n\t\t\t}\n\t\t\tif len(connz.Conns) != 1 {\n\t\t\t\treturn fmt.Errorf(\"Expected 1 connection, got %v\", connz.Conns)\n\t\t\t}\n\t\t\tconn := connz.Conns[0]\n\t\t\tif conn.Reason != reason.String() {\n\t\t\t\treturn fmt.Errorf(\"Expected reason %q, got %q\", reason.String(), conn.Reason)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tkp nkeys.KeyPair\n\t\tbsig bool\n\t\tok bool\n\t}{\n\t\t{\"key present first\", u1, false, true},\n\t\t{\"key present last\", u2, false, true},\n\t\t{\"key not present\", u3, false, false},\n\t\t{\"bad signature\", u2, true, false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tfor _, subTest := range []struct {\n\t\t\t\tname string\n\t\t\t\tport int\n\t\t\t\tisClient bool\n\t\t\t}{\n\t\t\t\t{\"client\", o.Port, true},\n\t\t\t\t{\"leafnodes\", o.LeafNode.Port, false},\n\t\t\t} {\n\t\t\t\tt.Run(subTest.name, func(t *testing.T) {\n\t\t\t\t\texpect, c := connect(t, test.kp, test.bsig, subTest.port, subTest.isClient, \"user\")\n\t\t\t\t\tdefer c.Close()\n\t\t\t\t\tif test.ok {\n\t\t\t\t\t\tvar p *server.ProxyInfo\n\t\t\t\t\t\tif subTest.isClient {\n\t\t\t\t\t\t\texpect(pongRe)\n\n\t\t\t\t\t\t\tconnz, err := s.Connz(nil)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Error getting connz: %v\", err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif n := len(connz.Conns); n != 1 {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expected 1 ConnInfo, got %v\", n)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tci := connz.Conns[0]\n\t\t\t\t\t\t\tp = ci.Proxy\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\texpect(infoRe)\n\n\t\t\t\t\t\t\tleafz, err := s.Leafz(nil)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Error getting leafz: %v\", err)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif n := len(leafz.Leafs); n != 1 {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Expected 1 LeafInfo, got %v\", n)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tli := leafz.Leafs[0]\n\t\t\t\t\t\t\tif li.Proxy == nil {\n\t\t\t\t\t\t\t\tt.Fatalf(\"Required Proxy to be present, was not: %+v\", li)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tp = li.Proxy\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif p == nil {\n\t\t\t\t\t\t\tt.Fatalf(\"Required Proxy to be present, was not\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\tpub, _ := test.kp.PublicKey()\n\t\t\t\t\t\tif p.Key != pub {\n\t\t\t\t\t\t\tt.Fatalf(\"Expected public key to be %q, got %q\", pub, p.Key)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\texpect(errRe)\n\t\t\t\t\t\tcheckClosedState(t, currentCID, server.ProxyNotTrusted)\n\t\t\t\t\t}\n\t\t\t\t})\n\t\t\t}\n\t\t})\n\t}\n\n\t// Create a connection using u1Pub and we will config reload and remove\n\t// that key from the list. The client should get disconnected.\n\texpectc, c := connect(t, u1, false, o.Port, true, \"user\")\n\tdefer c.Close()\n\texpectc(pongRe)\n\t// Capture this connection CID before the next connect.\n\tcid1 := currentCID\n\n\texpectl, l := connect(t, u1, false, o.LeafNode.Port, false, \"user\")\n\tdefer l.Close()\n\texpectl(infoRe)\n\tcid2 := currentCID\n\tcheckLeafNodeConnected(t, s)\n\n\tlogger := &captureProxiesReloadLogger{ch: make(chan string, 10)}\n\ts.SetLogger(logger, false, false)\n\n\tos.WriteFile(conf, fmt.Appendf(nil, tmpl, u3Pub, u2Pub), 0660)\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Reload failed: %v\", err)\n\t}\n\tfor range 2 {\n\t\tselect {\n\t\tcase str := <-logger.ch:\n\t\t\tif strings.Contains(str, \"removed\") {\n\t\t\t\tif !strings.Contains(str, u1Pub) {\n\t\t\t\t\tt.Fatalf(\"Expected removed trace to include %q, it did not: %s\", u1Pub, str)\n\t\t\t\t}\n\t\t\t} else if strings.Contains(str, \"added\") {\n\t\t\t\tif !strings.Contains(str, u3Pub) {\n\t\t\t\t\tt.Fatalf(\"Expected added trace to include %q, it did not: %s\", u3Pub, str)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"Unexpected log: %q\", str)\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Fatal(\"Expected a log, did not get one\")\n\t\t}\n\t}\n\n\t// Connections should get disconnected.\n\t// We need to consume what is sent by the server, but for leaf we may\n\t// get some LS+, etc... so just consumer until we get the io.EOF\n\treadAll := func(c net.Conn) {\n\t\tfor {\n\t\t\tvar buf [1024]byte\n\t\t\tif _, err := c.Read(buf[:]); err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treadAll(c)\n\texpectDisconnect(t, c)\n\treadAll(l)\n\texpectDisconnect(t, l)\n\tc.Close()\n\tl.Close()\n\n\t// Now check that the connection with CID `cid` was closed because the\n\t// proxy is (no longer) trusted.\n\tcheckClosedState(t, cid1, server.ProxyNotTrusted)\n\tcheckClosedState(t, cid2, server.ProxyNotTrusted)\n\n\tvarz, err = s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting varz: %v\", err)\n\t}\n\tvproxies = varz.Proxies\n\tif vproxies == nil {\n\t\tt.Fatal(\"Expected proxies to be not nil\")\n\t}\n\ttrusted = vproxies.Trusted\n\tif len(trusted) != 2 {\n\t\tt.Fatalf(\"Expected 2 trusted proxies, got %+v\", trusted)\n\t}\n\tif k := trusted[0].Key; k != u3Pub {\n\t\tt.Fatalf(\"Expected key to be %q, got %q\", u3Pub, k)\n\t}\n\tif k := trusted[1].Key; k != u2Pub {\n\t\tt.Fatalf(\"Expected key to be %q, got %q\", u2Pub, k)\n\t}\n\n\t// And we should be able to connect using u3 key pair.\n\texpectc, c = connect(t, u3, false, o.Port, true, \"user\")\n\texpectc(pongRe)\n\tc.Close()\n\texpectl, l = connect(t, u3, false, o.LeafNode.Port, false, \"user\")\n\texpectl(infoRe)\n\tl.Close()\n\n\t// But now using u1 key pair should fail.\n\texpectc, c = connect(t, u1, false, o.Port, true, \"user\")\n\texpectc(errRe)\n\tc.Close()\n\texpectl, l = connect(t, u1, false, o.LeafNode.Port, false, \"user\")\n\texpectl(errRe)\n\tl.Close()\n\n\ts.Shutdown()\n\n\t// Now check that if no trusted proxy are provided, a proxy connection is\n\t// accepted. Use leafnodes since we always send NONCE for this type of connection.\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tbsig bool\n\t}{\n\t\t{\"no trusted proxy and correct signature\", false},\n\t\t{\"no trusted proxy and invalid signature\", true},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\texpect, c := connect(t, u1, test.bsig, o.LeafNode.Port, false, \"user\")\n\t\t\tdefer c.Close()\n\t\t\t// Does not matter if bad signature, since there is no trusted keys\n\t\t\t// in the server, we accept without signature check.\n\t\t\texpect(infoRe)\n\t\t})\n\t}\n\n\ts.Shutdown()\n\n\t// Finally, check that if there are no configured trusted proxies\n\t// and a user requires a proxy connection, then the connection will fail.\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tusers [\n\t\t\t\t\t{user: u1, password: pwd}\n\t\t\t\t\t{user: u2, password: pwd, proxy_required: true}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuser string\n\t\tok bool\n\t}{\n\t\t{\"no trusted proxy configured no proxy required\", \"u1\", true},\n\t\t{\"no trusted proxy configured and proxy required\", \"u2\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// We can use any key pair here, it is not important.\n\t\t\texpect, c := connect(t, u1, false, o.LeafNode.Port, false, test.user)\n\t\t\tdefer c.Close()\n\t\t\tif test.ok {\n\t\t\t\texpect(infoRe)\n\t\t\t} else {\n\t\t\t\texpect(errRe)\n\t\t\t\t// Check that the reason is \"proxy required\"\n\t\t\t\tcheckClosedState(t, currentCID, server.ProxyRequired)\n\t\t\t}\n\t\t})\n\t}\n\n\t// Same test, but the \"proxy_required\" is at the authorization{} to-level,\n\t// so any user should fail.\n\tconf = createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization {\n\t\t\t\tusers [\n\t\t\t\t\t{user: u1, password: pwd}\n\t\t\t\t\t{user: u2, password: pwd}\n\t\t\t\t]\n\t\t\t\tproxy_required: true\n\t\t\t}\n\t\t}\n\t`))\n\ts, o = RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tt.Run(\"all users required to use trusted proxy\", func(t *testing.T) {\n\t\tfor _, user := range []string{\"u1\", \"u2\"} {\n\t\t\t// We can use any key pair here, it is not important.\n\t\t\texpect, c := connect(t, u1, false, o.LeafNode.Port, false, user)\n\t\t\tdefer c.Close()\n\t\t\texpect(errRe)\n\t\t\t// Check that the reason is \"proxy required\"\n\t\t\tcheckClosedState(t, currentCID, server.ProxyRequired)\n\t\t}\n\t})\n\n\tvarz, err = s.Varz(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting varz: %v\", err)\n\t}\n\tvproxies = varz.Proxies\n\tif vproxies != nil {\n\t\tt.Fatalf(\"Expected proxies to be nil, got %+v\", vproxies)\n\t}\n}\n" }, { "path": "test/client_cluster_test.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc TestServerRestartReSliceIssue(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\t// msg to send..\n\tmsg := []byte(\"Hello World\")\n\n\tservers := []string{urlA, urlB}\n\n\topts := nats.GetDefaultOptions()\n\topts.Timeout = (5 * time.Second)\n\topts.ReconnectWait = (50 * time.Millisecond)\n\topts.MaxReconnect = 1000\n\n\tnumClients := 20\n\n\treconnects := int32(0)\n\treconnectsDone := make(chan bool, numClients)\n\topts.ReconnectedCB = func(nc *nats.Conn) {\n\t\tatomic.AddInt32(&reconnects, 1)\n\t\treconnectsDone <- true\n\t}\n\n\tclients := make([]*nats.Conn, numClients)\n\n\t// Create 20 random clients.\n\t// Half connected to A and half to B..\n\tfor i := 0; i < numClients; i++ {\n\t\topts.Url = servers[i%2]\n\t\tnc, err := opts.Connect()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to create connection: %v\\n\", err)\n\t\t}\n\t\tclients[i] = nc\n\t\tdefer nc.Close()\n\n\t\t// Create 10 subscriptions each..\n\t\tfor x := 0; x < 10; x++ {\n\t\t\tsubject := fmt.Sprintf(\"foo.%d\", (rand.Int()%50)+1)\n\t\t\tnc.Subscribe(subject, func(m *nats.Msg) {\n\t\t\t\t// Just eat it..\n\t\t\t})\n\t\t}\n\t\t// Pick one subject to send to..\n\t\tsubject := fmt.Sprintf(\"foo.%d\", (rand.Int()%50)+1)\n\t\tgo func() {\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\tfor i := 1; i <= 100; i++ {\n\t\t\t\tif err := nc.Publish(subject, msg); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif i%10 == 0 {\n\t\t\t\t\ttime.Sleep(time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// Wait for a short bit..\n\ttime.Sleep(20 * time.Millisecond)\n\n\t// Restart SrvB\n\tsrvB.Shutdown()\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Check that all expected clients have reconnected\n\tdone := false\n\tfor i := 0; i < numClients/2 && !done; i++ {\n\t\tselect {\n\t\tcase <-reconnectsDone:\n\t\t\tdone = true\n\t\tcase <-time.After(3 * time.Second):\n\t\t\tt.Fatalf(\"Expected %d reconnects, got %d\\n\", numClients/2, reconnects)\n\t\t}\n\t}\n\n\t// Since srvB was restarted, its defer Shutdown() was last, so will\n\t// exectue first, which would cause clients that have reconnected to\n\t// it to try to reconnect (causing delays on Windows). So let's\n\t// explicitly close them here.\n\t// NOTE: With fix of NATS GO client (reconnect loop yields to Close()),\n\t// this change would not be required, however, it still speeeds up\n\t// the test, from more than 7s to less than one.\n\tfor i := 0; i < numClients; i++ {\n\t\tnc := clients[i]\n\t\tnc.Close()\n\t}\n}\n\n// This will test queue subscriber semantics across a cluster in the presence\n// of server restarts.\nfunc TestServerRestartAndQueueSubs(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\t// Client options\n\topts := nats.GetDefaultOptions()\n\topts.Timeout = 5 * time.Second\n\topts.ReconnectWait = 20 * time.Millisecond\n\topts.MaxReconnect = 1000\n\topts.NoRandomize = true\n\n\t// Allow us to block on a reconnect completion.\n\treconnectsDone := make(chan bool)\n\topts.ReconnectedCB = func(nc *nats.Conn) {\n\t\treconnectsDone <- true\n\t}\n\n\t// Helper to wait on a reconnect.\n\twaitOnReconnect := func() {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase <-reconnectsDone:\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Expected a reconnect, timedout!\\n\")\n\t\t}\n\t}\n\n\t// Create two clients..\n\topts.Servers = []string{urlA, urlB}\n\tc1, err := opts.Connect()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for c1: %v\\n\", err)\n\t}\n\tdefer c1.Close()\n\n\topts.Servers = []string{urlB, urlA}\n\tc2, err := opts.Connect()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for c2: %v\\n\", err)\n\t}\n\tdefer c2.Close()\n\n\t// Flusher helper function.\n\tflush := func() {\n\t\t// Wait for processing.\n\t\tc1.Flush()\n\t\tc2.Flush()\n\t\t// Wait for a short bit for cluster propagation.\n\t\ttime.Sleep(50 * time.Millisecond)\n\t}\n\n\t// To hold queue results.\n\tresults := make(map[int]int)\n\tvar mu sync.Mutex\n\n\t// This corresponds to the subsriptions below.\n\tconst ExpectedMsgCount = 3\n\n\t// Make sure we got what we needed, 1 msg only and all seqnos accounted for..\n\tcheckResults := func(numSent int) {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\n\t\tfor i := 0; i < numSent; i++ {\n\t\t\tif results[i] != ExpectedMsgCount {\n\t\t\t\tt.Fatalf(\"Received incorrect number of messages, [%d] vs [%d] for seq: %d\\n\", results[i], ExpectedMsgCount, i)\n\t\t\t}\n\t\t}\n\n\t\t// Auto reset results map\n\t\tresults = make(map[int]int)\n\t}\n\n\tsubj := \"foo.bar\"\n\tqgroup := \"workers\"\n\n\tcb := func(msg *nats.Msg) {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tseqno, _ := strconv.Atoi(string(msg.Data))\n\t\tresults[seqno] = results[seqno] + 1\n\t}\n\n\t// Create queue subscribers\n\tc1.QueueSubscribe(subj, qgroup, cb)\n\tc2.QueueSubscribe(subj, qgroup, cb)\n\n\t// Do a wildcard subscription.\n\tc1.Subscribe(\"foo.*\", cb)\n\tc2.Subscribe(\"foo.*\", cb)\n\n\t// Wait for processing.\n\tflush()\n\n\tsendAndCheckMsgs := func(numToSend int) {\n\t\tfor i := 0; i < numToSend; i++ {\n\t\t\tif i%2 == 0 {\n\t\t\t\tc1.Publish(subj, []byte(strconv.Itoa(i)))\n\t\t\t} else {\n\t\t\t\tc2.Publish(subj, []byte(strconv.Itoa(i)))\n\t\t\t}\n\t\t}\n\t\t// Wait for processing.\n\t\tflush()\n\t\t// Check Results\n\t\tcheckResults(numToSend)\n\t}\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// Base Test\n\t////////////////////////////////////////////////////////////////////////////\n\n\t// Make sure subscriptions are propagated in the cluster\n\tif err := checkExpectedSubs(4, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\t// Now send 10 messages, from each client..\n\tsendAndCheckMsgs(10)\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// Now restart SrvA and srvB, re-run test\n\t////////////////////////////////////////////////////////////////////////////\n\n\tsrvA.Shutdown()\n\t// Wait for client on A to reconnect to B.\n\twaitOnReconnect()\n\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB.Shutdown()\n\t// Now both clients should reconnect to A.\n\twaitOnReconnect()\n\twaitOnReconnect()\n\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\t// Make sure the cluster is reformed\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Make sure subscriptions are propagated in the cluster\n\t// Clients will be connected to srvA, so that will be 4,\n\t// but srvB will only have 2 now since we coaelsce.\n\tif err := checkExpectedSubs(4, srvA); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\tif err := checkExpectedSubs(2, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\t// Now send another 10 messages, from each client..\n\tsendAndCheckMsgs(10)\n\n\t// Since servers are restarted after all client's close defer calls,\n\t// their defer Shutdown() are last, and so will be executed first,\n\t// which would cause clients to try to reconnect on exit, causing\n\t// delays on Windows. So let's explicitly close them here.\n\tc1.Close()\n\tc2.Close()\n}\n\n// This will test request semantics across a route\nfunc TestRequestsAcrossRoutes(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc1: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc2: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tresponse := []byte(\"I will help you\")\n\n\t// Connect responder to srvA\n\tnc1.Subscribe(\"foo-req\", func(m *nats.Msg) {\n\t\tnc1.Publish(m.Reply, response)\n\t})\n\t// Make sure the route and the subscription are propagated.\n\tnc1.Flush()\n\n\tif err = checkExpectedSubs(1, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err = nc2.Request(\"foo-req\", []byte(strconv.Itoa(i)), 250*time.Millisecond); err != nil {\n\t\t\tt.Fatalf(\"Received an error on Request test [%d]: %s\", i, err)\n\t\t}\n\t}\n}\n\n// This will test request semantics across a route to queues\nfunc TestRequestsAcrossRoutesToQueues(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc1: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc2: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tresponse := []byte(\"I will help you\")\n\n\t// Connect one responder to srvA\n\tnc1.QueueSubscribe(\"foo-req\", \"booboo\", func(m *nats.Msg) {\n\t\tnc1.Publish(m.Reply, response)\n\t})\n\t// Make sure the route and the subscription are propagated.\n\tnc1.Flush()\n\n\t// Connect the other responder to srvB\n\tnc2.QueueSubscribe(\"foo-req\", \"booboo\", func(m *nats.Msg) {\n\t\tnc2.Publish(m.Reply, response)\n\t})\n\n\tif err = checkExpectedSubs(2, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err = nc2.Request(\"foo-req\", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil {\n\t\t\tt.Fatalf(\"Received an error on Request test [%d]: %s\", i, err)\n\t\t}\n\t}\n\n\tfor i := 0; i < 100; i++ {\n\t\tif _, err = nc1.Request(\"foo-req\", []byte(strconv.Itoa(i)), 500*time.Millisecond); err != nil {\n\t\t\tt.Fatalf(\"Received an error on Request test [%d]: %s\", i, err)\n\t\t}\n\t}\n}\n\n// This is in response to Issue #1144\n// https://github.com/nats-io/nats-server/issues/1144\nfunc TestQueueDistributionAcrossRoutes(t *testing.T) {\n\tsrvA, srvB, _, _ := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\turlA := srvA.ClientURL()\n\turlB := srvB.ClientURL()\n\n\tnc1, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc1: %v\\n\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for nc2: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\tvar qsubs []*nats.Subscription\n\n\t// Connect queue subscriptions as mentioned in the issue. 2(A) - 6(B) - 4(A)\n\tfor i := 0; i < 2; i++ {\n\t\tsub, _ := nc1.QueueSubscribeSync(\"foo\", \"bar\")\n\t\tqsubs = append(qsubs, sub)\n\t}\n\tnc1.Flush()\n\tfor i := 0; i < 6; i++ {\n\t\tsub, _ := nc2.QueueSubscribeSync(\"foo\", \"bar\")\n\t\tqsubs = append(qsubs, sub)\n\t}\n\tnc2.Flush()\n\tfor i := 0; i < 4; i++ {\n\t\tsub, _ := nc1.QueueSubscribeSync(\"foo\", \"bar\")\n\t\tqsubs = append(qsubs, sub)\n\t}\n\tnc1.Flush()\n\n\tif err := checkExpectedSubs(7, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsend := 10000\n\tfor i := 0; i < send; i++ {\n\t\tnc2.Publish(\"foo\", nil)\n\t}\n\tnc2.Flush()\n\n\ttp := func() int {\n\t\tvar total int\n\t\tfor i := 0; i < len(qsubs); i++ {\n\t\t\tpending, _, _ := qsubs[i].Pending()\n\t\t\ttotal += pending\n\t\t}\n\t\treturn total\n\t}\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif total := tp(); total != send {\n\t\t\treturn fmt.Errorf(\"Number of total received %d\", total)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// The bug is essentially that when we deliver across a route, we\n\t// prefer locals, but if we randomize to a block of bounce backs, then\n\t// we walk to the end and find the same local for all the remote options.\n\t// So what you will see in this case is a large value at #9 (2+6, next one local).\n\n\tavg := send / len(qsubs)\n\tfor i := 0; i < len(qsubs); i++ {\n\t\ttotal, _, _ := qsubs[i].Pending()\n\t\tif total > avg+(avg*3/10) {\n\t\t\tif i == 8 {\n\t\t\t\tt.Fatalf(\"Qsub in 8th position gets majority of the messages (prior 6 spots) in this test\")\n\t\t\t}\n\t\t\tt.Fatalf(\"Received too high, %d vs %d\", total, avg)\n\t\t}\n\t}\n}\n" }, { "path": "test/cluster_test.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\n// Helper function to check that a cluster is formed\nfunc checkClusterFormed(t testing.TB, servers ...*server.Server) {\n\tt.Helper()\n\texpectedNumRoutes := len(servers) - 1\n\tcheckFor(t, 10*time.Second, 100*time.Millisecond, func() error {\n\t\tfor _, s := range servers {\n\t\t\tif numRoutes := s.NumRoutes(); numRoutes != expectedNumRoutes {\n\t\t\t\treturn fmt.Errorf(\"Expected %d routes for server %q, got %d\", expectedNumRoutes, s.ID(), numRoutes)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc checkNumRoutes(t *testing.T, s *server.Server, expected int) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 15*time.Millisecond, func() error {\n\t\tif nr := s.NumRoutes(); nr != expected {\n\t\t\treturn fmt.Errorf(\"Expected %v routes, got %v\", expected, nr)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Helper function to check that a server (or list of servers) have the\n// expected number of subscriptions.\nfunc checkExpectedSubs(expected int, servers ...*server.Server) error {\n\tvar err string\n\tmaxTime := time.Now().Add(10 * time.Second)\n\tfor time.Now().Before(maxTime) {\n\t\terr = \"\"\n\t\tfor _, s := range servers {\n\t\t\tif numSubs := int(s.NumSubscriptions()); numSubs != expected {\n\t\t\t\terr = fmt.Sprintf(\"Expected %d subscriptions for server %q, got %d\", expected, s.ID(), numSubs)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif err != \"\" {\n\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tif err != \"\" {\n\t\treturn errors.New(err)\n\t}\n\treturn nil\n}\n\nfunc checkSubInterest(t testing.TB, s *server.Server, accName, subject string, timeout time.Duration) {\n\tt.Helper()\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tacc, err := s.LookupAccount(accName)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error looking up account %q: %v\", accName, err)\n\t\t}\n\t\tif acc.SubscriptionInterest(subject) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"no subscription interest for account %q on %q\", accName, subject)\n\t})\n}\n\nfunc checkNoSubInterest(t *testing.T, s *server.Server, accName, subject string, timeout time.Duration) {\n\tt.Helper()\n\tacc, err := s.LookupAccount(accName)\n\tif err != nil {\n\t\tt.Fatalf(\"error looking up account %q: %v\", accName, err)\n\t}\n\n\tstart := time.Now()\n\tfor time.Now().Before(start.Add(timeout)) {\n\t\tif acc.SubscriptionInterest(subject) {\n\t\t\tt.Fatalf(\"Did not expect interest for %q\", subject)\n\t\t}\n\t\ttime.Sleep(5 * time.Millisecond)\n\t}\n}\n\nfunc runThreeServers(t *testing.T) (srvA, srvB, srvC *server.Server, optsA, optsB, optsC *server.Options) {\n\tsrvA, optsA = RunServerWithConfig(\"./configs/srv_a.conf\")\n\tsrvB, optsB = RunServerWithConfig(\"./configs/srv_b.conf\")\n\tsrvC, optsC = RunServerWithConfig(\"./configs/srv_c.conf\")\n\n\tcheckClusterFormed(t, srvA, srvB, srvC)\n\treturn\n}\n\nfunc runServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) {\n\tsrvA, optsA = RunServerWithConfig(\"./configs/srv_a.conf\")\n\tsrvB, optsB = RunServerWithConfig(\"./configs/srv_b.conf\")\n\n\tcheckClusterFormed(t, srvA, srvB)\n\treturn\n}\n\nfunc TestProperServerWithRoutesShutdown(t *testing.T) {\n\tbefore := runtime.NumGoroutine()\n\tsrvA, srvB, _, _ := runServers(t)\n\tsrvA.Shutdown()\n\tsrvB.Shutdown()\n\ttime.Sleep(100 * time.Millisecond)\n\n\tafter := runtime.NumGoroutine()\n\tdelta := after - before\n\t// There may be some finalizers or IO, but in general more than\n\t// 2 as a delta represents a problem.\n\tif delta > 2 {\n\t\tt.Fatalf(\"Expected same number of goroutines, %d vs %d\\n\", before, after)\n\t}\n}\n\nfunc TestDoubleRouteConfig(t *testing.T) {\n\tsrvA, srvB, _, _ := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n}\n\nfunc TestBasicClusterPubSub(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\tsendA(\"SUB foo 22\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(1, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB, expectB := setupConn(t, clientB)\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"22\", \"\", \"2\", \"ok\")\n}\n\nfunc TestClusterQueueSubs(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\tsendB, expectB := setupConn(t, clientB)\n\n\texpectMsgsA := expectMsgsCommand(t, expectA)\n\texpectMsgsB := expectMsgsCommand(t, expectB)\n\n\t// Capture sids for checking later.\n\tqg1SidsA := []string{\"1\", \"2\", \"3\"}\n\n\t// Three queue subscribers\n\tfor _, sid := range qg1SidsA {\n\t\tsendA(fmt.Sprintf(\"SUB foo qg1 %s\\r\\n\", sid))\n\t}\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Make sure the subs have propagated to srvB before continuing\n\t// New cluster proto this will only be 1.\n\tif err := checkExpectedSubs(1, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Make sure we get only 1.\n\tmatches := expectMsgsA(1)\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\n\t// Capture sids for checking later.\n\tpSids := []string{\"4\", \"5\", \"6\"}\n\n\t// Create 3 normal subscribers\n\tfor _, sid := range pSids {\n\t\tsendA(fmt.Sprintf(\"SUB foo %s\\r\\n\", sid))\n\t}\n\n\t// Create a FWC Subscriber\n\tpSids = append(pSids, \"7\")\n\tsendA(\"SUB > 7\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Make sure the subs have propagated to srvB before continuing\n\t// Normal foo and the queue group will be one a piece, so 2 + wc == 3\n\tif err := checkExpectedSubs(3, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\t// Send to B\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Give plenty of time for the messages to flush, so that we don't\n\t// accidentally only read some of them.\n\ttime.Sleep(time.Millisecond * 250)\n\n\t// Should receive 5.\n\tmatches = expectMsgsA(5)\n\tcheckForQueueSid(t, matches, qg1SidsA)\n\tcheckForPubSids(t, matches, pSids)\n\n\t// Send to A\n\tsendA(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\t// Give plenty of time for the messages to flush, so that we don't\n\t// accidentally only read some of them.\n\ttime.Sleep(time.Millisecond * 250)\n\n\t// Should receive 5.\n\tmatches = expectMsgsA(5)\n\tcheckForQueueSid(t, matches, qg1SidsA)\n\tcheckForPubSids(t, matches, pSids)\n\n\t// Now add queue subscribers to B\n\tqg2SidsB := []string{\"1\", \"2\", \"3\"}\n\tfor _, sid := range qg2SidsB {\n\t\tsendB(fmt.Sprintf(\"SUB foo qg2 %s\\r\\n\", sid))\n\t}\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Make sure the subs have propagated to srvA before continuing\n\t// This will be all the subs on A and just 1 from B that gets coalesced.\n\tif err := checkExpectedSubs(len(qg1SidsA)+len(pSids)+1, srvA); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\t// Send to B\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\t// Give plenty of time for the messages to flush, so that we don't\n\t// accidentally only read some of them.\n\ttime.Sleep(time.Millisecond * 250)\n\n\t// Should receive 1 from B.\n\tmatches = expectMsgsB(1)\n\tcheckForQueueSid(t, matches, qg2SidsB)\n\n\t// Should receive 5 still from A.\n\tmatches = expectMsgsA(5)\n\tcheckForQueueSid(t, matches, qg1SidsA)\n\tcheckForPubSids(t, matches, pSids)\n\n\t// Now drop queue subscribers from A\n\tfor _, sid := range qg1SidsA {\n\t\tsendA(fmt.Sprintf(\"UNSUB %s\\r\\n\", sid))\n\t}\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Make sure the subs have propagated to srvB before continuing\n\tif err := checkExpectedSubs(1+1+len(qg2SidsB), srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\t// Send to B\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\t// Should receive 1 from B.\n\tmatches = expectMsgsB(1)\n\tcheckForQueueSid(t, matches, qg2SidsB)\n\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Should receive 4 now.\n\tmatches = expectMsgsA(4)\n\tcheckForPubSids(t, matches, pSids)\n\n\t// Send to A\n\tsendA(\"PUB foo 2\\r\\nok\\r\\n\")\n\n\t// Give plenty of time for the messages to flush, so that we don't\n\t// accidentally only read some of them.\n\ttime.Sleep(time.Millisecond * 250)\n\n\t// Should receive 4 now.\n\tmatches = expectMsgsA(4)\n\tcheckForPubSids(t, matches, pSids)\n}\n\n// Issue #22\nfunc TestClusterDoubleMsgs(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA1 := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA1.Close()\n\n\tclientA2 := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA2.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA1, expectA1 := setupConn(t, clientA1)\n\tsendA2, expectA2 := setupConn(t, clientA2)\n\tsendB, expectB := setupConn(t, clientB)\n\n\texpectMsgsA1 := expectMsgsCommand(t, expectA1)\n\texpectMsgsA2 := expectMsgsCommand(t, expectA2)\n\n\t// Capture sids for checking later.\n\tqg1SidsA := []string{\"1\", \"2\", \"3\"}\n\n\t// Three queue subscribers\n\tfor _, sid := range qg1SidsA {\n\t\tsendA1(fmt.Sprintf(\"SUB foo qg1 %s\\r\\n\", sid))\n\t}\n\tsendA1(\"PING\\r\\n\")\n\texpectA1(pongRe)\n\n\t// Make sure the subs have propagated to srvB before continuing\n\tif err := checkExpectedSubs(1, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Make sure we get only 1.\n\tmatches := expectMsgsA1(1)\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\tcheckForQueueSid(t, matches, qg1SidsA)\n\n\t// Add a FWC subscriber on A2\n\tsendA2(\"SUB > 1\\r\\n\")\n\tsendA2(\"SUB foo 2\\r\\n\")\n\tsendA2(\"PING\\r\\n\")\n\texpectA2(pongRe)\n\tpSids := []string{\"1\", \"2\"}\n\n\t// Make sure the subs have propagated to srvB before continuing\n\tif err := checkExpectedSubs(1+2, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tmatches = expectMsgsA1(1)\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\tcheckForQueueSid(t, matches, qg1SidsA)\n\n\tmatches = expectMsgsA2(2)\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\tcheckForPubSids(t, matches, pSids)\n\n\t// Close ClientA1\n\tclientA1.Close()\n\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\ttime.Sleep(10 * time.Millisecond)\n\n\tmatches = expectMsgsA2(2)\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\tcheckForPubSids(t, matches, pSids)\n}\n\n// This will test that we drop remote sids correctly.\nfunc TestClusterDropsRemoteSids(t *testing.T) {\n\tsrvA, srvB, optsA, _ := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\n\t// Add a subscription\n\tsendA(\"SUB foo 1\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Wait for propagation.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tif sc := srvA.NumSubscriptions(); sc != 1 {\n\t\tt.Fatalf(\"Expected one subscription for srvA, got %d\\n\", sc)\n\t}\n\tif sc := srvB.NumSubscriptions(); sc != 1 {\n\t\tt.Fatalf(\"Expected one subscription for srvB, got %d\\n\", sc)\n\t}\n\n\t// Add another subscription\n\tsendA(\"SUB bar 2\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Wait for propagation.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tif sc := srvA.NumSubscriptions(); sc != 2 {\n\t\tt.Fatalf(\"Expected two subscriptions for srvA, got %d\\n\", sc)\n\t}\n\tif sc := srvB.NumSubscriptions(); sc != 2 {\n\t\tt.Fatalf(\"Expected two subscriptions for srvB, got %d\\n\", sc)\n\t}\n\n\t// unsubscription\n\tsendA(\"UNSUB 1\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Wait for propagation.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tif sc := srvA.NumSubscriptions(); sc != 1 {\n\t\tt.Fatalf(\"Expected one subscription for srvA, got %d\\n\", sc)\n\t}\n\tif sc := srvB.NumSubscriptions(); sc != 1 {\n\t\tt.Fatalf(\"Expected one subscription for srvB, got %d\\n\", sc)\n\t}\n\n\t// Close the client and make sure we remove subscription state.\n\tclientA.Close()\n\n\t// Wait for propagation.\n\ttime.Sleep(100 * time.Millisecond)\n\tif sc := srvA.NumSubscriptions(); sc != 0 {\n\t\tt.Fatalf(\"Expected no subscriptions for srvA, got %d\\n\", sc)\n\t}\n\tif sc := srvB.NumSubscriptions(); sc != 0 {\n\t\tt.Fatalf(\"Expected no subscriptions for srvB, got %d\\n\", sc)\n\t}\n}\n\n// This will test that we drop remote sids correctly.\nfunc TestAutoUnsubscribePropagation(t *testing.T) {\n\tsrvA, srvB, optsA, _ := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\n\t// We will create subscriptions that will auto-unsubscribe and make sure\n\t// we are not accumulating orphan subscriptions on the other side.\n\tfor i := 1; i <= 100; i++ {\n\t\tsub := fmt.Sprintf(\"SUB foo %d\\r\\n\", i)\n\t\tauto := fmt.Sprintf(\"UNSUB %d 1\\r\\n\", i)\n\t\tsendA(sub)\n\t\tsendA(auto)\n\t\t// This will trip the auto-unsubscribe\n\t\tsendA(\"PUB foo 2\\r\\nok\\r\\n\")\n\t\texpectMsgs(1)\n\t}\n\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\ttime.Sleep(50 * time.Millisecond)\n\n\t// Make sure number of subscriptions on B is correct\n\tif subs := srvB.NumSubscriptions(); subs != 0 {\n\t\tt.Fatalf(\"Expected no subscriptions on remote server, got %d\\n\", subs)\n\t}\n}\n\nfunc TestAutoUnsubscribePropagationOnClientDisconnect(t *testing.T) {\n\tsrvA, srvB, optsA, _ := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tcluster := []*server.Server{srvA, srvB}\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\n\t// No subscriptions. Ready to test.\n\tif err := checkExpectedSubs(0, cluster...); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendA(\"SUB foo 1\\r\\n\")\n\tsendA(\"UNSUB 1 1\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Waiting cluster subs propagation\n\tif err := checkExpectedSubs(1, cluster...); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tclientA.Close()\n\n\t// No subs should be on the cluster when all clients is disconnected\n\tif err := checkExpectedSubs(0, cluster...); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n}\n\nfunc TestClusterNameOption(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tname: MyCluster\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsi := checkInfoMsg(t, c)\n\tif si.Cluster != \"MyCluster\" {\n\t\tt.Fatalf(\"Expected a cluster name of %q, got %q\", \"MyCluster\", si.Cluster)\n\t}\n}\n\nfunc TestEphemeralClusterName(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t}\n\t`))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsi := checkInfoMsg(t, c)\n\tif si.Cluster == \"\" {\n\t\tt.Fatalf(\"Expected an ephemeral cluster name to be set\")\n\t}\n}\n\ntype captureErrLogger struct {\n\tdummyLogger\n\tch chan string\n}\n\nfunc (c *captureErrLogger) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tselect {\n\tcase c.ch <- msg:\n\tdefault:\n\t}\n}\n\nfunc TestClusterNameConflictsDropRoutes(t *testing.T) {\n\tll := &captureErrLogger{ch: make(chan string, 4)}\n\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tname: MyCluster33\n\t\t\tlisten: 127.0.0.1:5244\n\t\t}\n\t`))\n\n\ts1, _ := RunServerWithConfig(conf)\n\tdefer s1.Shutdown()\n\ts1.SetLogger(ll, false, false)\n\n\tconf2 := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tname: MyCluster22\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes = [nats-route://127.0.0.1:5244]\n\t\t}\n\t`))\n\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\ts2.SetLogger(ll, false, false)\n\n\tselect {\n\tcase msg := <-ll.ch:\n\t\tif !strings.Contains(msg, \"Rejecting connection\") || !strings.Contains(msg, \"does not match\") {\n\t\t\tt.Fatalf(\"Got bad error about cluster name mismatch\")\n\t\t}\n\tcase <-time.After(time.Second):\n\t\tt.Fatalf(\"Expected an error, timed out\")\n\t}\n}\n\nfunc TestClusterNameDynamicNegotiation(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {listen: 127.0.0.1:5244}\n\t`))\n\n\tseed, _ := RunServerWithConfig(conf)\n\tdefer seed.Shutdown()\n\n\toconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tcluster {\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\troutes = [nats-route://127.0.0.1:5244]\n\t\t}\n\t`))\n\n\t// Create a random number of additional servers, up to 20.\n\tnumServers := rand.Intn(20) + 1\n\tservers := make([]*server.Server, 0, numServers+1)\n\tservers = append(servers, seed)\n\n\tfor i := 0; i < numServers; i++ {\n\t\ts, _ := RunServerWithConfig(oconf)\n\t\tdefer s.Shutdown()\n\t\tservers = append(servers, s)\n\t}\n\n\t// If this passes we should have all the same name.\n\tcheckClusterFormed(t, servers...)\n\n\tclusterName := seed.ClusterName()\n\tfor _, s := range servers {\n\t\tif s.ClusterName() != clusterName {\n\t\t\tt.Fatalf(\"Expected the cluster names to all be the same as %q, got %q\", clusterName, s.ClusterName())\n\t\t}\n\t}\n}\n" }, { "path": "test/cluster_tls_test.go", "content": "// Copyright 2013-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc runTLSServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) {\n\tsrvA, optsA = RunServerWithConfig(\"./configs/srv_a_tls.conf\")\n\tsrvB, optsB = RunServerWithConfig(\"./configs/srv_b_tls.conf\")\n\tcheckClusterFormed(t, srvA, srvB)\n\treturn\n}\n\nfunc TestTLSClusterConfig(t *testing.T) {\n\tsrvA, srvB, _, _ := runTLSServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n}\n\nfunc TestBasicTLSClusterPubSub(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runTLSServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\tsendA(\"SUB foo 22\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(1, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB, expectB := setupConn(t, clientB)\n\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"22\", \"\", \"2\", \"ok\")\n}\n\ntype captureTLSError struct {\n\tdummyLogger\n\tch chan struct{}\n}\n\nfunc (c *captureTLSError) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"handshake error\") {\n\t\tselect {\n\t\tcase c.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n}\n\ntype captureClusterTLSInsecureLogger struct {\n\tdummyLogger\n\tch chan struct{}\n}\n\nfunc (c *captureClusterTLSInsecureLogger) Warnf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"solicited routes will not be verified\") {\n\t\tselect {\n\t\tcase c.ch <- struct{}{}:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc TestClusterTLSInsecure(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"xyz\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tpool_size: -1\n\t\t\tcompression: \"disabled\"\n\t\t\ttls {\n\t\t\t cert_file: \"./configs/certs/server-noip.pem\"\n\t\t\t\tkey_file: \"./configs/certs/server-key-noip.pem\"\n\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t`))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tl := &captureTLSError{ch: make(chan struct{}, 1)}\n\tsrvA.SetLogger(l, false, false)\n\n\tbConfigTemplate := `\n\t\tport: -1\n\t\tcluster {\n\t\t\tname: \"xyz\"\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tpool_size: -1\n\t\t\tcompression: \"disabled\"\n\t\t\ttls {\n\t\t\t cert_file: \"./configs/certs/server-noip.pem\"\n\t\t\t\tkey_file: \"./configs/certs/server-key-noip.pem\"\n\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t\t%s\n\t\t\t}\n\t\t\troutes [\n\t\t\t\t\"nats://%s:%d\"\n\t\t\t]\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate,\n\t\t\"\", optsA.Cluster.Host, optsA.Cluster.Port)))\n\tsrvB, _ := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\t// We should get errors\n\tselect {\n\tcase <-l.ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get handshake error\")\n\t}\n\n\t// Set a logger that will capture the warning\n\twl := &captureClusterTLSInsecureLogger{ch: make(chan struct{}, 1)}\n\tsrvB.SetLogger(wl, false, false)\n\n\t// Need to add \"insecure: true\" and reload\n\tif err := os.WriteFile(confB,\n\t\t[]byte(fmt.Sprintf(bConfigTemplate, \"insecure: true\", optsA.Cluster.Host, optsA.Cluster.Port)),\n\t\t0666); err != nil {\n\t\tt.Fatalf(\"Error rewriting file: %v\", err)\n\t}\n\tif err := srvB.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Make sure we have the tracing\n\tselect {\n\tcase <-wl.ch:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Fatalf(\"Did not get warning about using cluster's insecure setting\")\n\t}\n}\n" }, { "path": "test/configs/certs/ca.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEkDCCA3igAwIBAgIUSZwW7btc9EUbrMWtjHpbM0C2bSEwDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmlj\nYXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAqilVqyY8rmCpTwAsLF7DEtWEq37KbljBWVjmlp2Wo6TgMd3b537t\n6iO8+SbI8KH75i63RcxV3Uzt1/L9Yb6enDXF52A/U5ugmDhaa+Vsoo2HBTbCczmp\nqndp7znllQqn7wNLv6aGSvaeIUeYS5Dmlh3kt7Vqbn4YRANkOUTDYGSpMv7jYKSu\n1ee05Rco3H674zdwToYto8L8V7nVMrky42qZnGrJTaze+Cm9tmaIyHCwUq362CxS\ndkmaEuWx11MOIFZvL80n7ci6pveDxe5MIfwMC3/oGn7mbsSqidPMcTtjw6ey5NEu\nZ0UrC/2lL1FtF4gnVMKUSaEhU2oKjj0ZAQIDAQABo4IBHjCCARowHQYDVR0OBBYE\nFP7Pfz4u7sSt6ltviEVsx4hIFIs6MIGuBgNVHSMEgaYwgaOAFP7Pfz4u7sSt6ltv\niEVsx4hIFIs6oXWkczBxMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5p\nYTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEpMCcGA1UEAwwg\nQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMjItMDgtMjeCFEmcFu27XPRFG6zFrYx6\nWzNAtm0hMAwGA1UdEwQFMAMBAf8wOgYJYIZIAYb4QgENBC0WK25hdHMuaW8gbmF0\ncy1zZXJ2ZXIgdGVzdC1zdWl0ZSB0cmFuc2llbnQgQ0EwDQYJKoZIhvcNAQELBQAD\nggEBAHDCHLQklYZlnzHDaSwxgGSiPUrCf2zhk2DNIYSDyBgdzrIapmaVYQRrCBtA\nj/4jVFesgw5WDoe4TKsyha0QeVwJDIN8qg2pvpbmD8nOtLApfl0P966vcucxDwqO\ndQWrIgNsaUdHdwdo0OfvAlTfG0v/y2X0kbL7h/el5W9kWpxM/rfbX4IHseZL2sLq\nFH69SN3FhMbdIm1ldrcLBQVz8vJAGI+6B9hSSFQWljssE0JfAX+8VW/foJgMSx7A\nvBTq58rLkAko56Jlzqh/4QT+ckayg9I73v1Q5/44jP1mHw35s5ZrzpDQt2sVv4l5\nlwRPJFXMwe64flUs9sM+/vqJaIY=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/client-cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 38:4c:16:24:9b:04:1c:b3:db:e0:4c:3c:ed:b7:40:7d:68:b5:fa:1f\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:ac:9c:3e:9d:3b:7a:12:56:85:78:ca:df:9c:fc:\n 0c:7e:5e:f2:4f:22:33:46:81:38:53:d7:a7:25:8f:\n d7:ee:16:13:e2:67:49:88:f6:94:99:f0:a9:a6:db:\n fe:7a:17:c9:e3:df:31:73:71:38:70:3a:96:1e:99:\n 7b:5d:07:e3:63:e4:e8:bf:99:f7:3d:5c:27:f5:b7:\n 37:29:da:ee:82:80:00:d4:c8:d3:1b:36:0d:8b:d3:\n 8a:9b:8e:12:a1:4d:0c:c5:22:f8:56:3b:6a:1a:fb:\n e9:3d:08:1e:13:7f:55:6e:2e:65:93:9a:90:54:03:\n 6d:0d:e6:44:d6:f7:c0:d7:d8:e1:c7:1e:c2:9b:a3:\n 6e:88:f1:7c:58:08:a2:9f:13:cc:5b:b9:11:2c:1d:\n 23:6f:3a:ae:47:9a:0f:6a:ce:e5:80:34:09:e6:e3:\n fd:76:4a:cf:5a:18:bb:9c:c5:c1:74:49:67:77:1b:\n ba:28:86:31:a6:fc:12:af:4a:85:1b:73:5b:f4:d6:\n 42:ff:0c:1c:49:e7:31:f2:5a:2a:1e:cd:87:cb:22:\n ff:70:1c:48:ed:ba:e0:be:f0:bc:9e:e0:dc:59:db:\n a5:74:25:58:b3:61:04:f6:33:28:6b:07:25:60:0f:\n 72:93:16:6c:9f:b0:ad:4a:18:f7:9e:29:1e:b7:61:\n 34:17\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n 1F:14:EF:2B:53:AB:28:4A:93:42:98:AE:85:06:0F:B4:7D:DC:36:AE\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io\n Netscape Cert Type: \n SSL Client\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 60:43:0b:c6:11:0b:96:ae:03:dc:77:26:9a:4a:bd:6a:d7:03:\n ec:43:16:2d:ba:8c:e5:50:fa:57:a9:1f:2f:a4:15:c3:a8:13:\n b9:d3:59:2a:97:7c:ae:ce:a9:f8:44:e4:97:ee:7d:09:dc:74:\n 38:80:94:cf:47:e0:84:52:2a:91:44:8a:85:55:da:42:6a:f1:\n 91:1a:6e:5a:63:e6:0b:61:3c:0d:b0:aa:17:b8:77:94:32:20:\n 4d:20:8f:84:56:64:ae:ef:d8:8d:42:b5:52:4d:b0:1c:46:97:\n bc:4c:77:8c:3f:a3:73:43:87:27:71:62:e7:fe:02:de:a1:27:\n 77:be:86:29:8f:62:a1:d9:e7:ea:61:33:73:f4:1f:0a:12:14:\n 68:eb:7d:8c:71:5b:42:e7:48:10:c9:df:30:3b:5b:eb:69:29:\n b6:95:bc:09:fc:01:b0:be:fc:9f:ee:c4:f3:df:a0:01:c5:68:\n 20:f5:2f:f8:e7:1c:a5:4c:a8:a8:a2:20:a1:d2:0f:f6:f6:c4:\n 0d:f5:26:fd:ea:8b:b5:06:a9:9e:17:35:47:f7:fd:6e:78:3d:\n 5f:7a:87:ed:21:b2:4e:e9:6a:d1:d9:ed:0e:cf:43:61:83:7c:\n fe:0d:b1:ad:ff:fa:2d:2b:36:9d:99:9c:20:48:21:0d:36:c8:\n dd:b6:0a:d8\n-----BEGIN CERTIFICATE-----\nMIIE5zCCA8+gAwIBAgIUOEwWJJsEHLPb4Ew87bdAfWi1+h8wDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKycPp07ehJWhXjK35z8\nDH5e8k8iM0aBOFPXpyWP1+4WE+JnSYj2lJnwqabb/noXyePfMXNxOHA6lh6Ze10H\n42Pk6L+Z9z1cJ/W3Nyna7oKAANTI0xs2DYvTipuOEqFNDMUi+FY7ahr76T0IHhN/\nVW4uZZOakFQDbQ3mRNb3wNfY4ccewpujbojxfFgIop8TzFu5ESwdI286rkeaD2rO\n5YA0Cebj/XZKz1oYu5zFwXRJZ3cbuiiGMab8Eq9KhRtzW/TWQv8MHEnnMfJaKh7N\nh8si/3AcSO264L7wvJ7g3FnbpXQlWLNhBPYzKGsHJWAPcpMWbJ+wrUoY954pHrdh\nNBcCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nHxTvK1OrKEqTQpiuhQYPtH3cNq4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA\nAAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC\nBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAGBDC8YR\nC5auA9x3JppKvWrXA+xDFi26jOVQ+lepHy+kFcOoE7nTWSqXfK7OqfhE5JfufQnc\ndDiAlM9H4IRSKpFEioVV2kJq8ZEablpj5gthPA2wqhe4d5QyIE0gj4RWZK7v2I1C\ntVJNsBxGl7xMd4w/o3NDhydxYuf+At6hJ3e+himPYqHZ5+phM3P0HwoSFGjrfYxx\nW0LnSBDJ3zA7W+tpKbaVvAn8AbC+/J/uxPPfoAHFaCD1L/jnHKVMqKiiIKHSD/b2\nxA31Jv3qi7UGqZ4XNUf3/W54PV96h+0hsk7patHZ7Q7PQ2GDfP4Nsa3/+i0rNp2Z\nnCBIIQ02yN22Ctg=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/client-id-auth-cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 74:4b:fa:47:3b:79:81:c8:07:a4:c8:d9:70:df:d2:98:bc:a1:27:76\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:e4:2f:ec:ba:66:87:36:a8:09:ab:66:9b:cc:77:\n 24:50:e0:29:4d:c9:98:c8:9f:08:1c:a5:66:0a:b6:\n 83:38:8a:3a:f3:3c:16:64:fe:13:b6:02:bd:50:64:\n 7d:bb:19:a5:ec:e1:1c:2f:1c:30:be:cd:07:c4:33:\n 1f:ae:e2:7f:c5:11:f0:d3:1d:2a:4b:28:66:c1:a0:\n 09:52:77:64:75:b7:86:a9:63:cf:a4:29:c8:ad:87:\n 3d:79:71:6b:19:34:86:23:80:ed:ed:e6:eb:29:c4:\n 6f:2c:9a:c8:ef:da:6a:4e:e5:c9:ce:ef:48:87:26:\n 11:d3:f2:2c:4f:d2:a8:b6:26:b1:1d:48:49:9f:f5:\n 5a:d4:8a:30:a8:67:1c:8a:b9:3a:13:5d:04:82:e4:\n 89:aa:4b:ec:13:59:f7:92:32:33:8f:28:df:50:98:\n 65:92:0a:35:cc:c5:04:e3:86:ac:69:cc:6a:84:30:\n 91:af:af:44:4a:e8:9f:26:0a:f6:1b:78:f6:90:81:\n b1:eb:11:e6:85:1e:42:0e:09:15:e5:e3:36:86:b2:\n 46:38:b7:81:c2:c4:16:72:b9:a7:55:89:03:78:28:\n e0:2e:29:17:ce:58:2f:16:80:34:b2:5e:02:00:11:\n 01:2c:35:ac:bb:0e:2d:41:00:6d:0c:85:94:1a:f6:\n 7a:8b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n 86:48:10:75:95:44:01:FC:76:A0:96:E9:35:52:55:A6:38:8C:5F:37\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, email:derek@nats.io\n Netscape Cert Type: \n SSL Client\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 0e:37:24:36:db:25:73:c2:8d:71:e0:44:05:c1:2f:ac:e7:a4:\n 20:26:2a:15:69:20:2f:ea:4c:fb:0c:74:4b:29:e0:92:0f:12:\n 8a:a0:4a:73:78:ba:81:eb:4d:91:1c:3f:8e:58:cb:3f:30:77:\n 6e:95:5d:a6:a6:22:b3:f9:20:0a:75:93:bc:95:6c:b3:1c:c2:\n 6a:72:9a:a6:62:1f:ed:a1:df:9c:2f:d7:28:02:08:49:07:c5:\n 3b:5b:58:0f:f6:7b:b5:39:de:05:c7:be:c7:89:bf:62:77:56:\n eb:96:b9:74:d6:bf:ce:20:8d:ef:0c:08:bb:ec:0d:b2:55:6d:\n cb:94:83:88:89:a5:47:c4:95:91:72:10:c5:3c:6a:be:e5:86:\n bb:be:96:74:fd:67:d2:08:55:4e:df:e9:96:c4:62:58:93:6a:\n 0c:2c:68:a1:93:b9:61:38:98:eb:c4:4a:1c:3f:9f:9d:c2:3c:\n b0:eb:62:7f:a0:aa:e9:53:a3:f7:55:5d:43:1d:5a:63:8b:97:\n 32:4c:dc:b8:11:0b:65:35:de:2b:f9:af:8e:28:8c:6f:dd:4d:\n 13:3e:b9:f8:7b:7f:27:0d:43:a9:da:b6:59:10:d5:9c:a6:97:\n 79:97:19:94:f0:8d:0e:13:f7:c1:ee:50:f2:4c:81:6a:e6:0f:\n d7:42:c3:67\n-----BEGIN CERTIFICATE-----\nMIIE5zCCA8+gAwIBAgIUdEv6Rzt5gcgHpMjZcN/SmLyhJ3YwDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOQv7LpmhzaoCatmm8x3\nJFDgKU3JmMifCBylZgq2gziKOvM8FmT+E7YCvVBkfbsZpezhHC8cML7NB8QzH67i\nf8UR8NMdKksoZsGgCVJ3ZHW3hqljz6QpyK2HPXlxaxk0hiOA7e3m6ynEbyyayO/a\nak7lyc7vSIcmEdPyLE/SqLYmsR1ISZ/1WtSKMKhnHIq5OhNdBILkiapL7BNZ95Iy\nM48o31CYZZIKNczFBOOGrGnMaoQwka+vREronyYK9ht49pCBsesR5oUeQg4JFeXj\nNoayRji3gcLEFnK5p1WJA3go4C4pF85YLxaANLJeAgARASw1rLsOLUEAbQyFlBr2\neosCAwEAAaOCAYwwggGIMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nhkgQdZVEAfx2oJbpNVJVpjiMXzcwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwOwYDVR0RBDQwMoIJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA\nAAABgQ1kZXJla0BuYXRzLmlvMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMC\nBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAA43JDbb\nJXPCjXHgRAXBL6znpCAmKhVpIC/qTPsMdEsp4JIPEoqgSnN4uoHrTZEcP45Yyz8w\nd26VXaamIrP5IAp1k7yVbLMcwmpymqZiH+2h35wv1ygCCEkHxTtbWA/2e7U53gXH\nvseJv2J3VuuWuXTWv84gje8MCLvsDbJVbcuUg4iJpUfElZFyEMU8ar7lhru+lnT9\nZ9IIVU7f6ZbEYliTagwsaKGTuWE4mOvEShw/n53CPLDrYn+gqulTo/dVXUMdWmOL\nlzJM3LgRC2U13iv5r44ojG/dTRM+ufh7fycNQ6natlkQ1Zyml3mXGZTwjQ4T98Hu\nUPJMgWrmD9dCw2c=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/client-id-auth-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDkL+y6Zoc2qAmr\nZpvMdyRQ4ClNyZjInwgcpWYKtoM4ijrzPBZk/hO2Ar1QZH27GaXs4RwvHDC+zQfE\nMx+u4n/FEfDTHSpLKGbBoAlSd2R1t4apY8+kKcithz15cWsZNIYjgO3t5uspxG8s\nmsjv2mpO5cnO70iHJhHT8ixP0qi2JrEdSEmf9VrUijCoZxyKuToTXQSC5ImqS+wT\nWfeSMjOPKN9QmGWSCjXMxQTjhqxpzGqEMJGvr0RK6J8mCvYbePaQgbHrEeaFHkIO\nCRXl4zaGskY4t4HCxBZyuadViQN4KOAuKRfOWC8WgDSyXgIAEQEsNay7Di1BAG0M\nhZQa9nqLAgMBAAECggEBAI3c3aflJhrszVYqLKIpUAKX2hXqR3oypLBqg84VOe9k\nwNGHgcS30TlO6rOYRjKT93wVV5hSRlvYzANGZWQsnJLAXKBjeW/QZlHqVOFYKLSm\nrKmSy/ybnY+EjMt7n8HDzcE03rcQ4RLOdO+eK14yw/TZF2X3jXe4S00hOjtGFG1o\n5BvIaN1HMnrV4G5WsbD2n4gE17lLGLlmn1FhbZaI3/uJgSvM/RgRzzSOSZL+XOzo\npyoMAm9DTI8U020qMuU23KEnjY2Wg68bMNMF38H7z73W+whRVQTyHq/mw0HhD4K7\nP15eWaBUytmRsAkeoybO1hEfGQbUYVdPOtqrvxCCp8ECgYEA83Jcmeiv5DRaH8Sk\n/K+O0LM1CDfdWoM/w+Z2frooKqjHKsSa8B8fqI/2AIafgLorksVb2WA8nzzhBvT+\ncIO9NiFbFkFVeLbrGc7supzPVjT1ZAFj7Qj9T0FmrMK937lnOmRNp6DU9t0hEJuF\n4wUly0smzVZiBiFRfXsPzANX2T8CgYEA7/Qg401pp99rQfjgCC70n/jVgI1cOFKh\npAdJHU+64GiozP4adMN/fd3fqqG3XnvFG8mRViwgL57iiIvxbbhx5pXUIZZO5LKC\nxwuMBzFhX8f4JAR7KANetOIsNF/ML7wTNvGhlUgc3CYXc/Fa1Wh78Xf4vhSd04KA\nOdkYuuVk37UCgYEAhCb4jbP6h27D3arpxSGn7TLa/vMUfiXxX26jtHdphn7IXzcK\nxH6guOgtKmvp+f8V2D66dW4AepCZtyUXWgypkdDZmWMt+rGRPhlN+J9XDf0BmKAI\nlovitjtSeUXdvKzwlIoOfYiZEslHQbSrIWmR8qGBFsZlv94mVm+PS7gk4BECgYEA\nts6t054hlXSAGYXK4FEtq0Z/Ge4YSQyi+v1V9Y/NlrQFjg81Bqn+Ul0bzrpfogr3\ncyEQqa76Ym1QtqivKWEw3XReZaxGtLNPMOeaKcy0G62UXZRQY36Vw4bgGJK3U9Kc\nbOqPqNSEsDARBBLnmdh9PMyi4+V/DCnLGMdNsO2c+VUCgYEAqz79qg3S+GON2woM\nNpe72bNBJC9UCciA4XpWkFB0f4AiPtFU0VW8fjZ4ZT6rd4sxvl727Y/FYgPcl3Dg\nr0zWWZAsaZ4LQRFk7EG9kMYe3Qsc/1211c5fg5dwmJwEybgW1gVvx/cb9qnbBifo\nhyWhzlSGwd/pB4skREo/fbNM3hA=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/client-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCsnD6dO3oSVoV4\nyt+c/Ax+XvJPIjNGgThT16clj9fuFhPiZ0mI9pSZ8Kmm2/56F8nj3zFzcThwOpYe\nmXtdB+Nj5Oi/mfc9XCf1tzcp2u6CgADUyNMbNg2L04qbjhKhTQzFIvhWO2oa++k9\nCB4Tf1VuLmWTmpBUA20N5kTW98DX2OHHHsKbo26I8XxYCKKfE8xbuREsHSNvOq5H\nmg9qzuWANAnm4/12Ss9aGLucxcF0SWd3G7oohjGm/BKvSoUbc1v01kL/DBxJ5zHy\nWioezYfLIv9wHEjtuuC+8Lye4NxZ26V0JVizYQT2MyhrByVgD3KTFmyfsK1KGPee\nKR63YTQXAgMBAAECggEBAKc6FHt2NPTxOAxn2C6aDmycBftesfiblnu8EWaVrmgu\noYMV+CsmYZ+mhmZu+mNFCsam5JzoUvp/+BKbNeZSjx2nl0qRmvOqhdhLcbkuLybl\nZmjAS64wNv2Bq+a6xRfaswWGtLuugkS0TCph4+mV0qmVb7mJ5ExQqWXu8kCl9QHn\nuKacp1wVFok9rmEI+byL1+Z01feKrkf/hcF6dk62U7zHNPajViJFTDww7hiHyfUH\n6qsxIe1UWSNKtE61haEHkzqbDIDAy79jX4t3JobLToeVNCbJ7BSPf2IQSPJxELVL\nsidIJhndEjsbDR2CLpIF/EjsiSIaP7jh2zC9fxFpgSkCgYEA1qH0PH1JD5FqRV/p\nn9COYa6EifvSymGo4u/2FHgtX7wNSIQvqAVXenrQs41mz9E65womeqFXT/AZglaM\n1PEjjwcFlDuLvUEYYJNgdXrIC515ZXS6TdvJ0JpQJLx28GzZ7h31tZXfwn68C3/i\nUGEHp+nN1BfBBQnsqvmGFFvHZFUCgYEAzeDlZHHijBlgHU+kGzKm7atJfAGsrv6/\ntw7CIMEsL+z/y7pl3nwDLdZF+mLIvGuKlwIRajEzbYcEuVymCyG2/SmPMQEUf6j+\nC1OmorX9CW8OwHmVCajkIgKn0ICFsF9iFv6aYZmm1kG48AIuYiQ7HOvY/MlilqFs\n1p8sw6ZpQrsCgYEAj7Z9fQs+omfxymYAXnwc+hcKtAGkENL3bIzULryRVSrrkgTA\njDaXbnFR0Qf7MWedkxnezfm+Js5TpkwhnGuiLaC8AZclaCFwGypTShZeYDifEmno\nXT2vkjfhNdfjo/Ser6vr3BxwaSDG9MQ6Wyu9HpeUtFD7c05D4++T8YnKpskCgYEA\npCkcoIAStcWSFy0m3K0B3+dBvAiVyh/FfNDeyEFf24Mt4CPsEIBwBH+j4ugbyeoy\nYwC6JCPBLyeHA8q1d5DVmX4m+Fs1HioBD8UOzRUyA/CzIZSQ21f5OIlHiIDCmQUl\ncNJpBUQAfT2AmpgSphzfqcsBhWeLHjLvVx8rEYLC0fsCgYAiHdPZ3C0f7rWZP93N\ngY4DuldiO4d+KVsWAdBxeNgPznisUI7/ZZ/9NvCxGvA5NynyZr0qlpiKzVvtFJG8\n1ZPUuFFRMAaWn9h5C+CwMPgk65tFC6lw/el0hpmcocSXVdiJEbkV0rnv9iGh0CYX\nHMACGrYlyZdDYM0CH/JAM+K/QQ==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/ca-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDqDCCApCgAwIBAgIBATANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMG0xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu\nYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEVMBMGA1UEAwwMbG9jYWxob3N0IGNhMIIB\nIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwy+fupDc9MZldhetmGqPJtuM\nsp5VV6W9amlzkTck15B9Vc3laC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjM\nFpkjkLIVdFkbcWiErXYWDBHdA6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJ\ngljFl2ymHaQEhta/87tSvPULZ7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXp\ni8kHqPCkImlv/s7Jw/QRIknV/kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF\n6O/4LvOC8UCbu50eW7OudppN8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQID\nAQABo1MwUTAdBgNVHQ4EFgQU7upCnRG44j5THcgKd28H4ESXBFkwHwYDVR0jBBgw\nFoAU7upCnRG44j5THcgKd28H4ESXBFkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG\n9w0BAQsFAAOCAQEAfBHCa8sm0e767+oIZj3JIRi9MWN24hB9i4lVjDrwdOMaapMC\nYLLj5urqIgjOULjdsxBMzdNgNgH1vPenRYUUvIQcq7tk1q8DpfvmHEg2DHajpTAC\nDroutE5fYtlmFPSQ5UGG1if237osd6pDarVhGAdxex4YhwM+y+OXgpLqk6oC85oI\nfatf+hcovwFOlNeOTUqNZW6fEC+iFdH5g4+dtlx2LAJLpW57+5z25iTH7z16nUwB\nVi76fezpaGA3xwkP/NMujgD4MbpVpF22a0YdK5fjUjXFwRI4Vu1zAjyJFhVuOWCS\nyT9yNzidtD5pho+Iv3JMzu54VWSq7nSUoPmKHQ==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/ca-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAwy+fupDc9MZldhetmGqPJtuMsp5VV6W9amlzkTck15B9Vc3l\naC6ph7Ble7FrT2L0sjG3U94MwU9/AHTXOmZdmbjMFpkjkLIVdFkbcWiErXYWDBHd\nA6dzOu+dagn0OyxRDjfqo1QUVKYVNu8Jw6MyWHXJgljFl2ymHaQEhta/87tSvPUL\nZ7gcEZ5CPFLENHWOlJPtQrPhJHDKjS8XHlbE1uXpi8kHqPCkImlv/s7Jw/QRIknV\n/kiAXAWGJCMbqLDG9JEatp7ektytcwMCr9pz9VzF6O/4LvOC8UCbu50eW7OudppN\n8G18IF3cMgH9jWsJpgVmXfJR+VZNe92/6ePTgQIDAQABAoIBAC5RXtYnCkgLzIf5\nlnhU0SOndfvtFtN1wT0/SO1s6JE++H8kHQxcBl7svShdMdnk4axnn9mHF//HnZu+\nHlT9dbjE4al7LbVojS7O9nQzGUkQfKrgklILqoyR0AkZ05s3KQT2v/eCPFDaGK6w\niuCiGZBkYy1LY5hLcCAYi/pze5mar1m6S2ZqvfpZWjlrlZOVi73cZOq91Es2g/iU\nTYaiR4HGHJ0McXFNjL6q7DxCBkRLWb6i+Xy+9+84XFZAVzRBAo1EaFJBPg0p88EC\nVKxQt4X0jgWYgLRABAqoQ+DkNEEoaCEQuUV67aIcIyV++ddn1FsegPtGYo38z7mr\nM+fzUAECgYEA/XL53y3eNnZ/U90gfr1HYIvyo6WShXEoIFw/s9QJnd5/ZkxcQFWr\nwUtMDNyjKFPnTTSLPr+vc3CGYqh8wFNxfids9KP95bMpVN8XGoTI99QqYK2tIgFl\nT8q46igOTcrg0f49ecqtQjL9F/dnZqzMlh5nJLMVGqRpNoZz9YpiS0kCgYEAxSaH\nuvaW7WxiBC+xf2vvgf585Wn9jh6QDX+MQjA7Ao7Tk4ZjDwAPMJkYd475BWp+DHSl\nb74x/nbRHwdLAfkxvYEsv4KrPR0yzdzGrXVATWcOP2bftEGPYVQnlgjjzMpLVXdt\nQErRS1vVnkavJLsxOHw++qiiyGENQk8LR2yOTnkCgYBBzy30dluBtskfBIbggdNb\nzVrmhSKDhbtOk8VyszcAB/r6nA9EITqkySFpIY039nlTwbX6SBmNlwU97tPduIz6\nndAbwc02bIvp3reICjyIpU2PpukSsFwXGONk4Zu9NVWlESfzTN4qF0VCiNoPfgTt\nYd2UWO+86D3ti4HmmtUlCQKBgDRFYvdPKfUJJ3O0sXr3QylUMAkjcPadY9QwXR+v\nafXjqHUUzG7NtTlNXg9U+PFWqtTimHpoExlEp21yoZCEYYu9FAAyxPQPKckrIAId\ndE8RY9WrkORZ/Ynwpg5BjSRe/lpKr8y8CYHRd3Hfi9BRUVuIlaofzAkUsk9CZdsq\nDREBAoGBAKfvd2PpW9B8Wu248h/zxkK4M58XQUubkFTvUm6ErH0GWFBwfq8b8Q5Y\n7/KnzZ3BefqQcNQKyoM06bTDT/YjlPLZdZtgN8UulGZQjBbLV9EIZJwRj+AUwMQV\nX+U8NAibD6yFqnOoJ0P5r8rlKPMg9+BkdZfnbhaCXb/KygYxqGcH\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/client-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIID0jCCArqgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu\nYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IGNsaWVu\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMn1VyxBY4AkODPmOxK5\nVG3F2qQ+0jNFeikwcgJPHvFamqn3cA5AIJIUVmMtBiUfjnperHVKeuPfmW1bJw4E\nne3V2eccDySoAR/BTX4kw0SPtIO3hnHyhOLX4bY4/Xw5OWgw2HMEwEwuoWxd+jpc\nGGzXY49J9gRKqxJFXR9tXD6T+1ABZPynqrTm3SYYCJoWq/C6feTSkf13HvnTnf8k\nfWcFum1Y5FegAObqbPqJwA0TGiuXSFkqw5oV0uAZzRQ7zqB6V8MB3W1U1pw86F1h\n09EN78PrW1yXX1LZrLKwlqPTVh53Y1HuT+mwJkdQjFbOGXwh3x7rmp8+A3QLD4pR\n5tsCAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI\nKwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA\nAAAAAAGCCWxvY2FsaG9zdIIQY2xpZW50LmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF\nAAOCAQEAWie6Pz2iJP6F9HfVH7anKVHeIXecwXJj4iLgEONaIcOyMcLPU4cthx1S\nOdvKAh+D9tT2PhVaIeDyYTUgFg/aaZUqI/W3odRH5HwQmE2YJDfXQusRtdFDTAUV\nXDqFkkNoJo4w3OQmlnQGm6QVReedyQ3jMTvqDRV+pa8gx6aH64jhP9fQRS4WkpYX\nd0HjWarV9/GzCP/+vGVZhwrhRG9p4F2ZCsflBzTx0YMGdo+vLDCSjwMbIT9t0T6/\nmt07Q70QSk8M3QAClrqarvLk+5z5XSZjtM06s/Z6opyqK2X8KYcOYX4WQyNFbOpy\n0YHy3iqmx/Ii0Zn5XZUXzAVGyJk5Yg==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/client-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAyfVXLEFjgCQ4M+Y7ErlUbcXapD7SM0V6KTByAk8e8Vqaqfdw\nDkAgkhRWYy0GJR+Oel6sdUp649+ZbVsnDgSd7dXZ5xwPJKgBH8FNfiTDRI+0g7eG\ncfKE4tfhtjj9fDk5aDDYcwTATC6hbF36OlwYbNdjj0n2BEqrEkVdH21cPpP7UAFk\n/KeqtObdJhgImhar8Lp95NKR/Xce+dOd/yR9ZwW6bVjkV6AA5ups+onADRMaK5dI\nWSrDmhXS4BnNFDvOoHpXwwHdbVTWnDzoXWHT0Q3vw+tbXJdfUtmssrCWo9NWHndj\nUe5P6bAmR1CMVs4ZfCHfHuuanz4DdAsPilHm2wIDAQABAoIBAHu5DodxI5i8F6ZL\n1EK7QOri+/gE+FcqqBUVtbKOcCFh5UBc8sv4Izv6s5WcXphdhbaXy0UrtK9nKyIg\nZoOi9nFewlhgCzLkrZObo3K06N9Wvjq3MukZrqkdogw1S61PjUi0K9YCwh+prYCq\n7gHUq636IecFY27ro3PVBKCdKZa3jwNIg4Oy5H0dsLgA1ma+tJaZMxDQZGyXvPqD\nwGXdkjdIefo7WQHi3yyujSs3Bu76ChmkGQfma48clWiu4HNlWI3Uc8aLBwDWVmjF\nS4T8s9HNw+nDWNHvnD45dkgKyTNAGA4rmD01OtuiC44GfZkRtupXcI0xA7d0lzd+\nyvMBlfkCgYEA/9zpUgL2Uw24H28pDXiV6Lc0PmCmHVLPQxQpDzsudX/ukaUl0tMC\ntOL9575sluXOXDSkuoMX5nDX/8C6fPNSbi4AzrqDxQ7Dc05W41MvlJ9DdIzlozEm\nLFJ4RfXccxcVCXRrgNDNHpkMKQkk3Z+zmlZeNbbhWKs2CgZGrvyFnW8CgYEAyhEJ\nZ1aRC6XrP+GbMCiN7w2X6ZcDNm89o8jXR/94hTDGvyyQ4XIs0245XjHNYVP/ghzi\niFpIRlph4QpKZebWiqgkwlsNsPKQmiBn5uqobo3VsUt+rFIhaX50GyM5cxSbSRkl\n/i9Z7ZZdj/dcsviURthKZiBn2uw+aROLCgKm71UCgYEA102h2K09cm4c/fagaQGL\nxCRGBid2IT7Jwfx5AKQgWCerLUv3JA0EPgq09gm7fs8qc1SpOXmO5w8V89TOGM74\nElcLvuocb/oYZjMJ0ojxhPLv5Geb5VM6eBl9tAFL3F0UCry4qdEKijDnlrBnIUd9\n7uW2qSSXQ/Huq0jUufMszGkCgYEAnayLnPJkviUbG77svLh4gHgn+SNYY2qMO7il\nnE3R+oRkIZsh9nmEVvtkkobkDzVfZGUrs2BXk2ZFiDfic/+bm5i3Dl3EojW09j+h\nNAQZqCLPA8i4MLjpz4rYCLEEzDLhNToFdoH2dzllCsjnsdPcyCdQbr6Mq7y6un2A\nejA1mP0CgYABUHeLemQTvsciXVAy7ZggDYmFIVPvxwYejAXWAeUdq2FEFWIqmywy\ngTISeMrWpaCpwRr97Rez5bLgYe1Crqujd03uzYzoMtuiE0027XE6CaqeDGEk7jWQ\nbnbanxoy7Ax6yjcEbqyaaG4ZbVXu5EbsCMPFE3mws3AqyF1/54YmbQ==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/ca-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL\nBQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy\nYW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z\ndCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI\nhvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw\nMzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g\nRnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI\nb3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq\nhkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM\nW5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk\ncM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa\nDPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu\nNIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3\nNqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz\nmJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i\nMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB\nu0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b\nXkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F\ngF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp\npZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh\nrU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH\ntCztUKY=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/ca-chain-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx\nCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM\nb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw\nEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z\ndDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM\nCkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc\nMBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et\naW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo\njeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q\nttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa\nAyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq\nNEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP\n1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G\nA1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j\nBBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH\nAwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA\nAAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI\nhvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp\nJE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx\nW7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy\n/i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr\ny6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+\nTp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEITCCAwmgAwIBAgIUBAbcj/g36HWhhWkPhGuGuEczfycwDQYJKoZIhvcNAQEL\nBQAwgZ8xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZy\nYW5jaXNjbzETMBEGA1UECgwKTG9jYWwgSG9zdDEiMCAGA1UECwwZTG9jYWwgSG9z\ndCBUcnVzdCBTZXJ2aWNlczEVMBMGA1UEAwwMY2EubG9jYWxob3N0MRswGQYJKoZI\nhvcNAQkBFgxjYUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjExMjE2MTgw\nMzMwWjCBnzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4g\nRnJhbmNpc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBI\nb3N0IFRydXN0IFNlcnZpY2VzMRUwEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkq\nhkiG9w0BCQEWDGNhQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAMK9Y88QNOVAyN43ESTQVCFJkBnpgAup/1oFAAv77pLuV0Wl1UVO8gKM\nW5k1qTahVSZXkX+gL0uoRe7n7XZb40iIcza1emEdMKcIOxSQ/jzj8xikziBdZrtk\ncM/MkpkDCg/45wuPUuq4umjXn7sJ3XwpKmR87PTNcSVRG7YCj8Vqze/KBhCDmTPa\nDPs0GGWBqlGHjx8LmG9WYPIxbV9wi5SYUQ4Iww29xEoZBcTPd1YluaAzF9OsYsFu\nNIuaYjh/XpZFbxGA99BfoYtIK1/X+CO1OEkCTcfS2DRwPzGV9uj39yGEmd/cOji3\nNqj55DglphjbdGR87CTKiwinKaLwXzMCAwEAAaNTMFEwHQYDVR0OBBYEFHTL54rz\nmJCUnDGtWCqne0MRIS0iMB8GA1UdIwQYMBaAFHTL54rzmJCUnDGtWCqne0MRIS0i\nMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJinrS32gInu3tTB\nu0Q3nXj99YMjHfyIO7kFrvyoZo4IEJhcPefhtt40Gbmnzo8ZJsx6HjAVDMnkj57b\nXkdRH23JKpcBeTHz1xqIcf93ij9HjO6AcVDCl5Ew3Tir2F81j/skXKqUZL4KeD0F\ngF5nQTEkpH9vlIg35T0eWygchRumKmRb+aw6/QvAmnfEWZskghV0zzPvG51B99bp\npZPaqZ25GyxqxwBcA6pUdbmBKoXRATyDwGcLhX40Rb6nCBeb2vQQTJ51lrEsCurh\nrU67Tcf+r9nS9fIWPoasmdaamEXFZNoZbw2g/EeH1OTXfR87sGCXZpBXIgsg9QIH\ntCztUKY=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/ca-interm-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEmTCCA4GgAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnzELMAkGA1UEBhMCVVMx\nCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM\nb2NhbCBIb3N0MSIwIAYDVQQLDBlMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2VzMRUw\nEwYDVQQDDAxjYS5sb2NhbGhvc3QxGzAZBgkqhkiG9w0BCQEWDGNhQGxvY2FsaG9z\ndDAeFw0yMTExMTYxODAzMzBaFw0yOTEwMTQxMzU3MjZaMIGsMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEzARBgNVBAoM\nCkxvY2FsIEhvc3QxITAfBgNVBAsMGExvY2FsIEhvc3QgVHJ1c3QgU2VydmljZTEc\nMBoGA1UEAwwTY2EtaW50ZXJtLmxvY2FsaG9zdDEiMCAGCSqGSIb3DQEJARYTY2Et\naW50ZXJtQGxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALljsfg9PNMYfmCgDQ4z+7VsfqUEHB+t/6NkACkBQi6NjNKhXxD1APfI6K5ZeCMo\njeSTmjmIDB7Shu9oU8Db5LFBxe7PE+dbHpX0/SAzBkW5UyCoX691MHhb2VbrS94q\nttBtl0U/DgtCwfVxHWf1GBhsJqsPUnBLRInxB/BMXlzhw8FSS6B5PkHmEyJgiUoa\nAyN60sEOUswa8IDzHlH/mDuYDUJKCHIgenVH7N8qZZfkK68bJfSjNHoPWT3sWPYq\nNEyPgvLqsb1ewTSAsKWTAEefhgE/DzVpsxEw613VvW/n/Tl9jay4IpQ38cuL8HhP\n1d7pF5kpg+tCDIjK6zjCCm8CAwEAAaOB0DCBzTAPBgNVHRMBAf8EBTADAQH/MB0G\nA1UdDgQWBBRMb9Of/qEuaW3cAAI66e/fsBYy2zALBgNVHQ8EBAMCAcYwHwYDVR0j\nBBgwFoAUdMvnivOYkJScMa1YKqd7QxEhLSIwKgYDVR0lAQH/BCAwHgYIKwYBBQUH\nAwkGCCsGAQUFBwMBBggrBgEFBQcDAjBBBgNVHREEOjA4hwR/AAABhxAAAAAAAAAA\nAAAAAAAAAAABghNjYS1pbnRlcm0ubG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZI\nhvcNAQELBQADggEBACjDl6q3l6F1YouwalK2sQJyn8gBjx2W5w95n+zuQrw4UGkp\nJE+wOVCmlj7mv77GOjZUhTPqW6cDKQfjCDiLWiw2Gw0HiLxgxiyOB3hS9hGCSiPx\nW7MLJ81dINq+ogEO22gC8fb0BoyqdkqteyLsjzz6aKaUnCB8UCMD/Ysjm/beVvPy\n/i1K7Ki1NnqeiCGOO8WTHdNOn2YWrt6Exbh/nsFMB7/wE8poQs2ynotTsFNBwuTr\ny6iwdnVnJLyQd6AzJiE5gOdvqRobKhUzh6C1d7wwDl3V4WAydsPXjTboRzJ0lFS+\nTp0aK3KjLHaWIehFoVgLuZhaLwXKpyRlyK54z14=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/ca-interm-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAuWOx+D080xh+YKANDjP7tWx+pQQcH63/o2QAKQFCLo2M0qFf\nEPUA98jorll4IyiN5JOaOYgMHtKG72hTwNvksUHF7s8T51selfT9IDMGRblTIKhf\nr3UweFvZVutL3iq20G2XRT8OC0LB9XEdZ/UYGGwmqw9ScEtEifEH8ExeXOHDwVJL\noHk+QeYTImCJShoDI3rSwQ5SzBrwgPMeUf+YO5gNQkoIciB6dUfs3ypll+Qrrxsl\n9KM0eg9ZPexY9io0TI+C8uqxvV7BNICwpZMAR5+GAT8PNWmzETDrXdW9b+f9OX2N\nrLgilDfxy4vweE/V3ukXmSmD60IMiMrrOMIKbwIDAQABAoIBAHrcuHCk3O+llvOg\nfqaACvvkaFYiUCUqonX9ayHBxMOnacGZ0rAhPz/39US+5KCgWtE2hQpFwIqYyXZW\ndNMmp/xVc8DdmfpE6BNHpo21Yx3IQXAuRiO5DaNmc7ZoAAK/2iJtzsfEjyElX/ey\nvGNOCmb9He9WGzNynnywsasuYYc5sr6+2b5AN0yMxCggJY9tdyxAK5Tgh0ahMumf\ngmNNziSqwRhJ8V9natT7TzFqheQe8u6NuC1uSZ7ARqHsf9v9nfjq3jscBjkcQexx\nkuBSXS43DoksMangyDRK7epbQfQf0KmmrScdlCy3vJquKWG4wsSaob+Wsyg/XvOa\nk/bCsuECgYEA55wfpvuzB4J5S9nes7D9Fd3mGi/HBjEflWFXIv0uns4zljDTEd1k\n/Ye13UfvIyVcKMiLzmVsGpf6sESdDo6Fgd/g6gjOfeGerc3t/MZLYsXVukheABhF\nYJX+4xvwVp7QxEXcftSbZjcoTL4HdTYDeKyfn0KftTm3C26MHJhron8CgYEAzOmI\nq8rqW/6CbGb12zxDNKnKgANS0MNFeLUKdJqwrX8LxYTXmXypdlwOJ66D1pAvhblR\nj7A7QBItbkvr0Qwv56a9fa2F2Nuot5VmJyHz6mL9cRGrhIZL6aFIVZ3C1iqhQKoS\n/6mAxPSWjD3u7eCFBwKeZwmWdSx3zM9VaefDwBECgYEAwWDycJqRJUEEA5faQNAS\nz/IhEFY552qWg0Pt3DHmfgOOwOTtJmpiyuhHqYVJHmAwLYEccezusNmaHxh6xc+r\nmv+RK/bEagg6U8Wv4jCyerrRs6J+kbeyHW2/jmIibkBV8Lqf2mmrglGlXUYAthWu\nGlCPSgr3i/mvYmUfqTR+EgMCgYEAvpX7GyWpIpUug1qkExwSufmuMbBlp2vnwqRI\nLDnwV/4RWc37pXNwPnjSZZAIaVlEChFaTdWw1h/SB4MvuwilycSo/CqXkiKD4vRe\nxcjrj7YwWakAqUsrcgojOBZ6sC2IO5e0AfyKmyWOnLPB9Zfcwq9p2xXszeDlMCYr\nIEyDIbECgYBT8aN5dimyIduEk4XTQ377l5i/Ng9EVOaxsuX4MhHUUScaWogo18Fq\neggKOwFXD08J0xVk0WORdkyt01/EEwhUmUUPolya8HZK2rpikpp0eIX68qSCMB6q\nIEuHf42ip+4d8IbQp7i8SFh8LNaxkUPdJi92fFuwbQ4WeNRiL4CoVg==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/ca-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAwr1jzxA05UDI3jcRJNBUIUmQGemAC6n/WgUAC/vuku5XRaXV\nRU7yAoxbmTWpNqFVJleRf6AvS6hF7uftdlvjSIhzNrV6YR0wpwg7FJD+POPzGKTO\nIF1mu2Rwz8ySmQMKD/jnC49S6ri6aNefuwndfCkqZHzs9M1xJVEbtgKPxWrN78oG\nEIOZM9oM+zQYZYGqUYePHwuYb1Zg8jFtX3CLlJhRDgjDDb3EShkFxM93ViW5oDMX\n06xiwW40i5piOH9elkVvEYD30F+hi0grX9f4I7U4SQJNx9LYNHA/MZX26Pf3IYSZ\n39w6OLc2qPnkOCWmGNt0ZHzsJMqLCKcpovBfMwIDAQABAoIBAANogyWtl+9nOdzB\nw/gL+Vz3x6DceFQ77m8p5T1QdHV33GkopNnors2inTvKN6LwIehg4fgE2q8NS+QJ\nhRtsMkcjtDiB5plqhF54A2ixvPFd3/RPdhgU6mZfNKY+Y1ZwOi3bYlfOc1oOT5zk\nITSJ0KmKouZNVWxXaCKKD90YoGGJHCxZGPkAQlX/GT3h7g8h6QUYovgKwn788EVi\nn78ldLq8AeqSiLZNK76DpnniKtaIwshslUkqPDW22EWk8YM6uvTNQQMuZzV1oBlQ\nTa5rJizOuWcIi+Mu2/QAhSVjhMSjXMyLoIrO5WkHpj9LzRwys9SQM7A1njZ9Xatl\niWWPmKECgYEA6pm1qgMG84ZhsFTxojYkrR/bs7LuLCSAIUfEoyypXJTrWNIJv5eb\nnZ/HL5o90KQHYN5T/LpZJfe/Etz+4npS6ahTXM71o51EpIN9DhNzJNduPSDaBqHP\nLiomPg+ipgPJ3ziFu4m2dK2y2TW/u9IO8+6Ah/2ozkXX4/kf+DFq+GMCgYEA1IDf\nKybhAgWImgf6m3GoZMDQOQLUUcn1cAagUUe2eNUa24lXx67ZmpJZpoHqixYylhfx\nBp8fmMD3q/OGe+j427OlVcomzlDTqnahJ9m2KFfn2tkovfJKYqZqx8r7F7kuqlgM\nr9DOAkR5s50tHz+EuQZqC1VZ796YcqnuIVoWbvECgYAhzJdxsRH1T+0PHI3bkvVh\nw+9BSowp6/BR2ycnYy3bWtE1cL1azxrqcLSf1RcG0jsF58It7SMe5zyuGQzX0EvV\nwhyQiHi3Y9cZ5J/FwWObcTY+tFb1EabpvcTYuCP0yyLweBI8XLDeyo+z35yKEM96\nsWfvL8p8PW/HNoM2nNgOhQKBgQCkxHPForSue5nqTKt84Yi+7l3FBrOX4y8iOJEP\n1LngQOQ9OuXMF3/0AOvwViWEyKZaiJ/DEZhPObgQJJeu9foXZ9iXh1HFgRhNwQO9\nwWojJ93Ha7/SX85bZUvANFuyjkxnmjPkEtPZIDz5DrLQ2tBPInEQ7pH4kjDEH4xb\nYd2pEQKBgATSX2zyg3POs3958Ls/4oegfnVclFeUf8zQpwGtsDxP6NtpVti9YGgM\nrO63dfeCuAfapxyX+I6iNw5pbDWK3TC8STjWnhhzAUZnZSoIGRB8q0yKCK8TZSMG\n3rMw+2LuO+br5tu/Hq+d6YlHbhdYW8JJFEF6QfNT31ltGE5+Nr6D\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/server-01-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEiTCCA3GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx\nCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM\nb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa\nBgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu\ndGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB\nrDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp\nc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy\ndXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMS5sb2NhbGhvc3QxIjAgBgkq\nhkiG9w0BCQEWE3NlcnZlci0wMUBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDF2MDDwVEZqC/dje8+soPRapPa9C4V8hmHuCQS9sg27HZv\nbJlywJMJE0PKm+612JoVB9p5tsXlV29gKcIK7zLEhesGlhiW9d2/7MOrPBSwqJfo\nIeabwlUfUTZ2M3XWGvU5Z65cK+cK7f0FABUzwTqb4/M3UR41cRvNtSK95nwl48Kb\nx4Dew4MIqvIN/H1qmjkNM8lsDHrCV2IyyYc1W1zDqZVU1JdpjO7tBBL6pkwlM31k\n8xjMr+hiVA64N+fESCauf4xArnqcHdhAkfnX+8DWCfSLFjP1lBZZTw1ByLEBBeNr\nkHt22RhO1mvBUHnML78k7TEFIWeKuNmOv1RGksfzAgMBAAGjgbMwgbAwDAYDVR0T\nAQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB\nhhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG\nAQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl\nci0wMS5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAngLW\nTZx4PVsbAQ+saPRJErZU4JfA8Csx1MpdOkmvqbTwpIvnkgE6mER6DA31z+Fml2J1\nq+TpGh83mZTaH2yivF1i7/EU4h/lDhCXOns+yPto1t8GbIGNRTvG9VwJjhjgbILm\n5l5cZQY7NeIryU4LmbELJCwPoHt52HnK0pfbhjCbD3Q6ugEx7xzESeqIwVS4gP5M\nso/S+lvlQi1YWn1JJrj+SM8E5k/7x1kjTQZ/sHXAsUTwdtE6Qk/xbT5dbZYuhhPU\nTF6sSfQpIO3Ju5Z+MTbcaCbJmqSe/y6J21e0Di1ve1BORKAy833F4leyPX8Ziv1x\niB8gZs4dMh88tJHqgw==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/server-01-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAxdjAw8FRGagv3Y3vPrKD0WqT2vQuFfIZh7gkEvbINux2b2yZ\ncsCTCRNDypvutdiaFQfaebbF5VdvYCnCCu8yxIXrBpYYlvXdv+zDqzwUsKiX6CHm\nm8JVH1E2djN11hr1OWeuXCvnCu39BQAVM8E6m+PzN1EeNXEbzbUiveZ8JePCm8eA\n3sODCKryDfx9apo5DTPJbAx6wldiMsmHNVtcw6mVVNSXaYzu7QQS+qZMJTN9ZPMY\nzK/oYlQOuDfnxEgmrn+MQK56nB3YQJH51/vA1gn0ixYz9ZQWWU8NQcixAQXja5B7\ndtkYTtZrwVB5zC+/JO0xBSFnirjZjr9URpLH8wIDAQABAoIBAA/+b8bjb9z1Hbl5\naefVopZhWUaQCtG3Wp0AI9psnM7j2sczLkx6iOho6EgOxwNoWTxuABlqzyC1KsnJ\nVmhv4djFQrSrmZ1Kjvye9Up4duu5FV6srunUkfEQLajsjBAc6coUOaI14l2d06B9\n2zjt3AESMm08X4bOeALvK7nSUiL13CAzKfv/ufM+bqi3nXxzNCBhtd51sHDaYqr7\nnbhFMrX1frhRb0a5k0Q0UPf3V8pUf04difa0MZJ8tQTW8HYLrEVjOE8NPp5yGQj0\nUX0d4XqS5LVDQl6Y1DAOQ7IsDq91gGKa8sk48HVvnwRjez0igAtbTq5I2DTydFDf\nD19fpCECgYEA8eyHbdojrawzQM9P5QADRTLN+r2rk8Tv0cW2CdJTmxZiU6HhVqxU\nNrP5G9FpCkn0jOikzhGdJalTgd3TKWMbZS9gPdezpj1viIT7v7NsK7Gnc8EcF+v8\n4H+Rj7jsxiu6ryNoNPmXZSsIyfGIvbOE9SwUki1Cc2NVRYf0g6aLfXsCgYEA0Vuw\n2DRg7FCP9Ck08eburI163OEKE2sNIc2z4O6I0gy/u/U86Y/bLBrpRypiSEAf1r/p\nyEEzzFqSNf7UNCLumdKk4LXUk3FMPygMZY/D7s1j/MVW/6HWeMeMy7IKKqOclTT1\nR1nNjbjRkorBRyHDkYqDrAGLuQUleWLrqSzIyekCgYBjxm5wcvgmB5A32YiU5LV0\nk6h6EkGyNxXFiWozkMgkfU3eOjRqf7ZXvVAvFeXhdXDjsItP5dnPD4++TtNpDVPe\nHnTt7IlONaZLQrVlccVaG/H4/prsjsqDeHl7MgSNErnyw4KV3p5+/gmo5/HCc0iR\nqTVuuDXgywX/IDxLE6QSAwKBgQChDDWXu4MrhjWWjvRJeWn0lskCjKJhmaH6dPCA\ngT4CxwffIKGA0ca5wOHeer4r8hgL7Il8IJwmAS4kFylKCe0dqypmKbmiyi7rDnWq\n0tLYKmtWEMAB2Y7QTkECmKy1bDKRnLFp96zl7lxYrCBOBa0ZkkID2RSQeWMAY5YB\nB5BVsQKBgQDRIIKN0UhGqHZRcHb26c5lE5Rs/8nX2nyFgxKlwno6Dk5I0Cdo9CDh\nz8f5klwm6Rgmsvr41crTg4IOimvCYGu94vyJTOKo+ln8Ky+02mdReomCVKoFdIB/\nWR+1aFvk1+r9aRr7MyPcHUGuIdUDJBnWv0pfNH1TUyvjpE3rMoaIqg==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/server-02-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEiTCCA3GgAwIBAgIBAzANBgkqhkiG9w0BAQsFADCBrDELMAkGA1UEBhMCVVMx\nCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRMwEQYDVQQKDApM\nb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRydXN0IFNlcnZpY2UxHDAa\nBgNVBAMME2NhLWludGVybS5sb2NhbGhvc3QxIjAgBgkqhkiG9w0BCQEWE2NhLWlu\ndGVybUBsb2NhbGhvc3QwHhcNMjExMTE2MTgwMzMwWhcNMjkxMDE0MTM1NzI2WjCB\nrDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNp\nc2NvMRMwEQYDVQQKDApMb2NhbCBIb3N0MSEwHwYDVQQLDBhMb2NhbCBIb3N0IFRy\ndXN0IFNlcnZpY2UxHDAaBgNVBAMME3NlcnZlci0wMi5sb2NhbGhvc3QxIjAgBgkq\nhkiG9w0BCQEWE3NlcnZlci0wMkBsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDWzAyE9A/EzJv5TcQlM9o4C/UOIQD4Rzs8S80x4ckGyF00\n8xF5FMmGzGMka4a9FNPDBzbBM41hlhJXcDhxRfvQodzvpOUXqi8Siha2SsyFzhD5\no3exxV8ETkHMtST1nW/bwuYqQbDf3PLpVV5qZgv4jIByyGE4nCPDqHMKX0h1xXIQ\nE3fv2qTcs5j72c3jcTlINFJ9aqTaFgSvvGwg+0KLrOjs1Og3408zTIHYSjTScFHd\nheDLHjfmIx8oqXj4+g3F+5BgLNvO2E6if9YYGOFjxzRCPOrRw4JB8nveI1gZhTD9\n+OmUkIMoNLX/+4Kbq+a47/kqrXv5GxLxwola8do3AgMBAAGjgbMwgbAwDAYDVR0T\nAQH/BAIwADALBgNVHQ8EBAMCBeAwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzAB\nhhVodHRwOi8vMTI3LjAuMC4xOjg4ODgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG\nAQUFBwMCMEEGA1UdEQQ6MDiHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCE3NlcnZl\nci0wMi5sb2NhbGhvc3SCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAH9jr\nP5EepmTiJzxYsrGE9q5BqhspBYqomgCbLWH3vDX50MwmxZ4ETe1icpHbXiddDA1k\ndfBxn/1HXEWXxwHCTgoRsSzufgacHdk/LgXCxkFB0JBZ9c+ewOFdUQihxFUx1rMt\nSz0QnO40aopTk7PN8YlinihAvil/JC8fN54T0myCSjtE54dtZq2wwFiiorxR6hyi\nhLL4zYKi/kCFa6DG0qmR6Yl2VsKiq3yp+6QxZ8w5srXTeeOoTOEyUkYKCTND7U1G\n5vyCdMFNIDWHSB35I+7e0D1QFaH5XQlFztUfmLoOMSMLbOuzG2cFO6J+Y0AgTP5h\npZ5mw1ixmVdNmkSf3g==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/desgsign/server-02-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpgIBAAKCAQEA1swMhPQPxMyb+U3EJTPaOAv1DiEA+Ec7PEvNMeHJBshdNPMR\neRTJhsxjJGuGvRTTwwc2wTONYZYSV3A4cUX70KHc76TlF6ovEooWtkrMhc4Q+aN3\nscVfBE5BzLUk9Z1v28LmKkGw39zy6VVeamYL+IyAcshhOJwjw6hzCl9IdcVyEBN3\n79qk3LOY+9nN43E5SDRSfWqk2hYEr7xsIPtCi6zo7NToN+NPM0yB2Eo00nBR3YXg\nyx435iMfKKl4+PoNxfuQYCzbzthOon/WGBjhY8c0Qjzq0cOCQfJ73iNYGYUw/fjp\nlJCDKDS1//uCm6vmuO/5Kq17+RsS8cKJWvHaNwIDAQABAoIBAQChTG2CUbydroDa\nP9mxjjSqreACEiqmRudmqg2XDdRl8YR8xKM0Z7XGMimpKc3uo3s6E2q8vrfGtmlj\nm/PmrAUjcMl2dD9M/BGJPIU9swHO4SXCMU0rA+oMU6/5x3XPs8BSKROqW9Y05bjM\nG71g7OzEYs36ZBsN5cK9pPtuqIrDVHfPQI+Y3eP7wBnLscK/AqUoapbc6oQqarl4\n9rdHY4IHsab3xTxt/riSooia17R6xrrSpDcg+tdx8CphjD53zLN2Y8jHpl6LVJEN\nrRFud68sDEtdU5EOj0WiQIke0Z8yBOgAt2YKqrU+O+Wuh2ad9pBhhp7oLvaRi0C8\ns+URKM2BAoGBAO4LtDgpWyMC3FfMDuSGT7hwp5+TtKM9Js2mZsQYZbJRX3BVK/G1\n+/LhZzpqcKqGAarZ/yz1cLUEprbEXX7+TFbPKKpBsEwTOQXxlvhmjSYxyHVfBCbB\n+o43EolRey0Zdnnawk2OpqOVWSQafrp5JyJumyvfwES3Oc2iT8288bXhAoGBAOb/\ncwLtk5I9Bx0BFUDFNgn8g8QXfWJ0axyCUQE2UGn/tiXodpkrvFF2QrpEQ+OG+iKB\n0d6I9lLNThw/7uKvecYN1HliWnJ9dzUe59nguwG+PNZJgSmBhnPs4Wg/gsG6h7K1\nJZ3VPeyNaSRK+pCn8sWkcQjumImZR/PHKzsDdOMXAoGBANp18UoNYi9qY69LfWtq\ne+Unth30HzYkW+UlznAud75DgZQFBlRIkFWhWOw6XPWSEBus/stS4MGv6BQZeDig\nxoxwh8BgkpvulEmJIuUKsIUZ8P8OWS/8m6ZCkodlOOb11E4WXnVw8it0V5+TlTQr\nFag627tTGA+4G5tFV3nX6ffhAoGBANloGOXrlhVvzK8WotsYATk66QT4mrC8I0ds\nuzKp3Ns2qUdaV6znhdEhvcGzmDWfhvJNPqn0O+lIgziBT6MYRkMKJyyrTbctsLFV\nSh88rKUCWB3Shnb7CgE3NBq6k6Ujmq2uYh3/Yc2udgOLcfINr6cmkqA2d2gh1J9y\nl5RuN2e3AoGBAN9FEe+OLRbq+qnMHkEoozoshxgEqv/M/BgXvoX8kelcp1zyBaWX\nt5yelDn7UjANhJNBsripzvaLilMxracm1Oy0fCtRJ3dvwm6djUelhSPBV9HJSEN8\nt2Sv1z45SNq3l2cz8pVvoQDNmUbh0QCQnScrLN8L3vBJSibAJgSdml51\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/gen.sh", "content": "#!/usr/bin/env bash\nset -euo pipefail\n\n# gen.sh generates certificates used in OCSP tests. It generates a CA, client\n# certs, and a few different types of server certs with different OCSP\n# settings. This requires OpenSSL, not LibreSSL.\n#\n# usage: ./gen.sh\n\n################################################################################\n# Setup CA\n################################################################################\nmkdir -p ./demoCA/newcerts\nrm -f demoCA/index.txt\ntouch demoCA/index.txt\necho \"01\" > demoCA/serial\n\nprefix=\"ca\"\nopenssl genrsa -out ${prefix}-key.pem\nopenssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \\\n\t-config <(echo \"\n\t\t[ req ]\n\t\tprompt = no\n\t\tdistinguished_name = req_distinguished_name\n\t\tstring_mask = utf8only\n\t\tutf8 = yes\n\t\tx509_extensions\t= v3_ca\n\n\t\t[ req_distinguished_name ]\n\t\tC = US\n\t\tST = CA\n\t\tL = San Francisco\n\t\tO = Synadia\n\t\tOU = nats.io\n\t\tCN = localhost ca\n\n\t\t[ v3_ca ]\n\t\tsubjectKeyIdentifier=hash\n\t\tauthorityKeyIdentifier=keyid:always,issuer\n\t\tbasicConstraints = critical,CA:true\n\t\")\nopenssl ca -batch -keyfile ${prefix}-key.pem -selfsign -notext \\\n\t-config <(echo \"\n\t\t[ ca ]\n\t\tdefault_ca = ca_default\n\n\t\t[ ca_default ]\n\t\tdir = ./demoCA\n\t\tdatabase = ./demoCA/index.txt\n\t\tnew_certs_dir = ./demoCA/newcerts\n\t\tserial = ./demoCA/serial\n\t\tdefault_md = default\n\t\tpolicy = policy_anything\n\t\tx509_extensions\t= v3_ca\n\t\tdefault_md = sha256\n\n\t\tdefault_enddate = 20291014135726Z\n\t\tcopy_extensions = copy\n\n\t\t[ policy_anything ]\n\t\tcountryName = optional\n\t\tstateOrProvinceName = optional\n\t\tlocalityName = optional\n\t\torganizationName = optional\n\t\torganizationalUnitName = optional\n\t\tcommonName = supplied\n\t\temailAddress = optional\n\n\t\t[ v3_ca ]\n\t\tsubjectKeyIdentifier=hash\n\t\tauthorityKeyIdentifier=keyid:always,issuer\n\t\tbasicConstraints = critical,CA:true\n\t\") \\\n\t-out ${prefix}-cert.pem -infiles ${prefix}-csr.pem\n\n################################################################################\n# Client cert\n################################################################################\nprefix=\"client\"\nopenssl genrsa -out ${prefix}-key.pem\nopenssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \\\n\t-config <(echo \"\n\t\t[ req ]\n\t\tprompt = no\n\t\tdistinguished_name = req_distinguished_name\n\t\treq_extensions = v3_req\n\t\tstring_mask = utf8only\n\t\tutf8 = yes\n\n\t\t[ req_distinguished_name ]\n\t\tC = US\n\t\tST = CA\n\t\tL = San Francisco\n\t\tO = Synadia\n\t\tOU = nats.io\n\t\tCN = localhost client\n\n\t\t[ v3_req ]\n\t\tsubjectAltName = @alt_names\n\n\t\t[ alt_names ]\n\t\tIP.1 = 127.0.0.1\n\t\tIP.2 = 0:0:0:0:0:0:0:1\n\t\tDNS.1 = localhost\n\t\tDNS.2 = client.localhost\n\t\")\nopenssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \\\n\t-config <(echo \"\n\t\t[ ca ]\n\t\tdefault_ca = ca_default\n\n\t\t[ ca_default ]\n\t\tdir = ./demoCA\n\t\tdatabase = ./demoCA/index.txt\n\t\tnew_certs_dir = ./demoCA/newcerts\n\t\tserial = ./demoCA/serial\n\t\tdefault_md = default\n\t\tpolicy = policy_anything\n\t\tx509_extensions\t= ext_ca\n\t\tdefault_md = sha256\n\n\t\tdefault_enddate = 20291014135726Z\n\t\tcopy_extensions = copy\n\n\t\t[ policy_anything ]\n\t\tcountryName = optional\n\t\tstateOrProvinceName = optional\n\t\tlocalityName = optional\n\t\torganizationName = optional\n\t\torganizationalUnitName = optional\n\t\tcommonName = supplied\n\t\temailAddress = optional\n\n\t\t[ ext_ca ]\n\t\tbasicConstraints = CA:FALSE\n\t\tkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n\t\textendedKeyUsage = serverAuth, clientAuth\n\t\") \\\n\t-out ${prefix}-cert.pem -infiles ${prefix}-csr.pem\n\n################################################################################\n# Server cert\n################################################################################\nprefix=\"server\"\nopenssl genrsa -out ${prefix}-key.pem\nopenssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \\\n\t-config <(echo \"\n\t\t[ req ]\n\t\tprompt = no\n\t\tdistinguished_name = req_distinguished_name\n\t\treq_extensions = v3_req\n\t\tstring_mask = utf8only\n\t\tutf8 = yes\n\n\t\t[ req_distinguished_name ]\n\t\tC = US\n\t\tST = CA\n\t\tL = San Francisco\n\t\tO = Synadia\n\t\tOU = nats.io\n\t\tCN = localhost server\n\n\t\t[ v3_req ]\n\t\tsubjectAltName = @alt_names\n\n\t\t[ alt_names ]\n\t\tIP.1 = 127.0.0.1\n\t\tIP.2 = 0:0:0:0:0:0:0:1\n\t\tDNS.1 = localhost\n\t\tDNS.2 = server.localhost\n\t\")\nopenssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \\\n\t-config <(echo \"\n\t\t[ ca ]\n\t\tdefault_ca = ca_default\n\n\t\t[ ca_default ]\n\t\tdir = ./demoCA\n\t\tdatabase = ./demoCA/index.txt\n\t\tnew_certs_dir = ./demoCA/newcerts\n\t\tserial = ./demoCA/serial\n\t\tdefault_md = default\n\t\tpolicy = policy_anything\n\t\tx509_extensions\t= ext_ca\n\t\tdefault_md = sha256\n\n\t\tdefault_enddate = 20291014135726Z\n\t\tcopy_extensions = copy\n\n\t\t[ policy_anything ]\n\t\tcountryName = optional\n\t\tstateOrProvinceName = optional\n\t\tlocalityName = optional\n\t\torganizationName = optional\n\t\torganizationalUnitName = optional\n\t\tcommonName = supplied\n\t\temailAddress = optional\n\n\t\t[ ext_ca ]\n\t\tbasicConstraints = CA:FALSE\n\t\tkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n\t\textendedKeyUsage = serverAuth, clientAuth\n\t\") \\\n\t-out ${prefix}-cert.pem -infiles ${prefix}-csr.pem\n\n################################################################################\n# Server cert (tlsfeature)\n################################################################################\nprefix=\"server-status-request\"\nopenssl genrsa -out ${prefix}-key.pem\nopenssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \\\n\t-config <(echo \"\n\t\t[ req ]\n\t\tprompt = no\n\t\tdistinguished_name = req_distinguished_name\n\t\treq_extensions = v3_req\n\t\tstring_mask = utf8only\n\t\tutf8 = yes\n\n\t\t[ req_distinguished_name ]\n\t\tC = US\n\t\tST = CA\n\t\tL = San Francisco\n\t\tO = Synadia\n\t\tOU = nats.io\n\t\tCN = localhost server status request\n\n\t\t[ v3_req ]\n\t\tsubjectAltName = @alt_names\n\n\t\t[ alt_names ]\n\t\tIP.1 = 127.0.0.1\n\t\tIP.2 = 0:0:0:0:0:0:0:1\n\t\tDNS.1 = localhost\n\t\tDNS.2 = server-status-request.localhost\n\t\")\nopenssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \\\n\t-config <(echo \"\n\t\t[ ca ]\n\t\tdefault_ca = ca_default\n\n\t\t[ ca_default ]\n\t\tdir = ./demoCA\n\t\tdatabase = ./demoCA/index.txt\n\t\tnew_certs_dir = ./demoCA/newcerts\n\t\tserial = ./demoCA/serial\n\t\tdefault_md = default\n\t\tpolicy = policy_anything\n\t\tx509_extensions\t= ext_ca\n\t\tdefault_md = sha256\n\n\t\tdefault_enddate = 20291014135726Z\n\t\tcopy_extensions = copy\n\n\t\t[ policy_anything ]\n\t\tcountryName = optional\n\t\tstateOrProvinceName = optional\n\t\tlocalityName = optional\n\t\torganizationName = optional\n\t\torganizationalUnitName = optional\n\t\tcommonName = supplied\n\t\temailAddress = optional\n\n\t\t[ ext_ca ]\n\t\tbasicConstraints = CA:FALSE\n\t\tkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n\t\ttlsfeature = status_request\n\t\textendedKeyUsage = serverAuth, clientAuth\n\t\") \\\n\t-out ${prefix}-cert.pem -infiles ${prefix}-csr.pem\n\n################################################################################\n# Server cert (authorityInfoAccess and tlsfeature)\n################################################################################\nfor n in {01..08}; do\n\tprefix=\"server-status-request-url-${n}\"\n\n\topenssl genrsa -out ${prefix}-key.pem\n\topenssl req -new -key ${prefix}-key.pem -out ${prefix}-csr.pem \\\n\t\t-config <(echo \"\n\t\t\t[ req ]\n\t\t\tprompt = no\n\t\t\tdistinguished_name = req_distinguished_name\n\t\t\treq_extensions = v3_req\n\t\t\tstring_mask = utf8only\n\t\t\tutf8 = yes\n\n\t\t\t[ req_distinguished_name ]\n\t\t\tC = US\n\t\t\tST = CA\n\t\t\tL = San Francisco\n\t\t\tO = Synadia\n\t\t\tOU = nats.io\n\t\t\tCN = localhost ${prefix}\n\n\t\t\t[ v3_req ]\n\t\t\tsubjectAltName = @alt_names\n\n\t\t\t[ alt_names ]\n\t\t\tIP.1 = 127.0.0.1\n\t\t\tIP.2 = 0:0:0:0:0:0:0:1\n\t\t\tDNS.1 = localhost\n\t\t\tDNS.2 = ${prefix}.localhost\n\t\t\")\n\topenssl ca -batch -keyfile ca-key.pem -cert ca-cert.pem -notext \\\n\t\t-config <(echo \"\n\t\t\t[ ca ]\n\t\t\tdefault_ca = ca_default\n\n\t\t\t[ ca_default ]\n\t\t\tdir = ./demoCA\n\t\t\tdatabase = ./demoCA/index.txt\n\t\t\tnew_certs_dir = ./demoCA/newcerts\n\t\t\tserial = ./demoCA/serial\n\t\t\tdefault_md = default\n\t\t\tpolicy = policy_anything\n\t\t\tx509_extensions\t= ext_ca\n\t\t\tdefault_md = sha256\n\n\t\t\tdefault_enddate = 20291014135726Z\n\t\t\tcopy_extensions = copy\n\n\t\t\t[ policy_anything ]\n\t\t\tcountryName = optional\n\t\t\tstateOrProvinceName = optional\n\t\t\tlocalityName = optional\n\t\t\torganizationName = optional\n\t\t\torganizationalUnitName = optional\n\t\t\tcommonName = supplied\n\t\t\temailAddress = optional\n\n\t\t\t[ ext_ca ]\n\t\t\tbasicConstraints = CA:FALSE\n\t\t\tkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n\t\t\tauthorityInfoAccess = OCSP;URI:http://127.0.0.1:8888\n\t\t\ttlsfeature = status_request\n\t\t\textendedKeyUsage = serverAuth, clientAuth\n\t\t\") \\\n\t\t-out ${prefix}-cert.pem -infiles ${prefix}-csr.pem\ndone\n\n################################################################################\n# Clean up\n################################################################################\nrm -f *-csr.pem\nrm -rf ./demoCA\n" }, { "path": "test/configs/certs/ocsp/server-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIID0jCCArqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMHExCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwHU3lu\nYWRpYTEQMA4GA1UECwwHbmF0cy5pbzEZMBcGA1UEAwwQbG9jYWxob3N0IHNlcnZl\ncjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN/v4lwsEQqEr8gqG6Xu\nUjNfvUBN/enc/26FqtsAF6ms0r4oHcyR3RZQGQj+Z0RF3Wu0Kq9692gk7FD/QulE\nhYJTjq6lEwvETuUHbkNmIAppNJW1JvgLsTOfm38VorBVU5PUMbrcfsVsFijXVACj\n9VMZ23So4dxtlvnqrd5/fVx0Pql5EjY87bJEKH5Zngy1v+AR5kybZaorOX9T4/Nl\ne0P184GwGs15hKAokoQMPm9uIhG527JMyhQh5J/2wooY2DBZ9jDt5FVXNpb0C+nr\nM+AULk5QHQsobTtmC3RSNHiNw5B5w+gmauhGziurq8gcx0DctqAslKFBkCLkL9fc\nF30CAwEAAaN5MHcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYI\nKwYBBQUHAwEGCCsGAQUFBwMCMD4GA1UdEQQ3MDWHBH8AAAGHEAAAAAAAAAAAAAAA\nAAAAAAGCCWxvY2FsaG9zdIIQc2VydmVyLmxvY2FsaG9zdDANBgkqhkiG9w0BAQsF\nAAOCAQEATM/K671w3aHt665HBMawzMIZZPq/ZoBfEUkSUW9KdnQHgTxatHcZonsL\naFn4XZBYQ0Pqkz7H1w39mHdvpURQ5ZMnsmn4jH3LECsOtQ4ztrLk2fhLSoMQBVdb\nUjdYhrM8AuILKRCzOBNsDm/ZB/vPSlmYhnaEBUjO0t+I/A0X1z5eDcYPLl578kfJ\nWjlvRluWr7Uku1DaZUy7TByYvUuOjP4c33DAnbZ5Sldx18repZ20REASxsCpa/CW\ntptxVfUvLcGRHIY0FxOn+5Pfm1QDo2uh6yVYHgsOCh1qW8FHfJvgnrMlvvXniKXu\n5H6A5GeyCkIVvAENDfl1cN9LaV5eQg==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA3+/iXCwRCoSvyCobpe5SM1+9QE396dz/boWq2wAXqazSvigd\nzJHdFlAZCP5nREXda7Qqr3r3aCTsUP9C6USFglOOrqUTC8RO5QduQ2YgCmk0lbUm\n+AuxM5+bfxWisFVTk9Qxutx+xWwWKNdUAKP1UxnbdKjh3G2W+eqt3n99XHQ+qXkS\nNjztskQoflmeDLW/4BHmTJtlqis5f1Pj82V7Q/XzgbAazXmEoCiShAw+b24iEbnb\nskzKFCHkn/bCihjYMFn2MO3kVVc2lvQL6esz4BQuTlAdCyhtO2YLdFI0eI3DkHnD\n6CZq6EbOK6uryBzHQNy2oCyUoUGQIuQv19wXfQIDAQABAoIBAC5rWl/C3rFwecOj\nPuHxeeaeVOuMfzLIFcbCPH1zEnSgl3rFdA/guJSUe+wKWDulw9U8npCLi9dxt+6+\nSw9xnb87NNts6nrI8ZW2KZwdAk1GK5lQ2MgYHF5YGnKIeJXWyiFdngVfCYRA+IL0\nx7vuQL8+H+iZzV/U87PQesQhZ0oPh03n/4OtSo1bnPOoiPLAC+GVVpSSKoh13beb\nGpIRXhGNXonGPWnj1t/oBHQwLDfDndnljkRaWTzxeCZGcQ1wYucf6jne5dI1E21q\nvrdyK8GL4SqzHC2SHnJO24aKam/Xr7YqOM2T+XsVywKxE4I1icT2YhQAgRuN8Nlg\n7CELdAECgYEA+CEhY7JYfmRb/+PIvogRr67uaYtxYygvDcYfse5O/V/HOzotIXIT\nt99deD8XOFpWhJHdDzDD/TStb/rmwu5kavMQzmQafyesu7okk1NtB+fZtMLQfF9/\n0+bW9uJQFyc1EVHKgyya3UidXTcmKcExKQZT+jdgFz614zndcOKl0H0CgYEA5wpO\nWIjjxVqke5ucrVSRsm37l0BZYeea2vxLAzUV4txVw3hcOgJo+FhEGFy6EjNSV2OR\ny589gZDEPZ/LNeaj7nbNUqf2xZV7MAXBgUCtj7fJqMbVhbXZB0w0hIknDL+td2WZ\nbocuVExTwRwXyuSCgUFo0Bz++L0cR3JK6kNBEwECgYEAyOfKaTbWgEAyXZbJy7vQ\n1jcFw1+sh2TZ9IUe1KroOi962XHZaOM9I/wvalVrL621r9GK8+nARxyH8cttXRg5\nJn94dCSJb7toGPg29TLvbR9FHx8+P/XzQlf+ZhgIUTbluQhIuL09Bz7sa7VjqRtL\n+rOs+0QrAac9DqajretV5uECgYEAgaXe4P+wEQcUVei0uu9B8waUsAOEJNR6qXf6\nAArCBVPvLIlV95dyoCmnzKP8Jkp2YmOVZNYvBY3fEVWiCtUqGJ7CCSgH6kg/oGsa\ncxWAT62qk/M/zpCFAPtaXSU5rIXDKcTxnHxvGw7Z0PuavlgMg8vYrTAYRCyaud0A\n/QRQeAECgYBuRDM+mk5EDbsi6MOD24x6krRDHr5/Ch5xCQXK2FhU8zcQ8P24UixU\nRe71LBsYLBHkhB/slofGdBvgeiVHwJyVWA9c3+kb+IwSilNRLV7IwxrfLg3xSVBu\n0KEwWLSXlJmPnGWObpBmz62HrfquyMME4srQrNfW1q+Qm8OlZIeInw==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIEBjCCAu6gAwIBAgIBBDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGAMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKDAmBgNVBAMMH2xvY2FsaG9zdCBzZXJ2\nZXIgc3RhdHVzIHJlcXVlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB\nAQDViockMMFbHDOsiD+JLalZc1g3rd10xbXwHFHsgFQUdr+g6guGnnQhKMHEsQWf\nbGmn5G+K0QG2jxpz2IpfXA2UpMIrYLhIcf5faEbUgmwd8zaNXNs0I6sZZk8GrKYw\nPLHhXam94FUfVwsecJ/V+sK7limOB/T1AuPamBhgqIgubz40N3Qmkh9J/rkhq2b1\nffgr2v6qKjQ8bIyPPDh9OB10KffjqoaN8ogXuE3hZdQTniRWW1nT38NqQdJg6T+N\ncEheH0H4pYUSx/TdF6AHXMxWFD6lX9nLU9UGpXLAHY63rVJCv2KIMMsCGBUPKkKK\nTujdFu2KHh52CstdasQ3gpnRAgMBAAGjgZwwgZkwCQYDVR0TBAIwADALBgNVHQ8E\nBAMCBeAwEQYIKwYBBQUHARgEBTADAgEFMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr\nBgEFBQcDAjBNBgNVHREERjBEhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABgglsb2Nh\nbGhvc3SCH3NlcnZlci1zdGF0dXMtcmVxdWVzdC5sb2NhbGhvc3QwDQYJKoZIhvcN\nAQELBQADggEBAIObz14Fkbn0Cp6qo1Sbvv8owyKxPULnV3i6qx9abThlAusbMvjt\nI4MXVmgn+mNV8raivSauqKKSp0QWKVrToMUIYurJOHqDBf7idL/g6ZP9u0RIcxE2\nb4gK54xdfh2gnc4BS+5LITB0bS96zFouz5gfz+pj7Yoe8dteylSDeG7rlO6g6qgI\nY2EkS9eayCTfr99joCfvHhuJxqWFQq++OAPujbMSC03CKb87Jg7jk3WKq8RzFgut\ntuTtJjPGvDgv15axrQk1zktIzzfMG3//gOAVH185AUbIA5tsiUmXY+q3mHxJejPS\nsr1dXay2Kw2scEgWeYiu4X7SrJg2C9ksCtw=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1YqHJDDBWxwzrIg/iS2pWXNYN63ddMW18BxR7IBUFHa/oOoL\nhp50ISjBxLEFn2xpp+RvitEBto8ac9iKX1wNlKTCK2C4SHH+X2hG1IJsHfM2jVzb\nNCOrGWZPBqymMDyx4V2pveBVH1cLHnCf1frCu5Ypjgf09QLj2pgYYKiILm8+NDd0\nJpIfSf65Iatm9X34K9r+qio0PGyMjzw4fTgddCn346qGjfKIF7hN4WXUE54kVltZ\n09/DakHSYOk/jXBIXh9B+KWFEsf03RegB1zMVhQ+pV/Zy1PVBqVywB2Ot61SQr9i\niDDLAhgVDypCik7o3Rbtih4edgrLXWrEN4KZ0QIDAQABAoIBAQCS0tXKr28y+vgY\np/Gvm2K2a1V7qyL6eDp5Zq95gl7NVzy5IlNcczR73C8m02R/UiZvVuTOuJiJ8mkI\nwBmcKYn46zPKaY0r2p1A8kzJLwexrBmVJwRdHC03oJ2zhCAcSI8x2pmsQUOl8c5m\nVp+/m9Qq8LuPua7Wi+8ozKzuQNKpwXcdPlD8jXbQNVmjbFAzSnFukxzHDxkEV2MO\n9seOf0b+2ByoBsAEKW6whdHX7qEJ68OjAbv6lYu0dJmAA3v9rXdKk1tYKorvGRms\nsRcSgvjGol+SwM7daa1V+Pa4c8DszSfieocCWJt1wde4BEMqmdgZVRXxjJVHzEPa\nYkZsLTCxAoGBAPfUARuZlH6yfVi2kK3rpjjqToAzSTQW3GRiEAb1/UeE3XKUor/N\nbJxHwB8Zxkk06Qfjc1Y7MP45qTfDUwk6pLJSCYsdX/c1rtKPHKTGKrg7qy6vYqNN\nsNv/E2C8spTCe0FD3hOFnRPBFtZrfRuWl8r7eI/uiTzES0E2QX/s1u/tAoGBANyV\nGJSyvs7vZHXTjaTd5prfpcGZFgdB0DivGjFISfTJ4cqFlxb7doDUh0//xIhoKMP9\nBnWpV7c+FN5sOReB+wkbqiWqRaflm8BhEJmHddYfpz3kgBBrfOHAA6reeG1kBJ5h\nNqSagt3VMmmkHSrGtuzeZAHF/owH/cAqgEIaE2z1AoGAIgWrxUM3PJF6XcRqZkX0\ngtm/vx/LS+hbhzhjJOF+TOQzlnhLQ3OLFoVPHbXnH8OwvhF+kvb7SdtWnL4m1xyC\n4awbfUqiEwj+oA9fikteL/6ZCIaxTuPqhLkmyt/80ClGzHXptdpg7wZSAUuuWCw7\nWHVfXrsLghkcj65IHazA8R0CgYEA0mQPPv1CS4RAULYIewD/zCaXJhHK1f8rCHXT\nSMBHcgkAqLdExjHw9K5BpccxgF3AzDbRa3aq6Gd5ZjDZP5fFhglx/1zp9VtmdFbg\ni1+NwD8OuFTy5TZwta38kYSCXuwwD5Rvlw6c1dNcszBKdZt1rHXt81cTFCMnH5wq\nG2JdtuUCgYBtUJVhPpUn23Lnmc/UEH+kVHAx+YytfOngUbJoPBe8HgNUnB4Gczii\nuQtKfJMTEXz/e45tAmNTaA2MwTAgbD2sH1gJDnALqUTxoWfFW4qTwl5ZB4CVue2X\nimQj/a/PallX0KENE+0VCQRhSwdhO60tMHJhK1KEycorHo+57QHJrA==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-01-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBBTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9MT5rT\n8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr/BqU\nBQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIWzgxr\nUzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUrWFwN\njngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxFfzwH\n4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAxLmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAPyxQodHHHRAW9t9A3jR1LE9FkEzN\nFmzFb3sZpYIzw/wTmD0KvbW3D8YVe/lj5e/LxhtHzun+AnFa/JXoK0t5uLzItges\nzGEyOGFj3zP+E9Chr1fr2X8sHJjqnUrknC9dQPoiBKlNRCWTP2eeKC+yODgEmgzS\nfEdcuuqGuM7MFbf3eVwtkWNCv1R14GZN4cuSeQn3xA8/8aLcGEHV2N1GLVxK55x0\nhbB1hjTaq/t9a66ByIiaeEex832H1MRRkPMhEQxAIx8HZFQJSX+yf9CtTJTc18Z+\nI/1utSjX9VW+4YW/vN7kvYgQ3YQUbMSGjqkROe/hb12TCTSbSt6O7P3JTw==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-01-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA3yzlI0YFwf0JWbT1kOtRc2HbTfU4ktVmfYAgNHYry1ARxw9M\nT5rT8qSV/xZR7EAj6lmE2AtYFeUI8WIVIYd8yrEegM4U5Foytbjem+xgZOJ4n7Mr\n/BqUBQQ/7IjPJ/EymIdvTJ6LEtZQNWyuHgNL2Z1MCJPPUBLwSujS+cPebKxrYMIW\nzgxrUzT8EK8VPJQGo/GJdQLk53GVS15Khtn+eymviFoNYEToLY9geaoewy1DWxUr\nWFwNjngB7iMBvrIjq+Pn9exObVHSpbdPHYDqKvdEykzpRl+bPS5O371z59/IQbxF\nfzwH4k3blC/M4JSN/HRsZQvnERyjHTjjwgadowIDAQABAoIBADIx0seANENROg0n\nGiQiCQu+cqiyAqtju812R8gE3Ay6UYVH943f8NOeGO5qgrp0Ip/OyoYc5w7lmIFR\n4hbMdlDQTWAC+cfyUX11uvuuym9mUGL61WW6RJhsQERe8Ni28v5DV9EPx4/RC/E4\nJGEztJeZPUSgmfwUhl8SxoDOgPRH18ilJ2kkBAC1UrS3jFTTSiIENXfpCkXcaBQN\n+DFstygHragvXXIRjpSAYo1hv/YNJHAvdsPDGfEYvL0G5MTwLtFVBMZVzjQqkriZ\npTYke2epm1cNY2+PrW3yrZhgaM+dZkVAaRVqsAKbP6L6z48PlqSOIKJtceOKYVMT\n1awxrUECgYEA8P3qOt7R6ICu9taxW+pVxScLr4UUtpvERXigUjbTCW2VTXBS18oe\n2MwihcRtRa5DI+4qCNC+mIRIKWKLqTyqiL733fVykEaXJQGnUtbHe1TmwT5TipQ3\ncc9XJ+8di7BsygVGrw6S3Yu20HjSbg8pTF3CqpHfGYd8hUl5Fb5bBCsCgYEA7RLv\nem1T/KlfrsT2jNp7aIH/frhPZu701UTHEPw6xbr/wXrMRQ25rKB0BrjGhA2IDmHO\nC3qXc/6J6uyl+Q7UnioPJ5uPIWQO9zmOqzUeaqpZi4ZDzKbZOcvDXGEh75SAcA2F\nG8rWgUsSG1MJrvRyc6LrMFflzFboNlM1RFKZuGkCgYAPcMVcJkCeu/f52sGcWQRL\n0HegNE+ib5KAvPzQp7CXzwyc3JCCQPH/A/1GQtKZf1PP7oeE7xL3JKqW9DhsjHWE\ns2+gKgTAF7XBajy1Qwue7E6onPyvSVXQFe/IoDptY81kmDfyKWXhif0ZmFQrNw93\nVB6P3S7FLPgedhmq+fpM5wKBgQDQxmYi2Hin/riGfmnpLNaU+DZ57/Hs4e62ibUM\n3jMbwkxMpMmS3j1vKnZDHSGlVQ1GLRd4wGL0AWqPegvHwym2h6+E39cyPp13Vjav\nyz2YDaCB4KGAbkbKQQkwJ1HtXEenZiCckwpIHkY5zeyvKBJKh2S61A1I4BfrD2fP\nXTaGUQKBgQCNb+1O1VNwkxAXYTSo8VDoRAcN+RagRGRPIWk38zAirN7+RZ6LzWQe\n8co0SD2Ex8eI5U7bHGObFIzNKgWNkZFX1CkyQMV96GWMfwdaIygOpXG9kbyaJBLo\n/oZMeSRMIN+OSMOsE7G+93y23sU5s/fI1sXn+U89n1frpnyVO2d9Zw==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-02-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBBjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3bLi/\nF1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1ZojJbB\nbt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUhGlWP\nrNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB8rIC\nSUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmCUUmm\nBc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAyLmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAjbMrXjneMWFNr4IHuVsjso86Ay83\n6E8ltbjOeqsyWI6uyWzaJvBlec2Gyr4rib+wgFOAGFDAGPB/hvyveas6wg4JUbQ/\nP3e8a+0Ls6eTYz6ijrwUGFxVsEu2VQTF7oy7SEbfabsC2g9sQSiaWJ0Js1txIhaH\nbqR2bzRDorgQ3HoLjiKbVGNvifg6qjLE2M8AhJ7FhnOpUpewsanBn4p0BCjfG8Ne\nv2EaxdJxK+kHiWX4D7ybOeIKRzX9E0HMmCh1KyEkV+qGsOrYq3ErWPcoPODOT/f3\nCLrfKmdHHGdxCc+NTxxSyc/CMZceMnV8yplX2ixPY7o1gFQyT3CvJ40Vnw==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-02-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAvpS9BNTijovvNY6lTOYqAdsYKjraohGUTHgtt+6NL5S9bnm3\nbLi/F1PWpqaUrR6pT6YtHBIO6acjYc0XgyvW2ECStpmpiDW1GS11A4vtnJwdk1Zo\njJbBbt2didXHM4I82hwf1Zn75syqIffNGz2re8X+H4MRPeLmoS0ubZBYk2+WlEUh\nGlWPrNRIWi9OxKPNXX7WBBzaF4QFnbVvkAlNfyOgH8QryRxVTUeLvO/QbOutPVyB\n8rICSUkq/PAqNrMbhpV4Wro3zhl3JIfnXKjLTITEpGcf2z7VgdZxqtZOARtY6lmC\nUUmmBc9voyT7QjSXalynFFjU70UguNc7soEAZQIDAQABAoIBAC1hE0T0P+H7VgEl\nrxyJbZU1iyJ7ExYmI2616wTx63JZ82U73D4qG249i03xOlOiimQpyHH3ps9h+d3O\nzPtx0914Orb6DdEeILoXBdbLWc/BqJUtrQVU3Eg+wsVQPvd32m+A5N+io8WIFDa0\nX4VOAOge8+yi89cNkSbtsDFsfnrUXQf/CzeNKutEHWqC3GmKtfxMyEVT+KbpV82b\nghoe+MrhILyMoMppTz1QOoo2ESBHzLz3WxhsJFMqhoECFmatNsq7etQ2fw3h5B6G\noOStbFB7AqYALuwYNdL4UkLA6k0hifM8ew/iNNSl9E9s+GYfVJfAE7lneI9PDwJ9\n9OLGNWECgYEA4qNCay5zV1MuJFdEFbZV8v7Nd+B7tIHE/kA14+rxNkhhMee8zLsc\nVe79UWWtVLhDsZSTkdUw/4r+DF7/TEl1q423aKD5Ar2jyPpW8rWOelB9GZ9R6Eui\nRutJclSWSBO40epOpIRIQjgFzN+gDyfvM1rQo3ZJ7dZy6bPaEHudlH0CgYEA10Wc\nJKEFd3xk62uYNv+nBZ0AucrGqlYLsaqUiEFRVEJkVng/le3DNNJt/kIvi9vhunQK\nksZCHjupsyfMfYS2sgkzzRfyZsgZH5DTNexX1dE9ijngfh1EO9rPQN+nzN60fEIy\nXWBiLx0G/b1YqhqSHyiNtmQlNs/9PXDzZtF0aAkCgYBPngjkPFI6uDb6f0mk0wRI\ntCicV1k95WS96PLFmbCZYD382SsjRQxESAvnv29v1re5N7fLwHhRHZP35puLQjc7\nSDLJZ9tykgpqvT80ToG6CHKaQLT2hTOB9IA90OdmdL81xzlPZEU6NhIbkefZyy6l\n/N5UvmZkTjTaUttldPe7xQKBgEiL9vEJKtR3oXMNEYEHavwjSwlR4t2oncxEFkZM\n7OGedj4FzDf0pqJ4gAT9vRQ/B7VUQfPwyHtz097CfNGYFhGttD1b4p7stDrFDcjQ\nW1F9cGXS12ro5gPd25abSOtr6hsuG9cIEk6aU67TTrwUtEW33vomibwNH+TC2eQ+\nliIxAoGAd+hpl56dAFSEFgaHn/cq+nKgrT/wkJjjIukAL7X9GCM/+qsXQbKjCh+B\npSi8G8ClSOoLLW0Of/j4uMrD3yNW/3Mww6Rkrl8Pemlno4+mt7ZJW0jGImM1tiFk\nQBwTY2HA9w7HhaOn2eDJ/pG1FSdbCeDBTuNMrAtVKsHbJFvWTdw=\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-03-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBBzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU2NTlaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatSMQdn\nRvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbNXDzY\n9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqnuk8Q\nR/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbLLVYy\ndXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBIoYTK\nFpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTAzLmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAM0XD1w1gEzD/1AubpOF5r0zvRGl3\nCr9ucLLAXA7/7kq0KsL3FKDU4ybVRZauZVEQYVKXAJGu8mGHM0VIRXtvEjriGw8o\n8acKXoMWfKVriAzaKPzHDgPfc1Cq6ejsLrsFMge4BqSua7OVmMNedshEU+Z7bvRD\nw+ikh+S0DxWcZxFVnKQqn9WSGvlCF+n0RG0yVjOHt2tLygfzcUVAifAuVs3ktyeH\nenX54T/drMIUaUBRjwlMOBXwehRBfOnPpx+RZ/W3IIpf4Pi2XfTEAHVXiDSPz3vM\nl+Kqp4ntmJII1XJfoC2+NS7sR2OGDJoIFlrpmetFWhUkSMojPxoeKieR9g==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-03-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAydHQHjnTS8Anevj4KyqNrionnZgRcwh275OIwJ5o9v8qxatS\nMQdnRvQBehShr1R2QGtPPVvXxlIYoKr3H+EGXZEpcyYPuaZ1dKhSZPQpLo+gIQbN\nXDzY9754HhCxBLHwx0wtb+flKbb+pgsWQI9t954LWnrNqZUtzPfKh9qg3ODG5sqn\nuk8QR/E7reifsWs2x/iiza8HavTJXZlvED8r7fUEVoxA4UCSHvWogR2VZxCPprbL\nLVYydXthGepvWNSGGZKOMrnJspe0d1zJXUTrzGFc2G6wm+LOJDl02XsfUCP1jHBI\noYTKFpWsPhMZJp5v6Ucw8UPKVvdD3FMoN4XYnwIDAQABAoIBADTGqzfcQ0cB3tEn\nni4bmKU83NM3WwjL495OpGpKgoRkwdijLoEnwHgrgt9b2dQxsCK0bSpMhCaWfV42\nlx09CR8awM4d2+refsSc578CompMoMFCOB2ww8Q1iM/D6aaiqaZUY2VqLOE1u23M\nZGtJlqY0LB/jETkRi8KX4dyY7YwUvHE2aDIkz0BKxWudmSbVNT5C0Ykujrm8EJpr\nKtRPUoBtw1+gGzEjuIRLxKJbWb5NS8+WA9ePQsS/10u9/qxH6VZ2SHJ2BzBaTHJr\n7z0VLtu7VqwXz+N8/N/K6roSlJWihRufPwNcaPagBzWVCURyodyWcSr65VnJB7t4\n9tcnnJECgYEA9a8sIAsmNMb+Kb3AESpxmdFxgX/GJrP8EPis+PlFqTx6EM6J9dbd\ne6r9hXAB9tvVouEEi56hRwfv3f0OEhlwdYMK5ar3T2mKrzFoOAT53YwULPj4moH0\nM6hta8B+YBzkWeOC/S6NRzGEkB1ORFI25nZeUtqJ3WvAxDXumxEerDMCgYEA0ksl\nGLCbxCefXT9fPySAUafdnyNtBFJPeP3ykSoXrBqbXmdc+1hRUS/weZz7xVtazMrt\nUeg52+wtMhj81RKZbW8wpfrU4tAQcb02DyJFzGwvZT0F4ChCTuNJn3HBtCxNCm9K\n4k83BV+4JOrH5VQXncaOMz3U9R/ROqkpJ6mT9eUCgYB/D7d8YwfBZ+Du6Ym51v+l\nk6JmXRS304HbrSCYKyMR4YsnwgmrsRIe8VHofwMlDpCwijt1kfbK65nsbwGl7q4w\nuDMckI0S7vygmqbRwhEPuXB9yc2Y6vjG9qaQgZ9aSdb3fiiylC7Q0RVEC0P9JZ/r\nFPC3XPrMHvw0/ceHPxVj7QKBgBfGGQaGiMWPOSwAixMHXF5e3OYtyhhP+d4Sz4Zr\nvMtfIqt0ggWEQYUtJ0GIZsoz/rriQhwdZdUgCSf9vS+Vb7T4egZ5qfGOVh2Vp3Pk\niwGGRYFreauSwZRLi7oz0RM+YuNIG64kCHNaE+ZQiJK7hAP5O4A9gELJ+wxnrVhn\nimulAoGAOtDcYdA2gb7rKHTWwVhMEmFtztPGDOzQOMsKJjilRpab88syGtH/Eycd\n1nD11MsLGQjrxaZo4eFWEDnomtMVsCixXzr2mmciDw3ZdRdlBrEUfeUjTSBtxSWA\nDSx94pwGGVaT+C+cDaFAOms9k3B96AwjomWnbH94w5aufOVy+ak=\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-04-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBCDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauKVv2x\nm+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2Dm/C3\ndAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE6XQw\nG7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZs6Cp\nLmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfvBK1I\ngbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA0Lmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAGWF7FwDv6NE0oEBsjmOoLXNw9qyJ\nTEiTfrWgHfJ2XFDIbpj8dmquX3TmSneWTFRS8LUwR73zl/JJ3Op6Q5ISBpwSXCh4\n8V9XWOB02fhJN1NhXOqxPgNH4EM2m+x/C1wdpZuOrmZjuH3uAW4lxi3lXS+H2AQu\nixD5dXgzrR8l7LIo6RAibqUs2aNG393ck19BF5ghAL+iZtZs0klTrsAyQnzHIGMP\nZa2DnK9yjir0M7n4AXspTvtvFj8Zr3WzCAwKW4w025iGxrbJ7Nu4ED9p65s/KZr0\nu8ES5UVaVM1v+7Y+fLzt0v7EX5bSx4okWbNbcMJ3e14JqUGoGMhB8o7IPw==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-04-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAsSTz1GlWezoGY6Soz2hGAP4gvsWnWdMl8xoYdmUtorcEsauK\nVv2xm+9XRX/OxjR9Kq9IyFooMBHJE/Eavft2LkT0BQ33g+NAp/NUGew5hQ/jlN2D\nm/C3dAbHBeHAvwR9z2XUE6lf58BVsK83xPJau1PuVWC3yxiRC2B08tWm1Ign2soE\n6XQwG7tEjPHbgldtgD/dxcXuuovU5UsFB4gB0rRu06M2j2J5sfWvLarWV8sJ2PXZ\ns6CpLmkAVf5EtRnfugqbCJ0Xsb5rgfMJ4iWWyYrIVEWbQ6JmNFDuBEWakh3fKSfv\nBK1IgbWYeLik5hQUt4WZVZ8q9jjtyewgBgZDvQIDAQABAoIBABJ6PETHyBGKd+gk\n4MbhpYus9lVv4IjqEWdOLYcL/rqeRIsTzZZNWB1f1caWEkdyaivtpLXhUZfSAxtl\nZtiyRh2Fa76rOkozhib+pqMdlNJgWejJQlUdwsUDf/cJGUXTfhwIDxsN68cMtON9\nI2ATt4sash2NvR4eLeL16Lz7tC8u6ANGleI2PjG+17qtTSnSoy5wO10M2aK+SrBn\nV0C0pT1BqapSTM6NkZ9Q/Z8gPH38Q5Eqc1iabYh0tHop6DLgudh+Au+z9q9NITgL\nfkEYxBIcBZ+1b/6HT5XPd1j5sh+VbG7y/xtoIU6YCguWkcxem/JFJWSgSVTyfK57\nBg284rECgYEA59RnZbyEC2VALPrHAnW7+Ik6LM6fhRED2MlIy7swzrjoT5s5u9OS\nli83xgVsSRhwWCf3UoDTyG7ArIF87hwaPXbH8PfZEsV2K8P0ZyaMIz2vQcotT9+A\nQmbXbKiXSg6MoSSJic+MrnCspF4c2YpCRnc3h285wUwH9ZvOVfkc6GsCgYEAw5z7\nXpV9iO0eA5dfIeHsTwbFz6AcbPkTPZ5nf4zn+o6HaAV7oyV6f7bF0JYf26guvvg4\noSYAU9VLRQrw/zzXAyJ16mnJBdlqO6dZU1G7HI6zPZlovuJvze2Q+tNUBZGD6Ouf\n0PhWzQtxL+LcmpCul4sDCP7mUCdG7aiClNg5LncCgYBRL8JSD5XSg8/YsK4W/3In\nlK8p1+ZnbFEDj2IN7u0lx/2bO0oZq/tO2xRWJboJUySsyrpDS1hffeG9x88fd/n8\ngmd8zN9ZQouiwgoKQaacBNMmYA5ERYoeNvPEWro4tiWrnScJewTSZiUfntHNoSya\njs+Ef4AjYGP9MGYvF4F3JwKBgQCEl7kPwNbp5IhuuMFkTyaF3rpg5U+/UgoHv4K6\nQ8HO5aPFD8phqPri7PojTE9l7hdZnRmNqhuYt8CgmS0IZa380vQIpBH95ASNUP9M\nad8iGVxHyd/lW8mbVYfrbSnL6Hn4fRbEaEE2FaZwZh9QqfeegzzYcSeedzEt0QZK\nbFI+OwKBgQCDqh0JwrL1WLXlor4N45EImHbeMbFzIZ4ARQuGptQ/m66MKN0cdMG2\nqCysRlj7eac6rEJo4gsQ/ASR0NLPZAxDnHk63PP15ObgALDUppa7fQgTpUgNKgD6\nLqWoCGvCm5etr5WUu621mlZoZpi4IWPU5zJfSoxNwGOJKy0ExNws5A==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-05-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBCTANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67ZOxt\npZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5UOZEA\nMvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Ynoi+V\nu1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD0DF0\nmmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGAKlld\n+ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA1Lmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAkmk/YJoaCGip+h+kUjmUQHT929TP\nIx3Jd/nA1LT1QQ7b8rP+jImw51CSLbWwNGihVRKkOOR/kkKZLgWdsa77jySKvqnf\n2QX4YcpaSzXitctiVoJAXhyJwEo4y6sRJS27IwFOh+X3gHzPdz7y5KYRfj6OHutC\nArqD6Ohy5dRg1Ixwf2+2go7I1pZIEfzhFHzFUylFmh9ko5EyTQ7cQw8I/D3UI9TB\n3TCPUrlSNGOHuREsGMtF7An5Idxytx6Rsx45sJq89tB1eipS//Pk/obRGCZYIb2v\nZmsOSYeppa+97tr2j3ox1cXgXlIvt7ANroZzVHrPhiOMCztrlkFhlqUibA==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-05-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEApT8gUAdz5Zgt6Zn+MEqRVn+6R/K9RNKZn7w+KXix7Khfi/67\nZOxtpZnWtUdjgGqPj/Zq9r93OrSb4ZXkquFox1XYdVfyWnq1seIOJfWea2s9iK5U\nOZEAMvFHbQOVhV3b2OAtnJEEpuvtbwJsvIRZGYZKxwE2E4rXXeXRzWOWM91KV0Yn\noi+Vu1klX/UVB4NPH/ktsm5lzay1otybAEUwy5W2NOY2NhayigV3NHD8vsDnRolD\n0DF0mmlaLfinrzyR8/mPrWkCf58sACwIuc1+f8eaQ+j6MxRZHJzX/lzAXlMMDIGA\nKlld+ThiCbbOIvTQDdmQnwI8+iRzQLS8SgnTXwIDAQABAoIBAH6YfgjRrNiYXZQ5\nhaM5fV7nO68jKB0xur/FV1ouPSExXQHDhY0sFFRqJzN6HaCkApgoEIo45P70nveT\n/jtrtPoBqnM7jVAD8m704CH0qglhfyLXa28uxxhAAJhjxokF/weHt2nbL9UxqhUB\npoxGWHxVhbQwGV2fYJ9vEexn/wp7ok1/gU7E/TVuKAal/PGMjOgkWijDUWHFRzZi\nEbaEfXBhLROOX7P3+rCjd7pfPw8oc6ktGYBpQXJmSS6jiDGypPm5XGjvb5lU0wTM\nhVEUlKIiZgwtdUieM3vwENoX8z5s3PXkJouGGWhUN63o3HTMD7WppGmYitSTO93b\n1u8uRgECgYEA204MDJkLZt10r95+d5N9tnD2gjENJHQzoaziKOOBoTDaTP2+69CI\nNpAYOKXcbxY7oOp1a0FbQEcmbF9QsMMFFPzm/+9kn8ekrPIxPmO6o1DB8PTcWOBD\nbr7x3OgyNtClRa+91D++/CkxBLgZz11uo/VW4UDl5KApuMEzGs4FpKcCgYEAwOV8\nRgmZRZMbS03VOvJsZVhybHVRnyQqtvsm8o7dE1URvHwxon3o9sryGLbtWLh43DcJ\nOg87B1CchuiydB9QAof32nVZmAcW0snxmg1/fhDiDO0Nmggm69097INwdypQJNGV\nBo3c7T3uExHQgFVjjyA2hNj8lnLz9H8PJhQOWokCgYB2em3MCIq+Yfn+YYeVdls+\nSz0CDRn6jcSvHcV5LaAEw23MlnGk2J4eTC6pvAGhCjPgtYoGDeMMkOPTPJNNS67d\nLdxmfKCyKZtvy9CK0josrR553O+GHHKRzBrCq7clIgeH5G/70QyPEnnnAMcA3os9\njPgI7ZTFtTmVFNtVKeSmlQKBgGJr9TfLKABe6dtHcFfuX65qLZ/3UFkx2/WXOi0j\nBdwaWL1iZPTy544cyOAhXgMZxNkf375XDRhTYjpXus4TOADCnY1CuR8f5t5Qmcyv\nvHB3bs+HmJwSMsHAFht6iUTUOyY+JZq9gY8vPS1PtqH0b6MUnZy1FlQxfRYwSmt0\nakp5AoGAJMvbFeQtX+1v8ronYwSZ1RK5RrvhUp9myP+4naeG9RvvlmmhRf7y+ohe\nIRXrvANTbbOTabygbLvuIVCmBus6fbtkH6ez4cLn+CnUpHCbpzrrXKLT2T9r9tl+\nIIgVtavl+Ags+X5XvmP5eGViFZ4oqSECM5Se1oqBX52ljZ/FQmU=\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-06-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBCjANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJnWZVQ\n/Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K5Yez\nG3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lYirw+\nhhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/LgL2Pa\nz2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicEtGLd\nw5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA2Lmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAYGlcBAP3Hdc7sruYdxXk8FYFIBrh\nvqLLElpR2qQZHfnXBTDjRf4ZyHbrEPsv7JWDlQMXI3U5SBReiqvv4LYgQ3W6PvZ/\nW3nKUlClu5L64E7ZGGQvGh2lyNWEdG21dlpves3Sb5UNtZdAuOBBJihHcJsUJGSE\nP2+lKy4O6lTTba0H9wt0uOiqwUVkwHMF6rqirncRLihXaLcAOFb+NSuFDOMMEX7D\nbAZBFDkk38Y7fBlWQ3sR2EvWpXp1uzg0ug8zl+JbFnlDswGSom1Y/NXBzIDYx87c\nBlPrhizNUKrCYT1OFz+a6VIYTNM01vyNZ2QEMYgcL4Ug3wYeb88HxEEeMA==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-06-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAoVGwsG/awAsfsLb0Sy/GPapkjGnAbumZh9c/9tEKa5ZmbKJn\nWZVQ/Qju30IlEiGtPrvBsdpQKowExT6aSK64DytS/aJndT9K9lvzv+Gs4m8eSx1K\n5YezG3QoIxp8TtZOOPMQtSVesUjDOcgmrYXtg4qHayKsKTFzs30t0rzGfjX4V8lY\nirw+hhVQAPRAyTtNOG+Xh66t+nUMtOYbM5QWIn5WDPPqzMU1H+DZd7HrOX008/Lg\nL2Paz2FNSGhxz6y8re1hF+G+CC68kMayftYRy3B2NX2E6mLI+Rq0mW7iQR/FNicE\ntGLdw5LYzIJ460FUc1iRXyO9We1+IgLaLKbeuwIDAQABAoIBADCLDXk6X4AUmNer\nZj3NxYC+6nzaQb7eaoXW/vd9lf4kBvQ/tovdg75w1ri2BcO8AI71HucvA5qaAcF9\nAh5G/ToLVnDeUvWUMzGJIAMonMaSm3INAYGqYPos7PsLOEbc9DLs6aUo9qKwe5bC\npOjoeT1xDSjvCRFoQSHM1GIqziwiBSYl+30wB6PtE/ww2HtLm1i7z3u8AOB4MHYY\nhpDm8r0VMys290goZGSUgz4OD7lXV3Is3F4DO1KJAcP06XDhJhUhIn54xhGJsj85\n24p7hYx1D1gFnbsCpWN2MPj1ZcXgYHR0brW2kEvf4stq5uYqDuQyT1VtvP/31jtW\nkmTZBIkCgYEAzY9fsDT8c6DU+Pk2kAiDyKWfEFwSVy0FY1Zs1H0DzL0go7szQGAY\nB3bzuuHhvHV450zwLVBec3H8iooi/2H50nnQg6IAjSjvu1qWjmuK6zS/jMcJ1ONX\nr7mscyPmeYgnEG4k1AcKnbBVQ0mCYw2u1nJH/jxpavp8BXjfAdhAfqcCgYEAyOc9\nuKZk2WwC+sHXC/gUfjiorFql2KVqJUG/K7hL4Yv/no6HPnI8ksHF9TUFtOGMRR36\nbDd+DwFuaix8IZJw0M8XU7koOrh81iDifbfkljaY35qMlqrVdKIa+fUUh8hgzBWj\nDLRj0PoymzguaSoOrnjj+lNhR2hZs0tDayPsVc0CgYEAyJhLFpb+bI/chQIdfrzw\nbVNCQCK5Ox0SDvh9x3YfySIeweiigQXFLTOlcQ+Qy+oPDBiGoJG3Og1YFpHqyTEB\nK6X4Fxx2UUpLd5dVKLJFpHWbH3P0Yi1gmnkkkk/MT3jQNjZl9grRD0TOT5ViiesG\nXPq0yqKFdQHvGR/078XqWi0CgYAjs2P2cistr9H/uX57dARAQdVHe3xJOWvZXwuX\nu1JQrE4qYO2LnUVCVwjUgC2ZbRM+HQupO1s9U2XJnEoatSkzEfn1OMv7U1lru8BD\n0u/d+anE4nPlOkpgRYZBsNmLH4KEAbxNv6iVNEDV8G/e5EhjnLv2eeaph9OY1uEO\nIv4WaQKBgQDElaNA6WQXsZAH1u1LLC6EWAKAJ/SyJzhMzrs+51MXXB7dIFlpM0BO\nDEmXkXSUO5aZQ+fsBgvDJlQhp6yheoLb9K+eCLgELh40sChg9jDFKdVRDShJFh5O\nUZYOrHYUSdy/gSzhaF47c+MZrNwwMfE6fbLSOtQrGYS4w5Hh5i+RHA==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-07-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBCzANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnroEDT/\n/ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaNHlkg\niar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbMyyCh\nMwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1niQB\nh80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam3dHv\nt8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA3Lmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAUcf3kUOmWALEcw/2DPQWzY7nn0mv\nRGlgQP4xl3W0MLXg2IywAqpOsrdR8sx9sNBA7WYtUT0fdR4gs1PWbddQeJqIwjLZ\nkOxjNwKVCp4DWiKStYA6bIdzNnxXvMKD4FHAUMBo7jsEGzvln0IfutNeE8WKloAB\nnezVDTefi+kbB8npk93yag3HXQldAKjUt8VZwebJKu1TvCSLiq4BXXV2DjGIrmTb\n0Zhdbar1HeEhf1IsnUyxfuCS+eXrVmF6XesRiaWux7Y79SGD37bBAH+dMwEuKH7+\n025W+IXGls4RvwfoXjVe2GNT9G5aLzzKwY2SE+A3GeeAqf8WCeCAA10nCg==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-07-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAqAFsSyRNQXcB3wCzvNycy+hkNORi9gjJBM0lu0BnxVgvlnro\nEDT//ZXwWF4Z/D5jbqCqi9iharH1O2wa0/zofAs0DYxLuSEp9efF/CXCURcQviaN\nHlkgiar8KeIB0r31ump0KwRUJgZZuDUgCkgNpn2J6QtjwIe7EB/0wssQE5IemHbM\nyyChMwSoMzwDGtpY2tDRfORPN8WMTnIh6Rfl4naszQ2gvveW0z6Ill8O5OmhOZA1\nniQBh80f77MYfC9let5wV+yqHFA6++ro4yiSY4+VLAVEIxw7KZt5nmFn5h+v6Gam\n3dHvt8MW7yBzA1vDvDO1+fCywV7F3MGpq/bRTwIDAQABAoIBAC13dR9Ay8eTtWXs\ngrfx3F0ynyvRQxbiecRa4IM0guDdZMOMr5DzRvQRx+GiVdX17GLcCEugbM67KmrS\nbtA/+YrYXVXtSfDoELsD5oi+jz/yxg0v0bEM3clkBKoxB91Im1+/v4CEw2EnBtDb\nT7h4l3kfMFtpQa2xDebnPw4YdllVGcKEwG5xqMe/EqMg+QpXbqsjvC4lCTJutdHD\nNpdHiz/GVq/UdrsM2Wsw1nlBokbt92Gn0LMx/R9kiegwRr0b2Mf1hxIQ/7lStAwB\nZIaWPqQn/nzCZrdn3cx3yzqKDmYKeXSME6SNsKEY465wO1iq0iTdPTdDN40bFZrb\n8g31GAECgYEAz2NCn+p+12G9EoeHm1ozdahaF6MZfTp/5cgJwPWeY5TYy3qFFCvE\nXPqaOo3EpdEYWjmNBgBqFrlgkuUpOsZekMEyAm6YmG+EPDK9XFuRNTBRaPaDp8bK\nRMYh3oDToDzzYVig7MJKi1a05rbo/oGogqex0kMEdToAFIcL56PL588CgYEAz2Lz\nsx/hTw0YjaviL+6aSVomttJE4gwrUcMheWUgGQDOK12wSsNqW1UWXdKtLmjqlwbz\ny1fxVsCLIdDFdvp7397+xkC9PNPoF78Nq9Kbj6BiC870dGGm/V6o9Bw1FZpDwT0h\naJNecMCHP7PUeGtw0U7x3eBBXB/BCQHa+/iXXoECgYBjDm85QOE68RVFL3UHMAta\nTIJTvfSjyvhiAa0e/HKd7++pKSk9XDZbjttx0ls2NGxkVA7W6gXCOuM05r5Ns4hh\nrdW8MaUzUjigJEAsSBRQinaoIu75iUr9lIGC7JeodtRtD881pwvCCDU56e3Z+oZJ\ned3Gp3oOoBh/tY1rI+J2IwKBgQCEtMw/NkpF/JpWpcOyalueTqrxWDIt+B2MT3JP\nLS/R8Br90xDpdozLbvJGDXc2eHqNNCyzVU4g9krR07dYQEceZaLgmDLABtXAxEfq\nSHW0/atf5Qm2o3ppLbatppMthK6QrB6BvbO0MO8bC2cNu6rQtVS+Zy9L1SIAMoi/\nrj6mgQKBgQDDx62JtXHSfaiIOEkpfZW1kOvzYdkjwYT92mJgrpso9ZDr/FhJuAj+\nNxPcO/e6Iie3pijTKbLpJQdE6n2srJ9GEdLPdWUC94ecwmKBA/m0GIO94hm4oIka\n+B5U5rkTWxn9xUtOfX4O4MBXpSvgf2a2ZrD34JNftYwzbB7pCkRxGg==\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-08-cert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIERzCCAy+gAwIBAgIBDDANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xFTATBgNVBAMMDGxvY2FsaG9zdCBjYTAe\nFw0yMTA3MDExNTU3MDBaFw0yOTEwMTQxMzU3MjZaMIGHMQswCQYDVQQGEwJVUzEL\nMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1N5\nbmFkaWExEDAOBgNVBAsMB25hdHMuaW8xLzAtBgNVBAMMJmxvY2FsaG9zdCBzZXJ2\nZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5Uz/kP\nF5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4NjancD\nRmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Qnc9m\numtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2gwrUV\nW6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU7zcJ\nu0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABo4HWMIHTMAkGA1UdEwQCMAAw\nCwYDVR0PBAQDAgXgMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDov\nLzEyNy4wLjAuMTo4ODg4MBEGCCsGAQUFBwEYBAUwAwIBBTAdBgNVHSUEFjAUBggr\nBgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4cEfwAAAYcQAAAAAAAAAAAAAAAA\nAAAAAYIJbG9jYWxob3N0giZzZXJ2ZXItc3RhdHVzLXJlcXVlc3QtdXJsLTA4Lmxv\nY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAgIB/wKMd38ZiSGP+Pj3t/h7H3uDV\nonmoqGw5+By0B555k3g8AHaGzRw2DTWz+1ncxkDZlZvt/GNDX+gg3B/MxA/HKlQp\nnf6Ctbo4MFkHtw2DlLPZSo4KHn5TZnBw26R6SApCIJspGBTjCd71l2i50LsaLC9S\nfX5OWl2Abgz5xjX7iGBGbR/4S5tWOc/JmSQJtawLGF1w6JDyCHIjWlmz0EjzKu5d\nw0+8BGzR8s116TA9PNn/BMf7lxD3bEM2Vh0wchikliFNO5Bdtzz8CCakvBrHhUK4\nPu28NEST4868VmxhBpn0Im3/3QFfaKuQN2AeAZAojdLorqXwzDZ8wc4XHA==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp/server-status-request-url-08-key.pem", "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAuDPg2JZudC65wAhHA8hQ/PydGDk8V7ocO1RDKFTp7Lw0Jb5U\nz/kPF5PnZ763kKUA8AIp88AKTxFw4+GTiQJRV5Mg4mvprHzHHkqk1fTSfvWQU4Nj\nancDRmevCNf9k5H272saSUTNi6wE7vay8nKNEgsWlsJUVGrQHreSXc8VKgNtSl3Q\nnc9mumtPHPfS98DvtAWZzCRfTMpeJoAXbntV+TmsSKu4eJPR57u9ZDbMcNnYSB2g\nwrUVW6FkRhhrgfCPhBkRw4/WaZcBMoDdg/fjf4LmGeqdrJIwzcoaeAZQhW2+HzsU\n7zcJu0oKFLtTWlySevVPZnOwi06z4uBGv5f/bwIDAQABAoIBAEW39QFlJpc43DtP\nprlKx0SJxFfXyfwWR/s4sFaClStyLHagrG8vAERXSb2dlNLdcRZma0SDyiMVIdpO\nlc3En876s/afC4mqjJ7td6g3irhc72x8jVNNimLKeZhqT+Lb20/RBNj4fqe/yvo7\nS42yR6ytn7YeBVcSOoV1y6NP7t3AW2L84snfXKBqddOmwoaodV13IDBAEsmwjhVy\nXkwRhiuPGCA+A/9x8YWqC+/BJMXQ0zEaAtE9tPnebWbbY8aHhCEhEpo36uG5Wypk\nlDS6Bg/w34IBbuUho0NFFVLXkM0/yGVS8UQW8eqEEWLJq6H2t8xwDWNNbXhUgl95\nB85B7bECgYEA60ZLMjYC3BWl+EDlxxxv7DT3FkhFAblWH5IRfCc4ilMJaQxPdwYj\nn2Zx/jjrpbk86AiuI4YdE1jM4FIxrkAAqiD91/uTSxX9U81jujuR7GDvBxY73SLY\nJvinO+6jpyyDrik03EZh2vZYr5VlwK0qTt588GeA5JGatVxfF6ID3JcCgYEAyG3a\n4+/qHW804XkfLrcM5esmBwPateleY1IeQPgv5W3ODlb5uS5C+8eLToM5MCkxvUaN\n92siQk/o5+yrNuVUB3Ao4et2iBGpASC2UZ5i5v0mctYOafFz2t1rhPyISI2ivbSe\nq272qY7P2qtPMMeyHe3xF33nHcc3JnSiu62C1ukCgYEAttYPxdt3aXVhX4V/i7ar\nu9KFWkmbZvWS2kH3WJZaOBTDsWEhuAuLT2qbl8bASi+kB2YHfg/RNKHDxWfat+GB\nIrU0z83v72ANWDy7DZURl7LUzpsWtolHlTGTNN4FS/sp8gSP0cbYcQMUdI8TXiK0\nSEpaqbCl1/rXUa2RMJp1ic0CgYBJX64ac+IJFIUPZf/8YhbetM8fElIm9mAPjCh3\nMGSYYTJmHYEeQclT0yE0hOWStAH5gLLIOPg5vndNMF8doaa5cl4FFuY1ugFc4FTe\nXFVoqpRAxgxQzIvVO+n4rVpW9UL2oADWUbELMbT2IHDFMtYKDumL6BOL2zpdYaWR\nf1u+wQKBgDk9d7b/D53ThOnW+0vYMjo4YMxwg1dcdOBB6p1TrpEBjoqZSPOaY4Ht\nOC/cYzvxh5H7YOsfCYzTEea+y3r5lpmpQi6EfvNpbCVsdKVCHFLivFb9/6mirdJq\nrUMrTPqCM+Q6aI2/GAwFc//3ItktChuwOxJtH0w/AirvFYR/NHaP\n-----END RSA PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 17:37:00:a1:ce:35:e0:84:dd:e9:30:0c:a7:12:b9:50:88:9c:16:07\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:02:58 2023 GMT\n Not After : Apr 28 19:02:58 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=CA OCSP Responder\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:d6:10:15:61:34:1e:97:0d:c6:c2:7d:f2:0f:9a:\n 35:56:54:7a:9b:9e:a3:0b:ff:31:0d:db:49:4b:98:\n e0:64:3a:3c:7f:4f:4b:d0:a8:01:80:c9:68:4e:76:\n 3b:be:7b:d9:56:8d:d4:fd:bf:e1:6f:d0:5c:88:07:\n 3f:05:a8:83:b3:7e:0b:ba:e0:36:f6:1c:e0:75:fd:\n be:38:26:33:1b:42:96:4e:62:0b:88:36:ef:cc:14:\n e3:97:86:dd:c2:78:d3:05:b7:4d:cd:2b:52:f2:11:\n 16:d2:7e:8f:f3:47:8c:f9:0f:1e:cd:5e:f7:a4:1c:\n 62:34:03:70:74:89:6b:bc:75:e3:30:82:c1:5b:67:\n f3:d1:ca:81:13:10:d8:c5:d8:20:05:6d:d1:e7:51:\n 19:ac:03:96:2a:a1:21:ff:88:2e:d2:e9:67:79:cf:\n ef:17:b5:2b:7c:10:1f:5e:79:3e:08:98:7f:42:bb:\n 8a:13:17:2d:9a:1a:8d:ff:36:c2:e9:c0:07:ea:cb:\n 4f:72:35:f7:f2:d9:86:d2:ab:6b:70:2b:57:82:c8:\n 02:93:aa:04:aa:00:3a:53:23:3d:61:82:32:0e:68:\n 33:7e:5f:03:52:c9:53:db:e3:26:46:8a:ab:e0:e5:\n 54:57:0d:e3:e3:24:b8:d9:69:92:0a:fb:bd:51:25:\n 89:fd\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n D2:00:A2:C3:AA:00:76:1C:E7:67:37:96:89:77:38:69:C5:1B:5E:45\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:FALSE\n X509v3 Key Usage: critical\n Digital Signature\n X509v3 Extended Key Usage: critical\n OCSP Signing\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b0:36:29:84:91:de:14:e5:db:bf:55:fc:d8:0a:81:b5:df:84:\n e4:5c:ae:e2:3c:1d:05:09:8a:85:7a:9e:f4:82:61:1b:7b:8a:\n 0f:1d:e3:ad:b0:60:45:12:2e:38:6d:9c:95:d2:42:fe:2e:1a:\n d2:a5:2c:82:40:1e:6c:4b:35:d1:3c:a6:4c:1c:73:c9:d0:32:\n e9:47:c9:9a:fa:d0:1a:ef:86:c7:1e:49:ca:62:f1:81:9d:4e:\n 38:35:56:1b:53:fe:4a:f4:4c:91:31:8f:32:70:64:ee:91:f7:\n 4e:fe:ab:c5:1e:84:d1:43:cd:af:f6:5d:2a:b1:4f:b1:f4:1f:\n 5a:9d:33:7a:48:94:c8:88:23:e5:b9:c8:a1:4d:51:4c:d5:3b:\n 5f:f7:e8:e5:e1:53:a6:de:c8:95:14:32:e0:52:db:43:d6:c9:\n 2f:7f:96:07:fb:87:0a:f0:53:3d:ce:e1:56:6f:dc:0e:84:f3:\n e2:ef:dc:17:0f:59:1f:1a:70:d5:7f:08:36:3d:7e:8e:f8:1f:\n 55:47:9a:96:1b:11:25:d9:27:7f:bf:e1:65:e5:16:ca:d9:bc:\n 6f:5c:5e:a6:4c:d0:7a:24:8d:42:c4:dc:b5:4a:75:4a:7c:88:\n da:21:5e:27:e1:0c:36:64:69:10:58:81:3d:cd:74:df:50:85:\n c2:71:fe:43\n-----BEGIN CERTIFICATE-----\nMIIEGzCCAwOgAwIBAgIUFzcAoc414ITd6TAMpxK5UIicFgcwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDI1OFoXDTMzMDQyODE5MDI1OFowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFD\nQSBPQ1NQIFJlc3BvbmRlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANYQFWE0HpcNxsJ98g+aNVZUepueowv/MQ3bSUuY4GQ6PH9PS9CoAYDJaE52O757\n2VaN1P2/4W/QXIgHPwWog7N+C7rgNvYc4HX9vjgmMxtClk5iC4g278wU45eG3cJ4\n0wW3Tc0rUvIRFtJ+j/NHjPkPHs1e96QcYjQDcHSJa7x14zCCwVtn89HKgRMQ2MXY\nIAVt0edRGawDliqhIf+ILtLpZ3nP7xe1K3wQH155PgiYf0K7ihMXLZoajf82wunA\nB+rLT3I19/LZhtKra3ArV4LIApOqBKoAOlMjPWGCMg5oM35fA1LJU9vjJkaKq+Dl\nVFcN4+MkuNlpkgr7vVElif0CAwEAAaOB4jCB3zAdBgNVHQ4EFgQU0gCiw6oAdhzn\nZzeWiXc4acUbXkUwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwDAYD\nVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUH\nAwkwMwYDVR0fBCwwKjAooCagJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3Rf\nY3JsLmRlcjAyBggrBgEFBQcBAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcu\nMC4wLjE6ODg4OC8wDQYJKoZIhvcNAQELBQADggEBALA2KYSR3hTl279V/NgKgbXf\nhORcruI8HQUJioV6nvSCYRt7ig8d462wYEUSLjhtnJXSQv4uGtKlLIJAHmxLNdE8\npkwcc8nQMulHyZr60BrvhsceScpi8YGdTjg1VhtT/kr0TJExjzJwZO6R907+q8Ue\nhNFDza/2XSqxT7H0H1qdM3pIlMiII+W5yKFNUUzVO1/36OXhU6beyJUUMuBS20PW\nyS9/lgf7hwrwUz3O4VZv3A6E8+Lv3BcPWR8acNV/CDY9fo74H1VHmpYbESXZJ3+/\n4WXlFsrZvG9cXqZM0HokjULE3LVKdUp8iNohXifhDDZkaRBYgT3NdN9QhcJx/kM=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWEBVhNB6XDcbC\nffIPmjVWVHqbnqML/zEN20lLmOBkOjx/T0vQqAGAyWhOdju+e9lWjdT9v+Fv0FyI\nBz8FqIOzfgu64Db2HOB1/b44JjMbQpZOYguINu/MFOOXht3CeNMFt03NK1LyERbS\nfo/zR4z5Dx7NXvekHGI0A3B0iWu8deMwgsFbZ/PRyoETENjF2CAFbdHnURmsA5Yq\noSH/iC7S6Wd5z+8XtSt8EB9eeT4ImH9Cu4oTFy2aGo3/NsLpwAfqy09yNffy2YbS\nq2twK1eCyAKTqgSqADpTIz1hgjIOaDN+XwNSyVPb4yZGiqvg5VRXDePjJLjZaZIK\n+71RJYn9AgMBAAECggEACnoECdtaqervMOKoH7Jc7Oo6i/ZCJZqqRHLYjf4f8VfW\nUSbI35/xXuO8mqZ3uxVlqDJN29NxzZ6lgLTWFUlPlM/U9CL4HaiBJdUy452e/7UN\nFS4AQXzq1JKrJuXfYZ63OT7k7Gcz6owCkW/HTNFSKXhfeg6tURdgiQooDVQSdUk6\nxX4gVEK3skozRXf4mrjTaNnFCOk2+sZdqrRn19ZAUGRisv6ECf8/wQlh3+ySfPYV\nu+BHQqzntToYP0HUZAO6rezcTVayW25E+AaOqNdNmSqcOX218ohVCzwzFpzIk8LW\njYLyGQBhHHcw+RHJeitcHrDuTTpOZFznQxzHiGH3AwKBgQD91TNHx9Y9jUBkISSi\nXylSiZEAOjPl4VrhRfI5OUx1l3XTqB/e3xBYLwxEpXjs7m3qyXhCe+rVuIwSjLzc\nmLCspPZw/fxdRefWW5B+v1HbHxC3/lBOhqaDfLL6x4A/q3n/itG9X0GjpfvRkdJY\nGYOJea/2rJuMsFs3atX160p4cwKBgQDX4/VXJWxxWUJbObwoxxABG9VTZdI6Dsqr\n8tgg+7NPqw3PAo5W+XLsGZCSWQfJTD49AHcHBon5IfEDa5srfKsOXFXoiNEdCjIG\nzJ9mNtGMokXOWLKgxMoqHz+WnqWgxi9D7QwWWNq5hWnACJUqeqelRMzoNkmr96DX\nNloqHREHzwKBgQC0jKnlLOfe8FIU5t5AAKBL7T4Og1fW8+zIwBADVBZmrk1JOBUz\nWkct8okvauQQ46ebkaLQ54OqcZJwv1q3LoS8yLnitUaEseyuNIMbJMr8qaQiu+oz\ncOOQM2q7ppw6raYhdoSpxs/Rr4bnEmoj8EH3z26ybyRVdjvrtzppqetWsQKBgQCa\nYogGA9siy6PqPMVTm9bUFCVfeEb4Aa/pesYYACbgaAB98uP7SnNmZ3m9TjGFQCKZ\n2QVFXuW35Q/HVGIonQRuRpWgroZr7+iKeDXdEIKVwU2OHFvRICk6KhJ9EYJ8EH2o\nY5HrQStY1BElpH2XXRMZ2rN1s6zHb1Pz0whzaUnOfQKBgQCpfJYh1Yzpryb0hkfa\nMAL2Rsw+mpYeJ27Bmv+taW5iEVMQr2AEYNJhQx1SjNZOml2mqY6un4UPwhUwqAqg\nSOgWNQGD5g6xoM6Hom+nZG03QacCYUOaD6xDmVKTY0LnzVBwspfvrIgLKgZ7IWBx\nKlqvY5FJ+NXg3wHLNwgGzkgVPg==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/System_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0:\n 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7:\n 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f:\n 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3:\n 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76:\n 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66:\n 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd:\n 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98:\n a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08:\n c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77:\n 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a:\n 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d:\n e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93:\n 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46:\n 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c:\n eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba:\n de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63:\n be:59\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:System@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c:\n 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64:\n e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71:\n f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55:\n a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca:\n 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c:\n 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3:\n 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59:\n ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02:\n 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5:\n a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8:\n 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30:\n a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad:\n fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca:\n 95:a0:88:62\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\noyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS\nfhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ\n6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F\nEVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw\napMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re\nVMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h\nu+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz\n5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j\ngrlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth\nWf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS\nU4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl\nnYT/mVDKFU/UMMiFypWgiGI=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/System_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7c:43:65:c7:cf:27:e3:83:ae:2f:60:ac:03:e5:f2:b6:22:88:bc:a2\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=System\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a3:21:2f:74:34:c1:1b:41:90:b6:4e:41:72:e0:\n 3f:9e:49:94:55:ec:02:4c:dd:14:80:b8:3d:c6:c7:\n 47:bb:a5:59:c3:35:86:89:17:08:ce:fe:71:e6:2f:\n 9c:c1:db:d2:7e:14:24:da:61:30:3a:e7:6e:b1:e3:\n 21:38:81:bc:47:df:b2:7f:1f:60:be:3d:c5:ed:76:\n 03:94:e3:c4:b3:3e:bf:f8:43:ba:c2:54:bc:bb:66:\n 59:98:a3:f9:aa:e3:10:e8:c3:88:dc:1a:18:6f:dd:\n 90:eb:6f:a3:4b:d4:af:34:5c:43:20:d5:5b:e7:98:\n a5:7c:7b:a9:15:86:bb:28:bf:ba:e0:bb:f7:1c:08:\n c4:26:eb:c1:ac:05:1f:74:4f:05:11:57:e0:12:77:\n 17:9e:89:dd:a5:38:ee:cf:cf:67:be:0c:5e:6a:4a:\n 74:61:21:79:8e:c3:28:f1:e2:06:00:2d:ea:3a:6d:\n e2:a6:25:fd:2d:8b:f5:82:36:91:8a:21:f0:6a:93:\n 19:d6:76:08:fd:cd:ee:90:a9:a9:cf:99:30:71:46:\n 57:ea:fb:c5:65:4f:7c:86:5c:9d:d7:b4:c3:27:3c:\n eb:27:dd:bc:55:76:1f:25:0d:cb:6f:43:9a:9f:ba:\n de:54:c1:90:03:9e:e5:0d:d9:cd:84:d4:58:74:63:\n be:59\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n A0:FA:B5:24:42:70:DF:E1:BB:E6:10:62:BE:FE:F5:81:13:2F:31:9B\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:System@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n ad:00:40:7a:34:ad:07:e9:ed:fa:8f:1f:48:08:79:81:a8:3c:\n 90:da:05:95:74:05:51:9c:17:a8:5c:03:09:c8:f8:2c:09:64:\n e2:7c:fc:69:e1:c0:5d:8a:d9:f0:f3:e4:cd:2c:5e:43:77:71:\n f8:58:20:88:8f:63:e1:b4:86:db:7a:54:df:ce:be:01:e2:55:\n a2:70:a8:89:64:cf:2a:13:78:91:de:83:ed:d6:74:24:00:ca:\n 3d:67:4a:cd:e3:82:b9:56:a3:3a:b4:80:b2:ac:61:e9:75:6c:\n 30:1c:81:96:2f:f0:99:b2:7b:73:b5:45:b0:3c:20:ed:54:b3:\n 87:37:9f:5e:07:c4:8a:72:94:53:4e:a2:a0:83:bc:fb:61:59:\n ff:8c:91:1c:db:ad:7a:e0:12:e3:a3:b1:91:97:d4:c7:ed:02:\n 6e:7e:01:d8:d6:d5:6d:81:a2:32:ca:8c:6d:32:91:40:97:e5:\n a1:ad:22:7d:af:ab:ce:68:0b:69:52:53:8a:80:dd:f3:9f:a8:\n 1f:34:a7:1f:37:58:cb:6c:da:54:cf:cc:0b:67:95:e9:6e:30:\n a4:ce:12:c4:5a:e0:d4:92:fb:0b:67:a8:51:ad:dc:4a:d0:ad:\n fb:92:77:85:a5:9d:84:ff:99:50:ca:15:4f:d4:30:c8:85:ca:\n 95:a0:88:62\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUfENlx88n44OuL2CsA+XytiKIvKIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGU3lzdGVtMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\noyEvdDTBG0GQtk5BcuA/nkmUVewCTN0UgLg9xsdHu6VZwzWGiRcIzv5x5i+cwdvS\nfhQk2mEwOuduseMhOIG8R9+yfx9gvj3F7XYDlOPEsz6/+EO6wlS8u2ZZmKP5quMQ\n6MOI3BoYb92Q62+jS9SvNFxDINVb55ilfHupFYa7KL+64Lv3HAjEJuvBrAUfdE8F\nEVfgEncXnondpTjuz89nvgxeakp0YSF5jsMo8eIGAC3qOm3ipiX9LYv1gjaRiiHw\napMZ1nYI/c3ukKmpz5kwcUZX6vvFZU98hlyd17TDJzzrJ928VXYfJQ3Lb0Oan7re\nVMGQA57lDdnNhNRYdGO+WQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFKD6tSRCcN/h\nu+YQYr7+9YETLzGbMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9TeXN0ZW1AdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAK0AQHo0rQfp7fqPH0gIeYGoPJDaBZV0BVGcF6hcAwnI+CwJZOJ8/GnhwF2K2fDz\n5M0sXkN3cfhYIIiPY+G0htt6VN/OvgHiVaJwqIlkzyoTeJHeg+3WdCQAyj1nSs3j\ngrlWozq0gLKsYel1bDAcgZYv8Jmye3O1RbA8IO1Us4c3n14HxIpylFNOoqCDvPth\nWf+MkRzbrXrgEuOjsZGX1MftAm5+AdjW1W2BojLKjG0ykUCX5aGtIn2vq85oC2lS\nU4qA3fOfqB80px83WMts2lTPzAtnleluMKTOEsRa4NSS+wtnqFGt3ErQrfuSd4Wl\nnYT/mVDKFU/UMMiFypWgiGI=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69:\n 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c:\n 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f:\n b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd:\n 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26:\n c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d:\n be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28:\n ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3:\n e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15:\n 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08:\n 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77:\n 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21:\n fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19:\n 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f:\n 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44:\n c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3:\n 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8:\n c5:ef\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:UserA1@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2:\n fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9:\n 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96:\n 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee:\n 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40:\n 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2:\n 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43:\n 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a:\n 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a:\n 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53:\n 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c:\n b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb:\n d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c:\n 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81:\n aa:42:9f:53\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\ntOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC\npt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP\nVPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz\nQxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX\nihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO\nCRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z\nOUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb\ncS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW\nX/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB\nChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a\nAC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v\nseLqxt4dsJfJNh2FgapCn1M=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 5c:a1:af:d5:7c:bb:16:ef:c2:c7:e6:53:fc:94:1a:ed:24:bb:b4:17\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b4:eb:22:e2:c4:ba:7f:33:aa:57:ab:13:f1:69:\n 09:98:28:3c:7d:a7:e2:41:2a:28:2f:f9:85:a1:6c:\n 94:ee:0a:eb:4d:01:4c:28:7c:9d:05:4d:d8:10:7f:\n b7:cf:13:c2:a6:de:11:0c:97:38:97:cd:6d:11:fd:\n 16:76:c0:eb:5a:b7:7b:17:13:45:9d:4b:00:4f:26:\n c5:b1:9b:67:93:2c:d6:d5:33:37:e1:50:1d:7b:0d:\n be:8c:cb:bd:29:99:8f:54:f6:7e:04:84:82:2a:28:\n ee:71:3e:8d:5f:72:b2:6a:77:6b:47:3e:ba:4d:b3:\n e2:96:14:71:0a:1e:26:16:8f:6c:1b:07:2a:ac:15:\n 89:1e:88:63:c3:81:3b:91:e9:f3:43:1b:f0:ec:08:\n 24:96:46:27:21:2a:56:25:2c:b6:cc:d9:02:70:77:\n 9d:e4:7c:44:8c:93:04:85:a3:09:0a:8e:f5:e7:21:\n fa:bd:56:28:b7:52:20:09:ec:9a:c4:d4:d7:8a:19:\n 4e:7a:10:e9:b2:10:36:68:ce:ce:78:8b:79:3f:6f:\n 70:3b:75:6d:70:59:3a:c9:85:a8:f8:23:d4:ab:44:\n c2:ae:f5:1c:6e:38:11:e1:5f:cc:8f:e2:43:f5:b3:\n 0e:09:17:b3:c6:ee:47:fb:39:c4:58:62:ba:e3:a8:\n c5:ef\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 70:55:CA:CA:A5:8F:4D:73:39:47:E2:97:A3:1F:F6:3E:33:C9:7A:BF\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:UserA1@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 99:81:61:3a:f1:c2:de:05:ad:ab:f3:fd:e0:d5:97:5b:fe:b2:\n fa:e2:5f:ab:41:9d:71:1d:10:54:0b:bc:b5:c9:8d:26:91:a9:\n 45:71:51:14:61:a7:3c:ef:1d:f7:db:71:2f:1f:c1:d7:80:96:\n 03:5d:0d:69:81:fa:be:ca:f7:56:70:7b:89:ca:8f:b6:16:ee:\n 4a:83:fc:70:2e:4b:0c:50:ba:c6:06:5e:58:bb:25:d6:19:40:\n 82:b4:18:57:16:5f:f2:98:3e:5d:9d:72:7a:8f:20:de:25:c2:\n 06:a7:46:b2:cc:4c:f9:da:a7:43:f5:a0:92:e4:e2:05:49:43:\n 9d:58:9f:20:5d:e2:88:77:f1:10:0c:f5:fc:a2:85:b6:41:0a:\n 1a:12:75:1e:47:3b:b3:4f:c9:45:71:99:b6:14:e9:6b:7d:7a:\n 98:ee:82:dd:59:f6:af:fa:a5:d1:1c:24:db:66:e7:82:bb:53:\n 70:4f:27:96:dc:19:c0:9e:2d:df:da:00:2f:c3:22:9e:71:9c:\n b3:89:da:0a:79:c3:f6:e3:9b:ca:b7:db:b6:5c:8f:e9:29:cb:\n d0:9c:e3:0e:0f:7c:2c:b5:b0:36:a9:13:38:d2:8e:6f:6a:6c:\n 0a:7f:3f:dd:af:b1:e2:ea:c6:de:1d:b0:97:c9:36:1d:85:81:\n aa:42:9f:53\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBcwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\ntOsi4sS6fzOqV6sT8WkJmCg8fafiQSooL/mFoWyU7grrTQFMKHydBU3YEH+3zxPC\npt4RDJc4l81tEf0WdsDrWrd7FxNFnUsATybFsZtnkyzW1TM34VAdew2+jMu9KZmP\nVPZ+BISCKijucT6NX3KyandrRz66TbPilhRxCh4mFo9sGwcqrBWJHohjw4E7kenz\nQxvw7AgklkYnISpWJSy2zNkCcHed5HxEjJMEhaMJCo715yH6vVYot1IgCeyaxNTX\nihlOehDpshA2aM7OeIt5P29wO3VtcFk6yYWo+CPUq0TCrvUcbjgR4V/Mj+JD9bMO\nCRezxu5H+znEWGK646jF7wIDAQABo4IBJDCCASAwHQYDVR0OBBYEFHBVysqlj01z\nOUfil6Mf9j4zyXq/MB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQTFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAJmBYTrxwt4Fravz/eDVl1v+svriX6tBnXEdEFQLvLXJjSaRqUVxURRhpzzvHffb\ncS8fwdeAlgNdDWmB+r7K91Zwe4nKj7YW7kqD/HAuSwxQusYGXli7JdYZQIK0GFcW\nX/KYPl2dcnqPIN4lwganRrLMTPnap0P1oJLk4gVJQ51YnyBd4oh38RAM9fyihbZB\nChoSdR5HO7NPyUVxmbYU6Wt9epjugt1Z9q/6pdEcJNtm54K7U3BPJ5bcGcCeLd/a\nAC/DIp5xnLOJ2gp5w/bjm8q327Zcj+kpy9Cc4w4PfCy1sDapEzjSjm9qbAp/P92v\nseLqxt4dsJfJNh2FgapCn1M=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e:\n 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac:\n 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7:\n a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9:\n 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02:\n 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d:\n 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c:\n 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a:\n e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35:\n 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91:\n 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71:\n c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1:\n 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0:\n 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2:\n 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2:\n d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99:\n b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8:\n fd:8d\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:UserA2@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5:\n 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1:\n 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f:\n 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff:\n 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a:\n 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac:\n 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1:\n cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e:\n ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16:\n da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92:\n 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91:\n 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2:\n 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01:\n 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78:\n a4:b7:fd:c2\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\npnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4\nIXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q\n1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055\nL88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+\nbsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5\nqOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl\nI8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em\nPaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3\nqcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+\nbsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB\nNaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp\nHTG1T5fg6jht4JhLeKS3/cI=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/UserA2_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7a:3d:fa:5b:9b:df:69:55:6e:9c:53:4c:fc:86:75:65:bc:78:4c:24\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:37:36 2023 GMT\n Not After : Apr 28 19:37:36 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserA2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a6:7c:40:80:2b:44:00:33:11:c6:c2:9d:67:3e:\n 87:8e:7e:40:d3:f5:d3:27:b6:7d:18:3c:c0:86:ac:\n 96:3a:ad:d8:c3:cb:ab:72:5e:4c:b7:24:45:da:c7:\n a8:cc:74:b8:21:75:62:9e:81:88:96:54:6e:db:f9:\n 8c:2f:4c:97:0d:ce:21:42:2f:92:57:7f:34:2b:02:\n 43:4c:22:ae:14:ca:fc:b2:2c:d0:67:0e:52:e0:6d:\n 61:96:a6:3b:cc:4f:6a:d6:ef:45:9c:74:92:25:6c:\n 0a:10:62:1b:22:2b:11:6b:d1:52:4d:da:8d:c3:4a:\n e6:74:a7:1b:1e:ef:8a:f4:96:88:02:0d:b7:57:35:\n 9f:a3:ff:a2:2c:b7:0e:27:4e:79:2f:cf:0c:f1:91:\n 0e:bf:01:d7:a2:71:2c:b7:0e:4b:7e:50:91:89:71:\n c2:17:aa:cb:29:80:9e:d7:2b:fa:33:41:e8:82:d1:\n 3a:97:3d:6c:de:66:9b:b4:ea:1a:eb:94:be:6e:c0:\n 66:e8:77:3d:72:d5:5c:a5:e8:ab:3b:33:f4:b3:c2:\n 26:49:bc:08:55:cf:16:b6:12:22:91:fe:c1:5a:b2:\n d7:77:e3:f4:47:bc:c4:77:6b:f5:7f:c3:e8:48:99:\n b9:a8:ea:b1:ae:e6:cc:3a:12:fa:4d:2f:5f:0f:a8:\n fd:8d\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B8:8E:4F:76:F1:F8:3C:A5:23:C5:8F:A1:2E:64:3E:48:53:02:CD:6B\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n email:UserA2@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 7c:1b:ae:98:16:42:f3:b2:a6:66:e9:a4:4f:61:04:a8:23:d5:\n 55:ea:d4:68:b5:98:fd:66:ff:10:dc:54:b7:01:78:4f:fc:e1:\n 75:e8:09:6d:ad:ac:57:b0:33:41:26:3d:ac:b0:17:46:c4:6f:\n 5b:c7:fa:ad:d2:94:13:ef:5e:bb:f5:ad:2d:39:85:d3:af:ff:\n 56:8e:f6:d1:20:12:03:86:cd:e8:ad:38:49:30:fb:98:de:3a:\n 5f:61:5a:08:37:a9:c3:10:ed:a3:60:3c:46:68:30:d8:4a:ac:\n 5d:eb:fd:d9:5d:90:b1:f0:b8:a8:68:5e:c8:41:6f:de:eb:a1:\n cc:33:98:2d:06:17:26:c4:24:bf:62:82:a9:13:04:71:3e:6e:\n ca:20:cf:5c:c5:47:67:f5:db:2e:56:60:4c:52:0c:4e:59:16:\n da:6a:e3:b2:e4:cb:d6:65:26:df:26:2e:e0:f4:11:b1:36:92:\n 7c:ab:c3:c3:97:a5:06:26:54:5c:c1:35:a1:2f:e5:0f:2f:91:\n 2d:cd:c5:dd:a7:f2:4c:e1:4d:0d:5c:bd:25:4f:c8:52:79:c2:\n 29:78:ef:88:10:43:a4:c4:df:97:48:22:09:db:48:19:85:01:\n 48:39:28:20:69:1d:31:b5:4f:97:e0:ea:38:6d:e0:98:4b:78:\n a4:b7:fd:c2\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUej36W5vfaVVunFNM/IZ1Zbx4TCQwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTM3MzZaFw0zMzA0MjgxOTM3MzZaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckEyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\npnxAgCtEADMRxsKdZz6Hjn5A0/XTJ7Z9GDzAhqyWOq3Yw8urcl5MtyRF2seozHS4\nIXVinoGIllRu2/mML0yXDc4hQi+SV380KwJDTCKuFMr8sizQZw5S4G1hlqY7zE9q\n1u9FnHSSJWwKEGIbIisRa9FSTdqNw0rmdKcbHu+K9JaIAg23VzWfo/+iLLcOJ055\nL88M8ZEOvwHXonEstw5LflCRiXHCF6rLKYCe1yv6M0HogtE6lz1s3mabtOoa65S+\nbsBm6Hc9ctVcpeirOzP0s8ImSbwIVc8WthIikf7BWrLXd+P0R7zEd2v1f8PoSJm5\nqOqxrubMOhL6TS9fD6j9jQIDAQABo4IBJDCCASAwHQYDVR0OBBYEFLiOT3bx+Dyl\nI8WPoS5kPkhTAs1rMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWYGo6dMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToxODg4OC9pbnRlcm1lZGlhdGUxX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjE4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQTJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAHwbrpgWQvOypmbppE9hBKgj1VXq1Gi1mP1m/xDcVLcBeE/84XXoCW2trFewM0Em\nPaywF0bEb1vH+q3SlBPvXrv1rS05hdOv/1aO9tEgEgOGzeitOEkw+5jeOl9hWgg3\nqcMQ7aNgPEZoMNhKrF3r/dldkLHwuKhoXshBb97rocwzmC0GFybEJL9igqkTBHE+\nbsogz1zFR2f12y5WYExSDE5ZFtpq47Lky9ZlJt8mLuD0EbE2knyrw8OXpQYmVFzB\nNaEv5Q8vkS3Nxd2n8kzhTQ1cvSVPyFJ5wil474gQQ6TE35dIIgnbSBmFAUg5KCBp\nHTG1T5fg6jht4JhLeKS3/cI=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/certfile.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/private/System_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjIS90NMEbQZC2\nTkFy4D+eSZRV7AJM3RSAuD3Gx0e7pVnDNYaJFwjO/nHmL5zB29J+FCTaYTA6526x\n4yE4gbxH37J/H2C+PcXtdgOU48SzPr/4Q7rCVLy7ZlmYo/mq4xDow4jcGhhv3ZDr\nb6NL1K80XEMg1VvnmKV8e6kVhrsov7rgu/ccCMQm68GsBR90TwURV+ASdxeeid2l\nOO7Pz2e+DF5qSnRhIXmOwyjx4gYALeo6beKmJf0ti/WCNpGKIfBqkxnWdgj9ze6Q\nqanPmTBxRlfq+8VlT3yGXJ3XtMMnPOsn3bxVdh8lDctvQ5qfut5UwZADnuUN2c2E\n1Fh0Y75ZAgMBAAECggEAGJh8EGwU0pB56nbVmOW1Sd8jsanGNgMeYIMG83Xf+6uk\nY1GqcXiK4DTOhQuYOcV0UQSmAtQlAriawNDzVRMAiaCxh8e6HSzwrws8YoJOCc2U\nAbFqkvrWQvYdW62bive1+LZkp/T6SsGQJGNebmRIr18a0vRAaWSjTOfTOFbqWKwD\n640JDw2KmJmba6JtOaEL4QWrvbugTNwh3OEHugBVTCiRTdruVpCpLSxW1yZEpwB2\nBmxQxHvbtIjiOmHuNrsh21jzi7IEx+TFawJ0EV6Wm9XCbjX62XETraILg5bWVGv7\nX+TIDE2JBCC9GZMm9Qj1EfCojRmKfxopv7sA1yBYRQKBgQC5K5NzQzk14G64tyvW\n61BteydWlBzFbgiMjYUq9wqgf2WbDVONBUB5x3MOArOmYuJOy0Xbt+pF3pF8wrkl\nhMt/hZqKzDtdDWo3+PnNFgcWmB+T76Jei9khIk3D9ENcGaNwGAS3l8sxqXNLVVBJ\nu5qHKeKFreXSra7xlXOuw5IMbQKBgQDhh1QqpQMloXOPCQWWzPtrEa8BRGNMFQTU\nyZFHeetQjjX5opxxMbbXU/wNz/dRgdfe2VLVo9e4dtQbzCKenuwWeivuDxd4YOsF\nVon9XDOzVWoXuP01MxDcU+sRoBLwWbpCWMe7r4Ny98C78+/5GssvkFUo+hd2vPo6\nU20pVZfuHQKBgGYD1eZooLpH/XgSojpzxgmrEc8nJnq21krpJPa4x8gIp+e2fdNx\nk0YEViTf5C3EyL10S/Zy6sS3jBvaA7rh4GNPLgdN4V6wp1ZS+vy8KAeQo8US/rds\nAKG6jnFovzucfGijMuYa4L1ph7V3ORaGHupcbwoK9lUNjxZVqjgcUvg5AoGBANOU\nzpWjcaxgJ7XNVP0BGe59DJ43tqCuJ3YqFK3l56oPgPvOXs6jQVIKbLHYpcJF+mwL\nnvbnW36nnJ7niKMfnYYI4CXa6r34zwSXB6Y2Vhqsy3euCX9bhTnvUN2cO6hZxbBw\n8hFWvA+j96FdXYlqZa0dz4c9+b1f1bHaitL4hizRAoGAVlH2lJr6s+ZmzkDY7D+Y\n6YKyjXaxhHBIqB2oLK1KuxGMiQnRADs9UOC4x2PQPOfemjVTJ3eN3rwxdqSh+Y2v\nK+RejHBJzbd4JIv0QRxpPAm9sezaNEHa7ss387cLZEBEYUI9HkIuPunKX+2lHITn\nWpVRyzYjVkFUUcRe3DyTlh8=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC06yLixLp/M6pX\nqxPxaQmYKDx9p+JBKigv+YWhbJTuCutNAUwofJ0FTdgQf7fPE8Km3hEMlziXzW0R\n/RZ2wOtat3sXE0WdSwBPJsWxm2eTLNbVMzfhUB17Db6My70pmY9U9n4EhIIqKO5x\nPo1fcrJqd2tHPrpNs+KWFHEKHiYWj2wbByqsFYkeiGPDgTuR6fNDG/DsCCSWRich\nKlYlLLbM2QJwd53kfESMkwSFowkKjvXnIfq9Vii3UiAJ7JrE1NeKGU56EOmyEDZo\nzs54i3k/b3A7dW1wWTrJhaj4I9SrRMKu9RxuOBHhX8yP4kP1sw4JF7PG7kf7OcRY\nYrrjqMXvAgMBAAECggEATFRQOaCKlpQzsB0rotSQCbQgIVutZ5Tjs6nwqTRoeS3+\nLFT5zrMUhGJdYEiiQimyHDjgtJEwfUtcUxSWX6/xHCsBMbEd08kK7loLWm2Ye02V\nrgmX7+WfKoWX+UsUGfMBt/TvIfTN/f+a6ghcGQMJJ0YO6tYaQCI+3NbvAjfKFgXi\nnWWZA+ipjh+Nu3YhVAy/uMInMi0qGWmpomU1yS+04E3OQksKYc3OmER7zFwbmNbF\n0LanWlLURUeHIS1BY+V4yXw6yBJaCDUpVA37mfLqRQshGtGjmWLtMt/AuSFokwHd\nyewoORlpVkZnE4Igv1JDggFdEI5lZ4PTmOjEXfntYQKBgQDH6sBr24OMUceNWyvf\nk03pqUaoiJkivAcUI/krfY7mdSLkiqs+UPuRrikGbvKT3R+iJVbTB4dXzGG6nzBc\nes7xwvzDGNHHXe0KAFhyIXwNMZmLGTmsNVfnKPAQ1BfKG9MtD5ck2gI1L1DkpaRz\nX57YONvG05HYmY7TaV2VOKK1iQKBgQDnq/zW6P9WHpIHBjZRN0V4yFl1dMfn2VwZ\nc3QWBd+kTwVBBhlJlqYeRIt4kmwExPnd3OX8Y7N18RttIc+k4dZgTA4w8G3xzvgk\n0sHgf3EBbrkUuS23BJ2IPIb4LmWckH6+KJkvBrlZOfoLBj8uxQwz/wmWA2IoQgKv\nCvDNr6G5twKBgECWSgZOjAhoX1T+0ITRvUkxJB/MydSb9JmAKb7wOJuh2l0Fo99l\nIHFnV9+5Nmuo89BZydwxwXsPD7/QMDqgfn1C5pBNU3Damnsxs2FkCgTlMlrrEmPd\ndAG9ixmUu/7S0H3tXIJOYIo4OCU2kpOnn9TxQafRsHvO2ILatp5ABukpAoGBALgP\nKJ4GF3bwaswx302/P+6qHoj28yv8wPNnir9Eg14jeeUjV0vj6K77fmOY0UEozeu6\n6O4QuC/oEwYtaq9wzcVMJ6oyGueWrAd1eptGJR4iPeF9DhjuDcqDbCgZlJlDI68o\nyitWiEOfkEzZ9bDO1NcqtQ7+OSoK597yLkb8Vt0ZAoGAN2dHPkTiNFlbzCefv/EP\nA4xQUAUiwfQ9ZlhMtD9Tlea8cMAD901rxy52YrgCvBPxw3HmKG2H0NOpa7BwrgA8\nuODxi6xBRExRhvaqZe1aP1xn4XKw2VVsMlIlJQj2Wmuxeknfm9R1sfRD797c4nuN\nntLUOPAWtDkLoJLrTd9EqFk=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmfECAK0QAMxHG\nwp1nPoeOfkDT9dMntn0YPMCGrJY6rdjDy6tyXky3JEXax6jMdLghdWKegYiWVG7b\n+YwvTJcNziFCL5JXfzQrAkNMIq4UyvyyLNBnDlLgbWGWpjvMT2rW70WcdJIlbAoQ\nYhsiKxFr0VJN2o3DSuZ0pxse74r0logCDbdXNZ+j/6Istw4nTnkvzwzxkQ6/Adei\ncSy3Dkt+UJGJccIXqsspgJ7XK/ozQeiC0TqXPWzeZpu06hrrlL5uwGbodz1y1Vyl\n6Ks7M/SzwiZJvAhVzxa2EiKR/sFastd34/RHvMR3a/V/w+hImbmo6rGu5sw6EvpN\nL18PqP2NAgMBAAECggEAHsf4UPou326RydLvsUgRXhofuFDKEpyd8l5BJmVAfWbp\nHgJJF6Mxwea196ZUokCuTplae33tmAXSXV99OL2LbCUBZzBOeVjud0k60hfTYcrJ\n/9NjULqIPjBbC7R+d97zHPwuPagb4UlhbvgElkOqO+n+sqBG96WgiE7hJ84YPfJO\nY+Is9vRbESkMVK1TH5PxfDE0Yu/i2vm/Fv+Ekgqe+GiZgbDw+L98D7ZX1xb8W2Ix\nWnM2Skd22pit1ftpixuHbdHcPX2NNdhFKy8r/ZYK0SCLwkb8yJhDLQF8Q01YXd6q\nFHtuE+MGXsr7dkcqYtc2QvigJdHs72WCjZwcpA+vgQKBgQC9qt3AIXPjPckhTEEK\n97tg0zqFVPHyhiy23qsKJ/egMIhYESQngLOPcQ0Q/bG5OJqe5sx31rmKQ368QUSX\nlIPG9WrRxCh3BTo7nOOEmAh4uGnKtvDJbTRP56fPQhkKlDywua8vKs0moUdcact7\njjXYxXSPGEqHQrjPkuurPJvc7QKBgQDgtd4/kYGM9R2ltSLWm/TZnE6LM1EtBWrA\nHNAYV7WxxKdUvTtxBIXDKer0RAbDKHIoZ6HI3lon5siuBVtIFoq2VLxS1jm3rEJv\nqV6USxxDnEkbLla6Jzmd5eqFPZErWfNqmdXP1sqC8fs5q1PUJXNtEIdJulXQnHP2\n5lJxq8ovIQKBgQC1HBerg0YZ08HfHeVuB6jRiGH1N2vhXeYMqQtCI2/9ctp+3b9c\nSTUs35LOirHOYBKlcVYFiPCa6mB2ewx4gcRjk61wqJLLNB6rFeDbmCFexRmgDJhY\nfwLY2igPbNpkk7BwQJ7bt082eAKgaBV54g3g9IucqGFiT4ASFgUb+kAK8QKBgFYJ\nrJgAWW8kJv7clQNA4YY0j+pCctFfIpl+LrszUhFHr54Fem3ygljQgvKV3VT59oO7\n8jkb0b83YR0oVeQLJX9cgGLjPWQzI5jna5wyChdlDqTGoFRUUn4/mwT7JstHfKkT\nT8dtgUqT5lIVZFp1IHXg/zveiZ7/WHNvip+VXCuhAoGAE3aA/rPBYEHJFLHaSgcR\nE+ggLP3HjQPN6347NXBZSaWSXqXtdLDlgnnysRnEd+JHSDI3lNuFa6d6nY7x7Mc0\nBn54Tf3KLLHcyUrwQTCjY230212gYGqWXMgeaTPJRtl4K0PchWzKzZ1m9RQAZHOQ\nOaBsh0IA+LCDTmsPsbzh6U4=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:40:31 2023 GMT\n Not After : Apr 28 19:40:31 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a:\n c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00:\n 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28:\n 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5:\n 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0:\n 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6:\n d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1:\n df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3:\n ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7:\n ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da:\n af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76:\n b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd:\n 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4:\n 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f:\n 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c:\n 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4:\n ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4:\n 66:6b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n email:UserB1@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8:\n d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1:\n 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29:\n 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77:\n 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c:\n eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65:\n 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c:\n e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73:\n bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1:\n 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f:\n 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3:\n fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5:\n c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86:\n 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6:\n 5a:e4:81:74\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nuhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF\n7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT\nfzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl\ndwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T\n/LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK\nS9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ\nY2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6\nS12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22\nPQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN\nc7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2\ngygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko\n4iDBH6wH/Kzg9e56xlrkgXQ=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3a:d5:76:e0:a4:4b:67:ba:da:f2:9b:15:09:4c:ff:54:58:1d:e9:92\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:40:31 2023 GMT\n Not After : Apr 28 19:40:31 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:ba:19:65:ab:3f:2e:f2:7a:93:ea:06:eb:a2:9a:\n c5:b9:20:66:2e:74:1b:94:5a:43:1c:8c:22:72:00:\n 79:2d:20:18:e3:4a:35:a6:df:8a:58:33:73:2c:28:\n 20:e7:d9:85:ec:f5:81:ae:44:44:55:66:65:d6:b5:\n 78:71:c4:d8:c2:7b:4c:2d:8b:18:b6:86:fc:50:c0:\n 7e:b6:6e:f7:76:c0:30:6c:67:09:53:2d:87:98:d6:\n d4:d8:b3:a9:80:45:93:7f:33:3f:41:2a:70:f3:e1:\n df:a0:85:64:4b:25:e4:91:e9:e6:c8:c3:a0:3e:b3:\n ef:97:1f:ae:9d:44:84:35:26:26:4e:0c:7a:1d:c7:\n ef:b6:46:8d:82:b8:b0:18:fb:25:77:04:20:8c:da:\n af:fa:9e:a2:b0:67:b6:a6:5b:d7:95:a5:3c:3e:76:\n b4:37:4a:48:98:34:96:9d:d2:ff:36:6a:f4:2a:cd:\n 85:b3:e3:71:74:0f:e0:25:f1:06:cb:9d:53:fc:b4:\n 5d:c4:8d:7a:0b:bd:16:ee:5c:58:21:ad:49:34:9f:\n 9e:1b:6d:f6:47:52:1f:a0:74:00:fe:3c:4d:5f:4c:\n 5a:23:4a:d5:4c:ff:3f:42:5d:85:df:f6:3b:32:c4:\n ca:4b:d0:9d:4b:9e:86:a6:64:44:b8:ae:24:1a:f4:\n 66:6b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n EC:AB:7B:4D:CD:62:D6:89:63:69:FE:97:34:5A:96:58:A5:94:A6:D9\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n email:UserB1@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n a8:78:fa:c2:44:e0:b9:c7:af:d5:cc:b6:b4:2b:3d:74:ae:b8:\n d1:e1:22:d0:63:7d:77:97:db:97:2f:f1:f0:ce:e3:9e:5e:e1:\n 2a:19:54:00:38:7b:30:0b:8b:95:3a:4b:5d:83:08:80:fe:29:\n 85:72:fd:c9:80:6b:c3:fd:a3:00:4f:b5:f2:34:a3:42:54:77:\n 77:70:43:40:fe:1f:7a:b7:7f:55:c3:c0:e2:44:d1:95:fb:4c:\n eb:f8:39:dd:b6:3d:07:27:39:8e:89:e4:a8:49:fd:02:70:65:\n 72:6f:c7:d4:12:57:bd:47:ea:7d:2d:63:b4:fe:81:33:20:3c:\n e0:36:a2:60:58:79:5e:ce:6c:ed:7c:97:6e:6b:52:25:8d:73:\n bb:ea:b5:8b:1e:d2:97:24:88:59:ea:a4:29:a3:ea:04:45:e1:\n 6a:cd:c8:b9:13:44:57:f8:7e:1a:85:34:11:71:f9:10:a4:6f:\n 07:d4:7d:21:84:f1:52:6f:f9:e8:36:83:28:32:aa:ad:2a:c3:\n fb:98:02:c7:2e:2c:49:08:21:af:fe:15:0e:f3:ce:e7:24:b5:\n c8:08:d6:20:e8:8c:24:ce:1f:84:0b:9a:46:07:8c:05:d0:86:\n 04:06:2b:a2:a8:e2:20:c1:1f:ac:07:fc:ac:e0:f5:ee:7a:c6:\n 5a:e4:81:74\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUOtV24KRLZ7ra8psVCUz/VFgd6ZIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckIxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nuhllqz8u8nqT6gbroprFuSBmLnQblFpDHIwicgB5LSAY40o1pt+KWDNzLCgg59mF\n7PWBrkREVWZl1rV4ccTYwntMLYsYtob8UMB+tm73dsAwbGcJUy2HmNbU2LOpgEWT\nfzM/QSpw8+HfoIVkSyXkkenmyMOgPrPvlx+unUSENSYmTgx6HcfvtkaNgriwGPsl\ndwQgjNqv+p6isGe2plvXlaU8Pna0N0pImDSWndL/Nmr0Ks2Fs+NxdA/gJfEGy51T\n/LRdxI16C70W7lxYIa1JNJ+eG232R1IfoHQA/jxNX0xaI0rVTP8/Ql2F3/Y7MsTK\nS9CdS56GpmREuK4kGvRmawIDAQABo4IBJDCCASAwHQYDVR0OBBYEFOyre03NYtaJ\nY2n+lzRallillKbZMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQjFAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAKh4+sJE4LnHr9XMtrQrPXSuuNHhItBjfXeX25cv8fDO455e4SoZVAA4ezALi5U6\nS12DCID+KYVy/cmAa8P9owBPtfI0o0JUd3dwQ0D+H3q3f1XDwOJE0ZX7TOv4Od22\nPQcnOY6J5KhJ/QJwZXJvx9QSV71H6n0tY7T+gTMgPOA2omBYeV7ObO18l25rUiWN\nc7vqtYse0pckiFnqpCmj6gRF4WrNyLkTRFf4fhqFNBFx+RCkbwfUfSGE8VJv+eg2\ngygyqq0qw/uYAscuLEkIIa/+FQ7zzucktcgI1iDojCTOH4QLmkYHjAXQhgQGK6Ko\n4iDBH6wH/Kzg9e56xlrkgXQ=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:40:31 2023 GMT\n Not After : Apr 28 19:40:31 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea:\n d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa:\n e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1:\n 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0:\n 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e:\n 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e:\n 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9:\n 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a:\n 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84:\n 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05:\n ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38:\n e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66:\n d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c:\n 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b:\n 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86:\n 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6:\n 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03:\n a5:5b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n email:UserB2@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f:\n 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef:\n 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a:\n a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7:\n 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab:\n 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc:\n 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4:\n 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51:\n 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8:\n ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0:\n 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78:\n 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48:\n 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42:\n 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b:\n 13:5f:cf:4c\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nsh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb\nQmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/\nJB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R\nqeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ\nilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl\n6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW\nZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL\nGhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA\nv3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF\nUW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr\nmHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO\noFsGb/GFJ4xrIN/gaxNfz0w=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/UserB2_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 1e:dc:a2:b9:fd:aa:6e:73:ae:1c:7d:8d:13:73:d1:cd:16:bb:40:90\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:40:31 2023 GMT\n Not After : Apr 28 19:40:31 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=UserB2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b2:1d:92:83:be:0f:40:5c:b8:34:93:66:28:ea:\n d3:85:1e:ec:66:e3:97:d0:fe:a7:2d:2c:89:c4:aa:\n e0:ff:62:a2:8b:19:19:8a:1f:bb:a9:24:2f:a8:a1:\n 16:95:a7:5b:42:65:2f:03:27:12:ac:44:fb:2f:e0:\n 9b:19:52:32:a7:db:83:d0:1a:d6:36:d7:b7:40:0e:\n 85:c6:a7:75:5c:d1:71:a9:99:d3:da:2b:70:f9:9e:\n 9d:0b:a8:35:bc:3c:7f:24:1e:b5:2e:83:31:07:c9:\n 9b:4a:0e:a3:32:36:bd:a6:2c:55:79:f8:71:66:6a:\n 2a:8f:f9:f9:67:b0:06:21:e4:2a:02:44:b6:39:84:\n 18:7a:00:5e:34:36:f4:61:0d:11:a9:e2:0c:b8:05:\n ed:67:97:bc:29:e7:69:ac:48:6e:fb:78:e9:3b:38:\n e3:db:09:cb:22:0f:9a:57:1c:cc:06:f1:f7:44:66:\n d0:01:c4:c1:14:65:29:e5:cf:19:26:73:c9:8a:5c:\n 2b:25:a9:d1:c6:3e:d8:4d:f5:f3:67:c7:23:b9:7b:\n 2b:f5:97:28:89:81:99:9d:82:45:21:27:f4:ca:86:\n 02:22:2f:26:4b:61:8a:cb:76:fb:b1:7b:4c:42:b6:\n 25:e8:3e:cb:ab:2c:60:a7:a3:82:fb:ef:05:59:03:\n a5:5b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C6:25:DB:6C:4E:18:89:96:67:30:E8:5F:EC:0C:03:70:A4:4C:07:98\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, S/MIME\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Client Authentication, E-mail Protection\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n email:UserB2@user.net\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 7d:93:8d:17:4b:fe:9e:5d:d0:4e:c3:47:dc:6c:05:1b:10:7f:\n 9d:24:75:ea:30:27:c3:b1:26:2c:38:c3:c9:18:ec:21:d2:ef:\n 07:b2:d4:f9:2e:a1:a2:1a:a5:68:cb:1a:14:55:7f:82:05:8a:\n a3:0d:11:f0:ed:f2:e2:c0:e3:6a:1c:76:42:01:92:68:2b:f7:\n 4d:98:ae:7b:02:f1:36:2e:44:67:43:39:8e:08:91:f1:f0:ab:\n 9c:84:df:08:80:bf:76:6b:37:3f:e8:70:e0:d6:27:73:e9:bc:\n 49:1f:c2:4a:15:51:22:c6:f3:85:52:e3:a6:93:aa:f6:c9:b4:\n 96:f2:09:e6:62:53:0e:87:76:fd:7a:38:69:e2:41:54:c5:51:\n 6e:cf:bc:1a:7b:0a:ef:c6:6e:be:b5:72:4d:f4:6f:fd:a5:a8:\n ba:23:15:80:fa:b6:37:8d:68:d8:3e:36:c5:ae:f6:6c:22:a0:\n 00:0d:93:e1:ae:41:9a:d7:35:d0:ab:98:71:1b:6b:8d:da:78:\n 65:3c:97:be:9c:9e:d7:32:a1:0c:2b:60:ac:74:18:18:e4:48:\n 87:40:dd:bf:eb:0e:27:17:96:a1:aa:32:a9:58:b5:ee:fc:42:\n 7e:d7:71:a4:8e:a0:5b:06:6f:f1:85:27:8c:6b:20:df:e0:6b:\n 13:5f:cf:4c\n-----BEGIN CERTIFICATE-----\nMIIEXTCCA0WgAwIBAgIUHtyiuf2qbnOuHH2NE3PRzRa7QJAwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTQwMzFaFw0zMzA0MjgxOTQwMzFaME8xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEP\nMA0GA1UEAwwGVXNlckIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nsh2Sg74PQFy4NJNmKOrThR7sZuOX0P6nLSyJxKrg/2KiixkZih+7qSQvqKEWladb\nQmUvAycSrET7L+CbGVIyp9uD0BrWNte3QA6Fxqd1XNFxqZnT2itw+Z6dC6g1vDx/\nJB61LoMxB8mbSg6jMja9pixVefhxZmoqj/n5Z7AGIeQqAkS2OYQYegBeNDb0YQ0R\nqeIMuAXtZ5e8KedprEhu+3jpOzjj2wnLIg+aVxzMBvH3RGbQAcTBFGUp5c8ZJnPJ\nilwrJanRxj7YTfXzZ8cjuXsr9ZcoiYGZnYJFISf0yoYCIi8mS2GKy3b7sXtMQrYl\n6D7Lqyxgp6OC++8FWQOlWwIDAQABo4IBJDCCASAwHQYDVR0OBBYEFMYl22xOGImW\nZzDoX+wMA3CkTAeYMB8GA1UdIwQYMBaAFHVV4o7nraXdgD3JMwssold37RWsMAwG\nA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgEBBAQDAgWgMA4GA1UdDwEB/wQEAwIF4DAd\nBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwPQYDVR0fBDYwNDAyoDCgLoYs\naHR0cDovLzEyNy4wLjAuMToyODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYI\nKwYBBQUHAQEEJzAlMCMGCCsGAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4\nLzAaBgNVHREEEzARgQ9Vc2VyQjJAdXNlci5uZXQwDQYJKoZIhvcNAQELBQADggEB\nAH2TjRdL/p5d0E7DR9xsBRsQf50kdeowJ8OxJiw4w8kY7CHS7wey1PkuoaIapWjL\nGhRVf4IFiqMNEfDt8uLA42ocdkIBkmgr902YrnsC8TYuRGdDOY4IkfHwq5yE3wiA\nv3ZrNz/ocODWJ3PpvEkfwkoVUSLG84VS46aTqvbJtJbyCeZiUw6Hdv16OGniQVTF\nUW7PvBp7Cu/Gbr61ck30b/2lqLojFYD6tjeNaNg+NsWu9mwioAANk+GuQZrXNdCr\nmHEba43aeGU8l76cntcyoQwrYKx0GBjkSIdA3b/rDicXlqGqMqlYte78Qn7XcaSO\noFsGb/GFJ4xrIN/gaxNfz0w=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/certfile.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6GWWrPy7yepPq\nBuuimsW5IGYudBuUWkMcjCJyAHktIBjjSjWm34pYM3MsKCDn2YXs9YGuRERVZmXW\ntXhxxNjCe0wtixi2hvxQwH62bvd2wDBsZwlTLYeY1tTYs6mARZN/Mz9BKnDz4d+g\nhWRLJeSR6ebIw6A+s++XH66dRIQ1JiZODHodx++2Ro2CuLAY+yV3BCCM2q/6nqKw\nZ7amW9eVpTw+drQ3SkiYNJad0v82avQqzYWz43F0D+Al8QbLnVP8tF3EjXoLvRbu\nXFghrUk0n54bbfZHUh+gdAD+PE1fTFojStVM/z9CXYXf9jsyxMpL0J1LnoamZES4\nriQa9GZrAgMBAAECggEAAVnSLX+MajFnQj3our9FMrZSGx4bbAhQz9dndUWc1HT4\nd4AgPFpAfqpof6vVycHx2jSnILhuseJykSGzwoHgynrVpI82T6f9EzhRmkLbK1Y5\n6t6jC9uwXDvv37RgYcW02o1avD8VdHtN+qXtO4Db22P1p7zeA6LzSscmmLjf4QcY\n15O5DFUsVD6jfjI+edTKY4OgqblwD/t5EqApBI/KhAypSRD/NDzKdtHZO+K3eJW0\napznw5wrzPVX1xk4p+1LnM5nLBRnwECqRyzlmxjX3rJr7tVVWqOkTHs807wK+7AW\no9rujmS/J8I86BtZdj938VGVyuyqhJndANF8rOh6nQKBgQD09ZFmj/SMIeJIa2Xj\nMiK1JMU1rcr2h8NxYhQqZV/sj8TD+Sm/ljCDDClqyo5wAvBdIkFO689sIDEFT1W1\nvUOnE8xa4kkoSf4TVADiGAt4aLHiPiRAoX0aPqgBSy9IcXg7p/iG5qFLp72CNEFg\n3vM5vgjX+xio42Hqdo6+ruE1pwKBgQDCfK4KpR2BAv6dbuGNF8qZHWkgBDpSSlug\nWMEZe6c9l44EAIHgJNr4nBviVZTZAHD+H5qSC8STQ6Y4ccOZYnG4dGxAztKYnX9Z\nT6R+zOkisK+Zhq9noj8veBwS6F2fGTL7cagBkj2q3SveagGtutkV6kOKUw5uu8dI\nGnSxaiNpnQKBgQDrzURlVWgUST3ZdsECvq1YcIgCj0TUooYKLF67HREE2LSR7dU5\nXytdyyRHb6tDuiCFlscFYMwwCqEFuoQISaPJPq62QiQoS2nwUynyezD3fNjXr/gX\n2xxhWjVB4Y0nkEssKhp8SaC1AkjUANd6l8PNLti2iDkJwrDsEaqBdjjG+wKBgAVM\nEg12K9SMuVSeZYRLRphfBbL6ioAdSFuYr0G7bXWvAA452U+6kUA+OEA05oX2jh1N\nzQ73RRZhvFBDQPmXhdNpUF1/hJrlh0dudOODP0JTn6TF11cyQxhO5CzbqVkg/ZN9\np/7K9eUGeyBmsL8DnNAM/mPxGS6I7MeY+N6wLmC9AoGBAPL97OOwtkfCqCXBzIua\neNFIPvW8cKEM1ggxUPGar36TuaKnDt8bdGL3/ZEAD28XMGCbUwo99WrW0J4r9b95\nRrs1FzUW9iVIqB+4W35lMfSbFOC/2GsSUf95ANT4wihu2QbVQU7iqjXw+w8ZN9Vx\nQkiwv6M/K0lzm6Q1H1pb7urx\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/client2/private/UserB2_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyHZKDvg9AXLg0\nk2Yo6tOFHuxm45fQ/qctLInEquD/YqKLGRmKH7upJC+ooRaVp1tCZS8DJxKsRPsv\n4JsZUjKn24PQGtY217dADoXGp3Vc0XGpmdPaK3D5np0LqDW8PH8kHrUugzEHyZtK\nDqMyNr2mLFV5+HFmaiqP+flnsAYh5CoCRLY5hBh6AF40NvRhDRGp4gy4Be1nl7wp\n52msSG77eOk7OOPbCcsiD5pXHMwG8fdEZtABxMEUZSnlzxkmc8mKXCslqdHGPthN\n9fNnxyO5eyv1lyiJgZmdgkUhJ/TKhgIiLyZLYYrLdvuxe0xCtiXoPsurLGCno4L7\n7wVZA6VbAgMBAAECggEACzGbuulEMPd1DwetATtNZTbHBOoMe3vVj0A7dEiIXokG\nzc2tl10Td26EVEBFvTpI5YiaqwzElYNMTo2M7TjizvTynZyGusPnisl6SoWoh0U5\n2HWIAHkSKCAww1RbGL+HbEuO5Wy3R7FMC0C6PuQPP3Bo+swVnqn1s6wf88U/zWml\nNthu0uQSj+pxW4tK/p7IoUVBnSqKExODDLG4LpO3meSaZIr36wC6bJZ8w8lZfRBy\nDkPJu9NNknL6qSoVGozLzgtg1//yCkU+LX0OcDgTNeup5DlA08jglQY8p3Xo3FPn\nevofoPvDnku4H1gCXT/djERRSlPdcGPEcy7xMQx12QKBgQDqdoL8hkp/DUzoKZyM\nu2Vud5E1jULal3SmRB1XFzqxEiFsAT6UBH2feVweBOKTjLBqIuC+teQ+JgC5TsYP\nCGbclQG/XBTYzOPfn3bBJWS4j7Jd68uXDQvkM9+RroFVaCXn75UGWEMqcbtgTNyU\nwUrAVgfTtz07iHf2oUy+IreW7wKBgQDCegdlOojhn4juC+B5ROJHXzwI1qEznpJa\nftI7RERUbDFRIaucwvI6y95nduIRORO1bzpBhHZzJDPNBhZZya9wkaLElXktgi1Z\nIwF6eb3m/FtOxx7DtI9daCVsuZsoPEw08NJq6UYQqeauaJ3LM5rDSMX0DN3V//2m\n7tULbZn4VQKBgQCT4dwMWsdyC3mOlXBgc3IuksvL8yVPqmew1xWKcORb+wuJi99k\njNCPXYR0irA+UGaVCxqmLyOe72lVeBIEOVBnoLRRdkrP06uGyJWmjWdR4ZCnHKp0\nw43UicNhp6d7rwz5lWtxbQowIzwEKXaXfLMhTSHyr4i3nAPOUz6MTmltkQKBgB6z\nePtoFDfaIZnC0jsSvs4ZoLace3JUtDIJF1M34bmaIub188uZkvfpO0EGKYYihpP7\n7SxupuxiaLMTJPAjwMh6lUGHf0vJ4zLRLeiR04Llj9yN3rNyi7dpO49AddgSPM2W\nvwEVtnPm/n3GEjMEAIiXsnhml5azBO4XghZ9xPLJAoGBALctm1sK8MdawZ+cnc1i\n4P3VP2/nzGeODF29MbJefYrg0hlKHZSfKsWMKg3Dk9jDUplwsVjK5oBgN1vg/zOV\nysTtyn1q/RBbe96lYkPHzdYPWDD5Rg80/t0n6jItTOQr6QCshDLrMB3bruIQz7V9\n6PPhzvdQu3v3e07wrKDa1F3t\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQC8xoQtwqtdBddl\nqOIVdNjy8VURRZOWTKXcy0T19BR+RgJZ6K54WWkhWPcWOLnCwmDYdquhOboLowMX\n5KHLXRoMYnEkZLAA8G9MrwhijNxP4NfUVSzbNvypqtdYJ+SZy9wp2eo1FssuvgSy\nglj05VwH2xKO4zyaXpBLxaPUIZZf4Y/3y57b4BCgbKIeMBdsMp97Q6Sf02szGxjN\npK0zSKOYsCvIInQXcdjxZCFV4TO8f3Rfpaaim1gv2+3HweU2LoYmrcb+uACFbnzt\n/UrGoNmyP069+ghSyF0xE4a9P+x62DoV4nGv7ACIfqbo4Z2rV1qKH/jiTSlYU3kl\n8J7ZGEAnAgMBAAECgf9Now4nMXr/fdU8+hNvCMPnuMbV5ewWCN2bzEa04K1D09BI\nTmm78MCVGwGoRoeNJBr5fdTPMMoJ/yVrG+W34iSvzqgnT4rJ/KqlA6CTwsiPyFay\nRgxRQHCpVuLwp8ClyQ0wu26XQlrgJ480trAoUQdj6pC3V+ICdk90R/j0RW5JtsSu\ne0ML3jNA9C4OgKlt2ia/MLqriaHXOf30EPONvtyqyKeGUFL7Un4eYKh4euRFEEMb\nMKngNonefDCIdYA1wVFa3wT8bNBbpuHl3ghkokv6VpdHIVn9wC1l6HY5nPRjgmo7\nsguRI1bRa2TFkOIVwZjCJTyfANyQw14pRS6rxIkCgYEAwzSYHRpJlPHAD7wi3tls\nbw7cBF9Q1P9PYKmVD9fAjx6eOjzDVOCdpGDijEkYoQoX1yYK3JaS8Vvp8V1wZ5Uh\nHTTr6Y5uS6CPh37wGTJc9XhXdJpeN67fEOBZGU04FUlASVFeCiV3Ga6YX0HQ/yKd\nVSc2JMX9mzxZjwhKRHmCEr0CgYEA95FFAxPxPNzYU3yHIdBlQWB1Z6AUFn+D4FgF\nxeFOGmul1E+0PnPH78IlYanMjhhJ1nkc6X71rdX4ylonB/x6ldUpghWW9VWrqRGG\n76S010aaZgOinwVE7+eeoelsIuma2W0QDwWrUT+RAsJBvZpGx1keo1qZEAaocs9V\nR2lvHrMCgYBNMTMl7wtB9wd4MXGoploW4M1ofTi9wehl1Sm5BhyDfBwd84FawygT\npKxxxUYUCKW80rJg4Lpi73HnnIeirnpVzmOsDELZbTjU4AGaNSxFdb0/wvuXEXPs\nfIs/UiXnZPwjAiYp5P7gDQb8RE6dVdbZoZPrns/W31qbETAtO8+QEQKBgQDgA710\nyYjSz+uXr+j/OflFrSjPedRzfzMvv7aJlhP8aEgH049/q3jRhNYah3Enaub1gWYe\nCtn4UNPtFqKW4WlzRw1mPm741Gqec9Or6VgSLDrt8IAocLYud2HdlMBa3xNVhxCu\n5yxcOq7W1jxyerVtEUFeA07ZZ4zpRp8eHVOFbQKBgGJGU7xoJWO9P17SUGNfmSEF\n6VIYFX6orA1Fi/kAJiqiFf98T4jnUWnL8LXVckt9FNw6KQqBCB6JuKXBFVkG2Bkr\nf5IIhziTuDVpdLQSf0Z2i59TspgYjiKs4WEN3N0HGtCXfbyPO6Tt08d4icxL5Myt\nW84T6Uof3+QQaqQnGvBE\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/intermediate2/private/intermediate2_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDaX/8d940anprz\nK2iPwQwzBkEAyT7kGuHgcGr1L63f8+mZ7cXXqpMTN/9HqvPFife3rTpH5ZxOn4zi\nQe2kfJ2IMq71ioSfDBigs/6O3CqIavUvnIaS+ntus1p4Z1MLIWwNbIAaDh7uBsTS\n5yTG5XS+Hi4XVSvlnwugWMz+v1M399yViPR3plm0uHyiS7dqZ6qE3Cnx+deJBU0L\n84stUplX7W8Rnq8oo2FEwuxuf589C9z3GW0UiqW4tikCNJC0lsHLp0JGl8+NWf0X\nsaYnp3uKR2/6AyQcEiXuNNZc2kWYIzDhSMma3zeqG3Bssg+VOdZtPiUgqAcsSFcM\nmVLLiQhBAgMBAAECggEABFew29ByrKMXSzsjgOpSmwgmjkSyPLiBIeSyZ85LK58u\n18oH62Y/tvvf1nXCk7zO4YbvGANrpI+dLlmnx2PYAR+a5ZSb1wrXSYjSyNX9fYl8\n9zWqYm1bO4QTCj5pwximzKyJ7pq1yD93tgb1LwRcmjRA7+NYdGBBi66AYxd8aOo6\nQB7JoME+hzYAWB+foCOAPGAxYe7EFCPkPEyz08oxRCvDua0xa0+tWkU77MhUSCu+\n/uSq/Og9C9TfzCX0W91TNDnq8VeXbLDJoPNzgfSWIeYxSw/X5dUkYU8N2LuPLQOO\n84Xv5UqU9YV22TEjg22YAL8/GMZ160K1xzXnQb1LPQKBgQDs/jOBp9NFiFlcNbJ8\nMKdfv+sktQR7onGehOaz/dEFEKiHNO8UmSlkAk+aZoTmYXKgIT4uytKRSOfZUWSl\nkY64sKJ7KTvVq/Dzm4KsyH8VgYYQ3OrNbqSCSK7DiOiKJxQ+Jhm2+a+io16B8ZbM\nRXLoaQ5+8oET6BgM5R6IMe4iFQKBgQDr44q7I7nhQdNz4h7OJ4HZ8X8WmQG7YpSX\nEMLb5sX5wymfPi5uUDTcthUw7dpleO9js5iT93fB6+rd5yyiDPIes/dWjqULprvR\nzIIr0u+cyt1TRxrNSa6dz/dJO3t/g/fTPKeM9j7ON4RvEGW4LPA+PbEUU0Q6xfSq\nOZ0sZSXUfQKBgQDh8+r/rxbLsJgiRkAKEAlETSLQOJYxmkthq6yZ52ElxyAm6N0Z\ncn34EAv9VclYLYiwC4HR8yaXxj7m/6dKBGFizWXcrw+RRQHSAW6xdedUhc1gvoBP\npTHL1ahqXVn4fhHav1C9F4nRMpmkosX3tC8+Twu3FVbjt+FWSgy2JYS5kQKBgD5B\n6u6jaj7Skc2HA5xjfvkXrPQ44+UiCpeoW9WQHfZilQyra7O/xYPvJr6oODkJ5xzI\nXN/Is7nh2zY/+l62zfxegUw+D794fR/NOxn37TfTrwB4xtEhvk12gwy3/0tTeEgv\nPQWORFtG+dQaXs5yReIXhDIaG+rrLjzzQdFizM49AoGBAOulUGVDBpUFDVJn8S5r\nbqss/PXW+5xj5g8b9/tzBuyfL0NJ9p3q6EWlELPRTX3zXuVRYjSe9cBUm5FXPxP2\ns1TsGUILjSw21dOtodahvXRDN3Uw2ALQy1MTDy8xLhr9Le+e6xF1T2muzg0vDT6L\nVXAYfY5NPUOiPaYAj792oZk/\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:33:37 2023 GMT\n Not After : Apr 28 19:33:37 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51:\n 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c:\n e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5:\n 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2:\n 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1:\n 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5:\n 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2:\n 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13:\n 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25:\n c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd:\n 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07:\n 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8:\n e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26:\n b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1:\n 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5:\n 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5:\n fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76:\n 82:5f\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6:\n ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec:\n dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf:\n 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d:\n f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df:\n 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72:\n 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50:\n 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3:\n b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce:\n 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06:\n b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa:\n d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40:\n 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9:\n cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c:\n 96:a5:e5:d9\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT\npV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga\nOURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS\nZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB\nNRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd\nA29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm\nA683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga\njp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ\n57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf\nhnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi\nDTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N\nvk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ\ny/ax19PH5nUVAP/H2VRjJxlclqXl2Q==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:28:39 2023 GMT\n Not After : Apr 28 19:28:39 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9:\n 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1:\n 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d:\n 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79:\n 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45:\n f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30:\n 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17:\n f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94:\n f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91:\n 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6:\n 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6:\n 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef:\n 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4:\n a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56:\n fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08:\n 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3:\n 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d:\n 38:21\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n X509v3 Key Usage: critical\n Digital Signature\n X509v3 Extended Key Usage: critical\n OCSP Signing\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32:\n 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d:\n bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3:\n 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60:\n c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b:\n 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24:\n 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3:\n a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb:\n ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc:\n 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b:\n 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f:\n e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21:\n 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9:\n 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e:\n cf:28:5c:69\n-----BEGIN CERTIFICATE-----\nMIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX\nMBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp\nPEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj\nMCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8\nQ+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd\n/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux\nVt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte\nUGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY\nGo6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG\nCCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4\nODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF\nBQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB\nAEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf\nc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S\n9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ\nu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3\nadZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9\nnOaFOvjJTDP96aqIjs8oXGk=\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 29:e1:52:8d:fd:a5:2a:87:eb:1d:e4:1d:47:6c:e1:8a:58:69:73:ab\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:28:39 2023 GMT\n Not After : Apr 28 19:28:39 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:a3:0c:ca:eb:80:eb:a1:0e:1e:71:9b:d3:b3:f9:\n 65:ce:70:c2:21:06:3c:31:c1:06:7e:a5:a8:4a:e1:\n 21:a3:74:54:9f:57:ce:50:d6:c3:29:3c:43:b0:9d:\n 3e:54:94:ee:8d:fa:0d:71:6c:df:5e:9e:01:30:79:\n 6c:bb:97:5d:af:bb:5b:05:77:72:9f:55:e6:66:45:\n f4:e2:c2:cf:7b:0e:58:d6:14:6a:76:29:ac:e3:30:\n 28:0d:ee:bd:ca:aa:ae:1f:1e:ef:40:f3:c3:ab:17:\n f2:d7:ec:0d:e1:fb:68:9a:09:83:99:11:58:42:94:\n f8:0d:d4:9a:6f:9f:3b:e8:56:f0:a9:b7:18:1a:91:\n 41:7c:43:e3:db:b1:01:f1:ad:0b:39:d7:65:98:e6:\n 15:b0:17:a9:56:6e:fb:84:7a:c0:cc:67:75:fc:f6:\n 75:84:31:78:c5:6d:51:8f:d0:19:d3:16:4f:87:ef:\n 5b:33:b9:7a:dd:fe:5f:a8:6a:fd:44:54:00:f3:a4:\n a6:5b:fd:3b:65:38:4f:82:4f:b9:c4:bd:c9:9a:56:\n fc:54:f1:58:2f:cb:ee:f4:08:fd:b7:ec:ad:28:08:\n 66:9b:f8:78:98:32:db:b1:56:dd:0e:31:ba:c6:e3:\n 56:f5:02:2f:fb:76:28:bb:c4:8b:f3:6b:da:aa:1d:\n 38:21\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n CB:5E:50:60:7A:AB:2F:A9:3B:1E:24:AB:02:42:8D:EC:81:60:48:13\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n X509v3 Key Usage: critical\n Digital Signature\n X509v3 Extended Key Usage: critical\n OCSP Signing\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 48:65:ce:6d:91:46:30:37:b6:f2:76:c0:42:e3:f5:ee:e9:32:\n 0e:46:b5:d5:9d:ac:b0:f2:23:f5:35:a8:1c:61:66:81:c0:0d:\n bc:a4:bb:b5:be:47:58:8b:f1:d1:5f:73:83:d2:99:da:3e:a3:\n 0b:32:81:96:a4:bd:a8:57:8e:fe:3d:c4:93:57:ef:05:77:60:\n c9:88:1c:2e:25:7e:ea:c8:95:8d:a6:4a:73:e5:bb:6c:c4:3b:\n 01:03:90:8d:12:f5:69:13:c5:79:87:ae:45:cb:49:c8:90:24:\n 39:30:cf:27:ba:31:1e:5f:5b:e0:0f:93:82:66:28:33:dc:e3:\n a1:a8:fc:ad:40:d0:48:31:63:fb:a0:6a:13:18:b1:8b:59:bb:\n ef:96:f8:83:98:6c:4a:18:37:1a:02:ad:c2:42:1d:7e:1c:dc:\n 4a:77:b7:f5:ae:97:3e:17:e8:35:96:85:a0:e4:30:c5:03:0b:\n 62:55:13:c1:3f:df:15:1b:c3:45:f7:69:d6:5e:f5:77:fc:4f:\n e8:28:3b:3e:f0:2c:20:22:81:72:a3:d6:1b:d1:52:63:86:21:\n 22:06:7a:5b:f4:2a:c7:e5:b9:97:ac:1b:56:b5:4c:62:e9:f9:\n 6f:49:5f:43:3d:9c:e6:85:3a:f8:c9:4c:33:fd:e9:aa:88:8e:\n cf:28:5c:69\n-----BEGIN CERTIFICATE-----\nMIIELTCCAxWgAwIBAgIUKeFSjf2lKofrHeQdR2zhilhpc6swDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTI4MzlaFw0zMzA0MjgxOTI4MzlaMFcxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEX\nMBUGA1UEAwwOT0NTUCBSZXNwb25kZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw\nggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMp\nPEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2Kazj\nMCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8\nQ+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd\n/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtux\nVt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMte\nUGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAFLWRbk9ktxaEdvm0vpnOYJWY\nGo6dMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoG\nCCsGAQUFBwMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4\nODgvaW50ZXJtZWRpYXRlMV9jcmwuZGVyMDMGCCsGAQUFBwEBBCcwJTAjBggrBgEF\nBQcwAYYXaHR0cDovLzEyNy4wLjAuMToxODg4OC8wDQYJKoZIhvcNAQELBQADggEB\nAEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFf\nc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S\n9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZ\nu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3\nadZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9\nnOaFOvjJTDP96aqIjs8oXGk=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjDMrrgOuhDh5x\nm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEw\neWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX\n7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lW\nbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+C\nT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvz\na9qqHTghAgMBAAECggEAE++sPxPuG6zzhX4hakvYCiAo6GtQAGBi6CjetTsmRwti\nDnKoyCMeTUQwXZ+4X5SvP35f1urSPAozSIdMR3qoSqSsqjQy+G8DIyWyHejmgBwe\nuhxYcRbC7Ct29k8m9ykb7bO1WtqDZf/hYkvbXbKFFXKM2/IuOcPnuZ8xe+z7IPsQ\nODHnrQs45wQyi2i2/+AbvEJjb3bb3oS8MfoZfvO8F06ejTOmv/ATZSxX0T6ppCPj\nHdmKqKDXlYQNA/LQeM4cs2FaQH170R1vGHppDjcs2ezqElB7/HKfKWeEn0Eytu9E\neWw9tZteisnzfqEvDMgOM2eWwAzfIhXSQYMWlVBicQKBgQC6MPaLd4r82BBMj7qx\nChdBxB7LXptvx/q3SrMjZ6GKmrGdXMbsos50XexajktBqkXfUMa8hGqmlciN5xL1\n+w//p7oSzb3VorOyHVXZpc8p79eUeX8ONcwySOYwO+CpqFBBDlvPn1OuPnlUL1pv\nIgCMT66flWJxRklDMIJsHr+iWQKBgQDgLq3I2cj4q+3121ECPXKLt+VCHUY0aygc\ntl6lvQw61UnmyLQ+k53/MmyPGGCxIFr18DsoKeWYwt3kWTW0MCDrQuO6PZkB268v\ngdsmN3nhAKiR0gUwJDrFjpPWr0GAhw9LE7HqpvkQ3fG5YSnXTUibhm6smHg7dzVL\nER+QJ+Y7CQKBgHIDN4WRjy9jEx/+x0BPwIwKDx1TcnURjQoeGPHuLHJWZbrJrBoN\nW8TQGsIc7iJopN6pdPjNUQ1vHN8gB3FO6q4PRBbtm3gtaEICSqa7LM8uSeFmQJIw\nCTklgKc6k0jwgyxDIZ9SnghNwzf0wzjYJmPFC1Y3QI/CjWwyUTrp3UkJAoGBANHc\nIKcS6MWQ/RPYGP+F0kLlBWJc0Smk3knylquES3yPybyXSdQCkDcjVuilo25soXn1\nRwuUHPBiCyIGOPXS0B4r4c6odyF8K4THhQVDjX6KBUNsXZrxb2scy1x/d0wAItrf\nNwA5CpM1kWE+idKY8E1XDSfZG0Rfla4N+4QRNb8xAoGAQrVe80TpPpzDH846xaPF\nBAhjKz7dRrUQ1n7ZI6yw8I5bU2ky9jSF3PRsDismM23aNbYhIgjcYv0tG0LMlWLV\n2eIrU9OoA7aepDPozyhEkENaWqYXX/T8AjD+Kaw7XJnt/NX8eS0RF2qgDA/HEwWw\nuf1ecRqpjZ9cxNGLZ+/pOkM=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:29:28 2023 GMT\n Not After : Apr 28 19:29:28 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2:\n 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33:\n 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9:\n 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95:\n ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44:\n 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5:\n c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2:\n c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0:\n 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e:\n 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5:\n 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7:\n d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0:\n af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5:\n 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a:\n 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd:\n 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69:\n be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be:\n 7e:89\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n X509v3 Key Usage: critical\n Digital Signature\n X509v3 Extended Key Usage: critical\n OCSP Signing\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33:\n 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76:\n fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd:\n c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05:\n 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba:\n 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78:\n e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d:\n 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c:\n 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5:\n a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad:\n 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09:\n f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52:\n 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf:\n ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77:\n 4d:ef:b5:43\n-----BEGIN CERTIFICATE-----\nMIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ\nMBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN\n06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B\nf4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO\nbEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3\nzlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l\n56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU\n5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi\nV3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww\nCgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy\nODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG\nAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC\nAQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN\n/ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7\nlpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv\nsjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0\n8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8\nsnFsePS65Lm3HxFl1E93Te+1Qw==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 7b:97:35:73:2b:2b:5f:74:c6:43:83:8f:ae:65:5b:a0:f5:f4:ff:1f\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: May 1 19:29:28 2023 GMT\n Not After : Apr 28 19:29:28 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=OCSP Responder 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:b8:98:3d:03:4d:5e:b2:66:5e:51:3b:f9:3d:f2:\n 7a:24:6b:70:5c:2f:7a:05:b2:51:77:62:45:e7:33:\n 75:77:db:31:6f:2d:13:32:cd:d3:a0:03:84:ee:f9:\n 2b:81:9d:e5:c9:ba:e2:25:c9:a7:18:2b:fd:f1:95:\n ad:d3:46:90:d9:7b:7f:39:2d:85:b4:70:7c:72:44:\n 99:fb:df:9f:22:4c:81:77:35:bb:fe:41:7f:86:f5:\n c7:29:53:7c:ee:d4:cc:09:54:fa:cc:b1:4d:4b:c2:\n c7:c7:3e:1a:13:59:66:36:31:ae:60:1b:6a:05:b0:\n 5b:64:96:77:9d:74:cc:42:6e:13:d1:21:83:94:8e:\n 6c:4c:d8:42:57:94:17:ff:26:d4:d1:2f:64:58:b5:\n 47:1a:22:38:69:bf:c0:5a:9c:c3:88:01:0a:1d:f7:\n d8:68:88:7c:57:5d:44:c4:71:d0:66:8d:1c:39:e0:\n af:e8:f7:ce:51:60:7c:1d:b7:d5:e7:b5:3e:6a:a5:\n 2b:46:c3:4e:b9:ef:de:bd:a6:be:e2:66:79:a9:6a:\n 0d:c1:b2:e7:5e:03:9d:de:dd:41:b9:c9:80:2c:bd:\n 6d:1f:09:5f:4e:25:e7:ac:ff:23:47:8f:5f:74:69:\n be:81:42:5c:e6:1a:f7:65:1f:eb:a1:d0:69:6f:be:\n 7e:89\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n E4:4D:EE:6A:A3:30:91:37:3E:5C:1D:BD:26:96:5F:FF:DB:D3:E2:15\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n X509v3 Key Usage: critical\n Digital Signature\n X509v3 Extended Key Usage: critical\n OCSP Signing\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 6c:d6:fa:8f:6f:c9:0a:99:0b:ee:6c:27:1f:75:52:b8:82:33:\n 41:fe:01:a1:f8:c5:24:4e:9e:3b:e2:89:0f:01:2b:8e:c4:76:\n fb:d9:75:5a:b2:9c:e0:36:8d:fd:90:9f:28:92:1b:a3:74:fd:\n c5:39:28:51:06:ab:95:f7:64:95:e8:7b:d9:97:35:33:97:05:\n 38:87:e6:e6:d7:a5:0b:a1:11:0c:b7:8b:76:b8:a9:46:33:ba:\n 50:b3:3b:96:90:65:4b:ea:14:20:c9:f7:0d:8d:5e:89:c6:78:\n e3:0b:4f:d2:db:10:46:8a:c4:81:6f:20:13:30:83:a8:45:4d:\n 2b:ef:f0:ce:18:a7:96:fc:b9:67:79:e9:a9:f0:2f:b2:33:1c:\n 83:cf:a3:4b:df:fd:c5:58:ae:87:83:d9:be:22:85:58:41:f5:\n a0:a2:2d:56:98:40:12:78:c5:43:b0:50:34:0f:6c:0b:52:ad:\n 68:e1:7a:9e:c1:54:58:bf:b4:f1:c5:3b:bf:97:e4:f9:44:09:\n f5:c7:67:7d:dc:3d:ea:a9:9f:0f:3a:aa:9c:4a:c1:ef:a1:52:\n 25:e4:57:22:d6:af:c6:c9:c8:02:91:4b:ec:a2:d6:ba:b5:bf:\n ed:22:7c:b2:71:6c:78:f4:ba:e4:b9:b7:1f:11:65:d4:4f:77:\n 4d:ef:b5:43\n-----BEGIN CERTIFICATE-----\nMIIELzCCAxegAwIBAgIUe5c1cysrX3TGQ4OPrmVboPX0/x8wDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA1MDExOTI5MjhaFw0zMzA0MjgxOTI5MjhaMFkxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEZ\nMBcGA1UEAwwQT0NTUCBSZXNwb25kZXIgMjCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBALiYPQNNXrJmXlE7+T3yeiRrcFwvegWyUXdiReczdXfbMW8tEzLN\n06ADhO75K4Gd5cm64iXJpxgr/fGVrdNGkNl7fzkthbRwfHJEmfvfnyJMgXc1u/5B\nf4b1xylTfO7UzAlU+syxTUvCx8c+GhNZZjYxrmAbagWwW2SWd510zEJuE9Ehg5SO\nbEzYQleUF/8m1NEvZFi1RxoiOGm/wFqcw4gBCh332GiIfFddRMRx0GaNHDngr+j3\nzlFgfB231ee1PmqlK0bDTrnv3r2mvuJmealqDcGy514Dnd7dQbnJgCy9bR8JX04l\n56z/I0ePX3RpvoFCXOYa92Uf66HQaW++fokCAwEAAaOB7TCB6jAdBgNVHQ4EFgQU\n5E3uaqMwkTc+XB29JpZf/9vT4hUwHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyi\nV3ftFawwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAww\nCgYIKwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovLzEyNy4wLjAuMToy\nODg4OC9pbnRlcm1lZGlhdGUyX2NybC5kZXIwMwYIKwYBBQUHAQEEJzAlMCMGCCsG\nAQUFBzABhhdodHRwOi8vMTI3LjAuMC4xOjI4ODg4LzANBgkqhkiG9w0BAQsFAAOC\nAQEAbNb6j2/JCpkL7mwnH3VSuIIzQf4BofjFJE6eO+KJDwErjsR2+9l1WrKc4DaN\n/ZCfKJIbo3T9xTkoUQarlfdkleh72Zc1M5cFOIfm5telC6ERDLeLdripRjO6ULM7\nlpBlS+oUIMn3DY1eicZ44wtP0tsQRorEgW8gEzCDqEVNK+/wzhinlvy5Z3npqfAv\nsjMcg8+jS9/9xViuh4PZviKFWEH1oKItVphAEnjFQ7BQNA9sC1KtaOF6nsFUWL+0\n8cU7v5fk+UQJ9cdnfdw96qmfDzqqnErB76FSJeRXItavxsnIApFL7KLWurW/7SJ8\nsnFsePS65Lm3HxFl1E93Te+1Qw==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4mD0DTV6yZl5R\nO/k98noka3BcL3oFslF3YkXnM3V32zFvLRMyzdOgA4Tu+SuBneXJuuIlyacYK/3x\nla3TRpDZe385LYW0cHxyRJn7358iTIF3Nbv+QX+G9ccpU3zu1MwJVPrMsU1LwsfH\nPhoTWWY2Ma5gG2oFsFtklneddMxCbhPRIYOUjmxM2EJXlBf/JtTRL2RYtUcaIjhp\nv8BanMOIAQod99hoiHxXXUTEcdBmjRw54K/o985RYHwdt9XntT5qpStGw0657969\npr7iZnmpag3BsudeA53e3UG5yYAsvW0fCV9OJees/yNHj190ab6BQlzmGvdlH+uh\n0Glvvn6JAgMBAAECggEAIlCyruV4ICPljqZefASSbijG12w/+8UdXdsX8ZXgVWqa\n8vbnJb+bgpiE4sPRMaQ/rlOebLXi6RxsdbeEe80XakaJ7QAoZdWvXLKiCW+VrpOY\nUafcjbRxV45i+qy5gdBvKaDxipG/M8E+0CwcPtKUrKhpqRYPjIUvSDCshcnLmuF3\nzztB/4VyVEUUaM0pEqSZhxSyraRmGARvF1iOSu1npe3AzWTrrjrSkbk6fi4GyECL\nIf0EQ1ZD+ZXQ6tcGDyNtmPox7lPMZOgwLJZ5zISXZ6QBjn0JvSzE+e4z0IFinLgx\nq5yBz2BhJEN8OBcs3J2N/ivQetWil64YbrbK6WbocQKBgQD/b4uHOuJVVifjIf6/\nkJ0UHhki4Q2Fj164royDigyyzaZmMzrlReZ5rAQLk8wGqw2hI+9gYoYBYqHm71kd\nWrwLS1TVZJ6x8TBh0sYOG2CPndqIjWFx9Wjjf1xNknwYdIoEdAAKZ/M1E71V0tZb\n+Ampl+lHPnKqYRSCd7gbYBU/TQKBgQC5AKGJusjRRRRWQqQ0RdJuxusZrtAAUd7l\nwOGMC0zVQSSvUuegFtWEaZUbByhCARtYp8o4rT6Fw9yOvMaMNcfd8tV5nYVHDsrw\nMurPhPitgI0/LdVvkAOO4fgPZHIXV9GbUDGq4uqB61daBSLQg1JjtzG8GvlGiYZl\nmKOWEXjWLQKBgQC3nHHaehxZpT20yin5f7U50czVwppaqE05Sdcdcq1gFe2Hx0mN\npypdyaV6wPnGzUxVyaP3T7rt4f1pKCGRtTg4kiTf450jYbEakEzntQw7EAgXYjFq\nnjKQXWt3I1XqqlLPkqa41DIBtDfEKnMF1wzzCIyaNqxsBq6cffwsSWvcfQKBgF/y\nUNUCd0X5Yqu+EjU+BP4I0kNWo2+XBlf36cHc1nM/Psxi3dfsH751l6wV0S4yLsGS\n+9DbILL1On0YsIxlFAwq9cYGCOoqZNugPKF1oBcztY2PssMSWJYQ4brx6C3tELtR\nIwEygFby/DGmukCT6vXmO7gH8UJA7t/gAu9Ajn/dAoGAI/Ejqb7HborIvCw/p+kB\nJkPIhTUuT5XonDm8h6KHWUESPikS7SMeRM/4V+AL/Y5MiiCBfjh3tCOup/16x6GQ\n4z6FvcIaYusxKup+afQaDyv1Phv5/mr74liLhC5Qp9EGU2FZrMZwG3EZjSn/0IE+\ndBJeWNtNHiFPcyTzYMMhDBw=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/root/private/root_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDiIWuf70i53iL7\nWzcJaMe1kldSJO+FAOhxhU0PW4zG508Z9uMLcKNBfnHUD9b98hrKqleRdpqygmJg\nzvIALtS8WNNgMEKmKLJQe1gBn/sKZbBA1nzit9qNGdmlUdJGfhRGq/rfzv6ECJhj\nRh1NindXZ9oWizIMfEHipezufSAo6wNf9eYF2IuWeG+uKZpQ99yWMYaBsXjo6+9d\nu+1C7JTGVEbsBW8bDDYkxqgGflxWuEM7EfQGCgUVGTsfyGcx6ztbKhUKe/lr5BDu\nRL4Z2NtEAfo6VvVsTvNgquTNsq13B0Xv8df1+lKEXANOcuCpkcXZ1gqEM5gx8gJb\nPxAVZXbXAgMBAAECggEAW6YC6i+/cIFs+SW3cStT4a29kU/h+axsCPJnUIWg0U6X\nWyUaUR0mNZmrRbDjyEmS/Te7xPtmaFn6yFSndVaFpw5zIQV+RbyxxHexK/tscgLT\nw/uKYxLz04M6GExIpoRb8Gash3/r3JRlOrsEjlRD2RuAoulob/H+e/8Wv3PcEGio\nR8jwCj5DEnWiMxDzgtxsVgR4OeRYqg3zKjWrLALEYoRbFTVncCVA40OnmGJZ3+E5\n+3OOX6p9y/nY36888345yuwiCOTdNwQVaCXnLDZlAIVpB8QmjXVB35RSs+r2H5SF\np/KRbZ/JNKdNrbTKfJyvbnIpyTAtJB9OkhyiR9AegQKBgQDkKAplyZ6ChT3l53nn\n4ngFi/nSTfrfJepmA5lVJk1Wxk0a4W++HxJkdKY2sUP7WuQ1xaPdcHxKzfp2HQE5\nL95jObU5dtY64QD4q0xqOw1ISDQi1euqZEmZziupEgPcMtw4sAVhHohzvTWo6a8o\nfGMSkLTd+2303xgBCZo2I/hZVwKBgQD9uha6pQmCg4Oi2i/38Vm3ByArnQvgkELC\neGBBJrCE8pSm+ToPtgL+gSuA8PlWzIYlIf0UVOJFyws8GkUFm6n0nUlN0NmK8Rhm\nBg4IvasxdRgtySJzZO7ipAqGIaWJIBi1Vj4/rnAVggkadbQgyw+eCZNc5Pg3D9MV\nTJ7d/xHegQKBgQCprGVfITuyUSihKy3rlu4vIdPd5IQnI2lYCGElg+CMIdkBnpmd\nSDpDXsSlc9rcuNFyc9LTQW4Nq3USFavtPX4jSK1PWOMk0mQIiku/zL6p/JhZN8GU\n7BQYP80UZQNd5K0Fs1Gs0ioj+JhJT9AlSavcCKWZV/yD2M1fKCb5EHMG7QKBgQDV\nSvtSeeytp8sgOtU6VMz7fOUBZOsYI43Ll5ArFNAtYxOt7jNuA68urf2ZTnn9Cr/2\nNUVgMx9oVpEiPF8roLlV5mc6IEjQcW72TT69AF0KnYnu63enlADxy78BFQXoaW/7\n+P0pYYXdvsvST4JWUv3U9+3GmMFE4GutKxUeQA+QgQKBgQCauejVixhfKcmkM9nn\nMGLSOUuFyd9HpQk3efxylphFNjpohk+k3fVKXBhmE4BDXbSlYUmMemm27tuQ/I6Z\nbWOjGl57ZbCgJ7LdXLanJhyJJ6cSmkX8+oD+fwPMrD8yaAfh37MdTnriZKIDMXp2\n7HtfLcz0evmbW06b/dReyvcqyQ==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/root/root_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 27:5e:cf:7e:be:aa:02:b9:a9:c7:42:30:43:fe:0e:80:05:91:dd:0b\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 18:57:57 2023 GMT\n Not After : Apr 28 18:57:57 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:e2:21:6b:9f:ef:48:b9:de:22:fb:5b:37:09:68:\n c7:b5:92:57:52:24:ef:85:00:e8:71:85:4d:0f:5b:\n 8c:c6:e7:4f:19:f6:e3:0b:70:a3:41:7e:71:d4:0f:\n d6:fd:f2:1a:ca:aa:57:91:76:9a:b2:82:62:60:ce:\n f2:00:2e:d4:bc:58:d3:60:30:42:a6:28:b2:50:7b:\n 58:01:9f:fb:0a:65:b0:40:d6:7c:e2:b7:da:8d:19:\n d9:a5:51:d2:46:7e:14:46:ab:fa:df:ce:fe:84:08:\n 98:63:46:1d:4d:8a:77:57:67:da:16:8b:32:0c:7c:\n 41:e2:a5:ec:ee:7d:20:28:eb:03:5f:f5:e6:05:d8:\n 8b:96:78:6f:ae:29:9a:50:f7:dc:96:31:86:81:b1:\n 78:e8:eb:ef:5d:bb:ed:42:ec:94:c6:54:46:ec:05:\n 6f:1b:0c:36:24:c6:a8:06:7e:5c:56:b8:43:3b:11:\n f4:06:0a:05:15:19:3b:1f:c8:67:31:eb:3b:5b:2a:\n 15:0a:7b:f9:6b:e4:10:ee:44:be:19:d8:db:44:01:\n fa:3a:56:f5:6c:4e:f3:60:aa:e4:cd:b2:ad:77:07:\n 45:ef:f1:d7:f5:fa:52:84:5c:03:4e:72:e0:a9:91:\n c5:d9:d6:0a:84:33:98:31:f2:02:5b:3f:10:15:65:\n 76:d7\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 22:79:1a:b9:5d:fa:f5:c9:a3:88:22:c4:92:e6:64:6d:ce:a5:\n ae:2e:69:48:6a:9e:d5:11:c5:bb:b0:de:38:1b:5b:04:85:60:\n d6:64:14:ed:c2:62:02:7d:ad:d2:17:ad:ef:40:27:2b:50:59:\n 4a:ff:88:c6:b3:16:5c:55:30:d9:23:bd:4f:0f:34:b7:7b:ed:\n 7a:e1:f3:39:35:e9:18:6d:70:b1:2b:2a:e2:e5:cd:a1:54:8a:\n f9:f4:95:81:29:84:3f:95:2f:48:e0:35:3e:d9:cb:84:4d:3d:\n 3e:3c:0e:8d:24:42:5f:19:e6:06:a5:87:ae:ba:af:07:02:e7:\n 6a:83:0a:89:d4:a4:38:ce:05:6e:f6:15:f1:7a:53:bb:50:28:\n 89:51:3f:f2:54:f1:d3:c4:28:07:a1:3e:55:e5:84:b8:df:58:\n af:c3:e7:81:c2:08:9c:35:e4:c4:86:75:a8:17:99:2c:a6:7f:\n 46:30:9b:23:55:c5:d8:e2:6a:e4:08:a1:8b:dc:bc:5b:86:95:\n 4a:79:fe:a6:93:3d:1a:5b:10:9a:2f:6a:45:2f:5d:c9:fa:95:\n 2e:66:eb:52:df:88:a7:5f:42:8f:5f:46:07:79:8b:a7:49:82:\n d3:81:c6:3e:c2:5a:15:c4:83:69:30:49:4d:6e:ea:05:1e:d8:\n dc:29:ac:17\n-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIUJ17Pfr6qArmpx0IwQ/4OgAWR3QswDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE4\nNTc1N1oXDTMzMDQyODE4NTc1N1owUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdS\nb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4iFrn+9Iud4i\n+1s3CWjHtZJXUiTvhQDocYVND1uMxudPGfbjC3CjQX5x1A/W/fIayqpXkXaasoJi\nYM7yAC7UvFjTYDBCpiiyUHtYAZ/7CmWwQNZ84rfajRnZpVHSRn4URqv6387+hAiY\nY0YdTYp3V2faFosyDHxB4qXs7n0gKOsDX/XmBdiLlnhvrimaUPfcljGGgbF46Ovv\nXbvtQuyUxlRG7AVvGww2JMaoBn5cVrhDOxH0BgoFFRk7H8hnMes7WyoVCnv5a+QQ\n7kS+GdjbRAH6Olb1bE7zYKrkzbKtdwdF7/HX9fpShFwDTnLgqZHF2dYKhDOYMfIC\nWz8QFWV21wIDAQABo4GZMIGWMB0GA1UdDgQWBBTDEkK6qdhN4MM+utdHQaYJL220\n4TAfBgNVHSMEGDAWgBTDEkK6qdhN4MM+utdHQaYJL2204TAPBgNVHRMBAf8EBTAD\nAQH/MA4GA1UdDwEB/wQEAwIBhjAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vMTI3\nLjAuMC4xOjg4ODgvcm9vdF9jcmwuZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQAieRq5\nXfr1yaOIIsSS5mRtzqWuLmlIap7VEcW7sN44G1sEhWDWZBTtwmICfa3SF63vQCcr\nUFlK/4jGsxZcVTDZI71PDzS3e+164fM5NekYbXCxKyri5c2hVIr59JWBKYQ/lS9I\n4DU+2cuETT0+PA6NJEJfGeYGpYeuuq8HAudqgwqJ1KQ4zgVu9hXxelO7UCiJUT/y\nVPHTxCgHoT5V5YS431ivw+eBwgicNeTEhnWoF5kspn9GMJsjVcXY4mrkCKGL3Lxb\nhpVKef6mkz0aWxCaL2pFL13J+pUuZutS34inX0KPX0YHeYunSYLTgcY+wloVxINp\nMElNbuoFHtjcKawX\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:33:37 2023 GMT\n Not After : Apr 28 19:33:37 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51:\n 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c:\n e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5:\n 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2:\n 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1:\n 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5:\n 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2:\n 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13:\n 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25:\n c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd:\n 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07:\n 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8:\n e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26:\n b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1:\n 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5:\n 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5:\n fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76:\n 82:5f\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6:\n ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec:\n dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf:\n 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d:\n f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df:\n 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72:\n 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50:\n 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3:\n b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce:\n 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06:\n b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa:\n d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40:\n 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9:\n cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c:\n 96:a5:e5:d9\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT\npV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga\nOURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS\nZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB\nNRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd\nA29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm\nA683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga\njp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ\n57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf\nhnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi\nDTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N\nvk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ\ny/ax19PH5nUVAP/H2VRjJxlclqXl2Q==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:c4:82:66:f8:5d:a6:b6:c7:66:e1:b2:01:3f:e0:72:fc:72:61:33\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:33:37 2023 GMT\n Not After : Apr 28 19:33:37 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:af:26:5c:50:c0:fa:62:b5:fd:3d:c1:9e:26:51:\n 58:62:04:37:b0:b5:6a:9b:6a:e3:22:3c:cd:ee:3c:\n e7:8b:d3:e2:4c:08:1a:4d:63:c1:81:20:f4:53:a5:\n 5d:2f:d2:71:d8:af:e3:26:95:b4:27:14:46:7f:e2:\n 0a:73:12:a7:0e:ff:99:5a:29:f5:d0:65:96:b1:d1:\n 96:7f:0c:43:b8:71:f2:4b:21:e1:97:6c:1b:01:e5:\n 38:1a:39:44:72:d5:19:20:87:fe:90:4f:3b:97:f2:\n 7d:bd:57:97:4d:9d:56:50:89:5b:79:29:7a:3a:13:\n 97:08:61:c2:0c:a6:02:49:c9:8a:41:ab:8e:9f:25:\n c9:33:18:f8:92:64:58:04:cc:a3:9d:cf:d4:d2:bd:\n 20:ab:8b:9d:55:df:fb:5b:23:ac:95:12:fa:6f:07:\n 93:3f:0e:03:86:c4:9b:25:06:21:9b:03:96:32:b8:\n e0:0f:63:e2:1d:34:d1:41:35:19:09:c1:a0:dc:26:\n b9:c8:66:fa:87:67:22:6e:0c:a6:e7:0f:24:64:b1:\n 4f:84:05:ef:ad:8e:1b:f2:f4:38:87:d3:e3:48:a5:\n 82:e0:66:89:1d:92:9a:59:67:a4:1d:03:6f:4d:a5:\n fb:3b:c0:0b:73:a7:ab:8f:b4:10:25:8e:69:42:76:\n 82:5f\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 43:16:E6:03:AF:37:B2:7B:BD:B3:C8:A2:9C:95:D7:FA:32:F8:9E:6F\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n a3:87:9f:05:e4:38:61:f7:c4:5b:17:13:4b:2c:9d:a2:4d:e6:\n ad:93:54:c5:a3:00:27:0b:5c:45:c5:bd:f8:b6:a7:5a:2a:ec:\n dc:9b:59:8a:c7:59:e7:b9:86:f7:27:be:45:0d:d9:86:76:cf:\n 00:71:ad:aa:cc:73:50:8c:68:63:b0:e2:3a:59:dd:85:fa:0d:\n f0:82:51:05:79:e6:d5:0e:0b:bb:ed:23:65:8f:d0:8b:01:df:\n 86:74:bc:3a:22:90:e4:59:44:91:d5:44:d8:21:4d:4e:10:72:\n 0a:12:2e:4a:20:5f:15:e7:16:0b:6f:76:f3:04:1f:da:44:50:\n 3b:c3:b3:0f:fa:05:cf:6e:64:9c:65:e2:0d:38:28:31:c3:c3:\n b6:66:ef:80:d3:c4:5f:e9:f9:01:e9:ce:e6:99:46:a0:9d:ce:\n 90:63:77:d2:85:21:d7:88:32:55:38:fe:10:07:69:cd:c8:06:\n b7:6f:49:98:bf:cd:be:4f:ab:44:ea:78:af:ab:01:c8:3e:fa:\n d9:54:bc:59:28:db:03:9b:1c:ee:e4:c3:ed:f3:97:30:c6:40:\n 33:76:84:40:b2:b8:4d:b4:ca:a9:2d:d1:4d:17:92:ea:c0:c9:\n cb:f6:b1:d7:d3:c7:e6:75:15:00:ff:c7:d9:54:63:27:19:5c:\n 96:a5:e5:d9\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUPMSCZvhdprbHZuGyAT/gcvxyYTMwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTMzMzdaFw0zMzA0MjgxOTMzMzdaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCvJlxQwPpitf09wZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRT\npV0v0nHYr+MmlbQnFEZ/4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5Tga\nOURy1Rkgh/6QTzuX8n29V5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiS\nZFgEzKOdz9TSvSCri51V3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFB\nNRkJwaDcJrnIZvqHZyJuDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6Qd\nA29Npfs7wAtzp6uPtBAljmlCdoJfAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUQxbm\nA683snu9s8iinJXX+jL4nm8wHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga\njp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAo4efBeQ4YffEWxcTSyydok3mrZNUxaMAJwtcRcW9+LanWirs3JtZisdZ\n57mG9ye+RQ3ZhnbPAHGtqsxzUIxoY7DiOlndhfoN8IJRBXnm1Q4Lu+0jZY/QiwHf\nhnS8OiKQ5FlEkdVE2CFNThByChIuSiBfFecWC2928wQf2kRQO8OzD/oFz25knGXi\nDTgoMcPDtmbvgNPEX+n5AenO5plGoJ3OkGN30oUh14gyVTj+EAdpzcgGt29JmL/N\nvk+rROp4r6sByD762VS8WSjbA5sc7uTD7fOXMMZAM3aEQLK4TbTKqS3RTReS6sDJ\ny/ax19PH5nUVAP/H2VRjJxlclqXl2Q==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:33:53 2023 GMT\n Not After : Apr 28 19:33:53 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e:\n bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb:\n bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9:\n ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66:\n 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6:\n a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4:\n 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31:\n 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff:\n 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5:\n 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3:\n 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e:\n 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18:\n 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55:\n 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86:\n 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f:\n 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51:\n 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4:\n b1:e9\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4:\n 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa:\n ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b:\n 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5:\n 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41:\n 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4:\n 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1:\n d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95:\n df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f:\n b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85:\n aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08:\n 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6:\n f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31:\n 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49:\n fc:f5:61:1c\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3\nqeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4\n71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq\nP5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS\nhQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ\nOaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5\nXEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga\njp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu\nMp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB\nOhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P\ntlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z\ntSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx\neTjjUIregZeAnkZxXJ/l3gxJ/PVhHA==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 55:57:db:45:43:06:ce:52:63:59:b9:5a:26:78:fd:0d:94:68:95:9c\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:15 2023 GMT\n Not After : Apr 28 19:01:15 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:bc:c6:84:2d:c2:ab:5d:05:d7:65:a8:e2:15:74:\n d8:f2:f1:55:11:45:93:96:4c:a5:dc:cb:44:f5:f4:\n 14:7e:46:02:59:e8:ae:78:59:69:21:58:f7:16:38:\n b9:c2:c2:60:d8:76:ab:a1:39:ba:0b:a3:03:17:e4:\n a1:cb:5d:1a:0c:62:71:24:64:b0:00:f0:6f:4c:af:\n 08:62:8c:dc:4f:e0:d7:d4:55:2c:db:36:fc:a9:aa:\n d7:58:27:e4:99:cb:dc:29:d9:ea:35:16:cb:2e:be:\n 04:b2:82:58:f4:e5:5c:07:db:12:8e:e3:3c:9a:5e:\n 90:4b:c5:a3:d4:21:96:5f:e1:8f:f7:cb:9e:db:e0:\n 10:a0:6c:a2:1e:30:17:6c:32:9f:7b:43:a4:9f:d3:\n 6b:33:1b:18:cd:a4:ad:33:48:a3:98:b0:2b:c8:22:\n 74:17:71:d8:f1:64:21:55:e1:33:bc:7f:74:5f:a5:\n a6:a2:9b:58:2f:db:ed:c7:c1:e5:36:2e:86:26:ad:\n c6:fe:b8:00:85:6e:7c:ed:fd:4a:c6:a0:d9:b2:3f:\n 4e:bd:fa:08:52:c8:5d:31:13:86:bd:3f:ec:7a:d8:\n 3a:15:e2:71:af:ec:00:88:7e:a6:e8:e1:9d:ab:57:\n 5a:8a:1f:f8:e2:4d:29:58:53:79:25:f0:9e:d9:18:\n 40:27\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b1:48:16:3b:d7:91:d0:4d:54:09:cb:ab:c7:41:4f:35:12:8b:\n a6:e8:84:11:49:a9:04:91:41:25:7c:02:38:b2:19:a0:e9:2e:\n d5:d6:7a:26:c1:1a:f8:f1:c6:51:92:68:af:c8:6e:5b:df:28:\n 40:b8:99:94:d5:43:7d:e3:68:75:94:26:56:11:21:9e:50:b3:\n 36:7b:f8:5f:33:76:64:71:04:26:2b:bb:2c:83:33:89:ba:74:\n c1:e9:9d:eb:c0:86:4b:4d:6f:f8:4d:55:5a:3d:f6:55:95:33:\n 0f:b8:f0:53:2b:93:a6:da:8d:5c:1a:e8:30:22:55:67:44:6e:\n 17:c4:57:05:0d:ce:fc:61:dd:b1:3c:b0:66:55:f4:42:d0:ce:\n 94:7d:6a:82:bd:32:ed:2f:21:ff:c7:70:ff:48:9d:10:4a:71:\n be:a8:37:e5:0f:f4:79:1e:7d:a2:f1:6a:6b:2c:e8:03:20:ce:\n 80:94:d2:38:80:bc:7e:56:c5:77:62:94:c0:b7:40:11:4d:ba:\n 98:4b:2e:52:03:66:68:36:ab:d1:0f:3e:b5:92:a3:95:9d:a4:\n ea:d3:8a:14:41:6d:86:24:89:aa:d7:29:20:c8:52:d5:bf:8d:\n 3b:09:52:dd:89:8c:2c:85:40:b5:9f:cc:47:63:ca:3a:e0:c9:\n 91:5c:43:a9\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUVVfbRUMGzlJjWblaJnj9DZRolZwwDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDExNVoXDTMzMDQyODE5MDExNVowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nALzGhC3Cq10F12Wo4hV02PLxVRFFk5ZMpdzLRPX0FH5GAlnornhZaSFY9xY4ucLC\nYNh2q6E5ugujAxfkoctdGgxicSRksADwb0yvCGKM3E/g19RVLNs2/Kmq11gn5JnL\n3CnZ6jUWyy6+BLKCWPTlXAfbEo7jPJpekEvFo9Qhll/hj/fLntvgEKBsoh4wF2wy\nn3tDpJ/TazMbGM2krTNIo5iwK8gidBdx2PFkIVXhM7x/dF+lpqKbWC/b7cfB5TYu\nhiatxv64AIVufO39Ssag2bI/Tr36CFLIXTEThr0/7HrYOhXica/sAIh+pujhnatX\nWoof+OJNKVhTeSXwntkYQCcCAwEAAaOB0DCBzTAdBgNVHQ4EFgQUtZFuT2S3FoR2\n+bS+mc5glZgajp0wHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBALFIFjvXkdBNVAnLq8dBTzUSi6bohBFJqQSRQSV8AjiyGaDp\nLtXWeibBGvjxxlGSaK/IblvfKEC4mZTVQ33jaHWUJlYRIZ5QszZ7+F8zdmRxBCYr\nuyyDM4m6dMHpnevAhktNb/hNVVo99lWVMw+48FMrk6bajVwa6DAiVWdEbhfEVwUN\nzvxh3bE8sGZV9ELQzpR9aoK9Mu0vIf/HcP9InRBKcb6oN+UP9HkefaLxamss6AMg\nzoCU0jiAvH5WxXdilMC3QBFNuphLLlIDZmg2q9EPPrWSo5WdpOrTihRBbYYkiarX\nKSDIUtW/jTsJUt2JjCyFQLWfzEdjyjrgyZFcQ6k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 2e:91:da:29:59:ff:c4:64:bf:02:bc:27:bb:e3:35:4e:5b:36:f7:91\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 1\n Validity\n Not Before: May 1 19:33:53 2023 GMT\n Not After : Apr 28 19:33:53 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:ac:48:ce:a7:b2:ad:7a:68:01:55:3f:86:20:7e:\n bb:26:e6:88:f3:ae:04:15:7d:d9:64:98:85:bc:eb:\n bd:d8:0a:c7:26:c4:8e:27:56:8c:a8:9f:51:37:a9:\n ec:8a:dc:af:27:05:0c:f5:c0:19:b1:2c:0d:56:66:\n 7b:7e:b1:8f:ab:34:61:56:37:a8:ab:51:d6:1d:e6:\n a7:56:b2:51:72:57:9b:c5:87:84:6c:ef:e6:18:d4:\n 45:b8:ef:52:72:11:02:81:61:f2:36:63:25:18:31:\n 7f:c7:91:89:c3:b0:73:13:f0:26:1f:a1:4f:8c:ff:\n 94:1c:75:a6:be:38:7d:81:06:33:dd:7b:86:81:c5:\n 1f:d2:5d:f6:ea:3f:9f:ab:fb:e7:97:3c:72:ea:b3:\n 83:ab:49:88:ac:a9:4b:81:db:fa:e3:bf:79:d9:6e:\n 90:bf:8f:68:d8:05:f8:52:ad:98:41:29:e0:2a:18:\n 98:b6:b2:61:78:02:02:52:85:02:e0:63:f4:a0:55:\n 80:c9:66:8b:ac:4f:8b:36:f4:56:8f:cf:bd:67:86:\n 72:92:0b:f9:73:7b:05:cc:3d:91:ed:ed:4f:f0:8f:\n 36:99:e5:51:7f:ee:9e:fb:e5:5c:d0:39:a2:f5:51:\n 06:92:3c:ad:cc:59:9d:0a:81:50:26:30:01:e9:f4:\n b1:e9\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n CD:65:B9:5C:48:35:F7:1E:85:6E:94:50:78:72:BB:3F:F7:BC:22:A6\n X509v3 Authority Key Identifier: \n B5:91:6E:4F:64:B7:16:84:76:F9:B4:BE:99:CE:60:95:98:1A:8E:9D\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:18888/intermediate1_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:18888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 6f:de:f3:92:b2:8b:57:61:7a:b9:06:49:3e:af:e0:1c:3a:d4:\n 42:52:fe:d0:7d:97:8a:9b:d0:6d:b9:f3:e6:8b:2a:40:ce:aa:\n ed:bb:ce:21:e8:ae:32:9d:eb:5a:00:e0:c1:3a:d7:40:74:1b:\n 43:e4:43:f0:61:bf:40:06:75:52:1b:b9:f4:b5:32:55:94:f5:\n 84:98:90:cc:27:92:91:b7:3d:8e:f1:12:bf:37:1a:8a:50:41:\n 3a:14:0c:cf:93:fe:57:97:7b:fe:af:b9:c0:c2:d6:bb:20:e4:\n 0a:6f:12:0b:60:a6:cc:59:46:db:99:db:61:71:d3:a7:f5:a1:\n d0:d6:81:87:57:a3:dd:b6:e1:ab:2f:4f:b6:51:21:ec:a6:95:\n df:d3:ab:e5:a1:67:a3:ba:b1:b9:71:39:a1:3b:db:5e:c5:6f:\n b1:34:27:ae:6d:f6:67:4c:7d:7c:6d:12:37:6f:b5:0b:5a:85:\n aa:5d:fd:03:de:59:b5:20:7a:ea:84:a0:a5:75:60:12:12:08:\n 77:0e:46:d6:fa:57:fa:b1:43:42:54:38:d7:66:67:cd:fc:b6:\n f9:4c:fe:99:71:2b:d5:a6:13:2f:2e:f0:a3:9e:fc:47:03:31:\n 79:38:e3:50:8a:de:81:97:80:9e:46:71:5c:9f:e5:de:0c:49:\n fc:f5:61:1c\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIULpHaKVn/xGS/Arwnu+M1Tls295EwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMTAe\nFw0yMzA1MDExOTMzNTNaFw0zMzA0MjgxOTMzNTNaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCsSM6nsq16aAFVP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3\nqeyK3K8nBQz1wBmxLA1WZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW4\n71JyEQKBYfI2YyUYMX/HkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbq\nP5+r++eXPHLqs4OrSYisqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJS\nhQLgY/SgVYDJZousT4s29FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQ\nOaL1UQaSPK3MWZ0KgVAmMAHp9LHpAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUzWW5\nXEg19x6FbpRQeHK7P/e8IqYwHwYDVR0jBBgwFoAUtZFuT2S3FoR2+bS+mc5glZga\njp0wDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ludGVybWVkaWF0ZTFfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMTg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAb97zkrKLV2F6uQZJPq/gHDrUQlL+0H2XipvQbbnz5osqQM6q7bvOIeiu\nMp3rWgDgwTrXQHQbQ+RD8GG/QAZ1Uhu59LUyVZT1hJiQzCeSkbc9jvESvzcailBB\nOhQMz5P+V5d7/q+5wMLWuyDkCm8SC2CmzFlG25nbYXHTp/Wh0NaBh1ej3bbhqy9P\ntlEh7KaV39Or5aFno7qxuXE5oTvbXsVvsTQnrm32Z0x9fG0SN2+1C1qFql39A95Z\ntSB66oSgpXVgEhIIdw5G1vpX+rFDQlQ412Znzfy2+Uz+mXEr1aYTLy7wo578RwMx\neTjjUIregZeAnkZxXJ/l3gxJ/PVhHA==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCvJlxQwPpitf09\nwZ4mUVhiBDewtWqbauMiPM3uPOeL0+JMCBpNY8GBIPRTpV0v0nHYr+MmlbQnFEZ/\n4gpzEqcO/5laKfXQZZax0ZZ/DEO4cfJLIeGXbBsB5TgaOURy1Rkgh/6QTzuX8n29\nV5dNnVZQiVt5KXo6E5cIYcIMpgJJyYpBq46fJckzGPiSZFgEzKOdz9TSvSCri51V\n3/tbI6yVEvpvB5M/DgOGxJslBiGbA5YyuOAPY+IdNNFBNRkJwaDcJrnIZvqHZyJu\nDKbnDyRksU+EBe+tjhvy9DiH0+NIpYLgZokdkppZZ6QdA29Npfs7wAtzp6uPtBAl\njmlCdoJfAgMBAAECggEAQLRoOEECfwMCehUUKs20XAl41WQ/7QiQvm4+GXwQgjyV\nhkccCGkI7H5TJK+bfHY/LrDTtsZpVmKMJORJvfcvFkBg08lakVFmWWy3L1pFjlcy\nDoWGxJzgYVPf5PgxDEcjUDxNU9yhhGHGB/Pa5oZwg7Iqw9kJ2XixPBx5RpjxkXYw\ntR8V3IaKq0YRI5lpUfuaofmJnHJnWCMTmawWMxWuTlzlbDDZTHQs8aTDUnwZ26kD\n6tYB2Tp3aP3zUE8MQZwOEyhRH1WQeS3kcIWh4UnPyA09g0aTb6YK8qacnTL2CixF\nVJpLDtlkQk0TCo06AZkcvWkPTQyFXnVsgkG8rRUlEQKBgQDrTHyf6merJAohUeBV\n5IIfoKHWbGc1DXSdmHtCSN9wFGkhCYtfCZ7YaSLjFF7GOvd6mfHJVnIp3aFONqM7\ndk/MZDsAvogO6lU+zgQc+EcKk+e6zyfsUYghy/R3+QKsYtd4SyNDq6cl80MUujjG\npE2b41O57sNCVZgywCCGXvt/ZwKBgQC+jyufgKRIptM+OOhHlKUaxkTDaMHA1KKY\niFPLuLgWmyCYHQq2D6uoCRGnEguEnXtbtOz6SYlMMNfeHtX0SATkdCGae/bh5ibG\nuQoWwRMkRkAgl1gyAh7h669pDUiD2gh0q56cS8El7Jgze7NRF4hUyY2mWc5nGhVR\n7rHKlOCiSQKBgHBiWevvg5BkaEo91w5vVA9TI7lMkYbvZFGZcNXaBI590TCsZFsC\nN1JZ9QXMxu+bXnS6bpehqGmCp/a5dgGCot6WyO+0ETw+hHS45ZIIq7XLqxS4uPLQ\nhlrOFXfwAWzg0NVt3ewGYpFnvRR7VX7bHw5j56uY9L4ML+OdjGthlnHlAoGAZAm7\nR/f7xtw1h7POVU22w3CUxtUm6jl2xobDHu7xTYTQvqp4Zg2h+wwPxVqWy171VLaN\ntfOG7YWyvbwIbD6mutwwi+5KNFtjve2EW1+u0dtDbRimx1IPrmDRbF/50qZSzBUQ\nplKqqmMjn9tvzsGA46oP/+WjksLBsIqTsZsotmkCgYAn8Ap+e6ZNX2uM8Kg7LB+T\nhBNGczNOGQX8SpfCeH9eV4VzfpEHn8Fxk+lcI2WpYkandQ8ju2s0mT5OoQ2VjxGT\neql9jMd8MQZTx/aWridt5qG3hsFcx9GILlcXTUqyRH0SFAU7xDO5HzzKP3tiW6BN\nYE3GakolPPymOR9q69sT0Q==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsSM6nsq16aAFV\nP4Ygfrsm5ojzrgQVfdlkmIW8673YCscmxI4nVoyon1E3qeyK3K8nBQz1wBmxLA1W\nZnt+sY+rNGFWN6irUdYd5qdWslFyV5vFh4Rs7+YY1EW471JyEQKBYfI2YyUYMX/H\nkYnDsHMT8CYfoU+M/5Qcdaa+OH2BBjPde4aBxR/SXfbqP5+r++eXPHLqs4OrSYis\nqUuB2/rjv3nZbpC/j2jYBfhSrZhBKeAqGJi2smF4AgJShQLgY/SgVYDJZousT4s2\n9FaPz71nhnKSC/lzewXMPZHt7U/wjzaZ5VF/7p775VzQOaL1UQaSPK3MWZ0KgVAm\nMAHp9LHpAgMBAAECggEABbGGbngQX9Un+x8FQesSPgHnGM92wtY6J5Gwn2qhJy6M\nVYwwFZ3Nz5pBPbrOY9SRGhPihrdixKOWgWppA8ne0WB4JC26HnGZnFAbAQRVqPbQ\nduhd4ILpOpzpkh1K6b+vvU0addXpsUlHJjYZmdy+9tPBkhtwz1xDCFGShrguR0Pa\nWTudsee4skdGfw6wMyHEfM4IXXuSfb1hIse1xlnZMPXMMi3ebCqpOy4IzJ4ML7sF\nRySdrdAHcWJqOQjPkDTOPCXpthBn3iQ8Fa7Znd0GGLZvdRbq3p10H5LNhMg+LBc7\noRVQ67qAfQKPHKQMSsR4x2fWo8/hw/QEi3cj6CohYQKBgQDtjDBm7VfbLojZeCMx\n+32EZ0bLUTob5qInTKpDbdKcYmxP857LRAglaGu+pkOTnHi6lOjJYSiBDd1+vWm/\n1lgMUjKerI0l5ol5yRHWNDFyeQoh10TqEUbIUqB8E5Vi4gl0DlpnsfEm899rlfhP\ndmi1rNpc/C7ZK8Zpt7l4eLbqYQKBgQC5qs+K01WwjtrjoqkEwKqjy7ASrbBuZ56u\nwOe+iO7pYVP4/VdAvOsfEYCWfjhoETYGKob9ZZlo3StpQ5Ku5CigpWQVSCvJhO2T\nKQe75DfXXxaqoPmlNcqAFpqY383Sm+1r3a815sg83XhQAu7GdCyTrLocBLM9SFWX\nfVbojv/EiQKBgBlOpCFzC7cYIBA7ElTS3C5s6kfi4XPzgDb7nfANFTD/81YZOEOj\nfdKuazwmbnCdbOdD0gESTsRg+8Xy2/9KEJtPboElFOyCwQauey385X+ykXfFfVwK\ndyYEV4CgfXvJZQRuOwdtF6n0tUq68XdVwBYK0kCxxTPxy/ObVTEWezZBAoGAPPX2\nevB0vCnLeN5ZfHP+ExW31AovWbCwC1TPQmIXf40zUxdrZJgi4uqOO9tpjdHI2TFx\nbRXEzwd/T2qeaMoFBOoI+Gvf5KS+lIjuPyTpqM9R0+hSz4nf2TqSvAsPu0zzIW2C\nL8J8kG9vJ2YvG/3c/QfDe5uXdlGfuMOwm18IX3ECgYAelsVWNSm9YR2H7S6dlgjm\n8U1IZO342Ab5qlrIWn9Vr/X9MRsAASlVVESeNeQTToBoub5pakUrpz9nNQy29+TX\nxYju72RsCjKywKXWZrCAdHfY+wJJWVo5XkdDZJVl2AYrnP3C07S9aKIjhpGHwz7n\njbbCEkHZREMbQJCQjuKT1w==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: Aug 2 22:15:27 2023 GMT\n Not After : Jul 30 22:15:27 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76:\n ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b:\n c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2:\n dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a:\n 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab:\n 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f:\n 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78:\n 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed:\n 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2:\n 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35:\n 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca:\n 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c:\n 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e:\n 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d:\n 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71:\n 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b:\n f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7:\n 0b:73\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62:\n 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43:\n 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c:\n 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51:\n 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7:\n 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9:\n 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00:\n ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae:\n 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5:\n 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a:\n 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16:\n cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a:\n 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6:\n df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74:\n 30:be:fb:f7\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y\n8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508\numx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz\nTBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3\nrfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC\n26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM\nnvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft\nFawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv\n9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3\nkP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ\nVKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E\nLv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG\n3xz8gjuuH/clyN/AxdGNUZR0ML779w==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3e:1f:9b:cd:c8:7b:95:f1:64:e6:41:9c:df:6e:03:da:92:9a:90:b7\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: Aug 2 22:15:27 2023 GMT\n Not After : Jul 30 22:15:27 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer3\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:9a:3c:db:76:c9:19:0f:7b:e6:d3:ed:d1:0b:76:\n ae:15:d4:11:1c:66:b8:5d:2a:7d:e3:1f:65:d8:1b:\n c4:63:62:f6:5c:8b:18:66:a8:1c:c2:a6:5e:72:f2:\n dd:57:42:8a:ab:5d:bd:37:b6:f1:4b:51:f0:b3:6a:\n 37:e9:55:78:01:23:ea:53:09:83:2f:7d:59:36:ab:\n 33:4f:4c:bc:ef:a9:1c:db:94:79:4c:0d:4a:7c:3f:\n 9d:3c:ba:6c:76:82:47:25:eb:79:22:f4:09:6c:78:\n 3c:a6:ef:4b:30:90:29:b3:5f:ba:69:b1:1a:95:ed:\n 53:e0:c6:24:78:6e:52:af:8e:bc:db:4a:f0:19:d2:\n 00:5a:a8:b6:73:4c:17:92:d1:8d:81:9b:4c:b8:35:\n 4d:91:dd:df:d3:85:a6:9f:c4:91:19:ec:47:d1:ca:\n 4e:0b:c3:06:8c:27:42:95:83:e3:28:6a:3b:74:9c:\n 68:b0:55:a5:91:91:cb:37:ad:fa:d8:69:8b:de:2e:\n 4a:51:59:32:4b:3d:06:21:04:65:d2:f5:8b:e8:4d:\n 45:96:de:63:97:47:81:85:ea:48:f0:9d:23:2d:71:\n 87:6f:d2:75:3d:45:bf:de:ad:43:82:db:a5:29:9b:\n f9:5e:38:0a:39:a9:38:71:ec:40:40:b5:dc:69:c7:\n 0b:73\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 7F:47:8C:9E:F1:73:7E:34:B9:5B:1E:ED:AD:3A:87:42:80:D4:E3:FD\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n b9:b4:05:48:a6:ba:6c:99:8b:23:c4:9b:b3:8a:32:3f:ca:62:\n 89:81:1e:5d:04:ba:2d:22:a3:0f:5a:5d:a0:ab:40:a4:87:43:\n 26:36:0a:09:64:ef:f5:b0:a7:6f:7a:1f:cc:06:6c:f7:8d:9c:\n 64:5e:c2:ae:e7:45:39:dc:bc:87:06:e6:d5:aa:6b:32:76:51:\n 64:e1:ac:d9:9a:dd:17:47:9b:4e:31:1c:93:f5:c5:ca:d6:b7:\n 90:ff:64:97:59:df:2b:7f:ee:2d:7d:73:ef:95:ad:b5:1e:a9:\n 0c:48:38:29:0b:39:4f:05:fb:07:cf:ec:94:a3:b3:d5:eb:00:\n ed:b2:b9:71:a0:59:b5:3f:7c:f5:20:90:54:a8:ea:36:4c:ae:\n 62:5b:2b:6d:05:8d:76:78:87:c9:90:f3:b2:d1:72:fc:87:f5:\n 28:4c:ec:19:50:0f:02:32:d4:57:75:d9:c1:b2:dc:0e:d4:9a:\n 3a:cd:48:70:1e:c4:2e:fd:4f:b0:89:6a:de:f0:90:91:23:16:\n cd:04:fc:61:87:9c:c3:5c:7e:0f:19:ff:26:3e:fb:1b:65:2a:\n 49:ae:47:9f:d5:e6:c8:30:bb:13:b9:48:d0:67:57:0f:fb:c6:\n df:1c:fc:82:3b:ae:1f:f7:25:c8:df:c0:c5:d1:8d:51:94:74:\n 30:be:fb:f7\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUPh+bzch7lfFk5kGc324D2pKakLcwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA4MDIyMjE1MjdaFw0zMzA3MzAyMjE1MjdaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQCaPNt2yRkPe+bT7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y\n8t1XQoqrXb03tvFLUfCzajfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508\numx2gkcl63ki9AlseDym70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZz\nTBeS0Y2Bm0y4NU2R3d/ThaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3\nrfrYaYveLkpRWTJLPQYhBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC\n26Upm/leOAo5qThx7EBAtdxpxwtzAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUf0eM\nnvFzfjS5Wx7trTqHQoDU4/0wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft\nFawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAubQFSKa6bJmLI8Sbs4oyP8piiYEeXQS6LSKjD1pdoKtApIdDJjYKCWTv\n9bCnb3ofzAZs942cZF7CrudFOdy8hwbm1aprMnZRZOGs2ZrdF0ebTjEck/XFyta3\nkP9kl1nfK3/uLX1z75WttR6pDEg4KQs5TwX7B8/slKOz1esA7bK5caBZtT989SCQ\nVKjqNkyuYlsrbQWNdniHyZDzstFy/If1KEzsGVAPAjLUV3XZwbLcDtSaOs1IcB7E\nLv1PsIlq3vCQkSMWzQT8YYecw1x+Dxn/Jj77G2UqSa5Hn9XmyDC7E7lI0GdXD/vG\n3xz8gjuuH/clyN/AxdGNUZR0ML779w==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_bundle.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: Aug 2 22:15:38 2023 GMT\n Not After : Jul 30 22:15:38 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af:\n 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3:\n 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52:\n 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80:\n 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53:\n 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48:\n 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f:\n dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35:\n be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82:\n 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc:\n 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54:\n cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f:\n 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d:\n 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42:\n 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27:\n 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5:\n 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a:\n 5a:71\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6:\n 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95:\n 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84:\n 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d:\n 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b:\n 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d:\n 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e:\n 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25:\n 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d:\n 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8:\n 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3:\n 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a:\n 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f:\n 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10:\n 7d:2e:97:de\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK\nUohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr\nVbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo\naJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir\nzOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1\n+YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh\nQuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft\nFawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO\ns7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr\nHi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/\nHkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS\nVKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP\nbPF0Y4sRt/Npt9bgcrId4RUQfS6X3g==\n-----END CERTIFICATE-----\nCertificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 3c:d7:16:fb:15:99:81:4e:53:f8:80:7c:b6:7c:77:a6:06:a4:3e:ea\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Root CA\n Validity\n Not Before: May 1 19:01:43 2023 GMT\n Not After : Apr 28 19:01:43 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:da:5f:ff:1d:f7:8d:1a:9e:9a:f3:2b:68:8f:c1:\n 0c:33:06:41:00:c9:3e:e4:1a:e1:e0:70:6a:f5:2f:\n ad:df:f3:e9:99:ed:c5:d7:aa:93:13:37:ff:47:aa:\n f3:c5:89:f7:b7:ad:3a:47:e5:9c:4e:9f:8c:e2:41:\n ed:a4:7c:9d:88:32:ae:f5:8a:84:9f:0c:18:a0:b3:\n fe:8e:dc:2a:88:6a:f5:2f:9c:86:92:fa:7b:6e:b3:\n 5a:78:67:53:0b:21:6c:0d:6c:80:1a:0e:1e:ee:06:\n c4:d2:e7:24:c6:e5:74:be:1e:2e:17:55:2b:e5:9f:\n 0b:a0:58:cc:fe:bf:53:37:f7:dc:95:88:f4:77:a6:\n 59:b4:b8:7c:a2:4b:b7:6a:67:aa:84:dc:29:f1:f9:\n d7:89:05:4d:0b:f3:8b:2d:52:99:57:ed:6f:11:9e:\n af:28:a3:61:44:c2:ec:6e:7f:9f:3d:0b:dc:f7:19:\n 6d:14:8a:a5:b8:b6:29:02:34:90:b4:96:c1:cb:a7:\n 42:46:97:cf:8d:59:fd:17:b1:a6:27:a7:7b:8a:47:\n 6f:fa:03:24:1c:12:25:ee:34:d6:5c:da:45:98:23:\n 30:e1:48:c9:9a:df:37:aa:1b:70:6c:b2:0f:95:39:\n d6:6d:3e:25:20:a8:07:2c:48:57:0c:99:52:cb:89:\n 08:41\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Authority Key Identifier: \n C3:12:42:BA:A9:D8:4D:E0:C3:3E:BA:D7:47:41:A6:09:2F:6D:B4:E1\n X509v3 Basic Constraints: critical\n CA:TRUE, pathlen:0\n X509v3 Key Usage: critical\n Digital Signature, Certificate Sign, CRL Sign\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:8888/root_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:8888/\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 1f:c6:fc:1c:a1:a5:6d:76:f0:7d:28:1f:e1:15:ab:86:e0:c3:\n dd:a0:17:96:0a:c0:16:32:52:37:a4:b6:ad:24:d7:fd:3c:01:\n 34:3b:a9:a2:ea:81:05:e7:06:5f:a3:af:7b:fa:b2:a9:c3:63:\n 89:bb:0c:70:48:e9:73:cc:33:64:cd:b3:71:88:d1:d1:a1:5a:\n 22:a6:ed:03:46:8e:9a:c0:92:37:46:9b:e5:37:78:a5:43:d5:\n 46:99:1b:34:40:27:8f:95:dd:c6:9a:55:d9:60:25:8d:b8:e9:\n 6e:c9:b3:ee:e8:f0:d9:11:ef:4e:ae:1e:03:70:03:60:66:fd:\n ab:b0:f4:74:b6:27:7c:7a:96:9d:86:58:5f:5c:d3:04:ab:16:\n 57:12:53:51:c7:93:ca:0b:4e:67:27:2d:b7:20:79:b6:b7:8c:\n e7:c3:d9:25:5e:25:63:cf:93:f0:6e:31:c0:d5:4f:05:1c:8d:\n 14:1b:6a:d5:01:b6:7a:09:6f:38:f3:e5:e2:5a:e4:e2:42:d5:\n 8a:8d:de:ef:73:25:85:3c:e3:a9:ef:f7:f7:23:4f:d3:27:c2:\n 3a:c6:c0:6f:2a:9b:1e:fe:fc:31:73:10:e1:08:62:98:2b:6d:\n 2f:cc:ab:dd:3a:65:c2:00:7f:29:18:32:cd:8f:56:a9:1d:86:\n f1:5e:60:55\n-----BEGIN CERTIFICATE-----\nMIIECTCCAvGgAwIBAgIUPNcW+xWZgU5T+IB8tnx3pgakPuowDQYJKoZIhvcNAQEL\nBQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRAwDgYDVQQDDAdSb290IENBMB4XDTIzMDUwMTE5\nMDE0M1oXDTMzMDQyODE5MDE0M1owWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldB\nMQ8wDQYDVQQHDAZUYWNvbWExETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJ\nbnRlcm1lZGlhdGUgQ0EgMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\nANpf/x33jRqemvMraI/BDDMGQQDJPuQa4eBwavUvrd/z6ZntxdeqkxM3/0eq88WJ\n97etOkflnE6fjOJB7aR8nYgyrvWKhJ8MGKCz/o7cKohq9S+chpL6e26zWnhnUwsh\nbA1sgBoOHu4GxNLnJMbldL4eLhdVK+WfC6BYzP6/Uzf33JWI9HemWbS4fKJLt2pn\nqoTcKfH514kFTQvziy1SmVftbxGeryijYUTC7G5/nz0L3PcZbRSKpbi2KQI0kLSW\nwcunQkaXz41Z/Rexpiene4pHb/oDJBwSJe401lzaRZgjMOFIyZrfN6obcGyyD5U5\n1m0+JSCoByxIVwyZUsuJCEECAwEAAaOB0DCBzTAdBgNVHQ4EFgQUdVXijuetpd2A\nPckzCyyiV3ftFawwHwYDVR0jBBgwFoAUwxJCuqnYTeDDPrrXR0GmCS9ttOEwEgYD\nVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwMwYDVR0fBCwwKjAooCag\nJIYiaHR0cDovLzEyNy4wLjAuMTo4ODg4L3Jvb3RfY3JsLmRlcjAyBggrBgEFBQcB\nAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly8xMjcuMC4wLjE6ODg4OC8wDQYJKoZI\nhvcNAQELBQADggEBAB/G/ByhpW128H0oH+EVq4bgw92gF5YKwBYyUjektq0k1/08\nATQ7qaLqgQXnBl+jr3v6sqnDY4m7DHBI6XPMM2TNs3GI0dGhWiKm7QNGjprAkjdG\nm+U3eKVD1UaZGzRAJ4+V3caaVdlgJY246W7Js+7o8NkR706uHgNwA2Bm/auw9HS2\nJ3x6lp2GWF9c0wSrFlcSU1HHk8oLTmcnLbcgeba3jOfD2SVeJWPPk/BuMcDVTwUc\njRQbatUBtnoJbzjz5eJa5OJC1YqN3u9zJYU846nv9/cjT9MnwjrGwG8qmx7+/DFz\nEOEIYpgrbS/Mq906ZcIAfykYMs2PVqkdhvFeYFU=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/TestServer4_cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 16:5e:ab:1c:8b:dc:fc:97:d9:34:9d:fd:cd:7d:b3:3c:51:83:ce:d2\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=WA, L=Tacoma, O=Testnats, CN=Intermediate CA 2\n Validity\n Not Before: Aug 2 22:15:38 2023 GMT\n Not After : Jul 30 22:15:38 2033 GMT\n Subject: C=US, ST=WA, L=Tacoma, O=Testnats, CN=TestServer4\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n Public-Key: (2048 bit)\n Modulus:\n 00:d5:fd:fb:3f:42:c7:ca:02:37:72:6e:78:d5:af:\n 8d:b4:4d:f4:4c:0c:8f:8f:67:da:62:c0:2a:0f:f3:\n 73:3b:83:c1:3a:df:9e:df:1d:26:12:95:41:ca:52:\n 88:4d:8b:38:7f:78:ce:ed:aa:48:b0:dc:57:62:80:\n 7a:fc:1f:43:c8:d8:2d:4f:38:c3:22:fc:bb:16:53:\n 84:9e:44:0c:f9:51:00:a0:57:97:3f:df:57:08:48:\n 3b:2b:55:b3:90:98:98:e6:a6:eb:ca:8f:ec:f8:4f:\n dc:4d:7e:71:2e:03:ff:cd:fa:ef:65:7e:6d:8c:35:\n be:df:fb:c1:0b:e9:f0:3b:89:24:4d:b4:02:7f:82:\n 8e:0a:34:ea:a8:68:9e:f8:4b:39:9a:8f:d5:eb:bc:\n 59:68:c9:f0:a5:eb:e9:be:7c:03:49:bd:b5:d9:54:\n cf:88:29:b0:2c:a3:e9:08:b6:66:37:57:ef:66:5f:\n 6b:0f:34:6d:02:bf:92:2b:cc:e9:9d:c0:a8:92:0d:\n 76:8f:ae:f6:3f:24:38:e9:5b:fc:12:a2:ab:fa:42:\n 3f:5a:05:e3:5e:bb:08:43:5d:55:18:17:13:0a:27:\n 84:5f:05:69:18:a9:45:68:37:a7:35:f9:8c:ef:c5:\n 9f:b1:8d:aa:3c:b7:cc:47:b6:e5:85:e2:73:f5:8a:\n 5a:71\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Subject Key Identifier: \n C4:BB:A1:42:EA:15:3E:0E:D1:48:5F:B5:E2:01:42:D0:72:BE:B0:CE\n X509v3 Authority Key Identifier: \n 75:55:E2:8E:E7:AD:A5:DD:80:3D:C9:33:0B:2C:A2:57:77:ED:15:AC\n X509v3 Basic Constraints: critical\n CA:FALSE\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: critical\n Digital Signature, Non Repudiation, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, TLS Web Client Authentication\n X509v3 CRL Distribution Points: \n Full Name:\n URI:http://127.0.0.1:28888/intermediate2_crl.der\n Authority Information Access: \n OCSP - URI:http://127.0.0.1:28888/\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1\n Signature Algorithm: sha256WithRSAEncryption\n Signature Value:\n 85:c2:1a:b0:94:8b:a0:f8:2c:85:1e:17:88:4e:ca:2c:d1:f6:\n 69:26:e3:a6:94:9f:62:eb:68:54:da:2b:f2:67:23:be:4b:95:\n 56:28:08:7a:52:8e:b3:b2:70:2f:c9:db:06:74:b4:8b:8e:84:\n 23:0a:74:f7:c1:67:81:69:11:36:2b:0e:4c:0f:2c:76:e6:2d:\n 50:f3:e8:59:0d:3a:6c:30:eb:31:16:74:c8:34:d1:62:97:6b:\n 1e:2f:5c:56:b0:6e:bc:5e:08:8f:d4:ce:4a:d3:8e:91:70:7d:\n 18:d4:3f:40:39:39:67:95:68:f7:16:c6:19:69:41:c2:20:2e:\n 45:e3:9d:31:c2:da:67:8d:2c:1f:a2:3f:1e:46:23:19:fd:25:\n 16:69:5c:80:09:1b:f7:7f:50:47:1d:d9:6b:aa:7b:0f:20:8d:\n 5a:f4:37:f0:c3:a7:31:5f:4d:41:70:c8:c4:aa:2a:69:d0:a8:\n 7b:3c:cc:b4:a4:12:54:a3:bf:ce:ea:22:20:58:ae:eb:29:f3:\n 15:da:22:05:46:cd:26:ef:63:84:4a:5b:86:47:fe:cb:fa:4a:\n 0c:fe:82:e0:db:81:dc:3e:87:8f:93:23:32:de:37:3d:d7:0f:\n 6c:f1:74:63:8b:11:b7:f3:69:b7:d6:e0:72:b2:1d:e1:15:10:\n 7d:2e:97:de\n-----BEGIN CERTIFICATE-----\nMIIEYjCCA0qgAwIBAgIUFl6rHIvc/JfZNJ39zX2zPFGDztIwDQYJKoZIhvcNAQEL\nBQAwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMQ8wDQYDVQQHDAZUYWNvbWEx\nETAPBgNVBAoMCFRlc3RuYXRzMRowGAYDVQQDDBFJbnRlcm1lZGlhdGUgQ0EgMjAe\nFw0yMzA4MDIyMjE1MzhaFw0zMzA3MzAyMjE1MzhaMFQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJXQTEPMA0GA1UEBwwGVGFjb21hMREwDwYDVQQKDAhUZXN0bmF0czEU\nMBIGA1UEAwwLVGVzdFNlcnZlcjQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK\nAoIBAQDV/fs/QsfKAjdybnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHK\nUohNizh/eM7tqkiw3FdigHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsr\nVbOQmJjmpuvKj+z4T9xNfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqo\naJ74Szmaj9XrvFloyfCl6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5Ir\nzOmdwKiSDXaPrvY/JDjpW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1\n+YzvxZ+xjao8t8xHtuWF4nP1ilpxAgMBAAGjggEkMIIBIDAdBgNVHQ4EFgQUxLuh\nQuoVPg7RSF+14gFC0HK+sM4wHwYDVR0jBBgwFoAUdVXijuetpd2APckzCyyiV3ft\nFawwDAYDVR0TAQH/BAIwADARBglghkgBhvhCAQEEBAMCBsAwDgYDVR0PAQH/BAQD\nAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKg\nMKAuhixodHRwOi8vMTI3LjAuMC4xOjI4ODg4L2ludGVybWVkaWF0ZTJfY3JsLmRl\ncjAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly8xMjcuMC4wLjE6\nMjg4ODgvMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF\nAAOCAQEAhcIasJSLoPgshR4XiE7KLNH2aSbjppSfYutoVNor8mcjvkuVVigIelKO\ns7JwL8nbBnS0i46EIwp098FngWkRNisOTA8sduYtUPPoWQ06bDDrMRZ0yDTRYpdr\nHi9cVrBuvF4Ij9TOStOOkXB9GNQ/QDk5Z5Vo9xbGGWlBwiAuReOdMcLaZ40sH6I/\nHkYjGf0lFmlcgAkb939QRx3Za6p7DyCNWvQ38MOnMV9NQXDIxKoqadCoezzMtKQS\nVKO/zuoiIFiu6ynzFdoiBUbNJu9jhEpbhkf+y/pKDP6C4NuB3D6Hj5MjMt43PdcP\nbPF0Y4sRt/Npt9bgcrId4RUQfS6X3g==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCaPNt2yRkPe+bT\n7dELdq4V1BEcZrhdKn3jH2XYG8RjYvZcixhmqBzCpl5y8t1XQoqrXb03tvFLUfCz\najfpVXgBI+pTCYMvfVk2qzNPTLzvqRzblHlMDUp8P508umx2gkcl63ki9AlseDym\n70swkCmzX7ppsRqV7VPgxiR4blKvjrzbSvAZ0gBaqLZzTBeS0Y2Bm0y4NU2R3d/T\nhaafxJEZ7EfRyk4LwwaMJ0KVg+Moajt0nGiwVaWRkcs3rfrYaYveLkpRWTJLPQYh\nBGXS9YvoTUWW3mOXR4GF6kjwnSMtcYdv0nU9Rb/erUOC26Upm/leOAo5qThx7EBA\ntdxpxwtzAgMBAAECggEALjBPYLE0SgjGxWyQj6hI1cyeGy0/xNa2wE9kxmT6WPEH\n6grVkdiCVGBSJIZKdpk8wbjes1Kby/yL4o7Kk5u+xkilIZzVpmEZWF/Ii9TlN7gj\nJja+ZGIOjkrWoZsKZCr7d4WezzLZp5wSPcOndrGVa1wdjQ02cvORjNyJi28uX9gd\n8uBK5AIXS1lbkt/v+8mrBPgZUttz6gxhlHwxKs6JWWlIpGemNddE39UxuGDGHmVA\naw/gH/G4LNXtbAIPq5zDtFbfCKnQVgU1ppWILehoFqIs8JLtz4LPuvIxeztzKff4\nDU31rs14Zati5ykq9CVqY/d+4nKdstwhRPcPfsvgYQKBgQDBNVPn73A7fRoURpzV\nsdJPA4RDbrbiZj0x/cAskuzzx/mmJUuNyuJxGizJU0ebT3VxtdCR2LqpgGEQEaKS\nwYmMlSJ4NccugWgRl7/of5d5oY2m6f4W4YaNp4RebdVhNPJ4wSbeW7pH+2OKr2xd\nmy+m1WJUvRBbPq5kV2BdHNw62QKBgQDMXTqaOjsC9jpOOIjsUHmV55MbMmwK8For\nH6e3Dn1ZO0Tpcg33GMLO5wHwzH6dlT2JVJAOdr5HqZgdIqjt30ACZsdf2VkutH94\nOvZmEAbwI9A+TAoxE8QlLYyz/qjJSGopJRU0x+KqEORxBmjO6LVV1GL9VVdoYrlH\nZ7mrJ+7RKwKBgQC87LyDS2rfgNEDipjJjPwtLy8iERzb/UVRoONNss3pA15mzIk4\nuW77UbEBnGGkyOn6quKr+tVr8ZD3+YaTIpSx1xLBoTSHkRqGOXD6k+k2knbFBIHl\nNdowoeGZxKSmTPPciGLNg7x/rp4Des3oKltKM9XXLpjT4FL+40HjStk+4QKBgQC8\n71AXd9BIy7VZzaCgwUG3GhIBadtDPbRO/AQFFAtE7KuoGz7X+/dWa3F62sQQEgKD\nLT/Fb3g5LoyoGvwMdoJp9fVLItj1egAC+pgEAbs4VhPXFFuzxa9oI7VaTwxikmU7\nRsJVOprOWbGo4KES8Ud8Y09lIHof0m2ymy2nE9MRYwKBgDn86ZcbBr6sBXgc6PEM\nrq4JXBCX8O17id9rJO37PkhPsOKpNf7YbQwHlHjwkUq5+g7Ec/LbeZ/tssEBY0ab\nzUXwgWFMUKJVTEZUFwl2aTBqW8+LSu1TgzGMx2H/sxrvS4ElxC04jpPWUQstcuRH\ny3yIz1HsmlMEg7qCiQ4maZE3\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/ocsp_peer/mini-ca/server2/private/TestServer4_keypair.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDV/fs/QsfKAjdy\nbnjVr420TfRMDI+PZ9piwCoP83M7g8E6357fHSYSlUHKUohNizh/eM7tqkiw3Fdi\ngHr8H0PI2C1POMMi/LsWU4SeRAz5UQCgV5c/31cISDsrVbOQmJjmpuvKj+z4T9xN\nfnEuA//N+u9lfm2MNb7f+8EL6fA7iSRNtAJ/go4KNOqoaJ74Szmaj9XrvFloyfCl\n6+m+fANJvbXZVM+IKbAso+kItmY3V+9mX2sPNG0Cv5IrzOmdwKiSDXaPrvY/JDjp\nW/wSoqv6Qj9aBeNeuwhDXVUYFxMKJ4RfBWkYqUVoN6c1+YzvxZ+xjao8t8xHtuWF\n4nP1ilpxAgMBAAECggEABmE7dr39Ep3ZDRdz0QwaNY5O6p8Dvy7llQTdZCsaDAPQ\nNJsC46w87LgoNVnbUDOGwE8n3TBS2ToCfXBu6joc5V2jkS10LOR7x+0+wpCtEdhL\nRFyEKP51u+yaXf8Aut5/zX2bwUbj9d28p89NnMV4AIo7Dau0pKXcDlW1Qk+LztyI\nhKFN6hrSFqAurmSt/pu3oo9kI9WJkrCxoj+VjQdVi420uAYOFR22aFaHrzpuHouW\n4IzFbLhVF+c33xSbs1OEIpZSFzNucWYEKSwEREcyFgIXfWpDaXjoqWcrvXkeqyo9\nvGytQ3YaEsZPzfzgcViwa30g7WAA7kO9RuwcCPK4wQKBgQDpVmbVnmTlRwFbtdkD\n4rjd5vtAB3nfsl0Ex11nU8+Oo0kZWeg8mm+Gba4vjEKfVyojbjFmm0ytQG0OGEK7\nUQ13mE1wueMn5qEVX9nTXIxVwcS7+rQAUrC5a6SSg81WIWzeclkqNc1J1EVC7jtl\nzqy3PtC94g4tV68urpD86RRxUQKBgQDqxpWscN1u7GeuYf8rSPhPcoZTupqyrV3L\nh+w7jUt5O/vfNPOYIXVfo2u05jiK0mTvLf5tVjYoQDF+x6odA2oBH2yz1ED0DZsf\n2AhdtCSrMbxazcl/5fPrIIa1GRBp6y5i0ddX8T19twr/PVoYGRqkU4xoN+KoOKz+\nHLFUUgQPIQKBgG5N9v0DDMVKRL0bAQUSN7xGxf1ly1pRUiHBMUl4WEUgsZy3YM7N\nXu1YiiBWGOSEaxomrFnKDnxUWXlxRJKSZWBk8i7Y4SZqozmcfzeop3qeyCbpBBCn\nBn4RAdJ1VitiT7n0qmwG1Q4St89FGXUuN33Exx8MbxFGQz05LrcwZAaRAoGAVFez\nPZfudQMI3GToPqygSCpkh3/qQ3Z008Go5FwGWS9rdOyY9nZOrGURNJPgjD65dBOZ\n672lByDIpzsjqfioBG89pf0CuKqKqA38M22cHsRnXle/o+sAjd/JhRXUB7ktmOK5\n8iYAaUFw+fEYhL/ACnjZYDdzfeueekvkiN5OBwECgYB90hQJ2lw5s6GFJd+9T5xS\nOMngfLAWDvW8+0hvtWCTLAVpMDWRGhGmvj532jWfkgqnvUemyF541RkV0Hy5K1Xl\n0icXtpuZ+REh7NCXFJlEiOd+69OEdu78s5Zy8V1zCkEsgxzl2q6PkBDWfxepgdRC\nLbwiAF8h2mxCwvvHbaBiKA==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/ca.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDd8u+Lbq5qjxj0\nD8YdouHVlsnH+8UH1spKRr53wRqdPX7qC9eyBDMJ8ltUVAfQcKjXqP3EN22rAm+1\nL8vSXcdqxO2pGStXbfrdvDIeiMuJJ2zpz4m3g5T98AcNFUxyUQet4rLHtQGyzK+S\nsTSOKCg7hULMvnXMLjUJTBsDgSYPNxLw3jlbppTaEVhna74d0aRNBkHVJvO8BJsT\nJ39iz2SrhvGzGfhK2Jbov0QPC44hIV28AFrwKbJI8gB7kD9BWe4hNfqoECLP2K/Z\nqYw27Ybr9nUVC3ZSnTudAueHwsT+Pa5qbBoNw4QGwyCc7y9nolIOosi7RK1LxxOS\n477I9ORrAgMBAAECggEAB2C0csOWm6cvEqnVkZWhASsWgQXO+mA68DYizbNHEbQC\nHICRRnMaSTnbrFVwvw6HpkeRS+z1wcn+cZzpnxAL+XIbXlQeWzUmMim31skATwpW\n9fy2nLmMgdZxPY+YL3KfGTogbEAJkup36Z3nW9Rc5gGbg1fPbZ6Zl7oGadroq3e+\n0mKdUzKNw46VVUpoHIq5ty/5mGyiew0E6IJEv3tD8MtFZ9r29zyDF6thsj3k1K+l\nelRc6hgvP74RI4vJ6yNGWgJRoLK4mHAxhS6Zew3aVr/5P9P0WRnj2WD65geUgaFv\nhCcJzqkGQPUl2CGRkspIUErpJkAZRyJyjuw/XU91aQKBgQD9dy6nqqfTpf/+dWzc\nVltdhlEuGfXN1Yfg9A4w+o3+goeFwt0dim6plX/iSLbohylXbpPcugx+OMIw0xHs\nZsT8L1NIUSiXIri+VIU+5iU5X+tQUlCXnTy1MhqRSI0BD02Ac/+MMp36RmYXgghd\nMIiMN11wDbeGsCzjDecziALi0wKBgQDgKxPmUniPe5eBF3QrpDpBnaxZEz1/e0Hq\nxKcN7ClCbfPSvyIGJhB2NJR597vfaHCwsp7ReUK1RrvmRBaBg8bFffCbEfFSyB5c\npZ6r9BXBhPTB9NMsnMn/e3Al8sVcYWbZdgkkxuVE5DdWZS3zXscmTzls/RPvDlOK\nyLI+GaiJCQKBgHxjGtBJnabuVhzoP446CwhwvDIlp01mNxKccsJyk3CNWji4ko7G\nutwac/H2wVyHyDASIho2a58d957CVo9vN8iS6QoaWhMhhQxSqjld9HKdsftvCgH9\ntict+X/G3PviKKSbSJPi20hReBSdScGB6eD14rL06FX+62haEFZnrxLlAoGAYrK0\nzUjsagg5mY3xCTICsFcyxflRr2peiHZTMy8Sr3vnyZd77Icf4lueL+FiZ6f9Td8n\nTAV+2H+vRWAfJKEXiGa95BjPTupra9FD/mO2nIDknu0jeVYDHiiEJUExok4EUaTF\nu9qSoXV14+UiiS/msThaiWEYQL3nDIj26Z60dLkCgYBBrZN/M8NwT111KFGXgYzm\nCxFCu1WPCRUXIKVJjbj9reJgfHMhj58NFpqtVSS4KNeoc6ZGOfkAT/Nb1/H001TF\nAK3CBTGzFQiqNmvmWP+0O6JaqVXPgHXXRCdXz9CnJmyIeJqvQbQEZPlIC4cTOi3U\nYr+e93q7SOS6wbf446ij5A==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/ca.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDBzCCAe+gAwIBAgIUC6fLO8KV763IBygl6/0w7EKectswDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBAN3y74turmqPGPQPxh2i4dWWycf7xQfWykpGvnfBGp09fuoL\n17IEMwnyW1RUB9BwqNeo/cQ3basCb7Uvy9Jdx2rE7akZK1dt+t28Mh6Iy4knbOnP\nibeDlP3wBw0VTHJRB63isse1AbLMr5KxNI4oKDuFQsy+dcwuNQlMGwOBJg83EvDe\nOVumlNoRWGdrvh3RpE0GQdUm87wEmxMnf2LPZKuG8bMZ+ErYlui/RA8LjiEhXbwA\nWvApskjyAHuQP0FZ7iE1+qgQIs/Yr9mpjDbthuv2dRULdlKdO50C54fCxP49rmps\nGg3DhAbDIJzvL2eiUg6iyLtErUvHE5Ljvsj05GsCAwEAAaNTMFEwHQYDVR0OBBYE\nFHMfJfiA8Nb10G8rKcO/am9Ogo6tMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/\nam9Ogo6tMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABMr45yW\nSDHQ+zOW6MXCn2CAqagr9TaRjkqYmaaNh0uDBI+oaFKi8WHCi7jt1iSb2f8wp6dv\njQ97osGsAsU5KgyyliweIaTftd58oegpmADwpQTpVa2RIz6o4z2FKnDm6ZtH5fm+\nRRS+FpcS81s1m6e3gJ3Ie4nIqQRrBcvKpQEgMgiwJH4v2rIB0RvkTztA2EeVyyH/\nIjbrbO6Rc7EpEJNbsVWHcKt0tMNx9F1qicgscrcEgAPG77yz1bP4jKSqhXt6OhM9\nlhXFq+EbzAhbPzkgQuxghHxJuDNZDafKvf1cl/b7jEykLEiKo3s1oto+9yzRKZ2y\nuKJ+WESZ0V6XOU4=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-a.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC2yUu/qqBD+rKF\nq2f2ISuw7KZlpne5NN4RXhAK1fe1b6TtZi/hDeDaBo0A5ttFn+lmb99DWwQh5UZO\nOrjueeZLjhkMIBqmVY0Z8wu0BudYFWrjbEEFz2oAN4UXxu6PvKdcHsJQHbMjVvxS\nj7GauUafZ1MZmcxRizQDW0+bD0NDKERR4JropXC+9Cths9MG8VKXGzYEBX1g9j2r\npRZ064ZDUEM3ImZI5fAdeUPZuutg2J9swgXvppVpXOMQPGR6TUZsQZC8Z39zm3OK\nvsyqKE519gpC5ZNTMm9xaD+Dd8tyv3OhMnVZvIEBvtJysKS076cGqxwIeFNZ5zky\niNzZ6A2jAgMBAAECggEATs2pDSiEMNqf5bYYu4ngesYEFRX4Ts5GoSky9GD8Cupn\nqfvgz3+6vXSOjOF6jmFbcFDx6emO9ggRT//MI/BFA7TvAoXCx4xhqVrZ5FMHoTeW\nnhbZT6Z09kP5at9WkXyNkM8oCf9L8IP7g6K0uNaA5o9WydY/OravhXTz9I9Xeddi\n6Cgjr42PVqpdlN/CvQQ+ISYGSYzg6r/o8gaIg1zypJPm3DqZ+TFBxalv7d0FoH54\neagk+blA6kUGvJ/CX5Fr19hliycDNfxT/uofdVKEUXSWlPQgzvGk6Pb0B/Jco0C5\nBDuvA97LigY4AR5FXZa2wmYhdlBontjixiZNFgeqHQKBgQDn4XriO8ZNkERW4e2V\naLrkRH5nd3IqRX9aUn7Qz1xXV+nE9+984Mwk0xtTkwtmmiII8v+2cfz1J7I+HVB9\nNRFGtcKX2REEM6wp+UPJi8HFx1uOr3Mi/JM8i+1/vq9YPXR4DJ0CBJwydTQo0zIz\njJrqKPUiEAtrAmT4YCz/hdrARQKBgQDJzIdWWCv8LeyFBdJJHikjSy1PETFMoeRj\nFEtdi3Oqninn99ovV43fgVXK2C38OaiO0X7N1OwXtJhWaBU5TkmlEbXe7eNlayFc\nRFfEmg3RFItFEU348oyNuhqDU7RRG6FapMOX6fAIhaF0Ox+qLlwsW+uNbjxcq/5Q\nJIhLji+4xwKBgFk9V9OVeZ4ENajSTmM/6R1MlvEu8Qr/sCGMui2WgE1FEjyxxsNi\nqG5LqKqjXrHgA9U539vRRq761gg9s+pb8Agsj3VtHrVlRY7p1YLNfIenT788Gq2O\nUwPsPjz5n/XvwNEq2YobSBnLLYXAsgNQy5XuqViwoBRQ2ZcVPHwigauZAoGASzFu\n7HmGSj1CeH6m0J/wbDpC88dQO2HHnDOfsYeY3eZ5bvonzqQtNS8YLLFI0Ucay7y8\nJy5DmsGsUeYOon/NaTikMCEsLkow8BR3L8bHhzTEgEPmbfDS9qx13KF/+wj0orXq\nO9zrmAFTG+A7+Em2BsFpkBWXYGCmLm6uapjcp1kCgYA1cGEgpOXGv3HR21ItrpWh\nwx/iEqybIdYU5bOr1OxZJE6PgWD3k0DcPAFGbfJX8I5AEQQHhbufv+QMwfdm4irt\n/EMWE+5MzmCOPKQzb3jLhvdqAKie2rnDFuFvlhT2woDxQaY5gaS3Tay380WZ3+rW\ntHA+wi2+w8REX4KLuaY25A==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-a.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcIwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL\nTG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV\nBAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is\nZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtslLv6qg\nQ/qyhatn9iErsOymZaZ3uTTeEV4QCtX3tW+k7WYv4Q3g2gaNAObbRZ/pZm/fQ1sE\nIeVGTjq47nnmS44ZDCAaplWNGfMLtAbnWBVq42xBBc9qADeFF8buj7ynXB7CUB2z\nI1b8Uo+xmrlGn2dTGZnMUYs0A1tPmw9DQyhEUeCa6KVwvvQrYbPTBvFSlxs2BAV9\nYPY9q6UWdOuGQ1BDNyJmSOXwHXlD2brrYNifbMIF76aVaVzjEDxkek1GbEGQvGd/\nc5tzir7MqihOdfYKQuWTUzJvcWg/g3fLcr9zoTJ1WbyBAb7ScrCktO+nBqscCHhT\nWec5Mojc2egNowIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w\nbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFCkA7CbJMs70HE5rQxHm\nqHL/UgkOMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3\nDQEBCwUAA4IBAQC87KaD2UuvQWIbSPOKy2iPc5bHrsmzxtPzT219NqzI4JZ9QeAz\nbtvLlMxgACreCb6wzYxaOjbU+O2LqO7mq/M4UyY1+wrAwYn+c3rVcimuPa2bGKin\nLe+aBr/4yXAahqH4DW5K4x3x/7c7wyNlj9MHUQDl9A/JHsit68hw1YY/1ALPaOpg\n9L5K52gGHVXrxb8In8OkJEoM38G5Zstiuh1umsWNWBp6Nd/FlNr+XbvNuFsWLC/V\nlTaHxFt9WRQNINc8RpK4YCHRdaOiEqAGexDkHgNaLHeHEwCvP3Zhj7Zpp+svlmm8\ncLL2A5wl0T0OEehT9wxKaG5cyP7jfYfJmENW\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-b.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSHSxxgyq3Yd4H\nLpzmpRceCQ8y4qqsysiZ2C/9HhwcvvoUXRJvLB5VzYb1702AZNgbeffrz8RRGcaa\nYuPRXwkitRGb/MdmC/AFi2aALk1M4EKzGhw5IjyL6tfk+qahxm0IACJVQW9W7Ts8\nO6odqnqenvXLzvTNn0UWWv7jZonOpqnR36CoxIj9WGNXIlwDtcTwCoLfd9V+kUzX\nPybIVrAL13tLPypZ7fpyAkwd/4qYAZ59HbRsQDNvYItkwe4JQiFt4PQ94+eGDXtk\nDqttKPi4uUUgSGqRzTNkvkM+KAghRD45XN2J1ugw16GGUt4sQKPViTDh06yQ/OTR\nX7CyuvmHAgMBAAECggEAGBhPXe8QSXpYJ1Oxm4OIHKImU4wJzYntqjLQAgNfbu55\nqDcqI4SYL/fDR9W6+gPWgoQgqbKoTTKbA15eyMZQumGwVMaOqUWqL0CWrxFgPPYk\nv/w5C1ZBQjxaqMATts6V90popYU8+ud2mQgwkHlZ2XHaNxaTHTdsCU3DGBnMqN1f\ntq3glwkHZqUxKsGXwWqSKsDwEUXpcs2ww0fbuzpdZ5IokPX9e9TdFkw10rKXboRn\nowcTxHI8kuTVMFg9ht4G39V/LVpqrPsHGz4Zj0tJ8PkwxowcwLMcxg2KVsHRWzC+\ni5UVlzjWVK/Gu1mMMCfySJLBi4GaO88hZzK8acdu2QKBgQDwgN5zkgOELSulfGDQ\nx6spyjbEdN9dn/cwCpolMJMg3mdcKP/XjPTTEe1dsWT7mTa9qxjWgcxOzr1A3h4B\nZPJTXgOQyt6rPYw2WffEh6+5H28+629VlhQary7Sp5rZAXiJoHgzHpb9sURwliTL\n62GzaCOxJd1AI3qd8atjgI/JmQKBgQDfpwcGmcpAhQMA/owdowCoJhQ7Jc5h3+RD\n55WO6sFFxigxa+gSt6NVw3ef5TB/zKyovPGceunaMyVOeKA3RSNYNX3gKKhpugeg\nwv465jGw84jPj7b1LMc7ydWHKWQ4M6JLPxlSM9lIV0Zdb+gNu64hCNMGOQljcCuh\na0Fi7XgQHwKBgES1XFgCF/UT/ospjHM5B+JJffoENagGwlS8QqtrRC04vKisoeLP\nx0XhskF5I5NpgyUOk2r87AR7tb+pdUMKttwwWK5t9s1PKRaB/3HrHb2yiFKealE2\nLgotkB/oeCmU7P3MlWptS8+wbBAKJPsJBQF/N+stGRdwy2ACIeesW+fJAoGAbG6w\ni1S4qDtUz4CaMiw7P0rm+B5GR1GjfACllBrhBZEjH4Lvi1OZTxVrxAv8TZnQBCdH\nBNTa2D1/0uHM278becLFeo57yHa3CAxB8hB0+xO3uto0m/3Pwn8ClXtN6amu/8hh\n4Gbe66HUrax5116s97sthJDWqm1R2fsEQpBba4UCgYEAitrHGT/Kr/91Ttccc7Ww\nEee/xEJR8iMFU010zkX9MAu1JExXOaMWVJRWlzSIJFhDjEovaq9qxdWPsoDT1C/7\nJIs7RGDVWX2a6uun5Oz9JJJ9KmNBzOAicTBBrWMT67TGWZhVYjEN9PvRyX/kMssP\nyqRcbXt4Unw93950LphbVOw=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-b.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDejCCAmKgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcMwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQHDAtM\nb3MgQW5nZWxlczENMAsGA1UECgwETkFUUzENMAsGA1UECwwETkFUUzESMBAGA1UE\nAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0h0s\ncYMqt2HeBy6c5qUXHgkPMuKqrMrImdgv/R4cHL76FF0SbyweVc2G9e9NgGTYG3n3\n68/EURnGmmLj0V8JIrURm/zHZgvwBYtmgC5NTOBCsxocOSI8i+rX5PqmocZtCAAi\nVUFvVu07PDuqHap6np71y870zZ9FFlr+42aJzqap0d+gqMSI/VhjVyJcA7XE8AqC\n33fVfpFM1z8myFawC9d7Sz8qWe36cgJMHf+KmAGefR20bEAzb2CLZMHuCUIhbeD0\nPePnhg17ZA6rbSj4uLlFIEhqkc0zZL5DPigIIUQ+OVzdidboMNehhlLeLECj1Ykw\n4dOskPzk0V+wsrr5hwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4\nYW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFBN5/1B/G8UTcl5O\ntqEHRzrAJ4IHMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqG\nSIb3DQEBCwUAA4IBAQAioN2YIK8yKkyn3x/UH+YhgLJj9CZfkCPa2uJTBaXCmZcv\nKBPfjs4bQ/c6omLzbGnIVEDHjUEzwxUf40cVQbciPXvrVxB5PIcUY/e8RAgFux7j\nYnP4F4fM4DC1yOA5AIqWZGX66GVnw7rslxz5Pko6QKNaHuVgwHHEizN0d8hHexdH\nrGHtX3tsFmI7GOwsVgLJNV3VcpT+W8ZdviHtbjL2gR3N/KpXSU3FHmDC56Zi8HyA\niUICVuCo06LCEq5J8M8f5dBEMtLJ31gDX1c/arLJuS6VS/+XrC7lKNT571SXPa9x\ngRmY0EtzyYut/yfG2qXWV9Xi0DbrZUKZVIUcqd8y\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-c.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDG0eLmkpj4R7o7\nPhx+EUuInMKkHD2yoYVIDc3+ipvj/3bptxWVcRZoWroFLrUroSGnPBI2stXHGFti\n5qkCCjmsWLsido8luDsLnr9kBQafBoV+tj+u/ZkJSeTJ+bVXaGS9GR4rGvYPgwRD\ndyuebRSrqW/GB5Chp0X4JqxNGrX+SrtU4fz23eBpg1+Tv9Td3xhlw4FdVAphSYS9\nNYLg7fNpIVwXLcvyXgLya2IOvItHGM1TyWq4jsOrgU23z640GzTUEIic34KkgF78\nBe3ZN+0oEiIrojpUb9I0k52NhHL6Jjxu01/IIgan4n7h/tr3O44YOeNfpRtn7RSA\nCelkiXUpAgMBAAECggEAAwbQ7cek3Sq3srfYVob237vbLq+2ChEcEU5nKLfSywlp\niwHxfyvl4RgjJXm3jP7P9KwdGYi8E7goM/zk7RYGGLyPug+UpZ4EsBbuPPHipEBv\nfyVRGjMxcDXCw4cHSVmv0ZJ0ph1Wn5fT/x7s+BAbCh3zM1A21cR7yTJK2dr36Ei+\nCgUEvVJQh3rult/4JJ5E74oXq6rXRqAjO7i/DXImLiV7YePjixhy4YGpYH8T1Da9\nCdhA53Ec2wqcEfaIZC+fMtz+4XQnCOJJEjJGfGApGadu52sqA6dB0I0D+DWy3H84\nV4MPrJMvllK1GH3pk5zqm+tWlt62R7hc9O4a7OOVgQKBgQD9ydnmPv5Zf03UAzT3\nVLuwvM+5H1EthWV0CsjPf9M5QGfAm5tuxrqWRfZ4mPX8YRJFc0RXBy5WD+UkBwDo\nGvmmD96t6FJ+xsXTz0Oq58cUAWiyuqPldbe+NdOIHasam31jBss/jrL61YHU4eec\ntCtTZtJQsvQ/X3wjWSF4fOmaOQKBgQDIjWlmxnxC+AhdrtlLYVhg87NoVgAOAZcf\nMCnJT9xMbhfmZko75vUdT4OnTXVjtxfqLP8yZE6lgb7GaTum3YLhhfZjnRs3buGk\ny+6mKlNNhfrOFCw8tgVzaeGQi+dE0WyBHqBpiKxU6EixY/28oV1+einxu9agZCNy\nUSU03LdycQKBgQCkSdwGAfdzhkainaTXC4IpCkFKLKzHVBh2A1k1giEBaeEAPXtk\nPb+h8g54yURMKabDULgHwn1LdyS1qtb6aiP8TT+wwrlMmm6MDBtY8ovcNoFJWisp\nKohU2NFjxxjxs4B673X5Ye4ZFMfkQI1H0xZM+j3hwcb9k1lwJI3XSr2KeQKBgEUA\nAkeN+qq/04EH61L4BwQ0VIGNNS+cdHYSiA3vIAhbyHVItDmv2J1hAhbJm53XHK9B\nE/wubrCa1xxEkHV0uNcG1CKppveHerLMRyt8XHTLp+LHJgEtTurKfwTQXZ1bwE0c\nUGx+zWvZD0mY8W+4xQYC2fOFgO5mBZwLNxXc6nDBAoGAHJaD/ZSSjtAqQnBYN7Wt\n/QOpWx3UrZ4xAKM6KPp0jAKNHgFlH1j7KXIT+fC54TdYw6K9TOzMOG83BFNBs1vf\nGTRc7NbMW5fW5NqLEwqBK6lWChqXMwjmz/W7qxRBPqtsmAkubgUO1fBw/XReut7C\nXOa0n2b/bfIb5jCCFAH/Gw8=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-c.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDpzCCAo+gAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcQwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL\nTG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV\nBAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzMxFDASBgoJkiaJk/Is\nZAEZFgRmb280MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxtHi5pKY\n+Ee6Oz4cfhFLiJzCpBw9sqGFSA3N/oqb4/926bcVlXEWaFq6BS61K6EhpzwSNrLV\nxxhbYuapAgo5rFi7InaPJbg7C56/ZAUGnwaFfrY/rv2ZCUnkyfm1V2hkvRkeKxr2\nD4MEQ3crnm0Uq6lvxgeQoadF+CasTRq1/kq7VOH89t3gaYNfk7/U3d8YZcOBXVQK\nYUmEvTWC4O3zaSFcFy3L8l4C8mtiDryLRxjNU8lquI7Dq4FNt8+uNBs01BCInN+C\npIBe/AXt2TftKBIiK6I6VG/SNJOdjYRy+iY8btNfyCIGp+J+4f7a9zuOGDnjX6Ub\nZ+0UgAnpZIl1KQIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1w\nbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFNZnXD/At0QIFoqsEUNn\nhd0XqkJWMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0GCSqGSIb3\nDQEBCwUAA4IBAQCiIEr4+IiHo4PLK41l6Yuo6QKETZg5tNKluqkT7r5ulYaTss+C\nO7nBIZaVAK3h5+uRTzrL2mEUyme4tYBUaIxqVTnjDgIbeKvg3Z0k07Zld7+eeE/B\nvnPk3c4aZaKshcbKoA+tZUeHk+BYZ28YkDH70OYsyVhemvTVDDy2EZGdgeaWJAbx\nRcQ7iVAIR8SEgJy9PWZMAFChNxa2N7Q1AcEqnU9UV0+XOHVHe+PYgK7SfuNOxM6x\nrP2NnkLenDoZ7IXEvl2WYVlnJpIZiNTRz94nkTz5MqjCuWS+YGYlPz76gpGPJ5NT\nVEfAZXP0EW8NWJ1XEIM6+u6c1SWnSJfTt6im\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-d.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCxkPAQkDuzNvqJ\n/iVOuucGi7n2N3VI0CXBoAIUTlyWnf73m2VOjeXUCBL95so3uGBd1Q+R9BwEnPH3\nw8P7zEn007tryzskGywjEqCn2WYW3TKBGuFhuTc5sFE4IesCjglRwebsCn78V7kE\nRnz9/hRgULEIRPjWZkdwnxO03S/4n/lzMxCkZdLjyIWpr99B9l5Z3dAFlMchRqKK\n7+IJGorNjEIkVLQIHdWFUAnI2ZBh0Xkuebqxg061dhEnvo8aBi/gqcc/8IVfnUbX\nmmQM4nPwRFzOY1/+hfCKOIR1MAQZ7ydXK1C0tuoR3rSvUEPgT7MbAoi8PZIaoLCB\nbFDsQ0LjAgMBAAECggEAGDmobvIRRbWPVTVDB2yQvqq1z8V9hFYVg0bzvFZ746FM\nRaFsnwd/9BS36+5JPMne/TOSFhJiTNomruwFTdHvbcNXSMZnhq5dsAPyFC72R9Hy\nJSgILhTJV98EwwOUSaXG2Ax8F0xP9S3N/wB+PAl749oOamI9OsRQedPSKfTv/wYw\nzPNhr4MgnnD34VDnkoYTFIU/n9OKeSgh8okx3PFaPkfYKjaOoIFHA4j1lMMVa7kl\nq27MrL/vpWF0MUu88ip3lDmZQC9t6BovoIidwRhNEYo4Qf+igcwcIce2MY77gg3Q\nIvWpsG7r/jyr6KFRONvyJWpyIgAq0Y6aRb4cNzkDxQKBgQDWrOdLVg6xw+I8BVrM\n9gDJyfeF8zEXu6Mi3zrypyOOd11ZqlVFaeq/sT2s7HCVEZOiqTgWE0ueBry6FGX3\nDkzGpTo1+kd5HiKawViPHnIIVTV/F1Hifrx8bDHRRPVqyh/iTV2c4S8Jdf90zgZJ\nvW2wErGTrpTVvgnb5GhZBZIQbQKBgQDTv082UDFD3fpA98t7DptqmYn8hpdAa4Yy\nMFMHBNbNntB5G4RDdDut0eWA01MKNRjKrxBw+YjuOgTIRx5oVre73y83932rGSiQ\ng1/y4PidwPVEAnVbbLqvvHkkKTbBbHENSh2pWCikJhAGKhVI665bAc84o8o4ikN9\nIQ20TUqujwKBgBt1rHRPgrusYcDpsm6rPPiS1A2XSP1PLBPm2AR36q1riU5R4mxp\nmvSAOHJpIBGBWRAicyEnwLFrDTMELvLGKn3yXprO89uDRkvjVW+hJlb4h0pFclz1\ndyi3MjwhhP7u8dshqErL5xdft5h6TgWarHAsQ/ivCSy6DUrKUaqpcsCVAoGASq8A\nNkzkg2ZonL+JYlbNlDShSZMDB/Kku1D2B9S6Gn34U67T4KK/ZdhRVTWz5TbDDsHe\nT6qDlFqcUzNaUzy6wyW91sSQ62cNOqNLlTOqHKHxH9KqJ9vaoJ8eLxXmPSSNXz2T\n5qW0d+kA39u59CVEMs8ZkoWajoSdtyWQWtakD5sCgYAUgWP7Z0CbUeNyaH811Aef\n/P7inGoQEqTYrmnyAFmRyBnmq3C3OjTRiz7+R7cERpgVVmLt4xpNWjmTbcngfvau\nc2YViXlsFx9s/xXFQBtppmdH1WeZBIOZbN6EaA5q0oHGOXAW5SmZR3sGKeG5Mz/m\nVDNDxg7Fspw4ZdHu6IeOCg==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-d.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDrTCCApWgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcUwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owgZQxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL\nTG9zIEFuZ2VsZXMxDTALBgNVBAsMBE5BVFMxDTALBgNVBAoMBE5BVFMxFjAUBgNV\nBAMMDSouZXhhbXBsZS5jb20xFzAVBgoJkiaJk/IsZAEZFgdleGFtcGxlMRMwEQYK\nCZImiZPyLGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nsZDwEJA7szb6if4lTrrnBou59jd1SNAlwaACFE5clp3+95tlTo3l1AgS/ebKN7hg\nXdUPkfQcBJzx98PD+8xJ9NO7a8s7JBssIxKgp9lmFt0ygRrhYbk3ObBROCHrAo4J\nUcHm7Ap+/Fe5BEZ8/f4UYFCxCET41mZHcJ8TtN0v+J/5czMQpGXS48iFqa/fQfZe\nWd3QBZTHIUaiiu/iCRqKzYxCJFS0CB3VhVAJyNmQYdF5Lnm6sYNOtXYRJ76PGgYv\n4KnHP/CFX51G15pkDOJz8ERczmNf/oXwijiEdTAEGe8nVytQtLbqEd60r1BD4E+z\nGwKIvD2SGqCwgWxQ7ENC4wIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SC\nC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFM4b9lmfPrfM\ntHVxnMbxR8oyE0ZtMB8GA1UdIwQYMBaAFHMfJfiA8Nb10G8rKcO/am9Ogo6tMA0G\nCSqGSIb3DQEBCwUAA4IBAQB3O3Q+oTfAVitx8K4+ZzXILszfblsGO4kvwvxV6EUB\nBUsat6mjawFTGhB1TdBR0CflA/nCjTZpoXNY+nCoJmr/Lxk1V+UNn5zhFTTfhZjU\nPA+++5QcYxXz3ukCEqTHLPZKvv2xB24xQUmsdwLlX3VGE1VqBfaz/2x6jFTVT3lz\n4W5JPltgFSVOLqQC1T0MY8L0h4eL+JpynHPjMkEwb0U4RPaSBhiRSjHD8dgoZ8Ft\nZbV6WCJeBYey0edKNwujuLl+McVpx6DIy8BPGPEcuo20Sy1BP3JAoN1Wu5J3zLBA\nAXwUZ6r7jAjU5v1nx6ijwtv3qavBVdclOlW0aDu3+Y3z\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-e.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDRoOr56ToxIMnw\npalVuFJwDnPotA0toHmQe2+iGUtsOaDQW5USKynV7YrP2DkBTzai+KyoA6cW6b3M\ni8FIivJk9hkxnWp7CuSn0dFZVoyMwSuyelw+ff+8Cn6hNmg9s0BgQklk6pe5LsGh\nRwun6CNPnlJapYycwaH3e7W4fiFrs5njz4yrYQTQZPA6fBybtcT5NatPMWBzCHl9\nHoBzLWEOphN7D+Pa5Cx0zwqHf8h5/NJUWXnNO89/Kca9oGZOm6t2+Dk0OYa+nw1s\nd1CWlzSYobilQiEEiYLnPgRLoNiJt7hDZqE4ubVcIDptN/SfiOr5cg6aao2Vdb0P\niNcKn531AgMBAAECggEAJj6R8AMtYy4kuXs3wBRPMfNpESDcMKFQ3Fuwu6WavYVs\nAedbdIOcb9OVGiNMDyGZCbWNpl94OfVt86Xhnu7+lw3Q/dzGwujzW2yroX1pSfqq\nErdBIgORxwgxHw9SiEMuMQGW60rjYF0X5WkVRs5StuFfC/udAjJPbtvKhPy58zBu\nC2ZtcZFmf4KuwzCkZT8K8zvEFcOOXNIhtKMdPRSsbO7EpWn+J3SA+mG3nKo36swe\n6EptIgFo4Z0dSzS1xK50iNOw10dKgxOSJpDjg6vH/cGduUxGvGWW0kiV1vD0ndUU\nU1qyQSwlfFfFUk9WrtxoWFCXgZWyhoSONQmG9eRlgQKBgQDtLYVNIKK4n/TEiz9s\nBtkFmh+ohQ6vjorjkzrj88Szxy9MivCD5vvU2Q61V15H9255RxgRPZ1lczGP++N2\nIckY+iC4vmeEyKOyFX8XYaJILEXPRkoyj6/PMyS8eWciZ08QW/bhOEDSiIlFrZ+4\nC/0KfMaT3MHRFRNV+jlza2RzNQKBgQDiQ7W+xtCqsvSXbVjPrC4dmxXlo1B/O4P3\nKsdiNoGpzu1I9nfKNKlfwfnzI5vtk7dS85kZPuCQ18fOQ7aO6w3VReg91zoEDSAV\nGSJLVHeiRr+XOBCV4Cpy3PloYD+m2f4niT/Ru6CfEmHq6IoouWCqSobZYlkIHtAe\nRfjg1b8XwQKBgQDU8e7iDbArZWok4QTjX86QKg3MhxJlauymYPUZ3y63XtnHtmM1\nejiWpP+Ar1217aChgMRKDnD3uhJAvR4/XRwpscGZPVLCNEZMyOIfwwgqFp3i+K/7\nb2ig9sE/+xwvFmQ1QuyIz7HblLzy820YmKLrPJYqAaV+rJZ8tjnIuB4rFQKBgF69\nPS9JbfvbfKCfD229SX3p/uwtSrpLgEEQ73VHH1mrpB1F3fiTvkuzG+ZbhaGflUYt\na3BvrHXZc/cA0ULcVulzIQAry0YA/Or4oCxsjL8s4nH97qvitcNslR3IFz7Uzh9d\nz5QQ/Ps+JjPG9HqCzF/hXr9M97x3smrKn5/8v8vBAoGBAIRJPSik2ILc8V/1FnPa\naOfcBJwcuKdRJ4yVWiwe6meQCLWBva19rH8qx/D7h+r96a8OlSDZuYltM/UF5rV5\nlEKUs7uH9Xqw8QMMI17fQ//aRx265PJvdLtoO302IUQiC5qZx9YZKSLIzetVEapD\nSfKVumS7pEPZgZI6WD5YoahG\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-e.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDnzCCAoegAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcYwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owgYYxETAPBgNVBAMMCEpvaG4gRG9lMQ8wDQYDVQQDDAYxMjM0NTYx\nDTALBgNVBAMMBGpkb2UxDjAMBgNVBAsMBVVzZXJzMRYwFAYDVQQLDA1Pcmdhbmlj\nIFVuaXRzMRQwEgYKCZImiZPyLGQBGRYEYWNtZTETMBEGCgmSJomT8ixkARkWA2Nv\nbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANGg6vnpOjEgyfClqVW4\nUnAOc+i0DS2geZB7b6IZS2w5oNBblRIrKdXtis/YOQFPNqL4rKgDpxbpvcyLwUiK\n8mT2GTGdansK5KfR0VlWjIzBK7J6XD59/7wKfqE2aD2zQGBCSWTql7kuwaFHC6fo\nI0+eUlqljJzBofd7tbh+IWuzmePPjKthBNBk8Dp8HJu1xPk1q08xYHMIeX0egHMt\nYQ6mE3sP49rkLHTPCod/yHn80lRZec07z38pxr2gZk6bq3b4OTQ5hr6fDWx3UJaX\nNJihuKVCIQSJguc+BEug2Im3uENmoTi5tVwgOm039J+I6vlyDppqjZV1vQ+I1wqf\nnfUCAwEAAaN2MHQwMgYDVR0RBCswKYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIP\nd3d3LmV4YW1wbGUuY29tMB0GA1UdDgQWBBToYUZkIK4kXJRs9Af7V10od3oJiDAf\nBgNVHSMEGDAWgBRzHyX4gPDW9dBvKynDv2pvToKOrTANBgkqhkiG9w0BAQsFAAOC\nAQEAKMs8gDYeTAITUMJ2cfQqU2t1I7DGIt4eQ/bJ1LWOhC6eUXpUrwjjNo4pq95P\ncct0haleycD4vjZF4/Jv81oSIUsGzQ8r3LZhJlPnzbZzaA4i4Tpxnw8JRE5iiF+z\nlm0Cl783Nh8voVKE1uJrSx+pMTH9Ihwu7vBklNI192zLPLPtDFSZ497oLVt1e8Wt\n2urYXF4/Wb1pyGL49gp/eOAEpw2j6pyxGSK/QyXyvvPnDLqNXJCOTyx7YQRvZnBy\nh9vTtHg4HSKmgrOHA/taW051lh2PQbi3pbi+ik3rk68QY/Y7b3fKjc/6lvXPRoh+\n94valHgOVFJaSG+Qil0A8i6bzQ==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/client-f.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCgF4OXW1j7wJKD\nKq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDwT7p/TL1FpGtK\nykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGqRSWzbGcXt5GR\nLpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPfynWmmHKf80/F\nkIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587a0DilwR1Pcr2\nq7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrYK74q1uMUJogc\n45EXMhGfAgMBAAECggEAHC+XpQXZuFVS1lIlAF5zW9Mv+k20pB80moRdZro/h6mV\nQFoKAq6SMvBofRQ7pqq746N959zIdCL7GutbXF6sU6rYxzFBk6FhzDdJZnI7Qc4z\nPpE0JKa2/WOjpodEPvsZvNs4xIrA6d2RfPiw9Wl7tR2M/AQCrC8Bvse5sRq/+dP4\nU24eetvPJDTYLk3PFnZJPShAUEOcKUaL/73nIwCFbC8ez3ae9EIt4CO4ljHORp6h\ng2zJFei7ie6VLHAZx4WUeGTVlhgF2bYjsjM1h254ZTIfNgqGs/DaOwzO2kpC5b+m\nTE3Tuq1r0b7uMEnsCJb4gj39RYOqThZ9IG3I1oXKiQKBgQDMp67wfZm4jKeNfwr2\ngUxR7Btn0XqB6aMBofAgQiUuq9z8lRr8PTM1wPnw/ZKN6mYunIhXHGP+t8628hJx\nFAy0fSy2sy9ucxipcDlyPBalSjtj1i1jqmZJtvmal7zfi0S/wgBn/J0X0lw7927Z\nai1VPVoHOeTouV4PhcJPHkqOuQKBgQDIQa+N4q4kCvmoMJSIHT9/vUV60uOO09QN\n4WEtSUdrZVYnSYvO52f11bh1ZLRiZayEfov3vJ9LsLhnil5pNhEvxCZ3v+z5Fqgi\nPRxY8+PnWleMC4QhCQOywpUbC5HqvqyEhGVVYt/CKbD0BKJoK/iVSkVUIhQuvh32\nW1x3wcG3FwKBgQCC9iL6fkVqVYe7Ajsvt1nxape2/dPZYnsPRmN1IR7OGOiXMYtN\nMpScp1rKHlo0OnUdCsoshFxw2YqMg6fNeLkQnrGIUG3fzgNQGiKIuW78Yt5SavIs\nvxQpw89CYCtbGbHqy+iaooqcfd7L+PCUbF+KFnKQATo8urI4WK4ZFxc68QKBgBZs\n3Kbr6opYvEpsXkW72L+KR3yQnzEAYa/IPGSg3yGUsIgnwUNDQK4T59slktmt/xq7\nPRtaARCt0oitwxLPHi+WLKSeVoAyXFOxOjpv3WasBD/Hjl1QsBxVk/L6YbXC8njI\nhryAHQSWPJ2m4zT9L5IwRgE867usSJis9HbouLOzAoGBAIZtLLPx8KYgjiVx8QPx\n4uggZ7iblh1aI5ZO0XbsmWQfVDJ7HKpLFc9vd+nX2++Mv0vm3yv2WslMVliFvjXl\ntgZNx10zn2usg+Cqy5728ldTXnXKA9q2KZOfhVPHv3sTao9bji5tywjnvt4ZLp+j\nngI3derKCSC10cXUiDEs3dXI\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/client-f.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDfDCCAmSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfccwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owZDETMBEGCgmSJomT8ixkARkWA29yZzEXMBUGCgmSJomT8ixkARkW\nB09wZW5TU0wxITAMBgNVBAoMBXVzZXJzMBEGCgmSJomT8ixkARkWA0RFVjERMA8G\nA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg\nF4OXW1j7wJKDKq7dKn4MU19yTzR01MkX1rsrC5kkFsihxNYd2pc8MdGB+VaJXaDw\nT7p/TL1FpGtKykgIp01nzceJNrfV6T79aslhvB/nt0BTKlCDDZTRogS6viJUAZGq\nRSWzbGcXt5GRLpRGeXO6OpjVdQQqDwTMdbuHRTZ99VDfy1x8TwzWX4NA7xNUuoPf\nynWmmHKf80/FkIgxWaRHDwUtKxn4ItVKNfMDAWfArdTMH7ZO4AYfIyM2WS82H587\na0DilwR1Pcr2q7o6mFB807sdSrjiClUhWwM6A0KVsxnUvQAWkyYh9n01vnDYHrrY\nK74q1uMUJogc45EXMhGfAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIIL\nZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQU1Fzej0nnnMw9\nAai0AC5g2XsdiDcwHwYDVR0jBBgwFoAUcx8l+IDw1vXQbyspw79qb06Cjq0wDQYJ\nKoZIhvcNAQELBQADggEBAKqH6YEyYgW5CnhHNOlbvB7kCNrQN/nAITWJILkJRtch\nhMhsODV49fj593Rp3vTJXJ+fRCwJeF5pCfHIyUaEC9FM8IERzK7yZUW3h6X/KfwW\nvHdRabjsuNuXk92wvslZK82jjosDBvdLv0pnApVv5OPnSXDUr/kGFnXhe7OQgLAb\nOE5o3jRHWGCiSa1CnXrY4fzjDNR/MRgk7N7nQ29cY5fC//3jtBlYlcY7smjH81OF\n9kFgQK1Mf83cN3zFLkVejLpquVM1CX210Z0GV7hEg6zr0jS8v1YRnCQ9xp+Euhd+\nTIU2SuidadW4ww/yCWsBfBuoX8ijByvG8sYyQ/jXL3s=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/rdns/server.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjc0LulDTiYpXX\ntTTXW8ubZueU3abq16rixbIbpQ+hZlpGhPOMDQbBEJdFuurt6GiwUJYvPonsuLxM\nVhwDBGwKAs8Sw0vjwIHoPNBo8oKa7lW6Be2vv1+VBiDOd8ua7xnrLBmpKgADP40a\nJ+VAgSSCAVBA19tG05CJbxGnGHBgXqNwMamR+DhQYp1JV8JqivGA4UqOAchHOyp9\nucFItKJ6EeB/qq1O53KnEcB5WqHYGy06/tJPQ/jsdQxakSaVMrKvrhTY4XkSMPAA\nsOcrX/BFpWkloNthICYP2ROzV2HYTDfQgGt1SU4wdNseOsRfs0TQRKMa29gx7iQI\npH3D2i/FAgMBAAECggEAArqksjOWy3Xebx7XLkZZ2LMn6DgoNpEZyRYIcoY3w9ff\n54AGSf8YrwYOJIupqwW/o7wAufK3HlT6TeL9Lj63aBZiazuIC5o2ARDs4Oe1rw0O\nMOAy9wWAvM4Ao7nj1CcsXV0b45NwMx2vRzHF1eUiHSqU8rQ39M+JAQZx+sKBc9Zx\n2oiL2Dy2q6YBqgJA24dDk2S2bez32ZWBdyJ+/kMzanhsHZb4a8StuxMjX201vhmG\nQzV9PR+oi2dn9Kee/GX4yZUVDZ9A1jOdusk0+SnAPJDMi2JEp5kj898GXW22PZCw\nmQ8iEqJCmsB5zWUiYiF0IkmTeSkukZoX3YaTts9jYQKBgQDRBnGrFimcM9QJVbeR\nObGix2TxmgKm8qkLuUb4LX/AgLHCNAy5SdodMOFHdV+zZlrqwY3d54yiFwix5ujD\nAaJcAhOKQGX0aTEz3Vd4XIj9Y/uMwhU9XG5zTbbT0QlVWLDy93KJYeVa9ateJsg5\nDBcF025ApLQv/Z6bT2R7VIjdYQKBgQDILtFktkfvQS/uMcJDX35Vw8VDsJWc2Aas\noBQBfHwFM7mzfoofmHnUFh/901uSXvj3YVgNHnSQx7+sJQhViO3RFHSNHwyTs7aV\nRErs/vseFpYf9SorASygkOR9bH4z+KIw2GpSMUIWq5c8E1xe7lkZUuHJhQht8Vzv\ndVwHBfUo5QKBgQDHwObD30wokIcj3Jyu1nnh27emA85hCSlvoMInziN2LgayVteK\nAv/EQcAocAzi/wAHtK0E0ZFeHbEbglYSde9ZCkruJhjI9/YjYHWE+rmXngL5Q0jU\n+Q48dpov1maa/0UrDDqS+9EZmgkI0vspOVqPIL0OXdgsvKzkM30NN34MQQKBgEzn\nN4giQsZWW6x9Ly0kzWrzV6AmgYOMthuxL55WjWqOMYQUU8nJkNv6V/XyMZasp6aZ\nnfMERTGtmyPt4iLCBOKyogfo6rL/cmArqUEcv6oScT/7tmRpAhT5NN4+RMmAdgaf\nzVCgHcyJxQLOSUkq9c21uonpcpCSDersQhtsnX9FAoGBAMLm40fsn5hqeOSHfF7W\nZEX0eKSBPuCCWTtUi5c7pPmyj0B8+9o3wiP6YHcs91ysfAWgnJqBlZkPxkwC5zvT\nAlS1GNsCCQzZT8c60QiOESTMWa+398AGtFRYVNjbamGcQOW3lprfeXiIQCAyQ2d1\nGNZICja01bMZsLidFoyUIH95\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/rdns/server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDLDCCAhSgAwIBAgIURF3HIP6jjAixTsMisMxARhqRfcEwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMjNaGA8yMTI0MDgx\nMzIyMzAyM1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEAo3NC7pQ04mKV17U011vLm2bnlN2m6teq4sWyG6UPoWZa\nRoTzjA0GwRCXRbrq7ehosFCWLz6J7Li8TFYcAwRsCgLPEsNL48CB6DzQaPKCmu5V\nugXtr79flQYgznfLmu8Z6ywZqSoAAz+NGiflQIEkggFQQNfbRtOQiW8RpxhwYF6j\ncDGpkfg4UGKdSVfCaorxgOFKjgHIRzsqfbnBSLSiehHgf6qtTudypxHAeVqh2Bst\nOv7ST0P47HUMWpEmlTKyr64U2OF5EjDwALDnK1/wRaVpJaDbYSAmD9kTs1dh2Ew3\n0IBrdUlOMHTbHjrEX7NE0ESjGtvYMe4kCKR9w9ovxQIDAQABo3YwdDAyBgNVHREE\nKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD\nVR0OBBYEFBTXnZytLW93IszwKUGgXXh7OdNtMB8GA1UdIwQYMBaAFHMfJfiA8Nb1\n0G8rKcO/am9Ogo6tMA0GCSqGSIb3DQEBCwUAA4IBAQDCrwa/UTnBOoJ/X2FIfaWP\ng0llr0OSQx3l2RweWte6O90VB9AWTgstYRVErmYyXV70lYNp+HOPpxak1DEGr+P8\n4REsMjX+odz9+UGOq/n5N+0VAfLTsQ9CG5EnHsfzwamgL/Ax3czrzgvmP4lz0tvp\n07le9YUWkuG9UsUhN4/qe65LVweg8AfhihiijlcQe3WnrB7WvyyOZO81lBnLIIar\nqq62NSNxPWNa/TGX/og+E5HwTPcMNeMOsok5D5TQtx1zNX+2Zj7i2Py3ScTCCY/G\n4U1nO4k+APE/3BKuvnrz8ZJ1UglT0lrVEkVqzW7TNCJpDz37A9QIQVLpw8oOr+Yc\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/regenerate_rdns_svid.sh", "content": "#!/usr/bin/env bash\nset -euo pipefail\n#\n# regenerate_rnds_svid: just remake the certs in the rdns & svid dirs.\n#\n# We're getting the hard requirements down in scripts, can integrate all into\n# one all-singing all-dancing script later, so that anyone can regenerate\n# without having to read test source code.\n#\n\nprogname=\"$(basename \"$0\" .sh)\"\nnote() { printf >&2 '%s: %s\\n' \"$progname\" \"$*\"; }\nwarn() { note \"$@\"; }\ndie() { warn \"$@\"; exit 1; }\n\nreadonly CERT_SUBDIR='test/configs/certs'\nreadonly RDNS_SUBDIR='rdns'\nreadonly SVID_SUBDIR='svid'\n\n# WARNING:\n# This data is hard-coded into tests such as TestTLSClientAuthWithRDNSequence\n# so do not \"fix\" it without editing the tests too!\nreadonly COMMON_SUB_COUNTRY=US\nreadonly COMMON_SUB_STATE=CA\nreadonly COMMON_SUB_LOCALITY='Los Angeles'\nreadonly COMMON_SUB_ORG=NATS\nreadonly COMMON_SUB_ORGUNIT=NATS\nreadonly COMMON_SUBJECT_OOU=\"/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT\"\nreadonly COMMON_SUBJECT_OUO=\"/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/L=$COMMON_SUB_LOCALITY/OU=$COMMON_SUB_ORGUNIT/O=$COMMON_SUB_ORG\"\nreadonly RDNS_COMMON_SAN='subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com'\n\nreadonly CA_KEYFILE=ca.key CA_CERTFILE=ca.pem CA_NAME='NATS CA' CA_SERIAL_FILE='ca.srl'\nreadonly RSA_SIZE=2048\nreadonly DIGEST_ALG=sha256\nreadonly CERT_DURATION=$((2 * 365))\n\nREPO_TOP=\"$(git rev-parse --show-toplevel)\"\nCERT_ABSOLUTE_DIR=\"$REPO_TOP/$CERT_SUBDIR\"\nreadonly REPO_TOP CERT_ABSOLUTE_DIR\n\nokay=true\nfor cmd in openssl ; do\n if command -v \"$cmd\" >/dev/null 2>&1; then\n continue\n fi\n okay=false\n warn \"missing command: $cmd\"\ndone\n$okay || die \"missing necessary commands\"\n\nmake_keyfile() {\n local keyfile=\"${1:?need a keyfile to create}\"\n (umask 077; openssl genrsa \"$RSA_SIZE\" > \"$keyfile\")\n}\n\nensure_keyfile() {\n local keyfile=\"${1:?need a keyfile to create}\"\n local description=\"${2:?need a description}\"\n if [ -f \"$keyfile\" ]; then\n note \"reusing EXISTING $description file: $keyfile\"\n return 0\n fi\n note \"creating NEW $description file: $keyfile\"\n make_keyfile \"$keyfile\"\n}\n\no_req_newkey() { openssl req -newkey \"rsa:$RSA_SIZE\" -nodes \"$@\"; }\n\no_x509_casign() {\n local extfile_contents=\"$1\"\n shift\n openssl x509 -req -days \"$CERT_DURATION\" \\\n -CA \"$CA_CERTFILE\" -CAkey \"$CA_KEYFILE\" -CAcreateserial \\\n -sha256 \\\n -extfile <(printf '%s\\n' \"$extfile_contents\") \\\n \"$@\"\n}\n\no_new_cafile() {\n local keyfile=\"$1\" cacertfile=\"$2\"\n shift 2\n openssl req -x509 -new -key \"$CA_KEYFILE\" -out \"$CA_CERTFILE\" -outform pem \\\n -days \"$CERT_DURATION\" -subj \"/CN=$CA_NAME\" \\\n \"$@\"\n# We want these:\n# -addext subjectKeyIdentifier=hash \\\n# -addext authorityKeyIdentifier=keyid:always,issuer \\\n# -addext basicConstraints=critical,CA:true \\\n# but even without an extensions section, those seem to have been included anyway,\n# resulting in a doubling when I created them, and Go cert parsing did not like\n# the doubled X509v3 extensions data, leading to a panic.\n# So, removed.\n}\n\n# ########################################################################\n# RDNS\n\nnote \"Working on rdns files\"\ncd \"$CERT_ABSOLUTE_DIR/$RDNS_SUBDIR\" || die \"unable to chdir($CERT_ABSOLUTE_DIR/$RDNS_SUBDIR)\"\n\nnote \"creating: CA\"\n# This one is kept in-git, so we don't force-recreate.\n# TBD: should we delete, as we do in the parent dir?\nensure_keyfile \"$CA_KEYFILE\" \"rdns CA key\"\no_new_cafile \"$CA_KEYFILE\" \"$CA_CERTFILE\"\n\nmake_rdns_client_pair() {\n local client_id=\"$1\"\n local subject=\"$2\"\n shift 2\n local prefix=\"client-$client_id\"\n note \"creating: $prefix\"\n rm -fv -- \"$prefix.key\" \"$prefix.csr\" \"$prefix.pem\"\n # TBD: preserve the .key if it already exists?\n # For now, just using the same ultimate command as was documented in the tls_test.go comments,\n # so that we minimize moving parts to debug. Key preservation is a future optimization.\n # The \"$@\" goes into the req invocation to let us specify -multivalue-rdn\n o_req_newkey -keyout \"$prefix.key\" -out \"$prefix.csr\" \"$@\" -subj \"$subject\" -addext extendedKeyUsage=clientAuth\n o_x509_casign \"$RDNS_COMMON_SAN\" -in \"$prefix.csr\" -out \"$prefix.pem\"\n rm -v -- \"$prefix.csr\"\n echo >&2\n}\n\nmake_svid_rsa_pair() {\n local prefix=\"$1\"\n local subject=\"$2\"\n local san_addition=\"$3\"\n shift 3\n note \"creating: $prefix\"\n rm -fv -- \"$prefix.key\" \"$prefix.csr\" \"$prefix.pem\"\n # TBD: preserve the .key if it already exists?\n # For now, just using the same ultimate command as was documented in the tls_test.go comments,\n # so that we minimize moving parts to debug. Key preservation is a future optimization.\n o_req_newkey -keyout \"$prefix.key\" -out \"$prefix.csr\" -subj \"$subject\" -addext extendedKeyUsage=clientAuth\n o_x509_casign \"$RDNS_COMMON_SAN${san_addition:+,}${san_addition:-}\" -in \"$prefix.csr\" -out \"$prefix.pem\"\n rm -v -- \"$prefix.csr\"\n echo >&2\n}\n\n# KEEP DN STRINGS HERE MATCHING THOSE IN tls_test.go SO THAT IT's COPY/PASTE!\n\n# C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2\nmake_rdns_client_pair a \"$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2\"\n\n# C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost\nmake_rdns_client_pair b \"$COMMON_SUBJECT_OOU/CN=localhost\"\n\n# C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4\nmake_rdns_client_pair c \"$COMMON_SUBJECT_OOU/CN=localhost/DC=foo3/DC=foo4\"\n\n# C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com\nmake_rdns_client_pair d \"$COMMON_SUBJECT_OUO/CN=*.example.com/DC=example/DC=com\"\n\n# OpenSSL: -subj \"/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com\"\nmake_rdns_client_pair e \"/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com\"\n\n# OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\"\n# WITH -multivalue-rdn for the -req\nmake_rdns_client_pair f \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\nnote \"creating: server\"\nrm -fv -- \"server.key\" \"server.csr\" \"server.pem\"\no_req_newkey -keyout \"server.key\" -out \"server.csr\" -subj \"/CN=localhost\"\no_x509_casign \"$RDNS_COMMON_SAN\" -in \"server.csr\" -out \"server.pem\"\nrm -v -- \"server.csr\"\necho >&2\n\nrm -rfv \"$CA_SERIAL_FILE\"\n\n# ########################################################################\n# SVID\n\nnote \"Working on svid files\"\ncd \"$CERT_ABSOLUTE_DIR/$SVID_SUBDIR\" || die \"unable to chdir($CERT_ABSOLUTE_DIR/$SVID_SUBDIR)\"\n\nnote \"creating: CA\"\nensure_keyfile \"$CA_KEYFILE\" \"svid CA key\"\no_new_cafile \"$CA_KEYFILE\" \"$CA_CERTFILE\"\n\nmake_svid_rsa_pair client-a \"$COMMON_SUBJECT_OOU/CN=localhost/DC=foo1/DC=foo2\" 'URI:spiffe://localhost/my-nats-service/user-a'\n\nmake_svid_rsa_pair client-b \"/C=US/O=SPIRE\" 'URI:spiffe://localhost/my-nats-service/user-b'\n\nmake_svid_rsa_pair server \"/CN=localhost\" ''\n\nrm -rfv \"$CA_SERIAL_FILE\"\n\n# FIXME: svid-user-a and svid-user-b are ECC certs, but not expiring at the\n# same time as the rest and with differing requirements, so not coding that up\n# now.\n" }, { "path": "test/configs/certs/regenerate_top.sh", "content": "#!/bin/sh\nset -eu\n#\n# regenerate_top: just remake the certs in this top-dir\n# we don't (currently) handle any sub-dirs\n#\n\nprogname=\"$(basename \"$0\" .sh)\"\nnote() { printf >&2 '%s: %s\\n' \"$progname\" \"$*\"; }\nwarn() { note \"$@\"; }\ndie() { warn \"$@\"; exit 1; }\n\nreadonly COMMON_SUB_COUNTRY=US\nreadonly COMMON_SUB_STATE=California\nreadonly COMMON_SUB_ORG=Synadia\nreadonly COMMON_SUB_ORGUNIT=nats.io\nreadonly COMMON_SUBJECT=\"/C=$COMMON_SUB_COUNTRY/ST=$COMMON_SUB_STATE/O=$COMMON_SUB_ORG/OU=$COMMON_SUB_ORGUNIT\"\n\nreadonly TEMP_CONFIG=openssl.cnf\nreadonly TEMP_CA_KEY_REL=ca-key.pem\nreadonly CA_FILE=ca.pem\nCA_NAME=\"Certificate Authority $(date +%Y-%m-%d)\"\nreadonly CA_NAME\nreadonly RSA_SIZE=2048\nreadonly DIGEST_ALG=sha256\nreadonly CERT_DURATION=$((10 * 365))\n\nokay=true\nfor cmd in openssl ; do\n if command -v \"$cmd\" >/dev/null 2>&1; then\n continue\n fi\n okay=false\n warn \"missing command: $cmd\"\ndone\n$okay || die \"missing necessary commands\"\n\ndelete_list=\"\"\ntrap 'if test -n \"$delete_list\"; then rm -rfv $delete_list; fi' EXIT\nadd_delete() {\n delete_list=\"${delete_list:-}${delete_list:+ }$*\"\n}\n\n# Issuer: C = US, ST = CA, O = Synadia, OU = nats.io, CN = localhost, emailAddress = derek@nats.io\n\nCA_DIR=\"$(mktemp -d)\"\nadd_delete \"$CA_DIR\"\nmkdir \"$CA_DIR/copies\"\ntouch \"$CA_DIR/index.txt\"\n\nreadonly CA_DIR\nreadonly CA_KEY=\"$CA_DIR/$TEMP_CA_KEY_REL\"\n\nCOMMON_X509V3='\nbasicConstraints = CA:FALSE\nnsComment = \"nats.io nats-server test-suite certificate\"\nsubjectKeyIdentifier = hash\nauthorityKeyIdentifier = keyid,issuer:always\nsubjectAltName = ${ENV::SUBJECTALTNAME}\n'\n\ncat > \"$TEMP_CONFIG\" < \"$keyfile\")\n}\n\nensure_keyfile() {\n local keyfile=\"${1:?need a keyfile to create}\"\n local description=\"${2:?need a description}\"\n if [ -f \"$keyfile\" ]; then\n note \"reusing EXISTING $description file: $keyfile\"\n return 0\n fi\n note \"creating NEW $description file: $keyfile\"\n make_keyfile \"$keyfile\"\n}\n\no_req() { openssl req -config \"$TEMP_CONFIG\" \"$@\"; }\n\nsign_csr() {\n local san=\"${1:?need subjectAltName}\"\n shift\n env SUBJECTALTNAME=\"$san\" openssl ca -config \"$TEMP_CONFIG\" -policy policy_anything -batch \"$@\"\n}\n\nmake_keyfile \"$CA_KEY\"\no_req -x509 -new -key \"$CA_KEY\" -out \"$CA_FILE\" -outform PEM -days \"$CERT_DURATION\" -subj \"$COMMON_SUBJECT/CN=$CA_NAME\" -extensions v3_ca\n\necho\nreadonly CLIENT_KEY=client-key.pem\nBASE=client-cert\nensure_keyfile \"$CLIENT_KEY\" \"client key\"\no_req -new -key \"$CLIENT_KEY\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=localhost\"\nadd_delete \"$BASE.csr\"\nsign_csr \"DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_client\n\necho\nreadonly CLIENT_ID_AUTH_KEY=client-id-auth-key.pem\nBASE=client-id-auth-cert\nensure_keyfile \"$CLIENT_ID_AUTH_KEY\" \"client id auth key\"\no_req -new -key \"$CLIENT_ID_AUTH_KEY\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=localhost\"\nadd_delete \"$BASE.csr\"\nsign_csr \"DNS:localhost, IP:127.0.0.1, IP:::1, email:derek@nats.io\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_client\n\necho\nreadonly SERVER_KEY=server-key.pem\nBASE=server-cert\nensure_keyfile \"$SERVER_KEY\" \"server key\"\no_req -new -key \"$SERVER_KEY\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=localhost\"\nadd_delete \"$BASE.csr\"\nsign_csr \"DNS:localhost, IP:127.0.0.1, IP:::1\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_server\n\necho\nreadonly SK_IPONLY=server-key-iponly.pem\nBASE=server-iponly\nensure_keyfile \"$SK_IPONLY\" \"server key, IP-only\"\n# Be careful not to put something verifiable that's not an IP into the CN field, for verifiers which check CN\no_req -new -key \"$SK_IPONLY\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=ip-only-localhost\"\nadd_delete \"$BASE.csr\"\nsign_csr \"IP:127.0.0.1, IP:::1\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_server\n\necho\nreadonly SK_NOIP=server-key-noip.pem\nBASE=server-noip\nensure_keyfile \"$SK_NOIP\" \"server key, no IPs\"\no_req -new -key \"$SK_NOIP\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=localhost\"\nadd_delete \"$BASE.csr\"\nsign_csr \"DNS:localhost\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_server\n\nfor SRV in srva srvb; do\n echo\n KEY=\"${SRV}-key.pem\"\n BASE=\"${SRV}-cert\"\n ensure_keyfile \"$KEY\" \"server key, variant $SRV\"\n o_req -new -key \"$KEY\" -out \"$BASE.csr\" -subj \"$COMMON_SUBJECT/CN=localhost\"\n add_delete \"$BASE.csr\"\n sign_csr \"DNS:localhost, IP:127.0.0.1, IP:::1\" -in \"$BASE.csr\" -out \"$BASE.pem\" -extensions nats_server\ndone\n\n" }, { "path": "test/configs/certs/sans/ca.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDBzCCAe+gAwIBAgIUc56AEpW980qCeFPAyYjDMQmgD4MwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy\nNDA3NTg1OVowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBALcG8Uka9njBYwtVEachD6ZzpiEZV8CVHU4lfXv2e0VD6D/z\nQ8DTWc7TRMy+15u/h8ZSNSQm2mNVDUA5SbJPoeka6LqXl47ZMhY9fwIa860KxphQ\nODdwu75v6jjh2IZz4BW3PGjeVAN6pJfejmn07y/lGa+jDvKPl0BEee+Vj2APpDu0\nJRtnNoLvJryuddQq9AaagPXP5wsvS5mRVqjM3w9dvvxGJUZW+nEKDX0cEw3nv1h4\nK57Z/fIHOk9cTH2b960QG/MjadJi/8tvVduBRt6ZZj9M5xQ4dg5+28lJbVq4eoAi\nlN4XIbah1tg2q0zWBprYieVLWEgT6680rFMrI2cCAwEAAaNTMFEwHQYDVR0OBBYE\nFK442m9BmE189lFzd57Hbz/b95psMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57H\nbz/b95psMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEx7HfRg\nQoInmdQwN8IKVB21RU7gdkFT6DxmS/ZEW3SFXctrkPftyToJ3PoFlp7rYooKCI8e\nMPOo2GeSt8Ocv2A9qgV+X5euZzKDrOW53kvk6BpvqiadfmOjw/aa4HwjSDh3lUwP\nhgQWriNYK7NBXINIxHXT6tRv9NvIvR3UeJ2DBujwAF5rBmTIBI9b5N1oQnE2jeY/\n2LsV8rcnWRcYEoyhb55AJ5aD640saGJ0StmdwRK9nOFzg02xUP4z8QuentlwPVVo\nDhCGGbMLY9fneXjrD4jEUjAHn4CWuPLdn/agIrC96qv+PZYUl9hJoDJnshuPczyR\nUQFYI1rGa9JuwRw=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/sans/client-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkjw4sZLkZJPTP\n/c4gIPXr0IacaCvNJne8fivl1FnVg0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJm\nsjkCGVSsNTSY3yqZCM2mLIgQmaSCuKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVT\nkJd2qDALXh6cHI8esCigBAZsnUS90mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PA\nSIakuGFvCWgv6uS57BOgn+c3134fCYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfy\nvkXIt51WoMm04gNymiVYYZxbNpe9M33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG\n8J8Rzl39AgMBAAECggEARvhc0FAeYb5l/kezP6q8chL4GY0HPNabC36qJt1/X5s6\ns5rM23JjAYz4XfSG9P5mTGI6JEvclgRazsS8ivhOoIKM6IMzeP2Qze65+V2cBbNO\nFhIl9RKOkPADjfwgaLdhHISBOzBilb/NxFr7jS7AbGc3XgZIMqFEnBtj20oiVEZu\nl9BAoiDlAiMKt18AMGxsI6Kr5Px/vuNhQpxjKDw+47wbhHP7EW+6ywVFn+Nb9kV+\n8QC/14GZ9EbDPfKhlU+NqTIAKZbvI+NvAyETB8ezEnEkEcSr3FaWhA4WpHauhijx\nqAJSBVsVinoFQVRHwDjZGDJ+M2QkFk24Jm+X3VQySQKBgQD7qFMRYF7+KbXBp8uK\nYpW3pyM+28oHaeVjKckCtFuAIJS02yYo2neANPeVIthurBwl4pRSZchNiqW70nSW\naYPw+4JCprYCkfNZFFP6X5FzIlmGRCpgjuWzXHFgkSRnOg+GckcrMu1VcvVYuosC\nKqQWkXSbupvrQTK1f8XZQTnR1wKBgQDogLG++I1+wLUFQmPw6c+RrTwj9i1SUEt6\n50pUp+9/KKDXDQTyjV/XZILmaLcn4jcqCXDIvEddkEUngkrazS6quwQtB4wpQgYZ\nSSQ/eWsRCvA0VUI+6tHFThPGg1SZREXorhflm7YYa2nsLF118d1pHqKO/gUhBJX+\nPv5Pzdy8SwKBgQC5E7PzxZJXDcnUIFk67wH4zPzIz8+m4CSJZ0Ojr4zTkCKNV9aa\nmQDl0w56KeROkFkrK5W1e5FyJZN5rG995x/X7MCB0CVvgnMbgi24puxLZmm8qwkX\ndkBMRqJDSLsjB7o/QHBCvvN4slDp7lcpQr7msha2KOlefNaUUOHqw2OIhQKBgQC5\n4go5kCYv7InNRqL4fTYCVen7JlpdsOxnunrm68zCcQ0GYdZOxVCWuDkfVSD5thY8\neYe+NSkpWKqxR63o+JoSzaotBhe89JhDpwJf7Qb4fTJF5NQt0Tcc86tDzsPYNYle\n2bEpVTBknZv1whKGtXQ7Es8MW3JmT3BL8LkJvKB5uQKBgQCIUUm3zKTXo4fxfR84\n485La1x9ThnOM7aDgLhKJLg0hQTMPSozsKS2wbOIbg45BS9BL/HGkjqHXryY35Ee\nwphElPZGLtw4bd7xjSQykXjsCHJIg1lxnEG0sdTaVBD3iQnx4L2pCg8fY0iUC1m4\nKnOwsB6IfaV1EnouUebes3ydjQ==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/sans/client.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDLjCCAhagAwIBAgIUTOgYUed1dyp+Gan755liL7b29wEwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy\nNDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB\nAQUAA4IBDwAwggEKAoIBAQDkjw4sZLkZJPTP/c4gIPXr0IacaCvNJne8fivl1FnV\ng0mG80zXQcXuPKi1HF3Qg/qZu38p6oZTfeJmsjkCGVSsNTSY3yqZCM2mLIgQmaSC\nuKtnvIWjeBUtmMAx6bA0OphD5rYaItflqaVTkJd2qDALXh6cHI8esCigBAZsnUS9\n0mAgDCQVahMIDkz4jFC/AXwnJUlNzqa7w9PASIakuGFvCWgv6uS57BOgn+c3134f\nCYi2s+A8xhg4o+GanV6+5sbLuwMPo+0DvBfyvkXIt51WoMm04gNymiVYYZxbNpe9\nM33ICQuFAoTb7RdN7mUylBVM9MDfdG7ne6TG8J8Rzl39AgMBAAGjdjB0MDIGA1Ud\nEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbTAd\nBgNVHQ4EFgQUiUGJiS99EVH6/iPsQESqZSfXxCIwHwYDVR0jBBgwFoAUrjjab0GY\nTXz2UXN3nsdvP9v3mmwwDQYJKoZIhvcNAQELBQADggEBAAugrGg7+6LX7UZ6YhCM\nP/CGBnxIMODeuikNf3OAEjwNRJjdcy8OyHKd9pSwEzNZ0jZCN127UDPHQzZGw5JE\nKCe3B4a0Lh0G5BQVOOTcUJM9et6aFtBDvNbKhiiwi11d5aAEQX7k3ugPPaEjP9si\nSM1b8UBcaO3mGSnveMVjBiInzSkJdc8kPaAsssqvledmQa9RyM1y3ZEcvixbF+6d\npfQG61UXR3rnGEUAoHWmnKzJ+o/kFlThd2jtKT8T4qp2Ws1ga43V5kacFqyWcpfy\n6dK1lfr3Mx4qHMlnFp7O9L4ciFLtLiv4ua9fi4zH8zS7EALkwQKxhaDHo6Afi1mI\nASw=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/sans/dev-email-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCoFArKsSQLasob\nBGo4+d2WUxjW3rp7qMZ2M92RvefInqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM\n56/QtNx6fapKltO8FSOs7dSWLQquvOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9\nF22R+MNLhCYacSytogTz9ZoCLLuqfaAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT\n7loxW9+VDXN1wKP3U+ziHJVPX2+438OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQ\nqEsY+mcjh/+JZk2xbhScSB/bsZFFIIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mS\nyH1rT3x9AgMBAAECggEAGk4GsVm/XyUPq+JJQJEEZ7gqGUjC7UMJCmtMGDjdK2X/\ns8thZKjpaZEO8MfcsGQSRGutAo5XT4c8BQImnzaLEgWUTvejfBpf2rrfDEDQ3O+W\nbINSaYYVIk1gX7hMwFZBVaCK48d5YLOMSW8u+Aszf2BXeUhwml0Swt6Tg5ceKMye\nfZa9geVZhC61Gtxbk+L4P4cnKLAvlxjFSn3iAS0YNGGsIVIHmpFQ5l0JdMLUF/4h\nC6enmgzs9fxnL67Wy54MoICH4qEekW09AHbDI2byKgBNEmmm/xsQlRGoHdwo3QFV\naCDmul7V9LX1DMs3aamzr/+jw0/4bixZvRN9zGgH6QKBgQDraDBtrY2Fk+uFNkPL\nzf29H3+W0bibfOKSoyTkvYHYxK44sYAN/ly7BRkH5CaYiTNQWJ26lVwi8nanSnNV\nNl7Hevx1C9Yn++guQtnUNOBvz1CSuRQ0L52US/YF4NKmnVN3Ny4DO99+bHXoWESh\n9mMlerG2Axr/FKf8E8bdG81RuQKBgQC2yBBGAJolYaGZyDMgu3CjXuJP1hPuePtC\nuXa2hp/uTf0D0bUg2vxDJjoZVSBFmpSoKaybsUip8gI3GFjJXGmu+No3gX8bRDaq\nH42+I507I1wbYqaWtVW8HmveAK3qi5ed886p7JGYZW9olTj6sfRlvoWa+P5A/9uz\nAUTOlMly5QKBgQDNVNfJEvS6aseoLij8f/SvHeZgWxW3KjtGxF7N1i6IMSX19X1I\nt3GS/2NR6sNvkVzc3C7YLKdtJCgyy1HGJeKOBMxoG6b0wVlH4K+31Vder2oMULs6\nub2tOISjo/KZueivt8W+tF7BG0HNJBDZZNweOOMBa7wEerP7wBRZkIKKoQKBgQCt\n+JpUproRHm4b2wue+glp1iP97TsnXgt5JOGzNUwAHEbYXb/St9wnZbki531CArG/\npXre7czFxM0K96d6cPU+TyoUbrM2lqSZJFNbSLac1TkT77+z7oDd/u6YbXkbpyX2\nd1qbLcoejV2O44lKRBrkxISSTrBh2aWZKXn+Tmu3aQKBgQCu7PIEgAfX78QrQWje\n2owrnV/3DQCtBOh1PyQ5a4LgxsSuvO+qRdsK07tYVxKLc5TOq4DJcmbIzrkEptfO\nWqMUoRk6BT7EfE7AIm2W3K9ohy6U7Jsej7YxhqYvMZllrksqluKbz51K/q1xDDFa\nkkwCvAfMIRx8Z8+2lZkc6yjGug==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/sans/dev-email.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDPDCCAiSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29wAwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy\nNDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB\nAQUAA4IBDwAwggEKAoIBAQCoFArKsSQLasobBGo4+d2WUxjW3rp7qMZ2M92RvefI\nnqsAq3MBstcsKIV6RrvJYiMeCw4gXkrcnYGM56/QtNx6fapKltO8FSOs7dSWLQqu\nvOmYbPHmhDzVyb4/Wcied0i/LyeP5HlUtz/9F22R+MNLhCYacSytogTz9ZoCLLuq\nfaAfP7MdqajDeewSn9fO4AAtGbR7ysYU5lkT7loxW9+VDXN1wKP3U+ziHJVPX2+4\n38OXEVFlM2FCk8qdOja7lmHJnvZcQD7qOywQqEsY+mcjh/+JZk2xbhScSB/bsZFF\nIIfSu5qwUJm2lP9uBksk51FCxWjB1ldAh9mSyH1rT3x9AgMBAAGjgYMwgYAwPgYD\nVR0RBDcwNYIMYXBwLm5hdHMuZGV2gRJhZG1pbkBhcHAubmF0cy5kZXaBEXJvb3RA\nYXBwLm5hdHMuZGV2MB0GA1UdDgQWBBQfZUwRFVCxJAU4eNYLvovRmH6WezAfBgNV\nHSMEGDAWgBSuONpvQZhNfPZRc3eex28/2/eabDANBgkqhkiG9w0BAQsFAAOCAQEA\ndVs5iZ1SPtR7fGQoNnMocd2GECOMZQ5cQz27GtLb8jKVbblBjWYNECUpHA44gWhx\nYhT4gEBww0w7FeQEnWyNMj8vidEm/Fkz4c01zDqwCd5Lh4UyQlWRwppHnV9+j9Xa\nYe2T89L9txLFzkaHZDhbZJy5/SDs2lRy+6IE1vxmiMZdVIzVzHhLQUIBfC1gh0Yc\nqfn2RVbulW11CUgSNXj7G5TPkHw/RG/vs3AsDdgXaCKfwttTirrpFl4OvGKH+xSV\nXMykd7Ck3ScD3oPpp3oY/Nq8iHBB8EgB6SZSSVVL9zl/4jo1s5IBr2ekAhSv9HGj\n29VtK+VVixHazzkGdvS/OA==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/sans/dev-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdqfE5jKdpe3cJ\n1Pd2mwBtfaDnaGsfOxyy7VsIFMGdekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/Y\nGzlntFaAZ7cZMaeCuce6HsE/vofOmMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8\nlRCQLVqHwkuhjlcr8r0A9CWfx0bc2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD2\n7X42IX3tUU5sljVbDd3TFKSyS4l2EebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoH\nftyCbvG/Rxd8LlAeWqgqkdKXUj8gsL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq\n6bvzPXoDAgMBAAECggEAOHkfRQhbDN7jCBwG2a5yjEV0BX9y41hmkraPJUleApzD\nHbFraIXW+zXcWjcVSUDbLat5CpamZWHnd2Zk/P7s/whnXrhY8mu+HS4X9U+3UGqN\nNfINHIrlcZqSak5hQVXeA8uL0v9zsFiU9ckFCkechkUzlvYnxj+nsHhDI8Sa3s95\n5POvYg1VrxfTM7yq8jo9hNoCKd16ghxPXHLHpimZ32qnmje0GqGHvQD1aL83fcaY\nSjSI0OhVf+rkKy8uy63xp20rKj+zCMmLIuV9AiFapiznp5ui7+WAH6oEjMjCzCKH\nSzE4arFgYOjHDF8ElMDhzyOrFg50uGA5Cl1kfn9x2QKBgQDXNSqqZQNCFU5Ah7K5\nwLSedQkHmofN8moH44YXLlK2zfEknXsmu56qoGBRlIVPwYqkXTUg9uDiv7IzE6Fh\neADCB+hiwnDPw6vB17aLJc+uE+XtLYWjjPMpOHyR4HAbgq/s3GGzAxqvtPOLrc26\ngzJReRpN/0/7sodj271LitbCuwKBgQC7jH4SiLuRZWobOrFNDtPBpPBvA56/YNWe\n0gvHfWYCPwt14aF2mkTo+wvY6bzSgy+KmRjCXSyQD9/5GOUtSCIrM5rQvqI3iYMw\n8kkJMDB6b7GEustNydGgpEVl5KqfwUzy+asOnjlqVO3FZSwvJlLNYVtgjRzGWsZX\n1pcL/0BlWQKBgQC3u5yBfVwavzXfwfrEu9F4t1LOOxBWs+/ybD2+7L2RdKG8MPdD\nSktcQS/6dmKahRJo4WrKifvVmvP4x3mwTVPYVAgCyR6nQtcQ16nxgoaciEB1Dbha\nuaugNamkoYkU865e+ogu3Sebe1GynuBVrEz5YfsjPCZ9LR6KEsC6Df7soQKBgQCo\naNrYh8FuKRPjiYumN1c8/oAMH8MP9MM+Dz0WkGrmP3hqDQaw+oxAbRXRXOn6WmR5\nX0pVVddrMWYcRxeb1rcf9gHhyhzeI/QTIq0kvAn8F4nfNuDSZBSB3KCYg4IXXDtd\nv4Wz5a6G5eZwp43KdO5LkE2+YFhjYSXTwFT4J+fKWQKBgCyoF/Za1xghz/+dmH/K\nvZtARsJgZjWIknsBdTwPEcqZjV7OBe7IKRutPgVnzHGt+3gWLqXQtz9uLxAMw/8B\nk1JQuEKEeuAltA2bE6vp8NXP7kC38oR/9Z7aHKru9+8nrrG/dVK28PrSymdd4keB\nJ+vQW0oEllTH7M7Wgyy3r4O6\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/sans/dev.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDIzCCAgugAwIBAgIUTOgYUed1dyp+Gan755liL7b29v8wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy\nNDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB\nAQUAA4IBDwAwggEKAoIBAQCdqfE5jKdpe3cJ1Pd2mwBtfaDnaGsfOxyy7VsIFMGd\nekkJ7KIN6wX+O0nFT8X8lpC7YnIgTzpZl2/YGzlntFaAZ7cZMaeCuce6HsE/vofO\nmMrNnT2pEp7z0DZn7TzjMdCAyeCnG3N4XGe8lRCQLVqHwkuhjlcr8r0A9CWfx0bc\n2zA1ZkiprnF3Qf0KbN4SfR+oHczsEO48yXD27X42IX3tUU5sljVbDd3TFKSyS4l2\nEebzhX1bNUj68jvxrDgo5ZTrMSIaiMtLSsoHftyCbvG/Rxd8LlAeWqgqkdKXUj8g\nsL4dCfzRjj1MjWSx1Mjcjp6ujHRyXeFrbvSq6bvzPXoDAgMBAAGjazBpMCcGA1Ud\nEQQgMB6CDGFwcC5uYXRzLmRldoIOKi5hcHAubmF0cy5kZXYwHQYDVR0OBBYEFDvM\nqcu0rwkeE32c0A2VgPLmf4eDMB8GA1UdIwQYMBaAFK442m9BmE189lFzd57Hbz/b\n95psMA0GCSqGSIb3DQEBCwUAA4IBAQBxVdfxsnSvL3WTuZroo+inViR8T7RV2Dys\n8NiwvO1o+uNATmzV5v2iLux2INEXBgt+71OqzcPPiTvjHcBkh6aTCk2N54mYRK6Q\nL3d7wfeYJhHUfeC/idM5c8UQVHkKa7Em+6D/IoWnzen6QTs2CtxJ+UE4Ie56DTHQ\n1qc96IqNUKVAq61tZe9u89sCsn9S5r/cyBa8pTMzfjZ6FHJFC/MZHu8InfNJ13MB\nq5edoCH761+MbyXz/JyrU2hTcno0z1ZSS6VteehDPdnDPjysoZPCjn1jP7G4rZ3W\neevtdw2p2SP+jdFt5Hy6tvy4sXtj79C4MHVbsAVw6hkE5PNoNRM3\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/sans/prod-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDQhGgjBMhK3sIJ\nIAKwXPUd/QrxbHQ+fGkXsHm9EIMzm9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0Aiu\nV+tHoilSzEZhfTRsVz+CPDBotyW8rK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6\nY0+klARJFsOp2kDhCyjW2Vz+M4pJIMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtn\nU4uypTXqKAYUQX88TS4mCBgxPYh7XZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUck\njPJkJNBaLcGgAJq3Msd7t2kXulmIJTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyF\nLe6T6NopAgMBAAECggEADzlRlJfzazL5Gfs0Mw3waAZ68246jPrenegItV70o1Ka\nQB2cKDHDTxi8fWHi5oB/2zhJLPrHxuS7s4XwA8iBSZ4oN9MxPgU/Oeoc0zJm/w/g\n9a2jh8xhUYpGwkiiSthjIZO7EEGiJOG5AlNQo+iaKP9wXPD2Sm/7BPe1AErBSo4o\nIyXhnx3e32sGAQekR/1AqAl2elXsFeO/nTYOBABd1Tv3RQW7JFtSiDKIIQV50Vhl\n2kdFqRlqfEt0JJeRB/hJZloQrKNOR5JUymn497kvmHeLOzNMq7+65naY86BLy+pu\nB6uBGgo3AZibfB8R95YpPTj+hn5YsGzYztMKRZ5PUQKBgQDwjXDqTMRdtoWVblvF\n89PY2bF4dvyo1WW5FWN7kbjjOmwz+rhsc3ct8IDhxdTh84gxv6+nQfNaXCmPl7LP\nY4Qe6PC1GlhgC/XP0CSb4YhqIlk8Ja1yLOVttT/u77Fd9WRLYHWQqWBMVJvPg+9q\npdXsfIvjQg3cYWrM8l7wzKuJEQKBgQDd6FJ0H9bTFYnMc60BHSujd/xTlxEgkS/2\nc+cFvVJ+Z8+kmNRxLWGeuj/pFh1Lg2o1dP5WH+1oT+gq50MHNyAqLP2+mY/iZTaD\nxp6+ChAZpg6FLxKfMr6vv8rfA/6259RjqrGKzVT7ZD9yHAkG86n+ZHvXxBIkVEOF\nWslsBhz/mQKBgAKYVAaDC39DT8+b4CuThM6LEqoNBo6/tpg3jdowaUEySSaKZuDs\nofWB6bIF01UYrnhy6+8u+/QWmHAvH8Oy/CLlOmuJvDhVCLLUOQDhfvo+Ip5Ofb+E\nknkLgoQOW+h6lDln8fy8hwhrxT6I2tVYsqUZdtzdwo6FEoThfHq2iocxAoGAS0Ri\nuzqVbhFvJObTdn5Db/XhoySpTYeRYiGb6Yl2sqNZfbXM0PqYkvMPfGrg0t0nKCyf\nm+zGZMw9rle0l3zuLwAFrSF++UZlQDsdWs4k+d4mLKxzv4XBwfDmydAcFEP7+TZL\ndA4DViWBk53Ivg/lyACjNOMvWB0wrcnGahqEuXkCgYAh47V2/4fEHLWwbsnvWL8T\n6C56/8RUBTvlF2vFtRA/THmYV34pGoM0EWLQ8th4mQQ4mPo6JcQ6L6K1zjw87RJ2\nzvlWhCNZdOUFtrsoVwuRsP3U8PW+qYiDasbNHJoy5UC5jTI5byluM802Vw+9uQ+l\nwHleMjtW69bGbRbHWDOVvw==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/sans/prod.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDJTCCAg2gAwIBAgIUTOgYUed1dyp+Gan755liL7b29v4wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU5MDBaGA8yMTI0MDMy\nNDA3NTkwMFowFjEUMBIGA1UEAwwLd3d3Lm5hdHMuaW8wggEiMA0GCSqGSIb3DQEB\nAQUAA4IBDwAwggEKAoIBAQDQhGgjBMhK3sIJIAKwXPUd/QrxbHQ+fGkXsHm9EIMz\nm9krVfVvsAqjrZyWHqiJ16N66+ysWqVe0AiuV+tHoilSzEZhfTRsVz+CPDBotyW8\nrK25SgLRoQ3AnmG2Co6s1Y05HNMyuUIpoXg6Y0+klARJFsOp2kDhCyjW2Vz+M4pJ\nIMY69y4IGjlCWOc4IiQKKIJjD12FxAe0/wtnU4uypTXqKAYUQX88TS4mCBgxPYh7\nXZrWfwZzYkXnUAxXy3OIqXtI2ZuQ3MkgaUckjPJkJNBaLcGgAJq3Msd7t2kXulmI\nJTnpw0D4jd12v19wDJCxTuaBdiUhhkZX9fyFLe6T6NopAgMBAAGjbTBrMCkGA1Ud\nEQQiMCCCDWFwcC5uYXRzLnByb2SCDyouYXBwLm5hdHMucHJvZDAdBgNVHQ4EFgQU\nKnk97WhPStd4LcWf52kZxmG4PVowHwYDVR0jBBgwFoAUrjjab0GYTXz2UXN3nsdv\nP9v3mmwwDQYJKoZIhvcNAQELBQADggEBACiIPya5Mgy2y4+PrBv6KbYBvpSMA0+g\n6OCq3pX4+okIGBoytjL+0DJchEAVGHxf6qnMWSQKnBFgjCHbi805JgJu819PDWFr\nlyWsH/YuWP/Df9zl6KiXbrLUBBnGT+bxaqc/FVEsJrhB+29kIrDmxdcL/CcmihwF\npAeYMDs7zjn3W8BQiBG+odapzP7VHsA0L/HpWOZrvJnSIU4vAujnn6SYRyk5y1T1\n7o7nSKo7Fh4YjteTUGzOKFKDS+Mk3SxXA+TIqskV0ZWgZD4KebN69VZWRXgDwTpT\nE9jt46Yzmk3RD1NsrfwmgNSGet1+mOzjOLzzXjlxl7qmpYvbyx8U+9c=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/sans/server-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLLgBYDBSeNemh\n8b9yPmGiaJkSkJtaXL7ESe1h/aNpdCRrkja7XdHqQ20a2H9X94nXuzvFbpJw8VGT\nq8ifGet3+++AJvEP+LfK7HSIvnyewKXZ+Oqf5AJg1YIb3k31VGzGEjjjnW7F/Gw4\n2qKyZ7+CEnIFbo4lLFBoTGvzJo2OwDVKTCby9E428te7rH1+F0OOANSG+SLzl5Vj\nbidr4OryTPjO2qJSs22rVnvjZdcG38Z1ams6s4fxwQWvuD9qBBrb0GGtvVP2cYBe\nXfOTOydS5IxOerb0EbbijudXVh+xH3dRgvfeJZWTqegCktT6OEa1pe7Q3BoMLtk0\nHJO77RtrAgMBAAECggEAMe91YC5f5t5jNE0A+2f/gPXqIRjvRY8jx4RP1lSLFADa\nbqG51+TgEY5Ow1lQOuN4uk+nCgf5784veXav0QKCz7NYkot/lahQK4xfu7ftUusQ\nF/lIWNGmD/yJhKXnTRNZxHh4COEJd5lpU7PQZr5+383+vf0E7HfuryORKGpiQiHl\nF1ZOYFG2GjFZychMTe+ggQISJqX8U2bE1mIhZNSkBvStt9LDIyWsvbud3j5DHPv/\nnX3EEymKwDiMrkNc+HTodJ3Ci9cSVGJ6igAODeu5wwddn9rpP4obOvJW5B2NuNXR\nOirU9o4Hd4cPcUqE8FHloEmIKmALqCQgH3gu/+euUQKBgQDoR4IMFRe4d+G/FJLE\nMUnQ1H1ipHOaZcLl0bBFeEtkpuqC5MDoCDJuQCeNrohSrE0dh4z15lq8G79eucYH\nE4vbmsyXAm+KgpYZ5S0wvMDoPaXxOSUJe95eUqeGDl3vcexSOXph7/syxXft7MpA\nJGFirKH16WiKn4u30siaNK8feQKBgQDf7b1pEdUr+ajhrJ0+LwC+/Q12nBgrULvf\nN1rOBXrXX+yI0esk0/i6EWThTZP0iyh/CpxZQyfsqQXBG94spJLGa/C3dcOWohQ4\ntTIYZN9uSozO8OXU+WEHM41kdOByItIpFVzJ2VQ711NizZxchsVUPexUxW99rxuy\nJYPe891lAwKBgDX1JoiX/cKkVpSEuvNIX9VbByV6/j/Hk9a/NytsHldrTt7JNOax\nwMVLseR/vrs1u1Q4wM3+jOVVrMbccNwD9mE1PIF312FdpHCvhCfmMQnCCGJY2/mw\nBJ/0o2XIwJl25WbUY1GM5dWNeaaTcwgja1v4rkbpyZcqKNKy7FamFmhBAoGADTg+\nz14jJJZ3luMW4V3rAFt2GQN8FcqNXM8qyDDgoKhkEWu7IxB0hZ+TQg3PZ0dz25Qn\nyMlDKSCNW2omwqHwnFAxBNOp+VGWOgbQb0o9OoiiKXFlUB5s0P0FIuIXggV2PMCr\nGqt+4o2CubmDup6mNaP2OMbzHwtgajL2xwnEjzsCgYEAhEQwa0t+x2+/vZ1NOQW7\nhp7vM2/DyAZnqgAdCwXnlmVayxR4r9OSDZmz5vqJWamQ0UIWYnTkvLN962hYdIC1\neIQ3yEjxIgZj9Kb+6vnHI81kkjg8LK/XG87HFBnL9juNgdbNQVIyMI84FM2jm+AP\nxReBzjjjrY3NrcJ8B6a4AUA=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/sans/server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDLDCCAhSgAwIBAgIUTOgYUed1dyp+Gan755liL7b29v0wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA0MTcwNzU4NTlaGA8yMTI0MDMy\nNDA3NTg1OVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEAyy4AWAwUnjXpofG/cj5homiZEpCbWly+xEntYf2jaXQk\na5I2u13R6kNtGth/V/eJ17s7xW6ScPFRk6vInxnrd/vvgCbxD/i3yux0iL58nsCl\n2fjqn+QCYNWCG95N9VRsxhI4451uxfxsONqisme/ghJyBW6OJSxQaExr8yaNjsA1\nSkwm8vRONvLXu6x9fhdDjgDUhvki85eVY24na+Dq8kz4ztqiUrNtq1Z742XXBt/G\ndWprOrOH8cEFr7g/agQa29Bhrb1T9nGAXl3zkzsnUuSMTnq29BG24o7nV1YfsR93\nUYL33iWVk6noApLU+jhGtaXu0NwaDC7ZNByTu+0bawIDAQABo3YwdDAyBgNVHREE\nKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD\nVR0OBBYEFG5wB9htqpiQze/q1+9rPaq1HlcpMB8GA1UdIwQYMBaAFK442m9BmE18\n9lFzd57Hbz/b95psMA0GCSqGSIb3DQEBCwUAA4IBAQA1tni0pG6LtlStmEUmn9xJ\nmcWlWgZtmUbZD7aE5P+SFLt02TY9ea7IoMy3Rv1yRihxLgURNB9Y8aON9IrraVRf\nU+EizfG9goQQINs652TNMoxjNEwbF19TzgY7k445gkcl8aZ5fGJJhxP7UGBC5xYL\n1Cx0uINiqs0fb3pPrcvhxD+I5tHDwJY7maW0fks/JD1M738hl5zpV1POMvarDFW0\nL4LUDweUuNeylOWyCBv6WFV7avSsMmoQmUvUZoPqp2gk0A17pDeuufI84fHhEghU\nUigXcElvZkaoThbE3YA0Hq8UJHSzAe8x0bDSRz4WfNzeqq1RtdZ+H1uPoMIJbZ4D\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/server-cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 1d:d9:1f:06:dd:fd:90:26:4e:27:ea:2e:01:4b:31:e6:d2:49:31:1f\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:e6:fb:47:65:cd:c9:a2:2d:af:8b:cd:d5:6a:79:\n 54:3c:07:5f:eb:5a:71:2b:2b:e5:6f:be:31:fb:16:\n 65:68:76:0e:59:e7:e4:57:ca:88:e9:77:d6:41:ad:\n 57:7a:42:b2:d2:54:c4:0f:7c:5b:c1:bc:61:97:e3:\n 22:3a:3e:1e:4a:5d:47:9f:6b:7d:6f:34:e3:8c:86:\n 9d:85:19:29:9a:11:58:44:4c:a1:90:d3:14:61:e1:\n 57:da:01:ea:ce:3f:90:ae:9e:5d:13:6d:2c:89:ca:\n 39:15:6b:b6:9e:32:d7:2a:4c:48:85:2f:b0:1e:d8:\n 4b:62:32:14:eb:32:b6:29:04:34:3c:af:39:b6:8b:\n 52:32:4d:bf:43:5f:9b:fb:0d:43:a6:ad:2c:a7:41:\n 29:55:c9:70:b3:b5:15:46:34:bf:e4:1e:52:2d:a4:\n 49:2e:d5:21:ed:fc:00:f7:a2:0b:bc:12:0a:90:64:\n 50:7c:c5:14:70:f5:fb:9b:62:08:78:43:49:31:f3:\n 47:b8:93:d4:2d:4c:a9:dc:17:70:76:34:66:ff:65:\n c1:39:67:e9:a6:1c:80:6a:f0:9d:b3:28:c8:a3:3a:\n b7:5d:de:6e:53:6d:09:b3:0d:b1:13:10:e8:ec:e0:\n bd:5e:a1:94:4b:70:bf:dc:bd:8b:b9:82:65:dd:af:\n 81:7b\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n 2B:8C:A3:8B:DB:DB:5C:CE:18:DB:F6:A8:31:4E:C2:3E:EE:D3:40:7E\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 54:49:34:2b:38:d1:aa:3b:43:60:4c:3f:6a:f8:74:ca:49:53:\n a1:af:12:d3:a8:17:90:7b:9d:a3:69:13:6e:da:2c:b7:61:31:\n ac:eb:00:93:92:fc:0c:10:d4:18:a0:16:61:94:4b:42:cb:eb:\n 7a:f6:80:c6:45:c0:9c:09:aa:a9:48:e8:36:e3:c5:be:36:e0:\n e9:78:2a:bb:ab:64:9b:20:eb:e6:0f:63:2b:59:c3:58:0b:3a:\n 84:15:04:c1:7e:12:03:1b:09:25:8d:4c:03:e8:18:26:c0:6c:\n b7:90:b1:fd:bc:f1:cf:d0:d5:4a:03:15:71:0c:7d:c1:76:87:\n 92:f1:3e:bc:75:51:5a:c4:36:a4:ff:91:98:df:33:5d:a7:38:\n de:50:29:fd:0f:c8:55:e6:8f:24:c2:2e:98:ab:d9:5d:65:2f:\n 50:cc:25:f6:84:f2:21:2e:5e:76:d0:86:1e:69:8b:cb:8a:3a:\n 2d:79:21:5e:e7:f7:2d:06:18:a1:13:cb:01:c3:46:91:2a:de:\n b4:82:d7:c3:62:6f:08:a1:d5:90:19:30:9d:64:8e:e4:f8:ba:\n 4f:2f:ba:13:b4:a3:9f:d1:d5:77:64:8a:3e:eb:53:c5:47:ac:\n ab:3e:0e:7a:9b:a6:f4:48:25:66:eb:c7:4c:f9:50:24:eb:71:\n e0:75:ae:e6\n-----BEGIN CERTIFICATE-----\nMIIE+TCCA+GgAwIBAgIUHdkfBt39kCZOJ+ouAUsx5tJJMR8wDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOb7R2XNyaItr4vN1Wp5\nVDwHX+tacSsr5W++MfsWZWh2Dlnn5FfKiOl31kGtV3pCstJUxA98W8G8YZfjIjo+\nHkpdR59rfW8044yGnYUZKZoRWERMoZDTFGHhV9oB6s4/kK6eXRNtLInKORVrtp4y\n1ypMSIUvsB7YS2IyFOsytikENDyvObaLUjJNv0Nfm/sNQ6atLKdBKVXJcLO1FUY0\nv+QeUi2kSS7VIe38APeiC7wSCpBkUHzFFHD1+5tiCHhDSTHzR7iT1C1MqdwXcHY0\nZv9lwTln6aYcgGrwnbMoyKM6t13eblNtCbMNsRMQ6OzgvV6hlEtwv9y9i7mCZd2v\ngXsCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nK4yji9vbXM4Y2/aoMU7CPu7TQH4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA\nAAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI\nKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI\nhvcNAQELBQADggEBAFRJNCs40ao7Q2BMP2r4dMpJU6GvEtOoF5B7naNpE27aLLdh\nMazrAJOS/AwQ1BigFmGUS0LL63r2gMZFwJwJqqlI6Dbjxb424Ol4KrurZJsg6+YP\nYytZw1gLOoQVBMF+EgMbCSWNTAPoGCbAbLeQsf288c/Q1UoDFXEMfcF2h5LxPrx1\nUVrENqT/kZjfM12nON5QKf0PyFXmjyTCLpir2V1lL1DMJfaE8iEuXnbQhh5pi8uK\nOi15IV7n9y0GGKETywHDRpEq3rSC18Nibwih1ZAZMJ1kjuT4uk8vuhO0o5/R1Xdk\nij7rU8VHrKs+DnqbpvRIJWbrx0z5UCTrceB1ruY=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/server-iponly.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 21:8e:d4:f9:10:fc:32:5f:46:45:23:53:14:03:f0:b2:dc:a1:d7:e5\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=ip-only-localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:bd:98:dc:14:49:92:43:1a:73:36:75:6d:c1:33:\n 73:b3:e7:f6:8a:e2:03:f7:24:93:eb:36:da:8f:64:\n c6:56:19:b7:92:d2:f1:df:d3:4f:7e:78:ce:db:dc:\n 6e:d4:5c:58:a4:a7:79:6b:49:ff:30:37:d9:71:92:\n 21:d7:91:72:8d:1a:21:75:0f:80:23:f0:2f:0e:6a:\n 1a:62:9b:eb:1e:74:df:df:61:8a:05:a8:31:b7:d4:\n 97:ee:c6:60:1b:6f:b1:85:9b:ac:3d:cd:a1:9e:a8:\n b5:56:84:ce:9f:8c:64:a5:57:41:75:81:c6:00:c8:\n d8:3d:ef:85:9c:78:04:49:5a:94:29:af:27:0a:be:\n 56:60:6b:a0:52:06:53:f5:cf:ce:47:9c:cf:50:8a:\n 1d:92:3a:21:d1:24:ed:81:e8:85:97:8c:41:46:96:\n da:02:d7:b7:f5:ab:92:4c:ab:0a:e2:19:9f:b8:21:\n 07:5c:f7:d1:7e:6e:49:70:a9:c4:45:c3:b2:10:ab:\n ca:50:e3:c9:ea:6f:b5:19:64:12:32:0e:93:59:f8:\n 8e:fc:a2:bc:33:f3:25:e4:59:98:6d:0e:97:76:b6:\n da:9e:50:cf:4d:a1:6b:62:ce:47:84:1d:f7:82:8a:\n da:d0:90:e7:a6:1f:43:25:61:57:c3:15:7e:e0:b8:\n 5c:17\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n 0A:AC:82:F2:35:73:89:FB:49:07:EB:9C:87:72:D9:91:26:DF:84:8A\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 87:00:2d:ec:ea:ab:13:3e:21:6c:4a:3e:f1:9b:c3:ab:c5:81:\n d3:34:2d:cb:f1:e7:98:96:13:a3:7f:29:00:83:70:20:81:56:\n a1:31:b7:d9:fa:35:02:f4:6f:32:8a:58:cf:68:39:e1:f9:fb:\n 50:20:60:85:f1:bd:22:73:e6:ac:b6:36:38:75:86:de:8a:cf:\n e5:41:df:0f:bd:61:94:3d:3a:59:81:50:c5:44:39:47:d9:e8:\n 56:2a:b4:2c:59:b7:89:f5:5d:a7:28:15:6e:98:1e:47:47:7d:\n a6:35:65:de:b7:06:9e:04:67:98:04:f9:57:54:82:2a:dc:e2:\n 6a:76:df:06:05:b4:61:84:ec:7d:5e:46:34:ea:14:b7:14:16:\n fe:51:0e:65:52:c6:74:96:8e:30:3a:b9:c2:d5:05:2d:a8:08:\n 11:ba:7e:65:c1:89:c3:61:ec:d6:06:e0:7e:ac:11:ee:18:89:\n 67:59:1d:6b:dc:cc:1e:48:c5:79:5b:8f:18:7e:e4:c4:9f:4f:\n 3d:a9:11:60:28:77:49:e7:23:4d:fa:49:8d:6b:5a:52:57:9b:\n bf:a7:d7:da:13:45:64:dd:d3:2f:4c:14:eb:79:46:8f:fd:41:\n ab:88:17:d4:8e:77:45:8c:7f:cb:e5:d8:19:7c:1b:1e:cd:21:\n 32:43:a3:c9\n-----BEGIN CERTIFICATE-----\nMIIE9jCCA96gAwIBAgIUIY7U+RD8Ml9GRSNTFAPwstyh1+UwDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowYjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xGjAYBgNVBAMMEWlwLW9ubHkt\nbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvZjcFEmS\nQxpzNnVtwTNzs+f2iuID9yST6zbaj2TGVhm3ktLx39NPfnjO29xu1FxYpKd5a0n/\nMDfZcZIh15FyjRohdQ+AI/AvDmoaYpvrHnTf32GKBagxt9SX7sZgG2+xhZusPc2h\nnqi1VoTOn4xkpVdBdYHGAMjYPe+FnHgESVqUKa8nCr5WYGugUgZT9c/OR5zPUIod\nkjoh0STtgeiFl4xBRpbaAte39auSTKsK4hmfuCEHXPfRfm5JcKnERcOyEKvKUOPJ\n6m+1GWQSMg6TWfiO/KK8M/Ml5FmYbQ6XdrbanlDPTaFrYs5HhB33gora0JDnph9D\nJWFXwxV+4LhcFwIDAQABo4IBkzCCAY8wCQYDVR0TBAIwADA5BglghkgBhvhCAQ0E\nLBYqbmF0cy5pbyBuYXRzLXNlcnZlciB0ZXN0LXN1aXRlIGNlcnRpZmljYXRlMB0G\nA1UdDgQWBBQKrILyNXOJ+0kH65yHctmRJt+EijCBrgYDVR0jBIGmMIGjgBT+z38+\nLu7Erepbb4hFbMeISBSLOqF1pHMwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh\nbGlmb3JuaWExEDAOBgNVBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAn\nBgNVBAMMIENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDIyLTA4LTI3ghRJnBbtu1z0\nRRusxa2MelszQLZtITAhBgNVHREEGjAYhwR/AAABhxAAAAAAAAAAAAAAAAAAAAAB\nMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYB\nBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcN\nAQELBQADggEBAIcALezqqxM+IWxKPvGbw6vFgdM0Lcvx55iWE6N/KQCDcCCBVqEx\nt9n6NQL0bzKKWM9oOeH5+1AgYIXxvSJz5qy2Njh1ht6Kz+VB3w+9YZQ9OlmBUMVE\nOUfZ6FYqtCxZt4n1XacoFW6YHkdHfaY1Zd63Bp4EZ5gE+VdUgirc4mp23wYFtGGE\n7H1eRjTqFLcUFv5RDmVSxnSWjjA6ucLVBS2oCBG6fmXBicNh7NYG4H6sEe4YiWdZ\nHWvczB5IxXlbjxh+5MSfTz2pEWAod0nnI036SY1rWlJXm7+n19oTRWTd0y9MFOt5\nRo/9QauIF9SOd0WMf8vl2Bl8Gx7NITJDo8k=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/server-key-iponly.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC9mNwUSZJDGnM2\ndW3BM3Oz5/aK4gP3JJPrNtqPZMZWGbeS0vHf009+eM7b3G7UXFikp3lrSf8wN9lx\nkiHXkXKNGiF1D4Aj8C8Oahpim+sedN/fYYoFqDG31JfuxmAbb7GFm6w9zaGeqLVW\nhM6fjGSlV0F1gcYAyNg974WceARJWpQprycKvlZga6BSBlP1z85HnM9Qih2SOiHR\nJO2B6IWXjEFGltoC17f1q5JMqwriGZ+4IQdc99F+bklwqcRFw7IQq8pQ48nqb7UZ\nZBIyDpNZ+I78orwz8yXkWZhtDpd2ttqeUM9NoWtizkeEHfeCitrQkOemH0MlYVfD\nFX7guFwXAgMBAAECggEAE6qdeYVAJLHDrax0nAvIPqsbCcD0BFjI9ycTeLhNUnUM\nc7Bp4nu6zTWez3OIE4MYtsugbp6YV9oTNhKgbAnsRfKl8cyP0CqD1wzue7gMpXYe\nGr+1X2zY62aj8+Kj6XSmh2NkdGy2DQ0W8kiIXkhj0DrC0XuKnF45AAOualKQr0MG\nJRFXi/AEEDkyWwfmqteD60HBzdweBqzYvsIyA8cQAgI4LguRKHKmLexqNDL0h0K+\nnPbQl5pg7es3kOaQ7DMICqd+E11btSXBJop95MLVhPXkzAyktIdfc1M7LsBNb7Dw\nja5G+wmmGG0KxCbqMVKCACxs63lWakIxFlrsxi+Q4QKBgQDwrM4f/j289+PBKlED\nUCXHsGBgE1vNZMAinfAazWAImTAwWoY0FDrMBBPJGx+Azke5teI82o+Sq4jTL44Y\nqvmqblHb9XEQLnRxSdl5WP794o1K9pP7nEvIpL+RngRIC1qt9BEGd5Pmn8FcF4BT\nZo9sdP8Hh6EJLGb33iYxUrTNTwKBgQDJq3GclCnVukbaGbCU2w54EDTJoghIY+W4\nkXeGL1VbSXD/NnyuhH/mua+UwBM7XSFJXKV2vfIhI1Qe9VW+7p4vyiLFp3atSb86\nAirajUPrCJNX1X/sm7fIss9U1ko/JruXSh1DxzEZvBQa52SwYH5bM6LbE6NJRuhS\nxMY2ipyiuQKBgQCm0gCl6GH+w4wYbi5tL3agbT7AGWr+eSE8XWD6EvTHwPbH7Vcs\nbgE7PHBCawxxCYppzQqdx5jQvxk92K6Tpp8bZRBUeFIAN1L624dkNy236PqqxTNZ\nqcJVtuwaEP9CuKwH+y553xSjPISYQqnuJR6wvH+xRm92nlJY6KBse7lavQKBgCEV\nNeMIz0AXec4HjtcshFgf2HkHUrKFaMb5XhEuLKN4DchgKN38MHsqFOqjA8SmR3Kg\ndyheipzzDbayammS/XI7h67DBQ3yXiNm/Z6ys+SXmIw9IuoutVyAMNDrAm0PrpBo\nARsAT0a4etfbA8KHYdMWSm4D77JypmQFkbqazI1JAoGBAKZTn6KQaBcZ5JrSya17\nsClZPlSsYhIW8EbxqDgNwDippcNrLuJCsWiwMNW+glQoDIcGAo557PwAXrridH89\n5mjFCVggHeso3T6/nsBB86Q8B1hq5E8AszhEKH6IeKd5tFNZkK83eWjAlbIUrEBH\nxXZFMGnYPPS7korkEzzuGJ+7\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/server-key-noip.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCt8Ic/MmaHejGb\nylQKrqYayiXVxfxJayEL3qcVyJw8zUEdMiV3aHuD6F0Uei4L6kGRpCDsIBcPy41M\nG4ig0ndGZX7RoOZMS8aMOaGzWzRXyKEQDBNUOnSQezu62kFigfXctXNsgzj0oVKr\nvcKVPnn/r6Su39YR2SkguLQV4zKTXDbOVrQBAqFFMaOhHuq4xAEEVxFE9FXq4q5o\nCHCFwFv/ur/ei7yhxgOiL4rrnrd5OmdqsHDT6AinEiTVu1eIcjfI5i7bh+AqcRos\nkJyIKQx1KITWf3UtUAg2K8/zujNyHnoH2yDamDs5hpZM4kpCYRqbC2dNbRPRn0Df\nEseNnVBpAgMBAAECggEAcmiqXRwmqmfqZ4Ge4+Pap/ZdCo6OkjAf7XHHTyHD+o47\njRul3zPfQnU9fDGdRgMQm95sNUQqRx5pUy0tIjMtdyVdVD9UG80fzK4/uPx9olv5\n7Nc0g4trjnkwYYgbx9KZyFGlmTN67BWMjiBj88zDbDW4ybm7UcQYNEipU1g8tQW1\ntUwcZ1oahXfzO75vcMqDVlS2IE0s0AD9sh+AaJIwxV9kSLNjlSwkpsH6PBKKB/3r\nWvG2p6Og1whdQ54PGADUVSx1yWFyXQDeygqLmryEWaHJQz1jt7bvaaAMy2PTdwVf\nA5LVG3VHkoQOBv8imtpCbU2J7zAk9ypDuRUlpa8h/QKBgQDdCCCbV02BhrqDYchm\nojB95Vx8KtvQdXhvsxShxyuIktuB7W+NnheBmLY0TNcYSQyzithCUBhtmyaC5S4f\ndHmT52e7HS0xaL9r9BhAQrtWReMcplKB1IIXtdYXEY3qOjZMxX3seJo0iBWS3hMH\nEG6tC6tlr5ZXOKJOrBMGuMgplwKBgQDJdSYkC3AX2p+4BNf3hgQyzotuSVSbx/zu\n0ZHhi8Wp7yF49c8+9+ahO9AMrVM0ZSh2buznfF46FNC/C55M7a9Rn60sFQQ16b5L\nrJTzlPoUGTnPLt8C3TdMIFg/5cAW6ZgZWNlU3aVU0W34NVh/H2m/M72tGrk250zs\nYhZ8/RGV/wKBgQCKlMfs3YXoyhIywaImR1Zj+ORNrYl4X86NKhirffbbgEhEZBvn\nDNHsHVVP4UWTImnmQA1rNlC6l+ZDd3G9owd/Jj0xYg+txOEPzFFQKQbQBq1ojxd3\n80dFmmqKuCTkUG8vHzvegZcdjJ0KIlaHvVPHB2QFM1vtf8Kz1MtxEXXeLQKBgDn0\nBm3WEH/8N3gzhIFDP0/yVO/8DmfmByAYj5PHpqw1C3cFl4HwxJrbXwVWkxn+g75W\nOLZ684xX0pky2W4d7hJYEfQdc6GixUh1tD/COpKvkw7D2Am146N1po1zJWgx+LxJ\n7/NW86nLuYvupK+lNMF5O/ZhOqjNrzZNHVUFZBq3AoGAPwixh7/ZMX6mmm8foImh\nqibytx72gl1jhHWSaX3rwrSOO9dxO2rlI7LOZQrarU632Y9KMkP3HNbBHPRkA4MI\n6I9wqawRzGjcpeXIMlPzOHDHYLyrTpEzo8nrSNk/cM8P4RxE12FqySzQIkiN06J7\nAxJ7hVqtX6wZIoqoOa9aK1E=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/server-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDm+0dlzcmiLa+L\nzdVqeVQ8B1/rWnErK+VvvjH7FmVodg5Z5+RXyojpd9ZBrVd6QrLSVMQPfFvBvGGX\n4yI6Ph5KXUefa31vNOOMhp2FGSmaEVhETKGQ0xRh4VfaAerOP5Cunl0TbSyJyjkV\na7aeMtcqTEiFL7Ae2EtiMhTrMrYpBDQ8rzm2i1IyTb9DX5v7DUOmrSynQSlVyXCz\ntRVGNL/kHlItpEku1SHt/AD3ogu8EgqQZFB8xRRw9fubYgh4Q0kx80e4k9QtTKnc\nF3B2NGb/ZcE5Z+mmHIBq8J2zKMijOrdd3m5TbQmzDbETEOjs4L1eoZRLcL/cvYu5\ngmXdr4F7AgMBAAECggEBAK4sr3MiEbjcsHJAvXyzjwRRH1Bu+8VtLW7swe2vvrpd\nw4aiKXrV/BXpSsRtvPgxkXyvdMSkpuBZeFI7cVTwAJFc86RQPt77x9bwr5ltFwTZ\nrXCbRH3b3ZPNhByds3zhS+2Q92itu5cPyanQdn2mor9/lHPyOOGZgobCcynELL6R\nwRElkeDyf5ODuWEd7ADC5IFyZuwb3azNVexIK+0yqnMmv+QzEW3hsycFmFGAeB7v\nMIMjb2BhLrRr6Y5Nh+k58yM5DCf9h/OJhDpeXwLkxyK4BFg+aZffEbUX0wHDMR7f\n/nMv1g6cKvDWiLU8xLzez4t2qNIBNdxw5ZSLyQRRolECgYEA+ySTKrBAqI0Uwn8H\nsUFH95WhWUXryeRyGyQsnWAjZGF1+d67sSY2un2W6gfZrxRgiNLWEFq9AaUs0MuH\n6syF4Xwx/aZgU/gvsGtkgzuKw1bgvekT9pS/+opmHRCZyQAFEHj0IEpzyB6rW1u/\nLdlR3ShEENnmXilFv/uF/uXP5tMCgYEA63LiT0w46aGPA/E+aLRWU10c1eZ7KdhR\nc3En6zfgIxgFs8J38oLdkOR0CF6T53DSuvGR/OprVKdlnUhhDxBgT1oQjK2GlhPx\nJV5uMvarJDJxAwsF+7T4H2QtZ00BtEfpyp790+TlypSG1jo/BnSMmX2uEbV722lY\nhzINLY49obkCgYBEpN2YyG4T4+PtuXznxRkfogVk+kiVeVx68KtFJLbnw//UGT4i\nEHjbBmLOevDT+vTb0QzzkWmh3nzeYRM4aUiatjCPzP79VJPsW54whIDMHZ32KpPr\nTQMgPt3kSdpO5zN7KiRIAzGcXE2n/e7GYGUQ1uWr2XMu/4byD5SzdCscQwJ/Ymii\nLoKtRvk/zWYHr7uwWSeR5dVvpQ3E/XtONAImrIRd3cRqXfJUqTrTRKxDJXkCmyBc\n5FkWg0t0LUkTSDiQCJqcUDA3EINFR1kwthxja72pfpwc5Be/nV9BmuuUysVD8myB\nqw8A/KsXsHKn5QrRuVXOa5hvLEXbuqYw29mX6QKBgDGDzIzpR9uPtBCqzWJmc+IJ\nz4m/1NFlEz0N0QNwZ/TlhyT60ytJNcmW8qkgOSTHG7RDueEIzjQ8LKJYH7kXjfcF\n6AJczUG5PQo9cdJKo9JP3e1037P/58JpLcLe8xxQ4ce03zZpzhsxR2G/tz8DstJs\nb8jpnLyqfGrcV2feUtIZ\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/server-noip.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 1d:5c:7c:59:0c:cd:27:83:dd:97:64:53:b0:44:3c:b4:5b:d4:fc:d1\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:ad:f0:87:3f:32:66:87:7a:31:9b:ca:54:0a:ae:\n a6:1a:ca:25:d5:c5:fc:49:6b:21:0b:de:a7:15:c8:\n 9c:3c:cd:41:1d:32:25:77:68:7b:83:e8:5d:14:7a:\n 2e:0b:ea:41:91:a4:20:ec:20:17:0f:cb:8d:4c:1b:\n 88:a0:d2:77:46:65:7e:d1:a0:e6:4c:4b:c6:8c:39:\n a1:b3:5b:34:57:c8:a1:10:0c:13:54:3a:74:90:7b:\n 3b:ba:da:41:62:81:f5:dc:b5:73:6c:83:38:f4:a1:\n 52:ab:bd:c2:95:3e:79:ff:af:a4:ae:df:d6:11:d9:\n 29:20:b8:b4:15:e3:32:93:5c:36:ce:56:b4:01:02:\n a1:45:31:a3:a1:1e:ea:b8:c4:01:04:57:11:44:f4:\n 55:ea:e2:ae:68:08:70:85:c0:5b:ff:ba:bf:de:8b:\n bc:a1:c6:03:a2:2f:8a:eb:9e:b7:79:3a:67:6a:b0:\n 70:d3:e8:08:a7:12:24:d5:bb:57:88:72:37:c8:e6:\n 2e:db:87:e0:2a:71:1a:2c:90:9c:88:29:0c:75:28:\n 84:d6:7f:75:2d:50:08:36:2b:cf:f3:ba:33:72:1e:\n 7a:07:db:20:da:98:3b:39:86:96:4c:e2:4a:42:61:\n 1a:9b:0b:67:4d:6d:13:d1:9f:40:df:12:c7:8d:9d:\n 50:69\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n C9:AA:3C:08:39:7E:C1:42:C0:3D:B7:2F:84:21:E7:8A:30:E7:C7:B1\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 9b:63:ae:ec:56:ec:0c:7a:d5:88:d1:0a:0a:81:29:37:4f:a6:\n 08:b8:78:78:23:af:5b:b7:65:61:d7:64:2a:c9:e7:a6:d2:b1:\n cb:36:bf:23:2e:2d:48:85:7f:16:0f:64:af:03:db:5d:0e:a7:\n 14:c5:f6:04:b2:6b:92:27:ba:cb:d2:13:25:a2:15:b0:8e:4a:\n 2d:eb:41:18:09:b1:68:d5:0f:6b:56:da:86:ed:4a:7a:29:30:\n 09:77:63:a4:64:3d:e3:2e:d7:6f:1a:8c:96:c9:cb:81:fe:a3:\n 6d:35:e3:09:ea:9b:2e:da:8c:8e:c8:c9:69:b1:83:e7:6f:2d:\n 5f:a1:ac:32:ae:29:57:a9:5c:9b:7d:f0:fd:47:3c:f3:6a:d0:\n eb:77:8d:70:06:a2:74:3d:d6:37:1e:7b:e7:d9:e4:33:c9:9d:\n ad:fa:24:c6:4d:e2:2c:c9:25:cb:75:be:8d:e9:83:7e:ad:db:\n 53:9e:97:be:d5:7f:83:90:fc:75:1d:02:29:b7:99:18:a3:39:\n 25:a2:54:b7:21:7d:be:0b:4c:ea:ff:80:b9:4b:5e:21:ed:25:\n ad:d4:62:52:59:79:83:32:df:30:a1:64:68:05:cc:35:ad:8b:\n d3:66:6b:b1:31:b7:b3:b2:d8:0f:5b:96:40:ef:57:1d:7f:b0:\n b0:f4:e9:db\n-----BEGIN CERTIFICATE-----\nMIIE4TCCA8mgAwIBAgIUHVx8WQzNJ4Pdl2RTsEQ8tFvU/NEwDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3whz8yZod6MZvKVAqu\nphrKJdXF/ElrIQvepxXInDzNQR0yJXdoe4PoXRR6LgvqQZGkIOwgFw/LjUwbiKDS\nd0ZlftGg5kxLxow5obNbNFfIoRAME1Q6dJB7O7raQWKB9dy1c2yDOPShUqu9wpU+\nef+vpK7f1hHZKSC4tBXjMpNcNs5WtAECoUUxo6Ee6rjEAQRXEUT0VerirmgIcIXA\nW/+6v96LvKHGA6Iviuuet3k6Z2qwcNPoCKcSJNW7V4hyN8jmLtuH4CpxGiyQnIgp\nDHUohNZ/dS1QCDYrz/O6M3IeegfbINqYOzmGlkziSkJhGpsLZ01tE9GfQN8Sx42d\nUGkCAwEAAaOCAYYwggGCMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nyao8CDl+wULAPbcvhCHnijDnx7Ewga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MBEGCWCGSAGG+EIBAQQEAwIGwDAL\nBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYIKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYB\nBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADggEBAJtjruxW7Ax61YjR\nCgqBKTdPpgi4eHgjr1u3ZWHXZCrJ56bSscs2vyMuLUiFfxYPZK8D210OpxTF9gSy\na5InusvSEyWiFbCOSi3rQRgJsWjVD2tW2obtSnopMAl3Y6RkPeMu128ajJbJy4H+\no2014wnqmy7ajI7IyWmxg+dvLV+hrDKuKVepXJt98P1HPPNq0Ot3jXAGonQ91jce\ne+fZ5DPJna36JMZN4izJJct1vo3pg36t21Oel77Vf4OQ/HUdAim3mRijOSWiVLch\nfb4LTOr/gLlLXiHtJa3UYlJZeYMy3zChZGgFzDWti9Nma7Ext7Oy2A9blkDvVx1/\nsLD06ds=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/srva-cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 0a:7d:45:32:e2:24:e3:81:f9:5a:64:e1:f2:50:92:d8:e2:3e:14:8e\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:9f:b5:d0:95:c7:8c:ee:16:13:fc:d6:90:8c:75:\n a9:3b:2c:ab:ff:b6:b6:ae:39:8f:c3:d4:75:7d:01:\n ba:0b:f0:db:8a:72:2d:41:49:e1:f1:b5:f6:5c:32:\n af:b0:46:89:ee:33:5d:84:47:69:a6:4b:de:3f:8b:\n a4:29:4c:98:20:6b:94:29:bc:4b:cb:84:73:d1:6d:\n 35:15:63:f8:55:a7:50:b4:2e:3e:e5:61:8a:c4:a5:\n fb:7a:88:aa:14:b2:06:36:8f:93:3b:8c:29:d2:ee:\n f6:8f:e7:83:68:33:59:17:d6:1f:b4:05:4d:02:ff:\n f3:a0:1f:15:2c:ad:8a:33:13:95:95:d7:b6:54:6f:\n ac:cb:98:8c:fb:69:c7:a6:cc:77:5f:20:08:e7:8b:\n 3b:79:5d:7d:63:2e:6e:cf:ce:b6:04:d0:0a:c5:19:\n 6d:ed:aa:45:7d:32:c3:53:ba:5d:d6:5c:e5:30:e0:\n c7:3c:2f:88:9e:0d:33:8a:c8:30:40:af:48:a6:aa:\n 4d:68:81:55:4f:43:cc:f2:79:3a:c2:23:0d:fe:09:\n dc:c0:85:94:cf:c0:f4:e2:7e:c2:64:7a:55:41:f1:\n 3f:3d:e3:46:4a:ef:55:33:d2:ea:72:56:25:15:6c:\n 47:1b:4d:c5:da:d3:6a:cb:62:db:d4:7c:86:64:81:\n 86:61\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n C7:6D:8A:C9:67:13:C9:63:AB:01:87:58:33:DC:FB:24:AC:61:65:4E\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n a1:3f:db:97:03:ca:e3:34:36:01:8a:d9:34:1e:1e:9e:d5:bf:\n 7b:9f:37:cf:61:2a:8e:fa:c3:cb:68:ca:f3:4e:04:78:8d:5a:\n 0c:c0:5b:24:fe:56:a5:c0:7a:b1:bd:63:1e:ff:bf:72:9a:fa:\n 08:33:4b:e0:2b:35:8b:03:c4:23:7f:54:3a:8a:f2:a9:28:d7:\n a8:7f:10:96:93:a3:b5:82:59:f1:b6:05:3b:58:cd:a1:8d:a9:\n 77:fa:71:36:f7:68:c9:50:46:4c:0f:c3:f8:f8:90:0c:69:eb:\n 53:e6:d0:5d:32:2f:d3:f7:b0:8d:75:68:59:37:d7:de:ee:da:\n 42:99:1c:24:03:28:c4:42:8c:17:e4:ed:5a:d8:70:32:01:fb:\n f8:57:f8:1a:2d:bd:f6:e1:35:12:28:65:ef:00:05:1b:e2:50:\n d3:1e:3c:25:34:56:5a:86:8c:30:9d:e0:f4:79:d8:09:7d:74:\n 0d:70:3a:2c:67:2a:d6:d2:6b:ed:d4:bd:1d:f2:d6:02:74:2a:\n 75:b9:8b:6b:a4:c1:c7:d8:42:58:ad:85:d1:0c:0f:14:88:12:\n 57:34:60:9f:18:32:f2:75:77:69:30:42:65:3a:f7:c1:16:49:\n 76:21:ea:cd:95:b1:29:d1:66:e2:90:e9:86:c4:7b:2d:aa:2b:\n 00:6e:59:f6\n-----BEGIN CERTIFICATE-----\nMIIE+TCCA+GgAwIBAgIUCn1FMuIk44H5WmTh8lCS2OI+FI4wDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ+10JXHjO4WE/zWkIx1\nqTssq/+2tq45j8PUdX0Bugvw24pyLUFJ4fG19lwyr7BGie4zXYRHaaZL3j+LpClM\nmCBrlCm8S8uEc9FtNRVj+FWnULQuPuVhisSl+3qIqhSyBjaPkzuMKdLu9o/ng2gz\nWRfWH7QFTQL/86AfFSytijMTlZXXtlRvrMuYjPtpx6bMd18gCOeLO3ldfWMubs/O\ntgTQCsUZbe2qRX0yw1O6XdZc5TDgxzwviJ4NM4rIMECvSKaqTWiBVU9DzPJ5OsIj\nDf4J3MCFlM/A9OJ+wmR6VUHxPz3jRkrvVTPS6nJWJRVsRxtNxdrTasti29R8hmSB\nhmECAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nx22KyWcTyWOrAYdYM9z7JKxhZU4wga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA\nAAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI\nKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI\nhvcNAQELBQADggEBAKE/25cDyuM0NgGK2TQeHp7Vv3ufN89hKo76w8toyvNOBHiN\nWgzAWyT+VqXAerG9Yx7/v3Ka+ggzS+ArNYsDxCN/VDqK8qko16h/EJaTo7WCWfG2\nBTtYzaGNqXf6cTb3aMlQRkwPw/j4kAxp61Pm0F0yL9P3sI11aFk3197u2kKZHCQD\nKMRCjBfk7VrYcDIB+/hX+BotvfbhNRIoZe8ABRviUNMePCU0VlqGjDCd4PR52Al9\ndA1wOixnKtbSa+3UvR3y1gJ0KnW5i2ukwcfYQlithdEMDxSIElc0YJ8YMvJ1d2kw\nQmU698EWSXYh6s2VsSnRZuKQ6YbEey2qKwBuWfY=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/srva-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCftdCVx4zuFhP8\n1pCMdak7LKv/trauOY/D1HV9AboL8NuKci1BSeHxtfZcMq+wRonuM12ER2mmS94/\ni6QpTJgga5QpvEvLhHPRbTUVY/hVp1C0Lj7lYYrEpft6iKoUsgY2j5M7jCnS7vaP\n54NoM1kX1h+0BU0C//OgHxUsrYozE5WV17ZUb6zLmIz7acemzHdfIAjnizt5XX1j\nLm7PzrYE0ArFGW3tqkV9MsNTul3WXOUw4Mc8L4ieDTOKyDBAr0imqk1ogVVPQ8zy\neTrCIw3+CdzAhZTPwPTifsJkelVB8T8940ZK71Uz0upyViUVbEcbTcXa02rLYtvU\nfIZkgYZhAgMBAAECggEAAcqJN1TQmM74lGpXnqunJTnACBMlg1iz6w9T7f359y5R\nWAElzsikVxCiXbBX1hNEcr5yuwwN/EZ3jKDlS21He32ZYMpy0yp+Hggqgz3myBcj\nSZIHNI1egAwYgOxNdAP3G9+KWC2fPnVdoImJKL8CrcjB/EPe9Dyon9AZbbHYjfAS\nwOCy06KTCvwJrNO1mYFSftEOe1S3TVGHp3KX96YWxR/yIgXNC+5bJWzf+X8Ca0i7\nYHsgoF1CqoLg7sq+nMo1xzGEapy2bkx8DRdxigIXBBNnKQdVVO7qjmUOw8EdJZ9/\nluZ+/lb6Oc9RT6+V3B97kpYP7Z+frItGZWuMtifLAQKBgQDP1eI2w/VW6CW0ODjQ\nO1ivNzK1CK4L9FNA07tnhfzEOlF/NMqfBz7pYyn+1ObxMcXzFAZAmh5E/Q8b8QEP\n4b51IUqGL6FSQlKSlWJlszD7QhUM6nXFmvDo1g3pBZ9w8EW12cL+FM/KBpXKB0aF\nS8wcekbvGkgqI6SQEvkcRrM6EQKBgQDEuNXn8B7YTlkw7eXu/F87ZQkWb1bes8VI\naQzVDH/ywu5Cbou8iJYsdfpZFwnJ4ojIl0eBleqXpgb+SiEx8q5zmR/Tt0NDIjfP\nlVdlcf3oLxjz6LKwHWPnXmz2WEeYez2/lkBc0XmOYrAF8jy2h1RX4kkvVGEBNNKE\nooiX2Kq3UQKBgFPQQCqvubezZMkZxyeKV0hJrUQ+XNnrTMue1zt3WCjWD9mJs5CQ\nPnmmCOCAgSWRWe+eCqtwu3Y8yyQMe84ozkK0Czaz7I2cu2nrnkO/EKiyzIRp5Nas\nlix/bhqjrtm/u40LZUxLnkQthah0aKcGmyh80zBHXATKXOnRDO6vQEXRAoGAKmuB\nFMaeoB4k9XwXPPWGw5QB5aiUnuk8/WRJtBdB7+NA5WpZD4qUd3npRhTFs5b5z+dv\nL+5X6+ONVoA9sDuYiqzWRB0cj/ls94JImkN+cPbW9qGTBo8P3/BopqZaqfLJWerY\nbittftqhAmc92E5OTbUp+t7DHDN7wcYalkPObEECgYB0sNTdWVl78AayFqI/5Bg0\nFfZYRTdVyx6qTPsQHqmPGHNn+ZyPIzVJJyfT3Fp8cToJDyYZtp5lUcnsSb2Q3j/C\nriF99UpDJfetEFJATlSWjgWkMgQVHm97qIZtr7XLzM2gWJ1/fE0IxIx3ZFGUi19x\nl6nGMJ23s64br4XiyNqHIw==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/srvb-cert.pem", "content": "Certificate:\n Data:\n Version: 3 (0x2)\n Serial Number:\n 20:98:70:f3:55:1c:7e:a3:15:0c:64:c1:a2:8e:56:4d:4a:77:70:e8\n Signature Algorithm: sha256WithRSAEncryption\n Issuer: C=US, ST=California, O=Synadia, OU=nats.io, CN=Certificate Authority 2022-08-27\n Validity\n Not Before: Aug 27 20:23:02 2022 GMT\n Not After : Aug 24 20:23:02 2032 GMT\n Subject: C=US, ST=California, O=Synadia, OU=nats.io, CN=localhost\n Subject Public Key Info:\n Public Key Algorithm: rsaEncryption\n RSA Public-Key: (2048 bit)\n Modulus:\n 00:bc:15:41:85:42:e2:da:58:c4:d4:2b:8a:92:88:\n cf:a8:81:5b:03:5d:8b:d5:44:32:07:ae:ef:a2:ff:\n 0f:29:14:dd:bf:7e:6b:37:e4:31:e1:94:7e:5e:be:\n 74:40:68:ce:3b:d5:bc:00:a6:42:fc:0f:e1:c5:8d:\n 4f:09:32:fc:af:6a:17:c8:92:3d:b0:2d:bd:ce:de:\n 7f:d8:bd:9e:5b:84:f8:b6:8c:a3:7e:98:37:f1:37:\n 1e:aa:cd:87:92:3f:33:34:17:0e:a1:78:3b:1a:88:\n d6:73:c2:ce:41:71:81:ea:83:2d:f7:e5:e6:32:f1:\n 18:33:6b:ad:9d:ec:0a:3e:8e:74:a6:b7:7b:4b:22:\n cb:0e:6b:57:14:cc:4d:c0:27:a1:38:30:df:35:e6:\n fe:49:8d:4f:18:4d:be:5d:df:46:d0:4e:3d:00:93:\n f8:e7:48:4c:6f:fb:13:c0:fb:6c:8a:9e:53:66:24:\n 3f:c3:93:66:57:6a:f6:66:88:94:b7:46:e1:e8:d6:\n 8a:04:02:92:e1:c5:c9:b1:71:f0:d2:1e:4a:69:8b:\n 0c:ac:a1:01:89:5f:1f:ac:79:da:8d:ad:bf:03:81:\n b6:8c:70:5c:1e:74:f5:a4:d8:d1:8a:41:d7:bf:27:\n 51:d1:fc:75:b9:34:73:03:ae:10:2e:ab:78:a2:af:\n e7:99\n Exponent: 65537 (0x10001)\n X509v3 extensions:\n X509v3 Basic Constraints: \n CA:FALSE\n Netscape Comment: \n nats.io nats-server test-suite certificate\n X509v3 Subject Key Identifier: \n 5F:CD:62:19:69:22:48:0B:79:B2:8F:53:D7:EA:C6:9B:C2:5E:58:94\n X509v3 Authority Key Identifier: \n keyid:FE:CF:7F:3E:2E:EE:C4:AD:EA:5B:6F:88:45:6C:C7:88:48:14:8B:3A\n DirName:/C=US/ST=California/O=Synadia/OU=nats.io/CN=Certificate Authority 2022-08-27\n serial:49:9C:16:ED:BB:5C:F4:45:1B:AC:C5:AD:8C:7A:5B:33:40:B6:6D:21\n\n X509v3 Subject Alternative Name: \n DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1\n Netscape Cert Type: \n SSL Client, SSL Server\n X509v3 Key Usage: \n Digital Signature, Key Encipherment\n X509v3 Extended Key Usage: \n TLS Web Server Authentication, Netscape Server Gated Crypto, Microsoft Server Gated Crypto, TLS Web Client Authentication\n Signature Algorithm: sha256WithRSAEncryption\n 08:6c:2f:cd:9d:b4:44:42:51:29:c2:55:92:e9:52:69:af:ae:\n c6:c4:4b:81:7a:a9:7c:81:f9:dd:27:fe:c1:e2:9b:59:92:38:\n 1f:f4:d3:04:b7:ed:59:78:da:09:33:c0:c0:1d:60:79:07:78:\n 15:aa:50:c4:f8:7c:47:96:a7:76:3f:ac:5f:eb:7d:8b:05:81:\n 83:f7:be:7d:1e:6f:05:d4:bb:7b:ee:14:a8:af:e2:22:82:bf:\n 25:7c:f2:30:38:98:68:43:c0:4a:08:80:11:81:be:c1:7e:43:\n 22:cb:85:c3:b6:da:37:86:c5:7f:5b:8c:d9:b4:61:3a:08:c0:\n 12:a0:a1:ea:2f:cd:58:da:41:8c:1e:10:1f:54:19:c2:6a:2e:\n 3a:d9:0e:f6:7c:34:95:72:9a:41:ed:1e:79:84:1f:8e:89:8c:\n 43:84:9d:32:42:e9:b5:db:1a:ab:2e:1a:f3:d1:b0:2b:38:fc:\n e3:c7:4f:1b:6f:9c:2e:3f:83:7a:fc:22:23:aa:64:6c:6f:c9:\n 26:81:81:5b:f1:8c:ca:0a:ba:2d:91:f7:2f:ba:d4:7f:dc:53:\n e2:f6:1f:38:50:6e:6b:87:10:b6:02:af:56:f3:83:9f:3e:3d:\n 83:e6:59:cf:3b:f8:bd:21:0a:8c:90:d4:26:74:70:1c:b1:cc:\n a6:3a:27:e1\n-----BEGIN CERTIFICATE-----\nMIIE+TCCA+GgAwIBAgIUIJhw81UcfqMVDGTBoo5WTUp3cOgwDQYJKoZIhvcNAQEL\nBQAwcTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoM\nB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xKTAnBgNVBAMMIENlcnRpZmljYXRl\nIEF1dGhvcml0eSAyMDIyLTA4LTI3MB4XDTIyMDgyNzIwMjMwMloXDTMyMDgyNDIw\nMjMwMlowWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNV\nBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMuaW8xEjAQBgNVBAMMCWxvY2FsaG9z\ndDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALwVQYVC4tpYxNQripKI\nz6iBWwNdi9VEMgeu76L/DykU3b9+azfkMeGUfl6+dEBozjvVvACmQvwP4cWNTwky\n/K9qF8iSPbAtvc7ef9i9nluE+LaMo36YN/E3HqrNh5I/MzQXDqF4OxqI1nPCzkFx\ngeqDLffl5jLxGDNrrZ3sCj6OdKa3e0siyw5rVxTMTcAnoTgw3zXm/kmNTxhNvl3f\nRtBOPQCT+OdITG/7E8D7bIqeU2YkP8OTZldq9maIlLdG4ejWigQCkuHFybFx8NIe\nSmmLDKyhAYlfH6x52o2tvwOBtoxwXB509aTY0YpB178nUdH8dbk0cwOuEC6reKKv\n55kCAwEAAaOCAZ4wggGaMAkGA1UdEwQCMAAwOQYJYIZIAYb4QgENBCwWKm5hdHMu\naW8gbmF0cy1zZXJ2ZXIgdGVzdC1zdWl0ZSBjZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQU\nX81iGWkiSAt5so9T1+rGm8JeWJQwga4GA1UdIwSBpjCBo4AU/s9/Pi7uxK3qW2+I\nRWzHiEgUizqhdaRzMHExCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh\nMRAwDgYDVQQKDAdTeW5hZGlhMRAwDgYDVQQLDAduYXRzLmlvMSkwJwYDVQQDDCBD\nZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAyMi0wOC0yN4IUSZwW7btc9EUbrMWtjHpb\nM0C2bSEwLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAA\nAAABMBEGCWCGSAGG+EIBAQQEAwIGwDALBgNVHQ8EBAMCBaAwNAYDVR0lBC0wKwYI\nKwYBBQUHAwEGCWCGSAGG+EIEAQYKKwYBBAGCNwoDAwYIKwYBBQUHAwIwDQYJKoZI\nhvcNAQELBQADggEBAAhsL82dtERCUSnCVZLpUmmvrsbES4F6qXyB+d0n/sHim1mS\nOB/00wS37Vl42gkzwMAdYHkHeBWqUMT4fEeWp3Y/rF/rfYsFgYP3vn0ebwXUu3vu\nFKiv4iKCvyV88jA4mGhDwEoIgBGBvsF+QyLLhcO22jeGxX9bjNm0YToIwBKgoeov\nzVjaQYweEB9UGcJqLjrZDvZ8NJVymkHtHnmEH46JjEOEnTJC6bXbGqsuGvPRsCs4\n/OPHTxtvnC4/g3r8IiOqZGxvySaBgVvxjMoKui2R9y+61H/cU+L2HzhQbmuHELYC\nr1bzg58+PYPmWc87+L0hCoyQ1CZ0cByxzKY6J+E=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/srvb-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC8FUGFQuLaWMTU\nK4qSiM+ogVsDXYvVRDIHru+i/w8pFN2/fms35DHhlH5evnRAaM471bwApkL8D+HF\njU8JMvyvahfIkj2wLb3O3n/YvZ5bhPi2jKN+mDfxNx6qzYeSPzM0Fw6heDsaiNZz\nws5BcYHqgy335eYy8Rgza62d7Ao+jnSmt3tLIssOa1cUzE3AJ6E4MN815v5JjU8Y\nTb5d30bQTj0Ak/jnSExv+xPA+2yKnlNmJD/Dk2ZXavZmiJS3RuHo1ooEApLhxcmx\ncfDSHkppiwysoQGJXx+sedqNrb8DgbaMcFwedPWk2NGKQde/J1HR/HW5NHMDrhAu\nq3iir+eZAgMBAAECggEAfFfVJEPO9Clhnx9WEoBOQQmb4QK0Un8uUDQQC4NyQ2ef\nHl12htHfRFJGpV5mPrrLNXWdQBSrDjd1vIQqb1t9AH80UA6Wp+XlqWEhxmm9yqWD\nRDSAVk5OFbOqG6ObcE1GDbb8njV9ZZJ3rLpqX4uqJx8ogc5EvQL1S/FxK4mEUEv8\nbq4Iswp590+KwnaYtkXEoieUbE1QC0y6z9BEpEe3WkNVGAmnuJ6wCiE2pW6yzK94\nNxGvnsejgBVDZZprWbLdIztHDJ4+VDDctkxnqsx3cb2o+mK7CCP6C7v9MAvgrMzH\nhEDu2WZC6T6GrVysYXSNmlWYPTsPh89F5Dyrp5vq8QKBgQDwTnMq4SBbrq3tsJEV\nChX3pqQ9icyaT/yLI1oWcrdJKpyMdDVxOBPjB04i7pUZFNKqHS/aKjozKmXHVFNy\nX9wy5D0der7gqHmMeEV4B7UeO31E2B6nftlntt+V8U3Nu6r9mi3QCbH5x5vBjDH+\nulx4Qma+CeBL+bP1Z8Vm8a6qNQKBgQDIXbY3jm3Gl2hE2+Ci7JceloRlwp3CYlHS\nKejhxx+secKLbbeXUp2A2uj6lz9DX/Gx+qw8zbhoodh6FkGJVrZchatUxci3P9rs\n6c2rVMzcPGSt67tFtGGm04cbZgD83QDj0xPtlolZlrGV55PpUFJkS2AThcitMYkx\noNOZf3hUVQKBgQDK3n/NoFbv74z4D8q7h1M6Sl13ckDfuU0LraGdN+VSCH42+Ngx\nJ9VqIT2usdn/XZvnb1J5jJKrpUxdNexor1K3SAXsQDaqdLCAjPygs2kNB83ec9GV\nN7qUG0ewTxGO8LMO+71XVwgARapRgGiokm4EPJJ+Nto7aaeqMpvtJ3V8TQKBgQCW\ntZ5M1Y4OtW4RdU3XXpwgvtihbiyZvLiMm0d3I/7YDdHebE2Ove6IsKBTdRZ5yB4V\nYFbTeW2/DhW1Zy4ubpHGsCh6IDbkV1yFnSs9NxDMi4SzKT/4o0BcankSi7wsdtOn\niepQaE2xnF7BQaoI8Yi3tdZzeMrEMyFdufnPWIka7QKBgQC9nCUtLl3FCaeq6PMv\nKrNQRu5dPbpy5SJI1SSXKCUU9VaA6XMKrNeQHkgsnsj6KZ9Rl+yNe31L/GHNRiSj\n4O/vChYMk1bXgESenvNafSMbl6agwi/Y1GFF6CM6/vDas4tB7bsFQPh+cnb9uIvq\n5IFkU1NTCSSydk03li94ZF81Bw==\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/ca.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDXYllrRiGR1WPO\nrT07fN7bY5q8Pnsb1zaO13ZlTdQpiNsEa9AkOL7NaAZFZJ9+KdhQYQgbAUb5Cvyw\nUIrydTvDrqzghQLaHn9VEHx7xJWJRaCYSqB7MBZCsc7ov/4ntIQO4H/aYhBdcXzj\nELpqeMSSLpnH9+9KmZU4ArxU0dqx+2YzUoCDXszsD8541797ksrxrZnLkuNxDmtZ\nULn55pY/MT7Tq0gkUA+ReDpHeElQuf9L40ODdueSpteR4mDNPQmLoTE5qwTsyhjF\nBght+121Chu3tRwprJqxEjwe/YB6rAFVnlgWwLumfYer/jI8UNoSsiVlj8ToMR11\nkIVF6NGTAgMBAAECggEAVJJoJSCxDsfY/G7207R493FZcWAhS/HKGPWOpkGUtMhp\no4dH/+kONq6Jg9b6b3oz13/6RExQl3qSdRLzNDgHQxOEC1b/IzGFcxvZnWm4A+JE\nqHfpIuXjX8XcxnSY8fNGRWcunMwTu/VTgf9wC12nZfUJakU2/zZmKsucMrwCIQ8q\nrYBoHlVVs/R+3Hj6FYj5yFeoCsWyPswFd07llu6/h8ADY+P1xnOeilJ8pdNR6vaQ\nwZQhBc0m414XRp2OzpKAX0+7sATSUhhGnU5VdlCcP6whHKmva2U3RusIN1Qy7UIA\n35sNxMsSzOvozwZXbWMpWaUU69uLtGjeX6qhh+bk9QKBgQD6XyViXijKHN56xFx9\n4inTPlqOrgtuZDm+9pQ/TFaZJfK4GoG/8QyUOXmLAnxbbTAAKWXc/zHXhVr71wsW\n4pgsUxypWfND9yQ7dYfJCR7PhkYqdRm3WG+JOss9aL+FULdqjD/4tw4Y5EHc2vpO\nplQ0p0/xJtLAlg0aqjAonjDUJwKBgQDcOdrrklov3IvKw7LgKYmvDRjLGEyMm3qs\n0fPgWDG7BzZ8nfDQ7o2Pglf/ypuLYTQyZV4DXAXdiQGjkM/SzStsSV0hmEFfaLtX\nJPSbk6gjW1+9+sxq1xJfxvu9tlUeLoRwA0Ck0rwGn3y3OjcQCCVL+5jkqmmERihM\n+3IY8+/etQKBgG4yZHToNpHGpRFpzb/GRFxqrFbyOavLxzLKurMleVQMxMjNOeBu\nSvgOV/WcEXn4E9FuZxwe4iQW2NXRb1sSPqH7rrjHE3ANnc/hyfLs5be3RLi5M6lj\n7it7SirsKKeXwDLQKfPhNf40sNxxMQxe3t8R6Pid+tKy+G+NCodGIP2vAoGBALWE\nsl8tGE0O2eJc/5koY6X3g5ITCzVyCeFchElMKGFiBpUckZuEpQW+h838L8zoZ9+K\nEb1vIh6BiOpKuCGmnr34klzYZDnB13hyq3N4d0P9UqMCj28YqrprBqBOI0ZnTR4O\ncD/qurQyzNcxkqUSMbu5O0Ju+93c/ebF8juDBRlpAoGAM14TPQSUz4oiMVB5VWAj\n75OK2rFQLC+mMx2T7u8UqpHVb+A6lUZcj0aN6TgRQxQo63bRXaEc/13BTMKmmqze\nrM7bpj4+b08Fr0yhw7fjO2b/q+zn8Gagk6UWIptpY/8WC4OSgaSjMC6ZErPWqRJs\ncJXNOa3YaoQcl8SfLW45RNg=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/ca.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDBzCCAe+gAwIBAgIUHba56kOIMIRpaKPovfEccmejyJcwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx\nMzIyMzAzNlowEjEQMA4GA1UEAwwHTkFUUyBDQTCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBANdiWWtGIZHVY86tPTt83ttjmrw+exvXNo7XdmVN1CmI2wRr\n0CQ4vs1oBkVkn34p2FBhCBsBRvkK/LBQivJ1O8OurOCFAtoef1UQfHvElYlFoJhK\noHswFkKxzui//ie0hA7gf9piEF1xfOMQump4xJIumcf370qZlTgCvFTR2rH7ZjNS\ngINezOwPznjXv3uSyvGtmcuS43EOa1lQufnmlj8xPtOrSCRQD5F4Okd4SVC5/0vj\nQ4N255Km15HiYM09CYuhMTmrBOzKGMUGCG37XbUKG7e1HCmsmrESPB79gHqsAVWe\nWBbAu6Z9h6v+MjxQ2hKyJWWPxOgxHXWQhUXo0ZMCAwEAAaNTMFEwHQYDVR0OBBYE\nFCcUWYVGF5cPrQMR/VNkapjuXMRbMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNk\napjuXMRbMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBALdP3pUX\nSkBo3xJ1m7KskL6Xtic9eDxu9aJKYUH4qw16Aoyj+tZm5R1UvyaEsHMG+hFOZRRp\ntArbOknL4RZG7Tx/ATTiSQl6ozmkZ8GRpKY6VsN5c6Y1e9sbdXeGgfitWMUkQQhD\nFtqps2TM6IfJBdj4BhVGAjG45tUhOTfMjZliaNZexVyR6XjuhDHcDDpcdaFgDfDA\nuAYDDbfhO7MIKxplzrPcI7e9zDdb9mgB8UExNltxrH9w3J1cG4VpKxFtXYs5FO4d\nswXWTr2cY4sTT/bhVJET+YnHyyZgg+Ear4n7eujA9klUHEMwPy/QDCn8u/091yew\nNnOmSiP2A+dWQB4=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/svid/client-a.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUuqRV+zUywU/g\n8Q5z6i+mJxzHCXjM1XPKu3gmWHtTTfRbQtvGdFg5mWHL8rgvEkeYkK+yn6NmGFPP\nz8vJDLyVzz70YQpi1Qv2Awe0bHTgwlCXAKUlHfEapDh3068hEonz2Fp8n652ArNd\noaKHJCUZtKIfm10cs+nX+cuiv+trfJUVYxPyrQhu7lhWhNRwDWedI3pc7382mwze\nSwFft0Mi1AyK7JhVMRM6ld91MXuJlLNfHKEejrumkt6dIP3o7cJJGxW4IFjDs4gh\nIK9cd6JMeYQXs9Bc03hKazCKCWITrPPiz0BD7beiqFUNcPgOlzqveBnqsXyRLEjE\niQ9a9QwDAgMBAAECggEAEC8994FOU7HyRacrDHVyg/aDoivSB8WT7n7MtKzc7BQd\nQC/GshIyZj+JAzmWzhXY0Vhaq5RwTTQz1/cOf61cJEKY4N0NKp4byW9LpQLxtmJO\n2hp0M4AjnuOPUDLVPu3BMgDcL+2XKDkOWGULcQCqQjbZjEOgM/cRaQhCHvlwM+er\nJqZ0CN6zPrcQTVlwEZLiL+aQ3igZwXaEPBq3VmuEW6rjpq7n7AE19GCmtwzo9PXW\nBxZFKMMDxsiiqTY/UyM0Sfci+oGYmDMwVMFh5wDdsyAv6ulF8g0U8YTV4w8uBj0I\nBRsSS8P6nwUIWGS9pz/1mgnsESPQtj9cs2dDUp5P0QKBgQDRCdVvlObEeka/rlIM\nto9J2iThHPecXHHfD5S5AxC6/fU+n/X2k3Q6GzlYIkFdYQzic8oTtQaODtib9eFL\nXHDCNyj4x3Gb7UPBl/f9uqIseUhpAJk6GRiUBgQlCJGp4btMVNx1bhoU+nWgysEo\neGvC/B8WB+04MLGURn1k3TTnawKBgQC2JE/dLa7bYxKH1K7h5HxwDjrdYQGFfWIb\n9/uFyZa+1NAWYj0TD3vy/pox7PI/VMMa6oWl2C6Mv7aSLlFTwcZz3YKPv8iB7WHR\nF12/s5xcKNID37J4Fo9EFq2751OqKyhW0aU65a7W/mjJY1loLo0fZVmuc1ZGPcDG\nv39nwJ1LyQKBgQCY8Ub2Qs2tB4HoENT7dD7WXAMLqbQI9SJjf/TigLmm9tSOo17E\nsBIjyXNlZnrIpY9VyD4buSE4ougdBAN4rgPaNZ2Gl/Ypjak7ZcMOI5UwUPSHzuZN\nObtf4h76MZDR6NSu/o/mY0419yJFKNO2lpTRrsHXzqjjqczjncd1GtvwTQKBgCVV\nUg1axMFxECZJQMNKavso34Xq0T4EUiy7apGknUJmKnS8FBqpDNXku8RJ9elnPUpy\nEYzJv9jXnttdpQvO5xduqsk+HSIMfwU1jeCyNvVo5IsoudMFhL4HC/s80hhKGk2Z\nUQ2+cFTe9ql4tKW3DPQHeRdZyrBBDhduvvWMNHGJAoGAD2x8zNEhBqOuBn5prtsC\nWPsjXRIjhXIlQbWwda30an74Bw5Z6OWgZr8nUjBtHn7RkeS8PqTE/UzMTLqD1+TD\nPvnA8GZ2WWdxhdqy04qkYS5SnkSSLeU0hUYZkLrjZJv5QdTnhBBVG/kDDI7TO8NA\ngKMQVjTCSStdZm02D6EqEfw=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/client-a.pem", "content": "-----BEGIN CERTIFICATE-----\nMIID1DCCArygAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe4wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx\nMzIyMzAzNlowgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEUMBIGA1UEBwwL\nTG9zIEFuZ2VsZXMxDTALBgNVBAoMBE5BVFMxDTALBgNVBAsMBE5BVFMxEjAQBgNV\nBAMMCWxvY2FsaG9zdDEUMBIGCgmSJomT8ixkARkWBGZvbzExFDASBgoJkiaJk/Is\nZAEZFgRmb28yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlLqkVfs1\nMsFP4PEOc+ovpiccxwl4zNVzyrt4Jlh7U030W0LbxnRYOZlhy/K4LxJHmJCvsp+j\nZhhTz8/LyQy8lc8+9GEKYtUL9gMHtGx04MJQlwClJR3xGqQ4d9OvIRKJ89hafJ+u\ndgKzXaGihyQlGbSiH5tdHLPp1/nLor/ra3yVFWMT8q0Ibu5YVoTUcA1nnSN6XO9/\nNpsM3ksBX7dDItQMiuyYVTETOpXfdTF7iZSzXxyhHo67ppLenSD96O3CSRsVuCBY\nw7OIISCvXHeiTHmEF7PQXNN4SmswigliE6zz4s9AQ+23oqhVDXD4Dpc6r3gZ6rF8\nkSxIxIkPWvUMAwIDAQABo4GiMIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhh\nbXBsZS5jb22CD3d3dy5leGFtcGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215\nLW5hdHMtc2VydmljZS91c2VyLWEwHQYDVR0OBBYEFDFDpnqTvAQQBUIBMzip3XCT\nFdsTMB8GA1UdIwQYMBaAFCcUWYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEB\nCwUAA4IBAQA8lg93kYHKkc4iF+mB8xHtyT1Pmy8CoxbDGy98cac1ny9s9K5kROTZ\nrE9p2FaC00jf5T/+si5diXa0EnIjWCk0uRlrW5PL85UbsqHJkWav7zCk7cDa9YrR\nnk4LMABf40RP3SOM4yxjjoVU13jFwIZ5M69Hce3CJ6ZkSoWRzWlKf6ECAbX/JIcJ\nu8wLBXbJaAvc1CQiZpiSWDvItg+sUn1dU1nmk2o1YmyBPN6uVF874f3Ihazb9wmJ\ndkWO48O+FaahjR6GzfjEx7w765ATqvTVYAgGZ55ps65VwfjcQbP1jYO4Sb3Aeeub\nyapE1595oBWoWdwtEeOTJrzSIEhkhyiC\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/svid/client-b.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7bOmV7/qGy+1L\n+cgBawm55lxS2w8QQr/lZrq4k/plQC2UDRvGPDaIihbvbP+pfiAgfSiWd/s+Qhdr\noOL/gkAJo/iD8HYu9FN+Y4vPlJyoEguNJzOyRe7U9YnW+Gsefwr60PoeiKfXK5B8\nJ5KahSQSka3hgW+yp4ECzAqpjobGUpzNoFVoQ0o9gBsvBWVOCJ02h5kcwsLFSSbh\nh76FPiedKHtA3VAj+vrg7rYT4wHpmSAdJZ7uNvbjbB4Jv35ebTnlkywBcPJi3YGt\nUI8O8HTDP1K8mlTitzvqHGMDtiT6OQJpffuJhOJ+PnXd4nOwKN+AAMQ8RxPx7lv5\nn2oYmq0vAgMBAAECggEAQNdYJIBGivXoHZ3HnP37+m2AyUVx+60sGGAm/w5q1TaT\nziGO11EjiK9FagWqmWfCE5tZPF079XkGAz1A6yaTowIwSu0Y5fuvGw2zY9hFLjw3\n9Nh4IRfinZho3I9K99z53nYH4iJi9qTisqE6Le0b4rS1lEukaglbL62Jjsl5RQje\nEogelhVsVh1Lv2GxZw2GUZHogTuWK3ZW3objq6HE0oReS9vUDwSL/rwuOcrGFdnb\nXkVD9Z+1eD8QD4Jp0rP0Abc8hkotb/alStbHTl3MzBEwVZXyc13HFDB4loOfhJU/\nbgPBgeFh1xXCR15+ouZsbsPkRZ5kwLSsbskUpkXeSQKBgQDe60LcdI0w7edfl3bl\npEOyA4Volh5net4dpQhLZFbjFAoTG1OA0Z9hBoZwoRLi1xkuMdLUzJOH6d9sBni2\nqn2sbHVKXTtb7Ie+RMb3XMpN34tlpqH2+vnrVkxkrPwWluhynVZm4AsefMHRAza2\nK3JBl2M+IAzgsfLUYNj4xWh35QKBgQDXPT5xjdr+NOADocrlf80w2iVnvLah0yVb\nFgVenkX1u5ZUiv9calJagfaVQNm4VwxBXGr1/3+PoPdKNWFjnlPO8R1E3Eia1Iu+\nFberWhEpAJuctT+lcPS8sFDh4r5edgyOXE3ikuI4tDRsQErGATfWkIfmuKUThW2u\nA8YZWmTXgwKBgQC0VUu0fhEOSRcIzMYhhh5BbvnIntCf09TF1JD+NIDZmIeB53RI\n3Mt7OUUe7h8SJQI4zZOKFKkwpTDZ+e5Dj261QtQrwXpYpeM2N/thds4t6y6goAmc\nfHIbQKfyNcKFH7WRqtiPxqUYF8LPLDNhcYovp6FgS5s/VpVj26kKruARRQKBgAP6\nKrw4bdKCsZz/kVs6SFVdwpEvKITG/Qq41nKQJCTNSZLYWq2ffVQU0LSjVSKV4TEX\n/xX6maXSyJ8Da4BGva+2Pt813tWVldt2VUWCTYzfaQC1TK7G7o7KB0SVRwVB0yuS\nQxIruqhnlAxoB7mu7hQ16Xz27n302jj78t4nEHP3AoGBAJMEOropyzzX3sYp4E9c\nu+VPyqcEMXfFimh2il0hijgB17dSW2UXeoQ85gevbPfGURL7a549xar1cNgEfQO0\nT82PecgRs83SEcgHT4mntytGY/lsiowAQZ2rXtlyG4ADPwbQGV3UXqV1Bm7yBT38\nYJJKJNKjyMHA8auKGG37WDhu\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/client-b.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDYjCCAkqgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe8wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx\nMzIyMzAzNlowHTELMAkGA1UEBhMCVVMxDjAMBgNVBAoMBVNQSVJFMIIBIjANBgkq\nhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu2zple/6hsvtS/nIAWsJueZcUtsPEEK/\n5Wa6uJP6ZUAtlA0bxjw2iIoW72z/qX4gIH0olnf7PkIXa6Di/4JACaP4g/B2LvRT\nfmOLz5ScqBILjSczskXu1PWJ1vhrHn8K+tD6Hoin1yuQfCeSmoUkEpGt4YFvsqeB\nAswKqY6GxlKczaBVaENKPYAbLwVlTgidNoeZHMLCxUkm4Ye+hT4nnSh7QN1QI/r6\n4O62E+MB6ZkgHSWe7jb242weCb9+Xm055ZMsAXDyYt2BrVCPDvB0wz9SvJpU4rc7\n6hxjA7Yk+jkCaX37iYTifj513eJzsCjfgADEPEcT8e5b+Z9qGJqtLwIDAQABo4Gi\nMIGfMF0GA1UdEQRWMFSCCWxvY2FsaG9zdIILZXhhbXBsZS5jb22CD3d3dy5leGFt\ncGxlLmNvbYYpc3BpZmZlOi8vbG9jYWxob3N0L215LW5hdHMtc2VydmljZS91c2Vy\nLWIwHQYDVR0OBBYEFAiaPOQsBb3+rAOlSevDy5kp7zCnMB8GA1UdIwQYMBaAFCcU\nWYVGF5cPrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQBq7BxhZMCTl6Xe\ni3GcPlOLp9JPP8bcyYZVB6nAf38eErpwYSPvp4I3IFU+KyZgtDIAb6Oy04BNV9eT\nBlpX300ZbylO/TLCrlMIJDYLIt5NciVe8IxsE//uLXFq5wZpcCcL9aQA2g0wW8hi\npkK1dQd3W1ryR/LwKiy0fcZJw/EFskoqq6vPJATIFvH/O0OxdKP9T24YVGgLBzwj\nxzUqVlU0CuP0snx6x4F3Oha6kJGwc90RlXszh2ELhO+o4sk4wtfAlEYaM+H3kbAK\nlRe8FENyGprOjlxjy6N1noJW9Mgisx9kNcsCzW8auGY6l8Vs3wW3vO3n4ZUnq9qc\nt95/tstH\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/svid/server.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmmTq1fDtsMKVU\nkc9jWb/JlOXU2i7s5uPzETulgrQRhHCWE6gACW3VYhTCecSf6TyLaFeTkwVsqv+r\nteOld0gRyA/NdCN0fY04OGfr7eg3dDyK+ovGInP1rh+6UrMl2Y+Iy5n/lymFPFkd\nEoqaKjXv5npGQScvrvTYN8hFu6ww3EdPI3rAYwolujGadbXttprLRIZvZQuYI30J\nLydCVsFMwfm0lnegEHtr/lx2UbxXeyf9nIOle7UsPAdR7oRXomf5FCoXUuWajv3Q\nW9NZEM4E0O2SbH/RO12MmmeXH0mwQuvrZAvgEtc8ZDMzmRZ+AOyCXxp6F7ME9SrB\npQqS7ljPAgMBAAECggEAFHvephXiUIkiquVKtJKCbEysfaFeB67T9Dz4Mn+BmG3K\nUgmUrHx8TrJCjy7dqDPLR/DKY9C96ACWcwxoMx4ZdGY+2yoeeeI+DN2SmYWk+byG\n/SeVLAVscJa6O/STJmI7rbwq8Swq5MDozBRNimsKOrcB7cVxBgHjBPrG2xwaLd+U\nzExqarE89tfvtFVUQobyeOimcSEerMIkOngwtRxbhvF956VpvVEaJDsm1rHqeiZA\nrKoky01IKMTXcf6g1bERKKXc/nPbMMOMYDD0FbQM5bK7OqSuN1+XFgLbIQOgShWv\nOhlENsa8TttAUq6dst+bv+zxnXTLNhV1INEdJWgSQQKBgQDfUktQoMFRCfwnFCNJ\n6r4jR108hLH7e8/2AfEJWAW8iwfvnpf4/fW7kqttcZ0FwLOGDb3TKJbXx0Jlz3Fr\nbkD3BIb3+19ludqI+hZanoSUqXBt/iTWRJpOGwpoLSIGNCR01GePMCva5MvETPj5\nnh7la+6AXngvxx8cR7RJMcxAMQKBgQC++hLaJ+cec3YpJGAAIFnXRgtFyzKn+ToE\nTvcbJEtHn1HO1MLJhd3E04A0BufjglxJ16Kpc95252O8GyveIKm408vbXjIY3Fxa\ny0zlnUB3UNdZ21HdZU3vr8T0C2s1c6tKTStuxUjtzfu1LXElJp/MxfodnyL0UE4L\nOiOmza7o/wKBgQC17uY6rmwvaDILaCWDn4D9XrM/jV8uYOTAJz7F/PbXbnCtrwdi\nKwtZn5iXFdr1h9L5YBKIh2W0LroSFVjyS6UTWJXYSuqiBjJaK1uwwbmDHzLf+Q7r\nnIIVJYRp8HLqmOomX7Bqf7UKpCC5MHgZa88B2tc3rz68tbzLnkLq6m3a0QKBgQCL\n8MVVtH2oE6VXGn96ODp9A6eI+g/hhD/eHlr2OY8sKXOR4tflQcGy0+SSuyi6zrB/\nr5JTJ0Oxz3aGxUZM25RiilhMI+cDzp2iLmznqGYvyD41av+/AtesNhlVVvS1U1AF\n/yw/XDNHhnPWXDkXpwjbzYqsoArGCf5WPcEYHon30wKBgDhFrftmpc8AgXoCDaAN\nYmb23qURm8m/+FTc+49xX3F/Lk2stW36M1Q2d0Liu30DTj1Yzkeb62ROycCdMN4l\nSG87HwV0A2Fs4lAKhaYhd505+3keNg+tAr5puySRaysaT6S9uw9QsoskA8+AZzq4\nmzR28luIuvUmtSP+UBrQ6hyW\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDLDCCAhSgAwIBAgIUU+CvNNI2Hg/kyKWUkP+fw3bsFe0wDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAgFw0yNDA5MDYyMjMwMzZaGA8yMTI0MDgx\nMzIyMzAzNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEAppk6tXw7bDClVJHPY1m/yZTl1Nou7Obj8xE7pYK0EYRw\nlhOoAAlt1WIUwnnEn+k8i2hXk5MFbKr/q7XjpXdIEcgPzXQjdH2NODhn6+3oN3Q8\nivqLxiJz9a4fulKzJdmPiMuZ/5cphTxZHRKKmio17+Z6RkEnL6702DfIRbusMNxH\nTyN6wGMKJboxmnW17baay0SGb2ULmCN9CS8nQlbBTMH5tJZ3oBB7a/5cdlG8V3sn\n/ZyDpXu1LDwHUe6EV6Jn+RQqF1Llmo790FvTWRDOBNDtkmx/0TtdjJpnlx9JsELr\n62QL4BLXPGQzM5kWfgDsgl8aehezBPUqwaUKku5YzwIDAQABo3YwdDAyBgNVHREE\nKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cuZXhhbXBsZS5jb20wHQYD\nVR0OBBYEFO2EJhb54LzOxx6W7lqphByWFT8pMB8GA1UdIwQYMBaAFCcUWYVGF5cP\nrQMR/VNkapjuXMRbMA0GCSqGSIb3DQEBCwUAA4IBAQA86URTSVJU6k6VjcnT9Fzh\nfnid+OV2NPoKzczw4pTc7aGjkZxtCD1ENlYhHlcni0ZFMIRtLiDARjwhBkVJ5S84\n1NS5l4J86ymazkSFZ27m8y0UeSDuPxZJFA/yBAmt/BoKRNMAAmonepdx73JpbiGE\nyMD9RU5qI2E6BGo0B2khRYuY+POPFGPueVbqg3qR+LJPlxp8OIet9HGagEcUK7lG\nPeFNKSUCfmuHHD/QO/gmG4ZM9/qB7M3McYh4/+CIihEmhfVK9Odo6Fs0t5MQdcEo\nv6++7DlnpwRnmgC8GtEBMK5XJAILb6cI11TearTphFoP7xpvz0VHp3Gy9mA5cdwx\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/svid/svid-user-a.key", "content": "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgST6YP9hyfw/Vmoxo\nMFp6MJFZu4xaYK3OweYcANEFTkmhRANCAAQCY7xD5sWZDVSRmBu2l4sjJYzpGVqg\nd7M8I6LnFjkhkJFc0h9n8jPud8POip9BfXJyLBzmtW+CfZC84zlFSknN\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/svid-user-a.pem", "content": "-----BEGIN CERTIFICATE-----\nMIICAzCCAamgAwIBAgIRAJXUSiQv6UVx+RHn17gl6xswCgYIKoZIzj0EAwIwMDEL\nMAkGA1UEBhMCVVMxDTALBgNVBAoTBE5BVFMxEjAQBgNVBAMTCWxvY2FsaG9zdDAe\nFw0yMDA1MjcxODI3MTRaFw0yNTA1MDkyMTA1MTFaMB0xCzAJBgNVBAYTAlVTMQ4w\nDAYDVQQKEwVTUElSRTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABAJjvEPmxZkN\nVJGYG7aXiyMljOkZWqB3szwjoucWOSGQkVzSH2fyM+53w86Kn0F9cnIsHOa1b4J9\nkLzjOUVKSc2jgbYwgbMwDgYDVR0PAQH/BAQDAgOoMB0GA1UdJQQWMBQGCCsGAQUF\nBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBS9gO5XK6fzpkTH\nDuhPV7lB2AAbNjAfBgNVHSMEGDAWgBSeUg2uZMN8Eio3bHcxv7zJIzclhzA0BgNV\nHREELTArhilzcGlmZmU6Ly9sb2NhbGhvc3QvbXktbmF0cy1zZXJ2aWNlL3VzZXIt\nYTAKBggqhkjOPQQDAgNIADBFAiA2TvD3xhOCvn9E2QF42o7gTjqGicTeNInKTEKe\nA6AMzgIhAKdpmH5367YqHijKhtfklnM7g8WhdPhn38xWL7jG+5+a\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw\najELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv\ncyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD\nDAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw\nCQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw\nEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp\nEHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO\nBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD\nfBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw\nHQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB\nAQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm\npOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK\ndXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5\nUQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw\nK8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6\nrnloUNEr2o9DlZp8EvK2I4dU\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/svid/svid-user-b.key", "content": "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEiQo4GXKbViodiF2\nLltOkXLauMoyKJu01c/FUoGpnXahRANCAASiSiVhimnedxcnXY1ffLWV6Ez9XIkq\n3pXxtk6q6jvDfn3OPPjIB47OH4KCqNaMoIsKxwK/mtOEETb0/gFqeQWa\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/svid/svid-user-b.pem", "content": "-----BEGIN CERTIFICATE-----\nMIICAjCCAaigAwIBAgIQUGwbDXAjCmdvfiGGjS/+PzAKBggqhkjOPQQDAjAwMQsw\nCQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MB4X\nDTIwMDUyNzE4MjkxMFoXDTI1MDUwOTIxMDUxMVowHTELMAkGA1UEBhMCVVMxDjAM\nBgNVBAoTBVNQSVJFMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEokolYYpp3ncX\nJ12NX3y1lehM/VyJKt6V8bZOquo7w359zjz4yAeOzh+CgqjWjKCLCscCv5rThBE2\n9P4BankFmqOBtjCBszAOBgNVHQ8BAf8EBAMCA6gwHQYDVR0lBBYwFAYIKwYBBQUH\nAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFOZVW2w2T+3afeJU\nJMuZg6Q8FXc/MB8GA1UdIwQYMBaAFJ5SDa5kw3wSKjdsdzG/vMkjNyWHMDQGA1Ud\nEQQtMCuGKXNwaWZmZTovL2xvY2FsaG9zdC9teS1uYXRzLXNlcnZpY2UvdXNlci1i\nMAoGCCqGSM49BAMCA0gAMEUCIQD81ueLXy2MerMclzKoMnP9VDjOLuHVHf7RkLYb\nOdqBigIgH0XT2q5pVmDQgCBP2bKaWZndvXlb5kkPw17XcSD2cKs=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIC3jCCAcagAwIBAgIRAP6sMTwSA5gCsJWO5fSCokUwDQYJKoZIhvcNAQELBQAw\najELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC0xv\ncyBBbmdlbGVzMQ0wCwYDVQQKDAROQVRTMQ0wCwYDVQQLDAROQVRTMRIwEAYDVQQD\nDAlsb2NhbGhvc3QwHhcNMjAwNTI3MTgxNzAyWhcNMjUwNTA5MjEwNTExWjAwMQsw\nCQYDVQQGEwJVUzENMAsGA1UEChMETkFUUzESMBAGA1UEAxMJbG9jYWxob3N0MFkw\nEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJhF2UV33hUpg53uGmy/GkXEI2ZR8EQmp\nEHxG1GWbjHR7FBdVP/HmPyVKu5vfegXZp/hD3H7UYHjiNeKMYyGT4qOBgzCBgDAO\nBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUnlINrmTD\nfBIqN2x3Mb+8ySM3JYcwHwYDVR0jBBgwFoAUlDSkRxR41mV4yHobEDnU1bjsywgw\nHQYDVR0RBBYwFIYSc3BpZmZlOi8vbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB\nAQA7nxOrNqGZ4U72qkB9YYXSi89HNgYoz1R0sLdRuh0BDpSPLASNymZrzbw1CuZm\npOJ6b6blxDlLKBx+tDBgYejRmVCZq+hD8mIVBT0Vg3uZhmPOo2URQmUcfsas9UXK\ndXGh/9FIqq4u3dBA1bCHlKk/bDIu/VkGMkTaHaDXNEcLBSWLdVkMOuuF6YHgKJh5\nUQEsbWt+kfL3MzeMuAQYVuskKWE19+oLfY41jTQUzPY83r9nJkEZaUyVBShj8CAw\nK8QHfKrQ1BE6ALrM1zvMS9zMopoalMtNJ1ILL1nYLD2teVv4iSRGyD7JgHUYYax6\nrnloUNEr2o9DlZp8EvK2I4dU\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/ca.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDBTCCAe2gAwIBAgIUYm7HlYXiXY8u67v2vKjU5LUCHDUwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MjRaFw0zNDAxMzEy\nMTQ4MjRaMBIxEDAOBgNVBAMMB05BVFMgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB\nDwAwggEKAoIBAQC9GEoJ84UWZTsnovuz3StEtri3NdKlOqzNCGvurd3rZgop/jZB\nRtJXQz9ZxdKx1ARpDV1m3CrQe63UbBRXJxA2XGfmBQ/BPo3IPEXJOsNEs9x5RsSL\nRiJ2re7jbWKeQv/ucQPdmLJumAp+TGAzdOM/AnDaVTSLPoARp/Va8Frs7iFfPpuJ\ntObvux4qnb/hxS2z39MWjyeM0dVOmjGwx9opxcE0hNI5ZutkoNxpmRayZqJSe85V\nBSPGsuBwgncvA2GWTNIGFfN2oxQhSuI8yM7+l/0+BHFWfm2G7/09tWDvFnWTSpTQ\nVISM3+6Wh91c6qSd0wsIb8q6jADAD6H8yhT/AgMBAAGjUzBRMB0GA1UdDgQWBBQk\nvMZGyNfVHU3oTUASSfYiT4arNTAfBgNVHSMEGDAWgBQkvMZGyNfVHU3oTUASSfYi\nT4arNTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBxN8yaacok\n1lyg7eTEv7/Sn4VGkfbuWm9ZQ3DPLKJhbe7uVxNChRrR8nKshwlGv1Wa8i0c0lJ3\nO+Uw24gjfhl/8zhFyxh4ijllj6/FVBNqsTvnGQqtKJP4h8kScUIH21mQ6JQAvh4e\nRY1sjPwZp+6vvogSrgQQ32jBaa8vfzcL2wECvnT1YqePVZYuRqEBjIvyG0ALlmE9\nDqZ8gH+W8E5IVulLVJxnYArCT1dW5AyM2fBETLB3PAWvSBkaCBl6QR+hLuyeR4vT\nm6Qx9EKr8MgIpiH7psnx8C9eF5j5HiwHfhwAdWD9W2tRzTxSZP2LJ9E+qjaohdLf\n6NYxXL8AHa17\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/certstore/delete-cert-from-store.ps1", "content": "$issuer=\"NATS CA\"\nGet-ChildItem Cert:\\CurrentUser\\My | Where-Object {$_.Issuer -match $issuer} | Remove-Item\nGet-ChildItem Cert:\\CurrentUser\\CA| Where-Object {$_.Issuer -match $issuer} | Remove-Item\nGet-ChildItem Cert:\\CurrentUser\\AuthRoot | Where-Object {$_.Issuer -match $issuer} | Remove-Item\nGet-ChildItem Cert:\\CurrentUser\\Root | Where-Object {$_.Issuer -match $issuer} | Remove-Item\n" }, { "path": "test/configs/certs/tlsauth/certstore/ecdsa_server.key", "content": "-----BEGIN PRIVATE KEY-----\nMIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAfrq5ri+W7sYQp/6xc\nlH6YbTy43dnnrKUbbdlzsDn4DPzO1k15LVXx8EPK+7vuh5uhZANiAAR6V4nqBt3k\nZfO9H664fPB8PkuDhphBfzxbSFFcr2DXj11g0ZV56Yjnh3RMC4Lud29ofpTQd8IP\n9bspEvjnBvOw60tH9WiquWqxLgSREUZVLEMD1dZ3JSVUfDCI2zzf00s=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/tlsauth/certstore/ecdsa_server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIB2jCCAWCgAwIBAgIUKRYMoky98mN3mpyL6PMIY8/d2OswCgYIKoZIzj0EAwMw\nFjEUMBIGA1UEAwwLbmF0cy1zZXJ2ZXIwHhcNMjUwMjA3MTgxNDM5WhcNMzUwMjA1\nMTgxNDM5WjAWMRQwEgYDVQQDDAtuYXRzLXNlcnZlcjB2MBAGByqGSM49AgEGBSuB\nBAAiA2IABHpXieoG3eRl870frrh88Hw+S4OGmEF/PFtIUVyvYNePXWDRlXnpiOeH\ndEwLgu53b2h+lNB3wg/1uykS+OcG87DrS0f1aKq5arEuBJERRlUsQwPV1nclJVR8\nMIjbPN/TS6NvMG0wHQYDVR0OBBYEFHbcfCfGs+l2bVg22WLTdV10AnpTMB8GA1Ud\nIwQYMBaAFHbcfCfGs+l2bVg22WLTdV10AnpTMA8GA1UdEwEB/wQFMAMBAf8wGgYD\nVR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49BAMDA2gAMGUCMQDhzRyw\nQ+m2fMFyqIgFc890jLIzh2bGqlmdkUpb+/Z/y9zKZQPSG5xhXp7A/FhvM24CMHVW\nZIWBCJJRhw/L3s73QHX1d+M6mNqES16cnnht6j9DF1AddIipcsnBcpo4s7K/Xg==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/certstore/generate_ecdsa_test_cert.sh", "content": "#!/usr/bin/env bash\n\nset -eou pipefail\n\nSCRIPT_ROOT=\"$(cd -P \"$(dirname \"${BASH_SOURCE[0]}\")\" >/dev/null 2>&1 && pwd)\"\n\ncert_file_prefix=\"${SCRIPT_ROOT}/ecdsa_server\"\nexport_password=\"s3cr3t\"\n\nopenssl req -x509 \\\n -days 3650 \\\n -newkey ec \\\n -pkeyopt ec_paramgen_curve:secp384r1 \\\n -sha384 \\\n -subj \"/CN=nats-server\" \\\n --addext \"subjectAltName=IP:127.0.0.1,DNS:localhost\" \\\n -nodes \\\n -out \"${cert_file_prefix}.pem\" \\\n -keyout \"${cert_file_prefix}.key\" \\\n -outform PEM >/dev/null 2>&1\n\nopenssl pkcs12 \\\n -inkey \"${cert_file_prefix}.key\" \\\n -in \"${cert_file_prefix}.pem\" \\\n -export \\\n -password \"pass:${export_password}\" \\\n -out \"${cert_file_prefix}.pfx\" >/dev/null 2>&1\n" }, { "path": "test/configs/certs/tlsauth/certstore/import-p12-ca.ps1", "content": "$fileLocale = $PSScriptRoot + \"\\ca.p12\"\n$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText\n$User = \"whatever\"\n$Cred = New-Object -TypeName \"System.Management.Automation.PSCredential\" -ArgumentList $User, $Pass\nImport-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\\CurrentUser\\My -Password $Cred.Password\n#Import-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\\LocalMachine\\Root -Password $Cred.Password\n# TODO? Move to trusted enterprise? Requires some fingerprint parsing." }, { "path": "test/configs/certs/tlsauth/certstore/import-p12-client.ps1", "content": "$fileLocale = $PSScriptRoot + \"\\client.p12\"\n$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText\n$User = \"whatever\"\n$Cred = New-Object -TypeName \"System.Management.Automation.PSCredential\" -ArgumentList $User, $Pass\nImport-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\\CurrentUser\\My -Password $Cred.Password" }, { "path": "test/configs/certs/tlsauth/certstore/import-p12-server.ps1", "content": "$file=$args[0]\nif (!$file) { $file=\"server.p12 \"}\n$fileLocale = $PSScriptRoot + \"\\\" + $file\necho \"Installing certificate $fileLocale\"\n$Pass = ConvertTo-SecureString -String 's3cr3t' -Force -AsPlainText\n$User = \"whatever\"\n$Cred = New-Object -TypeName \"System.Management.Automation.PSCredential\" -ArgumentList $User, $Pass\nImport-PfxCertificate -FilePath $filelocale -CertStoreLocation Cert:\\CurrentUser\\My -Password $Cred.Password" }, { "path": "test/configs/certs/tlsauth/certstore/pkcs12.md", "content": "# PKCS12 Files\n\nRefresh PKCS12 files when test certificates and keys (PEM files) are refreshed (e.g. expiry workflow)\n\n- `client.p12` is a p12/pfx packaging of `client.pem` and `client-key.pem`\n\n`openssl pkcs12 -export -inkey ./client-key.pem -in ./client.pem -out client.p12`\n\nTo add the CA, use the following:\n\n`openssl pkcs12 -export -nokeys -in ..\\ca.pem -out ca.p12`\n\n> Note: set the PKCS12 bundle password to `s3cr3t` as required by provisioning scripts\n\n## Cert Store Provisioning Scripts\n\nWindows cert store supports p12/pfx bundle for certificate-with-key import. Windows cert store tests will execute \na Powershell script to import relevant PKCS12 bundle into the Windows store before the test. Equivalent to:\n\n`powershell.exe -command \"& '..\\test\\configs\\certs\\tlsauth\\certstore\\import--p12.ps1'\"`\n\nThe `delete-cert-from-store.ps1` script deletes imported certificates from the Windows store (if present) that can\ncause side-effects and impact the validity of different use tests.\n\n> Note: Tests are configured for \"current user\" store context. Execute tests with appropriate Windows permissions\n> (e.g. as Admin) if adding tests with \"local machine\" store context specified." }, { "path": "test/configs/certs/tlsauth/client-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCjAaejJ5wqwsst\n7T71LQCdsbRpctZU2ZbTIOalEvHNikkdB2gIqxlmswxITFVtNp2IvVhcd4KaNGdu\na4uUc7BifG6XwFtmw54SPTlbFtZbr1y4mjF6+Wm3tV0XnMtM18wvkRm7wbYG5fz8\nBfWl4tbCDF8lnuC3YHFwVJst+o1HoZ4YNutDFJqMXkHuxEPsI5PmS3XHHpttPQGr\nvKnoYR0y0Cf4y3LCfQCQ+bFe2JS335WmJy2KVm+dpN+L2ASO8VvV/YTltSy8f4HQ\n3Dd/VjZtri7ANP61Xrrx4OJS6IRkPo3uxaR306oKyF9a4irX1KY+Vi1gpYOX9foc\nmVumCWCrAgMBAAECggEARk+F7CG/QlCQjEhb0ixtqheHPr7KhYHvhTUZV5XC2Aow\nfEWAEdEfnUVY5GyMopWewOcPUJ86JeK5xI69/7QhHnIWz/0oT7zMF4jyDwDcSGLt\nRzE3a5heid/Aflli9cvVZqUbaPnm1rXoeBrn+PxN7xigB92uh1qhw7ay0tPSkdUL\nsIA2fkouwAYIE5TDhKk4lF8Xg6KlYJEesFDGTVsaGL8/RH7VEH/Z7EssiHF60m4l\nOj6Vg7gxMB1UWQFw2/LDMrDw0bLVdAoOC0M7NLXlFdoeKPREvFpIqw8BWMF3NxRV\nOpUiLB/tv6A7nqevaa8KoWGUlCVY2DkCkgCmg+K8pQKBgQDVtb3nEJrOR496BaLn\npHKV+pEQjlVfcy36CoTa2PblbjkgSyA6wL9Itpkw4OF/Q8dop0qlbqZM9gUdjEhB\n6DfVINNMGx4SkWDtBvPwouCRIWRvZUXdVuZMgDpUY+5n2nEkUHbmSUEtPbhHn5dp\n9Tn1dsjNjweDTx2Q6nDuKUlSVwKBgQDDQ1fyyp6qI6opeFkfOsZ78Yk2lM6tRSkI\nJqwdOwZoSptZfiaoAQtmwU3zxioFgIv4FjbPCosJyej0Egv7oyMucZHPGhP6kZDJ\nLxZ0Jp0kDYYFCmUjOEpDf0mSmFEwmyPDb8052To3EuA4anEQd1KURG/0yKE4efwP\n0m3ml9R3zQKBgAh1cxjMPXRgvLsVsgb9KVPqYQeIurRWeMFm3S9UWyFlpXkzwAjT\nTD7yi0m1/Pbuldv8kyXNJWPycO1keg+xw1P6QqLGiAAwJOf82Hbz23OjILiQB53l\nLKRmhuiENBGEQeowDSS8TYoe4UZkeLfG7w5aL0SDnsaBwSfVP7cNh0ttAoGAEJDO\nDVMTUuvjq9EB/pxF6o37Th4hyqFrcb2WLIStbnul4lnJfcdY6EbODjhpqD3XohyA\nWeBTG2l90fcV/StB+Na5wBA+Uau31Nmh1gjQnBZpoFPZcLt90WwjGcTCXpVK23HI\nv3emcLWxQBgHr5Xv85Q6y1GaG+h9cfowSLfo1qECgYEAmqGyMa9OZ44+zni3nq2A\nJM2t4B5QJdG36R/2m4J7odUnmsaRVxw6JxaoF0CctFxGs/zdumwuhYeTfq6Lj/ly\nN1kJbvJJpguN1h4sOOLEk646mZzK2h5bMxsmPU2kNZFf1EUlB+Ro8i1TDvGY26Qs\nqgjd6RlSZFBmSfVS47ihEQA=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/tlsauth/client.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDPjCCAiagAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipYwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ5MzRaFw0zNDAxMzEy\nMTQ5MzRaMCgxEDAOBgNVBAsMB05BVFMuaW8xFDASBgNVBAMMC2V4YW1wbGUuY29t\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAowGnoyecKsLLLe0+9S0A\nnbG0aXLWVNmW0yDmpRLxzYpJHQdoCKsZZrMMSExVbTadiL1YXHeCmjRnbmuLlHOw\nYnxul8BbZsOeEj05WxbWW69cuJoxevlpt7VdF5zLTNfML5EZu8G2BuX8/AX1peLW\nwgxfJZ7gt2BxcFSbLfqNR6GeGDbrQxSajF5B7sRD7COT5kt1xx6bbT0Bq7yp6GEd\nMtAn+Mtywn0AkPmxXtiUt9+VpictilZvnaTfi9gEjvFb1f2E5bUsvH+B0Nw3f1Y2\nba4uwDT+tV668eDiUuiEZD6N7sWkd9OqCshfWuIq19SmPlYtYKWDl/X6HJlbpglg\nqwIDAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93\nd3cuZXhhbXBsZS5jb20wHQYDVR0OBBYEFA5GJsoT3uf5GCyL5KSFoyEuRVltMB8G\nA1UdIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IB\nAQAmBOQOp3CR5/P27RUonsgf5MCzblt6tBPJBMiZwB8sJyl9fXCTkzdqD0rJQb7q\n5s7p00rUXdeh13oRjFcoFuopMDCk3kBYnKJHXRREJNLHfp6CPUMKlg0GJUZ6v04A\nV7gVuhvmynHrmlbibMwbgZtZMnRU3x8JjawEUsEhoj3O2Qfen3sNfaOBlnwVUCBQ\nygSHQ0Pto1kQS+1Pc5DCwnOZ/qh5lORPdO1MNKqeu8HiiSJfuaCrQQM9zm72CHHY\nF755qy8OvWjwK0H9rCFBYSrAnYk/pTvXIeBsgNRlURS/qv1rqIEvAbXhRnw7oyvl\nP4bYY4pcpk32Ir2mFQFRQnSh\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/client2-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMsHsIYdxGt8ws\neoxGQQs4K+AfvznwuW+Qbnx+eGTS+OSaLqL9u81JdXxheeAVsVjRtz5qP3X9Vu75\ngSQU+AVhFp689VNR41vaOsKvX2dzVCaWCEwv66CekAop89vyB6fm14uSr9IusixH\nu6vfjuze2FZE5xcw5mSYMysqKTbM4DMjXFJXFg5mw1YC9zkh3zCWUZdmidafwOB6\nMiJkVZsAQjAwc5oaD4I/ubCdmWr8COYbbwUl32k+t0eQfj9cvd4+R7uBYVNWu1BN\ng0RqNZZlIM4EVonc1J+eN7oGo57jLrwf4jCOkcg2wd6X62CuuQHhwddQek4KGthM\nJUIH7Q9TAgMBAAECggEAE80u2cy9xomZUuQ4FcPNFg4IjImvTT5jMJG/sWxsNIyn\ncNL6KZm1blnTQorLxs11TjRv8U9aVrvGOpTnrK+htZa+nIEPImjgRehRVS3hkCKf\n6Pu8gxZEX5KHqS9SI8Ph1k8bzYD80E+kQPxC0Em/WH+NOPUyJSTkrmSk1FtQVdle\nKFk0+dRm10guI/rcInPUjpY6Cv9nSiAhanRPAtAANUvwYxVEMKzx5ns/JnQfAYAK\ngYTopXAT61VqW1jn4opjD+OqparjbK362q/spggHGPhSamXq8y/N9JadSx+5591u\no0oZm/UxefwP1CkE4Q6Pv9n3yiZOwVaOiLLVzmr6wQKBgQD4kC953if336Ga+BdB\nHj83qUk5U2Avp6qWLptd6z/VQXDwlL2oQaaXP83T6XfpXTMwDabUYkpo1JZrFOJl\ns9eZfxIDa0eH6evphqp6s/V6EhZkI9lL59ptZlK1hv1glmFDfn7WB1Yo7d8GgUx/\nS7CKoBK2WaX8YNsW9fIunadJeQKBgQDS0EDR46KiQBrjvNK4UxdfERl2af+6Sf+J\nBRlfj+YA9/zKEL3jZGWVa5rgrD8BJrTsa86QSSACsxQx5J9vPogKgogmghd9RQVd\ntHvIV0C2C6zCmZIvNuzn5+wMV7bWdoABPgbuZGTpc/LCwm//1EbwdNCTom8ssE19\nNTIBb7t4KwKBgQDXOqKheAhLzkz1D1WzgSlkXSWWieeD3D8OBBVsYcPIOP4+k80V\n4KML3KexkzvNynIEbg3DYcjktQ/6cP8I6Y0K0MkcRMyPl7I7Z+w+i41Hwlm5JIGI\nBJ9Sk4OSw+yqsgxOkT3qvjeRAUhZLaS7pSKdJraNR1s/Ce8sFpM6YjD0oQKBgAUc\ngYXVRBs0/LHq4R0Q/q8SZhCl70pgAu8ajYvwnD4HxTxM/Z2m0IO38TBjXL+1ZYuZ\n7Y84BquqFeJDzc3PsVK36X8thk5GPyQPfTTVUL9ZNx4cxRuZ9FKHIAUIl2lJxD7D\ndz2Od5fldMxeFIMabYHlAy2hMZrex3IyuPyp7dyzAoGAKCqmXStl7QzV6ovrjyJ1\noMP3BcQhTe1CvOEW7pGce/IGxqB0p7gICIH4UUDEVpitgvGLkOGScsd1rS86Fn+/\nRUlQQdgRIwzrlF1USV1kJ/SUI+7BbhiXUeRUmKuzPLgLSq8K3xEW0z21juSvq64W\ngtiiGjt3D3I47v7ecjJ9dIA=\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/tlsauth/client2.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDOzCCAiOgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipcwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTUwMzlaFw0zNDAxMzEy\nMTUwMzlaMCUxDTALBgNVBAsMBENOQ0YxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIB\nIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzLB7CGHcRrfMLHqMRkELOCvg\nH7858LlvkG58fnhk0vjkmi6i/bvNSXV8YXngFbFY0bc+aj91/Vbu+YEkFPgFYRae\nvPVTUeNb2jrCr19nc1QmlghML+ugnpAKKfPb8gen5teLkq/SLrIsR7ur347s3thW\nROcXMOZkmDMrKik2zOAzI1xSVxYOZsNWAvc5Id8wllGXZonWn8DgejIiZFWbAEIw\nMHOaGg+CP7mwnZlq/AjmG28FJd9pPrdHkH4/XL3ePke7gWFTVrtQTYNEajWWZSDO\nBFaJ3NSfnje6BqOe4y68H+IwjpHINsHel+tgrrkB4cHXUHpOChrYTCVCB+0PUwID\nAQABo3YwdDAyBgNVHREEKzApgglsb2NhbGhvc3SCC2V4YW1wbGUuY29tgg93d3cu\nZXhhbXBsZS5jb20wHQYDVR0OBBYEFEbKWjY1gwvxJusV+M5wUn+7MKR5MB8GA1Ud\nIwQYMBaAFCS8xkbI19UdTehNQBJJ9iJPhqs1MA0GCSqGSIb3DQEBCwUAA4IBAQAo\n873zVMG6tfoPRUZ/kEJcPEIaLmaolALyLEx3sZHe/B8hszuuMecBEfa02HwTSlzq\nfKrkME95LGE9D+4hxyPEqPTqruESShUvBFQIoTQxePAhhUG9icF4gqUpYvRHXMiR\nxIyfH3/KojDlBXRfDOaoXEXshiXfcYqbeh2qFdaoN24Vyh6lkNa2K3SUDAtKVFiF\njjBtNXuH/IJ3EWbs5AOOy98QtBMlT7kmummJVeaRR4QUfnzFrlj5nMSIopoxDm4N\nQeoQSC+63fce6ZJLGQqEFQNR6howBcDQ/8fMR/oLsJ1Hr9VshIsu3kGTk4RUqHI9\nipGt1UTvVf/uMUzA3yoC\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/server-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAgrLiRyYcP54+\nAlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1Puv8\nJ9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTCk/tV\nkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0+pM+\n5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbhC/Gp\nOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKOiZC5\nxYXBxO4DAgMBAAECggEAA+ILzXOfVneHPHuO4acuRvoIG/yoT1OgHcWoPl16ywBX\nBBUPYh9Yqm/lEQrDDwRCpuX54HEEVO+WRY/gpEHv/I6z8k3FTgWu66h6qhZGW/eo\nU02yRzdFBz9AV/xzWpvUEvUq3Uk/GkBvRxKyiN7RPMODpXktXxKTo9Kg/XTYnjIb\n2RWzKdoeptX6q/RRdPkBXxYV5XIaZLXPy+VaGgI7QcwApmCcin20c5smtLmGE7TV\n4hh6wxPwGEiDkfMQGnqWd+VdzbnZjZF2/4ZVCNAbJJaFffTxnoC0uGjE3ImAztmT\nTgmFbdfO/5qHTkJmfZaAbmo461k1O2Vd/p6LCN1mhQKBgQDokSJxOMuYurf1STAb\nom6E5lGXAYR/2TsBxzfWPvUlZDzaSZ7IU+/zpv9GTATc0mvdIpnT2kZvRoz7nbbR\nLFGjc2d1ITRBuDEX+qDkX1puFyhoYPyaz0AuC3AMqg28e3Z1I8nq3Op1TUKAMr9d\nZdOfRUcL65OM+YHPZ197AVwC9wKBgQDT6FgGceqBr3XJDikjU+kJfyrZrcE7sfrE\n3RIV6uMj6Myw4WpUpuH7hwqQ0xAir8UKHG6SDScymXtrJCoG1xJmGGDyoLt80VsF\nfXVhzL8rhIl48TVWptRUiIeZ2dXvianLkLPlLhVUcTThpjjAAbnNYhod1bHygX12\nw+Bd8lkeVQKBgBYLS7x3qbS8Xht96HV2HAu02R77IdgMey9b9sr0BMCak7oNKGPM\nsP3jYmcDZaKYv2iikvolwm9hvJNNC7sf/E0F71SG5TEliGHBe+apsySkRUw/hTIX\nWvoCU4ifxdWLzlqkHcuJTR/5RshoBwOPV1PNeUKD/eRq8gb6wW4jXtlZAoGBAK2o\nC3MEqcQrUSA53ZaY7jGdKDWJQgC0oyfvbyHNAuVro0sU/3lt5WWmTg9PGDsExjm6\nARbpdoTt6Ilt8o72c5p9Qf2zoNHyE2CVZruF+egkzi/xo99mCj1YQZ/gN4T80MwE\nwpf+wvYXa9m7yWf4QhbA3VwzwodUfMf2T4lN0KCdAoGBAM04ecwBXCNIXzTgJDeK\nzxfI95lphcNywdhUNw6KJPDg4ozPu72XrVToAtl7l6+2RPygNhSj117002AAgz30\nl0cfZoogXmIhZt+R39/2e9y31k+GDdEmNtPLsvbLPQ+Ttnab87pUABn4UC1XXGhF\nVt/jfLEw8ZDjB+aO4kFyZPIQ\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/tlsauth/server-no-ou-key.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDepPh4tWYw66pP\nGCEgpYzBTW36/s7cRnsuqhkiUolc5RB//JYaRz/sbIlY7tM8Z21nmjU0IXvlKPUt\n+6Winv7VqDU7S1gOssnIPbVhXGCb4fC3JGjQ2qD/pO5WfoUh8yYph9tFg07+zMyQ\nXgJiE8JglKCeD7kjna0/lkk63ddQP0npbH5943oMvHDj7UjJyBdE8nWX1LXWAmYJ\nWxym6MDfjWTVnyzwqm8fkl/BHp9fh98Nf9U6/gpq0ckRUHL4prhMw0ChhBVjkBQ8\nCrXrp82TQZbeFPm3uWTV3xf5bnbeO87dgiXP47yvqVFSQI2cFyo2JzEq6LVAeVPT\ndKzENXyFAgMBAAECggEAHlLdvKMIPhV65rbknCuwFgvTtOHLjtjSojJspe4T42EX\ndDcUwpN9s1e9BS3R+2Ii1n98S5Nb6oQ/kHm7v4BkOPll9qN2ZNoY/XraH16Tkeed\n/3OoCvob/3WZOJKW017ojbOBO+B8e9us6OTE8lK6oKjdj2mYz68ED6sKYkggsT8M\nah4KkzrX8xNiQwMRs0VMfXb1M8DUd+nWcamWdUytf3ZPFWxymzYzjjxDuisK4faV\nFuqwk2FRtOuyMeygC7KuPwjBtqXuFuHLYtOlibs5ZFG9Nwd9lWQEaVCnOlgW35zz\nCVkdszfCgimCL/K42vcmbdnI3GUO/g3szw9Vy95sAQKBgQDx/PHZks4FdxXSkuxZ\nu3Qjh5d/sG9NMK0QUNgF/MnORQjvmVfQOSXObiMxoXBBNW25St00mO0W+ZzXee5z\n3ANw6cPgSXla29RKC4yVuAXwOn/kPnuASQxdoGWEKuZJqUKbpK+FDgFbK9cZlSd/\nVNS9eW2j00CCqXK0FQ7FvfCoBQKBgQDriUonm0EwCoNrBlbllOynFYjFwg2yu/ti\nH/1SAdmTcD+NZFRwhBmSKxni2rYyMoyqkHN93t8Sz8ELIK4sEsA/XUpEioeGIwMG\nHQSRga2XYUVafuI/TQRHeMke703rtbIN7dkjEiWsM1gHoPcTRxZU9hRGq0p89GHq\nzOf4tuAqgQKBgA3ceVYHLLnvalaXh+ZT8IEggTMVPirjwOYQW29sXXrtRWfEFt2c\niGfcszNilfWGQ/S7LxSWNe58+dj16QzF64SKP2gXjVYBBZYAN1tUITLzhuPiGFzu\n0kCCsY3yjyJlCaW0t0Ed3kIErtuOSabnixAXZopdzXIulp1uH1yAVsqpAoGBAL+7\nua62FoGp/ULRHUm0SlTVFcqN5iK6Ha/KBKeOM/Ruan2Jz6bsEfjHt0HQ8oG4XoO2\nJR2woHyqvCV3y/C6rt6l7YAQGLRbqel/E6nzG0FggFljcn8/DZ20uFvDR/X5qWDn\nXlvLOPmNrjo/kQGTW5172BOS+obvVQoTFT6Ed8SBAoGBAI5802KpwzgN9IjPM1Zf\nrBZ6PshZ14PFxeQ2tOhvFcr+N+ceSY19lex+UyQn7F214Kz2U3n35zgNdLhdHLK6\nWokXp21WDqeg8n1cUrQ43yWqVX8cjdNx3P84B5Mdnb0nhbM2smUSuhrO+fOzr37A\nJegHloScSH3BcemRFfK31+ba\n-----END PRIVATE KEY-----\n" }, { "path": "test/configs/certs/tlsauth/server-no-ou.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDKjCCAhKgAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipUwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMTQ4MzJaFw0zNDAxMzEy\nMTQ4MzJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBAN6k+Hi1ZjDrqk8YISCljMFNbfr+ztxGey6qGSJSiVzlEH/8\nlhpHP+xsiVju0zxnbWeaNTQhe+Uo9S37paKe/tWoNTtLWA6yycg9tWFcYJvh8Lck\naNDaoP+k7lZ+hSHzJimH20WDTv7MzJBeAmITwmCUoJ4PuSOdrT+WSTrd11A/Sels\nfn3jegy8cOPtSMnIF0TydZfUtdYCZglbHKbowN+NZNWfLPCqbx+SX8Een1+H3w1/\n1Tr+CmrRyRFQcvimuEzDQKGEFWOQFDwKteunzZNBlt4U+be5ZNXfF/ludt47zt2C\nJc/jvK+pUVJAjZwXKjYnMSrotUB5U9N0rMQ1fIUCAwEAAaN2MHQwMgYDVR0RBCsw\nKYIJbG9jYWxob3N0ggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tMB0GA1Ud\nDgQWBBS/WuyC1fvhLL8rnKIXWFdQFnGpozAfBgNVHSMEGDAWgBQkvMZGyNfVHU3o\nTUASSfYiT4arNTANBgkqhkiG9w0BAQsFAAOCAQEAVgdcRJgo95MEDgEACekziOHn\nn86DdgNin4FDkL7Y2sBDrpej4B0jKPm2H/M4qdB2Z17Ru2TdcXOyk/sMc395GHsN\nBAdKuAcysvQ+USR3UXasJmC/CvoKGBOmFf9/Jor8U4Rs01bkXSd6pW8ytT3kyMak\n3r5tNugzRxpJvVDgjHlUkfhBoLeeCr+k1cN1OvR4cFhY6vxqS6GBdopFGC3DlnTL\nLPetNhQCd+r2mH1RT/56aLLRawy76GkBEZm/+mg+mYjxN3J1hWibouF4ccutvxtt\nh2/4PJNsXv5yt4wibazFixJ843KPdfw6pafXbYZsvvgNfvLrpp8beCUwHEYq+w==\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/certs/tlsauth/server.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDRjCCAi6gAwIBAgIUTKhkrEgzI82ef4hYCjeQsZ2FipgwDQYJKoZIhvcNAQEL\nBQAwEjEQMA4GA1UEAwwHTkFUUyBDQTAeFw0yNDAyMDMyMjA0NTdaFw0zNDAxMzEy\nMjA0NTdaMDAxGjAYBgNVBAsMEU5BVFMuaW8gT3BlcmF0b3JzMRIwEAYDVQQDDAls\nb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAgrLiRyYc\nP54+AlElXoMjhOMg+VZjkVnC4CVXk+3jmKqAzI6bum9QHBnSdN/ULaPFU4Vgoqt1\nPuv8J9snNlSSE4CcgCCmWihFIwSpIaXW/GWCZVvCQDn3QxAZt48vTtEI27tMsRTC\nk/tVkuShJ7OxrJyXcntBWbS+TeSOMVIF/v6lqWGjaDauPK0cpesa/qLElaHNlIV0\n+pM+5b3LIcEpuaECVwu5n5c9m/qdX7ZyPF2x8Z2I0zO5nLyFxb3WolPjok/qnZbh\nC/GpOX459F6yCVBmjkakaJocV1Ue7V0dFB7u3aYW0dwa+7Zb0613ZHMWKfAOzJKO\niZC5xYXBxO4DAgMBAAGjdjB0MDIGA1UdEQQrMCmCCWxvY2FsaG9zdIILZXhhbXBs\nZS5jb22CD3d3dy5leGFtcGxlLmNvbTAdBgNVHQ4EFgQUcpTmScu/KZHhs0KbdxtN\nGXndXBEwHwYDVR0jBBgwFoAUJLzGRsjX1R1N6E1AEkn2Ik+GqzUwDQYJKoZIhvcN\nAQELBQADggEBAKdqG82z3JBim/hiGf4LZT90ZHhw7ngPT9HUV4jYRIk0ngJ37ogK\nKCYW0UBCkugdf0elxcggjAsJZGlz+hW2j8MynEqJ9UU7jPPp4AKJqZHy5x49Y1iL\nkFlJE5a3LFJUaVG4JeYMqTL2zDtoj+hk7QPPoz88moDUbOHg3HccObHlISelVPON\nK/kvnJ2NfXImYkh7MusRxVuB4LcRRi5rwT0pOdtSPBCeSH96BOeCHTriPHGecgc4\n71tgSaELXPM1YnaM2WmXoGU1MZ7Dx6c2q97FI+SWgKfm7B1GQGyAghgKxlRyhfNj\nUvCrbaZDInrMWpMo3+upIBWpHzfmJVvUcYI=\n-----END CERTIFICATE-----\n" }, { "path": "test/configs/jetstream/restore_bad_stream/backup.json", "content": "{\n \"config\": {\n \"name\": \"TEST\",\n \"subjects\": [\n \"foo\"\n ],\n \"retention\": \"limits\",\n \"max_consumers\": -1,\n \"max_msgs_per_subject\": -1,\n \"max_msgs\": -1,\n \"max_bytes\": -1,\n \"max_age\": 0,\n \"max_msg_size\": -1,\n \"storage\": \"file\",\n \"discard\": \"old\",\n \"num_replicas\": 1,\n \"duplicate_window\": 120000000000,\n \"sealed\": false,\n \"deny_delete\": false,\n \"deny_purge\": false,\n \"allow_rollup_hdrs\": false\n },\n \"state\": {\n \"messages\": 10,\n \"bytes\": 381,\n \"first_seq\": 1,\n \"first_ts\": \"2022-03-07T23:59:01.710801Z\",\n \"last_seq\": 10,\n \"last_ts\": \"2022-03-07T23:59:01.712378Z\",\n \"num_subjects\": 1,\n \"consumer_count\": 1\n }\n}" }, { "path": "test/configs/jetstream/restore_empty_R1F_stream/backup.json", "content": "{\n \"config\": {\n \"name\": \"STREAM\",\n \"subjects\": [\n \"stream\"\n ],\n \"retention\": \"limits\",\n \"max_consumers\": -1,\n \"max_msgs_per_subject\": -1,\n \"max_msgs\": -1,\n \"max_bytes\": -1,\n \"max_age\": 0,\n \"max_msg_size\": -1,\n \"storage\": \"file\",\n \"discard\": \"old\",\n \"num_replicas\": 1,\n \"duplicate_window\": 120000000000,\n \"sealed\": false,\n \"deny_delete\": false,\n \"deny_purge\": false,\n \"allow_rollup_hdrs\": false,\n \"allow_direct\": true,\n \"mirror_direct\": false,\n \"consumer_limits\": {}\n },\n \"state\": {\n \"messages\": 0,\n \"bytes\": 0,\n \"first_seq\": 0,\n \"first_ts\": \"0001-01-01T00:00:00Z\",\n \"last_seq\": 0,\n \"last_ts\": \"0001-01-01T00:00:00Z\",\n \"consumer_count\": 0\n }\n}" }, { "path": "test/configs/jetstream/restore_empty_R3F_stream/backup.json", "content": "{\n \"config\": {\n \"name\": \"STREAM\",\n \"subjects\": [\n \"stream\"\n ],\n \"retention\": \"limits\",\n \"max_consumers\": -1,\n \"max_msgs_per_subject\": -1,\n \"max_msgs\": -1,\n \"max_bytes\": -1,\n \"max_age\": 0,\n \"max_msg_size\": -1,\n \"storage\": \"file\",\n \"discard\": \"old\",\n \"num_replicas\": 3,\n \"duplicate_window\": 120000000000,\n \"sealed\": false,\n \"deny_delete\": false,\n \"deny_purge\": false,\n \"allow_rollup_hdrs\": false,\n \"allow_direct\": true,\n \"mirror_direct\": false,\n \"consumer_limits\": {}\n },\n \"state\": {\n \"messages\": 0,\n \"bytes\": 0,\n \"first_seq\": 0,\n \"first_ts\": \"0001-01-01T00:00:00Z\",\n \"last_seq\": 0,\n \"last_ts\": \"0001-01-01T00:00:00Z\",\n \"consumer_count\": 0\n }\n}" }, { "path": "test/configs/nkeys/op.jwt", "content": "-----BEGIN TEST OPERATOR JWT-----\neyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJhdWQiOiJURVNUUyIsImV4cCI6MTg1OTEyMTI3NSwianRpIjoiWE5MWjZYWVBIVE1ESlFSTlFPSFVPSlFHV0NVN01JNVc1SlhDWk5YQllVS0VRVzY3STI1USIsImlhdCI6MTU0Mzc2MTI3NSwiaXNzIjoiT0NBVDMzTVRWVTJWVU9JTUdOR1VOWEo2NkFIMlJMU0RBRjNNVUJDWUFZNVFNSUw2NU5RTTZYUUciLCJuYW1lIjoiU3luYWRpYSBDb21tdW5pY2F0aW9ucyBJbmMuIiwibmJmIjoxNTQzNzYxMjc1LCJzdWIiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInR5cGUiOiJvcGVyYXRvciIsIm5hdHMiOnsic2lnbmluZ19rZXlzIjpbIk9EU0tSN01ZRlFaNU1NQUo2RlBNRUVUQ1RFM1JJSE9GTFRZUEpSTUFWVk40T0xWMllZQU1IQ0FDIiwiT0RTS0FDU1JCV1A1MzdEWkRSVko2NTdKT0lHT1BPUTZLRzdUNEhONk9LNEY2SUVDR1hEQUhOUDIiLCJPRFNLSTM2TFpCNDRPWTVJVkNSNlA1MkZaSlpZTVlXWlZXTlVEVExFWjVUSzJQTjNPRU1SVEFCUiJdfX0.hyfz6E39BMUh0GLzovFfk3wT4OfualftjdJ_eYkLfPvu5tZubYQ_Pn9oFYGCV_6yKy3KMGhWGUCyCdHaPhalBw\n------END TEST OPERATOR JWT------" }, { "path": "test/configs/nkeys/sigkeys.txt", "content": "\n########################################################\n# TESTS ONLY #\n########################################################\n\n# These are the public signing keys\n\n-----BEGIN SIGNING KEYS-----\nODSKR7MYFQZ5MMAJ6FPMEETCTE3RIHOFLTYPJRMAVVN4OLV2YYAMHCAC\nODSKACSRBWP537DZDRVJ657JOIGOPOQ6KG7T4HN6OK4F6IECGXDAHNP2\nODSKI36LZB44OY5IVCR6P52FZJZYMYWZVWNUDTLEZ5TK2PN3OEMRTABR\n------END SIGNING KEYS------\n\n# These are the seeds.\n\n----BEGIN SIGNING SEEDS-----\nSOAO7RDW6CLJORHHBS4DPYYIIIAASEIUJ5WWS5FMWLNTFHUCKQ5CAC45AA\nSOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ\nSOACSMP662P2BZDKVF6WCB6FIQYORADDWWWEAI55QY24CQRTY4METUING4\n------END SIGING SEEDS------\n" }, { "path": "test/configs/nkeys/test.seed", "content": "########################################################\n# TESTS ONLY #\n########################################################\n\n-----BEGIN TEST OPERATOR SEED-----\nSOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU\n------END TEST OPERATOR SEED------\n" }, { "path": "test/fanout_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race && !skipnoracetests\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\n// IMPORTANT: Tests in this file are not executed when running with the -race flag.\n// The test name should be prefixed with TestNoRace so we can run only\n// those tests: go test -run=TestNoRace ...\n\n// As we look to improve high fanout situations make sure we\n// have a test that checks ordering for all subscriptions from a single subscriber.\nfunc TestNoRaceHighFanoutOrdering(t *testing.T) {\n\topts := &server.Options{Host: \"127.0.0.1\", Port: server.RANDOM_PORT}\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s\", s.Addr())\n\n\tconst (\n\t\tnconns = 100\n\t\tnsubs = 100\n\t\tnpubs = 500\n\t)\n\n\t// make unique\n\tsubj := nats.NewInbox()\n\n\tvar wg sync.WaitGroup\n\twg.Add(nconns * nsubs)\n\n\tfor i := 0; i < nconns; i++ {\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected a successful connect on %d, got %v\\n\", i, err)\n\t\t}\n\n\t\tnc.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, e error) {\n\t\t\tt.Fatalf(\"Got an error %v for %+v\\n\", s, err)\n\t\t})\n\n\t\tfor y := 0; y < nsubs; y++ {\n\t\t\texpected := 0\n\t\t\tnc.Subscribe(subj, func(msg *nats.Msg) {\n\t\t\t\tn, _ := strconv.Atoi(string(msg.Data))\n\t\t\t\tif n != expected {\n\t\t\t\t\tt.Fatalf(\"Expected %d but received %d\\n\", expected, n)\n\t\t\t\t}\n\t\t\t\texpected++\n\t\t\t\tif expected >= npubs {\n\t\t\t\t\twg.Done()\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t\tnc.Flush()\n\t\tdefer nc.Close()\n\t}\n\n\tnc, _ := nats.Connect(url)\n\n\tfor i := 0; i < npubs; i++ {\n\t\tnc.Publish(subj, []byte(strconv.Itoa(i)))\n\t}\n\tdefer nc.Close()\n\n\twg.Wait()\n}\n\nfunc TestNoRaceRouteFormTimeWithHighSubscriptions(t *testing.T) {\n\tsrvA, optsA := RunServerWithConfig(\"./configs/srv_a.conf\")\n\tdefer srvA.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\n\t// Now add lots of subscriptions. These will need to be forwarded\n\t// to new routes when they are added.\n\tsubsTotal := 100000\n\tfor i := 0; i < subsTotal; i++ {\n\t\tsubject := fmt.Sprintf(\"FOO.BAR.BAZ.%d\", i)\n\t\tsendA(fmt.Sprintf(\"SUB %s %d\\r\\n\", subject, i))\n\t}\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tsrvB, _ := RunServerWithConfig(\"./configs/srv_b.conf\")\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Now wait for all subscriptions to be processed.\n\tif err := checkExpectedSubs(subsTotal, srvB); err != nil {\n\t\t// Make sure we are not a slow consumer\n\t\t// Check for slow consumer status\n\t\tif srvA.NumSlowConsumers() > 0 {\n\t\t\tt.Fatal(\"Did not receive all subscriptions due to slow consumer\")\n\t\t} else {\n\t\t\tt.Fatalf(\"%v\", err)\n\t\t}\n\t}\n\t// Just double check the slow consumer status.\n\tif srvA.NumSlowConsumers() > 0 {\n\t\tt.Fatalf(\"Received a slow consumer notification: %d\", srvA.NumSlowConsumers())\n\t}\n}\n" }, { "path": "test/gateway_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"regexp\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc testDefaultOptionsForGateway(name string) *server.Options {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.Cluster.Name = name\n\to.Gateway.Name = name\n\to.Gateway.Host = \"127.0.0.1\"\n\to.Gateway.Port = -1\n\treturn &o\n}\n\nfunc runGatewayServer(o *server.Options) *server.Server {\n\ts := RunServer(o)\n\treturn s\n}\n\nfunc createGatewayConn(t testing.TB, host string, port int) net.Conn {\n\tt.Helper()\n\treturn createClientConn(t, host, port)\n}\n\nfunc setupGatewayConn(t testing.TB, c net.Conn, org, dst string) (sendFun, expectFun) {\n\tt.Helper()\n\tdstInfo := checkInfoMsg(t, c)\n\tif dstInfo.Gateway != dst {\n\t\tt.Fatalf(\"Expected to connect to %q, got %q\", dst, dstInfo.Gateway)\n\t}\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"gateway\\\":%q}\\r\\n\",\n\t\tfalse, false, false, org)\n\tsendProto(t, c, cs)\n\tsendProto(t, c, fmt.Sprintf(\"INFO {\\\"gateway\\\":%q}\\r\\n\", org))\n\treturn sendCommand(t, c), expectCommand(t, c)\n}\n\nfunc expectNumberOfProtos(t *testing.T, expFn expectFun, proto *regexp.Regexp, expected int, ignore ...*regexp.Regexp) {\n\tt.Helper()\n\tbuf := []byte(nil)\n\tfor count := 0; count != expected; {\n\t\tbuf = append(buf, expFn(anyRe)...)\n\t\tfor _, skip := range ignore {\n\t\t\tbuf = skip.ReplaceAll(buf, []byte(``))\n\t\t}\n\t\tcount += len(proto.FindAllSubmatch(buf, -1))\n\t\tif count > expected {\n\t\t\tt.Fatalf(\"Expected %v matches, got %v\", expected, count)\n\t\t}\n\t\tbuf = proto.ReplaceAll(buf, []byte(``))\n\t}\n\tif len(buf) != 0 {\n\t\tt.Fatalf(\"did not consume everything, left with: %q\", buf)\n\t}\n}\n\nfunc TestGatewayAccountInterest(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Sending a bunch of messages. On the first, \"B\" will send an A-\n\t// protocol.\n\tfor i := 0; i < 100; i++ {\n\t\tgASend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\t}\n\t// We expect single A- followed by PONG. If \"B\" was sending more\n\t// this expect call would fail.\n\tgAExpect(aunsubRe)\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Start gateway C that connects to B\n\tgC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect := setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\t// Send more messages, C should get A-, but A should not (already\n\t// got it).\n\tfor i := 0; i < 100; i++ {\n\t\tgCSend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\t}\n\tgCExpect(aunsubRe)\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\texpectNothing(t, gA)\n\n\t// Restart one of the gateway, and resend a message, verify\n\t// that it receives A- (things get cleared on reconnect)\n\tgC.Close()\n\tgC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect = setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\tgCSend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\tgCExpect(aunsubRe)\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\texpectNothing(t, gA)\n\n\t// Close again and re-create, but this time don't send anything.\n\tgC.Close()\n\tgC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect = setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\n\t// Now register the $foo account on B and create a subscription,\n\t// A should receive an A+ because B knows that it previously sent\n\t// an A-, but since it did not send one to C, C should not receive\n\t// the A+.\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConnWithAccount(t, sb, client, \"$foo\")\n\tclientSend(\"SUB not.used 1234567\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tgAExpect(asubRe)\n\texpectNothing(t, gC)\n}\n\nfunc TestGatewaySubjectInterest(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tfooAcc := server.NewAccount(\"$foo\")\n\tob.Accounts = []*server.Account{fooAcc}\n\tob.Users = []*server.User{{Username: \"ivan\", Password: \"password\", Account: fooAcc}}\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\t// Create a client on B\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConnWithUserPass(t, client, \"ivan\", \"password\")\n\t// Since we want to test RS+/-, we need to have at\n\t// least a subscription on B so that sending from A does\n\t// not result in A-\n\tclientSend(\"SUB not.used 1234567\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\tfor i := 0; i < 100; i++ {\n\t\tgASend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\t}\n\t// We expect single RS- followed by PONG. If \"B\" was sending more\n\t// this expect call would fail.\n\tgAExpect(runsubRe)\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Start gateway C that connects to B\n\tgC := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect := setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\t// Send more messages, C should get RS-, but A should not (already\n\t// got it).\n\tfor i := 0; i < 100; i++ {\n\t\tgCSend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\t}\n\tgCExpect(runsubRe)\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\texpectNothing(t, gA)\n\n\t// Restart one of the gateway, and resend a message, verify\n\t// that it receives RS- (things get cleared on reconnect)\n\tgC.Close()\n\tgC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect = setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\tgCSend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\tgCExpect(runsubRe)\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\texpectNothing(t, gA)\n\n\t// Close again and re-create, but this time don't send anything.\n\tgC.Close()\n\tgC = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gC.Close()\n\n\tgCSend, gCExpect = setupGatewayConn(t, gC, \"C\", \"B\")\n\tgCSend(\"PING\\r\\n\")\n\tgCExpect(pongRe)\n\n\t// Now register a subscription on foo for account $foo on B.\n\t// A should receive a RS+ because B knows that it previously\n\t// sent a RS-, but since it did not send one to C, C should\n\t// not receive the RS+.\n\tclientSend(\"SUB foo 1\\r\\nSUB foo 2\\r\\n\")\n\t// Also subscribe to subject that was not used before,\n\t// so there should be no RS+ for this one.\n\tclientSend(\"SUB bar 3\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\tgAExpect(rsubRe)\n\texpectNothing(t, gC)\n\t// Check that we get only one protocol\n\texpectNothing(t, gA)\n\n\t// Unsubscribe the 2 subs on foo, expect to receive nothing.\n\tclientSend(\"UNSUB 1\\r\\nUNSUB 2\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\texpectNothing(t, gC)\n\texpectNothing(t, gA)\n\n\tgC.Close()\n\n\t// Send on foo, should get an RS-\n\tgASend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\tgAExpect(runsubRe)\n\t// Subscribe on foo, should get an RS+ that removes the no-interest\n\tclientSend(\"SUB foo 4\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tgAExpect(rsubRe)\n\t// Send on bar, message should be received.\n\tgASend(\"RMSG $foo bar 2\\r\\nok\\r\\n\")\n\tclientExpect(msgRe)\n\t// Unsub foo and bar\n\tclientSend(\"UNSUB 3\\r\\nUNSUB 4\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\texpectNothing(t, gA)\n\t// Send on both foo and bar expect RS-\n\tgASend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\tgAExpect(runsubRe)\n\tgASend(\"RMSG $foo bar 2\\r\\nok\\r\\n\")\n\tgAExpect(runsubRe)\n\t// Now have client create sub on \"*\", this should cause RS+ on *\n\t// The remote will have cleared its no-interest on foo and bar\n\t// and this receiving side is supposed to be doing the same.\n\tclientSend(\"SUB * 5\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tbuf := gAExpect(rsubRe)\n\tif !bytes.Contains(buf, []byte(\"$foo *\")) {\n\t\tt.Fatalf(\"Expected RS+ on %q, got %q\", \"*\", buf)\n\t}\n\t// Check that the remote has cleared by sending from the client\n\t// on foo and bar\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\n\")\n\tclientExpect(msgRe)\n\tclientSend(\"PUB bar 2\\r\\nok\\r\\n\")\n\tclientExpect(msgRe)\n\t// Check that A can send too and does not receive an RS-\n\tgASend(\"RMSG $foo foo 2\\r\\nok\\r\\n\")\n\texpectNothing(t, gA)\n\tclientExpect(msgRe)\n\tgASend(\"RMSG $foo bar 2\\r\\nok\\r\\n\")\n\texpectNothing(t, gA)\n\tclientExpect(msgRe)\n}\n\nfunc TestGatewayQueue(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tfooAcc := server.NewAccount(\"$foo\")\n\tob.Accounts = []*server.Account{fooAcc}\n\tob.Users = []*server.User{{Username: \"ivan\", Password: \"password\", Account: fooAcc}}\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConnWithUserPass(t, client, \"ivan\", \"password\")\n\n\t// Create one queue sub on foo.* for group bar.\n\tclientSend(\"SUB foo.* bar 1\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\t// Expect RS+\n\tgAExpect(rsubRe)\n\t// Add another queue sub on same group\n\tclientSend(\"SUB foo.* bar 2\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\t// Should not receive another RS+ for that one\n\texpectNothing(t, gA)\n\t// However, if subject is different, we can expect to receive another RS+\n\tclientSend(\"SUB foo.> bar 3\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tgAExpect(rsubRe)\n\n\t// Unsub one of the foo.* qsub, no RS- should be received\n\tclientSend(\"UNSUB 1\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\texpectNothing(t, gA)\n\t// Remove the other one, now we should get the RS-\n\tclientSend(\"UNSUB 2\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tgAExpect(runsubRe)\n\t// Remove last one\n\tclientSend(\"UNSUB 3\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tgAExpect(runsubRe)\n\n\t// Create some queues and check that interest is sent\n\t// when GW reconnects.\n\tclientSend(\"SUB foo bar 4\\r\\n\")\n\tgAExpect(rsubRe)\n\tclientSend(\"SUB foo baz 5\\r\\n\")\n\tgAExpect(rsubRe)\n\tclientSend(\"SUB foo bat 6\\r\\n\")\n\tgAExpect(rsubRe)\n\t// There is already one on foo/bar, so nothing sent\n\tclientSend(\"SUB foo bar 7\\r\\n\")\n\texpectNothing(t, gA)\n\t// Add regular sub that should not cause RS+\n\tclientSend(\"SUB foo 8\\r\\n\")\n\texpectNothing(t, gA)\n\n\t// Recreate gA\n\tgA.Close()\n\tgA = createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\tgASend, gAExpect = setupGatewayConn(t, gA, \"A\", \"B\")\n\t// A should receive 3 RS+\n\texpectNumberOfProtos(t, gAExpect, rsubRe, 3)\n\t// Nothing more\n\texpectNothing(t, gA)\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Have A send a message on subject that has no sub\n\tgASend(\"RMSG $foo new.subject 2\\r\\nok\\r\\n\")\n\tgAExpect(runsubRe)\n\t// Now create a queue sub and check that we do not receive\n\t// an RS+ without the queue name.\n\tclientSend(\"SUB new.* queue 9\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\tbuf := gAExpect(rsubRe)\n\tif !bytes.Contains(buf, []byte(\"new.* queue\")) {\n\t\tt.Fatalf(\"Should have receives RS+ for new.* for queue, did not: %v\", buf)\n\t}\n\t// Check for no other RS+. A should still keep an RS- for plain\n\t// sub on new.subject\n\texpectNothing(t, gA)\n\t// Send message, expected to be received by client\n\tgASend(\"RMSG $foo new.subject | queue 2\\r\\nok\\r\\n\")\n\tclientExpect(msgRe)\n\t// Unsubscribe the queue sub\n\tclientSend(\"UNSUB 9\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\t// A should receive RS- for this queue sub\n\tbuf = gAExpect(runsubRe)\n\tif !bytes.Contains(buf, []byte(\"new.* queue\")) {\n\t\tt.Fatalf(\"Should have receives RS- for new.* for queue, did not: %v\", buf)\n\t}\n\texpectNothing(t, gA)\n}\n\nfunc TestGatewaySendAllSubs(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\t// Create a client on B\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConn(t, client)\n\t// Since we want to test RS+/-, we need to have at\n\t// least a subscription on B so that sending from A does\n\t// not result in A-\n\tclientSend(\"SUB not.used 1234567\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Bombard B with messages on different subjects.\n\t// TODO(ik): Adapt if/when we change the conditions for the\n\t// switch.\n\tfor i := 0; i < 1010; i++ {\n\t\tgASend(fmt.Sprintf(\"RMSG $G foo.%d 2\\r\\nok\\r\\n\", i))\n\t\tif i < 1000 {\n\t\t\tgAExpect(runsubRe)\n\t\t}\n\t}\n\t// Expect an INFO + RS+ $G not.used + INFO\n\tbuf := bufio.NewReader(gA)\n\tfor i := 0; i < 3; i++ {\n\t\tline, _, err := buf.ReadLine()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading: %v\", err)\n\t\t}\n\t\tswitch i {\n\t\tcase 0:\n\t\tcase 2:\n\t\t\tif !bytes.HasPrefix(line, []byte(\"INFO {\")) {\n\t\t\t\tt.Fatalf(\"Expected INFO, got: %s\", line)\n\t\t\t}\n\t\tcase 1:\n\t\t\tif !bytes.HasPrefix(line, []byte(\"RS+ \")) {\n\t\t\t\tt.Fatalf(\"Expected RS+, got: %s\", line)\n\t\t\t}\n\t\t}\n\t}\n\t// After this point, any new sub or unsub on B should be\n\t// sent to A.\n\tclientSend(\"SUB foo 1\\r\\n\")\n\tgAExpect(rsubRe)\n\tclientSend(\"UNSUB 1\\r\\n\")\n\tgAExpect(runsubRe)\n}\n\nfunc TestGatewayNoPanicOnBadProtocol(t *testing.T) {\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto string\n\t}{\n\t\t{\"sub\", \"SUB > 1\\r\\n\"},\n\t\t{\"unsub\", \"UNSUB 1\\r\\n\"},\n\t\t{\"rsub\", \"RS+ $foo foo 2\\r\\n\"},\n\t\t{\"runsub\", \"RS- $foo foo 2\\r\\n\"},\n\t\t{\"pub\", \"PUB foo 2\\r\\nok\\r\\n\"},\n\t\t{\"msg\", \"MSG foo 2\\r\\nok\\r\\n\"},\n\t\t{\"rmsg\", \"RMSG $foo foo 2\\r\\nok\\r\\n\"},\n\t\t{\"anything\", \"xxxx\\r\\n\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Create raw tcp connection to gateway port\n\t\t\tclient := createClientConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\t\t\tdefer client.Close()\n\t\t\tclientSend := sendCommand(t, client)\n\t\t\tclientSend(test.proto)\n\t\t})\n\t}\n\n\t// Server should not have crashed.\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConn(t, client)\n\tclientSend(\"PING\\r\\n\")\n\tclientExpect(pongRe)\n}\n\nfunc TestGatewayNoAccUnsubAfterQSub(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\t// Simulate a client connecting to A and publishing a message\n\t// so we get an A- from B since there is no interest.\n\tgASend(\"RMSG $G foo 2\\r\\nok\\r\\n\")\n\tgAExpect(runsubRe)\n\n\t// Now create client on B and create queue sub.\n\tclient := createClientConn(t, ob.Host, ob.Port)\n\tdefer client.Close()\n\tclientSend, clientExpect := setupConn(t, client)\n\n\tclientSend(\"SUB bar queue 1\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\t// A should receive an RS+ for this queue sub.\n\tgAExpect(rsubRe)\n\n\t// On B, create a plain sub now. We should get nothing.\n\tclientSend(\"SUB baz 2\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\texpectNothing(t, gA)\n}\n\nfunc TestGatewayErrorOnRSentFromOutbound(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\tsb := runGatewayServer(ob)\n\tdefer sb.Shutdown()\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tproto string\n\t}{\n\t\t{\"RS+\", \"RS+\"},\n\t\t{\"RS-\", \"RS-\"},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tgA := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\t\t\tdefer gA.Close()\n\n\t\t\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\t\t\tgASend(\"PING\\r\\n\")\n\t\t\tgAExpect(pongRe)\n\n\t\t\tgASend(fmt.Sprintf(\"%s foo bar\\r\\n\", test.proto))\n\t\t\texpectDisconnect(t, gA)\n\t\t})\n\t}\n}\n\nfunc TestGatewaySystemConnectionAllowedToPublishOnGWPrefix(t *testing.T) {\n\tsc := createSuperCluster(t, 2, 2)\n\tdefer sc.shutdown()\n\n\to := sc.clusters[1].opts[1]\n\turl := fmt.Sprintf(\"nats://sys:pass@%s:%d\", o.Host, o.Port)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\treply := nats.NewInbox()\n\tsub, err := nc.SubscribeSync(reply)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif err := nc.PublishRequest(\"$SYS.REQ.SERVER.PING\", reply, nil); err != nil {\n\t\tt.Fatalf(\"Failed to send request: %v\", err)\n\t}\n\tfor i := 0; i < 4; i++ {\n\t\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected to get a response, got %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestGatewayTLSMixedIPAndDNS(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(5 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\t// Run this test extra times to make sure not flaky since it\n\t// on solicit time.\n\tfor i := 0; i < 10; i++ {\n\t\tt.Run(\"\", func(t *testing.T) {\n\t\t\tconfA1 := createConfFile(t, []byte(`\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: A1\n\t\t\tgateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server-iponly.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/server-key-iponly.pem\"\n\t\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\t\ttimeout: 2\n\t\t\t\t}\n\t\t\t\tconnect_retries: 3\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t}`))\n\t\t\tsrvA1, optsA1 := RunServerWithConfig(confA1)\n\t\t\tdefer srvA1.Shutdown()\n\n\t\t\tconfA2Template := `\n\t\t\tlisten: 127.0.0.1:-1\n\t\t\tserver_name: A2\n\t\t\tgateway {\n\t\t\t\tname: \"A\"\n\t\t\t\tlisten: \"localhost:-1\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\t\ttimeout: 2\n\t\t\t\t}\n\t\t\t\tconnect_retries: 3\n\t\t\t}\n\t\t\tcluster {\n\t\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\t\troutes [\n\t\t\t\t\t\"nats://%s:%d\"\n\t\t\t\t]\n\t\t\t}`\n\t\t\tconfA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template,\n\t\t\t\toptsA1.Cluster.Host, optsA1.Cluster.Port)))\n\t\t\tsrvA2, optsA2 := RunServerWithConfig(confA2)\n\t\t\tdefer srvA2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, srvA1, srvA2)\n\n\t\t\t// Create a GW connection to cluster \"A\". Don't use the helper since we need verification etc.\n\t\t\to := DefaultTestOptions\n\t\t\to.Port = -1\n\t\t\to.ServerName = \"B1\"\n\t\t\to.Gateway.Name = \"B\"\n\t\t\to.Gateway.Host = \"127.0.0.1\"\n\t\t\to.Gateway.Port = -1\n\n\t\t\ttc := &server.TLSConfigOpts{}\n\t\t\ttc.CertFile = \"./configs/certs/server-cert.pem\"\n\t\t\ttc.KeyFile = \"./configs/certs/server-key.pem\"\n\t\t\ttc.CaFile = \"./configs/certs/ca.pem\"\n\t\t\ttc.Timeout = 2.0\n\t\t\ttlsConfig, err := server.GenTLSConfig(tc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error generating TLS config: %v\", err)\n\t\t\t}\n\t\t\ttlsConfig.ClientAuth = tls.RequireAndVerifyClientCert\n\t\t\ttlsConfig.RootCAs = tlsConfig.ClientCAs\n\n\t\t\to.Gateway.TLSConfig = tlsConfig.Clone()\n\n\t\t\trurl, _ := url.Parse(fmt.Sprintf(\"nats://%s:%d\", optsA2.Gateway.Host, optsA2.Gateway.Port))\n\t\t\tremote := &server.RemoteGatewayOpts{Name: \"A\", URLs: []*url.URL{rurl}}\n\t\t\tremote.TLSConfig = tlsConfig.Clone()\n\t\t\to.Gateway.Gateways = []*server.RemoteGatewayOpts{remote}\n\n\t\t\tsrvB := RunServer(&o)\n\t\t\tdefer srvB.Shutdown()\n\n\t\t\twaitForOutboundGateways(t, srvB, 1, 10*time.Second)\n\t\t\twaitForOutboundGateways(t, srvA1, 1, 10*time.Second)\n\t\t\twaitForOutboundGateways(t, srvA2, 1, 10*time.Second)\n\n\t\t\t// Now kill off srvA2 and force serverB to connect to srvA1.\n\t\t\tsrvA2.Shutdown()\n\n\t\t\t// Make sure this works.\n\t\t\twaitForOutboundGateways(t, srvB, 1, 30*time.Second)\n\t\t})\n\t}\n}\n\nfunc TestGatewayAdvertiseInCluster(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob1 := testDefaultOptionsForGateway(\"B\")\n\tob1.Cluster.Name = \"B\"\n\tob1.Cluster.Host = \"127.0.0.1\"\n\tob1.Cluster.Port = -1\n\tsb1 := runGatewayServer(ob1)\n\tdefer sb1.Shutdown()\n\n\tgA := createGatewayConn(t, ob1.Gateway.Host, ob1.Gateway.Port)\n\tdefer gA.Close()\n\n\tgASend, gAExpect := setupGatewayConn(t, gA, \"A\", \"B\")\n\tgASend(\"PING\\r\\n\")\n\tgAExpect(pongRe)\n\n\tob2 := testDefaultOptionsForGateway(\"B\")\n\tob2.Cluster.Name = \"B\"\n\tob2.Cluster.Host = \"127.0.0.1\"\n\tob2.Cluster.Port = -1\n\tob2.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tob2.Gateway.Advertise = \"srvB:7222\"\n\tsb2 := runGatewayServer(ob2)\n\tdefer sb2.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2)\n\n\tbuf := gAExpect(infoRe)\n\tsi := &server.Info{}\n\tjson.Unmarshal(buf[5:], si)\n\tvar ok bool\n\tfor _, u := range si.GatewayURLs {\n\t\tif u == \"srvB:7222\" {\n\t\t\tok = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !ok {\n\t\tt.Fatalf(\"Url srvB:7222 was not found: %q\", si.GatewayURLs)\n\t}\n\n\tob3 := testDefaultOptionsForGateway(\"B\")\n\tob3.Cluster.Name = \"B\"\n\tob3.Cluster.Host = \"127.0.0.1\"\n\tob3.Cluster.Port = -1\n\tob3.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", ob1.Cluster.Port))\n\tob3.Gateway.Advertise = \"srvB:7222\"\n\tsb3 := runGatewayServer(ob3)\n\tdefer sb3.Shutdown()\n\n\tcheckClusterFormed(t, sb1, sb2, sb3)\n\n\t// Since it is the save srvB:7222 url, we should not get an update.\n\texpectNothing(t, gA)\n\n\t// Now shutdown sb2 and make sure that we are not getting an update\n\t// with srvB:7222 missing.\n\tsb2.Shutdown()\n\texpectNothing(t, gA)\n}\n\nfunc TestGatewayAuthTimeout(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsetAuth bool //\n\t\twait time.Duration\n\t}{\n\t\t{\"auth not explicitly set\", false, 2500 * time.Millisecond},\n\t\t{\"auth set\", true, 500 * time.Millisecond},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tob := testDefaultOptionsForGateway(\"B\")\n\t\t\tif test.setAuth {\n\t\t\t\tob.Gateway.AuthTimeout = 0.25\n\t\t\t}\n\t\t\tsb := RunServer(ob)\n\t\t\tdefer sb.Shutdown()\n\n\t\t\tsa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\t\t\tdefer sa.Close()\n\n\t\t\tgAExpect := expectCommand(t, sa)\n\n\t\t\tdstInfo := checkInfoMsg(t, sa)\n\t\t\tif dstInfo.Gateway != \"B\" {\n\t\t\t\tt.Fatalf(\"Expected to connect to %q, got %q\", \"B\", dstInfo.Gateway)\n\t\t\t}\n\n\t\t\t// Don't send our CONNECT and we should be disconnected due to auth timeout.\n\t\t\ttime.Sleep(test.wait)\n\t\t\tgAExpect(errRe)\n\t\t\texpectDisconnect(t, sa)\n\t\t})\n\t}\n}\n\nfunc TestGatewayFirstPingGoesAfterConnect(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\tob := testDefaultOptionsForGateway(\"B\")\n\t// For this test, we want the first ping to NOT be disabled.\n\tob.DisableShortFirstPing = false\n\t// Also, for this test increase auth_timeout so that it does not disconnect\n\t// while checking...\n\tob.Gateway.AuthTimeout = 10.0\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\tsa := createGatewayConn(t, ob.Gateway.Host, ob.Gateway.Port)\n\tdefer sa.Close()\n\n\tgASend, gAExpect := sendCommand(t, sa), expectCommand(t, sa)\n\tdstInfo := checkInfoMsg(t, sa)\n\tif dstInfo.Gateway != \"B\" {\n\t\tt.Fatalf(\"Expected to connect to %q, got %q\", \"B\", dstInfo.Gateway)\n\t}\n\n\t// Wait and we should not be receiving a PING from server B until we send\n\t// a CONNECT. We need to wait for more than the initial PING, so cannot\n\t// use expectNothing() helper here.\n\tbuf := make([]byte, 256)\n\tsa.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tif n, err := sa.Read(buf); err == nil {\n\t\tt.Fatalf(\"Expected nothing, got %s\", buf[:n])\n\t}\n\n\t// Now send connect and INFO\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"gateway\\\":%q}\\r\\n\",\n\t\tfalse, false, false, \"A\")\n\tgASend(cs)\n\tgASend(fmt.Sprintf(\"INFO {\\\"gateway\\\":%q}\\r\\n\", \"A\"))\n\n\t// We should get the first PING\n\tgAExpect(pingRe)\n}\n" }, { "path": "test/gosrv_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestSimpleGoServerShutdown(t *testing.T) {\n\tbase := runtime.NumGoroutine()\n\topts := DefaultTestOptions\n\topts.Port = -1\n\ts := RunServer(&opts)\n\ts.Shutdown()\n\tcheckFor(t, time.Second, 100*time.Millisecond, func() error {\n\t\tdelta := (runtime.NumGoroutine() - base)\n\t\tif delta > 1 {\n\t\t\treturn fmt.Errorf(\"%d go routines still exist post Shutdown()\", delta)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestGoServerShutdownWithClients(t *testing.T) {\n\tbase := runtime.NumGoroutine()\n\topts := DefaultTestOptions\n\topts.Port = -1\n\ts := RunServer(&opts)\n\taddr := s.Addr().(*net.TCPAddr)\n\tfor i := 0; i < 50; i++ {\n\t\tcreateClientConn(t, \"127.0.0.1\", addr.Port)\n\t}\n\ts.Shutdown()\n\t// Wait longer for client connections\n\ttime.Sleep(1 * time.Second)\n\tdelta := (runtime.NumGoroutine() - base)\n\t// There may be some finalizers or IO, but in general more than\n\t// 2 as a delta represents a problem.\n\tif delta > 2 {\n\t\tt.Fatalf(\"%d Go routines still exist post Shutdown()\", delta)\n\t}\n}\n\nfunc TestGoServerMultiShutdown(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\ts := RunServer(&opts)\n\ts.Shutdown()\n\ts.Shutdown()\n}\n" }, { "path": "test/leafnode_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/logger\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n\t\"github.com/nats-io/nuid\"\n)\n\nfunc createLeafConn(t tLogger, host string, port int) net.Conn {\n\treturn createClientConn(t, host, port)\n}\n\nfunc testDefaultOptionsForLeafNodes() *server.Options {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.LeafNode.Host = o.Host\n\to.LeafNode.Port = -1\n\to.NoSystemAccount = true\n\treturn &o\n}\n\nfunc runLeafServer() (*server.Server, *server.Options) {\n\to := testDefaultOptionsForLeafNodes()\n\treturn RunServer(o), o\n}\n\nfunc runLeafServerOnPort(port int) (*server.Server, *server.Options) {\n\to := testDefaultOptionsForLeafNodes()\n\to.LeafNode.Port = port\n\treturn RunServer(o), o\n}\n\nfunc runSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) {\n\treturn runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://%s:%d\", lso.LeafNode.Host, lso.LeafNode.Port))\n}\n\nfunc runSolicitLeafServerToURL(surl string) (*server.Server, *server.Options) {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.NoSystemAccount = true\n\trurl, _ := url.Parse(surl)\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\to.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\treturn RunServer(&o), &o\n}\n\nfunc TestLeafNodeInfo(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tinfo := checkInfoMsg(t, lc)\n\tif !info.AuthRequired {\n\t\tt.Fatalf(\"AuthRequired should always be true for leaf nodes\")\n\t}\n\t// By default headers should be true.\n\tif !info.Headers {\n\t\tt.Fatalf(\"Expected to have headers on by default\")\n\t}\n\n\tsendProto(t, lc, \"CONNECT {}\\r\\n\")\n\tcheckLeafNodeConnected(t, s)\n\n\t// Now close connection, make sure we are doing the right accounting in the server.\n\tlc.Close()\n\n\tcheckLeafNodeConnections(t, s, 0)\n}\n\nfunc TestLeafNodeSplitBuffer(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(s.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc.QueueSubscribe(\"foo\", \"bar\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"ok\"))\n\t})\n\tnc.Flush()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\tsendProto(t, lc, \"CONNECT {}\\r\\n\")\n\tcheckLeafNodeConnected(t, s)\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 2)\n\n\tleafSend(\"LS+ reply\\r\\nPING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tleafSend(\"LMSG foo \")\n\ttime.Sleep(time.Millisecond)\n\tleafSend(\"+ reply bar 2\\r\\n\")\n\ttime.Sleep(time.Millisecond)\n\tleafSend(\"OK\\r\")\n\ttime.Sleep(time.Millisecond)\n\tleafSend(\"\\n\")\n\tleafExpect(lmsgRe)\n}\n\nfunc TestNumLeafNodes(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tcreateNewLeafNode := func() net.Conn {\n\t\tt.Helper()\n\t\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\tcheckInfoMsg(t, lc)\n\t\tsendProto(t, lc, \"CONNECT {}\\r\\n\")\n\t\treturn lc\n\t}\n\tcheckLeafNodeConnections(t, s, 0)\n\n\tlc1 := createNewLeafNode()\n\tdefer lc1.Close()\n\tcheckLeafNodeConnections(t, s, 1)\n\n\tlc2 := createNewLeafNode()\n\tdefer lc2.Close()\n\tcheckLeafNodeConnections(t, s, 2)\n\n\t// Now test remove works.\n\tlc1.Close()\n\tcheckLeafNodeConnections(t, s, 1)\n\n\tlc2.Close()\n\tcheckLeafNodeConnections(t, s, 0)\n}\n\nfunc TestLeafNodeRequiresConnect(t *testing.T) {\n\topts := testDefaultOptionsForLeafNodes()\n\topts.LeafNode.AuthTimeout = 0.001\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tinfo := checkInfoMsg(t, lc)\n\tif !info.AuthRequired {\n\t\tt.Fatalf(\"Expected AuthRequired to force CONNECT\")\n\t}\n\tif info.TLSRequired {\n\t\tt.Fatalf(\"Expected TLSRequired to be false\")\n\t}\n\tif info.TLSVerify {\n\t\tt.Fatalf(\"Expected TLSVerify to be false\")\n\t}\n\n\t// Now wait and make sure we get disconnected.\n\terrBuf := expectResult(t, lc, errRe)\n\n\tif !strings.Contains(string(errBuf), \"Authentication Timeout\") {\n\t\tt.Fatalf(\"Authentication Timeout response incorrect: %q\", errBuf)\n\t}\n\texpectDisconnect(t, lc)\n}\n\nfunc setupLeaf(t *testing.T, lc net.Conn, expectedSubs int) (sendFun, expectFun) {\n\tt.Helper()\n\tsend, expect := setupConn(t, lc)\n\t// A loop detection subscription is sent, so consume this here, along\n\t// with the ones that caller expect on setup.\n\texpectNumberOfProtos(t, expect, lsubRe, expectedSubs, infoStartRe, pingRe)\n\treturn send, expect\n}\n\nfunc TestLeafNodeSendsSubsAfterConnect(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"SUB foo 1\\r\\n\")\n\tsend(\"SUB bar 2\\r\\n\")\n\tsend(\"SUB foo baz 3\\r\\n\")\n\tsend(\"SUB foo baz 4\\r\\n\")\n\tsend(\"SUB bar 5\\r\\n\")\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This should compress down to 1 for foo, 1 for bar, and 1 for foo [baz]\n\t// and one for the loop detection subject.\n\tsetupLeaf(t, lc, 4)\n}\n\nfunc TestLeafNodeSendsSubsOngoing(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"SUB foo 1\\r\\n\")\n\tleafExpect(lsubRe)\n\n\t// Check queues send updates each time.\n\t// TODO(dlc) - If we decide to suppress this with a timer approach this test will break.\n\tsend(\"SUB foo bar 2\\r\\n\")\n\tleafExpect(lsubRe)\n\tsend(\"SUB foo bar 3\\r\\n\")\n\tleafExpect(lsubRe)\n\tsend(\"SUB foo bar 4\\r\\n\")\n\tleafExpect(lsubRe)\n\n\t// Now check more normal subs do nothing.\n\tsend(\"SUB foo 5\\r\\n\")\n\texpectNothing(t, lc)\n\n\t// Check going back down does nothing til we hit 0.\n\tsend(\"UNSUB 5\\r\\n\")\n\texpectNothing(t, lc)\n\tsend(\"UNSUB 1\\r\\n\")\n\tleafExpect(lunsubRe)\n\n\t// Queues going down should always send updates.\n\tsend(\"UNSUB 2\\r\\n\")\n\tleafExpect(lsubRe)\n\tsend(\"UNSUB 3\\r\\n\")\n\tleafExpect(lsubRe)\n\tsend(\"UNSUB 4\\r\\n\")\n\tleafExpect(lunsubRe)\n}\n\nfunc TestLeafNodeSubs(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tleafSend(\"LS+ foo\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tsend(\"PUB foo 2\\r\\nOK\\r\\n\")\n\tmatches := lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"foo\", \"\", \"2\", \"OK\")\n\n\t// Second sub should not change delivery\n\tleafSend(\"LS+ foo\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB foo 3\\r\\nOK!\\r\\n\")\n\tmatches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"foo\", \"\", \"3\", \"OK!\")\n\n\t// Now add in a queue sub with weight 4.\n\tleafSend(\"LS+ foo bar 4\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB foo 4\\r\\nOKOK\\r\\n\")\n\tmatches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"foo\", \"| bar\", \"4\", \"OKOK\")\n\n\t// Now add in a queue sub with weight 4.\n\tleafSend(\"LS+ foo baz 2\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB foo 5\\r\\nHELLO\\r\\n\")\n\tmatches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"foo\", \"| bar baz\", \"5\", \"HELLO\")\n\n\t// Test Unsub\n\tleafSend(\"LS- foo\\r\\n\")\n\tleafSend(\"LS- foo bar\\r\\n\")\n\tleafSend(\"LS- foo baz\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB foo 5\\r\\nHELLO\\r\\n\")\n\texpectNothing(t, lc)\n}\n\nfunc TestLeafNodeMsgDelivery(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\t// Now send from leaf side.\n\tleafSend(\"LMSG foo 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\n\tmatches := msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"OK\")\n\n\tsend(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lunsubRe)\n\tsend(\"SUB foo bar 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\t// Now send again from leaf side. This is targeted so this should\n\t// not be delivered.\n\tleafSend(\"LMSG foo 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\texpectNothing(t, c)\n\n\t// Now send targeted, and we should receive it.\n\tleafSend(\"LMSG foo | bar 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\n\tmatches = msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckMsg(t, matches[0], \"foo\", \"2\", \"\", \"2\", \"OK\")\n\n\t// Check reply + queues\n\tleafSend(\"LMSG foo + myreply bar 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\n\tmatches = msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckMsg(t, matches[0], \"foo\", \"2\", \"myreply\", \"2\", \"OK\")\n}\n\nfunc TestLeafNodeAndRoutes(t *testing.T) {\n\toptsA := LoadConfig(\"./configs/srv_a_leaf.conf\")\n\toptsA.DisableShortFirstPing = true\n\toptsB := LoadConfig(\"./configs/srv_b.conf\")\n\toptsB.DisableShortFirstPing = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tlc := createLeafConn(t, optsA.LeafNode.Host, optsA.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 6)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tc := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\tsend(\"SUB foo 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\texpectNothing(t, lc)\n\n\tsend(\"UNSUB 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\texpectNothing(t, lc)\n\tsend(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lunsubRe)\n\n\t// Now put it back and test msg flow.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\tleafSend(\"LMSG foo + myreply bar 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\n\tmatches := msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"myreply\", \"2\", \"OK\")\n\n\t// Now check reverse.\n\tleafSend(\"LS+ bar\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB bar 2\\r\\nOK\\r\\n\")\n\tmatches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"bar\", \"\", \"2\", \"OK\")\n}\n\n// Helper function to check that a leaf node has connected to our server.\nfunc checkLeafNodeConnected(t *testing.T, s *server.Server) {\n\tt.Helper()\n\tcheckLeafNodeConnections(t, s, 1)\n}\n\nfunc checkLeafNodeConnections(t *testing.T, s *server.Server, expected int) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif nln := s.NumLeafNodes(); nln != expected {\n\t\t\treturn fmt.Errorf(\"Expected a connected leafnode for server %q, got %d\", s.ID(), nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeSolicit(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tsl, _ := runSolicitLeafServer(opts)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Now test reconnect.\n\ts.Shutdown()\n\t// Need to restart it on the same port.\n\ts, _ = runLeafServerOnPort(opts.LeafNode.Port)\n\tdefer s.Shutdown()\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc TestLeafNodeNoEcho(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// We should not echo back to ourselves. Set up 'foo' subscriptions\n\t// on both sides and send message across the leafnode connection. It\n\t// should not come back.\n\n\tsend(\"SUB foo 1\\r\\n\")\n\tleafExpect(lsubRe)\n\n\tleafSend(\"LS+ foo\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tleafSend(\"LMSG foo 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n}\n\nfunc TestLeafNodeHeaderSupport(t *testing.T) {\n\tsrvA, optsA := runLeafServer()\n\tdefer srvA.Shutdown()\n\n\tsrvB, optsB := runSolicitLeafServer(optsA)\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupHeaderConn(t, clientA)\n\tsendA(\"SUB foo bar 22\\r\\n\")\n\tsendA(\"SUB bar 11\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(3, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB, expectB := setupHeaderConn(t, clientB)\n\t// Can not have \\r\\n in payload fyi for regex.\n\t// With reply\n\tsendB(\"HPUB foo reply 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectHeaderMsgs := expectHeaderMsgsCommand(t, expectA)\n\tmatches := expectHeaderMsgs(1)\n\tcheckHmsg(t, matches[0], \"foo\", \"22\", \"reply\", \"12\", \"14\", \"K1:V1,K2:V2 \", \"ok\")\n\n\t// Without reply\n\tsendB(\"HPUB foo 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tmatches = expectHeaderMsgs(1)\n\tcheckHmsg(t, matches[0], \"foo\", \"22\", \"\", \"12\", \"14\", \"K1:V1,K2:V2 \", \"ok\")\n\n\t// Without queues or reply\n\tsendB(\"HPUB bar 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tmatches = expectHeaderMsgs(1)\n\tcheckHmsg(t, matches[0], \"bar\", \"11\", \"\", \"12\", \"14\", \"K1:V1,K2:V2 \", \"ok\")\n\n\t// Without queues but with reply\n\tsendB(\"HPUB bar reply 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tmatches = expectHeaderMsgs(1)\n\tcheckHmsg(t, matches[0], \"bar\", \"11\", \"reply\", \"12\", \"14\", \"K1:V1,K2:V2 \", \"ok\")\n}\n\n// Used to setup clusters of clusters for tests.\ntype cluster struct {\n\tservers []*server.Server\n\topts []*server.Options\n\tname string\n\tt *testing.T\n}\n\nfunc testDefaultClusterOptionsForLeafNodes() *server.Options {\n\to := DefaultTestOptions\n\to.Port = -1\n\to.Cluster.Host = o.Host\n\to.Cluster.Port = -1\n\to.Gateway.Host = o.Host\n\to.Gateway.Port = -1\n\to.LeafNode.Host = o.Host\n\to.LeafNode.Port = -1\n\treturn &o\n}\n\nfunc (c *cluster) shutdown() {\n\tif c == nil {\n\t\treturn\n\t}\n\tfor i, s := range c.servers {\n\t\tif cf := c.opts[i].ConfigFile; cf != \"\" {\n\t\t\tos.RemoveAll(cf)\n\t\t}\n\t\tif sd := s.StoreDir(); sd != \"\" {\n\t\t\tos.RemoveAll(sd)\n\t\t}\n\t\ts.Shutdown()\n\t}\n}\n\nfunc shutdownCluster(c *cluster) {\n\tc.shutdown()\n}\n\nfunc (c *cluster) totalSubs() int {\n\ttotalSubs := 0\n\tfor _, s := range c.servers {\n\t\ttotalSubs += int(s.NumSubscriptions())\n\t}\n\treturn totalSubs\n}\n\n// Wait for the expected number of outbound gateways, or fails.\nfunc waitForOutboundGateways(t *testing.T, s *server.Server, expected int, timeout time.Duration) {\n\tt.Helper()\n\tif timeout < 2*time.Second {\n\t\ttimeout = 2 * time.Second\n\t}\n\tcheckFor(t, timeout, 15*time.Millisecond, func() error {\n\t\tif n := s.NumOutboundGateways(); n != expected {\n\t\t\treturn fmt.Errorf(\"Expected %v outbound gateway(s), got %v (ulimit -n too low?)\",\n\t\t\t\texpected, n)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Creates a full cluster with numServers and given name and makes sure its well formed.\n// Will have Gateways and Leaf Node connections active.\nfunc createClusterWithName(t *testing.T, clusterName string, numServers int, connectTo ...*cluster) *cluster {\n\tt.Helper()\n\treturn createClusterEx(t, false, 5*time.Millisecond, true, clusterName, numServers, connectTo...)\n}\n\n// Creates a cluster and optionally additional accounts and users.\n// Will have Gateways and Leaf Node connections active.\nfunc createClusterEx(t *testing.T, doAccounts bool, gwSolicit time.Duration, waitOnGWs bool, clusterName string, numServers int, connectTo ...*cluster) *cluster {\n\tt.Helper()\n\n\tif clusterName == \"\" || numServers < 1 {\n\t\tt.Fatalf(\"Bad params\")\n\t}\n\n\t// Setup some accounts and users.\n\t// $SYS is always the system account. And we have default FOO and BAR accounts, as well\n\t// as DLC and NGS which do a service import.\n\tcreateAccountsAndUsers := func() ([]*server.Account, []*server.User) {\n\t\tif !doAccounts {\n\t\t\treturn []*server.Account{server.NewAccount(\"$SYS\")}, nil\n\t\t}\n\n\t\tsys := server.NewAccount(\"$SYS\")\n\t\tngs := server.NewAccount(\"NGS\")\n\t\tdlc := server.NewAccount(\"DLC\")\n\t\tfoo := server.NewAccount(\"FOO\")\n\t\tbar := server.NewAccount(\"BAR\")\n\n\t\taccounts := []*server.Account{sys, foo, bar, ngs, dlc}\n\n\t\tngs.AddServiceExport(\"ngs.usage.*\", nil)\n\t\tdlc.AddServiceImport(ngs, \"ngs.usage\", \"ngs.usage.dlc\")\n\n\t\t// Setup users\n\t\tusers := []*server.User{\n\t\t\t{Username: \"dlc\", Password: \"pass\", Permissions: nil, Account: dlc},\n\t\t\t{Username: \"ngs\", Password: \"pass\", Permissions: nil, Account: ngs},\n\t\t\t{Username: \"foo\", Password: \"pass\", Permissions: nil, Account: foo},\n\t\t\t{Username: \"bar\", Password: \"pass\", Permissions: nil, Account: bar},\n\t\t\t{Username: \"sys\", Password: \"pass\", Permissions: nil, Account: sys},\n\t\t}\n\t\treturn accounts, users\n\t}\n\n\tbindGlobal := func(s *server.Server) {\n\t\tngs, err := s.LookupAccount(\"NGS\")\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\t// Bind global to service import\n\t\tgacc, _ := s.LookupAccount(\"$G\")\n\t\tgacc.AddServiceImport(ngs, \"ngs.usage\", \"ngs.usage.$G\")\n\t}\n\n\t// If we are going to connect to another cluster set that up now for options.\n\tvar gws []*server.RemoteGatewayOpts\n\tfor _, c := range connectTo {\n\t\t// Gateways autodiscover here too, so just need one address from the set.\n\t\tgwAddr := fmt.Sprintf(\"nats-gw://%s:%d\", c.opts[0].Gateway.Host, c.opts[0].Gateway.Port)\n\t\tgwurl, _ := url.Parse(gwAddr)\n\t\tgws = append(gws, &server.RemoteGatewayOpts{Name: c.name, URLs: []*url.URL{gwurl}})\n\t}\n\n\t// Make the GWs form faster for the tests.\n\tserver.SetGatewaysSolicitDelay(gwSolicit)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\t// Create seed first.\n\to := testDefaultClusterOptionsForLeafNodes()\n\to.Gateway.Name = clusterName\n\to.Gateway.Gateways = gws\n\t// All of these need system accounts.\n\to.Accounts, o.Users = createAccountsAndUsers()\n\to.SystemAccount = \"$SYS\"\n\to.ServerName = fmt.Sprintf(\"%s1\", clusterName)\n\t// Run the server\n\ts := RunServer(o)\n\tbindGlobal(s)\n\n\tc := &cluster{servers: make([]*server.Server, 0, numServers), opts: make([]*server.Options, 0, numServers), name: clusterName}\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, o)\n\n\t// For connecting to seed server above.\n\trouteAddr := fmt.Sprintf(\"nats-route://%s:%d\", o.Cluster.Host, o.Cluster.Port)\n\trurl, _ := url.Parse(routeAddr)\n\troutes := []*url.URL{rurl}\n\n\tfor i := 1; i < numServers; i++ {\n\t\to := testDefaultClusterOptionsForLeafNodes()\n\t\to.Gateway.Name = clusterName\n\t\to.Gateway.Gateways = gws\n\t\to.Routes = routes\n\t\t// All of these need system accounts.\n\t\to.Accounts, o.Users = createAccountsAndUsers()\n\t\to.SystemAccount = \"$SYS\"\n\t\to.ServerName = fmt.Sprintf(\"%s%d\", clusterName, i+1)\n\t\ts := RunServer(o)\n\t\tbindGlobal(s)\n\n\t\tc.servers = append(c.servers, s)\n\t\tc.opts = append(c.opts, o)\n\t}\n\tcheckClusterFormed(t, c.servers...)\n\n\tif waitOnGWs {\n\t\t// Wait on gateway connections if we were asked to connect to other gateways.\n\t\tif numGWs := len(connectTo); numGWs > 0 {\n\t\t\tfor _, s := range c.servers {\n\t\t\t\twaitForOutboundGateways(t, s, numGWs, 2*time.Second)\n\t\t\t}\n\t\t}\n\t}\n\tc.t = t\n\treturn c\n}\n\nfunc TestLeafNodeGatewayRequiresSystemAccount(t *testing.T) {\n\to := testDefaultClusterOptionsForLeafNodes()\n\to.Gateway.Name = \"CLUSTER-A\"\n\t_, err := server.NewServer(o)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected an error with no system account defined\")\n\t}\n}\n\nfunc TestLeafNodeGatewaySendsSystemEvent(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(50 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 1)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 1, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create client on a server in cluster A\n\topts := ca.opts[0]\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\t// Listen for the leaf node event.\n\tsend, expect := setupConnWithAccount(t, ca.servers[0], c, \"$SYS\")\n\tsend(\"SUB $SYS.ACCOUNT.$G.LEAFNODE.CONNECT 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\topts = cb.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This is for our global responses since we are setting up GWs above.\n\tleafSend, leafExpect := setupLeaf(t, lc, 8)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tmatches := rawMsgRe.FindAllSubmatch(expect(rawMsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tm := matches[0]\n\tif string(m[subIndex]) != \"$SYS.ACCOUNT.$G.LEAFNODE.CONNECT\" {\n\t\tt.Fatalf(\"Got wrong subject for leaf node event, got %q, wanted %q\",\n\t\t\tm[subIndex], \"$SYS.ACCOUNT.$G.LEAFNODE.CONNECT\")\n\t}\n}\n\nfunc TestLeafNodeGatewayInterestPropagation(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\tsl1, sl1Opts := runSolicitLeafServer(ca.opts[1])\n\tdefer sl1.Shutdown()\n\n\tc := createClientConn(t, sl1Opts.Host, sl1Opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"SUB foo 1\\r\\n\")\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Now we will create a new leaf node on cluster B, expect to get the\n\t// interest for \"foo\".\n\topts := cb.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\t_, leafExpect := setupConn(t, lc)\n\tbuf := leafExpect(infoStartRe)\n\tbuf = infoStartRe.ReplaceAll(buf, []byte(nil))\n\n\tfoundFoo := false\n\tfor count := 0; count < 10; {\n\t\t// skip first time if we still have data (buf from above may already have some left)\n\t\tif count != 0 || len(buf) == 0 {\n\t\t\tbuf = append(buf, leafExpect(anyRe)...)\n\t\t}\n\t\tcount += len(lsubRe.FindAllSubmatch(buf, -1))\n\t\tif count > 10 {\n\t\t\tt.Fatalf(\"Expected %v matches, got %v (buf=%s)\", 10, count, buf)\n\t\t}\n\t\tif strings.Contains(string(buf), \"foo\") {\n\t\t\tfoundFoo = true\n\t\t}\n\t\tbuf = lsubRe.ReplaceAll(buf, []byte(nil))\n\t}\n\tif len(buf) != 0 {\n\t\tt.Fatalf(\"did not consume everything, left with: %q\", buf)\n\t}\n\tif !foundFoo {\n\t\tt.Fatalf(\"Expected interest for 'foo' as 'LS+ foo\\\\r\\\\n', got %q\", buf)\n\t}\n}\n\nfunc TestLeafNodeAuthSystemEventNoCrash(t *testing.T) {\n\tca := createClusterWithName(t, \"A\", 1)\n\tdefer shutdownCluster(ca)\n\n\topts := ca.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend := sendCommand(t, lc)\n\tleafSend(\"LS+ foo\\r\\n\")\n\tcheckInfoMsg(t, lc)\n}\n\nfunc TestLeafNodeWithRouteAndGateway(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(50 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create client on a server in cluster A\n\topts := ca.opts[0]\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Create a leaf node connection on a server in cluster B\n\topts = cb.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This is for our global responses since we are setting up GWs above.\n\tleafSend, leafExpect := setupLeaf(t, lc, 8)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// Make sure we see interest graph propagation on the leaf node\n\t// connection. This is required since leaf nodes only send data\n\t// in the presence of interest.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\tsend(\"SUB foo 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\texpectNothing(t, lc)\n\n\tsend(\"UNSUB 2\\r\\n\")\n\texpectNothing(t, lc)\n\tsend(\"UNSUB 1\\r\\n\")\n\tleafExpect(lunsubRe)\n\n\t// Now put it back and test msg flow.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\tleafSend(\"LMSG foo 2\\r\\nOK\\r\\n\")\n\texpectNothing(t, lc)\n\n\tmatches := msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"OK\")\n\n\t// Now check reverse.\n\tleafSend(\"LS+ bar\\r\\n\")\n\texpectNothing(t, lc)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\tsend(\"PUB bar 2\\r\\nOK\\r\\n\")\n\tmatches = lmsgRe.FindAllSubmatch(leafExpect(lmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckLmsg(t, matches[0], \"bar\", \"\", \"2\", \"OK\")\n}\n\n// This will test that we propagate interest only mode after a leafnode\n// has been established and a new server joins a remote cluster.\nfunc TestLeafNodeWithGatewaysAndStaggeredStart(t *testing.T) {\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\n\t// Create the leafnode on a server in cluster A.\n\topts := ca.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 8)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// Now setup the cluster B.\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create client on a server in cluster B\n\topts = cb.opts[0]\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Make sure we see interest graph propagation on the leaf node\n\t// connection. This is required since leaf nodes only send data\n\t// in the presence of interest.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n}\n\n// This will test that we propagate interest only mode after a leafnode\n// has been established and a server is restarted..\nfunc TestLeafNodeWithGatewaysServerRestart(t *testing.T) {\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\n\t// Now setup the cluster B.\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create the leafnode on a server in cluster B.\n\topts := cb.opts[1]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 8)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// Create client on a server in cluster A\n\topts = ca.opts[1]\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Make sure we see interest graph propagation on the leaf node\n\t// connection. This is required since leaf nodes only send data\n\t// in the presence of interest.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tleafExpect(lsubRe)\n\n\t// Close old leaf connection and simulate a reconnect.\n\tlc.Close()\n\n\t// Shutdown and recreate B and the leafnode connection to it.\n\tshutdownCluster(cb)\n\n\t// Create new cluster with longer solicit and don't wait for GW connect.\n\tcb = createClusterEx(t, false, 500*time.Millisecond, false, \"B\", 1, ca)\n\tdefer shutdownCluster(cb)\n\n\topts = cb.opts[0]\n\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t_, leafExpect = setupLeaf(t, lc, 8)\n\n\t// Now wait on GW solicit to fire\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// We should see the interest for 'foo' here.\n\tleafExpect(lsubRe)\n}\n\nfunc TestLeafNodeLocalizedDQ(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tsl, slOpts := runSolicitLeafServer(opts)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\tc := createClientConn(t, slOpts.Host, slOpts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"SUB foo bar 1\\r\\n\")\n\tsend(\"SUB foo bar 2\\r\\n\")\n\tsend(\"SUB foo bar 3\\r\\n\")\n\tsend(\"SUB foo bar 4\\r\\n\")\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Now create another client on the main leaf server.\n\tsc := createClientConn(t, opts.Host, opts.Port)\n\tdefer sc.Close()\n\n\tsendL, expectL := setupConn(t, sc)\n\tsendL(\"SUB foo bar 11\\r\\n\")\n\tsendL(\"SUB foo bar 12\\r\\n\")\n\tsendL(\"SUB foo bar 13\\r\\n\")\n\tsendL(\"SUB foo bar 14\\r\\n\")\n\tsendL(\"PING\\r\\n\")\n\texpectL(pongRe)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsend(\"PUB foo 2\\r\\nOK\\r\\n\")\n\t}\n\texpectNothing(t, sc)\n\n\tmatches := msgRe.FindAllSubmatch(expect(msgRe), -1)\n\tif len(matches) != 10 {\n\t\tt.Fatalf(\"Expected 10 msgs, got %d\", len(matches))\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tcheckMsg(t, matches[i], \"foo\", \"\", \"\", \"2\", \"OK\")\n\t}\n}\n\nfunc TestLeafNodeBasicAuth(t *testing.T) {\n\tcontent := `\n listen: \"127.0.0.1:-1\"\n\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tauthorization {\n\t\t\tuser: \"derek\"\n\t\t\tpassword: \"s3cr3t!\"\n\t\t\ttimeout: 2.2\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This should fail since we want u/p\n\tsetupConn(t, lc)\n\terrBuf := expectResult(t, lc, errRe)\n\tif !strings.Contains(string(errBuf), \"Authorization Violation\") {\n\t\tt.Fatalf(\"Authentication Timeout response incorrect: %q\", errBuf)\n\t}\n\texpectDisconnect(t, lc)\n\n\t// Try bad password as well.\n\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This should fail since we want u/p\n\tsetupConnWithUserPass(t, lc, \"derek\", \"badpassword\")\n\terrBuf = expectResult(t, lc, errRe)\n\tif !strings.Contains(string(errBuf), \"Authorization Violation\") {\n\t\tt.Fatalf(\"Authentication Timeout response incorrect: %q\", errBuf)\n\t}\n\texpectDisconnect(t, lc)\n\n\t// This one should work.\n\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\tleafSend, leafExpect := setupConnWithUserPass(t, lc, \"derek\", \"s3cr3t!\")\n\tleafExpect(infoRe)\n\tleafExpect(lsubRe)\n\tleafSend(\"PING\\r\\n\")\n\texpectResult(t, lc, pongRe)\n\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc runTLSSolicitLeafServer(lso *server.Options) (*server.Server, *server.Options) {\n\to := DefaultTestOptions\n\to.Port = -1\n\trurl, _ := url.Parse(fmt.Sprintf(\"nats-leaf://%s:%d\", lso.LeafNode.Host, lso.LeafNode.Port))\n\tremote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}}\n\tremote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12}\n\thost, _, _ := net.SplitHostPort(lso.LeafNode.Host)\n\tremote.TLSConfig.ServerName = host\n\tremote.TLSConfig.InsecureSkipVerify = true\n\tremote.Compression.Mode = server.CompressionOff\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{remote}\n\to.LeafNode.Compression.Mode = server.CompressionOff\n\treturn RunServer(&o), &o\n}\n\nfunc TestLeafNodeTLS(t *testing.T) {\n\tcontent := `\n\tlisten: \"127.0.0.1:-1\"\n\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\ttimeout: 0.1\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tinfo := checkInfoMsg(t, lc)\n\tif !info.TLSRequired {\n\t\tt.Fatalf(\"Expected TLSRequired to be true\")\n\t}\n\tif info.TLSVerify {\n\t\tt.Fatalf(\"Expected TLSVerify to be false\")\n\t}\n\t// We should get a disconnect here since we have not upgraded to TLS.\n\texpectDisconnect(t, lc)\n\n\t// This should work ok.\n\tsl, _ := runTLSSolicitLeafServer(opts)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc TestLeafNodeTLSConnCloseEarly(t *testing.T) {\n\tcontent := `\n\tlisten: \"127.0.0.1:-1\"\n\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\ttimeout: 2.0\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tlc, err := net.Dial(\"tcp\", fmt.Sprintf(\"127.0.0.1:%d\", opts.LeafNode.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to connect: %v\", err)\n\t}\n\t// Then close right away\n\tlc.Close()\n\n\t// Check server does not crash...\n\ttime.Sleep(250 * time.Millisecond)\n\tif s.ID() == \"\" {\n\t\tt.Fatalf(\"should not happen\")\n\t}\n}\n\ntype captureLeafNodeErrLogger struct {\n\tdummyLogger\n\tch chan string\n}\n\nfunc (c *captureLeafNodeErrLogger) Errorf(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tselect {\n\tcase c.ch <- msg:\n\tdefault:\n\t}\n}\n\nfunc TestLeafNodeTLSMixIP(t *testing.T) {\n\tcontent := `\n\tlisten: \"127.0.0.1:-1\"\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n authorization {\n\t\t\tuser: dlc\n\t\t\tpass: monkey\n\t\t}\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server-noip.pem\"\n\t\t\tkey_file: \"./configs/certs/server-key-noip.pem\"\n\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t}\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tslContent := `\n\tlisten: \"127.0.0.1:-1\"\n\tleafnodes {\n\t\treconnect: 1\n\t\tremotes: [\n\t\t\t{\n\t\t\t\turl: [tls://127.0.0.1:%d, \"tls://localhost:%d\"]\n\t\t\t\ttls { ca_file: \"./configs/certs/ca.pem\" }\n\t\t\t}\n\t\t]\n\t}\n\t`\n\tslconf := createConfFile(t, []byte(fmt.Sprintf(slContent, opts.LeafNode.Port, opts.LeafNode.Port)))\n\n\t// This will fail but we want to make sure in the correct way, not with\n\t// TLS issue because we used an IP for serverName.\n\tsl, _ := RunServerWithConfig(slconf)\n\tdefer sl.Shutdown()\n\n\tll := &captureLeafNodeErrLogger{ch: make(chan string, 2)}\n\tsl.SetLogger(ll, false, false)\n\n\t// We may or may not get an error depending on timing. For the handshake bug\n\t// we would always get it, so make sure if we have anything it is not that.\n\tselect {\n\tcase msg := <-ll.ch:\n\t\tif strings.Contains(msg, \"TLS handshake error\") && strings.Contains(msg, \"doesn't contain any IP SANs\") {\n\t\t\tt.Fatalf(\"Got bad error about TLS handshake\")\n\t\t}\n\tdefault:\n\t}\n}\n\nfunc runLeafNodeOperatorServer(t *testing.T) (*server.Server, *server.Options, string) {\n\tt.Helper()\n\tcontent := `\n\tport: -1\n\tserver_name: OP\n\toperator = \"./configs/nkeys/op.jwt\"\n\tresolver = MEMORY\n\tlisten: \"127.0.0.1:-1\"\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\treturn s, opts, conf\n}\n\nfunc genCredsFile(t *testing.T, jwt string, seed []byte) string {\n\tcreds := `\n\t\t-----BEGIN NATS USER JWT-----\n\t\t%s\n\t\t------END NATS USER JWT------\n\n\t\t************************* IMPORTANT *************************\n\t\tNKEY Seed printed below can be used to sign and prove identity.\n\t\tNKEYs are sensitive and should be treated as secrets.\n\n\t\t-----BEGIN USER NKEY SEED-----\n\t\t%s\n\t\t------END USER NKEY SEED------\n\n\t\t*************************************************************\n\t\t`\n\treturn createConfFile(t, []byte(strings.Replace(fmt.Sprintf(creds, jwt, seed), \"\\t\\t\", \"\", -1)))\n}\n\nfunc runSolicitWithCredentials(t *testing.T, opts *server.Options, creds string) (*server.Server, *server.Options, string) {\n\tcontent := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content, opts.LeafNode.Port, creds)\n\tconf := createConfFile(t, []byte(config))\n\ts, opts := RunServerWithConfig(conf)\n\treturn s, opts, conf\n}\n\nfunc TestLeafNodeOperatorModel(t *testing.T) {\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Make sure we get disconnected without proper credentials etc.\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This should fail since we want user jwt, signed nonce etc.\n\tsetupConn(t, lc)\n\terrBuf := expectResult(t, lc, errRe)\n\tif !strings.Contains(string(errBuf), \"Authorization Violation\") {\n\t\tt.Fatalf(\"Authentication Timeout response incorrect: %q\", errBuf)\n\t}\n\texpectDisconnect(t, lc)\n\n\t// Setup account and a user that will be used by the remote leaf node server.\n\t// createAccount automatically registers with resolver etc..\n\t_, akp := createAccount(t, s)\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\tsl, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc TestLeafNodeUserPermsForConnection(t *testing.T) {\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Setup account and a user that will be used by the remote leaf node server.\n\t// createAccount automatically registers with resolver etc..\n\tacc, akp := createAccount(t, s)\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tnuc.Permissions.Pub.Allow.Add(\"foo.>\")\n\tnuc.Permissions.Pub.Allow.Add(\"baz.>\")\n\tnuc.Permissions.Sub.Allow.Add(\"foo.>\")\n\t// we would be immediately disconnected if that would not work\n\tnuc.Permissions.Sub.Deny.Add(\"$SYS.>\")\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\tcontent := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t\tdeny_import: \"foo.33\"\n\t\t\t\t\tdeny_export: \"foo.33\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content, opts.LeafNode.Port, mycreds)\n\tlnconf := createConfFile(t, []byte(config))\n\tsl, _ := RunServerWithConfig(lnconf)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Create credentials for a normal unrestricted user that we will connect to the op server.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Create a user on the leafnode server that solicited.\n\tnc2, err := nats.Connect(sl.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Make sure subscriptions properly do or do not make it to the hub.\n\tnc2.SubscribeSync(\"bar\")\n\tcheckNoSubInterest(t, s, acc.GetName(), \"bar\", 20*time.Millisecond)\n\t// This one should.\n\tnc2.SubscribeSync(\"foo.22\")\n\tcheckSubInterest(t, s, acc.GetName(), \"foo.22\", 100*time.Millisecond)\n\n\t// Capture everything.\n\tsub, _ := nc.SubscribeSync(\">\")\n\tnc.Flush()\n\n\tcheckSubInterest(t, sl, \"$G\", \">\", 100*time.Millisecond)\n\t// Now check local pubs are not forwarded.\n\tnc2.Publish(\"baz.22\", nil)\n\tm, err := sub.NextMsg(1 * time.Second)\n\tif err != nil || m.Subject != \"baz.22\" {\n\t\tt.Fatalf(\"Expected to received this message\")\n\t}\n\tnc2.Publish(\"bar.22\", nil)\n\tif _, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Did not expect to receive this message\")\n\t}\n\n\t// Check local overrides work.\n\tnc2.Publish(\"foo.33\", nil)\n\tif _, err := sub.NextMsg(100 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Did not expect to receive this message\")\n\t}\n\n\t// This would trigger the sub interest below.\n\tsub.Unsubscribe()\n\tnc.Flush()\n\n\tnc2.SubscribeSync(\"foo.33\")\n\tcheckNoSubInterest(t, s, acc.GetName(), \"foo.33\", 20*time.Millisecond)\n}\n\nfunc TestLeafNodeMultipleAccounts(t *testing.T) {\n\t// So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have\n\t// the $G global account and no auth. Make sure things work properly here.\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Setup the two accounts for this server.\n\ta, akp1 := createAccount(t, s)\n\tkp1, _ := nkeys.CreateUser()\n\tpub1, _ := kp1.PublicKey()\n\tnuc1 := jwt.NewUserClaims(pub1)\n\tujwt1, err := nuc1.Encode(akp1)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\t// Create second account.\n\tcreateAccount(t, s)\n\n\t// Create the leaf node server using the first account.\n\tseed, _ := kp1.Seed()\n\tmycreds := genCredsFile(t, ujwt1, seed)\n\n\tsl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// To connect to main server.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\tnc1, err := nats.Connect(url, createUserCreds(t, s, akp1))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// This is a client connected to the leaf node with no auth,\n\t// binding to account1 via leafnode connection.\n\t// To connect to leafnode server.\n\tlurl := fmt.Sprintf(\"nats://%s:%d\", lopts.Host, lopts.Port)\n\tncl, err := nats.Connect(lurl)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\tlsub, _ := ncl.SubscribeSync(\"foo.test\")\n\n\t// Wait for the subs to propagate. LDS + foo.test\n\tcheckSubInterest(t, s, a.GetName(), \"foo.test\", 2*time.Second)\n\n\t// Now send from nc1 with account 1, should be received by our leafnode subscriber.\n\tnc1.Publish(\"foo.test\", nil)\n\n\t_, err = lsub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n}\n\nfunc TestLeafNodeOperatorAndPermissions(t *testing.T) {\n\ts, opts, conf := runLeafNodeOperatorServer(t)\n\tdefer os.Remove(conf)\n\tdefer s.Shutdown()\n\n\tacc, akp := createAccount(t, s)\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\n\t// Create SRV user, with no limitations\n\tsrvnuc := jwt.NewUserClaims(pub)\n\tsrvujwt, err := srvnuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tsrvcreds := genCredsFile(t, srvujwt, seed)\n\tdefer os.Remove(srvcreds)\n\n\t// Create connection for SRV\n\tsrvnc, err := nats.Connect(s.ClientURL(), nats.UserCredentials(srvcreds))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer srvnc.Close()\n\n\t// Create on the server \"s\" a subscription on \"*\" and on \"foo\".\n\t// We check that the subscription on \"*\" will be able to receive\n\t// messages since LEAF has publish permissions on \"foo\", so msg\n\t// should be received.\n\tsrvsubStar, err := srvnc.SubscribeSync(\"*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tsrvsubFoo, err := srvnc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tsrvnc.Flush()\n\n\t// Create LEAF user, with pub perms on \"foo\" and sub perms on \"bar\"\n\tleafnuc := jwt.NewUserClaims(pub)\n\tleafnuc.Permissions.Pub.Allow.Add(\"foo\")\n\tleafnuc.Permissions.Sub.Allow.Add(\"bar\")\n\tleafnuc.Permissions.Sub.Allow.Add(\"baz\")\n\tleafujwt, err := leafnuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tleafcreds := genCredsFile(t, leafujwt, seed)\n\tdefer os.Remove(leafcreds)\n\n\tcontent := `\n\t\tport: -1\n\t\tserver_name: LN\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: nats-leaf://127.0.0.1:%d\n\t\t\t\t\tcredentials: '%s'\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\tconfig := fmt.Sprintf(content, opts.LeafNode.Port, leafcreds)\n\tlnconf := createConfFile(t, []byte(config))\n\tdefer os.Remove(lnconf)\n\tsl, _ := RunServerWithConfig(lnconf)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Check that interest makes it to \"sl\" server.\n\t// This helper does not check for wildcard interest...\n\tcheckSubInterest(t, sl, \"$G\", \"foo\", time.Second)\n\n\t// Create connection for LEAF and subscribe on \"bar\"\n\tleafnc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(leafcreds))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer leafnc.Close()\n\n\tleafsub, err := leafnc.SubscribeSync(\"bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\t// To check that we can pull in 'baz'.\n\tleafsubpwc, err := leafnc.SubscribeSync(\"*\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tleafnc.Flush()\n\n\t// Make sure the interest on \"bar\" from \"sl\" server makes it to the \"s\" server.\n\tcheckSubInterest(t, s, acc.GetName(), \"bar\", time.Second)\n\t// Check for local interest too.\n\tcheckSubInterest(t, sl, \"$G\", \"bar\", time.Second)\n\n\t// Now that we know that \"s\" has received interest on \"bar\", create\n\t// the sub on \"bar\" locally on \"s\"\n\tsrvsub, err := srvnc.SubscribeSync(\"bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\n\tsrvnc.Publish(\"bar\", []byte(\"hello\"))\n\tif _, err := srvsub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"SRV did not get message: %v\", err)\n\t}\n\tif _, err := leafsub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"LEAF did not get message: %v\", err)\n\t}\n\tif _, err := leafsubpwc.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"LEAF did not get message: %v\", err)\n\t}\n\n\t// The leafnode has a sub on '*', that should pull in a publish to 'baz'.\n\tsrvnc.Publish(\"baz\", []byte(\"hello\"))\n\tif _, err := leafsubpwc.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"LEAF did not get message: %v\", err)\n\t}\n\n\t// User LEAF user on \"sl\" server, publish on \"foo\"\n\tleafnc.Publish(\"foo\", []byte(\"hello\"))\n\t// The user SRV on \"s\" receives it because the LN connection\n\t// is allowed to publish on \"foo\".\n\tif _, err := srvsubFoo.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"SRV did not get message: %v\", err)\n\t}\n\t// The wildcard subscription should get it too.\n\tif _, err := srvsubStar.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"SRV did not get message: %v\", err)\n\t}\n\n\t// However, even when using an unrestricted user connects to \"sl\" and\n\t// publishes on \"bar\", the user SRV on \"s\" should not receive it because\n\t// the LN connection is not allowed to publish (send msg over) on \"bar\".\n\tnc, err := nats.Connect(sl.ClientURL(), nats.UserCredentials(srvcreds))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\tnc.Publish(\"bar\", []byte(\"should not received\"))\n\tif _, err := srvsub.NextMsg(250 * time.Millisecond); err == nil {\n\t\tt.Fatal(\"Should not have received message\")\n\t}\n}\n\nfunc TestLeafNodeSignerUser(t *testing.T) {\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Setup the two accounts for this server.\n\t_, akp1 := createAccount(t, s)\n\tapk1, _ := akp1.PublicKey()\n\n\t// add a signing key to the account\n\takp2, err := nkeys.CreateAccount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tapk2, _ := akp2.PublicKey()\n\n\ttoken, err := s.AccountResolver().Fetch(apk1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tac, err := jwt.DecodeAccountClaims(token)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tac.SigningKeys.Add(apk2)\n\tokp, _ := nkeys.FromSeed(oSeed)\n\ttoken, err = ac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// update the resolver\n\taccount, _ := s.LookupAccount(apk1)\n\terr = s.AccountResolver().Store(apk1, token)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ts.UpdateAccountClaims(account, ac)\n\n\ttt, err := s.AccountResolver().Fetch(apk1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tac2, err := jwt.DecodeAccountClaims(tt)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(ac2.SigningKeys) != 1 {\n\t\tt.Fatal(\"signing key is not added\")\n\t}\n\tif _, ok := ac2.SigningKeys[apk2]; !ok {\n\t\tt.Fatal(\"signing key is not added\")\n\t}\n\n\t// create an user signed by the signing key\n\tkp1, _ := nkeys.CreateUser()\n\tpub1, _ := kp1.PublicKey()\n\tnuc1 := jwt.NewUserClaims(pub1)\n\tnuc1.IssuerAccount = apk1\n\tujwt1, err := nuc1.Encode(akp2)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\t// Create the leaf node server using the first account.\n\tseed, _ := kp1.Seed()\n\tmycreds := genCredsFile(t, ujwt1, seed)\n\n\tsl, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc TestLeafNodeExportsImports(t *testing.T) {\n\t// So we will create a main server with two accounts. The remote server, acting as a leaf node, will simply have\n\t// the $G global account and no auth. Make sure things work properly here.\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Setup the two accounts for this server.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create second account with exports\n\tacc2, akp2 := createAccount(t, s)\n\takp2Pub, _ := akp2.PublicKey()\n\takp2AC := jwt.NewAccountClaims(akp2Pub)\n\tstreamExport := &jwt.Export{Subject: \"foo.stream\", Type: jwt.Stream}\n\tserviceExport := &jwt.Export{Subject: \"req.echo\", Type: jwt.Service}\n\takp2AC.Exports.Add(streamExport, serviceExport)\n\takp2ACJWT, err := akp2AC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tif err := s.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\ts.UpdateAccountClaims(acc2, akp2AC)\n\n\t// Now create the first account and add on the imports. This will be what is used in the leafnode.\n\tacc1, akp1 := createAccount(t, s)\n\takp1Pub, _ := akp1.PublicKey()\n\takp1AC := jwt.NewAccountClaims(akp1Pub)\n\tstreamImport := &jwt.Import{Account: akp2Pub, Subject: \"foo.stream\", To: \"import\", Type: jwt.Stream}\n\tserviceImport := &jwt.Import{Account: akp2Pub, Subject: \"import.request\", To: \"req.echo\", Type: jwt.Service}\n\takp1AC.Imports.Add(streamImport, serviceImport)\n\takp1ACJWT, err := akp1AC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tif err := s.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\ts.UpdateAccountClaims(acc1, akp1AC)\n\n\t// Create the user will we use to connect the leafnode.\n\tkp1, _ := nkeys.CreateUser()\n\tpub1, _ := kp1.PublicKey()\n\tnuc1 := jwt.NewUserClaims(pub1)\n\tujwt1, err := nuc1.Encode(akp1)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\t// Create the leaf node server using the first account.\n\tseed, _ := kp1.Seed()\n\tmycreds := genCredsFile(t, ujwt1, seed)\n\n\tsl, lopts, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// To connect to main server.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\n\t// Imported\n\tnc1, err := nats.Connect(url, createUserCreds(t, s, akp1))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Exported\n\tnc2, err := nats.Connect(url, createUserCreds(t, s, akp2))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Leaf node connection.\n\tlurl := fmt.Sprintf(\"nats://%s:%d\", lopts.Host, lopts.Port)\n\tncl, err := nats.Connect(lurl)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\t// So everything should be setup here. So let's test streams first.\n\tlsub, _ := ncl.SubscribeSync(\"import.foo.stream\")\n\n\t// Wait for all subs to propagate.\n\tcheckSubInterest(t, s, acc1.GetName(), \"import.foo.stream\", time.Second)\n\n\t// Pub to other account with export on original subject.\n\tnc2.Publish(\"foo.stream\", nil)\n\n\t_, err = lsub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// Services\n\t// Create listener on nc2\n\tnc2.Subscribe(\"req.echo\", func(msg *nats.Msg) {\n\t\tnc2.Publish(msg.Reply, []byte(\"WORKED\"))\n\t})\n\tnc2.Flush()\n\n\t// Now send the request on the leaf node client.\n\tif _, err := ncl.Request(\"import.request\", []byte(\"fingers crossed\"), 500*time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Did not receive response: %v\", err)\n\t}\n}\n\nfunc TestLeafNodeExportImportComplexSetup(t *testing.T) {\n\tcontent := `\n\tport: -1\n\toperator = \"./configs/nkeys/op.jwt\"\n\tresolver = MEMORY\n\tcluster {\n\t\tport: -1\n\t\tname: xyz\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts1, s1Opts := RunServerWithConfig(conf)\n\tdefer s1.Shutdown()\n\n\tcontent = fmt.Sprintf(`\n\tport: -1\n\toperator = \"./configs/nkeys/op.jwt\"\n\tresolver = MEMORY\n\tcluster {\n\t\tport: -1\n\t\tname: xyz\n\t\troutes: [\"nats://%s:%d\"]\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n\t`, s1Opts.Cluster.Host, s1Opts.Cluster.Port)\n\tconf = createConfFile(t, []byte(content))\n\ts2, s2Opts := RunServerWithConfig(conf)\n\tdefer s2.Shutdown()\n\n\t// Setup the two accounts for this server.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create second account with exports\n\tacc2, akp2 := createAccount(t, s1)\n\takp2Pub, _ := akp2.PublicKey()\n\takp2AC := jwt.NewAccountClaims(akp2Pub)\n\tstreamExport := &jwt.Export{Subject: \"foo.stream\", Type: jwt.Stream}\n\tserviceExport := &jwt.Export{Subject: \"req.echo\", Type: jwt.Service}\n\takp2AC.Exports.Add(streamExport, serviceExport)\n\takp2ACJWT, err := akp2AC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tif err := s1.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\ts1.UpdateAccountClaims(acc2, akp2AC)\n\n\t// Now create the first account and add on the imports. This will be what is used in the leafnode.\n\tacc1, akp1 := createAccount(t, s1)\n\takp1Pub, _ := akp1.PublicKey()\n\takp1AC := jwt.NewAccountClaims(akp1Pub)\n\tstreamImport := &jwt.Import{Account: akp2Pub, Subject: \"foo.stream\", To: \"import\", Type: jwt.Stream}\n\tserviceImport := &jwt.Import{Account: akp2Pub, Subject: \"import.request\", To: \"req.echo\", Type: jwt.Service}\n\takp1AC.Imports.Add(streamImport, serviceImport)\n\takp1ACJWT, err := akp1AC.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tif err := s1.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\ts1.UpdateAccountClaims(acc1, akp1AC)\n\n\tif err := s2.AccountResolver().Store(akp2Pub, akp2ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\t// Just make sure that account object registered in S2 is not acc2\n\tif a, err := s2.LookupAccount(acc2.Name); err != nil || a == acc2 {\n\t\tt.Fatalf(\"Lookup account error: %v - accounts are same: %v\", err, a == acc2)\n\t}\n\n\tif err := s2.AccountResolver().Store(akp1Pub, akp1ACJWT); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\t// Just make sure that account object registered in S2 is not acc1\n\tif a, err := s2.LookupAccount(acc1.Name); err != nil || a == acc1 {\n\t\tt.Fatalf(\"Lookup account error: %v - accounts are same: %v\", err, a == acc1)\n\t}\n\n\t// Create the user will we use to connect the leafnode.\n\tkp1, _ := nkeys.CreateUser()\n\tpub1, _ := kp1.PublicKey()\n\tnuc1 := jwt.NewUserClaims(pub1)\n\tujwt1, err := nuc1.Encode(akp1)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\n\t// Create the leaf node server using the first account.\n\tseed, _ := kp1.Seed()\n\tmycreds := genCredsFile(t, ujwt1, seed)\n\n\tsl, lopts, _ := runSolicitWithCredentials(t, s1Opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s1)\n\n\t// Url to server s2\n\ts2URL := fmt.Sprintf(\"nats://%s:%d\", s2Opts.Host, s2Opts.Port)\n\n\t// Imported\n\tnc1, err := nats.Connect(s2URL, createUserCreds(t, s2, akp1))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Exported\n\tnc2, err := nats.Connect(s2URL, createUserCreds(t, s2, akp2))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Leaf node connection.\n\tlurl := fmt.Sprintf(\"nats://%s:%d\", lopts.Host, lopts.Port)\n\tncl, err := nats.Connect(lurl)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\t// So everything should be setup here. So let's test streams first.\n\tlsub, _ := ncl.SubscribeSync(\"import.foo.stream\")\n\n\t// Wait for the sub to propagate to s2. LDS + subject above.\n\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\tif acc1.RoutedSubs() != 6 {\n\t\t\treturn fmt.Errorf(\"Still no routed subscription: %d\", acc1.RoutedSubs())\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Pub to other account with export on original subject.\n\tnc2.Publish(\"foo.stream\", nil)\n\n\tif _, err = lsub.NextMsg(1 * time.Second); err != nil {\n\t\tt.Fatalf(\"Did not receive stream message: %s\", err)\n\t}\n\n\t// Services\n\t// Create listener on nc2 (which connects to s2)\n\tgotIt := int32(0)\n\tnc2.Subscribe(\"req.echo\", func(msg *nats.Msg) {\n\t\tatomic.AddInt32(&gotIt, 1)\n\t\tnc2.Publish(msg.Reply, []byte(\"WORKED\"))\n\t})\n\tnc2.Flush()\n\n\t// Wait for it to make it across.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Now send the request on the leaf node client.\n\tif _, err := ncl.Request(\"import.request\", []byte(\"fingers crossed\"), 5500*time.Millisecond); err != nil {\n\t\tif atomic.LoadInt32(&gotIt) == 0 {\n\t\t\tt.Fatalf(\"Request was not received\")\n\t\t}\n\t\tt.Fatalf(\"Did not receive response: %v\", err)\n\t}\n}\n\nfunc TestLeafNodeInfoURLs(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tuseAdvertise bool\n\t}{\n\t\t{\n\t\t\t\"without advertise\",\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t\"with advertise\",\n\t\t\ttrue,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\topts := testDefaultOptionsForLeafNodes()\n\t\t\topts.Cluster.Name = \"A\"\n\t\t\topts.Cluster.Port = -1\n\t\t\topts.LeafNode.Host = \"127.0.0.1\"\n\t\t\tif test.useAdvertise {\n\t\t\t\topts.LeafNode.Advertise = \"me:1\"\n\t\t\t}\n\t\t\ts1 := RunServer(opts)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\t\tdefer lc.Close()\n\t\t\tinfo := checkInfoMsg(t, lc)\n\t\t\tif sz := len(info.LeafNodeURLs); sz != 1 {\n\t\t\t\tt.Fatalf(\"Expected LeafNodeURLs array to be size 1, got %v\", sz)\n\t\t\t}\n\t\t\tvar s1LNURL string\n\t\t\tif test.useAdvertise {\n\t\t\t\ts1LNURL = \"me:1\"\n\t\t\t} else {\n\t\t\t\ts1LNURL = net.JoinHostPort(opts.LeafNode.Host, strconv.Itoa(opts.LeafNode.Port))\n\t\t\t}\n\t\t\tif url := info.LeafNodeURLs[0]; url != s1LNURL {\n\t\t\t\tt.Fatalf(\"Expected URL to be %s, got %s\", s1LNURL, url)\n\t\t\t}\n\t\t\tlc.Close()\n\n\t\t\topts2 := testDefaultOptionsForLeafNodes()\n\t\t\topts2.Cluster.Name = \"A\"\n\t\t\topts2.Cluster.Port = -1\n\t\t\topts2.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", opts.Cluster.Host, opts.Cluster.Port))\n\t\t\topts2.LeafNode.Host = \"127.0.0.1\"\n\t\t\tif test.useAdvertise {\n\t\t\t\topts2.LeafNode.Advertise = \"me:2\"\n\t\t\t}\n\t\t\ts2 := RunServer(opts2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\t\tdefer lc.Close()\n\t\t\tinfo = checkInfoMsg(t, lc)\n\t\t\tif sz := len(info.LeafNodeURLs); sz != 2 {\n\t\t\t\tt.Fatalf(\"Expected LeafNodeURLs array to be size 2, got %v\", sz)\n\t\t\t}\n\t\t\tvar s2LNURL string\n\t\t\tif test.useAdvertise {\n\t\t\t\ts2LNURL = \"me:2\"\n\t\t\t} else {\n\t\t\t\ts2LNURL = net.JoinHostPort(opts2.LeafNode.Host, strconv.Itoa(opts2.LeafNode.Port))\n\t\t\t}\n\t\t\tvar ok [2]int\n\t\t\tfor _, url := range info.LeafNodeURLs {\n\t\t\t\tif url == s1LNURL {\n\t\t\t\t\tok[0]++\n\t\t\t\t} else if url == s2LNURL {\n\t\t\t\t\tok[1]++\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor i, res := range ok {\n\t\t\t\tif res != 1 {\n\t\t\t\t\tt.Fatalf(\"URL from server %v was found %v times\", i+1, res)\n\t\t\t\t}\n\t\t\t}\n\t\t\tlc.Close()\n\n\t\t\t// Remove s2, and wait for route to be lost on s1.\n\t\t\ts2.Shutdown()\n\t\t\tcheckNumRoutes(t, s1, 0)\n\n\t\t\t// Now check that s1 returns only itself in the URLs array.\n\t\t\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\t\tdefer lc.Close()\n\t\t\tinfo = checkInfoMsg(t, lc)\n\t\t\tif sz := len(info.LeafNodeURLs); sz != 1 {\n\t\t\t\tt.Fatalf(\"Expected LeafNodeURLs array to be size 1, got %v\", sz)\n\t\t\t}\n\t\t\tif url := info.LeafNodeURLs[0]; url != s1LNURL {\n\t\t\t\tt.Fatalf(\"Expected URL to be %s, got %s\", s1LNURL, url)\n\t\t\t}\n\t\t\tlc.Close()\n\n\t\t\ts1.Shutdown()\n\n\t\t\t// Now we need a configuration where both s1 and s2 have a route\n\t\t\t// to each other, so we need explicit configuration. We are trying\n\t\t\t// to get S1->S2 and S2->S1 so one of the route is dropped. This\n\t\t\t// should not affect the number of URLs reported in INFO.\n\n\t\t\topts.Cluster.Port = 5223\n\t\t\topts.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://%s:5224\", opts2.Host))\n\t\t\ts1, _ = server.NewServer(opts)\n\t\t\tdefer s1.Shutdown()\n\n\t\t\topts2.Cluster.Port = 5224\n\t\t\topts2.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://%s:5223\", opts.Host))\n\t\t\ts2, _ = server.NewServer(opts2)\n\t\t\tdefer s2.Shutdown()\n\n\t\t\t// Start this way to increase chance of having the two connect\n\t\t\t// to each other at the same time. This will cause one of the\n\t\t\t// route to be dropped.\n\t\t\twg := &sync.WaitGroup{}\n\t\t\twg.Add(2)\n\t\t\tgo func() {\n\t\t\t\ts1.Start()\n\t\t\t\twg.Done()\n\t\t\t}()\n\t\t\tgo func() {\n\t\t\t\ts2.Start()\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\tcheckClusterFormed(t, s1, s2)\n\n\t\t\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\t\tdefer lc.Close()\n\t\t\tinfo = checkInfoMsg(t, lc)\n\t\t\tif sz := len(info.LeafNodeURLs); sz != 2 {\n\t\t\t\tt.Fatalf(\"Expected LeafNodeURLs array to be size 2, got %v\", sz)\n\t\t\t}\n\t\t\tok[0], ok[1] = 0, 0\n\t\t\tfor _, url := range info.LeafNodeURLs {\n\t\t\t\tif url == s1LNURL {\n\t\t\t\t\tok[0]++\n\t\t\t\t} else if url == s2LNURL {\n\t\t\t\t\tok[1]++\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor i, res := range ok {\n\t\t\t\tif res != 1 {\n\t\t\t\t\tt.Fatalf(\"URL from server %v was found %v times\", i+1, res)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLeafNodeFailover(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(50 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 2)\n\tdefer shutdownCluster(ca)\n\n\tcb := createClusterWithName(t, \"B\", 1, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Start a server that creates LeafNode connection to first\n\t// server in cluster A.\n\ts, opts := runSolicitLeafServer(ca.opts[0])\n\tdefer s.Shutdown()\n\n\t// Shutdown that server on A.\n\tca.servers[0].Shutdown()\n\n\t// Make sure that s reconnects its LN connection\n\tcheckLeafNodeConnected(t, ca.servers[1])\n\n\t// Verify that LeafNode info protocol is sent to the server `s`\n\t// with list of new servers. To do that, we will restart\n\t// ca[0] but with a different LN listen port.\n\tca.opts[0].Port = -1\n\tca.opts[0].Cluster.Port = -1\n\tca.opts[0].Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", ca.opts[1].Cluster.Host, ca.opts[1].Cluster.Port))\n\tca.opts[0].LeafNode.Port = -1\n\tnewa0 := RunServer(ca.opts[0])\n\tdefer newa0.Shutdown()\n\n\tcheckClusterFormed(t, newa0, ca.servers[1])\n\n\t// Shutdown the server the LN is currently connected to. It should\n\t// reconnect to newa0.\n\tca.servers[1].Shutdown()\n\tcheckLeafNodeConnected(t, newa0)\n\n\t// Now shutdown newa0 and make sure `s` does not reconnect\n\t// to server in gateway.\n\tnewa0.Shutdown()\n\n\t// Wait for more than the reconnect attempts.\n\ttime.Sleep(opts.LeafNode.ReconnectInterval + 50*time.Millisecond)\n\n\tcheckLeafNodeConnections(t, cb.servers[0], 0)\n}\n\nfunc TestLeafNodeAdvertise(t *testing.T) {\n\t// Create a dummy listener which will we use for the advertise address.\n\t// We will then stop the server and the test will be a success if\n\t// this listener accepts a connection.\n\tch := make(chan struct{}, 1)\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error starting listener: %v\", err)\n\t}\n\tdefer l.Close()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tc, _ := l.Accept()\n\t\tif c != nil {\n\t\t\tc.Close()\n\t\t}\n\t\tl.Close()\n\t\tch <- struct{}{}\n\t}()\n\n\tport := l.Addr().(*net.TCPAddr).Port\n\n\to2 := testDefaultOptionsForLeafNodes()\n\to2.LeafNode.Advertise = fmt.Sprintf(\"127.0.0.1:%d\", port)\n\to2.Cluster.Name = \"A\"\n\to2.Cluster.Port = -1\n\ts2 := RunServer(o2)\n\tdefer s2.Shutdown()\n\n\to1 := testDefaultOptionsForLeafNodes()\n\to1.Cluster.Name = \"A\"\n\to1.Cluster.Port = -1\n\to1.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o2.Cluster.Port))\n\ts1 := RunServer(o1)\n\tdefer s1.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\t// Start a server that connects to s1. It should be made aware\n\t// of s2 (and its advertise address).\n\ts, _ := runSolicitLeafServer(o1)\n\tdefer s.Shutdown()\n\n\t// Wait for leaf node connection to be established on s1.\n\tcheckLeafNodeConnected(t, s1)\n\n\t// Shutdown s1. The listener that we created should be the one\n\t// receiving the connection from s.\n\ts1.Shutdown()\n\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatalf(\"Server did not reconnect to advertised address\")\n\t}\n\twg.Wait()\n}\n\nfunc TestLeafNodeConnectionLimitsSingleServer(t *testing.T) {\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Setup account and a user that will be used by the remote leaf node server.\n\t// createAccount automatically registers with resolver etc..\n\tacc, akp := createAccount(t, s)\n\n\t// Now update with limits for lead node connections.\n\tconst maxleafs = 2\n\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tnac.Limits.LeafNodeConn = maxleafs\n\ts.UpdateAccountClaims(acc, nac)\n\n\t// Make sure we have the limits updated in acc.\n\tif mleafs := acc.MaxActiveLeafNodes(); mleafs != maxleafs {\n\t\tt.Fatalf(\"Expected to have max leafnodes of %d, got %d\", maxleafs, mleafs)\n\t}\n\n\t// Create the user credentials for the leadnode connection.\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\tcheckAccConnectionCounts := func(t *testing.T, expected int) {\n\t\tt.Helper()\n\t\tcheckFor(t, 2*time.Second, 15*time.Millisecond, func() error {\n\t\t\t// Make sure we are accounting properly here.\n\t\t\tif nln := acc.NumLeafNodes(); nln != expected {\n\t\t\t\treturn fmt.Errorf(\"expected %v leaf node, got %d\", expected, nln)\n\t\t\t}\n\t\t\t// clients and leafnodes counted together.\n\t\t\tif nc := acc.NumConnections(); nc != expected {\n\t\t\t\treturn fmt.Errorf(\"expected %v for total connections, got %d\", expected, nc)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tcheckAccNLF := func(n int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tif nln := acc.NumLeafNodes(); nln != n {\n\t\t\t\treturn fmt.Errorf(\"Expected %d leaf node, got %d\", n, nln)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsl, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnections(t, s, 1)\n\n\t// Make sure we are accounting properly here.\n\tcheckAccNLF(1)\n\t// clients and leafnodes counted together.\n\tif nc := acc.NumConnections(); nc != 1 {\n\t\tt.Fatalf(\"Expected 1 for total connections, got %d\", nc)\n\t}\n\n\ts2, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer s2.Shutdown()\n\tcheckLeafNodeConnections(t, s, 2)\n\tcheckAccConnectionCounts(t, 2)\n\n\ts2.Shutdown()\n\tcheckLeafNodeConnections(t, s, 1)\n\tcheckAccConnectionCounts(t, 1)\n\n\t// Make sure we are accounting properly here.\n\tcheckAccNLF(1)\n\t// clients and leafnodes counted together.\n\tif nc := acc.NumConnections(); nc != 1 {\n\t\tt.Fatalf(\"Expected 1 for total connections, got %d\", nc)\n\t}\n\n\t// Now add back the second one as #3.\n\ts3, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer s3.Shutdown()\n\tcheckLeafNodeConnections(t, s, 2)\n\n\tcheckAccConnectionCounts(t, 2)\n\n\t// Once we are here we should not be able to create anymore. Limit == 2.\n\ts4, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer s4.Shutdown()\n\n\tcheckAccConnectionCounts(t, 2)\n\n\t// Make sure s4 has 0 still. We need checkFor because it is possible\n\t// that when we check we have actually the connection registered for\n\t// a short period before it is closed due to limit.\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif nln := s4.NumLeafNodes(); nln != 0 {\n\t\t\treturn fmt.Errorf(\"Expected no leafnodes accounted for in s4, got %d\", nln)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure this is still 2.\n\tcheckLeafNodeConnections(t, s, 2)\n}\n\nfunc TestLeafNodeConnectionLimitsCluster(t *testing.T) {\n\tcontent := `\n\tport: -1\n\toperator = \"./configs/nkeys/op.jwt\"\n system_account = \"AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG\"\n\tresolver = MEMORY\n\tcluster {\n\t\tport: -1\n\t\tname: xyz\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n resolver_preload = {\n AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : \"eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ\"\n }\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts1, s1Opts := RunServerWithConfig(conf)\n\tdefer s1.Shutdown()\n\n\tcontent = fmt.Sprintf(`\n\tport: -1\n\toperator = \"./configs/nkeys/op.jwt\"\n system_account = \"AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG\"\n\tresolver = MEMORY\n\tcluster {\n\t\tport: -1\n\t\tname: xyz\n\t\troutes: [\"nats://%s:%d\"]\n\t}\n\tleafnodes {\n\t\tlisten: \"127.0.0.1:-1\"\n\t}\n resolver_preload = {\n AD2VB6C25DQPEUUQ7KJBUFX2J4ZNVBPOHSCBISC7VFZXVWXZA7VASQZG : \"eyJ0eXAiOiJqd3QiLCJhbGciOiJlZDI1NTE5In0.eyJqdGkiOiJDSzU1UERKSUlTWU5QWkhLSUpMVURVVTdJT1dINlM3UkE0RUc2TTVGVUQzUEdGQ1RWWlJRIiwiaWF0IjoxNTQzOTU4NjU4LCJpc3MiOiJPQ0FUMzNNVFZVMlZVT0lNR05HVU5YSjY2QUgyUkxTREFGM01VQkNZQVk1UU1JTDY1TlFNNlhRRyIsInN1YiI6IkFEMlZCNkMyNURRUEVVVVE3S0pCVUZYMko0Wk5WQlBPSFNDQklTQzdWRlpYVldYWkE3VkFTUVpHIiwidHlwZSI6ImFjY291bnQiLCJuYXRzIjp7ImxpbWl0cyI6e319fQ.7m1fysYUsBw15Lj88YmYoHxOI4HlOzu6qgP8Zg-1q9mQXUURijuDGVZrtb7gFYRlo-nG9xZyd2ZTRpMA-b0xCQ\"\n }\n\t`, s1Opts.Cluster.Host, s1Opts.Cluster.Port)\n\tconf = createConfFile(t, []byte(content))\n\ts2, s2Opts := RunServerWithConfig(conf)\n\tdefer s2.Shutdown()\n\n\t// Setup the two accounts for this server.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Setup account and a user that will be used by the remote leaf node server.\n\t// createAccount automatically registers with resolver etc..\n\tacc, akp := createAccount(t, s1)\n\n\t// Now update with limits for lead node connections.\n\tconst maxleafs = 10\n\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tnac.Limits.LeafNodeConn = maxleafs\n\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tif err := s1.AccountResolver().Store(apub, ajwt); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\ts1.UpdateAccountClaims(acc, nac)\n\n\tif err := s2.AccountResolver().Store(apub, ajwt); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\t// Make sure that account object registered in S2 is not acc2\n\tacc2, err := s2.LookupAccount(acc.Name)\n\tif err != nil || acc == acc2 {\n\t\tt.Fatalf(\"Lookup account error: %v - accounts are same: %v\", err, acc == acc2)\n\t}\n\n\t// Create the user credentials for the leadnode connection.\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\tloop := maxleafs / 2\n\n\t// Now create maxleafs/2 leaf node servers on each operator server.\n\tfor i := 0; i < loop; i++ {\n\t\tsl1, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds)\n\t\tdefer sl1.Shutdown()\n\n\t\tsl2, _, _ := runSolicitWithCredentials(t, s2Opts, mycreds)\n\t\tdefer sl2.Shutdown()\n\t}\n\n\tcheckLeafNodeConnections(t, s1, loop)\n\tcheckLeafNodeConnections(t, s2, loop)\n\n\t// Now check that we have the remotes registered. This will prove we are sending\n\t// and processing the leaf node connect events properly etc.\n\tcheckAccLFCount := func(acc *server.Account, remote bool, n int) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tstr := \"\"\n\t\t\tvar nln int\n\t\t\tif remote {\n\t\t\t\tnln = acc.NumRemoteLeafNodes()\n\t\t\t\tstr = \"remote\"\n\t\t\t} else {\n\t\t\t\tnln = acc.NumLeafNodes()\n\t\t\t}\n\t\t\tif nln != n {\n\t\t\t\treturn fmt.Errorf(\"number of expected %sleaf nodes is %v, got %v\", str, n, nln)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\tcheckAccLFCount(acc, true, loop)\n\tcheckAccLFCount(acc2, true, loop)\n\n\t// Now that we are here we should not be allowed anymore leaf nodes.\n\tl, _, _ := runSolicitWithCredentials(t, s1Opts, mycreds)\n\tdefer l.Shutdown()\n\n\tcheckAccLFCount(acc, false, maxleafs)\n\t// Should still be at loop size.\n\tcheckLeafNodeConnections(t, s1, loop)\n\n\tl, _, _ = runSolicitWithCredentials(t, s2Opts, mycreds)\n\tdefer l.Shutdown()\n\tcheckAccLFCount(acc2, false, maxleafs)\n\t// Should still be at loop size.\n\tcheckLeafNodeConnections(t, s2, loop)\n}\n\nfunc TestLeafNodeSwitchGatewayToInterestModeOnly(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(50 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create client on a server in cluster A\n\topts := ca.opts[0]\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Send a message from this client on \"foo\" so that B\n\t// registers a no-interest for account \"$G\"\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\t// Create a leaf node connection on a server in cluster B\n\topts = cb.opts[0]\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\t// This is for our global responses since we are setting up GWs above.\n\tleafSend, leafExpect := setupLeaf(t, lc, 8)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n}\n\n// The MSG proto for routes and gateways is RMSG, and we have an\n// optimization that a scratch buffer has RMSG and when doing a\n// client we just start at scratch[1]. For leaf nodes its LMSG and we\n// rewrite scratch[0], but never reset it which causes protocol\n// errors when used with routes or gateways after use to send\n// to a leafnode.\n// We will create a server with a leafnode connection and a route\n// and a gateway connection.\n\n// route connections to simulate.\nfunc TestLeafNodeResetsMSGProto(t *testing.T) {\n\tserver.GatewayDoNotForceInterestOnlyMode(true)\n\tdefer server.GatewayDoNotForceInterestOnlyMode(false)\n\n\topts := testDefaultOptionsForLeafNodes()\n\topts.Cluster.Name = \"xyz\"\n\topts.Cluster.Host = opts.Host\n\topts.Cluster.Port = -1\n\topts.Gateway.Name = \"xyz\"\n\topts.Gateway.Host = opts.Host\n\topts.Gateway.Port = -1\n\topts.Accounts = []*server.Account{server.NewAccount(\"$SYS\")}\n\topts.SystemAccount = \"$SYS\"\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupConn(t, lc)\n\n\t// To avoid possible INFO when switching to interest mode only,\n\t// delay start of gateway.\n\ttime.Sleep(500 * time.Millisecond)\n\n\tgw := createGatewayConn(t, opts.Gateway.Host, opts.Gateway.Port)\n\tdefer gw.Close()\n\n\tgwSend, gwExpect := setupGatewayConn(t, gw, \"A\", \"xyz\")\n\tgwSend(\"PING\\r\\n\")\n\tgwExpect(pongRe)\n\n\t// This is for our global responses since we are setting up GWs above.\n\tleafExpect(lsubRe)\n\n\t// Now setup interest in the leaf node for 'foo'.\n\tleafSend(\"LS+ foo\\r\\nPING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// Send msg from the gateway.\n\tgwSend(\"RMSG $G foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\tgwExpect(pongRe)\n\n\tleafExpect(lmsgRe)\n\n\t// At this point the gw inside our main server's scratch buffer is LMSG. When we do\n\t// same with a connected route with interest it should fail.\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\tcheckInfoMsg(t, rc)\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, \"RC\")\n\n\trouteSend(\"RS+ $G foo\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// This is for the route interest we just created.\n\tleafExpect(lsubRe)\n\n\t// Send msg from the gateway.\n\tgwSend(\"RMSG $G foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\tgwExpect(pongRe)\n\n\tleafExpect(lmsgRe)\n\n\t// Now make sure we get it on route. This will fail with the proto bug.\n\trouteExpect(rmsgRe)\n}\n\n// We need to make sure that as a remote server we also send our local subs on connect.\nfunc TestLeafNodeSendsRemoteSubsOnConnect(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tsl, slOpts := runSolicitLeafServer(opts)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\ts.Shutdown()\n\n\tc := createClientConn(t, slOpts.Host, slOpts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"SUB foo 1\\r\\n\")\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Need to restart it on the same port.\n\ts, _ = runLeafServerOnPort(opts.LeafNode.Port)\n\tdefer s.Shutdown()\n\tcheckLeafNodeConnected(t, s)\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tsetupLeaf(t, lc, 3)\n}\n\nfunc TestLeafNodeServiceImportLikeNGS(t *testing.T) {\n\tgwSolicit := 10 * time.Millisecond\n\tca := createClusterEx(t, true, gwSolicit, true, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterEx(t, true, gwSolicit, true, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Hang a responder off of cluster A.\n\topts := ca.opts[0]\n\turl := fmt.Sprintf(\"nats://ngs:pass@%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Create a queue subscriber to send results\n\tnc.QueueSubscribe(\"ngs.usage.*\", \"ngs\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"22\"))\n\t})\n\tnc.Flush()\n\n\t// Now create a leafnode server on B.\n\topts = cb.opts[1]\n\tsl, slOpts := runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://dlc:pass@%s:%d\", opts.LeafNode.Host, opts.LeafNode.Port))\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, sl)\n\n\t// Create a normal direct connect client on B.\n\turl = fmt.Sprintf(\"nats://dlc:pass@%s:%d\", opts.Host, opts.Port)\n\tnc2, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tif _, err := nc2.Request(\"ngs.usage\", []byte(\"fingers crossed\"), 500*time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Did not receive response: %v\", err)\n\t}\n\n\t// Now create a client on the leafnode.\n\turl = fmt.Sprintf(\"nats://%s:%d\", slOpts.Host, slOpts.Port)\n\tncl, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\tif _, err := ncl.Request(\"ngs.usage\", []byte(\"fingers crossed\"), 500*time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Did not receive response: %v\", err)\n\t}\n}\n\nfunc TestLeafNodeServiceImportResponderOnLeaf(t *testing.T) {\n\tgwSolicit := 10 * time.Millisecond\n\tca := createClusterEx(t, true, gwSolicit, true, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\n\t// Now create a leafnode server on A that will bind to the NGS account.\n\topts := ca.opts[1]\n\tsl, slOpts := runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://ngs:pass@%s:%d\", opts.LeafNode.Host, opts.LeafNode.Port))\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, sl)\n\n\t// Now create a client on the leafnode.\n\tncl, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", slOpts.Host, slOpts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\t// Create a queue subscriber to send results\n\tncl.QueueSubscribe(\"ngs.usage.*\", \"ngs\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"22\"))\n\t})\n\tncl.Flush()\n\n\t// Create a normal direct connect client on A. Needs to be same server as leafnode.\n\topts = ca.opts[1]\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://dlc:pass@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tif _, err := nc.Request(\"ngs.usage\", []byte(\"fingers crossed\"), 500*time.Millisecond); err != nil {\n\t\tt.Fatalf(\"Did not receive response: %v\", err)\n\t}\n}\n\nfunc TestLeafNodeSendsAccountingEvents(t *testing.T) {\n\ts, opts, _ := runLeafNodeOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// System account\n\tacc, akp := createAccount(t, s)\n\tif err := s.SetSystemAccount(acc.Name); err != nil {\n\t\tt.Fatalf(\"Expected this succeed, got %v\", err)\n\t}\n\n\t// Leafnode Account\n\tlacc, lakp := createAccount(t, s)\n\n\t// Create a system account user and connect a client to listen for the events.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Watch only for our leaf node account.\n\tcSub, _ := nc.SubscribeSync(fmt.Sprintf(\"$SYS.ACCOUNT.%s.CONNECT\", lacc.Name))\n\tdSub, _ := nc.SubscribeSync(fmt.Sprintf(\"$SYS.ACCOUNT.%s.DISCONNECT\", lacc.Name))\n\tnc.Flush()\n\n\t// Now create creds for the leafnode and connect the server.\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(lakp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tseed, _ := kp.Seed()\n\tmycreds := genCredsFile(t, ujwt, seed)\n\n\tsl, _, _ := runSolicitWithCredentials(t, opts, mycreds)\n\tdefer sl.Shutdown()\n\n\t// Wait for connect event\n\tmsg, err := cSub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error waiting for account connect event: %v\", err)\n\t}\n\tm := server.ConnectEventMsg{}\n\tif err := json.Unmarshal(msg.Data, &m); err != nil {\n\t\tt.Fatal(\"Did not get correctly formatted event\")\n\t}\n\n\t// Shutdown leafnode to generate disconnect event.\n\tsl.Shutdown()\n\n\tmsg, err = dSub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error waiting for account disconnect event: %v\", err)\n\t}\n\tdm := server.DisconnectEventMsg{}\n\tif err := json.Unmarshal(msg.Data, &dm); err != nil {\n\t\tt.Fatal(\"Did not get correctly formatted event\")\n\t}\n}\n\nfunc TestLeafNodeDistributedQueueAcrossGWs(t *testing.T) {\n\tgwSolicit := 10 * time.Millisecond\n\tca := createClusterEx(t, true, gwSolicit, true, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterEx(t, true, gwSolicit, true, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\tcc := createClusterEx(t, true, gwSolicit, true, \"C\", 3, ca, cb)\n\tdefer shutdownCluster(cc)\n\n\t// Create queue subscribers\n\tcreateQS := func(c *cluster) *nats.Conn {\n\t\tt.Helper()\n\t\topts := c.opts[rand.Intn(len(c.opts))]\n\t\turl := fmt.Sprintf(\"nats://ngs:pass@%s:%d\", opts.Host, opts.Port)\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tnc.QueueSubscribe(\"ngs.usage.*\", \"dq\", func(m *nats.Msg) {\n\t\t\tm.Respond([]byte(c.name))\n\t\t})\n\t\tnc.Flush()\n\t\treturn nc\n\t}\n\n\tncA := createQS(ca)\n\tdefer ncA.Close()\n\tncB := createQS(cb)\n\tdefer ncB.Close()\n\tncC := createQS(cc)\n\tdefer ncC.Close()\n\n\tconnectAndRequest := func(url, clusterName string, nreqs int) {\n\t\tt.Helper()\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tfor i := 0; i < nreqs; i++ {\n\t\t\tm, err := nc.Request(\"ngs.usage\", nil, 500*time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Did not receive a response: %v\", err)\n\t\t\t}\n\t\t\tif string(m.Data) != clusterName {\n\t\t\t\tt.Fatalf(\"Expected to prefer %q, but got response from %q\", clusterName, m.Data)\n\t\t\t}\n\t\t}\n\t}\n\n\tcheckClientDQ := func(c *cluster, nreqs int) {\n\t\tt.Helper()\n\t\t// Pick one at random.\n\t\topts := c.opts[rand.Intn(len(c.opts))]\n\t\turl := fmt.Sprintf(\"nats://dlc:pass@%s:%d\", opts.Host, opts.Port)\n\t\tconnectAndRequest(url, c.name, nreqs)\n\t}\n\n\t// First check that this works with direct connected clients.\n\tcheckClientDQ(ca, 100)\n\tcheckClientDQ(cb, 100)\n\tcheckClientDQ(cc, 100)\n\n\tcreateLNS := func(c *cluster) (*server.Server, *server.Options) {\n\t\tt.Helper()\n\t\t// Pick one at random.\n\t\tcopts := c.opts[rand.Intn(len(c.servers))]\n\t\ts, opts := runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://dlc:pass@%s:%d\", copts.LeafNode.Host, copts.LeafNode.Port))\n\t\tcheckLeafNodeConnected(t, s)\n\t\treturn s, opts\n\t}\n\n\tcheckLeafDQ := func(opts *server.Options, clusterName string, nreqs int) {\n\t\tt.Helper()\n\t\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t\tconnectAndRequest(url, clusterName, nreqs)\n\t}\n\n\t// Test leafnodes to all clusters.\n\tfor _, c := range []*cluster{ca, cb, cc} {\n\t\t// Now create a leafnode on cluster.\n\t\tsl, slOpts := createLNS(c)\n\t\tdefer sl.Shutdown()\n\t\t// Now connect to the leafnode server and run test.\n\t\tcheckLeafDQ(slOpts, c.name, 100)\n\t}\n}\n\nfunc TestLeafNodeDistributedQueueEvenly(t *testing.T) {\n\tgwSolicit := 10 * time.Millisecond\n\tca := createClusterEx(t, true, gwSolicit, true, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterEx(t, true, gwSolicit, true, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create queue subscribers\n\tcreateQS := func(c *cluster) *nats.Conn {\n\t\tt.Helper()\n\t\topts := c.opts[rand.Intn(len(c.opts))]\n\t\turl := fmt.Sprintf(\"nats://ngs:pass@%s:%d\", opts.Host, opts.Port)\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tcid, _ := nc.GetClientID()\n\t\tresponse := []byte(fmt.Sprintf(\"%s:%d:%d\", c.name, opts.Port, cid))\n\t\tnc.QueueSubscribe(\"ngs.usage.*\", \"dq\", func(m *nats.Msg) {\n\t\t\tm.Respond(response)\n\t\t})\n\t\tnc.Flush()\n\t\treturn nc\n\t}\n\n\tncA1 := createQS(ca)\n\tdefer ncA1.Close()\n\n\tncA2 := createQS(ca)\n\tdefer ncA2.Close()\n\n\tncA3 := createQS(ca)\n\tdefer ncA3.Close()\n\n\tresp := make(map[string]int)\n\n\tconnectAndRequest := func(url, clusterName string, nreqs int) {\n\t\tt.Helper()\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tfor i := 0; i < nreqs; i++ {\n\t\t\tm, err := nc.Request(\"ngs.usage\", nil, 500*time.Millisecond)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Did not receive a response: %v\", err)\n\t\t\t}\n\t\t\tif string(m.Data[0]) != clusterName {\n\t\t\t\tt.Fatalf(\"Expected to prefer %q, but got response from %q\", clusterName, m.Data[0])\n\t\t\t}\n\t\t\tresp[string(m.Data)]++\n\t\t}\n\t}\n\n\tcreateLNS := func(c *cluster) (*server.Server, *server.Options) {\n\t\tt.Helper()\n\t\t// Pick one at random.\n\t\tcopts := c.opts[rand.Intn(len(c.servers))]\n\t\ts, opts := runSolicitLeafServerToURL(fmt.Sprintf(\"nats-leaf://dlc:pass@%s:%d\", copts.LeafNode.Host, copts.LeafNode.Port))\n\t\tcheckLeafNodeConnected(t, s)\n\t\treturn s, opts\n\t}\n\n\tcheckLeafDQ := func(opts *server.Options, clusterName string, nreqs int) {\n\t\tt.Helper()\n\t\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t\tconnectAndRequest(url, clusterName, nreqs)\n\t}\n\n\t// Now create a leafnode on cluster A.\n\tsl, slOpts := createLNS(ca)\n\tdefer sl.Shutdown()\n\n\t// Now connect to the leafnode server and run test.\n\tcheckLeafDQ(slOpts, ca.name, 100)\n\n\t// Should have some for all 3 QS [ncA1, ncA2, ncA3]\n\tif lr := len(resp); lr != 3 {\n\t\tt.Fatalf(\"Expected all 3 queue subscribers to have received some messages, only got %d\", lr)\n\t}\n\t// Now check that we have at least 10% for each subscriber.\n\tfor _, r := range resp {\n\t\tif r < 10 {\n\t\t\tt.Fatalf(\"Got a subscriber with less than 10 responses: %d\", r)\n\t\t}\n\t}\n}\n\nfunc TestLeafNodeDefaultPort(t *testing.T) {\n\to := testDefaultOptionsForLeafNodes()\n\to.LeafNode.Port = server.DEFAULT_LEAFNODE_PORT\n\ts := RunServer(o)\n\tdefer s.Shutdown()\n\n\tconf := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tleaf {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turl: \"leafnode://127.0.0.1\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`))\n\n\tsl, _ := RunServerWithConfig(conf)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n}\n\n// Since leafnode's are true interest only we need to make sure that we\n// register the proper interest with global routing $GR.xxxxxx._INBOX.>\nfunc TestLeafNodeAndGatewayGlobalRouting(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(50 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\tsl, slOpts := runSolicitLeafServer(ca.opts[1])\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, ca.servers[1])\n\n\t// Create a client on the leafnode. This will listen for requests.\n\tncl, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", slOpts.Host, slOpts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\tncl.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"World\"))\n\t})\n\tncl.Flush()\n\n\t// Since for leafnodes the account becomes interest-only mode,\n\t// let's make sure that the interest on \"foo\" has time to propagate\n\t// to cluster B.\n\ttime.Sleep(250 * time.Millisecond)\n\n\t// Create a direct connect requestor. Try with all possible\n\t// servers in cluster B to make sure that we also receive the\n\t// reply when the accepting leafnode server does not have\n\t// its outbound GW connection to the requestor's server.\n\tfor i := 0; i < 3; i++ {\n\t\topts := cb.opts[i]\n\t\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\t\tnc, err := nats.Connect(url)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\n\t\t// We don't use an INBOX here because we had a bug where\n\t\t// leafnode would subscribe to _GR_.*.*.*.> instead of\n\t\t// _GR_.>, and inbox masked that because of their number\n\t\t// of tokens.\n\t\treply := nuid.Next()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t\t}\n\t\tif err := nc.PublishRequest(\"foo\", reply, []byte(\"Hello\")); err != nil {\n\t\t\tt.Fatalf(\"Failed to send request: %v\", err)\n\t\t}\n\t\tif _, err := sub.NextMsg(250 * time.Millisecond); err != nil {\n\t\t\tt.Fatalf(\"Did not get reply from server %d: %v\", i, err)\n\t\t}\n\t\tnc.Close()\n\t}\n}\n\nfunc checkLeafNode2Connected(t *testing.T, s *server.Server) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif nln := s.NumLeafNodes(); nln != 2 {\n\t\t\treturn fmt.Errorf(\"Expected a connected leafnode for server %q, got none\", s.ID())\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodesStaggeredSubPub(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tsl1, sl1Opts := runSolicitLeafServer(opts)\n\tdefer sl1.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// Create a client on the leafnode and a subscription.\n\tncl, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", sl1Opts.Host, sl1Opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\tsub, err := ncl.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to subscribe: %v\", err)\n\t}\n\n\tsl2, sl2Opts := runSolicitLeafServer(opts)\n\tdefer sl2.Shutdown()\n\n\tcheckLeafNode2Connected(t, s)\n\n\t// Create a client on the second leafnode and publish a message.\n\tncl2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", sl2Opts.Host, sl2Opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl2.Close()\n\n\tncl2.Publish(\"foo\", nil)\n\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif nmsgs, _, err := sub.Pending(); err != nil || nmsgs != 1 {\n\t\t\treturn fmt.Errorf(\"Did not receive the message: %v\", err)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestLeafNodeMultipleRemoteURLs(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tcontent := `\n\t\tport: -1\n\t\tleafnodes {\n\t\t\tremotes = [\n\t\t\t\t{\n\t\t\t\t\turls: [nats-leaf://127.0.0.1:%d,nats-leaf://localhost:%d]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t\t`\n\n\tconfig := fmt.Sprintf(content, opts.LeafNode.Port, opts.LeafNode.Port)\n\tconf := createConfFile(t, []byte(config))\n\tsl, _ := RunServerWithConfig(conf)\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n}\n\nfunc runSolicitLeafCluster(t *testing.T, clusterName string, d1, d2 *cluster) *cluster {\n\tc := &cluster{servers: make([]*server.Server, 0, 2), opts: make([]*server.Options, 0, 2), name: clusterName}\n\n\t// Who we will solicit for server 1\n\tci := rand.Intn(len(d1.opts))\n\topts := d1.opts[ci]\n\tsurl := fmt.Sprintf(\"nats-leaf://%s:%d\", opts.LeafNode.Host, opts.LeafNode.Port)\n\n\to := DefaultTestOptions\n\to.Port = -1\n\trurl, _ := url.Parse(surl)\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\to.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\to.Cluster.Name = clusterName\n\to.Cluster.Host = o.Host\n\to.Cluster.Port = -1\n\ts := RunServer(&o)\n\tcheckLeafNodeConnected(t, d1.servers[ci])\n\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, &o)\n\n\t// Grab route info\n\trouteAddr := fmt.Sprintf(\"nats-route://%s:%d\", o.Cluster.Host, o.Cluster.Port)\n\tcurl, _ := url.Parse(routeAddr)\n\n\t// Who we will solicit for server 2\n\tci = rand.Intn(len(d2.opts))\n\topts = d2.opts[ci]\n\tsurl = fmt.Sprintf(\"nats-leaf://%s:%d\", opts.LeafNode.Host, opts.LeafNode.Port)\n\n\t// This is for the case were d1 == d2 and we select the same server.\n\tplfn := d2.servers[ci].NumLeafNodes()\n\n\to2 := DefaultTestOptions\n\to2.Port = -1\n\trurl, _ = url.Parse(surl)\n\to2.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\to2.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\to2.Cluster.Name = clusterName\n\to2.Cluster.Host = o.Host\n\to2.Cluster.Port = -1\n\to2.Routes = []*url.URL{curl}\n\ts = RunServer(&o2)\n\n\tif plfn == 0 {\n\t\tcheckLeafNodeConnected(t, d2.servers[ci])\n\t} else {\n\t\tcheckLeafNode2Connected(t, d2.servers[ci])\n\t}\n\n\tc.servers = append(c.servers, s)\n\tc.opts = append(c.opts, &o2)\n\n\tcheckClusterFormed(t, c.servers...)\n\n\treturn c\n}\n\nfunc clientForCluster(t *testing.T, c *cluster) *nats.Conn {\n\tt.Helper()\n\topts := c.opts[rand.Intn(len(c.opts))]\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc TestLeafNodeCycleWithSolicited(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\t// Accepting leafnode cluster, e.g. NGS\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\tcb := createClusterWithName(t, \"B\", 3, ca)\n\tdefer shutdownCluster(cb)\n\n\t// Create the responders.\n\trequestsReceived := int32(0)\n\n\tnc := clientForCluster(t, ca)\n\tdefer nc.Close()\n\tnc.QueueSubscribe(\"request\", \"cycles\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&requestsReceived, 1)\n\t\tm.Respond([]byte(\"22\"))\n\t})\n\n\tnc = clientForCluster(t, cb)\n\tdefer nc.Close()\n\tnc.QueueSubscribe(\"request\", \"cycles\", func(m *nats.Msg) {\n\t\tatomic.AddInt32(&requestsReceived, 1)\n\t\tm.Respond([]byte(\"33\"))\n\t})\n\n\t// Soliciting cluster, both solicited connected to the \"A\" cluster\n\tsc := runSolicitLeafCluster(t, \"SC\", ca, ca)\n\tdefer shutdownCluster(sc)\n\n\tcheckInterest := func(s *server.Server, subject string) bool {\n\t\tt.Helper()\n\t\tacc, _ := s.LookupAccount(\"$G\")\n\t\treturn acc.SubscriptionInterest(subject)\n\t}\n\n\twaitForInterest := func(subject string, servers ...*server.Server) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tfor _, s := range servers {\n\t\t\t\tif !checkInterest(s, subject) {\n\t\t\t\t\treturn fmt.Errorf(\"No interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\twaitForInterest(\"request\",\n\t\tsc.servers[0], sc.servers[1],\n\t\tca.servers[0], ca.servers[1], ca.servers[2],\n\t\tcb.servers[0], cb.servers[1], cb.servers[2],\n\t)\n\n\t// Connect a client to a random server in sc\n\tcreateClientAndRequest := func(c *cluster) (*nats.Conn, *nats.Subscription) {\n\t\tnc := clientForCluster(t, c)\n\t\treply := nats.NewInbox()\n\t\tsub, err := nc.SubscribeSync(reply)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not subscribe: %v\", err)\n\t\t}\n\t\tif err := nc.PublishRequest(\"request\", reply, []byte(\"fingers crossed\")); err != nil {\n\t\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t\t}\n\t\treturn nc, sub\n\t}\n\n\tverifyOneResponse := func(sub *nats.Subscription) {\n\t\ttime.Sleep(250 * time.Millisecond)\n\t\tm, _, err := sub.Pending()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error calling Pending(): %v\", err)\n\t\t}\n\t\tif m > 1 {\n\t\t\tt.Fatalf(\"Received more then one response, cycle indicated: %d\", m)\n\t\t}\n\t}\n\n\tverifyRequestTotal := func(nre int32) {\n\t\tif nr := atomic.LoadInt32(&requestsReceived); nr != nre {\n\t\t\tt.Fatalf(\"Expected %d requests received, got %d\", nre, nr)\n\t\t}\n\t}\n\n\t// This should pass to here, but if we have a cycle things will be spinning and we will receive\n\t// too many responses when it should only be 1.\n\tnc, rsub := createClientAndRequest(sc)\n\tdefer nc.Close()\n\tverifyOneResponse(rsub)\n\tverifyRequestTotal(1)\n\n\t// Do a solicit across GW, so shut this one down.\n\tnc.Close()\n\tshutdownCluster(sc)\n\n\t// Soliciting cluster, connect to different clusters across a GW.\n\tsc = runSolicitLeafCluster(t, \"SC\", ca, cb)\n\tdefer shutdownCluster(sc)\n\n\tnc, rsub = createClientAndRequest(sc)\n\tdefer nc.Close()\n\tverifyOneResponse(rsub)\n\tverifyRequestTotal(2) // This is total since use same responders.\n}\n\nfunc TestLeafNodeNoRaceGeneratingNonce(t *testing.T) {\n\topts := testDefaultOptionsForLeafNodes()\n\topts.Cluster.Port = -1\n\topts.Cluster.Name = \"xyz\"\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tquitCh := make(chan struct{})\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\t\t\tcheckInfoMsg(t, lc)\n\t\t\tlc.Close()\n\t\t\tselect {\n\t\t\tcase <-quitCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\t\t\tcheckInfoMsg(t, rc)\n\t\t\trc.Close()\n\t\t\tselect {\n\t\t\tcase <-quitCh:\n\t\t\t\treturn\n\t\t\tcase <-time.After(time.Millisecond):\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Let this run for a bit to see if we get data race\n\ttime.Sleep(100 * time.Millisecond)\n\tclose(quitCh)\n\twg.Wait()\n}\n\nfunc runSolicitAndAcceptLeafServer(lso *server.Options) (*server.Server, *server.Options) {\n\tsurl := fmt.Sprintf(\"nats-leaf://%s:%d\", lso.LeafNode.Host, lso.LeafNode.Port)\n\to := testDefaultOptionsForLeafNodes()\n\to.Port = -1\n\trurl, _ := url.Parse(surl)\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{{URLs: []*url.URL{rurl}}}\n\to.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\treturn RunServer(o), o\n}\n\nfunc TestLeafNodeDaisyChain(t *testing.T) {\n\t// To quickly enable trace and debug logging\n\t// doLog, doTrace, doDebug = true, true, true\n\ts1, opts1 := runLeafServer()\n\tdefer s1.Shutdown()\n\n\ts2, opts2 := runSolicitAndAcceptLeafServer(opts1)\n\tdefer s2.Shutdown()\n\tcheckLeafNodeConnected(t, s1)\n\n\ts3, _ := runSolicitLeafServer(opts2)\n\tdefer s3.Shutdown()\n\tcheckLeafNodeConnections(t, s2, 2)\n\n\t// Make so we can tell the two apart since in same PID.\n\tif doLog {\n\t\ts1.SetLogger(logger.NewTestLogger(\"[S-1] - \", false), true, true)\n\t\ts2.SetLogger(logger.NewTestLogger(\"[S-2] - \", false), true, true)\n\t\ts3.SetLogger(logger.NewTestLogger(\"[S-3] - \", false), true, true)\n\t}\n\n\tnc1, err := nats.Connect(s1.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc1.Subscribe(\"ngs.usage\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\tnc1.Flush()\n\n\tcheckSubInterest(t, s3, \"$G\", \"ngs.usage\", time.Second)\n\n\tnc2, err := nats.Connect(s3.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tif _, err = nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n}\n\n// This will test failover to a server with a cert with only an IP after successfully connecting\n// to a server with a cert with both.\nfunc TestClusterTLSMixedIPAndDNS(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"./configs/certs/server-iponly.pem\"\n\t\t\t\tkey_file: \"./configs/certs/server-key-iponly.pem\"\n\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tname: xyz\n\t\t}\n\t`))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tbConfigTemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\ttls {\n\t\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\t\ttimeout: 2\n\t\t\t}\n\t\t}\n\t\tcluster {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tname: xyz\n\t\t\troutes [\n\t\t\t\t\"nats://%s:%d\"\n\t\t\t]\n\t\t}\n\t`\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate,\n\t\toptsA.Cluster.Host, optsA.Cluster.Port)))\n\tsrvB, optsB := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Solicit a leafnode server here. Don't use the helper since we need verification etc.\n\to := DefaultTestOptions\n\to.Port = -1\n\trurl, _ := url.Parse(fmt.Sprintf(\"nats-leaf://%s:%d\", optsB.LeafNode.Host, optsB.LeafNode.Port))\n\to.LeafNode.ReconnectInterval = 10 * time.Millisecond\n\tremote := &server.RemoteLeafOpts{URLs: []*url.URL{rurl}}\n\tremote.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12}\n\tpool := x509.NewCertPool()\n\trootPEM, err := os.ReadFile(\"./configs/certs/ca.pem\")\n\tif err != nil || rootPEM == nil {\n\t\tt.Fatalf(\"Error loading or parsing rootCA file: %v\", err)\n\t}\n\tok := pool.AppendCertsFromPEM(rootPEM)\n\tif !ok {\n\t\tt.Fatalf(\"Failed to parse root certificate from %q\", \"./configs/certs/ca.pem\")\n\t}\n\tremote.TLSConfig.RootCAs = pool\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{remote}\n\tsl, _ := RunServer(&o), &o\n\tdefer sl.Shutdown()\n\n\tcheckLeafNodeConnected(t, srvB)\n\n\t// Now kill off srvB and force client to connect to srvA.\n\tsrvB.Shutdown()\n\n\t// Make sure this works.\n\tcheckLeafNodeConnected(t, srvA)\n}\n\n// This will test for a bug in stream export/import with leafnodes.\n// https://github.com/nats-io/nats-server/issues/1332\nfunc TestStreamExportWithMultipleAccounts(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tbConfigTemplate := `\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tremotes = [\n\t\t\t{\n\t\t\t\turl:\"nats://127.0.0.1:%d\"\n\t\t\t\taccount:\"EXTERNAL\"\n\t\t\t}\n\t\t\t]\n\t\t}\n\t\taccounts: {\n\t\t INTERNAL: {\n\t\t users: [\n\t\t {user: good, password: pwd}\n\t\t ]\n\t\t\t imports: [\n\t\t {\n\t\t stream: { account: EXTERNAL, subject: \"foo\"}, prefix: \"bar\"\n\t\t }\n\t\t ]\n\t\t },\n\t\t EXTERNAL: {\n\t\t users: [\n\t\t {user: bad, password: pwd}\n\t\t ]\n\t\t exports: [{stream: \"foo\"}]\n\t\t },\n\t\t}\n\t`\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port)))\n\tsrvB, optsB := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://good:pwd@%s:%d\", optsB.Host, optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\twcsub, err := nc.SubscribeSync(\">\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer wcsub.Unsubscribe()\n\tnc.Flush()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Publish(\"foo\", nil)\n\n\tmsg, err := wcsub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Did not receive the message: %v\", err)\n\t}\n\tif msg.Subject != \"bar.foo\" {\n\t\tt.Fatalf(\"Received on wrong subject: %q\", msg.Subject)\n\t}\n}\n\n// This will test for a bug in service export/import with leafnodes.\n// https://github.com/nats-io/nats-server/issues/1336\nfunc TestServiceExportWithMultipleAccounts(t *testing.T) {\n\tconfA := createConfFile(t, []byte(`\n\t\tserver_name: A\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t}\n\t`))\n\n\tsrvA, optsA := RunServerWithConfig(confA)\n\tdefer srvA.Shutdown()\n\n\tbConfigTemplate := `\n\t\tserver_name: B\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tremotes = [\n\t\t\t{\n\t\t\t\turl:\"nats://127.0.0.1:%d\"\n\t\t\t\taccount:\"EXTERNAL\"\n\t\t\t}\n\t\t\t]\n\t\t}\n\t\taccounts: {\n\t\t\tINTERNAL: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: good, password: pwd}\n\t\t\t\t]\n\t\t\t\timports: [\n\t\t\t\t\t{\n\t\t\t\t\t\tservice: {\n\t\t\t\t\t\t\taccount: EXTERNAL\n\t\t\t\t\t\t\tsubject: \"foo\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t},\n\t\t\tEXTERNAL: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: evil, password: pwd}\n\t\t\t\t]\n\t\t\t\texports: [{service: \"foo\"}]\n\t\t\t},\n\t\t}\n\t`\n\n\tconfB := createConfFile(t, []byte(fmt.Sprintf(bConfigTemplate, optsA.LeafNode.Port)))\n\n\tsrvB, optsB := RunServerWithConfig(confB)\n\tdefer srvB.Shutdown()\n\n\tcheckLeafNodeConnected(t, srvB)\n\n\t// connect to confA, and offer a service\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%d\", optsA.Host, optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tif err := msg.Respond([]byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"Error on respond: %v\", err)\n\t\t}\n\t})\n\tnc2.Flush()\n\n\tcheckSubInterest(t, srvB, \"INTERNAL\", \"foo\", time.Second)\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://good:pwd@%s:%d\", optsB.Host, optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(\"foo\", []byte(\"hello\"), 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp == nil || strings.Compare(\"world\", string(resp.Data)) != 0 {\n\t\tt.Fatal(\"Did not receive the correct message\")\n\t}\n}\n\n// This will test for a bug in service export/import with leafnode restart.\n// https://github.com/nats-io/nats-server/issues/1344\nfunc TestServiceExportWithLeafnodeRestart(t *testing.T) {\n\tconfG := createConfFile(t, []byte(`\n\t\tserver_name: G\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tauthorization {\n\t\t\t\taccount:\"EXTERNAL\"\n\t\t\t\tuser: ln\n\t\t\t\tpassword: pass\n\t\t\t}\n\t\t}\n\n\t\taccounts: {\n\t\t INTERNAL: {\n\t\t users: [\n\t\t {user: good, password: pwd}\n\t\t ]\n\t\t exports: [{service: \"foo\", response: singleton}]\n\t\t\t imports: [\n\t\t {\n\t\t service: {\n\t\t account: EXTERNAL\n\t\t subject: \"evilfoo\"\n\t\t }, to: from_evilfoo\n\t\t }\n\t\t ]\n\t\t },\n\t\t EXTERNAL: {\n\t\t users: [\n\t\t {user: evil, password: pwd}\n\t\t ]\n\t\t exports: [{service: \"evilfoo\", response: singleton}]\n\t\t\t imports: [\n\t\t {\n\t\t service: {\n\t\t account: INTERNAL\n\t\t subject: \"foo\"\n\t\t }, to: goodfoo\n\t\t }\n\t\t ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrvG, optsG := RunServerWithConfig(confG)\n\tdefer srvG.Shutdown()\n\n\teConfigTemplate := `\n\t\tserver_name: E\n\t\tlisten: 127.0.0.1:-1\n\t\tleafnodes {\n\t\t\tlisten: \"127.0.0.1:-1\"\n\t\t\tremotes = [\n\t\t\t{\n\t\t\t\turl:\"nats://ln:pass@127.0.0.1:%d\"\n\t\t\t\taccount:\"EXTERNAL_GOOD\"\n\t\t\t}\n\t\t\t]\n\t\t}\n\n\t\taccounts: {\n\t\t INTERNAL_EVILINC: {\n\t\t users: [\n\t\t {user: evil, password: pwd}\n\t\t ]\n\t\t exports: [{service: \"foo\", response: singleton}]\n\t\t\t imports: [\n\t\t {\n\t\t service: {\n\t\t account: EXTERNAL_GOOD\n\t\t subject: \"goodfoo\"\n\t\t }, to: from_goodfoo\n\t\t }\n\t\t ]\n\t\t },\n\t\t EXTERNAL_GOOD: {\n\t\t users: [\n\t\t {user: good, password: pwd}\n\t\t ]\n\t\t exports: [{service: \"goodfoo\", response: singleton}]\n\t\t\t imports: [\n\t\t {\n\t\t service: {\n\t\t account: INTERNAL_EVILINC\n\t\t subject: \"foo\"\n\t\t }, to: evilfoo\n\t\t }\n\t\t ]\n\t\t },\n\t\t}\n\t`\n\n\tconfE := createConfFile(t, []byte(fmt.Sprintf(eConfigTemplate, optsG.LeafNode.Port)))\n\n\tsrvE, optsE := RunServerWithConfig(confE)\n\tdefer srvE.Shutdown()\n\n\t// connect to confE, and offer a service\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://evil:pwd@%s:%d\", optsE.Host, optsE.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tif err := msg.Respond([]byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"Error on respond: %v\", err)\n\t\t}\n\t})\n\tnc2.Flush()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://good:pwd@%s:%d\", optsG.Host, optsG.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tresp, err := nc.Request(\"from_evilfoo\", []byte(\"hello\"), 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp == nil || strings.Compare(\"world\", string(resp.Data)) != 0 {\n\t\tt.Fatal(\"Did not receive the correct message\")\n\t}\n\n\t// Now restart server E and requestor and replier.\n\tnc.Close()\n\tnc2.Close()\n\tsrvE.Shutdown()\n\n\tsrvE, optsE = RunServerWithConfig(confE)\n\tdefer srvE.Shutdown()\n\n\tnc2, err = nats.Connect(fmt.Sprintf(\"nats://evil:pwd@%s:%d\", optsE.Host, optsE.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tif err := msg.Respond([]byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"Error on respond: %v\", err)\n\t\t}\n\t})\n\tnc2.Flush()\n\n\tnc, err = nats.Connect(fmt.Sprintf(\"nats://good:pwd@%s:%d\", optsG.Host, optsG.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tresp, err = nc.Request(\"from_evilfoo\", []byte(\"hello\"), 2*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tif resp == nil || strings.Compare(\"world\", string(resp.Data)) != 0 {\n\t\tt.Fatal(\"Did not receive the correct message\")\n\t}\n}\n\nfunc TestLeafNodeQueueSubscriberUnsubscribe(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\t// Create a client on leaf server and create a queue sub\n\tc1 := createClientConn(t, opts.Host, opts.Port)\n\tdefer c1.Close()\n\n\tsend1, expect1 := setupConn(t, c1)\n\tsend1(\"SUB foo bar 1\\r\\nPING\\r\\n\")\n\texpect1(pongRe)\n\n\t// Leaf should receive an LS+ foo bar 1\n\tleafExpect(lsubRe)\n\n\t// Create a second client on leaf server and create queue sub on same group.\n\tc2 := createClientConn(t, opts.Host, opts.Port)\n\tdefer c2.Close()\n\n\tsend2, expect2 := setupConn(t, c2)\n\tsend2(\"SUB foo bar 1\\r\\nPING\\r\\n\")\n\texpect2(pongRe)\n\n\t// Leaf should receive an LS+ foo bar 2\n\tleafExpect(lsubRe)\n\n\t// Now close c1\n\tc1.Close()\n\n\t// Leaf should receive an indication that the queue group went to 1.\n\t// Which means LS+ foo bar 1.\n\tbuf := leafExpect(lsubRe)\n\tif matches := lsubRe.FindAllSubmatch(buf, -1); len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 LS+, got %v\", len(matches))\n\t}\n\t// Make sure that we did not get a LS- at the same time.\n\tif bytes.Contains(buf, []byte(\"LS-\")) {\n\t\tt.Fatalf(\"Unexpected LS- in response: %q\", buf)\n\t}\n\t// Make sure we receive nothing...\n\texpectNothing(t, lc)\n}\n\nfunc TestLeafNodeOriginClusterSingleHub(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tc1 := `\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1 }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, opts.LeafNode.Port)))\n\n\tln1, lopts1 := RunServerWithConfig(lconf1)\n\tdefer ln1.Shutdown()\n\n\tc2 := `\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, opts.LeafNode.Port)))\n\n\tln2, _ := RunServerWithConfig(lconf2)\n\tdefer ln2.Shutdown()\n\n\tln3, _ := RunServerWithConfig(lconf2)\n\tdefer ln3.Shutdown()\n\n\tcheckClusterFormed(t, ln1, ln2, ln3)\n\tcheckLeafNodeConnections(t, s, 3)\n\n\t// So now we are setup with 3 solicited leafnodes all connected to a hub.\n\t// We will create two clients, one on each leafnode server.\n\tnc1, err := nats.Connect(ln1.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(ln2.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckInterest := func(s *server.Server, subject string) bool {\n\t\tt.Helper()\n\t\tacc, _ := s.LookupAccount(\"$G\")\n\t\treturn acc.SubscriptionInterest(subject)\n\t}\n\n\twaitForInterest := func(subject string, servers ...*server.Server) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tfor _, s := range servers {\n\t\t\t\tif !checkInterest(s, subject) {\n\t\t\t\t\treturn fmt.Errorf(\"No interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsubj := \"foo.bar\"\n\n\tsub, _ := nc2.SubscribeSync(subj)\n\twaitForInterest(subj, ln1, ln2, ln3, s)\n\n\t// Make sure we truncated the subscription bouncing through the hub and back to other leafnodes.\n\tfor _, s := range []*server.Server{ln1, ln3} {\n\t\tacc, _ := s.LookupAccount(\"$G\")\n\t\tif nms := acc.Interest(subj); nms != 1 {\n\t\t\tt.Fatalf(\"Expected only one active subscription, got %d\", nms)\n\t\t}\n\t}\n\n\t// Send a message.\n\tnc1.Publish(subj, nil)\n\tnc1.Flush()\n\t// Wait to propagate\n\ttime.Sleep(25 * time.Millisecond)\n\n\t// Make sure we only get it once.\n\tif n, _, _ := sub.Pending(); n != 1 {\n\t\tt.Fatalf(\"Expected only one message, got %d\", n)\n\t}\n}\n\nfunc TestLeafNodeOriginCluster(t *testing.T) {\n\tca := createClusterWithName(t, \"A\", 3)\n\tdefer shutdownCluster(ca)\n\n\tc1 := `\n\tserver_name: L1\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1 }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf1 := createConfFile(t, []byte(fmt.Sprintf(c1, ca.opts[0].LeafNode.Port)))\n\n\tln1, lopts1 := RunServerWithConfig(lconf1)\n\tdefer ln1.Shutdown()\n\n\tc2 := `\n\tserver_name: L2\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf2 := createConfFile(t, []byte(fmt.Sprintf(c2, lopts1.Cluster.Port, ca.opts[1].LeafNode.Port)))\n\n\tln2, _ := RunServerWithConfig(lconf2)\n\tdefer ln2.Shutdown()\n\n\tc3 := `\n\tserver_name: L3\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln22, listen: 127.0.0.1:-1, routes = [ nats-route://127.0.0.1:%d] }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\t`\n\tlconf3 := createConfFile(t, []byte(fmt.Sprintf(c3, lopts1.Cluster.Port, ca.opts[2].LeafNode.Port)))\n\n\tln3, _ := RunServerWithConfig(lconf3)\n\tdefer ln3.Shutdown()\n\n\tcheckClusterFormed(t, ln1, ln2, ln3)\n\tcheckLeafNodeConnections(t, ca.servers[0], 1)\n\tcheckLeafNodeConnections(t, ca.servers[1], 1)\n\tcheckLeafNodeConnections(t, ca.servers[2], 1)\n\n\t// So now we are setup with 3 solicited leafnodes connected to different servers in the hub cluster.\n\t// We will create two clients, one on each leafnode server.\n\tnc1, err := nats.Connect(ln1.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(ln2.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tcheckInterest := func(s *server.Server, subject string) bool {\n\t\tt.Helper()\n\t\tacc, _ := s.LookupAccount(\"$G\")\n\t\treturn acc.SubscriptionInterest(subject)\n\t}\n\n\twaitForInterest := func(subject string, servers ...*server.Server) {\n\t\tt.Helper()\n\t\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\t\tfor _, s := range servers {\n\t\t\t\tif !checkInterest(s, subject) {\n\t\t\t\t\treturn fmt.Errorf(\"No interest\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tsubj := \"foo.bar\"\n\n\tsub, _ := nc2.SubscribeSync(subj)\n\twaitForInterest(subj, ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2])\n\n\t// Make sure we truncated the subscription bouncing through the hub and back to other leafnodes.\n\tfor _, s := range []*server.Server{ln1, ln3} {\n\t\tacc, _ := s.LookupAccount(\"$G\")\n\t\tif nms := acc.Interest(subj); nms != 1 {\n\t\t\tt.Fatalf(\"Expected only one active subscription, got %d\", nms)\n\t\t}\n\t}\n\n\t// Send a message.\n\tnc1.Publish(subj, nil)\n\tnc1.Flush()\n\t// Wait to propagate\n\ttime.Sleep(25 * time.Millisecond)\n\n\t// Make sure we only get it once.\n\tif n, _, _ := sub.Pending(); n != 1 {\n\t\tt.Fatalf(\"Expected only one message, got %d\", n)\n\t}\n\t// eat the msg\n\tsub.NextMsg(time.Second)\n\n\t// Now create interest on the hub side. This will draw the message from a leafnode\n\t// to the hub. We want to make sure that message does not bounce back to other leafnodes.\n\tnc3, err := nats.Connect(ca.servers[0].ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc3.Close()\n\n\twcSubj := \"foo.*\"\n\twcsub, _ := nc3.SubscribeSync(wcSubj)\n\t// This is a placeholder that we can use to check all interest has propagated.\n\tnc3.SubscribeSync(\"bar\")\n\twaitForInterest(\"bar\", ln1, ln2, ln3, ca.servers[0], ca.servers[1], ca.servers[2])\n\n\t// Send another message.\n\tm := nats.NewMsg(subj)\n\tm.Header.Add(\"Accept-Encoding\", \"json\")\n\tm.Header.Add(\"Authorization\", \"s3cr3t\")\n\tm.Data = []byte(\"Hello Headers!\")\n\n\tnc1.PublishMsg(m)\n\tnc1.Flush()\n\t// Wait to propagate\n\ttime.Sleep(25 * time.Millisecond)\n\n\t// Make sure we only get it once.\n\tif n, _, _ := sub.Pending(); n != 1 {\n\t\tt.Fatalf(\"Expected only one message, got %d\", n)\n\t}\n\t// Also for wc\n\tif n, _, _ := wcsub.Pending(); n != 1 {\n\t\tt.Fatalf(\"Expected only one message, got %d\", n)\n\t}\n\n\t// grab the msg\n\tmsg, _ := sub.NextMsg(time.Second)\n\tif !bytes.Equal(m.Data, msg.Data) {\n\t\tt.Fatalf(\"Expected the payloads to match, wanted %q, got %q\", m.Data, msg.Data)\n\t}\n\tif len(msg.Header) != 2 {\n\t\tt.Fatalf(\"Expected 2 header entries, got %d\", len(msg.Header))\n\t}\n\tif msg.Header.Get(\"Authorization\") != \"s3cr3t\" {\n\t\tt.Fatalf(\"Expected auth header to match, wanted %q, got %q\", \"s3cr3t\", msg.Header.Get(\"Authorization\"))\n\t}\n}\n\nfunc TestLeafNodeAdvertiseInCluster(t *testing.T) {\n\to1 := testDefaultOptionsForLeafNodes()\n\to1.Cluster.Name = \"abc\"\n\to1.Cluster.Host = \"127.0.0.1\"\n\to1.Cluster.Port = -1\n\ts1 := runGatewayServer(o1)\n\tdefer s1.Shutdown()\n\n\tlc := createLeafConn(t, o1.LeafNode.Host, o1.LeafNode.Port)\n\tdefer lc.Close()\n\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\tleafSend(\"PING\\r\\n\")\n\tleafExpect(pongRe)\n\n\to2 := testDefaultOptionsForLeafNodes()\n\to2.Cluster.Name = \"abc\"\n\to2.Cluster.Host = \"127.0.0.1\"\n\to2.Cluster.Port = -1\n\to2.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to2.LeafNode.Advertise = \"srvB:7222\"\n\ts2 := runGatewayServer(o2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tbuf := leafExpect(infoRe)\n\tsi := &server.Info{}\n\tjson.Unmarshal(buf[5:], si)\n\tvar ok bool\n\tfor _, u := range si.LeafNodeURLs {\n\t\tif u == \"srvB:7222\" {\n\t\t\tok = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif !ok {\n\t\tt.Fatalf(\"Url srvB:7222 was not found: %q\", si.GatewayURLs)\n\t}\n\n\to3 := testDefaultOptionsForLeafNodes()\n\to3.Cluster.Name = \"abc\"\n\to3.Cluster.Host = \"127.0.0.1\"\n\to3.Cluster.Port = -1\n\to3.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://127.0.0.1:%d\", o1.Cluster.Port))\n\to3.LeafNode.Advertise = \"srvB:7222\"\n\ts3 := runGatewayServer(o3)\n\tdefer s3.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2, s3)\n\n\t// Since it is the save srvB:7222 url, we should not get an update.\n\texpectNothing(t, lc)\n\n\t// Now shutdown s2 and make sure that we are not getting an update\n\t// with srvB:7222 missing.\n\ts2.Shutdown()\n\texpectNothing(t, lc)\n}\n\nfunc TestLeafNodeAndGatewaysStreamAndShadowSubs(t *testing.T) {\n\tserver.SetGatewaysSolicitDelay(10 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\tconf1 := createConfFile(t, []byte(`\n\t\tport: -1\n\t\tsystem_account: SYS\n\t\taccounts {\n\t\t\tSYS {}\n\t\t\tA: {\n\t\t\t\tusers: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }]\n\t\t\t\texports: [{ stream: A.b.>, accounts: [B] }]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tusers: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}]\n\t\t\t\timports: [{ stream: { account: A, subject: A.b.> } }]\n\t\t\t}\n\t\t}\n\t\tgateway {\n\t\t\tname: \"A\"\n\t\t\tport: -1\n\t\t}\n\t\tleafnodes {\n\t\t\tport: -1\n\t\t\tauthorization: {\n\t\t\t\tusers: [\n\t\t\t\t\t{user: a, password: pwd, account: A}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t`))\n\ts1, o1 := RunServerWithConfig(conf1)\n\tdefer s1.Shutdown()\n\n\tconf2 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tsystem_account: SYS\n\t\taccounts {\n\t\t\tSYS {}\n\t\t\tA: {\n\t\t\t\tusers: [{ user: a, password: pwd, permissions: {publish: [A.b.>]} }]\n\t\t\t\texports: [{ stream: A.b.>, accounts: [B] }]\n\t\t\t},\n\t\t\tB: {\n\t\t\t\tusers: [{ user: b, password: pwd, permissions: {subscribe: [ A.b.> ]}}]\n\t\t\t\timports: [{ stream: { account: A, subject: A.b.> } }]\n\t\t\t}\n\t\t}\n\t\tgateway {\n\t\t\tname: \"B\"\n\t\t\tport: -1\n\t\t\tgateways [\n\t\t\t\t{\n\t\t\t\t\tname: \"A\"\n\t\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.Gateway.Port)))\n\ts2, o2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\twaitForOutboundGateways(t, s1, 1, 2*time.Second)\n\twaitForOutboundGateways(t, s2, 1, 2*time.Second)\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@127.0.0.1:%d\", o2.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsub, err := nc.SubscribeSync(\"A.b.>\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscibe: %v\", err)\n\t}\n\tdefer sub.Unsubscribe()\n\n\tconf3 := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tport: -1\n\t\tsystem_account: SYS\n\t\taccounts: {\n\t\t\tSYS {}\n\t\t\tC: {\n\t\t\t\timports: [{ stream: { account: D, subject: b.> }, prefix: A }]\n\t\t\t}\n\t\t\tD: {\n\t\t\t\tusers: [{ user: d, password: pwd, permissions: {publish: [ b.> ]} }]\n\t\t\t\texports: [{ stream: b.>, accounts: [C] }]\n\t\t\t}\n\t\t}\n\t\tleafnodes {\n\t\t\tremotes [\n\t\t\t\t{\n\t\t\t\t\turl: \"nats://a:pwd@127.0.0.1:%d\"\n\t\t\t\t\taccount: C\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t`, o1.LeafNode.Port)))\n\ts3, o3 := RunServerWithConfig(conf3)\n\tdefer s3.Shutdown()\n\n\tcheckLeafNodeConnected(t, s1)\n\tcheckLeafNodeConnected(t, s3)\n\n\tncl, err := nats.Connect(fmt.Sprintf(\"nats://d:pwd@127.0.0.1:%d\", o3.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t}\n\tdefer ncl.Close()\n\n\tncl.Publish(\"b.c\", []byte(\"test\"))\n\tif _, err := sub.NextMsg(time.Second); err != nil {\n\t\tt.Fatalf(\"Did not receive message: %v\", err)\n\t}\n}\n\nfunc TestLeafnodeHeaders(t *testing.T) {\n\tsrv, opts := runLeafServer()\n\tdefer srv.Shutdown()\n\tleaf, _ := runSolicitLeafServer(opts)\n\tdefer leaf.Shutdown()\n\n\tcheckLeafNodeConnected(t, srv)\n\tcheckLeafNodeConnected(t, leaf)\n\n\tsnc, err := nats.Connect(srv.ClientURL())\n\tif err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\tdefer snc.Close()\n\n\tlnc, err := nats.Connect(leaf.ClientURL())\n\tif err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\tdefer lnc.Close()\n\n\t// Start with subscription on leaf so that we check that srv has the interest\n\t// (since we are going to publish from srv)\n\tlsub, err := lnc.SubscribeSync(\"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"subscribe failed: %s\", err)\n\t}\n\tlnc.Flush()\n\tcheckSubInterest(t, srv, \"$G\", \"test\", time.Second)\n\n\tssub, err := snc.SubscribeSync(\"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"subscribe failed: %s\", err)\n\t}\n\n\tmsg := nats.NewMsg(\"test\")\n\tmsg.Header.Add(\"Test\", \"Header\")\n\tif len(msg.Header) == 0 {\n\t\tt.Fatalf(\"msg header is empty\")\n\t}\n\terr = snc.PublishMsg(msg)\n\tif err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tsmsg, err := ssub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"next failed: %s\", err)\n\t}\n\tif len(smsg.Header) == 0 {\n\t\tt.Fatalf(\"server msgs header is empty\")\n\t}\n\n\tlmsg, err := lsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"next failed: %s\", err)\n\t}\n\tif len(lmsg.Header) == 0 {\n\t\tt.Fatalf(\"leaf msg header is empty\")\n\t}\n}\n\nfunc TestLeafNodeClusterNameWithSpacesRejected(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tcheckInfoMsg(t, lc)\n\tsendProto(t, lc, \"CONNECT {\\\"cluster\\\":\\\"my cluster\\\"}\\r\\n\")\n\texpect := expectCommand(t, lc)\n\texpect(errRe)\n\texpectDisconnect(t, lc)\n\n\tlc = createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\tleafSend, leafExpect := setupLeaf(t, lc, 1)\n\t// The accept side does expect an INFO from an handshake,\n\t// so it will \"ignore\" the first one.\n\tleafSend(\"INFO {\\\"server\\\":\\\"server\\\"}\\r\\n\")\n\tleafSend(\"INFO {\\\"cluster\\\":\\\"my cluster\\\"}\\r\\n\")\n\tleafExpect(errRe)\n\texpectDisconnect(t, lc)\n}\n\nfunc TestLeafNodeConnectInfo(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tsys string\n\t\thasSys bool\n\t}{\n\t\t{\"with explicit system account\", \"system_account: SYS\", true},\n\t\t{\"without explicit system account\", \"\", false},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\t\t\t\taccounts {\n\t\t\t\t\tSYS: { users: [{ user: sys, password: pwd}] }\n\t\t\t\t\tA: { users: [{ user: a, password: pwd}] }\n\t\t\t\t\tB: { users: [{ user: b, password: pwd}] }\n\t\t\t\t}\n\t\t\t\tleafnodes {\n\t\t\t\t\tport: -1\n\t\t\t\t}\n\t\t\t`, test.sys)))\n\t\t\thub, oHub := RunServerWithConfig(conf)\n\t\t\tdefer hub.Shutdown()\n\n\t\t\tcheckInfoOnConnect := func(user, acc string, isSys bool) {\n\t\t\t\tt.Helper()\n\t\t\t\tlc := createLeafConn(t, oHub.LeafNode.Host, oHub.LeafNode.Port)\n\t\t\t\tdefer lc.Close()\n\n\t\t\t\tcheckInfoMsg(t, lc)\n\n\t\t\t\tsendProto(t, lc, fmt.Sprintf(\"CONNECT {\\\"user\\\":%q,\\\"pass\\\":\\\"pwd\\\"}\\r\\n\", user))\n\t\t\t\tinfo := checkInfoMsg(t, lc)\n\t\t\t\tif !info.ConnectInfo {\n\t\t\t\t\tt.Fatal(\"Expected ConnectInfo to be true\")\n\t\t\t\t}\n\t\t\t\tif an := info.RemoteAccount; an != acc {\n\t\t\t\t\tt.Fatalf(\"Expected account %q, got %q\", acc, info.RemoteAccount)\n\t\t\t\t}\n\t\t\t\tif ais := info.IsSystemAccount; ais != isSys {\n\t\t\t\t\tt.Fatalf(\"Expected IsSystemAccount to be %v, got %v\", isSys, ais)\n\t\t\t\t}\n\t\t\t\tcheckLeafNodeConnected(t, hub)\n\t\t\t}\n\t\t\tcheckInfoOnConnect(\"a\", \"A\", false)\n\t\t\tcheckInfoOnConnect(\"sys\", \"SYS\", test.hasSys)\n\t\t\tcheckInfoOnConnect(\"b\", \"B\", false)\n\t\t})\n\t}\n}\n" }, { "path": "test/log_test.go", "content": "package test\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc RunServerWithLogging(opts *server.Options) *server.Server {\n\tif opts == nil {\n\t\topts = &DefaultTestOptions\n\t}\n\topts.NoLog = false\n\topts.Cluster.PoolSize = -1\n\topts.Cluster.Compression.Mode = server.CompressionOff\n\topts.LeafNode.Compression.Mode = server.CompressionOff\n\ts, err := server.NewServer(opts)\n\tif err != nil || s == nil {\n\t\tpanic(fmt.Sprintf(\"No NATS Server object returned: %v\", err))\n\t}\n\ts.ConfigureLogger()\n\tgo s.Start()\n\tif !s.ReadyForConnections(10 * time.Second) {\n\t\tpanic(\"Unable to start NATS Server in Go Routine\")\n\t}\n\treturn s\n}\n\nfunc TestLogMaxArchives(t *testing.T) {\n\t// With logfile_size_limit set to small 100 characters, plain startup rotates 8 times\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\ttotEntriesExpected int\n\t}{\n\t\t{\n\t\t\t\"Default implicit, no max logs, expect 0 purged logs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlogfile_size_limit: 100\n\t\t\t`,\n\t\t\t9,\n\t\t},\n\t\t{\n\t\t\t\"Default explicit, no max logs, expect 0 purged logs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlogfile_size_limit: 100\n\t\t\t\tlogfile_max_num: 0\n\t\t\t`,\n\t\t\t9,\n\t\t},\n\t\t{\n\t\t\t\"Default explicit - negative val, no max logs, expect 0 purged logs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlogfile_size_limit: 100\n\t\t\t\tlogfile_max_num: -42\n\t\t\t`,\n\t\t\t9,\n\t\t},\n\t\t{\n\t\t\t\"1-max num, expect 8 purged logs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlogfile_size_limit: 100\n\t\t\t\tlogfile_max_num: 1\n\t\t\t`,\n\t\t\t1,\n\t\t},\n\t\t{\n\t\t\t\"5-max num, expect 4 purged logs; use opt alias\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlog_size_limit: 100\n\t\t\t\tlog_max_num: 5\n\t\t\t`,\n\t\t\t5,\n\t\t},\n\t\t{\n\t\t\t\"100-max num, expect 0 purged logs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tlog_file: %s\n\t\t\t\tlogfile_size_limit: 100\n\t\t\t\tlogfile_max_num: 100\n\t\t\t`,\n\t\t\t9,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\td, err := os.MkdirTemp(\"\", \"logtest\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error creating temp dir: %v\", err)\n\t\t\t}\n\t\t\tcontent := fmt.Sprintf(test.config, filepath.Join(d, \"nats-server.log\"))\n\t\t\t// server config does not like plain windows backslash\n\t\t\tif runtime.GOOS == \"windows\" {\n\t\t\t\tcontent = filepath.ToSlash(content)\n\t\t\t}\n\t\t\topts, err := server.ProcessConfigFile(createConfFile(t, []byte(content)))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t\t\t}\n\t\t\ts := RunServerWithLogging(opts)\n\t\t\tif s == nil {\n\t\t\t\tt.Fatalf(\"No NATS Server object returned\")\n\t\t\t}\n\t\t\ts.Shutdown()\n\t\t\t// Windows filesystem can be a little pokey on the flush, so wait a bit after shutdown...\n\t\t\ttime.Sleep(500 * time.Millisecond)\n\t\t\tentries, err := os.ReadDir(d)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error reading dir: %v\", err)\n\t\t\t}\n\t\t\tif len(entries) != test.totEntriesExpected {\n\t\t\t\tt.Fatalf(\"Expected %d log files, got %d\", test.totEntriesExpected, len(entries))\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "test/maxpayload_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nfunc TestMaxPayload(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/override.conf\")\n\tdefer srv.Shutdown()\n\n\tendpoint := net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port))\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s/\", endpoint))\n\tif err != nil {\n\t\tt.Fatalf(\"Could not connect to server: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsize := 4 * 1024 * 1024\n\tbig := sizedBytes(size)\n\terr = nc.Publish(\"foo\", big)\n\n\tif err != nats.ErrMaxPayload {\n\t\tt.Fatalf(\"Expected a Max Payload error\")\n\t}\n\n\tconn, err := net.DialTimeout(\"tcp\", endpoint, nc.Opts.Timeout)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not make a raw connection to the server: %v\", err)\n\t}\n\tdefer conn.Close()\n\tinfo := make([]byte, 512)\n\t_, err = conn.Read(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected an info message to be sent by the server: %s\", err)\n\t}\n\tpub := fmt.Sprintf(\"PUB bar %d\\r\\n\", size)\n\t_, err = conn.Write([]byte(pub))\n\tif err != nil {\n\t\tt.Fatalf(\"Could not publish event to the server: %s\", err)\n\t}\n\n\terrMsg := make([]byte, 35)\n\t_, err = conn.Read(errMsg)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected an error message to be sent by the server: %s\", err)\n\t}\n\n\tif !strings.Contains(string(errMsg), \"Maximum Payload Violation\") {\n\t\tt.Errorf(\"Received wrong error message (%v)\\n\", string(errMsg))\n\t}\n\n\t// Client proactively omits sending the message so server\n\t// does not close the connection.\n\tif nc.IsClosed() {\n\t\tt.Errorf(\"Expected connection to not be closed.\")\n\t}\n\n\t// On the other hand client which did not proactively omitted\n\t// publishing the bytes following what is suggested by server\n\t// in the info message has its connection closed.\n\t_, err = conn.Write(big)\n\tif err == nil && runtime.GOOS != \"windows\" {\n\t\tt.Errorf(\"Expected error due to maximum payload transgression.\")\n\t}\n\n\t// On windows, the previous write will not fail because the connection\n\t// is not fully closed at this stage.\n\tif runtime.GOOS == \"windows\" {\n\t\t// Issuing a PING and not expecting the PONG.\n\t\t_, err = conn.Write([]byte(\"PING\\r\\n\"))\n\t\tif err == nil {\n\t\t\tconn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))\n\t\t\t_, err = conn.Read(big)\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"Expected closed connection due to maximum payload transgression.\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMaxPayloadOverrun(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\topts.MaxPayload = 10000\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\t// Overrun a int32\n\tc := createClientConn(t, \"127.0.0.1\", opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PUB foo 199380988\\r\\n\")\n\texpect(errRe)\n\n\t// Now overrun an int64, parseSize will have returned -1,\n\t// so we get disconnected.\n\tc = createClientConn(t, \"127.0.0.1\", opts.Port)\n\tdefer c.Close()\n\n\tsend, _ = setupConn(t, c)\n\tsend(\"PUB foo 18446744073709551615123\\r\\n\")\n\texpectDisconnect(t, c)\n}\n\nfunc TestAsyncInfoWithSmallerMaxPayload(t *testing.T) {\n\ts, opts := runOperatorServer(t)\n\tdefer s.Shutdown()\n\n\tconst testMaxPayload = 522\n\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tnac.Limits.Payload = testMaxPayload\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\tif err := s.AccountResolver().Store(apub, ajwt); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tnc.Flush()\n\tdefer nc.Close()\n\n\tif mp := nc.MaxPayload(); mp != testMaxPayload {\n\t\tt.Fatalf(\"Expected MaxPayload of %d, got %d\", testMaxPayload, mp)\n\t}\n}\n" }, { "path": "test/monitor_test.go", "content": "// Copyright 2012-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nconst CLIENT_PORT = 11422\nconst MONITOR_PORT = 11522\n\nfunc runMonitorServer() *server.Server {\n\tresetPreviousHTTPConnections()\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPPort = MONITOR_PORT\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.NoSystemAccount = true\n\treturn RunServer(&opts)\n}\n\n// Runs a clustered pair of monitor servers for testing the /routez endpoint\nfunc runMonitorServerClusteredPair(t *testing.T) (*server.Server, *server.Server) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPPort = MONITOR_PORT\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.Cluster = server.ClusterOpts{Name: \"M\", Host: \"127.0.0.1\", Port: 10223}\n\topts.Routes = server.RoutesFromStr(\"nats-route://127.0.0.1:10222\")\n\topts.NoSystemAccount = true\n\n\ts1 := RunServer(&opts)\n\n\topts2 := DefaultTestOptions\n\topts2.Port = CLIENT_PORT + 1\n\topts2.HTTPPort = MONITOR_PORT + 1\n\topts2.HTTPHost = \"127.0.0.1\"\n\topts2.Cluster = server.ClusterOpts{Name: \"M\", Host: \"127.0.0.1\", Port: 10222}\n\topts2.Routes = server.RoutesFromStr(\"nats-route://127.0.0.1:10223\")\n\topts2.NoSystemAccount = true\n\n\ts2 := RunServer(&opts2)\n\n\tcheckClusterFormed(t, s1, s2)\n\n\treturn s1, s2\n}\n\nfunc runMonitorServerNoHTTPPort() *server.Server {\n\tresetPreviousHTTPConnections()\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPPort = 0\n\topts.NoSystemAccount = true\n\n\treturn RunServer(&opts)\n}\n\nfunc resetPreviousHTTPConnections() {\n\thttp.DefaultTransport.(*http.Transport).CloseIdleConnections()\n}\n\n// Make sure that we do not run the http server for monitoring unless asked.\nfunc TestNoMonitorPort(t *testing.T) {\n\ts := runMonitorServerNoHTTPPort()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tif resp, err := http.Get(url + \"varz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n\tif resp, err := http.Get(url + \"healthz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n\tif resp, err := http.Get(url + \"connz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n}\n\n// testEndpointDataRace tests a monitoring endpoint for data races by polling\n// while client code acts to ensure statistics are updated. It is designed to\n// run under the -race flag to catch violations. The caller must start the\n// NATS server.\nfunc testEndpointDataRace(endpoint string, t *testing.T) {\n\tvar doneWg sync.WaitGroup\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\n\t// Poll as fast as we can, while creating connections, publishing,\n\t// and subscribing.\n\tclientDone := int64(0)\n\tdoneWg.Add(1)\n\tgo func() {\n\t\tfor atomic.LoadInt64(&clientDone) == 0 {\n\t\t\tresp, err := http.Get(url + endpoint)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Expected no error: Got %v\\n\", err)\n\t\t\t} else {\n\t\t\t\tresp.Body.Close()\n\t\t\t}\n\t\t}\n\t\tdoneWg.Done()\n\t}()\n\n\t// create connections, subscriptions, and publish messages to\n\t// update the monitor variables.\n\tvar conns []net.Conn\n\tfor i := 0; i < 50; i++ {\n\t\tcl := createClientConnSubscribeAndPublish(t)\n\t\t// keep a few connections around to test monitor variables.\n\t\tif i%10 == 0 {\n\t\t\tconns = append(conns, cl)\n\t\t} else {\n\t\t\tcl.Close()\n\t\t}\n\t}\n\tatomic.AddInt64(&clientDone, 1)\n\n\t// wait for the endpoint polling goroutine to exit\n\tdoneWg.Wait()\n\n\t// cleanup the conns\n\tfor _, cl := range conns {\n\t\tcl.Close()\n\t}\n}\n\nfunc TestEndpointDataRaces(t *testing.T) {\n\t// setup a small cluster to test /routez\n\ts1, s2 := runMonitorServerClusteredPair(t)\n\tdefer s1.Shutdown()\n\tdefer s2.Shutdown()\n\n\t// test all of our endpoints\n\ttestEndpointDataRace(\"varz\", t)\n\ttestEndpointDataRace(\"connz\", t)\n\ttestEndpointDataRace(\"routez\", t)\n\ttestEndpointDataRace(\"subsz\", t)\n\ttestEndpointDataRace(\"stacksz\", t)\n}\n\nfunc TestVarz(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tresp, err := http.Get(url + \"varz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tv := server.Varz{}\n\tif err := json.Unmarshal(body, &v); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\t// Do some sanity checks on values\n\tif time.Since(v.Start) > 10*time.Second {\n\t\tt.Fatal(\"Expected start time to be within 10 seconds.\")\n\t}\n\n\tcl := createClientConnSubscribeAndPublish(t)\n\tdefer cl.Close()\n\n\tresp, err = http.Get(url + \"varz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err = io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tif strings.Contains(string(body), \"cluster_port\") {\n\t\tt.Fatal(\"Varz body contains cluster information when no cluster is defined.\")\n\t}\n\n\tv = server.Varz{}\n\tif err := json.Unmarshal(body, &v); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\tif v.Connections != 1 {\n\t\tt.Fatalf(\"Expected Connections of 1, got %v\\n\", v.Connections)\n\t}\n\tif v.InMsgs != 1 {\n\t\tt.Fatalf(\"Expected InMsgs of 1, got %v\\n\", v.InMsgs)\n\t}\n\tif v.OutMsgs != 1 {\n\t\tt.Fatalf(\"Expected OutMsgs of 1, got %v\\n\", v.OutMsgs)\n\t}\n\tif v.InBytes != 5 {\n\t\tt.Fatalf(\"Expected InBytes of 5, got %v\\n\", v.InBytes)\n\t}\n\tif v.OutBytes != 5 {\n\t\tt.Fatalf(\"Expected OutBytes of 5, got %v\\n\", v.OutBytes)\n\t}\n\tif v.MaxPending != server.MAX_PENDING_SIZE {\n\t\tt.Fatalf(\"Expected MaxPending of %d, got %v\\n\",\n\t\t\tserver.MAX_PENDING_SIZE, v.MaxPending)\n\t}\n\tif v.WriteDeadline != server.DEFAULT_FLUSH_DEADLINE {\n\t\tt.Fatalf(\"Expected WriteDeadline of %d, got %v\\n\",\n\t\t\tserver.DEFAULT_FLUSH_DEADLINE, v.WriteDeadline)\n\t}\n}\n\nfunc TestConnz(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tresp, err := http.Get(url + \"connz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tc := server.Connz{}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\t// Test contents..\n\tif c.NumConns != 0 {\n\t\tt.Fatalf(\"Expected 0 connections, got %d\\n\", c.NumConns)\n\t}\n\tif c.Total != 0 {\n\t\tt.Fatalf(\"Expected 0 live connections, got %d\\n\", c.Total)\n\t}\n\tif c.Conns == nil || len(c.Conns) != 0 {\n\t\tt.Fatalf(\"Expected 0 connections in array, got %p\\n\", c.Conns)\n\t}\n\n\tcl := createClientConnSubscribeAndPublish(t)\n\tdefer cl.Close()\n\n\tresp, err = http.Get(url + \"connz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err = io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\tif c.NumConns != 1 {\n\t\tt.Fatalf(\"Expected 1 connection, got %d\\n\", c.NumConns)\n\t}\n\tif c.Total != 1 {\n\t\tt.Fatalf(\"Expected 1 live connection, got %d\\n\", c.Total)\n\t}\n\tif c.Conns == nil || len(c.Conns) != 1 {\n\t\tt.Fatalf(\"Expected 1 connection in array, got %p\\n\", c.Conns)\n\t}\n\n\tif c.Limit != server.DefaultConnListSize {\n\t\tt.Fatalf(\"Expected limit of %d, got %v\\n\", server.DefaultConnListSize, c.Limit)\n\t}\n\n\tif c.Offset != 0 {\n\t\tt.Fatalf(\"Expected offset of 0, got %v\\n\", c.Offset)\n\t}\n\n\t// Test inside details of each connection\n\tci := c.Conns[0]\n\n\tif ci.Cid == 0 {\n\t\tt.Fatalf(\"Expected non-zero cid, got %v\\n\", ci.Cid)\n\t}\n\tif ci.IP != \"127.0.0.1\" {\n\t\tt.Fatalf(\"Expected \\\"127.0.0.1\\\" for IP, got %v\\n\", ci.IP)\n\t}\n\tif ci.Port == 0 {\n\t\tt.Fatalf(\"Expected non-zero port, got %v\\n\", ci.Port)\n\t}\n\tif ci.NumSubs != 1 {\n\t\tt.Fatalf(\"Expected num_subs of 1, got %v\\n\", ci.NumSubs)\n\t}\n\tif len(ci.Subs) != 0 {\n\t\tt.Fatalf(\"Expected subs of 0, got %v\\n\", ci.Subs)\n\t}\n\tif ci.InMsgs != 1 {\n\t\tt.Fatalf(\"Expected InMsgs of 1, got %v\\n\", ci.InMsgs)\n\t}\n\tif ci.OutMsgs != 1 {\n\t\tt.Fatalf(\"Expected OutMsgs of 1, got %v\\n\", ci.OutMsgs)\n\t}\n\tif ci.InBytes != 5 {\n\t\tt.Fatalf(\"Expected InBytes of 1, got %v\\n\", ci.InBytes)\n\t}\n\tif ci.OutBytes != 5 {\n\t\tt.Fatalf(\"Expected OutBytes of 1, got %v\\n\", ci.OutBytes)\n\t}\n}\n\nfunc TestTLSConnz(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls.conf\")\n\tdefer srv.Shutdown()\n\trootCAFile := \"./configs/certs/ca.pem\"\n\tclientCertFile := \"./configs/certs/client-cert.pem\"\n\tclientKeyFile := \"./configs/certs/client-key.pem\"\n\n\t// Test with secure connection\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tnurl := fmt.Sprintf(\"tls://%s:%s@%s/\", opts.Username, opts.Password, endpoint)\n\tnc, err := nats.Connect(nurl, nats.RootCAs(rootCAFile))\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error on Connect with Secure Options: %+v\\n\", err)\n\t}\n\tdefer nc.Close()\n\tch := make(chan struct{})\n\tnc.Subscribe(\"foo\", func(m *nats.Msg) { ch <- struct{}{} })\n\tnc.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\t<-ch\n\n\turl := fmt.Sprintf(\"https://127.0.0.1:%d/\", opts.HTTPSPort)\n\ttlsConfig := &tls.Config{}\n\tcaCert, err := os.ReadFile(rootCAFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Got error reading RootCA file: %s\", err)\n\t}\n\tcaCertPool := x509.NewCertPool()\n\tcaCertPool.AppendCertsFromPEM(caCert)\n\ttlsConfig.RootCAs = caCertPool\n\n\tcert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Got error reading client certificates: %s\", err)\n\t}\n\ttlsConfig.Certificates = []tls.Certificate{cert}\n\ttransport := &http.Transport{TLSClientConfig: tlsConfig}\n\thttpClient := &http.Client{Transport: transport}\n\n\tresp, err := httpClient.Get(url + \"connz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\tc := server.Connz{}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\tif c.NumConns != 1 {\n\t\tt.Fatalf(\"Expected 1 connection, got %d\\n\", c.NumConns)\n\t}\n\tif c.Total != 1 {\n\t\tt.Fatalf(\"Expected 1 live connection, got %d\\n\", c.Total)\n\t}\n\tif c.Conns == nil || len(c.Conns) != 1 {\n\t\tt.Fatalf(\"Expected 1 connection in array, got %d\\n\", len(c.Conns))\n\t}\n\n\t// Test inside details of each connection\n\tci := c.Conns[0]\n\n\tif ci.Cid == 0 {\n\t\tt.Fatalf(\"Expected non-zero cid, got %v\\n\", ci.Cid)\n\t}\n\tif ci.IP != \"127.0.0.1\" {\n\t\tt.Fatalf(\"Expected \\\"127.0.0.1\\\" for IP, got %v\\n\", ci.IP)\n\t}\n\tif ci.Port == 0 {\n\t\tt.Fatalf(\"Expected non-zero port, got %v\\n\", ci.Port)\n\t}\n\tif ci.NumSubs != 1 {\n\t\tt.Fatalf(\"Expected num_subs of 1, got %v\\n\", ci.NumSubs)\n\t}\n\tif len(ci.Subs) != 0 {\n\t\tt.Fatalf(\"Expected subs of 0, got %v\\n\", ci.Subs)\n\t}\n\tif ci.InMsgs != 1 {\n\t\tt.Fatalf(\"Expected InMsgs of 1, got %v\\n\", ci.InMsgs)\n\t}\n\tif ci.OutMsgs != 1 {\n\t\tt.Fatalf(\"Expected OutMsgs of 1, got %v\\n\", ci.OutMsgs)\n\t}\n\tif ci.InBytes != 5 {\n\t\tt.Fatalf(\"Expected InBytes of 1, got %v\\n\", ci.InBytes)\n\t}\n\tif ci.OutBytes != 5 {\n\t\tt.Fatalf(\"Expected OutBytes of 1, got %v\\n\", ci.OutBytes)\n\t}\n\tif ci.Start.IsZero() {\n\t\tt.Fatalf(\"Expected Start to be valid\\n\")\n\t}\n\tif ci.Uptime == \"\" {\n\t\tt.Fatalf(\"Expected Uptime to be valid\\n\")\n\t}\n\tif ci.LastActivity.IsZero() {\n\t\tt.Fatalf(\"Expected LastActivity to be valid\\n\")\n\t}\n\tif ci.LastActivity.UnixNano() < ci.Start.UnixNano() {\n\t\tt.Fatalf(\"Expected LastActivity [%v] to be > Start [%v]\\n\", ci.LastActivity, ci.Start)\n\t}\n\tif ci.Idle == \"\" {\n\t\tt.Fatalf(\"Expected Idle to be valid\\n\")\n\t}\n}\n\nfunc TestConnzWithSubs(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tcl := createClientConnSubscribeAndPublish(t)\n\tdefer cl.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tresp, err := http.Get(url + \"connz?subs=1\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tc := server.Connz{}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\t// Test inside details of each connection\n\tci := c.Conns[0]\n\tif len(ci.Subs) != 1 || ci.Subs[0] != \"foo\" {\n\t\tt.Fatalf(\"Expected subs of 1, got %v\\n\", ci.Subs)\n\t}\n}\n\nfunc TestConnzWithAuth(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/multi_user.conf\")\n\tdefer srv.Shutdown()\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tcurl := fmt.Sprintf(\"nats://%s:%s@%s/\", opts.Users[0].Username, opts.Users[0].Password, endpoint)\n\tnc, err := nats.Connect(curl)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error on Connect: %+v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tch := make(chan struct{})\n\tnc.Subscribe(\"foo\", func(m *nats.Msg) { ch <- struct{}{} })\n\tnc.Publish(\"foo\", []byte(\"Hello\"))\n\n\t// Wait for message\n\t<-ch\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", opts.HTTPPort)\n\n\tresp, err := http.Get(url + \"connz?auth=1\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tc := server.Connz{}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\t// Test that we have authorized_user and its Alice.\n\tci := c.Conns[0]\n\tif ci.AuthorizedUser != opts.Users[0].Username {\n\t\tt.Fatalf(\"Expected authorized_user to be %q, got %q\\n\",\n\t\t\topts.Users[0].Username, ci.AuthorizedUser)\n\t}\n}\n\nfunc TestConnzWithAccounts(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\ts, opts := RunServerWithConfig(\"./configs/multi_accounts.conf\")\n\tdefer s.Shutdown()\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\n\t// Connect all the users. Tests depend on knowing users, accounts.\n\tif len(opts.Users) != 6 {\n\t\tt.Fatalf(\"Expected 6 total users, got %d\", len(opts.Users))\n\t}\n\tif len(opts.Accounts) != 3 {\n\t\tt.Fatalf(\"Expected 3 total accounts, got %d\", len(opts.Accounts))\n\t}\n\n\t// Map from user to account name.\n\tutoa := make(map[string]string)\n\tconns := make([]*nats.Conn, len(opts.Users))\n\tfor _, u := range opts.Users {\n\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://%s:%s@%s/\", u.Username, u.Password, endpoint))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Got an error on Connect: %+v\\n\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tutoa[u.Username] = u.Account.Name\n\t\tconns = append(conns, nc)\n\t}\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", opts.HTTPPort)\n\n\tgrabConnz := func(args string) *server.Connz {\n\t\tt.Helper()\n\t\tif args != \"\" {\n\t\t\targs = \"&\" + args\n\t\t}\n\t\tresp, err := http.Get(url + \"connz?auth=1\" + args)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t\t}\n\t\tif resp.StatusCode != 200 {\n\t\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tbody, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t\t}\n\t\tc := server.Connz{}\n\t\tif err := json.Unmarshal(body, &c); err != nil {\n\t\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t\t}\n\t\treturn &c\n\t}\n\n\tc := grabConnz(\"\")\n\tif c.NumConns != 6 {\n\t\tt.Fatalf(\"Expected 6 connection entries, got %d\", c.NumConns)\n\t}\n\n\tcheckConn := func(ci *server.ConnInfo) {\n\t\tt.Helper()\n\t\tuser := ci.AuthorizedUser\n\t\taccount := utoa[user]\n\t\tif user == \"\" || account == \"\" {\n\t\t\tt.Fatalf(\"Empty user or account: %q - %q\", user, account)\n\t\t}\n\t\tif ci.Account != account {\n\t\t\tt.Fatalf(\"Expected account of %q, got %q\", account, ci.Account)\n\t\t}\n\t}\n\n\tfor i, ci := range c.Conns {\n\t\tif ci.Cid != uint64(i+1) {\n\t\t\tt.Fatalf(\"Expected CID of %d, got %d\", i+1, ci.Cid)\n\t\t}\n\t\tcheckConn(ci)\n\t}\n\n\t// Now make sure we can pull connections by account and user\n\tpullByAccount := func(accName, state string) {\n\t\tt.Helper()\n\t\tc = grabConnz(\"acc=\" + accName + \"&state=\" + state)\n\t\tif c.NumConns != 2 {\n\t\t\tt.Fatalf(\"Expected 2 connection entries, got %d\", c.NumConns)\n\t\t}\n\t\tfor _, ci := range c.Conns {\n\t\t\tif ci.Account != accName {\n\t\t\t\tt.Fatalf(\"Expected %q account, go %q\", accName, ci.Account)\n\t\t\t}\n\t\t}\n\t}\n\n\tpullByUser := func(user, state string) {\n\t\tt.Helper()\n\t\tc = grabConnz(\"user=\" + user + \"&state=\" + state)\n\t\tif c.NumConns != 1 {\n\t\t\tt.Fatalf(\"Expected 1 connection, got %d\", c.NumConns)\n\t\t}\n\t\tif c.Conns[0].AuthorizedUser != user {\n\t\t\tt.Fatalf(\"Expected user %q, got %q\", user, c.Conns[0].AuthorizedUser)\n\t\t}\n\t}\n\n\tpullByAccount(\"engineering\", \"open\")\n\tpullByAccount(\"finance\", \"open\")\n\tpullByAccount(\"legal\", \"open\")\n\n\tpullByUser(\"alice\", \"open\")\n\tpullByUser(\"bob\", \"open\")\n\n\tpullByUser(\"john\", \"open\")\n\tpullByUser(\"mary\", \"open\")\n\n\tpullByUser(\"peter\", \"open\")\n\tpullByUser(\"paul\", \"open\")\n\n\t// Now closed and make sure these work on closed as well.\n\tfor _, nc := range conns {\n\t\tnc.Close()\n\t}\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif numClients := s.NumClients(); numClients != 0 {\n\t\t\treturn fmt.Errorf(\"Number of client is %d\", numClients)\n\t\t}\n\t\treturn nil\n\t})\n\n\tpullByAccount(\"engineering\", \"closed\")\n\tpullByAccount(\"finance\", \"closed\")\n\tpullByAccount(\"legal\", \"closed\")\n\n\tpullByUser(\"alice\", \"closed\")\n\tpullByUser(\"bob\", \"closed\")\n\n\tpullByUser(\"john\", \"closed\")\n\tpullByUser(\"mary\", \"closed\")\n\n\tpullByUser(\"peter\", \"closed\")\n\tpullByUser(\"paul\", \"closed\")\n}\n\nfunc TestConnzWithOffsetAndLimit(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tcl1 := createClientConnSubscribeAndPublish(t)\n\tdefer cl1.Close()\n\n\tcl2 := createClientConnSubscribeAndPublish(t)\n\tdefer cl2.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tresp, err := http.Get(url + \"connz?offset=1&limit=1\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tc := server.Connz{}\n\tif err := json.Unmarshal(body, &c); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\tif c.Limit != 1 {\n\t\tt.Fatalf(\"Expected limit of 1, got %v\\n\", c.Limit)\n\t}\n\n\tif c.Offset != 1 {\n\t\tt.Fatalf(\"Expected offset of 1, got %v\\n\", c.Offset)\n\t}\n\n\tif len(c.Conns) != 1 {\n\t\tt.Fatalf(\"Expected conns of 1, got %v\\n\", len(c.Conns))\n\t}\n}\n\nfunc TestSubsz(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\tcl := createClientConnSubscribeAndPublish(t)\n\tdefer cl.Close()\n\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", MONITOR_PORT)\n\tresp, err := http.Get(url + \"subscriptionsz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\n\tsu := server.Subsz{}\n\tif err := json.Unmarshal(body, &su); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\n\t// Do some sanity checks on values\n\tif su.NumSubs != 1 {\n\t\tt.Fatalf(\"Expected num_subs of 1, got %v\\n\", su.NumSubs)\n\t}\n}\n\nfunc TestHTTPHost(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\t// Grab non-127.0.0.1 address and try to use that to connect.\n\t// Should fail.\n\tvar ip net.IP\n\tifaces, _ := net.Interfaces()\n\tfor _, i := range ifaces {\n\t\taddrs, _ := i.Addrs()\n\t\tfor _, addr := range addrs {\n\t\t\tswitch v := addr.(type) {\n\t\t\tcase *net.IPNet:\n\t\t\t\tip = v.IP\n\t\t\tcase *net.IPAddr:\n\t\t\t\tip = v.IP\n\t\t\t}\n\t\t\t// Skip loopback/127.0.0.1 or any ipv6 for now.\n\t\t\tif ip.IsLoopback() || ip.To4() == nil {\n\t\t\t\tip = nil\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tif ip != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tif ip == nil {\n\t\tt.Fatalf(\"Could not find non-loopback IPV4 address\")\n\t}\n\turl := fmt.Sprintf(\"http://%v:%d/\", ip, MONITOR_PORT)\n\tif resp, err := http.Get(url + \"varz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n}\n\n// Create a connection to test ConnInfo\nfunc createClientConnSubscribeAndPublish(t *testing.T) net.Conn {\n\tcl := createClientConn(t, \"127.0.0.1\", CLIENT_PORT)\n\tsend, expect := setupConn(t, cl)\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\tsend(\"SUB foo 1\\r\\nPUB foo 5\\r\\nhello\\r\\n\")\n\texpectMsgs(1)\n\n\treturn cl\n}\n\nfunc TestMonitorNoTLSConfig(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.HTTPSPort = MONITOR_PORT\n\ts := server.New(&opts)\n\tdefer s.Shutdown()\n\t// Check with manually starting the monitoring, which should return an error\n\tif err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), \"TLS\") {\n\t\tt.Fatalf(\"Expected error about missing TLS config, got %v\", err)\n\t}\n\t// Also check by calling Start(), which should produce a fatal error\n\tdl := &dummyLogger{}\n\ts.SetLogger(dl, false, false)\n\tdefer s.SetLogger(nil, false, false)\n\ts.Start()\n\tif !strings.Contains(dl.msg, \"TLS\") {\n\t\tt.Fatalf(\"Expected error about missing TLS config, got %v\", dl.msg)\n\t}\n}\n\nfunc TestMonitorErrorOnListen(t *testing.T) {\n\ts := runMonitorServer()\n\tdefer s.Shutdown()\n\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT + 1\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.HTTPPort = MONITOR_PORT\n\ts2 := server.New(&opts)\n\tdefer s2.Shutdown()\n\tif err := s2.StartMonitoring(); err == nil || !strings.Contains(err.Error(), \"listen\") {\n\t\tt.Fatalf(\"Expected error about not able to start listener, got %v\", err)\n\t}\n}\n\nfunc TestMonitorBothPortsConfigured(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.HTTPPort = MONITOR_PORT\n\topts.HTTPSPort = MONITOR_PORT + 1\n\ts := server.New(&opts)\n\tdefer s.Shutdown()\n\tif err := s.StartMonitoring(); err == nil || !strings.Contains(err.Error(), \"specify both\") {\n\t\tt.Fatalf(\"Expected error about ports configured, got %v\", err)\n\t}\n}\n\nfunc TestMonitorStop(t *testing.T) {\n\tresetPreviousHTTPConnections()\n\topts := DefaultTestOptions\n\topts.Port = CLIENT_PORT\n\topts.HTTPHost = \"127.0.0.1\"\n\topts.HTTPPort = MONITOR_PORT\n\turl := fmt.Sprintf(\"http://%v:%d/\", opts.HTTPHost, MONITOR_PORT)\n\t// Create a server instance and start only the monitoring http server.\n\ts := server.New(&opts)\n\tif err := s.StartMonitoring(); err != nil {\n\t\tt.Fatalf(\"Error starting monitoring: %v\", err)\n\t}\n\t// Make sure http server is started\n\tresp, err := http.Get(url + \"varz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error on http request: %v\", err)\n\t}\n\tresp.Body.Close()\n\t// Although the server itself was not started (we did not call s.Start()),\n\t// Shutdown() should stop the http server.\n\ts.Shutdown()\n\t// HTTP request should now fail\n\tif resp, err := http.Get(url + \"varz\"); err == nil {\n\t\tt.Fatalf(\"Expected error: Got %+v\\n\", resp)\n\t}\n}\n" }, { "path": "test/new_routes_test.go", "content": "// Copyright 2018-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/logger\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nfunc runNewRouteServer(t *testing.T) (*server.Server, *server.Options) {\n\treturn RunServerWithConfig(\"./configs/new_cluster.conf\")\n}\n\nfunc TestNewRouteInfoOnConnect(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\tinfo := checkInfoMsg(t, rc)\n\tif info.Port != opts.Cluster.Port {\n\t\tt.Fatalf(\"Received wrong information for port, expected %d, got %d\",\n\t\t\tinfo.Port, opts.Cluster.Port)\n\t}\n\n\t// Make sure we advertise new proto.\n\tif info.Proto < server.RouteProtoV2 {\n\t\tt.Fatalf(\"Expected routeProtoV2 or above, got %d\", info.Proto)\n\t}\n\t// New proto should always send nonce too.\n\tif info.Nonce == \"\" {\n\t\tt.Fatalf(\"Expected a non empty nonce in new route INFO\")\n\t}\n\t// By default headers should be true.\n\tif !info.Headers {\n\t\tt.Fatalf(\"Expected to have headers on by default\")\n\t}\n\t// Leafnode origin cluster support.\n\tif !info.LNOC {\n\t\tt.Fatalf(\"Expected to have leafnode origin cluster support\")\n\t}\n\tif !info.LNOCU {\n\t\tt.Fatalf(\"Expected to have leafnode origin cluster in unsub protocol support\")\n\t}\n}\n\nfunc TestNewRouteHeaderSupport(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupHeaderConn(t, clientA)\n\tsendA(\"SUB foo bar 22\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(1, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB, expectB := setupHeaderConn(t, clientB)\n\t// Can not have \\r\\n in payload fyi for regex.\n\tsendB(\"HPUB foo reply 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectHeaderMsgs := expectHeaderMsgsCommand(t, expectA)\n\tmatches := expectHeaderMsgs(1)\n\tcheckHmsg(t, matches[0], \"foo\", \"22\", \"reply\", \"12\", \"14\", \"K1:V1,K2:V2 \", \"ok\")\n}\n\nfunc TestNewRouteHeaderSupportOldAndNew(t *testing.T) {\n\toptsA := LoadConfig(\"./configs/srv_a.conf\")\n\toptsA.NoHeaderSupport = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB, optsB := RunServerWithConfig(\"./configs/srv_b.conf\")\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupHeaderConn(t, clientA)\n\tsendA(\"SUB foo bar 22\\r\\n\")\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(1, srvA, srvB); err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\n\tsendB, expectB := setupHeaderConn(t, clientB)\n\t// Can not have \\r\\n in payload fyi for regex.\n\tsendB(\"HPUB foo reply 12 14\\r\\nK1:V1,K2:V2 ok\\r\\n\")\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"22\", \"reply\", \"2\", \"ok\")\n}\n\nfunc sendRouteInfo(t *testing.T, rc net.Conn, routeSend sendFun, routeID string) {\n\tinfo := checkInfoMsg(t, rc)\n\tinfo.ID = routeID\n\tinfo.Name = \"\"\n\tb, err := json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\trouteSend(fmt.Sprintf(\"INFO %s\\r\\n\", b))\n}\n\nfunc TestNewRouteConnectSubs(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\t// Create 10 normal subs and 10 queue subscribers.\n\tfor i := 0; i < 10; i++ {\n\t\tsend(fmt.Sprintf(\"SUB foo %d\\r\\n\", i))\n\t\tsend(fmt.Sprintf(\"SUB foo bar %d\\r\\n\", 100+i))\n\t}\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// This client should not be considered active since no subscriptions or\n\t// messages have been published.\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:22\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\tbuf := routeExpect(rsubRe)\n\n\tmatches := rsubRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 2 {\n\t\tt.Fatalf(\"Expected 2 results, got %d\", len(matches))\n\t}\n\tfor _, m := range matches {\n\t\tif string(m[1]) != \"$G\" {\n\t\t\tt.Fatalf(\"Expected global account name of '$G', got %q\", m[1])\n\t\t}\n\t\tif string(m[2]) != \"foo\" {\n\t\t\tt.Fatalf(\"Expected subject of 'foo', got %q\", m[2])\n\t\t}\n\t\tif m[3] != nil {\n\t\t\tif string(m[3]) != \"bar\" {\n\t\t\t\tt.Fatalf(\"Expected group of 'bar', got %q\", m[3])\n\t\t\t}\n\t\t\t// Expect a weighted count for the queue group\n\t\t\tif len(m) != 5 {\n\t\t\t\tt.Fatalf(\"Expected a weight for the queue group\")\n\t\t\t}\n\t\t\tif m[4] == nil || string(m[4]) != \"10\" {\n\t\t\t\tt.Fatalf(\"Expected Weight of '10', got %q\", m[4])\n\t\t\t}\n\t\t}\n\t}\n\n\t// Close the client connection, check the results.\n\tc.Close()\n\n\t// Expect 2\n\tfor numUnSubs := 0; numUnSubs != 2; {\n\t\tbuf := routeExpect(runsubRe)\n\t\tnumUnSubs += len(runsubRe.FindAllSubmatch(buf, -1))\n\t}\n}\n\nfunc TestNewRouteConnectSubsWithAccount(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\taccName := \"$FOO\"\n\ts.RegisterAccount(accName)\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConnWithAccount(t, s, c, accName)\n\n\t// Create 10 normal subs and 10 queue subscribers.\n\tfor i := 0; i < 10; i++ {\n\t\tsend(fmt.Sprintf(\"SUB foo %d\\r\\n\", i))\n\t\tsend(fmt.Sprintf(\"SUB foo bar %d\\r\\n\", 100+i))\n\t}\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// This client should not be considered active since no subscriptions or\n\t// messages have been published.\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:22\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\tbuf := routeExpect(rsubRe)\n\n\tmatches := rsubRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 2 {\n\t\tt.Fatalf(\"Expected 2 results, got %d\", len(matches))\n\t}\n\tfor _, m := range matches {\n\t\tif string(m[1]) != accName {\n\t\t\tt.Fatalf(\"Expected global account name of %q, got %q\", accName, m[1])\n\t\t}\n\t\tif string(m[2]) != \"foo\" {\n\t\t\tt.Fatalf(\"Expected subject of 'foo', got %q\", m[2])\n\t\t}\n\t\tif m[3] != nil {\n\t\t\tif string(m[3]) != \"bar\" {\n\t\t\t\tt.Fatalf(\"Expected group of 'bar', got %q\", m[3])\n\t\t\t}\n\t\t\t// Expect the SID to be the total weighted count for the queue group\n\t\t\tif len(m) != 5 {\n\t\t\t\tt.Fatalf(\"Expected a weight for the queue group\")\n\t\t\t}\n\t\t\tif m[4] == nil || string(m[4]) != \"10\" {\n\t\t\t\tt.Fatalf(\"Expected Weight of '10', got %q\", m[4])\n\t\t\t}\n\t\t}\n\t}\n\n\t// Close the client connection, check the results.\n\tc.Close()\n\n\t// Expect 2\n\tfor numUnSubs := 0; numUnSubs != 2; {\n\t\tbuf := routeExpect(runsubRe)\n\t\tnumUnSubs += len(runsubRe.FindAllSubmatch(buf, -1))\n\t}\n}\n\nfunc TestNewRouteRSubs(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tfoo, err := s.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$foo': %v\", err)\n\t}\n\tbar, err := s.RegisterAccount(\"$bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$bar': %v\", err)\n\t}\n\n\t// Create a client an account foo.\n\tclientA := createClientConn(t, opts.Host, opts.Port)\n\tsendA, expectA := setupConnWithAccount(t, s, clientA, \"$foo\")\n\tdefer clientA.Close()\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tif foonc := foo.NumConnections(); foonc != 1 {\n\t\tt.Fatalf(\"Expected foo account to have 1 client, got %d\", foonc)\n\t}\n\tif barnc := bar.NumConnections(); barnc != 0 {\n\t\tt.Fatalf(\"Expected bar account to have 0 clients, got %d\", barnc)\n\t}\n\n\t// Create a routeConn\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:33\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Have the client listen on foo.\n\tsendA(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Now create a new client for account $bar and have them subscribe.\n\tclientB := createClientConn(t, opts.Host, opts.Port)\n\tsendB, expectB := setupConnWithAccount(t, s, clientB, \"$bar\")\n\tdefer clientB.Close()\n\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tif foonc := foo.NumConnections(); foonc != 1 {\n\t\tt.Fatalf(\"Expected foo account to have 1 client, got %d\", foonc)\n\t}\n\tif barnc := bar.NumConnections(); barnc != 1 {\n\t\tt.Fatalf(\"Expected bar account to have 1 client, got %d\", barnc)\n\t}\n\n\t// Have the client listen on foo.\n\tsendB(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\trouteExpect(rsubRe)\n\n\t// Unsubscribe on clientA from foo subject.\n\tsendA(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\t// We should get an RUSUB here.\n\trouteExpect(runsubRe)\n\n\t// Now unsubscribe clientB, which should trigger an RS-.\n\tsendB(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\t// We should get an RUSUB here.\n\trouteExpect(runsubRe)\n\n\t// Now close down the clients.\n\tclientA.Close()\n\n\tsendB(\"SUB foo 2\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\trouteExpect(rsubRe)\n\n\t// Now close down client B.\n\tclientB.Close()\n\n\t// This should trigger an RS-\n\trouteExpect(runsubRe)\n}\n\nfunc TestNewRouteProgressiveNormalSubs(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:33\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// For progressive we will expect to receive first normal sub but\n\t// not subsequent ones.\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\trouteExpect(rsubRe)\n\n\tsend(\"SUB foo 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\texpectNothing(t, rc)\n\n\tvar buf []byte\n\n\t// Check that sid is showing us total number of subscriptions.\n\tcheckQueueSub := func(n string) {\n\t\tmatches := rsubRe.FindAllSubmatch(buf, -1)\n\t\tif len(matches) != 1 {\n\t\t\tt.Fatalf(\"Expected 1 result, got %d\", len(matches))\n\t\t}\n\t\tm := matches[0]\n\t\tif len(m) != 5 {\n\t\t\tt.Fatalf(\"Expected a SID for the queue group, only got %d elements\", len(m))\n\t\t}\n\t\tif string(m[4]) != n {\n\t\t\tt.Fatalf(\"Expected %q, got %q\", n, m[4])\n\t\t}\n\t}\n\n\t// We should always get the SUB info for QUEUES.\n\tsend(\"SUB foo bar 3\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tbuf = routeExpect(rsubRe)\n\tcheckQueueSub(\"1\")\n\n\tsend(\"SUB foo bar 4\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tbuf = routeExpect(rsubRe)\n\tcheckQueueSub(\"2\")\n\n\tsend(\"SUB foo bar 5\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tbuf = routeExpect(rsubRe)\n\tcheckQueueSub(\"3\")\n\n\t// Now walk them back down.\n\t// Again we should always get updates for queue subscribers.\n\t// And these will be RS+ protos walking the weighted count back down.\n\tsend(\"UNSUB 5\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tbuf = routeExpect(rsubRe)\n\tcheckQueueSub(\"2\")\n\n\tsend(\"UNSUB 4\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tbuf = routeExpect(rsubRe)\n\tcheckQueueSub(\"1\")\n\n\t// This one should send UNSUB\n\tsend(\"UNSUB 3\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(runsubRe)\n\n\t// Now normal ones.\n\tsend(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\texpectNothing(t, rc)\n\n\tsend(\"UNSUB 2\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(runsubRe)\n}\n\nfunc TestNewRouteClientClosedWithNormalSubscriptions(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:44\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(rsubRe)\n\n\tfor i := 2; i < 100; i++ {\n\t\tsend(fmt.Sprintf(\"SUB foo %d\\r\\n\", i))\n\t}\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Expect nothing from the route.\n\texpectNothing(t, rc)\n\n\t// Now close connection.\n\tc.Close()\n\texpectNothing(t, c)\n\n\tbuf := routeExpect(runsubRe)\n\tmatches := runsubRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 unsub response when closing client connection, got %d\", len(matches))\n\t}\n}\n\nfunc TestNewRouteClientClosedWithQueueSubscriptions(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:44\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tfor i := 0; i < 100; i++ {\n\t\tsend(fmt.Sprintf(\"SUB foo bar %d\\r\\n\", i))\n\t}\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Queue subscribers will send all updates.\n\tfor numRSubs := 0; numRSubs != 100; {\n\t\tbuf := routeExpect(rsubRe)\n\t\tnumRSubs += len(rsubRe.FindAllSubmatch(buf, -1))\n\t}\n\n\t// Now close connection.\n\tc.Close()\n\texpectNothing(t, c)\n\n\t// We should only get one unsub for the queue subscription.\n\tmatches := runsubRe.FindAllSubmatch(routeExpect(runsubRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 unsub response when closing client connection, got %d\", len(matches))\n\t}\n}\n\nfunc TestNewRouteRUnsubAccountSpecific(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\t// Create a routeConn\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:77\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\n\t// Now create 500 subs on same subject but all different accounts.\n\tfor i := 0; i < 500; i++ {\n\t\taccount := fmt.Sprintf(\"$foo.account.%d\", i)\n\t\ts.RegisterAccount(account)\n\t\trouteSend(fmt.Sprintf(\"RS+ %s foo\\r\\n\", account))\n\t}\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\trouteSend(\"RS- $foo.account.22 foo\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Do not expect a message on that account.\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConnWithAccount(t, s, c, \"$foo.account.22\")\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\tc.Close()\n\n\t// But make sure we still receive on others\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\tsend, expect = setupConnWithAccount(t, s, c, \"$foo.account.33\")\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches := rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$foo.account.33\", \"foo\", \"\", \"2\", \"ok\")\n}\n\nfunc TestNewRouteRSubCleanupOnDisconnect(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\t// Create a routeConn\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:77\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\n\t// Now create 100 subs on 3 different accounts.\n\tfor i := 0; i < 100; i++ {\n\t\tsubject := fmt.Sprintf(\"foo.%d\", i)\n\t\trouteSend(fmt.Sprintf(\"RS+ $foo %s\\r\\n\", subject))\n\t\trouteSend(fmt.Sprintf(\"RS+ $bar %s\\r\\n\", subject))\n\t\trouteSend(fmt.Sprintf(\"RS+ $baz %s bar %d\\r\\n\", subject, i+1))\n\t}\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\trc.Close()\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif ns := s.NumSubscriptions(); ns != 0 {\n\t\t\treturn fmt.Errorf(\"Number of subscriptions is %d\", ns)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNewRouteSendSubsAndMsgs(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:44\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Now let's send in interest from the new protocol.\n\t// Normal Subscription\n\trouteSend(\"RS+ $G foo\\r\\n\")\n\t// Make sure things were processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Now create a client and send a message, make sure we receive it\n\t// over the route connection.\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tbuf := routeExpect(rmsgRe)\n\tmatches := rmsgRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"\", \"2\", \"ok\")\n\n\t// Queue Subscription\n\trouteSend(\"RS+ $G foo bar 1\\r\\n\")\n\t// Make sure things were processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tsend(\"PUB foo reply 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"+ reply bar\", \"2\", \"ok\")\n\n\t// Another Queue Subscription\n\trouteSend(\"RS+ $G foo baz 1\\r\\n\")\n\t// Make sure things were processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tsend(\"PUB foo reply 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"+ reply bar baz\", \"2\", \"ok\")\n\n\t// Matching wildcard\n\trouteSend(\"RS+ $G *\\r\\n\")\n\t// Make sure things were processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tsend(\"PUB foo reply 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"+ reply bar baz\", \"2\", \"ok\")\n\n\t// No reply\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"| bar baz\", \"2\", \"ok\")\n\n\t// Now unsubscribe from the queue group.\n\trouteSend(\"RS- $G foo baz\\r\\n\")\n\trouteSend(\"RS- $G foo bar\\r\\n\")\n\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Now send and make sure they are removed. We should still get the message.\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\tmatches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"\", \"2\", \"ok\")\n\n\trouteSend(\"RS- $G foo\\r\\n\")\n\trouteSend(\"RS- $G *\\r\\n\")\n\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Now we should not receive messages anymore.\n\tsend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\texpectNothing(t, rc)\n}\n\nfunc TestNewRouteProcessRoutedMsgs(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:55\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Create a client\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\t// Normal sub to start\n\tsend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(rsubRe)\n\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\t// Now send in a RMSG to the route and make sure its delivered to the client.\n\trouteSend(\"RMSG $G foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\t// Now send in a RMSG to the route with a reply and make sure its delivered to the client.\n\trouteSend(\"RMSG $G foo reply 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tmatches = expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"reply\", \"2\", \"ok\")\n\n\t// Now add in a queue subscriber for the client.\n\tsend(\"SUB foo bar 11\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(rsubRe)\n\n\t// Now add in another queue subscriber for the client.\n\tsend(\"SUB foo baz 22\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\trouteExpect(rsubRe)\n\n\t// If we send from a route with no queues. Should only get one message.\n\trouteSend(\"RMSG $G foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\tmatches = expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\t// Now send to a specific queue group. We should get multiple messages now.\n\trouteSend(\"RMSG $G foo | bar 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\tmatches = expectMsgs(2)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\t// Now send to both queue groups. We should get all messages now.\n\trouteSend(\"RMSG $G foo | bar baz 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\tmatches = expectMsgs(3)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\t// Make sure we do the right thing with reply.\n\trouteSend(\"RMSG $G foo + reply bar baz 2\\r\\nok\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\tmatches = expectMsgs(3)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"reply\", \"2\", \"ok\")\n}\n\nfunc TestNewRouteQueueSubsDistribution(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendA, expectA := setupConn(t, clientA)\n\tsendB, expectB := setupConn(t, clientB)\n\n\t// Create 100 subscribers on each server.\n\tfor i := 0; i < 100; i++ {\n\t\tsproto := fmt.Sprintf(\"SUB foo bar %d\\r\\n\", i)\n\t\tsendA(sproto)\n\t\tsendB(sproto)\n\t}\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Each server should have its 100 local subscriptions, plus 1 for the route.\n\tif err := checkExpectedSubs(101, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tsender := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer sender.Close()\n\tsend, expect := setupConn(t, sender)\n\n\t// Send 100 messages from Sender\n\tfor i := 0; i < 100; i++ {\n\t\tsend(\"PUB foo 2\\r\\nok\\r\\n\")\n\t}\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tnumAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1))\n\tnumBReceived := len(msgRe.FindAllSubmatch(expectB(msgRe), -1))\n\n\t// We may not be able to properly time all messages being ready.\n\tfor numAReceived+numBReceived != 100 {\n\t\tif buf := peek(clientB); buf != nil {\n\t\t\tnumBReceived += len(msgRe.FindAllSubmatch(buf, -1))\n\t\t}\n\t\tif buf := peek(clientA); buf != nil {\n\t\t\tnumAReceived += len(msgRe.FindAllSubmatch(buf, -1))\n\t\t}\n\t}\n\t// These should be close to 50/50\n\tif numAReceived < 30 || numBReceived < 30 {\n\t\tt.Fatalf(\"Expected numbers to be close to 50/50, got %d/%d\", numAReceived, numBReceived)\n\t}\n}\n\n// Since we trade interest in accounts now, we have a potential issue with a new client\n// connecting via a brand new account, publishing and properly doing a flush, then exiting.\n// If existing subscribers were present but on a remote server they may not get the message.\nfunc TestNewRouteSinglePublishOnNewAccount(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tsrvA.RegisterAccount(\"$TEST22\")\n\tsrvB.RegisterAccount(\"$TEST22\")\n\n\t// Create and establish a listener on foo for $TEST22 account.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$TEST22\")\n\tsendA(\"SUB foo 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\tif err := checkExpectedSubs(1, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\t// Send a message, flush to make sure server processed and close connection.\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$TEST22\")\n\tsendB(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\tclientB.Close()\n\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n}\n\n// Same as above but make sure it works for queue subscribers as well.\nfunc TestNewRouteSinglePublishToQueueSubscriberOnNewAccount(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tsrvA.RegisterAccount(\"$TEST22\")\n\tsrvB.RegisterAccount(\"$TEST22\")\n\n\t// Create and establish a listener on foo for $TEST22 account.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$TEST22\")\n\tsendA(\"SUB foo bar 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\t// Send a message, flush to make sure server processed and close connection.\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$TEST22\")\n\tsendB(\"PUB foo bar 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\tdefer clientB.Close()\n\n\texpectMsgs := expectMsgsCommand(t, expectA)\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"bar\", \"2\", \"ok\")\n\n\tsendB(\"PUB foo bar 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\tmatches = expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"bar\", \"2\", \"ok\")\n}\n\n// Same as above but make sure it works for queue subscribers over multiple routes as well.\nfunc TestNewRouteSinglePublishToMultipleQueueSubscriberOnNewAccount(t *testing.T) {\n\tsrvA, srvB, srvC, optsA, optsB, optsC := runThreeServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\tdefer srvC.Shutdown()\n\n\tsrvA.RegisterAccount(\"$TEST22\")\n\tsrvB.RegisterAccount(\"$TEST22\")\n\tsrvC.RegisterAccount(\"$TEST22\")\n\n\t// Create and establish a listener on foo/bar for $TEST22 account. Do this on ClientA and ClientC.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$TEST22\")\n\tsendA(\"SUB foo bar 11\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\tclientC := createClientConn(t, optsC.Host, optsC.Port)\n\tdefer clientC.Close()\n\n\tsendC, expectC := setupConnWithAccount(t, srvC, clientC, \"$TEST22\")\n\tsendC(\"SUB foo bar 33\\r\\nPING\\r\\n\")\n\texpectC(pongRe)\n\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\tsendC(\"PING\\r\\n\")\n\texpectC(pongRe)\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Send a message, flush to make sure server processed and close connection.\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$TEST22\")\n\tsendB(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\tdefer clientB.Close()\n\n\t// This should trigger either clientA or clientC, but not both..\n\tbufA := peek(clientA)\n\tbufC := peek(clientC)\n\n\tif bufA != nil && bufC != nil {\n\t\tt.Fatalf(\"Expected one or the other, but got something on both\")\n\t}\n\tnumReceived := len(msgRe.FindAllSubmatch(bufA, -1))\n\tnumReceived += len(msgRe.FindAllSubmatch(bufC, -1))\n\tif numReceived != 1 {\n\t\tt.Fatalf(\"Expected only 1 msg, got %d\", numReceived)\n\t}\n\n\t// Now make sure that we are distributing correctly between A and C\n\t// Send 100 messages from Sender\n\tfor i := 0; i < 100; i++ {\n\t\tsendB(\"PUB foo 2\\r\\nok\\r\\n\")\n\t}\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tnumAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1))\n\tnumCReceived := len(msgRe.FindAllSubmatch(expectC(msgRe), -1))\n\n\t// We may not be able to properly time all messages being ready.\n\n\tfor numAReceived+numCReceived != 100 {\n\t\tif buf := peek(clientC); buf != nil {\n\t\t\tnumCReceived += len(msgRe.FindAllSubmatch(buf, -1))\n\t\t}\n\t\tif buf := peek(clientA); buf != nil {\n\t\t\tnumAReceived += len(msgRe.FindAllSubmatch(buf, -1))\n\t\t}\n\t}\n\n\t// These should be close to 50/50\n\tif numAReceived < 30 || numCReceived < 30 {\n\t\tt.Fatalf(\"Expected numbers to be close to 50/50, got %d/%d\", numAReceived, numCReceived)\n\t}\n}\n\nfunc registerAccounts(t *testing.T, s *server.Server) (*server.Account, *server.Account) {\n\t// Now create two accounts.\n\tf, err := s.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$foo': %v\", err)\n\t}\n\tb, err := s.RegisterAccount(\"$bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$bar': %v\", err)\n\t}\n\treturn f, b\n}\n\nfunc addStreamExport(subject string, authorized []*server.Account, targets ...*server.Account) {\n\tfor _, acc := range targets {\n\t\tacc.AddStreamExport(subject, authorized)\n\t}\n}\n\nfunc addServiceExport(subject string, authorized []*server.Account, targets ...*server.Account) {\n\tfor _, acc := range targets {\n\t\tacc.AddServiceExport(subject, authorized)\n\t}\n}\n\nvar isPublic = []*server.Account(nil)\n\nfunc TestNewRouteStreamImport(t *testing.T) {\n\ttestNewRouteStreamImport(t, false)\n}\n\nfunc testNewRouteStreamImport(t *testing.T, duplicateSub bool) {\n\tt.Helper()\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\t// Do Accounts for the servers.\n\tfooA, _ := registerAccounts(t, srvA)\n\tfooB, barB := registerAccounts(t, srvB)\n\n\t// Add export to both.\n\taddStreamExport(\"foo\", isPublic, fooA, fooB)\n\t// Add import abilities to server B's bar account from foo.\n\tbarB.AddStreamImport(fooB, \"foo\", \"\")\n\n\t// clientA will be connected to srvA and be the stream producer.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$foo\")\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$bar\")\n\tsendB(\"SUB foo 1\\r\\n\")\n\tif duplicateSub {\n\t\tsendB(\"SUB foo 1\\r\\n\")\n\t}\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\t// The subscription on \"foo\" for account $bar will also become\n\t// a subscription on \"foo\" for account $foo due to import.\n\t// So total of 2 subs.\n\tif err := checkExpectedSubs(2, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Send on clientA\n\tsendA(\"PING\\r\\n\")\n\texpectA(pongRe)\n\n\tsendA(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\texpectMsgs := expectMsgsCommand(t, expectB)\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\t// Send Again on clientA\n\tsendA(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\tmatches = expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n\n\tsendB(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\tif err := checkExpectedSubs(0, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tsendA(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\texpectNothing(t, clientA)\n}\n\nfunc TestNewRouteStreamImportLargeFanout(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\t// Do Accounts for the servers.\n\t// This account will export a stream.\n\tfooA, err := srvA.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$foo': %v\", err)\n\t}\n\tfooB, err := srvB.RegisterAccount(\"$foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating account '$foo': %v\", err)\n\t}\n\n\t// Add export to both.\n\taddStreamExport(\"foo\", isPublic, fooA, fooB)\n\n\t// Now we will create 100 accounts who will all import from foo.\n\tfanout := 100\n\tbarA := make([]*server.Account, fanout)\n\tfor i := 0; i < fanout; i++ {\n\t\tacc := fmt.Sprintf(\"$bar-%d\", i)\n\t\tbarA[i], err = srvB.RegisterAccount(acc)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error creating account %q: %v\", acc, err)\n\t\t}\n\t\t// Add import abilities to server B's bar account from foo.\n\t\tbarA[i].AddStreamImport(fooB, \"foo\", \"\")\n\t}\n\n\t// clientA will be connected to srvA and be the stream producer.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\t// Now setup fanout clients on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tclientB := make([]net.Conn, fanout)\n\tsendB := make([]sendFun, fanout)\n\texpectB := make([]expectFun, fanout)\n\n\tfor i := 0; i < fanout; i++ {\n\t\tclientB[i] = createClientConn(t, optsB.Host, optsB.Port)\n\t\tdefer clientB[i].Close()\n\t\tsendB[i], expectB[i] = setupConnWithAccount(t, srvB, clientB[i], barA[i].Name)\n\t\tsendB[i](\"SUB foo 1\\r\\nPING\\r\\n\")\n\t\texpectB[i](pongRe)\n\t}\n\n\t// Since we do not shadow all the bar acounts on srvA they will be dropped\n\t// when they hit the other side, which means we could only have one sub for\n\t// all the imports on srvA, and srvB will have 2*fanout, one normal and one\n\t// that represents the import.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif ns := srvA.NumSubscriptions(); ns != uint32(1) {\n\t\t\treturn fmt.Errorf(\"Number of subscriptions is %d\", ns)\n\t\t}\n\t\tif ns := srvB.NumSubscriptions(); ns != uint32(2*fanout) {\n\t\t\treturn fmt.Errorf(\"Number of subscriptions is %d\", ns)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNewRouteReservedReply(t *testing.T) {\n\ts, opts := runNewRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\t// Test that clients can't send to reserved service import replies.\n\tsend(\"PUB foo _R_.foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\texpect(errRe)\n}\n\nfunc TestNewRouteServiceImport(t *testing.T) {\n\t// To quickly enable trace and debug logging\n\t//doLog, doTrace, doDebug = true, true, true\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\t// Make so we can tell the two apart since in same PID.\n\tif doLog {\n\t\tsrvA.SetLogger(logger.NewTestLogger(\"[SRV-A] - \", false), true, true)\n\t\tsrvB.SetLogger(logger.NewTestLogger(\"[SRV-B] - \", false), true, true)\n\t}\n\n\t// Do Accounts for the servers.\n\tfooA, barA := registerAccounts(t, srvA)\n\tfooB, barB := registerAccounts(t, srvB)\n\n\t// Add export to both.\n\taddServiceExport(\"test.request\", isPublic, fooA, fooB)\n\n\t// Add import abilities to server B's bar account from foo.\n\t// Meaning that when a user sends a request on foo.request from account bar,\n\t// the request will be mapped to be received by the responder on account foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Do same on A.\n\tif err := barA.AddServiceImport(fooA, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint and responder.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$foo\")\n\tsendA(\"SUB test.request 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$bar\")\n\tsendB(\"SUB reply 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Wait for all subs to be propagated. (1 on foo, 2 on bar)\n\tif err := checkExpectedSubs(3, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Send the request from clientB on foo.request,\n\tsendB(\"PUB foo.request reply 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectMsgsA := expectMsgsCommand(t, expectA)\n\texpectMsgsB := expectMsgsCommand(t, expectB)\n\n\t// Expect the request on A\n\tmatches := expectMsgsA(1)\n\treply := string(matches[0][replyIndex])\n\tcheckMsg(t, matches[0], \"test.request\", \"1\", reply, \"2\", \"hi\")\n\tif reply == \"reply\" {\n\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t}\n\n\tsendA(fmt.Sprintf(\"PUB %s 2\\r\\nok\\r\\nPING\\r\\n\", reply))\n\texpectA(pongRe)\n\n\tmatches = expectMsgsB(1)\n\tcheckMsg(t, matches[0], \"reply\", \"1\", \"\", \"2\", \"ok\")\n\n\t// This will be the responder and the wildcard for all service replies.\n\tif ts := fooA.TotalSubs(); ts != 2 {\n\t\tt.Fatalf(\"Expected two subs to be left on fooA, but got %d\", ts)\n\t}\n\n\troutez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true})\n\tr := routez.Routes[0]\n\tif r == nil {\n\t\tt.Fatalf(\"Expected 1 route, got none\")\n\t}\n\tif r.NumSubs != 2 {\n\t\tt.Fatalf(\"Expected 2 subs in the route connection, got %v\", r.NumSubs)\n\t}\n}\n\nfunc TestNewRouteServiceExportWithWildcards(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tpublic bool\n\t}{\n\t\t{\n\t\t\tname: \"public\",\n\t\t\tpublic: true,\n\t\t},\n\t\t{\n\t\t\tname: \"private\",\n\t\t\tpublic: false,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tsrvA, srvB, optsA, optsB := runServers(t)\n\t\t\tdefer srvA.Shutdown()\n\t\t\tdefer srvB.Shutdown()\n\n\t\t\t// Do Accounts for the servers.\n\t\t\tfooA, barA := registerAccounts(t, srvA)\n\t\t\tfooB, barB := registerAccounts(t, srvB)\n\n\t\t\tvar accs []*server.Account\n\t\t\t// Add export to both.\n\t\t\tif !test.public {\n\t\t\t\taccs = []*server.Account{barA}\n\t\t\t}\n\t\t\taddServiceExport(\"ngs.update.*\", accs, fooA)\n\t\t\tif !test.public {\n\t\t\t\taccs = []*server.Account{barB}\n\t\t\t}\n\t\t\taddServiceExport(\"ngs.update.*\", accs, fooB)\n\n\t\t\t// Add import abilities to server B's bar account from foo.\n\t\t\tif err := barB.AddServiceImport(fooB, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\t\t\t// Do same on A.\n\t\t\tif err := barA.AddServiceImport(fooA, \"ngs.update\", \"ngs.update.$bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t\t\t}\n\n\t\t\t// clientA will be connected to srvA and be the service endpoint and responder.\n\t\t\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\t\t\tdefer clientA.Close()\n\n\t\t\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$foo\")\n\t\t\tsendA(\"SUB ngs.update.* 1\\r\\nPING\\r\\n\")\n\t\t\texpectA(pongRe)\n\n\t\t\t// Now setup client B on srvB who will do a sub from account $bar\n\t\t\t// that should map account $foo's foo subject.\n\t\t\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\t\t\tdefer clientB.Close()\n\n\t\t\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$bar\")\n\t\t\tsendB(\"SUB reply 1\\r\\nPING\\r\\n\")\n\t\t\texpectB(pongRe)\n\n\t\t\t// Wait for all subs to be propagated. (1 on foo, 2 on bar)\n\t\t\tif err := checkExpectedSubs(3, srvA, srvB); err != nil {\n\t\t\t\tt.Fatal(err.Error())\n\t\t\t}\n\n\t\t\t// Send the request from clientB on foo.request,\n\t\t\tsendB(\"PUB ngs.update reply 2\\r\\nhi\\r\\nPING\\r\\n\")\n\t\t\texpectB(pongRe)\n\n\t\t\texpectMsgsA := expectMsgsCommand(t, expectA)\n\t\t\texpectMsgsB := expectMsgsCommand(t, expectB)\n\n\t\t\t// Expect the request on A\n\t\t\tmatches := expectMsgsA(1)\n\t\t\treply := string(matches[0][replyIndex])\n\t\t\tcheckMsg(t, matches[0], \"ngs.update.$bar\", \"1\", reply, \"2\", \"hi\")\n\t\t\tif reply == \"reply\" {\n\t\t\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t\t\t}\n\n\t\t\tsendA(fmt.Sprintf(\"PUB %s 2\\r\\nok\\r\\nPING\\r\\n\", reply))\n\t\t\texpectA(pongRe)\n\n\t\t\tmatches = expectMsgsB(1)\n\t\t\tcheckMsg(t, matches[0], \"reply\", \"1\", \"\", \"2\", \"ok\")\n\n\t\t\tif ts := fooA.TotalSubs(); ts != 2 {\n\t\t\t\tt.Fatalf(\"Expected two subs to be left on fooA, but got %d\", ts)\n\t\t\t}\n\n\t\t\troutez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true})\n\t\t\tr := routez.Routes[0]\n\t\t\tif r == nil {\n\t\t\t\tt.Fatalf(\"Expected 1 route, got none\")\n\t\t\t}\n\t\t\tif r.NumSubs != 2 {\n\t\t\t\tt.Fatalf(\"Expected 2 subs in the route connection, got %v\", r.NumSubs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewRouteServiceImportQueueGroups(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\t// Do Accounts for the servers.\n\tfooA, barA := registerAccounts(t, srvA)\n\tfooB, barB := registerAccounts(t, srvB)\n\n\t// Add export to both.\n\taddServiceExport(\"test.request\", isPublic, fooA, fooB)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\t// Do same on A.\n\tif err := barA.AddServiceImport(fooA, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint and responder.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$foo\")\n\tsendA(\"SUB test.request QGROUP 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$bar\")\n\tsendB(\"SUB reply QGROUP_TOO 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Wait for all subs to be propagated. (1 on foo, 2 on bar)\n\tif err := checkExpectedSubs(3, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Send the request from clientB on foo.request,\n\tsendB(\"PUB foo.request reply 2\\r\\nhi\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\texpectMsgsA := expectMsgsCommand(t, expectA)\n\texpectMsgsB := expectMsgsCommand(t, expectB)\n\n\t// Expect the request on A\n\tmatches := expectMsgsA(1)\n\treply := string(matches[0][replyIndex])\n\tcheckMsg(t, matches[0], \"test.request\", \"1\", reply, \"2\", \"hi\")\n\tif reply == \"reply\" {\n\t\tt.Fatalf(\"Expected randomized reply, but got original\")\n\t}\n\n\tsendA(fmt.Sprintf(\"PUB %s 2\\r\\nok\\r\\nPING\\r\\n\", reply))\n\texpectA(pongRe)\n\n\tmatches = expectMsgsB(1)\n\tcheckMsg(t, matches[0], \"reply\", \"1\", \"\", \"2\", \"ok\")\n\n\tif ts := fooA.TotalSubs(); ts != 2 {\n\t\tt.Fatalf(\"Expected two subs to be left on fooA, but got %d\", ts)\n\t}\n\n\troutez, _ := srvA.Routez(&server.RoutezOptions{Subscriptions: true})\n\tr := routez.Routes[0]\n\tif r == nil {\n\t\tt.Fatalf(\"Expected 1 route, got none\")\n\t}\n\tif r.NumSubs != 2 {\n\t\tt.Fatalf(\"Expected 2 subs in the route connection, got %v\", r.NumSubs)\n\t}\n}\n\nfunc TestNewRouteServiceImportDanglingRemoteSubs(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\t// Do Accounts for the servers.\n\tfooA, _ := registerAccounts(t, srvA)\n\tfooB, barB := registerAccounts(t, srvB)\n\n\t// Add in the service export for the requests. Make it public.\n\tif err := fooA.AddServiceExport(\"test.request\", nil); err != nil {\n\t\tt.Fatalf(\"Error adding account service export to client foo: %v\", err)\n\t}\n\n\t// Add export to both.\n\taddServiceExport(\"test.request\", isPublic, fooA, fooB)\n\n\t// Add import abilities to server B's bar account from foo.\n\tif err := barB.AddServiceImport(fooB, \"foo.request\", \"test.request\"); err != nil {\n\t\tt.Fatalf(\"Error adding service import: %v\", err)\n\t}\n\n\t// clientA will be connected to srvA and be the service endpoint, but will not send responses.\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tsendA, expectA := setupConnWithAccount(t, srvA, clientA, \"$foo\")\n\t// Express interest.\n\tsendA(\"SUB test.request 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\t// Now setup client B on srvB who will do a sub from account $bar\n\t// that should map account $foo's foo subject.\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tsendB, expectB := setupConnWithAccount(t, srvB, clientB, \"$bar\")\n\tsendB(\"SUB reply 1\\r\\nPING\\r\\n\")\n\texpectB(pongRe)\n\n\t// Wait for all subs to be propagated (1 on foo and 1 on bar on srvA)\n\t// (note that srvA is not importing)\n\tif err := checkExpectedSubs(2, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// Wait for all subs to be propagated (1 on foo and 2 on bar)\n\tif err := checkExpectedSubs(3, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Send 100 requests from clientB on foo.request,\n\tfor i := 0; i < 100; i++ {\n\t\tsendB(\"PUB foo.request reply 2\\r\\nhi\\r\\n\")\n\t}\n\tsendB(\"PING\\r\\n\")\n\texpectB(pongRe)\n\n\tnumRequests := 0\n\t// Expect the request on A\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tbuf := expectA(msgRe)\n\t\tmatches := msgRe.FindAllSubmatch(buf, -1)\n\t\tnumRequests += len(matches)\n\t\tif numRequests != 100 {\n\t\t\treturn fmt.Errorf(\"Number of requests is %d\", numRequests)\n\t\t}\n\t\treturn nil\n\t})\n\n\texpectNothing(t, clientB)\n\n\t// These reply subjects will be dangling off of $foo account on serverA.\n\t// Remove our service endpoint and wait for the dangling replies to go to zero.\n\tsendA(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpectA(pongRe)\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif ts := fooA.TotalSubs(); ts != 1 {\n\t\t\treturn fmt.Errorf(\"Number of subs is %d, should be only 1\", ts)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNewRouteNoQueueSubscribersBounce(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tncA, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for ncA: %v\\n\", err)\n\t}\n\tdefer ncA.Close()\n\n\tncB, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for ncB: %v\\n\", err)\n\t}\n\tdefer ncB.Close()\n\n\tresponse := []byte(\"I will help you\")\n\n\t// Create a lot of queue subscribers on A, and have one on B.\n\tncB.QueueSubscribe(\"foo.request\", \"workers\", func(m *nats.Msg) {\n\t\tncB.Publish(m.Reply, response)\n\t})\n\n\tfor i := 0; i < 100; i++ {\n\t\tncA.QueueSubscribe(\"foo.request\", \"workers\", func(m *nats.Msg) {\n\t\t\tncA.Publish(m.Reply, response)\n\t\t})\n\t}\n\tncB.Flush()\n\tncA.Flush()\n\n\t// Send all requests from B\n\tnumAnswers := 0\n\tfor i := 0; i < 500; i++ {\n\t\tif _, err := ncB.Request(\"foo.request\", []byte(\"Help Me\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Received an error on Request test [%d]: %s\", i, err)\n\t\t}\n\t\tnumAnswers++\n\t\t// After we have sent 20 close the ncA client.\n\t\tif i == 20 {\n\t\t\tncA.Close()\n\t\t}\n\t}\n\n\tif numAnswers != 500 {\n\t\tt.Fatalf(\"Expect to get all 500 responses, got %d\", numAnswers)\n\t}\n}\n\nfunc TestNewRouteLargeDistinctQueueSubscribers(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tncA, err := nats.Connect(urlA)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for ncA: %v\\n\", err)\n\t}\n\tdefer ncA.Close()\n\n\tncB, err := nats.Connect(urlB)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create connection for ncB: %v\\n\", err)\n\t}\n\tdefer ncB.Close()\n\n\tconst nqsubs = 100\n\n\tqsubs := make([]*nats.Subscription, 100)\n\n\t// Create 100 queue subscribers on B all with different queue groups.\n\tfor i := 0; i < nqsubs; i++ {\n\t\tqg := fmt.Sprintf(\"worker-%d\", i)\n\t\tqsubs[i], _ = ncB.QueueSubscribeSync(\"foo\", qg)\n\t}\n\tncB.Flush()\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif ns := srvA.NumSubscriptions(); ns != 100 {\n\t\t\treturn fmt.Errorf(\"Number of subscriptions is %d\", ns)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Send 10 messages. We should receive 1000 responses.\n\tfor i := 0; i < 10; i++ {\n\t\tncA.Publish(\"foo\", nil)\n\t}\n\tncA.Flush()\n\n\tcheckFor(t, 2*time.Second, 10*time.Millisecond, func() error {\n\t\tfor i := 0; i < nqsubs; i++ {\n\t\t\tif n, _, _ := qsubs[i].Pending(); n != 10 {\n\t\t\t\treturn fmt.Errorf(\"Number of messages is %d\", n)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNewRouteLeafNodeOriginSupport(t *testing.T) {\n\tcontent := `\n\tlisten: 127.0.0.1:-1\n\tcluster { name: xyz, listen: 127.0.0.1:-1 }\n\tleafnodes { listen: 127.0.0.1:-1 }\n\tno_sys_acc: true\n\t`\n\tconf := createConfFile(t, []byte(content))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tgacc, _ := s.LookupAccount(\"$G\")\n\n\tlcontent := `\n\tlisten: 127.0.0.1:-1\n\tcluster { name: ln1, listen: 127.0.0.1:-1 }\n\tleafnodes { remotes = [{ url: nats-leaf://127.0.0.1:%d }] }\n\tno_sys_acc: true\n\t`\n\tlconf := createConfFile(t, []byte(fmt.Sprintf(lcontent, opts.LeafNode.Port)))\n\n\tln, _ := RunServerWithConfig(lconf)\n\tdefer ln.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\tlgacc, _ := ln.LookupAccount(\"$G\")\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"LNOC:22\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tpingPong := func() {\n\t\tt.Helper()\n\t\trouteSend(\"PING\\r\\n\")\n\t\trouteExpect(pongRe)\n\t}\n\n\tinfo := checkInfoMsg(t, rc)\n\tinfo.ID = routeID\n\tinfo.Name = \"\"\n\tinfo.LNOC = true\n\t// Overwrite to false to check that we are getting LS- without origin\n\t// if we are an old server.\n\tinfo.LNOCU = false\n\tb, err := json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\n\trouteSend(fmt.Sprintf(\"INFO %s\\r\\n\", b))\n\trouteExpect(rlsubRe)\n\tpingPong()\n\n\tsendLSProtosFromRoute := func(lnocu bool) {\n\t\tt.Helper()\n\n\t\t// Make sure it can process and LS+\n\t\trouteSend(\"LS+ ln1 $G foo\\r\\n\")\n\t\tpingPong()\n\n\t\t// Check interest is registered on remote server.\n\t\tif !gacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Expected interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// This should not have been sent to the leafnode since same origin cluster.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tif lgacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Did not expect interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// Now unsub. Either act as an old server that does not support origin\n\t\t// in the LS- or as a new server.\n\t\tif lnocu {\n\t\t\trouteSend(\"LS- ln1 $G foo\\r\\n\")\n\t\t} else {\n\t\t\trouteSend(\"LS- $G foo\\r\\n\")\n\t\t}\n\t\tpingPong()\n\n\t\t// Interest should be gone.\n\t\tif gacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Expected no interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// Make sure we did not incorrectly send an interest to the leaf.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tif lgacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Did not expect interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// Repeat with a queue.\n\t\trouteSend(\"LS+ ln1 $G foo bar 1\\r\\n\")\n\t\tpingPong()\n\n\t\tif !gacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Expected interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// This should not have been sent to the leafnode since same origin cluster.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tif lgacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Did not expect interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// Now unsub.\n\t\tif lnocu {\n\t\t\trouteSend(\"LS- ln1 $G foo bar\\r\\n\")\n\t\t} else {\n\t\t\trouteSend(\"LS- $G foo bar\\r\\n\")\n\t\t}\n\t\tpingPong()\n\n\t\t// Subscription should be gone.\n\t\tif gacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Expected no interest on \\\"foo\\\"\")\n\t\t}\n\n\t\t// Make sure we did not incorrectly send an interest to the leaf.\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tif lgacc.SubscriptionInterest(\"foo\") {\n\t\t\tt.Fatalf(\"Did not expect interest on \\\"foo\\\"\")\n\t\t}\n\t}\n\n\t// Check the LS+/- when not supporting origin in LS-\n\tsendLSProtosFromRoute(false)\n\n\t// Create a connection on the leafnode server.\n\tnc, err := nats.Connect(ln.ClientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error connecting %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(\"bar\")\n\t// Let it propagate to the main server\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif !gacc.SubscriptionInterest(\"bar\") {\n\t\t\treturn fmt.Errorf(\"No interest\")\n\t\t}\n\t\treturn nil\n\t})\n\t// For \"bar\"\n\trouteExpect(rlsubRe)\n\n\t// Now pretend like we send a message to the main server over the\n\t// route but from the same origin cluster, should not be delivered\n\t// to the leafnode.\n\n\t// Make sure it can process and LMSG.\n\t// LMSG for routes is like HMSG with an origin cluster before the account.\n\trouteSend(\"LMSG ln1 $G bar 0 2\\r\\nok\\r\\n\")\n\tpingPong()\n\n\t// Let it propagate if not properly truncated.\n\ttime.Sleep(10 * time.Millisecond)\n\tif n, _, _ := sub.Pending(); n != 0 {\n\t\tt.Fatalf(\"Should not have received the message on bar\")\n\t}\n\n\t// Try one with all the bells and whistles.\n\trouteSend(\"LMSG ln1 $G foo + reply bar baz 0 2\\r\\nok\\r\\n\")\n\tpingPong()\n\n\t// Let it propagate if not properly truncated.\n\ttime.Sleep(10 * time.Millisecond)\n\tif n, _, _ := sub.Pending(); n != 0 {\n\t\tt.Fatalf(\"Should not have received the message on bar\")\n\t}\n\n\t// Now unsubscribe, we should receive an LS- without origin.\n\tsub.Unsubscribe()\n\trouteExpect(lunsubRe)\n\n\t// Quick check for queues\n\tsub, _ = nc.QueueSubscribeSync(\"baz\", \"bat\")\n\t// Let it propagate to the main server\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif !gacc.SubscriptionInterest(\"baz\") {\n\t\t\treturn fmt.Errorf(\"No interest\")\n\t\t}\n\t\treturn nil\n\t})\n\t// For \"baz\"\n\trouteExpect(rlsubRe)\n\tsub.Unsubscribe()\n\trouteExpect(lunsubRe)\n\n\t// Restart our routed server, but this time indicate support\n\t// for LS- with origin cluster.\n\trc.Close()\n\trc = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteSend, routeExpect = setupRouteEx(t, rc, opts, routeID)\n\n\tinfo = checkInfoMsg(t, rc)\n\tinfo.ID = routeID\n\tinfo.Name = \"\"\n\t// These should be already set to true since the server that sends the\n\t// INFO has them enabled, but just be explicit.\n\tinfo.LNOC = true\n\tinfo.LNOCU = true\n\tb, err = json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\n\trouteSend(fmt.Sprintf(\"INFO %s\\r\\n\", b))\n\trouteExpect(rlsubRe)\n\tpingPong()\n\n\t// Check the LS+/LS-\n\tsendLSProtosFromRoute(true)\n\n\tsub, _ = nc.SubscribeSync(\"bar\")\n\trouteExpect(rlsubRe)\n\tsub.Unsubscribe()\n\trouteExpect(rlunsubRe)\n\n\tsub, _ = nc.QueueSubscribeSync(\"baz\", \"bat\")\n\trouteExpect(rlsubRe)\n\tsub.Unsubscribe()\n\trouteExpect(rlunsubRe)\n}\n\n// Check that real duplicate subscription (that is, sent by client with same sid)\n// are ignored and do not register multiple shadow subscriptions.\nfunc TestNewRouteDuplicateSubscription(t *testing.T) {\n\t// This is same test than TestNewRouteStreamImport but calling \"SUB foo 1\" twice.\n\ttestNewRouteStreamImport(t, true)\n\n\topts := LoadConfig(\"./configs/new_cluster.conf\")\n\topts.DisableShortFirstPing = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_DUPLICATE:22\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\tsendRouteInfo(t, rc, routeSend, routeID)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\tsend, expect := setupConn(t, c)\n\n\t// Create a real duplicate subscriptions (same sid)\n\tsend(\"SUB foo 1\\r\\nSUB foo 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\t// Route should receive single RS+\n\trouteExpect(rsubRe)\n\n\t// Unsubscribe.\n\tsend(\"UNSUB 1\\r\\nPING\\r\\n\")\n\texpect(pongRe)\n\n\t// Route should receive RS-.\n\t// With defect, only 1 subscription would be found during the unsubscribe,\n\t// however route map would have been updated twice when processing the\n\t// duplicate SUB, which means that the RS- would not be received because\n\t// the count would still be 1.\n\trouteExpect(runsubRe)\n}\n" }, { "path": "test/norace_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests\n\npackage test\n\nimport (\n\t\"context\"\n\tcrand \"crypto/rand\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nuid\"\n)\n\n// IMPORTANT: Tests in this file are not executed when running with the -race flag.\n// The test name should be prefixed with TestNoRace so we can run only\n// those tests: go test -run=TestNoRace ...\n\nfunc TestNoRaceRouteSendSubs(t *testing.T) {\n\ttemplate := `\n\t\t\tport: -1\n\t\t\twrite_deadline: \"2s\"\n\t\t\tcluster {\n\t\t\t\tport: -1\n\t\t\t\tpool_size: -1\n\t\t\t\tcompression: disabled\n\t\t\t\t%s\n\t\t\t}\n\t\t\tno_sys_acc: true\n\t`\n\tcfa := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\tsrvA, optsA := RunServerWithConfig(cfa)\n\tsrvA.Shutdown()\n\toptsA.DisableShortFirstPing = true\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tcfb := createConfFile(t, []byte(fmt.Sprintf(template, \"\")))\n\tsrvB, optsB := RunServerWithConfig(cfb)\n\tsrvB.Shutdown()\n\toptsB.DisableShortFirstPing = true\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tclientA := createClientConn(t, optsA.Host, optsA.Port)\n\tdefer clientA.Close()\n\n\tclientASend, clientAExpect := setupConn(t, clientA)\n\tclientASend(\"PING\\r\\n\")\n\tclientAExpect(pongRe)\n\n\tclientB := createClientConn(t, optsB.Host, optsB.Port)\n\tdefer clientB.Close()\n\n\tclientBSend, clientBExpect := setupConn(t, clientB)\n\tclientBSend(\"PING\\r\\n\")\n\tclientBExpect(pongRe)\n\n\t// total number of subscriptions per server\n\ttotalPerServer := 100000\n\tfor i := 0; i < totalPerServer/2; i++ {\n\t\tproto := fmt.Sprintf(\"SUB foo.%d %d\\r\\n\", i, i*2+1)\n\t\tclientASend(proto)\n\t\tclientBSend(proto)\n\t\tproto = fmt.Sprintf(\"SUB bar.%d queue.%d %d\\r\\n\", i, i, i*2+2)\n\t\tclientASend(proto)\n\t\tclientBSend(proto)\n\t}\n\tclientASend(\"PING\\r\\n\")\n\tclientAExpect(pongRe)\n\tclientBSend(\"PING\\r\\n\")\n\tclientBExpect(pongRe)\n\n\tif err := checkExpectedSubs(totalPerServer, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\troutes := fmt.Sprintf(`\n\t\troutes: [\n\t\t\t\"nats://%s:%d\"\n\t\t]\n\t`, optsA.Cluster.Host, optsA.Cluster.Port)\n\tif err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, routes)), 0600); err != nil {\n\t\tt.Fatalf(\"Error rewriting B's config file: %v\", err)\n\t}\n\tif err := srvB.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\tcheckClusterFormed(t, srvA, srvB)\n\tif err := checkExpectedSubs(2*totalPerServer, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\tcheckSlowConsumers := func(t *testing.T) {\n\t\tt.Helper()\n\t\tif srvB.NumSlowConsumers() != 0 || srvA.NumSlowConsumers() != 0 {\n\t\t\tt.Fatalf(\"Expected no slow consumers, got %d for srvA and %df for srvB\",\n\t\t\t\tsrvA.NumSlowConsumers(), srvB.NumSlowConsumers())\n\t\t}\n\t}\n\tcheckSlowConsumers(t)\n\n\ttype sender struct {\n\t\tc net.Conn\n\t\tsf sendFun\n\t\tef expectFun\n\t}\n\tvar senders []*sender\n\tcreateSenders := func(t *testing.T, host string, port int) {\n\t\tt.Helper()\n\t\tfor i := 0; i < 25; i++ {\n\t\t\ts := &sender{}\n\t\t\ts.c = createClientConn(t, host, port)\n\t\t\ts.sf, s.ef = setupConn(t, s.c)\n\t\t\ts.sf(\"PING\\r\\n\")\n\t\t\ts.ef(pongRe)\n\t\t\tsenders = append(senders, s)\n\t\t}\n\t}\n\tcreateSenders(t, optsA.Host, optsA.Port)\n\tcreateSenders(t, optsB.Host, optsB.Port)\n\tfor _, s := range senders {\n\t\tdefer s.c.Close()\n\t}\n\n\t// Now create SUBs on A and B for \"ping.replies\" and simulate\n\t// that there are thousands of replies being sent on\n\t// both sides.\n\tcreateSubOnReplies := func(t *testing.T, host string, port int) net.Conn {\n\t\tt.Helper()\n\t\tc := createClientConn(t, host, port)\n\t\tsend, expect := setupConn(t, c)\n\t\tsend(\"SUB ping.replies 123456789\\r\\nPING\\r\\n\")\n\t\texpect(pongRe)\n\t\treturn c\n\t}\n\trequestorOnA := createSubOnReplies(t, optsA.Host, optsA.Port)\n\tdefer requestorOnA.Close()\n\n\trequestorOnB := createSubOnReplies(t, optsB.Host, optsB.Port)\n\tdefer requestorOnB.Close()\n\n\tif err := checkExpectedSubs(2*totalPerServer+2, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\ttotalReplies := 120000\n\tpayload := sizedBytes(400)\n\texpectedBytes := (len(fmt.Sprintf(\"MSG ping.replies 123456789 %d\\r\\n\\r\\n\", len(payload))) + len(payload)) * totalReplies\n\tch := make(chan error, 2)\n\trecvReplies := func(c net.Conn) {\n\t\tvar buf [32 * 1024]byte\n\n\t\tfor total := 0; total < expectedBytes; {\n\t\t\tn, err := c.Read(buf[:])\n\t\t\tif err != nil {\n\t\t\t\tch <- fmt.Errorf(\"read error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttotal += n\n\t\t}\n\t\tch <- nil\n\t}\n\tgo recvReplies(requestorOnA)\n\tgo recvReplies(requestorOnB)\n\n\twg := sync.WaitGroup{}\n\twg.Add(len(senders))\n\treplyMsg := fmt.Sprintf(\"PUB ping.replies %d\\r\\n%s\\r\\n\", len(payload), payload)\n\tfor _, s := range senders {\n\t\tgo func(s *sender, count int) {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\ts.sf(replyMsg)\n\t\t\t}\n\t\t\ts.sf(\"PING\\r\\n\")\n\t\t\ts.ef(pongRe)\n\t\t}(s, totalReplies/len(senders))\n\t}\n\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase e := <-ch:\n\t\t\tif e != nil {\n\t\t\t\tt.Fatalf(\"Error: %v\", e)\n\t\t\t}\n\t\tcase <-time.After(10 * time.Second):\n\t\t\tt.Fatalf(\"Did not receive all %v replies\", totalReplies)\n\t\t}\n\t}\n\tcheckSlowConsumers(t)\n\twg.Wait()\n\tcheckSlowConsumers(t)\n\n\t// Let's remove the route and do a config reload.\n\t// Otherwise, on test shutdown the client close\n\t// will cause the server to try to send unsubs and\n\t// this can delay the test.\n\tif err := os.WriteFile(cfb, []byte(fmt.Sprintf(template, \"\")), 0600); err != nil {\n\t\tt.Fatalf(\"Error rewriting B's config file: %v\", err)\n\t}\n\tif err := srvB.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n}\n\nfunc TestNoRaceDynamicResponsePermsMemory(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\tdefer srv.Shutdown()\n\n\t// We will test the timeout to make sure that we are not showing excessive growth\n\t// when a reply subject is not utilized by the responder.\n\n\t// Alice can do anything, so she will be our requestor\n\trc := createClientConn(t, opts.Host, opts.Port)\n\tdefer rc.Close()\n\texpectAuthRequired(t, rc)\n\tdoAuthConnect(t, rc, \"\", \"alice\", DefaultPass)\n\texpectResult(t, rc, okRe)\n\n\t// MY_STREAM_SERVICE has an expiration of 10ms for the response permissions.\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"svcb\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB my.service.req 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tvar m runtime.MemStats\n\n\truntime.GC()\n\truntime.ReadMemStats(&m)\n\tpta := m.TotalAlloc\n\n\t// Need this so we do not blow the allocs on expectResult which makes 32k each time.\n\texpBuf := make([]byte, 32768)\n\texpect := func(c net.Conn, re *regexp.Regexp) {\n\t\tt.Helper()\n\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\tn, _ := c.Read(expBuf)\n\t\tc.SetReadDeadline(time.Time{})\n\t\tbuf := expBuf[:n]\n\t\tif !re.Match(buf) {\n\t\t\tt.Fatalf(\"Response did not match expected: \\n\\tReceived:'%q'\\n\\tExpected:'%s'\", buf, re)\n\t\t}\n\t}\n\n\t// Now send off some requests. We will not answer them and this will build up reply\n\t// permissions in the server.\n\tfor i := 0; i < 10000; i++ {\n\t\tpub := fmt.Sprintf(\"PUB my.service.req resp.%d 2\\r\\nok\\r\\n\", i)\n\t\tsendProto(t, rc, pub)\n\t\texpect(rc, okRe)\n\t\texpect(c, msgRe)\n\t}\n\n\tconst max = 20 * 1024 * 1024 // 20MB\n\tcheckFor(t, time.Second, 25*time.Millisecond, func() error {\n\t\truntime.GC()\n\t\truntime.ReadMemStats(&m)\n\t\tused := m.TotalAlloc - pta\n\t\tif used > max {\n\t\t\treturn fmt.Errorf(\"Using too much memory, expect < 20MB, got %dMB\", used/(1024*1024))\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceLargeClusterMem(t *testing.T) {\n\t// Try to clean up.\n\truntime.GC()\n\tvar m runtime.MemStats\n\truntime.ReadMemStats(&m)\n\tpta := m.TotalAlloc\n\n\topts := func() *server.Options {\n\t\to := DefaultTestOptions\n\t\to.Host = \"127.0.0.1\"\n\t\to.Port = -1\n\t\to.Cluster.Host = o.Host\n\t\to.Cluster.Port = -1\n\t\treturn &o\n\t}\n\n\tvar servers []*server.Server\n\n\t// Create seed first.\n\to := opts()\n\ts := RunServer(o)\n\tservers = append(servers, s)\n\n\t// For connecting to seed server above.\n\trouteAddr := fmt.Sprintf(\"nats-route://%s:%d\", o.Cluster.Host, o.Cluster.Port)\n\trurl, _ := url.Parse(routeAddr)\n\troutes := []*url.URL{rurl}\n\n\tnumServers := 15\n\n\tfor i := 1; i < numServers; i++ {\n\t\to := opts()\n\t\to.Routes = routes\n\t\ts := RunServer(o)\n\t\tservers = append(servers, s)\n\t}\n\tcheckClusterFormed(t, servers...)\n\n\t// Calculate in MB what we are using now.\n\tconst max = 80 * 1024 * 1024 // 80MB\n\truntime.ReadMemStats(&m)\n\tused := m.TotalAlloc - pta\n\tif used > max {\n\t\tt.Fatalf(\"Cluster using too much memory, expect < 80MB, got %dMB\", used/(1024*1024))\n\t}\n\n\tfor _, s := range servers {\n\t\ts.Shutdown()\n\t}\n}\n\n// Make sure we have the correct remote state when dealing with queue subscribers\n// across many client connections.\nfunc TestNoRaceQueueSubWeightOrderMultipleConnections(t *testing.T) {\n\topts, err := server.ProcessConfigFile(\"./configs/new_cluster.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.DisableShortFirstPing = true\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\t// Create 100 connections to s\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tclients := make([]*nats.Conn, 0, 100)\n\tfor i := 0; i < 100; i++ {\n\t\tnc, err := nats.Connect(url, nats.NoReconnect())\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error connecting: %v\", err)\n\t\t}\n\t\tdefer nc.Close()\n\t\tclients = append(clients, nc)\n\t}\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RTEST_NEW:22\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\tinfo := checkInfoMsg(t, rc)\n\n\tinfo.ID = routeID\n\tinfo.Name = routeID\n\n\tb, err := json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\t// Send our INFO and wait for a PONG. This will prevent a race\n\t// where server will have started processing queue subscriptions\n\t// and then processes the route's INFO (sending its current subs)\n\t// followed by updates.\n\trouteSend(fmt.Sprintf(\"INFO %s\\r\\nPING\\r\\n\", b))\n\trouteExpect(pongRe)\n\n\tstart := make(chan bool)\n\tfor _, nc := range clients {\n\t\tgo func(nc *nats.Conn) {\n\t\t\t<-start\n\t\t\t// Now create 100 identical queue subscribers on each connection.\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tif _, err := nc.QueueSubscribe(\"foo\", \"bar\", func(_ *nats.Msg) {}); err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tnc.Flush()\n\t\t}(nc)\n\t}\n\tclose(start)\n\n\t// We did have this where we wanted to get every update, but now with optimizations\n\t// we just want to make sure we always are increasing and that a previous update to\n\t// a lesser queue weight is never delivered for this test.\n\tmaxExpected := 10000\n\tupdates := 0\n\tfor qw := 0; qw < maxExpected; {\n\t\tbuf := routeExpect(rsubRe)\n\t\tmatches := rsubRe.FindAllSubmatch(buf, -1)\n\t\tfor _, m := range matches {\n\t\t\tif len(m) != 5 {\n\t\t\t\tt.Fatalf(\"Expected a weight for the queue group\")\n\t\t\t}\n\t\t\tnqw, err := strconv.Atoi(string(m[4]))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Got an error converting queue weight: %v\", err)\n\t\t\t}\n\t\t\t// Make sure the new value only increases, ok to skip since we will\n\t\t\t// optimize this now, but needs to always be increasing.\n\t\t\tif nqw <= qw {\n\t\t\t\tt.Fatalf(\"Was expecting increasing queue weight after %d, got %d\", qw, nqw)\n\t\t\t}\n\t\t\tqw = nqw\n\t\t\tupdates++\n\t\t}\n\t}\n\tif updates >= maxExpected {\n\t\tt.Fatalf(\"Was not expecting all %v updates to be received\", maxExpected)\n\t}\n}\n\nfunc TestNoRaceClusterLeaksSubscriptions(t *testing.T) {\n\tsrvA, srvB, optsA, optsB := runServers(t)\n\tdefer srvA.Shutdown()\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\turlA := fmt.Sprintf(\"nats://%s:%d/\", optsA.Host, optsA.Port)\n\turlB := fmt.Sprintf(\"nats://%s:%d/\", optsB.Host, optsB.Port)\n\n\tnumResponses := 100\n\trepliers := make([]*nats.Conn, 0, numResponses)\n\n\tvar noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {}\n\n\t// Create 100 repliers\n\tfor i := 0; i < 50; i++ {\n\t\tnc1, _ := nats.Connect(urlA)\n\t\tdefer nc1.Close()\n\t\tnc1.SetErrorHandler(noOpErrHandler)\n\t\tnc2, _ := nats.Connect(urlB)\n\t\tdefer nc2.Close()\n\t\tnc2.SetErrorHandler(noOpErrHandler)\n\t\trepliers = append(repliers, nc1, nc2)\n\t\tnc1.Subscribe(\"test.reply\", func(m *nats.Msg) {\n\t\t\tm.Respond([]byte(\"{\\\"sender\\\": 22 }\"))\n\t\t})\n\t\tnc2.Subscribe(\"test.reply\", func(m *nats.Msg) {\n\t\t\tm.Respond([]byte(\"{\\\"sender\\\": 33 }\"))\n\t\t})\n\t\tnc1.Flush()\n\t\tnc2.Flush()\n\t}\n\n\tservers := fmt.Sprintf(\"%s, %s\", urlA, urlB)\n\treq := sizedBytes(8 * 1024)\n\n\t// Now run a requestor in a loop, creating and tearing down each time to\n\t// simulate running a modified nats-req.\n\tdoReq := func() {\n\t\tmsgs := make(chan *nats.Msg, 1)\n\t\tinbox := nats.NewInbox()\n\t\tgrp := nuid.Next()\n\t\t// Create 8 queue Subscribers for responses.\n\t\tfor i := 0; i < 8; i++ {\n\t\t\tnc, _ := nats.Connect(servers)\n\t\t\tnc.SetErrorHandler(noOpErrHandler)\n\t\t\tnc.ChanQueueSubscribe(inbox, grp, msgs)\n\t\t\tnc.Flush()\n\t\t\tdefer nc.Close()\n\t\t}\n\t\tnc, _ := nats.Connect(servers)\n\t\tnc.SetErrorHandler(noOpErrHandler)\n\t\tnc.PublishRequest(\"test.reply\", inbox, req)\n\t\tdefer nc.Close()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)\n\t\tdefer cancel()\n\n\t\tvar received int\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-msgs:\n\t\t\t\treceived++\n\t\t\t\tif received >= numResponses {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tvar wg sync.WaitGroup\n\n\tdoRequests := func(n int) {\n\t\tfor i := 0; i < n; i++ {\n\t\t\tdoReq()\n\t\t}\n\t\twg.Done()\n\t}\n\n\tconcurrent := 10\n\twg.Add(concurrent)\n\tfor i := 0; i < concurrent; i++ {\n\t\tgo doRequests(10)\n\t}\n\twg.Wait()\n\n\t// Close responders too, should have zero(0) subs attached to routes.\n\tfor _, nc := range repliers {\n\t\tnc.Close()\n\t}\n\n\t// Make sure no clients remain. This is to make sure the test is correct and that\n\t// we have closed all the client connections.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tv1, _ := srvA.Varz(nil)\n\t\tv2, _ := srvB.Varz(nil)\n\t\tif v1.Connections != 0 || v2.Connections != 0 {\n\t\t\treturn fmt.Errorf(\"We have lingering client connections %d:%d\", v1.Connections, v2.Connections)\n\t\t}\n\t\treturn nil\n\t})\n\n\tloadRoutez := func() (*server.Routez, *server.Routez) {\n\t\tv1, err := srvA.Routez(&server.RoutezOptions{Subscriptions: true})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error getting Routez: %v\", err)\n\t\t}\n\t\tv2, err := srvB.Routez(&server.RoutezOptions{Subscriptions: true})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error getting Routez: %v\", err)\n\t\t}\n\t\treturn v1, v2\n\t}\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tr1, r2 := loadRoutez()\n\t\tif r1.Routes[0].NumSubs != 0 {\n\t\t\treturn fmt.Errorf(\"Leaked %d subs: %+v\", r1.Routes[0].NumSubs, r1.Routes[0].Subs)\n\t\t}\n\t\tif r2.Routes[0].NumSubs != 0 {\n\t\t\treturn fmt.Errorf(\"Leaked %d subs: %+v\", r2.Routes[0].NumSubs, r2.Routes[0].Subs)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestNoRaceLeafNodeSmapUpdate(t *testing.T) {\n\ts, opts := runLeafServer()\n\tdefer s.Shutdown()\n\n\t// Create a client on leaf server\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\tcsend, cexpect := setupConn(t, c)\n\n\tnumSubs := make(chan int, 1)\n\tdoneCh := make(chan struct{}, 1)\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 1; ; i++ {\n\t\t\tcsend(fmt.Sprintf(\"SUB foo.%d %d\\r\\n\", i, i))\n\t\t\tselect {\n\t\t\tcase <-doneCh:\n\t\t\t\tnumSubs <- i\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(5 * time.Millisecond)\n\t// Create leaf node\n\tlc := createLeafConn(t, opts.LeafNode.Host, opts.LeafNode.Port)\n\tdefer lc.Close()\n\n\tsetupConn(t, lc)\n\tcheckLeafNodeConnected(t, s)\n\n\tclose(doneCh)\n\tns := <-numSubs\n\tcsend(\"PING\\r\\n\")\n\tcexpect(pongRe)\n\twg.Wait()\n\n\t// Make sure we receive as many LS+ protocols (since all subs are unique).\n\t// But we also have to count for LDS subject.\n\t// There may be so many protocols and partials, that expectNumberOfProtos may\n\t// not work. Do a manual search here.\n\tcheckLS := func(proto string, expected int) {\n\t\tt.Helper()\n\t\tp := []byte(proto)\n\t\tcur := 0\n\t\tbuf := make([]byte, 32768)\n\t\tfor ls := 0; ls < expected; {\n\t\t\tlc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\tn, err := lc.Read(buf)\n\t\t\tlc.SetReadDeadline(time.Time{})\n\t\t\tif err == nil && n > 0 {\n\t\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\t\tif buf[i] == p[cur] {\n\t\t\t\t\t\tcur++\n\t\t\t\t\t\tif cur == len(p) {\n\t\t\t\t\t\t\tls++\n\t\t\t\t\t\t\tcur = 0\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tcur = 0\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err != nil || ls > expected {\n\t\t\t\tt.Fatalf(\"Expected %v %sgot %v, err: %v\", expected, proto, ls, err)\n\t\t\t}\n\t\t}\n\t}\n\tcheckLS(\"LS+ \", ns+1)\n\n\t// Now unsub all those subs...\n\tfor i := 1; i <= ns; i++ {\n\t\tcsend(fmt.Sprintf(\"UNSUB %d\\r\\n\", i))\n\t}\n\tcsend(\"PING\\r\\n\")\n\tcexpect(pongRe)\n\n\t// Expect that many LS-\n\tcheckLS(\"LS- \", ns)\n}\n\nfunc TestNoRaceSlowProxy(t *testing.T) {\n\tt.Skip()\n\n\topts := DefaultTestOptions\n\topts.Port = -1\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\trttTarget := 22 * time.Millisecond\n\tbwTarget := 10 * 1024 * 1024 / 8 // 10mbit\n\n\tsp := newSlowProxy(rttTarget, bwTarget, bwTarget, &opts)\n\tdefer sp.stop()\n\n\tnc, err := nats.Connect(sp.clientURL())\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tdoRTT := func() time.Duration {\n\t\tt.Helper()\n\t\tconst samples = 5\n\t\tvar total time.Duration\n\t\tfor i := 0; i < samples; i++ {\n\t\t\trtt, _ := nc.RTT()\n\t\t\ttotal += rtt\n\t\t}\n\t\treturn total / samples\n\t}\n\n\trtt := doRTT()\n\tif rtt < rttTarget || rtt > (rttTarget*3/2) {\n\t\tt.Fatalf(\"rtt is out of range, target of %v, actual %v\", rttTarget, rtt)\n\t}\n\n\t// Now test send BW.\n\tconst payloadSize = 64 * 1024\n\tvar payload [payloadSize]byte\n\tcrand.Read(payload[:])\n\n\t// 5MB total.\n\tbytesSent := (5 * 1024 * 1024)\n\ttoSend := bytesSent / payloadSize\n\n\tstart := time.Now()\n\tfor i := 0; i < toSend; i++ {\n\t\tnc.Publish(\"z\", payload[:])\n\t}\n\tnc.Flush()\n\ttt := time.Since(start)\n\tbps := float64(bytesSent) / tt.Seconds()\n\tmin, max := float64(bwTarget)*0.8, float64(bwTarget)*1.25\n\tif bps < min || bps > max {\n\t\tt.Fatalf(\"bps is off, target is %v, actual is %v\", bwTarget, bps)\n\t}\n}\n" }, { "path": "test/ocsp_peer_test.go", "content": "// Copyright 2023-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\t. \"github.com/nats-io/nats-server/v2/internal/ocsp\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"golang.org/x/crypto/ocsp\"\n)\n\nfunc NewOCSPResponderRootCA(t *testing.T) *http.Server {\n\tt.Helper()\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/caocsp/caocsp_cert.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/caocsp/private/caocsp_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\treturn NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, \"127.0.0.1:8888\")\n}\n\nfunc NewOCSPResponderIntermediateCA1(t *testing.T) *http.Server {\n\tt.Helper()\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\"\n\treturn NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, \"127.0.0.1:18888\")\n}\n\nfunc NewOCSPResponderIntermediateCA1Undelegated(t *testing.T) *http.Server {\n\tt.Helper()\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\"\n\tissuerCertKey := \"configs/certs/ocsp_peer/mini-ca/intermediate1/private/intermediate1_keypair.pem\"\n\treturn NewOCSPResponderCustomAddress(t, issuerCertPEM, issuerCertKey, \"127.0.0.1:18888\")\n}\n\nfunc NewOCSPResponderBadDelegateIntermediateCA1(t *testing.T) *http.Server {\n\tt.Helper()\n\t// UserA2 is a cert issued by intermediate1, but intermediate1 did not add OCSP signing extension\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/client1/UserA2_bundle.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/client1/private/UserA2_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\"\n\treturn NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, \"127.0.0.1:18888\")\n}\n\nfunc NewOCSPResponderIntermediateCA2(t *testing.T) *http.Server {\n\tt.Helper()\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp2/ocsp2_bundle.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp2/private/ocsp2_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem\"\n\treturn NewOCSPResponderDesignatedCustomAddress(t, issuerCertPEM, respCertPEM, respKeyPEM, \"127.0.0.1:28888\")\n}\n\n// TestOCSPPeerGoodClients is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates)\n// and default ocsp_cache implementation and oscp_cache=false configuration\nfunc TestOCSPPeerGoodClients(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# default ocsp_cache since omitted\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration, non-default ca_timeout\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Default cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 2\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# default ocsp_cache since omitted\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Explicit true cache: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Short form configuration\n\t\t\t\tocsp_cache: true\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestOCSPPeerUnknownClient is test of NATS client that is OCSP status Unknown from its OCSP Responder\nfunc TestOCSPPeerUnknownClient(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Default cache, mTLS OCSP peer check on inbound client connection, client unknown to intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tt.Errorf(\"Expected connection error, fell through\")\n\t\t})\n\t}\n}\n\n// TestOCSPPeerRevokedClient is test of NATS client that is OCSP status Revoked from its OCSP Responder\nfunc TestOCSPPeerRevokedClient(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Revoked)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so this revoked client should NOT be able to connect\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Explicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Explicit disable of OCSP peer check\n\t\t\t\t\tocsp_peer: false\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Implicit disable, mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Implicit disable of OCSP peer check (i.e. not configured)\n\t\t\t\t\t# ocsp_peer: false\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Explicit disable (long form), mTLS OCSP peer check on inbound client connection, client revoked by intermediate CA 1 but no OCSP check\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Explicit disable of OCSP peer check, long form\n\t\t\t\t\tocsp_peer: { verify: false }\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerUnknownAndRevokedIntermediate test of NATS client that is OCSP good but either its intermediate is unknown or revoked\nfunc TestOCSPPeerUnknownAndRevokedIntermediate(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Revoked)\n\t// No test OCSP status set on intermediate2, so unknown\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, client's intermediate is revoked\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, client's intermediate is unknown'\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tt.Errorf(\"Expected connection error, fell through\")\n\t\t})\n\t}\n}\n\n// TestOCSPPeerLeafGood tests Leaf Spoke peer checking Leaf Hub, Leaf Hub peer checking Leaf Spoke, and both peer checking\nfunc TestOCSPPeerLeafGood(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\thubconfig string\n\t\tspokeconfig string\n\t\texpected int\n\t}{\n\t\t{\n\t\t\t\"OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t1,\n\t\t},\n\t\t{\n\t\t\t\"OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t1,\n\t\t},\n\t\t{\n\t\t\t\"OCSP peer check bi-directionally\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t1,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\thubcontent := test.hubconfig\n\t\t\thubconf := createConfFile(t, []byte(hubcontent))\n\t\t\thub, _ := RunServerWithConfig(hubconf)\n\t\t\tdefer hub.Shutdown()\n\n\t\t\tspokecontent := test.spokeconfig\n\t\t\tspokeconf := createConfFile(t, []byte(spokecontent))\n\t\t\tspoke, _ := RunServerWithConfig(spokeconf)\n\t\t\tdefer spoke.Shutdown()\n\n\t\t\tcheckLeafNodeConnectedCount(t, hub, test.expected)\n\t\t})\n\t}\n}\n\n// TestOCSPPeerLeafReject tests rejected Leaf Hub, rejected Leaf Spoke, and both rejecting each other\nfunc TestOCSPPeerLeafReject(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\", ocsp.Revoked)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem\", ocsp.Revoked)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\thubconfig string\n\t\tspokeconfig string\n\t\texpected int\n\t}{\n\t\t{\n\t\t\t\"OCSP peer check on Leaf Hub by Leaf Spoke (TLS client OCSP verification of TLS server)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"OCSP peer check on Leaf Spoke by Leaf Hub (TLS server OCSP verification of TLS client)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"OCSP peer check bi-directionally\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tlisten: 127.0.0.1:7444\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tleaf: {\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats://127.0.0.1:7444\",\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t0,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\thubcontent := test.hubconfig\n\t\t\thubconf := createConfFile(t, []byte(hubcontent))\n\t\t\thub, _ := RunServerWithConfig(hubconf)\n\t\t\tdefer hub.Shutdown()\n\t\t\tspokecontent := test.spokeconfig\n\t\t\tspokeconf := createConfFile(t, []byte(spokecontent))\n\t\t\tspoke, _ := RunServerWithConfig(spokeconf)\n\t\t\tdefer spoke.Shutdown()\n\t\t\t// Need to inject some time for leaf connection attempts to complete, could refine this to better\n\t\t\t// negative test\n\t\t\ttime.Sleep(2000 * time.Millisecond)\n\t\t\tcheckLeafNodeConnectedCount(t, hub, test.expected)\n\t\t})\n\t}\n}\n\nfunc checkLeafNodeConnectedCount(t testing.TB, s *server.Server, lnCons int) {\n\tt.Helper()\n\tcheckFor(t, 5*time.Second, 15*time.Millisecond, func() error {\n\t\tif nln := s.NumLeafNodes(); nln != lnCons {\n\t\t\treturn fmt.Errorf(\"expected %d connected leafnode(s) for server %q, got %d\",\n\t\t\t\tlnCons, s.ID(), nln)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// TestOCSPPeerGoodClientsNoneCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates)\n// and ocsp cache type of none (no-op)\nfunc TestOCSPPeerGoodClientsNoneCache(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem\", ocsp.Good)\n\n\tdeleteLocalStore(t, \"\")\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"None cache explicit long form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Long form configuration\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: none\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"None cache explicit short form: mTLS OCSP peer check on inbound client connection, client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration\n\t\t\t\tocsp_cache: false\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestOCSPPeerGoodClientsLocalCache is test of two NATS client (AIA enabled at leaf and cert) under good path (different intermediates)\n// and leveraging the local ocsp cache type\nfunc TestOCSPPeerGoodClientsLocalCache(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration, local as default\n\t\t\t\tocsp_cache: true\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Local cache long form: mTLS OCSP peer check on inbound client connection, UserB1 client of intermediate CA 2\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t\t# Long form configuration\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Cleanup any previous test that saved a local cache\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnc.Close()\n\n\t\t\tv := monitorGetVarzHelper(t, 8222)\n\t\t\tif v.OCSPResponseCache == nil {\n\t\t\t\tt.Fatalf(\"Expected OCSP statistics to be in varz\")\n\t\t\t}\n\t\t\tif v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Responses != 2 {\n\t\t\t\tt.Errorf(\"Expected cache misses and cache items to be 2, got %d and %d\", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses)\n\t\t\t}\n\n\t\t\t// Should get a cache hit now\n\t\t\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\n\t\t\tv = monitorGetVarzHelper(t, 8222)\n\t\t\tif v.OCSPResponseCache == nil {\n\t\t\t\tt.Fatalf(\"Expected OCSP statistics to be in varz\")\n\t\t\t}\n\t\t\tif v.OCSPResponseCache.Misses != 2 || v.OCSPResponseCache.Hits != 2 || v.OCSPResponseCache.Responses != 2 {\n\t\t\t\tt.Errorf(\"Expected cache misses, hits and cache items to be 2, got %d and %d and %d\", v.OCSPResponseCache.Misses, v.OCSPResponseCache.Hits, v.OCSPResponseCache.Responses)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPPeerMonitor(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\tNATSClient bool\n\t\tWSClient bool\n\t\tMQTTClient bool\n\t\tLeafClient bool\n\t\tLeafRemotes bool\n\t\tNumTrueLeafRemotes int\n\t}{\n\t\t{\n\t\t\t\"Monitor peer config setting on NATS client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\t# Default cache configuration\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"Monitor peer config setting on Websockets client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\t# Default cache configuration\n\t\t\t\twebsocket: {\n\t\t\t\t\tport: 8443\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Long form configuration\n\t\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"Monitor peer config setting on MQTT client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\t# Default cache configuration\n\t\t\t\t# Required for MQTT\n\t\t\t\tserver_name: \"my_mqtt_server\"\n\t\t\t\tjetstream: {\n\t\t\t\t\tenabled: true\n\t\t\t\t}\n\t\t\t\tmqtt: {\n\t\t\t\t\tport: 1883\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Long form configuration\n\t\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"Monitor peer config setting on Leaf client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\t# Default cache configuration\n\t\t\t\tleaf: {\n\t\t\t\t\tport: 7422\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Long form configuration\n\t\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t\t0,\n\t\t},\n\t\t{\n\t\t\t\"Monitor peer config on some Leaf Remotes as well as Leaf client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\t# Default cache configuration\n\t\t\t\tleaf: {\n\t\t\t\t\tport: 7422\n\t\t\t\t\ttls: {\n\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t# Long form configuration\n\t\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tremotes: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats-leaf://bogus:7422\"\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Long form configuration\n\t\t\t\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\t\t\t\tverify: true\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats-leaf://anotherbogus:7422\"\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Short form configuration\n\t\t\t\t\t\t\t\tocsp_peer: true\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\turl: \"nats-leaf://yetanotherbogus:7422\"\n\t\t\t\t\t\t\ttls: {\n\t\t\t\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\t\t\t\ttimeout: 5\n\t\t\t\t\t\t\t\t# Peer not configured (default false)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t`,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t\ttrue,\n\t\t\t2,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tv := monitorGetVarzHelper(t, 8222)\n\t\t\tif test.NATSClient {\n\t\t\t\tif !v.TLSOCSPPeerVerify {\n\t\t\t\t\tt.Fatalf(\"Expected NATS Client TLSOCSPPeerVerify to be true, got false\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.WSClient {\n\t\t\t\tif !v.Websocket.TLSOCSPPeerVerify {\n\t\t\t\t\tt.Fatalf(\"Expected WS Client TLSOCSPPeerVerify to be true, got false\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.LeafClient {\n\t\t\t\tif !v.LeafNode.TLSOCSPPeerVerify {\n\t\t\t\t\tt.Fatalf(\"Expected Leaf Client TLSOCSPPeerVerify to be true, got false\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.LeafRemotes {\n\t\t\t\tcnt := 0\n\t\t\t\tfor _, r := range v.LeafNode.Remotes {\n\t\t\t\t\tif r.TLSOCSPPeerVerify {\n\t\t\t\t\t\tcnt++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif cnt != test.NumTrueLeafRemotes {\n\t\t\t\t\tt.Fatalf(\"Expected %d Leaf Remotes with TLSOCSPPeerVerify true, got %d\", test.NumTrueLeafRemotes, cnt)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPResponseCacheMonitor(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\texpect string\n\t}{\n\t\t{\n\t\t\t\"Monitor local cache enabled, explicit cache true\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration\n\t\t\t\tocsp_cache: true\n\t\t\t`,\n\t\t\t\"local\",\n\t\t},\n\t\t{\n\t\t\t\"Monitor local cache enabled, explicit cache type local\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Long form configuration\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t}\n\t\t\t`,\n\t\t\t\"local\",\n\t\t},\n\t\t{\n\t\t\t\"Monitor local cache enabled, implicit default\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration\n\t\t\t\t# ocsp_cache: true\n\t\t\t`,\n\t\t\t\"local\",\n\t\t},\n\t\t{\n\t\t\t\"Monitor none cache enabled, explicit cache false (short)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration\n\t\t\t\tocsp_cache: false\n\t\t\t`,\n\t\t\t\"\",\n\t\t},\n\t\t{\n\t\t\t\"Monitor none cache enabled, explicit cache false (long)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Long form configuration\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: none\n\t\t\t\t}\n\t\t\t`,\n\t\t\t\"\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, _ := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tv := monitorGetVarzHelper(t, 8222)\n\t\t\tvar ct string\n\t\t\tif v.OCSPResponseCache != nil {\n\t\t\t\tct = v.OCSPResponseCache.Type\n\t\t\t}\n\t\t\tif ct != test.expect {\n\t\t\t\tt.Fatalf(\"Expected OCSP Response Cache to be %s, got %s\", test.expect, ct)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPResponseCacheChangeAndReload(t *testing.T) {\n\tdeleteLocalStore(t, \"\")\n\n\t// Start with ocsp cache set to none\n\tcontent := `\n\t\tport: -1\n\t\thttp_port: 8222\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\ttimeout: 5\n\t\t\tverify: true\n\t\t\t# Short form configuration\n\t\t\tocsp_peer: true\n\t\t}\n\t\t# Long form configuration\n\t\tocsp_cache: {\n\t\t\ttype: none\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\tv := monitorGetVarzHelper(t, 8222)\n\tvar ct string\n\tif v.OCSPResponseCache != nil {\n\t\tct = v.OCSPResponseCache.Type\n\t}\n\tif ct != \"\" {\n\t\tt.Fatalf(\"Expected OCSP Response Cache to have empty type in varz indicating none\")\n\t}\n\n\t// Change to local cache\n\tcontent = `\n\t\tport: -1\n\t\thttp_port: 8222\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\ttimeout: 5\n\t\t\tverify: true\n\t\t\t# Short form configuration\n\t\t\tocsp_peer: true\n\t\t}\n\t\t# Long form configuration\n\t\tocsp_cache: {\n\t\t\ttype: local\n\t\t}\n\t`\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttime.Sleep(2 * time.Second)\n\tv = monitorGetVarzHelper(t, 8222)\n\tct = \"\"\n\tif v.OCSPResponseCache != nil {\n\t\tct = v.OCSPResponseCache.Type\n\t}\n\tif ct != \"local\" {\n\t\tt.Fatalf(\"Expected OCSP Response Cache type to be local, got %q\", ct)\n\t}\n}\n\nfunc deleteLocalStore(t *testing.T, dir string) {\n\tt.Helper()\n\tif dir == \"\" {\n\t\t// default\n\t\tdir = \"_rc_\"\n\t}\n\tif err := os.RemoveAll(dir); err != nil {\n\t\tt.Fatalf(\"Error cleaning up local store: %v\", err)\n\t}\n}\n\nfunc monitorGetVarzHelper(t *testing.T, httpPort int) *server.Varz {\n\tt.Helper()\n\turl := fmt.Sprintf(\"http://127.0.0.1:%d/\", httpPort)\n\tresp, err := http.Get(url + \"varz\")\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\tif resp.StatusCode != 200 {\n\t\tt.Fatalf(\"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tdefer resp.Body.Close()\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error reading the body: %v\\n\", err)\n\t}\n\tv := server.Varz{}\n\tif err := json.Unmarshal(body, &v); err != nil {\n\t\tt.Fatalf(\"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\treturn &v\n}\n\nfunc writeCacheFile(dir string, content []byte) error {\n\tif dir == \"\" {\n\t\tdir = \"_rc_\"\n\t}\n\terr := os.MkdirAll(filepath.Join(dir), os.ModePerm)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn os.WriteFile(filepath.Join(dir, \"cache.json\"), content, os.ModePerm)\n}\n\n// TestOCSPPeerPreserveRevokedCacheItem is test of the preserve_revoked cache policy\nfunc TestOCSPPeerPreserveRevokedCacheItem(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\tresponses int64\n\t\trevokes int64\n\t\tgoods int64\n\t\tunknowns int64\n\t\terr error\n\t\trerr error\n\t\tclean bool\n\t}{\n\t\t{\n\t\t\t\"Test expired revoked cert not actually deleted\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so this revoked client should NOT be able to connect\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t1,\n\t\t\t1,\n\t\t\t0,\n\t\t\t0,\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\ttrue,\n\t\t},\n\t\t{\n\t\t\t\"Test expired revoked cert replaced by current good cert\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so this revoked client should NOT be able to connect\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t2,\n\t\t\t0,\n\t\t\t2,\n\t\t\t0,\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfalse,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar intermediateCA1Responder *http.Server\n\t\t\t// clean slate starting the test and start the leaf CA responder for first run\n\t\t\tif test.clean {\n\t\t\t\tdeleteLocalStore(t, \"\")\n\t\t\t\t// establish the revoked item (expired) in cache\n\t\t\t\tc := []byte(`\n\t\t\t\t{\n\t\t\t\t \"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t \"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t \"cached_at\": \"2023-05-29T17:56:45Z\",\n\t\t\t\t \"resp_status\": \"revoked\",\n\t\t\t\t \"resp_expires\": \"2023-05-29T17:56:49Z\",\n\t\t\t\t \"resp\": \"/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t }\n\t\t\t\t}`)\n\t\t\t\terr := writeCacheFile(\"\", c)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tintermediateCA1Responder = NewOCSPResponderIntermediateCA1(t)\n\t\t\t\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\t\t\t\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\t\t\t\tdefer intermediateCA1Responder.Shutdown(ctx)\n\t\t\t}\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tv := monitorGetVarzHelper(t, 8222)\n\t\t\tif v.OCSPResponseCache == nil {\n\t\t\t\tt.Fatalf(\"Expected OCSP statistics to be in varz\")\n\t\t\t}\n\t\t\tresponses := v.OCSPResponseCache.Responses\n\t\t\trevokes := v.OCSPResponseCache.Revokes\n\t\t\tgoods := v.OCSPResponseCache.Goods\n\t\t\tunknowns := v.OCSPResponseCache.Unknowns\n\t\t\tif !(responses == test.responses && revokes == test.revokes && goods == test.goods && unknowns == test.unknowns) {\n\t\t\t\tt.Fatalf(\"Expected %d response, %d revoked, %d good, %d unknown; got [%d] and [%d] and [%d] and [%d]\", test.responses, test.revokes, test.goods, test.unknowns, responses, revokes, goods, unknowns)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestOCSPStapleFeatureInterop is a test of a NATS client (AIA enabled at leaf and cert) connecting to a NATS Server\n// in which both ocsp_peer is enabled on NATS client connections (verify client) and the ocsp staple is enabled such\n// that the NATS Server will staple its own OCSP response and make available to the NATS client during handshake.\nfunc TestOCSPStapleFeatureInterop(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Interop: Both Good: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tocsp_cache: true\n\t\t\t\tocsp: {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration, non-default ca_timeout\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"expected OCSP staple to be present\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\tresp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1])\n\t\t\t\t\t\tif err != nil || resp.Status != ocsp.Good {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"expected a valid GOOD stapled response\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"Interop: Bad Client: mTLS OCSP peer check on inbound client connection and server's OCSP staple validated at client\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tocsp_cache: true\n\t\t\t\tocsp: {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration, non-default ca_timeout\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"expected OCSP staple to be present\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\tresp, err := ocsp.ParseResponse(s.OCSPResponse, s.VerifiedChains[0][1])\n\t\t\t\t\t\tif err != nil || resp.Status != ocsp.Good {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"expected a valid GOOD stapled response\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tfmt.Errorf(\"remote error: tls: bad certificate\"),\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Revoked)\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestOCSPPeerWarnOnlyOption is test of NATS client that is OCSP Revoked status but allowed to pass with warn_only option\nfunc TestOCSPPeerWarnOnlyOption(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Revoked)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Revoked NATS client with warn_only explicitly set to false\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Enable OCSP peer but with warn_only option set to false\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\twarn_only: false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Revoked NATS client with warn_only explicitly set to true\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Enable OCSP peer but with warn_only option set to true\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\twarn_only: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerUnknownIsGoodOption is test of NATS client that is OCSP status Unknown from its OCSP Responder but we treat\n// status Unknown as \"Good\"\nfunc TestOCSPPeerUnknownIsGoodOption(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Unknown NATS client with no unknown_is_good option set (default false)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Short form configuration\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"Unknown NATS client with unknown_is_good set to true\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tunknown_is_good: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerAllowWhenCAUnreachableOption is test of the allow_when_ca_unreachable peer option\nfunc TestOCSPPeerAllowWhenCAUnreachableOption(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\tcachedResponse string\n\t\terr error\n\t\trerr error\n\t}{\n\t\t{\n\t\t\t\"Expired Revoked response in cache for UserA1 -- should be rejected connection (expired revoke honored)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t`\n\t\t\t\t{\n\t\t\t\t\t\"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t\t\t\"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t\t\t\"cached_at\": \"2023-05-29T17:56:45Z\",\n\t\t\t\t\t\t\"resp_status\": \"revoked\",\n\t\t\t\t\t\t\"resp_expires\": \"2023-05-29T17:56:49Z\",\n\t\t\t\t\t\t\"resp\": \"/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t},\n\t\t{\n\t\t\t\"Expired Good response in cache for UserA1 -- should be allowed connection (cached item irrelevant)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t`\n\t\t\t{\n\t\t\t\t\"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t\t\"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t\t\"cached_at\": \"2023-06-05T16:33:52Z\",\n\t\t\t\t\t\"resp_status\": \"good\",\n\t\t\t\t\t\"resp_expires\": \"2023-06-05T16:33:55Z\",\n\t\t\t\t\t\"resp\": \"/wYAAFMyc1R3TwBYBQBgpzMn1wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjMzMDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeAADZmABA1MVqgEToTAPRAATVaMA0GCSqGSIb3DQEBCwUAA4IBAQB9csJxA2VcpQYmC5lzI0dJJnEC1zcaP1oFbBm5VsZAbWh4mIGTqyEjMkucBqANTypnhWFRYcZE5nJE8tN8Aen0EBYkhk1wfEIhincaF3zs6x5RzigxQOqTPAanO55keKH5exYnfBcyCwAQiBbQaTXLJJdjerfqiRjqgsnLW8yl2IC8+kxTCfR4GHTK34mvqVWYYhP/xbaxTLJTp35t1vf1o78ct9R+z8W5sCSO4TsMNnExZlu4Ejeon/KivMR22j7nelTEaDCuaOl03WxKh9yhw2ix8V7lvR74wh5f4fjLZria6Y0+lUjwvvrZyBRwA62W/ihe3778q8KhLECFPQSaoIIENTCCBDEwggQtMIIDFaADAgECAhQp4VKN/aUqh+sd5B1HbOGKWGlzqzo/AQQwWlEk2jECCBowGEkxqBFJbnRlcm1lZGlhdGUgQ0EgMTAeFw0yMzA1MDExOTI4MzlaFw0zMzA0MjgND1GtFQBJDDCCASIuJwIAAQUA9EABDwAwggEKAoIBAQCjDMrrgOuhDh5xm9Oz+WXOcMIhBjwxwQZ+pahK4SGjdFSfV85Q1sMpPEOwnT5UlO6N+g1xbN9engEweWy7l12vu1sFd3KfVeZmRfTiws97DljWFGp2KazjMCgN7r3Kqq4fHu9A88OrF/LX7A3h+2iaCYOZEVhClPgN1JpvnzvoVvCptxgakUF8Q+PbsQHxrQs512WY5hWwF6lWbvuEesDMZ3X89nWEMXjFbVGP0BnTFk+H71szuXrd/l+oav1EVADzpKZb/TtlOE+CT7nEvcmaVvxU8Vgvy+70CP237K0oCGab+HiYMtuxVt0OMbrG41b1Ai/7dii7xIvza9qqHTghAgMBAAGjge0wgeowHQYDVR0OBBYEFMteUGB6qy+pOx4kqwJCjeyBYEgTMB8GA1UdIwQYMBaAUssDUDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEOGAQDAgeAMBYBHgAlARAQDDAKBgiJtrADCTA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vMTI3LjAuMC4xOjE4ODg4L2ldVigxX2NybC5kZXIwMxFLHAEBBCcwJTAjEVsMMAGGF1pKAC4SAgALBQD0AQEBAEhlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"Expired Unknown response in cache for UserA1 -- should be allowed connection (cached item irrelevant)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t`\n\t\t\t{\n\t\t\t\t\"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t\t\"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t\t\"cached_at\": \"2023-06-05T16:45:01Z\",\n\t\t\t\t\t\"resp_status\": \"unknown\",\n\t\t\t\t\t\"resp_expires\": \"2023-06-05T16:45:05Z\",\n\t\t\t\t\t\"resp\": \"/wYAAFMyc1R3TwBSBQBH1aW01wzUMIIGUwoBAKCCBkwwggZIBgkrBgEFBQcwAQEEggY5MIIGNTCB5aFZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HYDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA2MDUxNjQ1MDBaMHcwdTBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBeCADpmAAwxWqAROhMA9EABNVowDQYJKoZIhvcNAQELBQADggEBAAzwED88ngiFj3hLzIejZ8DE/ppticX0vgAjUM8oXjDjwNXpSQ5xdXHsDk4RAYAVpNyiAfL2fapwz91g3JuZot6npp8smtZC5D0YMKK8iMjrx2aMqVyv+ai/33WG8PRWpBNzSTYaLhlBFjhUrx8HDu97ozNbmfgDWzRS1LqkJRa5YXvkyppqYTFSX73DV9R9tOVOwZ0x5WEKst9IJ+88mXsOBGyuye2Gh9RK6KsLgaOwiD9FBf18WRKIixeVM1Y/xHc/iwFPi8k3Z6hZ6gHX8NboQ/djCyzYVSWsUedTo/62uuagHPRWoYci4HQl4bSFXfcEO/EkWGnqkWBHfYZ4soigggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSTaMQIIGjAYSTGoEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUa0VAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSywNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm2sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t}\n\t\t\t}`,\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"No response in cache for UserA1 -- should be allowed connection\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t\"\",\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\tc := []byte(test.cachedResponse)\n\t\t\terr := writeCacheFile(\"\", c)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPResponseCacheLocalStoreOpt is test of default and non-default local_store option\nfunc TestOCSPResponseCacheLocalStoreOpt(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\tcachedResponse string\n\t\terr error\n\t\trerr error\n\t\tstoreLocation string\n\t}{\n\t\t{\n\t\t\t\"Test load from non-default local store _custom_; connect will reject only if cache file found and loaded\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tlocal_store: \"_custom_\"\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t`\n\t\t\t\t{\n\t\t\t\t\t\"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t\t\t\"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t\t\t\"cached_at\": \"2023-05-29T17:56:45Z\",\n\t\t\t\t\t\t\"resp_status\": \"revoked\",\n\t\t\t\t\t\t\"resp_expires\": \"2023-05-29T17:56:49Z\",\n\t\t\t\t\t\t\"resp\": \"/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\t\"_custom_\",\n\t\t},\n\t\t{\n\t\t\t\"Test load from default local store when \\\"\\\" set; connect will reject only if cache file found and loaded\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check but allow when CA is unreachable\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 0.5\n\t\t\t\t\t\tallow_when_ca_unreachable: true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# preserve revoked true\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\tlocal_store: \"\"\n\t\t\t\t\tpreserve_revoked: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\t`\n\t\t\t\t{\n\t\t\t\t\t\"5xL/SuHl6JN0OmxrNMpzVMTA73JVYcRfGX8+HvJinEI=\": {\n\t\t\t\t\t\t\"subject\": \"CN=UserA1,O=Testnats,L=Tacoma,ST=WA,C=US\",\n\t\t\t\t\t\t\"cached_at\": \"2023-05-29T17:56:45Z\",\n\t\t\t\t\t\t\"resp_status\": \"revoked\",\n\t\t\t\t\t\t\"resp_expires\": \"2023-05-29T17:56:49Z\",\n\t\t\t\t\t\t\"resp\": \"/wYAAFMyc1R3TwBSBQBao1Qr1QzUMIIGUQoBAKCCBkowggZGBgkrBgEFBQcwAQEEggY3MIIGMzCB46FZMFcxCzAJBgNVBAYTAlVTEQ0gCAwCV0ExDzANAQ0wBwwGVGFjb21hMREwDwEROAoMCFRlc3RuYXRzMRcwFQET8HQDDA5PQ1NQIFJlc3BvbmRlchgPMjAyMzA1MjkxNzU2MDBaMHUwczBNMAkGBSsOAwIaBQAEFKgwn5fplwQy+DsulBg5SRpx0iaYBBS1kW5PZLcWhHb5tL6ZzmCVmBqOnQIUXKGv1Xy7Fu/Cx+ZT/JQa7SS7tBc2ZAAQNDVaoBE2dwD0QQE0OVowDQYJKoZIhvcNAQELBQADggEBAGAax/vkv3SBFNbxp2utc/N6Rje4E0ceC972sWgqYjzYrH0oc/acg+OAXaxUjwqoQWaT+dHaI4D5qoTkMx7XlWATjI2L72IUTf6Luo92jPzyDFwb10CdeFHtRtEYD54Qbi/nD4oxQ8cSoLKC3wft2l3E/mK/1I4Mxwq15CioK4MhfzTISoeGZbjDXPKgloJOG3rn9v64vFGV6dosbLgaXEs+MPcCsPQYkwhOOyazuewRmIDOBp5QSsKPhqsT8Rs20t8LGTMkvjZniFWJs90l9QL9F1m3obq5nyuxrGt+7Rf5zoj4T+0XCOGtE+b7cRCLg43tFuTbaAQG8Z+qkPzpza+gggQ1MIIEMTCCBC0wggMVoAMCAQICFCnhUo39pSqH6x3kHUds4YpYaXOrOj8BBDBaUSLaLwIIGjAYSS+oEUludGVybWVkaWF0ZSBDQSAxMB4XDTIzMDUwMTE5MjgzOVoXDTMzMDQyOA0PUasVAEkMMIIBIi4nAgABBQD0QAEPADCCAQoCggEBAKMMyuuA66EOHnGb07P5Zc5wwiEGPDHBBn6lqErhIaN0VJ9XzlDWwyk8Q7CdPlSU7o36DXFs316eATB5bLuXXa+7WwV3cp9V5mZF9OLCz3sOWNYUanYprOMwKA3uvcqqrh8e70Dzw6sX8tfsDeH7aJoJg5kRWEKU+A3Umm+fO+hW8Km3GBqRQXxD49uxAfGtCznXZZjmFbAXqVZu+4R6wMxndfz2dYQxeMVtUY/QGdMWT4fvWzO5et3+X6hq/URUAPOkplv9O2U4T4JPucS9yZpW/FTxWC/L7vQI/bfsrSgIZpv4eJgy27FW3Q4xusbjVvUCL/t2KLvEi/Nr2qodOCECAwEAAaOB7TCB6jAdBgNVHQ4EFgQUy15QYHqrL6k7HiSrAkKN7IFgSBMwHwYDVR0jBBgwFoBSyQNQMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQ4YBAMCB4AwFgEeACUBEBAMMAoGCIm0sAMJMD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly8xMjcuMC4wLjE6MTg4ODgvaV1WKDFfY3JsLmRlcjAzEUscAQEEJzAlMCMRWwwwAYYXWkoALhICkTnw/0hlzm2RRjA3tvJ2wELj9e7pMg5GtdWdrLDyI/U1qBxhZoHADbyku7W+R1iL8dFfc4PSmdo+owsygZakvahXjv49xJNX7wV3YMmIHC4lfurIlY2mSnPlu2zEOwEDkI0S9WkTxXmHrkXLSciQJDkwzye6MR5fW+APk4JmKDPc46Go/K1A0EgxY/ugahMYsYtZu++W+IOYbEoYNxoCrcJCHX4c3Ep3t/Wulz4X6DWWhaDkMMUDC2JVE8E/3xUbw0X3adZe9Xf8T+goOz7wLCAigXKj1hvRUmOGISIGelv0KsfluZesG1a1TGLp+W9JX0M9nOaFOvjJTDP96aqIjs8oXGk=\"\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\t\"_rc_\",\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, test.storeLocation)\n\t\t\tc := []byte(test.cachedResponse)\n\t\t\terr := writeCacheFile(test.storeLocation, c)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerIncrementalSaveLocalCache is test of timer-based response cache save as new entries added\nfunc TestOCSPPeerIncrementalSaveLocalCache(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate2/intermediate2_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client2/UserB1_cert.pem\", ocsp.Good)\n\n\tvar fi os.FileInfo\n\tvar err error\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts [][]nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Default cache, short form: mTLS OCSP peer check on inbound client connection, UserA1 client of intermediate CA 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Local cache with custom save_interval for testability\n\t\t\t\tocsp_cache: {\n\t\t\t\t\ttype: local\n\t\t\t\t\t# Save if dirty ever 1 second\n\t\t\t\t\tsave_interval: 1\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[][]nats.Option{\n\t\t\t\t{\n\t\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client2/UserB1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client2/private/UserB1_keypair.pem\"),\n\t\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t\t},\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Cleanup any previous test that saved a local cache\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\tfi, err = statCacheFile(\"\")\n\t\t\tif err != nil && fi != nil && fi.Size() != 0 {\n\t\t\t\tt.Fatalf(\"Expected no local cache file, got a FileInfo with size %d\", fi.Size())\n\t\t\t}\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\t// Connect with UserA1 client and get a CA Response\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts[0]...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnc.Close()\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\tfi, err = statCacheFile(\"\")\n\t\t\tif err == nil && fi != nil && fi.Size() > 0 {\n\t\t\t\t// good\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected an extant local cache file, got error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif fi != nil {\n\t\t\t\t\tt.Fatalf(\"Expected non-zero size local cache file, got a FileInfo with size %d\", fi.Size())\n\t\t\t\t}\n\t\t\t}\n\t\t\tfirstFi := fi\n\t\t\t// Connect with UserB1 client and get another CA Response\n\t\t\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts[1]...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnc.Close()\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\tfi, err = statCacheFile(\"\")\n\t\t\tif err == nil && fi != nil && fi.Size() > firstFi.Size() {\n\t\t\t\t// good\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Expected an extant local cache file, got error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif fi != nil {\n\t\t\t\t\tt.Fatalf(\"Expected non-zero size local cache file with more bytes, got a FileInfo with size %d\", fi.Size())\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc statCacheFile(dir string) (os.FileInfo, error) {\n\tif dir == \"\" {\n\t\tdir = \"_rc_\"\n\t}\n\treturn os.Stat(filepath.Join(dir, \"cache.json\"))\n}\n\n// TestOCSPPeerUndelegatedCAResponseSigner\nfunc TestOCSPPeerUndelegatedCAResponseSigner(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1Undelegated(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so unvalidated clients can't connect\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerDelegatedCAResponseSigner\nfunc TestOCSPPeerDelegatedCAResponseSigner(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, responder is CA (undelegated)\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so unvalidated clients can't connect\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerBadDelegatedCAResponseSigner\nfunc TestOCSPPeerBadDelegatedCAResponseSigner(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\tintermediateCA1Responder := NewOCSPResponderBadDelegateIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"mTLS OCSP peer check on inbound client connection, responder is not a legal delegate\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t# Cache configuration is default\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Turn on CA OCSP check so unvalidated clients can't connect\n\t\t\t\t\tocsp_peer: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\terrors.New(\"remote error: tls: bad certificate\"),\n\t\t\terrors.New(\"expect error\"),\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t})\n\t}\n}\n\n// TestOCSPPeerNextUpdateUnset is test of scenario when responder does not set NextUpdate and cache TTL option is used\nfunc TestOCSPPeerNextUpdateUnset(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\"\n\tintermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, \"127.0.0.1:18888\", 0, \"\")\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\texpectedMisses int64\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"TTL set to 4 seconds with second client connection leveraging cache from first client connect\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 0\n\t\t\t\t\t\tcache_ttl_when_next_update_unset: 4\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration, local as default\n\t\t\t\tocsp_cache: true\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\t2,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"TTL set to 1 seconds with second client connection not leveraging cache items from first client connect\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\thttp_port: 8222\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 0\n\t\t\t\t\t\tcache_ttl_when_next_update_unset: 1\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t# Short form configuration, local as default\n\t\t\t\tocsp_cache: true\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\t3,\n\t\t\tfunc() {},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Cleanup any previous test that saved a local cache\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnc.Close()\n\n\t\t\t// Wait interval shorter than first test, and longer than second test\n\t\t\ttime.Sleep(2 * time.Second)\n\n\t\t\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tv := monitorGetVarzHelper(t, 8222)\n\t\t\tif v.OCSPResponseCache == nil {\n\t\t\t\tt.Fatalf(\"Expected OCSP statistics to be in varz\")\n\t\t\t}\n\t\t\tif v.OCSPResponseCache.Misses != test.expectedMisses || v.OCSPResponseCache.Responses != 2 {\n\t\t\t\tt.Errorf(\"Expected cache misses to be %d and cache items to be 2, got %d and %d\", test.expectedMisses, v.OCSPResponseCache.Misses, v.OCSPResponseCache.Responses)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPMonitoringPort(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\trootCAResponder := NewOCSPResponderRootCA(t)\n\trootCAResponderURL := fmt.Sprintf(\"http://%s\", rootCAResponder.Addr)\n\tdefer rootCAResponder.Shutdown(ctx)\n\tSetOCSPStatus(t, rootCAResponderURL, \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\", ocsp.Good)\n\n\trespCertPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/ocsp1_bundle.pem\"\n\trespKeyPEM := \"configs/certs/ocsp_peer/mini-ca/ocsp1/private/ocsp1_keypair.pem\"\n\tissuerCertPEM := \"configs/certs/ocsp_peer/mini-ca/intermediate1/intermediate1_cert.pem\"\n\tintermediateCA1Responder := NewOCSPResponderBase(t, issuerCertPEM, respCertPEM, respKeyPEM, true, \"127.0.0.1:18888\", 0, \"\")\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/client1/UserA1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\", ocsp.Good)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t}{\n\t\t{\n\t\t\t\"https with ocsp_peer\",\n\t\t\t`\n\t\t\t\tnet: 127.0.0.1\n\t\t\t\tport: -1\n\t\t\t\thttps: -1\n\t\t\t\t# Short form configuration\n\t\t\t\tocsp_cache: true\n store_dir = %s\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t\t# Long form configuration\n\t\t\t\t\tocsp_peer: {\n\t\t\t\t\t\tverify: true\n\t\t\t\t\t\tca_timeout: 5\n\t\t\t\t\t\tallowed_clockskew: 30\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"https with just ocsp\",\n\t\t\t`\n\t\t\t\tnet: 127.0.0.1\n\t\t\t\tport: -1\n\t\t\t\thttps: -1\n\t\t\t\tocsp {\n mode = always\n url = http://127.0.0.1:18888\n }\n store_dir = %s\n\n\t\t\t\ttls: {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify: true\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\", \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"),\n\t\t\t\tnats.RootCAs(\"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdeleteLocalStore(t, \"\")\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(content, t.TempDir())))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\n\t\t\t// Make request to the HTTPS port using the client cert.\n\t\t\ttlsConfig := &tls.Config{}\n\t\t\tclientCertFile := \"./configs/certs/ocsp_peer/mini-ca/client1/UserA1_bundle.pem\"\n\t\t\tclientKeyFile := \"./configs/certs/ocsp_peer/mini-ca/client1/private/UserA1_keypair.pem\"\n\t\t\tcert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\ttlsConfig.Certificates = []tls.Certificate{cert}\n\t\t\tcaCertFile := \"./configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\tcaCert, err := os.ReadFile(caCertFile)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tcaCertPool := x509.NewCertPool()\n\t\t\tcaCertPool.AppendCertsFromPEM(caCert)\n\t\t\ttlsConfig.RootCAs = caCertPool\n\n\t\t\thc := &http.Client{\n\t\t\t\tTransport: &http.Transport{\n\t\t\t\t\tTLSClientConfig: tlsConfig,\n\t\t\t\t},\n\t\t\t}\n\t\t\tresp, err := hc.Get(\"https://\" + s.MonitorAddr().String())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tdefer resp.Body.Close()\n\t\t\tif resp.StatusCode != 200 {\n\t\t\t\tt.Errorf(\"Unexpected status: %v\", resp.Status)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "test/ocsp_test.go", "content": "// Copyright 2021-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\t. \"github.com/nats-io/nats-server/v2/internal/ocsp\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"golang.org/x/crypto/ocsp\"\n)\n\nfunc TestOCSPAlwaysMustStapleAndShutdown(t *testing.T) {\n\t// Certs that have must staple will auto shutdown the server.\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\topts := server.Options{}\n\topts.Host = \"127.0.0.1\"\n\topts.NoLog = true\n\topts.NoSigs = true\n\topts.MaxControlLine = 4096\n\topts.Port = -1\n\topts.TLSCert = serverCert\n\topts.TLSKey = serverKey\n\topts.TLSCaCert = caCert\n\topts.TLSTimeout = 5\n\ttcOpts := &server.TLSConfigOpts{\n\t\tCertFile: opts.TLSCert,\n\t\tKeyFile: opts.TLSKey,\n\t\tCaFile: opts.TLSCaCert,\n\t\tTimeout: opts.TLSTimeout,\n\t}\n\n\ttlsConf, err := server.GenTLSConfig(tcOpts)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\topts.TLSConfig = tlsConf\n\n\topts.OCSPConfig = &server.OCSPConfig{\n\t\tMode: server.OCSPModeAlways,\n\t\tOverrideURLs: []string{addr},\n\t}\n\tsrv := RunServer(&opts)\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\treturn fmt.Errorf(\"invalid staple\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// The server will shutdown because the server becomes revoked\n\t// and the policy is to always must-staple. The OCSP Responder\n\t// instructs the NATS Server to fetch OCSP Staples every 2 seconds.\n\ttime.Sleep(2 * time.Second)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Revoked)\n\ttime.Sleep(2 * time.Second)\n\n\t// Should be connection refused since server will abort now.\n\t_, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nats.ErrNoServers {\n\t\tt.Errorf(\"Expected connection refused\")\n\t}\n\t// Verify that the server finishes shutdown\n\tsrv.WaitForShutdown()\n}\n\nfunc TestOCSPMustStapleShutdown(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-status-request-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-status-request-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\topts := server.Options{}\n\topts.Host = \"127.0.0.1\"\n\topts.NoLog = true\n\topts.NoSigs = true\n\topts.MaxControlLine = 4096\n\topts.Port = -1\n\topts.TLSCert = serverCert\n\topts.TLSKey = serverKey\n\topts.TLSCaCert = caCert\n\topts.TLSTimeout = 5\n\ttlsConfigOpts := &server.TLSConfigOpts{\n\t\tCertFile: opts.TLSCert,\n\t\tKeyFile: opts.TLSKey,\n\t\tCaFile: opts.TLSCaCert,\n\t\tTimeout: opts.TLSTimeout,\n\t}\n\n\ttlsConf, err := server.GenTLSConfig(tlsConfigOpts)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\topts.TLSConfig = tlsConf\n\n\topts.OCSPConfig = &server.OCSPConfig{\n\t\tMode: server.OCSPModeMust,\n\t\tOverrideURLs: []string{addr},\n\t}\n\n\tsrv := RunServer(&opts)\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\treturn fmt.Errorf(\"invalid staple\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// The server will shutdown because the server becomes revoked\n\t// and the policy is to always must-staple. The OCSP Responder\n\t// instructs the NATS Server to fetch OCSP Staples every 2 seconds.\n\ttime.Sleep(2 * time.Second)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Revoked)\n\ttime.Sleep(2 * time.Second)\n\n\t// Should be connection refused since server will abort now.\n\t_, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nats.ErrNoServers {\n\t\tt.Errorf(\"Expected connection refused\")\n\t}\n}\n\nfunc TestOCSPMustStapleAutoDoesNotShutdown(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\tcontent := `\n\t\tport: -1\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\tt.Errorf(\"Expected valid OCSP staple status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// The server will shutdown because the server becomes revoked\n\t// and the policy is to always must-staple. The OCSP Responder\n\t// instructs the NATS Server to fetch OCSP Staples every 2 seconds.\n\ttime.Sleep(2 * time.Second)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Revoked)\n\ttime.Sleep(2 * time.Second)\n\n\t// Should not be connection refused, the client will continue running and\n\t// be served the stale OCSP staple instead.\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Revoked {\n\t\t\t\t\tt.Errorf(\"Expected revoked status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOCSPAutoWithoutMustStapleDoesNotShutdownOnRevoke(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\topts := server.Options{}\n\topts.Host = \"127.0.0.1\"\n\topts.NoLog = true\n\topts.NoSigs = true\n\topts.MaxControlLine = 4096\n\topts.Port = -1\n\topts.TLSCert = serverCert\n\topts.TLSKey = serverKey\n\topts.TLSCaCert = caCert\n\topts.TLSTimeout = 5\n\ttlsConfigOpts := &server.TLSConfigOpts{\n\t\tCertFile: opts.TLSCert,\n\t\tKeyFile: opts.TLSKey,\n\t\tCaFile: opts.TLSCaCert,\n\t\tTimeout: opts.TLSTimeout,\n\t}\n\ttlsConf, err := server.GenTLSConfig(tlsConfigOpts)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\topts.TLSConfig = tlsConf\n\n\topts.OCSPConfig = &server.OCSPConfig{\n\t\tMode: server.OCSPModeAuto,\n\t\tOverrideURLs: []string{addr},\n\t}\n\n\tsrv := RunServer(&opts)\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\treturn fmt.Errorf(\"Unexpected OCSP staple for certificate\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Revoke the client certificate, nothing will happens since does\n\t// not have MustStaple.\n\ttime.Sleep(2 * time.Second)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Revoked)\n\ttime.Sleep(2 * time.Second)\n\n\t// Should not be connection refused since server will continue running.\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Errorf(\"Unexpected error: %s\", err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOCSPClient(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tocspURL := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tdefer ocspr.Shutdown(ctx)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"OCSP Stapling makes server fail to boot if status is unknown\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\t# Enable OCSP stapling with policy to honor must staple if present.\n\t\t\t\tocsp: true\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {},\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling ignored by default if server without must staple status\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp: true\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) },\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling honored by default if server has must staple status\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling can be disabled even if server has must staple status\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp: false\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Revoked)\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPReloadRotateTLSCertWithNoURL(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\tupdatedServerCert = \"configs/certs/ocsp/server-status-request-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\tcontent := `\n\t\tport: -1\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\tt.Errorf(\"Expected valid OCSP staple status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Change the contents with another that will fail to get a staple\n\t// since it does not have an URL.\n\tcontent = `\n\t\tport: -1\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\t// Reload show warning because of cert missing OCSP Url so cannot be used\n\t// with OCSP stapling.\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpectedErr := fmt.Errorf(\"missing OCSP response\")\n\t// The server will not shutdown because the reload will fail.\n\t_, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t// The new certificate does not have OCSP Staples since\n\t\t\t\t// it could not fetch one from a OCSP server.\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn expectedErr\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != expectedErr {\n\t\tt.Fatalf(\"Unexpected error: %s\", expectedErr)\n\t}\n}\n\nfunc TestOCSPReloadRotateTLSCertDisableMustStaple(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\tupdatedServerCert = \"configs/certs/ocsp/server-status-request-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\tstoreDir := t.TempDir()\n\n\toriginalContent := `\n\t\tport: -1\n\n\t\tstore_dir: '%s'\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\n\tcontent := fmt.Sprintf(originalContent, storeDir)\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tvar staple []byte\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tstaple = s.OCSPResponse\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\tt.Errorf(\"Expected valid OCSP staple status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\tfiles := []string{}\n\terr = filepath.Walk(storeDir+\"/ocsp/\", func(path string, info os.FileInfo, err error) error {\n\t\tif info.IsDir() {\n\t\t\treturn nil\n\t\t}\n\t\tfiles = append(files, path)\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfound := false\n\tfor _, file := range files {\n\t\tdata, err := os.ReadFile(file)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif bytes.Equal(staple, data) {\n\t\t\tfound = true\n\t\t}\n\t}\n\tif !found {\n\t\tt.Error(\"Could not find OCSP Staple\")\n\t}\n\n\t// Change the contents with another that has OCSP Stapling disabled.\n\tupdatedContent := `\n\t\tport: -1\n\n\t\tstore_dir: '%s'\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tcontent = fmt.Sprintf(updatedContent, storeDir)\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// The new certificate does not have must staple so they will be missing.\n\ttime.Sleep(4 * time.Second)\n\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\treturn fmt.Errorf(\"unexpected OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Re-enable OCSP Stapling\n\tcontent = fmt.Sprintf(originalContent, storeDir)\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar newStaple []byte\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tnewStaple = s.OCSPResponse\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\tt.Errorf(\"Expected valid OCSP staple status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Confirm that it got a new staple.\n\tfiles = []string{}\n\terr = filepath.Walk(storeDir+\"/ocsp/\", func(path string, info os.FileInfo, err error) error {\n\t\tif info.IsDir() {\n\t\t\treturn nil\n\t\t}\n\t\tfiles = append(files, path)\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfound = false\n\tfor _, file := range files {\n\t\tdata, err := os.ReadFile(file)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif bytes.Equal(newStaple, data) {\n\t\t\tfound = true\n\t\t}\n\t}\n\tif !found {\n\t\tt.Error(\"Could not find OCSP Staple\")\n\t}\n\tif bytes.Equal(staple, newStaple) {\n\t\tt.Error(\"Expected new OCSP Staple\")\n\t}\n}\n\nfunc TestOCSPReloadRotateTLSCertEnableMustStaple(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tupdatedServerCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\tSetOCSPStatus(t, addr, updatedServerCert, ocsp.Good)\n\n\t// Start without OCSP Stapling MustStaple\n\tcontent := `\n\t\tport: -1\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\treturn fmt.Errorf(\"unexpected OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Change the contents with another that has OCSP Stapling enabled.\n\tcontent = `\n\t\tport: -1\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// The new certificate does not have must staple so they will be missing.\n\ttime.Sleep(2 * time.Second)\n\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\tt.Errorf(\"Expected valid OCSP staple status\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOCSPCluster(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// Seed server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tcluster {\n\t\t\tname: AB\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// The rest\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tcluster {\n\t\t\tname: AB\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\n\t\t\troutes: [ nats://127.0.0.1:%d ]\n\t\t\tconnect_retries: 30\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Revoke the seed server cluster certificate, following servers will not be able to verify connection.\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Revoked)\n\n\t// Original set of servers still can communicate to each other, even though the cert has been revoked.\n\t// NOTE: Should we unplug from the cluster in case our server is revoke and OCSP policy is always or must?\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Wait for seed server to notice that its certificate has been revoked,\n\t// so that new routes can't connect to it.\n\ttime.Sleep(6 * time.Second)\n\n\t// Start another server against the seed server that has an invalid OCSP Staple\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-05-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-05-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tcluster {\n\t\t\tname: AB\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\n\t\t\troutes: [ nats://127.0.0.1:%d ]\n\t\t\tconnect_retries: 30\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Cluster.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t// There should be no connectivity between the clients due to the revoked staple.\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\tresp, err := cC.Request(\"foo\", nil, 2*time.Second)\n\tif err == nil {\n\t\tt.Errorf(\"Unexpected success, response: %+v\", resp)\n\t}\n\tresp, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err == nil {\n\t\tt.Errorf(\"Unexpected success, response: %+v\", resp)\n\t}\n\n\t// Switch the certs from the seed server to new ones that are not revoked,\n\t// this should restart OCSP Stapling for the cluster routes.\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-07-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-07-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tcluster {\n\t\t\tport: -1\n\t\t\tpool_size: -1\n\t\t\tcompression: \"disabled\"\n\t\t\tname: AB\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tconnect_retries: 30\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// Wait to get a new OCSP Staple.\n\ttime.Sleep(10 * time.Second)\n\tcheckClusterFormed(t, srvA, srvB, srvC)\n\n\t// Now clients connect to C can communicate with B and A.\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n}\n\nfunc TestOCSPLeaf(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/client-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// LeafNode server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\t# Leaf connection must present certs.\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// LeafNode that has the original as a remote and running\n\t// without OCSP Stapling for the leaf remote.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\t# Cert without OCSP Stapling enabled is able to connect.\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/client-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/client-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\tcheckLeafNodeConnected(t, srvA)\n\n\t// Revoke the seed server cluster certificate, following servers will not be able to verify connection.\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Revoked)\n\n\t// Original set of servers still can communicate to each other via leafnode, even though the staple\n\t// for the leaf server has been revoked.\n\tcheckLeafNodeConnected(t, srvA)\n\n\t// Wait for seed server to notice that its certificate has been revoked.\n\ttime.Sleep(6 * time.Second)\n\n\t// Start another server against the seed server that has an revoked OCSP Staple.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-05-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-05-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t// There should be connectivity between the clients even if there is a revoked staple\n\t// from a leafnode connection.\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t// Ensure the subscriptions are known by the server we're connected to.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"Expected success, got: %+v\", err)\n\t}\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"Expected success, got: %+v\", err)\n\t}\n\n\t// Switch the certs from the leafnode server to new ones that are not revoked,\n\t// this should restart OCSP Stapling for the leafnode server.\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-07-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-07-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttime.Sleep(4 * time.Second)\n\n\t// A <-> A\n\t_, err = cA.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\n\t// B <-> A\n\t_, err = cB.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\n\t// C <-> A\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\t// C <-> B via leafnode A\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n}\n\nfunc TestOCSPLeafNoVerify(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/client-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// LeafNode server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\t# for this test, explicitly disable compression because we do it\n\t\t\t# in RunServer but here we do a config reload...\n\t\t\tcompression: off\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\t# Leaf server does not require certs for clients.\n\t\t\t\tverify: false\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// LeafNode remote that will connect to A and will not present certs.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\tcheckLeafNodeConnected(t, srvA)\n\n\t// Revoke the seed server cluster certificate, following servers will not be able to verify connection.\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Revoked)\n\n\t// Original set of servers still can communicate to each other, even though the cert has been revoked.\n\tcheckLeafNodeConnected(t, srvA)\n\n\t// Wait for seed server to notice that its certificate has been revoked.\n\ttime.Sleep(6 * time.Second)\n\n\t// Start another server against the seed server that has an revoked OCSP Staple.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-05-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-05-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.LeafNode.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t// There should be connectivity between the clients even if there is a revoked staple\n\t// from a leafnode connection.\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t// Ensure the subscriptions are known by the server we're connected to.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"Expected success, got: %+v\", err)\n\t}\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"Expected success, got: %+v\", err)\n\t}\n\n\t// Switch the certs from the leafnode server to new ones that are not revoked,\n\t// this should restart OCSP Stapling for the leafnode server.\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-07-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-07-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tcompression: off\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttime.Sleep(4 * time.Second)\n\n\t// A <-> A\n\t_, err = cA.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\n\t// B <-> A\n\t_, err = cB.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\n\t// C <-> A\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\t// C <-> B via leafnode A\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n}\n\nfunc TestOCSPLeafVerifyLeafRemote(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/client-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\n\t// LeafNode server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// LeafNode remote that will connect to A and will not present certs.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, _ := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\n\t// Should not have been able to connect.\n\tcheckLeafNodeConnections(t, srvA, 0)\n}\n\nfunc TestOCSPLeafVerifyAndMapLeafRemote(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/client-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\n\t// LeafNode server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t\tverify_and_map: true\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify_and_map: true\n\t\t\t}\n\t\t}\n\n\t\taccounts: {\n\t\t\tleaf: {\n\t\t\t users: [ {user: \"C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost server-status-request-url-04\"} ]\n\t\t\t}\n\t\t\tclient: {\n\t\t\t users: [ {user: \"C=US, ST=CA, L=San Francisco, O=Synadia, OU=nats.io, CN=localhost client\"} ]\n\t\t\t}\n\t\t}\n\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// LeafNode remote that will connect to A and will not present certs.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tleafnodes {\n\t\t\tremotes: [ {\n\t\t\t\turl: \"tls://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t} ]\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.LeafNode.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, _ := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\tcheckLeafNodeConnections(t, srvA, 1)\n}\n\nfunc TestOCSPGateway(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// LeafNode that has the original as a remote.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: B\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\"\n\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t}]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\t// Revoke the seed server cluster certificate, following servers will not be able to verify connection.\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Revoked)\n\n\t// Original set of servers still can communicate to each other, even though the cert has been revoked.\n\twaitForOutboundGateways(t, srvA, 1, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\t// Wait for gateway A to notice that its certificate has been revoked,\n\t// so that new gateways can't connect to it.\n\ttime.Sleep(6 * time.Second)\n\n\t// Start another server against the seed server that has an invalid OCSP Staple\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-05-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-05-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: C\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{name: \"A\", url: \"nats://127.0.0.1:%d\" }]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t// There should be no connectivity between the clients due to the revoked staple.\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t// Gateway C was not able to mesh with Gateway A because of the revoked OCSP staple\n\t// so these requests to A and B should fail.\n\tresp, err := cC.Request(\"foo\", nil, 2*time.Second)\n\tif err == nil {\n\t\tt.Errorf(\"Unexpected success, response: %+v\", resp)\n\t}\n\t// Make request to B\n\tresp, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err == nil {\n\t\tt.Errorf(\"Unexpected success, response: %+v\", resp)\n\t}\n\n\t// Switch the certs from the seed server to new ones that are not revoked,\n\t// this should restart OCSP Stapling for the cluster routes.\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-07-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-07-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttime.Sleep(4 * time.Second)\n\twaitForOutboundGateways(t, srvA, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvC, 2, 5*time.Second)\n\n\t// Now clients connect to C can communicate with B and A.\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n}\n\nfunc TestOCSPGatewayIntermediate(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_cert.pem\", ocsp.Good)\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\tocsp: {\n\t\t\tmode: always\n\t\t\turl: %s\n\t\t}\n\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, intermediateCA1ResponderURL)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\tocsp: {\n\t\t\tmode: always\n\t\t\turl: %s\n\t\t}\n\n\t\tgateway {\n\t\t\tname: B\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\"\n\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t}]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer2_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer2_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, intermediateCA1ResponderURL, optsA.Gateway.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\tcB, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n}\n\nfunc TestOCSPGatewayReload(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\n\t// Node A\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\n\t// Node B\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\n\t// Node C\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\n\t// Node A rotated certs\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\tocsp { mode = always }\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// Gateway B connects to Gateway A.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"BBB\"\n\n\t\tocsp { mode = always }\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: B\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\"\n\t\t\t\turl: \"nats://127.0.0.1:%d\"\n\t\t\t}]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-04-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-04-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Gateway.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cA.Close()\n\n\t// Wait for connectivity between A and B.\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\t// Gateway C also connects to Gateway A.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"CCC\"\n\n\t\tocsp { mode = always }\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: C\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{name: \"A\", url: \"nats://127.0.0.1:%d\" }]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\t////////////////////////////////////////////////////////////////////////////\n\t// //\n\t// A and B are connected at this point and C is starting with certs that //\n\t// will be rotated, in v2.10.8 on reload now all OCSP monitors are also //\n\t// always restarted. //\n\t// //\n\t////////////////////////////////////////////////////////////////////////////\n\tcB, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\tcC, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsC.Port),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t/////////////////////////////////////////////////////////////////////////////////\n\t// //\n\t// Switch all the certs from server A, all OCSP monitors should be restarted //\n\t// so it should have new staples. //\n\t// //\n\t/////////////////////////////////////////////////////////////////////////////////\n\tsrvConfA = `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"AAA\"\n\n\t\tocsp { mode = always }\n\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\t`\n\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tif err := os.WriteFile(sconfA, []byte(srvConfA), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := srvA.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\twaitForOutboundGateways(t, srvA, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 2, 5*time.Second)\n\twaitForOutboundGateways(t, srvC, 2, 5*time.Second)\n\n\t// Now clients connect to C can communicate with B and A.\n\t_, err = cC.Request(\"foo\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\t_, err = cC.Request(\"bar\", nil, 2*time.Second)\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n}\n\nfunc TestOCSPCustomConfig(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tocspURL := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tdefer ocspr.Shutdown(ctx)\n\n\tvar (\n\t\terrExpectedNoStaple = fmt.Errorf(\"expected no staple\")\n\t\terrMissingStaple = fmt.Errorf(\"missing OCSP Staple from server\")\n\t)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"OCSP Stapling in auto mode makes server fail to boot if status is revoked\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: auto\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\t\t\treturn errExpectedNoStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Revoked) },\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling must staple ignored if disabled with ocsp: false\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp: false\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\t\t\treturn errExpectedNoStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling must staple ignored if disabled with ocsp mode never\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp: { mode: never }\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\t\t\treturn errExpectedNoStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling in always mode fetches a staple even if cert does not have one\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t\turl: \"http://127.0.0.1:8888\"\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) },\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling in must staple mode does not fetch staple if there is no must staple flag\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: must\n\t\t\t\t\turl: \"http://127.0.0.1:8888\"\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\t\t\treturn errExpectedNoStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() { SetOCSPStatus(t, ocspURL, serverCert, ocsp.Good) },\n\t\t},\n\t\t{\n\t\t\t\"OCSP Stapling in must staple mode fetches staple if there is a must staple flag\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: must\n\t\t\t\t\turl: \"http://127.0.0.1:8888\"\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(caCert),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestOCSPCustomConfigReloadDisable(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tupdatedServerCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\tSetOCSPStatus(t, addr, updatedServerCert, ocsp.Good)\n\n\t// Start with server without OCSP Stapling MustStaple\n\tcontent := `\n\t\tport: -1\n\n\t\tocsp: { mode: always, url: \"http://127.0.0.1:8888\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Change and disable OCSP Stapling.\n\tcontent = `\n\t\tport: -1\n\n\t\tocsp: { mode: never }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// The new certificate has must staple but OCSP Stapling is disabled.\n\ttime.Sleep(2 * time.Second)\n\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\treturn fmt.Errorf(\"unexpected OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOCSPCustomConfigReloadEnable(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tupdatedServerCert = \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\tSetOCSPStatus(t, addr, updatedServerCert, ocsp.Good)\n\n\t// Start with server without OCSP Stapling MustStaple\n\tcontent := `\n\t\tport: -1\n\n\t\tocsp: { mode: never, url: \"http://127.0.0.1:8888\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tconf := createConfFile(t, []byte(content))\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse != nil {\n\t\t\t\t\treturn fmt.Errorf(\"unexpected OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n\n\t// Change and disable OCSP Stapling.\n\tcontent = `\n\t\tport: -1\n\n\t\tocsp: { mode: always, url: \"http://127.0.0.1:8888\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t`\n\tif err := os.WriteFile(conf, []byte(content), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing config: %v\", err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttime.Sleep(2 * time.Second)\n\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple!\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOCSPTLSConfigNoLeafSet(t *testing.T) {\n\to := DefaultTestOptions\n\to.HTTPHost = \"127.0.0.1\"\n\to.HTTPSPort = -1\n\to.TLSConfig = &tls.Config{ServerName: \"localhost\"}\n\tcert, err := tls.LoadX509KeyPair(\"configs/certs/server-cert.pem\", \"configs/certs/server-key.pem\")\n\tif err != nil {\n\t\tt.Fatalf(\"Got error reading certificates: %s\", err)\n\t}\n\to.TLSConfig.Certificates = []tls.Certificate{cert}\n\ts := RunServer(&o)\n\ts.Shutdown()\n}\n\nfunc TestOCSPSuperCluster(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponder(t, caCert, caKey)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-01-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-02-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-03-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-04-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-05-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-06-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-07-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-status-request-url-08-cert.pem\", ocsp.Good)\n\tSetOCSPStatus(t, addr, \"configs/certs/ocsp/server-cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\tstoreDirD := t.TempDir()\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"A\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tcluster {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// Server that has the original as a cluster.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"B\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-01-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-01-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tcluster {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\n\t\t\troutes: [ nats://127.0.0.1:%d ]\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-02-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-02-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-03-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-03-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\n\t}\n\tdefer cA.Close()\n\n\t// Start another server that will make connect as a gateway to cluster A.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"C\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-05-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-05-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: C\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\",\n\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t}]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-06-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-06-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\t// Check that server is connected to any server from the other cluster.\n\tcheckClusterFormed(t, srvA, srvB)\n\twaitForOutboundGateways(t, srvC, 1, 5*time.Second)\n\n\t// Start one more server that will become another gateway.\n\tsrvConfD := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"D\"\n\n\t\tocsp { mode: \"auto\", url: \"%s\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-07-cert.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-07-key.pem\"\n\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: D\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\",\n\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}},\n\t\t\t\t{\n\t\t\t\tname: \"C\",\n\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\n\t\t\t\t####################################################################\n\t\t\t\t## TEST NOTE: This cert does not have an OCSP Staple intentionally##\n\t\t\t\t####################################################################\n\t\t\t\ttls {\n\t\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\t\tcert_file: \"configs/certs/ocsp/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp/server-key.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}}\n\t\t\t]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp/server-status-request-url-08-cert.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp/server-status-request-url-08-key.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp/ca-cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfD = fmt.Sprintf(srvConfD, addr, storeDirD, optsA.Gateway.Port, optsC.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfD))\n\tsrvD, _ := RunServerWithConfig(conf)\n\tdefer srvD.Shutdown()\n\n\t// There should be a single gateway here because one of the gateway connections does not have a OCSP staple.\n\twaitForOutboundGateways(t, srvD, 1, 10*time.Second)\n\n\t// Connect to cluster A using server B.\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\n\t// Connects to cluster C using server C.\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From Server A\"))\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From Server B\"))\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t// Confirm that a message from server C can flow back to server A via gateway..\n\tvar (\n\t\tresp *nats.Msg\n\t\tlerr error\n\t)\n\tfor i := 0; i < 10; i++ {\n\t\tresp, lerr = cC.Request(\"foo\", nil, 500*time.Millisecond)\n\t\tif lerr != nil {\n\t\t\tcontinue\n\t\t}\n\t\tgot := string(resp.Data)\n\t\texpected := \"From Server A\"\n\t\tif got != expected {\n\t\t\tt.Fatalf(\"Expected %v, got: %v\", expected, got)\n\t\t}\n\n\t\t// Make request to B\n\t\tresp, lerr = cC.Request(\"bar\", nil, 500*time.Millisecond)\n\t\tif lerr != nil {\n\t\t\tcontinue\n\t\t}\n\t\tgot = string(resp.Data)\n\t\texpected = \"From Server B\"\n\t\tif got != expected {\n\t\t\tt.Errorf(\"Expected %v, got: %v\", expected, got)\n\t\t}\n\t\tlerr = nil\n\t\tbreak\n\t}\n\tif lerr != nil {\n\t\tt.Errorf(\"Unexpected error: %v\", lerr)\n\t}\n\tif n := srvD.NumOutboundGateways(); n > 1 {\n\t\tt.Errorf(\"Expected single gateway, got: %v\", n)\n\t}\n}\n\nfunc TestOCSPLocalIssuerDetermination(t *testing.T) {\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\n\t// Test constants\n\tocspURL := intermediateCA1ResponderURL\n\tclientTrustBundle := \"configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem\"\n\tserverCert := \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\"\n\n\tvar (\n\t\terrMissingStaple = fmt.Errorf(\"missing OCSP Staple from server\")\n\t)\n\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\topts []nats.Option\n\t\terr error\n\t\trerr error\n\t\tserverStart bool\n\t\tconfigure func()\n\t}{\n\t\t{\n\t\t\t\"Correct issuer configured in cert bundle\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(clientTrustBundle),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, serverCert, ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"Wrong issuer configured in cert bundle, server no start\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/misc/misconfig_TestServer1_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(clientTrustBundle),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tfalse,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, serverCert, ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"Issuer configured in CA bundle only, configuration 1\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/misc/trust_config1_bundle.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(clientTrustBundle),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, serverCert, ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"Issuer configured in CA bundle only, configuration 2\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/misc/trust_config2_bundle.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(clientTrustBundle),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, serverCert, ocsp.Good)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"Issuer configured in CA bundle only, configuration 3\",\n\t\t\t`\n\t\t\t\tport: -1\n\n\t\t\t\tocsp {\n\t\t\t\t\tmode: always\n\t\t\t\t}\n\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/misc/trust_config3_bundle.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`,\n\t\t\t[]nats.Option{\n\t\t\t\tnats.Secure(&tls.Config{\n\t\t\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\t\t\treturn errMissingStaple\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\tnats.ClientCert(\"./configs/certs/ocsp/client-cert.pem\", \"./configs/certs/ocsp/client-key.pem\"),\n\t\t\t\tnats.RootCAs(clientTrustBundle),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\ttrue,\n\t\t\tfunc() {\n\t\t\t\tSetOCSPStatus(t, ocspURL, serverCert, ocsp.Good)\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdefer func() {\n\t\t\t\tr := recover()\n\t\t\t\tif r != nil && test.serverStart {\n\t\t\t\t\tt.Fatalf(\"Expected server start, unexpected panic: %v\", r)\n\t\t\t\t}\n\t\t\t\tif r == nil && !test.serverStart {\n\t\t\t\t\tt.Fatalf(\"Expected server to not start and panic thrown\")\n\t\t\t\t}\n\t\t\t}()\n\t\t\ttest.configure()\n\t\t\tcontent := test.config\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\t// server may not start for some tests\n\t\t\tif s != nil {\n\t\t\t\tdefer s.Shutdown()\n\t\t\t}\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port), test.opts...)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMixedCAOCSPSuperCluster(t *testing.T) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t)\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\n\tintermediateCA1Responder := NewOCSPResponderIntermediateCA1(t)\n\tintermediateCA1ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA1Responder.Addr)\n\tdefer intermediateCA1Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA1ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_cert.pem\", ocsp.Good)\n\n\tintermediateCA2Responder := NewOCSPResponderIntermediateCA2(t)\n\tintermediateCA2ResponderURL := fmt.Sprintf(\"http://%s\", intermediateCA2Responder.Addr)\n\tdefer intermediateCA2Responder.Shutdown(ctx)\n\tSetOCSPStatus(t, intermediateCA2ResponderURL, \"configs/certs/ocsp_peer/mini-ca/server2/TestServer3_cert.pem\", ocsp.Good)\n\n\t// Store Dirs\n\tstoreDirA := t.TempDir()\n\tstoreDirB := t.TempDir()\n\tstoreDirC := t.TempDir()\n\n\t// Gateway server configuration\n\tsrvConfA := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"A\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tcluster {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tadvertise: \"127.0.0.1\"\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfA = fmt.Sprintf(srvConfA, storeDirA)\n\tsconfA := createConfFile(t, []byte(srvConfA))\n\tsrvA, optsA := RunServerWithConfig(sconfA)\n\tdefer srvA.Shutdown()\n\n\t// Server that has the original as a cluster.\n\tsrvConfB := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"B\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\n\t\tcluster {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: 127.0.0.1\n\t\t\tport: -1\n\n\t\t\troutes: [ nats://127.0.0.1:%d ]\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t}\n\t\t}\n\n\t\tgateway {\n\t\t\tname: A\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server1/TestServer1_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server1/private/TestServer1_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfB = fmt.Sprintf(srvConfB, storeDirB, optsA.Cluster.Port)\n\tconf := createConfFile(t, []byte(srvConfB))\n\tsrvB, optsB := RunServerWithConfig(conf)\n\tdefer srvB.Shutdown()\n\n\t// Client connects to server A.\n\tcA, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsA.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\n\t}\n\tdefer cA.Close()\n\n\t// Start another server that will make connect as a gateway to cluster A but with different CA issuer.\n\tsrvConfC := `\n\t\thost: \"127.0.0.1\"\n\t\tport: -1\n\n\t\tserver_name: \"C\"\n\n\t\tocsp { mode: \"always\" }\n\n\t\ttls {\n\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem\"\n\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem\"\n\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\ttimeout: 5\n\t\t}\n\t\tstore_dir: '%s'\n\t\tgateway {\n\t\t\tname: C\n\t\t\thost: \"127.0.0.1\"\n\t\t\tadvertise: \"127.0.0.1\"\n\t\t\tport: -1\n\t\t\tgateways: [{\n\t\t\t\tname: \"A\",\n\t\t\t\turls: [\"nats://127.0.0.1:%d\"]\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t}]\n\t\t\ttls {\n\t\t\t\tcert_file: \"configs/certs/ocsp_peer/mini-ca/server2/TestServer3_bundle.pem\"\n\t\t\t\tkey_file: \"configs/certs/ocsp_peer/mini-ca/server2/private/TestServer3_keypair.pem\"\n\t\t\t\tca_file: \"configs/certs/ocsp_peer/mini-ca/root/root_cert.pem\"\n\t\t\t\ttimeout: 5\n\t\t\t\tverify: true\n\t\t\t}\n\t\t}\n\t`\n\tsrvConfC = fmt.Sprintf(srvConfC, storeDirC, optsA.Gateway.Port)\n\tconf = createConfFile(t, []byte(srvConfC))\n\tsrvC, optsC := RunServerWithConfig(conf)\n\tdefer srvC.Shutdown()\n\n\t// Check that server is connected to any server from the other cluster.\n\tcheckClusterFormed(t, srvA, srvB)\n\twaitForOutboundGateways(t, srvC, 1, 5*time.Second)\n\n\t// Connect to cluster A using server B.\n\tcB, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsB.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cB.Close()\n\n\t// Connects to cluster C using server C.\n\tcC, err := nats.Connect(fmt.Sprintf(\"tls://127.0.0.1:%d\", optsC.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tif s.OCSPResponse == nil {\n\t\t\t\t\treturn fmt.Errorf(\"missing OCSP Staple from server\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer cC.Close()\n\n\t_, err = cA.Subscribe(\"foo\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From Server A\"))\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"%v\", err)\n\t}\n\tcA.Flush()\n\n\t_, err = cB.Subscribe(\"bar\", func(m *nats.Msg) {\n\t\tm.Respond([]byte(\"From Server B\"))\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcB.Flush()\n\n\t// Confirm that a message from server C can flow back to server A via gateway..\n\tvar (\n\t\tresp *nats.Msg\n\t\tlerr error\n\t)\n\tfor i := 0; i < 10; i++ {\n\t\tresp, lerr = cC.Request(\"foo\", nil, 500*time.Millisecond)\n\t\tif lerr != nil {\n\t\t\tcontinue\n\t\t}\n\t\tgot := string(resp.Data)\n\t\texpected := \"From Server A\"\n\t\tif got != expected {\n\t\t\tt.Fatalf(\"Expected %v, got: %v\", expected, got)\n\t\t}\n\n\t\t// Make request to B\n\t\tresp, lerr = cC.Request(\"bar\", nil, 500*time.Millisecond)\n\t\tif lerr != nil {\n\t\t\tcontinue\n\t\t}\n\t\tgot = string(resp.Data)\n\t\texpected = \"From Server B\"\n\t\tif got != expected {\n\t\t\tt.Errorf(\"Expected %v, got: %v\", expected, got)\n\t\t}\n\t\tlerr = nil\n\t\tbreak\n\t}\n\tif lerr != nil {\n\t\tt.Errorf(\"Unexpected error: %v\", lerr)\n\t}\n}\n\nfunc TestOCSPResponderHTTPMethods(t *testing.T) {\n\tt.Run(\"prefer get\", func(t *testing.T) {\n\t\ttestOCSPResponderHTTPMethods(t, \"GET\")\n\t})\n\tt.Run(\"prefer post\", func(t *testing.T) {\n\t\ttestOCSPResponderHTTPMethods(t, \"POST\")\n\t})\n\tt.Run(\"all methods failing\", func(t *testing.T) {\n\t\ttestOCSPResponderFailing(t, \"TEST\")\n\t})\n}\n\nfunc testOCSPResponderHTTPMethods(t *testing.T, method string) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\t// Add another responder that fails.\n\tbadaddr := \"http://127.0.0.1:8889\"\n\tbadocsp := NewOCSPResponderCustomAddress(t, caCert, caKey, badaddr)\n\tdefer badocsp.Shutdown(ctx)\n\n\topts := server.Options{}\n\topts.Host = \"127.0.0.1\"\n\topts.NoLog = true\n\topts.NoSigs = true\n\topts.MaxControlLine = 4096\n\topts.Port = -1\n\topts.TLSCert = serverCert\n\topts.TLSKey = serverKey\n\topts.TLSCaCert = caCert\n\topts.TLSTimeout = 5\n\ttcOpts := &server.TLSConfigOpts{\n\t\tCertFile: opts.TLSCert,\n\t\tKeyFile: opts.TLSKey,\n\t\tCaFile: opts.TLSCaCert,\n\t\tTimeout: opts.TLSTimeout,\n\t}\n\n\ttlsConf, err := server.GenTLSConfig(tcOpts)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\topts.TLSConfig = tlsConf\n\n\topts.OCSPConfig = &server.OCSPConfig{\n\t\tMode: server.OCSPModeAlways,\n\t\tOverrideURLs: []string{badaddr, addr},\n\t}\n\tsrv := RunServer(&opts)\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\tnats.Secure(&tls.Config{\n\t\t\tVerifyConnection: func(s tls.ConnectionState) error {\n\t\t\t\tresp, err := GetOCSPStatus(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif resp.Status != ocsp.Good {\n\t\t\t\t\treturn fmt.Errorf(\"invalid staple\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t}),\n\t\tnats.RootCAs(caCert),\n\t\tnats.ErrorHandler(noOpErrHandler),\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Publish(\"foo\", []byte(\"hello world\"))\n\tnc.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Close()\n}\n\nfunc testOCSPResponderFailing(t *testing.T, method string) {\n\tconst (\n\t\tcaCert = \"configs/certs/ocsp/ca-cert.pem\"\n\t\tcaKey = \"configs/certs/ocsp/ca-key.pem\"\n\t\tserverCert = \"configs/certs/ocsp/server-cert.pem\"\n\t\tserverKey = \"configs/certs/ocsp/server-key.pem\"\n\t)\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tdefer cancel()\n\tocspr := NewOCSPResponderPreferringHTTPMethod(t, caCert, caKey, method)\n\tdefer ocspr.Shutdown(ctx)\n\taddr := fmt.Sprintf(\"http://%s\", ocspr.Addr)\n\tSetOCSPStatus(t, addr, serverCert, ocsp.Good)\n\n\topts := server.Options{}\n\topts.Host = \"127.0.0.1\"\n\topts.NoLog = true\n\topts.NoSigs = true\n\topts.MaxControlLine = 4096\n\topts.Port = -1\n\topts.TLSCert = serverCert\n\topts.TLSKey = serverKey\n\topts.TLSCaCert = caCert\n\topts.TLSTimeout = 5\n\ttcOpts := &server.TLSConfigOpts{\n\t\tCertFile: opts.TLSCert,\n\t\tKeyFile: opts.TLSKey,\n\t\tCaFile: opts.TLSCaCert,\n\t\tTimeout: opts.TLSTimeout,\n\t}\n\n\ttlsConf, err := server.GenTLSConfig(tcOpts)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\topts.TLSConfig = tlsConf\n\n\topts.OCSPConfig = &server.OCSPConfig{\n\t\tMode: server.OCSPModeAlways,\n\t\tOverrideURLs: []string{addr},\n\t}\n\texpected := \"bad OCSP status update for certificate at 'configs/certs/ocsp/server-cert.pem': \"\n\texpected += \"exhausted ocsp servers: non-ok http status on POST request (reqlen=68): 400\\nnon-ok http status on GET request (reqlen=92): 400\"\n\t_, err = server.NewServer(&opts)\n\tif err == nil {\n\t\tt.Error(\"Unexpected success setting up server\")\n\t} else if err.Error() != expected {\n\t\tt.Errorf(\"Expected %q, got: %q\", expected, err.Error())\n\t}\n}\n" }, { "path": "test/operator_test.go", "content": "// Copyright 2018-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\nconst (\n\ttestOpConfig = \"./configs/operator.conf\"\n\ttestOpInlineConfig = \"./configs/operator_inline.conf\"\n)\n\n// This matches ./configs/nkeys_jwts/test.seed\n// Test operator seed.\nvar oSeed = []byte(\"SOAFYNORQLQFJYBYNUGC5D7SH2MXMUX5BFEWWGHN3EK4VGG5TPT5DZP7QU\")\n\n// This is a signing key seed.\nvar skSeed = []byte(\"SOAEL3NFOTU6YK3DBTEKQYZ2C5IWSVZWWZCQDASBUOHJKBFLVANK27JMMQ\")\n\nfunc checkKeys(t *testing.T, opts *server.Options, opc *jwt.OperatorClaims, expected int) {\n\t// We should have filled in the TrustedKeys here.\n\tif len(opts.TrustedKeys) != expected {\n\t\tt.Fatalf(\"Should have %d trusted keys, got %d\", expected, len(opts.TrustedKeys))\n\t}\n\t// Check that we properly placed all keys from the opc into TrustedKeys\n\tchkMember := func(s string) {\n\t\tfor _, c := range opts.TrustedKeys {\n\t\t\tif s == c {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tt.Fatalf(\"Expected %q to be in TrustedKeys\", s)\n\t}\n\tchkMember(opc.Issuer)\n\tfor _, sk := range opc.SigningKeys {\n\t\tchkMember(sk)\n\t}\n}\n\n// This will test that we enforce certain restrictions when you use trusted operators.\n// Like auth is always true, can't define accounts or users, required to define an account resolver, etc.\nfunc TestOperatorRestrictions(t *testing.T) {\n\topts, err := server.ProcessConfigFile(testOpConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoSigs = true\n\tif _, err := server.NewServer(opts); err != nil {\n\t\tt.Fatalf(\"Expected to create a server successfully\")\n\t}\n\t// TrustedKeys get defined when processing from above, trying again with\n\t// same opts should not work.\n\tif _, err := server.NewServer(opts); err == nil {\n\t\tt.Fatalf(\"Expected an error with TrustedKeys defined\")\n\t}\n\t// Must wipe and rebuild to succeed.\n\twipeOpts := func() {\n\t\topts.TrustedKeys = nil\n\t\topts.Accounts = nil\n\t\topts.Users = nil\n\t\topts.Nkeys = nil\n\t}\n\n\twipeOpts()\n\topts.Accounts = []*server.Account{{Name: \"TEST\"}}\n\tif _, err := server.NewServer(opts); err == nil {\n\t\tt.Fatalf(\"Expected an error with Accounts defined\")\n\t}\n\twipeOpts()\n\topts.Users = []*server.User{{Username: \"TEST\"}}\n\tif _, err := server.NewServer(opts); err == nil {\n\t\tt.Fatalf(\"Expected an error with Users defined\")\n\t}\n\twipeOpts()\n\topts.Nkeys = []*server.NkeyUser{{Nkey: \"TEST\"}}\n\tif _, err := server.NewServer(opts); err == nil {\n\t\tt.Fatalf(\"Expected an error with Nkey Users defined\")\n\t}\n\twipeOpts()\n\n\topts.AccountResolver = nil\n\tif _, err := server.NewServer(opts); err == nil {\n\t\tt.Fatalf(\"Expected an error without an AccountResolver defined\")\n\t}\n}\n\nfunc TestOperatorConfig(t *testing.T) {\n\topts, err := server.ProcessConfigFile(testOpConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoSigs = true\n\t// Check we have the TrustedOperators\n\tif len(opts.TrustedOperators) != 1 {\n\t\tt.Fatalf(\"Expected to load the operator\")\n\t}\n\t_, err = server.NewServer(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to create a server: %v\", err)\n\t}\n\t// We should have filled in the public TrustedKeys here.\n\t// Our master public key (issuer) plus the signing keys (3).\n\tcheckKeys(t, opts, opts.TrustedOperators[0], 4)\n}\n\nfunc TestOperatorConfigInline(t *testing.T) {\n\topts, err := server.ProcessConfigFile(testOpInlineConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing config file: %v\", err)\n\t}\n\topts.NoSigs = true\n\t// Check we have the TrustedOperators\n\tif len(opts.TrustedOperators) != 1 {\n\t\tt.Fatalf(\"Expected to load the operator\")\n\t}\n\t_, err = server.NewServer(opts)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to create a server: %v\", err)\n\t}\n\t// We should have filled in the public TrustedKeys here.\n\t// Our master public key (issuer) plus the signing keys (3).\n\tcheckKeys(t, opts, opts.TrustedOperators[0], 4)\n}\n\nfunc runOperatorServer(t *testing.T) (*server.Server, *server.Options) {\n\treturn RunServerWithConfig(testOpConfig)\n}\n\nfunc publicKeyFromKeyPair(t *testing.T, pair nkeys.KeyPair) (pkey string) {\n\tvar err error\n\tif pkey, err = pair.PublicKey(); err != nil {\n\t\tt.Fatalf(\"Expected no error %v\", err)\n\t}\n\treturn\n}\n\nfunc createAccountForOperatorKey(t *testing.T, s *server.Server, seed []byte) nkeys.KeyPair {\n\tt.Helper()\n\tokp, _ := nkeys.FromSeed(seed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tjwt, _ := nac.Encode(okp)\n\tif err := s.AccountResolver().Store(pub, jwt); err != nil {\n\t\tt.Fatalf(\"Account Resolver returned an error: %v\", err)\n\t}\n\treturn akp\n}\n\nfunc createAccount(t *testing.T, s *server.Server) (acc *server.Account, akp nkeys.KeyPair) {\n\tt.Helper()\n\takp = createAccountForOperatorKey(t, s, oSeed)\n\tif pub, err := akp.PublicKey(); err != nil {\n\t\tt.Fatalf(\"Expected this to pass, got %v\", err)\n\t} else if acc, err = s.LookupAccount(pub); err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\treturn acc, akp\n}\n\nfunc createUserCreds(t *testing.T, s *server.Server, akp nkeys.KeyPair) nats.Option {\n\tt.Helper()\n\topt, _ := createUserCredsOption(t, s, akp)\n\treturn opt\n}\n\nfunc createUserCredsOption(t *testing.T, s *server.Server, akp nkeys.KeyPair) (nats.Option, string) {\n\tt.Helper()\n\tkp, _ := nkeys.CreateUser()\n\tpub, _ := kp.PublicKey()\n\tnuc := jwt.NewUserClaims(pub)\n\tujwt, err := nuc.Encode(akp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating user JWT: %v\", err)\n\t}\n\tuserCB := func() (string, error) {\n\t\treturn ujwt, nil\n\t}\n\tsigCB := func(nonce []byte) ([]byte, error) {\n\t\tsig, _ := kp.Sign(nonce)\n\t\treturn sig, nil\n\t}\n\treturn nats.UserJWT(userCB, sigCB), pub\n}\n\nfunc TestOperatorServer(t *testing.T) {\n\ts, opts := runOperatorServer(t)\n\tdefer s.Shutdown()\n\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tif _, err := nats.Connect(url); err == nil {\n\t\tt.Fatalf(\"Expected to fail with no credentials\")\n\t}\n\n\t_, akp := createAccount(t, s)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tnc.Close()\n\n\t// Now create an account from another operator, this should fail.\n\tokp, _ := nkeys.CreateOperator()\n\tseed, _ := okp.Seed()\n\takp = createAccountForOperatorKey(t, s, seed)\n\t_, err = nats.Connect(url, createUserCreds(t, s, akp))\n\tif err == nil {\n\t\tt.Fatalf(\"Expected error on connect\")\n\t}\n\t// The account should not be in memory either\n\tif v, err := s.LookupAccount(publicKeyFromKeyPair(t, akp)); err == nil {\n\t\tt.Fatalf(\"Expected account to NOT be in memory: %v\", v)\n\t}\n}\n\nfunc TestOperatorSystemAccount(t *testing.T) {\n\ts, _ := runOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Create an account from another operator, this should fail if used as a system account.\n\tokp, _ := nkeys.CreateOperator()\n\tseed, _ := okp.Seed()\n\takp := createAccountForOperatorKey(t, s, seed)\n\tif err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err == nil {\n\t\tt.Fatalf(\"Expected this to fail\")\n\t}\n\tif acc := s.SystemAccount(); acc != nil {\n\t\tt.Fatalf(\"Expected no account to be set for system account\")\n\t}\n\n\tacc, _ := createAccount(t, s)\n\tif err := s.SetSystemAccount(acc.Name); err != nil {\n\t\tt.Fatalf(\"Expected this succeed, got %v\", err)\n\t}\n\tif sysAcc := s.SystemAccount(); sysAcc != acc {\n\t\tt.Fatalf(\"Did not get matching account for system account\")\n\t}\n}\n\nfunc TestOperatorSigningKeys(t *testing.T) {\n\ts, opts := runOperatorServer(t)\n\tdefer s.Shutdown()\n\n\t// Create an account with a signing key, not the master key.\n\takp := createAccountForOperatorKey(t, s, skSeed)\n\t// Make sure we can set system account.\n\tif err := s.SetSystemAccount(publicKeyFromKeyPair(t, akp)); err != nil {\n\t\tt.Fatalf(\"Expected this succeed, got %v\", err)\n\t}\n\n\t// Make sure we can create users with it too.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, akp))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tnc.Close()\n}\n\nfunc TestOperatorMemResolverPreload(t *testing.T) {\n\ts, opts := RunServerWithConfig(\"./configs/resolver_preload.conf\")\n\tdefer s.Shutdown()\n\n\t// Make sure we can look up the account.\n\tacc, _ := s.LookupAccount(\"ADM2CIIL3RWXBA6T2HW3FODNCQQOUJEHHQD6FKCPVAMHDNTTSMO73ROX\")\n\tif acc == nil {\n\t\tt.Fatalf(\"Expected to properly lookup account\")\n\t}\n\tsacc := s.SystemAccount()\n\tif sacc == nil {\n\t\tt.Fatalf(\"Expected to have system account registered\")\n\t}\n\tif sacc.Name != opts.SystemAccount {\n\t\tt.Fatalf(\"System account does not match, wanted %q, got %q\", opts.SystemAccount, sacc.Name)\n\t}\n}\n\nfunc TestOperatorConfigReloadDoesntKillNonce(t *testing.T) {\n\ts, _ := runOperatorServer(t)\n\tdefer s.Shutdown()\n\n\tif !s.NonceRequired() {\n\t\tt.Fatalf(\"Error nonce should be required\")\n\t}\n\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\n\tif !s.NonceRequired() {\n\t\tt.Fatalf(\"Error nonce should still be required after reload\")\n\t}\n}\n\nfunc createAccountForConfig(t *testing.T) (string, nkeys.KeyPair) {\n\tt.Helper()\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tjwt, _ := nac.Encode(okp)\n\treturn jwt, akp\n}\n\nfunc TestReloadDoesNotWipeAccountsWithOperatorMode(t *testing.T) {\n\t// We will run an operator mode server that forms a cluster. We will\n\t// make sure that a reload does not wipe account information.\n\t// We will force reload of auth by changing cluster auth timeout.\n\n\t// Create two accounts, system and normal account.\n\tsysJWT, sysKP := createAccountForConfig(t)\n\tsysPub, _ := sysKP.PublicKey()\n\n\taccJWT, accKP := createAccountForConfig(t)\n\taccPub, _ := accKP.PublicKey()\n\n\tcf := `\n\tlisten: 127.0.0.1:-1\n\tcluster {\n\t\tname: \"A\"\n\t\tlisten: 127.0.0.1:-1\n\t\tpool_size: -1\n\t\tcompression: \"disabled\"\n\t\tauthorization {\n\t\t\ttimeout: 2.2\n\t\t} %s\n\t}\n\n\toperator = \"./configs/nkeys/op.jwt\"\n\tsystem_account = \"%s\"\n\n\tresolver = MEMORY\n\tresolver_preload = {\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t}\n\t`\n\tcontents := strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\tconf := createConfFile(t, []byte(contents))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create a new server and route to main one.\n\trouteStr := fmt.Sprintf(\"\\n\\t\\troutes = [nats-route://%s:%d]\", opts.Cluster.Host, opts.Cluster.Port)\n\tcontents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\n\tconf2 := createConfFile(t, []byte(contents2))\n\n\ts2, opts2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s, s2)\n\n\t// Create a client on the first server and subscribe.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tch := make(chan bool)\n\tnc.Subscribe(\"foo\", func(m *nats.Msg) { ch <- true })\n\tnc.Flush()\n\n\t// Use this to check for message.\n\tcheckForMsg := func() {\n\t\tt.Helper()\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatal(\"Timeout waiting for message across route\")\n\t\t}\n\t}\n\n\t// Wait for \"foo\" interest to be propagated to s2's account `accPub`\n\tcheckSubInterest(t, s2, accPub, \"foo\", 2*time.Second)\n\n\t// Create second client and send message from this one. Interest should be here.\n\turl2 := fmt.Sprintf(\"nats://%s:%d/\", opts2.Host, opts2.Port)\n\tnc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Check that we can send messages.\n\tnc2.Publish(\"foo\", nil)\n\tcheckForMsg()\n\n\t// Now shutdown nc2 and srvA.\n\tnc2.Close()\n\ts2.Shutdown()\n\n\t// Now change config and do reload which will do an auth change.\n\tb, err := os.ReadFile(conf)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnewConf := bytes.Replace(b, []byte(\"2.2\"), []byte(\"3.3\"), 1)\n\terr = os.WriteFile(conf, newConf, 0644)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// This will cause reloadAuthorization to kick in and reprocess accounts.\n\ts.Reload()\n\n\ts2, opts2 = RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s, s2)\n\n\tcheckSubInterest(t, s2, accPub, \"foo\", 2*time.Second)\n\n\t// Reconnect and make sure this works. If accounts blown away this will fail.\n\turl2 = fmt.Sprintf(\"nats://%s:%d/\", opts2.Host, opts2.Port)\n\tnc2, err = nats.Connect(url2, createUserCreds(t, s2, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Check that we can send messages.\n\tnc2.Publish(\"foo\", nil)\n\tcheckForMsg()\n}\n\nfunc TestReloadDoesUpdateAccountsWithMemoryResolver(t *testing.T) {\n\t// We will run an operator mode server with a memory resolver.\n\t// Reloading should behave similar to configured accounts.\n\n\t// Create two accounts, system and normal account.\n\tsysJWT, sysKP := createAccountForConfig(t)\n\tsysPub, _ := sysKP.PublicKey()\n\n\taccJWT, accKP := createAccountForConfig(t)\n\taccPub, _ := accKP.PublicKey()\n\n\tcf := `\n\tlisten: 127.0.0.1:-1\n\tcluster {\n\t\tname: \"A\"\n\t\tlisten: 127.0.0.1:-1\n\t\tpool_size: -1\n\t\tauthorization {\n\t\t\ttimeout: 2.2\n\t\t} %s\n\t}\n\n\toperator = \"./configs/nkeys/op.jwt\"\n\tsystem_account = \"%s\"\n\n\tresolver = MEMORY\n\tresolver_preload = {\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t}\n\t`\n\tcontents := strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\tconf := createConfFile(t, []byte(contents))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create a client on the first server and subscribe.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, createUserCreds(t, s, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tasyncErr := make(chan error, 1)\n\tnc.SetErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) {\n\t\tasyncErr <- err\n\t})\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"foo\", func(m *nats.Msg) {})\n\tnc.Flush()\n\n\t// Now update and remove normal account and make sure we get disconnected.\n\taccJWT2, accKP2 := createAccountForConfig(t)\n\taccPub2, _ := accKP2.PublicKey()\n\tcontents = strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, accPub2, accJWT2), \"\\n\\t\", \"\\n\", -1)\n\terr = os.WriteFile(conf, []byte(contents), 0644)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// This will cause reloadAuthorization to kick in and reprocess accounts.\n\ts.Reload()\n\n\tselect {\n\tcase err := <-asyncErr:\n\t\tif err != nats.ErrAuthorization {\n\t\t\tt.Fatalf(\"Expected ErrAuthorization, got %v\", err)\n\t\t}\n\tcase <-time.After(2 * time.Second):\n\t\t// Give it up to 2 sec.\n\t\tt.Fatal(\"Expected connection to be disconnected\")\n\t}\n\n\t// Make sure we can lool up new account and not old one.\n\tif _, err := s.LookupAccount(accPub2); err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\tif _, err := s.LookupAccount(accPub); err == nil {\n\t\tt.Fatalf(\"Expected error looking up old account\")\n\t}\n}\n\nfunc TestReloadFailsWithBadAccountsWithMemoryResolver(t *testing.T) {\n\t// Create two accounts, system and normal account.\n\tsysJWT, sysKP := createAccountForConfig(t)\n\tsysPub, _ := sysKP.PublicKey()\n\n\t// Create an expired account by hand here. We want to make sure we start up correctly\n\t// with expired or otherwise accounts with validation issues.\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\takp, _ := nkeys.CreateAccount()\n\tapub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(apub)\n\tnac.IssuedAt = time.Now().Add(-10 * time.Second).Unix()\n\tnac.Expires = time.Now().Add(-2 * time.Second).Unix()\n\tajwt, err := nac.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tcf := `\n\tlisten: 127.0.0.1:-1\n\tcluster {\n\t\tname: \"A\"\n\t\tlisten: 127.0.0.1:-1\n\t\tpool_size: -1\n\t\tauthorization {\n\t\t\ttimeout: 2.2\n\t\t} %s\n\t}\n\n\toperator = \"./configs/nkeys/op.jwt\"\n\tsystem_account = \"%s\"\n\n\tresolver = MEMORY\n\tresolver_preload = {\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t}\n\t`\n\tcontents := strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, apub, ajwt), \"\\n\\t\", \"\\n\", -1)\n\tconf := createConfFile(t, []byte(contents))\n\n\ts, _ := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Now add in bogus account for second item and make sure reload fails.\n\tcontents = strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, \"foo\", \"bar\"), \"\\n\\t\", \"\\n\", -1)\n\terr = os.WriteFile(conf, []byte(contents), 0644)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := s.Reload(); err == nil {\n\t\tt.Fatalf(\"Expected fatal error with bad account on reload\")\n\t}\n\n\t// Put it back with a normal account and reload should succeed.\n\taccJWT, accKP := createAccountForConfig(t)\n\taccPub, _ := accKP.PublicKey()\n\n\tcontents = strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\terr = os.WriteFile(conf, []byte(contents), 0644)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Got unexpected error on reload: %v\", err)\n\t}\n}\n\nfunc TestConnsRequestDoesNotLoadAccountCheckingConnLimits(t *testing.T) {\n\t// Create two accounts, system and normal account.\n\tsysJWT, sysKP := createAccountForConfig(t)\n\tsysPub, _ := sysKP.PublicKey()\n\n\t// Do this account by hand to add in connection limits\n\tokp, _ := nkeys.FromSeed(oSeed)\n\taccKP, _ := nkeys.CreateAccount()\n\taccPub, _ := accKP.PublicKey()\n\tnac := jwt.NewAccountClaims(accPub)\n\tnac.Limits.Conn = 10\n\taccJWT, _ := nac.Encode(okp)\n\n\tcf := `\n\tlisten: 127.0.0.1:-1\n\tcluster {\n\t\tname: \"A\"\n\t\tlisten: 127.0.0.1:-1\n\t\tauthorization {\n\t\t\ttimeout: 2.2\n\t\t} %s\n\t}\n\n\toperator = \"./configs/nkeys/op.jwt\"\n\tsystem_account = \"%s\"\n\n\tresolver = MEMORY\n\tresolver_preload = {\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t}\n\t`\n\tcontents := strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\tconf := createConfFile(t, []byte(contents))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create a new server and route to main one.\n\trouteStr := fmt.Sprintf(\"\\n\\t\\troutes = [nats-route://%s:%d]\", opts.Cluster.Host, opts.Cluster.Port)\n\tcontents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\n\tconf2 := createConfFile(t, []byte(contents2))\n\n\ts2, _ := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s, s2)\n\n\t// Make sure that we do not have the account loaded.\n\t// Just SYS and $G\n\tif nla := s.NumLoadedAccounts(); nla != 2 {\n\t\tt.Fatalf(\"Expected only 2 loaded accounts, got %d\", nla)\n\t}\n\tif nla := s2.NumLoadedAccounts(); nla != 2 {\n\t\tt.Fatalf(\"Expected only 2 loaded accounts, got %d\", nla)\n\t}\n\n\t// Now connect to first server on accPub.\n\tnc, err := nats.Connect(s.ClientURL(), createUserCreds(t, s, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Just wait for the request for connections to move to S2 and cause a fetch.\n\t// This is what we want to fix.\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// We should have 3 here for sure.\n\tif nla := s.NumLoadedAccounts(); nla != 3 {\n\t\tt.Fatalf(\"Expected 3 loaded accounts, got %d\", nla)\n\t}\n\n\t// Now make sure that we still only have 2 loaded accounts on server 2.\n\tif nla := s2.NumLoadedAccounts(); nla != 2 {\n\t\tt.Fatalf(\"Expected only 2 loaded accounts, got %d\", nla)\n\t}\n}\n" }, { "path": "test/opts_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"testing\"\n)\n\nfunc TestServerConfig(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/override.conf\")\n\tdefer srv.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsinfo := checkInfoMsg(t, c)\n\tif sinfo.MaxPayload != opts.MaxPayload {\n\t\tt.Fatalf(\"Expected max_payload from server, got %d vs %d\",\n\t\t\topts.MaxPayload, sinfo.MaxPayload)\n\t}\n}\n\nfunc TestTLSConfig(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls.conf\")\n\tdefer srv.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsinfo := checkInfoMsg(t, c)\n\tif !sinfo.TLSRequired {\n\t\tt.Fatal(\"Expected TLSRequired to be true when configured\")\n\t}\n}\n" }, { "path": "test/pedantic_test.go", "content": "// Copyright 2012-2019 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc runPedanticServer() *server.Server {\n\topts := DefaultTestOptions\n\n\topts.NoLog = false\n\topts.Trace = true\n\n\topts.Port = PROTO_TEST_PORT\n\treturn RunServer(&opts)\n}\n\nfunc TestPedanticSub(t *testing.T) {\n\ts := runPedanticServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend := sendCommand(t, c)\n\texpect := expectCommand(t, c)\n\tdoConnect(t, c, false, true, false)\n\n\t// Ping should still be same\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Test malformed subjects for SUB\n\t// Sub can contain wildcards, but\n\t// subject must still be legit.\n\n\t// Empty terminal token\n\tsend(\"SUB foo. 1\\r\\n\")\n\texpect(errRe)\n\n\t// Empty beginning token\n\tsend(\"SUB .foo. 1\\r\\n\")\n\texpect(errRe)\n\n\t// Empty middle token\n\tsend(\"SUB foo..bar 1\\r\\n\")\n\texpect(errRe)\n\n\t// Bad non-terminal FWC\n\tsend(\"SUB foo.>.bar 1\\r\\n\")\n\tbuf := expect(errRe)\n\n\t// Check that itr is 'Invalid Subject'\n\tmatches := errRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 1 {\n\t\tt.Fatal(\"Wanted one overall match\")\n\t}\n\tif string(matches[0][1]) != \"'Invalid Subject'\" {\n\t\tt.Fatalf(\"Expected 'Invalid Subject', got %s\", string(matches[0][1]))\n\t}\n}\n\nfunc TestPedanticPub(t *testing.T) {\n\ts := runPedanticServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend := sendCommand(t, c)\n\texpect := expectCommand(t, c)\n\tdoConnect(t, c, false, true, false)\n\n\t// Ping should still be same\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Test malformed subjects for PUB\n\t// PUB subjects can not have wildcards\n\t// This will error in pedantic mode\n\tsend(\"PUB foo.* 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n\n\tsend(\"PUB foo.> 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n\n\tsend(\"PUB foo. 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n\n\tsend(\"PUB .foo 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n\n\tsend(\"PUB foo..* 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n}\n" }, { "path": "test/pid_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestPidFile(t *testing.T) {\n\topts := DefaultTestOptions\n\n\tfile := createTempFile(t, \"nats-server:pid_\")\n\tfile.Close()\n\topts.PidFile = file.Name()\n\n\ts := RunServer(&opts)\n\ts.Shutdown()\n\n\tbuf, err := os.ReadFile(opts.PidFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read pid_file: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length pid_file\")\n\t}\n\n\tpid := 0\n\tfmt.Sscanf(string(buf), \"%d\", &pid)\n\tif pid != os.Getpid() {\n\t\tt.Fatalf(\"Expected pid to be %d, got %d\\n\", os.Getpid(), pid)\n\t}\n}\n" }, { "path": "test/ping_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nconst (\n\tPING_TEST_PORT = 9972\n\tPING_INTERVAL = 50 * time.Millisecond\n\tPING_MAX = 2\n)\n\nfunc runPingServer() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = PING_TEST_PORT\n\topts.PingInterval = PING_INTERVAL\n\topts.MaxPingsOut = PING_MAX\n\treturn RunServer(&opts)\n}\n\nfunc TestPingSentToTLSConnection(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = PING_TEST_PORT\n\topts.PingInterval = PING_INTERVAL\n\topts.MaxPingsOut = PING_MAX\n\topts.TLSCert = \"configs/certs/server-cert.pem\"\n\topts.TLSKey = \"configs/certs/server-key.pem\"\n\topts.TLSCaCert = \"configs/certs/ca.pem\"\n\n\ttc := server.TLSConfigOpts{}\n\ttc.CertFile = opts.TLSCert\n\ttc.KeyFile = opts.TLSKey\n\ttc.CaFile = opts.TLSCaCert\n\n\topts.TLSConfig, _ = server.GenTLSConfig(&tc)\n\topts.TLSTimeout = 5\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PING_TEST_PORT)\n\tdefer c.Close()\n\n\tcheckInfoMsg(t, c)\n\tc = tls.Client(c, &tls.Config{InsecureSkipVerify: true})\n\ttlsConn := c.(*tls.Conn)\n\ttlsConn.Handshake()\n\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v}\\r\\n\", false, false, true)\n\tsendProto(t, c, cs)\n\n\texpect := expectCommand(t, c)\n\n\t// Expect the max to be delivered correctly..\n\tfor i := 0; i < PING_MAX; i++ {\n\t\ttime.Sleep(PING_INTERVAL / 2)\n\t\texpect(pingRe)\n\t}\n\n\t// We should get an error from the server\n\ttime.Sleep(PING_INTERVAL)\n\texpect(errRe)\n\n\t// Server should close the connection at this point..\n\ttime.Sleep(PING_INTERVAL)\n\tc.SetWriteDeadline(time.Now().Add(PING_INTERVAL))\n\n\tvar err error\n\tfor {\n\t\t_, err = c.Write([]byte(\"PING\\r\\n\"))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tc.SetWriteDeadline(time.Time{})\n\n\tif err == nil {\n\t\tt.Fatal(\"No error: Expected to have connection closed\")\n\t}\n\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\tt.Fatal(\"timeout: Expected to have connection closed\")\n\t}\n}\n\nfunc TestPingInterval(t *testing.T) {\n\ts := runPingServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PING_TEST_PORT)\n\tdefer c.Close()\n\n\tdoConnect(t, c, false, false, false)\n\n\texpect := expectCommand(t, c)\n\n\t// Expect the max to be delivered correctly..\n\tfor i := 0; i < PING_MAX; i++ {\n\t\ttime.Sleep(PING_INTERVAL / 2)\n\t\texpect(pingRe)\n\t}\n\n\t// We should get an error from the server\n\ttime.Sleep(PING_INTERVAL)\n\texpect(errRe)\n\n\t// Server should close the connection at this point..\n\ttime.Sleep(PING_INTERVAL)\n\tc.SetWriteDeadline(time.Now().Add(PING_INTERVAL))\n\n\tvar err error\n\tfor {\n\t\t_, err = c.Write([]byte(\"PING\\r\\n\"))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tc.SetWriteDeadline(time.Time{})\n\n\tif err == nil {\n\t\tt.Fatal(\"No error: Expected to have connection closed\")\n\t}\n\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\tt.Fatal(\"timeout: Expected to have connection closed\")\n\t}\n}\n\nfunc TestUnpromptedPong(t *testing.T) {\n\ts := runPingServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PING_TEST_PORT)\n\tdefer c.Close()\n\n\tdoConnect(t, c, false, false, false)\n\n\texpect := expectCommand(t, c)\n\n\t// Send lots of PONGs in a row...\n\tfor i := 0; i < 100; i++ {\n\t\tc.Write([]byte(\"PONG\\r\\n\"))\n\t}\n\n\t// The server should still send the max number of PINGs and then\n\t// close the connection.\n\tfor i := 0; i < PING_MAX; i++ {\n\t\ttime.Sleep(PING_INTERVAL / 2)\n\t\texpect(pingRe)\n\t}\n\n\t// We should get an error from the server\n\ttime.Sleep(PING_INTERVAL)\n\texpect(errRe)\n\n\t// Server should close the connection at this point..\n\tc.SetWriteDeadline(time.Now().Add(PING_INTERVAL))\n\tvar err error\n\tfor {\n\t\t_, err = c.Write([]byte(\"PING\\r\\n\"))\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\tc.SetWriteDeadline(time.Time{})\n\n\tif err == nil {\n\t\tt.Fatal(\"No error: Expected to have connection closed\")\n\t}\n\tif ne, ok := err.(net.Error); ok && ne.Timeout() {\n\t\tt.Fatal(\"timeout: Expected to have connection closed\")\n\t}\n}\n\nfunc TestPingSuppresion(t *testing.T) {\n\tpingInterval := 100 * time.Millisecond\n\thighWater := 130 * time.Millisecond\n\topts := DefaultTestOptions\n\topts.Port = PING_TEST_PORT\n\topts.PingInterval = pingInterval\n\topts.DisableShortFirstPing = true\n\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PING_TEST_PORT)\n\tdefer c.Close()\n\n\tconnectTime := time.Now()\n\n\tsend, expect := setupConn(t, c)\n\n\texpect(pingRe)\n\tpingTime := time.Since(connectTime)\n\tsend(\"PONG\\r\\n\")\n\n\t// Should be > 100 but less then 120(ish)\n\tif pingTime < pingInterval {\n\t\tt.Fatalf(\"pingTime too low: %v\", pingTime)\n\t}\n\t// +5 is just for fudging in case things are slow in the testing system.\n\tif pingTime > highWater {\n\t\tt.Fatalf(\"pingTime too high: %v\", pingTime)\n\t}\n\n\ttime.Sleep(pingInterval / 2)\n\n\t// Sending a PING should suppress.\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// This will wait for the time period where a PING should have fired\n\t// and been delivered. We expect nothing here since it should be suppressed.\n\texpectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond))\n}\n" }, { "path": "test/port_test.go", "content": "// Copyright 2014-2019 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc TestResolveRandomPort(t *testing.T) {\n\topts := &server.Options{Host: \"127.0.0.1\", Port: server.RANDOM_PORT, NoSigs: true}\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\taddr := s.Addr()\n\t_, port, err := net.SplitHostPort(addr.String())\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\n\tportNum, err := strconv.Atoi(port)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error: Got %v\\n\", err)\n\t}\n\n\tif portNum == server.DEFAULT_PORT {\n\t\tt.Fatalf(\"Expected server to choose a random port\\nGot: %d\", server.DEFAULT_PORT)\n\t}\n\n\tif portNum == server.RANDOM_PORT {\n\t\tt.Fatalf(\"Expected server to choose a random port\\nGot: %d\", server.RANDOM_PORT)\n\t}\n\n\tif opts.Port != portNum {\n\t\tt.Fatalf(\"Options port (%d) should have been overridden by chosen random port (%d)\",\n\t\t\topts.Port, portNum)\n\t}\n}\n" }, { "path": "test/ports_test.go", "content": "// Copyright 2018-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\n// waits until a calculated list of listeners is resolved or a timeout\nfunc waitForFile(path string, dur time.Duration) ([]byte, error) {\n\tend := time.Now().Add(dur)\n\tfor time.Now().Before(end) {\n\t\tif _, err := os.Stat(path); os.IsNotExist(err) {\n\t\t\ttime.Sleep(25 * time.Millisecond)\n\t\t\tcontinue\n\t\t} else {\n\t\t\treturn os.ReadFile(path)\n\t\t}\n\t}\n\treturn nil, errors.New(\"Timeout\")\n}\n\nfunc portFile(dirname string) string {\n\treturn filepath.Join(dirname, fmt.Sprintf(\"%s_%d.ports\", filepath.Base(os.Args[0]), os.Getpid()))\n}\n\nfunc TestPortsFile(t *testing.T) {\n\tportFileDir := t.TempDir()\n\n\topts := DefaultTestOptions\n\topts.PortsFileDir = portFileDir\n\topts.Port = -1\n\topts.HTTPPort = -1\n\topts.ProfPort = -1\n\topts.Cluster.Port = -1\n\topts.Websocket.Port = -1\n\ttc := &server.TLSConfigOpts{\n\t\tCertFile: \"./configs/certs/server-cert.pem\",\n\t\tKeyFile: \"./configs/certs/server-key.pem\",\n\t}\n\topts.Websocket.TLSConfig, _ = server.GenTLSConfig(tc)\n\n\ts := RunServer(&opts)\n\t// this for test cleanup in case we fail - will be ignored if server already shutdown\n\tdefer s.Shutdown()\n\n\tports := s.PortsInfo(5 * time.Second)\n\n\tif ports == nil {\n\t\tt.Fatal(\"services failed to start in 5 seconds\")\n\t}\n\n\t// the pid file should be\n\tportsFile := portFile(portFileDir)\n\n\tif portsFile == \"\" {\n\t\tt.Fatal(\"Expected a ports file\")\n\t}\n\n\t// try to read a file here - the file should be a json\n\tbuf, err := waitForFile(portsFile, 5*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read ports file: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length ports file\")\n\t}\n\n\treadPorts := server.Ports{}\n\tjson.Unmarshal(buf, &readPorts)\n\n\tif len(readPorts.Nats) == 0 || !strings.HasPrefix(readPorts.Nats[0], \"nats://\") {\n\t\tt.Fatal(\"Expected at least one nats url\")\n\t}\n\n\tif len(readPorts.Monitoring) == 0 || !strings.HasPrefix(readPorts.Monitoring[0], \"http://\") {\n\t\tt.Fatal(\"Expected at least one monitoring url\")\n\t}\n\n\tif len(readPorts.Cluster) == 0 || !strings.HasPrefix(readPorts.Cluster[0], \"nats://\") {\n\t\tt.Fatal(\"Expected at least one cluster listen url\")\n\t}\n\n\tif len(readPorts.Profile) == 0 || !strings.HasPrefix(readPorts.Profile[0], \"http://\") {\n\t\tt.Fatal(\"Expected at least one profile listen url\")\n\t}\n\n\tif len(readPorts.WebSocket) == 0 || !strings.HasPrefix(readPorts.WebSocket[0], \"wss://\") {\n\t\tt.Fatal(\"Expected at least one ws listen url\")\n\t}\n\n\t// testing cleanup\n\ts.Shutdown()\n\t// if we called shutdown, the cleanup code should have kicked\n\tif _, err := os.Stat(portsFile); os.IsNotExist(err) {\n\t\t// good\n\t} else {\n\t\tt.Fatalf(\"the port file %s was not deleted\", portsFile)\n\t}\n}\n\n// makes a temp directory with two directories 'A' and 'B'\n// the location of the ports file is changed from dir A to dir B.\nfunc TestPortsFileReload(t *testing.T) {\n\ttempDir := t.TempDir()\n\n\t// make child temp dir A\n\tdirA := filepath.Join(tempDir, \"A\")\n\tos.MkdirAll(dirA, 0777)\n\n\t// write the config file with a reference to A\n\tconfig := fmt.Sprintf(`\n\t\tports_file_dir: \"%s\"\n\t\tport: -1\n\t`, dirA)\n\tconfig = strings.Replace(config, \"\\\\\", \"/\", -1)\n\tconfPath := filepath.Join(tempDir, fmt.Sprintf(\"%d.conf\", os.Getpid()))\n\tif err := os.WriteFile(confPath, []byte(config), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing ports file (%s): %v\", confPath, err)\n\t}\n\n\topts, err := server.ProcessConfigFile(confPath)\n\tif err != nil {\n\t\tt.Fatalf(\"Error processing the configuration: %v\", err)\n\t}\n\n\ts := RunServer(opts)\n\tdefer s.Shutdown()\n\n\tports := s.PortsInfo(5 * time.Second)\n\tif ports == nil {\n\t\tt.Fatal(\"services failed to start in 5 seconds\")\n\t}\n\n\t// get the ports file path name\n\tportsFileInA := portFile(dirA)\n\t// the file should be in dirA\n\tif !strings.HasPrefix(portsFileInA, dirA) {\n\t\tt.Fatalf(\"expected ports file to be in [%s] but was in [%s]\", dirA, portsFileInA)\n\t}\n\t// wait for it\n\tbuf, err := waitForFile(portsFileInA, 5*time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read ports file: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length ports file\")\n\t}\n\n\t// change the configuration for the ports file to dirB\n\tdirB := filepath.Join(tempDir, \"B\")\n\tos.MkdirAll(dirB, 0777)\n\n\tconfig = fmt.Sprintf(`\n\t\tports_file_dir: \"%s\"\n\t\tport: -1\n\t`, dirB)\n\tconfig = strings.Replace(config, \"\\\\\", \"/\", -1)\n\tif err := os.WriteFile(confPath, []byte(config), 0666); err != nil {\n\t\tt.Fatalf(\"Error writing ports file (%s): %v\", confPath, err)\n\t}\n\n\t// reload the server\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"error reloading server: %v\", err)\n\t}\n\n\t// wait for the new file to show up\n\tportsFileInB := portFile(dirB)\n\tbuf, err = waitForFile(portsFileInB, 5*time.Second)\n\tif !strings.HasPrefix(portsFileInB, dirB) {\n\t\tt.Fatalf(\"expected ports file to be in [%s] but was in [%s]\", dirB, portsFileInB)\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"Could not read ports file: %v\", err)\n\t}\n\tif len(buf) <= 0 {\n\t\tt.Fatal(\"Expected a non-zero length ports file\")\n\t}\n\n\t// the file in dirA should have deleted\n\tif _, err := os.Stat(portsFileInA); os.IsNotExist(err) {\n\t\t// good\n\t} else {\n\t\tt.Fatalf(\"the port file %s was not deleted\", portsFileInA)\n\t}\n}\n" }, { "path": "test/proto_test.go", "content": "// Copyright 2012-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nconst PROTO_TEST_PORT = 9922\n\nfunc runProtoServer() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = PROTO_TEST_PORT\n\topts.MaxControlLine = 256\n\topts.NoSystemAccount = true\n\treturn RunServer(&opts)\n}\n\nfunc TestProtoBasics(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\t// Ping\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\t// Single Msg\n\tsend(\"SUB foo 1\\r\\nPUB foo 5\\r\\nhello\\r\\n\")\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"5\", \"hello\")\n\n\t// 2 Messages\n\tsend(\"SUB * 2\\r\\nPUB foo 2\\r\\nok\\r\\n\")\n\tmatches = expectMsgs(2)\n\t// Could arrive in any order\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n\tcheckMsg(t, matches[1], \"foo\", \"\", \"\", \"2\", \"ok\")\n}\n\nfunc TestProtoErr(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\t// Make sure we get an error on bad proto\n\tsend(\"ZZZ\")\n\texpect(errRe)\n}\n\nfunc TestUnsubMax(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\tsend(\"SUB foo 22\\r\\n\")\n\tsend(\"UNSUB 22 2\\r\\n\")\n\tfor i := 0; i < 100; i++ {\n\t\tsend(\"PUB foo 2\\r\\nok\\r\\n\")\n\t}\n\n\ttime.Sleep(50 * time.Millisecond)\n\n\tmatches := expectMsgs(2)\n\tcheckMsg(t, matches[0], \"foo\", \"22\", \"\", \"2\", \"ok\")\n\tcheckMsg(t, matches[1], \"foo\", \"22\", \"\", \"2\", \"ok\")\n}\n\nfunc TestQueueSub(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\tsent := 100\n\tsend(\"SUB foo qgroup1 22\\r\\n\")\n\tsend(\"SUB foo qgroup1 32\\r\\n\")\n\tfor i := 0; i < sent; i++ {\n\t\tsend(\"PUB foo 2\\r\\nok\\r\\n\")\n\t}\n\t// Wait for responses\n\ttime.Sleep(250 * time.Millisecond)\n\n\tmatches := expectMsgs(sent)\n\tsids := make(map[string]int)\n\tfor _, m := range matches {\n\t\tsids[string(m[sidIndex])]++\n\t}\n\tif len(sids) != 2 {\n\t\tt.Fatalf(\"Expected only 2 sids, got %d\\n\", len(sids))\n\t}\n\tfor k, c := range sids {\n\t\tif c < 35 {\n\t\t\tt.Fatalf(\"Expected ~50 (+-15) msgs for sid:'%s', got %d\\n\", k, c)\n\t\t}\n\t}\n}\n\nfunc TestMultipleQueueSub(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\texpectMsgs := expectMsgsCommand(t, expect)\n\n\tsent := 100\n\tsend(\"SUB foo g1 1\\r\\n\")\n\tsend(\"SUB foo g1 2\\r\\n\")\n\tsend(\"SUB foo g2 3\\r\\n\")\n\tsend(\"SUB foo g2 4\\r\\n\")\n\n\tfor i := 0; i < sent; i++ {\n\t\tsend(\"PUB foo 2\\r\\nok\\r\\n\")\n\t}\n\t// Wait for responses\n\ttime.Sleep(250 * time.Millisecond)\n\n\tmatches := expectMsgs(sent * 2)\n\tsids := make(map[string]int)\n\tfor _, m := range matches {\n\t\tsids[string(m[sidIndex])]++\n\t}\n\tif len(sids) != 4 {\n\t\tt.Fatalf(\"Expected 4 sids, got %d\\n\", len(sids))\n\t}\n\tfor k, c := range sids {\n\t\tif c < 35 {\n\t\t\tt.Fatalf(\"Expected ~50 (+-15) msgs for '%s', got %d\\n\", k, c)\n\t\t}\n\t}\n}\n\nfunc TestPubToArgState(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\tsend(\"PUBS foo 2\\r\\nok\\r\\n\")\n\texpect(errRe)\n}\n\nfunc TestSubToArgState(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\tsend(\"SUBZZZ foo 1\\r\\n\")\n\texpect(errRe)\n}\n\n// Issue #63\nfunc TestProtoCrash(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := sendCommand(t, c), expectCommand(t, c)\n\n\tcheckInfoMsg(t, c)\n\n\tsend(\"CONNECT {\\\"verbose\\\":true,\\\"tls_required\\\":false,\\\"user\\\":\\\"test\\\",\\\"pedantic\\\":true,\\\"pass\\\":\\\"password\\\"}\")\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tsend(\"\\r\\n\")\n\texpect(okRe)\n}\n\n// Issue #136\nfunc TestDuplicateProtoSub(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tsend(\"SUB foo 1\\r\\n\")\n\n\tsend(\"SUB foo 1\\r\\n\")\n\n\tns := 0\n\n\tfor i := 0; i < 5; i++ {\n\t\tns = int(s.NumSubscriptions())\n\t\tif ns == 0 {\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif ns != 1 {\n\t\tt.Fatalf(\"Expected 1 subscription, got %d\\n\", ns)\n\t}\n}\n\nfunc TestIncompletePubArg(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\tsend, expect := setupConn(t, c)\n\n\tsize := 10000\n\tgoodBuf := \"\"\n\tfor i := 0; i < size; i++ {\n\t\tgoodBuf += \"A\"\n\t}\n\tgoodBuf += \"\\r\\n\"\n\n\tbadSize := 3371\n\tbadBuf := \"\"\n\tfor i := 0; i < badSize; i++ {\n\t\tbadBuf += \"B\"\n\t}\n\t// Message is corrupted and since we are still reading from client,\n\t// next PUB accidentally becomes part of the payload of the\n\t// incomplete message thus breaking the protocol.\n\tbadBuf2 := \"\"\n\tfor i := 0; i < size; i++ {\n\t\tbadBuf2 += \"C\"\n\t}\n\tbadBuf2 += \"\\r\\n\"\n\n\tpub := \"PUB example 10000\\r\\n\"\n\tsend(pub + goodBuf + pub + goodBuf + pub + badBuf + pub + badBuf2)\n\texpect(errRe)\n}\n\nfunc TestControlLineMaximums(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tsend, expect := setupConn(t, c)\n\n\tpubTooLong := \"PUB foo \"\n\tfor i := 0; i < 32; i++ {\n\t\tpubTooLong += \"2222222222\"\n\t}\n\tsend(pubTooLong)\n\texpect(errRe)\n}\n\nfunc TestServerInfoWithClientAdvertise(t *testing.T) {\n\topts := DefaultTestOptions\n\topts.Port = PROTO_TEST_PORT\n\topts.ClientAdvertise = \"me:1\"\n\ts := RunServer(&opts)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tbuf := expectResult(t, c, infoRe)\n\tjs := infoRe.FindAllSubmatch(buf, 1)[0][1]\n\tvar sinfo server.Info\n\terr := json.Unmarshal(js, &sinfo)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not unmarshal INFO json: %v\\n\", err)\n\t}\n\tif sinfo.Host != \"me\" || sinfo.Port != 1 {\n\t\tt.Fatalf(\"Expected INFO Host:Port to be me:1, got %s:%d\", sinfo.Host, sinfo.Port)\n\t}\n}\n" }, { "path": "test/route_discovery_test.go", "content": "// Copyright 2015-2024 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nfunc runSeedServer(t *testing.T) (*server.Server, *server.Options) {\n\treturn RunServerWithConfig(\"./configs/seed.conf\")\n}\n\nfunc runAuthSeedServer(t *testing.T) (*server.Server, *server.Options) {\n\treturn RunServerWithConfig(\"./configs/auth_seed.conf\")\n}\n\nfunc TestSeedFirstRouteInfo(t *testing.T) {\n\ts, opts := runSeedServer(t)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\t_, routeExpect := setupRoute(t, rc, opts)\n\tbuf := routeExpect(infoRe)\n\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif info.ID != s.ID() {\n\t\tt.Fatalf(\"Expected seed's ID %q, got %q\", s.ID(), info.ID)\n\t}\n}\n\nfunc TestSeedMultipleRouteInfo(t *testing.T) {\n\ts, opts := runSeedServer(t)\n\tdefer s.Shutdown()\n\n\trc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc1.Close()\n\n\trc1ID := \"2222\"\n\trc1Port := 22\n\trc1Host := \"127.0.0.1\"\n\n\trouteSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID)\n\troute1Expect(infoRe)\n\n\t// register ourselves via INFO\n\tr1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port}\n\tb, _ := json.Marshal(r1Info)\n\tinfoJSON := fmt.Sprintf(server.InfoProto, b)\n\trouteSend1(infoJSON)\n\trouteSend1(\"PING\\r\\n\")\n\troute1Expect(pongRe)\n\n\trc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc2.Close()\n\n\trc2ID := \"2224\"\n\trc2Port := 24\n\trc2Host := \"127.0.0.1\"\n\n\trouteSend2, route2Expect := setupRouteEx(t, rc2, opts, rc2ID)\n\n\thp2 := fmt.Sprintf(\"nats-route://%s/\", net.JoinHostPort(rc2Host, strconv.Itoa(rc2Port)))\n\n\t// register ourselves via INFO\n\tr2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port}\n\tb, _ = json.Marshal(r2Info)\n\tinfoJSON = fmt.Sprintf(server.InfoProto, b)\n\trouteSend2(infoJSON)\n\n\t// Now read back the second INFO route1 should receive letting\n\t// it know about route2\n\tbuf := route1Expect(infoRe)\n\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif info.ID != rc2ID {\n\t\tt.Fatalf(\"Expected info.ID to be %q, got %q\", rc2ID, info.ID)\n\t}\n\tif info.IP == \"\" {\n\t\tt.Fatalf(\"Expected a IP for the implicit route\")\n\t}\n\tif info.IP != hp2 {\n\t\tt.Fatalf(\"Expected IP Host of %s, got %s\\n\", hp2, info.IP)\n\t}\n\n\troute2Expect(infoRe)\n\trouteSend2(\"PING\\r\\n\")\n\troute2Expect(pongRe)\n\n\t// Now let's do a third.\n\trc3 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc3.Close()\n\n\trc3ID := \"2226\"\n\trc3Port := 26\n\trc3Host := \"127.0.0.1\"\n\n\trouteSend3, _ := setupRouteEx(t, rc3, opts, rc3ID)\n\n\t// register ourselves via INFO\n\tr3Info := server.Info{ID: rc3ID, Host: rc3Host, Port: rc3Port}\n\tb, _ = json.Marshal(r3Info)\n\tinfoJSON = fmt.Sprintf(server.InfoProto, b)\n\trouteSend3(infoJSON)\n\n\t// Now read back out the info from the seed route\n\tbuf = route1Expect(infoRe)\n\n\tinfo = server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif info.ID != rc3ID {\n\t\tt.Fatalf(\"Expected info.ID to be %q, got %q\", rc3ID, info.ID)\n\t}\n\n\t// Now read back out the info from the seed route\n\tbuf = route2Expect(infoRe)\n\n\tinfo = server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif info.ID != rc3ID {\n\t\tt.Fatalf(\"Expected info.ID to be %q, got %q\", rc3ID, info.ID)\n\t}\n}\n\nfunc TestSeedSolicitWorks(t *testing.T) {\n\ts1, opts := runSeedServer(t)\n\tdefer s1.Shutdown()\n\n\t// Create the routes string for others to connect to the seed.\n\troutesStr := fmt.Sprintf(\"nats-route://%s:%d/\", opts.Cluster.Host, opts.Cluster.Port)\n\n\t// Run Server #2\n\ts2Opts := nextServerOpts(opts)\n\ts2Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts2 := RunServer(s2Opts)\n\tdefer s2.Shutdown()\n\n\t// Run Server #3\n\ts3Opts := nextServerOpts(s2Opts)\n\n\ts3 := RunServer(s3Opts)\n\tdefer s3.Shutdown()\n\n\t// Wait for a bit for graph to connect\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Grab Routez from monitor ports, make sure we are fully connected.\n\turl := fmt.Sprintf(\"http://%s:%d/\", opts.Host, opts.HTTPPort)\n\trz := readHTTPRoutez(t, url)\n\tfor _, route := range rz.Routes {\n\t\tif route.LastActivity.IsZero() {\n\t\t\tt.Error(\"Expected LastActivity to be valid\\n\")\n\t\t}\n\t}\n\tris := expectRids(t, rz, []string{s2.ID(), s3.ID()})\n\tif ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\t// Server 2 did solicit routes to Server 1.\n\turl = fmt.Sprintf(\"http://%s:%d/\", s2Opts.Host, s2Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tfor _, route := range rz.Routes {\n\t\tif route.LastActivity.IsZero() {\n\t\t\tt.Error(\"Expected LastActivity to be valid\")\n\t\t}\n\t}\n\tris = expectRids(t, rz, []string{s1.ID(), s3.ID()})\n\tif !ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\turl = fmt.Sprintf(\"http://%s:%d/\", s3Opts.Host, s3Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tfor _, route := range rz.Routes {\n\t\tif route.LastActivity.IsZero() {\n\t\t\tt.Error(\"Expected LastActivity to be valid\")\n\t\t}\n\t}\n\tris = expectRids(t, rz, []string{s1.ID(), s2.ID()})\n\tif !ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server to be configured\\n\")\n\t}\n\tif ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n}\n\ntype serverInfo struct {\n\tserver *server.Server\n\topts *server.Options\n}\n\nfunc checkConnected(t *testing.T, servers []serverInfo, current int, oneSeed bool) error {\n\ts := servers[current]\n\n\t// Grab Routez from monitor ports, make sure we are fully connected\n\turl := fmt.Sprintf(\"http://%s:%d/\", s.opts.Host, s.opts.HTTPPort)\n\trz := readHTTPRoutez(t, url)\n\ttotal := len(servers)\n\tvar ids []string\n\tfor i := 0; i < total; i++ {\n\t\tif i == current {\n\t\t\tcontinue\n\t\t}\n\t\tids = append(ids, servers[i].server.ID())\n\t}\n\tris, err := expectRidsNoFatal(t, true, rz, ids)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor i := 0; i < total; i++ {\n\t\tif i == current {\n\t\t\tcontinue\n\t\t}\n\t\ts := servers[i]\n\t\tif current == 0 || ((oneSeed && i > 0) || (!oneSeed && (i != current-1))) {\n\t\t\tif ris[s.server.ID()].IsConfigured {\n\t\t\t\treturn fmt.Errorf(\"Expected server %s:%d not to be configured\", s.opts.Host, s.opts.Port)\n\t\t\t}\n\t\t} else if oneSeed || (i == current-1) {\n\t\t\tif !ris[s.server.ID()].IsConfigured {\n\t\t\t\treturn fmt.Errorf(\"Expected server %s:%d to be configured\", s.opts.Host, s.opts.Port)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc TestStressSeedSolicitWorks(t *testing.T) {\n\ts1, opts := runSeedServer(t)\n\tdefer s1.Shutdown()\n\n\t// Create the routes string for others to connect to the seed.\n\troutesStr := fmt.Sprintf(\"nats-route://%s:%d/\", opts.Cluster.Host, opts.Cluster.Port)\n\n\ts2Opts := nextServerOpts(opts)\n\ts2Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts3Opts := nextServerOpts(s2Opts)\n\ts4Opts := nextServerOpts(s3Opts)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfunc() {\n\t\t\t// Run these servers manually, because we want them to start and\n\t\t\t// connect to s1 as fast as possible.\n\n\t\t\ts2 := server.New(s2Opts)\n\t\t\tif s2 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s2.Shutdown()\n\t\t\tgo s2.Start()\n\n\t\t\ts3 := server.New(s3Opts)\n\t\t\tif s3 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s3.Shutdown()\n\t\t\tgo s3.Start()\n\n\t\t\ts4 := server.New(s4Opts)\n\t\t\tif s4 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s4.Shutdown()\n\t\t\tgo s4.Start()\n\n\t\t\tserversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}}\n\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tfor j := 0; j < len(serversInfo); j++ {\n\t\t\t\t\tif err := checkConnected(t, serversInfo, j, true); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}()\n\t\tcheckNumRoutes(t, s1, 0)\n\t}\n}\n\nfunc TestChainedSolicitWorks(t *testing.T) {\n\ts1, opts := runSeedServer(t)\n\tdefer s1.Shutdown()\n\n\t// Create the routes string for others to connect to the seed.\n\troutesStr := fmt.Sprintf(\"nats-route://%s:%d/\", opts.Cluster.Host, opts.Cluster.Port)\n\n\t// Run Server #2\n\ts2Opts := nextServerOpts(opts)\n\ts2Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts2 := RunServer(s2Opts)\n\tdefer s2.Shutdown()\n\n\t// Run Server #3\n\ts3Opts := nextServerOpts(s2Opts)\n\t// We will have s3 connect to s2, not the seed.\n\troutesStr = fmt.Sprintf(\"nats-route://%s:%d/\", s2Opts.Cluster.Host, s2Opts.Cluster.Port)\n\ts3Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts3 := RunServer(s3Opts)\n\tdefer s3.Shutdown()\n\n\t// Wait for a bit for graph to connect\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Grab Routez from monitor ports, make sure we are fully connected\n\turl := fmt.Sprintf(\"http://%s:%d/\", opts.Host, opts.HTTPPort)\n\trz := readHTTPRoutez(t, url)\n\tris := expectRids(t, rz, []string{s2.ID(), s3.ID()})\n\tif ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\turl = fmt.Sprintf(\"http://%s:%d/\", s2Opts.Host, s2Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tris = expectRids(t, rz, []string{s1.ID(), s3.ID()})\n\tif !ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\turl = fmt.Sprintf(\"http://%s:%d/\", s3Opts.Host, s3Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tris = expectRids(t, rz, []string{s1.ID(), s2.ID()})\n\tif !ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected s2 server to be configured\\n\")\n\t}\n\tif ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server not to be configured\\n\")\n\t}\n}\n\nfunc TestStressChainedSolicitWorks(t *testing.T) {\n\ts1, opts := runSeedServer(t)\n\tdefer s1.Shutdown()\n\n\t// Create the routes string for s2 to connect to the seed\n\troutesStr := fmt.Sprintf(\"nats-route://%s:%d/\", opts.Cluster.Host, opts.Cluster.Port)\n\ts2Opts := nextServerOpts(opts)\n\ts2Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts3Opts := nextServerOpts(s2Opts)\n\t// Create the routes string for s3 to connect to s2\n\troutesStr = fmt.Sprintf(\"nats-route://%s:%d/\", s2Opts.Cluster.Host, s2Opts.Cluster.Port)\n\ts3Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts4Opts := nextServerOpts(s3Opts)\n\t// Create the routes string for s4 to connect to s3\n\troutesStr = fmt.Sprintf(\"nats-route://%s:%d/\", s3Opts.Cluster.Host, s3Opts.Cluster.Port)\n\ts4Opts.Routes = server.RoutesFromStr(routesStr)\n\n\tfor i := 0; i < 10; i++ {\n\t\tfunc() {\n\t\t\t// Run these servers manually, because we want them to start and\n\t\t\t// connect to s1 as fast as possible.\n\n\t\t\ts2 := server.New(s2Opts)\n\t\t\tif s2 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s2.Shutdown()\n\t\t\tgo s2.Start()\n\n\t\t\ts3 := server.New(s3Opts)\n\t\t\tif s3 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s3.Shutdown()\n\t\t\tgo s3.Start()\n\n\t\t\ts4 := server.New(s4Opts)\n\t\t\tif s4 == nil {\n\t\t\t\tpanic(\"No NATS Server object returned.\")\n\t\t\t}\n\t\t\tdefer s4.Shutdown()\n\t\t\tgo s4.Start()\n\n\t\t\tserversInfo := []serverInfo{{s1, opts}, {s2, s2Opts}, {s3, s3Opts}, {s4, s4Opts}}\n\n\t\t\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\t\t\tfor j := 0; j < len(serversInfo); j++ {\n\t\t\t\t\tif err := checkConnected(t, serversInfo, j, false); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}()\n\t\tcheckNumRoutes(t, s1, 0)\n\t}\n}\n\nfunc TestAuthSeedSolicitWorks(t *testing.T) {\n\ts1, opts := runAuthSeedServer(t)\n\tdefer s1.Shutdown()\n\n\t// Create the routes string for others to connect to the seed.\n\troutesStr := fmt.Sprintf(\"nats-route://%s:%s@%s:%d/\", opts.Cluster.Username, opts.Cluster.Password, opts.Cluster.Host, opts.Cluster.Port)\n\n\t// Run Server #2\n\ts2Opts := nextServerOpts(opts)\n\ts2Opts.Routes = server.RoutesFromStr(routesStr)\n\n\ts2 := RunServer(s2Opts)\n\tdefer s2.Shutdown()\n\n\t// Run Server #3\n\ts3Opts := nextServerOpts(s2Opts)\n\n\ts3 := RunServer(s3Opts)\n\tdefer s3.Shutdown()\n\n\t// Wait for a bit for graph to connect\n\ttime.Sleep(500 * time.Millisecond)\n\n\t// Grab Routez from monitor ports, make sure we are fully connected\n\turl := fmt.Sprintf(\"http://%s:%d/\", opts.Host, opts.HTTPPort)\n\trz := readHTTPRoutez(t, url)\n\tris := expectRids(t, rz, []string{s2.ID(), s3.ID()})\n\tif ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\turl = fmt.Sprintf(\"http://%s:%d/\", s2Opts.Host, s2Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tris = expectRids(t, rz, []string{s1.ID(), s3.ID()})\n\tif !ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server to be configured\\n\")\n\t}\n\tif ris[s3.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n\n\turl = fmt.Sprintf(\"http://%s:%d/\", s3Opts.Host, s3Opts.HTTPPort)\n\trz = readHTTPRoutez(t, url)\n\tris = expectRids(t, rz, []string{s1.ID(), s2.ID()})\n\tif !ris[s1.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected seed server to be configured\\n\")\n\t}\n\tif ris[s2.ID()].IsConfigured {\n\t\tt.Fatalf(\"Expected server not to be configured\\n\")\n\t}\n}\n\n// Helper to check for correct route memberships\nfunc expectRids(t *testing.T, rz *server.Routez, rids []string) map[string]*server.RouteInfo {\n\tri, err := expectRidsNoFatal(t, false, rz, rids)\n\tif err != nil {\n\t\tt.Fatalf(\"%v\", err)\n\t}\n\treturn ri\n}\n\nfunc expectRidsNoFatal(t *testing.T, direct bool, rz *server.Routez, rids []string) (map[string]*server.RouteInfo, error) {\n\tcaller := 1\n\tif !direct {\n\t\tcaller++\n\t}\n\tif len(rids) != rz.NumRoutes {\n\t\t_, fn, line, _ := runtime.Caller(caller)\n\t\treturn nil, fmt.Errorf(\"[%s:%d] Expecting %d routes, got %d\\n\", fn, line, len(rids), rz.NumRoutes)\n\t}\n\tset := make(map[string]bool)\n\tfor _, v := range rids {\n\t\tset[v] = true\n\t}\n\t// Make result map for additional checking\n\tri := make(map[string]*server.RouteInfo)\n\tfor _, r := range rz.Routes {\n\t\tif !set[r.RemoteID] {\n\t\t\t_, fn, line, _ := runtime.Caller(caller)\n\t\t\treturn nil, fmt.Errorf(\"[%s:%d] Route with rid %s unexpected, expected %+v\\n\", fn, line, r.RemoteID, rids)\n\t\t}\n\t\tri[r.RemoteID] = r\n\t}\n\treturn ri, nil\n}\n\n// Helper to easily grab routez info.\nfunc readHTTPRoutez(t *testing.T, url string) *server.Routez {\n\tresetPreviousHTTPConnections()\n\tresp, err := http.Get(url + \"routez\")\n\tif err != nil {\n\t\tstackFatalf(t, \"Expected no error: Got %v\\n\", err)\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != 200 {\n\t\tstackFatalf(t, \"Expected a 200 response, got %d\\n\", resp.StatusCode)\n\t}\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\tstackFatalf(t, \"Got an error reading the body: %v\\n\", err)\n\t}\n\tr := server.Routez{}\n\tif err := json.Unmarshal(body, &r); err != nil {\n\t\tstackFatalf(t, \"Got an error unmarshalling the body: %v\\n\", err)\n\t}\n\treturn &r\n}\n\nfunc TestSeedReturnIPInInfo(t *testing.T) {\n\ts, opts := runSeedServer(t)\n\tdefer s.Shutdown()\n\n\trc1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc1.Close()\n\n\trc1ID := \"2222\"\n\trc1Port := 22\n\trc1Host := \"127.0.0.1\"\n\n\trouteSend1, route1Expect := setupRouteEx(t, rc1, opts, rc1ID)\n\troute1Expect(infoRe)\n\n\t// register ourselves via INFO\n\tr1Info := server.Info{ID: rc1ID, Host: rc1Host, Port: rc1Port}\n\tb, _ := json.Marshal(r1Info)\n\tinfoJSON := fmt.Sprintf(server.InfoProto, b)\n\trouteSend1(infoJSON)\n\trouteSend1(\"PING\\r\\n\")\n\troute1Expect(pongRe)\n\n\trc2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc2.Close()\n\n\trc2ID := \"2224\"\n\trc2Port := 24\n\trc2Host := \"127.0.0.1\"\n\n\trouteSend2, _ := setupRouteEx(t, rc2, opts, rc2ID)\n\n\t// register ourselves via INFO\n\tr2Info := server.Info{ID: rc2ID, Host: rc2Host, Port: rc2Port}\n\tb, _ = json.Marshal(r2Info)\n\tinfoJSON = fmt.Sprintf(server.InfoProto, b)\n\trouteSend2(infoJSON)\n\n\t// Now read info that route1 should have received from the seed\n\tbuf := route1Expect(infoRe)\n\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif info.IP == \"\" {\n\t\tt.Fatal(\"Expected to have IP in INFO\")\n\t}\n\trip, _, err := net.SplitHostPort(strings.TrimPrefix(info.IP, \"nats-route://\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error parsing url: %v\", err)\n\t}\n\taddr, ok := rc1.RemoteAddr().(*net.TCPAddr)\n\tif !ok {\n\t\tt.Fatal(\"Unable to get IP address from route\")\n\t}\n\ts1 := strings.ToLower(addr.IP.String())\n\ts2 := strings.ToLower(rip)\n\tif s1 != s2 {\n\t\tt.Fatalf(\"Expected IP %s, got %s\", s1, s2)\n\t}\n}\n\nfunc TestImplicitRouteRetry(t *testing.T) {\n\tsrvSeed, optsSeed := runSeedServer(t)\n\tdefer srvSeed.Shutdown()\n\n\toptsA := nextServerOpts(optsSeed)\n\toptsA.Routes = server.RoutesFromStr(fmt.Sprintf(\"nats://%s:%d\", optsSeed.Cluster.Host, optsSeed.Cluster.Port))\n\toptsA.Cluster.ConnectRetries = 5\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB := nextServerOpts(optsA)\n\trcb := createRouteConn(t, optsSeed.Cluster.Host, optsSeed.Cluster.Port)\n\tdefer rcb.Close()\n\trcbID := \"ServerB\"\n\trouteBSend, routeBExpect := setupRouteEx(t, rcb, optsB, rcbID)\n\trouteBExpect(infoRe)\n\t// register ourselves via INFO\n\trbInfo := server.Info{ID: rcbID, Host: optsB.Cluster.Host, Port: optsB.Cluster.Port}\n\tb, _ := json.Marshal(rbInfo)\n\tinfoJSON := fmt.Sprintf(server.InfoProto, b)\n\trouteBSend(infoJSON)\n\trouteBSend(\"PING\\r\\n\")\n\trouteBExpect(pongRe)\n\n\t// srvA should try to connect. Wait to make sure that it fails.\n\ttime.Sleep(1200 * time.Millisecond)\n\n\t// Setup a fake route listen for routeB\n\trbListen, err := net.Listen(\"tcp\", fmt.Sprintf(\"%s:%d\", optsB.Cluster.Host, optsB.Cluster.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error during listen: %v\", err)\n\t}\n\tdefer rbListen.Close()\n\tc, err := rbListen.Accept()\n\tif err != nil {\n\t\tt.Fatalf(\"Error during accept: %v\", err)\n\t}\n\tdefer c.Close()\n\n\tbr := bufio.NewReaderSize(c, 32768)\n\t// Consume CONNECT and INFO\n\tfor i := 0; i < 2; i++ {\n\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\tbuf, _, err := br.ReadLine()\n\t\tc.SetReadDeadline(time.Time{})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error reading: %v\", err)\n\t\t}\n\t\tif i == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tbuf = buf[len(\"INFO \"):]\n\t\tinfo := &server.Info{}\n\t\tif err := json.Unmarshal(buf, info); err != nil {\n\t\t\tt.Fatalf(\"Error during unmarshal: %v\", err)\n\t\t}\n\t\t// Check INFO is from server A.\n\t\tif info.ID != srvA.ID() {\n\t\t\tt.Fatalf(\"Expected CONNECT from %v, got CONNECT from %v\", srvA.ID(), info.ID)\n\t\t}\n\t}\n}\n" }, { "path": "test/routes_test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/internal/testhelper\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nconst clientProtoInfo = 1\n\nfunc runRouteServer(t *testing.T) (*server.Server, *server.Options) {\n\treturn RunServerWithConfig(\"./configs/cluster.conf\")\n}\n\nfunc runRouteServerOverrides(t *testing.T, cbo func(*server.Options), cbs func(*server.Server)) (*server.Server, *server.Options) {\n\treturn RunServerWithConfigOverrides(\"./configs/cluster.conf\", cbo, cbs)\n}\n\nfunc TestRouterListeningSocket(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\t// Check that the cluster socket is able to be connected.\n\taddr := fmt.Sprintf(\"%s:%d\", opts.Cluster.Host, opts.Cluster.Port)\n\tcheckSocket(t, addr, 2*time.Second)\n}\n\nfunc TestRouteGoServerShutdown(t *testing.T) {\n\tbase := runtime.NumGoroutine()\n\ts, _ := runRouteServer(t)\n\ts.Shutdown()\n\ttime.Sleep(50 * time.Millisecond)\n\tdelta := (runtime.NumGoroutine() - base)\n\tif delta > 1 {\n\t\tt.Fatalf(\"%d Go routines still exist post Shutdown()\", delta)\n\t}\n}\n\nfunc TestSendRouteInfoOnConnect(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteID := \"RouteID\"\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\tbuf := routeExpect(infoRe)\n\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif !info.AuthRequired {\n\t\tt.Fatal(\"Expected to see AuthRequired\")\n\t}\n\tif info.Port != opts.Cluster.Port {\n\t\tt.Fatalf(\"Received wrong information for port, expected %d, got %d\",\n\t\t\tinfo.Port, opts.Cluster.Port)\n\t}\n\n\t// Need to send a different INFO than the one received, otherwise the server\n\t// will detect as a \"cycle\" and close the connection.\n\tinfo.ID = routeID\n\tinfo.Name = \"\"\n\tb, err := json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\tinfoJSON := fmt.Sprintf(\"INFO %s\\r\\n\", b)\n\trouteSend(infoJSON)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n}\n\nfunc TestRouteToSelf(t *testing.T) {\n\tl := testhelper.NewDummyLogger(100)\n\ts, opts := runRouteServerOverrides(t, nil,\n\t\tfunc(s *server.Server) {\n\t\t\ts.SetLogger(l, true, true)\n\t\t})\n\tdefer s.Shutdown()\n\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, s.ID())\n\tbuf := routeExpect(infoRe)\n\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\n\tif !info.AuthRequired {\n\t\tt.Fatal(\"Expected to see AuthRequired\")\n\t}\n\tif info.Port != opts.Cluster.Port {\n\t\tt.Fatalf(\"Received wrong information for port, expected %d, got %d\",\n\t\t\tinfo.Port, opts.Cluster.Port)\n\t}\n\n\t// Now send it back and that should be detected as a route to self and the\n\t// connection closed.\n\trouteSend(string(buf))\n\trouteSend(\"PING\\r\\n\")\n\trc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tif _, err := rc.Read(buf); err == nil {\n\t\tt.Fatal(\"Expected route connection to be closed\")\n\t}\n\t// This should have been removed by removePassFromTrace(), but we also check debug logs here\n\tl.CheckForProhibited(t, \"route authorization password found\", \"top_secret\")\n}\n\nfunc TestSendRouteSubAndUnsub(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\n\tsend, _ := setupConn(t, c)\n\n\t// We connect to the route.\n\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer rc.Close()\n\n\texpectAuthRequired(t, rc)\n\trouteSend, routeExpect := setupRouteEx(t, rc, opts, \"ROUTER:xyz\")\n\trouteSend(\"INFO {\\\"server_id\\\":\\\"ROUTER:xyz\\\"}\\r\\n\")\n\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Send SUB via client connection\n\tsend(\"SUB foo 22\\r\\n\")\n\n\t// Make sure the RS+ is broadcast via the route\n\texpectResult(t, rc, rsubRe)\n\n\t// Send UNSUB via client connection\n\tsend(\"UNSUB 22\\r\\n\")\n\n\t// Make sure the RS- is broadcast via the route\n\texpectResult(t, rc, runsubRe)\n\n\t// Explicitly shutdown the server, otherwise this test would\n\t// cause following test to fail.\n\ts.Shutdown()\n}\n\nfunc TestSendRouteSolicit(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\t// Listen for a connection from the server on the first route.\n\tif len(opts.Routes) <= 0 {\n\t\tt.Fatalf(\"Need an outbound solicted route for this test\")\n\t}\n\trURL := opts.Routes[0]\n\n\tconn := acceptRouteConn(t, rURL.Host, server.DEFAULT_ROUTE_CONNECT)\n\tdefer conn.Close()\n\n\t// We should receive a connect message right away due to auth.\n\tbuf := expectResult(t, conn, connectRe)\n\n\t// Check INFO follows. Could be inline, with first result, if not\n\t// check follow-on buffer.\n\tif !infoRe.Match(buf) {\n\t\texpectResult(t, conn, infoRe)\n\t}\n}\n\nfunc TestRouteForwardsMsgFromClients(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tdefer client.Close()\n\n\tclientSend, clientExpect := setupConn(t, client)\n\n\troute := acceptRouteConn(t, opts.Routes[0].Host, server.DEFAULT_ROUTE_CONNECT)\n\tdefer route.Close()\n\n\trouteSend, routeExpect := setupRoute(t, route, opts)\n\texpectMsgs := expectMsgsCommand(t, routeExpect)\n\n\t// Eat the CONNECT and INFO protos\n\tbuf := routeExpect(connectRe)\n\tif !infoRe.Match(buf) {\n\t\trouteExpect(infoRe)\n\t}\n\n\t// Send SUB via route connection, RS+\n\trouteSend(\"RS+ $G foo\\r\\nPING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Send PUB via client connection\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\tmatches := expectMsgs(1)\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"\", \"2\", \"ok\")\n}\n\nfunc TestRouteForwardsMsgToClients(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tdefer client.Close()\n\n\tclientSend, clientExpect := setupConn(t, client)\n\texpectMsgs := expectMsgsCommand(t, clientExpect)\n\n\troute := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\texpectAuthRequired(t, route)\n\trouteSend, _ := setupRoute(t, route, opts)\n\n\t// Subscribe to foo\n\tclientSend(\"SUB foo 1\\r\\nPING\\r\\n\")\n\t// Use ping roundtrip to make sure its processed.\n\tclientExpect(pongRe)\n\n\t// Send RMSG proto via route connection\n\trouteSend(\"RMSG $G foo 2\\r\\nok\\r\\n\")\n\n\tmatches := expectMsgs(1)\n\tcheckMsg(t, matches[0], \"foo\", \"1\", \"\", \"2\", \"ok\")\n}\n\nfunc TestRouteOneHopSemantics(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\troute := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\n\texpectAuthRequired(t, route)\n\trouteSend, _ := setupRoute(t, route, opts)\n\n\t// Express interest on this route for foo.\n\trouteSend(\"RS+ $G foo\\r\\n\")\n\n\t// Send MSG proto via route connection\n\trouteSend(\"RMSG foo 2\\r\\nok\\r\\n\")\n\n\t// Make sure it does not come back!\n\texpectNothing(t, route)\n}\n\nfunc TestRouteOnlySendOnce(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tdefer client.Close()\n\n\tclientSend, clientExpect := setupConn(t, client)\n\n\troute := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\n\texpectAuthRequired(t, route)\n\trouteSend, routeExpect := setupRoute(t, route, opts)\n\texpectMsgs := expectMsgsCommand(t, routeExpect)\n\n\t// Express multiple interest on this route for foo.\n\trouteSend(\"RS+ $G foo\\r\\n\")\n\trouteSend(\"RS+ $G foo\\r\\n\")\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Send PUB via client connection\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\texpectMsgs(1)\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n}\n\nfunc TestRouteQueueSemantics(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tclientSend, clientExpect := setupConn(t, client)\n\tclientExpectMsgs := expectMsgsCommand(t, clientExpect)\n\n\tdefer client.Close()\n\n\troute := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\n\texpectAuthRequired(t, route)\n\trouteSend, routeExpect := setupRouteEx(t, route, opts, \"ROUTER:xyz\")\n\trouteSend(\"INFO {\\\"server_id\\\":\\\"ROUTER:xyz\\\"}\\r\\n\")\n\texpectMsgs := expectRmsgsCommand(t, routeExpect)\n\n\t// Express multiple interest on this route for foo, queue group bar.\n\trouteSend(\"RS+ $G foo bar 1\\r\\n\")\n\trouteSend(\"RS+ $G foo bar 2\\r\\n\")\n\n\t// Use ping roundtrip to make sure its processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Send PUB via client connection\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\n\")\n\t// Use ping roundtrip to make sure its processed.\n\tclientSend(\"PING\\r\\n\")\n\tclientExpect(pongRe)\n\n\t// Only 1\n\tmatches := expectMsgs(1)\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"| bar\", \"2\", \"ok\")\n\n\t// Add normal Interest as well to route interest.\n\trouteSend(\"RS+ $G foo\\r\\n\")\n\n\t// Use ping roundtrip to make sure its processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Send PUB via client connection\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\t// Use ping roundtrip to make sure its processed.\n\tclientExpect(pongRe)\n\n\t// Should be 1 now for everything. Always receive 1 message.\n\tmatches = expectMsgs(1)\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"| bar\", \"2\", \"ok\")\n\n\t// Now create a queue subscription for the client as well as a normal one.\n\tclientSend(\"SUB foo 1\\r\\n\")\n\t// Use ping roundtrip to make sure its processed.\n\tclientSend(\"PING\\r\\n\")\n\tclientExpect(pongRe)\n\trouteExpect(rsubRe)\n\n\tclientSend(\"SUB foo bar 2\\r\\nPING\\r\\n\")\n\t// Use ping roundtrip to make sure its processed.\n\tclientExpect(pongRe)\n\trouteExpect(rsubRe)\n\n\t// Deliver a MSG from the route itself, make sure the client receives both.\n\trouteSend(\"RMSG $G foo | bar 2\\r\\nok\\r\\n\")\n\n\t// Use ping roundtrip to make sure its processed.\n\trouteSend(\"PING\\r\\n\")\n\trouteExpect(pongRe)\n\n\t// Should get 2 msgs.\n\tmatches = clientExpectMsgs(2)\n\n\t// Expect first to be the normal subscriber, next will be the queue one.\n\tcheckMsg(t, matches[0], \"foo\", \"\", \"\", \"2\", \"ok\")\n}\n\nfunc TestSolicitRouteReconnect(t *testing.T) {\n\tl := testhelper.NewDummyLogger(100)\n\ts, opts := runRouteServerOverrides(t, nil,\n\t\tfunc(s *server.Server) {\n\t\t\ts.SetLogger(l, true, true)\n\t\t})\n\tdefer s.Shutdown()\n\n\trURL := opts.Routes[0]\n\n\troute := acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT)\n\n\t// Go ahead and close the Route.\n\troute.Close()\n\n\t// We expect to get called back..\n\troute = acceptRouteConn(t, rURL.Host, 2*server.DEFAULT_ROUTE_CONNECT)\n\troute.Close()\n\n\t// Now we want to check for the debug logs when it tries to reconnect\n\tl.CheckForProhibited(t, \"route authorization password found\", \":bar\")\n}\n\nfunc TestMultipleRoutesSameId(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\troute1 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route1.Close()\n\n\texpectAuthRequired(t, route1)\n\troute1Send, route1Expect := setupRouteEx(t, route1, opts, \"ROUTE:2222\")\n\n\troute2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route2.Close()\n\n\texpectAuthRequired(t, route2)\n\troute2Send, route2Expect := setupRouteEx(t, route2, opts, \"ROUTE:2222\")\n\n\t// Send SUB via route connections\n\tsub := \"RS+ $G foo\\r\\nPING\\r\\n\"\n\troute1Send(sub)\n\troute2Send(sub)\n\troute1Expect(pongRe)\n\troute2Expect(pongRe)\n\n\t// Make sure we do not get anything on a RMSG send to a router.\n\t// Send RMSG proto via route connection\n\troute1Send(\"RMSG $G foo 2\\r\\nok\\r\\n\")\n\n\texpectNothing(t, route1)\n\texpectNothing(t, route2)\n\n\t// Setup a client\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tclientSend, clientExpect := setupConn(t, client)\n\tdefer client.Close()\n\n\t// Send PUB via client connection\n\tclientSend(\"PUB foo 2\\r\\nok\\r\\nPING\\r\\n\")\n\tclientExpect(pongRe)\n\n\t// We should only receive on one route, not both.\n\t// Check both manually.\n\troute1.SetReadDeadline(time.Now().Add(100 * time.Millisecond))\n\tbuf, _ := io.ReadAll(route1)\n\troute1.SetReadDeadline(time.Time{})\n\tif len(buf) <= 0 {\n\t\troute2.SetReadDeadline(time.Now().Add(100 * time.Millisecond))\n\t\tbuf, _ = io.ReadAll(route2)\n\t\troute2.SetReadDeadline(time.Time{})\n\t\tif len(buf) <= 0 {\n\t\t\tt.Fatal(\"Expected to get one message on a route, received none.\")\n\t\t}\n\t}\n\n\tmatches := rmsgRe.FindAllSubmatch(buf, -1)\n\tif len(matches) != 1 {\n\t\tt.Fatalf(\"Expected 1 msg, got %d\\n\", len(matches))\n\t}\n\tcheckRmsg(t, matches[0], \"$G\", \"foo\", \"\", \"2\", \"ok\")\n}\n\nfunc TestRouteResendsLocalSubsOnReconnect(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tclient := createClientConn(t, opts.Host, opts.Port)\n\tdefer client.Close()\n\n\tclientSend, clientExpect := setupConn(t, client)\n\n\t// Setup a local subscription, make sure it reaches.\n\tclientSend(\"SUB foo 1\\r\\n\")\n\tclientSend(\"PING\\r\\n\")\n\tclientExpect(pongRe)\n\n\troute := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\trouteSend, routeExpect := setupRouteEx(t, route, opts, \"ROUTE:1234\")\n\n\t// Expect to see the local sub echoed through after we send our INFO.\n\ttime.Sleep(50 * time.Millisecond)\n\tbuf := routeExpect(infoRe)\n\n\t// Generate our own INFO so we can send one to trigger the local subs.\n\tinfo := server.Info{}\n\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\tt.Fatalf(\"Could not unmarshal route info: %v\", err)\n\t}\n\tinfo.ID = \"ROUTE:1234\"\n\tinfo.Name = \"\"\n\tb, err := json.Marshal(info)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not marshal test route info: %v\", err)\n\t}\n\tinfoJSON := fmt.Sprintf(\"INFO %s\\r\\n\", b)\n\n\t// Trigger the send of local subs.\n\trouteSend(infoJSON)\n\n\trouteExpect(rsubRe)\n\n\t// Close and then re-open\n\troute.Close()\n\n\t// Give some time for the route close to be processed before trying to recreate.\n\tcheckNumRoutes(t, s, 0)\n\n\troute = createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\tdefer route.Close()\n\n\trouteSend, routeExpect = setupRouteEx(t, route, opts, \"ROUTE:1234\")\n\n\trouteExpect(infoRe)\n\n\trouteSend(infoJSON)\n\trouteExpect(rsubRe)\n}\n\ntype ignoreLogger struct{}\n\nfunc (l *ignoreLogger) Fatalf(f string, args ...any) {}\nfunc (l *ignoreLogger) Errorf(f string, args ...any) {}\n\nfunc TestRouteConnectOnShutdownRace(t *testing.T) {\n\ts, opts := runRouteServer(t)\n\tdefer s.Shutdown()\n\n\tl := &ignoreLogger{}\n\n\tvar wg sync.WaitGroup\n\n\tcQuit := make(chan bool, 1)\n\n\twg.Add(1)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\troute := createRouteConn(l, opts.Cluster.Host, opts.Cluster.Port)\n\t\t\tif route != nil {\n\t\t\t\tsetupRouteEx(l, route, opts, \"ROUTE:1234\")\n\t\t\t\troute.Close()\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase <-cQuit:\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t}()\n\n\ttime.Sleep(5 * time.Millisecond)\n\ts.Shutdown()\n\n\tcQuit <- true\n\n\twg.Wait()\n}\n\nfunc TestRouteSendAsyncINFOToClients(t *testing.T) {\n\tf := func(opts *server.Options) {\n\t\ts := RunServer(opts)\n\t\tdefer s.Shutdown()\n\n\t\tclientURL := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))\n\n\t\toldClient := createClientConn(t, opts.Host, opts.Port)\n\t\tdefer oldClient.Close()\n\n\t\toldClientSend, oldClientExpect := setupConn(t, oldClient)\n\t\toldClientSend(\"PING\\r\\n\")\n\t\toldClientExpect(pongRe)\n\n\t\tnewClient := createClientConn(t, opts.Host, opts.Port)\n\t\tdefer newClient.Close()\n\n\t\tnewClientSend, newClientExpect := setupConnWithProto(t, newClient, clientProtoInfo)\n\t\tnewClientSend(\"PING\\r\\n\")\n\n\t\t// For new clients, the initial PING (after a CONNECT) will result in a\n\t\t// PONG followed by an INFO, so make sure we receive them separately.\n\t\tgetPongAlone := func(c net.Conn) {\n\t\t\tt.Helper()\n\t\t\tpongBuf := make([]byte, len(\"PONG\\r\\n\"))\n\t\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\t\tn, err := c.Read(pongBuf)\n\t\t\tc.SetReadDeadline(time.Time{})\n\t\t\tif n <= 0 && err != nil {\n\t\t\t\tt.Fatalf(\"Error reading from conn: %v\\n\", err)\n\t\t\t}\n\t\t\tif !pongRe.Match(pongBuf) {\n\t\t\t\tt.Fatalf(\"Response did not match expected: \\n\\tReceived:'%q'\\n\\tExpected:'%s'\\n\", pongBuf, pongRe)\n\t\t\t}\n\t\t}\n\t\tgetPongAlone(newClient)\n\t\t// The new client should receive an INFO with the ConnectInfo boolean set to true.\n\t\tnewClientExpect(infoRe)\n\n\t\trouteID := \"Server-B\"\n\n\t\tcreateRoute := func() (net.Conn, sendFun, expectFun) {\n\t\t\trc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port)\n\t\t\trouteSend, routeExpect := setupRouteEx(t, rc, opts, routeID)\n\n\t\t\tbuf := routeExpect(infoRe)\n\n\t\t\tinfo := server.Info{}\n\t\t\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\t\t\tstackFatalf(t, \"Could not unmarshal route info: %v\", err)\n\t\t\t}\n\t\t\tif opts.Cluster.NoAdvertise {\n\t\t\t\tif len(info.ClientConnectURLs) != 0 {\n\t\t\t\t\tstackFatalf(t, \"Expected ClientConnectURLs to be empty, got %v\", info.ClientConnectURLs)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif len(info.ClientConnectURLs) == 0 {\n\t\t\t\t\tstackFatalf(t, \"Expected a list of URLs, got none\")\n\t\t\t\t}\n\t\t\t\tif info.ClientConnectURLs[0] != clientURL {\n\t\t\t\t\tstackFatalf(t, \"Expected ClientConnectURLs to be %q, got %q\", clientURL, info.ClientConnectURLs[0])\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn rc, routeSend, routeExpect\n\t\t}\n\n\t\tsendRouteINFO := func(routeSend sendFun, routeExpect expectFun, urls []string) {\n\t\t\trouteInfo := server.Info{}\n\t\t\trouteInfo.ID = routeID\n\t\t\trouteInfo.Cluster = \"xyz\"\n\t\t\trouteInfo.Host = \"127.0.0.1\"\n\t\t\trouteInfo.Port = 5222\n\t\t\trouteInfo.ClientConnectURLs = urls\n\t\t\tb, err := json.Marshal(routeInfo)\n\t\t\tif err != nil {\n\t\t\t\tstackFatalf(t, \"Could not marshal test route info: %v\", err)\n\t\t\t}\n\t\t\tinfoJSON := fmt.Sprintf(\"INFO %s\\r\\n\", b)\n\t\t\trouteSend(infoJSON)\n\t\t\trouteSend(\"PING\\r\\n\")\n\t\t\trouteExpect(pongRe)\n\t\t}\n\n\t\tcheckClientConnectURLS := func(urls, expected []string) {\n\t\t\t// Order of array is not guaranteed.\n\t\t\tok := false\n\t\t\tif len(urls) == len(expected) {\n\t\t\t\tm := make(map[string]struct{}, len(expected))\n\t\t\t\tfor _, url := range expected {\n\t\t\t\t\tm[url] = struct{}{}\n\t\t\t\t}\n\t\t\t\tok = true\n\t\t\t\tfor _, url := range urls {\n\t\t\t\t\tif _, present := m[url]; !present {\n\t\t\t\t\t\tok = false\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\tstackFatalf(t, \"Expected ClientConnectURLs to be %v, got %v\", expected, urls)\n\t\t\t}\n\t\t}\n\n\t\tvar checkConnectInfo bool\n\t\tcheckINFOReceived := func(client net.Conn, clientExpect expectFun, expectedURLs []string) {\n\t\t\tif opts.Cluster.NoAdvertise {\n\t\t\t\tif !checkConnectInfo {\n\t\t\t\t\texpectNothing(t, client)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\t// Since it is no advertise, but we still get an INFO, the URLs\n\t\t\t\t// should be empty.\n\t\t\t\texpectedURLs = nil\n\t\t\t}\n\t\t\tbuf := clientExpect(infoRe)\n\t\t\tinfo := server.Info{}\n\t\t\tif err := json.Unmarshal(buf[4:], &info); err != nil {\n\t\t\t\tstackFatalf(t, \"Could not unmarshal route info: %v\", err)\n\t\t\t}\n\t\t\tcheckClientConnectURLS(info.ClientConnectURLs, expectedURLs)\n\t\t\tif checkConnectInfo {\n\t\t\t\tif !info.ConnectInfo {\n\t\t\t\t\tstackFatalf(t, \"ConnectInfo should have been true\")\n\t\t\t\t}\n\t\t\t\tif info.RemoteAccount != \"$G\" {\n\t\t\t\t\tstackFatalf(t, \"RemoteAccount should be %q, got %q\", \"$G\", info.RemoteAccount)\n\t\t\t\t}\n\t\t\t\tif info.IsSystemAccount {\n\t\t\t\t\tstackFatalf(t, \"IsSystemAccount should have been false\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Create a route\n\t\trc, routeSend, routeExpect := createRoute()\n\t\tdefer rc.Close()\n\n\t\t// Send an INFO with single URL\n\t\trouteClientConnectURLs := []string{\"127.0.0.1:5222\"}\n\t\tsendRouteINFO(routeSend, routeExpect, routeClientConnectURLs)\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// We expect to get the one from the server we connect to and the other route.\n\t\texpectedURLs := []string{clientURL, routeClientConnectURLs[0]}\n\n\t\t// Expect new client to receive an INFO (unless disabled)\n\t\tcheckINFOReceived(newClient, newClientExpect, expectedURLs)\n\n\t\t// Disconnect the route\n\t\trc.Close()\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// Expect new client to receive an INFO (unless disabled).\n\t\t// The content will now have the disconnected route ClientConnectURLs\n\t\t// removed from the INFO. So it should be the one from the server the\n\t\t// client is connected to.\n\t\tcheckINFOReceived(newClient, newClientExpect, []string{clientURL})\n\n\t\t// Reconnect the route.\n\t\trc, routeSend, routeExpect = createRoute()\n\t\tdefer rc.Close()\n\n\t\t// Resend the same route INFO json. The server will now send\n\t\t// the INFO since the disconnected route ClientConnectURLs was\n\t\t// removed in previous step.\n\t\tsendRouteINFO(routeSend, routeExpect, routeClientConnectURLs)\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// Expect new client to receive an INFO (unless disabled)\n\t\tcheckINFOReceived(newClient, newClientExpect, expectedURLs)\n\n\t\t// Now stop the route and restart with an additional URL\n\t\trc.Close()\n\n\t\t// On route disconnect, clients will receive an updated INFO\n\t\texpectNothing(t, oldClient)\n\t\tcheckINFOReceived(newClient, newClientExpect, []string{clientURL})\n\n\t\trc, routeSend, routeExpect = createRoute()\n\t\tdefer rc.Close()\n\n\t\t// Create a client not sending the CONNECT until after route is added\n\t\tclientNoConnect := createClientConn(t, opts.Host, opts.Port)\n\t\tdefer clientNoConnect.Close()\n\n\t\t// Create a client that does not send the first PING yet\n\t\tclientNoPing := createClientConn(t, opts.Host, opts.Port)\n\t\tdefer clientNoPing.Close()\n\t\tclientNoPingSend, clientNoPingExpect := setupConnWithProto(t, clientNoPing, clientProtoInfo)\n\n\t\t// The route now has an additional URL\n\t\trouteClientConnectURLs = append(routeClientConnectURLs, \"127.0.0.1:7777\")\n\t\texpectedURLs = append(expectedURLs, \"127.0.0.1:7777\")\n\t\t// This causes the server to add the route and send INFO to clients\n\t\tsendRouteINFO(routeSend, routeExpect, routeClientConnectURLs)\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// Expect new client to receive an INFO, and verify content as expected.\n\t\tcheckINFOReceived(newClient, newClientExpect, expectedURLs)\n\n\t\t// Expect nothing yet for client that did not send the PING\n\t\texpectNothing(t, clientNoPing)\n\n\t\t// Now send the first PING\n\t\tclientNoPingSend(\"PING\\r\\n\")\n\t\tgetPongAlone(clientNoPing)\n\t\t// We should receive an INFO but with ConnectInfo==true, etc..\n\t\tcheckConnectInfo = true\n\t\tcheckINFOReceived(clientNoPing, clientNoPingExpect, expectedURLs)\n\t\tcheckConnectInfo = false\n\n\t\t// Have the client that did not send the connect do it now\n\t\tclientNoConnectSend, clientNoConnectExpect := setupConnWithProto(t, clientNoConnect, clientProtoInfo)\n\t\t// Send the PING\n\t\tclientNoConnectSend(\"PING\\r\\n\")\n\t\tgetPongAlone(clientNoConnect)\n\t\tcheckConnectInfo = true\n\t\tcheckINFOReceived(clientNoConnect, clientNoConnectExpect, expectedURLs)\n\t\tcheckConnectInfo = false\n\n\t\t// Create a client connection and verify content of initial INFO contains array\n\t\t// (but empty if no advertise option is set)\n\t\tcli := createClientConn(t, opts.Host, opts.Port)\n\t\tdefer cli.Close()\n\t\tbuf := expectResult(t, cli, infoRe)\n\t\tjs := infoRe.FindAllSubmatch(buf, 1)[0][1]\n\t\tvar sinfo server.Info\n\t\terr := json.Unmarshal(js, &sinfo)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Could not unmarshal INFO json: %v\\n\", err)\n\t\t}\n\t\tif opts.Cluster.NoAdvertise {\n\t\t\tif len(sinfo.ClientConnectURLs) != 0 {\n\t\t\t\tt.Fatalf(\"Expected ClientConnectURLs to be empty, got %v\", sinfo.ClientConnectURLs)\n\t\t\t}\n\t\t} else {\n\t\t\tcheckClientConnectURLS(sinfo.ClientConnectURLs, expectedURLs)\n\t\t}\n\n\t\t// Add a new route\n\t\trouteID = \"Server-C\"\n\t\trc2, route2Send, route2Expect := createRoute()\n\t\tdefer rc2.Close()\n\n\t\t// Send an INFO with single URL\n\t\trc2ConnectURLs := []string{\"127.0.0.1:8888\"}\n\t\tsendRouteINFO(route2Send, route2Expect, rc2ConnectURLs)\n\n\t\t// This is the combined client connect URLs array\n\t\ttotalConnectURLs := expectedURLs\n\t\ttotalConnectURLs = append(totalConnectURLs, rc2ConnectURLs...)\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// Expect new client to receive an INFO (unless disabled)\n\t\tcheckINFOReceived(newClient, newClientExpect, totalConnectURLs)\n\n\t\t// Make first route disconnect\n\t\trc.Close()\n\n\t\t// Expect nothing for old clients\n\t\texpectNothing(t, oldClient)\n\n\t\t// Expect new client to receive an INFO (unless disabled)\n\t\t// The content should be the server client is connected to and the last route\n\t\tcheckINFOReceived(newClient, newClientExpect, []string{\"127.0.0.1:5242\", \"127.0.0.1:8888\"})\n\t}\n\n\topts := LoadConfig(\"./configs/cluster.conf\")\n\t// For this test, be explicit about listen spec.\n\topts.Host = \"127.0.0.1\"\n\topts.Port = 5242\n\topts.DisableShortFirstPing = true\n\n\tf(opts)\n\topts.Cluster.NoAdvertise = true\n\tf(opts)\n}\n\nfunc TestRouteBasicPermissions(t *testing.T) {\n\tsrvA, optsA := RunServerWithConfig(\"./configs/srv_a_perms.conf\")\n\tdefer srvA.Shutdown()\n\n\tsrvB, optsB := RunServerWithConfig(\"./configs/srv_b.conf\")\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\t// Create a connection to server B\n\tncb, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncb.Close()\n\tch := make(chan bool, 1)\n\tcb := func(_ *nats.Msg) {\n\t\tch <- true\n\t}\n\t// Subscribe on Server B on \"bar\" and \"baz\", which should be accepted by server A across the route\n\t// Due to allowing \"*\"\n\tsubBbar, err := ncb.Subscribe(\"bar\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subBbar.Unsubscribe()\n\tsubBbaz, err := ncb.Subscribe(\"baz\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subBbaz.Unsubscribe()\n\tncb.Flush()\n\tif err := checkExpectedSubs(2, srvA, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Create a connection to server A\n\tnca, err := nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nca.Close()\n\t// Publish on bar and baz, messages should be received.\n\tif err := nca.Publish(\"bar\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tif err := nca.Publish(\"baz\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Did not get the messages\")\n\t\t}\n\t}\n\n\t// From B, start a subscription on \"foo\", which server A should drop since\n\t// it only exports on \"bar\" and \"baz\"\n\tsubBfoo, err := ncb.Subscribe(\"foo\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subBfoo.Unsubscribe()\n\tncb.Flush()\n\t// B should have now 3 subs\n\tif err := checkExpectedSubs(3, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// and A still 2.\n\tif err := checkExpectedSubs(2, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// So producing on \"foo\" from A should not be forwarded to B.\n\tif err := nca.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tselect {\n\tcase <-ch:\n\t\tt.Fatal(\"Message should not have been received\")\n\tcase <-time.After(100 * time.Millisecond):\n\t}\n\n\t// Now on A, create a subscription on something that A does not import,\n\t// like \"bat\".\n\tsubAbat, err := nca.Subscribe(\"bat\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subAbat.Unsubscribe()\n\tnca.Flush()\n\t// A should have 3 subs\n\tif err := checkExpectedSubs(3, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// And from B, send a message on that subject and make sure it is not received.\n\tif err := ncb.Publish(\"bat\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tselect {\n\tcase <-ch:\n\t\tt.Fatal(\"Message should not have been received\")\n\tcase <-time.After(100 * time.Millisecond):\n\t}\n\n\t// Stop subscription on foo from B\n\tsubBfoo.Unsubscribe()\n\tncb.Flush()\n\t// Back to 2 subs on B\n\tif err := checkExpectedSubs(2, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Create subscription on foo from A, this should be forwared to B.\n\tsubAfoo, err := nca.Subscribe(\"foo\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subAfoo.Unsubscribe()\n\t// Create another one so that test the import permissions cache\n\tsubAfoo2, err := nca.Subscribe(\"foo\", cb)\n\tif err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tdefer subAfoo2.Unsubscribe()\n\tnca.Flush()\n\t// A should have 5 subs\n\tif err := checkExpectedSubs(5, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// B should have 3 since we coalesce te two for 'foo'\n\tif err := checkExpectedSubs(3, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// Send a message from B and check that it is received.\n\tif err := ncb.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Did not get the message\")\n\t\t}\n\t}\n\n\t// Close connection from B, and restart server B too.\n\t// We want to make sure that\n\tncb.Close()\n\tsrvB.Shutdown()\n\n\t// Since B had 2 local subs, A should still only go from 4 to 3\n\tif err := checkExpectedSubs(3, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Restart server B\n\tsrvB, optsB = RunServerWithConfig(\"./configs/srv_b.conf\")\n\tdefer srvB.Shutdown()\n\t// Check that subs from A that can be sent to B are sent.\n\t// That would be 2 (the 2 subscriptions on foo) as one.\n\tif err := checkExpectedSubs(1, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Connect to B and send on \"foo\" and make sure we receive\n\tncb, err = nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncb.Close()\n\tif err := ncb.Publish(\"foo\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tselect {\n\t\tcase <-ch:\n\t\tcase <-time.After(time.Second):\n\t\t\tt.Fatal(\"Did not get the message\")\n\t\t}\n\t}\n\n\t// Send on \"bat\" and make sure that this is not received.\n\tif err := ncb.Publish(\"bat\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tselect {\n\tcase <-ch:\n\t\tt.Fatal(\"Message should not have been received\")\n\tcase <-time.After(100 * time.Millisecond):\n\t}\n\n\tnca.Close()\n\tncb.Close()\n\n\tsrvA.Shutdown()\n\tsrvB.Shutdown()\n\n\toptsA.Cluster.Permissions.Export = nil\n\tsrvA = RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB = RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tnca, err = nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsA.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nca.Close()\n\t// Subscribe on \"bar\" which is not imported\n\tif _, err := nca.Subscribe(\"bar\", cb); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif err := checkExpectedSubs(1, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Publish from B, should not be received\n\tncb, err = nats.Connect(fmt.Sprintf(\"nats://127.0.0.1:%d\", optsB.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncb.Close()\n\tif err := ncb.Publish(\"bar\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tselect {\n\tcase <-ch:\n\t\tt.Fatal(\"Message should not have been received\")\n\tcase <-time.After(100 * time.Millisecond):\n\t\t//ok\n\t}\n\t// Subscribe on \"baz\" on B\n\tif _, err := ncb.Subscribe(\"baz\", cb); err != nil {\n\t\tt.Fatalf(\"Error on subscribe: %v\", err)\n\t}\n\tif err := checkExpectedSubs(1, srvB); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\tif err := checkExpectedSubs(2, srvA); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\t// Publish from A, since there is no export restriction, message should be received.\n\tif err := nca.Publish(\"baz\", []byte(\"hello\")); err != nil {\n\t\tt.Fatalf(\"Error on publish: %v\", err)\n\t}\n\tselect {\n\tcase <-ch:\n\t// ok\n\tcase <-time.After(250 * time.Millisecond):\n\t\tt.Fatal(\"Message should have been received\")\n\t}\n}\n\nfunc createConfFile(t testing.TB, content []byte) string {\n\tt.Helper()\n\tconf := createTempFile(t, \"\")\n\tfName := conf.Name()\n\tconf.Close()\n\tif err := os.WriteFile(fName, content, 0666); err != nil {\n\t\tt.Fatalf(\"Error writing conf file: %v\", err)\n\t}\n\treturn fName\n}\n\nfunc TestRoutesOnlyImportOrExport(t *testing.T) {\n\tcontents := []string{\n\t\t`import: \"foo\"`,\n\t\t`import: {\n\t\t\tallow: \"foo\"\n\t\t}`,\n\t\t`import: {\n\t\t\tdeny: \"foo\"\n\t\t}`,\n\t\t`import: {\n\t\t\tallow: \"foo\"\n\t\t\tdeny: \"foo\"\n\t\t}`,\n\t\t`export: \"foo\"`,\n\t\t`export: {\n\t\t\tallow: \"foo\"\n\t\t}`,\n\t\t`export: {\n\t\t\tdeny: \"foo\"\n\t\t}`,\n\t\t`export: {\n\t\t\tallow: \"foo\"\n\t\t\tdeny: \"foo\"\n\t\t}`,\n\t}\n\tf := func(c string) {\n\t\tcf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\tport: -1\n\t\t\tcluster {\n\t\t\t\tname: \"Z\"\n\t\t\t\tport: -1\n\t\t\t\tauthorization {\n\t\t\t\t\tuser: ivan\n\t\t\t\t\tpassword: pwd\n\t\t\t\t\tpermissions {\n\t\t\t\t\t\t%s\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t`, c)))\n\t\ts, _ := RunServerWithConfig(cf)\n\t\ts.Shutdown()\n\t}\n\tfor _, c := range contents {\n\t\tf(c)\n\t}\n}\n" }, { "path": "test/service_latency_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/jwt/v2\"\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n\t\"github.com/nats-io/nkeys\"\n)\n\n// Used to setup superclusters for tests.\ntype supercluster struct {\n\tt *testing.T\n\tclusters []*cluster\n}\n\nfunc (sc *supercluster) shutdown() {\n\tif sc == nil {\n\t\treturn\n\t}\n\tfor _, c := range sc.clusters {\n\t\tshutdownCluster(c)\n\t}\n}\n\nconst digits = \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\nconst base = 36\nconst cnlen = 8\n\nfunc randClusterName() string {\n\tvar name []byte\n\trn := rand.Int63()\n\tfor i := 0; i < cnlen; i++ {\n\t\tname = append(name, digits[rn%base])\n\t\trn /= base\n\t}\n\treturn string(name[:cnlen])\n}\n\nfunc createSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster {\n\tclusters := []*cluster{}\n\n\tfor i := 0; i < numClusters; i++ {\n\t\t// Pick cluster name and setup default accounts.\n\t\tc := createClusterEx(t, true, 5*time.Millisecond, true, randClusterName(), numServersPer, clusters...)\n\t\tclusters = append(clusters, c)\n\t}\n\treturn &supercluster{t, clusters}\n}\n\nfunc (sc *supercluster) setResponseThreshold(t *testing.T, maxTime time.Duration) {\n\tt.Helper()\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\tfoo, err := s.LookupAccount(\"FOO\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'FOO': %v\", err)\n\t\t\t}\n\t\t\tif err := foo.SetServiceExportResponseThreshold(\"ngs.usage.*\", maxTime); err != nil {\n\t\t\t\tt.Fatalf(\"Error setting response threshold\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (sc *supercluster) setImportShare(t *testing.T) {\n\tt.Helper()\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\tfoo, err := s.LookupAccount(\"FOO\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'FOO': %v\", err)\n\t\t\t}\n\t\t\tbar, err := s.LookupAccount(\"BAR\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'BAR': %v\", err)\n\t\t\t}\n\t\t\tif err := bar.SetServiceImportSharing(foo, \"ngs.usage.bar\", true); err != nil {\n\t\t\t\tt.Fatalf(\"Error setting import sharing: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (sc *supercluster) setupLatencyTracking(t *testing.T, p int) {\n\tt.Helper()\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\tfoo, err := s.LookupAccount(\"FOO\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'FOO': %v\", err)\n\t\t\t}\n\t\t\tbar, err := s.LookupAccount(\"BAR\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'BAR': %v\", err)\n\t\t\t}\n\t\t\tif err := foo.AddServiceExport(\"ngs.usage.*\", nil); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service export to 'FOO': %v\", err)\n\t\t\t}\n\t\t\tif err := foo.TrackServiceExportWithSampling(\"ngs.usage.*\", \"results\", p); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding latency tracking to 'FOO': %v\", err)\n\t\t\t}\n\t\t\tif err := bar.AddServiceImport(foo, \"ngs.usage\", \"ngs.usage.bar\"); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding service import to 'ngs.usage': %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (sc *supercluster) removeLatencyTracking(t *testing.T) {\n\tt.Helper()\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\tfoo, err := s.LookupAccount(\"FOO\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account 'FOO': %v\", err)\n\t\t\t}\n\t\t\tfoo.UnTrackServiceExport(\"ngs.usage.*\")\n\t\t}\n\t}\n}\n\nfunc (sc *supercluster) totalSubs() int {\n\ttotalSubs := 0\n\tfor _, c := range sc.clusters {\n\t\ttotalSubs += c.totalSubs()\n\t}\n\treturn totalSubs\n}\n\nfunc clientConnectWithName(t *testing.T, opts *server.Options, user, appname string) *nats.Conn {\n\tt.Helper()\n\turl := fmt.Sprintf(\"nats://%s:pass@%s:%d\", user, opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, nats.Name(appname))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc clientConnect(t *testing.T, opts *server.Options, user string) *nats.Conn {\n\tt.Helper()\n\treturn clientConnectWithName(t, opts, user, \"\")\n}\n\nfunc clientConnectOldRequest(t *testing.T, opts *server.Options, user string) *nats.Conn {\n\tt.Helper()\n\turl := fmt.Sprintf(\"nats://%s:pass@%s:%d\", user, opts.Host, opts.Port)\n\tnc, err := nats.Connect(url, nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\treturn nc\n}\n\nfunc checkServiceLatency(t *testing.T, sl server.ServiceLatency, start time.Time, serviceTime time.Duration) {\n\tt.Helper()\n\n\tif sl.Status != 200 {\n\t\tt.Fatalf(\"Bad status received, wanted 200 got %d\", sl.Status)\n\t}\n\n\tserviceTime = serviceTime.Round(time.Millisecond)\n\n\tstartDelta := sl.RequestStart.Sub(start)\n\t// Original test was 5ms, but got GitHub Action failure with \"Bad start delta 5.033929ms\",\n\t// and Travis will get something like: \"Bad start delta 15.046059ms\", so be more generous.\n\tif startDelta > 20*time.Millisecond {\n\t\tt.Fatalf(\"Bad start delta %v\", startDelta)\n\t}\n\t// Since RTT during tests is estimate we remove from calculation.\n\tif (sl.ServiceLatency + sl.Responder.RTT) < time.Duration(float64(serviceTime)*0.8) {\n\t\tt.Fatalf(\"Bad service latency: %v (%v)\", sl.ServiceLatency, serviceTime)\n\t}\n\tif sl.TotalLatency < sl.ServiceLatency {\n\t\tt.Fatalf(\"Bad total latency: %v (%v)\", sl.TotalLatency, sl.ServiceLatency)\n\t}\n\t// We should have NATS latency here that is non-zero with real clients.\n\tif sl.Requestor.RTT == 0 {\n\t\tt.Fatalf(\"Expected non-zero NATS Requestor latency\")\n\t}\n\tif sl.Responder.RTT == 0 {\n\t\tt.Fatalf(\"Expected non-zero NATS Requestor latency\")\n\t}\n\n\t// Make sure they add up\n\tgot := sl.TotalLatency\n\texpected := sl.ServiceLatency + sl.NATSTotalTime()\n\tif got != expected {\n\t\tt.Fatalf(\"Numbers do not add up: %+v,\\ngot: %v\\nexpected: %v\", sl, got, expected)\n\t}\n}\n\nfunc extendedCheck(t *testing.T, lc *server.ClientInfo, eUser, appName, eServer string) {\n\tt.Helper()\n\tif lc.User != eUser {\n\t\tt.Fatalf(\"Expected user of %q, got %q\", eUser, lc.User)\n\t}\n\tif appName != \"\" && appName != lc.Name {\n\t\tt.Fatalf(\"Expected appname of %q, got %q\\n\", appName, lc.Name)\n\t}\n\tif lc.Host == \"\" {\n\t\tt.Fatalf(\"Expected non-empty IP\")\n\t}\n\tif lc.ID < 1 || lc.ID > 20 {\n\t\tt.Fatalf(\"Expected a ID in range, got %d\", lc.ID)\n\t}\n\tif eServer != \"\" && eServer != lc.Server {\n\t\tt.Fatalf(\"Expected server of %q, got %q\", eServer, lc.Server)\n\t}\n}\n\nfunc noShareCheck(t *testing.T, lc *server.ClientInfo) {\n\tt.Helper()\n\tif lc.Name != \"\" {\n\t\tt.Fatalf(\"appname should not have been shared, got %q\", lc.Name)\n\t}\n\tif lc.User != \"\" {\n\t\tt.Fatalf(\"user should not have been shared, got %q\", lc.User)\n\t}\n\tif lc.Host != \"\" {\n\t\tt.Fatalf(\"client ip should not have been shared, got %q\", lc.Host)\n\t}\n\tif lc.ID != 0 {\n\t\tt.Fatalf(\"client id should not have been shared, got %d\", lc.ID)\n\t}\n\tif lc.Server != \"\" {\n\t\tt.Fatalf(\"client' server should not have been shared, got %q\", lc.Server)\n\t}\n}\n\nfunc TestServiceLatencySingleServerConnect(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\t// Now we can setup and test, do single node only first.\n\t// This is the service provider.\n\tnc := clientConnectWithName(t, sc.clusters[0].opts[0], \"foo\", \"service22\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tserviceTime := 25 * time.Millisecond\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\ttime.Sleep(serviceTime)\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\n\t// Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[0], \"bar\")\n\tdefer nc2.Close()\n\n\t// Send the request.\n\tstart := time.Now()\n\tif _, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl server.ServiceLatency\n\trmsg, _ := rsub.NextMsg(time.Second)\n\tjson.Unmarshal(rmsg.Data, &sl)\n\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\n\trs := sc.clusters[0].servers[0]\n\textendedCheck(t, sl.Responder, \"foo\", \"service22\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n\n\t// Now make sure normal use case works with old request style.\n\tnc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], \"bar\")\n\tdefer nc3.Close()\n\n\tstart = time.Now()\n\tif _, err := nc3.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\tnc3.Close()\n\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\textendedCheck(t, sl.Responder, \"foo\", \"service22\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n}\n\n// If a client has a longer RTT that the effective RTT for NATS + responder\n// the requestor RTT will be marked as 0. This can happen quite often with\n// utility programs that are far away from a cluster like NGS but the service\n// response time has a shorter RTT.\nfunc TestServiceLatencyClientRTTSlowerVsServiceRTT(t *testing.T) {\n\tsc := createSuperCluster(t, 2, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have BAR import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// The service listener. Mostly instant response.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\ttime.Sleep(time.Millisecond)\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\tnc.Flush()\n\n\t// Requestor and processing\n\trequestAndCheck := func(cindex, sindex int) {\n\t\tt.Helper()\n\t\tsopts := sc.clusters[cindex].opts[sindex]\n\n\t\tif nmsgs, _, err := rsub.Pending(); err != nil || nmsgs != 0 {\n\t\t\tt.Fatalf(\"Did not expect any latency results, got %d\", nmsgs)\n\t\t}\n\n\t\trtt := 10 * time.Millisecond\n\t\tbw := 1024 * 1024\n\t\tsp := newSlowProxy(rtt+5*time.Millisecond, bw, bw, sopts)\n\t\tdefer sp.stop()\n\n\t\tnc2 := clientConnect(t, sp.opts(), \"bar\")\n\t\tdefer nc2.Close()\n\n\t\tstart := time.Now()\n\t\tnc2.Flush()\n\t\t// Check rtt for slow proxy\n\t\tif d := time.Since(start); d < rtt {\n\t\t\tt.Fatalf(\"Expected an rtt of at least %v, got %v\", rtt, d)\n\t\t}\n\n\t\t// Send the request.\n\t\t_, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected a response\")\n\t\t}\n\n\t\tvar sl server.ServiceLatency\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil || rmsg == nil {\n\t\t\tt.Fatalf(\"Did not receive latency results\")\n\t\t}\n\t\tjson.Unmarshal(rmsg.Data, &sl)\n\n\t\tif sl.Status != 200 {\n\t\t\tt.Fatalf(\"Expected a status 200 Ok, got [%d] %q\", sl.Status, sl.Error)\n\t\t}\n\t\t// We want to test here that when the client requestor RTT is larger then the response time\n\t\t// we still deliver a requestor value > 0.\n\t\t// Now check that it is close to rtt.\n\t\tif sl.Requestor.RTT < rtt {\n\t\t\tt.Fatalf(\"Expected requestor latency to be > %v, got %v\", rtt, sl.Requestor.RTT)\n\t\t}\n\t\tif sl.TotalLatency < rtt {\n\t\t\tt.Fatalf(\"Expected total latency to be > %v, got %v\", rtt, sl.TotalLatency)\n\t\t}\n\n\t\trs := sc.clusters[0].servers[0]\n\t\textendedCheck(t, sl.Responder, \"foo\", \"\", rs.Name())\n\t\t// Normally requestor's don't share\n\t\tnoShareCheck(t, sl.Requestor)\n\n\t\t// Check for trailing duplicates..\n\t\trmsg, err = rsub.NextMsg(100 * time.Millisecond)\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"Duplicate metric result, %q\", rmsg.Data)\n\t\t}\n\t}\n\n\t// Check same server.\n\trequestAndCheck(0, 0)\n\t// Check from remote server across GW.\n\trequestAndCheck(1, 1)\n\t// Same cluster but different server\n\trequestAndCheck(0, 1)\n}\n\nfunc connRTT(nc *nats.Conn) time.Duration {\n\t// Do 5x to flatten\n\ttotal := time.Duration(0)\n\tfor i := 0; i < 5; i++ {\n\t\tstart := time.Now()\n\t\tnc.Flush()\n\t\ttotal += time.Since(start)\n\t}\n\treturn total / 5\n}\n\nfunc TestServiceLatencyRemoteConnect(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\t// Now we can setup and test, do single node only first.\n\t// This is the service provider.\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tsubsBefore := int(sc.clusters[0].servers[0].NumSubscriptions())\n\n\t// The service listener.\n\tserviceTime := 25 * time.Millisecond\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\ttime.Sleep(serviceTime)\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\tnc.Flush()\n\n\tif err := checkExpectedSubs(subsBefore+2, sc.clusters[0].servers...); err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n\n\t// Same Cluster Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[2], \"bar\")\n\tdefer nc2.Close()\n\n\t// Send the request.\n\tstart := time.Now()\n\t_, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl server.ServiceLatency\n\trmsg, err := rsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting latency measurement: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &sl)\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\trs := sc.clusters[0].servers[0]\n\textendedCheck(t, sl.Responder, \"foo\", \"\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n\n\t// Lastly here, we need to make sure we are properly tracking the extra hops.\n\t// We will make sure that NATS latency is close to what we see from the outside in terms of RTT.\n\tif crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt {\n\t\tt.Fatalf(\"Not tracking second measurement for NATS latency across servers: %v vs %v\", sl.NATSTotalTime(), crtt)\n\t}\n\n\t// Gateway Requestor\n\tnc2 = clientConnect(t, sc.clusters[1].opts[1], \"bar\")\n\tdefer nc2.Close()\n\n\t// Send the request.\n\tstart = time.Now()\n\t_, err = nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\trmsg, err = rsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting latency measurement: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &sl)\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\textendedCheck(t, sl.Responder, \"foo\", \"\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n\n\t// Lastly here, we need to make sure we are properly tracking the extra hops.\n\t// We will make sure that NATS latency is close to what we see from the outside in terms of RTT.\n\tif crtt := connRTT(nc) + connRTT(nc2); sl.NATSTotalTime() < crtt {\n\t\tt.Fatalf(\"Not tracking second measurement for NATS latency across servers: %v vs %v\", sl.NATSTotalTime(), crtt)\n\t}\n\n\t// Now turn off and make sure we no longer receive updates.\n\tsc.removeLatencyTracking(t)\n\t_, err = nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\t_, err = rsub.NextMsg(100 * time.Millisecond)\n\tif err == nil {\n\t\tt.Fatalf(\"Did not expect to receive a latency metric\")\n\t}\n}\n\nfunc TestServiceLatencySampling(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 50)\n\n\t// Now we can setup and test, do single node only first.\n\t// This is the service provider.\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\treceived := int32(0)\n\n\tnc.Subscribe(\"results\", func(msg *nats.Msg) {\n\t\tatomic.AddInt32(&received, 1)\n\t})\n\n\t// Same Cluster Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[2], \"bar\")\n\tdefer nc2.Close()\n\n\ttoSend := 1000\n\tfor i := 0; i < toSend; i++ {\n\t\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\t}\n\t// Wait for results to flow in.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tmid := toSend / 2\n\tdelta := toSend / 10 // 10%\n\tgot := int(atomic.LoadInt32(&received))\n\n\tif got > mid+delta || got < mid-delta {\n\t\tt.Fatalf(\"Sampling number incorrect: %d vs %d\", mid, got)\n\t}\n}\n\nfunc TestServiceLatencyNoSubsLeak(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\tnc := clientConnectWithName(t, sc.clusters[0].opts[1], \"foo\", \"dlc22\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\tnc.Flush()\n\t// Propagation of sub through super cluster.\n\ttime.Sleep(100 * time.Millisecond)\n\n\tstartSubs := sc.totalSubs()\n\n\tfooAcc, _ := sc.clusters[1].servers[1].LookupAccount(\"FOO\")\n\tstartNumSis := fooAcc.NumServiceImports()\n\n\tfor i := 0; i < 100; i++ {\n\t\tnc := clientConnect(t, sc.clusters[1].opts[1], \"bar\")\n\t\tdefer nc.Close()\n\t\tif _, err := nc.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Error on request: %v\", err)\n\t\t}\n\t\tnc.Close()\n\t}\n\n\t// We are adding 3 here for the wildcard response subject for service replies.\n\t// we only have one but it will show in three places.\n\tstartSubs += 3\n\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif numSubs := sc.totalSubs(); numSubs != startSubs {\n\t\t\treturn fmt.Errorf(\"Leaked %d subs\", numSubs-startSubs)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now also check to make sure the service imports created for the request go away as well.\n\tcheckFor(t, time.Second, 50*time.Millisecond, func() error {\n\t\tif numSis := fooAcc.NumServiceImports(); numSis != startNumSis {\n\t\t\treturn fmt.Errorf(\"Leaked %d service imports\", numSis-startNumSis)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestServiceLatencyWithName(t *testing.T) {\n\tsc := createSuperCluster(t, 1, 1)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\topts := sc.clusters[0].opts[0]\n\n\tnc := clientConnectWithName(t, opts, \"foo\", \"dlc22\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\tnc.Flush()\n\n\tnc2 := clientConnect(t, opts, \"bar\")\n\tdefer nc2.Close()\n\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\n\tvar sl server.ServiceLatency\n\trmsg, err := rsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error getting message: %v\", err)\n\t}\n\tjson.Unmarshal(rmsg.Data, &sl)\n\n\t// Make sure we have AppName set.\n\trs := sc.clusters[0].servers[0]\n\textendedCheck(t, sl.Responder, \"foo\", \"dlc22\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n}\n\nfunc TestServiceLatencyWithNameMultiServer(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\tnc := clientConnectWithName(t, sc.clusters[0].opts[1], \"foo\", \"dlc22\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\tnc.Flush()\n\n\tnc2 := clientConnect(t, sc.clusters[1].opts[1], \"bar\")\n\tdefer nc2.Close()\n\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\n\tvar sl server.ServiceLatency\n\tcheckFor(t, 3*time.Second, time.Second, func() error {\n\t\trmsg, err := rsub.NextMsg(500 * time.Millisecond)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tjson.Unmarshal(rmsg.Data, &sl)\n\t\treturn nil\n\t})\n\n\t// Make sure we have AppName set.\n\trs := sc.clusters[0].servers[1]\n\textendedCheck(t, sl.Responder, \"foo\", \"dlc22\", rs.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n}\n\nfunc createAccountWithJWT(t *testing.T) (string, nkeys.KeyPair, *jwt.AccountClaims) {\n\tt.Helper()\n\tokp, _ := nkeys.FromSeed(oSeed)\n\takp, _ := nkeys.CreateAccount()\n\tpub, _ := akp.PublicKey()\n\tnac := jwt.NewAccountClaims(pub)\n\tjwt, _ := nac.Encode(okp)\n\treturn jwt, akp, nac\n}\n\nfunc TestServiceLatencyWithJWT(t *testing.T) {\n\tokp, _ := nkeys.FromSeed(oSeed)\n\n\t// Create three accounts, system, service and normal account.\n\tsysJWT, sysKP, _ := createAccountWithJWT(t)\n\tsysPub, _ := sysKP.PublicKey()\n\n\t_, svcKP, svcAcc := createAccountWithJWT(t)\n\tsvcPub, _ := svcKP.PublicKey()\n\n\t// Add in the service export with latency tracking here.\n\tserviceExport := &jwt.Export{Subject: \"req.*\", Type: jwt.Service}\n\tsvcAcc.Exports.Add(serviceExport)\n\tsvcJWT, err := svcAcc.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\t_, accKP, accAcc := createAccountWithJWT(t)\n\taccPub, _ := accKP.PublicKey()\n\n\t// Add in the import.\n\tserviceImport := &jwt.Import{Account: svcPub, Subject: \"request\", To: \"req.echo\", Type: jwt.Service}\n\taccAcc.Imports.Add(serviceImport)\n\taccJWT, err := accAcc.Encode(okp)\n\tif err != nil {\n\t\tt.Fatalf(\"Error generating account JWT: %v\", err)\n\t}\n\n\tcf := `\n\tlisten: 127.0.0.1:-1\n\tcluster {\n\t\tname: \"A\"\n\t\tlisten: 127.0.0.1:-1\n\t\tauthorization {\n\t\t\ttimeout: 2.2\n\t\t} %s\n\t}\n\n\toperator = \"./configs/nkeys/op.jwt\"\n\tsystem_account = \"%s\"\n\n\tresolver = MEMORY\n\tresolver_preload = {\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t\t%s : \"%s\"\n\t}\n\t`\n\tcontents := strings.Replace(fmt.Sprintf(cf, \"\", sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\tconf := createConfFile(t, []byte(contents))\n\n\ts, opts := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\t// Create a new server and route to main one.\n\trouteStr := fmt.Sprintf(\"\\n\\t\\troutes = [nats-route://%s:%d]\", opts.Cluster.Host, opts.Cluster.Port)\n\tcontents2 := strings.Replace(fmt.Sprintf(cf, routeStr, sysPub, sysPub, sysJWT, svcPub, svcJWT, accPub, accJWT), \"\\n\\t\", \"\\n\", -1)\n\n\tconf2 := createConfFile(t, []byte(contents2))\n\n\ts2, opts2 := RunServerWithConfig(conf2)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s, s2)\n\n\t// Create service provider.\n\turl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tcopt, pubUser := createUserCredsOption(t, s, svcKP)\n\tnc, err := nats.Connect(url, copt, nats.Name(\"fooService\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// The service listener.\n\tserviceTime := 25 * time.Millisecond\n\tnc.Subscribe(\"req.echo\", func(msg *nats.Msg) {\n\t\ttime.Sleep(serviceTime)\n\t\tmsg.Respond(msg.Data)\n\t})\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"results\")\n\tnc.Flush()\n\n\t// Create second client and send request from this one.\n\turl2 := fmt.Sprintf(\"nats://%s:%d/\", opts2.Host, opts2.Port)\n\tnc2, err := nats.Connect(url2, createUserCreds(t, s2, accKP))\n\tif err != nil {\n\t\tt.Fatalf(\"Error creating client: %v\\n\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Send the request.\n\t_, err = nc2.Request(\"request\", []byte(\"hello\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\t// We should not receive latency at this time.\n\t_, err = rsub.NextMsg(100 * time.Millisecond)\n\tif err == nil {\n\t\tt.Fatalf(\"Did not expect to receive a latency metric\")\n\t}\n\n\t// Now turn it on..\n\tupdateAccount := func() {\n\t\tt.Helper()\n\t\tfor _, s := range []*server.Server{s, s2} {\n\t\t\tsvcAccount, err := s.LookupAccount(svcPub)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Could not lookup service account from server %+v\", s)\n\t\t\t}\n\t\t\ts.UpdateAccountClaims(svcAccount, svcAcc)\n\t\t}\n\t}\n\tserviceExport.Latency = &jwt.ServiceLatency{Sampling: 100, Results: \"results\"}\n\tupdateAccount()\n\n\t// Grab the service responder's user.\n\n\t// Send the request.\n\tstart := time.Now()\n\t_, err = nc2.Request(\"request\", []byte(\"hello\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl server.ServiceLatency\n\trmsg, err := rsub.NextMsg(time.Second)\n\tif err != nil || rmsg == nil {\n\t\tt.Fatalf(\"Did not receive a latency metric\")\n\t}\n\tjson.Unmarshal(rmsg.Data, &sl)\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\textendedCheck(t, sl.Responder, pubUser, \"fooService\", s.Name())\n\t// Normally requestor's don't share\n\tnoShareCheck(t, sl.Requestor)\n\n\t// Now we will remove tracking. Do this by simulating a JWT update.\n\tserviceExport.Latency = nil\n\tupdateAccount()\n\n\t// Now we should not get any tracking data.\n\t_, err = nc2.Request(\"request\", []byte(\"hello\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\t_, err = rsub.NextMsg(100 * time.Millisecond)\n\tif err == nil {\n\t\tt.Fatalf(\"Did not expect to receive a latency metric\")\n\t}\n}\n\nfunc TestServiceLatencyAdjustNegativeLatencyValues(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import\n\t// that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\t// Now we can setup and test, do single node only first.\n\t// This is the service provider.\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, err := nc.SubscribeSync(\"results\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Flush()\n\n\t// Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[0], \"bar\")\n\tdefer nc2.Close()\n\n\t// Send the request.\n\ttotalSamples := 50\n\tfor i := 0; i < totalSamples; i++ {\n\t\tif _, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected a response\")\n\t\t}\n\t}\n\n\tvar sl server.ServiceLatency\n\tfor i := 0; i < totalSamples; i++ {\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to receive latency metric: %d, %s\", i, err)\n\t\t}\n\t\tif err := json.Unmarshal(rmsg.Data, &sl); err != nil {\n\t\t\tt.Errorf(\"Unexpected error processing latency metric: %s\", err)\n\t\t}\n\t\tif sl.ServiceLatency < 0 {\n\t\t\tt.Fatalf(\"Unexpected negative latency value: %v\", sl)\n\t\t}\n\t}\n}\n\nfunc TestServiceLatencyRemoteConnectAdjustNegativeValues(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 2)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\t// Now we can setup and test, do single node only first.\n\t// This is the service provider.\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// The service listener.\n\tnc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\n\t// Listen for metrics\n\trsub, err := nc.SubscribeSync(\"results\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Flush()\n\n\t// Same Cluster Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[2], \"bar\")\n\tdefer nc2.Close()\n\n\t// Gateway Requestor\n\tnc3 := clientConnect(t, sc.clusters[1].opts[1], \"bar\")\n\tdefer nc3.Close()\n\n\t// Send a few initial requests to ensure interest is propagated\n\t// both for cluster and gateway requestors.\n\tcheckFor(t, 3*time.Second, time.Second, func() error {\n\t\t_, err1 := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\t\t_, err2 := nc3.Request(\"ngs.usage\", []byte(\"1h\"), time.Second)\n\n\t\tif err1 != nil || err2 != nil {\n\t\t\treturn fmt.Errorf(\"Timed out waiting for super cluster to be ready\")\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Send the request.\n\ttotalSamples := 20\n\tfor i := 0; i < totalSamples; i++ {\n\t\tif _, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected a response\")\n\t\t}\n\t}\n\n\tfor i := 0; i < totalSamples; i++ {\n\t\tif _, err := nc3.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\t\tt.Fatalf(\"Expected a response\")\n\t\t}\n\t}\n\n\tvar sl server.ServiceLatency\n\tfor i := 0; i < totalSamples*2; i++ {\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to receive latency metric: %d, %s\", i, err)\n\t\t}\n\t\tif err = json.Unmarshal(rmsg.Data, &sl); err != nil {\n\t\t\tt.Errorf(\"Unexpected error processing latency metric: %s\", err)\n\t\t}\n\t\tif sl.ServiceLatency < 0 {\n\t\t\tt.Fatalf(\"Unexpected negative service latency value: %v\", sl)\n\t\t}\n\t\tif sl.SystemLatency < 0 {\n\t\t\tt.Fatalf(\"Unexpected negative system latency value: %v\", sl)\n\t\t}\n\t}\n}\n\nfunc TestServiceLatencyFailureReportingSingleServer(t *testing.T) {\n\tsc := createSuperCluster(t, 1, 1)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\tsc.setResponseThreshold(t, 20*time.Millisecond)\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// Listen for metrics\n\trsub, err := nc.SubscribeSync(\"results\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Flush()\n\n\tgetMetricResult := func() *server.ServiceLatency {\n\t\tt.Helper()\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to receive latency metric: %v\", err)\n\t\t}\n\t\tvar sl server.ServiceLatency\n\t\tif err = json.Unmarshal(rmsg.Data, &sl); err != nil {\n\t\t\tt.Errorf(\"Unexpected error processing latency metric: %s\", err)\n\t\t}\n\t\treturn &sl\n\t}\n\n\t// Same server\n\tnc2 := clientConnect(t, sc.clusters[0].opts[0], \"bar\")\n\tdefer nc2.Close()\n\n\t// Test a request with no reply subject\n\tnc2.Publish(\"ngs.usage\", []byte(\"1h\"))\n\tsl := getMetricResult()\n\n\tif sl.Status != 400 {\n\t\tt.Fatalf(\"Expected to get a bad request status [400], got %d\", sl.Status)\n\t}\n\n\t// Proper request, but no responders.\n\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 20*time.Millisecond)\n\tsl = getMetricResult()\n\tif sl.Status != 503 {\n\t\tt.Fatalf(\"Expected to get a service unavailable status [503], got %d\", sl.Status)\n\t}\n\n\t// The service listener. Make it slow. 20ms is respThreshold, so take 2X\n\tsub, _ := nc.Subscribe(\"ngs.usage.bar\", func(msg *nats.Msg) {\n\t\ttime.Sleep(40 * time.Millisecond)\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\tnc.Flush()\n\n\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 20*time.Millisecond)\n\tsl = getMetricResult()\n\tif sl.Status != 504 {\n\t\tt.Fatalf(\"Expected to get a service timeout status [504], got %d\", sl.Status)\n\t}\n\n\t// Make sure we do not get duplicates.\n\tif rmsg, err := rsub.NextMsg(50 * time.Millisecond); err == nil {\n\t\tt.Fatalf(\"Unexpected second response metric: %q\\n\", rmsg.Data)\n\t}\n\n\t// Now setup a responder that will respond under the threshold.\n\tsub.Unsubscribe()\n\tnc.Subscribe(\"ngs.usage.bar\", func(msg *nats.Msg) {\n\t\ttime.Sleep(5 * time.Millisecond)\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t})\n\tnc.Flush()\n\ttime.Sleep(100 * time.Millisecond)\n\n\t// Now create a responder using old request and we will do a short timeout.\n\tnc3 := clientConnectOldRequest(t, sc.clusters[0].opts[0], \"bar\")\n\tdefer nc3.Close()\n\n\tnc3.Request(\"ngs.usage\", []byte(\"1h\"), time.Millisecond)\n\tsl = getMetricResult()\n\tif sl.Status != 408 {\n\t\tt.Fatalf(\"Expected to get a request timeout status [408], got %d\", sl.Status)\n\t}\n}\n\nfunc TestServiceLatencyFailureReportingMultipleServers(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\tsc.setResponseThreshold(t, 10*time.Millisecond)\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// Listen for metrics\n\trsub, err := nc.SubscribeSync(\"results\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Flush()\n\n\tgetMetricResult := func() *server.ServiceLatency {\n\t\tt.Helper()\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to receive latency metric: %v\", err)\n\t\t}\n\t\tvar sl server.ServiceLatency\n\t\tif err = json.Unmarshal(rmsg.Data, &sl); err != nil {\n\t\t\tt.Errorf(\"Unexpected error processing latency metric: %s\", err)\n\t\t}\n\t\treturn &sl\n\t}\n\n\tcases := []struct {\n\t\tci, si int\n\t\tdesc string\n\t}{\n\t\t{0, 0, \"same server\"},\n\t\t{0, 1, \"same cluster, different server\"},\n\t\t{1, 1, \"different cluster\"},\n\t}\n\n\tfor _, cs := range cases {\n\t\t// Select the server to send request from.\n\t\tnc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], \"bar\")\n\t\tdefer nc2.Close()\n\n\t\t// Test a request with no reply subject\n\t\tnc2.Publish(\"ngs.usage\", []byte(\"1h\"))\n\t\tsl := getMetricResult()\n\t\tif sl.Status != 400 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a bad request status [400], got %d\", cs.desc, sl.Status)\n\t\t}\n\n\t\t// We wait here for the gateways to report no interest b/c optimistic mode.\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\t// Proper request, but no responders.\n\t\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 10*time.Millisecond)\n\t\tsl = getMetricResult()\n\t\tif sl.Status != 503 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a service unavailable status [503], got %d\", cs.desc, sl.Status)\n\t\t}\n\n\t\t// The service listener. Make it slow. 10ms is respThreshold, so make 3X\n\t\tsub, _ := nc.Subscribe(\"ngs.usage.bar\", func(msg *nats.Msg) {\n\t\t\ttime.Sleep(30 * time.Millisecond)\n\t\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t\t})\n\t\tdefer sub.Unsubscribe()\n\t\tnc.Flush()\n\t\t// Wait to propagate.\n\t\ttime.Sleep(200 * time.Millisecond)\n\n\t\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 10*time.Millisecond)\n\t\tsl = getMetricResult()\n\t\tif sl.Status != 504 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a service timeout status [504], got %d\", cs.desc, sl.Status)\n\t\t}\n\n\t\t// Clean up subscriber and requestor\n\t\tnc2.Close()\n\t\tsub.Unsubscribe()\n\t\tnc.Flush()\n\t\t// Wait to propagate.\n\t\ttime.Sleep(200 * time.Millisecond)\n\t}\n}\n\n// To test a bug rip@nats.io is seeing.\nfunc TestServiceLatencyOldRequestStyleSingleServer(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t SVC: {\n\t\t users: [ {user: svc, password: pass} ]\n\t\t exports: [ {\n\t\t\t\t\tservice: \"svc.echo\"\n\t\t\t\t\taccounts: [CLIENT]\n\t\t\t\t\tlatency: {\n\t\t\t\t\t\tsampling: 100%\n\t\t\t\t\t\tsubject: latency.svc\n\t\t\t\t\t}\n\t\t\t\t} ]\n\t\t },\n\t\t CLIENT: {\n\t\t users: [{user: client, password: pass} ]\n\t\t\t imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ]\n\t\t },\n\t\t SYS: { users: [{user: admin, password: pass}] }\n\t\t}\n\n\t\tsystem_account: SYS\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://svc:pass@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"latency.svc\")\n\n\t// Requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://client:pass@%s:%d\", opts.Host, opts.Port), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Setup responder\n\tserviceTime := 25 * time.Millisecond\n\tsub, _ := nc.Subscribe(\"svc.echo\", func(msg *nats.Msg) {\n\t\ttime.Sleep(serviceTime)\n\t\tmsg.Respond([]byte(\"world\"))\n\t})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\t// Send a request\n\tstart := time.Now()\n\tif _, err := nc2.Request(\"SVC\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl server.ServiceLatency\n\trmsg, _ := rsub.NextMsg(time.Second)\n\tjson.Unmarshal(rmsg.Data, &sl)\n\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\textendedCheck(t, sl.Responder, \"svc\", \"\", srv.Name())\n\tnoShareCheck(t, sl.Requestor)\n}\n\n// To test a bug wally@nats.io is seeing.\nfunc TestServiceAndStreamStackOverflow(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t STATIC {\n\t\t users = [ { user: \"static\", pass: \"foo\" } ]\n\t\t exports [\n\t\t { stream: > }\n\t\t { service: my.service }\n\t\t ]\n\t\t }\n\t\t DYN {\n\t\t users = [ { user: \"foo\", pass: \"bar\" } ]\n\t\t imports [\n\t\t { stream { subject: >, account: STATIC } }\n\t\t { service { subject: my.service, account: STATIC } }\n\t\t ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder (just request sub)\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://static:foo@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tsub, _ := nc.SubscribeSync(\"my.service\")\n\tnc.Flush()\n\n\t// Requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://foo:bar@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Send a single request.\n\tnc2.PublishRequest(\"my.service\", \"foo\", []byte(\"hi\"))\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif nm, _, err := sub.Pending(); err != nil || nm != 1 {\n\t\t\treturn fmt.Errorf(\"Expected one request, got %d\", nm)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Make sure works for queue subscribers as well.\n\tsub.Unsubscribe()\n\tsub, _ = nc.QueueSubscribeSync(\"my.service\", \"prod\")\n\tnc.Flush()\n\n\t// Send a single request.\n\tnc2.PublishRequest(\"my.service\", \"foo\", []byte(\"hi\"))\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif nm, _, err := sub.Pending(); err != nil || nm != 1 {\n\t\t\treturn fmt.Errorf(\"Expected one request, got %d\", nm)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now create an interest in the stream from nc2. that is a queue subscriber.\n\tsub2, _ := nc2.QueueSubscribeSync(\"my.service\", \"prod\")\n\tdefer sub2.Unsubscribe()\n\tnc2.Flush()\n\n\t// Send a single request.\n\tnc2.PublishRequest(\"my.service\", \"foo\", []byte(\"hi\"))\n\ttime.Sleep(10 * time.Millisecond)\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif nm, _, err := sub.Pending(); err != nil || nm != 2 {\n\t\t\treturn fmt.Errorf(\"Expected two requests, got %d\", nm)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Check we get the proper detailed information for the requestor when allowed.\nfunc TestServiceLatencyRequestorSharesDetailedInfo(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 3)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\tsc.setResponseThreshold(t, 10*time.Millisecond)\n\tsc.setImportShare(t)\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\t// Listen for metrics\n\trsub, err := nc.SubscribeSync(\"results\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc.Flush()\n\n\tgetMetricResult := func() *server.ServiceLatency {\n\t\tt.Helper()\n\t\trmsg, err := rsub.NextMsg(time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Expected to receive latency metric: %v\", err)\n\t\t}\n\t\tvar sl server.ServiceLatency\n\t\tif err = json.Unmarshal(rmsg.Data, &sl); err != nil {\n\t\t\tt.Errorf(\"Unexpected error processing latency metric: %s\", err)\n\t\t}\n\t\treturn &sl\n\t}\n\n\tcases := []struct {\n\t\tci, si int\n\t\tdesc string\n\t}{\n\t\t{0, 0, \"same server\"},\n\t\t{0, 1, \"same cluster, different server\"},\n\t\t{1, 1, \"different cluster\"},\n\t}\n\n\tfor _, cs := range cases {\n\t\t// Select the server to send request from.\n\t\tnc2 := clientConnect(t, sc.clusters[cs.ci].opts[cs.si], \"bar\")\n\t\tdefer nc2.Close()\n\n\t\trs := sc.clusters[cs.ci].servers[cs.si]\n\n\t\t// Test a request with no reply subject\n\t\tnc2.Publish(\"ngs.usage\", []byte(\"1h\"))\n\t\tsl := getMetricResult()\n\t\tif sl.Status != 400 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a bad request status [400], got %d\", cs.desc, sl.Status)\n\t\t}\n\t\textendedCheck(t, sl.Requestor, \"bar\", \"\", rs.Name())\n\n\t\t// We wait here for the gateways to report no interest b/c optimistic mode.\n\t\ttime.Sleep(50 * time.Millisecond)\n\n\t\t// Proper request, but no responders.\n\t\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 10*time.Millisecond)\n\t\tsl = getMetricResult()\n\t\tif sl.Status != 503 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a service unavailable status [503], got %d\", cs.desc, sl.Status)\n\t\t}\n\t\textendedCheck(t, sl.Requestor, \"bar\", \"\", rs.Name())\n\n\t\t// The service listener. Make it slow. 10ms is respThreshold, so take 2.5X\n\t\tsub, _ := nc.Subscribe(\"ngs.usage.bar\", func(msg *nats.Msg) {\n\t\t\ttime.Sleep(25 * time.Millisecond)\n\t\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t\t})\n\t\tdefer sub.Unsubscribe()\n\t\tnc.Flush()\n\t\t// Wait to propagate.\n\t\ttime.Sleep(200 * time.Millisecond)\n\n\t\tnc2.Request(\"ngs.usage\", []byte(\"1h\"), 10*time.Millisecond)\n\t\tsl = getMetricResult()\n\t\tif sl.Status != 504 {\n\t\t\tt.Fatalf(\"Test %q, Expected to get a service timeout status [504], got %d\", cs.desc, sl.Status)\n\t\t}\n\t\textendedCheck(t, sl.Requestor, \"bar\", \"\", rs.Name())\n\n\t\t// Clean up subscriber and requestor\n\t\tnc2.Close()\n\t\tsub.Unsubscribe()\n\t\tnc.Flush()\n\t\t// Wait to propagate.\n\t\ttime.Sleep(200 * time.Millisecond)\n\t}\n}\n\nfunc TestServiceLatencyRequestorSharesConfig(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t SVC: {\n\t\t users: [ {user: svc, password: pass} ]\n\t\t exports: [ {\n\t\t\t\t\tservice: \"svc.echo\"\n\t\t\t\t\taccounts: [CLIENT]\n\t\t\t\t\tlatency: {\n\t\t\t\t\t\tsampling: 100%\n\t\t\t\t\t\tsubject: latency.svc\n\t\t\t\t\t}\n\t\t\t\t} ]\n\t\t },\n\t\t CLIENT: {\n\t\t users: [{user: client, password: pass} ]\n\t\t\t imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ]\n\t\t },\n\t\t SYS: { users: [{user: admin, password: pass}] }\n\t\t}\n\n\t\tsystem_account: SYS\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://svc:pass@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"latency.svc\")\n\n\t// Requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://client:pass@%s:%d\", opts.Host, opts.Port), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Setup responder\n\tserviceTime := 25 * time.Millisecond\n\tsub, _ := nc.Subscribe(\"svc.echo\", func(msg *nats.Msg) {\n\t\ttime.Sleep(serviceTime)\n\t\tmsg.Respond([]byte(\"world\"))\n\t})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\t// Send a request\n\tstart := time.Now()\n\tif _, err := nc2.Request(\"SVC\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl server.ServiceLatency\n\trmsg, _ := rsub.NextMsg(time.Second)\n\tjson.Unmarshal(rmsg.Data, &sl)\n\n\tcheckServiceLatency(t, sl, start, serviceTime)\n\textendedCheck(t, sl.Responder, \"svc\", \"\", srv.Name())\n\textendedCheck(t, sl.Requestor, \"client\", \"\", srv.Name())\n\n\t// Check reload.\n\tnewConf := []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t SVC: {\n\t\t users: [ {user: svc, password: pass} ]\n\t\t exports: [ {\n\t\t\t\t\tservice: \"svc.echo\"\n\t\t\t\t\taccounts: [CLIENT]\n\t\t\t\t\tlatency: {\n\t\t\t\t\t\tsampling: 100%\n\t\t\t\t\t\tsubject: latency.svc\n\t\t\t\t\t}\n\t\t\t\t} ]\n\t\t },\n\t\t CLIENT: {\n\t\t users: [{user: client, password: pass} ]\n\t\t\t imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC} ]\n\t\t },\n\t\t SYS: { users: [{user: admin, password: pass}] }\n\t\t}\n\n\t\tsystem_account: SYS\n\t`)\n\tif err := os.WriteFile(conf, newConf, 0600); err != nil {\n\t\tt.Fatalf(\"Error rewriting server's config file: %v\", err)\n\t}\n\tif err := srv.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on server reload: %v\", err)\n\t}\n\n\tif _, err = nc2.Request(\"SVC\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\tvar sl2 server.ServiceLatency\n\trmsg, _ = rsub.NextMsg(time.Second)\n\tjson.Unmarshal(rmsg.Data, &sl2)\n\tnoShareCheck(t, sl2.Requestor)\n}\n\nfunc TestServiceLatencyLossTest(t *testing.T) {\n\t// assure that behavior with respect to requests timing out (and samples being reordered) is as expected.\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t SVC: {\n\t\t users: [ {user: svc, password: pass} ]\n\t\t exports: [ {\n\t\t\t\t\tservice: \"svc.echo\"\n\t\t\t\t\tthreshold: \"500ms\"\n\t\t\t\t\taccounts: [CLIENT]\n\t\t\t\t\tlatency: {\n\t\t\t\t\t\tsampling: headers\n\t\t\t\t\t\tsubject: latency.svc\n\t\t\t\t\t}\n\t\t\t\t} ]\n\t\t },\n\t\t CLIENT: {\n\t\t users: [{user: client, password: pass} ]\n\t\t\t imports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:true} ]\n\t\t },\n\t\t SYS: { users: [{user: admin, password: pass}] }\n\t\t}\n\t\tsystem_account: SYS\n\t`))\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder connection\n\tncr, err := nats.Connect(fmt.Sprintf(\"nats://svc:pass@%s:%d\", opts.Host, opts.Port), nats.Name(\"responder\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncr.Close()\n\n\tncl, err := nats.Connect(fmt.Sprintf(\"nats://svc:pass@%s:%d\", opts.Host, opts.Port), nats.Name(\"latency\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer ncl.Close()\n\t// Table of expected state for which message.\n\t// This also codifies that the first message, in respsonse to second request is ok.\n\t// Second message, in response to first request times out.\n\texpectedState := map[int]int{1: http.StatusOK, 2: http.StatusGatewayTimeout}\n\tmsgCnt := 0\n\tstart := time.Now().Add(250 * time.Millisecond)\n\n\tvar latErr []error\n\t// Listen for metrics\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\trsub, _ := ncl.Subscribe(\"latency.svc\", func(rmsg *nats.Msg) {\n\t\tdefer wg.Done()\n\t\tvar sl server.ServiceLatency\n\t\tjson.Unmarshal(rmsg.Data, &sl)\n\t\tmsgCnt++\n\t\tif want := expectedState[msgCnt]; want != sl.Status {\n\t\t\tlatErr = append(latErr, fmt.Errorf(\"Expected different status for msg #%d: %d != %d\", msgCnt, want, sl.Status))\n\t\t}\n\t\tif msgCnt > 1 {\n\t\t\tif start.Before(sl.RequestStart) {\n\t\t\t\tlatErr = append(latErr, fmt.Errorf(\"start times should indicate reordering %v : %v\", start, sl.RequestStart))\n\t\t\t}\n\t\t}\n\t\tstart = sl.RequestStart\n\t\tif strings.EqualFold(sl.RequestHeader.Get(\"Uber-Trace-Id\"), fmt.Sprintf(\"msg-%d\", msgCnt)) {\n\t\t\tlatErr = append(latErr, fmt.Errorf(\"no header present\"))\n\t\t}\n\t})\n\tdefer rsub.Unsubscribe()\n\t// Setup requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://client:pass@%s:%d\", opts.Host, opts.Port),\n\t\tnats.UseOldRequestStyle(), nats.Name(\"requestor\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\trespCnt := int64(0)\n\treply := nc2.NewRespInbox()\n\trepSub, _ := nc2.Subscribe(reply, func(msg *nats.Msg) {\n\t\tatomic.AddInt64(&respCnt, 1)\n\t})\n\tdefer repSub.Unsubscribe()\n\tnc2.Flush()\n\t// use dedicated send that publishes requests using same reply subject\n\tsend := func(msg string) {\n\t\tif err := nc2.PublishMsg(&nats.Msg{Subject: \"SVC\", Data: []byte(msg), Reply: reply,\n\t\t\tHeader: nats.Header{\"X-B3-Sampled\": []string{\"1\"}}}); err != nil {\n\t\t\tt.Fatalf(\"Expected a response got: %v\", err)\n\t\t}\n\t}\n\t// Setup responder that skips responding and triggers next request OR responds\n\tsub, _ := ncr.Subscribe(\"svc.echo\", func(msg *nats.Msg) {\n\t\tif string(msg.Data) != \"msg2\" {\n\t\t\tmsg.Respond([]byte(\"response\"))\n\t\t} else {\n\t\t\twg.Add(1)\n\t\t\tgo func() { // second request (use go routine to not block in responders callback)\n\t\t\t\tdefer wg.Done()\n\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\tsend(\"msg1\") // will cause the first latency measurement\n\t\t\t}()\n\t\t}\n\t})\n\tncr.Flush()\n\tncl.Flush()\n\tnc2.Flush()\n\tdefer sub.Unsubscribe()\n\t// Send first request, which is expected to timeout\n\tsend(\"msg2\")\n\t// wait till we got enough responses\n\twg.Wait()\n\tif len(latErr) > 0 {\n\t\tt.Fatalf(\"Got errors %v\", latErr)\n\t}\n\tif atomic.LoadInt64(&respCnt) != 1 {\n\t\tt.Fatalf(\"Expected only one message\")\n\t}\n}\n\nfunc TestServiceLatencyHeaderTriggered(t *testing.T) {\n\treceiveAndTest := func(t *testing.T, rsub *nats.Subscription, shared bool, header nats.Header, status int, srvName string) server.ServiceLatency {\n\t\tt.Helper()\n\t\tvar sl server.ServiceLatency\n\t\trmsg, _ := rsub.NextMsg(time.Second)\n\t\tif rmsg == nil {\n\t\t\tt.Fatal(\"Expected message\")\n\t\t\treturn sl\n\t\t}\n\t\tjson.Unmarshal(rmsg.Data, &sl)\n\t\tif sl.Status != status {\n\t\t\tt.Fatalf(\"Expected different status %d != %d\", status, sl.Status)\n\t\t}\n\t\tif status == http.StatusOK {\n\t\t\textendedCheck(t, sl.Responder, \"svc\", \"\", srvName)\n\t\t}\n\t\tif shared {\n\t\t\textendedCheck(t, sl.Requestor, \"client\", \"\", srvName)\n\t\t} else {\n\t\t\tnoShareCheck(t, sl.Requestor)\n\t\t}\n\t\t// header are always included\n\t\tif v := sl.RequestHeader.Get(\"Some-Other\"); v != \"\" {\n\t\t\tt.Fatalf(\"Expected header to be gone\")\n\t\t}\n\t\tfor k, value := range header {\n\t\t\tif v := sl.RequestHeader.Get(k); v != value[0] {\n\t\t\t\tt.Fatalf(\"Expected header %q to be set\", k)\n\t\t\t}\n\t\t}\n\t\treturn sl\n\t}\n\tfor _, v := range []struct {\n\t\tshared bool\n\t\theader nats.Header\n\t}{\n\t\t{true, nats.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:479fefe9525eddb:0:1\"}}},\n\t\t{true, nats.Header{\"X-B3-Sampled\": []string{\"1\"}}},\n\t\t{true, nats.Header{\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"}}},\n\t\t{true, nats.Header{\"B3\": []string{\"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90\"}}},\n\t\t{true, nats.Header{\"Traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}},\n\t\t{false, nats.Header{\"Uber-Trace-Id\": []string{\"479fefe9525eddb:479fefe9525eddb:0:1\"}}},\n\t\t{false, nats.Header{\"X-B3-Sampled\": []string{\"1\"}}},\n\t\t{false, nats.Header{\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"}}},\n\t\t{false, nats.Header{\"B3\": []string{\"80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1-1-05e3ac9a4f6e3b90\"}}},\n\t\t{false, nats.Header{\"Traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"}}},\n\t\t{false, nats.Header{\n\t\t\t\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"},\n\t\t\t\"X-B3-ParentSpanId\": []string{\"05e3ac9a4f6e3b90\"},\n\t\t\t\"X-B3-SpanId\": []string{\"e457b5a2e4d86bd1\"},\n\t\t\t\"X-B3-Sampled\": []string{\"1\"},\n\t\t}},\n\t\t{false, nats.Header{\n\t\t\t\"X-B3-TraceId\": []string{\"80f198ee56343ba864fe8b2a57d3eff7\"},\n\t\t\t\"X-B3-ParentSpanId\": []string{\"05e3ac9a4f6e3b90\"},\n\t\t\t\"X-B3-SpanId\": []string{\"e457b5a2e4d86bd1\"},\n\t\t}},\n\t\t{false, nats.Header{\n\t\t\t\"Uber-Trace-Id\": []string{\"479fefe9525eddb:479fefe9525eddb:0:1\"},\n\t\t\t\"Uberctx-X\": []string{\"foo\"},\n\t\t}},\n\t\t{false, nats.Header{\n\t\t\t\"Traceparent\": []string{\"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\"},\n\t\t\t\"Tracestate\": []string{\"rojo=00f067aa0ba902b7,congo=t61rcWkgMzE\"},\n\t\t}},\n\t} {\n\t\tt.Run(fmt.Sprintf(\"%s_%t_%s\", t.Name(), v.shared, v.header), func(t *testing.T) {\n\t\t\tconf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\t\t\tlisten: 127.0.0.1:-1\n\t\t\t\taccounts: {\n\t\t\t\t\tSVC: {\n\t\t\t\t\t\tusers: [ {user: svc, password: pass} ]\n\t\t\t\t\t\texports: [ {\n\t\t\t\t\t\t\tservice: \"svc.echo\"\n\t\t\t\t\t\t\taccounts: [CLIENT]\n\t\t\t\t\t\t\tlatency: {\n\t\t\t\t\t\t\t\tsampling: headers\n\t\t\t\t\t\t\t\tsubject: latency.svc\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}]\n\t\t\t\t\t},\n\t\t\t\t\tCLIENT: {\n\t\t\t\t\t\tusers: [{user: client, password: pass} ]\n\t\t\t\t\t\timports: [ {service: {account: SVC, subject: svc.echo}, to: SVC, share:%t} ]\n\t\t\t\t\t},\n\t\t\t\t\tSYS: { users: [{user: admin, password: pass}] }\n\t\t\t\t}\n\n\t\t\t\tsystem_account: SYS\n\t\t\t`, v.shared)))\n\t\t\tsrv, opts := RunServerWithConfig(conf)\n\t\t\tdefer srv.Shutdown()\n\n\t\t\t// Responder\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"nats://svc:pass@%s:%d\", opts.Host, opts.Port))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\t// Listen for metrics\n\t\t\trsub, _ := nc.SubscribeSync(\"latency.svc\")\n\t\t\tdefer rsub.Unsubscribe()\n\n\t\t\t// Setup responder\n\t\t\tserviceTime := 25 * time.Millisecond\n\t\t\tsub, _ := nc.Subscribe(\"svc.echo\", func(msg *nats.Msg) {\n\t\t\t\ttime.Sleep(serviceTime)\n\t\t\t\tmsg.Respond([]byte(\"world\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\t\t\tdefer sub.Unsubscribe()\n\n\t\t\t// Setup requestor\n\t\t\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://client:pass@%s:%d\", opts.Host, opts.Port), nats.UseOldRequestStyle())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t\t\t}\n\t\t\tdefer nc2.Close()\n\n\t\t\t// Send a request\n\t\t\tstart := time.Now()\n\t\t\tmsg := &nats.Msg{\n\t\t\t\tSubject: \"SVC\",\n\t\t\t\tData: []byte(\"1h\"),\n\t\t\t\tHeader: make(nats.Header),\n\t\t\t}\n\t\t\tfor k, v := range v.header {\n\t\t\t\tfor _, val := range v {\n\t\t\t\t\tmsg.Header.Add(k, val)\n\t\t\t\t}\n\t\t\t}\n\t\t\tmsg.Header.Add(\"Some-Other\", \"value\")\n\t\t\tif _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err != nil {\n\t\t\t\tt.Fatalf(\"Expected a response\")\n\t\t\t}\n\t\t\tsl := receiveAndTest(t, rsub, v.shared, v.header, http.StatusOK, srv.Name())\n\t\t\tcheckServiceLatency(t, sl, start, serviceTime)\n\t\t\t// shut down responder to test various error scenarios\n\t\t\tsub.Unsubscribe()\n\t\t\tnc.Flush()\n\t\t\t// Send a request without responder\n\t\t\tif _, err := nc2.RequestMsg(msg, 50*time.Millisecond); err == nil {\n\t\t\t\tt.Fatalf(\"Expected no response\")\n\t\t\t}\n\t\t\treceiveAndTest(t, rsub, v.shared, v.header, http.StatusServiceUnavailable, srv.Name())\n\n\t\t\t// send a message without a response\n\t\t\tmsg.Reply = \"\"\n\t\t\tif err := nc2.PublishMsg(msg); err != nil {\n\t\t\t\tt.Fatalf(\"Expected no error got %v\", err)\n\t\t\t}\n\t\t\treceiveAndTest(t, rsub, v.shared, v.header, http.StatusBadRequest, srv.Name())\n\t\t})\n\t}\n}\n\n// From a report by rip@nats.io on simple latency reporting missing in 2 server cluster setup.\nfunc TestServiceLatencyMissingResults(t *testing.T) {\n\taccConf := createConfFile(t, []byte(`\n\t\taccounts {\n\t\t one: {\n\t\t users = [ {user: one, password: password} ]\n\t\t imports = [ {service: {account: weather, subject: service.weather.requests.>}, to: service.weather.>, share: true} ]\n\t\t }\n\t\t weather: {\n\t\t users = [ {user: weather, password: password} ]\n\t\t exports = [ {\n\t\t service: service.weather.requests.>\n\t\t accounts: [one]\n\t\t latency: { sampling: 100%, subject: service.weather.latency }\n\t\t } ]\n\t\t }\n\t\t}\n\t`))\n\n\ts1Conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s1\n\t\tcluster { port: -1 }\n\t\tinclude %q\n\t`, filepath.Base(accConf))))\n\n\t// Link accConf for relative import from s1Conf\n\tos.Link(accConf, filepath.Join(filepath.Dir(s1Conf), filepath.Base(accConf)))\n\n\ts1, opts1 := RunServerWithConfig(s1Conf)\n\tdefer s1.Shutdown()\n\n\ts2Conf := createConfFile(t, []byte(fmt.Sprintf(`\n\t\tlisten: 127.0.0.1:-1\n\t\tserver_name: s2\n\t\tcluster {\n\t\t\tport: -1\n\t\t\troutes = [ nats-route://127.0.0.1:%d ]\n\t\t}\n\t\tinclude %q\n\t`, opts1.Cluster.Port, filepath.Base(accConf))))\n\n\t// Link accConf for relative import from s2Conf\n\tos.Link(accConf, filepath.Join(filepath.Dir(s2Conf), filepath.Base(accConf)))\n\n\ts2, opts2 := RunServerWithConfig(s2Conf)\n\tdefer s2.Shutdown()\n\n\tcheckClusterFormed(t, s1, s2)\n\n\tnc1, err := nats.Connect(fmt.Sprintf(\"nats://%s:%s@%s:%d\", \"weather\", \"password\", opts1.Host, opts1.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Create responder\n\tsub, _ := nc1.Subscribe(\"service.weather.requests.>\", func(msg *nats.Msg) {\n\t\ttime.Sleep(25 * time.Millisecond)\n\t\tmsg.Respond([]byte(\"sunny!\"))\n\t})\n\tdefer sub.Unsubscribe()\n\n\t// Create sync listener for latency.\n\tlatSubj := \"service.weather.latency\"\n\tlsub, _ := nc1.SubscribeSync(latSubj)\n\tdefer lsub.Unsubscribe()\n\tnc1.Flush()\n\n\t// Make sure the subscription propagates to s2 server.\n\tcheckSubInterest(t, s2, \"weather\", latSubj, time.Second)\n\n\t// Create requestor on s2.\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://%s:%s@%s:%d\", \"one\", \"password\", opts2.Host, opts2.Port), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc2.Request(\"service.weather.los_angeles\", nil, time.Second)\n\n\tlr, err := lsub.NextMsg(time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected a latency result, got %v\", err)\n\t}\n\t// Make sure we reported ok and have valid results for service, system and total.\n\tvar sl server.ServiceLatency\n\tjson.Unmarshal(lr.Data, &sl)\n\n\tif sl.Status != 200 {\n\t\tt.Fatalf(\"Expected a 200 status, got %d\\n\", sl.Status)\n\t}\n\tif sl.ServiceLatency == 0 || sl.SystemLatency == 0 || sl.TotalLatency == 0 {\n\t\tt.Fatalf(\"Received invalid tracking measurements, %d %d %d\", sl.ServiceLatency, sl.SystemLatency, sl.TotalLatency)\n\t}\n}\n\n// To test a bug I was seeing.\nfunc TestServiceLatencyDoubleResponse(t *testing.T) {\n\tsc := createSuperCluster(t, 3, 1)\n\tdefer sc.shutdown()\n\n\t// Now add in new service export to FOO and have bar import that with tracking enabled.\n\tsc.setupLatencyTracking(t, 100)\n\n\t// Responder\n\tnc := clientConnectWithName(t, sc.clusters[0].opts[0], \"foo\", \"service22\")\n\tdefer nc.Close()\n\n\t// Setup responder\n\tsub, _ := nc.Subscribe(\"ngs.usage.*\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"22 msgs\"))\n\t\tmsg.Respond([]byte(\"boom\"))\n\t})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\t// Listen for metrics\n\trsub, _ := nc.SubscribeSync(\"latency.svc\")\n\n\t// Requestor\n\tnc2 := clientConnect(t, sc.clusters[0].opts[2], \"bar\")\n\tdefer nc2.Close()\n\n\t// Send a request\n\tif _, err := nc2.Request(\"ngs.usage\", []byte(\"1h\"), time.Second); err != nil {\n\t\tt.Fatalf(\"Expected a response\")\n\t}\n\n\trsub.NextMsg(time.Second)\n\ttime.Sleep(time.Second)\n}\n" }, { "path": "test/services_test.go", "content": "// Copyright 2020-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nvar basicMASetupContents = []byte(`\n\tserver_name: A\n\tlisten: 127.0.0.1:-1\n\n\taccounts: {\n\t A: {\n\t users: [ {user: a, password: pwd} ]\n\t exports: [{service: \"foo\", response: stream}]\n\t },\n\t B: {\n\t users: [{user: b, password: pwd} ]\n\t\t imports: [{ service: { account: A, subject: \"foo\"}, to: \"foo_request\" }]\n\t }\n\t}\n`)\n\nfunc TestServiceImportWithStreamed(t *testing.T) {\n\tconf := createConfFile(t, basicMASetupContents)\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Limit max response maps here for the test.\n\taccB, err := srv.LookupAccount(\"B\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\t// connect and offer a service\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tif err := msg.Respond([]byte(\"world\")); err != nil {\n\t\t\tt.Fatalf(\"Error on respond: %v\", err)\n\t\t}\n\t})\n\tnc.Flush()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnumRequests := 10\n\tfor i := 0; i < numRequests; i++ {\n\t\tresp, err := nc2.Request(\"foo_request\", []byte(\"hello\"), 2*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif resp == nil || strings.Compare(\"world\", string(resp.Data)) != 0 {\n\t\t\tt.Fatal(\"Did not receive the correct message\")\n\t\t}\n\t}\n\n\t// Since we are using a new client that multiplexes, until the client itself goes away\n\t// we will have the full number of entries, even with the tighter ResponseEntriesPruneThreshold.\n\taccA, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\t// These should always be the same now.\n\tif nre := accB.NumPendingReverseResponses(); nre != numRequests {\n\t\tt.Fatalf(\"Expected %d entries, got %d\", numRequests, nre)\n\t}\n\tif nre := accA.NumPendingAllResponses(); nre != numRequests {\n\t\tt.Fatalf(\"Expected %d entries, got %d\", numRequests, nre)\n\t}\n\n\t// Now kill of the client that was doing the requests.\n\tnc2.Close()\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\taNrssi := accA.NumPendingAllResponses()\n\t\tbNre := accB.NumPendingReverseResponses()\n\t\tif aNrssi != 0 || bNre != 0 {\n\t\t\treturn fmt.Errorf(\"Response imports and response entries should all be 0, got %d %d\", aNrssi, bNre)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Now let's test old style request and reply that uses a new inbox each time. This should work ok..\n\tnc2, err = nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tfor i := 0; i < numRequests; i++ {\n\t\tresp, err := nc2.Request(\"foo_request\", []byte(\"hello\"), 2*time.Second)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t}\n\t\tif resp == nil || strings.Compare(\"world\", string(resp.Data)) != 0 {\n\t\t\tt.Fatal(\"Did not receive the correct message\")\n\t\t}\n\t}\n\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\taNrssi := accA.NumPendingAllResponses()\n\t\tbNre := accB.NumPendingReverseResponses()\n\t\tif aNrssi != 0 || bNre != 0 {\n\t\t\treturn fmt.Errorf(\"Response imports and response entries should all be 0, got %d %d\", aNrssi, bNre)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestServiceImportWithStreamedResponseAndEOF(t *testing.T) {\n\tconf := createConfFile(t, basicMASetupContents)\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\taccA, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\taccB, err := srv.LookupAccount(\"B\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// We will send four responses and then and nil message signaling EOF\n\tnc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\t// Streamed response.\n\t\tmsg.Respond([]byte(\"world-1\"))\n\t\tmsg.Respond([]byte(\"world-2\"))\n\t\tmsg.Respond([]byte(\"world-3\"))\n\t\tmsg.Respond([]byte(\"world-4\"))\n\t\tmsg.Respond(nil)\n\t})\n\tnc.Flush()\n\n\t// Now setup requester.\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnumRequests := 10\n\texpectedResponses := 5\n\n\tfor i := 0; i < numRequests; i++ {\n\t\t// Create an inbox\n\t\treply := nats.NewInbox()\n\t\tsub, _ := nc2.SubscribeSync(reply)\n\t\tdefer sub.Unsubscribe()\n\n\t\tif err := nc2.PublishRequest(\"foo_request\", reply, []byte(\"XOXO\")); err != nil {\n\t\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t\t}\n\n\t\t// Wait and make sure we get all the responses. Should be five.\n\t\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != expectedResponses {\n\t\t\t\treturn fmt.Errorf(\"Did not receive correct number of messages: %d vs %d\", nmsgs, expectedResponses)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\tif nre := accA.NumPendingAllResponses(); nre != 0 {\n\t\tt.Fatalf(\"Expected no entries, got %d\", nre)\n\t}\n\tif nre := accB.NumPendingReverseResponses(); nre != 0 {\n\t\tt.Fatalf(\"Expected no entries, got %d\", nre)\n\t}\n}\n\nfunc TestServiceExportsResponseFiltering(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tserver_name: A\n\t\tlisten: 127.0.0.1:-1\n\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd} ]\n\t\t exports: [ {service: \"foo\"}, {service: \"bar\"} ]\n\t\t },\n\t\t B: {\n\t\t users: [{user: b, password: pwd} ]\n\t\t\t imports: [ {service: { account: A, subject: \"foo\"}}, {service: { account: A, subject: \"bar\"}, to: \"baz\"} ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// If we do not subscribe the system is now smart enough to not setup the response service imports.\n\tnc.SubscribeSync(\"foo\")\n\tnc.SubscribeSync(\"bar\")\n\tnc.Flush()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// We don't expect responses, so just do publishes.\n\t// 5 for foo\n\tsendFoo := 5\n\tfor i := 0; i < sendFoo; i++ {\n\t\tnc2.PublishRequest(\"foo\", \"reply\", nil)\n\t}\n\t// 17 for bar\n\tsendBar := 17\n\tfor i := 0; i < sendBar; i++ {\n\t\tnc2.PublishRequest(\"baz\", \"reply\", nil)\n\t}\n\tnc2.Flush()\n\n\taccA, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\tsendTotal := sendFoo + sendBar\n\tif nre := accA.NumPendingAllResponses(); nre != sendTotal {\n\t\tt.Fatalf(\"Expected %d entries, got %d\", sendTotal, nre)\n\t}\n\n\tif nre := accA.NumPendingResponses(\"foo\"); nre != sendFoo {\n\t\tt.Fatalf(\"Expected %d entries, got %d\", sendFoo, nre)\n\t}\n\n\tif nre := accA.NumPendingResponses(\"bar\"); nre != sendBar {\n\t\tt.Fatalf(\"Expected %d entries, got %d\", sendBar, nre)\n\t}\n}\n\nfunc TestServiceExportsAutoDirectCleanup(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd} ]\n\t\t exports: [ {service: \"foo\"} ]\n\t\t },\n\t\t B: {\n\t\t users: [{user: b, password: pwd} ]\n\t\t\t imports: [ {service: { account: A, subject: \"foo\"}} ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\tacc, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\t// Potential resonder.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\texpectNone := func() {\n\t\tt.Helper()\n\t\tif nre := acc.NumPendingAllResponses(); nre != 0 {\n\t\t\tt.Fatalf(\"Expected no entries, got %d\", nre)\n\t\t}\n\t}\n\n\ttoSend := 10\n\n\t// With no responders we should never register service import responses etc.\n\tfor i := 0; i < toSend; i++ {\n\t\tnc2.PublishRequest(\"foo\", \"reply\", nil)\n\t}\n\tnc2.Flush()\n\texpectNone()\n\n\t// Now register a responder.\n\tsub, _ := nc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tmsg.Respond([]byte(\"world\"))\n\t})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\t// With no reply subject on a request we should never register service import responses etc.\n\tfor i := 0; i < toSend; i++ {\n\t\tnc2.Publish(\"foo\", nil)\n\t}\n\tnc2.Flush()\n\texpectNone()\n\n\t// Create an old request style client.\n\tnc3, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port), nats.UseOldRequestStyle())\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc3.Close()\n\n\t// If the request loses interest before the response we should not queue up service import responses either.\n\t// This only works for old style requests at the moment where we can detect interest going away.\n\tdelay := 25 * time.Millisecond\n\tsub.Unsubscribe()\n\tsub, _ = nc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\ttime.Sleep(delay)\n\t\tmsg.Respond([]byte(\"world\"))\n\t})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\tfor i := 0; i < toSend; i++ {\n\t\tnc3.Request(\"foo\", nil, time.Millisecond)\n\t}\n\tnc3.Flush()\n\ttime.Sleep(time.Duration(toSend) * delay * 2)\n\texpectNone()\n}\n\n// In some instances we do not have a forceful trigger that signals us to clean up.\n// Like a stream that does not send EOF or a responder who receives requests but does\n// not answer. For these we will have an expectation of a response threshold which\n// tells the system we should have seen a response by T, say 2 minutes, 30 seconds etc.\nfunc TestServiceExportsPruningCleanup(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd} ]\n\t\t exports: [ {service: \"foo\", response: stream} ]\n\t\t },\n\t\t B: {\n\t\t users: [{user: b, password: pwd} ]\n\t\t\t imports: [ {service: { account: A, subject: \"foo\"}} ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Potential resonder.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// We will subscribe but not answer.\n\tsub, _ := nc.Subscribe(\"foo\", func(msg *nats.Msg) {})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\tacc, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\t// Check on response thresholds.\n\trt, err := acc.ServiceExportResponseThreshold(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error retrieving response threshold, %v\", err)\n\t}\n\n\tif rt != server.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD {\n\t\tt.Fatalf(\"Expected the response threshold to be %v, got %v\",\n\t\t\tserver.DEFAULT_SERVICE_EXPORT_RESPONSE_THRESHOLD, rt)\n\t}\n\t// now set it\n\tnewRt := 10 * time.Millisecond\n\tif err := acc.SetServiceExportResponseThreshold(\"foo\", newRt); err != nil {\n\t\tt.Fatalf(\"Expected no error setting response threshold, got %v\", err)\n\t}\n\n\texpectedPending := func(expected int) {\n\t\tt.Helper()\n\t\t// Caller is sleeping a bit before, but avoid flappers.\n\t\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\t\tif nre := acc.NumPendingResponses(\"foo\"); nre != expected {\n\t\t\t\treturn fmt.Errorf(\"Expected %d entries, got %d\", expected, nre)\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}\n\n\t// Requestor\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\ttoSend := 10\n\n\t// This should register and they will dangle. Make sure we clean them up.\n\tfor i := 0; i < toSend; i++ {\n\t\tnc2.PublishRequest(\"foo\", \"reply\", nil)\n\t}\n\tnc2.Flush()\n\n\texpectedPending(10)\n\ttime.Sleep(4 * newRt)\n\texpectedPending(0)\n\n\t// Do it again.\n\tfor i := 0; i < toSend; i++ {\n\t\tnc2.PublishRequest(\"foo\", \"reply\", nil)\n\t}\n\tnc2.Flush()\n\n\texpectedPending(10)\n\ttime.Sleep(4 * newRt)\n\texpectedPending(0)\n}\n\nfunc TestServiceExportsResponseThreshold(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd} ]\n\t\t exports: [ {service: \"foo\", response: stream, threshold: \"1s\"} ]\n\t\t },\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Potential responder.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// We will subscribe but not answer.\n\tsub, _ := nc.Subscribe(\"foo\", func(msg *nats.Msg) {})\n\tnc.Flush()\n\tdefer sub.Unsubscribe()\n\n\tacc, err := srv.LookupAccount(\"A\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error looking up account: %v\", err)\n\t}\n\n\t// Check on response thresholds.\n\trt, err := acc.ServiceExportResponseThreshold(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error retrieving response threshold, %v\", err)\n\t}\n\tif rt != 1*time.Second {\n\t\tt.Fatalf(\"Expected response threshold to be %v, got %v\", 1*time.Second, rt)\n\t}\n}\n\nfunc TestServiceExportsResponseThresholdChunked(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd} ]\n\t\t exports: [ {service: \"foo\", response: chunked, threshold: \"10ms\"} ]\n\t\t },\n\t\t B: {\n\t\t users: [{user: b, password: pwd} ]\n\t\t\t imports: [ {service: { account: A, subject: \"foo\"}} ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\tnumChunks := 10\n\n\t// Respond with 5ms gaps for total response time for all chunks and EOF > 50ms.\n\tnc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\t// Streamed response.\n\t\tfor i := 1; i <= numChunks; i++ {\n\t\t\ttime.Sleep(5 * time.Millisecond)\n\t\t\tmsg.Respond([]byte(fmt.Sprintf(\"chunk-%d\", i)))\n\t\t}\n\t\tmsg.Respond(nil)\n\t})\n\tnc.Flush()\n\n\t// Now setup requester.\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Create an inbox\n\treply := nats.NewInbox()\n\tsub, _ := nc2.SubscribeSync(reply)\n\tdefer sub.Unsubscribe()\n\n\tif err := nc2.PublishRequest(\"foo\", reply, nil); err != nil {\n\t\tt.Fatalf(\"Error sending request: %v\", err)\n\t}\n\n\tcheckFor(t, 250*time.Millisecond, 10*time.Millisecond, func() error {\n\t\tif nmsgs, _, _ := sub.Pending(); err != nil || nmsgs != numChunks+1 {\n\t\t\treturn fmt.Errorf(\"Did not receive correct number of chunks: %d vs %d\", nmsgs, numChunks+1)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc TestServiceAllowResponsesPerms(t *testing.T) {\n\tconf := createConfFile(t, []byte(`\n\t\tlisten: 127.0.0.1:-1\n\t\taccounts: {\n\t\t A: {\n\t\t users: [ {user: a, password: pwd, permissions = {subscribe=foo, allow_responses=true}} ]\n\t\t exports: [ {service: \"foo\"} ]\n\t\t },\n\t\t B: {\n\t\t users: [{user: b, password: pwd} ]\n\t\t\t imports: [ {service: { account: A, subject: \"foo\"}} ]\n\t\t }\n\t\t}\n\t`))\n\n\tsrv, opts := RunServerWithConfig(conf)\n\tdefer srv.Shutdown()\n\n\t// Responder.\n\tnc, err := nats.Connect(fmt.Sprintf(\"nats://a:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\treply := []byte(\"Hello\")\n\t// Respond with 5ms gaps for total response time for all chunks and EOF > 50ms.\n\tnc.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tmsg.Respond(reply)\n\t})\n\tnc.Flush()\n\n\t// Now setup requester.\n\tnc2, err := nats.Connect(fmt.Sprintf(\"nats://b:pwd@%s:%d\", opts.Host, opts.Port))\n\tif err != nil {\n\t\tt.Fatalf(\"Error on connect: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tresp, err := nc2.Request(\"foo\", []byte(\"help\"), time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error expecting response %v\", err)\n\t}\n\tif !bytes.Equal(resp.Data, reply) {\n\t\tt.Fatalf(\"Did not get correct response, %q vs %q\", resp.Data, reply)\n\t}\n}\n" }, { "path": "test/system_services_test.go", "content": "// Copyright 2019-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"github.com/nats-io/nats.go\"\n)\n\nconst dbgSubs = \"$SYS.DEBUG.SUBSCRIBERS\"\n\nfunc (sc *supercluster) selectRandomServer() *server.Options {\n\tci := rand.Int31n(int32(len(sc.clusters)))\n\tsi := rand.Int31n(int32(len(sc.clusters[ci].servers)))\n\treturn sc.clusters[ci].opts[si]\n}\n\nfunc (sc *supercluster) setupSystemServicesImports(t *testing.T, account string) {\n\tt.Helper()\n\tfor _, c := range sc.clusters {\n\t\tfor _, s := range c.servers {\n\t\t\tsysAcc := s.SystemAccount()\n\t\t\tif sysAcc == nil {\n\t\t\t\tt.Fatalf(\"System account not set\")\n\t\t\t}\n\t\t\tacc, err := s.LookupAccount(account)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error looking up account '%s': %v\", account, err)\n\t\t\t}\n\t\t\tif err := acc.AddServiceImport(sysAcc, dbgSubs, dbgSubs); err != nil {\n\t\t\t\tt.Fatalf(\"Error adding subscribers debug service to '%s': %v\", account, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc numSubs(t *testing.T, msg *nats.Msg) int {\n\tt.Helper()\n\tif msg == nil || msg.Data == nil {\n\t\tt.Fatalf(\"No response\")\n\t}\n\tn, err := strconv.Atoi(string(msg.Data))\n\tif err != nil {\n\t\tt.Fatalf(\"Got non-number response: %v\", err)\n\t}\n\treturn n\n}\n\nfunc checkDbgNumSubs(t *testing.T, nc *nats.Conn, subj string, expected int) {\n\tt.Helper()\n\tvar n int\n\tfor i := 0; i < 3; i++ {\n\t\tresponse, err := nc.Request(dbgSubs, []byte(subj), 250*time.Millisecond)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif n = numSubs(t, response); n == expected {\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatalf(\"Expected %d subscribers, got %d\", expected, n)\n}\n\nfunc TestSystemServiceSubscribers(t *testing.T) {\n\tnumServers, numClusters := 3, 3\n\tsc := createSuperCluster(t, numServers, numClusters)\n\tdefer sc.shutdown()\n\n\tsc.setupSystemServicesImports(t, \"FOO\")\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tcheckInterest := func(expected int) {\n\t\tt.Helper()\n\t\tcheckDbgNumSubs(t, nc, \"foo.bar\", expected)\n\t}\n\n\tcheckInterest(0)\n\n\t// Now add in local subscribers.\n\tfor i := 0; i < 5; i++ {\n\t\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(\"foo.bar\")\n\t\tnc.SubscribeSync(\"foo.*\")\n\t\tnc.Flush()\n\t}\n\n\tcheckInterest(10)\n\n\t// Now create remote subscribers at random.\n\tfor i := 0; i < 90; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(\"foo.bar\")\n\t\tnc.Flush()\n\t}\n\n\tcheckInterest(100)\n}\n\n// Test that we can match wildcards. So sub may be foo.bar and we ask about foo.*, that should work.\nfunc TestSystemServiceSubscribersWildcards(t *testing.T) {\n\tnumServers, numClusters := 3, 3\n\tsc := createSuperCluster(t, numServers, numClusters)\n\tdefer sc.shutdown()\n\n\tsc.setupSystemServicesImports(t, \"FOO\")\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tfor i := 0; i < 50; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(fmt.Sprintf(\"foo.bar.%d\", i+1))\n\t\tnc.SubscribeSync(fmt.Sprintf(\"%d\", i+1))\n\t\tnc.Flush()\n\t}\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.*\", 50)\n\n\tcheckDbgNumSubs(t, nc, \"foo.>\", 50)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.22\", 1)\n\n\tresponse, _ := nc.Request(dbgSubs, []byte(\"_INBOX.*.*\"), time.Second)\n\thasInbox := numSubs(t, response)\n\n\tcheckDbgNumSubs(t, nc, \">\", 100+hasInbox)\n}\n\n// Test that we can match on queue groups as well. Separate request payload with any whitespace.\nfunc TestSystemServiceSubscribersQueueGroups(t *testing.T) {\n\tnumServers, numClusters := 3, 3\n\tsc := createSuperCluster(t, numServers, numClusters)\n\tdefer sc.shutdown()\n\n\tsc.setupSystemServicesImports(t, \"FOO\")\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tsubj := fmt.Sprintf(\"foo.bar.%d\", i+1)\n\t\tnc.QueueSubscribeSync(subj, \"QG.11\")\n\t\tnc.QueueSubscribeSync(\"foo.baz\", \"QG.33\")\n\t\tnc.Flush()\n\t}\n\n\tfor i := 0; i < 23; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tsubj := fmt.Sprintf(\"foo.bar.%d\", i+1)\n\t\tnc.QueueSubscribeSync(subj, \"QG.22\")\n\t\tnc.QueueSubscribeSync(\"foo.baz\", \"QG.22\")\n\t\tnc.Flush()\n\t}\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.*\", 33)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.22 QG.22\", 1)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.2\", 2)\n\n\tcheckDbgNumSubs(t, nc, \"foo.baz\", 33)\n\n\tcheckDbgNumSubs(t, nc, \"foo.baz QG.22\", 23)\n\n\t// Now check qfilters work on wildcards too.\n\tcheckDbgNumSubs(t, nc, \"foo.bar.> QG.11\", 10)\n\n\tcheckDbgNumSubs(t, nc, \"*.baz QG.22\", 23)\n\n\tcheckDbgNumSubs(t, nc, \"foo.*.22 QG.22\", 1)\n}\n\nfunc TestSystemServiceSubscribersLeafNodesWithoutSystem(t *testing.T) {\n\tnumServers, numClusters := 3, 3\n\tsc := createSuperCluster(t, numServers, numClusters)\n\tdefer sc.shutdown()\n\n\tsc.setupSystemServicesImports(t, \"FOO\")\n\n\tci := rand.Int31n(int32(len(sc.clusters)))\n\tsi := rand.Int31n(int32(len(sc.clusters[ci].servers)))\n\ts, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si]\n\turl := fmt.Sprintf(\"nats://%s:pass@%s:%d\", \"foo\", opts.Host, opts.LeafNode.Port)\n\tls, lopts := runSolicitLeafServerToURL(url)\n\tdefer ls.Shutdown()\n\n\tcheckLeafNodeConnected(t, s)\n\n\t// This is so we can test when the subs on a leafnode are flushed to the connected supercluster.\n\tfsubj := \"__leaf.flush__\"\n\tfc := clientConnect(t, opts, \"foo\")\n\tdefer fc.Close()\n\tfc.Subscribe(fsubj, func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\n\tlnc := clientConnect(t, lopts, \"$G\")\n\tdefer lnc.Close()\n\n\tflushLeaf := func() {\n\t\tif _, err := lnc.Request(fsubj, nil, time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not flush through to the supercluster: %v\", err)\n\t\t}\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(fmt.Sprintf(\"foo.bar.%d\", i+1))\n\t\tnc.QueueSubscribeSync(\"foo.bar.baz\", \"QG.22\")\n\t\tnc.Flush()\n\t}\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.*\", 20)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.3\", 1)\n\n\tlnc.SubscribeSync(\"foo.bar.3\")\n\tlnc.QueueSubscribeSync(\"foo.bar.baz\", \"QG.22\")\n\n\t// We could flush here but that does not guarantee we have flushed through to the supercluster.\n\tflushLeaf()\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.3\", 2)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.baz QG.22\", 11)\n\n\tlnc.SubscribeSync(\"foo.bar.3\")\n\tlnc.QueueSubscribeSync(\"foo.bar.baz\", \"QG.22\")\n\tflushLeaf()\n\n\t// For now we do not see all the details behind a leafnode if the leafnode is not enabled.\n\tcheckDbgNumSubs(t, nc, \"foo.bar.3\", 2)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.baz QG.22\", 12)\n}\n\nfunc runSolicitLeafServerWithSystemToURL(surl string) (*server.Server, *server.Options) {\n\to := DefaultTestOptions\n\to.Port = -1\n\tfooAcc := server.NewAccount(\"FOO\")\n\to.Accounts = []*server.Account{server.NewAccount(\"$SYS\"), fooAcc}\n\to.SystemAccount = \"$SYS\"\n\to.Users = []*server.User{\n\t\t{Username: \"foo\", Password: \"pass\", Permissions: nil, Account: fooAcc},\n\t}\n\trurl, _ := url.Parse(surl)\n\tsysUrl, _ := url.Parse(strings.Replace(surl, rurl.User.Username(), \"sys\", -1))\n\to.LeafNode.Remotes = []*server.RemoteLeafOpts{\n\t\t{\n\t\t\tURLs: []*url.URL{rurl},\n\t\t\tLocalAccount: \"FOO\",\n\t\t},\n\t\t{\n\t\t\tURLs: []*url.URL{sysUrl},\n\t\t\tLocalAccount: \"$SYS\",\n\t\t},\n\t}\n\to.LeafNode.ReconnectInterval = 100 * time.Millisecond\n\treturn RunServer(&o), &o\n}\n\nfunc TestSystemServiceSubscribersLeafNodesWithSystem(t *testing.T) {\n\tnumServers, numClusters := 3, 3\n\tsc := createSuperCluster(t, numServers, numClusters)\n\tdefer sc.shutdown()\n\n\tsc.setupSystemServicesImports(t, \"FOO\")\n\n\tci := rand.Int31n(int32(len(sc.clusters)))\n\tsi := rand.Int31n(int32(len(sc.clusters[ci].servers)))\n\ts, opts := sc.clusters[ci].servers[si], sc.clusters[ci].opts[si]\n\turl := fmt.Sprintf(\"nats://%s:pass@%s:%d\", \"foo\", opts.Host, opts.LeafNode.Port)\n\tls, lopts := runSolicitLeafServerWithSystemToURL(url)\n\tdefer ls.Shutdown()\n\n\tcheckFor(t, 5*time.Second, 100*time.Millisecond, func() error {\n\t\tif nln := s.NumLeafNodes(); nln != 2 {\n\t\t\treturn fmt.Errorf(\"Expected a connected leafnode for server %q, got none\", s.ID())\n\t\t}\n\t\treturn nil\n\t})\n\n\t// This is so we can test when the subs on a leafnode are flushed to the connected supercluster.\n\tfsubj := \"__leaf.flush__\"\n\tfc := clientConnect(t, opts, \"foo\")\n\tdefer fc.Close()\n\tfc.Subscribe(fsubj, func(m *nats.Msg) {\n\t\tm.Respond(nil)\n\t})\n\n\tlnc := clientConnect(t, lopts, \"foo\")\n\tdefer lnc.Close()\n\n\tflushLeaf := func() {\n\t\tif _, err := lnc.Request(fsubj, nil, time.Second); err != nil {\n\t\t\tt.Fatalf(\"Did not flush through to the supercluster: %v\", err)\n\t\t}\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tnc := clientConnect(t, sc.selectRandomServer(), \"foo\")\n\t\tdefer nc.Close()\n\t\tnc.SubscribeSync(fmt.Sprintf(\"foo.bar.%d\", i+1))\n\t\tnc.QueueSubscribeSync(\"foo.bar.baz\", \"QG.22\")\n\t\tnc.Flush()\n\t}\n\n\tnc := clientConnect(t, sc.clusters[0].opts[0], \"foo\")\n\tdefer nc.Close()\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.3\", 1)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.*\", 20)\n\n\tlnc.SubscribeSync(\"foo.bar.3\")\n\tlnc.QueueSubscribeSync(\"foo.bar.baz\", \"QG.22\")\n\tflushLeaf()\n\n\t// Since we are doing real tracking now on the other side, this will be off by 1 since we are counting\n\t// the leaf and the real sub.\n\tcheckDbgNumSubs(t, nc, \"foo.bar.3\", 3)\n\n\tcheckDbgNumSubs(t, nc, \"foo.bar.baz QG.22\", 12)\n}\n" }, { "path": "test/test.go", "content": "// Copyright 2012-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"crypto/rand\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"os\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\n\tsrvlog \"github.com/nats-io/nats-server/v2/logger\"\n)\n\n// So we can pass tests and benchmarks..\ntype tLogger interface {\n\tFatalf(format string, args ...any)\n\tErrorf(format string, args ...any)\n}\n\n// DefaultTestOptions are default options for the unit tests.\nvar DefaultTestOptions = server.Options{\n\tHost: \"127.0.0.1\",\n\tPort: 4222,\n\tNoLog: true,\n\tNoSigs: true,\n\tMaxControlLine: 4096,\n\tDisableShortFirstPing: true,\n}\n\n// RunDefaultServer starts a new Go routine based server using the default options\nfunc RunDefaultServer() *server.Server {\n\treturn RunServer(&DefaultTestOptions)\n}\n\nfunc RunRandClientPortServer() *server.Server {\n\topts := DefaultTestOptions\n\topts.Port = -1\n\treturn RunServer(&opts)\n}\n\n// To turn on server tracing and debugging and logging which are\n// normally suppressed.\nvar (\n\tdoLog = false\n\tdoTrace = false\n\tdoDebug = false\n)\n\n// RunServer starts a new Go routine based server\nfunc RunServer(opts *server.Options) *server.Server {\n\treturn RunServerCallback(opts, nil)\n}\n\nfunc RunServerCallback(opts *server.Options, callback func(*server.Server)) *server.Server {\n\tif opts == nil {\n\t\topts = &DefaultTestOptions\n\t}\n\t// Optionally override for individual debugging of tests\n\topts.NoLog = !doLog\n\topts.Trace = doTrace\n\topts.Debug = doDebug\n\t// For all tests in the \"test\" package, we will disable route pooling.\n\topts.Cluster.PoolSize = -1\n\t// Also disable compression for \"test\" package.\n\topts.Cluster.Compression.Mode = server.CompressionOff\n\topts.LeafNode.Compression.Mode = server.CompressionOff\n\n\ts, err := server.NewServer(opts)\n\tif err != nil || s == nil {\n\t\tpanic(fmt.Sprintf(\"No NATS Server object returned: %v\", err))\n\t}\n\n\tif doLog {\n\t\ts.ConfigureLogger()\n\t}\n\n\tif ll := os.Getenv(\"NATS_LOGGING\"); ll != \"\" {\n\t\tlog := srvlog.NewTestLogger(fmt.Sprintf(\"[%s] | \", s), true)\n\t\tdebug := ll == \"debug\" || ll == \"trace\"\n\t\ttrace := ll == \"trace\"\n\t\ts.SetLoggerV2(log, debug, trace, false)\n\t}\n\n\tif callback != nil {\n\t\tcallback(s)\n\t}\n\n\t// Run server in Go routine.\n\tgo s.Start()\n\n\t// Wait for accept loop(s) to be started\n\tif !s.ReadyForConnections(10 * time.Second) {\n\t\tpanic(\"Unable to start NATS Server in Go Routine\")\n\t}\n\treturn s\n}\n\n// LoadConfig loads a configuration from a filename\nfunc LoadConfig(configFile string) *server.Options {\n\topts, err := server.ProcessConfigFile(configFile)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Error processing configuration file: %v\", err))\n\t}\n\treturn opts\n}\n\n// RunServerWithConfig starts a new Go routine based server with a configuration file.\nfunc RunServerWithConfig(configFile string) (srv *server.Server, opts *server.Options) {\n\topts = LoadConfig(configFile)\n\tsrv = RunServer(opts)\n\treturn\n}\n\n// RunServerWithConfigOverrides starts a new Go routine based server with a configuration file,\n// providing a callback to update the options configured.\nfunc RunServerWithConfigOverrides(configFile string, optsCallback func(*server.Options), svrCallback func(*server.Server)) (srv *server.Server, opts *server.Options) {\n\topts = LoadConfig(configFile)\n\tif optsCallback != nil {\n\t\toptsCallback(opts)\n\t}\n\tsrv = RunServerCallback(opts, svrCallback)\n\treturn\n}\n\nfunc stackFatalf(t tLogger, f string, args ...any) {\n\tlines := make([]string, 0, 32)\n\tmsg := fmt.Sprintf(f, args...)\n\tlines = append(lines, msg)\n\n\t// Ignore ourselves\n\t_, testFile, _, _ := runtime.Caller(0)\n\n\t// Generate the Stack of callers:\n\tfor i := 0; true; i++ {\n\t\t_, file, line, ok := runtime.Caller(i)\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tif file == testFile {\n\t\t\tcontinue\n\t\t}\n\t\tmsg := fmt.Sprintf(\"%d - %s:%d\", i, file, line)\n\t\tlines = append(lines, msg)\n\t}\n\n\tt.Fatalf(\"%s\", strings.Join(lines, \"\\n\"))\n}\n\nfunc acceptRouteConn(t tLogger, host string, timeout time.Duration) net.Conn {\n\tl, e := net.Listen(\"tcp\", host)\n\tif e != nil {\n\t\tstackFatalf(t, \"Error listening for route connection on %v: %v\", host, e)\n\t}\n\tdefer l.Close()\n\n\ttl := l.(*net.TCPListener)\n\ttl.SetDeadline(time.Now().Add(timeout))\n\tconn, err := l.Accept()\n\ttl.SetDeadline(time.Time{})\n\n\tif err != nil {\n\t\tstackFatalf(t, \"Did not receive a route connection request: %v\", err)\n\t}\n\treturn conn\n}\n\nfunc createRouteConn(t tLogger, host string, port int) net.Conn {\n\treturn createClientConn(t, host, port)\n}\n\nfunc createClientConn(t tLogger, host string, port int) net.Conn {\n\taddr := net.JoinHostPort(host, fmt.Sprintf(\"%d\", port))\n\tc, err := net.DialTimeout(\"tcp\", addr, 3*time.Second)\n\tif err != nil {\n\t\tstackFatalf(t, \"Could not connect to server: %v\\n\", err)\n\t}\n\treturn c\n}\n\nfunc checkSocket(t tLogger, addr string, wait time.Duration) {\n\tend := time.Now().Add(wait)\n\tfor time.Now().Before(end) {\n\t\tconn, err := net.Dial(\"tcp\", addr)\n\t\tif err != nil {\n\t\t\t// Retry after 50ms\n\t\t\ttime.Sleep(50 * time.Millisecond)\n\t\t\tcontinue\n\t\t}\n\t\tconn.Close()\n\t\t// Wait a bit to give a chance to the server to remove this\n\t\t// \"client\" from its state, which may otherwise interfere with\n\t\t// some tests.\n\t\ttime.Sleep(25 * time.Millisecond)\n\t\treturn\n\t}\n\t// We have failed to bind the socket in the time allowed.\n\tt.Fatalf(\"Failed to connect to the socket: %q\", addr)\n}\n\nfunc checkInfoMsg(t tLogger, c net.Conn) server.Info {\n\tbuf := expectResult(t, c, infoRe)\n\tjs := infoRe.FindAllSubmatch(buf, 1)[0][1]\n\tvar sinfo server.Info\n\terr := json.Unmarshal(js, &sinfo)\n\tif err != nil {\n\t\tstackFatalf(t, \"Could not unmarshal INFO json: %v\\n\", err)\n\t}\n\treturn sinfo\n}\n\nfunc doHeadersConnect(t tLogger, c net.Conn, verbose, pedantic, ssl, headers bool) {\n\tcheckInfoMsg(t, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"headers\\\":%v}\\r\\n\",\n\t\tverbose, pedantic, ssl, headers)\n\tsendProto(t, c, cs)\n}\n\nfunc doConnect(t tLogger, c net.Conn, verbose, pedantic, ssl bool) {\n\tdoHeadersConnect(t, c, verbose, pedantic, ssl, false)\n}\n\nfunc doDefaultHeadersConnect(t tLogger, c net.Conn) {\n\tdoHeadersConnect(t, c, false, false, false, true)\n}\n\nfunc doDefaultConnect(t tLogger, c net.Conn) {\n\t// Basic Connect\n\tdoConnect(t, c, false, false, false)\n}\n\nconst routeConnectProto = \"CONNECT {\\\"verbose\\\":false,\\\"user\\\":\\\"%s\\\",\\\"pass\\\":\\\"%s\\\",\\\"name\\\":\\\"%s\\\",\\\"cluster\\\":\\\"xyz\\\"}\\r\\n\"\n\nfunc doRouteAuthConnect(t tLogger, c net.Conn, user, pass, id string) {\n\tcs := fmt.Sprintf(routeConnectProto, user, pass, id)\n\tsendProto(t, c, cs)\n}\n\nfunc setupRouteEx(t tLogger, c net.Conn, opts *server.Options, id string) (sendFun, expectFun) {\n\tuser := opts.Cluster.Username\n\tpass := opts.Cluster.Password\n\tdoRouteAuthConnect(t, c, user, pass, id)\n\treturn sendCommand(t, c), expectCommand(t, c)\n}\n\nfunc setupRoute(t tLogger, c net.Conn, opts *server.Options) (sendFun, expectFun) {\n\tu := make([]byte, 16)\n\tio.ReadFull(rand.Reader, u)\n\tid := fmt.Sprintf(\"ROUTER:%s\", hex.EncodeToString(u))\n\treturn setupRouteEx(t, c, opts, id)\n}\n\nfunc setupHeaderConn(t tLogger, c net.Conn) (sendFun, expectFun) {\n\tdoDefaultHeadersConnect(t, c)\n\treturn sendCommand(t, c), expectCommand(t, c)\n}\n\nfunc setupConn(t tLogger, c net.Conn) (sendFun, expectFun) {\n\tdoDefaultConnect(t, c)\n\treturn sendCommand(t, c), expectCommand(t, c)\n}\n\nfunc setupConnWithProto(t tLogger, c net.Conn, proto int) (sendFun, expectFun) {\n\tcheckInfoMsg(t, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"protocol\\\":%d}\\r\\n\", false, false, false, proto)\n\tsendProto(t, c, cs)\n\treturn sendCommand(t, c), expectCommand(t, c)\n}\n\nfunc setupConnWithAccount(t tLogger, s *server.Server, c net.Conn, account string) (sendFun, expectFun) {\n\tinfo := checkInfoMsg(t, c)\n\ts.RegisterAccount(account)\n\tacc, err := s.LookupAccount(account)\n\tif err != nil {\n\t\tt.Fatalf(\"Unexpected Error: %v\", err)\n\t}\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v}\\r\\n\", false, false, false)\n\tsendProto(t, c, cs)\n\n\tsend, expect := sendCommand(t, c), expectCommand(t, c)\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n\n\tnc := s.GetClient(info.CID)\n\tif nc == nil {\n\t\tt.Fatalf(\"Could not get client for CID:%d\", info.CID)\n\t}\n\tnc.RegisterUser(&server.User{Account: acc})\n\n\treturn send, expect\n}\n\nfunc setupConnWithUserPass(t tLogger, c net.Conn, username, password string) (sendFun, expectFun) {\n\tcheckInfoMsg(t, c)\n\tcs := fmt.Sprintf(\"CONNECT {\\\"verbose\\\":%v,\\\"pedantic\\\":%v,\\\"tls_required\\\":%v,\\\"user\\\":%q,\\\"pass\\\":%q}\\r\\n\",\n\t\tfalse, false, false, username, password)\n\tsendProto(t, c, cs)\n\treturn sendCommand(t, c), expectLefMostCommand(t, c)\n}\n\ntype sendFun func(string)\ntype expectFun func(*regexp.Regexp) []byte\n\n// Closure version for easier reading\nfunc sendCommand(t tLogger, c net.Conn) sendFun {\n\treturn func(op string) {\n\t\tsendProto(t, c, op)\n\t}\n}\n\n// Closure version for easier reading\nfunc expectCommand(t tLogger, c net.Conn) expectFun {\n\treturn func(re *regexp.Regexp) []byte {\n\t\treturn expectResult(t, c, re)\n\t}\n}\n\n// Closure version for easier reading\nfunc expectLefMostCommand(t tLogger, c net.Conn) expectFun {\n\tvar buf []byte\n\treturn func(re *regexp.Regexp) []byte {\n\t\treturn expectLeftMostResult(t, c, re, &buf)\n\t}\n}\n\n// Send the protocol command to the server.\nfunc sendProto(t tLogger, c net.Conn, op string) {\n\tn, err := c.Write([]byte(op))\n\tif err != nil {\n\t\tstackFatalf(t, \"Error writing command to conn: %v\\n\", err)\n\t}\n\tif n != len(op) {\n\t\tstackFatalf(t, \"Partial write: %d vs %d\\n\", n, len(op))\n\t}\n}\n\nvar (\n\tanyRe = regexp.MustCompile(`.*`)\n\tinfoRe = regexp.MustCompile(`INFO\\s+([^\\r\\n]+)\\r\\n`)\n\tinfoStartRe = regexp.MustCompile(`^INFO\\s+([^\\r\\n]+)\\r\\n`)\n\tpingRe = regexp.MustCompile(`^PING\\r\\n`)\n\tpongRe = regexp.MustCompile(`^PONG\\r\\n`)\n\thmsgRe = regexp.MustCompile(`(?:(?:HMSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\s+(\\d+)\\s*\\r\\n([^\\\\r\\\\n]*?)\\r\\n)+?)`)\n\tmsgRe = regexp.MustCompile(`(?:(?:MSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\s*\\r\\n([^\\\\r\\\\n]*?)\\r\\n)+?)`)\n\trawMsgRe = regexp.MustCompile(`(?:(?:MSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(([^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\s*\\r\\n(.*?)))`)\n\tokRe = regexp.MustCompile(`\\A\\+OK\\r\\n`)\n\terrRe = regexp.MustCompile(`\\A\\-ERR\\s+([^\\r\\n]+)\\r\\n`)\n\tconnectRe = regexp.MustCompile(`CONNECT\\s+([^\\r\\n]+)\\r\\n`)\n\trsubRe = regexp.MustCompile(`RS\\+\\s+([^\\s]+)\\s+([^\\s]+)\\s*([^\\s]+)?\\s*(\\d+)?\\r\\n`)\n\trunsubRe = regexp.MustCompile(`RS\\-\\s+([^\\s]+)\\s+([^\\s]+)\\s*([^\\s]+)?\\r\\n`)\n\trmsgRe = regexp.MustCompile(`(?:(?:RMSG\\s+([^\\s]+)\\s+([^\\s]+)\\s+(?:([|+]\\s+([\\w\\s]+)|[^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\s*\\r\\n([^\\\\r\\\\n]*?)\\r\\n)+?)`)\n\tasubRe = regexp.MustCompile(`A\\+\\s+([^\\r\\n]+)\\r\\n`)\n\taunsubRe = regexp.MustCompile(`A\\-\\s+([^\\r\\n]+)\\r\\n`)\n\tlsubRe = regexp.MustCompile(`LS\\+\\s+([^\\s]+)\\s*([^\\s]+)?\\s*(\\d+)?\\r\\n`)\n\tlunsubRe = regexp.MustCompile(`LS\\-\\s+([^\\s]+)\\s*([^\\s]+)\\s*([^\\s]+)?\\r\\n`)\n\tlmsgRe = regexp.MustCompile(`(?:(?:LMSG\\s+([^\\s]+)\\s+(?:([|+]\\s+([\\w\\s]+)|[^\\s]+)[^\\S\\r\\n]+)?(\\d+)\\s*\\r\\n([^\\\\r\\\\n]*?)\\r\\n)+?)`)\n\trlsubRe = regexp.MustCompile(`LS\\+\\s+([^\\s]+)\\s+([^\\s]+)\\s+([^\\s]+)\\s*([^\\s]+)?\\s*(\\d+)?\\r\\n`)\n\trlunsubRe = regexp.MustCompile(`LS\\-\\s+([^\\s]+)\\s+([^\\s]+)\\s+([^\\s]+)\\s*([^\\s]+)?\\r\\n`)\n)\n\nconst (\n\t// Regular Messages\n\tsubIndex = 1\n\tsidIndex = 2\n\treplyIndex = 4\n\tlenIndex = 5\n\tmsgIndex = 6\n\t// Headers\n\thlenIndex = 5\n\ttlenIndex = 6\n\thmsgIndex = 7\n\n\t// Routed Messages\n\taccIndex = 1\n\trsubIndex = 2\n\treplyAndQueueIndex = 3\n)\n\n// Test result from server against regexp and return left most match\nfunc expectLeftMostResult(t tLogger, c net.Conn, re *regexp.Regexp, buf *[]byte) []byte {\n\trecv := func() []byte {\n\t\texpBuf := make([]byte, 32768)\n\t\t// Wait for commands to be processed and results queued for read\n\t\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\t\tn, err := c.Read(expBuf)\n\t\tc.SetReadDeadline(time.Time{})\n\n\t\tif n <= 0 && err != nil {\n\t\t\tstackFatalf(t, \"Error reading from conn: %v\\n\", err)\n\t\t}\n\t\treturn expBuf[:n]\n\t}\n\tif len(*buf) == 0 {\n\t\t*buf = recv()\n\t}\n\temptyCnt := 0\n\tfor {\n\t\tresult := re.Find(*buf)\n\t\tif result == nil {\n\t\t\temptyCnt++\n\t\t\tif emptyCnt > 5 {\n\t\t\t\tstackFatalf(t, \"Reading empty data too often\\n\")\n\t\t\t}\n\t\t\t*buf = append(*buf, recv()...)\n\t\t} else {\n\t\t\tcutIdx := strings.Index(string(*buf), string(result)) + len(result)\n\t\t\t*buf = (*buf)[cutIdx:]\n\t\t\treturn result\n\t\t}\n\t}\n}\n\n// Test result from server against regexp\nfunc expectResult(t tLogger, c net.Conn, re *regexp.Regexp) []byte {\n\texpBuf := make([]byte, 32768)\n\t// Wait for commands to be processed and results queued for read\n\tc.SetReadDeadline(time.Now().Add(2 * time.Second))\n\tn, err := c.Read(expBuf)\n\tc.SetReadDeadline(time.Time{})\n\n\tif n <= 0 && err != nil {\n\t\tstackFatalf(t, \"Error reading from conn: %v\\n\", err)\n\t}\n\tbuf := expBuf[:n]\n\tif !re.Match(buf) {\n\t\tstackFatalf(t, \"Response did not match expected: \\n\\tReceived:'%q'\\n\\tExpected:'%s'\", buf, re)\n\t}\n\treturn buf\n}\n\nfunc peek(c net.Conn) []byte {\n\texpBuf := make([]byte, 32768)\n\tc.SetReadDeadline(time.Now().Add(50 * time.Millisecond))\n\tn, err := c.Read(expBuf)\n\tc.SetReadDeadline(time.Time{})\n\tif err != nil || n <= 0 {\n\t\treturn nil\n\t}\n\treturn expBuf\n}\n\nfunc expectDisconnect(t *testing.T, c net.Conn) {\n\tt.Helper()\n\tvar b [8]byte\n\tc.SetReadDeadline(time.Now().Add(200 * time.Millisecond))\n\t_, err := c.Read(b[:])\n\tc.SetReadDeadline(time.Time{})\n\tif err != io.EOF {\n\t\tt.Fatalf(\"Expected a disconnect\")\n\t}\n}\n\nfunc expectNothing(t tLogger, c net.Conn) {\n\texpectNothingTimeout(t, c, time.Now().Add(100*time.Millisecond))\n}\n\nfunc expectNothingTimeout(t tLogger, c net.Conn, dl time.Time) {\n\texpBuf := make([]byte, 32)\n\tc.SetReadDeadline(dl)\n\tn, err := c.Read(expBuf)\n\tc.SetReadDeadline(time.Time{})\n\tif err == nil && n > 0 {\n\t\tstackFatalf(t, \"Expected nothing, received: '%q'\\n\", expBuf[:n])\n\t}\n}\n\n// This will check that we got what we expected from a normal message.\nfunc checkMsg(t tLogger, m [][]byte, subject, sid, reply, len, msg string) {\n\tif string(m[subIndex]) != subject {\n\t\tstackFatalf(t, \"Did not get correct subject: expected '%s' got '%s'\\n\", subject, m[subIndex])\n\t}\n\tif sid != \"\" && string(m[sidIndex]) != sid {\n\t\tstackFatalf(t, \"Did not get correct sid: expected '%s' got '%s'\\n\", sid, m[sidIndex])\n\t}\n\tif string(m[replyIndex]) != reply {\n\t\tstackFatalf(t, \"Did not get correct reply: expected '%s' got '%s'\\n\", reply, m[replyIndex])\n\t}\n\tif string(m[lenIndex]) != len {\n\t\tstackFatalf(t, \"Did not get correct msg length: expected '%s' got '%s'\\n\", len, m[lenIndex])\n\t}\n\tif string(m[msgIndex]) != msg {\n\t\tstackFatalf(t, \"Did not get correct msg: expected '%s' got '%s'\\n\", msg, m[msgIndex])\n\t}\n}\n\nfunc checkRmsg(t tLogger, m [][]byte, account, subject, replyAndQueues, len, msg string) {\n\tif string(m[accIndex]) != account {\n\t\tstackFatalf(t, \"Did not get correct account: expected '%s' got '%s'\\n\", account, m[accIndex])\n\t}\n\tif string(m[rsubIndex]) != subject {\n\t\tstackFatalf(t, \"Did not get correct subject: expected '%s' got '%s'\\n\", subject, m[rsubIndex])\n\t}\n\tif string(m[lenIndex]) != len {\n\t\tstackFatalf(t, \"Did not get correct msg length: expected '%s' got '%s'\\n\", len, m[lenIndex])\n\t}\n\tif string(m[replyAndQueueIndex]) != replyAndQueues {\n\t\tstackFatalf(t, \"Did not get correct reply/queues: expected '%s' got '%s'\\n\", replyAndQueues, m[replyAndQueueIndex])\n\t}\n}\n\nfunc checkLmsg(t tLogger, m [][]byte, subject, replyAndQueues, len, msg string) {\n\tif string(m[rsubIndex-1]) != subject {\n\t\tstackFatalf(t, \"Did not get correct subject: expected '%s' got '%s'\\n\", subject, m[rsubIndex-1])\n\t}\n\tif string(m[lenIndex-1]) != len {\n\t\tstackFatalf(t, \"Did not get correct msg length: expected '%s' got '%s'\\n\", len, m[lenIndex-1])\n\t}\n\tif string(m[replyAndQueueIndex-1]) != replyAndQueues {\n\t\tstackFatalf(t, \"Did not get correct reply/queues: expected '%s' got '%s'\\n\", replyAndQueues, m[replyAndQueueIndex-1])\n\t}\n}\n\n// This will check that we got what we expected from a header message.\nfunc checkHmsg(t tLogger, m [][]byte, subject, sid, reply, hlen, len, hdr, msg string) {\n\tif string(m[subIndex]) != subject {\n\t\tstackFatalf(t, \"Did not get correct subject: expected '%s' got '%s'\\n\", subject, m[subIndex])\n\t}\n\tif sid != \"\" && string(m[sidIndex]) != sid {\n\t\tstackFatalf(t, \"Did not get correct sid: expected '%s' got '%s'\\n\", sid, m[sidIndex])\n\t}\n\tif string(m[replyIndex]) != reply {\n\t\tstackFatalf(t, \"Did not get correct reply: expected '%s' got '%s'\\n\", reply, m[replyIndex])\n\t}\n\tif string(m[hlenIndex]) != hlen {\n\t\tstackFatalf(t, \"Did not get correct header length: expected '%s' got '%s'\\n\", hlen, m[hlenIndex])\n\t}\n\tif string(m[tlenIndex]) != len {\n\t\tstackFatalf(t, \"Did not get correct msg length: expected '%s' got '%s'\\n\", len, m[tlenIndex])\n\t}\n\t// Extract the payload and break up the headers and msg.\n\tpayload := string(m[hmsgIndex])\n\thi, _ := strconv.Atoi(hlen)\n\trhdr, rmsg := payload[:hi], payload[hi:]\n\tif rhdr != hdr {\n\t\tstackFatalf(t, \"Did not get correct headers: expected '%s' got '%s'\\n\", hdr, rhdr)\n\t}\n\tif rmsg != msg {\n\t\tstackFatalf(t, \"Did not get correct msg: expected '%s' got '%s'\\n\", msg, rmsg)\n\t}\n}\n\n// Closure for expectMsgs\nfunc expectRmsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte {\n\treturn func(expected int) [][][]byte {\n\t\tbuf := ef(rmsgRe)\n\t\tmatches := rmsgRe.FindAllSubmatch(buf, -1)\n\t\tif len(matches) != expected {\n\t\t\tstackFatalf(t, \"Did not get correct # routed msgs: %d vs %d\\n\", len(matches), expected)\n\t\t}\n\t\treturn matches\n\t}\n}\n\n// Closure for expectHMsgs\nfunc expectHeaderMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte {\n\treturn func(expected int) [][][]byte {\n\t\tbuf := ef(hmsgRe)\n\t\tmatches := hmsgRe.FindAllSubmatch(buf, -1)\n\t\tif len(matches) != expected {\n\t\t\tstackFatalf(t, \"Did not get correct # msgs: %d vs %d\\n\", len(matches), expected)\n\t\t}\n\t\treturn matches\n\t}\n}\n\n// Closure for expectMsgs\nfunc expectMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte {\n\treturn func(expected int) [][][]byte {\n\t\tbuf := ef(msgRe)\n\t\tmatches := msgRe.FindAllSubmatch(buf, -1)\n\t\tif len(matches) != expected {\n\t\t\tstackFatalf(t, \"Did not get correct # msgs: %d vs %d\\n\", len(matches), expected)\n\t\t}\n\t\treturn matches\n\t}\n}\n\n// This will check that the matches include at least one of the sids. Useful for checking\n// that we received messages on a certain queue group.\nfunc checkForQueueSid(t tLogger, matches [][][]byte, sids []string) {\n\tseen := make(map[string]int, len(sids))\n\tfor _, sid := range sids {\n\t\tseen[sid] = 0\n\t}\n\tfor _, m := range matches {\n\t\tsid := string(m[sidIndex])\n\t\tif _, ok := seen[sid]; ok {\n\t\t\tseen[sid]++\n\t\t}\n\t}\n\t// Make sure we only see one and exactly one.\n\ttotal := 0\n\tfor _, n := range seen {\n\t\ttotal += n\n\t}\n\tif total != 1 {\n\t\tstackFatalf(t, \"Did not get a msg for queue sids group: expected 1 got %d\\n\", total)\n\t}\n}\n\n// This will check that the matches include all of the sids. Useful for checking\n// that we received messages on all subscribers.\nfunc checkForPubSids(t tLogger, matches [][][]byte, sids []string) {\n\tseen := make(map[string]int, len(sids))\n\tfor _, sid := range sids {\n\t\tseen[sid] = 0\n\t}\n\tfor _, m := range matches {\n\t\tsid := string(m[sidIndex])\n\t\tif _, ok := seen[sid]; ok {\n\t\t\tseen[sid]++\n\t\t}\n\t}\n\t// Make sure we only see one and exactly one for each sid.\n\tfor sid, n := range seen {\n\t\tif n != 1 {\n\t\t\tstackFatalf(t, \"Did not get a msg for sid[%s]: expected 1 got %d\\n\", sid, n)\n\n\t\t}\n\t}\n}\n\n// Helper function to generate next opts to make sure no port conflicts etc.\nfunc nextServerOpts(opts *server.Options) *server.Options {\n\tnopts := opts.Clone()\n\tnopts.Port++\n\tnopts.Cluster.Port++\n\tnopts.HTTPPort++\n\treturn nopts\n}\n\nfunc createTempFile(t testing.TB, prefix string) *os.File {\n\tt.Helper()\n\tfile, err := os.CreateTemp(t.TempDir(), prefix)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn file\n}\n" }, { "path": "test/test_test.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n\t\"golang.org/x/time/rate\"\n)\n\nfunc checkFor(t testing.TB, totalWait, sleepDur time.Duration, f func() error) {\n\tt.Helper()\n\ttimeout := time.Now().Add(totalWait)\n\tvar err error\n\tfor time.Now().Before(timeout) {\n\t\terr = f()\n\t\tif err == nil {\n\t\t\treturn\n\t\t}\n\t\ttime.Sleep(sleepDur)\n\t}\n\tif err != nil {\n\t\tt.Fatal(err.Error())\n\t}\n}\n\n// Slow Proxy - For introducing RTT and BW constraints.\ntype slowProxy struct {\n\tlistener net.Listener\n\tconns []net.Conn\n\to *server.Options\n\tu string\n}\n\nfunc newSlowProxy(rtt time.Duration, up, down int, opts *server.Options) *slowProxy {\n\tsaddr := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))\n\thp := net.JoinHostPort(\"127.0.0.1\", \"0\")\n\tl, e := net.Listen(\"tcp\", hp)\n\tif e != nil {\n\t\tpanic(fmt.Sprintf(\"Error listening on port: %s, %q\", hp, e))\n\t}\n\tport := l.Addr().(*net.TCPAddr).Port\n\tsp := &slowProxy{listener: l}\n\tgo func() {\n\t\tclient, err := l.Accept()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tserver, err := net.DialTimeout(\"tcp\", saddr, time.Second)\n\t\tif err != nil {\n\t\t\tpanic(\"Can't connect to server\")\n\t\t}\n\t\tsp.conns = append(sp.conns, client, server)\n\t\tgo sp.loop(rtt, up, client, server)\n\t\tgo sp.loop(rtt, down, server, client)\n\t}()\n\tsp.o = &server.Options{Host: \"127.0.0.1\", Port: port}\n\tsp.u = fmt.Sprintf(\"nats://%s:%d\", sp.o.Host, sp.o.Port)\n\treturn sp\n}\n\nfunc (sp *slowProxy) opts() *server.Options {\n\treturn sp.o\n}\n\n//lint:ignore U1000 Referenced in norace_test.go\nfunc (sp *slowProxy) clientURL() string {\n\treturn sp.u\n}\n\nfunc (sp *slowProxy) loop(rtt time.Duration, tbw int, r, w net.Conn) {\n\tdelay := rtt / 2\n\tconst rbl = 1024\n\tvar buf [rbl]byte\n\tctx := context.Background()\n\n\trl := rate.NewLimiter(rate.Limit(tbw), rbl)\n\n\tfor fr := true; ; {\n\t\tsr := time.Now()\n\t\tn, err := r.Read(buf[:])\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\t// RTT delays\n\t\tif fr || time.Since(sr) > 2*time.Millisecond {\n\t\t\tfr = false\n\t\t\ttime.Sleep(delay)\n\t\t}\n\t\tif err := rl.WaitN(ctx, n); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif _, err = w.Write(buf[:n]); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (sp *slowProxy) stop() {\n\tif sp.listener != nil {\n\t\tsp.listener.Close()\n\t\tsp.listener = nil\n\t\tfor _, c := range sp.conns {\n\t\t\tc.Close()\n\t\t}\n\t}\n}\n\n// Dummy Logger\ntype dummyLogger struct {\n\tsync.Mutex\n\tmsg string\n}\n\nfunc (d *dummyLogger) Fatalf(format string, args ...any) {\n\td.Lock()\n\td.msg = fmt.Sprintf(format, args...)\n\td.Unlock()\n}\n\nfunc (d *dummyLogger) Errorf(format string, args ...any) {\n}\n\nfunc (d *dummyLogger) Debugf(format string, args ...any) {\n}\n\nfunc (d *dummyLogger) Tracef(format string, args ...any) {\n}\n\nfunc (d *dummyLogger) Noticef(format string, args ...any) {\n}\n\nfunc (d *dummyLogger) Warnf(format string, args ...any) {\n}\n\nfunc TestStackFatal(t *testing.T) {\n\td := &dummyLogger{}\n\tstackFatalf(d, \"test stack %d\", 1)\n\tif !strings.HasPrefix(d.msg, \"test stack 1\") {\n\t\tt.Fatalf(\"Unexpected start of stack: %v\", d.msg)\n\t}\n\tif !strings.Contains(d.msg, \"test_test.go\") {\n\t\tt.Fatalf(\"Unexpected stack: %v\", d.msg)\n\t}\n}\n" }, { "path": "test/tls_test.go", "content": "// Copyright 2015-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bufio\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nats-io/nats.go\"\n\n\t\"github.com/nats-io/nats-server/v2/server\"\n)\n\nvar noOpErrHandler = func(_ *nats.Conn, _ *nats.Subscription, _ error) {}\n\nfunc TestTLSConnection(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls.conf\")\n\tdefer srv.Shutdown()\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tnurl := fmt.Sprintf(\"tls://%s:%s@%s/\", opts.Username, opts.Password, endpoint)\n\tnc, err := nats.Connect(nurl)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatalf(\"Expected error trying to connect to secure server\")\n\t}\n\n\t// Do simple SecureConnect\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://%s/\", endpoint))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatalf(\"Expected error trying to connect to secure server with no auth\")\n\t}\n\n\t// Now do more advanced checking, verifying servername and using rootCA.\n\n\tnc, err = nats.Connect(nurl, nats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error on Connect with Secure Options: %+v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tsubj := \"foo-tls\"\n\tsub, _ := nc.SubscribeSync(subj)\n\n\tnc.Publish(subj, []byte(\"We are Secure!\"))\n\tnc.Flush()\n\tnmsgs, _, _ := sub.Pending()\n\tif nmsgs != 1 {\n\t\tt.Fatalf(\"Expected to receive a message over the TLS connection\")\n\t}\n}\n\n// TestTLSInProcessConnection checks that even if TLS is enabled on the server,\n// that an in-process connection that does *not* use TLS still connects successfully.\nfunc TestTLSInProcessConnection(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls.conf\")\n\tdefer srv.Shutdown()\n\n\tnc, err := nats.Connect(\"\", nats.InProcessServer(srv), nats.UserInfo(opts.Username, opts.Password))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer nc.Close()\n\n\tif nc.TLSRequired() {\n\t\tt.Fatalf(\"Shouldn't have required TLS for in-process connection\")\n\t}\n\n\tif _, err = nc.TLSConnectionState(); err == nil {\n\t\tt.Fatal(\"Should have got an error retrieving TLS connection state\")\n\t}\n}\n\nfunc TestTLSClientCertificate(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tlsverify.conf\")\n\tdefer srv.Shutdown()\n\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\n\t_, err := nats.Connect(nurl)\n\tif err == nil {\n\t\tt.Fatalf(\"Expected error trying to connect to secure server without a certificate\")\n\t}\n\n\t// Load client certificate to successfully connect.\n\tcertFile := \"./configs/certs/client-cert.pem\"\n\tkeyFile := \"./configs/certs/client-key.pem\"\n\tcert, err := tls.LoadX509KeyPair(certFile, keyFile)\n\tif err != nil {\n\t\tt.Fatalf(\"error parsing X509 certificate/key pair: %v\", err)\n\t}\n\n\t// Load in root CA for server verification\n\trootPEM, err := os.ReadFile(\"./configs/certs/ca.pem\")\n\tif err != nil || rootPEM == nil {\n\t\tt.Fatalf(\"failed to read root certificate\")\n\t}\n\tpool := x509.NewCertPool()\n\tok := pool.AppendCertsFromPEM([]byte(rootPEM))\n\tif !ok {\n\t\tt.Fatalf(\"failed to parse root certificate\")\n\t}\n\n\tconfig := &tls.Config{\n\t\tCertificates: []tls.Certificate{cert},\n\t\tServerName: opts.Host,\n\t\tRootCAs: pool,\n\t\tMinVersion: tls.VersionTLS12,\n\t}\n\n\tcopts := nats.GetDefaultOptions()\n\tcopts.Url = nurl\n\tcopts.Secure = true\n\tcopts.TLSConfig = config\n\n\tnc, err := copts.Connect()\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error on Connect with Secure Options: %+v\\n\", err)\n\t}\n\tnc.Flush()\n\tdefer nc.Close()\n}\n\nfunc TestTLSClientCertificateHasUserID(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls_cert_id.conf\")\n\tdefer srv.Shutdown()\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/client-id-auth-cert.pem\", \"./configs/certs/client-id-auth-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc.Close()\n}\n\nfunc TestTLSClientCertificateCheckWithAllowedConnectionTypes(t *testing.T) {\n\tconf := createConfFile(t, []byte(\n\t\t`\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\ttimeout: 2\n\t\t\tca_file: \"./configs/certs/ca.pem\"\n\t\t\tverify_and_map: true\n\t\t}\n\t\tauthorization {\n\t\t\tusers = [\n\t\t\t\t{user: derek@nats.io, permissions: { publish:\"foo\" }, allowed_connection_types: [\"WEBSOCKET\"]}\n\t\t\t]\n\t\t}\n\t`))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", o.Host, o.Port)\n\tnc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/client-id-auth-cert.pem\", \"./configs/certs/client-id-auth-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err == nil {\n\t\tif nc != nil {\n\t\t\tnc.Close()\n\t\t}\n\t\tt.Fatal(\"Expected connection to fail, it did not\")\n\t}\n}\n\nfunc TestTLSClientCertificateCNBasedAuth(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls_cert_cn.conf\")\n\tdefer srv.Shutdown()\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\terrCh1 := make(chan error)\n\terrCh2 := make(chan error)\n\n\t// Using the default permissions\n\tnc1, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/tlsauth/client.pem\", \"./configs/certs/tlsauth/client-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/tlsauth/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\terrCh1 <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\t// Admin permissions can publish to '>'\n\tnc2, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/tlsauth/client2.pem\", \"./configs/certs/tlsauth/client2-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/tlsauth/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\terrCh2 <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\terr = nc1.Publish(\"foo.bar\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = nc1.SubscribeSync(\"foo.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc1.Flush()\n\n\tsub, err := nc2.SubscribeSync(\">\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc2.Flush()\n\terr = nc2.Publish(\"hello\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc2.Flush()\n\n\t_, err = sub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// Wait for a couple of errors\n\tvar count int\nLoop:\n\tfor {\n\t\tselect {\n\t\tcase err := <-errCh1:\n\t\t\tif err != nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count == 2 {\n\t\t\t\tbreak Loop\n\t\t\t}\n\t\tcase err := <-errCh2:\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected auth error from client: %s\", err)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Timed out expecting auth errors\")\n\t\t}\n\t}\n}\n\nfunc TestTLSClientCertificateSANsBasedAuth(t *testing.T) {\n\t// In this test we have 3 clients, one with permissions defined\n\t// for SAN 'app.nats.dev', other for SAN 'app.nats.prod' and another\n\t// one without the default permissions for the CN.\n\tsrv, opts := RunServerWithConfig(\"./configs/tls_cert_san_auth.conf\")\n\tdefer srv.Shutdown()\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\tdefaultErrCh := make(chan error)\n\tdevErrCh := make(chan error)\n\tprodErrCh := make(chan error)\n\n\t// default: Using the default permissions (no SANs)\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\tdefaultc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/client.pem\", \"./configs/certs/sans/client-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tdefaultErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer defaultc.Close()\n\n\t// dev: Using SAN 'app.nats.dev' with permissions to its own sandbox.\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\t// X509v3 Subject Alternative Name:\n\t// DNS:app.nats.dev, DNS:*.app.nats.dev\n\t//\n\tdevc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/dev.pem\", \"./configs/certs/sans/dev-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tdevErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer devc.Close()\n\n\t// prod: Using SAN 'app.nats.prod' with all permissions.\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\t// X509v3 Subject Alternative Name:\n\t// DNS:app.nats.prod, DNS:*.app.nats.prod\n\t//\n\tprodc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/prod.pem\", \"./configs/certs/sans/prod-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tprodErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer prodc.Close()\n\n\t// No permissions to publish or subscribe on foo.>\n\terr = devc.Publish(\"foo.bar\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = devc.SubscribeSync(\"foo.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdevc.Flush()\n\n\tprodSub, err := prodc.SubscribeSync(\">\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tprodc.Flush()\n\terr = prodc.Publish(\"hello\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tprodc.Flush()\n\n\t// prod: can receive message on wildcard subscription.\n\t_, err = prodSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// dev: enough permissions to publish to sandbox subject.\n\tdevSub, err := devc.SubscribeSync(\"sandbox.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdevc.Flush()\n\terr = devc.Publish(\"sandbox.foo.bar\", []byte(\"hi!\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// both dev and prod clients can receive message\n\t_, err = devSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\t_, err = prodSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// default: no enough permissions\n\t_, err = defaultc.SubscribeSync(\"sandbox.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefaultSub, err := defaultc.SubscribeSync(\"public.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefaultc.Flush()\n\terr = devc.Publish(\"public.foo.bar\", []byte(\"hi!\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = defaultSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// Wait for a couple of errors\n\tvar count int\nLoop:\n\tfor {\n\t\tselect {\n\t\tcase err := <-defaultErrCh:\n\t\t\tif err != nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count == 3 {\n\t\t\t\tbreak Loop\n\t\t\t}\n\t\tcase err := <-devErrCh:\n\t\t\tif err != nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count == 3 {\n\t\t\t\tbreak Loop\n\t\t\t}\n\t\tcase err := <-prodErrCh:\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected auth error from client: %s\", err)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Timed out expecting auth errors\")\n\t\t}\n\t}\n}\n\nfunc TestTLSClientCertificateTLSAuthMultipleOptions(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls_cert_san_emails.conf\")\n\tdefer srv.Shutdown()\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\tdefaultErrCh := make(chan error)\n\tdevErrCh := make(chan error)\n\tprodErrCh := make(chan error)\n\n\t// default: Using the default permissions, there are SANs\n\t// present in the cert but they are not users in the NATS config\n\t// so the subject is used instead.\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\t// X509v3 Subject Alternative Name:\n\t// DNS:app.nats.dev, DNS:*.app.nats.dev\n\t//\n\tdefaultc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/dev.pem\", \"./configs/certs/sans/dev-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tdefaultErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer defaultc.Close()\n\n\t// dev: Using SAN to validate user, even if emails are present in the config.\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\t// X509v3 Subject Alternative Name:\n\t// DNS:app.nats.dev, email:admin@app.nats.dev, email:root@app.nats.dev\n\t//\n\tdevc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/dev-email.pem\", \"./configs/certs/sans/dev-email-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tdevErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer devc.Close()\n\n\t// prod: Using SAN '*.app.nats.prod' with all permissions, which is not the first SAN.\n\t//\n\t// Subject: CN = www.nats.io\n\t//\n\t// X509v3 Subject Alternative Name:\n\t// DNS:app.nats.prod, DNS:*.app.nats.prod\n\t//\n\tprodc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/sans/prod.pem\", \"./configs/certs/sans/prod-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/sans/ca.pem\"),\n\t\tnats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {\n\t\t\tprodErrCh <- err\n\t\t}),\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer prodc.Close()\n\n\t// No permissions to publish or subscribe on foo.>\n\terr = devc.Publish(\"foo.bar\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = devc.SubscribeSync(\"foo.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdevc.Flush()\n\n\tprodSub, err := prodc.SubscribeSync(\">\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tprodc.Flush()\n\terr = prodc.Publish(\"hello\", []byte(\"hi\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tprodc.Flush()\n\n\t// prod: can receive message on wildcard subscription.\n\t_, err = prodSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// dev: enough permissions to publish to sandbox subject.\n\tdevSub, err := devc.SubscribeSync(\"sandbox.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdevc.Flush()\n\terr = devc.Publish(\"sandbox.foo.bar\", []byte(\"hi!\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// both dev and prod clients can receive message\n\t_, err = devSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\t_, err = prodSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// default: no enough permissions\n\t_, err = defaultc.SubscribeSync(\"sandbox.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefaultSub, err := defaultc.SubscribeSync(\"public.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefaultc.Flush()\n\terr = devc.Publish(\"public.foo.bar\", []byte(\"hi!\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = defaultSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\n\t// Wait for a couple of errors\n\tvar count int\nLoop:\n\tfor {\n\t\tselect {\n\t\tcase err := <-defaultErrCh:\n\t\t\tif err != nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count == 3 {\n\t\t\t\tbreak Loop\n\t\t\t}\n\t\tcase err := <-devErrCh:\n\t\t\tif err != nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tif count == 3 {\n\t\t\t\tbreak Loop\n\t\t\t}\n\t\tcase err := <-prodErrCh:\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Received unexpected auth error from client: %s\", err)\n\t\t\t}\n\t\tcase <-time.After(2 * time.Second):\n\t\t\tt.Fatalf(\"Timed out expecting auth errors\")\n\t\t}\n\t}\n}\n\nfunc TestTLSRoutesCertificateCNBasedAuth(t *testing.T) {\n\t// Base config for the servers\n\toptsA := LoadConfig(\"./configs/tls_cert_cn_routes.conf\")\n\toptsB := LoadConfig(\"./configs/tls_cert_cn_routes.conf\")\n\toptsC := LoadConfig(\"./configs/tls_cert_cn_routes_invalid_auth.conf\")\n\n\t// TLS map should have been enabled\n\tif !optsA.Cluster.TLSMap || !optsB.Cluster.TLSMap || !optsC.Cluster.TLSMap {\n\t\tt.Error(\"Expected Cluster TLS verify and map feature to be activated\")\n\t}\n\n\trouteURLs := \"nats://localhost:9935, nats://localhost:9936, nats://localhost:9937\"\n\n\toptsA.Host = \"127.0.0.1\"\n\toptsA.Port = 9335\n\toptsA.Cluster.Name = \"xyz\"\n\toptsA.Cluster.Host = optsA.Host\n\toptsA.Cluster.Port = 9935\n\toptsA.Routes = server.RoutesFromStr(routeURLs)\n\toptsA.NoSystemAccount = true\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\toptsB.Host = \"127.0.0.1\"\n\toptsB.Port = 9336\n\toptsB.Cluster.Name = \"xyz\"\n\toptsB.Cluster.Host = optsB.Host\n\toptsB.Cluster.Port = 9936\n\toptsB.Routes = server.RoutesFromStr(routeURLs)\n\toptsB.NoSystemAccount = true\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\toptsC.Host = \"127.0.0.1\"\n\toptsC.Port = 9337\n\toptsC.Cluster.Name = \"xyz\"\n\toptsC.Cluster.Host = optsC.Host\n\toptsC.Cluster.Port = 9937\n\toptsC.Routes = server.RoutesFromStr(routeURLs)\n\toptsC.NoSystemAccount = true\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\t// srvC is not connected to srvA and srvB due to wrong cert\n\tcheckClusterFormed(t, srvA, srvB)\n\n\tnc1, err := nats.Connect(fmt.Sprintf(\"%s:%d\", optsA.Host, optsA.Port), nats.Name(\"A\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(fmt.Sprintf(\"%s:%d\", optsB.Host, optsB.Port), nats.Name(\"B\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// This client is partitioned from rest of cluster due to using wrong cert.\n\tnc3, err := nats.Connect(fmt.Sprintf(\"%s:%d\", optsC.Host, optsC.Port), nats.Name(\"C\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc3.Close()\n\n\t// Allowed via permissions to broadcast to other members of cluster.\n\tpublicSub, err := nc1.SubscribeSync(\"public.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Not part of cluster permissions so message is not forwarded.\n\tprivateSub, err := nc1.SubscribeSync(\"private.>\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// This ensures that srvA has received the sub, not srvB\n\tnc1.Flush()\n\n\tcheckFor(t, time.Second, 15*time.Millisecond, func() error {\n\t\tif n := srvB.NumSubscriptions(); n != 1 {\n\t\t\treturn fmt.Errorf(\"Expected 1 sub, got %v\", n)\n\t\t}\n\t\treturn nil\n\t})\n\n\t// Not forwarded by cluster so won't be received.\n\terr = nc2.Publish(\"private.foo\", []byte(\"private message on server B\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = nc2.Publish(\"public.foo\", []byte(\"public message from server B to server A\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc2.Flush()\n\terr = nc3.Publish(\"public.foo\", []byte(\"dropped message since unauthorized server\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnc3.Flush()\n\n\t// Message will be received since allowed via permissions\n\t_, err = publicSub.NextMsg(1 * time.Second)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during wait for next message: %s\", err)\n\t}\n\tmsg, err := privateSub.NextMsg(500 * time.Millisecond)\n\tif err == nil {\n\t\tt.Errorf(\"Received unexpected message from private sub: %+v\", msg)\n\t}\n\tmsg, err = publicSub.NextMsg(500 * time.Millisecond)\n\tif err == nil {\n\t\tt.Errorf(\"Received unexpected message from private sub: %+v\", msg)\n\t}\n}\n\nfunc TestTLSGatewaysCertificateCNBasedAuth(t *testing.T) {\n\t// Base config for the servers\n\toptsA := LoadConfig(\"./configs/tls_cert_cn_gateways.conf\")\n\toptsB := LoadConfig(\"./configs/tls_cert_cn_gateways.conf\")\n\toptsC := LoadConfig(\"./configs/tls_cert_cn_gateways_invalid_auth.conf\")\n\n\t// TLS map should have been enabled\n\tif !optsA.Gateway.TLSMap || !optsB.Gateway.TLSMap || !optsC.Gateway.TLSMap {\n\t\tt.Error(\"Expected Cluster TLS verify and map feature to be activated\")\n\t}\n\n\tgwA, err := url.Parse(\"nats://localhost:9995\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tgwB, err := url.Parse(\"nats://localhost:9996\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tgwC, err := url.Parse(\"nats://localhost:9997\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\toptsA.Host = \"127.0.0.1\"\n\toptsA.Port = -1\n\toptsA.Gateway.Name = \"A\"\n\toptsA.Gateway.Port = 9995\n\n\toptsB.Host = \"127.0.0.1\"\n\toptsB.Port = -1\n\toptsB.Gateway.Name = \"B\"\n\toptsB.Gateway.Port = 9996\n\n\toptsC.Host = \"127.0.0.1\"\n\toptsC.Port = -1\n\toptsC.Gateway.Name = \"C\"\n\toptsC.Gateway.Port = 9997\n\n\tgateways := make([]*server.RemoteGatewayOpts, 3)\n\tgateways[0] = &server.RemoteGatewayOpts{\n\t\tName: optsA.Gateway.Name,\n\t\tURLs: []*url.URL{gwA},\n\t}\n\tgateways[1] = &server.RemoteGatewayOpts{\n\t\tName: optsB.Gateway.Name,\n\t\tURLs: []*url.URL{gwB},\n\t}\n\tgateways[2] = &server.RemoteGatewayOpts{\n\t\tName: optsC.Gateway.Name,\n\t\tURLs: []*url.URL{gwC},\n\t}\n\toptsA.Gateway.Gateways = gateways\n\toptsB.Gateway.Gateways = gateways\n\toptsC.Gateway.Gateways = gateways\n\n\tserver.SetGatewaysSolicitDelay(100 * time.Millisecond)\n\tdefer server.ResetGatewaysSolicitDelay()\n\n\tsrvA := RunServer(optsA)\n\tdefer srvA.Shutdown()\n\n\tsrvB := RunServer(optsB)\n\tdefer srvB.Shutdown()\n\n\tsrvC := RunServer(optsC)\n\tdefer srvC.Shutdown()\n\n\t// Because we need to use \"localhost\" in the gw URLs (to match\n\t// hostname in the user/CN), the server may try to connect to\n\t// a [::1], etc.. that may or may not work, so give a lot of\n\t// time for that to complete ok.\n\twaitForOutboundGateways(t, srvA, 1, 5*time.Second)\n\twaitForOutboundGateways(t, srvB, 1, 5*time.Second)\n\n\tnc1, err := nats.Connect(srvA.Addr().String(), nats.Name(\"A\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc1.Close()\n\n\tnc2, err := nats.Connect(srvB.Addr().String(), nats.Name(\"B\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\tnc3, err := nats.Connect(srvC.Addr().String(), nats.Name(\"C\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Expected to connect, got %v\", err)\n\t}\n\tdefer nc3.Close()\n\n\treceived := make(chan *nats.Msg)\n\t_, err = nc1.Subscribe(\"foo\", func(msg *nats.Msg) {\n\t\tselect {\n\t\tcase received <- msg:\n\t\tdefault:\n\t\t}\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t_, err = nc3.Subscribe(\"help\", func(msg *nats.Msg) {\n\t\tnc3.Publish(msg.Reply, []byte(\"response\"))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tgo func() {\n\t\tfor range time.NewTicker(10 * time.Millisecond).C {\n\t\t\tif nc2.IsClosed() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnc2.Publish(\"foo\", []byte(\"bar\"))\n\t\t}\n\t}()\n\n\tselect {\n\tcase <-received:\n\tcase <-time.After(2 * time.Second):\n\t\tt.Error(\"Timed out waiting for gateway messages\")\n\t}\n\n\tmsg, err := nc2.Request(\"help\", []byte(\"should fail\"), 100*time.Millisecond)\n\tif err == nil {\n\t\tt.Errorf(\"Expected to not receive any messages, got: %+v\", msg)\n\t}\n}\n\nfunc TestTLSVerifyClientCertificate(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tlsverify_noca.conf\")\n\tdefer srv.Shutdown()\n\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\n\t// The client is configured properly, but the server has no CA\n\t// to verify the client certificate. Connection should fail.\n\tnc, err := nats.Connect(nurl,\n\t\tnats.ClientCert(\"./configs/certs/client-cert.pem\", \"./configs/certs/client-key.pem\"),\n\t\tnats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatal(\"Expected failure to connect, did not\")\n\t}\n}\n\nfunc TestTLSConnectionTimeout(t *testing.T) {\n\topts := LoadConfig(\"./configs/tls.conf\")\n\topts.TLSTimeout = 0.25\n\n\tsrv := RunServer(opts)\n\tdefer srv.Shutdown()\n\n\t// Dial with normal TCP\n\tendpoint := net.JoinHostPort(opts.Host, fmt.Sprintf(\"%d\", opts.Port))\n\tconn, err := net.Dial(\"tcp\", endpoint)\n\tif err != nil {\n\t\tt.Fatalf(\"Could not connect to %q\", endpoint)\n\t}\n\tdefer conn.Close()\n\n\t// Read deadlines\n\tconn.SetReadDeadline(time.Now().Add(2 * time.Second))\n\n\t// Read the INFO string.\n\tbr := bufio.NewReader(conn)\n\tinfo, err := br.ReadString('\\n')\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read INFO - %v\", err)\n\t}\n\tif !strings.HasPrefix(info, \"INFO \") {\n\t\tt.Fatalf(\"INFO response incorrect: %s\\n\", info)\n\t}\n\twait := time.Duration(opts.TLSTimeout * float64(time.Second))\n\ttime.Sleep(wait)\n\t// Read deadlines\n\tconn.SetReadDeadline(time.Now().Add(2 * time.Second))\n\ttlsErr, err := br.ReadString('\\n')\n\tif err == nil && !strings.Contains(tlsErr, \"-ERR 'Secure Connection - TLS Required\") {\n\t\tt.Fatalf(\"TLS Timeout response incorrect: %q\\n\", tlsErr)\n\t}\n}\n\n// Ensure there is no race between authorization timeout and TLS handshake.\nfunc TestTLSAuthorizationShortTimeout(t *testing.T) {\n\topts := LoadConfig(\"./configs/tls.conf\")\n\topts.AuthTimeout = 0.001\n\n\tsrv := RunServer(opts)\n\tdefer srv.Shutdown()\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tnurl := fmt.Sprintf(\"tls://%s:%s@%s/\", opts.Username, opts.Password, endpoint)\n\n\t// Expect an error here (no CA) but not a TLS oversized record error which\n\t// indicates the authorization timeout fired too soon.\n\t_, err := nats.Connect(nurl)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error trying to connect to secure server\")\n\t}\n\tif strings.Contains(err.Error(), \"oversized record\") {\n\t\tt.Fatal(\"Corrupted TLS handshake:\", err)\n\t}\n}\n\nfunc TestClientTLSAndNonTLSConnections(t *testing.T) {\n\ts, opts := RunServerWithConfig(\"./configs/tls_mixed.conf\")\n\tdefer s.Shutdown()\n\n\tsurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\tnc, err := nats.Connect(surl, nats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to connect with TLS: %v\", err)\n\t}\n\tdefer nc.Close()\n\n\t// Now also make sure we can connect through plain text.\n\tnurl := fmt.Sprintf(\"nats://%s:%d\", opts.Host, opts.Port)\n\tnc2, err := nats.Connect(nurl)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to connect without TLS: %v\", err)\n\t}\n\tdefer nc2.Close()\n\n\t// Make sure they can go back and forth.\n\tsub, err := nc.SubscribeSync(\"foo\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tsub2, err := nc2.SubscribeSync(\"bar\")\n\tif err != nil {\n\t\tt.Fatalf(\"Error subscribing: %v\", err)\n\t}\n\tnc.Flush()\n\tnc2.Flush()\n\n\tnmsgs := 100\n\tfor i := 0; i < nmsgs; i++ {\n\t\tnc2.Publish(\"foo\", []byte(\"HELLO FROM PLAINTEXT\"))\n\t\tnc.Publish(\"bar\", []byte(\"HELLO FROM TLS\"))\n\t}\n\t// Now wait for the messages.\n\tcheckFor(t, time.Second, 10*time.Millisecond, func() error {\n\t\tif msgs, _, err := sub.Pending(); err != nil || msgs != nmsgs {\n\t\t\treturn fmt.Errorf(\"Did not receive the correct number of messages: %d\", msgs)\n\t\t}\n\t\tif msgs, _, err := sub2.Pending(); err != nil || msgs != nmsgs {\n\t\t\treturn fmt.Errorf(\"Did not receive the correct number of messages: %d\", msgs)\n\t\t}\n\t\treturn nil\n\t})\n\n}\n\nfunc stressConnect(t *testing.T, wg *sync.WaitGroup, errCh chan error, url string, index int) {\n\tdefer wg.Done()\n\n\tsubName := fmt.Sprintf(\"foo.%d\", index)\n\n\tfor i := 0; i < 33; i++ {\n\t\tnc, err := nats.Connect(url, nats.RootCAs(\"./configs/certs/ca.pem\"))\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"Unable to create TLS connection: %v\\n\", err)\n\t\t\treturn\n\t\t}\n\t\tdefer nc.Close()\n\n\t\tsub, err := nc.SubscribeSync(subName)\n\t\tif err != nil {\n\t\t\terrCh <- fmt.Errorf(\"Unable to subscribe on '%s': %v\\n\", subName, err)\n\t\t\treturn\n\t\t}\n\n\t\tif err := nc.Publish(subName, []byte(\"secure data\")); err != nil {\n\t\t\terrCh <- fmt.Errorf(\"Unable to send on '%s': %v\\n\", subName, err)\n\t\t}\n\n\t\tif _, err := sub.NextMsg(2 * time.Second); err != nil {\n\t\t\terrCh <- fmt.Errorf(\"Unable to get next message: %v\\n\", err)\n\t\t}\n\n\t\tnc.Close()\n\t}\n\n\terrCh <- nil\n}\n\nfunc TestTLSStressConnect(t *testing.T) {\n\topts, err := server.ProcessConfigFile(\"./configs/tls.conf\")\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"Error processing configuration file: %v\", err))\n\t}\n\topts.NoSigs, opts.NoLog = true, true\n\n\t// For this test, remove the authorization\n\topts.Username = \"\"\n\topts.Password = \"\"\n\n\t// Increase ssl timeout\n\topts.TLSTimeout = 2.0\n\n\tsrv := RunServer(opts)\n\tdefer srv.Shutdown()\n\n\tnurl := fmt.Sprintf(\"tls://%s:%d\", opts.Host, opts.Port)\n\n\tthreadCount := 3\n\n\terrCh := make(chan error, threadCount)\n\n\tvar wg sync.WaitGroup\n\twg.Add(threadCount)\n\n\tfor i := 0; i < threadCount; i++ {\n\t\tgo stressConnect(t, &wg, errCh, nurl, i)\n\t}\n\n\twg.Wait()\n\n\tvar lastError error\n\tfor i := 0; i < threadCount; i++ {\n\t\terr := <-errCh\n\t\tif err != nil {\n\t\t\tlastError = err\n\t\t}\n\t}\n\n\tif lastError != nil {\n\t\tt.Fatalf(\"%v\\n\", lastError)\n\t}\n}\n\nfunc TestTLSBadAuthError(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls.conf\")\n\tdefer srv.Shutdown()\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tnurl := fmt.Sprintf(\"tls://%s:%s@%s/\", opts.Username, \"NOT_THE_PASSWORD\", endpoint)\n\n\t_, err := nats.Connect(nurl, nats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err == nil {\n\t\tt.Fatalf(\"Expected error trying to connect to secure server\")\n\t}\n\tif strings.ToLower(err.Error()) != nats.ErrAuthorization.Error() {\n\t\tt.Fatalf(\"Expected and auth violation, got %v\\n\", err)\n\t}\n}\n\nfunc TestTLSConnectionCurvePref(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/tls_curve_pref.conf\")\n\tdefer srv.Shutdown()\n\n\tif len(opts.TLSConfig.CurvePreferences) != 1 {\n\t\tt.Fatal(\"Invalid curve preference loaded.\")\n\t}\n\n\tif opts.TLSConfig.CurvePreferences[0] != tls.CurveP256 {\n\t\tt.Fatalf(\"Invalid curve preference loaded [%v].\", opts.TLSConfig.CurvePreferences[0])\n\t}\n\n\tendpoint := fmt.Sprintf(\"%s:%d\", opts.Host, opts.Port)\n\tnurl := fmt.Sprintf(\"tls://%s:%s@%s/\", opts.Username, opts.Password, endpoint)\n\tnc, err := nats.Connect(nurl)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatalf(\"Expected error trying to connect to secure server\")\n\t}\n\n\t// Do simple SecureConnect\n\tnc, err = nats.Connect(fmt.Sprintf(\"tls://%s/\", endpoint))\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatalf(\"Expected error trying to connect to secure server with no auth\")\n\t}\n\n\t// Now do more advanced checking, verifying servername and using rootCA.\n\n\tnc, err = nats.Connect(nurl, nats.RootCAs(\"./configs/certs/ca.pem\"))\n\tif err != nil {\n\t\tt.Fatalf(\"Got an error on Connect with Secure Options: %+v\\n\", err)\n\t}\n\tdefer nc.Close()\n\n\tsubj := \"foo-tls\"\n\tsub, _ := nc.SubscribeSync(subj)\n\n\tnc.Publish(subj, []byte(\"We are Secure!\"))\n\tnc.Flush()\n\tnmsgs, _, _ := sub.Pending()\n\tif nmsgs != 1 {\n\t\tt.Fatalf(\"Expected to receive a message over the TLS connection\")\n\t}\n}\n\ntype captureSlowConsumerLogger struct {\n\tdummyLogger\n\tch chan string\n\tgotIt bool\n}\n\nfunc (l *captureSlowConsumerLogger) Noticef(format string, v ...any) {\n\tmsg := fmt.Sprintf(format, v...)\n\tif strings.Contains(msg, \"Slow Consumer\") {\n\t\tl.Lock()\n\t\tif !l.gotIt {\n\t\t\tl.gotIt = true\n\t\t\tl.ch <- msg\n\t\t}\n\t\tl.Unlock()\n\t}\n}\n\nfunc TestTLSTimeoutNotReportSlowConsumer(t *testing.T) {\n\toa, err := server.ProcessConfigFile(\"./configs/srv_a_tls.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to load config file: %v\", err)\n\t}\n\n\t// Override TLSTimeout to very small value so that handshake fails\n\toa.Cluster.TLSTimeout = 0.0000001\n\tsa := RunServer(oa)\n\tdefer sa.Shutdown()\n\n\tch := make(chan string, 1)\n\tcscl := &captureSlowConsumerLogger{ch: ch}\n\tsa.SetLogger(cscl, false, false)\n\n\tob, err := server.ProcessConfigFile(\"./configs/srv_b_tls.conf\")\n\tif err != nil {\n\t\tt.Fatalf(\"Unable to load config file: %v\", err)\n\t}\n\n\tsb := RunServer(ob)\n\tdefer sb.Shutdown()\n\n\t// Watch the logger for a bit and make sure we don't get any\n\t// Slow Consumer error.\n\tselect {\n\tcase e := <-ch:\n\t\tt.Fatalf(\"Unexpected slow consumer error: %s\", e)\n\tcase <-time.After(500 * time.Millisecond):\n\t\t// ok\n\t}\n}\n\nfunc TestTLSHandshakeFailureMemUsage(t *testing.T) {\n\tfor i, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t}{\n\t\t{\n\t\t\t\"connect to TLS client port\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\t\t\t`,\n\t\t},\n\t\t{\n\t\t\t\"connect to TLS route port\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tcluster {\n\t\t\t\t\tport: -1\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`,\n\t\t},\n\t\t{\n\t\t\t\"connect to TLS gateway port\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\tgateway {\n\t\t\t\t\tname: \"A\"\n\t\t\t\t\tport: -1\n\t\t\t\t\t%s\n\t\t\t\t}\n\t\t\t`,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := fmt.Sprintf(test.config, `\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/server-cert.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/server-key.pem\"\n\t\t\t\t\tca_file: \"configs/certs/ca.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t}\n\t\t\t`)\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tvar port int\n\t\t\tswitch i {\n\t\t\tcase 0:\n\t\t\t\tport = opts.Port\n\t\t\tcase 1:\n\t\t\t\tport = s.ClusterAddr().Port\n\t\t\tcase 2:\n\t\t\t\tport = s.GatewayAddr().Port\n\t\t\t}\n\n\t\t\tvarz, _ := s.Varz(nil)\n\t\t\tbase := varz.Mem\n\t\t\tbuf := make([]byte, 1024*1024)\n\t\t\tfor i := 0; i < 100; i++ {\n\t\t\t\tconn, err := net.Dial(\"tcp\", fmt.Sprintf(\"localhost:%d\", port))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Error on dial: %v\", err)\n\t\t\t\t}\n\t\t\t\tconn.SetWriteDeadline(time.Now().Add(10 * time.Millisecond))\n\t\t\t\tconn.Write(buf)\n\t\t\t\tconn.Close()\n\t\t\t}\n\n\t\t\tvarz, _ = s.Varz(nil)\n\t\t\tnewval := (varz.Mem - base) / (1024 * 1024)\n\t\t\t// May need to adjust that, but right now, with 100 clients\n\t\t\t// we are at about 20MB for client port, 11MB for route\n\t\t\t// and 6MB for gateways, so pick 50MB as the threshold\n\t\t\tif newval >= 50 {\n\t\t\t\tt.Fatalf(\"Consumed too much memory: %v MB\", newval)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTLSClientAuthWithRDNSequence(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\tcerts nats.Option\n\t\terr error\n\t\trerr error\n\t}{\n\t\t// To generate certs for these tests:\n\t\t// USE `regenerate_rdns_svid.sh` TO REGENERATE\n\t\t//\n\t\t// ```\n\t\t// openssl req -newkey rsa:2048 -nodes -keyout client-$CLIENT_ID.key -subj \"/C=US/ST=CA/L=Los Angeles/OU=NATS/O=NATS/CN=*.example.com/DC=example/DC=com\" -addext extendedKeyUsage=clientAuth -out client-$CLIENT_ID.csr\n\t\t// openssl x509 -req -extfile <(printf \"subjectAltName=DNS:localhost,DNS:example.com,DNS:www.example.com\") -days 1825 -in client-$CLIENT_ID.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client-$CLIENT_ID.pem\n\t\t// ```\n\t\t//\n\t\t// To confirm subject from cert:\n\t\t//\n\t\t// ```\n\t\t// openssl x509 -in client-$CLIENT_ID.pem -text | grep Subject:\n\t\t// ```\n\t\t//\n\t\t{\n\t\t\t\"connect with tls using full RDN sequence\",\n\t\t\t`\n port: -1\n %s\n\n authorization {\n users = [\n { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2\" }\n ]\n }\n `,\n\t\t\t// C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-a.pem\", \"./configs/certs/rdns/client-a.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls using full RDN sequence in original order\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\" }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t// C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo1, DC = foo2\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-a.pem\", \"./configs/certs/rdns/client-a.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls using partial RDN sequence has different permissions\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\" },\n\t\t\t\t { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2\" },\n\t\t\t\t { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\",\n permissions = { subscribe = { deny = \">\" }} }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t// C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-b.pem\", \"./configs/certs/rdns/client-b.key\"),\n\t\t\tnil,\n\t\t\terrors.New(\"nats: timeout\"),\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and RDN sequence partially matches\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\" }\n\t\t\t\t { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2\" }\n\t\t\t\t { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\"},\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t// Cert is:\n\t\t\t//\n\t\t\t// C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4\n\t\t\t//\n\t\t\t// but it will actually match the user without DCs so will not get an error:\n\t\t\t//\n\t\t\t// C = US, ST = CA, L = Los Angeles, O = NATS, OU = NATS, CN = localhost\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-c.pem\", \"./configs/certs/rdns/client-c.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and RDN sequence does not match\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=foo1,DC=foo2\" },\n\t\t\t\t { user = \"DC=foo2,DC=foo1,CN=localhost,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\" }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t// C = US, ST = California, L = Los Angeles, O = NATS, OU = NATS, CN = localhost, DC = foo3, DC = foo4\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-c.pem\", \"./configs/certs/rdns/client-c.key\"),\n\t\t\terrors.New(\"nats: Authorization Violation\"),\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and RDN sequence with space after comma not should matter\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=foo2, DC=foo1, CN=localhost, OU=NATS, O=NATS, L=Los Angeles, ST=CA, C=US\" }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t// C=US/ST=California/L=Los Angeles/O=NATS/OU=NATS/CN=localhost/DC=foo1/DC=foo2\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-a.pem\", \"./configs/certs/rdns/client-a.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and full RDN sequence respects order\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=com, DC=example, CN=*.example.com, O=NATS, OU=NATS, L=Los Angeles, ST=CA, C=US\" }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-d.pem\", \"./configs/certs/rdns/client-d.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and full RDN sequence with added domainComponents and spaces also matches\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com\" }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-d.pem\", \"./configs/certs/rdns/client-d.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and full RDN sequence with correct order takes precedence over others matches\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US,DC=example,DC=com\",\n permissions = { subscribe = { deny = \">\" }} }\n\n\t\t\t\t # This should take precedence since it is in the RFC2253 order.\n\t\t\t\t { user = \"DC=com,DC=example,CN=*.example.com,O=NATS,OU=NATS,L=Los Angeles,ST=CA,C=US\" }\n\n\t\t\t\t { user = \"CN=*.example.com,OU=NATS,O=NATS,L=Los Angeles,ST=CA,C=US\",\n permissions = { subscribe = { deny = \">\" }} }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// C = US, ST = CA, L = Los Angeles, OU = NATS, O = NATS, CN = *.example.com, DC = example, DC = com\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-d.pem\", \"./configs/certs/rdns/client-d.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and RDN includes multiple CN elements\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/CN=John Doe/CN=123456/CN=jdoe/OU=Users/OU=Organic Units/DC=acme/DC=com\"\n\t\t\t// Go: CN=jdoe,OU=Users+OU=Organic Units\n\t\t\t// RFC2253: DC=com,DC=acme,OU=Organic Units,OU=Users,CN=jdoe,CN=123456,CN=John Doe\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-e.pem\", \"./configs/certs/rdns/client-e.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN but there is no match\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=John Doe,DC=DEV,DC=OpenSSL,DC=org\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\terrors.New(\"nats: Authorization Violation\"),\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN that are reordered\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"CN=John Doe,O=users+DC=DEV,DC=OpenSSL,DC=org\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := fmt.Sprintf(test.config, `\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/rdns/server.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/rdns/server.key\"\n\t\t\t\t\tca_file: \"configs/certs/rdns/ca.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify_and_map: true\n\t\t\t\t}\n\t\t\t`)\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\t\t\ttest.certs,\n\t\t\t\tnats.RootCAs(\"./configs/certs/rdns/ca.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTLSClientAuthWithRDNSequenceReordered(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\tcerts nats.Option\n\t\terr error\n\t\trerr error\n\t}{\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN that are reordered\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=org,DC=OpenSSL,O=users+DC=DEV,CN=John Doe\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=org,DC=OpenSSL,O=users,CN=John Doe\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\terrors.New(\"nats: Authorization Violation\"),\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls and DN includes a multi value RDN that are reordered but not equal RDNs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t { user = \"DC=OpenSSL, DC=org, O=users, CN=John Doe\" }\n ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\t//\n\t\t\t// OpenSSL: -subj \"/DC=org/DC=OpenSSL/DC=DEV+O=users/CN=John Doe\" -multivalue-rdn\n\t\t\t// Go: CN=John Doe,O=users\n\t\t\t// RFC2253: CN=John Doe,DC=DEV+O=users,DC=OpenSSL,DC=org\n\t\t\t//\n\t\t\tnats.ClientCert(\"./configs/certs/rdns/client-f.pem\", \"./configs/certs/rdns/client-f.key\"),\n\t\t\terrors.New(\"nats: Authorization Violation\"),\n\t\t\tnil,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := fmt.Sprintf(test.config, `\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/rdns/server.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/rdns/server.key\"\n\t\t\t\t\tca_file: \"configs/certs/rdns/ca.pem\"\n\t\t\t\t\ttimeout: 5\n\t\t\t\t\tverify_and_map: true\n\t\t\t\t}\n\t\t\t`)\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\t\t\ttest.certs,\n\t\t\t\tnats.RootCAs(\"./configs/certs/rdns/ca.pem\"),\n\t\t\t)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTLSClientSVIDAuth(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tname string\n\t\tconfig string\n\t\tcerts nats.Option\n\t\terr error\n\t\trerr error\n\t}{\n\t\t{\n\t\t\t\"connect with tls using certificate with URIs\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t {\n user = \"spiffe://localhost/my-nats-service/user-a\"\n }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\tnats.ClientCert(\"./configs/certs/svid/client-a.pem\", \"./configs/certs/svid/client-a.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls using certificate with limited different permissions\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t {\n user = \"spiffe://localhost/my-nats-service/user-a\"\n },\n\t\t\t\t {\n user = \"spiffe://localhost/my-nats-service/user-b\"\n permissions = { subscribe = { deny = \">\" }}\n }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\tnats.ClientCert(\"./configs/certs/svid/client-b.pem\", \"./configs/certs/svid/client-b.key\"),\n\t\t\tnil,\n\t\t\terrors.New(\"nats: timeout\"),\n\t\t},\n\t\t{\n\t\t\t\"connect with tls without URIs in permissions will still match SAN\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t {\n user = \"O=SPIRE,C=US\"\n }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\tnats.ClientCert(\"./configs/certs/svid/client-b.pem\", \"./configs/certs/svid/client-b.key\"),\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"connect with tls but no permissions\",\n\t\t\t`\n\t\t\t\tport: -1\n\t\t\t\t%s\n\n\t\t\t\tauthorization {\n\t\t\t\t users = [\n\t\t\t\t {\n user = \"spiffe://localhost/my-nats-service/user-c\"\n }\n\t\t\t\t ]\n\t\t\t\t}\n\t\t\t`,\n\t\t\tnats.ClientCert(\"./configs/certs/svid/client-a.pem\", \"./configs/certs/svid/client-a.key\"),\n\t\t\terrors.New(\"nats: Authorization Violation\"),\n\t\t\tnil,\n\t\t},\n\t} {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcontent := fmt.Sprintf(test.config, `\n\t\t\t\ttls {\n\t\t\t\t\tcert_file: \"configs/certs/svid/server.pem\"\n\t\t\t\t\tkey_file: \"configs/certs/svid/server.key\"\n\t\t\t\t\tca_file: \"configs/certs/svid/ca.pem\"\n\t\t\t\t\ttimeout: 5\n insecure: true\n\t\t\t\t\tverify_and_map: true\n\t\t\t\t}\n\t\t\t`)\n\t\t\tconf := createConfFile(t, []byte(content))\n\t\t\ts, opts := RunServerWithConfig(conf)\n\t\t\tdefer s.Shutdown()\n\n\t\t\tnc, err := nats.Connect(fmt.Sprintf(\"tls://localhost:%d\", opts.Port),\n\t\t\t\ttest.certs,\n\t\t\t\tnats.RootCAs(\"./configs/certs/svid/ca.pem\"),\n\t\t\t\tnats.ErrorHandler(noOpErrHandler),\n\t\t\t)\n\t\t\tif test.err == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected to connect, got %v\", err)\n\t\t\t} else if test.err != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error on connect\")\n\t\t\t} else if test.err != nil && err != nil {\n\t\t\t\t// Error on connect was expected\n\t\t\t\tif test.err.Error() != err.Error() {\n\t\t\t\t\tt.Errorf(\"Expected error %s, got: %s\", test.err, err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer nc.Close()\n\n\t\t\tnc.Subscribe(\"ping\", func(m *nats.Msg) {\n\t\t\t\tm.Respond([]byte(\"pong\"))\n\t\t\t})\n\t\t\tnc.Flush()\n\n\t\t\t_, err = nc.Request(\"ping\", []byte(\"ping\"), 250*time.Millisecond)\n\t\t\tif test.rerr != nil && err == nil {\n\t\t\t\tt.Errorf(\"Expected error getting response\")\n\t\t\t} else if test.rerr == nil && err != nil {\n\t\t\t\tt.Errorf(\"Expected response\")\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTLSPinnedCertsClient(t *testing.T) {\n\ttmpl := `\n\thost: localhost\n\tport: -1\n\ttls {\n\t\tca_file: \"configs/certs/ca.pem\"\n\t\tcert_file: \"configs/certs/server-cert.pem\"\n\t\tkey_file: \"configs/certs/server-key.pem\"\n\t\t# Require a client certificate and map user id from certificate\n\t\tverify: true\n\t\tpinned_certs: [\"%s\"]\n\t}`\n\n\tconfFileName := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9\")))\n\tsrv, o := RunServerWithConfig(confFileName)\n\tdefer srv.Shutdown()\n\n\tif len(o.TLSPinnedCerts) != 1 {\n\t\tt.Fatal(\"expected one pinned cert\")\n\t}\n\n\topts := []nats.Option{\n\t\tnats.RootCAs(\"configs/certs/ca.pem\"),\n\t\tnats.ClientCert(\"./configs/certs/client-cert.pem\", \"./configs/certs/client-key.pem\"),\n\t}\n\n\tnc, err := nats.Connect(srv.ClientURL(), opts...)\n\tif err == nil {\n\t\tnc.Close()\n\t\tt.Fatalf(\"Expected error trying to connect without a certificate in pinned_certs\")\n\t}\n\n\tos.WriteFile(confFileName, []byte(fmt.Sprintf(tmpl, \"bf6f821f09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9\")), 0660)\n\tif err := srv.Reload(); err != nil {\n\t\tt.Fatalf(\"on Reload got %v\", err)\n\t}\n\t// reload pinned to the certs used\n\tnc, err = nats.Connect(srv.ClientURL(), opts...)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected no error, got: %v\", err)\n\t}\n\tnc.Close()\n}\n\ntype captureWarnLogger struct {\n\tdummyLogger\n\treceive chan string\n}\n\nfunc newCaptureWarnLogger() *captureWarnLogger {\n\treturn &captureWarnLogger{\n\t\treceive: make(chan string, 100),\n\t}\n}\n\nfunc (l *captureWarnLogger) Warnf(format string, v ...any) {\n\tl.receive <- fmt.Sprintf(format, v...)\n}\n\nfunc (l *captureWarnLogger) waitFor(expect string, timeout time.Duration) bool {\n\tfor {\n\t\tselect {\n\t\tcase msg := <-l.receive:\n\t\t\tif strings.Contains(msg, expect) {\n\t\t\t\treturn true\n\t\t\t}\n\t\tcase <-time.After(timeout):\n\t\t\treturn false\n\t\t}\n\t}\n}\n\nfunc TestTLSConnectionRate(t *testing.T) {\n\tconfig := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\ttls {\n\t\t\tcert_file: \"./configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"./configs/certs/server-key.pem\"\n\t\t\tconnection_rate_limit: 3\n\t\t}\n\t`\n\n\tconfFileName := createConfFile(t, []byte(config))\n\n\tsrv, _ := RunServerWithConfig(confFileName)\n\tlogger := newCaptureWarnLogger()\n\tsrv.SetLogger(logger, false, false)\n\tdefer srv.Shutdown()\n\n\tvar err error\n\tcount := 0\n\tfor count < 10 {\n\t\tvar nc *nats.Conn\n\t\tnc, err = nats.Connect(srv.ClientURL(), nats.RootCAs(\"./configs/certs/ca.pem\"))\n\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tnc.Close()\n\t\tcount++\n\t}\n\n\tif count != 3 {\n\t\tt.Fatalf(\"Expected 3 connections per second, got %d (%v)\", count, err)\n\t}\n\n\tif !logger.waitFor(\"connections due to TLS rate limiting\", time.Second) {\n\t\tt.Fatalf(\"did not log 'TLS rate limiting' warning\")\n\t}\n}\n\nfunc TestTLSPinnedCertsRoute(t *testing.T) {\n\ttmplSeed := `\n\thost: localhost\n\tport: -1\n\tcluster {\n\t\tport: -1\n\t\tpool_size: -1\n\t\ttls {\n\t\t\tca_file: \"configs/certs/ca.pem\"\n\t\t\tcert_file: \"configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/server-key.pem\"\n\t\t}\n\t}`\n\t// this server connects to seed, but is set up to not trust seeds cert\n\ttmplSrv := `\n\thost: localhost\n\tport: -1\n\tcluster {\n\t\tport: -1\n\t\troutes = [nats-route://localhost:%d]\n\t\ttls {\n\t\t\tca_file: \"configs/certs/ca.pem\"\n\t\t\tcert_file: \"configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/server-key.pem\"\n\t\t\t# Require a client certificate and map user id from certificate\n\t\t\tverify: true\n\t\t\t# expected to fail the seed server\n\t\t\tpinned_certs: [\"%s\"]\n\t\t}\n\t}`\n\n\tconfSeed := createConfFile(t, []byte(tmplSeed))\n\tsrvSeed, o := RunServerWithConfig(confSeed)\n\tdefer srvSeed.Shutdown()\n\n\tconfSrv := createConfFile(t, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, \"89386860ea1222698ea676fc97310bdf2bff6f7e2b0420fac3b3f8f5a08fede5\")))\n\tsrv, _ := RunServerWithConfig(confSrv)\n\tdefer srv.Shutdown()\n\n\tcheckClusterFormed(t, srvSeed, srv)\n\n\t// this change will result in the server being and remaining disconnected\n\tos.WriteFile(confSrv, []byte(fmt.Sprintf(tmplSrv, o.Cluster.Port, \"aaaaaaaa09fde09451411ba3b42c0f74727d61a974c69fd3cf5257f39c75f0e9\")), 0660)\n\tif err := srv.Reload(); err != nil {\n\t\tt.Fatalf(\"on Reload got %v\", err)\n\t}\n\n\tcheckNumRoutes(t, srvSeed, 0)\n\tcheckNumRoutes(t, srv, 0)\n}\n\nfunc TestAllowNonTLSReload(t *testing.T) {\n\ttmpl := `\n\t\tlisten: \"127.0.0.1:-1\"\n\t\tping_interval: \"%s\"\n\t\ttls {\n\t\t\tca_file: \"configs/certs/ca.pem\"\n\t\t\tcert_file: \"configs/certs/server-cert.pem\"\n\t\t\tkey_file: \"configs/certs/server-key.pem\"\n\t\t}\n\t\tallow_non_tls: true\n\t`\n\tconf := createConfFile(t, []byte(fmt.Sprintf(tmpl, \"10s\")))\n\ts, o := RunServerWithConfig(conf)\n\tdefer s.Shutdown()\n\n\tcheck := func() {\n\t\tt.Helper()\n\t\tnc := createClientConn(t, \"127.0.0.1\", o.Port)\n\t\tdefer nc.Close()\n\t\tinfo := checkInfoMsg(t, nc)\n\t\tif !info.TLSAvailable {\n\t\t\tt.Fatal(\"TLSAvailable should be true, was false\")\n\t\t}\n\t\tif info.TLSRequired {\n\t\t\tt.Fatal(\"TLSRequired should be false, was true\")\n\t\t}\n\t}\n\tcheck()\n\n\tos.WriteFile(conf, []byte(fmt.Sprintf(tmpl, \"20s\")), 0660)\n\tif err := s.Reload(); err != nil {\n\t\tt.Fatalf(\"Error on reload: %v\", err)\n\t}\n\tcheck()\n}\n" }, { "path": "test/user_authorization_test.go", "content": "// Copyright 2016-2025 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"regexp\"\n\t\"testing\"\n\t\"time\"\n)\n\nconst DefaultPass = \"foo\"\n\nvar permErrRe = regexp.MustCompile(`\\A\\-ERR\\s+'Permissions Violation([^\\r\\n]+)\\r\\n`)\n\nfunc TestUserAuthorizationProto(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\tdefer srv.Shutdown()\n\n\t// Alice can do anything, check a few for OK result.\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"alice\", DefaultPass)\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"SUB foo 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// Check that _ is ok\n\tsendProto(t, c, \"PUB _ 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tc.Close()\n\n\t// Bob is a requestor only, e.g. req.foo, req.bar for publish, subscribe only to INBOXes.\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"bob\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\t// These should error.\n\tsendProto(t, c, \"SUB foo 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\n\t// These should work ok.\n\tsendProto(t, c, \"SUB _INBOX.abcd 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB req.foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB req.bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tc.Close()\n\n\t// Joe is a default user\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"joe\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\t// These should error.\n\tsendProto(t, c, \"SUB foo.bar.* 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB foo.bar.baz 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\n\t// These should work ok.\n\tsendProto(t, c, \"SUB _INBOX.abcd 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"SUB PUBLIC.abcd 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"PUB SANDBOX.foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB SANDBOX.bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// Since only PWC, this should fail (too many tokens).\n\tsendProto(t, c, \"PUB SANDBOX.foo.bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\n\tc.Close()\n\n\t// This is the new style permissions with allow and deny clauses.\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"ns\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\t// These should work\n\tsendProto(t, c, \"PUB SANDBOX.foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB baz.bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"PUB baz.foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// These should error.\n\tsendProto(t, c, \"PUB foo 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB foo.bar 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB foo.bar.baz 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"PUB SYS.1 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\n\t// Subscriptions\n\n\t// These should work ok.\n\tsendProto(t, c, \"SUB foo.bar 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\tsendProto(t, c, \"SUB foo.foo 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// These should error.\n\tsendProto(t, c, \"SUB foo 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"SUB foo.baz 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"SUB foo.baz 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\tsendProto(t, c, \"SUB foo.baz 1\\r\\n\")\n\texpectResult(t, c, permErrRe)\n\n\t// Deny clauses for subscriptions need to be able to allow subscriptions\n\t// on larger scoped wildcards, but prevent delivery of a message whose\n\t// subject matches a deny clause.\n\n\t// Clear old stuff\n\tc.Close()\n\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"ns\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB foo.* 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB foo.* bar 2\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// Now send on foo.baz which should not be received on first client.\n\t// Joe is a default user\n\tnc := createClientConn(t, opts.Host, opts.Port)\n\tdefer nc.Close()\n\texpectAuthRequired(t, nc)\n\tdoAuthConnect(t, nc, \"\", \"ns-pub\", DefaultPass)\n\texpectResult(t, nc, okRe)\n\n\tsendProto(t, nc, \"PUB foo.baz 2\\r\\nok\\r\\n\")\n\texpectResult(t, nc, okRe)\n\n\t// Expect nothing from the wildcard subscription.\n\texpectNothing(t, c)\n\n\tsendProto(t, c, \"PING\\r\\n\")\n\texpectResult(t, c, pongRe)\n\n\t// Now create a queue sub on our ns-pub user. We want to test that\n\t// queue subscribers can be denied and delivery will route around.\n\tsendProto(t, nc, \"SUB foo.baz bar 2\\r\\n\")\n\texpectResult(t, nc, okRe)\n\n\t// Make sure we always get the message on our queue subscriber.\n\t// Do this several times since we should select the other subscriber\n\t// but get permission denied..\n\tfor i := 0; i < 20; i++ {\n\t\tsendProto(t, nc, \"PUB foo.baz 2\\r\\nok\\r\\n\")\n\t\tbuf := expectResult(t, nc, okRe)\n\t\tif msgRe.Match(buf) {\n\t\t\tcontinue\n\t\t} else {\n\t\t\texpectResult(t, nc, msgRe)\n\t\t}\n\t}\n\n\t// Clear old stuff\n\tc.Close()\n\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"ns\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB foo.bar 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB foo.bar.baz 2\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\tsendProto(t, c, \"SUB > 3\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\tsendProto(t, c, \"SUB SYS.> 4\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\tsendProto(t, c, \"SUB SYS.TEST.foo 5\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB SYS.bar 5\\r\\n\")\n\texpectResult(t, c, errRe)\n}\n\nfunc TestUserAuthorizationAllowResponses(t *testing.T) {\n\tsrv, opts := RunServerWithConfig(\"./configs/authorization.conf\")\n\tdefer srv.Shutdown()\n\n\t// Alice can do anything, so she will be our requestor\n\trc := createClientConn(t, opts.Host, opts.Port)\n\tdefer rc.Close()\n\texpectAuthRequired(t, rc)\n\tdoAuthConnect(t, rc, \"\", \"alice\", DefaultPass)\n\texpectResult(t, rc, okRe)\n\n\t// MY_SERVICE can subscribe to a single request subject but can\n\t// respond to any reply subject that it receives, but only\n\t// for one response.\n\tc := createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"svca\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB my.service.req 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, rc, \"PUB my.service.req resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, rc, okRe)\n\n\tmatches := msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1)\n\tcheckMsg(t, matches[0], \"my.service.req\", \"1\", \"resp.bar.22\", \"2\", \"ok\")\n\n\t// This should be allowed\n\tsendProto(t, c, \"PUB resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// This should not be allowed\n\tsendProto(t, c, \"PUB resp.bar.33 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\t// This should also not be allowed now since we already sent a response and max is 1.\n\tsendProto(t, c, \"PUB resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\tc.Close() // from MY_SERVICE\n\n\t// MY_STREAM_SERVICE can subscribe to a single request subject but can\n\t// respond to any reply subject that it receives, and send up to 10 responses.\n\t// Each permission for a response can last up to 10ms.\n\tc = createClientConn(t, opts.Host, opts.Port)\n\tdefer c.Close()\n\texpectAuthRequired(t, c)\n\tdoAuthConnect(t, c, \"\", \"svcb\", DefaultPass)\n\texpectResult(t, c, okRe)\n\n\tsendProto(t, c, \"SUB my.service.req 1\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// Same rules as above.\n\tsendProto(t, rc, \"PUB my.service.req resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, rc, okRe)\n\n\tmatches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1)\n\tcheckMsg(t, matches[0], \"my.service.req\", \"1\", \"resp.bar.22\", \"2\", \"ok\")\n\n\t// This should be allowed\n\tsendProto(t, c, \"PUB resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, okRe)\n\n\t// This should not be allowed\n\tsendProto(t, c, \"PUB resp.bar.33 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\t// We should be able to send 9 more here since we are allowed 10 total\n\tfor i := 0; i < 9; i++ {\n\t\tsendProto(t, c, \"PUB resp.bar.22 2\\r\\nok\\r\\n\")\n\t\texpectResult(t, c, okRe)\n\t}\n\t// Now this should fail since we already sent 10 responses.\n\tsendProto(t, c, \"PUB resp.bar.22 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n\n\t// Now test timeout.\n\tsendProto(t, rc, \"PUB my.service.req resp.bar.11 2\\r\\nok\\r\\n\")\n\texpectResult(t, rc, okRe)\n\n\tmatches = msgRe.FindAllSubmatch(expectResult(t, c, msgRe), -1)\n\tcheckMsg(t, matches[0], \"my.service.req\", \"1\", \"resp.bar.11\", \"2\", \"ok\")\n\n\ttime.Sleep(100 * time.Millisecond)\n\n\tsendProto(t, c, \"PUB resp.bar.11 2\\r\\nok\\r\\n\")\n\texpectResult(t, c, errRe)\n}\n" }, { "path": "test/verbose_test.go", "content": "// Copyright 2012-2018 The NATS Authors\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"testing\"\n)\n\nfunc TestVerbosePing(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tdoConnect(t, c, true, false, false)\n\n\tsend := sendCommand(t, c)\n\texpect := expectCommand(t, c)\n\n\texpect(okRe)\n\n\t// Ping should still be same\n\tsend(\"PING\\r\\n\")\n\texpect(pongRe)\n}\n\nfunc TestVerboseConnect(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tdoConnect(t, c, true, false, false)\n\n\tsend := sendCommand(t, c)\n\texpect := expectCommand(t, c)\n\n\texpect(okRe)\n\n\t// Connect\n\tsend(\"CONNECT {\\\"verbose\\\":true,\\\"pedantic\\\":true,\\\"tls_required\\\":false}\\r\\n\")\n\texpect(okRe)\n}\n\nfunc TestVerbosePubSub(t *testing.T) {\n\ts := runProtoServer()\n\tdefer s.Shutdown()\n\n\tc := createClientConn(t, \"127.0.0.1\", PROTO_TEST_PORT)\n\tdefer c.Close()\n\n\tdoConnect(t, c, true, false, false)\n\tsend := sendCommand(t, c)\n\texpect := expectCommand(t, c)\n\n\texpect(okRe)\n\n\t// Pub\n\tsend(\"PUB foo 2\\r\\nok\\r\\n\")\n\texpect(okRe)\n\n\t// Sub\n\tsend(\"SUB foo 1\\r\\n\")\n\texpect(okRe)\n\n\t// UnSub\n\tsend(\"UNSUB 1\\r\\n\")\n\texpect(okRe)\n}\n" }, { "path": "util/nats-server-hardened.service", "content": "[Unit]\nDescription=NATS Server\nAfter=network-online.target ntp.service\n\n# If you use a dedicated filesystem for JetStream data, then you might use something like:\n# ConditionPathIsMountPoint=/srv/jetstream\n# See also Service.ReadWritePaths\n\n[Service]\nType=simple\nEnvironmentFile=-/etc/default/nats-server\nExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf\nExecReload=/bin/kill -s HUP $MAINPID\n\n# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown\n# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode\nExecStop=/bin/kill -s SIGUSR2 $MAINPID\n\nUser=nats\nGroup=nats\n\nRestart=on-failure\nRestartSec=5\n\n# This should be `lame_duck_duration` + some buffer to finish the shutdown.\n# By default, `lame_duck_duration` is 2 mins.\nTimeoutStopSec=150\n\n# Capacity Limits\n# JetStream requires 2 FDs open per stream.\nLimitNOFILE=800000\n# Environment=GOMEMLIMIT=12GiB\n# You might find it better to set GOMEMLIMIT via /etc/default/nats-server,\n# so that you can change limits without needing a systemd daemon-reload.\n\n# Hardening\nCapabilityBoundingSet=\nLockPersonality=true\nMemoryDenyWriteExecute=true\nNoNewPrivileges=true\nPrivateDevices=true\nPrivateTmp=true\nPrivateUsers=true\nProcSubset=pid\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectHostname=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nProtectSystem=strict\nReadOnlyPaths=\nRestrictAddressFamilies=AF_INET AF_INET6\nRestrictNamespaces=true\nRestrictRealtime=true\nRestrictSUIDSGID=true\nSystemCallFilter=@system-service ~@privileged ~@resources\nUMask=0077\n\n# Consider locking down all areas of /etc which hold machine identity keys, etc\nInaccessiblePaths=/etc/ssh\n\n# If you have systemd >= 247\nProtectProc=invisible\n\n# If you have systemd >= 248\nPrivateIPC=true\n\n# Optional: writable directory for JetStream.\n# See also: Unit.ConditionPathIsMountPoint\nReadWritePaths=/var/lib/nats\n\n# Optional: resource control.\n# Replace weights by values that make sense for your situation.\n# For a list of all options see:\n# https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html\n#CPUAccounting=true\n#CPUWeight=100 # of 10000\n#IOAccounting=true\n#IOWeight=100 # of 10000\n#MemoryAccounting=true\n#MemoryMax=1GB\n#IPAccounting=true\n\n[Install]\nWantedBy=multi-user.target\n# If you install this service as nats-server.service and want 'nats'\n# to work as an alias, then uncomment this next line:\n#Alias=nats.service\n" }, { "path": "util/nats-server.service", "content": "[Unit]\nDescription=NATS Server\nAfter=network-online.target ntp.service\n\n[Service]\nPrivateTmp=true\nType=simple\nExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf\nExecReload=/bin/kill -s HUP $MAINPID\n\n# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown\n# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode\nExecStop=/bin/kill -s SIGUSR2 $MAINPID\n\n# This should be `lame_duck_duration` + some buffer to finish the shutdown.\n# By default, `lame_duck_duration` is 2 mins.\nTimeoutStopSec=150\n\nRestart=on-failure\n\nUser=nats\nGroup=nats\n\n[Install]\nWantedBy=multi-user.target\n" } ]